2023-10-01T09:53:24.072 INFO:root:teuthology version: 0.0.1.dev194+g54e62bc 2023-10-01T09:53:24.072 DEBUG:teuthology.run:Teuthology command: teuthology --verbose --description upgrade:pacific-x/stress-split/{0-distro/ubuntu_20.04 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity} --name teuthology-2023-09-30_14:23:11-upgrade:pacific-x-reef-distro-default-smithi --archive /home/teuthworker/archive/teuthology-2023-09-30_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7407998 --owner scheduled_teuthology@teuthology -- /home/teuthworker/archive/teuthology-2023-09-30_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7407998/orig.config.yaml 2023-10-01T09:53:24.132 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-10-01T09:53:24.227 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2023-09-30_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7407998 branch: reef description: upgrade:pacific-x/stress-split/{0-distro/ubuntu_20.04 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity} email: ceph-qa@ceph.io first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' first_in_suite: false job_id: '7407998' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run basic import/export cli tests for rbd ' name: teuthology-2023-09-30_14:23:11-upgrade:pacific-x-reef-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: ubuntu os_version: '20.04' overrides: admin_socket: branch: reef ceph: conf: global: mon election default strategy: 3 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: b56822191ad0416d0c05ed59720df03ad9c2a44e ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: b56822191ad0416d0c05ed59720df03ad9c2a44e kernel: hwe: true workunit: branch: reef sha1: fc1e90a395a6a5631142eb70fd9bf3d22740a9da owner: scheduled_teuthology@teuthology priority: 100 repo: https://github.com/ceph/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - workunit: branch: pacific clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' seed: 3147 sha1: b56822191ad0416d0c05ed59720df03ad9c2a44e sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: pacific clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: pacific clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: pacific clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' suite: upgrade:pacific-x suite_branch: reef suite_path: /home/teuthworker/src/github.com_ceph_ceph_fc1e90a395a6a5631142eb70fd9bf3d22740a9da/qa suite_relpath: qa suite_repo: https://github.com/ceph/ceph.git suite_sha1: fc1e90a395a6a5631142eb70fd9bf3d22740a9da targets: smithi078.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFyT/r7xuKfmg50w0OdQebr4A/ptJo1Zk6DyZNZNZM/q5rW+sznh+wvHqeT3e4YtCdZLaXyvMms8o8u8H5nXOYM= smithi173.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFyT/r7xuKfmg50w0OdQebr4A/ptJo1Zk6DyZNZNZM/q5rW+sznh+wvHqeT3e4YtCdZLaXyvMms8o8u8H5nXOYM= tasks: - install: branch: pacific exclude_packages: - ceph-volume - cephadm: cephadm_branch: pacific cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:pacific - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 54e62bcbac4e53d9685e08328b790d3b20d71cae timestamp: 2023-09-30_14:23:11 tube: smithi user: teuthology verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2830 2023-10-01T09:53:24.228 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_ceph_ceph_fc1e90a395a6a5631142eb70fd9bf3d22740a9da/qa; will attempt to use it 2023-10-01T09:53:24.229 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_ceph_ceph_fc1e90a395a6a5631142eb70fd9bf3d22740a9da/qa/tasks 2023-10-01T09:53:24.229 INFO:teuthology.run_tasks:Running task internal.check_packages... 2023-10-01T09:53:24.230 INFO:teuthology.task.internal:Checking packages... 2023-10-01T09:53:24.253 INFO:teuthology.task.internal:Checking packages for os_type 'ubuntu', flavor 'default' and ceph hash 'b56822191ad0416d0c05ed59720df03ad9c2a44e' 2023-10-01T09:53:24.253 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-01T09:53:24.253 INFO:teuthology.packaging:ref: None 2023-10-01T09:53:24.253 INFO:teuthology.packaging:tag: None 2023-10-01T09:53:24.253 INFO:teuthology.packaging:branch: reef 2023-10-01T09:53:24.254 INFO:teuthology.packaging:sha1: b56822191ad0416d0c05ed59720df03ad9c2a44e 2023-10-01T09:53:24.254 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&ref=reef 2023-10-01T09:53:24.546 INFO:teuthology.task.internal:Found packages for ceph version 18.2.0-501-gfc1e90a3-1focal 2023-10-01T09:53:24.546 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2023-10-01T09:53:24.553 INFO:teuthology.task.internal:no buildpackages task found 2023-10-01T09:53:24.553 INFO:teuthology.run_tasks:Running task internal.save_config... 2023-10-01T09:53:24.569 INFO:teuthology.task.internal:Saving configuration 2023-10-01T09:53:24.592 INFO:teuthology.run_tasks:Running task internal.check_lock... 2023-10-01T09:53:24.597 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-10-01T09:53:24.625 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi078.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-09-30_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7407998', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '20.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-01 09:47:34.191666', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGETOy3xONfbx7kUFCNiUtRLNfkcrcEsWzPdq6Np/6Bw'} 2023-10-01T09:53:24.647 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi173.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-09-30_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7407998', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '20.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-01 09:47:34.189897', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGETOy3xONfbx7kUFCNiUtRLNfkcrcEsWzPdq6Np/6Bw'} 2023-10-01T09:53:24.648 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2023-10-01T09:53:24.653 INFO:teuthology.task.internal:roles: ubuntu@smithi078.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2023-10-01T09:53:24.653 INFO:teuthology.task.internal:roles: ubuntu@smithi173.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2023-10-01T09:53:24.654 INFO:teuthology.run_tasks:Running task console_log... 2023-10-01T09:53:24.728 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f70afa9e160>, signals=[15]) 2023-10-01T09:53:24.729 INFO:teuthology.run_tasks:Running task internal.connect... 2023-10-01T09:53:24.735 INFO:teuthology.task.internal:Opening connections... 2023-10-01T09:53:24.735 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi078.front.sepia.ceph.com 2023-10-01T09:53:24.737 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi078.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-01T09:53:24.787 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi173.front.sepia.ceph.com 2023-10-01T09:53:24.788 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi173.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-01T09:53:24.836 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2023-10-01T09:53:24.842 DEBUG:teuthology.orchestra.run.smithi078:> uname -m 2023-10-01T09:53:24.864 INFO:teuthology.orchestra.run.smithi078.stdout:x86_64 2023-10-01T09:53:24.865 DEBUG:teuthology.orchestra.run.smithi078:> cat /etc/os-release 2023-10-01T09:53:24.911 INFO:teuthology.orchestra.run.smithi078.stdout:NAME="Ubuntu" 2023-10-01T09:53:24.911 INFO:teuthology.orchestra.run.smithi078.stdout:VERSION="20.04.4 LTS (Focal Fossa)" 2023-10-01T09:53:24.911 INFO:teuthology.orchestra.run.smithi078.stdout:ID=ubuntu 2023-10-01T09:53:24.911 INFO:teuthology.orchestra.run.smithi078.stdout:ID_LIKE=debian 2023-10-01T09:53:24.911 INFO:teuthology.orchestra.run.smithi078.stdout:PRETTY_NAME="Ubuntu 20.04.4 LTS" 2023-10-01T09:53:24.911 INFO:teuthology.orchestra.run.smithi078.stdout:VERSION_ID="20.04" 2023-10-01T09:53:24.911 INFO:teuthology.orchestra.run.smithi078.stdout:HOME_URL="https://www.ubuntu.com/" 2023-10-01T09:53:24.911 INFO:teuthology.orchestra.run.smithi078.stdout:SUPPORT_URL="https://help.ubuntu.com/" 2023-10-01T09:53:24.911 INFO:teuthology.orchestra.run.smithi078.stdout:BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" 2023-10-01T09:53:24.911 INFO:teuthology.orchestra.run.smithi078.stdout:PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" 2023-10-01T09:53:24.912 INFO:teuthology.orchestra.run.smithi078.stdout:VERSION_CODENAME=focal 2023-10-01T09:53:24.912 INFO:teuthology.orchestra.run.smithi078.stdout:UBUNTU_CODENAME=focal 2023-10-01T09:53:24.912 INFO:teuthology.lock.ops:Updating smithi078.front.sepia.ceph.com on lock server 2023-10-01T09:53:24.937 DEBUG:teuthology.orchestra.run.smithi173:> uname -m 2023-10-01T09:53:24.944 INFO:teuthology.orchestra.run.smithi173.stdout:x86_64 2023-10-01T09:53:24.944 DEBUG:teuthology.orchestra.run.smithi173:> cat /etc/os-release 2023-10-01T09:53:24.992 INFO:teuthology.orchestra.run.smithi173.stdout:NAME="Ubuntu" 2023-10-01T09:53:24.992 INFO:teuthology.orchestra.run.smithi173.stdout:VERSION="20.04.4 LTS (Focal Fossa)" 2023-10-01T09:53:24.992 INFO:teuthology.orchestra.run.smithi173.stdout:ID=ubuntu 2023-10-01T09:53:24.992 INFO:teuthology.orchestra.run.smithi173.stdout:ID_LIKE=debian 2023-10-01T09:53:24.992 INFO:teuthology.orchestra.run.smithi173.stdout:PRETTY_NAME="Ubuntu 20.04.4 LTS" 2023-10-01T09:53:24.992 INFO:teuthology.orchestra.run.smithi173.stdout:VERSION_ID="20.04" 2023-10-01T09:53:24.992 INFO:teuthology.orchestra.run.smithi173.stdout:HOME_URL="https://www.ubuntu.com/" 2023-10-01T09:53:24.993 INFO:teuthology.orchestra.run.smithi173.stdout:SUPPORT_URL="https://help.ubuntu.com/" 2023-10-01T09:53:24.993 INFO:teuthology.orchestra.run.smithi173.stdout:BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" 2023-10-01T09:53:24.993 INFO:teuthology.orchestra.run.smithi173.stdout:PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" 2023-10-01T09:53:24.993 INFO:teuthology.orchestra.run.smithi173.stdout:VERSION_CODENAME=focal 2023-10-01T09:53:24.993 INFO:teuthology.orchestra.run.smithi173.stdout:UBUNTU_CODENAME=focal 2023-10-01T09:53:24.993 INFO:teuthology.lock.ops:Updating smithi173.front.sepia.ceph.com on lock server 2023-10-01T09:53:25.012 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2023-10-01T09:53:25.021 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2023-10-01T09:53:25.026 INFO:teuthology.task.internal:Checking for old test directory... 2023-10-01T09:53:25.026 DEBUG:teuthology.orchestra.run.smithi078:> test '!' -e /home/ubuntu/cephtest 2023-10-01T09:53:25.028 DEBUG:teuthology.orchestra.run.smithi173:> test '!' -e /home/ubuntu/cephtest 2023-10-01T09:53:25.034 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2023-10-01T09:53:25.039 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2023-10-01T09:53:25.040 DEBUG:teuthology.orchestra.run.smithi078:> test -z $(ls -A /var/lib/ceph) 2023-10-01T09:53:25.074 DEBUG:teuthology.orchestra.run.smithi173:> test -z $(ls -A /var/lib/ceph) 2023-10-01T09:53:25.086 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2023-10-01T09:53:25.145 INFO:teuthology.run_tasks:Running task kernel... 2023-10-01T09:53:25.158 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2023-10-01T09:53:25.159 INFO:teuthology.task.kernel:normalize config orig: {'hwe': True} 2023-10-01T09:53:25.159 DEBUG:teuthology.task.kernel:normalized overrides {'mon.a': {'hwe': True}, 'mon.c': {'hwe': True}, 'mgr.y': {'hwe': True}, 'osd.0': {'hwe': True}, 'osd.1': {'hwe': True}, 'osd.2': {'hwe': True}, 'osd.3': {'hwe': True}, 'client.0': {'hwe': True}, 'node-exporter.a': {'hwe': True}, 'alertmanager.a': {'hwe': True}, 'mon.b': {'hwe': True}, 'mgr.x': {'hwe': True}, 'osd.4': {'hwe': True}, 'osd.5': {'hwe': True}, 'osd.6': {'hwe': True}, 'osd.7': {'hwe': True}, 'client.1': {'hwe': True}, 'prometheus.a': {'hwe': True}, 'grafana.a': {'hwe': True}, 'node-exporter.b': {'hwe': True}} 2023-10-01T09:53:25.159 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': True, 'sha1': 'distro', 'hwe': True}, 'mon.b': {'kdb': True, 'sha1': 'distro', 'hwe': True}}, timeout 300 2023-10-01T09:53:25.159 DEBUG:teuthology.orchestra.run.smithi078:> test -f /run/.containerenv -o -f /.dockerenv 2023-10-01T09:53:25.160 DEBUG:teuthology.orchestra.run.smithi173:> test -f /run/.containerenv -o -f /.dockerenv 2023-10-01T09:53:25.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T09:53:25.165 DEBUG:teuthology.orchestra.run.smithi078:> uname -r 2023-10-01T09:53:25.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T09:53:25.166 DEBUG:teuthology.orchestra.run.smithi173:> uname -r 2023-10-01T09:53:25.172 INFO:teuthology.orchestra.run.smithi173.stdout:5.4.0-124-generic 2023-10-01T09:53:25.172 INFO:teuthology.task.kernel:Running kernel on smithi173: 5.4.0-124-generic 2023-10-01T09:53:25.173 DEBUG:teuthology.orchestra.run.smithi173:> sudo apt-get clean 2023-10-01T09:53:25.214 INFO:teuthology.orchestra.run.smithi078.stdout:5.4.0-124-generic 2023-10-01T09:53:25.215 INFO:teuthology.task.kernel:Running kernel on smithi078: 5.4.0-124-generic 2023-10-01T09:53:25.215 DEBUG:teuthology.orchestra.run.smithi078:> sudo apt-get clean 2023-10-01T09:53:25.325 DEBUG:teuthology.orchestra.run.smithi078:> sudo apt-get update 2023-10-01T09:53:25.336 DEBUG:teuthology.orchestra.run.smithi173:> sudo apt-get update 2023-10-01T09:53:25.521 INFO:teuthology.orchestra.run.smithi173.stdout:Hit:1 http://archive.ubuntu.com/ubuntu focal InRelease 2023-10-01T09:53:25.522 INFO:teuthology.orchestra.run.smithi173.stdout:Get:2 http://archive.ubuntu.com/ubuntu focal-updates InRelease [114 kB] 2023-10-01T09:53:25.522 INFO:teuthology.orchestra.run.smithi173.stdout:Get:3 http://security.ubuntu.com/ubuntu focal-security InRelease [114 kB] 2023-10-01T09:53:25.589 INFO:teuthology.orchestra.run.smithi173.stdout:Get:4 http://archive.ubuntu.com/ubuntu focal-backports InRelease [108 kB] 2023-10-01T09:53:25.663 INFO:teuthology.orchestra.run.smithi078.stdout:Get:1 http://security.ubuntu.com/ubuntu focal-security InRelease [114 kB] 2023-10-01T09:53:25.776 INFO:teuthology.orchestra.run.smithi078.stdout:Hit:2 http://archive.ubuntu.com/ubuntu focal InRelease 2023-10-01T09:53:25.858 INFO:teuthology.orchestra.run.smithi078.stdout:Get:3 http://archive.ubuntu.com/ubuntu focal-updates InRelease [114 kB] 2023-10-01T09:53:25.906 INFO:teuthology.orchestra.run.smithi173.stdout:Get:5 http://security.ubuntu.com/ubuntu focal-security/main i386 Packages [652 kB] 2023-10-01T09:53:25.950 INFO:teuthology.orchestra.run.smithi078.stdout:Get:4 http://security.ubuntu.com/ubuntu focal-security/main amd64 Packages [2,464 kB] 2023-10-01T09:53:25.999 INFO:teuthology.orchestra.run.smithi173.stdout:Get:6 http://security.ubuntu.com/ubuntu focal-security/main amd64 Packages [2,464 kB] 2023-10-01T09:53:26.046 INFO:teuthology.orchestra.run.smithi173.stdout:Get:7 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 Packages [2,865 kB] 2023-10-01T09:53:26.046 INFO:teuthology.orchestra.run.smithi173.stdout:Get:8 http://security.ubuntu.com/ubuntu focal-security/main Translation-en [385 kB] 2023-10-01T09:53:26.051 INFO:teuthology.orchestra.run.smithi173.stdout:Get:9 http://security.ubuntu.com/ubuntu focal-security/main amd64 c-n-f Metadata [13.2 kB] 2023-10-01T09:53:26.052 INFO:teuthology.orchestra.run.smithi173.stdout:Get:10 http://security.ubuntu.com/ubuntu focal-security/restricted i386 Packages [33.3 kB] 2023-10-01T09:53:26.052 INFO:teuthology.orchestra.run.smithi173.stdout:Get:11 http://security.ubuntu.com/ubuntu focal-security/restricted amd64 Packages [2,218 kB] 2023-10-01T09:53:26.075 INFO:teuthology.orchestra.run.smithi173.stdout:Get:12 http://security.ubuntu.com/ubuntu focal-security/restricted Translation-en [309 kB] 2023-10-01T09:53:26.078 INFO:teuthology.orchestra.run.smithi173.stdout:Get:13 http://security.ubuntu.com/ubuntu focal-security/restricted amd64 c-n-f Metadata [572 B] 2023-10-01T09:53:26.078 INFO:teuthology.orchestra.run.smithi173.stdout:Get:14 http://security.ubuntu.com/ubuntu focal-security/universe amd64 Packages [886 kB] 2023-10-01T09:53:26.079 INFO:teuthology.orchestra.run.smithi078.stdout:Get:5 http://security.ubuntu.com/ubuntu focal-security/main i386 Packages [652 kB] 2023-10-01T09:53:26.084 INFO:teuthology.orchestra.run.smithi173.stdout:Get:15 http://security.ubuntu.com/ubuntu focal-security/universe i386 Packages [615 kB] 2023-10-01T09:53:26.087 INFO:teuthology.orchestra.run.smithi173.stdout:Get:16 http://security.ubuntu.com/ubuntu focal-security/universe Translation-en [185 kB] 2023-10-01T09:53:26.089 INFO:teuthology.orchestra.run.smithi173.stdout:Get:17 http://security.ubuntu.com/ubuntu focal-security/universe amd64 c-n-f Metadata [19.1 kB] 2023-10-01T09:53:26.090 INFO:teuthology.orchestra.run.smithi078.stdout:Get:6 http://security.ubuntu.com/ubuntu focal-security/main Translation-en [385 kB] 2023-10-01T09:53:26.093 INFO:teuthology.orchestra.run.smithi173.stdout:Get:18 http://security.ubuntu.com/ubuntu focal-security/multiverse i386 Packages [7,196 B] 2023-10-01T09:53:26.094 INFO:teuthology.orchestra.run.smithi173.stdout:Get:19 http://security.ubuntu.com/ubuntu focal-security/multiverse amd64 Packages [23.6 kB] 2023-10-01T09:53:26.094 INFO:teuthology.orchestra.run.smithi173.stdout:Get:20 http://security.ubuntu.com/ubuntu focal-security/multiverse Translation-en [5,504 B] 2023-10-01T09:53:26.095 INFO:teuthology.orchestra.run.smithi078.stdout:Get:7 http://security.ubuntu.com/ubuntu focal-security/main amd64 c-n-f Metadata [13.2 kB] 2023-10-01T09:53:26.095 INFO:teuthology.orchestra.run.smithi078.stdout:Get:8 http://security.ubuntu.com/ubuntu focal-security/restricted i386 Packages [33.3 kB] 2023-10-01T09:53:26.095 INFO:teuthology.orchestra.run.smithi078.stdout:Get:9 http://security.ubuntu.com/ubuntu focal-security/restricted amd64 Packages [2,218 kB] 2023-10-01T09:53:26.110 INFO:teuthology.orchestra.run.smithi078.stdout:Get:10 http://security.ubuntu.com/ubuntu focal-security/restricted Translation-en [309 kB] 2023-10-01T09:53:26.112 INFO:teuthology.orchestra.run.smithi078.stdout:Get:11 http://security.ubuntu.com/ubuntu focal-security/restricted amd64 c-n-f Metadata [572 B] 2023-10-01T09:53:26.112 INFO:teuthology.orchestra.run.smithi078.stdout:Get:12 http://security.ubuntu.com/ubuntu focal-security/universe i386 Packages [615 kB] 2023-10-01T09:53:26.113 INFO:teuthology.orchestra.run.smithi173.stdout:Get:21 http://security.ubuntu.com/ubuntu focal-security/multiverse amd64 c-n-f Metadata [548 B] 2023-10-01T09:53:26.116 INFO:teuthology.orchestra.run.smithi078.stdout:Get:13 http://security.ubuntu.com/ubuntu focal-security/universe amd64 Packages [886 kB] 2023-10-01T09:53:26.122 INFO:teuthology.orchestra.run.smithi078.stdout:Get:14 http://security.ubuntu.com/ubuntu focal-security/universe Translation-en [185 kB] 2023-10-01T09:53:26.123 INFO:teuthology.orchestra.run.smithi078.stdout:Get:15 http://security.ubuntu.com/ubuntu focal-security/universe amd64 c-n-f Metadata [19.1 kB] 2023-10-01T09:53:26.129 INFO:teuthology.orchestra.run.smithi078.stdout:Get:16 http://security.ubuntu.com/ubuntu focal-security/multiverse amd64 Packages [23.6 kB] 2023-10-01T09:53:26.130 INFO:teuthology.orchestra.run.smithi078.stdout:Get:17 http://security.ubuntu.com/ubuntu focal-security/multiverse i386 Packages [7,196 B] 2023-10-01T09:53:26.130 INFO:teuthology.orchestra.run.smithi078.stdout:Get:18 http://security.ubuntu.com/ubuntu focal-security/multiverse Translation-en [5,504 B] 2023-10-01T09:53:26.130 INFO:teuthology.orchestra.run.smithi078.stdout:Get:19 http://security.ubuntu.com/ubuntu focal-security/multiverse amd64 c-n-f Metadata [548 B] 2023-10-01T09:53:26.166 INFO:teuthology.orchestra.run.smithi173.stdout:Get:22 http://archive.ubuntu.com/ubuntu focal-updates/main i386 Packages [885 kB] 2023-10-01T09:53:26.176 INFO:teuthology.orchestra.run.smithi173.stdout:Get:23 http://archive.ubuntu.com/ubuntu focal-updates/main Translation-en [469 kB] 2023-10-01T09:53:26.180 INFO:teuthology.orchestra.run.smithi173.stdout:Get:24 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 c-n-f Metadata [17.1 kB] 2023-10-01T09:53:26.180 INFO:teuthology.orchestra.run.smithi173.stdout:Get:25 http://archive.ubuntu.com/ubuntu focal-updates/restricted i386 Packages [34.6 kB] 2023-10-01T09:53:26.180 INFO:teuthology.orchestra.run.smithi173.stdout:Get:26 http://archive.ubuntu.com/ubuntu focal-updates/restricted amd64 Packages [2,330 kB] 2023-10-01T09:53:26.200 INFO:teuthology.orchestra.run.smithi173.stdout:Get:27 http://archive.ubuntu.com/ubuntu focal-updates/restricted Translation-en [325 kB] 2023-10-01T09:53:26.200 INFO:teuthology.orchestra.run.smithi078.stdout:Get:20 http://archive.ubuntu.com/ubuntu focal-backports InRelease [108 kB] 2023-10-01T09:53:26.206 INFO:teuthology.orchestra.run.smithi173.stdout:Get:28 http://archive.ubuntu.com/ubuntu focal-updates/restricted amd64 c-n-f Metadata [572 B] 2023-10-01T09:53:26.206 INFO:teuthology.orchestra.run.smithi173.stdout:Get:29 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 Packages [1,117 kB] 2023-10-01T09:53:26.214 INFO:teuthology.orchestra.run.smithi173.stdout:Get:30 http://archive.ubuntu.com/ubuntu focal-updates/universe i386 Packages [748 kB] 2023-10-01T09:53:26.220 INFO:teuthology.orchestra.run.smithi173.stdout:Get:31 http://archive.ubuntu.com/ubuntu focal-updates/universe Translation-en [267 kB] 2023-10-01T09:53:26.221 INFO:teuthology.orchestra.run.smithi173.stdout:Get:32 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 c-n-f Metadata [25.6 kB] 2023-10-01T09:53:26.222 INFO:teuthology.orchestra.run.smithi173.stdout:Get:33 http://archive.ubuntu.com/ubuntu focal-updates/multiverse amd64 Packages [25.8 kB] 2023-10-01T09:53:26.222 INFO:teuthology.orchestra.run.smithi173.stdout:Get:34 http://archive.ubuntu.com/ubuntu focal-updates/multiverse i386 Packages [8,436 B] 2023-10-01T09:53:26.223 INFO:teuthology.orchestra.run.smithi173.stdout:Get:35 http://archive.ubuntu.com/ubuntu focal-updates/multiverse Translation-en [7,484 B] 2023-10-01T09:53:26.223 INFO:teuthology.orchestra.run.smithi173.stdout:Get:36 http://archive.ubuntu.com/ubuntu focal-updates/multiverse amd64 c-n-f Metadata [620 B] 2023-10-01T09:53:26.242 INFO:teuthology.orchestra.run.smithi173.stdout:Get:37 http://archive.ubuntu.com/ubuntu focal-backports/main i386 Packages [36.1 kB] 2023-10-01T09:53:26.242 INFO:teuthology.orchestra.run.smithi173.stdout:Get:38 http://archive.ubuntu.com/ubuntu focal-backports/main amd64 Packages [45.7 kB] 2023-10-01T09:53:26.243 INFO:teuthology.orchestra.run.smithi173.stdout:Get:39 http://archive.ubuntu.com/ubuntu focal-backports/main amd64 c-n-f Metadata [1,420 B] 2023-10-01T09:53:26.243 INFO:teuthology.orchestra.run.smithi173.stdout:Get:40 http://archive.ubuntu.com/ubuntu focal-backports/universe amd64 Packages [25.0 kB] 2023-10-01T09:53:26.261 INFO:teuthology.orchestra.run.smithi173.stdout:Get:41 http://archive.ubuntu.com/ubuntu focal-backports/universe i386 Packages [13.8 kB] 2023-10-01T09:53:26.262 INFO:teuthology.orchestra.run.smithi173.stdout:Get:42 http://archive.ubuntu.com/ubuntu focal-backports/universe amd64 c-n-f Metadata [880 B] 2023-10-01T09:53:26.342 INFO:teuthology.orchestra.run.smithi078.stdout:Get:21 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 Packages [2,865 kB] 2023-10-01T09:53:26.721 INFO:teuthology.orchestra.run.smithi078.stdout:Get:22 http://archive.ubuntu.com/ubuntu focal-updates/main i386 Packages [885 kB] 2023-10-01T09:53:26.751 INFO:teuthology.orchestra.run.smithi078.stdout:Get:23 http://archive.ubuntu.com/ubuntu focal-updates/main Translation-en [469 kB] 2023-10-01T09:53:26.764 INFO:teuthology.orchestra.run.smithi078.stdout:Get:24 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 c-n-f Metadata [17.1 kB] 2023-10-01T09:53:26.764 INFO:teuthology.orchestra.run.smithi078.stdout:Get:25 http://archive.ubuntu.com/ubuntu focal-updates/restricted i386 Packages [34.6 kB] 2023-10-01T09:53:26.765 INFO:teuthology.orchestra.run.smithi078.stdout:Get:26 http://archive.ubuntu.com/ubuntu focal-updates/restricted amd64 Packages [2,330 kB] 2023-10-01T09:53:26.824 INFO:teuthology.orchestra.run.smithi078.stdout:Get:27 http://archive.ubuntu.com/ubuntu focal-updates/restricted Translation-en [325 kB] 2023-10-01T09:53:26.844 INFO:teuthology.orchestra.run.smithi078.stdout:Get:28 http://archive.ubuntu.com/ubuntu focal-updates/restricted amd64 c-n-f Metadata [572 B] 2023-10-01T09:53:27.008 INFO:teuthology.orchestra.run.smithi078.stdout:Get:29 http://archive.ubuntu.com/ubuntu focal-updates/universe i386 Packages [748 kB] 2023-10-01T09:53:27.595 INFO:teuthology.orchestra.run.smithi078.stdout:Get:30 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 Packages [1,117 kB] 2023-10-01T09:53:27.703 INFO:teuthology.orchestra.run.smithi078.stdout:Get:31 http://archive.ubuntu.com/ubuntu focal-updates/universe Translation-en [267 kB] 2023-10-01T09:53:27.724 INFO:teuthology.orchestra.run.smithi078.stdout:Get:32 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 c-n-f Metadata [25.6 kB] 2023-10-01T09:53:27.726 INFO:teuthology.orchestra.run.smithi078.stdout:Get:33 http://archive.ubuntu.com/ubuntu focal-updates/multiverse i386 Packages [8,436 B] 2023-10-01T09:53:27.727 INFO:teuthology.orchestra.run.smithi078.stdout:Get:34 http://archive.ubuntu.com/ubuntu focal-updates/multiverse amd64 Packages [25.8 kB] 2023-10-01T09:53:27.728 INFO:teuthology.orchestra.run.smithi078.stdout:Get:35 http://archive.ubuntu.com/ubuntu focal-updates/multiverse Translation-en [7,484 B] 2023-10-01T09:53:27.729 INFO:teuthology.orchestra.run.smithi078.stdout:Get:36 http://archive.ubuntu.com/ubuntu focal-updates/multiverse amd64 c-n-f Metadata [620 B] 2023-10-01T09:53:27.729 INFO:teuthology.orchestra.run.smithi078.stdout:Get:37 http://archive.ubuntu.com/ubuntu focal-backports/main i386 Packages [36.1 kB] 2023-10-01T09:53:27.731 INFO:teuthology.orchestra.run.smithi078.stdout:Get:38 http://archive.ubuntu.com/ubuntu focal-backports/main amd64 Packages [45.7 kB] 2023-10-01T09:53:27.734 INFO:teuthology.orchestra.run.smithi078.stdout:Get:39 http://archive.ubuntu.com/ubuntu focal-backports/main amd64 c-n-f Metadata [1,420 B] 2023-10-01T09:53:27.899 INFO:teuthology.orchestra.run.smithi078.stdout:Get:40 http://archive.ubuntu.com/ubuntu focal-backports/universe i386 Packages [13.8 kB] 2023-10-01T09:53:28.062 INFO:teuthology.orchestra.run.smithi078.stdout:Get:41 http://archive.ubuntu.com/ubuntu focal-backports/universe amd64 Packages [25.0 kB] 2023-10-01T09:53:28.113 INFO:teuthology.orchestra.run.smithi078.stdout:Get:42 http://archive.ubuntu.com/ubuntu focal-backports/universe amd64 c-n-f Metadata [880 B] 2023-10-01T09:53:29.884 INFO:teuthology.orchestra.run.smithi173.stdout:Fetched 17.4 MB in 3s (6,901 kB/s) 2023-10-01T09:53:30.078 INFO:teuthology.orchestra.run.smithi078.stdout:Fetched 17.4 MB in 3s (6,405 kB/s) 2023-10-01T09:53:31.389 INFO:teuthology.orchestra.run.smithi173.stdout:Reading package lists... 2023-10-01T09:53:31.416 DEBUG:teuthology.orchestra.run.smithi173:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic-hwe-20.04 2023-10-01T09:53:31.491 INFO:teuthology.orchestra.run.smithi173.stdout:Reading package lists... 2023-10-01T09:53:31.551 INFO:teuthology.orchestra.run.smithi078.stdout:Reading package lists... 2023-10-01T09:53:31.577 DEBUG:teuthology.orchestra.run.smithi078:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic-hwe-20.04 2023-10-01T09:53:31.644 INFO:teuthology.orchestra.run.smithi078.stdout:Reading package lists... 2023-10-01T09:53:31.700 INFO:teuthology.orchestra.run.smithi173.stdout:Building dependency tree... 2023-10-01T09:53:31.701 INFO:teuthology.orchestra.run.smithi173.stdout:Reading state information... 2023-10-01T09:53:31.845 INFO:teuthology.orchestra.run.smithi078.stdout:Building dependency tree... 2023-10-01T09:53:31.846 INFO:teuthology.orchestra.run.smithi078.stdout:Reading state information... 2023-10-01T09:53:31.887 INFO:teuthology.orchestra.run.smithi173.stdout:The following additional packages will be installed: 2023-10-01T09:53:31.887 INFO:teuthology.orchestra.run.smithi173.stdout: libdbus-glib-1-2 libevdev2 libimobiledevice6 libplist3 libupower-glib3 2023-10-01T09:53:31.887 INFO:teuthology.orchestra.run.smithi173.stdout: libusbmuxd6 linux-image-5.15.0-84-generic linux-modules-5.15.0-84-generic 2023-10-01T09:53:31.887 INFO:teuthology.orchestra.run.smithi173.stdout: linux-modules-extra-5.15.0-84-generic thermald upower usbmuxd 2023-10-01T09:53:31.888 INFO:teuthology.orchestra.run.smithi173.stdout:Suggested packages: 2023-10-01T09:53:31.888 INFO:teuthology.orchestra.run.smithi173.stdout: libusbmuxd-tools fdutils linux-doc | linux-hwe-5.15-source-5.15.0 2023-10-01T09:53:31.888 INFO:teuthology.orchestra.run.smithi173.stdout: linux-hwe-5.15-tools linux-headers-5.15.0-84-generic 2023-10-01T09:53:31.932 INFO:teuthology.orchestra.run.smithi173.stdout:The following NEW packages will be installed: 2023-10-01T09:53:31.932 INFO:teuthology.orchestra.run.smithi173.stdout: libdbus-glib-1-2 libevdev2 libimobiledevice6 libplist3 libupower-glib3 2023-10-01T09:53:31.932 INFO:teuthology.orchestra.run.smithi173.stdout: libusbmuxd6 linux-image-5.15.0-84-generic linux-image-generic-hwe-20.04 2023-10-01T09:53:31.933 INFO:teuthology.orchestra.run.smithi173.stdout: linux-modules-5.15.0-84-generic linux-modules-extra-5.15.0-84-generic 2023-10-01T09:53:31.933 INFO:teuthology.orchestra.run.smithi173.stdout: thermald upower usbmuxd 2023-10-01T09:53:32.014 INFO:teuthology.orchestra.run.smithi078.stdout:The following additional packages will be installed: 2023-10-01T09:53:32.014 INFO:teuthology.orchestra.run.smithi078.stdout: libdbus-glib-1-2 libevdev2 libimobiledevice6 libplist3 libupower-glib3 2023-10-01T09:53:32.014 INFO:teuthology.orchestra.run.smithi078.stdout: libusbmuxd6 linux-image-5.15.0-84-generic linux-modules-5.15.0-84-generic 2023-10-01T09:53:32.015 INFO:teuthology.orchestra.run.smithi078.stdout: linux-modules-extra-5.15.0-84-generic thermald upower usbmuxd 2023-10-01T09:53:32.015 INFO:teuthology.orchestra.run.smithi078.stdout:Suggested packages: 2023-10-01T09:53:32.015 INFO:teuthology.orchestra.run.smithi078.stdout: libusbmuxd-tools fdutils linux-doc | linux-hwe-5.15-source-5.15.0 2023-10-01T09:53:32.015 INFO:teuthology.orchestra.run.smithi078.stdout: linux-hwe-5.15-tools linux-headers-5.15.0-84-generic 2023-10-01T09:53:32.069 INFO:teuthology.orchestra.run.smithi078.stdout:The following NEW packages will be installed: 2023-10-01T09:53:32.070 INFO:teuthology.orchestra.run.smithi078.stdout: libdbus-glib-1-2 libevdev2 libimobiledevice6 libplist3 libupower-glib3 2023-10-01T09:53:32.070 INFO:teuthology.orchestra.run.smithi078.stdout: libusbmuxd6 linux-image-5.15.0-84-generic linux-image-generic-hwe-20.04 2023-10-01T09:53:32.070 INFO:teuthology.orchestra.run.smithi078.stdout: linux-modules-5.15.0-84-generic linux-modules-extra-5.15.0-84-generic 2023-10-01T09:53:32.070 INFO:teuthology.orchestra.run.smithi078.stdout: thermald upower usbmuxd 2023-10-01T09:53:32.115 INFO:teuthology.orchestra.run.smithi173.stdout:0 upgraded, 13 newly installed, 0 to remove and 266 not upgraded. 2023-10-01T09:53:32.115 INFO:teuthology.orchestra.run.smithi173.stdout:Need to get 90.9 MB of archives. 2023-10-01T09:53:32.115 INFO:teuthology.orchestra.run.smithi173.stdout:After this operation, 497 MB of additional disk space will be used. 2023-10-01T09:53:32.115 INFO:teuthology.orchestra.run.smithi173.stdout:Get:1 http://archive.ubuntu.com/ubuntu focal/main amd64 libdbus-glib-1-2 amd64 0.110-5fakssync1 [59.1 kB] 2023-10-01T09:53:32.255 INFO:teuthology.orchestra.run.smithi078.stdout:0 upgraded, 13 newly installed, 0 to remove and 266 not upgraded. 2023-10-01T09:53:32.255 INFO:teuthology.orchestra.run.smithi078.stdout:Need to get 90.9 MB of archives. 2023-10-01T09:53:32.255 INFO:teuthology.orchestra.run.smithi078.stdout:After this operation, 497 MB of additional disk space will be used. 2023-10-01T09:53:32.255 INFO:teuthology.orchestra.run.smithi078.stdout:Get:1 http://archive.ubuntu.com/ubuntu focal/main amd64 libdbus-glib-1-2 amd64 0.110-5fakssync1 [59.1 kB] 2023-10-01T09:53:32.385 INFO:teuthology.orchestra.run.smithi173.stdout:Get:2 http://archive.ubuntu.com/ubuntu focal/main amd64 libplist3 amd64 2.1.0-4build2 [31.6 kB] 2023-10-01T09:53:32.413 INFO:teuthology.orchestra.run.smithi173.stdout:Get:3 http://archive.ubuntu.com/ubuntu focal/main amd64 libusbmuxd6 amd64 2.0.1-2 [19.1 kB] 2023-10-01T09:53:32.431 INFO:teuthology.orchestra.run.smithi173.stdout:Get:4 http://archive.ubuntu.com/ubuntu focal/main amd64 libimobiledevice6 amd64 1.2.1~git20191129.9f79242-1build1 [65.2 kB] 2023-10-01T09:53:32.504 INFO:teuthology.orchestra.run.smithi173.stdout:Get:5 http://archive.ubuntu.com/ubuntu focal/main amd64 libupower-glib3 amd64 0.99.11-1build2 [43.2 kB] 2023-10-01T09:53:32.526 INFO:teuthology.orchestra.run.smithi078.stdout:Get:2 http://archive.ubuntu.com/ubuntu focal/main amd64 libplist3 amd64 2.1.0-4build2 [31.6 kB] 2023-10-01T09:53:32.540 INFO:teuthology.orchestra.run.smithi173.stdout:Get:6 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-modules-5.15.0-84-generic amd64 5.15.0-84.93~20.04.1 [20.9 MB] 2023-10-01T09:53:32.554 INFO:teuthology.orchestra.run.smithi078.stdout:Get:3 http://archive.ubuntu.com/ubuntu focal/main amd64 libusbmuxd6 amd64 2.0.1-2 [19.1 kB] 2023-10-01T09:53:32.572 INFO:teuthology.orchestra.run.smithi078.stdout:Get:4 http://archive.ubuntu.com/ubuntu focal/main amd64 libimobiledevice6 amd64 1.2.1~git20191129.9f79242-1build1 [65.2 kB] 2023-10-01T09:53:32.646 INFO:teuthology.orchestra.run.smithi078.stdout:Get:5 http://archive.ubuntu.com/ubuntu focal/main amd64 libupower-glib3 amd64 0.99.11-1build2 [43.2 kB] 2023-10-01T09:53:32.682 INFO:teuthology.orchestra.run.smithi078.stdout:Get:6 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-modules-5.15.0-84-generic amd64 5.15.0-84.93~20.04.1 [20.9 MB] 2023-10-01T09:53:33.429 INFO:teuthology.orchestra.run.smithi173.stdout:Get:7 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-image-5.15.0-84-generic amd64 5.15.0-84.93~20.04.1 [11.4 MB] 2023-10-01T09:53:33.593 INFO:teuthology.orchestra.run.smithi078.stdout:Get:7 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-image-5.15.0-84-generic amd64 5.15.0-84.93~20.04.1 [11.4 MB] 2023-10-01T09:53:33.718 INFO:teuthology.orchestra.run.smithi173.stdout:Get:8 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-modules-extra-5.15.0-84-generic amd64 5.15.0-84.93~20.04.1 [58.0 MB] 2023-10-01T09:53:33.898 INFO:teuthology.orchestra.run.smithi078.stdout:Get:8 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-modules-extra-5.15.0-84-generic amd64 5.15.0-84.93~20.04.1 [58.0 MB] 2023-10-01T09:53:35.636 INFO:teuthology.orchestra.run.smithi173.stdout:Get:9 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-image-generic-hwe-20.04 amd64 5.15.0.84.93~20.04.42 [2,692 B] 2023-10-01T09:53:35.636 INFO:teuthology.orchestra.run.smithi173.stdout:Get:10 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 libevdev2 amd64 1.9.0+dfsg-1ubuntu0.2 [31.6 kB] 2023-10-01T09:53:35.637 INFO:teuthology.orchestra.run.smithi173.stdout:Get:11 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 thermald amd64 1.9.1-1ubuntu0.6 [233 kB] 2023-10-01T09:53:35.806 INFO:teuthology.orchestra.run.smithi173.stdout:Get:12 http://archive.ubuntu.com/ubuntu focal/main amd64 upower amd64 0.99.11-1build2 [104 kB] 2023-10-01T09:53:35.848 INFO:teuthology.orchestra.run.smithi078.stdout:Get:9 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 linux-image-generic-hwe-20.04 amd64 5.15.0.84.93~20.04.42 [2,692 B] 2023-10-01T09:53:35.848 INFO:teuthology.orchestra.run.smithi078.stdout:Get:10 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 libevdev2 amd64 1.9.0+dfsg-1ubuntu0.2 [31.6 kB] 2023-10-01T09:53:35.849 INFO:teuthology.orchestra.run.smithi078.stdout:Get:11 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 thermald amd64 1.9.1-1ubuntu0.6 [233 kB] 2023-10-01T09:53:36.019 INFO:teuthology.orchestra.run.smithi078.stdout:Get:12 http://archive.ubuntu.com/ubuntu focal/main amd64 upower amd64 0.99.11-1build2 [104 kB] 2023-10-01T09:53:36.151 INFO:teuthology.orchestra.run.smithi173.stdout:Get:13 http://archive.ubuntu.com/ubuntu focal/main amd64 usbmuxd amd64 1.1.1~git20191130.9af2b12-1 [38.4 kB] 2023-10-01T09:53:36.363 INFO:teuthology.orchestra.run.smithi078.stdout:Get:13 http://archive.ubuntu.com/ubuntu focal/main amd64 usbmuxd amd64 1.1.1~git20191130.9af2b12-1 [38.4 kB] 2023-10-01T09:53:36.858 INFO:teuthology.orchestra.run.smithi173.stdout:Fetched 90.9 MB in 4s (21.5 MB/s) 2023-10-01T09:53:37.044 INFO:teuthology.orchestra.run.smithi078.stdout:Fetched 90.9 MB in 4s (21.1 MB/s) 2023-10-01T09:53:37.074 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package libdbus-glib-1-2:amd64. 2023-10-01T09:53:37.275 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package libdbus-glib-1-2:amd64. 2023-10-01T09:53:38.805 INFO:teuthology.orchestra.run.smithi173.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 104373 files and directories currently installed.) 2023-10-01T09:53:38.811 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../00-libdbus-glib-1-2_0.110-5fakssync1_amd64.deb ... 2023-10-01T09:53:38.964 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking libdbus-glib-1-2:amd64 (0.110-5fakssync1) ... 2023-10-01T09:53:38.972 INFO:teuthology.orchestra.run.smithi078.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 104373 files and directories currently installed.) 2023-10-01T09:53:38.976 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../00-libdbus-glib-1-2_0.110-5fakssync1_amd64.deb ... 2023-10-01T09:53:39.148 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking libdbus-glib-1-2:amd64 (0.110-5fakssync1) ... 2023-10-01T09:53:39.350 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package libplist3:amd64. 2023-10-01T09:53:39.365 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../01-libplist3_2.1.0-4build2_amd64.deb ... 2023-10-01T09:53:39.409 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking libplist3:amd64 (2.1.0-4build2) ... 2023-10-01T09:53:39.484 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package libplist3:amd64. 2023-10-01T09:53:39.498 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../01-libplist3_2.1.0-4build2_amd64.deb ... 2023-10-01T09:53:39.542 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking libplist3:amd64 (2.1.0-4build2) ... 2023-10-01T09:53:39.754 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package libusbmuxd6:amd64. 2023-10-01T09:53:39.767 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../02-libusbmuxd6_2.0.1-2_amd64.deb ... 2023-10-01T09:53:39.803 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking libusbmuxd6:amd64 (2.0.1-2) ... 2023-10-01T09:53:39.920 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package libusbmuxd6:amd64. 2023-10-01T09:53:39.934 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../02-libusbmuxd6_2.0.1-2_amd64.deb ... 2023-10-01T09:53:39.978 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking libusbmuxd6:amd64 (2.0.1-2) ... 2023-10-01T09:53:40.157 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package libimobiledevice6:amd64. 2023-10-01T09:53:40.170 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../03-libimobiledevice6_1.2.1~git20191129.9f79242-1build1_amd64.deb ... 2023-10-01T09:53:40.207 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking libimobiledevice6:amd64 (1.2.1~git20191129.9f79242-1build1) ... 2023-10-01T09:53:40.331 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package libimobiledevice6:amd64. 2023-10-01T09:53:40.345 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../03-libimobiledevice6_1.2.1~git20191129.9f79242-1build1_amd64.deb ... 2023-10-01T09:53:40.397 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking libimobiledevice6:amd64 (1.2.1~git20191129.9f79242-1build1) ... 2023-10-01T09:53:40.585 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package libupower-glib3:amd64. 2023-10-01T09:53:40.599 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../04-libupower-glib3_0.99.11-1build2_amd64.deb ... 2023-10-01T09:53:40.644 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking libupower-glib3:amd64 (0.99.11-1build2) ... 2023-10-01T09:53:40.717 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package libupower-glib3:amd64. 2023-10-01T09:53:40.730 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../04-libupower-glib3_0.99.11-1build2_amd64.deb ... 2023-10-01T09:53:40.759 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking libupower-glib3:amd64 (0.99.11-1build2) ... 2023-10-01T09:53:40.988 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package linux-modules-5.15.0-84-generic. 2023-10-01T09:53:41.001 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../05-linux-modules-5.15.0-84-generic_5.15.0-84.93~20.04.1_amd64.deb ... 2023-10-01T09:53:41.038 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking linux-modules-5.15.0-84-generic (5.15.0-84.93~20.04.1) ... 2023-10-01T09:53:41.111 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package linux-modules-5.15.0-84-generic. 2023-10-01T09:53:41.125 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../05-linux-modules-5.15.0-84-generic_5.15.0-84.93~20.04.1_amd64.deb ... 2023-10-01T09:53:41.170 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking linux-modules-5.15.0-84-generic (5.15.0-84.93~20.04.1) ... 2023-10-01T09:53:43.633 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package linux-image-5.15.0-84-generic. 2023-10-01T09:53:43.649 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../06-linux-image-5.15.0-84-generic_5.15.0-84.93~20.04.1_amd64.deb ... 2023-10-01T09:53:43.747 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking linux-image-5.15.0-84-generic (5.15.0-84.93~20.04.1) ... 2023-10-01T09:53:43.753 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package linux-image-5.15.0-84-generic. 2023-10-01T09:53:43.767 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../06-linux-image-5.15.0-84-generic_5.15.0-84.93~20.04.1_amd64.deb ... 2023-10-01T09:53:43.874 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking linux-image-5.15.0-84-generic (5.15.0-84.93~20.04.1) ... 2023-10-01T09:53:44.237 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package linux-modules-extra-5.15.0-84-generic. 2023-10-01T09:53:44.250 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../07-linux-modules-extra-5.15.0-84-generic_5.15.0-84.93~20.04.1_amd64.deb ... 2023-10-01T09:53:44.320 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking linux-modules-extra-5.15.0-84-generic (5.15.0-84.93~20.04.1) ... 2023-10-01T09:53:44.441 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package linux-modules-extra-5.15.0-84-generic. 2023-10-01T09:53:44.453 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../07-linux-modules-extra-5.15.0-84-generic_5.15.0-84.93~20.04.1_amd64.deb ... 2023-10-01T09:53:44.498 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking linux-modules-extra-5.15.0-84-generic (5.15.0-84.93~20.04.1) ... 2023-10-01T09:53:51.380 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package linux-image-generic-hwe-20.04. 2023-10-01T09:53:51.398 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../08-linux-image-generic-hwe-20.04_5.15.0.84.93~20.04.42_amd64.deb ... 2023-10-01T09:53:51.438 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking linux-image-generic-hwe-20.04 (5.15.0.84.93~20.04.42) ... 2023-10-01T09:53:51.552 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package linux-image-generic-hwe-20.04. 2023-10-01T09:53:51.568 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../08-linux-image-generic-hwe-20.04_5.15.0.84.93~20.04.42_amd64.deb ... 2023-10-01T09:53:51.631 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking linux-image-generic-hwe-20.04 (5.15.0.84.93~20.04.42) ... 2023-10-01T09:53:51.807 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package libevdev2:amd64. 2023-10-01T09:53:51.822 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../09-libevdev2_1.9.0+dfsg-1ubuntu0.2_amd64.deb ... 2023-10-01T09:53:51.891 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking libevdev2:amd64 (1.9.0+dfsg-1ubuntu0.2) ... 2023-10-01T09:53:51.938 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package libevdev2:amd64. 2023-10-01T09:53:51.954 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../09-libevdev2_1.9.0+dfsg-1ubuntu0.2_amd64.deb ... 2023-10-01T09:53:52.018 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking libevdev2:amd64 (1.9.0+dfsg-1ubuntu0.2) ... 2023-10-01T09:53:52.294 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package thermald. 2023-10-01T09:53:52.309 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../10-thermald_1.9.1-1ubuntu0.6_amd64.deb ... 2023-10-01T09:53:52.352 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking thermald (1.9.1-1ubuntu0.6) ... 2023-10-01T09:53:52.406 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package thermald. 2023-10-01T09:53:52.422 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../10-thermald_1.9.1-1ubuntu0.6_amd64.deb ... 2023-10-01T09:53:52.465 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking thermald (1.9.1-1ubuntu0.6) ... 2023-10-01T09:53:52.931 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package upower. 2023-10-01T09:53:52.946 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../11-upower_0.99.11-1build2_amd64.deb ... 2023-10-01T09:53:52.996 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package upower. 2023-10-01T09:53:53.004 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking upower (0.99.11-1build2) ... 2023-10-01T09:53:53.012 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../11-upower_0.99.11-1build2_amd64.deb ... 2023-10-01T09:53:53.123 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking upower (0.99.11-1build2) ... 2023-10-01T09:53:53.453 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package usbmuxd. 2023-10-01T09:53:53.466 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../12-usbmuxd_1.1.1~git20191130.9af2b12-1_amd64.deb ... 2023-10-01T09:53:53.502 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking usbmuxd (1.1.1~git20191130.9af2b12-1) ... 2023-10-01T09:53:53.550 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package usbmuxd. 2023-10-01T09:53:53.566 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../12-usbmuxd_1.1.1~git20191130.9af2b12-1_amd64.deb ... 2023-10-01T09:53:53.608 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking usbmuxd (1.1.1~git20191130.9af2b12-1) ... 2023-10-01T09:53:53.815 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up libplist3:amd64 (2.1.0-4build2) ... 2023-10-01T09:53:53.896 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up libplist3:amd64 (2.1.0-4build2) ... 2023-10-01T09:53:53.940 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up libdbus-glib-1-2:amd64 (0.110-5fakssync1) ... 2023-10-01T09:53:54.054 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up libdbus-glib-1-2:amd64 (0.110-5fakssync1) ... 2023-10-01T09:53:54.108 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up libusbmuxd6:amd64 (2.0.1-2) ... 2023-10-01T09:53:54.180 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up libusbmuxd6:amd64 (2.0.1-2) ... 2023-10-01T09:53:54.234 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up libupower-glib3:amd64 (0.99.11-1build2) ... 2023-10-01T09:53:54.331 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up libupower-glib3:amd64 (0.99.11-1build2) ... 2023-10-01T09:53:54.385 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up libimobiledevice6:amd64 (1.2.1~git20191129.9f79242-1build1) ... 2023-10-01T09:53:54.457 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up libimobiledevice6:amd64 (1.2.1~git20191129.9f79242-1build1) ... 2023-10-01T09:53:54.527 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up libevdev2:amd64 (1.9.0+dfsg-1ubuntu0.2) ... 2023-10-01T09:53:54.583 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up libevdev2:amd64 (1.9.0+dfsg-1ubuntu0.2) ... 2023-10-01T09:53:54.678 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up upower (0.99.11-1build2) ... 2023-10-01T09:53:54.743 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up upower (0.99.11-1build2) ... 2023-10-01T09:53:55.235 INFO:teuthology.orchestra.run.smithi078.stdout:upower.service is a disabled or a static unit, not starting it. 2023-10-01T09:53:55.284 INFO:teuthology.orchestra.run.smithi173.stdout:upower.service is a disabled or a static unit, not starting it. 2023-10-01T09:53:55.291 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up usbmuxd (1.1.1~git20191130.9af2b12-1) ... 2023-10-01T09:53:55.355 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up usbmuxd (1.1.1~git20191130.9af2b12-1) ... 2023-10-01T09:53:55.550 INFO:teuthology.orchestra.run.smithi173.stdout:Warning: The home dir /var/lib/usbmux you specified can't be accessed: No such file or directory 2023-10-01T09:53:55.570 INFO:teuthology.orchestra.run.smithi173.stdout:Adding system user `usbmux' (UID 114) ... 2023-10-01T09:53:55.570 INFO:teuthology.orchestra.run.smithi173.stdout:Adding new user `usbmux' (UID 114) with group `plugdev' ... 2023-10-01T09:53:55.593 INFO:teuthology.orchestra.run.smithi078.stdout:Warning: The home dir /var/lib/usbmux you specified can't be accessed: No such file or directory 2023-10-01T09:53:55.605 INFO:teuthology.orchestra.run.smithi078.stdout:Adding system user `usbmux' (UID 114) ... 2023-10-01T09:53:55.605 INFO:teuthology.orchestra.run.smithi078.stdout:Adding new user `usbmux' (UID 114) with group `plugdev' ... 2023-10-01T09:53:56.106 INFO:teuthology.orchestra.run.smithi173.stdout:Not creating home directory `/var/lib/usbmux'. 2023-10-01T09:53:56.141 INFO:teuthology.orchestra.run.smithi078.stdout:Not creating home directory `/var/lib/usbmux'. 2023-10-01T09:53:56.322 INFO:teuthology.orchestra.run.smithi173.stdout:usbmuxd.service is a disabled or a static unit, not starting it. 2023-10-01T09:53:56.361 INFO:teuthology.orchestra.run.smithi078.stdout:usbmuxd.service is a disabled or a static unit, not starting it. 2023-10-01T09:53:56.369 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up thermald (1.9.1-1ubuntu0.6) ... 2023-10-01T09:53:56.404 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up thermald (1.9.1-1ubuntu0.6) ... 2023-10-01T09:53:56.676 INFO:teuthology.orchestra.run.smithi173.stdout:Created symlink /etc/systemd/system/dbus-org.freedesktop.thermald.service -> /lib/systemd/system/thermald.service. 2023-10-01T09:53:56.676 INFO:teuthology.orchestra.run.smithi173.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/thermald.service -> /lib/systemd/system/thermald.service. 2023-10-01T09:53:56.719 INFO:teuthology.orchestra.run.smithi078.stdout:Created symlink /etc/systemd/system/dbus-org.freedesktop.thermald.service -> /lib/systemd/system/thermald.service. 2023-10-01T09:53:56.719 INFO:teuthology.orchestra.run.smithi078.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/thermald.service -> /lib/systemd/system/thermald.service. 2023-10-01T09:53:57.141 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up linux-image-5.15.0-84-generic (5.15.0-84.93~20.04.1) ... 2023-10-01T09:53:57.167 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up linux-image-5.15.0-84-generic (5.15.0-84.93~20.04.1) ... 2023-10-01T09:53:58.488 INFO:teuthology.orchestra.run.smithi078.stdout:I: /vmlinuz is now a symlink to boot/vmlinuz-5.15.0-84-generic 2023-10-01T09:53:58.489 INFO:teuthology.orchestra.run.smithi078.stdout:I: /initrd.img is now a symlink to boot/initrd.img-5.15.0-84-generic 2023-10-01T09:53:58.645 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up linux-modules-extra-5.15.0-84-generic (5.15.0-84.93~20.04.1) ... 2023-10-01T09:53:58.667 INFO:teuthology.orchestra.run.smithi173.stdout:I: /vmlinuz is now a symlink to boot/vmlinuz-5.15.0-84-generic 2023-10-01T09:53:58.668 INFO:teuthology.orchestra.run.smithi173.stdout:I: /initrd.img is now a symlink to boot/initrd.img-5.15.0-84-generic 2023-10-01T09:53:59.334 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up linux-modules-extra-5.15.0-84-generic (5.15.0-84.93~20.04.1) ... 2023-10-01T09:54:00.030 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up linux-image-generic-hwe-20.04 (5.15.0.84.93~20.04.42) ... 2023-10-01T09:54:00.155 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up linux-modules-5.15.0-84-generic (5.15.0-84.93~20.04.1) ... 2023-10-01T09:54:00.752 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up linux-image-generic-hwe-20.04 (5.15.0.84.93~20.04.42) ... 2023-10-01T09:54:00.877 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up linux-modules-5.15.0-84-generic (5.15.0-84.93~20.04.1) ... 2023-10-01T09:54:02.279 INFO:teuthology.orchestra.run.smithi173.stdout:Processing triggers for libc-bin (2.31-0ubuntu9.9) ... 2023-10-01T09:54:02.382 INFO:teuthology.orchestra.run.smithi078.stdout:Processing triggers for libc-bin (2.31-0ubuntu9.9) ... 2023-10-01T09:54:02.592 INFO:teuthology.orchestra.run.smithi173.stdout:Processing triggers for man-db (2.9.1-1) ... 2023-10-01T09:54:02.696 INFO:teuthology.orchestra.run.smithi078.stdout:Processing triggers for man-db (2.9.1-1) ... 2023-10-01T09:54:03.362 INFO:teuthology.orchestra.run.smithi173.stdout:Processing triggers for dbus (1.12.16-2ubuntu2.2) ... 2023-10-01T09:54:03.471 INFO:teuthology.orchestra.run.smithi173.stdout:Processing triggers for linux-image-5.15.0-84-generic (5.15.0-84.93~20.04.1) ... 2023-10-01T09:54:03.474 INFO:teuthology.orchestra.run.smithi078.stdout:Processing triggers for dbus (1.12.16-2ubuntu2.2) ... 2023-10-01T09:54:03.538 INFO:teuthology.orchestra.run.smithi173.stdout:/etc/kernel/postinst.d/initramfs-tools: 2023-10-01T09:54:03.538 INFO:teuthology.orchestra.run.smithi173.stdout:update-initramfs: Generating /boot/initrd.img-5.15.0-84-generic 2023-10-01T09:54:03.616 INFO:teuthology.orchestra.run.smithi078.stdout:Processing triggers for linux-image-5.15.0-84-generic (5.15.0-84.93~20.04.1) ... 2023-10-01T09:54:03.704 INFO:teuthology.orchestra.run.smithi078.stdout:/etc/kernel/postinst.d/initramfs-tools: 2023-10-01T09:54:03.704 INFO:teuthology.orchestra.run.smithi078.stdout:update-initramfs: Generating /boot/initrd.img-5.15.0-84-generic 2023-10-01T09:54:29.013 INFO:teuthology.orchestra.run.smithi173.stdout:/etc/kernel/postinst.d/zz-update-grub: 2023-10-01T09:54:29.013 INFO:teuthology.orchestra.run.smithi173.stdout:Sourcing file `/etc/default/grub' 2023-10-01T09:54:29.025 INFO:teuthology.orchestra.run.smithi173.stdout:Sourcing file `/etc/default/grub.d/init-select.cfg' 2023-10-01T09:54:29.057 INFO:teuthology.orchestra.run.smithi173.stdout:Generating grub configuration file ... 2023-10-01T09:54:29.298 INFO:teuthology.orchestra.run.smithi078.stdout:/etc/kernel/postinst.d/zz-update-grub: 2023-10-01T09:54:29.298 INFO:teuthology.orchestra.run.smithi078.stdout:Sourcing file `/etc/default/grub' 2023-10-01T09:54:29.310 INFO:teuthology.orchestra.run.smithi078.stdout:Sourcing file `/etc/default/grub.d/init-select.cfg' 2023-10-01T09:54:29.331 INFO:teuthology.orchestra.run.smithi078.stdout:Generating grub configuration file ... 2023-10-01T09:54:29.381 INFO:teuthology.orchestra.run.smithi173.stdout:Found linux image: /boot/vmlinuz-5.15.0-84-generic 2023-10-01T09:54:29.404 INFO:teuthology.orchestra.run.smithi173.stdout:Found initrd image: /boot/initrd.img-5.15.0-84-generic 2023-10-01T09:54:29.667 INFO:teuthology.orchestra.run.smithi078.stdout:Found linux image: /boot/vmlinuz-5.15.0-84-generic 2023-10-01T09:54:29.686 INFO:teuthology.orchestra.run.smithi078.stdout:Found initrd image: /boot/initrd.img-5.15.0-84-generic 2023-10-01T09:54:29.830 INFO:teuthology.orchestra.run.smithi173.stdout:Found linux image: /boot/vmlinuz-5.4.0-124-generic 2023-10-01T09:54:29.841 INFO:teuthology.orchestra.run.smithi173.stdout:Found initrd image: /boot/initrd.img-5.4.0-124-generic 2023-10-01T09:54:30.110 INFO:teuthology.orchestra.run.smithi078.stdout:Found linux image: /boot/vmlinuz-5.4.0-124-generic 2023-10-01T09:54:30.120 INFO:teuthology.orchestra.run.smithi078.stdout:Found initrd image: /boot/initrd.img-5.4.0-124-generic 2023-10-01T09:54:31.863 INFO:teuthology.orchestra.run.smithi173.stdout:done 2023-10-01T09:54:32.214 INFO:teuthology.orchestra.run.smithi078.stdout:done 2023-10-01T09:54:32.302 DEBUG:teuthology.orchestra.run.smithi173:> dpkg -s linux-image-generic-hwe-20.04 2023-10-01T09:54:32.359 INFO:teuthology.orchestra.run.smithi173.stdout:Package: linux-image-generic-hwe-20.04 2023-10-01T09:54:32.360 INFO:teuthology.orchestra.run.smithi173.stdout:Status: install ok installed 2023-10-01T09:54:32.360 INFO:teuthology.orchestra.run.smithi173.stdout:Priority: optional 2023-10-01T09:54:32.360 INFO:teuthology.orchestra.run.smithi173.stdout:Section: kernel 2023-10-01T09:54:32.360 INFO:teuthology.orchestra.run.smithi173.stdout:Installed-Size: 20 2023-10-01T09:54:32.360 INFO:teuthology.orchestra.run.smithi173.stdout:Maintainer: Ubuntu Kernel Team 2023-10-01T09:54:32.360 INFO:teuthology.orchestra.run.smithi173.stdout:Architecture: amd64 2023-10-01T09:54:32.361 INFO:teuthology.orchestra.run.smithi173.stdout:Source: linux-meta-hwe-5.15 2023-10-01T09:54:32.361 INFO:teuthology.orchestra.run.smithi173.stdout:Version: 5.15.0.84.93~20.04.42 2023-10-01T09:54:32.361 INFO:teuthology.orchestra.run.smithi173.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.1), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-84), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.1) 2023-10-01T09:54:32.361 INFO:teuthology.orchestra.run.smithi173.stdout:Depends: linux-image-5.15.0-84-generic, linux-modules-extra-5.15.0-84-generic, linux-firmware, intel-microcode, amd64-microcode 2023-10-01T09:54:32.361 INFO:teuthology.orchestra.run.smithi173.stdout:Recommends: thermald 2023-10-01T09:54:32.361 INFO:teuthology.orchestra.run.smithi173.stdout:Description: Generic Linux kernel image 2023-10-01T09:54:32.362 INFO:teuthology.orchestra.run.smithi173.stdout: This package will always depend on the latest generic kernel image 2023-10-01T09:54:32.362 INFO:teuthology.orchestra.run.smithi173.stdout: available. 2023-10-01T09:54:32.362 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.4.0-124-generic Expected: 5.15.0-84-generic 2023-10-01T09:54:32.362 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2023-10-01T09:54:32.362 DEBUG:teuthology.task.kernel:src is distro, skipping download 2023-10-01T09:54:32.363 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2023-10-01T09:54:32.363 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi173.front.sepia.ceph.com, path=None, version=distro) 2023-10-01T09:54:32.363 DEBUG:teuthology.orchestra.run.smithi173:> sudo apt-get clean 2023-10-01T09:54:32.442 DEBUG:teuthology.orchestra.run.smithi173:> sudo apt-get update 2023-10-01T09:54:32.605 INFO:teuthology.orchestra.run.smithi173.stdout:Hit:1 http://security.ubuntu.com/ubuntu focal-security InRelease 2023-10-01T09:54:32.677 DEBUG:teuthology.orchestra.run.smithi078:> dpkg -s linux-image-generic-hwe-20.04 2023-10-01T09:54:32.719 INFO:teuthology.orchestra.run.smithi078.stdout:Package: linux-image-generic-hwe-20.04 2023-10-01T09:54:32.720 INFO:teuthology.orchestra.run.smithi078.stdout:Status: install ok installed 2023-10-01T09:54:32.720 INFO:teuthology.orchestra.run.smithi078.stdout:Priority: optional 2023-10-01T09:54:32.720 INFO:teuthology.orchestra.run.smithi078.stdout:Section: kernel 2023-10-01T09:54:32.720 INFO:teuthology.orchestra.run.smithi078.stdout:Installed-Size: 20 2023-10-01T09:54:32.720 INFO:teuthology.orchestra.run.smithi078.stdout:Maintainer: Ubuntu Kernel Team 2023-10-01T09:54:32.720 INFO:teuthology.orchestra.run.smithi078.stdout:Architecture: amd64 2023-10-01T09:54:32.720 INFO:teuthology.orchestra.run.smithi078.stdout:Source: linux-meta-hwe-5.15 2023-10-01T09:54:32.720 INFO:teuthology.orchestra.run.smithi078.stdout:Version: 5.15.0.84.93~20.04.42 2023-10-01T09:54:32.720 INFO:teuthology.orchestra.run.smithi078.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.1), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-84), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.1) 2023-10-01T09:54:32.720 INFO:teuthology.orchestra.run.smithi078.stdout:Depends: linux-image-5.15.0-84-generic, linux-modules-extra-5.15.0-84-generic, linux-firmware, intel-microcode, amd64-microcode 2023-10-01T09:54:32.721 INFO:teuthology.orchestra.run.smithi078.stdout:Recommends: thermald 2023-10-01T09:54:32.721 INFO:teuthology.orchestra.run.smithi078.stdout:Description: Generic Linux kernel image 2023-10-01T09:54:32.721 INFO:teuthology.orchestra.run.smithi078.stdout: This package will always depend on the latest generic kernel image 2023-10-01T09:54:32.721 INFO:teuthology.orchestra.run.smithi078.stdout: available. 2023-10-01T09:54:32.721 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.4.0-124-generic Expected: 5.15.0-84-generic 2023-10-01T09:54:32.721 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2023-10-01T09:54:32.721 DEBUG:teuthology.task.kernel:src is distro, skipping download 2023-10-01T09:54:32.721 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2023-10-01T09:54:32.721 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi078.front.sepia.ceph.com, path=None, version=distro) 2023-10-01T09:54:32.721 DEBUG:teuthology.orchestra.run.smithi078:> sudo apt-get clean 2023-10-01T09:54:32.722 INFO:teuthology.orchestra.run.smithi173.stdout:Hit:2 http://archive.ubuntu.com/ubuntu focal InRelease 2023-10-01T09:54:32.802 INFO:teuthology.orchestra.run.smithi173.stdout:Hit:3 http://archive.ubuntu.com/ubuntu focal-updates InRelease 2023-10-01T09:54:32.824 DEBUG:teuthology.orchestra.run.smithi078:> sudo apt-get update 2023-10-01T09:54:32.884 INFO:teuthology.orchestra.run.smithi173.stdout:Hit:4 http://archive.ubuntu.com/ubuntu focal-backports InRelease 2023-10-01T09:54:33.101 INFO:teuthology.orchestra.run.smithi078.stdout:Hit:1 http://archive.ubuntu.com/ubuntu focal InRelease 2023-10-01T09:54:33.101 INFO:teuthology.orchestra.run.smithi078.stdout:Hit:2 http://security.ubuntu.com/ubuntu focal-security InRelease 2023-10-01T09:54:33.171 INFO:teuthology.orchestra.run.smithi078.stdout:Hit:3 http://archive.ubuntu.com/ubuntu focal-updates InRelease 2023-10-01T09:54:33.253 INFO:teuthology.orchestra.run.smithi078.stdout:Hit:4 http://archive.ubuntu.com/ubuntu focal-backports InRelease 2023-10-01T09:54:34.323 INFO:teuthology.orchestra.run.smithi173.stdout:Reading package lists... 2023-10-01T09:54:34.349 DEBUG:teuthology.orchestra.run.smithi173:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic-hwe-20.04 2023-10-01T09:54:34.417 INFO:teuthology.orchestra.run.smithi173.stdout:Reading package lists... 2023-10-01T09:54:34.617 INFO:teuthology.orchestra.run.smithi173.stdout:Building dependency tree... 2023-10-01T09:54:34.617 INFO:teuthology.orchestra.run.smithi173.stdout:Reading state information... 2023-10-01T09:54:34.839 INFO:teuthology.orchestra.run.smithi173.stdout:linux-image-generic-hwe-20.04 is already the newest version (5.15.0.84.93~20.04.42). 2023-10-01T09:54:34.840 INFO:teuthology.orchestra.run.smithi173.stdout:0 upgraded, 0 newly installed, 0 to remove and 266 not upgraded. 2023-10-01T09:54:34.841 DEBUG:teuthology.orchestra.run.smithi173:> dpkg -s linux-image-generic-hwe-20.04 2023-10-01T09:54:34.845 INFO:teuthology.orchestra.run.smithi078.stdout:Reading package lists... 2023-10-01T09:54:34.865 INFO:teuthology.orchestra.run.smithi173.stdout:Package: linux-image-generic-hwe-20.04 2023-10-01T09:54:34.865 INFO:teuthology.orchestra.run.smithi173.stdout:Status: install ok installed 2023-10-01T09:54:34.865 INFO:teuthology.orchestra.run.smithi173.stdout:Priority: optional 2023-10-01T09:54:34.866 INFO:teuthology.orchestra.run.smithi173.stdout:Section: kernel 2023-10-01T09:54:34.866 INFO:teuthology.orchestra.run.smithi173.stdout:Installed-Size: 20 2023-10-01T09:54:34.866 INFO:teuthology.orchestra.run.smithi173.stdout:Maintainer: Ubuntu Kernel Team 2023-10-01T09:54:34.866 INFO:teuthology.orchestra.run.smithi173.stdout:Architecture: amd64 2023-10-01T09:54:34.866 INFO:teuthology.orchestra.run.smithi173.stdout:Source: linux-meta-hwe-5.15 2023-10-01T09:54:34.866 INFO:teuthology.orchestra.run.smithi173.stdout:Version: 5.15.0.84.93~20.04.42 2023-10-01T09:54:34.867 INFO:teuthology.orchestra.run.smithi173.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.1), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-84), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.1) 2023-10-01T09:54:34.867 INFO:teuthology.orchestra.run.smithi173.stdout:Depends: linux-image-5.15.0-84-generic, linux-modules-extra-5.15.0-84-generic, linux-firmware, intel-microcode, amd64-microcode 2023-10-01T09:54:34.867 INFO:teuthology.orchestra.run.smithi173.stdout:Recommends: thermald 2023-10-01T09:54:34.867 INFO:teuthology.orchestra.run.smithi173.stdout:Description: Generic Linux kernel image 2023-10-01T09:54:34.867 INFO:teuthology.orchestra.run.smithi173.stdout: This package will always depend on the latest generic kernel image 2023-10-01T09:54:34.867 INFO:teuthology.orchestra.run.smithi173.stdout: available. 2023-10-01T09:54:34.868 DEBUG:teuthology.orchestra.run.smithi173:> mktemp 2023-10-01T09:54:34.869 DEBUG:teuthology.orchestra.run.smithi078:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic-hwe-20.04 2023-10-01T09:54:34.873 INFO:teuthology.orchestra.run.smithi173.stdout:/tmp/tmp.7ZyO8EiH35 2023-10-01T09:54:34.874 DEBUG:teuthology.orchestra.run.smithi173:> sudo cp /boot/grub/grub.cfg /tmp/tmp.7ZyO8EiH35 2023-10-01T09:54:34.929 DEBUG:teuthology.orchestra.run.smithi173:> sudo chmod 0666 /tmp/tmp.7ZyO8EiH35 2023-10-01T09:54:34.939 INFO:teuthology.orchestra.run.smithi078.stdout:Reading package lists... 2023-10-01T09:54:34.963 DEBUG:teuthology.orchestra.remote:smithi173:/tmp/tmp.7ZyO8EiH35 is 9KB 2023-10-01T09:54:34.977 DEBUG:teuthology.orchestra.run.smithi173:> rm -fr /tmp/tmp.7ZyO8EiH35 2023-10-01T09:54:34.983 DEBUG:teuthology.orchestra.run.smithi173:> sudo rm -f -- /etc/grub.d/01_ceph_kernel 2023-10-01T09:54:35.038 DEBUG:teuthology.orchestra.run.smithi173:> set -ex 2023-10-01T09:54:35.039 DEBUG:teuthology.orchestra.run.smithi173:> sudo dd of=/etc/grub.d/01_ceph_kernel 2023-10-01T09:54:35.039 DEBUG:teuthology.orchestra.run.smithi173:> sudo chmod 755 /etc/grub.d/01_ceph_kernel 2023-10-01T09:54:35.071 INFO:teuthology.task.kernel:Distro Kernel Version: 5.15.0-84-generic 2023-10-01T09:54:35.071 DEBUG:teuthology.orchestra.run.smithi173:> sudo update-grub 2023-10-01T09:54:35.139 INFO:teuthology.orchestra.run.smithi078.stdout:Building dependency tree... 2023-10-01T09:54:35.139 INFO:teuthology.orchestra.run.smithi078.stdout:Reading state information... 2023-10-01T09:54:35.348 INFO:teuthology.orchestra.run.smithi078.stdout:linux-image-generic-hwe-20.04 is already the newest version (5.15.0.84.93~20.04.42). 2023-10-01T09:54:35.348 INFO:teuthology.orchestra.run.smithi078.stdout:0 upgraded, 0 newly installed, 0 to remove and 266 not upgraded. 2023-10-01T09:54:35.350 DEBUG:teuthology.orchestra.run.smithi078:> dpkg -s linux-image-generic-hwe-20.04 2023-10-01T09:54:35.368 INFO:teuthology.orchestra.run.smithi078.stdout:Package: linux-image-generic-hwe-20.04 2023-10-01T09:54:35.369 INFO:teuthology.orchestra.run.smithi078.stdout:Status: install ok installed 2023-10-01T09:54:35.369 INFO:teuthology.orchestra.run.smithi078.stdout:Priority: optional 2023-10-01T09:54:35.369 INFO:teuthology.orchestra.run.smithi078.stdout:Section: kernel 2023-10-01T09:54:35.369 INFO:teuthology.orchestra.run.smithi078.stdout:Installed-Size: 20 2023-10-01T09:54:35.369 INFO:teuthology.orchestra.run.smithi078.stdout:Maintainer: Ubuntu Kernel Team 2023-10-01T09:54:35.369 INFO:teuthology.orchestra.run.smithi078.stdout:Architecture: amd64 2023-10-01T09:54:35.369 INFO:teuthology.orchestra.run.smithi078.stdout:Source: linux-meta-hwe-5.15 2023-10-01T09:54:35.369 INFO:teuthology.orchestra.run.smithi078.stdout:Version: 5.15.0.84.93~20.04.42 2023-10-01T09:54:35.369 INFO:teuthology.orchestra.run.smithi078.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.1), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-84), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.1) 2023-10-01T09:54:35.370 INFO:teuthology.orchestra.run.smithi078.stdout:Depends: linux-image-5.15.0-84-generic, linux-modules-extra-5.15.0-84-generic, linux-firmware, intel-microcode, amd64-microcode 2023-10-01T09:54:35.370 INFO:teuthology.orchestra.run.smithi078.stdout:Recommends: thermald 2023-10-01T09:54:35.370 INFO:teuthology.orchestra.run.smithi078.stdout:Description: Generic Linux kernel image 2023-10-01T09:54:35.370 INFO:teuthology.orchestra.run.smithi078.stdout: This package will always depend on the latest generic kernel image 2023-10-01T09:54:35.370 INFO:teuthology.orchestra.run.smithi078.stdout: available. 2023-10-01T09:54:35.370 DEBUG:teuthology.orchestra.run.smithi078:> mktemp 2023-10-01T09:54:35.415 INFO:teuthology.orchestra.run.smithi078.stdout:/tmp/tmp.xGCnjvra1k 2023-10-01T09:54:35.416 DEBUG:teuthology.orchestra.run.smithi078:> sudo cp /boot/grub/grub.cfg /tmp/tmp.xGCnjvra1k 2023-10-01T09:54:35.430 DEBUG:teuthology.orchestra.run.smithi078:> sudo chmod 0666 /tmp/tmp.xGCnjvra1k 2023-10-01T09:54:35.470 DEBUG:teuthology.orchestra.remote:smithi078:/tmp/tmp.xGCnjvra1k is 9KB 2023-10-01T09:54:35.484 DEBUG:teuthology.orchestra.run.smithi078:> rm -fr /tmp/tmp.xGCnjvra1k 2023-10-01T09:54:35.490 DEBUG:teuthology.orchestra.run.smithi078:> sudo rm -f -- /etc/grub.d/01_ceph_kernel 2023-10-01T09:54:35.548 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2023-10-01T09:54:35.548 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd of=/etc/grub.d/01_ceph_kernel 2023-10-01T09:54:35.548 DEBUG:teuthology.orchestra.run.smithi078:> sudo chmod 755 /etc/grub.d/01_ceph_kernel 2023-10-01T09:54:35.586 INFO:teuthology.task.kernel:Distro Kernel Version: 5.15.0-84-generic 2023-10-01T09:54:35.587 DEBUG:teuthology.orchestra.run.smithi078:> sudo update-grub 2023-10-01T09:54:36.493 INFO:teuthology.orchestra.run.smithi173.stderr:Sourcing file `/etc/default/grub' 2023-10-01T09:54:36.498 INFO:teuthology.orchestra.run.smithi173.stderr:Sourcing file `/etc/default/grub.d/init-select.cfg' 2023-10-01T09:54:36.514 INFO:teuthology.orchestra.run.smithi173.stderr:Generating grub configuration file ... 2023-10-01T09:54:36.775 INFO:teuthology.orchestra.run.smithi173.stderr:Found linux image: /boot/vmlinuz-5.15.0-84-generic 2023-10-01T09:54:36.786 INFO:teuthology.orchestra.run.smithi173.stderr:Found initrd image: /boot/initrd.img-5.15.0-84-generic 2023-10-01T09:54:36.993 INFO:teuthology.orchestra.run.smithi078.stderr:Sourcing file `/etc/default/grub' 2023-10-01T09:54:37.001 INFO:teuthology.orchestra.run.smithi078.stderr:Sourcing file `/etc/default/grub.d/init-select.cfg' 2023-10-01T09:54:37.013 INFO:teuthology.orchestra.run.smithi078.stderr:Generating grub configuration file ... 2023-10-01T09:54:37.138 INFO:teuthology.orchestra.run.smithi173.stderr:Found linux image: /boot/vmlinuz-5.4.0-124-generic 2023-10-01T09:54:37.150 INFO:teuthology.orchestra.run.smithi173.stderr:Found initrd image: /boot/initrd.img-5.4.0-124-generic 2023-10-01T09:54:37.313 INFO:teuthology.orchestra.run.smithi078.stderr:Found linux image: /boot/vmlinuz-5.15.0-84-generic 2023-10-01T09:54:37.325 INFO:teuthology.orchestra.run.smithi078.stderr:Found initrd image: /boot/initrd.img-5.15.0-84-generic 2023-10-01T09:54:37.707 INFO:teuthology.orchestra.run.smithi078.stderr:Found linux image: /boot/vmlinuz-5.4.0-124-generic 2023-10-01T09:54:37.718 INFO:teuthology.orchestra.run.smithi078.stderr:Found initrd image: /boot/initrd.img-5.4.0-124-generic 2023-10-01T09:54:38.195 INFO:teuthology.orchestra.run.smithi173.stderr:done 2023-10-01T09:54:38.197 DEBUG:teuthology.orchestra.run.smithi173:> sudo shutdown -r now 2023-10-01T09:54:38.680 INFO:teuthology.orchestra.run.smithi078.stderr:done 2023-10-01T09:54:38.681 DEBUG:teuthology.orchestra.run.smithi078:> sudo shutdown -r now 2023-10-01T09:55:08.227 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2023-10-01T09:55:08.228 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi173.front.sepia.ceph.com' 2023-10-01T09:55:08.228 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi173.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-01T09:55:08.684 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2023-10-01T09:55:08.685 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi078.front.sepia.ceph.com' 2023-10-01T09:55:08.685 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi078.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-01T09:55:26.714 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.173 2023-10-01T09:55:27.226 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.78 2023-10-01T09:55:35.720 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi173.front.sepia.ceph.com' 2023-10-01T09:55:35.720 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi173.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-01T09:55:36.227 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi078.front.sepia.ceph.com' 2023-10-01T09:55:36.228 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi078.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-01T09:55:38.782 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.173 2023-10-01T09:55:39.294 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.78 2023-10-01T09:55:50.794 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi173.front.sepia.ceph.com' 2023-10-01T09:55:50.795 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi173.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-01T09:55:51.296 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi078.front.sepia.ceph.com' 2023-10-01T09:55:51.296 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi078.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-01T09:56:09.726 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.78 2023-10-01T09:56:22.577 DEBUG:teuthology.orchestra.run.smithi173:> true 2023-10-01T09:56:23.081 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi173.front.sepia.ceph.com' 2023-10-01T09:56:23.081 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "5.15.0-84-generic"... 2023-10-01T09:56:23.082 DEBUG:teuthology.orchestra.run.smithi173:> uname -r 2023-10-01T09:56:23.130 INFO:teuthology.orchestra.run.smithi173.stdout:5.15.0-84-generic 2023-10-01T09:56:23.130 DEBUG:teuthology.task.kernel:current kernel version is 5.15.0-84-generic vs 5.15.0-84-generic 2023-10-01T09:56:23.130 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2023-10-01T09:56:23.130 DEBUG:teuthology.task.kernel:Distro of this test job: ubuntu 2023-10-01T09:56:24.131 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2023-10-01T09:56:24.132 DEBUG:teuthology.orchestra.run.smithi173:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-10-01T09:56:24.465 INFO:teuthology.orchestra.run.smithi173.stdout:ttyS1 2023-10-01T09:56:24.490 DEBUG:teuthology.parallel:result is None 2023-10-01T09:56:24.727 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi078.front.sepia.ceph.com' 2023-10-01T09:56:24.728 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi078.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-01T09:56:25.163 DEBUG:teuthology.orchestra.run.smithi078:> true 2023-10-01T09:56:25.804 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi078.front.sepia.ceph.com' 2023-10-01T09:56:25.804 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.15.0-84-generic"... 2023-10-01T09:56:25.804 DEBUG:teuthology.orchestra.run.smithi078:> uname -r 2023-10-01T09:56:25.854 INFO:teuthology.orchestra.run.smithi078.stdout:5.15.0-84-generic 2023-10-01T09:56:25.854 DEBUG:teuthology.task.kernel:current kernel version is 5.15.0-84-generic vs 5.15.0-84-generic 2023-10-01T09:56:25.854 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2023-10-01T09:56:25.855 DEBUG:teuthology.task.kernel:Distro of this test job: ubuntu 2023-10-01T09:56:26.855 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2023-10-01T09:56:26.855 DEBUG:teuthology.orchestra.run.smithi078:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-10-01T09:56:26.965 INFO:teuthology.orchestra.run.smithi078.stdout:ttyS1 2023-10-01T09:56:26.986 DEBUG:teuthology.parallel:result is None 2023-10-01T09:56:26.986 INFO:teuthology.run_tasks:Running task internal.base... 2023-10-01T09:56:26.994 INFO:teuthology.task.internal:Creating test directory... 2023-10-01T09:56:26.994 DEBUG:teuthology.orchestra.run.smithi078:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-10-01T09:56:26.996 DEBUG:teuthology.orchestra.run.smithi173:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-10-01T09:56:27.004 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2023-10-01T09:56:27.011 INFO:teuthology.run_tasks:Running task internal.archive... 2023-10-01T09:56:27.018 INFO:teuthology.task.internal:Creating archive directory... 2023-10-01T09:56:27.018 DEBUG:teuthology.orchestra.run.smithi078:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-10-01T09:56:27.045 DEBUG:teuthology.orchestra.run.smithi173:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-10-01T09:56:27.077 INFO:teuthology.run_tasks:Running task internal.coredump... 2023-10-01T09:56:27.083 INFO:teuthology.task.internal:Enabling coredump saving... 2023-10-01T09:56:27.084 DEBUG:teuthology.orchestra.run.smithi078:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-10-01T09:56:27.113 DEBUG:teuthology.orchestra.run.smithi173:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-10-01T09:56:27.129 INFO:teuthology.orchestra.run.smithi173.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-01T09:56:27.138 INFO:teuthology.orchestra.run.smithi173.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-01T09:56:27.157 INFO:teuthology.orchestra.run.smithi078.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-01T09:56:27.169 INFO:teuthology.orchestra.run.smithi078.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-10-01T09:56:27.170 INFO:teuthology.run_tasks:Running task internal.sudo... 2023-10-01T09:56:27.177 INFO:teuthology.task.internal:Configuring sudo... 2023-10-01T09:56:27.178 DEBUG:teuthology.orchestra.run.smithi078:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-10-01T09:56:27.214 DEBUG:teuthology.orchestra.run.smithi173:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-10-01T09:56:27.229 INFO:teuthology.run_tasks:Running task internal.syslog... 2023-10-01T09:56:27.238 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2023-10-01T09:56:27.238 DEBUG:teuthology.orchestra.run.smithi078:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-10-01T09:56:27.273 DEBUG:teuthology.orchestra.run.smithi173:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-10-01T09:56:27.281 DEBUG:teuthology.orchestra.run.smithi078:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-10-01T09:56:27.327 DEBUG:teuthology.orchestra.run.smithi078:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-10-01T09:56:27.375 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2023-10-01T09:56:27.375 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-10-01T09:56:27.445 DEBUG:teuthology.orchestra.run.smithi173:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-10-01T09:56:27.453 DEBUG:teuthology.orchestra.run.smithi173:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-10-01T09:56:27.502 DEBUG:teuthology.orchestra.run.smithi173:> set -ex 2023-10-01T09:56:27.503 DEBUG:teuthology.orchestra.run.smithi173:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-10-01T09:56:27.576 DEBUG:teuthology.orchestra.run.smithi078:> sudo service rsyslog restart 2023-10-01T09:56:27.578 DEBUG:teuthology.orchestra.run.smithi173:> sudo service rsyslog restart 2023-10-01T09:56:27.660 INFO:teuthology.run_tasks:Running task internal.timer... 2023-10-01T09:56:27.667 INFO:teuthology.task.internal:Starting timer... 2023-10-01T09:56:27.668 INFO:teuthology.run_tasks:Running task pcp... 2023-10-01T09:56:27.677 INFO:teuthology.run_tasks:Running task selinux... 2023-10-01T09:56:27.686 DEBUG:teuthology.task.selinux:Excluding smithi078: OS 'ubuntu' does not support SELinux 2023-10-01T09:56:27.686 DEBUG:teuthology.task.selinux:Excluding smithi173: OS 'ubuntu' does not support SELinux 2023-10-01T09:56:27.686 DEBUG:teuthology.task.selinux:Getting current SELinux state 2023-10-01T09:56:27.686 DEBUG:teuthology.task.selinux:Existing SELinux modes: {} 2023-10-01T09:56:27.687 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2023-10-01T09:56:27.687 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2023-10-01T09:56:27.695 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2023-10-01T09:56:27.701 INFO:teuthology.repo_utils:Fetching main from origin 2023-10-01T09:56:27.811 INFO:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2023-10-01T09:56:27.827 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2023-10-01T09:56:27.828 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi078.front.sepia.ceph.com,smithi173.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2023-10-01T10:02:04.883 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi078.front.sepia.ceph.com'), Remote(name='ubuntu@smithi173.front.sepia.ceph.com')] 2023-10-01T10:02:04.883 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi078.front.sepia.ceph.com' 2023-10-01T10:02:04.884 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi078.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-01T10:02:04.984 DEBUG:teuthology.orchestra.run.smithi078:> true 2023-10-01T10:02:05.103 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi078.front.sepia.ceph.com' 2023-10-01T10:02:05.104 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi173.front.sepia.ceph.com' 2023-10-01T10:02:05.104 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi173.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-10-01T10:02:05.184 DEBUG:teuthology.orchestra.run.smithi173:> true 2023-10-01T10:02:05.308 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi173.front.sepia.ceph.com' 2023-10-01T10:02:05.308 INFO:teuthology.run_tasks:Running task clock... 2023-10-01T10:02:05.318 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2023-10-01T10:02:05.318 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-10-01T10:02:05.319 DEBUG:teuthology.orchestra.run.smithi078:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-01T10:02:05.321 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-10-01T10:02:05.321 DEBUG:teuthology.orchestra.run.smithi173:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-01T10:02:05.350 INFO:teuthology.orchestra.run.smithi078.stdout: 1 Oct 10:02:05 ntpd[22508]: ntpd 4.2.8p12@1.3728-o (1): Starting 2023-10-01T10:02:05.350 INFO:teuthology.orchestra.run.smithi078.stdout: 1 Oct 10:02:05 ntpd[22508]: Command line: ntpd -gq 2023-10-01T10:02:05.351 INFO:teuthology.orchestra.run.smithi078.stdout: 1 Oct 10:02:05 ntpd[22508]: proto: precision = 0.094 usec (-23) 2023-10-01T10:02:05.351 INFO:teuthology.orchestra.run.smithi078.stderr:restrict 0.0.0.0: KOD does nothing without LIMITED. 2023-10-01T10:02:05.352 INFO:teuthology.orchestra.run.smithi078.stdout: 1 Oct 10:02:05 ntpd[22508]: restrict 0.0.0.0: KOD does nothing without LIMITED. 2023-10-01T10:02:05.352 INFO:teuthology.orchestra.run.smithi078.stderr:restrict ::: KOD does nothing without LIMITED. 2023-10-01T10:02:05.353 INFO:teuthology.orchestra.run.smithi078.stdout: 1 Oct 10:02:05 ntpd[22508]: restrict ::: KOD does nothing without LIMITED. 2023-10-01T10:02:05.353 INFO:teuthology.orchestra.run.smithi078.stdout: 1 Oct 10:02:05 ntpd[22508]: Listen and drop on 0 v6wildcard [::]:123 2023-10-01T10:02:05.353 INFO:teuthology.orchestra.run.smithi078.stdout: 1 Oct 10:02:05 ntpd[22508]: Listen and drop on 1 v4wildcard 0.0.0.0:123 2023-10-01T10:02:05.354 INFO:teuthology.orchestra.run.smithi078.stdout: 1 Oct 10:02:05 ntpd[22508]: Listen normally on 2 lo 127.0.0.1:123 2023-10-01T10:02:05.354 INFO:teuthology.orchestra.run.smithi078.stdout: 1 Oct 10:02:05 ntpd[22508]: Listen normally on 3 enp3s0f1 172.21.15.78:123 2023-10-01T10:02:05.354 INFO:teuthology.orchestra.run.smithi078.stdout: 1 Oct 10:02:05 ntpd[22508]: Listen normally on 4 lo [::1]:123 2023-10-01T10:02:05.354 INFO:teuthology.orchestra.run.smithi078.stdout: 1 Oct 10:02:05 ntpd[22508]: Listen normally on 5 enp3s0f1 [fe80::ec4:7aff:fe8f:d4dd%5]:123 2023-10-01T10:02:05.354 INFO:teuthology.orchestra.run.smithi078.stdout: 1 Oct 10:02:05 ntpd[22508]: Listening on routing socket on fd #22 for interface updates 2023-10-01T10:02:05.389 INFO:teuthology.orchestra.run.smithi173.stdout: 1 Oct 10:02:05 ntpd[22507]: ntpd 4.2.8p12@1.3728-o (1): Starting 2023-10-01T10:02:05.390 INFO:teuthology.orchestra.run.smithi173.stdout: 1 Oct 10:02:05 ntpd[22507]: Command line: ntpd -gq 2023-10-01T10:02:05.390 INFO:teuthology.orchestra.run.smithi173.stdout: 1 Oct 10:02:05 ntpd[22507]: proto: precision = 0.100 usec (-23) 2023-10-01T10:02:05.390 INFO:teuthology.orchestra.run.smithi173.stderr:restrict 0.0.0.0: KOD does nothing without LIMITED. 2023-10-01T10:02:05.391 INFO:teuthology.orchestra.run.smithi173.stdout: 1 Oct 10:02:05 ntpd[22507]: restrict 0.0.0.0: KOD does nothing without LIMITED. 2023-10-01T10:02:05.391 INFO:teuthology.orchestra.run.smithi173.stderr:restrict ::: KOD does nothing without LIMITED. 2023-10-01T10:02:05.392 INFO:teuthology.orchestra.run.smithi173.stdout: 1 Oct 10:02:05 ntpd[22507]: restrict ::: KOD does nothing without LIMITED. 2023-10-01T10:02:05.392 INFO:teuthology.orchestra.run.smithi173.stdout: 1 Oct 10:02:05 ntpd[22507]: Listen and drop on 0 v6wildcard [::]:123 2023-10-01T10:02:05.392 INFO:teuthology.orchestra.run.smithi173.stdout: 1 Oct 10:02:05 ntpd[22507]: Listen and drop on 1 v4wildcard 0.0.0.0:123 2023-10-01T10:02:05.392 INFO:teuthology.orchestra.run.smithi173.stdout: 1 Oct 10:02:05 ntpd[22507]: Listen normally on 2 lo 127.0.0.1:123 2023-10-01T10:02:05.392 INFO:teuthology.orchestra.run.smithi173.stdout: 1 Oct 10:02:05 ntpd[22507]: Listen normally on 3 enp3s0f1 172.21.15.173:123 2023-10-01T10:02:05.393 INFO:teuthology.orchestra.run.smithi173.stdout: 1 Oct 10:02:05 ntpd[22507]: Listen normally on 4 lo [::1]:123 2023-10-01T10:02:05.393 INFO:teuthology.orchestra.run.smithi173.stdout: 1 Oct 10:02:05 ntpd[22507]: Listen normally on 5 enp3s0f1 [fe80::ec4:7aff:fe88:7259%5]:123 2023-10-01T10:02:05.393 INFO:teuthology.orchestra.run.smithi173.stdout: 1 Oct 10:02:05 ntpd[22507]: Listening on routing socket on fd #22 for interface updates 2023-10-01T10:02:06.351 INFO:teuthology.orchestra.run.smithi078.stderr: 1 Oct 10:02:06 ntpd[22508]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-01T10:02:06.351 INFO:teuthology.orchestra.run.smithi078.stderr: 1 Oct 10:02:06 ntpd[22508]: can't open /var/log/ntpstats/rawstats.20231001: Permission denied 2023-10-01T10:02:06.351 INFO:teuthology.orchestra.run.smithi078.stderr: 1 Oct 10:02:06 ntpd[22508]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-01T10:02:06.351 INFO:teuthology.orchestra.run.smithi078.stderr: 1 Oct 10:02:06 ntpd[22508]: can't open /var/log/ntpstats/rawstats.20231001: Permission denied 2023-10-01T10:02:06.351 INFO:teuthology.orchestra.run.smithi078.stderr: 1 Oct 10:02:06 ntpd[22508]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-01T10:02:06.352 INFO:teuthology.orchestra.run.smithi078.stderr: 1 Oct 10:02:06 ntpd[22508]: can't open /var/log/ntpstats/peerstats.20231001: Permission denied 2023-10-01T10:02:06.390 INFO:teuthology.orchestra.run.smithi173.stderr: 1 Oct 10:02:06 ntpd[22507]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-01T10:02:06.390 INFO:teuthology.orchestra.run.smithi173.stderr: 1 Oct 10:02:06 ntpd[22507]: can't open /var/log/ntpstats/rawstats.20231001: Permission denied 2023-10-01T10:02:06.391 INFO:teuthology.orchestra.run.smithi173.stderr: 1 Oct 10:02:06 ntpd[22507]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-01T10:02:06.391 INFO:teuthology.orchestra.run.smithi173.stderr: 1 Oct 10:02:06 ntpd[22507]: can't open /var/log/ntpstats/rawstats.20231001: Permission denied 2023-10-01T10:02:06.391 INFO:teuthology.orchestra.run.smithi173.stderr: 1 Oct 10:02:06 ntpd[22507]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-01T10:02:06.391 INFO:teuthology.orchestra.run.smithi173.stderr: 1 Oct 10:02:06 ntpd[22507]: can't open /var/log/ntpstats/rawstats.20231001: Permission denied 2023-10-01T10:02:06.391 INFO:teuthology.orchestra.run.smithi173.stderr: 1 Oct 10:02:06 ntpd[22507]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-01T10:02:06.391 INFO:teuthology.orchestra.run.smithi173.stderr: 1 Oct 10:02:06 ntpd[22507]: can't open /var/log/ntpstats/peerstats.20231001: Permission denied 2023-10-01T10:02:07.351 INFO:teuthology.orchestra.run.smithi078.stderr: 1 Oct 10:02:07 ntpd[22508]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-01T10:02:07.351 INFO:teuthology.orchestra.run.smithi078.stderr: 1 Oct 10:02:07 ntpd[22508]: can't open /var/log/ntpstats/rawstats.20231001: Permission denied 2023-10-01T10:02:08.351 INFO:teuthology.orchestra.run.smithi078.stderr: 1 Oct 10:02:08 ntpd[22508]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-01T10:02:08.352 INFO:teuthology.orchestra.run.smithi078.stderr: 1 Oct 10:02:08 ntpd[22508]: can't open /var/log/ntpstats/rawstats.20231001: Permission denied 2023-10-01T10:02:08.352 INFO:teuthology.orchestra.run.smithi078.stderr: 1 Oct 10:02:08 ntpd[22508]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-01T10:02:08.352 INFO:teuthology.orchestra.run.smithi078.stderr: 1 Oct 10:02:08 ntpd[22508]: can't open /var/log/ntpstats/rawstats.20231001: Permission denied 2023-10-01T10:02:08.352 INFO:teuthology.orchestra.run.smithi078.stderr: 1 Oct 10:02:08 ntpd[22508]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-01T10:02:08.352 INFO:teuthology.orchestra.run.smithi078.stderr: 1 Oct 10:02:08 ntpd[22508]: can't open /var/log/ntpstats/rawstats.20231001: Permission denied 2023-10-01T10:02:08.352 INFO:teuthology.orchestra.run.smithi078.stderr: 1 Oct 10:02:08 ntpd[22508]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-01T10:02:08.352 INFO:teuthology.orchestra.run.smithi078.stderr: 1 Oct 10:02:08 ntpd[22508]: can't open /var/log/ntpstats/peerstats.20231001: Permission denied 2023-10-01T10:02:08.352 INFO:teuthology.orchestra.run.smithi078.stderr: 1 Oct 10:02:08 ntpd[22508]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-01T10:02:08.352 INFO:teuthology.orchestra.run.smithi078.stderr: 1 Oct 10:02:08 ntpd[22508]: can't open /var/log/ntpstats/rawstats.20231001: Permission denied 2023-10-01T10:02:08.352 INFO:teuthology.orchestra.run.smithi078.stderr: 1 Oct 10:02:08 ntpd[22508]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-01T10:02:08.352 INFO:teuthology.orchestra.run.smithi078.stderr: 1 Oct 10:02:08 ntpd[22508]: can't open /var/log/ntpstats/peerstats.20231001: Permission denied 2023-10-01T10:02:08.391 INFO:teuthology.orchestra.run.smithi173.stderr: 1 Oct 10:02:08 ntpd[22507]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-01T10:02:08.391 INFO:teuthology.orchestra.run.smithi173.stderr: 1 Oct 10:02:08 ntpd[22507]: can't open /var/log/ntpstats/rawstats.20231001: Permission denied 2023-10-01T10:02:08.391 INFO:teuthology.orchestra.run.smithi173.stderr: 1 Oct 10:02:08 ntpd[22507]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-01T10:02:08.391 INFO:teuthology.orchestra.run.smithi173.stderr: 1 Oct 10:02:08 ntpd[22507]: can't open /var/log/ntpstats/rawstats.20231001: Permission denied 2023-10-01T10:02:08.391 INFO:teuthology.orchestra.run.smithi173.stderr: 1 Oct 10:02:08 ntpd[22507]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-01T10:02:08.391 INFO:teuthology.orchestra.run.smithi173.stderr: 1 Oct 10:02:08 ntpd[22507]: can't open /var/log/ntpstats/rawstats.20231001: Permission denied 2023-10-01T10:02:08.392 INFO:teuthology.orchestra.run.smithi173.stderr: 1 Oct 10:02:08 ntpd[22507]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-01T10:02:08.392 INFO:teuthology.orchestra.run.smithi173.stderr: 1 Oct 10:02:08 ntpd[22507]: can't open /var/log/ntpstats/peerstats.20231001: Permission denied 2023-10-01T10:02:08.392 INFO:teuthology.orchestra.run.smithi173.stderr: 1 Oct 10:02:08 ntpd[22507]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-01T10:02:08.392 INFO:teuthology.orchestra.run.smithi173.stderr: 1 Oct 10:02:08 ntpd[22507]: can't open /var/log/ntpstats/rawstats.20231001: Permission denied 2023-10-01T10:02:08.392 INFO:teuthology.orchestra.run.smithi173.stderr: 1 Oct 10:02:08 ntpd[22507]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-01T10:02:08.392 INFO:teuthology.orchestra.run.smithi173.stderr: 1 Oct 10:02:08 ntpd[22507]: can't open /var/log/ntpstats/peerstats.20231001: Permission denied 2023-10-01T10:02:10.352 INFO:teuthology.orchestra.run.smithi078.stderr: 1 Oct 10:02:10 ntpd[22508]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-01T10:02:10.352 INFO:teuthology.orchestra.run.smithi078.stderr: 1 Oct 10:02:10 ntpd[22508]: can't open /var/log/ntpstats/rawstats.20231001: Permission denied 2023-10-01T10:02:10.353 INFO:teuthology.orchestra.run.smithi078.stderr: 1 Oct 10:02:10 ntpd[22508]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-01T10:02:10.353 INFO:teuthology.orchestra.run.smithi078.stderr: 1 Oct 10:02:10 ntpd[22508]: can't open /var/log/ntpstats/rawstats.20231001: Permission denied 2023-10-01T10:02:10.353 INFO:teuthology.orchestra.run.smithi078.stderr: 1 Oct 10:02:10 ntpd[22508]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-01T10:02:10.353 INFO:teuthology.orchestra.run.smithi078.stderr: 1 Oct 10:02:10 ntpd[22508]: can't open /var/log/ntpstats/rawstats.20231001: Permission denied 2023-10-01T10:02:10.353 INFO:teuthology.orchestra.run.smithi078.stderr: 1 Oct 10:02:10 ntpd[22508]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-01T10:02:10.353 INFO:teuthology.orchestra.run.smithi078.stderr: 1 Oct 10:02:10 ntpd[22508]: can't open /var/log/ntpstats/rawstats.20231001: Permission denied 2023-10-01T10:02:10.353 INFO:teuthology.orchestra.run.smithi078.stderr: 1 Oct 10:02:10 ntpd[22508]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-01T10:02:10.353 INFO:teuthology.orchestra.run.smithi078.stderr: 1 Oct 10:02:10 ntpd[22508]: can't open /var/log/ntpstats/peerstats.20231001: Permission denied 2023-10-01T10:02:10.353 INFO:teuthology.orchestra.run.smithi078.stderr: 1 Oct 10:02:10 ntpd[22508]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-01T10:02:10.353 INFO:teuthology.orchestra.run.smithi078.stderr: 1 Oct 10:02:10 ntpd[22508]: can't open /var/log/ntpstats/rawstats.20231001: Permission denied 2023-10-01T10:02:10.354 INFO:teuthology.orchestra.run.smithi078.stderr: 1 Oct 10:02:10 ntpd[22508]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-01T10:02:10.354 INFO:teuthology.orchestra.run.smithi078.stderr: 1 Oct 10:02:10 ntpd[22508]: can't open /var/log/ntpstats/peerstats.20231001: Permission denied 2023-10-01T10:02:10.354 INFO:teuthology.orchestra.run.smithi078.stderr: 1 Oct 10:02:10 ntpd[22508]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-01T10:02:10.354 INFO:teuthology.orchestra.run.smithi078.stderr: 1 Oct 10:02:10 ntpd[22508]: can't open /var/log/ntpstats/rawstats.20231001: Permission denied 2023-10-01T10:02:10.354 INFO:teuthology.orchestra.run.smithi078.stderr: 1 Oct 10:02:10 ntpd[22508]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-01T10:02:10.354 INFO:teuthology.orchestra.run.smithi078.stderr: 1 Oct 10:02:10 ntpd[22508]: can't open /var/log/ntpstats/peerstats.20231001: Permission denied 2023-10-01T10:02:10.392 INFO:teuthology.orchestra.run.smithi173.stderr: 1 Oct 10:02:10 ntpd[22507]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-01T10:02:10.392 INFO:teuthology.orchestra.run.smithi173.stderr: 1 Oct 10:02:10 ntpd[22507]: can't open /var/log/ntpstats/rawstats.20231001: Permission denied 2023-10-01T10:02:10.392 INFO:teuthology.orchestra.run.smithi173.stderr: 1 Oct 10:02:10 ntpd[22507]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-01T10:02:10.392 INFO:teuthology.orchestra.run.smithi173.stderr: 1 Oct 10:02:10 ntpd[22507]: can't open /var/log/ntpstats/rawstats.20231001: Permission denied 2023-10-01T10:02:10.393 INFO:teuthology.orchestra.run.smithi173.stderr: 1 Oct 10:02:10 ntpd[22507]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-01T10:02:10.393 INFO:teuthology.orchestra.run.smithi173.stderr: 1 Oct 10:02:10 ntpd[22507]: can't open /var/log/ntpstats/rawstats.20231001: Permission denied 2023-10-01T10:02:10.393 INFO:teuthology.orchestra.run.smithi173.stderr: 1 Oct 10:02:10 ntpd[22507]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-01T10:02:10.393 INFO:teuthology.orchestra.run.smithi173.stderr: 1 Oct 10:02:10 ntpd[22507]: can't open /var/log/ntpstats/rawstats.20231001: Permission denied 2023-10-01T10:02:10.393 INFO:teuthology.orchestra.run.smithi173.stderr: 1 Oct 10:02:10 ntpd[22507]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-01T10:02:10.393 INFO:teuthology.orchestra.run.smithi173.stderr: 1 Oct 10:02:10 ntpd[22507]: can't open /var/log/ntpstats/peerstats.20231001: Permission denied 2023-10-01T10:02:10.393 INFO:teuthology.orchestra.run.smithi173.stderr: 1 Oct 10:02:10 ntpd[22507]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-01T10:02:10.393 INFO:teuthology.orchestra.run.smithi173.stderr: 1 Oct 10:02:10 ntpd[22507]: can't open /var/log/ntpstats/rawstats.20231001: Permission denied 2023-10-01T10:02:10.394 INFO:teuthology.orchestra.run.smithi173.stderr: 1 Oct 10:02:10 ntpd[22507]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-01T10:02:10.394 INFO:teuthology.orchestra.run.smithi173.stderr: 1 Oct 10:02:10 ntpd[22507]: can't open /var/log/ntpstats/peerstats.20231001: Permission denied 2023-10-01T10:02:10.394 INFO:teuthology.orchestra.run.smithi173.stderr: 1 Oct 10:02:10 ntpd[22507]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-01T10:02:10.394 INFO:teuthology.orchestra.run.smithi173.stderr: 1 Oct 10:02:10 ntpd[22507]: can't open /var/log/ntpstats/rawstats.20231001: Permission denied 2023-10-01T10:02:10.394 INFO:teuthology.orchestra.run.smithi173.stderr: 1 Oct 10:02:10 ntpd[22507]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-01T10:02:10.394 INFO:teuthology.orchestra.run.smithi173.stderr: 1 Oct 10:02:10 ntpd[22507]: can't open /var/log/ntpstats/peerstats.20231001: Permission denied 2023-10-01T10:02:12.351 INFO:teuthology.orchestra.run.smithi078.stderr: 1 Oct 10:02:12 ntpd[22508]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-01T10:02:12.352 INFO:teuthology.orchestra.run.smithi078.stderr: 1 Oct 10:02:12 ntpd[22508]: can't open /var/log/ntpstats/rawstats.20231001: Permission denied 2023-10-01T10:02:12.352 INFO:teuthology.orchestra.run.smithi078.stderr: 1 Oct 10:02:12 ntpd[22508]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-01T10:02:12.352 INFO:teuthology.orchestra.run.smithi078.stderr: 1 Oct 10:02:12 ntpd[22508]: can't open /var/log/ntpstats/rawstats.20231001: Permission denied 2023-10-01T10:02:12.352 INFO:teuthology.orchestra.run.smithi078.stderr: 1 Oct 10:02:12 ntpd[22508]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-01T10:02:12.352 INFO:teuthology.orchestra.run.smithi078.stderr: 1 Oct 10:02:12 ntpd[22508]: can't open /var/log/ntpstats/rawstats.20231001: Permission denied 2023-10-01T10:02:12.352 INFO:teuthology.orchestra.run.smithi078.stderr: 1 Oct 10:02:12 ntpd[22508]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-01T10:02:12.352 INFO:teuthology.orchestra.run.smithi078.stderr: 1 Oct 10:02:12 ntpd[22508]: can't open /var/log/ntpstats/rawstats.20231001: Permission denied 2023-10-01T10:02:12.352 INFO:teuthology.orchestra.run.smithi078.stderr: 1 Oct 10:02:12 ntpd[22508]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-01T10:02:12.352 INFO:teuthology.orchestra.run.smithi078.stderr: 1 Oct 10:02:12 ntpd[22508]: can't open /var/log/ntpstats/peerstats.20231001: Permission denied 2023-10-01T10:02:12.353 INFO:teuthology.orchestra.run.smithi078.stdout: 1 Oct 10:02:12 ntpd[22508]: ntpd: time slew +0.001021 s 2023-10-01T10:02:12.353 INFO:teuthology.orchestra.run.smithi078.stdout:ntpd: time slew +0.001021s 2023-10-01T10:02:12.353 INFO:teuthology.orchestra.run.smithi078.stderr: 1 Oct 10:02:12 ntpd[22508]: couldn't unlink /var/log/ntpstats/loopstats: Permission denied 2023-10-01T10:02:12.353 INFO:teuthology.orchestra.run.smithi078.stderr: 1 Oct 10:02:12 ntpd[22508]: can't open /var/log/ntpstats/loopstats.20231001: Permission denied 2023-10-01T10:02:12.391 INFO:teuthology.orchestra.run.smithi173.stderr: 1 Oct 10:02:12 ntpd[22507]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-01T10:02:12.391 INFO:teuthology.orchestra.run.smithi173.stderr: 1 Oct 10:02:12 ntpd[22507]: can't open /var/log/ntpstats/rawstats.20231001: Permission denied 2023-10-01T10:02:12.391 INFO:teuthology.orchestra.run.smithi173.stderr: 1 Oct 10:02:12 ntpd[22507]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-01T10:02:12.391 INFO:teuthology.orchestra.run.smithi173.stderr: 1 Oct 10:02:12 ntpd[22507]: can't open /var/log/ntpstats/rawstats.20231001: Permission denied 2023-10-01T10:02:12.391 INFO:teuthology.orchestra.run.smithi173.stderr: 1 Oct 10:02:12 ntpd[22507]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-01T10:02:12.391 INFO:teuthology.orchestra.run.smithi173.stderr: 1 Oct 10:02:12 ntpd[22507]: can't open /var/log/ntpstats/rawstats.20231001: Permission denied 2023-10-01T10:02:12.391 INFO:teuthology.orchestra.run.smithi173.stderr: 1 Oct 10:02:12 ntpd[22507]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2023-10-01T10:02:12.391 INFO:teuthology.orchestra.run.smithi173.stderr: 1 Oct 10:02:12 ntpd[22507]: can't open /var/log/ntpstats/rawstats.20231001: Permission denied 2023-10-01T10:02:12.391 INFO:teuthology.orchestra.run.smithi173.stderr: 1 Oct 10:02:12 ntpd[22507]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2023-10-01T10:02:12.391 INFO:teuthology.orchestra.run.smithi173.stderr: 1 Oct 10:02:12 ntpd[22507]: can't open /var/log/ntpstats/peerstats.20231001: Permission denied 2023-10-01T10:02:12.392 INFO:teuthology.orchestra.run.smithi173.stdout: 1 Oct 10:02:12 ntpd[22507]: ntpd: time slew -0.000748 s 2023-10-01T10:02:12.392 INFO:teuthology.orchestra.run.smithi173.stdout:ntpd: time slew -0.000748s 2023-10-01T10:02:12.392 INFO:teuthology.orchestra.run.smithi173.stderr: 1 Oct 10:02:12 ntpd[22507]: couldn't unlink /var/log/ntpstats/loopstats: Permission denied 2023-10-01T10:02:12.392 INFO:teuthology.orchestra.run.smithi173.stderr: 1 Oct 10:02:12 ntpd[22507]: can't open /var/log/ntpstats/loopstats.20231001: Permission denied 2023-10-01T10:02:12.426 INFO:teuthology.orchestra.run.smithi078.stdout: remote refid st t when poll reach delay offset jitter 2023-10-01T10:02:12.426 INFO:teuthology.orchestra.run.smithi078.stdout:============================================================================== 2023-10-01T10:02:12.426 INFO:teuthology.orchestra.run.smithi078.stdout: hv01.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-10-01T10:02:12.426 INFO:teuthology.orchestra.run.smithi078.stdout: hv02.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-10-01T10:02:12.426 INFO:teuthology.orchestra.run.smithi078.stdout: hv03.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-10-01T10:02:12.426 INFO:teuthology.orchestra.run.smithi078.stdout: hv04.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-10-01T10:02:12.463 INFO:teuthology.orchestra.run.smithi173.stdout: remote refid st t when poll reach delay offset jitter 2023-10-01T10:02:12.463 INFO:teuthology.orchestra.run.smithi173.stdout:============================================================================== 2023-10-01T10:02:12.463 INFO:teuthology.orchestra.run.smithi173.stdout: hv01.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-10-01T10:02:12.463 INFO:teuthology.orchestra.run.smithi173.stdout: hv02.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-10-01T10:02:12.463 INFO:teuthology.orchestra.run.smithi173.stdout: hv03.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-10-01T10:02:12.463 INFO:teuthology.orchestra.run.smithi173.stdout: hv04.front.sepi .INIT. 16 u - 64 0 0.000 0.000 0.000 2023-10-01T10:02:12.464 INFO:teuthology.run_tasks:Running task install... 2023-10-01T10:02:12.472 DEBUG:teuthology.task.install:project ceph 2023-10-01T10:02:12.472 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'b56822191ad0416d0c05ed59720df03ad9c2a44e'}} 2023-10-01T10:02:12.472 DEBUG:teuthology.task.install:config {'branch': 'pacific', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': 'b56822191ad0416d0c05ed59720df03ad9c2a44e'} 2023-10-01T10:02:12.472 INFO:teuthology.task.install:Using flavor: default 2023-10-01T10:02:12.478 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2023-10-01T10:02:12.478 INFO:teuthology.task.install:extra packages: [] 2023-10-01T10:02:12.479 DEBUG:teuthology.orchestra.run.smithi078:> sudo apt-key list | grep Ceph 2023-10-01T10:02:12.479 DEBUG:teuthology.orchestra.run.smithi173:> sudo apt-key list | grep Ceph 2023-10-01T10:02:12.578 INFO:teuthology.orchestra.run.smithi078.stderr:Warning: apt-key output should not be parsed (stdout is not a terminal) 2023-10-01T10:02:12.609 INFO:teuthology.orchestra.run.smithi173.stderr:Warning: apt-key output should not be parsed (stdout is not a terminal) 2023-10-01T10:02:12.636 INFO:teuthology.orchestra.run.smithi078.stdout:uid [ unknown] Ceph automated package build (Ceph automated package build) 2023-10-01T10:02:12.636 INFO:teuthology.orchestra.run.smithi078.stdout:uid [ unknown] Ceph.com (release key) 2023-10-01T10:02:12.636 INFO:teuthology.task.install.deb:Installing packages: ceph, cephadm, ceph-mds, ceph-mgr, ceph-common, ceph-fuse, ceph-test, radosgw, python3-rados, python3-rgw, python3-cephfs, python3-rbd, libcephfs2, libcephfs-dev, librados2, librbd1, rbd-fuse on remote deb x86_64 2023-10-01T10:02:12.636 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-01T10:02:12.637 INFO:teuthology.packaging:ref: None 2023-10-01T10:02:12.637 INFO:teuthology.packaging:tag: None 2023-10-01T10:02:12.637 INFO:teuthology.packaging:branch: pacific 2023-10-01T10:02:12.637 INFO:teuthology.packaging:sha1: b56822191ad0416d0c05ed59720df03ad9c2a44e 2023-10-01T10:02:12.637 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&ref=pacific 2023-10-01T10:02:12.662 INFO:teuthology.orchestra.run.smithi173.stdout:uid [ unknown] Ceph automated package build (Ceph automated package build) 2023-10-01T10:02:12.662 INFO:teuthology.orchestra.run.smithi173.stdout:uid [ unknown] Ceph.com (release key) 2023-10-01T10:02:12.663 INFO:teuthology.task.install.deb:Installing packages: ceph, cephadm, ceph-mds, ceph-mgr, ceph-common, ceph-fuse, ceph-test, radosgw, python3-rados, python3-rgw, python3-cephfs, python3-rbd, libcephfs2, libcephfs-dev, librados2, librbd1, rbd-fuse on remote deb x86_64 2023-10-01T10:02:12.663 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-01T10:02:12.664 INFO:teuthology.packaging:ref: None 2023-10-01T10:02:12.664 INFO:teuthology.packaging:tag: None 2023-10-01T10:02:12.664 INFO:teuthology.packaging:branch: pacific 2023-10-01T10:02:12.664 INFO:teuthology.packaging:sha1: b56822191ad0416d0c05ed59720df03ad9c2a44e 2023-10-01T10:02:12.664 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&ref=pacific 2023-10-01T10:02:12.892 INFO:teuthology.task.install.deb:Pulling from https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default/ 2023-10-01T10:02:12.893 INFO:teuthology.task.install.deb:Package version is 16.2.14-44-geeb13ac9-1focal 2023-10-01T10:02:13.034 DEBUG:teuthology.orchestra.run.smithi173:> set -ex 2023-10-01T10:02:13.034 DEBUG:teuthology.orchestra.run.smithi173:> sudo dd of=/etc/apt/sources.list.d/ceph.list 2023-10-01T10:02:13.049 DEBUG:teuthology.orchestra.run.smithi173:> sudo apt-get update 2023-10-01T10:02:13.176 INFO:teuthology.task.install.deb:Pulling from https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default/ 2023-10-01T10:02:13.176 INFO:teuthology.task.install.deb:Package version is 16.2.14-44-geeb13ac9-1focal 2023-10-01T10:02:13.232 INFO:teuthology.orchestra.run.smithi173.stdout:Hit:1 http://archive.ubuntu.com/ubuntu focal InRelease 2023-10-01T10:02:13.239 INFO:teuthology.orchestra.run.smithi173.stdout:Hit:2 http://archive.ubuntu.com/ubuntu focal-updates InRelease 2023-10-01T10:02:13.256 INFO:teuthology.orchestra.run.smithi173.stdout:Hit:3 http://archive.ubuntu.com/ubuntu focal-backports InRelease 2023-10-01T10:02:13.288 INFO:teuthology.orchestra.run.smithi173.stdout:Ign:4 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal InRelease 2023-10-01T10:02:13.309 INFO:teuthology.orchestra.run.smithi173.stdout:Get:5 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal Release [7,680 B] 2023-10-01T10:02:13.316 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2023-10-01T10:02:13.316 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd of=/etc/apt/sources.list.d/ceph.list 2023-10-01T10:02:13.332 INFO:teuthology.orchestra.run.smithi173.stdout:Ign:6 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal Release.gpg 2023-10-01T10:02:13.333 DEBUG:teuthology.orchestra.run.smithi078:> sudo apt-get update 2023-10-01T10:02:13.352 INFO:teuthology.orchestra.run.smithi173.stdout:Hit:7 http://security.ubuntu.com/ubuntu focal-security InRelease 2023-10-01T10:02:13.353 INFO:teuthology.orchestra.run.smithi173.stdout:Get:8 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main i386 Packages [3,119 B] 2023-10-01T10:02:13.399 INFO:teuthology.orchestra.run.smithi173.stdout:Get:9 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 Packages [17.2 kB] 2023-10-01T10:02:13.519 INFO:teuthology.orchestra.run.smithi078.stdout:Hit:1 http://security.ubuntu.com/ubuntu focal-security InRelease 2023-10-01T10:02:13.635 INFO:teuthology.orchestra.run.smithi078.stdout:Hit:2 http://archive.ubuntu.com/ubuntu focal InRelease 2023-10-01T10:02:13.717 INFO:teuthology.orchestra.run.smithi078.stdout:Hit:3 http://archive.ubuntu.com/ubuntu focal-updates InRelease 2023-10-01T10:02:13.789 INFO:teuthology.orchestra.run.smithi078.stdout:Ign:4 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal InRelease 2023-10-01T10:02:13.798 INFO:teuthology.orchestra.run.smithi078.stdout:Hit:5 http://archive.ubuntu.com/ubuntu focal-backports InRelease 2023-10-01T10:02:13.860 INFO:teuthology.orchestra.run.smithi173.stdout:Fetched 28.0 kB in 1s (46.1 kB/s) 2023-10-01T10:02:13.860 INFO:teuthology.orchestra.run.smithi078.stdout:Get:6 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal Release [7,680 B] 2023-10-01T10:02:13.934 INFO:teuthology.orchestra.run.smithi078.stdout:Ign:7 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal Release.gpg 2023-10-01T10:02:14.006 INFO:teuthology.orchestra.run.smithi078.stdout:Get:8 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 Packages [17.2 kB] 2023-10-01T10:02:14.096 INFO:teuthology.orchestra.run.smithi078.stdout:Get:9 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main i386 Packages [3,119 B] 2023-10-01T10:02:14.194 INFO:teuthology.orchestra.run.smithi078.stdout:Fetched 28.0 kB in 1s (43.0 kB/s) 2023-10-01T10:02:14.906 INFO:teuthology.orchestra.run.smithi173.stdout:Reading package lists... 2023-10-01T10:02:14.935 DEBUG:teuthology.orchestra.run.smithi173:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=16.2.14-44-geeb13ac9-1focal cephadm=16.2.14-44-geeb13ac9-1focal ceph-mds=16.2.14-44-geeb13ac9-1focal ceph-mgr=16.2.14-44-geeb13ac9-1focal ceph-common=16.2.14-44-geeb13ac9-1focal ceph-fuse=16.2.14-44-geeb13ac9-1focal ceph-test=16.2.14-44-geeb13ac9-1focal radosgw=16.2.14-44-geeb13ac9-1focal python3-rados=16.2.14-44-geeb13ac9-1focal python3-rgw=16.2.14-44-geeb13ac9-1focal python3-cephfs=16.2.14-44-geeb13ac9-1focal python3-rbd=16.2.14-44-geeb13ac9-1focal libcephfs2=16.2.14-44-geeb13ac9-1focal libcephfs-dev=16.2.14-44-geeb13ac9-1focal librados2=16.2.14-44-geeb13ac9-1focal librbd1=16.2.14-44-geeb13ac9-1focal rbd-fuse=16.2.14-44-geeb13ac9-1focal 2023-10-01T10:02:15.009 INFO:teuthology.orchestra.run.smithi173.stdout:Reading package lists... 2023-10-01T10:02:15.218 INFO:teuthology.orchestra.run.smithi173.stdout:Building dependency tree... 2023-10-01T10:02:15.218 INFO:teuthology.orchestra.run.smithi173.stdout:Reading state information... 2023-10-01T10:02:15.230 INFO:teuthology.orchestra.run.smithi078.stdout:Reading package lists... 2023-10-01T10:02:15.259 DEBUG:teuthology.orchestra.run.smithi078:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=16.2.14-44-geeb13ac9-1focal cephadm=16.2.14-44-geeb13ac9-1focal ceph-mds=16.2.14-44-geeb13ac9-1focal ceph-mgr=16.2.14-44-geeb13ac9-1focal ceph-common=16.2.14-44-geeb13ac9-1focal ceph-fuse=16.2.14-44-geeb13ac9-1focal ceph-test=16.2.14-44-geeb13ac9-1focal radosgw=16.2.14-44-geeb13ac9-1focal python3-rados=16.2.14-44-geeb13ac9-1focal python3-rgw=16.2.14-44-geeb13ac9-1focal python3-cephfs=16.2.14-44-geeb13ac9-1focal python3-rbd=16.2.14-44-geeb13ac9-1focal libcephfs2=16.2.14-44-geeb13ac9-1focal libcephfs-dev=16.2.14-44-geeb13ac9-1focal librados2=16.2.14-44-geeb13ac9-1focal librbd1=16.2.14-44-geeb13ac9-1focal rbd-fuse=16.2.14-44-geeb13ac9-1focal 2023-10-01T10:02:15.334 INFO:teuthology.orchestra.run.smithi078.stdout:Reading package lists... 2023-10-01T10:02:15.394 INFO:teuthology.orchestra.run.smithi173.stdout:The following packages were automatically installed and are no longer required: 2023-10-01T10:02:15.395 INFO:teuthology.orchestra.run.smithi173.stdout: libboost-iostreams1.71.0 libboost-thread1.71.0 2023-10-01T10:02:15.395 INFO:teuthology.orchestra.run.smithi173.stdout:Use 'sudo apt autoremove' to remove them. 2023-10-01T10:02:15.395 INFO:teuthology.orchestra.run.smithi173.stdout:The following additional packages will be installed: 2023-10-01T10:02:15.395 INFO:teuthology.orchestra.run.smithi173.stdout: ceph-base ceph-mgr-cephadm ceph-mgr-dashboard ceph-mgr-diskprediction-local 2023-10-01T10:02:15.396 INFO:teuthology.orchestra.run.smithi173.stdout: ceph-mgr-k8sevents ceph-mgr-modules-core ceph-mon ceph-osd jq libjaeger 2023-10-01T10:02:15.396 INFO:teuthology.orchestra.run.smithi173.stdout: libjq1 libjs-sphinxdoc libjs-underscore liblbfgsb0 libleveldb1d 2023-10-01T10:02:15.396 INFO:teuthology.orchestra.run.smithi173.stdout: liblttng-ust0 liblua5.3-0 liboath0 libonig5 librabbitmq4 libradosstriper1 2023-10-01T10:02:15.396 INFO:teuthology.orchestra.run.smithi173.stdout: librdkafka1 librgw2 libsqlite3-mod-ceph nvme-cli python-pastedeploy-tpl 2023-10-01T10:02:15.396 INFO:teuthology.orchestra.run.smithi173.stdout: python3-atomicwrites python3-attr python3-bcrypt python3-blinker python3-bs4 2023-10-01T10:02:15.396 INFO:teuthology.orchestra.run.smithi173.stdout: python3-cachetools python3-ceph-argparse python3-ceph-common 2023-10-01T10:02:15.396 INFO:teuthology.orchestra.run.smithi173.stdout: python3-cherrypy3 python3-cryptography python3-decorator python3-google-auth 2023-10-01T10:02:15.396 INFO:teuthology.orchestra.run.smithi173.stdout: python3-html5lib python3-jinja2 python3-joblib python3-jwt 2023-10-01T10:02:15.397 INFO:teuthology.orchestra.run.smithi173.stdout: python3-kubernetes python3-logutils python3-lxml python3-mako 2023-10-01T10:02:15.397 INFO:teuthology.orchestra.run.smithi173.stdout: python3-markupsafe python3-oauthlib python3-openssl python3-packaging 2023-10-01T10:02:15.397 INFO:teuthology.orchestra.run.smithi173.stdout: python3-paste python3-pastedeploy python3-pastescript python3-pecan 2023-10-01T10:02:15.397 INFO:teuthology.orchestra.run.smithi173.stdout: python3-pluggy python3-prettytable python3-psutil python3-py python3-pyasn1 2023-10-01T10:02:15.397 INFO:teuthology.orchestra.run.smithi173.stdout: python3-pyasn1-modules python3-pyinotify python3-pytest python3-repoze.lru 2023-10-01T10:02:15.397 INFO:teuthology.orchestra.run.smithi173.stdout: python3-requests-oauthlib python3-routes python3-rsa python3-scipy 2023-10-01T10:02:15.397 INFO:teuthology.orchestra.run.smithi173.stdout: python3-simplegeneric python3-simplejson python3-singledispatch 2023-10-01T10:02:15.397 INFO:teuthology.orchestra.run.smithi173.stdout: python3-sklearn python3-sklearn-lib python3-soupsieve python3-tempita 2023-10-01T10:02:15.397 INFO:teuthology.orchestra.run.smithi173.stdout: python3-waitress python3-wcwidth python3-webencodings python3-webob 2023-10-01T10:02:15.397 INFO:teuthology.orchestra.run.smithi173.stdout: python3-websocket python3-webtest python3-werkzeug socat xmlstarlet 2023-10-01T10:02:15.398 INFO:teuthology.orchestra.run.smithi173.stdout:Suggested packages: 2023-10-01T10:02:15.398 INFO:teuthology.orchestra.run.smithi173.stdout: python3-influxdb python-attr-doc python-blinker-doc python-cryptography-doc 2023-10-01T10:02:15.398 INFO:teuthology.orchestra.run.smithi173.stdout: python3-cryptography-vectors python3-genshi python-jinja2-doc python3-crypto 2023-10-01T10:02:15.398 INFO:teuthology.orchestra.run.smithi173.stdout: python3-lxml-dbg python-lxml-doc python3-beaker python-mako-doc 2023-10-01T10:02:15.398 INFO:teuthology.orchestra.run.smithi173.stdout: python-openssl-doc python3-openssl-dbg httpd-wsgi libapache2-mod-python 2023-10-01T10:02:15.398 INFO:teuthology.orchestra.run.smithi173.stdout: libapache2-mod-scgi libjs-mochikit python-pecan-doc python-psutil-doc 2023-10-01T10:02:15.398 INFO:teuthology.orchestra.run.smithi173.stdout: subversion python-pyinotify-doc python-scipy-doc python3-dap 2023-10-01T10:02:15.398 INFO:teuthology.orchestra.run.smithi173.stdout: python-sklearn-doc ipython3 python-waitress-doc python-webob-doc 2023-10-01T10:02:15.398 INFO:teuthology.orchestra.run.smithi173.stdout: python-webtest-doc python-werkzeug-doc python3-termcolor python3-watchdog 2023-10-01T10:02:15.398 INFO:teuthology.orchestra.run.smithi173.stdout:Recommended packages: 2023-10-01T10:02:15.399 INFO:teuthology.orchestra.run.smithi173.stdout: btrfs-tools 2023-10-01T10:02:15.510 INFO:teuthology.orchestra.run.smithi078.stdout:Building dependency tree... 2023-10-01T10:02:15.510 INFO:teuthology.orchestra.run.smithi078.stdout:Reading state information... 2023-10-01T10:02:15.516 INFO:teuthology.orchestra.run.smithi173.stdout:The following NEW packages will be installed: 2023-10-01T10:02:15.516 INFO:teuthology.orchestra.run.smithi173.stdout: ceph ceph-base ceph-common ceph-fuse ceph-mds ceph-mgr ceph-mgr-cephadm 2023-10-01T10:02:15.516 INFO:teuthology.orchestra.run.smithi173.stdout: ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-k8sevents 2023-10-01T10:02:15.516 INFO:teuthology.orchestra.run.smithi173.stdout: ceph-mgr-modules-core ceph-mon ceph-osd ceph-test cephadm jq libcephfs-dev 2023-10-01T10:02:15.517 INFO:teuthology.orchestra.run.smithi173.stdout: libcephfs2 libjaeger libjq1 libjs-sphinxdoc libjs-underscore liblbfgsb0 2023-10-01T10:02:15.517 INFO:teuthology.orchestra.run.smithi173.stdout: libleveldb1d liblttng-ust0 liblua5.3-0 liboath0 libonig5 librabbitmq4 2023-10-01T10:02:15.517 INFO:teuthology.orchestra.run.smithi173.stdout: libradosstriper1 librdkafka1 librgw2 libsqlite3-mod-ceph nvme-cli 2023-10-01T10:02:15.517 INFO:teuthology.orchestra.run.smithi173.stdout: python-pastedeploy-tpl python3-atomicwrites python3-attr python3-bcrypt 2023-10-01T10:02:15.517 INFO:teuthology.orchestra.run.smithi173.stdout: python3-blinker python3-bs4 python3-cachetools python3-ceph-argparse 2023-10-01T10:02:15.517 INFO:teuthology.orchestra.run.smithi173.stdout: python3-ceph-common python3-cephfs python3-cherrypy3 python3-cryptography 2023-10-01T10:02:15.518 INFO:teuthology.orchestra.run.smithi173.stdout: python3-decorator python3-google-auth python3-html5lib python3-jinja2 2023-10-01T10:02:15.518 INFO:teuthology.orchestra.run.smithi173.stdout: python3-joblib python3-jwt python3-kubernetes python3-logutils python3-lxml 2023-10-01T10:02:15.518 INFO:teuthology.orchestra.run.smithi173.stdout: python3-mako python3-markupsafe python3-oauthlib python3-openssl 2023-10-01T10:02:15.518 INFO:teuthology.orchestra.run.smithi173.stdout: python3-packaging python3-paste python3-pastedeploy python3-pastescript 2023-10-01T10:02:15.518 INFO:teuthology.orchestra.run.smithi173.stdout: python3-pecan python3-pluggy python3-prettytable python3-psutil python3-py 2023-10-01T10:02:15.518 INFO:teuthology.orchestra.run.smithi173.stdout: python3-pyasn1 python3-pyasn1-modules python3-pyinotify python3-pytest 2023-10-01T10:02:15.518 INFO:teuthology.orchestra.run.smithi173.stdout: python3-rados python3-rbd python3-repoze.lru python3-requests-oauthlib 2023-10-01T10:02:15.518 INFO:teuthology.orchestra.run.smithi173.stdout: python3-rgw python3-routes python3-rsa python3-scipy python3-simplegeneric 2023-10-01T10:02:15.518 INFO:teuthology.orchestra.run.smithi173.stdout: python3-simplejson python3-singledispatch python3-sklearn 2023-10-01T10:02:15.518 INFO:teuthology.orchestra.run.smithi173.stdout: python3-sklearn-lib python3-soupsieve python3-tempita python3-waitress 2023-10-01T10:02:15.518 INFO:teuthology.orchestra.run.smithi173.stdout: python3-wcwidth python3-webencodings python3-webob python3-websocket 2023-10-01T10:02:15.518 INFO:teuthology.orchestra.run.smithi173.stdout: python3-webtest python3-werkzeug radosgw rbd-fuse socat xmlstarlet 2023-10-01T10:02:15.519 INFO:teuthology.orchestra.run.smithi173.stdout:The following packages will be upgraded: 2023-10-01T10:02:15.519 INFO:teuthology.orchestra.run.smithi173.stdout: librados2 librbd1 2023-10-01T10:02:15.647 INFO:teuthology.orchestra.run.smithi173.stdout:2 upgraded, 98 newly installed, 0 to remove and 248 not upgraded. 2023-10-01T10:02:15.648 INFO:teuthology.orchestra.run.smithi173.stdout:Need to get 174 MB of archives. 2023-10-01T10:02:15.648 INFO:teuthology.orchestra.run.smithi173.stdout:After this operation, 804 MB of additional disk space will be used. 2023-10-01T10:02:15.648 INFO:teuthology.orchestra.run.smithi173.stdout:Get:1 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 libjaeger amd64 16.2.14-44-geeb13ac9-1focal [3,956 B] 2023-10-01T10:02:15.669 INFO:teuthology.orchestra.run.smithi173.stdout:Get:2 http://archive.ubuntu.com/ubuntu focal/main amd64 liblttng-ust0 amd64 2.11.0-1 [160 kB] 2023-10-01T10:02:15.676 INFO:teuthology.orchestra.run.smithi078.stdout:The following packages were automatically installed and are no longer required: 2023-10-01T10:02:15.677 INFO:teuthology.orchestra.run.smithi078.stdout: libboost-iostreams1.71.0 libboost-thread1.71.0 2023-10-01T10:02:15.677 INFO:teuthology.orchestra.run.smithi078.stdout:Use 'sudo apt autoremove' to remove them. 2023-10-01T10:02:15.677 INFO:teuthology.orchestra.run.smithi078.stdout:The following additional packages will be installed: 2023-10-01T10:02:15.677 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-base ceph-mgr-cephadm ceph-mgr-dashboard ceph-mgr-diskprediction-local 2023-10-01T10:02:15.678 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-mgr-k8sevents ceph-mgr-modules-core ceph-mon ceph-osd jq libjaeger 2023-10-01T10:02:15.678 INFO:teuthology.orchestra.run.smithi078.stdout: libjq1 libjs-sphinxdoc libjs-underscore liblbfgsb0 libleveldb1d 2023-10-01T10:02:15.678 INFO:teuthology.orchestra.run.smithi078.stdout: liblttng-ust0 liblua5.3-0 liboath0 libonig5 librabbitmq4 libradosstriper1 2023-10-01T10:02:15.678 INFO:teuthology.orchestra.run.smithi078.stdout: librdkafka1 librgw2 libsqlite3-mod-ceph nvme-cli python-pastedeploy-tpl 2023-10-01T10:02:15.679 INFO:teuthology.orchestra.run.smithi078.stdout: python3-atomicwrites python3-attr python3-bcrypt python3-blinker python3-bs4 2023-10-01T10:02:15.679 INFO:teuthology.orchestra.run.smithi078.stdout: python3-cachetools python3-ceph-argparse python3-ceph-common 2023-10-01T10:02:15.679 INFO:teuthology.orchestra.run.smithi078.stdout: python3-cherrypy3 python3-cryptography python3-decorator python3-google-auth 2023-10-01T10:02:15.679 INFO:teuthology.orchestra.run.smithi078.stdout: python3-html5lib python3-jinja2 python3-joblib python3-jwt 2023-10-01T10:02:15.679 INFO:teuthology.orchestra.run.smithi078.stdout: python3-kubernetes python3-logutils python3-lxml python3-mako 2023-10-01T10:02:15.679 INFO:teuthology.orchestra.run.smithi078.stdout: python3-markupsafe python3-oauthlib python3-openssl python3-packaging 2023-10-01T10:02:15.679 INFO:teuthology.orchestra.run.smithi078.stdout: python3-paste python3-pastedeploy python3-pastescript python3-pecan 2023-10-01T10:02:15.679 INFO:teuthology.orchestra.run.smithi078.stdout: python3-pluggy python3-prettytable python3-psutil python3-py python3-pyasn1 2023-10-01T10:02:15.679 INFO:teuthology.orchestra.run.smithi078.stdout: python3-pyasn1-modules python3-pyinotify python3-pytest python3-repoze.lru 2023-10-01T10:02:15.679 INFO:teuthology.orchestra.run.smithi078.stdout: python3-requests-oauthlib python3-routes python3-rsa python3-scipy 2023-10-01T10:02:15.679 INFO:teuthology.orchestra.run.smithi078.stdout: python3-simplegeneric python3-simplejson python3-singledispatch 2023-10-01T10:02:15.680 INFO:teuthology.orchestra.run.smithi078.stdout: python3-sklearn python3-sklearn-lib python3-soupsieve python3-tempita 2023-10-01T10:02:15.680 INFO:teuthology.orchestra.run.smithi078.stdout: python3-waitress python3-wcwidth python3-webencodings python3-webob 2023-10-01T10:02:15.680 INFO:teuthology.orchestra.run.smithi078.stdout: python3-websocket python3-webtest python3-werkzeug socat xmlstarlet 2023-10-01T10:02:15.680 INFO:teuthology.orchestra.run.smithi078.stdout:Suggested packages: 2023-10-01T10:02:15.680 INFO:teuthology.orchestra.run.smithi078.stdout: python3-influxdb python-attr-doc python-blinker-doc python-cryptography-doc 2023-10-01T10:02:15.680 INFO:teuthology.orchestra.run.smithi078.stdout: python3-cryptography-vectors python3-genshi python-jinja2-doc python3-crypto 2023-10-01T10:02:15.680 INFO:teuthology.orchestra.run.smithi078.stdout: python3-lxml-dbg python-lxml-doc python3-beaker python-mako-doc 2023-10-01T10:02:15.680 INFO:teuthology.orchestra.run.smithi078.stdout: python-openssl-doc python3-openssl-dbg httpd-wsgi libapache2-mod-python 2023-10-01T10:02:15.680 INFO:teuthology.orchestra.run.smithi078.stdout: libapache2-mod-scgi libjs-mochikit python-pecan-doc python-psutil-doc 2023-10-01T10:02:15.681 INFO:teuthology.orchestra.run.smithi078.stdout: subversion python-pyinotify-doc python-scipy-doc python3-dap 2023-10-01T10:02:15.681 INFO:teuthology.orchestra.run.smithi078.stdout: python-sklearn-doc ipython3 python-waitress-doc python-webob-doc 2023-10-01T10:02:15.681 INFO:teuthology.orchestra.run.smithi078.stdout: python-webtest-doc python-werkzeug-doc python3-termcolor python3-watchdog 2023-10-01T10:02:15.681 INFO:teuthology.orchestra.run.smithi078.stdout:Recommended packages: 2023-10-01T10:02:15.681 INFO:teuthology.orchestra.run.smithi078.stdout: btrfs-tools 2023-10-01T10:02:15.691 INFO:teuthology.orchestra.run.smithi173.stdout:Get:3 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 librbd1 amd64 16.2.14-44-geeb13ac9-1focal [3,384 kB] 2023-10-01T10:02:15.787 INFO:teuthology.orchestra.run.smithi078.stdout:The following NEW packages will be installed: 2023-10-01T10:02:15.788 INFO:teuthology.orchestra.run.smithi078.stdout: ceph ceph-base ceph-common ceph-fuse ceph-mds ceph-mgr ceph-mgr-cephadm 2023-10-01T10:02:15.788 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-k8sevents 2023-10-01T10:02:15.788 INFO:teuthology.orchestra.run.smithi078.stdout: ceph-mgr-modules-core ceph-mon ceph-osd ceph-test cephadm jq libcephfs-dev 2023-10-01T10:02:15.788 INFO:teuthology.orchestra.run.smithi078.stdout: libcephfs2 libjaeger libjq1 libjs-sphinxdoc libjs-underscore liblbfgsb0 2023-10-01T10:02:15.788 INFO:teuthology.orchestra.run.smithi078.stdout: libleveldb1d liblttng-ust0 liblua5.3-0 liboath0 libonig5 librabbitmq4 2023-10-01T10:02:15.789 INFO:teuthology.orchestra.run.smithi078.stdout: libradosstriper1 librdkafka1 librgw2 libsqlite3-mod-ceph nvme-cli 2023-10-01T10:02:15.789 INFO:teuthology.orchestra.run.smithi078.stdout: python-pastedeploy-tpl python3-atomicwrites python3-attr python3-bcrypt 2023-10-01T10:02:15.789 INFO:teuthology.orchestra.run.smithi078.stdout: python3-blinker python3-bs4 python3-cachetools python3-ceph-argparse 2023-10-01T10:02:15.789 INFO:teuthology.orchestra.run.smithi078.stdout: python3-ceph-common python3-cephfs python3-cherrypy3 python3-cryptography 2023-10-01T10:02:15.789 INFO:teuthology.orchestra.run.smithi078.stdout: python3-decorator python3-google-auth python3-html5lib python3-jinja2 2023-10-01T10:02:15.789 INFO:teuthology.orchestra.run.smithi078.stdout: python3-joblib python3-jwt python3-kubernetes python3-logutils python3-lxml 2023-10-01T10:02:15.789 INFO:teuthology.orchestra.run.smithi078.stdout: python3-mako python3-markupsafe python3-oauthlib python3-openssl 2023-10-01T10:02:15.789 INFO:teuthology.orchestra.run.smithi078.stdout: python3-packaging python3-paste python3-pastedeploy python3-pastescript 2023-10-01T10:02:15.789 INFO:teuthology.orchestra.run.smithi078.stdout: python3-pecan python3-pluggy python3-prettytable python3-psutil python3-py 2023-10-01T10:02:15.789 INFO:teuthology.orchestra.run.smithi078.stdout: python3-pyasn1 python3-pyasn1-modules python3-pyinotify python3-pytest 2023-10-01T10:02:15.789 INFO:teuthology.orchestra.run.smithi078.stdout: python3-rados python3-rbd python3-repoze.lru python3-requests-oauthlib 2023-10-01T10:02:15.790 INFO:teuthology.orchestra.run.smithi078.stdout: python3-rgw python3-routes python3-rsa python3-scipy python3-simplegeneric 2023-10-01T10:02:15.790 INFO:teuthology.orchestra.run.smithi078.stdout: python3-simplejson python3-singledispatch python3-sklearn 2023-10-01T10:02:15.790 INFO:teuthology.orchestra.run.smithi078.stdout: python3-sklearn-lib python3-soupsieve python3-tempita python3-waitress 2023-10-01T10:02:15.790 INFO:teuthology.orchestra.run.smithi078.stdout: python3-wcwidth python3-webencodings python3-webob python3-websocket 2023-10-01T10:02:15.790 INFO:teuthology.orchestra.run.smithi078.stdout: python3-webtest python3-werkzeug radosgw rbd-fuse socat xmlstarlet 2023-10-01T10:02:15.790 INFO:teuthology.orchestra.run.smithi078.stdout:The following packages will be upgraded: 2023-10-01T10:02:15.790 INFO:teuthology.orchestra.run.smithi078.stdout: librados2 librbd1 2023-10-01T10:02:15.918 INFO:teuthology.orchestra.run.smithi078.stdout:2 upgraded, 98 newly installed, 0 to remove and 248 not upgraded. 2023-10-01T10:02:15.918 INFO:teuthology.orchestra.run.smithi078.stdout:Need to get 174 MB of archives. 2023-10-01T10:02:15.918 INFO:teuthology.orchestra.run.smithi078.stdout:After this operation, 804 MB of additional disk space will be used. 2023-10-01T10:02:15.918 INFO:teuthology.orchestra.run.smithi078.stdout:Get:1 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 libjaeger amd64 16.2.14-44-geeb13ac9-1focal [3,956 B] 2023-10-01T10:02:15.941 INFO:teuthology.orchestra.run.smithi173.stdout:Get:4 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 librados2 amd64 16.2.14-44-geeb13ac9-1focal [3,390 kB] 2023-10-01T10:02:15.961 INFO:teuthology.orchestra.run.smithi078.stdout:Get:2 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 librbd1 amd64 16.2.14-44-geeb13ac9-1focal [3,384 kB] 2023-10-01T10:02:15.962 INFO:teuthology.orchestra.run.smithi173.stdout:Get:5 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-prettytable all 0.7.2-5 [20.1 kB] 2023-10-01T10:02:15.968 INFO:teuthology.orchestra.run.smithi173.stdout:Get:6 http://archive.ubuntu.com/ubuntu focal/main amd64 liblua5.3-0 amd64 5.3.3-1.1ubuntu2 [116 kB] 2023-10-01T10:02:15.985 INFO:teuthology.orchestra.run.smithi078.stdout:Get:3 http://archive.ubuntu.com/ubuntu focal/main amd64 liblttng-ust0 amd64 2.11.0-1 [160 kB] 2023-10-01T10:02:16.001 INFO:teuthology.orchestra.run.smithi173.stdout:Get:7 http://archive.ubuntu.com/ubuntu focal/main amd64 librabbitmq4 amd64 0.10.0-1 [35.8 kB] 2023-10-01T10:02:16.012 INFO:teuthology.orchestra.run.smithi173.stdout:Get:8 http://archive.ubuntu.com/ubuntu focal/universe amd64 librdkafka1 amd64 1.2.1-1ubuntu1 [410 kB] 2023-10-01T10:02:16.087 INFO:teuthology.orchestra.run.smithi173.stdout:Get:9 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 libcephfs2 amd64 16.2.14-44-geeb13ac9-1focal [729 kB] 2023-10-01T10:02:16.102 INFO:teuthology.orchestra.run.smithi173.stdout:Get:10 http://archive.ubuntu.com/ubuntu focal/main amd64 libleveldb1d amd64 1.22-3ubuntu2 [139 kB] 2023-10-01T10:02:16.117 INFO:teuthology.orchestra.run.smithi173.stdout:Get:11 http://archive.ubuntu.com/ubuntu focal/main amd64 liboath0 amd64 2.6.1-1.3 [44.7 kB] 2023-10-01T10:02:16.121 INFO:teuthology.orchestra.run.smithi173.stdout:Get:12 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 python3-rados amd64 16.2.14-44-geeb13ac9-1focal [413 kB] 2023-10-01T10:02:16.122 INFO:teuthology.orchestra.run.smithi173.stdout:Get:13 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-cryptography amd64 2.8-3ubuntu0.1 [211 kB] 2023-10-01T10:02:16.145 INFO:teuthology.orchestra.run.smithi173.stdout:Get:14 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-openssl all 19.0.0-1build1 [43.3 kB] 2023-10-01T10:02:16.147 INFO:teuthology.orchestra.run.smithi173.stdout:Get:15 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 python3-ceph-argparse all 16.2.14-44-geeb13ac9-1focal [22.2 kB] 2023-10-01T10:02:16.148 INFO:teuthology.orchestra.run.smithi173.stdout:Get:16 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 python3-cephfs amd64 16.2.14-44-geeb13ac9-1focal [212 kB] 2023-10-01T10:02:16.148 INFO:teuthology.orchestra.run.smithi173.stdout:Get:17 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-bcrypt amd64 3.1.7-2ubuntu1 [30.4 kB] 2023-10-01T10:02:16.151 INFO:teuthology.orchestra.run.smithi173.stdout:Get:18 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-cherrypy3 all 8.9.1-6 [160 kB] 2023-10-01T10:02:16.155 INFO:teuthology.orchestra.run.smithi173.stdout:Get:19 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 python3-ceph-common all 16.2.14-44-geeb13ac9-1focal [38.4 kB] 2023-10-01T10:02:16.156 INFO:teuthology.orchestra.run.smithi173.stdout:Get:20 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 python3-rbd amd64 16.2.14-44-geeb13ac9-1focal [386 kB] 2023-10-01T10:02:16.162 INFO:teuthology.orchestra.run.smithi173.stdout:Get:21 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-jwt all 1.7.1-2ubuntu2.1 [18.0 kB] 2023-10-01T10:02:16.163 INFO:teuthology.orchestra.run.smithi173.stdout:Get:22 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-logutils all 0.3.3-7 [17.3 kB] 2023-10-01T10:02:16.164 INFO:teuthology.orchestra.run.smithi173.stdout:Get:23 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-markupsafe amd64 1.1.0-1build2 [13.9 kB] 2023-10-01T10:02:16.166 INFO:teuthology.orchestra.run.smithi173.stdout:Get:24 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-mako all 1.1.0+ds1-1ubuntu2.1 [59.2 kB] 2023-10-01T10:02:16.181 INFO:teuthology.orchestra.run.smithi173.stdout:Get:25 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 librgw2 amd64 16.2.14-44-geeb13ac9-1focal [3,691 kB] 2023-10-01T10:02:16.224 INFO:teuthology.orchestra.run.smithi173.stdout:Get:26 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-simplegeneric all 0.8.1-3 [11.3 kB] 2023-10-01T10:02:16.225 INFO:teuthology.orchestra.run.smithi173.stdout:Get:27 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-singledispatch all 3.4.0.3-2build1 [7,128 B] 2023-10-01T10:02:16.226 INFO:teuthology.orchestra.run.smithi173.stdout:Get:28 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webob all 1:1.8.5-2 [84.4 kB] 2023-10-01T10:02:16.228 INFO:teuthology.orchestra.run.smithi173.stdout:Get:29 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-soupsieve all 1.9.5+dfsg-1 [29.1 kB] 2023-10-01T10:02:16.229 INFO:teuthology.orchestra.run.smithi173.stdout:Get:30 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-bs4 all 4.8.2-1 [83.0 kB] 2023-10-01T10:02:16.232 INFO:teuthology.orchestra.run.smithi173.stdout:Get:31 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-waitress all 1.4.1-1ubuntu0.1 [81.4 kB] 2023-10-01T10:02:16.235 INFO:teuthology.orchestra.run.smithi173.stdout:Get:32 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-tempita all 0.5.2-6 [14.8 kB] 2023-10-01T10:02:16.280 INFO:teuthology.orchestra.run.smithi173.stdout:Get:33 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-paste all 3.4.0+dfsg1-1 [456 kB] 2023-10-01T10:02:16.301 INFO:teuthology.orchestra.run.smithi173.stdout:Get:34 http://archive.ubuntu.com/ubuntu focal/main amd64 python-pastedeploy-tpl all 2.1.0-1 [5,016 B] 2023-10-01T10:02:16.301 INFO:teuthology.orchestra.run.smithi173.stdout:Get:35 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pastedeploy all 2.1.0-1 [26.5 kB] 2023-10-01T10:02:16.319 INFO:teuthology.orchestra.run.smithi078.stdout:Get:4 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 librados2 amd64 16.2.14-44-geeb13ac9-1focal [3,390 kB] 2023-10-01T10:02:16.336 INFO:teuthology.orchestra.run.smithi173.stdout:Get:36 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webtest all 2.0.32-2 [28.2 kB] 2023-10-01T10:02:16.338 INFO:teuthology.orchestra.run.smithi173.stdout:Get:37 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pecan all 1.3.3-3ubuntu1 [85.8 kB] 2023-10-01T10:02:16.340 INFO:teuthology.orchestra.run.smithi173.stdout:Get:38 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-werkzeug all 0.16.1+dfsg1-2ubuntu0.1 [183 kB] 2023-10-01T10:02:16.345 INFO:teuthology.orchestra.run.smithi173.stdout:Get:39 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-jinja2 all 2.10.1-2 [95.5 kB] 2023-10-01T10:02:16.346 INFO:teuthology.orchestra.run.smithi173.stdout:Get:40 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-repoze.lru all 0.7-2 [12.1 kB] 2023-10-01T10:02:16.346 INFO:teuthology.orchestra.run.smithi173.stdout:Get:41 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 libjs-underscore all 1.9.1~dfsg-1ubuntu0.20.04.1 [99.5 kB] 2023-10-01T10:02:16.353 INFO:teuthology.orchestra.run.smithi173.stdout:Get:42 http://archive.ubuntu.com/ubuntu focal/main amd64 libjs-sphinxdoc all 1.8.5-7ubuntu3 [97.1 kB] 2023-10-01T10:02:16.366 INFO:teuthology.orchestra.run.smithi173.stdout:Get:43 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 python3-rgw amd64 16.2.14-44-geeb13ac9-1focal [116 kB] 2023-10-01T10:02:16.377 INFO:teuthology.orchestra.run.smithi173.stdout:Get:44 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 libradosstriper1 amd64 16.2.14-44-geeb13ac9-1focal [449 kB] 2023-10-01T10:02:16.384 INFO:teuthology.orchestra.run.smithi078.stdout:Get:5 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-prettytable all 0.7.2-5 [20.1 kB] 2023-10-01T10:02:16.389 INFO:teuthology.orchestra.run.smithi173.stdout:Get:45 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-routes all 2.4.1-2 [88.1 kB] 2023-10-01T10:02:16.391 INFO:teuthology.orchestra.run.smithi173.stdout:Get:46 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 ceph-common amd64 16.2.14-44-geeb13ac9-1focal [22.6 MB] 2023-10-01T10:02:16.392 INFO:teuthology.orchestra.run.smithi078.stdout:Get:6 http://archive.ubuntu.com/ubuntu focal/main amd64 liblua5.3-0 amd64 5.3.3-1.1ubuntu2 [116 kB] 2023-10-01T10:02:16.392 INFO:teuthology.orchestra.run.smithi173.stdout:Get:47 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-decorator all 4.4.2-0ubuntu1 [10.3 kB] 2023-10-01T10:02:16.393 INFO:teuthology.orchestra.run.smithi173.stdout:Get:48 http://archive.ubuntu.com/ubuntu focal/universe amd64 liblbfgsb0 amd64 3.0+dfsg.3-7build1 [29.5 kB] 2023-10-01T10:02:16.436 INFO:teuthology.orchestra.run.smithi078.stdout:Get:7 http://archive.ubuntu.com/ubuntu focal/main amd64 librabbitmq4 amd64 0.10.0-1 [35.8 kB] 2023-10-01T10:02:16.447 INFO:teuthology.orchestra.run.smithi173.stdout:Get:49 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-scipy amd64 1.3.3-3build1 [11.1 MB] 2023-10-01T10:02:16.449 INFO:teuthology.orchestra.run.smithi078.stdout:Get:8 http://archive.ubuntu.com/ubuntu focal/universe amd64 librdkafka1 amd64 1.2.1-1ubuntu1 [410 kB] 2023-10-01T10:02:16.576 INFO:teuthology.orchestra.run.smithi078.stdout:Get:9 http://archive.ubuntu.com/ubuntu focal/main amd64 libleveldb1d amd64 1.22-3ubuntu2 [139 kB] 2023-10-01T10:02:16.599 INFO:teuthology.orchestra.run.smithi078.stdout:Get:10 http://archive.ubuntu.com/ubuntu focal/main amd64 liboath0 amd64 2.6.1-1.3 [44.7 kB] 2023-10-01T10:02:16.605 INFO:teuthology.orchestra.run.smithi078.stdout:Get:11 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-cryptography amd64 2.8-3ubuntu0.1 [211 kB] 2023-10-01T10:02:16.624 INFO:teuthology.orchestra.run.smithi078.stdout:Get:12 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 libcephfs2 amd64 16.2.14-44-geeb13ac9-1focal [729 kB] 2023-10-01T10:02:16.636 INFO:teuthology.orchestra.run.smithi078.stdout:Get:13 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-openssl all 19.0.0-1build1 [43.3 kB] 2023-10-01T10:02:16.641 INFO:teuthology.orchestra.run.smithi078.stdout:Get:14 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-bcrypt amd64 3.1.7-2ubuntu1 [30.4 kB] 2023-10-01T10:02:16.644 INFO:teuthology.orchestra.run.smithi078.stdout:Get:15 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-cherrypy3 all 8.9.1-6 [160 kB] 2023-10-01T10:02:16.685 INFO:teuthology.orchestra.run.smithi078.stdout:Get:16 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 python3-rados amd64 16.2.14-44-geeb13ac9-1focal [413 kB] 2023-10-01T10:02:16.736 INFO:teuthology.orchestra.run.smithi078.stdout:Get:17 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 python3-ceph-argparse all 16.2.14-44-geeb13ac9-1focal [22.2 kB] 2023-10-01T10:02:16.739 INFO:teuthology.orchestra.run.smithi078.stdout:Get:18 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 python3-cephfs amd64 16.2.14-44-geeb13ac9-1focal [212 kB] 2023-10-01T10:02:16.746 INFO:teuthology.orchestra.run.smithi078.stdout:Get:19 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 python3-ceph-common all 16.2.14-44-geeb13ac9-1focal [38.4 kB] 2023-10-01T10:02:16.747 INFO:teuthology.orchestra.run.smithi078.stdout:Get:20 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 python3-rbd amd64 16.2.14-44-geeb13ac9-1focal [386 kB] 2023-10-01T10:02:16.799 INFO:teuthology.orchestra.run.smithi078.stdout:Get:21 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 librgw2 amd64 16.2.14-44-geeb13ac9-1focal [3,691 kB] 2023-10-01T10:02:16.824 INFO:teuthology.orchestra.run.smithi078.stdout:Get:22 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-jwt all 1.7.1-2ubuntu2.1 [18.0 kB] 2023-10-01T10:02:16.886 INFO:teuthology.orchestra.run.smithi173.stdout:Get:50 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-sklearn-lib amd64 0.22.2.post1+dfsg-5 [2,358 kB] 2023-10-01T10:02:16.902 INFO:teuthology.orchestra.run.smithi173.stdout:Get:51 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-joblib all 0.14.0-3 [192 kB] 2023-10-01T10:02:16.903 INFO:teuthology.orchestra.run.smithi173.stdout:Get:52 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-sklearn all 0.22.2.post1+dfsg-5 [1,773 kB] 2023-10-01T10:02:16.939 INFO:teuthology.orchestra.run.smithi173.stdout:Get:53 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-cachetools all 4.0.0-1 [9,820 B] 2023-10-01T10:02:16.939 INFO:teuthology.orchestra.run.smithi173.stdout:Get:54 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyasn1 all 0.4.2-3build1 [46.7 kB] 2023-10-01T10:02:16.939 INFO:teuthology.orchestra.run.smithi173.stdout:Get:55 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyasn1-modules all 0.2.1-0.2build1 [32.9 kB] 2023-10-01T10:02:16.940 INFO:teuthology.orchestra.run.smithi173.stdout:Get:56 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-rsa all 4.0-3 [24.7 kB] 2023-10-01T10:02:16.940 INFO:teuthology.orchestra.run.smithi173.stdout:Get:57 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-google-auth all 1.5.1-2 [35.6 kB] 2023-10-01T10:02:16.941 INFO:teuthology.orchestra.run.smithi173.stdout:Get:58 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-blinker all 1.4+dfsg1-0.3ubuntu1 [13.2 kB] 2023-10-01T10:02:16.941 INFO:teuthology.orchestra.run.smithi173.stdout:Get:59 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-oauthlib all 3.1.0-1ubuntu2 [84.8 kB] 2023-10-01T10:02:16.968 INFO:teuthology.orchestra.run.smithi173.stdout:Get:60 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-requests-oauthlib all 1.0.0-1.1build1 [16.8 kB] 2023-10-01T10:02:16.969 INFO:teuthology.orchestra.run.smithi173.stdout:Get:61 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-websocket all 0.53.0-2ubuntu1 [32.3 kB] 2023-10-01T10:02:16.991 INFO:teuthology.orchestra.run.smithi078.stdout:Get:23 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-logutils all 0.3.3-7 [17.3 kB] 2023-10-01T10:02:17.004 INFO:teuthology.orchestra.run.smithi173.stdout:Get:62 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-kubernetes all 7.0.0~a1-2 [271 kB] 2023-10-01T10:02:17.010 INFO:teuthology.orchestra.run.smithi173.stdout:Get:63 http://archive.ubuntu.com/ubuntu focal/universe amd64 libonig5 amd64 6.9.4-1 [142 kB] 2023-10-01T10:02:17.011 INFO:teuthology.orchestra.run.smithi173.stdout:Get:64 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 libjq1 amd64 1.6-1ubuntu0.20.04.1 [121 kB] 2023-10-01T10:02:17.013 INFO:teuthology.orchestra.run.smithi173.stdout:Get:65 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 jq amd64 1.6-1ubuntu0.20.04.1 [50.2 kB] 2023-10-01T10:02:17.014 INFO:teuthology.orchestra.run.smithi173.stdout:Get:66 http://archive.ubuntu.com/ubuntu focal/main amd64 socat amd64 1.7.3.3-2 [323 kB] 2023-10-01T10:02:17.018 INFO:teuthology.orchestra.run.smithi173.stdout:Get:67 http://archive.ubuntu.com/ubuntu focal/universe amd64 xmlstarlet amd64 1.6.1-2build1 [267 kB] 2023-10-01T10:02:17.020 INFO:teuthology.orchestra.run.smithi078.stdout:Get:24 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-markupsafe amd64 1.1.0-1build2 [13.9 kB] 2023-10-01T10:02:17.021 INFO:teuthology.orchestra.run.smithi173.stdout:Get:68 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 nvme-cli amd64 1.9-1ubuntu0.1 [283 kB] 2023-10-01T10:02:17.026 INFO:teuthology.orchestra.run.smithi173.stdout:Get:69 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-atomicwrites all 1.1.5-2build1 [7,016 B] 2023-10-01T10:02:17.045 INFO:teuthology.orchestra.run.smithi078.stdout:Get:25 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-mako all 1.1.0+ds1-1ubuntu2.1 [59.2 kB] 2023-10-01T10:02:17.047 INFO:teuthology.orchestra.run.smithi173.stdout:Get:70 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-attr all 19.3.0-2 [33.9 kB] 2023-10-01T10:02:17.048 INFO:teuthology.orchestra.run.smithi173.stdout:Get:71 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webencodings all 0.5.1-1ubuntu1 [11.0 kB] 2023-10-01T10:02:17.083 INFO:teuthology.orchestra.run.smithi173.stdout:Get:72 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-html5lib all 1.0.1-2 [84.3 kB] 2023-10-01T10:02:17.085 INFO:teuthology.orchestra.run.smithi173.stdout:Get:73 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-lxml amd64 4.5.0-1ubuntu0.5 [1,384 kB] 2023-10-01T10:02:17.106 INFO:teuthology.orchestra.run.smithi173.stdout:Get:74 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-packaging all 20.3-1 [26.8 kB] 2023-10-01T10:02:17.106 INFO:teuthology.orchestra.run.smithi173.stdout:Get:75 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pastescript all 2.0.2-4 [54.6 kB] 2023-10-01T10:02:17.107 INFO:teuthology.orchestra.run.smithi173.stdout:Get:76 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-pluggy all 0.13.0-2 [18.4 kB] 2023-10-01T10:02:17.107 INFO:teuthology.orchestra.run.smithi173.stdout:Get:77 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-psutil amd64 5.5.1-1ubuntu4 [141 kB] 2023-10-01T10:02:17.109 INFO:teuthology.orchestra.run.smithi173.stdout:Get:78 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 python3-py all 1.8.1-1ubuntu0.1 [65.6 kB] 2023-10-01T10:02:17.110 INFO:teuthology.orchestra.run.smithi173.stdout:Get:79 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyinotify all 0.9.6-1.2ubuntu1 [24.8 kB] 2023-10-01T10:02:17.129 INFO:teuthology.orchestra.run.smithi173.stdout:Get:80 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-wcwidth all 0.1.8+dfsg1-3 [17.4 kB] 2023-10-01T10:02:17.129 INFO:teuthology.orchestra.run.smithi173.stdout:Get:81 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-pytest all 4.6.9-1 [178 kB] 2023-10-01T10:02:17.155 INFO:teuthology.orchestra.run.smithi078.stdout:Get:26 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 python3-rgw amd64 16.2.14-44-geeb13ac9-1focal [116 kB] 2023-10-01T10:02:17.161 INFO:teuthology.orchestra.run.smithi078.stdout:Get:27 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-simplegeneric all 0.8.1-3 [11.3 kB] 2023-10-01T10:02:17.166 INFO:teuthology.orchestra.run.smithi173.stdout:Get:82 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-simplejson amd64 3.16.0-2ubuntu2 [50.8 kB] 2023-10-01T10:02:17.174 INFO:teuthology.orchestra.run.smithi078.stdout:Get:28 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-singledispatch all 3.4.0.3-2build1 [7,128 B] 2023-10-01T10:02:17.178 INFO:teuthology.orchestra.run.smithi078.stdout:Get:29 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webob all 1:1.8.5-2 [84.4 kB] 2023-10-01T10:02:17.232 INFO:teuthology.orchestra.run.smithi078.stdout:Get:30 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 libradosstriper1 amd64 16.2.14-44-geeb13ac9-1focal [449 kB] 2023-10-01T10:02:17.243 INFO:teuthology.orchestra.run.smithi078.stdout:Get:31 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 ceph-common amd64 16.2.14-44-geeb13ac9-1focal [22.6 MB] 2023-10-01T10:02:17.245 INFO:teuthology.orchestra.run.smithi078.stdout:Get:32 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-soupsieve all 1.9.5+dfsg-1 [29.1 kB] 2023-10-01T10:02:17.261 INFO:teuthology.orchestra.run.smithi078.stdout:Get:33 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-bs4 all 4.8.2-1 [83.0 kB] 2023-10-01T10:02:17.296 INFO:teuthology.orchestra.run.smithi078.stdout:Get:34 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-waitress all 1.4.1-1ubuntu0.1 [81.4 kB] 2023-10-01T10:02:17.323 INFO:teuthology.orchestra.run.smithi078.stdout:Get:35 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-tempita all 0.5.2-6 [14.8 kB] 2023-10-01T10:02:17.491 INFO:teuthology.orchestra.run.smithi078.stdout:Get:36 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-paste all 3.4.0+dfsg1-1 [456 kB] 2023-10-01T10:02:17.627 INFO:teuthology.orchestra.run.smithi173.stdout:Get:83 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 ceph-base amd64 16.2.14-44-geeb13ac9-1focal [5,914 kB] 2023-10-01T10:02:17.856 INFO:teuthology.orchestra.run.smithi173.stdout:Get:84 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-modules-core all 16.2.14-44-geeb13ac9-1focal [204 kB] 2023-10-01T10:02:17.861 INFO:teuthology.orchestra.run.smithi173.stdout:Get:85 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 libsqlite3-mod-ceph amd64 16.2.14-44-geeb13ac9-1focal [126 kB] 2023-10-01T10:02:17.865 INFO:teuthology.orchestra.run.smithi173.stdout:Get:86 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr amd64 16.2.14-44-geeb13ac9-1focal [1,536 kB] 2023-10-01T10:02:17.938 INFO:teuthology.orchestra.run.smithi173.stdout:Get:87 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 ceph-mon amd64 16.2.14-44-geeb13ac9-1focal [6,994 kB] 2023-10-01T10:02:18.017 INFO:teuthology.orchestra.run.smithi078.stdout:Get:37 http://archive.ubuntu.com/ubuntu focal/main amd64 python-pastedeploy-tpl all 2.1.0-1 [5,016 B] 2023-10-01T10:02:18.017 INFO:teuthology.orchestra.run.smithi078.stdout:Get:38 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pastedeploy all 2.1.0-1 [26.5 kB] 2023-10-01T10:02:18.020 INFO:teuthology.orchestra.run.smithi078.stdout:Get:39 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webtest all 2.0.32-2 [28.2 kB] 2023-10-01T10:02:18.024 INFO:teuthology.orchestra.run.smithi078.stdout:Get:40 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pecan all 1.3.3-3ubuntu1 [85.8 kB] 2023-10-01T10:02:18.035 INFO:teuthology.orchestra.run.smithi078.stdout:Get:41 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-werkzeug all 0.16.1+dfsg1-2ubuntu0.1 [183 kB] 2023-10-01T10:02:18.059 INFO:teuthology.orchestra.run.smithi078.stdout:Get:42 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-jinja2 all 2.10.1-2 [95.5 kB] 2023-10-01T10:02:18.071 INFO:teuthology.orchestra.run.smithi078.stdout:Get:43 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-repoze.lru all 0.7-2 [12.1 kB] 2023-10-01T10:02:18.073 INFO:teuthology.orchestra.run.smithi078.stdout:Get:44 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 libjs-underscore all 1.9.1~dfsg-1ubuntu0.20.04.1 [99.5 kB] 2023-10-01T10:02:18.107 INFO:teuthology.orchestra.run.smithi078.stdout:Get:45 http://archive.ubuntu.com/ubuntu focal/main amd64 libjs-sphinxdoc all 1.8.5-7ubuntu3 [97.1 kB] 2023-10-01T10:02:18.120 INFO:teuthology.orchestra.run.smithi078.stdout:Get:46 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-routes all 2.4.1-2 [88.1 kB] 2023-10-01T10:02:18.295 INFO:teuthology.orchestra.run.smithi078.stdout:Get:47 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-decorator all 4.4.2-0ubuntu1 [10.3 kB] 2023-10-01T10:02:18.309 INFO:teuthology.orchestra.run.smithi173.stdout:Get:88 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 ceph-osd amd64 16.2.14-44-geeb13ac9-1focal [26.2 MB] 2023-10-01T10:02:18.381 INFO:teuthology.orchestra.run.smithi078.stdout:Get:48 http://archive.ubuntu.com/ubuntu focal/universe amd64 liblbfgsb0 amd64 3.0+dfsg.3-7build1 [29.5 kB] 2023-10-01T10:02:18.466 INFO:teuthology.orchestra.run.smithi078.stdout:Get:49 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-scipy amd64 1.3.3-3build1 [11.1 MB] 2023-10-01T10:02:19.237 INFO:teuthology.orchestra.run.smithi078.stdout:Get:50 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-sklearn-lib amd64 0.22.2.post1+dfsg-5 [2,358 kB] 2023-10-01T10:02:19.306 INFO:teuthology.orchestra.run.smithi078.stdout:Get:51 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-joblib all 0.14.0-3 [192 kB] 2023-10-01T10:02:19.346 INFO:teuthology.orchestra.run.smithi078.stdout:Get:52 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-sklearn all 0.22.2.post1+dfsg-5 [1,773 kB] 2023-10-01T10:02:19.364 INFO:teuthology.orchestra.run.smithi078.stdout:Get:53 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-cachetools all 4.0.0-1 [9,820 B] 2023-10-01T10:02:19.364 INFO:teuthology.orchestra.run.smithi078.stdout:Get:54 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyasn1 all 0.4.2-3build1 [46.7 kB] 2023-10-01T10:02:19.364 INFO:teuthology.orchestra.run.smithi078.stdout:Get:55 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyasn1-modules all 0.2.1-0.2build1 [32.9 kB] 2023-10-01T10:02:19.365 INFO:teuthology.orchestra.run.smithi078.stdout:Get:56 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-rsa all 4.0-3 [24.7 kB] 2023-10-01T10:02:19.416 INFO:teuthology.orchestra.run.smithi078.stdout:Get:57 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 ceph-base amd64 16.2.14-44-geeb13ac9-1focal [5,914 kB] 2023-10-01T10:02:19.416 INFO:teuthology.orchestra.run.smithi078.stdout:Get:58 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-google-auth all 1.5.1-2 [35.6 kB] 2023-10-01T10:02:19.580 INFO:teuthology.orchestra.run.smithi078.stdout:Get:59 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-blinker all 1.4+dfsg1-0.3ubuntu1 [13.2 kB] 2023-10-01T10:02:19.664 INFO:teuthology.orchestra.run.smithi078.stdout:Get:60 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-oauthlib all 3.1.0-1ubuntu2 [84.8 kB] 2023-10-01T10:02:19.722 INFO:teuthology.orchestra.run.smithi173.stdout:Get:89 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 ceph amd64 16.2.14-44-geeb13ac9-1focal [4,048 B] 2023-10-01T10:02:19.722 INFO:teuthology.orchestra.run.smithi173.stdout:Get:90 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 ceph-fuse amd64 16.2.14-44-geeb13ac9-1focal [834 kB] 2023-10-01T10:02:19.728 INFO:teuthology.orchestra.run.smithi173.stdout:Get:91 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 ceph-mds amd64 16.2.14-44-geeb13ac9-1focal [2,363 kB] 2023-10-01T10:02:19.793 INFO:teuthology.orchestra.run.smithi173.stdout:Get:92 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 cephadm amd64 16.2.14-44-geeb13ac9-1focal [76.4 kB] 2023-10-01T10:02:19.795 INFO:teuthology.orchestra.run.smithi173.stdout:Get:93 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-cephadm all 16.2.14-44-geeb13ac9-1focal [100 kB] 2023-10-01T10:02:19.809 INFO:teuthology.orchestra.run.smithi078.stdout:Get:61 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-modules-core all 16.2.14-44-geeb13ac9-1focal [204 kB] 2023-10-01T10:02:19.810 INFO:teuthology.orchestra.run.smithi173.stdout:Get:94 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-dashboard all 16.2.14-44-geeb13ac9-1focal [2,823 kB] 2023-10-01T10:02:19.842 INFO:teuthology.orchestra.run.smithi078.stdout:Get:62 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-requests-oauthlib all 1.0.0-1.1build1 [16.8 kB] 2023-10-01T10:02:19.843 INFO:teuthology.orchestra.run.smithi078.stdout:Get:63 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 libsqlite3-mod-ceph amd64 16.2.14-44-geeb13ac9-1focal [126 kB] 2023-10-01T10:02:19.846 INFO:teuthology.orchestra.run.smithi078.stdout:Get:64 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr amd64 16.2.14-44-geeb13ac9-1focal [1,536 kB] 2023-10-01T10:02:19.858 INFO:teuthology.orchestra.run.smithi078.stdout:Get:65 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-websocket all 0.53.0-2ubuntu1 [32.3 kB] 2023-10-01T10:02:19.890 INFO:teuthology.orchestra.run.smithi078.stdout:Get:66 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-kubernetes all 7.0.0~a1-2 [271 kB] 2023-10-01T10:02:19.944 INFO:teuthology.orchestra.run.smithi173.stdout:Get:95 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-diskprediction-local all 16.2.14-44-geeb13ac9-1focal [7,890 kB] 2023-10-01T10:02:19.993 INFO:teuthology.orchestra.run.smithi078.stdout:Get:67 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 ceph-mon amd64 16.2.14-44-geeb13ac9-1focal [6,994 kB] 2023-10-01T10:02:20.020 INFO:teuthology.orchestra.run.smithi078.stdout:Get:68 http://archive.ubuntu.com/ubuntu focal/universe amd64 libonig5 amd64 6.9.4-1 [142 kB] 2023-10-01T10:02:20.056 INFO:teuthology.orchestra.run.smithi078.stdout:Get:69 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 libjq1 amd64 1.6-1ubuntu0.20.04.1 [121 kB] 2023-10-01T10:02:20.080 INFO:teuthology.orchestra.run.smithi078.stdout:Get:70 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 jq amd64 1.6-1ubuntu0.20.04.1 [50.2 kB] 2023-10-01T10:02:20.088 INFO:teuthology.orchestra.run.smithi078.stdout:Get:71 http://archive.ubuntu.com/ubuntu focal/main amd64 socat amd64 1.7.3.3-2 [323 kB] 2023-10-01T10:02:20.131 INFO:teuthology.orchestra.run.smithi078.stdout:Get:72 http://archive.ubuntu.com/ubuntu focal/universe amd64 xmlstarlet amd64 1.6.1-2build1 [267 kB] 2023-10-01T10:02:20.158 INFO:teuthology.orchestra.run.smithi078.stdout:Get:73 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 nvme-cli amd64 1.9-1ubuntu0.1 [283 kB] 2023-10-01T10:02:20.344 INFO:teuthology.orchestra.run.smithi078.stdout:Get:74 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-atomicwrites all 1.1.5-2build1 [7,016 B] 2023-10-01T10:02:20.351 INFO:teuthology.orchestra.run.smithi173.stdout:Get:96 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-k8sevents all 16.2.14-44-geeb13ac9-1focal [4,164 B] 2023-10-01T10:02:20.351 INFO:teuthology.orchestra.run.smithi173.stdout:Get:97 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 ceph-test amd64 16.2.14-44-geeb13ac9-1focal [49.3 MB] 2023-10-01T10:02:20.428 INFO:teuthology.orchestra.run.smithi078.stdout:Get:75 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-attr all 19.3.0-2 [33.9 kB] 2023-10-01T10:02:20.529 INFO:teuthology.orchestra.run.smithi078.stdout:Get:76 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-webencodings all 0.5.1-1ubuntu1 [11.0 kB] 2023-10-01T10:02:20.552 INFO:teuthology.orchestra.run.smithi078.stdout:Get:77 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-html5lib all 1.0.1-2 [84.3 kB] 2023-10-01T10:02:20.633 INFO:teuthology.orchestra.run.smithi078.stdout:Get:78 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 ceph-osd amd64 16.2.14-44-geeb13ac9-1focal [26.2 MB] 2023-10-01T10:02:20.668 INFO:teuthology.orchestra.run.smithi078.stdout:Get:79 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 python3-lxml amd64 4.5.0-1ubuntu0.5 [1,384 kB] 2023-10-01T10:02:20.954 INFO:teuthology.orchestra.run.smithi078.stdout:Get:80 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-packaging all 20.3-1 [26.8 kB] 2023-10-01T10:02:20.955 INFO:teuthology.orchestra.run.smithi078.stdout:Get:81 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pastescript all 2.0.2-4 [54.6 kB] 2023-10-01T10:02:20.960 INFO:teuthology.orchestra.run.smithi078.stdout:Get:82 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-pluggy all 0.13.0-2 [18.4 kB] 2023-10-01T10:02:20.961 INFO:teuthology.orchestra.run.smithi078.stdout:Get:83 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-psutil amd64 5.5.1-1ubuntu4 [141 kB] 2023-10-01T10:02:20.971 INFO:teuthology.orchestra.run.smithi078.stdout:Get:84 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 python3-py all 1.8.1-1ubuntu0.1 [65.6 kB] 2023-10-01T10:02:20.976 INFO:teuthology.orchestra.run.smithi078.stdout:Get:85 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyinotify all 0.9.6-1.2ubuntu1 [24.8 kB] 2023-10-01T10:02:21.141 INFO:teuthology.orchestra.run.smithi078.stdout:Get:86 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-wcwidth all 0.1.8+dfsg1-3 [17.4 kB] 2023-10-01T10:02:21.308 INFO:teuthology.orchestra.run.smithi078.stdout:Get:87 http://archive.ubuntu.com/ubuntu focal/universe amd64 python3-pytest all 4.6.9-1 [178 kB] 2023-10-01T10:02:21.560 INFO:teuthology.orchestra.run.smithi078.stdout:Get:88 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-simplejson amd64 3.16.0-2ubuntu2 [50.8 kB] 2023-10-01T10:02:23.004 INFO:teuthology.orchestra.run.smithi173.stdout:Get:98 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 libcephfs-dev amd64 16.2.14-44-geeb13ac9-1focal [21.5 kB] 2023-10-01T10:02:23.005 INFO:teuthology.orchestra.run.smithi173.stdout:Get:99 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 radosgw amd64 16.2.14-44-geeb13ac9-1focal [11.3 MB] 2023-10-01T10:02:23.247 INFO:teuthology.orchestra.run.smithi078.stdout:Get:89 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 ceph amd64 16.2.14-44-geeb13ac9-1focal [4,048 B] 2023-10-01T10:02:23.247 INFO:teuthology.orchestra.run.smithi078.stdout:Get:90 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 ceph-fuse amd64 16.2.14-44-geeb13ac9-1focal [834 kB] 2023-10-01T10:02:23.275 INFO:teuthology.orchestra.run.smithi078.stdout:Get:91 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 ceph-mds amd64 16.2.14-44-geeb13ac9-1focal [2,363 kB] 2023-10-01T10:02:23.345 INFO:teuthology.orchestra.run.smithi173.stdout:Get:100 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 rbd-fuse amd64 16.2.14-44-geeb13ac9-1focal [78.7 kB] 2023-10-01T10:02:23.405 INFO:teuthology.orchestra.run.smithi078.stdout:Get:92 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 cephadm amd64 16.2.14-44-geeb13ac9-1focal [76.4 kB] 2023-10-01T10:02:23.407 INFO:teuthology.orchestra.run.smithi078.stdout:Get:93 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-cephadm all 16.2.14-44-geeb13ac9-1focal [100 kB] 2023-10-01T10:02:23.410 INFO:teuthology.orchestra.run.smithi078.stdout:Get:94 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-dashboard all 16.2.14-44-geeb13ac9-1focal [2,823 kB] 2023-10-01T10:02:23.503 INFO:teuthology.orchestra.run.smithi078.stdout:Get:95 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-diskprediction-local all 16.2.14-44-geeb13ac9-1focal [7,890 kB] 2023-10-01T10:02:23.763 INFO:teuthology.orchestra.run.smithi078.stdout:Get:96 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 ceph-mgr-k8sevents all 16.2.14-44-geeb13ac9-1focal [4,164 B] 2023-10-01T10:02:23.763 INFO:teuthology.orchestra.run.smithi078.stdout:Get:97 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 ceph-test amd64 16.2.14-44-geeb13ac9-1focal [49.3 MB] 2023-10-01T10:02:23.979 INFO:teuthology.orchestra.run.smithi173.stdout:Fetched 174 MB in 8s (22.2 MB/s) 2023-10-01T10:02:24.074 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package libjaeger. 2023-10-01T10:02:24.120 INFO:teuthology.orchestra.run.smithi173.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 112648 files and directories currently installed.) 2023-10-01T10:02:24.123 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../00-libjaeger_16.2.14-44-geeb13ac9-1focal_amd64.deb ... 2023-10-01T10:02:24.164 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking libjaeger (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:02:24.509 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package liblttng-ust0:amd64. 2023-10-01T10:02:24.522 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../01-liblttng-ust0_2.11.0-1_amd64.deb ... 2023-10-01T10:02:24.609 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking liblttng-ust0:amd64 (2.11.0-1) ... 2023-10-01T10:02:24.954 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../02-librbd1_16.2.14-44-geeb13ac9-1focal_amd64.deb ... 2023-10-01T10:02:25.081 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking librbd1 (16.2.14-44-geeb13ac9-1focal) over (15.2.17-0ubuntu0.20.04.4) ... 2023-10-01T10:02:25.634 INFO:teuthology.orchestra.run.smithi078.stdout:Get:98 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 libcephfs-dev amd64 16.2.14-44-geeb13ac9-1focal [21.5 kB] 2023-10-01T10:02:25.635 INFO:teuthology.orchestra.run.smithi078.stdout:Get:99 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 radosgw amd64 16.2.14-44-geeb13ac9-1focal [11.3 MB] 2023-10-01T10:02:25.731 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../03-librados2_16.2.14-44-geeb13ac9-1focal_amd64.deb ... 2023-10-01T10:02:25.852 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking librados2 (16.2.14-44-geeb13ac9-1focal) over (15.2.17-0ubuntu0.20.04.4) ... 2023-10-01T10:02:25.900 INFO:teuthology.orchestra.run.smithi078.stdout:Get:100 https://3.chacra.ceph.com/r/ceph/pacific/eeb13ac974775bd0cce96a2c687e8bcc2faca43d/ubuntu/focal/flavors/default focal/main amd64 rbd-fuse amd64 16.2.14-44-geeb13ac9-1focal [78.7 kB] 2023-10-01T10:02:26.532 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package libcephfs2. 2023-10-01T10:02:26.544 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../04-libcephfs2_16.2.14-44-geeb13ac9-1focal_amd64.deb ... 2023-10-01T10:02:26.567 INFO:teuthology.orchestra.run.smithi078.stdout:Fetched 174 MB in 10s (17.2 MB/s) 2023-10-01T10:02:26.581 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking libcephfs2 (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:02:26.668 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package libjaeger. 2023-10-01T10:02:26.715 INFO:teuthology.orchestra.run.smithi078.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 112648 files and directories currently installed.) 2023-10-01T10:02:26.721 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../00-libjaeger_16.2.14-44-geeb13ac9-1focal_amd64.deb ... 2023-10-01T10:02:26.758 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking libjaeger (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:02:26.909 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-rados. 2023-10-01T10:02:26.923 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../05-python3-rados_16.2.14-44-geeb13ac9-1focal_amd64.deb ... 2023-10-01T10:02:26.967 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-rados (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:02:27.044 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package liblttng-ust0:amd64. 2023-10-01T10:02:27.058 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../01-liblttng-ust0_2.11.0-1_amd64.deb ... 2023-10-01T10:02:27.144 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking liblttng-ust0:amd64 (2.11.0-1) ... 2023-10-01T10:02:27.338 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-ceph-argparse. 2023-10-01T10:02:27.350 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../06-python3-ceph-argparse_16.2.14-44-geeb13ac9-1focal_all.deb ... 2023-10-01T10:02:27.396 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-ceph-argparse (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:02:27.494 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../02-librbd1_16.2.14-44-geeb13ac9-1focal_amd64.deb ... 2023-10-01T10:02:27.623 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking librbd1 (16.2.14-44-geeb13ac9-1focal) over (15.2.17-0ubuntu0.20.04.4) ... 2023-10-01T10:02:27.698 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-cephfs. 2023-10-01T10:02:27.714 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../07-python3-cephfs_16.2.14-44-geeb13ac9-1focal_amd64.deb ... 2023-10-01T10:02:27.757 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-cephfs (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:02:28.085 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-ceph-common. 2023-10-01T10:02:28.097 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../08-python3-ceph-common_16.2.14-44-geeb13ac9-1focal_all.deb ... 2023-10-01T10:02:28.135 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-ceph-common (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:02:28.323 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../03-librados2_16.2.14-44-geeb13ac9-1focal_amd64.deb ... 2023-10-01T10:02:28.452 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking librados2 (16.2.14-44-geeb13ac9-1focal) over (15.2.17-0ubuntu0.20.04.4) ... 2023-10-01T10:02:28.471 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-prettytable. 2023-10-01T10:02:28.484 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../09-python3-prettytable_0.7.2-5_all.deb ... 2023-10-01T10:02:28.521 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-prettytable (0.7.2-5) ... 2023-10-01T10:02:28.757 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-rbd. 2023-10-01T10:02:28.770 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../10-python3-rbd_16.2.14-44-geeb13ac9-1focal_amd64.deb ... 2023-10-01T10:02:28.807 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-rbd (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:02:29.114 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package libcephfs2. 2023-10-01T10:02:29.130 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../04-libcephfs2_16.2.14-44-geeb13ac9-1focal_amd64.deb ... 2023-10-01T10:02:29.135 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package liblua5.3-0:amd64. 2023-10-01T10:02:29.150 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../11-liblua5.3-0_5.3.3-1.1ubuntu2_amd64.deb ... 2023-10-01T10:02:29.172 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking libcephfs2 (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:02:29.193 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking liblua5.3-0:amd64 (5.3.3-1.1ubuntu2) ... 2023-10-01T10:02:29.541 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package python3-rados. 2023-10-01T10:02:29.544 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package librabbitmq4:amd64. 2023-10-01T10:02:29.554 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../05-python3-rados_16.2.14-44-geeb13ac9-1focal_amd64.deb ... 2023-10-01T10:02:29.557 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../12-librabbitmq4_0.10.0-1_amd64.deb ... 2023-10-01T10:02:29.602 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking python3-rados (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:02:29.611 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking librabbitmq4:amd64 (0.10.0-1) ... 2023-10-01T10:02:29.952 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package librdkafka1:amd64. 2023-10-01T10:02:29.966 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../13-librdkafka1_1.2.1-1ubuntu1_amd64.deb ... 2023-10-01T10:02:29.977 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package python3-ceph-argparse. 2023-10-01T10:02:29.991 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../06-python3-ceph-argparse_16.2.14-44-geeb13ac9-1focal_all.deb ... 2023-10-01T10:02:30.011 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking librdkafka1:amd64 (1.2.1-1ubuntu1) ... 2023-10-01T10:02:30.035 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking python3-ceph-argparse (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:02:30.328 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package librgw2. 2023-10-01T10:02:30.338 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package python3-cephfs. 2023-10-01T10:02:30.343 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../14-librgw2_16.2.14-44-geeb13ac9-1focal_amd64.deb ... 2023-10-01T10:02:30.353 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../07-python3-cephfs_16.2.14-44-geeb13ac9-1focal_amd64.deb ... 2023-10-01T10:02:30.387 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking librgw2 (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:02:30.396 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking python3-cephfs (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:02:30.774 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package python3-ceph-common. 2023-10-01T10:02:30.790 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../08-python3-ceph-common_16.2.14-44-geeb13ac9-1focal_all.deb ... 2023-10-01T10:02:30.841 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking python3-ceph-common (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:02:30.973 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-rgw. 2023-10-01T10:02:30.987 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../15-python3-rgw_16.2.14-44-geeb13ac9-1focal_amd64.deb ... 2023-10-01T10:02:31.031 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-rgw (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:02:31.219 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package python3-prettytable. 2023-10-01T10:02:31.234 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../09-python3-prettytable_0.7.2-5_all.deb ... 2023-10-01T10:02:31.269 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking python3-prettytable (0.7.2-5) ... 2023-10-01T10:02:31.351 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package libleveldb1d:amd64. 2023-10-01T10:02:31.366 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../16-libleveldb1d_1.22-3ubuntu2_amd64.deb ... 2023-10-01T10:02:31.409 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking libleveldb1d:amd64 (1.22-3ubuntu2) ... 2023-10-01T10:02:31.505 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package python3-rbd. 2023-10-01T10:02:31.519 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../10-python3-rbd_16.2.14-44-geeb13ac9-1focal_amd64.deb ... 2023-10-01T10:02:31.563 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking python3-rbd (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:02:31.765 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package liboath0. 2023-10-01T10:02:31.779 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../17-liboath0_2.6.1-1.3_amd64.deb ... 2023-10-01T10:02:31.823 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking liboath0 (2.6.1-1.3) ... 2023-10-01T10:02:31.913 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package liblua5.3-0:amd64. 2023-10-01T10:02:31.927 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../11-liblua5.3-0_5.3.3-1.1ubuntu2_amd64.deb ... 2023-10-01T10:02:31.965 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking liblua5.3-0:amd64 (5.3.3-1.1ubuntu2) ... 2023-10-01T10:02:32.159 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package libradosstriper1. 2023-10-01T10:02:32.176 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../18-libradosstriper1_16.2.14-44-geeb13ac9-1focal_amd64.deb ... 2023-10-01T10:02:32.217 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking libradosstriper1 (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:02:32.311 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package librabbitmq4:amd64. 2023-10-01T10:02:32.326 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../12-librabbitmq4_0.10.0-1_amd64.deb ... 2023-10-01T10:02:32.361 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking librabbitmq4:amd64 (0.10.0-1) ... 2023-10-01T10:02:32.670 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package ceph-common. 2023-10-01T10:02:32.686 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../19-ceph-common_16.2.14-44-geeb13ac9-1focal_amd64.deb ... 2023-10-01T10:02:32.728 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking ceph-common (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:02:32.744 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package librdkafka1:amd64. 2023-10-01T10:02:32.759 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../13-librdkafka1_1.2.1-1ubuntu1_amd64.deb ... 2023-10-01T10:02:32.803 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking librdkafka1:amd64 (1.2.1-1ubuntu1) ... 2023-10-01T10:02:33.217 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package librgw2. 2023-10-01T10:02:33.231 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../14-librgw2_16.2.14-44-geeb13ac9-1focal_amd64.deb ... 2023-10-01T10:02:33.337 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking librgw2 (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:02:33.925 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package python3-rgw. 2023-10-01T10:02:33.939 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../15-python3-rgw_16.2.14-44-geeb13ac9-1focal_amd64.deb ... 2023-10-01T10:02:33.983 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking python3-rgw (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:02:34.286 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package libleveldb1d:amd64. 2023-10-01T10:02:34.299 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../16-libleveldb1d_1.22-3ubuntu2_amd64.deb ... 2023-10-01T10:02:34.328 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking libleveldb1d:amd64 (1.22-3ubuntu2) ... 2023-10-01T10:02:34.672 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package liboath0. 2023-10-01T10:02:34.685 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../17-liboath0_2.6.1-1.3_amd64.deb ... 2023-10-01T10:02:34.730 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking liboath0 (2.6.1-1.3) ... 2023-10-01T10:02:34.943 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package ceph-base. 2023-10-01T10:02:34.956 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../20-ceph-base_16.2.14-44-geeb13ac9-1focal_amd64.deb ... 2023-10-01T10:02:35.011 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking ceph-base (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:02:35.058 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package libradosstriper1. 2023-10-01T10:02:35.074 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../18-libradosstriper1_16.2.14-44-geeb13ac9-1focal_amd64.deb ... 2023-10-01T10:02:35.116 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking libradosstriper1 (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:02:35.554 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package ceph-common. 2023-10-01T10:02:35.567 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../19-ceph-common_16.2.14-44-geeb13ac9-1focal_amd64.deb ... 2023-10-01T10:02:35.611 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking ceph-common (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:02:35.914 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-cryptography. 2023-10-01T10:02:35.929 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../21-python3-cryptography_2.8-3ubuntu0.1_amd64.deb ... 2023-10-01T10:02:35.997 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-cryptography (2.8-3ubuntu0.1) ... 2023-10-01T10:02:36.359 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-openssl. 2023-10-01T10:02:36.374 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../22-python3-openssl_19.0.0-1build1_all.deb ... 2023-10-01T10:02:36.417 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-openssl (19.0.0-1build1) ... 2023-10-01T10:02:36.645 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package ceph-mgr-modules-core. 2023-10-01T10:02:36.659 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../23-ceph-mgr-modules-core_16.2.14-44-geeb13ac9-1focal_all.deb ... 2023-10-01T10:02:36.703 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking ceph-mgr-modules-core (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:02:37.066 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-bcrypt. 2023-10-01T10:02:37.079 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../24-python3-bcrypt_3.1.7-2ubuntu1_amd64.deb ... 2023-10-01T10:02:37.124 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-bcrypt (3.1.7-2ubuntu1) ... 2023-10-01T10:02:37.428 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-cherrypy3. 2023-10-01T10:02:37.442 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../25-python3-cherrypy3_8.9.1-6_all.deb ... 2023-10-01T10:02:37.486 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-cherrypy3 (8.9.1-6) ... 2023-10-01T10:02:37.823 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-jwt. 2023-10-01T10:02:37.833 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package ceph-base. 2023-10-01T10:02:37.836 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../26-python3-jwt_1.7.1-2ubuntu2.1_all.deb ... 2023-10-01T10:02:37.847 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../20-ceph-base_16.2.14-44-geeb13ac9-1focal_amd64.deb ... 2023-10-01T10:02:37.881 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-jwt (1.7.1-2ubuntu2.1) ... 2023-10-01T10:02:37.901 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking ceph-base (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:02:38.176 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-logutils. 2023-10-01T10:02:38.190 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../27-python3-logutils_0.3.3-7_all.deb ... 2023-10-01T10:02:38.234 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-logutils (0.3.3-7) ... 2023-10-01T10:02:38.537 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-markupsafe. 2023-10-01T10:02:38.550 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../28-python3-markupsafe_1.1.0-1build2_amd64.deb ... 2023-10-01T10:02:38.586 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-markupsafe (1.1.0-1build2) ... 2023-10-01T10:02:38.896 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package python3-cryptography. 2023-10-01T10:02:38.898 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-mako. 2023-10-01T10:02:38.909 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../21-python3-cryptography_2.8-3ubuntu0.1_amd64.deb ... 2023-10-01T10:02:38.911 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../29-python3-mako_1.1.0+ds1-1ubuntu2.1_all.deb ... 2023-10-01T10:02:38.954 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking python3-cryptography (2.8-3ubuntu0.1) ... 2023-10-01T10:02:38.956 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-mako (1.1.0+ds1-1ubuntu2.1) ... 2023-10-01T10:02:39.259 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-simplegeneric. 2023-10-01T10:02:39.272 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../30-python3-simplegeneric_0.8.1-3_all.deb ... 2023-10-01T10:02:39.309 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-simplegeneric (0.8.1-3) ... 2023-10-01T10:02:39.400 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package python3-openssl. 2023-10-01T10:02:39.413 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../22-python3-openssl_19.0.0-1build1_all.deb ... 2023-10-01T10:02:39.458 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking python3-openssl (19.0.0-1build1) ... 2023-10-01T10:02:39.603 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-singledispatch. 2023-10-01T10:02:39.618 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../31-python3-singledispatch_3.4.0.3-2build1_all.deb ... 2023-10-01T10:02:39.661 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-singledispatch (3.4.0.3-2build1) ... 2023-10-01T10:02:39.760 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package ceph-mgr-modules-core. 2023-10-01T10:02:39.774 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../23-ceph-mgr-modules-core_16.2.14-44-geeb13ac9-1focal_all.deb ... 2023-10-01T10:02:39.819 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking ceph-mgr-modules-core (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:02:39.989 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-webob. 2023-10-01T10:02:40.003 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../32-python3-webob_1%3a1.8.5-2_all.deb ... 2023-10-01T10:02:40.056 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-webob (1:1.8.5-2) ... 2023-10-01T10:02:40.183 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package python3-bcrypt. 2023-10-01T10:02:40.198 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../24-python3-bcrypt_3.1.7-2ubuntu1_amd64.deb ... 2023-10-01T10:02:40.241 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking python3-bcrypt (3.1.7-2ubuntu1) ... 2023-10-01T10:02:40.376 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-soupsieve. 2023-10-01T10:02:40.391 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../33-python3-soupsieve_1.9.5+dfsg-1_all.deb ... 2023-10-01T10:02:40.434 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-soupsieve (1.9.5+dfsg-1) ... 2023-10-01T10:02:40.544 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package python3-cherrypy3. 2023-10-01T10:02:40.557 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../25-python3-cherrypy3_8.9.1-6_all.deb ... 2023-10-01T10:02:40.593 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking python3-cherrypy3 (8.9.1-6) ... 2023-10-01T10:02:40.745 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-bs4. 2023-10-01T10:02:40.759 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../34-python3-bs4_4.8.2-1_all.deb ... 2023-10-01T10:02:40.803 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-bs4 (4.8.2-1) ... 2023-10-01T10:02:40.930 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package python3-jwt. 2023-10-01T10:02:40.943 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../26-python3-jwt_1.7.1-2ubuntu2.1_all.deb ... 2023-10-01T10:02:40.980 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking python3-jwt (1.7.1-2ubuntu2.1) ... 2023-10-01T10:02:41.165 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-waitress. 2023-10-01T10:02:41.179 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../35-python3-waitress_1.4.1-1ubuntu0.1_all.deb ... 2023-10-01T10:02:41.226 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-waitress (1.4.1-1ubuntu0.1) ... 2023-10-01T10:02:41.308 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package python3-logutils. 2023-10-01T10:02:41.323 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../27-python3-logutils_0.3.3-7_all.deb ... 2023-10-01T10:02:41.367 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking python3-logutils (0.3.3-7) ... 2023-10-01T10:02:41.535 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-tempita. 2023-10-01T10:02:41.550 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../36-python3-tempita_0.5.2-6_all.deb ... 2023-10-01T10:02:41.593 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-tempita (0.5.2-6) ... 2023-10-01T10:02:41.669 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package python3-markupsafe. 2023-10-01T10:02:41.684 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../28-python3-markupsafe_1.1.0-1build2_amd64.deb ... 2023-10-01T10:02:41.727 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking python3-markupsafe (1.1.0-1build2) ... 2023-10-01T10:02:41.895 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-paste. 2023-10-01T10:02:41.908 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../37-python3-paste_3.4.0+dfsg1-1_all.deb ... 2023-10-01T10:02:41.945 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-paste (3.4.0+dfsg1-1) ... 2023-10-01T10:02:42.013 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package python3-mako. 2023-10-01T10:02:42.027 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../29-python3-mako_1.1.0+ds1-1ubuntu2.1_all.deb ... 2023-10-01T10:02:42.071 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking python3-mako (1.1.0+ds1-1ubuntu2.1) ... 2023-10-01T10:02:42.257 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python-pastedeploy-tpl. 2023-10-01T10:02:42.270 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../38-python-pastedeploy-tpl_2.1.0-1_all.deb ... 2023-10-01T10:02:42.315 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python-pastedeploy-tpl (2.1.0-1) ... 2023-10-01T10:02:42.400 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package python3-simplegeneric. 2023-10-01T10:02:42.413 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../30-python3-simplegeneric_0.8.1-3_all.deb ... 2023-10-01T10:02:42.449 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking python3-simplegeneric (0.8.1-3) ... 2023-10-01T10:02:42.618 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-pastedeploy. 2023-10-01T10:02:42.631 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../39-python3-pastedeploy_2.1.0-1_all.deb ... 2023-10-01T10:02:42.676 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-pastedeploy (2.1.0-1) ... 2023-10-01T10:02:42.769 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package python3-singledispatch. 2023-10-01T10:02:42.783 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../31-python3-singledispatch_3.4.0.3-2build1_all.deb ... 2023-10-01T10:02:42.827 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking python3-singledispatch (3.4.0.3-2build1) ... 2023-10-01T10:02:42.979 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-webtest. 2023-10-01T10:02:42.992 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../40-python3-webtest_2.0.32-2_all.deb ... 2023-10-01T10:02:43.037 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-webtest (2.0.32-2) ... 2023-10-01T10:02:43.113 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package python3-webob. 2023-10-01T10:02:43.127 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../32-python3-webob_1%3a1.8.5-2_all.deb ... 2023-10-01T10:02:43.163 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking python3-webob (1:1.8.5-2) ... 2023-10-01T10:02:43.365 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-pecan. 2023-10-01T10:02:43.378 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../41-python3-pecan_1.3.3-3ubuntu1_all.deb ... 2023-10-01T10:02:43.415 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-pecan (1.3.3-3ubuntu1) ... 2023-10-01T10:02:43.483 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package python3-soupsieve. 2023-10-01T10:02:43.496 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../33-python3-soupsieve_1.9.5+dfsg-1_all.deb ... 2023-10-01T10:02:43.541 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking python3-soupsieve (1.9.5+dfsg-1) ... 2023-10-01T10:02:43.786 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-werkzeug. 2023-10-01T10:02:43.795 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../42-python3-werkzeug_0.16.1+dfsg1-2ubuntu0.1_all.deb ... 2023-10-01T10:02:43.835 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package python3-bs4. 2023-10-01T10:02:43.836 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-werkzeug (0.16.1+dfsg1-2ubuntu0.1) ... 2023-10-01T10:02:43.850 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../34-python3-bs4_4.8.2-1_all.deb ... 2023-10-01T10:02:43.893 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking python3-bs4 (4.8.2-1) ... 2023-10-01T10:02:44.157 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package libsqlite3-mod-ceph. 2023-10-01T10:02:44.170 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../43-libsqlite3-mod-ceph_16.2.14-44-geeb13ac9-1focal_amd64.deb ... 2023-10-01T10:02:44.214 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking libsqlite3-mod-ceph (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:02:44.230 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package python3-waitress. 2023-10-01T10:02:44.244 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../35-python3-waitress_1.4.1-1ubuntu0.1_all.deb ... 2023-10-01T10:02:44.291 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking python3-waitress (1.4.1-1ubuntu0.1) ... 2023-10-01T10:02:44.559 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package ceph-mgr. 2023-10-01T10:02:44.572 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../44-ceph-mgr_16.2.14-44-geeb13ac9-1focal_amd64.deb ... 2023-10-01T10:02:44.610 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package python3-tempita. 2023-10-01T10:02:44.617 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking ceph-mgr (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:02:44.624 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../36-python3-tempita_0.5.2-6_all.deb ... 2023-10-01T10:02:44.668 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking python3-tempita (0.5.2-6) ... 2023-10-01T10:02:44.971 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package python3-paste. 2023-10-01T10:02:44.985 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../37-python3-paste_3.4.0+dfsg1-1_all.deb ... 2023-10-01T10:02:45.029 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking python3-paste (3.4.0+dfsg1-1) ... 2023-10-01T10:02:45.095 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package ceph-mon. 2023-10-01T10:02:45.108 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../45-ceph-mon_16.2.14-44-geeb13ac9-1focal_amd64.deb ... 2023-10-01T10:02:45.153 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking ceph-mon (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:02:45.349 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package python-pastedeploy-tpl. 2023-10-01T10:02:45.363 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../38-python-pastedeploy-tpl_2.1.0-1_all.deb ... 2023-10-01T10:02:45.407 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking python-pastedeploy-tpl (2.1.0-1) ... 2023-10-01T10:02:45.744 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package python3-pastedeploy. 2023-10-01T10:02:45.758 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../39-python3-pastedeploy_2.1.0-1_all.deb ... 2023-10-01T10:02:45.802 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking python3-pastedeploy (2.1.0-1) ... 2023-10-01T10:02:46.096 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package python3-webtest. 2023-10-01T10:02:46.111 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../40-python3-webtest_2.0.32-2_all.deb ... 2023-10-01T10:02:46.154 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking python3-webtest (2.0.32-2) ... 2023-10-01T10:02:46.215 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package ceph-osd. 2023-10-01T10:02:46.229 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../46-ceph-osd_16.2.14-44-geeb13ac9-1focal_amd64.deb ... 2023-10-01T10:02:46.273 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking ceph-osd (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:02:46.449 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package python3-pecan. 2023-10-01T10:02:46.462 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../41-python3-pecan_1.3.3-3ubuntu1_all.deb ... 2023-10-01T10:02:46.499 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking python3-pecan (1.3.3-3ubuntu1) ... 2023-10-01T10:02:46.854 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package python3-werkzeug. 2023-10-01T10:02:46.867 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../42-python3-werkzeug_0.16.1+dfsg1-2ubuntu0.1_all.deb ... 2023-10-01T10:02:46.912 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking python3-werkzeug (0.16.1+dfsg1-2ubuntu0.1) ... 2023-10-01T10:02:47.265 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package libsqlite3-mod-ceph. 2023-10-01T10:02:47.279 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../43-libsqlite3-mod-ceph_16.2.14-44-geeb13ac9-1focal_amd64.deb ... 2023-10-01T10:02:47.323 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking libsqlite3-mod-ceph (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:02:47.668 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package ceph-mgr. 2023-10-01T10:02:47.683 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../44-ceph-mgr_16.2.14-44-geeb13ac9-1focal_amd64.deb ... 2023-10-01T10:02:47.726 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking ceph-mgr (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:02:48.205 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package ceph-mon. 2023-10-01T10:02:48.219 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../45-ceph-mon_16.2.14-44-geeb13ac9-1focal_amd64.deb ... 2023-10-01T10:02:48.262 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking ceph-mon (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:02:49.337 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package ceph. 2023-10-01T10:02:49.350 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../47-ceph_16.2.14-44-geeb13ac9-1focal_amd64.deb ... 2023-10-01T10:02:49.395 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking ceph (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:02:49.714 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package ceph-fuse. 2023-10-01T10:02:49.729 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../48-ceph-fuse_16.2.14-44-geeb13ac9-1focal_amd64.deb ... 2023-10-01T10:02:49.781 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking ceph-fuse (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:02:50.100 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package ceph-osd. 2023-10-01T10:02:50.113 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../46-ceph-osd_16.2.14-44-geeb13ac9-1focal_amd64.deb ... 2023-10-01T10:02:50.158 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking ceph-osd (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:02:50.184 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package ceph-mds. 2023-10-01T10:02:50.199 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../49-ceph-mds_16.2.14-44-geeb13ac9-1focal_amd64.deb ... 2023-10-01T10:02:50.242 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking ceph-mds (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:02:50.762 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package cephadm. 2023-10-01T10:02:50.778 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../50-cephadm_16.2.14-44-geeb13ac9-1focal_amd64.deb ... 2023-10-01T10:02:50.820 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking cephadm (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:02:51.123 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-jinja2. 2023-10-01T10:02:51.138 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../51-python3-jinja2_2.10.1-2_all.deb ... 2023-10-01T10:02:51.190 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-jinja2 (2.10.1-2) ... 2023-10-01T10:02:51.534 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package ceph-mgr-cephadm. 2023-10-01T10:02:51.550 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../52-ceph-mgr-cephadm_16.2.14-44-geeb13ac9-1focal_all.deb ... 2023-10-01T10:02:51.592 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking ceph-mgr-cephadm (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:02:51.937 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-repoze.lru. 2023-10-01T10:02:51.951 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../53-python3-repoze.lru_0.7-2_all.deb ... 2023-10-01T10:02:51.995 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-repoze.lru (0.7-2) ... 2023-10-01T10:02:52.248 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package libjs-underscore. 2023-10-01T10:02:52.262 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../54-libjs-underscore_1.9.1~dfsg-1ubuntu0.20.04.1_all.deb ... 2023-10-01T10:02:52.331 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking libjs-underscore (1.9.1~dfsg-1ubuntu0.20.04.1) ... 2023-10-01T10:02:52.600 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package libjs-sphinxdoc. 2023-10-01T10:02:52.615 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../55-libjs-sphinxdoc_1.8.5-7ubuntu3_all.deb ... 2023-10-01T10:02:52.658 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking libjs-sphinxdoc (1.8.5-7ubuntu3) ... 2023-10-01T10:02:52.813 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package ceph. 2023-10-01T10:02:52.827 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../47-ceph_16.2.14-44-geeb13ac9-1focal_amd64.deb ... 2023-10-01T10:02:52.871 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking ceph (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:02:52.961 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-routes. 2023-10-01T10:02:52.975 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../56-python3-routes_2.4.1-2_all.deb ... 2023-10-01T10:02:53.019 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-routes (2.4.1-2) ... 2023-10-01T10:02:53.191 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package ceph-fuse. 2023-10-01T10:02:53.206 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../48-ceph-fuse_16.2.14-44-geeb13ac9-1focal_amd64.deb ... 2023-10-01T10:02:53.249 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking ceph-fuse (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:02:53.347 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package ceph-mgr-dashboard. 2023-10-01T10:02:53.360 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../57-ceph-mgr-dashboard_16.2.14-44-geeb13ac9-1focal_all.deb ... 2023-10-01T10:02:53.405 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking ceph-mgr-dashboard (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:02:53.678 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package ceph-mds. 2023-10-01T10:02:53.691 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../49-ceph-mds_16.2.14-44-geeb13ac9-1focal_amd64.deb ... 2023-10-01T10:02:53.736 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking ceph-mds (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:02:54.289 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package cephadm. 2023-10-01T10:02:54.302 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../50-cephadm_16.2.14-44-geeb13ac9-1focal_amd64.deb ... 2023-10-01T10:02:54.338 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking cephadm (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:02:54.446 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-decorator. 2023-10-01T10:02:54.461 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../58-python3-decorator_4.4.2-0ubuntu1_all.deb ... 2023-10-01T10:02:54.504 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-decorator (4.4.2-0ubuntu1) ... 2023-10-01T10:02:54.666 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package python3-jinja2. 2023-10-01T10:02:54.757 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../51-python3-jinja2_2.10.1-2_all.deb ... 2023-10-01T10:02:54.757 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking python3-jinja2 (2.10.1-2) ... 2023-10-01T10:02:54.816 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package liblbfgsb0:amd64. 2023-10-01T10:02:54.830 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../59-liblbfgsb0_3.0+dfsg.3-7build1_amd64.deb ... 2023-10-01T10:02:54.875 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking liblbfgsb0:amd64 (3.0+dfsg.3-7build1) ... 2023-10-01T10:02:55.028 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package ceph-mgr-cephadm. 2023-10-01T10:02:55.043 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../52-ceph-mgr-cephadm_16.2.14-44-geeb13ac9-1focal_all.deb ... 2023-10-01T10:02:55.086 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking ceph-mgr-cephadm (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:02:55.201 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-scipy. 2023-10-01T10:02:55.215 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../60-python3-scipy_1.3.3-3build1_amd64.deb ... 2023-10-01T10:02:55.259 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-scipy (1.3.3-3build1) ... 2023-10-01T10:02:55.431 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package python3-repoze.lru. 2023-10-01T10:02:55.446 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../53-python3-repoze.lru_0.7-2_all.deb ... 2023-10-01T10:02:55.489 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking python3-repoze.lru (0.7-2) ... 2023-10-01T10:02:55.708 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package libjs-underscore. 2023-10-01T10:02:55.723 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../54-libjs-underscore_1.9.1~dfsg-1ubuntu0.20.04.1_all.deb ... 2023-10-01T10:02:55.758 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking libjs-underscore (1.9.1~dfsg-1ubuntu0.20.04.1) ... 2023-10-01T10:02:56.011 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package libjs-sphinxdoc. 2023-10-01T10:02:56.024 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../55-libjs-sphinxdoc_1.8.5-7ubuntu3_all.deb ... 2023-10-01T10:02:56.069 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking libjs-sphinxdoc (1.8.5-7ubuntu3) ... 2023-10-01T10:02:56.380 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package python3-routes. 2023-10-01T10:02:56.393 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../56-python3-routes_2.4.1-2_all.deb ... 2023-10-01T10:02:56.438 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking python3-routes (2.4.1-2) ... 2023-10-01T10:02:56.501 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-sklearn-lib. 2023-10-01T10:02:56.515 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../61-python3-sklearn-lib_0.22.2.post1+dfsg-5_amd64.deb ... 2023-10-01T10:02:56.559 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-sklearn-lib (0.22.2.post1+dfsg-5) ... 2023-10-01T10:02:56.749 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package ceph-mgr-dashboard. 2023-10-01T10:02:56.763 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../57-ceph-mgr-dashboard_16.2.14-44-geeb13ac9-1focal_all.deb ... 2023-10-01T10:02:56.807 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking ceph-mgr-dashboard (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:02:57.105 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-joblib. 2023-10-01T10:02:57.119 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../62-python3-joblib_0.14.0-3_all.deb ... 2023-10-01T10:02:57.162 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-joblib (0.14.0-3) ... 2023-10-01T10:02:57.533 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-sklearn. 2023-10-01T10:02:57.547 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../63-python3-sklearn_0.22.2.post1+dfsg-5_all.deb ... 2023-10-01T10:02:57.592 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-sklearn (0.22.2.post1+dfsg-5) ... 2023-10-01T10:02:57.857 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package python3-decorator. 2023-10-01T10:02:57.872 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../58-python3-decorator_4.4.2-0ubuntu1_all.deb ... 2023-10-01T10:02:57.915 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking python3-decorator (4.4.2-0ubuntu1) ... 2023-10-01T10:02:58.227 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package liblbfgsb0:amd64. 2023-10-01T10:02:58.241 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../59-liblbfgsb0_3.0+dfsg.3-7build1_amd64.deb ... 2023-10-01T10:02:58.285 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking liblbfgsb0:amd64 (3.0+dfsg.3-7build1) ... 2023-10-01T10:02:58.368 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package ceph-mgr-diskprediction-local. 2023-10-01T10:02:58.381 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../64-ceph-mgr-diskprediction-local_16.2.14-44-geeb13ac9-1focal_all.deb ... 2023-10-01T10:02:58.426 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking ceph-mgr-diskprediction-local (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:02:58.588 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package python3-scipy. 2023-10-01T10:02:58.603 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../60-python3-scipy_1.3.3-3build1_amd64.deb ... 2023-10-01T10:02:58.656 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking python3-scipy (1.3.3-3build1) ... 2023-10-01T10:02:59.510 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-cachetools. 2023-10-01T10:02:59.519 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../65-python3-cachetools_4.0.0-1_all.deb ... 2023-10-01T10:02:59.560 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-cachetools (4.0.0-1) ... 2023-10-01T10:02:59.846 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-pyasn1. 2023-10-01T10:02:59.854 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../66-python3-pyasn1_0.4.2-3build1_all.deb ... 2023-10-01T10:02:59.896 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-pyasn1 (0.4.2-3build1) ... 2023-10-01T10:02:59.988 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package python3-sklearn-lib. 2023-10-01T10:03:00.003 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../61-python3-sklearn-lib_0.22.2.post1+dfsg-5_amd64.deb ... 2023-10-01T10:03:00.046 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking python3-sklearn-lib (0.22.2.post1+dfsg-5) ... 2023-10-01T10:03:00.232 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-pyasn1-modules. 2023-10-01T10:03:00.241 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../67-python3-pyasn1-modules_0.2.1-0.2build1_all.deb ... 2023-10-01T10:03:00.282 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-pyasn1-modules (0.2.1-0.2build1) ... 2023-10-01T10:03:00.593 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-rsa. 2023-10-01T10:03:00.599 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package python3-joblib. 2023-10-01T10:03:00.608 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../68-python3-rsa_4.0-3_all.deb ... 2023-10-01T10:03:00.613 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../62-python3-joblib_0.14.0-3_all.deb ... 2023-10-01T10:03:00.651 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-rsa (4.0-3) ... 2023-10-01T10:03:00.657 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking python3-joblib (0.14.0-3) ... 2023-10-01T10:03:00.963 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-google-auth. 2023-10-01T10:03:00.976 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../69-python3-google-auth_1.5.1-2_all.deb ... 2023-10-01T10:03:01.012 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-google-auth (1.5.1-2) ... 2023-10-01T10:03:01.019 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package python3-sklearn. 2023-10-01T10:03:01.033 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../63-python3-sklearn_0.22.2.post1+dfsg-5_all.deb ... 2023-10-01T10:03:01.077 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking python3-sklearn (0.22.2.post1+dfsg-5) ... 2023-10-01T10:03:01.341 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-blinker. 2023-10-01T10:03:01.355 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../70-python3-blinker_1.4+dfsg1-0.3ubuntu1_all.deb ... 2023-10-01T10:03:01.399 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-blinker (1.4+dfsg1-0.3ubuntu1) ... 2023-10-01T10:03:01.718 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-oauthlib. 2023-10-01T10:03:01.734 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../71-python3-oauthlib_3.1.0-1ubuntu2_all.deb ... 2023-10-01T10:03:01.784 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-oauthlib (3.1.0-1ubuntu2) ... 2023-10-01T10:03:01.794 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package ceph-mgr-diskprediction-local. 2023-10-01T10:03:01.808 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../64-ceph-mgr-diskprediction-local_16.2.14-44-geeb13ac9-1focal_all.deb ... 2023-10-01T10:03:01.852 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking ceph-mgr-diskprediction-local (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:02.105 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-requests-oauthlib. 2023-10-01T10:03:02.118 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../72-python3-requests-oauthlib_1.0.0-1.1build1_all.deb ... 2023-10-01T10:03:02.162 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-requests-oauthlib (1.0.0-1.1build1) ... 2023-10-01T10:03:02.482 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-websocket. 2023-10-01T10:03:02.498 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../73-python3-websocket_0.53.0-2ubuntu1_all.deb ... 2023-10-01T10:03:02.540 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-websocket (0.53.0-2ubuntu1) ... 2023-10-01T10:03:02.860 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-kubernetes. 2023-10-01T10:03:02.874 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../74-python3-kubernetes_7.0.0~a1-2_all.deb ... 2023-10-01T10:03:02.918 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-kubernetes (7.0.0~a1-2) ... 2023-10-01T10:03:02.997 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package python3-cachetools. 2023-10-01T10:03:03.005 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../65-python3-cachetools_4.0.0-1_all.deb ... 2023-10-01T10:03:03.046 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking python3-cachetools (4.0.0-1) ... 2023-10-01T10:03:03.374 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package python3-pyasn1. 2023-10-01T10:03:03.388 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../66-python3-pyasn1_0.4.2-3build1_all.deb ... 2023-10-01T10:03:03.432 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking python3-pyasn1 (0.4.2-3build1) ... 2023-10-01T10:03:03.439 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package ceph-mgr-k8sevents. 2023-10-01T10:03:03.455 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../75-ceph-mgr-k8sevents_16.2.14-44-geeb13ac9-1focal_all.deb ... 2023-10-01T10:03:03.497 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking ceph-mgr-k8sevents (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:03.744 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package python3-pyasn1-modules. 2023-10-01T10:03:03.759 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../67-python3-pyasn1-modules_0.2.1-0.2build1_all.deb ... 2023-10-01T10:03:03.794 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking python3-pyasn1-modules (0.2.1-0.2build1) ... 2023-10-01T10:03:03.825 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package libonig5:amd64. 2023-10-01T10:03:03.839 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../76-libonig5_6.9.4-1_amd64.deb ... 2023-10-01T10:03:03.884 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking libonig5:amd64 (6.9.4-1) ... 2023-10-01T10:03:04.088 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package python3-rsa. 2023-10-01T10:03:04.102 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../68-python3-rsa_4.0-3_all.deb ... 2023-10-01T10:03:04.146 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking python3-rsa (4.0-3) ... 2023-10-01T10:03:04.228 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package libjq1:amd64. 2023-10-01T10:03:04.241 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../77-libjq1_1.6-1ubuntu0.20.04.1_amd64.deb ... 2023-10-01T10:03:04.286 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking libjq1:amd64 (1.6-1ubuntu0.20.04.1) ... 2023-10-01T10:03:04.441 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package python3-google-auth. 2023-10-01T10:03:04.455 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../69-python3-google-auth_1.5.1-2_all.deb ... 2023-10-01T10:03:04.507 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking python3-google-auth (1.5.1-2) ... 2023-10-01T10:03:04.836 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package python3-blinker. 2023-10-01T10:03:04.845 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../70-python3-blinker_1.4+dfsg1-0.3ubuntu1_all.deb ... 2023-10-01T10:03:04.872 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package jq. 2023-10-01T10:03:04.887 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../78-jq_1.6-1ubuntu0.20.04.1_amd64.deb ... 2023-10-01T10:03:04.921 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking jq (1.6-1ubuntu0.20.04.1) ... 2023-10-01T10:03:04.931 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking python3-blinker (1.4+dfsg1-0.3ubuntu1) ... 2023-10-01T10:03:05.157 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package socat. 2023-10-01T10:03:05.171 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../79-socat_1.7.3.3-2_amd64.deb ... 2023-10-01T10:03:05.224 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking socat (1.7.3.3-2) ... 2023-10-01T10:03:05.535 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package xmlstarlet. 2023-10-01T10:03:05.548 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package python3-oauthlib. 2023-10-01T10:03:05.553 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../80-xmlstarlet_1.6.1-2build1_amd64.deb ... 2023-10-01T10:03:05.563 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../71-python3-oauthlib_3.1.0-1ubuntu2_all.deb ... 2023-10-01T10:03:05.594 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking xmlstarlet (1.6.1-2build1) ... 2023-10-01T10:03:05.605 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking python3-oauthlib (3.1.0-1ubuntu2) ... 2023-10-01T10:03:06.492 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package ceph-test. 2023-10-01T10:03:06.492 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../81-ceph-test_16.2.14-44-geeb13ac9-1focal_amd64.deb ... 2023-10-01T10:03:06.492 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking ceph-test (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:06.493 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package python3-requests-oauthlib. 2023-10-01T10:03:06.493 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../72-python3-requests-oauthlib_1.0.0-1.1build1_all.deb ... 2023-10-01T10:03:06.493 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking python3-requests-oauthlib (1.0.0-1.1build1) ... 2023-10-01T10:03:06.493 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package python3-websocket. 2023-10-01T10:03:06.493 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../73-python3-websocket_0.53.0-2ubuntu1_all.deb ... 2023-10-01T10:03:06.493 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking python3-websocket (0.53.0-2ubuntu1) ... 2023-10-01T10:03:06.657 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package python3-kubernetes. 2023-10-01T10:03:06.669 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../74-python3-kubernetes_7.0.0~a1-2_all.deb ... 2023-10-01T10:03:06.714 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking python3-kubernetes (7.0.0~a1-2) ... 2023-10-01T10:03:07.277 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package ceph-mgr-k8sevents. 2023-10-01T10:03:07.291 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../75-ceph-mgr-k8sevents_16.2.14-44-geeb13ac9-1focal_all.deb ... 2023-10-01T10:03:07.335 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking ceph-mgr-k8sevents (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:07.654 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package libonig5:amd64. 2023-10-01T10:03:07.668 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../76-libonig5_6.9.4-1_amd64.deb ... 2023-10-01T10:03:07.712 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking libonig5:amd64 (6.9.4-1) ... 2023-10-01T10:03:08.074 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package libjq1:amd64. 2023-10-01T10:03:08.087 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../77-libjq1_1.6-1ubuntu0.20.04.1_amd64.deb ... 2023-10-01T10:03:08.132 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking libjq1:amd64 (1.6-1ubuntu0.20.04.1) ... 2023-10-01T10:03:08.401 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package jq. 2023-10-01T10:03:08.414 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../78-jq_1.6-1ubuntu0.20.04.1_amd64.deb ... 2023-10-01T10:03:08.459 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking jq (1.6-1ubuntu0.20.04.1) ... 2023-10-01T10:03:08.720 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package socat. 2023-10-01T10:03:08.735 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../79-socat_1.7.3.3-2_amd64.deb ... 2023-10-01T10:03:08.778 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking socat (1.7.3.3-2) ... 2023-10-01T10:03:09.115 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package xmlstarlet. 2023-10-01T10:03:09.129 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../80-xmlstarlet_1.6.1-2build1_amd64.deb ... 2023-10-01T10:03:09.173 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking xmlstarlet (1.6.1-2build1) ... 2023-10-01T10:03:09.613 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package ceph-test. 2023-10-01T10:03:09.626 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../81-ceph-test_16.2.14-44-geeb13ac9-1focal_amd64.deb ... 2023-10-01T10:03:09.679 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking ceph-test (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:10.137 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package libcephfs-dev. 2023-10-01T10:03:10.151 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../82-libcephfs-dev_16.2.14-44-geeb13ac9-1focal_amd64.deb ... 2023-10-01T10:03:10.194 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking libcephfs-dev (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:10.614 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package nvme-cli. 2023-10-01T10:03:10.630 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../83-nvme-cli_1.9-1ubuntu0.1_amd64.deb ... 2023-10-01T10:03:10.689 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking nvme-cli (1.9-1ubuntu0.1) ... 2023-10-01T10:03:11.062 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-atomicwrites. 2023-10-01T10:03:11.078 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../84-python3-atomicwrites_1.1.5-2build1_all.deb ... 2023-10-01T10:03:11.120 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-atomicwrites (1.1.5-2build1) ... 2023-10-01T10:03:11.423 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-attr. 2023-10-01T10:03:11.437 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../85-python3-attr_19.3.0-2_all.deb ... 2023-10-01T10:03:11.481 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-attr (19.3.0-2) ... 2023-10-01T10:03:11.800 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-webencodings. 2023-10-01T10:03:11.814 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../86-python3-webencodings_0.5.1-1ubuntu1_all.deb ... 2023-10-01T10:03:11.858 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-webencodings (0.5.1-1ubuntu1) ... 2023-10-01T10:03:12.169 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-html5lib. 2023-10-01T10:03:12.185 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../87-python3-html5lib_1.0.1-2_all.deb ... 2023-10-01T10:03:12.227 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-html5lib (1.0.1-2) ... 2023-10-01T10:03:12.589 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-lxml:amd64. 2023-10-01T10:03:12.603 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../88-python3-lxml_4.5.0-1ubuntu0.5_amd64.deb ... 2023-10-01T10:03:12.647 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-lxml:amd64 (4.5.0-1ubuntu0.5) ... 2023-10-01T10:03:13.117 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-packaging. 2023-10-01T10:03:13.126 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../89-python3-packaging_20.3-1_all.deb ... 2023-10-01T10:03:13.166 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-packaging (20.3-1) ... 2023-10-01T10:03:13.486 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-pastescript. 2023-10-01T10:03:13.496 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../90-python3-pastescript_2.0.2-4_all.deb ... 2023-10-01T10:03:13.536 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-pastescript (2.0.2-4) ... 2023-10-01T10:03:13.857 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-pluggy. 2023-10-01T10:03:13.871 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../91-python3-pluggy_0.13.0-2_all.deb ... 2023-10-01T10:03:13.915 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-pluggy (0.13.0-2) ... 2023-10-01T10:03:13.958 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package libcephfs-dev. 2023-10-01T10:03:13.972 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../82-libcephfs-dev_16.2.14-44-geeb13ac9-1focal_amd64.deb ... 2023-10-01T10:03:14.016 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking libcephfs-dev (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:14.221 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-psutil. 2023-10-01T10:03:14.236 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../92-python3-psutil_5.5.1-1ubuntu4_amd64.deb ... 2023-10-01T10:03:14.279 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-psutil (5.5.1-1ubuntu4) ... 2023-10-01T10:03:14.400 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package nvme-cli. 2023-10-01T10:03:14.416 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../83-nvme-cli_1.9-1ubuntu0.1_amd64.deb ... 2023-10-01T10:03:14.474 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking nvme-cli (1.9-1ubuntu0.1) ... 2023-10-01T10:03:14.657 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-py. 2023-10-01T10:03:14.671 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../93-python3-py_1.8.1-1ubuntu0.1_all.deb ... 2023-10-01T10:03:14.715 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-py (1.8.1-1ubuntu0.1) ... 2023-10-01T10:03:14.832 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package python3-atomicwrites. 2023-10-01T10:03:14.847 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../84-python3-atomicwrites_1.1.5-2build1_all.deb ... 2023-10-01T10:03:14.890 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking python3-atomicwrites (1.1.5-2build1) ... 2023-10-01T10:03:15.035 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-pyinotify. 2023-10-01T10:03:15.049 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../94-python3-pyinotify_0.9.6-1.2ubuntu1_all.deb ... 2023-10-01T10:03:15.092 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-pyinotify (0.9.6-1.2ubuntu1) ... 2023-10-01T10:03:15.176 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package python3-attr. 2023-10-01T10:03:15.192 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../85-python3-attr_19.3.0-2_all.deb ... 2023-10-01T10:03:15.234 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking python3-attr (19.3.0-2) ... 2023-10-01T10:03:15.395 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-wcwidth. 2023-10-01T10:03:15.411 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../95-python3-wcwidth_0.1.8+dfsg1-3_all.deb ... 2023-10-01T10:03:15.461 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-wcwidth (0.1.8+dfsg1-3) ... 2023-10-01T10:03:15.570 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package python3-webencodings. 2023-10-01T10:03:15.585 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../86-python3-webencodings_0.5.1-1ubuntu1_all.deb ... 2023-10-01T10:03:15.628 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking python3-webencodings (0.5.1-1ubuntu1) ... 2023-10-01T10:03:15.747 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-pytest. 2023-10-01T10:03:15.763 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../96-python3-pytest_4.6.9-1_all.deb ... 2023-10-01T10:03:15.805 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-pytest (4.6.9-1) ... 2023-10-01T10:03:15.948 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package python3-html5lib. 2023-10-01T10:03:15.962 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../87-python3-html5lib_1.0.1-2_all.deb ... 2023-10-01T10:03:16.006 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking python3-html5lib (1.0.1-2) ... 2023-10-01T10:03:16.150 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package python3-simplejson. 2023-10-01T10:03:16.166 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../97-python3-simplejson_3.16.0-2ubuntu2_amd64.deb ... 2023-10-01T10:03:16.208 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking python3-simplejson (3.16.0-2ubuntu2) ... 2023-10-01T10:03:16.351 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package python3-lxml:amd64. 2023-10-01T10:03:16.367 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../88-python3-lxml_4.5.0-1ubuntu0.5_amd64.deb ... 2023-10-01T10:03:16.409 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking python3-lxml:amd64 (4.5.0-1ubuntu0.5) ... 2023-10-01T10:03:16.612 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package radosgw. 2023-10-01T10:03:16.627 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../98-radosgw_16.2.14-44-geeb13ac9-1focal_amd64.deb ... 2023-10-01T10:03:16.670 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking radosgw (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:16.879 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package python3-packaging. 2023-10-01T10:03:16.893 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../89-python3-packaging_20.3-1_all.deb ... 2023-10-01T10:03:16.937 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking python3-packaging (20.3-1) ... 2023-10-01T10:03:17.234 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package python3-pastescript. 2023-10-01T10:03:17.248 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../90-python3-pastescript_2.0.2-4_all.deb ... 2023-10-01T10:03:17.291 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking python3-pastescript (2.0.2-4) ... 2023-10-01T10:03:17.597 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package python3-pluggy. 2023-10-01T10:03:17.612 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../91-python3-pluggy_0.13.0-2_all.deb ... 2023-10-01T10:03:17.655 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking python3-pluggy (0.13.0-2) ... 2023-10-01T10:03:17.857 INFO:teuthology.orchestra.run.smithi173.stdout:Selecting previously unselected package rbd-fuse. 2023-10-01T10:03:17.872 INFO:teuthology.orchestra.run.smithi173.stdout:Preparing to unpack .../99-rbd-fuse_16.2.14-44-geeb13ac9-1focal_amd64.deb ... 2023-10-01T10:03:17.915 INFO:teuthology.orchestra.run.smithi173.stdout:Unpacking rbd-fuse (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:17.975 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package python3-psutil. 2023-10-01T10:03:17.991 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../92-python3-psutil_5.5.1-1ubuntu4_amd64.deb ... 2023-10-01T10:03:18.033 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking python3-psutil (5.5.1-1ubuntu4) ... 2023-10-01T10:03:18.226 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-attr (19.3.0-2) ... 2023-10-01T10:03:18.360 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package python3-py. 2023-10-01T10:03:18.375 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../93-python3-py_1.8.1-1ubuntu0.1_all.deb ... 2023-10-01T10:03:18.418 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking python3-py (1.8.1-1ubuntu0.1) ... 2023-10-01T10:03:18.511 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up libleveldb1d:amd64 (1.22-3ubuntu2) ... 2023-10-01T10:03:18.637 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up librabbitmq4:amd64 (0.10.0-1) ... 2023-10-01T10:03:18.746 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package python3-pyinotify. 2023-10-01T10:03:18.760 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../94-python3-pyinotify_0.9.6-1.2ubuntu1_all.deb ... 2023-10-01T10:03:18.763 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up nvme-cli (1.9-1ubuntu0.1) ... 2023-10-01T10:03:18.804 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking python3-pyinotify (0.9.6-1.2ubuntu1) ... 2023-10-01T10:03:18.956 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up cephadm (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:19.123 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package python3-wcwidth. 2023-10-01T10:03:19.140 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../95-python3-wcwidth_0.1.8+dfsg1-3_all.deb ... 2023-10-01T10:03:19.182 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking python3-wcwidth (0.1.8+dfsg1-3) ... 2023-10-01T10:03:19.468 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package python3-pytest. 2023-10-01T10:03:19.483 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../96-python3-pytest_4.6.9-1_all.deb ... 2023-10-01T10:03:19.526 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking python3-pytest (4.6.9-1) ... 2023-10-01T10:03:19.558 INFO:teuthology.orchestra.run.smithi173.stdout:Adding system user cephadm....done 2023-10-01T10:03:19.712 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-waitress (1.4.1-1ubuntu0.1) ... 2023-10-01T10:03:19.855 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package python3-simplejson. 2023-10-01T10:03:19.869 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../97-python3-simplejson_3.16.0-2ubuntu2_amd64.deb ... 2023-10-01T10:03:19.913 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking python3-simplejson (3.16.0-2ubuntu2) ... 2023-10-01T10:03:20.063 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-repoze.lru (0.7-2) ... 2023-10-01T10:03:20.348 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up liboath0 (2.6.1-1.3) ... 2023-10-01T10:03:20.383 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package radosgw. 2023-10-01T10:03:20.397 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../98-radosgw_16.2.14-44-geeb13ac9-1focal_amd64.deb ... 2023-10-01T10:03:20.441 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking radosgw (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:20.473 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-py (1.8.1-1ubuntu0.1) ... 2023-10-01T10:03:20.800 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-joblib (0.14.0-3) ... 2023-10-01T10:03:21.185 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-jwt (1.7.1-2ubuntu2.1) ... 2023-10-01T10:03:21.462 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-cachetools (4.0.0-1) ... 2023-10-01T10:03:21.620 INFO:teuthology.orchestra.run.smithi078.stdout:Selecting previously unselected package rbd-fuse. 2023-10-01T10:03:21.633 INFO:teuthology.orchestra.run.smithi078.stdout:Preparing to unpack .../99-rbd-fuse_16.2.14-44-geeb13ac9-1focal_amd64.deb ... 2023-10-01T10:03:21.669 INFO:teuthology.orchestra.run.smithi078.stdout:Unpacking rbd-fuse (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:21.763 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-bcrypt (3.1.7-2ubuntu1) ... 2023-10-01T10:03:21.979 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up python3-attr (19.3.0-2) ... 2023-10-01T10:03:22.064 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-pyinotify (0.9.6-1.2ubuntu1) ... 2023-10-01T10:03:22.276 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up libleveldb1d:amd64 (1.22-3ubuntu2) ... 2023-10-01T10:03:22.348 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-ceph-argparse (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:22.402 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up librabbitmq4:amd64 (0.10.0-1) ... 2023-10-01T10:03:22.528 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up nvme-cli (1.9-1ubuntu0.1) ... 2023-10-01T10:03:22.641 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-sklearn-lib (0.22.2.post1+dfsg-5) ... 2023-10-01T10:03:22.721 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up cephadm (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:22.766 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-markupsafe (1.1.0-1build2) ... 2023-10-01T10:03:23.062 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-webencodings (0.5.1-1ubuntu1) ... 2023-10-01T10:03:23.331 INFO:teuthology.orchestra.run.smithi078.stdout:Adding system user cephadm....done 2023-10-01T10:03:23.344 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-psutil (5.5.1-1ubuntu4) ... 2023-10-01T10:03:23.485 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up python3-waitress (1.4.1-1ubuntu0.1) ... 2023-10-01T10:03:23.713 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-atomicwrites (1.1.5-2build1) ... 2023-10-01T10:03:23.828 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up python3-repoze.lru (0.7-2) ... 2023-10-01T10:03:23.989 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-simplejson (3.16.0-2ubuntu2) ... 2023-10-01T10:03:24.129 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up liboath0 (2.6.1-1.3) ... 2023-10-01T10:03:24.263 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up python3-py (1.8.1-1ubuntu0.1) ... 2023-10-01T10:03:24.315 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-decorator (4.4.2-0ubuntu1) ... 2023-10-01T10:03:24.573 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up python3-joblib (0.14.0-3) ... 2023-10-01T10:03:24.608 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up libjaeger (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:24.734 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-jinja2 (2.10.1-2) ... 2023-10-01T10:03:25.000 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up python3-jwt (1.7.1-2ubuntu2.1) ... 2023-10-01T10:03:25.060 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-tempita (0.5.2-6) ... 2023-10-01T10:03:25.268 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up python3-cachetools (4.0.0-1) ... 2023-10-01T10:03:25.386 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-packaging (20.3-1) ... 2023-10-01T10:03:25.569 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up python3-bcrypt (3.1.7-2ubuntu1) ... 2023-10-01T10:03:25.670 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python-pastedeploy-tpl (2.1.0-1) ... 2023-10-01T10:03:25.796 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-wcwidth (0.1.8+dfsg1-3) ... 2023-10-01T10:03:25.845 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up python3-pyinotify (0.9.6-1.2ubuntu1) ... 2023-10-01T10:03:26.106 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-paste (3.4.0+dfsg1-1) ... 2023-10-01T10:03:26.141 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up python3-ceph-argparse (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:26.432 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up python3-sklearn-lib (0.22.2.post1+dfsg-5) ... 2023-10-01T10:03:26.507 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-werkzeug (0.16.1+dfsg1-2ubuntu0.1) ... 2023-10-01T10:03:26.558 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up python3-markupsafe (1.1.0-1build2) ... 2023-10-01T10:03:26.843 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up python3-webencodings (0.5.1-1ubuntu1) ... 2023-10-01T10:03:27.808 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up python3-psutil (5.5.1-1ubuntu4) ... 2023-10-01T10:03:27.809 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up python3-atomicwrites (1.1.5-2build1) ... 2023-10-01T10:03:27.809 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up python3-simplejson (3.16.0-2ubuntu2) ... 2023-10-01T10:03:27.809 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-cryptography (2.8-3ubuntu0.1) ... 2023-10-01T10:03:27.809 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up socat (1.7.3.3-2) ... 2023-10-01T10:03:27.809 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-ceph-common (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:27.809 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-html5lib (1.0.1-2) ... 2023-10-01T10:03:27.989 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up librdkafka1:amd64 (1.2.1-1ubuntu1) ... 2023-10-01T10:03:28.097 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up python3-decorator (4.4.2-0ubuntu1) ... 2023-10-01T10:03:28.115 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up xmlstarlet (1.6.1-2build1) ... 2023-10-01T10:03:28.241 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-pluggy (0.13.0-2) ... 2023-10-01T10:03:28.373 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up libjaeger (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:28.499 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up python3-jinja2 (2.10.1-2) ... 2023-10-01T10:03:28.533 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-lxml:amd64 (4.5.0-1ubuntu0.5) ... 2023-10-01T10:03:28.825 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up python3-tempita (0.5.2-6) ... 2023-10-01T10:03:28.852 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up liblua5.3-0:amd64 (5.3.3-1.1ubuntu2) ... 2023-10-01T10:03:28.978 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-pyasn1 (0.4.2-3build1) ... 2023-10-01T10:03:29.126 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up python3-packaging (20.3-1) ... 2023-10-01T10:03:29.304 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-singledispatch (3.4.0.3-2build1) ... 2023-10-01T10:03:29.419 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up python-pastedeploy-tpl (2.1.0-1) ... 2023-10-01T10:03:29.545 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up python3-wcwidth (0.1.8+dfsg1-3) ... 2023-10-01T10:03:29.588 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-logutils (0.3.3-7) ... 2023-10-01T10:03:29.821 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up python3-paste (3.4.0+dfsg1-1) ... 2023-10-01T10:03:29.881 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up liblttng-ust0:amd64 (2.11.0-1) ... 2023-10-01T10:03:30.007 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-simplegeneric (0.8.1-3) ... 2023-10-01T10:03:30.222 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up python3-werkzeug (0.16.1+dfsg1-2ubuntu0.1) ... 2023-10-01T10:03:30.300 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-prettytable (0.7.2-5) ... 2023-10-01T10:03:30.574 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up python3-cryptography (2.8-3ubuntu0.1) ... 2023-10-01T10:03:30.592 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-soupsieve (1.9.5+dfsg-1) ... 2023-10-01T10:03:30.885 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-websocket (0.53.0-2ubuntu1) ... 2023-10-01T10:03:30.926 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up socat (1.7.3.3-2) ... 2023-10-01T10:03:31.052 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up python3-ceph-common (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:31.289 INFO:teuthology.orchestra.run.smithi173.stdout:update-alternatives: using /usr/bin/python3-wsdump to provide /usr/bin/wsdump (wsdump) in auto mode 2023-10-01T10:03:31.361 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up python3-html5lib (1.0.1-2) ... 2023-10-01T10:03:31.529 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up libonig5:amd64 (6.9.4-1) ... 2023-10-01T10:03:31.655 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-mako (1.1.0+ds1-1ubuntu2.1) ... 2023-10-01T10:03:31.704 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up librdkafka1:amd64 (1.2.1-1ubuntu1) ... 2023-10-01T10:03:31.830 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up xmlstarlet (1.6.1-2build1) ... 2023-10-01T10:03:31.956 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up python3-pluggy (0.13.0-2) ... 2023-10-01T10:03:31.973 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-cherrypy3 (8.9.1-6) ... 2023-10-01T10:03:32.250 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up python3-lxml:amd64 (4.5.0-1ubuntu0.5) ... 2023-10-01T10:03:32.333 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up libjs-underscore (1.9.1~dfsg-1ubuntu0.20.04.1) ... 2023-10-01T10:03:32.459 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up liblbfgsb0:amd64 (3.0+dfsg.3-7build1) ... 2023-10-01T10:03:32.534 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up liblua5.3-0:amd64 (5.3.3-1.1ubuntu2) ... 2023-10-01T10:03:32.585 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-webob (1:1.8.5-2) ... 2023-10-01T10:03:32.660 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up python3-pyasn1 (0.4.2-3build1) ... 2023-10-01T10:03:32.912 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-blinker (1.4+dfsg1-0.3ubuntu1) ... 2023-10-01T10:03:32.986 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up python3-singledispatch (3.4.0.3-2build1) ... 2023-10-01T10:03:33.188 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up librados2 (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:33.280 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up python3-logutils (0.3.3-7) ... 2023-10-01T10:03:33.316 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-scipy (1.3.3-3build1) ... 2023-10-01T10:03:33.606 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up liblttng-ust0:amd64 (2.11.0-1) ... 2023-10-01T10:03:33.732 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up python3-simplegeneric (0.8.1-3) ... 2023-10-01T10:03:34.024 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up python3-prettytable (0.7.2-5) ... 2023-10-01T10:03:34.300 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up python3-soupsieve (1.9.5+dfsg-1) ... 2023-10-01T10:03:34.593 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up python3-websocket (0.53.0-2ubuntu1) ... 2023-10-01T10:03:34.811 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up librgw2 (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:34.937 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up libsqlite3-mod-ceph (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:34.970 INFO:teuthology.orchestra.run.smithi078.stdout:update-alternatives: using /usr/bin/python3-wsdump to provide /usr/bin/wsdump (wsdump) in auto mode 2023-10-01T10:03:35.063 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up libjq1:amd64 (1.6-1ubuntu0.20.04.1) ... 2023-10-01T10:03:35.198 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-pytest (4.6.9-1) ... 2023-10-01T10:03:35.204 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up libonig5:amd64 (6.9.4-1) ... 2023-10-01T10:03:35.338 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up python3-mako (1.1.0+ds1-1ubuntu2.1) ... 2023-10-01T10:03:35.766 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-openssl (19.0.0-1build1) ... 2023-10-01T10:03:35.849 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up python3-cherrypy3 (8.9.1-6) ... 2023-10-01T10:03:36.067 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-pastedeploy (2.1.0-1) ... 2023-10-01T10:03:36.360 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-bs4 (4.8.2-1) ... 2023-10-01T10:03:36.517 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up libjs-underscore (1.9.1~dfsg-1ubuntu0.20.04.1) ... 2023-10-01T10:03:36.643 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up liblbfgsb0:amd64 (3.0+dfsg.3-7build1) ... 2023-10-01T10:03:36.694 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up libcephfs2 (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:36.769 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up python3-webob (1:1.8.5-2) ... 2023-10-01T10:03:36.820 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-oauthlib (3.1.0-1ubuntu2) ... 2023-10-01T10:03:37.095 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up python3-blinker (1.4+dfsg1-0.3ubuntu1) ... 2023-10-01T10:03:37.149 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up libradosstriper1 (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:37.286 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-pyasn1-modules (0.2.1-0.2build1) ... 2023-10-01T10:03:37.379 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up librados2 (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:37.539 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up python3-scipy (1.3.3-3build1) ... 2023-10-01T10:03:37.603 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-sklearn (0.22.2.post1+dfsg-5) ... 2023-10-01T10:03:38.633 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up librbd1 (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:38.759 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up libjs-sphinxdoc (1.8.5-7ubuntu3) ... 2023-10-01T10:03:38.893 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up jq (1.6-1ubuntu0.20.04.1) ... 2023-10-01T10:03:39.011 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up librgw2 (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:39.036 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-rsa (4.0-3) ... 2023-10-01T10:03:39.137 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up libsqlite3-mod-ceph (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:39.264 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up libjq1:amd64 (1.6-1ubuntu0.20.04.1) ... 2023-10-01T10:03:39.329 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-webtest (2.0.32-2) ... 2023-10-01T10:03:39.390 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up python3-pytest (4.6.9-1) ... 2023-10-01T10:03:39.638 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up ceph-mgr-modules-core (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:39.750 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up python3-openssl (19.0.0-1build1) ... 2023-10-01T10:03:39.764 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up ceph-fuse (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:39.979 INFO:teuthology.orchestra.run.smithi173.stdout:Created symlink /etc/systemd/system/remote-fs.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2023-10-01T10:03:39.979 INFO:teuthology.orchestra.run.smithi173.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2023-10-01T10:03:40.063 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up python3-pastedeploy (2.1.0-1) ... 2023-10-01T10:03:40.356 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up python3-bs4 (4.8.2-1) ... 2023-10-01T10:03:40.433 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up libcephfs-dev (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:40.558 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-requests-oauthlib (1.0.0-1.1build1) ... 2023-10-01T10:03:40.699 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up libcephfs2 (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:40.800 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up python3-oauthlib (3.1.0-1ubuntu2) ... 2023-10-01T10:03:40.851 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-pastescript (2.0.2-4) ... 2023-10-01T10:03:41.126 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up libradosstriper1 (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:41.161 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-pecan (1.3.3-3ubuntu1) ... 2023-10-01T10:03:41.252 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up python3-pyasn1-modules (0.2.1-0.2build1) ... 2023-10-01T10:03:41.541 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-rados (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:41.579 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up python3-sklearn (0.22.2.post1+dfsg-5) ... 2023-10-01T10:03:41.667 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-rbd (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:41.793 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up rbd-fuse (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:41.928 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-routes (2.4.1-2) ... 2023-10-01T10:03:42.230 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-google-auth (1.5.1-2) ... 2023-10-01T10:03:42.616 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-rgw (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:42.626 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up librbd1 (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:42.745 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-cephfs (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:42.753 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up libjs-sphinxdoc (1.8.5-7ubuntu3) ... 2023-10-01T10:03:42.879 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up jq (1.6-1ubuntu0.20.04.1) ... 2023-10-01T10:03:43.004 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up python3-rsa (4.0-3) ... 2023-10-01T10:03:43.045 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up python3-kubernetes (7.0.0~a1-2) ... 2023-10-01T10:03:43.306 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up python3-webtest (2.0.32-2) ... 2023-10-01T10:03:43.599 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up ceph-mgr-modules-core (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:43.720 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up ceph-fuse (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:43.936 INFO:teuthology.orchestra.run.smithi078.stdout:Created symlink /etc/systemd/system/remote-fs.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2023-10-01T10:03:43.936 INFO:teuthology.orchestra.run.smithi078.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2023-10-01T10:03:43.982 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up ceph-common (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:44.413 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up libcephfs-dev (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:44.526 INFO:teuthology.orchestra.run.smithi173.stdout:Adding group ceph....done 2023-10-01T10:03:44.547 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up python3-requests-oauthlib (1.0.0-1.1build1) ... 2023-10-01T10:03:44.841 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up python3-pastescript (2.0.2-4) ... 2023-10-01T10:03:44.895 INFO:teuthology.orchestra.run.smithi173.stdout:Adding system user ceph....done 2023-10-01T10:03:45.099 INFO:teuthology.orchestra.run.smithi173.stdout:Setting system user ceph properties....done 2023-10-01T10:03:45.116 INFO:teuthology.orchestra.run.smithi173.stdout:chown: cannot access '/var/log/ceph/*.log*': No such file or directory 2023-10-01T10:03:45.159 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up python3-pecan (1.3.3-3ubuntu1) ... 2023-10-01T10:03:45.257 INFO:teuthology.orchestra.run.smithi173.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /lib/systemd/system/ceph.target. 2023-10-01T10:03:45.521 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up python3-rados (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:45.567 INFO:teuthology.orchestra.run.smithi173.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/rbdmap.service -> /lib/systemd/system/rbdmap.service. 2023-10-01T10:03:45.645 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up python3-rbd (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:45.773 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up rbd-fuse (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:45.899 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up python3-routes (2.4.1-2) ... 2023-10-01T10:03:46.033 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up ceph-test (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:46.159 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up radosgw (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:46.200 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up python3-google-auth (1.5.1-2) ... 2023-10-01T10:03:46.424 INFO:teuthology.orchestra.run.smithi173.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2023-10-01T10:03:46.425 INFO:teuthology.orchestra.run.smithi173.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2023-10-01T10:03:46.612 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up python3-rgw (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:46.738 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up python3-cephfs (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:47.048 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up python3-kubernetes (7.0.0~a1-2) ... 2023-10-01T10:03:47.078 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up ceph-base (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:47.780 INFO:teuthology.orchestra.run.smithi173.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service -> /lib/systemd/system/ceph-crash.service. 2023-10-01T10:03:47.977 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up ceph-common (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:48.247 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up ceph-mds (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:48.470 INFO:teuthology.orchestra.run.smithi173.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2023-10-01T10:03:48.470 INFO:teuthology.orchestra.run.smithi173.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2023-10-01T10:03:48.521 INFO:teuthology.orchestra.run.smithi078.stdout:Adding group ceph....done 2023-10-01T10:03:48.915 INFO:teuthology.orchestra.run.smithi078.stdout:Adding system user ceph....done 2023-10-01T10:03:48.957 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up ceph-mgr (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:49.090 INFO:teuthology.orchestra.run.smithi078.stdout:Setting system user ceph properties....done 2023-10-01T10:03:49.113 INFO:teuthology.orchestra.run.smithi078.stdout:chown: cannot access '/var/log/ceph/*.log*': No such file or directory 2023-10-01T10:03:49.173 INFO:teuthology.orchestra.run.smithi173.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2023-10-01T10:03:49.173 INFO:teuthology.orchestra.run.smithi173.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2023-10-01T10:03:49.260 INFO:teuthology.orchestra.run.smithi078.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /lib/systemd/system/ceph.target. 2023-10-01T10:03:49.583 INFO:teuthology.orchestra.run.smithi078.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/rbdmap.service -> /lib/systemd/system/rbdmap.service. 2023-10-01T10:03:49.641 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up ceph-osd (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:50.053 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up ceph-test (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:50.132 INFO:teuthology.orchestra.run.smithi173.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2023-10-01T10:03:50.132 INFO:teuthology.orchestra.run.smithi173.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2023-10-01T10:03:50.179 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up radosgw (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:50.444 INFO:teuthology.orchestra.run.smithi078.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2023-10-01T10:03:50.444 INFO:teuthology.orchestra.run.smithi078.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2023-10-01T10:03:50.568 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up ceph-mgr-k8sevents (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:50.695 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up ceph-mgr-diskprediction-local (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:50.829 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up ceph-mon (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:51.046 INFO:teuthology.orchestra.run.smithi173.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2023-10-01T10:03:51.046 INFO:teuthology.orchestra.run.smithi173.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2023-10-01T10:03:51.098 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up ceph-base (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:51.468 INFO:teuthology.orchestra.run.smithi078.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service -> /lib/systemd/system/ceph-crash.service. 2023-10-01T10:03:51.491 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up ceph-mgr-cephadm (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:51.634 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up ceph (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:51.760 INFO:teuthology.orchestra.run.smithi173.stdout:Setting up ceph-mgr-dashboard (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:51.904 INFO:teuthology.orchestra.run.smithi173.stdout:Processing triggers for systemd (245.4-4ubuntu3.17) ... 2023-10-01T10:03:51.942 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up ceph-mds (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:52.163 INFO:teuthology.orchestra.run.smithi078.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2023-10-01T10:03:52.165 INFO:teuthology.orchestra.run.smithi078.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2023-10-01T10:03:52.187 INFO:teuthology.orchestra.run.smithi173.stdout:Processing triggers for man-db (2.9.1-1) ... 2023-10-01T10:03:52.637 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up ceph-mgr (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:52.883 INFO:teuthology.orchestra.run.smithi078.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2023-10-01T10:03:52.883 INFO:teuthology.orchestra.run.smithi078.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2023-10-01T10:03:53.332 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up ceph-osd (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:53.828 INFO:teuthology.orchestra.run.smithi078.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2023-10-01T10:03:53.828 INFO:teuthology.orchestra.run.smithi078.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2023-10-01T10:03:54.300 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up ceph-mgr-k8sevents (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:54.426 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up ceph-mgr-diskprediction-local (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:54.569 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up ceph-mon (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:54.786 INFO:teuthology.orchestra.run.smithi078.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2023-10-01T10:03:54.786 INFO:teuthology.orchestra.run.smithi078.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2023-10-01T10:03:55.091 INFO:teuthology.orchestra.run.smithi173.stdout:Processing triggers for libc-bin (2.31-0ubuntu9.9) ... 2023-10-01T10:03:55.245 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up ceph-mgr-cephadm (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:55.391 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up ceph (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:55.517 INFO:teuthology.orchestra.run.smithi078.stdout:Setting up ceph-mgr-dashboard (16.2.14-44-geeb13ac9-1focal) ... 2023-10-01T10:03:55.555 INFO:teuthology.orchestra.run.smithi173.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2023-10-01T10:03:55.558 DEBUG:teuthology.parallel:result is None 2023-10-01T10:03:55.661 INFO:teuthology.orchestra.run.smithi078.stdout:Processing triggers for systemd (245.4-4ubuntu3.17) ... 2023-10-01T10:03:55.961 INFO:teuthology.orchestra.run.smithi078.stdout:Processing triggers for man-db (2.9.1-1) ... 2023-10-01T10:03:58.957 INFO:teuthology.orchestra.run.smithi078.stdout:Processing triggers for libc-bin (2.31-0ubuntu9.9) ... 2023-10-01T10:03:59.407 INFO:teuthology.orchestra.run.smithi078.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2023-10-01T10:03:59.409 DEBUG:teuthology.parallel:result is None 2023-10-01T10:03:59.410 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-01T10:03:59.410 INFO:teuthology.packaging:ref: None 2023-10-01T10:03:59.410 INFO:teuthology.packaging:tag: None 2023-10-01T10:03:59.410 INFO:teuthology.packaging:branch: pacific 2023-10-01T10:03:59.410 INFO:teuthology.packaging:sha1: b56822191ad0416d0c05ed59720df03ad9c2a44e 2023-10-01T10:03:59.411 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&ref=pacific 2023-10-01T10:03:59.810 DEBUG:teuthology.orchestra.run.smithi078:> dpkg-query -W -f '${Version}' ceph 2023-10-01T10:03:59.832 INFO:teuthology.orchestra.run.smithi078.stdout:16.2.14-44-geeb13ac9-1focal 2023-10-01T10:03:59.833 INFO:teuthology.packaging:The installed version of ceph is 16.2.14-44-geeb13ac9-1focal 2023-10-01T10:03:59.833 INFO:teuthology.task.install:The correct ceph version 16.2.14-44-geeb13ac9-1focal is installed. 2023-10-01T10:03:59.833 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-10-01T10:03:59.834 INFO:teuthology.packaging:ref: None 2023-10-01T10:03:59.834 INFO:teuthology.packaging:tag: None 2023-10-01T10:03:59.834 INFO:teuthology.packaging:branch: pacific 2023-10-01T10:03:59.834 INFO:teuthology.packaging:sha1: b56822191ad0416d0c05ed59720df03ad9c2a44e 2023-10-01T10:03:59.834 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F20.04%2Fx86_64&ref=pacific 2023-10-01T10:04:00.212 DEBUG:teuthology.orchestra.run.smithi173:> dpkg-query -W -f '${Version}' ceph 2023-10-01T10:04:00.235 INFO:teuthology.orchestra.run.smithi173.stdout:16.2.14-44-geeb13ac9-1focal 2023-10-01T10:04:00.235 INFO:teuthology.packaging:The installed version of ceph is 16.2.14-44-geeb13ac9-1focal 2023-10-01T10:04:00.235 INFO:teuthology.task.install:The correct ceph version 16.2.14-44-geeb13ac9-1focal is installed. 2023-10-01T10:04:00.235 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2023-10-01T10:04:00.235 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2023-10-01T10:04:00.236 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-10-01T10:04:00.253 DEBUG:teuthology.orchestra.run.smithi173:> set -ex 2023-10-01T10:04:00.253 DEBUG:teuthology.orchestra.run.smithi173:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-10-01T10:04:00.291 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2023-10-01T10:04:00.292 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2023-10-01T10:04:00.292 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd of=/usr/bin/daemon-helper 2023-10-01T10:04:00.310 DEBUG:teuthology.orchestra.run.smithi078:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-10-01T10:04:00.369 DEBUG:teuthology.orchestra.run.smithi173:> set -ex 2023-10-01T10:04:00.370 DEBUG:teuthology.orchestra.run.smithi173:> sudo dd of=/usr/bin/daemon-helper 2023-10-01T10:04:00.384 DEBUG:teuthology.orchestra.run.smithi173:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-10-01T10:04:00.440 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2023-10-01T10:04:00.441 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2023-10-01T10:04:00.441 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd of=/usr/bin/adjust-ulimits 2023-10-01T10:04:00.457 DEBUG:teuthology.orchestra.run.smithi078:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-10-01T10:04:00.513 DEBUG:teuthology.orchestra.run.smithi173:> set -ex 2023-10-01T10:04:00.513 DEBUG:teuthology.orchestra.run.smithi173:> sudo dd of=/usr/bin/adjust-ulimits 2023-10-01T10:04:00.529 DEBUG:teuthology.orchestra.run.smithi173:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-10-01T10:04:00.588 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2023-10-01T10:04:00.589 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2023-10-01T10:04:00.589 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd of=/usr/bin/stdin-killer 2023-10-01T10:04:00.605 DEBUG:teuthology.orchestra.run.smithi078:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-10-01T10:04:00.661 DEBUG:teuthology.orchestra.run.smithi173:> set -ex 2023-10-01T10:04:00.661 DEBUG:teuthology.orchestra.run.smithi173:> sudo dd of=/usr/bin/stdin-killer 2023-10-01T10:04:00.676 DEBUG:teuthology.orchestra.run.smithi173:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-10-01T10:04:00.732 INFO:teuthology.run_tasks:Running task cephadm... 2023-10-01T10:04:00.816 INFO:tasks.cephadm:Config: {'cephadm_branch': 'pacific', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:pacific', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'b56822191ad0416d0c05ed59720df03ad9c2a44e'} 2023-10-01T10:04:00.817 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:pacific 2023-10-01T10:04:00.817 INFO:tasks.cephadm:Cluster fsid is d8016480-6041-11ee-8db5-212e2dc638e7 2023-10-01T10:04:00.817 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2023-10-01T10:04:00.817 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.78', 'mon.c': '[v2:172.21.15.78:3301,v1:172.21.15.78:6790]', 'mon.b': '172.21.15.173'} 2023-10-01T10:04:00.818 INFO:tasks.cephadm:First mon is mon.a on smithi078 2023-10-01T10:04:00.818 INFO:tasks.cephadm:First mgr is y 2023-10-01T10:04:00.818 INFO:tasks.cephadm:Normalizing hostnames... 2023-10-01T10:04:00.818 DEBUG:teuthology.orchestra.run.smithi078:> sudo hostname $(hostname -s) 2023-10-01T10:04:00.835 DEBUG:teuthology.orchestra.run.smithi173:> sudo hostname $(hostname -s) 2023-10-01T10:04:00.852 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref pacific)... 2023-10-01T10:04:00.852 DEBUG:teuthology.orchestra.run.smithi078:> curl --silent https://raw.githubusercontent.com/ceph/ceph/pacific/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-10-01T10:04:01.096 INFO:teuthology.orchestra.run.smithi078.stdout:-rw-rw-r-- 1 ubuntu ubuntu 350873 Oct 1 10:04 /home/ubuntu/cephtest/cephadm 2023-10-01T10:04:01.096 DEBUG:teuthology.orchestra.run.smithi173:> curl --silent https://raw.githubusercontent.com/ceph/ceph/pacific/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-10-01T10:04:01.375 INFO:teuthology.orchestra.run.smithi173.stdout:-rw-rw-r-- 1 ubuntu ubuntu 350873 Oct 1 10:04 /home/ubuntu/cephtest/cephadm 2023-10-01T10:04:01.375 DEBUG:teuthology.orchestra.run.smithi078:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-10-01T10:04:01.385 DEBUG:teuthology.orchestra.run.smithi173:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-10-01T10:04:01.405 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:pacific on all hosts... 2023-10-01T10:04:01.405 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific pull 2023-10-01T10:04:01.428 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific pull 2023-10-01T10:04:01.670 INFO:teuthology.orchestra.run.smithi078.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:pacific... 2023-10-01T10:04:01.709 INFO:teuthology.orchestra.run.smithi173.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:pacific... 2023-10-01T10:04:41.851 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-10-01T10:04:41.852 INFO:teuthology.orchestra.run.smithi173.stdout: "ceph_version": "ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)", 2023-10-01T10:04:41.852 INFO:teuthology.orchestra.run.smithi173.stdout: "image_id": "94a6c4498547c13992866c6d194f294b2191daf4f02c7d4d133790c792e33bf5", 2023-10-01T10:04:41.852 INFO:teuthology.orchestra.run.smithi173.stdout: "repo_digests": [ 2023-10-01T10:04:41.852 INFO:teuthology.orchestra.run.smithi173.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:8e9fcccea6e0115c1572f3a4163cdf39971d9e270ff3987b3049e8aad314c194" 2023-10-01T10:04:41.852 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:04:41.852 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-10-01T10:04:46.271 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2023-10-01T10:04:46.271 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph_version": "ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)", 2023-10-01T10:04:46.272 INFO:teuthology.orchestra.run.smithi078.stdout: "image_id": "94a6c4498547c13992866c6d194f294b2191daf4f02c7d4d133790c792e33bf5", 2023-10-01T10:04:46.272 INFO:teuthology.orchestra.run.smithi078.stdout: "repo_digests": [ 2023-10-01T10:04:46.272 INFO:teuthology.orchestra.run.smithi078.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:8e9fcccea6e0115c1572f3a4163cdf39971d9e270ff3987b3049e8aad314c194" 2023-10-01T10:04:46.272 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:04:46.272 INFO:teuthology.orchestra.run.smithi078.stdout:} 2023-10-01T10:04:46.296 DEBUG:teuthology.orchestra.run.smithi078:> sudo mkdir -p /etc/ceph 2023-10-01T10:04:46.311 DEBUG:teuthology.orchestra.run.smithi173:> sudo mkdir -p /etc/ceph 2023-10-01T10:04:46.330 DEBUG:teuthology.orchestra.run.smithi078:> sudo chmod 777 /etc/ceph 2023-10-01T10:04:46.368 DEBUG:teuthology.orchestra.run.smithi173:> sudo chmod 777 /etc/ceph 2023-10-01T10:04:46.388 INFO:tasks.cephadm:Writing seed config... 2023-10-01T10:04:46.390 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2023-10-01T10:04:46.390 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2023-10-01T10:04:46.390 INFO:tasks.cephadm: override: [osd] debug ms = 1 2023-10-01T10:04:46.390 INFO:tasks.cephadm: override: [osd] debug osd = 20 2023-10-01T10:04:46.390 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2023-10-01T10:04:46.390 INFO:tasks.cephadm: override: [global] mon election default strategy = 3 2023-10-01T10:04:46.390 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2023-10-01T10:04:46.390 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2023-10-01T10:04:46.390 INFO:tasks.cephadm: override: [mon] debug mon = 20 2023-10-01T10:04:46.390 INFO:tasks.cephadm: override: [mon] debug ms = 1 2023-10-01T10:04:46.391 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2023-10-01T10:04:46.391 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2023-10-01T10:04:46.391 DEBUG:teuthology.orchestra.run.smithi078:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2023-10-01T10:04:46.416 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = d8016480-6041-11ee-8db5-212e2dc638e7 mon election default strategy = 3 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2023-10-01T10:04:46.417 DEBUG:teuthology.orchestra.run.smithi078:mon.a> sudo journalctl -f -n 0 -u ceph-d8016480-6041-11ee-8db5-212e2dc638e7@mon.a.service 2023-10-01T10:04:46.460 DEBUG:teuthology.orchestra.run.smithi078:mgr.y> sudo journalctl -f -n 0 -u ceph-d8016480-6041-11ee-8db5-212e2dc638e7@mgr.y.service 2023-10-01T10:04:46.503 INFO:tasks.cephadm:Bootstrapping... 2023-10-01T10:04:46.504 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific -v bootstrap --fsid d8016480-6041-11ee-8db5-212e2dc638e7 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.78 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2023-10-01T10:04:46.713 INFO:teuthology.orchestra.run.smithi078.stderr:-------------------------------------------------------------------------------- 2023-10-01T10:04:46.713 INFO:teuthology.orchestra.run.smithi078.stderr:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:pacific', '-v', 'bootstrap', '--fsid', 'd8016480-6041-11ee-8db5-212e2dc638e7', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.78', '--skip-admin-label'] 2023-10-01T10:04:46.713 INFO:teuthology.orchestra.run.smithi078.stderr:Verifying podman|docker is present... 2023-10-01T10:04:46.713 INFO:teuthology.orchestra.run.smithi078.stderr:Verifying lvm2 is present... 2023-10-01T10:04:46.713 INFO:teuthology.orchestra.run.smithi078.stderr:Verifying time synchronization is in place... 2023-10-01T10:04:46.718 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2023-10-01T10:04:46.725 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: inactive 2023-10-01T10:04:46.730 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: Failed to get unit file state for chronyd.service: No such file or directory 2023-10-01T10:04:46.735 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: inactive 2023-10-01T10:04:46.740 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: masked 2023-10-01T10:04:46.745 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: inactive 2023-10-01T10:04:46.749 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: Failed to get unit file state for ntpd.service: No such file or directory 2023-10-01T10:04:46.753 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: inactive 2023-10-01T10:04:46.758 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: enabled 2023-10-01T10:04:46.762 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: active 2023-10-01T10:04:46.763 INFO:teuthology.orchestra.run.smithi078.stderr:Unit ntp.service is enabled and running 2023-10-01T10:04:46.764 INFO:teuthology.orchestra.run.smithi078.stderr:Repeating the final host check... 2023-10-01T10:04:46.764 INFO:teuthology.orchestra.run.smithi078.stderr:docker (/usr/bin/docker) is present 2023-10-01T10:04:46.764 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl is present 2023-10-01T10:04:46.764 INFO:teuthology.orchestra.run.smithi078.stderr:lvcreate is present 2023-10-01T10:04:46.769 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2023-10-01T10:04:46.774 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: inactive 2023-10-01T10:04:46.778 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: Failed to get unit file state for chronyd.service: No such file or directory 2023-10-01T10:04:46.782 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: inactive 2023-10-01T10:04:46.789 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: masked 2023-10-01T10:04:46.790 INFO:journalctl@ceph.mon.a.smithi078.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-01T10:04:46.795 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: inactive 2023-10-01T10:04:46.796 INFO:journalctl@ceph.mgr.y.smithi078.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-01T10:04:46.799 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: Failed to get unit file state for ntpd.service: No such file or directory 2023-10-01T10:04:46.804 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: inactive 2023-10-01T10:04:46.810 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: enabled 2023-10-01T10:04:46.816 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: active 2023-10-01T10:04:46.817 INFO:teuthology.orchestra.run.smithi078.stderr:Unit ntp.service is enabled and running 2023-10-01T10:04:46.817 INFO:teuthology.orchestra.run.smithi078.stderr:Host looks OK 2023-10-01T10:04:46.817 INFO:teuthology.orchestra.run.smithi078.stderr:Cluster fsid: d8016480-6041-11ee-8db5-212e2dc638e7 2023-10-01T10:04:46.817 INFO:teuthology.orchestra.run.smithi078.stderr:Acquiring lock 140647475720144 on /run/cephadm/d8016480-6041-11ee-8db5-212e2dc638e7.lock 2023-10-01T10:04:46.818 INFO:teuthology.orchestra.run.smithi078.stderr:Lock 140647475720144 acquired on /run/cephadm/d8016480-6041-11ee-8db5-212e2dc638e7.lock 2023-10-01T10:04:46.818 INFO:teuthology.orchestra.run.smithi078.stderr:Verifying IP 172.21.15.78 port 3300 ... 2023-10-01T10:04:46.818 INFO:teuthology.orchestra.run.smithi078.stderr:Verifying IP 172.21.15.78 port 6789 ... 2023-10-01T10:04:46.818 INFO:teuthology.orchestra.run.smithi078.stderr:Base mon IP(s) is [172.21.15.78:3300, 172.21.15.78:6789], mon addrv is [v2:172.21.15.78:3300,v1:172.21.15.78:6789] 2023-10-01T10:04:46.823 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/sbin/ip: default via 172.21.15.254 dev enp3s0f1 2023-10-01T10:04:46.823 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/sbin/ip: default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.78 metric 100 2023-10-01T10:04:46.823 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/sbin/ip: 172.17.0.0/16 dev docker0 proto kernel scope link src 172.17.0.1 linkdown 2023-10-01T10:04:46.823 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/sbin/ip: 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.78 2023-10-01T10:04:46.824 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/sbin/ip: 172.21.15.254 dev enp3s0f1 proto dhcp scope link src 172.21.15.78 metric 100 2023-10-01T10:04:46.827 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2023-10-01T10:04:46.827 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/sbin/ip: fe80::/64 dev enp3s0f1 proto kernel metric 256 pref medium 2023-10-01T10:04:46.827 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 100 expires 1778sec pref medium 2023-10-01T10:04:46.831 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2023-10-01T10:04:46.831 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/sbin/ip: inet6 ::1/128 scope host 2023-10-01T10:04:46.831 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/sbin/ip: valid_lft forever preferred_lft forever 2023-10-01T10:04:46.831 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/sbin/ip: 5: enp3s0f1: mtu 1500 state UP qlen 1000 2023-10-01T10:04:46.831 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/sbin/ip: inet6 fe80::ec4:7aff:fe8f:d4dd/64 scope link 2023-10-01T10:04:46.831 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/sbin/ip: valid_lft forever preferred_lft forever 2023-10-01T10:04:46.832 INFO:teuthology.orchestra.run.smithi078.stderr:Mon IP `172.21.15.78` is in CIDR network `172.21.0.0/20` 2023-10-01T10:04:46.832 INFO:teuthology.orchestra.run.smithi078.stderr:Mon IP `172.21.15.78` is in CIDR network `172.21.0.0/20` 2023-10-01T10:04:46.832 INFO:teuthology.orchestra.run.smithi078.stderr:Mon IP `172.21.15.78` is in CIDR network `172.21.15.254/32` 2023-10-01T10:04:46.833 INFO:teuthology.orchestra.run.smithi078.stderr:Mon IP `172.21.15.78` is in CIDR network `172.21.15.254/32` 2023-10-01T10:04:46.833 INFO:teuthology.orchestra.run.smithi078.stderr:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20', '172.21.15.254/32', '172.21.15.254/32'] 2023-10-01T10:04:46.834 INFO:teuthology.orchestra.run.smithi078.stderr:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2023-10-01T10:04:46.834 INFO:teuthology.orchestra.run.smithi078.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:pacific... 2023-10-01T10:04:47.035 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/docker: pacific: Pulling from ceph-ci/ceph 2023-10-01T10:04:47.064 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/docker: Digest: sha256:8e9fcccea6e0115c1572f3a4163cdf39971d9e270ff3987b3049e8aad314c194 2023-10-01T10:04:47.064 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/docker: Status: Image is up to date for quay.ceph.io/ceph-ci/ceph:pacific 2023-10-01T10:04:47.097 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/docker: quay.ceph.io/ceph-ci/ceph:pacific 2023-10-01T10:04:48.040 INFO:teuthology.orchestra.run.smithi078.stderr:ceph: ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable) 2023-10-01T10:04:48.657 INFO:teuthology.orchestra.run.smithi078.stderr:Ceph version: ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable) 2023-10-01T10:04:48.657 INFO:teuthology.orchestra.run.smithi078.stderr:Extracting ceph user uid/gid from container image... 2023-10-01T10:04:49.428 INFO:teuthology.orchestra.run.smithi078.stderr:stat: 167 167 2023-10-01T10:04:49.987 INFO:teuthology.orchestra.run.smithi078.stderr:Creating initial keys... 2023-10-01T10:04:50.819 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-authtool: AQBCRBllj6evMBAAj6ifXtu5W9ikwN4aC85qbA== 2023-10-01T10:04:52.257 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-authtool: AQBERBllXjMzDxAATaA397CZTlqcyWcA5RavGg== 2023-10-01T10:04:53.636 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-authtool: AQBFRBllFI7KJRAAKoZUUBuj5yKluB1XTXpgFA== 2023-10-01T10:04:54.224 INFO:teuthology.orchestra.run.smithi078.stderr:Creating initial monmap... 2023-10-01T10:04:55.009 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2023-10-01T10:04:55.009 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to d8016480-6041-11ee-8db5-212e2dc638e7 2023-10-01T10:04:55.009 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-10-01T10:04:55.595 INFO:teuthology.orchestra.run.smithi078.stderr:monmaptool for a [v2:172.21.15.78:3300,v1:172.21.15.78:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2023-10-01T10:04:55.595 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/monmaptool: set fsid to d8016480-6041-11ee-8db5-212e2dc638e7 2023-10-01T10:04:55.596 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-10-01T10:04:55.596 INFO:teuthology.orchestra.run.smithi078.stderr: 2023-10-01T10:04:55.596 INFO:teuthology.orchestra.run.smithi078.stderr:Creating mon... 2023-10-01T10:04:56.380 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 0 set uid:gid to 167:167 (ceph:ceph) 2023-10-01T10:04:56.381 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 1 imported monmap: 2023-10-01T10:04:56.381 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: epoch 0 2023-10-01T10:04:56.381 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: fsid d8016480-6041-11ee-8db5-212e2dc638e7 2023-10-01T10:04:56.381 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: last_changed 2023-10-01T10:04:55.006877+0000 2023-10-01T10:04:56.382 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: created 2023-10-01T10:04:55.006877+0000 2023-10-01T10:04:56.382 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: min_mon_release 0 (unknown) 2023-10-01T10:04:56.382 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: election_strategy: 1 2023-10-01T10:04:56.383 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: 0: [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] mon.a 2023-10-01T10:04:56.383 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: 2023-10-01T10:04:56.383 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 0 /usr/bin/ceph-mon: set fsid to d8016480-6041-11ee-8db5-212e2dc638e7 2023-10-01T10:04:56.384 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: RocksDB version: 6.8.1 2023-10-01T10:04:56.384 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: 2023-10-01T10:04:56.384 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-10-01T10:04:56.385 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Compile date Sep 25 2023 2023-10-01T10:04:56.385 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: DB SUMMARY 2023-10-01T10:04:56.385 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: 2023-10-01T10:04:56.385 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 0, files: 2023-10-01T10:04:56.385 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: 2023-10-01T10:04:56.385 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 2023-10-01T10:04:56.386 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: 2023-10-01T10:04:56.390 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.error_if_exists: 0 2023-10-01T10:04:56.390 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.create_if_missing: 1 2023-10-01T10:04:56.390 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.paranoid_checks: 1 2023-10-01T10:04:56.390 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.env: 0x5592737c1080 2023-10-01T10:04:56.390 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.fs: Posix File System 2023-10-01T10:04:56.390 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.info_log: 0x55927562daa0 2023-10-01T10:04:56.391 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.max_file_opening_threads: 16 2023-10-01T10:04:56.391 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.statistics: (nil) 2023-10-01T10:04:56.391 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.use_fsync: 0 2023-10-01T10:04:56.391 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.max_log_file_size: 0 2023-10-01T10:04:56.391 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-01T10:04:56.391 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.log_file_time_to_roll: 0 2023-10-01T10:04:56.392 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.keep_log_file_num: 1000 2023-10-01T10:04:56.392 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.recycle_log_file_num: 0 2023-10-01T10:04:56.392 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.allow_fallocate: 1 2023-10-01T10:04:56.392 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.allow_mmap_reads: 0 2023-10-01T10:04:56.392 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.allow_mmap_writes: 0 2023-10-01T10:04:56.392 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.use_direct_reads: 0 2023-10-01T10:04:56.392 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-01T10:04:56.393 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.create_missing_column_families: 0 2023-10-01T10:04:56.393 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.db_log_dir: 2023-10-01T10:04:56.393 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2023-10-01T10:04:56.393 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.table_cache_numshardbits: 6 2023-10-01T10:04:56.393 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.max_subcompactions: 1 2023-10-01T10:04:56.393 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.max_background_flushes: -1 2023-10-01T10:04:56.394 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-10-01T10:04:56.394 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-10-01T10:04:56.394 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-01T10:04:56.394 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-01T10:04:56.394 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-10-01T10:04:56.394 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.advise_random_on_open: 1 2023-10-01T10:04:56.394 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.db_write_buffer_size: 0 2023-10-01T10:04:56.395 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.write_buffer_manager: 0x559275635890 2023-10-01T10:04:56.395 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-01T10:04:56.395 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-10-01T10:04:56.395 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-01T10:04:56.395 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.use_adaptive_mutex: 0 2023-10-01T10:04:56.395 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.rate_limiter: (nil) 2023-10-01T10:04:56.396 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-01T10:04:56.396 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.wal_recovery_mode: 2 2023-10-01T10:04:56.396 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.enable_thread_tracking: 0 2023-10-01T10:04:56.421 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.enable_pipelined_write: 0 2023-10-01T10:04:56.421 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.unordered_write: 0 2023-10-01T10:04:56.421 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-01T10:04:56.422 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-01T10:04:56.422 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-01T10:04:56.422 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-01T10:04:56.422 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.row_cache: None 2023-10-01T10:04:56.422 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.wal_filter: None 2023-10-01T10:04:56.422 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-01T10:04:56.422 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.allow_ingest_behind: 0 2023-10-01T10:04:56.422 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.preserve_deletes: 0 2023-10-01T10:04:56.422 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.two_write_queues: 0 2023-10-01T10:04:56.422 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.manual_wal_flush: 0 2023-10-01T10:04:56.423 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.atomic_flush: 0 2023-10-01T10:04:56.423 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-01T10:04:56.423 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.persist_stats_to_disk: 0 2023-10-01T10:04:56.423 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-10-01T10:04:56.423 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.log_readahead_size: 0 2023-10-01T10:04:56.423 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.sst_file_checksum_func: Unknown 2023-10-01T10:04:56.423 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.max_background_jobs: 2 2023-10-01T10:04:56.423 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.max_background_compactions: -1 2023-10-01T10:04:56.423 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-01T10:04:56.423 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-01T10:04:56.423 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.delayed_write_rate : 16777216 2023-10-01T10:04:56.423 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.max_total_wal_size: 0 2023-10-01T10:04:56.424 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-01T10:04:56.424 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.stats_dump_period_sec: 600 2023-10-01T10:04:56.424 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.stats_persist_period_sec: 600 2023-10-01T10:04:56.424 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-01T10:04:56.424 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.max_open_files: -1 2023-10-01T10:04:56.424 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.bytes_per_sync: 0 2023-10-01T10:04:56.424 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-10-01T10:04:56.424 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-10-01T10:04:56.424 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.compaction_readahead_size: 0 2023-10-01T10:04:56.424 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Compression algorithms supported: 2023-10-01T10:04:56.424 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-01T10:04:56.424 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: kZSTD supported: 0 2023-10-01T10:04:56.425 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: kXpressCompression supported: 0 2023-10-01T10:04:56.425 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: kLZ4HCCompression supported: 1 2023-10-01T10:04:56.425 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: kLZ4Compression supported: 1 2023-10-01T10:04:56.425 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: kBZip2Compression supported: 0 2023-10-01T10:04:56.425 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: kZlibCompression supported: 1 2023-10-01T10:04:56.425 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: kSnappyCompression supported: 1 2023-10-01T10:04:56.425 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-10-01T10:04:56.425 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: [db_impl/db_impl_open.cc:273] Creating manifest 1 2023-10-01T10:04:56.425 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: 2023-10-01T10:04:56.425 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000001 2023-10-01T10:04:56.425 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: 2023-10-01T10:04:56.426 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2023-10-01T10:04:56.426 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: 2023-10-01T10:04:56.426 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-01T10:04:56.426 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.merge_operator: 2023-10-01T10:04:56.426 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.compaction_filter: None 2023-10-01T10:04:56.426 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.compaction_filter_factory: None 2023-10-01T10:04:56.426 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-10-01T10:04:56.426 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.table_factory: BlockBasedTable 2023-10-01T10:04:56.426 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x559275543d20) 2023-10-01T10:04:56.426 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: cache_index_and_filter_blocks: 1 2023-10-01T10:04:56.426 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-01T10:04:56.427 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-01T10:04:56.427 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: pin_top_level_index_and_filter: 1 2023-10-01T10:04:56.427 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: index_type: 0 2023-10-01T10:04:56.427 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: data_block_index_type: 0 2023-10-01T10:04:56.427 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: index_shortening: 1 2023-10-01T10:04:56.427 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: data_block_hash_table_util_ratio: 0.750000 2023-10-01T10:04:56.427 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: hash_index_allow_collision: 1 2023-10-01T10:04:56.427 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: checksum: 1 2023-10-01T10:04:56.427 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: no_block_cache: 0 2023-10-01T10:04:56.427 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: block_cache: 0x55927557ad10 2023-10-01T10:04:56.427 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: block_cache_name: BinnedLRUCache 2023-10-01T10:04:56.427 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: block_cache_options: 2023-10-01T10:04:56.428 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: capacity : 536870912 2023-10-01T10:04:56.428 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: num_shard_bits : 4 2023-10-01T10:04:56.428 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: strict_capacity_limit : 0 2023-10-01T10:04:56.428 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: high_pri_pool_ratio: 0.000 2023-10-01T10:04:56.428 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: block_cache_compressed: (nil) 2023-10-01T10:04:56.428 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: persistent_cache: (nil) 2023-10-01T10:04:56.428 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: block_size: 4096 2023-10-01T10:04:56.428 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: block_size_deviation: 10 2023-10-01T10:04:56.428 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: block_restart_interval: 16 2023-10-01T10:04:56.428 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: index_block_restart_interval: 1 2023-10-01T10:04:56.428 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: metadata_block_size: 4096 2023-10-01T10:04:56.428 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: partition_filters: 0 2023-10-01T10:04:56.429 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: use_delta_encoding: 1 2023-10-01T10:04:56.429 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: filter_policy: rocksdb.BuiltinBloomFilter 2023-10-01T10:04:56.429 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: whole_key_filtering: 1 2023-10-01T10:04:56.429 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: verify_compression: 0 2023-10-01T10:04:56.429 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: read_amp_bytes_per_bit: 0 2023-10-01T10:04:56.429 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: format_version: 2 2023-10-01T10:04:56.429 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: enable_index_compression: 1 2023-10-01T10:04:56.429 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: block_align: 0 2023-10-01T10:04:56.429 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: 2023-10-01T10:04:56.429 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.write_buffer_size: 33554432 2023-10-01T10:04:56.429 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.max_write_buffer_number: 2 2023-10-01T10:04:56.429 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.compression: NoCompression 2023-10-01T10:04:56.430 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.bottommost_compression: Disabled 2023-10-01T10:04:56.430 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.prefix_extractor: nullptr 2023-10-01T10:04:56.430 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-01T10:04:56.430 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.num_levels: 7 2023-10-01T10:04:56.430 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-01T10:04:56.430 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-01T10:04:56.430 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-01T10:04:56.430 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-01T10:04:56.430 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-01T10:04:56.430 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-01T10:04:56.431 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-01T10:04:56.431 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-01T10:04:56.431 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-01T10:04:56.431 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.compression_opts.window_bits: -14 2023-10-01T10:04:56.431 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.compression_opts.level: 32767 2023-10-01T10:04:56.431 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.compression_opts.strategy: 0 2023-10-01T10:04:56.431 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-01T10:04:56.432 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-01T10:04:56.432 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.compression_opts.enabled: false 2023-10-01T10:04:56.432 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-01T10:04:56.432 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-01T10:04:56.432 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-01T10:04:56.432 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.target_file_size_base: 67108864 2023-10-01T10:04:56.432 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.target_file_size_multiplier: 1 2023-10-01T10:04:56.432 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-01T10:04:56.432 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-01T10:04:56.432 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-01T10:04:56.433 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-01T10:04:56.433 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-01T10:04:56.433 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-01T10:04:56.433 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-01T10:04:56.433 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-01T10:04:56.433 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-01T10:04:56.434 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-01T10:04:56.434 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-01T10:04:56.434 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-01T10:04:56.434 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.arena_block_size: 4194304 2023-10-01T10:04:56.434 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-01T10:04:56.434 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-01T10:04:56.434 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-10-01T10:04:56.434 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.disable_auto_compactions: 0 2023-10-01T10:04:56.434 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-01T10:04:56.434 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-01T10:04:56.435 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-01T10:04:56.435 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-01T10:04:56.435 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-01T10:04:56.435 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-01T10:04:56.435 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-01T10:04:56.435 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-01T10:04:56.435 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-01T10:04:56.435 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-01T10:04:56.435 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.table_properties_collectors: 2023-10-01T10:04:56.435 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.inplace_update_support: 0 2023-10-01T10:04:56.435 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-10-01T10:04:56.435 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-01T10:04:56.436 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-01T10:04:56.436 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.memtable_huge_page_size: 0 2023-10-01T10:04:56.436 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.bloom_locality: 0 2023-10-01T10:04:56.436 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.max_successive_merges: 0 2023-10-01T10:04:56.436 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-10-01T10:04:56.436 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.paranoid_file_checks: 0 2023-10-01T10:04:56.436 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.force_consistency_checks: 0 2023-10-01T10:04:56.436 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.report_bg_io_stats: 0 2023-10-01T10:04:56.436 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.ttl: 2592000 2023-10-01T10:04:56.436 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.376+0000 7fee1b906880 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-10-01T10:04:56.436 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.380+0000 7fee1b906880 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-01T10:04:56.437 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: 2023-10-01T10:04:56.437 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.380+0000 7fee1b906880 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 0 2023-10-01T10:04:56.437 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: 2023-10-01T10:04:56.437 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.380+0000 7fee1b906880 4 rocksdb: DB pointer 0x559275643800 2023-10-01T10:04:56.437 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.380+0000 7fee0478f700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2023-10-01T10:04:56.437 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.380+0000 7fee0478f700 4 rocksdb: [db_impl/db_impl.cc:851] 2023-10-01T10:04:56.437 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: ** DB Stats ** 2023-10-01T10:04:56.437 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: Uptime(secs): 0.0 total, 0.0 interval 2023-10-01T10:04:56.437 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-01T10:04:56.437 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-01T10:04:56.437 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-01T10:04:56.438 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-01T10:04:56.438 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-10-01T10:04:56.438 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-01T10:04:56.438 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: 2023-10-01T10:04:56.438 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-10-01T10:04:56.438 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-01T10:04:56.438 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-01T10:04:56.438 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-01T10:04:56.438 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-01T10:04:56.438 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: 2023-10-01T10:04:56.438 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-10-01T10:04:56.439 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-01T10:04:56.439 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-01T10:04:56.439 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: Uptime(secs): 0.0 total, 0.0 interval 2023-10-01T10:04:56.439 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: Flush(GB): cumulative 0.000, interval 0.000 2023-10-01T10:04:56.439 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-01T10:04:56.439 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: AddFile(Total Files): cumulative 0, interval 0 2023-10-01T10:04:56.439 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: AddFile(L0 Files): cumulative 0, interval 0 2023-10-01T10:04:56.439 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: AddFile(Keys): cumulative 0, interval 0 2023-10-01T10:04:56.439 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-01T10:04:56.439 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-01T10:04:56.439 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-01T10:04:56.439 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: 2023-10-01T10:04:56.440 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: ** File Read Latency Histogram By Level [default] ** 2023-10-01T10:04:56.440 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: 2023-10-01T10:04:56.440 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-10-01T10:04:56.440 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-01T10:04:56.440 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-01T10:04:56.440 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-01T10:04:56.440 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-01T10:04:56.440 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: 2023-10-01T10:04:56.440 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-10-01T10:04:56.440 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-01T10:04:56.440 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-01T10:04:56.441 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: Uptime(secs): 0.0 total, 0.0 interval 2023-10-01T10:04:56.441 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: Flush(GB): cumulative 0.000, interval 0.000 2023-10-01T10:04:56.441 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-01T10:04:56.441 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: AddFile(Total Files): cumulative 0, interval 0 2023-10-01T10:04:56.441 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: AddFile(L0 Files): cumulative 0, interval 0 2023-10-01T10:04:56.441 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: AddFile(Keys): cumulative 0, interval 0 2023-10-01T10:04:56.441 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-01T10:04:56.441 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-01T10:04:56.441 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-01T10:04:56.442 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: 2023-10-01T10:04:56.442 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: ** File Read Latency Histogram By Level [default] ** 2023-10-01T10:04:56.442 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: 2023-10-01T10:04:56.442 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.380+0000 7fee1b906880 4 rocksdb: [db_impl/db_impl.cc:397] Shutdown: canceling all background work 2023-10-01T10:04:56.442 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.380+0000 7fee1b906880 4 rocksdb: [db_impl/db_impl.cc:573] Shutdown complete 2023-10-01T10:04:56.443 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-mon: debug 2023-10-01T10:04:56.380+0000 7fee1b906880 0 /usr/bin/ceph-mon: created monfs at /var/lib/ceph/mon/ceph-a for mon.a 2023-10-01T10:04:56.985 INFO:teuthology.orchestra.run.smithi078.stderr:create mon.a on 2023-10-01T10:04:57.010 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: Removed /etc/systemd/system/multi-user.target.wants/ceph.target. 2023-10-01T10:04:57.224 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /etc/systemd/system/ceph.target. 2023-10-01T10:04:57.438 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-d8016480-6041-11ee-8db5-212e2dc638e7.target -> /etc/systemd/system/ceph-d8016480-6041-11ee-8db5-212e2dc638e7.target. 2023-10-01T10:04:57.440 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-d8016480-6041-11ee-8db5-212e2dc638e7.target -> /etc/systemd/system/ceph-d8016480-6041-11ee-8db5-212e2dc638e7.target. 2023-10-01T10:04:57.869 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: Failed to reset failed state of unit ceph-d8016480-6041-11ee-8db5-212e2dc638e7@mon.a.service: Unit ceph-d8016480-6041-11ee-8db5-212e2dc638e7@mon.a.service not loaded. 2023-10-01T10:04:57.878 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: Created symlink /etc/systemd/system/ceph-d8016480-6041-11ee-8db5-212e2dc638e7.target.wants/ceph-d8016480-6041-11ee-8db5-212e2dc638e7@mon.a.service -> /etc/systemd/system/ceph-d8016480-6041-11ee-8db5-212e2dc638e7@.service. 2023-10-01T10:04:58.088 INFO:teuthology.orchestra.run.smithi078.stderr:firewalld does not appear to be present 2023-10-01T10:04:58.088 INFO:teuthology.orchestra.run.smithi078.stderr:Not possible to enable service . firewalld.service is not available 2023-10-01T10:04:58.089 INFO:teuthology.orchestra.run.smithi078.stderr:Waiting for mon to start... 2023-10-01T10:04:58.089 INFO:teuthology.orchestra.run.smithi078.stderr:Waiting for mon... 2023-10-01T10:04:59.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:04:59 smithi078 bash[26790]: cluster 2023-10-01T10:04:59.310269+0000 mon.a (mon.0) 1 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2023-10-01T10:04:59.586 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: cluster: 2023-10-01T10:04:59.587 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: id: d8016480-6041-11ee-8db5-212e2dc638e7 2023-10-01T10:04:59.587 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: health: HEALTH_OK 2023-10-01T10:04:59.587 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2023-10-01T10:04:59.587 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: services: 2023-10-01T10:04:59.587 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mon: 1 daemons, quorum a (age 0.240325s) 2023-10-01T10:04:59.587 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mgr: no daemons active 2023-10-01T10:04:59.587 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2023-10-01T10:04:59.587 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2023-10-01T10:04:59.587 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: data: 2023-10-01T10:04:59.587 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2023-10-01T10:04:59.588 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2023-10-01T10:04:59.588 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2023-10-01T10:04:59.588 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: pgs: 2023-10-01T10:04:59.588 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2023-10-01T10:05:00.209 INFO:teuthology.orchestra.run.smithi078.stderr:mon is available 2023-10-01T10:05:00.209 INFO:teuthology.orchestra.run.smithi078.stderr:Assimilating anything we can from ceph.conf... 2023-10-01T10:05:00.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:00 smithi078 bash[26790]: cluster 2023-10-01T10:04:59.342363+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2023-10-01T10:05:00.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:00 smithi078 bash[26790]: cluster 2023-10-01T10:04:59.342851+0000 mon.a (mon.0) 3 : cluster [DBG] monmap e1: 1 mons at {a=[v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0]} removed_ranks: {} 2023-10-01T10:05:00.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:00 smithi078 bash[26790]: cluster 2023-10-01T10:04:59.348816+0000 mon.a (mon.0) 4 : cluster [DBG] fsmap 2023-10-01T10:05:00.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:00 smithi078 bash[26790]: cluster 2023-10-01T10:04:59.352908+0000 mon.a (mon.0) 5 : cluster [DBG] osdmap e1: 0 total, 0 up, 0 in 2023-10-01T10:05:00.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:00 smithi078 bash[26790]: cluster 2023-10-01T10:04:59.353637+0000 mon.a (mon.0) 6 : cluster [DBG] mgrmap e1: no daemons active 2023-10-01T10:05:00.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:00 smithi078 bash[26790]: audit 2023-10-01T10:04:59.582612+0000 mon.a (mon.0) 7 : audit [DBG] from='client.? 172.21.15.78:0/248820635' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2023-10-01T10:05:01.314 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2023-10-01T10:05:01.314 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: [global] 2023-10-01T10:05:01.315 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: fsid = d8016480-6041-11ee-8db5-212e2dc638e7 2023-10-01T10:05:01.315 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: log_to_journald = false 2023-10-01T10:05:01.315 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.78:3300,v1:172.21.15.78:6789] 2023-10-01T10:05:01.315 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2023-10-01T10:05:01.315 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2023-10-01T10:05:01.315 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2023-10-01T10:05:01.315 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2023-10-01T10:05:01.315 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2023-10-01T10:05:01.315 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: [mgr] 2023-10-01T10:05:01.315 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2023-10-01T10:05:01.316 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2023-10-01T10:05:01.318 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: [osd] 2023-10-01T10:05:01.318 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2023-10-01T10:05:01.318 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2023-10-01T10:05:01.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:01 smithi078 bash[26790]: audit 2023-10-01T10:05:01.305541+0000 mon.a (mon.0) 8 : audit [INF] from='client.? 172.21.15.78:0/841649602' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-10-01T10:05:01.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:01 smithi078 bash[26790]: audit 2023-10-01T10:05:01.309890+0000 mon.a (mon.0) 9 : audit [INF] from='client.? 172.21.15.78:0/841649602' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2023-10-01T10:05:01.913 INFO:teuthology.orchestra.run.smithi078.stderr:Generating new minimal ceph.conf... 2023-10-01T10:05:03.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:03 smithi078 bash[26790]: audit 2023-10-01T10:05:02.973508+0000 mon.a (mon.0) 10 : audit [DBG] from='client.? 172.21.15.78:0/2901384586' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:05:03.602 INFO:teuthology.orchestra.run.smithi078.stderr:Restarting the monitor... 2023-10-01T10:05:03.966 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:03 smithi078 systemd[1]: Stopping Ceph mon.a for d8016480-6041-11ee-8db5-212e2dc638e7... 2023-10-01T10:05:03.966 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:03 smithi078 bash[27193]: Error response from daemon: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-mon.a 2023-10-01T10:05:03.966 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:03 smithi078 bash[26790]: debug 2023-10-01T10:05:03.768+0000 7f46f5acf700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-10-01T10:05:03.966 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:03 smithi078 bash[26790]: debug 2023-10-01T10:05:03.768+0000 7f46f5acf700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2023-10-01T10:05:04.519 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:04 smithi078 bash[27230]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-mon-a 2023-10-01T10:05:04.538 INFO:teuthology.orchestra.run.smithi078.stderr:Setting mon public_network to 172.21.15.254/32,172.21.0.0/20 2023-10-01T10:05:04.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:04 smithi078 bash[27294]: Error response from daemon: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-mon-a 2023-10-01T10:05:04.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:04 smithi078 systemd[1]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7@mon.a.service: Succeeded. 2023-10-01T10:05:04.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:04 smithi078 systemd[1]: Stopped Ceph mon.a for d8016480-6041-11ee-8db5-212e2dc638e7. 2023-10-01T10:05:04.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:04 smithi078 systemd[1]: Started Ceph mon.a for d8016480-6041-11ee-8db5-212e2dc638e7. 2023-10-01T10:05:06.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.768+0000 7f54e5698880 0 set uid:gid to 167:167 (ceph:ceph) 2023-10-01T10:05:06.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.768+0000 7f54e5698880 0 ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable), process ceph-mon, pid 7 2023-10-01T10:05:06.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.768+0000 7f54e5698880 0 pidfile_write: ignore empty --pid-file 2023-10-01T10:05:06.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 0 load: jerasure load: lrc load: isa 2023-10-01T10:05:06.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: RocksDB version: 6.8.1 2023-10-01T10:05:06.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-10-01T10:05:06.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Compile date Sep 25 2023 2023-10-01T10:05:06.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: DB SUMMARY 2023-10-01T10:05:06.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: CURRENT file: CURRENT 2023-10-01T10:05:06.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: IDENTITY file: IDENTITY 2023-10-01T10:05:06.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: MANIFEST file: MANIFEST-000005 size: 131 Bytes 2023-10-01T10:05:06.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000004.sst 2023-10-01T10:05:06.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000006.log size: 86691 ; 2023-10-01T10:05:06.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.error_if_exists: 0 2023-10-01T10:05:06.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.create_if_missing: 0 2023-10-01T10:05:06.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.paranoid_checks: 1 2023-10-01T10:05:06.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.env: 0x5579775c1080 2023-10-01T10:05:06.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.fs: Posix File System 2023-10-01T10:05:06.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.info_log: 0x5579791e7a80 2023-10-01T10:05:06.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.max_file_opening_threads: 16 2023-10-01T10:05:06.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.statistics: (nil) 2023-10-01T10:05:06.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.use_fsync: 0 2023-10-01T10:05:06.085 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.max_log_file_size: 0 2023-10-01T10:05:06.085 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-01T10:05:06.085 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.log_file_time_to_roll: 0 2023-10-01T10:05:06.085 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.keep_log_file_num: 1000 2023-10-01T10:05:06.085 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.recycle_log_file_num: 0 2023-10-01T10:05:06.087 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.allow_fallocate: 1 2023-10-01T10:05:06.087 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.allow_mmap_reads: 0 2023-10-01T10:05:06.087 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.allow_mmap_writes: 0 2023-10-01T10:05:06.087 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.use_direct_reads: 0 2023-10-01T10:05:06.087 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-01T10:05:06.087 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.create_missing_column_families: 0 2023-10-01T10:05:06.087 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.db_log_dir: 2023-10-01T10:05:06.087 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2023-10-01T10:05:06.088 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.table_cache_numshardbits: 6 2023-10-01T10:05:06.088 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.max_subcompactions: 1 2023-10-01T10:05:06.088 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.max_background_flushes: -1 2023-10-01T10:05:06.088 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-10-01T10:05:06.088 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-10-01T10:05:06.088 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-01T10:05:06.088 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-01T10:05:06.088 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-10-01T10:05:06.089 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.advise_random_on_open: 1 2023-10-01T10:05:06.089 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.db_write_buffer_size: 0 2023-10-01T10:05:06.089 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.write_buffer_manager: 0x557979ee4330 2023-10-01T10:05:06.089 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-01T10:05:06.089 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-10-01T10:05:06.089 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-01T10:05:06.089 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.use_adaptive_mutex: 0 2023-10-01T10:05:06.089 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.rate_limiter: (nil) 2023-10-01T10:05:06.089 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-01T10:05:06.090 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.wal_recovery_mode: 2 2023-10-01T10:05:06.090 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.enable_thread_tracking: 0 2023-10-01T10:05:06.090 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.enable_pipelined_write: 0 2023-10-01T10:05:06.090 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.unordered_write: 0 2023-10-01T10:05:06.090 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-01T10:05:06.090 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-01T10:05:06.090 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-01T10:05:06.090 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-01T10:05:06.090 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.row_cache: None 2023-10-01T10:05:06.091 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.wal_filter: None 2023-10-01T10:05:06.091 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-01T10:05:06.091 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.allow_ingest_behind: 0 2023-10-01T10:05:06.091 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.preserve_deletes: 0 2023-10-01T10:05:06.091 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.two_write_queues: 0 2023-10-01T10:05:06.091 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.manual_wal_flush: 0 2023-10-01T10:05:06.091 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.atomic_flush: 0 2023-10-01T10:05:06.091 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-01T10:05:06.091 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.persist_stats_to_disk: 0 2023-10-01T10:05:06.092 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-10-01T10:05:06.092 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.log_readahead_size: 0 2023-10-01T10:05:06.092 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.sst_file_checksum_func: Unknown 2023-10-01T10:05:06.092 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.max_background_jobs: 2 2023-10-01T10:05:06.092 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.max_background_compactions: -1 2023-10-01T10:05:06.092 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-01T10:05:06.092 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-01T10:05:06.092 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.delayed_write_rate : 16777216 2023-10-01T10:05:06.092 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.max_total_wal_size: 0 2023-10-01T10:05:06.093 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-01T10:05:06.093 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.stats_dump_period_sec: 600 2023-10-01T10:05:06.093 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.stats_persist_period_sec: 600 2023-10-01T10:05:06.093 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-01T10:05:06.093 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.max_open_files: -1 2023-10-01T10:05:06.093 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.bytes_per_sync: 0 2023-10-01T10:05:06.093 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-10-01T10:05:06.093 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-10-01T10:05:06.094 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.compaction_readahead_size: 0 2023-10-01T10:05:06.094 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Compression algorithms supported: 2023-10-01T10:05:06.094 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-01T10:05:06.094 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: kZSTD supported: 0 2023-10-01T10:05:06.094 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: kXpressCompression supported: 0 2023-10-01T10:05:06.094 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: kLZ4HCCompression supported: 1 2023-10-01T10:05:06.094 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: kLZ4Compression supported: 1 2023-10-01T10:05:06.094 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: kBZip2Compression supported: 0 2023-10-01T10:05:06.095 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: kZlibCompression supported: 1 2023-10-01T10:05:06.095 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: kSnappyCompression supported: 1 2023-10-01T10:05:06.095 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-10-01T10:05:06.095 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000005 2023-10-01T10:05:06.095 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2023-10-01T10:05:06.095 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-01T10:05:06.095 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.merge_operator: 2023-10-01T10:05:06.095 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.compaction_filter: None 2023-10-01T10:05:06.096 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.compaction_filter_factory: None 2023-10-01T10:05:06.096 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-10-01T10:05:06.096 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.table_factory: BlockBasedTable 2023-10-01T10:05:06.096 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x557979195d18) 2023-10-01T10:05:06.096 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: cache_index_and_filter_blocks: 1 2023-10-01T10:05:06.096 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-01T10:05:06.096 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-01T10:05:06.096 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: pin_top_level_index_and_filter: 1 2023-10-01T10:05:06.097 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: index_type: 0 2023-10-01T10:05:06.097 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: data_block_index_type: 0 2023-10-01T10:05:06.097 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: index_shortening: 1 2023-10-01T10:05:06.097 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: data_block_hash_table_util_ratio: 0.750000 2023-10-01T10:05:06.097 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: hash_index_allow_collision: 1 2023-10-01T10:05:06.097 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: checksum: 1 2023-10-01T10:05:06.097 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: no_block_cache: 0 2023-10-01T10:05:06.097 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: block_cache: 0x5579791ccf10 2023-10-01T10:05:06.098 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: block_cache_name: BinnedLRUCache 2023-10-01T10:05:06.098 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: block_cache_options: 2023-10-01T10:05:06.098 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: capacity : 536870912 2023-10-01T10:05:06.098 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: num_shard_bits : 4 2023-10-01T10:05:06.098 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: strict_capacity_limit : 0 2023-10-01T10:05:06.098 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: high_pri_pool_ratio: 0.000 2023-10-01T10:05:06.098 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: block_cache_compressed: (nil) 2023-10-01T10:05:06.098 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: persistent_cache: (nil) 2023-10-01T10:05:06.099 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: block_size: 4096 2023-10-01T10:05:06.099 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: block_size_deviation: 10 2023-10-01T10:05:06.099 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: block_restart_interval: 16 2023-10-01T10:05:06.099 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: index_block_restart_interval: 1 2023-10-01T10:05:06.099 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: metadata_block_size: 4096 2023-10-01T10:05:06.099 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: partition_filters: 0 2023-10-01T10:05:06.099 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: use_delta_encoding: 1 2023-10-01T10:05:06.099 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: filter_policy: rocksdb.BuiltinBloomFilter 2023-10-01T10:05:06.100 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: whole_key_filtering: 1 2023-10-01T10:05:06.100 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: verify_compression: 0 2023-10-01T10:05:06.100 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: read_amp_bytes_per_bit: 0 2023-10-01T10:05:06.100 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: format_version: 2 2023-10-01T10:05:06.100 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: enable_index_compression: 1 2023-10-01T10:05:06.100 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: block_align: 0 2023-10-01T10:05:06.100 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.write_buffer_size: 33554432 2023-10-01T10:05:06.100 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.max_write_buffer_number: 2 2023-10-01T10:05:06.101 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.compression: NoCompression 2023-10-01T10:05:06.101 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.bottommost_compression: Disabled 2023-10-01T10:05:06.101 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.prefix_extractor: nullptr 2023-10-01T10:05:06.101 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-01T10:05:06.101 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.num_levels: 7 2023-10-01T10:05:06.101 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-01T10:05:06.101 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-01T10:05:06.101 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-01T10:05:06.101 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-01T10:05:06.102 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-01T10:05:06.102 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-01T10:05:06.102 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-01T10:05:06.102 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-01T10:05:06.102 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-01T10:05:06.102 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.compression_opts.window_bits: -14 2023-10-01T10:05:06.102 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.compression_opts.level: 32767 2023-10-01T10:05:06.102 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.compression_opts.strategy: 0 2023-10-01T10:05:06.103 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-01T10:05:06.103 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-01T10:05:06.103 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.compression_opts.enabled: false 2023-10-01T10:05:06.103 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-01T10:05:06.103 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-01T10:05:06.103 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-01T10:05:06.103 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.target_file_size_base: 67108864 2023-10-01T10:05:06.103 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.target_file_size_multiplier: 1 2023-10-01T10:05:06.104 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-01T10:05:06.104 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-01T10:05:06.104 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-01T10:05:06.104 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-01T10:05:06.104 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-01T10:05:06.104 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-01T10:05:06.104 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-01T10:05:06.104 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-01T10:05:06.105 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-01T10:05:06.105 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-01T10:05:06.105 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-01T10:05:06.105 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-01T10:05:06.105 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.arena_block_size: 4194304 2023-10-01T10:05:06.105 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-01T10:05:06.105 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-01T10:05:06.105 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-10-01T10:05:06.105 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.disable_auto_compactions: 0 2023-10-01T10:05:06.106 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-01T10:05:06.106 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-01T10:05:06.106 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-01T10:05:06.106 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-01T10:05:06.106 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-01T10:05:06.106 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-01T10:05:06.106 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-01T10:05:06.106 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-01T10:05:06.106 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-01T10:05:06.107 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-01T10:05:06.107 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.table_properties_collectors: 2023-10-01T10:05:06.107 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.inplace_update_support: 0 2023-10-01T10:05:06.107 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-10-01T10:05:06.107 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-01T10:05:06.107 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-01T10:05:06.107 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.memtable_huge_page_size: 0 2023-10-01T10:05:06.107 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.bloom_locality: 0 2023-10-01T10:05:06.107 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.max_successive_merges: 0 2023-10-01T10:05:06.107 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-10-01T10:05:06.107 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.paranoid_file_checks: 0 2023-10-01T10:05:06.107 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.force_consistency_checks: 0 2023-10-01T10:05:06.108 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.report_bg_io_stats: 0 2023-10-01T10:05:06.108 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.ttl: 2592000 2023-10-01T10:05:06.108 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-10-01T10:05:06.108 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 7, last_sequence is 5, log_number is 4,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-01T10:05:06.108 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 4 2023-10-01T10:05:06.108 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696154705776797, "job": 1, "event": "recovery_started", "log_files": [6]} 2023-10-01T10:05:06.108 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.772+0000 7f54e5698880 4 rocksdb: [db_impl/db_impl_open.cc:760] Recovering log #6 mode 2 2023-10-01T10:05:06.108 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.776+0000 7f54e5698880 3 rocksdb: [le/block_based/filter_policy.cc:584] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-10-01T10:05:06.108 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.776+0000 7f54e5698880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696154705777971, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 7, "file_size": 83291, "table_properties": {"data_size": 81564, "index_size": 332, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 0, "index_value_is_delta_encoded": 0, "filter_size": 581, "raw_key_size": 9829, "raw_average_key_size": 47, "raw_value_size": 75885, "raw_average_value_size": 368, "num_data_blocks": 11, "num_entries": 206, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1696154705, "oldest_key_time": 3, "file_creation_time": 0}} 2023-10-01T10:05:06.108 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.776+0000 7f54e5698880 4 rocksdb: [version_set.cc:3826] Creating manifest 8 2023-10-01T10:05:06.108 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.776+0000 7f54e5698880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696154705778439, "job": 1, "event": "recovery_finished"} 2023-10-01T10:05:06.109 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.776+0000 7f54e5698880 4 rocksdb: DB pointer 0x557979295800 2023-10-01T10:05:06.109 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.776+0000 7f54cc4a0700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2023-10-01T10:05:06.109 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.776+0000 7f54cc4a0700 4 rocksdb: [db_impl/db_impl.cc:851] 2023-10-01T10:05:06.109 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: ** DB Stats ** 2023-10-01T10:05:06.109 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-01T10:05:06.109 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-01T10:05:06.109 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-01T10:05:06.109 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-01T10:05:06.109 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-01T10:05:06.109 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-10-01T10:05:06.109 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-01T10:05:06.109 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: ** Compaction Stats [default] ** 2023-10-01T10:05:06.109 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-01T10:05:06.110 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-01T10:05:06.110 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: L0 2/0 82.99 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 98.9 0.00 0.00 1 0.001 0 0 2023-10-01T10:05:06.110 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: Sum 2/0 82.99 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 98.9 0.00 0.00 1 0.001 0 0 2023-10-01T10:05:06.110 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 98.9 0.00 0.00 1 0.001 0 0 2023-10-01T10:05:06.110 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: ** Compaction Stats [default] ** 2023-10-01T10:05:06.110 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-01T10:05:06.110 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-01T10:05:06.110 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 98.9 0.00 0.00 1 0.001 0 0 2023-10-01T10:05:06.110 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-01T10:05:06.110 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: Flush(GB): cumulative 0.000, interval 0.000 2023-10-01T10:05:06.110 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-01T10:05:06.110 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: AddFile(Total Files): cumulative 0, interval 0 2023-10-01T10:05:06.111 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: AddFile(L0 Files): cumulative 0, interval 0 2023-10-01T10:05:06.111 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: AddFile(Keys): cumulative 0, interval 0 2023-10-01T10:05:06.111 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: Cumulative compaction: 0.00 GB write, 19.31 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-01T10:05:06.111 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: Interval compaction: 0.00 GB write, 19.31 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-01T10:05:06.111 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-01T10:05:06.111 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: ** File Read Latency Histogram By Level [default] ** 2023-10-01T10:05:06.111 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: ** Compaction Stats [default] ** 2023-10-01T10:05:06.111 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-01T10:05:06.112 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-01T10:05:06.112 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: L0 2/0 82.99 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 98.9 0.00 0.00 1 0.001 0 0 2023-10-01T10:05:06.112 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: Sum 2/0 82.99 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 98.9 0.00 0.00 1 0.001 0 0 2023-10-01T10:05:06.112 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-01T10:05:06.112 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: ** Compaction Stats [default] ** 2023-10-01T10:05:06.112 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-01T10:05:06.112 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-01T10:05:06.112 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 98.9 0.00 0.00 1 0.001 0 0 2023-10-01T10:05:06.112 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-01T10:05:06.112 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: Flush(GB): cumulative 0.000, interval 0.000 2023-10-01T10:05:06.112 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-01T10:05:06.112 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: AddFile(Total Files): cumulative 0, interval 0 2023-10-01T10:05:06.113 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: AddFile(L0 Files): cumulative 0, interval 0 2023-10-01T10:05:06.113 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: AddFile(Keys): cumulative 0, interval 0 2023-10-01T10:05:06.113 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: Cumulative compaction: 0.00 GB write, 19.07 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-01T10:05:06.113 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-01T10:05:06.113 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-01T10:05:06.113 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: ** File Read Latency Histogram By Level [default] ** 2023-10-01T10:05:06.113 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.776+0000 7f54e5698880 0 starting mon.a rank 0 at public addrs [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] at bind addrs [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid d8016480-6041-11ee-8db5-212e2dc638e7 2023-10-01T10:05:06.113 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.780+0000 7f54e5698880 1 mon.a@-1(???) e1 preinit fsid d8016480-6041-11ee-8db5-212e2dc638e7 2023-10-01T10:05:06.113 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.780+0000 7f54e5698880 0 mon.a@-1(???).mds e1 new map 2023-10-01T10:05:06.113 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.780+0000 7f54e5698880 0 mon.a@-1(???).mds e1 print_map 2023-10-01T10:05:06.113 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: e1 2023-10-01T10:05:06.113 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: enable_multiple, ever_enabled_multiple: 1,1 2023-10-01T10:05:06.114 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-01T10:05:06.114 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: legacy client fscid: -1 2023-10-01T10:05:06.114 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: 2023-10-01T10:05:06.114 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: No filesystems configured 2023-10-01T10:05:06.114 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.780+0000 7f54e5698880 0 mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2023-10-01T10:05:06.114 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.780+0000 7f54e5698880 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-10-01T10:05:06.114 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.780+0000 7f54e5698880 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-10-01T10:05:06.114 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.780+0000 7f54e5698880 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-10-01T10:05:06.114 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: audit 2023-10-01T10:05:02.973508+0000 mon.a (mon.0) 10 : audit [DBG] from='client.? 172.21.15.78:0/2901384586' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:05:06.114 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: debug 2023-10-01T10:05:05.780+0000 7f54e5698880 1 mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2023-10-01T10:05:06.114 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: cluster 2023-10-01T10:05:05.786090+0000 mon.a (mon.0) 1 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2023-10-01T10:05:06.114 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: cluster 2023-10-01T10:05:05.786161+0000 mon.a (mon.0) 2 : cluster [DBG] monmap e1: 1 mons at {a=[v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0]} removed_ranks: {} 2023-10-01T10:05:06.115 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: cluster 2023-10-01T10:05:05.809684+0000 mon.a (mon.0) 3 : cluster [DBG] fsmap 2023-10-01T10:05:06.115 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: cluster 2023-10-01T10:05:05.809721+0000 mon.a (mon.0) 4 : cluster [DBG] osdmap e1: 0 total, 0 up, 0 in 2023-10-01T10:05:06.115 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:05 smithi078 bash[27336]: cluster 2023-10-01T10:05:05.810140+0000 mon.a (mon.0) 5 : cluster [DBG] mgrmap e1: no daemons active 2023-10-01T10:05:06.695 INFO:teuthology.orchestra.run.smithi078.stderr:Wrote config to /etc/ceph/ceph.conf 2023-10-01T10:05:06.695 INFO:teuthology.orchestra.run.smithi078.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2023-10-01T10:05:06.695 INFO:teuthology.orchestra.run.smithi078.stderr:Creating mgr... 2023-10-01T10:05:06.695 INFO:teuthology.orchestra.run.smithi078.stderr:Verifying port 9283 ... 2023-10-01T10:05:06.927 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: Failed to reset failed state of unit ceph-d8016480-6041-11ee-8db5-212e2dc638e7@mgr.y.service: Unit ceph-d8016480-6041-11ee-8db5-212e2dc638e7@mgr.y.service not loaded. 2023-10-01T10:05:06.936 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: Created symlink /etc/systemd/system/ceph-d8016480-6041-11ee-8db5-212e2dc638e7.target.wants/ceph-d8016480-6041-11ee-8db5-212e2dc638e7@mgr.y.service -> /etc/systemd/system/ceph-d8016480-6041-11ee-8db5-212e2dc638e7@.service. 2023-10-01T10:05:07.145 INFO:teuthology.orchestra.run.smithi078.stderr:firewalld does not appear to be present 2023-10-01T10:05:07.145 INFO:teuthology.orchestra.run.smithi078.stderr:Not possible to enable service . firewalld.service is not available 2023-10-01T10:05:07.145 INFO:teuthology.orchestra.run.smithi078.stderr:firewalld does not appear to be present 2023-10-01T10:05:07.145 INFO:teuthology.orchestra.run.smithi078.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2023-10-01T10:05:07.145 INFO:teuthology.orchestra.run.smithi078.stderr:Waiting for mgr to start... 2023-10-01T10:05:07.145 INFO:teuthology.orchestra.run.smithi078.stderr:Waiting for mgr... 2023-10-01T10:05:07.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:07 smithi078 bash[27336]: audit 2023-10-01T10:05:06.032636+0000 mon.a (mon.0) 6 : audit [INF] from='client.? 172.21.15.78:0/237801738' entity='client.admin' 2023-10-01T10:05:08.763 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2023-10-01T10:05:08.763 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: { 2023-10-01T10:05:08.764 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "fsid": "d8016480-6041-11ee-8db5-212e2dc638e7", 2023-10-01T10:05:08.764 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "health": { 2023-10-01T10:05:08.764 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2023-10-01T10:05:08.764 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "checks": {}, 2023-10-01T10:05:08.764 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "mutes": [] 2023-10-01T10:05:08.764 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2023-10-01T10:05:08.764 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "election_epoch": 5, 2023-10-01T10:05:08.764 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "quorum": [ 2023-10-01T10:05:08.765 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 0 2023-10-01T10:05:08.765 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ], 2023-10-01T10:05:08.765 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "quorum_names": [ 2023-10-01T10:05:08.765 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "a" 2023-10-01T10:05:08.765 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ], 2023-10-01T10:05:08.765 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "quorum_age": 2, 2023-10-01T10:05:08.765 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "monmap": { 2023-10-01T10:05:08.765 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-01T10:05:08.765 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2023-10-01T10:05:08.765 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_mons": 1 2023-10-01T10:05:08.765 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2023-10-01T10:05:08.766 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "osdmap": { 2023-10-01T10:05:08.766 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-01T10:05:08.766 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_osds": 0, 2023-10-01T10:05:08.766 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_up_osds": 0, 2023-10-01T10:05:08.766 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "osd_up_since": 0, 2023-10-01T10:05:08.766 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_in_osds": 0, 2023-10-01T10:05:08.769 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "osd_in_since": 0, 2023-10-01T10:05:08.769 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2023-10-01T10:05:08.770 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2023-10-01T10:05:08.770 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "pgmap": { 2023-10-01T10:05:08.770 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "pgs_by_state": [], 2023-10-01T10:05:08.770 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_pgs": 0, 2023-10-01T10:05:08.770 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_pools": 0, 2023-10-01T10:05:08.770 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_objects": 0, 2023-10-01T10:05:08.770 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "data_bytes": 0, 2023-10-01T10:05:08.770 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "bytes_used": 0, 2023-10-01T10:05:08.770 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "bytes_avail": 0, 2023-10-01T10:05:08.770 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "bytes_total": 0 2023-10-01T10:05:08.771 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2023-10-01T10:05:08.771 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "fsmap": { 2023-10-01T10:05:08.771 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-01T10:05:08.771 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "by_rank": [], 2023-10-01T10:05:08.771 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "up:standby": 0 2023-10-01T10:05:08.771 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2023-10-01T10:05:08.771 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "mgrmap": { 2023-10-01T10:05:08.771 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "available": false, 2023-10-01T10:05:08.771 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_standbys": 0, 2023-10-01T10:05:08.772 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "modules": [ 2023-10-01T10:05:08.772 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "iostat", 2023-10-01T10:05:08.772 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "nfs", 2023-10-01T10:05:08.772 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "restful" 2023-10-01T10:05:08.772 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ], 2023-10-01T10:05:08.772 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "services": {} 2023-10-01T10:05:08.772 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2023-10-01T10:05:08.772 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "servicemap": { 2023-10-01T10:05:08.772 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-01T10:05:08.773 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "modified": "2023-10-01T10:04:59.346620+0000", 2023-10-01T10:05:08.773 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "services": {} 2023-10-01T10:05:08.773 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2023-10-01T10:05:08.773 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "progress_events": {} 2023-10-01T10:05:08.773 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: } 2023-10-01T10:05:08.961 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:08 smithi078 bash[27336]: audit 2023-10-01T10:05:08.758542+0000 mon.a (mon.0) 7 : audit [DBG] from='client.? 172.21.15.78:0/2100518622' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-10-01T10:05:08.961 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:08 smithi078 bash[27601]: debug 2023-10-01T10:05:08.803+0000 7fb010cc0680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-01T10:05:09.386 INFO:teuthology.orchestra.run.smithi078.stderr:mgr not available, waiting (1/15)... 2023-10-01T10:05:09.582 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:09 smithi078 bash[27601]: debug 2023-10-01T10:05:09.275+0000 7fb010cc0680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-01T10:05:10.537 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:10 smithi078 bash[27601]: debug 2023-10-01T10:05:10.191+0000 7fb010cc0680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-01T10:05:10.537 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:10 smithi078 bash[27601]: debug 2023-10-01T10:05:10.279+0000 7fb010cc0680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-01T10:05:11.674 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:10 smithi078 bash[27601]: debug 2023-10-01T10:05:10.531+0000 7fb010cc0680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-01T10:05:11.927 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:11 smithi078 bash[27601]: debug 2023-10-01T10:05:11.667+0000 7fb010cc0680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-01T10:05:11.927 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:11 smithi078 bash[27601]: debug 2023-10-01T10:05:11.827+0000 7fb010cc0680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-01T10:05:12.222 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:11 smithi078 bash[27601]: debug 2023-10-01T10:05:11.919+0000 7fb010cc0680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-01T10:05:12.222 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:12 smithi078 bash[27601]: debug 2023-10-01T10:05:12.031+0000 7fb010cc0680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-01T10:05:12.582 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:12 smithi078 bash[27601]: debug 2023-10-01T10:05:12.259+0000 7fb010cc0680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-01T10:05:12.585 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2023-10-01T10:05:12.586 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: { 2023-10-01T10:05:12.586 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "fsid": "d8016480-6041-11ee-8db5-212e2dc638e7", 2023-10-01T10:05:12.586 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "health": { 2023-10-01T10:05:12.586 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2023-10-01T10:05:12.587 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "checks": {}, 2023-10-01T10:05:12.587 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "mutes": [] 2023-10-01T10:05:12.587 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2023-10-01T10:05:12.587 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "election_epoch": 5, 2023-10-01T10:05:12.587 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "quorum": [ 2023-10-01T10:05:12.587 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 0 2023-10-01T10:05:12.587 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ], 2023-10-01T10:05:12.587 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "quorum_names": [ 2023-10-01T10:05:12.589 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "a" 2023-10-01T10:05:12.589 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ], 2023-10-01T10:05:12.589 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "quorum_age": 6, 2023-10-01T10:05:12.589 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "monmap": { 2023-10-01T10:05:12.589 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-01T10:05:12.589 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2023-10-01T10:05:12.589 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_mons": 1 2023-10-01T10:05:12.589 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2023-10-01T10:05:12.590 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "osdmap": { 2023-10-01T10:05:12.590 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-01T10:05:12.590 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_osds": 0, 2023-10-01T10:05:12.590 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_up_osds": 0, 2023-10-01T10:05:12.590 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "osd_up_since": 0, 2023-10-01T10:05:12.593 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_in_osds": 0, 2023-10-01T10:05:12.593 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "osd_in_since": 0, 2023-10-01T10:05:12.593 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2023-10-01T10:05:12.593 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2023-10-01T10:05:12.593 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "pgmap": { 2023-10-01T10:05:12.594 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "pgs_by_state": [], 2023-10-01T10:05:12.594 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_pgs": 0, 2023-10-01T10:05:12.594 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_pools": 0, 2023-10-01T10:05:12.594 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_objects": 0, 2023-10-01T10:05:12.594 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "data_bytes": 0, 2023-10-01T10:05:12.594 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "bytes_used": 0, 2023-10-01T10:05:12.594 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "bytes_avail": 0, 2023-10-01T10:05:12.594 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "bytes_total": 0 2023-10-01T10:05:12.594 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2023-10-01T10:05:12.594 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "fsmap": { 2023-10-01T10:05:12.594 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-01T10:05:12.595 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "by_rank": [], 2023-10-01T10:05:12.595 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "up:standby": 0 2023-10-01T10:05:12.595 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2023-10-01T10:05:12.595 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "mgrmap": { 2023-10-01T10:05:12.595 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "available": false, 2023-10-01T10:05:12.595 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_standbys": 0, 2023-10-01T10:05:12.595 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "modules": [ 2023-10-01T10:05:12.595 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "iostat", 2023-10-01T10:05:12.595 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "nfs", 2023-10-01T10:05:12.595 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "restful" 2023-10-01T10:05:12.595 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ], 2023-10-01T10:05:12.595 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "services": {} 2023-10-01T10:05:12.596 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2023-10-01T10:05:12.596 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "servicemap": { 2023-10-01T10:05:12.596 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-01T10:05:12.596 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "modified": "2023-10-01T10:04:59.346620+0000", 2023-10-01T10:05:12.596 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "services": {} 2023-10-01T10:05:12.596 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2023-10-01T10:05:12.596 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "progress_events": {} 2023-10-01T10:05:12.596 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: } 2023-10-01T10:05:13.082 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:12 smithi078 bash[27601]: debug 2023-10-01T10:05:12.695+0000 7fb010cc0680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-01T10:05:13.082 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:12 smithi078 bash[27601]: debug 2023-10-01T10:05:12.815+0000 7fb010cc0680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-01T10:05:13.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:12 smithi078 bash[27336]: audit 2023-10-01T10:05:12.582490+0000 mon.a (mon.0) 8 : audit [DBG] from='client.? 172.21.15.78:0/1408215335' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-10-01T10:05:13.213 INFO:teuthology.orchestra.run.smithi078.stderr:mgr not available, waiting (2/15)... 2023-10-01T10:05:14.082 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:13 smithi078 bash[27601]: debug 2023-10-01T10:05:13.667+0000 7fb010cc0680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-01T10:05:14.082 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:13 smithi078 bash[27601]: debug 2023-10-01T10:05:13.775+0000 7fb010cc0680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-01T10:05:14.082 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:13 smithi078 bash[27601]: debug 2023-10-01T10:05:13.891+0000 7fb010cc0680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-01T10:05:14.406 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:14 smithi078 bash[27601]: debug 2023-10-01T10:05:14.107+0000 7fb010cc0680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-01T10:05:14.740 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:14 smithi078 bash[27601]: debug 2023-10-01T10:05:14.399+0000 7fb010cc0680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-01T10:05:14.741 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:14 smithi078 bash[27601]: debug 2023-10-01T10:05:14.563+0000 7fb010cc0680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-01T10:05:15.060 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:14 smithi078 bash[27601]: debug 2023-10-01T10:05:14.735+0000 7fb010cc0680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-01T10:05:15.060 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:14 smithi078 bash[27601]: debug 2023-10-01T10:05:14.947+0000 7fb010cc0680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-01T10:05:15.332 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:15 smithi078 bash[27601]: debug 2023-10-01T10:05:15.055+0000 7fb010cc0680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-01T10:05:15.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:15 smithi078 bash[27336]: cluster 2023-10-01T10:05:15.079481+0000 mon.a (mon.0) 9 : cluster [INF] Activating manager daemon y 2023-10-01T10:05:15.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:15 smithi078 bash[27336]: cluster 2023-10-01T10:05:15.086283+0000 mon.a (mon.0) 10 : cluster [DBG] mgrmap e2: y(active, starting, since 0.007013s) 2023-10-01T10:05:15.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:15 smithi078 bash[27336]: audit 2023-10-01T10:05:15.087766+0000 mon.a (mon.0) 11 : audit [DBG] from='mgr.14100 172.21.15.78:0/132139390' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-01T10:05:15.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:15 smithi078 bash[27336]: audit 2023-10-01T10:05:15.088109+0000 mon.a (mon.0) 12 : audit [DBG] from='mgr.14100 172.21.15.78:0/132139390' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-01T10:05:15.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:15 smithi078 bash[27336]: audit 2023-10-01T10:05:15.088418+0000 mon.a (mon.0) 13 : audit [DBG] from='mgr.14100 172.21.15.78:0/132139390' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-01T10:05:15.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:15 smithi078 bash[27336]: audit 2023-10-01T10:05:15.088724+0000 mon.a (mon.0) 14 : audit [DBG] from='mgr.14100 172.21.15.78:0/132139390' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T10:05:15.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:15 smithi078 bash[27336]: audit 2023-10-01T10:05:15.089046+0000 mon.a (mon.0) 15 : audit [DBG] from='mgr.14100 172.21.15.78:0/132139390' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-01T10:05:15.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:15 smithi078 bash[27336]: cluster 2023-10-01T10:05:15.092723+0000 mon.a (mon.0) 16 : cluster [INF] Manager daemon y is now available 2023-10-01T10:05:15.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:15 smithi078 bash[27336]: audit 2023-10-01T10:05:15.105811+0000 mon.a (mon.0) 17 : audit [INF] from='mgr.14100 172.21.15.78:0/132139390' entity='mgr.y' 2023-10-01T10:05:15.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:15 smithi078 bash[27336]: audit 2023-10-01T10:05:15.114022+0000 mon.a (mon.0) 18 : audit [INF] from='mgr.14100 172.21.15.78:0/132139390' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-01T10:05:15.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:15 smithi078 bash[27336]: audit 2023-10-01T10:05:15.116529+0000 mon.a (mon.0) 19 : audit [INF] from='mgr.14100 172.21.15.78:0/132139390' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-01T10:05:15.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:15 smithi078 bash[27336]: audit 2023-10-01T10:05:15.119855+0000 mon.a (mon.0) 20 : audit [INF] from='mgr.14100 172.21.15.78:0/132139390' entity='mgr.y' 2023-10-01T10:05:15.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:15 smithi078 bash[27336]: audit 2023-10-01T10:05:15.123705+0000 mon.a (mon.0) 21 : audit [INF] from='mgr.14100 172.21.15.78:0/132139390' entity='mgr.y' 2023-10-01T10:05:16.662 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2023-10-01T10:05:16.662 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: { 2023-10-01T10:05:16.662 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "fsid": "d8016480-6041-11ee-8db5-212e2dc638e7", 2023-10-01T10:05:16.662 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "health": { 2023-10-01T10:05:16.662 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2023-10-01T10:05:16.662 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "checks": {}, 2023-10-01T10:05:16.662 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "mutes": [] 2023-10-01T10:05:16.664 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2023-10-01T10:05:16.665 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "election_epoch": 5, 2023-10-01T10:05:16.665 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "quorum": [ 2023-10-01T10:05:16.665 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 0 2023-10-01T10:05:16.665 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ], 2023-10-01T10:05:16.665 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "quorum_names": [ 2023-10-01T10:05:16.665 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "a" 2023-10-01T10:05:16.665 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ], 2023-10-01T10:05:16.665 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "quorum_age": 10, 2023-10-01T10:05:16.665 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "monmap": { 2023-10-01T10:05:16.665 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-01T10:05:16.666 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2023-10-01T10:05:16.666 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_mons": 1 2023-10-01T10:05:16.666 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2023-10-01T10:05:16.666 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "osdmap": { 2023-10-01T10:05:16.666 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-01T10:05:16.666 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_osds": 0, 2023-10-01T10:05:16.666 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_up_osds": 0, 2023-10-01T10:05:16.666 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "osd_up_since": 0, 2023-10-01T10:05:16.669 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_in_osds": 0, 2023-10-01T10:05:16.670 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "osd_in_since": 0, 2023-10-01T10:05:16.670 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2023-10-01T10:05:16.670 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2023-10-01T10:05:16.670 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "pgmap": { 2023-10-01T10:05:16.670 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "pgs_by_state": [], 2023-10-01T10:05:16.670 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_pgs": 0, 2023-10-01T10:05:16.670 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_pools": 0, 2023-10-01T10:05:16.670 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_objects": 0, 2023-10-01T10:05:16.670 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "data_bytes": 0, 2023-10-01T10:05:16.671 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "bytes_used": 0, 2023-10-01T10:05:16.671 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "bytes_avail": 0, 2023-10-01T10:05:16.671 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "bytes_total": 0 2023-10-01T10:05:16.671 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2023-10-01T10:05:16.671 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "fsmap": { 2023-10-01T10:05:16.671 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-01T10:05:16.671 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "by_rank": [], 2023-10-01T10:05:16.671 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "up:standby": 0 2023-10-01T10:05:16.671 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2023-10-01T10:05:16.671 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "mgrmap": { 2023-10-01T10:05:16.671 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "available": true, 2023-10-01T10:05:16.671 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_standbys": 0, 2023-10-01T10:05:16.671 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "modules": [ 2023-10-01T10:05:16.672 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "iostat", 2023-10-01T10:05:16.672 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "nfs", 2023-10-01T10:05:16.672 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "restful" 2023-10-01T10:05:16.672 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ], 2023-10-01T10:05:16.672 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "services": {} 2023-10-01T10:05:16.672 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2023-10-01T10:05:16.672 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "servicemap": { 2023-10-01T10:05:16.672 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2023-10-01T10:05:16.672 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "modified": "2023-10-01T10:04:59.346620+0000", 2023-10-01T10:05:16.672 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "services": {} 2023-10-01T10:05:16.672 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2023-10-01T10:05:16.672 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "progress_events": {} 2023-10-01T10:05:16.673 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: } 2023-10-01T10:05:17.262 INFO:teuthology.orchestra.run.smithi078.stderr:mgr is available 2023-10-01T10:05:17.290 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:17 smithi078 bash[27336]: cluster 2023-10-01T10:05:16.087414+0000 mon.a (mon.0) 22 : cluster [DBG] mgrmap e3: y(active, since 1.00815s) 2023-10-01T10:05:17.290 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:17 smithi078 bash[27336]: audit 2023-10-01T10:05:16.658414+0000 mon.a (mon.0) 23 : audit [DBG] from='client.? 172.21.15.78:0/16389465' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-10-01T10:05:18.543 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2023-10-01T10:05:18.543 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: [global] 2023-10-01T10:05:18.543 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: fsid = d8016480-6041-11ee-8db5-212e2dc638e7 2023-10-01T10:05:18.544 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: log_to_journald = false 2023-10-01T10:05:18.544 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2023-10-01T10:05:18.544 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2023-10-01T10:05:18.544 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2023-10-01T10:05:18.544 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2023-10-01T10:05:18.544 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2023-10-01T10:05:18.544 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: [mgr] 2023-10-01T10:05:18.544 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2023-10-01T10:05:18.544 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2023-10-01T10:05:18.544 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: [osd] 2023-10-01T10:05:18.544 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2023-10-01T10:05:18.544 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2023-10-01T10:05:19.109 INFO:teuthology.orchestra.run.smithi078.stderr:Enabling cephadm module... 2023-10-01T10:05:19.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:19 smithi078 bash[27336]: cluster 2023-10-01T10:05:18.088527+0000 mon.a (mon.0) 24 : cluster [DBG] mgrmap e4: y(active, since 3s) 2023-10-01T10:05:19.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:19 smithi078 bash[27336]: audit 2023-10-01T10:05:18.538126+0000 mon.a (mon.0) 25 : audit [INF] from='client.? 172.21.15.78:0/3239883806' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-10-01T10:05:20.824 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:20 smithi078 bash[27336]: audit 2023-10-01T10:05:20.481712+0000 mon.a (mon.0) 26 : audit [INF] from='client.? 172.21.15.78:0/2207375002' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2023-10-01T10:05:20.824 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:20 smithi078 bash[27601]: ignoring --setuser ceph since I am not root 2023-10-01T10:05:20.824 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:20 smithi078 bash[27601]: ignoring --setgroup ceph since I am not root 2023-10-01T10:05:20.825 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:20 smithi078 bash[27601]: debug 2023-10-01T10:05:20.723+0000 7f8e24a0d680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-01T10:05:21.082 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:20 smithi078 bash[27601]: debug 2023-10-01T10:05:20.819+0000 7f8e24a0d680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-01T10:05:21.547 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:21 smithi078 bash[27601]: debug 2023-10-01T10:05:21.247+0000 7f8e24a0d680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-01T10:05:21.838 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:21 smithi078 bash[27336]: audit 2023-10-01T10:05:20.540517+0000 mon.a (mon.0) 27 : audit [INF] from='client.? 172.21.15.78:0/2207375002' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2023-10-01T10:05:21.839 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:21 smithi078 bash[27336]: cluster 2023-10-01T10:05:20.540592+0000 mon.a (mon.0) 28 : cluster [DBG] mgrmap e5: y(active, since 5s) 2023-10-01T10:05:22.182 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:22 smithi078 bash[27601]: debug 2023-10-01T10:05:22.175+0000 7f8e24a0d680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-01T10:05:22.534 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:22 smithi078 bash[27601]: debug 2023-10-01T10:05:22.279+0000 7f8e24a0d680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-01T10:05:22.626 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: { 2023-10-01T10:05:22.626 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 5, 2023-10-01T10:05:22.626 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "available": true, 2023-10-01T10:05:22.626 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "active_name": "y", 2023-10-01T10:05:22.629 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_standby": 0 2023-10-01T10:05:22.629 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: } 2023-10-01T10:05:22.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:22 smithi078 bash[27336]: audit 2023-10-01T10:05:22.621966+0000 mon.a (mon.0) 29 : audit [DBG] from='client.? 172.21.15.78:0/2502420845' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-10-01T10:05:22.833 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:22 smithi078 bash[27601]: debug 2023-10-01T10:05:22.527+0000 7f8e24a0d680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-01T10:05:23.236 INFO:teuthology.orchestra.run.smithi078.stderr:Waiting for the mgr to restart... 2023-10-01T10:05:23.237 INFO:teuthology.orchestra.run.smithi078.stderr:Waiting for mgr epoch 5... 2023-10-01T10:05:23.926 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:23 smithi078 bash[27601]: debug 2023-10-01T10:05:23.667+0000 7f8e24a0d680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-01T10:05:23.926 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:23 smithi078 bash[27601]: debug 2023-10-01T10:05:23.827+0000 7f8e24a0d680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-01T10:05:24.273 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:23 smithi078 bash[27601]: debug 2023-10-01T10:05:23.927+0000 7f8e24a0d680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-01T10:05:24.273 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:24 smithi078 bash[27601]: debug 2023-10-01T10:05:24.047+0000 7f8e24a0d680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-01T10:05:24.582 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:24 smithi078 bash[27601]: debug 2023-10-01T10:05:24.267+0000 7f8e24a0d680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-01T10:05:25.082 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:24 smithi078 bash[27601]: debug 2023-10-01T10:05:24.627+0000 7f8e24a0d680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-01T10:05:25.083 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:24 smithi078 bash[27601]: debug 2023-10-01T10:05:24.751+0000 7f8e24a0d680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-01T10:05:26.053 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:25 smithi078 bash[27601]: debug 2023-10-01T10:05:25.607+0000 7f8e24a0d680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-01T10:05:26.552 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:25 smithi078 bash[27601]: debug 2023-10-01T10:05:25.711+0000 7f8e24a0d680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-01T10:05:26.552 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:25 smithi078 bash[27601]: debug 2023-10-01T10:05:25.831+0000 7f8e24a0d680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-01T10:05:26.668 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:26 smithi078 bash[27601]: debug 2023-10-01T10:05:26.047+0000 7f8e24a0d680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-01T10:05:26.668 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:26 smithi078 bash[27601]: debug 2023-10-01T10:05:26.331+0000 7f8e24a0d680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-01T10:05:26.669 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:26 smithi078 bash[27601]: debug 2023-10-01T10:05:26.491+0000 7f8e24a0d680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-01T10:05:26.989 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:26 smithi078 bash[27601]: debug 2023-10-01T10:05:26.663+0000 7f8e24a0d680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-01T10:05:26.989 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:26 smithi078 bash[27601]: debug 2023-10-01T10:05:26.879+0000 7f8e24a0d680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-01T10:05:27.332 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:26 smithi078 bash[27601]: debug 2023-10-01T10:05:26.983+0000 7f8e24a0d680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-01T10:05:27.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:27 smithi078 bash[27336]: cluster 2023-10-01T10:05:27.007518+0000 mon.a (mon.0) 30 : cluster [INF] Active manager daemon y restarted 2023-10-01T10:05:27.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:27 smithi078 bash[27336]: cluster 2023-10-01T10:05:27.007907+0000 mon.a (mon.0) 31 : cluster [INF] Activating manager daemon y 2023-10-01T10:05:27.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:27 smithi078 bash[27336]: cluster 2023-10-01T10:05:27.014219+0000 mon.a (mon.0) 32 : cluster [DBG] osdmap e2: 0 total, 0 up, 0 in 2023-10-01T10:05:27.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:27 smithi078 bash[27336]: cluster 2023-10-01T10:05:27.014514+0000 mon.a (mon.0) 33 : cluster [DBG] mgrmap e6: y(active, starting, since 0.00673761s) 2023-10-01T10:05:27.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:27 smithi078 bash[27336]: audit 2023-10-01T10:05:27.015839+0000 mon.a (mon.0) 34 : audit [DBG] from='mgr.14116 172.21.15.78:0/3408980511' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T10:05:27.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:27 smithi078 bash[27336]: audit 2023-10-01T10:05:27.017356+0000 mon.a (mon.0) 35 : audit [DBG] from='mgr.14116 172.21.15.78:0/3408980511' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-01T10:05:27.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:27 smithi078 bash[27336]: audit 2023-10-01T10:05:27.018503+0000 mon.a (mon.0) 36 : audit [DBG] from='mgr.14116 172.21.15.78:0/3408980511' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-01T10:05:27.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:27 smithi078 bash[27336]: audit 2023-10-01T10:05:27.018768+0000 mon.a (mon.0) 37 : audit [DBG] from='mgr.14116 172.21.15.78:0/3408980511' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-01T10:05:27.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:27 smithi078 bash[27336]: audit 2023-10-01T10:05:27.018993+0000 mon.a (mon.0) 38 : audit [DBG] from='mgr.14116 172.21.15.78:0/3408980511' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-01T10:05:27.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:27 smithi078 bash[27336]: cluster 2023-10-01T10:05:27.022812+0000 mon.a (mon.0) 39 : cluster [INF] Manager daemon y is now available 2023-10-01T10:05:27.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:27 smithi078 bash[27336]: audit 2023-10-01T10:05:27.042238+0000 mon.a (mon.0) 40 : audit [INF] from='mgr.14116 172.21.15.78:0/3408980511' entity='mgr.y' 2023-10-01T10:05:28.035 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: { 2023-10-01T10:05:28.035 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2023-10-01T10:05:28.035 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "initialized": true 2023-10-01T10:05:28.035 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: } 2023-10-01T10:05:28.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:28 smithi078 bash[27336]: cephadm 2023-10-01T10:05:27.037606+0000 mgr.y (mgr.14116) 1 : cephadm [INF] Found migration_current of "None". Setting to last migration. 2023-10-01T10:05:28.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:28 smithi078 bash[27336]: audit 2023-10-01T10:05:27.167094+0000 mon.a (mon.0) 41 : audit [INF] from='mgr.14116 172.21.15.78:0/3408980511' entity='mgr.y' 2023-10-01T10:05:28.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:28 smithi078 bash[27336]: audit 2023-10-01T10:05:27.169171+0000 mon.a (mon.0) 42 : audit [DBG] from='mgr.14116 172.21.15.78:0/3408980511' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:05:28.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:28 smithi078 bash[27336]: audit 2023-10-01T10:05:27.178462+0000 mon.a (mon.0) 43 : audit [DBG] from='mgr.14116 172.21.15.78:0/3408980511' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:05:28.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:28 smithi078 bash[27336]: audit 2023-10-01T10:05:27.181223+0000 mon.a (mon.0) 44 : audit [DBG] from='mgr.14116 172.21.15.78:0/3408980511' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:05:28.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:28 smithi078 bash[27336]: audit 2023-10-01T10:05:27.198655+0000 mon.a (mon.0) 45 : audit [INF] from='mgr.14116 172.21.15.78:0/3408980511' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-01T10:05:28.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:28 smithi078 bash[27336]: audit 2023-10-01T10:05:27.202169+0000 mon.a (mon.0) 46 : audit [INF] from='mgr.14116 172.21.15.78:0/3408980511' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-01T10:05:28.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:28 smithi078 bash[27336]: cluster 2023-10-01T10:05:28.017870+0000 mon.a (mon.0) 47 : cluster [DBG] mgrmap e7: y(active, since 1.01012s) 2023-10-01T10:05:28.709 INFO:teuthology.orchestra.run.smithi078.stderr:mgr epoch 5 is available 2023-10-01T10:05:28.709 INFO:teuthology.orchestra.run.smithi078.stderr:Setting orchestrator backend to cephadm... 2023-10-01T10:05:29.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:29 smithi078 bash[27336]: audit 2023-10-01T10:05:28.018112+0000 mgr.y (mgr.14116) 2 : audit [DBG] from='client.14120 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-10-01T10:05:29.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:29 smithi078 bash[27336]: audit 2023-10-01T10:05:28.032091+0000 mgr.y (mgr.14116) 3 : audit [DBG] from='client.14120 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-10-01T10:05:30.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:30 smithi078 bash[27336]: cluster 2023-10-01T10:05:29.175251+0000 mon.a (mon.0) 48 : cluster [DBG] mgrmap e8: y(active, since 2s) 2023-10-01T10:05:30.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:30 smithi078 bash[27336]: audit 2023-10-01T10:05:29.950530+0000 mon.a (mon.0) 49 : audit [INF] from='mgr.14116 172.21.15.78:0/3408980511' entity='mgr.y' 2023-10-01T10:05:30.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:30 smithi078 bash[27336]: audit 2023-10-01T10:05:30.006797+0000 mon.a (mon.0) 50 : audit [DBG] from='mgr.14116 172.21.15.78:0/3408980511' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:05:31.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:31 smithi078 bash[27336]: audit 2023-10-01T10:05:29.942890+0000 mgr.y (mgr.14116) 4 : audit [DBG] from='client.14126 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:05:31.927 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: value unchanged 2023-10-01T10:05:32.554 INFO:teuthology.orchestra.run.smithi078.stderr:Generating ssh key... 2023-10-01T10:05:32.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:32 smithi078 bash[27336]: audit 2023-10-01T10:05:31.924273+0000 mgr.y (mgr.14116) 5 : audit [DBG] from='client.14128 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:05:34.582 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:34 smithi078 bash[27601]: Generating public/private rsa key pair. 2023-10-01T10:05:34.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:34 smithi078 bash[27601]: Your identification has been saved in /tmp/tmpjhm5yhtg/key. 2023-10-01T10:05:34.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:34 smithi078 bash[27601]: Your public key has been saved in /tmp/tmpjhm5yhtg/key.pub. 2023-10-01T10:05:34.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:34 smithi078 bash[27601]: The key fingerprint is: 2023-10-01T10:05:34.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:34 smithi078 bash[27601]: SHA256:4o3Mu4gRDY5IUssgJC+nBNZ2w5GM7BHP/9CBOd/etHk ceph-d8016480-6041-11ee-8db5-212e2dc638e7 2023-10-01T10:05:34.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:34 smithi078 bash[27601]: The key's randomart image is: 2023-10-01T10:05:34.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:34 smithi078 bash[27601]: +---[RSA 3072]----+ 2023-10-01T10:05:34.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:34 smithi078 bash[27601]: |*o+.=.o | 2023-10-01T10:05:34.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:34 smithi078 bash[27601]: |== Bo* o | 2023-10-01T10:05:34.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:34 smithi078 bash[27601]: |ooO oo.+ . | 2023-10-01T10:05:34.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:34 smithi078 bash[27601]: |=* + . + o | 2023-10-01T10:05:34.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:34 smithi078 bash[27601]: |+ o . .oSo . . | 2023-10-01T10:05:34.584 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:34 smithi078 bash[27601]: | . + +o . o o | 2023-10-01T10:05:34.584 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:34 smithi078 bash[27601]: | . = .. . + E | 2023-10-01T10:05:34.584 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:34 smithi078 bash[27601]: | o . . . | 2023-10-01T10:05:34.584 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:34 smithi078 bash[27601]: | . . o. | 2023-10-01T10:05:34.584 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:34 smithi078 bash[27601]: +----[SHA256]-----+ 2023-10-01T10:05:35.563 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:35 smithi078 bash[27336]: audit 2023-10-01T10:05:33.882517+0000 mgr.y (mgr.14116) 6 : audit [DBG] from='client.14130 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:05:35.563 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:35 smithi078 bash[27336]: cephadm 2023-10-01T10:05:33.882899+0000 mgr.y (mgr.14116) 7 : cephadm [INF] Generating ssh key... 2023-10-01T10:05:35.563 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:35 smithi078 bash[27336]: audit 2023-10-01T10:05:34.243172+0000 mon.a (mon.0) 51 : audit [INF] from='mgr.14116 172.21.15.78:0/3408980511' entity='mgr.y' 2023-10-01T10:05:35.563 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:35 smithi078 bash[27336]: audit 2023-10-01T10:05:34.246398+0000 mon.a (mon.0) 52 : audit [INF] from='mgr.14116 172.21.15.78:0/3408980511' entity='mgr.y' 2023-10-01T10:05:36.175 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC6j9m798Yw1DtojxhrUK6OvmE5D1znBw6JRATSi6yMFufLoGtCgnBGNybPjLMPbmClz7+aC0MMDJottF9fSSdOigTTRR6t6lB5v8w1gn5XmgGEfOKB9vrfcoWNOD9ZWFzHLTrSWSncGZDAwALC/kllKobOOokXY4Putaj+eqpMlGcTfAZ/l31oNTxwmfWDxhQMPCQi7xhujlGZGu8DNqPd7pKIN52MFyg4t8U4g64mH4dj+0OPyQXtDi00Ognoyi7K7rfJYE3FN2SP8JmWElOORNRRvzF4OYygy8Gjn4JwY3zOIIuY16ls0VccVQdLTtlH4ZKratWPttKOKzXqctZnnjmJWyMiYOoqChiz4Dm9f2tMvFC5B2dffAqIQ483AdeAxpjuel4EdO1wxRqczz76RDqumMPhHBd07MYGeS4MgAiazvEslG0w/qG6bnYbcgCqy0dYUQ8dhuZBvyr5DqoYPa9xFihNoct3++jr9ImhLwQT30CxTiJKhybcPWZ29bU= ceph-d8016480-6041-11ee-8db5-212e2dc638e7 2023-10-01T10:05:36.841 INFO:teuthology.orchestra.run.smithi078.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2023-10-01T10:05:36.841 INFO:teuthology.orchestra.run.smithi078.stderr:Adding key to root@localhost authorized_keys... 2023-10-01T10:05:36.841 INFO:teuthology.orchestra.run.smithi078.stderr:Adding host smithi078... 2023-10-01T10:05:37.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:36 smithi078 bash[27336]: audit 2023-10-01T10:05:36.172555+0000 mgr.y (mgr.14116) 8 : audit [DBG] from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:05:38.582 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:38 smithi078 bash[27601]: Warning: Permanently added '172.21.15.78' (ECDSA) to the list of known hosts. 2023-10-01T10:05:39.043 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: Added host 'smithi078' with addr '172.21.15.78' 2023-10-01T10:05:39.074 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:38 smithi078 bash[27336]: audit 2023-10-01T10:05:38.216941+0000 mgr.y (mgr.14116) 9 : audit [DBG] from='client.14134 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi078", "addr": "172.21.15.78", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:05:39.978 INFO:teuthology.orchestra.run.smithi078.stderr:Deploying unmanaged mon service... 2023-10-01T10:05:40.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:40 smithi078 bash[27336]: audit 2023-10-01T10:05:39.040207+0000 mon.a (mon.0) 53 : audit [INF] from='mgr.14116 172.21.15.78:0/3408980511' entity='mgr.y' 2023-10-01T10:05:40.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:40 smithi078 bash[27336]: cephadm 2023-10-01T10:05:39.040950+0000 mgr.y (mgr.14116) 10 : cephadm [INF] Added host smithi078 2023-10-01T10:05:40.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:40 smithi078 bash[27336]: audit 2023-10-01T10:05:39.050522+0000 mon.a (mon.0) 54 : audit [DBG] from='mgr.14116 172.21.15.78:0/3408980511' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:05:41.915 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: Scheduled mon update... 2023-10-01T10:05:42.847 INFO:teuthology.orchestra.run.smithi078.stderr:Deploying unmanaged mgr service... 2023-10-01T10:05:43.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:42 smithi078 bash[27336]: audit 2023-10-01T10:05:41.909169+0000 mgr.y (mgr.14116) 11 : audit [DBG] from='client.14136 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:05:43.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:42 smithi078 bash[27336]: cephadm 2023-10-01T10:05:41.910677+0000 mgr.y (mgr.14116) 12 : cephadm [INF] Saving service mon spec with placement count:5 2023-10-01T10:05:43.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:42 smithi078 bash[27336]: audit 2023-10-01T10:05:41.912981+0000 mon.a (mon.0) 55 : audit [INF] from='mgr.14116 172.21.15.78:0/3408980511' entity='mgr.y' 2023-10-01T10:05:43.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:42 smithi078 bash[27336]: audit 2023-10-01T10:05:42.429688+0000 mon.a (mon.0) 56 : audit [INF] from='mgr.14116 172.21.15.78:0/3408980511' entity='mgr.y' 2023-10-01T10:05:43.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:42 smithi078 bash[27336]: audit 2023-10-01T10:05:42.678795+0000 mon.a (mon.0) 57 : audit [INF] from='mgr.14116 172.21.15.78:0/3408980511' entity='mgr.y' 2023-10-01T10:05:44.212 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: Scheduled mgr update... 2023-10-01T10:05:45.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:45 smithi078 bash[27336]: audit 2023-10-01T10:05:44.205171+0000 mgr.y (mgr.14116) 13 : audit [DBG] from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:05:45.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:45 smithi078 bash[27336]: cephadm 2023-10-01T10:05:44.207526+0000 mgr.y (mgr.14116) 14 : cephadm [INF] Saving service mgr spec with placement count:2 2023-10-01T10:05:45.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:45 smithi078 bash[27336]: audit 2023-10-01T10:05:44.210132+0000 mon.a (mon.0) 58 : audit [INF] from='mgr.14116 172.21.15.78:0/3408980511' entity='mgr.y' 2023-10-01T10:05:47.557 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:47 smithi078 bash[27336]: audit 2023-10-01T10:05:46.197691+0000 mon.a (mon.0) 59 : audit [INF] from='client.? 172.21.15.78:0/242187146' entity='client.admin' 2023-10-01T10:05:48.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:48 smithi078 bash[27336]: cluster 2023-10-01T10:05:47.020426+0000 mgr.y (mgr.14116) 15 : cluster [DBG] pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:05:48.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:48 smithi078 bash[27336]: audit 2023-10-01T10:05:47.725510+0000 mon.a (mon.0) 60 : audit [INF] from='mgr.14116 172.21.15.78:0/3408980511' entity='mgr.y' 2023-10-01T10:05:48.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:48 smithi078 bash[27336]: audit 2023-10-01T10:05:48.180124+0000 mon.a (mon.0) 61 : audit [INF] from='client.? 172.21.15.78:0/3633324930' entity='client.admin' 2023-10-01T10:05:48.927 INFO:teuthology.orchestra.run.smithi078.stderr:Enabling the dashboard module... 2023-10-01T10:05:50.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:50 smithi078 bash[27336]: cluster 2023-10-01T10:05:49.020830+0000 mgr.y (mgr.14116) 16 : cluster [DBG] pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:05:51.471 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:51 smithi078 bash[27336]: audit 2023-10-01T10:05:50.341032+0000 mon.a (mon.0) 62 : audit [INF] from='client.? 172.21.15.78:0/3104697531' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2023-10-01T10:05:51.472 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:51 smithi078 bash[27601]: ignoring --setuser ceph since I am not root 2023-10-01T10:05:51.472 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:51 smithi078 bash[27601]: ignoring --setgroup ceph since I am not root 2023-10-01T10:05:51.472 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:51 smithi078 bash[27601]: debug 2023-10-01T10:05:51.411+0000 7f329f1ae680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-01T10:05:51.832 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:51 smithi078 bash[27601]: debug 2023-10-01T10:05:51.515+0000 7f329f1ae680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-01T10:05:52.206 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:51 smithi078 bash[27601]: debug 2023-10-01T10:05:51.967+0000 7f329f1ae680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-01T10:05:52.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:52 smithi078 bash[27336]: audit 2023-10-01T10:05:51.208103+0000 mon.a (mon.0) 63 : audit [INF] from='client.? 172.21.15.78:0/3104697531' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2023-10-01T10:05:52.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:52 smithi078 bash[27336]: cluster 2023-10-01T10:05:51.208190+0000 mon.a (mon.0) 64 : cluster [DBG] mgrmap e9: y(active, since 24s) 2023-10-01T10:05:53.081 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:52 smithi078 bash[27601]: debug 2023-10-01T10:05:52.895+0000 7f329f1ae680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-01T10:05:53.081 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:53 smithi078 bash[27601]: debug 2023-10-01T10:05:52.995+0000 7f329f1ae680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-01T10:05:53.488 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: { 2023-10-01T10:05:53.488 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 9, 2023-10-01T10:05:53.489 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "available": true, 2023-10-01T10:05:53.489 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "active_name": "y", 2023-10-01T10:05:53.489 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_standby": 0 2023-10-01T10:05:53.489 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: } 2023-10-01T10:05:53.519 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:53 smithi078 bash[27601]: debug 2023-10-01T10:05:53.251+0000 7f329f1ae680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-01T10:05:53.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:53 smithi078 bash[27336]: audit 2023-10-01T10:05:53.486421+0000 mon.a (mon.0) 65 : audit [DBG] from='client.? 172.21.15.78:0/3875390432' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-10-01T10:05:54.545 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:54 smithi078 bash[27601]: debug 2023-10-01T10:05:54.383+0000 7f329f1ae680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-01T10:05:54.546 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:54 smithi078 bash[27601]: debug 2023-10-01T10:05:54.539+0000 7f329f1ae680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-01T10:05:54.621 INFO:teuthology.orchestra.run.smithi078.stderr:Waiting for the mgr to restart... 2023-10-01T10:05:54.621 INFO:teuthology.orchestra.run.smithi078.stderr:Waiting for mgr epoch 9... 2023-10-01T10:05:54.831 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:54 smithi078 bash[27601]: debug 2023-10-01T10:05:54.635+0000 7f329f1ae680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-01T10:05:54.831 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:54 smithi078 bash[27601]: debug 2023-10-01T10:05:54.747+0000 7f329f1ae680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-01T10:05:55.291 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:54 smithi078 bash[27601]: debug 2023-10-01T10:05:54.971+0000 7f329f1ae680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-01T10:05:55.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:55 smithi078 bash[27601]: debug 2023-10-01T10:05:55.335+0000 7f329f1ae680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-01T10:05:55.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:55 smithi078 bash[27601]: debug 2023-10-01T10:05:55.459+0000 7f329f1ae680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-01T10:05:56.539 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:56 smithi078 bash[27601]: debug 2023-10-01T10:05:56.315+0000 7f329f1ae680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-01T10:05:56.540 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:56 smithi078 bash[27601]: debug 2023-10-01T10:05:56.415+0000 7f329f1ae680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-01T10:05:56.831 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:56 smithi078 bash[27601]: debug 2023-10-01T10:05:56.535+0000 7f329f1ae680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-01T10:05:56.831 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:56 smithi078 bash[27601]: debug 2023-10-01T10:05:56.759+0000 7f329f1ae680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-01T10:05:57.331 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:57 smithi078 bash[27601]: debug 2023-10-01T10:05:57.047+0000 7f329f1ae680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-01T10:05:57.331 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:57 smithi078 bash[27601]: debug 2023-10-01T10:05:57.211+0000 7f329f1ae680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-01T10:05:57.707 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:57 smithi078 bash[27601]: debug 2023-10-01T10:05:57.379+0000 7f329f1ae680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-01T10:05:57.707 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:57 smithi078 bash[27601]: debug 2023-10-01T10:05:57.595+0000 7f329f1ae680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-01T10:05:57.998 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:57 smithi078 bash[27601]: debug 2023-10-01T10:05:57.703+0000 7f329f1ae680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-01T10:05:57.999 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:05:57 smithi078 bash[27601]: Warning: Permanently added '172.21.15.78' (ECDSA) to the list of known hosts. 2023-10-01T10:05:57.999 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:57 smithi078 bash[27336]: cluster 2023-10-01T10:05:57.727289+0000 mon.a (mon.0) 66 : cluster [INF] Active manager daemon y restarted 2023-10-01T10:05:57.999 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:57 smithi078 bash[27336]: cluster 2023-10-01T10:05:57.727721+0000 mon.a (mon.0) 67 : cluster [INF] Activating manager daemon y 2023-10-01T10:05:57.999 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:57 smithi078 bash[27336]: cluster 2023-10-01T10:05:57.735429+0000 mon.a (mon.0) 68 : cluster [DBG] osdmap e3: 0 total, 0 up, 0 in 2023-10-01T10:05:57.999 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:57 smithi078 bash[27336]: cluster 2023-10-01T10:05:57.735699+0000 mon.a (mon.0) 69 : cluster [DBG] mgrmap e10: y(active, starting, since 0.00814502s) 2023-10-01T10:05:57.999 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:57 smithi078 bash[27336]: audit 2023-10-01T10:05:57.737300+0000 mon.a (mon.0) 70 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T10:05:57.999 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:57 smithi078 bash[27336]: audit 2023-10-01T10:05:57.738374+0000 mon.a (mon.0) 71 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-01T10:05:58.000 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:57 smithi078 bash[27336]: audit 2023-10-01T10:05:57.739390+0000 mon.a (mon.0) 72 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-01T10:05:58.000 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:57 smithi078 bash[27336]: audit 2023-10-01T10:05:57.739621+0000 mon.a (mon.0) 73 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-01T10:05:58.000 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:57 smithi078 bash[27336]: audit 2023-10-01T10:05:57.739831+0000 mon.a (mon.0) 74 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-01T10:05:58.000 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:57 smithi078 bash[27336]: cluster 2023-10-01T10:05:57.742843+0000 mon.a (mon.0) 75 : cluster [INF] Manager daemon y is now available 2023-10-01T10:05:58.754 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: { 2023-10-01T10:05:58.755 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2023-10-01T10:05:58.755 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "initialized": true 2023-10-01T10:05:58.755 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: } 2023-10-01T10:05:59.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:58 smithi078 bash[27336]: audit 2023-10-01T10:05:57.883765+0000 mon.a (mon.0) 76 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:05:59.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:58 smithi078 bash[27336]: audit 2023-10-01T10:05:57.888613+0000 mon.a (mon.0) 77 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:05:59.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:58 smithi078 bash[27336]: audit 2023-10-01T10:05:57.926885+0000 mon.a (mon.0) 78 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-01T10:05:59.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:58 smithi078 bash[27336]: audit 2023-10-01T10:05:57.930684+0000 mon.a (mon.0) 79 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-01T10:05:59.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:58 smithi078 bash[27336]: cluster 2023-10-01T10:05:58.738626+0000 mon.a (mon.0) 80 : cluster [DBG] mgrmap e11: y(active, since 1.01107s) 2023-10-01T10:05:59.468 INFO:teuthology.orchestra.run.smithi078.stderr:mgr epoch 9 is available 2023-10-01T10:05:59.469 INFO:teuthology.orchestra.run.smithi078.stderr:Generating a dashboard self-signed certificate... 2023-10-01T10:06:00.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:59 smithi078 bash[27336]: audit 2023-10-01T10:05:58.737045+0000 mgr.y (mgr.14146) 1 : audit [DBG] from='client.14150 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-10-01T10:06:00.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:05:59 smithi078 bash[27336]: audit 2023-10-01T10:05:58.752774+0000 mgr.y (mgr.14146) 2 : audit [DBG] from='client.14150 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-10-01T10:06:01.042 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: Self-signed certificate created 2023-10-01T10:06:01.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:00 smithi078 bash[27336]: cluster 2023-10-01T10:05:59.791214+0000 mon.a (mon.0) 81 : cluster [DBG] mgrmap e12: y(active, since 2s) 2023-10-01T10:06:01.738 INFO:teuthology.orchestra.run.smithi078.stderr:Creating initial admin user... 2023-10-01T10:06:02.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:02 smithi078 bash[27336]: audit 2023-10-01T10:06:00.815853+0000 mgr.y (mgr.14146) 3 : audit [DBG] from='client.14156 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:06:02.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:02 smithi078 bash[27336]: audit 2023-10-01T10:06:01.033822+0000 mon.a (mon.0) 82 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:06:02.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:02 smithi078 bash[27336]: audit 2023-10-01T10:06:01.039596+0000 mon.a (mon.0) 83 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:06:03.357 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$NmJ7/K.SR9gOTfoEXkAi9.LDglQMdhJtPlGFyEeEekUrMbg7cuGRG", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1696154763, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2023-10-01T10:06:03.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:03 smithi078 bash[27336]: audit 2023-10-01T10:06:03.124303+0000 mgr.y (mgr.14146) 4 : audit [DBG] from='client.14158 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:06:03.963 INFO:teuthology.orchestra.run.smithi078.stderr:Fetching dashboard port number... 2023-10-01T10:06:04.687 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:04 smithi078 bash[27336]: audit 2023-10-01T10:06:03.355694+0000 mon.a (mon.0) 84 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:06:05.287 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 8443 2023-10-01T10:06:05.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:05 smithi078 bash[27336]: cluster 2023-10-01T10:06:04.360247+0000 mon.a (mon.0) 85 : cluster [DBG] mgrmap e13: y(active, since 6s) 2023-10-01T10:06:05.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:05 smithi078 bash[27336]: audit 2023-10-01T10:06:04.709901+0000 mon.a (mon.0) 86 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:06:05.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:05 smithi078 bash[27336]: audit 2023-10-01T10:06:05.285759+0000 mon.a (mon.0) 87 : audit [DBG] from='client.? 172.21.15.78:0/1667455202' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2023-10-01T10:06:05.969 INFO:teuthology.orchestra.run.smithi078.stderr:firewalld does not appear to be present 2023-10-01T10:06:05.970 INFO:teuthology.orchestra.run.smithi078.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2023-10-01T10:06:05.972 INFO:teuthology.orchestra.run.smithi078.stderr:Ceph Dashboard is now available at: 2023-10-01T10:06:05.972 INFO:teuthology.orchestra.run.smithi078.stderr: 2023-10-01T10:06:05.973 INFO:teuthology.orchestra.run.smithi078.stderr: URL: https://smithi078.front.sepia.ceph.com:8443/ 2023-10-01T10:06:05.973 INFO:teuthology.orchestra.run.smithi078.stderr: User: admin 2023-10-01T10:06:05.973 INFO:teuthology.orchestra.run.smithi078.stderr: Password: p9q7lcstdm 2023-10-01T10:06:05.973 INFO:teuthology.orchestra.run.smithi078.stderr: 2023-10-01T10:06:05.973 INFO:teuthology.orchestra.run.smithi078.stderr:Enabling autotune for osd_memory_target 2023-10-01T10:06:09.681 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2023-10-01T10:06:10.771 INFO:teuthology.orchestra.run.smithi078.stderr:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2023-10-01T10:06:10.771 INFO:teuthology.orchestra.run.smithi078.stderr: 2023-10-01T10:06:10.771 INFO:teuthology.orchestra.run.smithi078.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2023-10-01T10:06:10.771 INFO:teuthology.orchestra.run.smithi078.stderr: 2023-10-01T10:06:10.771 INFO:teuthology.orchestra.run.smithi078.stderr:Or, if you are only running a single cluster on this host: 2023-10-01T10:06:10.771 INFO:teuthology.orchestra.run.smithi078.stderr: 2023-10-01T10:06:10.771 INFO:teuthology.orchestra.run.smithi078.stderr: sudo /home/ubuntu/cephtest/cephadm shell 2023-10-01T10:06:10.771 INFO:teuthology.orchestra.run.smithi078.stderr: 2023-10-01T10:06:10.772 INFO:teuthology.orchestra.run.smithi078.stderr:Please consider enabling telemetry to help improve Ceph: 2023-10-01T10:06:10.772 INFO:teuthology.orchestra.run.smithi078.stderr: 2023-10-01T10:06:10.772 INFO:teuthology.orchestra.run.smithi078.stderr: ceph telemetry on 2023-10-01T10:06:10.772 INFO:teuthology.orchestra.run.smithi078.stderr: 2023-10-01T10:06:10.772 INFO:teuthology.orchestra.run.smithi078.stderr:For more information see: 2023-10-01T10:06:10.772 INFO:teuthology.orchestra.run.smithi078.stderr: 2023-10-01T10:06:10.772 INFO:teuthology.orchestra.run.smithi078.stderr: https://docs.ceph.com/en/pacific/mgr/telemetry/ 2023-10-01T10:06:10.772 INFO:teuthology.orchestra.run.smithi078.stderr: 2023-10-01T10:06:10.772 INFO:teuthology.orchestra.run.smithi078.stderr:Bootstrap complete. 2023-10-01T10:06:10.803 INFO:tasks.cephadm:Fetching config... 2023-10-01T10:06:10.804 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2023-10-01T10:06:10.804 DEBUG:teuthology.orchestra.run.smithi078:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2023-10-01T10:06:10.810 INFO:tasks.cephadm:Fetching client.admin keyring... 2023-10-01T10:06:10.811 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2023-10-01T10:06:10.811 DEBUG:teuthology.orchestra.run.smithi078:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2023-10-01T10:06:10.863 INFO:tasks.cephadm:Fetching mon keyring... 2023-10-01T10:06:10.863 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2023-10-01T10:06:10.864 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd if=/var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/keyring of=/dev/stdout 2023-10-01T10:06:10.917 INFO:tasks.cephadm:Fetching pub ssh key... 2023-10-01T10:06:10.917 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2023-10-01T10:06:10.917 DEBUG:teuthology.orchestra.run.smithi078:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2023-10-01T10:06:10.967 INFO:tasks.cephadm:Installing pub ssh key for root users... 2023-10-01T10:06:10.968 DEBUG:teuthology.orchestra.run.smithi078:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC6j9m798Yw1DtojxhrUK6OvmE5D1znBw6JRATSi6yMFufLoGtCgnBGNybPjLMPbmClz7+aC0MMDJottF9fSSdOigTTRR6t6lB5v8w1gn5XmgGEfOKB9vrfcoWNOD9ZWFzHLTrSWSncGZDAwALC/kllKobOOokXY4Putaj+eqpMlGcTfAZ/l31oNTxwmfWDxhQMPCQi7xhujlGZGu8DNqPd7pKIN52MFyg4t8U4g64mH4dj+0OPyQXtDi00Ognoyi7K7rfJYE3FN2SP8JmWElOORNRRvzF4OYygy8Gjn4JwY3zOIIuY16ls0VccVQdLTtlH4ZKratWPttKOKzXqctZnnjmJWyMiYOoqChiz4Dm9f2tMvFC5B2dffAqIQ483AdeAxpjuel4EdO1wxRqczz76RDqumMPhHBd07MYGeS4MgAiazvEslG0w/qG6bnYbcgCqy0dYUQ8dhuZBvyr5DqoYPa9xFihNoct3++jr9ImhLwQT30CxTiJKhybcPWZ29bU= ceph-d8016480-6041-11ee-8db5-212e2dc638e7' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-10-01T10:06:11.018 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:10 smithi078 bash[27336]: audit 2023-10-01T10:06:09.672785+0000 mon.a (mon.0) 88 : audit [INF] from='client.? 172.21.15.78:0/2178350690' entity='client.admin' 2023-10-01T10:06:11.026 INFO:teuthology.orchestra.run.smithi078.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC6j9m798Yw1DtojxhrUK6OvmE5D1znBw6JRATSi6yMFufLoGtCgnBGNybPjLMPbmClz7+aC0MMDJottF9fSSdOigTTRR6t6lB5v8w1gn5XmgGEfOKB9vrfcoWNOD9ZWFzHLTrSWSncGZDAwALC/kllKobOOokXY4Putaj+eqpMlGcTfAZ/l31oNTxwmfWDxhQMPCQi7xhujlGZGu8DNqPd7pKIN52MFyg4t8U4g64mH4dj+0OPyQXtDi00Ognoyi7K7rfJYE3FN2SP8JmWElOORNRRvzF4OYygy8Gjn4JwY3zOIIuY16ls0VccVQdLTtlH4ZKratWPttKOKzXqctZnnjmJWyMiYOoqChiz4Dm9f2tMvFC5B2dffAqIQ483AdeAxpjuel4EdO1wxRqczz76RDqumMPhHBd07MYGeS4MgAiazvEslG0w/qG6bnYbcgCqy0dYUQ8dhuZBvyr5DqoYPa9xFihNoct3++jr9ImhLwQT30CxTiJKhybcPWZ29bU= ceph-d8016480-6041-11ee-8db5-212e2dc638e7 2023-10-01T10:06:11.037 DEBUG:teuthology.orchestra.run.smithi173:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC6j9m798Yw1DtojxhrUK6OvmE5D1znBw6JRATSi6yMFufLoGtCgnBGNybPjLMPbmClz7+aC0MMDJottF9fSSdOigTTRR6t6lB5v8w1gn5XmgGEfOKB9vrfcoWNOD9ZWFzHLTrSWSncGZDAwALC/kllKobOOokXY4Putaj+eqpMlGcTfAZ/l31oNTxwmfWDxhQMPCQi7xhujlGZGu8DNqPd7pKIN52MFyg4t8U4g64mH4dj+0OPyQXtDi00Ognoyi7K7rfJYE3FN2SP8JmWElOORNRRvzF4OYygy8Gjn4JwY3zOIIuY16ls0VccVQdLTtlH4ZKratWPttKOKzXqctZnnjmJWyMiYOoqChiz4Dm9f2tMvFC5B2dffAqIQ483AdeAxpjuel4EdO1wxRqczz76RDqumMPhHBd07MYGeS4MgAiazvEslG0w/qG6bnYbcgCqy0dYUQ8dhuZBvyr5DqoYPa9xFihNoct3++jr9ImhLwQT30CxTiJKhybcPWZ29bU= ceph-d8016480-6041-11ee-8db5-212e2dc638e7' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-10-01T10:06:11.061 INFO:teuthology.orchestra.run.smithi173.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC6j9m798Yw1DtojxhrUK6OvmE5D1znBw6JRATSi6yMFufLoGtCgnBGNybPjLMPbmClz7+aC0MMDJottF9fSSdOigTTRR6t6lB5v8w1gn5XmgGEfOKB9vrfcoWNOD9ZWFzHLTrSWSncGZDAwALC/kllKobOOokXY4Putaj+eqpMlGcTfAZ/l31oNTxwmfWDxhQMPCQi7xhujlGZGu8DNqPd7pKIN52MFyg4t8U4g64mH4dj+0OPyQXtDi00Ognoyi7K7rfJYE3FN2SP8JmWElOORNRRvzF4OYygy8Gjn4JwY3zOIIuY16ls0VccVQdLTtlH4ZKratWPttKOKzXqctZnnjmJWyMiYOoqChiz4Dm9f2tMvFC5B2dffAqIQ483AdeAxpjuel4EdO1wxRqczz76RDqumMPhHBd07MYGeS4MgAiazvEslG0w/qG6bnYbcgCqy0dYUQ8dhuZBvyr5DqoYPa9xFihNoct3++jr9ImhLwQT30CxTiJKhybcPWZ29bU= ceph-d8016480-6041-11ee-8db5-212e2dc638e7 2023-10-01T10:06:11.072 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2023-10-01T10:06:15.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:15 smithi078 bash[27336]: audit 2023-10-01T10:06:14.399532+0000 mon.a (mon.0) 89 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:06:15.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:15 smithi078 bash[27336]: audit 2023-10-01T10:06:14.771114+0000 mon.a (mon.0) 90 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi078", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:06:15.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:15 smithi078 bash[27336]: audit 2023-10-01T10:06:14.775878+0000 mon.a (mon.0) 91 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:06:15.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:15 smithi078 bash[27336]: audit 2023-10-01T10:06:14.777967+0000 mon.a (mon.0) 92 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:06:15.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:15 smithi078 bash[27336]: audit 2023-10-01T10:06:14.782949+0000 mon.a (mon.0) 93 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:06:15.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:15 smithi078 bash[27336]: audit 2023-10-01T10:06:15.185670+0000 mon.a (mon.0) 94 : audit [INF] from='client.? 172.21.15.78:0/4172120572' entity='client.admin' 2023-10-01T10:06:15.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:15 smithi078 bash[27336]: audit 2023-10-01T10:06:15.249284+0000 mon.a (mon.0) 95 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:06:15.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:15 smithi078 bash[27336]: audit 2023-10-01T10:06:15.256426+0000 mon.a (mon.0) 96 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:06:16.071 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2023-10-01T10:06:16.071 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph orch client-keyring set client.admin '*' --mode 0755 2023-10-01T10:06:19.055 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:18 smithi078 bash[27336]: cluster 2023-10-01T10:06:17.741346+0000 mgr.y (mgr.14146) 5 : cluster [DBG] pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:06:20.237 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi173 2023-10-01T10:06:20.237 DEBUG:teuthology.orchestra.run.smithi173:> set -ex 2023-10-01T10:06:20.237 DEBUG:teuthology.orchestra.run.smithi173:> dd of=/etc/ceph/ceph.conf 2023-10-01T10:06:20.246 DEBUG:teuthology.orchestra.run.smithi173:> set -ex 2023-10-01T10:06:20.246 DEBUG:teuthology.orchestra.run.smithi173:> dd of=/etc/ceph/ceph.client.admin.keyring 2023-10-01T10:06:20.344 INFO:tasks.cephadm:Adding host smithi173 to orchestrator... 2023-10-01T10:06:20.344 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph orch host add smithi173 2023-10-01T10:06:20.554 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:20 smithi078 bash[27336]: audit 2023-10-01T10:06:19.440260+0000 mon.a (mon.0) 97 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:06:20.554 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:20 smithi078 bash[27336]: audit 2023-10-01T10:06:19.482747+0000 mon.a (mon.0) 98 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:06:20.555 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:20 smithi078 bash[27336]: audit 2023-10-01T10:06:19.486068+0000 mon.a (mon.0) 99 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:06:20.555 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:20 smithi078 bash[27336]: audit 2023-10-01T10:06:19.487174+0000 mon.a (mon.0) 100 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:06:20.555 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:20 smithi078 bash[27336]: cephadm 2023-10-01T10:06:19.489581+0000 mgr.y (mgr.14146) 7 : cephadm [INF] Updating smithi078:/etc/ceph/ceph.conf 2023-10-01T10:06:20.555 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:20 smithi078 bash[27336]: cephadm 2023-10-01T10:06:19.531731+0000 mgr.y (mgr.14146) 8 : cephadm [INF] Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2023-10-01T10:06:20.555 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:20 smithi078 bash[27336]: audit 2023-10-01T10:06:19.593852+0000 mon.a (mon.0) 101 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:06:20.555 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:20 smithi078 bash[27336]: audit 2023-10-01T10:06:19.597965+0000 mon.a (mon.0) 102 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:06:20.555 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:20 smithi078 bash[27336]: cluster 2023-10-01T10:06:19.741777+0000 mgr.y (mgr.14146) 9 : cluster [DBG] pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:06:23.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:22 smithi078 bash[27336]: cluster 2023-10-01T10:06:21.742179+0000 mgr.y (mgr.14146) 10 : cluster [DBG] pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:06:24.081 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:06:23 smithi078 bash[27601]: Warning: Permanently added 'smithi173,172.21.15.173' (ECDSA) to the list of known hosts. 2023-10-01T10:06:24.420 INFO:teuthology.orchestra.run.smithi078.stdout:Added host 'smithi173' with addr '172.21.15.173' 2023-10-01T10:06:24.799 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:06:24 smithi078 bash[27601]: Warning: Permanently added '172.21.15.173' (ECDSA) to the list of known hosts. 2023-10-01T10:06:25.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:24 smithi078 bash[27336]: audit 2023-10-01T10:06:23.562774+0000 mgr.y (mgr.14146) 11 : audit [DBG] from='client.14170 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi173", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:06:25.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:24 smithi078 bash[27336]: cluster 2023-10-01T10:06:23.742788+0000 mgr.y (mgr.14146) 12 : cluster [DBG] pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:06:25.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:24 smithi078 bash[27336]: audit 2023-10-01T10:06:24.417726+0000 mon.a (mon.0) 103 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:06:25.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:24 smithi078 bash[27336]: audit 2023-10-01T10:06:24.429961+0000 mon.a (mon.0) 104 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:06:25.263 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph orch host ls --format=json 2023-10-01T10:06:26.287 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:25 smithi078 bash[27336]: cephadm 2023-10-01T10:06:24.418604+0000 mgr.y (mgr.14146) 13 : cephadm [INF] Added host smithi173 2023-10-01T10:06:26.287 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:25 smithi078 bash[27336]: audit 2023-10-01T10:06:24.978978+0000 mon.a (mon.0) 105 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:06:27.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:27 smithi078 bash[27336]: cluster 2023-10-01T10:06:25.743294+0000 mgr.y (mgr.14146) 14 : cluster [DBG] pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:06:27.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:27 smithi078 bash[27336]: audit 2023-10-01T10:06:26.242108+0000 mon.a (mon.0) 106 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:06:28.599 INFO:teuthology.orchestra.run.smithi078.stdout: 2023-10-01T10:06:28.599 INFO:teuthology.orchestra.run.smithi078.stdout:[{"addr": "172.21.15.78", "hostname": "smithi078", "labels": [], "status": ""}, {"addr": "172.21.15.173", "hostname": "smithi173", "labels": [], "status": ""}] 2023-10-01T10:06:29.317 INFO:tasks.cephadm:Setting crush tunables to default 2023-10-01T10:06:29.318 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd crush tunables default 2023-10-01T10:06:29.516 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:29 smithi078 bash[27336]: cluster 2023-10-01T10:06:27.743787+0000 mgr.y (mgr.14146) 15 : cluster [DBG] pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:06:30.577 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:30 smithi078 bash[27336]: audit 2023-10-01T10:06:28.597648+0000 mgr.y (mgr.14146) 16 : audit [DBG] from='client.14172 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:06:31.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:31 smithi078 bash[27336]: cluster 2023-10-01T10:06:29.744274+0000 mgr.y (mgr.14146) 17 : cluster [DBG] pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:06:32.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:32 smithi078 bash[27336]: audit 2023-10-01T10:06:31.473667+0000 mon.a (mon.0) 107 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:06:32.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:32 smithi078 bash[27336]: cluster 2023-10-01T10:06:31.744746+0000 mgr.y (mgr.14146) 18 : cluster [DBG] pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:06:32.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:32 smithi078 bash[27336]: audit 2023-10-01T10:06:31.785398+0000 mon.a (mon.0) 108 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:06:32.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:32 smithi078 bash[27336]: audit 2023-10-01T10:06:31.786491+0000 mon.a (mon.0) 109 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi173", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:06:32.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:32 smithi078 bash[27336]: audit 2023-10-01T10:06:31.788198+0000 mon.a (mon.0) 110 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:06:32.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:32 smithi078 bash[27336]: audit 2023-10-01T10:06:31.789633+0000 mon.a (mon.0) 111 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:06:32.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:32 smithi078 bash[27336]: cephadm 2023-10-01T10:06:31.791476+0000 mgr.y (mgr.14146) 19 : cephadm [INF] Updating smithi173:/etc/ceph/ceph.conf 2023-10-01T10:06:32.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:32 smithi078 bash[27336]: cephadm 2023-10-01T10:06:31.815544+0000 mgr.y (mgr.14146) 20 : cephadm [INF] Updating smithi173:/etc/ceph/ceph.client.admin.keyring 2023-10-01T10:06:32.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:32 smithi078 bash[27336]: audit 2023-10-01T10:06:31.845534+0000 mon.a (mon.0) 112 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:06:32.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:32 smithi078 bash[27336]: audit 2023-10-01T10:06:31.850010+0000 mon.a (mon.0) 113 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:06:33.488 INFO:teuthology.orchestra.run.smithi078.stderr:adjusted tunables profile to default 2023-10-01T10:06:33.758 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:33 smithi078 bash[27336]: audit 2023-10-01T10:06:32.566293+0000 mon.a (mon.0) 114 : audit [INF] from='client.? 172.21.15.78:0/3746440933' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2023-10-01T10:06:34.255 INFO:tasks.cephadm:Adding mon.a on smithi078 2023-10-01T10:06:34.255 INFO:tasks.cephadm:Adding mon.c on smithi078 2023-10-01T10:06:34.255 INFO:tasks.cephadm:Adding mon.b on smithi173 2023-10-01T10:06:34.256 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph orch apply mon '3;smithi078:172.21.15.78=a;smithi078:[v2:172.21.15.78:3301,v1:172.21.15.78:6790]=c;smithi173:172.21.15.173=b' 2023-10-01T10:06:34.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:34 smithi078 bash[27336]: audit 2023-10-01T10:06:33.482522+0000 mon.a (mon.0) 115 : audit [INF] from='client.? 172.21.15.78:0/3746440933' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-10-01T10:06:34.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:34 smithi078 bash[27336]: cluster 2023-10-01T10:06:33.482606+0000 mon.a (mon.0) 116 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-01T10:06:34.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:34 smithi078 bash[27336]: cluster 2023-10-01T10:06:33.745169+0000 mgr.y (mgr.14146) 21 : cluster [DBG] pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:06:37.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:36 smithi078 bash[27336]: cluster 2023-10-01T10:06:35.745544+0000 mgr.y (mgr.14146) 22 : cluster [DBG] pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:06:37.096 INFO:teuthology.orchestra.run.smithi173.stdout:Scheduled mon update... 2023-10-01T10:06:37.973 DEBUG:teuthology.orchestra.run.smithi078:mon.c> sudo journalctl -f -n 0 -u ceph-d8016480-6041-11ee-8db5-212e2dc638e7@mon.c.service 2023-10-01T10:06:37.976 DEBUG:teuthology.orchestra.run.smithi173:mon.b> sudo journalctl -f -n 0 -u ceph-d8016480-6041-11ee-8db5-212e2dc638e7@mon.b.service 2023-10-01T10:06:37.982 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-10-01T10:06:37.983 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph mon dump -f json 2023-10-01T10:06:38.194 INFO:journalctl@ceph.mon.c.smithi078.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-01T10:06:38.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:38 smithi078 bash[27336]: audit 2023-10-01T10:06:37.081991+0000 mgr.y (mgr.14146) 23 : audit [DBG] from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi078:172.21.15.78=a;smithi078:[v2:172.21.15.78:3301,v1:172.21.15.78:6790]=c;smithi173:172.21.15.173=b", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:06:38.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:38 smithi078 bash[27336]: cephadm 2023-10-01T10:06:37.087979+0000 mgr.y (mgr.14146) 24 : cephadm [INF] Saving service mon spec with placement smithi078:172.21.15.78=a;smithi078:[v2:172.21.15.78:3301,v1:172.21.15.78:6790]=c;smithi173:172.21.15.173=b;count:3 2023-10-01T10:06:38.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:38 smithi078 bash[27336]: audit 2023-10-01T10:06:37.094976+0000 mon.a (mon.0) 117 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:06:38.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:38 smithi078 bash[27336]: audit 2023-10-01T10:06:37.129031+0000 mon.a (mon.0) 118 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:06:38.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:38 smithi078 bash[27336]: audit 2023-10-01T10:06:37.132725+0000 mon.a (mon.0) 119 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:06:38.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:38 smithi078 bash[27336]: audit 2023-10-01T10:06:37.134540+0000 mon.a (mon.0) 120 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:06:38.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:38 smithi078 bash[27336]: audit 2023-10-01T10:06:37.140457+0000 mon.a (mon.0) 121 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:06:38.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:38 smithi078 bash[27336]: audit 2023-10-01T10:06:37.143246+0000 mon.a (mon.0) 122 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T10:06:38.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:38 smithi078 bash[27336]: audit 2023-10-01T10:06:37.144574+0000 mon.a (mon.0) 123 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:06:38.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:38 smithi078 bash[27336]: cephadm 2023-10-01T10:06:37.145722+0000 mgr.y (mgr.14146) 25 : cephadm [INF] Deploying daemon mon.b on smithi173 2023-10-01T10:06:39.076 INFO:journalctl@ceph.mon.b.smithi173.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-01T10:06:39.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:39 smithi078 bash[27336]: cluster 2023-10-01T10:06:37.745910+0000 mgr.y (mgr.14146) 26 : cluster [DBG] pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:06:41.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:41 smithi078 bash[27336]: cluster 2023-10-01T10:06:39.746334+0000 mgr.y (mgr.14146) 27 : cluster [DBG] pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:06:42.691 INFO:teuthology.orchestra.run.smithi173.stdout: 2023-10-01T10:06:42.691 INFO:teuthology.orchestra.run.smithi173.stdout:{"epoch":1,"fsid":"d8016480-6041-11ee-8db5-212e2dc638e7","modified":"2023-10-01T10:04:55.006877Z","created":"2023-10-01T10:04:55.006877Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-10-01T10:06:42.694 INFO:teuthology.orchestra.run.smithi173.stderr:dumped monmap epoch 1 2023-10-01T10:06:43.486 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:43 smithi078 bash[27336]: cluster 2023-10-01T10:06:41.746717+0000 mgr.y (mgr.14146) 28 : cluster [DBG] pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:06:43.487 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:43 smithi078 bash[27336]: audit 2023-10-01T10:06:42.690182+0000 mon.a (mon.0) 124 : audit [DBG] from='client.? 172.21.15.173:0/4164141511' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-01T10:06:43.601 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:43 smithi173 systemd[1]: Started Ceph mon.b for d8016480-6041-11ee-8db5-212e2dc638e7. 2023-10-01T10:06:44.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.136+0000 7f014512e880 0 set uid:gid to 167:167 (ceph:ceph) 2023-10-01T10:06:44.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.136+0000 7f014512e880 0 ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable), process ceph-mon, pid 7 2023-10-01T10:06:44.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.136+0000 7f014512e880 0 pidfile_write: ignore empty --pid-file 2023-10-01T10:06:44.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 0 load: jerasure load: lrc load: isa 2023-10-01T10:06:44.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: RocksDB version: 6.8.1 2023-10-01T10:06:44.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-10-01T10:06:44.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Compile date Sep 25 2023 2023-10-01T10:06:44.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: DB SUMMARY 2023-10-01T10:06:44.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: CURRENT file: CURRENT 2023-10-01T10:06:44.372 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: IDENTITY file: IDENTITY 2023-10-01T10:06:44.372 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: MANIFEST file: MANIFEST-000001 size: 13 Bytes 2023-10-01T10:06:44.372 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 0, files: 2023-10-01T10:06:44.372 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000003.log size: 511 ; 2023-10-01T10:06:44.372 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.error_if_exists: 0 2023-10-01T10:06:44.372 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.create_if_missing: 0 2023-10-01T10:06:44.372 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.paranoid_checks: 1 2023-10-01T10:06:44.372 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.env: 0x56029f7c1080 2023-10-01T10:06:44.372 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.fs: Posix File System 2023-10-01T10:06:44.372 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.info_log: 0x5602a1e3ba80 2023-10-01T10:06:44.372 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.max_file_opening_threads: 16 2023-10-01T10:06:44.373 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.statistics: (nil) 2023-10-01T10:06:44.373 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.use_fsync: 0 2023-10-01T10:06:44.373 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.max_log_file_size: 0 2023-10-01T10:06:44.373 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-01T10:06:44.373 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.log_file_time_to_roll: 0 2023-10-01T10:06:44.373 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.keep_log_file_num: 1000 2023-10-01T10:06:44.373 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.recycle_log_file_num: 0 2023-10-01T10:06:44.374 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.allow_fallocate: 1 2023-10-01T10:06:44.374 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.allow_mmap_reads: 0 2023-10-01T10:06:44.375 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.allow_mmap_writes: 0 2023-10-01T10:06:44.375 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.use_direct_reads: 0 2023-10-01T10:06:44.375 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-01T10:06:44.375 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.create_missing_column_families: 0 2023-10-01T10:06:44.375 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.db_log_dir: 2023-10-01T10:06:44.375 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-b/store.db 2023-10-01T10:06:44.375 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.table_cache_numshardbits: 6 2023-10-01T10:06:44.375 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.max_subcompactions: 1 2023-10-01T10:06:44.375 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.max_background_flushes: -1 2023-10-01T10:06:44.375 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-10-01T10:06:44.375 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-10-01T10:06:44.375 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-01T10:06:44.376 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-01T10:06:44.376 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-10-01T10:06:44.376 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.advise_random_on_open: 1 2023-10-01T10:06:44.376 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.db_write_buffer_size: 0 2023-10-01T10:06:44.376 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.write_buffer_manager: 0x5602a2b38330 2023-10-01T10:06:44.376 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-01T10:06:44.376 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-10-01T10:06:44.377 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-01T10:06:44.377 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.use_adaptive_mutex: 0 2023-10-01T10:06:44.377 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.rate_limiter: (nil) 2023-10-01T10:06:44.377 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-01T10:06:44.377 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.wal_recovery_mode: 2 2023-10-01T10:06:44.377 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.enable_thread_tracking: 0 2023-10-01T10:06:44.377 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.enable_pipelined_write: 0 2023-10-01T10:06:44.377 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.unordered_write: 0 2023-10-01T10:06:44.377 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-01T10:06:44.377 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-01T10:06:44.377 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-01T10:06:44.378 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-01T10:06:44.378 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.row_cache: None 2023-10-01T10:06:44.378 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.wal_filter: None 2023-10-01T10:06:44.378 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-01T10:06:44.378 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.allow_ingest_behind: 0 2023-10-01T10:06:44.378 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.preserve_deletes: 0 2023-10-01T10:06:44.378 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.two_write_queues: 0 2023-10-01T10:06:44.378 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.manual_wal_flush: 0 2023-10-01T10:06:44.378 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.atomic_flush: 0 2023-10-01T10:06:44.378 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-01T10:06:44.378 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.persist_stats_to_disk: 0 2023-10-01T10:06:44.379 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-10-01T10:06:44.379 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.log_readahead_size: 0 2023-10-01T10:06:44.379 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.sst_file_checksum_func: Unknown 2023-10-01T10:06:44.379 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.max_background_jobs: 2 2023-10-01T10:06:44.379 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.max_background_compactions: -1 2023-10-01T10:06:44.379 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-01T10:06:44.379 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-01T10:06:44.379 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.delayed_write_rate : 16777216 2023-10-01T10:06:44.379 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.max_total_wal_size: 0 2023-10-01T10:06:44.379 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-01T10:06:44.379 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.stats_dump_period_sec: 600 2023-10-01T10:06:44.379 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.stats_persist_period_sec: 600 2023-10-01T10:06:44.380 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-01T10:06:44.380 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.max_open_files: -1 2023-10-01T10:06:44.380 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.bytes_per_sync: 0 2023-10-01T10:06:44.380 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-10-01T10:06:44.380 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-10-01T10:06:44.380 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.compaction_readahead_size: 0 2023-10-01T10:06:44.380 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Compression algorithms supported: 2023-10-01T10:06:44.380 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-01T10:06:44.380 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: kZSTD supported: 0 2023-10-01T10:06:44.380 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: kXpressCompression supported: 0 2023-10-01T10:06:44.380 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: kLZ4HCCompression supported: 1 2023-10-01T10:06:44.380 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: kLZ4Compression supported: 1 2023-10-01T10:06:44.380 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: kBZip2Compression supported: 0 2023-10-01T10:06:44.381 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: kZlibCompression supported: 1 2023-10-01T10:06:44.381 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: kSnappyCompression supported: 1 2023-10-01T10:06:44.381 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-10-01T10:06:44.381 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000001 2023-10-01T10:06:44.381 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2023-10-01T10:06:44.381 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-01T10:06:44.381 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.merge_operator: 2023-10-01T10:06:44.381 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.compaction_filter: None 2023-10-01T10:06:44.381 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.compaction_filter_factory: None 2023-10-01T10:06:44.381 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-10-01T10:06:44.381 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.table_factory: BlockBasedTable 2023-10-01T10:06:44.381 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5602a1de9d18) 2023-10-01T10:06:44.382 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: cache_index_and_filter_blocks: 1 2023-10-01T10:06:44.382 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-01T10:06:44.382 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-01T10:06:44.382 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: pin_top_level_index_and_filter: 1 2023-10-01T10:06:44.382 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: index_type: 0 2023-10-01T10:06:44.382 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: data_block_index_type: 0 2023-10-01T10:06:44.382 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: index_shortening: 1 2023-10-01T10:06:44.382 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: data_block_hash_table_util_ratio: 0.750000 2023-10-01T10:06:44.382 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: hash_index_allow_collision: 1 2023-10-01T10:06:44.382 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: checksum: 1 2023-10-01T10:06:44.382 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: no_block_cache: 0 2023-10-01T10:06:44.383 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: block_cache: 0x5602a1e20f10 2023-10-01T10:06:44.383 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: block_cache_name: BinnedLRUCache 2023-10-01T10:06:44.383 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: block_cache_options: 2023-10-01T10:06:44.383 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: capacity : 536870912 2023-10-01T10:06:44.383 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: num_shard_bits : 4 2023-10-01T10:06:44.383 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: strict_capacity_limit : 0 2023-10-01T10:06:44.383 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: high_pri_pool_ratio: 0.000 2023-10-01T10:06:44.383 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: block_cache_compressed: (nil) 2023-10-01T10:06:44.383 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: persistent_cache: (nil) 2023-10-01T10:06:44.383 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: block_size: 4096 2023-10-01T10:06:44.383 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: block_size_deviation: 10 2023-10-01T10:06:44.383 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: block_restart_interval: 16 2023-10-01T10:06:44.383 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: index_block_restart_interval: 1 2023-10-01T10:06:44.384 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: metadata_block_size: 4096 2023-10-01T10:06:44.384 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: partition_filters: 0 2023-10-01T10:06:44.384 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: use_delta_encoding: 1 2023-10-01T10:06:44.384 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: filter_policy: rocksdb.BuiltinBloomFilter 2023-10-01T10:06:44.384 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: whole_key_filtering: 1 2023-10-01T10:06:44.384 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: verify_compression: 0 2023-10-01T10:06:44.384 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: read_amp_bytes_per_bit: 0 2023-10-01T10:06:44.384 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: format_version: 2 2023-10-01T10:06:44.384 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: enable_index_compression: 1 2023-10-01T10:06:44.384 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: block_align: 0 2023-10-01T10:06:44.384 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.write_buffer_size: 33554432 2023-10-01T10:06:44.384 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.max_write_buffer_number: 2 2023-10-01T10:06:44.385 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.compression: NoCompression 2023-10-01T10:06:44.385 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.bottommost_compression: Disabled 2023-10-01T10:06:44.385 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.prefix_extractor: nullptr 2023-10-01T10:06:44.385 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-01T10:06:44.385 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.num_levels: 7 2023-10-01T10:06:44.385 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-01T10:06:44.385 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-01T10:06:44.385 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-01T10:06:44.385 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-01T10:06:44.385 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-01T10:06:44.385 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-01T10:06:44.385 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-01T10:06:44.385 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-01T10:06:44.386 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-01T10:06:44.386 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.compression_opts.window_bits: -14 2023-10-01T10:06:44.386 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.compression_opts.level: 32767 2023-10-01T10:06:44.386 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.compression_opts.strategy: 0 2023-10-01T10:06:44.386 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-01T10:06:44.386 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-01T10:06:44.386 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.compression_opts.enabled: false 2023-10-01T10:06:44.386 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-01T10:06:44.386 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-01T10:06:44.386 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-01T10:06:44.386 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.target_file_size_base: 67108864 2023-10-01T10:06:44.387 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.target_file_size_multiplier: 1 2023-10-01T10:06:44.387 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-01T10:06:44.387 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-01T10:06:44.387 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-01T10:06:44.387 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-01T10:06:44.387 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-01T10:06:44.387 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-01T10:06:44.388 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-01T10:06:44.388 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-01T10:06:44.388 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-01T10:06:44.388 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-01T10:06:44.388 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-01T10:06:44.388 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-01T10:06:44.388 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.arena_block_size: 4194304 2023-10-01T10:06:44.388 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-01T10:06:44.389 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-01T10:06:44.389 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-10-01T10:06:44.389 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.disable_auto_compactions: 0 2023-10-01T10:06:44.389 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-01T10:06:44.389 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-01T10:06:44.390 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-01T10:06:44.390 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-01T10:06:44.390 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-01T10:06:44.390 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-01T10:06:44.390 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-01T10:06:44.391 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-01T10:06:44.391 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-01T10:06:44.391 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-01T10:06:44.391 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.table_properties_collectors: 2023-10-01T10:06:44.391 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.inplace_update_support: 0 2023-10-01T10:06:44.391 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-10-01T10:06:44.392 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-01T10:06:44.392 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-01T10:06:44.392 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.memtable_huge_page_size: 0 2023-10-01T10:06:44.392 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.bloom_locality: 0 2023-10-01T10:06:44.392 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.max_successive_merges: 0 2023-10-01T10:06:44.392 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-10-01T10:06:44.392 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.paranoid_file_checks: 0 2023-10-01T10:06:44.392 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.force_consistency_checks: 0 2023-10-01T10:06:44.393 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.report_bg_io_stats: 0 2023-10-01T10:06:44.393 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.ttl: 2592000 2023-10-01T10:06:44.394 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-10-01T10:06:44.394 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-01T10:06:44.394 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 0 2023-10-01T10:06:44.394 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696154804147702, "job": 1, "event": "recovery_started", "log_files": [3]} 2023-10-01T10:06:44.394 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: [db_impl/db_impl_open.cc:760] Recovering log #3 mode 2 2023-10-01T10:06:44.394 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 3 rocksdb: [le/block_based/filter_policy.cc:584] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-10-01T10:06:44.394 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696154804148455, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 4, "file_size": 1437, "table_properties": {"data_size": 523, "index_size": 40, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 0, "index_value_is_delta_encoded": 0, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1696154804, "oldest_key_time": 3, "file_creation_time": 0}} 2023-10-01T10:06:44.395 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.144+0000 7f014512e880 4 rocksdb: [version_set.cc:3826] Creating manifest 5 2023-10-01T10:06:44.395 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.148+0000 7f014512e880 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696154804149112, "job": 1, "event": "recovery_finished"} 2023-10-01T10:06:44.395 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.148+0000 7f014512e880 4 rocksdb: DB pointer 0x5602a1ee9800 2023-10-01T10:06:44.395 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.148+0000 7f012bf36700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2023-10-01T10:06:44.395 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.148+0000 7f012bf36700 4 rocksdb: [db_impl/db_impl.cc:851] 2023-10-01T10:06:44.395 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: ** DB Stats ** 2023-10-01T10:06:44.395 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-01T10:06:44.396 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-01T10:06:44.396 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-01T10:06:44.396 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-01T10:06:44.396 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-01T10:06:44.396 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-10-01T10:06:44.396 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-01T10:06:44.396 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: ** Compaction Stats [default] ** 2023-10-01T10:06:44.396 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-01T10:06:44.397 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-01T10:06:44.397 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: L0 1/0 1.40 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 2.0 0.00 0.00 1 0.001 0 0 2023-10-01T10:06:44.397 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: Sum 1/0 1.40 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 2.0 0.00 0.00 1 0.001 0 0 2023-10-01T10:06:44.397 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 2.0 0.00 0.00 1 0.001 0 0 2023-10-01T10:06:44.397 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: ** Compaction Stats [default] ** 2023-10-01T10:06:44.397 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-01T10:06:44.397 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-01T10:06:44.397 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2.0 0.00 0.00 1 0.001 0 0 2023-10-01T10:06:44.398 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-01T10:06:44.398 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: Flush(GB): cumulative 0.000, interval 0.000 2023-10-01T10:06:44.398 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-01T10:06:44.398 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: AddFile(Total Files): cumulative 0, interval 0 2023-10-01T10:06:44.398 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: AddFile(L0 Files): cumulative 0, interval 0 2023-10-01T10:06:44.398 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: AddFile(Keys): cumulative 0, interval 0 2023-10-01T10:06:44.398 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: Cumulative compaction: 0.00 GB write, 0.35 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-01T10:06:44.399 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: Interval compaction: 0.00 GB write, 0.35 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-01T10:06:44.399 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-01T10:06:44.399 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: ** File Read Latency Histogram By Level [default] ** 2023-10-01T10:06:44.399 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: ** Compaction Stats [default] ** 2023-10-01T10:06:44.399 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-01T10:06:44.399 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-01T10:06:44.399 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: L0 1/0 1.40 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 2.0 0.00 0.00 1 0.001 0 0 2023-10-01T10:06:44.400 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: Sum 1/0 1.40 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 2.0 0.00 0.00 1 0.001 0 0 2023-10-01T10:06:44.400 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-10-01T10:06:44.400 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: ** Compaction Stats [default] ** 2023-10-01T10:06:44.400 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-10-01T10:06:44.400 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-01T10:06:44.400 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 2.0 0.00 0.00 1 0.001 0 0 2023-10-01T10:06:44.401 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-01T10:06:44.401 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: Flush(GB): cumulative 0.000, interval 0.000 2023-10-01T10:06:44.401 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-01T10:06:44.401 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: AddFile(Total Files): cumulative 0, interval 0 2023-10-01T10:06:44.401 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: AddFile(L0 Files): cumulative 0, interval 0 2023-10-01T10:06:44.401 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: AddFile(Keys): cumulative 0, interval 0 2023-10-01T10:06:44.401 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: Cumulative compaction: 0.00 GB write, 0.33 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-01T10:06:44.401 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-01T10:06:44.402 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-01T10:06:44.402 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: ** File Read Latency Histogram By Level [default] ** 2023-10-01T10:06:44.402 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.148+0000 7f014512e880 0 mon.b does not exist in monmap, will attempt to join an existing cluster 2023-10-01T10:06:44.402 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.148+0000 7f014512e880 0 using public_addr v2:172.21.15.173:0/0 -> [v2:172.21.15.173:3300/0,v1:172.21.15.173:6789/0] 2023-10-01T10:06:44.402 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.148+0000 7f014512e880 0 starting mon.b rank -1 at public addrs [v2:172.21.15.173:3300/0,v1:172.21.15.173:6789/0] at bind addrs [v2:172.21.15.173:3300/0,v1:172.21.15.173:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid d8016480-6041-11ee-8db5-212e2dc638e7 2023-10-01T10:06:44.403 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.152+0000 7f014512e880 1 mon.b@-1(???) e0 preinit fsid d8016480-6041-11ee-8db5-212e2dc638e7 2023-10-01T10:06:44.403 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.176+0000 7f012e73b700 0 mon.b@-1(synchronizing).mds e1 new map 2023-10-01T10:06:44.403 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.176+0000 7f012e73b700 0 mon.b@-1(synchronizing).mds e1 print_map 2023-10-01T10:06:44.403 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: e1 2023-10-01T10:06:44.403 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: enable_multiple, ever_enabled_multiple: 1,1 2023-10-01T10:06:44.403 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-01T10:06:44.403 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: legacy client fscid: -1 2023-10-01T10:06:44.404 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: 2023-10-01T10:06:44.404 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: No filesystems configured 2023-10-01T10:06:44.404 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.176+0000 7f012e73b700 1 mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2023-10-01T10:06:44.404 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.176+0000 7f012e73b700 1 mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2023-10-01T10:06:44.404 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.176+0000 7f012e73b700 1 mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2023-10-01T10:06:44.404 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.176+0000 7f012e73b700 1 mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2023-10-01T10:06:44.405 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.176+0000 7f012e73b700 1 mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2023-10-01T10:06:44.405 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.176+0000 7f012e73b700 1 mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2023-10-01T10:06:44.405 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.180+0000 7f012e73b700 0 mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2023-10-01T10:06:44.405 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.180+0000 7f012e73b700 0 mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-01T10:06:44.405 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.180+0000 7f012e73b700 0 mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-01T10:06:44.406 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.180+0000 7f012e73b700 0 mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-10-01T10:06:44.406 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: cluster 2023-10-01T10:06:41.746717+0000 mgr.y (mgr.14146) 28 : cluster [DBG] pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:06:44.406 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: audit 2023-10-01T10:06:42.690182+0000 mon.a (mon.0) 124 : audit [DBG] from='client.? 172.21.15.173:0/4164141511' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-01T10:06:44.406 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:44 smithi173 bash[27612]: debug 2023-10-01T10:06:44.180+0000 7f012e73b700 1 mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-10-01T10:06:44.603 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-10-01T10:06:44.604 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph mon dump -f json 2023-10-01T10:06:48.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:06:48 smithi078 bash[33821]: debug 2023-10-01T10:06:48.395+0000 7f9c1a222700 1 mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-10-01T10:06:49.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:49 smithi078 bash[27336]: cluster 2023-10-01T10:06:43.747131+0000 mgr.y (mgr.14146) 30 : cluster [DBG] pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:06:49.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:49 smithi078 bash[27336]: audit 2023-10-01T10:06:44.188587+0000 mon.a (mon.0) 128 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T10:06:49.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:49 smithi078 bash[27336]: audit 2023-10-01T10:06:44.193019+0000 mon.a (mon.0) 129 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T10:06:49.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:49 smithi078 bash[27336]: cluster 2023-10-01T10:06:44.193277+0000 mon.a (mon.0) 130 : cluster [INF] mon.a calling monitor election 2023-10-01T10:06:49.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:49 smithi078 bash[27336]: audit 2023-10-01T10:06:44.214554+0000 mon.a (mon.0) 131 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T10:06:49.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:49 smithi078 bash[27336]: audit 2023-10-01T10:06:45.188664+0000 mon.a (mon.0) 132 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T10:06:49.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:49 smithi078 bash[27336]: cluster 2023-10-01T10:06:45.747565+0000 mgr.y (mgr.14146) 31 : cluster [DBG] pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:06:49.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:49 smithi078 bash[27336]: audit 2023-10-01T10:06:46.188625+0000 mon.a (mon.0) 133 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T10:06:49.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:49 smithi078 bash[27336]: cluster 2023-10-01T10:06:46.189478+0000 mon.b (mon.1) 1 : cluster [INF] mon.b calling monitor election 2023-10-01T10:06:49.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:49 smithi078 bash[27336]: audit 2023-10-01T10:06:47.188575+0000 mon.a (mon.0) 134 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T10:06:49.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:49 smithi078 bash[27336]: cluster 2023-10-01T10:06:47.747999+0000 mgr.y (mgr.14146) 32 : cluster [DBG] pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:06:49.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:49 smithi078 bash[27336]: audit 2023-10-01T10:06:48.188868+0000 mon.a (mon.0) 135 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T10:06:49.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:49 smithi078 bash[27336]: audit 2023-10-01T10:06:48.401597+0000 mon.a (mon.0) 136 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T10:06:49.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:49 smithi078 bash[27336]: audit 2023-10-01T10:06:49.189440+0000 mon.a (mon.0) 137 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T10:06:49.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:49 smithi078 bash[27336]: cluster 2023-10-01T10:06:49.235472+0000 mon.a (mon.0) 138 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-10-01T10:06:49.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:49 smithi078 bash[27336]: cluster 2023-10-01T10:06:49.248011+0000 mon.a (mon.0) 139 : cluster [DBG] monmap e2: 2 mons at {a=[v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0],b=[v2:172.21.15.173:3300/0,v1:172.21.15.173:6789/0]} removed_ranks: {} 2023-10-01T10:06:49.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:49 smithi078 bash[27336]: cluster 2023-10-01T10:06:49.248827+0000 mon.a (mon.0) 140 : cluster [DBG] fsmap 2023-10-01T10:06:49.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:49 smithi078 bash[27336]: cluster 2023-10-01T10:06:49.248906+0000 mon.a (mon.0) 141 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-01T10:06:49.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:49 smithi078 bash[27336]: cluster 2023-10-01T10:06:49.249223+0000 mon.a (mon.0) 142 : cluster [DBG] mgrmap e13: y(active, since 51s) 2023-10-01T10:06:49.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:49 smithi078 bash[27336]: cluster 2023-10-01T10:06:49.256757+0000 mon.a (mon.0) 143 : cluster [INF] overall HEALTH_OK 2023-10-01T10:06:49.585 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:49 smithi078 bash[27336]: audit 2023-10-01T10:06:49.261963+0000 mon.a (mon.0) 144 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:06:49.585 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:49 smithi078 bash[27336]: audit 2023-10-01T10:06:49.263990+0000 mon.a (mon.0) 145 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:06:49.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:49 smithi173 bash[27612]: cluster 2023-10-01T10:06:43.747131+0000 mgr.y (mgr.14146) 30 : cluster [DBG] pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:06:49.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:49 smithi173 bash[27612]: audit 2023-10-01T10:06:44.188587+0000 mon.a (mon.0) 128 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T10:06:49.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:49 smithi173 bash[27612]: audit 2023-10-01T10:06:44.193019+0000 mon.a (mon.0) 129 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T10:06:49.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:49 smithi173 bash[27612]: cluster 2023-10-01T10:06:44.193277+0000 mon.a (mon.0) 130 : cluster [INF] mon.a calling monitor election 2023-10-01T10:06:49.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:49 smithi173 bash[27612]: audit 2023-10-01T10:06:44.214554+0000 mon.a (mon.0) 131 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T10:06:49.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:49 smithi173 bash[27612]: audit 2023-10-01T10:06:45.188664+0000 mon.a (mon.0) 132 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T10:06:49.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:49 smithi173 bash[27612]: cluster 2023-10-01T10:06:45.747565+0000 mgr.y (mgr.14146) 31 : cluster [DBG] pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:06:49.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:49 smithi173 bash[27612]: audit 2023-10-01T10:06:46.188625+0000 mon.a (mon.0) 133 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T10:06:49.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:49 smithi173 bash[27612]: cluster 2023-10-01T10:06:46.189478+0000 mon.b (mon.1) 1 : cluster [INF] mon.b calling monitor election 2023-10-01T10:06:49.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:49 smithi173 bash[27612]: audit 2023-10-01T10:06:47.188575+0000 mon.a (mon.0) 134 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T10:06:49.622 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:49 smithi173 bash[27612]: cluster 2023-10-01T10:06:47.747999+0000 mgr.y (mgr.14146) 32 : cluster [DBG] pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:06:49.622 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:49 smithi173 bash[27612]: audit 2023-10-01T10:06:48.188868+0000 mon.a (mon.0) 135 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T10:06:49.622 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:49 smithi173 bash[27612]: audit 2023-10-01T10:06:48.401597+0000 mon.a (mon.0) 136 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T10:06:49.622 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:49 smithi173 bash[27612]: audit 2023-10-01T10:06:49.189440+0000 mon.a (mon.0) 137 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T10:06:49.622 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:49 smithi173 bash[27612]: cluster 2023-10-01T10:06:49.235472+0000 mon.a (mon.0) 138 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-10-01T10:06:49.622 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:49 smithi173 bash[27612]: cluster 2023-10-01T10:06:49.248011+0000 mon.a (mon.0) 139 : cluster [DBG] monmap e2: 2 mons at {a=[v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0],b=[v2:172.21.15.173:3300/0,v1:172.21.15.173:6789/0]} removed_ranks: {} 2023-10-01T10:06:49.622 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:49 smithi173 bash[27612]: cluster 2023-10-01T10:06:49.248827+0000 mon.a (mon.0) 140 : cluster [DBG] fsmap 2023-10-01T10:06:49.623 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:49 smithi173 bash[27612]: cluster 2023-10-01T10:06:49.248906+0000 mon.a (mon.0) 141 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-01T10:06:49.623 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:49 smithi173 bash[27612]: cluster 2023-10-01T10:06:49.249223+0000 mon.a (mon.0) 142 : cluster [DBG] mgrmap e13: y(active, since 51s) 2023-10-01T10:06:49.623 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:49 smithi173 bash[27612]: cluster 2023-10-01T10:06:49.256757+0000 mon.a (mon.0) 143 : cluster [INF] overall HEALTH_OK 2023-10-01T10:06:49.623 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:49 smithi173 bash[27612]: audit 2023-10-01T10:06:49.261963+0000 mon.a (mon.0) 144 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:06:49.623 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:49 smithi173 bash[27612]: audit 2023-10-01T10:06:49.263990+0000 mon.a (mon.0) 145 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:06:50.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:50 smithi078 bash[27336]: audit 2023-10-01T10:06:49.402406+0000 mon.a (mon.0) 146 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T10:06:50.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:50 smithi078 bash[27336]: cluster 2023-10-01T10:06:49.748448+0000 mgr.y (mgr.14146) 33 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:06:50.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:50 smithi078 bash[27336]: audit 2023-10-01T10:06:50.189358+0000 mon.a (mon.0) 147 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T10:06:50.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:50 smithi173 bash[27612]: audit 2023-10-01T10:06:49.402406+0000 mon.a (mon.0) 146 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T10:06:50.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:50 smithi173 bash[27612]: cluster 2023-10-01T10:06:49.748448+0000 mgr.y (mgr.14146) 33 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:06:50.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:50 smithi173 bash[27612]: audit 2023-10-01T10:06:50.189358+0000 mon.a (mon.0) 147 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T10:06:55.477 INFO:teuthology.orchestra.run.smithi173.stdout: 2023-10-01T10:06:55.477 INFO:teuthology.orchestra.run.smithi173.stdout:{"epoch":3,"fsid":"d8016480-6041-11ee-8db5-212e2dc638e7","modified":"2023-10-01T10:06:50.402906Z","created":"2023-10-01T10:04:55.006877Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:3300","nonce":0},{"type":"v1","addr":"172.21.15.173:6789","nonce":0}]},"addr":"172.21.15.173:6789/0","public_addr":"172.21.15.173:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3301","nonce":0},{"type":"v1","addr":"172.21.15.78:6790","nonce":0}]},"addr":"172.21.15.78:6790/0","public_addr":"172.21.15.78:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2023-10-01T10:06:55.482 INFO:teuthology.orchestra.run.smithi173.stderr:dumped monmap epoch 3 2023-10-01T10:06:55.767 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:06:55 smithi078 bash[33821]: audit 2023-10-01T10:06:49.402406+0000 mon.a (mon.0) 146 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T10:06:55.768 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:06:55 smithi078 bash[33821]: cluster 2023-10-01T10:06:49.748448+0000 mgr.y (mgr.14146) 33 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:06:55.768 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:06:55 smithi078 bash[33821]: audit 2023-10-01T10:06:50.189358+0000 mon.a (mon.0) 147 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T10:06:55.768 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:06:55 smithi078 bash[33821]: audit 2023-10-01T10:06:50.407847+0000 mon.a (mon.0) 149 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T10:06:55.768 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:06:55 smithi078 bash[33821]: cluster 2023-10-01T10:06:50.408275+0000 mon.a (mon.0) 150 : cluster [INF] mon.a calling monitor election 2023-10-01T10:06:55.768 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:06:55 smithi078 bash[33821]: cluster 2023-10-01T10:06:50.409084+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2023-10-01T10:06:55.769 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:06:55 smithi078 bash[33821]: audit 2023-10-01T10:06:50.426777+0000 mon.a (mon.0) 151 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T10:06:55.769 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:06:55 smithi078 bash[33821]: audit 2023-10-01T10:06:50.427101+0000 mon.a (mon.0) 152 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T10:06:55.769 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:06:55 smithi078 bash[33821]: audit 2023-10-01T10:06:50.508229+0000 mon.a (mon.0) 153 : audit [DBG] from='client.? 172.21.15.173:0/1274939728' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-01T10:06:55.769 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:06:55 smithi078 bash[33821]: audit 2023-10-01T10:06:51.402771+0000 mon.a (mon.0) 154 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T10:06:55.770 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:06:55 smithi078 bash[33821]: cluster 2023-10-01T10:06:51.748886+0000 mgr.y (mgr.14146) 34 : cluster [DBG] pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:06:55.770 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:06:55 smithi078 bash[33821]: audit 2023-10-01T10:06:52.402944+0000 mon.a (mon.0) 155 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T10:06:55.770 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:06:55 smithi078 bash[33821]: cluster 2023-10-01T10:06:52.405074+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2023-10-01T10:06:55.770 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:06:55 smithi078 bash[33821]: audit 2023-10-01T10:06:53.402766+0000 mon.a (mon.0) 156 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T10:06:55.770 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:06:55 smithi078 bash[33821]: cluster 2023-10-01T10:06:53.749293+0000 mgr.y (mgr.14146) 35 : cluster [DBG] pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:06:55.770 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:06:55 smithi078 bash[33821]: audit 2023-10-01T10:06:54.402649+0000 mon.a (mon.0) 157 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T10:06:55.771 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:06:55 smithi078 bash[33821]: audit 2023-10-01T10:06:55.402948+0000 mon.a (mon.0) 158 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T10:06:55.771 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:06:55 smithi078 bash[33821]: cluster 2023-10-01T10:06:55.447246+0000 mon.a (mon.0) 159 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-01T10:06:55.771 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:06:55 smithi078 bash[33821]: cluster 2023-10-01T10:06:55.456313+0000 mon.a (mon.0) 160 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0],b=[v2:172.21.15.173:3300/0,v1:172.21.15.173:6789/0],c=[v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0]} removed_ranks: {} 2023-10-01T10:06:55.771 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:06:55 smithi078 bash[33821]: cluster 2023-10-01T10:06:55.474163+0000 mon.a (mon.0) 161 : cluster [DBG] fsmap 2023-10-01T10:06:55.772 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:55 smithi078 bash[27336]: audit 2023-10-01T10:06:50.407847+0000 mon.a (mon.0) 149 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T10:06:55.772 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:55 smithi078 bash[27336]: cluster 2023-10-01T10:06:50.408275+0000 mon.a (mon.0) 150 : cluster [INF] mon.a calling monitor election 2023-10-01T10:06:55.772 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:55 smithi078 bash[27336]: cluster 2023-10-01T10:06:50.409084+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2023-10-01T10:06:55.772 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:55 smithi078 bash[27336]: audit 2023-10-01T10:06:50.426777+0000 mon.a (mon.0) 151 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T10:06:55.773 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:55 smithi078 bash[27336]: audit 2023-10-01T10:06:50.427101+0000 mon.a (mon.0) 152 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T10:06:55.773 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:55 smithi078 bash[27336]: audit 2023-10-01T10:06:50.508229+0000 mon.a (mon.0) 153 : audit [DBG] from='client.? 172.21.15.173:0/1274939728' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-01T10:06:55.773 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:55 smithi078 bash[27336]: audit 2023-10-01T10:06:51.402771+0000 mon.a (mon.0) 154 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T10:06:55.773 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:55 smithi078 bash[27336]: cluster 2023-10-01T10:06:51.748886+0000 mgr.y (mgr.14146) 34 : cluster [DBG] pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:06:55.773 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:55 smithi078 bash[27336]: audit 2023-10-01T10:06:52.402944+0000 mon.a (mon.0) 155 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T10:06:55.773 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:55 smithi078 bash[27336]: cluster 2023-10-01T10:06:52.405074+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2023-10-01T10:06:55.774 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:55 smithi078 bash[27336]: audit 2023-10-01T10:06:53.402766+0000 mon.a (mon.0) 156 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T10:06:55.774 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:55 smithi078 bash[27336]: cluster 2023-10-01T10:06:53.749293+0000 mgr.y (mgr.14146) 35 : cluster [DBG] pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:06:55.774 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:55 smithi078 bash[27336]: audit 2023-10-01T10:06:54.402649+0000 mon.a (mon.0) 157 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T10:06:55.774 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:55 smithi078 bash[27336]: audit 2023-10-01T10:06:55.402948+0000 mon.a (mon.0) 158 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T10:06:55.774 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:55 smithi078 bash[27336]: cluster 2023-10-01T10:06:55.447246+0000 mon.a (mon.0) 159 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-01T10:06:55.774 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:55 smithi078 bash[27336]: cluster 2023-10-01T10:06:55.456313+0000 mon.a (mon.0) 160 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0],b=[v2:172.21.15.173:3300/0,v1:172.21.15.173:6789/0],c=[v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0]} removed_ranks: {} 2023-10-01T10:06:55.775 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:55 smithi078 bash[27336]: cluster 2023-10-01T10:06:55.474163+0000 mon.a (mon.0) 161 : cluster [DBG] fsmap 2023-10-01T10:06:55.775 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:55 smithi078 bash[27336]: cluster 2023-10-01T10:06:55.474259+0000 mon.a (mon.0) 162 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-01T10:06:55.775 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:55 smithi078 bash[27336]: cluster 2023-10-01T10:06:55.474596+0000 mon.a (mon.0) 163 : cluster [DBG] mgrmap e13: y(active, since 57s) 2023-10-01T10:06:55.775 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:55 smithi078 bash[27336]: cluster 2023-10-01T10:06:55.482276+0000 mon.a (mon.0) 164 : cluster [INF] overall HEALTH_OK 2023-10-01T10:06:55.775 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:55 smithi078 bash[27336]: audit 2023-10-01T10:06:55.487453+0000 mon.a (mon.0) 165 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:06:55.775 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:55 smithi078 bash[27336]: audit 2023-10-01T10:06:55.493323+0000 mon.a (mon.0) 166 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:06:55.775 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:55 smithi078 bash[27336]: audit 2023-10-01T10:06:55.498993+0000 mon.a (mon.0) 167 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:06:55.776 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:55 smithi078 bash[27336]: audit 2023-10-01T10:06:55.500206+0000 mon.a (mon.0) 168 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:06:55.776 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:55 smithi078 bash[27336]: audit 2023-10-01T10:06:55.501414+0000 mon.a (mon.0) 169 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:06:55.776 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:06:55 smithi078 bash[33821]: cluster 2023-10-01T10:06:55.474259+0000 mon.a (mon.0) 162 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-01T10:06:55.776 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:06:55 smithi078 bash[33821]: cluster 2023-10-01T10:06:55.474596+0000 mon.a (mon.0) 163 : cluster [DBG] mgrmap e13: y(active, since 57s) 2023-10-01T10:06:55.776 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:06:55 smithi078 bash[33821]: cluster 2023-10-01T10:06:55.482276+0000 mon.a (mon.0) 164 : cluster [INF] overall HEALTH_OK 2023-10-01T10:06:55.777 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:06:55 smithi078 bash[33821]: audit 2023-10-01T10:06:55.487453+0000 mon.a (mon.0) 165 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:06:55.777 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:06:55 smithi078 bash[33821]: audit 2023-10-01T10:06:55.493323+0000 mon.a (mon.0) 166 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:06:55.777 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:06:55 smithi078 bash[33821]: audit 2023-10-01T10:06:55.498993+0000 mon.a (mon.0) 167 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:06:55.777 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:06:55 smithi078 bash[33821]: audit 2023-10-01T10:06:55.500206+0000 mon.a (mon.0) 168 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:06:55.777 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:06:55 smithi078 bash[33821]: audit 2023-10-01T10:06:55.501414+0000 mon.a (mon.0) 169 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:06:55.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:55 smithi173 bash[27612]: audit 2023-10-01T10:06:50.407847+0000 mon.a (mon.0) 149 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T10:06:55.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:55 smithi173 bash[27612]: cluster 2023-10-01T10:06:50.408275+0000 mon.a (mon.0) 150 : cluster [INF] mon.a calling monitor election 2023-10-01T10:06:55.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:55 smithi173 bash[27612]: cluster 2023-10-01T10:06:50.409084+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2023-10-01T10:06:55.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:55 smithi173 bash[27612]: audit 2023-10-01T10:06:50.426777+0000 mon.a (mon.0) 151 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T10:06:55.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:55 smithi173 bash[27612]: audit 2023-10-01T10:06:50.427101+0000 mon.a (mon.0) 152 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T10:06:55.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:55 smithi173 bash[27612]: audit 2023-10-01T10:06:50.508229+0000 mon.a (mon.0) 153 : audit [DBG] from='client.? 172.21.15.173:0/1274939728' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-01T10:06:55.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:55 smithi173 bash[27612]: audit 2023-10-01T10:06:51.402771+0000 mon.a (mon.0) 154 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T10:06:55.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:55 smithi173 bash[27612]: cluster 2023-10-01T10:06:51.748886+0000 mgr.y (mgr.14146) 34 : cluster [DBG] pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:06:55.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:55 smithi173 bash[27612]: audit 2023-10-01T10:06:52.402944+0000 mon.a (mon.0) 155 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T10:06:55.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:55 smithi173 bash[27612]: cluster 2023-10-01T10:06:52.405074+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2023-10-01T10:06:55.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:55 smithi173 bash[27612]: audit 2023-10-01T10:06:53.402766+0000 mon.a (mon.0) 156 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T10:06:55.872 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:55 smithi173 bash[27612]: cluster 2023-10-01T10:06:53.749293+0000 mgr.y (mgr.14146) 35 : cluster [DBG] pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:06:55.872 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:55 smithi173 bash[27612]: audit 2023-10-01T10:06:54.402649+0000 mon.a (mon.0) 157 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T10:06:55.872 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:55 smithi173 bash[27612]: audit 2023-10-01T10:06:55.402948+0000 mon.a (mon.0) 158 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T10:06:55.872 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:55 smithi173 bash[27612]: cluster 2023-10-01T10:06:55.447246+0000 mon.a (mon.0) 159 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-01T10:06:55.872 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:55 smithi173 bash[27612]: cluster 2023-10-01T10:06:55.456313+0000 mon.a (mon.0) 160 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0],b=[v2:172.21.15.173:3300/0,v1:172.21.15.173:6789/0],c=[v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0]} removed_ranks: {} 2023-10-01T10:06:55.872 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:55 smithi173 bash[27612]: cluster 2023-10-01T10:06:55.474163+0000 mon.a (mon.0) 161 : cluster [DBG] fsmap 2023-10-01T10:06:55.872 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:55 smithi173 bash[27612]: cluster 2023-10-01T10:06:55.474259+0000 mon.a (mon.0) 162 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-10-01T10:06:55.872 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:55 smithi173 bash[27612]: cluster 2023-10-01T10:06:55.474596+0000 mon.a (mon.0) 163 : cluster [DBG] mgrmap e13: y(active, since 57s) 2023-10-01T10:06:55.872 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:55 smithi173 bash[27612]: cluster 2023-10-01T10:06:55.482276+0000 mon.a (mon.0) 164 : cluster [INF] overall HEALTH_OK 2023-10-01T10:06:55.873 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:55 smithi173 bash[27612]: audit 2023-10-01T10:06:55.487453+0000 mon.a (mon.0) 165 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:06:55.873 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:55 smithi173 bash[27612]: audit 2023-10-01T10:06:55.493323+0000 mon.a (mon.0) 166 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:06:55.873 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:55 smithi173 bash[27612]: audit 2023-10-01T10:06:55.498993+0000 mon.a (mon.0) 167 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:06:55.873 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:55 smithi173 bash[27612]: audit 2023-10-01T10:06:55.500206+0000 mon.a (mon.0) 168 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:06:55.873 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:55 smithi173 bash[27612]: audit 2023-10-01T10:06:55.501414+0000 mon.a (mon.0) 169 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:06:56.216 INFO:tasks.cephadm:Generating final ceph.conf file... 2023-10-01T10:06:56.217 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph config generate-minimal-conf 2023-10-01T10:06:56.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:56 smithi078 bash[27336]: cephadm 2023-10-01T10:06:55.503160+0000 mgr.y (mgr.14146) 36 : cephadm [INF] Updating smithi078:/etc/ceph/ceph.conf 2023-10-01T10:06:56.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:56 smithi078 bash[27336]: cephadm 2023-10-01T10:06:55.503562+0000 mgr.y (mgr.14146) 37 : cephadm [INF] Updating smithi173:/etc/ceph/ceph.conf 2023-10-01T10:06:56.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:56 smithi078 bash[27336]: audit 2023-10-01T10:06:55.546527+0000 mon.a (mon.0) 170 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:06:56.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:56 smithi078 bash[27336]: audit 2023-10-01T10:06:55.552320+0000 mon.a (mon.0) 171 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:06:56.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:56 smithi078 bash[27336]: audit 2023-10-01T10:06:55.556612+0000 mon.a (mon.0) 172 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:06:56.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:56 smithi078 bash[27336]: cephadm 2023-10-01T10:06:55.558665+0000 mgr.y (mgr.14146) 38 : cephadm [INF] Reconfiguring mon.a (unknown last config time)... 2023-10-01T10:06:56.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:56 smithi078 bash[27336]: audit 2023-10-01T10:06:55.559095+0000 mon.a (mon.0) 173 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T10:06:56.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:56 smithi078 bash[27336]: audit 2023-10-01T10:06:55.560256+0000 mon.a (mon.0) 174 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T10:06:56.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:56 smithi078 bash[27336]: audit 2023-10-01T10:06:55.561369+0000 mon.a (mon.0) 175 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:06:56.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:56 smithi078 bash[27336]: cephadm 2023-10-01T10:06:55.562314+0000 mgr.y (mgr.14146) 39 : cephadm [INF] Reconfiguring daemon mon.a on smithi078 2023-10-01T10:06:56.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:56 smithi078 bash[27336]: cluster 2023-10-01T10:06:55.749676+0000 mgr.y (mgr.14146) 40 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:06:56.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:56 smithi078 bash[27336]: audit 2023-10-01T10:06:56.402807+0000 mon.a (mon.0) 176 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T10:06:56.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:06:56 smithi078 bash[33821]: cephadm 2023-10-01T10:06:55.503160+0000 mgr.y (mgr.14146) 36 : cephadm [INF] Updating smithi078:/etc/ceph/ceph.conf 2023-10-01T10:06:56.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:06:56 smithi078 bash[33821]: cephadm 2023-10-01T10:06:55.503562+0000 mgr.y (mgr.14146) 37 : cephadm [INF] Updating smithi173:/etc/ceph/ceph.conf 2023-10-01T10:06:56.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:06:56 smithi078 bash[33821]: audit 2023-10-01T10:06:55.546527+0000 mon.a (mon.0) 170 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:06:56.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:06:56 smithi078 bash[33821]: audit 2023-10-01T10:06:55.552320+0000 mon.a (mon.0) 171 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:06:56.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:06:56 smithi078 bash[33821]: audit 2023-10-01T10:06:55.556612+0000 mon.a (mon.0) 172 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:06:56.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:06:56 smithi078 bash[33821]: cephadm 2023-10-01T10:06:55.558665+0000 mgr.y (mgr.14146) 38 : cephadm [INF] Reconfiguring mon.a (unknown last config time)... 2023-10-01T10:06:56.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:06:56 smithi078 bash[33821]: audit 2023-10-01T10:06:55.559095+0000 mon.a (mon.0) 173 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T10:06:56.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:06:56 smithi078 bash[33821]: audit 2023-10-01T10:06:55.560256+0000 mon.a (mon.0) 174 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T10:06:56.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:06:56 smithi078 bash[33821]: audit 2023-10-01T10:06:55.561369+0000 mon.a (mon.0) 175 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:06:56.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:06:56 smithi078 bash[33821]: cephadm 2023-10-01T10:06:55.562314+0000 mgr.y (mgr.14146) 39 : cephadm [INF] Reconfiguring daemon mon.a on smithi078 2023-10-01T10:06:56.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:06:56 smithi078 bash[33821]: cluster 2023-10-01T10:06:55.749676+0000 mgr.y (mgr.14146) 40 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:06:56.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:06:56 smithi078 bash[33821]: audit 2023-10-01T10:06:56.402807+0000 mon.a (mon.0) 176 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T10:06:56.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:56 smithi173 bash[27612]: cephadm 2023-10-01T10:06:55.503160+0000 mgr.y (mgr.14146) 36 : cephadm [INF] Updating smithi078:/etc/ceph/ceph.conf 2023-10-01T10:06:56.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:56 smithi173 bash[27612]: cephadm 2023-10-01T10:06:55.503562+0000 mgr.y (mgr.14146) 37 : cephadm [INF] Updating smithi173:/etc/ceph/ceph.conf 2023-10-01T10:06:56.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:56 smithi173 bash[27612]: audit 2023-10-01T10:06:55.546527+0000 mon.a (mon.0) 170 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:06:56.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:56 smithi173 bash[27612]: audit 2023-10-01T10:06:55.552320+0000 mon.a (mon.0) 171 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:06:56.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:56 smithi173 bash[27612]: audit 2023-10-01T10:06:55.556612+0000 mon.a (mon.0) 172 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:06:56.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:56 smithi173 bash[27612]: cephadm 2023-10-01T10:06:55.558665+0000 mgr.y (mgr.14146) 38 : cephadm [INF] Reconfiguring mon.a (unknown last config time)... 2023-10-01T10:06:56.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:56 smithi173 bash[27612]: audit 2023-10-01T10:06:55.559095+0000 mon.a (mon.0) 173 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T10:06:56.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:56 smithi173 bash[27612]: audit 2023-10-01T10:06:55.560256+0000 mon.a (mon.0) 174 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T10:06:56.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:56 smithi173 bash[27612]: audit 2023-10-01T10:06:55.561369+0000 mon.a (mon.0) 175 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:06:56.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:56 smithi173 bash[27612]: cephadm 2023-10-01T10:06:55.562314+0000 mgr.y (mgr.14146) 39 : cephadm [INF] Reconfiguring daemon mon.a on smithi078 2023-10-01T10:06:56.872 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:56 smithi173 bash[27612]: cluster 2023-10-01T10:06:55.749676+0000 mgr.y (mgr.14146) 40 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:06:56.872 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:56 smithi173 bash[27612]: audit 2023-10-01T10:06:56.402807+0000 mon.a (mon.0) 176 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T10:06:58.807 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:06:58 smithi078 bash[33821]: cluster 2023-10-01T10:06:57.749926+0000 mgr.y (mgr.14146) 41 : cluster [DBG] pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:06:58.807 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:06:58 smithi078 bash[33821]: audit 2023-10-01T10:06:58.643928+0000 mon.a (mon.0) 177 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:06:58.808 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:06:58 smithi078 bash[33821]: audit 2023-10-01T10:06:58.645423+0000 mon.a (mon.0) 178 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T10:06:58.808 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:06:58 smithi078 bash[33821]: audit 2023-10-01T10:06:58.646834+0000 mon.a (mon.0) 179 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T10:06:59.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:06:58 smithi078 bash[33821]: audit 2023-10-01T10:06:58.648161+0000 mon.a (mon.0) 180 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:06:59.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:58 smithi078 bash[27336]: cluster 2023-10-01T10:06:57.749926+0000 mgr.y (mgr.14146) 41 : cluster [DBG] pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:06:59.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:58 smithi078 bash[27336]: audit 2023-10-01T10:06:58.643928+0000 mon.a (mon.0) 177 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:06:59.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:58 smithi078 bash[27336]: audit 2023-10-01T10:06:58.645423+0000 mon.a (mon.0) 178 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T10:06:59.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:58 smithi078 bash[27336]: audit 2023-10-01T10:06:58.646834+0000 mon.a (mon.0) 179 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T10:06:59.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:58 smithi078 bash[27336]: audit 2023-10-01T10:06:58.648161+0000 mon.a (mon.0) 180 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:06:59.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:58 smithi173 bash[27612]: cluster 2023-10-01T10:06:57.749926+0000 mgr.y (mgr.14146) 41 : cluster [DBG] pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:06:59.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:58 smithi173 bash[27612]: audit 2023-10-01T10:06:58.643928+0000 mon.a (mon.0) 177 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:06:59.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:58 smithi173 bash[27612]: audit 2023-10-01T10:06:58.645423+0000 mon.a (mon.0) 178 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T10:06:59.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:58 smithi173 bash[27612]: audit 2023-10-01T10:06:58.646834+0000 mon.a (mon.0) 179 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T10:06:59.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:58 smithi173 bash[27612]: audit 2023-10-01T10:06:58.648161+0000 mon.a (mon.0) 180 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:07:00.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:59 smithi078 bash[27336]: cephadm 2023-10-01T10:06:58.644922+0000 mgr.y (mgr.14146) 42 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2023-10-01T10:07:00.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:06:59 smithi078 bash[27336]: cephadm 2023-10-01T10:06:58.649240+0000 mgr.y (mgr.14146) 43 : cephadm [INF] Reconfiguring daemon mon.c on smithi078 2023-10-01T10:07:00.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:06:59 smithi078 bash[33821]: cephadm 2023-10-01T10:06:58.644922+0000 mgr.y (mgr.14146) 42 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2023-10-01T10:07:00.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:06:59 smithi078 bash[33821]: cephadm 2023-10-01T10:06:58.649240+0000 mgr.y (mgr.14146) 43 : cephadm [INF] Reconfiguring daemon mon.c on smithi078 2023-10-01T10:07:00.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:59 smithi173 bash[27612]: cephadm 2023-10-01T10:06:58.644922+0000 mgr.y (mgr.14146) 42 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2023-10-01T10:07:00.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:06:59 smithi173 bash[27612]: cephadm 2023-10-01T10:06:58.649240+0000 mgr.y (mgr.14146) 43 : cephadm [INF] Reconfiguring daemon mon.c on smithi078 2023-10-01T10:07:00.549 INFO:teuthology.orchestra.run.smithi078.stdout:# minimal ceph.conf for d8016480-6041-11ee-8db5-212e2dc638e7 2023-10-01T10:07:00.549 INFO:teuthology.orchestra.run.smithi078.stdout:[global] 2023-10-01T10:07:00.549 INFO:teuthology.orchestra.run.smithi078.stdout: fsid = d8016480-6041-11ee-8db5-212e2dc638e7 2023-10-01T10:07:00.549 INFO:teuthology.orchestra.run.smithi078.stdout: mon_host = [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] [v2:172.21.15.173:3300/0,v1:172.21.15.173:6789/0] [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] 2023-10-01T10:07:01.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:00 smithi078 bash[27336]: cluster 2023-10-01T10:06:59.750184+0000 mgr.y (mgr.14146) 44 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:01.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:00 smithi078 bash[27336]: audit 2023-10-01T10:07:00.548807+0000 mon.c (mon.2) 2 : audit [DBG] from='client.? 172.21.15.78:0/3625892728' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:07:01.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:00 smithi078 bash[33821]: cluster 2023-10-01T10:06:59.750184+0000 mgr.y (mgr.14146) 44 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:01.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:00 smithi078 bash[33821]: audit 2023-10-01T10:07:00.548807+0000 mon.c (mon.2) 2 : audit [DBG] from='client.? 172.21.15.78:0/3625892728' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:07:01.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:00 smithi173 bash[27612]: cluster 2023-10-01T10:06:59.750184+0000 mgr.y (mgr.14146) 44 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:01.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:00 smithi173 bash[27612]: audit 2023-10-01T10:07:00.548807+0000 mon.c (mon.2) 2 : audit [DBG] from='client.? 172.21.15.78:0/3625892728' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:07:01.645 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2023-10-01T10:07:01.645 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2023-10-01T10:07:01.646 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd of=/etc/ceph/ceph.conf 2023-10-01T10:07:01.662 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2023-10-01T10:07:01.662 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-10-01T10:07:01.725 DEBUG:teuthology.orchestra.run.smithi173:> set -ex 2023-10-01T10:07:01.725 DEBUG:teuthology.orchestra.run.smithi173:> sudo dd of=/etc/ceph/ceph.conf 2023-10-01T10:07:01.736 DEBUG:teuthology.orchestra.run.smithi173:> set -ex 2023-10-01T10:07:01.736 DEBUG:teuthology.orchestra.run.smithi173:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-10-01T10:07:01.792 INFO:tasks.cephadm:Adding mgr.y on smithi078 2023-10-01T10:07:01.792 INFO:tasks.cephadm:Adding mgr.x on smithi173 2023-10-01T10:07:01.792 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph orch apply mgr '2;smithi078=y;smithi173=x' 2023-10-01T10:07:02.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:02 smithi173 bash[27612]: audit 2023-10-01T10:07:01.209454+0000 mon.a (mon.0) 181 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:07:02.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:02 smithi173 bash[27612]: cephadm 2023-10-01T10:07:01.210328+0000 mgr.y (mgr.14146) 45 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2023-10-01T10:07:02.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:02 smithi173 bash[27612]: audit 2023-10-01T10:07:01.210643+0000 mon.a (mon.0) 182 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T10:07:02.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:02 smithi173 bash[27612]: audit 2023-10-01T10:07:01.211693+0000 mon.a (mon.0) 183 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T10:07:02.372 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:02 smithi173 bash[27612]: audit 2023-10-01T10:07:01.212624+0000 mon.a (mon.0) 184 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:07:02.372 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:02 smithi173 bash[27612]: cephadm 2023-10-01T10:07:01.213432+0000 mgr.y (mgr.14146) 46 : cephadm [INF] Reconfiguring daemon mon.b on smithi173 2023-10-01T10:07:02.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:02 smithi078 bash[27336]: audit 2023-10-01T10:07:01.209454+0000 mon.a (mon.0) 181 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:07:02.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:02 smithi078 bash[27336]: cephadm 2023-10-01T10:07:01.210328+0000 mgr.y (mgr.14146) 45 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2023-10-01T10:07:02.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:02 smithi078 bash[27336]: audit 2023-10-01T10:07:01.210643+0000 mon.a (mon.0) 182 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T10:07:02.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:02 smithi078 bash[27336]: audit 2023-10-01T10:07:01.211693+0000 mon.a (mon.0) 183 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T10:07:02.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:02 smithi078 bash[27336]: audit 2023-10-01T10:07:01.212624+0000 mon.a (mon.0) 184 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:07:02.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:02 smithi078 bash[27336]: cephadm 2023-10-01T10:07:01.213432+0000 mgr.y (mgr.14146) 46 : cephadm [INF] Reconfiguring daemon mon.b on smithi173 2023-10-01T10:07:02.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:02 smithi078 bash[33821]: audit 2023-10-01T10:07:01.209454+0000 mon.a (mon.0) 181 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:07:02.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:02 smithi078 bash[33821]: cephadm 2023-10-01T10:07:01.210328+0000 mgr.y (mgr.14146) 45 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2023-10-01T10:07:02.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:02 smithi078 bash[33821]: audit 2023-10-01T10:07:01.210643+0000 mon.a (mon.0) 182 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T10:07:02.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:02 smithi078 bash[33821]: audit 2023-10-01T10:07:01.211693+0000 mon.a (mon.0) 183 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T10:07:02.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:02 smithi078 bash[33821]: audit 2023-10-01T10:07:01.212624+0000 mon.a (mon.0) 184 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:07:02.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:02 smithi078 bash[33821]: cephadm 2023-10-01T10:07:01.213432+0000 mgr.y (mgr.14146) 46 : cephadm [INF] Reconfiguring daemon mon.b on smithi173 2023-10-01T10:07:03.544 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:03 smithi173 bash[27612]: cluster 2023-10-01T10:07:01.750581+0000 mgr.y (mgr.14146) 47 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:03.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:03 smithi078 bash[27336]: cluster 2023-10-01T10:07:01.750581+0000 mgr.y (mgr.14146) 47 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:03.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:03 smithi078 bash[33821]: cluster 2023-10-01T10:07:01.750581+0000 mgr.y (mgr.14146) 47 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:05.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:04 smithi078 bash[27336]: audit 2023-10-01T10:07:03.707441+0000 mon.a (mon.0) 185 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:07:05.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:04 smithi078 bash[27336]: audit 2023-10-01T10:07:03.709192+0000 mon.a (mon.0) 186 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:07:05.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:04 smithi078 bash[27336]: audit 2023-10-01T10:07:03.711988+0000 mon.a (mon.0) 187 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:07:05.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:04 smithi078 bash[27336]: audit 2023-10-01T10:07:03.713355+0000 mon.a (mon.0) 188 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:07:05.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:04 smithi078 bash[27336]: audit 2023-10-01T10:07:03.719886+0000 mon.a (mon.0) 189 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:07:05.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:04 smithi078 bash[27336]: cluster 2023-10-01T10:07:03.751029+0000 mgr.y (mgr.14146) 48 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:05.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:04 smithi078 bash[33821]: audit 2023-10-01T10:07:03.707441+0000 mon.a (mon.0) 185 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:07:05.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:04 smithi078 bash[33821]: audit 2023-10-01T10:07:03.709192+0000 mon.a (mon.0) 186 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:07:05.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:04 smithi078 bash[33821]: audit 2023-10-01T10:07:03.711988+0000 mon.a (mon.0) 187 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:07:05.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:04 smithi078 bash[33821]: audit 2023-10-01T10:07:03.713355+0000 mon.a (mon.0) 188 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:07:05.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:04 smithi078 bash[33821]: audit 2023-10-01T10:07:03.719886+0000 mon.a (mon.0) 189 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:07:05.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:04 smithi078 bash[33821]: cluster 2023-10-01T10:07:03.751029+0000 mgr.y (mgr.14146) 48 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:05.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:04 smithi173 bash[27612]: audit 2023-10-01T10:07:03.707441+0000 mon.a (mon.0) 185 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:07:05.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:04 smithi173 bash[27612]: audit 2023-10-01T10:07:03.709192+0000 mon.a (mon.0) 186 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:07:05.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:04 smithi173 bash[27612]: audit 2023-10-01T10:07:03.711988+0000 mon.a (mon.0) 187 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:07:05.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:04 smithi173 bash[27612]: audit 2023-10-01T10:07:03.713355+0000 mon.a (mon.0) 188 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:07:05.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:04 smithi173 bash[27612]: audit 2023-10-01T10:07:03.719886+0000 mon.a (mon.0) 189 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:07:05.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:04 smithi173 bash[27612]: cluster 2023-10-01T10:07:03.751029+0000 mgr.y (mgr.14146) 48 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:05.270 INFO:teuthology.orchestra.run.smithi173.stdout:Scheduled mgr update... 2023-10-01T10:07:06.193 DEBUG:teuthology.orchestra.run.smithi173:mgr.x> sudo journalctl -f -n 0 -u ceph-d8016480-6041-11ee-8db5-212e2dc638e7@mgr.x.service 2023-10-01T10:07:06.195 INFO:tasks.cephadm:Deploying OSDs... 2023-10-01T10:07:06.196 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2023-10-01T10:07:06.196 DEBUG:teuthology.orchestra.run.smithi078:> dd if=/scratch_devs of=/dev/stdout 2023-10-01T10:07:06.203 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-10-01T10:07:06.204 DEBUG:teuthology.orchestra.run.smithi078:> stat /dev/vg_nvme/lv_1 2023-10-01T10:07:06.252 INFO:teuthology.orchestra.run.smithi078.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-10-01T10:07:06.252 INFO:teuthology.orchestra.run.smithi078.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-01T10:07:06.252 INFO:teuthology.orchestra.run.smithi078.stdout:Device: 5h/5d Inode: 491 Links: 1 2023-10-01T10:07:06.252 INFO:teuthology.orchestra.run.smithi078.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-01T10:07:06.252 INFO:teuthology.orchestra.run.smithi078.stdout:Access: 2023-10-01 10:00:35.981240185 +0000 2023-10-01T10:07:06.253 INFO:teuthology.orchestra.run.smithi078.stdout:Modify: 2023-10-01 10:00:35.665247472 +0000 2023-10-01T10:07:06.253 INFO:teuthology.orchestra.run.smithi078.stdout:Change: 2023-10-01 10:00:35.665247472 +0000 2023-10-01T10:07:06.253 INFO:teuthology.orchestra.run.smithi078.stdout: Birth: - 2023-10-01T10:07:06.253 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-10-01T10:07:06.308 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records in 2023-10-01T10:07:06.308 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records out 2023-10-01T10:07:06.309 INFO:teuthology.orchestra.run.smithi078.stderr:512 bytes copied, 0.00059729 s, 857 kB/s 2023-10-01T10:07:06.309 DEBUG:teuthology.orchestra.run.smithi078:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-10-01T10:07:06.362 DEBUG:teuthology.orchestra.run.smithi078:> stat /dev/vg_nvme/lv_2 2023-10-01T10:07:06.412 INFO:teuthology.orchestra.run.smithi078.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-10-01T10:07:06.412 INFO:teuthology.orchestra.run.smithi078.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-01T10:07:06.412 INFO:teuthology.orchestra.run.smithi078.stdout:Device: 5h/5d Inode: 497 Links: 1 2023-10-01T10:07:06.412 INFO:teuthology.orchestra.run.smithi078.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-01T10:07:06.413 INFO:teuthology.orchestra.run.smithi078.stdout:Access: 2023-10-01 10:00:36.513227916 +0000 2023-10-01T10:07:06.413 INFO:teuthology.orchestra.run.smithi078.stdout:Modify: 2023-10-01 10:00:36.197235204 +0000 2023-10-01T10:07:06.413 INFO:teuthology.orchestra.run.smithi078.stdout:Change: 2023-10-01 10:00:36.197235204 +0000 2023-10-01T10:07:06.413 INFO:teuthology.orchestra.run.smithi078.stdout: Birth: - 2023-10-01T10:07:06.413 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-10-01T10:07:06.465 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records in 2023-10-01T10:07:06.465 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records out 2023-10-01T10:07:06.466 INFO:teuthology.orchestra.run.smithi078.stderr:512 bytes copied, 0.000344079 s, 1.5 MB/s 2023-10-01T10:07:06.467 DEBUG:teuthology.orchestra.run.smithi078:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-10-01T10:07:06.518 DEBUG:teuthology.orchestra.run.smithi078:> stat /dev/vg_nvme/lv_3 2023-10-01T10:07:06.551 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:06 smithi173 bash[27612]: audit 2023-10-01T10:07:05.258740+0000 mgr.y (mgr.14146) 49 : audit [DBG] from='client.24109 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi078=y;smithi173=x", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:07:06.551 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:06 smithi173 bash[27612]: cephadm 2023-10-01T10:07:05.262234+0000 mgr.y (mgr.14146) 50 : cephadm [INF] Saving service mgr spec with placement smithi078=y;smithi173=x;count:2 2023-10-01T10:07:06.551 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:06 smithi173 bash[27612]: audit 2023-10-01T10:07:05.268027+0000 mon.a (mon.0) 190 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:07:06.551 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:06 smithi173 bash[27612]: audit 2023-10-01T10:07:05.290987+0000 mon.a (mon.0) 191 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:07:06.551 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:06 smithi173 bash[27612]: audit 2023-10-01T10:07:05.293965+0000 mon.a (mon.0) 192 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:07:06.551 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:06 smithi173 bash[27612]: audit 2023-10-01T10:07:05.295395+0000 mon.a (mon.0) 193 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:07:06.551 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:06 smithi173 bash[27612]: audit 2023-10-01T10:07:05.302267+0000 mon.a (mon.0) 194 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:07:06.551 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:06 smithi173 bash[27612]: audit 2023-10-01T10:07:05.304466+0000 mon.a (mon.0) 195 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-01T10:07:06.551 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:06 smithi173 bash[27612]: audit 2023-10-01T10:07:05.308838+0000 mon.a (mon.0) 196 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-10-01T10:07:06.551 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:06 smithi173 bash[27612]: audit 2023-10-01T10:07:05.310003+0000 mon.a (mon.0) 197 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-01T10:07:06.552 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:06 smithi173 bash[27612]: audit 2023-10-01T10:07:05.311396+0000 mon.a (mon.0) 198 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:07:06.572 INFO:teuthology.orchestra.run.smithi078.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-10-01T10:07:06.572 INFO:teuthology.orchestra.run.smithi078.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-01T10:07:06.572 INFO:teuthology.orchestra.run.smithi078.stdout:Device: 5h/5d Inode: 504 Links: 1 2023-10-01T10:07:06.572 INFO:teuthology.orchestra.run.smithi078.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-01T10:07:06.572 INFO:teuthology.orchestra.run.smithi078.stdout:Access: 2023-10-01 10:00:37.061215279 +0000 2023-10-01T10:07:06.572 INFO:teuthology.orchestra.run.smithi078.stdout:Modify: 2023-10-01 10:00:36.741222658 +0000 2023-10-01T10:07:06.572 INFO:teuthology.orchestra.run.smithi078.stdout:Change: 2023-10-01 10:00:36.741222658 +0000 2023-10-01T10:07:06.573 INFO:teuthology.orchestra.run.smithi078.stdout: Birth: - 2023-10-01T10:07:06.573 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-10-01T10:07:06.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:06 smithi078 bash[33821]: audit 2023-10-01T10:07:05.258740+0000 mgr.y (mgr.14146) 49 : audit [DBG] from='client.24109 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi078=y;smithi173=x", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:07:06.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:06 smithi078 bash[33821]: cephadm 2023-10-01T10:07:05.262234+0000 mgr.y (mgr.14146) 50 : cephadm [INF] Saving service mgr spec with placement smithi078=y;smithi173=x;count:2 2023-10-01T10:07:06.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:06 smithi078 bash[33821]: audit 2023-10-01T10:07:05.268027+0000 mon.a (mon.0) 190 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:07:06.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:06 smithi078 bash[33821]: audit 2023-10-01T10:07:05.290987+0000 mon.a (mon.0) 191 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:07:06.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:06 smithi078 bash[33821]: audit 2023-10-01T10:07:05.293965+0000 mon.a (mon.0) 192 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:07:06.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:06 smithi078 bash[33821]: audit 2023-10-01T10:07:05.295395+0000 mon.a (mon.0) 193 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:07:06.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:06 smithi078 bash[33821]: audit 2023-10-01T10:07:05.302267+0000 mon.a (mon.0) 194 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:07:06.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:06 smithi078 bash[33821]: audit 2023-10-01T10:07:05.304466+0000 mon.a (mon.0) 195 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-01T10:07:06.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:06 smithi078 bash[33821]: audit 2023-10-01T10:07:05.308838+0000 mon.a (mon.0) 196 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-10-01T10:07:06.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:06 smithi078 bash[33821]: audit 2023-10-01T10:07:05.310003+0000 mon.a (mon.0) 197 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-01T10:07:06.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:06 smithi078 bash[33821]: audit 2023-10-01T10:07:05.311396+0000 mon.a (mon.0) 198 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:07:06.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:06 smithi078 bash[27336]: audit 2023-10-01T10:07:05.258740+0000 mgr.y (mgr.14146) 49 : audit [DBG] from='client.24109 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi078=y;smithi173=x", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:07:06.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:06 smithi078 bash[27336]: cephadm 2023-10-01T10:07:05.262234+0000 mgr.y (mgr.14146) 50 : cephadm [INF] Saving service mgr spec with placement smithi078=y;smithi173=x;count:2 2023-10-01T10:07:06.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:06 smithi078 bash[27336]: audit 2023-10-01T10:07:05.268027+0000 mon.a (mon.0) 190 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:07:06.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:06 smithi078 bash[27336]: audit 2023-10-01T10:07:05.290987+0000 mon.a (mon.0) 191 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:07:06.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:06 smithi078 bash[27336]: audit 2023-10-01T10:07:05.293965+0000 mon.a (mon.0) 192 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:07:06.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:06 smithi078 bash[27336]: audit 2023-10-01T10:07:05.295395+0000 mon.a (mon.0) 193 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:07:06.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:06 smithi078 bash[27336]: audit 2023-10-01T10:07:05.302267+0000 mon.a (mon.0) 194 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:07:06.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:06 smithi078 bash[27336]: audit 2023-10-01T10:07:05.304466+0000 mon.a (mon.0) 195 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-01T10:07:06.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:06 smithi078 bash[27336]: audit 2023-10-01T10:07:05.308838+0000 mon.a (mon.0) 196 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-10-01T10:07:06.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:06 smithi078 bash[27336]: audit 2023-10-01T10:07:05.310003+0000 mon.a (mon.0) 197 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-01T10:07:06.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:06 smithi078 bash[27336]: audit 2023-10-01T10:07:05.311396+0000 mon.a (mon.0) 198 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:07:06.593 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records in 2023-10-01T10:07:06.593 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records out 2023-10-01T10:07:06.593 INFO:teuthology.orchestra.run.smithi078.stderr:512 bytes copied, 0.000403305 s, 1.3 MB/s 2023-10-01T10:07:06.594 DEBUG:teuthology.orchestra.run.smithi078:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-10-01T10:07:06.636 INFO:journalctl@ceph.mgr.x.smithi173.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-01T10:07:06.646 DEBUG:teuthology.orchestra.run.smithi078:> stat /dev/vg_nvme/lv_4 2023-10-01T10:07:06.696 INFO:teuthology.orchestra.run.smithi078.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-10-01T10:07:06.696 INFO:teuthology.orchestra.run.smithi078.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-01T10:07:06.696 INFO:teuthology.orchestra.run.smithi078.stdout:Device: 5h/5d Inode: 510 Links: 1 2023-10-01T10:07:06.696 INFO:teuthology.orchestra.run.smithi078.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-01T10:07:06.696 INFO:teuthology.orchestra.run.smithi078.stdout:Access: 2023-10-01 10:00:37.293209929 +0000 2023-10-01T10:07:06.696 INFO:teuthology.orchestra.run.smithi078.stdout:Modify: 2023-10-01 10:00:37.289210022 +0000 2023-10-01T10:07:06.696 INFO:teuthology.orchestra.run.smithi078.stdout:Change: 2023-10-01 10:00:37.289210022 +0000 2023-10-01T10:07:06.697 INFO:teuthology.orchestra.run.smithi078.stdout: Birth: - 2023-10-01T10:07:06.697 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-10-01T10:07:06.749 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records in 2023-10-01T10:07:06.749 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records out 2023-10-01T10:07:06.749 INFO:teuthology.orchestra.run.smithi078.stderr:512 bytes copied, 0.000439814 s, 1.2 MB/s 2023-10-01T10:07:06.750 DEBUG:teuthology.orchestra.run.smithi078:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-10-01T10:07:06.801 DEBUG:teuthology.orchestra.run.smithi173:> set -ex 2023-10-01T10:07:06.801 DEBUG:teuthology.orchestra.run.smithi173:> dd if=/scratch_devs of=/dev/stdout 2023-10-01T10:07:06.809 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-10-01T10:07:06.809 DEBUG:teuthology.orchestra.run.smithi173:> stat /dev/vg_nvme/lv_1 2023-10-01T10:07:06.862 INFO:teuthology.orchestra.run.smithi173.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-10-01T10:07:06.862 INFO:teuthology.orchestra.run.smithi173.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-01T10:07:06.862 INFO:teuthology.orchestra.run.smithi173.stdout:Device: 5h/5d Inode: 491 Links: 1 2023-10-01T10:07:06.862 INFO:teuthology.orchestra.run.smithi173.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-01T10:07:06.862 INFO:teuthology.orchestra.run.smithi173.stdout:Access: 2023-10-01 10:00:35.437387811 +0000 2023-10-01T10:07:06.862 INFO:teuthology.orchestra.run.smithi173.stdout:Modify: 2023-10-01 10:00:35.117395191 +0000 2023-10-01T10:07:06.863 INFO:teuthology.orchestra.run.smithi173.stdout:Change: 2023-10-01 10:00:35.117395191 +0000 2023-10-01T10:07:06.863 INFO:teuthology.orchestra.run.smithi173.stdout: Birth: - 2023-10-01T10:07:06.863 DEBUG:teuthology.orchestra.run.smithi173:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-10-01T10:07:06.920 INFO:teuthology.orchestra.run.smithi173.stderr:1+0 records in 2023-10-01T10:07:06.920 INFO:teuthology.orchestra.run.smithi173.stderr:1+0 records out 2023-10-01T10:07:06.920 INFO:teuthology.orchestra.run.smithi173.stderr:512 bytes copied, 0.000369839 s, 1.4 MB/s 2023-10-01T10:07:06.921 DEBUG:teuthology.orchestra.run.smithi173:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-10-01T10:07:06.972 DEBUG:teuthology.orchestra.run.smithi173:> stat /dev/vg_nvme/lv_2 2023-10-01T10:07:07.017 INFO:teuthology.orchestra.run.smithi173.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-10-01T10:07:07.018 INFO:teuthology.orchestra.run.smithi173.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-01T10:07:07.018 INFO:teuthology.orchestra.run.smithi173.stdout:Device: 5h/5d Inode: 498 Links: 1 2023-10-01T10:07:07.018 INFO:teuthology.orchestra.run.smithi173.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-01T10:07:07.018 INFO:teuthology.orchestra.run.smithi173.stdout:Access: 2023-10-01 10:00:35.985375174 +0000 2023-10-01T10:07:07.018 INFO:teuthology.orchestra.run.smithi173.stdout:Modify: 2023-10-01 10:00:35.657382738 +0000 2023-10-01T10:07:07.018 INFO:teuthology.orchestra.run.smithi173.stdout:Change: 2023-10-01 10:00:35.657382738 +0000 2023-10-01T10:07:07.018 INFO:teuthology.orchestra.run.smithi173.stdout: Birth: - 2023-10-01T10:07:07.019 DEBUG:teuthology.orchestra.run.smithi173:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-10-01T10:07:07.069 INFO:teuthology.orchestra.run.smithi173.stderr:1+0 records in 2023-10-01T10:07:07.069 INFO:teuthology.orchestra.run.smithi173.stderr:1+0 records out 2023-10-01T10:07:07.069 INFO:teuthology.orchestra.run.smithi173.stderr:512 bytes copied, 0.00023098 s, 2.2 MB/s 2023-10-01T10:07:07.070 DEBUG:teuthology.orchestra.run.smithi173:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-10-01T10:07:07.120 DEBUG:teuthology.orchestra.run.smithi173:> stat /dev/vg_nvme/lv_3 2023-10-01T10:07:07.168 INFO:teuthology.orchestra.run.smithi173.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-10-01T10:07:07.168 INFO:teuthology.orchestra.run.smithi173.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-01T10:07:07.168 INFO:teuthology.orchestra.run.smithi173.stdout:Device: 5h/5d Inode: 502 Links: 1 2023-10-01T10:07:07.168 INFO:teuthology.orchestra.run.smithi173.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-01T10:07:07.168 INFO:teuthology.orchestra.run.smithi173.stdout:Access: 2023-10-01 10:00:36.537362445 +0000 2023-10-01T10:07:07.168 INFO:teuthology.orchestra.run.smithi173.stdout:Modify: 2023-10-01 10:00:36.185370561 +0000 2023-10-01T10:07:07.168 INFO:teuthology.orchestra.run.smithi173.stdout:Change: 2023-10-01 10:00:36.185370561 +0000 2023-10-01T10:07:07.169 INFO:teuthology.orchestra.run.smithi173.stdout: Birth: - 2023-10-01T10:07:07.169 DEBUG:teuthology.orchestra.run.smithi173:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-10-01T10:07:07.222 INFO:teuthology.orchestra.run.smithi173.stderr:1+0 records in 2023-10-01T10:07:07.222 INFO:teuthology.orchestra.run.smithi173.stderr:1+0 records out 2023-10-01T10:07:07.222 INFO:teuthology.orchestra.run.smithi173.stderr:512 bytes copied, 0.000407605 s, 1.3 MB/s 2023-10-01T10:07:07.223 DEBUG:teuthology.orchestra.run.smithi173:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-10-01T10:07:07.273 DEBUG:teuthology.orchestra.run.smithi173:> stat /dev/vg_nvme/lv_4 2023-10-01T10:07:07.325 INFO:teuthology.orchestra.run.smithi173.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-10-01T10:07:07.325 INFO:teuthology.orchestra.run.smithi173.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-10-01T10:07:07.325 INFO:teuthology.orchestra.run.smithi173.stdout:Device: 5h/5d Inode: 508 Links: 1 2023-10-01T10:07:07.325 INFO:teuthology.orchestra.run.smithi173.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-10-01T10:07:07.325 INFO:teuthology.orchestra.run.smithi173.stdout:Access: 2023-10-01 10:00:37.081349898 +0000 2023-10-01T10:07:07.326 INFO:teuthology.orchestra.run.smithi173.stdout:Modify: 2023-10-01 10:00:36.753357463 +0000 2023-10-01T10:07:07.326 INFO:teuthology.orchestra.run.smithi173.stdout:Change: 2023-10-01 10:00:36.753357463 +0000 2023-10-01T10:07:07.326 INFO:teuthology.orchestra.run.smithi173.stdout: Birth: - 2023-10-01T10:07:07.326 DEBUG:teuthology.orchestra.run.smithi173:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-10-01T10:07:07.387 INFO:teuthology.orchestra.run.smithi173.stderr:1+0 records in 2023-10-01T10:07:07.387 INFO:teuthology.orchestra.run.smithi173.stderr:1+0 records out 2023-10-01T10:07:07.387 INFO:teuthology.orchestra.run.smithi173.stderr:512 bytes copied, 0.000352436 s, 1.5 MB/s 2023-10-01T10:07:07.388 DEBUG:teuthology.orchestra.run.smithi173:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-10-01T10:07:07.438 INFO:tasks.cephadm:Deploying osd.0 on smithi078 with /dev/vg_nvme/lv_4... 2023-10-01T10:07:07.438 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-10-01T10:07:07.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:07 smithi078 bash[33821]: cephadm 2023-10-01T10:07:05.312350+0000 mgr.y (mgr.14146) 51 : cephadm [INF] Deploying daemon mgr.x on smithi173 2023-10-01T10:07:07.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:07 smithi078 bash[33821]: cluster 2023-10-01T10:07:05.751453+0000 mgr.y (mgr.14146) 52 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:07.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:07 smithi078 bash[27336]: cephadm 2023-10-01T10:07:05.312350+0000 mgr.y (mgr.14146) 51 : cephadm [INF] Deploying daemon mgr.x on smithi173 2023-10-01T10:07:07.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:07 smithi078 bash[27336]: cluster 2023-10-01T10:07:05.751453+0000 mgr.y (mgr.14146) 52 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:07.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:07 smithi173 bash[27612]: cephadm 2023-10-01T10:07:05.312350+0000 mgr.y (mgr.14146) 51 : cephadm [INF] Deploying daemon mgr.x on smithi173 2023-10-01T10:07:07.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:07 smithi173 bash[27612]: cluster 2023-10-01T10:07:05.751453+0000 mgr.y (mgr.14146) 52 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:07.897 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:07:07 smithi173 systemd[1]: Started Ceph mgr.x for d8016480-6041-11ee-8db5-212e2dc638e7. 2023-10-01T10:07:08.859 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:07:08 smithi173 bash[28473]: debug 2023-10-01T10:07:08.715+0000 7fe4bb1de700 1 -- 172.21.15.173:0/84803101 <== mon.1 v2:172.21.15.173:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55ce27b504e0 con 0x55ce27c44800 2023-10-01T10:07:09.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:08 smithi173 bash[27612]: cluster 2023-10-01T10:07:07.751906+0000 mgr.y (mgr.14146) 53 : cluster [DBG] pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:09.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:08 smithi173 bash[27612]: audit 2023-10-01T10:07:07.858627+0000 mon.a (mon.0) 199 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:07:09.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:08 smithi173 bash[27612]: audit 2023-10-01T10:07:07.861133+0000 mon.a (mon.0) 200 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:07:09.121 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:07:08 smithi173 bash[28473]: debug 2023-10-01T10:07:08.871+0000 7fe4cb7c3680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-01T10:07:09.121 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:07:08 smithi173 bash[28473]: debug 2023-10-01T10:07:08.975+0000 7fe4cb7c3680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-01T10:07:09.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:08 smithi078 bash[27336]: cluster 2023-10-01T10:07:07.751906+0000 mgr.y (mgr.14146) 53 : cluster [DBG] pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:09.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:08 smithi078 bash[27336]: audit 2023-10-01T10:07:07.858627+0000 mon.a (mon.0) 199 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:07:09.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:08 smithi078 bash[27336]: audit 2023-10-01T10:07:07.861133+0000 mon.a (mon.0) 200 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:07:09.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:08 smithi078 bash[33821]: cluster 2023-10-01T10:07:07.751906+0000 mgr.y (mgr.14146) 53 : cluster [DBG] pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:09.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:08 smithi078 bash[33821]: audit 2023-10-01T10:07:07.858627+0000 mon.a (mon.0) 199 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:07:09.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:08 smithi078 bash[33821]: audit 2023-10-01T10:07:07.861133+0000 mon.a (mon.0) 200 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:07:09.870 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:07:09 smithi173 bash[28473]: debug 2023-10-01T10:07:09.447+0000 7fe4cb7c3680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-01T10:07:10.620 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:07:10 smithi173 bash[28473]: debug 2023-10-01T10:07:10.355+0000 7fe4cb7c3680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-01T10:07:10.620 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:07:10 smithi173 bash[28473]: debug 2023-10-01T10:07:10.451+0000 7fe4cb7c3680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-01T10:07:10.763 INFO:teuthology.orchestra.run.smithi078.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-10-01T10:07:10.763 INFO:teuthology.orchestra.run.smithi078.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-10-01T10:07:10.764 INFO:teuthology.orchestra.run.smithi078.stderr: stderr: 10+0 records in 2023-10-01T10:07:10.764 INFO:teuthology.orchestra.run.smithi078.stderr:10+0 records out 2023-10-01T10:07:10.764 INFO:teuthology.orchestra.run.smithi078.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0159326 s, 658 MB/s 2023-10-01T10:07:10.764 INFO:teuthology.orchestra.run.smithi078.stderr:--> Zapping successful for: 2023-10-01T10:07:11.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:10 smithi078 bash[27336]: cluster 2023-10-01T10:07:09.752380+0000 mgr.y (mgr.14146) 54 : cluster [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:11.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:10 smithi078 bash[27336]: audit 2023-10-01T10:07:10.497688+0000 mon.a (mon.0) 201 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:07:11.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:10 smithi078 bash[33821]: cluster 2023-10-01T10:07:09.752380+0000 mgr.y (mgr.14146) 54 : cluster [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:11.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:10 smithi078 bash[33821]: audit 2023-10-01T10:07:10.497688+0000 mon.a (mon.0) 201 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:07:11.120 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:07:10 smithi173 bash[28473]: debug 2023-10-01T10:07:10.707+0000 7fe4cb7c3680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-01T10:07:11.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:10 smithi173 bash[27612]: cluster 2023-10-01T10:07:09.752380+0000 mgr.y (mgr.14146) 54 : cluster [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:11.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:10 smithi173 bash[27612]: audit 2023-10-01T10:07:10.497688+0000 mon.a (mon.0) 201 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:07:11.515 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi078:vg_nvme/lv_4 2023-10-01T10:07:12.098 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:07:11 smithi173 bash[28473]: debug 2023-10-01T10:07:11.835+0000 7fe4cb7c3680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-01T10:07:12.098 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:07:12 smithi173 bash[28473]: debug 2023-10-01T10:07:11.995+0000 7fe4cb7c3680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-01T10:07:12.370 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:07:12 smithi173 bash[28473]: debug 2023-10-01T10:07:12.091+0000 7fe4cb7c3680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-01T10:07:12.371 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:07:12 smithi173 bash[28473]: debug 2023-10-01T10:07:12.203+0000 7fe4cb7c3680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-01T10:07:12.825 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:07:12 smithi173 bash[28473]: debug 2023-10-01T10:07:12.423+0000 7fe4cb7c3680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-01T10:07:13.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:12 smithi173 bash[27612]: cluster 2023-10-01T10:07:11.752958+0000 mgr.y (mgr.14146) 55 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:13.120 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:07:12 smithi173 bash[28473]: debug 2023-10-01T10:07:12.819+0000 7fe4cb7c3680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-01T10:07:13.120 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:07:12 smithi173 bash[28473]: debug 2023-10-01T10:07:12.955+0000 7fe4cb7c3680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-01T10:07:13.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:12 smithi078 bash[27336]: cluster 2023-10-01T10:07:11.752958+0000 mgr.y (mgr.14146) 55 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:13.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:12 smithi078 bash[33821]: cluster 2023-10-01T10:07:11.752958+0000 mgr.y (mgr.14146) 55 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:14.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:13 smithi078 bash[27336]: audit 2023-10-01T10:07:12.887393+0000 mon.a (mon.0) 202 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:07:14.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:13 smithi078 bash[27336]: audit 2023-10-01T10:07:12.888425+0000 mon.a (mon.0) 203 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:07:14.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:13 smithi078 bash[27336]: audit 2023-10-01T10:07:12.889646+0000 mon.a (mon.0) 204 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:07:14.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:13 smithi078 bash[27336]: audit 2023-10-01T10:07:12.894777+0000 mon.a (mon.0) 205 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:07:14.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:13 smithi078 bash[27336]: cephadm 2023-10-01T10:07:12.896804+0000 mgr.y (mgr.14146) 56 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2023-10-01T10:07:14.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:13 smithi078 bash[27336]: audit 2023-10-01T10:07:12.897208+0000 mon.a (mon.0) 206 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-01T10:07:14.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:13 smithi078 bash[27336]: audit 2023-10-01T10:07:12.898284+0000 mon.a (mon.0) 207 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-01T10:07:14.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:13 smithi078 bash[27336]: audit 2023-10-01T10:07:12.899059+0000 mon.a (mon.0) 208 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:07:14.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:13 smithi078 bash[27336]: cephadm 2023-10-01T10:07:12.899803+0000 mgr.y (mgr.14146) 57 : cephadm [INF] Reconfiguring daemon mgr.y on smithi078 2023-10-01T10:07:14.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:13 smithi078 bash[33821]: audit 2023-10-01T10:07:12.887393+0000 mon.a (mon.0) 202 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:07:14.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:13 smithi078 bash[33821]: audit 2023-10-01T10:07:12.888425+0000 mon.a (mon.0) 203 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:07:14.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:13 smithi078 bash[33821]: audit 2023-10-01T10:07:12.889646+0000 mon.a (mon.0) 204 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:07:14.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:13 smithi078 bash[33821]: audit 2023-10-01T10:07:12.894777+0000 mon.a (mon.0) 205 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:07:14.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:13 smithi078 bash[33821]: cephadm 2023-10-01T10:07:12.896804+0000 mgr.y (mgr.14146) 56 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2023-10-01T10:07:14.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:13 smithi078 bash[33821]: audit 2023-10-01T10:07:12.897208+0000 mon.a (mon.0) 206 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-01T10:07:14.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:13 smithi078 bash[33821]: audit 2023-10-01T10:07:12.898284+0000 mon.a (mon.0) 207 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-01T10:07:14.085 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:13 smithi078 bash[33821]: audit 2023-10-01T10:07:12.899059+0000 mon.a (mon.0) 208 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:07:14.085 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:13 smithi078 bash[33821]: cephadm 2023-10-01T10:07:12.899803+0000 mgr.y (mgr.14146) 57 : cephadm [INF] Reconfiguring daemon mgr.y on smithi078 2023-10-01T10:07:14.120 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:07:13 smithi173 bash[28473]: debug 2023-10-01T10:07:13.835+0000 7fe4cb7c3680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-01T10:07:14.120 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:07:13 smithi173 bash[28473]: debug 2023-10-01T10:07:13.935+0000 7fe4cb7c3680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-01T10:07:14.120 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:07:14 smithi173 bash[28473]: debug 2023-10-01T10:07:14.051+0000 7fe4cb7c3680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-01T10:07:14.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:13 smithi173 bash[27612]: audit 2023-10-01T10:07:12.887393+0000 mon.a (mon.0) 202 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:07:14.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:13 smithi173 bash[27612]: audit 2023-10-01T10:07:12.888425+0000 mon.a (mon.0) 203 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:07:14.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:13 smithi173 bash[27612]: audit 2023-10-01T10:07:12.889646+0000 mon.a (mon.0) 204 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:07:14.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:13 smithi173 bash[27612]: audit 2023-10-01T10:07:12.894777+0000 mon.a (mon.0) 205 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:07:14.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:13 smithi173 bash[27612]: cephadm 2023-10-01T10:07:12.896804+0000 mgr.y (mgr.14146) 56 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2023-10-01T10:07:14.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:13 smithi173 bash[27612]: audit 2023-10-01T10:07:12.897208+0000 mon.a (mon.0) 206 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-01T10:07:14.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:13 smithi173 bash[27612]: audit 2023-10-01T10:07:12.898284+0000 mon.a (mon.0) 207 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-01T10:07:14.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:13 smithi173 bash[27612]: audit 2023-10-01T10:07:12.899059+0000 mon.a (mon.0) 208 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:07:14.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:13 smithi173 bash[27612]: cephadm 2023-10-01T10:07:12.899803+0000 mgr.y (mgr.14146) 57 : cephadm [INF] Reconfiguring daemon mgr.y on smithi078 2023-10-01T10:07:14.566 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:07:14 smithi173 bash[28473]: debug 2023-10-01T10:07:14.271+0000 7fe4cb7c3680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-01T10:07:14.870 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:07:14 smithi173 bash[28473]: debug 2023-10-01T10:07:14.563+0000 7fe4cb7c3680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-01T10:07:14.870 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:07:14 smithi173 bash[28473]: debug 2023-10-01T10:07:14.723+0000 7fe4cb7c3680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-01T10:07:15.215 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:14 smithi173 bash[27612]: cluster 2023-10-01T10:07:13.753322+0000 mgr.y (mgr.14146) 58 : cluster [DBG] pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:15.215 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:07:14 smithi173 bash[28473]: debug 2023-10-01T10:07:14.891+0000 7fe4cb7c3680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-01T10:07:15.215 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:07:15 smithi173 bash[28473]: debug 2023-10-01T10:07:15.103+0000 7fe4cb7c3680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-01T10:07:15.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:14 smithi078 bash[27336]: cluster 2023-10-01T10:07:13.753322+0000 mgr.y (mgr.14146) 58 : cluster [DBG] pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:15.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:14 smithi078 bash[33821]: cluster 2023-10-01T10:07:13.753322+0000 mgr.y (mgr.14146) 58 : cluster [DBG] pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:15.620 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:07:15 smithi173 bash[28473]: debug 2023-10-01T10:07:15.211+0000 7fe4cb7c3680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-01T10:07:16.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:15 smithi078 bash[27336]: cluster 2023-10-01T10:07:15.237798+0000 mon.a (mon.0) 209 : cluster [DBG] Standby manager daemon x started 2023-10-01T10:07:16.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:15 smithi078 bash[27336]: audit 2023-10-01T10:07:15.238135+0000 mon.b (mon.1) 3 : audit [DBG] from='mgr.? 172.21.15.173:0/826810942' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-01T10:07:16.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:15 smithi078 bash[27336]: audit 2023-10-01T10:07:15.239026+0000 mon.b (mon.1) 4 : audit [DBG] from='mgr.? 172.21.15.173:0/826810942' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-01T10:07:16.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:15 smithi078 bash[27336]: audit 2023-10-01T10:07:15.241095+0000 mon.b (mon.1) 5 : audit [DBG] from='mgr.? 172.21.15.173:0/826810942' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-01T10:07:16.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:15 smithi078 bash[27336]: audit 2023-10-01T10:07:15.241874+0000 mon.b (mon.1) 6 : audit [DBG] from='mgr.? 172.21.15.173:0/826810942' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-01T10:07:16.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:15 smithi078 bash[27336]: audit 2023-10-01T10:07:15.352127+0000 mon.a (mon.0) 210 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T10:07:16.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:15 smithi078 bash[27336]: audit 2023-10-01T10:07:15.354840+0000 mon.a (mon.0) 211 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T10:07:16.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:15 smithi078 bash[27336]: audit 2023-10-01T10:07:15.355957+0000 mon.a (mon.0) 212 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:07:16.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:15 smithi078 bash[27336]: audit 2023-10-01T10:07:15.596859+0000 mon.a (mon.0) 213 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:07:16.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:15 smithi078 bash[27336]: audit 2023-10-01T10:07:15.598866+0000 mon.a (mon.0) 214 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:07:16.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:15 smithi078 bash[27336]: audit 2023-10-01T10:07:15.846152+0000 mon.a (mon.0) 215 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:07:16.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:15 smithi078 bash[27336]: audit 2023-10-01T10:07:15.847291+0000 mon.a (mon.0) 216 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:07:16.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:15 smithi078 bash[27336]: audit 2023-10-01T10:07:15.854854+0000 mon.a (mon.0) 217 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:07:16.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:15 smithi078 bash[33821]: cluster 2023-10-01T10:07:15.237798+0000 mon.a (mon.0) 209 : cluster [DBG] Standby manager daemon x started 2023-10-01T10:07:16.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:15 smithi078 bash[33821]: audit 2023-10-01T10:07:15.238135+0000 mon.b (mon.1) 3 : audit [DBG] from='mgr.? 172.21.15.173:0/826810942' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-01T10:07:16.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:15 smithi078 bash[33821]: audit 2023-10-01T10:07:15.239026+0000 mon.b (mon.1) 4 : audit [DBG] from='mgr.? 172.21.15.173:0/826810942' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-01T10:07:16.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:15 smithi078 bash[33821]: audit 2023-10-01T10:07:15.241095+0000 mon.b (mon.1) 5 : audit [DBG] from='mgr.? 172.21.15.173:0/826810942' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-01T10:07:16.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:15 smithi078 bash[33821]: audit 2023-10-01T10:07:15.241874+0000 mon.b (mon.1) 6 : audit [DBG] from='mgr.? 172.21.15.173:0/826810942' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-01T10:07:16.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:15 smithi078 bash[33821]: audit 2023-10-01T10:07:15.352127+0000 mon.a (mon.0) 210 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T10:07:16.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:15 smithi078 bash[33821]: audit 2023-10-01T10:07:15.354840+0000 mon.a (mon.0) 211 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T10:07:16.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:15 smithi078 bash[33821]: audit 2023-10-01T10:07:15.355957+0000 mon.a (mon.0) 212 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:07:16.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:15 smithi078 bash[33821]: audit 2023-10-01T10:07:15.596859+0000 mon.a (mon.0) 213 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:07:16.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:15 smithi078 bash[33821]: audit 2023-10-01T10:07:15.598866+0000 mon.a (mon.0) 214 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:07:16.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:15 smithi078 bash[33821]: audit 2023-10-01T10:07:15.846152+0000 mon.a (mon.0) 215 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:07:16.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:15 smithi078 bash[33821]: audit 2023-10-01T10:07:15.847291+0000 mon.a (mon.0) 216 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:07:16.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:15 smithi078 bash[33821]: audit 2023-10-01T10:07:15.854854+0000 mon.a (mon.0) 217 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:07:16.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:15 smithi173 bash[27612]: cluster 2023-10-01T10:07:15.237798+0000 mon.a (mon.0) 209 : cluster [DBG] Standby manager daemon x started 2023-10-01T10:07:16.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:15 smithi173 bash[27612]: audit 2023-10-01T10:07:15.238135+0000 mon.b (mon.1) 3 : audit [DBG] from='mgr.? 172.21.15.173:0/826810942' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-01T10:07:16.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:15 smithi173 bash[27612]: audit 2023-10-01T10:07:15.239026+0000 mon.b (mon.1) 4 : audit [DBG] from='mgr.? 172.21.15.173:0/826810942' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-01T10:07:16.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:15 smithi173 bash[27612]: audit 2023-10-01T10:07:15.241095+0000 mon.b (mon.1) 5 : audit [DBG] from='mgr.? 172.21.15.173:0/826810942' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-01T10:07:16.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:15 smithi173 bash[27612]: audit 2023-10-01T10:07:15.241874+0000 mon.b (mon.1) 6 : audit [DBG] from='mgr.? 172.21.15.173:0/826810942' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-01T10:07:16.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:15 smithi173 bash[27612]: audit 2023-10-01T10:07:15.352127+0000 mon.a (mon.0) 210 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T10:07:16.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:15 smithi173 bash[27612]: audit 2023-10-01T10:07:15.354840+0000 mon.a (mon.0) 211 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T10:07:16.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:15 smithi173 bash[27612]: audit 2023-10-01T10:07:15.355957+0000 mon.a (mon.0) 212 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:07:16.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:15 smithi173 bash[27612]: audit 2023-10-01T10:07:15.596859+0000 mon.a (mon.0) 213 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:07:16.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:15 smithi173 bash[27612]: audit 2023-10-01T10:07:15.598866+0000 mon.a (mon.0) 214 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:07:16.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:15 smithi173 bash[27612]: audit 2023-10-01T10:07:15.846152+0000 mon.a (mon.0) 215 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:07:16.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:15 smithi173 bash[27612]: audit 2023-10-01T10:07:15.847291+0000 mon.a (mon.0) 216 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:07:16.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:15 smithi173 bash[27612]: audit 2023-10-01T10:07:15.854854+0000 mon.a (mon.0) 217 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:07:17.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:16 smithi078 bash[27336]: audit 2023-10-01T10:07:15.349240+0000 mgr.y (mgr.14146) 59 : audit [DBG] from='client.24119 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:07:17.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:16 smithi078 bash[27336]: cluster 2023-10-01T10:07:15.753673+0000 mgr.y (mgr.14146) 60 : cluster [DBG] pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:17.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:16 smithi078 bash[27336]: cluster 2023-10-01T10:07:15.910547+0000 mon.a (mon.0) 218 : cluster [DBG] mgrmap e14: y(active, since 78s), standbys: x 2023-10-01T10:07:17.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:16 smithi078 bash[27336]: audit 2023-10-01T10:07:15.910691+0000 mon.a (mon.0) 219 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-01T10:07:17.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:16 smithi078 bash[33821]: audit 2023-10-01T10:07:15.349240+0000 mgr.y (mgr.14146) 59 : audit [DBG] from='client.24119 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:07:17.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:16 smithi078 bash[33821]: cluster 2023-10-01T10:07:15.753673+0000 mgr.y (mgr.14146) 60 : cluster [DBG] pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:17.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:16 smithi078 bash[33821]: cluster 2023-10-01T10:07:15.910547+0000 mon.a (mon.0) 218 : cluster [DBG] mgrmap e14: y(active, since 78s), standbys: x 2023-10-01T10:07:17.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:16 smithi078 bash[33821]: audit 2023-10-01T10:07:15.910691+0000 mon.a (mon.0) 219 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-01T10:07:17.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:16 smithi173 bash[27612]: audit 2023-10-01T10:07:15.349240+0000 mgr.y (mgr.14146) 59 : audit [DBG] from='client.24119 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:07:17.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:16 smithi173 bash[27612]: cluster 2023-10-01T10:07:15.753673+0000 mgr.y (mgr.14146) 60 : cluster [DBG] pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:17.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:16 smithi173 bash[27612]: cluster 2023-10-01T10:07:15.910547+0000 mon.a (mon.0) 218 : cluster [DBG] mgrmap e14: y(active, since 78s), standbys: x 2023-10-01T10:07:17.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:16 smithi173 bash[27612]: audit 2023-10-01T10:07:15.910691+0000 mon.a (mon.0) 219 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-01T10:07:19.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:18 smithi078 bash[27336]: cluster 2023-10-01T10:07:17.754097+0000 mgr.y (mgr.14146) 61 : cluster [DBG] pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:19.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:18 smithi078 bash[33821]: cluster 2023-10-01T10:07:17.754097+0000 mgr.y (mgr.14146) 61 : cluster [DBG] pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:19.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:18 smithi173 bash[27612]: cluster 2023-10-01T10:07:17.754097+0000 mgr.y (mgr.14146) 61 : cluster [DBG] pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:21.176 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:20 smithi078 bash[27336]: cluster 2023-10-01T10:07:19.754668+0000 mgr.y (mgr.14146) 62 : cluster [DBG] pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:21.176 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:20 smithi078 bash[33821]: cluster 2023-10-01T10:07:19.754668+0000 mgr.y (mgr.14146) 62 : cluster [DBG] pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:21.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:20 smithi173 bash[27612]: cluster 2023-10-01T10:07:19.754668+0000 mgr.y (mgr.14146) 62 : cluster [DBG] pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:23.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:22 smithi078 bash[27336]: cluster 2023-10-01T10:07:21.755140+0000 mgr.y (mgr.14146) 63 : cluster [DBG] pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:23.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:22 smithi078 bash[33821]: cluster 2023-10-01T10:07:21.755140+0000 mgr.y (mgr.14146) 63 : cluster [DBG] pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:23.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:22 smithi173 bash[27612]: cluster 2023-10-01T10:07:21.755140+0000 mgr.y (mgr.14146) 63 : cluster [DBG] pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:24.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:23 smithi078 bash[27336]: audit 2023-10-01T10:07:23.464632+0000 mon.a (mon.0) 220 : audit [INF] from='client.? 172.21.15.78:0/1604581601' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3cdf50f0-cae3-4e38-a7ca-c24c4a07b615"}]: dispatch 2023-10-01T10:07:24.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:23 smithi078 bash[27336]: audit 2023-10-01T10:07:23.471691+0000 mon.a (mon.0) 221 : audit [INF] from='client.? 172.21.15.78:0/1604581601' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3cdf50f0-cae3-4e38-a7ca-c24c4a07b615"}]': finished 2023-10-01T10:07:24.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:23 smithi078 bash[27336]: cluster 2023-10-01T10:07:23.471737+0000 mon.a (mon.0) 222 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2023-10-01T10:07:24.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:23 smithi078 bash[27336]: audit 2023-10-01T10:07:23.471907+0000 mon.a (mon.0) 223 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T10:07:24.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:23 smithi078 bash[33821]: audit 2023-10-01T10:07:23.464632+0000 mon.a (mon.0) 220 : audit [INF] from='client.? 172.21.15.78:0/1604581601' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3cdf50f0-cae3-4e38-a7ca-c24c4a07b615"}]: dispatch 2023-10-01T10:07:24.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:23 smithi078 bash[33821]: audit 2023-10-01T10:07:23.471691+0000 mon.a (mon.0) 221 : audit [INF] from='client.? 172.21.15.78:0/1604581601' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3cdf50f0-cae3-4e38-a7ca-c24c4a07b615"}]': finished 2023-10-01T10:07:24.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:23 smithi078 bash[33821]: cluster 2023-10-01T10:07:23.471737+0000 mon.a (mon.0) 222 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2023-10-01T10:07:24.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:23 smithi078 bash[33821]: audit 2023-10-01T10:07:23.471907+0000 mon.a (mon.0) 223 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T10:07:24.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:23 smithi173 bash[27612]: audit 2023-10-01T10:07:23.464632+0000 mon.a (mon.0) 220 : audit [INF] from='client.? 172.21.15.78:0/1604581601' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3cdf50f0-cae3-4e38-a7ca-c24c4a07b615"}]: dispatch 2023-10-01T10:07:24.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:23 smithi173 bash[27612]: audit 2023-10-01T10:07:23.471691+0000 mon.a (mon.0) 221 : audit [INF] from='client.? 172.21.15.78:0/1604581601' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3cdf50f0-cae3-4e38-a7ca-c24c4a07b615"}]': finished 2023-10-01T10:07:24.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:23 smithi173 bash[27612]: cluster 2023-10-01T10:07:23.471737+0000 mon.a (mon.0) 222 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2023-10-01T10:07:24.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:23 smithi173 bash[27612]: audit 2023-10-01T10:07:23.471907+0000 mon.a (mon.0) 223 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T10:07:25.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:24 smithi078 bash[33821]: cluster 2023-10-01T10:07:23.755581+0000 mgr.y (mgr.14146) 64 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:25.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:24 smithi078 bash[33821]: audit 2023-10-01T10:07:24.182425+0000 mon.a (mon.0) 224 : audit [DBG] from='client.? 172.21.15.78:0/1233222822' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T10:07:25.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:24 smithi078 bash[27336]: cluster 2023-10-01T10:07:23.755581+0000 mgr.y (mgr.14146) 64 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:25.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:24 smithi078 bash[27336]: audit 2023-10-01T10:07:24.182425+0000 mon.a (mon.0) 224 : audit [DBG] from='client.? 172.21.15.78:0/1233222822' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T10:07:25.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:24 smithi173 bash[27612]: cluster 2023-10-01T10:07:23.755581+0000 mgr.y (mgr.14146) 64 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:25.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:24 smithi173 bash[27612]: audit 2023-10-01T10:07:24.182425+0000 mon.a (mon.0) 224 : audit [DBG] from='client.? 172.21.15.78:0/1233222822' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T10:07:27.080 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:26 smithi078 bash[27336]: cluster 2023-10-01T10:07:25.756031+0000 mgr.y (mgr.14146) 65 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:27.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:26 smithi078 bash[33821]: cluster 2023-10-01T10:07:25.756031+0000 mgr.y (mgr.14146) 65 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:27.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:26 smithi173 bash[27612]: cluster 2023-10-01T10:07:25.756031+0000 mgr.y (mgr.14146) 65 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:29.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:28 smithi078 bash[33821]: cluster 2023-10-01T10:07:27.756486+0000 mgr.y (mgr.14146) 66 : cluster [DBG] pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:29.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:28 smithi078 bash[27336]: cluster 2023-10-01T10:07:27.756486+0000 mgr.y (mgr.14146) 66 : cluster [DBG] pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:29.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:28 smithi173 bash[27612]: cluster 2023-10-01T10:07:27.756486+0000 mgr.y (mgr.14146) 66 : cluster [DBG] pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:31.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:30 smithi078 bash[27336]: cluster 2023-10-01T10:07:29.756942+0000 mgr.y (mgr.14146) 67 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:31.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:30 smithi078 bash[33821]: cluster 2023-10-01T10:07:29.756942+0000 mgr.y (mgr.14146) 67 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:31.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:30 smithi173 bash[27612]: cluster 2023-10-01T10:07:29.756942+0000 mgr.y (mgr.14146) 67 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:33.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:32 smithi078 bash[27336]: cluster 2023-10-01T10:07:31.757450+0000 mgr.y (mgr.14146) 68 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:33.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:32 smithi078 bash[33821]: cluster 2023-10-01T10:07:31.757450+0000 mgr.y (mgr.14146) 68 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:33.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:32 smithi173 bash[27612]: cluster 2023-10-01T10:07:31.757450+0000 mgr.y (mgr.14146) 68 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:35.277 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:34 smithi078 bash[27336]: cluster 2023-10-01T10:07:33.757744+0000 mgr.y (mgr.14146) 69 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:35.278 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:34 smithi078 bash[33821]: cluster 2023-10-01T10:07:33.757744+0000 mgr.y (mgr.14146) 69 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:35.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:34 smithi173 bash[27612]: cluster 2023-10-01T10:07:33.757744+0000 mgr.y (mgr.14146) 69 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:36.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:35 smithi078 bash[33821]: audit 2023-10-01T10:07:35.800707+0000 mon.a (mon.0) 225 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-01T10:07:36.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:35 smithi078 bash[33821]: audit 2023-10-01T10:07:35.802079+0000 mon.a (mon.0) 226 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:07:36.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:35 smithi078 bash[27336]: audit 2023-10-01T10:07:35.800707+0000 mon.a (mon.0) 225 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-01T10:07:36.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:35 smithi078 bash[27336]: audit 2023-10-01T10:07:35.802079+0000 mon.a (mon.0) 226 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:07:36.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:35 smithi173 bash[27612]: audit 2023-10-01T10:07:35.800707+0000 mon.a (mon.0) 225 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-01T10:07:36.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:35 smithi173 bash[27612]: audit 2023-10-01T10:07:35.802079+0000 mon.a (mon.0) 226 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:07:37.140 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:36 smithi078 bash[33821]: cluster 2023-10-01T10:07:35.758129+0000 mgr.y (mgr.14146) 70 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:37.140 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:36 smithi078 bash[33821]: cephadm 2023-10-01T10:07:35.803216+0000 mgr.y (mgr.14146) 71 : cephadm [INF] Deploying daemon osd.0 on smithi078 2023-10-01T10:07:37.140 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:36 smithi078 bash[27336]: cluster 2023-10-01T10:07:35.758129+0000 mgr.y (mgr.14146) 70 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:37.140 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:36 smithi078 bash[27336]: cephadm 2023-10-01T10:07:35.803216+0000 mgr.y (mgr.14146) 71 : cephadm [INF] Deploying daemon osd.0 on smithi078 2023-10-01T10:07:37.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:36 smithi173 bash[27612]: cluster 2023-10-01T10:07:35.758129+0000 mgr.y (mgr.14146) 70 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:37.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:36 smithi173 bash[27612]: cephadm 2023-10-01T10:07:35.803216+0000 mgr.y (mgr.14146) 71 : cephadm [INF] Deploying daemon osd.0 on smithi078 2023-10-01T10:07:39.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:38 smithi078 bash[27336]: cluster 2023-10-01T10:07:37.758614+0000 mgr.y (mgr.14146) 72 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:39.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:38 smithi078 bash[33821]: cluster 2023-10-01T10:07:37.758614+0000 mgr.y (mgr.14146) 72 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:39.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:38 smithi173 bash[27612]: cluster 2023-10-01T10:07:37.758614+0000 mgr.y (mgr.14146) 72 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:41.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:41 smithi078 bash[27336]: cluster 2023-10-01T10:07:39.759078+0000 mgr.y (mgr.14146) 73 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:41.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:41 smithi078 bash[27336]: audit 2023-10-01T10:07:40.217005+0000 mon.a (mon.0) 227 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:07:41.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:41 smithi078 bash[27336]: audit 2023-10-01T10:07:40.234463+0000 mon.a (mon.0) 228 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:07:41.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:41 smithi078 bash[33821]: cluster 2023-10-01T10:07:39.759078+0000 mgr.y (mgr.14146) 73 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:41.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:41 smithi078 bash[33821]: audit 2023-10-01T10:07:40.217005+0000 mon.a (mon.0) 227 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:07:41.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:41 smithi078 bash[33821]: audit 2023-10-01T10:07:40.234463+0000 mon.a (mon.0) 228 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:07:41.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:41 smithi173 bash[27612]: cluster 2023-10-01T10:07:39.759078+0000 mgr.y (mgr.14146) 73 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:41.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:41 smithi173 bash[27612]: audit 2023-10-01T10:07:40.217005+0000 mon.a (mon.0) 227 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:07:41.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:41 smithi173 bash[27612]: audit 2023-10-01T10:07:40.234463+0000 mon.a (mon.0) 228 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:07:43.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:43 smithi078 bash[27336]: cluster 2023-10-01T10:07:41.759520+0000 mgr.y (mgr.14146) 74 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:43.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:43 smithi078 bash[33821]: cluster 2023-10-01T10:07:41.759520+0000 mgr.y (mgr.14146) 74 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:43.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:43 smithi173 bash[27612]: cluster 2023-10-01T10:07:41.759520+0000 mgr.y (mgr.14146) 74 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:45.227 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:45 smithi078 bash[27336]: cluster 2023-10-01T10:07:43.759928+0000 mgr.y (mgr.14146) 75 : cluster [DBG] pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:45.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:45 smithi078 bash[33821]: cluster 2023-10-01T10:07:43.759928+0000 mgr.y (mgr.14146) 75 : cluster [DBG] pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:45.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:45 smithi173 bash[27612]: cluster 2023-10-01T10:07:43.759928+0000 mgr.y (mgr.14146) 75 : cluster [DBG] pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:46.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:46 smithi173 bash[27612]: audit 2023-10-01T10:07:45.339521+0000 mon.a (mon.0) 229 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:07:46.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:46 smithi173 bash[27612]: audit 2023-10-01T10:07:45.340542+0000 mon.a (mon.0) 230 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:07:46.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:46 smithi173 bash[27612]: audit 2023-10-01T10:07:45.341752+0000 mon.a (mon.0) 231 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:07:46.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:46 smithi173 bash[27612]: audit 2023-10-01T10:07:45.348732+0000 mon.a (mon.0) 232 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:07:46.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:46 smithi173 bash[27612]: cluster 2023-10-01T10:07:45.760223+0000 mgr.y (mgr.14146) 76 : cluster [DBG] pgmap v49: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:46.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:46 smithi078 bash[27336]: audit 2023-10-01T10:07:45.339521+0000 mon.a (mon.0) 229 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:07:46.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:46 smithi078 bash[27336]: audit 2023-10-01T10:07:45.340542+0000 mon.a (mon.0) 230 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:07:46.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:46 smithi078 bash[27336]: audit 2023-10-01T10:07:45.341752+0000 mon.a (mon.0) 231 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:07:46.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:46 smithi078 bash[27336]: audit 2023-10-01T10:07:45.348732+0000 mon.a (mon.0) 232 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:07:46.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:46 smithi078 bash[27336]: cluster 2023-10-01T10:07:45.760223+0000 mgr.y (mgr.14146) 76 : cluster [DBG] pgmap v49: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:46.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:46 smithi078 bash[33821]: audit 2023-10-01T10:07:45.339521+0000 mon.a (mon.0) 229 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:07:46.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:46 smithi078 bash[33821]: audit 2023-10-01T10:07:45.340542+0000 mon.a (mon.0) 230 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:07:46.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:46 smithi078 bash[33821]: audit 2023-10-01T10:07:45.341752+0000 mon.a (mon.0) 231 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:07:46.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:46 smithi078 bash[33821]: audit 2023-10-01T10:07:45.348732+0000 mon.a (mon.0) 232 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:07:46.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:46 smithi078 bash[33821]: cluster 2023-10-01T10:07:45.760223+0000 mgr.y (mgr.14146) 76 : cluster [DBG] pgmap v49: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:47.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:47 smithi173 bash[27612]: audit 2023-10-01T10:07:46.940836+0000 mon.c (mon.2) 3 : audit [INF] from='osd.0 [v2:172.21.15.78:6802/575557942,v1:172.21.15.78:6803/575557942]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-01T10:07:47.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:47 smithi173 bash[27612]: audit 2023-10-01T10:07:46.941330+0000 mon.a (mon.0) 233 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-01T10:07:47.728 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:47 smithi078 bash[27336]: audit 2023-10-01T10:07:46.940836+0000 mon.c (mon.2) 3 : audit [INF] from='osd.0 [v2:172.21.15.78:6802/575557942,v1:172.21.15.78:6803/575557942]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-01T10:07:47.728 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:47 smithi078 bash[27336]: audit 2023-10-01T10:07:46.941330+0000 mon.a (mon.0) 233 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-01T10:07:47.728 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:47 smithi078 bash[33821]: audit 2023-10-01T10:07:46.940836+0000 mon.c (mon.2) 3 : audit [INF] from='osd.0 [v2:172.21.15.78:6802/575557942,v1:172.21.15.78:6803/575557942]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-01T10:07:47.728 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:47 smithi078 bash[33821]: audit 2023-10-01T10:07:46.941330+0000 mon.a (mon.0) 233 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-01T10:07:48.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:48 smithi173 bash[27612]: audit 2023-10-01T10:07:47.358628+0000 mon.a (mon.0) 234 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-01T10:07:48.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:48 smithi173 bash[27612]: cluster 2023-10-01T10:07:47.358702+0000 mon.a (mon.0) 235 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2023-10-01T10:07:48.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:48 smithi173 bash[27612]: audit 2023-10-01T10:07:47.358845+0000 mon.a (mon.0) 236 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T10:07:48.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:48 smithi173 bash[27612]: audit 2023-10-01T10:07:47.359686+0000 mon.c (mon.2) 4 : audit [INF] from='osd.0 [v2:172.21.15.78:6802/575557942,v1:172.21.15.78:6803/575557942]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2023-10-01T10:07:48.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:48 smithi173 bash[27612]: audit 2023-10-01T10:07:47.360091+0000 mon.a (mon.0) 237 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2023-10-01T10:07:48.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:48 smithi173 bash[27612]: cluster 2023-10-01T10:07:47.760572+0000 mgr.y (mgr.14146) 77 : cluster [DBG] pgmap v51: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:48.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:48 smithi078 bash[27336]: audit 2023-10-01T10:07:47.358628+0000 mon.a (mon.0) 234 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-01T10:07:48.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:48 smithi078 bash[27336]: cluster 2023-10-01T10:07:47.358702+0000 mon.a (mon.0) 235 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2023-10-01T10:07:48.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:48 smithi078 bash[27336]: audit 2023-10-01T10:07:47.358845+0000 mon.a (mon.0) 236 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T10:07:48.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:48 smithi078 bash[27336]: audit 2023-10-01T10:07:47.359686+0000 mon.c (mon.2) 4 : audit [INF] from='osd.0 [v2:172.21.15.78:6802/575557942,v1:172.21.15.78:6803/575557942]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2023-10-01T10:07:48.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:48 smithi078 bash[27336]: audit 2023-10-01T10:07:47.360091+0000 mon.a (mon.0) 237 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2023-10-01T10:07:48.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:48 smithi078 bash[27336]: cluster 2023-10-01T10:07:47.760572+0000 mgr.y (mgr.14146) 77 : cluster [DBG] pgmap v51: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:48.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:48 smithi078 bash[33821]: audit 2023-10-01T10:07:47.358628+0000 mon.a (mon.0) 234 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-01T10:07:48.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:48 smithi078 bash[33821]: cluster 2023-10-01T10:07:47.358702+0000 mon.a (mon.0) 235 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2023-10-01T10:07:48.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:48 smithi078 bash[33821]: audit 2023-10-01T10:07:47.358845+0000 mon.a (mon.0) 236 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T10:07:48.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:48 smithi078 bash[33821]: audit 2023-10-01T10:07:47.359686+0000 mon.c (mon.2) 4 : audit [INF] from='osd.0 [v2:172.21.15.78:6802/575557942,v1:172.21.15.78:6803/575557942]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2023-10-01T10:07:48.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:48 smithi078 bash[33821]: audit 2023-10-01T10:07:47.360091+0000 mon.a (mon.0) 237 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2023-10-01T10:07:48.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:48 smithi078 bash[33821]: cluster 2023-10-01T10:07:47.760572+0000 mgr.y (mgr.14146) 77 : cluster [DBG] pgmap v51: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:49.096 INFO:teuthology.orchestra.run.smithi078.stdout:Created osd(s) 0 on host 'smithi078' 2023-10-01T10:07:49.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:49 smithi078 bash[27336]: audit 2023-10-01T10:07:48.362680+0000 mon.a (mon.0) 238 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2023-10-01T10:07:49.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:49 smithi078 bash[27336]: cluster 2023-10-01T10:07:48.362782+0000 mon.a (mon.0) 239 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2023-10-01T10:07:49.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:49 smithi078 bash[27336]: audit 2023-10-01T10:07:48.363482+0000 mon.a (mon.0) 240 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T10:07:49.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:49 smithi078 bash[27336]: audit 2023-10-01T10:07:48.370516+0000 mon.a (mon.0) 241 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T10:07:49.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:49 smithi078 bash[27336]: audit 2023-10-01T10:07:49.094651+0000 mon.a (mon.0) 242 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:07:49.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:49 smithi078 bash[27336]: audit 2023-10-01T10:07:49.124971+0000 mon.a (mon.0) 243 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:07:49.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:49 smithi078 bash[33821]: audit 2023-10-01T10:07:48.362680+0000 mon.a (mon.0) 238 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2023-10-01T10:07:49.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:49 smithi078 bash[33821]: cluster 2023-10-01T10:07:48.362782+0000 mon.a (mon.0) 239 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2023-10-01T10:07:49.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:49 smithi078 bash[33821]: audit 2023-10-01T10:07:48.363482+0000 mon.a (mon.0) 240 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T10:07:49.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:49 smithi078 bash[33821]: audit 2023-10-01T10:07:48.370516+0000 mon.a (mon.0) 241 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T10:07:49.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:49 smithi078 bash[33821]: audit 2023-10-01T10:07:49.094651+0000 mon.a (mon.0) 242 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:07:49.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:49 smithi078 bash[33821]: audit 2023-10-01T10:07:49.124971+0000 mon.a (mon.0) 243 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:07:49.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:49 smithi173 bash[27612]: audit 2023-10-01T10:07:48.362680+0000 mon.a (mon.0) 238 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2023-10-01T10:07:49.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:49 smithi173 bash[27612]: cluster 2023-10-01T10:07:48.362782+0000 mon.a (mon.0) 239 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2023-10-01T10:07:49.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:49 smithi173 bash[27612]: audit 2023-10-01T10:07:48.363482+0000 mon.a (mon.0) 240 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T10:07:49.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:49 smithi173 bash[27612]: audit 2023-10-01T10:07:48.370516+0000 mon.a (mon.0) 241 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T10:07:49.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:49 smithi173 bash[27612]: audit 2023-10-01T10:07:49.094651+0000 mon.a (mon.0) 242 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:07:49.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:49 smithi173 bash[27612]: audit 2023-10-01T10:07:49.124971+0000 mon.a (mon.0) 243 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:07:49.760 DEBUG:teuthology.orchestra.run.smithi078:osd.0> sudo journalctl -f -n 0 -u ceph-d8016480-6041-11ee-8db5-212e2dc638e7@osd.0.service 2023-10-01T10:07:49.763 INFO:tasks.cephadm:Deploying osd.1 on smithi078 with /dev/vg_nvme/lv_3... 2023-10-01T10:07:49.763 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-10-01T10:07:50.027 INFO:journalctl@ceph.osd.0.smithi078.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-01T10:07:50.816 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:50 smithi078 bash[27336]: cluster 2023-10-01T10:07:47.950306+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-01T10:07:50.816 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:50 smithi078 bash[27336]: cluster 2023-10-01T10:07:47.950409+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-01T10:07:50.816 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:50 smithi078 bash[27336]: audit 2023-10-01T10:07:49.370613+0000 mon.a (mon.0) 244 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T10:07:50.816 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:50 smithi078 bash[27336]: cluster 2023-10-01T10:07:49.379621+0000 mon.a (mon.0) 245 : cluster [INF] osd.0 [v2:172.21.15.78:6802/575557942,v1:172.21.15.78:6803/575557942] boot 2023-10-01T10:07:50.816 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:50 smithi078 bash[27336]: cluster 2023-10-01T10:07:49.379679+0000 mon.a (mon.0) 246 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2023-10-01T10:07:50.817 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:50 smithi078 bash[27336]: audit 2023-10-01T10:07:49.379830+0000 mon.a (mon.0) 247 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T10:07:50.817 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:50 smithi078 bash[27336]: cluster 2023-10-01T10:07:49.761006+0000 mgr.y (mgr.14146) 78 : cluster [DBG] pgmap v54: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:50.817 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:50 smithi078 bash[33821]: cluster 2023-10-01T10:07:47.950306+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-01T10:07:50.817 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:50 smithi078 bash[33821]: cluster 2023-10-01T10:07:47.950409+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-01T10:07:50.817 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:50 smithi078 bash[33821]: audit 2023-10-01T10:07:49.370613+0000 mon.a (mon.0) 244 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T10:07:50.817 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:50 smithi078 bash[33821]: cluster 2023-10-01T10:07:49.379621+0000 mon.a (mon.0) 245 : cluster [INF] osd.0 [v2:172.21.15.78:6802/575557942,v1:172.21.15.78:6803/575557942] boot 2023-10-01T10:07:50.817 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:50 smithi078 bash[33821]: cluster 2023-10-01T10:07:49.379679+0000 mon.a (mon.0) 246 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2023-10-01T10:07:50.817 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:50 smithi078 bash[33821]: audit 2023-10-01T10:07:49.379830+0000 mon.a (mon.0) 247 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T10:07:50.818 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:50 smithi078 bash[33821]: cluster 2023-10-01T10:07:49.761006+0000 mgr.y (mgr.14146) 78 : cluster [DBG] pgmap v54: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:50.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:50 smithi173 bash[27612]: cluster 2023-10-01T10:07:47.950306+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-01T10:07:50.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:50 smithi173 bash[27612]: cluster 2023-10-01T10:07:47.950409+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-01T10:07:50.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:50 smithi173 bash[27612]: audit 2023-10-01T10:07:49.370613+0000 mon.a (mon.0) 244 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T10:07:50.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:50 smithi173 bash[27612]: cluster 2023-10-01T10:07:49.379621+0000 mon.a (mon.0) 245 : cluster [INF] osd.0 [v2:172.21.15.78:6802/575557942,v1:172.21.15.78:6803/575557942] boot 2023-10-01T10:07:50.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:50 smithi173 bash[27612]: cluster 2023-10-01T10:07:49.379679+0000 mon.a (mon.0) 246 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2023-10-01T10:07:50.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:50 smithi173 bash[27612]: audit 2023-10-01T10:07:49.379830+0000 mon.a (mon.0) 247 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T10:07:50.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:50 smithi173 bash[27612]: cluster 2023-10-01T10:07:49.761006+0000 mgr.y (mgr.14146) 78 : cluster [DBG] pgmap v54: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-10-01T10:07:51.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:51 smithi078 bash[27336]: cluster 2023-10-01T10:07:50.384518+0000 mon.a (mon.0) 248 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2023-10-01T10:07:51.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:51 smithi078 bash[33821]: cluster 2023-10-01T10:07:50.384518+0000 mon.a (mon.0) 248 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2023-10-01T10:07:51.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:51 smithi173 bash[27612]: cluster 2023-10-01T10:07:50.384518+0000 mon.a (mon.0) 248 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2023-10-01T10:07:52.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:52 smithi078 bash[33821]: cluster 2023-10-01T10:07:51.761674+0000 mgr.y (mgr.14146) 79 : cluster [DBG] pgmap v56: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:07:52.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:52 smithi078 bash[27336]: cluster 2023-10-01T10:07:51.761674+0000 mgr.y (mgr.14146) 79 : cluster [DBG] pgmap v56: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:07:52.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:52 smithi173 bash[27612]: cluster 2023-10-01T10:07:51.761674+0000 mgr.y (mgr.14146) 79 : cluster [DBG] pgmap v56: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:07:53.222 INFO:teuthology.orchestra.run.smithi078.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-10-01T10:07:53.222 INFO:teuthology.orchestra.run.smithi078.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-10-01T10:07:53.222 INFO:teuthology.orchestra.run.smithi078.stderr: stderr: 10+0 records in 2023-10-01T10:07:53.222 INFO:teuthology.orchestra.run.smithi078.stderr:10+0 records out 2023-10-01T10:07:53.222 INFO:teuthology.orchestra.run.smithi078.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0189778 s, 553 MB/s 2023-10-01T10:07:53.222 INFO:teuthology.orchestra.run.smithi078.stderr:--> Zapping successful for: 2023-10-01T10:07:54.005 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi078:vg_nvme/lv_3 2023-10-01T10:07:55.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:54 smithi078 bash[33821]: cluster 2023-10-01T10:07:53.762240+0000 mgr.y (mgr.14146) 80 : cluster [DBG] pgmap v57: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:07:55.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:54 smithi078 bash[27336]: cluster 2023-10-01T10:07:53.762240+0000 mgr.y (mgr.14146) 80 : cluster [DBG] pgmap v57: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:07:55.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:54 smithi173 bash[27612]: cluster 2023-10-01T10:07:53.762240+0000 mgr.y (mgr.14146) 80 : cluster [DBG] pgmap v57: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:07:57.064 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:56 smithi078 bash[33821]: cluster 2023-10-01T10:07:55.762783+0000 mgr.y (mgr.14146) 81 : cluster [DBG] pgmap v58: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:07:57.064 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:56 smithi078 bash[27336]: cluster 2023-10-01T10:07:55.762783+0000 mgr.y (mgr.14146) 81 : cluster [DBG] pgmap v58: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:07:57.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:56 smithi173 bash[27612]: cluster 2023-10-01T10:07:55.762783+0000 mgr.y (mgr.14146) 81 : cluster [DBG] pgmap v58: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:07:59.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:58 smithi173 bash[27612]: cluster 2023-10-01T10:07:57.763269+0000 mgr.y (mgr.14146) 82 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:07:59.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:58 smithi173 bash[27612]: audit 2023-10-01T10:07:58.821547+0000 mon.a (mon.0) 249 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T10:07:59.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:58 smithi173 bash[27612]: audit 2023-10-01T10:07:58.826672+0000 mon.a (mon.0) 250 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T10:07:59.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:58 smithi173 bash[27612]: audit 2023-10-01T10:07:58.828173+0000 mon.a (mon.0) 251 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:07:59.186 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:58 smithi078 bash[27336]: cluster 2023-10-01T10:07:57.763269+0000 mgr.y (mgr.14146) 82 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:07:59.186 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:58 smithi078 bash[27336]: audit 2023-10-01T10:07:58.821547+0000 mon.a (mon.0) 249 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T10:07:59.186 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:58 smithi078 bash[27336]: audit 2023-10-01T10:07:58.826672+0000 mon.a (mon.0) 250 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T10:07:59.186 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:58 smithi078 bash[27336]: audit 2023-10-01T10:07:58.828173+0000 mon.a (mon.0) 251 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:07:59.186 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:58 smithi078 bash[33821]: cluster 2023-10-01T10:07:57.763269+0000 mgr.y (mgr.14146) 82 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:07:59.186 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:58 smithi078 bash[33821]: audit 2023-10-01T10:07:58.821547+0000 mon.a (mon.0) 249 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T10:07:59.186 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:58 smithi078 bash[33821]: audit 2023-10-01T10:07:58.826672+0000 mon.a (mon.0) 250 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T10:07:59.186 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:58 smithi078 bash[33821]: audit 2023-10-01T10:07:58.828173+0000 mon.a (mon.0) 251 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:08:00.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:07:59 smithi173 bash[27612]: audit 2023-10-01T10:07:58.813825+0000 mgr.y (mgr.14146) 83 : audit [DBG] from='client.14226 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:08:00.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:07:59 smithi078 bash[27336]: audit 2023-10-01T10:07:58.813825+0000 mgr.y (mgr.14146) 83 : audit [DBG] from='client.14226 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:08:00.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:07:59 smithi078 bash[33821]: audit 2023-10-01T10:07:58.813825+0000 mgr.y (mgr.14146) 83 : audit [DBG] from='client.14226 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:08:01.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:01 smithi173 bash[27612]: cluster 2023-10-01T10:07:59.763853+0000 mgr.y (mgr.14146) 84 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:08:01.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:01 smithi173 bash[27612]: cephadm 2023-10-01T10:08:00.107046+0000 mgr.y (mgr.14146) 85 : cephadm [INF] Detected new or changed devices on smithi078 2023-10-01T10:08:01.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:01 smithi173 bash[27612]: audit 2023-10-01T10:08:00.115320+0000 mon.a (mon.0) 252 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:08:01.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:01 smithi173 bash[27612]: audit 2023-10-01T10:08:00.116669+0000 mon.a (mon.0) 253 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:08:01.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:01 smithi173 bash[27612]: cephadm 2023-10-01T10:08:00.117647+0000 mgr.y (mgr.14146) 86 : cephadm [INF] Adjusting osd_memory_target on smithi078 to 16241M 2023-10-01T10:08:01.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:01 smithi173 bash[27612]: audit 2023-10-01T10:08:00.121681+0000 mon.a (mon.0) 254 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:08:01.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:01 smithi173 bash[27612]: audit 2023-10-01T10:08:00.122832+0000 mon.a (mon.0) 255 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:08:01.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:01 smithi173 bash[27612]: audit 2023-10-01T10:08:00.123767+0000 mon.a (mon.0) 256 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:08:01.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:01 smithi173 bash[27612]: audit 2023-10-01T10:08:00.129354+0000 mon.a (mon.0) 257 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:08:01.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:01 smithi078 bash[33821]: cluster 2023-10-01T10:07:59.763853+0000 mgr.y (mgr.14146) 84 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:08:01.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:01 smithi078 bash[33821]: cephadm 2023-10-01T10:08:00.107046+0000 mgr.y (mgr.14146) 85 : cephadm [INF] Detected new or changed devices on smithi078 2023-10-01T10:08:01.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:01 smithi078 bash[33821]: audit 2023-10-01T10:08:00.115320+0000 mon.a (mon.0) 252 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:08:01.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:01 smithi078 bash[33821]: audit 2023-10-01T10:08:00.116669+0000 mon.a (mon.0) 253 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:08:01.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:01 smithi078 bash[33821]: cephadm 2023-10-01T10:08:00.117647+0000 mgr.y (mgr.14146) 86 : cephadm [INF] Adjusting osd_memory_target on smithi078 to 16241M 2023-10-01T10:08:01.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:01 smithi078 bash[33821]: audit 2023-10-01T10:08:00.121681+0000 mon.a (mon.0) 254 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:08:01.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:01 smithi078 bash[33821]: audit 2023-10-01T10:08:00.122832+0000 mon.a (mon.0) 255 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:08:01.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:01 smithi078 bash[33821]: audit 2023-10-01T10:08:00.123767+0000 mon.a (mon.0) 256 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:08:01.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:01 smithi078 bash[33821]: audit 2023-10-01T10:08:00.129354+0000 mon.a (mon.0) 257 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:08:01.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:01 smithi078 bash[27336]: cluster 2023-10-01T10:07:59.763853+0000 mgr.y (mgr.14146) 84 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:08:01.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:01 smithi078 bash[27336]: cephadm 2023-10-01T10:08:00.107046+0000 mgr.y (mgr.14146) 85 : cephadm [INF] Detected new or changed devices on smithi078 2023-10-01T10:08:01.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:01 smithi078 bash[27336]: audit 2023-10-01T10:08:00.115320+0000 mon.a (mon.0) 252 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:08:01.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:01 smithi078 bash[27336]: audit 2023-10-01T10:08:00.116669+0000 mon.a (mon.0) 253 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:08:01.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:01 smithi078 bash[27336]: cephadm 2023-10-01T10:08:00.117647+0000 mgr.y (mgr.14146) 86 : cephadm [INF] Adjusting osd_memory_target on smithi078 to 16241M 2023-10-01T10:08:01.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:01 smithi078 bash[27336]: audit 2023-10-01T10:08:00.121681+0000 mon.a (mon.0) 254 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:08:01.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:01 smithi078 bash[27336]: audit 2023-10-01T10:08:00.122832+0000 mon.a (mon.0) 255 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:08:01.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:01 smithi078 bash[27336]: audit 2023-10-01T10:08:00.123767+0000 mon.a (mon.0) 256 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:08:01.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:01 smithi078 bash[27336]: audit 2023-10-01T10:08:00.129354+0000 mon.a (mon.0) 257 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:08:03.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:03 smithi078 bash[27336]: cluster 2023-10-01T10:08:01.764403+0000 mgr.y (mgr.14146) 87 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:08:03.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:03 smithi078 bash[33821]: cluster 2023-10-01T10:08:01.764403+0000 mgr.y (mgr.14146) 87 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:08:03.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:03 smithi173 bash[27612]: cluster 2023-10-01T10:08:01.764403+0000 mgr.y (mgr.14146) 87 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:08:05.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:05 smithi078 bash[27336]: cluster 2023-10-01T10:08:03.764749+0000 mgr.y (mgr.14146) 88 : cluster [DBG] pgmap v62: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:08:05.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:05 smithi078 bash[33821]: cluster 2023-10-01T10:08:03.764749+0000 mgr.y (mgr.14146) 88 : cluster [DBG] pgmap v62: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:08:05.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:05 smithi173 bash[27612]: cluster 2023-10-01T10:08:03.764749+0000 mgr.y (mgr.14146) 88 : cluster [DBG] pgmap v62: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:08:07.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:07 smithi078 bash[27336]: cluster 2023-10-01T10:08:05.765211+0000 mgr.y (mgr.14146) 89 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:08:07.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:07 smithi078 bash[27336]: audit 2023-10-01T10:08:07.079689+0000 mon.c (mon.2) 5 : audit [INF] from='client.? 172.21.15.78:0/164894040' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4e6bea65-4109-43d2-b5be-3811fdd2a803"}]: dispatch 2023-10-01T10:08:07.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:07 smithi078 bash[27336]: audit 2023-10-01T10:08:07.080360+0000 mon.a (mon.0) 258 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4e6bea65-4109-43d2-b5be-3811fdd2a803"}]: dispatch 2023-10-01T10:08:07.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:07 smithi078 bash[27336]: audit 2023-10-01T10:08:07.086072+0000 mon.a (mon.0) 259 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4e6bea65-4109-43d2-b5be-3811fdd2a803"}]': finished 2023-10-01T10:08:07.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:07 smithi078 bash[27336]: cluster 2023-10-01T10:08:07.086115+0000 mon.a (mon.0) 260 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2023-10-01T10:08:07.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:07 smithi078 bash[27336]: audit 2023-10-01T10:08:07.086299+0000 mon.a (mon.0) 261 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T10:08:07.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:07 smithi078 bash[33821]: cluster 2023-10-01T10:08:05.765211+0000 mgr.y (mgr.14146) 89 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:08:07.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:07 smithi078 bash[33821]: audit 2023-10-01T10:08:07.079689+0000 mon.c (mon.2) 5 : audit [INF] from='client.? 172.21.15.78:0/164894040' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4e6bea65-4109-43d2-b5be-3811fdd2a803"}]: dispatch 2023-10-01T10:08:07.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:07 smithi078 bash[33821]: audit 2023-10-01T10:08:07.080360+0000 mon.a (mon.0) 258 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4e6bea65-4109-43d2-b5be-3811fdd2a803"}]: dispatch 2023-10-01T10:08:07.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:07 smithi078 bash[33821]: audit 2023-10-01T10:08:07.086072+0000 mon.a (mon.0) 259 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4e6bea65-4109-43d2-b5be-3811fdd2a803"}]': finished 2023-10-01T10:08:07.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:07 smithi078 bash[33821]: cluster 2023-10-01T10:08:07.086115+0000 mon.a (mon.0) 260 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2023-10-01T10:08:07.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:07 smithi078 bash[33821]: audit 2023-10-01T10:08:07.086299+0000 mon.a (mon.0) 261 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T10:08:07.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:07 smithi173 bash[27612]: cluster 2023-10-01T10:08:05.765211+0000 mgr.y (mgr.14146) 89 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:08:07.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:07 smithi173 bash[27612]: audit 2023-10-01T10:08:07.079689+0000 mon.c (mon.2) 5 : audit [INF] from='client.? 172.21.15.78:0/164894040' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4e6bea65-4109-43d2-b5be-3811fdd2a803"}]: dispatch 2023-10-01T10:08:07.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:07 smithi173 bash[27612]: audit 2023-10-01T10:08:07.080360+0000 mon.a (mon.0) 258 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4e6bea65-4109-43d2-b5be-3811fdd2a803"}]: dispatch 2023-10-01T10:08:07.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:07 smithi173 bash[27612]: audit 2023-10-01T10:08:07.086072+0000 mon.a (mon.0) 259 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4e6bea65-4109-43d2-b5be-3811fdd2a803"}]': finished 2023-10-01T10:08:07.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:07 smithi173 bash[27612]: cluster 2023-10-01T10:08:07.086115+0000 mon.a (mon.0) 260 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2023-10-01T10:08:07.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:07 smithi173 bash[27612]: audit 2023-10-01T10:08:07.086299+0000 mon.a (mon.0) 261 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T10:08:08.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:08 smithi078 bash[27336]: audit 2023-10-01T10:08:07.810527+0000 mon.c (mon.2) 6 : audit [DBG] from='client.? 172.21.15.78:0/695720815' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T10:08:08.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:08 smithi078 bash[33821]: audit 2023-10-01T10:08:07.810527+0000 mon.c (mon.2) 6 : audit [DBG] from='client.? 172.21.15.78:0/695720815' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T10:08:08.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:08 smithi173 bash[27612]: audit 2023-10-01T10:08:07.810527+0000 mon.c (mon.2) 6 : audit [DBG] from='client.? 172.21.15.78:0/695720815' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T10:08:09.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:09 smithi078 bash[27336]: cluster 2023-10-01T10:08:07.765586+0000 mgr.y (mgr.14146) 90 : cluster [DBG] pgmap v65: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:08:09.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:09 smithi078 bash[33821]: cluster 2023-10-01T10:08:07.765586+0000 mgr.y (mgr.14146) 90 : cluster [DBG] pgmap v65: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:08:09.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:09 smithi173 bash[27612]: cluster 2023-10-01T10:08:07.765586+0000 mgr.y (mgr.14146) 90 : cluster [DBG] pgmap v65: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:08:11.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:11 smithi078 bash[27336]: cluster 2023-10-01T10:08:09.766048+0000 mgr.y (mgr.14146) 91 : cluster [DBG] pgmap v66: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:08:11.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:11 smithi078 bash[33821]: cluster 2023-10-01T10:08:09.766048+0000 mgr.y (mgr.14146) 91 : cluster [DBG] pgmap v66: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:08:11.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:11 smithi173 bash[27612]: cluster 2023-10-01T10:08:09.766048+0000 mgr.y (mgr.14146) 91 : cluster [DBG] pgmap v66: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:08:13.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:13 smithi078 bash[27336]: cluster 2023-10-01T10:08:11.766564+0000 mgr.y (mgr.14146) 92 : cluster [DBG] pgmap v67: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:08:13.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:13 smithi078 bash[33821]: cluster 2023-10-01T10:08:11.766564+0000 mgr.y (mgr.14146) 92 : cluster [DBG] pgmap v67: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:08:13.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:13 smithi173 bash[27612]: cluster 2023-10-01T10:08:11.766564+0000 mgr.y (mgr.14146) 92 : cluster [DBG] pgmap v67: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:08:15.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:15 smithi078 bash[33821]: cluster 2023-10-01T10:08:13.767025+0000 mgr.y (mgr.14146) 93 : cluster [DBG] pgmap v68: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:08:15.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:15 smithi078 bash[27336]: cluster 2023-10-01T10:08:13.767025+0000 mgr.y (mgr.14146) 93 : cluster [DBG] pgmap v68: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:08:15.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:15 smithi173 bash[27612]: cluster 2023-10-01T10:08:13.767025+0000 mgr.y (mgr.14146) 93 : cluster [DBG] pgmap v68: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:08:17.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:17 smithi078 bash[33821]: cluster 2023-10-01T10:08:15.767511+0000 mgr.y (mgr.14146) 94 : cluster [DBG] pgmap v69: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:08:17.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:17 smithi078 bash[27336]: cluster 2023-10-01T10:08:15.767511+0000 mgr.y (mgr.14146) 94 : cluster [DBG] pgmap v69: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:08:17.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:17 smithi173 bash[27612]: cluster 2023-10-01T10:08:15.767511+0000 mgr.y (mgr.14146) 94 : cluster [DBG] pgmap v69: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:08:19.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:19 smithi078 bash[33821]: cluster 2023-10-01T10:08:17.767841+0000 mgr.y (mgr.14146) 95 : cluster [DBG] pgmap v70: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:08:19.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:19 smithi078 bash[27336]: cluster 2023-10-01T10:08:17.767841+0000 mgr.y (mgr.14146) 95 : cluster [DBG] pgmap v70: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:08:19.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:19 smithi173 bash[27612]: cluster 2023-10-01T10:08:17.767841+0000 mgr.y (mgr.14146) 95 : cluster [DBG] pgmap v70: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:08:20.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:20 smithi078 bash[27336]: audit 2023-10-01T10:08:19.633313+0000 mon.a (mon.0) 262 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-01T10:08:20.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:20 smithi078 bash[27336]: audit 2023-10-01T10:08:19.634635+0000 mon.a (mon.0) 263 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:08:20.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:20 smithi078 bash[33821]: audit 2023-10-01T10:08:19.633313+0000 mon.a (mon.0) 262 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-01T10:08:20.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:20 smithi078 bash[33821]: audit 2023-10-01T10:08:19.634635+0000 mon.a (mon.0) 263 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:08:20.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:20 smithi173 bash[27612]: audit 2023-10-01T10:08:19.633313+0000 mon.a (mon.0) 262 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-01T10:08:20.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:20 smithi173 bash[27612]: audit 2023-10-01T10:08:19.634635+0000 mon.a (mon.0) 263 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:08:21.336 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:21 smithi078 bash[27336]: cephadm 2023-10-01T10:08:19.635680+0000 mgr.y (mgr.14146) 96 : cephadm [INF] Deploying daemon osd.1 on smithi078 2023-10-01T10:08:21.336 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:21 smithi078 bash[27336]: cluster 2023-10-01T10:08:19.768260+0000 mgr.y (mgr.14146) 97 : cluster [DBG] pgmap v71: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:08:21.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:21 smithi078 bash[33821]: cephadm 2023-10-01T10:08:19.635680+0000 mgr.y (mgr.14146) 96 : cephadm [INF] Deploying daemon osd.1 on smithi078 2023-10-01T10:08:21.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:21 smithi078 bash[33821]: cluster 2023-10-01T10:08:19.768260+0000 mgr.y (mgr.14146) 97 : cluster [DBG] pgmap v71: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:08:21.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:21 smithi173 bash[27612]: cephadm 2023-10-01T10:08:19.635680+0000 mgr.y (mgr.14146) 96 : cephadm [INF] Deploying daemon osd.1 on smithi078 2023-10-01T10:08:21.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:21 smithi173 bash[27612]: cluster 2023-10-01T10:08:19.768260+0000 mgr.y (mgr.14146) 97 : cluster [DBG] pgmap v71: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:08:23.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:23 smithi078 bash[33821]: cluster 2023-10-01T10:08:21.768773+0000 mgr.y (mgr.14146) 98 : cluster [DBG] pgmap v72: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:08:23.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:23 smithi078 bash[27336]: cluster 2023-10-01T10:08:21.768773+0000 mgr.y (mgr.14146) 98 : cluster [DBG] pgmap v72: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:08:23.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:23 smithi173 bash[27612]: cluster 2023-10-01T10:08:21.768773+0000 mgr.y (mgr.14146) 98 : cluster [DBG] pgmap v72: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:08:25.327 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:24 smithi078 bash[27336]: cluster 2023-10-01T10:08:23.769317+0000 mgr.y (mgr.14146) 99 : cluster [DBG] pgmap v73: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:08:25.327 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:24 smithi078 bash[27336]: audit 2023-10-01T10:08:23.923075+0000 mon.a (mon.0) 264 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:08:25.327 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:24 smithi078 bash[27336]: audit 2023-10-01T10:08:23.956828+0000 mon.a (mon.0) 265 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:08:25.327 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:24 smithi078 bash[33821]: cluster 2023-10-01T10:08:23.769317+0000 mgr.y (mgr.14146) 99 : cluster [DBG] pgmap v73: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:08:25.327 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:24 smithi078 bash[33821]: audit 2023-10-01T10:08:23.923075+0000 mon.a (mon.0) 264 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:08:25.327 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:24 smithi078 bash[33821]: audit 2023-10-01T10:08:23.956828+0000 mon.a (mon.0) 265 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:08:25.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:24 smithi173 bash[27612]: cluster 2023-10-01T10:08:23.769317+0000 mgr.y (mgr.14146) 99 : cluster [DBG] pgmap v73: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:08:25.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:24 smithi173 bash[27612]: audit 2023-10-01T10:08:23.923075+0000 mon.a (mon.0) 264 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:08:25.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:24 smithi173 bash[27612]: audit 2023-10-01T10:08:23.956828+0000 mon.a (mon.0) 265 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:08:27.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:26 smithi078 bash[27336]: cluster 2023-10-01T10:08:25.769824+0000 mgr.y (mgr.14146) 100 : cluster [DBG] pgmap v74: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:08:27.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:26 smithi078 bash[33821]: cluster 2023-10-01T10:08:25.769824+0000 mgr.y (mgr.14146) 100 : cluster [DBG] pgmap v74: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:08:27.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:26 smithi173 bash[27612]: cluster 2023-10-01T10:08:25.769824+0000 mgr.y (mgr.14146) 100 : cluster [DBG] pgmap v74: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:08:29.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:28 smithi078 bash[27336]: cluster 2023-10-01T10:08:27.770172+0000 mgr.y (mgr.14146) 101 : cluster [DBG] pgmap v75: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:08:29.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:28 smithi078 bash[33821]: cluster 2023-10-01T10:08:27.770172+0000 mgr.y (mgr.14146) 101 : cluster [DBG] pgmap v75: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:08:29.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:28 smithi173 bash[27612]: cluster 2023-10-01T10:08:27.770172+0000 mgr.y (mgr.14146) 101 : cluster [DBG] pgmap v75: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:08:30.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:30 smithi078 bash[27336]: audit 2023-10-01T10:08:29.090192+0000 mon.a (mon.0) 266 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:08:30.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:30 smithi078 bash[27336]: audit 2023-10-01T10:08:29.338740+0000 mon.a (mon.0) 267 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:08:30.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:30 smithi078 bash[27336]: audit 2023-10-01T10:08:29.339895+0000 mon.a (mon.0) 268 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:08:30.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:30 smithi078 bash[27336]: audit 2023-10-01T10:08:29.347713+0000 mon.a (mon.0) 269 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:08:30.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:30 smithi078 bash[33821]: audit 2023-10-01T10:08:29.090192+0000 mon.a (mon.0) 266 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:08:30.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:30 smithi078 bash[33821]: audit 2023-10-01T10:08:29.338740+0000 mon.a (mon.0) 267 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:08:30.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:30 smithi078 bash[33821]: audit 2023-10-01T10:08:29.339895+0000 mon.a (mon.0) 268 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:08:30.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:30 smithi078 bash[33821]: audit 2023-10-01T10:08:29.347713+0000 mon.a (mon.0) 269 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:08:30.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:30 smithi173 bash[27612]: audit 2023-10-01T10:08:29.090192+0000 mon.a (mon.0) 266 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:08:30.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:30 smithi173 bash[27612]: audit 2023-10-01T10:08:29.338740+0000 mon.a (mon.0) 267 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:08:30.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:30 smithi173 bash[27612]: audit 2023-10-01T10:08:29.339895+0000 mon.a (mon.0) 268 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:08:30.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:30 smithi173 bash[27612]: audit 2023-10-01T10:08:29.347713+0000 mon.a (mon.0) 269 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:08:31.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:31 smithi173 bash[27612]: cluster 2023-10-01T10:08:29.770632+0000 mgr.y (mgr.14146) 102 : cluster [DBG] pgmap v76: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:08:31.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:31 smithi173 bash[27612]: audit 2023-10-01T10:08:30.384687+0000 mon.c (mon.2) 7 : audit [INF] from='osd.1 [v2:172.21.15.78:6810/3462774695,v1:172.21.15.78:6811/3462774695]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-01T10:08:31.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:31 smithi173 bash[27612]: audit 2023-10-01T10:08:30.385132+0000 mon.a (mon.0) 270 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-01T10:08:31.391 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:31 smithi078 bash[27336]: cluster 2023-10-01T10:08:29.770632+0000 mgr.y (mgr.14146) 102 : cluster [DBG] pgmap v76: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:08:31.391 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:31 smithi078 bash[27336]: audit 2023-10-01T10:08:30.384687+0000 mon.c (mon.2) 7 : audit [INF] from='osd.1 [v2:172.21.15.78:6810/3462774695,v1:172.21.15.78:6811/3462774695]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-01T10:08:31.391 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:31 smithi078 bash[27336]: audit 2023-10-01T10:08:30.385132+0000 mon.a (mon.0) 270 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-01T10:08:31.392 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:31 smithi078 bash[33821]: cluster 2023-10-01T10:08:29.770632+0000 mgr.y (mgr.14146) 102 : cluster [DBG] pgmap v76: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:08:31.392 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:31 smithi078 bash[33821]: audit 2023-10-01T10:08:30.384687+0000 mon.c (mon.2) 7 : audit [INF] from='osd.1 [v2:172.21.15.78:6810/3462774695,v1:172.21.15.78:6811/3462774695]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-01T10:08:31.392 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:31 smithi078 bash[33821]: audit 2023-10-01T10:08:30.385132+0000 mon.a (mon.0) 270 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-01T10:08:32.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:32 smithi078 bash[33821]: audit 2023-10-01T10:08:31.352059+0000 mon.a (mon.0) 271 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-01T10:08:32.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:32 smithi078 bash[33821]: cluster 2023-10-01T10:08:31.352213+0000 mon.a (mon.0) 272 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2023-10-01T10:08:32.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:32 smithi078 bash[33821]: audit 2023-10-01T10:08:31.352442+0000 mon.a (mon.0) 273 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T10:08:32.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:32 smithi078 bash[33821]: audit 2023-10-01T10:08:31.353722+0000 mon.c (mon.2) 8 : audit [INF] from='osd.1 [v2:172.21.15.78:6810/3462774695,v1:172.21.15.78:6811/3462774695]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2023-10-01T10:08:32.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:32 smithi078 bash[33821]: audit 2023-10-01T10:08:31.354258+0000 mon.a (mon.0) 274 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2023-10-01T10:08:32.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:32 smithi078 bash[33821]: cluster 2023-10-01T10:08:31.770963+0000 mgr.y (mgr.14146) 103 : cluster [DBG] pgmap v78: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:08:32.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:32 smithi078 bash[27336]: audit 2023-10-01T10:08:31.352059+0000 mon.a (mon.0) 271 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-01T10:08:32.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:32 smithi078 bash[27336]: cluster 2023-10-01T10:08:31.352213+0000 mon.a (mon.0) 272 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2023-10-01T10:08:32.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:32 smithi078 bash[27336]: audit 2023-10-01T10:08:31.352442+0000 mon.a (mon.0) 273 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T10:08:32.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:32 smithi078 bash[27336]: audit 2023-10-01T10:08:31.353722+0000 mon.c (mon.2) 8 : audit [INF] from='osd.1 [v2:172.21.15.78:6810/3462774695,v1:172.21.15.78:6811/3462774695]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2023-10-01T10:08:32.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:32 smithi078 bash[27336]: audit 2023-10-01T10:08:31.354258+0000 mon.a (mon.0) 274 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2023-10-01T10:08:32.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:32 smithi078 bash[27336]: cluster 2023-10-01T10:08:31.770963+0000 mgr.y (mgr.14146) 103 : cluster [DBG] pgmap v78: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:08:32.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:32 smithi173 bash[27612]: audit 2023-10-01T10:08:31.352059+0000 mon.a (mon.0) 271 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-01T10:08:32.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:32 smithi173 bash[27612]: cluster 2023-10-01T10:08:31.352213+0000 mon.a (mon.0) 272 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2023-10-01T10:08:32.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:32 smithi173 bash[27612]: audit 2023-10-01T10:08:31.352442+0000 mon.a (mon.0) 273 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T10:08:32.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:32 smithi173 bash[27612]: audit 2023-10-01T10:08:31.353722+0000 mon.c (mon.2) 8 : audit [INF] from='osd.1 [v2:172.21.15.78:6810/3462774695,v1:172.21.15.78:6811/3462774695]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2023-10-01T10:08:32.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:32 smithi173 bash[27612]: audit 2023-10-01T10:08:31.354258+0000 mon.a (mon.0) 274 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2023-10-01T10:08:32.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:32 smithi173 bash[27612]: cluster 2023-10-01T10:08:31.770963+0000 mgr.y (mgr.14146) 103 : cluster [DBG] pgmap v78: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:08:32.669 INFO:teuthology.orchestra.run.smithi078.stdout:Created osd(s) 1 on host 'smithi078' 2023-10-01T10:08:33.332 DEBUG:teuthology.orchestra.run.smithi078:osd.1> sudo journalctl -f -n 0 -u ceph-d8016480-6041-11ee-8db5-212e2dc638e7@osd.1.service 2023-10-01T10:08:33.335 INFO:tasks.cephadm:Deploying osd.2 on smithi078 with /dev/vg_nvme/lv_2... 2023-10-01T10:08:33.335 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-10-01T10:08:33.435 INFO:journalctl@ceph.osd.1.smithi078.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-01T10:08:33.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:33 smithi078 bash[27336]: audit 2023-10-01T10:08:32.355146+0000 mon.a (mon.0) 275 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2023-10-01T10:08:33.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:33 smithi078 bash[27336]: cluster 2023-10-01T10:08:32.355268+0000 mon.a (mon.0) 276 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2023-10-01T10:08:33.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:33 smithi078 bash[27336]: audit 2023-10-01T10:08:32.355908+0000 mon.a (mon.0) 277 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T10:08:33.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:33 smithi078 bash[27336]: audit 2023-10-01T10:08:32.361948+0000 mon.a (mon.0) 278 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T10:08:33.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:33 smithi078 bash[27336]: audit 2023-10-01T10:08:32.666949+0000 mon.a (mon.0) 279 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:08:33.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:33 smithi078 bash[27336]: audit 2023-10-01T10:08:32.671143+0000 mon.a (mon.0) 280 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:08:33.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:33 smithi078 bash[33821]: audit 2023-10-01T10:08:32.355146+0000 mon.a (mon.0) 275 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2023-10-01T10:08:33.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:33 smithi078 bash[33821]: cluster 2023-10-01T10:08:32.355268+0000 mon.a (mon.0) 276 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2023-10-01T10:08:33.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:33 smithi078 bash[33821]: audit 2023-10-01T10:08:32.355908+0000 mon.a (mon.0) 277 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T10:08:33.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:33 smithi078 bash[33821]: audit 2023-10-01T10:08:32.361948+0000 mon.a (mon.0) 278 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T10:08:33.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:33 smithi078 bash[33821]: audit 2023-10-01T10:08:32.666949+0000 mon.a (mon.0) 279 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:08:33.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:33 smithi078 bash[33821]: audit 2023-10-01T10:08:32.671143+0000 mon.a (mon.0) 280 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:08:33.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:33 smithi173 bash[27612]: audit 2023-10-01T10:08:32.355146+0000 mon.a (mon.0) 275 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2023-10-01T10:08:33.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:33 smithi173 bash[27612]: cluster 2023-10-01T10:08:32.355268+0000 mon.a (mon.0) 276 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2023-10-01T10:08:33.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:33 smithi173 bash[27612]: audit 2023-10-01T10:08:32.355908+0000 mon.a (mon.0) 277 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T10:08:33.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:33 smithi173 bash[27612]: audit 2023-10-01T10:08:32.361948+0000 mon.a (mon.0) 278 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T10:08:33.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:33 smithi173 bash[27612]: audit 2023-10-01T10:08:32.666949+0000 mon.a (mon.0) 279 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:08:33.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:33 smithi173 bash[27612]: audit 2023-10-01T10:08:32.671143+0000 mon.a (mon.0) 280 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:08:34.733 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:34 smithi078 bash[27336]: cluster 2023-10-01T10:08:31.368315+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-01T10:08:34.733 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:34 smithi078 bash[27336]: cluster 2023-10-01T10:08:31.368393+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-01T10:08:34.733 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:34 smithi078 bash[27336]: cluster 2023-10-01T10:08:33.366453+0000 mon.a (mon.0) 281 : cluster [INF] osd.1 [v2:172.21.15.78:6810/3462774695,v1:172.21.15.78:6811/3462774695] boot 2023-10-01T10:08:34.734 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:34 smithi078 bash[27336]: cluster 2023-10-01T10:08:33.366508+0000 mon.a (mon.0) 282 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2023-10-01T10:08:34.734 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:34 smithi078 bash[27336]: audit 2023-10-01T10:08:33.367005+0000 mon.a (mon.0) 283 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T10:08:34.734 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:34 smithi078 bash[27336]: cluster 2023-10-01T10:08:33.771510+0000 mgr.y (mgr.14146) 104 : cluster [DBG] pgmap v81: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:08:34.734 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:34 smithi078 bash[33821]: cluster 2023-10-01T10:08:31.368315+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-01T10:08:34.734 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:34 smithi078 bash[33821]: cluster 2023-10-01T10:08:31.368393+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-01T10:08:34.734 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:34 smithi078 bash[33821]: cluster 2023-10-01T10:08:33.366453+0000 mon.a (mon.0) 281 : cluster [INF] osd.1 [v2:172.21.15.78:6810/3462774695,v1:172.21.15.78:6811/3462774695] boot 2023-10-01T10:08:34.734 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:34 smithi078 bash[33821]: cluster 2023-10-01T10:08:33.366508+0000 mon.a (mon.0) 282 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2023-10-01T10:08:34.734 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:34 smithi078 bash[33821]: audit 2023-10-01T10:08:33.367005+0000 mon.a (mon.0) 283 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T10:08:34.735 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:34 smithi078 bash[33821]: cluster 2023-10-01T10:08:33.771510+0000 mgr.y (mgr.14146) 104 : cluster [DBG] pgmap v81: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:08:34.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:34 smithi173 bash[27612]: cluster 2023-10-01T10:08:31.368315+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-01T10:08:34.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:34 smithi173 bash[27612]: cluster 2023-10-01T10:08:31.368393+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-01T10:08:34.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:34 smithi173 bash[27612]: cluster 2023-10-01T10:08:33.366453+0000 mon.a (mon.0) 281 : cluster [INF] osd.1 [v2:172.21.15.78:6810/3462774695,v1:172.21.15.78:6811/3462774695] boot 2023-10-01T10:08:34.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:34 smithi173 bash[27612]: cluster 2023-10-01T10:08:33.366508+0000 mon.a (mon.0) 282 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2023-10-01T10:08:34.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:34 smithi173 bash[27612]: audit 2023-10-01T10:08:33.367005+0000 mon.a (mon.0) 283 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T10:08:34.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:34 smithi173 bash[27612]: cluster 2023-10-01T10:08:33.771510+0000 mgr.y (mgr.14146) 104 : cluster [DBG] pgmap v81: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2023-10-01T10:08:35.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:35 smithi078 bash[33821]: cluster 2023-10-01T10:08:34.383829+0000 mon.a (mon.0) 284 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2023-10-01T10:08:35.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:35 smithi078 bash[27336]: cluster 2023-10-01T10:08:34.383829+0000 mon.a (mon.0) 284 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2023-10-01T10:08:35.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:35 smithi173 bash[27612]: cluster 2023-10-01T10:08:34.383829+0000 mon.a (mon.0) 284 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2023-10-01T10:08:36.618 INFO:teuthology.orchestra.run.smithi078.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-10-01T10:08:36.618 INFO:teuthology.orchestra.run.smithi078.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-10-01T10:08:36.618 INFO:teuthology.orchestra.run.smithi078.stderr: stderr: 10+0 records in 2023-10-01T10:08:36.618 INFO:teuthology.orchestra.run.smithi078.stderr:10+0 records out 2023-10-01T10:08:36.619 INFO:teuthology.orchestra.run.smithi078.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0189259 s, 554 MB/s 2023-10-01T10:08:36.619 INFO:teuthology.orchestra.run.smithi078.stderr:--> Zapping successful for: 2023-10-01T10:08:37.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:36 smithi078 bash[27336]: cluster 2023-10-01T10:08:35.772051+0000 mgr.y (mgr.14146) 105 : cluster [DBG] pgmap v83: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-01T10:08:37.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:36 smithi078 bash[33821]: cluster 2023-10-01T10:08:35.772051+0000 mgr.y (mgr.14146) 105 : cluster [DBG] pgmap v83: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-01T10:08:37.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:36 smithi173 bash[27612]: cluster 2023-10-01T10:08:35.772051+0000 mgr.y (mgr.14146) 105 : cluster [DBG] pgmap v83: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-01T10:08:37.329 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi078:vg_nvme/lv_2 2023-10-01T10:08:39.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:38 smithi173 bash[27612]: cluster 2023-10-01T10:08:37.772409+0000 mgr.y (mgr.14146) 106 : cluster [DBG] pgmap v84: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-01T10:08:39.152 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:38 smithi078 bash[33821]: cluster 2023-10-01T10:08:37.772409+0000 mgr.y (mgr.14146) 106 : cluster [DBG] pgmap v84: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-01T10:08:39.152 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:38 smithi078 bash[27336]: cluster 2023-10-01T10:08:37.772409+0000 mgr.y (mgr.14146) 106 : cluster [DBG] pgmap v84: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-01T10:08:41.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:40 smithi173 bash[27612]: cluster 2023-10-01T10:08:39.772897+0000 mgr.y (mgr.14146) 107 : cluster [DBG] pgmap v85: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-01T10:08:41.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:40 smithi078 bash[27336]: cluster 2023-10-01T10:08:39.772897+0000 mgr.y (mgr.14146) 107 : cluster [DBG] pgmap v85: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-01T10:08:41.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:40 smithi078 bash[33821]: cluster 2023-10-01T10:08:39.772897+0000 mgr.y (mgr.14146) 107 : cluster [DBG] pgmap v85: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-01T10:08:43.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:42 smithi078 bash[33821]: cluster 2023-10-01T10:08:41.773414+0000 mgr.y (mgr.14146) 108 : cluster [DBG] pgmap v86: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-01T10:08:43.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:42 smithi078 bash[33821]: audit 2023-10-01T10:08:42.057410+0000 mgr.y (mgr.14146) 109 : audit [DBG] from='client.24173 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:08:43.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:42 smithi078 bash[33821]: audit 2023-10-01T10:08:42.063078+0000 mon.a (mon.0) 285 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T10:08:43.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:42 smithi078 bash[33821]: audit 2023-10-01T10:08:42.067340+0000 mon.a (mon.0) 286 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T10:08:43.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:42 smithi078 bash[33821]: audit 2023-10-01T10:08:42.068467+0000 mon.a (mon.0) 287 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:08:43.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:42 smithi078 bash[27336]: cluster 2023-10-01T10:08:41.773414+0000 mgr.y (mgr.14146) 108 : cluster [DBG] pgmap v86: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-01T10:08:43.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:42 smithi078 bash[27336]: audit 2023-10-01T10:08:42.057410+0000 mgr.y (mgr.14146) 109 : audit [DBG] from='client.24173 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:08:43.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:42 smithi078 bash[27336]: audit 2023-10-01T10:08:42.063078+0000 mon.a (mon.0) 285 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T10:08:43.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:42 smithi078 bash[27336]: audit 2023-10-01T10:08:42.067340+0000 mon.a (mon.0) 286 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T10:08:43.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:42 smithi078 bash[27336]: audit 2023-10-01T10:08:42.068467+0000 mon.a (mon.0) 287 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:08:43.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:42 smithi173 bash[27612]: cluster 2023-10-01T10:08:41.773414+0000 mgr.y (mgr.14146) 108 : cluster [DBG] pgmap v86: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-01T10:08:43.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:42 smithi173 bash[27612]: audit 2023-10-01T10:08:42.057410+0000 mgr.y (mgr.14146) 109 : audit [DBG] from='client.24173 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:08:43.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:42 smithi173 bash[27612]: audit 2023-10-01T10:08:42.063078+0000 mon.a (mon.0) 285 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T10:08:43.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:42 smithi173 bash[27612]: audit 2023-10-01T10:08:42.067340+0000 mon.a (mon.0) 286 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T10:08:43.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:42 smithi173 bash[27612]: audit 2023-10-01T10:08:42.068467+0000 mon.a (mon.0) 287 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:08:44.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:44 smithi078 bash[27336]: cephadm 2023-10-01T10:08:43.424264+0000 mgr.y (mgr.14146) 110 : cephadm [INF] Detected new or changed devices on smithi078 2023-10-01T10:08:44.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:44 smithi078 bash[27336]: audit 2023-10-01T10:08:43.433472+0000 mon.a (mon.0) 288 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:08:44.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:44 smithi078 bash[27336]: audit 2023-10-01T10:08:43.435090+0000 mon.a (mon.0) 289 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:08:44.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:44 smithi078 bash[27336]: audit 2023-10-01T10:08:43.436480+0000 mon.a (mon.0) 290 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:08:44.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:44 smithi078 bash[27336]: cephadm 2023-10-01T10:08:43.437616+0000 mgr.y (mgr.14146) 111 : cephadm [INF] Adjusting osd_memory_target on smithi078 to 8120M 2023-10-01T10:08:44.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:44 smithi078 bash[27336]: audit 2023-10-01T10:08:43.444218+0000 mon.a (mon.0) 291 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:08:44.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:44 smithi078 bash[27336]: audit 2023-10-01T10:08:43.445795+0000 mon.a (mon.0) 292 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:08:44.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:44 smithi078 bash[27336]: audit 2023-10-01T10:08:43.446914+0000 mon.a (mon.0) 293 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:08:44.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:44 smithi078 bash[27336]: audit 2023-10-01T10:08:43.455408+0000 mon.a (mon.0) 294 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:08:44.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:44 smithi078 bash[27336]: cluster 2023-10-01T10:08:43.773888+0000 mgr.y (mgr.14146) 112 : cluster [DBG] pgmap v87: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-01T10:08:44.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:44 smithi078 bash[33821]: cephadm 2023-10-01T10:08:43.424264+0000 mgr.y (mgr.14146) 110 : cephadm [INF] Detected new or changed devices on smithi078 2023-10-01T10:08:44.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:44 smithi078 bash[33821]: audit 2023-10-01T10:08:43.433472+0000 mon.a (mon.0) 288 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:08:44.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:44 smithi078 bash[33821]: audit 2023-10-01T10:08:43.435090+0000 mon.a (mon.0) 289 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:08:44.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:44 smithi078 bash[33821]: audit 2023-10-01T10:08:43.436480+0000 mon.a (mon.0) 290 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:08:44.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:44 smithi078 bash[33821]: cephadm 2023-10-01T10:08:43.437616+0000 mgr.y (mgr.14146) 111 : cephadm [INF] Adjusting osd_memory_target on smithi078 to 8120M 2023-10-01T10:08:44.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:44 smithi078 bash[33821]: audit 2023-10-01T10:08:43.444218+0000 mon.a (mon.0) 291 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:08:44.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:44 smithi078 bash[33821]: audit 2023-10-01T10:08:43.445795+0000 mon.a (mon.0) 292 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:08:44.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:44 smithi078 bash[33821]: audit 2023-10-01T10:08:43.446914+0000 mon.a (mon.0) 293 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:08:44.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:44 smithi078 bash[33821]: audit 2023-10-01T10:08:43.455408+0000 mon.a (mon.0) 294 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:08:44.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:44 smithi078 bash[33821]: cluster 2023-10-01T10:08:43.773888+0000 mgr.y (mgr.14146) 112 : cluster [DBG] pgmap v87: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-01T10:08:44.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:44 smithi173 bash[27612]: cephadm 2023-10-01T10:08:43.424264+0000 mgr.y (mgr.14146) 110 : cephadm [INF] Detected new or changed devices on smithi078 2023-10-01T10:08:44.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:44 smithi173 bash[27612]: audit 2023-10-01T10:08:43.433472+0000 mon.a (mon.0) 288 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:08:44.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:44 smithi173 bash[27612]: audit 2023-10-01T10:08:43.435090+0000 mon.a (mon.0) 289 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:08:44.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:44 smithi173 bash[27612]: audit 2023-10-01T10:08:43.436480+0000 mon.a (mon.0) 290 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:08:44.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:44 smithi173 bash[27612]: cephadm 2023-10-01T10:08:43.437616+0000 mgr.y (mgr.14146) 111 : cephadm [INF] Adjusting osd_memory_target on smithi078 to 8120M 2023-10-01T10:08:44.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:44 smithi173 bash[27612]: audit 2023-10-01T10:08:43.444218+0000 mon.a (mon.0) 291 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:08:44.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:44 smithi173 bash[27612]: audit 2023-10-01T10:08:43.445795+0000 mon.a (mon.0) 292 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:08:44.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:44 smithi173 bash[27612]: audit 2023-10-01T10:08:43.446914+0000 mon.a (mon.0) 293 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:08:44.872 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:44 smithi173 bash[27612]: audit 2023-10-01T10:08:43.455408+0000 mon.a (mon.0) 294 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:08:44.872 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:44 smithi173 bash[27612]: cluster 2023-10-01T10:08:43.773888+0000 mgr.y (mgr.14146) 112 : cluster [DBG] pgmap v87: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-01T10:08:47.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:46 smithi173 bash[27612]: cluster 2023-10-01T10:08:45.774364+0000 mgr.y (mgr.14146) 113 : cluster [DBG] pgmap v88: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-01T10:08:47.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:46 smithi078 bash[33821]: cluster 2023-10-01T10:08:45.774364+0000 mgr.y (mgr.14146) 113 : cluster [DBG] pgmap v88: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-01T10:08:47.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:46 smithi078 bash[27336]: cluster 2023-10-01T10:08:45.774364+0000 mgr.y (mgr.14146) 113 : cluster [DBG] pgmap v88: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-01T10:08:49.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:48 smithi173 bash[27612]: cluster 2023-10-01T10:08:47.774808+0000 mgr.y (mgr.14146) 114 : cluster [DBG] pgmap v89: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-01T10:08:49.268 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:48 smithi078 bash[33821]: cluster 2023-10-01T10:08:47.774808+0000 mgr.y (mgr.14146) 114 : cluster [DBG] pgmap v89: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-01T10:08:49.268 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:48 smithi078 bash[27336]: cluster 2023-10-01T10:08:47.774808+0000 mgr.y (mgr.14146) 114 : cluster [DBG] pgmap v89: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-01T10:08:51.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:50 smithi173 bash[27612]: cluster 2023-10-01T10:08:49.775216+0000 mgr.y (mgr.14146) 115 : cluster [DBG] pgmap v90: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-01T10:08:51.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:50 smithi173 bash[27612]: audit 2023-10-01T10:08:50.227036+0000 mon.a (mon.0) 295 : audit [INF] from='client.? 172.21.15.78:0/454492136' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9f830594-31d6-4994-85fa-0615c280d5c3"}]: dispatch 2023-10-01T10:08:51.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:50 smithi173 bash[27612]: audit 2023-10-01T10:08:50.239099+0000 mon.a (mon.0) 296 : audit [INF] from='client.? 172.21.15.78:0/454492136' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9f830594-31d6-4994-85fa-0615c280d5c3"}]': finished 2023-10-01T10:08:51.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:50 smithi173 bash[27612]: cluster 2023-10-01T10:08:50.239157+0000 mon.a (mon.0) 297 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2023-10-01T10:08:51.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:50 smithi173 bash[27612]: audit 2023-10-01T10:08:50.239291+0000 mon.a (mon.0) 298 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T10:08:51.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:50 smithi078 bash[27336]: cluster 2023-10-01T10:08:49.775216+0000 mgr.y (mgr.14146) 115 : cluster [DBG] pgmap v90: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-01T10:08:51.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:50 smithi078 bash[27336]: audit 2023-10-01T10:08:50.227036+0000 mon.a (mon.0) 295 : audit [INF] from='client.? 172.21.15.78:0/454492136' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9f830594-31d6-4994-85fa-0615c280d5c3"}]: dispatch 2023-10-01T10:08:51.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:50 smithi078 bash[27336]: audit 2023-10-01T10:08:50.239099+0000 mon.a (mon.0) 296 : audit [INF] from='client.? 172.21.15.78:0/454492136' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9f830594-31d6-4994-85fa-0615c280d5c3"}]': finished 2023-10-01T10:08:51.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:50 smithi078 bash[27336]: cluster 2023-10-01T10:08:50.239157+0000 mon.a (mon.0) 297 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2023-10-01T10:08:51.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:50 smithi078 bash[27336]: audit 2023-10-01T10:08:50.239291+0000 mon.a (mon.0) 298 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T10:08:51.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:50 smithi078 bash[33821]: cluster 2023-10-01T10:08:49.775216+0000 mgr.y (mgr.14146) 115 : cluster [DBG] pgmap v90: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-01T10:08:51.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:50 smithi078 bash[33821]: audit 2023-10-01T10:08:50.227036+0000 mon.a (mon.0) 295 : audit [INF] from='client.? 172.21.15.78:0/454492136' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9f830594-31d6-4994-85fa-0615c280d5c3"}]: dispatch 2023-10-01T10:08:51.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:50 smithi078 bash[33821]: audit 2023-10-01T10:08:50.239099+0000 mon.a (mon.0) 296 : audit [INF] from='client.? 172.21.15.78:0/454492136' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9f830594-31d6-4994-85fa-0615c280d5c3"}]': finished 2023-10-01T10:08:51.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:50 smithi078 bash[33821]: cluster 2023-10-01T10:08:50.239157+0000 mon.a (mon.0) 297 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2023-10-01T10:08:51.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:50 smithi078 bash[33821]: audit 2023-10-01T10:08:50.239291+0000 mon.a (mon.0) 298 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T10:08:52.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:51 smithi173 bash[27612]: audit 2023-10-01T10:08:50.972207+0000 mon.a (mon.0) 299 : audit [DBG] from='client.? 172.21.15.78:0/1588636617' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T10:08:52.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:51 smithi078 bash[27336]: audit 2023-10-01T10:08:50.972207+0000 mon.a (mon.0) 299 : audit [DBG] from='client.? 172.21.15.78:0/1588636617' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T10:08:52.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:51 smithi078 bash[33821]: audit 2023-10-01T10:08:50.972207+0000 mon.a (mon.0) 299 : audit [DBG] from='client.? 172.21.15.78:0/1588636617' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T10:08:53.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:52 smithi173 bash[27612]: cluster 2023-10-01T10:08:51.775765+0000 mgr.y (mgr.14146) 116 : cluster [DBG] pgmap v92: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-01T10:08:53.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:52 smithi078 bash[27336]: cluster 2023-10-01T10:08:51.775765+0000 mgr.y (mgr.14146) 116 : cluster [DBG] pgmap v92: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-01T10:08:53.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:52 smithi078 bash[33821]: cluster 2023-10-01T10:08:51.775765+0000 mgr.y (mgr.14146) 116 : cluster [DBG] pgmap v92: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-01T10:08:55.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:54 smithi173 bash[27612]: cluster 2023-10-01T10:08:53.776126+0000 mgr.y (mgr.14146) 117 : cluster [DBG] pgmap v93: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-01T10:08:55.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:54 smithi078 bash[27336]: cluster 2023-10-01T10:08:53.776126+0000 mgr.y (mgr.14146) 117 : cluster [DBG] pgmap v93: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-01T10:08:55.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:54 smithi078 bash[33821]: cluster 2023-10-01T10:08:53.776126+0000 mgr.y (mgr.14146) 117 : cluster [DBG] pgmap v93: 0 pgs: ; 0 B data, 580 MiB used, 178 GiB / 179 GiB avail 2023-10-01T10:08:57.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:56 smithi078 bash[27336]: cluster 2023-10-01T10:08:55.776574+0000 mgr.y (mgr.14146) 118 : cluster [DBG] pgmap v94: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-01T10:08:57.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:56 smithi078 bash[33821]: cluster 2023-10-01T10:08:55.776574+0000 mgr.y (mgr.14146) 118 : cluster [DBG] pgmap v94: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-01T10:08:57.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:56 smithi173 bash[27612]: cluster 2023-10-01T10:08:55.776574+0000 mgr.y (mgr.14146) 118 : cluster [DBG] pgmap v94: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-01T10:08:59.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:08:58 smithi078 bash[27336]: cluster 2023-10-01T10:08:57.777066+0000 mgr.y (mgr.14146) 119 : cluster [DBG] pgmap v95: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-01T10:08:59.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:08:58 smithi078 bash[33821]: cluster 2023-10-01T10:08:57.777066+0000 mgr.y (mgr.14146) 119 : cluster [DBG] pgmap v95: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-01T10:08:59.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:08:58 smithi173 bash[27612]: cluster 2023-10-01T10:08:57.777066+0000 mgr.y (mgr.14146) 119 : cluster [DBG] pgmap v95: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-01T10:09:01.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:00 smithi078 bash[27336]: cluster 2023-10-01T10:08:59.777591+0000 mgr.y (mgr.14146) 120 : cluster [DBG] pgmap v96: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-01T10:09:01.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:00 smithi078 bash[33821]: cluster 2023-10-01T10:08:59.777591+0000 mgr.y (mgr.14146) 120 : cluster [DBG] pgmap v96: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-01T10:09:01.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:00 smithi173 bash[27612]: cluster 2023-10-01T10:08:59.777591+0000 mgr.y (mgr.14146) 120 : cluster [DBG] pgmap v96: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-01T10:09:03.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:02 smithi078 bash[27336]: cluster 2023-10-01T10:09:01.778057+0000 mgr.y (mgr.14146) 121 : cluster [DBG] pgmap v97: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-01T10:09:03.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:02 smithi078 bash[27336]: audit 2023-10-01T10:09:02.633125+0000 mon.a (mon.0) 300 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-01T10:09:03.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:02 smithi078 bash[27336]: audit 2023-10-01T10:09:02.634760+0000 mon.a (mon.0) 301 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:09:03.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:02 smithi078 bash[33821]: cluster 2023-10-01T10:09:01.778057+0000 mgr.y (mgr.14146) 121 : cluster [DBG] pgmap v97: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-01T10:09:03.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:02 smithi078 bash[33821]: audit 2023-10-01T10:09:02.633125+0000 mon.a (mon.0) 300 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-01T10:09:03.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:02 smithi078 bash[33821]: audit 2023-10-01T10:09:02.634760+0000 mon.a (mon.0) 301 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:09:03.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:02 smithi173 bash[27612]: cluster 2023-10-01T10:09:01.778057+0000 mgr.y (mgr.14146) 121 : cluster [DBG] pgmap v97: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-01T10:09:03.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:02 smithi173 bash[27612]: audit 2023-10-01T10:09:02.633125+0000 mon.a (mon.0) 300 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-01T10:09:03.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:02 smithi173 bash[27612]: audit 2023-10-01T10:09:02.634760+0000 mon.a (mon.0) 301 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:09:04.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:03 smithi078 bash[33821]: cephadm 2023-10-01T10:09:02.636086+0000 mgr.y (mgr.14146) 122 : cephadm [INF] Deploying daemon osd.2 on smithi078 2023-10-01T10:09:04.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:03 smithi078 bash[27336]: cephadm 2023-10-01T10:09:02.636086+0000 mgr.y (mgr.14146) 122 : cephadm [INF] Deploying daemon osd.2 on smithi078 2023-10-01T10:09:04.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:03 smithi173 bash[27612]: cephadm 2023-10-01T10:09:02.636086+0000 mgr.y (mgr.14146) 122 : cephadm [INF] Deploying daemon osd.2 on smithi078 2023-10-01T10:09:05.315 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:04 smithi078 bash[27336]: cluster 2023-10-01T10:09:03.778514+0000 mgr.y (mgr.14146) 123 : cluster [DBG] pgmap v98: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-01T10:09:05.315 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:04 smithi078 bash[33821]: cluster 2023-10-01T10:09:03.778514+0000 mgr.y (mgr.14146) 123 : cluster [DBG] pgmap v98: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-01T10:09:05.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:04 smithi173 bash[27612]: cluster 2023-10-01T10:09:03.778514+0000 mgr.y (mgr.14146) 123 : cluster [DBG] pgmap v98: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-01T10:09:07.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:06 smithi078 bash[27336]: cluster 2023-10-01T10:09:05.779085+0000 mgr.y (mgr.14146) 124 : cluster [DBG] pgmap v99: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-01T10:09:07.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:06 smithi078 bash[27336]: audit 2023-10-01T10:09:06.858474+0000 mon.a (mon.0) 302 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:09:07.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:06 smithi078 bash[27336]: audit 2023-10-01T10:09:06.882626+0000 mon.a (mon.0) 303 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:09:07.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:06 smithi078 bash[33821]: cluster 2023-10-01T10:09:05.779085+0000 mgr.y (mgr.14146) 124 : cluster [DBG] pgmap v99: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-01T10:09:07.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:06 smithi078 bash[33821]: audit 2023-10-01T10:09:06.858474+0000 mon.a (mon.0) 302 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:09:07.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:06 smithi078 bash[33821]: audit 2023-10-01T10:09:06.882626+0000 mon.a (mon.0) 303 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:09:07.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:06 smithi173 bash[27612]: cluster 2023-10-01T10:09:05.779085+0000 mgr.y (mgr.14146) 124 : cluster [DBG] pgmap v99: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-01T10:09:07.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:06 smithi173 bash[27612]: audit 2023-10-01T10:09:06.858474+0000 mon.a (mon.0) 302 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:09:07.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:06 smithi173 bash[27612]: audit 2023-10-01T10:09:06.882626+0000 mon.a (mon.0) 303 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:09:09.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:08 smithi078 bash[33821]: cluster 2023-10-01T10:09:07.779542+0000 mgr.y (mgr.14146) 125 : cluster [DBG] pgmap v100: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-01T10:09:09.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:08 smithi078 bash[27336]: cluster 2023-10-01T10:09:07.779542+0000 mgr.y (mgr.14146) 125 : cluster [DBG] pgmap v100: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-01T10:09:09.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:08 smithi173 bash[27612]: cluster 2023-10-01T10:09:07.779542+0000 mgr.y (mgr.14146) 125 : cluster [DBG] pgmap v100: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-01T10:09:11.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:10 smithi078 bash[27336]: cluster 2023-10-01T10:09:09.780064+0000 mgr.y (mgr.14146) 126 : cluster [DBG] pgmap v101: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-01T10:09:11.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:10 smithi078 bash[33821]: cluster 2023-10-01T10:09:09.780064+0000 mgr.y (mgr.14146) 126 : cluster [DBG] pgmap v101: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-01T10:09:11.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:10 smithi173 bash[27612]: cluster 2023-10-01T10:09:09.780064+0000 mgr.y (mgr.14146) 126 : cluster [DBG] pgmap v101: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-01T10:09:13.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:12 smithi078 bash[33821]: cluster 2023-10-01T10:09:11.780534+0000 mgr.y (mgr.14146) 127 : cluster [DBG] pgmap v102: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-01T10:09:13.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:12 smithi078 bash[27336]: cluster 2023-10-01T10:09:11.780534+0000 mgr.y (mgr.14146) 127 : cluster [DBG] pgmap v102: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-01T10:09:13.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:12 smithi173 bash[27612]: cluster 2023-10-01T10:09:11.780534+0000 mgr.y (mgr.14146) 127 : cluster [DBG] pgmap v102: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-01T10:09:14.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:14 smithi078 bash[27336]: audit 2023-10-01T10:09:13.190851+0000 mon.a (mon.0) 304 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:09:14.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:14 smithi078 bash[27336]: audit 2023-10-01T10:09:13.192094+0000 mon.a (mon.0) 305 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:09:14.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:14 smithi078 bash[27336]: audit 2023-10-01T10:09:13.193378+0000 mon.a (mon.0) 306 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:09:14.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:14 smithi078 bash[27336]: audit 2023-10-01T10:09:13.202106+0000 mon.a (mon.0) 307 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:09:14.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:14 smithi078 bash[27336]: audit 2023-10-01T10:09:13.846988+0000 mon.c (mon.2) 9 : audit [INF] from='osd.2 [v2:172.21.15.78:6818/1190788353,v1:172.21.15.78:6819/1190788353]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-01T10:09:14.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:14 smithi078 bash[27336]: audit 2023-10-01T10:09:13.847383+0000 mon.a (mon.0) 308 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-01T10:09:14.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:14 smithi078 bash[33821]: audit 2023-10-01T10:09:13.190851+0000 mon.a (mon.0) 304 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:09:14.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:14 smithi078 bash[33821]: audit 2023-10-01T10:09:13.192094+0000 mon.a (mon.0) 305 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:09:14.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:14 smithi078 bash[33821]: audit 2023-10-01T10:09:13.193378+0000 mon.a (mon.0) 306 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:09:14.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:14 smithi078 bash[33821]: audit 2023-10-01T10:09:13.202106+0000 mon.a (mon.0) 307 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:09:14.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:14 smithi078 bash[33821]: audit 2023-10-01T10:09:13.846988+0000 mon.c (mon.2) 9 : audit [INF] from='osd.2 [v2:172.21.15.78:6818/1190788353,v1:172.21.15.78:6819/1190788353]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-01T10:09:14.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:14 smithi078 bash[33821]: audit 2023-10-01T10:09:13.847383+0000 mon.a (mon.0) 308 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-01T10:09:14.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:14 smithi173 bash[27612]: audit 2023-10-01T10:09:13.190851+0000 mon.a (mon.0) 304 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:09:14.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:14 smithi173 bash[27612]: audit 2023-10-01T10:09:13.192094+0000 mon.a (mon.0) 305 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:09:14.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:14 smithi173 bash[27612]: audit 2023-10-01T10:09:13.193378+0000 mon.a (mon.0) 306 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:09:14.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:14 smithi173 bash[27612]: audit 2023-10-01T10:09:13.202106+0000 mon.a (mon.0) 307 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:09:14.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:14 smithi173 bash[27612]: audit 2023-10-01T10:09:13.846988+0000 mon.c (mon.2) 9 : audit [INF] from='osd.2 [v2:172.21.15.78:6818/1190788353,v1:172.21.15.78:6819/1190788353]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-01T10:09:14.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:14 smithi173 bash[27612]: audit 2023-10-01T10:09:13.847383+0000 mon.a (mon.0) 308 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-01T10:09:15.482 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:15 smithi078 bash[27336]: cluster 2023-10-01T10:09:13.781064+0000 mgr.y (mgr.14146) 128 : cluster [DBG] pgmap v103: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-01T10:09:15.482 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:15 smithi078 bash[27336]: audit 2023-10-01T10:09:14.211125+0000 mon.a (mon.0) 309 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-01T10:09:15.482 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:15 smithi078 bash[27336]: cluster 2023-10-01T10:09:14.211223+0000 mon.a (mon.0) 310 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2023-10-01T10:09:15.482 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:15 smithi078 bash[27336]: audit 2023-10-01T10:09:14.211463+0000 mon.a (mon.0) 311 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T10:09:15.483 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:15 smithi078 bash[27336]: audit 2023-10-01T10:09:14.211969+0000 mon.c (mon.2) 10 : audit [INF] from='osd.2 [v2:172.21.15.78:6818/1190788353,v1:172.21.15.78:6819/1190788353]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2023-10-01T10:09:15.483 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:15 smithi078 bash[27336]: audit 2023-10-01T10:09:14.212486+0000 mon.a (mon.0) 312 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2023-10-01T10:09:15.483 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:15 smithi078 bash[33821]: cluster 2023-10-01T10:09:13.781064+0000 mgr.y (mgr.14146) 128 : cluster [DBG] pgmap v103: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-01T10:09:15.483 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:15 smithi078 bash[33821]: audit 2023-10-01T10:09:14.211125+0000 mon.a (mon.0) 309 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-01T10:09:15.483 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:15 smithi078 bash[33821]: cluster 2023-10-01T10:09:14.211223+0000 mon.a (mon.0) 310 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2023-10-01T10:09:15.484 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:15 smithi078 bash[33821]: audit 2023-10-01T10:09:14.211463+0000 mon.a (mon.0) 311 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T10:09:15.484 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:15 smithi078 bash[33821]: audit 2023-10-01T10:09:14.211969+0000 mon.c (mon.2) 10 : audit [INF] from='osd.2 [v2:172.21.15.78:6818/1190788353,v1:172.21.15.78:6819/1190788353]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2023-10-01T10:09:15.484 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:15 smithi078 bash[33821]: audit 2023-10-01T10:09:14.212486+0000 mon.a (mon.0) 312 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2023-10-01T10:09:15.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:15 smithi173 bash[27612]: cluster 2023-10-01T10:09:13.781064+0000 mgr.y (mgr.14146) 128 : cluster [DBG] pgmap v103: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-01T10:09:15.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:15 smithi173 bash[27612]: audit 2023-10-01T10:09:14.211125+0000 mon.a (mon.0) 309 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-01T10:09:15.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:15 smithi173 bash[27612]: cluster 2023-10-01T10:09:14.211223+0000 mon.a (mon.0) 310 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2023-10-01T10:09:15.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:15 smithi173 bash[27612]: audit 2023-10-01T10:09:14.211463+0000 mon.a (mon.0) 311 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T10:09:15.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:15 smithi173 bash[27612]: audit 2023-10-01T10:09:14.211969+0000 mon.c (mon.2) 10 : audit [INF] from='osd.2 [v2:172.21.15.78:6818/1190788353,v1:172.21.15.78:6819/1190788353]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2023-10-01T10:09:15.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:15 smithi173 bash[27612]: audit 2023-10-01T10:09:14.212486+0000 mon.a (mon.0) 312 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2023-10-01T10:09:16.728 INFO:teuthology.orchestra.run.smithi078.stdout:Created osd(s) 2 on host 'smithi078' 2023-10-01T10:09:16.986 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:16 smithi078 bash[27336]: audit 2023-10-01T10:09:15.216592+0000 mon.a (mon.0) 313 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2023-10-01T10:09:16.986 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:16 smithi078 bash[27336]: cluster 2023-10-01T10:09:15.216764+0000 mon.a (mon.0) 314 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2023-10-01T10:09:16.986 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:16 smithi078 bash[27336]: audit 2023-10-01T10:09:15.217484+0000 mon.a (mon.0) 315 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T10:09:16.986 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:16 smithi078 bash[27336]: audit 2023-10-01T10:09:15.223318+0000 mon.a (mon.0) 316 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T10:09:16.987 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:16 smithi078 bash[33821]: audit 2023-10-01T10:09:15.216592+0000 mon.a (mon.0) 313 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2023-10-01T10:09:16.987 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:16 smithi078 bash[33821]: cluster 2023-10-01T10:09:15.216764+0000 mon.a (mon.0) 314 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2023-10-01T10:09:16.987 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:16 smithi078 bash[33821]: audit 2023-10-01T10:09:15.217484+0000 mon.a (mon.0) 315 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T10:09:16.987 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:16 smithi078 bash[33821]: audit 2023-10-01T10:09:15.223318+0000 mon.a (mon.0) 316 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T10:09:17.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:16 smithi173 bash[27612]: audit 2023-10-01T10:09:15.216592+0000 mon.a (mon.0) 313 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2023-10-01T10:09:17.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:16 smithi173 bash[27612]: cluster 2023-10-01T10:09:15.216764+0000 mon.a (mon.0) 314 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2023-10-01T10:09:17.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:16 smithi173 bash[27612]: audit 2023-10-01T10:09:15.217484+0000 mon.a (mon.0) 315 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T10:09:17.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:16 smithi173 bash[27612]: audit 2023-10-01T10:09:15.223318+0000 mon.a (mon.0) 316 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T10:09:17.520 DEBUG:teuthology.orchestra.run.smithi078:osd.2> sudo journalctl -f -n 0 -u ceph-d8016480-6041-11ee-8db5-212e2dc638e7@osd.2.service 2023-10-01T10:09:17.523 INFO:tasks.cephadm:Deploying osd.3 on smithi078 with /dev/vg_nvme/lv_1... 2023-10-01T10:09:17.523 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-10-01T10:09:17.749 INFO:journalctl@ceph.osd.2.smithi078.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-01T10:09:17.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:17 smithi078 bash[33821]: cluster 2023-10-01T10:09:14.866781+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-01T10:09:17.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:17 smithi078 bash[33821]: cluster 2023-10-01T10:09:14.866863+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-01T10:09:17.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:17 smithi078 bash[33821]: cluster 2023-10-01T10:09:15.781559+0000 mgr.y (mgr.14146) 129 : cluster [DBG] pgmap v106: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-01T10:09:17.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:17 smithi078 bash[33821]: cluster 2023-10-01T10:09:16.227356+0000 mon.a (mon.0) 317 : cluster [INF] osd.2 [v2:172.21.15.78:6818/1190788353,v1:172.21.15.78:6819/1190788353] boot 2023-10-01T10:09:17.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:17 smithi078 bash[33821]: cluster 2023-10-01T10:09:16.227432+0000 mon.a (mon.0) 318 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2023-10-01T10:09:17.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:17 smithi078 bash[33821]: audit 2023-10-01T10:09:16.227977+0000 mon.a (mon.0) 319 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T10:09:17.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:17 smithi078 bash[33821]: audit 2023-10-01T10:09:16.228472+0000 mon.a (mon.0) 320 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-01T10:09:17.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:17 smithi078 bash[33821]: audit 2023-10-01T10:09:16.726132+0000 mon.a (mon.0) 321 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:09:17.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:17 smithi078 bash[33821]: audit 2023-10-01T10:09:16.727766+0000 mon.a (mon.0) 322 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:09:17.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:17 smithi078 bash[33821]: audit 2023-10-01T10:09:17.227492+0000 mon.a (mon.0) 323 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-10-01T10:09:17.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:17 smithi078 bash[33821]: cluster 2023-10-01T10:09:17.227587+0000 mon.a (mon.0) 324 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2023-10-01T10:09:17.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:17 smithi078 bash[33821]: audit 2023-10-01T10:09:17.228496+0000 mon.a (mon.0) 325 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-10-01T10:09:17.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:17 smithi078 bash[27336]: cluster 2023-10-01T10:09:14.866781+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-01T10:09:17.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:17 smithi078 bash[27336]: cluster 2023-10-01T10:09:14.866863+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-01T10:09:17.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:17 smithi078 bash[27336]: cluster 2023-10-01T10:09:15.781559+0000 mgr.y (mgr.14146) 129 : cluster [DBG] pgmap v106: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-01T10:09:17.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:17 smithi078 bash[27336]: cluster 2023-10-01T10:09:16.227356+0000 mon.a (mon.0) 317 : cluster [INF] osd.2 [v2:172.21.15.78:6818/1190788353,v1:172.21.15.78:6819/1190788353] boot 2023-10-01T10:09:17.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:17 smithi078 bash[27336]: cluster 2023-10-01T10:09:16.227432+0000 mon.a (mon.0) 318 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2023-10-01T10:09:17.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:17 smithi078 bash[27336]: audit 2023-10-01T10:09:16.227977+0000 mon.a (mon.0) 319 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T10:09:17.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:17 smithi078 bash[27336]: audit 2023-10-01T10:09:16.228472+0000 mon.a (mon.0) 320 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-01T10:09:17.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:17 smithi078 bash[27336]: audit 2023-10-01T10:09:16.726132+0000 mon.a (mon.0) 321 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:09:17.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:17 smithi078 bash[27336]: audit 2023-10-01T10:09:16.727766+0000 mon.a (mon.0) 322 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:09:17.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:17 smithi078 bash[27336]: audit 2023-10-01T10:09:17.227492+0000 mon.a (mon.0) 323 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-10-01T10:09:17.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:17 smithi078 bash[27336]: cluster 2023-10-01T10:09:17.227587+0000 mon.a (mon.0) 324 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2023-10-01T10:09:17.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:17 smithi078 bash[27336]: audit 2023-10-01T10:09:17.228496+0000 mon.a (mon.0) 325 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-10-01T10:09:18.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:17 smithi173 bash[27612]: cluster 2023-10-01T10:09:14.866781+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-01T10:09:18.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:17 smithi173 bash[27612]: cluster 2023-10-01T10:09:14.866863+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-01T10:09:18.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:17 smithi173 bash[27612]: cluster 2023-10-01T10:09:15.781559+0000 mgr.y (mgr.14146) 129 : cluster [DBG] pgmap v106: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2023-10-01T10:09:18.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:17 smithi173 bash[27612]: cluster 2023-10-01T10:09:16.227356+0000 mon.a (mon.0) 317 : cluster [INF] osd.2 [v2:172.21.15.78:6818/1190788353,v1:172.21.15.78:6819/1190788353] boot 2023-10-01T10:09:18.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:17 smithi173 bash[27612]: cluster 2023-10-01T10:09:16.227432+0000 mon.a (mon.0) 318 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2023-10-01T10:09:18.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:17 smithi173 bash[27612]: audit 2023-10-01T10:09:16.227977+0000 mon.a (mon.0) 319 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T10:09:18.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:17 smithi173 bash[27612]: audit 2023-10-01T10:09:16.228472+0000 mon.a (mon.0) 320 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-01T10:09:18.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:17 smithi173 bash[27612]: audit 2023-10-01T10:09:16.726132+0000 mon.a (mon.0) 321 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:09:18.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:17 smithi173 bash[27612]: audit 2023-10-01T10:09:16.727766+0000 mon.a (mon.0) 322 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:09:18.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:17 smithi173 bash[27612]: audit 2023-10-01T10:09:17.227492+0000 mon.a (mon.0) 323 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-10-01T10:09:18.122 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:17 smithi173 bash[27612]: cluster 2023-10-01T10:09:17.227587+0000 mon.a (mon.0) 324 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2023-10-01T10:09:18.122 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:17 smithi173 bash[27612]: audit 2023-10-01T10:09:17.228496+0000 mon.a (mon.0) 325 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-10-01T10:09:19.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:19 smithi078 bash[33821]: cluster 2023-10-01T10:09:17.782073+0000 mgr.y (mgr.14146) 130 : cluster [DBG] pgmap v109: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:09:19.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:19 smithi078 bash[33821]: audit 2023-10-01T10:09:18.230511+0000 mon.a (mon.0) 326 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-10-01T10:09:19.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:19 smithi078 bash[33821]: cluster 2023-10-01T10:09:18.230763+0000 mon.a (mon.0) 327 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2023-10-01T10:09:19.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:19 smithi078 bash[27336]: cluster 2023-10-01T10:09:17.782073+0000 mgr.y (mgr.14146) 130 : cluster [DBG] pgmap v109: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:09:19.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:19 smithi078 bash[27336]: audit 2023-10-01T10:09:18.230511+0000 mon.a (mon.0) 326 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-10-01T10:09:19.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:19 smithi078 bash[27336]: cluster 2023-10-01T10:09:18.230763+0000 mon.a (mon.0) 327 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2023-10-01T10:09:19.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:19 smithi173 bash[27612]: cluster 2023-10-01T10:09:17.782073+0000 mgr.y (mgr.14146) 130 : cluster [DBG] pgmap v109: 1 pgs: 1 unknown; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:09:19.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:19 smithi173 bash[27612]: audit 2023-10-01T10:09:18.230511+0000 mon.a (mon.0) 326 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-10-01T10:09:19.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:19 smithi173 bash[27612]: cluster 2023-10-01T10:09:18.230763+0000 mon.a (mon.0) 327 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2023-10-01T10:09:21.057 INFO:teuthology.orchestra.run.smithi078.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-10-01T10:09:21.057 INFO:teuthology.orchestra.run.smithi078.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-10-01T10:09:21.057 INFO:teuthology.orchestra.run.smithi078.stderr: stderr: 10+0 records in 2023-10-01T10:09:21.057 INFO:teuthology.orchestra.run.smithi078.stderr:10+0 records out 2023-10-01T10:09:21.057 INFO:teuthology.orchestra.run.smithi078.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0172526 s, 608 MB/s 2023-10-01T10:09:21.057 INFO:teuthology.orchestra.run.smithi078.stderr:--> Zapping successful for: 2023-10-01T10:09:21.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:20 smithi078 bash[27336]: cluster 2023-10-01T10:09:19.699052+0000 mon.a (mon.0) 328 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2023-10-01T10:09:21.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:20 smithi078 bash[27336]: cluster 2023-10-01T10:09:19.782618+0000 mgr.y (mgr.14146) 131 : cluster [DBG] pgmap v112: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:09:21.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:20 smithi078 bash[33821]: cluster 2023-10-01T10:09:19.699052+0000 mon.a (mon.0) 328 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2023-10-01T10:09:21.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:20 smithi078 bash[33821]: cluster 2023-10-01T10:09:19.782618+0000 mgr.y (mgr.14146) 131 : cluster [DBG] pgmap v112: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:09:21.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:20 smithi173 bash[27612]: cluster 2023-10-01T10:09:19.699052+0000 mon.a (mon.0) 328 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2023-10-01T10:09:21.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:20 smithi173 bash[27612]: cluster 2023-10-01T10:09:19.782618+0000 mgr.y (mgr.14146) 131 : cluster [DBG] pgmap v112: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:09:21.841 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi078:vg_nvme/lv_1 2023-10-01T10:09:23.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:22 smithi173 bash[27612]: cluster 2023-10-01T10:09:21.783150+0000 mgr.y (mgr.14146) 132 : cluster [DBG] pgmap v113: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:09:23.302 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:22 smithi078 bash[27336]: cluster 2023-10-01T10:09:21.783150+0000 mgr.y (mgr.14146) 132 : cluster [DBG] pgmap v113: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:09:23.302 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:22 smithi078 bash[33821]: cluster 2023-10-01T10:09:21.783150+0000 mgr.y (mgr.14146) 132 : cluster [DBG] pgmap v113: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:09:25.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:24 smithi173 bash[27612]: cluster 2023-10-01T10:09:23.783666+0000 mgr.y (mgr.14146) 133 : cluster [DBG] pgmap v114: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:09:25.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:24 smithi078 bash[27336]: cluster 2023-10-01T10:09:23.783666+0000 mgr.y (mgr.14146) 133 : cluster [DBG] pgmap v114: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:09:25.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:24 smithi078 bash[33821]: cluster 2023-10-01T10:09:23.783666+0000 mgr.y (mgr.14146) 133 : cluster [DBG] pgmap v114: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:09:27.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:26 smithi173 bash[27612]: cluster 2023-10-01T10:09:25.784233+0000 mgr.y (mgr.14146) 134 : cluster [DBG] pgmap v115: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:09:27.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:26 smithi173 bash[27612]: audit 2023-10-01T10:09:26.522710+0000 mon.a (mon.0) 329 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T10:09:27.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:26 smithi173 bash[27612]: audit 2023-10-01T10:09:26.528535+0000 mon.a (mon.0) 330 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T10:09:27.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:26 smithi173 bash[27612]: audit 2023-10-01T10:09:26.529731+0000 mon.a (mon.0) 331 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:09:27.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:26 smithi078 bash[27336]: cluster 2023-10-01T10:09:25.784233+0000 mgr.y (mgr.14146) 134 : cluster [DBG] pgmap v115: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:09:27.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:26 smithi078 bash[27336]: audit 2023-10-01T10:09:26.522710+0000 mon.a (mon.0) 329 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T10:09:27.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:26 smithi078 bash[27336]: audit 2023-10-01T10:09:26.528535+0000 mon.a (mon.0) 330 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T10:09:27.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:26 smithi078 bash[27336]: audit 2023-10-01T10:09:26.529731+0000 mon.a (mon.0) 331 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:09:27.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:26 smithi078 bash[33821]: cluster 2023-10-01T10:09:25.784233+0000 mgr.y (mgr.14146) 134 : cluster [DBG] pgmap v115: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:09:27.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:26 smithi078 bash[33821]: audit 2023-10-01T10:09:26.522710+0000 mon.a (mon.0) 329 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T10:09:27.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:26 smithi078 bash[33821]: audit 2023-10-01T10:09:26.528535+0000 mon.a (mon.0) 330 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T10:09:27.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:26 smithi078 bash[33821]: audit 2023-10-01T10:09:26.529731+0000 mon.a (mon.0) 331 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:09:28.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:27 smithi078 bash[27336]: audit 2023-10-01T10:09:26.519742+0000 mgr.y (mgr.14146) 135 : audit [DBG] from='client.24200 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:09:28.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:27 smithi078 bash[33821]: audit 2023-10-01T10:09:26.519742+0000 mgr.y (mgr.14146) 135 : audit [DBG] from='client.24200 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:09:28.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:27 smithi173 bash[27612]: audit 2023-10-01T10:09:26.519742+0000 mgr.y (mgr.14146) 135 : audit [DBG] from='client.24200 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:09:29.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:28 smithi078 bash[27336]: cluster 2023-10-01T10:09:27.784844+0000 mgr.y (mgr.14146) 136 : cluster [DBG] pgmap v116: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:09:29.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:28 smithi078 bash[27336]: cephadm 2023-10-01T10:09:27.884020+0000 mgr.y (mgr.14146) 137 : cephadm [INF] Detected new or changed devices on smithi078 2023-10-01T10:09:29.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:28 smithi078 bash[27336]: audit 2023-10-01T10:09:27.894700+0000 mon.a (mon.0) 332 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:09:29.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:28 smithi078 bash[27336]: audit 2023-10-01T10:09:27.896387+0000 mon.a (mon.0) 333 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:09:29.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:28 smithi078 bash[27336]: audit 2023-10-01T10:09:27.897574+0000 mon.a (mon.0) 334 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:09:29.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:28 smithi078 bash[27336]: audit 2023-10-01T10:09:27.898737+0000 mon.a (mon.0) 335 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:09:29.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:28 smithi078 bash[27336]: cephadm 2023-10-01T10:09:27.899647+0000 mgr.y (mgr.14146) 138 : cephadm [INF] Adjusting osd_memory_target on smithi078 to 5413M 2023-10-01T10:09:29.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:28 smithi078 bash[27336]: audit 2023-10-01T10:09:27.905063+0000 mon.a (mon.0) 336 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:09:29.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:28 smithi078 bash[27336]: audit 2023-10-01T10:09:27.907002+0000 mon.a (mon.0) 337 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:09:29.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:28 smithi078 bash[27336]: audit 2023-10-01T10:09:27.907938+0000 mon.a (mon.0) 338 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:09:29.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:28 smithi078 bash[27336]: audit 2023-10-01T10:09:27.914260+0000 mon.a (mon.0) 339 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:09:29.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:28 smithi078 bash[33821]: cluster 2023-10-01T10:09:27.784844+0000 mgr.y (mgr.14146) 136 : cluster [DBG] pgmap v116: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:09:29.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:28 smithi078 bash[33821]: cephadm 2023-10-01T10:09:27.884020+0000 mgr.y (mgr.14146) 137 : cephadm [INF] Detected new or changed devices on smithi078 2023-10-01T10:09:29.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:28 smithi078 bash[33821]: audit 2023-10-01T10:09:27.894700+0000 mon.a (mon.0) 332 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:09:29.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:28 smithi078 bash[33821]: audit 2023-10-01T10:09:27.896387+0000 mon.a (mon.0) 333 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:09:29.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:28 smithi078 bash[33821]: audit 2023-10-01T10:09:27.897574+0000 mon.a (mon.0) 334 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:09:29.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:28 smithi078 bash[33821]: audit 2023-10-01T10:09:27.898737+0000 mon.a (mon.0) 335 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:09:29.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:28 smithi078 bash[33821]: cephadm 2023-10-01T10:09:27.899647+0000 mgr.y (mgr.14146) 138 : cephadm [INF] Adjusting osd_memory_target on smithi078 to 5413M 2023-10-01T10:09:29.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:28 smithi078 bash[33821]: audit 2023-10-01T10:09:27.905063+0000 mon.a (mon.0) 336 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:09:29.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:28 smithi078 bash[33821]: audit 2023-10-01T10:09:27.907002+0000 mon.a (mon.0) 337 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:09:29.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:28 smithi078 bash[33821]: audit 2023-10-01T10:09:27.907938+0000 mon.a (mon.0) 338 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:09:29.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:28 smithi078 bash[33821]: audit 2023-10-01T10:09:27.914260+0000 mon.a (mon.0) 339 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:09:29.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:28 smithi173 bash[27612]: cluster 2023-10-01T10:09:27.784844+0000 mgr.y (mgr.14146) 136 : cluster [DBG] pgmap v116: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:09:29.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:28 smithi173 bash[27612]: cephadm 2023-10-01T10:09:27.884020+0000 mgr.y (mgr.14146) 137 : cephadm [INF] Detected new or changed devices on smithi078 2023-10-01T10:09:29.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:28 smithi173 bash[27612]: audit 2023-10-01T10:09:27.894700+0000 mon.a (mon.0) 332 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:09:29.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:28 smithi173 bash[27612]: audit 2023-10-01T10:09:27.896387+0000 mon.a (mon.0) 333 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:09:29.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:28 smithi173 bash[27612]: audit 2023-10-01T10:09:27.897574+0000 mon.a (mon.0) 334 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:09:29.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:28 smithi173 bash[27612]: audit 2023-10-01T10:09:27.898737+0000 mon.a (mon.0) 335 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:09:29.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:28 smithi173 bash[27612]: cephadm 2023-10-01T10:09:27.899647+0000 mgr.y (mgr.14146) 138 : cephadm [INF] Adjusting osd_memory_target on smithi078 to 5413M 2023-10-01T10:09:29.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:28 smithi173 bash[27612]: audit 2023-10-01T10:09:27.905063+0000 mon.a (mon.0) 336 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:09:29.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:28 smithi173 bash[27612]: audit 2023-10-01T10:09:27.907002+0000 mon.a (mon.0) 337 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:09:29.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:28 smithi173 bash[27612]: audit 2023-10-01T10:09:27.907938+0000 mon.a (mon.0) 338 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:09:29.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:28 smithi173 bash[27612]: audit 2023-10-01T10:09:27.914260+0000 mon.a (mon.0) 339 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:09:31.308 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:30 smithi078 bash[27336]: cluster 2023-10-01T10:09:29.785460+0000 mgr.y (mgr.14146) 139 : cluster [DBG] pgmap v117: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:09:31.308 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:30 smithi078 bash[33821]: cluster 2023-10-01T10:09:29.785460+0000 mgr.y (mgr.14146) 139 : cluster [DBG] pgmap v117: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:09:31.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:30 smithi173 bash[27612]: cluster 2023-10-01T10:09:29.785460+0000 mgr.y (mgr.14146) 139 : cluster [DBG] pgmap v117: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:09:33.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:32 smithi078 bash[33821]: cluster 2023-10-01T10:09:31.786034+0000 mgr.y (mgr.14146) 140 : cluster [DBG] pgmap v118: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:09:33.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:32 smithi078 bash[27336]: cluster 2023-10-01T10:09:31.786034+0000 mgr.y (mgr.14146) 140 : cluster [DBG] pgmap v118: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:09:33.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:32 smithi173 bash[27612]: cluster 2023-10-01T10:09:31.786034+0000 mgr.y (mgr.14146) 140 : cluster [DBG] pgmap v118: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:09:35.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:34 smithi078 bash[27336]: cluster 2023-10-01T10:09:33.786661+0000 mgr.y (mgr.14146) 141 : cluster [DBG] pgmap v119: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:09:35.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:34 smithi078 bash[33821]: cluster 2023-10-01T10:09:33.786661+0000 mgr.y (mgr.14146) 141 : cluster [DBG] pgmap v119: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:09:35.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:34 smithi173 bash[27612]: cluster 2023-10-01T10:09:33.786661+0000 mgr.y (mgr.14146) 141 : cluster [DBG] pgmap v119: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:09:36.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:35 smithi078 bash[27336]: audit 2023-10-01T10:09:34.931218+0000 mon.c (mon.2) 11 : audit [INF] from='client.? 172.21.15.78:0/4086356172' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2780842b-2f4f-4190-b4a5-279a0444adf2"}]: dispatch 2023-10-01T10:09:36.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:35 smithi078 bash[27336]: audit 2023-10-01T10:09:34.931930+0000 mon.a (mon.0) 340 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2780842b-2f4f-4190-b4a5-279a0444adf2"}]: dispatch 2023-10-01T10:09:36.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:35 smithi078 bash[27336]: audit 2023-10-01T10:09:34.937577+0000 mon.a (mon.0) 341 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2780842b-2f4f-4190-b4a5-279a0444adf2"}]': finished 2023-10-01T10:09:36.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:35 smithi078 bash[27336]: cluster 2023-10-01T10:09:34.937647+0000 mon.a (mon.0) 342 : cluster [DBG] osdmap e22: 4 total, 3 up, 4 in 2023-10-01T10:09:36.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:35 smithi078 bash[27336]: audit 2023-10-01T10:09:34.937768+0000 mon.a (mon.0) 343 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T10:09:36.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:35 smithi078 bash[27336]: audit 2023-10-01T10:09:35.692737+0000 mon.a (mon.0) 344 : audit [DBG] from='client.? 172.21.15.78:0/249474074' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T10:09:36.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:35 smithi078 bash[33821]: audit 2023-10-01T10:09:34.931218+0000 mon.c (mon.2) 11 : audit [INF] from='client.? 172.21.15.78:0/4086356172' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2780842b-2f4f-4190-b4a5-279a0444adf2"}]: dispatch 2023-10-01T10:09:36.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:35 smithi078 bash[33821]: audit 2023-10-01T10:09:34.931930+0000 mon.a (mon.0) 340 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2780842b-2f4f-4190-b4a5-279a0444adf2"}]: dispatch 2023-10-01T10:09:36.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:35 smithi078 bash[33821]: audit 2023-10-01T10:09:34.937577+0000 mon.a (mon.0) 341 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2780842b-2f4f-4190-b4a5-279a0444adf2"}]': finished 2023-10-01T10:09:36.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:35 smithi078 bash[33821]: cluster 2023-10-01T10:09:34.937647+0000 mon.a (mon.0) 342 : cluster [DBG] osdmap e22: 4 total, 3 up, 4 in 2023-10-01T10:09:36.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:35 smithi078 bash[33821]: audit 2023-10-01T10:09:34.937768+0000 mon.a (mon.0) 343 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T10:09:36.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:35 smithi078 bash[33821]: audit 2023-10-01T10:09:35.692737+0000 mon.a (mon.0) 344 : audit [DBG] from='client.? 172.21.15.78:0/249474074' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T10:09:36.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:35 smithi173 bash[27612]: audit 2023-10-01T10:09:34.931218+0000 mon.c (mon.2) 11 : audit [INF] from='client.? 172.21.15.78:0/4086356172' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2780842b-2f4f-4190-b4a5-279a0444adf2"}]: dispatch 2023-10-01T10:09:36.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:35 smithi173 bash[27612]: audit 2023-10-01T10:09:34.931930+0000 mon.a (mon.0) 340 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2780842b-2f4f-4190-b4a5-279a0444adf2"}]: dispatch 2023-10-01T10:09:36.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:35 smithi173 bash[27612]: audit 2023-10-01T10:09:34.937577+0000 mon.a (mon.0) 341 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2780842b-2f4f-4190-b4a5-279a0444adf2"}]': finished 2023-10-01T10:09:36.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:35 smithi173 bash[27612]: cluster 2023-10-01T10:09:34.937647+0000 mon.a (mon.0) 342 : cluster [DBG] osdmap e22: 4 total, 3 up, 4 in 2023-10-01T10:09:36.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:35 smithi173 bash[27612]: audit 2023-10-01T10:09:34.937768+0000 mon.a (mon.0) 343 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T10:09:36.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:35 smithi173 bash[27612]: audit 2023-10-01T10:09:35.692737+0000 mon.a (mon.0) 344 : audit [DBG] from='client.? 172.21.15.78:0/249474074' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T10:09:37.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:36 smithi078 bash[27336]: cluster 2023-10-01T10:09:35.787207+0000 mgr.y (mgr.14146) 142 : cluster [DBG] pgmap v121: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:09:37.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:36 smithi078 bash[33821]: cluster 2023-10-01T10:09:35.787207+0000 mgr.y (mgr.14146) 142 : cluster [DBG] pgmap v121: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:09:37.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:36 smithi173 bash[27612]: cluster 2023-10-01T10:09:35.787207+0000 mgr.y (mgr.14146) 142 : cluster [DBG] pgmap v121: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:09:39.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:38 smithi078 bash[27336]: cluster 2023-10-01T10:09:37.787750+0000 mgr.y (mgr.14146) 143 : cluster [DBG] pgmap v122: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:09:39.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:38 smithi078 bash[33821]: cluster 2023-10-01T10:09:37.787750+0000 mgr.y (mgr.14146) 143 : cluster [DBG] pgmap v122: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:09:39.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:38 smithi173 bash[27612]: cluster 2023-10-01T10:09:37.787750+0000 mgr.y (mgr.14146) 143 : cluster [DBG] pgmap v122: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:09:41.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:40 smithi078 bash[33821]: cluster 2023-10-01T10:09:39.788254+0000 mgr.y (mgr.14146) 144 : cluster [DBG] pgmap v123: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:09:41.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:40 smithi078 bash[27336]: cluster 2023-10-01T10:09:39.788254+0000 mgr.y (mgr.14146) 144 : cluster [DBG] pgmap v123: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:09:41.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:40 smithi173 bash[27612]: cluster 2023-10-01T10:09:39.788254+0000 mgr.y (mgr.14146) 144 : cluster [DBG] pgmap v123: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:09:43.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:42 smithi078 bash[27336]: cluster 2023-10-01T10:09:41.788772+0000 mgr.y (mgr.14146) 145 : cluster [DBG] pgmap v124: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:09:43.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:42 smithi078 bash[33821]: cluster 2023-10-01T10:09:41.788772+0000 mgr.y (mgr.14146) 145 : cluster [DBG] pgmap v124: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:09:43.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:42 smithi173 bash[27612]: cluster 2023-10-01T10:09:41.788772+0000 mgr.y (mgr.14146) 145 : cluster [DBG] pgmap v124: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:09:45.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:44 smithi078 bash[27336]: cluster 2023-10-01T10:09:43.789276+0000 mgr.y (mgr.14146) 146 : cluster [DBG] pgmap v125: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:09:45.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:44 smithi078 bash[33821]: cluster 2023-10-01T10:09:43.789276+0000 mgr.y (mgr.14146) 146 : cluster [DBG] pgmap v125: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:09:45.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:44 smithi173 bash[27612]: cluster 2023-10-01T10:09:43.789276+0000 mgr.y (mgr.14146) 146 : cluster [DBG] pgmap v125: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:09:47.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:46 smithi078 bash[33821]: cluster 2023-10-01T10:09:45.789588+0000 mgr.y (mgr.14146) 147 : cluster [DBG] pgmap v126: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:09:47.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:46 smithi078 bash[27336]: cluster 2023-10-01T10:09:45.789588+0000 mgr.y (mgr.14146) 147 : cluster [DBG] pgmap v126: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:09:47.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:46 smithi173 bash[27612]: cluster 2023-10-01T10:09:45.789588+0000 mgr.y (mgr.14146) 147 : cluster [DBG] pgmap v126: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:09:48.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:47 smithi078 bash[27336]: audit 2023-10-01T10:09:47.378770+0000 mon.a (mon.0) 345 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-01T10:09:48.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:47 smithi078 bash[27336]: audit 2023-10-01T10:09:47.380154+0000 mon.a (mon.0) 346 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:09:48.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:47 smithi078 bash[33821]: audit 2023-10-01T10:09:47.378770+0000 mon.a (mon.0) 345 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-01T10:09:48.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:47 smithi078 bash[33821]: audit 2023-10-01T10:09:47.380154+0000 mon.a (mon.0) 346 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:09:48.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:47 smithi173 bash[27612]: audit 2023-10-01T10:09:47.378770+0000 mon.a (mon.0) 345 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-01T10:09:48.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:47 smithi173 bash[27612]: audit 2023-10-01T10:09:47.380154+0000 mon.a (mon.0) 346 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:09:49.091 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:48 smithi078 bash[27336]: cephadm 2023-10-01T10:09:47.381211+0000 mgr.y (mgr.14146) 148 : cephadm [INF] Deploying daemon osd.3 on smithi078 2023-10-01T10:09:49.091 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:48 smithi078 bash[27336]: cluster 2023-10-01T10:09:47.790106+0000 mgr.y (mgr.14146) 149 : cluster [DBG] pgmap v127: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:09:49.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:49 smithi078 bash[33821]: cephadm 2023-10-01T10:09:47.381211+0000 mgr.y (mgr.14146) 148 : cephadm [INF] Deploying daemon osd.3 on smithi078 2023-10-01T10:09:49.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:49 smithi078 bash[33821]: cluster 2023-10-01T10:09:47.790106+0000 mgr.y (mgr.14146) 149 : cluster [DBG] pgmap v127: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:09:49.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:48 smithi173 bash[27612]: cephadm 2023-10-01T10:09:47.381211+0000 mgr.y (mgr.14146) 148 : cephadm [INF] Deploying daemon osd.3 on smithi078 2023-10-01T10:09:49.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:48 smithi173 bash[27612]: cluster 2023-10-01T10:09:47.790106+0000 mgr.y (mgr.14146) 149 : cluster [DBG] pgmap v127: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:09:51.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:50 smithi078 bash[27336]: cluster 2023-10-01T10:09:49.790748+0000 mgr.y (mgr.14146) 150 : cluster [DBG] pgmap v128: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:09:51.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:50 smithi078 bash[33821]: cluster 2023-10-01T10:09:49.790748+0000 mgr.y (mgr.14146) 150 : cluster [DBG] pgmap v128: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:09:51.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:50 smithi173 bash[27612]: cluster 2023-10-01T10:09:49.790748+0000 mgr.y (mgr.14146) 150 : cluster [DBG] pgmap v128: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:09:53.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:52 smithi078 bash[33821]: cluster 2023-10-01T10:09:51.791289+0000 mgr.y (mgr.14146) 151 : cluster [DBG] pgmap v129: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:09:53.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:52 smithi078 bash[33821]: audit 2023-10-01T10:09:51.796859+0000 mon.a (mon.0) 347 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:09:53.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:52 smithi078 bash[33821]: audit 2023-10-01T10:09:51.797234+0000 mon.a (mon.0) 348 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:09:53.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:52 smithi078 bash[27336]: cluster 2023-10-01T10:09:51.791289+0000 mgr.y (mgr.14146) 151 : cluster [DBG] pgmap v129: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:09:53.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:52 smithi078 bash[27336]: audit 2023-10-01T10:09:51.796859+0000 mon.a (mon.0) 347 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:09:53.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:52 smithi078 bash[27336]: audit 2023-10-01T10:09:51.797234+0000 mon.a (mon.0) 348 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:09:53.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:52 smithi173 bash[27612]: cluster 2023-10-01T10:09:51.791289+0000 mgr.y (mgr.14146) 151 : cluster [DBG] pgmap v129: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:09:53.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:52 smithi173 bash[27612]: audit 2023-10-01T10:09:51.796859+0000 mon.a (mon.0) 347 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:09:53.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:52 smithi173 bash[27612]: audit 2023-10-01T10:09:51.797234+0000 mon.a (mon.0) 348 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:09:55.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:54 smithi078 bash[27336]: cluster 2023-10-01T10:09:53.791804+0000 mgr.y (mgr.14146) 152 : cluster [DBG] pgmap v130: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:09:55.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:54 smithi078 bash[33821]: cluster 2023-10-01T10:09:53.791804+0000 mgr.y (mgr.14146) 152 : cluster [DBG] pgmap v130: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:09:55.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:54 smithi173 bash[27612]: cluster 2023-10-01T10:09:53.791804+0000 mgr.y (mgr.14146) 152 : cluster [DBG] pgmap v130: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:09:57.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:56 smithi173 bash[27612]: cluster 2023-10-01T10:09:55.792350+0000 mgr.y (mgr.14146) 153 : cluster [DBG] pgmap v131: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:09:57.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:56 smithi078 bash[27336]: cluster 2023-10-01T10:09:55.792350+0000 mgr.y (mgr.14146) 153 : cluster [DBG] pgmap v131: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:09:57.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:56 smithi078 bash[33821]: cluster 2023-10-01T10:09:55.792350+0000 mgr.y (mgr.14146) 153 : cluster [DBG] pgmap v131: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:09:58.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:58 smithi078 bash[33821]: audit 2023-10-01T10:09:57.032544+0000 mon.a (mon.0) 349 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:09:58.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:58 smithi078 bash[33821]: audit 2023-10-01T10:09:57.286780+0000 mon.a (mon.0) 350 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:09:58.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:58 smithi078 bash[33821]: audit 2023-10-01T10:09:57.288181+0000 mon.a (mon.0) 351 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:09:58.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:58 smithi078 bash[33821]: audit 2023-10-01T10:09:57.296758+0000 mon.a (mon.0) 352 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:09:58.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:58 smithi078 bash[27336]: audit 2023-10-01T10:09:57.032544+0000 mon.a (mon.0) 349 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:09:58.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:58 smithi078 bash[27336]: audit 2023-10-01T10:09:57.286780+0000 mon.a (mon.0) 350 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:09:58.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:58 smithi078 bash[27336]: audit 2023-10-01T10:09:57.288181+0000 mon.a (mon.0) 351 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:09:58.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:58 smithi078 bash[27336]: audit 2023-10-01T10:09:57.296758+0000 mon.a (mon.0) 352 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:09:58.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:58 smithi173 bash[27612]: audit 2023-10-01T10:09:57.032544+0000 mon.a (mon.0) 349 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:09:58.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:58 smithi173 bash[27612]: audit 2023-10-01T10:09:57.286780+0000 mon.a (mon.0) 350 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:09:58.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:58 smithi173 bash[27612]: audit 2023-10-01T10:09:57.288181+0000 mon.a (mon.0) 351 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:09:58.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:58 smithi173 bash[27612]: audit 2023-10-01T10:09:57.296758+0000 mon.a (mon.0) 352 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:09:59.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:59 smithi078 bash[27336]: cluster 2023-10-01T10:09:57.792888+0000 mgr.y (mgr.14146) 154 : cluster [DBG] pgmap v132: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:09:59.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:59 smithi078 bash[27336]: audit 2023-10-01T10:09:58.471009+0000 mon.c (mon.2) 12 : audit [INF] from='osd.3 [v2:172.21.15.78:6826/53937366,v1:172.21.15.78:6827/53937366]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-01T10:09:59.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:09:59 smithi078 bash[27336]: audit 2023-10-01T10:09:58.471568+0000 mon.a (mon.0) 353 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-01T10:09:59.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:59 smithi078 bash[33821]: cluster 2023-10-01T10:09:57.792888+0000 mgr.y (mgr.14146) 154 : cluster [DBG] pgmap v132: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:09:59.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:59 smithi078 bash[33821]: audit 2023-10-01T10:09:58.471009+0000 mon.c (mon.2) 12 : audit [INF] from='osd.3 [v2:172.21.15.78:6826/53937366,v1:172.21.15.78:6827/53937366]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-01T10:09:59.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:09:59 smithi078 bash[33821]: audit 2023-10-01T10:09:58.471568+0000 mon.a (mon.0) 353 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-01T10:09:59.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:59 smithi173 bash[27612]: cluster 2023-10-01T10:09:57.792888+0000 mgr.y (mgr.14146) 154 : cluster [DBG] pgmap v132: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:09:59.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:59 smithi173 bash[27612]: audit 2023-10-01T10:09:58.471009+0000 mon.c (mon.2) 12 : audit [INF] from='osd.3 [v2:172.21.15.78:6826/53937366,v1:172.21.15.78:6827/53937366]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-01T10:09:59.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:09:59 smithi173 bash[27612]: audit 2023-10-01T10:09:58.471568+0000 mon.a (mon.0) 353 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-01T10:10:00.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:00 smithi078 bash[33821]: audit 2023-10-01T10:09:59.306638+0000 mon.a (mon.0) 354 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-01T10:10:00.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:00 smithi078 bash[33821]: cluster 2023-10-01T10:09:59.306750+0000 mon.a (mon.0) 355 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2023-10-01T10:10:00.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:00 smithi078 bash[33821]: audit 2023-10-01T10:09:59.307333+0000 mon.a (mon.0) 356 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T10:10:00.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:00 smithi078 bash[33821]: audit 2023-10-01T10:09:59.309909+0000 mon.c (mon.2) 13 : audit [INF] from='osd.3 [v2:172.21.15.78:6826/53937366,v1:172.21.15.78:6827/53937366]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2023-10-01T10:10:00.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:00 smithi078 bash[33821]: audit 2023-10-01T10:09:59.314749+0000 mon.a (mon.0) 357 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2023-10-01T10:10:00.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:00 smithi078 bash[33821]: cluster 2023-10-01T10:09:59.793520+0000 mgr.y (mgr.14146) 155 : cluster [DBG] pgmap v134: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:10:00.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:00 smithi078 bash[33821]: cluster 2023-10-01T10:10:00.000138+0000 mon.a (mon.0) 358 : cluster [INF] overall HEALTH_OK 2023-10-01T10:10:00.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:00 smithi078 bash[27336]: audit 2023-10-01T10:09:59.306638+0000 mon.a (mon.0) 354 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-01T10:10:00.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:00 smithi078 bash[27336]: cluster 2023-10-01T10:09:59.306750+0000 mon.a (mon.0) 355 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2023-10-01T10:10:00.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:00 smithi078 bash[27336]: audit 2023-10-01T10:09:59.307333+0000 mon.a (mon.0) 356 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T10:10:00.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:00 smithi078 bash[27336]: audit 2023-10-01T10:09:59.309909+0000 mon.c (mon.2) 13 : audit [INF] from='osd.3 [v2:172.21.15.78:6826/53937366,v1:172.21.15.78:6827/53937366]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2023-10-01T10:10:00.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:00 smithi078 bash[27336]: audit 2023-10-01T10:09:59.314749+0000 mon.a (mon.0) 357 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2023-10-01T10:10:00.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:00 smithi078 bash[27336]: cluster 2023-10-01T10:09:59.793520+0000 mgr.y (mgr.14146) 155 : cluster [DBG] pgmap v134: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:10:00.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:00 smithi078 bash[27336]: cluster 2023-10-01T10:10:00.000138+0000 mon.a (mon.0) 358 : cluster [INF] overall HEALTH_OK 2023-10-01T10:10:00.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:00 smithi173 bash[27612]: audit 2023-10-01T10:09:59.306638+0000 mon.a (mon.0) 354 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-01T10:10:00.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:00 smithi173 bash[27612]: cluster 2023-10-01T10:09:59.306750+0000 mon.a (mon.0) 355 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2023-10-01T10:10:00.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:00 smithi173 bash[27612]: audit 2023-10-01T10:09:59.307333+0000 mon.a (mon.0) 356 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T10:10:00.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:00 smithi173 bash[27612]: audit 2023-10-01T10:09:59.309909+0000 mon.c (mon.2) 13 : audit [INF] from='osd.3 [v2:172.21.15.78:6826/53937366,v1:172.21.15.78:6827/53937366]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2023-10-01T10:10:00.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:00 smithi173 bash[27612]: audit 2023-10-01T10:09:59.314749+0000 mon.a (mon.0) 357 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2023-10-01T10:10:00.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:00 smithi173 bash[27612]: cluster 2023-10-01T10:09:59.793520+0000 mgr.y (mgr.14146) 155 : cluster [DBG] pgmap v134: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:10:00.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:00 smithi173 bash[27612]: cluster 2023-10-01T10:10:00.000138+0000 mon.a (mon.0) 358 : cluster [INF] overall HEALTH_OK 2023-10-01T10:10:00.883 INFO:teuthology.orchestra.run.smithi078.stdout:Created osd(s) 3 on host 'smithi078' 2023-10-01T10:10:01.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:01 smithi078 bash[27336]: audit 2023-10-01T10:10:00.317123+0000 mon.a (mon.0) 359 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2023-10-01T10:10:01.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:01 smithi078 bash[27336]: cluster 2023-10-01T10:10:00.317449+0000 mon.a (mon.0) 360 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2023-10-01T10:10:01.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:01 smithi078 bash[27336]: audit 2023-10-01T10:10:00.318233+0000 mon.a (mon.0) 361 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T10:10:01.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:01 smithi078 bash[27336]: audit 2023-10-01T10:10:00.325438+0000 mon.a (mon.0) 362 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T10:10:01.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:01 smithi078 bash[27336]: audit 2023-10-01T10:10:00.880712+0000 mon.a (mon.0) 363 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:10:01.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:01 smithi078 bash[27336]: audit 2023-10-01T10:10:00.921439+0000 mon.a (mon.0) 364 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:10:01.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:01 smithi078 bash[33821]: audit 2023-10-01T10:10:00.317123+0000 mon.a (mon.0) 359 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2023-10-01T10:10:01.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:01 smithi078 bash[33821]: cluster 2023-10-01T10:10:00.317449+0000 mon.a (mon.0) 360 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2023-10-01T10:10:01.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:01 smithi078 bash[33821]: audit 2023-10-01T10:10:00.318233+0000 mon.a (mon.0) 361 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T10:10:01.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:01 smithi078 bash[33821]: audit 2023-10-01T10:10:00.325438+0000 mon.a (mon.0) 362 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T10:10:01.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:01 smithi078 bash[33821]: audit 2023-10-01T10:10:00.880712+0000 mon.a (mon.0) 363 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:10:01.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:01 smithi078 bash[33821]: audit 2023-10-01T10:10:00.921439+0000 mon.a (mon.0) 364 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:10:01.584 DEBUG:teuthology.orchestra.run.smithi078:osd.3> sudo journalctl -f -n 0 -u ceph-d8016480-6041-11ee-8db5-212e2dc638e7@osd.3.service 2023-10-01T10:10:01.586 INFO:tasks.cephadm:Deploying osd.4 on smithi173 with /dev/vg_nvme/lv_4... 2023-10-01T10:10:01.587 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-10-01T10:10:01.599 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:01 smithi173 bash[27612]: audit 2023-10-01T10:10:00.317123+0000 mon.a (mon.0) 359 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2023-10-01T10:10:01.599 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:01 smithi173 bash[27612]: cluster 2023-10-01T10:10:00.317449+0000 mon.a (mon.0) 360 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2023-10-01T10:10:01.599 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:01 smithi173 bash[27612]: audit 2023-10-01T10:10:00.318233+0000 mon.a (mon.0) 361 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T10:10:01.599 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:01 smithi173 bash[27612]: audit 2023-10-01T10:10:00.325438+0000 mon.a (mon.0) 362 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T10:10:01.600 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:01 smithi173 bash[27612]: audit 2023-10-01T10:10:00.880712+0000 mon.a (mon.0) 363 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:10:01.600 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:01 smithi173 bash[27612]: audit 2023-10-01T10:10:00.921439+0000 mon.a (mon.0) 364 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:10:01.732 INFO:journalctl@ceph.osd.3.smithi078.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-01T10:10:02.618 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:02 smithi173 bash[27612]: cluster 2023-10-01T10:09:59.445427+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-01T10:10:02.618 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:02 smithi173 bash[27612]: cluster 2023-10-01T10:09:59.445535+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-01T10:10:02.618 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:02 smithi173 bash[27612]: cluster 2023-10-01T10:10:01.326536+0000 mon.a (mon.0) 365 : cluster [INF] osd.3 [v2:172.21.15.78:6826/53937366,v1:172.21.15.78:6827/53937366] boot 2023-10-01T10:10:02.618 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:02 smithi173 bash[27612]: cluster 2023-10-01T10:10:01.326593+0000 mon.a (mon.0) 366 : cluster [DBG] osdmap e25: 4 total, 4 up, 4 in 2023-10-01T10:10:02.618 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:02 smithi173 bash[27612]: audit 2023-10-01T10:10:01.327538+0000 mon.a (mon.0) 367 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T10:10:02.618 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:02 smithi173 bash[27612]: cluster 2023-10-01T10:10:01.793958+0000 mgr.y (mgr.14146) 156 : cluster [DBG] pgmap v137: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:10:02.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:02 smithi078 bash[27336]: cluster 2023-10-01T10:09:59.445427+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-01T10:10:02.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:02 smithi078 bash[27336]: cluster 2023-10-01T10:09:59.445535+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-01T10:10:02.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:02 smithi078 bash[27336]: cluster 2023-10-01T10:10:01.326536+0000 mon.a (mon.0) 365 : cluster [INF] osd.3 [v2:172.21.15.78:6826/53937366,v1:172.21.15.78:6827/53937366] boot 2023-10-01T10:10:02.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:02 smithi078 bash[27336]: cluster 2023-10-01T10:10:01.326593+0000 mon.a (mon.0) 366 : cluster [DBG] osdmap e25: 4 total, 4 up, 4 in 2023-10-01T10:10:02.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:02 smithi078 bash[27336]: audit 2023-10-01T10:10:01.327538+0000 mon.a (mon.0) 367 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T10:10:02.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:02 smithi078 bash[27336]: cluster 2023-10-01T10:10:01.793958+0000 mgr.y (mgr.14146) 156 : cluster [DBG] pgmap v137: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:10:02.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:02 smithi078 bash[33821]: cluster 2023-10-01T10:09:59.445427+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-01T10:10:02.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:02 smithi078 bash[33821]: cluster 2023-10-01T10:09:59.445535+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-01T10:10:02.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:02 smithi078 bash[33821]: cluster 2023-10-01T10:10:01.326536+0000 mon.a (mon.0) 365 : cluster [INF] osd.3 [v2:172.21.15.78:6826/53937366,v1:172.21.15.78:6827/53937366] boot 2023-10-01T10:10:02.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:02 smithi078 bash[33821]: cluster 2023-10-01T10:10:01.326593+0000 mon.a (mon.0) 366 : cluster [DBG] osdmap e25: 4 total, 4 up, 4 in 2023-10-01T10:10:02.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:02 smithi078 bash[33821]: audit 2023-10-01T10:10:01.327538+0000 mon.a (mon.0) 367 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T10:10:02.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:02 smithi078 bash[33821]: cluster 2023-10-01T10:10:01.793958+0000 mgr.y (mgr.14146) 156 : cluster [DBG] pgmap v137: 1 pgs: 1 active+clean; 0 B data, 871 MiB used, 267 GiB / 268 GiB avail 2023-10-01T10:10:03.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:03 smithi173 bash[27612]: cluster 2023-10-01T10:10:02.347949+0000 mon.a (mon.0) 368 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2023-10-01T10:10:03.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:03 smithi078 bash[27336]: cluster 2023-10-01T10:10:02.347949+0000 mon.a (mon.0) 368 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2023-10-01T10:10:03.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:03 smithi078 bash[33821]: cluster 2023-10-01T10:10:02.347949+0000 mon.a (mon.0) 368 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2023-10-01T10:10:04.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:04 smithi173 bash[27612]: cluster 2023-10-01T10:10:03.794464+0000 mgr.y (mgr.14146) 157 : cluster [DBG] pgmap v139: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T10:10:04.775 INFO:teuthology.orchestra.run.smithi173.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-10-01T10:10:04.775 INFO:teuthology.orchestra.run.smithi173.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-10-01T10:10:04.775 INFO:teuthology.orchestra.run.smithi173.stderr: stderr: 10+0 records in 2023-10-01T10:10:04.775 INFO:teuthology.orchestra.run.smithi173.stderr:10+0 records out 2023-10-01T10:10:04.775 INFO:teuthology.orchestra.run.smithi173.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0170336 s, 616 MB/s 2023-10-01T10:10:04.775 INFO:teuthology.orchestra.run.smithi173.stderr:--> Zapping successful for: 2023-10-01T10:10:04.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:04 smithi078 bash[27336]: cluster 2023-10-01T10:10:03.794464+0000 mgr.y (mgr.14146) 157 : cluster [DBG] pgmap v139: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T10:10:04.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:04 smithi078 bash[33821]: cluster 2023-10-01T10:10:03.794464+0000 mgr.y (mgr.14146) 157 : cluster [DBG] pgmap v139: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T10:10:05.397 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi173:vg_nvme/lv_4 2023-10-01T10:10:07.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:06 smithi173 bash[27612]: cluster 2023-10-01T10:10:05.795011+0000 mgr.y (mgr.14146) 158 : cluster [DBG] pgmap v140: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T10:10:07.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:06 smithi078 bash[27336]: cluster 2023-10-01T10:10:05.795011+0000 mgr.y (mgr.14146) 158 : cluster [DBG] pgmap v140: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T10:10:07.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:06 smithi078 bash[33821]: cluster 2023-10-01T10:10:05.795011+0000 mgr.y (mgr.14146) 158 : cluster [DBG] pgmap v140: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T10:10:09.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:08 smithi078 bash[33821]: cluster 2023-10-01T10:10:07.795630+0000 mgr.y (mgr.14146) 159 : cluster [DBG] pgmap v141: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T10:10:09.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:08 smithi078 bash[33821]: audit 2023-10-01T10:10:08.254836+0000 mgr.y (mgr.14146) 160 : audit [DBG] from='client.24214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi173:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:10:09.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:08 smithi078 bash[33821]: audit 2023-10-01T10:10:08.262658+0000 mon.a (mon.0) 369 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T10:10:09.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:08 smithi078 bash[33821]: audit 2023-10-01T10:10:08.267952+0000 mon.a (mon.0) 370 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T10:10:09.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:08 smithi078 bash[33821]: audit 2023-10-01T10:10:08.269319+0000 mon.a (mon.0) 371 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:10:09.264 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:08 smithi078 bash[27336]: cluster 2023-10-01T10:10:07.795630+0000 mgr.y (mgr.14146) 159 : cluster [DBG] pgmap v141: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T10:10:09.264 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:08 smithi078 bash[27336]: audit 2023-10-01T10:10:08.254836+0000 mgr.y (mgr.14146) 160 : audit [DBG] from='client.24214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi173:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:10:09.264 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:08 smithi078 bash[27336]: audit 2023-10-01T10:10:08.262658+0000 mon.a (mon.0) 369 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T10:10:09.264 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:08 smithi078 bash[27336]: audit 2023-10-01T10:10:08.267952+0000 mon.a (mon.0) 370 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T10:10:09.264 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:08 smithi078 bash[27336]: audit 2023-10-01T10:10:08.269319+0000 mon.a (mon.0) 371 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:10:09.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:08 smithi173 bash[27612]: cluster 2023-10-01T10:10:07.795630+0000 mgr.y (mgr.14146) 159 : cluster [DBG] pgmap v141: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T10:10:09.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:08 smithi173 bash[27612]: audit 2023-10-01T10:10:08.254836+0000 mgr.y (mgr.14146) 160 : audit [DBG] from='client.24214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi173:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:10:09.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:08 smithi173 bash[27612]: audit 2023-10-01T10:10:08.262658+0000 mon.a (mon.0) 369 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T10:10:09.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:08 smithi173 bash[27612]: audit 2023-10-01T10:10:08.267952+0000 mon.a (mon.0) 370 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T10:10:09.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:08 smithi173 bash[27612]: audit 2023-10-01T10:10:08.269319+0000 mon.a (mon.0) 371 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:10:11.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:11 smithi078 bash[27336]: cluster 2023-10-01T10:10:09.796218+0000 mgr.y (mgr.14146) 161 : cluster [DBG] pgmap v142: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T10:10:11.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:11 smithi078 bash[27336]: cephadm 2023-10-01T10:10:10.009655+0000 mgr.y (mgr.14146) 162 : cephadm [INF] Detected new or changed devices on smithi078 2023-10-01T10:10:11.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:11 smithi078 bash[27336]: audit 2023-10-01T10:10:10.019791+0000 mon.a (mon.0) 372 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:10:11.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:11 smithi078 bash[27336]: audit 2023-10-01T10:10:10.021457+0000 mon.a (mon.0) 373 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:10:11.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:11 smithi078 bash[27336]: audit 2023-10-01T10:10:10.022641+0000 mon.a (mon.0) 374 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:10:11.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:11 smithi078 bash[27336]: audit 2023-10-01T10:10:10.023655+0000 mon.a (mon.0) 375 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:10:11.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:11 smithi078 bash[27336]: audit 2023-10-01T10:10:10.024613+0000 mon.a (mon.0) 376 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:10:11.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:11 smithi078 bash[27336]: cephadm 2023-10-01T10:10:10.025358+0000 mgr.y (mgr.14146) 163 : cephadm [INF] Adjusting osd_memory_target on smithi078 to 4060M 2023-10-01T10:10:11.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:11 smithi078 bash[27336]: audit 2023-10-01T10:10:10.029700+0000 mon.a (mon.0) 377 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:10:11.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:11 smithi078 bash[27336]: audit 2023-10-01T10:10:10.031204+0000 mon.a (mon.0) 378 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:10:11.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:11 smithi078 bash[27336]: audit 2023-10-01T10:10:10.032266+0000 mon.a (mon.0) 379 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:10:11.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:11 smithi078 bash[27336]: audit 2023-10-01T10:10:10.038008+0000 mon.a (mon.0) 380 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:10:11.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:11 smithi078 bash[33821]: cluster 2023-10-01T10:10:09.796218+0000 mgr.y (mgr.14146) 161 : cluster [DBG] pgmap v142: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T10:10:11.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:11 smithi078 bash[33821]: cephadm 2023-10-01T10:10:10.009655+0000 mgr.y (mgr.14146) 162 : cephadm [INF] Detected new or changed devices on smithi078 2023-10-01T10:10:11.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:11 smithi078 bash[33821]: audit 2023-10-01T10:10:10.019791+0000 mon.a (mon.0) 372 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:10:11.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:11 smithi078 bash[33821]: audit 2023-10-01T10:10:10.021457+0000 mon.a (mon.0) 373 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:10:11.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:11 smithi078 bash[33821]: audit 2023-10-01T10:10:10.022641+0000 mon.a (mon.0) 374 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:10:11.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:11 smithi078 bash[33821]: audit 2023-10-01T10:10:10.023655+0000 mon.a (mon.0) 375 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:10:11.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:11 smithi078 bash[33821]: audit 2023-10-01T10:10:10.024613+0000 mon.a (mon.0) 376 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:10:11.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:11 smithi078 bash[33821]: cephadm 2023-10-01T10:10:10.025358+0000 mgr.y (mgr.14146) 163 : cephadm [INF] Adjusting osd_memory_target on smithi078 to 4060M 2023-10-01T10:10:11.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:11 smithi078 bash[33821]: audit 2023-10-01T10:10:10.029700+0000 mon.a (mon.0) 377 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:10:11.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:11 smithi078 bash[33821]: audit 2023-10-01T10:10:10.031204+0000 mon.a (mon.0) 378 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:10:11.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:11 smithi078 bash[33821]: audit 2023-10-01T10:10:10.032266+0000 mon.a (mon.0) 379 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:10:11.337 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:11 smithi078 bash[33821]: audit 2023-10-01T10:10:10.038008+0000 mon.a (mon.0) 380 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:10:11.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:11 smithi173 bash[27612]: cluster 2023-10-01T10:10:09.796218+0000 mgr.y (mgr.14146) 161 : cluster [DBG] pgmap v142: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T10:10:11.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:11 smithi173 bash[27612]: cephadm 2023-10-01T10:10:10.009655+0000 mgr.y (mgr.14146) 162 : cephadm [INF] Detected new or changed devices on smithi078 2023-10-01T10:10:11.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:11 smithi173 bash[27612]: audit 2023-10-01T10:10:10.019791+0000 mon.a (mon.0) 372 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:10:11.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:11 smithi173 bash[27612]: audit 2023-10-01T10:10:10.021457+0000 mon.a (mon.0) 373 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:10:11.372 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:11 smithi173 bash[27612]: audit 2023-10-01T10:10:10.022641+0000 mon.a (mon.0) 374 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:10:11.372 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:11 smithi173 bash[27612]: audit 2023-10-01T10:10:10.023655+0000 mon.a (mon.0) 375 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:10:11.372 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:11 smithi173 bash[27612]: audit 2023-10-01T10:10:10.024613+0000 mon.a (mon.0) 376 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:10:11.372 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:11 smithi173 bash[27612]: cephadm 2023-10-01T10:10:10.025358+0000 mgr.y (mgr.14146) 163 : cephadm [INF] Adjusting osd_memory_target on smithi078 to 4060M 2023-10-01T10:10:11.372 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:11 smithi173 bash[27612]: audit 2023-10-01T10:10:10.029700+0000 mon.a (mon.0) 377 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:10:11.372 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:11 smithi173 bash[27612]: audit 2023-10-01T10:10:10.031204+0000 mon.a (mon.0) 378 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:10:11.373 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:11 smithi173 bash[27612]: audit 2023-10-01T10:10:10.032266+0000 mon.a (mon.0) 379 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:10:11.373 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:11 smithi173 bash[27612]: audit 2023-10-01T10:10:10.038008+0000 mon.a (mon.0) 380 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:10:13.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:13 smithi078 bash[27336]: cluster 2023-10-01T10:10:11.796722+0000 mgr.y (mgr.14146) 164 : cluster [DBG] pgmap v143: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T10:10:13.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:13 smithi078 bash[33821]: cluster 2023-10-01T10:10:11.796722+0000 mgr.y (mgr.14146) 164 : cluster [DBG] pgmap v143: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T10:10:13.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:13 smithi173 bash[27612]: cluster 2023-10-01T10:10:11.796722+0000 mgr.y (mgr.14146) 164 : cluster [DBG] pgmap v143: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T10:10:15.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:15 smithi078 bash[27336]: cluster 2023-10-01T10:10:13.797303+0000 mgr.y (mgr.14146) 165 : cluster [DBG] pgmap v144: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T10:10:15.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:15 smithi078 bash[33821]: cluster 2023-10-01T10:10:13.797303+0000 mgr.y (mgr.14146) 165 : cluster [DBG] pgmap v144: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T10:10:15.357 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:15 smithi173 bash[27612]: cluster 2023-10-01T10:10:13.797303+0000 mgr.y (mgr.14146) 165 : cluster [DBG] pgmap v144: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T10:10:17.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:17 smithi078 bash[27336]: cluster 2023-10-01T10:10:15.797923+0000 mgr.y (mgr.14146) 166 : cluster [DBG] pgmap v145: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T10:10:17.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:17 smithi078 bash[27336]: audit 2023-10-01T10:10:16.259103+0000 mon.b (mon.1) 7 : audit [INF] from='client.? 172.21.15.173:0/2661461026' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "57c30778-fd7e-4f3e-afd9-389ef377cb62"}]: dispatch 2023-10-01T10:10:17.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:17 smithi078 bash[27336]: audit 2023-10-01T10:10:16.259784+0000 mon.a (mon.0) 381 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "57c30778-fd7e-4f3e-afd9-389ef377cb62"}]: dispatch 2023-10-01T10:10:17.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:17 smithi078 bash[27336]: audit 2023-10-01T10:10:16.267371+0000 mon.a (mon.0) 382 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "57c30778-fd7e-4f3e-afd9-389ef377cb62"}]': finished 2023-10-01T10:10:17.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:17 smithi078 bash[27336]: cluster 2023-10-01T10:10:16.267564+0000 mon.a (mon.0) 383 : cluster [DBG] osdmap e27: 5 total, 4 up, 5 in 2023-10-01T10:10:17.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:17 smithi078 bash[27336]: audit 2023-10-01T10:10:16.267866+0000 mon.a (mon.0) 384 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T10:10:17.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:17 smithi078 bash[27336]: audit 2023-10-01T10:10:16.959690+0000 mon.b (mon.1) 8 : audit [DBG] from='client.? 172.21.15.173:0/1768471948' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T10:10:17.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:17 smithi078 bash[33821]: cluster 2023-10-01T10:10:15.797923+0000 mgr.y (mgr.14146) 166 : cluster [DBG] pgmap v145: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T10:10:17.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:17 smithi078 bash[33821]: audit 2023-10-01T10:10:16.259103+0000 mon.b (mon.1) 7 : audit [INF] from='client.? 172.21.15.173:0/2661461026' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "57c30778-fd7e-4f3e-afd9-389ef377cb62"}]: dispatch 2023-10-01T10:10:17.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:17 smithi078 bash[33821]: audit 2023-10-01T10:10:16.259784+0000 mon.a (mon.0) 381 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "57c30778-fd7e-4f3e-afd9-389ef377cb62"}]: dispatch 2023-10-01T10:10:17.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:17 smithi078 bash[33821]: audit 2023-10-01T10:10:16.267371+0000 mon.a (mon.0) 382 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "57c30778-fd7e-4f3e-afd9-389ef377cb62"}]': finished 2023-10-01T10:10:17.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:17 smithi078 bash[33821]: cluster 2023-10-01T10:10:16.267564+0000 mon.a (mon.0) 383 : cluster [DBG] osdmap e27: 5 total, 4 up, 5 in 2023-10-01T10:10:17.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:17 smithi078 bash[33821]: audit 2023-10-01T10:10:16.267866+0000 mon.a (mon.0) 384 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T10:10:17.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:17 smithi078 bash[33821]: audit 2023-10-01T10:10:16.959690+0000 mon.b (mon.1) 8 : audit [DBG] from='client.? 172.21.15.173:0/1768471948' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T10:10:17.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:17 smithi173 bash[27612]: cluster 2023-10-01T10:10:15.797923+0000 mgr.y (mgr.14146) 166 : cluster [DBG] pgmap v145: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T10:10:17.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:17 smithi173 bash[27612]: audit 2023-10-01T10:10:16.259103+0000 mon.b (mon.1) 7 : audit [INF] from='client.? 172.21.15.173:0/2661461026' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "57c30778-fd7e-4f3e-afd9-389ef377cb62"}]: dispatch 2023-10-01T10:10:17.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:17 smithi173 bash[27612]: audit 2023-10-01T10:10:16.259784+0000 mon.a (mon.0) 381 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "57c30778-fd7e-4f3e-afd9-389ef377cb62"}]: dispatch 2023-10-01T10:10:17.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:17 smithi173 bash[27612]: audit 2023-10-01T10:10:16.267371+0000 mon.a (mon.0) 382 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "57c30778-fd7e-4f3e-afd9-389ef377cb62"}]': finished 2023-10-01T10:10:17.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:17 smithi173 bash[27612]: cluster 2023-10-01T10:10:16.267564+0000 mon.a (mon.0) 383 : cluster [DBG] osdmap e27: 5 total, 4 up, 5 in 2023-10-01T10:10:17.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:17 smithi173 bash[27612]: audit 2023-10-01T10:10:16.267866+0000 mon.a (mon.0) 384 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T10:10:17.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:17 smithi173 bash[27612]: audit 2023-10-01T10:10:16.959690+0000 mon.b (mon.1) 8 : audit [DBG] from='client.? 172.21.15.173:0/1768471948' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T10:10:19.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:19 smithi078 bash[27336]: cluster 2023-10-01T10:10:17.798510+0000 mgr.y (mgr.14146) 167 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T10:10:19.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:19 smithi078 bash[33821]: cluster 2023-10-01T10:10:17.798510+0000 mgr.y (mgr.14146) 167 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T10:10:19.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:19 smithi173 bash[27612]: cluster 2023-10-01T10:10:17.798510+0000 mgr.y (mgr.14146) 167 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T10:10:21.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:21 smithi078 bash[33821]: cluster 2023-10-01T10:10:19.798978+0000 mgr.y (mgr.14146) 168 : cluster [DBG] pgmap v148: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T10:10:21.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:21 smithi078 bash[27336]: cluster 2023-10-01T10:10:19.798978+0000 mgr.y (mgr.14146) 168 : cluster [DBG] pgmap v148: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T10:10:21.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:21 smithi173 bash[27612]: cluster 2023-10-01T10:10:19.798978+0000 mgr.y (mgr.14146) 168 : cluster [DBG] pgmap v148: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T10:10:23.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:23 smithi078 bash[27336]: cluster 2023-10-01T10:10:21.799485+0000 mgr.y (mgr.14146) 169 : cluster [DBG] pgmap v149: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T10:10:23.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:23 smithi078 bash[33821]: cluster 2023-10-01T10:10:21.799485+0000 mgr.y (mgr.14146) 169 : cluster [DBG] pgmap v149: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T10:10:23.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:23 smithi173 bash[27612]: cluster 2023-10-01T10:10:21.799485+0000 mgr.y (mgr.14146) 169 : cluster [DBG] pgmap v149: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T10:10:25.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:25 smithi078 bash[27336]: cluster 2023-10-01T10:10:23.800011+0000 mgr.y (mgr.14146) 170 : cluster [DBG] pgmap v150: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T10:10:25.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:25 smithi078 bash[33821]: cluster 2023-10-01T10:10:23.800011+0000 mgr.y (mgr.14146) 170 : cluster [DBG] pgmap v150: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T10:10:25.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:25 smithi173 bash[27612]: cluster 2023-10-01T10:10:23.800011+0000 mgr.y (mgr.14146) 170 : cluster [DBG] pgmap v150: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T10:10:27.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:27 smithi078 bash[27336]: cluster 2023-10-01T10:10:25.800541+0000 mgr.y (mgr.14146) 171 : cluster [DBG] pgmap v151: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T10:10:27.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:27 smithi078 bash[33821]: cluster 2023-10-01T10:10:25.800541+0000 mgr.y (mgr.14146) 171 : cluster [DBG] pgmap v151: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T10:10:27.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:27 smithi173 bash[27612]: cluster 2023-10-01T10:10:25.800541+0000 mgr.y (mgr.14146) 171 : cluster [DBG] pgmap v151: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T10:10:29.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:29 smithi078 bash[27336]: cluster 2023-10-01T10:10:27.801087+0000 mgr.y (mgr.14146) 172 : cluster [DBG] pgmap v152: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T10:10:29.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:29 smithi078 bash[27336]: audit 2023-10-01T10:10:28.408273+0000 mon.a (mon.0) 385 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-01T10:10:29.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:29 smithi078 bash[27336]: audit 2023-10-01T10:10:28.409827+0000 mon.a (mon.0) 386 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:10:29.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:29 smithi078 bash[33821]: cluster 2023-10-01T10:10:27.801087+0000 mgr.y (mgr.14146) 172 : cluster [DBG] pgmap v152: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T10:10:29.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:29 smithi078 bash[33821]: audit 2023-10-01T10:10:28.408273+0000 mon.a (mon.0) 385 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-01T10:10:29.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:29 smithi078 bash[33821]: audit 2023-10-01T10:10:28.409827+0000 mon.a (mon.0) 386 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:10:29.333 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:29 smithi173 bash[27612]: cluster 2023-10-01T10:10:27.801087+0000 mgr.y (mgr.14146) 172 : cluster [DBG] pgmap v152: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T10:10:29.333 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:29 smithi173 bash[27612]: audit 2023-10-01T10:10:28.408273+0000 mon.a (mon.0) 385 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-01T10:10:29.333 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:29 smithi173 bash[27612]: audit 2023-10-01T10:10:28.409827+0000 mon.a (mon.0) 386 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:10:30.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:30 smithi078 bash[27336]: cephadm 2023-10-01T10:10:28.411168+0000 mgr.y (mgr.14146) 173 : cephadm [INF] Deploying daemon osd.4 on smithi173 2023-10-01T10:10:30.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:30 smithi078 bash[33821]: cephadm 2023-10-01T10:10:28.411168+0000 mgr.y (mgr.14146) 173 : cephadm [INF] Deploying daemon osd.4 on smithi173 2023-10-01T10:10:30.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:30 smithi173 bash[27612]: cephadm 2023-10-01T10:10:28.411168+0000 mgr.y (mgr.14146) 173 : cephadm [INF] Deploying daemon osd.4 on smithi173 2023-10-01T10:10:31.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:31 smithi078 bash[27336]: cluster 2023-10-01T10:10:29.801627+0000 mgr.y (mgr.14146) 174 : cluster [DBG] pgmap v153: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T10:10:31.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:31 smithi078 bash[33821]: cluster 2023-10-01T10:10:29.801627+0000 mgr.y (mgr.14146) 174 : cluster [DBG] pgmap v153: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T10:10:31.334 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:31 smithi173 bash[27612]: cluster 2023-10-01T10:10:29.801627+0000 mgr.y (mgr.14146) 174 : cluster [DBG] pgmap v153: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T10:10:33.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:33 smithi078 bash[27336]: cluster 2023-10-01T10:10:31.802154+0000 mgr.y (mgr.14146) 175 : cluster [DBG] pgmap v154: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T10:10:33.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:33 smithi078 bash[27336]: audit 2023-10-01T10:10:32.207423+0000 mon.a (mon.0) 387 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:10:33.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:33 smithi078 bash[27336]: audit 2023-10-01T10:10:32.219060+0000 mon.a (mon.0) 388 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:10:33.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:33 smithi078 bash[33821]: cluster 2023-10-01T10:10:31.802154+0000 mgr.y (mgr.14146) 175 : cluster [DBG] pgmap v154: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T10:10:33.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:33 smithi078 bash[33821]: audit 2023-10-01T10:10:32.207423+0000 mon.a (mon.0) 387 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:10:33.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:33 smithi078 bash[33821]: audit 2023-10-01T10:10:32.219060+0000 mon.a (mon.0) 388 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:10:33.583 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:33 smithi173 bash[27612]: cluster 2023-10-01T10:10:31.802154+0000 mgr.y (mgr.14146) 175 : cluster [DBG] pgmap v154: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T10:10:33.584 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:33 smithi173 bash[27612]: audit 2023-10-01T10:10:32.207423+0000 mon.a (mon.0) 387 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:10:33.584 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:33 smithi173 bash[27612]: audit 2023-10-01T10:10:32.219060+0000 mon.a (mon.0) 388 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:10:35.564 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:35 smithi173 bash[27612]: cluster 2023-10-01T10:10:33.802705+0000 mgr.y (mgr.14146) 176 : cluster [DBG] pgmap v155: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T10:10:35.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:35 smithi078 bash[27336]: cluster 2023-10-01T10:10:33.802705+0000 mgr.y (mgr.14146) 176 : cluster [DBG] pgmap v155: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T10:10:35.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:35 smithi078 bash[33821]: cluster 2023-10-01T10:10:33.802705+0000 mgr.y (mgr.14146) 176 : cluster [DBG] pgmap v155: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T10:10:37.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:37 smithi078 bash[27336]: cluster 2023-10-01T10:10:35.803238+0000 mgr.y (mgr.14146) 177 : cluster [DBG] pgmap v156: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T10:10:37.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:37 smithi078 bash[33821]: cluster 2023-10-01T10:10:35.803238+0000 mgr.y (mgr.14146) 177 : cluster [DBG] pgmap v156: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T10:10:37.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:37 smithi173 bash[27612]: cluster 2023-10-01T10:10:35.803238+0000 mgr.y (mgr.14146) 177 : cluster [DBG] pgmap v156: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T10:10:38.551 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:38 smithi173 bash[27612]: audit 2023-10-01T10:10:37.274575+0000 mon.a (mon.0) 389 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:10:38.552 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:38 smithi173 bash[27612]: audit 2023-10-01T10:10:37.527047+0000 mon.a (mon.0) 390 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:10:38.552 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:38 smithi173 bash[27612]: audit 2023-10-01T10:10:37.528747+0000 mon.a (mon.0) 391 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:10:38.552 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:38 smithi173 bash[27612]: audit 2023-10-01T10:10:37.536091+0000 mon.a (mon.0) 392 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:10:38.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:38 smithi078 bash[27336]: audit 2023-10-01T10:10:37.274575+0000 mon.a (mon.0) 389 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:10:38.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:38 smithi078 bash[27336]: audit 2023-10-01T10:10:37.527047+0000 mon.a (mon.0) 390 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:10:38.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:38 smithi078 bash[27336]: audit 2023-10-01T10:10:37.528747+0000 mon.a (mon.0) 391 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:10:38.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:38 smithi078 bash[27336]: audit 2023-10-01T10:10:37.536091+0000 mon.a (mon.0) 392 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:10:38.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:38 smithi078 bash[33821]: audit 2023-10-01T10:10:37.274575+0000 mon.a (mon.0) 389 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:10:38.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:38 smithi078 bash[33821]: audit 2023-10-01T10:10:37.527047+0000 mon.a (mon.0) 390 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:10:38.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:38 smithi078 bash[33821]: audit 2023-10-01T10:10:37.528747+0000 mon.a (mon.0) 391 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:10:38.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:38 smithi078 bash[33821]: audit 2023-10-01T10:10:37.536091+0000 mon.a (mon.0) 392 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:10:39.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:39 smithi078 bash[27336]: cluster 2023-10-01T10:10:37.803771+0000 mgr.y (mgr.14146) 178 : cluster [DBG] pgmap v157: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T10:10:39.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:39 smithi078 bash[27336]: audit 2023-10-01T10:10:38.555752+0000 mon.b (mon.1) 9 : audit [INF] from='osd.4 [v2:172.21.15.173:6800/2430591286,v1:172.21.15.173:6801/2430591286]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-01T10:10:39.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:39 smithi078 bash[27336]: audit 2023-10-01T10:10:38.556161+0000 mon.a (mon.0) 393 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-01T10:10:39.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:39 smithi078 bash[33821]: cluster 2023-10-01T10:10:37.803771+0000 mgr.y (mgr.14146) 178 : cluster [DBG] pgmap v157: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T10:10:39.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:39 smithi078 bash[33821]: audit 2023-10-01T10:10:38.555752+0000 mon.b (mon.1) 9 : audit [INF] from='osd.4 [v2:172.21.15.173:6800/2430591286,v1:172.21.15.173:6801/2430591286]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-01T10:10:39.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:39 smithi078 bash[33821]: audit 2023-10-01T10:10:38.556161+0000 mon.a (mon.0) 393 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-01T10:10:39.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:39 smithi173 bash[27612]: cluster 2023-10-01T10:10:37.803771+0000 mgr.y (mgr.14146) 178 : cluster [DBG] pgmap v157: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T10:10:39.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:39 smithi173 bash[27612]: audit 2023-10-01T10:10:38.555752+0000 mon.b (mon.1) 9 : audit [INF] from='osd.4 [v2:172.21.15.173:6800/2430591286,v1:172.21.15.173:6801/2430591286]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-01T10:10:39.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:39 smithi173 bash[27612]: audit 2023-10-01T10:10:38.556161+0000 mon.a (mon.0) 393 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-01T10:10:40.660 INFO:teuthology.orchestra.run.smithi173.stdout:Created osd(s) 4 on host 'smithi173' 2023-10-01T10:10:40.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:40 smithi078 bash[27336]: audit 2023-10-01T10:10:39.541976+0000 mon.a (mon.0) 394 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-10-01T10:10:40.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:40 smithi078 bash[27336]: cluster 2023-10-01T10:10:39.542191+0000 mon.a (mon.0) 395 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2023-10-01T10:10:40.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:40 smithi078 bash[27336]: audit 2023-10-01T10:10:39.542448+0000 mon.a (mon.0) 396 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T10:10:40.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:40 smithi078 bash[27336]: audit 2023-10-01T10:10:39.543839+0000 mon.b (mon.1) 10 : audit [INF] from='osd.4 [v2:172.21.15.173:6800/2430591286,v1:172.21.15.173:6801/2430591286]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-10-01T10:10:40.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:40 smithi078 bash[27336]: audit 2023-10-01T10:10:39.544256+0000 mon.a (mon.0) 397 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-10-01T10:10:40.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:40 smithi078 bash[27336]: cluster 2023-10-01T10:10:39.804261+0000 mgr.y (mgr.14146) 179 : cluster [DBG] pgmap v159: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T10:10:40.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:40 smithi078 bash[33821]: audit 2023-10-01T10:10:39.541976+0000 mon.a (mon.0) 394 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-10-01T10:10:40.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:40 smithi078 bash[33821]: cluster 2023-10-01T10:10:39.542191+0000 mon.a (mon.0) 395 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2023-10-01T10:10:40.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:40 smithi078 bash[33821]: audit 2023-10-01T10:10:39.542448+0000 mon.a (mon.0) 396 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T10:10:40.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:40 smithi078 bash[33821]: audit 2023-10-01T10:10:39.543839+0000 mon.b (mon.1) 10 : audit [INF] from='osd.4 [v2:172.21.15.173:6800/2430591286,v1:172.21.15.173:6801/2430591286]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-10-01T10:10:40.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:40 smithi078 bash[33821]: audit 2023-10-01T10:10:39.544256+0000 mon.a (mon.0) 397 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-10-01T10:10:40.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:40 smithi078 bash[33821]: cluster 2023-10-01T10:10:39.804261+0000 mgr.y (mgr.14146) 179 : cluster [DBG] pgmap v159: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T10:10:40.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:40 smithi173 bash[27612]: audit 2023-10-01T10:10:39.541976+0000 mon.a (mon.0) 394 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-10-01T10:10:40.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:40 smithi173 bash[27612]: cluster 2023-10-01T10:10:39.542191+0000 mon.a (mon.0) 395 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2023-10-01T10:10:40.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:40 smithi173 bash[27612]: audit 2023-10-01T10:10:39.542448+0000 mon.a (mon.0) 396 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T10:10:40.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:40 smithi173 bash[27612]: audit 2023-10-01T10:10:39.543839+0000 mon.b (mon.1) 10 : audit [INF] from='osd.4 [v2:172.21.15.173:6800/2430591286,v1:172.21.15.173:6801/2430591286]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-10-01T10:10:40.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:40 smithi173 bash[27612]: audit 2023-10-01T10:10:39.544256+0000 mon.a (mon.0) 397 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-10-01T10:10:40.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:40 smithi173 bash[27612]: cluster 2023-10-01T10:10:39.804261+0000 mgr.y (mgr.14146) 179 : cluster [DBG] pgmap v159: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T10:10:41.334 DEBUG:teuthology.orchestra.run.smithi173:osd.4> sudo journalctl -f -n 0 -u ceph-d8016480-6041-11ee-8db5-212e2dc638e7@osd.4.service 2023-10-01T10:10:41.336 INFO:tasks.cephadm:Deploying osd.5 on smithi173 with /dev/vg_nvme/lv_3... 2023-10-01T10:10:41.336 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-10-01T10:10:41.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:41 smithi173 bash[27612]: audit 2023-10-01T10:10:40.545049+0000 mon.a (mon.0) 398 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]': finished 2023-10-01T10:10:41.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:41 smithi173 bash[27612]: cluster 2023-10-01T10:10:40.545196+0000 mon.a (mon.0) 399 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2023-10-01T10:10:41.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:41 smithi173 bash[27612]: audit 2023-10-01T10:10:40.545855+0000 mon.a (mon.0) 400 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T10:10:41.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:41 smithi173 bash[27612]: audit 2023-10-01T10:10:40.552658+0000 mon.a (mon.0) 401 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T10:10:41.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:41 smithi173 bash[27612]: audit 2023-10-01T10:10:40.657675+0000 mon.a (mon.0) 402 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:10:41.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:41 smithi173 bash[27612]: audit 2023-10-01T10:10:40.661981+0000 mon.a (mon.0) 403 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:10:41.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:41 smithi078 bash[33821]: audit 2023-10-01T10:10:40.545049+0000 mon.a (mon.0) 398 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]': finished 2023-10-01T10:10:41.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:41 smithi078 bash[33821]: cluster 2023-10-01T10:10:40.545196+0000 mon.a (mon.0) 399 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2023-10-01T10:10:41.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:41 smithi078 bash[33821]: audit 2023-10-01T10:10:40.545855+0000 mon.a (mon.0) 400 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T10:10:41.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:41 smithi078 bash[33821]: audit 2023-10-01T10:10:40.552658+0000 mon.a (mon.0) 401 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T10:10:41.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:41 smithi078 bash[33821]: audit 2023-10-01T10:10:40.657675+0000 mon.a (mon.0) 402 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:10:41.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:41 smithi078 bash[33821]: audit 2023-10-01T10:10:40.661981+0000 mon.a (mon.0) 403 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:10:41.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:41 smithi078 bash[27336]: audit 2023-10-01T10:10:40.545049+0000 mon.a (mon.0) 398 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]': finished 2023-10-01T10:10:41.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:41 smithi078 bash[27336]: cluster 2023-10-01T10:10:40.545196+0000 mon.a (mon.0) 399 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2023-10-01T10:10:41.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:41 smithi078 bash[27336]: audit 2023-10-01T10:10:40.545855+0000 mon.a (mon.0) 400 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T10:10:41.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:41 smithi078 bash[27336]: audit 2023-10-01T10:10:40.552658+0000 mon.a (mon.0) 401 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T10:10:41.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:41 smithi078 bash[27336]: audit 2023-10-01T10:10:40.657675+0000 mon.a (mon.0) 402 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:10:41.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:41 smithi078 bash[27336]: audit 2023-10-01T10:10:40.661981+0000 mon.a (mon.0) 403 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:10:41.918 INFO:journalctl@ceph.osd.4.smithi173.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-01T10:10:42.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:42 smithi078 bash[27336]: cluster 2023-10-01T10:10:39.549971+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-01T10:10:42.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:42 smithi078 bash[27336]: cluster 2023-10-01T10:10:39.550030+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-01T10:10:42.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:42 smithi078 bash[27336]: cluster 2023-10-01T10:10:41.554491+0000 mon.a (mon.0) 404 : cluster [INF] osd.4 [v2:172.21.15.173:6800/2430591286,v1:172.21.15.173:6801/2430591286] boot 2023-10-01T10:10:42.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:42 smithi078 bash[27336]: cluster 2023-10-01T10:10:41.554666+0000 mon.a (mon.0) 405 : cluster [DBG] osdmap e30: 5 total, 5 up, 5 in 2023-10-01T10:10:42.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:42 smithi078 bash[27336]: audit 2023-10-01T10:10:41.555400+0000 mon.a (mon.0) 406 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T10:10:42.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:42 smithi078 bash[27336]: cluster 2023-10-01T10:10:41.804796+0000 mgr.y (mgr.14146) 180 : cluster [DBG] pgmap v162: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T10:10:42.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:42 smithi078 bash[33821]: cluster 2023-10-01T10:10:39.549971+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-01T10:10:42.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:42 smithi078 bash[33821]: cluster 2023-10-01T10:10:39.550030+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-01T10:10:42.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:42 smithi078 bash[33821]: cluster 2023-10-01T10:10:41.554491+0000 mon.a (mon.0) 404 : cluster [INF] osd.4 [v2:172.21.15.173:6800/2430591286,v1:172.21.15.173:6801/2430591286] boot 2023-10-01T10:10:42.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:42 smithi078 bash[33821]: cluster 2023-10-01T10:10:41.554666+0000 mon.a (mon.0) 405 : cluster [DBG] osdmap e30: 5 total, 5 up, 5 in 2023-10-01T10:10:42.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:42 smithi078 bash[33821]: audit 2023-10-01T10:10:41.555400+0000 mon.a (mon.0) 406 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T10:10:42.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:42 smithi078 bash[33821]: cluster 2023-10-01T10:10:41.804796+0000 mgr.y (mgr.14146) 180 : cluster [DBG] pgmap v162: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T10:10:42.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:42 smithi173 bash[27612]: cluster 2023-10-01T10:10:39.549971+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-01T10:10:42.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:42 smithi173 bash[27612]: cluster 2023-10-01T10:10:39.550030+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-01T10:10:42.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:42 smithi173 bash[27612]: cluster 2023-10-01T10:10:41.554491+0000 mon.a (mon.0) 404 : cluster [INF] osd.4 [v2:172.21.15.173:6800/2430591286,v1:172.21.15.173:6801/2430591286] boot 2023-10-01T10:10:42.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:42 smithi173 bash[27612]: cluster 2023-10-01T10:10:41.554666+0000 mon.a (mon.0) 405 : cluster [DBG] osdmap e30: 5 total, 5 up, 5 in 2023-10-01T10:10:42.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:42 smithi173 bash[27612]: audit 2023-10-01T10:10:41.555400+0000 mon.a (mon.0) 406 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T10:10:42.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:42 smithi173 bash[27612]: cluster 2023-10-01T10:10:41.804796+0000 mgr.y (mgr.14146) 180 : cluster [DBG] pgmap v162: 1 pgs: 1 active+clean; 0 B data, 1.1 GiB used, 356 GiB / 358 GiB avail 2023-10-01T10:10:43.863 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:43 smithi173 bash[27612]: cluster 2023-10-01T10:10:42.571832+0000 mon.a (mon.0) 407 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2023-10-01T10:10:44.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:43 smithi078 bash[27336]: cluster 2023-10-01T10:10:42.571832+0000 mon.a (mon.0) 407 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2023-10-01T10:10:44.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:43 smithi078 bash[33821]: cluster 2023-10-01T10:10:42.571832+0000 mon.a (mon.0) 407 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2023-10-01T10:10:44.491 INFO:teuthology.orchestra.run.smithi173.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-10-01T10:10:44.491 INFO:teuthology.orchestra.run.smithi173.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-10-01T10:10:44.491 INFO:teuthology.orchestra.run.smithi173.stderr: stderr: 10+0 records in 2023-10-01T10:10:44.491 INFO:teuthology.orchestra.run.smithi173.stderr:10+0 records out 2023-10-01T10:10:44.491 INFO:teuthology.orchestra.run.smithi173.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0183136 s, 573 MB/s 2023-10-01T10:10:44.492 INFO:teuthology.orchestra.run.smithi173.stderr:--> Zapping successful for: 2023-10-01T10:10:44.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:44 smithi173 bash[27612]: cluster 2023-10-01T10:10:43.575893+0000 mon.a (mon.0) 408 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2023-10-01T10:10:44.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:44 smithi173 bash[27612]: cluster 2023-10-01T10:10:43.805395+0000 mgr.y (mgr.14146) 181 : cluster [DBG] pgmap v165: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T10:10:45.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:44 smithi078 bash[27336]: cluster 2023-10-01T10:10:43.575893+0000 mon.a (mon.0) 408 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2023-10-01T10:10:45.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:44 smithi078 bash[27336]: cluster 2023-10-01T10:10:43.805395+0000 mgr.y (mgr.14146) 181 : cluster [DBG] pgmap v165: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T10:10:45.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:44 smithi078 bash[33821]: cluster 2023-10-01T10:10:43.575893+0000 mon.a (mon.0) 408 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2023-10-01T10:10:45.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:44 smithi078 bash[33821]: cluster 2023-10-01T10:10:43.805395+0000 mgr.y (mgr.14146) 181 : cluster [DBG] pgmap v165: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T10:10:45.228 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi173:vg_nvme/lv_3 2023-10-01T10:10:47.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:46 smithi078 bash[27336]: cluster 2023-10-01T10:10:45.805876+0000 mgr.y (mgr.14146) 182 : cluster [DBG] pgmap v166: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T10:10:47.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:46 smithi078 bash[33821]: cluster 2023-10-01T10:10:45.805876+0000 mgr.y (mgr.14146) 182 : cluster [DBG] pgmap v166: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T10:10:47.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:46 smithi173 bash[27612]: cluster 2023-10-01T10:10:45.805876+0000 mgr.y (mgr.14146) 182 : cluster [DBG] pgmap v166: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T10:10:49.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:48 smithi078 bash[33821]: cluster 2023-10-01T10:10:47.806416+0000 mgr.y (mgr.14146) 183 : cluster [DBG] pgmap v167: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T10:10:49.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:48 smithi078 bash[27336]: cluster 2023-10-01T10:10:47.806416+0000 mgr.y (mgr.14146) 183 : cluster [DBG] pgmap v167: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T10:10:49.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:48 smithi173 bash[27612]: cluster 2023-10-01T10:10:47.806416+0000 mgr.y (mgr.14146) 183 : cluster [DBG] pgmap v167: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T10:10:50.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:49 smithi173 bash[27612]: audit 2023-10-01T10:10:49.432734+0000 mon.a (mon.0) 409 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T10:10:50.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:49 smithi173 bash[27612]: audit 2023-10-01T10:10:49.437961+0000 mon.a (mon.0) 410 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T10:10:50.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:49 smithi173 bash[27612]: audit 2023-10-01T10:10:49.439222+0000 mon.a (mon.0) 411 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:10:50.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:49 smithi078 bash[27336]: audit 2023-10-01T10:10:49.432734+0000 mon.a (mon.0) 409 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T10:10:50.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:49 smithi078 bash[27336]: audit 2023-10-01T10:10:49.437961+0000 mon.a (mon.0) 410 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T10:10:50.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:49 smithi078 bash[27336]: audit 2023-10-01T10:10:49.439222+0000 mon.a (mon.0) 411 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:10:50.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:49 smithi078 bash[33821]: audit 2023-10-01T10:10:49.432734+0000 mon.a (mon.0) 409 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T10:10:50.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:49 smithi078 bash[33821]: audit 2023-10-01T10:10:49.437961+0000 mon.a (mon.0) 410 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T10:10:50.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:49 smithi078 bash[33821]: audit 2023-10-01T10:10:49.439222+0000 mon.a (mon.0) 411 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:10:51.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:50 smithi078 bash[27336]: audit 2023-10-01T10:10:49.426733+0000 mgr.y (mgr.14146) 184 : audit [DBG] from='client.24241 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi173:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:10:51.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:50 smithi078 bash[27336]: cluster 2023-10-01T10:10:49.806943+0000 mgr.y (mgr.14146) 185 : cluster [DBG] pgmap v168: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T10:10:51.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:50 smithi078 bash[27336]: audit 2023-10-01T10:10:50.593839+0000 mon.a (mon.0) 412 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:10:51.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:50 smithi078 bash[27336]: audit 2023-10-01T10:10:50.595602+0000 mon.a (mon.0) 413 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:10:51.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:50 smithi078 bash[27336]: audit 2023-10-01T10:10:50.602858+0000 mon.a (mon.0) 414 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:10:51.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:50 smithi078 bash[27336]: audit 2023-10-01T10:10:50.604265+0000 mon.a (mon.0) 415 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:10:51.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:50 smithi078 bash[27336]: audit 2023-10-01T10:10:50.605431+0000 mon.a (mon.0) 416 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:10:51.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:50 smithi078 bash[27336]: audit 2023-10-01T10:10:50.611017+0000 mon.a (mon.0) 417 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:10:51.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:50 smithi078 bash[33821]: audit 2023-10-01T10:10:49.426733+0000 mgr.y (mgr.14146) 184 : audit [DBG] from='client.24241 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi173:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:10:51.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:50 smithi078 bash[33821]: cluster 2023-10-01T10:10:49.806943+0000 mgr.y (mgr.14146) 185 : cluster [DBG] pgmap v168: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T10:10:51.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:50 smithi078 bash[33821]: audit 2023-10-01T10:10:50.593839+0000 mon.a (mon.0) 412 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:10:51.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:50 smithi078 bash[33821]: audit 2023-10-01T10:10:50.595602+0000 mon.a (mon.0) 413 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:10:51.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:50 smithi078 bash[33821]: audit 2023-10-01T10:10:50.602858+0000 mon.a (mon.0) 414 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:10:51.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:50 smithi078 bash[33821]: audit 2023-10-01T10:10:50.604265+0000 mon.a (mon.0) 415 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:10:51.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:50 smithi078 bash[33821]: audit 2023-10-01T10:10:50.605431+0000 mon.a (mon.0) 416 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:10:51.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:50 smithi078 bash[33821]: audit 2023-10-01T10:10:50.611017+0000 mon.a (mon.0) 417 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:10:51.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:50 smithi173 bash[27612]: audit 2023-10-01T10:10:49.426733+0000 mgr.y (mgr.14146) 184 : audit [DBG] from='client.24241 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi173:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:10:51.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:50 smithi173 bash[27612]: cluster 2023-10-01T10:10:49.806943+0000 mgr.y (mgr.14146) 185 : cluster [DBG] pgmap v168: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T10:10:51.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:50 smithi173 bash[27612]: audit 2023-10-01T10:10:50.593839+0000 mon.a (mon.0) 412 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:10:51.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:50 smithi173 bash[27612]: audit 2023-10-01T10:10:50.595602+0000 mon.a (mon.0) 413 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:10:51.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:50 smithi173 bash[27612]: audit 2023-10-01T10:10:50.602858+0000 mon.a (mon.0) 414 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:10:51.372 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:50 smithi173 bash[27612]: audit 2023-10-01T10:10:50.604265+0000 mon.a (mon.0) 415 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:10:51.372 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:50 smithi173 bash[27612]: audit 2023-10-01T10:10:50.605431+0000 mon.a (mon.0) 416 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:10:51.372 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:50 smithi173 bash[27612]: audit 2023-10-01T10:10:50.611017+0000 mon.a (mon.0) 417 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:10:52.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:51 smithi078 bash[33821]: cephadm 2023-10-01T10:10:50.584647+0000 mgr.y (mgr.14146) 186 : cephadm [INF] Detected new or changed devices on smithi173 2023-10-01T10:10:52.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:51 smithi078 bash[33821]: cephadm 2023-10-01T10:10:50.596665+0000 mgr.y (mgr.14146) 187 : cephadm [INF] Adjusting osd_memory_target on smithi173 to 17265M 2023-10-01T10:10:52.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:51 smithi078 bash[27336]: cephadm 2023-10-01T10:10:50.584647+0000 mgr.y (mgr.14146) 186 : cephadm [INF] Detected new or changed devices on smithi173 2023-10-01T10:10:52.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:51 smithi078 bash[27336]: cephadm 2023-10-01T10:10:50.596665+0000 mgr.y (mgr.14146) 187 : cephadm [INF] Adjusting osd_memory_target on smithi173 to 17265M 2023-10-01T10:10:52.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:51 smithi173 bash[27612]: cephadm 2023-10-01T10:10:50.584647+0000 mgr.y (mgr.14146) 186 : cephadm [INF] Detected new or changed devices on smithi173 2023-10-01T10:10:52.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:51 smithi173 bash[27612]: cephadm 2023-10-01T10:10:50.596665+0000 mgr.y (mgr.14146) 187 : cephadm [INF] Adjusting osd_memory_target on smithi173 to 17265M 2023-10-01T10:10:53.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:52 smithi078 bash[33821]: cluster 2023-10-01T10:10:51.807475+0000 mgr.y (mgr.14146) 188 : cluster [DBG] pgmap v169: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T10:10:53.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:52 smithi078 bash[27336]: cluster 2023-10-01T10:10:51.807475+0000 mgr.y (mgr.14146) 188 : cluster [DBG] pgmap v169: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T10:10:53.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:52 smithi173 bash[27612]: cluster 2023-10-01T10:10:51.807475+0000 mgr.y (mgr.14146) 188 : cluster [DBG] pgmap v169: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T10:10:55.231 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:54 smithi173 bash[27612]: cluster 2023-10-01T10:10:53.808010+0000 mgr.y (mgr.14146) 189 : cluster [DBG] pgmap v170: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T10:10:55.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:54 smithi078 bash[27336]: cluster 2023-10-01T10:10:53.808010+0000 mgr.y (mgr.14146) 189 : cluster [DBG] pgmap v170: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T10:10:55.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:54 smithi078 bash[33821]: cluster 2023-10-01T10:10:53.808010+0000 mgr.y (mgr.14146) 189 : cluster [DBG] pgmap v170: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T10:10:57.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:56 smithi078 bash[27336]: cluster 2023-10-01T10:10:55.808592+0000 mgr.y (mgr.14146) 190 : cluster [DBG] pgmap v171: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T10:10:57.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:56 smithi078 bash[33821]: cluster 2023-10-01T10:10:55.808592+0000 mgr.y (mgr.14146) 190 : cluster [DBG] pgmap v171: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T10:10:57.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:56 smithi173 bash[27612]: cluster 2023-10-01T10:10:55.808592+0000 mgr.y (mgr.14146) 190 : cluster [DBG] pgmap v171: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T10:10:58.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:57 smithi078 bash[27336]: audit 2023-10-01T10:10:57.289345+0000 mon.b (mon.1) 11 : audit [INF] from='client.? 172.21.15.173:0/3793157739' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c62aa7c3-2d09-4088-ba36-c84588dc3549"}]: dispatch 2023-10-01T10:10:58.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:57 smithi078 bash[27336]: audit 2023-10-01T10:10:57.290138+0000 mon.a (mon.0) 418 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c62aa7c3-2d09-4088-ba36-c84588dc3549"}]: dispatch 2023-10-01T10:10:58.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:57 smithi078 bash[27336]: audit 2023-10-01T10:10:57.297891+0000 mon.a (mon.0) 419 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c62aa7c3-2d09-4088-ba36-c84588dc3549"}]': finished 2023-10-01T10:10:58.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:57 smithi078 bash[27336]: cluster 2023-10-01T10:10:57.297983+0000 mon.a (mon.0) 420 : cluster [DBG] osdmap e33: 6 total, 5 up, 6 in 2023-10-01T10:10:58.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:57 smithi078 bash[27336]: audit 2023-10-01T10:10:57.298222+0000 mon.a (mon.0) 421 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T10:10:58.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:57 smithi078 bash[33821]: audit 2023-10-01T10:10:57.289345+0000 mon.b (mon.1) 11 : audit [INF] from='client.? 172.21.15.173:0/3793157739' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c62aa7c3-2d09-4088-ba36-c84588dc3549"}]: dispatch 2023-10-01T10:10:58.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:57 smithi078 bash[33821]: audit 2023-10-01T10:10:57.290138+0000 mon.a (mon.0) 418 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c62aa7c3-2d09-4088-ba36-c84588dc3549"}]: dispatch 2023-10-01T10:10:58.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:57 smithi078 bash[33821]: audit 2023-10-01T10:10:57.297891+0000 mon.a (mon.0) 419 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c62aa7c3-2d09-4088-ba36-c84588dc3549"}]': finished 2023-10-01T10:10:58.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:57 smithi078 bash[33821]: cluster 2023-10-01T10:10:57.297983+0000 mon.a (mon.0) 420 : cluster [DBG] osdmap e33: 6 total, 5 up, 6 in 2023-10-01T10:10:58.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:57 smithi078 bash[33821]: audit 2023-10-01T10:10:57.298222+0000 mon.a (mon.0) 421 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T10:10:58.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:57 smithi173 bash[27612]: audit 2023-10-01T10:10:57.289345+0000 mon.b (mon.1) 11 : audit [INF] from='client.? 172.21.15.173:0/3793157739' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c62aa7c3-2d09-4088-ba36-c84588dc3549"}]: dispatch 2023-10-01T10:10:58.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:57 smithi173 bash[27612]: audit 2023-10-01T10:10:57.290138+0000 mon.a (mon.0) 418 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c62aa7c3-2d09-4088-ba36-c84588dc3549"}]: dispatch 2023-10-01T10:10:58.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:57 smithi173 bash[27612]: audit 2023-10-01T10:10:57.297891+0000 mon.a (mon.0) 419 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c62aa7c3-2d09-4088-ba36-c84588dc3549"}]': finished 2023-10-01T10:10:58.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:57 smithi173 bash[27612]: cluster 2023-10-01T10:10:57.297983+0000 mon.a (mon.0) 420 : cluster [DBG] osdmap e33: 6 total, 5 up, 6 in 2023-10-01T10:10:58.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:57 smithi173 bash[27612]: audit 2023-10-01T10:10:57.298222+0000 mon.a (mon.0) 421 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T10:10:59.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:58 smithi078 bash[27336]: cluster 2023-10-01T10:10:57.809126+0000 mgr.y (mgr.14146) 191 : cluster [DBG] pgmap v173: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T10:10:59.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:10:58 smithi078 bash[27336]: audit 2023-10-01T10:10:58.025295+0000 mon.b (mon.1) 12 : audit [DBG] from='client.? 172.21.15.173:0/2925991798' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T10:10:59.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:58 smithi078 bash[33821]: cluster 2023-10-01T10:10:57.809126+0000 mgr.y (mgr.14146) 191 : cluster [DBG] pgmap v173: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T10:10:59.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:10:58 smithi078 bash[33821]: audit 2023-10-01T10:10:58.025295+0000 mon.b (mon.1) 12 : audit [DBG] from='client.? 172.21.15.173:0/2925991798' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T10:10:59.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:58 smithi173 bash[27612]: cluster 2023-10-01T10:10:57.809126+0000 mgr.y (mgr.14146) 191 : cluster [DBG] pgmap v173: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T10:10:59.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:10:58 smithi173 bash[27612]: audit 2023-10-01T10:10:58.025295+0000 mon.b (mon.1) 12 : audit [DBG] from='client.? 172.21.15.173:0/2925991798' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T10:11:01.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:00 smithi173 bash[27612]: cluster 2023-10-01T10:10:59.809683+0000 mgr.y (mgr.14146) 192 : cluster [DBG] pgmap v174: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T10:11:01.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:00 smithi078 bash[27336]: cluster 2023-10-01T10:10:59.809683+0000 mgr.y (mgr.14146) 192 : cluster [DBG] pgmap v174: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T10:11:01.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:00 smithi078 bash[33821]: cluster 2023-10-01T10:10:59.809683+0000 mgr.y (mgr.14146) 192 : cluster [DBG] pgmap v174: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T10:11:03.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:02 smithi078 bash[27336]: cluster 2023-10-01T10:11:01.810231+0000 mgr.y (mgr.14146) 193 : cluster [DBG] pgmap v175: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T10:11:03.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:02 smithi078 bash[33821]: cluster 2023-10-01T10:11:01.810231+0000 mgr.y (mgr.14146) 193 : cluster [DBG] pgmap v175: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T10:11:03.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:02 smithi173 bash[27612]: cluster 2023-10-01T10:11:01.810231+0000 mgr.y (mgr.14146) 193 : cluster [DBG] pgmap v175: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T10:11:05.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:04 smithi078 bash[27336]: cluster 2023-10-01T10:11:03.810825+0000 mgr.y (mgr.14146) 194 : cluster [DBG] pgmap v176: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T10:11:05.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:04 smithi078 bash[33821]: cluster 2023-10-01T10:11:03.810825+0000 mgr.y (mgr.14146) 194 : cluster [DBG] pgmap v176: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T10:11:05.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:04 smithi173 bash[27612]: cluster 2023-10-01T10:11:03.810825+0000 mgr.y (mgr.14146) 194 : cluster [DBG] pgmap v176: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T10:11:07.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:06 smithi078 bash[27336]: cluster 2023-10-01T10:11:05.811397+0000 mgr.y (mgr.14146) 195 : cluster [DBG] pgmap v177: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T10:11:07.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:06 smithi078 bash[33821]: cluster 2023-10-01T10:11:05.811397+0000 mgr.y (mgr.14146) 195 : cluster [DBG] pgmap v177: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T10:11:07.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:06 smithi173 bash[27612]: cluster 2023-10-01T10:11:05.811397+0000 mgr.y (mgr.14146) 195 : cluster [DBG] pgmap v177: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T10:11:09.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:08 smithi173 bash[27612]: cluster 2023-10-01T10:11:07.811942+0000 mgr.y (mgr.14146) 196 : cluster [DBG] pgmap v178: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T10:11:09.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:08 smithi078 bash[27336]: cluster 2023-10-01T10:11:07.811942+0000 mgr.y (mgr.14146) 196 : cluster [DBG] pgmap v178: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T10:11:09.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:08 smithi078 bash[33821]: cluster 2023-10-01T10:11:07.811942+0000 mgr.y (mgr.14146) 196 : cluster [DBG] pgmap v178: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T10:11:10.322 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:09 smithi173 bash[27612]: audit 2023-10-01T10:11:09.342164+0000 mon.a (mon.0) 422 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-01T10:11:10.322 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:09 smithi173 bash[27612]: audit 2023-10-01T10:11:09.343772+0000 mon.a (mon.0) 423 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:11:10.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:09 smithi078 bash[27336]: audit 2023-10-01T10:11:09.342164+0000 mon.a (mon.0) 422 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-01T10:11:10.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:09 smithi078 bash[27336]: audit 2023-10-01T10:11:09.343772+0000 mon.a (mon.0) 423 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:11:10.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:09 smithi078 bash[33821]: audit 2023-10-01T10:11:09.342164+0000 mon.a (mon.0) 422 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-01T10:11:10.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:09 smithi078 bash[33821]: audit 2023-10-01T10:11:09.343772+0000 mon.a (mon.0) 423 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:11:11.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:10 smithi173 bash[27612]: cephadm 2023-10-01T10:11:09.345145+0000 mgr.y (mgr.14146) 197 : cephadm [INF] Deploying daemon osd.5 on smithi173 2023-10-01T10:11:11.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:10 smithi173 bash[27612]: cluster 2023-10-01T10:11:09.812502+0000 mgr.y (mgr.14146) 198 : cluster [DBG] pgmap v179: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T10:11:11.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:10 smithi078 bash[27336]: cephadm 2023-10-01T10:11:09.345145+0000 mgr.y (mgr.14146) 197 : cephadm [INF] Deploying daemon osd.5 on smithi173 2023-10-01T10:11:11.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:10 smithi078 bash[27336]: cluster 2023-10-01T10:11:09.812502+0000 mgr.y (mgr.14146) 198 : cluster [DBG] pgmap v179: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T10:11:11.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:10 smithi078 bash[33821]: cephadm 2023-10-01T10:11:09.345145+0000 mgr.y (mgr.14146) 197 : cephadm [INF] Deploying daemon osd.5 on smithi173 2023-10-01T10:11:11.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:10 smithi078 bash[33821]: cluster 2023-10-01T10:11:09.812502+0000 mgr.y (mgr.14146) 198 : cluster [DBG] pgmap v179: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T10:11:13.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:12 smithi078 bash[27336]: cluster 2023-10-01T10:11:11.813062+0000 mgr.y (mgr.14146) 199 : cluster [DBG] pgmap v180: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T10:11:13.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:12 smithi078 bash[33821]: cluster 2023-10-01T10:11:11.813062+0000 mgr.y (mgr.14146) 199 : cluster [DBG] pgmap v180: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T10:11:13.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:12 smithi173 bash[27612]: cluster 2023-10-01T10:11:11.813062+0000 mgr.y (mgr.14146) 199 : cluster [DBG] pgmap v180: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T10:11:14.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:14 smithi173 bash[27612]: audit 2023-10-01T10:11:13.465072+0000 mon.a (mon.0) 424 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:11:14.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:14 smithi173 bash[27612]: audit 2023-10-01T10:11:13.490825+0000 mon.a (mon.0) 425 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:11:14.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:14 smithi173 bash[27612]: cluster 2023-10-01T10:11:13.813666+0000 mgr.y (mgr.14146) 200 : cluster [DBG] pgmap v181: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T10:11:14.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:14 smithi078 bash[27336]: audit 2023-10-01T10:11:13.465072+0000 mon.a (mon.0) 424 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:11:14.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:14 smithi078 bash[27336]: audit 2023-10-01T10:11:13.490825+0000 mon.a (mon.0) 425 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:11:14.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:14 smithi078 bash[27336]: cluster 2023-10-01T10:11:13.813666+0000 mgr.y (mgr.14146) 200 : cluster [DBG] pgmap v181: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T10:11:14.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:14 smithi078 bash[33821]: audit 2023-10-01T10:11:13.465072+0000 mon.a (mon.0) 424 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:11:14.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:14 smithi078 bash[33821]: audit 2023-10-01T10:11:13.490825+0000 mon.a (mon.0) 425 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:11:14.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:14 smithi078 bash[33821]: cluster 2023-10-01T10:11:13.813666+0000 mgr.y (mgr.14146) 200 : cluster [DBG] pgmap v181: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T10:11:17.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:16 smithi078 bash[27336]: cluster 2023-10-01T10:11:15.814321+0000 mgr.y (mgr.14146) 201 : cluster [DBG] pgmap v182: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T10:11:17.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:16 smithi078 bash[33821]: cluster 2023-10-01T10:11:15.814321+0000 mgr.y (mgr.14146) 201 : cluster [DBG] pgmap v182: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T10:11:17.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:16 smithi173 bash[27612]: cluster 2023-10-01T10:11:15.814321+0000 mgr.y (mgr.14146) 201 : cluster [DBG] pgmap v182: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T10:11:19.168 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:18 smithi173 bash[27612]: cluster 2023-10-01T10:11:17.814890+0000 mgr.y (mgr.14146) 202 : cluster [DBG] pgmap v183: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T10:11:19.168 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:18 smithi173 bash[27612]: audit 2023-10-01T10:11:18.584565+0000 mon.a (mon.0) 426 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:11:19.169 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:18 smithi173 bash[27612]: audit 2023-10-01T10:11:18.586039+0000 mon.a (mon.0) 427 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:11:19.169 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:18 smithi173 bash[27612]: audit 2023-10-01T10:11:18.587403+0000 mon.a (mon.0) 428 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:11:19.169 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:18 smithi173 bash[27612]: audit 2023-10-01T10:11:18.593924+0000 mon.a (mon.0) 429 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:11:19.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:18 smithi078 bash[27336]: cluster 2023-10-01T10:11:17.814890+0000 mgr.y (mgr.14146) 202 : cluster [DBG] pgmap v183: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T10:11:19.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:18 smithi078 bash[27336]: audit 2023-10-01T10:11:18.584565+0000 mon.a (mon.0) 426 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:11:19.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:18 smithi078 bash[27336]: audit 2023-10-01T10:11:18.586039+0000 mon.a (mon.0) 427 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:11:19.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:18 smithi078 bash[27336]: audit 2023-10-01T10:11:18.587403+0000 mon.a (mon.0) 428 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:11:19.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:18 smithi078 bash[27336]: audit 2023-10-01T10:11:18.593924+0000 mon.a (mon.0) 429 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:11:19.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:18 smithi078 bash[33821]: cluster 2023-10-01T10:11:17.814890+0000 mgr.y (mgr.14146) 202 : cluster [DBG] pgmap v183: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T10:11:19.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:18 smithi078 bash[33821]: audit 2023-10-01T10:11:18.584565+0000 mon.a (mon.0) 426 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:11:19.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:18 smithi078 bash[33821]: audit 2023-10-01T10:11:18.586039+0000 mon.a (mon.0) 427 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:11:19.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:18 smithi078 bash[33821]: audit 2023-10-01T10:11:18.587403+0000 mon.a (mon.0) 428 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:11:19.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:18 smithi078 bash[33821]: audit 2023-10-01T10:11:18.593924+0000 mon.a (mon.0) 429 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:11:21.227 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:20 smithi173 bash[27612]: cluster 2023-10-01T10:11:19.815425+0000 mgr.y (mgr.14146) 203 : cluster [DBG] pgmap v184: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T10:11:21.227 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:20 smithi173 bash[27612]: audit 2023-10-01T10:11:19.924126+0000 mon.b (mon.1) 13 : audit [INF] from='osd.5 [v2:172.21.15.173:6808/2798643181,v1:172.21.15.173:6809/2798643181]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-01T10:11:21.227 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:20 smithi173 bash[27612]: audit 2023-10-01T10:11:19.924337+0000 mon.a (mon.0) 430 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-01T10:11:21.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:20 smithi078 bash[27336]: cluster 2023-10-01T10:11:19.815425+0000 mgr.y (mgr.14146) 203 : cluster [DBG] pgmap v184: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T10:11:21.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:20 smithi078 bash[27336]: audit 2023-10-01T10:11:19.924126+0000 mon.b (mon.1) 13 : audit [INF] from='osd.5 [v2:172.21.15.173:6808/2798643181,v1:172.21.15.173:6809/2798643181]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-01T10:11:21.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:20 smithi078 bash[27336]: audit 2023-10-01T10:11:19.924337+0000 mon.a (mon.0) 430 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-01T10:11:21.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:20 smithi078 bash[33821]: cluster 2023-10-01T10:11:19.815425+0000 mgr.y (mgr.14146) 203 : cluster [DBG] pgmap v184: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T10:11:21.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:20 smithi078 bash[33821]: audit 2023-10-01T10:11:19.924126+0000 mon.b (mon.1) 13 : audit [INF] from='osd.5 [v2:172.21.15.173:6808/2798643181,v1:172.21.15.173:6809/2798643181]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-01T10:11:21.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:20 smithi078 bash[33821]: audit 2023-10-01T10:11:19.924337+0000 mon.a (mon.0) 430 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-10-01T10:11:21.712 INFO:teuthology.orchestra.run.smithi173.stdout:Created osd(s) 5 on host 'smithi173' 2023-10-01T10:11:22.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:21 smithi173 bash[27612]: audit 2023-10-01T10:11:20.900529+0000 mon.a (mon.0) 431 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-10-01T10:11:22.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:21 smithi173 bash[27612]: cluster 2023-10-01T10:11:20.900655+0000 mon.a (mon.0) 432 : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2023-10-01T10:11:22.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:21 smithi173 bash[27612]: audit 2023-10-01T10:11:20.900887+0000 mon.a (mon.0) 433 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T10:11:22.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:21 smithi173 bash[27612]: audit 2023-10-01T10:11:20.902452+0000 mon.a (mon.0) 434 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-10-01T10:11:22.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:21 smithi173 bash[27612]: audit 2023-10-01T10:11:20.902500+0000 mon.b (mon.1) 14 : audit [INF] from='osd.5 [v2:172.21.15.173:6808/2798643181,v1:172.21.15.173:6809/2798643181]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-10-01T10:11:22.122 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:21 smithi173 bash[27612]: audit 2023-10-01T10:11:21.709595+0000 mon.a (mon.0) 435 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:11:22.122 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:21 smithi173 bash[27612]: audit 2023-10-01T10:11:21.720213+0000 mon.a (mon.0) 436 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:11:22.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:21 smithi078 bash[27336]: audit 2023-10-01T10:11:20.900529+0000 mon.a (mon.0) 431 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-10-01T10:11:22.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:21 smithi078 bash[27336]: cluster 2023-10-01T10:11:20.900655+0000 mon.a (mon.0) 432 : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2023-10-01T10:11:22.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:21 smithi078 bash[27336]: audit 2023-10-01T10:11:20.900887+0000 mon.a (mon.0) 433 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T10:11:22.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:21 smithi078 bash[27336]: audit 2023-10-01T10:11:20.902452+0000 mon.a (mon.0) 434 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-10-01T10:11:22.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:21 smithi078 bash[27336]: audit 2023-10-01T10:11:20.902500+0000 mon.b (mon.1) 14 : audit [INF] from='osd.5 [v2:172.21.15.173:6808/2798643181,v1:172.21.15.173:6809/2798643181]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-10-01T10:11:22.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:21 smithi078 bash[27336]: audit 2023-10-01T10:11:21.709595+0000 mon.a (mon.0) 435 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:11:22.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:21 smithi078 bash[27336]: audit 2023-10-01T10:11:21.720213+0000 mon.a (mon.0) 436 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:11:22.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:21 smithi078 bash[33821]: audit 2023-10-01T10:11:20.900529+0000 mon.a (mon.0) 431 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-10-01T10:11:22.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:21 smithi078 bash[33821]: cluster 2023-10-01T10:11:20.900655+0000 mon.a (mon.0) 432 : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2023-10-01T10:11:22.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:21 smithi078 bash[33821]: audit 2023-10-01T10:11:20.900887+0000 mon.a (mon.0) 433 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T10:11:22.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:21 smithi078 bash[33821]: audit 2023-10-01T10:11:20.902452+0000 mon.a (mon.0) 434 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-10-01T10:11:22.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:21 smithi078 bash[33821]: audit 2023-10-01T10:11:20.902500+0000 mon.b (mon.1) 14 : audit [INF] from='osd.5 [v2:172.21.15.173:6808/2798643181,v1:172.21.15.173:6809/2798643181]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-10-01T10:11:22.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:21 smithi078 bash[33821]: audit 2023-10-01T10:11:21.709595+0000 mon.a (mon.0) 435 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:11:22.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:21 smithi078 bash[33821]: audit 2023-10-01T10:11:21.720213+0000 mon.a (mon.0) 436 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:11:22.381 DEBUG:teuthology.orchestra.run.smithi173:osd.5> sudo journalctl -f -n 0 -u ceph-d8016480-6041-11ee-8db5-212e2dc638e7@osd.5.service 2023-10-01T10:11:22.387 INFO:tasks.cephadm:Deploying osd.6 on smithi173 with /dev/vg_nvme/lv_2... 2023-10-01T10:11:22.387 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-10-01T10:11:22.606 INFO:journalctl@ceph.osd.5.smithi173.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-01T10:11:22.922 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:22 smithi173 bash[27612]: cluster 2023-10-01T10:11:21.815977+0000 mgr.y (mgr.14146) 204 : cluster [DBG] pgmap v186: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T10:11:22.922 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:22 smithi173 bash[27612]: audit 2023-10-01T10:11:21.905050+0000 mon.a (mon.0) 437 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]': finished 2023-10-01T10:11:23.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:22 smithi078 bash[33821]: cluster 2023-10-01T10:11:21.815977+0000 mgr.y (mgr.14146) 204 : cluster [DBG] pgmap v186: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T10:11:23.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:22 smithi078 bash[33821]: audit 2023-10-01T10:11:21.905050+0000 mon.a (mon.0) 437 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]': finished 2023-10-01T10:11:23.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:22 smithi078 bash[33821]: cluster 2023-10-01T10:11:21.905368+0000 mon.a (mon.0) 438 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2023-10-01T10:11:23.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:22 smithi078 bash[33821]: audit 2023-10-01T10:11:21.906272+0000 mon.a (mon.0) 439 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T10:11:23.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:22 smithi078 bash[33821]: audit 2023-10-01T10:11:21.913906+0000 mon.a (mon.0) 440 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T10:11:23.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:22 smithi078 bash[27336]: cluster 2023-10-01T10:11:21.815977+0000 mgr.y (mgr.14146) 204 : cluster [DBG] pgmap v186: 1 pgs: 1 active+clean; 0 B data, 1.4 GiB used, 446 GiB / 447 GiB avail 2023-10-01T10:11:23.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:22 smithi078 bash[27336]: audit 2023-10-01T10:11:21.905050+0000 mon.a (mon.0) 437 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]': finished 2023-10-01T10:11:23.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:22 smithi078 bash[27336]: cluster 2023-10-01T10:11:21.905368+0000 mon.a (mon.0) 438 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2023-10-01T10:11:23.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:22 smithi078 bash[27336]: audit 2023-10-01T10:11:21.906272+0000 mon.a (mon.0) 439 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T10:11:23.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:22 smithi078 bash[27336]: audit 2023-10-01T10:11:21.913906+0000 mon.a (mon.0) 440 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T10:11:23.351 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:22 smithi173 bash[27612]: cluster 2023-10-01T10:11:21.905368+0000 mon.a (mon.0) 438 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2023-10-01T10:11:23.351 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:22 smithi173 bash[27612]: audit 2023-10-01T10:11:21.906272+0000 mon.a (mon.0) 439 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T10:11:23.351 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:22 smithi173 bash[27612]: audit 2023-10-01T10:11:21.913906+0000 mon.a (mon.0) 440 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T10:11:24.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:23 smithi173 bash[27612]: cluster 2023-10-01T10:11:20.934679+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-01T10:11:24.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:23 smithi173 bash[27612]: cluster 2023-10-01T10:11:20.934730+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-01T10:11:24.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:23 smithi173 bash[27612]: cluster 2023-10-01T10:11:22.912467+0000 mon.a (mon.0) 441 : cluster [INF] osd.5 [v2:172.21.15.173:6808/2798643181,v1:172.21.15.173:6809/2798643181] boot 2023-10-01T10:11:24.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:23 smithi173 bash[27612]: cluster 2023-10-01T10:11:22.912529+0000 mon.a (mon.0) 442 : cluster [DBG] osdmap e36: 6 total, 6 up, 6 in 2023-10-01T10:11:24.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:23 smithi173 bash[27612]: audit 2023-10-01T10:11:22.913165+0000 mon.a (mon.0) 443 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T10:11:24.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:23 smithi173 bash[27612]: cluster 2023-10-01T10:11:23.910971+0000 mon.a (mon.0) 444 : cluster [DBG] osdmap e37: 6 total, 6 up, 6 in 2023-10-01T10:11:24.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:23 smithi078 bash[27336]: cluster 2023-10-01T10:11:20.934679+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-01T10:11:24.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:23 smithi078 bash[27336]: cluster 2023-10-01T10:11:20.934730+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-01T10:11:24.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:23 smithi078 bash[27336]: cluster 2023-10-01T10:11:22.912467+0000 mon.a (mon.0) 441 : cluster [INF] osd.5 [v2:172.21.15.173:6808/2798643181,v1:172.21.15.173:6809/2798643181] boot 2023-10-01T10:11:24.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:23 smithi078 bash[27336]: cluster 2023-10-01T10:11:22.912529+0000 mon.a (mon.0) 442 : cluster [DBG] osdmap e36: 6 total, 6 up, 6 in 2023-10-01T10:11:24.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:23 smithi078 bash[27336]: audit 2023-10-01T10:11:22.913165+0000 mon.a (mon.0) 443 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T10:11:24.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:23 smithi078 bash[27336]: cluster 2023-10-01T10:11:23.910971+0000 mon.a (mon.0) 444 : cluster [DBG] osdmap e37: 6 total, 6 up, 6 in 2023-10-01T10:11:24.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:23 smithi078 bash[33821]: cluster 2023-10-01T10:11:20.934679+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-01T10:11:24.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:23 smithi078 bash[33821]: cluster 2023-10-01T10:11:20.934730+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-01T10:11:24.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:23 smithi078 bash[33821]: cluster 2023-10-01T10:11:22.912467+0000 mon.a (mon.0) 441 : cluster [INF] osd.5 [v2:172.21.15.173:6808/2798643181,v1:172.21.15.173:6809/2798643181] boot 2023-10-01T10:11:24.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:23 smithi078 bash[33821]: cluster 2023-10-01T10:11:22.912529+0000 mon.a (mon.0) 442 : cluster [DBG] osdmap e36: 6 total, 6 up, 6 in 2023-10-01T10:11:24.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:23 smithi078 bash[33821]: audit 2023-10-01T10:11:22.913165+0000 mon.a (mon.0) 443 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T10:11:24.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:23 smithi078 bash[33821]: cluster 2023-10-01T10:11:23.910971+0000 mon.a (mon.0) 444 : cluster [DBG] osdmap e37: 6 total, 6 up, 6 in 2023-10-01T10:11:25.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:24 smithi173 bash[27612]: cluster 2023-10-01T10:11:23.816538+0000 mgr.y (mgr.14146) 205 : cluster [DBG] pgmap v189: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T10:11:25.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:24 smithi173 bash[27612]: cluster 2023-10-01T10:11:24.914638+0000 mon.a (mon.0) 445 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2023-10-01T10:11:25.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:24 smithi078 bash[27336]: cluster 2023-10-01T10:11:23.816538+0000 mgr.y (mgr.14146) 205 : cluster [DBG] pgmap v189: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T10:11:25.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:24 smithi078 bash[27336]: cluster 2023-10-01T10:11:24.914638+0000 mon.a (mon.0) 445 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2023-10-01T10:11:25.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:24 smithi078 bash[33821]: cluster 2023-10-01T10:11:23.816538+0000 mgr.y (mgr.14146) 205 : cluster [DBG] pgmap v189: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T10:11:25.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:24 smithi078 bash[33821]: cluster 2023-10-01T10:11:24.914638+0000 mon.a (mon.0) 445 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2023-10-01T10:11:25.537 INFO:teuthology.orchestra.run.smithi173.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-10-01T10:11:25.537 INFO:teuthology.orchestra.run.smithi173.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-10-01T10:11:25.537 INFO:teuthology.orchestra.run.smithi173.stderr: stderr: 10+0 records in 2023-10-01T10:11:25.537 INFO:teuthology.orchestra.run.smithi173.stderr:10+0 records out 2023-10-01T10:11:25.537 INFO:teuthology.orchestra.run.smithi173.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0164891 s, 636 MB/s 2023-10-01T10:11:25.537 INFO:teuthology.orchestra.run.smithi173.stderr:--> Zapping successful for: 2023-10-01T10:11:26.225 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi173:vg_nvme/lv_2 2023-10-01T10:11:27.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:26 smithi173 bash[27612]: cluster 2023-10-01T10:11:25.817125+0000 mgr.y (mgr.14146) 206 : cluster [DBG] pgmap v192: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T10:11:27.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:26 smithi078 bash[27336]: cluster 2023-10-01T10:11:25.817125+0000 mgr.y (mgr.14146) 206 : cluster [DBG] pgmap v192: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T10:11:27.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:26 smithi078 bash[33821]: cluster 2023-10-01T10:11:25.817125+0000 mgr.y (mgr.14146) 206 : cluster [DBG] pgmap v192: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T10:11:29.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:28 smithi078 bash[27336]: cluster 2023-10-01T10:11:27.817738+0000 mgr.y (mgr.14146) 207 : cluster [DBG] pgmap v193: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T10:11:29.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:28 smithi078 bash[33821]: cluster 2023-10-01T10:11:27.817738+0000 mgr.y (mgr.14146) 207 : cluster [DBG] pgmap v193: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T10:11:29.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:28 smithi173 bash[27612]: cluster 2023-10-01T10:11:27.817738+0000 mgr.y (mgr.14146) 207 : cluster [DBG] pgmap v193: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T10:11:31.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:30 smithi173 bash[27612]: cluster 2023-10-01T10:11:29.818448+0000 mgr.y (mgr.14146) 208 : cluster [DBG] pgmap v194: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T10:11:31.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:30 smithi173 bash[27612]: audit 2023-10-01T10:11:30.437381+0000 mon.a (mon.0) 446 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T10:11:31.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:30 smithi173 bash[27612]: audit 2023-10-01T10:11:30.442806+0000 mon.a (mon.0) 447 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T10:11:31.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:30 smithi173 bash[27612]: audit 2023-10-01T10:11:30.444220+0000 mon.a (mon.0) 448 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:11:31.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:30 smithi078 bash[27336]: cluster 2023-10-01T10:11:29.818448+0000 mgr.y (mgr.14146) 208 : cluster [DBG] pgmap v194: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T10:11:31.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:30 smithi078 bash[27336]: audit 2023-10-01T10:11:30.437381+0000 mon.a (mon.0) 446 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T10:11:31.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:30 smithi078 bash[27336]: audit 2023-10-01T10:11:30.442806+0000 mon.a (mon.0) 447 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T10:11:31.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:30 smithi078 bash[27336]: audit 2023-10-01T10:11:30.444220+0000 mon.a (mon.0) 448 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:11:31.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:30 smithi078 bash[33821]: cluster 2023-10-01T10:11:29.818448+0000 mgr.y (mgr.14146) 208 : cluster [DBG] pgmap v194: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T10:11:31.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:30 smithi078 bash[33821]: audit 2023-10-01T10:11:30.437381+0000 mon.a (mon.0) 446 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T10:11:31.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:30 smithi078 bash[33821]: audit 2023-10-01T10:11:30.442806+0000 mon.a (mon.0) 447 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T10:11:31.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:30 smithi078 bash[33821]: audit 2023-10-01T10:11:30.444220+0000 mon.a (mon.0) 448 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:11:32.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:31 smithi078 bash[33821]: audit 2023-10-01T10:11:30.430864+0000 mgr.y (mgr.14146) 209 : audit [DBG] from='client.24268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi173:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:11:32.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:31 smithi078 bash[33821]: audit 2023-10-01T10:11:31.636879+0000 mon.a (mon.0) 449 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:11:32.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:31 smithi078 bash[33821]: audit 2023-10-01T10:11:31.638753+0000 mon.a (mon.0) 450 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:11:32.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:31 smithi078 bash[33821]: audit 2023-10-01T10:11:31.640180+0000 mon.a (mon.0) 451 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:11:32.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:31 smithi078 bash[33821]: audit 2023-10-01T10:11:31.646312+0000 mon.a (mon.0) 452 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:11:32.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:31 smithi078 bash[33821]: audit 2023-10-01T10:11:31.647972+0000 mon.a (mon.0) 453 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:11:32.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:31 smithi078 bash[33821]: audit 2023-10-01T10:11:31.649176+0000 mon.a (mon.0) 454 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:11:32.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:31 smithi078 bash[33821]: audit 2023-10-01T10:11:31.657179+0000 mon.a (mon.0) 455 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:11:32.336 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:31 smithi078 bash[27336]: audit 2023-10-01T10:11:30.430864+0000 mgr.y (mgr.14146) 209 : audit [DBG] from='client.24268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi173:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:11:32.336 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:31 smithi078 bash[27336]: audit 2023-10-01T10:11:31.636879+0000 mon.a (mon.0) 449 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:11:32.336 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:31 smithi078 bash[27336]: audit 2023-10-01T10:11:31.638753+0000 mon.a (mon.0) 450 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:11:32.336 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:31 smithi078 bash[27336]: audit 2023-10-01T10:11:31.640180+0000 mon.a (mon.0) 451 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:11:32.337 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:31 smithi078 bash[27336]: audit 2023-10-01T10:11:31.646312+0000 mon.a (mon.0) 452 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:11:32.337 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:31 smithi078 bash[27336]: audit 2023-10-01T10:11:31.647972+0000 mon.a (mon.0) 453 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:11:32.337 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:31 smithi078 bash[27336]: audit 2023-10-01T10:11:31.649176+0000 mon.a (mon.0) 454 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:11:32.337 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:31 smithi078 bash[27336]: audit 2023-10-01T10:11:31.657179+0000 mon.a (mon.0) 455 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:11:32.373 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:31 smithi173 bash[27612]: audit 2023-10-01T10:11:30.430864+0000 mgr.y (mgr.14146) 209 : audit [DBG] from='client.24268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi173:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:11:32.373 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:31 smithi173 bash[27612]: audit 2023-10-01T10:11:31.636879+0000 mon.a (mon.0) 449 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:11:32.373 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:31 smithi173 bash[27612]: audit 2023-10-01T10:11:31.638753+0000 mon.a (mon.0) 450 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:11:32.373 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:31 smithi173 bash[27612]: audit 2023-10-01T10:11:31.640180+0000 mon.a (mon.0) 451 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:11:32.373 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:31 smithi173 bash[27612]: audit 2023-10-01T10:11:31.646312+0000 mon.a (mon.0) 452 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:11:32.373 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:31 smithi173 bash[27612]: audit 2023-10-01T10:11:31.647972+0000 mon.a (mon.0) 453 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:11:32.373 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:31 smithi173 bash[27612]: audit 2023-10-01T10:11:31.649176+0000 mon.a (mon.0) 454 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:11:32.373 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:31 smithi173 bash[27612]: audit 2023-10-01T10:11:31.657179+0000 mon.a (mon.0) 455 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:11:33.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:32 smithi078 bash[27336]: cephadm 2023-10-01T10:11:31.627255+0000 mgr.y (mgr.14146) 210 : cephadm [INF] Detected new or changed devices on smithi173 2023-10-01T10:11:33.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:32 smithi078 bash[27336]: cephadm 2023-10-01T10:11:31.641271+0000 mgr.y (mgr.14146) 211 : cephadm [INF] Adjusting osd_memory_target on smithi173 to 8632M 2023-10-01T10:11:33.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:32 smithi078 bash[27336]: cluster 2023-10-01T10:11:31.819035+0000 mgr.y (mgr.14146) 212 : cluster [DBG] pgmap v195: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T10:11:33.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:32 smithi078 bash[33821]: cephadm 2023-10-01T10:11:31.627255+0000 mgr.y (mgr.14146) 210 : cephadm [INF] Detected new or changed devices on smithi173 2023-10-01T10:11:33.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:32 smithi078 bash[33821]: cephadm 2023-10-01T10:11:31.641271+0000 mgr.y (mgr.14146) 211 : cephadm [INF] Adjusting osd_memory_target on smithi173 to 8632M 2023-10-01T10:11:33.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:32 smithi078 bash[33821]: cluster 2023-10-01T10:11:31.819035+0000 mgr.y (mgr.14146) 212 : cluster [DBG] pgmap v195: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T10:11:33.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:32 smithi173 bash[27612]: cephadm 2023-10-01T10:11:31.627255+0000 mgr.y (mgr.14146) 210 : cephadm [INF] Detected new or changed devices on smithi173 2023-10-01T10:11:33.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:32 smithi173 bash[27612]: cephadm 2023-10-01T10:11:31.641271+0000 mgr.y (mgr.14146) 211 : cephadm [INF] Adjusting osd_memory_target on smithi173 to 8632M 2023-10-01T10:11:33.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:32 smithi173 bash[27612]: cluster 2023-10-01T10:11:31.819035+0000 mgr.y (mgr.14146) 212 : cluster [DBG] pgmap v195: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T10:11:35.208 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:34 smithi173 bash[27612]: cluster 2023-10-01T10:11:33.819605+0000 mgr.y (mgr.14146) 213 : cluster [DBG] pgmap v196: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T10:11:35.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:34 smithi078 bash[27336]: cluster 2023-10-01T10:11:33.819605+0000 mgr.y (mgr.14146) 213 : cluster [DBG] pgmap v196: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T10:11:35.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:34 smithi078 bash[33821]: cluster 2023-10-01T10:11:33.819605+0000 mgr.y (mgr.14146) 213 : cluster [DBG] pgmap v196: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T10:11:37.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:36 smithi078 bash[27336]: cluster 2023-10-01T10:11:35.820233+0000 mgr.y (mgr.14146) 214 : cluster [DBG] pgmap v197: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T10:11:37.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:36 smithi078 bash[33821]: cluster 2023-10-01T10:11:35.820233+0000 mgr.y (mgr.14146) 214 : cluster [DBG] pgmap v197: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T10:11:37.336 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:36 smithi173 bash[27612]: cluster 2023-10-01T10:11:35.820233+0000 mgr.y (mgr.14146) 214 : cluster [DBG] pgmap v197: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T10:11:39.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:38 smithi078 bash[27336]: cluster 2023-10-01T10:11:37.820811+0000 mgr.y (mgr.14146) 215 : cluster [DBG] pgmap v198: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T10:11:39.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:38 smithi078 bash[27336]: audit 2023-10-01T10:11:38.222536+0000 mon.b (mon.1) 15 : audit [INF] from='client.? 172.21.15.173:0/1493546736' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b140330d-dd93-4ce4-9b9a-2ec6321419b3"}]: dispatch 2023-10-01T10:11:39.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:38 smithi078 bash[27336]: audit 2023-10-01T10:11:38.222850+0000 mon.a (mon.0) 456 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b140330d-dd93-4ce4-9b9a-2ec6321419b3"}]: dispatch 2023-10-01T10:11:39.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:38 smithi078 bash[27336]: audit 2023-10-01T10:11:38.235111+0000 mon.a (mon.0) 457 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b140330d-dd93-4ce4-9b9a-2ec6321419b3"}]': finished 2023-10-01T10:11:39.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:38 smithi078 bash[27336]: cluster 2023-10-01T10:11:38.235263+0000 mon.a (mon.0) 458 : cluster [DBG] osdmap e39: 7 total, 6 up, 7 in 2023-10-01T10:11:39.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:38 smithi078 bash[27336]: audit 2023-10-01T10:11:38.235442+0000 mon.a (mon.0) 459 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T10:11:39.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:38 smithi078 bash[27336]: audit 2023-10-01T10:11:38.914205+0000 mon.b (mon.1) 16 : audit [DBG] from='client.? 172.21.15.173:0/770913639' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T10:11:39.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:38 smithi078 bash[33821]: cluster 2023-10-01T10:11:37.820811+0000 mgr.y (mgr.14146) 215 : cluster [DBG] pgmap v198: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T10:11:39.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:38 smithi078 bash[33821]: audit 2023-10-01T10:11:38.222536+0000 mon.b (mon.1) 15 : audit [INF] from='client.? 172.21.15.173:0/1493546736' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b140330d-dd93-4ce4-9b9a-2ec6321419b3"}]: dispatch 2023-10-01T10:11:39.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:38 smithi078 bash[33821]: audit 2023-10-01T10:11:38.222850+0000 mon.a (mon.0) 456 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b140330d-dd93-4ce4-9b9a-2ec6321419b3"}]: dispatch 2023-10-01T10:11:39.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:38 smithi078 bash[33821]: audit 2023-10-01T10:11:38.235111+0000 mon.a (mon.0) 457 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b140330d-dd93-4ce4-9b9a-2ec6321419b3"}]': finished 2023-10-01T10:11:39.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:38 smithi078 bash[33821]: cluster 2023-10-01T10:11:38.235263+0000 mon.a (mon.0) 458 : cluster [DBG] osdmap e39: 7 total, 6 up, 7 in 2023-10-01T10:11:39.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:38 smithi078 bash[33821]: audit 2023-10-01T10:11:38.235442+0000 mon.a (mon.0) 459 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T10:11:39.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:38 smithi078 bash[33821]: audit 2023-10-01T10:11:38.914205+0000 mon.b (mon.1) 16 : audit [DBG] from='client.? 172.21.15.173:0/770913639' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T10:11:39.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:38 smithi173 bash[27612]: cluster 2023-10-01T10:11:37.820811+0000 mgr.y (mgr.14146) 215 : cluster [DBG] pgmap v198: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T10:11:39.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:38 smithi173 bash[27612]: audit 2023-10-01T10:11:38.222536+0000 mon.b (mon.1) 15 : audit [INF] from='client.? 172.21.15.173:0/1493546736' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b140330d-dd93-4ce4-9b9a-2ec6321419b3"}]: dispatch 2023-10-01T10:11:39.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:38 smithi173 bash[27612]: audit 2023-10-01T10:11:38.222850+0000 mon.a (mon.0) 456 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b140330d-dd93-4ce4-9b9a-2ec6321419b3"}]: dispatch 2023-10-01T10:11:39.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:38 smithi173 bash[27612]: audit 2023-10-01T10:11:38.235111+0000 mon.a (mon.0) 457 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b140330d-dd93-4ce4-9b9a-2ec6321419b3"}]': finished 2023-10-01T10:11:39.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:38 smithi173 bash[27612]: cluster 2023-10-01T10:11:38.235263+0000 mon.a (mon.0) 458 : cluster [DBG] osdmap e39: 7 total, 6 up, 7 in 2023-10-01T10:11:39.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:38 smithi173 bash[27612]: audit 2023-10-01T10:11:38.235442+0000 mon.a (mon.0) 459 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T10:11:39.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:38 smithi173 bash[27612]: audit 2023-10-01T10:11:38.914205+0000 mon.b (mon.1) 16 : audit [DBG] from='client.? 172.21.15.173:0/770913639' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T10:11:41.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:40 smithi078 bash[27336]: cluster 2023-10-01T10:11:39.821382+0000 mgr.y (mgr.14146) 216 : cluster [DBG] pgmap v200: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T10:11:41.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:40 smithi078 bash[33821]: cluster 2023-10-01T10:11:39.821382+0000 mgr.y (mgr.14146) 216 : cluster [DBG] pgmap v200: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T10:11:41.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:40 smithi173 bash[27612]: cluster 2023-10-01T10:11:39.821382+0000 mgr.y (mgr.14146) 216 : cluster [DBG] pgmap v200: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T10:11:43.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:42 smithi078 bash[27336]: cluster 2023-10-01T10:11:41.821888+0000 mgr.y (mgr.14146) 217 : cluster [DBG] pgmap v201: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T10:11:43.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:42 smithi078 bash[33821]: cluster 2023-10-01T10:11:41.821888+0000 mgr.y (mgr.14146) 217 : cluster [DBG] pgmap v201: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T10:11:43.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:42 smithi173 bash[27612]: cluster 2023-10-01T10:11:41.821888+0000 mgr.y (mgr.14146) 217 : cluster [DBG] pgmap v201: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T10:11:45.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:44 smithi078 bash[27336]: cluster 2023-10-01T10:11:43.822483+0000 mgr.y (mgr.14146) 218 : cluster [DBG] pgmap v202: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T10:11:45.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:44 smithi078 bash[33821]: cluster 2023-10-01T10:11:43.822483+0000 mgr.y (mgr.14146) 218 : cluster [DBG] pgmap v202: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T10:11:45.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:44 smithi173 bash[27612]: cluster 2023-10-01T10:11:43.822483+0000 mgr.y (mgr.14146) 218 : cluster [DBG] pgmap v202: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T10:11:47.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:46 smithi078 bash[27336]: cluster 2023-10-01T10:11:45.823029+0000 mgr.y (mgr.14146) 219 : cluster [DBG] pgmap v203: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T10:11:47.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:46 smithi078 bash[33821]: cluster 2023-10-01T10:11:45.823029+0000 mgr.y (mgr.14146) 219 : cluster [DBG] pgmap v203: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T10:11:47.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:46 smithi173 bash[27612]: cluster 2023-10-01T10:11:45.823029+0000 mgr.y (mgr.14146) 219 : cluster [DBG] pgmap v203: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T10:11:49.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:48 smithi078 bash[27336]: cluster 2023-10-01T10:11:47.823578+0000 mgr.y (mgr.14146) 220 : cluster [DBG] pgmap v204: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T10:11:49.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:48 smithi078 bash[33821]: cluster 2023-10-01T10:11:47.823578+0000 mgr.y (mgr.14146) 220 : cluster [DBG] pgmap v204: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T10:11:49.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:48 smithi173 bash[27612]: cluster 2023-10-01T10:11:47.823578+0000 mgr.y (mgr.14146) 220 : cluster [DBG] pgmap v204: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T10:11:51.308 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:50 smithi173 bash[27612]: cluster 2023-10-01T10:11:49.824148+0000 mgr.y (mgr.14146) 221 : cluster [DBG] pgmap v205: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T10:11:51.308 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:50 smithi173 bash[27612]: audit 2023-10-01T10:11:50.323706+0000 mon.a (mon.0) 460 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-01T10:11:51.309 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:50 smithi173 bash[27612]: audit 2023-10-01T10:11:50.325824+0000 mon.a (mon.0) 461 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:11:51.309 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:50 smithi173 bash[27612]: cephadm 2023-10-01T10:11:50.327398+0000 mgr.y (mgr.14146) 222 : cephadm [INF] Deploying daemon osd.6 on smithi173 2023-10-01T10:11:51.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:50 smithi078 bash[33821]: cluster 2023-10-01T10:11:49.824148+0000 mgr.y (mgr.14146) 221 : cluster [DBG] pgmap v205: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T10:11:51.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:50 smithi078 bash[33821]: audit 2023-10-01T10:11:50.323706+0000 mon.a (mon.0) 460 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-01T10:11:51.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:50 smithi078 bash[33821]: audit 2023-10-01T10:11:50.325824+0000 mon.a (mon.0) 461 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:11:51.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:50 smithi078 bash[33821]: cephadm 2023-10-01T10:11:50.327398+0000 mgr.y (mgr.14146) 222 : cephadm [INF] Deploying daemon osd.6 on smithi173 2023-10-01T10:11:51.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:50 smithi078 bash[27336]: cluster 2023-10-01T10:11:49.824148+0000 mgr.y (mgr.14146) 221 : cluster [DBG] pgmap v205: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T10:11:51.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:50 smithi078 bash[27336]: audit 2023-10-01T10:11:50.323706+0000 mon.a (mon.0) 460 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-01T10:11:51.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:50 smithi078 bash[27336]: audit 2023-10-01T10:11:50.325824+0000 mon.a (mon.0) 461 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:11:51.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:50 smithi078 bash[27336]: cephadm 2023-10-01T10:11:50.327398+0000 mgr.y (mgr.14146) 222 : cephadm [INF] Deploying daemon osd.6 on smithi173 2023-10-01T10:11:53.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:52 smithi078 bash[27336]: cluster 2023-10-01T10:11:51.824706+0000 mgr.y (mgr.14146) 223 : cluster [DBG] pgmap v206: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T10:11:53.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:52 smithi078 bash[33821]: cluster 2023-10-01T10:11:51.824706+0000 mgr.y (mgr.14146) 223 : cluster [DBG] pgmap v206: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T10:11:53.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:52 smithi173 bash[27612]: cluster 2023-10-01T10:11:51.824706+0000 mgr.y (mgr.14146) 223 : cluster [DBG] pgmap v206: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T10:11:55.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:55 smithi173 bash[27612]: cluster 2023-10-01T10:11:53.825300+0000 mgr.y (mgr.14146) 224 : cluster [DBG] pgmap v207: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T10:11:55.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:55 smithi173 bash[27612]: audit 2023-10-01T10:11:54.172980+0000 mon.a (mon.0) 462 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:11:55.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:55 smithi173 bash[27612]: audit 2023-10-01T10:11:54.185153+0000 mon.a (mon.0) 463 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:11:55.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:55 smithi078 bash[27336]: cluster 2023-10-01T10:11:53.825300+0000 mgr.y (mgr.14146) 224 : cluster [DBG] pgmap v207: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T10:11:55.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:55 smithi078 bash[27336]: audit 2023-10-01T10:11:54.172980+0000 mon.a (mon.0) 462 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:11:55.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:55 smithi078 bash[27336]: audit 2023-10-01T10:11:54.185153+0000 mon.a (mon.0) 463 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:11:55.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:55 smithi078 bash[33821]: cluster 2023-10-01T10:11:53.825300+0000 mgr.y (mgr.14146) 224 : cluster [DBG] pgmap v207: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T10:11:55.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:55 smithi078 bash[33821]: audit 2023-10-01T10:11:54.172980+0000 mon.a (mon.0) 462 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:11:55.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:55 smithi078 bash[33821]: audit 2023-10-01T10:11:54.185153+0000 mon.a (mon.0) 463 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:11:57.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:57 smithi173 bash[27612]: cluster 2023-10-01T10:11:55.825879+0000 mgr.y (mgr.14146) 225 : cluster [DBG] pgmap v208: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T10:11:57.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:57 smithi078 bash[27336]: cluster 2023-10-01T10:11:55.825879+0000 mgr.y (mgr.14146) 225 : cluster [DBG] pgmap v208: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T10:11:57.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:57 smithi078 bash[33821]: cluster 2023-10-01T10:11:55.825879+0000 mgr.y (mgr.14146) 225 : cluster [DBG] pgmap v208: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T10:11:59.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:11:59 smithi173 bash[27612]: cluster 2023-10-01T10:11:57.826512+0000 mgr.y (mgr.14146) 226 : cluster [DBG] pgmap v209: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T10:11:59.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:11:59 smithi078 bash[27336]: cluster 2023-10-01T10:11:57.826512+0000 mgr.y (mgr.14146) 226 : cluster [DBG] pgmap v209: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T10:11:59.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:11:59 smithi078 bash[33821]: cluster 2023-10-01T10:11:57.826512+0000 mgr.y (mgr.14146) 226 : cluster [DBG] pgmap v209: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T10:12:00.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:00 smithi173 bash[27612]: audit 2023-10-01T10:11:59.379927+0000 mon.a (mon.0) 464 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:12:00.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:00 smithi173 bash[27612]: audit 2023-10-01T10:11:59.641776+0000 mon.a (mon.0) 465 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:12:00.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:00 smithi173 bash[27612]: audit 2023-10-01T10:11:59.643961+0000 mon.a (mon.0) 466 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:12:00.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:00 smithi173 bash[27612]: audit 2023-10-01T10:11:59.652321+0000 mon.a (mon.0) 467 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:12:00.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:00 smithi173 bash[27612]: cluster 2023-10-01T10:11:59.827032+0000 mgr.y (mgr.14146) 227 : cluster [DBG] pgmap v210: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T10:12:00.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:00 smithi078 bash[27336]: audit 2023-10-01T10:11:59.379927+0000 mon.a (mon.0) 464 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:12:00.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:00 smithi078 bash[27336]: audit 2023-10-01T10:11:59.641776+0000 mon.a (mon.0) 465 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:12:00.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:00 smithi078 bash[27336]: audit 2023-10-01T10:11:59.643961+0000 mon.a (mon.0) 466 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:12:00.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:00 smithi078 bash[27336]: audit 2023-10-01T10:11:59.652321+0000 mon.a (mon.0) 467 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:12:00.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:00 smithi078 bash[27336]: cluster 2023-10-01T10:11:59.827032+0000 mgr.y (mgr.14146) 227 : cluster [DBG] pgmap v210: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T10:12:00.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:00 smithi078 bash[33821]: audit 2023-10-01T10:11:59.379927+0000 mon.a (mon.0) 464 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:12:00.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:00 smithi078 bash[33821]: audit 2023-10-01T10:11:59.641776+0000 mon.a (mon.0) 465 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:12:00.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:00 smithi078 bash[33821]: audit 2023-10-01T10:11:59.643961+0000 mon.a (mon.0) 466 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:12:00.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:00 smithi078 bash[33821]: audit 2023-10-01T10:11:59.652321+0000 mon.a (mon.0) 467 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:12:00.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:00 smithi078 bash[33821]: cluster 2023-10-01T10:11:59.827032+0000 mgr.y (mgr.14146) 227 : cluster [DBG] pgmap v210: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T10:12:01.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:01 smithi173 bash[27612]: audit 2023-10-01T10:12:00.664557+0000 mon.a (mon.0) 468 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-01T10:12:01.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:01 smithi173 bash[27612]: audit 2023-10-01T10:12:00.664724+0000 mon.b (mon.1) 17 : audit [INF] from='osd.6 [v2:172.21.15.173:6816/1568190351,v1:172.21.15.173:6817/1568190351]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-01T10:12:02.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:01 smithi078 bash[27336]: audit 2023-10-01T10:12:00.664557+0000 mon.a (mon.0) 468 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-01T10:12:02.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:01 smithi078 bash[27336]: audit 2023-10-01T10:12:00.664724+0000 mon.b (mon.1) 17 : audit [INF] from='osd.6 [v2:172.21.15.173:6816/1568190351,v1:172.21.15.173:6817/1568190351]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-01T10:12:02.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:01 smithi078 bash[33821]: audit 2023-10-01T10:12:00.664557+0000 mon.a (mon.0) 468 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-01T10:12:02.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:01 smithi078 bash[33821]: audit 2023-10-01T10:12:00.664724+0000 mon.b (mon.1) 17 : audit [INF] from='osd.6 [v2:172.21.15.173:6816/1568190351,v1:172.21.15.173:6817/1568190351]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-10-01T10:12:02.718 INFO:teuthology.orchestra.run.smithi173.stdout:Created osd(s) 6 on host 'smithi173' 2023-10-01T10:12:03.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:02 smithi078 bash[27336]: audit 2023-10-01T10:12:01.664209+0000 mon.a (mon.0) 469 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-10-01T10:12:03.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:02 smithi078 bash[27336]: cluster 2023-10-01T10:12:01.664313+0000 mon.a (mon.0) 470 : cluster [DBG] osdmap e40: 7 total, 6 up, 7 in 2023-10-01T10:12:03.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:02 smithi078 bash[27336]: audit 2023-10-01T10:12:01.664486+0000 mon.a (mon.0) 471 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T10:12:03.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:02 smithi078 bash[27336]: audit 2023-10-01T10:12:01.667992+0000 mon.a (mon.0) 472 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-10-01T10:12:03.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:02 smithi078 bash[27336]: audit 2023-10-01T10:12:01.668232+0000 mon.b (mon.1) 18 : audit [INF] from='osd.6 [v2:172.21.15.173:6816/1568190351,v1:172.21.15.173:6817/1568190351]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-10-01T10:12:03.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:02 smithi078 bash[27336]: cluster 2023-10-01T10:12:01.827574+0000 mgr.y (mgr.14146) 228 : cluster [DBG] pgmap v212: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T10:12:03.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:02 smithi078 bash[33821]: audit 2023-10-01T10:12:01.664209+0000 mon.a (mon.0) 469 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-10-01T10:12:03.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:02 smithi078 bash[33821]: cluster 2023-10-01T10:12:01.664313+0000 mon.a (mon.0) 470 : cluster [DBG] osdmap e40: 7 total, 6 up, 7 in 2023-10-01T10:12:03.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:02 smithi078 bash[33821]: audit 2023-10-01T10:12:01.664486+0000 mon.a (mon.0) 471 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T10:12:03.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:02 smithi078 bash[33821]: audit 2023-10-01T10:12:01.667992+0000 mon.a (mon.0) 472 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-10-01T10:12:03.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:02 smithi078 bash[33821]: audit 2023-10-01T10:12:01.668232+0000 mon.b (mon.1) 18 : audit [INF] from='osd.6 [v2:172.21.15.173:6816/1568190351,v1:172.21.15.173:6817/1568190351]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-10-01T10:12:03.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:02 smithi078 bash[33821]: cluster 2023-10-01T10:12:01.827574+0000 mgr.y (mgr.14146) 228 : cluster [DBG] pgmap v212: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T10:12:03.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:02 smithi173 bash[27612]: audit 2023-10-01T10:12:01.664209+0000 mon.a (mon.0) 469 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-10-01T10:12:03.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:02 smithi173 bash[27612]: cluster 2023-10-01T10:12:01.664313+0000 mon.a (mon.0) 470 : cluster [DBG] osdmap e40: 7 total, 6 up, 7 in 2023-10-01T10:12:03.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:02 smithi173 bash[27612]: audit 2023-10-01T10:12:01.664486+0000 mon.a (mon.0) 471 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T10:12:03.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:02 smithi173 bash[27612]: audit 2023-10-01T10:12:01.667992+0000 mon.a (mon.0) 472 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-10-01T10:12:03.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:02 smithi173 bash[27612]: audit 2023-10-01T10:12:01.668232+0000 mon.b (mon.1) 18 : audit [INF] from='osd.6 [v2:172.21.15.173:6816/1568190351,v1:172.21.15.173:6817/1568190351]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-10-01T10:12:03.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:02 smithi173 bash[27612]: cluster 2023-10-01T10:12:01.827574+0000 mgr.y (mgr.14146) 228 : cluster [DBG] pgmap v212: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T10:12:03.328 DEBUG:teuthology.orchestra.run.smithi173:osd.6> sudo journalctl -f -n 0 -u ceph-d8016480-6041-11ee-8db5-212e2dc638e7@osd.6.service 2023-10-01T10:12:03.331 INFO:tasks.cephadm:Deploying osd.7 on smithi173 with /dev/vg_nvme/lv_1... 2023-10-01T10:12:03.331 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-10-01T10:12:03.828 INFO:journalctl@ceph.osd.6.smithi173.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-01T10:12:03.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:03 smithi173 bash[27612]: cluster 2023-10-01T10:12:01.653957+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-01T10:12:03.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:03 smithi173 bash[27612]: cluster 2023-10-01T10:12:01.654039+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-01T10:12:03.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:03 smithi173 bash[27612]: audit 2023-10-01T10:12:02.668170+0000 mon.a (mon.0) 473 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]': finished 2023-10-01T10:12:03.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:03 smithi173 bash[27612]: cluster 2023-10-01T10:12:02.668352+0000 mon.a (mon.0) 474 : cluster [DBG] osdmap e41: 7 total, 6 up, 7 in 2023-10-01T10:12:03.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:03 smithi173 bash[27612]: audit 2023-10-01T10:12:02.669002+0000 mon.a (mon.0) 475 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T10:12:03.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:03 smithi173 bash[27612]: audit 2023-10-01T10:12:02.679604+0000 mon.a (mon.0) 476 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T10:12:03.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:03 smithi173 bash[27612]: audit 2023-10-01T10:12:02.715126+0000 mon.a (mon.0) 477 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:12:03.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:03 smithi173 bash[27612]: audit 2023-10-01T10:12:02.728444+0000 mon.a (mon.0) 478 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:12:04.086 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:03 smithi078 bash[27336]: cluster 2023-10-01T10:12:01.653957+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-01T10:12:04.086 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:03 smithi078 bash[27336]: cluster 2023-10-01T10:12:01.654039+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-01T10:12:04.086 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:03 smithi078 bash[27336]: audit 2023-10-01T10:12:02.668170+0000 mon.a (mon.0) 473 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]': finished 2023-10-01T10:12:04.086 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:03 smithi078 bash[27336]: cluster 2023-10-01T10:12:02.668352+0000 mon.a (mon.0) 474 : cluster [DBG] osdmap e41: 7 total, 6 up, 7 in 2023-10-01T10:12:04.086 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:03 smithi078 bash[27336]: audit 2023-10-01T10:12:02.669002+0000 mon.a (mon.0) 475 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T10:12:04.086 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:03 smithi078 bash[27336]: audit 2023-10-01T10:12:02.679604+0000 mon.a (mon.0) 476 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T10:12:04.086 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:03 smithi078 bash[27336]: audit 2023-10-01T10:12:02.715126+0000 mon.a (mon.0) 477 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:12:04.087 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:03 smithi078 bash[27336]: audit 2023-10-01T10:12:02.728444+0000 mon.a (mon.0) 478 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:12:04.087 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:03 smithi078 bash[33821]: cluster 2023-10-01T10:12:01.653957+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-01T10:12:04.087 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:03 smithi078 bash[33821]: cluster 2023-10-01T10:12:01.654039+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-01T10:12:04.087 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:03 smithi078 bash[33821]: audit 2023-10-01T10:12:02.668170+0000 mon.a (mon.0) 473 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]': finished 2023-10-01T10:12:04.087 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:03 smithi078 bash[33821]: cluster 2023-10-01T10:12:02.668352+0000 mon.a (mon.0) 474 : cluster [DBG] osdmap e41: 7 total, 6 up, 7 in 2023-10-01T10:12:04.087 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:03 smithi078 bash[33821]: audit 2023-10-01T10:12:02.669002+0000 mon.a (mon.0) 475 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T10:12:04.087 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:03 smithi078 bash[33821]: audit 2023-10-01T10:12:02.679604+0000 mon.a (mon.0) 476 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T10:12:04.088 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:03 smithi078 bash[33821]: audit 2023-10-01T10:12:02.715126+0000 mon.a (mon.0) 477 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:12:04.088 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:03 smithi078 bash[33821]: audit 2023-10-01T10:12:02.728444+0000 mon.a (mon.0) 478 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:12:05.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:04 smithi078 bash[27336]: cluster 2023-10-01T10:12:03.682740+0000 mon.a (mon.0) 479 : cluster [INF] osd.6 [v2:172.21.15.173:6816/1568190351,v1:172.21.15.173:6817/1568190351] boot 2023-10-01T10:12:05.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:04 smithi078 bash[27336]: cluster 2023-10-01T10:12:03.682809+0000 mon.a (mon.0) 480 : cluster [DBG] osdmap e42: 7 total, 7 up, 7 in 2023-10-01T10:12:05.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:04 smithi078 bash[27336]: audit 2023-10-01T10:12:03.688072+0000 mon.a (mon.0) 481 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T10:12:05.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:04 smithi078 bash[27336]: cluster 2023-10-01T10:12:03.828026+0000 mgr.y (mgr.14146) 229 : cluster [DBG] pgmap v215: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T10:12:05.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:04 smithi078 bash[33821]: cluster 2023-10-01T10:12:03.682740+0000 mon.a (mon.0) 479 : cluster [INF] osd.6 [v2:172.21.15.173:6816/1568190351,v1:172.21.15.173:6817/1568190351] boot 2023-10-01T10:12:05.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:04 smithi078 bash[33821]: cluster 2023-10-01T10:12:03.682809+0000 mon.a (mon.0) 480 : cluster [DBG] osdmap e42: 7 total, 7 up, 7 in 2023-10-01T10:12:05.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:04 smithi078 bash[33821]: audit 2023-10-01T10:12:03.688072+0000 mon.a (mon.0) 481 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T10:12:05.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:04 smithi078 bash[33821]: cluster 2023-10-01T10:12:03.828026+0000 mgr.y (mgr.14146) 229 : cluster [DBG] pgmap v215: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T10:12:05.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:04 smithi173 bash[27612]: cluster 2023-10-01T10:12:03.682740+0000 mon.a (mon.0) 479 : cluster [INF] osd.6 [v2:172.21.15.173:6816/1568190351,v1:172.21.15.173:6817/1568190351] boot 2023-10-01T10:12:05.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:04 smithi173 bash[27612]: cluster 2023-10-01T10:12:03.682809+0000 mon.a (mon.0) 480 : cluster [DBG] osdmap e42: 7 total, 7 up, 7 in 2023-10-01T10:12:05.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:04 smithi173 bash[27612]: audit 2023-10-01T10:12:03.688072+0000 mon.a (mon.0) 481 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T10:12:05.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:04 smithi173 bash[27612]: cluster 2023-10-01T10:12:03.828026+0000 mgr.y (mgr.14146) 229 : cluster [DBG] pgmap v215: 1 pgs: 1 active+clean; 0 B data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-10-01T10:12:06.022 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:05 smithi173 bash[27612]: cluster 2023-10-01T10:12:04.707639+0000 mon.a (mon.0) 482 : cluster [DBG] osdmap e43: 7 total, 7 up, 7 in 2023-10-01T10:12:06.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:05 smithi078 bash[27336]: cluster 2023-10-01T10:12:04.707639+0000 mon.a (mon.0) 482 : cluster [DBG] osdmap e43: 7 total, 7 up, 7 in 2023-10-01T10:12:06.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:05 smithi078 bash[33821]: cluster 2023-10-01T10:12:04.707639+0000 mon.a (mon.0) 482 : cluster [DBG] osdmap e43: 7 total, 7 up, 7 in 2023-10-01T10:12:06.677 INFO:teuthology.orchestra.run.smithi173.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-10-01T10:12:06.678 INFO:teuthology.orchestra.run.smithi173.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-10-01T10:12:06.678 INFO:teuthology.orchestra.run.smithi173.stderr: stderr: 10+0 records in 2023-10-01T10:12:06.678 INFO:teuthology.orchestra.run.smithi173.stderr:10+0 records out 2023-10-01T10:12:06.678 INFO:teuthology.orchestra.run.smithi173.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0171057 s, 613 MB/s 2023-10-01T10:12:06.678 INFO:teuthology.orchestra.run.smithi173.stderr:--> Zapping successful for: 2023-10-01T10:12:07.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:06 smithi078 bash[27336]: cluster 2023-10-01T10:12:05.715002+0000 mon.a (mon.0) 483 : cluster [DBG] osdmap e44: 7 total, 7 up, 7 in 2023-10-01T10:12:07.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:06 smithi078 bash[27336]: cluster 2023-10-01T10:12:05.828593+0000 mgr.y (mgr.14146) 230 : cluster [DBG] pgmap v218: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T10:12:07.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:06 smithi078 bash[33821]: cluster 2023-10-01T10:12:05.715002+0000 mon.a (mon.0) 483 : cluster [DBG] osdmap e44: 7 total, 7 up, 7 in 2023-10-01T10:12:07.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:06 smithi078 bash[33821]: cluster 2023-10-01T10:12:05.828593+0000 mgr.y (mgr.14146) 230 : cluster [DBG] pgmap v218: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T10:12:07.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:06 smithi173 bash[27612]: cluster 2023-10-01T10:12:05.715002+0000 mon.a (mon.0) 483 : cluster [DBG] osdmap e44: 7 total, 7 up, 7 in 2023-10-01T10:12:07.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:06 smithi173 bash[27612]: cluster 2023-10-01T10:12:05.828593+0000 mgr.y (mgr.14146) 230 : cluster [DBG] pgmap v218: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T10:12:07.356 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph orch daemon add osd smithi173:vg_nvme/lv_1 2023-10-01T10:12:09.084 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:08 smithi173 bash[27612]: cluster 2023-10-01T10:12:07.829164+0000 mgr.y (mgr.14146) 231 : cluster [DBG] pgmap v219: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T10:12:09.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:08 smithi078 bash[27336]: cluster 2023-10-01T10:12:07.829164+0000 mgr.y (mgr.14146) 231 : cluster [DBG] pgmap v219: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T10:12:09.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:08 smithi078 bash[33821]: cluster 2023-10-01T10:12:07.829164+0000 mgr.y (mgr.14146) 231 : cluster [DBG] pgmap v219: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T10:12:11.262 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:10 smithi173 bash[27612]: cluster 2023-10-01T10:12:09.829704+0000 mgr.y (mgr.14146) 232 : cluster [DBG] pgmap v220: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T10:12:11.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:10 smithi078 bash[27336]: cluster 2023-10-01T10:12:09.829704+0000 mgr.y (mgr.14146) 232 : cluster [DBG] pgmap v220: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T10:12:11.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:10 smithi078 bash[33821]: cluster 2023-10-01T10:12:09.829704+0000 mgr.y (mgr.14146) 232 : cluster [DBG] pgmap v220: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T10:12:12.194 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:11 smithi173 bash[27612]: audit 2023-10-01T10:12:11.723075+0000 mon.a (mon.0) 484 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T10:12:12.195 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:11 smithi173 bash[27612]: audit 2023-10-01T10:12:11.727858+0000 mon.a (mon.0) 485 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T10:12:12.195 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:11 smithi173 bash[27612]: audit 2023-10-01T10:12:11.729374+0000 mon.a (mon.0) 486 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:12:12.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:11 smithi078 bash[27336]: audit 2023-10-01T10:12:11.723075+0000 mon.a (mon.0) 484 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T10:12:12.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:11 smithi078 bash[27336]: audit 2023-10-01T10:12:11.727858+0000 mon.a (mon.0) 485 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T10:12:12.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:11 smithi078 bash[27336]: audit 2023-10-01T10:12:11.729374+0000 mon.a (mon.0) 486 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:12:12.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:11 smithi078 bash[33821]: audit 2023-10-01T10:12:11.723075+0000 mon.a (mon.0) 484 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-10-01T10:12:12.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:11 smithi078 bash[33821]: audit 2023-10-01T10:12:11.727858+0000 mon.a (mon.0) 485 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-10-01T10:12:12.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:11 smithi078 bash[33821]: audit 2023-10-01T10:12:11.729374+0000 mon.a (mon.0) 486 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:12:13.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:12 smithi078 bash[27336]: audit 2023-10-01T10:12:11.716172+0000 mgr.y (mgr.14146) 233 : audit [DBG] from='client.24295 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi173:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:12:13.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:12 smithi078 bash[27336]: cluster 2023-10-01T10:12:11.830225+0000 mgr.y (mgr.14146) 234 : cluster [DBG] pgmap v221: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T10:12:13.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:12 smithi078 bash[27336]: audit 2023-10-01T10:12:12.900805+0000 mon.a (mon.0) 487 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:12:13.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:12 smithi078 bash[27336]: audit 2023-10-01T10:12:12.902740+0000 mon.a (mon.0) 488 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:12:13.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:12 smithi078 bash[27336]: audit 2023-10-01T10:12:12.904182+0000 mon.a (mon.0) 489 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:12:13.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:12 smithi078 bash[33821]: audit 2023-10-01T10:12:11.716172+0000 mgr.y (mgr.14146) 233 : audit [DBG] from='client.24295 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi173:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:12:13.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:12 smithi078 bash[33821]: cluster 2023-10-01T10:12:11.830225+0000 mgr.y (mgr.14146) 234 : cluster [DBG] pgmap v221: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T10:12:13.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:12 smithi078 bash[33821]: audit 2023-10-01T10:12:12.900805+0000 mon.a (mon.0) 487 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:12:13.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:12 smithi078 bash[33821]: audit 2023-10-01T10:12:12.902740+0000 mon.a (mon.0) 488 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:12:13.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:12 smithi078 bash[33821]: audit 2023-10-01T10:12:12.904182+0000 mon.a (mon.0) 489 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:12:13.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:12 smithi173 bash[27612]: audit 2023-10-01T10:12:11.716172+0000 mgr.y (mgr.14146) 233 : audit [DBG] from='client.24295 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi173:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:12:13.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:12 smithi173 bash[27612]: cluster 2023-10-01T10:12:11.830225+0000 mgr.y (mgr.14146) 234 : cluster [DBG] pgmap v221: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T10:12:13.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:12 smithi173 bash[27612]: audit 2023-10-01T10:12:12.900805+0000 mon.a (mon.0) 487 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:12:13.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:12 smithi173 bash[27612]: audit 2023-10-01T10:12:12.902740+0000 mon.a (mon.0) 488 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:12:13.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:12 smithi173 bash[27612]: audit 2023-10-01T10:12:12.904182+0000 mon.a (mon.0) 489 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:12:14.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:13 smithi078 bash[27336]: cephadm 2023-10-01T10:12:12.891155+0000 mgr.y (mgr.14146) 235 : cephadm [INF] Detected new or changed devices on smithi173 2023-10-01T10:12:14.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:13 smithi078 bash[27336]: audit 2023-10-01T10:12:12.905379+0000 mon.a (mon.0) 490 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:12:14.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:13 smithi078 bash[27336]: cephadm 2023-10-01T10:12:12.906664+0000 mgr.y (mgr.14146) 236 : cephadm [INF] Adjusting osd_memory_target on smithi173 to 5755M 2023-10-01T10:12:14.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:13 smithi078 bash[27336]: audit 2023-10-01T10:12:12.916567+0000 mon.a (mon.0) 491 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:12:14.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:13 smithi078 bash[27336]: audit 2023-10-01T10:12:12.918170+0000 mon.a (mon.0) 492 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:12:14.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:13 smithi078 bash[27336]: audit 2023-10-01T10:12:12.919286+0000 mon.a (mon.0) 493 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:12:14.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:13 smithi078 bash[27336]: audit 2023-10-01T10:12:12.925232+0000 mon.a (mon.0) 494 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:12:14.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:13 smithi078 bash[33821]: cephadm 2023-10-01T10:12:12.891155+0000 mgr.y (mgr.14146) 235 : cephadm [INF] Detected new or changed devices on smithi173 2023-10-01T10:12:14.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:13 smithi078 bash[33821]: audit 2023-10-01T10:12:12.905379+0000 mon.a (mon.0) 490 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:12:14.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:13 smithi078 bash[33821]: cephadm 2023-10-01T10:12:12.906664+0000 mgr.y (mgr.14146) 236 : cephadm [INF] Adjusting osd_memory_target on smithi173 to 5755M 2023-10-01T10:12:14.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:13 smithi078 bash[33821]: audit 2023-10-01T10:12:12.916567+0000 mon.a (mon.0) 491 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:12:14.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:13 smithi078 bash[33821]: audit 2023-10-01T10:12:12.918170+0000 mon.a (mon.0) 492 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:12:14.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:13 smithi078 bash[33821]: audit 2023-10-01T10:12:12.919286+0000 mon.a (mon.0) 493 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:12:14.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:13 smithi078 bash[33821]: audit 2023-10-01T10:12:12.925232+0000 mon.a (mon.0) 494 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:12:14.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:13 smithi173 bash[27612]: cephadm 2023-10-01T10:12:12.891155+0000 mgr.y (mgr.14146) 235 : cephadm [INF] Detected new or changed devices on smithi173 2023-10-01T10:12:14.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:13 smithi173 bash[27612]: audit 2023-10-01T10:12:12.905379+0000 mon.a (mon.0) 490 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:12:14.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:13 smithi173 bash[27612]: cephadm 2023-10-01T10:12:12.906664+0000 mgr.y (mgr.14146) 236 : cephadm [INF] Adjusting osd_memory_target on smithi173 to 5755M 2023-10-01T10:12:14.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:13 smithi173 bash[27612]: audit 2023-10-01T10:12:12.916567+0000 mon.a (mon.0) 491 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:12:14.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:13 smithi173 bash[27612]: audit 2023-10-01T10:12:12.918170+0000 mon.a (mon.0) 492 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:12:14.372 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:13 smithi173 bash[27612]: audit 2023-10-01T10:12:12.919286+0000 mon.a (mon.0) 493 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:12:14.372 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:13 smithi173 bash[27612]: audit 2023-10-01T10:12:12.925232+0000 mon.a (mon.0) 494 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:12:15.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:14 smithi078 bash[27336]: cluster 2023-10-01T10:12:13.830858+0000 mgr.y (mgr.14146) 237 : cluster [DBG] pgmap v222: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T10:12:15.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:14 smithi078 bash[33821]: cluster 2023-10-01T10:12:13.830858+0000 mgr.y (mgr.14146) 237 : cluster [DBG] pgmap v222: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T10:12:15.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:14 smithi173 bash[27612]: cluster 2023-10-01T10:12:13.830858+0000 mgr.y (mgr.14146) 237 : cluster [DBG] pgmap v222: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T10:12:17.218 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:16 smithi173 bash[27612]: cluster 2023-10-01T10:12:15.831469+0000 mgr.y (mgr.14146) 238 : cluster [DBG] pgmap v223: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T10:12:17.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:16 smithi078 bash[27336]: cluster 2023-10-01T10:12:15.831469+0000 mgr.y (mgr.14146) 238 : cluster [DBG] pgmap v223: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T10:12:17.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:16 smithi078 bash[33821]: cluster 2023-10-01T10:12:15.831469+0000 mgr.y (mgr.14146) 238 : cluster [DBG] pgmap v223: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T10:12:19.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:18 smithi173 bash[27612]: cluster 2023-10-01T10:12:17.832026+0000 mgr.y (mgr.14146) 239 : cluster [DBG] pgmap v224: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T10:12:19.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:18 smithi078 bash[27336]: cluster 2023-10-01T10:12:17.832026+0000 mgr.y (mgr.14146) 239 : cluster [DBG] pgmap v224: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T10:12:19.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:18 smithi078 bash[33821]: cluster 2023-10-01T10:12:17.832026+0000 mgr.y (mgr.14146) 239 : cluster [DBG] pgmap v224: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T10:12:20.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:19 smithi078 bash[27336]: audit 2023-10-01T10:12:19.633478+0000 mon.b (mon.1) 19 : audit [INF] from='client.? 172.21.15.173:0/2347207320' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7590027f-50c4-4457-843d-cebaa734d2ea"}]: dispatch 2023-10-01T10:12:20.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:19 smithi078 bash[27336]: audit 2023-10-01T10:12:19.633690+0000 mon.a (mon.0) 495 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7590027f-50c4-4457-843d-cebaa734d2ea"}]: dispatch 2023-10-01T10:12:20.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:19 smithi078 bash[27336]: audit 2023-10-01T10:12:19.642535+0000 mon.a (mon.0) 496 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7590027f-50c4-4457-843d-cebaa734d2ea"}]': finished 2023-10-01T10:12:20.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:19 smithi078 bash[27336]: cluster 2023-10-01T10:12:19.642718+0000 mon.a (mon.0) 497 : cluster [DBG] osdmap e45: 8 total, 7 up, 8 in 2023-10-01T10:12:20.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:19 smithi078 bash[27336]: audit 2023-10-01T10:12:19.642947+0000 mon.a (mon.0) 498 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T10:12:20.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:19 smithi078 bash[33821]: audit 2023-10-01T10:12:19.633478+0000 mon.b (mon.1) 19 : audit [INF] from='client.? 172.21.15.173:0/2347207320' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7590027f-50c4-4457-843d-cebaa734d2ea"}]: dispatch 2023-10-01T10:12:20.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:19 smithi078 bash[33821]: audit 2023-10-01T10:12:19.633690+0000 mon.a (mon.0) 495 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7590027f-50c4-4457-843d-cebaa734d2ea"}]: dispatch 2023-10-01T10:12:20.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:19 smithi078 bash[33821]: audit 2023-10-01T10:12:19.642535+0000 mon.a (mon.0) 496 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7590027f-50c4-4457-843d-cebaa734d2ea"}]': finished 2023-10-01T10:12:20.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:19 smithi078 bash[33821]: cluster 2023-10-01T10:12:19.642718+0000 mon.a (mon.0) 497 : cluster [DBG] osdmap e45: 8 total, 7 up, 8 in 2023-10-01T10:12:20.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:19 smithi078 bash[33821]: audit 2023-10-01T10:12:19.642947+0000 mon.a (mon.0) 498 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T10:12:20.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:19 smithi173 bash[27612]: audit 2023-10-01T10:12:19.633478+0000 mon.b (mon.1) 19 : audit [INF] from='client.? 172.21.15.173:0/2347207320' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7590027f-50c4-4457-843d-cebaa734d2ea"}]: dispatch 2023-10-01T10:12:20.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:19 smithi173 bash[27612]: audit 2023-10-01T10:12:19.633690+0000 mon.a (mon.0) 495 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7590027f-50c4-4457-843d-cebaa734d2ea"}]: dispatch 2023-10-01T10:12:20.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:19 smithi173 bash[27612]: audit 2023-10-01T10:12:19.642535+0000 mon.a (mon.0) 496 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7590027f-50c4-4457-843d-cebaa734d2ea"}]': finished 2023-10-01T10:12:20.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:19 smithi173 bash[27612]: cluster 2023-10-01T10:12:19.642718+0000 mon.a (mon.0) 497 : cluster [DBG] osdmap e45: 8 total, 7 up, 8 in 2023-10-01T10:12:20.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:19 smithi173 bash[27612]: audit 2023-10-01T10:12:19.642947+0000 mon.a (mon.0) 498 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T10:12:21.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:20 smithi078 bash[27336]: cluster 2023-10-01T10:12:19.832613+0000 mgr.y (mgr.14146) 240 : cluster [DBG] pgmap v226: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T10:12:21.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:20 smithi078 bash[27336]: audit 2023-10-01T10:12:20.387331+0000 mon.b (mon.1) 20 : audit [DBG] from='client.? 172.21.15.173:0/1321691145' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T10:12:21.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:20 smithi078 bash[33821]: cluster 2023-10-01T10:12:19.832613+0000 mgr.y (mgr.14146) 240 : cluster [DBG] pgmap v226: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T10:12:21.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:20 smithi078 bash[33821]: audit 2023-10-01T10:12:20.387331+0000 mon.b (mon.1) 20 : audit [DBG] from='client.? 172.21.15.173:0/1321691145' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T10:12:21.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:20 smithi173 bash[27612]: cluster 2023-10-01T10:12:19.832613+0000 mgr.y (mgr.14146) 240 : cluster [DBG] pgmap v226: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T10:12:21.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:20 smithi173 bash[27612]: audit 2023-10-01T10:12:20.387331+0000 mon.b (mon.1) 20 : audit [DBG] from='client.? 172.21.15.173:0/1321691145' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-10-01T10:12:23.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:22 smithi078 bash[27336]: cluster 2023-10-01T10:12:21.833150+0000 mgr.y (mgr.14146) 241 : cluster [DBG] pgmap v227: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T10:12:23.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:22 smithi078 bash[33821]: cluster 2023-10-01T10:12:21.833150+0000 mgr.y (mgr.14146) 241 : cluster [DBG] pgmap v227: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T10:12:23.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:22 smithi173 bash[27612]: cluster 2023-10-01T10:12:21.833150+0000 mgr.y (mgr.14146) 241 : cluster [DBG] pgmap v227: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T10:12:25.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:24 smithi078 bash[27336]: cluster 2023-10-01T10:12:23.833712+0000 mgr.y (mgr.14146) 242 : cluster [DBG] pgmap v228: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T10:12:25.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:24 smithi078 bash[33821]: cluster 2023-10-01T10:12:23.833712+0000 mgr.y (mgr.14146) 242 : cluster [DBG] pgmap v228: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T10:12:25.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:24 smithi173 bash[27612]: cluster 2023-10-01T10:12:23.833712+0000 mgr.y (mgr.14146) 242 : cluster [DBG] pgmap v228: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T10:12:27.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:26 smithi078 bash[27336]: cluster 2023-10-01T10:12:25.834323+0000 mgr.y (mgr.14146) 243 : cluster [DBG] pgmap v229: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T10:12:27.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:26 smithi078 bash[33821]: cluster 2023-10-01T10:12:25.834323+0000 mgr.y (mgr.14146) 243 : cluster [DBG] pgmap v229: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T10:12:27.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:26 smithi173 bash[27612]: cluster 2023-10-01T10:12:25.834323+0000 mgr.y (mgr.14146) 243 : cluster [DBG] pgmap v229: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T10:12:29.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:28 smithi078 bash[27336]: cluster 2023-10-01T10:12:27.834876+0000 mgr.y (mgr.14146) 244 : cluster [DBG] pgmap v230: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T10:12:29.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:28 smithi078 bash[33821]: cluster 2023-10-01T10:12:27.834876+0000 mgr.y (mgr.14146) 244 : cluster [DBG] pgmap v230: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T10:12:29.340 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:28 smithi173 bash[27612]: cluster 2023-10-01T10:12:27.834876+0000 mgr.y (mgr.14146) 244 : cluster [DBG] pgmap v230: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T10:12:31.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:30 smithi078 bash[27336]: cluster 2023-10-01T10:12:29.835411+0000 mgr.y (mgr.14146) 245 : cluster [DBG] pgmap v231: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T10:12:31.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:30 smithi078 bash[33821]: cluster 2023-10-01T10:12:29.835411+0000 mgr.y (mgr.14146) 245 : cluster [DBG] pgmap v231: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T10:12:31.357 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:30 smithi173 bash[27612]: cluster 2023-10-01T10:12:29.835411+0000 mgr.y (mgr.14146) 245 : cluster [DBG] pgmap v231: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T10:12:32.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:31 smithi078 bash[27336]: audit 2023-10-01T10:12:31.826914+0000 mon.a (mon.0) 499 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-01T10:12:32.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:31 smithi078 bash[27336]: audit 2023-10-01T10:12:31.828618+0000 mon.a (mon.0) 500 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:12:32.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:31 smithi078 bash[33821]: audit 2023-10-01T10:12:31.826914+0000 mon.a (mon.0) 499 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-01T10:12:32.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:31 smithi078 bash[33821]: audit 2023-10-01T10:12:31.828618+0000 mon.a (mon.0) 500 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:12:32.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:31 smithi173 bash[27612]: audit 2023-10-01T10:12:31.826914+0000 mon.a (mon.0) 499 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-01T10:12:32.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:31 smithi173 bash[27612]: audit 2023-10-01T10:12:31.828618+0000 mon.a (mon.0) 500 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:12:33.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:32 smithi173 bash[27612]: cephadm 2023-10-01T10:12:31.830035+0000 mgr.y (mgr.14146) 246 : cephadm [INF] Deploying daemon osd.7 on smithi173 2023-10-01T10:12:33.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:32 smithi173 bash[27612]: cluster 2023-10-01T10:12:31.835860+0000 mgr.y (mgr.14146) 247 : cluster [DBG] pgmap v232: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T10:12:33.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:32 smithi078 bash[27336]: cephadm 2023-10-01T10:12:31.830035+0000 mgr.y (mgr.14146) 246 : cephadm [INF] Deploying daemon osd.7 on smithi173 2023-10-01T10:12:33.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:32 smithi078 bash[27336]: cluster 2023-10-01T10:12:31.835860+0000 mgr.y (mgr.14146) 247 : cluster [DBG] pgmap v232: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T10:12:33.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:32 smithi078 bash[33821]: cephadm 2023-10-01T10:12:31.830035+0000 mgr.y (mgr.14146) 246 : cephadm [INF] Deploying daemon osd.7 on smithi173 2023-10-01T10:12:33.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:32 smithi078 bash[33821]: cluster 2023-10-01T10:12:31.835860+0000 mgr.y (mgr.14146) 247 : cluster [DBG] pgmap v232: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T10:12:35.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:34 smithi173 bash[27612]: cluster 2023-10-01T10:12:33.836464+0000 mgr.y (mgr.14146) 248 : cluster [DBG] pgmap v233: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T10:12:35.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:34 smithi078 bash[27336]: cluster 2023-10-01T10:12:33.836464+0000 mgr.y (mgr.14146) 248 : cluster [DBG] pgmap v233: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T10:12:35.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:34 smithi078 bash[33821]: cluster 2023-10-01T10:12:33.836464+0000 mgr.y (mgr.14146) 248 : cluster [DBG] pgmap v233: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T10:12:37.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:36 smithi173 bash[27612]: cluster 2023-10-01T10:12:35.837089+0000 mgr.y (mgr.14146) 249 : cluster [DBG] pgmap v234: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T10:12:37.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:36 smithi173 bash[27612]: audit 2023-10-01T10:12:35.920290+0000 mon.a (mon.0) 501 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:12:37.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:36 smithi173 bash[27612]: audit 2023-10-01T10:12:35.958415+0000 mon.a (mon.0) 502 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:12:37.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:36 smithi078 bash[27336]: cluster 2023-10-01T10:12:35.837089+0000 mgr.y (mgr.14146) 249 : cluster [DBG] pgmap v234: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T10:12:37.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:36 smithi078 bash[27336]: audit 2023-10-01T10:12:35.920290+0000 mon.a (mon.0) 501 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:12:37.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:36 smithi078 bash[27336]: audit 2023-10-01T10:12:35.958415+0000 mon.a (mon.0) 502 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:12:37.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:36 smithi078 bash[33821]: cluster 2023-10-01T10:12:35.837089+0000 mgr.y (mgr.14146) 249 : cluster [DBG] pgmap v234: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T10:12:37.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:36 smithi078 bash[33821]: audit 2023-10-01T10:12:35.920290+0000 mon.a (mon.0) 501 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:12:37.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:36 smithi078 bash[33821]: audit 2023-10-01T10:12:35.958415+0000 mon.a (mon.0) 502 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:12:39.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:38 smithi173 bash[27612]: cluster 2023-10-01T10:12:37.837669+0000 mgr.y (mgr.14146) 250 : cluster [DBG] pgmap v235: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T10:12:39.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:38 smithi078 bash[33821]: cluster 2023-10-01T10:12:37.837669+0000 mgr.y (mgr.14146) 250 : cluster [DBG] pgmap v235: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T10:12:39.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:38 smithi078 bash[27336]: cluster 2023-10-01T10:12:37.837669+0000 mgr.y (mgr.14146) 250 : cluster [DBG] pgmap v235: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T10:12:41.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:40 smithi173 bash[27612]: cluster 2023-10-01T10:12:39.838225+0000 mgr.y (mgr.14146) 251 : cluster [DBG] pgmap v236: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T10:12:41.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:40 smithi078 bash[27336]: cluster 2023-10-01T10:12:39.838225+0000 mgr.y (mgr.14146) 251 : cluster [DBG] pgmap v236: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T10:12:41.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:40 smithi078 bash[33821]: cluster 2023-10-01T10:12:39.838225+0000 mgr.y (mgr.14146) 251 : cluster [DBG] pgmap v236: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T10:12:42.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:42 smithi173 bash[27612]: audit 2023-10-01T10:12:41.083674+0000 mon.a (mon.0) 503 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:12:42.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:42 smithi173 bash[27612]: audit 2023-10-01T10:12:41.085419+0000 mon.a (mon.0) 504 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:12:42.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:42 smithi173 bash[27612]: audit 2023-10-01T10:12:41.086913+0000 mon.a (mon.0) 505 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:12:42.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:42 smithi173 bash[27612]: audit 2023-10-01T10:12:41.093847+0000 mon.a (mon.0) 506 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:12:42.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:42 smithi078 bash[27336]: audit 2023-10-01T10:12:41.083674+0000 mon.a (mon.0) 503 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:12:42.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:42 smithi078 bash[27336]: audit 2023-10-01T10:12:41.085419+0000 mon.a (mon.0) 504 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:12:42.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:42 smithi078 bash[27336]: audit 2023-10-01T10:12:41.086913+0000 mon.a (mon.0) 505 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:12:42.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:42 smithi078 bash[27336]: audit 2023-10-01T10:12:41.093847+0000 mon.a (mon.0) 506 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:12:42.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:42 smithi078 bash[33821]: audit 2023-10-01T10:12:41.083674+0000 mon.a (mon.0) 503 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:12:42.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:42 smithi078 bash[33821]: audit 2023-10-01T10:12:41.085419+0000 mon.a (mon.0) 504 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:12:42.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:42 smithi078 bash[33821]: audit 2023-10-01T10:12:41.086913+0000 mon.a (mon.0) 505 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:12:42.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:42 smithi078 bash[33821]: audit 2023-10-01T10:12:41.093847+0000 mon.a (mon.0) 506 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:12:43.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:43 smithi173 bash[27612]: cluster 2023-10-01T10:12:41.838777+0000 mgr.y (mgr.14146) 252 : cluster [DBG] pgmap v237: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T10:12:43.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:43 smithi173 bash[27612]: audit 2023-10-01T10:12:42.410640+0000 mon.a (mon.0) 507 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-01T10:12:43.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:43 smithi173 bash[27612]: audit 2023-10-01T10:12:42.410932+0000 mon.b (mon.1) 21 : audit [INF] from='osd.7 [v2:172.21.15.173:6824/1958390745,v1:172.21.15.173:6825/1958390745]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-01T10:12:43.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:43 smithi078 bash[33821]: cluster 2023-10-01T10:12:41.838777+0000 mgr.y (mgr.14146) 252 : cluster [DBG] pgmap v237: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T10:12:43.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:43 smithi078 bash[33821]: audit 2023-10-01T10:12:42.410640+0000 mon.a (mon.0) 507 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-01T10:12:43.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:43 smithi078 bash[33821]: audit 2023-10-01T10:12:42.410932+0000 mon.b (mon.1) 21 : audit [INF] from='osd.7 [v2:172.21.15.173:6824/1958390745,v1:172.21.15.173:6825/1958390745]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-01T10:12:43.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:43 smithi078 bash[27336]: cluster 2023-10-01T10:12:41.838777+0000 mgr.y (mgr.14146) 252 : cluster [DBG] pgmap v237: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T10:12:43.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:43 smithi078 bash[27336]: audit 2023-10-01T10:12:42.410640+0000 mon.a (mon.0) 507 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-01T10:12:43.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:43 smithi078 bash[27336]: audit 2023-10-01T10:12:42.410932+0000 mon.b (mon.1) 21 : audit [INF] from='osd.7 [v2:172.21.15.173:6824/1958390745,v1:172.21.15.173:6825/1958390745]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-10-01T10:12:44.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:44 smithi173 bash[27612]: audit 2023-10-01T10:12:43.110302+0000 mon.a (mon.0) 508 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-10-01T10:12:44.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:44 smithi173 bash[27612]: cluster 2023-10-01T10:12:43.110406+0000 mon.a (mon.0) 509 : cluster [DBG] osdmap e46: 8 total, 7 up, 8 in 2023-10-01T10:12:44.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:44 smithi173 bash[27612]: audit 2023-10-01T10:12:43.110676+0000 mon.a (mon.0) 510 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T10:12:44.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:44 smithi173 bash[27612]: audit 2023-10-01T10:12:43.112230+0000 mon.a (mon.0) 511 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-10-01T10:12:44.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:44 smithi173 bash[27612]: audit 2023-10-01T10:12:43.112653+0000 mon.b (mon.1) 22 : audit [INF] from='osd.7 [v2:172.21.15.173:6824/1958390745,v1:172.21.15.173:6825/1958390745]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-10-01T10:12:44.517 INFO:teuthology.orchestra.run.smithi173.stdout:Created osd(s) 7 on host 'smithi173' 2023-10-01T10:12:44.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:44 smithi078 bash[27336]: audit 2023-10-01T10:12:43.110302+0000 mon.a (mon.0) 508 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-10-01T10:12:44.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:44 smithi078 bash[27336]: cluster 2023-10-01T10:12:43.110406+0000 mon.a (mon.0) 509 : cluster [DBG] osdmap e46: 8 total, 7 up, 8 in 2023-10-01T10:12:44.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:44 smithi078 bash[27336]: audit 2023-10-01T10:12:43.110676+0000 mon.a (mon.0) 510 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T10:12:44.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:44 smithi078 bash[27336]: audit 2023-10-01T10:12:43.112230+0000 mon.a (mon.0) 511 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-10-01T10:12:44.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:44 smithi078 bash[27336]: audit 2023-10-01T10:12:43.112653+0000 mon.b (mon.1) 22 : audit [INF] from='osd.7 [v2:172.21.15.173:6824/1958390745,v1:172.21.15.173:6825/1958390745]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-10-01T10:12:44.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:44 smithi078 bash[33821]: audit 2023-10-01T10:12:43.110302+0000 mon.a (mon.0) 508 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-10-01T10:12:44.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:44 smithi078 bash[33821]: cluster 2023-10-01T10:12:43.110406+0000 mon.a (mon.0) 509 : cluster [DBG] osdmap e46: 8 total, 7 up, 8 in 2023-10-01T10:12:44.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:44 smithi078 bash[33821]: audit 2023-10-01T10:12:43.110676+0000 mon.a (mon.0) 510 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T10:12:44.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:44 smithi078 bash[33821]: audit 2023-10-01T10:12:43.112230+0000 mon.a (mon.0) 511 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-10-01T10:12:44.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:44 smithi078 bash[33821]: audit 2023-10-01T10:12:43.112653+0000 mon.b (mon.1) 22 : audit [INF] from='osd.7 [v2:172.21.15.173:6824/1958390745,v1:172.21.15.173:6825/1958390745]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-10-01T10:12:45.166 DEBUG:teuthology.orchestra.run.smithi173:osd.7> sudo journalctl -f -n 0 -u ceph-d8016480-6041-11ee-8db5-212e2dc638e7@osd.7.service 2023-10-01T10:12:45.168 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2023-10-01T10:12:45.169 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd stat -f json 2023-10-01T10:12:45.340 INFO:journalctl@ceph.osd.7.smithi173.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-01T10:12:45.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:45 smithi078 bash[27336]: cluster 2023-10-01T10:12:43.839309+0000 mgr.y (mgr.14146) 253 : cluster [DBG] pgmap v239: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T10:12:45.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:45 smithi078 bash[27336]: audit 2023-10-01T10:12:44.114767+0000 mon.a (mon.0) 512 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]': finished 2023-10-01T10:12:45.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:45 smithi078 bash[27336]: cluster 2023-10-01T10:12:44.114856+0000 mon.a (mon.0) 513 : cluster [DBG] osdmap e47: 8 total, 7 up, 8 in 2023-10-01T10:12:45.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:45 smithi078 bash[27336]: audit 2023-10-01T10:12:44.115449+0000 mon.a (mon.0) 514 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T10:12:45.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:45 smithi078 bash[27336]: audit 2023-10-01T10:12:44.122931+0000 mon.a (mon.0) 515 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T10:12:45.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:45 smithi078 bash[27336]: audit 2023-10-01T10:12:44.513747+0000 mon.a (mon.0) 516 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:12:45.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:45 smithi078 bash[27336]: audit 2023-10-01T10:12:44.521712+0000 mon.a (mon.0) 517 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:12:45.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:45 smithi078 bash[33821]: cluster 2023-10-01T10:12:43.839309+0000 mgr.y (mgr.14146) 253 : cluster [DBG] pgmap v239: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T10:12:45.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:45 smithi078 bash[33821]: audit 2023-10-01T10:12:44.114767+0000 mon.a (mon.0) 512 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]': finished 2023-10-01T10:12:45.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:45 smithi078 bash[33821]: cluster 2023-10-01T10:12:44.114856+0000 mon.a (mon.0) 513 : cluster [DBG] osdmap e47: 8 total, 7 up, 8 in 2023-10-01T10:12:45.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:45 smithi078 bash[33821]: audit 2023-10-01T10:12:44.115449+0000 mon.a (mon.0) 514 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T10:12:45.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:45 smithi078 bash[33821]: audit 2023-10-01T10:12:44.122931+0000 mon.a (mon.0) 515 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T10:12:45.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:45 smithi078 bash[33821]: audit 2023-10-01T10:12:44.513747+0000 mon.a (mon.0) 516 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:12:45.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:45 smithi078 bash[33821]: audit 2023-10-01T10:12:44.521712+0000 mon.a (mon.0) 517 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:12:45.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:45 smithi173 bash[27612]: cluster 2023-10-01T10:12:43.839309+0000 mgr.y (mgr.14146) 253 : cluster [DBG] pgmap v239: 1 pgs: 1 active+clean; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T10:12:45.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:45 smithi173 bash[27612]: audit 2023-10-01T10:12:44.114767+0000 mon.a (mon.0) 512 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]': finished 2023-10-01T10:12:45.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:45 smithi173 bash[27612]: cluster 2023-10-01T10:12:44.114856+0000 mon.a (mon.0) 513 : cluster [DBG] osdmap e47: 8 total, 7 up, 8 in 2023-10-01T10:12:45.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:45 smithi173 bash[27612]: audit 2023-10-01T10:12:44.115449+0000 mon.a (mon.0) 514 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T10:12:45.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:45 smithi173 bash[27612]: audit 2023-10-01T10:12:44.122931+0000 mon.a (mon.0) 515 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T10:12:45.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:45 smithi173 bash[27612]: audit 2023-10-01T10:12:44.513747+0000 mon.a (mon.0) 516 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:12:45.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:45 smithi173 bash[27612]: audit 2023-10-01T10:12:44.521712+0000 mon.a (mon.0) 517 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:12:46.474 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:46 smithi078 bash[27336]: cluster 2023-10-01T10:12:43.447324+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-01T10:12:46.474 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:46 smithi078 bash[27336]: cluster 2023-10-01T10:12:43.447406+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-01T10:12:46.474 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:46 smithi078 bash[27336]: audit 2023-10-01T10:12:45.120870+0000 mon.a (mon.0) 518 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T10:12:46.474 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:46 smithi078 bash[27336]: cluster 2023-10-01T10:12:45.126351+0000 mon.a (mon.0) 519 : cluster [INF] osd.7 [v2:172.21.15.173:6824/1958390745,v1:172.21.15.173:6825/1958390745] boot 2023-10-01T10:12:46.474 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:46 smithi078 bash[27336]: cluster 2023-10-01T10:12:45.126463+0000 mon.a (mon.0) 520 : cluster [DBG] osdmap e48: 8 total, 8 up, 8 in 2023-10-01T10:12:46.475 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:46 smithi078 bash[27336]: audit 2023-10-01T10:12:45.127512+0000 mon.a (mon.0) 521 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T10:12:46.475 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:46 smithi078 bash[27336]: cluster 2023-10-01T10:12:45.913160+0000 mon.a (mon.0) 522 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2023-10-01T10:12:46.475 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:46 smithi078 bash[33821]: cluster 2023-10-01T10:12:43.447324+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-01T10:12:46.475 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:46 smithi078 bash[33821]: cluster 2023-10-01T10:12:43.447406+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-01T10:12:46.475 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:46 smithi078 bash[33821]: audit 2023-10-01T10:12:45.120870+0000 mon.a (mon.0) 518 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T10:12:46.475 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:46 smithi078 bash[33821]: cluster 2023-10-01T10:12:45.126351+0000 mon.a (mon.0) 519 : cluster [INF] osd.7 [v2:172.21.15.173:6824/1958390745,v1:172.21.15.173:6825/1958390745] boot 2023-10-01T10:12:46.475 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:46 smithi078 bash[33821]: cluster 2023-10-01T10:12:45.126463+0000 mon.a (mon.0) 520 : cluster [DBG] osdmap e48: 8 total, 8 up, 8 in 2023-10-01T10:12:46.475 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:46 smithi078 bash[33821]: audit 2023-10-01T10:12:45.127512+0000 mon.a (mon.0) 521 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T10:12:46.475 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:46 smithi078 bash[33821]: cluster 2023-10-01T10:12:45.913160+0000 mon.a (mon.0) 522 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2023-10-01T10:12:46.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:46 smithi173 bash[27612]: cluster 2023-10-01T10:12:43.447324+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2023-10-01T10:12:46.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:46 smithi173 bash[27612]: cluster 2023-10-01T10:12:43.447406+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2023-10-01T10:12:46.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:46 smithi173 bash[27612]: audit 2023-10-01T10:12:45.120870+0000 mon.a (mon.0) 518 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T10:12:46.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:46 smithi173 bash[27612]: cluster 2023-10-01T10:12:45.126351+0000 mon.a (mon.0) 519 : cluster [INF] osd.7 [v2:172.21.15.173:6824/1958390745,v1:172.21.15.173:6825/1958390745] boot 2023-10-01T10:12:46.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:46 smithi173 bash[27612]: cluster 2023-10-01T10:12:45.126463+0000 mon.a (mon.0) 520 : cluster [DBG] osdmap e48: 8 total, 8 up, 8 in 2023-10-01T10:12:46.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:46 smithi173 bash[27612]: audit 2023-10-01T10:12:45.127512+0000 mon.a (mon.0) 521 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T10:12:46.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:46 smithi173 bash[27612]: cluster 2023-10-01T10:12:45.913160+0000 mon.a (mon.0) 522 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2023-10-01T10:12:47.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:47 smithi078 bash[27336]: cluster 2023-10-01T10:12:45.839897+0000 mgr.y (mgr.14146) 254 : cluster [DBG] pgmap v242: 1 pgs: 1 remapped+peering; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T10:12:47.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:47 smithi078 bash[27336]: cluster 2023-10-01T10:12:46.132571+0000 mon.a (mon.0) 523 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-01T10:12:47.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:47 smithi078 bash[27336]: cluster 2023-10-01T10:12:46.913657+0000 mon.a (mon.0) 524 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2023-10-01T10:12:47.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:47 smithi078 bash[33821]: cluster 2023-10-01T10:12:45.839897+0000 mgr.y (mgr.14146) 254 : cluster [DBG] pgmap v242: 1 pgs: 1 remapped+peering; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T10:12:47.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:47 smithi078 bash[33821]: cluster 2023-10-01T10:12:46.132571+0000 mon.a (mon.0) 523 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-01T10:12:47.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:47 smithi078 bash[33821]: cluster 2023-10-01T10:12:46.913657+0000 mon.a (mon.0) 524 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2023-10-01T10:12:47.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:47 smithi173 bash[27612]: cluster 2023-10-01T10:12:45.839897+0000 mgr.y (mgr.14146) 254 : cluster [DBG] pgmap v242: 1 pgs: 1 remapped+peering; 0 B data, 2.0 GiB used, 624 GiB / 626 GiB avail 2023-10-01T10:12:47.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:47 smithi173 bash[27612]: cluster 2023-10-01T10:12:46.132571+0000 mon.a (mon.0) 523 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-10-01T10:12:47.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:47 smithi173 bash[27612]: cluster 2023-10-01T10:12:46.913657+0000 mon.a (mon.0) 524 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2023-10-01T10:12:48.543 INFO:teuthology.orchestra.run.smithi078.stdout: 2023-10-01T10:12:48.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:48 smithi078 bash[33821]: cluster 2023-10-01T10:12:48.145977+0000 mon.a (mon.0) 525 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-01T10:12:48.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:48 smithi078 bash[33821]: cluster 2023-10-01T10:12:48.146047+0000 mon.a (mon.0) 526 : cluster [INF] Cluster is now healthy 2023-10-01T10:12:48.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:48 smithi078 bash[27336]: cluster 2023-10-01T10:12:48.145977+0000 mon.a (mon.0) 525 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-01T10:12:48.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:48 smithi078 bash[27336]: cluster 2023-10-01T10:12:48.146047+0000 mon.a (mon.0) 526 : cluster [INF] Cluster is now healthy 2023-10-01T10:12:48.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:48 smithi173 bash[27612]: cluster 2023-10-01T10:12:48.145977+0000 mon.a (mon.0) 525 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-10-01T10:12:48.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:48 smithi173 bash[27612]: cluster 2023-10-01T10:12:48.146047+0000 mon.a (mon.0) 526 : cluster [INF] Cluster is now healthy 2023-10-01T10:12:49.344 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":50,"num_osds":8,"num_up_osds":8,"osd_up_since":1696155165,"num_in_osds":8,"osd_in_since":1696155139,"num_remapped_pgs":0} 2023-10-01T10:12:49.345 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-01T10:12:49.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:49 smithi078 bash[27336]: cluster 2023-10-01T10:12:47.840442+0000 mgr.y (mgr.14146) 255 : cluster [DBG] pgmap v245: 1 pgs: 1 peering; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:12:49.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:49 smithi078 bash[27336]: audit 2023-10-01T10:12:48.541250+0000 mon.c (mon.2) 14 : audit [DBG] from='client.? 172.21.15.78:0/3640417627' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-01T10:12:49.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:49 smithi078 bash[33821]: cluster 2023-10-01T10:12:47.840442+0000 mgr.y (mgr.14146) 255 : cluster [DBG] pgmap v245: 1 pgs: 1 peering; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:12:49.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:49 smithi078 bash[33821]: audit 2023-10-01T10:12:48.541250+0000 mon.c (mon.2) 14 : audit [DBG] from='client.? 172.21.15.78:0/3640417627' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-01T10:12:49.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:49 smithi173 bash[27612]: cluster 2023-10-01T10:12:47.840442+0000 mgr.y (mgr.14146) 255 : cluster [DBG] pgmap v245: 1 pgs: 1 peering; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:12:49.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:49 smithi173 bash[27612]: audit 2023-10-01T10:12:48.541250+0000 mon.c (mon.2) 14 : audit [DBG] from='client.? 172.21.15.78:0/3640417627' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-10-01T10:12:51.532 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:51 smithi173 bash[27612]: cluster 2023-10-01T10:12:49.841028+0000 mgr.y (mgr.14146) 256 : cluster [DBG] pgmap v246: 1 pgs: 1 peering; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:12:51.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:51 smithi078 bash[27336]: cluster 2023-10-01T10:12:49.841028+0000 mgr.y (mgr.14146) 256 : cluster [DBG] pgmap v246: 1 pgs: 1 peering; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:12:51.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:51 smithi078 bash[33821]: cluster 2023-10-01T10:12:49.841028+0000 mgr.y (mgr.14146) 256 : cluster [DBG] pgmap v246: 1 pgs: 1 peering; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:12:53.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:53 smithi078 bash[27336]: cluster 2023-10-01T10:12:51.841664+0000 mgr.y (mgr.14146) 257 : cluster [DBG] pgmap v247: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:12:53.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:53 smithi078 bash[33821]: cluster 2023-10-01T10:12:51.841664+0000 mgr.y (mgr.14146) 257 : cluster [DBG] pgmap v247: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:12:53.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:53 smithi173 bash[27612]: cluster 2023-10-01T10:12:51.841664+0000 mgr.y (mgr.14146) 257 : cluster [DBG] pgmap v247: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:12:54.097 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:12:54.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:54 smithi078 bash[27336]: cephadm 2023-10-01T10:12:53.263672+0000 mgr.y (mgr.14146) 258 : cephadm [INF] Detected new or changed devices on smithi173 2023-10-01T10:12:54.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:54 smithi078 bash[27336]: audit 2023-10-01T10:12:53.282332+0000 mon.a (mon.0) 527 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:12:54.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:54 smithi078 bash[27336]: audit 2023-10-01T10:12:53.283911+0000 mon.a (mon.0) 528 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:12:54.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:54 smithi078 bash[27336]: audit 2023-10-01T10:12:53.285032+0000 mon.a (mon.0) 529 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:12:54.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:54 smithi078 bash[27336]: audit 2023-10-01T10:12:53.286076+0000 mon.a (mon.0) 530 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:12:54.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:54 smithi078 bash[27336]: audit 2023-10-01T10:12:53.287095+0000 mon.a (mon.0) 531 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:12:54.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:54 smithi078 bash[27336]: cephadm 2023-10-01T10:12:53.287887+0000 mgr.y (mgr.14146) 259 : cephadm [INF] Adjusting osd_memory_target on smithi173 to 4316M 2023-10-01T10:12:54.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:54 smithi078 bash[27336]: audit 2023-10-01T10:12:53.294375+0000 mon.a (mon.0) 532 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:12:54.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:54 smithi078 bash[27336]: audit 2023-10-01T10:12:53.296038+0000 mon.a (mon.0) 533 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:12:54.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:54 smithi078 bash[27336]: audit 2023-10-01T10:12:53.297306+0000 mon.a (mon.0) 534 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:12:54.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:54 smithi078 bash[27336]: audit 2023-10-01T10:12:53.304778+0000 mon.a (mon.0) 535 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:12:54.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:54 smithi078 bash[33821]: cephadm 2023-10-01T10:12:53.263672+0000 mgr.y (mgr.14146) 258 : cephadm [INF] Detected new or changed devices on smithi173 2023-10-01T10:12:54.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:54 smithi078 bash[33821]: audit 2023-10-01T10:12:53.282332+0000 mon.a (mon.0) 527 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:12:54.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:54 smithi078 bash[33821]: audit 2023-10-01T10:12:53.283911+0000 mon.a (mon.0) 528 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:12:54.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:54 smithi078 bash[33821]: audit 2023-10-01T10:12:53.285032+0000 mon.a (mon.0) 529 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:12:54.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:54 smithi078 bash[33821]: audit 2023-10-01T10:12:53.286076+0000 mon.a (mon.0) 530 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:12:54.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:54 smithi078 bash[33821]: audit 2023-10-01T10:12:53.287095+0000 mon.a (mon.0) 531 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:12:54.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:54 smithi078 bash[33821]: cephadm 2023-10-01T10:12:53.287887+0000 mgr.y (mgr.14146) 259 : cephadm [INF] Adjusting osd_memory_target on smithi173 to 4316M 2023-10-01T10:12:54.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:54 smithi078 bash[33821]: audit 2023-10-01T10:12:53.294375+0000 mon.a (mon.0) 532 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:12:54.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:54 smithi078 bash[33821]: audit 2023-10-01T10:12:53.296038+0000 mon.a (mon.0) 533 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:12:54.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:54 smithi078 bash[33821]: audit 2023-10-01T10:12:53.297306+0000 mon.a (mon.0) 534 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:12:54.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:54 smithi078 bash[33821]: audit 2023-10-01T10:12:53.304778+0000 mon.a (mon.0) 535 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:12:54.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:54 smithi173 bash[27612]: cephadm 2023-10-01T10:12:53.263672+0000 mgr.y (mgr.14146) 258 : cephadm [INF] Detected new or changed devices on smithi173 2023-10-01T10:12:54.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:54 smithi173 bash[27612]: audit 2023-10-01T10:12:53.282332+0000 mon.a (mon.0) 527 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:12:54.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:54 smithi173 bash[27612]: audit 2023-10-01T10:12:53.283911+0000 mon.a (mon.0) 528 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:12:54.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:54 smithi173 bash[27612]: audit 2023-10-01T10:12:53.285032+0000 mon.a (mon.0) 529 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:12:54.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:54 smithi173 bash[27612]: audit 2023-10-01T10:12:53.286076+0000 mon.a (mon.0) 530 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:12:54.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:54 smithi173 bash[27612]: audit 2023-10-01T10:12:53.287095+0000 mon.a (mon.0) 531 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:12:54.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:54 smithi173 bash[27612]: cephadm 2023-10-01T10:12:53.287887+0000 mgr.y (mgr.14146) 259 : cephadm [INF] Adjusting osd_memory_target on smithi173 to 4316M 2023-10-01T10:12:54.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:54 smithi173 bash[27612]: audit 2023-10-01T10:12:53.294375+0000 mon.a (mon.0) 532 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:12:54.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:54 smithi173 bash[27612]: audit 2023-10-01T10:12:53.296038+0000 mon.a (mon.0) 533 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:12:54.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:54 smithi173 bash[27612]: audit 2023-10-01T10:12:53.297306+0000 mon.a (mon.0) 534 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:12:54.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:54 smithi173 bash[27612]: audit 2023-10-01T10:12:53.304778+0000 mon.a (mon.0) 535 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:12:55.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:55 smithi078 bash[27336]: cluster 2023-10-01T10:12:53.842188+0000 mgr.y (mgr.14146) 260 : cluster [DBG] pgmap v248: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:12:55.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:55 smithi078 bash[33821]: cluster 2023-10-01T10:12:53.842188+0000 mgr.y (mgr.14146) 260 : cluster [DBG] pgmap v248: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:12:55.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:55 smithi173 bash[27612]: cluster 2023-10-01T10:12:53.842188+0000 mgr.y (mgr.14146) 260 : cluster [DBG] pgmap v248: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:12:57.176 INFO:teuthology.orchestra.run.smithi078.stdout: 2023-10-01T10:12:57.176 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":50,"fsid":"d8016480-6041-11ee-8db5-212e2dc638e7","created":"2023-10-01T10:04:59.345633+0000","modified":"2023-10-01T10:12:46.906785+0000","last_up_change":"2023-10-01T10:12:45.118874+0000","last_in_change":"2023-10-01T10:12:19.634793+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-10-01T10:09:16.689172+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}}],"osds":[{"osd":0,"uuid":"3cdf50f0-cae3-4e38-a7ca-c24c4a07b615","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6802","nonce":575557942},{"type":"v1","addr":"172.21.15.78:6803","nonce":575557942}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6804","nonce":575557942},{"type":"v1","addr":"172.21.15.78:6805","nonce":575557942}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6808","nonce":575557942},{"type":"v1","addr":"172.21.15.78:6809","nonce":575557942}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6806","nonce":575557942},{"type":"v1","addr":"172.21.15.78:6807","nonce":575557942}]},"public_addr":"172.21.15.78:6803/575557942","cluster_addr":"172.21.15.78:6805/575557942","heartbeat_back_addr":"172.21.15.78:6809/575557942","heartbeat_front_addr":"172.21.15.78:6807/575557942","state":["exists","up"]},{"osd":1,"uuid":"4e6bea65-4109-43d2-b5be-3811fdd2a803","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6810","nonce":3462774695},{"type":"v1","addr":"172.21.15.78:6811","nonce":3462774695}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6812","nonce":3462774695},{"type":"v1","addr":"172.21.15.78:6813","nonce":3462774695}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6816","nonce":3462774695},{"type":"v1","addr":"172.21.15.78:6817","nonce":3462774695}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6814","nonce":3462774695},{"type":"v1","addr":"172.21.15.78:6815","nonce":3462774695}]},"public_addr":"172.21.15.78:6811/3462774695","cluster_addr":"172.21.15.78:6813/3462774695","heartbeat_back_addr":"172.21.15.78:6817/3462774695","heartbeat_front_addr":"172.21.15.78:6815/3462774695","state":["exists","up"]},{"osd":2,"uuid":"9f830594-31d6-4994-85fa-0615c280d5c3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6818","nonce":1190788353},{"type":"v1","addr":"172.21.15.78:6819","nonce":1190788353}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6820","nonce":1190788353},{"type":"v1","addr":"172.21.15.78:6821","nonce":1190788353}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6824","nonce":1190788353},{"type":"v1","addr":"172.21.15.78:6825","nonce":1190788353}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6822","nonce":1190788353},{"type":"v1","addr":"172.21.15.78:6823","nonce":1190788353}]},"public_addr":"172.21.15.78:6819/1190788353","cluster_addr":"172.21.15.78:6821/1190788353","heartbeat_back_addr":"172.21.15.78:6825/1190788353","heartbeat_front_addr":"172.21.15.78:6823/1190788353","state":["exists","up"]},{"osd":3,"uuid":"2780842b-2f4f-4190-b4a5-279a0444adf2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6826","nonce":53937366},{"type":"v1","addr":"172.21.15.78:6827","nonce":53937366}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6828","nonce":53937366},{"type":"v1","addr":"172.21.15.78:6829","nonce":53937366}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6832","nonce":53937366},{"type":"v1","addr":"172.21.15.78:6833","nonce":53937366}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6830","nonce":53937366},{"type":"v1","addr":"172.21.15.78:6831","nonce":53937366}]},"public_addr":"172.21.15.78:6827/53937366","cluster_addr":"172.21.15.78:6829/53937366","heartbeat_back_addr":"172.21.15.78:6833/53937366","heartbeat_front_addr":"172.21.15.78:6831/53937366","state":["exists","up"]},{"osd":4,"uuid":"57c30778-fd7e-4f3e-afd9-389ef377cb62","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6800","nonce":2430591286},{"type":"v1","addr":"172.21.15.173:6801","nonce":2430591286}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6802","nonce":2430591286},{"type":"v1","addr":"172.21.15.173:6803","nonce":2430591286}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6806","nonce":2430591286},{"type":"v1","addr":"172.21.15.173:6807","nonce":2430591286}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6804","nonce":2430591286},{"type":"v1","addr":"172.21.15.173:6805","nonce":2430591286}]},"public_addr":"172.21.15.173:6801/2430591286","cluster_addr":"172.21.15.173:6803/2430591286","heartbeat_back_addr":"172.21.15.173:6807/2430591286","heartbeat_front_addr":"172.21.15.173:6805/2430591286","state":["exists","up"]},{"osd":5,"uuid":"c62aa7c3-2d09-4088-ba36-c84588dc3549","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":37,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6808","nonce":2798643181},{"type":"v1","addr":"172.21.15.173:6809","nonce":2798643181}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6810","nonce":2798643181},{"type":"v1","addr":"172.21.15.173:6811","nonce":2798643181}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6814","nonce":2798643181},{"type":"v1","addr":"172.21.15.173:6815","nonce":2798643181}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6812","nonce":2798643181},{"type":"v1","addr":"172.21.15.173:6813","nonce":2798643181}]},"public_addr":"172.21.15.173:6809/2798643181","cluster_addr":"172.21.15.173:6811/2798643181","heartbeat_back_addr":"172.21.15.173:6815/2798643181","heartbeat_front_addr":"172.21.15.173:6813/2798643181","state":["exists","up"]},{"osd":6,"uuid":"b140330d-dd93-4ce4-9b9a-2ec6321419b3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6816","nonce":1568190351},{"type":"v1","addr":"172.21.15.173:6817","nonce":1568190351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6818","nonce":1568190351},{"type":"v1","addr":"172.21.15.173:6819","nonce":1568190351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6822","nonce":1568190351},{"type":"v1","addr":"172.21.15.173:6823","nonce":1568190351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6820","nonce":1568190351},{"type":"v1","addr":"172.21.15.173:6821","nonce":1568190351}]},"public_addr":"172.21.15.173:6817/1568190351","cluster_addr":"172.21.15.173:6819/1568190351","heartbeat_back_addr":"172.21.15.173:6823/1568190351","heartbeat_front_addr":"172.21.15.173:6821/1568190351","state":["exists","up"]},{"osd":7,"uuid":"7590027f-50c4-4457-843d-cebaa734d2ea","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6824","nonce":1958390745},{"type":"v1","addr":"172.21.15.173:6825","nonce":1958390745}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6826","nonce":1958390745},{"type":"v1","addr":"172.21.15.173:6827","nonce":1958390745}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6830","nonce":1958390745},{"type":"v1","addr":"172.21.15.173:6831","nonce":1958390745}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6828","nonce":1958390745},{"type":"v1","addr":"172.21.15.173:6829","nonce":1958390745}]},"public_addr":"172.21.15.173:6825/1958390745","cluster_addr":"172.21.15.173:6827/1958390745","heartbeat_back_addr":"172.21.15.173:6831/1958390745","heartbeat_front_addr":"172.21.15.173:6829/1958390745","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:07:47.950416+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:08:31.368397+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:09:14.866866+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:09:59.445542+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:10:39.550033+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:11:20.934732+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:12:01.654044+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:12:43.447412+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.78:0/3939471097":"2023-10-02T10:05:57.727322+0000","172.21.15.78:6800/1700562157":"2023-10-02T10:05:57.727322+0000","172.21.15.78:0/3635389419":"2023-10-02T10:05:57.727322+0000","172.21.15.78:6800/3027286102":"2023-10-02T10:05:27.007550+0000","172.21.15.78:6801/3027286102":"2023-10-02T10:05:27.007550+0000","172.21.15.78:0/308254153":"2023-10-02T10:05:27.007550+0000","172.21.15.78:6801/1700562157":"2023-10-02T10:05:57.727322+0000","172.21.15.78:0/158328292":"2023-10-02T10:05:27.007550+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-01T10:12:57.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:57 smithi078 bash[27336]: cluster 2023-10-01T10:12:55.842735+0000 mgr.y (mgr.14146) 261 : cluster [DBG] pgmap v249: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:12:57.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:57 smithi078 bash[27336]: audit 2023-10-01T10:12:57.172517+0000 mon.c (mon.2) 15 : audit [DBG] from='client.? 172.21.15.78:0/2852101983' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T10:12:57.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:57 smithi078 bash[33821]: cluster 2023-10-01T10:12:55.842735+0000 mgr.y (mgr.14146) 261 : cluster [DBG] pgmap v249: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:12:57.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:57 smithi078 bash[33821]: audit 2023-10-01T10:12:57.172517+0000 mon.c (mon.2) 15 : audit [DBG] from='client.? 172.21.15.78:0/2852101983' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T10:12:57.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:57 smithi173 bash[27612]: cluster 2023-10-01T10:12:55.842735+0000 mgr.y (mgr.14146) 261 : cluster [DBG] pgmap v249: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:12:57.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:57 smithi173 bash[27612]: audit 2023-10-01T10:12:57.172517+0000 mon.c (mon.2) 15 : audit [DBG] from='client.? 172.21.15.78:0/2852101983' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T10:12:57.916 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': 'device_health_metrics', 'create_time': '2023-10-01T10:09:16.689172+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr_devicehealth': {}}}] 2023-10-01T10:12:57.916 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd pool get device_health_metrics pg_num 2023-10-01T10:12:59.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:12:59 smithi078 bash[27336]: cluster 2023-10-01T10:12:57.843285+0000 mgr.y (mgr.14146) 262 : cluster [DBG] pgmap v250: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:12:59.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:12:59 smithi078 bash[33821]: cluster 2023-10-01T10:12:57.843285+0000 mgr.y (mgr.14146) 262 : cluster [DBG] pgmap v250: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:12:59.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:12:59 smithi173 bash[27612]: cluster 2023-10-01T10:12:57.843285+0000 mgr.y (mgr.14146) 262 : cluster [DBG] pgmap v250: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:01.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:01 smithi078 bash[27336]: cluster 2023-10-01T10:12:59.843846+0000 mgr.y (mgr.14146) 263 : cluster [DBG] pgmap v251: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:01.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:01 smithi078 bash[33821]: cluster 2023-10-01T10:12:59.843846+0000 mgr.y (mgr.14146) 263 : cluster [DBG] pgmap v251: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:01.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:01 smithi173 bash[27612]: cluster 2023-10-01T10:12:59.843846+0000 mgr.y (mgr.14146) 263 : cluster [DBG] pgmap v251: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:02.674 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:13:03.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:03 smithi078 bash[27336]: cluster 2023-10-01T10:13:01.844401+0000 mgr.y (mgr.14146) 264 : cluster [DBG] pgmap v252: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:03.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:03 smithi078 bash[33821]: cluster 2023-10-01T10:13:01.844401+0000 mgr.y (mgr.14146) 264 : cluster [DBG] pgmap v252: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:03.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:03 smithi173 bash[27612]: cluster 2023-10-01T10:13:01.844401+0000 mgr.y (mgr.14146) 264 : cluster [DBG] pgmap v252: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:05.577 INFO:teuthology.orchestra.run.smithi078.stdout:pg_num: 1 2023-10-01T10:13:05.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:05 smithi078 bash[27336]: cluster 2023-10-01T10:13:03.844936+0000 mgr.y (mgr.14146) 265 : cluster [DBG] pgmap v253: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:05.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:05 smithi078 bash[33821]: cluster 2023-10-01T10:13:03.844936+0000 mgr.y (mgr.14146) 265 : cluster [DBG] pgmap v253: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:05.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:05 smithi173 bash[27612]: cluster 2023-10-01T10:13:03.844936+0000 mgr.y (mgr.14146) 265 : cluster [DBG] pgmap v253: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:06.259 INFO:tasks.cephadm:Adding prometheus.a on smithi173 2023-10-01T10:13:06.259 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph orch apply prometheus '1;smithi173=a' 2023-10-01T10:13:06.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:06 smithi078 bash[27336]: audit 2023-10-01T10:13:05.574993+0000 mon.c (mon.2) 16 : audit [DBG] from='client.? 172.21.15.78:0/4206447205' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2023-10-01T10:13:06.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:06 smithi078 bash[33821]: audit 2023-10-01T10:13:05.574993+0000 mon.c (mon.2) 16 : audit [DBG] from='client.? 172.21.15.78:0/4206447205' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2023-10-01T10:13:06.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:06 smithi173 bash[27612]: audit 2023-10-01T10:13:05.574993+0000 mon.c (mon.2) 16 : audit [DBG] from='client.? 172.21.15.78:0/4206447205' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "device_health_metrics", "var": "pg_num"}]: dispatch 2023-10-01T10:13:07.610 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:07 smithi173 bash[27612]: cluster 2023-10-01T10:13:05.845408+0000 mgr.y (mgr.14146) 266 : cluster [DBG] pgmap v254: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:07.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:07 smithi078 bash[27336]: cluster 2023-10-01T10:13:05.845408+0000 mgr.y (mgr.14146) 266 : cluster [DBG] pgmap v254: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:07.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:07 smithi078 bash[33821]: cluster 2023-10-01T10:13:05.845408+0000 mgr.y (mgr.14146) 266 : cluster [DBG] pgmap v254: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:08.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:08 smithi173 bash[27612]: cluster 2023-10-01T10:13:07.845890+0000 mgr.y (mgr.14146) 267 : cluster [DBG] pgmap v255: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:08.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:08 smithi078 bash[27336]: cluster 2023-10-01T10:13:07.845890+0000 mgr.y (mgr.14146) 267 : cluster [DBG] pgmap v255: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:08.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:08 smithi078 bash[33821]: cluster 2023-10-01T10:13:07.845890+0000 mgr.y (mgr.14146) 267 : cluster [DBG] pgmap v255: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:09.221 INFO:teuthology.orchestra.run.smithi173.stdout:Scheduled prometheus update... 2023-10-01T10:13:09.885 DEBUG:teuthology.orchestra.run.smithi173:prometheus.a> sudo journalctl -f -n 0 -u ceph-d8016480-6041-11ee-8db5-212e2dc638e7@prometheus.a.service 2023-10-01T10:13:09.887 INFO:tasks.cephadm:Adding node-exporter.a on smithi078 2023-10-01T10:13:09.888 INFO:tasks.cephadm:Adding node-exporter.b on smithi173 2023-10-01T10:13:09.888 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph orch apply node-exporter '2;smithi078=a;smithi173=b' 2023-10-01T10:13:10.052 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-01T10:13:10.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:10 smithi078 bash[27336]: audit 2023-10-01T10:13:09.208895+0000 mgr.y (mgr.14146) 268 : audit [DBG] from='client.24337 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi173=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:13:10.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:10 smithi078 bash[27336]: cephadm 2023-10-01T10:13:09.212282+0000 mgr.y (mgr.14146) 269 : cephadm [INF] Saving service prometheus spec with placement smithi173=a;count:1 2023-10-01T10:13:10.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:10 smithi078 bash[27336]: audit 2023-10-01T10:13:09.218495+0000 mon.a (mon.0) 536 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:13:10.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:10 smithi078 bash[27336]: audit 2023-10-01T10:13:09.265717+0000 mon.a (mon.0) 537 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:13:10.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:10 smithi078 bash[27336]: audit 2023-10-01T10:13:09.542296+0000 mon.a (mon.0) 538 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:13:10.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:10 smithi078 bash[27336]: audit 2023-10-01T10:13:09.544061+0000 mon.a (mon.0) 539 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:13:10.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:10 smithi078 bash[27336]: audit 2023-10-01T10:13:09.552017+0000 mon.a (mon.0) 540 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:13:10.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:10 smithi078 bash[27336]: audit 2023-10-01T10:13:09.560471+0000 mon.a (mon.0) 541 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-10-01T10:13:10.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:10 smithi078 bash[33821]: audit 2023-10-01T10:13:09.208895+0000 mgr.y (mgr.14146) 268 : audit [DBG] from='client.24337 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi173=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:13:10.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:10 smithi078 bash[33821]: cephadm 2023-10-01T10:13:09.212282+0000 mgr.y (mgr.14146) 269 : cephadm [INF] Saving service prometheus spec with placement smithi173=a;count:1 2023-10-01T10:13:10.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:10 smithi078 bash[33821]: audit 2023-10-01T10:13:09.218495+0000 mon.a (mon.0) 536 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:13:10.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:10 smithi078 bash[33821]: audit 2023-10-01T10:13:09.265717+0000 mon.a (mon.0) 537 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:13:10.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:10 smithi078 bash[33821]: audit 2023-10-01T10:13:09.542296+0000 mon.a (mon.0) 538 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:13:10.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:10 smithi078 bash[33821]: audit 2023-10-01T10:13:09.544061+0000 mon.a (mon.0) 539 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:13:10.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:10 smithi078 bash[33821]: audit 2023-10-01T10:13:09.552017+0000 mon.a (mon.0) 540 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:13:10.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:10 smithi078 bash[33821]: audit 2023-10-01T10:13:09.560471+0000 mon.a (mon.0) 541 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-10-01T10:13:10.602 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:10 smithi173 bash[27612]: audit 2023-10-01T10:13:09.208895+0000 mgr.y (mgr.14146) 268 : audit [DBG] from='client.24337 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi173=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:13:10.602 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:10 smithi173 bash[27612]: cephadm 2023-10-01T10:13:09.212282+0000 mgr.y (mgr.14146) 269 : cephadm [INF] Saving service prometheus spec with placement smithi173=a;count:1 2023-10-01T10:13:10.603 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:10 smithi173 bash[27612]: audit 2023-10-01T10:13:09.218495+0000 mon.a (mon.0) 536 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:13:10.603 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:10 smithi173 bash[27612]: audit 2023-10-01T10:13:09.265717+0000 mon.a (mon.0) 537 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:13:10.603 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:10 smithi173 bash[27612]: audit 2023-10-01T10:13:09.542296+0000 mon.a (mon.0) 538 : audit [DBG] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:13:10.603 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:10 smithi173 bash[27612]: audit 2023-10-01T10:13:09.544061+0000 mon.a (mon.0) 539 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:13:10.603 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:10 smithi173 bash[27612]: audit 2023-10-01T10:13:09.552017+0000 mon.a (mon.0) 540 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' 2023-10-01T10:13:10.603 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:10 smithi173 bash[27612]: audit 2023-10-01T10:13:09.560471+0000 mon.a (mon.0) 541 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-10-01T10:13:10.871 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:13:10 smithi173 bash[28473]: ignoring --setuser ceph since I am not root 2023-10-01T10:13:11.988 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:13:10 smithi173 bash[28473]: ignoring --setgroup ceph since I am not root 2023-10-01T10:13:11.988 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:13:10 smithi173 bash[28473]: debug 2023-10-01T10:13:10.611+0000 7fd88730b700 1 -- 172.21.15.173:0/2599990607 <== mon.1 v2:172.21.15.173:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x56317d5d04e0 con 0x56317d6c4800 2023-10-01T10:13:11.988 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:13:10 smithi173 bash[28473]: debug 2023-10-01T10:13:10.767+0000 7fd8974d2680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-01T10:13:11.989 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:13:10 smithi173 bash[28473]: debug 2023-10-01T10:13:10.875+0000 7fd8974d2680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-01T10:13:11.989 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:13:10 smithi078 bash[27601]: ignoring --setuser ceph since I am not root 2023-10-01T10:13:11.989 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:13:10 smithi078 bash[27601]: ignoring --setgroup ceph since I am not root 2023-10-01T10:13:11.989 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:13:10 smithi078 bash[27601]: debug 2023-10-01T10:13:10.607+0000 7f3e04422700 1 -- 172.21.15.78:0/2956357301 <== mon.0 v2:172.21.15.78:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55c616df84e0 con 0x55c616eec800 2023-10-01T10:13:11.989 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:13:10 smithi078 bash[27601]: debug 2023-10-01T10:13:10.763+0000 7f3e145e9680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-01T10:13:11.990 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:13:10 smithi078 bash[27601]: debug 2023-10-01T10:13:10.871+0000 7f3e145e9680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-01T10:13:11.990 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:13:11 smithi078 bash[27601]: debug 2023-10-01T10:13:11.323+0000 7f3e145e9680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-01T10:13:11.990 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:11 smithi173 bash[27612]: cluster 2023-10-01T10:13:09.846375+0000 mgr.y (mgr.14146) 270 : cluster [DBG] pgmap v256: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:11.990 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:11 smithi173 bash[27612]: audit 2023-10-01T10:13:10.563975+0000 mon.a (mon.0) 542 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-10-01T10:13:11.990 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:11 smithi173 bash[27612]: cluster 2023-10-01T10:13:10.564079+0000 mon.a (mon.0) 543 : cluster [DBG] mgrmap e15: y(active, since 7m), standbys: x 2023-10-01T10:13:11.990 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:11 smithi078 bash[27336]: cluster 2023-10-01T10:13:09.846375+0000 mgr.y (mgr.14146) 270 : cluster [DBG] pgmap v256: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:11.991 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:11 smithi078 bash[27336]: audit 2023-10-01T10:13:10.563975+0000 mon.a (mon.0) 542 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-10-01T10:13:11.991 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:11 smithi078 bash[27336]: cluster 2023-10-01T10:13:10.564079+0000 mon.a (mon.0) 543 : cluster [DBG] mgrmap e15: y(active, since 7m), standbys: x 2023-10-01T10:13:11.991 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:11 smithi078 bash[33821]: cluster 2023-10-01T10:13:09.846375+0000 mgr.y (mgr.14146) 270 : cluster [DBG] pgmap v256: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:11.991 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:11 smithi078 bash[33821]: audit 2023-10-01T10:13:10.563975+0000 mon.a (mon.0) 542 : audit [INF] from='mgr.14146 172.21.15.78:0/129949741' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-10-01T10:13:11.991 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:11 smithi078 bash[33821]: cluster 2023-10-01T10:13:10.564079+0000 mon.a (mon.0) 543 : cluster [DBG] mgrmap e15: y(active, since 7m), standbys: x 2023-10-01T10:13:11.991 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:13:11 smithi173 bash[28473]: debug 2023-10-01T10:13:11.375+0000 7fd8974d2680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-01T10:13:12.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:13:12 smithi078 bash[27601]: debug 2023-10-01T10:13:12.243+0000 7f3e145e9680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-01T10:13:12.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:13:12 smithi078 bash[27601]: debug 2023-10-01T10:13:12.343+0000 7f3e145e9680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-01T10:13:12.620 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:13:12 smithi173 bash[28473]: debug 2023-10-01T10:13:12.343+0000 7fd8974d2680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-01T10:13:12.621 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:13:12 smithi173 bash[28473]: debug 2023-10-01T10:13:12.443+0000 7fd8974d2680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-01T10:13:13.082 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:13:12 smithi078 bash[27601]: debug 2023-10-01T10:13:12.599+0000 7f3e145e9680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-01T10:13:13.120 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:13:12 smithi173 bash[28473]: debug 2023-10-01T10:13:12.699+0000 7fd8974d2680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-01T10:13:14.014 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:13:13 smithi078 bash[27601]: debug 2023-10-01T10:13:13.739+0000 7f3e145e9680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-01T10:13:14.014 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:13:13 smithi078 bash[27601]: debug 2023-10-01T10:13:13.907+0000 7f3e145e9680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-01T10:13:14.119 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:13:13 smithi173 bash[28473]: debug 2023-10-01T10:13:13.851+0000 7fd8974d2680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-01T10:13:14.119 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:13:14 smithi173 bash[28473]: debug 2023-10-01T10:13:14.015+0000 7fd8974d2680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-01T10:13:14.332 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:13:14 smithi078 bash[27601]: debug 2023-10-01T10:13:14.007+0000 7f3e145e9680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-01T10:13:14.333 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:13:14 smithi078 bash[27601]: debug 2023-10-01T10:13:14.123+0000 7f3e145e9680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-01T10:13:14.370 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:13:14 smithi173 bash[28473]: debug 2023-10-01T10:13:14.115+0000 7fd8974d2680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-01T10:13:14.371 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:13:14 smithi173 bash[28473]: debug 2023-10-01T10:13:14.231+0000 7fd8974d2680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-01T10:13:14.714 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:13:14 smithi078 bash[27601]: debug 2023-10-01T10:13:14.343+0000 7f3e145e9680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-01T10:13:14.832 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:13:14 smithi173 bash[28473]: debug 2023-10-01T10:13:14.459+0000 7fd8974d2680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-01T10:13:15.082 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:13:14 smithi078 bash[27601]: debug 2023-10-01T10:13:14.707+0000 7f3e145e9680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-01T10:13:15.083 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:13:14 smithi078 bash[27601]: debug 2023-10-01T10:13:14.831+0000 7f3e145e9680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-01T10:13:15.120 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:13:14 smithi173 bash[28473]: debug 2023-10-01T10:13:14.827+0000 7fd8974d2680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-01T10:13:15.120 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:13:14 smithi173 bash[28473]: debug 2023-10-01T10:13:14.951+0000 7fd8974d2680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-01T10:13:16.082 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:13:15 smithi078 bash[27601]: debug 2023-10-01T10:13:15.695+0000 7f3e145e9680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-01T10:13:16.083 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:13:15 smithi078 bash[27601]: debug 2023-10-01T10:13:15.799+0000 7f3e145e9680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-01T10:13:16.083 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:13:15 smithi078 bash[27601]: debug 2023-10-01T10:13:15.919+0000 7f3e145e9680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-01T10:13:16.120 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:13:15 smithi173 bash[28473]: debug 2023-10-01T10:13:15.823+0000 7fd8974d2680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-01T10:13:16.120 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:13:15 smithi173 bash[28473]: debug 2023-10-01T10:13:15.931+0000 7fd8974d2680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-01T10:13:16.120 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:13:16 smithi173 bash[28473]: debug 2023-10-01T10:13:16.047+0000 7fd8974d2680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-01T10:13:16.457 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:13:16 smithi078 bash[27601]: debug 2023-10-01T10:13:16.159+0000 7f3e145e9680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-01T10:13:16.567 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:13:16 smithi173 bash[28473]: debug 2023-10-01T10:13:16.267+0000 7fd8974d2680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-01T10:13:16.794 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:13:16 smithi078 bash[27601]: debug 2023-10-01T10:13:16.451+0000 7f3e145e9680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-01T10:13:16.794 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:13:16 smithi078 bash[27601]: debug 2023-10-01T10:13:16.615+0000 7f3e145e9680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-01T10:13:16.870 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:13:16 smithi173 bash[28473]: debug 2023-10-01T10:13:16.563+0000 7fd8974d2680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-01T10:13:16.870 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:13:16 smithi173 bash[28473]: debug 2023-10-01T10:13:16.727+0000 7fd8974d2680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-01T10:13:17.082 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:13:16 smithi078 bash[27601]: debug 2023-10-01T10:13:16.787+0000 7f3e145e9680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-01T10:13:17.083 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:13:17 smithi078 bash[27601]: debug 2023-10-01T10:13:17.007+0000 7f3e145e9680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-01T10:13:17.193 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:13:16 smithi173 bash[28473]: debug 2023-10-01T10:13:16.899+0000 7fd8974d2680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-01T10:13:17.193 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:13:17 smithi173 bash[28473]: debug 2023-10-01T10:13:17.111+0000 7fd8974d2680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-01T10:13:17.376 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:17 smithi078 bash[27336]: cluster 2023-10-01T10:13:17.135523+0000 mon.a (mon.0) 544 : cluster [INF] Active manager daemon y restarted 2023-10-01T10:13:17.377 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:17 smithi078 bash[27336]: cluster 2023-10-01T10:13:17.136054+0000 mon.a (mon.0) 545 : cluster [INF] Activating manager daemon y 2023-10-01T10:13:17.377 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:17 smithi078 bash[27336]: cluster 2023-10-01T10:13:17.144891+0000 mon.a (mon.0) 546 : cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2023-10-01T10:13:17.377 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:17 smithi078 bash[27336]: cluster 2023-10-01T10:13:17.145460+0000 mon.a (mon.0) 547 : cluster [DBG] mgrmap e16: y(active, starting, since 0.00958142s), standbys: x 2023-10-01T10:13:17.377 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:17 smithi078 bash[27336]: audit 2023-10-01T10:13:17.147919+0000 mon.a (mon.0) 548 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T10:13:17.377 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:17 smithi078 bash[27336]: audit 2023-10-01T10:13:17.148217+0000 mon.a (mon.0) 549 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T10:13:17.377 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:17 smithi078 bash[27336]: audit 2023-10-01T10:13:17.148525+0000 mon.a (mon.0) 550 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T10:13:17.377 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:17 smithi078 bash[27336]: audit 2023-10-01T10:13:17.149698+0000 mon.a (mon.0) 551 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-01T10:13:17.377 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:17 smithi078 bash[27336]: audit 2023-10-01T10:13:17.149983+0000 mon.a (mon.0) 552 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-01T10:13:17.377 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:17 smithi078 bash[27336]: audit 2023-10-01T10:13:17.150263+0000 mon.a (mon.0) 553 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T10:13:17.377 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:17 smithi078 bash[27336]: audit 2023-10-01T10:13:17.150516+0000 mon.a (mon.0) 554 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T10:13:17.378 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:17 smithi078 bash[27336]: audit 2023-10-01T10:13:17.150760+0000 mon.a (mon.0) 555 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T10:13:17.378 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:17 smithi078 bash[27336]: audit 2023-10-01T10:13:17.151062+0000 mon.a (mon.0) 556 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T10:13:17.378 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:17 smithi078 bash[27336]: audit 2023-10-01T10:13:17.151315+0000 mon.a (mon.0) 557 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T10:13:17.378 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:17 smithi078 bash[27336]: audit 2023-10-01T10:13:17.151563+0000 mon.a (mon.0) 558 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T10:13:17.378 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:17 smithi078 bash[27336]: audit 2023-10-01T10:13:17.151808+0000 mon.a (mon.0) 559 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T10:13:17.378 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:17 smithi078 bash[27336]: audit 2023-10-01T10:13:17.152057+0000 mon.a (mon.0) 560 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T10:13:17.378 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:17 smithi078 bash[27336]: audit 2023-10-01T10:13:17.152313+0000 mon.a (mon.0) 561 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-01T10:13:17.378 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:17 smithi078 bash[27336]: audit 2023-10-01T10:13:17.152496+0000 mon.a (mon.0) 562 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-01T10:13:17.378 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:17 smithi078 bash[27336]: audit 2023-10-01T10:13:17.153089+0000 mon.a (mon.0) 563 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-01T10:13:17.378 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:17 smithi078 bash[27336]: cluster 2023-10-01T10:13:17.178162+0000 mon.a (mon.0) 564 : cluster [INF] Manager daemon y is now available 2023-10-01T10:13:17.379 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:17 smithi078 bash[33821]: cluster 2023-10-01T10:13:17.135523+0000 mon.a (mon.0) 544 : cluster [INF] Active manager daemon y restarted 2023-10-01T10:13:17.379 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:17 smithi078 bash[33821]: cluster 2023-10-01T10:13:17.136054+0000 mon.a (mon.0) 545 : cluster [INF] Activating manager daemon y 2023-10-01T10:13:17.379 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:17 smithi078 bash[33821]: cluster 2023-10-01T10:13:17.144891+0000 mon.a (mon.0) 546 : cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2023-10-01T10:13:17.379 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:17 smithi078 bash[33821]: cluster 2023-10-01T10:13:17.145460+0000 mon.a (mon.0) 547 : cluster [DBG] mgrmap e16: y(active, starting, since 0.00958142s), standbys: x 2023-10-01T10:13:17.379 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:17 smithi078 bash[33821]: audit 2023-10-01T10:13:17.147919+0000 mon.a (mon.0) 548 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T10:13:17.379 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:17 smithi078 bash[33821]: audit 2023-10-01T10:13:17.148217+0000 mon.a (mon.0) 549 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T10:13:17.379 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:17 smithi078 bash[33821]: audit 2023-10-01T10:13:17.148525+0000 mon.a (mon.0) 550 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T10:13:17.379 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:17 smithi078 bash[33821]: audit 2023-10-01T10:13:17.149698+0000 mon.a (mon.0) 551 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-01T10:13:17.379 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:17 smithi078 bash[33821]: audit 2023-10-01T10:13:17.149983+0000 mon.a (mon.0) 552 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-01T10:13:17.379 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:17 smithi078 bash[33821]: audit 2023-10-01T10:13:17.150263+0000 mon.a (mon.0) 553 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T10:13:17.379 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:17 smithi078 bash[33821]: audit 2023-10-01T10:13:17.150516+0000 mon.a (mon.0) 554 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T10:13:17.380 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:17 smithi078 bash[33821]: audit 2023-10-01T10:13:17.150760+0000 mon.a (mon.0) 555 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T10:13:17.380 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:17 smithi078 bash[33821]: audit 2023-10-01T10:13:17.151062+0000 mon.a (mon.0) 556 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T10:13:17.380 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:17 smithi078 bash[33821]: audit 2023-10-01T10:13:17.151315+0000 mon.a (mon.0) 557 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T10:13:17.380 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:17 smithi078 bash[33821]: audit 2023-10-01T10:13:17.151563+0000 mon.a (mon.0) 558 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T10:13:17.380 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:17 smithi078 bash[33821]: audit 2023-10-01T10:13:17.151808+0000 mon.a (mon.0) 559 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T10:13:17.380 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:17 smithi078 bash[33821]: audit 2023-10-01T10:13:17.152057+0000 mon.a (mon.0) 560 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T10:13:17.380 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:17 smithi078 bash[33821]: audit 2023-10-01T10:13:17.152313+0000 mon.a (mon.0) 561 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-01T10:13:17.380 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:17 smithi078 bash[33821]: audit 2023-10-01T10:13:17.152496+0000 mon.a (mon.0) 562 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-01T10:13:17.380 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:17 smithi078 bash[33821]: audit 2023-10-01T10:13:17.153089+0000 mon.a (mon.0) 563 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-01T10:13:17.380 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:17 smithi078 bash[33821]: cluster 2023-10-01T10:13:17.178162+0000 mon.a (mon.0) 564 : cluster [INF] Manager daemon y is now available 2023-10-01T10:13:17.380 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:13:17 smithi078 bash[27601]: debug 2023-10-01T10:13:17.111+0000 7f3e145e9680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-01T10:13:17.462 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:13:17 smithi173 bash[28473]: debug 2023-10-01T10:13:17.215+0000 7fd8974d2680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-01T10:13:17.463 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:13:17 smithi173 bash[28473]: [01/Oct/2023:10:13:17] ENGINE Bus STARTING 2023-10-01T10:13:17.463 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:13:17 smithi173 bash[28473]: CherryPy Checker: 2023-10-01T10:13:17.463 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:13:17 smithi173 bash[28473]: The Application mounted at '' has an empty config. 2023-10-01T10:13:17.463 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:13:17 smithi173 bash[28473]: [01/Oct/2023:10:13:17] ENGINE Serving on http://:::9283 2023-10-01T10:13:17.463 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:13:17 smithi173 bash[28473]: [01/Oct/2023:10:13:17] ENGINE Bus STARTED 2023-10-01T10:13:17.463 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:17 smithi173 bash[27612]: cluster 2023-10-01T10:13:17.135523+0000 mon.a (mon.0) 544 : cluster [INF] Active manager daemon y restarted 2023-10-01T10:13:17.463 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:17 smithi173 bash[27612]: cluster 2023-10-01T10:13:17.136054+0000 mon.a (mon.0) 545 : cluster [INF] Activating manager daemon y 2023-10-01T10:13:17.464 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:17 smithi173 bash[27612]: cluster 2023-10-01T10:13:17.144891+0000 mon.a (mon.0) 546 : cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2023-10-01T10:13:17.464 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:17 smithi173 bash[27612]: cluster 2023-10-01T10:13:17.145460+0000 mon.a (mon.0) 547 : cluster [DBG] mgrmap e16: y(active, starting, since 0.00958142s), standbys: x 2023-10-01T10:13:17.464 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:17 smithi173 bash[27612]: audit 2023-10-01T10:13:17.147919+0000 mon.a (mon.0) 548 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T10:13:17.464 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:17 smithi173 bash[27612]: audit 2023-10-01T10:13:17.148217+0000 mon.a (mon.0) 549 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T10:13:17.464 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:17 smithi173 bash[27612]: audit 2023-10-01T10:13:17.148525+0000 mon.a (mon.0) 550 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T10:13:17.464 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:17 smithi173 bash[27612]: audit 2023-10-01T10:13:17.149698+0000 mon.a (mon.0) 551 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-01T10:13:17.464 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:17 smithi173 bash[27612]: audit 2023-10-01T10:13:17.149983+0000 mon.a (mon.0) 552 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-01T10:13:17.464 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:17 smithi173 bash[27612]: audit 2023-10-01T10:13:17.150263+0000 mon.a (mon.0) 553 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T10:13:17.464 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:17 smithi173 bash[27612]: audit 2023-10-01T10:13:17.150516+0000 mon.a (mon.0) 554 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T10:13:17.464 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:17 smithi173 bash[27612]: audit 2023-10-01T10:13:17.150760+0000 mon.a (mon.0) 555 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T10:13:17.464 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:17 smithi173 bash[27612]: audit 2023-10-01T10:13:17.151062+0000 mon.a (mon.0) 556 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T10:13:17.464 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:17 smithi173 bash[27612]: audit 2023-10-01T10:13:17.151315+0000 mon.a (mon.0) 557 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T10:13:17.465 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:17 smithi173 bash[27612]: audit 2023-10-01T10:13:17.151563+0000 mon.a (mon.0) 558 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T10:13:17.465 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:17 smithi173 bash[27612]: audit 2023-10-01T10:13:17.151808+0000 mon.a (mon.0) 559 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T10:13:17.465 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:17 smithi173 bash[27612]: audit 2023-10-01T10:13:17.152057+0000 mon.a (mon.0) 560 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T10:13:17.465 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:17 smithi173 bash[27612]: audit 2023-10-01T10:13:17.152313+0000 mon.a (mon.0) 561 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-01T10:13:17.465 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:17 smithi173 bash[27612]: audit 2023-10-01T10:13:17.152496+0000 mon.a (mon.0) 562 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-01T10:13:17.465 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:17 smithi173 bash[27612]: audit 2023-10-01T10:13:17.153089+0000 mon.a (mon.0) 563 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-01T10:13:17.465 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:17 smithi173 bash[27612]: cluster 2023-10-01T10:13:17.178162+0000 mon.a (mon.0) 564 : cluster [INF] Manager daemon y is now available 2023-10-01T10:13:17.833 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:13:17 smithi078 bash[27601]: [01/Oct/2023:10:13:17] ENGINE Bus STARTING 2023-10-01T10:13:17.833 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:13:17 smithi078 bash[27601]: Warning: Permanently added '172.21.15.78' (ECDSA) to the list of known hosts. 2023-10-01T10:13:17.833 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:13:17 smithi078 bash[27601]: Warning: Permanently added '172.21.15.173' (ECDSA) to the list of known hosts. 2023-10-01T10:13:17.833 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:13:17 smithi078 bash[27601]: CherryPy Checker: 2023-10-01T10:13:17.833 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:13:17 smithi078 bash[27601]: The Application mounted at '' has an empty config. 2023-10-01T10:13:17.833 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:13:17 smithi078 bash[27601]: [01/Oct/2023:10:13:17] ENGINE Serving on http://:::9283 2023-10-01T10:13:17.834 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:13:17 smithi078 bash[27601]: [01/Oct/2023:10:13:17] ENGINE Bus STARTED 2023-10-01T10:13:18.170 INFO:teuthology.orchestra.run.smithi173.stdout:Scheduled node-exporter update... 2023-10-01T10:13:18.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:18 smithi078 bash[27336]: cluster 2023-10-01T10:13:17.243265+0000 mon.a (mon.0) 565 : cluster [DBG] Standby manager daemon x restarted 2023-10-01T10:13:18.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:18 smithi078 bash[27336]: cluster 2023-10-01T10:13:17.243441+0000 mon.a (mon.0) 566 : cluster [DBG] Standby manager daemon x started 2023-10-01T10:13:18.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:18 smithi078 bash[27336]: audit 2023-10-01T10:13:17.248927+0000 mon.b (mon.1) 23 : audit [DBG] from='mgr.? 172.21.15.173:0/36631260' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-01T10:13:18.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:18 smithi078 bash[27336]: audit 2023-10-01T10:13:17.250724+0000 mon.b (mon.1) 24 : audit [DBG] from='mgr.? 172.21.15.173:0/36631260' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-01T10:13:18.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:18 smithi078 bash[27336]: audit 2023-10-01T10:13:17.256411+0000 mon.b (mon.1) 25 : audit [DBG] from='mgr.? 172.21.15.173:0/36631260' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-01T10:13:18.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:18 smithi078 bash[27336]: audit 2023-10-01T10:13:17.257085+0000 mon.b (mon.1) 26 : audit [DBG] from='mgr.? 172.21.15.173:0/36631260' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-01T10:13:18.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:18 smithi078 bash[27336]: audit 2023-10-01T10:13:17.316315+0000 mon.a (mon.0) 567 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:13:18.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:18 smithi078 bash[27336]: audit 2023-10-01T10:13:17.322407+0000 mon.a (mon.0) 568 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:13:18.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:18 smithi078 bash[27336]: audit 2023-10-01T10:13:17.351606+0000 mon.a (mon.0) 569 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:13:18.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:18 smithi078 bash[27336]: audit 2023-10-01T10:13:17.389443+0000 mon.a (mon.0) 570 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-01T10:13:18.585 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:18 smithi078 bash[27336]: audit 2023-10-01T10:13:17.395952+0000 mon.a (mon.0) 571 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-01T10:13:18.585 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:18 smithi078 bash[27336]: cluster 2023-10-01T10:13:18.155358+0000 mon.a (mon.0) 572 : cluster [DBG] mgrmap e17: y(active, since 1.01948s), standbys: x 2023-10-01T10:13:18.585 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:18 smithi078 bash[27336]: audit 2023-10-01T10:13:18.167543+0000 mon.a (mon.0) 573 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:13:18.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:18 smithi078 bash[33821]: cluster 2023-10-01T10:13:17.243265+0000 mon.a (mon.0) 565 : cluster [DBG] Standby manager daemon x restarted 2023-10-01T10:13:18.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:18 smithi078 bash[33821]: cluster 2023-10-01T10:13:17.243441+0000 mon.a (mon.0) 566 : cluster [DBG] Standby manager daemon x started 2023-10-01T10:13:18.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:18 smithi078 bash[33821]: audit 2023-10-01T10:13:17.248927+0000 mon.b (mon.1) 23 : audit [DBG] from='mgr.? 172.21.15.173:0/36631260' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-01T10:13:18.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:18 smithi078 bash[33821]: audit 2023-10-01T10:13:17.250724+0000 mon.b (mon.1) 24 : audit [DBG] from='mgr.? 172.21.15.173:0/36631260' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-01T10:13:18.586 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:18 smithi078 bash[33821]: audit 2023-10-01T10:13:17.256411+0000 mon.b (mon.1) 25 : audit [DBG] from='mgr.? 172.21.15.173:0/36631260' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-01T10:13:18.586 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:18 smithi078 bash[33821]: audit 2023-10-01T10:13:17.257085+0000 mon.b (mon.1) 26 : audit [DBG] from='mgr.? 172.21.15.173:0/36631260' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-01T10:13:18.586 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:18 smithi078 bash[33821]: audit 2023-10-01T10:13:17.316315+0000 mon.a (mon.0) 567 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:13:18.586 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:18 smithi078 bash[33821]: audit 2023-10-01T10:13:17.322407+0000 mon.a (mon.0) 568 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:13:18.586 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:18 smithi078 bash[33821]: audit 2023-10-01T10:13:17.351606+0000 mon.a (mon.0) 569 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:13:18.586 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:18 smithi078 bash[33821]: audit 2023-10-01T10:13:17.389443+0000 mon.a (mon.0) 570 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-01T10:13:18.586 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:18 smithi078 bash[33821]: audit 2023-10-01T10:13:17.395952+0000 mon.a (mon.0) 571 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-01T10:13:18.586 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:18 smithi078 bash[33821]: cluster 2023-10-01T10:13:18.155358+0000 mon.a (mon.0) 572 : cluster [DBG] mgrmap e17: y(active, since 1.01948s), standbys: x 2023-10-01T10:13:18.586 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:18 smithi078 bash[33821]: audit 2023-10-01T10:13:18.167543+0000 mon.a (mon.0) 573 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:13:18.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:18 smithi173 bash[27612]: cluster 2023-10-01T10:13:17.243265+0000 mon.a (mon.0) 565 : cluster [DBG] Standby manager daemon x restarted 2023-10-01T10:13:18.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:18 smithi173 bash[27612]: cluster 2023-10-01T10:13:17.243441+0000 mon.a (mon.0) 566 : cluster [DBG] Standby manager daemon x started 2023-10-01T10:13:18.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:18 smithi173 bash[27612]: audit 2023-10-01T10:13:17.248927+0000 mon.b (mon.1) 23 : audit [DBG] from='mgr.? 172.21.15.173:0/36631260' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-01T10:13:18.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:18 smithi173 bash[27612]: audit 2023-10-01T10:13:17.250724+0000 mon.b (mon.1) 24 : audit [DBG] from='mgr.? 172.21.15.173:0/36631260' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-01T10:13:18.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:18 smithi173 bash[27612]: audit 2023-10-01T10:13:17.256411+0000 mon.b (mon.1) 25 : audit [DBG] from='mgr.? 172.21.15.173:0/36631260' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-01T10:13:18.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:18 smithi173 bash[27612]: audit 2023-10-01T10:13:17.257085+0000 mon.b (mon.1) 26 : audit [DBG] from='mgr.? 172.21.15.173:0/36631260' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-01T10:13:18.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:18 smithi173 bash[27612]: audit 2023-10-01T10:13:17.316315+0000 mon.a (mon.0) 567 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:13:18.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:18 smithi173 bash[27612]: audit 2023-10-01T10:13:17.322407+0000 mon.a (mon.0) 568 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:13:18.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:18 smithi173 bash[27612]: audit 2023-10-01T10:13:17.351606+0000 mon.a (mon.0) 569 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:13:18.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:18 smithi173 bash[27612]: audit 2023-10-01T10:13:17.389443+0000 mon.a (mon.0) 570 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-01T10:13:18.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:18 smithi173 bash[27612]: audit 2023-10-01T10:13:17.395952+0000 mon.a (mon.0) 571 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-01T10:13:18.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:18 smithi173 bash[27612]: cluster 2023-10-01T10:13:18.155358+0000 mon.a (mon.0) 572 : cluster [DBG] mgrmap e17: y(active, since 1.01948s), standbys: x 2023-10-01T10:13:18.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:18 smithi173 bash[27612]: audit 2023-10-01T10:13:18.167543+0000 mon.a (mon.0) 573 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:13:18.965 DEBUG:teuthology.orchestra.run.smithi078:node-exporter.a> sudo journalctl -f -n 0 -u ceph-d8016480-6041-11ee-8db5-212e2dc638e7@node-exporter.a.service 2023-10-01T10:13:18.968 DEBUG:teuthology.orchestra.run.smithi173:node-exporter.b> sudo journalctl -f -n 0 -u ceph-d8016480-6041-11ee-8db5-212e2dc638e7@node-exporter.b.service 2023-10-01T10:13:18.971 INFO:tasks.cephadm:Adding alertmanager.a on smithi078 2023-10-01T10:13:18.971 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph orch apply alertmanager '1;smithi078=a' 2023-10-01T10:13:19.021 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-01T10:13:19.148 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-01T10:13:19.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:19 smithi078 bash[27336]: cephadm 2023-10-01T10:13:18.163779+0000 mgr.y (mgr.14427) 3 : cephadm [INF] Saving service node-exporter spec with placement smithi078=a;smithi173=b;count:2 2023-10-01T10:13:19.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:19 smithi078 bash[33821]: cephadm 2023-10-01T10:13:18.163779+0000 mgr.y (mgr.14427) 3 : cephadm [INF] Saving service node-exporter spec with placement smithi078=a;smithi173=b;count:2 2023-10-01T10:13:19.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:19 smithi173 bash[27612]: cephadm 2023-10-01T10:13:18.163779+0000 mgr.y (mgr.14427) 3 : cephadm [INF] Saving service node-exporter spec with placement smithi078=a;smithi173=b;count:2 2023-10-01T10:13:20.316 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:20 smithi173 bash[27612]: cluster 2023-10-01T10:13:19.151299+0000 mgr.y (mgr.14427) 4 : cluster [DBG] pgmap v4: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:20.316 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:20 smithi173 bash[27612]: cluster 2023-10-01T10:13:20.163222+0000 mon.a (mon.0) 574 : cluster [DBG] mgrmap e18: y(active, since 3s), standbys: x 2023-10-01T10:13:20.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:20 smithi078 bash[27336]: cluster 2023-10-01T10:13:19.151299+0000 mgr.y (mgr.14427) 4 : cluster [DBG] pgmap v4: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:20.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:20 smithi078 bash[27336]: cluster 2023-10-01T10:13:20.163222+0000 mon.a (mon.0) 574 : cluster [DBG] mgrmap e18: y(active, since 3s), standbys: x 2023-10-01T10:13:20.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:20 smithi078 bash[33821]: cluster 2023-10-01T10:13:19.151299+0000 mgr.y (mgr.14427) 4 : cluster [DBG] pgmap v4: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:20.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:20 smithi078 bash[33821]: cluster 2023-10-01T10:13:20.163222+0000 mon.a (mon.0) 574 : cluster [DBG] mgrmap e18: y(active, since 3s), standbys: x 2023-10-01T10:13:21.864 INFO:teuthology.orchestra.run.smithi173.stdout:Scheduled alertmanager update... 2023-10-01T10:13:22.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:21 smithi078 bash[27336]: cluster 2023-10-01T10:13:21.151956+0000 mgr.y (mgr.14427) 5 : cluster [DBG] pgmap v5: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:22.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:21 smithi078 bash[27336]: audit 2023-10-01T10:13:21.861125+0000 mon.a (mon.0) 575 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:13:22.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:21 smithi078 bash[33821]: cluster 2023-10-01T10:13:21.151956+0000 mgr.y (mgr.14427) 5 : cluster [DBG] pgmap v5: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:22.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:21 smithi078 bash[33821]: audit 2023-10-01T10:13:21.861125+0000 mon.a (mon.0) 575 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:13:22.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:21 smithi173 bash[27612]: cluster 2023-10-01T10:13:21.151956+0000 mgr.y (mgr.14427) 5 : cluster [DBG] pgmap v5: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:22.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:21 smithi173 bash[27612]: audit 2023-10-01T10:13:21.861125+0000 mon.a (mon.0) 575 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:13:22.577 DEBUG:teuthology.orchestra.run.smithi078:alertmanager.a> sudo journalctl -f -n 0 -u ceph-d8016480-6041-11ee-8db5-212e2dc638e7@alertmanager.a.service 2023-10-01T10:13:22.591 INFO:tasks.cephadm:Adding grafana.a on smithi173 2023-10-01T10:13:22.592 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph orch apply grafana '1;smithi173=a' 2023-10-01T10:13:22.763 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-01T10:13:23.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:22 smithi173 bash[27612]: audit 2023-10-01T10:13:21.850987+0000 mgr.y (mgr.14427) 6 : audit [DBG] from='client.24373 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi078=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:13:23.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:22 smithi173 bash[27612]: cephadm 2023-10-01T10:13:21.854332+0000 mgr.y (mgr.14427) 7 : cephadm [INF] Saving service alertmanager spec with placement smithi078=a;count:1 2023-10-01T10:13:23.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:22 smithi078 bash[27336]: audit 2023-10-01T10:13:21.850987+0000 mgr.y (mgr.14427) 6 : audit [DBG] from='client.24373 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi078=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:13:23.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:22 smithi078 bash[27336]: cephadm 2023-10-01T10:13:21.854332+0000 mgr.y (mgr.14427) 7 : cephadm [INF] Saving service alertmanager spec with placement smithi078=a;count:1 2023-10-01T10:13:23.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:22 smithi078 bash[33821]: audit 2023-10-01T10:13:21.850987+0000 mgr.y (mgr.14427) 6 : audit [DBG] from='client.24373 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi078=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:13:23.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:22 smithi078 bash[33821]: cephadm 2023-10-01T10:13:21.854332+0000 mgr.y (mgr.14427) 7 : cephadm [INF] Saving service alertmanager spec with placement smithi078=a;count:1 2023-10-01T10:13:24.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:23 smithi078 bash[27336]: audit 2023-10-01T10:13:22.971628+0000 mon.a (mon.0) 576 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:13:24.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:23 smithi078 bash[27336]: cluster 2023-10-01T10:13:23.152680+0000 mgr.y (mgr.14427) 8 : cluster [DBG] pgmap v6: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:24.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:23 smithi078 bash[27336]: audit 2023-10-01T10:13:23.497023+0000 mon.a (mon.0) 577 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:13:24.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:23 smithi078 bash[27336]: audit 2023-10-01T10:13:23.768156+0000 mon.a (mon.0) 578 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:13:24.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:23 smithi078 bash[27336]: audit 2023-10-01T10:13:23.769855+0000 mon.a (mon.0) 579 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:13:24.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:23 smithi078 bash[27336]: audit 2023-10-01T10:13:23.833398+0000 mon.a (mon.0) 580 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:13:24.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:23 smithi078 bash[27336]: audit 2023-10-01T10:13:23.853898+0000 mon.a (mon.0) 581 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:13:24.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:23 smithi078 bash[27336]: audit 2023-10-01T10:13:23.859653+0000 mon.a (mon.0) 582 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:13:24.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:23 smithi078 bash[33821]: audit 2023-10-01T10:13:22.971628+0000 mon.a (mon.0) 576 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:13:24.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:23 smithi078 bash[33821]: cluster 2023-10-01T10:13:23.152680+0000 mgr.y (mgr.14427) 8 : cluster [DBG] pgmap v6: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:24.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:23 smithi078 bash[33821]: audit 2023-10-01T10:13:23.497023+0000 mon.a (mon.0) 577 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:13:24.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:23 smithi078 bash[33821]: audit 2023-10-01T10:13:23.768156+0000 mon.a (mon.0) 578 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:13:24.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:23 smithi078 bash[33821]: audit 2023-10-01T10:13:23.769855+0000 mon.a (mon.0) 579 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:13:24.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:23 smithi078 bash[33821]: audit 2023-10-01T10:13:23.833398+0000 mon.a (mon.0) 580 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:13:24.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:23 smithi078 bash[33821]: audit 2023-10-01T10:13:23.853898+0000 mon.a (mon.0) 581 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:13:24.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:23 smithi078 bash[33821]: audit 2023-10-01T10:13:23.859653+0000 mon.a (mon.0) 582 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:13:24.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:23 smithi173 bash[27612]: audit 2023-10-01T10:13:22.971628+0000 mon.a (mon.0) 576 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:13:24.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:23 smithi173 bash[27612]: cluster 2023-10-01T10:13:23.152680+0000 mgr.y (mgr.14427) 8 : cluster [DBG] pgmap v6: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:24.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:23 smithi173 bash[27612]: audit 2023-10-01T10:13:23.497023+0000 mon.a (mon.0) 577 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:13:24.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:23 smithi173 bash[27612]: audit 2023-10-01T10:13:23.768156+0000 mon.a (mon.0) 578 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:13:24.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:23 smithi173 bash[27612]: audit 2023-10-01T10:13:23.769855+0000 mon.a (mon.0) 579 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:13:24.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:23 smithi173 bash[27612]: audit 2023-10-01T10:13:23.833398+0000 mon.a (mon.0) 580 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:13:24.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:23 smithi173 bash[27612]: audit 2023-10-01T10:13:23.853898+0000 mon.a (mon.0) 581 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:13:24.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:23 smithi173 bash[27612]: audit 2023-10-01T10:13:23.859653+0000 mon.a (mon.0) 582 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:13:25.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:24 smithi078 bash[33821]: cephadm 2023-10-01T10:13:23.771995+0000 mgr.y (mgr.14427) 9 : cephadm [INF] Updating smithi078:/etc/ceph/ceph.conf 2023-10-01T10:13:25.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:24 smithi078 bash[33821]: cephadm 2023-10-01T10:13:23.772698+0000 mgr.y (mgr.14427) 10 : cephadm [INF] Updating smithi173:/etc/ceph/ceph.conf 2023-10-01T10:13:25.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:24 smithi078 bash[33821]: cephadm 2023-10-01T10:13:23.799167+0000 mgr.y (mgr.14427) 11 : cephadm [INF] Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2023-10-01T10:13:25.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:24 smithi078 bash[33821]: cephadm 2023-10-01T10:13:23.811949+0000 mgr.y (mgr.14427) 12 : cephadm [INF] Updating smithi173:/etc/ceph/ceph.client.admin.keyring 2023-10-01T10:13:25.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:24 smithi078 bash[33821]: cephadm 2023-10-01T10:13:23.883809+0000 mgr.y (mgr.14427) 13 : cephadm [INF] Deploying daemon prometheus.a on smithi173 2023-10-01T10:13:25.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:24 smithi078 bash[27336]: cephadm 2023-10-01T10:13:23.771995+0000 mgr.y (mgr.14427) 9 : cephadm [INF] Updating smithi078:/etc/ceph/ceph.conf 2023-10-01T10:13:25.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:24 smithi078 bash[27336]: cephadm 2023-10-01T10:13:23.772698+0000 mgr.y (mgr.14427) 10 : cephadm [INF] Updating smithi173:/etc/ceph/ceph.conf 2023-10-01T10:13:25.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:24 smithi078 bash[27336]: cephadm 2023-10-01T10:13:23.799167+0000 mgr.y (mgr.14427) 11 : cephadm [INF] Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2023-10-01T10:13:25.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:24 smithi078 bash[27336]: cephadm 2023-10-01T10:13:23.811949+0000 mgr.y (mgr.14427) 12 : cephadm [INF] Updating smithi173:/etc/ceph/ceph.client.admin.keyring 2023-10-01T10:13:25.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:24 smithi078 bash[27336]: cephadm 2023-10-01T10:13:23.883809+0000 mgr.y (mgr.14427) 13 : cephadm [INF] Deploying daemon prometheus.a on smithi173 2023-10-01T10:13:25.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:24 smithi173 bash[27612]: cephadm 2023-10-01T10:13:23.771995+0000 mgr.y (mgr.14427) 9 : cephadm [INF] Updating smithi078:/etc/ceph/ceph.conf 2023-10-01T10:13:25.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:24 smithi173 bash[27612]: cephadm 2023-10-01T10:13:23.772698+0000 mgr.y (mgr.14427) 10 : cephadm [INF] Updating smithi173:/etc/ceph/ceph.conf 2023-10-01T10:13:25.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:24 smithi173 bash[27612]: cephadm 2023-10-01T10:13:23.799167+0000 mgr.y (mgr.14427) 11 : cephadm [INF] Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2023-10-01T10:13:25.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:24 smithi173 bash[27612]: cephadm 2023-10-01T10:13:23.811949+0000 mgr.y (mgr.14427) 12 : cephadm [INF] Updating smithi173:/etc/ceph/ceph.client.admin.keyring 2023-10-01T10:13:25.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:24 smithi173 bash[27612]: cephadm 2023-10-01T10:13:23.883809+0000 mgr.y (mgr.14427) 13 : cephadm [INF] Deploying daemon prometheus.a on smithi173 2023-10-01T10:13:25.912 INFO:teuthology.orchestra.run.smithi173.stdout:Scheduled grafana update... 2023-10-01T10:13:26.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:25 smithi078 bash[27336]: cluster 2023-10-01T10:13:25.153360+0000 mgr.y (mgr.14427) 14 : cluster [DBG] pgmap v7: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:26.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:25 smithi078 bash[27336]: audit 2023-10-01T10:13:25.908891+0000 mon.a (mon.0) 583 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:13:26.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:25 smithi078 bash[33821]: cluster 2023-10-01T10:13:25.153360+0000 mgr.y (mgr.14427) 14 : cluster [DBG] pgmap v7: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:26.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:25 smithi078 bash[33821]: audit 2023-10-01T10:13:25.908891+0000 mon.a (mon.0) 583 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:13:26.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:25 smithi173 bash[27612]: cluster 2023-10-01T10:13:25.153360+0000 mgr.y (mgr.14427) 14 : cluster [DBG] pgmap v7: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:26.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:25 smithi173 bash[27612]: audit 2023-10-01T10:13:25.908891+0000 mon.a (mon.0) 583 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:13:27.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:26 smithi078 bash[33821]: audit 2023-10-01T10:13:25.896498+0000 mgr.y (mgr.14427) 15 : audit [DBG] from='client.24379 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi173=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:13:27.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:26 smithi078 bash[33821]: cephadm 2023-10-01T10:13:25.900021+0000 mgr.y (mgr.14427) 16 : cephadm [INF] Saving service grafana spec with placement smithi173=a;count:1 2023-10-01T10:13:27.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:26 smithi078 bash[27336]: audit 2023-10-01T10:13:25.896498+0000 mgr.y (mgr.14427) 15 : audit [DBG] from='client.24379 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi173=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:13:27.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:26 smithi078 bash[27336]: cephadm 2023-10-01T10:13:25.900021+0000 mgr.y (mgr.14427) 16 : cephadm [INF] Saving service grafana spec with placement smithi173=a;count:1 2023-10-01T10:13:27.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:26 smithi173 bash[27612]: audit 2023-10-01T10:13:25.896498+0000 mgr.y (mgr.14427) 15 : audit [DBG] from='client.24379 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi173=a", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:13:27.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:26 smithi173 bash[27612]: cephadm 2023-10-01T10:13:25.900021+0000 mgr.y (mgr.14427) 16 : cephadm [INF] Saving service grafana spec with placement smithi173=a;count:1 2023-10-01T10:13:27.427 DEBUG:teuthology.orchestra.run.smithi173:grafana.a> sudo journalctl -f -n 0 -u ceph-d8016480-6041-11ee-8db5-212e2dc638e7@grafana.a.service 2023-10-01T10:13:27.429 INFO:tasks.cephadm:Setting up client nodes... 2023-10-01T10:13:27.430 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-10-01T10:13:27.617 INFO:journalctl@ceph.grafana.a.smithi173.stdout:-- Logs begin at Sun 2022-08-14 12:20:24 UTC. -- 2023-10-01T10:13:28.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:27 smithi078 bash[27336]: cluster 2023-10-01T10:13:27.153856+0000 mgr.y (mgr.14427) 17 : cluster [DBG] pgmap v8: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:28.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:27 smithi078 bash[33821]: cluster 2023-10-01T10:13:27.153856+0000 mgr.y (mgr.14427) 17 : cluster [DBG] pgmap v8: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:28.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:27 smithi173 bash[27612]: cluster 2023-10-01T10:13:27.153856+0000 mgr.y (mgr.14427) 17 : cluster [DBG] pgmap v8: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:30.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:30 smithi078 bash[33821]: cluster 2023-10-01T10:13:29.154476+0000 mgr.y (mgr.14427) 18 : cluster [DBG] pgmap v9: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:30.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:30 smithi078 bash[27336]: cluster 2023-10-01T10:13:29.154476+0000 mgr.y (mgr.14427) 18 : cluster [DBG] pgmap v9: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:30.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:30 smithi173 bash[27612]: cluster 2023-10-01T10:13:29.154476+0000 mgr.y (mgr.14427) 18 : cluster [DBG] pgmap v9: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:30.817 INFO:teuthology.orchestra.run.smithi078.stdout:[client.0] 2023-10-01T10:13:30.817 INFO:teuthology.orchestra.run.smithi078.stdout: key = AQBKRhllP+YuMBAA5DzDzOSpTEHMejdIdR8x2w== 2023-10-01T10:13:31.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:31 smithi078 bash[27336]: audit 2023-10-01T10:13:30.807298+0000 mon.c (mon.2) 17 : audit [INF] from='client.? 172.21.15.78:0/1691487228' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-01T10:13:31.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:31 smithi078 bash[27336]: audit 2023-10-01T10:13:30.808027+0000 mon.a (mon.0) 584 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-01T10:13:31.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:31 smithi078 bash[27336]: audit 2023-10-01T10:13:30.814368+0000 mon.a (mon.0) 585 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-01T10:13:31.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:31 smithi078 bash[33821]: audit 2023-10-01T10:13:30.807298+0000 mon.c (mon.2) 17 : audit [INF] from='client.? 172.21.15.78:0/1691487228' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-01T10:13:31.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:31 smithi078 bash[33821]: audit 2023-10-01T10:13:30.808027+0000 mon.a (mon.0) 584 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-01T10:13:31.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:31 smithi078 bash[33821]: audit 2023-10-01T10:13:30.814368+0000 mon.a (mon.0) 585 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-01T10:13:31.502 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2023-10-01T10:13:31.502 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2023-10-01T10:13:31.502 DEBUG:teuthology.orchestra.run.smithi078:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2023-10-01T10:13:31.527 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-10-01T10:13:31.539 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:31 smithi173 bash[27612]: audit 2023-10-01T10:13:30.807298+0000 mon.c (mon.2) 17 : audit [INF] from='client.? 172.21.15.78:0/1691487228' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-01T10:13:31.539 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:31 smithi173 bash[27612]: audit 2023-10-01T10:13:30.808027+0000 mon.a (mon.0) 584 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-01T10:13:31.539 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:31 smithi173 bash[27612]: audit 2023-10-01T10:13:30.814368+0000 mon.a (mon.0) 585 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-01T10:13:32.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:32 smithi078 bash[33821]: cluster 2023-10-01T10:13:31.155068+0000 mgr.y (mgr.14427) 19 : cluster [DBG] pgmap v10: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:32.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:32 smithi078 bash[27336]: cluster 2023-10-01T10:13:31.155068+0000 mgr.y (mgr.14427) 19 : cluster [DBG] pgmap v10: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:32.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:32 smithi173 bash[27612]: cluster 2023-10-01T10:13:31.155068+0000 mgr.y (mgr.14427) 19 : cluster [DBG] pgmap v10: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:34.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:34 smithi078 bash[33821]: cluster 2023-10-01T10:13:33.155635+0000 mgr.y (mgr.14427) 20 : cluster [DBG] pgmap v11: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:34.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:34 smithi078 bash[27336]: cluster 2023-10-01T10:13:33.155635+0000 mgr.y (mgr.14427) 20 : cluster [DBG] pgmap v11: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:34.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:34 smithi173 bash[27612]: cluster 2023-10-01T10:13:33.155635+0000 mgr.y (mgr.14427) 20 : cluster [DBG] pgmap v11: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:36.556 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:36 smithi173 bash[27612]: cluster 2023-10-01T10:13:35.156298+0000 mgr.y (mgr.14427) 21 : cluster [DBG] pgmap v12: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:36.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:36 smithi078 bash[27336]: cluster 2023-10-01T10:13:35.156298+0000 mgr.y (mgr.14427) 21 : cluster [DBG] pgmap v12: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:36.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:36 smithi078 bash[33821]: cluster 2023-10-01T10:13:35.156298+0000 mgr.y (mgr.14427) 21 : cluster [DBG] pgmap v12: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:36.843 INFO:teuthology.orchestra.run.smithi173.stdout:[client.1] 2023-10-01T10:13:36.844 INFO:teuthology.orchestra.run.smithi173.stdout: key = AQBQRhlllM2WMRAAweQwAwi4m/pZME+VLoBEoA== 2023-10-01T10:13:37.484 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:37 smithi173 bash[27612]: audit 2023-10-01T10:13:36.831626+0000 mon.a (mon.0) 586 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-01T10:13:37.484 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:37 smithi173 bash[27612]: audit 2023-10-01T10:13:36.832033+0000 mon.b (mon.1) 27 : audit [INF] from='client.? 172.21.15.173:0/1313977991' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-01T10:13:37.484 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:37 smithi173 bash[27612]: audit 2023-10-01T10:13:36.840550+0000 mon.a (mon.0) 587 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-01T10:13:37.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:37 smithi078 bash[27336]: audit 2023-10-01T10:13:36.831626+0000 mon.a (mon.0) 586 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-01T10:13:37.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:37 smithi078 bash[27336]: audit 2023-10-01T10:13:36.832033+0000 mon.b (mon.1) 27 : audit [INF] from='client.? 172.21.15.173:0/1313977991' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-01T10:13:37.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:37 smithi078 bash[27336]: audit 2023-10-01T10:13:36.840550+0000 mon.a (mon.0) 587 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-01T10:13:37.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:37 smithi078 bash[33821]: audit 2023-10-01T10:13:36.831626+0000 mon.a (mon.0) 586 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-01T10:13:37.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:37 smithi078 bash[33821]: audit 2023-10-01T10:13:36.832033+0000 mon.b (mon.1) 27 : audit [INF] from='client.? 172.21.15.173:0/1313977991' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-10-01T10:13:37.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:37 smithi078 bash[33821]: audit 2023-10-01T10:13:36.840550+0000 mon.a (mon.0) 587 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-10-01T10:13:38.062 DEBUG:teuthology.orchestra.run.smithi173:> set -ex 2023-10-01T10:13:38.063 DEBUG:teuthology.orchestra.run.smithi173:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2023-10-01T10:13:38.063 DEBUG:teuthology.orchestra.run.smithi173:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2023-10-01T10:13:38.129 INFO:tasks.cephadm:Waiting for OSDs to come up 2023-10-01T10:13:38.129 DEBUG:teuthology.orchestra.run.smithi078:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2023-10-01T10:13:38.426 INFO:teuthology.misc.health.smithi078.stdout: 2023-10-01T10:13:38.426 INFO:teuthology.misc.health.smithi078.stdout:{"epoch":51,"fsid":"d8016480-6041-11ee-8db5-212e2dc638e7","created":"2023-10-01T10:04:59.345633+0000","modified":"2023-10-01T10:13:17.135626+0000","last_up_change":"2023-10-01T10:12:45.118874+0000","last_in_change":"2023-10-01T10:12:19.634793+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-10-01T10:09:16.689172+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}}],"osds":[{"osd":0,"uuid":"3cdf50f0-cae3-4e38-a7ca-c24c4a07b615","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6802","nonce":575557942},{"type":"v1","addr":"172.21.15.78:6803","nonce":575557942}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6804","nonce":575557942},{"type":"v1","addr":"172.21.15.78:6805","nonce":575557942}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6808","nonce":575557942},{"type":"v1","addr":"172.21.15.78:6809","nonce":575557942}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6806","nonce":575557942},{"type":"v1","addr":"172.21.15.78:6807","nonce":575557942}]},"public_addr":"172.21.15.78:6803/575557942","cluster_addr":"172.21.15.78:6805/575557942","heartbeat_back_addr":"172.21.15.78:6809/575557942","heartbeat_front_addr":"172.21.15.78:6807/575557942","state":["exists","up"]},{"osd":1,"uuid":"4e6bea65-4109-43d2-b5be-3811fdd2a803","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6810","nonce":3462774695},{"type":"v1","addr":"172.21.15.78:6811","nonce":3462774695}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6812","nonce":3462774695},{"type":"v1","addr":"172.21.15.78:6813","nonce":3462774695}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6816","nonce":3462774695},{"type":"v1","addr":"172.21.15.78:6817","nonce":3462774695}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6814","nonce":3462774695},{"type":"v1","addr":"172.21.15.78:6815","nonce":3462774695}]},"public_addr":"172.21.15.78:6811/3462774695","cluster_addr":"172.21.15.78:6813/3462774695","heartbeat_back_addr":"172.21.15.78:6817/3462774695","heartbeat_front_addr":"172.21.15.78:6815/3462774695","state":["exists","up"]},{"osd":2,"uuid":"9f830594-31d6-4994-85fa-0615c280d5c3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6818","nonce":1190788353},{"type":"v1","addr":"172.21.15.78:6819","nonce":1190788353}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6820","nonce":1190788353},{"type":"v1","addr":"172.21.15.78:6821","nonce":1190788353}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6824","nonce":1190788353},{"type":"v1","addr":"172.21.15.78:6825","nonce":1190788353}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6822","nonce":1190788353},{"type":"v1","addr":"172.21.15.78:6823","nonce":1190788353}]},"public_addr":"172.21.15.78:6819/1190788353","cluster_addr":"172.21.15.78:6821/1190788353","heartbeat_back_addr":"172.21.15.78:6825/1190788353","heartbeat_front_addr":"172.21.15.78:6823/1190788353","state":["exists","up"]},{"osd":3,"uuid":"2780842b-2f4f-4190-b4a5-279a0444adf2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6826","nonce":53937366},{"type":"v1","addr":"172.21.15.78:6827","nonce":53937366}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6828","nonce":53937366},{"type":"v1","addr":"172.21.15.78:6829","nonce":53937366}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6832","nonce":53937366},{"type":"v1","addr":"172.21.15.78:6833","nonce":53937366}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6830","nonce":53937366},{"type":"v1","addr":"172.21.15.78:6831","nonce":53937366}]},"public_addr":"172.21.15.78:6827/53937366","cluster_addr":"172.21.15.78:6829/53937366","heartbeat_back_addr":"172.21.15.78:6833/53937366","heartbeat_front_addr":"172.21.15.78:6831/53937366","state":["exists","up"]},{"osd":4,"uuid":"57c30778-fd7e-4f3e-afd9-389ef377cb62","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6800","nonce":2430591286},{"type":"v1","addr":"172.21.15.173:6801","nonce":2430591286}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6802","nonce":2430591286},{"type":"v1","addr":"172.21.15.173:6803","nonce":2430591286}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6806","nonce":2430591286},{"type":"v1","addr":"172.21.15.173:6807","nonce":2430591286}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6804","nonce":2430591286},{"type":"v1","addr":"172.21.15.173:6805","nonce":2430591286}]},"public_addr":"172.21.15.173:6801/2430591286","cluster_addr":"172.21.15.173:6803/2430591286","heartbeat_back_addr":"172.21.15.173:6807/2430591286","heartbeat_front_addr":"172.21.15.173:6805/2430591286","state":["exists","up"]},{"osd":5,"uuid":"c62aa7c3-2d09-4088-ba36-c84588dc3549","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":37,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6808","nonce":2798643181},{"type":"v1","addr":"172.21.15.173:6809","nonce":2798643181}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6810","nonce":2798643181},{"type":"v1","addr":"172.21.15.173:6811","nonce":2798643181}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6814","nonce":2798643181},{"type":"v1","addr":"172.21.15.173:6815","nonce":2798643181}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6812","nonce":2798643181},{"type":"v1","addr":"172.21.15.173:6813","nonce":2798643181}]},"public_addr":"172.21.15.173:6809/2798643181","cluster_addr":"172.21.15.173:6811/2798643181","heartbeat_back_addr":"172.21.15.173:6815/2798643181","heartbeat_front_addr":"172.21.15.173:6813/2798643181","state":["exists","up"]},{"osd":6,"uuid":"b140330d-dd93-4ce4-9b9a-2ec6321419b3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6816","nonce":1568190351},{"type":"v1","addr":"172.21.15.173:6817","nonce":1568190351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6818","nonce":1568190351},{"type":"v1","addr":"172.21.15.173:6819","nonce":1568190351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6822","nonce":1568190351},{"type":"v1","addr":"172.21.15.173:6823","nonce":1568190351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6820","nonce":1568190351},{"type":"v1","addr":"172.21.15.173:6821","nonce":1568190351}]},"public_addr":"172.21.15.173:6817/1568190351","cluster_addr":"172.21.15.173:6819/1568190351","heartbeat_back_addr":"172.21.15.173:6823/1568190351","heartbeat_front_addr":"172.21.15.173:6821/1568190351","state":["exists","up"]},{"osd":7,"uuid":"7590027f-50c4-4457-843d-cebaa734d2ea","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6824","nonce":1958390745},{"type":"v1","addr":"172.21.15.173:6825","nonce":1958390745}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6826","nonce":1958390745},{"type":"v1","addr":"172.21.15.173:6827","nonce":1958390745}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6830","nonce":1958390745},{"type":"v1","addr":"172.21.15.173:6831","nonce":1958390745}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6828","nonce":1958390745},{"type":"v1","addr":"172.21.15.173:6829","nonce":1958390745}]},"public_addr":"172.21.15.173:6825/1958390745","cluster_addr":"172.21.15.173:6827/1958390745","heartbeat_back_addr":"172.21.15.173:6831/1958390745","heartbeat_front_addr":"172.21.15.173:6829/1958390745","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:07:47.950416+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:08:31.368397+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:09:14.866866+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:09:59.445542+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:10:39.550033+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:11:20.934732+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:12:01.654044+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:12:43.447412+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.78:0/2359078223":"2023-10-02T10:13:17.135564+0000","172.21.15.78:0/3939471097":"2023-10-02T10:05:57.727322+0000","172.21.15.78:6800/1224621968":"2023-10-02T10:13:17.135564+0000","172.21.15.78:6800/1700562157":"2023-10-02T10:05:57.727322+0000","172.21.15.78:0/3635389419":"2023-10-02T10:05:57.727322+0000","172.21.15.78:0/3655343641":"2023-10-02T10:13:17.135564+0000","172.21.15.78:6800/3027286102":"2023-10-02T10:05:27.007550+0000","172.21.15.78:6801/3027286102":"2023-10-02T10:05:27.007550+0000","172.21.15.78:6801/1224621968":"2023-10-02T10:13:17.135564+0000","172.21.15.78:0/308254153":"2023-10-02T10:05:27.007550+0000","172.21.15.78:6801/1700562157":"2023-10-02T10:05:57.727322+0000","172.21.15.78:0/158328292":"2023-10-02T10:05:27.007550+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-01T10:13:38.443 DEBUG:teuthology.misc:8 of 8 OSDs are up 2023-10-01T10:13:38.443 INFO:tasks.cephadm:Creating RBD pool 2023-10-01T10:13:38.443 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- sudo ceph --cluster ceph osd pool create rbd 8 2023-10-01T10:13:38.489 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:38 smithi173 bash[27612]: cluster 2023-10-01T10:13:37.156837+0000 mgr.y (mgr.14427) 22 : cluster [DBG] pgmap v13: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:38.489 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:13:38 smithi173 systemd[1]: Started Ceph prometheus.a for d8016480-6041-11ee-8db5-212e2dc638e7. 2023-10-01T10:13:38.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:38 smithi078 bash[33821]: cluster 2023-10-01T10:13:37.156837+0000 mgr.y (mgr.14427) 22 : cluster [DBG] pgmap v13: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:38.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:38 smithi078 bash[27336]: cluster 2023-10-01T10:13:37.156837+0000 mgr.y (mgr.14427) 22 : cluster [DBG] pgmap v13: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:39.197 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:39 smithi078 systemd[1]: Started Ceph node-exporter.a for d8016480-6041-11ee-8db5-212e2dc638e7. 2023-10-01T10:13:39.333 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:39 smithi078 bash[52454]: Unable to find image 'quay.io/prometheus/node-exporter:v1.3.1' locally 2023-10-01T10:13:39.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:39 smithi078 bash[33821]: audit 2023-10-01T10:13:38.422084+0000 mon.c (mon.2) 18 : audit [DBG] from='client.? 172.21.15.78:0/1245902745' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T10:13:39.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:39 smithi078 bash[33821]: audit 2023-10-01T10:13:38.469784+0000 mon.a (mon.0) 588 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:13:39.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:39 smithi078 bash[33821]: cephadm 2023-10-01T10:13:38.472160+0000 mgr.y (mgr.14427) 23 : cephadm [INF] Deploying daemon node-exporter.a on smithi078 2023-10-01T10:13:39.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:39 smithi078 bash[33821]: audit 2023-10-01T10:13:39.176835+0000 mon.a (mon.0) 589 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:13:39.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:39 smithi078 bash[27336]: audit 2023-10-01T10:13:38.422084+0000 mon.c (mon.2) 18 : audit [DBG] from='client.? 172.21.15.78:0/1245902745' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T10:13:39.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:39 smithi078 bash[27336]: audit 2023-10-01T10:13:38.469784+0000 mon.a (mon.0) 588 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:13:39.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:39 smithi078 bash[27336]: cephadm 2023-10-01T10:13:38.472160+0000 mgr.y (mgr.14427) 23 : cephadm [INF] Deploying daemon node-exporter.a on smithi078 2023-10-01T10:13:39.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:39 smithi078 bash[27336]: audit 2023-10-01T10:13:39.176835+0000 mon.a (mon.0) 589 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:13:39.519 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:39 smithi173 bash[27612]: audit 2023-10-01T10:13:38.422084+0000 mon.c (mon.2) 18 : audit [DBG] from='client.? 172.21.15.78:0/1245902745' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T10:13:39.519 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:39 smithi173 bash[27612]: audit 2023-10-01T10:13:38.469784+0000 mon.a (mon.0) 588 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:13:39.519 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:39 smithi173 bash[27612]: cephadm 2023-10-01T10:13:38.472160+0000 mgr.y (mgr.14427) 23 : cephadm [INF] Deploying daemon node-exporter.a on smithi078 2023-10-01T10:13:39.519 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:39 smithi173 bash[27612]: audit 2023-10-01T10:13:39.176835+0000 mon.a (mon.0) 589 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:13:39.871 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:13:39 smithi173 bash[46761]: ts=2023-10-01T10:13:39.517Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2023-10-01T10:13:39.871 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:13:39 smithi173 bash[46761]: ts=2023-10-01T10:13:39.517Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2023-10-01T10:13:39.871 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:13:39 smithi173 bash[46761]: ts=2023-10-01T10:13:39.517Z caller=main.go:518 level=info host_details="(Linux 5.15.0-84-generic #93~20.04.1-Ubuntu SMP Wed Sep 6 16:15:40 UTC 2023 x86_64 smithi173 (none))" 2023-10-01T10:13:39.871 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:13:39 smithi173 bash[46761]: ts=2023-10-01T10:13:39.517Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-10-01T10:13:39.871 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:13:39 smithi173 bash[46761]: ts=2023-10-01T10:13:39.517Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-10-01T10:13:39.871 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:13:39 smithi173 bash[46761]: ts=2023-10-01T10:13:39.520Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2023-10-01T10:13:39.871 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:13:39 smithi173 bash[46761]: ts=2023-10-01T10:13:39.520Z caller=main.go:923 level=info msg="Starting TSDB ..." 2023-10-01T10:13:39.871 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:13:39 smithi173 bash[46761]: ts=2023-10-01T10:13:39.522Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2023-10-01T10:13:39.871 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:13:39 smithi173 bash[46761]: ts=2023-10-01T10:13:39.526Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-10-01T10:13:39.872 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:13:39 smithi173 bash[46761]: ts=2023-10-01T10:13:39.526Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.405µs 2023-10-01T10:13:39.872 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:13:39 smithi173 bash[46761]: ts=2023-10-01T10:13:39.526Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-10-01T10:13:39.872 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:13:39 smithi173 bash[46761]: ts=2023-10-01T10:13:39.526Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2023-10-01T10:13:39.872 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:13:39 smithi173 bash[46761]: ts=2023-10-01T10:13:39.527Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=62.539µs wal_replay_duration=300.791µs total_replay_duration=386.511µs 2023-10-01T10:13:39.872 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:13:39 smithi173 bash[46761]: ts=2023-10-01T10:13:39.528Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2023-10-01T10:13:39.872 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:13:39 smithi173 bash[46761]: ts=2023-10-01T10:13:39.528Z caller=main.go:947 level=info msg="TSDB started" 2023-10-01T10:13:39.872 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:13:39 smithi173 bash[46761]: ts=2023-10-01T10:13:39.528Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-10-01T10:13:39.872 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:13:39 smithi173 bash[46761]: ts=2023-10-01T10:13:39.554Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=25.285181ms db_storage=1.088µs remote_storage=1.483µs web_handler=330ns query_engine=804ns scrape=351.736µs scrape_sd=23.543µs notify=1.159µs notify_sd=1.991µs rules=24.49636ms 2023-10-01T10:13:39.872 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:13:39 smithi173 bash[46761]: ts=2023-10-01T10:13:39.554Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2023-10-01T10:13:40.083 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:39 smithi078 bash[52454]: v1.3.1: Pulling from prometheus/node-exporter 2023-10-01T10:13:40.370 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:39 smithi173 systemd[1]: Started Ceph node-exporter.b for d8016480-6041-11ee-8db5-212e2dc638e7. 2023-10-01T10:13:40.371 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:39 smithi173 bash[46929]: Unable to find image 'quay.io/prometheus/node-exporter:v1.3.1' locally 2023-10-01T10:13:40.477 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:40 smithi078 bash[27336]: cluster 2023-10-01T10:13:39.157428+0000 mgr.y (mgr.14427) 24 : cluster [DBG] pgmap v14: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:40.478 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:40 smithi078 bash[52454]: aa2a8d90b84c: Pulling fs layer 2023-10-01T10:13:40.478 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:40 smithi078 bash[52454]: b45d31ee2d7f: Pulling fs layer 2023-10-01T10:13:40.478 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:40 smithi078 bash[52454]: b5db1e299295: Pulling fs layer 2023-10-01T10:13:40.478 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:40 smithi078 bash[52454]: b45d31ee2d7f: Verifying Checksum 2023-10-01T10:13:40.478 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:40 smithi078 bash[52454]: b45d31ee2d7f: Download complete 2023-10-01T10:13:40.478 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:40 smithi078 bash[52454]: aa2a8d90b84c: Download complete 2023-10-01T10:13:40.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:40 smithi078 bash[27336]: cephadm 2023-10-01T10:13:39.178045+0000 mgr.y (mgr.14427) 25 : cephadm [INF] Deploying daemon node-exporter.b on smithi173 2023-10-01T10:13:40.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:40 smithi078 bash[27336]: audit 2023-10-01T10:13:39.953674+0000 mon.a (mon.0) 590 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:13:40.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:40 smithi078 bash[33821]: cluster 2023-10-01T10:13:39.157428+0000 mgr.y (mgr.14427) 24 : cluster [DBG] pgmap v14: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:40.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:40 smithi078 bash[33821]: cephadm 2023-10-01T10:13:39.178045+0000 mgr.y (mgr.14427) 25 : cephadm [INF] Deploying daemon node-exporter.b on smithi173 2023-10-01T10:13:40.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:40 smithi078 bash[33821]: audit 2023-10-01T10:13:39.953674+0000 mon.a (mon.0) 590 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:13:40.584 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:40 smithi078 bash[52454]: b5db1e299295: Verifying Checksum 2023-10-01T10:13:40.584 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:40 smithi078 bash[52454]: b5db1e299295: Download complete 2023-10-01T10:13:40.805 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:40 smithi173 bash[46929]: v1.3.1: Pulling from prometheus/node-exporter 2023-10-01T10:13:40.805 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:40 smithi173 bash[27612]: cluster 2023-10-01T10:13:39.157428+0000 mgr.y (mgr.14427) 24 : cluster [DBG] pgmap v14: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:40.805 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:40 smithi173 bash[27612]: cephadm 2023-10-01T10:13:39.178045+0000 mgr.y (mgr.14427) 25 : cephadm [INF] Deploying daemon node-exporter.b on smithi173 2023-10-01T10:13:40.805 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:40 smithi173 bash[27612]: audit 2023-10-01T10:13:39.953674+0000 mon.a (mon.0) 590 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:13:41.120 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:40 smithi173 bash[46929]: aa2a8d90b84c: Pulling fs layer 2023-10-01T10:13:41.120 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:40 smithi173 bash[46929]: b45d31ee2d7f: Pulling fs layer 2023-10-01T10:13:41.120 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:40 smithi173 bash[46929]: b5db1e299295: Pulling fs layer 2023-10-01T10:13:41.120 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:40 smithi173 bash[46929]: b45d31ee2d7f: Verifying Checksum 2023-10-01T10:13:41.121 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:41 smithi173 bash[46929]: aa2a8d90b84c: Download complete 2023-10-01T10:13:41.333 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:40 smithi078 bash[52454]: aa2a8d90b84c: Pull complete 2023-10-01T10:13:41.477 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:41 smithi173 bash[46929]: aa2a8d90b84c: Pull complete 2023-10-01T10:13:41.477 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:41 smithi173 bash[46929]: b5db1e299295: Verifying Checksum 2023-10-01T10:13:41.477 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:41 smithi173 bash[46929]: b5db1e299295: Download complete 2023-10-01T10:13:41.480 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:41 smithi173 bash[27612]: cephadm 2023-10-01T10:13:39.964895+0000 mgr.y (mgr.14427) 26 : cephadm [INF] Deploying daemon alertmanager.a on smithi078 2023-10-01T10:13:41.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:41 smithi078 bash[33821]: cephadm 2023-10-01T10:13:39.964895+0000 mgr.y (mgr.14427) 26 : cephadm [INF] Deploying daemon alertmanager.a on smithi078 2023-10-01T10:13:41.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:41 smithi078 bash[27336]: cephadm 2023-10-01T10:13:39.964895+0000 mgr.y (mgr.14427) 26 : cephadm [INF] Deploying daemon alertmanager.a on smithi078 2023-10-01T10:13:41.872 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:41 smithi173 bash[46929]: b45d31ee2d7f: Pull complete 2023-10-01T10:13:42.370 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:41 smithi173 bash[46929]: b5db1e299295: Pull complete 2023-10-01T10:13:42.370 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:42 smithi173 bash[46929]: Digest: sha256:f2269e73124dd0f60a7d19a2ce1264d33d08a985aed0ee6b0b89d0be470592cd 2023-10-01T10:13:42.371 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:42 smithi173 bash[46929]: Status: Downloaded newer image for quay.io/prometheus/node-exporter:v1.3.1 2023-10-01T10:13:42.478 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:42 smithi078 bash[52454]: b45d31ee2d7f: Pull complete 2023-10-01T10:13:42.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:42 smithi078 bash[27336]: cluster 2023-10-01T10:13:41.157844+0000 mgr.y (mgr.14427) 27 : cluster [DBG] pgmap v15: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:42.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:42 smithi078 bash[27336]: audit 2023-10-01T10:13:42.366961+0000 mon.a (mon.0) 591 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:13:42.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:42 smithi078 bash[33821]: cluster 2023-10-01T10:13:41.157844+0000 mgr.y (mgr.14427) 27 : cluster [DBG] pgmap v15: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:42.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:42 smithi078 bash[33821]: audit 2023-10-01T10:13:42.366961+0000 mon.a (mon.0) 591 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:13:42.834 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:42 smithi078 bash[52454]: b5db1e299295: Pull complete 2023-10-01T10:13:42.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:42 smithi173 bash[27612]: cluster 2023-10-01T10:13:41.157844+0000 mgr.y (mgr.14427) 27 : cluster [DBG] pgmap v15: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:42.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:42 smithi173 bash[27612]: audit 2023-10-01T10:13:42.366961+0000 mon.a (mon.0) 591 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:13:43.333 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:42 smithi078 bash[52454]: Digest: sha256:f2269e73124dd0f60a7d19a2ce1264d33d08a985aed0ee6b0b89d0be470592cd 2023-10-01T10:13:43.333 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:42 smithi078 bash[52454]: Status: Downloaded newer image for quay.io/prometheus/node-exporter:v1.3.1 2023-10-01T10:13:43.660 INFO:teuthology.orchestra.run.smithi078.stderr:pool 'rbd' created 2023-10-01T10:13:43.661 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:43 smithi173 bash[46929]: ts=2023-10-01T10:13:43.509Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2023-10-01T10:13:43.661 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:43 smithi173 bash[46929]: ts=2023-10-01T10:13:43.509Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2023-10-01T10:13:43.661 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:43 smithi173 bash[46929]: ts=2023-10-01T10:13:43.510Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2023-10-01T10:13:43.661 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:43 smithi173 bash[46929]: ts=2023-10-01T10:13:43.510Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-10-01T10:13:43.661 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:43 smithi173 bash[46929]: ts=2023-10-01T10:13:43.511Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2023-10-01T10:13:43.661 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:43 smithi173 bash[46929]: ts=2023-10-01T10:13:43.511Z caller=node_exporter.go:115 level=info collector=arp 2023-10-01T10:13:43.661 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:43 smithi173 bash[46929]: ts=2023-10-01T10:13:43.511Z caller=node_exporter.go:115 level=info collector=bcache 2023-10-01T10:13:43.662 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:43 smithi173 bash[46929]: ts=2023-10-01T10:13:43.511Z caller=node_exporter.go:115 level=info collector=bonding 2023-10-01T10:13:43.662 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:43 smithi173 bash[46929]: ts=2023-10-01T10:13:43.511Z caller=node_exporter.go:115 level=info collector=btrfs 2023-10-01T10:13:43.662 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:43 smithi173 bash[46929]: ts=2023-10-01T10:13:43.511Z caller=node_exporter.go:115 level=info collector=conntrack 2023-10-01T10:13:43.662 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:43 smithi173 bash[46929]: ts=2023-10-01T10:13:43.511Z caller=node_exporter.go:115 level=info collector=cpu 2023-10-01T10:13:43.662 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:43 smithi173 bash[46929]: ts=2023-10-01T10:13:43.511Z caller=node_exporter.go:115 level=info collector=cpufreq 2023-10-01T10:13:43.663 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:43 smithi173 bash[46929]: ts=2023-10-01T10:13:43.511Z caller=node_exporter.go:115 level=info collector=diskstats 2023-10-01T10:13:43.663 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:43 smithi173 bash[46929]: ts=2023-10-01T10:13:43.511Z caller=node_exporter.go:115 level=info collector=dmi 2023-10-01T10:13:43.663 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:43 smithi173 bash[46929]: ts=2023-10-01T10:13:43.511Z caller=node_exporter.go:115 level=info collector=edac 2023-10-01T10:13:43.663 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:43 smithi173 bash[46929]: ts=2023-10-01T10:13:43.511Z caller=node_exporter.go:115 level=info collector=entropy 2023-10-01T10:13:43.663 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:43 smithi173 bash[46929]: ts=2023-10-01T10:13:43.511Z caller=node_exporter.go:115 level=info collector=fibrechannel 2023-10-01T10:13:43.663 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:43 smithi173 bash[46929]: ts=2023-10-01T10:13:43.511Z caller=node_exporter.go:115 level=info collector=filefd 2023-10-01T10:13:43.663 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:43 smithi173 bash[46929]: ts=2023-10-01T10:13:43.511Z caller=node_exporter.go:115 level=info collector=filesystem 2023-10-01T10:13:43.663 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:43 smithi173 bash[46929]: ts=2023-10-01T10:13:43.511Z caller=node_exporter.go:115 level=info collector=hwmon 2023-10-01T10:13:43.663 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:43 smithi173 bash[46929]: ts=2023-10-01T10:13:43.511Z caller=node_exporter.go:115 level=info collector=infiniband 2023-10-01T10:13:43.663 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:43 smithi173 bash[46929]: ts=2023-10-01T10:13:43.511Z caller=node_exporter.go:115 level=info collector=ipvs 2023-10-01T10:13:43.663 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:43 smithi173 bash[46929]: ts=2023-10-01T10:13:43.511Z caller=node_exporter.go:115 level=info collector=loadavg 2023-10-01T10:13:43.664 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:43 smithi173 bash[46929]: ts=2023-10-01T10:13:43.511Z caller=node_exporter.go:115 level=info collector=mdadm 2023-10-01T10:13:43.664 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:43 smithi173 bash[46929]: ts=2023-10-01T10:13:43.511Z caller=node_exporter.go:115 level=info collector=meminfo 2023-10-01T10:13:43.664 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:43 smithi173 bash[46929]: ts=2023-10-01T10:13:43.511Z caller=node_exporter.go:115 level=info collector=netclass 2023-10-01T10:13:43.664 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:43 smithi173 bash[46929]: ts=2023-10-01T10:13:43.511Z caller=node_exporter.go:115 level=info collector=netdev 2023-10-01T10:13:43.664 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:43 smithi173 bash[46929]: ts=2023-10-01T10:13:43.511Z caller=node_exporter.go:115 level=info collector=netstat 2023-10-01T10:13:43.664 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:43 smithi173 bash[46929]: ts=2023-10-01T10:13:43.511Z caller=node_exporter.go:115 level=info collector=nfs 2023-10-01T10:13:43.664 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:43 smithi173 bash[46929]: ts=2023-10-01T10:13:43.511Z caller=node_exporter.go:115 level=info collector=nfsd 2023-10-01T10:13:43.664 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:43 smithi173 bash[46929]: ts=2023-10-01T10:13:43.511Z caller=node_exporter.go:115 level=info collector=nvme 2023-10-01T10:13:43.664 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:43 smithi173 bash[46929]: ts=2023-10-01T10:13:43.511Z caller=node_exporter.go:115 level=info collector=os 2023-10-01T10:13:43.664 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:43 smithi173 bash[46929]: ts=2023-10-01T10:13:43.511Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2023-10-01T10:13:43.664 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:43 smithi173 bash[46929]: ts=2023-10-01T10:13:43.511Z caller=node_exporter.go:115 level=info collector=pressure 2023-10-01T10:13:43.664 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:43 smithi173 bash[46929]: ts=2023-10-01T10:13:43.511Z caller=node_exporter.go:115 level=info collector=rapl 2023-10-01T10:13:43.665 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:43 smithi173 bash[46929]: ts=2023-10-01T10:13:43.511Z caller=node_exporter.go:115 level=info collector=schedstat 2023-10-01T10:13:43.665 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:43 smithi173 bash[46929]: ts=2023-10-01T10:13:43.511Z caller=node_exporter.go:115 level=info collector=sockstat 2023-10-01T10:13:43.665 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:43 smithi173 bash[46929]: ts=2023-10-01T10:13:43.511Z caller=node_exporter.go:115 level=info collector=softnet 2023-10-01T10:13:43.665 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:43 smithi173 bash[46929]: ts=2023-10-01T10:13:43.511Z caller=node_exporter.go:115 level=info collector=stat 2023-10-01T10:13:43.665 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:43 smithi173 bash[46929]: ts=2023-10-01T10:13:43.511Z caller=node_exporter.go:115 level=info collector=tapestats 2023-10-01T10:13:43.665 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:43 smithi173 bash[46929]: ts=2023-10-01T10:13:43.511Z caller=node_exporter.go:115 level=info collector=textfile 2023-10-01T10:13:43.665 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:43 smithi173 bash[46929]: ts=2023-10-01T10:13:43.511Z caller=node_exporter.go:115 level=info collector=thermal_zone 2023-10-01T10:13:43.665 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:43 smithi173 bash[46929]: ts=2023-10-01T10:13:43.511Z caller=node_exporter.go:115 level=info collector=time 2023-10-01T10:13:43.665 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:43 smithi173 bash[46929]: ts=2023-10-01T10:13:43.511Z caller=node_exporter.go:115 level=info collector=udp_queues 2023-10-01T10:13:43.665 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:43 smithi173 bash[46929]: ts=2023-10-01T10:13:43.511Z caller=node_exporter.go:115 level=info collector=uname 2023-10-01T10:13:43.665 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:43 smithi173 bash[46929]: ts=2023-10-01T10:13:43.511Z caller=node_exporter.go:115 level=info collector=vmstat 2023-10-01T10:13:43.665 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:43 smithi173 bash[46929]: ts=2023-10-01T10:13:43.511Z caller=node_exporter.go:115 level=info collector=xfs 2023-10-01T10:13:43.666 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:43 smithi173 bash[46929]: ts=2023-10-01T10:13:43.511Z caller=node_exporter.go:115 level=info collector=zfs 2023-10-01T10:13:43.666 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:43 smithi173 bash[46929]: ts=2023-10-01T10:13:43.512Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2023-10-01T10:13:43.666 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:13:43 smithi173 bash[46929]: ts=2023-10-01T10:13:43.512Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2023-10-01T10:13:44.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:43 smithi078 bash[33821]: audit 2023-10-01T10:13:43.147963+0000 mon.a (mon.0) 592 : audit [INF] from='client.? 172.21.15.78:0/3979806997' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-10-01T10:13:44.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:43 smithi078 bash[33821]: cluster 2023-10-01T10:13:43.158168+0000 mgr.y (mgr.14427) 28 : cluster [DBG] pgmap v16: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:44.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:43 smithi078 bash[27336]: audit 2023-10-01T10:13:43.147963+0000 mon.a (mon.0) 592 : audit [INF] from='client.? 172.21.15.78:0/3979806997' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-10-01T10:13:44.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:43 smithi078 bash[27336]: cluster 2023-10-01T10:13:43.158168+0000 mgr.y (mgr.14427) 28 : cluster [DBG] pgmap v16: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:44.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:43 smithi173 bash[27612]: audit 2023-10-01T10:13:43.147963+0000 mon.a (mon.0) 592 : audit [INF] from='client.? 172.21.15.78:0/3979806997' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-10-01T10:13:44.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:43 smithi173 bash[27612]: cluster 2023-10-01T10:13:43.158168+0000 mgr.y (mgr.14427) 28 : cluster [DBG] pgmap v16: 1 pgs: 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:45.032 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:44 smithi078 bash[33821]: audit 2023-10-01T10:13:43.651059+0000 mon.a (mon.0) 593 : audit [INF] from='client.? 172.21.15.78:0/3979806997' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-10-01T10:13:45.032 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:44 smithi078 bash[33821]: cluster 2023-10-01T10:13:43.651117+0000 mon.a (mon.0) 594 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2023-10-01T10:13:45.032 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:44 smithi078 bash[33821]: audit 2023-10-01T10:13:43.651897+0000 mon.a (mon.0) 595 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-01T10:13:45.032 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:44 smithi078 bash[33821]: audit 2023-10-01T10:13:43.652983+0000 mon.a (mon.0) 596 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-10-01T10:13:45.032 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:44 smithi078 bash[27336]: audit 2023-10-01T10:13:43.651059+0000 mon.a (mon.0) 593 : audit [INF] from='client.? 172.21.15.78:0/3979806997' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-10-01T10:13:45.033 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:44 smithi078 bash[27336]: cluster 2023-10-01T10:13:43.651117+0000 mon.a (mon.0) 594 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2023-10-01T10:13:45.033 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:44 smithi078 bash[27336]: audit 2023-10-01T10:13:43.651897+0000 mon.a (mon.0) 595 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-01T10:13:45.033 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:44 smithi078 bash[27336]: audit 2023-10-01T10:13:43.652983+0000 mon.a (mon.0) 596 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-10-01T10:13:45.034 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- sudo ceph --cluster ceph osd pool application enable rbd rbd --yes-i-really-mean-it 2023-10-01T10:13:45.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:44 smithi173 bash[27612]: audit 2023-10-01T10:13:43.651059+0000 mon.a (mon.0) 593 : audit [INF] from='client.? 172.21.15.78:0/3979806997' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-10-01T10:13:45.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:44 smithi173 bash[27612]: cluster 2023-10-01T10:13:43.651117+0000 mon.a (mon.0) 594 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2023-10-01T10:13:45.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:44 smithi173 bash[27612]: audit 2023-10-01T10:13:43.651897+0000 mon.a (mon.0) 595 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": "device_health_metrics", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-10-01T10:13:45.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:44 smithi173 bash[27612]: audit 2023-10-01T10:13:43.652983+0000 mon.a (mon.0) 596 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]: dispatch 2023-10-01T10:13:45.332 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:13:45 smithi078 bash[27601]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:13:45] "GET /metrics HTTP/1.1" 200 153276 "" "Prometheus/2.33.4" 2023-10-01T10:13:45.583 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:45 smithi078 bash[52454]: ts=2023-10-01T10:13:45.450Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2023-10-01T10:13:45.584 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:45 smithi078 bash[52454]: ts=2023-10-01T10:13:45.450Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2023-10-01T10:13:45.584 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:45 smithi078 bash[52454]: ts=2023-10-01T10:13:45.450Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2023-10-01T10:13:45.584 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:45 smithi078 bash[52454]: ts=2023-10-01T10:13:45.450Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-10-01T10:13:45.584 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:45 smithi078 bash[52454]: ts=2023-10-01T10:13:45.451Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2023-10-01T10:13:45.584 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:45 smithi078 bash[52454]: ts=2023-10-01T10:13:45.451Z caller=node_exporter.go:115 level=info collector=arp 2023-10-01T10:13:45.584 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:45 smithi078 bash[52454]: ts=2023-10-01T10:13:45.451Z caller=node_exporter.go:115 level=info collector=bcache 2023-10-01T10:13:45.584 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:45 smithi078 bash[52454]: ts=2023-10-01T10:13:45.451Z caller=node_exporter.go:115 level=info collector=bonding 2023-10-01T10:13:45.584 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:45 smithi078 bash[52454]: ts=2023-10-01T10:13:45.451Z caller=node_exporter.go:115 level=info collector=btrfs 2023-10-01T10:13:45.584 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:45 smithi078 bash[52454]: ts=2023-10-01T10:13:45.451Z caller=node_exporter.go:115 level=info collector=conntrack 2023-10-01T10:13:45.584 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:45 smithi078 bash[52454]: ts=2023-10-01T10:13:45.451Z caller=node_exporter.go:115 level=info collector=cpu 2023-10-01T10:13:45.585 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:45 smithi078 bash[52454]: ts=2023-10-01T10:13:45.451Z caller=node_exporter.go:115 level=info collector=cpufreq 2023-10-01T10:13:45.585 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:45 smithi078 bash[52454]: ts=2023-10-01T10:13:45.451Z caller=node_exporter.go:115 level=info collector=diskstats 2023-10-01T10:13:45.585 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:45 smithi078 bash[52454]: ts=2023-10-01T10:13:45.451Z caller=node_exporter.go:115 level=info collector=dmi 2023-10-01T10:13:45.585 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:45 smithi078 bash[52454]: ts=2023-10-01T10:13:45.451Z caller=node_exporter.go:115 level=info collector=edac 2023-10-01T10:13:45.585 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:45 smithi078 bash[52454]: ts=2023-10-01T10:13:45.451Z caller=node_exporter.go:115 level=info collector=entropy 2023-10-01T10:13:45.585 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:45 smithi078 bash[52454]: ts=2023-10-01T10:13:45.451Z caller=node_exporter.go:115 level=info collector=fibrechannel 2023-10-01T10:13:45.585 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:45 smithi078 bash[52454]: ts=2023-10-01T10:13:45.451Z caller=node_exporter.go:115 level=info collector=filefd 2023-10-01T10:13:45.585 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:45 smithi078 bash[52454]: ts=2023-10-01T10:13:45.451Z caller=node_exporter.go:115 level=info collector=filesystem 2023-10-01T10:13:45.585 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:45 smithi078 bash[52454]: ts=2023-10-01T10:13:45.451Z caller=node_exporter.go:115 level=info collector=hwmon 2023-10-01T10:13:45.585 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:45 smithi078 bash[52454]: ts=2023-10-01T10:13:45.451Z caller=node_exporter.go:115 level=info collector=infiniband 2023-10-01T10:13:45.585 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:45 smithi078 bash[52454]: ts=2023-10-01T10:13:45.451Z caller=node_exporter.go:115 level=info collector=ipvs 2023-10-01T10:13:45.585 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:45 smithi078 bash[52454]: ts=2023-10-01T10:13:45.451Z caller=node_exporter.go:115 level=info collector=loadavg 2023-10-01T10:13:45.586 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:45 smithi078 bash[52454]: ts=2023-10-01T10:13:45.451Z caller=node_exporter.go:115 level=info collector=mdadm 2023-10-01T10:13:45.586 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:45 smithi078 bash[52454]: ts=2023-10-01T10:13:45.451Z caller=node_exporter.go:115 level=info collector=meminfo 2023-10-01T10:13:45.586 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:45 smithi078 bash[52454]: ts=2023-10-01T10:13:45.451Z caller=node_exporter.go:115 level=info collector=netclass 2023-10-01T10:13:45.586 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:45 smithi078 bash[52454]: ts=2023-10-01T10:13:45.451Z caller=node_exporter.go:115 level=info collector=netdev 2023-10-01T10:13:45.586 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:45 smithi078 bash[52454]: ts=2023-10-01T10:13:45.451Z caller=node_exporter.go:115 level=info collector=netstat 2023-10-01T10:13:45.586 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:45 smithi078 bash[52454]: ts=2023-10-01T10:13:45.451Z caller=node_exporter.go:115 level=info collector=nfs 2023-10-01T10:13:45.586 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:45 smithi078 bash[52454]: ts=2023-10-01T10:13:45.451Z caller=node_exporter.go:115 level=info collector=nfsd 2023-10-01T10:13:45.586 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:45 smithi078 bash[52454]: ts=2023-10-01T10:13:45.451Z caller=node_exporter.go:115 level=info collector=nvme 2023-10-01T10:13:45.586 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:45 smithi078 bash[52454]: ts=2023-10-01T10:13:45.451Z caller=node_exporter.go:115 level=info collector=os 2023-10-01T10:13:45.586 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:45 smithi078 bash[52454]: ts=2023-10-01T10:13:45.451Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2023-10-01T10:13:45.586 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:45 smithi078 bash[52454]: ts=2023-10-01T10:13:45.451Z caller=node_exporter.go:115 level=info collector=pressure 2023-10-01T10:13:45.586 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:45 smithi078 bash[52454]: ts=2023-10-01T10:13:45.451Z caller=node_exporter.go:115 level=info collector=rapl 2023-10-01T10:13:45.587 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:45 smithi078 bash[52454]: ts=2023-10-01T10:13:45.451Z caller=node_exporter.go:115 level=info collector=schedstat 2023-10-01T10:13:45.587 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:45 smithi078 bash[52454]: ts=2023-10-01T10:13:45.451Z caller=node_exporter.go:115 level=info collector=sockstat 2023-10-01T10:13:45.587 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:45 smithi078 bash[52454]: ts=2023-10-01T10:13:45.451Z caller=node_exporter.go:115 level=info collector=softnet 2023-10-01T10:13:45.587 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:45 smithi078 bash[52454]: ts=2023-10-01T10:13:45.451Z caller=node_exporter.go:115 level=info collector=stat 2023-10-01T10:13:45.587 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:45 smithi078 bash[52454]: ts=2023-10-01T10:13:45.451Z caller=node_exporter.go:115 level=info collector=tapestats 2023-10-01T10:13:45.587 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:45 smithi078 bash[52454]: ts=2023-10-01T10:13:45.451Z caller=node_exporter.go:115 level=info collector=textfile 2023-10-01T10:13:45.587 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:45 smithi078 bash[52454]: ts=2023-10-01T10:13:45.451Z caller=node_exporter.go:115 level=info collector=thermal_zone 2023-10-01T10:13:45.587 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:45 smithi078 bash[52454]: ts=2023-10-01T10:13:45.451Z caller=node_exporter.go:115 level=info collector=time 2023-10-01T10:13:45.587 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:45 smithi078 bash[52454]: ts=2023-10-01T10:13:45.451Z caller=node_exporter.go:115 level=info collector=udp_queues 2023-10-01T10:13:45.587 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:45 smithi078 bash[52454]: ts=2023-10-01T10:13:45.451Z caller=node_exporter.go:115 level=info collector=uname 2023-10-01T10:13:45.587 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:45 smithi078 bash[52454]: ts=2023-10-01T10:13:45.451Z caller=node_exporter.go:115 level=info collector=vmstat 2023-10-01T10:13:45.587 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:45 smithi078 bash[52454]: ts=2023-10-01T10:13:45.451Z caller=node_exporter.go:115 level=info collector=xfs 2023-10-01T10:13:45.587 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:45 smithi078 bash[52454]: ts=2023-10-01T10:13:45.451Z caller=node_exporter.go:115 level=info collector=zfs 2023-10-01T10:13:45.588 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:45 smithi078 bash[52454]: ts=2023-10-01T10:13:45.451Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2023-10-01T10:13:45.588 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:13:45 smithi078 bash[52454]: ts=2023-10-01T10:13:45.452Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2023-10-01T10:13:45.620 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:13:45 smithi173 bash[28473]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:13:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T10:13:46.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:45 smithi078 bash[33821]: audit 2023-10-01T10:13:44.656408+0000 mon.a (mon.0) 597 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-10-01T10:13:46.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:45 smithi078 bash[33821]: cluster 2023-10-01T10:13:44.656820+0000 mon.a (mon.0) 598 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2023-10-01T10:13:46.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:45 smithi078 bash[33821]: cluster 2023-10-01T10:13:45.158691+0000 mgr.y (mgr.14427) 29 : cluster [DBG] pgmap v19: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:46.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:45 smithi078 bash[27336]: audit 2023-10-01T10:13:44.656408+0000 mon.a (mon.0) 597 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-10-01T10:13:46.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:45 smithi078 bash[27336]: cluster 2023-10-01T10:13:44.656820+0000 mon.a (mon.0) 598 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2023-10-01T10:13:46.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:45 smithi078 bash[27336]: cluster 2023-10-01T10:13:45.158691+0000 mgr.y (mgr.14427) 29 : cluster [DBG] pgmap v19: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:46.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:45 smithi173 bash[27612]: audit 2023-10-01T10:13:44.656408+0000 mon.a (mon.0) 597 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": "device_health_metrics", "app": "mgr_devicehealth"}]': finished 2023-10-01T10:13:46.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:45 smithi173 bash[27612]: cluster 2023-10-01T10:13:44.656820+0000 mon.a (mon.0) 598 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2023-10-01T10:13:46.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:45 smithi173 bash[27612]: cluster 2023-10-01T10:13:45.158691+0000 mgr.y (mgr.14427) 29 : cluster [DBG] pgmap v19: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:47.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:46 smithi078 bash[33821]: cluster 2023-10-01T10:13:45.671120+0000 mon.a (mon.0) 599 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2023-10-01T10:13:47.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:46 smithi078 bash[27336]: cluster 2023-10-01T10:13:45.671120+0000 mon.a (mon.0) 599 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2023-10-01T10:13:47.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:46 smithi173 bash[27612]: cluster 2023-10-01T10:13:45.671120+0000 mon.a (mon.0) 599 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2023-10-01T10:13:48.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:47 smithi078 bash[27336]: cluster 2023-10-01T10:13:47.159221+0000 mgr.y (mgr.14427) 30 : cluster [DBG] pgmap v21: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:48.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:47 smithi078 bash[33821]: cluster 2023-10-01T10:13:47.159221+0000 mgr.y (mgr.14427) 30 : cluster [DBG] pgmap v21: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:48.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:47 smithi173 bash[27612]: cluster 2023-10-01T10:13:47.159221+0000 mgr.y (mgr.14427) 30 : cluster [DBG] pgmap v21: 9 pgs: 8 unknown, 1 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:50.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:50 smithi078 bash[27336]: cluster 2023-10-01T10:13:49.159867+0000 mgr.y (mgr.14427) 31 : cluster [DBG] pgmap v22: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:50.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:50 smithi078 bash[33821]: cluster 2023-10-01T10:13:49.159867+0000 mgr.y (mgr.14427) 31 : cluster [DBG] pgmap v22: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:50.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:50 smithi173 bash[27612]: cluster 2023-10-01T10:13:49.159867+0000 mgr.y (mgr.14427) 31 : cluster [DBG] pgmap v22: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:51.239 INFO:teuthology.orchestra.run.smithi078.stderr:enabled application 'rbd' on pool 'rbd' 2023-10-01T10:13:51.500 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:51 smithi078 bash[33821]: audit 2023-10-01T10:13:50.752794+0000 mon.c (mon.2) 19 : audit [INF] from='client.? 172.21.15.78:0/4160585621' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T10:13:51.500 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:51 smithi078 bash[33821]: audit 2023-10-01T10:13:50.753357+0000 mon.a (mon.0) 600 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T10:13:51.500 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:51 smithi078 bash[27336]: audit 2023-10-01T10:13:50.752794+0000 mon.c (mon.2) 19 : audit [INF] from='client.? 172.21.15.78:0/4160585621' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T10:13:51.500 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:51 smithi078 bash[27336]: audit 2023-10-01T10:13:50.753357+0000 mon.a (mon.0) 600 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T10:13:51.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:51 smithi173 bash[27612]: audit 2023-10-01T10:13:50.752794+0000 mon.c (mon.2) 19 : audit [INF] from='client.? 172.21.15.78:0/4160585621' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T10:13:51.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:51 smithi173 bash[27612]: audit 2023-10-01T10:13:50.753357+0000 mon.a (mon.0) 600 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T10:13:52.327 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-10-01T10:13:52.327 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-10-01T10:13:52.328 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph mgr dump --format=json 2023-10-01T10:13:52.502 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:52 smithi078 bash[27336]: cluster 2023-10-01T10:13:51.160473+0000 mgr.y (mgr.14427) 32 : cluster [DBG] pgmap v23: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:52.502 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:52 smithi078 bash[27336]: audit 2023-10-01T10:13:51.231427+0000 mon.a (mon.0) 601 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-10-01T10:13:52.502 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:52 smithi078 bash[27336]: cluster 2023-10-01T10:13:51.231498+0000 mon.a (mon.0) 602 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2023-10-01T10:13:52.502 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:52 smithi078 bash[33821]: cluster 2023-10-01T10:13:51.160473+0000 mgr.y (mgr.14427) 32 : cluster [DBG] pgmap v23: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:52.503 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:52 smithi078 bash[33821]: audit 2023-10-01T10:13:51.231427+0000 mon.a (mon.0) 601 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-10-01T10:13:52.503 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:52 smithi078 bash[33821]: cluster 2023-10-01T10:13:51.231498+0000 mon.a (mon.0) 602 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2023-10-01T10:13:52.622 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:52 smithi173 bash[27612]: cluster 2023-10-01T10:13:51.160473+0000 mgr.y (mgr.14427) 32 : cluster [DBG] pgmap v23: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:52.622 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:52 smithi173 bash[27612]: audit 2023-10-01T10:13:51.231427+0000 mon.a (mon.0) 601 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-10-01T10:13:52.622 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:52 smithi173 bash[27612]: cluster 2023-10-01T10:13:51.231498+0000 mon.a (mon.0) 602 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2023-10-01T10:13:53.333 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:Oct 01 10:13:52 smithi078 systemd[1]: Started Ceph alertmanager.a for d8016480-6041-11ee-8db5-212e2dc638e7. 2023-10-01T10:13:54.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:54 smithi078 bash[27336]: audit 2023-10-01T10:13:53.009832+0000 mon.a (mon.0) 603 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:13:54.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:54 smithi078 bash[27336]: audit 2023-10-01T10:13:53.011841+0000 mon.a (mon.0) 604 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:13:54.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:54 smithi078 bash[27336]: cluster 2023-10-01T10:13:53.161315+0000 mgr.y (mgr.14427) 33 : cluster [DBG] pgmap v25: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:54.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:54 smithi078 bash[33821]: audit 2023-10-01T10:13:53.009832+0000 mon.a (mon.0) 603 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:13:54.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:54 smithi078 bash[33821]: audit 2023-10-01T10:13:53.011841+0000 mon.a (mon.0) 604 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:13:54.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:54 smithi078 bash[33821]: cluster 2023-10-01T10:13:53.161315+0000 mgr.y (mgr.14427) 33 : cluster [DBG] pgmap v25: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:54.334 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:Oct 01 10:13:54 smithi078 bash[53177]: level=info ts=2023-10-01T10:13:54.130Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2023-10-01T10:13:54.334 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:Oct 01 10:13:54 smithi078 bash[53177]: level=info ts=2023-10-01T10:13:54.130Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2023-10-01T10:13:54.334 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:Oct 01 10:13:54 smithi078 bash[53177]: level=info ts=2023-10-01T10:13:54.134Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.78 port=9094 2023-10-01T10:13:54.334 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:Oct 01 10:13:54 smithi078 bash[53177]: level=info ts=2023-10-01T10:13:54.136Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-10-01T10:13:54.334 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:Oct 01 10:13:54 smithi078 bash[53177]: level=info ts=2023-10-01T10:13:54.184Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-01T10:13:54.334 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:Oct 01 10:13:54 smithi078 bash[53177]: level=info ts=2023-10-01T10:13:54.185Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-01T10:13:54.334 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:Oct 01 10:13:54 smithi078 bash[53177]: level=info ts=2023-10-01T10:13:54.189Z caller=main.go:518 msg=Listening address=:9093 2023-10-01T10:13:54.335 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:Oct 01 10:13:54 smithi078 bash[53177]: level=info ts=2023-10-01T10:13:54.189Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2023-10-01T10:13:54.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:54 smithi173 bash[27612]: audit 2023-10-01T10:13:53.009832+0000 mon.a (mon.0) 603 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:13:54.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:54 smithi173 bash[27612]: audit 2023-10-01T10:13:53.011841+0000 mon.a (mon.0) 604 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:13:54.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:54 smithi173 bash[27612]: cluster 2023-10-01T10:13:53.161315+0000 mgr.y (mgr.14427) 33 : cluster [DBG] pgmap v25: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:55.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:13:55 smithi078 bash[27601]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:13:55] "GET /metrics HTTP/1.1" 200 155663 "" "Prometheus/2.33.4" 2023-10-01T10:13:55.620 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:13:55 smithi173 bash[28473]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:13:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T10:13:56.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:56 smithi078 bash[27336]: cluster 2023-10-01T10:13:55.161911+0000 mgr.y (mgr.14427) 34 : cluster [DBG] pgmap v26: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:56.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:56 smithi078 bash[33821]: cluster 2023-10-01T10:13:55.161911+0000 mgr.y (mgr.14427) 34 : cluster [DBG] pgmap v26: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:56.584 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:Oct 01 10:13:56 smithi078 bash[53177]: level=info ts=2023-10-01T10:13:56.137Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.001043895s 2023-10-01T10:13:56.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:56 smithi173 bash[27612]: cluster 2023-10-01T10:13:55.161911+0000 mgr.y (mgr.14427) 34 : cluster [DBG] pgmap v26: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:58.099 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:13:58.387 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:58 smithi078 bash[33821]: cluster 2023-10-01T10:13:57.162542+0000 mgr.y (mgr.14427) 35 : cluster [DBG] pgmap v27: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:58.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:58 smithi078 bash[33821]: audit 2023-10-01T10:13:57.384569+0000 mon.a (mon.0) 605 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:13:58.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:58 smithi078 bash[27336]: cluster 2023-10-01T10:13:57.162542+0000 mgr.y (mgr.14427) 35 : cluster [DBG] pgmap v27: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:58.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:58 smithi078 bash[27336]: audit 2023-10-01T10:13:57.384569+0000 mon.a (mon.0) 605 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:13:58.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:58 smithi173 bash[27612]: cluster 2023-10-01T10:13:57.162542+0000 mgr.y (mgr.14427) 35 : cluster [DBG] pgmap v27: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:58.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:58 smithi173 bash[27612]: audit 2023-10-01T10:13:57.384569+0000 mon.a (mon.0) 605 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:13:59.348 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:13:59 smithi078 bash[27601]: [01/Oct/2023:10:13:59] ENGINE Bus STOPPING 2023-10-01T10:13:59.677 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:13:59 smithi078 bash[27601]: [01/Oct/2023:10:13:59] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-01T10:13:59.677 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:13:59 smithi078 bash[27601]: [01/Oct/2023:10:13:59] ENGINE Bus STOPPED 2023-10-01T10:13:59.677 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:13:59 smithi078 bash[27601]: [01/Oct/2023:10:13:59] ENGINE Bus STARTING 2023-10-01T10:13:59.677 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:13:59 smithi078 bash[27601]: [01/Oct/2023:10:13:59] ENGINE Serving on http://:::9283 2023-10-01T10:13:59.677 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:13:59 smithi078 bash[27601]: [01/Oct/2023:10:13:59] ENGINE Bus STARTED 2023-10-01T10:13:59.986 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:59 smithi078 bash[33821]: audit 2023-10-01T10:13:58.673974+0000 mon.a (mon.0) 606 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:13:59.986 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:59 smithi078 bash[33821]: audit 2023-10-01T10:13:58.933311+0000 mon.a (mon.0) 607 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:13:59.986 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:59 smithi078 bash[33821]: audit 2023-10-01T10:13:58.935174+0000 mon.a (mon.0) 608 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:13:59.986 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:59 smithi078 bash[33821]: audit 2023-10-01T10:13:58.936401+0000 mon.a (mon.0) 609 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:13:59.986 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:59 smithi078 bash[33821]: audit 2023-10-01T10:13:58.942578+0000 mon.a (mon.0) 610 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:13:59.986 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:59 smithi078 bash[33821]: cluster 2023-10-01T10:13:59.163169+0000 mgr.y (mgr.14427) 36 : cluster [DBG] pgmap v28: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:59.986 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:59 smithi078 bash[33821]: audit 2023-10-01T10:13:59.290453+0000 mon.a (mon.0) 611 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:13:59.986 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:59 smithi078 bash[33821]: audit 2023-10-01T10:13:59.294655+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:13:59.987 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:59 smithi078 bash[33821]: audit 2023-10-01T10:13:59.298628+0000 mon.a (mon.0) 613 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-01T10:13:59.987 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:59 smithi078 bash[33821]: audit 2023-10-01T10:13:59.299042+0000 mgr.y (mgr.14427) 37 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-01T10:13:59.987 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:59 smithi078 bash[33821]: audit 2023-10-01T10:13:59.303064+0000 mon.a (mon.0) 614 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:13:59.987 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:13:59 smithi078 bash[33821]: cephadm 2023-10-01T10:13:59.319298+0000 mgr.y (mgr.14427) 38 : cephadm [INF] Deploying daemon grafana.a on smithi173 2023-10-01T10:13:59.987 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:59 smithi078 bash[27336]: audit 2023-10-01T10:13:58.673974+0000 mon.a (mon.0) 606 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:13:59.987 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:59 smithi078 bash[27336]: audit 2023-10-01T10:13:58.933311+0000 mon.a (mon.0) 607 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:13:59.987 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:59 smithi078 bash[27336]: audit 2023-10-01T10:13:58.935174+0000 mon.a (mon.0) 608 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:13:59.987 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:59 smithi078 bash[27336]: audit 2023-10-01T10:13:58.936401+0000 mon.a (mon.0) 609 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:13:59.987 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:59 smithi078 bash[27336]: audit 2023-10-01T10:13:58.942578+0000 mon.a (mon.0) 610 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:13:59.987 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:59 smithi078 bash[27336]: cluster 2023-10-01T10:13:59.163169+0000 mgr.y (mgr.14427) 36 : cluster [DBG] pgmap v28: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:13:59.988 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:59 smithi078 bash[27336]: audit 2023-10-01T10:13:59.290453+0000 mon.a (mon.0) 611 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:13:59.988 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:59 smithi078 bash[27336]: audit 2023-10-01T10:13:59.294655+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:13:59.988 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:59 smithi078 bash[27336]: audit 2023-10-01T10:13:59.298628+0000 mon.a (mon.0) 613 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-01T10:13:59.988 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:59 smithi078 bash[27336]: audit 2023-10-01T10:13:59.299042+0000 mgr.y (mgr.14427) 37 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-01T10:13:59.988 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:59 smithi078 bash[27336]: audit 2023-10-01T10:13:59.303064+0000 mon.a (mon.0) 614 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:13:59.988 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:13:59 smithi078 bash[27336]: cephadm 2023-10-01T10:13:59.319298+0000 mgr.y (mgr.14427) 38 : cephadm [INF] Deploying daemon grafana.a on smithi173 2023-10-01T10:14:00.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:59 smithi173 bash[27612]: audit 2023-10-01T10:13:58.673974+0000 mon.a (mon.0) 606 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:14:00.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:59 smithi173 bash[27612]: audit 2023-10-01T10:13:58.933311+0000 mon.a (mon.0) 607 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:14:00.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:59 smithi173 bash[27612]: audit 2023-10-01T10:13:58.935174+0000 mon.a (mon.0) 608 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:14:00.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:59 smithi173 bash[27612]: audit 2023-10-01T10:13:58.936401+0000 mon.a (mon.0) 609 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:14:00.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:59 smithi173 bash[27612]: audit 2023-10-01T10:13:58.942578+0000 mon.a (mon.0) 610 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:14:00.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:59 smithi173 bash[27612]: cluster 2023-10-01T10:13:59.163169+0000 mgr.y (mgr.14427) 36 : cluster [DBG] pgmap v28: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:00.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:59 smithi173 bash[27612]: audit 2023-10-01T10:13:59.290453+0000 mon.a (mon.0) 611 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:14:00.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:59 smithi173 bash[27612]: audit 2023-10-01T10:13:59.294655+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:14:00.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:59 smithi173 bash[27612]: audit 2023-10-01T10:13:59.298628+0000 mon.a (mon.0) 613 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-01T10:14:00.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:59 smithi173 bash[27612]: audit 2023-10-01T10:13:59.299042+0000 mgr.y (mgr.14427) 37 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-10-01T10:14:00.122 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:59 smithi173 bash[27612]: audit 2023-10-01T10:13:59.303064+0000 mon.a (mon.0) 614 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:14:00.122 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:13:59 smithi173 bash[27612]: cephadm 2023-10-01T10:13:59.319298+0000 mgr.y (mgr.14427) 38 : cephadm [INF] Deploying daemon grafana.a on smithi173 2023-10-01T10:14:01.240 INFO:teuthology.orchestra.run.smithi078.stdout: 2023-10-01T10:14:01.974 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":18,"active_gid":14427,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6800","nonce":516444166},{"type":"v1","addr":"172.21.15.78:6801","nonce":516444166}]},"active_addr":"172.21.15.78:6801/516444166","active_change":"2023-10-01T10:13:17.135858+0000","active_mgr_features":4540138314316775423,"available":true,"standbys":[{"gid":24349,"name":"x","mgr_features":4540138314316775423,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.78:8443/","prometheus":"http://172.21.15.78:9283/"},"always_on_modules":{"nautilus":["balancer","crash","devicehealth","orchestrator_cli","progress","rbd_support","status","volumes"],"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":51,"active_clients":[{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.78:0","nonce":3274439700}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.78:0","nonce":702157702}]}]} 2023-10-01T10:14:01.979 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-10-01T10:14:01.979 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-10-01T10:14:01.979 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-01T10:14:02.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:14:01 smithi078 bash[33821]: cluster 2023-10-01T10:14:01.163579+0000 mgr.y (mgr.14427) 39 : cluster [DBG] pgmap v29: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:02.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:14:01 smithi078 bash[33821]: audit 2023-10-01T10:14:01.228920+0000 mon.c (mon.2) 20 : audit [DBG] from='client.? 172.21.15.78:0/2810962441' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-01T10:14:02.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:14:01 smithi078 bash[27336]: cluster 2023-10-01T10:14:01.163579+0000 mgr.y (mgr.14427) 39 : cluster [DBG] pgmap v29: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:02.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:14:01 smithi078 bash[27336]: audit 2023-10-01T10:14:01.228920+0000 mon.c (mon.2) 20 : audit [DBG] from='client.? 172.21.15.78:0/2810962441' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-01T10:14:02.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:14:01 smithi173 bash[27612]: cluster 2023-10-01T10:14:01.163579+0000 mgr.y (mgr.14427) 39 : cluster [DBG] pgmap v29: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:02.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:14:01 smithi173 bash[27612]: audit 2023-10-01T10:14:01.228920+0000 mon.c (mon.2) 20 : audit [DBG] from='client.? 172.21.15.78:0/2810962441' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-01T10:14:03.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:14:03 smithi078 bash[27336]: audit 2023-10-01T10:14:02.395837+0000 mon.a (mon.0) 615 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:14:03.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:14:03 smithi078 bash[33821]: audit 2023-10-01T10:14:02.395837+0000 mon.a (mon.0) 615 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:14:03.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:14:03 smithi173 bash[27612]: audit 2023-10-01T10:14:02.395837+0000 mon.a (mon.0) 615 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:14:04.404 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:Oct 01 10:14:04 smithi078 bash[53177]: level=info ts=2023-10-01T10:14:04.140Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.004531773s 2023-10-01T10:14:04.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:14:04 smithi078 bash[27336]: cluster 2023-10-01T10:14:03.164113+0000 mgr.y (mgr.14427) 40 : cluster [DBG] pgmap v30: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:04.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:14:04 smithi078 bash[33821]: cluster 2023-10-01T10:14:03.164113+0000 mgr.y (mgr.14427) 40 : cluster [DBG] pgmap v30: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:04.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:14:04 smithi173 bash[27612]: cluster 2023-10-01T10:14:03.164113+0000 mgr.y (mgr.14427) 40 : cluster [DBG] pgmap v30: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:05.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:14:05 smithi078 bash[27601]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:14:05] "GET /metrics HTTP/1.1" 200 155655 "" "Prometheus/2.33.4" 2023-10-01T10:14:05.620 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:14:05 smithi173 bash[28473]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:14:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T10:14:06.751 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:14:06.783 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:14:06 smithi078 bash[33821]: cluster 2023-10-01T10:14:05.164759+0000 mgr.y (mgr.14427) 41 : cluster [DBG] pgmap v31: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:06.783 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:14:06 smithi078 bash[27336]: cluster 2023-10-01T10:14:05.164759+0000 mgr.y (mgr.14427) 41 : cluster [DBG] pgmap v31: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:06.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:14:06 smithi173 bash[27612]: cluster 2023-10-01T10:14:05.164759+0000 mgr.y (mgr.14427) 41 : cluster [DBG] pgmap v31: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:08.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:14:08 smithi078 bash[27336]: cluster 2023-10-01T10:14:07.165392+0000 mgr.y (mgr.14427) 42 : cluster [DBG] pgmap v32: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:08.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:14:08 smithi078 bash[33821]: cluster 2023-10-01T10:14:07.165392+0000 mgr.y (mgr.14427) 42 : cluster [DBG] pgmap v32: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:08.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:14:08 smithi173 bash[27612]: cluster 2023-10-01T10:14:07.165392+0000 mgr.y (mgr.14427) 42 : cluster [DBG] pgmap v32: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:09.782 INFO:teuthology.orchestra.run.smithi078.stdout: 2023-10-01T10:14:09.782 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":55,"fsid":"d8016480-6041-11ee-8db5-212e2dc638e7","created":"2023-10-01T10:04:59.345633+0000","modified":"2023-10-01T10:13:51.216859+0000","last_up_change":"2023-10-01T10:12:45.118874+0000","last_in_change":"2023-10-01T10:12:19.634793+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-10-01T10:09:16.689172+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-01T10:13:43.595790+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"3cdf50f0-cae3-4e38-a7ca-c24c4a07b615","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6802","nonce":575557942},{"type":"v1","addr":"172.21.15.78:6803","nonce":575557942}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6804","nonce":575557942},{"type":"v1","addr":"172.21.15.78:6805","nonce":575557942}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6808","nonce":575557942},{"type":"v1","addr":"172.21.15.78:6809","nonce":575557942}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6806","nonce":575557942},{"type":"v1","addr":"172.21.15.78:6807","nonce":575557942}]},"public_addr":"172.21.15.78:6803/575557942","cluster_addr":"172.21.15.78:6805/575557942","heartbeat_back_addr":"172.21.15.78:6809/575557942","heartbeat_front_addr":"172.21.15.78:6807/575557942","state":["exists","up"]},{"osd":1,"uuid":"4e6bea65-4109-43d2-b5be-3811fdd2a803","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6810","nonce":3462774695},{"type":"v1","addr":"172.21.15.78:6811","nonce":3462774695}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6812","nonce":3462774695},{"type":"v1","addr":"172.21.15.78:6813","nonce":3462774695}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6816","nonce":3462774695},{"type":"v1","addr":"172.21.15.78:6817","nonce":3462774695}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6814","nonce":3462774695},{"type":"v1","addr":"172.21.15.78:6815","nonce":3462774695}]},"public_addr":"172.21.15.78:6811/3462774695","cluster_addr":"172.21.15.78:6813/3462774695","heartbeat_back_addr":"172.21.15.78:6817/3462774695","heartbeat_front_addr":"172.21.15.78:6815/3462774695","state":["exists","up"]},{"osd":2,"uuid":"9f830594-31d6-4994-85fa-0615c280d5c3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6818","nonce":1190788353},{"type":"v1","addr":"172.21.15.78:6819","nonce":1190788353}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6820","nonce":1190788353},{"type":"v1","addr":"172.21.15.78:6821","nonce":1190788353}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6824","nonce":1190788353},{"type":"v1","addr":"172.21.15.78:6825","nonce":1190788353}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6822","nonce":1190788353},{"type":"v1","addr":"172.21.15.78:6823","nonce":1190788353}]},"public_addr":"172.21.15.78:6819/1190788353","cluster_addr":"172.21.15.78:6821/1190788353","heartbeat_back_addr":"172.21.15.78:6825/1190788353","heartbeat_front_addr":"172.21.15.78:6823/1190788353","state":["exists","up"]},{"osd":3,"uuid":"2780842b-2f4f-4190-b4a5-279a0444adf2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6826","nonce":53937366},{"type":"v1","addr":"172.21.15.78:6827","nonce":53937366}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6828","nonce":53937366},{"type":"v1","addr":"172.21.15.78:6829","nonce":53937366}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6832","nonce":53937366},{"type":"v1","addr":"172.21.15.78:6833","nonce":53937366}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6830","nonce":53937366},{"type":"v1","addr":"172.21.15.78:6831","nonce":53937366}]},"public_addr":"172.21.15.78:6827/53937366","cluster_addr":"172.21.15.78:6829/53937366","heartbeat_back_addr":"172.21.15.78:6833/53937366","heartbeat_front_addr":"172.21.15.78:6831/53937366","state":["exists","up"]},{"osd":4,"uuid":"57c30778-fd7e-4f3e-afd9-389ef377cb62","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6800","nonce":2430591286},{"type":"v1","addr":"172.21.15.173:6801","nonce":2430591286}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6802","nonce":2430591286},{"type":"v1","addr":"172.21.15.173:6803","nonce":2430591286}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6806","nonce":2430591286},{"type":"v1","addr":"172.21.15.173:6807","nonce":2430591286}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6804","nonce":2430591286},{"type":"v1","addr":"172.21.15.173:6805","nonce":2430591286}]},"public_addr":"172.21.15.173:6801/2430591286","cluster_addr":"172.21.15.173:6803/2430591286","heartbeat_back_addr":"172.21.15.173:6807/2430591286","heartbeat_front_addr":"172.21.15.173:6805/2430591286","state":["exists","up"]},{"osd":5,"uuid":"c62aa7c3-2d09-4088-ba36-c84588dc3549","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6808","nonce":2798643181},{"type":"v1","addr":"172.21.15.173:6809","nonce":2798643181}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6810","nonce":2798643181},{"type":"v1","addr":"172.21.15.173:6811","nonce":2798643181}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6814","nonce":2798643181},{"type":"v1","addr":"172.21.15.173:6815","nonce":2798643181}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6812","nonce":2798643181},{"type":"v1","addr":"172.21.15.173:6813","nonce":2798643181}]},"public_addr":"172.21.15.173:6809/2798643181","cluster_addr":"172.21.15.173:6811/2798643181","heartbeat_back_addr":"172.21.15.173:6815/2798643181","heartbeat_front_addr":"172.21.15.173:6813/2798643181","state":["exists","up"]},{"osd":6,"uuid":"b140330d-dd93-4ce4-9b9a-2ec6321419b3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6816","nonce":1568190351},{"type":"v1","addr":"172.21.15.173:6817","nonce":1568190351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6818","nonce":1568190351},{"type":"v1","addr":"172.21.15.173:6819","nonce":1568190351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6822","nonce":1568190351},{"type":"v1","addr":"172.21.15.173:6823","nonce":1568190351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6820","nonce":1568190351},{"type":"v1","addr":"172.21.15.173:6821","nonce":1568190351}]},"public_addr":"172.21.15.173:6817/1568190351","cluster_addr":"172.21.15.173:6819/1568190351","heartbeat_back_addr":"172.21.15.173:6823/1568190351","heartbeat_front_addr":"172.21.15.173:6821/1568190351","state":["exists","up"]},{"osd":7,"uuid":"7590027f-50c4-4457-843d-cebaa734d2ea","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6824","nonce":1958390745},{"type":"v1","addr":"172.21.15.173:6825","nonce":1958390745}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6826","nonce":1958390745},{"type":"v1","addr":"172.21.15.173:6827","nonce":1958390745}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6830","nonce":1958390745},{"type":"v1","addr":"172.21.15.173:6831","nonce":1958390745}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6828","nonce":1958390745},{"type":"v1","addr":"172.21.15.173:6829","nonce":1958390745}]},"public_addr":"172.21.15.173:6825/1958390745","cluster_addr":"172.21.15.173:6827/1958390745","heartbeat_back_addr":"172.21.15.173:6831/1958390745","heartbeat_front_addr":"172.21.15.173:6829/1958390745","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:07:47.950416+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:08:31.368397+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:09:14.866866+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:09:59.445542+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:10:39.550033+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:11:20.934732+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:12:01.654044+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:12:43.447412+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.78:0/2359078223":"2023-10-02T10:13:17.135564+0000","172.21.15.78:0/3939471097":"2023-10-02T10:05:57.727322+0000","172.21.15.78:6800/1224621968":"2023-10-02T10:13:17.135564+0000","172.21.15.78:6800/1700562157":"2023-10-02T10:05:57.727322+0000","172.21.15.78:0/3635389419":"2023-10-02T10:05:57.727322+0000","172.21.15.78:0/3655343641":"2023-10-02T10:13:17.135564+0000","172.21.15.78:6800/3027286102":"2023-10-02T10:05:27.007550+0000","172.21.15.78:6801/3027286102":"2023-10-02T10:05:27.007550+0000","172.21.15.78:6801/1224621968":"2023-10-02T10:13:17.135564+0000","172.21.15.78:0/308254153":"2023-10-02T10:05:27.007550+0000","172.21.15.78:6801/1700562157":"2023-10-02T10:05:57.727322+0000","172.21.15.78:0/158328292":"2023-10-02T10:05:27.007550+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-01T10:14:10.514 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-10-01T10:14:10.514 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-01T10:14:10.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:14:10 smithi078 bash[27336]: cluster 2023-10-01T10:14:09.166162+0000 mgr.y (mgr.14427) 43 : cluster [DBG] pgmap v33: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:10.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:14:10 smithi078 bash[27336]: audit 2023-10-01T10:14:09.778259+0000 mon.c (mon.2) 21 : audit [DBG] from='client.? 172.21.15.78:0/4260570829' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T10:14:10.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:14:10 smithi078 bash[33821]: cluster 2023-10-01T10:14:09.166162+0000 mgr.y (mgr.14427) 43 : cluster [DBG] pgmap v33: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:10.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:14:10 smithi078 bash[33821]: audit 2023-10-01T10:14:09.778259+0000 mon.c (mon.2) 21 : audit [DBG] from='client.? 172.21.15.78:0/4260570829' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T10:14:10.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:14:10 smithi173 bash[27612]: cluster 2023-10-01T10:14:09.166162+0000 mgr.y (mgr.14427) 43 : cluster [DBG] pgmap v33: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:10.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:14:10 smithi173 bash[27612]: audit 2023-10-01T10:14:09.778259+0000 mon.c (mon.2) 21 : audit [DBG] from='client.? 172.21.15.78:0/4260570829' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T10:14:12.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:14:11 smithi078 bash[27336]: cluster 2023-10-01T10:14:11.166893+0000 mgr.y (mgr.14427) 44 : cluster [DBG] pgmap v34: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:12.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:14:11 smithi078 bash[33821]: cluster 2023-10-01T10:14:11.166893+0000 mgr.y (mgr.14427) 44 : cluster [DBG] pgmap v34: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:12.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:14:11 smithi173 bash[27612]: cluster 2023-10-01T10:14:11.166893+0000 mgr.y (mgr.14427) 44 : cluster [DBG] pgmap v34: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:14.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:14:14 smithi078 bash[27336]: cluster 2023-10-01T10:14:13.167635+0000 mgr.y (mgr.14427) 45 : cluster [DBG] pgmap v35: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:14.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:14:14 smithi078 bash[33821]: cluster 2023-10-01T10:14:13.167635+0000 mgr.y (mgr.14427) 45 : cluster [DBG] pgmap v35: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:14.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:14:14 smithi173 bash[27612]: cluster 2023-10-01T10:14:13.167635+0000 mgr.y (mgr.14427) 45 : cluster [DBG] pgmap v35: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:15.279 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:14:15.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:14:15 smithi078 bash[27601]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:14:15] "GET /metrics HTTP/1.1" 200 155655 "" "Prometheus/2.33.4" 2023-10-01T10:14:15.620 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:14:15 smithi173 bash[28473]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:14:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T10:14:16.522 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:14:16 smithi078 bash[33821]: cluster 2023-10-01T10:14:15.168401+0000 mgr.y (mgr.14427) 46 : cluster [DBG] pgmap v36: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:16.523 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:14:16 smithi078 bash[27336]: cluster 2023-10-01T10:14:15.168401+0000 mgr.y (mgr.14427) 46 : cluster [DBG] pgmap v36: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:16.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:14:16 smithi173 bash[27612]: cluster 2023-10-01T10:14:15.168401+0000 mgr.y (mgr.14427) 46 : cluster [DBG] pgmap v36: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:18.274 INFO:teuthology.orchestra.run.smithi078.stdout: 2023-10-01T10:14:18.274 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":55,"fsid":"d8016480-6041-11ee-8db5-212e2dc638e7","created":"2023-10-01T10:04:59.345633+0000","modified":"2023-10-01T10:13:51.216859+0000","last_up_change":"2023-10-01T10:12:45.118874+0000","last_in_change":"2023-10-01T10:12:19.634793+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-10-01T10:09:16.689172+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-01T10:13:43.595790+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"3cdf50f0-cae3-4e38-a7ca-c24c4a07b615","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6802","nonce":575557942},{"type":"v1","addr":"172.21.15.78:6803","nonce":575557942}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6804","nonce":575557942},{"type":"v1","addr":"172.21.15.78:6805","nonce":575557942}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6808","nonce":575557942},{"type":"v1","addr":"172.21.15.78:6809","nonce":575557942}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6806","nonce":575557942},{"type":"v1","addr":"172.21.15.78:6807","nonce":575557942}]},"public_addr":"172.21.15.78:6803/575557942","cluster_addr":"172.21.15.78:6805/575557942","heartbeat_back_addr":"172.21.15.78:6809/575557942","heartbeat_front_addr":"172.21.15.78:6807/575557942","state":["exists","up"]},{"osd":1,"uuid":"4e6bea65-4109-43d2-b5be-3811fdd2a803","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6810","nonce":3462774695},{"type":"v1","addr":"172.21.15.78:6811","nonce":3462774695}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6812","nonce":3462774695},{"type":"v1","addr":"172.21.15.78:6813","nonce":3462774695}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6816","nonce":3462774695},{"type":"v1","addr":"172.21.15.78:6817","nonce":3462774695}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6814","nonce":3462774695},{"type":"v1","addr":"172.21.15.78:6815","nonce":3462774695}]},"public_addr":"172.21.15.78:6811/3462774695","cluster_addr":"172.21.15.78:6813/3462774695","heartbeat_back_addr":"172.21.15.78:6817/3462774695","heartbeat_front_addr":"172.21.15.78:6815/3462774695","state":["exists","up"]},{"osd":2,"uuid":"9f830594-31d6-4994-85fa-0615c280d5c3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6818","nonce":1190788353},{"type":"v1","addr":"172.21.15.78:6819","nonce":1190788353}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6820","nonce":1190788353},{"type":"v1","addr":"172.21.15.78:6821","nonce":1190788353}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6824","nonce":1190788353},{"type":"v1","addr":"172.21.15.78:6825","nonce":1190788353}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6822","nonce":1190788353},{"type":"v1","addr":"172.21.15.78:6823","nonce":1190788353}]},"public_addr":"172.21.15.78:6819/1190788353","cluster_addr":"172.21.15.78:6821/1190788353","heartbeat_back_addr":"172.21.15.78:6825/1190788353","heartbeat_front_addr":"172.21.15.78:6823/1190788353","state":["exists","up"]},{"osd":3,"uuid":"2780842b-2f4f-4190-b4a5-279a0444adf2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6826","nonce":53937366},{"type":"v1","addr":"172.21.15.78:6827","nonce":53937366}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6828","nonce":53937366},{"type":"v1","addr":"172.21.15.78:6829","nonce":53937366}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6832","nonce":53937366},{"type":"v1","addr":"172.21.15.78:6833","nonce":53937366}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6830","nonce":53937366},{"type":"v1","addr":"172.21.15.78:6831","nonce":53937366}]},"public_addr":"172.21.15.78:6827/53937366","cluster_addr":"172.21.15.78:6829/53937366","heartbeat_back_addr":"172.21.15.78:6833/53937366","heartbeat_front_addr":"172.21.15.78:6831/53937366","state":["exists","up"]},{"osd":4,"uuid":"57c30778-fd7e-4f3e-afd9-389ef377cb62","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6800","nonce":2430591286},{"type":"v1","addr":"172.21.15.173:6801","nonce":2430591286}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6802","nonce":2430591286},{"type":"v1","addr":"172.21.15.173:6803","nonce":2430591286}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6806","nonce":2430591286},{"type":"v1","addr":"172.21.15.173:6807","nonce":2430591286}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6804","nonce":2430591286},{"type":"v1","addr":"172.21.15.173:6805","nonce":2430591286}]},"public_addr":"172.21.15.173:6801/2430591286","cluster_addr":"172.21.15.173:6803/2430591286","heartbeat_back_addr":"172.21.15.173:6807/2430591286","heartbeat_front_addr":"172.21.15.173:6805/2430591286","state":["exists","up"]},{"osd":5,"uuid":"c62aa7c3-2d09-4088-ba36-c84588dc3549","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6808","nonce":2798643181},{"type":"v1","addr":"172.21.15.173:6809","nonce":2798643181}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6810","nonce":2798643181},{"type":"v1","addr":"172.21.15.173:6811","nonce":2798643181}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6814","nonce":2798643181},{"type":"v1","addr":"172.21.15.173:6815","nonce":2798643181}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6812","nonce":2798643181},{"type":"v1","addr":"172.21.15.173:6813","nonce":2798643181}]},"public_addr":"172.21.15.173:6809/2798643181","cluster_addr":"172.21.15.173:6811/2798643181","heartbeat_back_addr":"172.21.15.173:6815/2798643181","heartbeat_front_addr":"172.21.15.173:6813/2798643181","state":["exists","up"]},{"osd":6,"uuid":"b140330d-dd93-4ce4-9b9a-2ec6321419b3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6816","nonce":1568190351},{"type":"v1","addr":"172.21.15.173:6817","nonce":1568190351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6818","nonce":1568190351},{"type":"v1","addr":"172.21.15.173:6819","nonce":1568190351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6822","nonce":1568190351},{"type":"v1","addr":"172.21.15.173:6823","nonce":1568190351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6820","nonce":1568190351},{"type":"v1","addr":"172.21.15.173:6821","nonce":1568190351}]},"public_addr":"172.21.15.173:6817/1568190351","cluster_addr":"172.21.15.173:6819/1568190351","heartbeat_back_addr":"172.21.15.173:6823/1568190351","heartbeat_front_addr":"172.21.15.173:6821/1568190351","state":["exists","up"]},{"osd":7,"uuid":"7590027f-50c4-4457-843d-cebaa734d2ea","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6824","nonce":1958390745},{"type":"v1","addr":"172.21.15.173:6825","nonce":1958390745}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6826","nonce":1958390745},{"type":"v1","addr":"172.21.15.173:6827","nonce":1958390745}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6830","nonce":1958390745},{"type":"v1","addr":"172.21.15.173:6831","nonce":1958390745}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6828","nonce":1958390745},{"type":"v1","addr":"172.21.15.173:6829","nonce":1958390745}]},"public_addr":"172.21.15.173:6825/1958390745","cluster_addr":"172.21.15.173:6827/1958390745","heartbeat_back_addr":"172.21.15.173:6831/1958390745","heartbeat_front_addr":"172.21.15.173:6829/1958390745","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:07:47.950416+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:08:31.368397+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:09:14.866866+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:09:59.445542+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:10:39.550033+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:11:20.934732+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:12:01.654044+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:12:43.447412+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.78:0/2359078223":"2023-10-02T10:13:17.135564+0000","172.21.15.78:0/3939471097":"2023-10-02T10:05:57.727322+0000","172.21.15.78:6800/1224621968":"2023-10-02T10:13:17.135564+0000","172.21.15.78:6800/1700562157":"2023-10-02T10:05:57.727322+0000","172.21.15.78:0/3635389419":"2023-10-02T10:05:57.727322+0000","172.21.15.78:0/3655343641":"2023-10-02T10:13:17.135564+0000","172.21.15.78:6800/3027286102":"2023-10-02T10:05:27.007550+0000","172.21.15.78:6801/3027286102":"2023-10-02T10:05:27.007550+0000","172.21.15.78:6801/1224621968":"2023-10-02T10:13:17.135564+0000","172.21.15.78:0/308254153":"2023-10-02T10:05:27.007550+0000","172.21.15.78:6801/1700562157":"2023-10-02T10:05:57.727322+0000","172.21.15.78:0/158328292":"2023-10-02T10:05:27.007550+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-01T10:14:18.521 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:14:18 smithi078 bash[33821]: cluster 2023-10-01T10:14:17.168961+0000 mgr.y (mgr.14427) 47 : cluster [DBG] pgmap v37: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:18.522 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:14:18 smithi078 bash[27336]: cluster 2023-10-01T10:14:17.168961+0000 mgr.y (mgr.14427) 47 : cluster [DBG] pgmap v37: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:18.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:14:18 smithi173 bash[27612]: cluster 2023-10-01T10:14:17.168961+0000 mgr.y (mgr.14427) 47 : cluster [DBG] pgmap v37: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:19.052 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-10-01T10:14:19.052 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-10-01T10:14:19.053 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-10-01T10:14:19.053 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-10-01T10:14:19.054 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-10-01T10:14:19.054 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-10-01T10:14:19.055 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-10-01T10:14:19.055 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-10-01T10:14:19.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:14:19 smithi078 bash[33821]: audit 2023-10-01T10:14:18.269532+0000 mon.a (mon.0) 616 : audit [DBG] from='client.? 172.21.15.78:0/251387637' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T10:14:19.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:14:19 smithi078 bash[27336]: audit 2023-10-01T10:14:18.269532+0000 mon.a (mon.0) 616 : audit [DBG] from='client.? 172.21.15.78:0/251387637' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T10:14:19.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:14:19 smithi173 bash[27612]: audit 2023-10-01T10:14:18.269532+0000 mon.a (mon.0) 616 : audit [DBG] from='client.? 172.21.15.78:0/251387637' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T10:14:20.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:14:20 smithi078 bash[27336]: cluster 2023-10-01T10:14:19.169481+0000 mgr.y (mgr.14427) 48 : cluster [DBG] pgmap v38: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:20.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:14:20 smithi078 bash[33821]: cluster 2023-10-01T10:14:19.169481+0000 mgr.y (mgr.14427) 48 : cluster [DBG] pgmap v38: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:20.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:14:20 smithi173 bash[27612]: cluster 2023-10-01T10:14:19.169481+0000 mgr.y (mgr.14427) 48 : cluster [DBG] pgmap v38: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:22.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:14:21 smithi173 bash[27612]: cluster 2023-10-01T10:14:21.170059+0000 mgr.y (mgr.14427) 49 : cluster [DBG] pgmap v39: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:22.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:14:21 smithi078 bash[27336]: cluster 2023-10-01T10:14:21.170059+0000 mgr.y (mgr.14427) 49 : cluster [DBG] pgmap v39: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:22.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:14:21 smithi078 bash[33821]: cluster 2023-10-01T10:14:21.170059+0000 mgr.y (mgr.14427) 49 : cluster [DBG] pgmap v39: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:23.963 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:14:23.966 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:14:23.967 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:14:23.973 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:14:23.974 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:14:23.974 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:14:23.974 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:14:23.975 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:14:24.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:14:24 smithi078 bash[27336]: cluster 2023-10-01T10:14:23.170790+0000 mgr.y (mgr.14427) 50 : cluster [DBG] pgmap v40: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:24.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:14:24 smithi078 bash[33821]: cluster 2023-10-01T10:14:23.170790+0000 mgr.y (mgr.14427) 50 : cluster [DBG] pgmap v40: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:24.397 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:24 smithi173 systemd[1]: Started Ceph grafana.a for d8016480-6041-11ee-8db5-212e2dc638e7. 2023-10-01T10:14:24.397 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:14:24 smithi173 bash[27612]: cluster 2023-10-01T10:14:23.170790+0000 mgr.y (mgr.14427) 50 : cluster [DBG] pgmap v40: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:25.383 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:14:25 smithi078 bash[27601]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:14:25] "GET /metrics HTTP/1.1" 200 155661 "" "Prometheus/2.33.4" 2023-10-01T10:14:25.450 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:14:25 smithi173 bash[28473]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:14:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T10:14:25.451 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:14:25 smithi173 bash[27612]: audit 2023-10-01T10:14:24.373117+0000 mon.a (mon.0) 617 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:14:25.451 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:14:25 smithi173 bash[27612]: audit 2023-10-01T10:14:24.374983+0000 mon.a (mon.0) 618 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:14:25.451 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:25 smithi173 bash[47537]: t=2023-10-01T10:14:25+0000 lvl=info msg="The state of unified alerting is still not defined. The decision will be made during as we run the database migrations" logger=settings 2023-10-01T10:14:25.451 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:25 smithi173 bash[47537]: t=2023-10-01T10:14:25+0000 lvl=warn msg="falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled." logger=settings 2023-10-01T10:14:25.452 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:25 smithi173 bash[47537]: t=2023-10-01T10:14:25+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2023-10-01T10:14:25.452 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:25 smithi173 bash[47537]: t=2023-10-01T10:14:25+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2023-10-01T10:14:25.452 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:25 smithi173 bash[47537]: t=2023-10-01T10:14:25+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2023-10-01T10:14:25.452 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:25 smithi173 bash[47537]: t=2023-10-01T10:14:25+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2023-10-01T10:14:25.452 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:25 smithi173 bash[47537]: t=2023-10-01T10:14:25+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2023-10-01T10:14:25.452 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:25 smithi173 bash[47537]: t=2023-10-01T10:14:25+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2023-10-01T10:14:25.452 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:25 smithi173 bash[47537]: t=2023-10-01T10:14:25+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2023-10-01T10:14:25.452 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:25 smithi173 bash[47537]: t=2023-10-01T10:14:25+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2023-10-01T10:14:25.452 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:25 smithi173 bash[47537]: t=2023-10-01T10:14:25+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2023-10-01T10:14:25.452 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:25 smithi173 bash[47537]: t=2023-10-01T10:14:25+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2023-10-01T10:14:25.452 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:25 smithi173 bash[47537]: t=2023-10-01T10:14:25+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2023-10-01T10:14:25.453 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:25 smithi173 bash[47537]: t=2023-10-01T10:14:25+0000 lvl=info msg="App mode production" logger=settings 2023-10-01T10:14:25.453 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:25 smithi173 bash[47537]: t=2023-10-01T10:14:25+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2023-10-01T10:14:25.453 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:25 smithi173 bash[47537]: t=2023-10-01T10:14:25+0000 lvl=warn msg="SQLite database file has broader permissions than it should" logger=sqlstore path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2023-10-01T10:14:25.453 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:25 smithi173 bash[47537]: t=2023-10-01T10:14:25+0000 lvl=info msg="Starting DB migrations" logger=migrator 2023-10-01T10:14:25.453 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:25 smithi173 bash[47537]: t=2023-10-01T10:14:25+0000 lvl=info msg="Executing migration" logger=migrator id="create migration_log table" 2023-10-01T10:14:25.453 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:25 smithi173 bash[47537]: t=2023-10-01T10:14:25+0000 lvl=info msg="Executing migration" logger=migrator id="create user table" 2023-10-01T10:14:25.453 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:25 smithi173 bash[47537]: t=2023-10-01T10:14:25+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.login" 2023-10-01T10:14:25.453 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:25 smithi173 bash[47537]: t=2023-10-01T10:14:25+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.email" 2023-10-01T10:14:25.712 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:25 smithi173 bash[47537]: t=2023-10-01T10:14:25+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_login - v1" 2023-10-01T10:14:25.712 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:25 smithi173 bash[47537]: t=2023-10-01T10:14:25+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_email - v1" 2023-10-01T10:14:25.712 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:25 smithi173 bash[47537]: t=2023-10-01T10:14:25+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table user to user_v1 - v1" 2023-10-01T10:14:25.712 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:25 smithi173 bash[47537]: t=2023-10-01T10:14:25+0000 lvl=info msg="Executing migration" logger=migrator id="create user table v2" 2023-10-01T10:14:25.712 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:25 smithi173 bash[47537]: t=2023-10-01T10:14:25+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_login - v2" 2023-10-01T10:14:25.712 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:25 smithi173 bash[47537]: t=2023-10-01T10:14:25+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_email - v2" 2023-10-01T10:14:25.713 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:25 smithi173 bash[47537]: t=2023-10-01T10:14:25+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2023-10-01T10:14:25.713 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:25 smithi173 bash[47537]: t=2023-10-01T10:14:25+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table user_v1" 2023-10-01T10:14:25.713 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:25 smithi173 bash[47537]: t=2023-10-01T10:14:25+0000 lvl=info msg="Executing migration" logger=migrator id="Add column help_flags1 to user table" 2023-10-01T10:14:25.745 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:14:25 smithi078 bash[33821]: audit 2023-10-01T10:14:24.373117+0000 mon.a (mon.0) 617 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:14:25.745 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:14:25 smithi078 bash[33821]: audit 2023-10-01T10:14:24.374983+0000 mon.a (mon.0) 618 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:14:25.746 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:14:25 smithi078 bash[27336]: audit 2023-10-01T10:14:24.373117+0000 mon.a (mon.0) 617 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:14:25.746 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:14:25 smithi078 bash[27336]: audit 2023-10-01T10:14:24.374983+0000 mon.a (mon.0) 618 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:14:25.981 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:25 smithi173 bash[47537]: t=2023-10-01T10:14:25+0000 lvl=info msg="Executing migration" logger=migrator id="Update user table charset" 2023-10-01T10:14:25.981 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:25 smithi173 bash[47537]: t=2023-10-01T10:14:25+0000 lvl=info msg="Executing migration" logger=migrator id="Add last_seen_at column to user" 2023-10-01T10:14:25.981 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:25 smithi173 bash[47537]: t=2023-10-01T10:14:25+0000 lvl=info msg="Executing migration" logger=migrator id="Add missing user data" 2023-10-01T10:14:25.981 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:25 smithi173 bash[47537]: t=2023-10-01T10:14:25+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_disabled column to user" 2023-10-01T10:14:25.982 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:25 smithi173 bash[47537]: t=2023-10-01T10:14:25+0000 lvl=info msg="Executing migration" logger=migrator id="Add index user.login/user.email" 2023-10-01T10:14:25.982 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:25 smithi173 bash[47537]: t=2023-10-01T10:14:25+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_service_account column to user" 2023-10-01T10:14:25.982 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:25 smithi173 bash[47537]: t=2023-10-01T10:14:25+0000 lvl=info msg="Executing migration" logger=migrator id="create temp user table v1-7" 2023-10-01T10:14:26.251 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:26 smithi173 bash[47537]: t=2023-10-01T10:14:26+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v1-7" 2023-10-01T10:14:26.251 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:26 smithi173 bash[47537]: t=2023-10-01T10:14:26+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v1-7" 2023-10-01T10:14:26.251 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:26 smithi173 bash[47537]: t=2023-10-01T10:14:26+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v1-7" 2023-10-01T10:14:26.251 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:26 smithi173 bash[47537]: t=2023-10-01T10:14:26+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v1-7" 2023-10-01T10:14:26.251 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:26 smithi173 bash[47537]: t=2023-10-01T10:14:26+0000 lvl=info msg="Executing migration" logger=migrator id="Update temp_user table charset" 2023-10-01T10:14:26.251 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:26 smithi173 bash[47537]: t=2023-10-01T10:14:26+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_email - v1" 2023-10-01T10:14:26.251 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:26 smithi173 bash[47537]: t=2023-10-01T10:14:26+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_org_id - v1" 2023-10-01T10:14:26.252 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:26 smithi173 bash[47537]: t=2023-10-01T10:14:26+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_code - v1" 2023-10-01T10:14:26.252 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:26 smithi173 bash[47537]: t=2023-10-01T10:14:26+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_status - v1" 2023-10-01T10:14:26.252 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:26 smithi173 bash[47537]: t=2023-10-01T10:14:26+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2023-10-01T10:14:26.458 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:14:26 smithi078 bash[33821]: cluster 2023-10-01T10:14:25.171293+0000 mgr.y (mgr.14427) 51 : cluster [DBG] pgmap v41: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:26.458 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:14:26 smithi078 bash[27336]: cluster 2023-10-01T10:14:25.171293+0000 mgr.y (mgr.14427) 51 : cluster [DBG] pgmap v41: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:26.520 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:14:26 smithi173 bash[27612]: cluster 2023-10-01T10:14:25.171293+0000 mgr.y (mgr.14427) 51 : cluster [DBG] pgmap v41: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:26.520 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:26 smithi173 bash[47537]: t=2023-10-01T10:14:26+0000 lvl=info msg="Executing migration" logger=migrator id="create temp_user v2" 2023-10-01T10:14:26.521 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:26 smithi173 bash[47537]: t=2023-10-01T10:14:26+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v2" 2023-10-01T10:14:26.521 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:26 smithi173 bash[47537]: t=2023-10-01T10:14:26+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v2" 2023-10-01T10:14:26.521 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:26 smithi173 bash[47537]: t=2023-10-01T10:14:26+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v2" 2023-10-01T10:14:26.521 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:26 smithi173 bash[47537]: t=2023-10-01T10:14:26+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v2" 2023-10-01T10:14:26.521 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:26 smithi173 bash[47537]: t=2023-10-01T10:14:26+0000 lvl=info msg="Executing migration" logger=migrator id="copy temp_user v1 to v2" 2023-10-01T10:14:26.521 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:26 smithi173 bash[47537]: t=2023-10-01T10:14:26+0000 lvl=info msg="Executing migration" logger=migrator id="drop temp_user_tmp_qwerty" 2023-10-01T10:14:26.790 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:26 smithi173 bash[47537]: t=2023-10-01T10:14:26+0000 lvl=info msg="Executing migration" logger=migrator id="Set created for temp users that will otherwise prematurely expire" 2023-10-01T10:14:26.790 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:26 smithi173 bash[47537]: t=2023-10-01T10:14:26+0000 lvl=info msg="Executing migration" logger=migrator id="create star table" 2023-10-01T10:14:26.790 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:26 smithi173 bash[47537]: t=2023-10-01T10:14:26+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index star.user_id_dashboard_id" 2023-10-01T10:14:26.790 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:26 smithi173 bash[47537]: t=2023-10-01T10:14:26+0000 lvl=info msg="Executing migration" logger=migrator id="create org table v1" 2023-10-01T10:14:26.791 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:26 smithi173 bash[47537]: t=2023-10-01T10:14:26+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_name - v1" 2023-10-01T10:14:26.791 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:26 smithi173 bash[47537]: t=2023-10-01T10:14:26+0000 lvl=info msg="Executing migration" logger=migrator id="create org_user table v1" 2023-10-01T10:14:26.791 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:26 smithi173 bash[47537]: t=2023-10-01T10:14:26+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_org_id - v1" 2023-10-01T10:14:26.791 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:26 smithi173 bash[47537]: t=2023-10-01T10:14:26+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_user_org_id_user_id - v1" 2023-10-01T10:14:26.791 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:26 smithi173 bash[47537]: t=2023-10-01T10:14:26+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_user_id - v1" 2023-10-01T10:14:27.051 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:26 smithi173 bash[47537]: t=2023-10-01T10:14:26+0000 lvl=info msg="Executing migration" logger=migrator id="Update org table charset" 2023-10-01T10:14:27.051 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:26 smithi173 bash[47537]: t=2023-10-01T10:14:26+0000 lvl=info msg="Executing migration" logger=migrator id="Update org_user table charset" 2023-10-01T10:14:27.051 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:26 smithi173 bash[47537]: t=2023-10-01T10:14:26+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate all Read Only Viewers to Viewers" 2023-10-01T10:14:27.051 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:26 smithi173 bash[47537]: t=2023-10-01T10:14:26+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard table" 2023-10-01T10:14:27.052 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:26 smithi173 bash[47537]: t=2023-10-01T10:14:26+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard.account_id" 2023-10-01T10:14:27.052 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:26 smithi173 bash[47537]: t=2023-10-01T10:14:26+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_account_id_slug" 2023-10-01T10:14:27.052 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:26 smithi173 bash[47537]: t=2023-10-01T10:14:26+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_tag table" 2023-10-01T10:14:27.052 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:27 smithi173 bash[47537]: t=2023-10-01T10:14:27+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_tag.dasboard_id_term" 2023-10-01T10:14:27.052 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:27 smithi173 bash[47537]: t=2023-10-01T10:14:27+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2023-10-01T10:14:27.312 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:27 smithi173 bash[47537]: t=2023-10-01T10:14:27+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard to dashboard_v1 - v1" 2023-10-01T10:14:27.313 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:27 smithi173 bash[47537]: t=2023-10-01T10:14:27+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard v2" 2023-10-01T10:14:27.313 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:27 smithi173 bash[47537]: t=2023-10-01T10:14:27+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_org_id - v2" 2023-10-01T10:14:27.313 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:27 smithi173 bash[47537]: t=2023-10-01T10:14:27+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_org_id_slug - v2" 2023-10-01T10:14:27.313 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:27 smithi173 bash[47537]: t=2023-10-01T10:14:27+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard v1 to v2" 2023-10-01T10:14:27.313 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:27 smithi173 bash[47537]: t=2023-10-01T10:14:27+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_v1" 2023-10-01T10:14:27.313 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:27 smithi173 bash[47537]: t=2023-10-01T10:14:27+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard.data to mediumtext v1" 2023-10-01T10:14:27.313 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:27 smithi173 bash[47537]: t=2023-10-01T10:14:27+0000 lvl=info msg="Executing migration" logger=migrator id="Add column updated_by in dashboard - v2" 2023-10-01T10:14:27.313 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:27 smithi173 bash[47537]: t=2023-10-01T10:14:27+0000 lvl=info msg="Executing migration" logger=migrator id="Add column created_by in dashboard - v2" 2023-10-01T10:14:27.565 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:27 smithi173 bash[47537]: t=2023-10-01T10:14:27+0000 lvl=info msg="Executing migration" logger=migrator id="Add column gnetId in dashboard" 2023-10-01T10:14:27.565 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:27 smithi173 bash[47537]: t=2023-10-01T10:14:27+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for gnetId in dashboard" 2023-10-01T10:14:27.565 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:27 smithi173 bash[47537]: t=2023-10-01T10:14:27+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_id in dashboard" 2023-10-01T10:14:27.566 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:27 smithi173 bash[47537]: t=2023-10-01T10:14:27+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for plugin_id in dashboard" 2023-10-01T10:14:27.566 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:27 smithi173 bash[47537]: t=2023-10-01T10:14:27+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_id in dashboard_tag" 2023-10-01T10:14:27.566 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:27 smithi173 bash[47537]: t=2023-10-01T10:14:27+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard table charset" 2023-10-01T10:14:27.566 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:27 smithi173 bash[47537]: t=2023-10-01T10:14:27+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_tag table charset" 2023-10-01T10:14:27.566 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:27 smithi173 bash[47537]: t=2023-10-01T10:14:27+0000 lvl=info msg="Executing migration" logger=migrator id="Add column folder_id in dashboard" 2023-10-01T10:14:27.818 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:27 smithi173 bash[47537]: t=2023-10-01T10:14:27+0000 lvl=info msg="Executing migration" logger=migrator id="Add column isFolder in dashboard" 2023-10-01T10:14:27.818 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:27 smithi173 bash[47537]: t=2023-10-01T10:14:27+0000 lvl=info msg="Executing migration" logger=migrator id="Add column has_acl in dashboard" 2023-10-01T10:14:27.818 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:27 smithi173 bash[47537]: t=2023-10-01T10:14:27+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in dashboard" 2023-10-01T10:14:27.818 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:27 smithi173 bash[47537]: t=2023-10-01T10:14:27+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in dashboard" 2023-10-01T10:14:27.819 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:27 smithi173 bash[47537]: t=2023-10-01T10:14:27+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index dashboard_org_id_uid" 2023-10-01T10:14:27.819 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:27 smithi173 bash[47537]: t=2023-10-01T10:14:27+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_slug" 2023-10-01T10:14:27.819 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:27 smithi173 bash[47537]: t=2023-10-01T10:14:27+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard title length" 2023-10-01T10:14:27.819 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:27 smithi173 bash[47537]: t=2023-10-01T10:14:27+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index for dashboard_org_id_title_folder_id" 2023-10-01T10:14:28.070 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:27 smithi173 bash[47537]: t=2023-10-01T10:14:27+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning" 2023-10-01T10:14:28.071 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:27 smithi173 bash[47537]: t=2023-10-01T10:14:27+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2023-10-01T10:14:28.071 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:27 smithi173 bash[47537]: t=2023-10-01T10:14:27+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning v2" 2023-10-01T10:14:28.071 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:27 smithi173 bash[47537]: t=2023-10-01T10:14:27+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2023-10-01T10:14:28.071 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:27 smithi173 bash[47537]: t=2023-10-01T10:14:27+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2023-10-01T10:14:28.071 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:28 smithi173 bash[47537]: t=2023-10-01T10:14:28+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard_provisioning v1 to v2" 2023-10-01T10:14:28.071 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:28 smithi173 bash[47537]: t=2023-10-01T10:14:28+0000 lvl=info msg="Executing migration" logger=migrator id="drop dashboard_provisioning_tmp_qwerty" 2023-10-01T10:14:28.349 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:28 smithi173 bash[47537]: t=2023-10-01T10:14:28+0000 lvl=info msg="Executing migration" logger=migrator id="Add check_sum column" 2023-10-01T10:14:28.349 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:28 smithi173 bash[47537]: t=2023-10-01T10:14:28+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_title" 2023-10-01T10:14:28.349 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:28 smithi173 bash[47537]: t=2023-10-01T10:14:28+0000 lvl=info msg="Executing migration" logger=migrator id="delete tags for deleted dashboards" 2023-10-01T10:14:28.349 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:28 smithi173 bash[47537]: t=2023-10-01T10:14:28+0000 lvl=info msg="Executing migration" logger=migrator id="delete stars for deleted dashboards" 2023-10-01T10:14:28.349 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:28 smithi173 bash[47537]: t=2023-10-01T10:14:28+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_is_folder" 2023-10-01T10:14:28.349 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:28 smithi173 bash[47537]: t=2023-10-01T10:14:28+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table" 2023-10-01T10:14:28.349 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:28 smithi173 bash[47537]: t=2023-10-01T10:14:28+0000 lvl=info msg="Executing migration" logger=migrator id="add index data_source.account_id" 2023-10-01T10:14:28.349 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:28 smithi173 bash[47537]: t=2023-10-01T10:14:28+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index data_source.account_id_name" 2023-10-01T10:14:28.350 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:28 smithi173 bash[47537]: t=2023-10-01T10:14:28+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_data_source_account_id - v1" 2023-10-01T10:14:28.350 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:28 smithi173 bash[47537]: t=2023-10-01T10:14:28+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_data_source_account_id_name - v1" 2023-10-01T10:14:28.610 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:14:28 smithi173 bash[27612]: cluster 2023-10-01T10:14:27.171863+0000 mgr.y (mgr.14427) 52 : cluster [DBG] pgmap v42: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:28.610 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:14:28 smithi173 bash[27612]: audit 2023-10-01T10:14:27.423144+0000 mon.a (mon.0) 619 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:14:28.610 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:28 smithi173 bash[47537]: t=2023-10-01T10:14:28+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table data_source to data_source_v1 - v1" 2023-10-01T10:14:28.611 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:28 smithi173 bash[47537]: t=2023-10-01T10:14:28+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table v2" 2023-10-01T10:14:28.611 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:28 smithi173 bash[47537]: t=2023-10-01T10:14:28+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_data_source_org_id - v2" 2023-10-01T10:14:28.611 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:28 smithi173 bash[47537]: t=2023-10-01T10:14:28+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_data_source_org_id_name - v2" 2023-10-01T10:14:28.611 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:28 smithi173 bash[47537]: t=2023-10-01T10:14:28+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2023-10-01T10:14:28.611 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:28 smithi173 bash[47537]: t=2023-10-01T10:14:28+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table data_source_v1 #2" 2023-10-01T10:14:28.611 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:28 smithi173 bash[47537]: t=2023-10-01T10:14:28+0000 lvl=info msg="Executing migration" logger=migrator id="Add column with_credentials" 2023-10-01T10:14:28.611 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:28 smithi173 bash[47537]: t=2023-10-01T10:14:28+0000 lvl=info msg="Executing migration" logger=migrator id="Add secure json data column" 2023-10-01T10:14:28.611 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:28 smithi173 bash[47537]: t=2023-10-01T10:14:28+0000 lvl=info msg="Executing migration" logger=migrator id="Update data_source table charset" 2023-10-01T10:14:28.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:14:28 smithi078 bash[27336]: cluster 2023-10-01T10:14:27.171863+0000 mgr.y (mgr.14427) 52 : cluster [DBG] pgmap v42: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:28.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:14:28 smithi078 bash[27336]: audit 2023-10-01T10:14:27.423144+0000 mon.a (mon.0) 619 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:14:28.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:14:28 smithi078 bash[33821]: cluster 2023-10-01T10:14:27.171863+0000 mgr.y (mgr.14427) 52 : cluster [DBG] pgmap v42: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:28.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:14:28 smithi078 bash[33821]: audit 2023-10-01T10:14:27.423144+0000 mon.a (mon.0) 619 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:14:28.871 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:28 smithi173 bash[47537]: t=2023-10-01T10:14:28+0000 lvl=info msg="Executing migration" logger=migrator id="Update initial version to 1" 2023-10-01T10:14:28.871 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:28 smithi173 bash[47537]: t=2023-10-01T10:14:28+0000 lvl=info msg="Executing migration" logger=migrator id="Add read_only data column" 2023-10-01T10:14:28.871 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:28 smithi173 bash[47537]: t=2023-10-01T10:14:28+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate logging ds to loki ds" 2023-10-01T10:14:28.871 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:28 smithi173 bash[47537]: t=2023-10-01T10:14:28+0000 lvl=info msg="Executing migration" logger=migrator id="Update json_data with nulls" 2023-10-01T10:14:28.871 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:28 smithi173 bash[47537]: t=2023-10-01T10:14:28+0000 lvl=info msg="Executing migration" logger=migrator id="Add uid column" 2023-10-01T10:14:28.871 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:28 smithi173 bash[47537]: t=2023-10-01T10:14:28+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid value" 2023-10-01T10:14:28.871 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:28 smithi173 bash[47537]: t=2023-10-01T10:14:28+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index datasource_org_id_uid" 2023-10-01T10:14:29.124 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:28 smithi173 bash[47537]: t=2023-10-01T10:14:28+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index datasource_org_id_is_default" 2023-10-01T10:14:29.124 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:28 smithi173 bash[47537]: t=2023-10-01T10:14:28+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table" 2023-10-01T10:14:29.124 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:28 smithi173 bash[47537]: t=2023-10-01T10:14:28+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id" 2023-10-01T10:14:29.124 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:28 smithi173 bash[47537]: t=2023-10-01T10:14:28+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.key" 2023-10-01T10:14:29.124 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:28 smithi173 bash[47537]: t=2023-10-01T10:14:28+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id_name" 2023-10-01T10:14:29.124 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:29 smithi173 bash[47537]: t=2023-10-01T10:14:29+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_api_key_account_id - v1" 2023-10-01T10:14:29.125 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:29 smithi173 bash[47537]: t=2023-10-01T10:14:29+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_key - v1" 2023-10-01T10:14:29.125 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:29 smithi173 bash[47537]: t=2023-10-01T10:14:29+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_account_id_name - v1" 2023-10-01T10:14:29.376 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:29 smithi173 bash[47537]: t=2023-10-01T10:14:29+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table api_key to api_key_v1 - v1" 2023-10-01T10:14:29.376 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:29 smithi173 bash[47537]: t=2023-10-01T10:14:29+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table v2" 2023-10-01T10:14:29.376 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:29 smithi173 bash[47537]: t=2023-10-01T10:14:29+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_api_key_org_id - v2" 2023-10-01T10:14:29.376 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:29 smithi173 bash[47537]: t=2023-10-01T10:14:29+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_key - v2" 2023-10-01T10:14:29.376 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:29 smithi173 bash[47537]: t=2023-10-01T10:14:29+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_org_id_name - v2" 2023-10-01T10:14:29.638 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:29 smithi173 bash[47537]: t=2023-10-01T10:14:29+0000 lvl=info msg="Executing migration" logger=migrator id="copy api_key v1 to v2" 2023-10-01T10:14:29.638 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:29 smithi173 bash[47537]: t=2023-10-01T10:14:29+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table api_key_v1" 2023-10-01T10:14:29.638 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:29 smithi173 bash[47537]: t=2023-10-01T10:14:29+0000 lvl=info msg="Executing migration" logger=migrator id="Update api_key table charset" 2023-10-01T10:14:29.638 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:29 smithi173 bash[47537]: t=2023-10-01T10:14:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add expires to api_key table" 2023-10-01T10:14:29.639 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:29 smithi173 bash[47537]: t=2023-10-01T10:14:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add service account foreign key" 2023-10-01T10:14:29.639 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:29 smithi173 bash[47537]: t=2023-10-01T10:14:29+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v4" 2023-10-01T10:14:29.915 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:29 smithi173 bash[47537]: t=2023-10-01T10:14:29+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_snapshot_v4 #1" 2023-10-01T10:14:29.915 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:29 smithi173 bash[47537]: t=2023-10-01T10:14:29+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v5 #2" 2023-10-01T10:14:29.915 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:29 smithi173 bash[47537]: t=2023-10-01T10:14:29+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_key - v5" 2023-10-01T10:14:29.915 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:29 smithi173 bash[47537]: t=2023-10-01T10:14:29+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_delete_key - v5" 2023-10-01T10:14:29.915 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:29 smithi173 bash[47537]: t=2023-10-01T10:14:29+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_snapshot_user_id - v5" 2023-10-01T10:14:29.915 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:29 smithi173 bash[47537]: t=2023-10-01T10:14:29+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_snapshot to mediumtext v2" 2023-10-01T10:14:29.916 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:29 smithi173 bash[47537]: t=2023-10-01T10:14:29+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_snapshot table charset" 2023-10-01T10:14:29.916 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:29 smithi173 bash[47537]: t=2023-10-01T10:14:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external_delete_url to dashboard_snapshots table" 2023-10-01T10:14:30.176 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:29 smithi173 bash[47537]: t=2023-10-01T10:14:29+0000 lvl=info msg="Executing migration" logger=migrator id="Add encrypted dashboard json column" 2023-10-01T10:14:30.176 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:29 smithi173 bash[47537]: t=2023-10-01T10:14:29+0000 lvl=info msg="Executing migration" logger=migrator id="Change dashboard_encrypted column to MEDIUMBLOB" 2023-10-01T10:14:30.177 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:30 smithi173 bash[47537]: t=2023-10-01T10:14:30+0000 lvl=info msg="Executing migration" logger=migrator id="create quota table v1" 2023-10-01T10:14:30.177 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:30 smithi173 bash[47537]: t=2023-10-01T10:14:30+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_quota_org_id_user_id_target - v1" 2023-10-01T10:14:30.177 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:30 smithi173 bash[47537]: t=2023-10-01T10:14:30+0000 lvl=info msg="Executing migration" logger=migrator id="Update quota table charset" 2023-10-01T10:14:30.177 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:30 smithi173 bash[47537]: t=2023-10-01T10:14:30+0000 lvl=info msg="Executing migration" logger=migrator id="create plugin_setting table" 2023-10-01T10:14:30.177 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:30 smithi173 bash[47537]: t=2023-10-01T10:14:30+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2023-10-01T10:14:30.428 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:30 smithi173 bash[47537]: t=2023-10-01T10:14:30+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_version to plugin_settings" 2023-10-01T10:14:30.428 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:30 smithi173 bash[47537]: t=2023-10-01T10:14:30+0000 lvl=info msg="Executing migration" logger=migrator id="Update plugin_setting table charset" 2023-10-01T10:14:30.428 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:30 smithi173 bash[47537]: t=2023-10-01T10:14:30+0000 lvl=info msg="Executing migration" logger=migrator id="create session table" 2023-10-01T10:14:30.428 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:30 smithi173 bash[47537]: t=2023-10-01T10:14:30+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist table" 2023-10-01T10:14:30.428 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:30 smithi173 bash[47537]: t=2023-10-01T10:14:30+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist_item table" 2023-10-01T10:14:30.428 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:30 smithi173 bash[47537]: t=2023-10-01T10:14:30+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist table v2" 2023-10-01T10:14:30.428 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:30 smithi173 bash[47537]: t=2023-10-01T10:14:30+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist item table v2" 2023-10-01T10:14:30.681 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:14:30 smithi173 bash[27612]: cluster 2023-10-01T10:14:29.172354+0000 mgr.y (mgr.14427) 53 : cluster [DBG] pgmap v43: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:30.681 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:14:30 smithi173 bash[27612]: audit 2023-10-01T10:14:30.173868+0000 mon.a (mon.0) 620 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:14:30.681 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:30 smithi173 bash[47537]: t=2023-10-01T10:14:30+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist table charset" 2023-10-01T10:14:30.682 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:30 smithi173 bash[47537]: t=2023-10-01T10:14:30+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist_item table charset" 2023-10-01T10:14:30.682 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:30 smithi173 bash[47537]: t=2023-10-01T10:14:30+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v2" 2023-10-01T10:14:30.682 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:30 smithi173 bash[47537]: t=2023-10-01T10:14:30+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v3" 2023-10-01T10:14:30.682 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:30 smithi173 bash[47537]: t=2023-10-01T10:14:30+0000 lvl=info msg="Executing migration" logger=migrator id="create preferences table v3" 2023-10-01T10:14:30.682 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:30 smithi173 bash[47537]: t=2023-10-01T10:14:30+0000 lvl=info msg="Executing migration" logger=migrator id="Update preferences table charset" 2023-10-01T10:14:30.682 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:30 smithi173 bash[47537]: t=2023-10-01T10:14:30+0000 lvl=info msg="Executing migration" logger=migrator id="Add column team_id in preferences" 2023-10-01T10:14:30.682 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:30 smithi173 bash[47537]: t=2023-10-01T10:14:30+0000 lvl=info msg="Executing migration" logger=migrator id="Update team_id column values in preferences" 2023-10-01T10:14:30.844 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:14:30 smithi078 bash[33821]: cluster 2023-10-01T10:14:29.172354+0000 mgr.y (mgr.14427) 53 : cluster [DBG] pgmap v43: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:30.845 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:14:30 smithi078 bash[33821]: audit 2023-10-01T10:14:30.173868+0000 mon.a (mon.0) 620 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:14:30.845 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:14:30 smithi078 bash[27336]: cluster 2023-10-01T10:14:29.172354+0000 mgr.y (mgr.14427) 53 : cluster [DBG] pgmap v43: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:30.845 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:14:30 smithi078 bash[27336]: audit 2023-10-01T10:14:30.173868+0000 mon.a (mon.0) 620 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:14:30.968 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:30 smithi173 bash[47537]: t=2023-10-01T10:14:30+0000 lvl=info msg="Executing migration" logger=migrator id="Add column week_start in preferences" 2023-10-01T10:14:30.968 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:30 smithi173 bash[47537]: t=2023-10-01T10:14:30+0000 lvl=info msg="Executing migration" logger=migrator id="create alert table v1" 2023-10-01T10:14:30.968 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:30 smithi173 bash[47537]: t=2023-10-01T10:14:30+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert org_id & id " 2023-10-01T10:14:30.968 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:30 smithi173 bash[47537]: t=2023-10-01T10:14:30+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert state" 2023-10-01T10:14:30.968 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:30 smithi173 bash[47537]: t=2023-10-01T10:14:30+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert dashboard_id" 2023-10-01T10:14:30.968 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:30 smithi173 bash[47537]: t=2023-10-01T10:14:30+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v1" 2023-10-01T10:14:30.968 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:30 smithi173 bash[47537]: t=2023-10-01T10:14:30+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_rule_tag.alert_id_tag_id" 2023-10-01T10:14:30.968 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:30 smithi173 bash[47537]: t=2023-10-01T10:14:30+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2023-10-01T10:14:30.969 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:30 smithi173 bash[47537]: t=2023-10-01T10:14:30+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2023-10-01T10:14:30.969 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:30 smithi173 bash[47537]: t=2023-10-01T10:14:30+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v2" 2023-10-01T10:14:31.220 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:31 smithi173 bash[47537]: t=2023-10-01T10:14:31+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2023-10-01T10:14:31.221 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:31 smithi173 bash[47537]: t=2023-10-01T10:14:31+0000 lvl=info msg="Executing migration" logger=migrator id="copy alert_rule_tag v1 to v2" 2023-10-01T10:14:31.221 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:31 smithi173 bash[47537]: t=2023-10-01T10:14:31+0000 lvl=info msg="Executing migration" logger=migrator id="drop table alert_rule_tag_v1" 2023-10-01T10:14:31.221 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:31 smithi173 bash[47537]: t=2023-10-01T10:14:31+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification table v1" 2023-10-01T10:14:31.221 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:31 smithi173 bash[47537]: t=2023-10-01T10:14:31+0000 lvl=info msg="Executing migration" logger=migrator id="Add column is_default" 2023-10-01T10:14:31.221 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:31 smithi173 bash[47537]: t=2023-10-01T10:14:31+0000 lvl=info msg="Executing migration" logger=migrator id="Add column frequency" 2023-10-01T10:14:31.221 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:31 smithi173 bash[47537]: t=2023-10-01T10:14:31+0000 lvl=info msg="Executing migration" logger=migrator id="Add column send_reminder" 2023-10-01T10:14:31.221 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:31 smithi173 bash[47537]: t=2023-10-01T10:14:31+0000 lvl=info msg="Executing migration" logger=migrator id="Add column disable_resolve_message" 2023-10-01T10:14:31.482 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:31 smithi173 bash[47537]: t=2023-10-01T10:14:31+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification org_id & name" 2023-10-01T10:14:31.482 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:31 smithi173 bash[47537]: t=2023-10-01T10:14:31+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert table charset" 2023-10-01T10:14:31.482 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:31 smithi173 bash[47537]: t=2023-10-01T10:14:31+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert_notification table charset" 2023-10-01T10:14:31.482 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:31 smithi173 bash[47537]: t=2023-10-01T10:14:31+0000 lvl=info msg="Executing migration" logger=migrator id="create notification_journal table v1" 2023-10-01T10:14:31.482 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:31 smithi173 bash[47537]: t=2023-10-01T10:14:31+0000 lvl=info msg="Executing migration" logger=migrator id="add index notification_journal org_id & alert_id & notifier_id" 2023-10-01T10:14:31.482 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:31 smithi173 bash[47537]: t=2023-10-01T10:14:31+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_notification_journal" 2023-10-01T10:14:31.483 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:31 smithi173 bash[47537]: t=2023-10-01T10:14:31+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification_state table v1" 2023-10-01T10:14:31.483 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:31 smithi173 bash[47537]: t=2023-10-01T10:14:31+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification_state org_id & alert_id & notifier_id" 2023-10-01T10:14:31.483 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:31 smithi173 bash[47537]: t=2023-10-01T10:14:31+0000 lvl=info msg="Executing migration" logger=migrator id="Add for to alert table" 2023-10-01T10:14:31.743 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:31 smithi173 bash[47537]: t=2023-10-01T10:14:31+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in alert_notification" 2023-10-01T10:14:31.743 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:31 smithi173 bash[47537]: t=2023-10-01T10:14:31+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in alert_notification" 2023-10-01T10:14:31.743 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:31 smithi173 bash[47537]: t=2023-10-01T10:14:31+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_notification_org_id_uid" 2023-10-01T10:14:31.743 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:31 smithi173 bash[47537]: t=2023-10-01T10:14:31+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_name" 2023-10-01T10:14:31.743 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:31 smithi173 bash[47537]: t=2023-10-01T10:14:31+0000 lvl=info msg="Executing migration" logger=migrator id="Add column secure_settings in alert_notification" 2023-10-01T10:14:31.743 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:31 smithi173 bash[47537]: t=2023-10-01T10:14:31+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert.settings to mediumtext" 2023-10-01T10:14:31.743 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:31 smithi173 bash[47537]: t=2023-10-01T10:14:31+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_notification_state_alert_id" 2023-10-01T10:14:31.743 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:31 smithi173 bash[47537]: t=2023-10-01T10:14:31+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_rule_tag_alert_id" 2023-10-01T10:14:32.004 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:14:31 smithi173 bash[27612]: cluster 2023-10-01T10:14:31.172769+0000 mgr.y (mgr.14427) 54 : cluster [DBG] pgmap v44: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:32.004 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:31 smithi173 bash[47537]: t=2023-10-01T10:14:31+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old annotation table v4" 2023-10-01T10:14:32.004 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:31 smithi173 bash[47537]: t=2023-10-01T10:14:31+0000 lvl=info msg="Executing migration" logger=migrator id="create annotation table v5" 2023-10-01T10:14:32.004 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:31 smithi173 bash[47537]: t=2023-10-01T10:14:31+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 0 v3" 2023-10-01T10:14:32.004 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:31 smithi173 bash[47537]: t=2023-10-01T10:14:31+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 1 v3" 2023-10-01T10:14:32.005 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:31 smithi173 bash[47537]: t=2023-10-01T10:14:31+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 2 v3" 2023-10-01T10:14:32.005 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:31 smithi173 bash[47537]: t=2023-10-01T10:14:31+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 3 v3" 2023-10-01T10:14:32.005 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:31 smithi173 bash[47537]: t=2023-10-01T10:14:31+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 4 v3" 2023-10-01T10:14:32.005 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:32 smithi173 bash[47537]: t=2023-10-01T10:14:32+0000 lvl=info msg="Executing migration" logger=migrator id="Update annotation table charset" 2023-10-01T10:14:32.201 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:14:31 smithi078 bash[27336]: cluster 2023-10-01T10:14:31.172769+0000 mgr.y (mgr.14427) 54 : cluster [DBG] pgmap v44: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:32.201 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:14:31 smithi078 bash[33821]: cluster 2023-10-01T10:14:31.172769+0000 mgr.y (mgr.14427) 54 : cluster [DBG] pgmap v44: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:32.274 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:32 smithi173 bash[47537]: t=2023-10-01T10:14:32+0000 lvl=info msg="Executing migration" logger=migrator id="Add column region_id to annotation table" 2023-10-01T10:14:32.274 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:32 smithi173 bash[47537]: t=2023-10-01T10:14:32+0000 lvl=info msg="Executing migration" logger=migrator id="Drop category_id index" 2023-10-01T10:14:32.274 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:32 smithi173 bash[47537]: t=2023-10-01T10:14:32+0000 lvl=info msg="Executing migration" logger=migrator id="Add column tags to annotation table" 2023-10-01T10:14:32.274 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:32 smithi173 bash[47537]: t=2023-10-01T10:14:32+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v2" 2023-10-01T10:14:32.274 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:32 smithi173 bash[47537]: t=2023-10-01T10:14:32+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index annotation_tag.annotation_id_tag_id" 2023-10-01T10:14:32.274 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:32 smithi173 bash[47537]: t=2023-10-01T10:14:32+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2023-10-01T10:14:32.274 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:32 smithi173 bash[47537]: t=2023-10-01T10:14:32+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table annotation_tag to annotation_tag_v2 - v2" 2023-10-01T10:14:32.274 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:32 smithi173 bash[47537]: t=2023-10-01T10:14:32+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v3" 2023-10-01T10:14:32.274 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:32 smithi173 bash[47537]: t=2023-10-01T10:14:32+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2023-10-01T10:14:32.526 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:32 smithi173 bash[47537]: t=2023-10-01T10:14:32+0000 lvl=info msg="Executing migration" logger=migrator id="copy annotation_tag v2 to v3" 2023-10-01T10:14:32.526 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:32 smithi173 bash[47537]: t=2023-10-01T10:14:32+0000 lvl=info msg="Executing migration" logger=migrator id="drop table annotation_tag_v2" 2023-10-01T10:14:32.526 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:32 smithi173 bash[47537]: t=2023-10-01T10:14:32+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert annotations and set TEXT to empty" 2023-10-01T10:14:32.527 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:32 smithi173 bash[47537]: t=2023-10-01T10:14:32+0000 lvl=info msg="Executing migration" logger=migrator id="Add created time to annotation table" 2023-10-01T10:14:32.527 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:32 smithi173 bash[47537]: t=2023-10-01T10:14:32+0000 lvl=info msg="Executing migration" logger=migrator id="Add updated time to annotation table" 2023-10-01T10:14:32.527 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:32 smithi173 bash[47537]: t=2023-10-01T10:14:32+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for created in annotation table" 2023-10-01T10:14:32.527 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:32 smithi173 bash[47537]: t=2023-10-01T10:14:32+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for updated in annotation table" 2023-10-01T10:14:32.527 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:32 smithi173 bash[47537]: t=2023-10-01T10:14:32+0000 lvl=info msg="Executing migration" logger=migrator id="Convert existing annotations from seconds to milliseconds" 2023-10-01T10:14:32.781 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:32 smithi173 bash[47537]: t=2023-10-01T10:14:32+0000 lvl=info msg="Executing migration" logger=migrator id="Add epoch_end column" 2023-10-01T10:14:32.781 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:32 smithi173 bash[47537]: t=2023-10-01T10:14:32+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for epoch_end" 2023-10-01T10:14:32.781 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:32 smithi173 bash[47537]: t=2023-10-01T10:14:32+0000 lvl=info msg="Executing migration" logger=migrator id="Make epoch_end the same as epoch" 2023-10-01T10:14:32.781 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:32 smithi173 bash[47537]: t=2023-10-01T10:14:32+0000 lvl=info msg="Executing migration" logger=migrator id="Move region to single row" 2023-10-01T10:14:32.781 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:32 smithi173 bash[47537]: t=2023-10-01T10:14:32+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch from annotation table" 2023-10-01T10:14:32.781 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:32 smithi173 bash[47537]: t=2023-10-01T10:14:32+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2023-10-01T10:14:32.782 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:32 smithi173 bash[47537]: t=2023-10-01T10:14:32+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2023-10-01T10:14:33.041 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:32 smithi173 bash[47537]: t=2023-10-01T10:14:32+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_epoch_end_epoch on annotation table" 2023-10-01T10:14:33.041 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:32 smithi173 bash[47537]: t=2023-10-01T10:14:32+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch_epoch_end from annotation table" 2023-10-01T10:14:33.041 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:32 smithi173 bash[47537]: t=2023-10-01T10:14:32+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for alert_id on annotation table" 2023-10-01T10:14:33.041 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:32 smithi173 bash[47537]: t=2023-10-01T10:14:32+0000 lvl=info msg="Executing migration" logger=migrator id="create test_data table" 2023-10-01T10:14:33.042 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:32 smithi173 bash[47537]: t=2023-10-01T10:14:32+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_version table v1" 2023-10-01T10:14:33.042 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:32 smithi173 bash[47537]: t=2023-10-01T10:14:32+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_version.dashboard_id" 2023-10-01T10:14:33.042 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:32 smithi173 bash[47537]: t=2023-10-01T10:14:32+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2023-10-01T10:14:33.042 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:32 smithi173 bash[47537]: t=2023-10-01T10:14:32+0000 lvl=info msg="Executing migration" logger=migrator id="Set dashboard version to 1 where 0" 2023-10-01T10:14:33.294 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:33 smithi173 bash[47537]: t=2023-10-01T10:14:33+0000 lvl=info msg="Executing migration" logger=migrator id="save existing dashboard data in dashboard_version table v1" 2023-10-01T10:14:33.294 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:33 smithi173 bash[47537]: t=2023-10-01T10:14:33+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_version.data to mediumtext v1" 2023-10-01T10:14:33.294 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:33 smithi173 bash[47537]: t=2023-10-01T10:14:33+0000 lvl=info msg="Executing migration" logger=migrator id="create team table" 2023-10-01T10:14:33.294 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:33 smithi173 bash[47537]: t=2023-10-01T10:14:33+0000 lvl=info msg="Executing migration" logger=migrator id="add index team.org_id" 2023-10-01T10:14:33.295 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:33 smithi173 bash[47537]: t=2023-10-01T10:14:33+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_org_id_name" 2023-10-01T10:14:33.295 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:33 smithi173 bash[47537]: t=2023-10-01T10:14:33+0000 lvl=info msg="Executing migration" logger=migrator id="create team member table" 2023-10-01T10:14:33.295 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:33 smithi173 bash[47537]: t=2023-10-01T10:14:33+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.org_id" 2023-10-01T10:14:33.295 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:33 smithi173 bash[47537]: t=2023-10-01T10:14:33+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_member_org_id_team_id_user_id" 2023-10-01T10:14:33.295 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:33 smithi173 bash[47537]: t=2023-10-01T10:14:33+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.team_id" 2023-10-01T10:14:33.572 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:33 smithi173 bash[47537]: t=2023-10-01T10:14:33+0000 lvl=info msg="Executing migration" logger=migrator id="Add column email to team table" 2023-10-01T10:14:33.572 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:33 smithi173 bash[47537]: t=2023-10-01T10:14:33+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external to team_member table" 2023-10-01T10:14:33.572 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:33 smithi173 bash[47537]: t=2023-10-01T10:14:33+0000 lvl=info msg="Executing migration" logger=migrator id="Add column permission to team_member table" 2023-10-01T10:14:33.572 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:33 smithi173 bash[47537]: t=2023-10-01T10:14:33+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard acl table" 2023-10-01T10:14:33.572 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:33 smithi173 bash[47537]: t=2023-10-01T10:14:33+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_dashboard_id" 2023-10-01T10:14:33.572 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:33 smithi173 bash[47537]: t=2023-10-01T10:14:33+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_user_id" 2023-10-01T10:14:33.572 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:33 smithi173 bash[47537]: t=2023-10-01T10:14:33+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_team_id" 2023-10-01T10:14:33.572 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:33 smithi173 bash[47537]: t=2023-10-01T10:14:33+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_user_id" 2023-10-01T10:14:33.573 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:33 smithi173 bash[47537]: t=2023-10-01T10:14:33+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_team_id" 2023-10-01T10:14:33.824 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:33 smithi173 bash[47537]: t=2023-10-01T10:14:33+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_org_id_role" 2023-10-01T10:14:33.824 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:33 smithi173 bash[47537]: t=2023-10-01T10:14:33+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_permission" 2023-10-01T10:14:33.824 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:33 smithi173 bash[47537]: t=2023-10-01T10:14:33+0000 lvl=info msg="Executing migration" logger=migrator id="save default acl rules in dashboard_acl table" 2023-10-01T10:14:33.824 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:33 smithi173 bash[47537]: t=2023-10-01T10:14:33+0000 lvl=info msg="Executing migration" logger=migrator id="delete acl rules for deleted dashboards and folders" 2023-10-01T10:14:33.824 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:33 smithi173 bash[47537]: t=2023-10-01T10:14:33+0000 lvl=info msg="Executing migration" logger=migrator id="create tag table" 2023-10-01T10:14:33.824 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:33 smithi173 bash[47537]: t=2023-10-01T10:14:33+0000 lvl=info msg="Executing migration" logger=migrator id="add index tag.key_value" 2023-10-01T10:14:33.824 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:33 smithi173 bash[47537]: t=2023-10-01T10:14:33+0000 lvl=info msg="Executing migration" logger=migrator id="create login attempt table" 2023-10-01T10:14:33.825 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:33 smithi173 bash[47537]: t=2023-10-01T10:14:33+0000 lvl=info msg="Executing migration" logger=migrator id="add index login_attempt.username" 2023-10-01T10:14:34.085 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:33 smithi173 bash[47537]: t=2023-10-01T10:14:33+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_login_attempt_username - v1" 2023-10-01T10:14:34.085 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:33 smithi173 bash[47537]: t=2023-10-01T10:14:33+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2023-10-01T10:14:34.085 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:33 smithi173 bash[47537]: t=2023-10-01T10:14:33+0000 lvl=info msg="Executing migration" logger=migrator id="create login_attempt v2" 2023-10-01T10:14:34.085 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:33 smithi173 bash[47537]: t=2023-10-01T10:14:33+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_login_attempt_username - v2" 2023-10-01T10:14:34.085 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:33 smithi173 bash[47537]: t=2023-10-01T10:14:33+0000 lvl=info msg="Executing migration" logger=migrator id="copy login_attempt v1 to v2" 2023-10-01T10:14:34.085 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:33 smithi173 bash[47537]: t=2023-10-01T10:14:33+0000 lvl=info msg="Executing migration" logger=migrator id="drop login_attempt_tmp_qwerty" 2023-10-01T10:14:34.086 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:34 smithi173 bash[47537]: t=2023-10-01T10:14:34+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth table" 2023-10-01T10:14:34.086 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:34 smithi173 bash[47537]: t=2023-10-01T10:14:34+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_user_auth_auth_module_auth_id - v1" 2023-10-01T10:14:34.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:14:34 smithi173 bash[27612]: cluster 2023-10-01T10:14:33.173401+0000 mgr.y (mgr.14427) 55 : cluster [DBG] pgmap v45: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:34.370 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:34 smithi173 bash[47537]: t=2023-10-01T10:14:34+0000 lvl=info msg="Executing migration" logger=migrator id="alter user_auth.auth_id to length 190" 2023-10-01T10:14:34.371 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:34 smithi173 bash[47537]: t=2023-10-01T10:14:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth access token to user_auth" 2023-10-01T10:14:34.371 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:34 smithi173 bash[47537]: t=2023-10-01T10:14:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth refresh token to user_auth" 2023-10-01T10:14:34.371 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:34 smithi173 bash[47537]: t=2023-10-01T10:14:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth token type to user_auth" 2023-10-01T10:14:34.371 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:34 smithi173 bash[47537]: t=2023-10-01T10:14:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth expiry to user_auth" 2023-10-01T10:14:34.371 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:34 smithi173 bash[47537]: t=2023-10-01T10:14:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add index to user_id column in user_auth" 2023-10-01T10:14:34.371 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:34 smithi173 bash[47537]: t=2023-10-01T10:14:34+0000 lvl=info msg="Executing migration" logger=migrator id="create server_lock table" 2023-10-01T10:14:34.371 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:34 smithi173 bash[47537]: t=2023-10-01T10:14:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index server_lock.operation_uid" 2023-10-01T10:14:34.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:14:34 smithi078 bash[27336]: cluster 2023-10-01T10:14:33.173401+0000 mgr.y (mgr.14427) 55 : cluster [DBG] pgmap v45: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:34.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:14:34 smithi078 bash[33821]: cluster 2023-10-01T10:14:33.173401+0000 mgr.y (mgr.14427) 55 : cluster [DBG] pgmap v45: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:34.657 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:34 smithi173 bash[47537]: t=2023-10-01T10:14:34+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth token table" 2023-10-01T10:14:34.657 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:34 smithi173 bash[47537]: t=2023-10-01T10:14:34+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.auth_token" 2023-10-01T10:14:34.657 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:34 smithi173 bash[47537]: t=2023-10-01T10:14:34+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.prev_auth_token" 2023-10-01T10:14:34.657 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:34 smithi173 bash[47537]: t=2023-10-01T10:14:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_auth_token.user_id" 2023-10-01T10:14:34.657 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:34 smithi173 bash[47537]: t=2023-10-01T10:14:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add revoked_at to the user auth token" 2023-10-01T10:14:34.658 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:34 smithi173 bash[47537]: t=2023-10-01T10:14:34+0000 lvl=info msg="Executing migration" logger=migrator id="create cache_data table" 2023-10-01T10:14:34.658 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:34 smithi173 bash[47537]: t=2023-10-01T10:14:34+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index cache_data.cache_key" 2023-10-01T10:14:34.658 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:34 smithi173 bash[47537]: t=2023-10-01T10:14:34+0000 lvl=info msg="Executing migration" logger=migrator id="create short_url table v1" 2023-10-01T10:14:34.658 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:34 smithi173 bash[47537]: t=2023-10-01T10:14:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index short_url.org_id-uid" 2023-10-01T10:14:34.658 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:34 smithi173 bash[47537]: t=2023-10-01T10:14:34+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition table" 2023-10-01T10:14:34.658 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:34 smithi173 bash[47537]: t=2023-10-01T10:14:34+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition table" 2023-10-01T10:14:34.909 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:34 smithi173 bash[47537]: t=2023-10-01T10:14:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and title columns" 2023-10-01T10:14:34.909 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:34 smithi173 bash[47537]: t=2023-10-01T10:14:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and uid columns" 2023-10-01T10:14:34.909 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:34 smithi173 bash[47537]: t=2023-10-01T10:14:34+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition table data column to mediumtext in mysql" 2023-10-01T10:14:34.909 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:34 smithi173 bash[47537]: t=2023-10-01T10:14:34+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and title columns" 2023-10-01T10:14:34.910 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:34 smithi173 bash[47537]: t=2023-10-01T10:14:34+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and uid columns" 2023-10-01T10:14:34.910 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:34 smithi173 bash[47537]: t=2023-10-01T10:14:34+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and title columns" 2023-10-01T10:14:34.910 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:34 smithi173 bash[47537]: t=2023-10-01T10:14:34+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and uid columns" 2023-10-01T10:14:34.910 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:34 smithi173 bash[47537]: t=2023-10-01T10:14:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add column paused in alert_definition" 2023-10-01T10:14:35.170 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:34 smithi173 bash[47537]: t=2023-10-01T10:14:34+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition table" 2023-10-01T10:14:35.174 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:34 smithi173 bash[47537]: t=2023-10-01T10:14:34+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition_version table" 2023-10-01T10:14:35.174 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:35 smithi173 bash[47537]: t=2023-10-01T10:14:35+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition_version table" 2023-10-01T10:14:35.174 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:35 smithi173 bash[47537]: t=2023-10-01T10:14:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_id and version columns" 2023-10-01T10:14:35.174 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:35 smithi173 bash[47537]: t=2023-10-01T10:14:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_uid and version columns" 2023-10-01T10:14:35.174 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:35 smithi173 bash[47537]: t=2023-10-01T10:14:35+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition_version table data column to mediumtext in mysql" 2023-10-01T10:14:35.174 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:35 smithi173 bash[47537]: t=2023-10-01T10:14:35+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition_version table" 2023-10-01T10:14:35.174 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:35 smithi173 bash[47537]: t=2023-10-01T10:14:35+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_instance table" 2023-10-01T10:14:35.174 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:35 smithi173 bash[47537]: t=2023-10-01T10:14:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2023-10-01T10:14:35.439 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:14:35 smithi173 bash[28473]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:14:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T10:14:35.440 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:35 smithi173 bash[47537]: t=2023-10-01T10:14:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, current_state columns" 2023-10-01T10:14:35.440 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:35 smithi173 bash[47537]: t=2023-10-01T10:14:35+0000 lvl=info msg="Executing migration" logger=migrator id="add column current_state_end to alert_instance" 2023-10-01T10:14:35.440 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:35 smithi173 bash[47537]: t=2023-10-01T10:14:35+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, def_uid, current_state on alert_instance" 2023-10-01T10:14:35.440 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:35 smithi173 bash[47537]: t=2023-10-01T10:14:35+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, current_state on alert_instance" 2023-10-01T10:14:35.440 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:35 smithi173 bash[47537]: t=2023-10-01T10:14:35+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_org_id to rule_org_id in alert_instance" 2023-10-01T10:14:35.440 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:35 smithi173 bash[47537]: t=2023-10-01T10:14:35+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_uid to rule_uid in alert_instance" 2023-10-01T10:14:35.440 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:35 smithi173 bash[47537]: t=2023-10-01T10:14:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, rule_uid, current_state on alert_instance" 2023-10-01T10:14:35.586 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:14:35 smithi078 bash[27601]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:14:35] "GET /metrics HTTP/1.1" 200 155666 "" "Prometheus/2.33.4" 2023-10-01T10:14:35.725 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:35 smithi173 bash[47537]: t=2023-10-01T10:14:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, current_state on alert_instance" 2023-10-01T10:14:35.726 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:35 smithi173 bash[47537]: t=2023-10-01T10:14:35+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule table" 2023-10-01T10:14:35.726 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:35 smithi173 bash[47537]: t=2023-10-01T10:14:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and title columns" 2023-10-01T10:14:35.726 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:35 smithi173 bash[47537]: t=2023-10-01T10:14:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and uid columns" 2023-10-01T10:14:35.726 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:35 smithi173 bash[47537]: t=2023-10-01T10:14:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2023-10-01T10:14:35.726 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:35 smithi173 bash[47537]: t=2023-10-01T10:14:35+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule table data column to mediumtext in mysql" 2023-10-01T10:14:35.726 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:35 smithi173 bash[47537]: t=2023-10-01T10:14:35+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule" 2023-10-01T10:14:35.726 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:35 smithi173 bash[47537]: t=2023-10-01T10:14:35+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule" 2023-10-01T10:14:35.726 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:35 smithi173 bash[47537]: t=2023-10-01T10:14:35+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule" 2023-10-01T10:14:35.726 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:35 smithi173 bash[47537]: t=2023-10-01T10:14:35+0000 lvl=info msg="Executing migration" logger=migrator id="remove unique index from alert_rule on org_id, title columns" 2023-10-01T10:14:35.978 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:35 smithi173 bash[47537]: t=2023-10-01T10:14:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespase_uid and title columns" 2023-10-01T10:14:35.978 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:35 smithi173 bash[47537]: t=2023-10-01T10:14:35+0000 lvl=info msg="Executing migration" logger=migrator id="add dashboard_uid column to alert_rule" 2023-10-01T10:14:35.978 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:35 smithi173 bash[47537]: t=2023-10-01T10:14:35+0000 lvl=info msg="Executing migration" logger=migrator id="add panel_id column to alert_rule" 2023-10-01T10:14:35.978 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:35 smithi173 bash[47537]: t=2023-10-01T10:14:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2023-10-01T10:14:35.978 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:35 smithi173 bash[47537]: t=2023-10-01T10:14:35+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule_version table" 2023-10-01T10:14:35.978 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:35 smithi173 bash[47537]: t=2023-10-01T10:14:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2023-10-01T10:14:35.979 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:35 smithi173 bash[47537]: t=2023-10-01T10:14:35+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2023-10-01T10:14:36.238 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:35 smithi173 bash[47537]: t=2023-10-01T10:14:35+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule_version table data column to mediumtext in mysql" 2023-10-01T10:14:36.238 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:36 smithi173 bash[47537]: t=2023-10-01T10:14:36+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule_version" 2023-10-01T10:14:36.238 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:36 smithi173 bash[47537]: t=2023-10-01T10:14:36+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule_version" 2023-10-01T10:14:36.238 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:36 smithi173 bash[47537]: t=2023-10-01T10:14:36+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule_version" 2023-10-01T10:14:36.238 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:36 smithi173 bash[47537]: t=2023-10-01T10:14:36+0000 lvl=info msg="Executing migration" logger=migrator id=create_alert_configuration_table 2023-10-01T10:14:36.238 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:36 smithi173 bash[47537]: t=2023-10-01T10:14:36+0000 lvl=info msg="Executing migration" logger=migrator id="Add column default in alert_configuration" 2023-10-01T10:14:36.239 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:36 smithi173 bash[47537]: t=2023-10-01T10:14:36+0000 lvl=info msg="Executing migration" logger=migrator id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2023-10-01T10:14:36.239 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:36 smithi173 bash[47537]: t=2023-10-01T10:14:36+0000 lvl=info msg="Executing migration" logger=migrator id="add column org_id in alert_configuration" 2023-10-01T10:14:36.239 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:36 smithi173 bash[47537]: t=2023-10-01T10:14:36+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_configuration table on org_id column" 2023-10-01T10:14:36.479 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:14:36 smithi078 bash[33821]: cluster 2023-10-01T10:14:35.173961+0000 mgr.y (mgr.14427) 56 : cluster [DBG] pgmap v46: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:36.479 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:14:36 smithi078 bash[27336]: cluster 2023-10-01T10:14:35.173961+0000 mgr.y (mgr.14427) 56 : cluster [DBG] pgmap v46: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:36.491 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:14:36 smithi173 bash[27612]: cluster 2023-10-01T10:14:35.173961+0000 mgr.y (mgr.14427) 56 : cluster [DBG] pgmap v46: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:36.491 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:36 smithi173 bash[47537]: t=2023-10-01T10:14:36+0000 lvl=info msg="Executing migration" logger=migrator id=create_ngalert_configuration_table 2023-10-01T10:14:36.491 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:36 smithi173 bash[47537]: t=2023-10-01T10:14:36+0000 lvl=info msg="Executing migration" logger=migrator id="add index in ngalert_configuration on org_id column" 2023-10-01T10:14:36.492 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:36 smithi173 bash[47537]: t=2023-10-01T10:14:36+0000 lvl=info msg="Executing migration" logger=migrator id="clear migration entry \"remove unified alerting data\"" 2023-10-01T10:14:36.492 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:36 smithi173 bash[47537]: t=2023-10-01T10:14:36+0000 lvl=info msg="Executing migration" logger=migrator id="move dashboard alerts to unified alerting" 2023-10-01T10:14:36.492 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:36 smithi173 bash[47537]: t=2023-10-01T10:14:36+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element table v1" 2023-10-01T10:14:36.492 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:36 smithi173 bash[47537]: t=2023-10-01T10:14:36+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element org_id-folder_id-name-kind" 2023-10-01T10:14:36.492 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:36 smithi173 bash[47537]: t=2023-10-01T10:14:36+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element_connection table v1" 2023-10-01T10:14:36.492 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:36 smithi173 bash[47537]: t=2023-10-01T10:14:36+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element_connection element_id-kind-connection_id" 2023-10-01T10:14:36.492 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:36 smithi173 bash[47537]: t=2023-10-01T10:14:36+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index library_element org_id_uid" 2023-10-01T10:14:36.743 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:36 smithi173 bash[47537]: t=2023-10-01T10:14:36+0000 lvl=info msg="Executing migration" logger=migrator id="clone move dashboard alerts to unified alerting" 2023-10-01T10:14:36.744 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:36 smithi173 bash[47537]: t=2023-10-01T10:14:36+0000 lvl=info msg="Executing migration" logger=migrator id="create data_keys table" 2023-10-01T10:14:36.744 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:36 smithi173 bash[47537]: t=2023-10-01T10:14:36+0000 lvl=info msg="Executing migration" logger=migrator id="create kv_store table v1" 2023-10-01T10:14:36.744 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:36 smithi173 bash[47537]: t=2023-10-01T10:14:36+0000 lvl=info msg="Executing migration" logger=migrator id="add index kv_store.org_id-namespace-key" 2023-10-01T10:14:36.744 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:36 smithi173 bash[47537]: t=2023-10-01T10:14:36+0000 lvl=info msg="Executing migration" logger=migrator id="update dashboard_uid and panel_id from existing annotations" 2023-10-01T10:14:36.744 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:36 smithi173 bash[47537]: t=2023-10-01T10:14:36+0000 lvl=info msg="Executing migration" logger=migrator id="create permission table" 2023-10-01T10:14:36.744 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:36 smithi173 bash[47537]: t=2023-10-01T10:14:36+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index permission.role_id" 2023-10-01T10:14:36.744 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:36 smithi173 bash[47537]: t=2023-10-01T10:14:36+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_id_action_scope" 2023-10-01T10:14:36.996 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:36 smithi173 bash[47537]: t=2023-10-01T10:14:36+0000 lvl=info msg="Executing migration" logger=migrator id="create role table" 2023-10-01T10:14:36.996 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:36 smithi173 bash[47537]: t=2023-10-01T10:14:36+0000 lvl=info msg="Executing migration" logger=migrator id="add column display_name" 2023-10-01T10:14:36.996 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:36 smithi173 bash[47537]: t=2023-10-01T10:14:36+0000 lvl=info msg="Executing migration" logger=migrator id="add column group_name" 2023-10-01T10:14:36.996 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:36 smithi173 bash[47537]: t=2023-10-01T10:14:36+0000 lvl=info msg="Executing migration" logger=migrator id="add index role.org_id" 2023-10-01T10:14:36.996 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:36 smithi173 bash[47537]: t=2023-10-01T10:14:36+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_org_id_name" 2023-10-01T10:14:36.996 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:36 smithi173 bash[47537]: t=2023-10-01T10:14:36+0000 lvl=info msg="Executing migration" logger=migrator id="add index role_org_id_uid" 2023-10-01T10:14:36.996 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:36 smithi173 bash[47537]: t=2023-10-01T10:14:36+0000 lvl=info msg="Executing migration" logger=migrator id="create team role table" 2023-10-01T10:14:37.256 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:37 smithi173 bash[47537]: t=2023-10-01T10:14:37+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.org_id" 2023-10-01T10:14:37.256 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:37 smithi173 bash[47537]: t=2023-10-01T10:14:37+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_role_org_id_team_id_role_id" 2023-10-01T10:14:37.257 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:37 smithi173 bash[47537]: t=2023-10-01T10:14:37+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.team_id" 2023-10-01T10:14:37.257 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:37 smithi173 bash[47537]: t=2023-10-01T10:14:37+0000 lvl=info msg="Executing migration" logger=migrator id="create user role table" 2023-10-01T10:14:37.257 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:37 smithi173 bash[47537]: t=2023-10-01T10:14:37+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.org_id" 2023-10-01T10:14:37.257 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:37 smithi173 bash[47537]: t=2023-10-01T10:14:37+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_role_org_id_user_id_role_id" 2023-10-01T10:14:37.257 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:37 smithi173 bash[47537]: t=2023-10-01T10:14:37+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.user_id" 2023-10-01T10:14:37.257 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:37 smithi173 bash[47537]: t=2023-10-01T10:14:37+0000 lvl=info msg="Executing migration" logger=migrator id="create builtin role table" 2023-10-01T10:14:37.517 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:37 smithi173 bash[47537]: t=2023-10-01T10:14:37+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.role_id" 2023-10-01T10:14:37.517 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:37 smithi173 bash[47537]: t=2023-10-01T10:14:37+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.name" 2023-10-01T10:14:37.517 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:37 smithi173 bash[47537]: t=2023-10-01T10:14:37+0000 lvl=info msg="Executing migration" logger=migrator id="Add column org_id to builtin_role table" 2023-10-01T10:14:37.517 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:37 smithi173 bash[47537]: t=2023-10-01T10:14:37+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.org_id" 2023-10-01T10:14:37.517 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:37 smithi173 bash[47537]: t=2023-10-01T10:14:37+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_org_id_role_id_role" 2023-10-01T10:14:37.517 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:37 smithi173 bash[47537]: t=2023-10-01T10:14:37+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index role_org_id_uid" 2023-10-01T10:14:37.518 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:37 smithi173 bash[47537]: t=2023-10-01T10:14:37+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role.uid" 2023-10-01T10:14:37.518 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:37 smithi173 bash[47537]: t=2023-10-01T10:14:37+0000 lvl=info msg="Executing migration" logger=migrator id="create seed assignment table" 2023-10-01T10:14:37.814 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:37 smithi173 bash[47537]: t=2023-10-01T10:14:37+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_role_name" 2023-10-01T10:14:37.814 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:37 smithi173 bash[47537]: t=2023-10-01T10:14:37+0000 lvl=info msg="migrations completed" logger=migrator performed=381 skipped=0 duration=12.245989692s 2023-10-01T10:14:37.814 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:37 smithi173 bash[47537]: t=2023-10-01T10:14:37+0000 lvl=info msg="Created default organization" logger=sqlstore 2023-10-01T10:14:37.814 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:37 smithi173 bash[47537]: t=2023-10-01T10:14:37+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2023-10-01T10:14:37.814 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:37 smithi173 bash[47537]: t=2023-10-01T10:14:37+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2023-10-01T10:14:37.814 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:37 smithi173 bash[47537]: t=2023-10-01T10:14:37+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2023-10-01T10:14:37.814 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:37 smithi173 bash[47537]: t=2023-10-01T10:14:37+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2023-10-01T10:14:37.815 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:37 smithi173 bash[47537]: t=2023-10-01T10:14:37+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2023-10-01T10:14:37.815 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:37 smithi173 bash[47537]: t=2023-10-01T10:14:37+0000 lvl=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" logger=provisioning.datasources filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2023-10-01T10:14:37.815 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:37 smithi173 bash[47537]: t=2023-10-01T10:14:37+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2023-10-01T10:14:38.114 INFO:teuthology.orchestra.run.smithi078.stdout:154618822695 2023-10-01T10:14:38.115 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-10-01T10:14:38.120 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:37 smithi173 bash[47537]: t=2023-10-01T10:14:37+0000 lvl=info msg="warming cache for startup" logger=ngalert 2023-10-01T10:14:38.120 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:37 smithi173 bash[47537]: t=2023-10-01T10:14:37+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2023-10-01T10:14:38.120 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:14:37 smithi173 bash[47537]: t=2023-10-01T10:14:37+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2023-10-01T10:14:38.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:14:38 smithi078 bash[27336]: cluster 2023-10-01T10:14:37.174564+0000 mgr.y (mgr.14427) 57 : cluster [DBG] pgmap v47: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:38.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:14:38 smithi078 bash[33821]: cluster 2023-10-01T10:14:37.174564+0000 mgr.y (mgr.14427) 57 : cluster [DBG] pgmap v47: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:38.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:14:38 smithi173 bash[27612]: cluster 2023-10-01T10:14:37.174564+0000 mgr.y (mgr.14427) 57 : cluster [DBG] pgmap v47: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:39.690 INFO:teuthology.orchestra.run.smithi078.stdout:55834574922 2023-10-01T10:14:39.691 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-10-01T10:14:39.907 INFO:teuthology.orchestra.run.smithi078.stdout:180388626465 2023-10-01T10:14:39.907 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-10-01T10:14:40.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:14:40 smithi078 bash[27336]: cluster 2023-10-01T10:14:39.175137+0000 mgr.y (mgr.14427) 58 : cluster [DBG] pgmap v48: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:40.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:14:40 smithi078 bash[33821]: cluster 2023-10-01T10:14:39.175137+0000 mgr.y (mgr.14427) 58 : cluster [DBG] pgmap v48: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:40.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:14:40 smithi173 bash[27612]: cluster 2023-10-01T10:14:39.175137+0000 mgr.y (mgr.14427) 58 : cluster [DBG] pgmap v48: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:41.599 INFO:teuthology.orchestra.run.smithi078.stdout:77309411393 2023-10-01T10:14:41.600 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-10-01T10:14:41.863 INFO:teuthology.orchestra.run.smithi078.stdout:206158430232 2023-10-01T10:14:41.863 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-10-01T10:14:42.304 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:14:41 smithi078 bash[33821]: cluster 2023-10-01T10:14:41.175571+0000 mgr.y (mgr.14427) 59 : cluster [DBG] pgmap v49: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:42.304 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:14:41 smithi078 bash[27336]: cluster 2023-10-01T10:14:41.175571+0000 mgr.y (mgr.14427) 59 : cluster [DBG] pgmap v49: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:42.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:14:41 smithi173 bash[27612]: cluster 2023-10-01T10:14:41.175571+0000 mgr.y (mgr.14427) 59 : cluster [DBG] pgmap v49: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:42.990 INFO:teuthology.orchestra.run.smithi078.stdout:34359738452 2023-10-01T10:14:42.991 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-10-01T10:14:43.121 INFO:teuthology.orchestra.run.smithi078.stdout:107374182458 2023-10-01T10:14:43.122 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-10-01T10:14:43.189 INFO:teuthology.orchestra.run.smithi078.stdout:128849018930 2023-10-01T10:14:43.189 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-10-01T10:14:44.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:14:44 smithi078 bash[27336]: cluster 2023-10-01T10:14:43.176009+0000 mgr.y (mgr.14427) 60 : cluster [DBG] pgmap v50: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:44.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:14:44 smithi078 bash[33821]: cluster 2023-10-01T10:14:43.176009+0000 mgr.y (mgr.14427) 60 : cluster [DBG] pgmap v50: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:44.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:14:44 smithi173 bash[27612]: cluster 2023-10-01T10:14:43.176009+0000 mgr.y (mgr.14427) 60 : cluster [DBG] pgmap v50: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:45.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:14:45 smithi078 bash[27601]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:14:45] "GET /metrics HTTP/1.1" 200 155666 "" "Prometheus/2.33.4" 2023-10-01T10:14:45.620 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:14:45 smithi173 bash[28473]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:14:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T10:14:45.922 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:14:45.923 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:14:45.928 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:14:45.929 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:14:45.929 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:14:46.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:14:46 smithi078 bash[27336]: cluster 2023-10-01T10:14:45.176541+0000 mgr.y (mgr.14427) 61 : cluster [DBG] pgmap v51: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:46.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:14:46 smithi078 bash[33821]: cluster 2023-10-01T10:14:45.176541+0000 mgr.y (mgr.14427) 61 : cluster [DBG] pgmap v51: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:46.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:14:46 smithi173 bash[27612]: cluster 2023-10-01T10:14:45.176541+0000 mgr.y (mgr.14427) 61 : cluster [DBG] pgmap v51: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:46.930 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:14:46.931 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:14:47.942 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:14:47.967 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:14:47 smithi078 bash[33821]: audit 2023-10-01T10:14:46.657452+0000 mon.a (mon.0) 621 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:14:47.967 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:14:47 smithi078 bash[33821]: audit 2023-10-01T10:14:46.901846+0000 mon.a (mon.0) 622 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:14:47.967 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:14:47 smithi078 bash[33821]: audit 2023-10-01T10:14:46.903368+0000 mon.a (mon.0) 623 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:14:47.967 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:14:47 smithi078 bash[33821]: audit 2023-10-01T10:14:46.910978+0000 mon.a (mon.0) 624 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:14:47.967 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:14:47 smithi078 bash[33821]: cephadm 2023-10-01T10:14:46.915352+0000 mgr.y (mgr.14427) 62 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-10-01T10:14:47.967 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:14:47 smithi078 bash[33821]: cephadm 2023-10-01T10:14:46.919228+0000 mgr.y (mgr.14427) 63 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi078 2023-10-01T10:14:47.967 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:14:47 smithi078 bash[33821]: cluster 2023-10-01T10:14:47.177025+0000 mgr.y (mgr.14427) 64 : cluster [DBG] pgmap v52: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:47.968 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:14:47 smithi078 bash[27336]: audit 2023-10-01T10:14:46.657452+0000 mon.a (mon.0) 621 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:14:47.968 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:14:47 smithi078 bash[27336]: audit 2023-10-01T10:14:46.901846+0000 mon.a (mon.0) 622 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:14:47.968 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:14:47 smithi078 bash[27336]: audit 2023-10-01T10:14:46.903368+0000 mon.a (mon.0) 623 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:14:47.968 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:14:47 smithi078 bash[27336]: audit 2023-10-01T10:14:46.910978+0000 mon.a (mon.0) 624 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:14:47.968 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:14:47 smithi078 bash[27336]: cephadm 2023-10-01T10:14:46.915352+0000 mgr.y (mgr.14427) 62 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-10-01T10:14:47.968 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:14:47 smithi078 bash[27336]: cephadm 2023-10-01T10:14:46.919228+0000 mgr.y (mgr.14427) 63 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi078 2023-10-01T10:14:47.968 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:14:47 smithi078 bash[27336]: cluster 2023-10-01T10:14:47.177025+0000 mgr.y (mgr.14427) 64 : cluster [DBG] pgmap v52: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:48.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:14:47 smithi173 bash[27612]: audit 2023-10-01T10:14:46.657452+0000 mon.a (mon.0) 621 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:14:48.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:14:47 smithi173 bash[27612]: audit 2023-10-01T10:14:46.901846+0000 mon.a (mon.0) 622 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:14:48.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:14:47 smithi173 bash[27612]: audit 2023-10-01T10:14:46.903368+0000 mon.a (mon.0) 623 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:14:48.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:14:47 smithi173 bash[27612]: audit 2023-10-01T10:14:46.910978+0000 mon.a (mon.0) 624 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:14:48.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:14:47 smithi173 bash[27612]: cephadm 2023-10-01T10:14:46.915352+0000 mgr.y (mgr.14427) 62 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-10-01T10:14:48.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:14:47 smithi173 bash[27612]: cephadm 2023-10-01T10:14:46.919228+0000 mgr.y (mgr.14427) 63 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi078 2023-10-01T10:14:48.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:14:47 smithi173 bash[27612]: cluster 2023-10-01T10:14:47.177025+0000 mgr.y (mgr.14427) 64 : cluster [DBG] pgmap v52: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:50.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:14:50 smithi078 bash[27336]: cluster 2023-10-01T10:14:49.177666+0000 mgr.y (mgr.14427) 65 : cluster [DBG] pgmap v53: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:50.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:14:50 smithi078 bash[33821]: cluster 2023-10-01T10:14:49.177666+0000 mgr.y (mgr.14427) 65 : cluster [DBG] pgmap v53: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:50.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:14:50 smithi173 bash[27612]: cluster 2023-10-01T10:14:49.177666+0000 mgr.y (mgr.14427) 65 : cluster [DBG] pgmap v53: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:52.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:14:51 smithi078 bash[27336]: cluster 2023-10-01T10:14:51.178149+0000 mgr.y (mgr.14427) 66 : cluster [DBG] pgmap v54: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:52.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:14:51 smithi078 bash[33821]: cluster 2023-10-01T10:14:51.178149+0000 mgr.y (mgr.14427) 66 : cluster [DBG] pgmap v54: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:52.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:14:51 smithi173 bash[27612]: cluster 2023-10-01T10:14:51.178149+0000 mgr.y (mgr.14427) 66 : cluster [DBG] pgmap v54: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:54.522 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:14:54 smithi078 bash[33821]: cluster 2023-10-01T10:14:53.178543+0000 mgr.y (mgr.14427) 67 : cluster [DBG] pgmap v55: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:54.523 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:14:54 smithi078 bash[27336]: cluster 2023-10-01T10:14:53.178543+0000 mgr.y (mgr.14427) 67 : cluster [DBG] pgmap v55: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:54.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:14:54 smithi173 bash[27612]: cluster 2023-10-01T10:14:53.178543+0000 mgr.y (mgr.14427) 67 : cluster [DBG] pgmap v55: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:55.333 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:14:55 smithi078 bash[27601]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:14:55] "GET /metrics HTTP/1.1" 200 155663 "" "Prometheus/2.33.4" 2023-10-01T10:14:55.620 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:14:55 smithi173 bash[28473]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:14:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T10:14:56.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:14:56 smithi078 bash[27336]: cluster 2023-10-01T10:14:55.179077+0000 mgr.y (mgr.14427) 68 : cluster [DBG] pgmap v56: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:56.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:14:56 smithi078 bash[33821]: cluster 2023-10-01T10:14:55.179077+0000 mgr.y (mgr.14427) 68 : cluster [DBG] pgmap v56: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:56.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:14:56 smithi173 bash[27612]: cluster 2023-10-01T10:14:55.179077+0000 mgr.y (mgr.14427) 68 : cluster [DBG] pgmap v56: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:58.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:14:58 smithi078 bash[33821]: cluster 2023-10-01T10:14:57.179523+0000 mgr.y (mgr.14427) 69 : cluster [DBG] pgmap v57: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:58.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:14:58 smithi078 bash[27336]: cluster 2023-10-01T10:14:57.179523+0000 mgr.y (mgr.14427) 69 : cluster [DBG] pgmap v57: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:14:58.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:14:58 smithi173 bash[27612]: cluster 2023-10-01T10:14:57.179523+0000 mgr.y (mgr.14427) 69 : cluster [DBG] pgmap v57: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:00.449 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:00 smithi078 bash[27336]: cluster 2023-10-01T10:14:59.179979+0000 mgr.y (mgr.14427) 70 : cluster [DBG] pgmap v58: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:00.449 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:00 smithi078 bash[33821]: cluster 2023-10-01T10:14:59.179979+0000 mgr.y (mgr.14427) 70 : cluster [DBG] pgmap v58: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:00.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:00 smithi173 bash[27612]: cluster 2023-10-01T10:14:59.179979+0000 mgr.y (mgr.14427) 70 : cluster [DBG] pgmap v58: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:00.668 INFO:teuthology.orchestra.run.smithi078.stdout:55834574927 2023-10-01T10:15:00.800 INFO:teuthology.orchestra.run.smithi078.stdout:180388626470 2023-10-01T10:15:01.174 INFO:teuthology.orchestra.run.smithi078.stdout:154618822700 2023-10-01T10:15:01.471 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:01 smithi078 bash[27336]: audit 2023-10-01T10:15:00.666239+0000 mon.c (mon.2) 22 : audit [DBG] from='client.? 172.21.15.78:0/3184111784' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-01T10:15:01.471 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:01 smithi078 bash[27336]: audit 2023-10-01T10:15:00.798082+0000 mon.c (mon.2) 23 : audit [DBG] from='client.? 172.21.15.78:0/1598483417' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-01T10:15:01.471 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:01 smithi078 bash[27336]: audit 2023-10-01T10:15:01.171711+0000 mon.c (mon.2) 24 : audit [DBG] from='client.? 172.21.15.78:0/3816196397' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-01T10:15:01.471 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:01 smithi078 bash[33821]: audit 2023-10-01T10:15:00.666239+0000 mon.c (mon.2) 22 : audit [DBG] from='client.? 172.21.15.78:0/3184111784' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-01T10:15:01.471 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:01 smithi078 bash[33821]: audit 2023-10-01T10:15:00.798082+0000 mon.c (mon.2) 23 : audit [DBG] from='client.? 172.21.15.78:0/1598483417' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-01T10:15:01.471 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:01 smithi078 bash[33821]: audit 2023-10-01T10:15:01.171711+0000 mon.c (mon.2) 24 : audit [DBG] from='client.? 172.21.15.78:0/3816196397' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-01T10:15:01.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:01 smithi173 bash[27612]: audit 2023-10-01T10:15:00.666239+0000 mon.c (mon.2) 22 : audit [DBG] from='client.? 172.21.15.78:0/3184111784' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-01T10:15:01.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:01 smithi173 bash[27612]: audit 2023-10-01T10:15:00.798082+0000 mon.c (mon.2) 23 : audit [DBG] from='client.? 172.21.15.78:0/1598483417' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-01T10:15:01.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:01 smithi173 bash[27612]: audit 2023-10-01T10:15:01.171711+0000 mon.c (mon.2) 24 : audit [DBG] from='client.? 172.21.15.78:0/3816196397' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-01T10:15:02.476 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:02 smithi078 bash[33821]: cluster 2023-10-01T10:15:01.180345+0000 mgr.y (mgr.14427) 71 : cluster [DBG] pgmap v59: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:02.477 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:02 smithi078 bash[27336]: cluster 2023-10-01T10:15:01.180345+0000 mgr.y (mgr.14427) 71 : cluster [DBG] pgmap v59: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:02.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:02 smithi173 bash[27612]: cluster 2023-10-01T10:15:01.180345+0000 mgr.y (mgr.14427) 71 : cluster [DBG] pgmap v59: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:02.773 INFO:teuthology.orchestra.run.smithi078.stdout:107374182462 2023-10-01T10:15:02.862 INFO:teuthology.orchestra.run.smithi078.stdout:206158430236 2023-10-01T10:15:03.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:03 smithi078 bash[27336]: audit 2023-10-01T10:15:02.770766+0000 mon.c (mon.2) 25 : audit [DBG] from='client.? 172.21.15.78:0/43796119' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-01T10:15:03.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:03 smithi078 bash[27336]: audit 2023-10-01T10:15:02.860135+0000 mon.a (mon.0) 625 : audit [DBG] from='client.? 172.21.15.78:0/2734104658' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-01T10:15:03.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:03 smithi078 bash[33821]: audit 2023-10-01T10:15:02.770766+0000 mon.c (mon.2) 25 : audit [DBG] from='client.? 172.21.15.78:0/43796119' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-01T10:15:03.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:03 smithi078 bash[33821]: audit 2023-10-01T10:15:02.860135+0000 mon.a (mon.0) 625 : audit [DBG] from='client.? 172.21.15.78:0/2734104658' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-01T10:15:03.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:03 smithi173 bash[27612]: audit 2023-10-01T10:15:02.770766+0000 mon.c (mon.2) 25 : audit [DBG] from='client.? 172.21.15.78:0/43796119' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-01T10:15:03.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:03 smithi173 bash[27612]: audit 2023-10-01T10:15:02.860135+0000 mon.a (mon.0) 625 : audit [DBG] from='client.? 172.21.15.78:0/2734104658' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-01T10:15:04.380 INFO:tasks.cephadm.ceph_manager.ceph:need seq 180388626465 got 180388626470 for osd.6 2023-10-01T10:15:04.381 DEBUG:teuthology.parallel:result is None 2023-10-01T10:15:04.383 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574922 got 55834574927 for osd.1 2023-10-01T10:15:04.383 DEBUG:teuthology.parallel:result is None 2023-10-01T10:15:04.524 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:04 smithi078 bash[33821]: cluster 2023-10-01T10:15:03.180904+0000 mgr.y (mgr.14427) 72 : cluster [DBG] pgmap v60: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:04.525 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:04 smithi078 bash[27336]: cluster 2023-10-01T10:15:03.180904+0000 mgr.y (mgr.14427) 72 : cluster [DBG] pgmap v60: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:04.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:04 smithi173 bash[27612]: cluster 2023-10-01T10:15:03.180904+0000 mgr.y (mgr.14427) 72 : cluster [DBG] pgmap v60: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:05.333 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:15:05 smithi078 bash[27601]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:15:05] "GET /metrics HTTP/1.1" 200 155648 "" "Prometheus/2.33.4" 2023-10-01T10:15:05.620 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:15:05 smithi173 bash[28473]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:15:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T10:15:05.832 INFO:tasks.cephadm.ceph_manager.ceph:need seq 154618822695 got 154618822700 for osd.5 2023-10-01T10:15:05.832 DEBUG:teuthology.parallel:result is None 2023-10-01T10:15:06.067 INFO:teuthology.orchestra.run.smithi078.stdout:128849018935 2023-10-01T10:15:06.103 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430232 got 206158430236 for osd.7 2023-10-01T10:15:06.103 DEBUG:teuthology.parallel:result is None 2023-10-01T10:15:06.196 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182458 got 107374182462 for osd.3 2023-10-01T10:15:06.196 DEBUG:teuthology.parallel:result is None 2023-10-01T10:15:06.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:06 smithi078 bash[33821]: cluster 2023-10-01T10:15:05.181387+0000 mgr.y (mgr.14427) 73 : cluster [DBG] pgmap v61: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:06.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:06 smithi078 bash[33821]: audit 2023-10-01T10:15:06.065183+0000 mon.c (mon.2) 26 : audit [DBG] from='client.? 172.21.15.78:0/1307476896' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-01T10:15:06.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:06 smithi078 bash[27336]: cluster 2023-10-01T10:15:05.181387+0000 mgr.y (mgr.14427) 73 : cluster [DBG] pgmap v61: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:06.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:06 smithi078 bash[27336]: audit 2023-10-01T10:15:06.065183+0000 mon.c (mon.2) 26 : audit [DBG] from='client.? 172.21.15.78:0/1307476896' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-01T10:15:06.476 INFO:teuthology.orchestra.run.smithi078.stdout:77309411398 2023-10-01T10:15:06.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:06 smithi173 bash[27612]: cluster 2023-10-01T10:15:05.181387+0000 mgr.y (mgr.14427) 73 : cluster [DBG] pgmap v61: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:06.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:06 smithi173 bash[27612]: audit 2023-10-01T10:15:06.065183+0000 mon.c (mon.2) 26 : audit [DBG] from='client.? 172.21.15.78:0/1307476896' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-01T10:15:06.781 INFO:teuthology.orchestra.run.smithi078.stdout:34359738457 2023-10-01T10:15:07.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:07 smithi078 bash[33821]: audit 2023-10-01T10:15:06.473805+0000 mon.c (mon.2) 27 : audit [DBG] from='client.? 172.21.15.78:0/1611506764' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-01T10:15:07.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:07 smithi078 bash[33821]: audit 2023-10-01T10:15:06.779004+0000 mon.c (mon.2) 28 : audit [DBG] from='client.? 172.21.15.78:0/1039024960' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-01T10:15:07.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:07 smithi078 bash[27336]: audit 2023-10-01T10:15:06.473805+0000 mon.c (mon.2) 27 : audit [DBG] from='client.? 172.21.15.78:0/1611506764' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-01T10:15:07.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:07 smithi078 bash[27336]: audit 2023-10-01T10:15:06.779004+0000 mon.c (mon.2) 28 : audit [DBG] from='client.? 172.21.15.78:0/1039024960' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-01T10:15:07.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:07 smithi173 bash[27612]: audit 2023-10-01T10:15:06.473805+0000 mon.c (mon.2) 27 : audit [DBG] from='client.? 172.21.15.78:0/1611506764' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-01T10:15:07.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:07 smithi173 bash[27612]: audit 2023-10-01T10:15:06.779004+0000 mon.c (mon.2) 28 : audit [DBG] from='client.? 172.21.15.78:0/1039024960' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-01T10:15:08.011 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018930 got 128849018935 for osd.4 2023-10-01T10:15:08.011 DEBUG:teuthology.parallel:result is None 2023-10-01T10:15:08.334 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411393 got 77309411398 for osd.2 2023-10-01T10:15:08.334 DEBUG:teuthology.parallel:result is None 2023-10-01T10:15:08.448 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:08 smithi078 bash[27336]: cluster 2023-10-01T10:15:07.181880+0000 mgr.y (mgr.14427) 74 : cluster [DBG] pgmap v62: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:08.449 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:08 smithi078 bash[33821]: cluster 2023-10-01T10:15:07.181880+0000 mgr.y (mgr.14427) 74 : cluster [DBG] pgmap v62: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:08.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:08 smithi173 bash[27612]: cluster 2023-10-01T10:15:07.181880+0000 mgr.y (mgr.14427) 74 : cluster [DBG] pgmap v62: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:09.071 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738452 got 34359738457 for osd.0 2023-10-01T10:15:09.071 DEBUG:teuthology.parallel:result is None 2023-10-01T10:15:09.071 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-10-01T10:15:09.072 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T10:15:09.320 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:Oct 01 10:15:09 smithi078 systemd[1]: Stopping Ceph alertmanager.a for d8016480-6041-11ee-8db5-212e2dc638e7... 2023-10-01T10:15:09.320 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:Oct 01 10:15:09 smithi078 bash[58081]: Error response from daemon: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-alertmanager.a 2023-10-01T10:15:09.320 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:Oct 01 10:15:09 smithi078 bash[53177]: level=info ts=2023-10-01T10:15:09.285Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2023-10-01T10:15:10.078 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:Oct 01 10:15:09 smithi078 bash[58118]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-alertmanager-a 2023-10-01T10:15:10.078 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:Oct 01 10:15:10 smithi078 bash[58202]: Error response from daemon: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-alertmanager-a 2023-10-01T10:15:10.079 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:Oct 01 10:15:10 smithi078 systemd[1]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-10-01T10:15:10.079 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:Oct 01 10:15:10 smithi078 systemd[1]: Stopped Ceph alertmanager.a for d8016480-6041-11ee-8db5-212e2dc638e7. 2023-10-01T10:15:10.079 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:Oct 01 10:15:10 smithi078 systemd[1]: Started Ceph alertmanager.a for d8016480-6041-11ee-8db5-212e2dc638e7. 2023-10-01T10:15:10.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:10 smithi078 bash[33821]: cluster 2023-10-01T10:15:09.182313+0000 mgr.y (mgr.14427) 75 : cluster [DBG] pgmap v63: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:10.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:10 smithi078 bash[33821]: audit 2023-10-01T10:15:10.050209+0000 mon.a (mon.0) 626 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:10.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:10 smithi078 bash[27336]: cluster 2023-10-01T10:15:09.182313+0000 mgr.y (mgr.14427) 75 : cluster [DBG] pgmap v63: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:10.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:10 smithi078 bash[27336]: audit 2023-10-01T10:15:10.050209+0000 mon.a (mon.0) 626 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:10.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:10 smithi173 bash[27612]: cluster 2023-10-01T10:15:09.182313+0000 mgr.y (mgr.14427) 75 : cluster [DBG] pgmap v63: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:10.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:10 smithi173 bash[27612]: audit 2023-10-01T10:15:10.050209+0000 mon.a (mon.0) 626 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:11.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:11 smithi078 bash[27336]: cephadm 2023-10-01T10:15:10.053015+0000 mgr.y (mgr.14427) 76 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-01T10:15:11.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:11 smithi078 bash[27336]: cephadm 2023-10-01T10:15:10.058667+0000 mgr.y (mgr.14427) 77 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi173 2023-10-01T10:15:11.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:11 smithi078 bash[33821]: cephadm 2023-10-01T10:15:10.053015+0000 mgr.y (mgr.14427) 76 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-01T10:15:11.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:11 smithi078 bash[33821]: cephadm 2023-10-01T10:15:10.058667+0000 mgr.y (mgr.14427) 77 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi173 2023-10-01T10:15:11.584 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:Oct 01 10:15:11 smithi078 bash[58231]: level=info ts=2023-10-01T10:15:11.256Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2023-10-01T10:15:11.584 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:Oct 01 10:15:11 smithi078 bash[58231]: level=info ts=2023-10-01T10:15:11.256Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2023-10-01T10:15:11.584 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:Oct 01 10:15:11 smithi078 bash[58231]: level=info ts=2023-10-01T10:15:11.260Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.78 port=9094 2023-10-01T10:15:11.585 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:Oct 01 10:15:11 smithi078 bash[58231]: level=info ts=2023-10-01T10:15:11.261Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-10-01T10:15:11.585 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:Oct 01 10:15:11 smithi078 bash[58231]: level=info ts=2023-10-01T10:15:11.307Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-01T10:15:11.585 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:Oct 01 10:15:11 smithi078 bash[58231]: level=info ts=2023-10-01T10:15:11.307Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-01T10:15:11.585 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:Oct 01 10:15:11 smithi078 bash[58231]: level=info ts=2023-10-01T10:15:11.311Z caller=main.go:518 msg=Listening address=:9093 2023-10-01T10:15:11.585 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:Oct 01 10:15:11 smithi078 bash[58231]: level=info ts=2023-10-01T10:15:11.311Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2023-10-01T10:15:11.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:11 smithi173 bash[27612]: cephadm 2023-10-01T10:15:10.053015+0000 mgr.y (mgr.14427) 76 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-01T10:15:11.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:11 smithi173 bash[27612]: cephadm 2023-10-01T10:15:10.058667+0000 mgr.y (mgr.14427) 77 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi173 2023-10-01T10:15:12.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:12 smithi078 bash[27336]: cluster 2023-10-01T10:15:11.182859+0000 mgr.y (mgr.14427) 78 : cluster [DBG] pgmap v64: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:12.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:12 smithi078 bash[33821]: cluster 2023-10-01T10:15:11.182859+0000 mgr.y (mgr.14427) 78 : cluster [DBG] pgmap v64: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:12.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:12 smithi173 bash[27612]: cluster 2023-10-01T10:15:11.182859+0000 mgr.y (mgr.14427) 78 : cluster [DBG] pgmap v64: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:13.583 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:Oct 01 10:15:13 smithi078 bash[58231]: level=info ts=2023-10-01T10:15:13.261Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000040308s 2023-10-01T10:15:13.826 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:15:14.265 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:15:13 smithi173 systemd[1]: Stopping Ceph prometheus.a for d8016480-6041-11ee-8db5-212e2dc638e7... 2023-10-01T10:15:14.265 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:15:13 smithi173 bash[48144]: Error response from daemon: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-prometheus.a 2023-10-01T10:15:14.266 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:15:14 smithi173 bash[46761]: ts=2023-10-01T10:15:14.069Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-10-01T10:15:14.266 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:15:14 smithi173 bash[46761]: ts=2023-10-01T10:15:14.070Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2023-10-01T10:15:14.266 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:15:14 smithi173 bash[46761]: ts=2023-10-01T10:15:14.070Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2023-10-01T10:15:14.266 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:15:14 smithi173 bash[46761]: ts=2023-10-01T10:15:14.070Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2023-10-01T10:15:14.266 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:15:14 smithi173 bash[46761]: ts=2023-10-01T10:15:14.070Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2023-10-01T10:15:14.266 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:15:14 smithi173 bash[46761]: ts=2023-10-01T10:15:14.070Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2023-10-01T10:15:14.266 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:15:14 smithi173 bash[46761]: ts=2023-10-01T10:15:14.070Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2023-10-01T10:15:14.266 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:15:14 smithi173 bash[46761]: ts=2023-10-01T10:15:14.070Z caller=main.go:828 level=info msg="Scrape manager stopped" 2023-10-01T10:15:14.266 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:15:14 smithi173 bash[46761]: ts=2023-10-01T10:15:14.070Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2023-10-01T10:15:14.266 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:15:14 smithi173 bash[46761]: ts=2023-10-01T10:15:14.071Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2023-10-01T10:15:14.266 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:15:14 smithi173 bash[46761]: ts=2023-10-01T10:15:14.071Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2023-10-01T10:15:14.266 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:15:14 smithi173 bash[46761]: ts=2023-10-01T10:15:14.071Z caller=main.go:1066 level=info msg="See you next time!" 2023-10-01T10:15:14.531 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:14 smithi173 bash[27612]: cluster 2023-10-01T10:15:13.183564+0000 mgr.y (mgr.14427) 79 : cluster [DBG] pgmap v65: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:14.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:14 smithi078 bash[33821]: cluster 2023-10-01T10:15:13.183564+0000 mgr.y (mgr.14427) 79 : cluster [DBG] pgmap v65: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:14.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:14 smithi078 bash[27336]: cluster 2023-10-01T10:15:13.183564+0000 mgr.y (mgr.14427) 79 : cluster [DBG] pgmap v65: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:14.791 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:15:14 smithi173 bash[48177]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-prometheus-a 2023-10-01T10:15:14.792 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:15:14 smithi173 bash[48243]: Error response from daemon: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-prometheus-a 2023-10-01T10:15:14.792 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:15:14 smithi173 systemd[1]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7@prometheus.a.service: Succeeded. 2023-10-01T10:15:14.792 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:15:14 smithi173 systemd[1]: Stopped Ceph prometheus.a for d8016480-6041-11ee-8db5-212e2dc638e7. 2023-10-01T10:15:14.792 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:15:14 smithi173 systemd[1]: Started Ceph prometheus.a for d8016480-6041-11ee-8db5-212e2dc638e7. 2023-10-01T10:15:15.057 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:15:14 smithi078 bash[27601]: [01/Oct/2023:10:15:14] ENGINE Bus STOPPING 2023-10-01T10:15:15.057 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:15:14 smithi078 bash[27601]: [01/Oct/2023:10:15:14] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-01T10:15:15.057 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:15:14 smithi078 bash[27601]: [01/Oct/2023:10:15:14] ENGINE Bus STOPPED 2023-10-01T10:15:15.057 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:15:14 smithi078 bash[27601]: [01/Oct/2023:10:15:14] ENGINE Bus STARTING 2023-10-01T10:15:15.057 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:15:14 smithi078 bash[27601]: [01/Oct/2023:10:15:14] ENGINE Serving on http://:::9283 2023-10-01T10:15:15.057 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:15:14 smithi078 bash[27601]: [01/Oct/2023:10:15:14] ENGINE Bus STARTED 2023-10-01T10:15:15.057 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:15:14 smithi078 bash[27601]: [01/Oct/2023:10:15:14] ENGINE Bus STOPPING 2023-10-01T10:15:15.058 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:15:14 smithi078 bash[27601]: [01/Oct/2023:10:15:14] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-01T10:15:15.058 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:15:14 smithi078 bash[27601]: [01/Oct/2023:10:15:14] ENGINE Bus STOPPED 2023-10-01T10:15:15.058 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:15:14 smithi078 bash[27601]: [01/Oct/2023:10:15:14] ENGINE Bus STARTING 2023-10-01T10:15:15.058 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:15:15 smithi078 bash[27601]: [01/Oct/2023:10:15:15] ENGINE Serving on http://:::9283 2023-10-01T10:15:15.058 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:15:15 smithi078 bash[27601]: [01/Oct/2023:10:15:15] ENGINE Bus STARTED 2023-10-01T10:15:15.058 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:15:15 smithi078 bash[27601]: [01/Oct/2023:10:15:15] ENGINE Bus STOPPING 2023-10-01T10:15:15.058 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:15:15 smithi078 bash[27601]: [01/Oct/2023:10:15:15] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-01T10:15:15.058 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:15:15 smithi078 bash[27601]: [01/Oct/2023:10:15:15] ENGINE Bus STOPPED 2023-10-01T10:15:15.333 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:15:15 smithi078 bash[27601]: [01/Oct/2023:10:15:15] ENGINE Bus STARTING 2023-10-01T10:15:15.352 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:15:15 smithi078 bash[27601]: [01/Oct/2023:10:15:15] ENGINE Serving on http://:::9283 2023-10-01T10:15:15.353 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:15:15 smithi078 bash[27601]: [01/Oct/2023:10:15:15] ENGINE Bus STARTED 2023-10-01T10:15:15.868 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:15:15 smithi173 bash[48271]: ts=2023-10-01T10:15:15.730Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2023-10-01T10:15:15.869 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:15:15 smithi173 bash[48271]: ts=2023-10-01T10:15:15.730Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2023-10-01T10:15:15.869 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:15:15 smithi173 bash[48271]: ts=2023-10-01T10:15:15.730Z caller=main.go:518 level=info host_details="(Linux 5.15.0-84-generic #93~20.04.1-Ubuntu SMP Wed Sep 6 16:15:40 UTC 2023 x86_64 smithi173 (none))" 2023-10-01T10:15:15.869 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:15:15 smithi173 bash[48271]: ts=2023-10-01T10:15:15.730Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-10-01T10:15:15.869 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:15:15 smithi173 bash[48271]: ts=2023-10-01T10:15:15.730Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-10-01T10:15:15.869 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:15:15 smithi173 bash[48271]: ts=2023-10-01T10:15:15.734Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2023-10-01T10:15:15.869 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:15:15 smithi173 bash[48271]: ts=2023-10-01T10:15:15.734Z caller=main.go:923 level=info msg="Starting TSDB ..." 2023-10-01T10:15:15.869 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:15:15 smithi173 bash[48271]: ts=2023-10-01T10:15:15.735Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2023-10-01T10:15:15.869 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:15:15 smithi173 bash[48271]: ts=2023-10-01T10:15:15.739Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-10-01T10:15:15.869 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:15:15 smithi173 bash[48271]: ts=2023-10-01T10:15:15.739Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.669µs 2023-10-01T10:15:15.869 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:15:15 smithi173 bash[48271]: ts=2023-10-01T10:15:15.739Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-10-01T10:15:15.870 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:15:15 smithi173 bash[48271]: ts=2023-10-01T10:15:15.866Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2023-10-01T10:15:15.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:15 smithi173 bash[27612]: audit 2023-10-01T10:15:14.767811+0000 mon.a (mon.0) 627 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:15.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:15 smithi173 bash[27612]: audit 2023-10-01T10:15:14.772275+0000 mon.a (mon.0) 628 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-01T10:15:15.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:15 smithi173 bash[27612]: audit 2023-10-01T10:15:14.772973+0000 mgr.y (mgr.14427) 80 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-01T10:15:15.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:15 smithi173 bash[27612]: audit 2023-10-01T10:15:14.774524+0000 mon.a (mon.0) 629 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi078.front.sepia.ceph.com:9093"}]: dispatch 2023-10-01T10:15:15.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:15 smithi173 bash[27612]: audit 2023-10-01T10:15:14.775099+0000 mgr.y (mgr.14427) 81 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi078.front.sepia.ceph.com:9093"}]: dispatch 2023-10-01T10:15:15.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:15 smithi173 bash[27612]: audit 2023-10-01T10:15:14.781383+0000 mon.a (mon.0) 630 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:15.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:15 smithi173 bash[27612]: audit 2023-10-01T10:15:14.792343+0000 mon.a (mon.0) 631 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-01T10:15:15.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:15 smithi173 bash[27612]: audit 2023-10-01T10:15:14.793312+0000 mgr.y (mgr.14427) 82 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-01T10:15:15.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:15 smithi173 bash[27612]: audit 2023-10-01T10:15:14.799668+0000 mon.a (mon.0) 632 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi173.front.sepia.ceph.com:9095"}]: dispatch 2023-10-01T10:15:15.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:15 smithi173 bash[27612]: audit 2023-10-01T10:15:14.800326+0000 mgr.y (mgr.14427) 83 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi173.front.sepia.ceph.com:9095"}]: dispatch 2023-10-01T10:15:15.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:15 smithi173 bash[27612]: audit 2023-10-01T10:15:14.806577+0000 mon.a (mon.0) 633 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:15.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:15 smithi173 bash[27612]: audit 2023-10-01T10:15:14.814062+0000 mon.a (mon.0) 634 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-01T10:15:15.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:15 smithi173 bash[27612]: audit 2023-10-01T10:15:14.814615+0000 mgr.y (mgr.14427) 84 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-01T10:15:15.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:15 smithi173 bash[27612]: audit 2023-10-01T10:15:14.821600+0000 mon.a (mon.0) 635 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi173.front.sepia.ceph.com:3000"}]: dispatch 2023-10-01T10:15:15.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:15 smithi173 bash[27612]: audit 2023-10-01T10:15:14.822401+0000 mgr.y (mgr.14427) 85 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi173.front.sepia.ceph.com:3000"}]: dispatch 2023-10-01T10:15:15.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:15 smithi173 bash[27612]: audit 2023-10-01T10:15:14.830794+0000 mon.a (mon.0) 636 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:15.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:15 smithi173 bash[27612]: audit 2023-10-01T10:15:14.835291+0000 mon.a (mon.0) 637 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:15:15.872 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:15 smithi173 bash[27612]: cluster 2023-10-01T10:15:15.184003+0000 mgr.y (mgr.14427) 86 : cluster [DBG] pgmap v66: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:16.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:15 smithi078 bash[27336]: audit 2023-10-01T10:15:14.767811+0000 mon.a (mon.0) 627 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:16.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:15 smithi078 bash[27336]: audit 2023-10-01T10:15:14.772275+0000 mon.a (mon.0) 628 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-01T10:15:16.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:15 smithi078 bash[27336]: audit 2023-10-01T10:15:14.772973+0000 mgr.y (mgr.14427) 80 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-01T10:15:16.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:15 smithi078 bash[27336]: audit 2023-10-01T10:15:14.774524+0000 mon.a (mon.0) 629 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi078.front.sepia.ceph.com:9093"}]: dispatch 2023-10-01T10:15:16.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:15 smithi078 bash[27336]: audit 2023-10-01T10:15:14.775099+0000 mgr.y (mgr.14427) 81 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi078.front.sepia.ceph.com:9093"}]: dispatch 2023-10-01T10:15:16.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:15 smithi078 bash[27336]: audit 2023-10-01T10:15:14.781383+0000 mon.a (mon.0) 630 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:16.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:15 smithi078 bash[27336]: audit 2023-10-01T10:15:14.792343+0000 mon.a (mon.0) 631 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-01T10:15:16.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:15 smithi078 bash[27336]: audit 2023-10-01T10:15:14.793312+0000 mgr.y (mgr.14427) 82 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-01T10:15:16.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:15 smithi078 bash[27336]: audit 2023-10-01T10:15:14.799668+0000 mon.a (mon.0) 632 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi173.front.sepia.ceph.com:9095"}]: dispatch 2023-10-01T10:15:16.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:15 smithi078 bash[27336]: audit 2023-10-01T10:15:14.800326+0000 mgr.y (mgr.14427) 83 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi173.front.sepia.ceph.com:9095"}]: dispatch 2023-10-01T10:15:16.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:15 smithi078 bash[27336]: audit 2023-10-01T10:15:14.806577+0000 mon.a (mon.0) 633 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:16.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:15 smithi078 bash[27336]: audit 2023-10-01T10:15:14.814062+0000 mon.a (mon.0) 634 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-01T10:15:16.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:15 smithi078 bash[27336]: audit 2023-10-01T10:15:14.814615+0000 mgr.y (mgr.14427) 84 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-01T10:15:16.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:15 smithi078 bash[27336]: audit 2023-10-01T10:15:14.821600+0000 mon.a (mon.0) 635 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi173.front.sepia.ceph.com:3000"}]: dispatch 2023-10-01T10:15:16.085 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:15 smithi078 bash[27336]: audit 2023-10-01T10:15:14.822401+0000 mgr.y (mgr.14427) 85 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi173.front.sepia.ceph.com:3000"}]: dispatch 2023-10-01T10:15:16.085 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:15 smithi078 bash[27336]: audit 2023-10-01T10:15:14.830794+0000 mon.a (mon.0) 636 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:16.085 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:15 smithi078 bash[27336]: audit 2023-10-01T10:15:14.835291+0000 mon.a (mon.0) 637 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:15:16.085 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:15 smithi078 bash[27336]: cluster 2023-10-01T10:15:15.184003+0000 mgr.y (mgr.14427) 86 : cluster [DBG] pgmap v66: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:16.085 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:15 smithi078 bash[33821]: audit 2023-10-01T10:15:14.767811+0000 mon.a (mon.0) 627 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:16.085 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:15 smithi078 bash[33821]: audit 2023-10-01T10:15:14.772275+0000 mon.a (mon.0) 628 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-01T10:15:16.085 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:15 smithi078 bash[33821]: audit 2023-10-01T10:15:14.772973+0000 mgr.y (mgr.14427) 80 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-01T10:15:16.085 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:15 smithi078 bash[33821]: audit 2023-10-01T10:15:14.774524+0000 mon.a (mon.0) 629 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi078.front.sepia.ceph.com:9093"}]: dispatch 2023-10-01T10:15:16.085 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:15 smithi078 bash[33821]: audit 2023-10-01T10:15:14.775099+0000 mgr.y (mgr.14427) 81 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi078.front.sepia.ceph.com:9093"}]: dispatch 2023-10-01T10:15:16.085 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:15 smithi078 bash[33821]: audit 2023-10-01T10:15:14.781383+0000 mon.a (mon.0) 630 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:16.086 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:15 smithi078 bash[33821]: audit 2023-10-01T10:15:14.792343+0000 mon.a (mon.0) 631 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-01T10:15:16.086 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:15 smithi078 bash[33821]: audit 2023-10-01T10:15:14.793312+0000 mgr.y (mgr.14427) 82 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-01T10:15:16.086 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:15 smithi078 bash[33821]: audit 2023-10-01T10:15:14.799668+0000 mon.a (mon.0) 632 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi173.front.sepia.ceph.com:9095"}]: dispatch 2023-10-01T10:15:16.086 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:15 smithi078 bash[33821]: audit 2023-10-01T10:15:14.800326+0000 mgr.y (mgr.14427) 83 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi173.front.sepia.ceph.com:9095"}]: dispatch 2023-10-01T10:15:16.086 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:15 smithi078 bash[33821]: audit 2023-10-01T10:15:14.806577+0000 mon.a (mon.0) 633 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:16.086 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:15 smithi078 bash[33821]: audit 2023-10-01T10:15:14.814062+0000 mon.a (mon.0) 634 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-01T10:15:16.086 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:15 smithi078 bash[33821]: audit 2023-10-01T10:15:14.814615+0000 mgr.y (mgr.14427) 84 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-01T10:15:16.086 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:15 smithi078 bash[33821]: audit 2023-10-01T10:15:14.821600+0000 mon.a (mon.0) 635 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi173.front.sepia.ceph.com:3000"}]: dispatch 2023-10-01T10:15:16.086 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:15 smithi078 bash[33821]: audit 2023-10-01T10:15:14.822401+0000 mgr.y (mgr.14427) 85 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi173.front.sepia.ceph.com:3000"}]: dispatch 2023-10-01T10:15:16.086 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:15 smithi078 bash[33821]: audit 2023-10-01T10:15:14.830794+0000 mon.a (mon.0) 636 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:16.086 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:15 smithi078 bash[33821]: audit 2023-10-01T10:15:14.835291+0000 mon.a (mon.0) 637 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:15:16.087 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:15 smithi078 bash[33821]: cluster 2023-10-01T10:15:15.184003+0000 mgr.y (mgr.14427) 86 : cluster [DBG] pgmap v66: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:16.120 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:15:15 smithi173 bash[48271]: ts=2023-10-01T10:15:15.867Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2023-10-01T10:15:16.120 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:15:15 smithi173 bash[48271]: ts=2023-10-01T10:15:15.867Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=48.39µs wal_replay_duration=127.387061ms total_replay_duration=127.460248ms 2023-10-01T10:15:16.120 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:15:15 smithi173 bash[48271]: ts=2023-10-01T10:15:15.871Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2023-10-01T10:15:16.120 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:15:15 smithi173 bash[48271]: ts=2023-10-01T10:15:15.871Z caller=main.go:947 level=info msg="TSDB started" 2023-10-01T10:15:16.120 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:15:15 smithi173 bash[48271]: ts=2023-10-01T10:15:15.871Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-10-01T10:15:16.121 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:15:15 smithi173 bash[48271]: ts=2023-10-01T10:15:15.894Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=23.395901ms db_storage=1.669µs remote_storage=2.183µs web_handler=671ns query_engine=1.272µs scrape=526.804µs scrape_sd=55.356µs notify=62.293µs notify_sd=21.091µs rules=22.13182ms 2023-10-01T10:15:16.121 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:15:15 smithi173 bash[48271]: ts=2023-10-01T10:15:15.894Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2023-10-01T10:15:16.917 INFO:teuthology.orchestra.run.smithi078.stdout: 2023-10-01T10:15:16.920 INFO:teuthology.orchestra.run.smithi078.stderr:dumped all 2023-10-01T10:15:17.613 INFO:teuthology.orchestra.run.smithi078.stdout:{"pg_ready":true,"pg_map":{"version":66,"stamp":"2023-10-01T10:15:15.183803+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2380984,"kb_used_data":4216,"kb_used_omap":0,"kb_used_meta":2376704,"kb_avail":747580232,"statfs":{"total":767960285184,"available":765522157568,"internally_reserved":0,"allocated":4317184,"data_stored":1763490,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2433744896},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.000000"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-01T10:13:46.495363+0000","last_change":"2023-10-01T10:13:44.666450+0000","last_active":"2023-10-01T10:13:46.495363+0000","last_peered":"2023-10-01T10:13:46.495363+0000","last_clean":"2023-10-01T10:13:46.495363+0000","last_became_active":"2023-10-01T10:13:44.665959+0000","last_became_peered":"2023-10-01T10:13:44.665959+0000","last_unstale":"2023-10-01T10:13:46.495363+0000","last_undegraded":"2023-10-01T10:13:46.495363+0000","last_fullsized":"2023-10-01T10:13:46.495363+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-01T10:13:46.493047+0000","last_change":"2023-10-01T10:13:44.825307+0000","last_active":"2023-10-01T10:13:46.493047+0000","last_peered":"2023-10-01T10:13:46.493047+0000","last_clean":"2023-10-01T10:13:46.493047+0000","last_became_active":"2023-10-01T10:13:44.824803+0000","last_became_peered":"2023-10-01T10:13:44.824803+0000","last_unstale":"2023-10-01T10:13:46.493047+0000","last_undegraded":"2023-10-01T10:13:46.493047+0000","last_fullsized":"2023-10-01T10:13:46.493047+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-01T10:13:45.924633+0000","last_change":"2023-10-01T10:13:44.827141+0000","last_active":"2023-10-01T10:13:45.924633+0000","last_peered":"2023-10-01T10:13:45.924633+0000","last_clean":"2023-10-01T10:13:45.924633+0000","last_became_active":"2023-10-01T10:13:44.826089+0000","last_became_peered":"2023-10-01T10:13:44.826089+0000","last_unstale":"2023-10-01T10:13:45.924633+0000","last_undegraded":"2023-10-01T10:13:45.924633+0000","last_fullsized":"2023-10-01T10:13:45.924633+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-01T10:13:45.924525+0000","last_change":"2023-10-01T10:13:44.825879+0000","last_active":"2023-10-01T10:13:45.924525+0000","last_peered":"2023-10-01T10:13:45.924525+0000","last_clean":"2023-10-01T10:13:45.924525+0000","last_became_active":"2023-10-01T10:13:44.825097+0000","last_became_peered":"2023-10-01T10:13:44.825097+0000","last_unstale":"2023-10-01T10:13:45.924525+0000","last_undegraded":"2023-10-01T10:13:45.924525+0000","last_fullsized":"2023-10-01T10:13:45.924525+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":15,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-01T10:13:45.924675+0000","last_change":"2023-10-01T10:12:47.477457+0000","last_active":"2023-10-01T10:13:45.924675+0000","last_peered":"2023-10-01T10:13:45.924675+0000","last_clean":"2023-10-01T10:13:45.924675+0000","last_became_active":"2023-10-01T10:12:47.476522+0000","last_became_peered":"2023-10-01T10:12:47.476522+0000","last_unstale":"2023-10-01T10:13:45.924675+0000","last_undegraded":"2023-10-01T10:13:45.924675+0000","last_fullsized":"2023-10-01T10:13:45.924675+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:09:17.221082+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:09:17.221082+0000","last_clean_scrub_stamp":"2023-10-01T10:09:17.221082+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-01T10:13:46.495403+0000","last_change":"2023-10-01T10:13:44.663992+0000","last_active":"2023-10-01T10:13:46.495403+0000","last_peered":"2023-10-01T10:13:46.495403+0000","last_clean":"2023-10-01T10:13:46.495403+0000","last_became_active":"2023-10-01T10:13:44.663669+0000","last_became_peered":"2023-10-01T10:13:44.663669+0000","last_unstale":"2023-10-01T10:13:46.495403+0000","last_undegraded":"2023-10-01T10:13:46.495403+0000","last_fullsized":"2023-10-01T10:13:46.495403+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-01T10:13:46.493149+0000","last_change":"2023-10-01T10:13:44.823885+0000","last_active":"2023-10-01T10:13:46.493149+0000","last_peered":"2023-10-01T10:13:46.493149+0000","last_clean":"2023-10-01T10:13:46.493149+0000","last_became_active":"2023-10-01T10:13:44.823314+0000","last_became_peered":"2023-10-01T10:13:44.823314+0000","last_unstale":"2023-10-01T10:13:46.493149+0000","last_undegraded":"2023-10-01T10:13:46.493149+0000","last_fullsized":"2023-10-01T10:13:46.493149+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-01T10:13:46.495302+0000","last_change":"2023-10-01T10:13:44.663534+0000","last_active":"2023-10-01T10:13:46.495302+0000","last_peered":"2023-10-01T10:13:46.495302+0000","last_clean":"2023-10-01T10:13:46.495302+0000","last_became_active":"2023-10-01T10:13:44.663014+0000","last_became_peered":"2023-10-01T10:13:44.663014+0000","last_unstale":"2023-10-01T10:13:46.495302+0000","last_undegraded":"2023-10-01T10:13:46.495302+0000","last_fullsized":"2023-10-01T10:13:46.495302+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-01T10:13:46.493139+0000","last_change":"2023-10-01T10:13:44.824140+0000","last_active":"2023-10-01T10:13:46.493139+0000","last_peered":"2023-10-01T10:13:46.493139+0000","last_clean":"2023-10-01T10:13:46.493139+0000","last_became_active":"2023-10-01T10:13:44.823462+0000","last_became_peered":"2023-10-01T10:13:44.823462+0000","last_unstale":"2023-10-01T10:13:46.493139+0000","last_undegraded":"2023-10-01T10:13:46.493139+0000","last_fullsized":"2023-10-01T10:13:46.493139+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430239,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:14:50 2023","interfaces":[{"interface":"back","average":{"1min":0.85299999999999998,"5min":0.91400000000000003,"15min":0.92400000000000004},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.115,"5min":1.161,"15min":1.161},"last":1.0820000000000001},{"interface":"front","average":{"1min":0.82999999999999996,"5min":0.91400000000000003,"15min":0.92800000000000005},"min":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"max":{"1min":0.98199999999999998,"5min":1.198,"15min":1.198},"last":1.2629999999999999}]},{"osd":1,"last update":"Sun Oct 1 10:14:50 2023","interfaces":[{"interface":"back","average":{"1min":0.85699999999999998,"5min":0.89900000000000002,"15min":0.90600000000000003},"min":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"max":{"1min":1.0960000000000001,"5min":1.444,"15min":1.444},"last":0.93400000000000005},{"interface":"front","average":{"1min":0.88500000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"min":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"max":{"1min":1.377,"5min":1.377,"15min":1.377},"last":0.95899999999999996}]},{"osd":2,"last update":"Sun Oct 1 10:14:50 2023","interfaces":[{"interface":"back","average":{"1min":0.89200000000000002,"5min":0.92400000000000004,"15min":0.93000000000000005},"min":{"1min":0.65400000000000003,"5min":0.61899999999999999,"15min":0.61899999999999999},"max":{"1min":1.4239999999999999,"5min":1.4239999999999999,"15min":1.4239999999999999},"last":1.024},{"interface":"front","average":{"1min":0.93200000000000005,"5min":0.95199999999999996,"15min":0.95599999999999996},"min":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"max":{"1min":1.238,"5min":1.2390000000000001,"15min":1.2390000000000001},"last":1.0389999999999999}]},{"osd":3,"last update":"Sun Oct 1 10:14:50 2023","interfaces":[{"interface":"back","average":{"1min":0.88400000000000001,"5min":0.90200000000000002,"15min":0.90500000000000003},"min":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.155,"5min":1.1990000000000001,"15min":1.1990000000000001},"last":0.86199999999999999},{"interface":"front","average":{"1min":1.016,"5min":1.008,"15min":1.006},"min":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"max":{"1min":3.2229999999999999,"5min":3.2229999999999999,"15min":3.2229999999999999},"last":1.0900000000000001}]},{"osd":4,"last update":"Sun Oct 1 10:14:50 2023","interfaces":[{"interface":"back","average":{"1min":0.89600000000000002,"5min":0.92800000000000005,"15min":0.93400000000000005},"min":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":1.3169999999999999,"5min":1.361,"15min":1.361},"last":1.1619999999999999},{"interface":"front","average":{"1min":0.86299999999999999,"5min":0.91100000000000003,"15min":0.91900000000000004},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":1.5840000000000001,"5min":1.607,"15min":1.607},"last":1.357}]},{"osd":5,"last update":"Sun Oct 1 10:14:50 2023","interfaces":[{"interface":"back","average":{"1min":0.873,"5min":0.97299999999999998,"15min":0.98899999999999999},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":1.5149999999999999,"5min":1.831,"15min":1.831},"last":1.3200000000000001},{"interface":"front","average":{"1min":0.88900000000000001,"5min":1.0589999999999999,"15min":1.087},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":1.4610000000000001,"5min":1.5,"15min":1.5},"last":1.1259999999999999}]},{"osd":6,"last update":"Sun Oct 1 10:14:50 2023","interfaces":[{"interface":"back","average":{"1min":0.91700000000000004,"5min":1.0609999999999999,"15min":1.085},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":1.6240000000000001,"5min":1.637,"15min":1.637},"last":1.173},{"interface":"front","average":{"1min":0.93100000000000005,"5min":1.0780000000000001,"15min":1.1020000000000001},"min":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"max":{"1min":1.6870000000000001,"5min":2.2869999999999999,"15min":2.2869999999999999},"last":1.407}]}]},{"osd":6,"up_from":42,"seq":180388626473,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297620,"kb_used_data":524,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447532,"statfs":{"total":95995035648,"available":95690272768,"internally_reserved":0,"allocated":536576,"data_stored":220200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:15:08 2023","interfaces":[{"interface":"back","average":{"1min":0.77100000000000002,"5min":0.871,"15min":0.89000000000000001},"min":{"1min":0.52300000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":1.038,"5min":1.359,"15min":1.359},"last":1.1180000000000001},{"interface":"front","average":{"1min":0.84399999999999997,"5min":0.86499999999999999,"15min":0.85599999999999998},"min":{"1min":0.61299999999999999,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":1.1279999999999999,"5min":1.1839999999999999,"15min":1.1839999999999999},"last":1.0369999999999999}]},{"osd":1,"last update":"Sun Oct 1 10:15:08 2023","interfaces":[{"interface":"back","average":{"1min":0.80200000000000005,"5min":0.90900000000000003,"15min":0.93100000000000005},"min":{"1min":0.57499999999999996,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":1.054,"5min":1.4299999999999999,"15min":1.4299999999999999},"last":1.2030000000000001},{"interface":"front","average":{"1min":0.84799999999999998,"5min":0.94399999999999995,"15min":0.97799999999999998},"min":{"1min":0.67700000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"max":{"1min":0.995,"5min":1.298,"15min":1.298},"last":0.99399999999999999}]},{"osd":2,"last update":"Sun Oct 1 10:15:08 2023","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":0.86899999999999999,"15min":0.875},"min":{"1min":0.59299999999999997,"5min":0.57799999999999996,"15min":0.57799999999999996},"max":{"1min":1.2330000000000001,"5min":1.2889999999999999,"15min":1.2889999999999999},"last":0.78300000000000003},{"interface":"front","average":{"1min":0.86899999999999999,"5min":0.91700000000000004,"15min":0.93300000000000005},"min":{"1min":0.60599999999999998,"5min":0.57099999999999995,"15min":0.57099999999999995},"max":{"1min":1.109,"5min":1.484,"15min":1.484},"last":0.96199999999999997}]},{"osd":3,"last update":"Sun Oct 1 10:15:08 2023","interfaces":[{"interface":"back","average":{"1min":0.84399999999999997,"5min":0.93899999999999995,"15min":0.95699999999999996},"min":{"1min":0.67300000000000004,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":1.1279999999999999,"5min":1.456,"15min":1.456},"last":1.1619999999999999},{"interface":"front","average":{"1min":0.88200000000000001,"5min":0.90800000000000003,"15min":0.92100000000000004},"min":{"1min":0.68899999999999995,"5min":0.59299999999999997,"15min":0.59299999999999997},"max":{"1min":1.1479999999999999,"5min":1.266,"15min":1.266},"last":0.89200000000000002}]},{"osd":4,"last update":"Sun Oct 1 10:15:08 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":0.86899999999999999,"15min":0.871},"min":{"1min":0.58899999999999997,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.1799999999999999,"5min":1.246,"15min":1.246},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.96099999999999997,"5min":0.90900000000000003,"15min":0.90300000000000002},"min":{"1min":0.79300000000000004,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":1.143,"5min":1.349,"15min":1.349},"last":1.0589999999999999}]},{"osd":5,"last update":"Sun Oct 1 10:15:08 2023","interfaces":[{"interface":"back","average":{"1min":1.026,"5min":0.92300000000000004,"15min":0.90700000000000003},"min":{"1min":0.67500000000000004,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":1.208,"5min":1.27,"15min":1.27},"last":0.70099999999999996},{"interface":"front","average":{"1min":1.0209999999999999,"5min":0.97499999999999998,"15min":0.97299999999999998},"min":{"1min":0.65500000000000003,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":1.1990000000000001,"5min":1.631,"15min":1.631},"last":0.92300000000000004}]},{"osd":7,"last update":"Sun Oct 1 10:14:48 2023","interfaces":[{"interface":"back","average":{"1min":1.0580000000000001,"5min":1.0780000000000001,"15min":1.081},"min":{"1min":0.77000000000000002,"5min":0.59699999999999998,"15min":0.59699999999999998},"max":{"1min":1.2310000000000001,"5min":1.7250000000000001,"15min":1.7250000000000001},"last":0.752},{"interface":"front","average":{"1min":1.032,"5min":1.038,"15min":1.0389999999999999},"min":{"1min":0.77400000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":1.246,"5min":1.5,"15min":1.5},"last":0.84899999999999998}]}]},{"osd":1,"up_from":13,"seq":55834574930,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:14:42 2023","interfaces":[{"interface":"back","average":{"1min":0.64900000000000002,"5min":0.65500000000000003,"15min":0.56599999999999995},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.92900000000000005,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.83299999999999996},{"interface":"front","average":{"1min":0.70699999999999996,"5min":0.72699999999999998,"15min":0.58499999999999996},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.075,"5min":1.1919999999999999,"15min":1.1919999999999999},"last":0.748}]},{"osd":2,"last update":"Sun Oct 1 10:14:24 2023","interfaces":[{"interface":"back","average":{"1min":0.68200000000000005,"5min":0.70099999999999996,"15min":0.64200000000000002},"min":{"1min":0.45500000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.996,"5min":1.1120000000000001,"15min":1.1120000000000001},"last":0.873},{"interface":"front","average":{"1min":0.77100000000000002,"5min":0.76100000000000001,"15min":0.67900000000000005},"min":{"1min":0.41299999999999998,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":1.474,"5min":1.474,"15min":1.474},"last":0.90800000000000003}]},{"osd":3,"last update":"Sun Oct 1 10:15:04 2023","interfaces":[{"interface":"back","average":{"1min":0.753,"5min":0.83199999999999996,"15min":0.81599999999999995},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":1.5780000000000001,"5min":2.5059999999999998,"15min":2.5059999999999998},"last":0.98299999999999998},{"interface":"front","average":{"1min":0.71099999999999997,"5min":0.77400000000000002,"15min":0.78100000000000003},"min":{"1min":0.42399999999999999,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":1.5620000000000001,"5min":2.153,"15min":2.153},"last":0.94799999999999995}]},{"osd":4,"last update":"Sun Oct 1 10:14:48 2023","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":0.97599999999999998,"15min":0.97399999999999998},"min":{"1min":0.749,"5min":0.505,"15min":0.505},"max":{"1min":1.2969999999999999,"5min":1.3500000000000001,"15min":1.3500000000000001},"last":1.0309999999999999},{"interface":"front","average":{"1min":0.97199999999999998,"5min":0.97399999999999998,"15min":0.96899999999999997},"min":{"1min":0.629,"5min":0.56799999999999995,"15min":0.56799999999999995},"max":{"1min":1.341,"5min":1.4730000000000001,"15min":1.4730000000000001},"last":1.0089999999999999}]},{"osd":5,"last update":"Sun Oct 1 10:14:24 2023","interfaces":[{"interface":"back","average":{"1min":1,"5min":1.0389999999999999,"15min":1.0609999999999999},"min":{"1min":0.71299999999999997,"5min":0.65500000000000003,"15min":0.65500000000000003},"max":{"1min":1.383,"5min":1.5389999999999999,"15min":1.5389999999999999},"last":1.079},{"interface":"front","average":{"1min":1.034,"5min":1.0600000000000001,"15min":1.0620000000000001},"min":{"1min":0.754,"5min":0.64900000000000002,"15min":0.64900000000000002},"max":{"1min":1.3300000000000001,"5min":1.3300000000000001,"15min":1.3300000000000001},"last":0.81399999999999995}]},{"osd":6,"last update":"Sun Oct 1 10:15:04 2023","interfaces":[{"interface":"back","average":{"1min":1.008,"5min":0.97799999999999998,"15min":0.96399999999999997},"min":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"max":{"1min":1.6060000000000001,"5min":1.6060000000000001,"15min":1.6060000000000001},"last":1.1919999999999999},{"interface":"front","average":{"1min":1.095,"5min":1.0620000000000001,"15min":1.0600000000000001},"min":{"1min":0.78400000000000003,"5min":0.68300000000000005,"15min":0.68300000000000005},"max":{"1min":2.0510000000000002,"5min":2.0510000000000002,"15min":2.0510000000000002},"last":1.0609999999999999}]},{"osd":7,"last update":"Sun Oct 1 10:14:48 2023","interfaces":[{"interface":"back","average":{"1min":1.0600000000000001,"5min":1.0049999999999999,"15min":0.996},"min":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"max":{"1min":1.5289999999999999,"5min":1.5289999999999999,"15min":1.5289999999999999},"last":1.4339999999999999},{"interface":"front","average":{"1min":1.0489999999999999,"5min":1.0669999999999999,"15min":1.0700000000000001},"min":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"max":{"1min":1.3879999999999999,"5min":1.3879999999999999,"15min":1.3879999999999999},"last":1.542}]}]},{"osd":0,"up_from":8,"seq":34359738459,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 10:14:46 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":0.66500000000000004,"15min":0.56699999999999995},"min":{"1min":0.33900000000000002,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":1.2490000000000001,"5min":1.603,"15min":1.603},"last":0.69099999999999995},{"interface":"front","average":{"1min":0.69199999999999995,"5min":0.71099999999999997,"15min":0.63},"min":{"1min":0.39300000000000002,"5min":0.312,"15min":0.312},"max":{"1min":1.0369999999999999,"5min":1.0780000000000001,"15min":1.0780000000000001},"last":0.59799999999999998}]},{"osd":2,"last update":"Sun Oct 1 10:14:31 2023","interfaces":[{"interface":"back","average":{"1min":0.79000000000000004,"5min":0.74299999999999999,"15min":0.66100000000000003},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":1.169,"5min":1.2050000000000001,"15min":1.2050000000000001},"last":0.747},{"interface":"front","average":{"1min":0.70899999999999996,"5min":0.70799999999999996,"15min":0.64600000000000002},"min":{"1min":0.498,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":1.0149999999999999,"5min":1.149,"15min":1.149},"last":0.71399999999999997}]},{"osd":3,"last update":"Sun Oct 1 10:15:05 2023","interfaces":[{"interface":"back","average":{"1min":0.68300000000000005,"5min":0.72399999999999998,"15min":0.68400000000000005},"min":{"1min":0.40100000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.90900000000000003,"5min":1.0960000000000001,"15min":1.0960000000000001},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.73799999999999999,"5min":0.82399999999999995,"15min":0.78200000000000003},"min":{"1min":0.45200000000000001,"5min":0.375,"15min":0.375},"max":{"1min":1.077,"5min":1.28,"15min":1.28},"last":0.65900000000000003}]},{"osd":4,"last update":"Sun Oct 1 10:14:46 2023","interfaces":[{"interface":"back","average":{"1min":0.83899999999999997,"5min":0.90600000000000003,"15min":0.91000000000000003},"min":{"1min":0.55400000000000005,"5min":0.55000000000000004,"15min":0.55000000000000004},"max":{"1min":1.248,"5min":1.3220000000000001,"15min":1.3220000000000001},"last":0.83099999999999996},{"interface":"front","average":{"1min":1.01,"5min":1.006,"15min":0.96599999999999997},"min":{"1min":0.79900000000000004,"5min":0.60799999999999998,"15min":0.60799999999999998},"max":{"1min":1.286,"5min":1.6499999999999999,"15min":1.6499999999999999},"last":0.81399999999999995}]},{"osd":5,"last update":"Sun Oct 1 10:14:31 2023","interfaces":[{"interface":"back","average":{"1min":1.0680000000000001,"5min":1.0409999999999999,"15min":1.0329999999999999},"min":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"max":{"1min":1.718,"5min":1.718,"15min":1.718},"last":0.77300000000000002},{"interface":"front","average":{"1min":1.0589999999999999,"5min":1.054,"15min":1.0389999999999999},"min":{"1min":0.72899999999999998,"5min":0.70999999999999996,"15min":0.70999999999999996},"max":{"1min":1.548,"5min":1.548,"15min":1.548},"last":1.119}]},{"osd":6,"last update":"Sun Oct 1 10:15:11 2023","interfaces":[{"interface":"back","average":{"1min":0.999,"5min":1.0720000000000001,"15min":1.0860000000000001},"min":{"1min":0.65100000000000002,"5min":0.58099999999999996,"15min":0.58099999999999996},"max":{"1min":1.448,"5min":1.5669999999999999,"15min":1.5669999999999999},"last":0.88300000000000001},{"interface":"front","average":{"1min":1.008,"5min":1.077,"15min":1.085},"min":{"1min":0.754,"5min":0.70699999999999996,"15min":0.70699999999999996},"max":{"1min":1.2929999999999999,"5min":1.6240000000000001,"15min":1.6240000000000001},"last":1.0609999999999999}]},{"osd":7,"last update":"Sun Oct 1 10:14:46 2023","interfaces":[{"interface":"back","average":{"1min":0.96499999999999997,"5min":1.0389999999999999,"15min":1.0509999999999999},"min":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"max":{"1min":1.1140000000000001,"5min":1.4299999999999999,"15min":1.4299999999999999},"last":1.0069999999999999},{"interface":"front","average":{"1min":1.111,"5min":1.1639999999999999,"15min":1.173},"min":{"1min":0.68999999999999995,"5min":0.53500000000000003,"15min":0.53500000000000003},"max":{"1min":1.4059999999999999,"5min":1.6539999999999999,"15min":1.6539999999999999},"last":1.0469999999999999}]}]},{"osd":2,"up_from":18,"seq":77309411400,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:14:25 2023","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":0.64300000000000002,"15min":0.59199999999999997},"min":{"1min":0.38400000000000001,"5min":0.318,"15min":0.318},"max":{"1min":0.85799999999999998,"5min":1.077,"15min":1.077},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.80200000000000005,"5min":0.73099999999999998,"15min":0.65600000000000003},"min":{"1min":0.46600000000000003,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":1.7809999999999999,"5min":1.7809999999999999,"15min":1.7809999999999999},"last":0.69199999999999995}]},{"osd":1,"last update":"Sun Oct 1 10:14:25 2023","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":0.71099999999999997,"15min":0.64700000000000002},"min":{"1min":0.41699999999999998,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.2529999999999999,"5min":1.2529999999999999,"15min":1.2529999999999999},"last":0.81299999999999994},{"interface":"front","average":{"1min":0.83799999999999997,"5min":0.75900000000000001,"15min":0.67100000000000004},"min":{"1min":0.52300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.2010000000000001,"5min":1.4019999999999999,"15min":1.4019999999999999},"last":0.754}]},{"osd":3,"last update":"Sun Oct 1 10:15:07 2023","interfaces":[{"interface":"back","average":{"1min":0.67000000000000004,"5min":0.75800000000000001,"15min":0.72599999999999998},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":1.1519999999999999,"5min":1.3029999999999999,"15min":1.3029999999999999},"last":0.64900000000000002},{"interface":"front","average":{"1min":0.76100000000000001,"5min":0.84399999999999997,"15min":0.80400000000000005},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":1.228,"5min":1.4710000000000001,"15min":1.4710000000000001},"last":0.58799999999999997}]},{"osd":4,"last update":"Sun Oct 1 10:14:51 2023","interfaces":[{"interface":"back","average":{"1min":0.99299999999999999,"5min":0.95999999999999996,"15min":0.91200000000000003},"min":{"1min":0.69899999999999995,"5min":0.68100000000000005,"15min":0.68100000000000005},"max":{"1min":1.381,"5min":1.425,"15min":1.425},"last":1.052},{"interface":"front","average":{"1min":1.081,"5min":1.02,"15min":0.96799999999999997},"min":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"max":{"1min":1.4550000000000001,"5min":1.6679999999999999,"15min":1.6679999999999999},"last":1.0720000000000001}]},{"osd":5,"last update":"Sun Oct 1 10:14:35 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":1.028,"15min":1.0049999999999999},"min":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"max":{"1min":1.3839999999999999,"5min":1.647,"15min":1.647},"last":0.86299999999999999},{"interface":"front","average":{"1min":1.006,"5min":1.0209999999999999,"15min":1.012},"min":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"max":{"1min":1.456,"5min":1.54,"15min":1.54},"last":0.98899999999999999}]},{"osd":6,"last update":"Sun Oct 1 10:15:07 2023","interfaces":[{"interface":"back","average":{"1min":1.0249999999999999,"5min":1.0920000000000001,"15min":1.1080000000000001},"min":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"max":{"1min":1.4950000000000001,"5min":1.532,"15min":1.532},"last":0.92900000000000005},{"interface":"front","average":{"1min":1.069,"5min":1.1419999999999999,"15min":1.1559999999999999},"min":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"max":{"1min":1.3620000000000001,"5min":1.647,"15min":1.647},"last":1.024}]},{"osd":7,"last update":"Sun Oct 1 10:14:51 2023","interfaces":[{"interface":"back","average":{"1min":1.109,"5min":1.073,"15min":1.0669999999999999},"min":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"max":{"1min":1.5409999999999999,"5min":1.5409999999999999,"15min":1.5409999999999999},"last":1.042},{"interface":"front","average":{"1min":1.127,"5min":1.1479999999999999,"15min":1.1519999999999999},"min":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"max":{"1min":1.5589999999999999,"5min":1.5589999999999999,"15min":1.5589999999999999},"last":1.1359999999999999}]}]},{"osd":3,"up_from":25,"seq":107374182465,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:14:14 2023","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":0.65600000000000003,"15min":0.629},"min":{"1min":0.436,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":1.097,"5min":1.097,"15min":1.097},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.66800000000000004,"5min":0.74199999999999999,"15min":0.72399999999999998},"min":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":0.94899999999999995,"5min":1.05,"15min":1.05},"last":0.66700000000000004}]},{"osd":1,"last update":"Sun Oct 1 10:14:14 2023","interfaces":[{"interface":"back","average":{"1min":0.72899999999999998,"5min":0.76700000000000002,"15min":0.75900000000000001},"min":{"1min":0.52600000000000002,"5min":0.50900000000000001,"15min":0.50900000000000001},"max":{"1min":1.2170000000000001,"5min":1.224,"15min":1.224},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.745,"5min":0.73999999999999999,"15min":0.73199999999999998},"min":{"1min":0.499,"5min":0.499,"15min":0.499},"max":{"1min":1.1120000000000001,"5min":1.1439999999999999,"15min":1.1439999999999999},"last":0.59999999999999998}]},{"osd":2,"last update":"Sun Oct 1 10:14:14 2023","interfaces":[{"interface":"back","average":{"1min":0.77400000000000002,"5min":0.79600000000000004,"15min":0.78600000000000003},"min":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"max":{"1min":1.133,"5min":1.133,"15min":1.133},"last":0.70299999999999996},{"interface":"front","average":{"1min":0.77700000000000002,"5min":0.80000000000000004,"15min":0.78600000000000003},"min":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":1.3129999999999999,"5min":1.3500000000000001,"15min":1.3500000000000001},"last":0.91500000000000004}]},{"osd":4,"last update":"Sun Oct 1 10:14:53 2023","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":0.92700000000000005,"15min":0.91300000000000003},"min":{"1min":0.67200000000000004,"5min":0.61099999999999999,"15min":0.61099999999999999},"max":{"1min":1.1890000000000001,"5min":1.2769999999999999,"15min":1.2769999999999999},"last":1.0960000000000001},{"interface":"front","average":{"1min":0.90500000000000003,"5min":0.95799999999999996,"15min":0.95499999999999996},"min":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"max":{"1min":1.268,"5min":1.4710000000000001,"15min":1.4710000000000001},"last":1.3140000000000001}]},{"osd":5,"last update":"Sun Oct 1 10:14:34 2023","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":1.0269999999999999,"15min":1.0329999999999999},"min":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"max":{"1min":1.25,"5min":1.603,"15min":1.603},"last":1.288},{"interface":"front","average":{"1min":1.028,"5min":1.054,"15min":1.042},"min":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"max":{"1min":1.335,"5min":1.4330000000000001,"15min":1.4330000000000001},"last":1.222}]},{"osd":6,"last update":"Sun Oct 1 10:15:06 2023","interfaces":[{"interface":"back","average":{"1min":0.89900000000000002,"5min":1.0660000000000001,"15min":1.0900000000000001},"min":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"max":{"1min":1.0680000000000001,"5min":1.4970000000000001,"15min":1.4970000000000001},"last":1.194},{"interface":"front","average":{"1min":0.94299999999999995,"5min":1.0569999999999999,"15min":1.0840000000000001},"min":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"max":{"1min":1.1000000000000001,"5min":1.399,"15min":1.399},"last":1.121}]},{"osd":7,"last update":"Sun Oct 1 10:14:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":1.161,"15min":1.1799999999999999},"min":{"1min":0.82299999999999995,"5min":0.80700000000000005,"15min":0.80700000000000005},"max":{"1min":1.242,"5min":1.46,"15min":1.46},"last":1.27},{"interface":"front","average":{"1min":1.014,"5min":1.1839999999999999,"15min":1.212},"min":{"1min":0.66500000000000004,"5min":0.56899999999999995,"15min":0.56899999999999995},"max":{"1min":1.345,"5min":1.661,"15min":1.661},"last":1.2529999999999999}]}]},{"osd":4,"up_from":30,"seq":128849018937,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:14:49 2023","interfaces":[{"interface":"back","average":{"1min":0.748,"5min":0.82499999999999996,"15min":0.83499999999999996},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":1.028,"5min":1.22,"15min":1.22},"last":0.76200000000000001},{"interface":"front","average":{"1min":0.81799999999999995,"5min":0.85399999999999998,"15min":0.86599999999999999},"min":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"max":{"1min":1.1439999999999999,"5min":1.1859999999999999,"15min":1.1859999999999999},"last":1.141}]},{"osd":1,"last update":"Sun Oct 1 10:14:49 2023","interfaces":[{"interface":"back","average":{"1min":0.79400000000000004,"5min":0.85199999999999998,"15min":0.86899999999999999},"min":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":1.484,"5min":1.484,"15min":1.484},"last":0.85199999999999998},{"interface":"front","average":{"1min":0.82199999999999995,"5min":0.88400000000000001,"15min":0.88900000000000001},"min":{"1min":0.67500000000000004,"5min":0.65200000000000002,"15min":0.65200000000000002},"max":{"1min":1.1899999999999999,"5min":1.325,"15min":1.325},"last":1.24}]},{"osd":2,"last update":"Sun Oct 1 10:14:49 2023","interfaces":[{"interface":"back","average":{"1min":0.88700000000000001,"5min":0.90000000000000002,"15min":0.89200000000000002},"min":{"1min":0.63900000000000001,"5min":0.58899999999999997,"15min":0.58899999999999997},"max":{"1min":1.095,"5min":1.3859999999999999,"15min":1.3859999999999999},"last":1.052},{"interface":"front","average":{"1min":0.83299999999999996,"5min":0.876,"15min":0.878},"min":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"max":{"1min":0.97599999999999998,"5min":1.659,"15min":1.659},"last":0.91200000000000003}]},{"osd":3,"last update":"Sun Oct 1 10:14:49 2023","interfaces":[{"interface":"back","average":{"1min":0.92900000000000005,"5min":0.96099999999999997,"15min":0.96499999999999997},"min":{"1min":0.67100000000000004,"5min":0.56799999999999995,"15min":0.56799999999999995},"max":{"1min":1.206,"5min":1.278,"15min":1.278},"last":0.995},{"interface":"front","average":{"1min":0.92600000000000005,"5min":0.96199999999999997,"15min":0.95799999999999996},"min":{"1min":0.68999999999999995,"5min":0.61699999999999999,"15min":0.61699999999999999},"max":{"1min":1.585,"5min":1.585,"15min":1.585},"last":1.1120000000000001}]},{"osd":5,"last update":"Sun Oct 1 10:14:29 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":0.82799999999999996,"15min":0.79300000000000004},"min":{"1min":0.55700000000000005,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":1.218,"5min":1.341,"15min":1.341},"last":0.97099999999999997},{"interface":"front","average":{"1min":0.96899999999999997,"5min":0.88200000000000001,"15min":0.84799999999999998},"min":{"1min":0.68500000000000005,"5min":0.51000000000000001,"15min":0.51000000000000001},"max":{"1min":1.2829999999999999,"5min":1.4219999999999999,"15min":1.4219999999999999},"last":1.0780000000000001}]},{"osd":6,"last update":"Sun Oct 1 10:15:11 2023","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":0.84999999999999998,"15min":0.82699999999999996},"min":{"1min":0.627,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":1.2549999999999999,"5min":1.3360000000000001,"15min":1.3360000000000001},"last":0.93799999999999994},{"interface":"front","average":{"1min":1.01,"5min":0.93799999999999994,"15min":0.92000000000000004},"min":{"1min":0.64200000000000002,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":1.1910000000000001,"5min":1.284,"15min":1.284},"last":1.1910000000000001}]},{"osd":7,"last update":"Sun Oct 1 10:14:49 2023","interfaces":[{"interface":"back","average":{"1min":0.95699999999999996,"5min":0.94399999999999995,"15min":0.94199999999999995},"min":{"1min":0.67700000000000005,"5min":0.64400000000000002,"15min":0.64400000000000002},"max":{"1min":1.2150000000000001,"5min":1.2749999999999999,"15min":1.2749999999999999},"last":0.82499999999999996},{"interface":"front","average":{"1min":0.99299999999999999,"5min":0.92900000000000005,"15min":0.91800000000000004},"min":{"1min":0.61399999999999999,"5min":0.495,"15min":0.495},"max":{"1min":1.3580000000000001,"5min":1.3580000000000001,"15min":1.3580000000000001},"last":1.0229999999999999}]}]},{"osd":5,"up_from":36,"seq":154618822703,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297620,"kb_used_data":524,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447532,"statfs":{"total":95995035648,"available":95690272768,"internally_reserved":0,"allocated":536576,"data_stored":220200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:14:31 2023","interfaces":[{"interface":"back","average":{"1min":0.85999999999999999,"5min":0.89200000000000002,"15min":0.90200000000000002},"min":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"max":{"1min":1.1240000000000001,"5min":1.339,"15min":1.339},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.79700000000000004,"5min":0.85499999999999998,"15min":0.873},"min":{"1min":0.627,"5min":0.58299999999999996,"15min":0.58299999999999996},"max":{"1min":0.996,"5min":1.1679999999999999,"15min":1.1679999999999999},"last":0.67300000000000004}]},{"osd":1,"last update":"Sun Oct 1 10:14:31 2023","interfaces":[{"interface":"back","average":{"1min":0.85199999999999998,"5min":0.85899999999999999,"15min":0.85299999999999998},"min":{"1min":0.627,"5min":0.627,"15min":0.627},"max":{"1min":1.119,"5min":1.119,"15min":1.119},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.86599999999999999,"5min":0.90300000000000002,"15min":0.90900000000000003},"min":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"max":{"1min":1.1990000000000001,"5min":1.1990000000000001,"15min":1.1990000000000001},"last":0.72699999999999998}]},{"osd":2,"last update":"Sun Oct 1 10:14:31 2023","interfaces":[{"interface":"back","average":{"1min":0.88600000000000001,"5min":0.86899999999999999,"15min":0.86399999999999999},"min":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"max":{"1min":1.155,"5min":1.155,"15min":1.155},"last":0.88200000000000001},{"interface":"front","average":{"1min":0.85999999999999999,"5min":0.91100000000000003,"15min":0.91700000000000004},"min":{"1min":0.67800000000000005,"5min":0.57199999999999995,"15min":0.57199999999999995},"max":{"1min":1.153,"5min":1.214,"15min":1.214},"last":0.65500000000000003}]},{"osd":3,"last update":"Sun Oct 1 10:14:31 2023","interfaces":[{"interface":"back","average":{"1min":0.86699999999999999,"5min":0.90000000000000002,"15min":0.91300000000000003},"min":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"max":{"1min":1.129,"5min":1.1339999999999999,"15min":1.1339999999999999},"last":0.83899999999999997},{"interface":"front","average":{"1min":0.86099999999999999,"5min":0.90100000000000002,"15min":0.90600000000000003},"min":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"max":{"1min":1.04,"5min":1.2490000000000001,"15min":1.2490000000000001},"last":0.80600000000000005}]},{"osd":4,"last update":"Sun Oct 1 10:14:31 2023","interfaces":[{"interface":"back","average":{"1min":0.81200000000000006,"5min":0.75900000000000001,"15min":0.73299999999999998},"min":{"1min":0.47099999999999997,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":1.256,"5min":1.256,"15min":1.256},"last":1.0489999999999999},{"interface":"front","average":{"1min":0.90100000000000002,"5min":0.84499999999999997,"15min":0.81200000000000006},"min":{"1min":0.61099999999999999,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":1.2929999999999999,"5min":1.431,"15min":1.431},"last":0.97899999999999998}]},{"osd":6,"last update":"Sun Oct 1 10:15:10 2023","interfaces":[{"interface":"back","average":{"1min":0.95599999999999996,"5min":0.92500000000000004,"15min":0.92900000000000005},"min":{"1min":0.59199999999999997,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":1.3149999999999999,"5min":1.333,"15min":1.333},"last":1},{"interface":"front","average":{"1min":0.95399999999999996,"5min":0.90700000000000003,"15min":0.89700000000000002},"min":{"1min":0.63600000000000001,"5min":0.434,"15min":0.434},"max":{"1min":1.238,"5min":1.2649999999999999,"15min":1.2649999999999999},"last":1.0269999999999999}]},{"osd":7,"last update":"Sun Oct 1 10:14:53 2023","interfaces":[{"interface":"back","average":{"1min":0.91500000000000004,"5min":0.90700000000000003,"15min":0.90600000000000003},"min":{"1min":0.56299999999999994,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":1.3009999999999999,"5min":1.3009999999999999,"15min":1.3009999999999999},"last":0.94999999999999996},{"interface":"front","average":{"1min":0.98999999999999999,"5min":1.0189999999999999,"15min":1.024},"min":{"1min":0.57599999999999996,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":1.377,"5min":1.377,"15min":1.377},"last":1.0960000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T10:15:17.616 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T10:15:18.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:18 smithi078 bash[27336]: audit 2023-10-01T10:15:16.904203+0000 mgr.y (mgr.14427) 87 : audit [DBG] from='client.24521 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:15:18.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:18 smithi078 bash[27336]: cluster 2023-10-01T10:15:17.184525+0000 mgr.y (mgr.14427) 88 : cluster [DBG] pgmap v67: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:18.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:18 smithi078 bash[33821]: audit 2023-10-01T10:15:16.904203+0000 mgr.y (mgr.14427) 87 : audit [DBG] from='client.24521 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:15:18.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:18 smithi078 bash[33821]: cluster 2023-10-01T10:15:17.184525+0000 mgr.y (mgr.14427) 88 : cluster [DBG] pgmap v67: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:18.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:18 smithi173 bash[27612]: audit 2023-10-01T10:15:16.904203+0000 mgr.y (mgr.14427) 87 : audit [DBG] from='client.24521 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:15:18.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:18 smithi173 bash[27612]: cluster 2023-10-01T10:15:17.184525+0000 mgr.y (mgr.14427) 88 : cluster [DBG] pgmap v67: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:20.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:20 smithi078 bash[27336]: cluster 2023-10-01T10:15:19.185338+0000 mgr.y (mgr.14427) 89 : cluster [DBG] pgmap v68: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:20.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:20 smithi078 bash[33821]: cluster 2023-10-01T10:15:19.185338+0000 mgr.y (mgr.14427) 89 : cluster [DBG] pgmap v68: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:20.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:20 smithi173 bash[27612]: cluster 2023-10-01T10:15:19.185338+0000 mgr.y (mgr.14427) 89 : cluster [DBG] pgmap v68: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:21.583 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:Oct 01 10:15:21 smithi078 bash[58231]: level=info ts=2023-10-01T10:15:21.264Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.002979955s 2023-10-01T10:15:22.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:21 smithi078 bash[33821]: audit 2023-10-01T10:15:20.726148+0000 mon.a (mon.0) 638 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:22.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:21 smithi078 bash[33821]: audit 2023-10-01T10:15:20.990773+0000 mon.a (mon.0) 639 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:22.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:21 smithi078 bash[33821]: audit 2023-10-01T10:15:20.992220+0000 mon.a (mon.0) 640 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:15:22.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:21 smithi078 bash[33821]: audit 2023-10-01T10:15:20.993706+0000 mon.a (mon.0) 641 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:15:22.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:21 smithi078 bash[33821]: audit 2023-10-01T10:15:21.003190+0000 mon.a (mon.0) 642 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:22.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:21 smithi078 bash[33821]: audit 2023-10-01T10:15:21.012812+0000 mon.a (mon.0) 643 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:15:22.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:21 smithi078 bash[33821]: cluster 2023-10-01T10:15:21.185879+0000 mgr.y (mgr.14427) 90 : cluster [DBG] pgmap v69: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:22.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:21 smithi078 bash[27336]: audit 2023-10-01T10:15:20.726148+0000 mon.a (mon.0) 638 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:22.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:21 smithi078 bash[27336]: audit 2023-10-01T10:15:20.990773+0000 mon.a (mon.0) 639 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:22.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:21 smithi078 bash[27336]: audit 2023-10-01T10:15:20.992220+0000 mon.a (mon.0) 640 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:15:22.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:21 smithi078 bash[27336]: audit 2023-10-01T10:15:20.993706+0000 mon.a (mon.0) 641 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:15:22.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:21 smithi078 bash[27336]: audit 2023-10-01T10:15:21.003190+0000 mon.a (mon.0) 642 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:22.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:21 smithi078 bash[27336]: audit 2023-10-01T10:15:21.012812+0000 mon.a (mon.0) 643 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:15:22.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:21 smithi078 bash[27336]: cluster 2023-10-01T10:15:21.185879+0000 mgr.y (mgr.14427) 90 : cluster [DBG] pgmap v69: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:22.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:21 smithi173 bash[27612]: audit 2023-10-01T10:15:20.726148+0000 mon.a (mon.0) 638 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:22.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:21 smithi173 bash[27612]: audit 2023-10-01T10:15:20.990773+0000 mon.a (mon.0) 639 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:22.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:21 smithi173 bash[27612]: audit 2023-10-01T10:15:20.992220+0000 mon.a (mon.0) 640 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:15:22.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:21 smithi173 bash[27612]: audit 2023-10-01T10:15:20.993706+0000 mon.a (mon.0) 641 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:15:22.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:21 smithi173 bash[27612]: audit 2023-10-01T10:15:21.003190+0000 mon.a (mon.0) 642 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:22.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:21 smithi173 bash[27612]: audit 2023-10-01T10:15:21.012812+0000 mon.a (mon.0) 643 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:15:22.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:21 smithi173 bash[27612]: cluster 2023-10-01T10:15:21.185879+0000 mgr.y (mgr.14427) 90 : cluster [DBG] pgmap v69: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:22.163 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:15:24.251 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:24 smithi078 bash[27336]: cluster 2023-10-01T10:15:23.186615+0000 mgr.y (mgr.14427) 91 : cluster [DBG] pgmap v70: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:24.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:24 smithi078 bash[33821]: cluster 2023-10-01T10:15:23.186615+0000 mgr.y (mgr.14427) 91 : cluster [DBG] pgmap v70: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:24.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:24 smithi173 bash[27612]: cluster 2023-10-01T10:15:23.186615+0000 mgr.y (mgr.14427) 91 : cluster [DBG] pgmap v70: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:25.290 INFO:teuthology.orchestra.run.smithi078.stdout: 2023-10-01T10:15:25.292 INFO:teuthology.orchestra.run.smithi078.stderr:dumped all 2023-10-01T10:15:25.333 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:15:25 smithi078 bash[27601]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:15:25] "GET /metrics HTTP/1.1" 200 155664 "" "Prometheus/2.33.4" 2023-10-01T10:15:25.620 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:15:25 smithi173 bash[28473]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:15:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T10:15:26.010 INFO:teuthology.orchestra.run.smithi078.stdout:{"pg_ready":true,"pg_map":{"version":71,"stamp":"2023-10-01T10:15:25.186975+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2380984,"kb_used_data":4216,"kb_used_omap":0,"kb_used_meta":2376704,"kb_avail":747580232,"statfs":{"total":767960285184,"available":765522157568,"internally_reserved":0,"allocated":4317184,"data_stored":1763490,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2433744896},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"0.000000"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-01T10:13:46.495363+0000","last_change":"2023-10-01T10:13:44.666450+0000","last_active":"2023-10-01T10:13:46.495363+0000","last_peered":"2023-10-01T10:13:46.495363+0000","last_clean":"2023-10-01T10:13:46.495363+0000","last_became_active":"2023-10-01T10:13:44.665959+0000","last_became_peered":"2023-10-01T10:13:44.665959+0000","last_unstale":"2023-10-01T10:13:46.495363+0000","last_undegraded":"2023-10-01T10:13:46.495363+0000","last_fullsized":"2023-10-01T10:13:46.495363+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-01T10:13:46.493047+0000","last_change":"2023-10-01T10:13:44.825307+0000","last_active":"2023-10-01T10:13:46.493047+0000","last_peered":"2023-10-01T10:13:46.493047+0000","last_clean":"2023-10-01T10:13:46.493047+0000","last_became_active":"2023-10-01T10:13:44.824803+0000","last_became_peered":"2023-10-01T10:13:44.824803+0000","last_unstale":"2023-10-01T10:13:46.493047+0000","last_undegraded":"2023-10-01T10:13:46.493047+0000","last_fullsized":"2023-10-01T10:13:46.493047+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-01T10:13:45.924633+0000","last_change":"2023-10-01T10:13:44.827141+0000","last_active":"2023-10-01T10:13:45.924633+0000","last_peered":"2023-10-01T10:13:45.924633+0000","last_clean":"2023-10-01T10:13:45.924633+0000","last_became_active":"2023-10-01T10:13:44.826089+0000","last_became_peered":"2023-10-01T10:13:44.826089+0000","last_unstale":"2023-10-01T10:13:45.924633+0000","last_undegraded":"2023-10-01T10:13:45.924633+0000","last_fullsized":"2023-10-01T10:13:45.924633+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-01T10:13:45.924525+0000","last_change":"2023-10-01T10:13:44.825879+0000","last_active":"2023-10-01T10:13:45.924525+0000","last_peered":"2023-10-01T10:13:45.924525+0000","last_clean":"2023-10-01T10:13:45.924525+0000","last_became_active":"2023-10-01T10:13:44.825097+0000","last_became_peered":"2023-10-01T10:13:44.825097+0000","last_unstale":"2023-10-01T10:13:45.924525+0000","last_undegraded":"2023-10-01T10:13:45.924525+0000","last_fullsized":"2023-10-01T10:13:45.924525+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":15,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-01T10:13:45.924675+0000","last_change":"2023-10-01T10:12:47.477457+0000","last_active":"2023-10-01T10:13:45.924675+0000","last_peered":"2023-10-01T10:13:45.924675+0000","last_clean":"2023-10-01T10:13:45.924675+0000","last_became_active":"2023-10-01T10:12:47.476522+0000","last_became_peered":"2023-10-01T10:12:47.476522+0000","last_unstale":"2023-10-01T10:13:45.924675+0000","last_undegraded":"2023-10-01T10:13:45.924675+0000","last_fullsized":"2023-10-01T10:13:45.924675+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:09:17.221082+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:09:17.221082+0000","last_clean_scrub_stamp":"2023-10-01T10:09:17.221082+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-01T10:13:46.495403+0000","last_change":"2023-10-01T10:13:44.663992+0000","last_active":"2023-10-01T10:13:46.495403+0000","last_peered":"2023-10-01T10:13:46.495403+0000","last_clean":"2023-10-01T10:13:46.495403+0000","last_became_active":"2023-10-01T10:13:44.663669+0000","last_became_peered":"2023-10-01T10:13:44.663669+0000","last_unstale":"2023-10-01T10:13:46.495403+0000","last_undegraded":"2023-10-01T10:13:46.495403+0000","last_fullsized":"2023-10-01T10:13:46.495403+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-01T10:13:46.493149+0000","last_change":"2023-10-01T10:13:44.823885+0000","last_active":"2023-10-01T10:13:46.493149+0000","last_peered":"2023-10-01T10:13:46.493149+0000","last_clean":"2023-10-01T10:13:46.493149+0000","last_became_active":"2023-10-01T10:13:44.823314+0000","last_became_peered":"2023-10-01T10:13:44.823314+0000","last_unstale":"2023-10-01T10:13:46.493149+0000","last_undegraded":"2023-10-01T10:13:46.493149+0000","last_fullsized":"2023-10-01T10:13:46.493149+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-01T10:13:46.495302+0000","last_change":"2023-10-01T10:13:44.663534+0000","last_active":"2023-10-01T10:13:46.495302+0000","last_peered":"2023-10-01T10:13:46.495302+0000","last_clean":"2023-10-01T10:13:46.495302+0000","last_became_active":"2023-10-01T10:13:44.663014+0000","last_became_peered":"2023-10-01T10:13:44.663014+0000","last_unstale":"2023-10-01T10:13:46.495302+0000","last_undegraded":"2023-10-01T10:13:46.495302+0000","last_fullsized":"2023-10-01T10:13:46.495302+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":54,"state":"active+clean","last_fresh":"2023-10-01T10:13:46.493139+0000","last_change":"2023-10-01T10:13:44.824140+0000","last_active":"2023-10-01T10:13:46.493139+0000","last_peered":"2023-10-01T10:13:46.493139+0000","last_clean":"2023-10-01T10:13:46.493139+0000","last_became_active":"2023-10-01T10:13:44.823462+0000","last_became_peered":"2023-10-01T10:13:44.823462+0000","last_unstale":"2023-10-01T10:13:46.493139+0000","last_undegraded":"2023-10-01T10:13:46.493139+0000","last_fullsized":"2023-10-01T10:13:46.493139+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430241,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:14:50 2023","interfaces":[{"interface":"back","average":{"1min":0.85299999999999998,"5min":0.91400000000000003,"15min":0.92400000000000004},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.115,"5min":1.161,"15min":1.161},"last":1.143},{"interface":"front","average":{"1min":0.82999999999999996,"5min":0.91400000000000003,"15min":0.92800000000000005},"min":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"max":{"1min":0.98199999999999998,"5min":1.198,"15min":1.198},"last":0.86099999999999999}]},{"osd":1,"last update":"Sun Oct 1 10:14:50 2023","interfaces":[{"interface":"back","average":{"1min":0.85699999999999998,"5min":0.89900000000000002,"15min":0.90600000000000003},"min":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"max":{"1min":1.0960000000000001,"5min":1.444,"15min":1.444},"last":0.65800000000000003},{"interface":"front","average":{"1min":0.88500000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"min":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"max":{"1min":1.377,"5min":1.377,"15min":1.377},"last":1.0329999999999999}]},{"osd":2,"last update":"Sun Oct 1 10:14:50 2023","interfaces":[{"interface":"back","average":{"1min":0.89200000000000002,"5min":0.92400000000000004,"15min":0.93000000000000005},"min":{"1min":0.65400000000000003,"5min":0.61899999999999999,"15min":0.61899999999999999},"max":{"1min":1.4239999999999999,"5min":1.4239999999999999,"15min":1.4239999999999999},"last":0.73899999999999999},{"interface":"front","average":{"1min":0.93200000000000005,"5min":0.95199999999999996,"15min":0.95599999999999996},"min":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"max":{"1min":1.238,"5min":1.2390000000000001,"15min":1.2390000000000001},"last":0.81499999999999995}]},{"osd":3,"last update":"Sun Oct 1 10:14:50 2023","interfaces":[{"interface":"back","average":{"1min":0.88400000000000001,"5min":0.90200000000000002,"15min":0.90500000000000003},"min":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.155,"5min":1.1990000000000001,"15min":1.1990000000000001},"last":0.93300000000000005},{"interface":"front","average":{"1min":1.016,"5min":1.008,"15min":1.006},"min":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"max":{"1min":3.2229999999999999,"5min":3.2229999999999999,"15min":3.2229999999999999},"last":0.94299999999999995}]},{"osd":4,"last update":"Sun Oct 1 10:14:50 2023","interfaces":[{"interface":"back","average":{"1min":0.89600000000000002,"5min":0.92800000000000005,"15min":0.93400000000000005},"min":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":1.3169999999999999,"5min":1.361,"15min":1.361},"last":0.95399999999999996},{"interface":"front","average":{"1min":0.86299999999999999,"5min":0.91100000000000003,"15min":0.91900000000000004},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":1.5840000000000001,"5min":1.607,"15min":1.607},"last":0.97699999999999998}]},{"osd":5,"last update":"Sun Oct 1 10:14:50 2023","interfaces":[{"interface":"back","average":{"1min":0.873,"5min":0.97299999999999998,"15min":0.98899999999999999},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":1.5149999999999999,"5min":1.831,"15min":1.831},"last":0.88600000000000001},{"interface":"front","average":{"1min":0.88900000000000001,"5min":1.0589999999999999,"15min":1.087},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":1.4610000000000001,"5min":1.5,"15min":1.5},"last":1.0920000000000001}]},{"osd":6,"last update":"Sun Oct 1 10:14:50 2023","interfaces":[{"interface":"back","average":{"1min":0.91700000000000004,"5min":1.0609999999999999,"15min":1.085},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":1.6240000000000001,"5min":1.637,"15min":1.637},"last":0.98999999999999999},{"interface":"front","average":{"1min":0.93100000000000005,"5min":1.0780000000000001,"15min":1.1020000000000001},"min":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"max":{"1min":1.6870000000000001,"5min":2.2869999999999999,"15min":2.2869999999999999},"last":1.0149999999999999}]}]},{"osd":6,"up_from":42,"seq":180388626475,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297620,"kb_used_data":524,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447532,"statfs":{"total":95995035648,"available":95690272768,"internally_reserved":0,"allocated":536576,"data_stored":220200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:15:08 2023","interfaces":[{"interface":"back","average":{"1min":0.77100000000000002,"5min":0.871,"15min":0.89000000000000001},"min":{"1min":0.52300000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":1.038,"5min":1.359,"15min":1.359},"last":1.107},{"interface":"front","average":{"1min":0.84399999999999997,"5min":0.86499999999999999,"15min":0.85599999999999998},"min":{"1min":0.61299999999999999,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":1.1279999999999999,"5min":1.1839999999999999,"15min":1.1839999999999999},"last":0.75900000000000001}]},{"osd":1,"last update":"Sun Oct 1 10:15:08 2023","interfaces":[{"interface":"back","average":{"1min":0.80200000000000005,"5min":0.90900000000000003,"15min":0.93100000000000005},"min":{"1min":0.57499999999999996,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":1.054,"5min":1.4299999999999999,"15min":1.4299999999999999},"last":1.032},{"interface":"front","average":{"1min":0.84799999999999998,"5min":0.94399999999999995,"15min":0.97799999999999998},"min":{"1min":0.67700000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"max":{"1min":0.995,"5min":1.298,"15min":1.298},"last":0.88600000000000001}]},{"osd":2,"last update":"Sun Oct 1 10:15:08 2023","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":0.86899999999999999,"15min":0.875},"min":{"1min":0.59299999999999997,"5min":0.57799999999999996,"15min":0.57799999999999996},"max":{"1min":1.2330000000000001,"5min":1.2889999999999999,"15min":1.2889999999999999},"last":0.98199999999999998},{"interface":"front","average":{"1min":0.86899999999999999,"5min":0.91700000000000004,"15min":0.93300000000000005},"min":{"1min":0.60599999999999998,"5min":0.57099999999999995,"15min":0.57099999999999995},"max":{"1min":1.109,"5min":1.484,"15min":1.484},"last":0.85199999999999998}]},{"osd":3,"last update":"Sun Oct 1 10:15:08 2023","interfaces":[{"interface":"back","average":{"1min":0.84399999999999997,"5min":0.93899999999999995,"15min":0.95699999999999996},"min":{"1min":0.67300000000000004,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":1.1279999999999999,"5min":1.456,"15min":1.456},"last":0.871},{"interface":"front","average":{"1min":0.88200000000000001,"5min":0.90800000000000003,"15min":0.92100000000000004},"min":{"1min":0.68899999999999995,"5min":0.59299999999999997,"15min":0.59299999999999997},"max":{"1min":1.1479999999999999,"5min":1.266,"15min":1.266},"last":1.0620000000000001}]},{"osd":4,"last update":"Sun Oct 1 10:15:08 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":0.86899999999999999,"15min":0.871},"min":{"1min":0.58899999999999997,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.1799999999999999,"5min":1.246,"15min":1.246},"last":0.94099999999999995},{"interface":"front","average":{"1min":0.96099999999999997,"5min":0.90900000000000003,"15min":0.90300000000000002},"min":{"1min":0.79300000000000004,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":1.143,"5min":1.349,"15min":1.349},"last":0.78500000000000003}]},{"osd":5,"last update":"Sun Oct 1 10:15:08 2023","interfaces":[{"interface":"back","average":{"1min":1.026,"5min":0.92300000000000004,"15min":0.90700000000000003},"min":{"1min":0.67500000000000004,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":1.208,"5min":1.27,"15min":1.27},"last":1.012},{"interface":"front","average":{"1min":1.0209999999999999,"5min":0.97499999999999998,"15min":0.97299999999999998},"min":{"1min":0.65500000000000003,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":1.1990000000000001,"5min":1.631,"15min":1.631},"last":0.92900000000000005}]},{"osd":7,"last update":"Sun Oct 1 10:14:48 2023","interfaces":[{"interface":"back","average":{"1min":1.0580000000000001,"5min":1.0780000000000001,"15min":1.081},"min":{"1min":0.77000000000000002,"5min":0.59699999999999998,"15min":0.59699999999999998},"max":{"1min":1.2310000000000001,"5min":1.7250000000000001,"15min":1.7250000000000001},"last":1.1539999999999999},{"interface":"front","average":{"1min":1.032,"5min":1.038,"15min":1.0389999999999999},"min":{"1min":0.77400000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":1.246,"5min":1.5,"15min":1.5},"last":1.085}]}]},{"osd":1,"up_from":13,"seq":55834574932,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:14:42 2023","interfaces":[{"interface":"back","average":{"1min":0.64900000000000002,"5min":0.65500000000000003,"15min":0.56599999999999995},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.92900000000000005,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.81999999999999995},{"interface":"front","average":{"1min":0.70699999999999996,"5min":0.72699999999999998,"15min":0.58499999999999996},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.075,"5min":1.1919999999999999,"15min":1.1919999999999999},"last":0.85799999999999998}]},{"osd":2,"last update":"Sun Oct 1 10:14:24 2023","interfaces":[{"interface":"back","average":{"1min":0.68200000000000005,"5min":0.70099999999999996,"15min":0.64200000000000002},"min":{"1min":0.45500000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.996,"5min":1.1120000000000001,"15min":1.1120000000000001},"last":0.67800000000000005},{"interface":"front","average":{"1min":0.77100000000000002,"5min":0.76100000000000001,"15min":0.67900000000000005},"min":{"1min":0.41299999999999998,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":1.474,"5min":1.474,"15min":1.474},"last":0.48099999999999998}]},{"osd":3,"last update":"Sun Oct 1 10:15:04 2023","interfaces":[{"interface":"back","average":{"1min":0.753,"5min":0.83199999999999996,"15min":0.81599999999999995},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":1.5780000000000001,"5min":2.5059999999999998,"15min":2.5059999999999998},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.71099999999999997,"5min":0.77400000000000002,"15min":0.78100000000000003},"min":{"1min":0.42399999999999999,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":1.5620000000000001,"5min":2.153,"15min":2.153},"last":0.58899999999999997}]},{"osd":4,"last update":"Sun Oct 1 10:14:48 2023","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":0.97599999999999998,"15min":0.97399999999999998},"min":{"1min":0.749,"5min":0.505,"15min":0.505},"max":{"1min":1.2969999999999999,"5min":1.3500000000000001,"15min":1.3500000000000001},"last":0.90700000000000003},{"interface":"front","average":{"1min":0.97199999999999998,"5min":0.97399999999999998,"15min":0.96899999999999997},"min":{"1min":0.629,"5min":0.56799999999999995,"15min":0.56799999999999995},"max":{"1min":1.341,"5min":1.4730000000000001,"15min":1.4730000000000001},"last":1.016}]},{"osd":5,"last update":"Sun Oct 1 10:14:24 2023","interfaces":[{"interface":"back","average":{"1min":1,"5min":1.0389999999999999,"15min":1.0609999999999999},"min":{"1min":0.71299999999999997,"5min":0.65500000000000003,"15min":0.65500000000000003},"max":{"1min":1.383,"5min":1.5389999999999999,"15min":1.5389999999999999},"last":0.69699999999999995},{"interface":"front","average":{"1min":1.034,"5min":1.0600000000000001,"15min":1.0620000000000001},"min":{"1min":0.754,"5min":0.64900000000000002,"15min":0.64900000000000002},"max":{"1min":1.3300000000000001,"5min":1.3300000000000001,"15min":1.3300000000000001},"last":0.76700000000000002}]},{"osd":6,"last update":"Sun Oct 1 10:15:04 2023","interfaces":[{"interface":"back","average":{"1min":1.008,"5min":0.97799999999999998,"15min":0.96399999999999997},"min":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"max":{"1min":1.6060000000000001,"5min":1.6060000000000001,"15min":1.6060000000000001},"last":0.873},{"interface":"front","average":{"1min":1.095,"5min":1.0620000000000001,"15min":1.0600000000000001},"min":{"1min":0.78400000000000003,"5min":0.68300000000000005,"15min":0.68300000000000005},"max":{"1min":2.0510000000000002,"5min":2.0510000000000002,"15min":2.0510000000000002},"last":0.997}]},{"osd":7,"last update":"Sun Oct 1 10:14:48 2023","interfaces":[{"interface":"back","average":{"1min":1.0600000000000001,"5min":1.0049999999999999,"15min":0.996},"min":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"max":{"1min":1.5289999999999999,"5min":1.5289999999999999,"15min":1.5289999999999999},"last":0.92500000000000004},{"interface":"front","average":{"1min":1.0489999999999999,"5min":1.0669999999999999,"15min":1.0700000000000001},"min":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"max":{"1min":1.3879999999999999,"5min":1.3879999999999999,"15min":1.3879999999999999},"last":0.97799999999999998}]}]},{"osd":0,"up_from":8,"seq":34359738461,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 10:14:46 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":0.66500000000000004,"15min":0.56699999999999995},"min":{"1min":0.33900000000000002,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":1.2490000000000001,"5min":1.603,"15min":1.603},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.69199999999999995,"5min":0.71099999999999997,"15min":0.63},"min":{"1min":0.39300000000000002,"5min":0.312,"15min":0.312},"max":{"1min":1.0369999999999999,"5min":1.0780000000000001,"15min":1.0780000000000001},"last":0.5}]},{"osd":2,"last update":"Sun Oct 1 10:14:31 2023","interfaces":[{"interface":"back","average":{"1min":0.79000000000000004,"5min":0.74299999999999999,"15min":0.66100000000000003},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":1.169,"5min":1.2050000000000001,"15min":1.2050000000000001},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.70899999999999996,"5min":0.70799999999999996,"15min":0.64600000000000002},"min":{"1min":0.498,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":1.0149999999999999,"5min":1.149,"15min":1.149},"last":0.54000000000000004}]},{"osd":3,"last update":"Sun Oct 1 10:15:05 2023","interfaces":[{"interface":"back","average":{"1min":0.68300000000000005,"5min":0.72399999999999998,"15min":0.68400000000000005},"min":{"1min":0.40100000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.90900000000000003,"5min":1.0960000000000001,"15min":1.0960000000000001},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.73799999999999999,"5min":0.82399999999999995,"15min":0.78200000000000003},"min":{"1min":0.45200000000000001,"5min":0.375,"15min":0.375},"max":{"1min":1.077,"5min":1.28,"15min":1.28},"last":0.69399999999999995}]},{"osd":4,"last update":"Sun Oct 1 10:14:46 2023","interfaces":[{"interface":"back","average":{"1min":0.83899999999999997,"5min":0.90600000000000003,"15min":0.91000000000000003},"min":{"1min":0.55400000000000005,"5min":0.55000000000000004,"15min":0.55000000000000004},"max":{"1min":1.248,"5min":1.3220000000000001,"15min":1.3220000000000001},"last":0.90300000000000002},{"interface":"front","average":{"1min":1.01,"5min":1.006,"15min":0.96599999999999997},"min":{"1min":0.79900000000000004,"5min":0.60799999999999998,"15min":0.60799999999999998},"max":{"1min":1.286,"5min":1.6499999999999999,"15min":1.6499999999999999},"last":0.66200000000000003}]},{"osd":5,"last update":"Sun Oct 1 10:14:31 2023","interfaces":[{"interface":"back","average":{"1min":1.0680000000000001,"5min":1.0409999999999999,"15min":1.0329999999999999},"min":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"max":{"1min":1.718,"5min":1.718,"15min":1.718},"last":0.78100000000000003},{"interface":"front","average":{"1min":1.0589999999999999,"5min":1.054,"15min":1.0389999999999999},"min":{"1min":0.72899999999999998,"5min":0.70999999999999996,"15min":0.70999999999999996},"max":{"1min":1.548,"5min":1.548,"15min":1.548},"last":0.76300000000000001}]},{"osd":6,"last update":"Sun Oct 1 10:15:11 2023","interfaces":[{"interface":"back","average":{"1min":0.999,"5min":1.0720000000000001,"15min":1.0860000000000001},"min":{"1min":0.65100000000000002,"5min":0.58099999999999996,"15min":0.58099999999999996},"max":{"1min":1.448,"5min":1.5669999999999999,"15min":1.5669999999999999},"last":0.82099999999999995},{"interface":"front","average":{"1min":1.008,"5min":1.077,"15min":1.085},"min":{"1min":0.754,"5min":0.70699999999999996,"15min":0.70699999999999996},"max":{"1min":1.2929999999999999,"5min":1.6240000000000001,"15min":1.6240000000000001},"last":0.67400000000000004}]},{"osd":7,"last update":"Sun Oct 1 10:14:46 2023","interfaces":[{"interface":"back","average":{"1min":0.96499999999999997,"5min":1.0389999999999999,"15min":1.0509999999999999},"min":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"max":{"1min":1.1140000000000001,"5min":1.4299999999999999,"15min":1.4299999999999999},"last":0.85899999999999999},{"interface":"front","average":{"1min":1.111,"5min":1.1639999999999999,"15min":1.173},"min":{"1min":0.68999999999999995,"5min":0.53500000000000003,"15min":0.53500000000000003},"max":{"1min":1.4059999999999999,"5min":1.6539999999999999,"15min":1.6539999999999999},"last":0.93700000000000006}]}]},{"osd":2,"up_from":18,"seq":77309411402,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:14:25 2023","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":0.64300000000000002,"15min":0.59199999999999997},"min":{"1min":0.38400000000000001,"5min":0.318,"15min":0.318},"max":{"1min":0.85799999999999998,"5min":1.077,"15min":1.077},"last":0.65300000000000002},{"interface":"front","average":{"1min":0.80200000000000005,"5min":0.73099999999999998,"15min":0.65600000000000003},"min":{"1min":0.46600000000000003,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":1.7809999999999999,"5min":1.7809999999999999,"15min":1.7809999999999999},"last":0.44700000000000001}]},{"osd":1,"last update":"Sun Oct 1 10:14:25 2023","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":0.71099999999999997,"15min":0.64700000000000002},"min":{"1min":0.41699999999999998,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.2529999999999999,"5min":1.2529999999999999,"15min":1.2529999999999999},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.83799999999999997,"5min":0.75900000000000001,"15min":0.67100000000000004},"min":{"1min":0.52300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.2010000000000001,"5min":1.4019999999999999,"15min":1.4019999999999999},"last":0.39800000000000002}]},{"osd":3,"last update":"Sun Oct 1 10:15:07 2023","interfaces":[{"interface":"back","average":{"1min":0.67000000000000004,"5min":0.75800000000000001,"15min":0.72599999999999998},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":1.1519999999999999,"5min":1.3029999999999999,"15min":1.3029999999999999},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.76100000000000001,"5min":0.84399999999999997,"15min":0.80400000000000005},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":1.228,"5min":1.4710000000000001,"15min":1.4710000000000001},"last":0.51200000000000001}]},{"osd":4,"last update":"Sun Oct 1 10:14:51 2023","interfaces":[{"interface":"back","average":{"1min":0.99299999999999999,"5min":0.95999999999999996,"15min":0.91200000000000003},"min":{"1min":0.69899999999999995,"5min":0.68100000000000005,"15min":0.68100000000000005},"max":{"1min":1.381,"5min":1.425,"15min":1.425},"last":0.64700000000000002},{"interface":"front","average":{"1min":1.081,"5min":1.02,"15min":0.96799999999999997},"min":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"max":{"1min":1.4550000000000001,"5min":1.6679999999999999,"15min":1.6679999999999999},"last":0.85499999999999998}]},{"osd":5,"last update":"Sun Oct 1 10:14:35 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":1.028,"15min":1.0049999999999999},"min":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"max":{"1min":1.3839999999999999,"5min":1.647,"15min":1.647},"last":0.629},{"interface":"front","average":{"1min":1.006,"5min":1.0209999999999999,"15min":1.012},"min":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"max":{"1min":1.456,"5min":1.54,"15min":1.54},"last":0.747}]},{"osd":6,"last update":"Sun Oct 1 10:15:07 2023","interfaces":[{"interface":"back","average":{"1min":1.0249999999999999,"5min":1.0920000000000001,"15min":1.1080000000000001},"min":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"max":{"1min":1.4950000000000001,"5min":1.532,"15min":1.532},"last":0.68600000000000005},{"interface":"front","average":{"1min":1.069,"5min":1.1419999999999999,"15min":1.1559999999999999},"min":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"max":{"1min":1.3620000000000001,"5min":1.647,"15min":1.647},"last":0.78000000000000003}]},{"osd":7,"last update":"Sun Oct 1 10:14:51 2023","interfaces":[{"interface":"back","average":{"1min":1.109,"5min":1.073,"15min":1.0669999999999999},"min":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"max":{"1min":1.5409999999999999,"5min":1.5409999999999999,"15min":1.5409999999999999},"last":0.81699999999999995},{"interface":"front","average":{"1min":1.127,"5min":1.1479999999999999,"15min":1.1519999999999999},"min":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"max":{"1min":1.5589999999999999,"5min":1.5589999999999999,"15min":1.5589999999999999},"last":0.91100000000000003}]}]},{"osd":3,"up_from":25,"seq":107374182467,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:15:17 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.65100000000000002,"15min":0.627},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":1.0860000000000001,"5min":1.097,"15min":1.097},"last":0.69399999999999995},{"interface":"front","average":{"1min":0.62,"5min":0.72299999999999998,"15min":0.71799999999999997},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.98699999999999999,"5min":1.05,"15min":1.05},"last":0.74299999999999999}]},{"osd":1,"last update":"Sun Oct 1 10:15:17 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.74199999999999999,"15min":0.751},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.92400000000000004,"5min":1.224,"15min":1.224},"last":0.91500000000000004},{"interface":"front","average":{"1min":0.66300000000000003,"5min":0.72699999999999998,"15min":0.72799999999999998},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":1.054,"5min":1.1439999999999999,"15min":1.1439999999999999},"last":0.67200000000000004}]},{"osd":2,"last update":"Sun Oct 1 10:15:17 2023","interfaces":[{"interface":"back","average":{"1min":0.65800000000000003,"5min":0.77100000000000002,"15min":0.77800000000000002},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.85599999999999998,"5min":1.133,"15min":1.133},"last":0.73099999999999998},{"interface":"front","average":{"1min":0.66500000000000004,"5min":0.77700000000000002,"15min":0.77800000000000002},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.91500000000000004,"5min":1.3500000000000001,"15min":1.3500000000000001},"last":0.80400000000000005}]},{"osd":4,"last update":"Sun Oct 1 10:14:53 2023","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":0.92700000000000005,"15min":0.91300000000000003},"min":{"1min":0.67200000000000004,"5min":0.61099999999999999,"15min":0.61099999999999999},"max":{"1min":1.1890000000000001,"5min":1.2769999999999999,"15min":1.2769999999999999},"last":0.90500000000000003},{"interface":"front","average":{"1min":0.90500000000000003,"5min":0.95799999999999996,"15min":0.95499999999999996},"min":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"max":{"1min":1.268,"5min":1.4710000000000001,"15min":1.4710000000000001},"last":1.01}]},{"osd":5,"last update":"Sun Oct 1 10:14:34 2023","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":1.0269999999999999,"15min":1.0329999999999999},"min":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"max":{"1min":1.25,"5min":1.603,"15min":1.603},"last":0.97299999999999998},{"interface":"front","average":{"1min":1.028,"5min":1.054,"15min":1.042},"min":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"max":{"1min":1.335,"5min":1.4330000000000001,"15min":1.4330000000000001},"last":1.0589999999999999}]},{"osd":6,"last update":"Sun Oct 1 10:15:06 2023","interfaces":[{"interface":"back","average":{"1min":0.89900000000000002,"5min":1.0660000000000001,"15min":1.0900000000000001},"min":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"max":{"1min":1.0680000000000001,"5min":1.4970000000000001,"15min":1.4970000000000001},"last":1.091},{"interface":"front","average":{"1min":0.94299999999999995,"5min":1.0569999999999999,"15min":1.0840000000000001},"min":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"max":{"1min":1.1000000000000001,"5min":1.399,"15min":1.399},"last":1.1890000000000001}]},{"osd":7,"last update":"Sun Oct 1 10:14:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":1.161,"15min":1.1799999999999999},"min":{"1min":0.82299999999999995,"5min":0.80700000000000005,"15min":0.80700000000000005},"max":{"1min":1.242,"5min":1.46,"15min":1.46},"last":1.3240000000000001},{"interface":"front","average":{"1min":1.014,"5min":1.1839999999999999,"15min":1.212},"min":{"1min":0.66500000000000004,"5min":0.56899999999999995,"15min":0.56899999999999995},"max":{"1min":1.345,"5min":1.661,"15min":1.661},"last":1.427}]}]},{"osd":4,"up_from":30,"seq":128849018939,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297624,"kb_used_data":528,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447528,"statfs":{"total":95995035648,"available":95690268672,"internally_reserved":0,"allocated":540672,"data_stored":220515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:14:49 2023","interfaces":[{"interface":"back","average":{"1min":0.748,"5min":0.82499999999999996,"15min":0.83499999999999996},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":1.028,"5min":1.22,"15min":1.22},"last":0.73899999999999999},{"interface":"front","average":{"1min":0.81799999999999995,"5min":0.85399999999999998,"15min":0.86599999999999999},"min":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"max":{"1min":1.1439999999999999,"5min":1.1859999999999999,"15min":1.1859999999999999},"last":0.95999999999999996}]},{"osd":1,"last update":"Sun Oct 1 10:14:49 2023","interfaces":[{"interface":"back","average":{"1min":0.79400000000000004,"5min":0.85199999999999998,"15min":0.86899999999999999},"min":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":1.484,"5min":1.484,"15min":1.484},"last":0.65400000000000003},{"interface":"front","average":{"1min":0.82199999999999995,"5min":0.88400000000000001,"15min":0.88900000000000001},"min":{"1min":0.67500000000000004,"5min":0.65200000000000002,"15min":0.65200000000000002},"max":{"1min":1.1899999999999999,"5min":1.325,"15min":1.325},"last":0.80800000000000005}]},{"osd":2,"last update":"Sun Oct 1 10:14:49 2023","interfaces":[{"interface":"back","average":{"1min":0.88700000000000001,"5min":0.90000000000000002,"15min":0.89200000000000002},"min":{"1min":0.63900000000000001,"5min":0.58899999999999997,"15min":0.58899999999999997},"max":{"1min":1.095,"5min":1.3859999999999999,"15min":1.3859999999999999},"last":0.68700000000000006},{"interface":"front","average":{"1min":0.83299999999999996,"5min":0.876,"15min":0.878},"min":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"max":{"1min":0.97599999999999998,"5min":1.659,"15min":1.659},"last":0.84699999999999998}]},{"osd":3,"last update":"Sun Oct 1 10:14:49 2023","interfaces":[{"interface":"back","average":{"1min":0.92900000000000005,"5min":0.96099999999999997,"15min":0.96499999999999997},"min":{"1min":0.67100000000000004,"5min":0.56799999999999995,"15min":0.56799999999999995},"max":{"1min":1.206,"5min":1.278,"15min":1.278},"last":0.754},{"interface":"front","average":{"1min":0.92600000000000005,"5min":0.96199999999999997,"15min":0.95799999999999996},"min":{"1min":0.68999999999999995,"5min":0.61699999999999999,"15min":0.61699999999999999},"max":{"1min":1.585,"5min":1.585,"15min":1.585},"last":1.022}]},{"osd":5,"last update":"Sun Oct 1 10:14:29 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":0.82799999999999996,"15min":0.79300000000000004},"min":{"1min":0.55700000000000005,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":1.218,"5min":1.341,"15min":1.341},"last":0.88800000000000001},{"interface":"front","average":{"1min":0.96899999999999997,"5min":0.88200000000000001,"15min":0.84799999999999998},"min":{"1min":0.68500000000000005,"5min":0.51000000000000001,"15min":0.51000000000000001},"max":{"1min":1.2829999999999999,"5min":1.4219999999999999,"15min":1.4219999999999999},"last":1.0029999999999999}]},{"osd":6,"last update":"Sun Oct 1 10:15:11 2023","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":0.84999999999999998,"15min":0.82699999999999996},"min":{"1min":0.627,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":1.2549999999999999,"5min":1.3360000000000001,"15min":1.3360000000000001},"last":0.92100000000000004},{"interface":"front","average":{"1min":1.01,"5min":0.93799999999999994,"15min":0.92000000000000004},"min":{"1min":0.64200000000000002,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":1.1910000000000001,"5min":1.284,"15min":1.284},"last":1.0580000000000001}]},{"osd":7,"last update":"Sun Oct 1 10:14:49 2023","interfaces":[{"interface":"back","average":{"1min":0.95699999999999996,"5min":0.94399999999999995,"15min":0.94199999999999995},"min":{"1min":0.67700000000000005,"5min":0.64400000000000002,"15min":0.64400000000000002},"max":{"1min":1.2150000000000001,"5min":1.2749999999999999,"15min":1.2749999999999999},"last":0.93700000000000006},{"interface":"front","average":{"1min":0.99299999999999999,"5min":0.92900000000000005,"15min":0.91800000000000004},"min":{"1min":0.61399999999999999,"5min":0.495,"15min":0.495},"max":{"1min":1.3580000000000001,"5min":1.3580000000000001,"15min":1.3580000000000001},"last":1.212}]}]},{"osd":5,"up_from":36,"seq":154618822705,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297620,"kb_used_data":524,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447532,"statfs":{"total":95995035648,"available":95690272768,"internally_reserved":0,"allocated":536576,"data_stored":220200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:14:31 2023","interfaces":[{"interface":"back","average":{"1min":0.85999999999999999,"5min":0.89200000000000002,"15min":0.90200000000000002},"min":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"max":{"1min":1.1240000000000001,"5min":1.339,"15min":1.339},"last":1.028},{"interface":"front","average":{"1min":0.79700000000000004,"5min":0.85499999999999998,"15min":0.873},"min":{"1min":0.627,"5min":0.58299999999999996,"15min":0.58299999999999996},"max":{"1min":0.996,"5min":1.1679999999999999,"15min":1.1679999999999999},"last":0.91600000000000004}]},{"osd":1,"last update":"Sun Oct 1 10:14:31 2023","interfaces":[{"interface":"back","average":{"1min":0.85199999999999998,"5min":0.85899999999999999,"15min":0.85299999999999998},"min":{"1min":0.627,"5min":0.627,"15min":0.627},"max":{"1min":1.119,"5min":1.119,"15min":1.119},"last":0.89900000000000002},{"interface":"front","average":{"1min":0.86599999999999999,"5min":0.90300000000000002,"15min":0.90900000000000003},"min":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"max":{"1min":1.1990000000000001,"5min":1.1990000000000001,"15min":1.1990000000000001},"last":0.97899999999999998}]},{"osd":2,"last update":"Sun Oct 1 10:14:31 2023","interfaces":[{"interface":"back","average":{"1min":0.88600000000000001,"5min":0.86899999999999999,"15min":0.86399999999999999},"min":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"max":{"1min":1.155,"5min":1.155,"15min":1.155},"last":0.83299999999999996},{"interface":"front","average":{"1min":0.85999999999999999,"5min":0.91100000000000003,"15min":0.91700000000000004},"min":{"1min":0.67800000000000005,"5min":0.57199999999999995,"15min":0.57199999999999995},"max":{"1min":1.153,"5min":1.214,"15min":1.214},"last":0.877}]},{"osd":3,"last update":"Sun Oct 1 10:14:31 2023","interfaces":[{"interface":"back","average":{"1min":0.86699999999999999,"5min":0.90000000000000002,"15min":0.91300000000000003},"min":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"max":{"1min":1.129,"5min":1.1339999999999999,"15min":1.1339999999999999},"last":0.93400000000000005},{"interface":"front","average":{"1min":0.86099999999999999,"5min":0.90100000000000002,"15min":0.90600000000000003},"min":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"max":{"1min":1.04,"5min":1.2490000000000001,"15min":1.2490000000000001},"last":0.74099999999999999}]},{"osd":4,"last update":"Sun Oct 1 10:14:31 2023","interfaces":[{"interface":"back","average":{"1min":0.81200000000000006,"5min":0.75900000000000001,"15min":0.73299999999999998},"min":{"1min":0.47099999999999997,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":1.256,"5min":1.256,"15min":1.256},"last":0.71999999999999997},{"interface":"front","average":{"1min":0.90100000000000002,"5min":0.84499999999999997,"15min":0.81200000000000006},"min":{"1min":0.61099999999999999,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":1.2929999999999999,"5min":1.431,"15min":1.431},"last":0.77900000000000003}]},{"osd":6,"last update":"Sun Oct 1 10:15:10 2023","interfaces":[{"interface":"back","average":{"1min":0.95599999999999996,"5min":0.92500000000000004,"15min":0.92900000000000005},"min":{"1min":0.59199999999999997,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":1.3149999999999999,"5min":1.333,"15min":1.333},"last":0.95099999999999996},{"interface":"front","average":{"1min":0.95399999999999996,"5min":0.90700000000000003,"15min":0.89700000000000002},"min":{"1min":0.63600000000000001,"5min":0.434,"15min":0.434},"max":{"1min":1.238,"5min":1.2649999999999999,"15min":1.2649999999999999},"last":0.85799999999999998}]},{"osd":7,"last update":"Sun Oct 1 10:14:53 2023","interfaces":[{"interface":"back","average":{"1min":0.91500000000000004,"5min":0.90700000000000003,"15min":0.90600000000000003},"min":{"1min":0.56299999999999994,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":1.3009999999999999,"5min":1.3009999999999999,"15min":1.3009999999999999},"last":1.008},{"interface":"front","average":{"1min":0.98999999999999999,"5min":1.0189999999999999,"15min":1.024},"min":{"1min":0.57599999999999996,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":1.377,"5min":1.377,"15min":1.377},"last":1.038}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T10:15:26.015 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-10-01T10:15:26.015 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-10-01T10:15:26.015 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-10-01T10:15:26.015 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph health --format=json 2023-10-01T10:15:26.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:26 smithi078 bash[27336]: cluster 2023-10-01T10:15:25.187305+0000 mgr.y (mgr.14427) 92 : cluster [DBG] pgmap v71: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:26.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:26 smithi078 bash[27336]: audit 2023-10-01T10:15:25.283124+0000 mgr.y (mgr.14427) 93 : audit [DBG] from='client.14604 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:15:26.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:26 smithi078 bash[33821]: cluster 2023-10-01T10:15:25.187305+0000 mgr.y (mgr.14427) 92 : cluster [DBG] pgmap v71: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:26.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:26 smithi078 bash[33821]: audit 2023-10-01T10:15:25.283124+0000 mgr.y (mgr.14427) 93 : audit [DBG] from='client.14604 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:15:26.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:26 smithi173 bash[27612]: cluster 2023-10-01T10:15:25.187305+0000 mgr.y (mgr.14427) 92 : cluster [DBG] pgmap v71: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:26.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:26 smithi173 bash[27612]: audit 2023-10-01T10:15:25.283124+0000 mgr.y (mgr.14427) 93 : audit [DBG] from='client.14604 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:15:28.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:27 smithi078 bash[27336]: audit 2023-10-01T10:15:26.878342+0000 mon.a (mon.0) 644 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:28.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:27 smithi078 bash[27336]: audit 2023-10-01T10:15:26.880211+0000 mon.a (mon.0) 645 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:15:28.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:27 smithi078 bash[27336]: audit 2023-10-01T10:15:26.881671+0000 mon.a (mon.0) 646 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:15:28.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:27 smithi078 bash[27336]: audit 2023-10-01T10:15:26.887989+0000 mon.a (mon.0) 647 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:28.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:27 smithi078 bash[27336]: cluster 2023-10-01T10:15:27.188017+0000 mgr.y (mgr.14427) 94 : cluster [DBG] pgmap v72: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:28.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:27 smithi078 bash[33821]: audit 2023-10-01T10:15:26.878342+0000 mon.a (mon.0) 644 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:28.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:27 smithi078 bash[33821]: audit 2023-10-01T10:15:26.880211+0000 mon.a (mon.0) 645 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:15:28.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:27 smithi078 bash[33821]: audit 2023-10-01T10:15:26.881671+0000 mon.a (mon.0) 646 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:15:28.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:27 smithi078 bash[33821]: audit 2023-10-01T10:15:26.887989+0000 mon.a (mon.0) 647 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:28.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:27 smithi078 bash[33821]: cluster 2023-10-01T10:15:27.188017+0000 mgr.y (mgr.14427) 94 : cluster [DBG] pgmap v72: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:28.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:27 smithi173 bash[27612]: audit 2023-10-01T10:15:26.878342+0000 mon.a (mon.0) 644 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:28.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:27 smithi173 bash[27612]: audit 2023-10-01T10:15:26.880211+0000 mon.a (mon.0) 645 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:15:28.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:27 smithi173 bash[27612]: audit 2023-10-01T10:15:26.881671+0000 mon.a (mon.0) 646 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:15:28.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:27 smithi173 bash[27612]: audit 2023-10-01T10:15:26.887989+0000 mon.a (mon.0) 647 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:28.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:27 smithi173 bash[27612]: cluster 2023-10-01T10:15:27.188017+0000 mgr.y (mgr.14427) 94 : cluster [DBG] pgmap v72: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:30.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:30 smithi078 bash[27336]: cluster 2023-10-01T10:15:29.188762+0000 mgr.y (mgr.14427) 95 : cluster [DBG] pgmap v73: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:30.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:30 smithi078 bash[33821]: cluster 2023-10-01T10:15:29.188762+0000 mgr.y (mgr.14427) 95 : cluster [DBG] pgmap v73: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:30.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:30 smithi173 bash[27612]: cluster 2023-10-01T10:15:29.188762+0000 mgr.y (mgr.14427) 95 : cluster [DBG] pgmap v73: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:30.778 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:15:32.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:31 smithi078 bash[33821]: cluster 2023-10-01T10:15:31.189395+0000 mgr.y (mgr.14427) 96 : cluster [DBG] pgmap v74: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:32.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:31 smithi078 bash[27336]: cluster 2023-10-01T10:15:31.189395+0000 mgr.y (mgr.14427) 96 : cluster [DBG] pgmap v74: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:32.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:31 smithi173 bash[27612]: cluster 2023-10-01T10:15:31.189395+0000 mgr.y (mgr.14427) 96 : cluster [DBG] pgmap v74: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:33.786 INFO:teuthology.orchestra.run.smithi078.stdout: 2023-10-01T10:15:34.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:34 smithi078 bash[27336]: cluster 2023-10-01T10:15:33.190063+0000 mgr.y (mgr.14427) 97 : cluster [DBG] pgmap v75: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:34.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:34 smithi078 bash[27336]: audit 2023-10-01T10:15:33.784144+0000 mon.c (mon.2) 29 : audit [DBG] from='client.? 172.21.15.78:0/1465329542' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-01T10:15:34.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:34 smithi078 bash[33821]: cluster 2023-10-01T10:15:33.190063+0000 mgr.y (mgr.14427) 97 : cluster [DBG] pgmap v75: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:34.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:34 smithi078 bash[33821]: audit 2023-10-01T10:15:33.784144+0000 mon.c (mon.2) 29 : audit [DBG] from='client.? 172.21.15.78:0/1465329542' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-01T10:15:34.538 INFO:teuthology.orchestra.run.smithi078.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-10-01T10:15:34.539 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-10-01T10:15:34.539 INFO:tasks.cephadm:Setup complete, yielding 2023-10-01T10:15:34.539 INFO:teuthology.run_tasks:Running task cephadm.shell... 2023-10-01T10:15:34.548 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi078.front.sepia.ceph.com 2023-10-01T10:15:34.549 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- bash -c 'ceph fs volume create foo' 2023-10-01T10:15:34.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:34 smithi173 bash[27612]: cluster 2023-10-01T10:15:33.190063+0000 mgr.y (mgr.14427) 97 : cluster [DBG] pgmap v75: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:34.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:34 smithi173 bash[27612]: audit 2023-10-01T10:15:33.784144+0000 mon.c (mon.2) 29 : audit [DBG] from='client.? 172.21.15.78:0/1465329542' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-01T10:15:35.571 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:15:35 smithi078 bash[27601]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:15:35] "GET /metrics HTTP/1.1" 200 155684 "" "Prometheus/2.33.4" 2023-10-01T10:15:35.620 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:15:35 smithi173 bash[28473]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:15:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T10:15:36.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:36 smithi078 bash[27336]: cluster 2023-10-01T10:15:35.190411+0000 mgr.y (mgr.14427) 98 : cluster [DBG] pgmap v76: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:36.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:36 smithi078 bash[33821]: cluster 2023-10-01T10:15:35.190411+0000 mgr.y (mgr.14427) 98 : cluster [DBG] pgmap v76: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:36.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:36 smithi173 bash[27612]: cluster 2023-10-01T10:15:35.190411+0000 mgr.y (mgr.14427) 98 : cluster [DBG] pgmap v76: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:38.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:38 smithi078 bash[33821]: cluster 2023-10-01T10:15:37.190941+0000 mgr.y (mgr.14427) 99 : cluster [DBG] pgmap v77: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:38.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:38 smithi078 bash[33821]: audit 2023-10-01T10:15:37.791106+0000 mon.a (mon.0) 648 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-10-01T10:15:38.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:38 smithi078 bash[27336]: cluster 2023-10-01T10:15:37.190941+0000 mgr.y (mgr.14427) 99 : cluster [DBG] pgmap v77: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:38.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:38 smithi078 bash[27336]: audit 2023-10-01T10:15:37.791106+0000 mon.a (mon.0) 648 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-10-01T10:15:38.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:38 smithi173 bash[27612]: cluster 2023-10-01T10:15:37.190941+0000 mgr.y (mgr.14427) 99 : cluster [DBG] pgmap v77: 9 pgs: 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:38.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:38 smithi173 bash[27612]: audit 2023-10-01T10:15:37.791106+0000 mon.a (mon.0) 648 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-10-01T10:15:39.569 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:39 smithi078 bash[27336]: audit 2023-10-01T10:15:37.790187+0000 mgr.y (mgr.14427) 100 : audit [DBG] from='client.14616 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:15:39.570 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:39 smithi078 bash[27336]: audit 2023-10-01T10:15:38.265005+0000 mon.a (mon.0) 649 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-10-01T10:15:39.570 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:39 smithi078 bash[27336]: cluster 2023-10-01T10:15:38.265194+0000 mon.a (mon.0) 650 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2023-10-01T10:15:39.570 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:39 smithi078 bash[27336]: audit 2023-10-01T10:15:38.266239+0000 mon.a (mon.0) 651 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-10-01T10:15:39.570 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:39 smithi078 bash[27336]: debug 2023-10-01T10:15:39.284+0000 7f54ceca5700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-10-01T10:15:39.570 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:39 smithi078 bash[33821]: audit 2023-10-01T10:15:37.790187+0000 mgr.y (mgr.14427) 100 : audit [DBG] from='client.14616 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:15:39.570 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:39 smithi078 bash[33821]: audit 2023-10-01T10:15:38.265005+0000 mon.a (mon.0) 649 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-10-01T10:15:39.570 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:39 smithi078 bash[33821]: cluster 2023-10-01T10:15:38.265194+0000 mon.a (mon.0) 650 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2023-10-01T10:15:39.571 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:39 smithi078 bash[33821]: audit 2023-10-01T10:15:38.266239+0000 mon.a (mon.0) 651 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-10-01T10:15:39.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:39 smithi173 bash[27612]: audit 2023-10-01T10:15:37.790187+0000 mgr.y (mgr.14427) 100 : audit [DBG] from='client.14616 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:15:39.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:39 smithi173 bash[27612]: audit 2023-10-01T10:15:38.265005+0000 mon.a (mon.0) 649 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-10-01T10:15:39.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:39 smithi173 bash[27612]: cluster 2023-10-01T10:15:38.265194+0000 mon.a (mon.0) 650 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2023-10-01T10:15:39.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:39 smithi173 bash[27612]: audit 2023-10-01T10:15:38.266239+0000 mon.a (mon.0) 651 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-10-01T10:15:40.214 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2023-10-01T10:15:40.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:40 smithi078 bash[27336]: cluster 2023-10-01T10:15:39.191520+0000 mgr.y (mgr.14427) 101 : cluster [DBG] pgmap v79: 41 pgs: 32 unknown, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:40.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:40 smithi078 bash[27336]: audit 2023-10-01T10:15:39.276663+0000 mon.a (mon.0) 652 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-10-01T10:15:40.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:40 smithi078 bash[27336]: cluster 2023-10-01T10:15:39.276862+0000 mon.a (mon.0) 653 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2023-10-01T10:15:40.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:40 smithi078 bash[27336]: audit 2023-10-01T10:15:39.282919+0000 mon.a (mon.0) 654 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-10-01T10:15:40.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:40 smithi078 bash[27336]: cluster 2023-10-01T10:15:39.285274+0000 mon.a (mon.0) 655 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-10-01T10:15:40.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:40 smithi078 bash[27336]: cluster 2023-10-01T10:15:39.285314+0000 mon.a (mon.0) 656 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-10-01T10:15:40.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:40 smithi078 bash[27336]: cluster 2023-10-01T10:15:39.296943+0000 mon.a (mon.0) 657 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2023-10-01T10:15:40.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:40 smithi078 bash[27336]: audit 2023-10-01T10:15:39.297423+0000 mon.a (mon.0) 658 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-10-01T10:15:40.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:40 smithi078 bash[27336]: cluster 2023-10-01T10:15:39.297572+0000 mon.a (mon.0) 659 : cluster [DBG] fsmap foo:0 2023-10-01T10:15:40.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:40 smithi078 bash[27336]: cephadm 2023-10-01T10:15:39.302618+0000 mgr.y (mgr.14427) 102 : cephadm [INF] Saving service mds.foo spec with placement count:2 2023-10-01T10:15:40.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:40 smithi078 bash[27336]: audit 2023-10-01T10:15:39.309798+0000 mon.a (mon.0) 660 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:40.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:40 smithi078 bash[27336]: audit 2023-10-01T10:15:39.339094+0000 mon.a (mon.0) 661 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:15:40.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:40 smithi078 bash[27336]: audit 2023-10-01T10:15:39.622529+0000 mon.a (mon.0) 662 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:15:40.585 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:40 smithi078 bash[27336]: audit 2023-10-01T10:15:39.623923+0000 mon.a (mon.0) 663 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:15:40.585 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:40 smithi078 bash[27336]: audit 2023-10-01T10:15:39.630260+0000 mon.a (mon.0) 664 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:40.585 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:40 smithi078 bash[27336]: audit 2023-10-01T10:15:39.638808+0000 mon.a (mon.0) 665 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:40.585 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:40 smithi078 bash[27336]: audit 2023-10-01T10:15:39.640398+0000 mon.a (mon.0) 666 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi078.vjqrth", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-01T10:15:40.585 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:40 smithi078 bash[27336]: audit 2023-10-01T10:15:39.645172+0000 mon.a (mon.0) 667 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi078.vjqrth", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-01T10:15:40.585 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:40 smithi078 bash[27336]: audit 2023-10-01T10:15:39.646416+0000 mon.a (mon.0) 668 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:15:40.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:40 smithi078 bash[33821]: cluster 2023-10-01T10:15:39.191520+0000 mgr.y (mgr.14427) 101 : cluster [DBG] pgmap v79: 41 pgs: 32 unknown, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:40.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:40 smithi078 bash[33821]: audit 2023-10-01T10:15:39.276663+0000 mon.a (mon.0) 652 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-10-01T10:15:40.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:40 smithi078 bash[33821]: cluster 2023-10-01T10:15:39.276862+0000 mon.a (mon.0) 653 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2023-10-01T10:15:40.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:40 smithi078 bash[33821]: audit 2023-10-01T10:15:39.282919+0000 mon.a (mon.0) 654 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-10-01T10:15:40.586 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:40 smithi078 bash[33821]: cluster 2023-10-01T10:15:39.285274+0000 mon.a (mon.0) 655 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-10-01T10:15:40.586 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:40 smithi078 bash[33821]: cluster 2023-10-01T10:15:39.285314+0000 mon.a (mon.0) 656 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-10-01T10:15:40.586 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:40 smithi078 bash[33821]: cluster 2023-10-01T10:15:39.296943+0000 mon.a (mon.0) 657 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2023-10-01T10:15:40.586 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:40 smithi078 bash[33821]: audit 2023-10-01T10:15:39.297423+0000 mon.a (mon.0) 658 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-10-01T10:15:40.586 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:40 smithi078 bash[33821]: cluster 2023-10-01T10:15:39.297572+0000 mon.a (mon.0) 659 : cluster [DBG] fsmap foo:0 2023-10-01T10:15:40.586 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:40 smithi078 bash[33821]: cephadm 2023-10-01T10:15:39.302618+0000 mgr.y (mgr.14427) 102 : cephadm [INF] Saving service mds.foo spec with placement count:2 2023-10-01T10:15:40.586 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:40 smithi078 bash[33821]: audit 2023-10-01T10:15:39.309798+0000 mon.a (mon.0) 660 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:40.586 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:40 smithi078 bash[33821]: audit 2023-10-01T10:15:39.339094+0000 mon.a (mon.0) 661 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:15:40.586 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:40 smithi078 bash[33821]: audit 2023-10-01T10:15:39.622529+0000 mon.a (mon.0) 662 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:15:40.586 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:40 smithi078 bash[33821]: audit 2023-10-01T10:15:39.623923+0000 mon.a (mon.0) 663 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:15:40.586 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:40 smithi078 bash[33821]: audit 2023-10-01T10:15:39.630260+0000 mon.a (mon.0) 664 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:40.586 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:40 smithi078 bash[33821]: audit 2023-10-01T10:15:39.638808+0000 mon.a (mon.0) 665 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:40.587 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:40 smithi078 bash[33821]: audit 2023-10-01T10:15:39.640398+0000 mon.a (mon.0) 666 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi078.vjqrth", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-01T10:15:40.587 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:40 smithi078 bash[33821]: audit 2023-10-01T10:15:39.645172+0000 mon.a (mon.0) 667 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi078.vjqrth", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-01T10:15:40.587 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:40 smithi078 bash[33821]: audit 2023-10-01T10:15:39.646416+0000 mon.a (mon.0) 668 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:15:40.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:40 smithi173 bash[27612]: cluster 2023-10-01T10:15:39.191520+0000 mgr.y (mgr.14427) 101 : cluster [DBG] pgmap v79: 41 pgs: 32 unknown, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:40.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:40 smithi173 bash[27612]: audit 2023-10-01T10:15:39.276663+0000 mon.a (mon.0) 652 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-10-01T10:15:40.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:40 smithi173 bash[27612]: cluster 2023-10-01T10:15:39.276862+0000 mon.a (mon.0) 653 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2023-10-01T10:15:40.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:40 smithi173 bash[27612]: audit 2023-10-01T10:15:39.282919+0000 mon.a (mon.0) 654 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-10-01T10:15:40.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:40 smithi173 bash[27612]: cluster 2023-10-01T10:15:39.285274+0000 mon.a (mon.0) 655 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-10-01T10:15:40.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:40 smithi173 bash[27612]: cluster 2023-10-01T10:15:39.285314+0000 mon.a (mon.0) 656 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-10-01T10:15:40.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:40 smithi173 bash[27612]: cluster 2023-10-01T10:15:39.296943+0000 mon.a (mon.0) 657 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2023-10-01T10:15:40.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:40 smithi173 bash[27612]: audit 2023-10-01T10:15:39.297423+0000 mon.a (mon.0) 658 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-10-01T10:15:40.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:40 smithi173 bash[27612]: cluster 2023-10-01T10:15:39.297572+0000 mon.a (mon.0) 659 : cluster [DBG] fsmap foo:0 2023-10-01T10:15:40.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:40 smithi173 bash[27612]: cephadm 2023-10-01T10:15:39.302618+0000 mgr.y (mgr.14427) 102 : cephadm [INF] Saving service mds.foo spec with placement count:2 2023-10-01T10:15:40.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:40 smithi173 bash[27612]: audit 2023-10-01T10:15:39.309798+0000 mon.a (mon.0) 660 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:40.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:40 smithi173 bash[27612]: audit 2023-10-01T10:15:39.339094+0000 mon.a (mon.0) 661 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:15:40.622 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:40 smithi173 bash[27612]: audit 2023-10-01T10:15:39.622529+0000 mon.a (mon.0) 662 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:15:40.622 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:40 smithi173 bash[27612]: audit 2023-10-01T10:15:39.623923+0000 mon.a (mon.0) 663 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:15:40.622 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:40 smithi173 bash[27612]: audit 2023-10-01T10:15:39.630260+0000 mon.a (mon.0) 664 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:40.622 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:40 smithi173 bash[27612]: audit 2023-10-01T10:15:39.638808+0000 mon.a (mon.0) 665 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:40.622 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:40 smithi173 bash[27612]: audit 2023-10-01T10:15:39.640398+0000 mon.a (mon.0) 666 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi078.vjqrth", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-01T10:15:40.622 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:40 smithi173 bash[27612]: audit 2023-10-01T10:15:39.645172+0000 mon.a (mon.0) 667 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi078.vjqrth", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-01T10:15:40.622 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:40 smithi173 bash[27612]: audit 2023-10-01T10:15:39.646416+0000 mon.a (mon.0) 668 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:15:41.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:41 smithi078 bash[27336]: cephadm 2023-10-01T10:15:39.647533+0000 mgr.y (mgr.14427) 103 : cephadm [INF] Deploying daemon mds.foo.smithi078.vjqrth on smithi078 2023-10-01T10:15:41.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:41 smithi078 bash[27336]: cluster 2023-10-01T10:15:40.299657+0000 mon.a (mon.0) 669 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2023-10-01T10:15:41.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:41 smithi078 bash[33821]: cephadm 2023-10-01T10:15:39.647533+0000 mgr.y (mgr.14427) 103 : cephadm [INF] Deploying daemon mds.foo.smithi078.vjqrth on smithi078 2023-10-01T10:15:41.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:41 smithi078 bash[33821]: cluster 2023-10-01T10:15:40.299657+0000 mon.a (mon.0) 669 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2023-10-01T10:15:41.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:41 smithi173 bash[27612]: cephadm 2023-10-01T10:15:39.647533+0000 mgr.y (mgr.14427) 103 : cephadm [INF] Deploying daemon mds.foo.smithi078.vjqrth on smithi078 2023-10-01T10:15:41.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:41 smithi173 bash[27612]: cluster 2023-10-01T10:15:40.299657+0000 mon.a (mon.0) 669 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2023-10-01T10:15:42.587 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:42 smithi078 bash[33821]: cluster 2023-10-01T10:15:41.192063+0000 mgr.y (mgr.14427) 104 : cluster [DBG] pgmap v83: 73 pgs: 64 unknown, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:42.588 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:42 smithi078 bash[33821]: cluster 2023-10-01T10:15:41.305618+0000 mon.a (mon.0) 670 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2023-10-01T10:15:42.588 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:42 smithi078 bash[27336]: cluster 2023-10-01T10:15:41.192063+0000 mgr.y (mgr.14427) 104 : cluster [DBG] pgmap v83: 73 pgs: 64 unknown, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:42.588 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:42 smithi078 bash[27336]: cluster 2023-10-01T10:15:41.305618+0000 mon.a (mon.0) 670 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2023-10-01T10:15:42.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:42 smithi173 bash[27612]: cluster 2023-10-01T10:15:41.192063+0000 mgr.y (mgr.14427) 104 : cluster [DBG] pgmap v83: 73 pgs: 64 unknown, 9 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:42.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:42 smithi173 bash[27612]: cluster 2023-10-01T10:15:41.305618+0000 mon.a (mon.0) 670 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2023-10-01T10:15:44.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:43 smithi078 bash[33821]: audit 2023-10-01T10:15:42.772039+0000 mon.a (mon.0) 671 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:44.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:43 smithi078 bash[33821]: audit 2023-10-01T10:15:42.773691+0000 mon.a (mon.0) 672 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi173.vrqqoc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-01T10:15:44.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:43 smithi078 bash[33821]: audit 2023-10-01T10:15:42.780029+0000 mon.a (mon.0) 673 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi173.vrqqoc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-01T10:15:44.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:43 smithi078 bash[33821]: audit 2023-10-01T10:15:42.781151+0000 mon.a (mon.0) 674 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:15:44.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:43 smithi078 bash[33821]: cephadm 2023-10-01T10:15:42.782056+0000 mgr.y (mgr.14427) 105 : cephadm [INF] Deploying daemon mds.foo.smithi173.vrqqoc on smithi173 2023-10-01T10:15:44.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:43 smithi078 bash[33821]: cluster 2023-10-01T10:15:43.192954+0000 mgr.y (mgr.14427) 106 : cluster [DBG] pgmap v85: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:44.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:43 smithi078 bash[27336]: audit 2023-10-01T10:15:42.772039+0000 mon.a (mon.0) 671 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:44.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:43 smithi078 bash[27336]: audit 2023-10-01T10:15:42.773691+0000 mon.a (mon.0) 672 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi173.vrqqoc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-01T10:15:44.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:43 smithi078 bash[27336]: audit 2023-10-01T10:15:42.780029+0000 mon.a (mon.0) 673 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi173.vrqqoc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-01T10:15:44.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:43 smithi078 bash[27336]: audit 2023-10-01T10:15:42.781151+0000 mon.a (mon.0) 674 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:15:44.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:43 smithi078 bash[27336]: cephadm 2023-10-01T10:15:42.782056+0000 mgr.y (mgr.14427) 105 : cephadm [INF] Deploying daemon mds.foo.smithi173.vrqqoc on smithi173 2023-10-01T10:15:44.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:43 smithi078 bash[27336]: cluster 2023-10-01T10:15:43.192954+0000 mgr.y (mgr.14427) 106 : cluster [DBG] pgmap v85: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:44.105 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:43 smithi173 bash[27612]: audit 2023-10-01T10:15:42.772039+0000 mon.a (mon.0) 671 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:44.105 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:43 smithi173 bash[27612]: audit 2023-10-01T10:15:42.773691+0000 mon.a (mon.0) 672 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi173.vrqqoc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-01T10:15:44.105 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:43 smithi173 bash[27612]: audit 2023-10-01T10:15:42.780029+0000 mon.a (mon.0) 673 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi173.vrqqoc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-10-01T10:15:44.105 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:43 smithi173 bash[27612]: audit 2023-10-01T10:15:42.781151+0000 mon.a (mon.0) 674 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:15:44.105 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:43 smithi173 bash[27612]: cephadm 2023-10-01T10:15:42.782056+0000 mgr.y (mgr.14427) 105 : cephadm [INF] Deploying daemon mds.foo.smithi173.vrqqoc on smithi173 2023-10-01T10:15:44.105 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:43 smithi173 bash[27612]: cluster 2023-10-01T10:15:43.192954+0000 mgr.y (mgr.14427) 106 : cluster [DBG] pgmap v85: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:45.333 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:15:45 smithi078 bash[27601]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:15:45] "GET /metrics HTTP/1.1" 200 155684 "" "Prometheus/2.33.4" 2023-10-01T10:15:45.620 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:15:45 smithi173 bash[28473]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:15:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T10:15:45.728 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2023-10-01T10:15:46.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:45 smithi078 bash[27336]: cluster 2023-10-01T10:15:44.785247+0000 mon.a (mon.0) 675 : cluster [DBG] mds.? [v2:172.21.15.78:6834/2348005403,v1:172.21.15.78:6835/2348005403] up:boot 2023-10-01T10:15:46.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:45 smithi078 bash[27336]: cluster 2023-10-01T10:15:44.785406+0000 mon.a (mon.0) 676 : cluster [INF] daemon mds.foo.smithi078.vjqrth assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-10-01T10:15:46.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:45 smithi078 bash[27336]: cluster 2023-10-01T10:15:44.785552+0000 mon.a (mon.0) 677 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-10-01T10:15:46.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:45 smithi078 bash[27336]: cluster 2023-10-01T10:15:44.785609+0000 mon.a (mon.0) 678 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-10-01T10:15:46.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:45 smithi078 bash[27336]: cluster 2023-10-01T10:15:44.785654+0000 mon.a (mon.0) 679 : cluster [INF] Cluster is now healthy 2023-10-01T10:15:46.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:45 smithi078 bash[27336]: cluster 2023-10-01T10:15:44.786028+0000 mon.a (mon.0) 680 : cluster [DBG] fsmap foo:0 1 up:standby 2023-10-01T10:15:46.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:45 smithi078 bash[27336]: audit 2023-10-01T10:15:44.786301+0000 mon.a (mon.0) 681 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi078.vjqrth"}]: dispatch 2023-10-01T10:15:46.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:45 smithi078 bash[27336]: cluster 2023-10-01T10:15:44.789592+0000 mon.a (mon.0) 682 : cluster [DBG] fsmap foo:1 {0=foo.smithi078.vjqrth=up:creating} 2023-10-01T10:15:46.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:45 smithi078 bash[27336]: cluster 2023-10-01T10:15:44.810629+0000 mon.a (mon.0) 683 : cluster [INF] daemon mds.foo.smithi078.vjqrth is now active in filesystem foo as rank 0 2023-10-01T10:15:46.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:45 smithi078 bash[27336]: cluster 2023-10-01T10:15:45.193570+0000 mgr.y (mgr.14427) 107 : cluster [DBG] pgmap v86: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:46.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:45 smithi078 bash[27336]: audit 2023-10-01T10:15:45.265309+0000 mon.a (mon.0) 684 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:46.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:45 smithi078 bash[27336]: audit 2023-10-01T10:15:45.267479+0000 mon.a (mon.0) 685 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:15:46.085 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:45 smithi078 bash[27336]: audit 2023-10-01T10:15:45.518576+0000 mon.a (mon.0) 686 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:46.085 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:45 smithi078 bash[33821]: cluster 2023-10-01T10:15:44.785247+0000 mon.a (mon.0) 675 : cluster [DBG] mds.? [v2:172.21.15.78:6834/2348005403,v1:172.21.15.78:6835/2348005403] up:boot 2023-10-01T10:15:46.085 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:45 smithi078 bash[33821]: cluster 2023-10-01T10:15:44.785406+0000 mon.a (mon.0) 676 : cluster [INF] daemon mds.foo.smithi078.vjqrth assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-10-01T10:15:46.085 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:45 smithi078 bash[33821]: cluster 2023-10-01T10:15:44.785552+0000 mon.a (mon.0) 677 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-10-01T10:15:46.085 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:45 smithi078 bash[33821]: cluster 2023-10-01T10:15:44.785609+0000 mon.a (mon.0) 678 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-10-01T10:15:46.085 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:45 smithi078 bash[33821]: cluster 2023-10-01T10:15:44.785654+0000 mon.a (mon.0) 679 : cluster [INF] Cluster is now healthy 2023-10-01T10:15:46.085 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:45 smithi078 bash[33821]: cluster 2023-10-01T10:15:44.786028+0000 mon.a (mon.0) 680 : cluster [DBG] fsmap foo:0 1 up:standby 2023-10-01T10:15:46.085 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:45 smithi078 bash[33821]: audit 2023-10-01T10:15:44.786301+0000 mon.a (mon.0) 681 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi078.vjqrth"}]: dispatch 2023-10-01T10:15:46.085 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:45 smithi078 bash[33821]: cluster 2023-10-01T10:15:44.789592+0000 mon.a (mon.0) 682 : cluster [DBG] fsmap foo:1 {0=foo.smithi078.vjqrth=up:creating} 2023-10-01T10:15:46.085 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:45 smithi078 bash[33821]: cluster 2023-10-01T10:15:44.810629+0000 mon.a (mon.0) 683 : cluster [INF] daemon mds.foo.smithi078.vjqrth is now active in filesystem foo as rank 0 2023-10-01T10:15:46.086 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:45 smithi078 bash[33821]: cluster 2023-10-01T10:15:45.193570+0000 mgr.y (mgr.14427) 107 : cluster [DBG] pgmap v86: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:46.086 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:45 smithi078 bash[33821]: audit 2023-10-01T10:15:45.265309+0000 mon.a (mon.0) 684 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:46.086 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:45 smithi078 bash[33821]: audit 2023-10-01T10:15:45.267479+0000 mon.a (mon.0) 685 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:15:46.086 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:45 smithi078 bash[33821]: audit 2023-10-01T10:15:45.518576+0000 mon.a (mon.0) 686 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:46.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:45 smithi173 bash[27612]: cluster 2023-10-01T10:15:44.785247+0000 mon.a (mon.0) 675 : cluster [DBG] mds.? [v2:172.21.15.78:6834/2348005403,v1:172.21.15.78:6835/2348005403] up:boot 2023-10-01T10:15:46.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:45 smithi173 bash[27612]: cluster 2023-10-01T10:15:44.785406+0000 mon.a (mon.0) 676 : cluster [INF] daemon mds.foo.smithi078.vjqrth assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-10-01T10:15:46.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:45 smithi173 bash[27612]: cluster 2023-10-01T10:15:44.785552+0000 mon.a (mon.0) 677 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-10-01T10:15:46.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:45 smithi173 bash[27612]: cluster 2023-10-01T10:15:44.785609+0000 mon.a (mon.0) 678 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-10-01T10:15:46.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:45 smithi173 bash[27612]: cluster 2023-10-01T10:15:44.785654+0000 mon.a (mon.0) 679 : cluster [INF] Cluster is now healthy 2023-10-01T10:15:46.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:45 smithi173 bash[27612]: cluster 2023-10-01T10:15:44.786028+0000 mon.a (mon.0) 680 : cluster [DBG] fsmap foo:0 1 up:standby 2023-10-01T10:15:46.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:45 smithi173 bash[27612]: audit 2023-10-01T10:15:44.786301+0000 mon.a (mon.0) 681 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi078.vjqrth"}]: dispatch 2023-10-01T10:15:46.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:45 smithi173 bash[27612]: cluster 2023-10-01T10:15:44.789592+0000 mon.a (mon.0) 682 : cluster [DBG] fsmap foo:1 {0=foo.smithi078.vjqrth=up:creating} 2023-10-01T10:15:46.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:45 smithi173 bash[27612]: cluster 2023-10-01T10:15:44.810629+0000 mon.a (mon.0) 683 : cluster [INF] daemon mds.foo.smithi078.vjqrth is now active in filesystem foo as rank 0 2023-10-01T10:15:46.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:45 smithi173 bash[27612]: cluster 2023-10-01T10:15:45.193570+0000 mgr.y (mgr.14427) 107 : cluster [DBG] pgmap v86: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:46.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:45 smithi173 bash[27612]: audit 2023-10-01T10:15:45.265309+0000 mon.a (mon.0) 684 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:46.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:45 smithi173 bash[27612]: audit 2023-10-01T10:15:45.267479+0000 mon.a (mon.0) 685 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:15:46.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:45 smithi173 bash[27612]: audit 2023-10-01T10:15:45.518576+0000 mon.a (mon.0) 686 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:47.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:46 smithi078 bash[27336]: cluster 2023-10-01T10:15:45.798079+0000 mon.a (mon.0) 687 : cluster [DBG] mds.? [v2:172.21.15.78:6834/2348005403,v1:172.21.15.78:6835/2348005403] up:active 2023-10-01T10:15:47.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:46 smithi078 bash[27336]: cluster 2023-10-01T10:15:45.798144+0000 mon.a (mon.0) 688 : cluster [DBG] fsmap foo:1 {0=foo.smithi078.vjqrth=up:active} 2023-10-01T10:15:47.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:46 smithi078 bash[33821]: cluster 2023-10-01T10:15:45.798079+0000 mon.a (mon.0) 687 : cluster [DBG] mds.? [v2:172.21.15.78:6834/2348005403,v1:172.21.15.78:6835/2348005403] up:active 2023-10-01T10:15:47.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:46 smithi078 bash[33821]: cluster 2023-10-01T10:15:45.798144+0000 mon.a (mon.0) 688 : cluster [DBG] fsmap foo:1 {0=foo.smithi078.vjqrth=up:active} 2023-10-01T10:15:47.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:46 smithi173 bash[27612]: cluster 2023-10-01T10:15:45.798079+0000 mon.a (mon.0) 687 : cluster [DBG] mds.? [v2:172.21.15.78:6834/2348005403,v1:172.21.15.78:6835/2348005403] up:active 2023-10-01T10:15:47.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:46 smithi173 bash[27612]: cluster 2023-10-01T10:15:45.798144+0000 mon.a (mon.0) 688 : cluster [DBG] fsmap foo:1 {0=foo.smithi078.vjqrth=up:active} 2023-10-01T10:15:48.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:47 smithi078 bash[27336]: cluster 2023-10-01T10:15:46.967008+0000 mon.a (mon.0) 689 : cluster [DBG] mds.? [v2:172.21.15.173:6832/2937343008,v1:172.21.15.173:6833/2937343008] up:boot 2023-10-01T10:15:48.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:47 smithi078 bash[27336]: cluster 2023-10-01T10:15:46.967421+0000 mon.a (mon.0) 690 : cluster [DBG] fsmap foo:1 {0=foo.smithi078.vjqrth=up:active} 1 up:standby 2023-10-01T10:15:48.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:47 smithi078 bash[27336]: audit 2023-10-01T10:15:46.967637+0000 mon.a (mon.0) 691 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi173.vrqqoc"}]: dispatch 2023-10-01T10:15:48.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:47 smithi078 bash[27336]: cluster 2023-10-01T10:15:46.973842+0000 mon.a (mon.0) 692 : cluster [DBG] fsmap foo:1 {0=foo.smithi078.vjqrth=up:active} 1 up:standby 2023-10-01T10:15:48.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:47 smithi078 bash[27336]: cluster 2023-10-01T10:15:47.194250+0000 mgr.y (mgr.14427) 108 : cluster [DBG] pgmap v87: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:48.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:47 smithi078 bash[27336]: audit 2023-10-01T10:15:47.465082+0000 mon.a (mon.0) 693 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:48.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:47 smithi078 bash[33821]: cluster 2023-10-01T10:15:46.967008+0000 mon.a (mon.0) 689 : cluster [DBG] mds.? [v2:172.21.15.173:6832/2937343008,v1:172.21.15.173:6833/2937343008] up:boot 2023-10-01T10:15:48.085 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:47 smithi078 bash[33821]: cluster 2023-10-01T10:15:46.967421+0000 mon.a (mon.0) 690 : cluster [DBG] fsmap foo:1 {0=foo.smithi078.vjqrth=up:active} 1 up:standby 2023-10-01T10:15:48.085 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:47 smithi078 bash[33821]: audit 2023-10-01T10:15:46.967637+0000 mon.a (mon.0) 691 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi173.vrqqoc"}]: dispatch 2023-10-01T10:15:48.085 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:47 smithi078 bash[33821]: cluster 2023-10-01T10:15:46.973842+0000 mon.a (mon.0) 692 : cluster [DBG] fsmap foo:1 {0=foo.smithi078.vjqrth=up:active} 1 up:standby 2023-10-01T10:15:48.085 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:47 smithi078 bash[33821]: cluster 2023-10-01T10:15:47.194250+0000 mgr.y (mgr.14427) 108 : cluster [DBG] pgmap v87: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:48.085 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:47 smithi078 bash[33821]: audit 2023-10-01T10:15:47.465082+0000 mon.a (mon.0) 693 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:48.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:47 smithi173 bash[27612]: cluster 2023-10-01T10:15:46.967008+0000 mon.a (mon.0) 689 : cluster [DBG] mds.? [v2:172.21.15.173:6832/2937343008,v1:172.21.15.173:6833/2937343008] up:boot 2023-10-01T10:15:48.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:47 smithi173 bash[27612]: cluster 2023-10-01T10:15:46.967421+0000 mon.a (mon.0) 690 : cluster [DBG] fsmap foo:1 {0=foo.smithi078.vjqrth=up:active} 1 up:standby 2023-10-01T10:15:48.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:47 smithi173 bash[27612]: audit 2023-10-01T10:15:46.967637+0000 mon.a (mon.0) 691 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi173.vrqqoc"}]: dispatch 2023-10-01T10:15:48.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:47 smithi173 bash[27612]: cluster 2023-10-01T10:15:46.973842+0000 mon.a (mon.0) 692 : cluster [DBG] fsmap foo:1 {0=foo.smithi078.vjqrth=up:active} 1 up:standby 2023-10-01T10:15:48.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:47 smithi173 bash[27612]: cluster 2023-10-01T10:15:47.194250+0000 mgr.y (mgr.14427) 108 : cluster [DBG] pgmap v87: 73 pgs: 73 active+clean; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:48.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:47 smithi173 bash[27612]: audit 2023-10-01T10:15:47.465082+0000 mon.a (mon.0) 693 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:49.683 INFO:teuthology.run_tasks:Running task ceph.healthy... 2023-10-01T10:15:49.692 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-10-01T10:15:49.692 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-10-01T10:15:49.693 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph mgr dump --format=json 2023-10-01T10:15:50.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:50 smithi078 bash[33821]: cluster 2023-10-01T10:15:49.195011+0000 mgr.y (mgr.14427) 109 : cluster [DBG] pgmap v88: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:50.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:50 smithi078 bash[27336]: cluster 2023-10-01T10:15:49.195011+0000 mgr.y (mgr.14427) 109 : cluster [DBG] pgmap v88: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:50.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:50 smithi173 bash[27612]: cluster 2023-10-01T10:15:49.195011+0000 mgr.y (mgr.14427) 109 : cluster [DBG] pgmap v88: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:52.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:52 smithi078 bash[27336]: audit 2023-10-01T10:15:51.148699+0000 mon.a (mon.0) 694 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:52.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:52 smithi078 bash[27336]: audit 2023-10-01T10:15:51.174871+0000 mon.a (mon.0) 695 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:52.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:52 smithi078 bash[27336]: cluster 2023-10-01T10:15:51.195570+0000 mgr.y (mgr.14427) 110 : cluster [DBG] pgmap v89: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:52.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:52 smithi078 bash[27336]: audit 2023-10-01T10:15:51.461714+0000 mon.a (mon.0) 696 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:15:52.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:52 smithi078 bash[27336]: audit 2023-10-01T10:15:51.463445+0000 mon.a (mon.0) 697 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:15:52.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:52 smithi078 bash[27336]: audit 2023-10-01T10:15:51.474430+0000 mon.a (mon.0) 698 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:52.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:52 smithi078 bash[27336]: audit 2023-10-01T10:15:51.483193+0000 mon.a (mon.0) 699 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:15:52.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:52 smithi078 bash[27336]: audit 2023-10-01T10:15:51.485519+0000 mon.a (mon.0) 700 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:15:52.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:52 smithi078 bash[27336]: audit 2023-10-01T10:15:51.486426+0000 mon.a (mon.0) 701 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:15:52.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:52 smithi078 bash[27336]: audit 2023-10-01T10:15:51.494831+0000 mon.a (mon.0) 702 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:52.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:52 smithi078 bash[33821]: audit 2023-10-01T10:15:51.148699+0000 mon.a (mon.0) 694 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:52.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:52 smithi078 bash[33821]: audit 2023-10-01T10:15:51.174871+0000 mon.a (mon.0) 695 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:52.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:52 smithi078 bash[33821]: cluster 2023-10-01T10:15:51.195570+0000 mgr.y (mgr.14427) 110 : cluster [DBG] pgmap v89: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:52.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:52 smithi078 bash[33821]: audit 2023-10-01T10:15:51.461714+0000 mon.a (mon.0) 696 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:15:52.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:52 smithi078 bash[33821]: audit 2023-10-01T10:15:51.463445+0000 mon.a (mon.0) 697 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:15:52.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:52 smithi078 bash[33821]: audit 2023-10-01T10:15:51.474430+0000 mon.a (mon.0) 698 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:52.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:52 smithi078 bash[33821]: audit 2023-10-01T10:15:51.483193+0000 mon.a (mon.0) 699 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:15:52.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:52 smithi078 bash[33821]: audit 2023-10-01T10:15:51.485519+0000 mon.a (mon.0) 700 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:15:52.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:52 smithi078 bash[33821]: audit 2023-10-01T10:15:51.486426+0000 mon.a (mon.0) 701 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:15:52.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:52 smithi078 bash[33821]: audit 2023-10-01T10:15:51.494831+0000 mon.a (mon.0) 702 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:52.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:52 smithi173 bash[27612]: audit 2023-10-01T10:15:51.148699+0000 mon.a (mon.0) 694 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:52.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:52 smithi173 bash[27612]: audit 2023-10-01T10:15:51.174871+0000 mon.a (mon.0) 695 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:52.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:52 smithi173 bash[27612]: cluster 2023-10-01T10:15:51.195570+0000 mgr.y (mgr.14427) 110 : cluster [DBG] pgmap v89: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:15:52.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:52 smithi173 bash[27612]: audit 2023-10-01T10:15:51.461714+0000 mon.a (mon.0) 696 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:15:52.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:52 smithi173 bash[27612]: audit 2023-10-01T10:15:51.463445+0000 mon.a (mon.0) 697 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:15:52.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:52 smithi173 bash[27612]: audit 2023-10-01T10:15:51.474430+0000 mon.a (mon.0) 698 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:52.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:52 smithi173 bash[27612]: audit 2023-10-01T10:15:51.483193+0000 mon.a (mon.0) 699 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:15:52.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:52 smithi173 bash[27612]: audit 2023-10-01T10:15:51.485519+0000 mon.a (mon.0) 700 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:15:52.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:52 smithi173 bash[27612]: audit 2023-10-01T10:15:51.486426+0000 mon.a (mon.0) 701 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:15:52.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:52 smithi173 bash[27612]: audit 2023-10-01T10:15:51.494831+0000 mon.a (mon.0) 702 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:54.342 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:15:54.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:54 smithi078 bash[27336]: cluster 2023-10-01T10:15:53.196433+0000 mgr.y (mgr.14427) 111 : cluster [DBG] pgmap v90: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-01T10:15:54.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:54 smithi078 bash[33821]: cluster 2023-10-01T10:15:53.196433+0000 mgr.y (mgr.14427) 111 : cluster [DBG] pgmap v90: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-01T10:15:54.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:54 smithi173 bash[27612]: cluster 2023-10-01T10:15:53.196433+0000 mgr.y (mgr.14427) 111 : cluster [DBG] pgmap v90: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-01T10:15:55.406 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:15:55 smithi078 bash[27601]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:15:55] "GET /metrics HTTP/1.1" 200 191015 "" "Prometheus/2.33.4" 2023-10-01T10:15:55.620 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:15:55 smithi173 bash[28473]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:15:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T10:15:56.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:56 smithi078 bash[33821]: cluster 2023-10-01T10:15:55.197084+0000 mgr.y (mgr.14427) 112 : cluster [DBG] pgmap v91: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-01T10:15:56.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:56 smithi078 bash[27336]: cluster 2023-10-01T10:15:55.197084+0000 mgr.y (mgr.14427) 112 : cluster [DBG] pgmap v91: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-01T10:15:56.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:56 smithi173 bash[27612]: cluster 2023-10-01T10:15:55.197084+0000 mgr.y (mgr.14427) 112 : cluster [DBG] pgmap v91: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-01T10:15:57.273 INFO:teuthology.orchestra.run.smithi078.stdout: 2023-10-01T10:15:58.027 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":18,"active_gid":14427,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6800","nonce":516444166},{"type":"v1","addr":"172.21.15.78:6801","nonce":516444166}]},"active_addr":"172.21.15.78:6801/516444166","active_change":"2023-10-01T10:13:17.135858+0000","active_mgr_features":4540138314316775423,"available":true,"standbys":[{"gid":24349,"name":"x","mgr_features":4540138314316775423,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"str","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"str","level":"advanced","flags":0,"default_value":"30","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"str","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"str","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"str","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"None","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.78:8443/","prometheus":"http://172.21.15.78:9283/"},"always_on_modules":{"nautilus":["balancer","crash","devicehealth","orchestrator_cli","progress","rbd_support","status","volumes"],"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":51,"active_clients":[{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.78:0","nonce":3274439700}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.78:0","nonce":702157702}]}]} 2023-10-01T10:15:58.032 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-10-01T10:15:58.032 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-10-01T10:15:58.032 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-01T10:15:58.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:58 smithi078 bash[27336]: cluster 2023-10-01T10:15:57.197682+0000 mgr.y (mgr.14427) 113 : cluster [DBG] pgmap v92: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-01T10:15:58.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:58 smithi078 bash[27336]: audit 2023-10-01T10:15:57.259633+0000 mon.c (mon.2) 30 : audit [DBG] from='client.? 172.21.15.78:0/933215667' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-01T10:15:58.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:15:58 smithi078 bash[27336]: audit 2023-10-01T10:15:57.487542+0000 mon.a (mon.0) 703 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:58.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:58 smithi078 bash[33821]: cluster 2023-10-01T10:15:57.197682+0000 mgr.y (mgr.14427) 113 : cluster [DBG] pgmap v92: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-01T10:15:58.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:58 smithi078 bash[33821]: audit 2023-10-01T10:15:57.259633+0000 mon.c (mon.2) 30 : audit [DBG] from='client.? 172.21.15.78:0/933215667' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-01T10:15:58.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:15:58 smithi078 bash[33821]: audit 2023-10-01T10:15:57.487542+0000 mon.a (mon.0) 703 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:15:58.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:58 smithi173 bash[27612]: cluster 2023-10-01T10:15:57.197682+0000 mgr.y (mgr.14427) 113 : cluster [DBG] pgmap v92: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-01T10:15:58.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:58 smithi173 bash[27612]: audit 2023-10-01T10:15:57.259633+0000 mon.c (mon.2) 30 : audit [DBG] from='client.? 172.21.15.78:0/933215667' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-01T10:15:58.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:15:58 smithi173 bash[27612]: audit 2023-10-01T10:15:57.487542+0000 mon.a (mon.0) 703 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:16:00.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:16:00 smithi078 bash[27336]: cluster 2023-10-01T10:15:59.198834+0000 mgr.y (mgr.14427) 114 : cluster [DBG] pgmap v93: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-01T10:16:00.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:16:00 smithi078 bash[33821]: cluster 2023-10-01T10:15:59.198834+0000 mgr.y (mgr.14427) 114 : cluster [DBG] pgmap v93: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-01T10:16:00.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:16:00 smithi173 bash[27612]: cluster 2023-10-01T10:15:59.198834+0000 mgr.y (mgr.14427) 114 : cluster [DBG] pgmap v93: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-01T10:16:02.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:16:01 smithi078 bash[33821]: cluster 2023-10-01T10:16:01.199593+0000 mgr.y (mgr.14427) 115 : cluster [DBG] pgmap v94: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-01T10:16:02.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:16:01 smithi078 bash[27336]: cluster 2023-10-01T10:16:01.199593+0000 mgr.y (mgr.14427) 115 : cluster [DBG] pgmap v94: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-01T10:16:02.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:16:01 smithi173 bash[27612]: cluster 2023-10-01T10:16:01.199593+0000 mgr.y (mgr.14427) 115 : cluster [DBG] pgmap v94: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-01T10:16:02.807 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:16:04.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:16:04 smithi078 bash[27336]: cluster 2023-10-01T10:16:03.200685+0000 mgr.y (mgr.14427) 116 : cluster [DBG] pgmap v95: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-01T10:16:04.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:16:04 smithi078 bash[33821]: cluster 2023-10-01T10:16:03.200685+0000 mgr.y (mgr.14427) 116 : cluster [DBG] pgmap v95: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-01T10:16:04.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:16:04 smithi173 bash[27612]: cluster 2023-10-01T10:16:03.200685+0000 mgr.y (mgr.14427) 116 : cluster [DBG] pgmap v95: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-10-01T10:16:05.519 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:16:05 smithi078 bash[27601]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:16:05] "GET /metrics HTTP/1.1" 200 191378 "" "Prometheus/2.33.4" 2023-10-01T10:16:05.620 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:16:05 smithi173 bash[28473]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:16:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T10:16:05.913 INFO:teuthology.orchestra.run.smithi078.stdout: 2023-10-01T10:16:05.914 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":60,"fsid":"d8016480-6041-11ee-8db5-212e2dc638e7","created":"2023-10-01T10:04:59.345633+0000","modified":"2023-10-01T10:15:41.294268+0000","last_up_change":"2023-10-01T10:12:45.118874+0000","last_in_change":"2023-10-01T10:12:19.634793+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-10-01T10:09:16.689172+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-01T10:13:43.595790+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-01T10:15:38.225217+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-01T10:15:38.676998+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"3cdf50f0-cae3-4e38-a7ca-c24c4a07b615","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6802","nonce":575557942},{"type":"v1","addr":"172.21.15.78:6803","nonce":575557942}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6804","nonce":575557942},{"type":"v1","addr":"172.21.15.78:6805","nonce":575557942}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6808","nonce":575557942},{"type":"v1","addr":"172.21.15.78:6809","nonce":575557942}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6806","nonce":575557942},{"type":"v1","addr":"172.21.15.78:6807","nonce":575557942}]},"public_addr":"172.21.15.78:6803/575557942","cluster_addr":"172.21.15.78:6805/575557942","heartbeat_back_addr":"172.21.15.78:6809/575557942","heartbeat_front_addr":"172.21.15.78:6807/575557942","state":["exists","up"]},{"osd":1,"uuid":"4e6bea65-4109-43d2-b5be-3811fdd2a803","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6810","nonce":3462774695},{"type":"v1","addr":"172.21.15.78:6811","nonce":3462774695}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6812","nonce":3462774695},{"type":"v1","addr":"172.21.15.78:6813","nonce":3462774695}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6816","nonce":3462774695},{"type":"v1","addr":"172.21.15.78:6817","nonce":3462774695}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6814","nonce":3462774695},{"type":"v1","addr":"172.21.15.78:6815","nonce":3462774695}]},"public_addr":"172.21.15.78:6811/3462774695","cluster_addr":"172.21.15.78:6813/3462774695","heartbeat_back_addr":"172.21.15.78:6817/3462774695","heartbeat_front_addr":"172.21.15.78:6815/3462774695","state":["exists","up"]},{"osd":2,"uuid":"9f830594-31d6-4994-85fa-0615c280d5c3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6818","nonce":1190788353},{"type":"v1","addr":"172.21.15.78:6819","nonce":1190788353}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6820","nonce":1190788353},{"type":"v1","addr":"172.21.15.78:6821","nonce":1190788353}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6824","nonce":1190788353},{"type":"v1","addr":"172.21.15.78:6825","nonce":1190788353}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6822","nonce":1190788353},{"type":"v1","addr":"172.21.15.78:6823","nonce":1190788353}]},"public_addr":"172.21.15.78:6819/1190788353","cluster_addr":"172.21.15.78:6821/1190788353","heartbeat_back_addr":"172.21.15.78:6825/1190788353","heartbeat_front_addr":"172.21.15.78:6823/1190788353","state":["exists","up"]},{"osd":3,"uuid":"2780842b-2f4f-4190-b4a5-279a0444adf2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6826","nonce":53937366},{"type":"v1","addr":"172.21.15.78:6827","nonce":53937366}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6828","nonce":53937366},{"type":"v1","addr":"172.21.15.78:6829","nonce":53937366}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6832","nonce":53937366},{"type":"v1","addr":"172.21.15.78:6833","nonce":53937366}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6830","nonce":53937366},{"type":"v1","addr":"172.21.15.78:6831","nonce":53937366}]},"public_addr":"172.21.15.78:6827/53937366","cluster_addr":"172.21.15.78:6829/53937366","heartbeat_back_addr":"172.21.15.78:6833/53937366","heartbeat_front_addr":"172.21.15.78:6831/53937366","state":["exists","up"]},{"osd":4,"uuid":"57c30778-fd7e-4f3e-afd9-389ef377cb62","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6800","nonce":2430591286},{"type":"v1","addr":"172.21.15.173:6801","nonce":2430591286}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6802","nonce":2430591286},{"type":"v1","addr":"172.21.15.173:6803","nonce":2430591286}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6806","nonce":2430591286},{"type":"v1","addr":"172.21.15.173:6807","nonce":2430591286}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6804","nonce":2430591286},{"type":"v1","addr":"172.21.15.173:6805","nonce":2430591286}]},"public_addr":"172.21.15.173:6801/2430591286","cluster_addr":"172.21.15.173:6803/2430591286","heartbeat_back_addr":"172.21.15.173:6807/2430591286","heartbeat_front_addr":"172.21.15.173:6805/2430591286","state":["exists","up"]},{"osd":5,"uuid":"c62aa7c3-2d09-4088-ba36-c84588dc3549","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6808","nonce":2798643181},{"type":"v1","addr":"172.21.15.173:6809","nonce":2798643181}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6810","nonce":2798643181},{"type":"v1","addr":"172.21.15.173:6811","nonce":2798643181}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6814","nonce":2798643181},{"type":"v1","addr":"172.21.15.173:6815","nonce":2798643181}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6812","nonce":2798643181},{"type":"v1","addr":"172.21.15.173:6813","nonce":2798643181}]},"public_addr":"172.21.15.173:6809/2798643181","cluster_addr":"172.21.15.173:6811/2798643181","heartbeat_back_addr":"172.21.15.173:6815/2798643181","heartbeat_front_addr":"172.21.15.173:6813/2798643181","state":["exists","up"]},{"osd":6,"uuid":"b140330d-dd93-4ce4-9b9a-2ec6321419b3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6816","nonce":1568190351},{"type":"v1","addr":"172.21.15.173:6817","nonce":1568190351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6818","nonce":1568190351},{"type":"v1","addr":"172.21.15.173:6819","nonce":1568190351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6822","nonce":1568190351},{"type":"v1","addr":"172.21.15.173:6823","nonce":1568190351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6820","nonce":1568190351},{"type":"v1","addr":"172.21.15.173:6821","nonce":1568190351}]},"public_addr":"172.21.15.173:6817/1568190351","cluster_addr":"172.21.15.173:6819/1568190351","heartbeat_back_addr":"172.21.15.173:6823/1568190351","heartbeat_front_addr":"172.21.15.173:6821/1568190351","state":["exists","up"]},{"osd":7,"uuid":"7590027f-50c4-4457-843d-cebaa734d2ea","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6824","nonce":1958390745},{"type":"v1","addr":"172.21.15.173:6825","nonce":1958390745}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6826","nonce":1958390745},{"type":"v1","addr":"172.21.15.173:6827","nonce":1958390745}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6830","nonce":1958390745},{"type":"v1","addr":"172.21.15.173:6831","nonce":1958390745}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6828","nonce":1958390745},{"type":"v1","addr":"172.21.15.173:6829","nonce":1958390745}]},"public_addr":"172.21.15.173:6825/1958390745","cluster_addr":"172.21.15.173:6827/1958390745","heartbeat_back_addr":"172.21.15.173:6831/1958390745","heartbeat_front_addr":"172.21.15.173:6829/1958390745","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:07:47.950416+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:08:31.368397+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:09:14.866866+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:09:59.445542+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:10:39.550033+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:11:20.934732+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:12:01.654044+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:12:43.447412+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.78:0/2359078223":"2023-10-02T10:13:17.135564+0000","172.21.15.78:0/3939471097":"2023-10-02T10:05:57.727322+0000","172.21.15.78:6800/1224621968":"2023-10-02T10:13:17.135564+0000","172.21.15.78:6800/1700562157":"2023-10-02T10:05:57.727322+0000","172.21.15.78:0/3635389419":"2023-10-02T10:05:57.727322+0000","172.21.15.78:0/3655343641":"2023-10-02T10:13:17.135564+0000","172.21.15.78:6800/3027286102":"2023-10-02T10:05:27.007550+0000","172.21.15.78:6801/3027286102":"2023-10-02T10:05:27.007550+0000","172.21.15.78:6801/1224621968":"2023-10-02T10:13:17.135564+0000","172.21.15.78:0/308254153":"2023-10-02T10:05:27.007550+0000","172.21.15.78:6801/1700562157":"2023-10-02T10:05:57.727322+0000","172.21.15.78:0/158328292":"2023-10-02T10:05:27.007550+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-01T10:16:06.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:16:06 smithi078 bash[33821]: cluster 2023-10-01T10:16:05.201366+0000 mgr.y (mgr.14427) 117 : cluster [DBG] pgmap v96: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:06.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:16:06 smithi078 bash[33821]: audit 2023-10-01T10:16:05.908539+0000 mon.c (mon.2) 31 : audit [DBG] from='client.? 172.21.15.78:0/2802735664' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T10:16:06.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:16:06 smithi078 bash[27336]: cluster 2023-10-01T10:16:05.201366+0000 mgr.y (mgr.14427) 117 : cluster [DBG] pgmap v96: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:06.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:16:06 smithi078 bash[27336]: audit 2023-10-01T10:16:05.908539+0000 mon.c (mon.2) 31 : audit [DBG] from='client.? 172.21.15.78:0/2802735664' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T10:16:06.597 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-10-01T10:16:06.597 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-01T10:16:06.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:16:06 smithi173 bash[27612]: cluster 2023-10-01T10:16:05.201366+0000 mgr.y (mgr.14427) 117 : cluster [DBG] pgmap v96: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:06.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:16:06 smithi173 bash[27612]: audit 2023-10-01T10:16:05.908539+0000 mon.c (mon.2) 31 : audit [DBG] from='client.? 172.21.15.78:0/2802735664' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T10:16:08.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:16:08 smithi078 bash[27336]: cluster 2023-10-01T10:16:07.202086+0000 mgr.y (mgr.14427) 118 : cluster [DBG] pgmap v97: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:08.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:16:08 smithi078 bash[33821]: cluster 2023-10-01T10:16:07.202086+0000 mgr.y (mgr.14427) 118 : cluster [DBG] pgmap v97: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:08.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:16:08 smithi173 bash[27612]: cluster 2023-10-01T10:16:07.202086+0000 mgr.y (mgr.14427) 118 : cluster [DBG] pgmap v97: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:10.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:16:10 smithi078 bash[33821]: cluster 2023-10-01T10:16:09.203220+0000 mgr.y (mgr.14427) 119 : cluster [DBG] pgmap v98: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:10.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:16:10 smithi078 bash[27336]: cluster 2023-10-01T10:16:09.203220+0000 mgr.y (mgr.14427) 119 : cluster [DBG] pgmap v98: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:10.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:16:10 smithi173 bash[27612]: cluster 2023-10-01T10:16:09.203220+0000 mgr.y (mgr.14427) 119 : cluster [DBG] pgmap v98: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:11.372 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:16:12.274 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:16:11 smithi078 bash[27336]: cluster 2023-10-01T10:16:11.203861+0000 mgr.y (mgr.14427) 120 : cluster [DBG] pgmap v99: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:12.275 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:16:11 smithi078 bash[33821]: cluster 2023-10-01T10:16:11.203861+0000 mgr.y (mgr.14427) 120 : cluster [DBG] pgmap v99: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:12.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:16:11 smithi173 bash[27612]: cluster 2023-10-01T10:16:11.203861+0000 mgr.y (mgr.14427) 120 : cluster [DBG] pgmap v99: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:14.430 INFO:teuthology.orchestra.run.smithi078.stdout: 2023-10-01T10:16:14.430 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":60,"fsid":"d8016480-6041-11ee-8db5-212e2dc638e7","created":"2023-10-01T10:04:59.345633+0000","modified":"2023-10-01T10:15:41.294268+0000","last_up_change":"2023-10-01T10:12:45.118874+0000","last_in_change":"2023-10-01T10:12:19.634793+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"pacific","pools":[{"pool":1,"pool_name":"device_health_metrics","create_time":"2023-10-01T10:09:16.689172+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"53","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr_devicehealth":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-01T10:13:43.595790+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-01T10:15:38.225217+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-01T10:15:38.676998+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"3cdf50f0-cae3-4e38-a7ca-c24c4a07b615","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6802","nonce":575557942},{"type":"v1","addr":"172.21.15.78:6803","nonce":575557942}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6804","nonce":575557942},{"type":"v1","addr":"172.21.15.78:6805","nonce":575557942}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6808","nonce":575557942},{"type":"v1","addr":"172.21.15.78:6809","nonce":575557942}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6806","nonce":575557942},{"type":"v1","addr":"172.21.15.78:6807","nonce":575557942}]},"public_addr":"172.21.15.78:6803/575557942","cluster_addr":"172.21.15.78:6805/575557942","heartbeat_back_addr":"172.21.15.78:6809/575557942","heartbeat_front_addr":"172.21.15.78:6807/575557942","state":["exists","up"]},{"osd":1,"uuid":"4e6bea65-4109-43d2-b5be-3811fdd2a803","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6810","nonce":3462774695},{"type":"v1","addr":"172.21.15.78:6811","nonce":3462774695}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6812","nonce":3462774695},{"type":"v1","addr":"172.21.15.78:6813","nonce":3462774695}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6816","nonce":3462774695},{"type":"v1","addr":"172.21.15.78:6817","nonce":3462774695}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6814","nonce":3462774695},{"type":"v1","addr":"172.21.15.78:6815","nonce":3462774695}]},"public_addr":"172.21.15.78:6811/3462774695","cluster_addr":"172.21.15.78:6813/3462774695","heartbeat_back_addr":"172.21.15.78:6817/3462774695","heartbeat_front_addr":"172.21.15.78:6815/3462774695","state":["exists","up"]},{"osd":2,"uuid":"9f830594-31d6-4994-85fa-0615c280d5c3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6818","nonce":1190788353},{"type":"v1","addr":"172.21.15.78:6819","nonce":1190788353}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6820","nonce":1190788353},{"type":"v1","addr":"172.21.15.78:6821","nonce":1190788353}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6824","nonce":1190788353},{"type":"v1","addr":"172.21.15.78:6825","nonce":1190788353}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6822","nonce":1190788353},{"type":"v1","addr":"172.21.15.78:6823","nonce":1190788353}]},"public_addr":"172.21.15.78:6819/1190788353","cluster_addr":"172.21.15.78:6821/1190788353","heartbeat_back_addr":"172.21.15.78:6825/1190788353","heartbeat_front_addr":"172.21.15.78:6823/1190788353","state":["exists","up"]},{"osd":3,"uuid":"2780842b-2f4f-4190-b4a5-279a0444adf2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6826","nonce":53937366},{"type":"v1","addr":"172.21.15.78:6827","nonce":53937366}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6828","nonce":53937366},{"type":"v1","addr":"172.21.15.78:6829","nonce":53937366}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6832","nonce":53937366},{"type":"v1","addr":"172.21.15.78:6833","nonce":53937366}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6830","nonce":53937366},{"type":"v1","addr":"172.21.15.78:6831","nonce":53937366}]},"public_addr":"172.21.15.78:6827/53937366","cluster_addr":"172.21.15.78:6829/53937366","heartbeat_back_addr":"172.21.15.78:6833/53937366","heartbeat_front_addr":"172.21.15.78:6831/53937366","state":["exists","up"]},{"osd":4,"uuid":"57c30778-fd7e-4f3e-afd9-389ef377cb62","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6800","nonce":2430591286},{"type":"v1","addr":"172.21.15.173:6801","nonce":2430591286}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6802","nonce":2430591286},{"type":"v1","addr":"172.21.15.173:6803","nonce":2430591286}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6806","nonce":2430591286},{"type":"v1","addr":"172.21.15.173:6807","nonce":2430591286}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6804","nonce":2430591286},{"type":"v1","addr":"172.21.15.173:6805","nonce":2430591286}]},"public_addr":"172.21.15.173:6801/2430591286","cluster_addr":"172.21.15.173:6803/2430591286","heartbeat_back_addr":"172.21.15.173:6807/2430591286","heartbeat_front_addr":"172.21.15.173:6805/2430591286","state":["exists","up"]},{"osd":5,"uuid":"c62aa7c3-2d09-4088-ba36-c84588dc3549","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6808","nonce":2798643181},{"type":"v1","addr":"172.21.15.173:6809","nonce":2798643181}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6810","nonce":2798643181},{"type":"v1","addr":"172.21.15.173:6811","nonce":2798643181}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6814","nonce":2798643181},{"type":"v1","addr":"172.21.15.173:6815","nonce":2798643181}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6812","nonce":2798643181},{"type":"v1","addr":"172.21.15.173:6813","nonce":2798643181}]},"public_addr":"172.21.15.173:6809/2798643181","cluster_addr":"172.21.15.173:6811/2798643181","heartbeat_back_addr":"172.21.15.173:6815/2798643181","heartbeat_front_addr":"172.21.15.173:6813/2798643181","state":["exists","up"]},{"osd":6,"uuid":"b140330d-dd93-4ce4-9b9a-2ec6321419b3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6816","nonce":1568190351},{"type":"v1","addr":"172.21.15.173:6817","nonce":1568190351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6818","nonce":1568190351},{"type":"v1","addr":"172.21.15.173:6819","nonce":1568190351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6822","nonce":1568190351},{"type":"v1","addr":"172.21.15.173:6823","nonce":1568190351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6820","nonce":1568190351},{"type":"v1","addr":"172.21.15.173:6821","nonce":1568190351}]},"public_addr":"172.21.15.173:6817/1568190351","cluster_addr":"172.21.15.173:6819/1568190351","heartbeat_back_addr":"172.21.15.173:6823/1568190351","heartbeat_front_addr":"172.21.15.173:6821/1568190351","state":["exists","up"]},{"osd":7,"uuid":"7590027f-50c4-4457-843d-cebaa734d2ea","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6824","nonce":1958390745},{"type":"v1","addr":"172.21.15.173:6825","nonce":1958390745}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6826","nonce":1958390745},{"type":"v1","addr":"172.21.15.173:6827","nonce":1958390745}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6830","nonce":1958390745},{"type":"v1","addr":"172.21.15.173:6831","nonce":1958390745}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6828","nonce":1958390745},{"type":"v1","addr":"172.21.15.173:6829","nonce":1958390745}]},"public_addr":"172.21.15.173:6825/1958390745","cluster_addr":"172.21.15.173:6827/1958390745","heartbeat_back_addr":"172.21.15.173:6831/1958390745","heartbeat_front_addr":"172.21.15.173:6829/1958390745","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:07:47.950416+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:08:31.368397+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:09:14.866866+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:09:59.445542+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:10:39.550033+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:11:20.934732+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:12:01.654044+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:12:43.447412+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.78:0/2359078223":"2023-10-02T10:13:17.135564+0000","172.21.15.78:0/3939471097":"2023-10-02T10:05:57.727322+0000","172.21.15.78:6800/1224621968":"2023-10-02T10:13:17.135564+0000","172.21.15.78:6800/1700562157":"2023-10-02T10:05:57.727322+0000","172.21.15.78:0/3635389419":"2023-10-02T10:05:57.727322+0000","172.21.15.78:0/3655343641":"2023-10-02T10:13:17.135564+0000","172.21.15.78:6800/3027286102":"2023-10-02T10:05:27.007550+0000","172.21.15.78:6801/3027286102":"2023-10-02T10:05:27.007550+0000","172.21.15.78:6801/1224621968":"2023-10-02T10:13:17.135564+0000","172.21.15.78:0/308254153":"2023-10-02T10:05:27.007550+0000","172.21.15.78:6801/1700562157":"2023-10-02T10:05:57.727322+0000","172.21.15.78:0/158328292":"2023-10-02T10:05:27.007550+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-01T10:16:14.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:16:14 smithi078 bash[27336]: cluster 2023-10-01T10:16:13.204634+0000 mgr.y (mgr.14427) 121 : cluster [DBG] pgmap v100: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:14.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:16:14 smithi078 bash[33821]: cluster 2023-10-01T10:16:13.204634+0000 mgr.y (mgr.14427) 121 : cluster [DBG] pgmap v100: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:14.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:16:14 smithi173 bash[27612]: cluster 2023-10-01T10:16:13.204634+0000 mgr.y (mgr.14427) 121 : cluster [DBG] pgmap v100: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:15.084 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-10-01T10:16:15.085 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-10-01T10:16:15.085 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-10-01T10:16:15.086 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-10-01T10:16:15.086 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-10-01T10:16:15.086 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-10-01T10:16:15.087 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-10-01T10:16:15.087 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-10-01T10:16:15.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:16:15 smithi078 bash[27336]: audit 2023-10-01T10:16:14.419080+0000 mon.a (mon.0) 704 : audit [DBG] from='client.? 172.21.15.78:0/1555163917' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T10:16:15.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:16:15 smithi078 bash[33821]: audit 2023-10-01T10:16:14.419080+0000 mon.a (mon.0) 704 : audit [DBG] from='client.? 172.21.15.78:0/1555163917' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T10:16:15.333 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:16:15 smithi078 bash[27601]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:16:15] "GET /metrics HTTP/1.1" 200 191378 "" "Prometheus/2.33.4" 2023-10-01T10:16:15.620 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:16:15 smithi173 bash[28473]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:16:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T10:16:15.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:16:15 smithi173 bash[27612]: audit 2023-10-01T10:16:14.419080+0000 mon.a (mon.0) 704 : audit [DBG] from='client.? 172.21.15.78:0/1555163917' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T10:16:16.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:16:16 smithi078 bash[27336]: cluster 2023-10-01T10:16:15.205108+0000 mgr.y (mgr.14427) 122 : cluster [DBG] pgmap v101: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:16.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:16:16 smithi078 bash[33821]: cluster 2023-10-01T10:16:15.205108+0000 mgr.y (mgr.14427) 122 : cluster [DBG] pgmap v101: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:16.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:16:16 smithi173 bash[27612]: cluster 2023-10-01T10:16:15.205108+0000 mgr.y (mgr.14427) 122 : cluster [DBG] pgmap v101: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:17.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:16:17 smithi078 bash[27336]: audit 2023-10-01T10:16:17.193752+0000 mon.a (mon.0) 705 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2023-10-01T10:16:17.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:16:17 smithi078 bash[27336]: audit 2023-10-01T10:16:17.194283+0000 mon.a (mon.0) 706 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2023-10-01T10:16:17.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:16:17 smithi078 bash[27336]: audit 2023-10-01T10:16:17.194721+0000 mon.a (mon.0) 707 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2023-10-01T10:16:17.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:16:17 smithi078 bash[33821]: audit 2023-10-01T10:16:17.193752+0000 mon.a (mon.0) 705 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2023-10-01T10:16:17.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:16:17 smithi078 bash[33821]: audit 2023-10-01T10:16:17.194283+0000 mon.a (mon.0) 706 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2023-10-01T10:16:17.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:16:17 smithi078 bash[33821]: audit 2023-10-01T10:16:17.194721+0000 mon.a (mon.0) 707 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2023-10-01T10:16:17.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:16:17 smithi173 bash[27612]: audit 2023-10-01T10:16:17.193752+0000 mon.a (mon.0) 705 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2023-10-01T10:16:17.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:16:17 smithi173 bash[27612]: audit 2023-10-01T10:16:17.194283+0000 mon.a (mon.0) 706 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2023-10-01T10:16:17.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:16:17 smithi173 bash[27612]: audit 2023-10-01T10:16:17.194721+0000 mon.a (mon.0) 707 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2023-10-01T10:16:18.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:16:18 smithi078 bash[33821]: cluster 2023-10-01T10:16:17.205627+0000 mgr.y (mgr.14427) 123 : cluster [DBG] pgmap v102: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:18.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:16:18 smithi078 bash[33821]: audit 2023-10-01T10:16:17.289496+0000 mon.a (mon.0) 708 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2023-10-01T10:16:18.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:16:18 smithi078 bash[33821]: audit 2023-10-01T10:16:17.289627+0000 mon.a (mon.0) 709 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2023-10-01T10:16:18.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:16:18 smithi078 bash[33821]: audit 2023-10-01T10:16:17.289715+0000 mon.a (mon.0) 710 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2023-10-01T10:16:18.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:16:18 smithi078 bash[33821]: cluster 2023-10-01T10:16:17.289823+0000 mon.a (mon.0) 711 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2023-10-01T10:16:18.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:16:18 smithi078 bash[27336]: cluster 2023-10-01T10:16:17.205627+0000 mgr.y (mgr.14427) 123 : cluster [DBG] pgmap v102: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:18.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:16:18 smithi078 bash[27336]: audit 2023-10-01T10:16:17.289496+0000 mon.a (mon.0) 708 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2023-10-01T10:16:18.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:16:18 smithi078 bash[27336]: audit 2023-10-01T10:16:17.289627+0000 mon.a (mon.0) 709 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2023-10-01T10:16:18.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:16:18 smithi078 bash[27336]: audit 2023-10-01T10:16:17.289715+0000 mon.a (mon.0) 710 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2023-10-01T10:16:18.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:16:18 smithi078 bash[27336]: cluster 2023-10-01T10:16:17.289823+0000 mon.a (mon.0) 711 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2023-10-01T10:16:18.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:16:18 smithi173 bash[27612]: cluster 2023-10-01T10:16:17.205627+0000 mgr.y (mgr.14427) 123 : cluster [DBG] pgmap v102: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:18.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:16:18 smithi173 bash[27612]: audit 2023-10-01T10:16:17.289496+0000 mon.a (mon.0) 708 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2023-10-01T10:16:18.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:16:18 smithi173 bash[27612]: audit 2023-10-01T10:16:17.289627+0000 mon.a (mon.0) 709 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2023-10-01T10:16:18.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:16:18 smithi173 bash[27612]: audit 2023-10-01T10:16:17.289715+0000 mon.a (mon.0) 710 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2023-10-01T10:16:18.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:16:18 smithi173 bash[27612]: cluster 2023-10-01T10:16:17.289823+0000 mon.a (mon.0) 711 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2023-10-01T10:16:19.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:16:19 smithi078 bash[27336]: cluster 2023-10-01T10:16:18.293978+0000 mon.a (mon.0) 712 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2023-10-01T10:16:19.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:16:19 smithi078 bash[33821]: cluster 2023-10-01T10:16:18.293978+0000 mon.a (mon.0) 712 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2023-10-01T10:16:19.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:16:19 smithi173 bash[27612]: cluster 2023-10-01T10:16:18.293978+0000 mon.a (mon.0) 712 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2023-10-01T10:16:19.985 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:16:19.985 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:16:19.995 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:16:19.997 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:16:19.998 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:16:19.998 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:16:19.999 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:16:20.000 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:16:20.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:16:20 smithi078 bash[27336]: cluster 2023-10-01T10:16:19.206277+0000 mgr.y (mgr.14427) 124 : cluster [DBG] pgmap v105: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:20.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:16:20 smithi078 bash[33821]: cluster 2023-10-01T10:16:19.206277+0000 mgr.y (mgr.14427) 124 : cluster [DBG] pgmap v105: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:20.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:16:20 smithi173 bash[27612]: cluster 2023-10-01T10:16:19.206277+0000 mgr.y (mgr.14427) 124 : cluster [DBG] pgmap v105: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:22.337 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:16:21 smithi078 bash[27336]: cluster 2023-10-01T10:16:21.206692+0000 mgr.y (mgr.14427) 125 : cluster [DBG] pgmap v106: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:22.338 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:16:21 smithi078 bash[33821]: cluster 2023-10-01T10:16:21.206692+0000 mgr.y (mgr.14427) 125 : cluster [DBG] pgmap v106: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:22.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:16:21 smithi173 bash[27612]: cluster 2023-10-01T10:16:21.206692+0000 mgr.y (mgr.14427) 125 : cluster [DBG] pgmap v106: 73 pgs: 3 peering, 70 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:24.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:16:24 smithi078 bash[27336]: cluster 2023-10-01T10:16:23.207213+0000 mgr.y (mgr.14427) 126 : cluster [DBG] pgmap v107: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:24.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:16:24 smithi078 bash[33821]: cluster 2023-10-01T10:16:23.207213+0000 mgr.y (mgr.14427) 126 : cluster [DBG] pgmap v107: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:24.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:16:24 smithi173 bash[27612]: cluster 2023-10-01T10:16:23.207213+0000 mgr.y (mgr.14427) 126 : cluster [DBG] pgmap v107: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:25.333 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:16:25 smithi078 bash[27601]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:16:25] "GET /metrics HTTP/1.1" 200 191371 "" "Prometheus/2.33.4" 2023-10-01T10:16:25.620 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:16:25 smithi173 bash[28473]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:16:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T10:16:26.576 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:16:26 smithi078 bash[33821]: cluster 2023-10-01T10:16:25.207628+0000 mgr.y (mgr.14427) 127 : cluster [DBG] pgmap v108: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:26.577 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:16:26 smithi078 bash[27336]: cluster 2023-10-01T10:16:25.207628+0000 mgr.y (mgr.14427) 127 : cluster [DBG] pgmap v108: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:26.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:16:26 smithi173 bash[27612]: cluster 2023-10-01T10:16:25.207628+0000 mgr.y (mgr.14427) 127 : cluster [DBG] pgmap v108: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:28.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:16:28 smithi078 bash[27336]: cluster 2023-10-01T10:16:27.208166+0000 mgr.y (mgr.14427) 128 : cluster [DBG] pgmap v109: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:28.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:16:28 smithi078 bash[33821]: cluster 2023-10-01T10:16:27.208166+0000 mgr.y (mgr.14427) 128 : cluster [DBG] pgmap v109: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:28.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:16:28 smithi173 bash[27612]: cluster 2023-10-01T10:16:27.208166+0000 mgr.y (mgr.14427) 128 : cluster [DBG] pgmap v109: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:30.578 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:16:30 smithi078 bash[33821]: cluster 2023-10-01T10:16:29.209237+0000 mgr.y (mgr.14427) 129 : cluster [DBG] pgmap v110: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:30.578 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:16:30 smithi078 bash[27336]: cluster 2023-10-01T10:16:29.209237+0000 mgr.y (mgr.14427) 129 : cluster [DBG] pgmap v110: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:30.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:16:30 smithi173 bash[27612]: cluster 2023-10-01T10:16:29.209237+0000 mgr.y (mgr.14427) 129 : cluster [DBG] pgmap v110: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:32.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:16:31 smithi078 bash[27336]: cluster 2023-10-01T10:16:31.209975+0000 mgr.y (mgr.14427) 130 : cluster [DBG] pgmap v111: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:32.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:16:31 smithi078 bash[33821]: cluster 2023-10-01T10:16:31.209975+0000 mgr.y (mgr.14427) 130 : cluster [DBG] pgmap v111: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:32.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:16:31 smithi173 bash[27612]: cluster 2023-10-01T10:16:31.209975+0000 mgr.y (mgr.14427) 130 : cluster [DBG] pgmap v111: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:33.078 INFO:teuthology.orchestra.run.smithi078.stdout:77309411416 2023-10-01T10:16:33.078 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-10-01T10:16:34.547 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:16:34 smithi078 bash[27336]: cluster 2023-10-01T10:16:33.210582+0000 mgr.y (mgr.14427) 131 : cluster [DBG] pgmap v112: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:34.548 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:16:34 smithi078 bash[33821]: cluster 2023-10-01T10:16:33.210582+0000 mgr.y (mgr.14427) 131 : cluster [DBG] pgmap v112: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:34.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:16:34 smithi173 bash[27612]: cluster 2023-10-01T10:16:33.210582+0000 mgr.y (mgr.14427) 131 : cluster [DBG] pgmap v112: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:34.653 INFO:teuthology.orchestra.run.smithi078.stdout:34359738475 2023-10-01T10:16:34.654 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-10-01T10:16:34.670 INFO:teuthology.orchestra.run.smithi078.stdout:55834574946 2023-10-01T10:16:34.671 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-10-01T10:16:35.274 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:16:35 smithi078 bash[27601]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:16:35] "GET /metrics HTTP/1.1" 200 191383 "" "Prometheus/2.33.4" 2023-10-01T10:16:35.621 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:16:35 smithi173 bash[28473]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:16:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T10:16:36.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:16:36 smithi078 bash[27336]: cluster 2023-10-01T10:16:35.211046+0000 mgr.y (mgr.14427) 132 : cluster [DBG] pgmap v113: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:36.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:16:36 smithi078 bash[33821]: cluster 2023-10-01T10:16:35.211046+0000 mgr.y (mgr.14427) 132 : cluster [DBG] pgmap v113: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:36.520 INFO:teuthology.orchestra.run.smithi078.stdout:154618822719 2023-10-01T10:16:36.521 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-10-01T10:16:36.586 INFO:teuthology.orchestra.run.smithi078.stdout:206158430256 2023-10-01T10:16:36.587 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-10-01T10:16:36.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:16:36 smithi173 bash[27612]: cluster 2023-10-01T10:16:35.211046+0000 mgr.y (mgr.14427) 132 : cluster [DBG] pgmap v113: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:36.753 INFO:teuthology.orchestra.run.smithi078.stdout:180388626490 2023-10-01T10:16:36.754 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-10-01T10:16:37.831 INFO:teuthology.orchestra.run.smithi078.stdout:128849018954 2023-10-01T10:16:37.832 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-10-01T10:16:37.852 INFO:teuthology.orchestra.run.smithi078.stdout:107374182482 2023-10-01T10:16:37.853 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-10-01T10:16:38.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:16:38 smithi078 bash[27336]: cluster 2023-10-01T10:16:37.211775+0000 mgr.y (mgr.14427) 133 : cluster [DBG] pgmap v114: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:38.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:16:38 smithi078 bash[33821]: cluster 2023-10-01T10:16:37.211775+0000 mgr.y (mgr.14427) 133 : cluster [DBG] pgmap v114: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:38.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:16:38 smithi173 bash[27612]: cluster 2023-10-01T10:16:37.211775+0000 mgr.y (mgr.14427) 133 : cluster [DBG] pgmap v114: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:40.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:16:40 smithi078 bash[27336]: cluster 2023-10-01T10:16:39.212870+0000 mgr.y (mgr.14427) 134 : cluster [DBG] pgmap v115: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:40.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:16:40 smithi078 bash[33821]: cluster 2023-10-01T10:16:39.212870+0000 mgr.y (mgr.14427) 134 : cluster [DBG] pgmap v115: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:40.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:16:40 smithi173 bash[27612]: cluster 2023-10-01T10:16:39.212870+0000 mgr.y (mgr.14427) 134 : cluster [DBG] pgmap v115: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:40.890 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:16:40.890 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:16:40.891 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:16:40.891 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:16:40.892 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:16:40.892 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:16:41.901 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:16:41.902 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:16:42.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:16:41 smithi173 bash[27612]: cluster 2023-10-01T10:16:41.213526+0000 mgr.y (mgr.14427) 135 : cluster [DBG] pgmap v116: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:42.432 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:16:41 smithi078 bash[27336]: cluster 2023-10-01T10:16:41.213526+0000 mgr.y (mgr.14427) 135 : cluster [DBG] pgmap v116: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:42.433 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:16:41 smithi078 bash[33821]: cluster 2023-10-01T10:16:41.213526+0000 mgr.y (mgr.14427) 135 : cluster [DBG] pgmap v116: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:44.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:16:44 smithi078 bash[27336]: cluster 2023-10-01T10:16:43.214518+0000 mgr.y (mgr.14427) 136 : cluster [DBG] pgmap v117: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:44.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:16:44 smithi078 bash[33821]: cluster 2023-10-01T10:16:43.214518+0000 mgr.y (mgr.14427) 136 : cluster [DBG] pgmap v117: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:44.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:16:44 smithi173 bash[27612]: cluster 2023-10-01T10:16:43.214518+0000 mgr.y (mgr.14427) 136 : cluster [DBG] pgmap v117: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:45.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:16:45 smithi078 bash[27601]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:16:45] "GET /metrics HTTP/1.1" 200 191383 "" "Prometheus/2.33.4" 2023-10-01T10:16:45.620 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:16:45 smithi173 bash[28473]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:16:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T10:16:46.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:16:46 smithi078 bash[27336]: cluster 2023-10-01T10:16:45.215108+0000 mgr.y (mgr.14427) 137 : cluster [DBG] pgmap v118: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:46.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:16:46 smithi078 bash[33821]: cluster 2023-10-01T10:16:45.215108+0000 mgr.y (mgr.14427) 137 : cluster [DBG] pgmap v118: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:46.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:16:46 smithi173 bash[27612]: cluster 2023-10-01T10:16:45.215108+0000 mgr.y (mgr.14427) 137 : cluster [DBG] pgmap v118: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:48.593 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:16:48 smithi078 bash[33821]: cluster 2023-10-01T10:16:47.215725+0000 mgr.y (mgr.14427) 138 : cluster [DBG] pgmap v119: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:48.594 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:16:48 smithi078 bash[27336]: cluster 2023-10-01T10:16:47.215725+0000 mgr.y (mgr.14427) 138 : cluster [DBG] pgmap v119: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:48.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:16:48 smithi173 bash[27612]: cluster 2023-10-01T10:16:47.215725+0000 mgr.y (mgr.14427) 138 : cluster [DBG] pgmap v119: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:50.585 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:16:50 smithi078 bash[27336]: cluster 2023-10-01T10:16:49.216429+0000 mgr.y (mgr.14427) 139 : cluster [DBG] pgmap v120: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:50.586 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:16:50 smithi078 bash[33821]: cluster 2023-10-01T10:16:49.216429+0000 mgr.y (mgr.14427) 139 : cluster [DBG] pgmap v120: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:50.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:16:50 smithi173 bash[27612]: cluster 2023-10-01T10:16:49.216429+0000 mgr.y (mgr.14427) 139 : cluster [DBG] pgmap v120: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:51.130 INFO:teuthology.orchestra.run.smithi078.stdout:154618822723 2023-10-01T10:16:52.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:16:51 smithi078 bash[33821]: audit 2023-10-01T10:16:51.127398+0000 mon.a (mon.0) 713 : audit [DBG] from='client.? 172.21.15.78:0/3896854439' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-01T10:16:52.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:16:51 smithi078 bash[33821]: cluster 2023-10-01T10:16:51.217153+0000 mgr.y (mgr.14427) 140 : cluster [DBG] pgmap v121: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:52.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:16:51 smithi078 bash[33821]: audit 2023-10-01T10:16:51.502987+0000 mon.a (mon.0) 714 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:16:52.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:16:51 smithi078 bash[33821]: audit 2023-10-01T10:16:51.772358+0000 mon.a (mon.0) 715 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:16:52.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:16:51 smithi078 bash[27336]: audit 2023-10-01T10:16:51.127398+0000 mon.a (mon.0) 713 : audit [DBG] from='client.? 172.21.15.78:0/3896854439' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-01T10:16:52.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:16:51 smithi078 bash[27336]: cluster 2023-10-01T10:16:51.217153+0000 mgr.y (mgr.14427) 140 : cluster [DBG] pgmap v121: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:52.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:16:51 smithi078 bash[27336]: audit 2023-10-01T10:16:51.502987+0000 mon.a (mon.0) 714 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:16:52.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:16:51 smithi078 bash[27336]: audit 2023-10-01T10:16:51.772358+0000 mon.a (mon.0) 715 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:16:52.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:16:51 smithi173 bash[27612]: audit 2023-10-01T10:16:51.127398+0000 mon.a (mon.0) 713 : audit [DBG] from='client.? 172.21.15.78:0/3896854439' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-01T10:16:52.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:16:51 smithi173 bash[27612]: cluster 2023-10-01T10:16:51.217153+0000 mgr.y (mgr.14427) 140 : cluster [DBG] pgmap v121: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:52.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:16:51 smithi173 bash[27612]: audit 2023-10-01T10:16:51.502987+0000 mon.a (mon.0) 714 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:16:52.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:16:51 smithi173 bash[27612]: audit 2023-10-01T10:16:51.772358+0000 mon.a (mon.0) 715 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:16:52.394 INFO:teuthology.orchestra.run.smithi078.stdout:77309411420 2023-10-01T10:16:53.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:16:52 smithi078 bash[27336]: audit 2023-10-01T10:16:52.062886+0000 mon.a (mon.0) 716 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:16:53.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:16:52 smithi078 bash[27336]: audit 2023-10-01T10:16:52.064068+0000 mon.a (mon.0) 717 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:16:53.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:16:52 smithi078 bash[27336]: audit 2023-10-01T10:16:52.070764+0000 mon.a (mon.0) 718 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:16:53.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:16:52 smithi078 bash[27336]: audit 2023-10-01T10:16:52.391510+0000 mon.a (mon.0) 719 : audit [DBG] from='client.? 172.21.15.78:0/287399053' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-01T10:16:53.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:16:52 smithi078 bash[33821]: audit 2023-10-01T10:16:52.062886+0000 mon.a (mon.0) 716 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:16:53.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:16:52 smithi078 bash[33821]: audit 2023-10-01T10:16:52.064068+0000 mon.a (mon.0) 717 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:16:53.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:16:52 smithi078 bash[33821]: audit 2023-10-01T10:16:52.070764+0000 mon.a (mon.0) 718 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:16:53.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:16:52 smithi078 bash[33821]: audit 2023-10-01T10:16:52.391510+0000 mon.a (mon.0) 719 : audit [DBG] from='client.? 172.21.15.78:0/287399053' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-01T10:16:53.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:16:52 smithi173 bash[27612]: audit 2023-10-01T10:16:52.062886+0000 mon.a (mon.0) 716 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:16:53.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:16:52 smithi173 bash[27612]: audit 2023-10-01T10:16:52.064068+0000 mon.a (mon.0) 717 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:16:53.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:16:52 smithi173 bash[27612]: audit 2023-10-01T10:16:52.070764+0000 mon.a (mon.0) 718 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:16:53.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:16:52 smithi173 bash[27612]: audit 2023-10-01T10:16:52.391510+0000 mon.a (mon.0) 719 : audit [DBG] from='client.? 172.21.15.78:0/287399053' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-01T10:16:53.931 INFO:tasks.cephadm.ceph_manager.ceph:need seq 154618822719 got 154618822723 for osd.5 2023-10-01T10:16:53.931 DEBUG:teuthology.parallel:result is None 2023-10-01T10:16:54.303 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:16:53 smithi078 bash[33821]: cluster 2023-10-01T10:16:53.217869+0000 mgr.y (mgr.14427) 141 : cluster [DBG] pgmap v122: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:54.304 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:16:53 smithi078 bash[27336]: cluster 2023-10-01T10:16:53.217869+0000 mgr.y (mgr.14427) 141 : cluster [DBG] pgmap v122: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:54.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:16:53 smithi173 bash[27612]: cluster 2023-10-01T10:16:53.217869+0000 mgr.y (mgr.14427) 141 : cluster [DBG] pgmap v122: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:54.499 INFO:teuthology.orchestra.run.smithi078.stdout:55834574951 2023-10-01T10:16:54.614 INFO:teuthology.orchestra.run.smithi078.stdout:206158430260 2023-10-01T10:16:55.267 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:16:54 smithi078 bash[27336]: audit 2023-10-01T10:16:54.496943+0000 mon.a (mon.0) 720 : audit [DBG] from='client.? 172.21.15.78:0/568577857' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-01T10:16:55.268 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:16:54 smithi078 bash[27336]: audit 2023-10-01T10:16:54.612010+0000 mon.a (mon.0) 721 : audit [DBG] from='client.? 172.21.15.78:0/652342523' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-01T10:16:55.268 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:16:55 smithi078 bash[27601]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:16:55] "GET /metrics HTTP/1.1" 200 191379 "" "Prometheus/2.33.4" 2023-10-01T10:16:55.268 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:16:54 smithi078 bash[33821]: audit 2023-10-01T10:16:54.496943+0000 mon.a (mon.0) 720 : audit [DBG] from='client.? 172.21.15.78:0/568577857' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-01T10:16:55.268 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:16:54 smithi078 bash[33821]: audit 2023-10-01T10:16:54.612010+0000 mon.a (mon.0) 721 : audit [DBG] from='client.? 172.21.15.78:0/652342523' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-01T10:16:55.285 INFO:teuthology.orchestra.run.smithi078.stdout:34359738480 2023-10-01T10:16:55.294 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411416 got 77309411420 for osd.2 2023-10-01T10:16:55.294 DEBUG:teuthology.parallel:result is None 2023-10-01T10:16:55.367 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:16:55 smithi173 bash[27612]: audit 2023-10-01T10:16:54.496943+0000 mon.a (mon.0) 720 : audit [DBG] from='client.? 172.21.15.78:0/568577857' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-01T10:16:55.367 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:16:55 smithi173 bash[27612]: audit 2023-10-01T10:16:54.612010+0000 mon.a (mon.0) 721 : audit [DBG] from='client.? 172.21.15.78:0/652342523' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-01T10:16:55.620 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:16:55 smithi173 bash[28473]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:16:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T10:16:55.631 INFO:teuthology.orchestra.run.smithi078.stdout:180388626494 2023-10-01T10:16:56.135 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:16:56 smithi078 bash[33821]: cluster 2023-10-01T10:16:55.218221+0000 mgr.y (mgr.14427) 142 : cluster [DBG] pgmap v123: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:56.135 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:16:56 smithi078 bash[33821]: audit 2023-10-01T10:16:55.282652+0000 mon.c (mon.2) 32 : audit [DBG] from='client.? 172.21.15.78:0/965245308' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-01T10:16:56.135 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:16:56 smithi078 bash[33821]: audit 2023-10-01T10:16:55.628244+0000 mon.c (mon.2) 33 : audit [DBG] from='client.? 172.21.15.78:0/2423156607' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-01T10:16:56.136 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:16:56 smithi078 bash[27336]: cluster 2023-10-01T10:16:55.218221+0000 mgr.y (mgr.14427) 142 : cluster [DBG] pgmap v123: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:56.136 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:16:56 smithi078 bash[27336]: audit 2023-10-01T10:16:55.282652+0000 mon.c (mon.2) 32 : audit [DBG] from='client.? 172.21.15.78:0/965245308' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-01T10:16:56.136 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:16:56 smithi078 bash[27336]: audit 2023-10-01T10:16:55.628244+0000 mon.c (mon.2) 33 : audit [DBG] from='client.? 172.21.15.78:0/2423156607' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-01T10:16:56.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:16:56 smithi173 bash[27612]: cluster 2023-10-01T10:16:55.218221+0000 mgr.y (mgr.14427) 142 : cluster [DBG] pgmap v123: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:56.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:16:56 smithi173 bash[27612]: audit 2023-10-01T10:16:55.282652+0000 mon.c (mon.2) 32 : audit [DBG] from='client.? 172.21.15.78:0/965245308' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-01T10:16:56.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:16:56 smithi173 bash[27612]: audit 2023-10-01T10:16:55.628244+0000 mon.c (mon.2) 33 : audit [DBG] from='client.? 172.21.15.78:0/2423156607' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-01T10:16:56.770 INFO:teuthology.orchestra.run.smithi078.stdout:128849018958 2023-10-01T10:16:57.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:16:57 smithi078 bash[27336]: audit 2023-10-01T10:16:56.766553+0000 mon.a (mon.0) 722 : audit [DBG] from='client.? 172.21.15.78:0/2927310142' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-01T10:16:57.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:16:57 smithi078 bash[33821]: audit 2023-10-01T10:16:56.766553+0000 mon.a (mon.0) 722 : audit [DBG] from='client.? 172.21.15.78:0/2927310142' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-01T10:16:57.113 INFO:teuthology.orchestra.run.smithi078.stdout:107374182486 2023-10-01T10:16:57.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:16:57 smithi173 bash[27612]: audit 2023-10-01T10:16:56.766553+0000 mon.a (mon.0) 722 : audit [DBG] from='client.? 172.21.15.78:0/2927310142' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-01T10:16:58.340 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:16:58 smithi078 bash[27336]: audit 2023-10-01T10:16:57.110311+0000 mon.c (mon.2) 34 : audit [DBG] from='client.? 172.21.15.78:0/607406887' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-01T10:16:58.340 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:16:58 smithi078 bash[27336]: cluster 2023-10-01T10:16:57.218567+0000 mgr.y (mgr.14427) 143 : cluster [DBG] pgmap v124: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:58.340 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:16:58 smithi078 bash[33821]: audit 2023-10-01T10:16:57.110311+0000 mon.c (mon.2) 34 : audit [DBG] from='client.? 172.21.15.78:0/607406887' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-01T10:16:58.341 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:16:58 smithi078 bash[33821]: cluster 2023-10-01T10:16:57.218567+0000 mgr.y (mgr.14427) 143 : cluster [DBG] pgmap v124: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:58.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:16:58 smithi173 bash[27612]: audit 2023-10-01T10:16:57.110311+0000 mon.c (mon.2) 34 : audit [DBG] from='client.? 172.21.15.78:0/607406887' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-01T10:16:58.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:16:58 smithi173 bash[27612]: cluster 2023-10-01T10:16:57.218567+0000 mgr.y (mgr.14427) 143 : cluster [DBG] pgmap v124: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:16:58.507 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430256 got 206158430260 for osd.7 2023-10-01T10:16:58.507 DEBUG:teuthology.parallel:result is None 2023-10-01T10:16:58.508 INFO:tasks.cephadm.ceph_manager.ceph:need seq 180388626490 got 180388626494 for osd.6 2023-10-01T10:16:58.508 DEBUG:teuthology.parallel:result is None 2023-10-01T10:16:58.508 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574946 got 55834574951 for osd.1 2023-10-01T10:16:58.509 DEBUG:teuthology.parallel:result is None 2023-10-01T10:16:58.570 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738475 got 34359738480 for osd.0 2023-10-01T10:16:58.570 DEBUG:teuthology.parallel:result is None 2023-10-01T10:16:58.730 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018954 got 128849018958 for osd.4 2023-10-01T10:16:58.730 DEBUG:teuthology.parallel:result is None 2023-10-01T10:16:58.776 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182482 got 107374182486 for osd.3 2023-10-01T10:16:58.777 DEBUG:teuthology.parallel:result is None 2023-10-01T10:16:58.777 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-10-01T10:16:58.777 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T10:17:00.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:17:00 smithi078 bash[33821]: cluster 2023-10-01T10:16:59.219646+0000 mgr.y (mgr.14427) 144 : cluster [DBG] pgmap v125: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:00.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:17:00 smithi078 bash[27336]: cluster 2023-10-01T10:16:59.219646+0000 mgr.y (mgr.14427) 144 : cluster [DBG] pgmap v125: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:00.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:17:00 smithi173 bash[27612]: cluster 2023-10-01T10:16:59.219646+0000 mgr.y (mgr.14427) 144 : cluster [DBG] pgmap v125: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:02.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:17:02 smithi078 bash[27336]: cluster 2023-10-01T10:17:01.220413+0000 mgr.y (mgr.14427) 145 : cluster [DBG] pgmap v126: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:02.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:17:02 smithi078 bash[33821]: cluster 2023-10-01T10:17:01.220413+0000 mgr.y (mgr.14427) 145 : cluster [DBG] pgmap v126: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:02.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:17:02 smithi173 bash[27612]: cluster 2023-10-01T10:17:01.220413+0000 mgr.y (mgr.14427) 145 : cluster [DBG] pgmap v126: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:03.553 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:17:04.532 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:17:04 smithi078 bash[27336]: cluster 2023-10-01T10:17:03.221551+0000 mgr.y (mgr.14427) 146 : cluster [DBG] pgmap v127: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:04.532 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:17:04 smithi078 bash[33821]: cluster 2023-10-01T10:17:03.221551+0000 mgr.y (mgr.14427) 146 : cluster [DBG] pgmap v127: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:04.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:17:04 smithi173 bash[27612]: cluster 2023-10-01T10:17:03.221551+0000 mgr.y (mgr.14427) 146 : cluster [DBG] pgmap v127: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:05.333 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:17:05 smithi078 bash[27601]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:17:05] "GET /metrics HTTP/1.1" 200 191376 "" "Prometheus/2.33.4" 2023-10-01T10:17:05.620 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:17:05 smithi173 bash[28473]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:17:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T10:17:06.578 INFO:teuthology.orchestra.run.smithi078.stdout: 2023-10-01T10:17:06.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:17:06 smithi078 bash[33821]: cluster 2023-10-01T10:17:05.222215+0000 mgr.y (mgr.14427) 147 : cluster [DBG] pgmap v128: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:06.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:17:06 smithi078 bash[27336]: cluster 2023-10-01T10:17:05.222215+0000 mgr.y (mgr.14427) 147 : cluster [DBG] pgmap v128: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:06.584 INFO:teuthology.orchestra.run.smithi078.stderr:dumped all 2023-10-01T10:17:06.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:17:06 smithi173 bash[27612]: cluster 2023-10-01T10:17:05.222215+0000 mgr.y (mgr.14427) 147 : cluster [DBG] pgmap v128: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:07.399 INFO:teuthology.orchestra.run.smithi078.stdout:{"pg_ready":true,"pg_map":{"version":128,"stamp":"2023-10-01T10:17:05.221887+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2381752,"kb_used_data":4984,"kb_used_omap":0,"kb_used_meta":2376704,"kb_avail":747579464,"statfs":{"total":767960285184,"available":765521371136,"internally_reserved":0,"allocated":5103616,"data_stored":2165872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2433744896},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004429"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.771834+0000","last_change":"2023-10-01T10:15:40.316537+0000","last_active":"2023-10-01T10:16:17.771834+0000","last_peered":"2023-10-01T10:16:17.771834+0000","last_clean":"2023-10-01T10:16:17.771834+0000","last_became_active":"2023-10-01T10:15:40.316156+0000","last_became_peered":"2023-10-01T10:15:40.316156+0000","last_unstale":"2023-10-01T10:16:17.771834+0000","last_undegraded":"2023-10-01T10:16:17.771834+0000","last_fullsized":"2023-10-01T10:16:17.771834+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.290543+0000","last_change":"2023-10-01T10:15:39.289865+0000","last_active":"2023-10-01T10:16:17.290543+0000","last_peered":"2023-10-01T10:16:17.290543+0000","last_clean":"2023-10-01T10:16:17.290543+0000","last_became_active":"2023-10-01T10:15:39.289486+0000","last_became_peered":"2023-10-01T10:15:39.289486+0000","last_unstale":"2023-10-01T10:16:17.290543+0000","last_undegraded":"2023-10-01T10:16:17.290543+0000","last_fullsized":"2023-10-01T10:16:17.290543+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":4,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.771556+0000","last_change":"2023-10-01T10:15:40.306898+0000","last_active":"2023-10-01T10:16:17.771556+0000","last_peered":"2023-10-01T10:16:17.771556+0000","last_clean":"2023-10-01T10:16:17.771556+0000","last_became_active":"2023-10-01T10:15:40.306060+0000","last_became_peered":"2023-10-01T10:15:40.306060+0000","last_unstale":"2023-10-01T10:16:17.771556+0000","last_undegraded":"2023-10-01T10:16:17.771556+0000","last_fullsized":"2023-10-01T10:16:17.771556+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.773376+0000","last_change":"2023-10-01T10:15:39.291017+0000","last_active":"2023-10-01T10:16:17.773376+0000","last_peered":"2023-10-01T10:16:17.773376+0000","last_clean":"2023-10-01T10:16:17.773376+0000","last_became_active":"2023-10-01T10:15:39.289792+0000","last_became_peered":"2023-10-01T10:15:39.289792+0000","last_unstale":"2023-10-01T10:16:17.773376+0000","last_undegraded":"2023-10-01T10:16:17.773376+0000","last_fullsized":"2023-10-01T10:16:17.773376+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.772307+0000","last_change":"2023-10-01T10:15:40.310483+0000","last_active":"2023-10-01T10:16:17.772307+0000","last_peered":"2023-10-01T10:16:17.772307+0000","last_clean":"2023-10-01T10:16:17.772307+0000","last_became_active":"2023-10-01T10:15:40.310173+0000","last_became_peered":"2023-10-01T10:15:40.310173+0000","last_unstale":"2023-10-01T10:16:17.772307+0000","last_undegraded":"2023-10-01T10:16:17.772307+0000","last_fullsized":"2023-10-01T10:16:17.772307+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.291810+0000","last_change":"2023-10-01T10:15:39.296653+0000","last_active":"2023-10-01T10:16:17.291810+0000","last_peered":"2023-10-01T10:16:17.291810+0000","last_clean":"2023-10-01T10:16:17.291810+0000","last_became_active":"2023-10-01T10:15:39.296347+0000","last_became_peered":"2023-10-01T10:15:39.296347+0000","last_unstale":"2023-10-01T10:16:17.291810+0000","last_undegraded":"2023-10-01T10:16:17.291810+0000","last_fullsized":"2023-10-01T10:16:17.291810+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.772977+0000","last_change":"2023-10-01T10:15:40.317311+0000","last_active":"2023-10-01T10:16:17.772977+0000","last_peered":"2023-10-01T10:16:17.772977+0000","last_clean":"2023-10-01T10:16:17.772977+0000","last_became_active":"2023-10-01T10:15:40.315986+0000","last_became_peered":"2023-10-01T10:15:40.315986+0000","last_unstale":"2023-10-01T10:16:17.772977+0000","last_undegraded":"2023-10-01T10:16:17.772977+0000","last_fullsized":"2023-10-01T10:16:17.772977+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.291869+0000","last_change":"2023-10-01T10:15:39.301714+0000","last_active":"2023-10-01T10:16:17.291869+0000","last_peered":"2023-10-01T10:16:17.291869+0000","last_clean":"2023-10-01T10:16:17.291869+0000","last_became_active":"2023-10-01T10:15:39.301333+0000","last_became_peered":"2023-10-01T10:15:39.301333+0000","last_unstale":"2023-10-01T10:16:17.291869+0000","last_undegraded":"2023-10-01T10:16:17.291869+0000","last_fullsized":"2023-10-01T10:16:17.291869+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.770286+0000","last_change":"2023-10-01T10:15:40.311730+0000","last_active":"2023-10-01T10:16:17.770286+0000","last_peered":"2023-10-01T10:16:17.770286+0000","last_clean":"2023-10-01T10:16:17.770286+0000","last_became_active":"2023-10-01T10:15:40.311413+0000","last_became_peered":"2023-10-01T10:15:40.311413+0000","last_unstale":"2023-10-01T10:16:17.770286+0000","last_undegraded":"2023-10-01T10:16:17.770286+0000","last_fullsized":"2023-10-01T10:16:17.770286+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.289345+0000","last_change":"2023-10-01T10:15:39.294402+0000","last_active":"2023-10-01T10:16:17.289345+0000","last_peered":"2023-10-01T10:16:17.289345+0000","last_clean":"2023-10-01T10:16:17.289345+0000","last_became_active":"2023-10-01T10:15:39.294133+0000","last_became_peered":"2023-10-01T10:15:39.294133+0000","last_unstale":"2023-10-01T10:16:17.289345+0000","last_undegraded":"2023-10-01T10:16:17.289345+0000","last_fullsized":"2023-10-01T10:16:17.289345+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.772455+0000","last_change":"2023-10-01T10:15:40.318970+0000","last_active":"2023-10-01T10:16:17.772455+0000","last_peered":"2023-10-01T10:16:17.772455+0000","last_clean":"2023-10-01T10:16:17.772455+0000","last_became_active":"2023-10-01T10:15:40.318666+0000","last_became_peered":"2023-10-01T10:15:40.318666+0000","last_unstale":"2023-10-01T10:16:17.772455+0000","last_undegraded":"2023-10-01T10:16:17.772455+0000","last_fullsized":"2023-10-01T10:16:17.772455+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.289908+0000","last_change":"2023-10-01T10:15:39.290055+0000","last_active":"2023-10-01T10:16:17.289908+0000","last_peered":"2023-10-01T10:16:17.289908+0000","last_clean":"2023-10-01T10:16:17.289908+0000","last_became_active":"2023-10-01T10:15:39.289657+0000","last_became_peered":"2023-10-01T10:15:39.289657+0000","last_unstale":"2023-10-01T10:16:17.289908+0000","last_undegraded":"2023-10-01T10:16:17.289908+0000","last_fullsized":"2023-10-01T10:16:17.289908+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":17,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T10:15:41.421134+0000","last_change":"2023-10-01T10:13:44.666450+0000","last_active":"2023-10-01T10:15:41.421134+0000","last_peered":"2023-10-01T10:15:41.421134+0000","last_clean":"2023-10-01T10:15:41.421134+0000","last_became_active":"2023-10-01T10:13:44.665959+0000","last_became_peered":"2023-10-01T10:13:44.665959+0000","last_unstale":"2023-10-01T10:15:41.421134+0000","last_undegraded":"2023-10-01T10:15:41.421134+0000","last_fullsized":"2023-10-01T10:15:41.421134+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.771101+0000","last_change":"2023-10-01T10:15:40.317645+0000","last_active":"2023-10-01T10:16:17.771101+0000","last_peered":"2023-10-01T10:16:17.771101+0000","last_clean":"2023-10-01T10:16:17.771101+0000","last_became_active":"2023-10-01T10:15:40.317310+0000","last_became_peered":"2023-10-01T10:15:40.317310+0000","last_unstale":"2023-10-01T10:16:17.771101+0000","last_undegraded":"2023-10-01T10:16:17.771101+0000","last_fullsized":"2023-10-01T10:16:17.771101+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.767445+0000","last_change":"2023-10-01T10:13:44.825307+0000","last_active":"2023-10-01T10:16:17.767445+0000","last_peered":"2023-10-01T10:16:17.767445+0000","last_clean":"2023-10-01T10:16:17.767445+0000","last_became_active":"2023-10-01T10:13:44.824803+0000","last_became_peered":"2023-10-01T10:13:44.824803+0000","last_unstale":"2023-10-01T10:16:17.767445+0000","last_undegraded":"2023-10-01T10:16:17.767445+0000","last_fullsized":"2023-10-01T10:16:17.767445+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.773776+0000","last_change":"2023-10-01T10:15:39.299726+0000","last_active":"2023-10-01T10:16:17.773776+0000","last_peered":"2023-10-01T10:16:17.773776+0000","last_clean":"2023-10-01T10:16:17.773776+0000","last_became_active":"2023-10-01T10:15:39.299316+0000","last_became_peered":"2023-10-01T10:15:39.299316+0000","last_unstale":"2023-10-01T10:16:17.773776+0000","last_undegraded":"2023-10-01T10:16:17.773776+0000","last_fullsized":"2023-10-01T10:16:17.773776+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.289073+0000","last_change":"2023-10-01T10:15:40.314007+0000","last_active":"2023-10-01T10:16:17.289073+0000","last_peered":"2023-10-01T10:16:17.289073+0000","last_clean":"2023-10-01T10:16:17.289073+0000","last_became_active":"2023-10-01T10:15:40.313624+0000","last_became_peered":"2023-10-01T10:15:40.313624+0000","last_unstale":"2023-10-01T10:16:17.289073+0000","last_undegraded":"2023-10-01T10:16:17.289073+0000","last_fullsized":"2023-10-01T10:16:17.289073+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.767423+0000","last_change":"2023-10-01T10:15:39.298273+0000","last_active":"2023-10-01T10:16:17.767423+0000","last_peered":"2023-10-01T10:16:17.767423+0000","last_clean":"2023-10-01T10:16:17.767423+0000","last_became_active":"2023-10-01T10:15:39.297927+0000","last_became_peered":"2023-10-01T10:15:39.297927+0000","last_unstale":"2023-10-01T10:16:17.767423+0000","last_undegraded":"2023-10-01T10:16:17.767423+0000","last_fullsized":"2023-10-01T10:16:17.767423+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.777042+0000","last_change":"2023-10-01T10:13:44.827141+0000","last_active":"2023-10-01T10:16:17.777042+0000","last_peered":"2023-10-01T10:16:17.777042+0000","last_clean":"2023-10-01T10:16:17.777042+0000","last_became_active":"2023-10-01T10:13:44.826089+0000","last_became_peered":"2023-10-01T10:13:44.826089+0000","last_unstale":"2023-10-01T10:16:17.777042+0000","last_undegraded":"2023-10-01T10:16:17.777042+0000","last_fullsized":"2023-10-01T10:16:17.777042+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.290041+0000","last_change":"2023-10-01T10:15:40.313188+0000","last_active":"2023-10-01T10:16:17.290041+0000","last_peered":"2023-10-01T10:16:17.290041+0000","last_clean":"2023-10-01T10:16:17.290041+0000","last_became_active":"2023-10-01T10:15:40.312671+0000","last_became_peered":"2023-10-01T10:15:40.312671+0000","last_unstale":"2023-10-01T10:16:17.290041+0000","last_undegraded":"2023-10-01T10:16:17.290041+0000","last_fullsized":"2023-10-01T10:16:17.290041+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.289054+0000","last_change":"2023-10-01T10:15:39.295743+0000","last_active":"2023-10-01T10:16:17.289054+0000","last_peered":"2023-10-01T10:16:17.289054+0000","last_clean":"2023-10-01T10:16:17.289054+0000","last_became_active":"2023-10-01T10:15:39.295398+0000","last_became_peered":"2023-10-01T10:15:39.295398+0000","last_unstale":"2023-10-01T10:16:17.289054+0000","last_undegraded":"2023-10-01T10:16:17.289054+0000","last_fullsized":"2023-10-01T10:16:17.289054+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.776437+0000","last_change":"2023-10-01T10:13:44.825879+0000","last_active":"2023-10-01T10:16:17.776437+0000","last_peered":"2023-10-01T10:16:17.776437+0000","last_clean":"2023-10-01T10:16:17.776437+0000","last_became_active":"2023-10-01T10:13:44.825097+0000","last_became_peered":"2023-10-01T10:13:44.825097+0000","last_unstale":"2023-10-01T10:16:17.776437+0000","last_undegraded":"2023-10-01T10:16:17.776437+0000","last_fullsized":"2023-10-01T10:16:17.776437+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T10:15:41.421241+0000","last_change":"2023-10-01T10:15:40.316207+0000","last_active":"2023-10-01T10:15:41.421241+0000","last_peered":"2023-10-01T10:15:41.421241+0000","last_clean":"2023-10-01T10:15:41.421241+0000","last_became_active":"2023-10-01T10:15:40.315843+0000","last_became_peered":"2023-10-01T10:15:40.315843+0000","last_unstale":"2023-10-01T10:15:41.421241+0000","last_undegraded":"2023-10-01T10:15:41.421241+0000","last_fullsized":"2023-10-01T10:15:41.421241+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.771335+0000","last_change":"2023-10-01T10:15:39.293526+0000","last_active":"2023-10-01T10:16:17.771335+0000","last_peered":"2023-10-01T10:16:17.771335+0000","last_clean":"2023-10-01T10:16:17.771335+0000","last_became_active":"2023-10-01T10:15:39.293196+0000","last_became_peered":"2023-10-01T10:15:39.293196+0000","last_unstale":"2023-10-01T10:16:17.771335+0000","last_undegraded":"2023-10-01T10:16:17.771335+0000","last_fullsized":"2023-10-01T10:16:17.771335+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":22,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.776338+0000","last_change":"2023-10-01T10:12:47.477457+0000","last_active":"2023-10-01T10:16:17.776338+0000","last_peered":"2023-10-01T10:16:17.776338+0000","last_clean":"2023-10-01T10:16:17.776338+0000","last_became_active":"2023-10-01T10:12:47.476522+0000","last_became_peered":"2023-10-01T10:12:47.476522+0000","last_unstale":"2023-10-01T10:16:17.776338+0000","last_undegraded":"2023-10-01T10:16:17.776338+0000","last_fullsized":"2023-10-01T10:16:17.776338+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:09:17.221082+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:09:17.221082+0000","last_clean_scrub_stamp":"2023-10-01T10:09:17.221082+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.291528+0000","last_change":"2023-10-01T10:13:44.663992+0000","last_active":"2023-10-01T10:16:17.291528+0000","last_peered":"2023-10-01T10:16:17.291528+0000","last_clean":"2023-10-01T10:16:17.291528+0000","last_became_active":"2023-10-01T10:13:44.663669+0000","last_became_peered":"2023-10-01T10:13:44.663669+0000","last_unstale":"2023-10-01T10:16:17.291528+0000","last_undegraded":"2023-10-01T10:16:17.291528+0000","last_fullsized":"2023-10-01T10:16:17.291528+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.767341+0000","last_change":"2023-10-01T10:15:39.727816+0000","last_active":"2023-10-01T10:16:17.767341+0000","last_peered":"2023-10-01T10:16:17.767341+0000","last_clean":"2023-10-01T10:16:17.767341+0000","last_became_active":"2023-10-01T10:15:39.727202+0000","last_became_peered":"2023-10-01T10:15:39.727202+0000","last_unstale":"2023-10-01T10:16:17.767341+0000","last_undegraded":"2023-10-01T10:16:17.767341+0000","last_fullsized":"2023-10-01T10:16:17.767341+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.766733+0000","last_change":"2023-10-01T10:15:39.295490+0000","last_active":"2023-10-01T10:16:17.766733+0000","last_peered":"2023-10-01T10:16:17.766733+0000","last_clean":"2023-10-01T10:16:17.766733+0000","last_became_active":"2023-10-01T10:15:39.294900+0000","last_became_peered":"2023-10-01T10:15:39.294900+0000","last_unstale":"2023-10-01T10:16:17.766733+0000","last_undegraded":"2023-10-01T10:16:17.766733+0000","last_fullsized":"2023-10-01T10:16:17.766733+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.770202+0000","last_change":"2023-10-01T10:13:44.823885+0000","last_active":"2023-10-01T10:16:17.770202+0000","last_peered":"2023-10-01T10:16:17.770202+0000","last_clean":"2023-10-01T10:16:17.770202+0000","last_became_active":"2023-10-01T10:13:44.823314+0000","last_became_peered":"2023-10-01T10:13:44.823314+0000","last_unstale":"2023-10-01T10:16:17.770202+0000","last_undegraded":"2023-10-01T10:16:17.770202+0000","last_fullsized":"2023-10-01T10:16:17.770202+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.767203+0000","last_change":"2023-10-01T10:15:39.670779+0000","last_active":"2023-10-01T10:16:17.767203+0000","last_peered":"2023-10-01T10:16:17.767203+0000","last_clean":"2023-10-01T10:16:17.767203+0000","last_became_active":"2023-10-01T10:15:39.670136+0000","last_became_peered":"2023-10-01T10:15:39.670136+0000","last_unstale":"2023-10-01T10:16:17.767203+0000","last_undegraded":"2023-10-01T10:16:17.767203+0000","last_fullsized":"2023-10-01T10:16:17.767203+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.772485+0000","last_change":"2023-10-01T10:15:39.291273+0000","last_active":"2023-10-01T10:16:17.772485+0000","last_peered":"2023-10-01T10:16:17.772485+0000","last_clean":"2023-10-01T10:16:17.772485+0000","last_became_active":"2023-10-01T10:15:39.290959+0000","last_became_peered":"2023-10-01T10:15:39.290959+0000","last_unstale":"2023-10-01T10:16:17.772485+0000","last_undegraded":"2023-10-01T10:16:17.772485+0000","last_fullsized":"2023-10-01T10:16:17.772485+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.291428+0000","last_change":"2023-10-01T10:13:44.663534+0000","last_active":"2023-10-01T10:16:17.291428+0000","last_peered":"2023-10-01T10:16:17.291428+0000","last_clean":"2023-10-01T10:16:17.291428+0000","last_became_active":"2023-10-01T10:13:44.663014+0000","last_became_peered":"2023-10-01T10:13:44.663014+0000","last_unstale":"2023-10-01T10:16:17.291428+0000","last_undegraded":"2023-10-01T10:16:17.291428+0000","last_fullsized":"2023-10-01T10:16:17.291428+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.766946+0000","last_change":"2023-10-01T10:15:39.343985+0000","last_active":"2023-10-01T10:16:17.766946+0000","last_peered":"2023-10-01T10:16:17.766946+0000","last_clean":"2023-10-01T10:16:17.766946+0000","last_became_active":"2023-10-01T10:15:39.343351+0000","last_became_peered":"2023-10-01T10:15:39.343351+0000","last_unstale":"2023-10-01T10:16:17.766946+0000","last_undegraded":"2023-10-01T10:16:17.766946+0000","last_fullsized":"2023-10-01T10:16:17.766946+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.291766+0000","last_change":"2023-10-01T10:15:39.297597+0000","last_active":"2023-10-01T10:16:17.291766+0000","last_peered":"2023-10-01T10:16:17.291766+0000","last_clean":"2023-10-01T10:16:17.291766+0000","last_became_active":"2023-10-01T10:15:39.297235+0000","last_became_peered":"2023-10-01T10:15:39.297235+0000","last_unstale":"2023-10-01T10:16:17.291766+0000","last_undegraded":"2023-10-01T10:16:17.291766+0000","last_fullsized":"2023-10-01T10:16:17.291766+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.767016+0000","last_change":"2023-10-01T10:13:44.824140+0000","last_active":"2023-10-01T10:16:17.767016+0000","last_peered":"2023-10-01T10:16:17.767016+0000","last_clean":"2023-10-01T10:16:17.767016+0000","last_became_active":"2023-10-01T10:13:44.823462+0000","last_became_peered":"2023-10-01T10:13:44.823462+0000","last_unstale":"2023-10-01T10:16:17.767016+0000","last_undegraded":"2023-10-01T10:16:17.767016+0000","last_fullsized":"2023-10-01T10:16:17.767016+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":10,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-01T10:16:18.299852+0000","last_change":"2023-10-01T10:16:18.299852+0000","last_active":"2023-10-01T10:16:18.299852+0000","last_peered":"2023-10-01T10:16:18.299852+0000","last_clean":"2023-10-01T10:16:18.299852+0000","last_became_active":"2023-10-01T10:16:18.299391+0000","last_became_peered":"2023-10-01T10:16:18.299391+0000","last_unstale":"2023-10-01T10:16:18.299852+0000","last_undegraded":"2023-10-01T10:16:18.299852+0000","last_fullsized":"2023-10-01T10:16:18.299852+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.771364+0000","last_change":"2023-10-01T10:15:39.297452+0000","last_active":"2023-10-01T10:16:17.771364+0000","last_peered":"2023-10-01T10:16:17.771364+0000","last_clean":"2023-10-01T10:16:17.771364+0000","last_became_active":"2023-10-01T10:15:39.297021+0000","last_became_peered":"2023-10-01T10:15:39.297021+0000","last_unstale":"2023-10-01T10:16:17.771364+0000","last_undegraded":"2023-10-01T10:16:17.771364+0000","last_fullsized":"2023-10-01T10:16:17.771364+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.773153+0000","last_change":"2023-10-01T10:15:40.313526+0000","last_active":"2023-10-01T10:16:17.773153+0000","last_peered":"2023-10-01T10:16:17.773153+0000","last_clean":"2023-10-01T10:16:17.773153+0000","last_became_active":"2023-10-01T10:15:40.313153+0000","last_became_peered":"2023-10-01T10:15:40.313153+0000","last_unstale":"2023-10-01T10:16:17.773153+0000","last_undegraded":"2023-10-01T10:16:17.773153+0000","last_fullsized":"2023-10-01T10:16:17.773153+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.772536+0000","last_change":"2023-10-01T10:15:39.296739+0000","last_active":"2023-10-01T10:16:17.772536+0000","last_peered":"2023-10-01T10:16:17.772536+0000","last_clean":"2023-10-01T10:16:17.772536+0000","last_became_active":"2023-10-01T10:15:39.296339+0000","last_became_peered":"2023-10-01T10:15:39.296339+0000","last_unstale":"2023-10-01T10:16:17.772536+0000","last_undegraded":"2023-10-01T10:16:17.772536+0000","last_fullsized":"2023-10-01T10:16:17.772536+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":22,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-01T10:16:18.364581+0000","last_change":"2023-10-01T10:16:18.364581+0000","last_active":"2023-10-01T10:16:18.364581+0000","last_peered":"2023-10-01T10:16:18.364581+0000","last_clean":"2023-10-01T10:16:18.364581+0000","last_became_active":"2023-10-01T10:16:18.363621+0000","last_became_peered":"2023-10-01T10:16:18.363621+0000","last_unstale":"2023-10-01T10:16:18.364581+0000","last_undegraded":"2023-10-01T10:16:18.364581+0000","last_fullsized":"2023-10-01T10:16:18.364581+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T10:15:41.420588+0000","last_change":"2023-10-01T10:15:39.299457+0000","last_active":"2023-10-01T10:15:41.420588+0000","last_peered":"2023-10-01T10:15:41.420588+0000","last_clean":"2023-10-01T10:15:41.420588+0000","last_became_active":"2023-10-01T10:15:39.298456+0000","last_became_peered":"2023-10-01T10:15:39.298456+0000","last_unstale":"2023-10-01T10:15:41.420588+0000","last_undegraded":"2023-10-01T10:15:41.420588+0000","last_fullsized":"2023-10-01T10:15:41.420588+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.772970+0000","last_change":"2023-10-01T10:15:40.309865+0000","last_active":"2023-10-01T10:16:17.772970+0000","last_peered":"2023-10-01T10:16:17.772970+0000","last_clean":"2023-10-01T10:16:17.772970+0000","last_became_active":"2023-10-01T10:15:40.309490+0000","last_became_peered":"2023-10-01T10:15:40.309490+0000","last_unstale":"2023-10-01T10:16:17.772970+0000","last_undegraded":"2023-10-01T10:16:17.772970+0000","last_fullsized":"2023-10-01T10:16:17.772970+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.772480+0000","last_change":"2023-10-01T10:15:39.300710+0000","last_active":"2023-10-01T10:16:17.772480+0000","last_peered":"2023-10-01T10:16:17.772480+0000","last_clean":"2023-10-01T10:16:17.772480+0000","last_became_active":"2023-10-01T10:15:39.300387+0000","last_became_peered":"2023-10-01T10:15:39.300387+0000","last_unstale":"2023-10-01T10:16:17.772480+0000","last_undegraded":"2023-10-01T10:16:17.772480+0000","last_fullsized":"2023-10-01T10:16:17.772480+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.289505+0000","last_change":"2023-10-01T10:15:40.311169+0000","last_active":"2023-10-01T10:16:17.289505+0000","last_peered":"2023-10-01T10:16:17.289505+0000","last_clean":"2023-10-01T10:16:17.289505+0000","last_became_active":"2023-10-01T10:15:40.310611+0000","last_became_peered":"2023-10-01T10:15:40.310611+0000","last_unstale":"2023-10-01T10:16:17.289505+0000","last_undegraded":"2023-10-01T10:16:17.289505+0000","last_fullsized":"2023-10-01T10:16:17.289505+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.291644+0000","last_change":"2023-10-01T10:15:39.295299+0000","last_active":"2023-10-01T10:16:17.291644+0000","last_peered":"2023-10-01T10:16:17.291644+0000","last_clean":"2023-10-01T10:16:17.291644+0000","last_became_active":"2023-10-01T10:15:39.294974+0000","last_became_peered":"2023-10-01T10:15:39.294974+0000","last_unstale":"2023-10-01T10:16:17.291644+0000","last_undegraded":"2023-10-01T10:16:17.291644+0000","last_fullsized":"2023-10-01T10:16:17.291644+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T10:15:41.420342+0000","last_change":"2023-10-01T10:15:40.319834+0000","last_active":"2023-10-01T10:15:41.420342+0000","last_peered":"2023-10-01T10:15:41.420342+0000","last_clean":"2023-10-01T10:15:41.420342+0000","last_became_active":"2023-10-01T10:15:40.319460+0000","last_became_peered":"2023-10-01T10:15:40.319460+0000","last_unstale":"2023-10-01T10:15:41.420342+0000","last_undegraded":"2023-10-01T10:15:41.420342+0000","last_fullsized":"2023-10-01T10:15:41.420342+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.776935+0000","last_change":"2023-10-01T10:15:39.293587+0000","last_active":"2023-10-01T10:16:17.776935+0000","last_peered":"2023-10-01T10:16:17.776935+0000","last_clean":"2023-10-01T10:16:17.776935+0000","last_became_active":"2023-10-01T10:15:39.293273+0000","last_became_peered":"2023-10-01T10:15:39.293273+0000","last_unstale":"2023-10-01T10:16:17.776935+0000","last_undegraded":"2023-10-01T10:16:17.776935+0000","last_fullsized":"2023-10-01T10:16:17.776935+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.772238+0000","last_change":"2023-10-01T10:15:40.312616+0000","last_active":"2023-10-01T10:16:17.772238+0000","last_peered":"2023-10-01T10:16:17.772238+0000","last_clean":"2023-10-01T10:16:17.772238+0000","last_became_active":"2023-10-01T10:15:40.312249+0000","last_became_peered":"2023-10-01T10:15:40.312249+0000","last_unstale":"2023-10-01T10:16:17.772238+0000","last_undegraded":"2023-10-01T10:16:17.772238+0000","last_fullsized":"2023-10-01T10:16:17.772238+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.776891+0000","last_change":"2023-10-01T10:15:39.302221+0000","last_active":"2023-10-01T10:16:17.776891+0000","last_peered":"2023-10-01T10:16:17.776891+0000","last_clean":"2023-10-01T10:16:17.776891+0000","last_became_active":"2023-10-01T10:15:39.301850+0000","last_became_peered":"2023-10-01T10:15:39.301850+0000","last_unstale":"2023-10-01T10:16:17.776891+0000","last_undegraded":"2023-10-01T10:16:17.776891+0000","last_fullsized":"2023-10-01T10:16:17.776891+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.291406+0000","last_change":"2023-10-01T10:15:40.318702+0000","last_active":"2023-10-01T10:16:17.291406+0000","last_peered":"2023-10-01T10:16:17.291406+0000","last_clean":"2023-10-01T10:16:17.291406+0000","last_became_active":"2023-10-01T10:15:40.318333+0000","last_became_peered":"2023-10-01T10:15:40.318333+0000","last_unstale":"2023-10-01T10:16:17.291406+0000","last_undegraded":"2023-10-01T10:16:17.291406+0000","last_fullsized":"2023-10-01T10:16:17.291406+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.777054+0000","last_change":"2023-10-01T10:15:39.300486+0000","last_active":"2023-10-01T10:16:17.777054+0000","last_peered":"2023-10-01T10:16:17.777054+0000","last_clean":"2023-10-01T10:16:17.777054+0000","last_became_active":"2023-10-01T10:15:39.300129+0000","last_became_peered":"2023-10-01T10:15:39.300129+0000","last_unstale":"2023-10-01T10:16:17.777054+0000","last_undegraded":"2023-10-01T10:16:17.777054+0000","last_fullsized":"2023-10-01T10:16:17.777054+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.772864+0000","last_change":"2023-10-01T10:15:40.317579+0000","last_active":"2023-10-01T10:16:17.772864+0000","last_peered":"2023-10-01T10:16:17.772864+0000","last_clean":"2023-10-01T10:16:17.772864+0000","last_became_active":"2023-10-01T10:15:40.317213+0000","last_became_peered":"2023-10-01T10:15:40.317213+0000","last_unstale":"2023-10-01T10:16:17.772864+0000","last_undegraded":"2023-10-01T10:16:17.772864+0000","last_fullsized":"2023-10-01T10:16:17.772864+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":21,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T10:15:44.811930+0000","last_change":"2023-10-01T10:15:39.291400+0000","last_active":"2023-10-01T10:15:44.811930+0000","last_peered":"2023-10-01T10:15:44.811930+0000","last_clean":"2023-10-01T10:15:44.811930+0000","last_became_active":"2023-10-01T10:15:39.291018+0000","last_became_peered":"2023-10-01T10:15:39.291018+0000","last_unstale":"2023-10-01T10:15:44.811930+0000","last_undegraded":"2023-10-01T10:15:44.811930+0000","last_fullsized":"2023-10-01T10:15:44.811930+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":8,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.289641+0000","last_change":"2023-10-01T10:15:40.316610+0000","last_active":"2023-10-01T10:16:17.289641+0000","last_peered":"2023-10-01T10:16:17.289641+0000","last_clean":"2023-10-01T10:16:17.289641+0000","last_became_active":"2023-10-01T10:15:40.316284+0000","last_became_peered":"2023-10-01T10:15:40.316284+0000","last_unstale":"2023-10-01T10:16:17.289641+0000","last_undegraded":"2023-10-01T10:16:17.289641+0000","last_fullsized":"2023-10-01T10:16:17.289641+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.776247+0000","last_change":"2023-10-01T10:15:39.302235+0000","last_active":"2023-10-01T10:16:17.776247+0000","last_peered":"2023-10-01T10:16:17.776247+0000","last_clean":"2023-10-01T10:16:17.776247+0000","last_became_active":"2023-10-01T10:15:39.301830+0000","last_became_peered":"2023-10-01T10:15:39.301830+0000","last_unstale":"2023-10-01T10:16:17.776247+0000","last_undegraded":"2023-10-01T10:16:17.776247+0000","last_fullsized":"2023-10-01T10:16:17.776247+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.291632+0000","last_change":"2023-10-01T10:15:40.315419+0000","last_active":"2023-10-01T10:16:17.291632+0000","last_peered":"2023-10-01T10:16:17.291632+0000","last_clean":"2023-10-01T10:16:17.291632+0000","last_became_active":"2023-10-01T10:15:40.314924+0000","last_became_peered":"2023-10-01T10:15:40.314924+0000","last_unstale":"2023-10-01T10:16:17.291632+0000","last_undegraded":"2023-10-01T10:16:17.291632+0000","last_fullsized":"2023-10-01T10:16:17.291632+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.289124+0000","last_change":"2023-10-01T10:15:39.293968+0000","last_active":"2023-10-01T10:16:17.289124+0000","last_peered":"2023-10-01T10:16:17.289124+0000","last_clean":"2023-10-01T10:16:17.289124+0000","last_became_active":"2023-10-01T10:15:39.293630+0000","last_became_peered":"2023-10-01T10:15:39.293630+0000","last_unstale":"2023-10-01T10:16:17.289124+0000","last_undegraded":"2023-10-01T10:16:17.289124+0000","last_fullsized":"2023-10-01T10:16:17.289124+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":22,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-01T10:16:18.300879+0000","last_change":"2023-10-01T10:16:18.300879+0000","last_active":"2023-10-01T10:16:18.300879+0000","last_peered":"2023-10-01T10:16:18.300879+0000","last_clean":"2023-10-01T10:16:18.300879+0000","last_became_active":"2023-10-01T10:16:18.300505+0000","last_became_peered":"2023-10-01T10:16:18.300505+0000","last_unstale":"2023-10-01T10:16:18.300879+0000","last_undegraded":"2023-10-01T10:16:18.300879+0000","last_fullsized":"2023-10-01T10:16:18.300879+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.776614+0000","last_change":"2023-10-01T10:15:39.300411+0000","last_active":"2023-10-01T10:16:17.776614+0000","last_peered":"2023-10-01T10:16:17.776614+0000","last_clean":"2023-10-01T10:16:17.776614+0000","last_became_active":"2023-10-01T10:15:39.300066+0000","last_became_peered":"2023-10-01T10:15:39.300066+0000","last_unstale":"2023-10-01T10:16:17.776614+0000","last_undegraded":"2023-10-01T10:16:17.776614+0000","last_fullsized":"2023-10-01T10:16:17.776614+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.772758+0000","last_change":"2023-10-01T10:15:40.317496+0000","last_active":"2023-10-01T10:16:17.772758+0000","last_peered":"2023-10-01T10:16:17.772758+0000","last_clean":"2023-10-01T10:16:17.772758+0000","last_became_active":"2023-10-01T10:15:40.316915+0000","last_became_peered":"2023-10-01T10:15:40.316915+0000","last_unstale":"2023-10-01T10:16:17.772758+0000","last_undegraded":"2023-10-01T10:16:17.772758+0000","last_fullsized":"2023-10-01T10:16:17.772758+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"60'6","reported_seq":20,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.772907+0000","last_change":"2023-10-01T10:15:39.296588+0000","last_active":"2023-10-01T10:16:17.772907+0000","last_peered":"2023-10-01T10:16:17.772907+0000","last_clean":"2023-10-01T10:16:17.772907+0000","last_became_active":"2023-10-01T10:15:39.294601+0000","last_became_peered":"2023-10-01T10:15:39.294601+0000","last_unstale":"2023-10-01T10:16:17.772907+0000","last_undegraded":"2023-10-01T10:16:17.772907+0000","last_fullsized":"2023-10-01T10:16:17.772907+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":6,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.766703+0000","last_change":"2023-10-01T10:15:39.310807+0000","last_active":"2023-10-01T10:16:17.766703+0000","last_peered":"2023-10-01T10:16:17.766703+0000","last_clean":"2023-10-01T10:16:17.766703+0000","last_became_active":"2023-10-01T10:15:39.310474+0000","last_became_peered":"2023-10-01T10:15:39.310474+0000","last_unstale":"2023-10-01T10:16:17.766703+0000","last_undegraded":"2023-10-01T10:16:17.766703+0000","last_fullsized":"2023-10-01T10:16:17.766703+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":17,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.776723+0000","last_change":"2023-10-01T10:15:39.300951+0000","last_active":"2023-10-01T10:16:17.776723+0000","last_peered":"2023-10-01T10:16:17.776723+0000","last_clean":"2023-10-01T10:16:17.776723+0000","last_became_active":"2023-10-01T10:15:39.300626+0000","last_became_peered":"2023-10-01T10:15:39.300626+0000","last_unstale":"2023-10-01T10:16:17.776723+0000","last_undegraded":"2023-10-01T10:16:17.776723+0000","last_fullsized":"2023-10-01T10:16:17.776723+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.771751+0000","last_change":"2023-10-01T10:15:40.317104+0000","last_active":"2023-10-01T10:16:17.771751+0000","last_peered":"2023-10-01T10:16:17.771751+0000","last_clean":"2023-10-01T10:16:17.771751+0000","last_became_active":"2023-10-01T10:15:40.316787+0000","last_became_peered":"2023-10-01T10:15:40.316787+0000","last_unstale":"2023-10-01T10:16:17.771751+0000","last_undegraded":"2023-10-01T10:16:17.771751+0000","last_fullsized":"2023-10-01T10:16:17.771751+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.291729+0000","last_change":"2023-10-01T10:15:39.294083+0000","last_active":"2023-10-01T10:16:17.291729+0000","last_peered":"2023-10-01T10:16:17.291729+0000","last_clean":"2023-10-01T10:16:17.291729+0000","last_became_active":"2023-10-01T10:15:39.293744+0000","last_became_peered":"2023-10-01T10:15:39.293744+0000","last_unstale":"2023-10-01T10:16:17.291729+0000","last_undegraded":"2023-10-01T10:16:17.291729+0000","last_fullsized":"2023-10-01T10:16:17.291729+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.771584+0000","last_change":"2023-10-01T10:15:40.315049+0000","last_active":"2023-10-01T10:16:17.771584+0000","last_peered":"2023-10-01T10:16:17.771584+0000","last_clean":"2023-10-01T10:16:17.771584+0000","last_became_active":"2023-10-01T10:15:40.314510+0000","last_became_peered":"2023-10-01T10:15:40.314510+0000","last_unstale":"2023-10-01T10:16:17.771584+0000","last_undegraded":"2023-10-01T10:16:17.771584+0000","last_fullsized":"2023-10-01T10:16:17.771584+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.290468+0000","last_change":"2023-10-01T10:15:39.297202+0000","last_active":"2023-10-01T10:16:17.290468+0000","last_peered":"2023-10-01T10:16:17.290468+0000","last_clean":"2023-10-01T10:16:17.290468+0000","last_became_active":"2023-10-01T10:15:39.296867+0000","last_became_peered":"2023-10-01T10:15:39.296867+0000","last_unstale":"2023-10-01T10:16:17.290468+0000","last_undegraded":"2023-10-01T10:16:17.290468+0000","last_fullsized":"2023-10-01T10:16:17.290468+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T10:15:41.421148+0000","last_change":"2023-10-01T10:15:40.315154+0000","last_active":"2023-10-01T10:15:41.421148+0000","last_peered":"2023-10-01T10:15:41.421148+0000","last_clean":"2023-10-01T10:15:41.421148+0000","last_became_active":"2023-10-01T10:15:40.314777+0000","last_became_peered":"2023-10-01T10:15:40.314777+0000","last_unstale":"2023-10-01T10:15:41.421148+0000","last_undegraded":"2023-10-01T10:15:41.421148+0000","last_fullsized":"2023-10-01T10:15:41.421148+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.771811+0000","last_change":"2023-10-01T10:15:39.298982+0000","last_active":"2023-10-01T10:16:17.771811+0000","last_peered":"2023-10-01T10:16:17.771811+0000","last_clean":"2023-10-01T10:16:17.771811+0000","last_became_active":"2023-10-01T10:15:39.298695+0000","last_became_peered":"2023-10-01T10:15:39.298695+0000","last_unstale":"2023-10-01T10:16:17.771811+0000","last_undegraded":"2023-10-01T10:16:17.771811+0000","last_fullsized":"2023-10-01T10:16:17.771811+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.289766+0000","last_change":"2023-10-01T10:15:40.315110+0000","last_active":"2023-10-01T10:16:17.289766+0000","last_peered":"2023-10-01T10:16:17.289766+0000","last_clean":"2023-10-01T10:16:17.289766+0000","last_became_active":"2023-10-01T10:15:40.314775+0000","last_became_peered":"2023-10-01T10:15:40.314775+0000","last_unstale":"2023-10-01T10:16:17.289766+0000","last_undegraded":"2023-10-01T10:16:17.289766+0000","last_fullsized":"2023-10-01T10:16:17.289766+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.766436+0000","last_change":"2023-10-01T10:15:39.294176+0000","last_active":"2023-10-01T10:16:17.766436+0000","last_peered":"2023-10-01T10:16:17.766436+0000","last_clean":"2023-10-01T10:16:17.766436+0000","last_became_active":"2023-10-01T10:15:39.293850+0000","last_became_peered":"2023-10-01T10:15:39.293850+0000","last_unstale":"2023-10-01T10:16:17.766436+0000","last_undegraded":"2023-10-01T10:16:17.766436+0000","last_fullsized":"2023-10-01T10:16:17.766436+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.773060+0000","last_change":"2023-10-01T10:15:40.311002+0000","last_active":"2023-10-01T10:16:17.773060+0000","last_peered":"2023-10-01T10:16:17.773060+0000","last_clean":"2023-10-01T10:16:17.773060+0000","last_became_active":"2023-10-01T10:15:40.310679+0000","last_became_peered":"2023-10-01T10:15:40.310679+0000","last_unstale":"2023-10-01T10:16:17.773060+0000","last_undegraded":"2023-10-01T10:16:17.773060+0000","last_fullsized":"2023-10-01T10:16:17.773060+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.772341+0000","last_change":"2023-10-01T10:15:39.298785+0000","last_active":"2023-10-01T10:16:17.772341+0000","last_peered":"2023-10-01T10:16:17.772341+0000","last_clean":"2023-10-01T10:16:17.772341+0000","last_became_active":"2023-10-01T10:15:39.297156+0000","last_became_peered":"2023-10-01T10:15:39.297156+0000","last_unstale":"2023-10-01T10:16:17.772341+0000","last_undegraded":"2023-10-01T10:16:17.772341+0000","last_fullsized":"2023-10-01T10:16:17.772341+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430262,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297736,"kb_used_data":640,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447416,"statfs":{"total":95995035648,"available":95690153984,"internally_reserved":0,"allocated":655360,"data_stored":271700,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:16:55 2023","interfaces":[{"interface":"back","average":{"1min":0.86199999999999999,"5min":0.89500000000000002,"15min":0.91800000000000004},"min":{"1min":0.56699999999999995,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.1859999999999999,"5min":1.2170000000000001,"15min":1.2170000000000001},"last":1.073},{"interface":"front","average":{"1min":0.88300000000000001,"5min":0.88600000000000001,"15min":0.91900000000000004},"min":{"1min":0.68300000000000005,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":1.0720000000000001,"5min":1.2629999999999999,"15min":1.2629999999999999},"last":1.0129999999999999}]},{"osd":1,"last update":"Sun Oct 1 10:16:55 2023","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":0.91100000000000003,"15min":0.91000000000000003},"min":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"max":{"1min":1.3260000000000001,"5min":1.444,"15min":1.444},"last":0.79000000000000004},{"interface":"front","average":{"1min":0.91100000000000003,"5min":0.89300000000000002,"15min":0.88700000000000001},"min":{"1min":0.74099999999999999,"5min":0.56599999999999995,"15min":0.56599999999999995},"max":{"1min":1.1140000000000001,"5min":1.377,"15min":1.377},"last":1.1559999999999999}]},{"osd":2,"last update":"Sun Oct 1 10:16:55 2023","interfaces":[{"interface":"back","average":{"1min":0.995,"5min":0.94199999999999995,"15min":0.93600000000000005},"min":{"1min":0.71599999999999997,"5min":0.61899999999999999,"15min":0.61899999999999999},"max":{"1min":1.631,"5min":1.631,"15min":1.631},"last":1.1299999999999999},{"interface":"front","average":{"1min":1.0249999999999999,"5min":0.97099999999999997,"15min":0.96199999999999997},"min":{"1min":0.82799999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"max":{"1min":1.218,"5min":1.2390000000000001,"15min":1.2390000000000001},"last":0.99099999999999999}]},{"osd":3,"last update":"Sun Oct 1 10:16:55 2023","interfaces":[{"interface":"back","average":{"1min":0.96599999999999997,"5min":0.91900000000000004,"15min":0.91100000000000003},"min":{"1min":0.66600000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.2450000000000001,"5min":1.2450000000000001,"15min":1.2450000000000001},"last":0.84499999999999997},{"interface":"front","average":{"1min":1.0569999999999999,"5min":1.0189999999999999,"15min":1.01},"min":{"1min":0.73399999999999999,"5min":0.57199999999999995,"15min":0.57199999999999995},"max":{"1min":1.3919999999999999,"5min":3.2229999999999999,"15min":3.2229999999999999},"last":1.2190000000000001}]},{"osd":4,"last update":"Sun Oct 1 10:16:55 2023","interfaces":[{"interface":"back","average":{"1min":1.0009999999999999,"5min":0.95699999999999996,"15min":0.94299999999999995},"min":{"1min":0.55200000000000005,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":1.2929999999999999,"5min":1.5629999999999999,"15min":1.5629999999999999},"last":0.88200000000000001},{"interface":"front","average":{"1min":1.028,"5min":0.96299999999999997,"15min":0.93600000000000005},"min":{"1min":0.72699999999999998,"5min":0.437,"15min":0.437},"max":{"1min":1.266,"5min":1.7470000000000001,"15min":1.7470000000000001},"last":0.755}]},{"osd":5,"last update":"Sun Oct 1 10:16:55 2023","interfaces":[{"interface":"back","average":{"1min":1.0740000000000001,"5min":1.01,"15min":1.002},"min":{"1min":0.69499999999999995,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":1.3089999999999999,"5min":1.831,"15min":1.831},"last":1.0489999999999999},{"interface":"front","average":{"1min":1.161,"5min":1.0740000000000001,"15min":1.0920000000000001},"min":{"1min":0.73999999999999999,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":1.3700000000000001,"5min":1.5,"15min":1.5},"last":1.208}]},{"osd":6,"last update":"Sun Oct 1 10:16:55 2023","interfaces":[{"interface":"back","average":{"1min":1.1519999999999999,"5min":1.081,"15min":1.0920000000000001},"min":{"1min":0.83399999999999996,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":1.381,"5min":1.7150000000000001,"15min":1.7150000000000001},"last":0.94399999999999995},{"interface":"front","average":{"1min":1.1299999999999999,"5min":1.099,"15min":1.109},"min":{"1min":0.68999999999999995,"5min":0.60499999999999998,"15min":0.60499999999999998},"max":{"1min":1.343,"5min":2.2869999999999999,"15min":2.2869999999999999},"last":0.92100000000000004}]}]},{"osd":6,"up_from":42,"seq":180388626496,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297720,"kb_used_data":624,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447432,"statfs":{"total":95995035648,"available":95690170368,"internally_reserved":0,"allocated":638976,"data_stored":271239,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:16:12 2023","interfaces":[{"interface":"back","average":{"1min":0.89300000000000002,"5min":0.86899999999999999,"15min":0.88900000000000001},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":1.278,"5min":1.359,"15min":1.359},"last":0.82499999999999996},{"interface":"front","average":{"1min":0.82799999999999996,"5min":0.85999999999999999,"15min":0.85399999999999998},"min":{"1min":0.55400000000000005,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":1.173,"5min":1.1839999999999999,"15min":1.1839999999999999},"last":1.153}]},{"osd":1,"last update":"Sun Oct 1 10:16:12 2023","interfaces":[{"interface":"back","average":{"1min":0.995,"5min":0.92000000000000004,"15min":0.93400000000000005},"min":{"1min":0.502,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":1.6279999999999999,"5min":1.6279999999999999,"15min":1.6279999999999999},"last":1.0620000000000001},{"interface":"front","average":{"1min":0.94399999999999995,"5min":0.93300000000000005,"15min":0.97499999999999998},"min":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":1.198,"5min":1.298,"15min":1.298},"last":1.014}]},{"osd":2,"last update":"Sun Oct 1 10:16:12 2023","interfaces":[{"interface":"back","average":{"1min":0.877,"5min":0.86899999999999999,"15min":0.875},"min":{"1min":0.67000000000000004,"5min":0.57799999999999996,"15min":0.57799999999999996},"max":{"1min":1.121,"5min":1.2889999999999999,"15min":1.2889999999999999},"last":0.95899999999999996},{"interface":"front","average":{"1min":1.0069999999999999,"5min":0.93000000000000005,"15min":0.93700000000000006},"min":{"1min":0.623,"5min":0.57099999999999995,"15min":0.57099999999999995},"max":{"1min":2.8149999999999999,"5min":2.8149999999999999,"15min":2.8149999999999999},"last":0.85099999999999998}]},{"osd":3,"last update":"Sun Oct 1 10:16:12 2023","interfaces":[{"interface":"back","average":{"1min":0.97599999999999998,"5min":0.94099999999999995,"15min":0.95799999999999996},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":1.583,"5min":1.583,"15min":1.583},"last":0.97999999999999998},{"interface":"front","average":{"1min":0.92400000000000004,"5min":0.90800000000000003,"15min":0.92100000000000004},"min":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"max":{"1min":1.113,"5min":1.266,"15min":1.266},"last":0.88400000000000001}]},{"osd":4,"last update":"Sun Oct 1 10:16:12 2023","interfaces":[{"interface":"back","average":{"1min":0.93799999999999994,"5min":0.88200000000000001,"15min":0.875},"min":{"1min":0.49199999999999999,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.4890000000000001,"5min":1.4890000000000001,"15min":1.4890000000000001},"last":1.0920000000000001},{"interface":"front","average":{"1min":1.0129999999999999,"5min":0.93100000000000005,"15min":0.91000000000000003},"min":{"1min":0.55300000000000005,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":1.2070000000000001,"5min":1.349,"15min":1.349},"last":0.91500000000000004}]},{"osd":5,"last update":"Sun Oct 1 10:16:12 2023","interfaces":[{"interface":"back","average":{"1min":0.96799999999999997,"5min":0.93700000000000006,"15min":0.91100000000000003},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":1.5629999999999999,"5min":1.5629999999999999,"15min":1.5629999999999999},"last":0.93600000000000005},{"interface":"front","average":{"1min":1.0049999999999999,"5min":0.98199999999999998,"15min":0.97499999999999998},"min":{"1min":0.51100000000000001,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":1.6639999999999999,"5min":1.6639999999999999,"15min":1.6639999999999999},"last":1.1240000000000001}]},{"osd":7,"last update":"Sun Oct 1 10:16:53 2023","interfaces":[{"interface":"back","average":{"1min":1.1020000000000001,"5min":1.077,"15min":1.081},"min":{"1min":0.72899999999999998,"5min":0.59199999999999997,"15min":0.59199999999999997},"max":{"1min":1.706,"5min":1.7250000000000001,"15min":1.7250000000000001},"last":1.169},{"interface":"front","average":{"1min":1.085,"5min":1.0509999999999999,"15min":1.0429999999999999},"min":{"1min":0.69399999999999995,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":1.4359999999999999,"5min":1.5,"15min":1.5},"last":1.03}]}]},{"osd":1,"up_from":13,"seq":55834574953,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297708,"kb_used_data":612,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447444,"statfs":{"total":95995035648,"available":95690182656,"internally_reserved":0,"allocated":626688,"data_stored":269948,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:16:47 2023","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.64800000000000002,"15min":0.58099999999999996},"min":{"1min":0.39700000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.99299999999999999,"5min":1.698,"15min":1.698},"last":0.90500000000000003},{"interface":"front","average":{"1min":0.65100000000000002,"5min":0.70399999999999996,"15min":0.60499999999999998},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.99099999999999999,"5min":1.8100000000000001,"15min":1.8100000000000001},"last":1.3140000000000001}]},{"osd":2,"last update":"Sun Oct 1 10:16:29 2023","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":0.68500000000000005,"15min":0.64500000000000002},"min":{"1min":0.41299999999999998,"5min":0.376,"15min":0.33600000000000002},"max":{"1min":1.0449999999999999,"5min":1.1120000000000001,"15min":1.1120000000000001},"last":1.0589999999999999},{"interface":"front","average":{"1min":0.72399999999999998,"5min":0.76500000000000001,"15min":0.68899999999999995},"min":{"1min":0.50900000000000001,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.94699999999999995,"5min":1.5009999999999999,"15min":1.5009999999999999},"last":1.087}]},{"osd":3,"last update":"Sun Oct 1 10:16:06 2023","interfaces":[{"interface":"back","average":{"1min":0.71899999999999997,"5min":0.81399999999999995,"15min":0.81000000000000005},"min":{"1min":0.46300000000000002,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":1.0109999999999999,"5min":2.5059999999999998,"15min":2.5059999999999998},"last":0.94399999999999995},{"interface":"front","average":{"1min":0.72199999999999998,"5min":0.76200000000000001,"15min":0.77700000000000002},"min":{"1min":0.42499999999999999,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":1.0940000000000001,"5min":2.153,"15min":2.153},"last":1.1519999999999999}]},{"osd":4,"last update":"Sun Oct 1 10:16:50 2023","interfaces":[{"interface":"back","average":{"1min":0.90300000000000002,"5min":0.95799999999999996,"15min":0.96799999999999997},"min":{"1min":0.51100000000000001,"5min":0.505,"15min":0.505},"max":{"1min":1.1659999999999999,"5min":1.6759999999999999,"15min":1.6759999999999999},"last":1.2270000000000001},{"interface":"front","average":{"1min":0.96299999999999997,"5min":0.98599999999999999,"15min":0.97299999999999998},"min":{"1min":0.69199999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"max":{"1min":1.2050000000000001,"5min":1.804,"15min":1.804},"last":1.0249999999999999}]},{"osd":5,"last update":"Sun Oct 1 10:16:29 2023","interfaces":[{"interface":"back","average":{"1min":0.95999999999999996,"5min":0.99399999999999999,"15min":1.046},"min":{"1min":0.64500000000000002,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":1.839,"5min":1.839,"15min":1.839},"last":1.2010000000000001},{"interface":"front","average":{"1min":1.0189999999999999,"5min":1.0429999999999999,"15min":1.056},"min":{"1min":0.81200000000000006,"5min":0.64900000000000002,"15min":0.64900000000000002},"max":{"1min":1.264,"5min":1.863,"15min":1.863},"last":1.3680000000000001}]},{"osd":6,"last update":"Sun Oct 1 10:16:06 2023","interfaces":[{"interface":"back","average":{"1min":0.996,"5min":0.98499999999999999,"15min":0.96699999999999997},"min":{"1min":0.69199999999999995,"5min":0.61899999999999999,"15min":0.61899999999999999},"max":{"1min":2.1000000000000001,"5min":2.1000000000000001,"15min":2.1000000000000001},"last":1.27},{"interface":"front","average":{"1min":0.99399999999999999,"5min":1.048,"15min":1.056},"min":{"1min":0.753,"5min":0.68300000000000005,"15min":0.68300000000000005},"max":{"1min":1.3089999999999999,"5min":2.0510000000000002,"15min":2.0510000000000002},"last":1.1859999999999999}]},{"osd":7,"last update":"Sun Oct 1 10:16:50 2023","interfaces":[{"interface":"back","average":{"1min":0.97499999999999998,"5min":1.02,"15min":1.0009999999999999},"min":{"1min":0.72199999999999998,"5min":0.58699999999999997,"15min":0.58699999999999997},"max":{"1min":1.3,"5min":2.0230000000000001,"15min":2.0230000000000001},"last":1.127},{"interface":"front","average":{"1min":1.0089999999999999,"5min":1.0389999999999999,"15min":1.0609999999999999},"min":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"max":{"1min":1.6040000000000001,"5min":1.623,"15min":1.623},"last":1.292}]}]},{"osd":0,"up_from":8,"seq":34359738482,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297716,"kb_used_data":620,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447436,"statfs":{"total":95995035648,"available":95690174464,"internally_reserved":0,"allocated":634880,"data_stored":270524,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 10:16:47 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.66900000000000004,"15min":0.58099999999999996},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.83299999999999996,"5min":1.603,"15min":1.603},"last":0.69399999999999995},{"interface":"front","average":{"1min":0.622,"5min":0.68700000000000006,"15min":0.63600000000000001},"min":{"1min":0.374,"5min":0.312,"15min":0.312},"max":{"1min":0.94099999999999995,"5min":1.0369999999999999,"15min":1.0780000000000001},"last":0.84299999999999997}]},{"osd":2,"last update":"Sun Oct 1 10:16:34 2023","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":0.72399999999999998,"15min":0.66400000000000003},"min":{"1min":0.41999999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.88700000000000001,"5min":1.169,"15min":1.2050000000000001},"last":0.73699999999999999},{"interface":"front","average":{"1min":0.626,"5min":0.68300000000000005,"15min":0.64700000000000002},"min":{"1min":0.46000000000000002,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.94399999999999995,"5min":1.149,"15min":1.149},"last":0.79400000000000004}]},{"osd":3,"last update":"Sun Oct 1 10:16:09 2023","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":0.71899999999999997,"15min":0.68300000000000005},"min":{"1min":0.40200000000000002,"5min":0.40100000000000002,"15min":0.33000000000000002},"max":{"1min":0.82899999999999996,"5min":1.0960000000000001,"15min":1.0960000000000001},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.76100000000000001,"5min":0.82399999999999995,"15min":0.78200000000000003},"min":{"1min":0.58299999999999996,"5min":0.375,"15min":0.375},"max":{"1min":1.1220000000000001,"5min":1.28,"15min":1.28},"last":1.0700000000000001}]},{"osd":4,"last update":"Sun Oct 1 10:16:47 2023","interfaces":[{"interface":"back","average":{"1min":0.88900000000000001,"5min":0.90500000000000003,"15min":0.91000000000000003},"min":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":1.1279999999999999,"5min":1.476,"15min":1.476},"last":1.0469999999999999},{"interface":"front","average":{"1min":0.94799999999999995,"5min":1.0169999999999999,"15min":0.96899999999999997},"min":{"1min":0.66800000000000004,"5min":0.66200000000000003,"15min":0.60799999999999998},"max":{"1min":1.276,"5min":1.6499999999999999,"15min":1.6499999999999999},"last":1.2529999999999999}]},{"osd":5,"last update":"Sun Oct 1 10:16:34 2023","interfaces":[{"interface":"back","average":{"1min":0.98399999999999999,"5min":1.0349999999999999,"15min":1.0309999999999999},"min":{"1min":0.63100000000000001,"5min":0.60599999999999998,"15min":0.60599999999999998},"max":{"1min":1.637,"5min":1.718,"15min":1.718},"last":1.179},{"interface":"front","average":{"1min":1.0289999999999999,"5min":1.046,"15min":1.036},"min":{"1min":0.72099999999999997,"5min":0.70999999999999996,"15min":0.70999999999999996},"max":{"1min":1.548,"5min":1.548,"15min":1.548},"last":0.96499999999999997}]},{"osd":6,"last update":"Sun Oct 1 10:16:15 2023","interfaces":[{"interface":"back","average":{"1min":0.96599999999999997,"5min":1.046,"15min":1.0780000000000001},"min":{"1min":0.67000000000000004,"5min":0.58099999999999996,"15min":0.58099999999999996},"max":{"1min":1.3480000000000001,"5min":1.5669999999999999,"15min":1.5669999999999999},"last":1.306},{"interface":"front","average":{"1min":0.98599999999999999,"5min":1.056,"15min":1.0780000000000001},"min":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"max":{"1min":1.3340000000000001,"5min":1.6240000000000001,"15min":1.6240000000000001},"last":1.024}]},{"osd":7,"last update":"Sun Oct 1 10:16:47 2023","interfaces":[{"interface":"back","average":{"1min":0.93700000000000006,"5min":0.999,"15min":1.038},"min":{"1min":0.66200000000000003,"5min":0.58799999999999997,"15min":0.58799999999999997},"max":{"1min":1.145,"5min":1.4299999999999999,"15min":1.4299999999999999},"last":1.278},{"interface":"front","average":{"1min":0.99099999999999999,"5min":1.115,"15min":1.157},"min":{"1min":0.64300000000000002,"5min":0.53500000000000003,"15min":0.53500000000000003},"max":{"1min":1.3979999999999999,"5min":1.6539999999999999,"15min":1.6539999999999999},"last":1.3480000000000001}]}]},{"osd":2,"up_from":18,"seq":77309411423,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297716,"kb_used_data":620,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447436,"statfs":{"total":95995035648,"available":95690174464,"internally_reserved":0,"allocated":634880,"data_stored":270536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:16:31 2023","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":0.66300000000000003,"15min":0.60299999999999998},"min":{"1min":0.46800000000000003,"5min":0.38400000000000001,"15min":0.318},"max":{"1min":1.0649999999999999,"5min":1.077,"15min":1.077},"last":0.68799999999999994},{"interface":"front","average":{"1min":0.83999999999999997,"5min":0.754,"15min":0.66900000000000004},"min":{"1min":0.53000000000000003,"5min":0.314,"15min":0.314},"max":{"1min":2.927,"5min":2.927,"15min":2.927},"last":0.82799999999999996}]},{"osd":1,"last update":"Sun Oct 1 10:16:31 2023","interfaces":[{"interface":"back","average":{"1min":0.71199999999999997,"5min":0.71599999999999997,"15min":0.65200000000000002},"min":{"1min":0.52300000000000002,"5min":0.37,"15min":0.36599999999999999},"max":{"1min":0.98899999999999999,"5min":1.2529999999999999,"15min":1.2529999999999999},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.77300000000000002,"5min":0.77700000000000002,"15min":0.68300000000000005},"min":{"1min":0.46899999999999997,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":1.0429999999999999,"5min":1.4019999999999999,"15min":1.4019999999999999},"last":0.58599999999999997}]},{"osd":3,"last update":"Sun Oct 1 10:16:09 2023","interfaces":[{"interface":"back","average":{"1min":0.746,"5min":0.76500000000000001,"15min":0.72899999999999998},"min":{"1min":0.45900000000000002,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":1.069,"5min":1.3029999999999999,"15min":1.3029999999999999},"last":0.89300000000000002},{"interface":"front","average":{"1min":0.83099999999999996,"5min":0.85399999999999998,"15min":0.80800000000000005},"min":{"1min":0.51200000000000001,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":1.0820000000000001,"5min":1.4710000000000001,"15min":1.4710000000000001},"last":0.73599999999999999}]},{"osd":4,"last update":"Sun Oct 1 10:16:54 2023","interfaces":[{"interface":"back","average":{"1min":0.99299999999999999,"5min":0.999,"15min":0.92500000000000004},"min":{"1min":0.68200000000000005,"5min":0.64700000000000002,"15min":0.64700000000000002},"max":{"1min":1.238,"5min":1.425,"15min":1.425},"last":1.01},{"interface":"front","average":{"1min":1.0589999999999999,"5min":1.0640000000000001,"15min":0.98299999999999998},"min":{"1min":0.68799999999999994,"5min":0.66200000000000003,"15min":0.66200000000000003},"max":{"1min":1.3520000000000001,"5min":1.6679999999999999,"15min":1.6679999999999999},"last":0.98999999999999999}]},{"osd":5,"last update":"Sun Oct 1 10:16:36 2023","interfaces":[{"interface":"back","average":{"1min":1.0649999999999999,"5min":1.0409999999999999,"15min":1.0089999999999999},"min":{"1min":0.71399999999999997,"5min":0.629,"15min":0.629},"max":{"1min":1.4239999999999999,"5min":1.647,"15min":1.647},"last":0.93100000000000005},{"interface":"front","average":{"1min":1.032,"5min":1.024,"15min":1.0129999999999999},"min":{"1min":0.621,"5min":0.621,"15min":0.621},"max":{"1min":1.373,"5min":1.54,"15min":1.54},"last":0.86599999999999999}]},{"osd":6,"last update":"Sun Oct 1 10:16:09 2023","interfaces":[{"interface":"back","average":{"1min":1.028,"5min":1.0740000000000001,"15min":1.1020000000000001},"min":{"1min":0.625,"5min":0.61799999999999999,"15min":0.61799999999999999},"max":{"1min":1.4910000000000001,"5min":1.532,"15min":1.532},"last":1.179},{"interface":"front","average":{"1min":1.1559999999999999,"5min":1.1399999999999999,"15min":1.155},"min":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"max":{"1min":1.5169999999999999,"5min":1.647,"15min":1.647},"last":1.032}]},{"osd":7,"last update":"Sun Oct 1 10:16:54 2023","interfaces":[{"interface":"back","average":{"1min":1.0409999999999999,"5min":1.0700000000000001,"15min":1.0660000000000001},"min":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"max":{"1min":1.3520000000000001,"5min":1.5409999999999999,"15min":1.5409999999999999},"last":1.0960000000000001},{"interface":"front","average":{"1min":1.0980000000000001,"5min":1.1220000000000001,"15min":1.143},"min":{"1min":0.76800000000000002,"5min":0.58099999999999996,"15min":0.58099999999999996},"max":{"1min":1.4930000000000001,"5min":1.5589999999999999,"15min":1.5589999999999999},"last":1.2749999999999999}]}]},{"osd":3,"up_from":25,"seq":107374182488,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297712,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447440,"statfs":{"total":95995035648,"available":95690178560,"internally_reserved":0,"allocated":630784,"data_stored":270038,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:16:21 2023","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":0.65900000000000003,"15min":0.63},"min":{"1min":0.502,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.80600000000000005,"5min":1.097,"15min":1.097},"last":0.75900000000000001},{"interface":"front","average":{"1min":0.71999999999999997,"5min":0.72399999999999998,"15min":0.71799999999999997},"min":{"1min":0.53600000000000003,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.997,"5min":1.05,"15min":1.05},"last":0.38700000000000001}]},{"osd":1,"last update":"Sun Oct 1 10:16:21 2023","interfaces":[{"interface":"back","average":{"1min":0.745,"5min":0.73999999999999999,"15min":0.75},"min":{"1min":0.55000000000000004,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":1.018,"5min":1.2170000000000001,"15min":1.224},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.70699999999999996,"5min":0.72199999999999998,"15min":0.72599999999999998},"min":{"1min":0.51900000000000002,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.99399999999999999,"5min":1.1439999999999999,"15min":1.1439999999999999},"last":0.66700000000000004}]},{"osd":2,"last update":"Sun Oct 1 10:16:21 2023","interfaces":[{"interface":"back","average":{"1min":0.71999999999999997,"5min":0.75900000000000001,"15min":0.77400000000000002},"min":{"1min":0.53400000000000003,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":1.0489999999999999,"5min":1.133,"15min":1.133},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.77300000000000002,"5min":0.77600000000000002,"15min":0.77800000000000002},"min":{"1min":0.59999999999999998,"5min":0.318,"15min":0.318},"max":{"1min":0.91400000000000003,"5min":1.3500000000000001,"15min":1.3500000000000001},"last":0.85499999999999998}]},{"osd":4,"last update":"Sun Oct 1 10:16:58 2023","interfaces":[{"interface":"back","average":{"1min":0.91400000000000003,"5min":0.93200000000000005,"15min":0.91500000000000004},"min":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"max":{"1min":1.2869999999999999,"5min":1.2869999999999999,"15min":1.2869999999999999},"last":1.0109999999999999},{"interface":"front","average":{"1min":0.93100000000000005,"5min":0.95699999999999996,"15min":0.95499999999999996},"min":{"1min":0.69599999999999995,"5min":0.51100000000000001,"15min":0.51100000000000001},"max":{"1min":1.3580000000000001,"5min":1.4710000000000001,"15min":1.4710000000000001},"last":0.81100000000000005}]},{"osd":5,"last update":"Sun Oct 1 10:16:37 2023","interfaces":[{"interface":"back","average":{"1min":0.998,"5min":1.0049999999999999,"15min":1.026},"min":{"1min":0.752,"5min":0.52700000000000002,"15min":0.52700000000000002},"max":{"1min":1.3149999999999999,"5min":1.603,"15min":1.603},"last":0.94799999999999995},{"interface":"front","average":{"1min":1.0029999999999999,"5min":1.04,"15min":1.0369999999999999},"min":{"1min":0.72199999999999998,"5min":0.68300000000000005,"15min":0.68300000000000005},"max":{"1min":1.3560000000000001,"5min":1.4330000000000001,"15min":1.4330000000000001},"last":0.82199999999999995}]},{"osd":6,"last update":"Sun Oct 1 10:16:09 2023","interfaces":[{"interface":"back","average":{"1min":1.109,"5min":1.0680000000000001,"15min":1.091},"min":{"1min":0.83099999999999996,"5min":0.69799999999999995,"15min":0.69799999999999995},"max":{"1min":1.3520000000000001,"5min":1.4970000000000001,"15min":1.4970000000000001},"last":0.876},{"interface":"front","average":{"1min":1.0900000000000001,"5min":1.0549999999999999,"15min":1.083},"min":{"1min":0.89800000000000002,"5min":0.75700000000000001,"15min":0.75700000000000001},"max":{"1min":1.383,"5min":1.399,"15min":1.399},"last":1.246}]},{"osd":7,"last update":"Sun Oct 1 10:16:52 2023","interfaces":[{"interface":"back","average":{"1min":1.0780000000000001,"5min":1.115,"15min":1.165},"min":{"1min":0.76300000000000001,"5min":0.629,"15min":0.629},"max":{"1min":1.337,"5min":1.46,"15min":1.46},"last":1.194},{"interface":"front","average":{"1min":1.0429999999999999,"5min":1.119,"15min":1.1910000000000001},"min":{"1min":0.61899999999999999,"5min":0.56899999999999995,"15min":0.56899999999999995},"max":{"1min":1.4770000000000001,"5min":1.661,"15min":1.661},"last":0.83699999999999997}]}]},{"osd":4,"up_from":30,"seq":128849018960,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297732,"kb_used_data":636,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447420,"statfs":{"total":95995035648,"available":95690158080,"internally_reserved":0,"allocated":651264,"data_stored":271630,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:16:52 2023","interfaces":[{"interface":"back","average":{"1min":0.86699999999999999,"5min":0.82999999999999996,"15min":0.83699999999999997},"min":{"1min":0.59199999999999997,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":1.1499999999999999,"5min":1.1499999999999999,"15min":1.22},"last":0.95499999999999996},{"interface":"front","average":{"1min":0.92400000000000004,"5min":0.879,"15min":0.874},"min":{"1min":0.69999999999999996,"5min":0.52100000000000002,"15min":0.52100000000000002},"max":{"1min":1.2030000000000001,"5min":1.873,"15min":1.873},"last":1.083}]},{"osd":1,"last update":"Sun Oct 1 10:16:52 2023","interfaces":[{"interface":"back","average":{"1min":0.96099999999999997,"5min":0.86699999999999999,"15min":0.874},"min":{"1min":0.65600000000000003,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":1.298,"5min":1.484,"15min":1.484},"last":0.79500000000000004},{"interface":"front","average":{"1min":1.0029999999999999,"5min":0.91300000000000003,"15min":0.89900000000000002},"min":{"1min":0.626,"5min":0.626,"15min":0.626},"max":{"1min":1.214,"5min":1.9650000000000001,"15min":1.9650000000000001},"last":1.034}]},{"osd":2,"last update":"Sun Oct 1 10:16:52 2023","interfaces":[{"interface":"back","average":{"1min":0.98899999999999999,"5min":0.94099999999999995,"15min":0.90600000000000003},"min":{"1min":0.63600000000000001,"5min":0.58899999999999997,"15min":0.58899999999999997},"max":{"1min":1.228,"5min":2,"15min":2},"last":0.98199999999999998},{"interface":"front","average":{"1min":0.83999999999999997,"5min":0.874,"15min":0.878},"min":{"1min":0.56999999999999995,"5min":0.53600000000000003,"15min":0.53600000000000003},"max":{"1min":1.024,"5min":2.335,"15min":2.335},"last":0.82899999999999996}]},{"osd":3,"last update":"Sun Oct 1 10:16:52 2023","interfaces":[{"interface":"back","average":{"1min":1.0109999999999999,"5min":0.97599999999999998,"15min":0.96999999999999997},"min":{"1min":0.73199999999999998,"5min":0.67100000000000004,"15min":0.56799999999999995},"max":{"1min":1.4019999999999999,"5min":2.4169999999999998,"15min":2.4169999999999998},"last":1.0449999999999999},{"interface":"front","average":{"1min":0.98699999999999999,"5min":0.96599999999999997,"15min":0.95999999999999996},"min":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"max":{"1min":1.1599999999999999,"5min":1.585,"15min":1.585},"last":1.022}]},{"osd":5,"last update":"Sun Oct 1 10:16:32 2023","interfaces":[{"interface":"back","average":{"1min":0.98599999999999999,"5min":0.90600000000000003,"15min":0.81899999999999995},"min":{"1min":0.68600000000000005,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":1.923,"5min":1.923,"15min":1.923},"last":0.77100000000000002},{"interface":"front","average":{"1min":1.095,"5min":0.98099999999999998,"15min":0.88100000000000001},"min":{"1min":0.79600000000000004,"5min":0.51000000000000001,"15min":0.51000000000000001},"max":{"1min":2.3780000000000001,"5min":2.3780000000000001,"15min":2.3780000000000001},"last":1.1459999999999999}]},{"osd":6,"last update":"Sun Oct 1 10:16:16 2023","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":0.871,"15min":0.83399999999999996},"min":{"1min":0.625,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":1.232,"5min":1.3360000000000001,"15min":1.3360000000000001},"last":0.875},{"interface":"front","average":{"1min":1.085,"5min":0.97299999999999998,"15min":0.93200000000000005},"min":{"1min":0.75900000000000001,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":2.8599999999999999,"5min":2.8599999999999999,"15min":2.8599999999999999},"last":0.91000000000000003}]},{"osd":7,"last update":"Sun Oct 1 10:16:52 2023","interfaces":[{"interface":"back","average":{"1min":0.98999999999999999,"5min":0.97199999999999998,"15min":0.95099999999999996},"min":{"1min":0.80700000000000005,"5min":0.64400000000000002,"15min":0.64400000000000002},"max":{"1min":1.3240000000000001,"5min":2.4550000000000001,"15min":2.4550000000000001},"last":0.96999999999999997},{"interface":"front","average":{"1min":1.0820000000000001,"5min":0.98999999999999999,"15min":0.93799999999999994},"min":{"1min":0.84799999999999998,"5min":0.495,"15min":0.495},"max":{"1min":1.3620000000000001,"5min":1.3620000000000001,"15min":1.3620000000000001},"last":1.113}]}]},{"osd":5,"up_from":36,"seq":154618822726,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297712,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447440,"statfs":{"total":95995035648,"available":95690178560,"internally_reserved":0,"allocated":630784,"data_stored":270257,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:16:34 2023","interfaces":[{"interface":"back","average":{"1min":0.91000000000000003,"5min":0.877,"15min":0.89700000000000002},"min":{"1min":0.61499999999999999,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":1.1539999999999999,"5min":1.339,"15min":1.339},"last":0.71899999999999997},{"interface":"front","average":{"1min":0.87,"5min":0.84199999999999997,"15min":0.86899999999999999},"min":{"1min":0.64700000000000002,"5min":0.53100000000000003,"15min":0.53100000000000003},"max":{"1min":1.167,"5min":1.1679999999999999,"15min":1.1679999999999999},"last":0.67000000000000004}]},{"osd":1,"last update":"Sun Oct 1 10:16:34 2023","interfaces":[{"interface":"back","average":{"1min":0.98599999999999999,"5min":0.88400000000000001,"15min":0.86199999999999999},"min":{"1min":0.58999999999999997,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":1.282,"5min":1.282,"15min":1.282},"last":0.69799999999999995},{"interface":"front","average":{"1min":0.91800000000000004,"5min":0.90100000000000002,"15min":0.90900000000000003},"min":{"1min":0.63100000000000001,"5min":0.57699999999999996,"15min":0.57699999999999996},"max":{"1min":1.1879999999999999,"5min":1.266,"15min":1.266},"last":0.96099999999999997}]},{"osd":2,"last update":"Sun Oct 1 10:16:34 2023","interfaces":[{"interface":"back","average":{"1min":0.92000000000000004,"5min":0.879,"15min":0.86699999999999999},"min":{"1min":0.749,"5min":0.48999999999999999,"15min":0.48999999999999999},"max":{"1min":1.1739999999999999,"5min":1.1739999999999999,"15min":1.1739999999999999},"last":0.79500000000000004},{"interface":"front","average":{"1min":0.98999999999999999,"5min":0.93300000000000005,"15min":0.92400000000000004},"min":{"1min":0.60499999999999998,"5min":0.57199999999999995,"15min":0.57199999999999995},"max":{"1min":1.228,"5min":1.228,"15min":1.228},"last":0.82999999999999996}]},{"osd":3,"last update":"Sun Oct 1 10:16:34 2023","interfaces":[{"interface":"back","average":{"1min":0.97299999999999998,"5min":0.92200000000000004,"15min":0.92000000000000004},"min":{"1min":0.80000000000000004,"5min":0.60699999999999998,"15min":0.60699999999999998},"max":{"1min":1.2030000000000001,"5min":1.236,"15min":1.236},"last":0.91700000000000004},{"interface":"front","average":{"1min":0.94799999999999995,"5min":0.90900000000000003,"15min":0.90900000000000003},"min":{"1min":0.63700000000000001,"5min":0.58099999999999996,"15min":0.58099999999999996},"max":{"1min":1.196,"5min":1.2490000000000001,"15min":1.2490000000000001},"last":0.76600000000000001}]},{"osd":4,"last update":"Sun Oct 1 10:16:34 2023","interfaces":[{"interface":"back","average":{"1min":0.93100000000000005,"5min":0.84699999999999998,"15min":0.76200000000000001},"min":{"1min":0.60899999999999999,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":1.286,"5min":1.286,"15min":1.286},"last":0.873},{"interface":"front","average":{"1min":0.91900000000000004,"5min":0.93100000000000005,"15min":0.84099999999999997},"min":{"1min":0.50600000000000001,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":1.3069999999999999,"5min":1.431,"15min":1.431},"last":1.022}]},{"osd":6,"last update":"Sun Oct 1 10:16:11 2023","interfaces":[{"interface":"back","average":{"1min":0.95699999999999996,"5min":0.93000000000000005,"15min":0.93000000000000005},"min":{"1min":0.66500000000000004,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":1.24,"5min":1.333,"15min":1.333},"last":0.94599999999999995},{"interface":"front","average":{"1min":0.92100000000000004,"5min":0.91300000000000003,"15min":0.89900000000000002},"min":{"1min":0.70099999999999996,"5min":0.434,"15min":0.434},"max":{"1min":1.117,"5min":1.2649999999999999,"15min":1.2649999999999999},"last":1.052}]},{"osd":7,"last update":"Sun Oct 1 10:16:57 2023","interfaces":[{"interface":"back","average":{"1min":1.0129999999999999,"5min":0.95599999999999996,"15min":0.92200000000000004},"min":{"1min":0.81499999999999995,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":1.2569999999999999,"5min":1.3480000000000001,"15min":1.3480000000000001},"last":1.107},{"interface":"front","average":{"1min":1.056,"5min":1.0349999999999999,"15min":1.0289999999999999},"min":{"1min":0.83099999999999996,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":1.2809999999999999,"5min":1.3959999999999999,"15min":1.3959999999999999},"last":1.2050000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T10:17:07.408 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T10:17:07.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:17:07 smithi078 bash[27336]: audit 2023-10-01T10:17:06.565656+0000 mgr.y (mgr.14427) 148 : audit [DBG] from='client.24683 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:17:07.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:17:07 smithi078 bash[33821]: audit 2023-10-01T10:17:06.565656+0000 mgr.y (mgr.14427) 148 : audit [DBG] from='client.24683 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:17:07.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:17:07 smithi173 bash[27612]: audit 2023-10-01T10:17:06.565656+0000 mgr.y (mgr.14427) 148 : audit [DBG] from='client.24683 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:17:08.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:17:08 smithi078 bash[27336]: cluster 2023-10-01T10:17:07.222774+0000 mgr.y (mgr.14427) 149 : cluster [DBG] pgmap v129: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:08.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:17:08 smithi078 bash[33821]: cluster 2023-10-01T10:17:07.222774+0000 mgr.y (mgr.14427) 149 : cluster [DBG] pgmap v129: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:08.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:17:08 smithi173 bash[27612]: cluster 2023-10-01T10:17:07.222774+0000 mgr.y (mgr.14427) 149 : cluster [DBG] pgmap v129: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:10.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:17:10 smithi078 bash[27336]: cluster 2023-10-01T10:17:09.223895+0000 mgr.y (mgr.14427) 150 : cluster [DBG] pgmap v130: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:10.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:17:10 smithi078 bash[33821]: cluster 2023-10-01T10:17:09.223895+0000 mgr.y (mgr.14427) 150 : cluster [DBG] pgmap v130: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:10.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:17:10 smithi173 bash[27612]: cluster 2023-10-01T10:17:09.223895+0000 mgr.y (mgr.14427) 150 : cluster [DBG] pgmap v130: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:12.224 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:17:12.256 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:17:11 smithi078 bash[27336]: cluster 2023-10-01T10:17:11.224646+0000 mgr.y (mgr.14427) 151 : cluster [DBG] pgmap v131: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:12.256 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:17:11 smithi078 bash[33821]: cluster 2023-10-01T10:17:11.224646+0000 mgr.y (mgr.14427) 151 : cluster [DBG] pgmap v131: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:12.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:17:11 smithi173 bash[27612]: cluster 2023-10-01T10:17:11.224646+0000 mgr.y (mgr.14427) 151 : cluster [DBG] pgmap v131: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:14.586 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:17:14 smithi078 bash[33821]: cluster 2023-10-01T10:17:13.225629+0000 mgr.y (mgr.14427) 152 : cluster [DBG] pgmap v132: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:14.586 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:17:14 smithi078 bash[27336]: cluster 2023-10-01T10:17:13.225629+0000 mgr.y (mgr.14427) 152 : cluster [DBG] pgmap v132: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:14.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:17:14 smithi173 bash[27612]: cluster 2023-10-01T10:17:13.225629+0000 mgr.y (mgr.14427) 152 : cluster [DBG] pgmap v132: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:15.205 INFO:teuthology.orchestra.run.smithi078.stdout: 2023-10-01T10:17:15.207 INFO:teuthology.orchestra.run.smithi078.stderr:dumped all 2023-10-01T10:17:15.415 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:17:15 smithi078 bash[27601]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:17:15] "GET /metrics HTTP/1.1" 200 191376 "" "Prometheus/2.33.4" 2023-10-01T10:17:15.621 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:17:15 smithi173 bash[28473]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:17:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T10:17:15.968 INFO:teuthology.orchestra.run.smithi078.stdout:{"pg_ready":true,"pg_map":{"version":132,"stamp":"2023-10-01T10:17:13.224933+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2381752,"kb_used_data":4984,"kb_used_omap":0,"kb_used_meta":2376704,"kb_avail":747579464,"statfs":{"total":767960285184,"available":765521371136,"internally_reserved":0,"allocated":5103616,"data_stored":2165872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":2433744896},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004931"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.771834+0000","last_change":"2023-10-01T10:15:40.316537+0000","last_active":"2023-10-01T10:16:17.771834+0000","last_peered":"2023-10-01T10:16:17.771834+0000","last_clean":"2023-10-01T10:16:17.771834+0000","last_became_active":"2023-10-01T10:15:40.316156+0000","last_became_peered":"2023-10-01T10:15:40.316156+0000","last_unstale":"2023-10-01T10:16:17.771834+0000","last_undegraded":"2023-10-01T10:16:17.771834+0000","last_fullsized":"2023-10-01T10:16:17.771834+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.290543+0000","last_change":"2023-10-01T10:15:39.289865+0000","last_active":"2023-10-01T10:16:17.290543+0000","last_peered":"2023-10-01T10:16:17.290543+0000","last_clean":"2023-10-01T10:16:17.290543+0000","last_became_active":"2023-10-01T10:15:39.289486+0000","last_became_peered":"2023-10-01T10:15:39.289486+0000","last_unstale":"2023-10-01T10:16:17.290543+0000","last_undegraded":"2023-10-01T10:16:17.290543+0000","last_fullsized":"2023-10-01T10:16:17.290543+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":4,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.771556+0000","last_change":"2023-10-01T10:15:40.306898+0000","last_active":"2023-10-01T10:16:17.771556+0000","last_peered":"2023-10-01T10:16:17.771556+0000","last_clean":"2023-10-01T10:16:17.771556+0000","last_became_active":"2023-10-01T10:15:40.306060+0000","last_became_peered":"2023-10-01T10:15:40.306060+0000","last_unstale":"2023-10-01T10:16:17.771556+0000","last_undegraded":"2023-10-01T10:16:17.771556+0000","last_fullsized":"2023-10-01T10:16:17.771556+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.773376+0000","last_change":"2023-10-01T10:15:39.291017+0000","last_active":"2023-10-01T10:16:17.773376+0000","last_peered":"2023-10-01T10:16:17.773376+0000","last_clean":"2023-10-01T10:16:17.773376+0000","last_became_active":"2023-10-01T10:15:39.289792+0000","last_became_peered":"2023-10-01T10:15:39.289792+0000","last_unstale":"2023-10-01T10:16:17.773376+0000","last_undegraded":"2023-10-01T10:16:17.773376+0000","last_fullsized":"2023-10-01T10:16:17.773376+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.772307+0000","last_change":"2023-10-01T10:15:40.310483+0000","last_active":"2023-10-01T10:16:17.772307+0000","last_peered":"2023-10-01T10:16:17.772307+0000","last_clean":"2023-10-01T10:16:17.772307+0000","last_became_active":"2023-10-01T10:15:40.310173+0000","last_became_peered":"2023-10-01T10:15:40.310173+0000","last_unstale":"2023-10-01T10:16:17.772307+0000","last_undegraded":"2023-10-01T10:16:17.772307+0000","last_fullsized":"2023-10-01T10:16:17.772307+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.291810+0000","last_change":"2023-10-01T10:15:39.296653+0000","last_active":"2023-10-01T10:16:17.291810+0000","last_peered":"2023-10-01T10:16:17.291810+0000","last_clean":"2023-10-01T10:16:17.291810+0000","last_became_active":"2023-10-01T10:15:39.296347+0000","last_became_peered":"2023-10-01T10:15:39.296347+0000","last_unstale":"2023-10-01T10:16:17.291810+0000","last_undegraded":"2023-10-01T10:16:17.291810+0000","last_fullsized":"2023-10-01T10:16:17.291810+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.772977+0000","last_change":"2023-10-01T10:15:40.317311+0000","last_active":"2023-10-01T10:16:17.772977+0000","last_peered":"2023-10-01T10:16:17.772977+0000","last_clean":"2023-10-01T10:16:17.772977+0000","last_became_active":"2023-10-01T10:15:40.315986+0000","last_became_peered":"2023-10-01T10:15:40.315986+0000","last_unstale":"2023-10-01T10:16:17.772977+0000","last_undegraded":"2023-10-01T10:16:17.772977+0000","last_fullsized":"2023-10-01T10:16:17.772977+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.291869+0000","last_change":"2023-10-01T10:15:39.301714+0000","last_active":"2023-10-01T10:16:17.291869+0000","last_peered":"2023-10-01T10:16:17.291869+0000","last_clean":"2023-10-01T10:16:17.291869+0000","last_became_active":"2023-10-01T10:15:39.301333+0000","last_became_peered":"2023-10-01T10:15:39.301333+0000","last_unstale":"2023-10-01T10:16:17.291869+0000","last_undegraded":"2023-10-01T10:16:17.291869+0000","last_fullsized":"2023-10-01T10:16:17.291869+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.770286+0000","last_change":"2023-10-01T10:15:40.311730+0000","last_active":"2023-10-01T10:16:17.770286+0000","last_peered":"2023-10-01T10:16:17.770286+0000","last_clean":"2023-10-01T10:16:17.770286+0000","last_became_active":"2023-10-01T10:15:40.311413+0000","last_became_peered":"2023-10-01T10:15:40.311413+0000","last_unstale":"2023-10-01T10:16:17.770286+0000","last_undegraded":"2023-10-01T10:16:17.770286+0000","last_fullsized":"2023-10-01T10:16:17.770286+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.289345+0000","last_change":"2023-10-01T10:15:39.294402+0000","last_active":"2023-10-01T10:16:17.289345+0000","last_peered":"2023-10-01T10:16:17.289345+0000","last_clean":"2023-10-01T10:16:17.289345+0000","last_became_active":"2023-10-01T10:15:39.294133+0000","last_became_peered":"2023-10-01T10:15:39.294133+0000","last_unstale":"2023-10-01T10:16:17.289345+0000","last_undegraded":"2023-10-01T10:16:17.289345+0000","last_fullsized":"2023-10-01T10:16:17.289345+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.772455+0000","last_change":"2023-10-01T10:15:40.318970+0000","last_active":"2023-10-01T10:16:17.772455+0000","last_peered":"2023-10-01T10:16:17.772455+0000","last_clean":"2023-10-01T10:16:17.772455+0000","last_became_active":"2023-10-01T10:15:40.318666+0000","last_became_peered":"2023-10-01T10:15:40.318666+0000","last_unstale":"2023-10-01T10:16:17.772455+0000","last_undegraded":"2023-10-01T10:16:17.772455+0000","last_fullsized":"2023-10-01T10:16:17.772455+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.289908+0000","last_change":"2023-10-01T10:15:39.290055+0000","last_active":"2023-10-01T10:16:17.289908+0000","last_peered":"2023-10-01T10:16:17.289908+0000","last_clean":"2023-10-01T10:16:17.289908+0000","last_became_active":"2023-10-01T10:15:39.289657+0000","last_became_peered":"2023-10-01T10:15:39.289657+0000","last_unstale":"2023-10-01T10:16:17.289908+0000","last_undegraded":"2023-10-01T10:16:17.289908+0000","last_fullsized":"2023-10-01T10:16:17.289908+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":17,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T10:15:41.421134+0000","last_change":"2023-10-01T10:13:44.666450+0000","last_active":"2023-10-01T10:15:41.421134+0000","last_peered":"2023-10-01T10:15:41.421134+0000","last_clean":"2023-10-01T10:15:41.421134+0000","last_became_active":"2023-10-01T10:13:44.665959+0000","last_became_peered":"2023-10-01T10:13:44.665959+0000","last_unstale":"2023-10-01T10:15:41.421134+0000","last_undegraded":"2023-10-01T10:15:41.421134+0000","last_fullsized":"2023-10-01T10:15:41.421134+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.771101+0000","last_change":"2023-10-01T10:15:40.317645+0000","last_active":"2023-10-01T10:16:17.771101+0000","last_peered":"2023-10-01T10:16:17.771101+0000","last_clean":"2023-10-01T10:16:17.771101+0000","last_became_active":"2023-10-01T10:15:40.317310+0000","last_became_peered":"2023-10-01T10:15:40.317310+0000","last_unstale":"2023-10-01T10:16:17.771101+0000","last_undegraded":"2023-10-01T10:16:17.771101+0000","last_fullsized":"2023-10-01T10:16:17.771101+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.767445+0000","last_change":"2023-10-01T10:13:44.825307+0000","last_active":"2023-10-01T10:16:17.767445+0000","last_peered":"2023-10-01T10:16:17.767445+0000","last_clean":"2023-10-01T10:16:17.767445+0000","last_became_active":"2023-10-01T10:13:44.824803+0000","last_became_peered":"2023-10-01T10:13:44.824803+0000","last_unstale":"2023-10-01T10:16:17.767445+0000","last_undegraded":"2023-10-01T10:16:17.767445+0000","last_fullsized":"2023-10-01T10:16:17.767445+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.773776+0000","last_change":"2023-10-01T10:15:39.299726+0000","last_active":"2023-10-01T10:16:17.773776+0000","last_peered":"2023-10-01T10:16:17.773776+0000","last_clean":"2023-10-01T10:16:17.773776+0000","last_became_active":"2023-10-01T10:15:39.299316+0000","last_became_peered":"2023-10-01T10:15:39.299316+0000","last_unstale":"2023-10-01T10:16:17.773776+0000","last_undegraded":"2023-10-01T10:16:17.773776+0000","last_fullsized":"2023-10-01T10:16:17.773776+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.289073+0000","last_change":"2023-10-01T10:15:40.314007+0000","last_active":"2023-10-01T10:16:17.289073+0000","last_peered":"2023-10-01T10:16:17.289073+0000","last_clean":"2023-10-01T10:16:17.289073+0000","last_became_active":"2023-10-01T10:15:40.313624+0000","last_became_peered":"2023-10-01T10:15:40.313624+0000","last_unstale":"2023-10-01T10:16:17.289073+0000","last_undegraded":"2023-10-01T10:16:17.289073+0000","last_fullsized":"2023-10-01T10:16:17.289073+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.767423+0000","last_change":"2023-10-01T10:15:39.298273+0000","last_active":"2023-10-01T10:16:17.767423+0000","last_peered":"2023-10-01T10:16:17.767423+0000","last_clean":"2023-10-01T10:16:17.767423+0000","last_became_active":"2023-10-01T10:15:39.297927+0000","last_became_peered":"2023-10-01T10:15:39.297927+0000","last_unstale":"2023-10-01T10:16:17.767423+0000","last_undegraded":"2023-10-01T10:16:17.767423+0000","last_fullsized":"2023-10-01T10:16:17.767423+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.777042+0000","last_change":"2023-10-01T10:13:44.827141+0000","last_active":"2023-10-01T10:16:17.777042+0000","last_peered":"2023-10-01T10:16:17.777042+0000","last_clean":"2023-10-01T10:16:17.777042+0000","last_became_active":"2023-10-01T10:13:44.826089+0000","last_became_peered":"2023-10-01T10:13:44.826089+0000","last_unstale":"2023-10-01T10:16:17.777042+0000","last_undegraded":"2023-10-01T10:16:17.777042+0000","last_fullsized":"2023-10-01T10:16:17.777042+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.290041+0000","last_change":"2023-10-01T10:15:40.313188+0000","last_active":"2023-10-01T10:16:17.290041+0000","last_peered":"2023-10-01T10:16:17.290041+0000","last_clean":"2023-10-01T10:16:17.290041+0000","last_became_active":"2023-10-01T10:15:40.312671+0000","last_became_peered":"2023-10-01T10:15:40.312671+0000","last_unstale":"2023-10-01T10:16:17.290041+0000","last_undegraded":"2023-10-01T10:16:17.290041+0000","last_fullsized":"2023-10-01T10:16:17.290041+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.289054+0000","last_change":"2023-10-01T10:15:39.295743+0000","last_active":"2023-10-01T10:16:17.289054+0000","last_peered":"2023-10-01T10:16:17.289054+0000","last_clean":"2023-10-01T10:16:17.289054+0000","last_became_active":"2023-10-01T10:15:39.295398+0000","last_became_peered":"2023-10-01T10:15:39.295398+0000","last_unstale":"2023-10-01T10:16:17.289054+0000","last_undegraded":"2023-10-01T10:16:17.289054+0000","last_fullsized":"2023-10-01T10:16:17.289054+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.776437+0000","last_change":"2023-10-01T10:13:44.825879+0000","last_active":"2023-10-01T10:16:17.776437+0000","last_peered":"2023-10-01T10:16:17.776437+0000","last_clean":"2023-10-01T10:16:17.776437+0000","last_became_active":"2023-10-01T10:13:44.825097+0000","last_became_peered":"2023-10-01T10:13:44.825097+0000","last_unstale":"2023-10-01T10:16:17.776437+0000","last_undegraded":"2023-10-01T10:16:17.776437+0000","last_fullsized":"2023-10-01T10:16:17.776437+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T10:15:41.421241+0000","last_change":"2023-10-01T10:15:40.316207+0000","last_active":"2023-10-01T10:15:41.421241+0000","last_peered":"2023-10-01T10:15:41.421241+0000","last_clean":"2023-10-01T10:15:41.421241+0000","last_became_active":"2023-10-01T10:15:40.315843+0000","last_became_peered":"2023-10-01T10:15:40.315843+0000","last_unstale":"2023-10-01T10:15:41.421241+0000","last_undegraded":"2023-10-01T10:15:41.421241+0000","last_fullsized":"2023-10-01T10:15:41.421241+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.771335+0000","last_change":"2023-10-01T10:15:39.293526+0000","last_active":"2023-10-01T10:16:17.771335+0000","last_peered":"2023-10-01T10:16:17.771335+0000","last_clean":"2023-10-01T10:16:17.771335+0000","last_became_active":"2023-10-01T10:15:39.293196+0000","last_became_peered":"2023-10-01T10:15:39.293196+0000","last_unstale":"2023-10-01T10:16:17.771335+0000","last_undegraded":"2023-10-01T10:16:17.771335+0000","last_fullsized":"2023-10-01T10:16:17.771335+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":22,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.776338+0000","last_change":"2023-10-01T10:12:47.477457+0000","last_active":"2023-10-01T10:16:17.776338+0000","last_peered":"2023-10-01T10:16:17.776338+0000","last_clean":"2023-10-01T10:16:17.776338+0000","last_became_active":"2023-10-01T10:12:47.476522+0000","last_became_peered":"2023-10-01T10:12:47.476522+0000","last_unstale":"2023-10-01T10:16:17.776338+0000","last_undegraded":"2023-10-01T10:16:17.776338+0000","last_fullsized":"2023-10-01T10:16:17.776338+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:09:17.221082+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:09:17.221082+0000","last_clean_scrub_stamp":"2023-10-01T10:09:17.221082+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.291528+0000","last_change":"2023-10-01T10:13:44.663992+0000","last_active":"2023-10-01T10:16:17.291528+0000","last_peered":"2023-10-01T10:16:17.291528+0000","last_clean":"2023-10-01T10:16:17.291528+0000","last_became_active":"2023-10-01T10:13:44.663669+0000","last_became_peered":"2023-10-01T10:13:44.663669+0000","last_unstale":"2023-10-01T10:16:17.291528+0000","last_undegraded":"2023-10-01T10:16:17.291528+0000","last_fullsized":"2023-10-01T10:16:17.291528+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.767341+0000","last_change":"2023-10-01T10:15:39.727816+0000","last_active":"2023-10-01T10:16:17.767341+0000","last_peered":"2023-10-01T10:16:17.767341+0000","last_clean":"2023-10-01T10:16:17.767341+0000","last_became_active":"2023-10-01T10:15:39.727202+0000","last_became_peered":"2023-10-01T10:15:39.727202+0000","last_unstale":"2023-10-01T10:16:17.767341+0000","last_undegraded":"2023-10-01T10:16:17.767341+0000","last_fullsized":"2023-10-01T10:16:17.767341+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.766733+0000","last_change":"2023-10-01T10:15:39.295490+0000","last_active":"2023-10-01T10:16:17.766733+0000","last_peered":"2023-10-01T10:16:17.766733+0000","last_clean":"2023-10-01T10:16:17.766733+0000","last_became_active":"2023-10-01T10:15:39.294900+0000","last_became_peered":"2023-10-01T10:15:39.294900+0000","last_unstale":"2023-10-01T10:16:17.766733+0000","last_undegraded":"2023-10-01T10:16:17.766733+0000","last_fullsized":"2023-10-01T10:16:17.766733+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.770202+0000","last_change":"2023-10-01T10:13:44.823885+0000","last_active":"2023-10-01T10:16:17.770202+0000","last_peered":"2023-10-01T10:16:17.770202+0000","last_clean":"2023-10-01T10:16:17.770202+0000","last_became_active":"2023-10-01T10:13:44.823314+0000","last_became_peered":"2023-10-01T10:13:44.823314+0000","last_unstale":"2023-10-01T10:16:17.770202+0000","last_undegraded":"2023-10-01T10:16:17.770202+0000","last_fullsized":"2023-10-01T10:16:17.770202+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.767203+0000","last_change":"2023-10-01T10:15:39.670779+0000","last_active":"2023-10-01T10:16:17.767203+0000","last_peered":"2023-10-01T10:16:17.767203+0000","last_clean":"2023-10-01T10:16:17.767203+0000","last_became_active":"2023-10-01T10:15:39.670136+0000","last_became_peered":"2023-10-01T10:15:39.670136+0000","last_unstale":"2023-10-01T10:16:17.767203+0000","last_undegraded":"2023-10-01T10:16:17.767203+0000","last_fullsized":"2023-10-01T10:16:17.767203+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.772485+0000","last_change":"2023-10-01T10:15:39.291273+0000","last_active":"2023-10-01T10:16:17.772485+0000","last_peered":"2023-10-01T10:16:17.772485+0000","last_clean":"2023-10-01T10:16:17.772485+0000","last_became_active":"2023-10-01T10:15:39.290959+0000","last_became_peered":"2023-10-01T10:15:39.290959+0000","last_unstale":"2023-10-01T10:16:17.772485+0000","last_undegraded":"2023-10-01T10:16:17.772485+0000","last_fullsized":"2023-10-01T10:16:17.772485+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.291428+0000","last_change":"2023-10-01T10:13:44.663534+0000","last_active":"2023-10-01T10:16:17.291428+0000","last_peered":"2023-10-01T10:16:17.291428+0000","last_clean":"2023-10-01T10:16:17.291428+0000","last_became_active":"2023-10-01T10:13:44.663014+0000","last_became_peered":"2023-10-01T10:13:44.663014+0000","last_unstale":"2023-10-01T10:16:17.291428+0000","last_undegraded":"2023-10-01T10:16:17.291428+0000","last_fullsized":"2023-10-01T10:16:17.291428+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.766946+0000","last_change":"2023-10-01T10:15:39.343985+0000","last_active":"2023-10-01T10:16:17.766946+0000","last_peered":"2023-10-01T10:16:17.766946+0000","last_clean":"2023-10-01T10:16:17.766946+0000","last_became_active":"2023-10-01T10:15:39.343351+0000","last_became_peered":"2023-10-01T10:15:39.343351+0000","last_unstale":"2023-10-01T10:16:17.766946+0000","last_undegraded":"2023-10-01T10:16:17.766946+0000","last_fullsized":"2023-10-01T10:16:17.766946+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.291766+0000","last_change":"2023-10-01T10:15:39.297597+0000","last_active":"2023-10-01T10:16:17.291766+0000","last_peered":"2023-10-01T10:16:17.291766+0000","last_clean":"2023-10-01T10:16:17.291766+0000","last_became_active":"2023-10-01T10:15:39.297235+0000","last_became_peered":"2023-10-01T10:15:39.297235+0000","last_unstale":"2023-10-01T10:16:17.291766+0000","last_undegraded":"2023-10-01T10:16:17.291766+0000","last_fullsized":"2023-10-01T10:16:17.291766+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":18,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.767016+0000","last_change":"2023-10-01T10:13:44.824140+0000","last_active":"2023-10-01T10:16:17.767016+0000","last_peered":"2023-10-01T10:16:17.767016+0000","last_clean":"2023-10-01T10:16:17.767016+0000","last_became_active":"2023-10-01T10:13:44.823462+0000","last_became_peered":"2023-10-01T10:13:44.823462+0000","last_unstale":"2023-10-01T10:16:17.767016+0000","last_undegraded":"2023-10-01T10:16:17.767016+0000","last_fullsized":"2023-10-01T10:16:17.767016+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":10,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-01T10:16:18.299852+0000","last_change":"2023-10-01T10:16:18.299852+0000","last_active":"2023-10-01T10:16:18.299852+0000","last_peered":"2023-10-01T10:16:18.299852+0000","last_clean":"2023-10-01T10:16:18.299852+0000","last_became_active":"2023-10-01T10:16:18.299391+0000","last_became_peered":"2023-10-01T10:16:18.299391+0000","last_unstale":"2023-10-01T10:16:18.299852+0000","last_undegraded":"2023-10-01T10:16:18.299852+0000","last_fullsized":"2023-10-01T10:16:18.299852+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.771364+0000","last_change":"2023-10-01T10:15:39.297452+0000","last_active":"2023-10-01T10:16:17.771364+0000","last_peered":"2023-10-01T10:16:17.771364+0000","last_clean":"2023-10-01T10:16:17.771364+0000","last_became_active":"2023-10-01T10:15:39.297021+0000","last_became_peered":"2023-10-01T10:15:39.297021+0000","last_unstale":"2023-10-01T10:16:17.771364+0000","last_undegraded":"2023-10-01T10:16:17.771364+0000","last_fullsized":"2023-10-01T10:16:17.771364+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.773153+0000","last_change":"2023-10-01T10:15:40.313526+0000","last_active":"2023-10-01T10:16:17.773153+0000","last_peered":"2023-10-01T10:16:17.773153+0000","last_clean":"2023-10-01T10:16:17.773153+0000","last_became_active":"2023-10-01T10:15:40.313153+0000","last_became_peered":"2023-10-01T10:15:40.313153+0000","last_unstale":"2023-10-01T10:16:17.773153+0000","last_undegraded":"2023-10-01T10:16:17.773153+0000","last_fullsized":"2023-10-01T10:16:17.773153+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.772536+0000","last_change":"2023-10-01T10:15:39.296739+0000","last_active":"2023-10-01T10:16:17.772536+0000","last_peered":"2023-10-01T10:16:17.772536+0000","last_clean":"2023-10-01T10:16:17.772536+0000","last_became_active":"2023-10-01T10:15:39.296339+0000","last_became_peered":"2023-10-01T10:15:39.296339+0000","last_unstale":"2023-10-01T10:16:17.772536+0000","last_undegraded":"2023-10-01T10:16:17.772536+0000","last_fullsized":"2023-10-01T10:16:17.772536+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":22,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-01T10:16:18.364581+0000","last_change":"2023-10-01T10:16:18.364581+0000","last_active":"2023-10-01T10:16:18.364581+0000","last_peered":"2023-10-01T10:16:18.364581+0000","last_clean":"2023-10-01T10:16:18.364581+0000","last_became_active":"2023-10-01T10:16:18.363621+0000","last_became_peered":"2023-10-01T10:16:18.363621+0000","last_unstale":"2023-10-01T10:16:18.364581+0000","last_undegraded":"2023-10-01T10:16:18.364581+0000","last_fullsized":"2023-10-01T10:16:18.364581+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":13,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T10:15:41.420588+0000","last_change":"2023-10-01T10:15:39.299457+0000","last_active":"2023-10-01T10:15:41.420588+0000","last_peered":"2023-10-01T10:15:41.420588+0000","last_clean":"2023-10-01T10:15:41.420588+0000","last_became_active":"2023-10-01T10:15:39.298456+0000","last_became_peered":"2023-10-01T10:15:39.298456+0000","last_unstale":"2023-10-01T10:15:41.420588+0000","last_undegraded":"2023-10-01T10:15:41.420588+0000","last_fullsized":"2023-10-01T10:15:41.420588+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.772970+0000","last_change":"2023-10-01T10:15:40.309865+0000","last_active":"2023-10-01T10:16:17.772970+0000","last_peered":"2023-10-01T10:16:17.772970+0000","last_clean":"2023-10-01T10:16:17.772970+0000","last_became_active":"2023-10-01T10:15:40.309490+0000","last_became_peered":"2023-10-01T10:15:40.309490+0000","last_unstale":"2023-10-01T10:16:17.772970+0000","last_undegraded":"2023-10-01T10:16:17.772970+0000","last_fullsized":"2023-10-01T10:16:17.772970+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.772480+0000","last_change":"2023-10-01T10:15:39.300710+0000","last_active":"2023-10-01T10:16:17.772480+0000","last_peered":"2023-10-01T10:16:17.772480+0000","last_clean":"2023-10-01T10:16:17.772480+0000","last_became_active":"2023-10-01T10:15:39.300387+0000","last_became_peered":"2023-10-01T10:15:39.300387+0000","last_unstale":"2023-10-01T10:16:17.772480+0000","last_undegraded":"2023-10-01T10:16:17.772480+0000","last_fullsized":"2023-10-01T10:16:17.772480+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.289505+0000","last_change":"2023-10-01T10:15:40.311169+0000","last_active":"2023-10-01T10:16:17.289505+0000","last_peered":"2023-10-01T10:16:17.289505+0000","last_clean":"2023-10-01T10:16:17.289505+0000","last_became_active":"2023-10-01T10:15:40.310611+0000","last_became_peered":"2023-10-01T10:15:40.310611+0000","last_unstale":"2023-10-01T10:16:17.289505+0000","last_undegraded":"2023-10-01T10:16:17.289505+0000","last_fullsized":"2023-10-01T10:16:17.289505+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.291644+0000","last_change":"2023-10-01T10:15:39.295299+0000","last_active":"2023-10-01T10:16:17.291644+0000","last_peered":"2023-10-01T10:16:17.291644+0000","last_clean":"2023-10-01T10:16:17.291644+0000","last_became_active":"2023-10-01T10:15:39.294974+0000","last_became_peered":"2023-10-01T10:15:39.294974+0000","last_unstale":"2023-10-01T10:16:17.291644+0000","last_undegraded":"2023-10-01T10:16:17.291644+0000","last_fullsized":"2023-10-01T10:16:17.291644+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T10:15:41.420342+0000","last_change":"2023-10-01T10:15:40.319834+0000","last_active":"2023-10-01T10:15:41.420342+0000","last_peered":"2023-10-01T10:15:41.420342+0000","last_clean":"2023-10-01T10:15:41.420342+0000","last_became_active":"2023-10-01T10:15:40.319460+0000","last_became_peered":"2023-10-01T10:15:40.319460+0000","last_unstale":"2023-10-01T10:15:41.420342+0000","last_undegraded":"2023-10-01T10:15:41.420342+0000","last_fullsized":"2023-10-01T10:15:41.420342+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.776935+0000","last_change":"2023-10-01T10:15:39.293587+0000","last_active":"2023-10-01T10:16:17.776935+0000","last_peered":"2023-10-01T10:16:17.776935+0000","last_clean":"2023-10-01T10:16:17.776935+0000","last_became_active":"2023-10-01T10:15:39.293273+0000","last_became_peered":"2023-10-01T10:15:39.293273+0000","last_unstale":"2023-10-01T10:16:17.776935+0000","last_undegraded":"2023-10-01T10:16:17.776935+0000","last_fullsized":"2023-10-01T10:16:17.776935+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.772238+0000","last_change":"2023-10-01T10:15:40.312616+0000","last_active":"2023-10-01T10:16:17.772238+0000","last_peered":"2023-10-01T10:16:17.772238+0000","last_clean":"2023-10-01T10:16:17.772238+0000","last_became_active":"2023-10-01T10:15:40.312249+0000","last_became_peered":"2023-10-01T10:15:40.312249+0000","last_unstale":"2023-10-01T10:16:17.772238+0000","last_undegraded":"2023-10-01T10:16:17.772238+0000","last_fullsized":"2023-10-01T10:16:17.772238+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.776891+0000","last_change":"2023-10-01T10:15:39.302221+0000","last_active":"2023-10-01T10:16:17.776891+0000","last_peered":"2023-10-01T10:16:17.776891+0000","last_clean":"2023-10-01T10:16:17.776891+0000","last_became_active":"2023-10-01T10:15:39.301850+0000","last_became_peered":"2023-10-01T10:15:39.301850+0000","last_unstale":"2023-10-01T10:16:17.776891+0000","last_undegraded":"2023-10-01T10:16:17.776891+0000","last_fullsized":"2023-10-01T10:16:17.776891+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.291406+0000","last_change":"2023-10-01T10:15:40.318702+0000","last_active":"2023-10-01T10:16:17.291406+0000","last_peered":"2023-10-01T10:16:17.291406+0000","last_clean":"2023-10-01T10:16:17.291406+0000","last_became_active":"2023-10-01T10:15:40.318333+0000","last_became_peered":"2023-10-01T10:15:40.318333+0000","last_unstale":"2023-10-01T10:16:17.291406+0000","last_undegraded":"2023-10-01T10:16:17.291406+0000","last_fullsized":"2023-10-01T10:16:17.291406+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.777054+0000","last_change":"2023-10-01T10:15:39.300486+0000","last_active":"2023-10-01T10:16:17.777054+0000","last_peered":"2023-10-01T10:16:17.777054+0000","last_clean":"2023-10-01T10:16:17.777054+0000","last_became_active":"2023-10-01T10:15:39.300129+0000","last_became_peered":"2023-10-01T10:15:39.300129+0000","last_unstale":"2023-10-01T10:16:17.777054+0000","last_undegraded":"2023-10-01T10:16:17.777054+0000","last_fullsized":"2023-10-01T10:16:17.777054+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.772864+0000","last_change":"2023-10-01T10:15:40.317579+0000","last_active":"2023-10-01T10:16:17.772864+0000","last_peered":"2023-10-01T10:16:17.772864+0000","last_clean":"2023-10-01T10:16:17.772864+0000","last_became_active":"2023-10-01T10:15:40.317213+0000","last_became_peered":"2023-10-01T10:15:40.317213+0000","last_unstale":"2023-10-01T10:16:17.772864+0000","last_undegraded":"2023-10-01T10:16:17.772864+0000","last_fullsized":"2023-10-01T10:16:17.772864+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":21,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T10:15:44.811930+0000","last_change":"2023-10-01T10:15:39.291400+0000","last_active":"2023-10-01T10:15:44.811930+0000","last_peered":"2023-10-01T10:15:44.811930+0000","last_clean":"2023-10-01T10:15:44.811930+0000","last_became_active":"2023-10-01T10:15:39.291018+0000","last_became_peered":"2023-10-01T10:15:39.291018+0000","last_unstale":"2023-10-01T10:15:44.811930+0000","last_undegraded":"2023-10-01T10:15:44.811930+0000","last_fullsized":"2023-10-01T10:15:44.811930+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":8,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.289641+0000","last_change":"2023-10-01T10:15:40.316610+0000","last_active":"2023-10-01T10:16:17.289641+0000","last_peered":"2023-10-01T10:16:17.289641+0000","last_clean":"2023-10-01T10:16:17.289641+0000","last_became_active":"2023-10-01T10:15:40.316284+0000","last_became_peered":"2023-10-01T10:15:40.316284+0000","last_unstale":"2023-10-01T10:16:17.289641+0000","last_undegraded":"2023-10-01T10:16:17.289641+0000","last_fullsized":"2023-10-01T10:16:17.289641+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.776247+0000","last_change":"2023-10-01T10:15:39.302235+0000","last_active":"2023-10-01T10:16:17.776247+0000","last_peered":"2023-10-01T10:16:17.776247+0000","last_clean":"2023-10-01T10:16:17.776247+0000","last_became_active":"2023-10-01T10:15:39.301830+0000","last_became_peered":"2023-10-01T10:15:39.301830+0000","last_unstale":"2023-10-01T10:16:17.776247+0000","last_undegraded":"2023-10-01T10:16:17.776247+0000","last_fullsized":"2023-10-01T10:16:17.776247+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.291632+0000","last_change":"2023-10-01T10:15:40.315419+0000","last_active":"2023-10-01T10:16:17.291632+0000","last_peered":"2023-10-01T10:16:17.291632+0000","last_clean":"2023-10-01T10:16:17.291632+0000","last_became_active":"2023-10-01T10:15:40.314924+0000","last_became_peered":"2023-10-01T10:15:40.314924+0000","last_unstale":"2023-10-01T10:16:17.291632+0000","last_undegraded":"2023-10-01T10:16:17.291632+0000","last_fullsized":"2023-10-01T10:16:17.291632+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.289124+0000","last_change":"2023-10-01T10:15:39.293968+0000","last_active":"2023-10-01T10:16:17.289124+0000","last_peered":"2023-10-01T10:16:17.289124+0000","last_clean":"2023-10-01T10:16:17.289124+0000","last_became_active":"2023-10-01T10:15:39.293630+0000","last_became_peered":"2023-10-01T10:15:39.293630+0000","last_unstale":"2023-10-01T10:16:17.289124+0000","last_undegraded":"2023-10-01T10:16:17.289124+0000","last_fullsized":"2023-10-01T10:16:17.289124+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":22,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-10-01T10:16:18.300879+0000","last_change":"2023-10-01T10:16:18.300879+0000","last_active":"2023-10-01T10:16:18.300879+0000","last_peered":"2023-10-01T10:16:18.300879+0000","last_clean":"2023-10-01T10:16:18.300879+0000","last_became_active":"2023-10-01T10:16:18.300505+0000","last_became_peered":"2023-10-01T10:16:18.300505+0000","last_unstale":"2023-10-01T10:16:18.300879+0000","last_undegraded":"2023-10-01T10:16:18.300879+0000","last_fullsized":"2023-10-01T10:16:18.300879+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":16,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.776614+0000","last_change":"2023-10-01T10:15:39.300411+0000","last_active":"2023-10-01T10:16:17.776614+0000","last_peered":"2023-10-01T10:16:17.776614+0000","last_clean":"2023-10-01T10:16:17.776614+0000","last_became_active":"2023-10-01T10:15:39.300066+0000","last_became_peered":"2023-10-01T10:15:39.300066+0000","last_unstale":"2023-10-01T10:16:17.776614+0000","last_undegraded":"2023-10-01T10:16:17.776614+0000","last_fullsized":"2023-10-01T10:16:17.776614+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.772758+0000","last_change":"2023-10-01T10:15:40.317496+0000","last_active":"2023-10-01T10:16:17.772758+0000","last_peered":"2023-10-01T10:16:17.772758+0000","last_clean":"2023-10-01T10:16:17.772758+0000","last_became_active":"2023-10-01T10:15:40.316915+0000","last_became_peered":"2023-10-01T10:15:40.316915+0000","last_unstale":"2023-10-01T10:16:17.772758+0000","last_undegraded":"2023-10-01T10:16:17.772758+0000","last_fullsized":"2023-10-01T10:16:17.772758+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"60'6","reported_seq":20,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.772907+0000","last_change":"2023-10-01T10:15:39.296588+0000","last_active":"2023-10-01T10:16:17.772907+0000","last_peered":"2023-10-01T10:16:17.772907+0000","last_clean":"2023-10-01T10:16:17.772907+0000","last_became_active":"2023-10-01T10:15:39.294601+0000","last_became_peered":"2023-10-01T10:15:39.294601+0000","last_unstale":"2023-10-01T10:16:17.772907+0000","last_undegraded":"2023-10-01T10:16:17.772907+0000","last_fullsized":"2023-10-01T10:16:17.772907+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":6,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.766703+0000","last_change":"2023-10-01T10:15:39.310807+0000","last_active":"2023-10-01T10:16:17.766703+0000","last_peered":"2023-10-01T10:16:17.766703+0000","last_clean":"2023-10-01T10:16:17.766703+0000","last_became_active":"2023-10-01T10:15:39.310474+0000","last_became_peered":"2023-10-01T10:15:39.310474+0000","last_unstale":"2023-10-01T10:16:17.766703+0000","last_undegraded":"2023-10-01T10:16:17.766703+0000","last_fullsized":"2023-10-01T10:16:17.766703+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":17,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.776723+0000","last_change":"2023-10-01T10:15:39.300951+0000","last_active":"2023-10-01T10:16:17.776723+0000","last_peered":"2023-10-01T10:16:17.776723+0000","last_clean":"2023-10-01T10:16:17.776723+0000","last_became_active":"2023-10-01T10:15:39.300626+0000","last_became_peered":"2023-10-01T10:15:39.300626+0000","last_unstale":"2023-10-01T10:16:17.776723+0000","last_undegraded":"2023-10-01T10:16:17.776723+0000","last_fullsized":"2023-10-01T10:16:17.776723+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.771751+0000","last_change":"2023-10-01T10:15:40.317104+0000","last_active":"2023-10-01T10:16:17.771751+0000","last_peered":"2023-10-01T10:16:17.771751+0000","last_clean":"2023-10-01T10:16:17.771751+0000","last_became_active":"2023-10-01T10:15:40.316787+0000","last_became_peered":"2023-10-01T10:15:40.316787+0000","last_unstale":"2023-10-01T10:16:17.771751+0000","last_undegraded":"2023-10-01T10:16:17.771751+0000","last_fullsized":"2023-10-01T10:16:17.771751+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.291729+0000","last_change":"2023-10-01T10:15:39.294083+0000","last_active":"2023-10-01T10:16:17.291729+0000","last_peered":"2023-10-01T10:16:17.291729+0000","last_clean":"2023-10-01T10:16:17.291729+0000","last_became_active":"2023-10-01T10:15:39.293744+0000","last_became_peered":"2023-10-01T10:15:39.293744+0000","last_unstale":"2023-10-01T10:16:17.291729+0000","last_undegraded":"2023-10-01T10:16:17.291729+0000","last_fullsized":"2023-10-01T10:16:17.291729+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.771584+0000","last_change":"2023-10-01T10:15:40.315049+0000","last_active":"2023-10-01T10:16:17.771584+0000","last_peered":"2023-10-01T10:16:17.771584+0000","last_clean":"2023-10-01T10:16:17.771584+0000","last_became_active":"2023-10-01T10:15:40.314510+0000","last_became_peered":"2023-10-01T10:15:40.314510+0000","last_unstale":"2023-10-01T10:16:17.771584+0000","last_undegraded":"2023-10-01T10:16:17.771584+0000","last_fullsized":"2023-10-01T10:16:17.771584+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.290468+0000","last_change":"2023-10-01T10:15:39.297202+0000","last_active":"2023-10-01T10:16:17.290468+0000","last_peered":"2023-10-01T10:16:17.290468+0000","last_clean":"2023-10-01T10:16:17.290468+0000","last_became_active":"2023-10-01T10:15:39.296867+0000","last_became_peered":"2023-10-01T10:15:39.296867+0000","last_unstale":"2023-10-01T10:16:17.290468+0000","last_undegraded":"2023-10-01T10:16:17.290468+0000","last_fullsized":"2023-10-01T10:16:17.290468+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":12,"reported_epoch":60,"state":"active+clean","last_fresh":"2023-10-01T10:15:41.421148+0000","last_change":"2023-10-01T10:15:40.315154+0000","last_active":"2023-10-01T10:15:41.421148+0000","last_peered":"2023-10-01T10:15:41.421148+0000","last_clean":"2023-10-01T10:15:41.421148+0000","last_became_active":"2023-10-01T10:15:40.314777+0000","last_became_peered":"2023-10-01T10:15:40.314777+0000","last_unstale":"2023-10-01T10:15:41.421148+0000","last_undegraded":"2023-10-01T10:15:41.421148+0000","last_fullsized":"2023-10-01T10:15:41.421148+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.771811+0000","last_change":"2023-10-01T10:15:39.298982+0000","last_active":"2023-10-01T10:16:17.771811+0000","last_peered":"2023-10-01T10:16:17.771811+0000","last_clean":"2023-10-01T10:16:17.771811+0000","last_became_active":"2023-10-01T10:15:39.298695+0000","last_became_peered":"2023-10-01T10:15:39.298695+0000","last_unstale":"2023-10-01T10:16:17.771811+0000","last_undegraded":"2023-10-01T10:16:17.771811+0000","last_fullsized":"2023-10-01T10:16:17.771811+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.289766+0000","last_change":"2023-10-01T10:15:40.315110+0000","last_active":"2023-10-01T10:16:17.289766+0000","last_peered":"2023-10-01T10:16:17.289766+0000","last_clean":"2023-10-01T10:16:17.289766+0000","last_became_active":"2023-10-01T10:15:40.314775+0000","last_became_peered":"2023-10-01T10:15:40.314775+0000","last_unstale":"2023-10-01T10:16:17.289766+0000","last_undegraded":"2023-10-01T10:16:17.289766+0000","last_fullsized":"2023-10-01T10:16:17.289766+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.766436+0000","last_change":"2023-10-01T10:15:39.294176+0000","last_active":"2023-10-01T10:16:17.766436+0000","last_peered":"2023-10-01T10:16:17.766436+0000","last_clean":"2023-10-01T10:16:17.766436+0000","last_became_active":"2023-10-01T10:15:39.293850+0000","last_became_peered":"2023-10-01T10:15:39.293850+0000","last_unstale":"2023-10-01T10:16:17.766436+0000","last_undegraded":"2023-10-01T10:16:17.766436+0000","last_fullsized":"2023-10-01T10:16:17.766436+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":13,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.773060+0000","last_change":"2023-10-01T10:15:40.311002+0000","last_active":"2023-10-01T10:16:17.773060+0000","last_peered":"2023-10-01T10:16:17.773060+0000","last_clean":"2023-10-01T10:16:17.773060+0000","last_became_active":"2023-10-01T10:15:40.310679+0000","last_became_peered":"2023-10-01T10:15:40.310679+0000","last_unstale":"2023-10-01T10:16:17.773060+0000","last_undegraded":"2023-10-01T10:16:17.773060+0000","last_fullsized":"2023-10-01T10:16:17.773060+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":15,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-10-01T10:16:17.772341+0000","last_change":"2023-10-01T10:15:39.298785+0000","last_active":"2023-10-01T10:16:17.772341+0000","last_peered":"2023-10-01T10:16:17.772341+0000","last_clean":"2023-10-01T10:16:17.772341+0000","last_became_active":"2023-10-01T10:15:39.297156+0000","last_became_peered":"2023-10-01T10:15:39.297156+0000","last_unstale":"2023-10-01T10:16:17.772341+0000","last_undegraded":"2023-10-01T10:16:17.772341+0000","last_fullsized":"2023-10-01T10:16:17.772341+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430264,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297736,"kb_used_data":640,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447416,"statfs":{"total":95995035648,"available":95690153984,"internally_reserved":0,"allocated":655360,"data_stored":271700,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:16:55 2023","interfaces":[{"interface":"back","average":{"1min":0.86199999999999999,"5min":0.89500000000000002,"15min":0.91800000000000004},"min":{"1min":0.56699999999999995,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.1859999999999999,"5min":1.2170000000000001,"15min":1.2170000000000001},"last":1.0720000000000001},{"interface":"front","average":{"1min":0.88300000000000001,"5min":0.88600000000000001,"15min":0.91900000000000004},"min":{"1min":0.68300000000000005,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":1.0720000000000001,"5min":1.2629999999999999,"15min":1.2629999999999999},"last":0.86199999999999999}]},{"osd":1,"last update":"Sun Oct 1 10:16:55 2023","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":0.91100000000000003,"15min":0.91000000000000003},"min":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"max":{"1min":1.3260000000000001,"5min":1.444,"15min":1.444},"last":1.125},{"interface":"front","average":{"1min":0.91100000000000003,"5min":0.89300000000000002,"15min":0.88700000000000001},"min":{"1min":0.74099999999999999,"5min":0.56599999999999995,"15min":0.56599999999999995},"max":{"1min":1.1140000000000001,"5min":1.377,"15min":1.377},"last":0.84499999999999997}]},{"osd":2,"last update":"Sun Oct 1 10:16:55 2023","interfaces":[{"interface":"back","average":{"1min":0.995,"5min":0.94199999999999995,"15min":0.93600000000000005},"min":{"1min":0.71599999999999997,"5min":0.61899999999999999,"15min":0.61899999999999999},"max":{"1min":1.631,"5min":1.631,"15min":1.631},"last":0.97399999999999998},{"interface":"front","average":{"1min":1.0249999999999999,"5min":0.97099999999999997,"15min":0.96199999999999997},"min":{"1min":0.82799999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"max":{"1min":1.218,"5min":1.2390000000000001,"15min":1.2390000000000001},"last":0.998}]},{"osd":3,"last update":"Sun Oct 1 10:16:55 2023","interfaces":[{"interface":"back","average":{"1min":0.96599999999999997,"5min":0.91900000000000004,"15min":0.91100000000000003},"min":{"1min":0.66600000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.2450000000000001,"5min":1.2450000000000001,"15min":1.2450000000000001},"last":0.77400000000000002},{"interface":"front","average":{"1min":1.0569999999999999,"5min":1.0189999999999999,"15min":1.01},"min":{"1min":0.73399999999999999,"5min":0.57199999999999995,"15min":0.57199999999999995},"max":{"1min":1.3919999999999999,"5min":3.2229999999999999,"15min":3.2229999999999999},"last":1.0529999999999999}]},{"osd":4,"last update":"Sun Oct 1 10:16:55 2023","interfaces":[{"interface":"back","average":{"1min":1.0009999999999999,"5min":0.95699999999999996,"15min":0.94299999999999995},"min":{"1min":0.55200000000000005,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":1.2929999999999999,"5min":1.5629999999999999,"15min":1.5629999999999999},"last":0.72699999999999998},{"interface":"front","average":{"1min":1.028,"5min":0.96299999999999997,"15min":0.93600000000000005},"min":{"1min":0.72699999999999998,"5min":0.437,"15min":0.437},"max":{"1min":1.266,"5min":1.7470000000000001,"15min":1.7470000000000001},"last":0.61799999999999999}]},{"osd":5,"last update":"Sun Oct 1 10:16:55 2023","interfaces":[{"interface":"back","average":{"1min":1.0740000000000001,"5min":1.01,"15min":1.002},"min":{"1min":0.69499999999999995,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":1.3089999999999999,"5min":1.831,"15min":1.831},"last":0.88500000000000001},{"interface":"front","average":{"1min":1.161,"5min":1.0740000000000001,"15min":1.0920000000000001},"min":{"1min":0.73999999999999999,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":1.3700000000000001,"5min":1.5,"15min":1.5},"last":0.91500000000000004}]},{"osd":6,"last update":"Sun Oct 1 10:16:55 2023","interfaces":[{"interface":"back","average":{"1min":1.1519999999999999,"5min":1.081,"15min":1.0920000000000001},"min":{"1min":0.83399999999999996,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":1.381,"5min":1.7150000000000001,"15min":1.7150000000000001},"last":0.75700000000000001},{"interface":"front","average":{"1min":1.1299999999999999,"5min":1.099,"15min":1.109},"min":{"1min":0.68999999999999995,"5min":0.60499999999999998,"15min":0.60499999999999998},"max":{"1min":1.343,"5min":2.2869999999999999,"15min":2.2869999999999999},"last":0.93999999999999995}]}]},{"osd":6,"up_from":42,"seq":180388626498,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297720,"kb_used_data":624,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447432,"statfs":{"total":95995035648,"available":95690170368,"internally_reserved":0,"allocated":638976,"data_stored":271239,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:17:12 2023","interfaces":[{"interface":"back","average":{"1min":0.82899999999999996,"5min":0.85499999999999998,"15min":0.88500000000000001},"min":{"1min":0.55900000000000005,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":1.1899999999999999,"5min":1.359,"15min":1.359},"last":0.93100000000000005},{"interface":"front","average":{"1min":0.83699999999999997,"5min":0.85699999999999998,"15min":0.85299999999999998},"min":{"1min":0.503,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":1.153,"5min":1.1839999999999999,"15min":1.1839999999999999},"last":0.86699999999999999}]},{"osd":1,"last update":"Sun Oct 1 10:17:12 2023","interfaces":[{"interface":"back","average":{"1min":0.91100000000000003,"5min":0.91400000000000003,"15min":0.93200000000000005},"min":{"1min":0.53300000000000003,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":1.1870000000000001,"5min":1.6279999999999999,"15min":1.6279999999999999},"last":0.80600000000000005},{"interface":"front","average":{"1min":0.92100000000000004,"5min":0.91800000000000004,"15min":0.96999999999999997},"min":{"1min":0.64100000000000001,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":1.1379999999999999,"5min":1.298,"15min":1.298},"last":0.83599999999999997}]},{"osd":2,"last update":"Sun Oct 1 10:17:12 2023","interfaces":[{"interface":"back","average":{"1min":0.89000000000000001,"5min":0.871,"15min":0.876},"min":{"1min":0.63700000000000001,"5min":0.57799999999999996,"15min":0.57799999999999996},"max":{"1min":2.274,"5min":2.274,"15min":2.274},"last":0.745},{"interface":"front","average":{"1min":0.89200000000000002,"5min":0.92000000000000004,"15min":0.93400000000000005},"min":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"max":{"1min":1.159,"5min":2.8149999999999999,"15min":2.8149999999999999},"last":1.0740000000000001}]},{"osd":3,"last update":"Sun Oct 1 10:17:12 2023","interfaces":[{"interface":"back","average":{"1min":0.99099999999999999,"5min":0.94599999999999995,"15min":0.95999999999999996},"min":{"1min":0.73599999999999999,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":2.1059999999999999,"5min":2.1059999999999999,"15min":2.1059999999999999},"last":1.0069999999999999},{"interface":"front","average":{"1min":0.94499999999999995,"5min":0.91100000000000003,"15min":0.92200000000000004},"min":{"1min":0.69399999999999995,"5min":0.57899999999999996,"15min":0.57899999999999996},"max":{"1min":2.1379999999999999,"5min":2.1379999999999999,"15min":2.1379999999999999},"last":0.89900000000000002}]},{"osd":4,"last update":"Sun Oct 1 10:17:12 2023","interfaces":[{"interface":"back","average":{"1min":0.93899999999999995,"5min":0.89600000000000002,"15min":0.88},"min":{"1min":0.57499999999999996,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.2330000000000001,"5min":1.4890000000000001,"15min":1.4890000000000001},"last":0.96099999999999997},{"interface":"front","average":{"1min":1.024,"5min":0.95599999999999996,"15min":0.91800000000000004},"min":{"1min":0.73999999999999999,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":1.2470000000000001,"5min":1.349,"15min":1.349},"last":1.0549999999999999}]},{"osd":5,"last update":"Sun Oct 1 10:17:12 2023","interfaces":[{"interface":"back","average":{"1min":1.022,"5min":0.96199999999999997,"15min":0.92000000000000004},"min":{"1min":0.74399999999999999,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":1.304,"5min":1.5629999999999999,"15min":1.5629999999999999},"last":1.0349999999999999},{"interface":"front","average":{"1min":1.0629999999999999,"5min":1,"15min":0.98099999999999998},"min":{"1min":0.69599999999999995,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":1.288,"5min":1.6639999999999999,"15min":1.6639999999999999},"last":1.1040000000000001}]},{"osd":7,"last update":"Sun Oct 1 10:16:53 2023","interfaces":[{"interface":"back","average":{"1min":1.1020000000000001,"5min":1.077,"15min":1.081},"min":{"1min":0.72899999999999998,"5min":0.59199999999999997,"15min":0.59199999999999997},"max":{"1min":1.706,"5min":1.7250000000000001,"15min":1.7250000000000001},"last":1.1539999999999999},{"interface":"front","average":{"1min":1.085,"5min":1.0509999999999999,"15min":1.0429999999999999},"min":{"1min":0.69399999999999995,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":1.4359999999999999,"5min":1.5,"15min":1.5},"last":1.123}]}]},{"osd":1,"up_from":13,"seq":55834574955,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297708,"kb_used_data":612,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447444,"statfs":{"total":95995035648,"available":95690182656,"internally_reserved":0,"allocated":626688,"data_stored":269948,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:16:47 2023","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.64800000000000002,"15min":0.58099999999999996},"min":{"1min":0.39700000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.99299999999999999,"5min":1.698,"15min":1.698},"last":0.76100000000000001},{"interface":"front","average":{"1min":0.65100000000000002,"5min":0.70399999999999996,"15min":0.60499999999999998},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.99099999999999999,"5min":1.8100000000000001,"15min":1.8100000000000001},"last":0.54900000000000004}]},{"osd":2,"last update":"Sun Oct 1 10:16:29 2023","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":0.68500000000000005,"15min":0.64500000000000002},"min":{"1min":0.41299999999999998,"5min":0.376,"15min":0.33600000000000002},"max":{"1min":1.0449999999999999,"5min":1.1120000000000001,"15min":1.1120000000000001},"last":0.83499999999999996},{"interface":"front","average":{"1min":0.72399999999999998,"5min":0.76500000000000001,"15min":0.68899999999999995},"min":{"1min":0.50900000000000001,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.94699999999999995,"5min":1.5009999999999999,"15min":1.5009999999999999},"last":0.80700000000000005}]},{"osd":3,"last update":"Sun Oct 1 10:17:11 2023","interfaces":[{"interface":"back","average":{"1min":0.73499999999999999,"5min":0.80000000000000004,"15min":0.80500000000000005},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":1.089,"5min":2.5059999999999998,"15min":2.5059999999999998},"last":0.88900000000000001},{"interface":"front","average":{"1min":0.70499999999999996,"5min":0.745,"15min":0.77200000000000002},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":1.1519999999999999,"5min":2.153,"15min":2.153},"last":0.95499999999999996}]},{"osd":4,"last update":"Sun Oct 1 10:16:50 2023","interfaces":[{"interface":"back","average":{"1min":0.90300000000000002,"5min":0.95799999999999996,"15min":0.96799999999999997},"min":{"1min":0.51100000000000001,"5min":0.505,"15min":0.505},"max":{"1min":1.1659999999999999,"5min":1.6759999999999999,"15min":1.6759999999999999},"last":1.0589999999999999},{"interface":"front","average":{"1min":0.96299999999999997,"5min":0.98599999999999999,"15min":0.97299999999999998},"min":{"1min":0.69199999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"max":{"1min":1.2050000000000001,"5min":1.804,"15min":1.804},"last":1.0209999999999999}]},{"osd":5,"last update":"Sun Oct 1 10:16:29 2023","interfaces":[{"interface":"back","average":{"1min":0.95999999999999996,"5min":0.99399999999999999,"15min":1.046},"min":{"1min":0.64500000000000002,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":1.839,"5min":1.839,"15min":1.839},"last":0.92500000000000004},{"interface":"front","average":{"1min":1.0189999999999999,"5min":1.0429999999999999,"15min":1.056},"min":{"1min":0.81200000000000006,"5min":0.64900000000000002,"15min":0.64900000000000002},"max":{"1min":1.264,"5min":1.863,"15min":1.863},"last":1.115}]},{"osd":6,"last update":"Sun Oct 1 10:17:11 2023","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":0.998,"15min":0.97099999999999997},"min":{"1min":0.64800000000000002,"5min":0.61899999999999999,"15min":0.61899999999999999},"max":{"1min":1.3,"5min":2.1000000000000001,"15min":2.1000000000000001},"last":1.2230000000000001},{"interface":"front","average":{"1min":1.022,"5min":1.0409999999999999,"15min":1.0529999999999999},"min":{"1min":0.71499999999999997,"5min":0.68300000000000005,"15min":0.68300000000000005},"max":{"1min":1.2989999999999999,"5min":2.0510000000000002,"15min":2.0510000000000002},"last":1.196}]},{"osd":7,"last update":"Sun Oct 1 10:16:50 2023","interfaces":[{"interface":"back","average":{"1min":0.97499999999999998,"5min":1.02,"15min":1.0009999999999999},"min":{"1min":0.72199999999999998,"5min":0.58699999999999997,"15min":0.58699999999999997},"max":{"1min":1.3,"5min":2.0230000000000001,"15min":2.0230000000000001},"last":1.091},{"interface":"front","average":{"1min":1.0089999999999999,"5min":1.0389999999999999,"15min":1.0609999999999999},"min":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"max":{"1min":1.6040000000000001,"5min":1.623,"15min":1.623},"last":1.147}]}]},{"osd":0,"up_from":8,"seq":34359738484,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297716,"kb_used_data":620,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447436,"statfs":{"total":95995035648,"available":95690174464,"internally_reserved":0,"allocated":634880,"data_stored":270524,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 10:16:47 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.66900000000000004,"15min":0.58099999999999996},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.83299999999999996,"5min":1.603,"15min":1.603},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.622,"5min":0.68700000000000006,"15min":0.63600000000000001},"min":{"1min":0.374,"5min":0.312,"15min":0.312},"max":{"1min":0.94099999999999995,"5min":1.0369999999999999,"15min":1.0780000000000001},"last":0.40400000000000003}]},{"osd":2,"last update":"Sun Oct 1 10:16:34 2023","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":0.72399999999999998,"15min":0.66400000000000003},"min":{"1min":0.41999999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.88700000000000001,"5min":1.169,"15min":1.2050000000000001},"last":0.64700000000000002},{"interface":"front","average":{"1min":0.626,"5min":0.68300000000000005,"15min":0.64700000000000002},"min":{"1min":0.46000000000000002,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.94399999999999995,"5min":1.149,"15min":1.149},"last":0.50700000000000001}]},{"osd":3,"last update":"Sun Oct 1 10:17:12 2023","interfaces":[{"interface":"back","average":{"1min":0.67000000000000004,"5min":0.70199999999999996,"15min":0.68300000000000005},"min":{"1min":0.38,"5min":0.38,"15min":0.33000000000000002},"max":{"1min":1.1379999999999999,"5min":1.1379999999999999,"15min":1.1379999999999999},"last":0.68500000000000005},{"interface":"front","average":{"1min":0.73799999999999999,"5min":0.80700000000000005,"15min":0.78100000000000003},"min":{"1min":0.42299999999999999,"5min":0.375,"15min":0.375},"max":{"1min":1.0700000000000001,"5min":1.28,"15min":1.28},"last":0.86699999999999999}]},{"osd":4,"last update":"Sun Oct 1 10:16:47 2023","interfaces":[{"interface":"back","average":{"1min":0.88900000000000001,"5min":0.90500000000000003,"15min":0.91000000000000003},"min":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":1.1279999999999999,"5min":1.476,"15min":1.476},"last":0.71199999999999997},{"interface":"front","average":{"1min":0.94799999999999995,"5min":1.0169999999999999,"15min":0.96899999999999997},"min":{"1min":0.66800000000000004,"5min":0.66200000000000003,"15min":0.60799999999999998},"max":{"1min":1.276,"5min":1.6499999999999999,"15min":1.6499999999999999},"last":1.0369999999999999}]},{"osd":5,"last update":"Sun Oct 1 10:16:34 2023","interfaces":[{"interface":"back","average":{"1min":0.98399999999999999,"5min":1.0349999999999999,"15min":1.0309999999999999},"min":{"1min":0.63100000000000001,"5min":0.60599999999999998,"15min":0.60599999999999998},"max":{"1min":1.637,"5min":1.718,"15min":1.718},"last":1.2509999999999999},{"interface":"front","average":{"1min":1.0289999999999999,"5min":1.046,"15min":1.036},"min":{"1min":0.72099999999999997,"5min":0.70999999999999996,"15min":0.70999999999999996},"max":{"1min":1.548,"5min":1.548,"15min":1.548},"last":0.996}]},{"osd":6,"last update":"Sun Oct 1 10:16:15 2023","interfaces":[{"interface":"back","average":{"1min":0.96599999999999997,"5min":1.046,"15min":1.0780000000000001},"min":{"1min":0.67000000000000004,"5min":0.58099999999999996,"15min":0.58099999999999996},"max":{"1min":1.3480000000000001,"5min":1.5669999999999999,"15min":1.5669999999999999},"last":1.1180000000000001},{"interface":"front","average":{"1min":0.98599999999999999,"5min":1.056,"15min":1.0780000000000001},"min":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"max":{"1min":1.3340000000000001,"5min":1.6240000000000001,"15min":1.6240000000000001},"last":0.94599999999999995}]},{"osd":7,"last update":"Sun Oct 1 10:16:47 2023","interfaces":[{"interface":"back","average":{"1min":0.93700000000000006,"5min":0.999,"15min":1.038},"min":{"1min":0.66200000000000003,"5min":0.58799999999999997,"15min":0.58799999999999997},"max":{"1min":1.145,"5min":1.4299999999999999,"15min":1.4299999999999999},"last":0.80600000000000005},{"interface":"front","average":{"1min":0.99099999999999999,"5min":1.115,"15min":1.157},"min":{"1min":0.64300000000000002,"5min":0.53500000000000003,"15min":0.53500000000000003},"max":{"1min":1.3979999999999999,"5min":1.6539999999999999,"15min":1.6539999999999999},"last":1.1859999999999999}]}]},{"osd":2,"up_from":18,"seq":77309411425,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297716,"kb_used_data":620,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447436,"statfs":{"total":95995035648,"available":95690174464,"internally_reserved":0,"allocated":634880,"data_stored":270536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:16:31 2023","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":0.66300000000000003,"15min":0.60299999999999998},"min":{"1min":0.46800000000000003,"5min":0.38400000000000001,"15min":0.318},"max":{"1min":1.0649999999999999,"5min":1.077,"15min":1.077},"last":0.80900000000000005},{"interface":"front","average":{"1min":0.83999999999999997,"5min":0.754,"15min":0.66900000000000004},"min":{"1min":0.53000000000000003,"5min":0.314,"15min":0.314},"max":{"1min":2.927,"5min":2.927,"15min":2.927},"last":1.0409999999999999}]},{"osd":1,"last update":"Sun Oct 1 10:16:31 2023","interfaces":[{"interface":"back","average":{"1min":0.71199999999999997,"5min":0.71599999999999997,"15min":0.65200000000000002},"min":{"1min":0.52300000000000002,"5min":0.37,"15min":0.36599999999999999},"max":{"1min":0.98899999999999999,"5min":1.2529999999999999,"15min":1.2529999999999999},"last":0.90200000000000002},{"interface":"front","average":{"1min":0.77300000000000002,"5min":0.77700000000000002,"15min":0.68300000000000005},"min":{"1min":0.46899999999999997,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":1.0429999999999999,"5min":1.4019999999999999,"15min":1.4019999999999999},"last":0.84799999999999998}]},{"osd":3,"last update":"Sun Oct 1 10:17:10 2023","interfaces":[{"interface":"back","average":{"1min":0.68899999999999995,"5min":0.745,"15min":0.72699999999999998},"min":{"1min":0.40699999999999997,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.92300000000000004,"5min":1.3029999999999999,"15min":1.3029999999999999},"last":1.012},{"interface":"front","average":{"1min":0.77800000000000002,"5min":0.83299999999999996,"15min":0.80700000000000005},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":1.1799999999999999,"5min":1.4710000000000001,"15min":1.4710000000000001},"last":1.1020000000000001}]},{"osd":4,"last update":"Sun Oct 1 10:16:54 2023","interfaces":[{"interface":"back","average":{"1min":0.99299999999999999,"5min":0.999,"15min":0.92500000000000004},"min":{"1min":0.68200000000000005,"5min":0.64700000000000002,"15min":0.64700000000000002},"max":{"1min":1.238,"5min":1.425,"15min":1.425},"last":1.1739999999999999},{"interface":"front","average":{"1min":1.0589999999999999,"5min":1.0640000000000001,"15min":0.98299999999999998},"min":{"1min":0.68799999999999994,"5min":0.66200000000000003,"15min":0.66200000000000003},"max":{"1min":1.3520000000000001,"5min":1.6679999999999999,"15min":1.6679999999999999},"last":1.1479999999999999}]},{"osd":5,"last update":"Sun Oct 1 10:16:36 2023","interfaces":[{"interface":"back","average":{"1min":1.0649999999999999,"5min":1.0409999999999999,"15min":1.0089999999999999},"min":{"1min":0.71399999999999997,"5min":0.629,"15min":0.629},"max":{"1min":1.4239999999999999,"5min":1.647,"15min":1.647},"last":1.4199999999999999},{"interface":"front","average":{"1min":1.032,"5min":1.024,"15min":1.0129999999999999},"min":{"1min":0.621,"5min":0.621,"15min":0.621},"max":{"1min":1.373,"5min":1.54,"15min":1.54},"last":1.3899999999999999}]},{"osd":6,"last update":"Sun Oct 1 10:17:10 2023","interfaces":[{"interface":"back","average":{"1min":1.0349999999999999,"5min":1.0569999999999999,"15min":1.097},"min":{"1min":0.64300000000000002,"5min":0.61799999999999999,"15min":0.61799999999999999},"max":{"1min":1.319,"5min":1.532,"15min":1.532},"last":1.8979999999999999},{"interface":"front","average":{"1min":1.0720000000000001,"5min":1.1220000000000001,"15min":1.149},"min":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"max":{"1min":1.4870000000000001,"5min":1.647,"15min":1.647},"last":1.3500000000000001}]},{"osd":7,"last update":"Sun Oct 1 10:16:54 2023","interfaces":[{"interface":"back","average":{"1min":1.0409999999999999,"5min":1.0700000000000001,"15min":1.0660000000000001},"min":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"max":{"1min":1.3520000000000001,"5min":1.5409999999999999,"15min":1.5409999999999999},"last":1.448},{"interface":"front","average":{"1min":1.0980000000000001,"5min":1.1220000000000001,"15min":1.143},"min":{"1min":0.76800000000000002,"5min":0.58099999999999996,"15min":0.58099999999999996},"max":{"1min":1.4930000000000001,"5min":1.5589999999999999,"15min":1.5589999999999999},"last":1.9510000000000001}]}]},{"osd":3,"up_from":25,"seq":107374182490,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297712,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447440,"statfs":{"total":95995035648,"available":95690178560,"internally_reserved":0,"allocated":630784,"data_stored":270038,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:16:21 2023","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":0.65900000000000003,"15min":0.63},"min":{"1min":0.502,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.80600000000000005,"5min":1.097,"15min":1.097},"last":0.72199999999999998},{"interface":"front","average":{"1min":0.71999999999999997,"5min":0.72399999999999998,"15min":0.71799999999999997},"min":{"1min":0.53600000000000003,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.997,"5min":1.05,"15min":1.05},"last":0.85799999999999998}]},{"osd":1,"last update":"Sun Oct 1 10:16:21 2023","interfaces":[{"interface":"back","average":{"1min":0.745,"5min":0.73999999999999999,"15min":0.75},"min":{"1min":0.55000000000000004,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":1.018,"5min":1.2170000000000001,"15min":1.224},"last":0.67800000000000005},{"interface":"front","average":{"1min":0.70699999999999996,"5min":0.72199999999999998,"15min":0.72599999999999998},"min":{"1min":0.51900000000000002,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.99399999999999999,"5min":1.1439999999999999,"15min":1.1439999999999999},"last":0.90900000000000003}]},{"osd":2,"last update":"Sun Oct 1 10:16:21 2023","interfaces":[{"interface":"back","average":{"1min":0.71999999999999997,"5min":0.75900000000000001,"15min":0.77400000000000002},"min":{"1min":0.53400000000000003,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":1.0489999999999999,"5min":1.133,"15min":1.133},"last":0.82399999999999995},{"interface":"front","average":{"1min":0.77300000000000002,"5min":0.77600000000000002,"15min":0.77800000000000002},"min":{"1min":0.59999999999999998,"5min":0.318,"15min":0.318},"max":{"1min":0.91400000000000003,"5min":1.3500000000000001,"15min":1.3500000000000001},"last":1.22}]},{"osd":4,"last update":"Sun Oct 1 10:16:58 2023","interfaces":[{"interface":"back","average":{"1min":0.91400000000000003,"5min":0.93200000000000005,"15min":0.91500000000000004},"min":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"max":{"1min":1.2869999999999999,"5min":1.2869999999999999,"15min":1.2869999999999999},"last":1.0900000000000001},{"interface":"front","average":{"1min":0.93100000000000005,"5min":0.95699999999999996,"15min":0.95499999999999996},"min":{"1min":0.69599999999999995,"5min":0.51100000000000001,"15min":0.51100000000000001},"max":{"1min":1.3580000000000001,"5min":1.4710000000000001,"15min":1.4710000000000001},"last":0.96699999999999997}]},{"osd":5,"last update":"Sun Oct 1 10:16:37 2023","interfaces":[{"interface":"back","average":{"1min":0.998,"5min":1.0049999999999999,"15min":1.026},"min":{"1min":0.752,"5min":0.52700000000000002,"15min":0.52700000000000002},"max":{"1min":1.3149999999999999,"5min":1.603,"15min":1.603},"last":1.0649999999999999},{"interface":"front","average":{"1min":1.0029999999999999,"5min":1.04,"15min":1.0369999999999999},"min":{"1min":0.72199999999999998,"5min":0.68300000000000005,"15min":0.68300000000000005},"max":{"1min":1.3560000000000001,"5min":1.4330000000000001,"15min":1.4330000000000001},"last":1.137}]},{"osd":6,"last update":"Sun Oct 1 10:17:09 2023","interfaces":[{"interface":"back","average":{"1min":0.95099999999999996,"5min":1.0369999999999999,"15min":1.081},"min":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"max":{"1min":1.419,"5min":1.4970000000000001,"15min":1.4970000000000001},"last":0.995},{"interface":"front","average":{"1min":1.012,"5min":1.038,"15min":1.0780000000000001},"min":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"max":{"1min":1.246,"5min":1.399,"15min":1.399},"last":1.2889999999999999}]},{"osd":7,"last update":"Sun Oct 1 10:16:52 2023","interfaces":[{"interface":"back","average":{"1min":1.0780000000000001,"5min":1.115,"15min":1.165},"min":{"1min":0.76300000000000001,"5min":0.629,"15min":0.629},"max":{"1min":1.337,"5min":1.46,"15min":1.46},"last":1.333},{"interface":"front","average":{"1min":1.0429999999999999,"5min":1.119,"15min":1.1910000000000001},"min":{"1min":0.61899999999999999,"5min":0.56899999999999995,"15min":0.56899999999999995},"max":{"1min":1.4770000000000001,"5min":1.661,"15min":1.661},"last":1.1970000000000001}]}]},{"osd":4,"up_from":30,"seq":128849018962,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297732,"kb_used_data":636,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447420,"statfs":{"total":95995035648,"available":95690158080,"internally_reserved":0,"allocated":651264,"data_stored":271630,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:16:52 2023","interfaces":[{"interface":"back","average":{"1min":0.86699999999999999,"5min":0.82999999999999996,"15min":0.83699999999999997},"min":{"1min":0.59199999999999997,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":1.1499999999999999,"5min":1.1499999999999999,"15min":1.22},"last":0.748},{"interface":"front","average":{"1min":0.92400000000000004,"5min":0.879,"15min":0.874},"min":{"1min":0.69999999999999996,"5min":0.52100000000000002,"15min":0.52100000000000002},"max":{"1min":1.2030000000000001,"5min":1.873,"15min":1.873},"last":1.208}]},{"osd":1,"last update":"Sun Oct 1 10:16:52 2023","interfaces":[{"interface":"back","average":{"1min":0.96099999999999997,"5min":0.86699999999999999,"15min":0.874},"min":{"1min":0.65600000000000003,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":1.298,"5min":1.484,"15min":1.484},"last":0.85599999999999998},{"interface":"front","average":{"1min":1.0029999999999999,"5min":0.91300000000000003,"15min":0.89900000000000002},"min":{"1min":0.626,"5min":0.626,"15min":0.626},"max":{"1min":1.214,"5min":1.9650000000000001,"15min":1.9650000000000001},"last":1.1100000000000001}]},{"osd":2,"last update":"Sun Oct 1 10:16:52 2023","interfaces":[{"interface":"back","average":{"1min":0.98899999999999999,"5min":0.94099999999999995,"15min":0.90600000000000003},"min":{"1min":0.63600000000000001,"5min":0.58899999999999997,"15min":0.58899999999999997},"max":{"1min":1.228,"5min":2,"15min":2},"last":0.96299999999999997},{"interface":"front","average":{"1min":0.83999999999999997,"5min":0.874,"15min":0.878},"min":{"1min":0.56999999999999995,"5min":0.53600000000000003,"15min":0.53600000000000003},"max":{"1min":1.024,"5min":2.335,"15min":2.335},"last":0.91500000000000004}]},{"osd":3,"last update":"Sun Oct 1 10:16:52 2023","interfaces":[{"interface":"back","average":{"1min":1.0109999999999999,"5min":0.97599999999999998,"15min":0.96999999999999997},"min":{"1min":0.73199999999999998,"5min":0.67100000000000004,"15min":0.56799999999999995},"max":{"1min":1.4019999999999999,"5min":2.4169999999999998,"15min":2.4169999999999998},"last":1.0740000000000001},{"interface":"front","average":{"1min":0.98699999999999999,"5min":0.96599999999999997,"15min":0.95999999999999996},"min":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"max":{"1min":1.1599999999999999,"5min":1.585,"15min":1.585},"last":1.1399999999999999}]},{"osd":5,"last update":"Sun Oct 1 10:16:32 2023","interfaces":[{"interface":"back","average":{"1min":0.98599999999999999,"5min":0.90600000000000003,"15min":0.81899999999999995},"min":{"1min":0.68600000000000005,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":1.923,"5min":1.923,"15min":1.923},"last":1.1679999999999999},{"interface":"front","average":{"1min":1.095,"5min":0.98099999999999998,"15min":0.88100000000000001},"min":{"1min":0.79600000000000004,"5min":0.51000000000000001,"15min":0.51000000000000001},"max":{"1min":2.3780000000000001,"5min":2.3780000000000001,"15min":2.3780000000000001},"last":1.1839999999999999}]},{"osd":6,"last update":"Sun Oct 1 10:16:16 2023","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":0.871,"15min":0.83399999999999996},"min":{"1min":0.625,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":1.232,"5min":1.3360000000000001,"15min":1.3360000000000001},"last":1.024},{"interface":"front","average":{"1min":1.085,"5min":0.97299999999999998,"15min":0.93200000000000005},"min":{"1min":0.75900000000000001,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":2.8599999999999999,"5min":2.8599999999999999,"15min":2.8599999999999999},"last":1.268}]},{"osd":7,"last update":"Sun Oct 1 10:16:52 2023","interfaces":[{"interface":"back","average":{"1min":0.98999999999999999,"5min":0.97199999999999998,"15min":0.95099999999999996},"min":{"1min":0.80700000000000005,"5min":0.64400000000000002,"15min":0.64400000000000002},"max":{"1min":1.3240000000000001,"5min":2.4550000000000001,"15min":2.4550000000000001},"last":1.2470000000000001},{"interface":"front","average":{"1min":1.0820000000000001,"5min":0.98999999999999999,"15min":0.93799999999999994},"min":{"1min":0.84799999999999998,"5min":0.495,"15min":0.495},"max":{"1min":1.3620000000000001,"5min":1.3620000000000001,"15min":1.3620000000000001},"last":1.228}]}]},{"osd":5,"up_from":36,"seq":154618822728,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297712,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":297088,"kb_avail":93447440,"statfs":{"total":95995035648,"available":95690178560,"internally_reserved":0,"allocated":630784,"data_stored":270257,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304218112},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:16:34 2023","interfaces":[{"interface":"back","average":{"1min":0.91000000000000003,"5min":0.877,"15min":0.89700000000000002},"min":{"1min":0.61499999999999999,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":1.1539999999999999,"5min":1.339,"15min":1.339},"last":0.65700000000000003},{"interface":"front","average":{"1min":0.87,"5min":0.84199999999999997,"15min":0.86899999999999999},"min":{"1min":0.64700000000000002,"5min":0.53100000000000003,"15min":0.53100000000000003},"max":{"1min":1.167,"5min":1.1679999999999999,"15min":1.1679999999999999},"last":0.98199999999999998}]},{"osd":1,"last update":"Sun Oct 1 10:16:34 2023","interfaces":[{"interface":"back","average":{"1min":0.98599999999999999,"5min":0.88400000000000001,"15min":0.86199999999999999},"min":{"1min":0.58999999999999997,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":1.282,"5min":1.282,"15min":1.282},"last":1.028},{"interface":"front","average":{"1min":0.91800000000000004,"5min":0.90100000000000002,"15min":0.90900000000000003},"min":{"1min":0.63100000000000001,"5min":0.57699999999999996,"15min":0.57699999999999996},"max":{"1min":1.1879999999999999,"5min":1.266,"15min":1.266},"last":0.79900000000000004}]},{"osd":2,"last update":"Sun Oct 1 10:16:34 2023","interfaces":[{"interface":"back","average":{"1min":0.92000000000000004,"5min":0.879,"15min":0.86699999999999999},"min":{"1min":0.749,"5min":0.48999999999999999,"15min":0.48999999999999999},"max":{"1min":1.1739999999999999,"5min":1.1739999999999999,"15min":1.1739999999999999},"last":0.72199999999999998},{"interface":"front","average":{"1min":0.98999999999999999,"5min":0.93300000000000005,"15min":0.92400000000000004},"min":{"1min":0.60499999999999998,"5min":0.57199999999999995,"15min":0.57199999999999995},"max":{"1min":1.228,"5min":1.228,"15min":1.228},"last":1.1200000000000001}]},{"osd":3,"last update":"Sun Oct 1 10:16:34 2023","interfaces":[{"interface":"back","average":{"1min":0.97299999999999998,"5min":0.92200000000000004,"15min":0.92000000000000004},"min":{"1min":0.80000000000000004,"5min":0.60699999999999998,"15min":0.60699999999999998},"max":{"1min":1.2030000000000001,"5min":1.236,"15min":1.236},"last":1.0129999999999999},{"interface":"front","average":{"1min":0.94799999999999995,"5min":0.90900000000000003,"15min":0.90900000000000003},"min":{"1min":0.63700000000000001,"5min":0.58099999999999996,"15min":0.58099999999999996},"max":{"1min":1.196,"5min":1.2490000000000001,"15min":1.2490000000000001},"last":1.083}]},{"osd":4,"last update":"Sun Oct 1 10:16:34 2023","interfaces":[{"interface":"back","average":{"1min":0.93100000000000005,"5min":0.84699999999999998,"15min":0.76200000000000001},"min":{"1min":0.60899999999999999,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":1.286,"5min":1.286,"15min":1.286},"last":0.93100000000000005},{"interface":"front","average":{"1min":0.91900000000000004,"5min":0.93100000000000005,"15min":0.84099999999999997},"min":{"1min":0.50600000000000001,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":1.3069999999999999,"5min":1.431,"15min":1.431},"last":0.88600000000000001}]},{"osd":6,"last update":"Sun Oct 1 10:16:11 2023","interfaces":[{"interface":"back","average":{"1min":0.95699999999999996,"5min":0.93000000000000005,"15min":0.93000000000000005},"min":{"1min":0.66500000000000004,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":1.24,"5min":1.333,"15min":1.333},"last":1.2310000000000001},{"interface":"front","average":{"1min":0.92100000000000004,"5min":0.91300000000000003,"15min":0.89900000000000002},"min":{"1min":0.70099999999999996,"5min":0.434,"15min":0.434},"max":{"1min":1.117,"5min":1.2649999999999999,"15min":1.2649999999999999},"last":0.86499999999999999}]},{"osd":7,"last update":"Sun Oct 1 10:16:57 2023","interfaces":[{"interface":"back","average":{"1min":1.0129999999999999,"5min":0.95599999999999996,"15min":0.92200000000000004},"min":{"1min":0.81499999999999995,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":1.2569999999999999,"5min":1.3480000000000001,"15min":1.3480000000000001},"last":1.1779999999999999},{"interface":"front","average":{"1min":1.056,"5min":1.0349999999999999,"15min":1.0289999999999999},"min":{"1min":0.83099999999999996,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":1.2809999999999999,"5min":1.3959999999999999,"15min":1.3959999999999999},"last":1.1499999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T10:17:15.972 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-10-01T10:17:15.973 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-10-01T10:17:15.973 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-10-01T10:17:15.973 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph health --format=json 2023-10-01T10:17:16.289 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:17:16 smithi078 bash[27336]: audit 2023-10-01T10:17:15.183516+0000 mgr.y (mgr.14427) 153 : audit [DBG] from='client.24689 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:17:16.290 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:17:16 smithi078 bash[27336]: cluster 2023-10-01T10:17:15.226135+0000 mgr.y (mgr.14427) 154 : cluster [DBG] pgmap v133: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:16.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:17:16 smithi078 bash[33821]: audit 2023-10-01T10:17:15.183516+0000 mgr.y (mgr.14427) 153 : audit [DBG] from='client.24689 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:17:16.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:17:16 smithi078 bash[33821]: cluster 2023-10-01T10:17:15.226135+0000 mgr.y (mgr.14427) 154 : cluster [DBG] pgmap v133: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:16.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:17:16 smithi173 bash[27612]: audit 2023-10-01T10:17:15.183516+0000 mgr.y (mgr.14427) 153 : audit [DBG] from='client.24689 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:17:16.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:17:16 smithi173 bash[27612]: cluster 2023-10-01T10:17:15.226135+0000 mgr.y (mgr.14427) 154 : cluster [DBG] pgmap v133: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:18.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:17:18 smithi078 bash[27336]: cluster 2023-10-01T10:17:17.226644+0000 mgr.y (mgr.14427) 155 : cluster [DBG] pgmap v134: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:18.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:17:18 smithi078 bash[33821]: cluster 2023-10-01T10:17:17.226644+0000 mgr.y (mgr.14427) 155 : cluster [DBG] pgmap v134: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:18.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:17:18 smithi173 bash[27612]: cluster 2023-10-01T10:17:17.226644+0000 mgr.y (mgr.14427) 155 : cluster [DBG] pgmap v134: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:20.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:17:20 smithi078 bash[27336]: cluster 2023-10-01T10:17:19.227651+0000 mgr.y (mgr.14427) 156 : cluster [DBG] pgmap v135: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:20.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:17:20 smithi078 bash[33821]: cluster 2023-10-01T10:17:19.227651+0000 mgr.y (mgr.14427) 156 : cluster [DBG] pgmap v135: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:20.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:17:20 smithi173 bash[27612]: cluster 2023-10-01T10:17:19.227651+0000 mgr.y (mgr.14427) 156 : cluster [DBG] pgmap v135: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:20.787 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:17:22.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:17:21 smithi078 bash[27336]: cluster 2023-10-01T10:17:21.228243+0000 mgr.y (mgr.14427) 157 : cluster [DBG] pgmap v136: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:22.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:17:21 smithi078 bash[33821]: cluster 2023-10-01T10:17:21.228243+0000 mgr.y (mgr.14427) 157 : cluster [DBG] pgmap v136: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:22.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:17:21 smithi173 bash[27612]: cluster 2023-10-01T10:17:21.228243+0000 mgr.y (mgr.14427) 157 : cluster [DBG] pgmap v136: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:23.970 INFO:teuthology.orchestra.run.smithi078.stdout: 2023-10-01T10:17:24.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:17:24 smithi078 bash[33821]: cluster 2023-10-01T10:17:23.228841+0000 mgr.y (mgr.14427) 158 : cluster [DBG] pgmap v137: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:24.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:17:24 smithi078 bash[33821]: audit 2023-10-01T10:17:23.967929+0000 mon.c (mon.2) 35 : audit [DBG] from='client.? 172.21.15.78:0/557772358' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-01T10:17:24.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:17:24 smithi078 bash[27336]: cluster 2023-10-01T10:17:23.228841+0000 mgr.y (mgr.14427) 158 : cluster [DBG] pgmap v137: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:24.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:17:24 smithi078 bash[27336]: audit 2023-10-01T10:17:23.967929+0000 mon.c (mon.2) 35 : audit [DBG] from='client.? 172.21.15.78:0/557772358' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-01T10:17:24.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:17:24 smithi173 bash[27612]: cluster 2023-10-01T10:17:23.228841+0000 mgr.y (mgr.14427) 158 : cluster [DBG] pgmap v137: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:24.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:17:24 smithi173 bash[27612]: audit 2023-10-01T10:17:23.967929+0000 mon.c (mon.2) 35 : audit [DBG] from='client.? 172.21.15.78:0/557772358' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-01T10:17:24.690 INFO:teuthology.orchestra.run.smithi078.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-10-01T10:17:24.690 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-10-01T10:17:24.691 INFO:teuthology.run_tasks:Running task print... 2023-10-01T10:17:24.700 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2023-10-01T10:17:24.700 INFO:teuthology.run_tasks:Running task parallel... 2023-10-01T10:17:24.720 INFO:teuthology.task.parallel:starting parallel... 2023-10-01T10:17:24.720 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-10-01T10:17:24.721 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2023-10-01T10:17:24.721 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-01T10:17:24.723 INFO:tasks.radosbench:Beginning radosbench... 2023-10-01T10:17:24.723 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_0 2023-10-01T10:17:24.723 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_0 16 2023-10-01T10:17:24.724 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-10-01T10:17:24.724 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2023-10-01T10:17:24.724 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi078.front.sepia.ceph.com 2023-10-01T10:17:24.724 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -e sha1=b56822191ad0416d0c05ed59720df03ad9c2a44e -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2023-10-01T10:17:25.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:17:25 smithi078 bash[27601]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:17:25] "GET /metrics HTTP/1.1" 200 191399 "" "Prometheus/2.33.4" 2023-10-01T10:17:25.620 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:17:25 smithi173 bash[28473]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:17:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T10:17:26.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:17:26 smithi078 bash[33821]: cluster 2023-10-01T10:17:25.229448+0000 mgr.y (mgr.14427) 159 : cluster [DBG] pgmap v138: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:26.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:17:26 smithi078 bash[27336]: cluster 2023-10-01T10:17:25.229448+0000 mgr.y (mgr.14427) 159 : cluster [DBG] pgmap v138: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:26.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:17:26 smithi173 bash[27612]: cluster 2023-10-01T10:17:25.229448+0000 mgr.y (mgr.14427) 159 : cluster [DBG] pgmap v138: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:28.295 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:17:27 smithi078 bash[27601]: [01/Oct/2023:10:17:27] ENGINE Bus STOPPING 2023-10-01T10:17:28.296 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:17:27 smithi078 bash[27601]: [01/Oct/2023:10:17:27] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-01T10:17:28.296 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:17:27 smithi078 bash[27601]: [01/Oct/2023:10:17:27] ENGINE Bus STOPPED 2023-10-01T10:17:28.296 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:17:27 smithi078 bash[27601]: [01/Oct/2023:10:17:27] ENGINE Bus STARTING 2023-10-01T10:17:28.296 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:17:28 smithi078 bash[27601]: [01/Oct/2023:10:17:28] ENGINE Serving on http://:::9283 2023-10-01T10:17:28.296 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:17:28 smithi078 bash[27601]: [01/Oct/2023:10:17:28] ENGINE Bus STARTED 2023-10-01T10:17:28.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:17:28 smithi078 bash[33821]: cluster 2023-10-01T10:17:27.230028+0000 mgr.y (mgr.14427) 160 : cluster [DBG] pgmap v139: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:28.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:17:28 smithi078 bash[33821]: audit 2023-10-01T10:17:27.912382+0000 mon.a (mon.0) 723 : audit [INF] from='client.? 172.21.15.78:0/3883461224' entity='client.admin' 2023-10-01T10:17:28.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:17:28 smithi078 bash[33821]: audit 2023-10-01T10:17:27.932195+0000 mon.a (mon.0) 724 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:17:28.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:17:28 smithi078 bash[27336]: cluster 2023-10-01T10:17:27.230028+0000 mgr.y (mgr.14427) 160 : cluster [DBG] pgmap v139: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:28.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:17:28 smithi078 bash[27336]: audit 2023-10-01T10:17:27.912382+0000 mon.a (mon.0) 723 : audit [INF] from='client.? 172.21.15.78:0/3883461224' entity='client.admin' 2023-10-01T10:17:28.585 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:17:28 smithi078 bash[27336]: audit 2023-10-01T10:17:27.932195+0000 mon.a (mon.0) 724 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:17:28.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:17:28 smithi173 bash[27612]: cluster 2023-10-01T10:17:27.230028+0000 mgr.y (mgr.14427) 160 : cluster [DBG] pgmap v139: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:28.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:17:28 smithi173 bash[27612]: audit 2023-10-01T10:17:27.912382+0000 mon.a (mon.0) 723 : audit [INF] from='client.? 172.21.15.78:0/3883461224' entity='client.admin' 2023-10-01T10:17:28.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:17:28 smithi173 bash[27612]: audit 2023-10-01T10:17:27.932195+0000 mon.a (mon.0) 724 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:17:28.677 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -e sha1=b56822191ad0416d0c05ed59720df03ad9c2a44e -- bash -c 'ceph config set global log_to_journald false --force' 2023-10-01T10:17:30.170 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:17:30.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:17:30 smithi078 bash[27336]: cluster 2023-10-01T10:17:29.230689+0000 mgr.y (mgr.14427) 161 : cluster [DBG] pgmap v140: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:30.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:17:30 smithi078 bash[33821]: cluster 2023-10-01T10:17:29.230689+0000 mgr.y (mgr.14427) 161 : cluster [DBG] pgmap v140: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:30.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:17:30 smithi173 bash[27612]: cluster 2023-10-01T10:17:29.230689+0000 mgr.y (mgr.14427) 161 : cluster [DBG] pgmap v140: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:32.232 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:17:32 smithi078 bash[27336]: cluster 2023-10-01T10:17:31.231020+0000 mgr.y (mgr.14427) 162 : cluster [DBG] pgmap v141: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:32.232 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:17:32 smithi078 bash[33821]: cluster 2023-10-01T10:17:31.231020+0000 mgr.y (mgr.14427) 162 : cluster [DBG] pgmap v141: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:32.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:17:32 smithi173 bash[27612]: cluster 2023-10-01T10:17:31.231020+0000 mgr.y (mgr.14427) 162 : cluster [DBG] pgmap v141: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:33.072 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:17:32 smithi078 bash[27601]: [01/Oct/2023:10:17:32] ENGINE Bus STOPPING 2023-10-01T10:17:33.072 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:17:32 smithi078 bash[27601]: [01/Oct/2023:10:17:32] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-01T10:17:33.072 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:17:32 smithi078 bash[27601]: [01/Oct/2023:10:17:32] ENGINE Bus STOPPED 2023-10-01T10:17:33.072 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:17:32 smithi078 bash[27601]: [01/Oct/2023:10:17:32] ENGINE Bus STARTING 2023-10-01T10:17:33.072 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:17:32 smithi078 bash[27601]: [01/Oct/2023:10:17:32] ENGINE Serving on http://:::9283 2023-10-01T10:17:33.072 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:17:32 smithi078 bash[27601]: [01/Oct/2023:10:17:32] ENGINE Bus STARTED 2023-10-01T10:17:34.076 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:17:33 smithi078 bash[27336]: audit 2023-10-01T10:17:32.627091+0000 mon.a (mon.0) 725 : audit [INF] from='client.? ' entity='client.admin' 2023-10-01T10:17:34.076 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:17:33 smithi078 bash[33821]: audit 2023-10-01T10:17:32.627091+0000 mon.a (mon.0) 725 : audit [INF] from='client.? ' entity='client.admin' 2023-10-01T10:17:34.077 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -e sha1=b56822191ad0416d0c05ed59720df03ad9c2a44e -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2023-10-01T10:17:34.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:17:33 smithi173 bash[27612]: audit 2023-10-01T10:17:32.627091+0000 mon.a (mon.0) 725 : audit [INF] from='client.? ' entity='client.admin' 2023-10-01T10:17:35.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:17:34 smithi078 bash[27336]: cluster 2023-10-01T10:17:33.231587+0000 mgr.y (mgr.14427) 163 : cluster [DBG] pgmap v142: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:35.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:17:34 smithi078 bash[27336]: audit 2023-10-01T10:17:33.877713+0000 mon.a (mon.0) 726 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:17:35.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:17:34 smithi078 bash[33821]: cluster 2023-10-01T10:17:33.231587+0000 mgr.y (mgr.14427) 163 : cluster [DBG] pgmap v142: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:35.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:17:34 smithi078 bash[33821]: audit 2023-10-01T10:17:33.877713+0000 mon.a (mon.0) 726 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:17:35.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:17:34 smithi173 bash[27612]: cluster 2023-10-01T10:17:33.231587+0000 mgr.y (mgr.14427) 163 : cluster [DBG] pgmap v142: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:35.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:17:34 smithi173 bash[27612]: audit 2023-10-01T10:17:33.877713+0000 mon.a (mon.0) 726 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:17:35.365 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:17:35 smithi078 bash[27601]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:17:35] "GET /metrics HTTP/1.1" 200 191400 "" "Prometheus/2.33.4" 2023-10-01T10:17:35.620 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:17:35 smithi173 bash[28473]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:17:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T10:17:36.488 INFO:teuthology.orchestra.run.smithi078.stderr:pool 'unique_pool_0' created 2023-10-01T10:17:36.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:17:36 smithi078 bash[27336]: cluster 2023-10-01T10:17:35.231995+0000 mgr.y (mgr.14427) 164 : cluster [DBG] pgmap v143: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:37.019 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:17:36 smithi078 bash[27336]: audit 2023-10-01T10:17:35.441360+0000 mon.a (mon.0) 727 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:17:37.019 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:17:36 smithi078 bash[27336]: audit 2023-10-01T10:17:35.442809+0000 mon.a (mon.0) 728 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:17:37.019 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:17:36 smithi078 bash[27336]: audit 2023-10-01T10:17:35.444061+0000 mon.a (mon.0) 729 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:17:37.019 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:17:36 smithi078 bash[27336]: audit 2023-10-01T10:17:35.451530+0000 mon.a (mon.0) 730 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:17:37.019 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:17:36 smithi078 bash[27336]: audit 2023-10-01T10:17:35.459147+0000 mon.a (mon.0) 731 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:17:37.019 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:17:36 smithi078 bash[27336]: audit 2023-10-01T10:17:35.461152+0000 mon.a (mon.0) 732 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:17:37.019 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:17:36 smithi078 bash[27336]: audit 2023-10-01T10:17:35.462344+0000 mon.a (mon.0) 733 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:17:37.020 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:17:36 smithi078 bash[27336]: audit 2023-10-01T10:17:35.472913+0000 mon.a (mon.0) 734 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:17:37.020 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:17:36 smithi078 bash[27336]: audit 2023-10-01T10:17:35.602153+0000 mon.c (mon.2) 36 : audit [INF] from='client.? 172.21.15.78:0/589920535' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-10-01T10:17:37.020 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:17:36 smithi078 bash[27336]: audit 2023-10-01T10:17:35.602698+0000 mon.a (mon.0) 735 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-10-01T10:17:37.020 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:17:36 smithi078 bash[33821]: cluster 2023-10-01T10:17:35.231995+0000 mgr.y (mgr.14427) 164 : cluster [DBG] pgmap v143: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:37.020 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:17:36 smithi078 bash[33821]: audit 2023-10-01T10:17:35.441360+0000 mon.a (mon.0) 727 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:17:37.020 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:17:36 smithi078 bash[33821]: audit 2023-10-01T10:17:35.442809+0000 mon.a (mon.0) 728 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:17:37.020 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:17:36 smithi078 bash[33821]: audit 2023-10-01T10:17:35.444061+0000 mon.a (mon.0) 729 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:17:37.020 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:17:36 smithi078 bash[33821]: audit 2023-10-01T10:17:35.451530+0000 mon.a (mon.0) 730 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:17:37.020 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:17:36 smithi078 bash[33821]: audit 2023-10-01T10:17:35.459147+0000 mon.a (mon.0) 731 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:17:37.021 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:17:36 smithi078 bash[33821]: audit 2023-10-01T10:17:35.461152+0000 mon.a (mon.0) 732 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:17:37.021 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:17:36 smithi078 bash[33821]: audit 2023-10-01T10:17:35.462344+0000 mon.a (mon.0) 733 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:17:37.021 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:17:36 smithi078 bash[33821]: audit 2023-10-01T10:17:35.472913+0000 mon.a (mon.0) 734 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:17:37.021 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:17:36 smithi078 bash[33821]: audit 2023-10-01T10:17:35.602153+0000 mon.c (mon.2) 36 : audit [INF] from='client.? 172.21.15.78:0/589920535' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-10-01T10:17:37.021 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:17:36 smithi078 bash[33821]: audit 2023-10-01T10:17:35.602698+0000 mon.a (mon.0) 735 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-10-01T10:17:37.021 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:17:36 smithi173 bash[27612]: cluster 2023-10-01T10:17:35.231995+0000 mgr.y (mgr.14427) 164 : cluster [DBG] pgmap v143: 73 pgs: 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:37.021 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:17:36 smithi173 bash[27612]: audit 2023-10-01T10:17:35.441360+0000 mon.a (mon.0) 727 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:17:37.022 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:17:36 smithi173 bash[27612]: audit 2023-10-01T10:17:35.442809+0000 mon.a (mon.0) 728 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:17:37.022 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:17:36 smithi173 bash[27612]: audit 2023-10-01T10:17:35.444061+0000 mon.a (mon.0) 729 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:17:37.022 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:17:36 smithi173 bash[27612]: audit 2023-10-01T10:17:35.451530+0000 mon.a (mon.0) 730 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:17:37.022 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:17:36 smithi173 bash[27612]: audit 2023-10-01T10:17:35.459147+0000 mon.a (mon.0) 731 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:17:37.022 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:17:36 smithi173 bash[27612]: audit 2023-10-01T10:17:35.461152+0000 mon.a (mon.0) 732 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:17:37.022 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:17:36 smithi173 bash[27612]: audit 2023-10-01T10:17:35.462344+0000 mon.a (mon.0) 733 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:17:37.022 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:17:36 smithi173 bash[27612]: audit 2023-10-01T10:17:35.472913+0000 mon.a (mon.0) 734 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:17:37.022 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:17:36 smithi173 bash[27612]: audit 2023-10-01T10:17:35.602153+0000 mon.c (mon.2) 36 : audit [INF] from='client.? 172.21.15.78:0/589920535' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-10-01T10:17:37.022 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:17:36 smithi173 bash[27612]: audit 2023-10-01T10:17:35.602698+0000 mon.a (mon.0) 735 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-10-01T10:17:37.825 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:17:37 smithi078 bash[27336]: audit 2023-10-01T10:17:36.478737+0000 mon.a (mon.0) 736 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-10-01T10:17:37.825 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:17:37 smithi078 bash[27336]: cluster 2023-10-01T10:17:36.478955+0000 mon.a (mon.0) 737 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2023-10-01T10:17:37.825 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:17:37 smithi078 bash[33821]: audit 2023-10-01T10:17:36.478737+0000 mon.a (mon.0) 736 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-10-01T10:17:37.825 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:17:37 smithi078 bash[33821]: cluster 2023-10-01T10:17:36.478955+0000 mon.a (mon.0) 737 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2023-10-01T10:17:37.826 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_0 rados --yes-i-really-mean-it || true 2023-10-01T10:17:37.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:17:37 smithi173 bash[27612]: audit 2023-10-01T10:17:36.478737+0000 mon.a (mon.0) 736 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-10-01T10:17:37.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:17:37 smithi173 bash[27612]: cluster 2023-10-01T10:17:36.478955+0000 mon.a (mon.0) 737 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2023-10-01T10:17:38.313 INFO:teuthology.orchestra.run.smithi078.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:b56822191ad0416d0c05ed59720df03ad9c2a44e 2023-10-01T10:17:38.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:17:38 smithi078 bash[27336]: cluster 2023-10-01T10:17:37.232599+0000 mgr.y (mgr.14427) 165 : cluster [DBG] pgmap v145: 89 pgs: 16 unknown, 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:38.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:17:38 smithi078 bash[27336]: cluster 2023-10-01T10:17:37.483639+0000 mon.a (mon.0) 738 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2023-10-01T10:17:38.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:17:38 smithi078 bash[27336]: audit 2023-10-01T10:17:38.309936+0000 mon.a (mon.0) 739 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:17:38.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:17:38 smithi078 bash[27336]: audit 2023-10-01T10:17:38.353910+0000 mon.a (mon.0) 740 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:17:38.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:17:38 smithi078 bash[27336]: audit 2023-10-01T10:17:38.356172+0000 mon.a (mon.0) 741 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:17:38.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:17:38 smithi078 bash[27336]: audit 2023-10-01T10:17:38.357098+0000 mon.a (mon.0) 742 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:17:38.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:17:38 smithi078 bash[27336]: audit 2023-10-01T10:17:38.366218+0000 mon.a (mon.0) 743 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:17:38.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:17:38 smithi078 bash[33821]: cluster 2023-10-01T10:17:37.232599+0000 mgr.y (mgr.14427) 165 : cluster [DBG] pgmap v145: 89 pgs: 16 unknown, 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:38.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:17:38 smithi078 bash[33821]: cluster 2023-10-01T10:17:37.483639+0000 mon.a (mon.0) 738 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2023-10-01T10:17:38.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:17:38 smithi078 bash[33821]: audit 2023-10-01T10:17:38.309936+0000 mon.a (mon.0) 739 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:17:38.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:17:38 smithi078 bash[33821]: audit 2023-10-01T10:17:38.353910+0000 mon.a (mon.0) 740 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:17:38.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:17:38 smithi078 bash[33821]: audit 2023-10-01T10:17:38.356172+0000 mon.a (mon.0) 741 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:17:38.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:17:38 smithi078 bash[33821]: audit 2023-10-01T10:17:38.357098+0000 mon.a (mon.0) 742 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:17:38.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:17:38 smithi078 bash[33821]: audit 2023-10-01T10:17:38.366218+0000 mon.a (mon.0) 743 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:17:38.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:17:38 smithi173 bash[27612]: cluster 2023-10-01T10:17:37.232599+0000 mgr.y (mgr.14427) 165 : cluster [DBG] pgmap v145: 89 pgs: 16 unknown, 73 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:38.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:17:38 smithi173 bash[27612]: cluster 2023-10-01T10:17:37.483639+0000 mon.a (mon.0) 738 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2023-10-01T10:17:38.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:17:38 smithi173 bash[27612]: audit 2023-10-01T10:17:38.309936+0000 mon.a (mon.0) 739 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:17:38.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:17:38 smithi173 bash[27612]: audit 2023-10-01T10:17:38.353910+0000 mon.a (mon.0) 740 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:17:38.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:17:38 smithi173 bash[27612]: audit 2023-10-01T10:17:38.356172+0000 mon.a (mon.0) 741 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:17:38.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:17:38 smithi173 bash[27612]: audit 2023-10-01T10:17:38.357098+0000 mon.a (mon.0) 742 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:17:38.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:17:38 smithi173 bash[27612]: audit 2023-10-01T10:17:38.366218+0000 mon.a (mon.0) 743 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:17:39.363 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -e sha1=b56822191ad0416d0c05ed59720df03ad9c2a44e -- bash -c 'ceph orch ps' 2023-10-01T10:17:39.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:17:39 smithi078 bash[27336]: audit 2023-10-01T10:17:38.304557+0000 mgr.y (mgr.14427) 166 : audit [DBG] from='client.24719 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b56822191ad0416d0c05ed59720df03ad9c2a44e", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:17:39.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:17:39 smithi078 bash[27336]: cephadm 2023-10-01T10:17:38.305329+0000 mgr.y (mgr.14427) 167 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b56822191ad0416d0c05ed59720df03ad9c2a44e 2023-10-01T10:17:39.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:17:39 smithi078 bash[27336]: cephadm 2023-10-01T10:17:38.372642+0000 mgr.y (mgr.14427) 168 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b56822191ad0416d0c05ed59720df03ad9c2a44e 2023-10-01T10:17:39.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:17:39 smithi078 bash[27336]: cluster 2023-10-01T10:17:38.486758+0000 mon.a (mon.0) 744 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2023-10-01T10:17:39.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:17:39 smithi078 bash[33821]: audit 2023-10-01T10:17:38.304557+0000 mgr.y (mgr.14427) 166 : audit [DBG] from='client.24719 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b56822191ad0416d0c05ed59720df03ad9c2a44e", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:17:39.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:17:39 smithi078 bash[33821]: cephadm 2023-10-01T10:17:38.305329+0000 mgr.y (mgr.14427) 167 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b56822191ad0416d0c05ed59720df03ad9c2a44e 2023-10-01T10:17:39.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:17:39 smithi078 bash[33821]: cephadm 2023-10-01T10:17:38.372642+0000 mgr.y (mgr.14427) 168 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b56822191ad0416d0c05ed59720df03ad9c2a44e 2023-10-01T10:17:39.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:17:39 smithi078 bash[33821]: cluster 2023-10-01T10:17:38.486758+0000 mon.a (mon.0) 744 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2023-10-01T10:17:39.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:17:39 smithi173 bash[27612]: audit 2023-10-01T10:17:38.304557+0000 mgr.y (mgr.14427) 166 : audit [DBG] from='client.24719 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b56822191ad0416d0c05ed59720df03ad9c2a44e", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:17:39.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:17:39 smithi173 bash[27612]: cephadm 2023-10-01T10:17:38.305329+0000 mgr.y (mgr.14427) 167 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b56822191ad0416d0c05ed59720df03ad9c2a44e 2023-10-01T10:17:39.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:17:39 smithi173 bash[27612]: cephadm 2023-10-01T10:17:38.372642+0000 mgr.y (mgr.14427) 168 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b56822191ad0416d0c05ed59720df03ad9c2a44e 2023-10-01T10:17:39.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:17:39 smithi173 bash[27612]: cluster 2023-10-01T10:17:38.486758+0000 mon.a (mon.0) 744 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2023-10-01T10:17:40.758 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:17:40 smithi078 bash[33821]: cluster 2023-10-01T10:17:39.233107+0000 mgr.y (mgr.14427) 169 : cluster [DBG] pgmap v148: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:40.759 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:17:40 smithi078 bash[27336]: cluster 2023-10-01T10:17:39.233107+0000 mgr.y (mgr.14427) 169 : cluster [DBG] pgmap v148: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:40.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:17:40 smithi173 bash[27612]: cluster 2023-10-01T10:17:39.233107+0000 mgr.y (mgr.14427) 169 : cluster [DBG] pgmap v148: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:42.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:17:42 smithi078 bash[27336]: cluster 2023-10-01T10:17:41.233572+0000 mgr.y (mgr.14427) 170 : cluster [DBG] pgmap v149: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:42.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:17:42 smithi078 bash[33821]: cluster 2023-10-01T10:17:41.233572+0000 mgr.y (mgr.14427) 170 : cluster [DBG] pgmap v149: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:42.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:17:42 smithi173 bash[27612]: cluster 2023-10-01T10:17:41.233572+0000 mgr.y (mgr.14427) 170 : cluster [DBG] pgmap v149: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:42.884 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:17:43.244 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-01T10:17:43.245 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.a smithi078 *:9093,9094 running (2m) 7s ago 3m 12.8M - 0.23.0 ba2b418f427c bd6f258697c7 2023-10-01T10:17:43.245 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.a smithi173 *:3000 running (3m) 9s ago 3m 43.2M - 8.3.5 dad864ee21e9 1b3629350a14 2023-10-01T10:17:43.245 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foo.smithi078.vjqrth smithi078 running (2m) 7s ago 2m 13.4M - 16.2.14-44-geeb13ac9 94a6c4498547 aedf3352e433 2023-10-01T10:17:43.245 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foo.smithi173.vrqqoc smithi173 running (117s) 9s ago 117s 11.5M - 16.2.14-44-geeb13ac9 94a6c4498547 3815ca08717b 2023-10-01T10:17:43.245 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.x smithi173 *:8443 running (10m) 9s ago 10m 377M - 16.2.14-44-geeb13ac9 94a6c4498547 98f9f045e4d5 2023-10-01T10:17:43.245 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.y smithi078 *:9283 running (12m) 7s ago 12m 426M - 16.2.14-44-geeb13ac9 94a6c4498547 f2c6b6b2b335 2023-10-01T10:17:43.245 INFO:teuthology.orchestra.run.smithi078.stdout:mon.a smithi078 running (12m) 7s ago 12m 50.1M 2048M 16.2.14-44-geeb13ac9 94a6c4498547 6de9da5ca74a 2023-10-01T10:17:43.245 INFO:teuthology.orchestra.run.smithi078.stdout:mon.b smithi173 running (10m) 9s ago 11m 42.6M 2048M 16.2.14-44-geeb13ac9 94a6c4498547 9ab8649fee56 2023-10-01T10:17:43.245 INFO:teuthology.orchestra.run.smithi078.stdout:mon.c smithi078 running (10m) 7s ago 10m 42.7M 2048M 16.2.14-44-geeb13ac9 94a6c4498547 39d4065bb273 2023-10-01T10:17:43.245 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.a smithi078 *:9100 running (4m) 7s ago 4m 9451k - 1.3.1 1dbe0e931976 1ee8f68e9d28 2023-10-01T10:17:43.245 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.b smithi173 *:9100 running (4m) 9s ago 4m 9652k - 1.3.1 1dbe0e931976 bafc9a69a8ba 2023-10-01T10:17:43.246 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (10m) 7s ago 10m 39.6M 4060M 16.2.14-44-geeb13ac9 94a6c4498547 f6a90fa5df1a 2023-10-01T10:17:43.246 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (9m) 7s ago 9m 37.8M 4060M 16.2.14-44-geeb13ac9 94a6c4498547 da5330a1eff3 2023-10-01T10:17:43.246 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (8m) 7s ago 8m 37.2M 4060M 16.2.14-44-geeb13ac9 94a6c4498547 685fab0bfbf6 2023-10-01T10:17:43.246 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (7m) 7s ago 7m 38.9M 4060M 16.2.14-44-geeb13ac9 94a6c4498547 c3823edbc5f1 2023-10-01T10:17:43.246 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi173 running (7m) 9s ago 7m 38.4M 4316M 16.2.14-44-geeb13ac9 94a6c4498547 f6a3e2fc7289 2023-10-01T10:17:43.246 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi173 running (6m) 9s ago 6m 37.3M 4316M 16.2.14-44-geeb13ac9 94a6c4498547 b4ab64b3b760 2023-10-01T10:17:43.246 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi173 running (5m) 9s ago 5m 37.5M 4316M 16.2.14-44-geeb13ac9 94a6c4498547 67631580752d 2023-10-01T10:17:43.246 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi173 running (5m) 9s ago 5m 38.1M 4316M 16.2.14-44-geeb13ac9 94a6c4498547 86157c8ed4b2 2023-10-01T10:17:43.246 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.a smithi173 *:9095 running (2m) 9s ago 4m 51.8M - 2.33.4 514e6a882f6e f4d009830691 2023-10-01T10:17:44.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:17:44 smithi078 bash[27336]: audit 2023-10-01T10:17:43.223892+0000 mgr.y (mgr.14427) 171 : audit [DBG] from='client.24725 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:17:44.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:17:44 smithi078 bash[27336]: cluster 2023-10-01T10:17:43.234463+0000 mgr.y (mgr.14427) 172 : cluster [DBG] pgmap v150: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:44.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:17:44 smithi078 bash[33821]: audit 2023-10-01T10:17:43.223892+0000 mgr.y (mgr.14427) 171 : audit [DBG] from='client.24725 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:17:44.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:17:44 smithi078 bash[33821]: cluster 2023-10-01T10:17:43.234463+0000 mgr.y (mgr.14427) 172 : cluster [DBG] pgmap v150: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:44.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:17:44 smithi173 bash[27612]: audit 2023-10-01T10:17:43.223892+0000 mgr.y (mgr.14427) 171 : audit [DBG] from='client.24725 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:17:44.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:17:44 smithi173 bash[27612]: cluster 2023-10-01T10:17:43.234463+0000 mgr.y (mgr.14427) 172 : cluster [DBG] pgmap v150: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:45.023 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -e sha1=b56822191ad0416d0c05ed59720df03ad9c2a44e -- bash -c 'echo wait for minority of mons to upgrade' 2023-10-01T10:17:45.289 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:17:45 smithi078 bash[27601]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:17:45] "GET /metrics HTTP/1.1" 200 191400 "" "Prometheus/2.33.4" 2023-10-01T10:17:45.620 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:17:45 smithi173 bash[28473]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:17:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T10:17:46.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:17:46 smithi078 bash[33821]: cluster 2023-10-01T10:17:45.234889+0000 mgr.y (mgr.14427) 173 : cluster [DBG] pgmap v151: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:46.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:17:46 smithi078 bash[27336]: cluster 2023-10-01T10:17:45.234889+0000 mgr.y (mgr.14427) 173 : cluster [DBG] pgmap v151: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:46.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:17:46 smithi173 bash[27612]: cluster 2023-10-01T10:17:45.234889+0000 mgr.y (mgr.14427) 173 : cluster [DBG] pgmap v151: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:48.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:17:48 smithi078 bash[27336]: cluster 2023-10-01T10:17:47.235312+0000 mgr.y (mgr.14427) 174 : cluster [DBG] pgmap v152: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:48.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:17:48 smithi078 bash[33821]: cluster 2023-10-01T10:17:47.235312+0000 mgr.y (mgr.14427) 174 : cluster [DBG] pgmap v152: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:48.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:17:48 smithi173 bash[27612]: cluster 2023-10-01T10:17:47.235312+0000 mgr.y (mgr.14427) 174 : cluster [DBG] pgmap v152: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:50.310 INFO:teuthology.orchestra.run.smithi078.stderr:enabled application 'rados' on pool 'unique_pool_0' 2023-10-01T10:17:50.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:17:50 smithi078 bash[27336]: cluster 2023-10-01T10:17:49.236278+0000 mgr.y (mgr.14427) 175 : cluster [DBG] pgmap v153: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:50.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:17:50 smithi078 bash[27336]: audit 2023-10-01T10:17:50.236110+0000 mon.a (mon.0) 745 : audit [INF] from='client.? 172.21.15.78:0/1765071448' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T10:17:50.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:17:50 smithi078 bash[33821]: cluster 2023-10-01T10:17:49.236278+0000 mgr.y (mgr.14427) 175 : cluster [DBG] pgmap v153: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:50.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:17:50 smithi078 bash[33821]: audit 2023-10-01T10:17:50.236110+0000 mon.a (mon.0) 745 : audit [INF] from='client.? 172.21.15.78:0/1765071448' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T10:17:50.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:17:50 smithi173 bash[27612]: cluster 2023-10-01T10:17:49.236278+0000 mgr.y (mgr.14427) 175 : cluster [DBG] pgmap v153: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:50.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:17:50 smithi173 bash[27612]: audit 2023-10-01T10:17:50.236110+0000 mon.a (mon.0) 745 : audit [INF] from='client.? 172.21.15.78:0/1765071448' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T10:17:51.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:17:51 smithi078 bash[27336]: audit 2023-10-01T10:17:50.304102+0000 mon.a (mon.0) 746 : audit [INF] from='client.? 172.21.15.78:0/1765071448' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-01T10:17:51.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:17:51 smithi078 bash[27336]: cluster 2023-10-01T10:17:50.304199+0000 mon.a (mon.0) 747 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2023-10-01T10:17:51.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:17:51 smithi078 bash[33821]: audit 2023-10-01T10:17:50.304102+0000 mon.a (mon.0) 746 : audit [INF] from='client.? 172.21.15.78:0/1765071448' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-01T10:17:51.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:17:51 smithi078 bash[33821]: cluster 2023-10-01T10:17:50.304199+0000 mon.a (mon.0) 747 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2023-10-01T10:17:51.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:17:51 smithi173 bash[27612]: audit 2023-10-01T10:17:50.304102+0000 mon.a (mon.0) 746 : audit [INF] from='client.? 172.21.15.78:0/1765071448' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-01T10:17:51.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:17:51 smithi173 bash[27612]: cluster 2023-10-01T10:17:50.304199+0000 mon.a (mon.0) 747 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2023-10-01T10:17:52.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:17:52 smithi078 bash[33821]: cluster 2023-10-01T10:17:51.236839+0000 mgr.y (mgr.14427) 176 : cluster [DBG] pgmap v155: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:52.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:17:52 smithi078 bash[27336]: cluster 2023-10-01T10:17:51.236839+0000 mgr.y (mgr.14427) 176 : cluster [DBG] pgmap v155: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:52.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:17:52 smithi173 bash[27612]: cluster 2023-10-01T10:17:51.236839+0000 mgr.y (mgr.14427) 176 : cluster [DBG] pgmap v155: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:54.571 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:17:54 smithi078 bash[33821]: cluster 2023-10-01T10:17:53.237486+0000 mgr.y (mgr.14427) 177 : cluster [DBG] pgmap v156: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:54.572 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:17:54 smithi078 bash[27336]: cluster 2023-10-01T10:17:53.237486+0000 mgr.y (mgr.14427) 177 : cluster [DBG] pgmap v156: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:54.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:17:54 smithi173 bash[27612]: cluster 2023-10-01T10:17:53.237486+0000 mgr.y (mgr.14427) 177 : cluster [DBG] pgmap v156: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:54.684 INFO:teuthology.orchestra.run.smithi078.stdout:wait for minority of mons to upgrade 2023-10-01T10:17:55.314 DEBUG:teuthology.orchestra.run.smithi078:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_0 bench 90 write' 2023-10-01T10:17:55.316 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-01T10:17:55.317 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-01T10:17:55.333 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:17:55 smithi078 bash[27601]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:17:55] "GET /metrics HTTP/1.1" 200 193815 "" "Prometheus/2.33.4" 2023-10-01T10:17:55.358 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:hints = 1 2023-10-01T10:17:55.358 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-01T10:17:55.358 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Object prefix: benchmark_data_smithi078_66777 2023-10-01T10:17:55.359 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T10:17:55.359 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 0 0 0 0 0 0 - 0 2023-10-01T10:17:55.863 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -e sha1=b56822191ad0416d0c05ed59720df03ad9c2a44e -- bash -c 'while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done' 2023-10-01T10:17:55.869 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:17:55 smithi173 bash[28473]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:17:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T10:17:56.359 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 1 16 2828 2812 175.741 175.75 0.00402276 0.00566548 2023-10-01T10:17:56.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:17:56 smithi078 bash[27336]: cluster 2023-10-01T10:17:55.237870+0000 mgr.y (mgr.14427) 178 : cluster [DBG] pgmap v157: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:56.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:17:56 smithi078 bash[33821]: cluster 2023-10-01T10:17:55.237870+0000 mgr.y (mgr.14427) 178 : cluster [DBG] pgmap v157: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:56.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:17:56 smithi173 bash[27612]: cluster 2023-10-01T10:17:55.237870+0000 mgr.y (mgr.14427) 178 : cluster [DBG] pgmap v157: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:57.359 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 2 16 5771 5755 179.829 183.938 0.0142989 0.00554146 2023-10-01T10:17:58.359 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 3 16 8537 8521 177.506 172.875 0.0115055 0.00562339 2023-10-01T10:17:58.586 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:17:58 smithi078 bash[27336]: cluster 2023-10-01T10:17:57.238327+0000 mgr.y (mgr.14427) 179 : cluster [DBG] pgmap v158: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:58.587 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:17:58 smithi078 bash[33821]: cluster 2023-10-01T10:17:57.238327+0000 mgr.y (mgr.14427) 179 : cluster [DBG] pgmap v158: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:58.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:17:58 smithi173 bash[27612]: cluster 2023-10-01T10:17:57.238327+0000 mgr.y (mgr.14427) 179 : cluster [DBG] pgmap v158: 89 pgs: 89 active+clean; 2.3 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:17:59.359 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 4 16 11023 11007 171.969 155.375 0.0056045 0.00580701 2023-10-01T10:18:00.360 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 5 16 13217 13201 164.998 137.125 0.012298 0.00605488 2023-10-01T10:18:00.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:18:00 smithi078 bash[27336]: cluster 2023-10-01T10:17:59.239602+0000 mgr.y (mgr.14427) 180 : cluster [DBG] pgmap v159: 89 pgs: 89 active+clean; 505 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 50 MiB/s wr, 807 op/s 2023-10-01T10:18:00.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:18:00 smithi078 bash[33821]: cluster 2023-10-01T10:17:59.239602+0000 mgr.y (mgr.14427) 180 : cluster [DBG] pgmap v159: 89 pgs: 89 active+clean; 505 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 50 MiB/s wr, 807 op/s 2023-10-01T10:18:00.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:18:00 smithi173 bash[27612]: cluster 2023-10-01T10:17:59.239602+0000 mgr.y (mgr.14427) 180 : cluster [DBG] pgmap v159: 89 pgs: 89 active+clean; 505 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 50 MiB/s wr, 807 op/s 2023-10-01T10:18:01.359 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 6 16 15368 15352 159.902 134.438 0.00310654 0.00624642 2023-10-01T10:18:02.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:18:02 smithi078 bash[27336]: cluster 2023-10-01T10:18:01.240267+0000 mgr.y (mgr.14427) 181 : cluster [DBG] pgmap v160: 89 pgs: 89 active+clean; 505 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 46 MiB/s wr, 738 op/s 2023-10-01T10:18:02.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:18:02 smithi078 bash[33821]: cluster 2023-10-01T10:18:01.240267+0000 mgr.y (mgr.14427) 181 : cluster [DBG] pgmap v160: 89 pgs: 89 active+clean; 505 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 46 MiB/s wr, 738 op/s 2023-10-01T10:18:02.360 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 7 16 17471 17455 155.834 131.438 0.0102956 0.00640579 2023-10-01T10:18:02.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:18:02 smithi173 bash[27612]: cluster 2023-10-01T10:18:01.240267+0000 mgr.y (mgr.14427) 181 : cluster [DBG] pgmap v160: 89 pgs: 89 active+clean; 505 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 46 MiB/s wr, 738 op/s 2023-10-01T10:18:03.363 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 8 16 19462 19446 151.908 124.438 0.00410737 0.00656724 2023-10-01T10:18:04.360 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 9 16 21467 21451 148.952 125.312 0.00325857 0.00670666 2023-10-01T10:18:04.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:18:04 smithi078 bash[33821]: cluster 2023-10-01T10:18:03.241023+0000 mgr.y (mgr.14427) 182 : cluster [DBG] pgmap v161: 89 pgs: 89 active+clean; 1.2 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 100 MiB/s wr, 1.59k op/s 2023-10-01T10:18:04.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:18:04 smithi078 bash[27336]: cluster 2023-10-01T10:18:03.241023+0000 mgr.y (mgr.14427) 182 : cluster [DBG] pgmap v161: 89 pgs: 89 active+clean; 1.2 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 100 MiB/s wr, 1.59k op/s 2023-10-01T10:18:04.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:18:04 smithi173 bash[27612]: cluster 2023-10-01T10:18:03.241023+0000 mgr.y (mgr.14427) 182 : cluster [DBG] pgmap v161: 89 pgs: 89 active+clean; 1.2 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 100 MiB/s wr, 1.59k op/s 2023-10-01T10:18:05.360 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 10 16 23301 23285 145.518 114.625 0.00328738 0.0068613 2023-10-01T10:18:05.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:18:05 smithi078 bash[27601]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:18:05] "GET /metrics HTTP/1.1" 200 194337 "" "Prometheus/2.33.4" 2023-10-01T10:18:05.620 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:18:05 smithi173 bash[28473]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:18:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T10:18:06.360 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 11 16 24851 24835 141.095 96.875 0.00963762 0.00708081 2023-10-01T10:18:06.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:18:06 smithi078 bash[27336]: cluster 2023-10-01T10:18:05.241440+0000 mgr.y (mgr.14427) 183 : cluster [DBG] pgmap v162: 89 pgs: 89 active+clean; 1.2 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 100 MiB/s wr, 1.59k op/s 2023-10-01T10:18:06.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:18:06 smithi078 bash[27336]: audit 2023-10-01T10:18:06.126162+0000 mon.c (mon.2) 37 : audit [DBG] from='client.? 172.21.15.78:0/684420824' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:18:06.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:18:06 smithi078 bash[33821]: cluster 2023-10-01T10:18:05.241440+0000 mgr.y (mgr.14427) 183 : cluster [DBG] pgmap v162: 89 pgs: 89 active+clean; 1.2 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 100 MiB/s wr, 1.59k op/s 2023-10-01T10:18:06.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:18:06 smithi078 bash[33821]: audit 2023-10-01T10:18:06.126162+0000 mon.c (mon.2) 37 : audit [DBG] from='client.? 172.21.15.78:0/684420824' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:18:06.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:18:06 smithi173 bash[27612]: cluster 2023-10-01T10:18:05.241440+0000 mgr.y (mgr.14427) 183 : cluster [DBG] pgmap v162: 89 pgs: 89 active+clean; 1.2 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 100 MiB/s wr, 1.59k op/s 2023-10-01T10:18:06.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:18:06 smithi173 bash[27612]: audit 2023-10-01T10:18:06.126162+0000 mon.c (mon.2) 37 : audit [DBG] from='client.? 172.21.15.78:0/684420824' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:18:07.360 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 12 16 26594 26578 138.414 108.938 0.00960288 0.00721689 2023-10-01T10:18:08.360 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 13 16 28157 28141 135.281 97.6875 0.00771374 0.007382 2023-10-01T10:18:08.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:18:08 smithi078 bash[27336]: cluster 2023-10-01T10:18:07.241938+0000 mgr.y (mgr.14427) 184 : cluster [DBG] pgmap v163: 89 pgs: 89 active+clean; 1.2 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 100 MiB/s wr, 1.59k op/s 2023-10-01T10:18:08.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:18:08 smithi078 bash[33821]: cluster 2023-10-01T10:18:07.241938+0000 mgr.y (mgr.14427) 184 : cluster [DBG] pgmap v163: 89 pgs: 89 active+clean; 1.2 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 100 MiB/s wr, 1.59k op/s 2023-10-01T10:18:08.622 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:18:08 smithi173 bash[27612]: cluster 2023-10-01T10:18:07.241938+0000 mgr.y (mgr.14427) 184 : cluster [DBG] pgmap v163: 89 pgs: 89 active+clean; 1.2 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 100 MiB/s wr, 1.59k op/s 2023-10-01T10:18:09.360 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 14 16 29662 29646 132.336 94.0625 0.00484739 0.00754901 2023-10-01T10:18:09.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:18:09 smithi078 bash[33821]: audit 2023-10-01T10:18:08.884873+0000 mon.c (mon.2) 38 : audit [DBG] from='client.? 172.21.15.78:0/737807883' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:18:09.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:18:09 smithi078 bash[27336]: audit 2023-10-01T10:18:08.884873+0000 mon.c (mon.2) 38 : audit [DBG] from='client.? 172.21.15.78:0/737807883' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:18:09.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:18:09 smithi173 bash[27612]: audit 2023-10-01T10:18:08.884873+0000 mon.c (mon.2) 38 : audit [DBG] from='client.? 172.21.15.78:0/737807883' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:18:10.360 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 15 16 31214 31198 129.98 97 0.00364933 0.00768626 2023-10-01T10:18:10.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:18:10 smithi078 bash[27336]: cluster 2023-10-01T10:18:09.243825+0000 mgr.y (mgr.14427) 185 : cluster [DBG] pgmap v164: 89 pgs: 89 active+clean; 1.7 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 145 MiB/s wr, 2.33k op/s 2023-10-01T10:18:10.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:18:10 smithi078 bash[33821]: cluster 2023-10-01T10:18:09.243825+0000 mgr.y (mgr.14427) 185 : cluster [DBG] pgmap v164: 89 pgs: 89 active+clean; 1.7 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 145 MiB/s wr, 2.33k op/s 2023-10-01T10:18:10.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:18:10 smithi173 bash[27612]: cluster 2023-10-01T10:18:09.243825+0000 mgr.y (mgr.14427) 185 : cluster [DBG] pgmap v164: 89 pgs: 89 active+clean; 1.7 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 145 MiB/s wr, 2.33k op/s 2023-10-01T10:18:11.361 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 16 16 32698 32682 127.652 92.75 0.00362784 0.00782495 2023-10-01T10:18:12.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:18:12 smithi078 bash[27336]: cluster 2023-10-01T10:18:11.244611+0000 mgr.y (mgr.14427) 186 : cluster [DBG] pgmap v165: 89 pgs: 89 active+clean; 1.7 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2023-10-01T10:18:12.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:18:12 smithi078 bash[27336]: audit 2023-10-01T10:18:11.608665+0000 mon.c (mon.2) 39 : audit [DBG] from='client.? 172.21.15.78:0/3424814594' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:18:12.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:18:12 smithi078 bash[33821]: cluster 2023-10-01T10:18:11.244611+0000 mgr.y (mgr.14427) 186 : cluster [DBG] pgmap v165: 89 pgs: 89 active+clean; 1.7 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2023-10-01T10:18:12.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:18:12 smithi078 bash[33821]: audit 2023-10-01T10:18:11.608665+0000 mon.c (mon.2) 39 : audit [DBG] from='client.? 172.21.15.78:0/3424814594' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:18:12.361 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 17 16 34168 34152 125.547 91.875 0.00714338 0.0079584 2023-10-01T10:18:12.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:18:12 smithi173 bash[27612]: cluster 2023-10-01T10:18:11.244611+0000 mgr.y (mgr.14427) 186 : cluster [DBG] pgmap v165: 89 pgs: 89 active+clean; 1.7 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2023-10-01T10:18:12.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:18:12 smithi173 bash[27612]: audit 2023-10-01T10:18:11.608665+0000 mon.c (mon.2) 39 : audit [DBG] from='client.? 172.21.15.78:0/3424814594' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:18:13.361 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 18 15 35808 35793 124.27 102.562 0.0037009 0.00804107 2023-10-01T10:18:14.361 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 19 16 37467 37451 123.182 103.625 0.00732256 0.00811164 2023-10-01T10:18:14.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:18:14 smithi078 bash[27336]: cluster 2023-10-01T10:18:13.245520+0000 mgr.y (mgr.14427) 187 : cluster [DBG] pgmap v166: 89 pgs: 89 active+clean; 2.2 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 143 MiB/s wr, 2.28k op/s 2023-10-01T10:18:14.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:18:14 smithi078 bash[27336]: audit 2023-10-01T10:18:14.268576+0000 mon.c (mon.2) 40 : audit [DBG] from='client.? 172.21.15.78:0/44729092' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:18:14.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:18:14 smithi078 bash[33821]: cluster 2023-10-01T10:18:13.245520+0000 mgr.y (mgr.14427) 187 : cluster [DBG] pgmap v166: 89 pgs: 89 active+clean; 2.2 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 143 MiB/s wr, 2.28k op/s 2023-10-01T10:18:14.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:18:14 smithi078 bash[33821]: audit 2023-10-01T10:18:14.268576+0000 mon.c (mon.2) 40 : audit [DBG] from='client.? 172.21.15.78:0/44729092' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:18:14.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:18:14 smithi173 bash[27612]: cluster 2023-10-01T10:18:13.245520+0000 mgr.y (mgr.14427) 187 : cluster [DBG] pgmap v166: 89 pgs: 89 active+clean; 2.2 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 143 MiB/s wr, 2.28k op/s 2023-10-01T10:18:14.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:18:14 smithi173 bash[27612]: audit 2023-10-01T10:18:14.268576+0000 mon.c (mon.2) 40 : audit [DBG] from='client.? 172.21.15.78:0/44729092' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:18:15.361 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:2023-10-01T10:18:15.359067+0000 min lat: 0.00135626 max lat: 0.0733663 avg lat: 0.00815046 2023-10-01T10:18:15.361 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T10:18:15.361 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 20 16 39252 39236 122.601 111.562 0.0139313 0.00815046 2023-10-01T10:18:15.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:18:15 smithi078 bash[27601]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:18:15] "GET /metrics HTTP/1.1" 200 194337 "" "Prometheus/2.33.4" 2023-10-01T10:18:15.620 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:18:15 smithi173 bash[28473]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:18:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T10:18:16.361 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 21 16 40981 40965 121.908 108.062 0.0172419 0.0081977 2023-10-01T10:18:16.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:18:16 smithi078 bash[33821]: cluster 2023-10-01T10:18:15.245919+0000 mgr.y (mgr.14427) 188 : cluster [DBG] pgmap v167: 89 pgs: 89 active+clean; 2.2 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-01T10:18:16.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:18:16 smithi078 bash[27336]: cluster 2023-10-01T10:18:15.245919+0000 mgr.y (mgr.14427) 188 : cluster [DBG] pgmap v167: 89 pgs: 89 active+clean; 2.2 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-01T10:18:16.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:18:16 smithi173 bash[27612]: cluster 2023-10-01T10:18:15.245919+0000 mgr.y (mgr.14427) 188 : cluster [DBG] pgmap v167: 89 pgs: 89 active+clean; 2.2 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-01T10:18:17.361 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 22 16 42614 42598 121.006 102.062 0.00454855 0.00825841 2023-10-01T10:18:17.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:18:17 smithi078 bash[27336]: audit 2023-10-01T10:18:16.890961+0000 mon.a (mon.0) 748 : audit [DBG] from='client.? 172.21.15.78:0/3609596066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:18:17.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:18:17 smithi078 bash[33821]: audit 2023-10-01T10:18:16.890961+0000 mon.a (mon.0) 748 : audit [DBG] from='client.? 172.21.15.78:0/3609596066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:18:17.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:18:17 smithi173 bash[27612]: audit 2023-10-01T10:18:16.890961+0000 mon.a (mon.0) 748 : audit [DBG] from='client.? 172.21.15.78:0/3609596066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:18:18.361 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 23 16 44279 44263 120.268 104.062 0.0068001 0.00830867 2023-10-01T10:18:18.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:18:18 smithi078 bash[33821]: cluster 2023-10-01T10:18:17.246348+0000 mgr.y (mgr.14427) 189 : cluster [DBG] pgmap v168: 89 pgs: 89 active+clean; 2.2 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-01T10:18:18.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:18:18 smithi078 bash[27336]: cluster 2023-10-01T10:18:17.246348+0000 mgr.y (mgr.14427) 189 : cluster [DBG] pgmap v168: 89 pgs: 89 active+clean; 2.2 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-01T10:18:18.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:18:18 smithi173 bash[27612]: cluster 2023-10-01T10:18:17.246348+0000 mgr.y (mgr.14427) 189 : cluster [DBG] pgmap v168: 89 pgs: 89 active+clean; 2.2 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-01T10:18:19.361 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 24 16 45875 45859 119.413 99.75 0.0104403 0.00836892 2023-10-01T10:18:20.361 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 25 16 47473 47457 118.631 99.875 0.0125402 0.00842418 2023-10-01T10:18:20.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:18:20 smithi078 bash[33821]: cluster 2023-10-01T10:18:19.247015+0000 mgr.y (mgr.14427) 190 : cluster [DBG] pgmap v169: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 129 MiB/s wr, 2.07k op/s 2023-10-01T10:18:20.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:18:20 smithi078 bash[33821]: audit 2023-10-01T10:18:19.524741+0000 mon.a (mon.0) 749 : audit [DBG] from='client.? 172.21.15.78:0/3580814394' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:18:20.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:18:20 smithi078 bash[27336]: cluster 2023-10-01T10:18:19.247015+0000 mgr.y (mgr.14427) 190 : cluster [DBG] pgmap v169: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 129 MiB/s wr, 2.07k op/s 2023-10-01T10:18:20.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:18:20 smithi078 bash[27336]: audit 2023-10-01T10:18:19.524741+0000 mon.a (mon.0) 749 : audit [DBG] from='client.? 172.21.15.78:0/3580814394' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:18:20.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:18:20 smithi173 bash[27612]: cluster 2023-10-01T10:18:19.247015+0000 mgr.y (mgr.14427) 190 : cluster [DBG] pgmap v169: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 129 MiB/s wr, 2.07k op/s 2023-10-01T10:18:20.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:18:20 smithi173 bash[27612]: audit 2023-10-01T10:18:19.524741+0000 mon.a (mon.0) 749 : audit [DBG] from='client.? 172.21.15.78:0/3580814394' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:18:21.362 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 26 16 48995 48979 117.727 95.125 0.0121751 0.00848835 2023-10-01T10:18:22.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:18:22 smithi078 bash[33821]: cluster 2023-10-01T10:18:21.247461+0000 mgr.y (mgr.14427) 191 : cluster [DBG] pgmap v170: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-01T10:18:22.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:18:22 smithi078 bash[27336]: cluster 2023-10-01T10:18:21.247461+0000 mgr.y (mgr.14427) 191 : cluster [DBG] pgmap v170: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-01T10:18:22.362 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 27 15 50427 50412 116.683 89.5625 0.00926813 0.00856439 2023-10-01T10:18:22.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:18:22 smithi173 bash[27612]: cluster 2023-10-01T10:18:21.247461+0000 mgr.y (mgr.14427) 191 : cluster [DBG] pgmap v170: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-01T10:18:23.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:18:23 smithi078 bash[33821]: audit 2023-10-01T10:18:22.152987+0000 mon.c (mon.2) 41 : audit [DBG] from='client.? 172.21.15.78:0/1179769203' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:18:23.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:18:23 smithi078 bash[27336]: audit 2023-10-01T10:18:22.152987+0000 mon.c (mon.2) 41 : audit [DBG] from='client.? 172.21.15.78:0/1179769203' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:18:23.362 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 28 16 51953 51937 115.92 95.3125 0.0066982 0.00862001 2023-10-01T10:18:23.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:18:23 smithi173 bash[27612]: audit 2023-10-01T10:18:22.152987+0000 mon.c (mon.2) 41 : audit [DBG] from='client.? 172.21.15.78:0/1179769203' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:18:24.362 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 29 16 53343 53327 114.918 86.875 0.00471023 0.00869397 2023-10-01T10:18:24.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:18:24 smithi173 bash[27612]: cluster 2023-10-01T10:18:23.248342+0000 mgr.y (mgr.14427) 192 : cluster [DBG] pgmap v171: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 124 MiB/s wr, 1.98k op/s 2023-10-01T10:18:24.567 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:18:24 smithi078 bash[27336]: cluster 2023-10-01T10:18:23.248342+0000 mgr.y (mgr.14427) 192 : cluster [DBG] pgmap v171: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 124 MiB/s wr, 1.98k op/s 2023-10-01T10:18:24.567 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:18:24 smithi078 bash[33821]: cluster 2023-10-01T10:18:23.248342+0000 mgr.y (mgr.14427) 192 : cluster [DBG] pgmap v171: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 124 MiB/s wr, 1.98k op/s 2023-10-01T10:18:25.362 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 30 16 54701 54685 113.916 84.875 0.00826772 0.00877247 2023-10-01T10:18:25.366 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:18:25 smithi173 bash[28473]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:18:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T10:18:25.367 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:18:25 smithi173 bash[27612]: audit 2023-10-01T10:18:24.831336+0000 mon.c (mon.2) 42 : audit [DBG] from='client.? 172.21.15.78:0/1156589312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:18:25.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:18:25 smithi078 bash[33821]: audit 2023-10-01T10:18:24.831336+0000 mon.c (mon.2) 42 : audit [DBG] from='client.? 172.21.15.78:0/1156589312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:18:25.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:18:25 smithi078 bash[27336]: audit 2023-10-01T10:18:24.831336+0000 mon.c (mon.2) 42 : audit [DBG] from='client.? 172.21.15.78:0/1156589312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:18:25.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:18:25 smithi078 bash[27601]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:18:25] "GET /metrics HTTP/1.1" 200 194632 "" "Prometheus/2.33.4" 2023-10-01T10:18:26.364 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 31 16 56018 56002 112.897 82.3125 0.0143657 0.00885233 2023-10-01T10:18:26.576 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:18:26 smithi173 bash[27612]: cluster 2023-10-01T10:18:25.250541+0000 mgr.y (mgr.14427) 193 : cluster [DBG] pgmap v172: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s 2023-10-01T10:18:26.577 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:18:26 smithi173 bash[27612]: audit 2023-10-01T10:18:25.569178+0000 mon.a (mon.0) 750 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:18:26.577 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:18:26 smithi173 bash[27612]: cephadm 2023-10-01T10:18:25.569916+0000 mgr.y (mgr.14427) 194 : cephadm [INF] Upgrade: Target is version 18.2.0-497-gb5682219 (unknown) 2023-10-01T10:18:26.577 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:18:26 smithi173 bash[27612]: cephadm 2023-10-01T10:18:25.570054+0000 mgr.y (mgr.14427) 195 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf, digests ['quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf'] 2023-10-01T10:18:26.577 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:18:26 smithi173 bash[27612]: audit 2023-10-01T10:18:25.573287+0000 mon.a (mon.0) 751 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:18:26.577 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:18:26 smithi173 bash[27612]: audit 2023-10-01T10:18:25.586064+0000 mon.a (mon.0) 752 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:18:26.577 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:18:26 smithi173 bash[27612]: cephadm 2023-10-01T10:18:25.587312+0000 mgr.y (mgr.14427) 196 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2023-10-01T10:18:26.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:18:26 smithi078 bash[33821]: cluster 2023-10-01T10:18:25.250541+0000 mgr.y (mgr.14427) 193 : cluster [DBG] pgmap v172: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s 2023-10-01T10:18:26.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:18:26 smithi078 bash[33821]: audit 2023-10-01T10:18:25.569178+0000 mon.a (mon.0) 750 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:18:26.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:18:26 smithi078 bash[33821]: cephadm 2023-10-01T10:18:25.569916+0000 mgr.y (mgr.14427) 194 : cephadm [INF] Upgrade: Target is version 18.2.0-497-gb5682219 (unknown) 2023-10-01T10:18:26.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:18:26 smithi078 bash[33821]: cephadm 2023-10-01T10:18:25.570054+0000 mgr.y (mgr.14427) 195 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf, digests ['quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf'] 2023-10-01T10:18:26.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:18:26 smithi078 bash[33821]: audit 2023-10-01T10:18:25.573287+0000 mon.a (mon.0) 751 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:18:26.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:18:26 smithi078 bash[33821]: audit 2023-10-01T10:18:25.586064+0000 mon.a (mon.0) 752 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:18:26.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:18:26 smithi078 bash[33821]: cephadm 2023-10-01T10:18:25.587312+0000 mgr.y (mgr.14427) 196 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2023-10-01T10:18:26.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:18:26 smithi078 bash[27336]: cluster 2023-10-01T10:18:25.250541+0000 mgr.y (mgr.14427) 193 : cluster [DBG] pgmap v172: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s 2023-10-01T10:18:26.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:18:26 smithi078 bash[27336]: audit 2023-10-01T10:18:25.569178+0000 mon.a (mon.0) 750 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:18:26.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:18:26 smithi078 bash[27336]: cephadm 2023-10-01T10:18:25.569916+0000 mgr.y (mgr.14427) 194 : cephadm [INF] Upgrade: Target is version 18.2.0-497-gb5682219 (unknown) 2023-10-01T10:18:26.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:18:26 smithi078 bash[27336]: cephadm 2023-10-01T10:18:25.570054+0000 mgr.y (mgr.14427) 195 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf, digests ['quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf'] 2023-10-01T10:18:26.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:18:26 smithi078 bash[27336]: audit 2023-10-01T10:18:25.573287+0000 mon.a (mon.0) 751 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:18:26.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:18:26 smithi078 bash[27336]: audit 2023-10-01T10:18:25.586064+0000 mon.a (mon.0) 752 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:18:26.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:18:26 smithi078 bash[27336]: cephadm 2023-10-01T10:18:25.587312+0000 mgr.y (mgr.14427) 196 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2023-10-01T10:18:27.362 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 32 16 57190 57174 111.657 73.25 0.0124616 0.00895065 2023-10-01T10:18:27.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:18:27 smithi078 bash[33821]: cephadm 2023-10-01T10:18:25.896422+0000 mgr.y (mgr.14427) 197 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf on smithi173 2023-10-01T10:18:27.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:18:27 smithi078 bash[33821]: audit 2023-10-01T10:18:27.431704+0000 mon.b (mon.1) 28 : audit [DBG] from='client.? 172.21.15.78:0/2440196766' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:18:27.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:18:27 smithi078 bash[27336]: cephadm 2023-10-01T10:18:25.896422+0000 mgr.y (mgr.14427) 197 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf on smithi173 2023-10-01T10:18:27.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:18:27 smithi078 bash[27336]: audit 2023-10-01T10:18:27.431704+0000 mon.b (mon.1) 28 : audit [DBG] from='client.? 172.21.15.78:0/2440196766' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:18:27.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:18:27 smithi173 bash[27612]: cephadm 2023-10-01T10:18:25.896422+0000 mgr.y (mgr.14427) 197 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf on smithi173 2023-10-01T10:18:27.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:18:27 smithi173 bash[27612]: audit 2023-10-01T10:18:27.431704+0000 mon.b (mon.1) 28 : audit [DBG] from='client.? 172.21.15.78:0/2440196766' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:18:28.362 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 33 16 58394 58378 110.554 75.25 0.00664066 0.00903974 2023-10-01T10:18:28.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:18:28 smithi173 bash[27612]: cluster 2023-10-01T10:18:27.250987+0000 mgr.y (mgr.14427) 198 : cluster [DBG] pgmap v173: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s 2023-10-01T10:18:29.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:18:28 smithi078 bash[33821]: cluster 2023-10-01T10:18:27.250987+0000 mgr.y (mgr.14427) 198 : cluster [DBG] pgmap v173: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s 2023-10-01T10:18:29.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:18:28 smithi078 bash[27336]: cluster 2023-10-01T10:18:27.250987+0000 mgr.y (mgr.14427) 198 : cluster [DBG] pgmap v173: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s 2023-10-01T10:18:29.363 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 34 16 59562 59546 109.449 73 0.00733369 0.00913115 2023-10-01T10:18:30.362 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 35 16 60704 60688 108.361 71.375 0.00944998 0.00922066 2023-10-01T10:18:30.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:18:30 smithi173 bash[27612]: cluster 2023-10-01T10:18:29.251769+0000 mgr.y (mgr.14427) 199 : cluster [DBG] pgmap v174: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2023-10-01T10:18:30.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:18:30 smithi173 bash[27612]: audit 2023-10-01T10:18:30.048743+0000 mon.c (mon.2) 43 : audit [DBG] from='client.? 172.21.15.78:0/2724933997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:18:31.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:18:30 smithi078 bash[27336]: cluster 2023-10-01T10:18:29.251769+0000 mgr.y (mgr.14427) 199 : cluster [DBG] pgmap v174: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2023-10-01T10:18:31.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:18:30 smithi078 bash[27336]: audit 2023-10-01T10:18:30.048743+0000 mon.c (mon.2) 43 : audit [DBG] from='client.? 172.21.15.78:0/2724933997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:18:31.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:18:30 smithi078 bash[33821]: cluster 2023-10-01T10:18:29.251769+0000 mgr.y (mgr.14427) 199 : cluster [DBG] pgmap v174: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2023-10-01T10:18:31.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:18:30 smithi078 bash[33821]: audit 2023-10-01T10:18:30.048743+0000 mon.c (mon.2) 43 : audit [DBG] from='client.? 172.21.15.78:0/2724933997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:18:31.362 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 36 16 61817 61801 107.283 69.5625 0.0114367 0.00931472 2023-10-01T10:18:32.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:18:32 smithi078 bash[27336]: cluster 2023-10-01T10:18:31.252318+0000 mgr.y (mgr.14427) 200 : cluster [DBG] pgmap v175: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-10-01T10:18:32.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:18:32 smithi078 bash[33821]: cluster 2023-10-01T10:18:31.252318+0000 mgr.y (mgr.14427) 200 : cluster [DBG] pgmap v175: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-10-01T10:18:32.363 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 37 16 62831 62815 106.096 63.375 0.0178344 0.00941982 2023-10-01T10:18:32.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:18:32 smithi173 bash[27612]: cluster 2023-10-01T10:18:31.252318+0000 mgr.y (mgr.14427) 200 : cluster [DBG] pgmap v175: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-10-01T10:18:33.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:18:33 smithi078 bash[27336]: audit 2023-10-01T10:18:32.679428+0000 mon.c (mon.2) 44 : audit [DBG] from='client.? 172.21.15.78:0/3394070720' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:18:33.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:18:33 smithi078 bash[33821]: audit 2023-10-01T10:18:32.679428+0000 mon.c (mon.2) 44 : audit [DBG] from='client.? 172.21.15.78:0/3394070720' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:18:33.363 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 38 16 63877 63861 105.025 65.375 0.01358 0.00951526 2023-10-01T10:18:33.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:18:33 smithi173 bash[27612]: audit 2023-10-01T10:18:32.679428+0000 mon.c (mon.2) 44 : audit [DBG] from='client.? 172.21.15.78:0/3394070720' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:18:34.363 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 39 16 64909 64893 103.985 64.5 0.0105281 0.00960979 2023-10-01T10:18:34.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:18:34 smithi173 bash[27612]: cluster 2023-10-01T10:18:33.253099+0000 mgr.y (mgr.14427) 201 : cluster [DBG] pgmap v176: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 103 MiB/s wr, 1.64k op/s 2023-10-01T10:18:34.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:18:34 smithi078 bash[27336]: cluster 2023-10-01T10:18:33.253099+0000 mgr.y (mgr.14427) 201 : cluster [DBG] pgmap v176: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 103 MiB/s wr, 1.64k op/s 2023-10-01T10:18:34.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:18:34 smithi078 bash[33821]: cluster 2023-10-01T10:18:33.253099+0000 mgr.y (mgr.14427) 201 : cluster [DBG] pgmap v176: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 103 MiB/s wr, 1.64k op/s 2023-10-01T10:18:35.363 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:2023-10-01T10:18:35.360975+0000 min lat: 0.00135626 max lat: 0.0967229 avg lat: 0.00970266 2023-10-01T10:18:35.363 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T10:18:35.363 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 40 16 65946 65930 103.006 64.8125 0.00588511 0.00970266 2023-10-01T10:18:35.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:18:35 smithi078 bash[33821]: audit 2023-10-01T10:18:35.248357+0000 mon.c (mon.2) 45 : audit [DBG] from='client.? 172.21.15.78:0/1974559876' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:18:35.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:18:35 smithi078 bash[27336]: audit 2023-10-01T10:18:35.248357+0000 mon.c (mon.2) 45 : audit [DBG] from='client.? 172.21.15.78:0/1974559876' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:18:35.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:18:35 smithi078 bash[27601]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:18:35] "GET /metrics HTTP/1.1" 200 194671 "" "Prometheus/2.33.4" 2023-10-01T10:18:35.619 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:18:35 smithi173 bash[28473]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:18:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T10:18:35.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:18:35 smithi173 bash[27612]: audit 2023-10-01T10:18:35.248357+0000 mon.c (mon.2) 45 : audit [DBG] from='client.? 172.21.15.78:0/1974559876' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:18:36.363 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 41 16 67082 67066 102.225 71 0.0191184 0.00977649 2023-10-01T10:18:36.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:18:36 smithi078 bash[27336]: cluster 2023-10-01T10:18:35.253555+0000 mgr.y (mgr.14427) 202 : cluster [DBG] pgmap v177: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 62 MiB/s wr, 999 op/s 2023-10-01T10:18:36.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:18:36 smithi078 bash[33821]: cluster 2023-10-01T10:18:35.253555+0000 mgr.y (mgr.14427) 202 : cluster [DBG] pgmap v177: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 62 MiB/s wr, 999 op/s 2023-10-01T10:18:36.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:18:36 smithi173 bash[27612]: cluster 2023-10-01T10:18:35.253555+0000 mgr.y (mgr.14427) 202 : cluster [DBG] pgmap v177: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 62 MiB/s wr, 999 op/s 2023-10-01T10:18:37.363 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 42 16 68196 68180 101.449 69.625 0.0166854 0.00985149 2023-10-01T10:18:38.363 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 43 16 69274 69258 100.656 67.375 0.030794 0.00992868 2023-10-01T10:18:38.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:18:38 smithi078 bash[27336]: cluster 2023-10-01T10:18:37.253985+0000 mgr.y (mgr.14427) 203 : cluster [DBG] pgmap v178: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 62 MiB/s wr, 999 op/s 2023-10-01T10:18:38.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:18:38 smithi078 bash[27336]: audit 2023-10-01T10:18:37.846301+0000 mon.a (mon.0) 753 : audit [DBG] from='client.? 172.21.15.78:0/2333944995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:18:38.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:18:38 smithi078 bash[33821]: cluster 2023-10-01T10:18:37.253985+0000 mgr.y (mgr.14427) 203 : cluster [DBG] pgmap v178: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 62 MiB/s wr, 999 op/s 2023-10-01T10:18:38.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:18:38 smithi078 bash[33821]: audit 2023-10-01T10:18:37.846301+0000 mon.a (mon.0) 753 : audit [DBG] from='client.? 172.21.15.78:0/2333944995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:18:38.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:18:38 smithi173 bash[27612]: cluster 2023-10-01T10:18:37.253985+0000 mgr.y (mgr.14427) 203 : cluster [DBG] pgmap v178: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 62 MiB/s wr, 999 op/s 2023-10-01T10:18:38.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:18:38 smithi173 bash[27612]: audit 2023-10-01T10:18:37.846301+0000 mon.a (mon.0) 753 : audit [DBG] from='client.? 172.21.15.78:0/2333944995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:18:39.363 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 44 16 70357 70341 99.9067 67.6875 0.0189027 0.0100039 2023-10-01T10:18:40.363 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 45 16 71406 71390 99.1434 65.5625 0.0106265 0.0100803 2023-10-01T10:18:40.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:18:40 smithi078 bash[27336]: cluster 2023-10-01T10:18:39.254835+0000 mgr.y (mgr.14427) 204 : cluster [DBG] pgmap v179: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2023-10-01T10:18:40.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:18:40 smithi078 bash[33821]: cluster 2023-10-01T10:18:39.254835+0000 mgr.y (mgr.14427) 204 : cluster [DBG] pgmap v179: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2023-10-01T10:18:40.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:18:40 smithi173 bash[27612]: cluster 2023-10-01T10:18:39.254835+0000 mgr.y (mgr.14427) 204 : cluster [DBG] pgmap v179: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2023-10-01T10:18:41.363 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 46 16 72498 72482 98.4716 68.25 0.0112106 0.0101492 2023-10-01T10:18:41.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:18:41 smithi173 bash[27612]: audit 2023-10-01T10:18:40.417752+0000 mon.a (mon.0) 754 : audit [DBG] from='client.? 172.21.15.78:0/737388725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:18:41.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:18:41 smithi078 bash[27336]: audit 2023-10-01T10:18:40.417752+0000 mon.a (mon.0) 754 : audit [DBG] from='client.? 172.21.15.78:0/737388725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:18:41.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:18:41 smithi078 bash[33821]: audit 2023-10-01T10:18:40.417752+0000 mon.a (mon.0) 754 : audit [DBG] from='client.? 172.21.15.78:0/737388725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:18:42.364 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 47 16 73524 73508 97.7407 64.125 0.00590998 0.0102241 2023-10-01T10:18:42.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:18:42 smithi173 bash[27612]: cluster 2023-10-01T10:18:41.255272+0000 mgr.y (mgr.14427) 205 : cluster [DBG] pgmap v180: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 57 MiB/s wr, 909 op/s 2023-10-01T10:18:42.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:18:42 smithi078 bash[27336]: cluster 2023-10-01T10:18:41.255272+0000 mgr.y (mgr.14427) 205 : cluster [DBG] pgmap v180: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 57 MiB/s wr, 909 op/s 2023-10-01T10:18:42.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:18:42 smithi078 bash[33821]: cluster 2023-10-01T10:18:41.255272+0000 mgr.y (mgr.14427) 205 : cluster [DBG] pgmap v180: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 57 MiB/s wr, 909 op/s 2023-10-01T10:18:43.364 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 48 16 74481 74465 96.9504 59.8125 0.00509907 0.0103078 2023-10-01T10:18:43.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:18:43 smithi173 bash[27612]: audit 2023-10-01T10:18:42.993701+0000 mon.c (mon.2) 46 : audit [DBG] from='client.? 172.21.15.78:0/3239817798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:18:43.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:18:43 smithi078 bash[27336]: audit 2023-10-01T10:18:42.993701+0000 mon.c (mon.2) 46 : audit [DBG] from='client.? 172.21.15.78:0/3239817798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:18:43.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:18:43 smithi078 bash[33821]: audit 2023-10-01T10:18:42.993701+0000 mon.c (mon.2) 46 : audit [DBG] from='client.? 172.21.15.78:0/3239817798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:18:44.364 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 49 16 75442 75426 96.1975 60.0625 0.0125527 0.0103893 2023-10-01T10:18:44.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:18:44 smithi173 bash[27612]: cluster 2023-10-01T10:18:43.255894+0000 mgr.y (mgr.14427) 206 : cluster [DBG] pgmap v181: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-01T10:18:44.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:18:44 smithi078 bash[27336]: cluster 2023-10-01T10:18:43.255894+0000 mgr.y (mgr.14427) 206 : cluster [DBG] pgmap v181: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-01T10:18:44.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:18:44 smithi078 bash[33821]: cluster 2023-10-01T10:18:43.255894+0000 mgr.y (mgr.14427) 206 : cluster [DBG] pgmap v181: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-01T10:18:45.364 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 50 15 76350 76335 95.4097 56.8125 0.0105466 0.0104738 2023-10-01T10:18:45.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:18:45 smithi078 bash[27601]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:18:45] "GET /metrics HTTP/1.1" 200 194671 "" "Prometheus/2.33.4" 2023-10-01T10:18:45.619 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:18:45 smithi173 bash[28473]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:18:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T10:18:46.367 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 51 16 77269 77253 94.6638 57.375 0.0117617 0.0105558 2023-10-01T10:18:46.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:18:46 smithi173 bash[27612]: cluster 2023-10-01T10:18:45.256310+0000 mgr.y (mgr.14427) 207 : cluster [DBG] pgmap v182: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 55 MiB/s wr, 886 op/s 2023-10-01T10:18:46.622 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:18:46 smithi173 bash[27612]: audit 2023-10-01T10:18:45.558869+0000 mon.a (mon.0) 755 : audit [DBG] from='client.? 172.21.15.78:0/1378800211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:18:46.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:18:46 smithi078 bash[33821]: cluster 2023-10-01T10:18:45.256310+0000 mgr.y (mgr.14427) 207 : cluster [DBG] pgmap v182: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 55 MiB/s wr, 886 op/s 2023-10-01T10:18:46.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:18:46 smithi078 bash[33821]: audit 2023-10-01T10:18:45.558869+0000 mon.a (mon.0) 755 : audit [DBG] from='client.? 172.21.15.78:0/1378800211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:18:46.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:18:46 smithi078 bash[27336]: cluster 2023-10-01T10:18:45.256310+0000 mgr.y (mgr.14427) 207 : cluster [DBG] pgmap v182: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 55 MiB/s wr, 886 op/s 2023-10-01T10:18:46.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:18:46 smithi078 bash[27336]: audit 2023-10-01T10:18:45.558869+0000 mon.a (mon.0) 755 : audit [DBG] from='client.? 172.21.15.78:0/1378800211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:18:47.364 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 52 16 78165 78149 93.9202 56 0.00729906 0.0106409 2023-10-01T10:18:48.364 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 53 16 79058 79042 93.2011 55.8125 0.020992 0.0107234 2023-10-01T10:18:48.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:18:48 smithi173 bash[27612]: cluster 2023-10-01T10:18:47.256789+0000 mgr.y (mgr.14427) 208 : cluster [DBG] pgmap v183: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 55 MiB/s wr, 886 op/s 2023-10-01T10:18:48.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:18:48 smithi173 bash[27612]: audit 2023-10-01T10:18:48.110510+0000 mon.a (mon.0) 756 : audit [DBG] from='client.? 172.21.15.78:0/3611347242' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:18:48.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:18:48 smithi078 bash[33821]: cluster 2023-10-01T10:18:47.256789+0000 mgr.y (mgr.14427) 208 : cluster [DBG] pgmap v183: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 55 MiB/s wr, 886 op/s 2023-10-01T10:18:48.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:18:48 smithi078 bash[33821]: audit 2023-10-01T10:18:48.110510+0000 mon.a (mon.0) 756 : audit [DBG] from='client.? 172.21.15.78:0/3611347242' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:18:48.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:18:48 smithi078 bash[27336]: cluster 2023-10-01T10:18:47.256789+0000 mgr.y (mgr.14427) 208 : cluster [DBG] pgmap v183: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 55 MiB/s wr, 886 op/s 2023-10-01T10:18:48.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:18:48 smithi078 bash[27336]: audit 2023-10-01T10:18:48.110510+0000 mon.a (mon.0) 756 : audit [DBG] from='client.? 172.21.15.78:0/3611347242' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:18:49.364 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 54 16 79954 79938 92.5121 56 0.0413072 0.0108033 2023-10-01T10:18:50.364 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 55 16 80809 80793 91.8015 53.4375 0.0225748 0.0108869 2023-10-01T10:18:50.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:18:50 smithi078 bash[33821]: cluster 2023-10-01T10:18:49.257498+0000 mgr.y (mgr.14427) 209 : cluster [DBG] pgmap v184: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-10-01T10:18:50.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:18:50 smithi078 bash[27336]: cluster 2023-10-01T10:18:49.257498+0000 mgr.y (mgr.14427) 209 : cluster [DBG] pgmap v184: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-10-01T10:18:50.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:18:50 smithi173 bash[27612]: cluster 2023-10-01T10:18:49.257498+0000 mgr.y (mgr.14427) 209 : cluster [DBG] pgmap v184: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-10-01T10:18:51.365 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 56 16 81671 81655 91.1242 53.875 0.00719017 0.0109675 2023-10-01T10:18:51.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:18:51 smithi078 bash[27336]: audit 2023-10-01T10:18:50.678502+0000 mon.c (mon.2) 47 : audit [DBG] from='client.? 172.21.15.78:0/3677219297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:18:51.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:18:51 smithi078 bash[33821]: audit 2023-10-01T10:18:50.678502+0000 mon.c (mon.2) 47 : audit [DBG] from='client.? 172.21.15.78:0/3677219297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:18:51.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:18:51 smithi173 bash[27612]: audit 2023-10-01T10:18:50.678502+0000 mon.c (mon.2) 47 : audit [DBG] from='client.? 172.21.15.78:0/3677219297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:18:52.364 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 57 16 82569 82553 90.5101 56.125 0.0111291 0.011043 2023-10-01T10:18:52.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:18:52 smithi078 bash[27336]: cluster 2023-10-01T10:18:51.257891+0000 mgr.y (mgr.14427) 210 : cluster [DBG] pgmap v185: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 51 MiB/s wr, 818 op/s 2023-10-01T10:18:52.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:18:52 smithi078 bash[33821]: cluster 2023-10-01T10:18:51.257891+0000 mgr.y (mgr.14427) 210 : cluster [DBG] pgmap v185: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 51 MiB/s wr, 818 op/s 2023-10-01T10:18:52.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:18:52 smithi173 bash[27612]: cluster 2023-10-01T10:18:51.257891+0000 mgr.y (mgr.14427) 210 : cluster [DBG] pgmap v185: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 51 MiB/s wr, 818 op/s 2023-10-01T10:18:53.365 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 58 16 83442 83426 89.8902 54.5625 0.0129558 0.0111187 2023-10-01T10:18:53.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:18:53 smithi078 bash[33821]: audit 2023-10-01T10:18:53.250671+0000 mon.c (mon.2) 48 : audit [DBG] from='client.? 172.21.15.78:0/2208680589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:18:53.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:18:53 smithi078 bash[27336]: audit 2023-10-01T10:18:53.250671+0000 mon.c (mon.2) 48 : audit [DBG] from='client.? 172.21.15.78:0/2208680589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:18:53.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:18:53 smithi173 bash[27612]: audit 2023-10-01T10:18:53.250671+0000 mon.c (mon.2) 48 : audit [DBG] from='client.? 172.21.15.78:0/2208680589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:18:54.365 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 59 16 84317 84301 89.2934 54.6875 0.00472667 0.0111917 2023-10-01T10:18:54.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:18:54 smithi078 bash[27336]: cluster 2023-10-01T10:18:53.258706+0000 mgr.y (mgr.14427) 211 : cluster [DBG] pgmap v186: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-10-01T10:18:54.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:18:54 smithi078 bash[33821]: cluster 2023-10-01T10:18:53.258706+0000 mgr.y (mgr.14427) 211 : cluster [DBG] pgmap v186: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-10-01T10:18:54.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:18:54 smithi173 bash[27612]: cluster 2023-10-01T10:18:53.258706+0000 mgr.y (mgr.14427) 211 : cluster [DBG] pgmap v186: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-10-01T10:18:55.365 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:2023-10-01T10:18:55.362867+0000 min lat: 0.00135626 max lat: 0.0967229 avg lat: 0.0112649 2023-10-01T10:18:55.365 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T10:18:55.365 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 60 16 85188 85172 88.7124 54.4375 0.0504347 0.0112649 2023-10-01T10:18:55.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:18:55 smithi078 bash[27601]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:18:55] "GET /metrics HTTP/1.1" 200 194686 "" "Prometheus/2.33.4" 2023-10-01T10:18:55.619 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:18:55 smithi173 bash[28473]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:18:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T10:18:56.365 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 61 16 86069 86053 88.1607 55.0625 0.00904267 0.0113358 2023-10-01T10:18:56.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:18:56 smithi078 bash[27336]: cluster 2023-10-01T10:18:55.259147+0000 mgr.y (mgr.14427) 212 : cluster [DBG] pgmap v187: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 47 MiB/s wr, 747 op/s 2023-10-01T10:18:56.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:18:56 smithi078 bash[27336]: audit 2023-10-01T10:18:55.852919+0000 mon.c (mon.2) 49 : audit [DBG] from='client.? 172.21.15.78:0/2873750236' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:18:56.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:18:56 smithi078 bash[33821]: cluster 2023-10-01T10:18:55.259147+0000 mgr.y (mgr.14427) 212 : cluster [DBG] pgmap v187: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 47 MiB/s wr, 747 op/s 2023-10-01T10:18:56.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:18:56 smithi078 bash[33821]: audit 2023-10-01T10:18:55.852919+0000 mon.c (mon.2) 49 : audit [DBG] from='client.? 172.21.15.78:0/2873750236' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:18:56.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:18:56 smithi173 bash[27612]: cluster 2023-10-01T10:18:55.259147+0000 mgr.y (mgr.14427) 212 : cluster [DBG] pgmap v187: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 47 MiB/s wr, 747 op/s 2023-10-01T10:18:56.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:18:56 smithi173 bash[27612]: audit 2023-10-01T10:18:55.852919+0000 mon.c (mon.2) 49 : audit [DBG] from='client.? 172.21.15.78:0/2873750236' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:18:57.365 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 62 16 87004 86988 87.6812 58.4375 0.0144395 0.0113999 2023-10-01T10:18:58.365 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 63 16 88009 87993 87.2863 62.8125 0.0203324 0.0114505 2023-10-01T10:18:58.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:18:58 smithi078 bash[33821]: cluster 2023-10-01T10:18:57.259585+0000 mgr.y (mgr.14427) 213 : cluster [DBG] pgmap v188: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 47 MiB/s wr, 747 op/s 2023-10-01T10:18:58.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:18:58 smithi078 bash[27336]: cluster 2023-10-01T10:18:57.259585+0000 mgr.y (mgr.14427) 213 : cluster [DBG] pgmap v188: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 47 MiB/s wr, 747 op/s 2023-10-01T10:18:58.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:18:58 smithi173 bash[27612]: cluster 2023-10-01T10:18:57.259585+0000 mgr.y (mgr.14427) 213 : cluster [DBG] pgmap v188: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 47 MiB/s wr, 747 op/s 2023-10-01T10:18:59.365 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 64 16 89004 88988 86.8941 62.1875 0.015102 0.0115025 2023-10-01T10:18:59.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:18:59 smithi078 bash[27336]: audit 2023-10-01T10:18:58.467806+0000 mon.b (mon.1) 29 : audit [DBG] from='client.? 172.21.15.78:0/407828407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:18:59.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:18:59 smithi078 bash[33821]: audit 2023-10-01T10:18:58.467806+0000 mon.b (mon.1) 29 : audit [DBG] from='client.? 172.21.15.78:0/407828407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:18:59.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:18:59 smithi173 bash[27612]: audit 2023-10-01T10:18:58.467806+0000 mon.b (mon.1) 29 : audit [DBG] from='client.? 172.21.15.78:0/407828407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:00.365 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 65 16 90027 90011 86.5408 63.9375 0.0176426 0.0115495 2023-10-01T10:19:00.498 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:00 smithi173 bash[27612]: cluster 2023-10-01T10:18:59.260272+0000 mgr.y (mgr.14427) 214 : cluster [DBG] pgmap v189: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 70 MiB/s wr, 1.13k op/s 2023-10-01T10:19:00.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:00 smithi078 bash[33821]: cluster 2023-10-01T10:18:59.260272+0000 mgr.y (mgr.14427) 214 : cluster [DBG] pgmap v189: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 70 MiB/s wr, 1.13k op/s 2023-10-01T10:19:00.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:00 smithi078 bash[27336]: cluster 2023-10-01T10:18:59.260272+0000 mgr.y (mgr.14427) 214 : cluster [DBG] pgmap v189: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 70 MiB/s wr, 1.13k op/s 2023-10-01T10:19:01.365 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 66 16 91021 91005 86.1708 62.125 0.0253865 0.0115992 2023-10-01T10:19:01.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:01 smithi078 bash[27336]: audit 2023-10-01T10:19:01.071101+0000 mon.a (mon.0) 757 : audit [DBG] from='client.? 172.21.15.78:0/2619349442' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:01.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:01 smithi078 bash[33821]: audit 2023-10-01T10:19:01.071101+0000 mon.a (mon.0) 757 : audit [DBG] from='client.? 172.21.15.78:0/2619349442' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:01.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:01 smithi173 bash[27612]: audit 2023-10-01T10:19:01.071101+0000 mon.a (mon.0) 757 : audit [DBG] from='client.? 172.21.15.78:0/2619349442' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:02.366 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 67 16 92039 92023 85.8342 63.625 0.00677853 0.0116448 2023-10-01T10:19:02.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:02 smithi078 bash[33821]: cluster 2023-10-01T10:19:01.260696+0000 mgr.y (mgr.14427) 215 : cluster [DBG] pgmap v190: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 47 MiB/s wr, 744 op/s 2023-10-01T10:19:02.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:02 smithi078 bash[27336]: cluster 2023-10-01T10:19:01.260696+0000 mgr.y (mgr.14427) 215 : cluster [DBG] pgmap v190: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 47 MiB/s wr, 744 op/s 2023-10-01T10:19:02.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:02 smithi173 bash[27612]: cluster 2023-10-01T10:19:01.260696+0000 mgr.y (mgr.14427) 215 : cluster [DBG] pgmap v190: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 47 MiB/s wr, 744 op/s 2023-10-01T10:19:03.366 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 68 16 93014 92998 85.468 60.9375 0.0160509 0.0116943 2023-10-01T10:19:04.366 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 69 16 93982 93966 85.106 60.5 0.0207231 0.0117447 2023-10-01T10:19:04.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:04 smithi078 bash[33821]: cluster 2023-10-01T10:19:03.261456+0000 mgr.y (mgr.14427) 216 : cluster [DBG] pgmap v191: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-10-01T10:19:04.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:04 smithi078 bash[33821]: audit 2023-10-01T10:19:03.684133+0000 mon.c (mon.2) 50 : audit [DBG] from='client.? 172.21.15.78:0/858429609' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:04.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:04 smithi078 bash[27336]: cluster 2023-10-01T10:19:03.261456+0000 mgr.y (mgr.14427) 216 : cluster [DBG] pgmap v191: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-10-01T10:19:04.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:04 smithi078 bash[27336]: audit 2023-10-01T10:19:03.684133+0000 mon.c (mon.2) 50 : audit [DBG] from='client.? 172.21.15.78:0/858429609' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:04.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:04 smithi173 bash[27612]: cluster 2023-10-01T10:19:03.261456+0000 mgr.y (mgr.14427) 216 : cluster [DBG] pgmap v191: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-10-01T10:19:04.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:04 smithi173 bash[27612]: audit 2023-10-01T10:19:03.684133+0000 mon.c (mon.2) 50 : audit [DBG] from='client.? 172.21.15.78:0/858429609' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:05.366 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 70 16 94983 94967 84.7839 62.5625 0.0275781 0.011789 2023-10-01T10:19:05.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:19:05 smithi078 bash[27601]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:19:05] "GET /metrics HTTP/1.1" 200 194747 "" "Prometheus/2.33.4" 2023-10-01T10:19:05.619 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:19:05 smithi173 bash[28473]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:19:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T10:19:06.366 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 71 16 95942 95926 84.4338 59.9375 0.0178056 0.011838 2023-10-01T10:19:06.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:06 smithi078 bash[27336]: cluster 2023-10-01T10:19:05.262054+0000 mgr.y (mgr.14427) 217 : cluster [DBG] pgmap v192: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 53 MiB/s wr, 842 op/s 2023-10-01T10:19:06.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:06 smithi078 bash[27336]: audit 2023-10-01T10:19:06.312789+0000 mon.b (mon.1) 30 : audit [DBG] from='client.? 172.21.15.78:0/4095437493' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:06.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:06 smithi078 bash[33821]: cluster 2023-10-01T10:19:05.262054+0000 mgr.y (mgr.14427) 217 : cluster [DBG] pgmap v192: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 53 MiB/s wr, 842 op/s 2023-10-01T10:19:06.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:06 smithi078 bash[33821]: audit 2023-10-01T10:19:06.312789+0000 mon.b (mon.1) 30 : audit [DBG] from='client.? 172.21.15.78:0/4095437493' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:06.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:06 smithi173 bash[27612]: cluster 2023-10-01T10:19:05.262054+0000 mgr.y (mgr.14427) 217 : cluster [DBG] pgmap v192: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 53 MiB/s wr, 842 op/s 2023-10-01T10:19:06.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:06 smithi173 bash[27612]: audit 2023-10-01T10:19:06.312789+0000 mon.b (mon.1) 30 : audit [DBG] from='client.? 172.21.15.78:0/4095437493' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:07.366 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 72 16 96909 96893 84.1005 60.4375 0.0271619 0.011885 2023-10-01T10:19:08.366 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 73 16 97871 97855 83.772 60.125 0.0127074 0.0119317 2023-10-01T10:19:08.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:08 smithi078 bash[27336]: cluster 2023-10-01T10:19:07.262474+0000 mgr.y (mgr.14427) 218 : cluster [DBG] pgmap v193: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 50 MiB/s wr, 796 op/s 2023-10-01T10:19:08.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:08 smithi078 bash[33821]: cluster 2023-10-01T10:19:07.262474+0000 mgr.y (mgr.14427) 218 : cluster [DBG] pgmap v193: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 50 MiB/s wr, 796 op/s 2023-10-01T10:19:08.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:08 smithi173 bash[27612]: cluster 2023-10-01T10:19:07.262474+0000 mgr.y (mgr.14427) 218 : cluster [DBG] pgmap v193: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 50 MiB/s wr, 796 op/s 2023-10-01T10:19:09.366 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 74 16 98806 98790 83.4296 58.4375 0.0127469 0.0119807 2023-10-01T10:19:09.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:09 smithi078 bash[27336]: audit 2023-10-01T10:19:08.919460+0000 mon.c (mon.2) 51 : audit [DBG] from='client.? 172.21.15.78:0/3880165425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:09.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:09 smithi078 bash[33821]: audit 2023-10-01T10:19:08.919460+0000 mon.c (mon.2) 51 : audit [DBG] from='client.? 172.21.15.78:0/3880165425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:09.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:09 smithi173 bash[27612]: audit 2023-10-01T10:19:08.919460+0000 mon.c (mon.2) 51 : audit [DBG] from='client.? 172.21.15.78:0/3880165425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:10.366 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 75 16 99803 99787 83.1479 62.3125 0.00808366 0.0120208 2023-10-01T10:19:10.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:10 smithi078 bash[33821]: cluster 2023-10-01T10:19:09.265965+0000 mgr.y (mgr.14427) 219 : cluster [DBG] pgmap v194: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-01T10:19:10.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:10 smithi078 bash[27336]: cluster 2023-10-01T10:19:09.265965+0000 mgr.y (mgr.14427) 219 : cluster [DBG] pgmap v194: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-01T10:19:10.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:10 smithi173 bash[27612]: cluster 2023-10-01T10:19:09.265965+0000 mgr.y (mgr.14427) 219 : cluster [DBG] pgmap v194: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-10-01T10:19:11.366 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 76 16 100731 100715 82.817 58 0.0133462 0.0120689 2023-10-01T10:19:12.366 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 77 16 101636 101620 82.4759 56.5625 0.00993032 0.0121194 2023-10-01T10:19:12.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:12 smithi173 bash[27612]: cluster 2023-10-01T10:19:11.266397+0000 mgr.y (mgr.14427) 220 : cluster [DBG] pgmap v195: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 823 op/s 2023-10-01T10:19:12.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:12 smithi173 bash[27612]: audit 2023-10-01T10:19:11.558707+0000 mon.b (mon.1) 31 : audit [DBG] from='client.? 172.21.15.78:0/4108714923' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:12.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:12 smithi078 bash[27336]: cluster 2023-10-01T10:19:11.266397+0000 mgr.y (mgr.14427) 220 : cluster [DBG] pgmap v195: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 823 op/s 2023-10-01T10:19:12.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:12 smithi078 bash[27336]: audit 2023-10-01T10:19:11.558707+0000 mon.b (mon.1) 31 : audit [DBG] from='client.? 172.21.15.78:0/4108714923' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:12.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:12 smithi078 bash[33821]: cluster 2023-10-01T10:19:11.266397+0000 mgr.y (mgr.14427) 220 : cluster [DBG] pgmap v195: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 823 op/s 2023-10-01T10:19:12.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:12 smithi078 bash[33821]: audit 2023-10-01T10:19:11.558707+0000 mon.b (mon.1) 31 : audit [DBG] from='client.? 172.21.15.78:0/4108714923' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:13.367 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 78 16 102521 102505 82.1276 55.3125 0.0181706 0.0121699 2023-10-01T10:19:13.543 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:13 smithi173 bash[27612]: cephadm 2023-10-01T10:19:12.149298+0000 mgr.y (mgr.14427) 221 : cephadm [INF] Upgrade: Updating mgr.x 2023-10-01T10:19:13.543 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:13 smithi173 bash[27612]: audit 2023-10-01T10:19:12.154784+0000 mon.a (mon.0) 758 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:19:13.543 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:13 smithi173 bash[27612]: audit 2023-10-01T10:19:12.156361+0000 mon.a (mon.0) 759 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-01T10:19:13.543 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:13 smithi173 bash[27612]: audit 2023-10-01T10:19:12.157992+0000 mon.a (mon.0) 760 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-01T10:19:13.543 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:13 smithi173 bash[27612]: audit 2023-10-01T10:19:12.159313+0000 mon.a (mon.0) 761 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:19:13.544 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:13 smithi173 bash[27612]: cephadm 2023-10-01T10:19:12.160644+0000 mgr.y (mgr.14427) 222 : cephadm [INF] Deploying daemon mgr.x on smithi173 2023-10-01T10:19:13.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:13 smithi078 bash[27336]: cephadm 2023-10-01T10:19:12.149298+0000 mgr.y (mgr.14427) 221 : cephadm [INF] Upgrade: Updating mgr.x 2023-10-01T10:19:13.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:13 smithi078 bash[27336]: audit 2023-10-01T10:19:12.154784+0000 mon.a (mon.0) 758 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:19:13.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:13 smithi078 bash[27336]: audit 2023-10-01T10:19:12.156361+0000 mon.a (mon.0) 759 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-01T10:19:13.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:13 smithi078 bash[27336]: audit 2023-10-01T10:19:12.157992+0000 mon.a (mon.0) 760 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-01T10:19:13.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:13 smithi078 bash[27336]: audit 2023-10-01T10:19:12.159313+0000 mon.a (mon.0) 761 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:19:13.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:13 smithi078 bash[27336]: cephadm 2023-10-01T10:19:12.160644+0000 mgr.y (mgr.14427) 222 : cephadm [INF] Deploying daemon mgr.x on smithi173 2023-10-01T10:19:13.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:13 smithi078 bash[33821]: cephadm 2023-10-01T10:19:12.149298+0000 mgr.y (mgr.14427) 221 : cephadm [INF] Upgrade: Updating mgr.x 2023-10-01T10:19:13.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:13 smithi078 bash[33821]: audit 2023-10-01T10:19:12.154784+0000 mon.a (mon.0) 758 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:19:13.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:13 smithi078 bash[33821]: audit 2023-10-01T10:19:12.156361+0000 mon.a (mon.0) 759 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-01T10:19:13.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:13 smithi078 bash[33821]: audit 2023-10-01T10:19:12.157992+0000 mon.a (mon.0) 760 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-01T10:19:13.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:13 smithi078 bash[33821]: audit 2023-10-01T10:19:12.159313+0000 mon.a (mon.0) 761 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:19:13.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:13 smithi078 bash[33821]: cephadm 2023-10-01T10:19:12.160644+0000 mgr.y (mgr.14427) 222 : cephadm [INF] Deploying daemon mgr.x on smithi173 2023-10-01T10:19:14.367 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 79 16 103403 103387 81.7857 55.125 0.028764 0.0122217 2023-10-01T10:19:14.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:14 smithi078 bash[27336]: cluster 2023-10-01T10:19:13.267033+0000 mgr.y (mgr.14427) 223 : cluster [DBG] pgmap v196: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2023-10-01T10:19:14.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:14 smithi078 bash[33821]: cluster 2023-10-01T10:19:13.267033+0000 mgr.y (mgr.14427) 223 : cluster [DBG] pgmap v196: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2023-10-01T10:19:14.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:14 smithi173 bash[27612]: cluster 2023-10-01T10:19:13.267033+0000 mgr.y (mgr.14427) 223 : cluster [DBG] pgmap v196: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2023-10-01T10:19:14.993 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:19:14 smithi173 systemd[1]: Stopping Ceph mgr.x for d8016480-6041-11ee-8db5-212e2dc638e7... 2023-10-01T10:19:15.114 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:19:14 smithi173 bash[50293]: Error response from daemon: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-mgr.x 2023-10-01T10:19:15.367 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:2023-10-01T10:19:15.364782+0000 min lat: 0.00135626 max lat: 0.0967229 avg lat: 0.0122769 2023-10-01T10:19:15.367 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T10:19:15.367 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 80 16 104238 104222 81.4157 52.1875 0.0187665 0.0122769 2023-10-01T10:19:15.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:15 smithi173 bash[27612]: audit 2023-10-01T10:19:14.177326+0000 mon.a (mon.0) 762 : audit [DBG] from='client.? 172.21.15.78:0/3676200668' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:15.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:15 smithi078 bash[33821]: audit 2023-10-01T10:19:14.177326+0000 mon.a (mon.0) 762 : audit [DBG] from='client.? 172.21.15.78:0/3676200668' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:15.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:15 smithi078 bash[27336]: audit 2023-10-01T10:19:14.177326+0000 mon.a (mon.0) 762 : audit [DBG] from='client.? 172.21.15.78:0/3676200668' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:15.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:19:15 smithi078 bash[27601]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:19:15] "GET /metrics HTTP/1.1" 200 194747 "" "Prometheus/2.33.4" 2023-10-01T10:19:16.120 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:19:15 smithi173 bash[50321]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-mgr-x 2023-10-01T10:19:16.120 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:19:15 smithi173 bash[50380]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-mgr-x 2023-10-01T10:19:16.120 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:19:15 smithi173 systemd[1]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7@mgr.x.service: Main process exited, code=exited, status=143/n/a 2023-10-01T10:19:16.120 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:19:15 smithi173 systemd[1]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7@mgr.x.service: Failed with result 'exit-code'. 2023-10-01T10:19:16.120 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:19:15 smithi173 systemd[1]: Stopped Ceph mgr.x for d8016480-6041-11ee-8db5-212e2dc638e7. 2023-10-01T10:19:16.367 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 81 16 105045 105029 81.0332 50.4375 0.009701 0.0123347 2023-10-01T10:19:16.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:16 smithi078 bash[27336]: cluster 2023-10-01T10:19:15.267611+0000 mgr.y (mgr.14427) 224 : cluster [DBG] pgmap v197: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 53 MiB/s wr, 847 op/s 2023-10-01T10:19:16.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:16 smithi078 bash[33821]: cluster 2023-10-01T10:19:15.267611+0000 mgr.y (mgr.14427) 224 : cluster [DBG] pgmap v197: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 53 MiB/s wr, 847 op/s 2023-10-01T10:19:16.619 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:19:16 smithi173 systemd[1]: Started Ceph mgr.x for d8016480-6041-11ee-8db5-212e2dc638e7. 2023-10-01T10:19:16.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:16 smithi173 bash[27612]: cluster 2023-10-01T10:19:15.267611+0000 mgr.y (mgr.14427) 224 : cluster [DBG] pgmap v197: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 53 MiB/s wr, 847 op/s 2023-10-01T10:19:17.367 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 82 16 105888 105872 80.6874 52.6875 0.00584317 0.0123876 2023-10-01T10:19:17.563 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:19:17 smithi173 bash[50431]: debug 2023-10-01T10:19:17.316+0000 7f9f1260f700 1 -- 172.21.15.173:0/377458102 <== mon.2 v2:172.21.15.78:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x563ede3a11e0 con 0x563edf120c00 2023-10-01T10:19:17.563 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:17 smithi173 bash[27612]: audit 2023-10-01T10:19:16.194098+0000 mon.a (mon.0) 763 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:19:17.563 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:17 smithi173 bash[27612]: audit 2023-10-01T10:19:16.195211+0000 mon.a (mon.0) 764 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:19:17.563 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:17 smithi173 bash[27612]: audit 2023-10-01T10:19:16.842695+0000 mon.a (mon.0) 765 : audit [DBG] from='client.? 172.21.15.78:0/3070119507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:17.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:17 smithi078 bash[33821]: audit 2023-10-01T10:19:16.194098+0000 mon.a (mon.0) 763 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:19:17.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:17 smithi078 bash[33821]: audit 2023-10-01T10:19:16.195211+0000 mon.a (mon.0) 764 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:19:17.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:17 smithi078 bash[33821]: audit 2023-10-01T10:19:16.842695+0000 mon.a (mon.0) 765 : audit [DBG] from='client.? 172.21.15.78:0/3070119507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:17.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:17 smithi078 bash[27336]: audit 2023-10-01T10:19:16.194098+0000 mon.a (mon.0) 763 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:19:17.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:17 smithi078 bash[27336]: audit 2023-10-01T10:19:16.195211+0000 mon.a (mon.0) 764 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:19:17.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:17 smithi078 bash[27336]: audit 2023-10-01T10:19:16.842695+0000 mon.a (mon.0) 765 : audit [DBG] from='client.? 172.21.15.78:0/3070119507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:17.870 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:19:17 smithi173 bash[50431]: debug 2023-10-01T10:19:17.560+0000 7f9f1c75d200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-01T10:19:18.367 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 83 16 106676 106660 80.3086 49.25 0.00524474 0.0124444 2023-10-01T10:19:18.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:18 smithi078 bash[27336]: cluster 2023-10-01T10:19:17.268102+0000 mgr.y (mgr.14427) 225 : cluster [DBG] pgmap v198: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 794 op/s 2023-10-01T10:19:18.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:18 smithi078 bash[33821]: cluster 2023-10-01T10:19:17.268102+0000 mgr.y (mgr.14427) 225 : cluster [DBG] pgmap v198: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 794 op/s 2023-10-01T10:19:18.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:18 smithi173 bash[27612]: cluster 2023-10-01T10:19:17.268102+0000 mgr.y (mgr.14427) 225 : cluster [DBG] pgmap v198: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 794 op/s 2023-10-01T10:19:19.230 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:19:18 smithi173 bash[50431]: debug 2023-10-01T10:19:18.924+0000 7f9f1c75d200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-01T10:19:19.231 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:19:19 smithi173 bash[50431]: debug 2023-10-01T10:19:19.072+0000 7f9f1c75d200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-01T10:19:19.231 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:19:19 smithi173 bash[50431]: debug 2023-10-01T10:19:19.228+0000 7f9f1c75d200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-01T10:19:19.367 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 84 16 107472 107456 79.9448 49.75 0.0112536 0.0125026 2023-10-01T10:19:19.616 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:19:19 smithi173 bash[50431]: debug 2023-10-01T10:19:19.612+0000 7f9f1c75d200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-01T10:19:20.367 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 85 16 108268 108252 79.5895 49.75 0.0164239 0.0125576 2023-10-01T10:19:20.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:20 smithi078 bash[33821]: cluster 2023-10-01T10:19:19.268874+0000 mgr.y (mgr.14427) 226 : cluster [DBG] pgmap v199: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-10-01T10:19:20.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:20 smithi078 bash[33821]: audit 2023-10-01T10:19:19.445080+0000 mon.a (mon.0) 766 : audit [DBG] from='client.? 172.21.15.78:0/796236399' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:20.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:20 smithi078 bash[27336]: cluster 2023-10-01T10:19:19.268874+0000 mgr.y (mgr.14427) 226 : cluster [DBG] pgmap v199: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-10-01T10:19:20.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:20 smithi078 bash[27336]: audit 2023-10-01T10:19:19.445080+0000 mon.a (mon.0) 766 : audit [DBG] from='client.? 172.21.15.78:0/796236399' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:20.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:20 smithi173 bash[27612]: cluster 2023-10-01T10:19:19.268874+0000 mgr.y (mgr.14427) 226 : cluster [DBG] pgmap v199: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-10-01T10:19:20.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:20 smithi173 bash[27612]: audit 2023-10-01T10:19:19.445080+0000 mon.a (mon.0) 766 : audit [DBG] from='client.? 172.21.15.78:0/796236399' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:20.620 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:19:20 smithi173 bash[50431]: debug 2023-10-01T10:19:20.176+0000 7f9f1c75d200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-01T10:19:20.620 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:19:20 smithi173 bash[50431]: debug 2023-10-01T10:19:20.424+0000 7f9f1c75d200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-01T10:19:21.121 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:19:20 smithi173 bash[50431]: debug 2023-10-01T10:19:20.756+0000 7f9f1c75d200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-01T10:19:21.368 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 86 16 109018 109002 79.209 46.875 0.0140169 0.0126178 2023-10-01T10:19:22.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:22 smithi078 bash[27336]: cluster 2023-10-01T10:19:21.269338+0000 mgr.y (mgr.14427) 227 : cluster [DBG] pgmap v200: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 46 MiB/s wr, 735 op/s 2023-10-01T10:19:22.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:22 smithi078 bash[27336]: audit 2023-10-01T10:19:22.032419+0000 mon.a (mon.0) 767 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:19:22.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:22 smithi078 bash[27336]: audit 2023-10-01T10:19:22.065649+0000 mon.b (mon.1) 32 : audit [DBG] from='client.? 172.21.15.78:0/967034115' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:22.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:22 smithi078 bash[33821]: cluster 2023-10-01T10:19:21.269338+0000 mgr.y (mgr.14427) 227 : cluster [DBG] pgmap v200: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 46 MiB/s wr, 735 op/s 2023-10-01T10:19:22.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:22 smithi078 bash[33821]: audit 2023-10-01T10:19:22.032419+0000 mon.a (mon.0) 767 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:19:22.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:22 smithi078 bash[33821]: audit 2023-10-01T10:19:22.065649+0000 mon.b (mon.1) 32 : audit [DBG] from='client.? 172.21.15.78:0/967034115' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:22.368 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 87 16 109634 109618 78.7411 38.5 0.0136511 0.0126928 2023-10-01T10:19:22.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:22 smithi173 bash[27612]: cluster 2023-10-01T10:19:21.269338+0000 mgr.y (mgr.14427) 227 : cluster [DBG] pgmap v200: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 46 MiB/s wr, 735 op/s 2023-10-01T10:19:22.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:22 smithi173 bash[27612]: audit 2023-10-01T10:19:22.032419+0000 mon.a (mon.0) 767 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:19:22.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:22 smithi173 bash[27612]: audit 2023-10-01T10:19:22.065649+0000 mon.b (mon.1) 32 : audit [DBG] from='client.? 172.21.15.78:0/967034115' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:22.808 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:19:22 smithi173 bash[50431]: debug 2023-10-01T10:19:22.432+0000 7f9f1c75d200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-01T10:19:22.808 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:19:22 smithi173 bash[50431]: debug 2023-10-01T10:19:22.624+0000 7f9f1c75d200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-01T10:19:22.808 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:19:22 smithi173 bash[50431]: debug 2023-10-01T10:19:22.804+0000 7f9f1c75d200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-01T10:19:23.368 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 88 16 110362 110346 78.3633 45.5 0.024192 0.012755 2023-10-01T10:19:23.455 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:19:23 smithi078 bash[27601]: ignoring --setuser ceph since I am not root 2023-10-01T10:19:23.455 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:19:23 smithi078 bash[27601]: ignoring --setgroup ceph since I am not root 2023-10-01T10:19:23.455 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:19:23 smithi078 bash[27601]: debug 2023-10-01T10:19:23.212+0000 7f13600c7700 1 -- 172.21.15.78:0/2183603444 <== mon.0 v2:172.21.15.78:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55e084bb04e0 con 0x55e084ca4c00 2023-10-01T10:19:23.455 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:19:23 smithi078 bash[27601]: debug 2023-10-01T10:19:23.448+0000 7f137028e680 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-01T10:19:23.514 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:19:23 smithi173 bash[50431]: debug 2023-10-01T10:19:23.512+0000 7f9f1c75d200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-01T10:19:23.754 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:19:23 smithi078 bash[27601]: debug 2023-10-01T10:19:23.596+0000 7f137028e680 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-01T10:19:23.827 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:23 smithi173 bash[27612]: audit 2023-10-01T10:19:22.749795+0000 mon.a (mon.0) 768 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:19:23.828 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:23 smithi173 bash[27612]: audit 2023-10-01T10:19:23.109468+0000 mon.a (mon.0) 769 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:19:23.828 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:23 smithi173 bash[27612]: audit 2023-10-01T10:19:23.110819+0000 mon.a (mon.0) 770 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:19:23.828 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:23 smithi173 bash[27612]: audit 2023-10-01T10:19:23.117666+0000 mon.a (mon.0) 771 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:19:23.828 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:23 smithi173 bash[27612]: audit 2023-10-01T10:19:23.125751+0000 mon.a (mon.0) 772 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:19:23.828 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:23 smithi173 bash[27612]: audit 2023-10-01T10:19:23.133905+0000 mon.a (mon.0) 773 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-10-01T10:19:23.828 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:23 smithi173 bash[27612]: cluster 2023-10-01T10:19:23.142643+0000 mon.a (mon.0) 774 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2023-10-01T10:19:23.828 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:23 smithi173 bash[27612]: audit 2023-10-01T10:19:23.147544+0000 mon.a (mon.0) 775 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-10-01T10:19:23.828 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:23 smithi173 bash[27612]: cluster 2023-10-01T10:19:23.147610+0000 mon.a (mon.0) 776 : cluster [DBG] mgrmap e19: x(active, starting, since 0.0129939s) 2023-10-01T10:19:23.829 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:19:23 smithi173 bash[50431]: debug 2023-10-01T10:19:23.824+0000 7f9f1c75d200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-01T10:19:24.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:23 smithi078 bash[27336]: audit 2023-10-01T10:19:22.749795+0000 mon.a (mon.0) 768 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:19:24.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:23 smithi078 bash[27336]: audit 2023-10-01T10:19:23.109468+0000 mon.a (mon.0) 769 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:19:24.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:23 smithi078 bash[27336]: audit 2023-10-01T10:19:23.110819+0000 mon.a (mon.0) 770 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:19:24.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:23 smithi078 bash[27336]: audit 2023-10-01T10:19:23.117666+0000 mon.a (mon.0) 771 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:19:24.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:23 smithi078 bash[27336]: audit 2023-10-01T10:19:23.125751+0000 mon.a (mon.0) 772 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:19:24.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:23 smithi078 bash[27336]: audit 2023-10-01T10:19:23.133905+0000 mon.a (mon.0) 773 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-10-01T10:19:24.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:23 smithi078 bash[27336]: cluster 2023-10-01T10:19:23.142643+0000 mon.a (mon.0) 774 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2023-10-01T10:19:24.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:23 smithi078 bash[27336]: audit 2023-10-01T10:19:23.147544+0000 mon.a (mon.0) 775 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-10-01T10:19:24.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:23 smithi078 bash[27336]: cluster 2023-10-01T10:19:23.147610+0000 mon.a (mon.0) 776 : cluster [DBG] mgrmap e19: x(active, starting, since 0.0129939s) 2023-10-01T10:19:24.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:23 smithi078 bash[33821]: audit 2023-10-01T10:19:22.749795+0000 mon.a (mon.0) 768 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:19:24.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:23 smithi078 bash[33821]: audit 2023-10-01T10:19:23.109468+0000 mon.a (mon.0) 769 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:19:24.085 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:23 smithi078 bash[33821]: audit 2023-10-01T10:19:23.110819+0000 mon.a (mon.0) 770 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:19:24.085 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:23 smithi078 bash[33821]: audit 2023-10-01T10:19:23.117666+0000 mon.a (mon.0) 771 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' 2023-10-01T10:19:24.085 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:23 smithi078 bash[33821]: audit 2023-10-01T10:19:23.125751+0000 mon.a (mon.0) 772 : audit [DBG] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:19:24.085 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:23 smithi078 bash[33821]: audit 2023-10-01T10:19:23.133905+0000 mon.a (mon.0) 773 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-10-01T10:19:24.085 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:23 smithi078 bash[33821]: cluster 2023-10-01T10:19:23.142643+0000 mon.a (mon.0) 774 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2023-10-01T10:19:24.085 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:23 smithi078 bash[33821]: audit 2023-10-01T10:19:23.147544+0000 mon.a (mon.0) 775 : audit [INF] from='mgr.14427 172.21.15.78:0/3605470137' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-10-01T10:19:24.085 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:23 smithi078 bash[33821]: cluster 2023-10-01T10:19:23.147610+0000 mon.a (mon.0) 776 : cluster [DBG] mgrmap e19: x(active, starting, since 0.0129939s) 2023-10-01T10:19:24.368 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 89 16 111129 111113 78.0214 47.9375 0.0224096 0.0128106 2023-10-01T10:19:24.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:19:24 smithi078 bash[27601]: debug 2023-10-01T10:19:24.320+0000 7f137028e680 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-01T10:19:24.758 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:19:24 smithi173 bash[50431]: debug 2023-10-01T10:19:24.451+0000 7f9f1c75d200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-01T10:19:25.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:24 smithi078 bash[27336]: audit 2023-10-01T10:19:24.741117+0000 mon.a (mon.0) 777 : audit [DBG] from='client.? 172.21.15.78:0/134680753' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:25.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:24 smithi078 bash[33821]: audit 2023-10-01T10:19:24.741117+0000 mon.a (mon.0) 777 : audit [DBG] from='client.? 172.21.15.78:0/134680753' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:25.118 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:19:25 smithi173 bash[50431]: debug 2023-10-01T10:19:25.115+0000 7f9f1c75d200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-01T10:19:25.118 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:24 smithi173 bash[27612]: audit 2023-10-01T10:19:24.741117+0000 mon.a (mon.0) 777 : audit [DBG] from='client.? 172.21.15.78:0/134680753' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:25.368 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 90 11 111893 111882 77.6884 48.0625 0.0111716 0.0128663 2023-10-01T10:19:26.009 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:19:25 smithi078 bash[27601]: debug 2023-10-01T10:19:25.620+0000 7f137028e680 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-01T10:19:26.009 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:19:25 smithi078 bash[27601]: debug 2023-10-01T10:19:25.728+0000 7f137028e680 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-01T10:19:26.119 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:19:25 smithi173 bash[50431]: debug 2023-10-01T10:19:25.731+0000 7f9f1c75d200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-01T10:19:26.119 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:19:25 smithi173 bash[50431]: debug 2023-10-01T10:19:25.843+0000 7f9f1c75d200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-01T10:19:26.333 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:19:26 smithi078 bash[27601]: debug 2023-10-01T10:19:26.004+0000 7f137028e680 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-01T10:19:26.368 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Total time run: 90.0169 2023-10-01T10:19:26.368 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Total writes made: 111893 2023-10-01T10:19:26.368 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Write size: 65536 2023-10-01T10:19:26.369 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Object size: 65536 2023-10-01T10:19:26.369 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Bandwidth (MB/sec): 77.6889 2023-10-01T10:19:26.369 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Stddev Bandwidth: 30.7852 2023-10-01T10:19:26.369 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Max bandwidth (MB/sec): 183.938 2023-10-01T10:19:26.369 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Min bandwidth (MB/sec): 38.5 2023-10-01T10:19:26.369 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Average IOPS: 1243 2023-10-01T10:19:26.369 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Stddev IOPS: 492.563 2023-10-01T10:19:26.369 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Max IOPS: 2943 2023-10-01T10:19:26.369 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Min IOPS: 616 2023-10-01T10:19:26.369 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Average Latency(s): 0.0128668 2023-10-01T10:19:26.369 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Stddev Latency(s): 0.00854058 2023-10-01T10:19:26.370 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Max latency(s): 0.0967229 2023-10-01T10:19:26.370 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Min latency(s): 0.00135626 2023-10-01T10:19:26.381 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Cleaning up (deleting benchmark objects) 2023-10-01T10:19:26.491 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:19:26 smithi173 bash[50431]: debug 2023-10-01T10:19:26.195+0000 7f9f1c75d200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-01T10:19:26.491 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:19:26 smithi173 bash[50431]: debug 2023-10-01T10:19:26.307+0000 7f9f1c75d200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-01T10:19:26.491 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:19:26 smithi173 bash[50431]: debug 2023-10-01T10:19:26.487+0000 7f9f1c75d200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-01T10:19:26.819 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:19:26 smithi173 bash[50431]: debug 2023-10-01T10:19:26.639+0000 7f9f1c75d200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-01T10:19:26.819 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:19:26 smithi173 bash[50431]: debug 2023-10-01T10:19:26.815+0000 7f9f1c75d200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-01T10:19:27.079 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:19:27 smithi173 bash[50431]: debug 2023-10-01T10:19:27.075+0000 7f9f1c75d200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-01T10:19:27.370 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:19:27 smithi173 bash[50431]: debug 2023-10-01T10:19:27.239+0000 7f9f1c75d200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-01T10:19:27.816 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:27 smithi078 bash[27336]: audit 2023-10-01T10:19:27.386057+0000 mon.b (mon.1) 33 : audit [DBG] from='client.? 172.21.15.78:0/1171213368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:27.817 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:19:27 smithi078 bash[27601]: debug 2023-10-01T10:19:27.572+0000 7f137028e680 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-01T10:19:27.817 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:27 smithi078 bash[33821]: audit 2023-10-01T10:19:27.386057+0000 mon.b (mon.1) 33 : audit [DBG] from='client.? 172.21.15.78:0/1171213368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:27.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:27 smithi173 bash[27612]: audit 2023-10-01T10:19:27.386057+0000 mon.b (mon.1) 33 : audit [DBG] from='client.? 172.21.15.78:0/1171213368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:28.083 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:19:27 smithi078 bash[27601]: debug 2023-10-01T10:19:27.812+0000 7f137028e680 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-01T10:19:28.083 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:19:27 smithi078 bash[27601]: debug 2023-10-01T10:19:27.952+0000 7f137028e680 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-01T10:19:28.447 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:19:28 smithi078 bash[27601]: debug 2023-10-01T10:19:28.116+0000 7f137028e680 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-01T10:19:28.447 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:19:28 smithi078 bash[27601]: debug 2023-10-01T10:19:28.440+0000 7f137028e680 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-01T10:19:29.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:29 smithi078 bash[27336]: cluster 2023-10-01T10:19:28.951779+0000 mon.a (mon.0) 778 : cluster [INF] Active manager daemon x restarted 2023-10-01T10:19:29.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:29 smithi078 bash[27336]: cluster 2023-10-01T10:19:28.952439+0000 mon.a (mon.0) 779 : cluster [INF] Activating manager daemon x 2023-10-01T10:19:29.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:29 smithi078 bash[27336]: audit 2023-10-01T10:19:28.955559+0000 mon.b (mon.1) 34 : audit [DBG] from='mgr.? 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-01T10:19:29.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:29 smithi078 bash[27336]: audit 2023-10-01T10:19:28.956750+0000 mon.b (mon.1) 35 : audit [DBG] from='mgr.? 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-01T10:19:29.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:29 smithi078 bash[27336]: cluster 2023-10-01T10:19:28.964549+0000 mon.a (mon.0) 780 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2023-10-01T10:19:29.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:29 smithi078 bash[27336]: cluster 2023-10-01T10:19:28.965401+0000 mon.a (mon.0) 781 : cluster [DBG] mgrmap e20: x(active, starting, since 0.0132582s) 2023-10-01T10:19:29.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:29 smithi078 bash[27336]: audit 2023-10-01T10:19:28.974633+0000 mon.c (mon.2) 52 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-01T10:19:29.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:29 smithi078 bash[27336]: audit 2023-10-01T10:19:28.981492+0000 mon.c (mon.2) 53 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T10:19:29.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:29 smithi078 bash[27336]: audit 2023-10-01T10:19:28.982913+0000 mon.c (mon.2) 54 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T10:19:29.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:29 smithi078 bash[27336]: audit 2023-10-01T10:19:28.985024+0000 mon.c (mon.2) 55 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T10:19:29.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:29 smithi078 bash[27336]: audit 2023-10-01T10:19:28.987029+0000 mon.c (mon.2) 56 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi078.vjqrth"}]: dispatch 2023-10-01T10:19:29.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:29 smithi078 bash[27336]: audit 2023-10-01T10:19:28.988668+0000 mon.c (mon.2) 57 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi173.vrqqoc"}]: dispatch 2023-10-01T10:19:29.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:29 smithi078 bash[27336]: audit 2023-10-01T10:19:28.990322+0000 mon.c (mon.2) 58 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-01T10:19:29.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:29 smithi078 bash[27336]: audit 2023-10-01T10:19:28.991723+0000 mon.c (mon.2) 59 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T10:19:29.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:29 smithi078 bash[27336]: audit 2023-10-01T10:19:28.993186+0000 mon.c (mon.2) 60 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T10:19:29.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:29 smithi078 bash[27336]: audit 2023-10-01T10:19:28.994763+0000 mon.c (mon.2) 61 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T10:19:29.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:29 smithi078 bash[27336]: audit 2023-10-01T10:19:28.996310+0000 mon.c (mon.2) 62 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T10:19:29.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:29 smithi078 bash[27336]: audit 2023-10-01T10:19:28.997793+0000 mon.c (mon.2) 63 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T10:19:29.336 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:29 smithi078 bash[27336]: audit 2023-10-01T10:19:28.999145+0000 mon.c (mon.2) 64 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T10:19:29.336 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:29 smithi078 bash[27336]: audit 2023-10-01T10:19:29.000474+0000 mon.c (mon.2) 65 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T10:19:29.336 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:29 smithi078 bash[27336]: audit 2023-10-01T10:19:29.001913+0000 mon.c (mon.2) 66 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T10:19:29.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:29 smithi078 bash[33821]: cluster 2023-10-01T10:19:28.951779+0000 mon.a (mon.0) 778 : cluster [INF] Active manager daemon x restarted 2023-10-01T10:19:29.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:29 smithi078 bash[33821]: cluster 2023-10-01T10:19:28.952439+0000 mon.a (mon.0) 779 : cluster [INF] Activating manager daemon x 2023-10-01T10:19:29.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:29 smithi078 bash[33821]: audit 2023-10-01T10:19:28.955559+0000 mon.b (mon.1) 34 : audit [DBG] from='mgr.? 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-01T10:19:29.337 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:29 smithi078 bash[33821]: audit 2023-10-01T10:19:28.956750+0000 mon.b (mon.1) 35 : audit [DBG] from='mgr.? 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-01T10:19:29.337 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:29 smithi078 bash[33821]: cluster 2023-10-01T10:19:28.964549+0000 mon.a (mon.0) 780 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2023-10-01T10:19:29.337 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:29 smithi078 bash[33821]: cluster 2023-10-01T10:19:28.965401+0000 mon.a (mon.0) 781 : cluster [DBG] mgrmap e20: x(active, starting, since 0.0132582s) 2023-10-01T10:19:29.337 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:29 smithi078 bash[33821]: audit 2023-10-01T10:19:28.974633+0000 mon.c (mon.2) 52 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-01T10:19:29.337 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:29 smithi078 bash[33821]: audit 2023-10-01T10:19:28.981492+0000 mon.c (mon.2) 53 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T10:19:29.337 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:29 smithi078 bash[33821]: audit 2023-10-01T10:19:28.982913+0000 mon.c (mon.2) 54 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T10:19:29.337 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:29 smithi078 bash[33821]: audit 2023-10-01T10:19:28.985024+0000 mon.c (mon.2) 55 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T10:19:29.337 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:29 smithi078 bash[33821]: audit 2023-10-01T10:19:28.987029+0000 mon.c (mon.2) 56 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi078.vjqrth"}]: dispatch 2023-10-01T10:19:29.338 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:29 smithi078 bash[33821]: audit 2023-10-01T10:19:28.988668+0000 mon.c (mon.2) 57 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi173.vrqqoc"}]: dispatch 2023-10-01T10:19:29.338 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:29 smithi078 bash[33821]: audit 2023-10-01T10:19:28.990322+0000 mon.c (mon.2) 58 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-01T10:19:29.338 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:29 smithi078 bash[33821]: audit 2023-10-01T10:19:28.991723+0000 mon.c (mon.2) 59 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T10:19:29.338 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:29 smithi078 bash[33821]: audit 2023-10-01T10:19:28.993186+0000 mon.c (mon.2) 60 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T10:19:29.338 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:29 smithi078 bash[33821]: audit 2023-10-01T10:19:28.994763+0000 mon.c (mon.2) 61 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T10:19:29.338 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:29 smithi078 bash[33821]: audit 2023-10-01T10:19:28.996310+0000 mon.c (mon.2) 62 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T10:19:29.338 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:29 smithi078 bash[33821]: audit 2023-10-01T10:19:28.997793+0000 mon.c (mon.2) 63 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T10:19:29.338 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:29 smithi078 bash[33821]: audit 2023-10-01T10:19:28.999145+0000 mon.c (mon.2) 64 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T10:19:29.339 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:29 smithi078 bash[33821]: audit 2023-10-01T10:19:29.000474+0000 mon.c (mon.2) 65 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T10:19:29.339 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:29 smithi078 bash[33821]: audit 2023-10-01T10:19:29.001913+0000 mon.c (mon.2) 66 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T10:19:29.339 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:19:28 smithi078 bash[27601]: debug 2023-10-01T10:19:28.960+0000 7f137028e680 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-01T10:19:29.339 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:19:29 smithi078 bash[27601]: debug 2023-10-01T10:19:29.156+0000 7f137028e680 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-01T10:19:29.340 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:29 smithi173 bash[27612]: cluster 2023-10-01T10:19:28.951779+0000 mon.a (mon.0) 778 : cluster [INF] Active manager daemon x restarted 2023-10-01T10:19:29.340 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:29 smithi173 bash[27612]: cluster 2023-10-01T10:19:28.952439+0000 mon.a (mon.0) 779 : cluster [INF] Activating manager daemon x 2023-10-01T10:19:29.340 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:29 smithi173 bash[27612]: audit 2023-10-01T10:19:28.955559+0000 mon.b (mon.1) 34 : audit [DBG] from='mgr.? 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-01T10:19:29.340 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:29 smithi173 bash[27612]: audit 2023-10-01T10:19:28.956750+0000 mon.b (mon.1) 35 : audit [DBG] from='mgr.? 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-01T10:19:29.341 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:29 smithi173 bash[27612]: cluster 2023-10-01T10:19:28.964549+0000 mon.a (mon.0) 780 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2023-10-01T10:19:29.341 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:29 smithi173 bash[27612]: cluster 2023-10-01T10:19:28.965401+0000 mon.a (mon.0) 781 : cluster [DBG] mgrmap e20: x(active, starting, since 0.0132582s) 2023-10-01T10:19:29.341 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:29 smithi173 bash[27612]: audit 2023-10-01T10:19:28.974633+0000 mon.c (mon.2) 52 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-01T10:19:29.341 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:29 smithi173 bash[27612]: audit 2023-10-01T10:19:28.981492+0000 mon.c (mon.2) 53 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T10:19:29.341 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:29 smithi173 bash[27612]: audit 2023-10-01T10:19:28.982913+0000 mon.c (mon.2) 54 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T10:19:29.341 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:29 smithi173 bash[27612]: audit 2023-10-01T10:19:28.985024+0000 mon.c (mon.2) 55 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T10:19:29.342 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:29 smithi173 bash[27612]: audit 2023-10-01T10:19:28.987029+0000 mon.c (mon.2) 56 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi078.vjqrth"}]: dispatch 2023-10-01T10:19:29.342 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:29 smithi173 bash[27612]: audit 2023-10-01T10:19:28.988668+0000 mon.c (mon.2) 57 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi173.vrqqoc"}]: dispatch 2023-10-01T10:19:29.342 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:29 smithi173 bash[27612]: audit 2023-10-01T10:19:28.990322+0000 mon.c (mon.2) 58 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-01T10:19:29.342 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:29 smithi173 bash[27612]: audit 2023-10-01T10:19:28.991723+0000 mon.c (mon.2) 59 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T10:19:29.342 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:29 smithi173 bash[27612]: audit 2023-10-01T10:19:28.993186+0000 mon.c (mon.2) 60 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T10:19:29.342 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:29 smithi173 bash[27612]: audit 2023-10-01T10:19:28.994763+0000 mon.c (mon.2) 61 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T10:19:29.342 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:29 smithi173 bash[27612]: audit 2023-10-01T10:19:28.996310+0000 mon.c (mon.2) 62 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T10:19:29.342 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:29 smithi173 bash[27612]: audit 2023-10-01T10:19:28.997793+0000 mon.c (mon.2) 63 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T10:19:29.342 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:29 smithi173 bash[27612]: audit 2023-10-01T10:19:28.999145+0000 mon.c (mon.2) 64 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T10:19:29.342 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:29 smithi173 bash[27612]: audit 2023-10-01T10:19:29.000474+0000 mon.c (mon.2) 65 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T10:19:29.343 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:29 smithi173 bash[27612]: audit 2023-10-01T10:19:29.001913+0000 mon.c (mon.2) 66 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T10:19:29.343 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:19:28 smithi173 bash[50431]: debug 2023-10-01T10:19:28.927+0000 7f9f1c75d200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-01T10:19:29.343 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:19:28 smithi173 bash[50431]: [01/Oct/2023:10:19:28] ENGINE Bus STARTING 2023-10-01T10:19:29.343 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:19:29 smithi173 bash[50431]: CherryPy Checker: 2023-10-01T10:19:29.343 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:19:29 smithi173 bash[50431]: The Application mounted at '' has an empty config. 2023-10-01T10:19:29.343 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:19:29 smithi173 bash[50431]: [01/Oct/2023:10:19:29] ENGINE Serving on http://:::9283 2023-10-01T10:19:29.344 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:19:29 smithi173 bash[50431]: [01/Oct/2023:10:19:29] ENGINE Bus STARTED 2023-10-01T10:19:29.344 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:19:29 smithi173 bash[50431]: [01/Oct/2023:10:19:29] ENGINE Bus STOPPING 2023-10-01T10:19:29.344 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:19:29 smithi173 bash[50431]: [01/Oct/2023:10:19:29] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-01T10:19:29.344 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:19:29 smithi173 bash[50431]: [01/Oct/2023:10:19:29] ENGINE Bus STOPPED 2023-10-01T10:19:29.344 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:19:29 smithi173 bash[50431]: [01/Oct/2023:10:19:29] ENGINE Bus STARTING 2023-10-01T10:19:29.601 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:19:29 smithi173 bash[50431]: debug 2023-10-01T10:19:29.535+0000 7f9e7b3fa700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-01T10:19:29.601 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:19:29 smithi173 bash[50431]: debug 2023-10-01T10:19:29.535+0000 7f9e7b3fa700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-01T10:19:29.601 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:19:29 smithi173 bash[50431]: debug 2023-10-01T10:19:29.535+0000 7f9e7b3fa700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-01T10:19:29.601 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:19:29 smithi173 bash[50431]: debug 2023-10-01T10:19:29.535+0000 7f9e7b3fa700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-01T10:19:29.601 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:19:29 smithi173 bash[50431]: debug 2023-10-01T10:19:29.535+0000 7f9e7b3fa700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-01T10:19:29.601 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:19:29 smithi173 bash[50431]: debug 2023-10-01T10:19:29.539+0000 7f9e80c05700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-01T10:19:29.601 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:19:29 smithi173 bash[50431]: debug 2023-10-01T10:19:29.539+0000 7f9e80c05700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-01T10:19:29.602 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:19:29 smithi173 bash[50431]: debug 2023-10-01T10:19:29.539+0000 7f9e80c05700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-01T10:19:29.602 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:19:29 smithi173 bash[50431]: debug 2023-10-01T10:19:29.539+0000 7f9e80c05700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-01T10:19:29.602 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:19:29 smithi173 bash[50431]: debug 2023-10-01T10:19:29.539+0000 7f9e80c05700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-01T10:19:29.602 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:19:29 smithi173 bash[50431]: [01/Oct/2023:10:19:29] ENGINE Serving on http://:::9283 2023-10-01T10:19:29.602 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:19:29 smithi173 bash[50431]: [01/Oct/2023:10:19:29] ENGINE Bus STARTED 2023-10-01T10:19:30.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:30 smithi078 bash[27336]: audit 2023-10-01T10:19:29.003584+0000 mon.c (mon.2) 67 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-01T10:19:30.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:30 smithi078 bash[27336]: audit 2023-10-01T10:19:29.006087+0000 mon.c (mon.2) 68 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-01T10:19:30.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:30 smithi078 bash[27336]: audit 2023-10-01T10:19:29.006707+0000 mon.c (mon.2) 69 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-01T10:19:30.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:30 smithi078 bash[27336]: audit 2023-10-01T10:19:29.007266+0000 mon.c (mon.2) 70 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-01T10:19:30.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:30 smithi078 bash[27336]: audit 2023-10-01T10:19:29.017462+0000 mon.c (mon.2) 71 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-01T10:19:30.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:30 smithi078 bash[27336]: cluster 2023-10-01T10:19:29.230741+0000 mon.a (mon.0) 782 : cluster [INF] Manager daemon x is now available 2023-10-01T10:19:30.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:30 smithi078 bash[27336]: audit 2023-10-01T10:19:29.274363+0000 mon.a (mon.0) 783 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:30.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:30 smithi078 bash[27336]: audit 2023-10-01T10:19:29.315846+0000 mon.c (mon.2) 72 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:19:30.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:30 smithi078 bash[27336]: audit 2023-10-01T10:19:29.361829+0000 mon.c (mon.2) 73 : audit [INF] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T10:19:30.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:30 smithi078 bash[27336]: audit 2023-10-01T10:19:29.362527+0000 mon.a (mon.0) 784 : audit [INF] from='mgr.24697 ' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T10:19:30.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:30 smithi078 bash[27336]: audit 2023-10-01T10:19:29.375842+0000 mon.c (mon.2) 74 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:19:30.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:30 smithi078 bash[27336]: audit 2023-10-01T10:19:29.398994+0000 mon.c (mon.2) 75 : audit [INF] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-01T10:19:30.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:30 smithi078 bash[27336]: audit 2023-10-01T10:19:29.399417+0000 mon.a (mon.0) 785 : audit [INF] from='mgr.24697 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-01T10:19:30.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:30 smithi078 bash[27336]: audit 2023-10-01T10:19:29.401526+0000 mon.c (mon.2) 76 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:19:30.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:30 smithi078 bash[27336]: audit 2023-10-01T10:19:29.501310+0000 mon.a (mon.0) 786 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:30.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:30 smithi078 bash[27336]: audit 2023-10-01T10:19:29.518036+0000 mon.c (mon.2) 77 : audit [INF] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-01T10:19:30.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:30 smithi078 bash[27336]: audit 2023-10-01T10:19:29.518583+0000 mon.a (mon.0) 787 : audit [INF] from='mgr.24697 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-01T10:19:30.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:30 smithi078 bash[33821]: audit 2023-10-01T10:19:29.003584+0000 mon.c (mon.2) 67 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-01T10:19:30.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:30 smithi078 bash[33821]: audit 2023-10-01T10:19:29.006087+0000 mon.c (mon.2) 68 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-01T10:19:30.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:30 smithi078 bash[33821]: audit 2023-10-01T10:19:29.006707+0000 mon.c (mon.2) 69 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-01T10:19:30.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:30 smithi078 bash[33821]: audit 2023-10-01T10:19:29.007266+0000 mon.c (mon.2) 70 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-01T10:19:30.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:30 smithi078 bash[33821]: audit 2023-10-01T10:19:29.017462+0000 mon.c (mon.2) 71 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-01T10:19:30.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:30 smithi078 bash[33821]: cluster 2023-10-01T10:19:29.230741+0000 mon.a (mon.0) 782 : cluster [INF] Manager daemon x is now available 2023-10-01T10:19:30.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:30 smithi078 bash[33821]: audit 2023-10-01T10:19:29.274363+0000 mon.a (mon.0) 783 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:30.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:30 smithi078 bash[33821]: audit 2023-10-01T10:19:29.315846+0000 mon.c (mon.2) 72 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:19:30.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:30 smithi078 bash[33821]: audit 2023-10-01T10:19:29.361829+0000 mon.c (mon.2) 73 : audit [INF] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T10:19:30.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:30 smithi078 bash[33821]: audit 2023-10-01T10:19:29.362527+0000 mon.a (mon.0) 784 : audit [INF] from='mgr.24697 ' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T10:19:30.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:30 smithi078 bash[33821]: audit 2023-10-01T10:19:29.375842+0000 mon.c (mon.2) 74 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:19:30.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:30 smithi078 bash[33821]: audit 2023-10-01T10:19:29.398994+0000 mon.c (mon.2) 75 : audit [INF] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-01T10:19:30.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:30 smithi078 bash[33821]: audit 2023-10-01T10:19:29.399417+0000 mon.a (mon.0) 785 : audit [INF] from='mgr.24697 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-01T10:19:30.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:30 smithi078 bash[33821]: audit 2023-10-01T10:19:29.401526+0000 mon.c (mon.2) 76 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:19:30.337 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:30 smithi078 bash[33821]: audit 2023-10-01T10:19:29.501310+0000 mon.a (mon.0) 786 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:30.337 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:30 smithi078 bash[33821]: audit 2023-10-01T10:19:29.518036+0000 mon.c (mon.2) 77 : audit [INF] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-01T10:19:30.337 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:30 smithi078 bash[33821]: audit 2023-10-01T10:19:29.518583+0000 mon.a (mon.0) 787 : audit [INF] from='mgr.24697 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-01T10:19:30.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:30 smithi173 bash[27612]: audit 2023-10-01T10:19:29.003584+0000 mon.c (mon.2) 67 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-01T10:19:30.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:30 smithi173 bash[27612]: audit 2023-10-01T10:19:29.006087+0000 mon.c (mon.2) 68 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-01T10:19:30.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:30 smithi173 bash[27612]: audit 2023-10-01T10:19:29.006707+0000 mon.c (mon.2) 69 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-01T10:19:30.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:30 smithi173 bash[27612]: audit 2023-10-01T10:19:29.007266+0000 mon.c (mon.2) 70 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-01T10:19:30.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:30 smithi173 bash[27612]: audit 2023-10-01T10:19:29.017462+0000 mon.c (mon.2) 71 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-01T10:19:30.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:30 smithi173 bash[27612]: cluster 2023-10-01T10:19:29.230741+0000 mon.a (mon.0) 782 : cluster [INF] Manager daemon x is now available 2023-10-01T10:19:30.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:30 smithi173 bash[27612]: audit 2023-10-01T10:19:29.274363+0000 mon.a (mon.0) 783 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:30.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:30 smithi173 bash[27612]: audit 2023-10-01T10:19:29.315846+0000 mon.c (mon.2) 72 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:19:30.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:30 smithi173 bash[27612]: audit 2023-10-01T10:19:29.361829+0000 mon.c (mon.2) 73 : audit [INF] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T10:19:30.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:30 smithi173 bash[27612]: audit 2023-10-01T10:19:29.362527+0000 mon.a (mon.0) 784 : audit [INF] from='mgr.24697 ' entity='mgr.x' cmd=[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T10:19:30.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:30 smithi173 bash[27612]: audit 2023-10-01T10:19:29.375842+0000 mon.c (mon.2) 74 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:19:30.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:30 smithi173 bash[27612]: audit 2023-10-01T10:19:29.398994+0000 mon.c (mon.2) 75 : audit [INF] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-01T10:19:30.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:30 smithi173 bash[27612]: audit 2023-10-01T10:19:29.399417+0000 mon.a (mon.0) 785 : audit [INF] from='mgr.24697 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-10-01T10:19:30.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:30 smithi173 bash[27612]: audit 2023-10-01T10:19:29.401526+0000 mon.c (mon.2) 76 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:19:30.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:30 smithi173 bash[27612]: audit 2023-10-01T10:19:29.501310+0000 mon.a (mon.0) 786 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:30.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:30 smithi173 bash[27612]: audit 2023-10-01T10:19:29.518036+0000 mon.c (mon.2) 77 : audit [INF] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-01T10:19:30.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:30 smithi173 bash[27612]: audit 2023-10-01T10:19:29.518583+0000 mon.a (mon.0) 787 : audit [INF] from='mgr.24697 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-10-01T10:19:30.761 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:19:30 smithi078 bash[27601]: debug 2023-10-01T10:19:30.448+0000 7f137028e680 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-01T10:19:30.761 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:19:30 smithi078 bash[27601]: debug 2023-10-01T10:19:30.588+0000 7f137028e680 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-01T10:19:30.761 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:19:30 smithi078 bash[27601]: debug 2023-10-01T10:19:30.756+0000 7f137028e680 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-01T10:19:31.032 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:31 smithi078 bash[27336]: cephadm 2023-10-01T10:19:30.010427+0000 mgr.x (mgr.24697) 1 : cephadm [INF] Deploying cephadm binary to smithi173 2023-10-01T10:19:31.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:31 smithi078 bash[27336]: cephadm 2023-10-01T10:19:30.024663+0000 mgr.x (mgr.24697) 2 : cephadm [INF] Deploying cephadm binary to smithi078 2023-10-01T10:19:31.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:31 smithi078 bash[27336]: cluster 2023-10-01T10:19:30.032044+0000 mon.a (mon.0) 788 : cluster [DBG] mgrmap e21: x(active, since 1.07993s) 2023-10-01T10:19:31.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:31 smithi078 bash[27336]: audit 2023-10-01T10:19:30.077043+0000 mon.c (mon.2) 78 : audit [DBG] from='client.? 172.21.15.78:0/2936075757' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:31.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:31 smithi078 bash[27336]: cluster 2023-10-01T10:19:30.116723+0000 mgr.x (mgr.24697) 3 : cluster [DBG] pgmap v3: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail 2023-10-01T10:19:31.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:31 smithi078 bash[27336]: audit 2023-10-01T10:19:30.276924+0000 mon.a (mon.0) 789 : audit [INF] from='mgr.24697 ' entity='mgr.x' cmd='[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]': finished 2023-10-01T10:19:31.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:31 smithi078 bash[27336]: cluster 2023-10-01T10:19:30.277127+0000 mon.a (mon.0) 790 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2023-10-01T10:19:31.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:31 smithi078 bash[27336]: audit 2023-10-01T10:19:30.289433+0000 mon.c (mon.2) 79 : audit [INF] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T10:19:31.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:31 smithi078 bash[27336]: audit 2023-10-01T10:19:30.290089+0000 mon.a (mon.0) 791 : audit [INF] from='mgr.24697 ' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T10:19:31.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:31 smithi078 bash[27336]: audit 2023-10-01T10:19:30.298128+0000 mon.a (mon.0) 792 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:31.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:31 smithi078 bash[27336]: audit 2023-10-01T10:19:30.341996+0000 mon.a (mon.0) 793 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:31.334 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:19:31 smithi078 bash[27601]: debug 2023-10-01T10:19:31.080+0000 7f137028e680 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-01T10:19:31.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:31 smithi078 bash[33821]: cephadm 2023-10-01T10:19:30.010427+0000 mgr.x (mgr.24697) 1 : cephadm [INF] Deploying cephadm binary to smithi173 2023-10-01T10:19:31.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:31 smithi078 bash[33821]: cephadm 2023-10-01T10:19:30.024663+0000 mgr.x (mgr.24697) 2 : cephadm [INF] Deploying cephadm binary to smithi078 2023-10-01T10:19:31.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:31 smithi078 bash[33821]: cluster 2023-10-01T10:19:30.032044+0000 mon.a (mon.0) 788 : cluster [DBG] mgrmap e21: x(active, since 1.07993s) 2023-10-01T10:19:31.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:31 smithi078 bash[33821]: audit 2023-10-01T10:19:30.077043+0000 mon.c (mon.2) 78 : audit [DBG] from='client.? 172.21.15.78:0/2936075757' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:31.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:31 smithi078 bash[33821]: cluster 2023-10-01T10:19:30.116723+0000 mgr.x (mgr.24697) 3 : cluster [DBG] pgmap v3: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail 2023-10-01T10:19:31.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:31 smithi078 bash[33821]: audit 2023-10-01T10:19:30.276924+0000 mon.a (mon.0) 789 : audit [INF] from='mgr.24697 ' entity='mgr.x' cmd='[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]': finished 2023-10-01T10:19:31.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:31 smithi078 bash[33821]: cluster 2023-10-01T10:19:30.277127+0000 mon.a (mon.0) 790 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2023-10-01T10:19:31.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:31 smithi078 bash[33821]: audit 2023-10-01T10:19:30.289433+0000 mon.c (mon.2) 79 : audit [INF] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T10:19:31.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:31 smithi078 bash[33821]: audit 2023-10-01T10:19:30.290089+0000 mon.a (mon.0) 791 : audit [INF] from='mgr.24697 ' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T10:19:31.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:31 smithi078 bash[33821]: audit 2023-10-01T10:19:30.298128+0000 mon.a (mon.0) 792 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:31.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:31 smithi078 bash[33821]: audit 2023-10-01T10:19:30.341996+0000 mon.a (mon.0) 793 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:31.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:31 smithi173 bash[27612]: cephadm 2023-10-01T10:19:30.010427+0000 mgr.x (mgr.24697) 1 : cephadm [INF] Deploying cephadm binary to smithi173 2023-10-01T10:19:31.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:31 smithi173 bash[27612]: cephadm 2023-10-01T10:19:30.024663+0000 mgr.x (mgr.24697) 2 : cephadm [INF] Deploying cephadm binary to smithi078 2023-10-01T10:19:31.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:31 smithi173 bash[27612]: cluster 2023-10-01T10:19:30.032044+0000 mon.a (mon.0) 788 : cluster [DBG] mgrmap e21: x(active, since 1.07993s) 2023-10-01T10:19:31.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:31 smithi173 bash[27612]: audit 2023-10-01T10:19:30.077043+0000 mon.c (mon.2) 78 : audit [DBG] from='client.? 172.21.15.78:0/2936075757' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:31.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:31 smithi173 bash[27612]: cluster 2023-10-01T10:19:30.116723+0000 mgr.x (mgr.24697) 3 : cluster [DBG] pgmap v3: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail 2023-10-01T10:19:31.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:31 smithi173 bash[27612]: audit 2023-10-01T10:19:30.276924+0000 mon.a (mon.0) 789 : audit [INF] from='mgr.24697 ' entity='mgr.x' cmd='[{"prefix": "osd pool rename", "format": "json", "srcpool": "device_health_metrics", "destpool": ".mgr", "yes_i_really_mean_it": true}]': finished 2023-10-01T10:19:31.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:31 smithi173 bash[27612]: cluster 2023-10-01T10:19:30.277127+0000 mon.a (mon.0) 790 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2023-10-01T10:19:31.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:31 smithi173 bash[27612]: audit 2023-10-01T10:19:30.289433+0000 mon.c (mon.2) 79 : audit [INF] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T10:19:31.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:31 smithi173 bash[27612]: audit 2023-10-01T10:19:30.290089+0000 mon.a (mon.0) 791 : audit [INF] from='mgr.24697 ' entity='mgr.x' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T10:19:31.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:31 smithi173 bash[27612]: audit 2023-10-01T10:19:30.298128+0000 mon.a (mon.0) 792 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:31.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:31 smithi173 bash[27612]: audit 2023-10-01T10:19:30.341996+0000 mon.a (mon.0) 793 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:31.833 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:19:31 smithi078 bash[27601]: debug 2023-10-01T10:19:31.500+0000 7f137028e680 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-01T10:19:31.833 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:19:31 smithi078 bash[27601]: debug 2023-10-01T10:19:31.740+0000 7f137028e680 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-01T10:19:32.289 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:19:32 smithi078 bash[27601]: debug 2023-10-01T10:19:32.000+0000 7f137028e680 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-01T10:19:32.585 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:32 smithi078 bash[27336]: cluster 2023-10-01T10:19:30.989961+0000 mgr.x (mgr.24697) 4 : cluster [DBG] pgmap v5: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail 2023-10-01T10:19:32.585 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:32 smithi078 bash[27336]: cephadm 2023-10-01T10:19:31.222971+0000 mgr.x (mgr.24697) 5 : cephadm [INF] [01/Oct/2023:10:19:31] ENGINE Bus STARTING 2023-10-01T10:19:32.585 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:32 smithi078 bash[27336]: audit 2023-10-01T10:19:31.277309+0000 mon.a (mon.0) 794 : audit [INF] from='mgr.24697 ' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-10-01T10:19:32.586 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:32 smithi078 bash[27336]: cluster 2023-10-01T10:19:31.277397+0000 mon.a (mon.0) 795 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2023-10-01T10:19:32.586 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:32 smithi078 bash[27336]: cluster 2023-10-01T10:19:31.346556+0000 mon.a (mon.0) 796 : cluster [DBG] mgrmap e22: x(active, since 2s) 2023-10-01T10:19:32.586 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:19:32 smithi078 bash[27601]: debug 2023-10-01T10:19:32.340+0000 7f137028e680 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-01T10:19:32.586 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:19:32 smithi078 bash[27601]: debug 2023-10-01T10:19:32.492+0000 7f137028e680 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-01T10:19:32.586 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:19:32 smithi078 bash[27601]: [01/Oct/2023:10:19:32] ENGINE Bus STARTING 2023-10-01T10:19:32.587 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:32 smithi078 bash[33821]: cluster 2023-10-01T10:19:30.989961+0000 mgr.x (mgr.24697) 4 : cluster [DBG] pgmap v5: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail 2023-10-01T10:19:32.587 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:32 smithi078 bash[33821]: cephadm 2023-10-01T10:19:31.222971+0000 mgr.x (mgr.24697) 5 : cephadm [INF] [01/Oct/2023:10:19:31] ENGINE Bus STARTING 2023-10-01T10:19:32.587 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:32 smithi078 bash[33821]: audit 2023-10-01T10:19:31.277309+0000 mon.a (mon.0) 794 : audit [INF] from='mgr.24697 ' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-10-01T10:19:32.587 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:32 smithi078 bash[33821]: cluster 2023-10-01T10:19:31.277397+0000 mon.a (mon.0) 795 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2023-10-01T10:19:32.587 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:32 smithi078 bash[33821]: cluster 2023-10-01T10:19:31.346556+0000 mon.a (mon.0) 796 : cluster [DBG] mgrmap e22: x(active, since 2s) 2023-10-01T10:19:32.587 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:Oct 01 10:19:32 smithi078 bash[58231]: level=warn ts=2023-10-01T10:19:32.317Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi078.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.78:8443: connect: connection refused" 2023-10-01T10:19:32.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:32 smithi173 bash[27612]: cluster 2023-10-01T10:19:30.989961+0000 mgr.x (mgr.24697) 4 : cluster [DBG] pgmap v5: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail 2023-10-01T10:19:32.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:32 smithi173 bash[27612]: cephadm 2023-10-01T10:19:31.222971+0000 mgr.x (mgr.24697) 5 : cephadm [INF] [01/Oct/2023:10:19:31] ENGINE Bus STARTING 2023-10-01T10:19:32.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:32 smithi173 bash[27612]: audit 2023-10-01T10:19:31.277309+0000 mon.a (mon.0) 794 : audit [INF] from='mgr.24697 ' entity='mgr.x' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-10-01T10:19:32.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:32 smithi173 bash[27612]: cluster 2023-10-01T10:19:31.277397+0000 mon.a (mon.0) 795 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2023-10-01T10:19:32.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:32 smithi173 bash[27612]: cluster 2023-10-01T10:19:31.346556+0000 mon.a (mon.0) 796 : cluster [DBG] mgrmap e22: x(active, since 2s) 2023-10-01T10:19:32.995 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:Oct 01 10:19:32 smithi078 bash[58231]: level=info ts=2023-10-01T10:19:32.992Z caller=notify.go:735 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify success" attempts=2 2023-10-01T10:19:32.996 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:19:32 smithi078 bash[27601]: CherryPy Checker: 2023-10-01T10:19:32.996 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:19:32 smithi078 bash[27601]: The Application mounted at '' has an empty config. 2023-10-01T10:19:32.996 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:19:32 smithi078 bash[27601]: [01/Oct/2023:10:19:32] ENGINE Serving on http://:::9283 2023-10-01T10:19:32.996 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:19:32 smithi078 bash[27601]: [01/Oct/2023:10:19:32] ENGINE Bus STARTED 2023-10-01T10:19:33.300 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:33 smithi078 bash[27336]: cephadm 2023-10-01T10:19:31.408349+0000 mgr.x (mgr.24697) 6 : cephadm [INF] [01/Oct/2023:10:19:31] ENGINE Serving on https://172.21.15.173:7150 2023-10-01T10:19:33.300 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:33 smithi078 bash[27336]: cephadm 2023-10-01T10:19:31.537212+0000 mgr.x (mgr.24697) 7 : cephadm [INF] [01/Oct/2023:10:19:31] ENGINE Serving on http://172.21.15.173:8765 2023-10-01T10:19:33.300 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:33 smithi078 bash[27336]: cephadm 2023-10-01T10:19:31.537629+0000 mgr.x (mgr.24697) 8 : cephadm [INF] [01/Oct/2023:10:19:31] ENGINE Bus STARTED 2023-10-01T10:19:33.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:33 smithi078 bash[27336]: cluster 2023-10-01T10:19:32.520479+0000 mon.a (mon.0) 797 : cluster [DBG] Standby manager daemon y started 2023-10-01T10:19:33.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:33 smithi078 bash[27336]: audit 2023-10-01T10:19:32.520950+0000 mon.a (mon.0) 798 : audit [DBG] from='mgr.? 172.21.15.78:0/3303142635' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-01T10:19:33.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:33 smithi078 bash[27336]: audit 2023-10-01T10:19:32.527148+0000 mon.a (mon.0) 799 : audit [DBG] from='mgr.? 172.21.15.78:0/3303142635' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-01T10:19:33.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:33 smithi078 bash[27336]: audit 2023-10-01T10:19:32.531672+0000 mon.a (mon.0) 800 : audit [DBG] from='mgr.? 172.21.15.78:0/3303142635' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-01T10:19:33.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:33 smithi078 bash[27336]: audit 2023-10-01T10:19:32.542612+0000 mon.a (mon.0) 801 : audit [DBG] from='mgr.? 172.21.15.78:0/3303142635' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-01T10:19:33.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:33 smithi078 bash[27336]: audit 2023-10-01T10:19:32.799659+0000 mon.b (mon.1) 36 : audit [DBG] from='client.? 172.21.15.78:0/2235021475' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:33.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:33 smithi078 bash[33821]: cephadm 2023-10-01T10:19:31.408349+0000 mgr.x (mgr.24697) 6 : cephadm [INF] [01/Oct/2023:10:19:31] ENGINE Serving on https://172.21.15.173:7150 2023-10-01T10:19:33.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:33 smithi078 bash[33821]: cephadm 2023-10-01T10:19:31.537212+0000 mgr.x (mgr.24697) 7 : cephadm [INF] [01/Oct/2023:10:19:31] ENGINE Serving on http://172.21.15.173:8765 2023-10-01T10:19:33.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:33 smithi078 bash[33821]: cephadm 2023-10-01T10:19:31.537629+0000 mgr.x (mgr.24697) 8 : cephadm [INF] [01/Oct/2023:10:19:31] ENGINE Bus STARTED 2023-10-01T10:19:33.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:33 smithi078 bash[33821]: cluster 2023-10-01T10:19:32.520479+0000 mon.a (mon.0) 797 : cluster [DBG] Standby manager daemon y started 2023-10-01T10:19:33.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:33 smithi078 bash[33821]: audit 2023-10-01T10:19:32.520950+0000 mon.a (mon.0) 798 : audit [DBG] from='mgr.? 172.21.15.78:0/3303142635' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-01T10:19:33.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:33 smithi078 bash[33821]: audit 2023-10-01T10:19:32.527148+0000 mon.a (mon.0) 799 : audit [DBG] from='mgr.? 172.21.15.78:0/3303142635' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-01T10:19:33.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:33 smithi078 bash[33821]: audit 2023-10-01T10:19:32.531672+0000 mon.a (mon.0) 800 : audit [DBG] from='mgr.? 172.21.15.78:0/3303142635' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-01T10:19:33.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:33 smithi078 bash[33821]: audit 2023-10-01T10:19:32.542612+0000 mon.a (mon.0) 801 : audit [DBG] from='mgr.? 172.21.15.78:0/3303142635' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-01T10:19:33.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:33 smithi078 bash[33821]: audit 2023-10-01T10:19:32.799659+0000 mon.b (mon.1) 36 : audit [DBG] from='client.? 172.21.15.78:0/2235021475' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:33.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:33 smithi173 bash[27612]: cephadm 2023-10-01T10:19:31.408349+0000 mgr.x (mgr.24697) 6 : cephadm [INF] [01/Oct/2023:10:19:31] ENGINE Serving on https://172.21.15.173:7150 2023-10-01T10:19:33.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:33 smithi173 bash[27612]: cephadm 2023-10-01T10:19:31.537212+0000 mgr.x (mgr.24697) 7 : cephadm [INF] [01/Oct/2023:10:19:31] ENGINE Serving on http://172.21.15.173:8765 2023-10-01T10:19:33.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:33 smithi173 bash[27612]: cephadm 2023-10-01T10:19:31.537629+0000 mgr.x (mgr.24697) 8 : cephadm [INF] [01/Oct/2023:10:19:31] ENGINE Bus STARTED 2023-10-01T10:19:33.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:33 smithi173 bash[27612]: cluster 2023-10-01T10:19:32.520479+0000 mon.a (mon.0) 797 : cluster [DBG] Standby manager daemon y started 2023-10-01T10:19:33.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:33 smithi173 bash[27612]: audit 2023-10-01T10:19:32.520950+0000 mon.a (mon.0) 798 : audit [DBG] from='mgr.? 172.21.15.78:0/3303142635' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-01T10:19:33.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:33 smithi173 bash[27612]: audit 2023-10-01T10:19:32.527148+0000 mon.a (mon.0) 799 : audit [DBG] from='mgr.? 172.21.15.78:0/3303142635' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-01T10:19:33.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:33 smithi173 bash[27612]: audit 2023-10-01T10:19:32.531672+0000 mon.a (mon.0) 800 : audit [DBG] from='mgr.? 172.21.15.78:0/3303142635' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-01T10:19:33.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:33 smithi173 bash[27612]: audit 2023-10-01T10:19:32.542612+0000 mon.a (mon.0) 801 : audit [DBG] from='mgr.? 172.21.15.78:0/3303142635' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-01T10:19:33.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:33 smithi173 bash[27612]: audit 2023-10-01T10:19:32.799659+0000 mon.b (mon.1) 36 : audit [DBG] from='client.? 172.21.15.78:0/2235021475' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:34.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:34 smithi078 bash[33821]: cluster 2023-10-01T10:19:32.990400+0000 mgr.x (mgr.24697) 9 : cluster [DBG] pgmap v7: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail 2023-10-01T10:19:34.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:34 smithi078 bash[33821]: cluster 2023-10-01T10:19:33.308032+0000 mon.a (mon.0) 802 : cluster [DBG] mgrmap e23: x(active, since 4s), standbys: y 2023-10-01T10:19:34.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:34 smithi078 bash[33821]: audit 2023-10-01T10:19:33.308532+0000 mon.c (mon.2) 80 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-01T10:19:34.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:34 smithi078 bash[33821]: audit 2023-10-01T10:19:33.452975+0000 mon.a (mon.0) 803 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:34.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:34 smithi078 bash[27336]: cluster 2023-10-01T10:19:32.990400+0000 mgr.x (mgr.24697) 9 : cluster [DBG] pgmap v7: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail 2023-10-01T10:19:34.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:34 smithi078 bash[27336]: cluster 2023-10-01T10:19:33.308032+0000 mon.a (mon.0) 802 : cluster [DBG] mgrmap e23: x(active, since 4s), standbys: y 2023-10-01T10:19:34.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:34 smithi078 bash[27336]: audit 2023-10-01T10:19:33.308532+0000 mon.c (mon.2) 80 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-01T10:19:34.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:34 smithi078 bash[27336]: audit 2023-10-01T10:19:33.452975+0000 mon.a (mon.0) 803 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:34.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:34 smithi173 bash[27612]: cluster 2023-10-01T10:19:32.990400+0000 mgr.x (mgr.24697) 9 : cluster [DBG] pgmap v7: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail 2023-10-01T10:19:34.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:34 smithi173 bash[27612]: cluster 2023-10-01T10:19:33.308032+0000 mon.a (mon.0) 802 : cluster [DBG] mgrmap e23: x(active, since 4s), standbys: y 2023-10-01T10:19:34.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:34 smithi173 bash[27612]: audit 2023-10-01T10:19:33.308532+0000 mon.c (mon.2) 80 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-01T10:19:34.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:34 smithi173 bash[27612]: audit 2023-10-01T10:19:33.452975+0000 mon.a (mon.0) 803 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:35.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:19:35 smithi078 bash[27601]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:19:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T10:19:35.619 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:19:35 smithi173 bash[50431]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:19:35] "GET /metrics HTTP/1.1" 200 33165 "" "Prometheus/2.33.4" 2023-10-01T10:19:36.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:36 smithi078 bash[33821]: cluster 2023-10-01T10:19:34.990836+0000 mgr.x (mgr.24697) 10 : cluster [DBG] pgmap v8: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail 2023-10-01T10:19:36.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:36 smithi078 bash[33821]: audit 2023-10-01T10:19:35.519348+0000 mon.c (mon.2) 81 : audit [DBG] from='client.? 172.21.15.78:0/3354396980' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:36.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:36 smithi078 bash[27336]: cluster 2023-10-01T10:19:34.990836+0000 mgr.x (mgr.24697) 10 : cluster [DBG] pgmap v8: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail 2023-10-01T10:19:36.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:36 smithi078 bash[27336]: audit 2023-10-01T10:19:35.519348+0000 mon.c (mon.2) 81 : audit [DBG] from='client.? 172.21.15.78:0/3354396980' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:36.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:36 smithi173 bash[27612]: cluster 2023-10-01T10:19:34.990836+0000 mgr.x (mgr.24697) 10 : cluster [DBG] pgmap v8: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail 2023-10-01T10:19:36.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:36 smithi173 bash[27612]: audit 2023-10-01T10:19:35.519348+0000 mon.c (mon.2) 81 : audit [DBG] from='client.? 172.21.15.78:0/3354396980' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:38.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:38 smithi078 bash[33821]: cluster 2023-10-01T10:19:36.991574+0000 mgr.x (mgr.24697) 11 : cluster [DBG] pgmap v9: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 12 KiB/s rd, 237 KiB/s wr, 569 op/s 2023-10-01T10:19:38.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:38 smithi078 bash[33821]: audit 2023-10-01T10:19:38.129625+0000 mon.a (mon.0) 804 : audit [DBG] from='client.? 172.21.15.78:0/2074139019' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:38.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:38 smithi078 bash[27336]: cluster 2023-10-01T10:19:36.991574+0000 mgr.x (mgr.24697) 11 : cluster [DBG] pgmap v9: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 12 KiB/s rd, 237 KiB/s wr, 569 op/s 2023-10-01T10:19:38.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:38 smithi078 bash[27336]: audit 2023-10-01T10:19:38.129625+0000 mon.a (mon.0) 804 : audit [DBG] from='client.? 172.21.15.78:0/2074139019' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:38.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:38 smithi173 bash[27612]: cluster 2023-10-01T10:19:36.991574+0000 mgr.x (mgr.24697) 11 : cluster [DBG] pgmap v9: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 12 KiB/s rd, 237 KiB/s wr, 569 op/s 2023-10-01T10:19:38.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:38 smithi173 bash[27612]: audit 2023-10-01T10:19:38.129625+0000 mon.a (mon.0) 804 : audit [DBG] from='client.? 172.21.15.78:0/2074139019' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:40.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:39 smithi078 bash[27336]: audit 2023-10-01T10:19:38.935559+0000 mon.a (mon.0) 805 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:40.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:39 smithi078 bash[27336]: cluster 2023-10-01T10:19:38.991975+0000 mgr.x (mgr.24697) 12 : cluster [DBG] pgmap v10: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 9.4 KiB/s rd, 187 KiB/s wr, 449 op/s 2023-10-01T10:19:40.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:39 smithi078 bash[27336]: audit 2023-10-01T10:19:39.881927+0000 mon.a (mon.0) 806 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:40.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:39 smithi078 bash[27336]: audit 2023-10-01T10:19:39.897337+0000 mon.a (mon.0) 807 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:40.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:39 smithi078 bash[33821]: audit 2023-10-01T10:19:38.935559+0000 mon.a (mon.0) 805 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:40.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:39 smithi078 bash[33821]: cluster 2023-10-01T10:19:38.991975+0000 mgr.x (mgr.24697) 12 : cluster [DBG] pgmap v10: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 9.4 KiB/s rd, 187 KiB/s wr, 449 op/s 2023-10-01T10:19:40.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:39 smithi078 bash[33821]: audit 2023-10-01T10:19:39.881927+0000 mon.a (mon.0) 806 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:40.085 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:39 smithi078 bash[33821]: audit 2023-10-01T10:19:39.897337+0000 mon.a (mon.0) 807 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:40.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:39 smithi173 bash[27612]: audit 2023-10-01T10:19:38.935559+0000 mon.a (mon.0) 805 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:40.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:39 smithi173 bash[27612]: cluster 2023-10-01T10:19:38.991975+0000 mgr.x (mgr.24697) 12 : cluster [DBG] pgmap v10: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 9.4 KiB/s rd, 187 KiB/s wr, 449 op/s 2023-10-01T10:19:40.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:39 smithi173 bash[27612]: audit 2023-10-01T10:19:39.881927+0000 mon.a (mon.0) 806 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:40.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:39 smithi173 bash[27612]: audit 2023-10-01T10:19:39.897337+0000 mon.a (mon.0) 807 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:42.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:41 smithi078 bash[27336]: audit 2023-10-01T10:19:40.682303+0000 mon.a (mon.0) 808 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:42.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:41 smithi078 bash[27336]: audit 2023-10-01T10:19:40.698418+0000 mon.a (mon.0) 809 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:42.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:41 smithi078 bash[27336]: audit 2023-10-01T10:19:40.849507+0000 mon.b (mon.1) 37 : audit [DBG] from='client.? 172.21.15.78:0/545608699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:42.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:41 smithi078 bash[27336]: audit 2023-10-01T10:19:40.969938+0000 mon.a (mon.0) 810 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:42.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:41 smithi078 bash[27336]: audit 2023-10-01T10:19:40.981359+0000 mon.a (mon.0) 811 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:42.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:41 smithi078 bash[27336]: cluster 2023-10-01T10:19:40.992720+0000 mgr.x (mgr.24697) 13 : cluster [DBG] pgmap v11: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 9.1 KiB/s rd, 215 KiB/s wr, 760 op/s 2023-10-01T10:19:42.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:41 smithi078 bash[33821]: audit 2023-10-01T10:19:40.682303+0000 mon.a (mon.0) 808 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:42.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:41 smithi078 bash[33821]: audit 2023-10-01T10:19:40.698418+0000 mon.a (mon.0) 809 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:42.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:41 smithi078 bash[33821]: audit 2023-10-01T10:19:40.849507+0000 mon.b (mon.1) 37 : audit [DBG] from='client.? 172.21.15.78:0/545608699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:42.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:41 smithi078 bash[33821]: audit 2023-10-01T10:19:40.969938+0000 mon.a (mon.0) 810 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:42.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:41 smithi078 bash[33821]: audit 2023-10-01T10:19:40.981359+0000 mon.a (mon.0) 811 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:42.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:41 smithi078 bash[33821]: cluster 2023-10-01T10:19:40.992720+0000 mgr.x (mgr.24697) 13 : cluster [DBG] pgmap v11: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 9.1 KiB/s rd, 215 KiB/s wr, 760 op/s 2023-10-01T10:19:42.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:41 smithi173 bash[27612]: audit 2023-10-01T10:19:40.682303+0000 mon.a (mon.0) 808 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:42.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:41 smithi173 bash[27612]: audit 2023-10-01T10:19:40.698418+0000 mon.a (mon.0) 809 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:42.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:41 smithi173 bash[27612]: audit 2023-10-01T10:19:40.849507+0000 mon.b (mon.1) 37 : audit [DBG] from='client.? 172.21.15.78:0/545608699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:42.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:41 smithi173 bash[27612]: audit 2023-10-01T10:19:40.969938+0000 mon.a (mon.0) 810 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:42.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:41 smithi173 bash[27612]: audit 2023-10-01T10:19:40.981359+0000 mon.a (mon.0) 811 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:42.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:41 smithi173 bash[27612]: cluster 2023-10-01T10:19:40.992720+0000 mgr.x (mgr.24697) 13 : cluster [DBG] pgmap v11: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 9.1 KiB/s rd, 215 KiB/s wr, 760 op/s 2023-10-01T10:19:43.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:42 smithi173 bash[27612]: audit 2023-10-01T10:19:41.836214+0000 mon.a (mon.0) 812 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:43.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:42 smithi173 bash[27612]: audit 2023-10-01T10:19:41.844253+0000 mon.a (mon.0) 813 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:43.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:42 smithi078 bash[27336]: audit 2023-10-01T10:19:41.836214+0000 mon.a (mon.0) 812 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:43.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:42 smithi078 bash[27336]: audit 2023-10-01T10:19:41.844253+0000 mon.a (mon.0) 813 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:43.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:42 smithi078 bash[33821]: audit 2023-10-01T10:19:41.836214+0000 mon.a (mon.0) 812 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:43.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:42 smithi078 bash[33821]: audit 2023-10-01T10:19:41.844253+0000 mon.a (mon.0) 813 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:45.445 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:45 smithi078 bash[27336]: cluster 2023-10-01T10:19:42.993180+0000 mgr.x (mgr.24697) 14 : cluster [DBG] pgmap v12: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 7.8 KiB/s rd, 184 KiB/s wr, 649 op/s 2023-10-01T10:19:45.446 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:45 smithi078 bash[27336]: audit 2023-10-01T10:19:43.146657+0000 mon.a (mon.0) 814 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-01T10:19:45.446 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:45 smithi078 bash[27336]: audit 2023-10-01T10:19:43.147165+0000 mon.c (mon.2) 82 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T10:19:45.446 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:45 smithi078 bash[27336]: audit 2023-10-01T10:19:43.148017+0000 mon.c (mon.2) 83 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T10:19:45.446 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:45 smithi078 bash[27336]: audit 2023-10-01T10:19:43.148832+0000 mon.c (mon.2) 84 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T10:19:45.446 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:45 smithi078 bash[27336]: audit 2023-10-01T10:19:43.514180+0000 mon.c (mon.2) 85 : audit [DBG] from='client.? 172.21.15.78:0/2266329573' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:45.446 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:45 smithi078 bash[27336]: audit 2023-10-01T10:19:44.286013+0000 mon.a (mon.0) 815 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-01T10:19:45.446 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:45 smithi078 bash[27336]: audit 2023-10-01T10:19:44.289132+0000 mon.c (mon.2) 86 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T10:19:45.447 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:19:45 smithi078 bash[27601]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:19:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T10:19:45.619 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:19:45 smithi173 bash[50431]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:19:45] "GET /metrics HTTP/1.1" 200 33165 "" "Prometheus/2.33.4" 2023-10-01T10:19:45.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:45 smithi173 bash[27612]: cluster 2023-10-01T10:19:42.993180+0000 mgr.x (mgr.24697) 14 : cluster [DBG] pgmap v12: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 7.8 KiB/s rd, 184 KiB/s wr, 649 op/s 2023-10-01T10:19:45.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:45 smithi173 bash[27612]: audit 2023-10-01T10:19:43.146657+0000 mon.a (mon.0) 814 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-01T10:19:45.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:45 smithi173 bash[27612]: audit 2023-10-01T10:19:43.147165+0000 mon.c (mon.2) 82 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T10:19:45.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:45 smithi173 bash[27612]: audit 2023-10-01T10:19:43.148017+0000 mon.c (mon.2) 83 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T10:19:45.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:45 smithi173 bash[27612]: audit 2023-10-01T10:19:43.148832+0000 mon.c (mon.2) 84 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T10:19:45.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:45 smithi173 bash[27612]: audit 2023-10-01T10:19:43.514180+0000 mon.c (mon.2) 85 : audit [DBG] from='client.? 172.21.15.78:0/2266329573' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:45.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:45 smithi173 bash[27612]: audit 2023-10-01T10:19:44.286013+0000 mon.a (mon.0) 815 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-01T10:19:45.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:45 smithi173 bash[27612]: audit 2023-10-01T10:19:44.289132+0000 mon.c (mon.2) 86 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T10:19:45.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:45 smithi078 bash[33821]: cluster 2023-10-01T10:19:42.993180+0000 mgr.x (mgr.24697) 14 : cluster [DBG] pgmap v12: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 7.8 KiB/s rd, 184 KiB/s wr, 649 op/s 2023-10-01T10:19:45.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:45 smithi078 bash[33821]: audit 2023-10-01T10:19:43.146657+0000 mon.a (mon.0) 814 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-01T10:19:45.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:45 smithi078 bash[33821]: audit 2023-10-01T10:19:43.147165+0000 mon.c (mon.2) 82 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T10:19:45.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:45 smithi078 bash[33821]: audit 2023-10-01T10:19:43.148017+0000 mon.c (mon.2) 83 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T10:19:45.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:45 smithi078 bash[33821]: audit 2023-10-01T10:19:43.148832+0000 mon.c (mon.2) 84 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T10:19:45.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:45 smithi078 bash[33821]: audit 2023-10-01T10:19:43.514180+0000 mon.c (mon.2) 85 : audit [DBG] from='client.? 172.21.15.78:0/2266329573' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:45.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:45 smithi078 bash[33821]: audit 2023-10-01T10:19:44.286013+0000 mon.a (mon.0) 815 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-01T10:19:45.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:45 smithi078 bash[33821]: audit 2023-10-01T10:19:44.289132+0000 mon.c (mon.2) 86 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T10:19:46.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:46 smithi173 bash[27612]: audit 2023-10-01T10:19:44.289812+0000 mon.c (mon.2) 87 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T10:19:46.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:46 smithi173 bash[27612]: audit 2023-10-01T10:19:44.289984+0000 mon.c (mon.2) 88 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T10:19:46.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:46 smithi173 bash[27612]: audit 2023-10-01T10:19:44.291197+0000 mon.b (mon.1) 38 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-01T10:19:46.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:46 smithi173 bash[27612]: audit 2023-10-01T10:19:44.315950+0000 mon.c (mon.2) 89 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:19:46.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:46 smithi173 bash[27612]: cluster 2023-10-01T10:19:44.993651+0000 mgr.x (mgr.24697) 15 : cluster [DBG] pgmap v13: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 7.6 KiB/s rd, 179 KiB/s wr, 634 op/s 2023-10-01T10:19:46.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:46 smithi173 bash[27612]: audit 2023-10-01T10:19:45.432309+0000 mon.b (mon.1) 39 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-01T10:19:46.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:46 smithi173 bash[27612]: audit 2023-10-01T10:19:46.214105+0000 mon.a (mon.0) 816 : audit [DBG] from='client.? 172.21.15.78:0/533593701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:47.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:46 smithi078 bash[27336]: audit 2023-10-01T10:19:44.289812+0000 mon.c (mon.2) 87 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T10:19:47.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:46 smithi078 bash[27336]: audit 2023-10-01T10:19:44.289984+0000 mon.c (mon.2) 88 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T10:19:47.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:46 smithi078 bash[27336]: audit 2023-10-01T10:19:44.291197+0000 mon.b (mon.1) 38 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-01T10:19:47.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:46 smithi078 bash[27336]: audit 2023-10-01T10:19:44.315950+0000 mon.c (mon.2) 89 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:19:47.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:46 smithi078 bash[27336]: cluster 2023-10-01T10:19:44.993651+0000 mgr.x (mgr.24697) 15 : cluster [DBG] pgmap v13: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 7.6 KiB/s rd, 179 KiB/s wr, 634 op/s 2023-10-01T10:19:47.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:46 smithi078 bash[27336]: audit 2023-10-01T10:19:45.432309+0000 mon.b (mon.1) 39 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-01T10:19:47.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:46 smithi078 bash[27336]: audit 2023-10-01T10:19:46.214105+0000 mon.a (mon.0) 816 : audit [DBG] from='client.? 172.21.15.78:0/533593701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:47.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:46 smithi078 bash[33821]: audit 2023-10-01T10:19:44.289812+0000 mon.c (mon.2) 87 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T10:19:47.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:46 smithi078 bash[33821]: audit 2023-10-01T10:19:44.289984+0000 mon.c (mon.2) 88 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T10:19:47.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:46 smithi078 bash[33821]: audit 2023-10-01T10:19:44.291197+0000 mon.b (mon.1) 38 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-01T10:19:47.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:46 smithi078 bash[33821]: audit 2023-10-01T10:19:44.315950+0000 mon.c (mon.2) 89 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:19:47.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:46 smithi078 bash[33821]: cluster 2023-10-01T10:19:44.993651+0000 mgr.x (mgr.24697) 15 : cluster [DBG] pgmap v13: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 7.6 KiB/s rd, 179 KiB/s wr, 634 op/s 2023-10-01T10:19:47.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:46 smithi078 bash[33821]: audit 2023-10-01T10:19:45.432309+0000 mon.b (mon.1) 39 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-01T10:19:47.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:46 smithi078 bash[33821]: audit 2023-10-01T10:19:46.214105+0000 mon.a (mon.0) 816 : audit [DBG] from='client.? 172.21.15.78:0/533593701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:47.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:47 smithi173 bash[27612]: audit 2023-10-01T10:19:45.454175+0000 mon.c (mon.2) 90 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-01T10:19:47.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:47 smithi173 bash[27612]: audit 2023-10-01T10:19:46.581613+0000 mon.c (mon.2) 91 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-01T10:19:47.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:47 smithi173 bash[27612]: audit 2023-10-01T10:19:46.584420+0000 mon.c (mon.2) 92 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T10:19:47.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:47 smithi173 bash[27612]: audit 2023-10-01T10:19:46.593951+0000 mon.c (mon.2) 93 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T10:19:47.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:47 smithi173 bash[27612]: audit 2023-10-01T10:19:46.594169+0000 mon.c (mon.2) 94 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T10:19:47.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:47 smithi173 bash[27612]: cluster 2023-10-01T10:19:46.994427+0000 mgr.x (mgr.24697) 16 : cluster [DBG] pgmap v14: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 7.6 KiB/s rd, 179 KiB/s wr, 929 op/s 2023-10-01T10:19:48.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:47 smithi078 bash[33821]: audit 2023-10-01T10:19:45.454175+0000 mon.c (mon.2) 90 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-01T10:19:48.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:47 smithi078 bash[33821]: audit 2023-10-01T10:19:46.581613+0000 mon.c (mon.2) 91 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-01T10:19:48.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:47 smithi078 bash[33821]: audit 2023-10-01T10:19:46.584420+0000 mon.c (mon.2) 92 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T10:19:48.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:47 smithi078 bash[33821]: audit 2023-10-01T10:19:46.593951+0000 mon.c (mon.2) 93 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T10:19:48.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:47 smithi078 bash[33821]: audit 2023-10-01T10:19:46.594169+0000 mon.c (mon.2) 94 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T10:19:48.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:47 smithi078 bash[33821]: cluster 2023-10-01T10:19:46.994427+0000 mgr.x (mgr.24697) 16 : cluster [DBG] pgmap v14: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 7.6 KiB/s rd, 179 KiB/s wr, 929 op/s 2023-10-01T10:19:48.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:47 smithi078 bash[27336]: audit 2023-10-01T10:19:45.454175+0000 mon.c (mon.2) 90 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-10-01T10:19:48.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:47 smithi078 bash[27336]: audit 2023-10-01T10:19:46.581613+0000 mon.c (mon.2) 91 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-10-01T10:19:48.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:47 smithi078 bash[27336]: audit 2023-10-01T10:19:46.584420+0000 mon.c (mon.2) 92 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T10:19:48.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:47 smithi078 bash[27336]: audit 2023-10-01T10:19:46.593951+0000 mon.c (mon.2) 93 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T10:19:48.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:47 smithi078 bash[27336]: audit 2023-10-01T10:19:46.594169+0000 mon.c (mon.2) 94 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T10:19:48.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:47 smithi078 bash[27336]: cluster 2023-10-01T10:19:46.994427+0000 mgr.x (mgr.24697) 16 : cluster [DBG] pgmap v14: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 7.6 KiB/s rd, 179 KiB/s wr, 929 op/s 2023-10-01T10:19:49.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:48 smithi078 bash[27336]: audit 2023-10-01T10:19:48.859640+0000 mon.c (mon.2) 95 : audit [DBG] from='client.? 172.21.15.78:0/606259523' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:49.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:48 smithi078 bash[33821]: audit 2023-10-01T10:19:48.859640+0000 mon.c (mon.2) 95 : audit [DBG] from='client.? 172.21.15.78:0/606259523' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:49.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:48 smithi173 bash[27612]: audit 2023-10-01T10:19:48.859640+0000 mon.c (mon.2) 95 : audit [DBG] from='client.? 172.21.15.78:0/606259523' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:50.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:49 smithi078 bash[27336]: cluster 2023-10-01T10:19:48.994843+0000 mgr.x (mgr.24697) 17 : cluster [DBG] pgmap v15: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 767 B/s rd, 43 KiB/s wr, 603 op/s 2023-10-01T10:19:50.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:49 smithi078 bash[33821]: cluster 2023-10-01T10:19:48.994843+0000 mgr.x (mgr.24697) 17 : cluster [DBG] pgmap v15: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 767 B/s rd, 43 KiB/s wr, 603 op/s 2023-10-01T10:19:50.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:49 smithi173 bash[27612]: cluster 2023-10-01T10:19:48.994843+0000 mgr.x (mgr.24697) 17 : cluster [DBG] pgmap v15: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 767 B/s rd, 43 KiB/s wr, 603 op/s 2023-10-01T10:19:52.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:52 smithi173 bash[27612]: cluster 2023-10-01T10:19:50.995565+0000 mgr.x (mgr.24697) 18 : cluster [DBG] pgmap v16: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 922 op/s 2023-10-01T10:19:52.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:52 smithi173 bash[27612]: audit 2023-10-01T10:19:51.507054+0000 mon.a (mon.0) 817 : audit [DBG] from='client.? 172.21.15.78:0/1575289016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:52.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:52 smithi173 bash[27612]: audit 2023-10-01T10:19:51.937937+0000 mon.a (mon.0) 818 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:52.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:52 smithi173 bash[27612]: audit 2023-10-01T10:19:51.955974+0000 mon.a (mon.0) 819 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:52.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:52 smithi173 bash[27612]: audit 2023-10-01T10:19:51.965183+0000 mon.c (mon.2) 96 : audit [INF] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:19:52.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:52 smithi173 bash[27612]: audit 2023-10-01T10:19:51.966012+0000 mon.a (mon.0) 820 : audit [INF] from='mgr.24697 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:19:52.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:52 smithi173 bash[27612]: audit 2023-10-01T10:19:51.967599+0000 mon.c (mon.2) 97 : audit [INF] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:19:52.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:52 smithi173 bash[27612]: audit 2023-10-01T10:19:51.968321+0000 mon.a (mon.0) 821 : audit [INF] from='mgr.24697 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:19:52.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:52 smithi173 bash[27612]: audit 2023-10-01T10:19:51.969650+0000 mon.c (mon.2) 98 : audit [INF] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:19:52.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:52 smithi173 bash[27612]: audit 2023-10-01T10:19:51.970670+0000 mon.a (mon.0) 822 : audit [INF] from='mgr.24697 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:19:52.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:52 smithi173 bash[27612]: audit 2023-10-01T10:19:51.972037+0000 mon.c (mon.2) 99 : audit [INF] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:19:52.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:52 smithi173 bash[27612]: audit 2023-10-01T10:19:51.972998+0000 mon.a (mon.0) 823 : audit [INF] from='mgr.24697 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:19:52.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:52 smithi173 bash[27612]: audit 2023-10-01T10:19:51.984628+0000 mon.a (mon.0) 824 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:52.371 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:19:52 smithi173 bash[48271]: ts=2023-10-01T10:19:52.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi173.front.sepia.ceph.com:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.78:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-01T10:19:52.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:52 smithi078 bash[27336]: cluster 2023-10-01T10:19:50.995565+0000 mgr.x (mgr.24697) 18 : cluster [DBG] pgmap v16: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 922 op/s 2023-10-01T10:19:52.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:52 smithi078 bash[27336]: audit 2023-10-01T10:19:51.507054+0000 mon.a (mon.0) 817 : audit [DBG] from='client.? 172.21.15.78:0/1575289016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:52.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:52 smithi078 bash[27336]: audit 2023-10-01T10:19:51.937937+0000 mon.a (mon.0) 818 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:52.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:52 smithi078 bash[27336]: audit 2023-10-01T10:19:51.955974+0000 mon.a (mon.0) 819 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:52.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:52 smithi078 bash[27336]: audit 2023-10-01T10:19:51.965183+0000 mon.c (mon.2) 96 : audit [INF] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:19:52.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:52 smithi078 bash[27336]: audit 2023-10-01T10:19:51.966012+0000 mon.a (mon.0) 820 : audit [INF] from='mgr.24697 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:19:52.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:52 smithi078 bash[27336]: audit 2023-10-01T10:19:51.967599+0000 mon.c (mon.2) 97 : audit [INF] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:19:52.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:52 smithi078 bash[27336]: audit 2023-10-01T10:19:51.968321+0000 mon.a (mon.0) 821 : audit [INF] from='mgr.24697 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:19:52.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:52 smithi078 bash[27336]: audit 2023-10-01T10:19:51.969650+0000 mon.c (mon.2) 98 : audit [INF] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:19:52.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:52 smithi078 bash[27336]: audit 2023-10-01T10:19:51.970670+0000 mon.a (mon.0) 822 : audit [INF] from='mgr.24697 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:19:52.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:52 smithi078 bash[27336]: audit 2023-10-01T10:19:51.972037+0000 mon.c (mon.2) 99 : audit [INF] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:19:52.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:52 smithi078 bash[27336]: audit 2023-10-01T10:19:51.972998+0000 mon.a (mon.0) 823 : audit [INF] from='mgr.24697 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:19:52.585 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:52 smithi078 bash[27336]: audit 2023-10-01T10:19:51.984628+0000 mon.a (mon.0) 824 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:52.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:52 smithi078 bash[33821]: cluster 2023-10-01T10:19:50.995565+0000 mgr.x (mgr.24697) 18 : cluster [DBG] pgmap v16: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 922 op/s 2023-10-01T10:19:52.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:52 smithi078 bash[33821]: audit 2023-10-01T10:19:51.507054+0000 mon.a (mon.0) 817 : audit [DBG] from='client.? 172.21.15.78:0/1575289016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:52.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:52 smithi078 bash[33821]: audit 2023-10-01T10:19:51.937937+0000 mon.a (mon.0) 818 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:52.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:52 smithi078 bash[33821]: audit 2023-10-01T10:19:51.955974+0000 mon.a (mon.0) 819 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:52.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:52 smithi078 bash[33821]: audit 2023-10-01T10:19:51.965183+0000 mon.c (mon.2) 96 : audit [INF] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:19:52.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:52 smithi078 bash[33821]: audit 2023-10-01T10:19:51.966012+0000 mon.a (mon.0) 820 : audit [INF] from='mgr.24697 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:19:52.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:52 smithi078 bash[33821]: audit 2023-10-01T10:19:51.967599+0000 mon.c (mon.2) 97 : audit [INF] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:19:52.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:52 smithi078 bash[33821]: audit 2023-10-01T10:19:51.968321+0000 mon.a (mon.0) 821 : audit [INF] from='mgr.24697 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:19:52.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:52 smithi078 bash[33821]: audit 2023-10-01T10:19:51.969650+0000 mon.c (mon.2) 98 : audit [INF] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:19:52.586 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:52 smithi078 bash[33821]: audit 2023-10-01T10:19:51.970670+0000 mon.a (mon.0) 822 : audit [INF] from='mgr.24697 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:19:52.586 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:52 smithi078 bash[33821]: audit 2023-10-01T10:19:51.972037+0000 mon.c (mon.2) 99 : audit [INF] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:19:52.586 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:52 smithi078 bash[33821]: audit 2023-10-01T10:19:51.972998+0000 mon.a (mon.0) 823 : audit [INF] from='mgr.24697 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:19:52.586 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:52 smithi078 bash[33821]: audit 2023-10-01T10:19:51.984628+0000 mon.a (mon.0) 824 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:53.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:53 smithi078 bash[33821]: cephadm 2023-10-01T10:19:51.976418+0000 mgr.x (mgr.24697) 19 : cephadm [INF] Adjusting osd_memory_target on smithi173 to 2524M 2023-10-01T10:19:53.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:53 smithi078 bash[27336]: cephadm 2023-10-01T10:19:51.976418+0000 mgr.x (mgr.24697) 19 : cephadm [INF] Adjusting osd_memory_target on smithi173 to 2524M 2023-10-01T10:19:53.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:53 smithi173 bash[27612]: cephadm 2023-10-01T10:19:51.976418+0000 mgr.x (mgr.24697) 19 : cephadm [INF] Adjusting osd_memory_target on smithi173 to 2524M 2023-10-01T10:19:54.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:54 smithi173 bash[27612]: cluster 2023-10-01T10:19:52.996020+0000 mgr.x (mgr.24697) 20 : cluster [DBG] pgmap v17: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 614 op/s 2023-10-01T10:19:54.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:54 smithi173 bash[27612]: audit 2023-10-01T10:19:53.818627+0000 mon.a (mon.0) 825 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:54.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:54 smithi173 bash[27612]: audit 2023-10-01T10:19:53.829095+0000 mon.a (mon.0) 826 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:54.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:54 smithi173 bash[27612]: audit 2023-10-01T10:19:53.833446+0000 mon.c (mon.2) 100 : audit [INF] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:19:54.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:54 smithi173 bash[27612]: audit 2023-10-01T10:19:53.834190+0000 mon.a (mon.0) 827 : audit [INF] from='mgr.24697 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:19:54.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:54 smithi173 bash[27612]: audit 2023-10-01T10:19:53.838860+0000 mon.c (mon.2) 101 : audit [INF] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:19:54.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:54 smithi173 bash[27612]: audit 2023-10-01T10:19:53.840089+0000 mon.a (mon.0) 828 : audit [INF] from='mgr.24697 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:19:54.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:54 smithi173 bash[27612]: audit 2023-10-01T10:19:53.842172+0000 mon.c (mon.2) 102 : audit [INF] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:19:54.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:54 smithi173 bash[27612]: audit 2023-10-01T10:19:53.842821+0000 mon.a (mon.0) 829 : audit [INF] from='mgr.24697 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:19:54.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:54 smithi173 bash[27612]: audit 2023-10-01T10:19:53.844519+0000 mon.c (mon.2) 103 : audit [INF] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:19:54.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:54 smithi173 bash[27612]: audit 2023-10-01T10:19:53.845091+0000 mon.a (mon.0) 830 : audit [INF] from='mgr.24697 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:19:54.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:54 smithi173 bash[27612]: audit 2023-10-01T10:19:53.858055+0000 mon.a (mon.0) 831 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:54.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:54 smithi173 bash[27612]: audit 2023-10-01T10:19:53.860183+0000 mon.c (mon.2) 104 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:19:54.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:54 smithi173 bash[27612]: audit 2023-10-01T10:19:53.862753+0000 mon.c (mon.2) 105 : audit [INF] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:19:54.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:54 smithi078 bash[27336]: cluster 2023-10-01T10:19:52.996020+0000 mgr.x (mgr.24697) 20 : cluster [DBG] pgmap v17: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 614 op/s 2023-10-01T10:19:54.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:54 smithi078 bash[27336]: audit 2023-10-01T10:19:53.818627+0000 mon.a (mon.0) 825 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:54.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:54 smithi078 bash[27336]: audit 2023-10-01T10:19:53.829095+0000 mon.a (mon.0) 826 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:54.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:54 smithi078 bash[27336]: audit 2023-10-01T10:19:53.833446+0000 mon.c (mon.2) 100 : audit [INF] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:19:54.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:54 smithi078 bash[27336]: audit 2023-10-01T10:19:53.834190+0000 mon.a (mon.0) 827 : audit [INF] from='mgr.24697 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:19:54.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:54 smithi078 bash[27336]: audit 2023-10-01T10:19:53.838860+0000 mon.c (mon.2) 101 : audit [INF] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:19:54.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:54 smithi078 bash[27336]: audit 2023-10-01T10:19:53.840089+0000 mon.a (mon.0) 828 : audit [INF] from='mgr.24697 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:19:54.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:54 smithi078 bash[27336]: audit 2023-10-01T10:19:53.842172+0000 mon.c (mon.2) 102 : audit [INF] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:19:54.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:54 smithi078 bash[27336]: audit 2023-10-01T10:19:53.842821+0000 mon.a (mon.0) 829 : audit [INF] from='mgr.24697 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:19:54.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:54 smithi078 bash[27336]: audit 2023-10-01T10:19:53.844519+0000 mon.c (mon.2) 103 : audit [INF] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:19:54.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:54 smithi078 bash[27336]: audit 2023-10-01T10:19:53.845091+0000 mon.a (mon.0) 830 : audit [INF] from='mgr.24697 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:19:54.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:54 smithi078 bash[27336]: audit 2023-10-01T10:19:53.858055+0000 mon.a (mon.0) 831 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:54.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:54 smithi078 bash[27336]: audit 2023-10-01T10:19:53.860183+0000 mon.c (mon.2) 104 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:19:54.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:54 smithi078 bash[27336]: audit 2023-10-01T10:19:53.862753+0000 mon.c (mon.2) 105 : audit [INF] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:19:54.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:54 smithi078 bash[33821]: cluster 2023-10-01T10:19:52.996020+0000 mgr.x (mgr.24697) 20 : cluster [DBG] pgmap v17: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 614 op/s 2023-10-01T10:19:54.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:54 smithi078 bash[33821]: audit 2023-10-01T10:19:53.818627+0000 mon.a (mon.0) 825 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:54.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:54 smithi078 bash[33821]: audit 2023-10-01T10:19:53.829095+0000 mon.a (mon.0) 826 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:54.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:54 smithi078 bash[33821]: audit 2023-10-01T10:19:53.833446+0000 mon.c (mon.2) 100 : audit [INF] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:19:54.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:54 smithi078 bash[33821]: audit 2023-10-01T10:19:53.834190+0000 mon.a (mon.0) 827 : audit [INF] from='mgr.24697 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:19:54.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:54 smithi078 bash[33821]: audit 2023-10-01T10:19:53.838860+0000 mon.c (mon.2) 101 : audit [INF] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:19:54.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:54 smithi078 bash[33821]: audit 2023-10-01T10:19:53.840089+0000 mon.a (mon.0) 828 : audit [INF] from='mgr.24697 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:19:54.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:54 smithi078 bash[33821]: audit 2023-10-01T10:19:53.842172+0000 mon.c (mon.2) 102 : audit [INF] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:19:54.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:54 smithi078 bash[33821]: audit 2023-10-01T10:19:53.842821+0000 mon.a (mon.0) 829 : audit [INF] from='mgr.24697 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:19:54.586 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:54 smithi078 bash[33821]: audit 2023-10-01T10:19:53.844519+0000 mon.c (mon.2) 103 : audit [INF] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:19:54.586 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:54 smithi078 bash[33821]: audit 2023-10-01T10:19:53.845091+0000 mon.a (mon.0) 830 : audit [INF] from='mgr.24697 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-10-01T10:19:54.586 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:54 smithi078 bash[33821]: audit 2023-10-01T10:19:53.858055+0000 mon.a (mon.0) 831 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:54.586 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:54 smithi078 bash[33821]: audit 2023-10-01T10:19:53.860183+0000 mon.c (mon.2) 104 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:19:54.586 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:54 smithi078 bash[33821]: audit 2023-10-01T10:19:53.862753+0000 mon.c (mon.2) 105 : audit [INF] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:19:55.367 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:19:55 smithi173 bash[50431]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:19:55] "GET /metrics HTTP/1.1" 200 36077 "" "Prometheus/2.33.4" 2023-10-01T10:19:55.367 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:55 smithi173 bash[27612]: cephadm 2023-10-01T10:19:53.852299+0000 mgr.x (mgr.24697) 21 : cephadm [INF] Adjusting osd_memory_target on smithi078 to 2524M 2023-10-01T10:19:55.367 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:55 smithi173 bash[27612]: cephadm 2023-10-01T10:19:53.870304+0000 mgr.x (mgr.24697) 22 : cephadm [INF] Updating smithi078:/etc/ceph/ceph.conf 2023-10-01T10:19:55.368 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:55 smithi173 bash[27612]: cephadm 2023-10-01T10:19:53.870577+0000 mgr.x (mgr.24697) 23 : cephadm [INF] Updating smithi173:/etc/ceph/ceph.conf 2023-10-01T10:19:55.368 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:55 smithi173 bash[27612]: cephadm 2023-10-01T10:19:54.086149+0000 mgr.x (mgr.24697) 24 : cephadm [INF] Updating smithi173:/var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T10:19:55.368 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:55 smithi173 bash[27612]: cephadm 2023-10-01T10:19:54.091914+0000 mgr.x (mgr.24697) 25 : cephadm [INF] Updating smithi078:/var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T10:19:55.368 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:55 smithi173 bash[27612]: audit 2023-10-01T10:19:54.167957+0000 mon.c (mon.2) 106 : audit [DBG] from='client.? 172.21.15.78:0/2006251150' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:55.368 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:55 smithi173 bash[27612]: cephadm 2023-10-01T10:19:54.305371+0000 mgr.x (mgr.24697) 26 : cephadm [INF] Updating smithi173:/etc/ceph/ceph.client.admin.keyring 2023-10-01T10:19:55.368 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:55 smithi173 bash[27612]: cephadm 2023-10-01T10:19:54.315340+0000 mgr.x (mgr.24697) 27 : cephadm [INF] Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2023-10-01T10:19:55.368 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:55 smithi173 bash[27612]: audit 2023-10-01T10:19:54.740508+0000 mon.a (mon.0) 832 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:55.368 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:55 smithi173 bash[27612]: audit 2023-10-01T10:19:54.748523+0000 mon.a (mon.0) 833 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:55.368 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:55 smithi173 bash[27612]: audit 2023-10-01T10:19:54.760666+0000 mon.a (mon.0) 834 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:55.368 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:55 smithi173 bash[27612]: audit 2023-10-01T10:19:54.776960+0000 mon.a (mon.0) 835 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:55.368 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:55 smithi173 bash[27612]: audit 2023-10-01T10:19:54.784931+0000 mon.a (mon.0) 836 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:55.368 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:55 smithi173 bash[27612]: audit 2023-10-01T10:19:54.871062+0000 mon.a (mon.0) 837 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:55.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:55 smithi173 bash[27612]: audit 2023-10-01T10:19:54.879892+0000 mon.a (mon.0) 838 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:55.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:55 smithi173 bash[27612]: audit 2023-10-01T10:19:54.895756+0000 mon.a (mon.0) 839 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:55.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:55 smithi173 bash[27612]: audit 2023-10-01T10:19:54.911198+0000 mon.a (mon.0) 840 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:55.516 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:19:55 smithi078 bash[27601]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:19:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T10:19:55.516 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:55 smithi078 bash[33821]: cephadm 2023-10-01T10:19:53.852299+0000 mgr.x (mgr.24697) 21 : cephadm [INF] Adjusting osd_memory_target on smithi078 to 2524M 2023-10-01T10:19:55.517 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:55 smithi078 bash[33821]: cephadm 2023-10-01T10:19:53.870304+0000 mgr.x (mgr.24697) 22 : cephadm [INF] Updating smithi078:/etc/ceph/ceph.conf 2023-10-01T10:19:55.517 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:55 smithi078 bash[33821]: cephadm 2023-10-01T10:19:53.870577+0000 mgr.x (mgr.24697) 23 : cephadm [INF] Updating smithi173:/etc/ceph/ceph.conf 2023-10-01T10:19:55.517 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:55 smithi078 bash[33821]: cephadm 2023-10-01T10:19:54.086149+0000 mgr.x (mgr.24697) 24 : cephadm [INF] Updating smithi173:/var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T10:19:55.517 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:55 smithi078 bash[33821]: cephadm 2023-10-01T10:19:54.091914+0000 mgr.x (mgr.24697) 25 : cephadm [INF] Updating smithi078:/var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T10:19:55.517 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:55 smithi078 bash[33821]: audit 2023-10-01T10:19:54.167957+0000 mon.c (mon.2) 106 : audit [DBG] from='client.? 172.21.15.78:0/2006251150' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:55.517 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:55 smithi078 bash[33821]: cephadm 2023-10-01T10:19:54.305371+0000 mgr.x (mgr.24697) 26 : cephadm [INF] Updating smithi173:/etc/ceph/ceph.client.admin.keyring 2023-10-01T10:19:55.517 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:55 smithi078 bash[33821]: cephadm 2023-10-01T10:19:54.315340+0000 mgr.x (mgr.24697) 27 : cephadm [INF] Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2023-10-01T10:19:55.517 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:55 smithi078 bash[33821]: audit 2023-10-01T10:19:54.740508+0000 mon.a (mon.0) 832 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:55.517 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:55 smithi078 bash[33821]: audit 2023-10-01T10:19:54.748523+0000 mon.a (mon.0) 833 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:55.517 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:55 smithi078 bash[33821]: audit 2023-10-01T10:19:54.760666+0000 mon.a (mon.0) 834 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:55.517 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:55 smithi078 bash[33821]: audit 2023-10-01T10:19:54.776960+0000 mon.a (mon.0) 835 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:55.517 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:55 smithi078 bash[33821]: audit 2023-10-01T10:19:54.784931+0000 mon.a (mon.0) 836 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:55.518 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:55 smithi078 bash[33821]: audit 2023-10-01T10:19:54.871062+0000 mon.a (mon.0) 837 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:55.518 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:55 smithi078 bash[33821]: audit 2023-10-01T10:19:54.879892+0000 mon.a (mon.0) 838 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:55.518 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:55 smithi078 bash[33821]: audit 2023-10-01T10:19:54.895756+0000 mon.a (mon.0) 839 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:55.518 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:55 smithi078 bash[33821]: audit 2023-10-01T10:19:54.911198+0000 mon.a (mon.0) 840 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:55.518 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:55 smithi078 bash[27336]: cephadm 2023-10-01T10:19:53.852299+0000 mgr.x (mgr.24697) 21 : cephadm [INF] Adjusting osd_memory_target on smithi078 to 2524M 2023-10-01T10:19:55.518 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:55 smithi078 bash[27336]: cephadm 2023-10-01T10:19:53.870304+0000 mgr.x (mgr.24697) 22 : cephadm [INF] Updating smithi078:/etc/ceph/ceph.conf 2023-10-01T10:19:55.518 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:55 smithi078 bash[27336]: cephadm 2023-10-01T10:19:53.870577+0000 mgr.x (mgr.24697) 23 : cephadm [INF] Updating smithi173:/etc/ceph/ceph.conf 2023-10-01T10:19:55.519 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:55 smithi078 bash[27336]: cephadm 2023-10-01T10:19:54.086149+0000 mgr.x (mgr.24697) 24 : cephadm [INF] Updating smithi173:/var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T10:19:55.519 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:55 smithi078 bash[27336]: cephadm 2023-10-01T10:19:54.091914+0000 mgr.x (mgr.24697) 25 : cephadm [INF] Updating smithi078:/var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T10:19:55.519 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:55 smithi078 bash[27336]: audit 2023-10-01T10:19:54.167957+0000 mon.c (mon.2) 106 : audit [DBG] from='client.? 172.21.15.78:0/2006251150' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:55.519 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:55 smithi078 bash[27336]: cephadm 2023-10-01T10:19:54.305371+0000 mgr.x (mgr.24697) 26 : cephadm [INF] Updating smithi173:/etc/ceph/ceph.client.admin.keyring 2023-10-01T10:19:55.519 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:55 smithi078 bash[27336]: cephadm 2023-10-01T10:19:54.315340+0000 mgr.x (mgr.24697) 27 : cephadm [INF] Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2023-10-01T10:19:55.519 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:55 smithi078 bash[27336]: audit 2023-10-01T10:19:54.740508+0000 mon.a (mon.0) 832 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:55.519 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:55 smithi078 bash[27336]: audit 2023-10-01T10:19:54.748523+0000 mon.a (mon.0) 833 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:55.519 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:55 smithi078 bash[27336]: audit 2023-10-01T10:19:54.760666+0000 mon.a (mon.0) 834 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:55.519 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:55 smithi078 bash[27336]: audit 2023-10-01T10:19:54.776960+0000 mon.a (mon.0) 835 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:55.519 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:55 smithi078 bash[27336]: audit 2023-10-01T10:19:54.784931+0000 mon.a (mon.0) 836 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:55.519 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:55 smithi078 bash[27336]: audit 2023-10-01T10:19:54.871062+0000 mon.a (mon.0) 837 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:55.519 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:55 smithi078 bash[27336]: audit 2023-10-01T10:19:54.879892+0000 mon.a (mon.0) 838 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:55.520 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:55 smithi078 bash[27336]: audit 2023-10-01T10:19:54.895756+0000 mon.a (mon.0) 839 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:55.520 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:55 smithi078 bash[27336]: audit 2023-10-01T10:19:54.911198+0000 mon.a (mon.0) 840 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:55.520 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:19:55 smithi078 systemd[1]: Stopping Ceph node-exporter.a for d8016480-6041-11ee-8db5-212e2dc638e7... 2023-10-01T10:19:55.520 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:19:55 smithi078 bash[69630]: Error response from daemon: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-node-exporter.a 2023-10-01T10:19:56.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:56 smithi173 bash[27612]: cephadm 2023-10-01T10:19:54.520049+0000 mgr.x (mgr.24697) 28 : cephadm [INF] Updating smithi173:/var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-01T10:19:56.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:56 smithi173 bash[27612]: cephadm 2023-10-01T10:19:54.531173+0000 mgr.x (mgr.24697) 29 : cephadm [INF] Updating smithi078:/var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-01T10:19:56.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:56 smithi173 bash[27612]: cephadm 2023-10-01T10:19:54.919865+0000 mgr.x (mgr.24697) 30 : cephadm [INF] Reconfiguring node-exporter.a (dependencies changed)... 2023-10-01T10:19:56.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:56 smithi173 bash[27612]: cephadm 2023-10-01T10:19:54.920450+0000 mgr.x (mgr.24697) 31 : cephadm [INF] Reconfiguring daemon node-exporter.a on smithi078 2023-10-01T10:19:56.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:56 smithi173 bash[27612]: cluster 2023-10-01T10:19:54.996460+0000 mgr.x (mgr.24697) 32 : cluster [DBG] pgmap v18: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 614 op/s 2023-10-01T10:19:56.380 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:56 smithi078 bash[33821]: cephadm 2023-10-01T10:19:54.520049+0000 mgr.x (mgr.24697) 28 : cephadm [INF] Updating smithi173:/var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-01T10:19:56.380 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:56 smithi078 bash[33821]: cephadm 2023-10-01T10:19:54.531173+0000 mgr.x (mgr.24697) 29 : cephadm [INF] Updating smithi078:/var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-01T10:19:56.380 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:56 smithi078 bash[33821]: cephadm 2023-10-01T10:19:54.919865+0000 mgr.x (mgr.24697) 30 : cephadm [INF] Reconfiguring node-exporter.a (dependencies changed)... 2023-10-01T10:19:56.380 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:56 smithi078 bash[33821]: cephadm 2023-10-01T10:19:54.920450+0000 mgr.x (mgr.24697) 31 : cephadm [INF] Reconfiguring daemon node-exporter.a on smithi078 2023-10-01T10:19:56.380 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:56 smithi078 bash[33821]: cluster 2023-10-01T10:19:54.996460+0000 mgr.x (mgr.24697) 32 : cluster [DBG] pgmap v18: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 614 op/s 2023-10-01T10:19:56.381 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:19:56 smithi078 bash[69661]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-node-exporter-a 2023-10-01T10:19:56.381 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:56 smithi078 bash[27336]: cephadm 2023-10-01T10:19:54.520049+0000 mgr.x (mgr.24697) 28 : cephadm [INF] Updating smithi173:/var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-01T10:19:56.381 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:56 smithi078 bash[27336]: cephadm 2023-10-01T10:19:54.531173+0000 mgr.x (mgr.24697) 29 : cephadm [INF] Updating smithi078:/var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-01T10:19:56.381 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:56 smithi078 bash[27336]: cephadm 2023-10-01T10:19:54.919865+0000 mgr.x (mgr.24697) 30 : cephadm [INF] Reconfiguring node-exporter.a (dependencies changed)... 2023-10-01T10:19:56.381 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:56 smithi078 bash[27336]: cephadm 2023-10-01T10:19:54.920450+0000 mgr.x (mgr.24697) 31 : cephadm [INF] Reconfiguring daemon node-exporter.a on smithi078 2023-10-01T10:19:56.381 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:56 smithi078 bash[27336]: cluster 2023-10-01T10:19:54.996460+0000 mgr.x (mgr.24697) 32 : cluster [DBG] pgmap v18: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 614 op/s 2023-10-01T10:19:56.704 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:19:56 smithi078 systemd[1]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2023-10-01T10:19:56.704 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:19:56 smithi078 bash[69737]: Error response from daemon: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-node-exporter-a 2023-10-01T10:19:56.704 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:19:56 smithi078 systemd[1]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7@node-exporter.a.service: Failed with result 'exit-code'. 2023-10-01T10:19:56.704 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:19:56 smithi078 systemd[1]: Stopped Ceph node-exporter.a for d8016480-6041-11ee-8db5-212e2dc638e7. 2023-10-01T10:19:56.705 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:19:56 smithi078 systemd[1]: Started Ceph node-exporter.a for d8016480-6041-11ee-8db5-212e2dc638e7. 2023-10-01T10:19:58.033 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:57 smithi078 bash[27336]: audit 2023-10-01T10:19:56.653137+0000 mon.a (mon.0) 841 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:58.033 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:57 smithi078 bash[27336]: audit 2023-10-01T10:19:56.671161+0000 mon.a (mon.0) 842 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:58.033 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:57 smithi078 bash[27336]: cephadm 2023-10-01T10:19:56.695818+0000 mgr.x (mgr.24697) 33 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-10-01T10:19:58.033 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:57 smithi078 bash[27336]: cephadm 2023-10-01T10:19:56.731339+0000 mgr.x (mgr.24697) 34 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi078 2023-10-01T10:19:58.033 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:57 smithi078 bash[27336]: audit 2023-10-01T10:19:56.866760+0000 mon.a (mon.0) 843 : audit [DBG] from='client.? 172.21.15.78:0/2651623728' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:58.033 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:19:57 smithi078 bash[27336]: cluster 2023-10-01T10:19:56.997062+0000 mgr.x (mgr.24697) 35 : cluster [DBG] pgmap v19: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 930 op/s 2023-10-01T10:19:58.033 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:57 smithi078 bash[33821]: audit 2023-10-01T10:19:56.653137+0000 mon.a (mon.0) 841 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:58.034 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:57 smithi078 bash[33821]: audit 2023-10-01T10:19:56.671161+0000 mon.a (mon.0) 842 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:58.034 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:57 smithi078 bash[33821]: cephadm 2023-10-01T10:19:56.695818+0000 mgr.x (mgr.24697) 33 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-10-01T10:19:58.034 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:57 smithi078 bash[33821]: cephadm 2023-10-01T10:19:56.731339+0000 mgr.x (mgr.24697) 34 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi078 2023-10-01T10:19:58.034 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:57 smithi078 bash[33821]: audit 2023-10-01T10:19:56.866760+0000 mon.a (mon.0) 843 : audit [DBG] from='client.? 172.21.15.78:0/2651623728' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:58.034 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:19:57 smithi078 bash[33821]: cluster 2023-10-01T10:19:56.997062+0000 mgr.x (mgr.24697) 35 : cluster [DBG] pgmap v19: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 930 op/s 2023-10-01T10:19:58.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:57 smithi173 bash[27612]: audit 2023-10-01T10:19:56.653137+0000 mon.a (mon.0) 841 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:58.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:57 smithi173 bash[27612]: audit 2023-10-01T10:19:56.671161+0000 mon.a (mon.0) 842 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:19:58.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:57 smithi173 bash[27612]: cephadm 2023-10-01T10:19:56.695818+0000 mgr.x (mgr.24697) 33 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2023-10-01T10:19:58.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:57 smithi173 bash[27612]: cephadm 2023-10-01T10:19:56.731339+0000 mgr.x (mgr.24697) 34 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi078 2023-10-01T10:19:58.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:57 smithi173 bash[27612]: audit 2023-10-01T10:19:56.866760+0000 mon.a (mon.0) 843 : audit [DBG] from='client.? 172.21.15.78:0/2651623728' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:19:58.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:19:57 smithi173 bash[27612]: cluster 2023-10-01T10:19:56.997062+0000 mgr.x (mgr.24697) 35 : cluster [DBG] pgmap v19: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 930 op/s 2023-10-01T10:19:58.333 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:19:58 smithi078 bash[69758]: ts=2023-10-01T10:19:58.178Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2023-10-01T10:19:58.334 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:19:58 smithi078 bash[69758]: ts=2023-10-01T10:19:58.178Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2023-10-01T10:19:58.334 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:19:58 smithi078 bash[69758]: ts=2023-10-01T10:19:58.178Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2023-10-01T10:19:58.334 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:19:58 smithi078 bash[69758]: ts=2023-10-01T10:19:58.178Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-10-01T10:19:58.334 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:19:58 smithi078 bash[69758]: ts=2023-10-01T10:19:58.180Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2023-10-01T10:19:58.334 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:19:58 smithi078 bash[69758]: ts=2023-10-01T10:19:58.180Z caller=node_exporter.go:115 level=info collector=arp 2023-10-01T10:19:58.334 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:19:58 smithi078 bash[69758]: ts=2023-10-01T10:19:58.180Z caller=node_exporter.go:115 level=info collector=bcache 2023-10-01T10:19:58.334 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:19:58 smithi078 bash[69758]: ts=2023-10-01T10:19:58.180Z caller=node_exporter.go:115 level=info collector=bonding 2023-10-01T10:19:58.334 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:19:58 smithi078 bash[69758]: ts=2023-10-01T10:19:58.180Z caller=node_exporter.go:115 level=info collector=btrfs 2023-10-01T10:19:58.335 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:19:58 smithi078 bash[69758]: ts=2023-10-01T10:19:58.180Z caller=node_exporter.go:115 level=info collector=conntrack 2023-10-01T10:19:58.335 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:19:58 smithi078 bash[69758]: ts=2023-10-01T10:19:58.181Z caller=node_exporter.go:115 level=info collector=cpu 2023-10-01T10:19:58.335 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:19:58 smithi078 bash[69758]: ts=2023-10-01T10:19:58.181Z caller=node_exporter.go:115 level=info collector=cpufreq 2023-10-01T10:19:58.335 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:19:58 smithi078 bash[69758]: ts=2023-10-01T10:19:58.181Z caller=node_exporter.go:115 level=info collector=diskstats 2023-10-01T10:19:58.335 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:19:58 smithi078 bash[69758]: ts=2023-10-01T10:19:58.181Z caller=node_exporter.go:115 level=info collector=dmi 2023-10-01T10:19:58.335 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:19:58 smithi078 bash[69758]: ts=2023-10-01T10:19:58.181Z caller=node_exporter.go:115 level=info collector=edac 2023-10-01T10:19:58.335 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:19:58 smithi078 bash[69758]: ts=2023-10-01T10:19:58.181Z caller=node_exporter.go:115 level=info collector=entropy 2023-10-01T10:19:58.335 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:19:58 smithi078 bash[69758]: ts=2023-10-01T10:19:58.181Z caller=node_exporter.go:115 level=info collector=fibrechannel 2023-10-01T10:19:58.335 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:19:58 smithi078 bash[69758]: ts=2023-10-01T10:19:58.181Z caller=node_exporter.go:115 level=info collector=filefd 2023-10-01T10:19:58.335 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:19:58 smithi078 bash[69758]: ts=2023-10-01T10:19:58.181Z caller=node_exporter.go:115 level=info collector=filesystem 2023-10-01T10:19:58.336 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:19:58 smithi078 bash[69758]: ts=2023-10-01T10:19:58.181Z caller=node_exporter.go:115 level=info collector=hwmon 2023-10-01T10:19:58.336 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:19:58 smithi078 bash[69758]: ts=2023-10-01T10:19:58.181Z caller=node_exporter.go:115 level=info collector=infiniband 2023-10-01T10:19:58.336 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:19:58 smithi078 bash[69758]: ts=2023-10-01T10:19:58.181Z caller=node_exporter.go:115 level=info collector=ipvs 2023-10-01T10:19:58.336 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:19:58 smithi078 bash[69758]: ts=2023-10-01T10:19:58.181Z caller=node_exporter.go:115 level=info collector=loadavg 2023-10-01T10:19:58.336 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:19:58 smithi078 bash[69758]: ts=2023-10-01T10:19:58.181Z caller=node_exporter.go:115 level=info collector=mdadm 2023-10-01T10:19:58.336 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:19:58 smithi078 bash[69758]: ts=2023-10-01T10:19:58.181Z caller=node_exporter.go:115 level=info collector=meminfo 2023-10-01T10:19:58.336 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:19:58 smithi078 bash[69758]: ts=2023-10-01T10:19:58.181Z caller=node_exporter.go:115 level=info collector=netclass 2023-10-01T10:19:58.336 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:19:58 smithi078 bash[69758]: ts=2023-10-01T10:19:58.181Z caller=node_exporter.go:115 level=info collector=netdev 2023-10-01T10:19:58.336 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:19:58 smithi078 bash[69758]: ts=2023-10-01T10:19:58.181Z caller=node_exporter.go:115 level=info collector=netstat 2023-10-01T10:19:58.337 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:19:58 smithi078 bash[69758]: ts=2023-10-01T10:19:58.181Z caller=node_exporter.go:115 level=info collector=nfs 2023-10-01T10:19:58.337 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:19:58 smithi078 bash[69758]: ts=2023-10-01T10:19:58.181Z caller=node_exporter.go:115 level=info collector=nfsd 2023-10-01T10:19:58.337 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:19:58 smithi078 bash[69758]: ts=2023-10-01T10:19:58.181Z caller=node_exporter.go:115 level=info collector=nvme 2023-10-01T10:19:58.337 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:19:58 smithi078 bash[69758]: ts=2023-10-01T10:19:58.181Z caller=node_exporter.go:115 level=info collector=os 2023-10-01T10:19:58.337 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:19:58 smithi078 bash[69758]: ts=2023-10-01T10:19:58.181Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2023-10-01T10:19:58.337 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:19:58 smithi078 bash[69758]: ts=2023-10-01T10:19:58.181Z caller=node_exporter.go:115 level=info collector=pressure 2023-10-01T10:19:58.337 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:19:58 smithi078 bash[69758]: ts=2023-10-01T10:19:58.182Z caller=node_exporter.go:115 level=info collector=rapl 2023-10-01T10:19:58.337 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:19:58 smithi078 bash[69758]: ts=2023-10-01T10:19:58.182Z caller=node_exporter.go:115 level=info collector=schedstat 2023-10-01T10:19:58.337 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:19:58 smithi078 bash[69758]: ts=2023-10-01T10:19:58.182Z caller=node_exporter.go:115 level=info collector=sockstat 2023-10-01T10:19:58.337 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:19:58 smithi078 bash[69758]: ts=2023-10-01T10:19:58.182Z caller=node_exporter.go:115 level=info collector=softnet 2023-10-01T10:19:58.338 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:19:58 smithi078 bash[69758]: ts=2023-10-01T10:19:58.182Z caller=node_exporter.go:115 level=info collector=stat 2023-10-01T10:19:58.338 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:19:58 smithi078 bash[69758]: ts=2023-10-01T10:19:58.182Z caller=node_exporter.go:115 level=info collector=tapestats 2023-10-01T10:19:58.338 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:19:58 smithi078 bash[69758]: ts=2023-10-01T10:19:58.182Z caller=node_exporter.go:115 level=info collector=textfile 2023-10-01T10:19:58.338 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:19:58 smithi078 bash[69758]: ts=2023-10-01T10:19:58.182Z caller=node_exporter.go:115 level=info collector=thermal_zone 2023-10-01T10:19:58.338 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:19:58 smithi078 bash[69758]: ts=2023-10-01T10:19:58.182Z caller=node_exporter.go:115 level=info collector=time 2023-10-01T10:19:58.338 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:19:58 smithi078 bash[69758]: ts=2023-10-01T10:19:58.182Z caller=node_exporter.go:115 level=info collector=udp_queues 2023-10-01T10:19:58.338 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:19:58 smithi078 bash[69758]: ts=2023-10-01T10:19:58.182Z caller=node_exporter.go:115 level=info collector=uname 2023-10-01T10:19:58.338 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:19:58 smithi078 bash[69758]: ts=2023-10-01T10:19:58.182Z caller=node_exporter.go:115 level=info collector=vmstat 2023-10-01T10:19:58.338 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:19:58 smithi078 bash[69758]: ts=2023-10-01T10:19:58.182Z caller=node_exporter.go:115 level=info collector=xfs 2023-10-01T10:19:58.338 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:19:58 smithi078 bash[69758]: ts=2023-10-01T10:19:58.182Z caller=node_exporter.go:115 level=info collector=zfs 2023-10-01T10:19:58.338 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:19:58 smithi078 bash[69758]: ts=2023-10-01T10:19:58.182Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2023-10-01T10:19:58.339 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 10:19:58 smithi078 bash[69758]: ts=2023-10-01T10:19:58.183Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2023-10-01T10:20:00.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:00 smithi078 bash[33821]: cluster 2023-10-01T10:19:59.001302+0000 mgr.x (mgr.24697) 36 : cluster [DBG] pgmap v20: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 634 op/s 2023-10-01T10:20:00.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:00 smithi078 bash[33821]: audit 2023-10-01T10:19:59.319505+0000 mon.c (mon.2) 107 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:20:00.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:00 smithi078 bash[33821]: audit 2023-10-01T10:19:59.542726+0000 mon.c (mon.2) 108 : audit [DBG] from='client.? 172.21.15.78:0/1028408809' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:00.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:00 smithi078 bash[33821]: cluster 2023-10-01T10:20:00.000110+0000 mon.a (mon.0) 844 : cluster [INF] overall HEALTH_OK 2023-10-01T10:20:00.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:00 smithi078 bash[27336]: cluster 2023-10-01T10:19:59.001302+0000 mgr.x (mgr.24697) 36 : cluster [DBG] pgmap v20: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 634 op/s 2023-10-01T10:20:00.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:00 smithi078 bash[27336]: audit 2023-10-01T10:19:59.319505+0000 mon.c (mon.2) 107 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:20:00.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:00 smithi078 bash[27336]: audit 2023-10-01T10:19:59.542726+0000 mon.c (mon.2) 108 : audit [DBG] from='client.? 172.21.15.78:0/1028408809' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:00.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:00 smithi078 bash[27336]: cluster 2023-10-01T10:20:00.000110+0000 mon.a (mon.0) 844 : cluster [INF] overall HEALTH_OK 2023-10-01T10:20:00.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:00 smithi173 bash[27612]: cluster 2023-10-01T10:19:59.001302+0000 mgr.x (mgr.24697) 36 : cluster [DBG] pgmap v20: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 634 op/s 2023-10-01T10:20:00.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:00 smithi173 bash[27612]: audit 2023-10-01T10:19:59.319505+0000 mon.c (mon.2) 107 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:20:00.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:00 smithi173 bash[27612]: audit 2023-10-01T10:19:59.542726+0000 mon.c (mon.2) 108 : audit [DBG] from='client.? 172.21.15.78:0/1028408809' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:00.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:00 smithi173 bash[27612]: cluster 2023-10-01T10:20:00.000110+0000 mon.a (mon.0) 844 : cluster [INF] overall HEALTH_OK 2023-10-01T10:20:02.369 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:20:02 smithi173 bash[48271]: ts=2023-10-01T10:20:02.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi173.front.sepia.ceph.com:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.78:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-01T10:20:02.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:02 smithi173 bash[27612]: cluster 2023-10-01T10:20:01.001950+0000 mgr.x (mgr.24697) 37 : cluster [DBG] pgmap v21: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 947 op/s 2023-10-01T10:20:02.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:02 smithi078 bash[33821]: cluster 2023-10-01T10:20:01.001950+0000 mgr.x (mgr.24697) 37 : cluster [DBG] pgmap v21: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 947 op/s 2023-10-01T10:20:02.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:02 smithi078 bash[27336]: cluster 2023-10-01T10:20:01.001950+0000 mgr.x (mgr.24697) 37 : cluster [DBG] pgmap v21: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 767 B/s rd, 22 KiB/s wr, 947 op/s 2023-10-01T10:20:03.310 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:03 smithi078 bash[27336]: audit 2023-10-01T10:20:02.216941+0000 mon.c (mon.2) 109 : audit [DBG] from='client.? 172.21.15.78:0/2630660162' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:03.310 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:03 smithi078 bash[33821]: audit 2023-10-01T10:20:02.216941+0000 mon.c (mon.2) 109 : audit [DBG] from='client.? 172.21.15.78:0/2630660162' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:03.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:03 smithi173 bash[27612]: audit 2023-10-01T10:20:02.216941+0000 mon.c (mon.2) 109 : audit [DBG] from='client.? 172.21.15.78:0/2630660162' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:04.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:04 smithi078 bash[27336]: cluster 2023-10-01T10:20:03.002403+0000 mgr.x (mgr.24697) 38 : cluster [DBG] pgmap v22: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 628 op/s 2023-10-01T10:20:04.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:04 smithi078 bash[33821]: cluster 2023-10-01T10:20:03.002403+0000 mgr.x (mgr.24697) 38 : cluster [DBG] pgmap v22: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 628 op/s 2023-10-01T10:20:04.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:04 smithi173 bash[27612]: cluster 2023-10-01T10:20:03.002403+0000 mgr.x (mgr.24697) 38 : cluster [DBG] pgmap v22: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 628 op/s 2023-10-01T10:20:05.368 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:05 smithi078 bash[33821]: audit 2023-10-01T10:20:04.897896+0000 mon.c (mon.2) 110 : audit [DBG] from='client.? 172.21.15.78:0/3908876798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:05.369 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:05 smithi078 bash[27336]: audit 2023-10-01T10:20:04.897896+0000 mon.c (mon.2) 110 : audit [DBG] from='client.? 172.21.15.78:0/3908876798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:05.369 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:20:05 smithi078 bash[27601]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:20:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T10:20:05.631 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:20:05 smithi173 bash[50431]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:20:05] "GET /metrics HTTP/1.1" 200 36075 "" "Prometheus/2.33.4" 2023-10-01T10:20:05.632 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:05 smithi173 bash[27612]: audit 2023-10-01T10:20:04.897896+0000 mon.c (mon.2) 110 : audit [DBG] from='client.? 172.21.15.78:0/3908876798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:06.406 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:06 smithi078 bash[33821]: cluster 2023-10-01T10:20:05.002886+0000 mgr.x (mgr.24697) 39 : cluster [DBG] pgmap v23: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 628 op/s 2023-10-01T10:20:06.407 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:Oct 01 10:20:06 smithi078 systemd[1]: Stopping Ceph alertmanager.a for d8016480-6041-11ee-8db5-212e2dc638e7... 2023-10-01T10:20:06.407 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:Oct 01 10:20:06 smithi078 bash[70078]: Error response from daemon: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-alertmanager.a 2023-10-01T10:20:06.407 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:Oct 01 10:20:06 smithi078 bash[58231]: level=info ts=2023-10-01T10:20:06.352Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2023-10-01T10:20:06.407 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:06 smithi078 bash[27336]: cluster 2023-10-01T10:20:05.002886+0000 mgr.x (mgr.24697) 39 : cluster [DBG] pgmap v23: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 628 op/s 2023-10-01T10:20:06.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:06 smithi173 bash[27612]: cluster 2023-10-01T10:20:05.002886+0000 mgr.x (mgr.24697) 39 : cluster [DBG] pgmap v23: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 628 op/s 2023-10-01T10:20:07.333 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:Oct 01 10:20:07 smithi078 bash[70106]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-alertmanager-a 2023-10-01T10:20:07.333 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:Oct 01 10:20:07 smithi078 bash[70187]: Error response from daemon: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-alertmanager-a 2023-10-01T10:20:07.333 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:Oct 01 10:20:07 smithi078 systemd[1]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-10-01T10:20:07.333 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:Oct 01 10:20:07 smithi078 systemd[1]: Stopped Ceph alertmanager.a for d8016480-6041-11ee-8db5-212e2dc638e7. 2023-10-01T10:20:07.333 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:Oct 01 10:20:07 smithi078 systemd[1]: Started Ceph alertmanager.a for d8016480-6041-11ee-8db5-212e2dc638e7. 2023-10-01T10:20:08.323 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:08 smithi078 bash[27336]: cluster 2023-10-01T10:20:07.003640+0000 mgr.x (mgr.24697) 40 : cluster [DBG] pgmap v24: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 948 op/s 2023-10-01T10:20:08.323 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:08 smithi078 bash[27336]: audit 2023-10-01T10:20:07.317319+0000 mon.a (mon.0) 845 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:20:08.323 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:08 smithi078 bash[27336]: audit 2023-10-01T10:20:07.330694+0000 mon.a (mon.0) 846 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:20:08.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:08 smithi078 bash[33821]: cluster 2023-10-01T10:20:07.003640+0000 mgr.x (mgr.24697) 40 : cluster [DBG] pgmap v24: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 948 op/s 2023-10-01T10:20:08.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:08 smithi078 bash[33821]: audit 2023-10-01T10:20:07.317319+0000 mon.a (mon.0) 845 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:20:08.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:08 smithi078 bash[33821]: audit 2023-10-01T10:20:07.330694+0000 mon.a (mon.0) 846 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:20:08.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:08 smithi078 bash[33821]: cephadm 2023-10-01T10:20:07.338524+0000 mgr.x (mgr.24697) 41 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-01T10:20:08.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:08 smithi078 bash[33821]: audit 2023-10-01T10:20:07.576806+0000 mon.a (mon.0) 847 : audit [DBG] from='client.? 172.21.15.78:0/262701975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:08.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:08 smithi078 bash[27336]: cephadm 2023-10-01T10:20:07.338524+0000 mgr.x (mgr.24697) 41 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-01T10:20:08.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:08 smithi078 bash[27336]: audit 2023-10-01T10:20:07.576806+0000 mon.a (mon.0) 847 : audit [DBG] from='client.? 172.21.15.78:0/262701975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:08.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:08 smithi173 bash[27612]: cluster 2023-10-01T10:20:07.003640+0000 mgr.x (mgr.24697) 40 : cluster [DBG] pgmap v24: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 948 op/s 2023-10-01T10:20:08.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:08 smithi173 bash[27612]: audit 2023-10-01T10:20:07.317319+0000 mon.a (mon.0) 845 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:20:08.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:08 smithi173 bash[27612]: audit 2023-10-01T10:20:07.330694+0000 mon.a (mon.0) 846 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:20:08.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:08 smithi173 bash[27612]: cephadm 2023-10-01T10:20:07.338524+0000 mgr.x (mgr.24697) 41 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-01T10:20:08.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:08 smithi173 bash[27612]: audit 2023-10-01T10:20:07.576806+0000 mon.a (mon.0) 847 : audit [DBG] from='client.? 172.21.15.78:0/262701975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:09.083 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:Oct 01 10:20:08 smithi078 bash[70209]: level=info ts=2023-10-01T10:20:08.695Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2023-10-01T10:20:09.083 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:Oct 01 10:20:08 smithi078 bash[70209]: level=info ts=2023-10-01T10:20:08.696Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2023-10-01T10:20:09.083 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:Oct 01 10:20:08 smithi078 bash[70209]: level=info ts=2023-10-01T10:20:08.698Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.78 port=9094 2023-10-01T10:20:09.083 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:Oct 01 10:20:08 smithi078 bash[70209]: level=info ts=2023-10-01T10:20:08.701Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-10-01T10:20:09.084 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:Oct 01 10:20:08 smithi078 bash[70209]: level=info ts=2023-10-01T10:20:08.768Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-01T10:20:09.084 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:Oct 01 10:20:08 smithi078 bash[70209]: level=info ts=2023-10-01T10:20:08.769Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-10-01T10:20:09.084 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:Oct 01 10:20:08 smithi078 bash[70209]: level=info ts=2023-10-01T10:20:08.772Z caller=main.go:518 msg=Listening address=:9093 2023-10-01T10:20:09.084 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:Oct 01 10:20:08 smithi078 bash[70209]: level=info ts=2023-10-01T10:20:08.772Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2023-10-01T10:20:09.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:09 smithi078 bash[27336]: cephadm 2023-10-01T10:20:07.594432+0000 mgr.x (mgr.24697) 42 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi173 2023-10-01T10:20:09.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:09 smithi078 bash[33821]: cephadm 2023-10-01T10:20:07.594432+0000 mgr.x (mgr.24697) 42 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi173 2023-10-01T10:20:09.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:09 smithi173 bash[27612]: cephadm 2023-10-01T10:20:07.594432+0000 mgr.x (mgr.24697) 42 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi173 2023-10-01T10:20:10.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:10 smithi078 bash[27336]: cluster 2023-10-01T10:20:09.007669+0000 mgr.x (mgr.24697) 43 : cluster [DBG] pgmap v25: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 632 op/s 2023-10-01T10:20:10.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:10 smithi078 bash[27336]: audit 2023-10-01T10:20:10.182770+0000 mon.c (mon.2) 111 : audit [DBG] from='client.? 172.21.15.78:0/3798814594' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:10.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:10 smithi078 bash[33821]: cluster 2023-10-01T10:20:09.007669+0000 mgr.x (mgr.24697) 43 : cluster [DBG] pgmap v25: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 632 op/s 2023-10-01T10:20:10.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:10 smithi078 bash[33821]: audit 2023-10-01T10:20:10.182770+0000 mon.c (mon.2) 111 : audit [DBG] from='client.? 172.21.15.78:0/3798814594' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:10.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:10 smithi173 bash[27612]: cluster 2023-10-01T10:20:09.007669+0000 mgr.x (mgr.24697) 43 : cluster [DBG] pgmap v25: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 632 op/s 2023-10-01T10:20:10.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:10 smithi173 bash[27612]: audit 2023-10-01T10:20:10.182770+0000 mon.c (mon.2) 111 : audit [DBG] from='client.? 172.21.15.78:0/3798814594' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:11.083 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:Oct 01 10:20:10 smithi078 bash[70209]: level=info ts=2023-10-01T10:20:10.701Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000071303s 2023-10-01T10:20:12.619 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:20:12 smithi173 bash[48271]: ts=2023-10-01T10:20:12.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi173.front.sepia.ceph.com:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.78:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-01T10:20:12.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:12 smithi173 bash[27612]: cluster 2023-10-01T10:20:11.008802+0000 mgr.x (mgr.24697) 44 : cluster [DBG] pgmap v26: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 926 op/s 2023-10-01T10:20:12.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:12 smithi078 bash[33821]: cluster 2023-10-01T10:20:11.008802+0000 mgr.x (mgr.24697) 44 : cluster [DBG] pgmap v26: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 926 op/s 2023-10-01T10:20:12.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:12 smithi078 bash[27336]: cluster 2023-10-01T10:20:11.008802+0000 mgr.x (mgr.24697) 44 : cluster [DBG] pgmap v26: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 926 op/s 2023-10-01T10:20:13.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:13 smithi173 bash[27612]: audit 2023-10-01T10:20:12.824783+0000 mon.a (mon.0) 848 : audit [DBG] from='client.? 172.21.15.78:0/3830473773' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:13.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:13 smithi078 bash[27336]: audit 2023-10-01T10:20:12.824783+0000 mon.a (mon.0) 848 : audit [DBG] from='client.? 172.21.15.78:0/3830473773' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:13.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:13 smithi078 bash[33821]: audit 2023-10-01T10:20:12.824783+0000 mon.a (mon.0) 848 : audit [DBG] from='client.? 172.21.15.78:0/3830473773' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:14.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:14 smithi173 bash[27612]: cluster 2023-10-01T10:20:13.009353+0000 mgr.x (mgr.24697) 45 : cluster [DBG] pgmap v27: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 614 op/s 2023-10-01T10:20:14.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:14 smithi173 bash[27612]: audit 2023-10-01T10:20:14.318864+0000 mon.c (mon.2) 112 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:20:14.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:14 smithi078 bash[27336]: cluster 2023-10-01T10:20:13.009353+0000 mgr.x (mgr.24697) 45 : cluster [DBG] pgmap v27: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 614 op/s 2023-10-01T10:20:14.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:14 smithi078 bash[27336]: audit 2023-10-01T10:20:14.318864+0000 mon.c (mon.2) 112 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:20:14.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:14 smithi078 bash[33821]: cluster 2023-10-01T10:20:13.009353+0000 mgr.x (mgr.24697) 45 : cluster [DBG] pgmap v27: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 614 op/s 2023-10-01T10:20:14.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:14 smithi078 bash[33821]: audit 2023-10-01T10:20:14.318864+0000 mon.c (mon.2) 112 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:20:15.407 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:15 smithi078 bash[27336]: cluster 2023-10-01T10:20:15.009803+0000 mgr.x (mgr.24697) 46 : cluster [DBG] pgmap v28: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 614 op/s 2023-10-01T10:20:15.408 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:20:15 smithi078 bash[27601]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:20:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T10:20:15.619 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:20:15 smithi173 bash[50431]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:20:15] "GET /metrics HTTP/1.1" 200 36073 "" "Prometheus/2.33.4" 2023-10-01T10:20:15.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:15 smithi173 bash[27612]: cluster 2023-10-01T10:20:15.009803+0000 mgr.x (mgr.24697) 46 : cluster [DBG] pgmap v28: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 614 op/s 2023-10-01T10:20:15.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:15 smithi078 bash[33821]: cluster 2023-10-01T10:20:15.009803+0000 mgr.x (mgr.24697) 46 : cluster [DBG] pgmap v28: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 614 op/s 2023-10-01T10:20:16.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:16 smithi078 bash[33821]: audit 2023-10-01T10:20:15.518396+0000 mon.a (mon.0) 849 : audit [DBG] from='client.? 172.21.15.78:0/4280385209' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:16.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:16 smithi078 bash[27336]: audit 2023-10-01T10:20:15.518396+0000 mon.a (mon.0) 849 : audit [DBG] from='client.? 172.21.15.78:0/4280385209' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:16.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:16 smithi173 bash[27612]: audit 2023-10-01T10:20:15.518396+0000 mon.a (mon.0) 849 : audit [DBG] from='client.? 172.21.15.78:0/4280385209' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:17.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:17 smithi078 bash[27336]: cluster 2023-10-01T10:20:17.010526+0000 mgr.x (mgr.24697) 47 : cluster [DBG] pgmap v29: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 906 op/s 2023-10-01T10:20:17.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:17 smithi078 bash[33821]: cluster 2023-10-01T10:20:17.010526+0000 mgr.x (mgr.24697) 47 : cluster [DBG] pgmap v29: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 906 op/s 2023-10-01T10:20:17.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:17 smithi173 bash[27612]: cluster 2023-10-01T10:20:17.010526+0000 mgr.x (mgr.24697) 47 : cluster [DBG] pgmap v29: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 906 op/s 2023-10-01T10:20:18.709 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:18 smithi078 bash[33821]: audit 2023-10-01T10:20:18.026365+0000 mon.c (mon.2) 113 : audit [DBG] from='client.? 172.21.15.78:0/4070561224' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:18.709 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:Oct 01 10:20:18 smithi078 bash[70209]: level=info ts=2023-10-01T10:20:18.705Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.003903116s 2023-10-01T10:20:18.709 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:18 smithi078 bash[27336]: audit 2023-10-01T10:20:18.026365+0000 mon.c (mon.2) 113 : audit [DBG] from='client.? 172.21.15.78:0/4070561224' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:18.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:18 smithi173 bash[27612]: audit 2023-10-01T10:20:18.026365+0000 mon.c (mon.2) 113 : audit [DBG] from='client.? 172.21.15.78:0/4070561224' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:19.594 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:19 smithi173 bash[27612]: cluster 2023-10-01T10:20:19.010975+0000 mgr.x (mgr.24697) 48 : cluster [DBG] pgmap v30: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 585 op/s 2023-10-01T10:20:19.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:19 smithi078 bash[33821]: cluster 2023-10-01T10:20:19.010975+0000 mgr.x (mgr.24697) 48 : cluster [DBG] pgmap v30: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 585 op/s 2023-10-01T10:20:19.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:19 smithi078 bash[27336]: cluster 2023-10-01T10:20:19.010975+0000 mgr.x (mgr.24697) 48 : cluster [DBG] pgmap v30: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 585 op/s 2023-10-01T10:20:21.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:20 smithi078 bash[33821]: audit 2023-10-01T10:20:20.730292+0000 mon.a (mon.0) 850 : audit [DBG] from='client.? 172.21.15.78:0/4193484976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:21.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:20 smithi078 bash[27336]: audit 2023-10-01T10:20:20.730292+0000 mon.a (mon.0) 850 : audit [DBG] from='client.? 172.21.15.78:0/4193484976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:21.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:20 smithi173 bash[27612]: audit 2023-10-01T10:20:20.730292+0000 mon.a (mon.0) 850 : audit [DBG] from='client.? 172.21.15.78:0/4193484976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:22.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:22 smithi173 bash[27612]: cluster 2023-10-01T10:20:21.011692+0000 mgr.x (mgr.24697) 49 : cluster [DBG] pgmap v31: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 881 op/s 2023-10-01T10:20:22.370 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:20:22 smithi173 systemd[1]: Stopping Ceph prometheus.a for d8016480-6041-11ee-8db5-212e2dc638e7... 2023-10-01T10:20:22.370 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:20:22 smithi173 bash[52591]: Error response from daemon: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-prometheus.a 2023-10-01T10:20:22.370 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:20:22 smithi173 bash[48271]: ts=2023-10-01T10:20:22.314Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi173.front.sepia.ceph.com:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.78:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-01T10:20:22.370 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:20:22 smithi173 bash[48271]: ts=2023-10-01T10:20:22.349Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-10-01T10:20:22.370 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:20:22 smithi173 bash[48271]: ts=2023-10-01T10:20:22.349Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2023-10-01T10:20:22.370 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:20:22 smithi173 bash[48271]: ts=2023-10-01T10:20:22.349Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2023-10-01T10:20:22.370 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:20:22 smithi173 bash[48271]: ts=2023-10-01T10:20:22.349Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2023-10-01T10:20:22.370 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:20:22 smithi173 bash[48271]: ts=2023-10-01T10:20:22.349Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2023-10-01T10:20:22.370 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:20:22 smithi173 bash[48271]: ts=2023-10-01T10:20:22.349Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2023-10-01T10:20:22.371 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:20:22 smithi173 bash[48271]: ts=2023-10-01T10:20:22.349Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2023-10-01T10:20:22.371 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:20:22 smithi173 bash[48271]: ts=2023-10-01T10:20:22.350Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2023-10-01T10:20:22.371 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:20:22 smithi173 bash[48271]: ts=2023-10-01T10:20:22.350Z caller=main.go:828 level=info msg="Scrape manager stopped" 2023-10-01T10:20:22.371 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:20:22 smithi173 bash[48271]: ts=2023-10-01T10:20:22.351Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2023-10-01T10:20:22.371 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:20:22 smithi173 bash[48271]: ts=2023-10-01T10:20:22.351Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2023-10-01T10:20:22.371 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:20:22 smithi173 bash[48271]: ts=2023-10-01T10:20:22.351Z caller=main.go:1066 level=info msg="See you next time!" 2023-10-01T10:20:22.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:22 smithi078 bash[27336]: cluster 2023-10-01T10:20:21.011692+0000 mgr.x (mgr.24697) 49 : cluster [DBG] pgmap v31: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 881 op/s 2023-10-01T10:20:22.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:22 smithi078 bash[33821]: cluster 2023-10-01T10:20:21.011692+0000 mgr.x (mgr.24697) 49 : cluster [DBG] pgmap v31: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 881 op/s 2023-10-01T10:20:23.351 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:20:23 smithi173 bash[52621]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-prometheus-a 2023-10-01T10:20:23.351 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:20:23 smithi173 bash[52678]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-prometheus-a 2023-10-01T10:20:23.351 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:20:23 smithi173 systemd[1]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7@prometheus.a.service: Succeeded. 2023-10-01T10:20:23.352 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:20:23 smithi173 systemd[1]: Stopped Ceph prometheus.a for d8016480-6041-11ee-8db5-212e2dc638e7. 2023-10-01T10:20:23.352 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:20:23 smithi173 systemd[1]: Started Ceph prometheus.a for d8016480-6041-11ee-8db5-212e2dc638e7. 2023-10-01T10:20:24.038 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:20:23 smithi173 systemd[1]: Stopping Ceph node-exporter.b for d8016480-6041-11ee-8db5-212e2dc638e7... 2023-10-01T10:20:24.038 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:20:23 smithi173 bash[52727]: Error response from daemon: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-node-exporter.b 2023-10-01T10:20:24.316 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:24 smithi173 bash[27612]: cluster 2023-10-01T10:20:23.012062+0000 mgr.x (mgr.24697) 50 : cluster [DBG] pgmap v32: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 587 op/s 2023-10-01T10:20:24.316 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:24 smithi173 bash[27612]: audit 2023-10-01T10:20:23.314278+0000 mon.a (mon.0) 851 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:20:24.317 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:24 smithi173 bash[27612]: audit 2023-10-01T10:20:23.331344+0000 mon.a (mon.0) 852 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:20:24.317 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:24 smithi173 bash[27612]: cephadm 2023-10-01T10:20:23.346660+0000 mgr.x (mgr.24697) 51 : cephadm [INF] Reconfiguring node-exporter.b (dependencies changed)... 2023-10-01T10:20:24.317 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:24 smithi173 bash[27612]: cephadm 2023-10-01T10:20:23.347123+0000 mgr.x (mgr.24697) 52 : cephadm [INF] Reconfiguring daemon node-exporter.b on smithi173 2023-10-01T10:20:24.317 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:24 smithi173 bash[27612]: audit 2023-10-01T10:20:23.361296+0000 mon.a (mon.0) 853 : audit [DBG] from='client.? 172.21.15.78:0/1314475531' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:24.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:24 smithi078 bash[33821]: cluster 2023-10-01T10:20:23.012062+0000 mgr.x (mgr.24697) 50 : cluster [DBG] pgmap v32: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 587 op/s 2023-10-01T10:20:24.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:24 smithi078 bash[33821]: audit 2023-10-01T10:20:23.314278+0000 mon.a (mon.0) 851 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:20:24.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:24 smithi078 bash[33821]: audit 2023-10-01T10:20:23.331344+0000 mon.a (mon.0) 852 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:20:24.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:24 smithi078 bash[33821]: cephadm 2023-10-01T10:20:23.346660+0000 mgr.x (mgr.24697) 51 : cephadm [INF] Reconfiguring node-exporter.b (dependencies changed)... 2023-10-01T10:20:24.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:24 smithi078 bash[33821]: cephadm 2023-10-01T10:20:23.347123+0000 mgr.x (mgr.24697) 52 : cephadm [INF] Reconfiguring daemon node-exporter.b on smithi173 2023-10-01T10:20:24.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:24 smithi078 bash[33821]: audit 2023-10-01T10:20:23.361296+0000 mon.a (mon.0) 853 : audit [DBG] from='client.? 172.21.15.78:0/1314475531' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:24.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:24 smithi078 bash[27336]: cluster 2023-10-01T10:20:23.012062+0000 mgr.x (mgr.24697) 50 : cluster [DBG] pgmap v32: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 587 op/s 2023-10-01T10:20:24.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:24 smithi078 bash[27336]: audit 2023-10-01T10:20:23.314278+0000 mon.a (mon.0) 851 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:20:24.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:24 smithi078 bash[27336]: audit 2023-10-01T10:20:23.331344+0000 mon.a (mon.0) 852 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:20:24.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:24 smithi078 bash[27336]: cephadm 2023-10-01T10:20:23.346660+0000 mgr.x (mgr.24697) 51 : cephadm [INF] Reconfiguring node-exporter.b (dependencies changed)... 2023-10-01T10:20:24.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:24 smithi078 bash[27336]: cephadm 2023-10-01T10:20:23.347123+0000 mgr.x (mgr.24697) 52 : cephadm [INF] Reconfiguring daemon node-exporter.b on smithi173 2023-10-01T10:20:24.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:24 smithi078 bash[27336]: audit 2023-10-01T10:20:23.361296+0000 mon.a (mon.0) 853 : audit [DBG] from='client.? 172.21.15.78:0/1314475531' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:25.206 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:20:25 smithi173 bash[52702]: ts=2023-10-01T10:20:25.142Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2023-10-01T10:20:25.206 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:20:25 smithi173 bash[52702]: ts=2023-10-01T10:20:25.142Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2023-10-01T10:20:25.206 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:20:25 smithi173 bash[52702]: ts=2023-10-01T10:20:25.142Z caller=main.go:518 level=info host_details="(Linux 5.15.0-84-generic #93~20.04.1-Ubuntu SMP Wed Sep 6 16:15:40 UTC 2023 x86_64 smithi173 (none))" 2023-10-01T10:20:25.206 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:20:25 smithi173 bash[52702]: ts=2023-10-01T10:20:25.142Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-10-01T10:20:25.206 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:20:25 smithi173 bash[52702]: ts=2023-10-01T10:20:25.142Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-10-01T10:20:25.206 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:20:25 smithi173 bash[52702]: ts=2023-10-01T10:20:25.143Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2023-10-01T10:20:25.206 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:20:25 smithi173 bash[52702]: ts=2023-10-01T10:20:25.145Z caller=main.go:923 level=info msg="Starting TSDB ..." 2023-10-01T10:20:25.206 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:20:25 smithi173 bash[52702]: ts=2023-10-01T10:20:25.146Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2023-10-01T10:20:25.206 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:20:25 smithi173 bash[52702]: ts=2023-10-01T10:20:25.148Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-10-01T10:20:25.207 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:20:25 smithi173 bash[52702]: ts=2023-10-01T10:20:25.148Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=4.098µs 2023-10-01T10:20:25.207 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:20:25 smithi173 bash[52702]: ts=2023-10-01T10:20:25.149Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-10-01T10:20:25.207 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:20:24 smithi173 bash[52759]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-node-exporter-b 2023-10-01T10:20:25.207 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:20:25 smithi173 systemd[1]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2023-10-01T10:20:25.207 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:20:25 smithi173 bash[52871]: Error response from daemon: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-node-exporter-b 2023-10-01T10:20:25.207 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:20:25 smithi173 systemd[1]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7@node-exporter.b.service: Failed with result 'exit-code'. 2023-10-01T10:20:25.207 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:20:25 smithi173 systemd[1]: Stopped Ceph node-exporter.b for d8016480-6041-11ee-8db5-212e2dc638e7. 2023-10-01T10:20:25.207 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:20:25 smithi173 systemd[1]: Started Ceph node-exporter.b for d8016480-6041-11ee-8db5-212e2dc638e7. 2023-10-01T10:20:25.619 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:20:25 smithi173 bash[52702]: ts=2023-10-01T10:20:25.459Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2023-10-01T10:20:26.254 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:26 smithi173 bash[27612]: cluster 2023-10-01T10:20:25.012677+0000 mgr.x (mgr.24697) 53 : cluster [DBG] pgmap v33: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 587 op/s 2023-10-01T10:20:26.255 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:26 smithi173 bash[27612]: audit 2023-10-01T10:20:25.248947+0000 mon.a (mon.0) 854 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:20:26.255 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:20:25 smithi173 bash[52702]: ts=2023-10-01T10:20:25.876Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2023-10-01T10:20:26.255 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:20:25 smithi173 bash[52702]: ts=2023-10-01T10:20:25.877Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2023-10-01T10:20:26.255 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:20:25 smithi173 bash[52702]: ts=2023-10-01T10:20:25.877Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=48.075µs wal_replay_duration=728.182465ms total_replay_duration=728.29867ms 2023-10-01T10:20:26.255 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:20:25 smithi173 bash[52702]: ts=2023-10-01T10:20:25.881Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2023-10-01T10:20:26.255 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:20:25 smithi173 bash[52702]: ts=2023-10-01T10:20:25.881Z caller=main.go:947 level=info msg="TSDB started" 2023-10-01T10:20:26.256 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:20:25 smithi173 bash[52702]: ts=2023-10-01T10:20:25.881Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-10-01T10:20:26.256 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:20:25 smithi173 bash[52702]: ts=2023-10-01T10:20:25.914Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=32.884592ms db_storage=1.153µs remote_storage=2.307µs web_handler=477ns query_engine=1.078µs scrape=367.836µs scrape_sd=142.567µs notify=20.698µs notify_sd=16.251µs rules=31.913131ms 2023-10-01T10:20:26.256 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:20:25 smithi173 bash[52702]: ts=2023-10-01T10:20:25.914Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2023-10-01T10:20:26.504 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:26 smithi173 bash[27612]: audit 2023-10-01T10:20:25.262597+0000 mon.a (mon.0) 855 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:20:26.505 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:26 smithi173 bash[27612]: cephadm 2023-10-01T10:20:25.272648+0000 mgr.x (mgr.24697) 54 : cephadm [INF] Reconfiguring grafana.a (dependencies changed)... 2023-10-01T10:20:26.505 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:26 smithi173 bash[27612]: cephadm 2023-10-01T10:20:25.296121+0000 mgr.x (mgr.24697) 55 : cephadm [INF] Reconfiguring daemon grafana.a on smithi173 2023-10-01T10:20:26.505 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:26 smithi173 bash[27612]: audit 2023-10-01T10:20:26.015094+0000 mon.a (mon.0) 856 : audit [DBG] from='client.? 172.21.15.78:0/1843610649' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:26.505 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:20:26 smithi173 bash[52894]: ts=2023-10-01T10:20:26.502Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2023-10-01T10:20:26.505 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:20:26 smithi173 bash[52894]: ts=2023-10-01T10:20:26.502Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2023-10-01T10:20:26.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:26 smithi078 bash[27336]: cluster 2023-10-01T10:20:25.012677+0000 mgr.x (mgr.24697) 53 : cluster [DBG] pgmap v33: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 587 op/s 2023-10-01T10:20:26.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:26 smithi078 bash[27336]: audit 2023-10-01T10:20:25.248947+0000 mon.a (mon.0) 854 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:20:26.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:26 smithi078 bash[27336]: audit 2023-10-01T10:20:25.262597+0000 mon.a (mon.0) 855 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:20:26.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:26 smithi078 bash[27336]: cephadm 2023-10-01T10:20:25.272648+0000 mgr.x (mgr.24697) 54 : cephadm [INF] Reconfiguring grafana.a (dependencies changed)... 2023-10-01T10:20:26.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:26 smithi078 bash[27336]: cephadm 2023-10-01T10:20:25.296121+0000 mgr.x (mgr.24697) 55 : cephadm [INF] Reconfiguring daemon grafana.a on smithi173 2023-10-01T10:20:26.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:26 smithi078 bash[27336]: audit 2023-10-01T10:20:26.015094+0000 mon.a (mon.0) 856 : audit [DBG] from='client.? 172.21.15.78:0/1843610649' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:26.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:26 smithi078 bash[33821]: cluster 2023-10-01T10:20:25.012677+0000 mgr.x (mgr.24697) 53 : cluster [DBG] pgmap v33: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 587 op/s 2023-10-01T10:20:26.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:26 smithi078 bash[33821]: audit 2023-10-01T10:20:25.248947+0000 mon.a (mon.0) 854 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:20:26.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:26 smithi078 bash[33821]: audit 2023-10-01T10:20:25.262597+0000 mon.a (mon.0) 855 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:20:26.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:26 smithi078 bash[33821]: cephadm 2023-10-01T10:20:25.272648+0000 mgr.x (mgr.24697) 54 : cephadm [INF] Reconfiguring grafana.a (dependencies changed)... 2023-10-01T10:20:26.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:26 smithi078 bash[33821]: cephadm 2023-10-01T10:20:25.296121+0000 mgr.x (mgr.24697) 55 : cephadm [INF] Reconfiguring daemon grafana.a on smithi173 2023-10-01T10:20:26.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:26 smithi078 bash[33821]: audit 2023-10-01T10:20:26.015094+0000 mon.a (mon.0) 856 : audit [DBG] from='client.? 172.21.15.78:0/1843610649' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:26.870 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:20:26 smithi173 bash[52894]: ts=2023-10-01T10:20:26.503Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2023-10-01T10:20:26.871 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:20:26 smithi173 bash[52894]: ts=2023-10-01T10:20:26.503Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-10-01T10:20:26.871 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:20:26 smithi173 bash[52894]: ts=2023-10-01T10:20:26.504Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2023-10-01T10:20:26.871 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:20:26 smithi173 bash[52894]: ts=2023-10-01T10:20:26.504Z caller=node_exporter.go:115 level=info collector=arp 2023-10-01T10:20:26.871 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:20:26 smithi173 bash[52894]: ts=2023-10-01T10:20:26.504Z caller=node_exporter.go:115 level=info collector=bcache 2023-10-01T10:20:26.871 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:20:26 smithi173 bash[52894]: ts=2023-10-01T10:20:26.505Z caller=node_exporter.go:115 level=info collector=bonding 2023-10-01T10:20:26.871 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:20:26 smithi173 bash[52894]: ts=2023-10-01T10:20:26.505Z caller=node_exporter.go:115 level=info collector=btrfs 2023-10-01T10:20:26.871 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:20:26 smithi173 bash[52894]: ts=2023-10-01T10:20:26.505Z caller=node_exporter.go:115 level=info collector=conntrack 2023-10-01T10:20:26.871 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:20:26 smithi173 bash[52894]: ts=2023-10-01T10:20:26.505Z caller=node_exporter.go:115 level=info collector=cpu 2023-10-01T10:20:26.871 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:20:26 smithi173 bash[52894]: ts=2023-10-01T10:20:26.506Z caller=node_exporter.go:115 level=info collector=cpufreq 2023-10-01T10:20:26.871 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:20:26 smithi173 bash[52894]: ts=2023-10-01T10:20:26.506Z caller=node_exporter.go:115 level=info collector=diskstats 2023-10-01T10:20:26.871 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:20:26 smithi173 bash[52894]: ts=2023-10-01T10:20:26.506Z caller=node_exporter.go:115 level=info collector=dmi 2023-10-01T10:20:26.871 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:20:26 smithi173 bash[52894]: ts=2023-10-01T10:20:26.506Z caller=node_exporter.go:115 level=info collector=edac 2023-10-01T10:20:26.872 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:20:26 smithi173 bash[52894]: ts=2023-10-01T10:20:26.507Z caller=node_exporter.go:115 level=info collector=entropy 2023-10-01T10:20:26.872 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:20:26 smithi173 bash[52894]: ts=2023-10-01T10:20:26.507Z caller=node_exporter.go:115 level=info collector=fibrechannel 2023-10-01T10:20:26.872 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:20:26 smithi173 bash[52894]: ts=2023-10-01T10:20:26.507Z caller=node_exporter.go:115 level=info collector=filefd 2023-10-01T10:20:26.872 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:20:26 smithi173 bash[52894]: ts=2023-10-01T10:20:26.507Z caller=node_exporter.go:115 level=info collector=filesystem 2023-10-01T10:20:26.872 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:20:26 smithi173 bash[52894]: ts=2023-10-01T10:20:26.508Z caller=node_exporter.go:115 level=info collector=hwmon 2023-10-01T10:20:26.872 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:20:26 smithi173 bash[52894]: ts=2023-10-01T10:20:26.508Z caller=node_exporter.go:115 level=info collector=infiniband 2023-10-01T10:20:26.872 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:20:26 smithi173 bash[52894]: ts=2023-10-01T10:20:26.508Z caller=node_exporter.go:115 level=info collector=ipvs 2023-10-01T10:20:26.872 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:20:26 smithi173 bash[52894]: ts=2023-10-01T10:20:26.508Z caller=node_exporter.go:115 level=info collector=loadavg 2023-10-01T10:20:26.872 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:20:26 smithi173 bash[52894]: ts=2023-10-01T10:20:26.509Z caller=node_exporter.go:115 level=info collector=mdadm 2023-10-01T10:20:26.872 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:20:26 smithi173 bash[52894]: ts=2023-10-01T10:20:26.509Z caller=node_exporter.go:115 level=info collector=meminfo 2023-10-01T10:20:26.872 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:20:26 smithi173 bash[52894]: ts=2023-10-01T10:20:26.509Z caller=node_exporter.go:115 level=info collector=netclass 2023-10-01T10:20:26.873 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:20:26 smithi173 bash[52894]: ts=2023-10-01T10:20:26.509Z caller=node_exporter.go:115 level=info collector=netdev 2023-10-01T10:20:26.873 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:20:26 smithi173 bash[52894]: ts=2023-10-01T10:20:26.510Z caller=node_exporter.go:115 level=info collector=netstat 2023-10-01T10:20:26.873 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:20:26 smithi173 bash[52894]: ts=2023-10-01T10:20:26.510Z caller=node_exporter.go:115 level=info collector=nfs 2023-10-01T10:20:26.873 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:20:26 smithi173 bash[52894]: ts=2023-10-01T10:20:26.510Z caller=node_exporter.go:115 level=info collector=nfsd 2023-10-01T10:20:26.873 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:20:26 smithi173 bash[52894]: ts=2023-10-01T10:20:26.510Z caller=node_exporter.go:115 level=info collector=nvme 2023-10-01T10:20:26.873 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:20:26 smithi173 bash[52894]: ts=2023-10-01T10:20:26.510Z caller=node_exporter.go:115 level=info collector=os 2023-10-01T10:20:26.873 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:20:26 smithi173 bash[52894]: ts=2023-10-01T10:20:26.511Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2023-10-01T10:20:26.873 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:20:26 smithi173 bash[52894]: ts=2023-10-01T10:20:26.511Z caller=node_exporter.go:115 level=info collector=pressure 2023-10-01T10:20:26.873 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:20:26 smithi173 bash[52894]: ts=2023-10-01T10:20:26.511Z caller=node_exporter.go:115 level=info collector=rapl 2023-10-01T10:20:26.873 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:20:26 smithi173 bash[52894]: ts=2023-10-01T10:20:26.511Z caller=node_exporter.go:115 level=info collector=schedstat 2023-10-01T10:20:26.873 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:20:26 smithi173 bash[52894]: ts=2023-10-01T10:20:26.512Z caller=node_exporter.go:115 level=info collector=sockstat 2023-10-01T10:20:26.873 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:20:26 smithi173 bash[52894]: ts=2023-10-01T10:20:26.512Z caller=node_exporter.go:115 level=info collector=softnet 2023-10-01T10:20:26.874 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:20:26 smithi173 bash[52894]: ts=2023-10-01T10:20:26.512Z caller=node_exporter.go:115 level=info collector=stat 2023-10-01T10:20:26.874 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:20:26 smithi173 bash[52894]: ts=2023-10-01T10:20:26.512Z caller=node_exporter.go:115 level=info collector=tapestats 2023-10-01T10:20:26.874 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:20:26 smithi173 bash[52894]: ts=2023-10-01T10:20:26.512Z caller=node_exporter.go:115 level=info collector=textfile 2023-10-01T10:20:26.874 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:20:26 smithi173 bash[52894]: ts=2023-10-01T10:20:26.512Z caller=node_exporter.go:115 level=info collector=thermal_zone 2023-10-01T10:20:26.874 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:20:26 smithi173 bash[52894]: ts=2023-10-01T10:20:26.513Z caller=node_exporter.go:115 level=info collector=time 2023-10-01T10:20:26.874 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:20:26 smithi173 bash[52894]: ts=2023-10-01T10:20:26.513Z caller=node_exporter.go:115 level=info collector=udp_queues 2023-10-01T10:20:26.874 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:20:26 smithi173 bash[52894]: ts=2023-10-01T10:20:26.513Z caller=node_exporter.go:115 level=info collector=uname 2023-10-01T10:20:26.874 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:20:26 smithi173 bash[52894]: ts=2023-10-01T10:20:26.513Z caller=node_exporter.go:115 level=info collector=vmstat 2023-10-01T10:20:26.874 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:20:26 smithi173 bash[52894]: ts=2023-10-01T10:20:26.514Z caller=node_exporter.go:115 level=info collector=xfs 2023-10-01T10:20:26.874 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:20:26 smithi173 bash[52894]: ts=2023-10-01T10:20:26.514Z caller=node_exporter.go:115 level=info collector=zfs 2023-10-01T10:20:26.874 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:20:26 smithi173 bash[52894]: ts=2023-10-01T10:20:26.514Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2023-10-01T10:20:26.875 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 10:20:26 smithi173 bash[52894]: ts=2023-10-01T10:20:26.515Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2023-10-01T10:20:28.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:28 smithi078 bash[27336]: cluster 2023-10-01T10:20:27.013410+0000 mgr.x (mgr.24697) 56 : cluster [DBG] pgmap v34: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 883 op/s 2023-10-01T10:20:28.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:28 smithi078 bash[33821]: cluster 2023-10-01T10:20:27.013410+0000 mgr.x (mgr.24697) 56 : cluster [DBG] pgmap v34: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 883 op/s 2023-10-01T10:20:28.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:28 smithi173 bash[27612]: cluster 2023-10-01T10:20:27.013410+0000 mgr.x (mgr.24697) 56 : cluster [DBG] pgmap v34: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 883 op/s 2023-10-01T10:20:29.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:29 smithi078 bash[33821]: audit 2023-10-01T10:20:28.681527+0000 mon.a (mon.0) 857 : audit [DBG] from='client.? 172.21.15.78:0/594347026' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:29.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:29 smithi078 bash[27336]: audit 2023-10-01T10:20:28.681527+0000 mon.a (mon.0) 857 : audit [DBG] from='client.? 172.21.15.78:0/594347026' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:29.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:29 smithi173 bash[27612]: audit 2023-10-01T10:20:28.681527+0000 mon.a (mon.0) 857 : audit [DBG] from='client.? 172.21.15.78:0/594347026' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:30.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:30 smithi078 bash[27336]: cluster 2023-10-01T10:20:29.013773+0000 mgr.x (mgr.24697) 57 : cluster [DBG] pgmap v35: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 592 op/s 2023-10-01T10:20:30.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:30 smithi078 bash[27336]: audit 2023-10-01T10:20:29.319743+0000 mon.c (mon.2) 114 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:20:30.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:30 smithi078 bash[33821]: cluster 2023-10-01T10:20:29.013773+0000 mgr.x (mgr.24697) 57 : cluster [DBG] pgmap v35: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 592 op/s 2023-10-01T10:20:30.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:30 smithi078 bash[33821]: audit 2023-10-01T10:20:29.319743+0000 mon.c (mon.2) 114 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:20:30.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:30 smithi173 bash[27612]: cluster 2023-10-01T10:20:29.013773+0000 mgr.x (mgr.24697) 57 : cluster [DBG] pgmap v35: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 592 op/s 2023-10-01T10:20:30.622 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:30 smithi173 bash[27612]: audit 2023-10-01T10:20:29.319743+0000 mon.c (mon.2) 114 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:20:32.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:32 smithi078 bash[33821]: cluster 2023-10-01T10:20:31.014624+0000 mgr.x (mgr.24697) 58 : cluster [DBG] pgmap v36: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 881 op/s 2023-10-01T10:20:32.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:32 smithi078 bash[33821]: audit 2023-10-01T10:20:31.349342+0000 mon.c (mon.2) 115 : audit [DBG] from='client.? 172.21.15.78:0/3838405064' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:32.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:32 smithi078 bash[27336]: cluster 2023-10-01T10:20:31.014624+0000 mgr.x (mgr.24697) 58 : cluster [DBG] pgmap v36: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 881 op/s 2023-10-01T10:20:32.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:32 smithi078 bash[27336]: audit 2023-10-01T10:20:31.349342+0000 mon.c (mon.2) 115 : audit [DBG] from='client.? 172.21.15.78:0/3838405064' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:32.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:32 smithi173 bash[27612]: cluster 2023-10-01T10:20:31.014624+0000 mgr.x (mgr.24697) 58 : cluster [DBG] pgmap v36: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 881 op/s 2023-10-01T10:20:32.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:32 smithi173 bash[27612]: audit 2023-10-01T10:20:31.349342+0000 mon.c (mon.2) 115 : audit [DBG] from='client.? 172.21.15.78:0/3838405064' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:33.119 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:20:32 smithi173 bash[50431]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:20:32] "GET /metrics HTTP/1.1" 200 36076 "" "Prometheus/2.33.4" 2023-10-01T10:20:34.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:34 smithi078 bash[27336]: cluster 2023-10-01T10:20:33.015083+0000 mgr.x (mgr.24697) 59 : cluster [DBG] pgmap v37: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 585 op/s 2023-10-01T10:20:34.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:34 smithi078 bash[27336]: audit 2023-10-01T10:20:33.999878+0000 mon.a (mon.0) 858 : audit [DBG] from='client.? 172.21.15.78:0/903588057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:34.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:34 smithi078 bash[33821]: cluster 2023-10-01T10:20:33.015083+0000 mgr.x (mgr.24697) 59 : cluster [DBG] pgmap v37: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 585 op/s 2023-10-01T10:20:34.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:34 smithi078 bash[33821]: audit 2023-10-01T10:20:33.999878+0000 mon.a (mon.0) 858 : audit [DBG] from='client.? 172.21.15.78:0/903588057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:34.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:34 smithi173 bash[27612]: cluster 2023-10-01T10:20:33.015083+0000 mgr.x (mgr.24697) 59 : cluster [DBG] pgmap v37: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 585 op/s 2023-10-01T10:20:34.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:34 smithi173 bash[27612]: audit 2023-10-01T10:20:33.999878+0000 mon.a (mon.0) 858 : audit [DBG] from='client.? 172.21.15.78:0/903588057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:36.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:36 smithi078 bash[33821]: cluster 2023-10-01T10:20:35.015517+0000 mgr.x (mgr.24697) 60 : cluster [DBG] pgmap v38: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 586 op/s 2023-10-01T10:20:36.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:36 smithi078 bash[27336]: cluster 2023-10-01T10:20:35.015517+0000 mgr.x (mgr.24697) 60 : cluster [DBG] pgmap v38: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 586 op/s 2023-10-01T10:20:36.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:36 smithi173 bash[27612]: cluster 2023-10-01T10:20:35.015517+0000 mgr.x (mgr.24697) 60 : cluster [DBG] pgmap v38: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 586 op/s 2023-10-01T10:20:37.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:37 smithi078 bash[27336]: audit 2023-10-01T10:20:36.705551+0000 mon.c (mon.2) 116 : audit [DBG] from='client.? 172.21.15.78:0/1147521997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:37.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:37 smithi078 bash[33821]: audit 2023-10-01T10:20:36.705551+0000 mon.c (mon.2) 116 : audit [DBG] from='client.? 172.21.15.78:0/1147521997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:37.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:37 smithi173 bash[27612]: audit 2023-10-01T10:20:36.705551+0000 mon.c (mon.2) 116 : audit [DBG] from='client.? 172.21.15.78:0/1147521997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:38.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:38 smithi078 bash[27336]: cluster 2023-10-01T10:20:37.016077+0000 mgr.x (mgr.24697) 61 : cluster [DBG] pgmap v39: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 878 op/s 2023-10-01T10:20:38.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:38 smithi078 bash[33821]: cluster 2023-10-01T10:20:37.016077+0000 mgr.x (mgr.24697) 61 : cluster [DBG] pgmap v39: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 878 op/s 2023-10-01T10:20:38.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:38 smithi173 bash[27612]: cluster 2023-10-01T10:20:37.016077+0000 mgr.x (mgr.24697) 61 : cluster [DBG] pgmap v39: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 878 op/s 2023-10-01T10:20:39.119 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:20:38 smithi173 bash[52702]: ts=2023-10-01T10:20:38.662Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"smithi173:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"smithi173.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-01T10:20:40.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:40 smithi078 bash[27336]: cluster 2023-10-01T10:20:39.016525+0000 mgr.x (mgr.24697) 62 : cluster [DBG] pgmap v40: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 582 op/s 2023-10-01T10:20:40.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:40 smithi078 bash[27336]: audit 2023-10-01T10:20:39.389294+0000 mon.c (mon.2) 117 : audit [DBG] from='client.? 172.21.15.78:0/1620327550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:40.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:40 smithi078 bash[33821]: cluster 2023-10-01T10:20:39.016525+0000 mgr.x (mgr.24697) 62 : cluster [DBG] pgmap v40: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 582 op/s 2023-10-01T10:20:40.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:40 smithi078 bash[33821]: audit 2023-10-01T10:20:39.389294+0000 mon.c (mon.2) 117 : audit [DBG] from='client.? 172.21.15.78:0/1620327550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:40.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:40 smithi173 bash[27612]: cluster 2023-10-01T10:20:39.016525+0000 mgr.x (mgr.24697) 62 : cluster [DBG] pgmap v40: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 582 op/s 2023-10-01T10:20:40.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:40 smithi173 bash[27612]: audit 2023-10-01T10:20:39.389294+0000 mon.c (mon.2) 117 : audit [DBG] from='client.? 172.21.15.78:0/1620327550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:42.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:42 smithi078 bash[33821]: cluster 2023-10-01T10:20:41.017203+0000 mgr.x (mgr.24697) 63 : cluster [DBG] pgmap v41: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 867 op/s 2023-10-01T10:20:42.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:42 smithi078 bash[33821]: audit 2023-10-01T10:20:41.993184+0000 mon.a (mon.0) 859 : audit [DBG] from='client.? 172.21.15.78:0/1309638193' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:42.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:42 smithi078 bash[27336]: cluster 2023-10-01T10:20:41.017203+0000 mgr.x (mgr.24697) 63 : cluster [DBG] pgmap v41: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 867 op/s 2023-10-01T10:20:42.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:42 smithi078 bash[27336]: audit 2023-10-01T10:20:41.993184+0000 mon.a (mon.0) 859 : audit [DBG] from='client.? 172.21.15.78:0/1309638193' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:42.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:42 smithi173 bash[27612]: cluster 2023-10-01T10:20:41.017203+0000 mgr.x (mgr.24697) 63 : cluster [DBG] pgmap v41: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 867 op/s 2023-10-01T10:20:42.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:42 smithi173 bash[27612]: audit 2023-10-01T10:20:41.993184+0000 mon.a (mon.0) 859 : audit [DBG] from='client.? 172.21.15.78:0/1309638193' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:43.119 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:20:42 smithi173 bash[50431]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:20:42] "GET /metrics HTTP/1.1" 200 36076 "" "Prometheus/2.33.4" 2023-10-01T10:20:44.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:44 smithi078 bash[33821]: cluster 2023-10-01T10:20:43.019930+0000 mgr.x (mgr.24697) 64 : cluster [DBG] pgmap v42: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 578 op/s 2023-10-01T10:20:44.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:44 smithi078 bash[27336]: cluster 2023-10-01T10:20:43.019930+0000 mgr.x (mgr.24697) 64 : cluster [DBG] pgmap v42: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 578 op/s 2023-10-01T10:20:44.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:44 smithi173 bash[27612]: cluster 2023-10-01T10:20:43.019930+0000 mgr.x (mgr.24697) 64 : cluster [DBG] pgmap v42: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 578 op/s 2023-10-01T10:20:45.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:45 smithi078 bash[33821]: audit 2023-10-01T10:20:44.319695+0000 mon.c (mon.2) 118 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:20:45.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:45 smithi078 bash[33821]: audit 2023-10-01T10:20:44.625138+0000 mon.a (mon.0) 860 : audit [DBG] from='client.? 172.21.15.78:0/2277785506' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:45.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:45 smithi078 bash[27336]: audit 2023-10-01T10:20:44.319695+0000 mon.c (mon.2) 118 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:20:45.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:45 smithi078 bash[27336]: audit 2023-10-01T10:20:44.625138+0000 mon.a (mon.0) 860 : audit [DBG] from='client.? 172.21.15.78:0/2277785506' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:45.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:45 smithi173 bash[27612]: audit 2023-10-01T10:20:44.319695+0000 mon.c (mon.2) 118 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:20:45.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:45 smithi173 bash[27612]: audit 2023-10-01T10:20:44.625138+0000 mon.a (mon.0) 860 : audit [DBG] from='client.? 172.21.15.78:0/2277785506' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:46.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:46 smithi173 bash[27612]: cluster 2023-10-01T10:20:45.020443+0000 mgr.x (mgr.24697) 65 : cluster [DBG] pgmap v43: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 578 op/s 2023-10-01T10:20:46.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:46 smithi078 bash[33821]: cluster 2023-10-01T10:20:45.020443+0000 mgr.x (mgr.24697) 65 : cluster [DBG] pgmap v43: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 578 op/s 2023-10-01T10:20:46.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:46 smithi078 bash[27336]: cluster 2023-10-01T10:20:45.020443+0000 mgr.x (mgr.24697) 65 : cluster [DBG] pgmap v43: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 578 op/s 2023-10-01T10:20:47.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:47 smithi173 bash[27612]: audit 2023-10-01T10:20:47.256188+0000 mon.c (mon.2) 119 : audit [DBG] from='client.? 172.21.15.78:0/1862988035' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:47.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:47 smithi078 bash[27336]: audit 2023-10-01T10:20:47.256188+0000 mon.c (mon.2) 119 : audit [DBG] from='client.? 172.21.15.78:0/1862988035' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:47.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:47 smithi078 bash[33821]: audit 2023-10-01T10:20:47.256188+0000 mon.c (mon.2) 119 : audit [DBG] from='client.? 172.21.15.78:0/1862988035' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:48.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:48 smithi173 bash[27612]: cluster 2023-10-01T10:20:47.021152+0000 mgr.x (mgr.24697) 66 : cluster [DBG] pgmap v44: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 833 op/s 2023-10-01T10:20:48.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:48 smithi078 bash[27336]: cluster 2023-10-01T10:20:47.021152+0000 mgr.x (mgr.24697) 66 : cluster [DBG] pgmap v44: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 833 op/s 2023-10-01T10:20:48.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:48 smithi078 bash[33821]: cluster 2023-10-01T10:20:47.021152+0000 mgr.x (mgr.24697) 66 : cluster [DBG] pgmap v44: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 833 op/s 2023-10-01T10:20:49.119 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:20:48 smithi173 bash[52702]: ts=2023-10-01T10:20:48.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"smithi173:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"smithi173.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-01T10:20:50.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:50 smithi173 bash[27612]: cluster 2023-10-01T10:20:49.022465+0000 mgr.x (mgr.24697) 67 : cluster [DBG] pgmap v45: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 540 op/s 2023-10-01T10:20:50.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:50 smithi173 bash[27612]: audit 2023-10-01T10:20:49.890230+0000 mon.c (mon.2) 120 : audit [DBG] from='client.? 172.21.15.78:0/2050536861' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:50.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:50 smithi078 bash[33821]: cluster 2023-10-01T10:20:49.022465+0000 mgr.x (mgr.24697) 67 : cluster [DBG] pgmap v45: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 540 op/s 2023-10-01T10:20:50.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:50 smithi078 bash[33821]: audit 2023-10-01T10:20:49.890230+0000 mon.c (mon.2) 120 : audit [DBG] from='client.? 172.21.15.78:0/2050536861' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:50.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:50 smithi078 bash[27336]: cluster 2023-10-01T10:20:49.022465+0000 mgr.x (mgr.24697) 67 : cluster [DBG] pgmap v45: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 540 op/s 2023-10-01T10:20:50.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:50 smithi078 bash[27336]: audit 2023-10-01T10:20:49.890230+0000 mon.c (mon.2) 120 : audit [DBG] from='client.? 172.21.15.78:0/2050536861' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:52.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:52 smithi173 bash[27612]: cluster 2023-10-01T10:20:51.023216+0000 mgr.x (mgr.24697) 68 : cluster [DBG] pgmap v46: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 795 op/s 2023-10-01T10:20:52.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:52 smithi078 bash[27336]: cluster 2023-10-01T10:20:51.023216+0000 mgr.x (mgr.24697) 68 : cluster [DBG] pgmap v46: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 795 op/s 2023-10-01T10:20:52.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:52 smithi078 bash[33821]: cluster 2023-10-01T10:20:51.023216+0000 mgr.x (mgr.24697) 68 : cluster [DBG] pgmap v46: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 795 op/s 2023-10-01T10:20:53.119 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:20:52 smithi173 bash[50431]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:20:52] "GET /metrics HTTP/1.1" 200 36077 "" "Prometheus/2.33.4" 2023-10-01T10:20:53.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:53 smithi078 bash[27336]: audit 2023-10-01T10:20:52.543902+0000 mon.b (mon.1) 40 : audit [DBG] from='client.? 172.21.15.78:0/2059361453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:53.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:53 smithi078 bash[33821]: audit 2023-10-01T10:20:52.543902+0000 mon.b (mon.1) 40 : audit [DBG] from='client.? 172.21.15.78:0/2059361453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:53.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:53 smithi173 bash[27612]: audit 2023-10-01T10:20:52.543902+0000 mon.b (mon.1) 40 : audit [DBG] from='client.? 172.21.15.78:0/2059361453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:54.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:54 smithi078 bash[27336]: cluster 2023-10-01T10:20:53.023707+0000 mgr.x (mgr.24697) 69 : cluster [DBG] pgmap v47: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 509 op/s 2023-10-01T10:20:54.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:54 smithi078 bash[33821]: cluster 2023-10-01T10:20:53.023707+0000 mgr.x (mgr.24697) 69 : cluster [DBG] pgmap v47: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 509 op/s 2023-10-01T10:20:54.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:54 smithi173 bash[27612]: cluster 2023-10-01T10:20:53.023707+0000 mgr.x (mgr.24697) 69 : cluster [DBG] pgmap v47: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 509 op/s 2023-10-01T10:20:55.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:55 smithi078 bash[33821]: cluster 2023-10-01T10:20:55.024142+0000 mgr.x (mgr.24697) 70 : cluster [DBG] pgmap v48: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 509 op/s 2023-10-01T10:20:55.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:55 smithi078 bash[33821]: audit 2023-10-01T10:20:55.225405+0000 mon.a (mon.0) 861 : audit [DBG] from='client.? 172.21.15.78:0/2316266884' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:55.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:55 smithi078 bash[27336]: cluster 2023-10-01T10:20:55.024142+0000 mgr.x (mgr.24697) 70 : cluster [DBG] pgmap v48: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 509 op/s 2023-10-01T10:20:55.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:55 smithi078 bash[27336]: audit 2023-10-01T10:20:55.225405+0000 mon.a (mon.0) 861 : audit [DBG] from='client.? 172.21.15.78:0/2316266884' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:55.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:55 smithi173 bash[27612]: cluster 2023-10-01T10:20:55.024142+0000 mgr.x (mgr.24697) 70 : cluster [DBG] pgmap v48: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 509 op/s 2023-10-01T10:20:55.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:55 smithi173 bash[27612]: audit 2023-10-01T10:20:55.225405+0000 mon.a (mon.0) 861 : audit [DBG] from='client.? 172.21.15.78:0/2316266884' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:58.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:58 smithi173 bash[27612]: cluster 2023-10-01T10:20:57.024865+0000 mgr.x (mgr.24697) 71 : cluster [DBG] pgmap v49: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 799 op/s 2023-10-01T10:20:58.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:20:58 smithi173 bash[27612]: audit 2023-10-01T10:20:57.895810+0000 mon.a (mon.0) 862 : audit [DBG] from='client.? 172.21.15.78:0/3335572054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:58.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:58 smithi078 bash[33821]: cluster 2023-10-01T10:20:57.024865+0000 mgr.x (mgr.24697) 71 : cluster [DBG] pgmap v49: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 799 op/s 2023-10-01T10:20:58.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:20:58 smithi078 bash[33821]: audit 2023-10-01T10:20:57.895810+0000 mon.a (mon.0) 862 : audit [DBG] from='client.? 172.21.15.78:0/3335572054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:58.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:58 smithi078 bash[27336]: cluster 2023-10-01T10:20:57.024865+0000 mgr.x (mgr.24697) 71 : cluster [DBG] pgmap v49: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 799 op/s 2023-10-01T10:20:58.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:20:58 smithi078 bash[27336]: audit 2023-10-01T10:20:57.895810+0000 mon.a (mon.0) 862 : audit [DBG] from='client.? 172.21.15.78:0/3335572054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:20:58.869 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:20:58 smithi173 bash[52702]: ts=2023-10-01T10:20:58.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"smithi173:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"smithi173.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-01T10:21:00.417 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:00 smithi173 bash[27612]: cluster 2023-10-01T10:20:59.025281+0000 mgr.x (mgr.24697) 72 : cluster [DBG] pgmap v50: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 543 op/s 2023-10-01T10:21:00.418 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:00 smithi173 bash[27612]: audit 2023-10-01T10:20:59.320026+0000 mon.c (mon.2) 121 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:21:00.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:00 smithi078 bash[33821]: cluster 2023-10-01T10:20:59.025281+0000 mgr.x (mgr.24697) 72 : cluster [DBG] pgmap v50: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 543 op/s 2023-10-01T10:21:00.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:00 smithi078 bash[33821]: audit 2023-10-01T10:20:59.320026+0000 mon.c (mon.2) 121 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:21:00.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:00 smithi078 bash[27336]: cluster 2023-10-01T10:20:59.025281+0000 mgr.x (mgr.24697) 72 : cluster [DBG] pgmap v50: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 543 op/s 2023-10-01T10:21:00.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:00 smithi078 bash[27336]: audit 2023-10-01T10:20:59.320026+0000 mon.c (mon.2) 121 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:21:01.515 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:21:01 smithi173 systemd[1]: Stopping Ceph grafana.a for d8016480-6041-11ee-8db5-212e2dc638e7... 2023-10-01T10:21:01.515 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:21:01 smithi173 bash[53132]: Error response from daemon: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-grafana.a 2023-10-01T10:21:01.515 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:21:01 smithi173 bash[47537]: t=2023-10-01T10:21:01+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2023-10-01T10:21:01.515 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:01 smithi173 bash[27612]: cluster 2023-10-01T10:21:00.135932+0000 mon.a (mon.0) 863 : cluster [DBG] mgrmap e24: x(active, since 91s), standbys: y 2023-10-01T10:21:01.516 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:01 smithi173 bash[27612]: audit 2023-10-01T10:21:00.600751+0000 mon.a (mon.0) 864 : audit [DBG] from='client.? 172.21.15.78:0/869328172' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:01.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:01 smithi078 bash[33821]: cluster 2023-10-01T10:21:00.135932+0000 mon.a (mon.0) 863 : cluster [DBG] mgrmap e24: x(active, since 91s), standbys: y 2023-10-01T10:21:01.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:01 smithi078 bash[33821]: audit 2023-10-01T10:21:00.600751+0000 mon.a (mon.0) 864 : audit [DBG] from='client.? 172.21.15.78:0/869328172' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:01.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:01 smithi078 bash[27336]: cluster 2023-10-01T10:21:00.135932+0000 mon.a (mon.0) 863 : cluster [DBG] mgrmap e24: x(active, since 91s), standbys: y 2023-10-01T10:21:01.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:01 smithi078 bash[27336]: audit 2023-10-01T10:21:00.600751+0000 mon.a (mon.0) 864 : audit [DBG] from='client.? 172.21.15.78:0/869328172' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:02.472 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:02 smithi173 bash[27612]: cluster 2023-10-01T10:21:01.026090+0000 mgr.x (mgr.24697) 73 : cluster [DBG] pgmap v51: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 255 B/s wr, 847 op/s 2023-10-01T10:21:02.472 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:21:02 smithi173 bash[53162]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-grafana-a 2023-10-01T10:21:02.472 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:21:02 smithi173 bash[53218]: Error response from daemon: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-grafana-a 2023-10-01T10:21:02.473 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:21:02 smithi173 systemd[1]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7@grafana.a.service: Succeeded. 2023-10-01T10:21:02.473 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:21:02 smithi173 systemd[1]: Stopped Ceph grafana.a for d8016480-6041-11ee-8db5-212e2dc638e7. 2023-10-01T10:21:02.473 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:21:02 smithi173 systemd[1]: Started Ceph grafana.a for d8016480-6041-11ee-8db5-212e2dc638e7. 2023-10-01T10:21:02.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:02 smithi078 bash[33821]: cluster 2023-10-01T10:21:01.026090+0000 mgr.x (mgr.24697) 73 : cluster [DBG] pgmap v51: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 255 B/s wr, 847 op/s 2023-10-01T10:21:02.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:02 smithi078 bash[27336]: cluster 2023-10-01T10:21:01.026090+0000 mgr.x (mgr.24697) 73 : cluster [DBG] pgmap v51: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 255 B/s wr, 847 op/s 2023-10-01T10:21:02.869 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:21:02 smithi173 bash[50431]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:21:02] "GET /metrics HTTP/1.1" 200 36061 "" "Prometheus/2.33.4" 2023-10-01T10:21:03.745 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:03 smithi173 bash[27612]: audit 2023-10-01T10:21:02.429933+0000 mon.a (mon.0) 865 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:21:03.745 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:03 smithi173 bash[27612]: audit 2023-10-01T10:21:02.443262+0000 mon.a (mon.0) 866 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:21:03.746 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:03 smithi173 bash[27612]: audit 2023-10-01T10:21:02.478954+0000 mon.c (mon.2) 122 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-01T10:21:03.746 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:03 smithi173 bash[27612]: audit 2023-10-01T10:21:02.481864+0000 mgr.x (mgr.24697) 74 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-01T10:21:03.746 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:03 smithi173 bash[27612]: audit 2023-10-01T10:21:02.485504+0000 mon.c (mon.2) 123 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-01T10:21:03.746 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:03 smithi173 bash[27612]: audit 2023-10-01T10:21:02.488333+0000 mgr.x (mgr.24697) 75 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-01T10:21:03.746 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:03 smithi173 bash[27612]: audit 2023-10-01T10:21:02.491940+0000 mon.c (mon.2) 124 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-01T10:21:03.746 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:03 smithi173 bash[27612]: audit 2023-10-01T10:21:02.494704+0000 mgr.x (mgr.24697) 76 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-01T10:21:03.746 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:03 smithi173 bash[27612]: audit 2023-10-01T10:21:02.504284+0000 mon.c (mon.2) 125 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:21:03.746 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:03 smithi173 bash[27612]: cephadm 2023-10-01T10:21:02.510561+0000 mgr.x (mgr.24697) 77 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-10-01T10:21:03.746 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:03 smithi173 bash[27612]: cephadm 2023-10-01T10:21:02.511607+0000 mgr.x (mgr.24697) 78 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-10-01T10:21:03.746 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:03 smithi173 bash[27612]: cluster 2023-10-01T10:21:03.026551+0000 mgr.x (mgr.24697) 79 : cluster [DBG] pgmap v52: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 255 B/s wr, 593 op/s 2023-10-01T10:21:03.747 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:03 smithi173 bash[27612]: audit 2023-10-01T10:21:03.294974+0000 mon.c (mon.2) 126 : audit [DBG] from='client.? 172.21.15.78:0/1701651643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:03.747 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:21:03 smithi173 bash[53242]: t=2023-10-01T10:21:03+0000 lvl=info msg="The state of unified alerting is still not defined. The decision will be made during as we run the database migrations" logger=settings 2023-10-01T10:21:03.747 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:21:03 smithi173 bash[53242]: t=2023-10-01T10:21:03+0000 lvl=warn msg="falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled." logger=settings 2023-10-01T10:21:03.747 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:21:03 smithi173 bash[53242]: t=2023-10-01T10:21:03+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2023-10-01T10:21:03.747 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:21:03 smithi173 bash[53242]: t=2023-10-01T10:21:03+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2023-10-01T10:21:03.747 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:21:03 smithi173 bash[53242]: t=2023-10-01T10:21:03+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2023-10-01T10:21:03.747 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:21:03 smithi173 bash[53242]: t=2023-10-01T10:21:03+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2023-10-01T10:21:03.747 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:21:03 smithi173 bash[53242]: t=2023-10-01T10:21:03+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2023-10-01T10:21:03.748 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:21:03 smithi173 bash[53242]: t=2023-10-01T10:21:03+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2023-10-01T10:21:03.748 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:21:03 smithi173 bash[53242]: t=2023-10-01T10:21:03+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2023-10-01T10:21:03.748 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:21:03 smithi173 bash[53242]: t=2023-10-01T10:21:03+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2023-10-01T10:21:03.748 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:21:03 smithi173 bash[53242]: t=2023-10-01T10:21:03+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2023-10-01T10:21:03.748 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:21:03 smithi173 bash[53242]: t=2023-10-01T10:21:03+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2023-10-01T10:21:03.748 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:21:03 smithi173 bash[53242]: t=2023-10-01T10:21:03+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2023-10-01T10:21:03.748 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:21:03 smithi173 bash[53242]: t=2023-10-01T10:21:03+0000 lvl=info msg="App mode production" logger=settings 2023-10-01T10:21:03.748 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:21:03 smithi173 bash[53242]: t=2023-10-01T10:21:03+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2023-10-01T10:21:03.748 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:21:03 smithi173 bash[53242]: t=2023-10-01T10:21:03+0000 lvl=warn msg="SQLite database file has broader permissions than it should" logger=sqlstore path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2023-10-01T10:21:03.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:03 smithi078 bash[33821]: audit 2023-10-01T10:21:02.429933+0000 mon.a (mon.0) 865 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:21:03.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:03 smithi078 bash[33821]: audit 2023-10-01T10:21:02.443262+0000 mon.a (mon.0) 866 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:21:03.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:03 smithi078 bash[33821]: audit 2023-10-01T10:21:02.478954+0000 mon.c (mon.2) 122 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-01T10:21:03.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:03 smithi078 bash[33821]: audit 2023-10-01T10:21:02.481864+0000 mgr.x (mgr.24697) 74 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-01T10:21:03.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:03 smithi078 bash[33821]: audit 2023-10-01T10:21:02.485504+0000 mon.c (mon.2) 123 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-01T10:21:03.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:03 smithi078 bash[33821]: audit 2023-10-01T10:21:02.488333+0000 mgr.x (mgr.24697) 75 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-01T10:21:03.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:03 smithi078 bash[33821]: audit 2023-10-01T10:21:02.491940+0000 mon.c (mon.2) 124 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-01T10:21:03.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:03 smithi078 bash[33821]: audit 2023-10-01T10:21:02.494704+0000 mgr.x (mgr.24697) 76 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-01T10:21:03.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:03 smithi078 bash[33821]: audit 2023-10-01T10:21:02.504284+0000 mon.c (mon.2) 125 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:21:03.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:03 smithi078 bash[33821]: cephadm 2023-10-01T10:21:02.510561+0000 mgr.x (mgr.24697) 77 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-10-01T10:21:03.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:03 smithi078 bash[33821]: cephadm 2023-10-01T10:21:02.511607+0000 mgr.x (mgr.24697) 78 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-10-01T10:21:03.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:03 smithi078 bash[33821]: cluster 2023-10-01T10:21:03.026551+0000 mgr.x (mgr.24697) 79 : cluster [DBG] pgmap v52: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 255 B/s wr, 593 op/s 2023-10-01T10:21:03.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:03 smithi078 bash[33821]: audit 2023-10-01T10:21:03.294974+0000 mon.c (mon.2) 126 : audit [DBG] from='client.? 172.21.15.78:0/1701651643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:03.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:03 smithi078 bash[27336]: audit 2023-10-01T10:21:02.429933+0000 mon.a (mon.0) 865 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:21:03.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:03 smithi078 bash[27336]: audit 2023-10-01T10:21:02.443262+0000 mon.a (mon.0) 866 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:21:03.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:03 smithi078 bash[27336]: audit 2023-10-01T10:21:02.478954+0000 mon.c (mon.2) 122 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-01T10:21:03.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:03 smithi078 bash[27336]: audit 2023-10-01T10:21:02.481864+0000 mgr.x (mgr.24697) 74 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-10-01T10:21:03.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:03 smithi078 bash[27336]: audit 2023-10-01T10:21:02.485504+0000 mon.c (mon.2) 123 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-01T10:21:03.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:03 smithi078 bash[27336]: audit 2023-10-01T10:21:02.488333+0000 mgr.x (mgr.24697) 75 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-01T10:21:03.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:03 smithi078 bash[27336]: audit 2023-10-01T10:21:02.491940+0000 mon.c (mon.2) 124 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-01T10:21:03.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:03 smithi078 bash[27336]: audit 2023-10-01T10:21:02.494704+0000 mgr.x (mgr.24697) 76 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-10-01T10:21:03.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:03 smithi078 bash[27336]: audit 2023-10-01T10:21:02.504284+0000 mon.c (mon.2) 125 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:21:03.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:03 smithi078 bash[27336]: cephadm 2023-10-01T10:21:02.510561+0000 mgr.x (mgr.24697) 77 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-10-01T10:21:03.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:03 smithi078 bash[27336]: cephadm 2023-10-01T10:21:02.511607+0000 mgr.x (mgr.24697) 78 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2023-10-01T10:21:03.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:03 smithi078 bash[27336]: cluster 2023-10-01T10:21:03.026551+0000 mgr.x (mgr.24697) 79 : cluster [DBG] pgmap v52: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 255 B/s wr, 593 op/s 2023-10-01T10:21:03.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:03 smithi078 bash[27336]: audit 2023-10-01T10:21:03.294974+0000 mon.c (mon.2) 126 : audit [DBG] from='client.? 172.21.15.78:0/1701651643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:03.869 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:21:03 smithi173 bash[53242]: t=2023-10-01T10:21:03+0000 lvl=info msg="Starting DB migrations" logger=migrator 2023-10-01T10:21:03.870 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:21:03 smithi173 bash[53242]: t=2023-10-01T10:21:03+0000 lvl=info msg="migrations completed" logger=migrator performed=0 skipped=377 duration=1.029336ms 2023-10-01T10:21:03.870 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:21:03 smithi173 bash[53242]: t=2023-10-01T10:21:03+0000 lvl=info msg="Created default organization" logger=sqlstore 2023-10-01T10:21:03.870 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:21:03 smithi173 bash[53242]: t=2023-10-01T10:21:03+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2023-10-01T10:21:03.870 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:21:03 smithi173 bash[53242]: t=2023-10-01T10:21:03+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2023-10-01T10:21:04.083 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:21:03 smithi173 bash[53242]: t=2023-10-01T10:21:03+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2023-10-01T10:21:04.083 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:21:03 smithi173 bash[53242]: t=2023-10-01T10:21:03+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2023-10-01T10:21:04.083 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:21:03 smithi173 bash[53242]: t=2023-10-01T10:21:03+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2023-10-01T10:21:04.083 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:21:04 smithi173 bash[53242]: t=2023-10-01T10:21:04+0000 lvl=info msg="deleted datasource based on configuration" logger=provisioning.datasources name=Dashboard1 2023-10-01T10:21:04.369 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:21:04 smithi173 bash[53242]: t=2023-10-01T10:21:04+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2023-10-01T10:21:04.370 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:21:04 smithi173 bash[53242]: t=2023-10-01T10:21:04+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Loki uid=P8E80F9AEF21F6940 2023-10-01T10:21:04.370 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:21:04 smithi173 bash[53242]: t=2023-10-01T10:21:04+0000 lvl=info msg="warming cache for startup" logger=ngalert 2023-10-01T10:21:04.370 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:21:04 smithi173 bash[53242]: t=2023-10-01T10:21:04+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2023-10-01T10:21:04.370 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 10:21:04 smithi173 bash[53242]: t=2023-10-01T10:21:04+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2023-10-01T10:21:06.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:06 smithi173 bash[27612]: cluster 2023-10-01T10:21:05.027021+0000 mgr.x (mgr.24697) 80 : cluster [DBG] pgmap v53: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 255 B/s wr, 593 op/s 2023-10-01T10:21:06.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:06 smithi173 bash[27612]: audit 2023-10-01T10:21:05.914101+0000 mon.c (mon.2) 127 : audit [DBG] from='client.? 172.21.15.78:0/877828829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:06.491 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:06 smithi078 bash[27336]: cluster 2023-10-01T10:21:05.027021+0000 mgr.x (mgr.24697) 80 : cluster [DBG] pgmap v53: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 255 B/s wr, 593 op/s 2023-10-01T10:21:06.491 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:06 smithi078 bash[27336]: audit 2023-10-01T10:21:05.914101+0000 mon.c (mon.2) 127 : audit [DBG] from='client.? 172.21.15.78:0/877828829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:06.491 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:06 smithi078 bash[33821]: cluster 2023-10-01T10:21:05.027021+0000 mgr.x (mgr.24697) 80 : cluster [DBG] pgmap v53: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 255 B/s wr, 593 op/s 2023-10-01T10:21:06.491 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:06 smithi078 bash[33821]: audit 2023-10-01T10:21:05.914101+0000 mon.c (mon.2) 127 : audit [DBG] from='client.? 172.21.15.78:0/877828829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:08.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:08 smithi173 bash[27612]: cluster 2023-10-01T10:21:07.027752+0000 mgr.x (mgr.24697) 81 : cluster [DBG] pgmap v54: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 255 B/s wr, 886 op/s 2023-10-01T10:21:08.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:08 smithi173 bash[27612]: audit 2023-10-01T10:21:07.804309+0000 mon.a (mon.0) 867 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:21:08.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:08 smithi173 bash[27612]: audit 2023-10-01T10:21:07.806837+0000 mon.c (mon.2) 128 : audit [INF] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-01T10:21:08.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:08 smithi173 bash[27612]: audit 2023-10-01T10:21:07.807301+0000 mon.a (mon.0) 868 : audit [INF] from='mgr.24697 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-01T10:21:08.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:08 smithi173 bash[27612]: audit 2023-10-01T10:21:07.814100+0000 mon.c (mon.2) 129 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-01T10:21:08.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:08 smithi173 bash[27612]: audit 2023-10-01T10:21:07.815514+0000 mon.c (mon.2) 130 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:21:08.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:08 smithi078 bash[27336]: cluster 2023-10-01T10:21:07.027752+0000 mgr.x (mgr.24697) 81 : cluster [DBG] pgmap v54: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 255 B/s wr, 886 op/s 2023-10-01T10:21:08.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:08 smithi078 bash[27336]: audit 2023-10-01T10:21:07.804309+0000 mon.a (mon.0) 867 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:21:08.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:08 smithi078 bash[27336]: audit 2023-10-01T10:21:07.806837+0000 mon.c (mon.2) 128 : audit [INF] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-01T10:21:08.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:08 smithi078 bash[27336]: audit 2023-10-01T10:21:07.807301+0000 mon.a (mon.0) 868 : audit [INF] from='mgr.24697 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-01T10:21:08.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:08 smithi078 bash[27336]: audit 2023-10-01T10:21:07.814100+0000 mon.c (mon.2) 129 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-01T10:21:08.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:08 smithi078 bash[27336]: audit 2023-10-01T10:21:07.815514+0000 mon.c (mon.2) 130 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:21:08.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:08 smithi078 bash[33821]: cluster 2023-10-01T10:21:07.027752+0000 mgr.x (mgr.24697) 81 : cluster [DBG] pgmap v54: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 255 B/s wr, 886 op/s 2023-10-01T10:21:08.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:08 smithi078 bash[33821]: audit 2023-10-01T10:21:07.804309+0000 mon.a (mon.0) 867 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:21:08.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:08 smithi078 bash[33821]: audit 2023-10-01T10:21:07.806837+0000 mon.c (mon.2) 128 : audit [INF] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-01T10:21:08.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:08 smithi078 bash[33821]: audit 2023-10-01T10:21:07.807301+0000 mon.a (mon.0) 868 : audit [INF] from='mgr.24697 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-01T10:21:08.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:08 smithi078 bash[33821]: audit 2023-10-01T10:21:07.814100+0000 mon.c (mon.2) 129 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-01T10:21:08.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:08 smithi078 bash[33821]: audit 2023-10-01T10:21:07.815514+0000 mon.c (mon.2) 130 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:21:09.119 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:21:08 smithi173 bash[52702]: ts=2023-10-01T10:21:08.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"smithi173:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"smithi173.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-01T10:21:09.518 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:09 smithi078 bash[33821]: cephadm 2023-10-01T10:21:07.797945+0000 mgr.x (mgr.24697) 82 : cephadm [INF] Upgrade: Updating mgr.y 2023-10-01T10:21:09.518 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:09 smithi078 bash[33821]: cephadm 2023-10-01T10:21:07.819165+0000 mgr.x (mgr.24697) 83 : cephadm [INF] Deploying daemon mgr.y on smithi078 2023-10-01T10:21:09.519 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:09 smithi078 bash[33821]: audit 2023-10-01T10:21:08.649664+0000 mon.c (mon.2) 131 : audit [DBG] from='client.? 172.21.15.78:0/201258164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:09.519 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:09 smithi078 bash[27336]: cephadm 2023-10-01T10:21:07.797945+0000 mgr.x (mgr.24697) 82 : cephadm [INF] Upgrade: Updating mgr.y 2023-10-01T10:21:09.519 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:09 smithi078 bash[27336]: cephadm 2023-10-01T10:21:07.819165+0000 mgr.x (mgr.24697) 83 : cephadm [INF] Deploying daemon mgr.y on smithi078 2023-10-01T10:21:09.520 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:09 smithi078 bash[27336]: audit 2023-10-01T10:21:08.649664+0000 mon.c (mon.2) 131 : audit [DBG] from='client.? 172.21.15.78:0/201258164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:09.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:09 smithi173 bash[27612]: cephadm 2023-10-01T10:21:07.797945+0000 mgr.x (mgr.24697) 82 : cephadm [INF] Upgrade: Updating mgr.y 2023-10-01T10:21:09.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:09 smithi173 bash[27612]: cephadm 2023-10-01T10:21:07.819165+0000 mgr.x (mgr.24697) 83 : cephadm [INF] Deploying daemon mgr.y on smithi078 2023-10-01T10:21:09.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:09 smithi173 bash[27612]: audit 2023-10-01T10:21:08.649664+0000 mon.c (mon.2) 131 : audit [DBG] from='client.? 172.21.15.78:0/201258164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:10.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:10 smithi078 bash[27336]: cluster 2023-10-01T10:21:09.028293+0000 mgr.x (mgr.24697) 84 : cluster [DBG] pgmap v55: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 255 B/s wr, 596 op/s 2023-10-01T10:21:10.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:10 smithi078 bash[33821]: cluster 2023-10-01T10:21:09.028293+0000 mgr.x (mgr.24697) 84 : cluster [DBG] pgmap v55: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 255 B/s wr, 596 op/s 2023-10-01T10:21:10.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:10 smithi173 bash[27612]: cluster 2023-10-01T10:21:09.028293+0000 mgr.x (mgr.24697) 84 : cluster [DBG] pgmap v55: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 255 B/s wr, 596 op/s 2023-10-01T10:21:10.901 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:21:10 smithi078 systemd[1]: Stopping Ceph mgr.y for d8016480-6041-11ee-8db5-212e2dc638e7... 2023-10-01T10:21:12.083 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:21:11 smithi078 bash[71102]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-mgr-y 2023-10-01T10:21:12.083 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:21:11 smithi078 systemd[1]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7@mgr.y.service: Main process exited, code=exited, status=143/n/a 2023-10-01T10:21:12.083 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:21:12 smithi078 bash[71162]: Error response from daemon: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-mgr-y 2023-10-01T10:21:12.083 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:21:12 smithi078 systemd[1]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7@mgr.y.service: Failed with result 'exit-code'. 2023-10-01T10:21:12.083 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:21:12 smithi078 systemd[1]: Stopped Ceph mgr.y for d8016480-6041-11ee-8db5-212e2dc638e7. 2023-10-01T10:21:12.401 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:12 smithi078 bash[27336]: cluster 2023-10-01T10:21:11.029112+0000 mgr.x (mgr.24697) 85 : cluster [DBG] pgmap v56: 89 pgs: 89 active+clean; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 255 B/s wr, 882 op/s 2023-10-01T10:21:12.402 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:12 smithi078 bash[27336]: audit 2023-10-01T10:21:11.383279+0000 mon.b (mon.1) 41 : audit [DBG] from='client.? 172.21.15.78:0/247202427' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:12.402 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:21:12 smithi078 systemd[1]: Started Ceph mgr.y for d8016480-6041-11ee-8db5-212e2dc638e7. 2023-10-01T10:21:12.402 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:12 smithi078 bash[33821]: cluster 2023-10-01T10:21:11.029112+0000 mgr.x (mgr.24697) 85 : cluster [DBG] pgmap v56: 89 pgs: 89 active+clean; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 255 B/s wr, 882 op/s 2023-10-01T10:21:12.402 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:12 smithi078 bash[33821]: audit 2023-10-01T10:21:11.383279+0000 mon.b (mon.1) 41 : audit [DBG] from='client.? 172.21.15.78:0/247202427' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:12.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:12 smithi173 bash[27612]: cluster 2023-10-01T10:21:11.029112+0000 mgr.x (mgr.24697) 85 : cluster [DBG] pgmap v56: 89 pgs: 89 active+clean; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 255 B/s wr, 882 op/s 2023-10-01T10:21:12.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:12 smithi173 bash[27612]: audit 2023-10-01T10:21:11.383279+0000 mon.b (mon.1) 41 : audit [DBG] from='client.? 172.21.15.78:0/247202427' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:13.120 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:21:12 smithi173 bash[50431]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:21:12] "GET /metrics HTTP/1.1" 200 36061 "" "Prometheus/2.33.4" 2023-10-01T10:21:13.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:13 smithi078 bash[27336]: audit 2023-10-01T10:21:12.365250+0000 mon.a (mon.0) 869 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:21:13.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:13 smithi078 bash[27336]: audit 2023-10-01T10:21:12.377626+0000 mon.a (mon.0) 870 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:21:13.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:13 smithi078 bash[27336]: audit 2023-10-01T10:21:12.401451+0000 mon.c (mon.2) 132 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:21:13.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:13 smithi078 bash[33821]: audit 2023-10-01T10:21:12.365250+0000 mon.a (mon.0) 869 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:21:13.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:13 smithi078 bash[33821]: audit 2023-10-01T10:21:12.377626+0000 mon.a (mon.0) 870 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:21:13.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:13 smithi078 bash[33821]: audit 2023-10-01T10:21:12.401451+0000 mon.c (mon.2) 132 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:21:13.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:13 smithi173 bash[27612]: audit 2023-10-01T10:21:12.365250+0000 mon.a (mon.0) 869 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:21:13.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:13 smithi173 bash[27612]: audit 2023-10-01T10:21:12.377626+0000 mon.a (mon.0) 870 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:21:13.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:13 smithi173 bash[27612]: audit 2023-10-01T10:21:12.401451+0000 mon.c (mon.2) 132 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:21:14.333 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:21:13 smithi078 bash[71214]: debug 2023-10-01T10:21:13.847+0000 7fe217d09200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-01T10:21:14.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:14 smithi173 bash[27612]: cluster 2023-10-01T10:21:13.029559+0000 mgr.x (mgr.24697) 86 : cluster [DBG] pgmap v57: 89 pgs: 89 active+clean; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 578 op/s 2023-10-01T10:21:14.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:14 smithi173 bash[27612]: audit 2023-10-01T10:21:14.109765+0000 mon.a (mon.0) 871 : audit [DBG] from='client.? 172.21.15.78:0/1185857929' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:14.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:14 smithi173 bash[27612]: audit 2023-10-01T10:21:14.320627+0000 mon.c (mon.2) 133 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:21:14.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:14 smithi078 bash[33821]: cluster 2023-10-01T10:21:13.029559+0000 mgr.x (mgr.24697) 86 : cluster [DBG] pgmap v57: 89 pgs: 89 active+clean; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 578 op/s 2023-10-01T10:21:14.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:14 smithi078 bash[33821]: audit 2023-10-01T10:21:14.109765+0000 mon.a (mon.0) 871 : audit [DBG] from='client.? 172.21.15.78:0/1185857929' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:14.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:14 smithi078 bash[33821]: audit 2023-10-01T10:21:14.320627+0000 mon.c (mon.2) 133 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:21:14.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:14 smithi078 bash[27336]: cluster 2023-10-01T10:21:13.029559+0000 mgr.x (mgr.24697) 86 : cluster [DBG] pgmap v57: 89 pgs: 89 active+clean; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 578 op/s 2023-10-01T10:21:14.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:14 smithi078 bash[27336]: audit 2023-10-01T10:21:14.109765+0000 mon.a (mon.0) 871 : audit [DBG] from='client.? 172.21.15.78:0/1185857929' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:14.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:14 smithi078 bash[27336]: audit 2023-10-01T10:21:14.320627+0000 mon.c (mon.2) 133 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:21:15.518 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:15 smithi078 bash[27336]: cluster 2023-10-01T10:21:15.029997+0000 mgr.x (mgr.24697) 87 : cluster [DBG] pgmap v58: 89 pgs: 89 active+clean; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 578 op/s 2023-10-01T10:21:15.518 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:15 smithi078 bash[33821]: cluster 2023-10-01T10:21:15.029997+0000 mgr.x (mgr.24697) 87 : cluster [DBG] pgmap v58: 89 pgs: 89 active+clean; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 578 op/s 2023-10-01T10:21:15.519 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:21:15 smithi078 bash[71214]: debug 2023-10-01T10:21:15.199+0000 7fe217d09200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-01T10:21:15.519 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:21:15 smithi078 bash[71214]: debug 2023-10-01T10:21:15.355+0000 7fe217d09200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-01T10:21:15.519 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:21:15 smithi078 bash[71214]: debug 2023-10-01T10:21:15.511+0000 7fe217d09200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-01T10:21:15.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:15 smithi173 bash[27612]: cluster 2023-10-01T10:21:15.029997+0000 mgr.x (mgr.24697) 87 : cluster [DBG] pgmap v58: 89 pgs: 89 active+clean; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 578 op/s 2023-10-01T10:21:16.333 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:21:15 smithi078 bash[71214]: debug 2023-10-01T10:21:15.911+0000 7fe217d09200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-01T10:21:16.733 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:21:16 smithi078 bash[71214]: debug 2023-10-01T10:21:16.475+0000 7fe217d09200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-01T10:21:17.072 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:21:16 smithi078 bash[71214]: debug 2023-10-01T10:21:16.727+0000 7fe217d09200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-01T10:21:17.072 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:21:17 smithi078 bash[71214]: debug 2023-10-01T10:21:17.063+0000 7fe217d09200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-01T10:21:17.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:17 smithi078 bash[33821]: audit 2023-10-01T10:21:16.897568+0000 mon.c (mon.2) 134 : audit [DBG] from='client.? 172.21.15.78:0/909355386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:17.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:17 smithi078 bash[27336]: audit 2023-10-01T10:21:16.897568+0000 mon.c (mon.2) 134 : audit [DBG] from='client.? 172.21.15.78:0/909355386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:17.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:17 smithi173 bash[27612]: audit 2023-10-01T10:21:16.897568+0000 mon.c (mon.2) 134 : audit [DBG] from='client.? 172.21.15.78:0/909355386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:18.550 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:18 smithi078 bash[27336]: cluster 2023-10-01T10:21:17.030745+0000 mgr.x (mgr.24697) 88 : cluster [DBG] pgmap v59: 89 pgs: 89 active+clean; 2.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 863 op/s 2023-10-01T10:21:18.551 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:18 smithi078 bash[33821]: cluster 2023-10-01T10:21:17.030745+0000 mgr.x (mgr.24697) 88 : cluster [DBG] pgmap v59: 89 pgs: 89 active+clean; 2.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 863 op/s 2023-10-01T10:21:18.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:18 smithi173 bash[27612]: cluster 2023-10-01T10:21:17.030745+0000 mgr.x (mgr.24697) 88 : cluster [DBG] pgmap v59: 89 pgs: 89 active+clean; 2.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 863 op/s 2023-10-01T10:21:18.833 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:21:18 smithi078 bash[71214]: debug 2023-10-01T10:21:18.543+0000 7fe217d09200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-01T10:21:18.833 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:21:18 smithi078 bash[71214]: debug 2023-10-01T10:21:18.707+0000 7fe217d09200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-01T10:21:19.120 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:21:18 smithi173 bash[52702]: ts=2023-10-01T10:21:18.658Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"smithi173:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"smithi173.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-01T10:21:19.333 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:21:18 smithi078 bash[71214]: debug 2023-10-01T10:21:18.875+0000 7fe217d09200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-01T10:21:19.943 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:19 smithi078 bash[27336]: audit 2023-10-01T10:21:18.765940+0000 mon.a (mon.0) 872 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:21:19.943 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:19 smithi078 bash[27336]: audit 2023-10-01T10:21:18.781105+0000 mon.a (mon.0) 873 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:21:19.943 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:19 smithi078 bash[27336]: audit 2023-10-01T10:21:19.028280+0000 mon.a (mon.0) 874 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:21:19.943 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:19 smithi078 bash[27336]: cluster 2023-10-01T10:21:19.033571+0000 mgr.x (mgr.24697) 89 : cluster [DBG] pgmap v60: 89 pgs: 89 active+clean; 2.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 569 op/s 2023-10-01T10:21:19.943 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:19 smithi078 bash[27336]: audit 2023-10-01T10:21:19.046193+0000 mon.a (mon.0) 875 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:21:19.943 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:19 smithi078 bash[27336]: audit 2023-10-01T10:21:19.622890+0000 mon.a (mon.0) 876 : audit [DBG] from='client.? 172.21.15.78:0/3906441194' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:19.944 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:21:19 smithi078 bash[71214]: debug 2023-10-01T10:21:19.595+0000 7fe217d09200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-01T10:21:19.944 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:21:19 smithi078 bash[71214]: debug 2023-10-01T10:21:19.935+0000 7fe217d09200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-01T10:21:19.944 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:19 smithi078 bash[33821]: audit 2023-10-01T10:21:18.765940+0000 mon.a (mon.0) 872 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:21:19.944 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:19 smithi078 bash[33821]: audit 2023-10-01T10:21:18.781105+0000 mon.a (mon.0) 873 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:21:19.944 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:19 smithi078 bash[33821]: audit 2023-10-01T10:21:19.028280+0000 mon.a (mon.0) 874 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:21:19.944 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:19 smithi078 bash[33821]: cluster 2023-10-01T10:21:19.033571+0000 mgr.x (mgr.24697) 89 : cluster [DBG] pgmap v60: 89 pgs: 89 active+clean; 2.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 569 op/s 2023-10-01T10:21:19.944 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:19 smithi078 bash[33821]: audit 2023-10-01T10:21:19.046193+0000 mon.a (mon.0) 875 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:21:19.944 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:19 smithi078 bash[33821]: audit 2023-10-01T10:21:19.622890+0000 mon.a (mon.0) 876 : audit [DBG] from='client.? 172.21.15.78:0/3906441194' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:20.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:19 smithi173 bash[27612]: audit 2023-10-01T10:21:18.765940+0000 mon.a (mon.0) 872 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:21:20.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:19 smithi173 bash[27612]: audit 2023-10-01T10:21:18.781105+0000 mon.a (mon.0) 873 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:21:20.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:19 smithi173 bash[27612]: audit 2023-10-01T10:21:19.028280+0000 mon.a (mon.0) 874 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:21:20.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:19 smithi173 bash[27612]: cluster 2023-10-01T10:21:19.033571+0000 mgr.x (mgr.24697) 89 : cluster [DBG] pgmap v60: 89 pgs: 89 active+clean; 2.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 569 op/s 2023-10-01T10:21:20.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:19 smithi173 bash[27612]: audit 2023-10-01T10:21:19.046193+0000 mon.a (mon.0) 875 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:21:20.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:19 smithi173 bash[27612]: audit 2023-10-01T10:21:19.622890+0000 mon.a (mon.0) 876 : audit [DBG] from='client.? 172.21.15.78:0/3906441194' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:21.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:20 smithi078 bash[27336]: audit 2023-10-01T10:21:19.808506+0000 mon.a (mon.0) 877 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:21:21.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:20 smithi078 bash[27336]: audit 2023-10-01T10:21:19.839498+0000 mon.a (mon.0) 878 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:21:21.083 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:21:20 smithi078 bash[71214]: debug 2023-10-01T10:21:20.591+0000 7fe217d09200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-01T10:21:21.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:20 smithi078 bash[33821]: audit 2023-10-01T10:21:19.808506+0000 mon.a (mon.0) 877 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:21:21.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:20 smithi078 bash[33821]: audit 2023-10-01T10:21:19.839498+0000 mon.a (mon.0) 878 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:21:21.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:20 smithi173 bash[27612]: audit 2023-10-01T10:21:19.808506+0000 mon.a (mon.0) 877 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:21:21.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:20 smithi173 bash[27612]: audit 2023-10-01T10:21:19.839498+0000 mon.a (mon.0) 878 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:21:21.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:21:21 smithi078 bash[71214]: debug 2023-10-01T10:21:21.299+0000 7fe217d09200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-01T10:21:22.381 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:22 smithi078 bash[27336]: cluster 2023-10-01T10:21:21.034313+0000 mgr.x (mgr.24697) 90 : cluster [DBG] pgmap v61: 89 pgs: 89 active+clean; 1.9 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 829 op/s 2023-10-01T10:21:22.382 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:21:22 smithi078 bash[71214]: debug 2023-10-01T10:21:22.199+0000 7fe217d09200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-01T10:21:22.382 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:21:22 smithi078 bash[71214]: debug 2023-10-01T10:21:22.375+0000 7fe217d09200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-01T10:21:22.382 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:22 smithi078 bash[33821]: cluster 2023-10-01T10:21:21.034313+0000 mgr.x (mgr.24697) 90 : cluster [DBG] pgmap v61: 89 pgs: 89 active+clean; 1.9 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 829 op/s 2023-10-01T10:21:22.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:22 smithi173 bash[27612]: cluster 2023-10-01T10:21:21.034313+0000 mgr.x (mgr.24697) 90 : cluster [DBG] pgmap v61: 89 pgs: 89 active+clean; 1.9 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 829 op/s 2023-10-01T10:21:23.119 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:21:22 smithi173 bash[50431]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:21:22] "GET /metrics HTTP/1.1" 200 36071 "" "Prometheus/2.33.4" 2023-10-01T10:21:23.145 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:23 smithi078 bash[27336]: audit 2023-10-01T10:21:22.649104+0000 mon.b (mon.1) 42 : audit [DBG] from='client.? 172.21.15.78:0/684637415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:23.146 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:21:22 smithi078 bash[71214]: debug 2023-10-01T10:21:22.875+0000 7fe217d09200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-01T10:21:23.146 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:21:23 smithi078 bash[71214]: debug 2023-10-01T10:21:23.023+0000 7fe217d09200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-01T10:21:23.414 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:21:23 smithi078 bash[71214]: debug 2023-10-01T10:21:23.239+0000 7fe217d09200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-01T10:21:23.414 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:23 smithi078 bash[33821]: audit 2023-10-01T10:21:22.649104+0000 mon.b (mon.1) 42 : audit [DBG] from='client.? 172.21.15.78:0/684637415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:23.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:23 smithi173 bash[27612]: audit 2023-10-01T10:21:22.649104+0000 mon.b (mon.1) 42 : audit [DBG] from='client.? 172.21.15.78:0/684637415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:23.833 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:21:23 smithi078 bash[71214]: debug 2023-10-01T10:21:23.406+0000 7fe217d09200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-01T10:21:23.833 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:21:23 smithi078 bash[71214]: debug 2023-10-01T10:21:23.578+0000 7fe217d09200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-01T10:21:24.147 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:21:23 smithi078 bash[71214]: debug 2023-10-01T10:21:23.858+0000 7fe217d09200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-01T10:21:24.147 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:21:24 smithi078 bash[71214]: debug 2023-10-01T10:21:24.022+0000 7fe217d09200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-01T10:21:24.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:24 smithi078 bash[27336]: cluster 2023-10-01T10:21:23.034670+0000 mgr.x (mgr.24697) 91 : cluster [DBG] pgmap v62: 89 pgs: 89 active+clean; 1.9 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 543 op/s 2023-10-01T10:21:24.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:24 smithi078 bash[33821]: cluster 2023-10-01T10:21:23.034670+0000 mgr.x (mgr.24697) 91 : cluster [DBG] pgmap v62: 89 pgs: 89 active+clean; 1.9 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 543 op/s 2023-10-01T10:21:24.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:24 smithi173 bash[27612]: cluster 2023-10-01T10:21:23.034670+0000 mgr.x (mgr.24697) 91 : cluster [DBG] pgmap v62: 89 pgs: 89 active+clean; 1.9 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 543 op/s 2023-10-01T10:21:26.083 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:21:25 smithi078 bash[71214]: debug 2023-10-01T10:21:25.766+0000 7fe217d09200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-01T10:21:26.083 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:21:25 smithi078 bash[71214]: [01/Oct/2023:10:21:25] ENGINE Bus STARTING 2023-10-01T10:21:26.083 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:21:25 smithi078 bash[71214]: CherryPy Checker: 2023-10-01T10:21:26.083 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:21:25 smithi078 bash[71214]: The Application mounted at '' has an empty config. 2023-10-01T10:21:26.083 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:21:25 smithi078 bash[71214]: [01/Oct/2023:10:21:25] ENGINE Serving on http://:::9283 2023-10-01T10:21:26.083 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:21:25 smithi078 bash[71214]: [01/Oct/2023:10:21:25] ENGINE Bus STARTED 2023-10-01T10:21:26.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:26 smithi078 bash[27336]: cluster 2023-10-01T10:21:25.035096+0000 mgr.x (mgr.24697) 92 : cluster [DBG] pgmap v63: 89 pgs: 89 active+clean; 1.9 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 543 op/s 2023-10-01T10:21:26.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:26 smithi078 bash[27336]: audit 2023-10-01T10:21:25.348034+0000 mon.b (mon.1) 43 : audit [DBG] from='client.? 172.21.15.78:0/1309061401' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:26.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:26 smithi078 bash[27336]: cluster 2023-10-01T10:21:25.792411+0000 mon.a (mon.0) 879 : cluster [DBG] Standby manager daemon y restarted 2023-10-01T10:21:26.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:26 smithi078 bash[27336]: cluster 2023-10-01T10:21:25.792533+0000 mon.a (mon.0) 880 : cluster [DBG] Standby manager daemon y started 2023-10-01T10:21:26.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:26 smithi078 bash[27336]: audit 2023-10-01T10:21:25.795624+0000 mon.c (mon.2) 135 : audit [DBG] from='mgr.? 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-01T10:21:26.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:26 smithi078 bash[27336]: audit 2023-10-01T10:21:25.796323+0000 mon.c (mon.2) 136 : audit [DBG] from='mgr.? 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-01T10:21:26.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:26 smithi078 bash[27336]: audit 2023-10-01T10:21:25.797836+0000 mon.c (mon.2) 137 : audit [DBG] from='mgr.? 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-01T10:21:26.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:26 smithi078 bash[27336]: audit 2023-10-01T10:21:25.798902+0000 mon.c (mon.2) 138 : audit [DBG] from='mgr.? 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-01T10:21:26.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:26 smithi078 bash[33821]: cluster 2023-10-01T10:21:25.035096+0000 mgr.x (mgr.24697) 92 : cluster [DBG] pgmap v63: 89 pgs: 89 active+clean; 1.9 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 543 op/s 2023-10-01T10:21:26.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:26 smithi078 bash[33821]: audit 2023-10-01T10:21:25.348034+0000 mon.b (mon.1) 43 : audit [DBG] from='client.? 172.21.15.78:0/1309061401' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:26.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:26 smithi078 bash[33821]: cluster 2023-10-01T10:21:25.792411+0000 mon.a (mon.0) 879 : cluster [DBG] Standby manager daemon y restarted 2023-10-01T10:21:26.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:26 smithi078 bash[33821]: cluster 2023-10-01T10:21:25.792533+0000 mon.a (mon.0) 880 : cluster [DBG] Standby manager daemon y started 2023-10-01T10:21:26.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:26 smithi078 bash[33821]: audit 2023-10-01T10:21:25.795624+0000 mon.c (mon.2) 135 : audit [DBG] from='mgr.? 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-01T10:21:26.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:26 smithi078 bash[33821]: audit 2023-10-01T10:21:25.796323+0000 mon.c (mon.2) 136 : audit [DBG] from='mgr.? 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-01T10:21:26.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:26 smithi078 bash[33821]: audit 2023-10-01T10:21:25.797836+0000 mon.c (mon.2) 137 : audit [DBG] from='mgr.? 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-01T10:21:26.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:26 smithi078 bash[33821]: audit 2023-10-01T10:21:25.798902+0000 mon.c (mon.2) 138 : audit [DBG] from='mgr.? 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-01T10:21:26.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:26 smithi173 bash[27612]: cluster 2023-10-01T10:21:25.035096+0000 mgr.x (mgr.24697) 92 : cluster [DBG] pgmap v63: 89 pgs: 89 active+clean; 1.9 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 543 op/s 2023-10-01T10:21:26.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:26 smithi173 bash[27612]: audit 2023-10-01T10:21:25.348034+0000 mon.b (mon.1) 43 : audit [DBG] from='client.? 172.21.15.78:0/1309061401' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:26.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:26 smithi173 bash[27612]: cluster 2023-10-01T10:21:25.792411+0000 mon.a (mon.0) 879 : cluster [DBG] Standby manager daemon y restarted 2023-10-01T10:21:26.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:26 smithi173 bash[27612]: cluster 2023-10-01T10:21:25.792533+0000 mon.a (mon.0) 880 : cluster [DBG] Standby manager daemon y started 2023-10-01T10:21:26.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:26 smithi173 bash[27612]: audit 2023-10-01T10:21:25.795624+0000 mon.c (mon.2) 135 : audit [DBG] from='mgr.? 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-10-01T10:21:26.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:26 smithi173 bash[27612]: audit 2023-10-01T10:21:25.796323+0000 mon.c (mon.2) 136 : audit [DBG] from='mgr.? 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-01T10:21:26.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:26 smithi173 bash[27612]: audit 2023-10-01T10:21:25.797836+0000 mon.c (mon.2) 137 : audit [DBG] from='mgr.? 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-10-01T10:21:26.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:26 smithi173 bash[27612]: audit 2023-10-01T10:21:25.798902+0000 mon.c (mon.2) 138 : audit [DBG] from='mgr.? 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-01T10:21:27.406 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:27 smithi078 bash[33821]: cluster 2023-10-01T10:21:26.155062+0000 mon.a (mon.0) 881 : cluster [DBG] mgrmap e25: x(active, since 117s), standbys: y 2023-10-01T10:21:27.407 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:27 smithi078 bash[27336]: cluster 2023-10-01T10:21:26.155062+0000 mon.a (mon.0) 881 : cluster [DBG] mgrmap e25: x(active, since 117s), standbys: y 2023-10-01T10:21:27.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:27 smithi173 bash[27612]: cluster 2023-10-01T10:21:26.155062+0000 mon.a (mon.0) 881 : cluster [DBG] mgrmap e25: x(active, since 117s), standbys: y 2023-10-01T10:21:28.560 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:28 smithi078 bash[27336]: cluster 2023-10-01T10:21:27.036138+0000 mgr.x (mgr.24697) 93 : cluster [DBG] pgmap v64: 89 pgs: 89 active+clean; 1.7 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 821 op/s 2023-10-01T10:21:28.560 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:28 smithi078 bash[27336]: audit 2023-10-01T10:21:28.055651+0000 mon.c (mon.2) 139 : audit [DBG] from='client.? 172.21.15.78:0/3128129433' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:28.560 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:28 smithi078 bash[33821]: cluster 2023-10-01T10:21:27.036138+0000 mgr.x (mgr.24697) 93 : cluster [DBG] pgmap v64: 89 pgs: 89 active+clean; 1.7 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 821 op/s 2023-10-01T10:21:28.560 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:28 smithi078 bash[33821]: audit 2023-10-01T10:21:28.055651+0000 mon.c (mon.2) 139 : audit [DBG] from='client.? 172.21.15.78:0/3128129433' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:28.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:28 smithi173 bash[27612]: cluster 2023-10-01T10:21:27.036138+0000 mgr.x (mgr.24697) 93 : cluster [DBG] pgmap v64: 89 pgs: 89 active+clean; 1.7 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 821 op/s 2023-10-01T10:21:28.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:28 smithi173 bash[27612]: audit 2023-10-01T10:21:28.055651+0000 mon.c (mon.2) 139 : audit [DBG] from='client.? 172.21.15.78:0/3128129433' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:29.119 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:21:28 smithi173 bash[52702]: ts=2023-10-01T10:21:28.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"smithi173:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"smithi173.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-01T10:21:30.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:30 smithi078 bash[33821]: cluster 2023-10-01T10:21:29.036634+0000 mgr.x (mgr.24697) 94 : cluster [DBG] pgmap v65: 89 pgs: 89 active+clean; 1.7 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 537 op/s 2023-10-01T10:21:30.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:30 smithi078 bash[33821]: audit 2023-10-01T10:21:29.320832+0000 mon.c (mon.2) 140 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:21:30.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:30 smithi078 bash[27336]: cluster 2023-10-01T10:21:29.036634+0000 mgr.x (mgr.24697) 94 : cluster [DBG] pgmap v65: 89 pgs: 89 active+clean; 1.7 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 537 op/s 2023-10-01T10:21:30.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:30 smithi078 bash[27336]: audit 2023-10-01T10:21:29.320832+0000 mon.c (mon.2) 140 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:21:30.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:30 smithi173 bash[27612]: cluster 2023-10-01T10:21:29.036634+0000 mgr.x (mgr.24697) 94 : cluster [DBG] pgmap v65: 89 pgs: 89 active+clean; 1.7 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 537 op/s 2023-10-01T10:21:30.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:30 smithi173 bash[27612]: audit 2023-10-01T10:21:29.320832+0000 mon.c (mon.2) 140 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:21:31.475 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:21:31 smithi173 bash[50431]: debug 2023-10-01T10:21:31.389+0000 7f9f0fe0a700 -1 mgr handle_mgr_map I was active but no longer am 2023-10-01T10:21:31.475 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:31 smithi173 bash[27612]: audit 2023-10-01T10:21:30.769414+0000 mon.c (mon.2) 141 : audit [DBG] from='client.? 172.21.15.78:0/146994090' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:31.564 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:31 smithi078 bash[33821]: audit 2023-10-01T10:21:30.769414+0000 mon.c (mon.2) 141 : audit [DBG] from='client.? 172.21.15.78:0/146994090' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:31.564 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:31 smithi078 bash[27336]: audit 2023-10-01T10:21:30.769414+0000 mon.c (mon.2) 141 : audit [DBG] from='client.? 172.21.15.78:0/146994090' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:31.565 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:21:31 smithi078 bash[71214]: [01/Oct/2023:10:21:31] ENGINE Bus STOPPING 2023-10-01T10:21:31.565 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:21:31 smithi078 bash[71214]: [01/Oct/2023:10:21:31] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-01T10:21:31.565 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:21:31 smithi078 bash[71214]: [01/Oct/2023:10:21:31] ENGINE Bus STOPPED 2023-10-01T10:21:31.565 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:21:31 smithi078 bash[71214]: [01/Oct/2023:10:21:31] ENGINE Bus STARTING 2023-10-01T10:21:31.739 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:21:31 smithi173 bash[50431]: ignoring --setuser ceph since I am not root 2023-10-01T10:21:31.739 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:21:31 smithi173 bash[50431]: ignoring --setgroup ceph since I am not root 2023-10-01T10:21:31.739 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:21:31 smithi173 bash[50431]: debug 2023-10-01T10:21:31.709+0000 7fb5fcb61200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-01T10:21:31.833 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:21:31 smithi078 bash[71214]: debug 2023-10-01T10:21:31.734+0000 7fe1769a6700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-01T10:21:31.833 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:21:31 smithi078 bash[71214]: debug 2023-10-01T10:21:31.734+0000 7fe1769a6700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-01T10:21:31.833 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:21:31 smithi078 bash[71214]: debug 2023-10-01T10:21:31.734+0000 7fe1769a6700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-01T10:21:31.833 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:21:31 smithi078 bash[71214]: debug 2023-10-01T10:21:31.734+0000 7fe1769a6700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-01T10:21:31.833 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:21:31 smithi078 bash[71214]: debug 2023-10-01T10:21:31.734+0000 7fe1769a6700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-01T10:21:31.833 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:21:31 smithi078 bash[71214]: debug 2023-10-01T10:21:31.742+0000 7fe17b9b0700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-01T10:21:31.833 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:21:31 smithi078 bash[71214]: debug 2023-10-01T10:21:31.742+0000 7fe17b9b0700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-01T10:21:31.834 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:21:31 smithi078 bash[71214]: debug 2023-10-01T10:21:31.742+0000 7fe17b9b0700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-01T10:21:31.834 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:21:31 smithi078 bash[71214]: debug 2023-10-01T10:21:31.742+0000 7fe17b9b0700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-01T10:21:31.834 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:21:31 smithi078 bash[71214]: debug 2023-10-01T10:21:31.742+0000 7fe17b9b0700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-01T10:21:31.834 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:21:31 smithi078 bash[71214]: [01/Oct/2023:10:21:31] ENGINE Serving on http://:::9283 2023-10-01T10:21:31.834 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:21:31 smithi078 bash[71214]: [01/Oct/2023:10:21:31] ENGINE Bus STARTED 2023-10-01T10:21:32.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[27336]: cluster 2023-10-01T10:21:31.037381+0000 mgr.x (mgr.24697) 95 : cluster [DBG] pgmap v66: 89 pgs: 89 active+clean; 1.5 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 832 op/s 2023-10-01T10:21:32.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[27336]: audit 2023-10-01T10:21:31.259506+0000 mon.a (mon.0) 882 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:21:32.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[27336]: audit 2023-10-01T10:21:31.268393+0000 mon.a (mon.0) 883 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:21:32.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[27336]: audit 2023-10-01T10:21:31.273722+0000 mon.c (mon.2) 142 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:21:32.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[27336]: audit 2023-10-01T10:21:31.274926+0000 mon.c (mon.2) 143 : audit [INF] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:21:32.585 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[27336]: audit 2023-10-01T10:21:31.284927+0000 mon.a (mon.0) 884 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:21:32.585 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[27336]: audit 2023-10-01T10:21:31.369348+0000 mon.c (mon.2) 144 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:21:32.585 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[27336]: audit 2023-10-01T10:21:31.381353+0000 mon.c (mon.2) 145 : audit [INF] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-01T10:21:32.585 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[27336]: audit 2023-10-01T10:21:31.381797+0000 mon.a (mon.0) 885 : audit [INF] from='mgr.24697 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-01T10:21:32.585 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[27336]: cluster 2023-10-01T10:21:31.389966+0000 mon.a (mon.0) 886 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2023-10-01T10:21:32.585 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[27336]: audit 2023-10-01T10:21:31.390248+0000 mon.a (mon.0) 887 : audit [INF] from='mgr.24697 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-10-01T10:21:32.585 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[27336]: cluster 2023-10-01T10:21:31.390324+0000 mon.a (mon.0) 888 : cluster [DBG] mgrmap e26: y(active, starting, since 0.00785875s) 2023-10-01T10:21:32.585 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[27336]: audit 2023-10-01T10:21:31.395467+0000 mon.c (mon.2) 146 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T10:21:32.585 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[27336]: audit 2023-10-01T10:21:31.395713+0000 mon.c (mon.2) 147 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T10:21:32.585 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[27336]: audit 2023-10-01T10:21:31.395921+0000 mon.c (mon.2) 148 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T10:21:32.585 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[27336]: audit 2023-10-01T10:21:31.397721+0000 mon.c (mon.2) 149 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi078.vjqrth"}]: dispatch 2023-10-01T10:21:32.586 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[27336]: audit 2023-10-01T10:21:31.398019+0000 mon.c (mon.2) 150 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi173.vrqqoc"}]: dispatch 2023-10-01T10:21:32.586 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[27336]: audit 2023-10-01T10:21:31.398363+0000 mon.c (mon.2) 151 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-01T10:21:32.586 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[27336]: audit 2023-10-01T10:21:31.398629+0000 mon.c (mon.2) 152 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T10:21:32.586 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[27336]: audit 2023-10-01T10:21:31.398996+0000 mon.c (mon.2) 153 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T10:21:32.586 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[27336]: audit 2023-10-01T10:21:31.399339+0000 mon.c (mon.2) 154 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T10:21:32.586 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[27336]: audit 2023-10-01T10:21:31.399668+0000 mon.c (mon.2) 155 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T10:21:32.586 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[27336]: audit 2023-10-01T10:21:31.399951+0000 mon.c (mon.2) 156 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T10:21:32.586 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[27336]: audit 2023-10-01T10:21:31.400219+0000 mon.c (mon.2) 157 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T10:21:32.586 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[27336]: audit 2023-10-01T10:21:31.400502+0000 mon.c (mon.2) 158 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T10:21:32.586 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[27336]: audit 2023-10-01T10:21:31.400812+0000 mon.c (mon.2) 159 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T10:21:32.587 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[27336]: audit 2023-10-01T10:21:31.401114+0000 mon.c (mon.2) 160 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-01T10:21:32.587 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[27336]: audit 2023-10-01T10:21:31.401605+0000 mon.c (mon.2) 161 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-01T10:21:32.587 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[27336]: audit 2023-10-01T10:21:31.402240+0000 mon.c (mon.2) 162 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-01T10:21:32.587 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[27336]: cluster 2023-10-01T10:21:31.459226+0000 mon.a (mon.0) 889 : cluster [INF] Manager daemon y is now available 2023-10-01T10:21:32.587 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[27336]: audit 2023-10-01T10:21:31.502805+0000 mon.c (mon.2) 163 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:21:32.587 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[27336]: audit 2023-10-01T10:21:31.519776+0000 mon.c (mon.2) 164 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:21:32.587 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[27336]: audit 2023-10-01T10:21:31.556663+0000 mon.c (mon.2) 165 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:21:32.587 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[27336]: audit 2023-10-01T10:21:31.569492+0000 mon.c (mon.2) 166 : audit [INF] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-01T10:21:32.587 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[27336]: audit 2023-10-01T10:21:31.570169+0000 mon.a (mon.0) 890 : audit [INF] from='mgr.25115 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-01T10:21:32.587 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[27336]: audit 2023-10-01T10:21:31.689891+0000 mon.c (mon.2) 167 : audit [INF] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-01T10:21:32.587 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[27336]: audit 2023-10-01T10:21:31.690382+0000 mon.a (mon.0) 891 : audit [INF] from='mgr.25115 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-01T10:21:32.588 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[33821]: cluster 2023-10-01T10:21:31.037381+0000 mgr.x (mgr.24697) 95 : cluster [DBG] pgmap v66: 89 pgs: 89 active+clean; 1.5 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 832 op/s 2023-10-01T10:21:32.588 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[33821]: audit 2023-10-01T10:21:31.259506+0000 mon.a (mon.0) 882 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:21:32.588 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[33821]: audit 2023-10-01T10:21:31.268393+0000 mon.a (mon.0) 883 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:21:32.588 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[33821]: audit 2023-10-01T10:21:31.273722+0000 mon.c (mon.2) 142 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:21:32.588 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[33821]: audit 2023-10-01T10:21:31.274926+0000 mon.c (mon.2) 143 : audit [INF] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:21:32.589 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[33821]: audit 2023-10-01T10:21:31.284927+0000 mon.a (mon.0) 884 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:21:32.589 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[33821]: audit 2023-10-01T10:21:31.369348+0000 mon.c (mon.2) 144 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:21:32.589 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[33821]: audit 2023-10-01T10:21:31.381353+0000 mon.c (mon.2) 145 : audit [INF] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-01T10:21:32.589 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[33821]: audit 2023-10-01T10:21:31.381797+0000 mon.a (mon.0) 885 : audit [INF] from='mgr.24697 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-01T10:21:32.589 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[33821]: cluster 2023-10-01T10:21:31.389966+0000 mon.a (mon.0) 886 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2023-10-01T10:21:32.589 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[33821]: audit 2023-10-01T10:21:31.390248+0000 mon.a (mon.0) 887 : audit [INF] from='mgr.24697 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-10-01T10:21:32.589 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[33821]: cluster 2023-10-01T10:21:31.390324+0000 mon.a (mon.0) 888 : cluster [DBG] mgrmap e26: y(active, starting, since 0.00785875s) 2023-10-01T10:21:32.589 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[33821]: audit 2023-10-01T10:21:31.395467+0000 mon.c (mon.2) 146 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T10:21:32.589 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[33821]: audit 2023-10-01T10:21:31.395713+0000 mon.c (mon.2) 147 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T10:21:32.589 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[33821]: audit 2023-10-01T10:21:31.395921+0000 mon.c (mon.2) 148 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T10:21:32.589 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[33821]: audit 2023-10-01T10:21:31.397721+0000 mon.c (mon.2) 149 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi078.vjqrth"}]: dispatch 2023-10-01T10:21:32.590 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[33821]: audit 2023-10-01T10:21:31.398019+0000 mon.c (mon.2) 150 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi173.vrqqoc"}]: dispatch 2023-10-01T10:21:32.590 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[33821]: audit 2023-10-01T10:21:31.398363+0000 mon.c (mon.2) 151 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-01T10:21:32.590 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[33821]: audit 2023-10-01T10:21:31.398629+0000 mon.c (mon.2) 152 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T10:21:32.590 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[33821]: audit 2023-10-01T10:21:31.398996+0000 mon.c (mon.2) 153 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T10:21:32.590 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[33821]: audit 2023-10-01T10:21:31.399339+0000 mon.c (mon.2) 154 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T10:21:32.590 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[33821]: audit 2023-10-01T10:21:31.399668+0000 mon.c (mon.2) 155 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T10:21:32.590 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[33821]: audit 2023-10-01T10:21:31.399951+0000 mon.c (mon.2) 156 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T10:21:32.590 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[33821]: audit 2023-10-01T10:21:31.400219+0000 mon.c (mon.2) 157 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T10:21:32.590 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[33821]: audit 2023-10-01T10:21:31.400502+0000 mon.c (mon.2) 158 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T10:21:32.590 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[33821]: audit 2023-10-01T10:21:31.400812+0000 mon.c (mon.2) 159 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T10:21:32.591 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[33821]: audit 2023-10-01T10:21:31.401114+0000 mon.c (mon.2) 160 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-01T10:21:32.591 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[33821]: audit 2023-10-01T10:21:31.401605+0000 mon.c (mon.2) 161 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-01T10:21:32.591 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[33821]: audit 2023-10-01T10:21:31.402240+0000 mon.c (mon.2) 162 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-01T10:21:32.591 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[33821]: cluster 2023-10-01T10:21:31.459226+0000 mon.a (mon.0) 889 : cluster [INF] Manager daemon y is now available 2023-10-01T10:21:32.591 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[33821]: audit 2023-10-01T10:21:31.502805+0000 mon.c (mon.2) 163 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:21:32.591 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[33821]: audit 2023-10-01T10:21:31.519776+0000 mon.c (mon.2) 164 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:21:32.591 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[33821]: audit 2023-10-01T10:21:31.556663+0000 mon.c (mon.2) 165 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:21:32.591 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[33821]: audit 2023-10-01T10:21:31.569492+0000 mon.c (mon.2) 166 : audit [INF] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-01T10:21:32.591 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[33821]: audit 2023-10-01T10:21:31.570169+0000 mon.a (mon.0) 890 : audit [INF] from='mgr.25115 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-01T10:21:32.591 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[33821]: audit 2023-10-01T10:21:31.689891+0000 mon.c (mon.2) 167 : audit [INF] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-01T10:21:32.591 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:32 smithi078 bash[33821]: audit 2023-10-01T10:21:31.690382+0000 mon.a (mon.0) 891 : audit [INF] from='mgr.25115 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-01T10:21:32.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:32 smithi173 bash[27612]: cluster 2023-10-01T10:21:31.037381+0000 mgr.x (mgr.24697) 95 : cluster [DBG] pgmap v66: 89 pgs: 89 active+clean; 1.5 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 832 op/s 2023-10-01T10:21:32.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:32 smithi173 bash[27612]: audit 2023-10-01T10:21:31.259506+0000 mon.a (mon.0) 882 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:21:32.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:32 smithi173 bash[27612]: audit 2023-10-01T10:21:31.268393+0000 mon.a (mon.0) 883 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:21:32.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:32 smithi173 bash[27612]: audit 2023-10-01T10:21:31.273722+0000 mon.c (mon.2) 142 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:21:32.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:32 smithi173 bash[27612]: audit 2023-10-01T10:21:31.274926+0000 mon.c (mon.2) 143 : audit [INF] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:21:32.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:32 smithi173 bash[27612]: audit 2023-10-01T10:21:31.284927+0000 mon.a (mon.0) 884 : audit [INF] from='mgr.24697 ' entity='mgr.x' 2023-10-01T10:21:32.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:32 smithi173 bash[27612]: audit 2023-10-01T10:21:31.369348+0000 mon.c (mon.2) 144 : audit [DBG] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:21:32.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:32 smithi173 bash[27612]: audit 2023-10-01T10:21:31.381353+0000 mon.c (mon.2) 145 : audit [INF] from='mgr.24697 172.21.15.173:0/2096776878' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-01T10:21:32.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:32 smithi173 bash[27612]: audit 2023-10-01T10:21:31.381797+0000 mon.a (mon.0) 885 : audit [INF] from='mgr.24697 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-10-01T10:21:32.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:32 smithi173 bash[27612]: cluster 2023-10-01T10:21:31.389966+0000 mon.a (mon.0) 886 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2023-10-01T10:21:32.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:32 smithi173 bash[27612]: audit 2023-10-01T10:21:31.390248+0000 mon.a (mon.0) 887 : audit [INF] from='mgr.24697 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-10-01T10:21:32.622 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:32 smithi173 bash[27612]: cluster 2023-10-01T10:21:31.390324+0000 mon.a (mon.0) 888 : cluster [DBG] mgrmap e26: y(active, starting, since 0.00785875s) 2023-10-01T10:21:32.622 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:32 smithi173 bash[27612]: audit 2023-10-01T10:21:31.395467+0000 mon.c (mon.2) 146 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T10:21:32.622 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:32 smithi173 bash[27612]: audit 2023-10-01T10:21:31.395713+0000 mon.c (mon.2) 147 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T10:21:32.622 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:32 smithi173 bash[27612]: audit 2023-10-01T10:21:31.395921+0000 mon.c (mon.2) 148 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T10:21:32.622 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:32 smithi173 bash[27612]: audit 2023-10-01T10:21:31.397721+0000 mon.c (mon.2) 149 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi078.vjqrth"}]: dispatch 2023-10-01T10:21:32.622 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:32 smithi173 bash[27612]: audit 2023-10-01T10:21:31.398019+0000 mon.c (mon.2) 150 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi173.vrqqoc"}]: dispatch 2023-10-01T10:21:32.622 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:32 smithi173 bash[27612]: audit 2023-10-01T10:21:31.398363+0000 mon.c (mon.2) 151 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-01T10:21:32.623 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:32 smithi173 bash[27612]: audit 2023-10-01T10:21:31.398629+0000 mon.c (mon.2) 152 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T10:21:32.623 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:32 smithi173 bash[27612]: audit 2023-10-01T10:21:31.398996+0000 mon.c (mon.2) 153 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T10:21:32.623 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:32 smithi173 bash[27612]: audit 2023-10-01T10:21:31.399339+0000 mon.c (mon.2) 154 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T10:21:32.623 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:32 smithi173 bash[27612]: audit 2023-10-01T10:21:31.399668+0000 mon.c (mon.2) 155 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T10:21:32.623 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:32 smithi173 bash[27612]: audit 2023-10-01T10:21:31.399951+0000 mon.c (mon.2) 156 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T10:21:32.623 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:32 smithi173 bash[27612]: audit 2023-10-01T10:21:31.400219+0000 mon.c (mon.2) 157 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T10:21:32.623 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:32 smithi173 bash[27612]: audit 2023-10-01T10:21:31.400502+0000 mon.c (mon.2) 158 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T10:21:32.624 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:32 smithi173 bash[27612]: audit 2023-10-01T10:21:31.400812+0000 mon.c (mon.2) 159 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T10:21:32.624 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:32 smithi173 bash[27612]: audit 2023-10-01T10:21:31.401114+0000 mon.c (mon.2) 160 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-01T10:21:32.624 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:32 smithi173 bash[27612]: audit 2023-10-01T10:21:31.401605+0000 mon.c (mon.2) 161 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-01T10:21:32.624 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:32 smithi173 bash[27612]: audit 2023-10-01T10:21:31.402240+0000 mon.c (mon.2) 162 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-01T10:21:32.624 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:32 smithi173 bash[27612]: cluster 2023-10-01T10:21:31.459226+0000 mon.a (mon.0) 889 : cluster [INF] Manager daemon y is now available 2023-10-01T10:21:32.625 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:32 smithi173 bash[27612]: audit 2023-10-01T10:21:31.502805+0000 mon.c (mon.2) 163 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:21:32.625 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:32 smithi173 bash[27612]: audit 2023-10-01T10:21:31.519776+0000 mon.c (mon.2) 164 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:21:32.625 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:32 smithi173 bash[27612]: audit 2023-10-01T10:21:31.556663+0000 mon.c (mon.2) 165 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:21:32.625 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:32 smithi173 bash[27612]: audit 2023-10-01T10:21:31.569492+0000 mon.c (mon.2) 166 : audit [INF] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-01T10:21:32.625 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:32 smithi173 bash[27612]: audit 2023-10-01T10:21:31.570169+0000 mon.a (mon.0) 890 : audit [INF] from='mgr.25115 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-01T10:21:32.625 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:32 smithi173 bash[27612]: audit 2023-10-01T10:21:31.689891+0000 mon.c (mon.2) 167 : audit [INF] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-01T10:21:32.626 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:32 smithi173 bash[27612]: audit 2023-10-01T10:21:31.690382+0000 mon.a (mon.0) 891 : audit [INF] from='mgr.25115 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-01T10:21:33.397 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:21:33 smithi173 bash[50431]: debug 2023-10-01T10:21:33.121+0000 7fb5fcb61200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-01T10:21:33.397 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:21:33 smithi173 bash[50431]: debug 2023-10-01T10:21:33.273+0000 7fb5fcb61200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-01T10:21:33.819 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:33 smithi173 bash[27612]: cephadm 2023-10-01T10:21:32.255585+0000 mgr.y (mgr.25115) 1 : cephadm [INF] [01/Oct/2023:10:21:32] ENGINE Bus STARTING 2023-10-01T10:21:33.819 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:33 smithi173 bash[27612]: cluster 2023-10-01T10:21:32.398443+0000 mon.a (mon.0) 892 : cluster [DBG] mgrmap e27: y(active, since 1.01597s) 2023-10-01T10:21:33.819 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:33 smithi173 bash[27612]: cephadm 2023-10-01T10:21:32.429884+0000 mgr.y (mgr.25115) 2 : cephadm [INF] [01/Oct/2023:10:21:32] ENGINE Serving on https://172.21.15.78:7150 2023-10-01T10:21:33.819 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:33 smithi173 bash[27612]: cluster 2023-10-01T10:21:32.459613+0000 mgr.y (mgr.25115) 3 : cluster [DBG] pgmap v3: 89 pgs: 89 active+clean; 1.4 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2023-10-01T10:21:33.820 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:33 smithi173 bash[27612]: cephadm 2023-10-01T10:21:32.534004+0000 mgr.y (mgr.25115) 4 : cephadm [INF] [01/Oct/2023:10:21:32] ENGINE Serving on http://172.21.15.78:8765 2023-10-01T10:21:33.820 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:33 smithi173 bash[27612]: cephadm 2023-10-01T10:21:32.534138+0000 mgr.y (mgr.25115) 5 : cephadm [INF] [01/Oct/2023:10:21:32] ENGINE Bus STARTED 2023-10-01T10:21:33.820 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:21:33 smithi173 bash[50431]: debug 2023-10-01T10:21:33.425+0000 7fb5fcb61200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-01T10:21:33.820 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:21:33 smithi173 bash[50431]: debug 2023-10-01T10:21:33.813+0000 7fb5fcb61200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-01T10:21:33.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:33 smithi078 bash[27336]: cephadm 2023-10-01T10:21:32.255585+0000 mgr.y (mgr.25115) 1 : cephadm [INF] [01/Oct/2023:10:21:32] ENGINE Bus STARTING 2023-10-01T10:21:33.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:33 smithi078 bash[27336]: cluster 2023-10-01T10:21:32.398443+0000 mon.a (mon.0) 892 : cluster [DBG] mgrmap e27: y(active, since 1.01597s) 2023-10-01T10:21:33.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:33 smithi078 bash[27336]: cephadm 2023-10-01T10:21:32.429884+0000 mgr.y (mgr.25115) 2 : cephadm [INF] [01/Oct/2023:10:21:32] ENGINE Serving on https://172.21.15.78:7150 2023-10-01T10:21:33.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:33 smithi078 bash[27336]: cluster 2023-10-01T10:21:32.459613+0000 mgr.y (mgr.25115) 3 : cluster [DBG] pgmap v3: 89 pgs: 89 active+clean; 1.4 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2023-10-01T10:21:33.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:33 smithi078 bash[27336]: cephadm 2023-10-01T10:21:32.534004+0000 mgr.y (mgr.25115) 4 : cephadm [INF] [01/Oct/2023:10:21:32] ENGINE Serving on http://172.21.15.78:8765 2023-10-01T10:21:33.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:33 smithi078 bash[27336]: cephadm 2023-10-01T10:21:32.534138+0000 mgr.y (mgr.25115) 5 : cephadm [INF] [01/Oct/2023:10:21:32] ENGINE Bus STARTED 2023-10-01T10:21:33.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:33 smithi078 bash[33821]: cephadm 2023-10-01T10:21:32.255585+0000 mgr.y (mgr.25115) 1 : cephadm [INF] [01/Oct/2023:10:21:32] ENGINE Bus STARTING 2023-10-01T10:21:33.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:33 smithi078 bash[33821]: cluster 2023-10-01T10:21:32.398443+0000 mon.a (mon.0) 892 : cluster [DBG] mgrmap e27: y(active, since 1.01597s) 2023-10-01T10:21:33.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:33 smithi078 bash[33821]: cephadm 2023-10-01T10:21:32.429884+0000 mgr.y (mgr.25115) 2 : cephadm [INF] [01/Oct/2023:10:21:32] ENGINE Serving on https://172.21.15.78:7150 2023-10-01T10:21:33.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:33 smithi078 bash[33821]: cluster 2023-10-01T10:21:32.459613+0000 mgr.y (mgr.25115) 3 : cluster [DBG] pgmap v3: 89 pgs: 89 active+clean; 1.4 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2023-10-01T10:21:33.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:33 smithi078 bash[33821]: cephadm 2023-10-01T10:21:32.534004+0000 mgr.y (mgr.25115) 4 : cephadm [INF] [01/Oct/2023:10:21:32] ENGINE Serving on http://172.21.15.78:8765 2023-10-01T10:21:33.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:33 smithi078 bash[33821]: cephadm 2023-10-01T10:21:32.534138+0000 mgr.y (mgr.25115) 5 : cephadm [INF] [01/Oct/2023:10:21:32] ENGINE Bus STARTED 2023-10-01T10:21:34.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:34 smithi173 bash[27612]: cluster 2023-10-01T10:21:33.399995+0000 mgr.y (mgr.25115) 6 : cluster [DBG] pgmap v4: 89 pgs: 89 active+clean; 1.4 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2023-10-01T10:21:34.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:34 smithi173 bash[27612]: audit 2023-10-01T10:21:33.428254+0000 mon.c (mon.2) 168 : audit [DBG] from='client.? 172.21.15.78:0/1927243473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:34.620 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:21:34 smithi173 bash[50431]: debug 2023-10-01T10:21:34.341+0000 7fb5fcb61200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-01T10:21:34.620 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:21:34 smithi173 bash[50431]: debug 2023-10-01T10:21:34.577+0000 7fb5fcb61200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-01T10:21:34.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:34 smithi078 bash[27336]: cluster 2023-10-01T10:21:33.399995+0000 mgr.y (mgr.25115) 6 : cluster [DBG] pgmap v4: 89 pgs: 89 active+clean; 1.4 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2023-10-01T10:21:34.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:34 smithi078 bash[27336]: audit 2023-10-01T10:21:33.428254+0000 mon.c (mon.2) 168 : audit [DBG] from='client.? 172.21.15.78:0/1927243473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:34.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:34 smithi078 bash[33821]: cluster 2023-10-01T10:21:33.399995+0000 mgr.y (mgr.25115) 6 : cluster [DBG] pgmap v4: 89 pgs: 89 active+clean; 1.4 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2023-10-01T10:21:34.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:34 smithi078 bash[33821]: audit 2023-10-01T10:21:33.428254+0000 mon.c (mon.2) 168 : audit [DBG] from='client.? 172.21.15.78:0/1927243473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:35.369 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:21:34 smithi173 bash[50431]: debug 2023-10-01T10:21:34.909+0000 7fb5fcb61200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-01T10:21:36.733 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:21:36 smithi173 bash[50431]: debug 2023-10-01T10:21:36.385+0000 7fb5fcb61200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-01T10:21:36.734 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:21:36 smithi173 bash[50431]: debug 2023-10-01T10:21:36.553+0000 7fb5fcb61200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-01T10:21:36.734 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:36 smithi173 bash[27612]: cluster 2023-10-01T10:21:35.400473+0000 mgr.y (mgr.25115) 7 : cluster [DBG] pgmap v5: 89 pgs: 89 active+clean; 1.4 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2023-10-01T10:21:36.734 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:36 smithi173 bash[27612]: audit 2023-10-01T10:21:36.074552+0000 mon.a (mon.0) 893 : audit [DBG] from='client.? 172.21.15.78:0/1858924811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:36.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:36 smithi078 bash[33821]: cluster 2023-10-01T10:21:35.400473+0000 mgr.y (mgr.25115) 7 : cluster [DBG] pgmap v5: 89 pgs: 89 active+clean; 1.4 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2023-10-01T10:21:36.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:36 smithi078 bash[33821]: audit 2023-10-01T10:21:36.074552+0000 mon.a (mon.0) 893 : audit [DBG] from='client.? 172.21.15.78:0/1858924811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:36.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:36 smithi078 bash[27336]: cluster 2023-10-01T10:21:35.400473+0000 mgr.y (mgr.25115) 7 : cluster [DBG] pgmap v5: 89 pgs: 89 active+clean; 1.4 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2023-10-01T10:21:36.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:36 smithi078 bash[27336]: audit 2023-10-01T10:21:36.074552+0000 mon.a (mon.0) 893 : audit [DBG] from='client.? 172.21.15.78:0/1858924811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:37.119 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:21:36 smithi173 bash[50431]: debug 2023-10-01T10:21:36.729+0000 7fb5fcb61200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-01T10:21:37.737 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:21:37 smithi173 bash[50431]: debug 2023-10-01T10:21:37.413+0000 7fb5fcb61200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-01T10:21:38.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:38 smithi078 bash[27336]: cluster 2023-10-01T10:21:37.400999+0000 mgr.y (mgr.25115) 8 : cluster [DBG] pgmap v6: 89 pgs: 89 active+clean; 1.4 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2023-10-01T10:21:38.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:38 smithi078 bash[33821]: cluster 2023-10-01T10:21:37.400999+0000 mgr.y (mgr.25115) 8 : cluster [DBG] pgmap v6: 89 pgs: 89 active+clean; 1.4 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2023-10-01T10:21:38.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:38 smithi173 bash[27612]: cluster 2023-10-01T10:21:37.400999+0000 mgr.y (mgr.25115) 8 : cluster [DBG] pgmap v6: 89 pgs: 89 active+clean; 1.4 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2023-10-01T10:21:38.870 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:21:37 smithi173 bash[50431]: debug 2023-10-01T10:21:37.729+0000 7fb5fcb61200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-01T10:21:38.870 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:21:38 smithi173 bash[50431]: debug 2023-10-01T10:21:38.457+0000 7fb5fcb61200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-01T10:21:39.578 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:21:39 smithi173 bash[50431]: debug 2023-10-01T10:21:39.177+0000 7fb5fcb61200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-01T10:21:39.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:39 smithi078 bash[27336]: audit 2023-10-01T10:21:38.574905+0000 mon.a (mon.0) 894 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:21:39.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:39 smithi078 bash[27336]: audit 2023-10-01T10:21:38.590645+0000 mon.a (mon.0) 895 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:21:39.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:39 smithi078 bash[27336]: audit 2023-10-01T10:21:38.820160+0000 mon.a (mon.0) 896 : audit [DBG] from='client.? 172.21.15.78:0/1886158513' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:39.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:39 smithi078 bash[27336]: audit 2023-10-01T10:21:39.182615+0000 mon.a (mon.0) 897 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:21:39.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:39 smithi078 bash[27336]: audit 2023-10-01T10:21:39.198142+0000 mon.a (mon.0) 898 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:21:39.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:39 smithi078 bash[33821]: audit 2023-10-01T10:21:38.574905+0000 mon.a (mon.0) 894 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:21:39.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:39 smithi078 bash[33821]: audit 2023-10-01T10:21:38.590645+0000 mon.a (mon.0) 895 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:21:39.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:39 smithi078 bash[33821]: audit 2023-10-01T10:21:38.820160+0000 mon.a (mon.0) 896 : audit [DBG] from='client.? 172.21.15.78:0/1886158513' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:39.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:39 smithi078 bash[33821]: audit 2023-10-01T10:21:39.182615+0000 mon.a (mon.0) 897 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:21:39.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:39 smithi078 bash[33821]: audit 2023-10-01T10:21:39.198142+0000 mon.a (mon.0) 898 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:21:39.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:39 smithi173 bash[27612]: audit 2023-10-01T10:21:38.574905+0000 mon.a (mon.0) 894 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:21:39.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:39 smithi173 bash[27612]: audit 2023-10-01T10:21:38.590645+0000 mon.a (mon.0) 895 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:21:39.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:39 smithi173 bash[27612]: audit 2023-10-01T10:21:38.820160+0000 mon.a (mon.0) 896 : audit [DBG] from='client.? 172.21.15.78:0/1886158513' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:39.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:39 smithi173 bash[27612]: audit 2023-10-01T10:21:39.182615+0000 mon.a (mon.0) 897 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:21:39.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:39 smithi173 bash[27612]: audit 2023-10-01T10:21:39.198142+0000 mon.a (mon.0) 898 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:21:40.369 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:21:40 smithi173 bash[50431]: debug 2023-10-01T10:21:39.993+0000 7fb5fcb61200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-01T10:21:40.370 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:21:40 smithi173 bash[50431]: debug 2023-10-01T10:21:40.157+0000 7fb5fcb61200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-01T10:21:40.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:40 smithi173 bash[27612]: cluster 2023-10-01T10:21:39.406113+0000 mgr.y (mgr.25115) 9 : cluster [DBG] pgmap v7: 89 pgs: 89 active+clean; 1.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 467 op/s 2023-10-01T10:21:40.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:40 smithi173 bash[27612]: audit 2023-10-01T10:21:39.599911+0000 mon.a (mon.0) 899 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:21:40.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:40 smithi173 bash[27612]: audit 2023-10-01T10:21:39.613993+0000 mon.a (mon.0) 900 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:21:40.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:40 smithi173 bash[27612]: audit 2023-10-01T10:21:40.150597+0000 mon.a (mon.0) 901 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:21:40.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:40 smithi173 bash[27612]: audit 2023-10-01T10:21:40.162381+0000 mon.a (mon.0) 902 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:21:40.871 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:21:40 smithi173 bash[50431]: debug 2023-10-01T10:21:40.661+0000 7fb5fcb61200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-01T10:21:40.871 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:21:40 smithi173 bash[50431]: debug 2023-10-01T10:21:40.825+0000 7fb5fcb61200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-01T10:21:41.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:40 smithi078 bash[27336]: cluster 2023-10-01T10:21:39.406113+0000 mgr.y (mgr.25115) 9 : cluster [DBG] pgmap v7: 89 pgs: 89 active+clean; 1.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 467 op/s 2023-10-01T10:21:41.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:40 smithi078 bash[27336]: audit 2023-10-01T10:21:39.599911+0000 mon.a (mon.0) 899 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:21:41.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:40 smithi078 bash[27336]: audit 2023-10-01T10:21:39.613993+0000 mon.a (mon.0) 900 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:21:41.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:40 smithi078 bash[27336]: audit 2023-10-01T10:21:40.150597+0000 mon.a (mon.0) 901 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:21:41.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:40 smithi078 bash[27336]: audit 2023-10-01T10:21:40.162381+0000 mon.a (mon.0) 902 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:21:41.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:40 smithi078 bash[33821]: cluster 2023-10-01T10:21:39.406113+0000 mgr.y (mgr.25115) 9 : cluster [DBG] pgmap v7: 89 pgs: 89 active+clean; 1.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 467 op/s 2023-10-01T10:21:41.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:40 smithi078 bash[33821]: audit 2023-10-01T10:21:39.599911+0000 mon.a (mon.0) 899 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:21:41.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:40 smithi078 bash[33821]: audit 2023-10-01T10:21:39.613993+0000 mon.a (mon.0) 900 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:21:41.085 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:40 smithi078 bash[33821]: audit 2023-10-01T10:21:40.150597+0000 mon.a (mon.0) 901 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:21:41.085 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:40 smithi078 bash[33821]: audit 2023-10-01T10:21:40.162381+0000 mon.a (mon.0) 902 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:21:41.357 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:21:41 smithi173 bash[50431]: debug 2023-10-01T10:21:41.029+0000 7fb5fcb61200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-01T10:21:41.357 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:21:41 smithi173 bash[50431]: debug 2023-10-01T10:21:41.181+0000 7fb5fcb61200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-01T10:21:41.357 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:21:41 smithi173 bash[50431]: debug 2023-10-01T10:21:41.349+0000 7fb5fcb61200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-01T10:21:42.120 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:21:41 smithi173 bash[50431]: debug 2023-10-01T10:21:41.621+0000 7fb5fcb61200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-01T10:21:42.120 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:21:41 smithi173 bash[50431]: debug 2023-10-01T10:21:41.793+0000 7fb5fcb61200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-01T10:21:42.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:42 smithi078 bash[27336]: cluster 2023-10-01T10:21:41.406653+0000 mgr.y (mgr.25115) 10 : cluster [DBG] pgmap v8: 89 pgs: 89 active+clean; 1.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 362 op/s 2023-10-01T10:21:42.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:42 smithi078 bash[27336]: audit 2023-10-01T10:21:41.482515+0000 mon.c (mon.2) 169 : audit [DBG] from='client.? 172.21.15.78:0/3712255053' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:42.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:42 smithi078 bash[33821]: cluster 2023-10-01T10:21:41.406653+0000 mgr.y (mgr.25115) 10 : cluster [DBG] pgmap v8: 89 pgs: 89 active+clean; 1.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 362 op/s 2023-10-01T10:21:42.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:42 smithi078 bash[33821]: audit 2023-10-01T10:21:41.482515+0000 mon.c (mon.2) 169 : audit [DBG] from='client.? 172.21.15.78:0/3712255053' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:42.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:42 smithi173 bash[27612]: cluster 2023-10-01T10:21:41.406653+0000 mgr.y (mgr.25115) 10 : cluster [DBG] pgmap v8: 89 pgs: 89 active+clean; 1.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 362 op/s 2023-10-01T10:21:42.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:42 smithi173 bash[27612]: audit 2023-10-01T10:21:41.482515+0000 mon.c (mon.2) 169 : audit [DBG] from='client.? 172.21.15.78:0/3712255053' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:43.870 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:21:43 smithi173 bash[50431]: debug 2023-10-01T10:21:43.529+0000 7fb5fcb61200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-01T10:21:43.870 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:21:43 smithi173 bash[50431]: [01/Oct/2023:10:21:43] ENGINE Bus STARTING 2023-10-01T10:21:43.870 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:21:43 smithi173 bash[50431]: CherryPy Checker: 2023-10-01T10:21:43.870 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:21:43 smithi173 bash[50431]: The Application mounted at '' has an empty config. 2023-10-01T10:21:43.870 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:21:43 smithi173 bash[50431]: [01/Oct/2023:10:21:43] ENGINE Serving on http://:::9283 2023-10-01T10:21:43.870 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:21:43 smithi173 bash[50431]: [01/Oct/2023:10:21:43] ENGINE Bus STARTED 2023-10-01T10:21:44.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:44 smithi078 bash[27336]: cluster 2023-10-01T10:21:43.407462+0000 mgr.y (mgr.25115) 11 : cluster [DBG] pgmap v9: 89 pgs: 89 active+clean; 1.0 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 576 op/s 2023-10-01T10:21:44.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:44 smithi078 bash[27336]: cluster 2023-10-01T10:21:43.553568+0000 mon.a (mon.0) 903 : cluster [DBG] Standby manager daemon x started 2023-10-01T10:21:44.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:44 smithi078 bash[27336]: audit 2023-10-01T10:21:43.558634+0000 mon.b (mon.1) 44 : audit [DBG] from='mgr.? 172.21.15.173:0/1696412936' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-01T10:21:44.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:44 smithi078 bash[27336]: audit 2023-10-01T10:21:43.565102+0000 mon.b (mon.1) 45 : audit [DBG] from='mgr.? 172.21.15.173:0/1696412936' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-01T10:21:44.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:44 smithi078 bash[27336]: audit 2023-10-01T10:21:43.569669+0000 mon.b (mon.1) 46 : audit [DBG] from='mgr.? 172.21.15.173:0/1696412936' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-01T10:21:44.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:44 smithi078 bash[27336]: audit 2023-10-01T10:21:43.573579+0000 mon.b (mon.1) 47 : audit [DBG] from='mgr.? 172.21.15.173:0/1696412936' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-01T10:21:44.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:44 smithi078 bash[27336]: audit 2023-10-01T10:21:44.194040+0000 mon.b (mon.1) 48 : audit [DBG] from='client.? 172.21.15.78:0/3858569934' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:44.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:44 smithi078 bash[33821]: cluster 2023-10-01T10:21:43.407462+0000 mgr.y (mgr.25115) 11 : cluster [DBG] pgmap v9: 89 pgs: 89 active+clean; 1.0 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 576 op/s 2023-10-01T10:21:44.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:44 smithi078 bash[33821]: cluster 2023-10-01T10:21:43.553568+0000 mon.a (mon.0) 903 : cluster [DBG] Standby manager daemon x started 2023-10-01T10:21:44.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:44 smithi078 bash[33821]: audit 2023-10-01T10:21:43.558634+0000 mon.b (mon.1) 44 : audit [DBG] from='mgr.? 172.21.15.173:0/1696412936' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-01T10:21:44.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:44 smithi078 bash[33821]: audit 2023-10-01T10:21:43.565102+0000 mon.b (mon.1) 45 : audit [DBG] from='mgr.? 172.21.15.173:0/1696412936' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-01T10:21:44.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:44 smithi078 bash[33821]: audit 2023-10-01T10:21:43.569669+0000 mon.b (mon.1) 46 : audit [DBG] from='mgr.? 172.21.15.173:0/1696412936' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-01T10:21:44.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:44 smithi078 bash[33821]: audit 2023-10-01T10:21:43.573579+0000 mon.b (mon.1) 47 : audit [DBG] from='mgr.? 172.21.15.173:0/1696412936' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-01T10:21:44.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:44 smithi078 bash[33821]: audit 2023-10-01T10:21:44.194040+0000 mon.b (mon.1) 48 : audit [DBG] from='client.? 172.21.15.78:0/3858569934' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:44.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:44 smithi173 bash[27612]: cluster 2023-10-01T10:21:43.407462+0000 mgr.y (mgr.25115) 11 : cluster [DBG] pgmap v9: 89 pgs: 89 active+clean; 1.0 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 576 op/s 2023-10-01T10:21:44.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:44 smithi173 bash[27612]: cluster 2023-10-01T10:21:43.553568+0000 mon.a (mon.0) 903 : cluster [DBG] Standby manager daemon x started 2023-10-01T10:21:44.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:44 smithi173 bash[27612]: audit 2023-10-01T10:21:43.558634+0000 mon.b (mon.1) 44 : audit [DBG] from='mgr.? 172.21.15.173:0/1696412936' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-01T10:21:44.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:44 smithi173 bash[27612]: audit 2023-10-01T10:21:43.565102+0000 mon.b (mon.1) 45 : audit [DBG] from='mgr.? 172.21.15.173:0/1696412936' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-01T10:21:44.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:44 smithi173 bash[27612]: audit 2023-10-01T10:21:43.569669+0000 mon.b (mon.1) 46 : audit [DBG] from='mgr.? 172.21.15.173:0/1696412936' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-01T10:21:44.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:44 smithi173 bash[27612]: audit 2023-10-01T10:21:43.573579+0000 mon.b (mon.1) 47 : audit [DBG] from='mgr.? 172.21.15.173:0/1696412936' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-01T10:21:44.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:44 smithi173 bash[27612]: audit 2023-10-01T10:21:44.194040+0000 mon.b (mon.1) 48 : audit [DBG] from='client.? 172.21.15.78:0/3858569934' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:45.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:45 smithi078 bash[33821]: cluster 2023-10-01T10:21:44.492163+0000 mon.a (mon.0) 904 : cluster [DBG] mgrmap e28: y(active, since 13s), standbys: x 2023-10-01T10:21:45.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:45 smithi078 bash[33821]: audit 2023-10-01T10:21:44.498084+0000 mon.c (mon.2) 170 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-01T10:21:45.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:45 smithi078 bash[27336]: cluster 2023-10-01T10:21:44.492163+0000 mon.a (mon.0) 904 : cluster [DBG] mgrmap e28: y(active, since 13s), standbys: x 2023-10-01T10:21:45.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:45 smithi078 bash[27336]: audit 2023-10-01T10:21:44.498084+0000 mon.c (mon.2) 170 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-01T10:21:45.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:45 smithi173 bash[27612]: cluster 2023-10-01T10:21:44.492163+0000 mon.a (mon.0) 904 : cluster [DBG] mgrmap e28: y(active, since 13s), standbys: x 2023-10-01T10:21:45.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:45 smithi173 bash[27612]: audit 2023-10-01T10:21:44.498084+0000 mon.c (mon.2) 170 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-01T10:21:46.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:46 smithi078 bash[33821]: cluster 2023-10-01T10:21:45.407991+0000 mgr.y (mgr.25115) 12 : cluster [DBG] pgmap v10: 89 pgs: 89 active+clean; 1.0 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 525 op/s 2023-10-01T10:21:46.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:46 smithi078 bash[27336]: cluster 2023-10-01T10:21:45.407991+0000 mgr.y (mgr.25115) 12 : cluster [DBG] pgmap v10: 89 pgs: 89 active+clean; 1.0 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 525 op/s 2023-10-01T10:21:46.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:46 smithi173 bash[27612]: cluster 2023-10-01T10:21:45.407991+0000 mgr.y (mgr.25115) 12 : cluster [DBG] pgmap v10: 89 pgs: 89 active+clean; 1.0 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 525 op/s 2023-10-01T10:21:47.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:47 smithi078 bash[27336]: audit 2023-10-01T10:21:46.557200+0000 mon.c (mon.2) 171 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:21:47.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:47 smithi078 bash[27336]: audit 2023-10-01T10:21:46.872647+0000 mon.c (mon.2) 172 : audit [DBG] from='client.? 172.21.15.78:0/3749109272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:47.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:47 smithi078 bash[33821]: audit 2023-10-01T10:21:46.557200+0000 mon.c (mon.2) 171 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:21:47.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:47 smithi078 bash[33821]: audit 2023-10-01T10:21:46.872647+0000 mon.c (mon.2) 172 : audit [DBG] from='client.? 172.21.15.78:0/3749109272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:47.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:47 smithi173 bash[27612]: audit 2023-10-01T10:21:46.557200+0000 mon.c (mon.2) 171 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:21:47.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:47 smithi173 bash[27612]: audit 2023-10-01T10:21:46.872647+0000 mon.c (mon.2) 172 : audit [DBG] from='client.? 172.21.15.78:0/3749109272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:48.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:48 smithi078 bash[33821]: cluster 2023-10-01T10:21:47.408498+0000 mgr.y (mgr.25115) 13 : cluster [DBG] pgmap v11: 89 pgs: 89 active+clean; 1.0 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 525 op/s 2023-10-01T10:21:49.790 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:48 smithi078 bash[27336]: cluster 2023-10-01T10:21:47.408498+0000 mgr.y (mgr.25115) 13 : cluster [DBG] pgmap v11: 89 pgs: 89 active+clean; 1.0 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 525 op/s 2023-10-01T10:21:49.790 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:48 smithi173 bash[27612]: cluster 2023-10-01T10:21:47.408498+0000 mgr.y (mgr.25115) 13 : cluster [DBG] pgmap v11: 89 pgs: 89 active+clean; 1.0 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 525 op/s 2023-10-01T10:21:50.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:50 smithi078 bash[27336]: cluster 2023-10-01T10:21:49.409239+0000 mgr.y (mgr.25115) 14 : cluster [DBG] pgmap v12: 89 pgs: 89 active+clean; 853 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 816 op/s 2023-10-01T10:21:50.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:50 smithi078 bash[27336]: audit 2023-10-01T10:21:49.573399+0000 mon.a (mon.0) 905 : audit [DBG] from='client.? 172.21.15.78:0/759536' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:50.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:50 smithi078 bash[33821]: cluster 2023-10-01T10:21:49.409239+0000 mgr.y (mgr.25115) 14 : cluster [DBG] pgmap v12: 89 pgs: 89 active+clean; 853 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 816 op/s 2023-10-01T10:21:50.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:50 smithi078 bash[33821]: audit 2023-10-01T10:21:49.573399+0000 mon.a (mon.0) 905 : audit [DBG] from='client.? 172.21.15.78:0/759536' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:50.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:50 smithi173 bash[27612]: cluster 2023-10-01T10:21:49.409239+0000 mgr.y (mgr.25115) 14 : cluster [DBG] pgmap v12: 89 pgs: 89 active+clean; 853 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 816 op/s 2023-10-01T10:21:50.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:50 smithi173 bash[27612]: audit 2023-10-01T10:21:49.573399+0000 mon.a (mon.0) 905 : audit [DBG] from='client.? 172.21.15.78:0/759536' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:52.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:51 smithi078 bash[27336]: audit 2023-10-01T10:21:50.915274+0000 mon.a (mon.0) 906 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:21:52.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:51 smithi078 bash[27336]: audit 2023-10-01T10:21:50.926074+0000 mon.a (mon.0) 907 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:21:52.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:51 smithi078 bash[27336]: audit 2023-10-01T10:21:50.933029+0000 mon.c (mon.2) 173 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:21:52.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:51 smithi078 bash[27336]: audit 2023-10-01T10:21:50.935097+0000 mon.c (mon.2) 174 : audit [INF] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:21:52.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:51 smithi078 bash[27336]: cephadm 2023-10-01T10:21:50.936847+0000 mgr.y (mgr.25115) 15 : cephadm [INF] Updating smithi078:/etc/ceph/ceph.conf 2023-10-01T10:21:52.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:51 smithi078 bash[27336]: cephadm 2023-10-01T10:21:50.937090+0000 mgr.y (mgr.25115) 16 : cephadm [INF] Updating smithi173:/etc/ceph/ceph.conf 2023-10-01T10:21:52.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:51 smithi078 bash[27336]: cephadm 2023-10-01T10:21:51.153052+0000 mgr.y (mgr.25115) 17 : cephadm [INF] Updating smithi173:/var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T10:21:52.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:51 smithi078 bash[27336]: cephadm 2023-10-01T10:21:51.155158+0000 mgr.y (mgr.25115) 18 : cephadm [INF] Updating smithi078:/var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T10:21:52.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:51 smithi078 bash[27336]: cephadm 2023-10-01T10:21:51.384360+0000 mgr.y (mgr.25115) 19 : cephadm [INF] Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2023-10-01T10:21:52.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:51 smithi078 bash[27336]: cephadm 2023-10-01T10:21:51.384616+0000 mgr.y (mgr.25115) 20 : cephadm [INF] Updating smithi173:/etc/ceph/ceph.client.admin.keyring 2023-10-01T10:21:52.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:51 smithi078 bash[27336]: cluster 2023-10-01T10:21:51.409695+0000 mgr.y (mgr.25115) 21 : cluster [DBG] pgmap v13: 89 pgs: 89 active+clean; 853 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 546 op/s 2023-10-01T10:21:52.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:51 smithi078 bash[27336]: cephadm 2023-10-01T10:21:51.599568+0000 mgr.y (mgr.25115) 22 : cephadm [INF] Updating smithi173:/var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-01T10:21:52.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:51 smithi078 bash[27336]: cephadm 2023-10-01T10:21:51.599867+0000 mgr.y (mgr.25115) 23 : cephadm [INF] Updating smithi078:/var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-01T10:21:52.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:51 smithi078 bash[27336]: audit 2023-10-01T10:21:51.835979+0000 mon.a (mon.0) 908 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:21:52.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:51 smithi078 bash[27336]: audit 2023-10-01T10:21:51.845695+0000 mon.a (mon.0) 909 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:21:52.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:51 smithi078 bash[27336]: audit 2023-10-01T10:21:51.855846+0000 mon.a (mon.0) 910 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:21:52.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:51 smithi078 bash[27336]: audit 2023-10-01T10:21:51.868896+0000 mon.a (mon.0) 911 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:21:52.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:51 smithi078 bash[27336]: audit 2023-10-01T10:21:51.877817+0000 mon.a (mon.0) 912 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:21:52.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:51 smithi078 bash[33821]: audit 2023-10-01T10:21:50.915274+0000 mon.a (mon.0) 906 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:21:52.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:51 smithi078 bash[33821]: audit 2023-10-01T10:21:50.926074+0000 mon.a (mon.0) 907 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:21:52.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:51 smithi078 bash[33821]: audit 2023-10-01T10:21:50.933029+0000 mon.c (mon.2) 173 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:21:52.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:51 smithi078 bash[33821]: audit 2023-10-01T10:21:50.935097+0000 mon.c (mon.2) 174 : audit [INF] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:21:52.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:51 smithi078 bash[33821]: cephadm 2023-10-01T10:21:50.936847+0000 mgr.y (mgr.25115) 15 : cephadm [INF] Updating smithi078:/etc/ceph/ceph.conf 2023-10-01T10:21:52.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:51 smithi078 bash[33821]: cephadm 2023-10-01T10:21:50.937090+0000 mgr.y (mgr.25115) 16 : cephadm [INF] Updating smithi173:/etc/ceph/ceph.conf 2023-10-01T10:21:52.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:51 smithi078 bash[33821]: cephadm 2023-10-01T10:21:51.153052+0000 mgr.y (mgr.25115) 17 : cephadm [INF] Updating smithi173:/var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T10:21:52.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:51 smithi078 bash[33821]: cephadm 2023-10-01T10:21:51.155158+0000 mgr.y (mgr.25115) 18 : cephadm [INF] Updating smithi078:/var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T10:21:52.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:51 smithi078 bash[33821]: cephadm 2023-10-01T10:21:51.384360+0000 mgr.y (mgr.25115) 19 : cephadm [INF] Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2023-10-01T10:21:52.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:51 smithi078 bash[33821]: cephadm 2023-10-01T10:21:51.384616+0000 mgr.y (mgr.25115) 20 : cephadm [INF] Updating smithi173:/etc/ceph/ceph.client.admin.keyring 2023-10-01T10:21:52.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:51 smithi078 bash[33821]: cluster 2023-10-01T10:21:51.409695+0000 mgr.y (mgr.25115) 21 : cluster [DBG] pgmap v13: 89 pgs: 89 active+clean; 853 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 546 op/s 2023-10-01T10:21:52.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:51 smithi078 bash[33821]: cephadm 2023-10-01T10:21:51.599568+0000 mgr.y (mgr.25115) 22 : cephadm [INF] Updating smithi173:/var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-01T10:21:52.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:51 smithi078 bash[33821]: cephadm 2023-10-01T10:21:51.599867+0000 mgr.y (mgr.25115) 23 : cephadm [INF] Updating smithi078:/var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-01T10:21:52.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:51 smithi078 bash[33821]: audit 2023-10-01T10:21:51.835979+0000 mon.a (mon.0) 908 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:21:52.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:51 smithi078 bash[33821]: audit 2023-10-01T10:21:51.845695+0000 mon.a (mon.0) 909 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:21:52.337 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:51 smithi078 bash[33821]: audit 2023-10-01T10:21:51.855846+0000 mon.a (mon.0) 910 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:21:52.337 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:51 smithi078 bash[33821]: audit 2023-10-01T10:21:51.868896+0000 mon.a (mon.0) 911 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:21:52.337 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:51 smithi078 bash[33821]: audit 2023-10-01T10:21:51.877817+0000 mon.a (mon.0) 912 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:21:52.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:51 smithi173 bash[27612]: audit 2023-10-01T10:21:50.915274+0000 mon.a (mon.0) 906 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:21:52.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:51 smithi173 bash[27612]: audit 2023-10-01T10:21:50.926074+0000 mon.a (mon.0) 907 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:21:52.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:51 smithi173 bash[27612]: audit 2023-10-01T10:21:50.933029+0000 mon.c (mon.2) 173 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:21:52.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:51 smithi173 bash[27612]: audit 2023-10-01T10:21:50.935097+0000 mon.c (mon.2) 174 : audit [INF] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:21:52.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:51 smithi173 bash[27612]: cephadm 2023-10-01T10:21:50.936847+0000 mgr.y (mgr.25115) 15 : cephadm [INF] Updating smithi078:/etc/ceph/ceph.conf 2023-10-01T10:21:52.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:51 smithi173 bash[27612]: cephadm 2023-10-01T10:21:50.937090+0000 mgr.y (mgr.25115) 16 : cephadm [INF] Updating smithi173:/etc/ceph/ceph.conf 2023-10-01T10:21:52.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:51 smithi173 bash[27612]: cephadm 2023-10-01T10:21:51.153052+0000 mgr.y (mgr.25115) 17 : cephadm [INF] Updating smithi173:/var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T10:21:52.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:51 smithi173 bash[27612]: cephadm 2023-10-01T10:21:51.155158+0000 mgr.y (mgr.25115) 18 : cephadm [INF] Updating smithi078:/var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T10:21:52.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:51 smithi173 bash[27612]: cephadm 2023-10-01T10:21:51.384360+0000 mgr.y (mgr.25115) 19 : cephadm [INF] Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2023-10-01T10:21:52.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:51 smithi173 bash[27612]: cephadm 2023-10-01T10:21:51.384616+0000 mgr.y (mgr.25115) 20 : cephadm [INF] Updating smithi173:/etc/ceph/ceph.client.admin.keyring 2023-10-01T10:21:52.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:51 smithi173 bash[27612]: cluster 2023-10-01T10:21:51.409695+0000 mgr.y (mgr.25115) 21 : cluster [DBG] pgmap v13: 89 pgs: 89 active+clean; 853 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 546 op/s 2023-10-01T10:21:52.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:51 smithi173 bash[27612]: cephadm 2023-10-01T10:21:51.599568+0000 mgr.y (mgr.25115) 22 : cephadm [INF] Updating smithi173:/var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-01T10:21:52.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:51 smithi173 bash[27612]: cephadm 2023-10-01T10:21:51.599867+0000 mgr.y (mgr.25115) 23 : cephadm [INF] Updating smithi078:/var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-01T10:21:52.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:51 smithi173 bash[27612]: audit 2023-10-01T10:21:51.835979+0000 mon.a (mon.0) 908 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:21:52.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:51 smithi173 bash[27612]: audit 2023-10-01T10:21:51.845695+0000 mon.a (mon.0) 909 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:21:52.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:51 smithi173 bash[27612]: audit 2023-10-01T10:21:51.855846+0000 mon.a (mon.0) 910 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:21:52.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:51 smithi173 bash[27612]: audit 2023-10-01T10:21:51.868896+0000 mon.a (mon.0) 911 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:21:52.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:51 smithi173 bash[27612]: audit 2023-10-01T10:21:51.877817+0000 mon.a (mon.0) 912 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:21:53.119 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:21:52 smithi173 bash[50431]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:21:52] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2023-10-01T10:21:53.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:53 smithi078 bash[27336]: cephadm 2023-10-01T10:21:51.960654+0000 mgr.y (mgr.25115) 24 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-01T10:21:53.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:53 smithi078 bash[27336]: cephadm 2023-10-01T10:21:52.234562+0000 mgr.y (mgr.25115) 25 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi173 2023-10-01T10:21:53.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:53 smithi078 bash[27336]: audit 2023-10-01T10:21:52.320050+0000 mon.c (mon.2) 175 : audit [DBG] from='client.? 172.21.15.78:0/2602754845' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:53.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:53 smithi078 bash[33821]: cephadm 2023-10-01T10:21:51.960654+0000 mgr.y (mgr.25115) 24 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-01T10:21:53.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:53 smithi078 bash[33821]: cephadm 2023-10-01T10:21:52.234562+0000 mgr.y (mgr.25115) 25 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi173 2023-10-01T10:21:53.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:53 smithi078 bash[33821]: audit 2023-10-01T10:21:52.320050+0000 mon.c (mon.2) 175 : audit [DBG] from='client.? 172.21.15.78:0/2602754845' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:53.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:53 smithi173 bash[27612]: cephadm 2023-10-01T10:21:51.960654+0000 mgr.y (mgr.25115) 24 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2023-10-01T10:21:53.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:53 smithi173 bash[27612]: cephadm 2023-10-01T10:21:52.234562+0000 mgr.y (mgr.25115) 25 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi173 2023-10-01T10:21:53.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:53 smithi173 bash[27612]: audit 2023-10-01T10:21:52.320050+0000 mon.c (mon.2) 175 : audit [DBG] from='client.? 172.21.15.78:0/2602754845' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:54.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:54 smithi078 bash[27336]: cluster 2023-10-01T10:21:53.410450+0000 mgr.y (mgr.25115) 26 : cluster [DBG] pgmap v14: 89 pgs: 89 active+clean; 638 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 833 op/s 2023-10-01T10:21:54.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:54 smithi078 bash[33821]: cluster 2023-10-01T10:21:53.410450+0000 mgr.y (mgr.25115) 26 : cluster [DBG] pgmap v14: 89 pgs: 89 active+clean; 638 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 833 op/s 2023-10-01T10:21:54.702 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:54 smithi173 bash[27612]: cluster 2023-10-01T10:21:53.410450+0000 mgr.y (mgr.25115) 26 : cluster [DBG] pgmap v14: 89 pgs: 89 active+clean; 638 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 833 op/s 2023-10-01T10:21:55.495 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:55 smithi173 bash[27612]: audit 2023-10-01T10:21:54.977761+0000 mon.c (mon.2) 176 : audit [DBG] from='client.? 172.21.15.78:0/463412627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:55.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:55 smithi078 bash[27336]: audit 2023-10-01T10:21:54.977761+0000 mon.c (mon.2) 176 : audit [DBG] from='client.? 172.21.15.78:0/463412627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:55.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:55 smithi078 bash[33821]: audit 2023-10-01T10:21:54.977761+0000 mon.c (mon.2) 176 : audit [DBG] from='client.? 172.21.15.78:0/463412627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:56.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:56 smithi173 bash[27612]: cluster 2023-10-01T10:21:55.410874+0000 mgr.y (mgr.25115) 27 : cluster [DBG] pgmap v15: 89 pgs: 89 active+clean; 638 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 578 op/s 2023-10-01T10:21:56.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:56 smithi078 bash[27336]: cluster 2023-10-01T10:21:55.410874+0000 mgr.y (mgr.25115) 27 : cluster [DBG] pgmap v15: 89 pgs: 89 active+clean; 638 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 578 op/s 2023-10-01T10:21:56.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:56 smithi078 bash[33821]: cluster 2023-10-01T10:21:55.410874+0000 mgr.y (mgr.25115) 27 : cluster [DBG] pgmap v15: 89 pgs: 89 active+clean; 638 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 578 op/s 2023-10-01T10:21:58.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:58 smithi078 bash[33821]: cluster 2023-10-01T10:21:57.411283+0000 mgr.y (mgr.25115) 28 : cluster [DBG] pgmap v16: 89 pgs: 89 active+clean; 638 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 578 op/s 2023-10-01T10:21:58.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:21:58 smithi078 bash[33821]: audit 2023-10-01T10:21:57.689584+0000 mon.c (mon.2) 177 : audit [DBG] from='client.? 172.21.15.78:0/4022247492' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:58.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:58 smithi078 bash[27336]: cluster 2023-10-01T10:21:57.411283+0000 mgr.y (mgr.25115) 28 : cluster [DBG] pgmap v16: 89 pgs: 89 active+clean; 638 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 578 op/s 2023-10-01T10:21:58.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:21:58 smithi078 bash[27336]: audit 2023-10-01T10:21:57.689584+0000 mon.c (mon.2) 177 : audit [DBG] from='client.? 172.21.15.78:0/4022247492' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:58.853 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:21:58 smithi173 systemd[1]: Stopping Ceph prometheus.a for d8016480-6041-11ee-8db5-212e2dc638e7... 2023-10-01T10:21:58.853 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:21:58 smithi173 bash[54387]: Error response from daemon: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-prometheus.a 2023-10-01T10:21:58.853 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:21:58 smithi173 bash[52702]: ts=2023-10-01T10:21:58.851Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-10-01T10:21:58.853 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:21:58 smithi173 bash[52702]: ts=2023-10-01T10:21:58.851Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2023-10-01T10:21:58.853 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:58 smithi173 bash[27612]: cluster 2023-10-01T10:21:57.411283+0000 mgr.y (mgr.25115) 28 : cluster [DBG] pgmap v16: 89 pgs: 89 active+clean; 638 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 578 op/s 2023-10-01T10:21:58.854 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:21:58 smithi173 bash[27612]: audit 2023-10-01T10:21:57.689584+0000 mon.c (mon.2) 177 : audit [DBG] from='client.? 172.21.15.78:0/4022247492' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:21:59.120 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:21:58 smithi173 bash[52702]: ts=2023-10-01T10:21:58.851Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2023-10-01T10:21:59.120 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:21:58 smithi173 bash[52702]: ts=2023-10-01T10:21:58.851Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2023-10-01T10:21:59.120 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:21:58 smithi173 bash[52702]: ts=2023-10-01T10:21:58.851Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2023-10-01T10:21:59.120 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:21:58 smithi173 bash[52702]: ts=2023-10-01T10:21:58.851Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2023-10-01T10:21:59.120 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:21:58 smithi173 bash[52702]: ts=2023-10-01T10:21:58.851Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2023-10-01T10:21:59.120 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:21:58 smithi173 bash[52702]: ts=2023-10-01T10:21:58.851Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2023-10-01T10:21:59.120 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:21:58 smithi173 bash[52702]: ts=2023-10-01T10:21:58.852Z caller=main.go:828 level=info msg="Scrape manager stopped" 2023-10-01T10:21:59.120 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:21:58 smithi173 bash[52702]: ts=2023-10-01T10:21:58.852Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2023-10-01T10:21:59.121 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:21:58 smithi173 bash[52702]: ts=2023-10-01T10:21:58.852Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2023-10-01T10:21:59.121 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:21:58 smithi173 bash[52702]: ts=2023-10-01T10:21:58.852Z caller=main.go:1066 level=info msg="See you next time!" 2023-10-01T10:21:59.746 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:21:59 smithi173 bash[54417]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-prometheus-a 2023-10-01T10:21:59.746 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:21:59 smithi173 bash[54477]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-prometheus-a 2023-10-01T10:21:59.747 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:21:59 smithi173 systemd[1]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7@prometheus.a.service: Succeeded. 2023-10-01T10:21:59.747 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:21:59 smithi173 systemd[1]: Stopped Ceph prometheus.a for d8016480-6041-11ee-8db5-212e2dc638e7. 2023-10-01T10:21:59.747 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:21:59 smithi173 systemd[1]: Started Ceph prometheus.a for d8016480-6041-11ee-8db5-212e2dc638e7. 2023-10-01T10:22:00.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:00 smithi078 bash[27336]: cluster 2023-10-01T10:21:59.412101+0000 mgr.y (mgr.25115) 29 : cluster [DBG] pgmap v17: 89 pgs: 89 active+clean; 419 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 870 op/s 2023-10-01T10:22:00.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:00 smithi078 bash[27336]: audit 2023-10-01T10:21:59.706037+0000 mon.a (mon.0) 913 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:22:00.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:00 smithi078 bash[27336]: audit 2023-10-01T10:21:59.713319+0000 mon.a (mon.0) 914 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:22:00.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:00 smithi078 bash[27336]: audit 2023-10-01T10:21:59.739986+0000 mon.c (mon.2) 178 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-01T10:22:00.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:00 smithi078 bash[27336]: audit 2023-10-01T10:21:59.751960+0000 mon.c (mon.2) 179 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:22:00.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:00 smithi078 bash[27336]: audit 2023-10-01T10:22:00.384519+0000 mon.c (mon.2) 180 : audit [DBG] from='client.? 172.21.15.78:0/3766526958' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:22:00.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:00 smithi078 bash[33821]: cluster 2023-10-01T10:21:59.412101+0000 mgr.y (mgr.25115) 29 : cluster [DBG] pgmap v17: 89 pgs: 89 active+clean; 419 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 870 op/s 2023-10-01T10:22:00.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:00 smithi078 bash[33821]: audit 2023-10-01T10:21:59.706037+0000 mon.a (mon.0) 913 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:22:00.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:00 smithi078 bash[33821]: audit 2023-10-01T10:21:59.713319+0000 mon.a (mon.0) 914 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:22:00.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:00 smithi078 bash[33821]: audit 2023-10-01T10:21:59.739986+0000 mon.c (mon.2) 178 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-01T10:22:00.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:00 smithi078 bash[33821]: audit 2023-10-01T10:21:59.751960+0000 mon.c (mon.2) 179 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:22:00.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:00 smithi078 bash[33821]: audit 2023-10-01T10:22:00.384519+0000 mon.c (mon.2) 180 : audit [DBG] from='client.? 172.21.15.78:0/3766526958' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:22:00.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:00 smithi173 bash[27612]: cluster 2023-10-01T10:21:59.412101+0000 mgr.y (mgr.25115) 29 : cluster [DBG] pgmap v17: 89 pgs: 89 active+clean; 419 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 870 op/s 2023-10-01T10:22:00.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:00 smithi173 bash[27612]: audit 2023-10-01T10:21:59.706037+0000 mon.a (mon.0) 913 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:22:00.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:00 smithi173 bash[27612]: audit 2023-10-01T10:21:59.713319+0000 mon.a (mon.0) 914 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:22:00.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:00 smithi173 bash[27612]: audit 2023-10-01T10:21:59.739986+0000 mon.c (mon.2) 178 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-01T10:22:00.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:00 smithi173 bash[27612]: audit 2023-10-01T10:21:59.751960+0000 mon.c (mon.2) 179 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:22:00.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:00 smithi173 bash[27612]: audit 2023-10-01T10:22:00.384519+0000 mon.c (mon.2) 180 : audit [DBG] from='client.? 172.21.15.78:0/3766526958' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:22:01.370 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:22:01 smithi173 bash[54497]: ts=2023-10-01T10:22:01.211Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2023-10-01T10:22:01.370 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:22:01 smithi173 bash[54497]: ts=2023-10-01T10:22:01.211Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2023-10-01T10:22:01.370 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:22:01 smithi173 bash[54497]: ts=2023-10-01T10:22:01.211Z caller=main.go:518 level=info host_details="(Linux 5.15.0-84-generic #93~20.04.1-Ubuntu SMP Wed Sep 6 16:15:40 UTC 2023 x86_64 smithi173 (none))" 2023-10-01T10:22:01.370 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:22:01 smithi173 bash[54497]: ts=2023-10-01T10:22:01.211Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-10-01T10:22:01.370 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:22:01 smithi173 bash[54497]: ts=2023-10-01T10:22:01.211Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-10-01T10:22:01.370 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:22:01 smithi173 bash[54497]: ts=2023-10-01T10:22:01.213Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2023-10-01T10:22:01.370 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:22:01 smithi173 bash[54497]: ts=2023-10-01T10:22:01.237Z caller=main.go:923 level=info msg="Starting TSDB ..." 2023-10-01T10:22:01.370 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:22:01 smithi173 bash[54497]: ts=2023-10-01T10:22:01.238Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2023-10-01T10:22:01.370 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:22:01 smithi173 bash[54497]: ts=2023-10-01T10:22:01.242Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-10-01T10:22:01.371 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:22:01 smithi173 bash[54497]: ts=2023-10-01T10:22:01.242Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=4.527µs 2023-10-01T10:22:01.371 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:22:01 smithi173 bash[54497]: ts=2023-10-01T10:22:01.242Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-10-01T10:22:01.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:01 smithi078 bash[33821]: audit 2023-10-01T10:21:59.740673+0000 mgr.y (mgr.25115) 30 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-01T10:22:01.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:01 smithi078 bash[27336]: audit 2023-10-01T10:21:59.740673+0000 mgr.y (mgr.25115) 30 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-01T10:22:01.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:01 smithi173 bash[27612]: audit 2023-10-01T10:21:59.740673+0000 mgr.y (mgr.25115) 30 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-10-01T10:22:01.870 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:22:01 smithi173 bash[54497]: ts=2023-10-01T10:22:01.516Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2023-10-01T10:22:02.370 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:22:01 smithi173 bash[54497]: ts=2023-10-01T10:22:01.980Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2023-10-01T10:22:02.370 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:22:02 smithi173 bash[54497]: ts=2023-10-01T10:22:02.056Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2023-10-01T10:22:02.370 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:22:02 smithi173 bash[54497]: ts=2023-10-01T10:22:02.056Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2023-10-01T10:22:02.370 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:22:02 smithi173 bash[54497]: ts=2023-10-01T10:22:02.056Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=98.243µs wal_replay_duration=813.680903ms total_replay_duration=814.048193ms 2023-10-01T10:22:02.371 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:22:02 smithi173 bash[54497]: ts=2023-10-01T10:22:02.062Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2023-10-01T10:22:02.371 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:22:02 smithi173 bash[54497]: ts=2023-10-01T10:22:02.062Z caller=main.go:947 level=info msg="TSDB started" 2023-10-01T10:22:02.371 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:22:02 smithi173 bash[54497]: ts=2023-10-01T10:22:02.062Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-10-01T10:22:02.371 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:22:02 smithi173 bash[54497]: ts=2023-10-01T10:22:02.093Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=31.134903ms db_storage=1.091µs remote_storage=2.074µs web_handler=383ns query_engine=1.085µs scrape=361.243µs scrape_sd=134.711µs notify=18.95µs notify_sd=15.798µs rules=30.212552ms 2023-10-01T10:22:02.371 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:22:02 smithi173 bash[54497]: ts=2023-10-01T10:22:02.093Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2023-10-01T10:22:02.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:02 smithi078 bash[27336]: cluster 2023-10-01T10:22:01.413771+0000 mgr.y (mgr.25115) 31 : cluster [DBG] pgmap v18: 89 pgs: 89 active+clean; 419 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 578 op/s 2023-10-01T10:22:02.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:02 smithi078 bash[27336]: audit 2023-10-01T10:22:01.557149+0000 mon.c (mon.2) 181 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:22:02.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:02 smithi078 bash[27336]: audit 2023-10-01T10:22:02.287415+0000 mon.c (mon.2) 182 : audit [INF] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-01T10:22:02.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:02 smithi078 bash[27336]: audit 2023-10-01T10:22:02.288109+0000 mon.a (mon.0) 915 : audit [INF] from='mgr.25115 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-01T10:22:02.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:02 smithi078 bash[27336]: audit 2023-10-01T10:22:02.290269+0000 mon.c (mon.2) 183 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-01T10:22:02.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:02 smithi078 bash[27336]: audit 2023-10-01T10:22:02.297540+0000 mon.c (mon.2) 184 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:22:02.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:02 smithi078 bash[33821]: cluster 2023-10-01T10:22:01.413771+0000 mgr.y (mgr.25115) 31 : cluster [DBG] pgmap v18: 89 pgs: 89 active+clean; 419 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 578 op/s 2023-10-01T10:22:02.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:02 smithi078 bash[33821]: audit 2023-10-01T10:22:01.557149+0000 mon.c (mon.2) 181 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:22:02.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:02 smithi078 bash[33821]: audit 2023-10-01T10:22:02.287415+0000 mon.c (mon.2) 182 : audit [INF] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-01T10:22:02.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:02 smithi078 bash[33821]: audit 2023-10-01T10:22:02.288109+0000 mon.a (mon.0) 915 : audit [INF] from='mgr.25115 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-01T10:22:02.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:02 smithi078 bash[33821]: audit 2023-10-01T10:22:02.290269+0000 mon.c (mon.2) 183 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-01T10:22:02.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:02 smithi078 bash[33821]: audit 2023-10-01T10:22:02.297540+0000 mon.c (mon.2) 184 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:22:02.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:02 smithi173 bash[27612]: cluster 2023-10-01T10:22:01.413771+0000 mgr.y (mgr.25115) 31 : cluster [DBG] pgmap v18: 89 pgs: 89 active+clean; 419 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 578 op/s 2023-10-01T10:22:02.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:02 smithi173 bash[27612]: audit 2023-10-01T10:22:01.557149+0000 mon.c (mon.2) 181 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:22:02.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:02 smithi173 bash[27612]: audit 2023-10-01T10:22:02.287415+0000 mon.c (mon.2) 182 : audit [INF] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-01T10:22:02.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:02 smithi173 bash[27612]: audit 2023-10-01T10:22:02.288109+0000 mon.a (mon.0) 915 : audit [INF] from='mgr.25115 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-01T10:22:02.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:02 smithi173 bash[27612]: audit 2023-10-01T10:22:02.290269+0000 mon.c (mon.2) 183 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-01T10:22:02.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:02 smithi173 bash[27612]: audit 2023-10-01T10:22:02.297540+0000 mon.c (mon.2) 184 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:22:03.767 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:03 smithi173 bash[27612]: cephadm 2023-10-01T10:22:02.286629+0000 mgr.y (mgr.25115) 32 : cephadm [INF] Upgrade: Updating mgr.x 2023-10-01T10:22:03.767 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:03 smithi173 bash[27612]: cephadm 2023-10-01T10:22:02.299272+0000 mgr.y (mgr.25115) 33 : cephadm [INF] Deploying daemon mgr.x on smithi173 2023-10-01T10:22:03.767 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:03 smithi173 bash[27612]: audit 2023-10-01T10:22:03.058294+0000 mon.a (mon.0) 916 : audit [DBG] from='client.? 172.21.15.78:0/749396508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:22:03.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:03 smithi078 bash[27336]: cephadm 2023-10-01T10:22:02.286629+0000 mgr.y (mgr.25115) 32 : cephadm [INF] Upgrade: Updating mgr.x 2023-10-01T10:22:03.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:03 smithi078 bash[27336]: cephadm 2023-10-01T10:22:02.299272+0000 mgr.y (mgr.25115) 33 : cephadm [INF] Deploying daemon mgr.x on smithi173 2023-10-01T10:22:03.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:03 smithi078 bash[27336]: audit 2023-10-01T10:22:03.058294+0000 mon.a (mon.0) 916 : audit [DBG] from='client.? 172.21.15.78:0/749396508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:22:03.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:03 smithi078 bash[33821]: cephadm 2023-10-01T10:22:02.286629+0000 mgr.y (mgr.25115) 32 : cephadm [INF] Upgrade: Updating mgr.x 2023-10-01T10:22:03.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:03 smithi078 bash[33821]: cephadm 2023-10-01T10:22:02.299272+0000 mgr.y (mgr.25115) 33 : cephadm [INF] Deploying daemon mgr.x on smithi173 2023-10-01T10:22:03.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:03 smithi078 bash[33821]: audit 2023-10-01T10:22:03.058294+0000 mon.a (mon.0) 916 : audit [DBG] from='client.? 172.21.15.78:0/749396508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:22:04.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:04 smithi078 bash[27336]: cluster 2023-10-01T10:22:03.414537+0000 mgr.y (mgr.25115) 34 : cluster [DBG] pgmap v19: 89 pgs: 89 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 863 op/s 2023-10-01T10:22:04.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:04 smithi078 bash[33821]: cluster 2023-10-01T10:22:03.414537+0000 mgr.y (mgr.25115) 34 : cluster [DBG] pgmap v19: 89 pgs: 89 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 863 op/s 2023-10-01T10:22:04.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:04 smithi173 bash[27612]: cluster 2023-10-01T10:22:03.414537+0000 mgr.y (mgr.25115) 34 : cluster [DBG] pgmap v19: 89 pgs: 89 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 863 op/s 2023-10-01T10:22:05.288 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:05 smithi173 systemd[1]: Stopping Ceph mgr.x for d8016480-6041-11ee-8db5-212e2dc638e7... 2023-10-01T10:22:06.369 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:05 smithi173 bash[54786]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-mgr-x 2023-10-01T10:22:06.370 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:06 smithi173 systemd[1]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7@mgr.x.service: Main process exited, code=exited, status=143/n/a 2023-10-01T10:22:06.370 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:06 smithi173 bash[54845]: Error response from daemon: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-mgr-x 2023-10-01T10:22:06.370 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:06 smithi173 systemd[1]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7@mgr.x.service: Failed with result 'exit-code'. 2023-10-01T10:22:06.370 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:06 smithi173 systemd[1]: Stopped Ceph mgr.x for d8016480-6041-11ee-8db5-212e2dc638e7. 2023-10-01T10:22:06.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:06 smithi078 bash[27336]: cluster 2023-10-01T10:22:05.414955+0000 mgr.y (mgr.25115) 35 : cluster [DBG] pgmap v20: 89 pgs: 89 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 576 op/s 2023-10-01T10:22:06.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:06 smithi078 bash[27336]: audit 2023-10-01T10:22:05.728470+0000 mon.a (mon.0) 917 : audit [DBG] from='client.? 172.21.15.78:0/2543680317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:22:06.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:06 smithi078 bash[33821]: cluster 2023-10-01T10:22:05.414955+0000 mgr.y (mgr.25115) 35 : cluster [DBG] pgmap v20: 89 pgs: 89 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 576 op/s 2023-10-01T10:22:06.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:06 smithi078 bash[33821]: audit 2023-10-01T10:22:05.728470+0000 mon.a (mon.0) 917 : audit [DBG] from='client.? 172.21.15.78:0/2543680317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:22:06.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:06 smithi173 bash[27612]: cluster 2023-10-01T10:22:05.414955+0000 mgr.y (mgr.25115) 35 : cluster [DBG] pgmap v20: 89 pgs: 89 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 576 op/s 2023-10-01T10:22:06.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:06 smithi173 bash[27612]: audit 2023-10-01T10:22:05.728470+0000 mon.a (mon.0) 917 : audit [DBG] from='client.? 172.21.15.78:0/2543680317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:22:06.870 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:06 smithi173 systemd[1]: Started Ceph mgr.x for d8016480-6041-11ee-8db5-212e2dc638e7. 2023-10-01T10:22:07.277 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Removed 111893 objects 2023-10-01T10:22:07.278 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Clean up completed and total clean up time :160.897 2023-10-01T10:22:07.809 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:07 smithi173 bash[27612]: audit 2023-10-01T10:22:06.530677+0000 mon.a (mon.0) 918 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:22:07.809 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:07 smithi173 bash[27612]: audit 2023-10-01T10:22:06.545920+0000 mon.a (mon.0) 919 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:22:07.809 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:07 smithi173 bash[27612]: audit 2023-10-01T10:22:06.581535+0000 mon.c (mon.2) 185 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:22:07.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:07 smithi078 bash[33821]: audit 2023-10-01T10:22:06.530677+0000 mon.a (mon.0) 918 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:22:07.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:07 smithi078 bash[33821]: audit 2023-10-01T10:22:06.545920+0000 mon.a (mon.0) 919 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:22:07.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:07 smithi078 bash[33821]: audit 2023-10-01T10:22:06.581535+0000 mon.c (mon.2) 185 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:22:07.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:07 smithi078 bash[27336]: audit 2023-10-01T10:22:06.530677+0000 mon.a (mon.0) 918 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:22:07.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:07 smithi078 bash[27336]: audit 2023-10-01T10:22:06.545920+0000 mon.a (mon.0) 919 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:22:07.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:07 smithi078 bash[27336]: audit 2023-10-01T10:22:06.581535+0000 mon.c (mon.2) 185 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:22:08.119 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:07 smithi173 bash[54895]: debug 2023-10-01T10:22:07.804+0000 7efd89696700 1 -- 172.21.15.173:0/119621224 <== mon.0 v2:172.21.15.78:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5617967ef1e0 con 0x56179756e800 2023-10-01T10:22:08.120 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:07 smithi173 bash[54895]: debug 2023-10-01T10:22:07.964+0000 7efd937e4200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-01T10:22:08.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:08 smithi078 bash[27336]: cluster 2023-10-01T10:22:07.415675+0000 mgr.y (mgr.25115) 36 : cluster [DBG] pgmap v21: 89 pgs: 89 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 576 op/s 2023-10-01T10:22:08.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:08 smithi078 bash[27336]: audit 2023-10-01T10:22:08.193706+0000 mon.c (mon.2) 186 : audit [DBG] from='client.? 172.21.15.78:0/1126825503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:22:08.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:08 smithi078 bash[33821]: cluster 2023-10-01T10:22:07.415675+0000 mgr.y (mgr.25115) 36 : cluster [DBG] pgmap v21: 89 pgs: 89 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 576 op/s 2023-10-01T10:22:08.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:08 smithi078 bash[33821]: audit 2023-10-01T10:22:08.193706+0000 mon.c (mon.2) 186 : audit [DBG] from='client.? 172.21.15.78:0/1126825503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:22:08.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:08 smithi173 bash[27612]: cluster 2023-10-01T10:22:07.415675+0000 mgr.y (mgr.25115) 36 : cluster [DBG] pgmap v21: 89 pgs: 89 active+clean; 205 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 576 op/s 2023-10-01T10:22:08.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:08 smithi173 bash[27612]: audit 2023-10-01T10:22:08.193706+0000 mon.c (mon.2) 186 : audit [DBG] from='client.? 172.21.15.78:0/1126825503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:22:09.349 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:08 smithi173 bash[54895]: debug 2023-10-01T10:22:08.868+0000 7efd937e4200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-01T10:22:09.349 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:08 smithi173 bash[54895]: debug 2023-10-01T10:22:08.980+0000 7efd937e4200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-01T10:22:09.349 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:09 smithi173 bash[54895]: debug 2023-10-01T10:22:09.084+0000 7efd937e4200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-01T10:22:09.619 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:09 smithi173 bash[54895]: debug 2023-10-01T10:22:09.344+0000 7efd937e4200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-01T10:22:10.119 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:09 smithi173 bash[54895]: debug 2023-10-01T10:22:09.728+0000 7efd937e4200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-01T10:22:10.120 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:09 smithi173 bash[54895]: debug 2023-10-01T10:22:09.904+0000 7efd937e4200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-01T10:22:10.553 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:22:10 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:22:10] "GET /metrics HTTP/1.1" 200 36072 "" "Prometheus/2.33.4" 2023-10-01T10:22:10.553 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:10 smithi173 bash[54895]: debug 2023-10-01T10:22:10.128+0000 7efd937e4200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-01T10:22:10.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:10 smithi078 bash[27336]: cluster 2023-10-01T10:22:09.416837+0000 mgr.y (mgr.25115) 37 : cluster [DBG] pgmap v22: 89 pgs: 89 active+clean; 581 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 849 op/s 2023-10-01T10:22:10.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:10 smithi078 bash[33821]: cluster 2023-10-01T10:22:09.416837+0000 mgr.y (mgr.25115) 37 : cluster [DBG] pgmap v22: 89 pgs: 89 active+clean; 581 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 849 op/s 2023-10-01T10:22:10.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:10 smithi173 bash[27612]: cluster 2023-10-01T10:22:09.416837+0000 mgr.y (mgr.25115) 37 : cluster [DBG] pgmap v22: 89 pgs: 89 active+clean; 581 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 849 op/s 2023-10-01T10:22:11.555 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:11 smithi173 bash[54895]: debug 2023-10-01T10:22:11.148+0000 7efd937e4200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-01T10:22:11.555 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:11 smithi173 bash[54895]: debug 2023-10-01T10:22:11.260+0000 7efd937e4200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-01T10:22:11.556 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:11 smithi173 bash[54895]: debug 2023-10-01T10:22:11.380+0000 7efd937e4200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-01T10:22:11.556 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:11 smithi173 bash[27612]: audit 2023-10-01T10:22:10.642125+0000 mon.a (mon.0) 920 : audit [DBG] from='client.? 172.21.15.78:0/381786601' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:22:11.573 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_0 2023-10-01T10:22:11.574 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_0 unique_pool_0 --yes-i-really-really-mean-it 2023-10-01T10:22:11.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:11 smithi078 bash[27336]: audit 2023-10-01T10:22:10.642125+0000 mon.a (mon.0) 920 : audit [DBG] from='client.? 172.21.15.78:0/381786601' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:22:11.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:11 smithi078 bash[33821]: audit 2023-10-01T10:22:10.642125+0000 mon.a (mon.0) 920 : audit [DBG] from='client.? 172.21.15.78:0/381786601' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:22:12.369 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:11 smithi173 bash[54895]: debug 2023-10-01T10:22:11.880+0000 7efd937e4200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-01T10:22:12.369 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:12 smithi173 bash[54895]: debug 2023-10-01T10:22:12.124+0000 7efd937e4200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-01T10:22:12.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:12 smithi078 bash[27336]: cluster 2023-10-01T10:22:11.417569+0000 mgr.y (mgr.25115) 38 : cluster [DBG] pgmap v23: 89 pgs: 89 active+clean; 581 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 557 op/s 2023-10-01T10:22:12.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:12 smithi078 bash[33821]: cluster 2023-10-01T10:22:11.417569+0000 mgr.y (mgr.25115) 38 : cluster [DBG] pgmap v23: 89 pgs: 89 active+clean; 581 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 557 op/s 2023-10-01T10:22:12.869 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:12 smithi173 bash[54895]: debug 2023-10-01T10:22:12.644+0000 7efd937e4200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-01T10:22:12.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:12 smithi173 bash[27612]: cluster 2023-10-01T10:22:11.417569+0000 mgr.y (mgr.25115) 38 : cluster [DBG] pgmap v23: 89 pgs: 89 active+clean; 581 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 557 op/s 2023-10-01T10:22:13.619 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:13 smithi173 bash[54895]: debug 2023-10-01T10:22:13.188+0000 7efd937e4200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-01T10:22:14.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:13 smithi078 bash[27336]: audit 2023-10-01T10:22:12.830060+0000 mon.a (mon.0) 921 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:22:14.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:13 smithi078 bash[27336]: audit 2023-10-01T10:22:12.839548+0000 mon.a (mon.0) 922 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:22:14.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:13 smithi078 bash[27336]: audit 2023-10-01T10:22:13.088714+0000 mon.a (mon.0) 923 : audit [DBG] from='client.? 172.21.15.78:0/291165188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:22:14.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:13 smithi078 bash[27336]: cluster 2023-10-01T10:22:13.418766+0000 mgr.y (mgr.25115) 39 : cluster [DBG] pgmap v24: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 557 op/s 2023-10-01T10:22:14.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:13 smithi078 bash[27336]: audit 2023-10-01T10:22:13.633415+0000 mon.a (mon.0) 924 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:22:14.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:13 smithi078 bash[27336]: audit 2023-10-01T10:22:13.646564+0000 mon.a (mon.0) 925 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:22:14.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:13 smithi078 bash[33821]: audit 2023-10-01T10:22:12.830060+0000 mon.a (mon.0) 921 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:22:14.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:13 smithi078 bash[33821]: audit 2023-10-01T10:22:12.839548+0000 mon.a (mon.0) 922 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:22:14.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:13 smithi078 bash[33821]: audit 2023-10-01T10:22:13.088714+0000 mon.a (mon.0) 923 : audit [DBG] from='client.? 172.21.15.78:0/291165188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:22:14.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:13 smithi078 bash[33821]: cluster 2023-10-01T10:22:13.418766+0000 mgr.y (mgr.25115) 39 : cluster [DBG] pgmap v24: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 557 op/s 2023-10-01T10:22:14.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:13 smithi078 bash[33821]: audit 2023-10-01T10:22:13.633415+0000 mon.a (mon.0) 924 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:22:14.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:13 smithi078 bash[33821]: audit 2023-10-01T10:22:13.646564+0000 mon.a (mon.0) 925 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:22:14.119 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:13 smithi173 bash[54895]: debug 2023-10-01T10:22:13.812+0000 7efd937e4200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-01T10:22:14.120 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:13 smithi173 bash[54895]: debug 2023-10-01T10:22:13.944+0000 7efd937e4200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-01T10:22:14.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:13 smithi173 bash[27612]: audit 2023-10-01T10:22:12.830060+0000 mon.a (mon.0) 921 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:22:14.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:13 smithi173 bash[27612]: audit 2023-10-01T10:22:12.839548+0000 mon.a (mon.0) 922 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:22:14.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:13 smithi173 bash[27612]: audit 2023-10-01T10:22:13.088714+0000 mon.a (mon.0) 923 : audit [DBG] from='client.? 172.21.15.78:0/291165188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:22:14.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:13 smithi173 bash[27612]: cluster 2023-10-01T10:22:13.418766+0000 mgr.y (mgr.25115) 39 : cluster [DBG] pgmap v24: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 557 op/s 2023-10-01T10:22:14.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:13 smithi173 bash[27612]: audit 2023-10-01T10:22:13.633415+0000 mon.a (mon.0) 924 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:22:14.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:13 smithi173 bash[27612]: audit 2023-10-01T10:22:13.646564+0000 mon.a (mon.0) 925 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:22:14.572 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:14 smithi173 bash[54895]: debug 2023-10-01T10:22:14.304+0000 7efd937e4200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-01T10:22:14.572 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:14 smithi173 bash[54895]: debug 2023-10-01T10:22:14.424+0000 7efd937e4200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-01T10:22:14.869 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:14 smithi173 bash[54895]: debug 2023-10-01T10:22:14.568+0000 7efd937e4200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-01T10:22:14.869 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:14 smithi173 bash[54895]: debug 2023-10-01T10:22:14.684+0000 7efd937e4200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-01T10:22:14.870 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:14 smithi173 bash[54895]: debug 2023-10-01T10:22:14.808+0000 7efd937e4200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-01T10:22:15.369 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:15 smithi173 bash[54895]: debug 2023-10-01T10:22:15.004+0000 7efd937e4200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-01T10:22:15.370 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:15 smithi173 bash[54895]: debug 2023-10-01T10:22:15.128+0000 7efd937e4200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-01T10:22:16.351 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:22:16.620 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:16 smithi173 bash[54895]: debug 2023-10-01T10:22:16.352+0000 7efd937e4200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-01T10:22:16.620 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:16 smithi173 bash[54895]: [01/Oct/2023:10:22:16] ENGINE Bus STARTING 2023-10-01T10:22:16.620 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:16 smithi173 bash[54895]: CherryPy Checker: 2023-10-01T10:22:16.620 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:16 smithi173 bash[54895]: The Application mounted at '' has an empty config. 2023-10-01T10:22:16.620 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:16 smithi173 bash[54895]: [01/Oct/2023:10:22:16] ENGINE Serving on http://:::9283 2023-10-01T10:22:16.620 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:16 smithi173 bash[54895]: [01/Oct/2023:10:22:16] ENGINE Bus STARTED 2023-10-01T10:22:16.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:16 smithi173 bash[27612]: cluster 2023-10-01T10:22:15.419418+0000 mgr.y (mgr.25115) 40 : cluster [DBG] pgmap v25: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 272 op/s 2023-10-01T10:22:16.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:16 smithi173 bash[27612]: audit 2023-10-01T10:22:15.535570+0000 mon.a (mon.0) 926 : audit [DBG] from='client.? 172.21.15.78:0/443859839' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:22:16.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:16 smithi173 bash[27612]: cluster 2023-10-01T10:22:16.374979+0000 mon.a (mon.0) 927 : cluster [DBG] Standby manager daemon x restarted 2023-10-01T10:22:16.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:16 smithi173 bash[27612]: cluster 2023-10-01T10:22:16.375217+0000 mon.a (mon.0) 928 : cluster [DBG] Standby manager daemon x started 2023-10-01T10:22:16.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:16 smithi173 bash[27612]: audit 2023-10-01T10:22:16.383179+0000 mon.b (mon.1) 49 : audit [DBG] from='mgr.? 172.21.15.173:0/1506323087' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-01T10:22:16.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:16 smithi173 bash[27612]: audit 2023-10-01T10:22:16.384151+0000 mon.b (mon.1) 50 : audit [DBG] from='mgr.? 172.21.15.173:0/1506323087' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-01T10:22:16.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:16 smithi173 bash[27612]: audit 2023-10-01T10:22:16.388281+0000 mon.b (mon.1) 51 : audit [DBG] from='mgr.? 172.21.15.173:0/1506323087' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-01T10:22:16.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:16 smithi173 bash[27612]: audit 2023-10-01T10:22:16.388901+0000 mon.b (mon.1) 52 : audit [DBG] from='mgr.? 172.21.15.173:0/1506323087' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-01T10:22:16.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:16 smithi078 bash[33821]: cluster 2023-10-01T10:22:15.419418+0000 mgr.y (mgr.25115) 40 : cluster [DBG] pgmap v25: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 272 op/s 2023-10-01T10:22:16.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:16 smithi078 bash[33821]: audit 2023-10-01T10:22:15.535570+0000 mon.a (mon.0) 926 : audit [DBG] from='client.? 172.21.15.78:0/443859839' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:22:16.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:16 smithi078 bash[33821]: cluster 2023-10-01T10:22:16.374979+0000 mon.a (mon.0) 927 : cluster [DBG] Standby manager daemon x restarted 2023-10-01T10:22:16.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:16 smithi078 bash[33821]: cluster 2023-10-01T10:22:16.375217+0000 mon.a (mon.0) 928 : cluster [DBG] Standby manager daemon x started 2023-10-01T10:22:16.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:16 smithi078 bash[33821]: audit 2023-10-01T10:22:16.383179+0000 mon.b (mon.1) 49 : audit [DBG] from='mgr.? 172.21.15.173:0/1506323087' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-01T10:22:16.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:16 smithi078 bash[33821]: audit 2023-10-01T10:22:16.384151+0000 mon.b (mon.1) 50 : audit [DBG] from='mgr.? 172.21.15.173:0/1506323087' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-01T10:22:16.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:16 smithi078 bash[33821]: audit 2023-10-01T10:22:16.388281+0000 mon.b (mon.1) 51 : audit [DBG] from='mgr.? 172.21.15.173:0/1506323087' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-01T10:22:16.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:16 smithi078 bash[33821]: audit 2023-10-01T10:22:16.388901+0000 mon.b (mon.1) 52 : audit [DBG] from='mgr.? 172.21.15.173:0/1506323087' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-01T10:22:16.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:16 smithi078 bash[27336]: cluster 2023-10-01T10:22:15.419418+0000 mgr.y (mgr.25115) 40 : cluster [DBG] pgmap v25: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 272 op/s 2023-10-01T10:22:16.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:16 smithi078 bash[27336]: audit 2023-10-01T10:22:15.535570+0000 mon.a (mon.0) 926 : audit [DBG] from='client.? 172.21.15.78:0/443859839' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:22:16.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:16 smithi078 bash[27336]: cluster 2023-10-01T10:22:16.374979+0000 mon.a (mon.0) 927 : cluster [DBG] Standby manager daemon x restarted 2023-10-01T10:22:16.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:16 smithi078 bash[27336]: cluster 2023-10-01T10:22:16.375217+0000 mon.a (mon.0) 928 : cluster [DBG] Standby manager daemon x started 2023-10-01T10:22:16.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:16 smithi078 bash[27336]: audit 2023-10-01T10:22:16.383179+0000 mon.b (mon.1) 49 : audit [DBG] from='mgr.? 172.21.15.173:0/1506323087' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-01T10:22:16.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:16 smithi078 bash[27336]: audit 2023-10-01T10:22:16.384151+0000 mon.b (mon.1) 50 : audit [DBG] from='mgr.? 172.21.15.173:0/1506323087' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-01T10:22:16.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:16 smithi078 bash[27336]: audit 2023-10-01T10:22:16.388281+0000 mon.b (mon.1) 51 : audit [DBG] from='mgr.? 172.21.15.173:0/1506323087' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-01T10:22:16.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:16 smithi078 bash[27336]: audit 2023-10-01T10:22:16.388901+0000 mon.b (mon.1) 52 : audit [DBG] from='mgr.? 172.21.15.173:0/1506323087' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-01T10:22:17.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:17 smithi078 bash[33821]: audit 2023-10-01T10:22:16.558490+0000 mon.c (mon.2) 187 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:22:17.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:17 smithi078 bash[33821]: cluster 2023-10-01T10:22:17.144939+0000 mon.a (mon.0) 929 : cluster [DBG] mgrmap e29: y(active, since 45s), standbys: x 2023-10-01T10:22:17.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:17 smithi078 bash[27336]: audit 2023-10-01T10:22:16.558490+0000 mon.c (mon.2) 187 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:22:17.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:17 smithi078 bash[27336]: cluster 2023-10-01T10:22:17.144939+0000 mon.a (mon.0) 929 : cluster [DBG] mgrmap e29: y(active, since 45s), standbys: x 2023-10-01T10:22:17.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:17 smithi173 bash[27612]: audit 2023-10-01T10:22:16.558490+0000 mon.c (mon.2) 187 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:22:17.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:17 smithi173 bash[27612]: cluster 2023-10-01T10:22:17.144939+0000 mon.a (mon.0) 929 : cluster [DBG] mgrmap e29: y(active, since 45s), standbys: x 2023-10-01T10:22:18.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:18 smithi078 bash[33821]: cluster 2023-10-01T10:22:17.420104+0000 mgr.y (mgr.25115) 41 : cluster [DBG] pgmap v26: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 272 op/s 2023-10-01T10:22:18.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:18 smithi078 bash[33821]: audit 2023-10-01T10:22:17.962833+0000 mon.c (mon.2) 188 : audit [DBG] from='client.? 172.21.15.78:0/1780330050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:22:18.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:18 smithi078 bash[27336]: cluster 2023-10-01T10:22:17.420104+0000 mgr.y (mgr.25115) 41 : cluster [DBG] pgmap v26: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 272 op/s 2023-10-01T10:22:18.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:18 smithi078 bash[27336]: audit 2023-10-01T10:22:17.962833+0000 mon.c (mon.2) 188 : audit [DBG] from='client.? 172.21.15.78:0/1780330050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:22:18.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:18 smithi173 bash[27612]: cluster 2023-10-01T10:22:17.420104+0000 mgr.y (mgr.25115) 41 : cluster [DBG] pgmap v26: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 272 op/s 2023-10-01T10:22:18.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:18 smithi173 bash[27612]: audit 2023-10-01T10:22:17.962833+0000 mon.c (mon.2) 188 : audit [DBG] from='client.? 172.21.15.78:0/1780330050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:22:18.870 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:22:18 smithi173 bash[54497]: ts=2023-10-01T10:22:18.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"smithi173.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"smithi078:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-01T10:22:20.582 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:22:20 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:22:20] "GET /metrics HTTP/1.1" 200 36023 "" "Prometheus/2.33.4" 2023-10-01T10:22:20.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:20 smithi078 bash[27336]: cluster 2023-10-01T10:22:19.421305+0000 mgr.y (mgr.25115) 42 : cluster [DBG] pgmap v27: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 272 op/s 2023-10-01T10:22:20.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:20 smithi078 bash[27336]: audit 2023-10-01T10:22:20.396112+0000 mon.c (mon.2) 189 : audit [DBG] from='client.? 172.21.15.78:0/407569777' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:22:20.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:20 smithi078 bash[33821]: cluster 2023-10-01T10:22:19.421305+0000 mgr.y (mgr.25115) 42 : cluster [DBG] pgmap v27: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 272 op/s 2023-10-01T10:22:20.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:20 smithi078 bash[33821]: audit 2023-10-01T10:22:20.396112+0000 mon.c (mon.2) 189 : audit [DBG] from='client.? 172.21.15.78:0/407569777' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:22:20.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:20 smithi173 bash[27612]: cluster 2023-10-01T10:22:19.421305+0000 mgr.y (mgr.25115) 42 : cluster [DBG] pgmap v27: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 272 op/s 2023-10-01T10:22:20.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:20 smithi173 bash[27612]: audit 2023-10-01T10:22:20.396112+0000 mon.c (mon.2) 189 : audit [DBG] from='client.? 172.21.15.78:0/407569777' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:22:21.513 INFO:teuthology.orchestra.run.smithi078.stderr:pool 'unique_pool_0' removed 2023-10-01T10:22:21.758 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:21 smithi078 bash[27336]: audit 2023-10-01T10:22:21.114140+0000 mon.c (mon.2) 190 : audit [INF] from='client.? 172.21.15.78:0/1457464531' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T10:22:21.758 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:21 smithi078 bash[27336]: audit 2023-10-01T10:22:21.114824+0000 mon.a (mon.0) 930 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T10:22:21.758 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:21 smithi078 bash[33821]: audit 2023-10-01T10:22:21.114140+0000 mon.c (mon.2) 190 : audit [INF] from='client.? 172.21.15.78:0/1457464531' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T10:22:21.759 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:21 smithi078 bash[33821]: audit 2023-10-01T10:22:21.114824+0000 mon.a (mon.0) 930 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T10:22:21.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:21 smithi173 bash[27612]: audit 2023-10-01T10:22:21.114140+0000 mon.c (mon.2) 190 : audit [INF] from='client.? 172.21.15.78:0/1457464531' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T10:22:21.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:21 smithi173 bash[27612]: audit 2023-10-01T10:22:21.114824+0000 mon.a (mon.0) 930 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T10:22:22.368 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-01T10:22:22.368 INFO:tasks.radosbench:Beginning radosbench... 2023-10-01T10:22:22.369 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_1 2023-10-01T10:22:22.369 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_1 16 2023-10-01T10:22:22.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:22 smithi078 bash[33821]: cluster 2023-10-01T10:22:21.421953+0000 mgr.y (mgr.25115) 43 : cluster [DBG] pgmap v28: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:22:22.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:22 smithi078 bash[33821]: audit 2023-10-01T10:22:21.503507+0000 mon.a (mon.0) 931 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-10-01T10:22:22.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:22 smithi078 bash[33821]: cluster 2023-10-01T10:22:21.503568+0000 mon.a (mon.0) 932 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2023-10-01T10:22:22.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:22 smithi078 bash[27336]: cluster 2023-10-01T10:22:21.421953+0000 mgr.y (mgr.25115) 43 : cluster [DBG] pgmap v28: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:22:22.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:22 smithi078 bash[27336]: audit 2023-10-01T10:22:21.503507+0000 mon.a (mon.0) 931 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-10-01T10:22:22.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:22 smithi078 bash[27336]: cluster 2023-10-01T10:22:21.503568+0000 mon.a (mon.0) 932 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2023-10-01T10:22:22.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:22 smithi173 bash[27612]: cluster 2023-10-01T10:22:21.421953+0000 mgr.y (mgr.25115) 43 : cluster [DBG] pgmap v28: 89 pgs: 89 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:22:22.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:22 smithi173 bash[27612]: audit 2023-10-01T10:22:21.503507+0000 mon.a (mon.0) 931 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-10-01T10:22:22.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:22 smithi173 bash[27612]: cluster 2023-10-01T10:22:21.503568+0000 mon.a (mon.0) 932 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2023-10-01T10:22:24.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:23 smithi173 bash[27612]: audit 2023-10-01T10:22:22.852546+0000 mon.a (mon.0) 933 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:22:24.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:23 smithi173 bash[27612]: audit 2023-10-01T10:22:22.852898+0000 mon.a (mon.0) 934 : audit [DBG] from='client.? 172.21.15.78:0/4074380738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:22:24.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:23 smithi173 bash[27612]: audit 2023-10-01T10:22:22.862496+0000 mon.a (mon.0) 935 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:22:24.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:23 smithi173 bash[27612]: audit 2023-10-01T10:22:22.864581+0000 mon.c (mon.2) 191 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:22:24.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:23 smithi173 bash[27612]: audit 2023-10-01T10:22:22.865775+0000 mon.c (mon.2) 192 : audit [INF] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:22:24.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:23 smithi173 bash[27612]: audit 2023-10-01T10:22:22.872970+0000 mon.a (mon.0) 936 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:22:24.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:23 smithi173 bash[27612]: audit 2023-10-01T10:22:22.953846+0000 mon.c (mon.2) 193 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:22:24.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:23 smithi173 bash[27612]: audit 2023-10-01T10:22:22.959665+0000 mon.c (mon.2) 194 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:22:24.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:23 smithi173 bash[27612]: cephadm 2023-10-01T10:22:22.960834+0000 mgr.y (mgr.25115) 44 : cephadm [INF] Upgrade: Setting container_image for all mgr 2023-10-01T10:22:24.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:23 smithi173 bash[27612]: audit 2023-10-01T10:22:22.968960+0000 mon.a (mon.0) 937 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:22:24.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:23 smithi173 bash[27612]: audit 2023-10-01T10:22:22.970604+0000 mon.c (mon.2) 195 : audit [INF] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-01T10:22:24.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:23 smithi173 bash[27612]: audit 2023-10-01T10:22:22.971032+0000 mon.a (mon.0) 938 : audit [INF] from='mgr.25115 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-01T10:22:24.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:23 smithi173 bash[27612]: audit 2023-10-01T10:22:22.975553+0000 mon.a (mon.0) 939 : audit [INF] from='mgr.25115 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-10-01T10:22:24.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:23 smithi173 bash[27612]: audit 2023-10-01T10:22:22.976922+0000 mon.c (mon.2) 196 : audit [INF] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-01T10:22:24.122 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:23 smithi173 bash[27612]: audit 2023-10-01T10:22:22.977403+0000 mon.a (mon.0) 940 : audit [INF] from='mgr.25115 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-01T10:22:24.122 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:23 smithi173 bash[27612]: audit 2023-10-01T10:22:22.981819+0000 mon.a (mon.0) 941 : audit [INF] from='mgr.25115 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-10-01T10:22:24.122 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:23 smithi173 bash[27612]: audit 2023-10-01T10:22:22.984174+0000 mon.c (mon.2) 197 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-01T10:22:24.122 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:23 smithi173 bash[27612]: audit 2023-10-01T10:22:22.985376+0000 mon.c (mon.2) 198 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-10-01T10:22:24.122 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:23 smithi173 bash[27612]: cephadm 2023-10-01T10:22:22.986335+0000 mgr.y (mgr.25115) 45 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2023-10-01T10:22:24.122 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:23 smithi173 bash[27612]: cluster 2023-10-01T10:22:23.423086+0000 mgr.y (mgr.25115) 46 : cluster [DBG] pgmap v30: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:22:24.243 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:23 smithi078 bash[33821]: audit 2023-10-01T10:22:22.852546+0000 mon.a (mon.0) 933 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:22:24.243 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:23 smithi078 bash[33821]: audit 2023-10-01T10:22:22.852898+0000 mon.a (mon.0) 934 : audit [DBG] from='client.? 172.21.15.78:0/4074380738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:22:24.243 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:23 smithi078 bash[33821]: audit 2023-10-01T10:22:22.862496+0000 mon.a (mon.0) 935 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:22:24.244 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:23 smithi078 bash[33821]: audit 2023-10-01T10:22:22.864581+0000 mon.c (mon.2) 191 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:22:24.244 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:23 smithi078 bash[33821]: audit 2023-10-01T10:22:22.865775+0000 mon.c (mon.2) 192 : audit [INF] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:22:24.244 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:23 smithi078 bash[33821]: audit 2023-10-01T10:22:22.872970+0000 mon.a (mon.0) 936 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:22:24.244 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:23 smithi078 bash[33821]: audit 2023-10-01T10:22:22.953846+0000 mon.c (mon.2) 193 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:22:24.244 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:23 smithi078 bash[33821]: audit 2023-10-01T10:22:22.959665+0000 mon.c (mon.2) 194 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:22:24.244 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:23 smithi078 bash[33821]: cephadm 2023-10-01T10:22:22.960834+0000 mgr.y (mgr.25115) 44 : cephadm [INF] Upgrade: Setting container_image for all mgr 2023-10-01T10:22:24.244 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:23 smithi078 bash[33821]: audit 2023-10-01T10:22:22.968960+0000 mon.a (mon.0) 937 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:22:24.244 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:23 smithi078 bash[33821]: audit 2023-10-01T10:22:22.970604+0000 mon.c (mon.2) 195 : audit [INF] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-01T10:22:24.244 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:23 smithi078 bash[33821]: audit 2023-10-01T10:22:22.971032+0000 mon.a (mon.0) 938 : audit [INF] from='mgr.25115 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-01T10:22:24.244 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:23 smithi078 bash[33821]: audit 2023-10-01T10:22:22.975553+0000 mon.a (mon.0) 939 : audit [INF] from='mgr.25115 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-10-01T10:22:24.244 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:23 smithi078 bash[33821]: audit 2023-10-01T10:22:22.976922+0000 mon.c (mon.2) 196 : audit [INF] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-01T10:22:24.245 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:23 smithi078 bash[33821]: audit 2023-10-01T10:22:22.977403+0000 mon.a (mon.0) 940 : audit [INF] from='mgr.25115 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-01T10:22:24.245 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:23 smithi078 bash[33821]: audit 2023-10-01T10:22:22.981819+0000 mon.a (mon.0) 941 : audit [INF] from='mgr.25115 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-10-01T10:22:24.245 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:23 smithi078 bash[33821]: audit 2023-10-01T10:22:22.984174+0000 mon.c (mon.2) 197 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-01T10:22:24.245 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:23 smithi078 bash[33821]: audit 2023-10-01T10:22:22.985376+0000 mon.c (mon.2) 198 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-10-01T10:22:24.245 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:23 smithi078 bash[33821]: cephadm 2023-10-01T10:22:22.986335+0000 mgr.y (mgr.25115) 45 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2023-10-01T10:22:24.245 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:23 smithi078 bash[33821]: cluster 2023-10-01T10:22:23.423086+0000 mgr.y (mgr.25115) 46 : cluster [DBG] pgmap v30: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:22:24.245 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:23 smithi078 bash[27336]: audit 2023-10-01T10:22:22.852546+0000 mon.a (mon.0) 933 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:22:24.245 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:23 smithi078 bash[27336]: audit 2023-10-01T10:22:22.852898+0000 mon.a (mon.0) 934 : audit [DBG] from='client.? 172.21.15.78:0/4074380738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:22:24.246 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:23 smithi078 bash[27336]: audit 2023-10-01T10:22:22.862496+0000 mon.a (mon.0) 935 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:22:24.246 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:23 smithi078 bash[27336]: audit 2023-10-01T10:22:22.864581+0000 mon.c (mon.2) 191 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:22:24.246 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:23 smithi078 bash[27336]: audit 2023-10-01T10:22:22.865775+0000 mon.c (mon.2) 192 : audit [INF] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:22:24.246 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:23 smithi078 bash[27336]: audit 2023-10-01T10:22:22.872970+0000 mon.a (mon.0) 936 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:22:24.246 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:23 smithi078 bash[27336]: audit 2023-10-01T10:22:22.953846+0000 mon.c (mon.2) 193 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:22:24.246 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:23 smithi078 bash[27336]: audit 2023-10-01T10:22:22.959665+0000 mon.c (mon.2) 194 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:22:24.246 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:23 smithi078 bash[27336]: cephadm 2023-10-01T10:22:22.960834+0000 mgr.y (mgr.25115) 44 : cephadm [INF] Upgrade: Setting container_image for all mgr 2023-10-01T10:22:24.246 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:23 smithi078 bash[27336]: audit 2023-10-01T10:22:22.968960+0000 mon.a (mon.0) 937 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:22:24.246 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:23 smithi078 bash[27336]: audit 2023-10-01T10:22:22.970604+0000 mon.c (mon.2) 195 : audit [INF] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-01T10:22:24.246 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:23 smithi078 bash[27336]: audit 2023-10-01T10:22:22.971032+0000 mon.a (mon.0) 938 : audit [INF] from='mgr.25115 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-10-01T10:22:24.246 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:23 smithi078 bash[27336]: audit 2023-10-01T10:22:22.975553+0000 mon.a (mon.0) 939 : audit [INF] from='mgr.25115 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-10-01T10:22:24.247 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:23 smithi078 bash[27336]: audit 2023-10-01T10:22:22.976922+0000 mon.c (mon.2) 196 : audit [INF] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-01T10:22:24.247 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:23 smithi078 bash[27336]: audit 2023-10-01T10:22:22.977403+0000 mon.a (mon.0) 940 : audit [INF] from='mgr.25115 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-10-01T10:22:24.247 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:23 smithi078 bash[27336]: audit 2023-10-01T10:22:22.981819+0000 mon.a (mon.0) 941 : audit [INF] from='mgr.25115 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-10-01T10:22:24.247 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:23 smithi078 bash[27336]: audit 2023-10-01T10:22:22.984174+0000 mon.c (mon.2) 197 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-01T10:22:24.247 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:23 smithi078 bash[27336]: audit 2023-10-01T10:22:22.985376+0000 mon.c (mon.2) 198 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-10-01T10:22:24.247 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:23 smithi078 bash[27336]: cephadm 2023-10-01T10:22:22.986335+0000 mgr.y (mgr.25115) 45 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2023-10-01T10:22:24.247 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:23 smithi078 bash[27336]: cluster 2023-10-01T10:22:23.423086+0000 mgr.y (mgr.25115) 46 : cluster [DBG] pgmap v30: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:22:25.609 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:25 smithi078 bash[33821]: audit 2023-10-01T10:22:25.298956+0000 mon.c (mon.2) 199 : audit [DBG] from='client.? 172.21.15.78:0/3308543545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:22:25.610 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:25 smithi078 bash[33821]: audit 2023-10-01T10:22:25.315787+0000 mon.a (mon.0) 942 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:22:25.610 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:25 smithi078 bash[33821]: audit 2023-10-01T10:22:25.317412+0000 mon.c (mon.2) 200 : audit [INF] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T10:22:25.610 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:25 smithi078 bash[33821]: audit 2023-10-01T10:22:25.318393+0000 mon.c (mon.2) 201 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T10:22:25.610 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:25 smithi078 bash[33821]: audit 2023-10-01T10:22:25.319565+0000 mon.c (mon.2) 202 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:22:25.610 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:25 smithi078 bash[27336]: audit 2023-10-01T10:22:25.298956+0000 mon.c (mon.2) 199 : audit [DBG] from='client.? 172.21.15.78:0/3308543545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:22:25.610 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:25 smithi078 bash[27336]: audit 2023-10-01T10:22:25.315787+0000 mon.a (mon.0) 942 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:22:25.610 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:25 smithi078 bash[27336]: audit 2023-10-01T10:22:25.317412+0000 mon.c (mon.2) 200 : audit [INF] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T10:22:25.610 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:25 smithi078 bash[27336]: audit 2023-10-01T10:22:25.318393+0000 mon.c (mon.2) 201 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T10:22:25.611 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:25 smithi078 bash[27336]: audit 2023-10-01T10:22:25.319565+0000 mon.c (mon.2) 202 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:22:25.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:25 smithi173 bash[27612]: audit 2023-10-01T10:22:25.298956+0000 mon.c (mon.2) 199 : audit [DBG] from='client.? 172.21.15.78:0/3308543545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:22:25.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:25 smithi173 bash[27612]: audit 2023-10-01T10:22:25.315787+0000 mon.a (mon.0) 942 : audit [INF] from='mgr.25115 ' entity='mgr.y' 2023-10-01T10:22:25.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:25 smithi173 bash[27612]: audit 2023-10-01T10:22:25.317412+0000 mon.c (mon.2) 200 : audit [INF] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T10:22:25.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:25 smithi173 bash[27612]: audit 2023-10-01T10:22:25.318393+0000 mon.c (mon.2) 201 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T10:22:25.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:25 smithi173 bash[27612]: audit 2023-10-01T10:22:25.319565+0000 mon.c (mon.2) 202 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:22:26.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:26 smithi173 bash[27612]: cephadm 2023-10-01T10:22:25.305462+0000 mgr.y (mgr.25115) 47 : cephadm [INF] Upgrade: Updating mon.a 2023-10-01T10:22:26.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:26 smithi173 bash[27612]: cephadm 2023-10-01T10:22:25.321118+0000 mgr.y (mgr.25115) 48 : cephadm [INF] Deploying daemon mon.a on smithi078 2023-10-01T10:22:26.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:26 smithi173 bash[27612]: cluster 2023-10-01T10:22:25.423581+0000 mgr.y (mgr.25115) 49 : cluster [DBG] pgmap v31: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:22:26.754 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:26 smithi078 bash[27336]: cephadm 2023-10-01T10:22:25.305462+0000 mgr.y (mgr.25115) 47 : cephadm [INF] Upgrade: Updating mon.a 2023-10-01T10:22:26.754 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:26 smithi078 bash[27336]: cephadm 2023-10-01T10:22:25.321118+0000 mgr.y (mgr.25115) 48 : cephadm [INF] Deploying daemon mon.a on smithi078 2023-10-01T10:22:26.754 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:26 smithi078 bash[27336]: cluster 2023-10-01T10:22:25.423581+0000 mgr.y (mgr.25115) 49 : cluster [DBG] pgmap v31: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:22:26.754 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:26 smithi078 bash[33821]: cephadm 2023-10-01T10:22:25.305462+0000 mgr.y (mgr.25115) 47 : cephadm [INF] Upgrade: Updating mon.a 2023-10-01T10:22:26.755 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:26 smithi078 bash[33821]: cephadm 2023-10-01T10:22:25.321118+0000 mgr.y (mgr.25115) 48 : cephadm [INF] Deploying daemon mon.a on smithi078 2023-10-01T10:22:26.755 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:26 smithi078 bash[33821]: cluster 2023-10-01T10:22:25.423581+0000 mgr.y (mgr.25115) 49 : cluster [DBG] pgmap v31: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:22:27.981 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:27 smithi078 systemd[1]: Stopping Ceph mon.a for d8016480-6041-11ee-8db5-212e2dc638e7... 2023-10-01T10:22:27.995 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:22:28.250 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:27 smithi078 bash[27336]: debug 2023-10-01T10:22:27.974+0000 7f54d54b2700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-10-01T10:22:28.250 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:27 smithi078 bash[27336]: debug 2023-10-01T10:22:27.974+0000 7f54d54b2700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2023-10-01T10:22:29.120 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:22:28 smithi173 bash[54497]: ts=2023-10-01T10:22:28.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"smithi173.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"smithi078:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-01T10:22:29.288 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:29 smithi078 bash[75104]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-mon-a 2023-10-01T10:22:29.576 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:29 smithi078 bash[75180]: Error response from daemon: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-mon-a 2023-10-01T10:22:29.576 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:29 smithi078 systemd[1]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7@mon.a.service: Succeeded. 2023-10-01T10:22:29.576 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:29 smithi078 systemd[1]: Stopped Ceph mon.a for d8016480-6041-11ee-8db5-212e2dc638e7. 2023-10-01T10:22:29.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:29 smithi078 systemd[1]: Started Ceph mon.a for d8016480-6041-11ee-8db5-212e2dc638e7. 2023-10-01T10:22:30.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:22:30 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:22:30] "GET /metrics HTTP/1.1" 200 36023 "" "Prometheus/2.33.4" 2023-10-01T10:22:31.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.554+0000 7f31d75bac80 0 set uid:gid to 167:167 (ceph:ceph) 2023-10-01T10:22:31.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.554+0000 7f31d75bac80 0 ceph version 18.2.0-497-gb5682219 (b56822191ad0416d0c05ed59720df03ad9c2a44e) reef (stable), process ceph-mon, pid 7 2023-10-01T10:22:31.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.554+0000 7f31d75bac80 0 pidfile_write: ignore empty --pid-file 2023-10-01T10:22:31.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 0 load: jerasure load: lrc 2023-10-01T10:22:31.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: RocksDB version: 7.9.2 2023-10-01T10:22:31.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Git sha 0 2023-10-01T10:22:31.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Compile date 2023-09-29 15:06:10 2023-10-01T10:22:31.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: DB SUMMARY 2023-10-01T10:22:31.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: DB Session ID: J018X9LM2MV003NE569N 2023-10-01T10:22:31.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: CURRENT file: CURRENT 2023-10-01T10:22:31.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: IDENTITY file: IDENTITY 2023-10-01T10:22:31.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: MANIFEST file: MANIFEST-000008 size: 966 Bytes 2023-10-01T10:22:31.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000023.sst 2023-10-01T10:22:31.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000021.log size: 16708727 ; 2023-10-01T10:22:31.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.error_if_exists: 0 2023-10-01T10:22:31.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.create_if_missing: 0 2023-10-01T10:22:31.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.paranoid_checks: 1 2023-10-01T10:22:31.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.flush_verify_memtable_count: 1 2023-10-01T10:22:31.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-10-01T10:22:31.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-10-01T10:22:31.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.env: 0x55ceab538720 2023-10-01T10:22:31.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.fs: PosixFileSystem 2023-10-01T10:22:31.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.info_log: 0x55ceae0ef480 2023-10-01T10:22:31.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.max_file_opening_threads: 16 2023-10-01T10:22:31.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.statistics: (nil) 2023-10-01T10:22:31.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.use_fsync: 0 2023-10-01T10:22:31.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.max_log_file_size: 0 2023-10-01T10:22:31.836 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-01T10:22:31.836 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.log_file_time_to_roll: 0 2023-10-01T10:22:31.837 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.keep_log_file_num: 1000 2023-10-01T10:22:31.837 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.recycle_log_file_num: 0 2023-10-01T10:22:31.837 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.allow_fallocate: 1 2023-10-01T10:22:31.837 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.allow_mmap_reads: 0 2023-10-01T10:22:31.837 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.allow_mmap_writes: 0 2023-10-01T10:22:31.837 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.use_direct_reads: 0 2023-10-01T10:22:31.837 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-01T10:22:31.837 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.create_missing_column_families: 0 2023-10-01T10:22:31.837 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.db_log_dir: 2023-10-01T10:22:31.837 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.wal_dir: 2023-10-01T10:22:31.837 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.table_cache_numshardbits: 6 2023-10-01T10:22:31.837 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-10-01T10:22:31.838 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-10-01T10:22:31.838 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-01T10:22:31.838 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-01T10:22:31.838 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-10-01T10:22:31.838 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.advise_random_on_open: 1 2023-10-01T10:22:31.838 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.db_write_buffer_size: 0 2023-10-01T10:22:31.838 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.write_buffer_manager: 0x55cead37cc80 2023-10-01T10:22:31.838 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-01T10:22:31.838 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-01T10:22:31.838 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.use_adaptive_mutex: 0 2023-10-01T10:22:31.838 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.rate_limiter: (nil) 2023-10-01T10:22:31.838 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-01T10:22:31.839 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.wal_recovery_mode: 2 2023-10-01T10:22:31.839 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.enable_thread_tracking: 0 2023-10-01T10:22:31.839 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.enable_pipelined_write: 0 2023-10-01T10:22:31.839 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.unordered_write: 0 2023-10-01T10:22:31.839 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-01T10:22:31.839 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-01T10:22:31.839 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-01T10:22:31.839 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-01T10:22:31.839 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.row_cache: None 2023-10-01T10:22:31.839 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.wal_filter: None 2023-10-01T10:22:31.839 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-01T10:22:31.839 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.allow_ingest_behind: 0 2023-10-01T10:22:31.840 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.two_write_queues: 0 2023-10-01T10:22:31.840 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.manual_wal_flush: 0 2023-10-01T10:22:31.840 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.wal_compression: 0 2023-10-01T10:22:31.840 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.atomic_flush: 0 2023-10-01T10:22:31.840 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-01T10:22:31.840 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.persist_stats_to_disk: 0 2023-10-01T10:22:31.840 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-10-01T10:22:31.840 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.log_readahead_size: 0 2023-10-01T10:22:31.840 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-01T10:22:31.840 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.best_efforts_recovery: 0 2023-10-01T10:22:31.840 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-01T10:22:31.840 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-01T10:22:31.841 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.allow_data_in_errors: 0 2023-10-01T10:22:31.841 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.db_host_id: __hostname__ 2023-10-01T10:22:31.841 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.enforce_single_del_contracts: true 2023-10-01T10:22:31.841 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.max_background_jobs: 2 2023-10-01T10:22:31.841 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.max_background_compactions: -1 2023-10-01T10:22:31.841 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.max_subcompactions: 1 2023-10-01T10:22:31.841 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-01T10:22:31.841 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-01T10:22:31.841 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.delayed_write_rate : 16777216 2023-10-01T10:22:31.841 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.max_total_wal_size: 0 2023-10-01T10:22:31.841 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-01T10:22:31.841 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.stats_dump_period_sec: 600 2023-10-01T10:22:31.841 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.stats_persist_period_sec: 600 2023-10-01T10:22:31.842 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-01T10:22:31.842 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.max_open_files: -1 2023-10-01T10:22:31.842 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.bytes_per_sync: 0 2023-10-01T10:22:31.842 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-10-01T10:22:31.842 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-10-01T10:22:31.842 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.compaction_readahead_size: 0 2023-10-01T10:22:31.842 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.max_background_flushes: -1 2023-10-01T10:22:31.842 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Compression algorithms supported: 2023-10-01T10:22:31.842 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-01T10:22:31.842 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: kZSTD supported: 0 2023-10-01T10:22:31.842 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: kXpressCompression supported: 0 2023-10-01T10:22:31.842 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: kLZ4HCCompression supported: 1 2023-10-01T10:22:31.843 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: kZlibCompression supported: 1 2023-10-01T10:22:31.843 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: kSnappyCompression supported: 1 2023-10-01T10:22:31.843 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: kLZ4Compression supported: 1 2023-10-01T10:22:31.843 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: kBZip2Compression supported: 0 2023-10-01T10:22:31.843 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-10-01T10:22:31.843 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: DMutex implementation: pthread_mutex_t 2023-10-01T10:22:31.843 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000008 2023-10-01T10:22:31.843 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-10-01T10:22:31.843 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-01T10:22:31.843 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.merge_operator: 2023-10-01T10:22:31.843 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.compaction_filter: None 2023-10-01T10:22:31.844 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.compaction_filter_factory: None 2023-10-01T10:22:31.844 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.sst_partitioner_factory: None 2023-10-01T10:22:31.844 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-10-01T10:22:31.844 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.table_factory: BlockBasedTable 2023-10-01T10:22:31.844 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55ceae0ef580) 2023-10-01T10:22:31.844 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: cache_index_and_filter_blocks: 1 2023-10-01T10:22:31.844 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-01T10:22:31.844 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-01T10:22:31.844 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: pin_top_level_index_and_filter: 1 2023-10-01T10:22:31.844 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: index_type: 0 2023-10-01T10:22:31.844 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: data_block_index_type: 0 2023-10-01T10:22:31.844 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: index_shortening: 1 2023-10-01T10:22:31.845 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: data_block_hash_table_util_ratio: 0.750000 2023-10-01T10:22:31.845 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: checksum: 4 2023-10-01T10:22:31.845 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: no_block_cache: 0 2023-10-01T10:22:31.845 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: block_cache: 0x55cead3ff1f0 2023-10-01T10:22:31.845 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: block_cache_name: BinnedLRUCache 2023-10-01T10:22:31.845 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: block_cache_options: 2023-10-01T10:22:31.845 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: capacity : 536870912 2023-10-01T10:22:31.845 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: num_shard_bits : 4 2023-10-01T10:22:31.845 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: strict_capacity_limit : 0 2023-10-01T10:22:31.845 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: high_pri_pool_ratio: 0.000 2023-10-01T10:22:31.845 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: block_cache_compressed: (nil) 2023-10-01T10:22:31.845 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: persistent_cache: (nil) 2023-10-01T10:22:31.846 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: block_size: 4096 2023-10-01T10:22:31.846 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: block_size_deviation: 10 2023-10-01T10:22:31.846 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: block_restart_interval: 16 2023-10-01T10:22:31.846 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: index_block_restart_interval: 1 2023-10-01T10:22:31.846 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: metadata_block_size: 4096 2023-10-01T10:22:31.846 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: partition_filters: 0 2023-10-01T10:22:31.846 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: use_delta_encoding: 1 2023-10-01T10:22:31.846 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: filter_policy: bloomfilter 2023-10-01T10:22:31.846 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: whole_key_filtering: 1 2023-10-01T10:22:31.846 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: verify_compression: 0 2023-10-01T10:22:31.846 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: read_amp_bytes_per_bit: 0 2023-10-01T10:22:31.847 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: format_version: 5 2023-10-01T10:22:31.847 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: enable_index_compression: 1 2023-10-01T10:22:31.847 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: block_align: 0 2023-10-01T10:22:31.847 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: max_auto_readahead_size: 262144 2023-10-01T10:22:31.847 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: prepopulate_block_cache: 0 2023-10-01T10:22:31.847 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: initial_auto_readahead_size: 8192 2023-10-01T10:22:31.847 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: num_file_reads_for_auto_readahead: 2 2023-10-01T10:22:31.847 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.write_buffer_size: 33554432 2023-10-01T10:22:31.847 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.max_write_buffer_number: 2 2023-10-01T10:22:31.848 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.compression: NoCompression 2023-10-01T10:22:31.848 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.bottommost_compression: Disabled 2023-10-01T10:22:31.848 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.prefix_extractor: nullptr 2023-10-01T10:22:31.848 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-01T10:22:31.849 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.num_levels: 7 2023-10-01T10:22:31.849 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-01T10:22:31.849 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-01T10:22:31.849 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-01T10:22:31.849 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-01T10:22:31.849 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-01T10:22:31.849 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-01T10:22:31.849 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-01T10:22:31.849 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-01T10:22:31.849 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-01T10:22:31.849 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-01T10:22:31.849 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-10-01T10:22:31.850 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-10-01T10:22:31.850 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.compression_opts.window_bits: -14 2023-10-01T10:22:31.850 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.compression_opts.level: 32767 2023-10-01T10:22:31.850 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.compression_opts.strategy: 0 2023-10-01T10:22:31.850 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-01T10:22:31.850 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-01T10:22:31.850 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-10-01T10:22:31.850 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-01T10:22:31.850 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.compression_opts.enabled: false 2023-10-01T10:22:31.850 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-10-01T10:22:31.850 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-01T10:22:31.851 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-01T10:22:31.851 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-01T10:22:31.851 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.target_file_size_base: 67108864 2023-10-01T10:22:31.851 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.target_file_size_multiplier: 1 2023-10-01T10:22:31.851 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-01T10:22:31.851 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-01T10:22:31.851 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-01T10:22:31.851 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-01T10:22:31.851 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-01T10:22:31.851 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-01T10:22:31.851 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-01T10:22:31.851 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-01T10:22:31.851 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-01T10:22:31.852 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-01T10:22:31.852 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-01T10:22:31.852 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-01T10:22:31.852 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-10-01T10:22:31.852 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.arena_block_size: 1048576 2023-10-01T10:22:31.852 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-01T10:22:31.852 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-01T10:22:31.852 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.disable_auto_compactions: 0 2023-10-01T10:22:31.852 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-01T10:22:31.852 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-01T10:22:31.852 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-01T10:22:31.852 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-01T10:22:31.853 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-01T10:22:31.853 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-01T10:22:31.853 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-01T10:22:31.853 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-01T10:22:31.853 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-01T10:22:31.853 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-01T10:22:31.853 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.table_properties_collectors: 2023-10-01T10:22:31.853 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.inplace_update_support: 0 2023-10-01T10:22:31.853 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-10-01T10:22:31.853 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-01T10:22:31.853 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-01T10:22:31.854 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.memtable_huge_page_size: 0 2023-10-01T10:22:31.854 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.bloom_locality: 0 2023-10-01T10:22:31.854 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.max_successive_merges: 0 2023-10-01T10:22:31.854 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-10-01T10:22:31.854 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.paranoid_file_checks: 0 2023-10-01T10:22:31.854 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.force_consistency_checks: 1 2023-10-01T10:22:31.854 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.report_bg_io_stats: 0 2023-10-01T10:22:31.854 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.ttl: 2592000 2023-10-01T10:22:31.854 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-10-01T10:22:31.854 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.preclude_last_level_data_seconds: 0 2023-10-01T10:22:31.854 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.preserve_internal_time_seconds: 0 2023-10-01T10:22:31.855 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.enable_blob_files: false 2023-10-01T10:22:31.855 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.min_blob_size: 0 2023-10-01T10:22:31.855 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.blob_file_size: 268435456 2023-10-01T10:22:31.855 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.blob_compression_type: NoCompression 2023-10-01T10:22:31.855 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.enable_blob_garbage_collection: false 2023-10-01T10:22:31.855 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-01T10:22:31.855 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-10-01T10:22:31.855 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.blob_compaction_readahead_size: 0 2023-10-01T10:22:31.855 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.blob_file_starting_level: 0 2023-10-01T10:22:31.855 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31d75bac80 4 rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-10-01T10:22:31.855 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.558+0000 7f31ccdd8700 3 rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 23.sst 2023-10-01T10:22:31.855 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.562+0000 7f31d75bac80 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-10-01T10:22:31.856 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.562+0000 7f31d75bac80 4 rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000008 succeeded,manifest_file_number is 8, next_file_number is 25, last_sequence is 11333, log_number is 21,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-01T10:22:31.856 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.562+0000 7f31d75bac80 4 rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 21 2023-10-01T10:22:31.856 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.562+0000 7f31d75bac80 4 rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 6f25e794-4750-4e09-9326-4244483fc9d7 2023-10-01T10:22:31.856 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.562+0000 7f31d75bac80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696155751565685, "job": 1, "event": "recovery_started", "wal_files": [21]} 2023-10-01T10:22:31.856 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.562+0000 7f31d75bac80 4 rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #21 mode 2 2023-10-01T10:22:31.856 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.614+0000 7f31d75bac80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696155751618094, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 26, "file_size": 16204780, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 11334, "largest_seqno": 15265, "table_properties": {"data_size": 16182187, "index_size": 18051, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 3525, "raw_key_size": 26854, "raw_average_key_size": 19, "raw_value_size": 16144649, "raw_average_value_size": 11540, "num_data_blocks": 785, "num_entries": 1399, "num_filter_entries": 1399, "num_deletions": 253, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1696155751, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "6f25e794-4750-4e09-9326-4244483fc9d7", "db_session_id": "J018X9LM2MV003NE569N", "orig_file_number": 26, "seqno_to_time_mapping": "N/A"}} 2023-10-01T10:22:31.856 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.614+0000 7f31d75bac80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696155751618283, "job": 1, "event": "recovery_finished"} 2023-10-01T10:22:31.856 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.614+0000 7f31d75bac80 4 rocksdb: [db/version_set.cc:5047] Creating manifest 28 2023-10-01T10:22:31.856 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.614+0000 7f31d75bac80 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-10-01T10:22:31.856 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.618+0000 7f31d75bac80 4 rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000021.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-10-01T10:22:31.856 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.618+0000 7f31d75bac80 4 rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55cead4a0000 2023-10-01T10:22:31.856 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.618+0000 7f31d75bac80 4 rocksdb: DB pointer 0x55cead48e000 2023-10-01T10:22:31.857 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.618+0000 7f31c5dca700 4 rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-10-01T10:22:31.857 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.618+0000 7f31c5dca700 4 rocksdb: [db/db_impl/db_impl.cc:1111] 2023-10-01T10:22:31.857 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: ** DB Stats ** 2023-10-01T10:22:31.857 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: Uptime(secs): 0.1 total, 0.1 interval 2023-10-01T10:22:31.857 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-01T10:22:31.857 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-01T10:22:31.857 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-01T10:22:31.857 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-01T10:22:31.857 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-01T10:22:31.857 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-01T10:22:31.857 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: ** Compaction Stats [default] ** 2023-10-01T10:22:31.857 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-01T10:22:31.858 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-10-01T10:22:31.858 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: L0 1/0 15.45 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 558.8 0.03 0.00 1 0.028 0 0 0.0 0.0 2023-10-01T10:22:31.858 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: L6 1/0 22.17 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-10-01T10:22:31.858 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: Sum 2/0 37.63 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 558.8 0.03 0.00 1 0.028 0 0 0.0 0.0 2023-10-01T10:22:31.858 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 558.8 0.03 0.00 1 0.028 0 0 0.0 0.0 2023-10-01T10:22:31.858 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: ** Compaction Stats [default] ** 2023-10-01T10:22:31.858 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-01T10:22:31.858 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-01T10:22:31.858 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 558.8 0.03 0.00 1 0.028 0 0 0.0 0.0 2023-10-01T10:22:31.858 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-10-01T10:22:31.858 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: Uptime(secs): 0.1 total, 0.1 interval 2023-10-01T10:22:31.858 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: Flush(GB): cumulative 0.015, interval 0.015 2023-10-01T10:22:31.859 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-01T10:22:31.859 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: AddFile(Total Files): cumulative 0, interval 0 2023-10-01T10:22:31.859 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: AddFile(L0 Files): cumulative 0, interval 0 2023-10-01T10:22:31.859 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: AddFile(Keys): cumulative 0, interval 0 2023-10-01T10:22:31.859 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: Cumulative compaction: 0.02 GB write, 263.86 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-01T10:22:31.859 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: Interval compaction: 0.02 GB write, 263.86 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-01T10:22:31.859 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-01T10:22:31.859 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: Block cache BinnedLRUCache@0x55cead3ff1f0#7 capacity: 512.00 MB usage: 24.11 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.5e-05 secs_since: 0 2023-10-01T10:22:31.859 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: Block cache entry stats(count,size,portion): FilterBlock(1,4.03 KB,0.0007689%) IndexBlock(1,20.08 KB,0.0038296%) Misc(1,0.00 KB,0%) 2023-10-01T10:22:31.859 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: ** File Read Latency Histogram By Level [default] ** 2023-10-01T10:22:31.859 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.618+0000 7f31d75bac80 0 starting mon.a rank 0 at public addrs [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] at bind addrs [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid d8016480-6041-11ee-8db5-212e2dc638e7 2023-10-01T10:22:31.859 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.622+0000 7f31d75bac80 1 mon.a@-1(???) e3 preinit fsid d8016480-6041-11ee-8db5-212e2dc638e7 2023-10-01T10:22:31.860 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.622+0000 7f31d75bac80 0 mon.a@-1(???).mds e7 new map 2023-10-01T10:22:31.860 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.622+0000 7f31d75bac80 0 mon.a@-1(???).mds e7 print_map 2023-10-01T10:22:31.860 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: e7 2023-10-01T10:22:31.860 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: enable_multiple, ever_enabled_multiple: 1,1 2023-10-01T10:22:31.860 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-01T10:22:31.860 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: legacy client fscid: 1 2023-10-01T10:22:31.860 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: 2023-10-01T10:22:31.860 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: Filesystem 'foo' (1) 2023-10-01T10:22:31.860 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: fs_name foo 2023-10-01T10:22:31.860 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: epoch 5 2023-10-01T10:22:31.860 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: flags 12 joinable allow_snaps allow_multimds_snaps 2023-10-01T10:22:31.860 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: created 2023-10-01T10:15:39.285114+0000 2023-10-01T10:22:31.861 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: modified 2023-10-01T10:15:45.790305+0000 2023-10-01T10:22:31.861 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: tableserver 0 2023-10-01T10:22:31.861 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: root 0 2023-10-01T10:22:31.861 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: session_timeout 60 2023-10-01T10:22:31.861 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: session_autoclose 300 2023-10-01T10:22:31.861 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: max_file_size 1099511627776 2023-10-01T10:22:31.861 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: required_client_features {} 2023-10-01T10:22:31.861 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: last_failure 0 2023-10-01T10:22:31.861 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: last_failure_osd_epoch 0 2023-10-01T10:22:31.861 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-01T10:22:31.861 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: max_mds 1 2023-10-01T10:22:31.861 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: in 0 2023-10-01T10:22:31.862 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: up {0=14622} 2023-10-01T10:22:31.862 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: failed 2023-10-01T10:22:31.862 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: damaged 2023-10-01T10:22:31.862 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: stopped 2023-10-01T10:22:31.862 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: data_pools [4] 2023-10-01T10:22:31.862 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: metadata_pool 3 2023-10-01T10:22:31.862 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: inline_data disabled 2023-10-01T10:22:31.862 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: balancer 2023-10-01T10:22:31.862 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: bal_rank_mask -1 2023-10-01T10:22:31.862 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: standby_count_wanted 1 2023-10-01T10:22:31.862 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: [mds.foo.smithi078.vjqrth{0:14622} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.78:6834/2348005403,v1:172.21.15.78:6835/2348005403] compat {c=[1],r=[1],i=[7ff]}] 2023-10-01T10:22:31.862 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: 2023-10-01T10:22:31.863 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: 2023-10-01T10:22:31.863 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: Standby daemons: 2023-10-01T10:22:31.863 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: 2023-10-01T10:22:31.863 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: [mds.foo.smithi173.vrqqoc{-1:24478} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.173:6832/2937343008,v1:172.21.15.173:6833/2937343008] compat {c=[1],r=[1],i=[7ff]}] 2023-10-01T10:22:31.863 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.622+0000 7f31d75bac80 0 mon.a@-1(???).osd e72 crush map has features 3314933000854323200, adjusting msgr requires 2023-10-01T10:22:31.863 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.622+0000 7f31d75bac80 0 mon.a@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2023-10-01T10:22:31.863 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.622+0000 7f31d75bac80 0 mon.a@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2023-10-01T10:22:31.863 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.622+0000 7f31d75bac80 0 mon.a@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2023-10-01T10:22:31.863 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:22:31 smithi078 bash[75299]: debug 2023-10-01T10:22:31.622+0000 7f31d75bac80 1 mon.a@-1(???).paxosservice(auth 1..20) refresh upgraded, format 0 -> 3 2023-10-01T10:22:38.400 INFO:teuthology.orchestra.run.smithi078.stderr:pool 'unique_pool_1' created 2023-10-01T10:22:38.597 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:22:38 smithi078 bash[71214]: ignoring --setuser ceph since I am not root 2023-10-01T10:22:38.597 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:22:38 smithi078 bash[71214]: ignoring --setgroup ceph since I am not root 2023-10-01T10:22:38.597 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:22:38 smithi078 bash[71214]: debug 2023-10-01T10:22:38.410+0000 7fda00776700 1 -- 172.21.15.78:0/2504333159 <== mon.2 v2:172.21.15.78:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55ae1f2f31e0 con 0x55ae20072c00 2023-10-01T10:22:38.597 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:22:38 smithi078 bash[71214]: debug 2023-10-01T10:22:38.570+0000 7fda0a4a6200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-01T10:22:38.598 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:38 smithi078 bash[33821]: cluster 2023-10-01T10:22:29.425411+0000 mgr.y (mgr.25115) 51 : cluster [DBG] pgmap v33: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:22:38.598 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:38 smithi078 bash[33821]: audit 2023-10-01T10:22:30.198021+0000 mon.c (mon.2) 204 : audit [DBG] from='client.? 172.21.15.78:0/883120638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:22:38.598 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:38 smithi078 bash[33821]: cluster 2023-10-01T10:22:31.426002+0000 mgr.y (mgr.25115) 52 : cluster [DBG] pgmap v34: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:22:38.598 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:38 smithi078 bash[33821]: audit 2023-10-01T10:22:31.557932+0000 mon.c (mon.2) 205 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:22:38.598 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:38 smithi078 bash[33821]: audit 2023-10-01T10:22:32.613336+0000 mon.c (mon.2) 206 : audit [INF] from='client.? 172.21.15.78:0/2062941886' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-01T10:22:38.598 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:38 smithi078 bash[33821]: cluster 2023-10-01T10:22:34.650894+0000 mon.a (mon.0) 1 : cluster [INF] mon.a calling monitor election 2023-10-01T10:22:38.598 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:38 smithi078 bash[33821]: cluster 2023-10-01T10:22:37.856990+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-01T10:22:38.599 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:38 smithi078 bash[33821]: cluster 2023-10-01T10:22:37.870342+0000 mon.a (mon.0) 3 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0],b=[v2:172.21.15.173:3300/0,v1:172.21.15.173:6789/0],c=[v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0]} removed_ranks: {} 2023-10-01T10:22:38.599 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:38 smithi078 bash[33821]: cluster 2023-10-01T10:22:37.871142+0000 mon.a (mon.0) 4 : cluster [DBG] fsmap foo:1 {0=foo.smithi078.vjqrth=up:active} 1 up:standby 2023-10-01T10:22:38.599 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:38 smithi078 bash[33821]: cluster 2023-10-01T10:22:37.871206+0000 mon.a (mon.0) 5 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2023-10-01T10:22:38.599 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:38 smithi078 bash[33821]: cluster 2023-10-01T10:22:37.872407+0000 mon.a (mon.0) 6 : cluster [DBG] mgrmap e29: y(active, since 66s), standbys: x 2023-10-01T10:22:38.599 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:38 smithi078 bash[33821]: debug 2023-10-01T10:22:38.378+0000 7f9c1a222700 -1 mon.c@2(peon).osd e73 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-01T10:22:38.620 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:38 smithi173 bash[54895]: ignoring --setuser ceph since I am not root 2023-10-01T10:22:38.620 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:38 smithi173 bash[54895]: ignoring --setgroup ceph since I am not root 2023-10-01T10:22:38.620 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:38 smithi173 bash[54895]: debug 2023-10-01T10:22:38.403+0000 7faeb4156700 1 -- 172.21.15.173:0/1606310982 <== mon.1 v2:172.21.15.173:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55574c2011e0 con 0x55574cf80800 2023-10-01T10:22:38.620 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:38 smithi173 bash[54895]: debug 2023-10-01T10:22:38.559+0000 7faebde86200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-10-01T10:22:38.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:38 smithi173 bash[27612]: cluster 2023-10-01T10:22:29.425411+0000 mgr.y (mgr.25115) 51 : cluster [DBG] pgmap v33: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:22:38.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:38 smithi173 bash[27612]: audit 2023-10-01T10:22:30.198021+0000 mon.c (mon.2) 204 : audit [DBG] from='client.? 172.21.15.78:0/883120638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:22:38.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:38 smithi173 bash[27612]: cluster 2023-10-01T10:22:31.426002+0000 mgr.y (mgr.25115) 52 : cluster [DBG] pgmap v34: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:22:38.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:38 smithi173 bash[27612]: audit 2023-10-01T10:22:31.557932+0000 mon.c (mon.2) 205 : audit [DBG] from='mgr.25115 172.21.15.78:0/435462654' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:22:38.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:38 smithi173 bash[27612]: audit 2023-10-01T10:22:32.613336+0000 mon.c (mon.2) 206 : audit [INF] from='client.? 172.21.15.78:0/2062941886' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-01T10:22:38.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:38 smithi173 bash[27612]: cluster 2023-10-01T10:22:34.650894+0000 mon.a (mon.0) 1 : cluster [INF] mon.a calling monitor election 2023-10-01T10:22:38.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:38 smithi173 bash[27612]: cluster 2023-10-01T10:22:37.856990+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-01T10:22:38.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:38 smithi173 bash[27612]: cluster 2023-10-01T10:22:37.870342+0000 mon.a (mon.0) 3 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0],b=[v2:172.21.15.173:3300/0,v1:172.21.15.173:6789/0],c=[v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0]} removed_ranks: {} 2023-10-01T10:22:38.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:38 smithi173 bash[27612]: cluster 2023-10-01T10:22:37.871142+0000 mon.a (mon.0) 4 : cluster [DBG] fsmap foo:1 {0=foo.smithi078.vjqrth=up:active} 1 up:standby 2023-10-01T10:22:38.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:38 smithi173 bash[27612]: cluster 2023-10-01T10:22:37.871206+0000 mon.a (mon.0) 5 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2023-10-01T10:22:38.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:38 smithi173 bash[27612]: cluster 2023-10-01T10:22:37.872407+0000 mon.a (mon.0) 6 : cluster [DBG] mgrmap e29: y(active, since 66s), standbys: x 2023-10-01T10:22:38.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:38 smithi173 bash[27612]: debug 2023-10-01T10:22:38.379+0000 7f012e73b700 -1 mon.b@1(peon).osd e73 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-01T10:22:39.120 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:22:38 smithi173 bash[54497]: ts=2023-10-01T10:22:38.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"smithi173.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"smithi078:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-01T10:22:39.268 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_1 rados --yes-i-really-mean-it || true 2023-10-01T10:22:39.571 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:39 smithi078 bash[33821]: audit 2023-10-01T10:22:32.659583+0000 mon.c (mon.2) 207 : audit [DBG] from='client.? 172.21.15.78:0/3501294221' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:22:39.571 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:39 smithi078 bash[33821]: cluster 2023-10-01T10:22:33.427181+0000 mgr.y (mgr.25115) 53 : cluster [DBG] pgmap v35: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:22:39.571 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:39 smithi078 bash[33821]: audit 2023-10-01T10:22:35.120363+0000 mon.b (mon.1) 53 : audit [DBG] from='client.? 172.21.15.78:0/825224361' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:22:39.572 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:39 smithi078 bash[33821]: cluster 2023-10-01T10:22:35.427829+0000 mgr.y (mgr.25115) 54 : cluster [DBG] pgmap v36: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:22:39.572 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:39 smithi078 bash[33821]: cluster 2023-10-01T10:22:37.428463+0000 mgr.y (mgr.25115) 55 : cluster [DBG] pgmap v37: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:22:39.572 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:39 smithi078 bash[33821]: cluster 2023-10-01T10:22:37.486867+0000 mon.b (mon.1) 54 : cluster [INF] mon.b calling monitor election 2023-10-01T10:22:39.572 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:39 smithi078 bash[33821]: audit 2023-10-01T10:22:37.566408+0000 mon.b (mon.1) 55 : audit [DBG] from='client.? 172.21.15.78:0/3624328531' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:22:39.572 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:39 smithi078 bash[33821]: audit 2023-10-01T10:22:37.886920+0000 mon.a (mon.0) 7 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-01T10:22:39.572 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:39 smithi078 bash[33821]: cluster 2023-10-01T10:22:38.342336+0000 mon.a (mon.0) 8 : cluster [DBG] mgrmap e30: y(active, since 66s), standbys: x 2023-10-01T10:22:39.572 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:39 smithi078 bash[33821]: cluster 2023-10-01T10:22:38.342392+0000 mon.a (mon.0) 9 : cluster [INF] overall HEALTH_OK 2023-10-01T10:22:39.572 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:39 smithi078 bash[33821]: audit 2023-10-01T10:22:38.342691+0000 mon.a (mon.0) 10 : audit [INF] from='mgr.25115 ' entity='' 2023-10-01T10:22:39.572 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:39 smithi078 bash[33821]: audit 2023-10-01T10:22:38.386974+0000 mon.a (mon.0) 11 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-10-01T10:22:39.572 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:39 smithi078 bash[33821]: cluster 2023-10-01T10:22:38.387042+0000 mon.a (mon.0) 12 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2023-10-01T10:22:39.573 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:39 smithi078 bash[33821]: cluster 2023-10-01T10:22:38.389099+0000 osd.2 (osd.2) 3 : cluster [WRN] failed to encode map e73 with expected crc 2023-10-01T10:22:39.573 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:39 smithi078 bash[33821]: cluster 2023-10-01T10:22:38.391192+0000 osd.6 (osd.6) 3 : cluster [WRN] failed to encode map e73 with expected crc 2023-10-01T10:22:39.573 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:39 smithi078 bash[33821]: debug 2023-10-01T10:22:39.386+0000 7f9c1a222700 -1 mon.c@2(peon).osd e74 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-01T10:22:39.620 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:39 smithi173 bash[54895]: debug 2023-10-01T10:22:39.487+0000 7faebde86200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-01T10:22:39.620 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:39 smithi173 bash[54895]: debug 2023-10-01T10:22:39.591+0000 7faebde86200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-01T10:22:39.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:39 smithi173 bash[27612]: audit 2023-10-01T10:22:32.659583+0000 mon.c (mon.2) 207 : audit [DBG] from='client.? 172.21.15.78:0/3501294221' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:22:39.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:39 smithi173 bash[27612]: cluster 2023-10-01T10:22:33.427181+0000 mgr.y (mgr.25115) 53 : cluster [DBG] pgmap v35: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:22:39.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:39 smithi173 bash[27612]: audit 2023-10-01T10:22:35.120363+0000 mon.b (mon.1) 53 : audit [DBG] from='client.? 172.21.15.78:0/825224361' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:22:39.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:39 smithi173 bash[27612]: cluster 2023-10-01T10:22:35.427829+0000 mgr.y (mgr.25115) 54 : cluster [DBG] pgmap v36: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:22:39.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:39 smithi173 bash[27612]: cluster 2023-10-01T10:22:37.428463+0000 mgr.y (mgr.25115) 55 : cluster [DBG] pgmap v37: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:22:39.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:39 smithi173 bash[27612]: cluster 2023-10-01T10:22:37.486867+0000 mon.b (mon.1) 54 : cluster [INF] mon.b calling monitor election 2023-10-01T10:22:39.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:39 smithi173 bash[27612]: audit 2023-10-01T10:22:37.566408+0000 mon.b (mon.1) 55 : audit [DBG] from='client.? 172.21.15.78:0/3624328531' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:22:39.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:39 smithi173 bash[27612]: audit 2023-10-01T10:22:37.886920+0000 mon.a (mon.0) 7 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-10-01T10:22:39.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:39 smithi173 bash[27612]: cluster 2023-10-01T10:22:38.342336+0000 mon.a (mon.0) 8 : cluster [DBG] mgrmap e30: y(active, since 66s), standbys: x 2023-10-01T10:22:39.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:39 smithi173 bash[27612]: cluster 2023-10-01T10:22:38.342392+0000 mon.a (mon.0) 9 : cluster [INF] overall HEALTH_OK 2023-10-01T10:22:39.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:39 smithi173 bash[27612]: audit 2023-10-01T10:22:38.342691+0000 mon.a (mon.0) 10 : audit [INF] from='mgr.25115 ' entity='' 2023-10-01T10:22:39.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:39 smithi173 bash[27612]: audit 2023-10-01T10:22:38.386974+0000 mon.a (mon.0) 11 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-10-01T10:22:39.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:39 smithi173 bash[27612]: cluster 2023-10-01T10:22:38.387042+0000 mon.a (mon.0) 12 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2023-10-01T10:22:39.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:39 smithi173 bash[27612]: cluster 2023-10-01T10:22:38.389099+0000 osd.2 (osd.2) 3 : cluster [WRN] failed to encode map e73 with expected crc 2023-10-01T10:22:39.622 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:39 smithi173 bash[27612]: cluster 2023-10-01T10:22:38.391192+0000 osd.6 (osd.6) 3 : cluster [WRN] failed to encode map e73 with expected crc 2023-10-01T10:22:39.622 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:39 smithi173 bash[27612]: debug 2023-10-01T10:22:39.383+0000 7f012e73b700 -1 mon.b@1(peon).osd e74 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-01T10:22:39.833 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:22:39 smithi078 bash[71214]: debug 2023-10-01T10:22:39.562+0000 7fda0a4a6200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-10-01T10:22:39.833 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:22:39 smithi078 bash[71214]: debug 2023-10-01T10:22:39.674+0000 7fda0a4a6200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-10-01T10:22:39.833 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:22:39 smithi078 bash[71214]: debug 2023-10-01T10:22:39.794+0000 7fda0a4a6200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-01T10:22:39.962 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:39 smithi173 bash[54895]: debug 2023-10-01T10:22:39.691+0000 7faebde86200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-10-01T10:22:40.034 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 18.2.0-497-gb5682219 (b56822191ad0416d0c05ed59720df03ad9c2a44e) reef (stable)": 1 2023-10-01T10:22:40.326 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:39 smithi173 bash[54895]: debug 2023-10-01T10:22:39.959+0000 7faebde86200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-01T10:22:40.332 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:22:40 smithi078 bash[71214]: debug 2023-10-01T10:22:40.090+0000 7fda0a4a6200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-10-01T10:22:40.582 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:22:40 smithi078 bash[71214]: debug 2023-10-01T10:22:40.474+0000 7fda0a4a6200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-01T10:22:40.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:40 smithi078 bash[33821]: cluster 2023-10-01T10:22:39.393636+0000 mon.a (mon.0) 13 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2023-10-01T10:22:40.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:40 smithi078 bash[33821]: cluster 2023-10-01T10:22:39.395325+0000 osd.4 (osd.4) 3 : cluster [WRN] failed to encode map e74 with expected crc 2023-10-01T10:22:40.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:40 smithi078 bash[33821]: cluster 2023-10-01T10:22:39.397013+0000 osd.1 (osd.1) 3 : cluster [WRN] failed to encode map e74 with expected crc 2023-10-01T10:22:40.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:40 smithi078 bash[33821]: cluster 2023-10-01T10:22:39.398364+0000 osd.7 (osd.7) 3 : cluster [WRN] failed to encode map e74 with expected crc 2023-10-01T10:22:40.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:40 smithi078 bash[33821]: cluster 2023-10-01T10:22:39.398441+0000 osd.4 (osd.4) 4 : cluster [WRN] failed to encode map e74 with expected crc 2023-10-01T10:22:40.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:40 smithi078 bash[33821]: cluster 2023-10-01T10:22:39.398528+0000 osd.5 (osd.5) 3 : cluster [WRN] failed to encode map e74 with expected crc 2023-10-01T10:22:40.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:40 smithi078 bash[33821]: cluster 2023-10-01T10:22:39.400446+0000 osd.3 (osd.3) 3 : cluster [WRN] failed to encode map e74 with expected crc 2023-10-01T10:22:40.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:40 smithi078 bash[33821]: cluster 2023-10-01T10:22:39.400475+0000 osd.2 (osd.2) 4 : cluster [WRN] failed to encode map e74 with expected crc 2023-10-01T10:22:40.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:40 smithi078 bash[33821]: cluster 2023-10-01T10:22:39.400622+0000 osd.1 (osd.1) 4 : cluster [WRN] failed to encode map e74 with expected crc 2023-10-01T10:22:40.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:40 smithi078 bash[33821]: cluster 2023-10-01T10:22:39.400752+0000 osd.0 (osd.0) 3 : cluster [WRN] failed to encode map e74 with expected crc 2023-10-01T10:22:40.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:40 smithi078 bash[33821]: audit 2023-10-01T10:22:40.016604+0000 mon.a (mon.0) 14 : audit [DBG] from='client.? 172.21.15.78:0/777361630' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:22:40.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:40 smithi078 bash[33821]: debug 2023-10-01T10:22:40.390+0000 7f9c1a222700 -1 mon.c@2(peon).osd e75 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-01T10:22:40.619 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:40 smithi173 bash[54895]: debug 2023-10-01T10:22:40.319+0000 7faebde86200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-10-01T10:22:40.620 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:40 smithi173 bash[54895]: debug 2023-10-01T10:22:40.483+0000 7faebde86200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-01T10:22:40.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:40 smithi173 bash[27612]: cluster 2023-10-01T10:22:39.393636+0000 mon.a (mon.0) 13 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2023-10-01T10:22:40.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:40 smithi173 bash[27612]: cluster 2023-10-01T10:22:39.395325+0000 osd.4 (osd.4) 3 : cluster [WRN] failed to encode map e74 with expected crc 2023-10-01T10:22:40.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:40 smithi173 bash[27612]: cluster 2023-10-01T10:22:39.397013+0000 osd.1 (osd.1) 3 : cluster [WRN] failed to encode map e74 with expected crc 2023-10-01T10:22:40.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:40 smithi173 bash[27612]: cluster 2023-10-01T10:22:39.398364+0000 osd.7 (osd.7) 3 : cluster [WRN] failed to encode map e74 with expected crc 2023-10-01T10:22:40.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:40 smithi173 bash[27612]: cluster 2023-10-01T10:22:39.398441+0000 osd.4 (osd.4) 4 : cluster [WRN] failed to encode map e74 with expected crc 2023-10-01T10:22:40.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:40 smithi173 bash[27612]: cluster 2023-10-01T10:22:39.398528+0000 osd.5 (osd.5) 3 : cluster [WRN] failed to encode map e74 with expected crc 2023-10-01T10:22:40.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:40 smithi173 bash[27612]: cluster 2023-10-01T10:22:39.400446+0000 osd.3 (osd.3) 3 : cluster [WRN] failed to encode map e74 with expected crc 2023-10-01T10:22:40.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:40 smithi173 bash[27612]: cluster 2023-10-01T10:22:39.400475+0000 osd.2 (osd.2) 4 : cluster [WRN] failed to encode map e74 with expected crc 2023-10-01T10:22:40.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:40 smithi173 bash[27612]: cluster 2023-10-01T10:22:39.400622+0000 osd.1 (osd.1) 4 : cluster [WRN] failed to encode map e74 with expected crc 2023-10-01T10:22:40.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:40 smithi173 bash[27612]: cluster 2023-10-01T10:22:39.400752+0000 osd.0 (osd.0) 3 : cluster [WRN] failed to encode map e74 with expected crc 2023-10-01T10:22:40.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:40 smithi173 bash[27612]: audit 2023-10-01T10:22:40.016604+0000 mon.a (mon.0) 14 : audit [DBG] from='client.? 172.21.15.78:0/777361630' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:22:40.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:40 smithi173 bash[27612]: debug 2023-10-01T10:22:40.391+0000 7f012e73b700 -1 mon.b@1(peon).osd e75 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-01T10:22:40.834 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -e sha1=b56822191ad0416d0c05ed59720df03ad9c2a44e -- bash -c 'ceph orch ps' 2023-10-01T10:22:41.018 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:22:40 smithi078 bash[71214]: debug 2023-10-01T10:22:40.654+0000 7fda0a4a6200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-10-01T10:22:41.019 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:22:40 smithi078 bash[71214]: debug 2023-10-01T10:22:40.897+0000 7fda0a4a6200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-01T10:22:41.119 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:40 smithi173 bash[54895]: debug 2023-10-01T10:22:40.695+0000 7faebde86200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-10-01T10:22:41.710 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:41 smithi173 bash[27612]: cluster 2023-10-01T10:22:40.397805+0000 mon.a (mon.0) 15 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2023-10-01T10:22:41.710 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:41 smithi173 bash[27612]: cluster 2023-10-01T10:22:40.399773+0000 osd.1 (osd.1) 5 : cluster [WRN] failed to encode map e75 with expected crc 2023-10-01T10:22:41.710 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:41 smithi173 bash[27612]: cluster 2023-10-01T10:22:40.404093+0000 osd.4 (osd.4) 5 : cluster [WRN] failed to encode map e75 with expected crc 2023-10-01T10:22:41.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:41 smithi078 bash[33821]: cluster 2023-10-01T10:22:40.397805+0000 mon.a (mon.0) 15 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2023-10-01T10:22:41.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:41 smithi078 bash[33821]: cluster 2023-10-01T10:22:40.399773+0000 osd.1 (osd.1) 5 : cluster [WRN] failed to encode map e75 with expected crc 2023-10-01T10:22:41.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:41 smithi078 bash[33821]: cluster 2023-10-01T10:22:40.404093+0000 osd.4 (osd.4) 5 : cluster [WRN] failed to encode map e75 with expected crc 2023-10-01T10:22:42.119 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:41 smithi173 bash[54895]: debug 2023-10-01T10:22:41.707+0000 7faebde86200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-01T10:22:42.119 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:41 smithi173 bash[54895]: debug 2023-10-01T10:22:41.819+0000 7faebde86200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-01T10:22:42.120 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:41 smithi173 bash[54895]: debug 2023-10-01T10:22:41.939+0000 7faebde86200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-01T10:22:42.333 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:22:42 smithi078 bash[71214]: debug 2023-10-01T10:22:42.021+0000 7fda0a4a6200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-10-01T10:22:42.333 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:22:42 smithi078 bash[71214]: debug 2023-10-01T10:22:42.141+0000 7fda0a4a6200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-10-01T10:22:42.333 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:22:42 smithi078 bash[71214]: debug 2023-10-01T10:22:42.261+0000 7fda0a4a6200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-10-01T10:22:42.832 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:22:42 smithi078 bash[71214]: debug 2023-10-01T10:22:42.753+0000 7fda0a4a6200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-01T10:22:42.869 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:42 smithi173 bash[54895]: debug 2023-10-01T10:22:42.423+0000 7faebde86200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-10-01T10:22:42.870 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:42 smithi173 bash[54895]: debug 2023-10-01T10:22:42.655+0000 7faebde86200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-01T10:22:43.082 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:22:42 smithi078 bash[71214]: debug 2023-10-01T10:22:42.981+0000 7fda0a4a6200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-10-01T10:22:43.369 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:43 smithi173 bash[54895]: debug 2023-10-01T10:22:43.111+0000 7faebde86200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-01T10:22:43.832 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:22:43 smithi078 bash[71214]: debug 2023-10-01T10:22:43.445+0000 7fda0a4a6200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-10-01T10:22:44.119 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:43 smithi173 bash[54895]: debug 2023-10-01T10:22:43.615+0000 7faebde86200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-01T10:22:44.332 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:22:43 smithi078 bash[71214]: debug 2023-10-01T10:22:43.949+0000 7fda0a4a6200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-10-01T10:22:44.360 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:22:44.619 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:44 smithi173 bash[54895]: debug 2023-10-01T10:22:44.171+0000 7faebde86200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-01T10:22:44.620 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:44 smithi173 bash[54895]: debug 2023-10-01T10:22:44.287+0000 7faebde86200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-01T10:22:44.685 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:22:44 smithi078 bash[71214]: debug 2023-10-01T10:22:44.553+0000 7fda0a4a6200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-10-01T10:22:44.905 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:44 smithi173 bash[54895]: debug 2023-10-01T10:22:44.635+0000 7faebde86200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-01T10:22:44.905 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:44 smithi173 bash[54895]: debug 2023-10-01T10:22:44.759+0000 7faebde86200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-01T10:22:45.071 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:22:44 smithi078 bash[71214]: debug 2023-10-01T10:22:44.677+0000 7fda0a4a6200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-10-01T10:22:45.157 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:44 smithi173 bash[54895]: debug 2023-10-01T10:22:44.899+0000 7faebde86200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-01T10:22:45.157 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:45 smithi173 bash[54895]: debug 2023-10-01T10:22:45.019+0000 7faebde86200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-01T10:22:45.332 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:22:45 smithi078 bash[71214]: debug 2023-10-01T10:22:45.061+0000 7fda0a4a6200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-10-01T10:22:45.332 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:22:45 smithi078 bash[71214]: debug 2023-10-01T10:22:45.181+0000 7fda0a4a6200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-10-01T10:22:45.479 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:45 smithi173 bash[54895]: debug 2023-10-01T10:22:45.151+0000 7faebde86200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-01T10:22:45.479 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:45 smithi173 bash[54895]: debug 2023-10-01T10:22:45.347+0000 7faebde86200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-01T10:22:45.606 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:22:45 smithi078 bash[71214]: debug 2023-10-01T10:22:45.333+0000 7fda0a4a6200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-10-01T10:22:45.606 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:22:45 smithi078 bash[71214]: debug 2023-10-01T10:22:45.461+0000 7fda0a4a6200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-10-01T10:22:45.869 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:45 smithi173 bash[54895]: debug 2023-10-01T10:22:45.475+0000 7faebde86200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-01T10:22:45.954 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:22:45 smithi078 bash[71214]: debug 2023-10-01T10:22:45.597+0000 7fda0a4a6200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-10-01T10:22:45.954 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:22:45 smithi078 bash[71214]: debug 2023-10-01T10:22:45.809+0000 7fda0a4a6200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-10-01T10:22:46.832 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:22:45 smithi078 bash[71214]: debug 2023-10-01T10:22:45.945+0000 7fda0a4a6200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-10-01T10:22:47.120 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:46 smithi173 bash[54895]: debug 2023-10-01T10:22:46.755+0000 7faebde86200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-01T10:22:47.120 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:46 smithi173 bash[54895]: [01/Oct/2023:10:22:46] ENGINE Bus STARTING 2023-10-01T10:22:47.120 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:46 smithi173 bash[54895]: CherryPy Checker: 2023-10-01T10:22:47.120 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:46 smithi173 bash[54895]: The Application mounted at '' has an empty config. 2023-10-01T10:22:47.120 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:46 smithi173 bash[54895]: [01/Oct/2023:10:22:46] ENGINE Serving on http://:::9283 2023-10-01T10:22:47.120 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 10:22:46 smithi173 bash[54895]: [01/Oct/2023:10:22:46] ENGINE Bus STARTED 2023-10-01T10:22:47.454 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:47 smithi078 bash[33821]: debug 2023-10-01T10:22:47.237+0000 7f9c1a222700 -1 mon.c@2(peon).osd e76 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-01T10:22:47.454 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:22:47 smithi078 bash[71214]: debug 2023-10-01T10:22:47.225+0000 7fda0a4a6200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-10-01T10:22:47.454 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:22:47 smithi078 bash[71214]: [01/Oct/2023:10:22:47] ENGINE Bus STARTING 2023-10-01T10:22:47.501 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:47 smithi173 bash[27612]: debug 2023-10-01T10:22:47.239+0000 7f012e73b700 -1 mon.b@1(peon).osd e76 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-01T10:22:47.833 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:22:47 smithi078 bash[71214]: debug 2023-10-01T10:22:47.457+0000 7fd963fb9700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-01T10:22:47.833 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:22:47 smithi078 bash[71214]: debug 2023-10-01T10:22:47.457+0000 7fd963fb9700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-01T10:22:47.833 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:22:47 smithi078 bash[71214]: debug 2023-10-01T10:22:47.457+0000 7fd963fb9700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-01T10:22:47.833 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:22:47 smithi078 bash[71214]: debug 2023-10-01T10:22:47.457+0000 7fd963fb9700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-01T10:22:47.833 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:22:47 smithi078 bash[71214]: debug 2023-10-01T10:22:47.457+0000 7fd963fb9700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-01T10:22:47.833 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:22:47 smithi078 bash[71214]: debug 2023-10-01T10:22:47.465+0000 7fd968fc3700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-01T10:22:47.833 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:22:47 smithi078 bash[71214]: debug 2023-10-01T10:22:47.465+0000 7fd968fc3700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-01T10:22:47.833 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:22:47 smithi078 bash[71214]: debug 2023-10-01T10:22:47.465+0000 7fd968fc3700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-01T10:22:47.834 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:22:47 smithi078 bash[71214]: debug 2023-10-01T10:22:47.465+0000 7fd968fc3700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-01T10:22:47.834 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:22:47 smithi078 bash[71214]: debug 2023-10-01T10:22:47.465+0000 7fd968fc3700 -1 client.0 error registering admin socket command: (17) File exists 2023-10-01T10:22:47.834 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:22:47 smithi078 bash[71214]: CherryPy Checker: 2023-10-01T10:22:47.834 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:22:47 smithi078 bash[71214]: The Application mounted at '' has an empty config. 2023-10-01T10:22:47.834 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:22:47 smithi078 bash[71214]: [01/Oct/2023:10:22:47] ENGINE Serving on http://:::9283 2023-10-01T10:22:47.834 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:22:47 smithi078 bash[71214]: [01/Oct/2023:10:22:47] ENGINE Bus STARTED 2023-10-01T10:22:47.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:47 smithi078 bash[33821]: cluster 2023-10-01T10:22:46.778912+0000 mon.a (mon.0) 16 : cluster [DBG] Standby manager daemon x restarted 2023-10-01T10:22:47.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:47 smithi078 bash[33821]: cluster 2023-10-01T10:22:46.778996+0000 mon.a (mon.0) 17 : cluster [DBG] Standby manager daemon x started 2023-10-01T10:22:47.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:47 smithi078 bash[33821]: audit 2023-10-01T10:22:46.785245+0000 mon.b (mon.1) 56 : audit [DBG] from='mgr.? 172.21.15.173:0/3887574547' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-01T10:22:47.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:47 smithi078 bash[33821]: audit 2023-10-01T10:22:46.786576+0000 mon.b (mon.1) 57 : audit [DBG] from='mgr.? 172.21.15.173:0/3887574547' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-01T10:22:47.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:47 smithi078 bash[33821]: audit 2023-10-01T10:22:46.790872+0000 mon.b (mon.1) 58 : audit [DBG] from='mgr.? 172.21.15.173:0/3887574547' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-01T10:22:47.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:47 smithi078 bash[33821]: audit 2023-10-01T10:22:46.791349+0000 mon.b (mon.1) 59 : audit [DBG] from='mgr.? 172.21.15.173:0/3887574547' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-01T10:22:47.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:47 smithi078 bash[33821]: cluster 2023-10-01T10:22:47.233916+0000 mon.a (mon.0) 18 : cluster [INF] Active manager daemon y restarted 2023-10-01T10:22:47.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:47 smithi078 bash[33821]: cluster 2023-10-01T10:22:47.234500+0000 mon.a (mon.0) 19 : cluster [INF] Activating manager daemon y 2023-10-01T10:22:47.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:47 smithi078 bash[33821]: cluster 2023-10-01T10:22:47.245639+0000 mon.a (mon.0) 20 : cluster [DBG] osdmap e76: 8 total, 8 up, 8 in 2023-10-01T10:22:47.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:47 smithi078 bash[33821]: cluster 2023-10-01T10:22:47.246125+0000 mon.a (mon.0) 21 : cluster [DBG] mgrmap e31: y(active, starting, since 0.0117723s), standbys: x 2023-10-01T10:22:47.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:47 smithi078 bash[33821]: cluster 2023-10-01T10:22:47.247306+0000 osd.1 (osd.1) 6 : cluster [WRN] failed to encode map e76 with expected crc 2023-10-01T10:22:47.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:47 smithi078 bash[33821]: audit 2023-10-01T10:22:47.257982+0000 mon.a (mon.0) 22 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T10:22:47.836 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:47 smithi078 bash[33821]: audit 2023-10-01T10:22:47.258207+0000 mon.a (mon.0) 23 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T10:22:47.836 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:47 smithi078 bash[33821]: audit 2023-10-01T10:22:47.258399+0000 mon.a (mon.0) 24 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T10:22:47.836 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:47 smithi078 bash[33821]: audit 2023-10-01T10:22:47.259009+0000 mon.a (mon.0) 25 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi078.vjqrth"}]: dispatch 2023-10-01T10:22:47.836 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:47 smithi078 bash[33821]: audit 2023-10-01T10:22:47.259214+0000 mon.a (mon.0) 26 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi173.vrqqoc"}]: dispatch 2023-10-01T10:22:47.836 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:47 smithi078 bash[33821]: audit 2023-10-01T10:22:47.259711+0000 mon.a (mon.0) 27 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-01T10:22:47.836 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:47 smithi078 bash[33821]: audit 2023-10-01T10:22:47.259860+0000 mon.a (mon.0) 28 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-01T10:22:47.836 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:47 smithi078 bash[33821]: audit 2023-10-01T10:22:47.260280+0000 mon.a (mon.0) 29 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T10:22:47.836 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:47 smithi078 bash[33821]: audit 2023-10-01T10:22:47.260520+0000 mon.a (mon.0) 30 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T10:22:47.836 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:47 smithi078 bash[33821]: audit 2023-10-01T10:22:47.260687+0000 mon.a (mon.0) 31 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T10:22:47.836 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:47 smithi078 bash[33821]: audit 2023-10-01T10:22:47.260929+0000 mon.a (mon.0) 32 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T10:22:47.836 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:47 smithi078 bash[33821]: audit 2023-10-01T10:22:47.261156+0000 mon.a (mon.0) 33 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T10:22:47.836 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:47 smithi078 bash[33821]: audit 2023-10-01T10:22:47.261479+0000 mon.a (mon.0) 34 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T10:22:47.837 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:47 smithi078 bash[33821]: audit 2023-10-01T10:22:47.261804+0000 mon.a (mon.0) 35 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T10:22:47.837 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:47 smithi078 bash[33821]: audit 2023-10-01T10:22:47.262024+0000 mon.a (mon.0) 36 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T10:22:47.837 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:47 smithi078 bash[33821]: audit 2023-10-01T10:22:47.262311+0000 mon.a (mon.0) 37 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-01T10:22:47.837 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:47 smithi078 bash[33821]: audit 2023-10-01T10:22:47.262537+0000 mon.a (mon.0) 38 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-01T10:22:47.837 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:47 smithi078 bash[33821]: audit 2023-10-01T10:22:47.262987+0000 mon.a (mon.0) 39 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-01T10:22:47.837 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:47 smithi078 bash[33821]: cluster 2023-10-01T10:22:47.292492+0000 mon.a (mon.0) 40 : cluster [INF] Manager daemon y is now available 2023-10-01T10:22:47.837 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:47 smithi078 bash[33821]: audit 2023-10-01T10:22:47.324716+0000 mon.a (mon.0) 41 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:22:47.837 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:47 smithi078 bash[33821]: audit 2023-10-01T10:22:47.328014+0000 mon.a (mon.0) 42 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:22:47.837 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:47 smithi078 bash[33821]: audit 2023-10-01T10:22:47.369447+0000 mon.a (mon.0) 43 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:22:47.837 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:47 smithi078 bash[33821]: audit 2023-10-01T10:22:47.373962+0000 mon.a (mon.0) 44 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-01T10:22:47.837 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:47 smithi078 bash[33821]: audit 2023-10-01T10:22:47.417036+0000 mon.a (mon.0) 45 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-01T10:22:47.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:47 smithi173 bash[27612]: cluster 2023-10-01T10:22:46.778912+0000 mon.a (mon.0) 16 : cluster [DBG] Standby manager daemon x restarted 2023-10-01T10:22:47.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:47 smithi173 bash[27612]: cluster 2023-10-01T10:22:46.778996+0000 mon.a (mon.0) 17 : cluster [DBG] Standby manager daemon x started 2023-10-01T10:22:47.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:47 smithi173 bash[27612]: audit 2023-10-01T10:22:46.785245+0000 mon.b (mon.1) 56 : audit [DBG] from='mgr.? 172.21.15.173:0/3887574547' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-10-01T10:22:47.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:47 smithi173 bash[27612]: audit 2023-10-01T10:22:46.786576+0000 mon.b (mon.1) 57 : audit [DBG] from='mgr.? 172.21.15.173:0/3887574547' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-10-01T10:22:47.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:47 smithi173 bash[27612]: audit 2023-10-01T10:22:46.790872+0000 mon.b (mon.1) 58 : audit [DBG] from='mgr.? 172.21.15.173:0/3887574547' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-10-01T10:22:47.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:47 smithi173 bash[27612]: audit 2023-10-01T10:22:46.791349+0000 mon.b (mon.1) 59 : audit [DBG] from='mgr.? 172.21.15.173:0/3887574547' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-10-01T10:22:47.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:47 smithi173 bash[27612]: cluster 2023-10-01T10:22:47.233916+0000 mon.a (mon.0) 18 : cluster [INF] Active manager daemon y restarted 2023-10-01T10:22:47.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:47 smithi173 bash[27612]: cluster 2023-10-01T10:22:47.234500+0000 mon.a (mon.0) 19 : cluster [INF] Activating manager daemon y 2023-10-01T10:22:47.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:47 smithi173 bash[27612]: cluster 2023-10-01T10:22:47.245639+0000 mon.a (mon.0) 20 : cluster [DBG] osdmap e76: 8 total, 8 up, 8 in 2023-10-01T10:22:47.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:47 smithi173 bash[27612]: cluster 2023-10-01T10:22:47.246125+0000 mon.a (mon.0) 21 : cluster [DBG] mgrmap e31: y(active, starting, since 0.0117723s), standbys: x 2023-10-01T10:22:47.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:47 smithi173 bash[27612]: cluster 2023-10-01T10:22:47.247306+0000 osd.1 (osd.1) 6 : cluster [WRN] failed to encode map e76 with expected crc 2023-10-01T10:22:47.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:47 smithi173 bash[27612]: audit 2023-10-01T10:22:47.257982+0000 mon.a (mon.0) 22 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T10:22:47.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:47 smithi173 bash[27612]: audit 2023-10-01T10:22:47.258207+0000 mon.a (mon.0) 23 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T10:22:47.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:47 smithi173 bash[27612]: audit 2023-10-01T10:22:47.258399+0000 mon.a (mon.0) 24 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T10:22:47.872 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:47 smithi173 bash[27612]: audit 2023-10-01T10:22:47.259009+0000 mon.a (mon.0) 25 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi078.vjqrth"}]: dispatch 2023-10-01T10:22:47.872 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:47 smithi173 bash[27612]: audit 2023-10-01T10:22:47.259214+0000 mon.a (mon.0) 26 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi173.vrqqoc"}]: dispatch 2023-10-01T10:22:47.872 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:47 smithi173 bash[27612]: audit 2023-10-01T10:22:47.259711+0000 mon.a (mon.0) 27 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-10-01T10:22:47.872 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:47 smithi173 bash[27612]: audit 2023-10-01T10:22:47.259860+0000 mon.a (mon.0) 28 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-10-01T10:22:47.872 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:47 smithi173 bash[27612]: audit 2023-10-01T10:22:47.260280+0000 mon.a (mon.0) 29 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T10:22:47.872 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:47 smithi173 bash[27612]: audit 2023-10-01T10:22:47.260520+0000 mon.a (mon.0) 30 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T10:22:47.872 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:47 smithi173 bash[27612]: audit 2023-10-01T10:22:47.260687+0000 mon.a (mon.0) 31 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T10:22:47.872 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:47 smithi173 bash[27612]: audit 2023-10-01T10:22:47.260929+0000 mon.a (mon.0) 32 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T10:22:47.872 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:47 smithi173 bash[27612]: audit 2023-10-01T10:22:47.261156+0000 mon.a (mon.0) 33 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T10:22:47.872 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:47 smithi173 bash[27612]: audit 2023-10-01T10:22:47.261479+0000 mon.a (mon.0) 34 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-10-01T10:22:47.872 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:47 smithi173 bash[27612]: audit 2023-10-01T10:22:47.261804+0000 mon.a (mon.0) 35 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-10-01T10:22:47.872 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:47 smithi173 bash[27612]: audit 2023-10-01T10:22:47.262024+0000 mon.a (mon.0) 36 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-10-01T10:22:47.873 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:47 smithi173 bash[27612]: audit 2023-10-01T10:22:47.262311+0000 mon.a (mon.0) 37 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-10-01T10:22:47.873 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:47 smithi173 bash[27612]: audit 2023-10-01T10:22:47.262537+0000 mon.a (mon.0) 38 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-10-01T10:22:47.873 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:47 smithi173 bash[27612]: audit 2023-10-01T10:22:47.262987+0000 mon.a (mon.0) 39 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-10-01T10:22:47.873 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:47 smithi173 bash[27612]: cluster 2023-10-01T10:22:47.292492+0000 mon.a (mon.0) 40 : cluster [INF] Manager daemon y is now available 2023-10-01T10:22:47.873 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:47 smithi173 bash[27612]: audit 2023-10-01T10:22:47.324716+0000 mon.a (mon.0) 41 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:22:47.873 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:47 smithi173 bash[27612]: audit 2023-10-01T10:22:47.328014+0000 mon.a (mon.0) 42 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:22:47.873 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:47 smithi173 bash[27612]: audit 2023-10-01T10:22:47.369447+0000 mon.a (mon.0) 43 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:22:47.873 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:47 smithi173 bash[27612]: audit 2023-10-01T10:22:47.373962+0000 mon.a (mon.0) 44 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-10-01T10:22:47.873 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:47 smithi173 bash[27612]: audit 2023-10-01T10:22:47.417036+0000 mon.a (mon.0) 45 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-10-01T10:22:48.276 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-01T10:22:48.276 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.a smithi078 *:9093,9094 running (2m) - 8m 14.8M - 0.23.0 ba2b418f427c 90745d6cf2c8 2023-10-01T10:22:48.276 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.a smithi173 *:3000 running (105s) - 8m 41.1M - 8.3.5 dad864ee21e9 4874aca3a861 2023-10-01T10:22:48.277 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foo.smithi078.vjqrth smithi078 running (7m) - 7m 15.3M - 16.2.14-44-geeb13ac9 94a6c4498547 aedf3352e433 2023-10-01T10:22:48.277 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foo.smithi173.vrqqoc smithi173 running (7m) - 7m 13.0M - 16.2.14-44-geeb13ac9 94a6c4498547 3815ca08717b 2023-10-01T10:22:48.277 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.x smithi173 *:8443,9283,8765 running (41s) - 15m 73.4M - 18.2.0-497-gb5682219 a3fc96fed5ac 0e4d4ec3a75c 2023-10-01T10:22:48.277 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.y smithi078 *:8443,9283,8765 running (95s) - 17m 457M - 18.2.0-497-gb5682219 a3fc96fed5ac bdb8227ce5b1 2023-10-01T10:22:48.277 INFO:teuthology.orchestra.run.smithi078.stdout:mon.a smithi078 running (17m) - 17m 63.7M 2048M 16.2.14-44-geeb13ac9 94a6c4498547 6de9da5ca74a 2023-10-01T10:22:48.277 INFO:teuthology.orchestra.run.smithi078.stdout:mon.b smithi173 running (16m) - 16m 60.8M 2048M 16.2.14-44-geeb13ac9 94a6c4498547 9ab8649fee56 2023-10-01T10:22:48.277 INFO:teuthology.orchestra.run.smithi078.stdout:mon.c smithi078 running (16m) - 16m 57.3M 2048M 16.2.14-44-geeb13ac9 94a6c4498547 39d4065bb273 2023-10-01T10:22:48.277 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.a smithi078 *:9100 running (2m) - 9m 12.5M - 1.3.1 1dbe0e931976 3c271e279ae2 2023-10-01T10:22:48.277 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.b smithi173 *:9100 running (2m) - 9m 11.1M - 1.3.1 1dbe0e931976 cf13fd39e3de 2023-10-01T10:22:48.277 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (15m) - 15m 367M 2524M 16.2.14-44-geeb13ac9 94a6c4498547 f6a90fa5df1a 2023-10-01T10:22:48.277 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (14m) - 14m 240M 2524M 16.2.14-44-geeb13ac9 94a6c4498547 da5330a1eff3 2023-10-01T10:22:48.278 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (13m) - 13m 329M 2524M 16.2.14-44-geeb13ac9 94a6c4498547 685fab0bfbf6 2023-10-01T10:22:48.278 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (12m) - 12m 167M 2524M 16.2.14-44-geeb13ac9 94a6c4498547 c3823edbc5f1 2023-10-01T10:22:48.278 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi173 running (12m) - 12m 368M 2524M 16.2.14-44-geeb13ac9 94a6c4498547 f6a3e2fc7289 2023-10-01T10:22:48.278 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi173 running (11m) - 11m 403M 2524M 16.2.14-44-geeb13ac9 94a6c4498547 b4ab64b3b760 2023-10-01T10:22:48.278 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi173 running (10m) - 10m 209M 2524M 16.2.14-44-geeb13ac9 94a6c4498547 67631580752d 2023-10-01T10:22:48.278 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi173 running (10m) - 10m 259M 2524M 16.2.14-44-geeb13ac9 94a6c4498547 86157c8ed4b2 2023-10-01T10:22:48.278 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.a smithi173 *:9095 running (48s) - 9m 43.5M - 2.33.4 514e6a882f6e 94c2cb9de105 2023-10-01T10:22:48.582 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:22:48 smithi078 bash[71214]: debug 2023-10-01T10:22:48.257+0000 7fd9c245e700 -1 mgr.server handle_report got status from non-daemon mon.a 2023-10-01T10:22:48.660 INFO:teuthology.orchestra.run.smithi078.stderr:enabled application 'rados' on pool 'unique_pool_1' 2023-10-01T10:22:48.733 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:48 smithi078 bash[33821]: cluster 2023-10-01T10:22:47.250583+0000 osd.4 (osd.4) 6 : cluster [WRN] failed to encode map e76 with expected crc 2023-10-01T10:22:48.734 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:48 smithi078 bash[33821]: cluster 2023-10-01T10:22:47.257786+0000 osd.4 (osd.4) 7 : cluster [WRN] failed to encode map e76 with expected crc 2023-10-01T10:22:48.734 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:48 smithi078 bash[33821]: cluster 2023-10-01T10:22:47.382920+0000 osd.5 (osd.5) 4 : cluster [WRN] failed to encode map e76 with expected crc 2023-10-01T10:22:48.734 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:48 smithi078 bash[33821]: cluster 2023-10-01T10:22:47.401632+0000 osd.2 (osd.2) 5 : cluster [WRN] failed to encode map e76 with expected crc 2023-10-01T10:22:48.734 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:48 smithi078 bash[33821]: audit 2023-10-01T10:22:47.651105+0000 mon.a (mon.0) 46 : audit [INF] from='client.? 172.21.15.78:0/3253819074' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T10:22:48.734 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:48 smithi078 bash[33821]: cephadm 2023-10-01T10:22:48.222444+0000 mgr.y (mgr.25072) 1 : cephadm [INF] [01/Oct/2023:10:22:48] ENGINE Bus STARTING 2023-10-01T10:22:48.734 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:48 smithi078 bash[33821]: cluster 2023-10-01T10:22:48.256381+0000 mon.a (mon.0) 47 : cluster [DBG] mgrmap e32: y(active, since 1.02202s), standbys: x 2023-10-01T10:22:48.734 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:48 smithi078 bash[33821]: audit 2023-10-01T10:22:48.257323+0000 mgr.y (mgr.25072) 2 : audit [DBG] from='client.25328 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:22:48.734 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:48 smithi078 bash[33821]: cluster 2023-10-01T10:22:48.266764+0000 mgr.y (mgr.25072) 3 : cluster [DBG] pgmap v3: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:22:48.734 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:48 smithi078 bash[33821]: cephadm 2023-10-01T10:22:48.344219+0000 mgr.y (mgr.25072) 4 : cephadm [INF] [01/Oct/2023:10:22:48] ENGINE Serving on https://172.21.15.78:7150 2023-10-01T10:22:48.734 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:48 smithi078 bash[33821]: debug 2023-10-01T10:22:48.645+0000 7f9c1a222700 -1 mon.c@2(peon).osd e77 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-01T10:22:49.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:48 smithi173 bash[27612]: cluster 2023-10-01T10:22:47.250583+0000 osd.4 (osd.4) 6 : cluster [WRN] failed to encode map e76 with expected crc 2023-10-01T10:22:49.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:48 smithi173 bash[27612]: cluster 2023-10-01T10:22:47.257786+0000 osd.4 (osd.4) 7 : cluster [WRN] failed to encode map e76 with expected crc 2023-10-01T10:22:49.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:48 smithi173 bash[27612]: cluster 2023-10-01T10:22:47.382920+0000 osd.5 (osd.5) 4 : cluster [WRN] failed to encode map e76 with expected crc 2023-10-01T10:22:49.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:48 smithi173 bash[27612]: cluster 2023-10-01T10:22:47.401632+0000 osd.2 (osd.2) 5 : cluster [WRN] failed to encode map e76 with expected crc 2023-10-01T10:22:49.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:48 smithi173 bash[27612]: audit 2023-10-01T10:22:47.651105+0000 mon.a (mon.0) 46 : audit [INF] from='client.? 172.21.15.78:0/3253819074' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T10:22:49.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:48 smithi173 bash[27612]: cephadm 2023-10-01T10:22:48.222444+0000 mgr.y (mgr.25072) 1 : cephadm [INF] [01/Oct/2023:10:22:48] ENGINE Bus STARTING 2023-10-01T10:22:49.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:48 smithi173 bash[27612]: cluster 2023-10-01T10:22:48.256381+0000 mon.a (mon.0) 47 : cluster [DBG] mgrmap e32: y(active, since 1.02202s), standbys: x 2023-10-01T10:22:49.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:48 smithi173 bash[27612]: audit 2023-10-01T10:22:48.257323+0000 mgr.y (mgr.25072) 2 : audit [DBG] from='client.25328 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:22:49.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:48 smithi173 bash[27612]: cluster 2023-10-01T10:22:48.266764+0000 mgr.y (mgr.25072) 3 : cluster [DBG] pgmap v3: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:22:49.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:48 smithi173 bash[27612]: cephadm 2023-10-01T10:22:48.344219+0000 mgr.y (mgr.25072) 4 : cephadm [INF] [01/Oct/2023:10:22:48] ENGINE Serving on https://172.21.15.78:7150 2023-10-01T10:22:49.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:48 smithi173 bash[27612]: debug 2023-10-01T10:22:48.647+0000 7f012e73b700 -1 mon.b@1(peon).osd e77 update_from_paxos full map CRC mismatch, resetting to canonical 2023-10-01T10:22:49.289 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -e sha1=b56822191ad0416d0c05ed59720df03ad9c2a44e -- bash -c 'ceph orch upgrade pause' 2023-10-01T10:22:50.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:49 smithi078 bash[33821]: cephadm 2023-10-01T10:22:48.446003+0000 mgr.y (mgr.25072) 5 : cephadm [INF] [01/Oct/2023:10:22:48] ENGINE Serving on http://172.21.15.78:8765 2023-10-01T10:22:50.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:49 smithi078 bash[33821]: cephadm 2023-10-01T10:22:48.446382+0000 mgr.y (mgr.25072) 6 : cephadm [INF] [01/Oct/2023:10:22:48] ENGINE Bus STARTED 2023-10-01T10:22:50.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:49 smithi078 bash[33821]: audit 2023-10-01T10:22:48.653247+0000 mon.a (mon.0) 48 : audit [INF] from='client.? 172.21.15.78:0/3253819074' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-01T10:22:50.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:49 smithi078 bash[33821]: cluster 2023-10-01T10:22:48.653363+0000 mon.a (mon.0) 49 : cluster [DBG] osdmap e77: 8 total, 8 up, 8 in 2023-10-01T10:22:50.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:49 smithi078 bash[33821]: cluster 2023-10-01T10:22:48.656211+0000 osd.0 (osd.0) 4 : cluster [WRN] failed to encode map e77 with expected crc 2023-10-01T10:22:50.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:49 smithi078 bash[33821]: cluster 2023-10-01T10:22:48.657991+0000 osd.5 (osd.5) 5 : cluster [WRN] failed to encode map e77 with expected crc 2023-10-01T10:22:50.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:49 smithi078 bash[33821]: cluster 2023-10-01T10:22:49.261878+0000 mgr.y (mgr.25072) 7 : cluster [DBG] pgmap v5: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:22:50.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:49 smithi173 bash[27612]: cephadm 2023-10-01T10:22:48.446003+0000 mgr.y (mgr.25072) 5 : cephadm [INF] [01/Oct/2023:10:22:48] ENGINE Serving on http://172.21.15.78:8765 2023-10-01T10:22:50.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:49 smithi173 bash[27612]: cephadm 2023-10-01T10:22:48.446382+0000 mgr.y (mgr.25072) 6 : cephadm [INF] [01/Oct/2023:10:22:48] ENGINE Bus STARTED 2023-10-01T10:22:50.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:49 smithi173 bash[27612]: audit 2023-10-01T10:22:48.653247+0000 mon.a (mon.0) 48 : audit [INF] from='client.? 172.21.15.78:0/3253819074' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-01T10:22:50.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:49 smithi173 bash[27612]: cluster 2023-10-01T10:22:48.653363+0000 mon.a (mon.0) 49 : cluster [DBG] osdmap e77: 8 total, 8 up, 8 in 2023-10-01T10:22:50.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:49 smithi173 bash[27612]: cluster 2023-10-01T10:22:48.656211+0000 osd.0 (osd.0) 4 : cluster [WRN] failed to encode map e77 with expected crc 2023-10-01T10:22:50.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:49 smithi173 bash[27612]: cluster 2023-10-01T10:22:48.657991+0000 osd.5 (osd.5) 5 : cluster [WRN] failed to encode map e77 with expected crc 2023-10-01T10:22:50.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:49 smithi173 bash[27612]: cluster 2023-10-01T10:22:49.261878+0000 mgr.y (mgr.25072) 7 : cluster [DBG] pgmap v5: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:22:50.582 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:22:50 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:22:50] "GET /metrics HTTP/1.1" 200 33353 "" "Prometheus/2.33.4" 2023-10-01T10:22:50.902 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:50 smithi078 bash[33821]: cluster 2023-10-01T10:22:49.664759+0000 mon.a (mon.0) 50 : cluster [DBG] mgrmap e33: y(active, since 2s), standbys: x 2023-10-01T10:22:51.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:50 smithi173 bash[27612]: cluster 2023-10-01T10:22:49.664759+0000 mon.a (mon.0) 50 : cluster [DBG] mgrmap e33: y(active, since 2s), standbys: x 2023-10-01T10:22:51.272 DEBUG:teuthology.orchestra.run.smithi078:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_1 bench 90 write' 2023-10-01T10:22:51.279 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-01T10:22:51.280 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-01T10:22:51.324 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:hints = 1 2023-10-01T10:22:51.324 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-01T10:22:51.324 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Object prefix: benchmark_data_smithi078_76465 2023-10-01T10:22:51.325 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T10:22:51.325 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 0 0 0 0 0 0 - 0 2023-10-01T10:22:52.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:51 smithi078 bash[33821]: cluster 2023-10-01T10:22:51.262449+0000 mgr.y (mgr.25072) 8 : cluster [DBG] pgmap v6: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:22:52.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:51 smithi173 bash[27612]: cluster 2023-10-01T10:22:51.262449+0000 mgr.y (mgr.25072) 8 : cluster [DBG] pgmap v6: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:22:52.326 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 1 16 3065 3049 190.394 190.562 0.00414642 0.00522622 2023-10-01T10:22:53.326 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 2 16 5845 5829 182.067 173.75 0.00549274 0.00547877 2023-10-01T10:22:53.991 INFO:teuthology.orchestra.run.smithi078.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T10:22:54.327 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 3 16 7886 7870 163.873 127.562 0.00669122 0.00609132 2023-10-01T10:22:54.330 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:54 smithi173 bash[27612]: cluster 2023-10-01T10:22:53.262997+0000 mgr.y (mgr.25072) 9 : cluster [DBG] pgmap v7: 89 pgs: 89 active+clean; 45 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 9.0 MiB/s wr, 143 op/s 2023-10-01T10:22:54.330 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:54 smithi173 bash[27612]: audit 2023-10-01T10:22:53.981438+0000 mon.a (mon.0) 51 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:22:54.330 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:54 smithi173 bash[27612]: audit 2023-10-01T10:22:54.182719+0000 mon.a (mon.0) 52 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:22:54.330 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:54 smithi173 bash[27612]: audit 2023-10-01T10:22:54.204725+0000 mon.a (mon.0) 53 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:22:54.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:54 smithi078 bash[33821]: cluster 2023-10-01T10:22:53.262997+0000 mgr.y (mgr.25072) 9 : cluster [DBG] pgmap v7: 89 pgs: 89 active+clean; 45 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 9.0 MiB/s wr, 143 op/s 2023-10-01T10:22:54.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:54 smithi078 bash[33821]: audit 2023-10-01T10:22:53.981438+0000 mon.a (mon.0) 51 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:22:54.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:54 smithi078 bash[33821]: audit 2023-10-01T10:22:54.182719+0000 mon.a (mon.0) 52 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:22:54.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:54 smithi078 bash[33821]: audit 2023-10-01T10:22:54.204725+0000 mon.a (mon.0) 53 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:22:55.327 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 4 16 9972 9956 155.498 130.375 0.00403415 0.00642213 2023-10-01T10:22:55.559 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -e sha1=b56822191ad0416d0c05ed59720df03ad9c2a44e -- bash -c 'sleep 60' 2023-10-01T10:22:55.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:55 smithi078 bash[33821]: audit 2023-10-01T10:22:53.974623+0000 mgr.y (mgr.25072) 10 : audit [DBG] from='client.25126 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:22:55.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:55 smithi078 bash[33821]: cephadm 2023-10-01T10:22:53.975096+0000 mgr.y (mgr.25072) 11 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T10:22:55.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:55 smithi078 bash[33821]: audit 2023-10-01T10:22:55.098403+0000 mon.a (mon.0) 54 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:22:55.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:55 smithi078 bash[33821]: audit 2023-10-01T10:22:55.107287+0000 mon.a (mon.0) 55 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:22:55.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:55 smithi078 bash[33821]: audit 2023-10-01T10:22:55.257800+0000 mon.a (mon.0) 56 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:22:55.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:55 smithi078 bash[33821]: audit 2023-10-01T10:22:55.271837+0000 mon.a (mon.0) 57 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:22:55.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:55 smithi173 bash[27612]: audit 2023-10-01T10:22:53.974623+0000 mgr.y (mgr.25072) 10 : audit [DBG] from='client.25126 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:22:55.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:55 smithi173 bash[27612]: cephadm 2023-10-01T10:22:53.975096+0000 mgr.y (mgr.25072) 11 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T10:22:55.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:55 smithi173 bash[27612]: audit 2023-10-01T10:22:55.098403+0000 mon.a (mon.0) 54 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:22:55.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:55 smithi173 bash[27612]: audit 2023-10-01T10:22:55.107287+0000 mon.a (mon.0) 55 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:22:55.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:55 smithi173 bash[27612]: audit 2023-10-01T10:22:55.257800+0000 mon.a (mon.0) 56 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:22:55.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:55 smithi173 bash[27612]: audit 2023-10-01T10:22:55.271837+0000 mon.a (mon.0) 57 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:22:56.330 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 5 16 12031 12015 150.135 128.688 0.00495376 0.00665148 2023-10-01T10:22:56.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:56 smithi173 bash[27612]: cluster 2023-10-01T10:22:55.263915+0000 mgr.y (mgr.25072) 12 : cluster [DBG] pgmap v8: 89 pgs: 89 active+clean; 357 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 51 MiB/s wr, 815 op/s 2023-10-01T10:22:56.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:56 smithi173 bash[27612]: audit 2023-10-01T10:22:56.273842+0000 mon.a (mon.0) 58 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:22:56.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:56 smithi173 bash[27612]: audit 2023-10-01T10:22:56.303055+0000 mon.a (mon.0) 59 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:22:56.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:56 smithi078 bash[33821]: cluster 2023-10-01T10:22:55.263915+0000 mgr.y (mgr.25072) 12 : cluster [DBG] pgmap v8: 89 pgs: 89 active+clean; 357 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 51 MiB/s wr, 815 op/s 2023-10-01T10:22:56.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:56 smithi078 bash[33821]: audit 2023-10-01T10:22:56.273842+0000 mon.a (mon.0) 58 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:22:56.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:56 smithi078 bash[33821]: audit 2023-10-01T10:22:56.303055+0000 mon.a (mon.0) 59 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:22:57.327 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 6 16 14449 14433 150.298 151.125 0.00999458 0.00664537 2023-10-01T10:22:57.803 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:22:57 smithi078 bash[33821]: cluster 2023-10-01T10:22:57.264479+0000 mgr.y (mgr.25072) 13 : cluster [DBG] pgmap v9: 89 pgs: 89 active+clean; 357 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 40 MiB/s wr, 633 op/s 2023-10-01T10:22:58.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:22:57 smithi173 bash[27612]: cluster 2023-10-01T10:22:57.264479+0000 mgr.y (mgr.25072) 13 : cluster [DBG] pgmap v9: 89 pgs: 89 active+clean; 357 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 40 MiB/s wr, 633 op/s 2023-10-01T10:22:58.327 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 7 16 16756 16740 149.422 144.188 0.00414136 0.00668535 2023-10-01T10:22:59.119 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:22:58 smithi173 bash[54497]: ts=2023-10-01T10:22:58.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"smithi173.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"smithi078:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-01T10:22:59.327 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 8 16 18931 18915 147.735 135.938 0.00554027 0.00675935 2023-10-01T10:23:00.327 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 9 16 21041 21025 145.972 131.875 0.00800631 0.00684323 2023-10-01T10:23:00.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:23:00 smithi078 bash[33821]: cluster 2023-10-01T10:22:59.265631+0000 mgr.y (mgr.25072) 14 : cluster [DBG] pgmap v10: 89 pgs: 89 active+clean; 1.0 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s 2023-10-01T10:23:00.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:23:00 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:23:00] "GET /metrics HTTP/1.1" 200 33353 "" "Prometheus/2.33.4" 2023-10-01T10:23:00.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:23:00 smithi173 bash[27612]: cluster 2023-10-01T10:22:59.265631+0000 mgr.y (mgr.25072) 14 : cluster [DBG] pgmap v10: 89 pgs: 89 active+clean; 1.0 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s 2023-10-01T10:23:01.327 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 10 16 23088 23072 144.168 127.938 0.00619171 0.00692958 2023-10-01T10:23:02.327 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 11 16 25114 25098 142.572 126.625 0.00599337 0.00700738 2023-10-01T10:23:02.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:23:02 smithi078 bash[33821]: cluster 2023-10-01T10:23:01.266270+0000 mgr.y (mgr.25072) 15 : cluster [DBG] pgmap v11: 89 pgs: 89 active+clean; 1.0 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-10-01T10:23:02.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:23:02 smithi173 bash[27612]: cluster 2023-10-01T10:23:01.266270+0000 mgr.y (mgr.25072) 15 : cluster [DBG] pgmap v11: 89 pgs: 89 active+clean; 1.0 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-10-01T10:23:03.327 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 12 16 27067 27051 140.862 122.062 0.00372098 0.00709203 2023-10-01T10:23:03.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:23:03 smithi173 bash[27612]: audit 2023-10-01T10:23:02.364678+0000 mon.a (mon.0) 60 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:23:03.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:23:03 smithi078 bash[33821]: audit 2023-10-01T10:23:02.364678+0000 mon.a (mon.0) 60 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:23:04.328 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 13 16 28941 28925 139.035 117.125 0.00826141 0.00718692 2023-10-01T10:23:04.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:23:04 smithi173 bash[27612]: cluster 2023-10-01T10:23:03.267018+0000 mgr.y (mgr.25072) 16 : cluster [DBG] pgmap v12: 89 pgs: 89 active+clean; 1.5 GiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 130 MiB/s wr, 2.08k op/s 2023-10-01T10:23:04.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:23:04 smithi078 bash[33821]: cluster 2023-10-01T10:23:03.267018+0000 mgr.y (mgr.25072) 16 : cluster [DBG] pgmap v12: 89 pgs: 89 active+clean; 1.5 GiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 130 MiB/s wr, 2.08k op/s 2023-10-01T10:23:05.327 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 14 16 30811 30795 137.452 116.875 0.00773538 0.00726755 2023-10-01T10:23:06.328 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 15 16 32607 32591 135.771 112.25 0.0130969 0.00735962 2023-10-01T10:23:06.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:23:06 smithi173 bash[27612]: cluster 2023-10-01T10:23:05.267602+0000 mgr.y (mgr.25072) 17 : cluster [DBG] pgmap v13: 89 pgs: 89 active+clean; 1.6 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 137 MiB/s wr, 2.19k op/s 2023-10-01T10:23:06.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:23:06 smithi078 bash[33821]: cluster 2023-10-01T10:23:05.267602+0000 mgr.y (mgr.25072) 17 : cluster [DBG] pgmap v13: 89 pgs: 89 active+clean; 1.6 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 137 MiB/s wr, 2.19k op/s 2023-10-01T10:23:07.328 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 16 16 34318 34302 133.969 106.938 0.0152846 0.00745818 2023-10-01T10:23:08.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:23:07 smithi078 bash[33821]: cluster 2023-10-01T10:23:07.268533+0000 mgr.y (mgr.25072) 18 : cluster [DBG] pgmap v14: 89 pgs: 89 active+clean; 1.6 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2023-10-01T10:23:08.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:23:07 smithi173 bash[27612]: cluster 2023-10-01T10:23:07.268533+0000 mgr.y (mgr.25072) 18 : cluster [DBG] pgmap v14: 89 pgs: 89 active+clean; 1.6 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2023-10-01T10:23:08.328 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 17 16 35883 35867 131.841 97.8125 0.00770594 0.00757731 2023-10-01T10:23:09.119 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:23:08 smithi173 bash[54497]: ts=2023-10-01T10:23:08.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"smithi173.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"smithi078:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-01T10:23:09.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:23:08 smithi173 bash[27612]: audit 2023-10-01T10:23:07.864090+0000 mon.a (mon.0) 61 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:23:09.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:23:08 smithi173 bash[27612]: audit 2023-10-01T10:23:07.880445+0000 mon.a (mon.0) 62 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:23:09.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:23:08 smithi173 bash[27612]: audit 2023-10-01T10:23:07.882971+0000 mon.a (mon.0) 63 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:23:09.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:23:08 smithi173 bash[27612]: audit 2023-10-01T10:23:07.884341+0000 mon.a (mon.0) 64 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:23:09.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:23:08 smithi173 bash[27612]: cephadm 2023-10-01T10:23:07.886128+0000 mgr.y (mgr.25072) 19 : cephadm [INF] Updating smithi078:/etc/ceph/ceph.conf 2023-10-01T10:23:09.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:23:08 smithi173 bash[27612]: cephadm 2023-10-01T10:23:07.886362+0000 mgr.y (mgr.25072) 20 : cephadm [INF] Updating smithi173:/etc/ceph/ceph.conf 2023-10-01T10:23:09.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:23:08 smithi173 bash[27612]: cephadm 2023-10-01T10:23:08.105307+0000 mgr.y (mgr.25072) 21 : cephadm [INF] Updating smithi173:/var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T10:23:09.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:23:08 smithi173 bash[27612]: cephadm 2023-10-01T10:23:08.105710+0000 mgr.y (mgr.25072) 22 : cephadm [INF] Updating smithi078:/var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T10:23:09.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:23:08 smithi173 bash[27612]: cephadm 2023-10-01T10:23:08.324481+0000 mgr.y (mgr.25072) 23 : cephadm [INF] Updating smithi173:/etc/ceph/ceph.client.admin.keyring 2023-10-01T10:23:09.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:23:08 smithi173 bash[27612]: cephadm 2023-10-01T10:23:08.324802+0000 mgr.y (mgr.25072) 24 : cephadm [INF] Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2023-10-01T10:23:09.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:23:08 smithi173 bash[27612]: audit 2023-10-01T10:23:08.763037+0000 mon.a (mon.0) 65 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:23:09.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:23:08 smithi173 bash[27612]: audit 2023-10-01T10:23:08.772553+0000 mon.a (mon.0) 66 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:23:09.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:23:08 smithi173 bash[27612]: audit 2023-10-01T10:23:08.784548+0000 mon.a (mon.0) 67 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:23:09.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:23:08 smithi173 bash[27612]: audit 2023-10-01T10:23:08.793750+0000 mon.a (mon.0) 68 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:23:09.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:23:08 smithi173 bash[27612]: audit 2023-10-01T10:23:08.802857+0000 mon.a (mon.0) 69 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:23:09.328 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 18 16 37382 37366 129.721 93.6875 0.0131601 0.007703 2023-10-01T10:23:09.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:23:08 smithi078 bash[33821]: audit 2023-10-01T10:23:07.864090+0000 mon.a (mon.0) 61 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:23:09.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:23:08 smithi078 bash[33821]: audit 2023-10-01T10:23:07.880445+0000 mon.a (mon.0) 62 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:23:09.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:23:08 smithi078 bash[33821]: audit 2023-10-01T10:23:07.882971+0000 mon.a (mon.0) 63 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:23:09.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:23:08 smithi078 bash[33821]: audit 2023-10-01T10:23:07.884341+0000 mon.a (mon.0) 64 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:23:09.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:23:08 smithi078 bash[33821]: cephadm 2023-10-01T10:23:07.886128+0000 mgr.y (mgr.25072) 19 : cephadm [INF] Updating smithi078:/etc/ceph/ceph.conf 2023-10-01T10:23:09.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:23:08 smithi078 bash[33821]: cephadm 2023-10-01T10:23:07.886362+0000 mgr.y (mgr.25072) 20 : cephadm [INF] Updating smithi173:/etc/ceph/ceph.conf 2023-10-01T10:23:09.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:23:08 smithi078 bash[33821]: cephadm 2023-10-01T10:23:08.105307+0000 mgr.y (mgr.25072) 21 : cephadm [INF] Updating smithi173:/var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T10:23:09.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:23:08 smithi078 bash[33821]: cephadm 2023-10-01T10:23:08.105710+0000 mgr.y (mgr.25072) 22 : cephadm [INF] Updating smithi078:/var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T10:23:09.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:23:08 smithi078 bash[33821]: cephadm 2023-10-01T10:23:08.324481+0000 mgr.y (mgr.25072) 23 : cephadm [INF] Updating smithi173:/etc/ceph/ceph.client.admin.keyring 2023-10-01T10:23:09.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:23:08 smithi078 bash[33821]: cephadm 2023-10-01T10:23:08.324802+0000 mgr.y (mgr.25072) 24 : cephadm [INF] Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2023-10-01T10:23:09.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:23:08 smithi078 bash[33821]: audit 2023-10-01T10:23:08.763037+0000 mon.a (mon.0) 65 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:23:09.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:23:08 smithi078 bash[33821]: audit 2023-10-01T10:23:08.772553+0000 mon.a (mon.0) 66 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:23:09.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:23:08 smithi078 bash[33821]: audit 2023-10-01T10:23:08.784548+0000 mon.a (mon.0) 67 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:23:09.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:23:08 smithi078 bash[33821]: audit 2023-10-01T10:23:08.793750+0000 mon.a (mon.0) 68 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:23:09.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:23:08 smithi078 bash[33821]: audit 2023-10-01T10:23:08.802857+0000 mon.a (mon.0) 69 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:23:10.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:23:09 smithi173 bash[27612]: cephadm 2023-10-01T10:23:08.540586+0000 mgr.y (mgr.25072) 25 : cephadm [INF] Updating smithi078:/var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-01T10:23:10.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:23:09 smithi173 bash[27612]: cephadm 2023-10-01T10:23:08.542580+0000 mgr.y (mgr.25072) 26 : cephadm [INF] Updating smithi173:/var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-01T10:23:10.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:23:09 smithi173 bash[27612]: audit 2023-10-01T10:23:08.888720+0000 mon.a (mon.0) 70 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:23:10.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:23:09 smithi173 bash[27612]: audit 2023-10-01T10:23:08.891289+0000 mon.a (mon.0) 71 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:23:10.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:23:09 smithi173 bash[27612]: audit 2023-10-01T10:23:08.892253+0000 mon.a (mon.0) 72 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:23:10.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:23:09 smithi173 bash[27612]: audit 2023-10-01T10:23:08.900995+0000 mon.a (mon.0) 73 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:23:10.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:23:09 smithi173 bash[27612]: cluster 2023-10-01T10:23:09.269287+0000 mgr.y (mgr.25072) 27 : cluster [DBG] pgmap v15: 89 pgs: 89 active+clean; 2.2 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 157 MiB/s wr, 2.51k op/s 2023-10-01T10:23:10.269 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:23:09 smithi078 bash[33821]: cephadm 2023-10-01T10:23:08.540586+0000 mgr.y (mgr.25072) 25 : cephadm [INF] Updating smithi078:/var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-01T10:23:10.269 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:23:09 smithi078 bash[33821]: cephadm 2023-10-01T10:23:08.542580+0000 mgr.y (mgr.25072) 26 : cephadm [INF] Updating smithi173:/var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-01T10:23:10.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:23:09 smithi078 bash[33821]: audit 2023-10-01T10:23:08.888720+0000 mon.a (mon.0) 70 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:23:10.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:23:09 smithi078 bash[33821]: audit 2023-10-01T10:23:08.891289+0000 mon.a (mon.0) 71 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:23:10.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:23:09 smithi078 bash[33821]: audit 2023-10-01T10:23:08.892253+0000 mon.a (mon.0) 72 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:23:10.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:23:09 smithi078 bash[33821]: audit 2023-10-01T10:23:08.900995+0000 mon.a (mon.0) 73 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:23:10.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:23:09 smithi078 bash[33821]: cluster 2023-10-01T10:23:09.269287+0000 mgr.y (mgr.25072) 27 : cluster [DBG] pgmap v15: 89 pgs: 89 active+clean; 2.2 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 157 MiB/s wr, 2.51k op/s 2023-10-01T10:23:10.328 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 19 16 39041 39025 128.351 103.688 0.00584369 0.00778577 2023-10-01T10:23:10.582 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:23:10 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:23:10] "GET /metrics HTTP/1.1" 200 36068 "" "Prometheus/2.33.4" 2023-10-01T10:23:11.328 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:2023-10-01T10:23:11.326610+0000 min lat: 0.00137921 max lat: 0.0672694 avg lat: 0.00786772 2023-10-01T10:23:11.328 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T10:23:11.328 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 20 16 40664 40648 127.005 101.438 0.0107699 0.00786772 2023-10-01T10:23:12.328 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 21 16 42254 42238 125.689 99.375 0.00543905 0.00794994 2023-10-01T10:23:12.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:23:12 smithi078 bash[33821]: cluster 2023-10-01T10:23:11.269748+0000 mgr.y (mgr.25072) 28 : cluster [DBG] pgmap v16: 89 pgs: 89 active+clean; 2.2 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s 2023-10-01T10:23:12.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:23:12 smithi173 bash[27612]: cluster 2023-10-01T10:23:11.269748+0000 mgr.y (mgr.25072) 28 : cluster [DBG] pgmap v16: 89 pgs: 89 active+clean; 2.2 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s 2023-10-01T10:23:13.328 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 22 16 43746 43730 124.214 93.25 0.00935989 0.00804486 2023-10-01T10:23:14.328 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 23 16 45246 45230 122.889 93.75 0.00611586 0.00813203 2023-10-01T10:23:14.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:23:14 smithi173 bash[27612]: cluster 2023-10-01T10:23:13.271837+0000 mgr.y (mgr.25072) 29 : cluster [DBG] pgmap v17: 89 pgs: 89 active+clean; 2.5 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 126 MiB/s wr, 2.01k op/s 2023-10-01T10:23:14.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:23:14 smithi078 bash[33821]: cluster 2023-10-01T10:23:13.271837+0000 mgr.y (mgr.25072) 29 : cluster [DBG] pgmap v17: 89 pgs: 89 active+clean; 2.5 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 126 MiB/s wr, 2.01k op/s 2023-10-01T10:23:15.328 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 24 16 46724 46708 121.617 92.375 0.0150696 0.00821696 2023-10-01T10:23:16.329 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 25 15 48213 48198 120.477 93.125 0.0138042 0.00829539 2023-10-01T10:23:16.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:23:16 smithi173 bash[27612]: cluster 2023-10-01T10:23:15.272407+0000 mgr.y (mgr.25072) 30 : cluster [DBG] pgmap v18: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2023-10-01T10:23:16.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:23:16 smithi078 bash[33821]: cluster 2023-10-01T10:23:15.272407+0000 mgr.y (mgr.25072) 30 : cluster [DBG] pgmap v18: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2023-10-01T10:23:17.329 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 26 16 49635 49619 119.259 88.8125 0.00786901 0.00837949 2023-10-01T10:23:18.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:23:17 smithi078 bash[33821]: cluster 2023-10-01T10:23:17.273042+0000 mgr.y (mgr.25072) 31 : cluster [DBG] pgmap v19: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-10-01T10:23:18.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:23:17 smithi078 bash[33821]: audit 2023-10-01T10:23:17.365225+0000 mon.a (mon.0) 74 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:23:18.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:23:17 smithi173 bash[27612]: cluster 2023-10-01T10:23:17.273042+0000 mgr.y (mgr.25072) 31 : cluster [DBG] pgmap v19: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-10-01T10:23:18.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:23:17 smithi173 bash[27612]: audit 2023-10-01T10:23:17.365225+0000 mon.a (mon.0) 74 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:23:18.329 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 27 16 51060 51044 118.14 89.0625 0.0165196 0.00845878 2023-10-01T10:23:19.120 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:23:18 smithi173 bash[54497]: ts=2023-10-01T10:23:18.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"smithi173.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"smithi078:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-01T10:23:19.329 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 28 16 52454 52438 117.032 87.125 0.0100888 0.00853818 2023-10-01T10:23:20.329 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 29 16 53821 53805 115.943 85.4375 0.00526903 0.00861868 2023-10-01T10:23:20.582 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:23:20 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:23:20] "GET /metrics HTTP/1.1" 200 36068 "" "Prometheus/2.33.4" 2023-10-01T10:23:20.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:23:20 smithi078 bash[33821]: cluster 2023-10-01T10:23:19.273783+0000 mgr.y (mgr.25072) 32 : cluster [DBG] pgmap v20: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 125 MiB/s wr, 2.00k op/s 2023-10-01T10:23:20.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:23:20 smithi173 bash[27612]: cluster 2023-10-01T10:23:19.273783+0000 mgr.y (mgr.25072) 32 : cluster [DBG] pgmap v20: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 125 MiB/s wr, 2.00k op/s 2023-10-01T10:23:21.329 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 30 16 55148 55132 114.842 82.9375 0.00868724 0.0087016 2023-10-01T10:23:22.329 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 31 16 56533 56517 113.93 86.5625 0.0136867 0.00877157 2023-10-01T10:23:22.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:23:22 smithi173 bash[27612]: cluster 2023-10-01T10:23:21.274395+0000 mgr.y (mgr.25072) 33 : cluster [DBG] pgmap v21: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-10-01T10:23:22.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:23:22 smithi078 bash[33821]: cluster 2023-10-01T10:23:21.274395+0000 mgr.y (mgr.25072) 33 : cluster [DBG] pgmap v21: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-10-01T10:23:23.329 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 32 16 57852 57836 112.946 82.4375 0.00735811 0.00884789 2023-10-01T10:23:24.329 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 33 16 59168 59152 112.015 82.25 0.00997937 0.00892119 2023-10-01T10:23:24.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:23:24 smithi173 bash[27612]: cluster 2023-10-01T10:23:23.275019+0000 mgr.y (mgr.25072) 34 : cluster [DBG] pgmap v22: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-10-01T10:23:24.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:23:24 smithi078 bash[33821]: cluster 2023-10-01T10:23:23.275019+0000 mgr.y (mgr.25072) 34 : cluster [DBG] pgmap v22: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-10-01T10:23:25.329 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 34 16 60501 60485 111.171 83.3125 0.0116621 0.00898953 2023-10-01T10:23:26.329 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 35 16 61743 61727 110.212 77.625 0.0126755 0.00906756 2023-10-01T10:23:26.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:23:26 smithi173 bash[27612]: cluster 2023-10-01T10:23:25.275585+0000 mgr.y (mgr.25072) 35 : cluster [DBG] pgmap v23: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-10-01T10:23:26.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:23:26 smithi078 bash[33821]: cluster 2023-10-01T10:23:25.275585+0000 mgr.y (mgr.25072) 35 : cluster [DBG] pgmap v23: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-10-01T10:23:27.330 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 36 16 62988 62972 109.312 77.8125 0.0195001 0.00914184 2023-10-01T10:23:28.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:23:27 smithi078 bash[33821]: cluster 2023-10-01T10:23:27.276054+0000 mgr.y (mgr.25072) 36 : cluster [DBG] pgmap v24: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-10-01T10:23:28.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:23:27 smithi173 bash[27612]: cluster 2023-10-01T10:23:27.276054+0000 mgr.y (mgr.25072) 36 : cluster [DBG] pgmap v24: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-10-01T10:23:28.330 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 37 16 64203 64187 108.41 75.9375 0.0243858 0.00921919 2023-10-01T10:23:29.120 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:23:28 smithi173 bash[54497]: ts=2023-10-01T10:23:28.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"smithi173.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"smithi078:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-01T10:23:29.330 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 38 16 65420 65404 107.559 76.0625 0.0153666 0.00929134 2023-10-01T10:23:30.330 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 39 16 66585 66569 106.667 72.8125 0.00568744 0.00936804 2023-10-01T10:23:30.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:23:30 smithi078 bash[33821]: cluster 2023-10-01T10:23:29.276917+0000 mgr.y (mgr.25072) 37 : cluster [DBG] pgmap v25: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s 2023-10-01T10:23:30.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:23:30 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:23:30] "GET /metrics HTTP/1.1" 200 36068 "" "Prometheus/2.33.4" 2023-10-01T10:23:30.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:23:30 smithi173 bash[27612]: cluster 2023-10-01T10:23:29.276917+0000 mgr.y (mgr.25072) 37 : cluster [DBG] pgmap v25: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s 2023-10-01T10:23:31.330 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:2023-10-01T10:23:31.328472+0000 min lat: 0.00137921 max lat: 0.0882167 avg lat: 0.00944264 2023-10-01T10:23:31.330 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T10:23:31.330 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 40 16 67760 67744 105.837 73.4375 0.0120181 0.00944264 2023-10-01T10:23:32.330 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 41 16 68905 68889 105 71.5625 0.00813747 0.00951828 2023-10-01T10:23:32.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:23:32 smithi173 bash[27612]: cluster 2023-10-01T10:23:31.277676+0000 mgr.y (mgr.25072) 38 : cluster [DBG] pgmap v26: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-01T10:23:32.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:23:32 smithi078 bash[33821]: cluster 2023-10-01T10:23:31.277676+0000 mgr.y (mgr.25072) 38 : cluster [DBG] pgmap v26: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-01T10:23:33.330 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 42 16 70043 70027 104.194 71.125 0.0164745 0.00959146 2023-10-01T10:23:33.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:23:33 smithi173 bash[27612]: audit 2023-10-01T10:23:32.367640+0000 mon.a (mon.0) 75 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:23:33.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:23:33 smithi078 bash[33821]: audit 2023-10-01T10:23:32.367640+0000 mon.a (mon.0) 75 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:23:34.330 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 43 16 71152 71136 103.382 69.3125 0.0138854 0.00966733 2023-10-01T10:23:34.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:23:34 smithi173 bash[27612]: cluster 2023-10-01T10:23:33.278479+0000 mgr.y (mgr.25072) 39 : cluster [DBG] pgmap v27: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-01T10:23:34.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:23:34 smithi078 bash[33821]: cluster 2023-10-01T10:23:33.278479+0000 mgr.y (mgr.25072) 39 : cluster [DBG] pgmap v27: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-01T10:23:35.330 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 44 16 72252 72236 102.595 68.75 0.0230754 0.00974122 2023-10-01T10:23:36.330 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 45 16 73362 73346 101.857 69.375 0.0164981 0.00981207 2023-10-01T10:23:36.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:23:36 smithi173 bash[27612]: cluster 2023-10-01T10:23:35.279040+0000 mgr.y (mgr.25072) 40 : cluster [DBG] pgmap v28: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-10-01T10:23:36.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:23:36 smithi078 bash[33821]: cluster 2023-10-01T10:23:35.279040+0000 mgr.y (mgr.25072) 40 : cluster [DBG] pgmap v28: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-10-01T10:23:37.330 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 46 16 74399 74383 101.051 64.8125 0.00462641 0.00989004 2023-10-01T10:23:38.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:23:37 smithi078 bash[33821]: cluster 2023-10-01T10:23:37.281722+0000 mgr.y (mgr.25072) 41 : cluster [DBG] pgmap v29: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-01T10:23:38.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:23:37 smithi173 bash[27612]: cluster 2023-10-01T10:23:37.281722+0000 mgr.y (mgr.25072) 41 : cluster [DBG] pgmap v29: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-01T10:23:38.330 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 47 16 75492 75476 100.355 68.3125 0.00727365 0.00995821 2023-10-01T10:23:39.119 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:23:38 smithi173 bash[54497]: ts=2023-10-01T10:23:38.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"smithi173.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"smithi078:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-01T10:23:39.331 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 48 16 76582 76566 99.6832 68.125 0.0148346 0.0100261 2023-10-01T10:23:40.331 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 49 16 77663 77647 99.0276 67.5625 0.00828892 0.0100927 2023-10-01T10:23:40.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:23:40 smithi078 bash[33821]: cluster 2023-10-01T10:23:39.282454+0000 mgr.y (mgr.25072) 42 : cluster [DBG] pgmap v30: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2023-10-01T10:23:40.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:23:40 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:23:40] "GET /metrics HTTP/1.1" 200 36071 "" "Prometheus/2.33.4" 2023-10-01T10:23:40.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:23:40 smithi173 bash[27612]: cluster 2023-10-01T10:23:39.282454+0000 mgr.y (mgr.25072) 42 : cluster [DBG] pgmap v30: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2023-10-01T10:23:41.331 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 50 16 78698 78682 98.3406 64.6875 0.0101925 0.010163 2023-10-01T10:23:42.331 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 51 16 79759 79743 97.7125 66.3125 0.0109439 0.0102281 2023-10-01T10:23:42.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:23:42 smithi173 bash[27612]: cluster 2023-10-01T10:23:41.282899+0000 mgr.y (mgr.25072) 43 : cluster [DBG] pgmap v31: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 59 MiB/s wr, 941 op/s 2023-10-01T10:23:42.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:23:42 smithi078 bash[33821]: cluster 2023-10-01T10:23:41.282899+0000 mgr.y (mgr.25072) 43 : cluster [DBG] pgmap v31: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 59 MiB/s wr, 941 op/s 2023-10-01T10:23:43.331 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 52 16 80792 80776 97.0749 64.5625 0.0217487 0.0102954 2023-10-01T10:23:44.331 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 53 16 81814 81798 96.4484 63.875 0.0238382 0.0103629 2023-10-01T10:23:44.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:23:44 smithi173 bash[27612]: cluster 2023-10-01T10:23:43.283470+0000 mgr.y (mgr.25072) 44 : cluster [DBG] pgmap v32: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-10-01T10:23:44.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:23:44 smithi078 bash[33821]: cluster 2023-10-01T10:23:43.283470+0000 mgr.y (mgr.25072) 44 : cluster [DBG] pgmap v32: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-10-01T10:23:45.331 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 54 16 82834 82818 95.8428 63.75 0.0114539 0.0104276 2023-10-01T10:23:46.333 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 55 16 83849 83833 95.2535 63.4375 0.0138214 0.0104927 2023-10-01T10:23:46.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:23:46 smithi173 bash[27612]: cluster 2023-10-01T10:23:45.284022+0000 mgr.y (mgr.25072) 45 : cluster [DBG] pgmap v33: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-10-01T10:23:46.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:23:46 smithi078 bash[33821]: cluster 2023-10-01T10:23:45.284022+0000 mgr.y (mgr.25072) 45 : cluster [DBG] pgmap v33: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-10-01T10:23:47.331 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 56 16 84869 84853 94.6908 63.75 0.0139543 0.010555 2023-10-01T10:23:48.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:23:47 smithi078 bash[33821]: cluster 2023-10-01T10:23:47.284435+0000 mgr.y (mgr.25072) 46 : cluster [DBG] pgmap v34: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 56 MiB/s wr, 897 op/s 2023-10-01T10:23:48.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:23:47 smithi078 bash[33821]: audit 2023-10-01T10:23:47.373503+0000 mon.a (mon.0) 76 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:23:48.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:23:47 smithi173 bash[27612]: cluster 2023-10-01T10:23:47.284435+0000 mgr.y (mgr.25072) 46 : cluster [DBG] pgmap v34: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 56 MiB/s wr, 897 op/s 2023-10-01T10:23:48.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:23:47 smithi173 bash[27612]: audit 2023-10-01T10:23:47.373503+0000 mon.a (mon.0) 76 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:23:48.331 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 57 16 85858 85842 94.114 61.8125 0.00997493 0.010619 2023-10-01T10:23:49.119 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:23:48 smithi173 bash[54497]: ts=2023-10-01T10:23:48.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"smithi173.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"smithi078:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-01T10:23:49.332 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 58 16 86840 86824 93.5494 61.375 0.00680098 0.0106839 2023-10-01T10:23:50.332 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 59 16 87835 87819 93.0178 62.1875 0.0085312 0.0107446 2023-10-01T10:23:50.582 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:23:50 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:23:50] "GET /metrics HTTP/1.1" 200 36070 "" "Prometheus/2.33.4" 2023-10-01T10:23:50.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:23:50 smithi078 bash[33821]: cluster 2023-10-01T10:23:49.285365+0000 mgr.y (mgr.25072) 47 : cluster [DBG] pgmap v35: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 82 MiB/s wr, 1.32k op/s 2023-10-01T10:23:50.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:23:50 smithi173 bash[27612]: cluster 2023-10-01T10:23:49.285365+0000 mgr.y (mgr.25072) 47 : cluster [DBG] pgmap v35: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 82 MiB/s wr, 1.32k op/s 2023-10-01T10:23:51.332 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:2023-10-01T10:23:51.330358+0000 min lat: 0.00137921 max lat: 0.0882167 avg lat: 0.0108062 2023-10-01T10:23:51.332 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T10:23:51.332 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 60 16 88815 88799 92.4882 61.25 0.017893 0.0108062 2023-10-01T10:23:52.332 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 61 16 89801 89785 91.9822 61.625 0.030478 0.0108661 2023-10-01T10:23:52.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:23:52 smithi173 bash[27612]: cluster 2023-10-01T10:23:51.285774+0000 mgr.y (mgr.25072) 48 : cluster [DBG] pgmap v36: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 54 MiB/s wr, 864 op/s 2023-10-01T10:23:52.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:23:52 smithi078 bash[33821]: cluster 2023-10-01T10:23:51.285774+0000 mgr.y (mgr.25072) 48 : cluster [DBG] pgmap v36: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 54 MiB/s wr, 864 op/s 2023-10-01T10:23:53.332 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 62 16 90780 90764 91.4854 61.1875 0.0203651 0.010925 2023-10-01T10:23:54.332 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 63 16 91747 91731 90.9925 60.4375 0.00570166 0.0109843 2023-10-01T10:23:54.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:23:54 smithi173 bash[27612]: cluster 2023-10-01T10:23:53.286208+0000 mgr.y (mgr.25072) 49 : cluster [DBG] pgmap v37: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 62 MiB/s wr, 994 op/s 2023-10-01T10:23:54.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:23:54 smithi078 bash[33821]: cluster 2023-10-01T10:23:53.286208+0000 mgr.y (mgr.25072) 49 : cluster [DBG] pgmap v37: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 62 MiB/s wr, 994 op/s 2023-10-01T10:23:55.332 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 64 16 92746 92730 90.5463 62.4375 0.00843106 0.011039 2023-10-01T10:23:56.332 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 65 16 93685 93669 90.0561 58.6875 0.00797141 0.0110978 2023-10-01T10:23:56.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:23:56 smithi078 bash[33821]: cluster 2023-10-01T10:23:55.286833+0000 mgr.y (mgr.25072) 50 : cluster [DBG] pgmap v38: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-10-01T10:23:56.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:23:56 smithi173 bash[27612]: cluster 2023-10-01T10:23:55.286833+0000 mgr.y (mgr.25072) 50 : cluster [DBG] pgmap v38: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-10-01T10:23:57.332 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 66 16 94636 94620 89.5921 59.4375 0.0289035 0.0111566 2023-10-01T10:23:58.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:23:57 smithi078 bash[33821]: cluster 2023-10-01T10:23:57.287330+0000 mgr.y (mgr.25072) 51 : cluster [DBG] pgmap v39: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 52 MiB/s wr, 832 op/s 2023-10-01T10:23:58.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:23:57 smithi173 bash[27612]: cluster 2023-10-01T10:23:57.287330+0000 mgr.y (mgr.25072) 51 : cluster [DBG] pgmap v39: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 52 MiB/s wr, 832 op/s 2023-10-01T10:23:58.333 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 67 16 95524 95508 89.0832 55.5 0.0240904 0.0112198 2023-10-01T10:23:59.119 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:23:58 smithi173 bash[54497]: ts=2023-10-01T10:23:58.661Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"smithi173.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"smithi078:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-01T10:23:59.332 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 68 16 96464 96448 88.637 58.75 0.0120428 0.0112759 2023-10-01T10:24:00.333 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 69 16 97370 97354 88.173 56.625 0.00783353 0.0113359 2023-10-01T10:24:00.582 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:24:00 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:24:00] "GET /metrics HTTP/1.1" 200 36070 "" "Prometheus/2.33.4" 2023-10-01T10:24:00.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:00 smithi078 bash[33821]: cluster 2023-10-01T10:23:59.288053+0000 mgr.y (mgr.25072) 52 : cluster [DBG] pgmap v40: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-10-01T10:24:00.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:00 smithi173 bash[27612]: cluster 2023-10-01T10:23:59.288053+0000 mgr.y (mgr.25072) 52 : cluster [DBG] pgmap v40: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-10-01T10:24:01.333 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 70 16 98261 98245 87.7089 55.6875 0.0120593 0.0113953 2023-10-01T10:24:02.229 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -e sha1=b56822191ad0416d0c05ed59720df03ad9c2a44e -- bash -c 'ceph orch upgrade resume' 2023-10-01T10:24:02.333 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 71 16 99209 99193 87.308 59.25 0.0196746 0.0114478 2023-10-01T10:24:02.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:02 smithi078 bash[33821]: cluster 2023-10-01T10:24:01.288529+0000 mgr.y (mgr.25072) 53 : cluster [DBG] pgmap v41: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 50 MiB/s wr, 806 op/s 2023-10-01T10:24:02.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:02 smithi173 bash[27612]: cluster 2023-10-01T10:24:01.288529+0000 mgr.y (mgr.25072) 53 : cluster [DBG] pgmap v41: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 50 MiB/s wr, 806 op/s 2023-10-01T10:24:03.333 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 72 16 100122 100106 86.8878 57.0625 0.0059578 0.0115036 2023-10-01T10:24:03.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:03 smithi173 bash[27612]: audit 2023-10-01T10:24:02.373918+0000 mon.a (mon.0) 77 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:24:03.716 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:03 smithi078 bash[33821]: audit 2023-10-01T10:24:02.373918+0000 mon.a (mon.0) 77 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:24:04.333 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 73 16 101024 101008 86.4698 56.375 0.0114039 0.011559 2023-10-01T10:24:04.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:04 smithi173 bash[27612]: cluster 2023-10-01T10:24:03.289037+0000 mgr.y (mgr.25072) 54 : cluster [DBG] pgmap v42: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 857 op/s 2023-10-01T10:24:04.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:04 smithi078 bash[33821]: cluster 2023-10-01T10:24:03.289037+0000 mgr.y (mgr.25072) 54 : cluster [DBG] pgmap v42: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 857 op/s 2023-10-01T10:24:05.333 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 74 15 101954 101939 86.0875 58.1875 0.0184124 0.0116103 2023-10-01T10:24:06.333 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 75 16 102833 102817 85.6713 54.875 0.00905481 0.0116664 2023-10-01T10:24:06.581 INFO:teuthology.orchestra.run.smithi078.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T10:24:06.637 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:06 smithi078 bash[33821]: cluster 2023-10-01T10:24:05.289750+0000 mgr.y (mgr.25072) 55 : cluster [DBG] pgmap v43: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-01T10:24:06.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:06 smithi173 bash[27612]: cluster 2023-10-01T10:24:05.289750+0000 mgr.y (mgr.25072) 55 : cluster [DBG] pgmap v43: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-01T10:24:07.333 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 76 16 103659 103643 85.2233 51.625 0.0185245 0.0117283 2023-10-01T10:24:07.636 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -e sha1=b56822191ad0416d0c05ed59720df03ad9c2a44e -- bash -c 'echo wait for majority of mons to upgrade' 2023-10-01T10:24:07.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:07 smithi173 bash[27612]: audit 2023-10-01T10:24:06.570408+0000 mgr.y (mgr.25072) 56 : audit [DBG] from='client.34182 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:24:07.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:07 smithi173 bash[27612]: cephadm 2023-10-01T10:24:06.570782+0000 mgr.y (mgr.25072) 57 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T10:24:07.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:07 smithi173 bash[27612]: audit 2023-10-01T10:24:06.577699+0000 mon.a (mon.0) 78 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:24:07.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:07 smithi173 bash[27612]: audit 2023-10-01T10:24:06.603811+0000 mon.a (mon.0) 79 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:24:07.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:07 smithi173 bash[27612]: cluster 2023-10-01T10:24:07.290129+0000 mgr.y (mgr.25072) 58 : cluster [DBG] pgmap v44: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 49 MiB/s wr, 779 op/s 2023-10-01T10:24:07.880 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:07 smithi078 bash[33821]: audit 2023-10-01T10:24:06.570408+0000 mgr.y (mgr.25072) 56 : audit [DBG] from='client.34182 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:24:07.881 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:07 smithi078 bash[33821]: cephadm 2023-10-01T10:24:06.570782+0000 mgr.y (mgr.25072) 57 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T10:24:07.881 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:07 smithi078 bash[33821]: audit 2023-10-01T10:24:06.577699+0000 mon.a (mon.0) 78 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:24:07.881 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:07 smithi078 bash[33821]: audit 2023-10-01T10:24:06.603811+0000 mon.a (mon.0) 79 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:24:07.881 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:07 smithi078 bash[33821]: cluster 2023-10-01T10:24:07.290129+0000 mgr.y (mgr.25072) 58 : cluster [DBG] pgmap v44: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 49 MiB/s wr, 779 op/s 2023-10-01T10:24:08.333 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 77 16 104529 104513 84.8226 54.375 0.0112629 0.011784 2023-10-01T10:24:09.119 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:24:08 smithi173 bash[54497]: ts=2023-10-01T10:24:08.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"smithi173.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"smithi078:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-01T10:24:09.333 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 78 16 105422 105406 84.4506 55.8125 0.029535 0.0118355 2023-10-01T10:24:10.333 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 79 16 106311 106295 84.0848 55.5625 0.0175162 0.0118871 2023-10-01T10:24:10.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:10 smithi078 bash[33821]: cluster 2023-10-01T10:24:09.294076+0000 mgr.y (mgr.25072) 59 : cluster [DBG] pgmap v45: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-01T10:24:10.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:24:10 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:24:10] "GET /metrics HTTP/1.1" 200 36065 "" "Prometheus/2.33.4" 2023-10-01T10:24:10.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:10 smithi173 bash[27612]: cluster 2023-10-01T10:24:09.294076+0000 mgr.y (mgr.25072) 59 : cluster [DBG] pgmap v45: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-01T10:24:11.334 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:2023-10-01T10:24:11.332238+0000 min lat: 0.00137921 max lat: 0.0882167 avg lat: 0.0119424 2023-10-01T10:24:11.334 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T10:24:11.334 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 80 16 107153 107137 83.6915 52.625 0.011428 0.0119424 2023-10-01T10:24:11.546 INFO:teuthology.orchestra.run.smithi078.stdout:wait for majority of mons to upgrade 2023-10-01T10:24:12.334 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 81 16 107914 107898 83.2454 47.5625 0.0106873 0.0120072 2023-10-01T10:24:12.546 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -e sha1=b56822191ad0416d0c05ed59720df03ad9c2a44e -- bash -c 'while ! ceph mon versions | grep $sha1 | egrep '"'"': [23]'"'"' ; do sleep 2 ; done' 2023-10-01T10:24:12.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:12 smithi173 bash[27612]: cluster 2023-10-01T10:24:11.294522+0000 mgr.y (mgr.25072) 60 : cluster [DBG] pgmap v46: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 47 MiB/s wr, 750 op/s 2023-10-01T10:24:12.827 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:12 smithi078 bash[33821]: cluster 2023-10-01T10:24:11.294522+0000 mgr.y (mgr.25072) 60 : cluster [DBG] pgmap v46: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 47 MiB/s wr, 750 op/s 2023-10-01T10:24:13.334 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 82 16 108619 108603 82.7676 44.0625 0.0263711 0.0120766 2023-10-01T10:24:14.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:13 smithi078 bash[33821]: audit 2023-10-01T10:24:12.928463+0000 mon.a (mon.0) 80 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:24:14.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:13 smithi078 bash[33821]: audit 2023-10-01T10:24:12.941057+0000 mon.a (mon.0) 81 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:24:14.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:13 smithi078 bash[33821]: audit 2023-10-01T10:24:13.033385+0000 mon.a (mon.0) 82 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:24:14.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:13 smithi078 bash[33821]: audit 2023-10-01T10:24:13.044950+0000 mon.a (mon.0) 83 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:24:14.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:13 smithi078 bash[33821]: cluster 2023-10-01T10:24:13.295015+0000 mgr.y (mgr.25072) 61 : cluster [DBG] pgmap v47: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 794 op/s 2023-10-01T10:24:14.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:13 smithi078 bash[33821]: audit 2023-10-01T10:24:13.641876+0000 mon.a (mon.0) 84 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:24:14.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:13 smithi078 bash[33821]: audit 2023-10-01T10:24:13.643131+0000 mon.a (mon.0) 85 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:24:14.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:13 smithi078 bash[33821]: audit 2023-10-01T10:24:13.656978+0000 mon.a (mon.0) 86 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:24:14.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:13 smithi078 bash[33821]: audit 2023-10-01T10:24:13.747978+0000 mon.a (mon.0) 87 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:24:14.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:13 smithi078 bash[33821]: audit 2023-10-01T10:24:13.752341+0000 mon.a (mon.0) 88 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:24:14.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:13 smithi078 bash[33821]: audit 2023-10-01T10:24:13.754366+0000 mon.a (mon.0) 89 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-01T10:24:14.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:13 smithi078 bash[33821]: audit 2023-10-01T10:24:13.755727+0000 mon.a (mon.0) 90 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-10-01T10:24:14.334 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 83 16 109380 109364 82.3434 47.5625 0.00623996 0.0121377 2023-10-01T10:24:14.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:13 smithi173 bash[27612]: audit 2023-10-01T10:24:12.928463+0000 mon.a (mon.0) 80 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:24:14.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:13 smithi173 bash[27612]: audit 2023-10-01T10:24:12.941057+0000 mon.a (mon.0) 81 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:24:14.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:13 smithi173 bash[27612]: audit 2023-10-01T10:24:13.033385+0000 mon.a (mon.0) 82 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:24:14.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:13 smithi173 bash[27612]: audit 2023-10-01T10:24:13.044950+0000 mon.a (mon.0) 83 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:24:14.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:13 smithi173 bash[27612]: cluster 2023-10-01T10:24:13.295015+0000 mgr.y (mgr.25072) 61 : cluster [DBG] pgmap v47: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 794 op/s 2023-10-01T10:24:14.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:13 smithi173 bash[27612]: audit 2023-10-01T10:24:13.641876+0000 mon.a (mon.0) 84 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:24:14.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:13 smithi173 bash[27612]: audit 2023-10-01T10:24:13.643131+0000 mon.a (mon.0) 85 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:24:14.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:13 smithi173 bash[27612]: audit 2023-10-01T10:24:13.656978+0000 mon.a (mon.0) 86 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:24:14.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:13 smithi173 bash[27612]: audit 2023-10-01T10:24:13.747978+0000 mon.a (mon.0) 87 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:24:14.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:13 smithi173 bash[27612]: audit 2023-10-01T10:24:13.752341+0000 mon.a (mon.0) 88 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:24:14.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:13 smithi173 bash[27612]: audit 2023-10-01T10:24:13.754366+0000 mon.a (mon.0) 89 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-01T10:24:14.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:13 smithi173 bash[27612]: audit 2023-10-01T10:24:13.755727+0000 mon.a (mon.0) 90 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-10-01T10:24:15.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:15 smithi078 bash[33821]: cephadm 2023-10-01T10:24:13.756352+0000 mgr.y (mgr.25072) 62 : cephadm [INF] Upgrade: It appears safe to stop mon.c 2023-10-01T10:24:15.334 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 84 16 110231 110215 81.9962 53.1875 0.00903491 0.0121897 2023-10-01T10:24:15.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:15 smithi173 bash[27612]: cephadm 2023-10-01T10:24:13.756352+0000 mgr.y (mgr.25072) 62 : cephadm [INF] Upgrade: It appears safe to stop mon.c 2023-10-01T10:24:16.321 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:16 smithi078 bash[33821]: cluster 2023-10-01T10:24:15.295895+0000 mgr.y (mgr.25072) 63 : cluster [DBG] pgmap v48: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-01T10:24:16.336 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 85 16 111057 111041 81.6371 51.625 0.0131097 0.0122425 2023-10-01T10:24:16.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:16 smithi173 bash[27612]: cluster 2023-10-01T10:24:15.295895+0000 mgr.y (mgr.25072) 63 : cluster [DBG] pgmap v48: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-01T10:24:17.336 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 86 16 111901 111885 81.3012 52.75 0.00949136 0.0122932 2023-10-01T10:24:17.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:17 smithi078 bash[33821]: cluster 2023-10-01T10:24:17.296343+0000 mgr.y (mgr.25072) 64 : cluster [DBG] pgmap v49: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 44 MiB/s wr, 709 op/s 2023-10-01T10:24:17.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:17 smithi078 bash[33821]: audit 2023-10-01T10:24:17.374109+0000 mon.a (mon.0) 91 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:24:18.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:17 smithi173 bash[27612]: cluster 2023-10-01T10:24:17.296343+0000 mgr.y (mgr.25072) 64 : cluster [DBG] pgmap v49: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 44 MiB/s wr, 709 op/s 2023-10-01T10:24:18.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:17 smithi173 bash[27612]: audit 2023-10-01T10:24:17.374109+0000 mon.a (mon.0) 91 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:24:18.336 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 87 16 112691 112675 80.9342 49.375 0.00965714 0.0123505 2023-10-01T10:24:19.121 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:24:18 smithi173 bash[54497]: ts=2023-10-01T10:24:18.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"smithi173.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"smithi078:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-01T10:24:19.336 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 88 16 113421 113405 80.5329 45.625 0.0188 0.0124111 2023-10-01T10:24:19.694 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:19 smithi078 bash[33821]: cluster 2023-10-01T10:24:18.703871+0000 mon.a (mon.0) 92 : cluster [DBG] mgrmap e34: y(active, since 91s), standbys: x 2023-10-01T10:24:19.694 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:19 smithi078 bash[33821]: cephadm 2023-10-01T10:24:18.986358+0000 mgr.y (mgr.25072) 65 : cephadm [INF] Upgrade: Updating mon.c 2023-10-01T10:24:19.694 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:19 smithi078 bash[33821]: audit 2023-10-01T10:24:18.993642+0000 mon.a (mon.0) 93 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:24:19.694 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:19 smithi078 bash[33821]: audit 2023-10-01T10:24:18.995336+0000 mon.a (mon.0) 94 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T10:24:19.694 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:19 smithi078 bash[33821]: audit 2023-10-01T10:24:19.001208+0000 mon.a (mon.0) 95 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T10:24:19.695 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:19 smithi078 bash[33821]: audit 2023-10-01T10:24:19.006185+0000 mon.a (mon.0) 96 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:24:19.695 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:19 smithi078 bash[33821]: cephadm 2023-10-01T10:24:19.007779+0000 mgr.y (mgr.25072) 66 : cephadm [INF] Deploying daemon mon.c on smithi078 2023-10-01T10:24:19.695 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:19 smithi078 bash[33821]: cluster 2023-10-01T10:24:19.297041+0000 mgr.y (mgr.25072) 67 : cluster [DBG] pgmap v50: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 65 MiB/s wr, 1.05k op/s 2023-10-01T10:24:19.695 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:19 smithi078 bash[33821]: audit 2023-10-01T10:24:19.347385+0000 mon.a (mon.0) 97 : audit [DBG] from='client.? 172.21.15.78:0/2883028663' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:24:20.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:19 smithi173 bash[27612]: cluster 2023-10-01T10:24:18.703871+0000 mon.a (mon.0) 92 : cluster [DBG] mgrmap e34: y(active, since 91s), standbys: x 2023-10-01T10:24:20.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:19 smithi173 bash[27612]: cephadm 2023-10-01T10:24:18.986358+0000 mgr.y (mgr.25072) 65 : cephadm [INF] Upgrade: Updating mon.c 2023-10-01T10:24:20.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:19 smithi173 bash[27612]: audit 2023-10-01T10:24:18.993642+0000 mon.a (mon.0) 93 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:24:20.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:19 smithi173 bash[27612]: audit 2023-10-01T10:24:18.995336+0000 mon.a (mon.0) 94 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T10:24:20.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:19 smithi173 bash[27612]: audit 2023-10-01T10:24:19.001208+0000 mon.a (mon.0) 95 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T10:24:20.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:19 smithi173 bash[27612]: audit 2023-10-01T10:24:19.006185+0000 mon.a (mon.0) 96 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:24:20.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:19 smithi173 bash[27612]: cephadm 2023-10-01T10:24:19.007779+0000 mgr.y (mgr.25072) 66 : cephadm [INF] Deploying daemon mon.c on smithi078 2023-10-01T10:24:20.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:19 smithi173 bash[27612]: cluster 2023-10-01T10:24:19.297041+0000 mgr.y (mgr.25072) 67 : cluster [DBG] pgmap v50: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 65 MiB/s wr, 1.05k op/s 2023-10-01T10:24:20.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:19 smithi173 bash[27612]: audit 2023-10-01T10:24:19.347385+0000 mon.a (mon.0) 97 : audit [DBG] from='client.? 172.21.15.78:0/2883028663' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:24:20.336 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 89 16 114250 114234 80.2102 51.8125 0.0207074 0.0124619 2023-10-01T10:24:20.582 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:24:20 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:24:20] "GET /metrics HTTP/1.1" 200 36069 "" "Prometheus/2.33.4" 2023-10-01T10:24:21.336 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Total time run: 90.0088 2023-10-01T10:24:21.337 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Total writes made: 115066 2023-10-01T10:24:21.337 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Write size: 65536 2023-10-01T10:24:21.337 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Object size: 65536 2023-10-01T10:24:21.337 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Bandwidth (MB/sec): 79.8991 2023-10-01T10:24:21.337 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Stddev Bandwidth: 30.1965 2023-10-01T10:24:21.337 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Max bandwidth (MB/sec): 190.562 2023-10-01T10:24:21.337 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Min bandwidth (MB/sec): 44.0625 2023-10-01T10:24:21.337 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Average IOPS: 1278 2023-10-01T10:24:21.337 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Stddev IOPS: 483.144 2023-10-01T10:24:21.337 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Max IOPS: 3049 2023-10-01T10:24:21.338 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Min IOPS: 705 2023-10-01T10:24:21.338 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Average Latency(s): 0.0125112 2023-10-01T10:24:21.338 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Stddev Latency(s): 0.0078272 2023-10-01T10:24:21.338 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Max latency(s): 0.120887 2023-10-01T10:24:21.338 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Min latency(s): 0.00137921 2023-10-01T10:24:21.346 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Cleaning up (deleting benchmark objects) 2023-10-01T10:24:22.527 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:22 smithi078 systemd[1]: Stopping Ceph mon.c for d8016480-6041-11ee-8db5-212e2dc638e7... 2023-10-01T10:24:22.527 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:22 smithi078 bash[33821]: debug 2023-10-01T10:24:22.248+0000 7f9c20a2f700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-10-01T10:24:22.527 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:22 smithi078 bash[33821]: debug 2023-10-01T10:24:22.248+0000 7f9c20a2f700 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2023-10-01T10:24:23.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:23 smithi078 bash[78172]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-mon-c 2023-10-01T10:24:23.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:23 smithi078 bash[78225]: Error response from daemon: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-mon-c 2023-10-01T10:24:23.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:23 smithi078 systemd[1]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7@mon.c.service: Succeeded. 2023-10-01T10:24:23.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:23 smithi078 systemd[1]: Stopped Ceph mon.c for d8016480-6041-11ee-8db5-212e2dc638e7. 2023-10-01T10:24:24.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:23 smithi078 systemd[1]: Started Ceph mon.c for d8016480-6041-11ee-8db5-212e2dc638e7. 2023-10-01T10:24:25.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.104+0000 7fe05cc72c80 0 set uid:gid to 167:167 (ceph:ceph) 2023-10-01T10:24:25.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.104+0000 7fe05cc72c80 0 ceph version 18.2.0-497-gb5682219 (b56822191ad0416d0c05ed59720df03ad9c2a44e) reef (stable), process ceph-mon, pid 7 2023-10-01T10:24:25.337 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.104+0000 7fe05cc72c80 0 pidfile_write: ignore empty --pid-file 2023-10-01T10:24:25.337 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 0 load: jerasure load: lrc 2023-10-01T10:24:25.337 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: RocksDB version: 7.9.2 2023-10-01T10:24:25.337 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Git sha 0 2023-10-01T10:24:25.337 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Compile date 2023-09-29 15:06:10 2023-10-01T10:24:25.337 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: DB SUMMARY 2023-10-01T10:24:25.337 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: DB Session ID: Y585ZSZM3SI22ZD5NLPH 2023-10-01T10:24:25.337 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: CURRENT file: CURRENT 2023-10-01T10:24:25.337 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: IDENTITY file: IDENTITY 2023-10-01T10:24:25.337 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: MANIFEST file: MANIFEST-000005 size: 1276 Bytes 2023-10-01T10:24:25.337 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000026.sst 2023-10-01T10:24:25.337 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000024.log size: 3136114 ; 2023-10-01T10:24:25.338 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.error_if_exists: 0 2023-10-01T10:24:25.338 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.create_if_missing: 0 2023-10-01T10:24:25.338 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.paranoid_checks: 1 2023-10-01T10:24:25.338 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.flush_verify_memtable_count: 1 2023-10-01T10:24:25.338 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-10-01T10:24:25.338 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-10-01T10:24:25.338 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.env: 0x565415297720 2023-10-01T10:24:25.338 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.fs: PosixFileSystem 2023-10-01T10:24:25.338 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.info_log: 0x565416a97480 2023-10-01T10:24:25.338 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.max_file_opening_threads: 16 2023-10-01T10:24:25.339 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.statistics: (nil) 2023-10-01T10:24:25.339 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.use_fsync: 0 2023-10-01T10:24:25.339 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.max_log_file_size: 0 2023-10-01T10:24:25.339 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-01T10:24:25.339 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.log_file_time_to_roll: 0 2023-10-01T10:24:25.339 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.keep_log_file_num: 1000 2023-10-01T10:24:25.339 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.recycle_log_file_num: 0 2023-10-01T10:24:25.339 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.allow_fallocate: 1 2023-10-01T10:24:25.339 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.allow_mmap_reads: 0 2023-10-01T10:24:25.339 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.allow_mmap_writes: 0 2023-10-01T10:24:25.339 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.use_direct_reads: 0 2023-10-01T10:24:25.339 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-01T10:24:25.340 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.create_missing_column_families: 0 2023-10-01T10:24:25.340 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.db_log_dir: 2023-10-01T10:24:25.340 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.wal_dir: 2023-10-01T10:24:25.340 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.table_cache_numshardbits: 6 2023-10-01T10:24:25.340 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-10-01T10:24:25.340 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-10-01T10:24:25.340 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-01T10:24:25.340 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-01T10:24:25.340 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-10-01T10:24:25.340 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.advise_random_on_open: 1 2023-10-01T10:24:25.340 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.db_write_buffer_size: 0 2023-10-01T10:24:25.340 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.write_buffer_manager: 0x565415d24c80 2023-10-01T10:24:25.341 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-01T10:24:25.341 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-01T10:24:25.341 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.use_adaptive_mutex: 0 2023-10-01T10:24:25.341 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.rate_limiter: (nil) 2023-10-01T10:24:25.341 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-01T10:24:25.341 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.wal_recovery_mode: 2 2023-10-01T10:24:25.341 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.enable_thread_tracking: 0 2023-10-01T10:24:25.341 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.enable_pipelined_write: 0 2023-10-01T10:24:25.341 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.unordered_write: 0 2023-10-01T10:24:25.341 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-01T10:24:25.341 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-01T10:24:25.342 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-01T10:24:25.342 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-01T10:24:25.343 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.row_cache: None 2023-10-01T10:24:25.343 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.wal_filter: None 2023-10-01T10:24:25.343 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-01T10:24:25.343 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.allow_ingest_behind: 0 2023-10-01T10:24:25.343 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.two_write_queues: 0 2023-10-01T10:24:25.344 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.manual_wal_flush: 0 2023-10-01T10:24:25.344 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.wal_compression: 0 2023-10-01T10:24:25.344 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.atomic_flush: 0 2023-10-01T10:24:25.344 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-01T10:24:25.344 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.persist_stats_to_disk: 0 2023-10-01T10:24:25.344 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-10-01T10:24:25.344 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.log_readahead_size: 0 2023-10-01T10:24:25.344 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-01T10:24:25.344 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.best_efforts_recovery: 0 2023-10-01T10:24:25.344 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-01T10:24:25.344 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-01T10:24:25.344 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.allow_data_in_errors: 0 2023-10-01T10:24:25.345 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.db_host_id: __hostname__ 2023-10-01T10:24:25.345 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.enforce_single_del_contracts: true 2023-10-01T10:24:25.345 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.max_background_jobs: 2 2023-10-01T10:24:25.345 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.max_background_compactions: -1 2023-10-01T10:24:25.345 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.max_subcompactions: 1 2023-10-01T10:24:25.345 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-01T10:24:25.345 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-01T10:24:25.345 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.delayed_write_rate : 16777216 2023-10-01T10:24:25.345 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.max_total_wal_size: 0 2023-10-01T10:24:25.345 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-01T10:24:25.345 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.stats_dump_period_sec: 600 2023-10-01T10:24:25.345 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.stats_persist_period_sec: 600 2023-10-01T10:24:25.346 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-01T10:24:25.346 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.max_open_files: -1 2023-10-01T10:24:25.346 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.bytes_per_sync: 0 2023-10-01T10:24:25.346 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-10-01T10:24:25.346 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-10-01T10:24:25.346 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.compaction_readahead_size: 0 2023-10-01T10:24:25.346 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.max_background_flushes: -1 2023-10-01T10:24:25.346 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Compression algorithms supported: 2023-10-01T10:24:25.346 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-01T10:24:25.346 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: kZSTD supported: 0 2023-10-01T10:24:25.346 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: kXpressCompression supported: 0 2023-10-01T10:24:25.347 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: kLZ4HCCompression supported: 1 2023-10-01T10:24:25.347 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: kZlibCompression supported: 1 2023-10-01T10:24:25.347 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: kSnappyCompression supported: 1 2023-10-01T10:24:25.347 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: kLZ4Compression supported: 1 2023-10-01T10:24:25.347 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: kBZip2Compression supported: 0 2023-10-01T10:24:25.347 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-10-01T10:24:25.347 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: DMutex implementation: pthread_mutex_t 2023-10-01T10:24:25.347 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 2023-10-01T10:24:25.347 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-10-01T10:24:25.347 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-01T10:24:25.347 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.merge_operator: 2023-10-01T10:24:25.347 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.compaction_filter: None 2023-10-01T10:24:25.348 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.compaction_filter_factory: None 2023-10-01T10:24:25.348 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.sst_partitioner_factory: None 2023-10-01T10:24:25.348 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-10-01T10:24:25.348 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.table_factory: BlockBasedTable 2023-10-01T10:24:25.348 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x565416a97580) 2023-10-01T10:24:25.348 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: cache_index_and_filter_blocks: 1 2023-10-01T10:24:25.348 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-01T10:24:25.348 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-01T10:24:25.348 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: pin_top_level_index_and_filter: 1 2023-10-01T10:24:25.348 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: index_type: 0 2023-10-01T10:24:25.348 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: data_block_index_type: 0 2023-10-01T10:24:25.348 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: index_shortening: 1 2023-10-01T10:24:25.349 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: data_block_hash_table_util_ratio: 0.750000 2023-10-01T10:24:25.349 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: checksum: 4 2023-10-01T10:24:25.349 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: no_block_cache: 0 2023-10-01T10:24:25.349 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: block_cache: 0x565415da71f0 2023-10-01T10:24:25.349 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: block_cache_name: BinnedLRUCache 2023-10-01T10:24:25.349 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: block_cache_options: 2023-10-01T10:24:25.349 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: capacity : 536870912 2023-10-01T10:24:25.349 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: num_shard_bits : 4 2023-10-01T10:24:25.349 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: strict_capacity_limit : 0 2023-10-01T10:24:25.349 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: high_pri_pool_ratio: 0.000 2023-10-01T10:24:25.349 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: block_cache_compressed: (nil) 2023-10-01T10:24:25.349 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: persistent_cache: (nil) 2023-10-01T10:24:25.350 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: block_size: 4096 2023-10-01T10:24:25.350 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: block_size_deviation: 10 2023-10-01T10:24:25.350 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: block_restart_interval: 16 2023-10-01T10:24:25.350 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: index_block_restart_interval: 1 2023-10-01T10:24:25.350 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: metadata_block_size: 4096 2023-10-01T10:24:25.350 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: partition_filters: 0 2023-10-01T10:24:25.350 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: use_delta_encoding: 1 2023-10-01T10:24:25.350 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: filter_policy: bloomfilter 2023-10-01T10:24:25.350 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: whole_key_filtering: 1 2023-10-01T10:24:25.350 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: verify_compression: 0 2023-10-01T10:24:25.350 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: read_amp_bytes_per_bit: 0 2023-10-01T10:24:25.350 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: format_version: 5 2023-10-01T10:24:25.351 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: enable_index_compression: 1 2023-10-01T10:24:25.351 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: block_align: 0 2023-10-01T10:24:25.351 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: max_auto_readahead_size: 262144 2023-10-01T10:24:25.351 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: prepopulate_block_cache: 0 2023-10-01T10:24:25.351 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: initial_auto_readahead_size: 8192 2023-10-01T10:24:25.351 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: num_file_reads_for_auto_readahead: 2 2023-10-01T10:24:25.351 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.write_buffer_size: 33554432 2023-10-01T10:24:25.351 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.max_write_buffer_number: 2 2023-10-01T10:24:25.351 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.compression: NoCompression 2023-10-01T10:24:25.351 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.bottommost_compression: Disabled 2023-10-01T10:24:25.351 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.prefix_extractor: nullptr 2023-10-01T10:24:25.351 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-01T10:24:25.352 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.num_levels: 7 2023-10-01T10:24:25.352 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-01T10:24:25.352 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-01T10:24:25.352 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-01T10:24:25.352 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-01T10:24:25.352 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-01T10:24:25.352 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-01T10:24:25.352 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-01T10:24:25.352 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-01T10:24:25.352 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-01T10:24:25.352 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-01T10:24:25.352 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-10-01T10:24:25.353 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-10-01T10:24:25.353 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.compression_opts.window_bits: -14 2023-10-01T10:24:25.353 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.compression_opts.level: 32767 2023-10-01T10:24:25.353 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.compression_opts.strategy: 0 2023-10-01T10:24:25.353 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-01T10:24:25.353 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-01T10:24:25.353 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-10-01T10:24:25.353 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-01T10:24:25.353 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.compression_opts.enabled: false 2023-10-01T10:24:25.353 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-10-01T10:24:25.353 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-01T10:24:25.353 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-01T10:24:25.353 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-01T10:24:25.354 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.target_file_size_base: 67108864 2023-10-01T10:24:25.354 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.target_file_size_multiplier: 1 2023-10-01T10:24:25.354 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-01T10:24:25.354 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-01T10:24:25.354 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-01T10:24:25.354 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-01T10:24:25.354 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-01T10:24:25.354 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-01T10:24:25.354 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-01T10:24:25.354 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-01T10:24:25.354 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-01T10:24:25.355 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-01T10:24:25.355 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-01T10:24:25.355 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-01T10:24:25.355 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-10-01T10:24:25.355 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.arena_block_size: 1048576 2023-10-01T10:24:25.355 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-01T10:24:25.355 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-01T10:24:25.355 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.disable_auto_compactions: 0 2023-10-01T10:24:25.355 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-01T10:24:25.355 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-01T10:24:25.355 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-01T10:24:25.355 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-01T10:24:25.356 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-01T10:24:25.356 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-01T10:24:25.356 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-01T10:24:25.356 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-01T10:24:25.356 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-01T10:24:25.356 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-01T10:24:25.356 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.table_properties_collectors: 2023-10-01T10:24:25.356 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.inplace_update_support: 0 2023-10-01T10:24:25.356 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-10-01T10:24:25.356 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-01T10:24:25.356 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-01T10:24:25.356 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.memtable_huge_page_size: 0 2023-10-01T10:24:25.356 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.bloom_locality: 0 2023-10-01T10:24:25.357 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.max_successive_merges: 0 2023-10-01T10:24:25.357 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-10-01T10:24:25.357 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.paranoid_file_checks: 0 2023-10-01T10:24:25.357 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.force_consistency_checks: 1 2023-10-01T10:24:25.357 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.report_bg_io_stats: 0 2023-10-01T10:24:25.357 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.ttl: 2592000 2023-10-01T10:24:25.357 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-10-01T10:24:25.357 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.preclude_last_level_data_seconds: 0 2023-10-01T10:24:25.357 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.preserve_internal_time_seconds: 0 2023-10-01T10:24:25.357 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.enable_blob_files: false 2023-10-01T10:24:25.357 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.min_blob_size: 0 2023-10-01T10:24:25.357 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.blob_file_size: 268435456 2023-10-01T10:24:25.357 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.blob_compression_type: NoCompression 2023-10-01T10:24:25.358 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.enable_blob_garbage_collection: false 2023-10-01T10:24:25.358 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-01T10:24:25.358 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-10-01T10:24:25.358 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.blob_compaction_readahead_size: 0 2023-10-01T10:24:25.358 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.blob_file_starting_level: 0 2023-10-01T10:24:25.358 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe05cc72c80 4 rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-10-01T10:24:25.358 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.112+0000 7fe052490700 3 rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 26.sst 2023-10-01T10:24:25.358 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.116+0000 7fe05cc72c80 4 rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 28, last_sequence is 17639, log_number is 24,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-01T10:24:25.358 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.116+0000 7fe05cc72c80 4 rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 24 2023-10-01T10:24:25.359 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.116+0000 7fe05cc72c80 4 rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 6d4da773-fdd0-4ab8-86ce-8779735d428b 2023-10-01T10:24:25.359 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.116+0000 7fe05cc72c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696155865117557, "job": 1, "event": "recovery_started", "wal_files": [24]} 2023-10-01T10:24:25.359 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.116+0000 7fe05cc72c80 4 rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #24 mode 2 2023-10-01T10:24:25.359 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.124+0000 7fe05cc72c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696155865128917, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 29, "file_size": 2083488, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 17634, "largest_seqno": 18206, "table_properties": {"data_size": 2079592, "index_size": 2434, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 453, "raw_key_size": 3531, "raw_average_key_size": 20, "raw_value_size": 2074476, "raw_average_value_size": 12131, "num_data_blocks": 107, "num_entries": 171, "num_filter_entries": 171, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1696155865, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "6d4da773-fdd0-4ab8-86ce-8779735d428b", "db_session_id": "Y585ZSZM3SI22ZD5NLPH", "orig_file_number": 29, "seqno_to_time_mapping": "N/A"}} 2023-10-01T10:24:25.359 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.124+0000 7fe05cc72c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696155865129076, "job": 1, "event": "recovery_finished"} 2023-10-01T10:24:25.359 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.124+0000 7fe05cc72c80 4 rocksdb: [db/version_set.cc:5047] Creating manifest 31 2023-10-01T10:24:25.359 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.136+0000 7fe05cc72c80 4 rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000024.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-10-01T10:24:25.359 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.136+0000 7fe05cc72c80 4 rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x565415e48000 2023-10-01T10:24:25.359 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.136+0000 7fe05cc72c80 4 rocksdb: DB pointer 0x565415e36000 2023-10-01T10:24:25.359 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.136+0000 7fe04b482700 4 rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-10-01T10:24:25.359 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.136+0000 7fe04b482700 4 rocksdb: [db/db_impl/db_impl.cc:1111] 2023-10-01T10:24:25.359 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: ** DB Stats ** 2023-10-01T10:24:25.360 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-01T10:24:25.360 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-01T10:24:25.360 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-01T10:24:25.360 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-01T10:24:25.360 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-01T10:24:25.360 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-01T10:24:25.360 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-01T10:24:25.360 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: ** Compaction Stats [default] ** 2023-10-01T10:24:25.360 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-01T10:24:25.360 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-10-01T10:24:25.360 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: L0 1/0 1.99 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 418.3 0.00 0.00 1 0.005 0 0 0.0 0.0 2023-10-01T10:24:25.360 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: L6 1/0 32.18 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-10-01T10:24:25.361 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: Sum 2/0 34.17 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 418.3 0.00 0.00 1 0.005 0 0 0.0 0.0 2023-10-01T10:24:25.361 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 418.3 0.00 0.00 1 0.005 0 0 0.0 0.0 2023-10-01T10:24:25.361 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: ** Compaction Stats [default] ** 2023-10-01T10:24:25.361 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-01T10:24:25.361 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-01T10:24:25.361 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 418.3 0.00 0.00 1 0.005 0 0 0.0 0.0 2023-10-01T10:24:25.361 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-10-01T10:24:25.361 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-01T10:24:25.361 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: Flush(GB): cumulative 0.002, interval 0.002 2023-10-01T10:24:25.361 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-01T10:24:25.361 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: AddFile(Total Files): cumulative 0, interval 0 2023-10-01T10:24:25.361 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: AddFile(L0 Files): cumulative 0, interval 0 2023-10-01T10:24:25.361 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: AddFile(Keys): cumulative 0, interval 0 2023-10-01T10:24:25.362 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: Cumulative compaction: 0.00 GB write, 85.55 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-01T10:24:25.362 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: Interval compaction: 0.00 GB write, 85.55 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-01T10:24:25.362 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-01T10:24:25.362 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: Block cache BinnedLRUCache@0x565415da71f0#7 capacity: 512.00 MB usage: 81.92 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.2e-05 secs_since: 0 2023-10-01T10:24:25.362 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: Block cache entry stats(count,size,portion): DataBlock(3,22.73 KB,0.00433624%) FilterBlock(2,8.53 KB,0.00162721%) IndexBlock(2,50.66 KB,0.00966191%) Misc(1,0.00 KB,0%) 2023-10-01T10:24:25.362 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: ** File Read Latency Histogram By Level [default] ** 2023-10-01T10:24:25.362 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.136+0000 7fe05cc72c80 0 starting mon.c rank 2 at public addrs [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] at bind addrs [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid d8016480-6041-11ee-8db5-212e2dc638e7 2023-10-01T10:24:25.362 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.140+0000 7fe05cc72c80 1 mon.c@-1(???) e3 preinit fsid d8016480-6041-11ee-8db5-212e2dc638e7 2023-10-01T10:24:25.362 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.140+0000 7fe05cc72c80 0 mon.c@-1(???).mds e7 new map 2023-10-01T10:24:25.362 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.140+0000 7fe05cc72c80 0 mon.c@-1(???).mds e7 print_map 2023-10-01T10:24:25.362 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: e7 2023-10-01T10:24:25.363 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: enable_multiple, ever_enabled_multiple: 1,1 2023-10-01T10:24:25.363 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-01T10:24:25.363 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: legacy client fscid: 1 2023-10-01T10:24:25.363 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: 2023-10-01T10:24:25.363 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: Filesystem 'foo' (1) 2023-10-01T10:24:25.363 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: fs_name foo 2023-10-01T10:24:25.363 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: epoch 5 2023-10-01T10:24:25.363 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: flags 12 joinable allow_snaps allow_multimds_snaps 2023-10-01T10:24:25.363 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: created 2023-10-01T10:15:39.285114+0000 2023-10-01T10:24:25.363 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: modified 2023-10-01T10:15:45.790305+0000 2023-10-01T10:24:25.363 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: tableserver 0 2023-10-01T10:24:25.363 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: root 0 2023-10-01T10:24:25.364 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: session_timeout 60 2023-10-01T10:24:25.364 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: session_autoclose 300 2023-10-01T10:24:25.364 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: max_file_size 1099511627776 2023-10-01T10:24:25.364 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: required_client_features {} 2023-10-01T10:24:25.364 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: last_failure 0 2023-10-01T10:24:25.364 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: last_failure_osd_epoch 0 2023-10-01T10:24:25.364 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-01T10:24:25.364 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: max_mds 1 2023-10-01T10:24:25.364 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: in 0 2023-10-01T10:24:25.364 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: up {0=14622} 2023-10-01T10:24:25.364 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: failed 2023-10-01T10:24:25.364 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: damaged 2023-10-01T10:24:25.365 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: stopped 2023-10-01T10:24:25.365 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: data_pools [4] 2023-10-01T10:24:25.365 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: metadata_pool 3 2023-10-01T10:24:25.365 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: inline_data disabled 2023-10-01T10:24:25.365 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: balancer 2023-10-01T10:24:25.365 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: bal_rank_mask -1 2023-10-01T10:24:25.365 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: standby_count_wanted 1 2023-10-01T10:24:25.365 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: [mds.foo.smithi078.vjqrth{0:14622} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.78:6834/2348005403,v1:172.21.15.78:6835/2348005403] compat {c=[1],r=[1],i=[7ff]}] 2023-10-01T10:24:25.365 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: 2023-10-01T10:24:25.365 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: 2023-10-01T10:24:25.365 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: Standby daemons: 2023-10-01T10:24:25.365 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: 2023-10-01T10:24:25.365 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: [mds.foo.smithi173.vrqqoc{-1:24478} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.173:6832/2937343008,v1:172.21.15.173:6833/2937343008] compat {c=[1],r=[1],i=[7ff]}] 2023-10-01T10:24:25.366 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.140+0000 7fe05cc72c80 0 mon.c@-1(???).osd e77 crush map has features 3314933000854323200, adjusting msgr requires 2023-10-01T10:24:25.366 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.140+0000 7fe05cc72c80 0 mon.c@-1(???).osd e77 crush map has features 432629239337189376, adjusting msgr requires 2023-10-01T10:24:25.366 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.140+0000 7fe05cc72c80 0 mon.c@-1(???).osd e77 crush map has features 432629239337189376, adjusting msgr requires 2023-10-01T10:24:25.366 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.140+0000 7fe05cc72c80 0 mon.c@-1(???).osd e77 crush map has features 432629239337189376, adjusting msgr requires 2023-10-01T10:24:25.366 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:24:25 smithi078 bash[78274]: debug 2023-10-01T10:24:25.140+0000 7fe05cc72c80 1 mon.c@-1(???).paxosservice(auth 1..21) refresh upgraded, format 0 -> 3 2023-10-01T10:24:29.119 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:24:28 smithi173 bash[54497]: ts=2023-10-01T10:24:28.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"smithi173.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"smithi078:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-01T10:24:30.582 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:24:30 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:24:30] "GET /metrics HTTP/1.1" 200 36069 "" "Prometheus/2.33.4" 2023-10-01T10:24:30.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:30 smithi173 bash[27612]: cluster 2023-10-01T10:24:21.297475+0000 mgr.y (mgr.25072) 68 : cluster [DBG] pgmap v51: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 43 MiB/s wr, 680 op/s 2023-10-01T10:24:30.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:30 smithi173 bash[27612]: audit 2023-10-01T10:24:22.106510+0000 mon.a (mon.0) 98 : audit [DBG] from='client.? 172.21.15.78:0/694448440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:24:31.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:31 smithi173 bash[27612]: cluster 2023-10-01T10:24:23.297942+0000 mgr.y (mgr.25072) 69 : cluster [DBG] pgmap v52: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 44 MiB/s wr, 720 op/s 2023-10-01T10:24:31.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:31 smithi173 bash[27612]: audit 2023-10-01T10:24:24.840083+0000 mon.a (mon.0) 99 : audit [DBG] from='client.? 172.21.15.78:0/3239132624' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:24:31.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:31 smithi173 bash[27612]: cluster 2023-10-01T10:24:25.298598+0000 mgr.y (mgr.25072) 70 : cluster [DBG] pgmap v53: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 52 MiB/s wr, 969 op/s 2023-10-01T10:24:31.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:31 smithi173 bash[27612]: cluster 2023-10-01T10:24:25.356617+0000 mon.b (mon.1) 60 : cluster [INF] mon.b calling monitor election 2023-10-01T10:24:31.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:31 smithi173 bash[27612]: cluster 2023-10-01T10:24:25.356805+0000 mon.a (mon.0) 100 : cluster [INF] mon.a calling monitor election 2023-10-01T10:24:31.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:31 smithi173 bash[27612]: cluster 2023-10-01T10:24:27.299019+0000 mgr.y (mgr.25072) 71 : cluster [DBG] pgmap v54: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 34 MiB/s wr, 671 op/s 2023-10-01T10:24:31.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:31 smithi173 bash[27612]: cluster 2023-10-01T10:24:29.299633+0000 mgr.y (mgr.25072) 72 : cluster [DBG] pgmap v55: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 34 MiB/s wr, 1.01k op/s 2023-10-01T10:24:31.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:31 smithi173 bash[27612]: cluster 2023-10-01T10:24:30.359149+0000 mon.a (mon.0) 101 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-10-01T10:24:31.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:31 smithi173 bash[27612]: cluster 2023-10-01T10:24:30.371322+0000 mon.a (mon.0) 102 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0],b=[v2:172.21.15.173:3300/0,v1:172.21.15.173:6789/0],c=[v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0]} removed_ranks: {} 2023-10-01T10:24:31.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:31 smithi173 bash[27612]: cluster 2023-10-01T10:24:30.389907+0000 mon.a (mon.0) 103 : cluster [DBG] fsmap foo:1 {0=foo.smithi078.vjqrth=up:active} 1 up:standby 2023-10-01T10:24:31.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:31 smithi173 bash[27612]: cluster 2023-10-01T10:24:30.390078+0000 mon.a (mon.0) 104 : cluster [DBG] osdmap e77: 8 total, 8 up, 8 in 2023-10-01T10:24:31.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:31 smithi173 bash[27612]: cluster 2023-10-01T10:24:30.390957+0000 mon.a (mon.0) 105 : cluster [DBG] mgrmap e34: y(active, since 103s), standbys: x 2023-10-01T10:24:31.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:31 smithi173 bash[27612]: cluster 2023-10-01T10:24:30.393613+0000 mon.a (mon.0) 106 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2023-10-01T10:24:31.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:31 smithi173 bash[27612]: cluster 2023-10-01T10:24:30.406485+0000 mon.a (mon.0) 107 : cluster [WRN] Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2023-10-01T10:24:31.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:31 smithi173 bash[27612]: cluster 2023-10-01T10:24:30.406568+0000 mon.a (mon.0) 108 : cluster [WRN] [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2023-10-01T10:24:31.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:31 smithi173 bash[27612]: cluster 2023-10-01T10:24:30.406669+0000 mon.a (mon.0) 109 : cluster [WRN] mon.c (rank 2) addr [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0] is down (out of quorum) 2023-10-01T10:24:31.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:31 smithi173 bash[27612]: audit 2023-10-01T10:24:30.414381+0000 mon.a (mon.0) 110 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:24:31.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:31 smithi173 bash[27612]: audit 2023-10-01T10:24:30.422206+0000 mon.a (mon.0) 111 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:24:31.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:31 smithi173 bash[27612]: audit 2023-10-01T10:24:30.426220+0000 mon.a (mon.0) 112 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:24:31.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:31 smithi173 bash[27612]: audit 2023-10-01T10:24:31.203512+0000 mon.a (mon.0) 113 : audit [DBG] from='client.? 172.21.15.78:0/744453728' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:24:32.889 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:32 smithi173 bash[27612]: cluster 2023-10-01T10:24:25.354320+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2023-10-01T10:24:32.889 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:32 smithi173 bash[27612]: cluster 2023-10-01T10:24:31.300073+0000 mgr.y (mgr.25072) 73 : cluster [DBG] pgmap v56: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 12 MiB/s wr, 673 op/s 2023-10-01T10:24:32.889 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:32 smithi173 bash[27612]: cluster 2023-10-01T10:24:31.386815+0000 mon.a (mon.0) 114 : cluster [INF] mon.a calling monitor election 2023-10-01T10:24:32.889 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:32 smithi173 bash[27612]: cluster 2023-10-01T10:24:31.392536+0000 mon.b (mon.1) 61 : cluster [INF] mon.b calling monitor election 2023-10-01T10:24:32.889 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:32 smithi173 bash[27612]: cluster 2023-10-01T10:24:31.414096+0000 mon.a (mon.0) 115 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-01T10:24:32.890 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:32 smithi173 bash[27612]: cluster 2023-10-01T10:24:31.420403+0000 mon.a (mon.0) 116 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0],b=[v2:172.21.15.173:3300/0,v1:172.21.15.173:6789/0],c=[v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0]} removed_ranks: {} 2023-10-01T10:24:32.890 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:32 smithi173 bash[27612]: cluster 2023-10-01T10:24:31.421234+0000 mon.a (mon.0) 117 : cluster [DBG] fsmap foo:1 {0=foo.smithi078.vjqrth=up:active} 1 up:standby 2023-10-01T10:24:32.890 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:32 smithi173 bash[27612]: cluster 2023-10-01T10:24:31.421394+0000 mon.a (mon.0) 118 : cluster [DBG] osdmap e77: 8 total, 8 up, 8 in 2023-10-01T10:24:32.890 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:32 smithi173 bash[27612]: cluster 2023-10-01T10:24:31.422287+0000 mon.a (mon.0) 119 : cluster [DBG] mgrmap e34: y(active, since 104s), standbys: x 2023-10-01T10:24:32.890 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:32 smithi173 bash[27612]: cluster 2023-10-01T10:24:31.422905+0000 mon.a (mon.0) 120 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-10-01T10:24:32.890 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:32 smithi173 bash[27612]: cluster 2023-10-01T10:24:31.423001+0000 mon.a (mon.0) 121 : cluster [INF] Cluster is now healthy 2023-10-01T10:24:32.890 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:32 smithi173 bash[27612]: cluster 2023-10-01T10:24:31.432792+0000 mon.a (mon.0) 122 : cluster [INF] overall HEALTH_OK 2023-10-01T10:24:32.890 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:32 smithi173 bash[27612]: audit 2023-10-01T10:24:32.374431+0000 mon.a (mon.0) 123 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:24:33.970 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 18.2.0-497-gb5682219 (b56822191ad0416d0c05ed59720df03ad9c2a44e) reef (stable)": 2 2023-10-01T10:24:34.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:34 smithi173 bash[27612]: cluster 2023-10-01T10:24:33.300742+0000 mgr.y (mgr.25072) 74 : cluster [DBG] pgmap v57: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 12 MiB/s wr, 673 op/s 2023-10-01T10:24:34.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:34 smithi173 bash[27612]: audit 2023-10-01T10:24:33.937824+0000 mon.a (mon.0) 124 : audit [DBG] from='client.? 172.21.15.78:0/448982698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:24:35.582 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:24:35 smithi078 bash[71214]: debug 2023-10-01T10:24:35.152+0000 7fd9c245e700 -1 mgr.server handle_report got status from non-daemon mon.c 2023-10-01T10:24:36.141 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -e sha1=b56822191ad0416d0c05ed59720df03ad9c2a44e -- bash -c 'ceph orch ps' 2023-10-01T10:24:36.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:36 smithi173 bash[27612]: cluster 2023-10-01T10:24:35.301499+0000 mgr.y (mgr.25072) 75 : cluster [DBG] pgmap v58: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 11 MiB/s wr, 962 op/s 2023-10-01T10:24:38.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:37 smithi173 bash[27612]: cluster 2023-10-01T10:24:37.301977+0000 mgr.y (mgr.25072) 76 : cluster [DBG] pgmap v59: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 669 op/s 2023-10-01T10:24:39.051 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:24:38 smithi173 bash[54497]: ts=2023-10-01T10:24:38.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"smithi173.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"smithi078:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-01T10:24:39.051 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:39 smithi173 bash[27612]: audit 2023-10-01T10:24:38.048101+0000 mon.a (mon.0) 125 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:24:39.052 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:39 smithi173 bash[27612]: audit 2023-10-01T10:24:38.061150+0000 mon.a (mon.0) 126 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:24:40.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:40 smithi173 bash[27612]: audit 2023-10-01T10:24:39.064412+0000 mon.a (mon.0) 127 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:24:40.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:40 smithi173 bash[27612]: audit 2023-10-01T10:24:39.071433+0000 mon.a (mon.0) 128 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:24:40.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:40 smithi173 bash[27612]: cluster 2023-10-01T10:24:39.302744+0000 mgr.y (mgr.25072) 77 : cluster [DBG] pgmap v60: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 976 op/s 2023-10-01T10:24:40.460 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-01T10:24:40.460 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.a smithi078 *:9093,9094 running (4m) - 10m 13.9M - 0.23.0 ba2b418f427c 90745d6cf2c8 2023-10-01T10:24:40.460 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.a smithi173 *:3000 running (3m) - 10m 40.5M - 8.3.5 dad864ee21e9 4874aca3a861 2023-10-01T10:24:40.460 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foo.smithi078.vjqrth smithi078 running (8m) - 8m 16.6M - 16.2.14-44-geeb13ac9 94a6c4498547 aedf3352e433 2023-10-01T10:24:40.460 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foo.smithi173.vrqqoc smithi173 running (8m) - 8m 13.6M - 16.2.14-44-geeb13ac9 94a6c4498547 3815ca08717b 2023-10-01T10:24:40.460 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.x smithi173 *:8443,9283,8765 running (2m) - 17m 426M - 18.2.0-497-gb5682219 a3fc96fed5ac 0e4d4ec3a75c 2023-10-01T10:24:40.460 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.y smithi078 *:8443,9283,8765 running (3m) - 19m 474M - 18.2.0-497-gb5682219 a3fc96fed5ac bdb8227ce5b1 2023-10-01T10:24:40.460 INFO:teuthology.orchestra.run.smithi078.stdout:mon.a smithi078 running (2m) - 19m 51.1M 2048M 18.2.0-497-gb5682219 a3fc96fed5ac 62398b61255b 2023-10-01T10:24:40.461 INFO:teuthology.orchestra.run.smithi078.stdout:mon.b smithi173 running (17m) - 17m 52.8M 2048M 16.2.14-44-geeb13ac9 94a6c4498547 9ab8649fee56 2023-10-01T10:24:40.461 INFO:teuthology.orchestra.run.smithi078.stdout:mon.c smithi078 running (16s) - 17m 19.3M 2048M 18.2.0-497-gb5682219 a3fc96fed5ac 5f02c5b8aecd 2023-10-01T10:24:40.461 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.a smithi078 *:9100 running (4m) - 11m 9396k - 1.3.1 1dbe0e931976 3c271e279ae2 2023-10-01T10:24:40.461 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.b smithi173 *:9100 running (4m) - 11m 9123k - 1.3.1 1dbe0e931976 cf13fd39e3de 2023-10-01T10:24:40.461 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (16m) - 17m 396M 2524M 16.2.14-44-geeb13ac9 94a6c4498547 f6a90fa5df1a 2023-10-01T10:24:40.461 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (16m) - 16m 335M 2524M 16.2.14-44-geeb13ac9 94a6c4498547 da5330a1eff3 2023-10-01T10:24:40.461 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (15m) - 15m 483M 2524M 16.2.14-44-geeb13ac9 94a6c4498547 685fab0bfbf6 2023-10-01T10:24:40.461 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (14m) - 14m 470M 2524M 16.2.14-44-geeb13ac9 94a6c4498547 c3823edbc5f1 2023-10-01T10:24:40.461 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi173 running (14m) - 14m 404M 2524M 16.2.14-44-geeb13ac9 94a6c4498547 f6a3e2fc7289 2023-10-01T10:24:40.461 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi173 running (13m) - 13m 437M 2524M 16.2.14-44-geeb13ac9 94a6c4498547 b4ab64b3b760 2023-10-01T10:24:40.461 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi173 running (12m) - 12m 277M 2524M 16.2.14-44-geeb13ac9 94a6c4498547 67631580752d 2023-10-01T10:24:40.461 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi173 running (12m) - 12m 375M 2524M 16.2.14-44-geeb13ac9 94a6c4498547 86157c8ed4b2 2023-10-01T10:24:40.462 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.a smithi173 *:9095 running (2m) - 11m 54.4M - 2.33.4 514e6a882f6e 94c2cb9de105 2023-10-01T10:24:40.523 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:24:40 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:24:40] "GET /metrics HTTP/1.1" 200 36069 "" "Prometheus/2.33.4" 2023-10-01T10:24:41.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:41 smithi173 bash[27612]: audit 2023-10-01T10:24:40.439142+0000 mgr.y (mgr.25072) 78 : audit [DBG] from='client.34215 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:24:41.457 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -e sha1=b56822191ad0416d0c05ed59720df03ad9c2a44e -- bash -c 'ceph orch upgrade pause' 2023-10-01T10:24:42.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:42 smithi173 bash[27612]: cluster 2023-10-01T10:24:41.303179+0000 mgr.y (mgr.25072) 79 : cluster [DBG] pgmap v61: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 635 op/s 2023-10-01T10:24:44.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:44 smithi173 bash[27612]: cluster 2023-10-01T10:24:43.303687+0000 mgr.y (mgr.25072) 80 : cluster [DBG] pgmap v62: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 635 op/s 2023-10-01T10:24:46.296 INFO:teuthology.orchestra.run.smithi078.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T10:24:46.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:46 smithi173 bash[27612]: cluster 2023-10-01T10:24:45.305489+0000 mgr.y (mgr.25072) 81 : cluster [DBG] pgmap v63: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 955 op/s 2023-10-01T10:24:46.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:46 smithi173 bash[27612]: audit 2023-10-01T10:24:46.291061+0000 mon.a (mon.0) 129 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:24:47.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:47 smithi173 bash[27612]: audit 2023-10-01T10:24:46.280316+0000 mgr.y (mgr.25072) 82 : audit [DBG] from='client.34218 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:24:47.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:47 smithi173 bash[27612]: cephadm 2023-10-01T10:24:46.281006+0000 mgr.y (mgr.25072) 83 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T10:24:48.161 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -e sha1=b56822191ad0416d0c05ed59720df03ad9c2a44e -- bash -c 'sleep 60' 2023-10-01T10:24:48.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:48 smithi173 bash[27612]: cluster 2023-10-01T10:24:47.305900+0000 mgr.y (mgr.25072) 84 : cluster [DBG] pgmap v64: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 626 op/s 2023-10-01T10:24:48.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:48 smithi173 bash[27612]: audit 2023-10-01T10:24:47.382805+0000 mon.a (mon.0) 130 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:24:49.119 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:24:48 smithi173 bash[54497]: ts=2023-10-01T10:24:48.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"smithi173.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"smithi078:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-01T10:24:50.585 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:24:50 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:24:50] "GET /metrics HTTP/1.1" 200 36065 "" "Prometheus/2.33.4" 2023-10-01T10:24:50.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:50 smithi173 bash[27612]: cluster 2023-10-01T10:24:49.306445+0000 mgr.y (mgr.25072) 85 : cluster [DBG] pgmap v65: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 942 op/s 2023-10-01T10:24:52.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:51 smithi173 bash[27612]: audit 2023-10-01T10:24:50.908342+0000 mon.a (mon.0) 131 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:24:52.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:51 smithi173 bash[27612]: audit 2023-10-01T10:24:50.918285+0000 mon.a (mon.0) 132 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:24:52.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:51 smithi173 bash[27612]: audit 2023-10-01T10:24:50.919599+0000 mon.a (mon.0) 133 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:24:52.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:51 smithi173 bash[27612]: audit 2023-10-01T10:24:50.923731+0000 mon.a (mon.0) 134 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:24:52.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:51 smithi173 bash[27612]: audit 2023-10-01T10:24:50.932637+0000 mon.a (mon.0) 135 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:24:52.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:51 smithi173 bash[27612]: audit 2023-10-01T10:24:51.019441+0000 mon.a (mon.0) 136 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:24:52.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:51 smithi173 bash[27612]: audit 2023-10-01T10:24:51.021592+0000 mon.a (mon.0) 137 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:24:52.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:51 smithi173 bash[27612]: audit 2023-10-01T10:24:51.029692+0000 mon.a (mon.0) 138 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:24:52.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:51 smithi173 bash[27612]: audit 2023-10-01T10:24:51.036687+0000 mon.a (mon.0) 139 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:24:52.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:51 smithi173 bash[27612]: cluster 2023-10-01T10:24:51.306854+0000 mgr.y (mgr.25072) 86 : cluster [DBG] pgmap v66: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 636 op/s 2023-10-01T10:24:54.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:54 smithi173 bash[27612]: cluster 2023-10-01T10:24:53.307323+0000 mgr.y (mgr.25072) 87 : cluster [DBG] pgmap v67: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 636 op/s 2023-10-01T10:24:56.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:56 smithi173 bash[27612]: cluster 2023-10-01T10:24:55.308104+0000 mgr.y (mgr.25072) 88 : cluster [DBG] pgmap v68: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 950 op/s 2023-10-01T10:24:58.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:24:57 smithi173 bash[27612]: cluster 2023-10-01T10:24:57.308530+0000 mgr.y (mgr.25072) 89 : cluster [DBG] pgmap v69: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 630 op/s 2023-10-01T10:24:59.119 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:24:58 smithi173 bash[54497]: ts=2023-10-01T10:24:58.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"smithi173.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"smithi078:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-01T10:25:00.582 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:25:00 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:25:00] "GET /metrics HTTP/1.1" 200 36065 "" "Prometheus/2.33.4" 2023-10-01T10:25:00.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:25:00 smithi173 bash[27612]: cluster 2023-10-01T10:24:59.309238+0000 mgr.y (mgr.25072) 90 : cluster [DBG] pgmap v70: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 950 op/s 2023-10-01T10:25:02.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:25:02 smithi173 bash[27612]: cluster 2023-10-01T10:25:01.309690+0000 mgr.y (mgr.25072) 91 : cluster [DBG] pgmap v71: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 633 op/s 2023-10-01T10:25:03.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:25:03 smithi173 bash[27612]: audit 2023-10-01T10:25:02.383036+0000 mon.a (mon.0) 140 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:25:04.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:25:04 smithi173 bash[27612]: cluster 2023-10-01T10:25:03.310268+0000 mgr.y (mgr.25072) 92 : cluster [DBG] pgmap v72: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 633 op/s 2023-10-01T10:25:06.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:25:06 smithi173 bash[27612]: cluster 2023-10-01T10:25:05.310995+0000 mgr.y (mgr.25072) 93 : cluster [DBG] pgmap v73: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 947 op/s 2023-10-01T10:25:08.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:25:07 smithi173 bash[27612]: cluster 2023-10-01T10:25:07.311409+0000 mgr.y (mgr.25072) 94 : cluster [DBG] pgmap v74: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 633 op/s 2023-10-01T10:25:09.119 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 10:25:08 smithi173 bash[54497]: ts=2023-10-01T10:25:08.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"smithi173.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)\", cluster_addr=\"172.21.15.78\", device_class=\"ssd\", hostname=\"smithi078\", instance=\"smithi078:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.78\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-10-01T10:25:10.582 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:25:10 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:25:10] "GET /metrics HTTP/1.1" 200 36080 "" "Prometheus/2.33.4" 2023-10-01T10:25:10.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:25:10 smithi173 bash[27612]: cluster 2023-10-01T10:25:09.312006+0000 mgr.y (mgr.25072) 95 : cluster [DBG] pgmap v75: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 943 op/s 2023-10-01T10:25:12.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:25:12 smithi173 bash[27612]: cluster 2023-10-01T10:25:11.315486+0000 mgr.y (mgr.25072) 96 : cluster [DBG] pgmap v76: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 622 op/s 2023-10-01T10:25:14.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:25:14 smithi173 bash[27612]: cluster 2023-10-01T10:25:13.315999+0000 mgr.y (mgr.25072) 97 : cluster [DBG] pgmap v77: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2023-10-01T10:25:16.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:25:16 smithi173 bash[27612]: cluster 2023-10-01T10:25:15.316770+0000 mgr.y (mgr.25072) 98 : cluster [DBG] pgmap v78: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 931 op/s 2023-10-01T10:25:18.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:25:17 smithi173 bash[27612]: cluster 2023-10-01T10:25:17.317197+0000 mgr.y (mgr.25072) 99 : cluster [DBG] pgmap v79: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 617 op/s 2023-10-01T10:25:18.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:25:17 smithi173 bash[27612]: audit 2023-10-01T10:25:17.383261+0000 mon.a (mon.0) 141 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:25:20.582 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:25:20 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:25:20] "GET /metrics HTTP/1.1" 200 36078 "" "Prometheus/2.33.4" 2023-10-01T10:25:20.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:25:20 smithi173 bash[27612]: cluster 2023-10-01T10:25:19.317958+0000 mgr.y (mgr.25072) 100 : cluster [DBG] pgmap v80: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 926 op/s 2023-10-01T10:25:22.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:25:22 smithi173 bash[27612]: cluster 2023-10-01T10:25:21.318401+0000 mgr.y (mgr.25072) 101 : cluster [DBG] pgmap v81: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 617 op/s 2023-10-01T10:25:24.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:25:24 smithi173 bash[27612]: cluster 2023-10-01T10:25:23.318859+0000 mgr.y (mgr.25072) 102 : cluster [DBG] pgmap v82: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 617 op/s 2023-10-01T10:25:26.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:25:26 smithi173 bash[27612]: cluster 2023-10-01T10:25:25.319596+0000 mgr.y (mgr.25072) 103 : cluster [DBG] pgmap v83: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 921 op/s 2023-10-01T10:25:28.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:25:27 smithi173 bash[27612]: cluster 2023-10-01T10:25:27.320098+0000 mgr.y (mgr.25072) 104 : cluster [DBG] pgmap v84: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 612 op/s 2023-10-01T10:25:30.582 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:25:30 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:25:30] "GET /metrics HTTP/1.1" 200 36078 "" "Prometheus/2.33.4" 2023-10-01T10:25:30.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:25:30 smithi173 bash[27612]: cluster 2023-10-01T10:25:29.320746+0000 mgr.y (mgr.25072) 105 : cluster [DBG] pgmap v85: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 914 op/s 2023-10-01T10:25:32.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:25:32 smithi173 bash[27612]: cluster 2023-10-01T10:25:31.321145+0000 mgr.y (mgr.25072) 106 : cluster [DBG] pgmap v86: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 605 op/s 2023-10-01T10:25:33.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:25:33 smithi173 bash[27612]: audit 2023-10-01T10:25:32.387554+0000 mon.a (mon.0) 142 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:25:34.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:25:34 smithi173 bash[27612]: cluster 2023-10-01T10:25:33.323839+0000 mgr.y (mgr.25072) 107 : cluster [DBG] pgmap v87: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 605 op/s 2023-10-01T10:25:36.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:25:36 smithi173 bash[27612]: cluster 2023-10-01T10:25:35.324747+0000 mgr.y (mgr.25072) 108 : cluster [DBG] pgmap v88: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 905 op/s 2023-10-01T10:25:38.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:25:37 smithi173 bash[27612]: cluster 2023-10-01T10:25:37.325219+0000 mgr.y (mgr.25072) 109 : cluster [DBG] pgmap v89: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 601 op/s 2023-10-01T10:25:40.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:25:40 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:25:40] "GET /metrics HTTP/1.1" 200 36073 "" "Prometheus/2.33.4" 2023-10-01T10:25:40.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:25:40 smithi173 bash[27612]: cluster 2023-10-01T10:25:39.325969+0000 mgr.y (mgr.25072) 110 : cluster [DBG] pgmap v90: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 900 op/s 2023-10-01T10:25:42.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:25:42 smithi173 bash[27612]: cluster 2023-10-01T10:25:41.326402+0000 mgr.y (mgr.25072) 111 : cluster [DBG] pgmap v91: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 599 op/s 2023-10-01T10:25:44.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:25:44 smithi173 bash[27612]: cluster 2023-10-01T10:25:43.326823+0000 mgr.y (mgr.25072) 112 : cluster [DBG] pgmap v92: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 599 op/s 2023-10-01T10:25:46.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:25:46 smithi173 bash[27612]: cluster 2023-10-01T10:25:45.327697+0000 mgr.y (mgr.25072) 113 : cluster [DBG] pgmap v93: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 902 op/s 2023-10-01T10:25:48.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:25:47 smithi173 bash[27612]: cluster 2023-10-01T10:25:47.328134+0000 mgr.y (mgr.25072) 114 : cluster [DBG] pgmap v94: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 602 op/s 2023-10-01T10:25:48.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:25:47 smithi173 bash[27612]: audit 2023-10-01T10:25:47.384139+0000 mon.a (mon.0) 143 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:25:50.582 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:25:50 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:25:50] "GET /metrics HTTP/1.1" 200 36078 "" "Prometheus/2.33.4" 2023-10-01T10:25:50.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:25:50 smithi173 bash[27612]: cluster 2023-10-01T10:25:49.328877+0000 mgr.y (mgr.25072) 115 : cluster [DBG] pgmap v95: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 908 op/s 2023-10-01T10:25:51.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:25:51 smithi173 bash[27612]: audit 2023-10-01T10:25:51.115625+0000 mon.a (mon.0) 144 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:25:52.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:25:52 smithi173 bash[27612]: cluster 2023-10-01T10:25:51.329319+0000 mgr.y (mgr.25072) 116 : cluster [DBG] pgmap v96: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 609 op/s 2023-10-01T10:25:52.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:25:52 smithi173 bash[27612]: audit 2023-10-01T10:25:51.647899+0000 mon.a (mon.0) 145 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:25:52.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:25:52 smithi173 bash[27612]: audit 2023-10-01T10:25:51.688342+0000 mon.a (mon.0) 146 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:25:54.646 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -e sha1=b56822191ad0416d0c05ed59720df03ad9c2a44e -- bash -c 'ceph orch upgrade resume' 2023-10-01T10:25:54.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:25:54 smithi173 bash[27612]: cluster 2023-10-01T10:25:53.329746+0000 mgr.y (mgr.25072) 117 : cluster [DBG] pgmap v97: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 609 op/s 2023-10-01T10:25:56.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:25:56 smithi173 bash[27612]: cluster 2023-10-01T10:25:55.332526+0000 mgr.y (mgr.25072) 118 : cluster [DBG] pgmap v98: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 898 op/s 2023-10-01T10:25:58.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:25:57 smithi173 bash[27612]: cluster 2023-10-01T10:25:57.332902+0000 mgr.y (mgr.25072) 119 : cluster [DBG] pgmap v99: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 595 op/s 2023-10-01T10:25:58.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:25:57 smithi173 bash[27612]: audit 2023-10-01T10:25:57.676339+0000 mon.a (mon.0) 147 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:25:58.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:25:57 smithi173 bash[27612]: audit 2023-10-01T10:25:57.691224+0000 mon.a (mon.0) 148 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:25:58.759 INFO:teuthology.orchestra.run.smithi078.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T10:25:59.767 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -e sha1=b56822191ad0416d0c05ed59720df03ad9c2a44e -- bash -c 'echo wait for all mons to upgrade' 2023-10-01T10:26:00.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:25:59 smithi173 bash[27612]: audit 2023-10-01T10:25:58.744984+0000 mgr.y (mgr.25072) 120 : audit [DBG] from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:26:00.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:25:59 smithi173 bash[27612]: cephadm 2023-10-01T10:25:58.745431+0000 mgr.y (mgr.25072) 121 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T10:26:00.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:25:59 smithi173 bash[27612]: audit 2023-10-01T10:25:58.753130+0000 mon.a (mon.0) 149 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:00.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:25:59 smithi173 bash[27612]: cluster 2023-10-01T10:25:59.333749+0000 mgr.y (mgr.25072) 122 : cluster [DBG] pgmap v100: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 891 op/s 2023-10-01T10:26:00.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:25:59 smithi173 bash[27612]: audit 2023-10-01T10:25:59.600092+0000 mon.a (mon.0) 150 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:00.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:25:59 smithi173 bash[27612]: audit 2023-10-01T10:25:59.607175+0000 mon.a (mon.0) 151 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:00.332 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:26:00 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:26:00] "GET /metrics HTTP/1.1" 200 36078 "" "Prometheus/2.33.4" 2023-10-01T10:26:01.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:00 smithi173 bash[27612]: audit 2023-10-01T10:26:00.245372+0000 mon.a (mon.0) 152 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:26:01.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:00 smithi173 bash[27612]: audit 2023-10-01T10:26:00.246702+0000 mon.a (mon.0) 153 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:26:01.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:00 smithi173 bash[27612]: audit 2023-10-01T10:26:00.256003+0000 mon.a (mon.0) 154 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:01.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:00 smithi173 bash[27612]: audit 2023-10-01T10:26:00.339374+0000 mon.a (mon.0) 155 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:26:01.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:00 smithi173 bash[27612]: audit 2023-10-01T10:26:00.349038+0000 mon.a (mon.0) 156 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:26:01.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:00 smithi173 bash[27612]: audit 2023-10-01T10:26:00.355427+0000 mon.a (mon.0) 157 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-01T10:26:01.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:00 smithi173 bash[27612]: audit 2023-10-01T10:26:00.356600+0000 mon.a (mon.0) 158 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-10-01T10:26:01.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:00 smithi173 bash[27612]: cephadm 2023-10-01T10:26:00.357370+0000 mgr.y (mgr.25072) 123 : cephadm [INF] Upgrade: It appears safe to stop mon.b 2023-10-01T10:26:02.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:01 smithi173 bash[27612]: cluster 2023-10-01T10:26:01.334179+0000 mgr.y (mgr.25072) 124 : cluster [DBG] pgmap v101: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 585 op/s 2023-10-01T10:26:03.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:02 smithi173 bash[27612]: audit 2023-10-01T10:26:02.384378+0000 mon.a (mon.0) 159 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:26:03.424 INFO:teuthology.orchestra.run.smithi078.stdout:wait for all mons to upgrade 2023-10-01T10:26:04.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:03 smithi173 bash[27612]: cluster 2023-10-01T10:26:03.334636+0000 mgr.y (mgr.25072) 125 : cluster [DBG] pgmap v102: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 585 op/s 2023-10-01T10:26:04.361 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -e sha1=b56822191ad0416d0c05ed59720df03ad9c2a44e -- bash -c 'while ! ceph mon versions | grep $sha1 | grep '"'"': 3'"'"' ; do sleep 2 ; done' 2023-10-01T10:26:06.564 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:06 smithi173 bash[27612]: cephadm 2023-10-01T10:26:05.246409+0000 mgr.y (mgr.25072) 126 : cephadm [INF] Upgrade: Updating mon.b 2023-10-01T10:26:06.564 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:06 smithi173 bash[27612]: audit 2023-10-01T10:26:05.262780+0000 mon.a (mon.0) 160 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:06.564 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:06 smithi173 bash[27612]: audit 2023-10-01T10:26:05.264561+0000 mon.a (mon.0) 161 : audit [INF] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T10:26:06.564 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:06 smithi173 bash[27612]: audit 2023-10-01T10:26:05.266210+0000 mon.a (mon.0) 162 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T10:26:06.564 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:06 smithi173 bash[27612]: audit 2023-10-01T10:26:05.271384+0000 mon.a (mon.0) 163 : audit [DBG] from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:26:06.564 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:06 smithi173 bash[27612]: cephadm 2023-10-01T10:26:05.273309+0000 mgr.y (mgr.25072) 127 : cephadm [INF] Deploying daemon mon.b on smithi173 2023-10-01T10:26:06.564 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:06 smithi173 bash[27612]: cluster 2023-10-01T10:26:05.335349+0000 mgr.y (mgr.25072) 128 : cluster [DBG] pgmap v103: 89 pgs: 89 active+clean; 2.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2023-10-01T10:26:07.767 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:07 smithi173 bash[27612]: cluster 2023-10-01T10:26:07.335768+0000 mgr.y (mgr.25072) 129 : cluster [DBG] pgmap v104: 89 pgs: 89 active+clean; 2.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 586 op/s 2023-10-01T10:26:07.767 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:07 smithi173 systemd[1]: Stopping Ceph mon.b for d8016480-6041-11ee-8db5-212e2dc638e7... 2023-10-01T10:26:08.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:07 smithi173 bash[27612]: debug 2023-10-01T10:26:07.911+0000 7f0134f48700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-10-01T10:26:08.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:07 smithi173 bash[27612]: debug 2023-10-01T10:26:07.911+0000 7f0134f48700 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2023-10-01T10:26:09.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:08 smithi173 bash[58705]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-mon-b 2023-10-01T10:26:09.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:08 smithi173 bash[58765]: Error response from daemon: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-mon-b 2023-10-01T10:26:09.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:08 smithi173 systemd[1]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7@mon.b.service: Succeeded. 2023-10-01T10:26:09.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:08 smithi173 systemd[1]: Stopped Ceph mon.b for d8016480-6041-11ee-8db5-212e2dc638e7. 2023-10-01T10:26:09.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:09 smithi173 systemd[1]: Started Ceph mon.b for d8016480-6041-11ee-8db5-212e2dc638e7. 2023-10-01T10:26:10.582 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:26:10 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:26:10] "GET /metrics HTTP/1.1" 200 36075 "" "Prometheus/2.33.4" 2023-10-01T10:26:10.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.295+0000 7f84e0026c80 0 set uid:gid to 167:167 (ceph:ceph) 2023-10-01T10:26:10.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.295+0000 7f84e0026c80 0 ceph version 18.2.0-497-gb5682219 (b56822191ad0416d0c05ed59720df03ad9c2a44e) reef (stable), process ceph-mon, pid 6 2023-10-01T10:26:10.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.295+0000 7f84e0026c80 0 pidfile_write: ignore empty --pid-file 2023-10-01T10:26:10.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.299+0000 7f84e0026c80 0 load: jerasure load: lrc 2023-10-01T10:26:10.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: RocksDB version: 7.9.2 2023-10-01T10:26:10.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Git sha 0 2023-10-01T10:26:10.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Compile date 2023-09-29 15:06:10 2023-10-01T10:26:10.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: DB SUMMARY 2023-10-01T10:26:10.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: DB Session ID: NTL49HT04N61T5TCZDBQ 2023-10-01T10:26:10.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: CURRENT file: CURRENT 2023-10-01T10:26:10.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: IDENTITY file: IDENTITY 2023-10-01T10:26:10.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: MANIFEST file: MANIFEST-000005 size: 1853 Bytes 2023-10-01T10:26:10.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000035.sst 2023-10-01T10:26:10.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000033.log size: 1403620 ; 2023-10-01T10:26:10.622 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.error_if_exists: 0 2023-10-01T10:26:10.622 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.create_if_missing: 0 2023-10-01T10:26:10.622 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.paranoid_checks: 1 2023-10-01T10:26:10.622 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.flush_verify_memtable_count: 1 2023-10-01T10:26:10.622 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-10-01T10:26:10.622 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-10-01T10:26:10.622 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.env: 0x556c2a2b4720 2023-10-01T10:26:10.622 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.fs: PosixFileSystem 2023-10-01T10:26:10.622 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.info_log: 0x556c2d2a7480 2023-10-01T10:26:10.623 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.max_file_opening_threads: 16 2023-10-01T10:26:10.623 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.statistics: (nil) 2023-10-01T10:26:10.623 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.use_fsync: 0 2023-10-01T10:26:10.623 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.max_log_file_size: 0 2023-10-01T10:26:10.623 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-10-01T10:26:10.623 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.log_file_time_to_roll: 0 2023-10-01T10:26:10.623 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.keep_log_file_num: 1000 2023-10-01T10:26:10.623 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.recycle_log_file_num: 0 2023-10-01T10:26:10.623 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.allow_fallocate: 1 2023-10-01T10:26:10.623 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.allow_mmap_reads: 0 2023-10-01T10:26:10.623 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.allow_mmap_writes: 0 2023-10-01T10:26:10.623 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.use_direct_reads: 0 2023-10-01T10:26:10.624 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-10-01T10:26:10.624 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.create_missing_column_families: 0 2023-10-01T10:26:10.624 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.db_log_dir: 2023-10-01T10:26:10.624 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.wal_dir: 2023-10-01T10:26:10.624 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.table_cache_numshardbits: 6 2023-10-01T10:26:10.624 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-10-01T10:26:10.624 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-10-01T10:26:10.624 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-10-01T10:26:10.624 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-10-01T10:26:10.624 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-10-01T10:26:10.624 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.advise_random_on_open: 1 2023-10-01T10:26:10.624 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.db_write_buffer_size: 0 2023-10-01T10:26:10.625 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.write_buffer_manager: 0x556c2c534c80 2023-10-01T10:26:10.625 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-10-01T10:26:10.625 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-10-01T10:26:10.625 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.use_adaptive_mutex: 0 2023-10-01T10:26:10.625 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.rate_limiter: (nil) 2023-10-01T10:26:10.625 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-10-01T10:26:10.625 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.wal_recovery_mode: 2 2023-10-01T10:26:10.627 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.enable_thread_tracking: 0 2023-10-01T10:26:10.627 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.enable_pipelined_write: 0 2023-10-01T10:26:10.627 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.unordered_write: 0 2023-10-01T10:26:10.627 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-10-01T10:26:10.627 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-10-01T10:26:10.627 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-10-01T10:26:10.627 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-10-01T10:26:10.627 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.row_cache: None 2023-10-01T10:26:10.627 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.wal_filter: None 2023-10-01T10:26:10.628 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-10-01T10:26:10.628 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.allow_ingest_behind: 0 2023-10-01T10:26:10.628 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.two_write_queues: 0 2023-10-01T10:26:10.628 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.manual_wal_flush: 0 2023-10-01T10:26:10.628 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.wal_compression: 0 2023-10-01T10:26:10.628 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.atomic_flush: 0 2023-10-01T10:26:10.628 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-10-01T10:26:10.628 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.persist_stats_to_disk: 0 2023-10-01T10:26:10.628 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-10-01T10:26:10.628 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.log_readahead_size: 0 2023-10-01T10:26:10.628 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2023-10-01T10:26:10.628 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.best_efforts_recovery: 0 2023-10-01T10:26:10.629 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-10-01T10:26:10.629 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-10-01T10:26:10.629 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.allow_data_in_errors: 0 2023-10-01T10:26:10.629 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.db_host_id: __hostname__ 2023-10-01T10:26:10.629 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.enforce_single_del_contracts: true 2023-10-01T10:26:10.629 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.max_background_jobs: 2 2023-10-01T10:26:10.629 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.max_background_compactions: -1 2023-10-01T10:26:10.629 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.max_subcompactions: 1 2023-10-01T10:26:10.629 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-10-01T10:26:10.629 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-10-01T10:26:10.629 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.delayed_write_rate : 16777216 2023-10-01T10:26:10.630 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.max_total_wal_size: 0 2023-10-01T10:26:10.630 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-10-01T10:26:10.630 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.stats_dump_period_sec: 600 2023-10-01T10:26:10.630 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.stats_persist_period_sec: 600 2023-10-01T10:26:10.630 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-10-01T10:26:10.630 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.max_open_files: -1 2023-10-01T10:26:10.630 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.bytes_per_sync: 0 2023-10-01T10:26:10.630 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-10-01T10:26:10.630 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-10-01T10:26:10.630 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.compaction_readahead_size: 0 2023-10-01T10:26:10.630 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.max_background_flushes: -1 2023-10-01T10:26:10.631 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Compression algorithms supported: 2023-10-01T10:26:10.631 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-10-01T10:26:10.631 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: kZSTD supported: 0 2023-10-01T10:26:10.631 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: kXpressCompression supported: 0 2023-10-01T10:26:10.631 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: kLZ4HCCompression supported: 1 2023-10-01T10:26:10.631 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: kZlibCompression supported: 1 2023-10-01T10:26:10.631 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: kSnappyCompression supported: 1 2023-10-01T10:26:10.631 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: kLZ4Compression supported: 1 2023-10-01T10:26:10.631 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: kBZip2Compression supported: 0 2023-10-01T10:26:10.631 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-10-01T10:26:10.631 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: DMutex implementation: pthread_mutex_t 2023-10-01T10:26:10.632 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000005 2023-10-01T10:26:10.632 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-10-01T10:26:10.632 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-10-01T10:26:10.632 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.merge_operator: 2023-10-01T10:26:10.632 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.compaction_filter: None 2023-10-01T10:26:10.632 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.compaction_filter_factory: None 2023-10-01T10:26:10.632 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.sst_partitioner_factory: None 2023-10-01T10:26:10.632 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-10-01T10:26:10.632 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.table_factory: BlockBasedTable 2023-10-01T10:26:10.632 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x556c2d2a7580) 2023-10-01T10:26:10.633 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: cache_index_and_filter_blocks: 1 2023-10-01T10:26:10.633 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: cache_index_and_filter_blocks_with_high_priority: 0 2023-10-01T10:26:10.633 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: pin_l0_filter_and_index_blocks_in_cache: 0 2023-10-01T10:26:10.633 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: pin_top_level_index_and_filter: 1 2023-10-01T10:26:10.633 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: index_type: 0 2023-10-01T10:26:10.633 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: data_block_index_type: 0 2023-10-01T10:26:10.633 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: index_shortening: 1 2023-10-01T10:26:10.633 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: data_block_hash_table_util_ratio: 0.750000 2023-10-01T10:26:10.633 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: checksum: 4 2023-10-01T10:26:10.633 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: no_block_cache: 0 2023-10-01T10:26:10.633 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: block_cache: 0x556c2c5b71f0 2023-10-01T10:26:10.633 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: block_cache_name: BinnedLRUCache 2023-10-01T10:26:10.633 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: block_cache_options: 2023-10-01T10:26:10.633 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: capacity : 536870912 2023-10-01T10:26:10.634 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: num_shard_bits : 4 2023-10-01T10:26:10.634 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: strict_capacity_limit : 0 2023-10-01T10:26:10.634 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: high_pri_pool_ratio: 0.000 2023-10-01T10:26:10.634 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: block_cache_compressed: (nil) 2023-10-01T10:26:10.634 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: persistent_cache: (nil) 2023-10-01T10:26:10.634 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: block_size: 4096 2023-10-01T10:26:10.634 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: block_size_deviation: 10 2023-10-01T10:26:10.634 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: block_restart_interval: 16 2023-10-01T10:26:10.634 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: index_block_restart_interval: 1 2023-10-01T10:26:10.634 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: metadata_block_size: 4096 2023-10-01T10:26:10.634 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: partition_filters: 0 2023-10-01T10:26:10.635 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: use_delta_encoding: 1 2023-10-01T10:26:10.635 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: filter_policy: bloomfilter 2023-10-01T10:26:10.635 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: whole_key_filtering: 1 2023-10-01T10:26:10.635 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: verify_compression: 0 2023-10-01T10:26:10.635 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: read_amp_bytes_per_bit: 0 2023-10-01T10:26:10.635 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: format_version: 5 2023-10-01T10:26:10.635 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: enable_index_compression: 1 2023-10-01T10:26:10.635 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: block_align: 0 2023-10-01T10:26:10.635 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: max_auto_readahead_size: 262144 2023-10-01T10:26:10.635 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: prepopulate_block_cache: 0 2023-10-01T10:26:10.636 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: initial_auto_readahead_size: 8192 2023-10-01T10:26:10.636 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: num_file_reads_for_auto_readahead: 2 2023-10-01T10:26:10.636 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.write_buffer_size: 33554432 2023-10-01T10:26:10.636 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.max_write_buffer_number: 2 2023-10-01T10:26:10.636 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.compression: NoCompression 2023-10-01T10:26:10.636 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.bottommost_compression: Disabled 2023-10-01T10:26:10.636 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.prefix_extractor: nullptr 2023-10-01T10:26:10.636 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-10-01T10:26:10.636 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.num_levels: 7 2023-10-01T10:26:10.636 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-10-01T10:26:10.636 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-10-01T10:26:10.636 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-10-01T10:26:10.637 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-10-01T10:26:10.637 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-10-01T10:26:10.637 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-10-01T10:26:10.637 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-10-01T10:26:10.637 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-10-01T10:26:10.637 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-10-01T10:26:10.637 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-10-01T10:26:10.637 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-10-01T10:26:10.637 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-10-01T10:26:10.637 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.compression_opts.window_bits: -14 2023-10-01T10:26:10.637 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.compression_opts.level: 32767 2023-10-01T10:26:10.638 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.303+0000 7f84e0026c80 4 rocksdb: Options.compression_opts.strategy: 0 2023-10-01T10:26:10.638 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-10-01T10:26:10.638 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-10-01T10:26:10.638 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-10-01T10:26:10.638 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.compression_opts.parallel_threads: 1 2023-10-01T10:26:10.638 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.compression_opts.enabled: false 2023-10-01T10:26:10.638 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-10-01T10:26:10.638 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-10-01T10:26:10.638 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-10-01T10:26:10.638 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-10-01T10:26:10.638 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.target_file_size_base: 67108864 2023-10-01T10:26:10.639 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.target_file_size_multiplier: 1 2023-10-01T10:26:10.639 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-10-01T10:26:10.639 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-10-01T10:26:10.639 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-10-01T10:26:10.639 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-10-01T10:26:10.639 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-10-01T10:26:10.639 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-10-01T10:26:10.639 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-10-01T10:26:10.639 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-10-01T10:26:10.639 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-10-01T10:26:10.639 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-10-01T10:26:10.640 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-10-01T10:26:10.640 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-10-01T10:26:10.640 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-10-01T10:26:10.640 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.arena_block_size: 1048576 2023-10-01T10:26:10.640 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-10-01T10:26:10.640 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-10-01T10:26:10.640 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.disable_auto_compactions: 0 2023-10-01T10:26:10.640 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-10-01T10:26:10.640 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-10-01T10:26:10.640 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-10-01T10:26:10.640 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-10-01T10:26:10.640 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-10-01T10:26:10.641 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-10-01T10:26:10.641 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-10-01T10:26:10.641 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-10-01T10:26:10.641 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-10-01T10:26:10.641 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-10-01T10:26:10.641 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.table_properties_collectors: 2023-10-01T10:26:10.641 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.inplace_update_support: 0 2023-10-01T10:26:10.641 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-10-01T10:26:10.641 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-10-01T10:26:10.641 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-10-01T10:26:10.642 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.memtable_huge_page_size: 0 2023-10-01T10:26:10.642 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.bloom_locality: 0 2023-10-01T10:26:10.642 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.max_successive_merges: 0 2023-10-01T10:26:10.642 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-10-01T10:26:10.642 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.paranoid_file_checks: 0 2023-10-01T10:26:10.642 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.force_consistency_checks: 1 2023-10-01T10:26:10.642 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.report_bg_io_stats: 0 2023-10-01T10:26:10.642 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.ttl: 2592000 2023-10-01T10:26:10.642 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-10-01T10:26:10.642 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.preclude_last_level_data_seconds: 0 2023-10-01T10:26:10.643 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.preserve_internal_time_seconds: 0 2023-10-01T10:26:10.643 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.enable_blob_files: false 2023-10-01T10:26:10.643 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.min_blob_size: 0 2023-10-01T10:26:10.643 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.blob_file_size: 268435456 2023-10-01T10:26:10.643 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.blob_compression_type: NoCompression 2023-10-01T10:26:10.643 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.enable_blob_garbage_collection: false 2023-10-01T10:26:10.643 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-10-01T10:26:10.643 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-10-01T10:26:10.643 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.blob_compaction_readahead_size: 0 2023-10-01T10:26:10.643 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.blob_file_starting_level: 0 2023-10-01T10:26:10.643 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-10-01T10:26:10.643 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84d5844700 3 rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 35.sst 2023-10-01T10:26:10.644 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 37, last_sequence is 20417, log_number is 33,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-10-01T10:26:10.644 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 33 2023-10-01T10:26:10.644 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 045f312c-4a2b-4bde-a61a-92f7d31e551c 2023-10-01T10:26:10.644 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696155970311755, "job": 1, "event": "recovery_started", "wal_files": [33]} 2023-10-01T10:26:10.644 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.307+0000 7f84e0026c80 4 rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #33 mode 2 2023-10-01T10:26:10.644 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.315+0000 7f84e0026c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696155970317390, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 38, "file_size": 923887, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 20411, "largest_seqno": 20679, "table_properties": {"data_size": 921458, "index_size": 1161, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 261, "raw_key_size": 2078, "raw_average_key_size": 22, "raw_value_size": 918732, "raw_average_value_size": 9878, "num_data_blocks": 50, "num_entries": 93, "num_filter_entries": 93, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1696155970, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "045f312c-4a2b-4bde-a61a-92f7d31e551c", "db_session_id": "NTL49HT04N61T5TCZDBQ", "orig_file_number": 38, "seqno_to_time_mapping": "N/A"}} 2023-10-01T10:26:10.644 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.315+0000 7f84e0026c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1696155970317543, "job": 1, "event": "recovery_finished"} 2023-10-01T10:26:10.644 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.315+0000 7f84e0026c80 4 rocksdb: [db/version_set.cc:5047] Creating manifest 40 2023-10-01T10:26:10.644 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.315+0000 7f84e0026c80 4 rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000033.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-10-01T10:26:10.644 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.315+0000 7f84e0026c80 4 rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x556c2c658000 2023-10-01T10:26:10.644 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.315+0000 7f84e0026c80 4 rocksdb: DB pointer 0x556c2c646000 2023-10-01T10:26:10.645 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.319+0000 7f84ce836700 4 rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-10-01T10:26:10.645 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.319+0000 7f84ce836700 4 rocksdb: [db/db_impl/db_impl.cc:1111] 2023-10-01T10:26:10.645 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: ** DB Stats ** 2023-10-01T10:26:10.645 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-01T10:26:10.645 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-10-01T10:26:10.645 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-01T10:26:10.645 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-01T10:26:10.645 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-10-01T10:26:10.645 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-10-01T10:26:10.645 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-10-01T10:26:10.645 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: ** Compaction Stats [default] ** 2023-10-01T10:26:10.646 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-01T10:26:10.646 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-10-01T10:26:10.646 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: L0 1/0 902.23 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 277.4 0.00 0.00 1 0.003 0 0 0.0 0.0 2023-10-01T10:26:10.646 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: L6 1/0 26.78 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-10-01T10:26:10.646 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: Sum 2/0 27.66 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 277.4 0.00 0.00 1 0.003 0 0 0.0 0.0 2023-10-01T10:26:10.646 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 277.4 0.00 0.00 1 0.003 0 0 0.0 0.0 2023-10-01T10:26:10.646 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: ** Compaction Stats [default] ** 2023-10-01T10:26:10.646 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-10-01T10:26:10.646 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-10-01T10:26:10.646 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 277.4 0.00 0.00 1 0.003 0 0 0.0 0.0 2023-10-01T10:26:10.646 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-10-01T10:26:10.647 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: Uptime(secs): 0.0 total, 0.0 interval 2023-10-01T10:26:10.647 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: Flush(GB): cumulative 0.001, interval 0.001 2023-10-01T10:26:10.647 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: AddFile(GB): cumulative 0.000, interval 0.000 2023-10-01T10:26:10.647 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: AddFile(Total Files): cumulative 0, interval 0 2023-10-01T10:26:10.647 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: AddFile(L0 Files): cumulative 0, interval 0 2023-10-01T10:26:10.647 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: AddFile(Keys): cumulative 0, interval 0 2023-10-01T10:26:10.647 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: Cumulative compaction: 0.00 GB write, 67.70 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-01T10:26:10.647 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: Interval compaction: 0.00 GB write, 67.70 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-10-01T10:26:10.647 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-10-01T10:26:10.647 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: Block cache BinnedLRUCache@0x556c2c5b71f0#6 capacity: 512.00 MB usage: 1.64 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.7e-05 secs_since: 0 2023-10-01T10:26:10.647 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: Block cache entry stats(count,size,portion): FilterBlock(1,0.31 KB,5.96046e-05%) IndexBlock(1,1.33 KB,0.00025332%) Misc(1,0.00 KB,0%) 2023-10-01T10:26:10.648 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: ** File Read Latency Histogram By Level [default] ** 2023-10-01T10:26:10.648 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.319+0000 7f84e0026c80 0 starting mon.b rank 1 at public addrs [v2:172.21.15.173:3300/0,v1:172.21.15.173:6789/0] at bind addrs [v2:172.21.15.173:3300/0,v1:172.21.15.173:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid d8016480-6041-11ee-8db5-212e2dc638e7 2023-10-01T10:26:10.648 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.319+0000 7f84e0026c80 1 mon.b@-1(???) e3 preinit fsid d8016480-6041-11ee-8db5-212e2dc638e7 2023-10-01T10:26:10.648 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.319+0000 7f84e0026c80 0 mon.b@-1(???).mds e7 new map 2023-10-01T10:26:10.648 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.319+0000 7f84e0026c80 0 mon.b@-1(???).mds e7 print_map 2023-10-01T10:26:10.648 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: e7 2023-10-01T10:26:10.648 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: enable_multiple, ever_enabled_multiple: 1,1 2023-10-01T10:26:10.648 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-01T10:26:10.648 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: legacy client fscid: 1 2023-10-01T10:26:10.648 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: 2023-10-01T10:26:10.649 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: Filesystem 'foo' (1) 2023-10-01T10:26:10.649 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: fs_name foo 2023-10-01T10:26:10.649 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: epoch 5 2023-10-01T10:26:10.649 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: flags 12 joinable allow_snaps allow_multimds_snaps 2023-10-01T10:26:10.649 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: created 2023-10-01T10:15:39.285114+0000 2023-10-01T10:26:10.649 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: modified 2023-10-01T10:15:45.790305+0000 2023-10-01T10:26:10.649 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: tableserver 0 2023-10-01T10:26:10.649 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: root 0 2023-10-01T10:26:10.649 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: session_timeout 60 2023-10-01T10:26:10.649 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: session_autoclose 300 2023-10-01T10:26:10.649 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: max_file_size 1099511627776 2023-10-01T10:26:10.650 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: required_client_features {} 2023-10-01T10:26:10.650 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: last_failure 0 2023-10-01T10:26:10.650 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: last_failure_osd_epoch 0 2023-10-01T10:26:10.650 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-10-01T10:26:10.650 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: max_mds 1 2023-10-01T10:26:10.650 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: in 0 2023-10-01T10:26:10.650 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: up {0=14622} 2023-10-01T10:26:10.650 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: failed 2023-10-01T10:26:10.650 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: damaged 2023-10-01T10:26:10.650 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: stopped 2023-10-01T10:26:10.650 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: data_pools [4] 2023-10-01T10:26:10.651 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: metadata_pool 3 2023-10-01T10:26:10.651 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: inline_data disabled 2023-10-01T10:26:10.651 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: balancer 2023-10-01T10:26:10.651 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: bal_rank_mask -1 2023-10-01T10:26:10.651 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: standby_count_wanted 1 2023-10-01T10:26:10.651 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: [mds.foo.smithi078.vjqrth{0:14622} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.78:6834/2348005403,v1:172.21.15.78:6835/2348005403] compat {c=[1],r=[1],i=[7ff]}] 2023-10-01T10:26:10.651 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: 2023-10-01T10:26:10.651 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: 2023-10-01T10:26:10.651 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: Standby daemons: 2023-10-01T10:26:10.651 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: 2023-10-01T10:26:10.651 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: [mds.foo.smithi173.vrqqoc{-1:24478} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.173:6832/2937343008,v1:172.21.15.173:6833/2937343008] compat {c=[1],r=[1],i=[7ff]}] 2023-10-01T10:26:10.651 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.319+0000 7f84e0026c80 0 mon.b@-1(???).osd e77 crush map has features 3314933000854323200, adjusting msgr requires 2023-10-01T10:26:10.652 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.319+0000 7f84e0026c80 0 mon.b@-1(???).osd e77 crush map has features 432629239337189376, adjusting msgr requires 2023-10-01T10:26:10.652 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.319+0000 7f84e0026c80 0 mon.b@-1(???).osd e77 crush map has features 432629239337189376, adjusting msgr requires 2023-10-01T10:26:10.652 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:10 smithi173 bash[58816]: debug 2023-10-01T10:26:10.319+0000 7f84e0026c80 0 mon.b@-1(???).osd e77 crush map has features 432629239337189376, adjusting msgr requires 2023-10-01T10:26:11.449 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 18.2.0-497-gb5682219 (b56822191ad0416d0c05ed59720df03ad9c2a44e) reef (stable)": 3 2023-10-01T10:26:12.008 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cluster 2023-10-01T10:04:59.345901+0000 mon.a (mon.0) 0 : 1 mkfs d8016480-6041-11ee-8db5-212e2dc638e7 2023-10-01T10:26:12.008 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cephadm 2023-10-01T10:20:23.346660+0000 mgr.x (mgr.24697) 51 : cephadm 1 Reconfiguring node-exporter.b (dependencies changed)... 2023-10-01T10:26:12.008 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cephadm 2023-10-01T10:20:23.347123+0000 mgr.x (mgr.24697) 52 : cephadm 1 Reconfiguring daemon node-exporter.b on smithi173 2023-10-01T10:26:12.008 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cephadm 2023-10-01T10:20:25.272648+0000 mgr.x (mgr.24697) 54 : cephadm 1 Reconfiguring grafana.a (dependencies changed)... 2023-10-01T10:26:12.008 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cephadm 2023-10-01T10:20:25.296121+0000 mgr.x (mgr.24697) 55 : cephadm 1 Reconfiguring daemon grafana.a on smithi173 2023-10-01T10:26:12.009 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cephadm 2023-10-01T10:21:02.510561+0000 mgr.x (mgr.24697) 77 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2023-10-01T10:26:12.009 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cephadm 2023-10-01T10:21:02.511607+0000 mgr.x (mgr.24697) 78 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2023-10-01T10:26:12.009 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cephadm 2023-10-01T10:21:07.797945+0000 mgr.x (mgr.24697) 82 : cephadm 1 Upgrade: Updating mgr.y 2023-10-01T10:26:12.009 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cephadm 2023-10-01T10:21:07.819165+0000 mgr.x (mgr.24697) 83 : cephadm 1 Deploying daemon mgr.y on smithi078 2023-10-01T10:26:12.009 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cephadm 2023-10-01T10:21:32.255585+0000 mgr.y (mgr.25115) 1 : cephadm 1 [01/Oct/2023:10:21:32] ENGINE Bus STARTING 2023-10-01T10:26:12.009 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cephadm 2023-10-01T10:21:32.429884+0000 mgr.y (mgr.25115) 2 : cephadm 1 [01/Oct/2023:10:21:32] ENGINE Serving on https://172.21.15.78:7150 2023-10-01T10:26:12.009 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cephadm 2023-10-01T10:21:32.534004+0000 mgr.y (mgr.25115) 4 : cephadm 1 [01/Oct/2023:10:21:32] ENGINE Serving on http://172.21.15.78:8765 2023-10-01T10:26:12.009 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cephadm 2023-10-01T10:21:32.534138+0000 mgr.y (mgr.25115) 5 : cephadm 1 [01/Oct/2023:10:21:32] ENGINE Bus STARTED 2023-10-01T10:26:12.009 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cephadm 2023-10-01T10:21:50.936847+0000 mgr.y (mgr.25115) 15 : cephadm 1 Updating smithi078:/etc/ceph/ceph.conf 2023-10-01T10:26:12.009 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cephadm 2023-10-01T10:21:50.937090+0000 mgr.y (mgr.25115) 16 : cephadm 1 Updating smithi173:/etc/ceph/ceph.conf 2023-10-01T10:26:12.009 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cephadm 2023-10-01T10:21:51.153052+0000 mgr.y (mgr.25115) 17 : cephadm 1 Updating smithi173:/var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T10:26:12.010 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cephadm 2023-10-01T10:21:51.155158+0000 mgr.y (mgr.25115) 18 : cephadm 1 Updating smithi078:/var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T10:26:12.010 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cephadm 2023-10-01T10:21:51.384360+0000 mgr.y (mgr.25115) 19 : cephadm 1 Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2023-10-01T10:26:12.010 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cephadm 2023-10-01T10:21:51.384616+0000 mgr.y (mgr.25115) 20 : cephadm 1 Updating smithi173:/etc/ceph/ceph.client.admin.keyring 2023-10-01T10:26:12.010 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cephadm 2023-10-01T10:21:51.599568+0000 mgr.y (mgr.25115) 22 : cephadm 1 Updating smithi173:/var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-01T10:26:12.010 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cephadm 2023-10-01T10:21:51.599867+0000 mgr.y (mgr.25115) 23 : cephadm 1 Updating smithi078:/var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-01T10:26:12.010 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cephadm 2023-10-01T10:21:51.960654+0000 mgr.y (mgr.25115) 24 : cephadm 1 Reconfiguring prometheus.a (dependencies changed)... 2023-10-01T10:26:12.010 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cephadm 2023-10-01T10:21:52.234562+0000 mgr.y (mgr.25115) 25 : cephadm 1 Reconfiguring daemon prometheus.a on smithi173 2023-10-01T10:26:12.010 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cephadm 2023-10-01T10:22:02.286629+0000 mgr.y (mgr.25115) 32 : cephadm 1 Upgrade: Updating mgr.x 2023-10-01T10:26:12.010 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cephadm 2023-10-01T10:22:02.299272+0000 mgr.y (mgr.25115) 33 : cephadm 1 Deploying daemon mgr.x on smithi173 2023-10-01T10:26:12.010 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cephadm 2023-10-01T10:22:22.960834+0000 mgr.y (mgr.25115) 44 : cephadm 1 Upgrade: Setting container_image for all mgr 2023-10-01T10:26:12.010 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cephadm 2023-10-01T10:22:22.986335+0000 mgr.y (mgr.25115) 45 : cephadm 1 Upgrade: It appears safe to stop mon.a 2023-10-01T10:26:12.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cephadm 2023-10-01T10:22:25.305462+0000 mgr.y (mgr.25115) 47 : cephadm 1 Upgrade: Updating mon.a 2023-10-01T10:26:12.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cephadm 2023-10-01T10:22:25.321118+0000 mgr.y (mgr.25115) 48 : cephadm 1 Deploying daemon mon.a on smithi078 2023-10-01T10:26:12.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cephadm 2023-10-01T10:22:48.222444+0000 mgr.y (mgr.25072) 1 : cephadm 1 [01/Oct/2023:10:22:48] ENGINE Bus STARTING 2023-10-01T10:26:12.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cephadm 2023-10-01T10:22:48.344219+0000 mgr.y (mgr.25072) 4 : cephadm 1 [01/Oct/2023:10:22:48] ENGINE Serving on https://172.21.15.78:7150 2023-10-01T10:26:12.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cephadm 2023-10-01T10:22:48.446003+0000 mgr.y (mgr.25072) 5 : cephadm 1 [01/Oct/2023:10:22:48] ENGINE Serving on http://172.21.15.78:8765 2023-10-01T10:26:12.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cephadm 2023-10-01T10:22:48.446382+0000 mgr.y (mgr.25072) 6 : cephadm 1 [01/Oct/2023:10:22:48] ENGINE Bus STARTED 2023-10-01T10:26:12.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cephadm 2023-10-01T10:22:53.975096+0000 mgr.y (mgr.25072) 11 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T10:26:12.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cephadm 2023-10-01T10:23:07.886128+0000 mgr.y (mgr.25072) 19 : cephadm 1 Updating smithi078:/etc/ceph/ceph.conf 2023-10-01T10:26:12.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cephadm 2023-10-01T10:23:07.886362+0000 mgr.y (mgr.25072) 20 : cephadm 1 Updating smithi173:/etc/ceph/ceph.conf 2023-10-01T10:26:12.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cluster 2023-10-01T10:04:59.345901+0000 mon.a (mon.0) 0 : 1 mkfs d8016480-6041-11ee-8db5-212e2dc638e7 2023-10-01T10:26:12.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cephadm 2023-10-01T10:20:23.346660+0000 mgr.x (mgr.24697) 51 : cephadm 1 Reconfiguring node-exporter.b (dependencies changed)... 2023-10-01T10:26:12.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cephadm 2023-10-01T10:20:23.347123+0000 mgr.x (mgr.24697) 52 : cephadm 1 Reconfiguring daemon node-exporter.b on smithi173 2023-10-01T10:26:12.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cephadm 2023-10-01T10:20:25.272648+0000 mgr.x (mgr.24697) 54 : cephadm 1 Reconfiguring grafana.a (dependencies changed)... 2023-10-01T10:26:12.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cephadm 2023-10-01T10:20:25.296121+0000 mgr.x (mgr.24697) 55 : cephadm 1 Reconfiguring daemon grafana.a on smithi173 2023-10-01T10:26:12.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cephadm 2023-10-01T10:21:02.510561+0000 mgr.x (mgr.24697) 77 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2023-10-01T10:26:12.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cephadm 2023-10-01T10:21:02.511607+0000 mgr.x (mgr.24697) 78 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2023-10-01T10:26:12.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cephadm 2023-10-01T10:21:07.797945+0000 mgr.x (mgr.24697) 82 : cephadm 1 Upgrade: Updating mgr.y 2023-10-01T10:26:12.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cephadm 2023-10-01T10:21:07.819165+0000 mgr.x (mgr.24697) 83 : cephadm 1 Deploying daemon mgr.y on smithi078 2023-10-01T10:26:12.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cephadm 2023-10-01T10:21:32.255585+0000 mgr.y (mgr.25115) 1 : cephadm 1 [01/Oct/2023:10:21:32] ENGINE Bus STARTING 2023-10-01T10:26:12.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cephadm 2023-10-01T10:21:32.429884+0000 mgr.y (mgr.25115) 2 : cephadm 1 [01/Oct/2023:10:21:32] ENGINE Serving on https://172.21.15.78:7150 2023-10-01T10:26:12.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cephadm 2023-10-01T10:21:32.534004+0000 mgr.y (mgr.25115) 4 : cephadm 1 [01/Oct/2023:10:21:32] ENGINE Serving on http://172.21.15.78:8765 2023-10-01T10:26:12.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cephadm 2023-10-01T10:21:32.534138+0000 mgr.y (mgr.25115) 5 : cephadm 1 [01/Oct/2023:10:21:32] ENGINE Bus STARTED 2023-10-01T10:26:12.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cephadm 2023-10-01T10:21:50.936847+0000 mgr.y (mgr.25115) 15 : cephadm 1 Updating smithi078:/etc/ceph/ceph.conf 2023-10-01T10:26:12.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cephadm 2023-10-01T10:23:08.105307+0000 mgr.y (mgr.25072) 21 : cephadm 1 Updating smithi173:/var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T10:26:12.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cephadm 2023-10-01T10:23:08.105710+0000 mgr.y (mgr.25072) 22 : cephadm 1 Updating smithi078:/var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T10:26:12.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cephadm 2023-10-01T10:23:08.324481+0000 mgr.y (mgr.25072) 23 : cephadm 1 Updating smithi173:/etc/ceph/ceph.client.admin.keyring 2023-10-01T10:26:12.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cephadm 2023-10-01T10:23:08.324802+0000 mgr.y (mgr.25072) 24 : cephadm 1 Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2023-10-01T10:26:12.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cephadm 2023-10-01T10:23:08.540586+0000 mgr.y (mgr.25072) 25 : cephadm 1 Updating smithi078:/var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-01T10:26:12.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cephadm 2023-10-01T10:23:08.542580+0000 mgr.y (mgr.25072) 26 : cephadm 1 Updating smithi173:/var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-01T10:26:12.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cephadm 2023-10-01T10:24:06.570782+0000 mgr.y (mgr.25072) 57 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T10:26:12.336 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cephadm 2023-10-01T10:24:13.756352+0000 mgr.y (mgr.25072) 62 : cephadm 1 Upgrade: It appears safe to stop mon.c 2023-10-01T10:26:12.336 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cephadm 2023-10-01T10:24:18.986358+0000 mgr.y (mgr.25072) 65 : cephadm 1 Upgrade: Updating mon.c 2023-10-01T10:26:12.336 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cephadm 2023-10-01T10:24:19.007779+0000 mgr.y (mgr.25072) 66 : cephadm 1 Deploying daemon mon.c on smithi078 2023-10-01T10:26:12.336 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: audit 2023-10-01T10:24:24.840083+0000 mon.a (mon.0) 99 : audit 0 from='client.? 172.21.15.78:0/3239132624' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:26:12.336 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: audit 2023-10-01T10:24:30.414381+0000 mon.a (mon.0) 110 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.336 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: audit 2023-10-01T10:24:30.422206+0000 mon.a (mon.0) 111 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.336 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: audit 2023-10-01T10:24:30.426220+0000 mon.a (mon.0) 112 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:26:12.336 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: audit 2023-10-01T10:24:31.203512+0000 mon.a (mon.0) 113 : audit 0 from='client.? 172.21.15.78:0/744453728' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:26:12.336 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cluster 2023-10-01T10:24:31.423001+0000 mon.a (mon.0) 121 : cluster 1 Cluster is now healthy 2023-10-01T10:26:12.336 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cluster 2023-10-01T10:24:31.432792+0000 mon.a (mon.0) 122 : cluster 1 overall HEALTH_OK 2023-10-01T10:26:12.336 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: audit 2023-10-01T10:24:32.374431+0000 mon.a (mon.0) 123 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:26:12.336 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cluster 2023-10-01T10:24:33.300742+0000 mgr.y (mgr.25072) 74 : cluster 0 pgmap v57: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 12 MiB/s wr, 673 op/s 2023-10-01T10:26:12.337 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: audit 2023-10-01T10:24:33.937824+0000 mon.a (mon.0) 124 : audit 0 from='client.? 172.21.15.78:0/448982698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:26:12.337 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cluster 2023-10-01T10:24:35.301499+0000 mgr.y (mgr.25072) 75 : cluster 0 pgmap v58: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 11 MiB/s wr, 962 op/s 2023-10-01T10:26:12.337 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cluster 2023-10-01T10:24:37.301977+0000 mgr.y (mgr.25072) 76 : cluster 0 pgmap v59: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 669 op/s 2023-10-01T10:26:12.337 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: audit 2023-10-01T10:24:38.048101+0000 mon.a (mon.0) 125 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.337 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: audit 2023-10-01T10:24:38.061150+0000 mon.a (mon.0) 126 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.337 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: audit 2023-10-01T10:24:39.064412+0000 mon.a (mon.0) 127 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.337 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: audit 2023-10-01T10:24:39.071433+0000 mon.a (mon.0) 128 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.337 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cluster 2023-10-01T10:24:39.302744+0000 mgr.y (mgr.25072) 77 : cluster 0 pgmap v60: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 976 op/s 2023-10-01T10:26:12.337 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: audit 2023-10-01T10:24:40.439142+0000 mgr.y (mgr.25072) 78 : audit 0 from='client.34215 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:26:12.337 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cluster 2023-10-01T10:24:41.303179+0000 mgr.y (mgr.25072) 79 : cluster 0 pgmap v61: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 635 op/s 2023-10-01T10:26:12.337 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cluster 2023-10-01T10:24:43.303687+0000 mgr.y (mgr.25072) 80 : cluster 0 pgmap v62: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 635 op/s 2023-10-01T10:26:12.337 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cluster 2023-10-01T10:24:45.305489+0000 mgr.y (mgr.25072) 81 : cluster 0 pgmap v63: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 955 op/s 2023-10-01T10:26:12.337 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: audit 2023-10-01T10:24:46.280316+0000 mgr.y (mgr.25072) 82 : audit 0 from='client.34218 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:26:12.338 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cephadm 2023-10-01T10:24:46.281006+0000 mgr.y (mgr.25072) 83 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T10:26:12.338 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: audit 2023-10-01T10:24:46.291061+0000 mon.a (mon.0) 129 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.338 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cluster 2023-10-01T10:24:47.305900+0000 mgr.y (mgr.25072) 84 : cluster 0 pgmap v64: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 626 op/s 2023-10-01T10:26:12.338 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: audit 2023-10-01T10:24:47.382805+0000 mon.a (mon.0) 130 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:26:12.338 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cluster 2023-10-01T10:24:49.306445+0000 mgr.y (mgr.25072) 85 : cluster 0 pgmap v65: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 942 op/s 2023-10-01T10:26:12.338 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: audit 2023-10-01T10:24:50.908342+0000 mon.a (mon.0) 131 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.338 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: audit 2023-10-01T10:24:50.918285+0000 mon.a (mon.0) 132 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.338 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: audit 2023-10-01T10:24:50.919599+0000 mon.a (mon.0) 133 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:26:12.338 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: audit 2023-10-01T10:24:50.923731+0000 mon.a (mon.0) 134 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:26:12.338 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: audit 2023-10-01T10:24:50.932637+0000 mon.a (mon.0) 135 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.338 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: audit 2023-10-01T10:24:51.019441+0000 mon.a (mon.0) 136 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:26:12.339 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: audit 2023-10-01T10:24:51.021592+0000 mon.a (mon.0) 137 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:26:12.339 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: audit 2023-10-01T10:24:51.029692+0000 mon.a (mon.0) 138 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:26:12.339 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: audit 2023-10-01T10:24:51.036687+0000 mon.a (mon.0) 139 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.339 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cluster 2023-10-01T10:24:51.306854+0000 mgr.y (mgr.25072) 86 : cluster 0 pgmap v66: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 636 op/s 2023-10-01T10:26:12.339 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cluster 2023-10-01T10:24:53.307323+0000 mgr.y (mgr.25072) 87 : cluster 0 pgmap v67: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 636 op/s 2023-10-01T10:26:12.339 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cluster 2023-10-01T10:24:55.308104+0000 mgr.y (mgr.25072) 88 : cluster 0 pgmap v68: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 950 op/s 2023-10-01T10:26:12.339 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cluster 2023-10-01T10:24:57.308530+0000 mgr.y (mgr.25072) 89 : cluster 0 pgmap v69: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 630 op/s 2023-10-01T10:26:12.339 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cluster 2023-10-01T10:24:59.309238+0000 mgr.y (mgr.25072) 90 : cluster 0 pgmap v70: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 950 op/s 2023-10-01T10:26:12.339 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cluster 2023-10-01T10:25:01.309690+0000 mgr.y (mgr.25072) 91 : cluster 0 pgmap v71: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 633 op/s 2023-10-01T10:26:12.339 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: audit 2023-10-01T10:25:02.383036+0000 mon.a (mon.0) 140 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:26:12.339 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cluster 2023-10-01T10:25:03.310268+0000 mgr.y (mgr.25072) 92 : cluster 0 pgmap v72: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 633 op/s 2023-10-01T10:26:12.339 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cluster 2023-10-01T10:25:05.310995+0000 mgr.y (mgr.25072) 93 : cluster 0 pgmap v73: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 947 op/s 2023-10-01T10:26:12.340 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cluster 2023-10-01T10:25:07.311409+0000 mgr.y (mgr.25072) 94 : cluster 0 pgmap v74: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 633 op/s 2023-10-01T10:26:12.340 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cluster 2023-10-01T10:25:09.312006+0000 mgr.y (mgr.25072) 95 : cluster 0 pgmap v75: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 943 op/s 2023-10-01T10:26:12.340 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cluster 2023-10-01T10:25:11.315486+0000 mgr.y (mgr.25072) 96 : cluster 0 pgmap v76: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 622 op/s 2023-10-01T10:26:12.340 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cluster 2023-10-01T10:25:13.315999+0000 mgr.y (mgr.25072) 97 : cluster 0 pgmap v77: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2023-10-01T10:26:12.340 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cluster 2023-10-01T10:25:15.316770+0000 mgr.y (mgr.25072) 98 : cluster 0 pgmap v78: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 931 op/s 2023-10-01T10:26:12.340 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cluster 2023-10-01T10:25:17.317197+0000 mgr.y (mgr.25072) 99 : cluster 0 pgmap v79: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 617 op/s 2023-10-01T10:26:12.340 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: audit 2023-10-01T10:25:17.383261+0000 mon.a (mon.0) 141 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:26:12.340 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cluster 2023-10-01T10:25:19.317958+0000 mgr.y (mgr.25072) 100 : cluster 0 pgmap v80: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 926 op/s 2023-10-01T10:26:12.340 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cluster 2023-10-01T10:25:21.318401+0000 mgr.y (mgr.25072) 101 : cluster 0 pgmap v81: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 617 op/s 2023-10-01T10:26:12.340 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cluster 2023-10-01T10:25:23.318859+0000 mgr.y (mgr.25072) 102 : cluster 0 pgmap v82: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 617 op/s 2023-10-01T10:26:12.340 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cluster 2023-10-01T10:25:25.319596+0000 mgr.y (mgr.25072) 103 : cluster 0 pgmap v83: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 921 op/s 2023-10-01T10:26:12.340 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cluster 2023-10-01T10:25:27.320098+0000 mgr.y (mgr.25072) 104 : cluster 0 pgmap v84: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 612 op/s 2023-10-01T10:26:12.340 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cluster 2023-10-01T10:25:29.320746+0000 mgr.y (mgr.25072) 105 : cluster 0 pgmap v85: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 914 op/s 2023-10-01T10:26:12.341 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cluster 2023-10-01T10:25:31.321145+0000 mgr.y (mgr.25072) 106 : cluster 0 pgmap v86: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 605 op/s 2023-10-01T10:26:12.341 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: audit 2023-10-01T10:25:32.387554+0000 mon.a (mon.0) 142 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:26:12.341 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cluster 2023-10-01T10:25:33.323839+0000 mgr.y (mgr.25072) 107 : cluster 0 pgmap v87: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 605 op/s 2023-10-01T10:26:12.341 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cluster 2023-10-01T10:25:35.324747+0000 mgr.y (mgr.25072) 108 : cluster 0 pgmap v88: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 905 op/s 2023-10-01T10:26:12.341 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cluster 2023-10-01T10:25:37.325219+0000 mgr.y (mgr.25072) 109 : cluster 0 pgmap v89: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 601 op/s 2023-10-01T10:26:12.341 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cluster 2023-10-01T10:25:39.325969+0000 mgr.y (mgr.25072) 110 : cluster 0 pgmap v90: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 900 op/s 2023-10-01T10:26:12.341 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cluster 2023-10-01T10:25:41.326402+0000 mgr.y (mgr.25072) 111 : cluster 0 pgmap v91: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 599 op/s 2023-10-01T10:26:12.341 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cluster 2023-10-01T10:25:43.326823+0000 mgr.y (mgr.25072) 112 : cluster 0 pgmap v92: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 599 op/s 2023-10-01T10:26:12.341 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cluster 2023-10-01T10:25:45.327697+0000 mgr.y (mgr.25072) 113 : cluster 0 pgmap v93: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 902 op/s 2023-10-01T10:26:12.341 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cluster 2023-10-01T10:25:47.328134+0000 mgr.y (mgr.25072) 114 : cluster 0 pgmap v94: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 602 op/s 2023-10-01T10:26:12.341 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: audit 2023-10-01T10:25:47.384139+0000 mon.a (mon.0) 143 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:26:12.342 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cephadm 2023-10-01T10:21:50.937090+0000 mgr.y (mgr.25115) 16 : cephadm 1 Updating smithi173:/etc/ceph/ceph.conf 2023-10-01T10:26:12.342 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cephadm 2023-10-01T10:21:51.153052+0000 mgr.y (mgr.25115) 17 : cephadm 1 Updating smithi173:/var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T10:26:12.342 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cephadm 2023-10-01T10:21:51.155158+0000 mgr.y (mgr.25115) 18 : cephadm 1 Updating smithi078:/var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T10:26:12.342 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cephadm 2023-10-01T10:21:51.384360+0000 mgr.y (mgr.25115) 19 : cephadm 1 Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2023-10-01T10:26:12.342 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cephadm 2023-10-01T10:21:51.384616+0000 mgr.y (mgr.25115) 20 : cephadm 1 Updating smithi173:/etc/ceph/ceph.client.admin.keyring 2023-10-01T10:26:12.342 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cephadm 2023-10-01T10:21:51.599568+0000 mgr.y (mgr.25115) 22 : cephadm 1 Updating smithi173:/var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-01T10:26:12.342 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cephadm 2023-10-01T10:21:51.599867+0000 mgr.y (mgr.25115) 23 : cephadm 1 Updating smithi078:/var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-01T10:26:12.342 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cephadm 2023-10-01T10:21:51.960654+0000 mgr.y (mgr.25115) 24 : cephadm 1 Reconfiguring prometheus.a (dependencies changed)... 2023-10-01T10:26:12.342 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cephadm 2023-10-01T10:21:52.234562+0000 mgr.y (mgr.25115) 25 : cephadm 1 Reconfiguring daemon prometheus.a on smithi173 2023-10-01T10:26:12.343 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cephadm 2023-10-01T10:22:02.286629+0000 mgr.y (mgr.25115) 32 : cephadm 1 Upgrade: Updating mgr.x 2023-10-01T10:26:12.343 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cephadm 2023-10-01T10:22:02.299272+0000 mgr.y (mgr.25115) 33 : cephadm 1 Deploying daemon mgr.x on smithi173 2023-10-01T10:26:12.343 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cephadm 2023-10-01T10:22:22.960834+0000 mgr.y (mgr.25115) 44 : cephadm 1 Upgrade: Setting container_image for all mgr 2023-10-01T10:26:12.343 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cephadm 2023-10-01T10:22:22.986335+0000 mgr.y (mgr.25115) 45 : cephadm 1 Upgrade: It appears safe to stop mon.a 2023-10-01T10:26:12.343 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cephadm 2023-10-01T10:22:25.305462+0000 mgr.y (mgr.25115) 47 : cephadm 1 Upgrade: Updating mon.a 2023-10-01T10:26:12.343 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cephadm 2023-10-01T10:22:25.321118+0000 mgr.y (mgr.25115) 48 : cephadm 1 Deploying daemon mon.a on smithi078 2023-10-01T10:26:12.343 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cephadm 2023-10-01T10:22:48.222444+0000 mgr.y (mgr.25072) 1 : cephadm 1 [01/Oct/2023:10:22:48] ENGINE Bus STARTING 2023-10-01T10:26:12.343 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cephadm 2023-10-01T10:22:48.344219+0000 mgr.y (mgr.25072) 4 : cephadm 1 [01/Oct/2023:10:22:48] ENGINE Serving on https://172.21.15.78:7150 2023-10-01T10:26:12.343 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cephadm 2023-10-01T10:22:48.446003+0000 mgr.y (mgr.25072) 5 : cephadm 1 [01/Oct/2023:10:22:48] ENGINE Serving on http://172.21.15.78:8765 2023-10-01T10:26:12.343 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cephadm 2023-10-01T10:22:48.446382+0000 mgr.y (mgr.25072) 6 : cephadm 1 [01/Oct/2023:10:22:48] ENGINE Bus STARTED 2023-10-01T10:26:12.343 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cephadm 2023-10-01T10:22:53.975096+0000 mgr.y (mgr.25072) 11 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T10:26:12.345 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cephadm 2023-10-01T10:23:07.886128+0000 mgr.y (mgr.25072) 19 : cephadm 1 Updating smithi078:/etc/ceph/ceph.conf 2023-10-01T10:26:12.345 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cephadm 2023-10-01T10:23:07.886362+0000 mgr.y (mgr.25072) 20 : cephadm 1 Updating smithi173:/etc/ceph/ceph.conf 2023-10-01T10:26:12.345 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cephadm 2023-10-01T10:23:08.105307+0000 mgr.y (mgr.25072) 21 : cephadm 1 Updating smithi173:/var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T10:26:12.345 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cephadm 2023-10-01T10:23:08.105710+0000 mgr.y (mgr.25072) 22 : cephadm 1 Updating smithi078:/var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T10:26:12.345 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cephadm 2023-10-01T10:23:08.324481+0000 mgr.y (mgr.25072) 23 : cephadm 1 Updating smithi173:/etc/ceph/ceph.client.admin.keyring 2023-10-01T10:26:12.345 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cephadm 2023-10-01T10:23:08.324802+0000 mgr.y (mgr.25072) 24 : cephadm 1 Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2023-10-01T10:26:12.345 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cephadm 2023-10-01T10:23:08.540586+0000 mgr.y (mgr.25072) 25 : cephadm 1 Updating smithi078:/var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-01T10:26:12.345 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cephadm 2023-10-01T10:23:08.542580+0000 mgr.y (mgr.25072) 26 : cephadm 1 Updating smithi173:/var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-01T10:26:12.345 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cephadm 2023-10-01T10:24:06.570782+0000 mgr.y (mgr.25072) 57 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T10:26:12.346 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cephadm 2023-10-01T10:24:13.756352+0000 mgr.y (mgr.25072) 62 : cephadm 1 Upgrade: It appears safe to stop mon.c 2023-10-01T10:26:12.346 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cephadm 2023-10-01T10:24:18.986358+0000 mgr.y (mgr.25072) 65 : cephadm 1 Upgrade: Updating mon.c 2023-10-01T10:26:12.346 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cephadm 2023-10-01T10:24:19.007779+0000 mgr.y (mgr.25072) 66 : cephadm 1 Deploying daemon mon.c on smithi078 2023-10-01T10:26:12.346 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: audit 2023-10-01T10:24:24.840083+0000 mon.a (mon.0) 99 : audit 0 from='client.? 172.21.15.78:0/3239132624' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:26:12.346 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: audit 2023-10-01T10:24:30.414381+0000 mon.a (mon.0) 110 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.346 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: audit 2023-10-01T10:24:30.422206+0000 mon.a (mon.0) 111 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.346 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: audit 2023-10-01T10:24:30.426220+0000 mon.a (mon.0) 112 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:26:12.346 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: audit 2023-10-01T10:24:31.203512+0000 mon.a (mon.0) 113 : audit 0 from='client.? 172.21.15.78:0/744453728' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:26:12.346 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cluster 2023-10-01T10:24:31.423001+0000 mon.a (mon.0) 121 : cluster 1 Cluster is now healthy 2023-10-01T10:26:12.346 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cluster 2023-10-01T10:24:31.432792+0000 mon.a (mon.0) 122 : cluster 1 overall HEALTH_OK 2023-10-01T10:26:12.346 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: audit 2023-10-01T10:24:32.374431+0000 mon.a (mon.0) 123 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:26:12.346 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cluster 2023-10-01T10:24:33.300742+0000 mgr.y (mgr.25072) 74 : cluster 0 pgmap v57: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 12 MiB/s wr, 673 op/s 2023-10-01T10:26:12.347 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: audit 2023-10-01T10:24:33.937824+0000 mon.a (mon.0) 124 : audit 0 from='client.? 172.21.15.78:0/448982698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:26:12.347 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cluster 2023-10-01T10:24:35.301499+0000 mgr.y (mgr.25072) 75 : cluster 0 pgmap v58: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 11 MiB/s wr, 962 op/s 2023-10-01T10:26:12.347 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cluster 2023-10-01T10:24:37.301977+0000 mgr.y (mgr.25072) 76 : cluster 0 pgmap v59: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 669 op/s 2023-10-01T10:26:12.347 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: audit 2023-10-01T10:24:38.048101+0000 mon.a (mon.0) 125 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.347 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: audit 2023-10-01T10:24:38.061150+0000 mon.a (mon.0) 126 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.347 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: audit 2023-10-01T10:24:39.064412+0000 mon.a (mon.0) 127 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.347 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: audit 2023-10-01T10:24:39.071433+0000 mon.a (mon.0) 128 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.347 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cluster 2023-10-01T10:24:39.302744+0000 mgr.y (mgr.25072) 77 : cluster 0 pgmap v60: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 976 op/s 2023-10-01T10:26:12.347 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: audit 2023-10-01T10:24:40.439142+0000 mgr.y (mgr.25072) 78 : audit 0 from='client.34215 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:26:12.347 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cluster 2023-10-01T10:24:41.303179+0000 mgr.y (mgr.25072) 79 : cluster 0 pgmap v61: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 635 op/s 2023-10-01T10:26:12.347 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cluster 2023-10-01T10:24:43.303687+0000 mgr.y (mgr.25072) 80 : cluster 0 pgmap v62: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 635 op/s 2023-10-01T10:26:12.347 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cluster 2023-10-01T10:24:45.305489+0000 mgr.y (mgr.25072) 81 : cluster 0 pgmap v63: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 955 op/s 2023-10-01T10:26:12.348 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: audit 2023-10-01T10:24:46.280316+0000 mgr.y (mgr.25072) 82 : audit 0 from='client.34218 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:26:12.348 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cephadm 2023-10-01T10:24:46.281006+0000 mgr.y (mgr.25072) 83 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T10:26:12.348 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: audit 2023-10-01T10:24:46.291061+0000 mon.a (mon.0) 129 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.348 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cluster 2023-10-01T10:24:47.305900+0000 mgr.y (mgr.25072) 84 : cluster 0 pgmap v64: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 626 op/s 2023-10-01T10:26:12.348 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: audit 2023-10-01T10:24:47.382805+0000 mon.a (mon.0) 130 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:26:12.348 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cluster 2023-10-01T10:24:49.306445+0000 mgr.y (mgr.25072) 85 : cluster 0 pgmap v65: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 942 op/s 2023-10-01T10:26:12.348 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: audit 2023-10-01T10:24:50.908342+0000 mon.a (mon.0) 131 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.348 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: audit 2023-10-01T10:24:50.918285+0000 mon.a (mon.0) 132 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.348 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: audit 2023-10-01T10:24:50.919599+0000 mon.a (mon.0) 133 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:26:12.348 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: audit 2023-10-01T10:24:50.923731+0000 mon.a (mon.0) 134 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:26:12.348 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: audit 2023-10-01T10:24:50.932637+0000 mon.a (mon.0) 135 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.348 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: audit 2023-10-01T10:24:51.019441+0000 mon.a (mon.0) 136 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:26:12.348 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: audit 2023-10-01T10:24:51.021592+0000 mon.a (mon.0) 137 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:26:12.349 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: audit 2023-10-01T10:24:51.029692+0000 mon.a (mon.0) 138 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:26:12.349 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: audit 2023-10-01T10:24:51.036687+0000 mon.a (mon.0) 139 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.349 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cluster 2023-10-01T10:24:51.306854+0000 mgr.y (mgr.25072) 86 : cluster 0 pgmap v66: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 636 op/s 2023-10-01T10:26:12.349 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cluster 2023-10-01T10:24:53.307323+0000 mgr.y (mgr.25072) 87 : cluster 0 pgmap v67: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 636 op/s 2023-10-01T10:26:12.349 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cluster 2023-10-01T10:24:55.308104+0000 mgr.y (mgr.25072) 88 : cluster 0 pgmap v68: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 950 op/s 2023-10-01T10:26:12.349 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cluster 2023-10-01T10:24:57.308530+0000 mgr.y (mgr.25072) 89 : cluster 0 pgmap v69: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 630 op/s 2023-10-01T10:26:12.349 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cluster 2023-10-01T10:24:59.309238+0000 mgr.y (mgr.25072) 90 : cluster 0 pgmap v70: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 950 op/s 2023-10-01T10:26:12.349 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cluster 2023-10-01T10:25:01.309690+0000 mgr.y (mgr.25072) 91 : cluster 0 pgmap v71: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 633 op/s 2023-10-01T10:26:12.349 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: audit 2023-10-01T10:25:02.383036+0000 mon.a (mon.0) 140 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:26:12.349 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cluster 2023-10-01T10:25:03.310268+0000 mgr.y (mgr.25072) 92 : cluster 0 pgmap v72: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 633 op/s 2023-10-01T10:26:12.349 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cluster 2023-10-01T10:25:05.310995+0000 mgr.y (mgr.25072) 93 : cluster 0 pgmap v73: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 947 op/s 2023-10-01T10:26:12.349 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cluster 2023-10-01T10:25:07.311409+0000 mgr.y (mgr.25072) 94 : cluster 0 pgmap v74: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 633 op/s 2023-10-01T10:26:12.350 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cluster 2023-10-01T10:25:09.312006+0000 mgr.y (mgr.25072) 95 : cluster 0 pgmap v75: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 943 op/s 2023-10-01T10:26:12.350 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cluster 2023-10-01T10:25:11.315486+0000 mgr.y (mgr.25072) 96 : cluster 0 pgmap v76: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 622 op/s 2023-10-01T10:26:12.350 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cluster 2023-10-01T10:25:13.315999+0000 mgr.y (mgr.25072) 97 : cluster 0 pgmap v77: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2023-10-01T10:26:12.350 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cluster 2023-10-01T10:25:15.316770+0000 mgr.y (mgr.25072) 98 : cluster 0 pgmap v78: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 931 op/s 2023-10-01T10:26:12.350 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cluster 2023-10-01T10:25:17.317197+0000 mgr.y (mgr.25072) 99 : cluster 0 pgmap v79: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 617 op/s 2023-10-01T10:26:12.350 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: audit 2023-10-01T10:25:17.383261+0000 mon.a (mon.0) 141 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:26:12.350 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cluster 2023-10-01T10:25:49.328877+0000 mgr.y (mgr.25072) 115 : cluster 0 pgmap v95: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 908 op/s 2023-10-01T10:26:12.351 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: audit 2023-10-01T10:25:51.115625+0000 mon.a (mon.0) 144 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:26:12.351 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cluster 2023-10-01T10:25:51.329319+0000 mgr.y (mgr.25072) 116 : cluster 0 pgmap v96: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 609 op/s 2023-10-01T10:26:12.351 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: audit 2023-10-01T10:25:51.647899+0000 mon.a (mon.0) 145 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.351 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: audit 2023-10-01T10:25:51.688342+0000 mon.a (mon.0) 146 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.351 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cluster 2023-10-01T10:25:53.329746+0000 mgr.y (mgr.25072) 117 : cluster 0 pgmap v97: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 609 op/s 2023-10-01T10:26:12.351 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cluster 2023-10-01T10:25:55.332526+0000 mgr.y (mgr.25072) 118 : cluster 0 pgmap v98: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 898 op/s 2023-10-01T10:26:12.351 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cluster 2023-10-01T10:25:57.332902+0000 mgr.y (mgr.25072) 119 : cluster 0 pgmap v99: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 595 op/s 2023-10-01T10:26:12.351 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: audit 2023-10-01T10:25:57.676339+0000 mon.a (mon.0) 147 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.351 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: audit 2023-10-01T10:25:57.691224+0000 mon.a (mon.0) 148 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.351 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: audit 2023-10-01T10:25:58.744984+0000 mgr.y (mgr.25072) 120 : audit 0 from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:26:12.351 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cephadm 2023-10-01T10:25:58.745431+0000 mgr.y (mgr.25072) 121 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T10:26:12.351 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: audit 2023-10-01T10:25:58.753130+0000 mon.a (mon.0) 149 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.352 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cluster 2023-10-01T10:25:59.333749+0000 mgr.y (mgr.25072) 122 : cluster 0 pgmap v100: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 891 op/s 2023-10-01T10:26:12.352 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: audit 2023-10-01T10:25:59.600092+0000 mon.a (mon.0) 150 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.352 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: audit 2023-10-01T10:25:59.607175+0000 mon.a (mon.0) 151 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.352 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: audit 2023-10-01T10:26:00.245372+0000 mon.a (mon.0) 152 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:26:12.352 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: audit 2023-10-01T10:26:00.246702+0000 mon.a (mon.0) 153 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:26:12.352 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: audit 2023-10-01T10:26:00.256003+0000 mon.a (mon.0) 154 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.352 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: audit 2023-10-01T10:26:00.339374+0000 mon.a (mon.0) 155 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:26:12.352 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: audit 2023-10-01T10:26:00.349038+0000 mon.a (mon.0) 156 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:26:12.352 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: audit 2023-10-01T10:26:00.355427+0000 mon.a (mon.0) 157 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-01T10:26:12.352 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: audit 2023-10-01T10:26:00.356600+0000 mon.a (mon.0) 158 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-10-01T10:26:12.352 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cephadm 2023-10-01T10:26:00.357370+0000 mgr.y (mgr.25072) 123 : cephadm 1 Upgrade: It appears safe to stop mon.b 2023-10-01T10:26:12.352 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cluster 2023-10-01T10:26:01.334179+0000 mgr.y (mgr.25072) 124 : cluster 0 pgmap v101: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 585 op/s 2023-10-01T10:26:12.353 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: audit 2023-10-01T10:26:02.384378+0000 mon.a (mon.0) 159 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:26:12.353 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cluster 2023-10-01T10:26:03.334636+0000 mgr.y (mgr.25072) 125 : cluster 0 pgmap v102: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 585 op/s 2023-10-01T10:26:12.353 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cephadm 2023-10-01T10:26:05.246409+0000 mgr.y (mgr.25072) 126 : cephadm 1 Upgrade: Updating mon.b 2023-10-01T10:26:12.353 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: audit 2023-10-01T10:26:05.262780+0000 mon.a (mon.0) 160 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.353 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: audit 2023-10-01T10:26:05.264561+0000 mon.a (mon.0) 161 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T10:26:12.353 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: audit 2023-10-01T10:26:05.266210+0000 mon.a (mon.0) 162 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T10:26:12.353 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: audit 2023-10-01T10:26:05.271384+0000 mon.a (mon.0) 163 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:26:12.353 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cephadm 2023-10-01T10:26:05.273309+0000 mgr.y (mgr.25072) 127 : cephadm 1 Deploying daemon mon.b on smithi173 2023-10-01T10:26:12.353 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cluster 2023-10-01T10:26:05.335349+0000 mgr.y (mgr.25072) 128 : cluster 0 pgmap v103: 89 pgs: 89 active+clean; 2.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2023-10-01T10:26:12.353 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cluster 2023-10-01T10:26:07.335768+0000 mgr.y (mgr.25072) 129 : cluster 0 pgmap v104: 89 pgs: 89 active+clean; 2.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 586 op/s 2023-10-01T10:26:12.353 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: audit 2023-10-01T10:26:08.661742+0000 mon.c (mon.2) 2 : audit 0 from='client.? 172.21.15.78:0/3583851419' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:26:12.353 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cluster 2023-10-01T10:26:11.004253+0000 mon.a (mon.0) 170 : cluster 1 overall HEALTH_OK 2023-10-01T10:26:12.353 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: audit 2023-10-01T10:26:11.004576+0000 mon.a (mon.0) 171 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T10:26:12.354 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cluster 2023-10-01T10:26:11.004907+0000 mon.a (mon.0) 172 : cluster 1 mon.a calling monitor election 2023-10-01T10:26:12.354 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cluster 2023-10-01T10:26:11.004952+0000 mon.c (mon.2) 3 : cluster 1 mon.c calling monitor election 2023-10-01T10:26:12.354 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: audit 2023-10-01T10:26:11.006433+0000 mon.a (mon.0) 173 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T10:26:12.354 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: audit 2023-10-01T10:26:11.006748+0000 mon.a (mon.0) 174 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T10:26:12.354 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cluster 2023-10-01T10:26:11.009717+0000 mon.b (mon.1) 2 : cluster 1 mon.b calling monitor election 2023-10-01T10:26:12.354 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cluster 2023-10-01T10:26:11.010564+0000 mon.a (mon.0) 175 : cluster 1 mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-01T10:26:12.354 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cluster 2023-10-01T10:26:11.021522+0000 mon.a (mon.0) 176 : cluster 0 monmap e4: 3 mons at {a=[v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0],b=[v2:172.21.15.173:3300/0,v1:172.21.15.173:6789/0],c=[v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0]} removed_ranks: {} 2023-10-01T10:26:12.354 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cluster 2023-10-01T10:26:11.022329+0000 mon.a (mon.0) 177 : cluster 0 fsmap foo:1 {0=foo.smithi078.vjqrth=up:active} 1 up:standby 2023-10-01T10:26:12.354 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cluster 2023-10-01T10:26:11.022378+0000 mon.a (mon.0) 178 : cluster 0 osdmap e77: 8 total, 8 up, 8 in 2023-10-01T10:26:12.354 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cluster 2023-10-01T10:26:11.026052+0000 mon.a (mon.0) 179 : cluster 0 mgrmap e34: y(active, since 3m), standbys: x 2023-10-01T10:26:12.355 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cluster 2023-10-01T10:26:11.034015+0000 mon.a (mon.0) 180 : cluster 1 overall HEALTH_OK 2023-10-01T10:26:12.355 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: audit 2023-10-01T10:26:11.041871+0000 mon.a (mon.0) 181 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.355 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: audit 2023-10-01T10:26:11.053072+0000 mon.a (mon.0) 182 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.355 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: audit 2023-10-01T10:26:11.065573+0000 mon.a (mon.0) 183 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:26:12.355 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: cluster 2023-10-01T10:26:11.336939+0000 mgr.y (mgr.25072) 131 : cluster 0 pgmap v106: 89 pgs: 89 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 593 op/s 2023-10-01T10:26:12.355 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[75299]: audit 2023-10-01T10:26:11.419118+0000 mon.c (mon.2) 4 : audit 0 from='client.? 172.21.15.78:0/2924871869' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:26:12.355 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cluster 2023-10-01T10:25:19.317958+0000 mgr.y (mgr.25072) 100 : cluster 0 pgmap v80: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 926 op/s 2023-10-01T10:26:12.355 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cluster 2023-10-01T10:25:21.318401+0000 mgr.y (mgr.25072) 101 : cluster 0 pgmap v81: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 617 op/s 2023-10-01T10:26:12.355 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cluster 2023-10-01T10:25:23.318859+0000 mgr.y (mgr.25072) 102 : cluster 0 pgmap v82: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 617 op/s 2023-10-01T10:26:12.355 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cluster 2023-10-01T10:25:25.319596+0000 mgr.y (mgr.25072) 103 : cluster 0 pgmap v83: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 921 op/s 2023-10-01T10:26:12.356 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cluster 2023-10-01T10:25:27.320098+0000 mgr.y (mgr.25072) 104 : cluster 0 pgmap v84: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 612 op/s 2023-10-01T10:26:12.356 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cluster 2023-10-01T10:25:29.320746+0000 mgr.y (mgr.25072) 105 : cluster 0 pgmap v85: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 914 op/s 2023-10-01T10:26:12.356 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cluster 2023-10-01T10:25:31.321145+0000 mgr.y (mgr.25072) 106 : cluster 0 pgmap v86: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 605 op/s 2023-10-01T10:26:12.356 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: audit 2023-10-01T10:25:32.387554+0000 mon.a (mon.0) 142 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:26:12.356 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cluster 2023-10-01T10:25:33.323839+0000 mgr.y (mgr.25072) 107 : cluster 0 pgmap v87: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 605 op/s 2023-10-01T10:26:12.356 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cluster 2023-10-01T10:25:35.324747+0000 mgr.y (mgr.25072) 108 : cluster 0 pgmap v88: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 905 op/s 2023-10-01T10:26:12.356 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cluster 2023-10-01T10:25:37.325219+0000 mgr.y (mgr.25072) 109 : cluster 0 pgmap v89: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 601 op/s 2023-10-01T10:26:12.356 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cluster 2023-10-01T10:25:39.325969+0000 mgr.y (mgr.25072) 110 : cluster 0 pgmap v90: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 900 op/s 2023-10-01T10:26:12.356 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cluster 2023-10-01T10:25:41.326402+0000 mgr.y (mgr.25072) 111 : cluster 0 pgmap v91: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 599 op/s 2023-10-01T10:26:12.356 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cluster 2023-10-01T10:25:43.326823+0000 mgr.y (mgr.25072) 112 : cluster 0 pgmap v92: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 599 op/s 2023-10-01T10:26:12.356 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cluster 2023-10-01T10:25:45.327697+0000 mgr.y (mgr.25072) 113 : cluster 0 pgmap v93: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 902 op/s 2023-10-01T10:26:12.356 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cluster 2023-10-01T10:25:47.328134+0000 mgr.y (mgr.25072) 114 : cluster 0 pgmap v94: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 602 op/s 2023-10-01T10:26:12.357 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: audit 2023-10-01T10:25:47.384139+0000 mon.a (mon.0) 143 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:26:12.357 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cluster 2023-10-01T10:25:49.328877+0000 mgr.y (mgr.25072) 115 : cluster 0 pgmap v95: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 908 op/s 2023-10-01T10:26:12.357 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: audit 2023-10-01T10:25:51.115625+0000 mon.a (mon.0) 144 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:26:12.357 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cluster 2023-10-01T10:25:51.329319+0000 mgr.y (mgr.25072) 116 : cluster 0 pgmap v96: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 609 op/s 2023-10-01T10:26:12.357 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: audit 2023-10-01T10:25:51.647899+0000 mon.a (mon.0) 145 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.357 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: audit 2023-10-01T10:25:51.688342+0000 mon.a (mon.0) 146 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.357 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cluster 2023-10-01T10:25:53.329746+0000 mgr.y (mgr.25072) 117 : cluster 0 pgmap v97: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 609 op/s 2023-10-01T10:26:12.357 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cluster 2023-10-01T10:25:55.332526+0000 mgr.y (mgr.25072) 118 : cluster 0 pgmap v98: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 898 op/s 2023-10-01T10:26:12.357 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cluster 2023-10-01T10:25:57.332902+0000 mgr.y (mgr.25072) 119 : cluster 0 pgmap v99: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 595 op/s 2023-10-01T10:26:12.357 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: audit 2023-10-01T10:25:57.676339+0000 mon.a (mon.0) 147 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.357 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: audit 2023-10-01T10:25:57.691224+0000 mon.a (mon.0) 148 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.357 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: audit 2023-10-01T10:25:58.744984+0000 mgr.y (mgr.25072) 120 : audit 0 from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:26:12.357 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cephadm 2023-10-01T10:25:58.745431+0000 mgr.y (mgr.25072) 121 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T10:26:12.358 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: audit 2023-10-01T10:25:58.753130+0000 mon.a (mon.0) 149 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.358 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cluster 2023-10-01T10:25:59.333749+0000 mgr.y (mgr.25072) 122 : cluster 0 pgmap v100: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 891 op/s 2023-10-01T10:26:12.358 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: audit 2023-10-01T10:25:59.600092+0000 mon.a (mon.0) 150 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.358 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: audit 2023-10-01T10:25:59.607175+0000 mon.a (mon.0) 151 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.358 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: audit 2023-10-01T10:26:00.245372+0000 mon.a (mon.0) 152 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:26:12.358 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: audit 2023-10-01T10:26:00.246702+0000 mon.a (mon.0) 153 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:26:12.358 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: audit 2023-10-01T10:26:00.256003+0000 mon.a (mon.0) 154 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.358 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: audit 2023-10-01T10:26:00.339374+0000 mon.a (mon.0) 155 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:26:12.358 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: audit 2023-10-01T10:26:00.349038+0000 mon.a (mon.0) 156 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:26:12.358 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: audit 2023-10-01T10:26:00.355427+0000 mon.a (mon.0) 157 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-01T10:26:12.358 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: audit 2023-10-01T10:26:00.356600+0000 mon.a (mon.0) 158 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-10-01T10:26:12.358 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cephadm 2023-10-01T10:26:00.357370+0000 mgr.y (mgr.25072) 123 : cephadm 1 Upgrade: It appears safe to stop mon.b 2023-10-01T10:26:12.359 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cluster 2023-10-01T10:26:01.334179+0000 mgr.y (mgr.25072) 124 : cluster 0 pgmap v101: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 585 op/s 2023-10-01T10:26:12.359 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: audit 2023-10-01T10:26:02.384378+0000 mon.a (mon.0) 159 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:26:12.359 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cluster 2023-10-01T10:26:03.334636+0000 mgr.y (mgr.25072) 125 : cluster 0 pgmap v102: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 585 op/s 2023-10-01T10:26:12.359 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cephadm 2023-10-01T10:26:05.246409+0000 mgr.y (mgr.25072) 126 : cephadm 1 Upgrade: Updating mon.b 2023-10-01T10:26:12.359 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: audit 2023-10-01T10:26:05.262780+0000 mon.a (mon.0) 160 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.359 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: audit 2023-10-01T10:26:05.264561+0000 mon.a (mon.0) 161 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T10:26:12.359 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: audit 2023-10-01T10:26:05.266210+0000 mon.a (mon.0) 162 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T10:26:12.359 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: audit 2023-10-01T10:26:05.271384+0000 mon.a (mon.0) 163 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:26:12.359 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cephadm 2023-10-01T10:26:05.273309+0000 mgr.y (mgr.25072) 127 : cephadm 1 Deploying daemon mon.b on smithi173 2023-10-01T10:26:12.359 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cluster 2023-10-01T10:26:05.335349+0000 mgr.y (mgr.25072) 128 : cluster 0 pgmap v103: 89 pgs: 89 active+clean; 2.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2023-10-01T10:26:12.359 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cluster 2023-10-01T10:26:07.335768+0000 mgr.y (mgr.25072) 129 : cluster 0 pgmap v104: 89 pgs: 89 active+clean; 2.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 586 op/s 2023-10-01T10:26:12.359 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: audit 2023-10-01T10:26:08.661742+0000 mon.c (mon.2) 2 : audit 0 from='client.? 172.21.15.78:0/3583851419' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:26:12.360 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cluster 2023-10-01T10:26:11.004253+0000 mon.a (mon.0) 170 : cluster 1 overall HEALTH_OK 2023-10-01T10:26:12.360 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: audit 2023-10-01T10:26:11.004576+0000 mon.a (mon.0) 171 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T10:26:12.360 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cluster 2023-10-01T10:26:11.004907+0000 mon.a (mon.0) 172 : cluster 1 mon.a calling monitor election 2023-10-01T10:26:12.360 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cluster 2023-10-01T10:26:11.004952+0000 mon.c (mon.2) 3 : cluster 1 mon.c calling monitor election 2023-10-01T10:26:12.360 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: audit 2023-10-01T10:26:11.006433+0000 mon.a (mon.0) 173 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T10:26:12.360 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: audit 2023-10-01T10:26:11.006748+0000 mon.a (mon.0) 174 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T10:26:12.360 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cluster 2023-10-01T10:26:11.009717+0000 mon.b (mon.1) 2 : cluster 1 mon.b calling monitor election 2023-10-01T10:26:12.360 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cluster 2023-10-01T10:26:11.010564+0000 mon.a (mon.0) 175 : cluster 1 mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-01T10:26:12.360 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cluster 2023-10-01T10:26:11.021522+0000 mon.a (mon.0) 176 : cluster 0 monmap e4: 3 mons at {a=[v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0],b=[v2:172.21.15.173:3300/0,v1:172.21.15.173:6789/0],c=[v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0]} removed_ranks: {} 2023-10-01T10:26:12.360 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cluster 2023-10-01T10:26:11.022329+0000 mon.a (mon.0) 177 : cluster 0 fsmap foo:1 {0=foo.smithi078.vjqrth=up:active} 1 up:standby 2023-10-01T10:26:12.360 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cluster 2023-10-01T10:26:11.022378+0000 mon.a (mon.0) 178 : cluster 0 osdmap e77: 8 total, 8 up, 8 in 2023-10-01T10:26:12.360 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cluster 2023-10-01T10:26:11.026052+0000 mon.a (mon.0) 179 : cluster 0 mgrmap e34: y(active, since 3m), standbys: x 2023-10-01T10:26:12.361 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cluster 2023-10-01T10:26:11.034015+0000 mon.a (mon.0) 180 : cluster 1 overall HEALTH_OK 2023-10-01T10:26:12.361 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: audit 2023-10-01T10:26:11.041871+0000 mon.a (mon.0) 181 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.361 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: audit 2023-10-01T10:26:11.053072+0000 mon.a (mon.0) 182 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.361 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: audit 2023-10-01T10:26:11.065573+0000 mon.a (mon.0) 183 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:26:12.361 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: cluster 2023-10-01T10:26:11.336939+0000 mgr.y (mgr.25072) 131 : cluster 0 pgmap v106: 89 pgs: 89 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 593 op/s 2023-10-01T10:26:12.361 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:12 smithi078 bash[78274]: audit 2023-10-01T10:26:11.419118+0000 mon.c (mon.2) 4 : audit 0 from='client.? 172.21.15.78:0/2924871869' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:26:12.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cluster 2023-10-01T10:04:59.345901+0000 mon.a (mon.0) 0 : 1 mkfs d8016480-6041-11ee-8db5-212e2dc638e7 2023-10-01T10:26:12.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cephadm 2023-10-01T10:20:23.346660+0000 mgr.x (mgr.24697) 51 : cephadm 1 Reconfiguring node-exporter.b (dependencies changed)... 2023-10-01T10:26:12.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cephadm 2023-10-01T10:20:23.347123+0000 mgr.x (mgr.24697) 52 : cephadm 1 Reconfiguring daemon node-exporter.b on smithi173 2023-10-01T10:26:12.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cephadm 2023-10-01T10:20:25.272648+0000 mgr.x (mgr.24697) 54 : cephadm 1 Reconfiguring grafana.a (dependencies changed)... 2023-10-01T10:26:12.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cephadm 2023-10-01T10:20:25.296121+0000 mgr.x (mgr.24697) 55 : cephadm 1 Reconfiguring daemon grafana.a on smithi173 2023-10-01T10:26:12.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cephadm 2023-10-01T10:21:02.510561+0000 mgr.x (mgr.24697) 77 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2023-10-01T10:26:12.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cephadm 2023-10-01T10:21:02.511607+0000 mgr.x (mgr.24697) 78 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2023-10-01T10:26:12.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cephadm 2023-10-01T10:21:07.797945+0000 mgr.x (mgr.24697) 82 : cephadm 1 Upgrade: Updating mgr.y 2023-10-01T10:26:12.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cephadm 2023-10-01T10:21:07.819165+0000 mgr.x (mgr.24697) 83 : cephadm 1 Deploying daemon mgr.y on smithi078 2023-10-01T10:26:12.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cephadm 2023-10-01T10:21:32.255585+0000 mgr.y (mgr.25115) 1 : cephadm 1 [01/Oct/2023:10:21:32] ENGINE Bus STARTING 2023-10-01T10:26:12.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cephadm 2023-10-01T10:21:32.429884+0000 mgr.y (mgr.25115) 2 : cephadm 1 [01/Oct/2023:10:21:32] ENGINE Serving on https://172.21.15.78:7150 2023-10-01T10:26:12.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cephadm 2023-10-01T10:21:32.534004+0000 mgr.y (mgr.25115) 4 : cephadm 1 [01/Oct/2023:10:21:32] ENGINE Serving on http://172.21.15.78:8765 2023-10-01T10:26:12.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cephadm 2023-10-01T10:21:32.534138+0000 mgr.y (mgr.25115) 5 : cephadm 1 [01/Oct/2023:10:21:32] ENGINE Bus STARTED 2023-10-01T10:26:12.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cephadm 2023-10-01T10:21:50.936847+0000 mgr.y (mgr.25115) 15 : cephadm 1 Updating smithi078:/etc/ceph/ceph.conf 2023-10-01T10:26:12.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cephadm 2023-10-01T10:21:50.937090+0000 mgr.y (mgr.25115) 16 : cephadm 1 Updating smithi173:/etc/ceph/ceph.conf 2023-10-01T10:26:12.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cephadm 2023-10-01T10:21:51.153052+0000 mgr.y (mgr.25115) 17 : cephadm 1 Updating smithi173:/var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T10:26:12.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cephadm 2023-10-01T10:21:51.155158+0000 mgr.y (mgr.25115) 18 : cephadm 1 Updating smithi078:/var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T10:26:12.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cephadm 2023-10-01T10:21:51.384360+0000 mgr.y (mgr.25115) 19 : cephadm 1 Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2023-10-01T10:26:12.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cephadm 2023-10-01T10:21:51.384616+0000 mgr.y (mgr.25115) 20 : cephadm 1 Updating smithi173:/etc/ceph/ceph.client.admin.keyring 2023-10-01T10:26:12.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cephadm 2023-10-01T10:21:51.599568+0000 mgr.y (mgr.25115) 22 : cephadm 1 Updating smithi173:/var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-01T10:26:12.372 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cephadm 2023-10-01T10:21:51.599867+0000 mgr.y (mgr.25115) 23 : cephadm 1 Updating smithi078:/var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-01T10:26:12.372 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cephadm 2023-10-01T10:21:51.960654+0000 mgr.y (mgr.25115) 24 : cephadm 1 Reconfiguring prometheus.a (dependencies changed)... 2023-10-01T10:26:12.372 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cephadm 2023-10-01T10:21:52.234562+0000 mgr.y (mgr.25115) 25 : cephadm 1 Reconfiguring daemon prometheus.a on smithi173 2023-10-01T10:26:12.372 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cephadm 2023-10-01T10:22:02.286629+0000 mgr.y (mgr.25115) 32 : cephadm 1 Upgrade: Updating mgr.x 2023-10-01T10:26:12.372 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cephadm 2023-10-01T10:22:02.299272+0000 mgr.y (mgr.25115) 33 : cephadm 1 Deploying daemon mgr.x on smithi173 2023-10-01T10:26:12.372 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cephadm 2023-10-01T10:22:22.960834+0000 mgr.y (mgr.25115) 44 : cephadm 1 Upgrade: Setting container_image for all mgr 2023-10-01T10:26:12.372 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cephadm 2023-10-01T10:22:22.986335+0000 mgr.y (mgr.25115) 45 : cephadm 1 Upgrade: It appears safe to stop mon.a 2023-10-01T10:26:12.372 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cephadm 2023-10-01T10:22:25.305462+0000 mgr.y (mgr.25115) 47 : cephadm 1 Upgrade: Updating mon.a 2023-10-01T10:26:12.372 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cephadm 2023-10-01T10:22:25.321118+0000 mgr.y (mgr.25115) 48 : cephadm 1 Deploying daemon mon.a on smithi078 2023-10-01T10:26:12.372 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cephadm 2023-10-01T10:22:48.222444+0000 mgr.y (mgr.25072) 1 : cephadm 1 [01/Oct/2023:10:22:48] ENGINE Bus STARTING 2023-10-01T10:26:12.372 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cephadm 2023-10-01T10:22:48.344219+0000 mgr.y (mgr.25072) 4 : cephadm 1 [01/Oct/2023:10:22:48] ENGINE Serving on https://172.21.15.78:7150 2023-10-01T10:26:12.372 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cephadm 2023-10-01T10:22:48.446003+0000 mgr.y (mgr.25072) 5 : cephadm 1 [01/Oct/2023:10:22:48] ENGINE Serving on http://172.21.15.78:8765 2023-10-01T10:26:12.373 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cephadm 2023-10-01T10:22:48.446382+0000 mgr.y (mgr.25072) 6 : cephadm 1 [01/Oct/2023:10:22:48] ENGINE Bus STARTED 2023-10-01T10:26:12.373 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cephadm 2023-10-01T10:22:53.975096+0000 mgr.y (mgr.25072) 11 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T10:26:12.373 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cephadm 2023-10-01T10:23:07.886128+0000 mgr.y (mgr.25072) 19 : cephadm 1 Updating smithi078:/etc/ceph/ceph.conf 2023-10-01T10:26:12.373 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cephadm 2023-10-01T10:23:07.886362+0000 mgr.y (mgr.25072) 20 : cephadm 1 Updating smithi173:/etc/ceph/ceph.conf 2023-10-01T10:26:12.373 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cephadm 2023-10-01T10:23:08.105307+0000 mgr.y (mgr.25072) 21 : cephadm 1 Updating smithi173:/var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T10:26:12.373 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cephadm 2023-10-01T10:23:08.105710+0000 mgr.y (mgr.25072) 22 : cephadm 1 Updating smithi078:/var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/config/ceph.conf 2023-10-01T10:26:12.373 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cephadm 2023-10-01T10:23:08.324481+0000 mgr.y (mgr.25072) 23 : cephadm 1 Updating smithi173:/etc/ceph/ceph.client.admin.keyring 2023-10-01T10:26:12.373 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cephadm 2023-10-01T10:23:08.324802+0000 mgr.y (mgr.25072) 24 : cephadm 1 Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2023-10-01T10:26:12.373 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cephadm 2023-10-01T10:23:08.540586+0000 mgr.y (mgr.25072) 25 : cephadm 1 Updating smithi078:/var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-01T10:26:12.373 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cephadm 2023-10-01T10:23:08.542580+0000 mgr.y (mgr.25072) 26 : cephadm 1 Updating smithi173:/var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/config/ceph.client.admin.keyring 2023-10-01T10:26:12.373 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cephadm 2023-10-01T10:24:06.570782+0000 mgr.y (mgr.25072) 57 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T10:26:12.373 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cephadm 2023-10-01T10:24:13.756352+0000 mgr.y (mgr.25072) 62 : cephadm 1 Upgrade: It appears safe to stop mon.c 2023-10-01T10:26:12.374 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cephadm 2023-10-01T10:24:18.986358+0000 mgr.y (mgr.25072) 65 : cephadm 1 Upgrade: Updating mon.c 2023-10-01T10:26:12.374 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cephadm 2023-10-01T10:24:19.007779+0000 mgr.y (mgr.25072) 66 : cephadm 1 Deploying daemon mon.c on smithi078 2023-10-01T10:26:12.374 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: audit 2023-10-01T10:24:24.840083+0000 mon.a (mon.0) 99 : audit 0 from='client.? 172.21.15.78:0/3239132624' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:26:12.374 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: audit 2023-10-01T10:24:30.414381+0000 mon.a (mon.0) 110 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.374 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: audit 2023-10-01T10:24:30.422206+0000 mon.a (mon.0) 111 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.374 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: audit 2023-10-01T10:24:30.426220+0000 mon.a (mon.0) 112 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:26:12.375 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: audit 2023-10-01T10:24:31.203512+0000 mon.a (mon.0) 113 : audit 0 from='client.? 172.21.15.78:0/744453728' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:26:12.375 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cluster 2023-10-01T10:24:31.423001+0000 mon.a (mon.0) 121 : cluster 1 Cluster is now healthy 2023-10-01T10:26:12.375 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cluster 2023-10-01T10:24:31.432792+0000 mon.a (mon.0) 122 : cluster 1 overall HEALTH_OK 2023-10-01T10:26:12.375 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: audit 2023-10-01T10:24:32.374431+0000 mon.a (mon.0) 123 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:26:12.375 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cluster 2023-10-01T10:24:33.300742+0000 mgr.y (mgr.25072) 74 : cluster 0 pgmap v57: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 12 MiB/s wr, 673 op/s 2023-10-01T10:26:12.375 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: audit 2023-10-01T10:24:33.937824+0000 mon.a (mon.0) 124 : audit 0 from='client.? 172.21.15.78:0/448982698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:26:12.375 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cluster 2023-10-01T10:24:35.301499+0000 mgr.y (mgr.25072) 75 : cluster 0 pgmap v58: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 11 MiB/s wr, 962 op/s 2023-10-01T10:26:12.375 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cluster 2023-10-01T10:24:37.301977+0000 mgr.y (mgr.25072) 76 : cluster 0 pgmap v59: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 669 op/s 2023-10-01T10:26:12.375 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: audit 2023-10-01T10:24:38.048101+0000 mon.a (mon.0) 125 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.375 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: audit 2023-10-01T10:24:38.061150+0000 mon.a (mon.0) 126 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.375 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: audit 2023-10-01T10:24:39.064412+0000 mon.a (mon.0) 127 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.375 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: audit 2023-10-01T10:24:39.071433+0000 mon.a (mon.0) 128 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.376 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cluster 2023-10-01T10:24:39.302744+0000 mgr.y (mgr.25072) 77 : cluster 0 pgmap v60: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 976 op/s 2023-10-01T10:26:12.376 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: audit 2023-10-01T10:24:40.439142+0000 mgr.y (mgr.25072) 78 : audit 0 from='client.34215 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:26:12.376 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cluster 2023-10-01T10:24:41.303179+0000 mgr.y (mgr.25072) 79 : cluster 0 pgmap v61: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 635 op/s 2023-10-01T10:26:12.376 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cluster 2023-10-01T10:24:43.303687+0000 mgr.y (mgr.25072) 80 : cluster 0 pgmap v62: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 635 op/s 2023-10-01T10:26:12.376 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cluster 2023-10-01T10:24:45.305489+0000 mgr.y (mgr.25072) 81 : cluster 0 pgmap v63: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 955 op/s 2023-10-01T10:26:12.376 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: audit 2023-10-01T10:24:46.280316+0000 mgr.y (mgr.25072) 82 : audit 0 from='client.34218 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:26:12.376 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cephadm 2023-10-01T10:24:46.281006+0000 mgr.y (mgr.25072) 83 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T10:26:12.376 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: audit 2023-10-01T10:24:46.291061+0000 mon.a (mon.0) 129 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.376 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cluster 2023-10-01T10:24:47.305900+0000 mgr.y (mgr.25072) 84 : cluster 0 pgmap v64: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 626 op/s 2023-10-01T10:26:12.376 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: audit 2023-10-01T10:24:47.382805+0000 mon.a (mon.0) 130 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:26:12.376 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cluster 2023-10-01T10:24:49.306445+0000 mgr.y (mgr.25072) 85 : cluster 0 pgmap v65: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 942 op/s 2023-10-01T10:26:12.376 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: audit 2023-10-01T10:24:50.908342+0000 mon.a (mon.0) 131 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.377 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: audit 2023-10-01T10:24:50.918285+0000 mon.a (mon.0) 132 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.377 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: audit 2023-10-01T10:24:50.919599+0000 mon.a (mon.0) 133 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:26:12.377 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: audit 2023-10-01T10:24:50.923731+0000 mon.a (mon.0) 134 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:26:12.377 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: audit 2023-10-01T10:24:50.932637+0000 mon.a (mon.0) 135 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.377 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: audit 2023-10-01T10:24:51.019441+0000 mon.a (mon.0) 136 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:26:12.377 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: audit 2023-10-01T10:24:51.021592+0000 mon.a (mon.0) 137 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:26:12.377 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: audit 2023-10-01T10:24:51.029692+0000 mon.a (mon.0) 138 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:26:12.377 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: audit 2023-10-01T10:24:51.036687+0000 mon.a (mon.0) 139 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.377 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cluster 2023-10-01T10:24:51.306854+0000 mgr.y (mgr.25072) 86 : cluster 0 pgmap v66: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 636 op/s 2023-10-01T10:26:12.377 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cluster 2023-10-01T10:24:53.307323+0000 mgr.y (mgr.25072) 87 : cluster 0 pgmap v67: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 636 op/s 2023-10-01T10:26:12.377 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cluster 2023-10-01T10:24:55.308104+0000 mgr.y (mgr.25072) 88 : cluster 0 pgmap v68: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 950 op/s 2023-10-01T10:26:12.377 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cluster 2023-10-01T10:24:57.308530+0000 mgr.y (mgr.25072) 89 : cluster 0 pgmap v69: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 630 op/s 2023-10-01T10:26:12.378 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cluster 2023-10-01T10:24:59.309238+0000 mgr.y (mgr.25072) 90 : cluster 0 pgmap v70: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 950 op/s 2023-10-01T10:26:12.378 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cluster 2023-10-01T10:25:01.309690+0000 mgr.y (mgr.25072) 91 : cluster 0 pgmap v71: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 633 op/s 2023-10-01T10:26:12.378 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: audit 2023-10-01T10:25:02.383036+0000 mon.a (mon.0) 140 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:26:12.378 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cluster 2023-10-01T10:25:03.310268+0000 mgr.y (mgr.25072) 92 : cluster 0 pgmap v72: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 633 op/s 2023-10-01T10:26:12.378 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cluster 2023-10-01T10:25:05.310995+0000 mgr.y (mgr.25072) 93 : cluster 0 pgmap v73: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 947 op/s 2023-10-01T10:26:12.378 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cluster 2023-10-01T10:25:07.311409+0000 mgr.y (mgr.25072) 94 : cluster 0 pgmap v74: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 633 op/s 2023-10-01T10:26:12.378 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cluster 2023-10-01T10:25:09.312006+0000 mgr.y (mgr.25072) 95 : cluster 0 pgmap v75: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 943 op/s 2023-10-01T10:26:12.378 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cluster 2023-10-01T10:25:11.315486+0000 mgr.y (mgr.25072) 96 : cluster 0 pgmap v76: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 622 op/s 2023-10-01T10:26:12.378 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cluster 2023-10-01T10:25:13.315999+0000 mgr.y (mgr.25072) 97 : cluster 0 pgmap v77: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2023-10-01T10:26:12.378 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cluster 2023-10-01T10:25:15.316770+0000 mgr.y (mgr.25072) 98 : cluster 0 pgmap v78: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 931 op/s 2023-10-01T10:26:12.378 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cluster 2023-10-01T10:25:17.317197+0000 mgr.y (mgr.25072) 99 : cluster 0 pgmap v79: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 617 op/s 2023-10-01T10:26:12.378 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: audit 2023-10-01T10:25:17.383261+0000 mon.a (mon.0) 141 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:26:12.379 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cluster 2023-10-01T10:25:19.317958+0000 mgr.y (mgr.25072) 100 : cluster 0 pgmap v80: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 926 op/s 2023-10-01T10:26:12.379 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cluster 2023-10-01T10:25:21.318401+0000 mgr.y (mgr.25072) 101 : cluster 0 pgmap v81: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 617 op/s 2023-10-01T10:26:12.379 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cluster 2023-10-01T10:25:23.318859+0000 mgr.y (mgr.25072) 102 : cluster 0 pgmap v82: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 617 op/s 2023-10-01T10:26:12.379 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cluster 2023-10-01T10:25:25.319596+0000 mgr.y (mgr.25072) 103 : cluster 0 pgmap v83: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 921 op/s 2023-10-01T10:26:12.379 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cluster 2023-10-01T10:25:27.320098+0000 mgr.y (mgr.25072) 104 : cluster 0 pgmap v84: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 612 op/s 2023-10-01T10:26:12.379 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cluster 2023-10-01T10:25:29.320746+0000 mgr.y (mgr.25072) 105 : cluster 0 pgmap v85: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 914 op/s 2023-10-01T10:26:12.379 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cluster 2023-10-01T10:25:31.321145+0000 mgr.y (mgr.25072) 106 : cluster 0 pgmap v86: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 605 op/s 2023-10-01T10:26:12.379 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: audit 2023-10-01T10:25:32.387554+0000 mon.a (mon.0) 142 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:26:12.379 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cluster 2023-10-01T10:25:33.323839+0000 mgr.y (mgr.25072) 107 : cluster 0 pgmap v87: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 605 op/s 2023-10-01T10:26:12.379 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cluster 2023-10-01T10:25:35.324747+0000 mgr.y (mgr.25072) 108 : cluster 0 pgmap v88: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 905 op/s 2023-10-01T10:26:12.379 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cluster 2023-10-01T10:25:37.325219+0000 mgr.y (mgr.25072) 109 : cluster 0 pgmap v89: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 601 op/s 2023-10-01T10:26:12.379 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cluster 2023-10-01T10:25:39.325969+0000 mgr.y (mgr.25072) 110 : cluster 0 pgmap v90: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 900 op/s 2023-10-01T10:26:12.380 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cluster 2023-10-01T10:25:41.326402+0000 mgr.y (mgr.25072) 111 : cluster 0 pgmap v91: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 599 op/s 2023-10-01T10:26:12.380 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cluster 2023-10-01T10:25:43.326823+0000 mgr.y (mgr.25072) 112 : cluster 0 pgmap v92: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 599 op/s 2023-10-01T10:26:12.380 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cluster 2023-10-01T10:25:45.327697+0000 mgr.y (mgr.25072) 113 : cluster 0 pgmap v93: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 902 op/s 2023-10-01T10:26:12.380 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cluster 2023-10-01T10:25:47.328134+0000 mgr.y (mgr.25072) 114 : cluster 0 pgmap v94: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 602 op/s 2023-10-01T10:26:12.380 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: audit 2023-10-01T10:25:47.384139+0000 mon.a (mon.0) 143 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:26:12.380 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cluster 2023-10-01T10:25:49.328877+0000 mgr.y (mgr.25072) 115 : cluster 0 pgmap v95: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 908 op/s 2023-10-01T10:26:12.380 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: audit 2023-10-01T10:25:51.115625+0000 mon.a (mon.0) 144 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:26:12.380 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cluster 2023-10-01T10:25:51.329319+0000 mgr.y (mgr.25072) 116 : cluster 0 pgmap v96: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 609 op/s 2023-10-01T10:26:12.380 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: audit 2023-10-01T10:25:51.647899+0000 mon.a (mon.0) 145 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.380 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: audit 2023-10-01T10:25:51.688342+0000 mon.a (mon.0) 146 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.380 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cluster 2023-10-01T10:25:53.329746+0000 mgr.y (mgr.25072) 117 : cluster 0 pgmap v97: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 609 op/s 2023-10-01T10:26:12.380 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cluster 2023-10-01T10:25:55.332526+0000 mgr.y (mgr.25072) 118 : cluster 0 pgmap v98: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 898 op/s 2023-10-01T10:26:12.381 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cluster 2023-10-01T10:25:57.332902+0000 mgr.y (mgr.25072) 119 : cluster 0 pgmap v99: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 595 op/s 2023-10-01T10:26:12.381 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: audit 2023-10-01T10:25:57.676339+0000 mon.a (mon.0) 147 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.381 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: audit 2023-10-01T10:25:57.691224+0000 mon.a (mon.0) 148 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.381 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: audit 2023-10-01T10:25:58.744984+0000 mgr.y (mgr.25072) 120 : audit 0 from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:26:12.381 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cephadm 2023-10-01T10:25:58.745431+0000 mgr.y (mgr.25072) 121 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T10:26:12.381 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: audit 2023-10-01T10:25:58.753130+0000 mon.a (mon.0) 149 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.381 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cluster 2023-10-01T10:25:59.333749+0000 mgr.y (mgr.25072) 122 : cluster 0 pgmap v100: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 891 op/s 2023-10-01T10:26:12.381 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: audit 2023-10-01T10:25:59.600092+0000 mon.a (mon.0) 150 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.381 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: audit 2023-10-01T10:25:59.607175+0000 mon.a (mon.0) 151 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.381 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: audit 2023-10-01T10:26:00.245372+0000 mon.a (mon.0) 152 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:26:12.381 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: audit 2023-10-01T10:26:00.246702+0000 mon.a (mon.0) 153 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:26:12.381 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: audit 2023-10-01T10:26:00.256003+0000 mon.a (mon.0) 154 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.381 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: audit 2023-10-01T10:26:00.339374+0000 mon.a (mon.0) 155 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:26:12.382 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: audit 2023-10-01T10:26:00.349038+0000 mon.a (mon.0) 156 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:26:12.382 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: audit 2023-10-01T10:26:00.355427+0000 mon.a (mon.0) 157 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-10-01T10:26:12.382 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: audit 2023-10-01T10:26:00.356600+0000 mon.a (mon.0) 158 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-10-01T10:26:12.382 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cephadm 2023-10-01T10:26:00.357370+0000 mgr.y (mgr.25072) 123 : cephadm 1 Upgrade: It appears safe to stop mon.b 2023-10-01T10:26:12.382 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cluster 2023-10-01T10:26:01.334179+0000 mgr.y (mgr.25072) 124 : cluster 0 pgmap v101: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 585 op/s 2023-10-01T10:26:12.382 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: audit 2023-10-01T10:26:02.384378+0000 mon.a (mon.0) 159 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:26:12.382 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cluster 2023-10-01T10:26:03.334636+0000 mgr.y (mgr.25072) 125 : cluster 0 pgmap v102: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 585 op/s 2023-10-01T10:26:12.382 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cephadm 2023-10-01T10:26:05.246409+0000 mgr.y (mgr.25072) 126 : cephadm 1 Upgrade: Updating mon.b 2023-10-01T10:26:12.382 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: audit 2023-10-01T10:26:05.262780+0000 mon.a (mon.0) 160 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.382 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: audit 2023-10-01T10:26:05.264561+0000 mon.a (mon.0) 161 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T10:26:12.382 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: audit 2023-10-01T10:26:05.266210+0000 mon.a (mon.0) 162 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T10:26:12.383 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: audit 2023-10-01T10:26:05.271384+0000 mon.a (mon.0) 163 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:26:12.383 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cephadm 2023-10-01T10:26:05.273309+0000 mgr.y (mgr.25072) 127 : cephadm 1 Deploying daemon mon.b on smithi173 2023-10-01T10:26:12.383 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cluster 2023-10-01T10:26:05.335349+0000 mgr.y (mgr.25072) 128 : cluster 0 pgmap v103: 89 pgs: 89 active+clean; 2.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2023-10-01T10:26:12.383 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cluster 2023-10-01T10:26:07.335768+0000 mgr.y (mgr.25072) 129 : cluster 0 pgmap v104: 89 pgs: 89 active+clean; 2.4 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 586 op/s 2023-10-01T10:26:12.383 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: audit 2023-10-01T10:26:08.661742+0000 mon.c (mon.2) 2 : audit 0 from='client.? 172.21.15.78:0/3583851419' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:26:12.383 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cluster 2023-10-01T10:26:11.004253+0000 mon.a (mon.0) 170 : cluster 1 overall HEALTH_OK 2023-10-01T10:26:12.383 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: audit 2023-10-01T10:26:11.004576+0000 mon.a (mon.0) 171 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-10-01T10:26:12.383 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cluster 2023-10-01T10:26:11.004907+0000 mon.a (mon.0) 172 : cluster 1 mon.a calling monitor election 2023-10-01T10:26:12.383 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cluster 2023-10-01T10:26:11.004952+0000 mon.c (mon.2) 3 : cluster 1 mon.c calling monitor election 2023-10-01T10:26:12.383 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: audit 2023-10-01T10:26:11.006433+0000 mon.a (mon.0) 173 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-10-01T10:26:12.383 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: audit 2023-10-01T10:26:11.006748+0000 mon.a (mon.0) 174 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-10-01T10:26:12.383 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cluster 2023-10-01T10:26:11.009717+0000 mon.b (mon.1) 2 : cluster 1 mon.b calling monitor election 2023-10-01T10:26:12.383 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cluster 2023-10-01T10:26:11.010564+0000 mon.a (mon.0) 175 : cluster 1 mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-10-01T10:26:12.384 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cluster 2023-10-01T10:26:11.021522+0000 mon.a (mon.0) 176 : cluster 0 monmap e4: 3 mons at {a=[v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0],b=[v2:172.21.15.173:3300/0,v1:172.21.15.173:6789/0],c=[v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0]} removed_ranks: {} 2023-10-01T10:26:12.384 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cluster 2023-10-01T10:26:11.022329+0000 mon.a (mon.0) 177 : cluster 0 fsmap foo:1 {0=foo.smithi078.vjqrth=up:active} 1 up:standby 2023-10-01T10:26:12.384 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cluster 2023-10-01T10:26:11.022378+0000 mon.a (mon.0) 178 : cluster 0 osdmap e77: 8 total, 8 up, 8 in 2023-10-01T10:26:12.384 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cluster 2023-10-01T10:26:11.026052+0000 mon.a (mon.0) 179 : cluster 0 mgrmap e34: y(active, since 3m), standbys: x 2023-10-01T10:26:12.384 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cluster 2023-10-01T10:26:11.034015+0000 mon.a (mon.0) 180 : cluster 1 overall HEALTH_OK 2023-10-01T10:26:12.384 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: audit 2023-10-01T10:26:11.041871+0000 mon.a (mon.0) 181 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.384 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: audit 2023-10-01T10:26:11.053072+0000 mon.a (mon.0) 182 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:12.384 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: audit 2023-10-01T10:26:11.065573+0000 mon.a (mon.0) 183 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:26:12.384 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: cluster 2023-10-01T10:26:11.336939+0000 mgr.y (mgr.25072) 131 : cluster 0 pgmap v106: 89 pgs: 89 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 593 op/s 2023-10-01T10:26:12.384 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:12 smithi173 bash[58816]: audit 2023-10-01T10:26:11.419118+0000 mon.c (mon.2) 4 : audit 0 from='client.? 172.21.15.78:0/2924871869' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-10-01T10:26:12.828 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -e sha1=b56822191ad0416d0c05ed59720df03ad9c2a44e -- bash -c 'ceph orch ps' 2023-10-01T10:26:14.394 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:14 smithi078 bash[75299]: cluster 2023-10-01T10:26:13.337386+0000 mgr.y (mgr.25072) 132 : cluster 0 pgmap v107: 89 pgs: 89 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 593 op/s 2023-10-01T10:26:14.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:14 smithi078 bash[78274]: cluster 2023-10-01T10:26:13.337386+0000 mgr.y (mgr.25072) 132 : cluster 0 pgmap v107: 89 pgs: 89 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 593 op/s 2023-10-01T10:26:14.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:14 smithi173 bash[58816]: cluster 2023-10-01T10:26:13.337386+0000 mgr.y (mgr.25072) 132 : cluster 0 pgmap v107: 89 pgs: 89 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 593 op/s 2023-10-01T10:26:15.833 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:26:15 smithi078 bash[71214]: debug 2023-10-01T10:26:15.326+0000 7fd9c245e700 -1 mgr.server handle_report got status from non-daemon mon.b 2023-10-01T10:26:16.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:16 smithi173 bash[58816]: cluster 2023-10-01T10:26:15.338191+0000 mgr.y (mgr.25072) 133 : cluster 0 pgmap v108: 89 pgs: 89 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 887 op/s 2023-10-01T10:26:16.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:16 smithi078 bash[75299]: cluster 2023-10-01T10:26:15.338191+0000 mgr.y (mgr.25072) 133 : cluster 0 pgmap v108: 89 pgs: 89 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 887 op/s 2023-10-01T10:26:16.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:16 smithi078 bash[78274]: cluster 2023-10-01T10:26:15.338191+0000 mgr.y (mgr.25072) 133 : cluster 0 pgmap v108: 89 pgs: 89 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 887 op/s 2023-10-01T10:26:17.196 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-01T10:26:17.196 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.a smithi078 *:9093,9094 running (6m) - 12m 13.9M - 0.23.0 ba2b418f427c 90745d6cf2c8 2023-10-01T10:26:17.196 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.a smithi173 *:3000 running (5m) - 11m 43.2M - 8.3.5 dad864ee21e9 4874aca3a861 2023-10-01T10:26:17.196 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foo.smithi078.vjqrth smithi078 running (10m) - 10m 17.0M - 16.2.14-44-geeb13ac9 94a6c4498547 aedf3352e433 2023-10-01T10:26:17.196 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foo.smithi173.vrqqoc smithi173 running (10m) - 10m 13.9M - 16.2.14-44-geeb13ac9 94a6c4498547 3815ca08717b 2023-10-01T10:26:17.197 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.x smithi173 *:8443,9283,8765 running (4m) - 19m 427M - 18.2.0-497-gb5682219 a3fc96fed5ac 0e4d4ec3a75c 2023-10-01T10:26:17.197 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.y smithi078 *:8443,9283,8765 running (5m) - 21m 476M - 18.2.0-497-gb5682219 a3fc96fed5ac bdb8227ce5b1 2023-10-01T10:26:17.197 INFO:teuthology.orchestra.run.smithi078.stdout:mon.a smithi078 running (3m) - 21m 65.4M 2048M 18.2.0-497-gb5682219 a3fc96fed5ac 62398b61255b 2023-10-01T10:26:17.197 INFO:teuthology.orchestra.run.smithi078.stdout:mon.b smithi173 starting - - - 2048M 2023-10-01T10:26:17.197 INFO:teuthology.orchestra.run.smithi078.stdout:mon.c smithi078 running (113s) - 19m 39.9M 2048M 18.2.0-497-gb5682219 a3fc96fed5ac 5f02c5b8aecd 2023-10-01T10:26:17.197 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.a smithi078 *:9100 running (6m) - 12m 13.0M - 1.3.1 1dbe0e931976 3c271e279ae2 2023-10-01T10:26:17.197 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.b smithi173 *:9100 running (5m) - 12m 9.80M - 1.3.1 1dbe0e931976 cf13fd39e3de 2023-10-01T10:26:17.197 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (18m) - 18m 418M 2524M 16.2.14-44-geeb13ac9 94a6c4498547 f6a90fa5df1a 2023-10-01T10:26:17.197 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (17m) - 17m 360M 2524M 16.2.14-44-geeb13ac9 94a6c4498547 da5330a1eff3 2023-10-01T10:26:17.197 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (17m) - 17m 516M 2524M 16.2.14-44-geeb13ac9 94a6c4498547 685fab0bfbf6 2023-10-01T10:26:17.197 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (16m) - 16m 508M 2524M 16.2.14-44-geeb13ac9 94a6c4498547 c3823edbc5f1 2023-10-01T10:26:17.198 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi173 running (15m) - 15m 441M 2524M 16.2.14-44-geeb13ac9 94a6c4498547 f6a3e2fc7289 2023-10-01T10:26:17.198 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi173 running (15m) - 15m 479M 2524M 16.2.14-44-geeb13ac9 94a6c4498547 b4ab64b3b760 2023-10-01T10:26:17.198 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi173 running (14m) - 14m 315M 2524M 16.2.14-44-geeb13ac9 94a6c4498547 67631580752d 2023-10-01T10:26:17.198 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi173 running (13m) - 13m 421M 2524M 16.2.14-44-geeb13ac9 94a6c4498547 86157c8ed4b2 2023-10-01T10:26:17.198 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.a smithi173 *:9095 running (4m) - 12m 57.6M - 2.33.4 514e6a882f6e 94c2cb9de105 2023-10-01T10:26:17.724 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:17 smithi078 bash[75299]: audit 2023-10-01T10:26:17.172977+0000 mgr.y (mgr.25072) 134 : audit 0 from='client.34236 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:26:17.725 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:17 smithi078 bash[75299]: cluster 2023-10-01T10:26:17.338672+0000 mgr.y (mgr.25072) 135 : cluster 0 pgmap v109: 89 pgs: 89 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 596 op/s 2023-10-01T10:26:17.725 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:17 smithi078 bash[75299]: audit 2023-10-01T10:26:17.384879+0000 mon.a (mon.0) 184 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:26:17.725 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:17 smithi078 bash[75299]: audit 2023-10-01T10:26:17.597069+0000 mon.a (mon.0) 185 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:17.725 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:17 smithi078 bash[75299]: audit 2023-10-01T10:26:17.609607+0000 mon.a (mon.0) 186 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:18.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:17 smithi078 bash[78274]: audit 2023-10-01T10:26:17.172977+0000 mgr.y (mgr.25072) 134 : audit 0 from='client.34236 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:26:18.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:17 smithi078 bash[78274]: cluster 2023-10-01T10:26:17.338672+0000 mgr.y (mgr.25072) 135 : cluster 0 pgmap v109: 89 pgs: 89 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 596 op/s 2023-10-01T10:26:18.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:17 smithi078 bash[78274]: audit 2023-10-01T10:26:17.384879+0000 mon.a (mon.0) 184 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:26:18.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:17 smithi078 bash[78274]: audit 2023-10-01T10:26:17.597069+0000 mon.a (mon.0) 185 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:18.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:17 smithi078 bash[78274]: audit 2023-10-01T10:26:17.609607+0000 mon.a (mon.0) 186 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:18.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:17 smithi173 bash[58816]: audit 2023-10-01T10:26:17.172977+0000 mgr.y (mgr.25072) 134 : audit 0 from='client.34236 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:26:18.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:17 smithi173 bash[58816]: cluster 2023-10-01T10:26:17.338672+0000 mgr.y (mgr.25072) 135 : cluster 0 pgmap v109: 89 pgs: 89 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 596 op/s 2023-10-01T10:26:18.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:17 smithi173 bash[58816]: audit 2023-10-01T10:26:17.384879+0000 mon.a (mon.0) 184 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:26:18.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:17 smithi173 bash[58816]: audit 2023-10-01T10:26:17.597069+0000 mon.a (mon.0) 185 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:18.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:17 smithi173 bash[58816]: audit 2023-10-01T10:26:17.609607+0000 mon.a (mon.0) 186 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:18.330 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -e sha1=b56822191ad0416d0c05ed59720df03ad9c2a44e -- bash -c 'ceph orch upgrade pause' 2023-10-01T10:26:19.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:19 smithi078 bash[75299]: audit 2023-10-01T10:26:18.571150+0000 mon.a (mon.0) 187 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:19.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:19 smithi078 bash[75299]: audit 2023-10-01T10:26:18.585730+0000 mon.a (mon.0) 188 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:19.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:19 smithi078 bash[75299]: cluster 2023-10-01T10:26:19.339363+0000 mgr.y (mgr.25072) 136 : cluster 0 pgmap v110: 89 pgs: 89 active+clean; 1.8 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 886 op/s 2023-10-01T10:26:19.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:19 smithi078 bash[78274]: audit 2023-10-01T10:26:18.571150+0000 mon.a (mon.0) 187 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:19.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:19 smithi078 bash[78274]: audit 2023-10-01T10:26:18.585730+0000 mon.a (mon.0) 188 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:19.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:19 smithi078 bash[78274]: cluster 2023-10-01T10:26:19.339363+0000 mgr.y (mgr.25072) 136 : cluster 0 pgmap v110: 89 pgs: 89 active+clean; 1.8 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 886 op/s 2023-10-01T10:26:19.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:19 smithi173 bash[58816]: audit 2023-10-01T10:26:18.571150+0000 mon.a (mon.0) 187 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:19.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:19 smithi173 bash[58816]: audit 2023-10-01T10:26:18.585730+0000 mon.a (mon.0) 188 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:19.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:19 smithi173 bash[58816]: cluster 2023-10-01T10:26:19.339363+0000 mgr.y (mgr.25072) 136 : cluster 0 pgmap v110: 89 pgs: 89 active+clean; 1.8 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 886 op/s 2023-10-01T10:26:20.332 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:26:20 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:26:20] "GET /metrics HTTP/1.1" 200 36069 "" "Prometheus/2.33.4" 2023-10-01T10:26:22.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:22 smithi078 bash[75299]: cluster 2023-10-01T10:26:21.339798+0000 mgr.y (mgr.25072) 137 : cluster 0 pgmap v111: 89 pgs: 89 active+clean; 1.8 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 583 op/s 2023-10-01T10:26:22.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:22 smithi078 bash[78274]: cluster 2023-10-01T10:26:21.339798+0000 mgr.y (mgr.25072) 137 : cluster 0 pgmap v111: 89 pgs: 89 active+clean; 1.8 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 583 op/s 2023-10-01T10:26:22.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:22 smithi173 bash[58816]: cluster 2023-10-01T10:26:21.339798+0000 mgr.y (mgr.25072) 137 : cluster 0 pgmap v111: 89 pgs: 89 active+clean; 1.8 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 583 op/s 2023-10-01T10:26:22.916 INFO:teuthology.orchestra.run.smithi078.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T10:26:24.070 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -e sha1=b56822191ad0416d0c05ed59720df03ad9c2a44e -- bash -c 'sleep 60' 2023-10-01T10:26:24.304 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:23 smithi078 bash[75299]: audit 2023-10-01T10:26:22.899835+0000 mgr.y (mgr.25072) 138 : audit 0 from='client.34242 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:26:24.304 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:23 smithi078 bash[75299]: cephadm 2023-10-01T10:26:22.904961+0000 mgr.y (mgr.25072) 139 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T10:26:24.304 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:23 smithi078 bash[75299]: audit 2023-10-01T10:26:22.913444+0000 mon.a (mon.0) 189 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:24.304 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:23 smithi078 bash[75299]: cluster 2023-10-01T10:26:23.340255+0000 mgr.y (mgr.25072) 140 : cluster 0 pgmap v112: 89 pgs: 89 active+clean; 1.8 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 583 op/s 2023-10-01T10:26:24.305 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:23 smithi078 bash[78274]: audit 2023-10-01T10:26:22.899835+0000 mgr.y (mgr.25072) 138 : audit 0 from='client.34242 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:26:24.305 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:23 smithi078 bash[78274]: cephadm 2023-10-01T10:26:22.904961+0000 mgr.y (mgr.25072) 139 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T10:26:24.305 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:23 smithi078 bash[78274]: audit 2023-10-01T10:26:22.913444+0000 mon.a (mon.0) 189 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:24.305 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:23 smithi078 bash[78274]: cluster 2023-10-01T10:26:23.340255+0000 mgr.y (mgr.25072) 140 : cluster 0 pgmap v112: 89 pgs: 89 active+clean; 1.8 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 583 op/s 2023-10-01T10:26:24.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:23 smithi173 bash[58816]: audit 2023-10-01T10:26:22.899835+0000 mgr.y (mgr.25072) 138 : audit 0 from='client.34242 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:26:24.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:23 smithi173 bash[58816]: cephadm 2023-10-01T10:26:22.904961+0000 mgr.y (mgr.25072) 139 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T10:26:24.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:23 smithi173 bash[58816]: audit 2023-10-01T10:26:22.913444+0000 mon.a (mon.0) 189 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:24.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:23 smithi173 bash[58816]: cluster 2023-10-01T10:26:23.340255+0000 mgr.y (mgr.25072) 140 : cluster 0 pgmap v112: 89 pgs: 89 active+clean; 1.8 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 583 op/s 2023-10-01T10:26:26.796 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:26 smithi078 bash[75299]: cluster 2023-10-01T10:26:25.341006+0000 mgr.y (mgr.25072) 141 : cluster 0 pgmap v113: 89 pgs: 89 active+clean; 1.5 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 882 op/s 2023-10-01T10:26:26.796 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:26 smithi078 bash[78274]: cluster 2023-10-01T10:26:25.341006+0000 mgr.y (mgr.25072) 141 : cluster 0 pgmap v113: 89 pgs: 89 active+clean; 1.5 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 882 op/s 2023-10-01T10:26:26.810 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:26 smithi173 bash[58816]: cluster 2023-10-01T10:26:25.341006+0000 mgr.y (mgr.25072) 141 : cluster 0 pgmap v113: 89 pgs: 89 active+clean; 1.5 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 882 op/s 2023-10-01T10:26:28.008 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:27 smithi173 bash[58816]: cluster 2023-10-01T10:26:27.341478+0000 mgr.y (mgr.25072) 142 : cluster 0 pgmap v114: 89 pgs: 89 active+clean; 1.5 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 588 op/s 2023-10-01T10:26:28.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:27 smithi078 bash[78274]: cluster 2023-10-01T10:26:27.341478+0000 mgr.y (mgr.25072) 142 : cluster 0 pgmap v114: 89 pgs: 89 active+clean; 1.5 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 588 op/s 2023-10-01T10:26:28.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:27 smithi078 bash[75299]: cluster 2023-10-01T10:26:27.341478+0000 mgr.y (mgr.25072) 142 : cluster 0 pgmap v114: 89 pgs: 89 active+clean; 1.5 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 588 op/s 2023-10-01T10:26:30.534 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:30 smithi078 bash[78274]: cluster 2023-10-01T10:26:29.342217+0000 mgr.y (mgr.25072) 143 : cluster 0 pgmap v115: 89 pgs: 89 active+clean; 1.3 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 890 op/s 2023-10-01T10:26:30.534 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:30 smithi078 bash[78274]: audit 2023-10-01T10:26:30.014923+0000 mon.a (mon.0) 190 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:30.534 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:30 smithi078 bash[78274]: audit 2023-10-01T10:26:30.025587+0000 mon.a (mon.0) 191 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:30.534 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:30 smithi078 bash[78274]: audit 2023-10-01T10:26:30.027240+0000 mon.a (mon.0) 192 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:26:30.534 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:30 smithi078 bash[78274]: audit 2023-10-01T10:26:30.031477+0000 mon.a (mon.0) 193 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:26:30.534 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:30 smithi078 bash[78274]: audit 2023-10-01T10:26:30.040981+0000 mon.a (mon.0) 194 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:30.534 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:30 smithi078 bash[78274]: audit 2023-10-01T10:26:30.116803+0000 mon.a (mon.0) 195 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T10:26:30.534 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:30 smithi078 bash[78274]: audit 2023-10-01T10:26:30.119856+0000 mon.a (mon.0) 196 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T10:26:30.534 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:30 smithi078 bash[78274]: audit 2023-10-01T10:26:30.120977+0000 mon.a (mon.0) 197 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:26:30.535 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:30 smithi078 bash[75299]: cluster 2023-10-01T10:26:29.342217+0000 mgr.y (mgr.25072) 143 : cluster 0 pgmap v115: 89 pgs: 89 active+clean; 1.3 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 890 op/s 2023-10-01T10:26:30.535 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:30 smithi078 bash[75299]: audit 2023-10-01T10:26:30.014923+0000 mon.a (mon.0) 190 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:30.535 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:30 smithi078 bash[75299]: audit 2023-10-01T10:26:30.025587+0000 mon.a (mon.0) 191 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:30.535 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:30 smithi078 bash[75299]: audit 2023-10-01T10:26:30.027240+0000 mon.a (mon.0) 192 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:26:30.535 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:30 smithi078 bash[75299]: audit 2023-10-01T10:26:30.031477+0000 mon.a (mon.0) 193 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:26:30.535 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:30 smithi078 bash[75299]: audit 2023-10-01T10:26:30.040981+0000 mon.a (mon.0) 194 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:30.535 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:30 smithi078 bash[75299]: audit 2023-10-01T10:26:30.116803+0000 mon.a (mon.0) 195 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T10:26:30.535 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:30 smithi078 bash[75299]: audit 2023-10-01T10:26:30.119856+0000 mon.a (mon.0) 196 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T10:26:30.535 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:30 smithi078 bash[75299]: audit 2023-10-01T10:26:30.120977+0000 mon.a (mon.0) 197 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:26:30.536 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:26:30 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:26:30] "GET /metrics HTTP/1.1" 200 36069 "" "Prometheus/2.33.4" 2023-10-01T10:26:30.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:30 smithi173 bash[58816]: cluster 2023-10-01T10:26:29.342217+0000 mgr.y (mgr.25072) 143 : cluster 0 pgmap v115: 89 pgs: 89 active+clean; 1.3 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 890 op/s 2023-10-01T10:26:30.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:30 smithi173 bash[58816]: audit 2023-10-01T10:26:30.014923+0000 mon.a (mon.0) 190 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:30.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:30 smithi173 bash[58816]: audit 2023-10-01T10:26:30.025587+0000 mon.a (mon.0) 191 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:30.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:30 smithi173 bash[58816]: audit 2023-10-01T10:26:30.027240+0000 mon.a (mon.0) 192 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:26:30.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:30 smithi173 bash[58816]: audit 2023-10-01T10:26:30.031477+0000 mon.a (mon.0) 193 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:26:30.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:30 smithi173 bash[58816]: audit 2023-10-01T10:26:30.040981+0000 mon.a (mon.0) 194 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:30.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:30 smithi173 bash[58816]: audit 2023-10-01T10:26:30.116803+0000 mon.a (mon.0) 195 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T10:26:30.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:30 smithi173 bash[58816]: audit 2023-10-01T10:26:30.119856+0000 mon.a (mon.0) 196 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T10:26:30.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:30 smithi173 bash[58816]: audit 2023-10-01T10:26:30.120977+0000 mon.a (mon.0) 197 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:26:31.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:31 smithi078 bash[78274]: cephadm 2023-10-01T10:26:30.116197+0000 mgr.y (mgr.25072) 144 : cephadm 1 Reconfiguring mon.a (monmap changed)... 2023-10-01T10:26:31.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:31 smithi078 bash[78274]: cephadm 2023-10-01T10:26:30.122146+0000 mgr.y (mgr.25072) 145 : cephadm 1 Reconfiguring daemon mon.a on smithi078 2023-10-01T10:26:31.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:31 smithi078 bash[75299]: cephadm 2023-10-01T10:26:30.116197+0000 mgr.y (mgr.25072) 144 : cephadm 1 Reconfiguring mon.a (monmap changed)... 2023-10-01T10:26:31.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:31 smithi078 bash[75299]: cephadm 2023-10-01T10:26:30.122146+0000 mgr.y (mgr.25072) 145 : cephadm 1 Reconfiguring daemon mon.a on smithi078 2023-10-01T10:26:31.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:31 smithi173 bash[58816]: cephadm 2023-10-01T10:26:30.116197+0000 mgr.y (mgr.25072) 144 : cephadm 1 Reconfiguring mon.a (monmap changed)... 2023-10-01T10:26:31.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:31 smithi173 bash[58816]: cephadm 2023-10-01T10:26:30.122146+0000 mgr.y (mgr.25072) 145 : cephadm 1 Reconfiguring daemon mon.a on smithi078 2023-10-01T10:26:32.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:32 smithi078 bash[78274]: cluster 2023-10-01T10:26:31.342650+0000 mgr.y (mgr.25072) 146 : cluster 0 pgmap v116: 89 pgs: 89 active+clean; 1.3 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2023-10-01T10:26:32.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:32 smithi078 bash[78274]: audit 2023-10-01T10:26:32.385175+0000 mon.a (mon.0) 198 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:26:32.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:32 smithi078 bash[75299]: cluster 2023-10-01T10:26:31.342650+0000 mgr.y (mgr.25072) 146 : cluster 0 pgmap v116: 89 pgs: 89 active+clean; 1.3 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2023-10-01T10:26:32.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:32 smithi078 bash[75299]: audit 2023-10-01T10:26:32.385175+0000 mon.a (mon.0) 198 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:26:32.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:32 smithi173 bash[58816]: cluster 2023-10-01T10:26:31.342650+0000 mgr.y (mgr.25072) 146 : cluster 0 pgmap v116: 89 pgs: 89 active+clean; 1.3 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2023-10-01T10:26:32.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:32 smithi173 bash[58816]: audit 2023-10-01T10:26:32.385175+0000 mon.a (mon.0) 198 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:26:34.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:34 smithi173 bash[58816]: cluster 2023-10-01T10:26:33.343026+0000 mgr.y (mgr.25072) 147 : cluster 0 pgmap v117: 89 pgs: 89 active+clean; 1.3 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2023-10-01T10:26:34.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:34 smithi173 bash[58816]: audit 2023-10-01T10:26:33.349001+0000 mon.a (mon.0) 199 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:34.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:34 smithi173 bash[58816]: audit 2023-10-01T10:26:33.356670+0000 mon.a (mon.0) 200 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:34.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:34 smithi173 bash[58816]: cephadm 2023-10-01T10:26:33.400927+0000 mgr.y (mgr.25072) 148 : cephadm 1 Reconfiguring mgr.y (monmap changed)... 2023-10-01T10:26:34.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:34 smithi173 bash[58816]: audit 2023-10-01T10:26:33.401692+0000 mon.a (mon.0) 201 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-01T10:26:34.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:34 smithi173 bash[58816]: audit 2023-10-01T10:26:33.403904+0000 mon.a (mon.0) 202 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-01T10:26:34.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:34 smithi173 bash[58816]: audit 2023-10-01T10:26:33.405149+0000 mon.a (mon.0) 203 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:26:34.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:34 smithi173 bash[58816]: cephadm 2023-10-01T10:26:33.406496+0000 mgr.y (mgr.25072) 149 : cephadm 1 Reconfiguring daemon mgr.y on smithi078 2023-10-01T10:26:34.788 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:34 smithi078 bash[78274]: cluster 2023-10-01T10:26:33.343026+0000 mgr.y (mgr.25072) 147 : cluster 0 pgmap v117: 89 pgs: 89 active+clean; 1.3 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2023-10-01T10:26:34.789 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:34 smithi078 bash[78274]: audit 2023-10-01T10:26:33.349001+0000 mon.a (mon.0) 199 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:34.789 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:34 smithi078 bash[78274]: audit 2023-10-01T10:26:33.356670+0000 mon.a (mon.0) 200 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:34.789 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:34 smithi078 bash[78274]: cephadm 2023-10-01T10:26:33.400927+0000 mgr.y (mgr.25072) 148 : cephadm 1 Reconfiguring mgr.y (monmap changed)... 2023-10-01T10:26:34.789 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:34 smithi078 bash[78274]: audit 2023-10-01T10:26:33.401692+0000 mon.a (mon.0) 201 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-01T10:26:34.789 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:34 smithi078 bash[78274]: audit 2023-10-01T10:26:33.403904+0000 mon.a (mon.0) 202 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-01T10:26:34.789 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:34 smithi078 bash[78274]: audit 2023-10-01T10:26:33.405149+0000 mon.a (mon.0) 203 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:26:34.789 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:34 smithi078 bash[78274]: cephadm 2023-10-01T10:26:33.406496+0000 mgr.y (mgr.25072) 149 : cephadm 1 Reconfiguring daemon mgr.y on smithi078 2023-10-01T10:26:34.790 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:34 smithi078 bash[75299]: cluster 2023-10-01T10:26:33.343026+0000 mgr.y (mgr.25072) 147 : cluster 0 pgmap v117: 89 pgs: 89 active+clean; 1.3 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2023-10-01T10:26:34.790 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:34 smithi078 bash[75299]: audit 2023-10-01T10:26:33.349001+0000 mon.a (mon.0) 199 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:34.790 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:34 smithi078 bash[75299]: audit 2023-10-01T10:26:33.356670+0000 mon.a (mon.0) 200 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:34.790 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:34 smithi078 bash[75299]: cephadm 2023-10-01T10:26:33.400927+0000 mgr.y (mgr.25072) 148 : cephadm 1 Reconfiguring mgr.y (monmap changed)... 2023-10-01T10:26:34.790 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:34 smithi078 bash[75299]: audit 2023-10-01T10:26:33.401692+0000 mon.a (mon.0) 201 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-01T10:26:34.790 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:34 smithi078 bash[75299]: audit 2023-10-01T10:26:33.403904+0000 mon.a (mon.0) 202 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-01T10:26:34.790 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:34 smithi078 bash[75299]: audit 2023-10-01T10:26:33.405149+0000 mon.a (mon.0) 203 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:26:34.791 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:34 smithi078 bash[75299]: cephadm 2023-10-01T10:26:33.406496+0000 mgr.y (mgr.25072) 149 : cephadm 1 Reconfiguring daemon mgr.y on smithi078 2023-10-01T10:26:36.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:36 smithi078 bash[78274]: cluster 2023-10-01T10:26:35.343784+0000 mgr.y (mgr.25072) 150 : cluster 0 pgmap v118: 89 pgs: 89 active+clean; 1.1 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 896 op/s 2023-10-01T10:26:36.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:36 smithi078 bash[78274]: audit 2023-10-01T10:26:35.897614+0000 mon.a (mon.0) 204 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:36.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:36 smithi078 bash[78274]: audit 2023-10-01T10:26:35.910811+0000 mon.a (mon.0) 205 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:36.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:36 smithi078 bash[78274]: audit 2023-10-01T10:26:35.927198+0000 mon.a (mon.0) 206 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T10:26:36.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:36 smithi078 bash[78274]: audit 2023-10-01T10:26:35.928281+0000 mon.a (mon.0) 207 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T10:26:36.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:36 smithi078 bash[78274]: audit 2023-10-01T10:26:35.929030+0000 mon.a (mon.0) 208 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:26:36.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:36 smithi078 bash[75299]: cluster 2023-10-01T10:26:35.343784+0000 mgr.y (mgr.25072) 150 : cluster 0 pgmap v118: 89 pgs: 89 active+clean; 1.1 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 896 op/s 2023-10-01T10:26:36.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:36 smithi078 bash[75299]: audit 2023-10-01T10:26:35.897614+0000 mon.a (mon.0) 204 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:36.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:36 smithi078 bash[75299]: audit 2023-10-01T10:26:35.910811+0000 mon.a (mon.0) 205 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:36.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:36 smithi078 bash[75299]: audit 2023-10-01T10:26:35.927198+0000 mon.a (mon.0) 206 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T10:26:36.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:36 smithi078 bash[75299]: audit 2023-10-01T10:26:35.928281+0000 mon.a (mon.0) 207 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T10:26:36.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:36 smithi078 bash[75299]: audit 2023-10-01T10:26:35.929030+0000 mon.a (mon.0) 208 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:26:36.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:36 smithi173 bash[58816]: cluster 2023-10-01T10:26:35.343784+0000 mgr.y (mgr.25072) 150 : cluster 0 pgmap v118: 89 pgs: 89 active+clean; 1.1 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 896 op/s 2023-10-01T10:26:36.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:36 smithi173 bash[58816]: audit 2023-10-01T10:26:35.897614+0000 mon.a (mon.0) 204 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:36.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:36 smithi173 bash[58816]: audit 2023-10-01T10:26:35.910811+0000 mon.a (mon.0) 205 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:36.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:36 smithi173 bash[58816]: audit 2023-10-01T10:26:35.927198+0000 mon.a (mon.0) 206 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T10:26:36.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:36 smithi173 bash[58816]: audit 2023-10-01T10:26:35.928281+0000 mon.a (mon.0) 207 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T10:26:36.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:36 smithi173 bash[58816]: audit 2023-10-01T10:26:35.929030+0000 mon.a (mon.0) 208 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:26:37.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:37 smithi173 bash[58816]: cephadm 2023-10-01T10:26:35.926707+0000 mgr.y (mgr.25072) 151 : cephadm 1 Reconfiguring mon.c (monmap changed)... 2023-10-01T10:26:37.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:37 smithi173 bash[58816]: cephadm 2023-10-01T10:26:35.930421+0000 mgr.y (mgr.25072) 152 : cephadm 1 Reconfiguring daemon mon.c on smithi078 2023-10-01T10:26:37.750 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:37 smithi078 bash[78274]: cephadm 2023-10-01T10:26:35.926707+0000 mgr.y (mgr.25072) 151 : cephadm 1 Reconfiguring mon.c (monmap changed)... 2023-10-01T10:26:37.751 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:37 smithi078 bash[78274]: cephadm 2023-10-01T10:26:35.930421+0000 mgr.y (mgr.25072) 152 : cephadm 1 Reconfiguring daemon mon.c on smithi078 2023-10-01T10:26:37.751 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:37 smithi078 bash[75299]: cephadm 2023-10-01T10:26:35.926707+0000 mgr.y (mgr.25072) 151 : cephadm 1 Reconfiguring mon.c (monmap changed)... 2023-10-01T10:26:37.751 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:37 smithi078 bash[75299]: cephadm 2023-10-01T10:26:35.930421+0000 mgr.y (mgr.25072) 152 : cephadm 1 Reconfiguring daemon mon.c on smithi078 2023-10-01T10:26:38.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:38 smithi173 bash[58816]: cluster 2023-10-01T10:26:37.344178+0000 mgr.y (mgr.25072) 153 : cluster 0 pgmap v119: 89 pgs: 89 active+clean; 1.1 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 597 op/s 2023-10-01T10:26:38.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:38 smithi078 bash[78274]: cluster 2023-10-01T10:26:37.344178+0000 mgr.y (mgr.25072) 153 : cluster 0 pgmap v119: 89 pgs: 89 active+clean; 1.1 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 597 op/s 2023-10-01T10:26:38.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:38 smithi078 bash[75299]: cluster 2023-10-01T10:26:37.344178+0000 mgr.y (mgr.25072) 153 : cluster 0 pgmap v119: 89 pgs: 89 active+clean; 1.1 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 597 op/s 2023-10-01T10:26:39.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:39 smithi078 bash[78274]: audit 2023-10-01T10:26:38.501783+0000 mon.a (mon.0) 209 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:39.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:39 smithi078 bash[78274]: audit 2023-10-01T10:26:38.511445+0000 mon.a (mon.0) 210 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:39.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:39 smithi078 bash[78274]: cephadm 2023-10-01T10:26:38.558754+0000 mgr.y (mgr.25072) 154 : cephadm 1 Reconfiguring osd.0 (monmap changed)... 2023-10-01T10:26:39.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:39 smithi078 bash[78274]: audit 2023-10-01T10:26:38.559211+0000 mon.a (mon.0) 211 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-01T10:26:39.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:39 smithi078 bash[78274]: audit 2023-10-01T10:26:38.560374+0000 mon.a (mon.0) 212 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:26:39.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:39 smithi078 bash[78274]: cephadm 2023-10-01T10:26:38.563637+0000 mgr.y (mgr.25072) 155 : cephadm 1 Reconfiguring daemon osd.0 on smithi078 2023-10-01T10:26:39.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:39 smithi078 bash[78274]: cluster 2023-10-01T10:26:39.345861+0000 mgr.y (mgr.25072) 156 : cluster 0 pgmap v120: 89 pgs: 89 active+clean; 891 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 903 op/s 2023-10-01T10:26:39.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:39 smithi078 bash[75299]: audit 2023-10-01T10:26:38.501783+0000 mon.a (mon.0) 209 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:39.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:39 smithi078 bash[75299]: audit 2023-10-01T10:26:38.511445+0000 mon.a (mon.0) 210 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:39.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:39 smithi078 bash[75299]: cephadm 2023-10-01T10:26:38.558754+0000 mgr.y (mgr.25072) 154 : cephadm 1 Reconfiguring osd.0 (monmap changed)... 2023-10-01T10:26:39.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:39 smithi078 bash[75299]: audit 2023-10-01T10:26:38.559211+0000 mon.a (mon.0) 211 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-01T10:26:39.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:39 smithi078 bash[75299]: audit 2023-10-01T10:26:38.560374+0000 mon.a (mon.0) 212 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:26:39.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:39 smithi078 bash[75299]: cephadm 2023-10-01T10:26:38.563637+0000 mgr.y (mgr.25072) 155 : cephadm 1 Reconfiguring daemon osd.0 on smithi078 2023-10-01T10:26:39.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:39 smithi078 bash[75299]: cluster 2023-10-01T10:26:39.345861+0000 mgr.y (mgr.25072) 156 : cluster 0 pgmap v120: 89 pgs: 89 active+clean; 891 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 903 op/s 2023-10-01T10:26:39.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:39 smithi173 bash[58816]: audit 2023-10-01T10:26:38.501783+0000 mon.a (mon.0) 209 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:39.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:39 smithi173 bash[58816]: audit 2023-10-01T10:26:38.511445+0000 mon.a (mon.0) 210 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:39.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:39 smithi173 bash[58816]: cephadm 2023-10-01T10:26:38.558754+0000 mgr.y (mgr.25072) 154 : cephadm 1 Reconfiguring osd.0 (monmap changed)... 2023-10-01T10:26:39.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:39 smithi173 bash[58816]: audit 2023-10-01T10:26:38.559211+0000 mon.a (mon.0) 211 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-01T10:26:39.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:39 smithi173 bash[58816]: audit 2023-10-01T10:26:38.560374+0000 mon.a (mon.0) 212 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:26:39.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:39 smithi173 bash[58816]: cephadm 2023-10-01T10:26:38.563637+0000 mgr.y (mgr.25072) 155 : cephadm 1 Reconfiguring daemon osd.0 on smithi078 2023-10-01T10:26:39.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:39 smithi173 bash[58816]: cluster 2023-10-01T10:26:39.345861+0000 mgr.y (mgr.25072) 156 : cluster 0 pgmap v120: 89 pgs: 89 active+clean; 891 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 903 op/s 2023-10-01T10:26:40.514 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:26:40 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:26:40] "GET /metrics HTTP/1.1" 200 36066 "" "Prometheus/2.33.4" 2023-10-01T10:26:42.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:42 smithi078 bash[75299]: audit 2023-10-01T10:26:41.310355+0000 mon.a (mon.0) 213 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:42.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:42 smithi078 bash[75299]: audit 2023-10-01T10:26:41.322004+0000 mon.a (mon.0) 214 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:42.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:42 smithi078 bash[75299]: cephadm 2023-10-01T10:26:41.344776+0000 mgr.y (mgr.25072) 157 : cephadm 1 Reconfiguring osd.1 (monmap changed)... 2023-10-01T10:26:42.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:42 smithi078 bash[75299]: audit 2023-10-01T10:26:41.345252+0000 mon.a (mon.0) 215 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-01T10:26:42.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:42 smithi078 bash[75299]: cluster 2023-10-01T10:26:41.346246+0000 mgr.y (mgr.25072) 158 : cluster 0 pgmap v121: 89 pgs: 89 active+clean; 891 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 601 op/s 2023-10-01T10:26:42.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:42 smithi078 bash[75299]: audit 2023-10-01T10:26:41.346290+0000 mon.a (mon.0) 216 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:26:42.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:42 smithi078 bash[75299]: cephadm 2023-10-01T10:26:41.349378+0000 mgr.y (mgr.25072) 159 : cephadm 1 Reconfiguring daemon osd.1 on smithi078 2023-10-01T10:26:42.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:42 smithi078 bash[78274]: audit 2023-10-01T10:26:41.310355+0000 mon.a (mon.0) 213 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:42.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:42 smithi078 bash[78274]: audit 2023-10-01T10:26:41.322004+0000 mon.a (mon.0) 214 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:42.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:42 smithi078 bash[78274]: cephadm 2023-10-01T10:26:41.344776+0000 mgr.y (mgr.25072) 157 : cephadm 1 Reconfiguring osd.1 (monmap changed)... 2023-10-01T10:26:42.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:42 smithi078 bash[78274]: audit 2023-10-01T10:26:41.345252+0000 mon.a (mon.0) 215 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-01T10:26:42.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:42 smithi078 bash[78274]: cluster 2023-10-01T10:26:41.346246+0000 mgr.y (mgr.25072) 158 : cluster 0 pgmap v121: 89 pgs: 89 active+clean; 891 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 601 op/s 2023-10-01T10:26:42.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:42 smithi078 bash[78274]: audit 2023-10-01T10:26:41.346290+0000 mon.a (mon.0) 216 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:26:42.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:42 smithi078 bash[78274]: cephadm 2023-10-01T10:26:41.349378+0000 mgr.y (mgr.25072) 159 : cephadm 1 Reconfiguring daemon osd.1 on smithi078 2023-10-01T10:26:42.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:42 smithi173 bash[58816]: audit 2023-10-01T10:26:41.310355+0000 mon.a (mon.0) 213 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:42.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:42 smithi173 bash[58816]: audit 2023-10-01T10:26:41.322004+0000 mon.a (mon.0) 214 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:42.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:42 smithi173 bash[58816]: cephadm 2023-10-01T10:26:41.344776+0000 mgr.y (mgr.25072) 157 : cephadm 1 Reconfiguring osd.1 (monmap changed)... 2023-10-01T10:26:42.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:42 smithi173 bash[58816]: audit 2023-10-01T10:26:41.345252+0000 mon.a (mon.0) 215 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-01T10:26:42.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:42 smithi173 bash[58816]: cluster 2023-10-01T10:26:41.346246+0000 mgr.y (mgr.25072) 158 : cluster 0 pgmap v121: 89 pgs: 89 active+clean; 891 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 601 op/s 2023-10-01T10:26:42.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:42 smithi173 bash[58816]: audit 2023-10-01T10:26:41.346290+0000 mon.a (mon.0) 216 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:26:42.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:42 smithi173 bash[58816]: cephadm 2023-10-01T10:26:41.349378+0000 mgr.y (mgr.25072) 159 : cephadm 1 Reconfiguring daemon osd.1 on smithi078 2023-10-01T10:26:44.780 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:44 smithi078 bash[75299]: cluster 2023-10-01T10:26:43.346675+0000 mgr.y (mgr.25072) 160 : cluster 0 pgmap v122: 89 pgs: 89 active+clean; 891 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 601 op/s 2023-10-01T10:26:44.781 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:44 smithi078 bash[75299]: audit 2023-10-01T10:26:43.921350+0000 mon.a (mon.0) 217 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:44.781 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:44 smithi078 bash[75299]: audit 2023-10-01T10:26:43.934997+0000 mon.a (mon.0) 218 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:44.781 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:44 smithi078 bash[75299]: audit 2023-10-01T10:26:43.981579+0000 mon.a (mon.0) 219 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-01T10:26:44.781 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:44 smithi078 bash[75299]: audit 2023-10-01T10:26:43.983167+0000 mon.a (mon.0) 220 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:26:44.781 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:44 smithi078 bash[78274]: cluster 2023-10-01T10:26:43.346675+0000 mgr.y (mgr.25072) 160 : cluster 0 pgmap v122: 89 pgs: 89 active+clean; 891 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 601 op/s 2023-10-01T10:26:44.781 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:44 smithi078 bash[78274]: audit 2023-10-01T10:26:43.921350+0000 mon.a (mon.0) 217 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:44.781 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:44 smithi078 bash[78274]: audit 2023-10-01T10:26:43.934997+0000 mon.a (mon.0) 218 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:44.781 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:44 smithi078 bash[78274]: audit 2023-10-01T10:26:43.981579+0000 mon.a (mon.0) 219 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-01T10:26:44.782 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:44 smithi078 bash[78274]: audit 2023-10-01T10:26:43.983167+0000 mon.a (mon.0) 220 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:26:44.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:44 smithi173 bash[58816]: cluster 2023-10-01T10:26:43.346675+0000 mgr.y (mgr.25072) 160 : cluster 0 pgmap v122: 89 pgs: 89 active+clean; 891 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 601 op/s 2023-10-01T10:26:44.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:44 smithi173 bash[58816]: audit 2023-10-01T10:26:43.921350+0000 mon.a (mon.0) 217 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:44.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:44 smithi173 bash[58816]: audit 2023-10-01T10:26:43.934997+0000 mon.a (mon.0) 218 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:44.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:44 smithi173 bash[58816]: audit 2023-10-01T10:26:43.981579+0000 mon.a (mon.0) 219 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-01T10:26:44.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:44 smithi173 bash[58816]: audit 2023-10-01T10:26:43.983167+0000 mon.a (mon.0) 220 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:26:45.670 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:45 smithi078 bash[75299]: cephadm 2023-10-01T10:26:43.979545+0000 mgr.y (mgr.25072) 161 : cephadm 1 Reconfiguring osd.2 (monmap changed)... 2023-10-01T10:26:45.670 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:45 smithi078 bash[75299]: cephadm 2023-10-01T10:26:43.986220+0000 mgr.y (mgr.25072) 162 : cephadm 1 Reconfiguring daemon osd.2 on smithi078 2023-10-01T10:26:45.670 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:45 smithi078 bash[78274]: cephadm 2023-10-01T10:26:43.979545+0000 mgr.y (mgr.25072) 161 : cephadm 1 Reconfiguring osd.2 (monmap changed)... 2023-10-01T10:26:45.671 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:45 smithi078 bash[78274]: cephadm 2023-10-01T10:26:43.986220+0000 mgr.y (mgr.25072) 162 : cephadm 1 Reconfiguring daemon osd.2 on smithi078 2023-10-01T10:26:45.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:45 smithi173 bash[58816]: cephadm 2023-10-01T10:26:43.979545+0000 mgr.y (mgr.25072) 161 : cephadm 1 Reconfiguring osd.2 (monmap changed)... 2023-10-01T10:26:45.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:45 smithi173 bash[58816]: cephadm 2023-10-01T10:26:43.986220+0000 mgr.y (mgr.25072) 162 : cephadm 1 Reconfiguring daemon osd.2 on smithi078 2023-10-01T10:26:46.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:46 smithi078 bash[75299]: cluster 2023-10-01T10:26:45.347384+0000 mgr.y (mgr.25072) 163 : cluster 0 pgmap v123: 89 pgs: 89 active+clean; 665 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 902 op/s 2023-10-01T10:26:46.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:46 smithi078 bash[78274]: cluster 2023-10-01T10:26:45.347384+0000 mgr.y (mgr.25072) 163 : cluster 0 pgmap v123: 89 pgs: 89 active+clean; 665 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 902 op/s 2023-10-01T10:26:46.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:46 smithi173 bash[58816]: cluster 2023-10-01T10:26:45.347384+0000 mgr.y (mgr.25072) 163 : cluster 0 pgmap v123: 89 pgs: 89 active+clean; 665 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 902 op/s 2023-10-01T10:26:47.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:47 smithi078 bash[78274]: audit 2023-10-01T10:26:46.674338+0000 mon.a (mon.0) 221 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:47.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:47 smithi078 bash[78274]: audit 2023-10-01T10:26:46.681842+0000 mon.a (mon.0) 222 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:47.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:47 smithi078 bash[78274]: cephadm 2023-10-01T10:26:46.711656+0000 mgr.y (mgr.25072) 164 : cephadm 1 Reconfiguring osd.3 (monmap changed)... 2023-10-01T10:26:47.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:47 smithi078 bash[78274]: audit 2023-10-01T10:26:46.712079+0000 mon.a (mon.0) 223 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-01T10:26:47.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:47 smithi078 bash[78274]: audit 2023-10-01T10:26:46.713008+0000 mon.a (mon.0) 224 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:26:47.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:47 smithi078 bash[78274]: cephadm 2023-10-01T10:26:46.715308+0000 mgr.y (mgr.25072) 165 : cephadm 1 Reconfiguring daemon osd.3 on smithi078 2023-10-01T10:26:47.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:47 smithi078 bash[78274]: cluster 2023-10-01T10:26:47.347794+0000 mgr.y (mgr.25072) 166 : cluster 0 pgmap v124: 89 pgs: 89 active+clean; 665 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 606 op/s 2023-10-01T10:26:47.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:47 smithi078 bash[78274]: audit 2023-10-01T10:26:47.385314+0000 mon.a (mon.0) 225 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:26:47.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:47 smithi078 bash[75299]: audit 2023-10-01T10:26:46.674338+0000 mon.a (mon.0) 221 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:47.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:47 smithi078 bash[75299]: audit 2023-10-01T10:26:46.681842+0000 mon.a (mon.0) 222 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:47.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:47 smithi078 bash[75299]: cephadm 2023-10-01T10:26:46.711656+0000 mgr.y (mgr.25072) 164 : cephadm 1 Reconfiguring osd.3 (monmap changed)... 2023-10-01T10:26:47.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:47 smithi078 bash[75299]: audit 2023-10-01T10:26:46.712079+0000 mon.a (mon.0) 223 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-01T10:26:47.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:47 smithi078 bash[75299]: audit 2023-10-01T10:26:46.713008+0000 mon.a (mon.0) 224 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:26:47.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:47 smithi078 bash[75299]: cephadm 2023-10-01T10:26:46.715308+0000 mgr.y (mgr.25072) 165 : cephadm 1 Reconfiguring daemon osd.3 on smithi078 2023-10-01T10:26:47.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:47 smithi078 bash[75299]: cluster 2023-10-01T10:26:47.347794+0000 mgr.y (mgr.25072) 166 : cluster 0 pgmap v124: 89 pgs: 89 active+clean; 665 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 606 op/s 2023-10-01T10:26:47.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:47 smithi078 bash[75299]: audit 2023-10-01T10:26:47.385314+0000 mon.a (mon.0) 225 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:26:48.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:47 smithi173 bash[58816]: audit 2023-10-01T10:26:46.674338+0000 mon.a (mon.0) 221 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:48.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:47 smithi173 bash[58816]: audit 2023-10-01T10:26:46.681842+0000 mon.a (mon.0) 222 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:48.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:47 smithi173 bash[58816]: cephadm 2023-10-01T10:26:46.711656+0000 mgr.y (mgr.25072) 164 : cephadm 1 Reconfiguring osd.3 (monmap changed)... 2023-10-01T10:26:48.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:47 smithi173 bash[58816]: audit 2023-10-01T10:26:46.712079+0000 mon.a (mon.0) 223 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-01T10:26:48.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:47 smithi173 bash[58816]: audit 2023-10-01T10:26:46.713008+0000 mon.a (mon.0) 224 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:26:48.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:47 smithi173 bash[58816]: cephadm 2023-10-01T10:26:46.715308+0000 mgr.y (mgr.25072) 165 : cephadm 1 Reconfiguring daemon osd.3 on smithi078 2023-10-01T10:26:48.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:47 smithi173 bash[58816]: cluster 2023-10-01T10:26:47.347794+0000 mgr.y (mgr.25072) 166 : cluster 0 pgmap v124: 89 pgs: 89 active+clean; 665 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 606 op/s 2023-10-01T10:26:48.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:47 smithi173 bash[58816]: audit 2023-10-01T10:26:47.385314+0000 mon.a (mon.0) 225 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:26:50.269 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:26:50 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:26:50] "GET /metrics HTTP/1.1" 200 36065 "" "Prometheus/2.33.4" 2023-10-01T10:26:50.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:50 smithi078 bash[78274]: cluster 2023-10-01T10:26:49.348516+0000 mgr.y (mgr.25072) 167 : cluster 0 pgmap v125: 89 pgs: 89 active+clean; 439 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 908 op/s 2023-10-01T10:26:50.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:50 smithi078 bash[78274]: audit 2023-10-01T10:26:49.560469+0000 mon.a (mon.0) 226 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:50.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:50 smithi078 bash[78274]: audit 2023-10-01T10:26:49.570972+0000 mon.a (mon.0) 227 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:50.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:50 smithi078 bash[78274]: audit 2023-10-01T10:26:49.600452+0000 mon.a (mon.0) 228 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi078.vjqrth", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-01T10:26:50.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:50 smithi078 bash[78274]: audit 2023-10-01T10:26:49.602077+0000 mon.a (mon.0) 229 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:26:50.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:50 smithi078 bash[75299]: cluster 2023-10-01T10:26:49.348516+0000 mgr.y (mgr.25072) 167 : cluster 0 pgmap v125: 89 pgs: 89 active+clean; 439 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 908 op/s 2023-10-01T10:26:50.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:50 smithi078 bash[75299]: audit 2023-10-01T10:26:49.560469+0000 mon.a (mon.0) 226 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:50.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:50 smithi078 bash[75299]: audit 2023-10-01T10:26:49.570972+0000 mon.a (mon.0) 227 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:50.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:50 smithi078 bash[75299]: audit 2023-10-01T10:26:49.600452+0000 mon.a (mon.0) 228 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi078.vjqrth", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-01T10:26:50.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:50 smithi078 bash[75299]: audit 2023-10-01T10:26:49.602077+0000 mon.a (mon.0) 229 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:26:50.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:50 smithi173 bash[58816]: cluster 2023-10-01T10:26:49.348516+0000 mgr.y (mgr.25072) 167 : cluster 0 pgmap v125: 89 pgs: 89 active+clean; 439 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 908 op/s 2023-10-01T10:26:50.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:50 smithi173 bash[58816]: audit 2023-10-01T10:26:49.560469+0000 mon.a (mon.0) 226 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:50.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:50 smithi173 bash[58816]: audit 2023-10-01T10:26:49.570972+0000 mon.a (mon.0) 227 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:50.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:50 smithi173 bash[58816]: audit 2023-10-01T10:26:49.600452+0000 mon.a (mon.0) 228 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi078.vjqrth", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-01T10:26:50.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:50 smithi173 bash[58816]: audit 2023-10-01T10:26:49.602077+0000 mon.a (mon.0) 229 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:26:51.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:51 smithi078 bash[78274]: cephadm 2023-10-01T10:26:49.599963+0000 mgr.y (mgr.25072) 168 : cephadm 1 Reconfiguring mds.foo.smithi078.vjqrth (monmap changed)... 2023-10-01T10:26:51.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:51 smithi078 bash[78274]: cephadm 2023-10-01T10:26:49.603372+0000 mgr.y (mgr.25072) 169 : cephadm 1 Reconfiguring daemon mds.foo.smithi078.vjqrth on smithi078 2023-10-01T10:26:51.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:51 smithi078 bash[75299]: cephadm 2023-10-01T10:26:49.599963+0000 mgr.y (mgr.25072) 168 : cephadm 1 Reconfiguring mds.foo.smithi078.vjqrth (monmap changed)... 2023-10-01T10:26:51.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:51 smithi078 bash[75299]: cephadm 2023-10-01T10:26:49.603372+0000 mgr.y (mgr.25072) 169 : cephadm 1 Reconfiguring daemon mds.foo.smithi078.vjqrth on smithi078 2023-10-01T10:26:51.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:51 smithi173 bash[58816]: cephadm 2023-10-01T10:26:49.599963+0000 mgr.y (mgr.25072) 168 : cephadm 1 Reconfiguring mds.foo.smithi078.vjqrth (monmap changed)... 2023-10-01T10:26:51.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:51 smithi173 bash[58816]: cephadm 2023-10-01T10:26:49.603372+0000 mgr.y (mgr.25072) 169 : cephadm 1 Reconfiguring daemon mds.foo.smithi078.vjqrth on smithi078 2023-10-01T10:26:52.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:52 smithi078 bash[75299]: cluster 2023-10-01T10:26:51.349035+0000 mgr.y (mgr.25072) 170 : cluster 0 pgmap v126: 89 pgs: 89 active+clean; 439 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 602 op/s 2023-10-01T10:26:52.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:52 smithi078 bash[78274]: cluster 2023-10-01T10:26:51.349035+0000 mgr.y (mgr.25072) 170 : cluster 0 pgmap v126: 89 pgs: 89 active+clean; 439 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 602 op/s 2023-10-01T10:26:52.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:52 smithi173 bash[58816]: cluster 2023-10-01T10:26:51.349035+0000 mgr.y (mgr.25072) 170 : cluster 0 pgmap v126: 89 pgs: 89 active+clean; 439 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 602 op/s 2023-10-01T10:26:53.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:53 smithi078 bash[75299]: audit 2023-10-01T10:26:52.458314+0000 mon.a (mon.0) 230 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:53.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:53 smithi078 bash[75299]: audit 2023-10-01T10:26:52.473141+0000 mon.a (mon.0) 231 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:53.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:53 smithi078 bash[75299]: audit 2023-10-01T10:26:52.523565+0000 mon.a (mon.0) 232 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T10:26:53.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:53 smithi078 bash[75299]: audit 2023-10-01T10:26:52.524688+0000 mon.a (mon.0) 233 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T10:26:53.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:53 smithi078 bash[75299]: audit 2023-10-01T10:26:52.525751+0000 mon.a (mon.0) 234 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:26:53.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:53 smithi078 bash[78274]: audit 2023-10-01T10:26:52.458314+0000 mon.a (mon.0) 230 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:53.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:53 smithi078 bash[78274]: audit 2023-10-01T10:26:52.473141+0000 mon.a (mon.0) 231 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:53.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:53 smithi078 bash[78274]: audit 2023-10-01T10:26:52.523565+0000 mon.a (mon.0) 232 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T10:26:53.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:53 smithi078 bash[78274]: audit 2023-10-01T10:26:52.524688+0000 mon.a (mon.0) 233 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T10:26:53.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:53 smithi078 bash[78274]: audit 2023-10-01T10:26:52.525751+0000 mon.a (mon.0) 234 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:26:53.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:53 smithi173 bash[58816]: audit 2023-10-01T10:26:52.458314+0000 mon.a (mon.0) 230 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:53.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:53 smithi173 bash[58816]: audit 2023-10-01T10:26:52.473141+0000 mon.a (mon.0) 231 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:53.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:53 smithi173 bash[58816]: audit 2023-10-01T10:26:52.523565+0000 mon.a (mon.0) 232 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-10-01T10:26:53.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:53 smithi173 bash[58816]: audit 2023-10-01T10:26:52.524688+0000 mon.a (mon.0) 233 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-10-01T10:26:53.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:53 smithi173 bash[58816]: audit 2023-10-01T10:26:52.525751+0000 mon.a (mon.0) 234 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:26:54.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:54 smithi078 bash[75299]: cephadm 2023-10-01T10:26:52.523112+0000 mgr.y (mgr.25072) 171 : cephadm 1 Reconfiguring mon.b (monmap changed)... 2023-10-01T10:26:54.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:54 smithi078 bash[75299]: cephadm 2023-10-01T10:26:52.526859+0000 mgr.y (mgr.25072) 172 : cephadm 1 Reconfiguring daemon mon.b on smithi173 2023-10-01T10:26:54.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:54 smithi078 bash[75299]: cluster 2023-10-01T10:26:53.349489+0000 mgr.y (mgr.25072) 173 : cluster 0 pgmap v127: 89 pgs: 89 active+clean; 439 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 602 op/s 2023-10-01T10:26:54.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:54 smithi078 bash[78274]: cephadm 2023-10-01T10:26:52.523112+0000 mgr.y (mgr.25072) 171 : cephadm 1 Reconfiguring mon.b (monmap changed)... 2023-10-01T10:26:54.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:54 smithi078 bash[78274]: cephadm 2023-10-01T10:26:52.526859+0000 mgr.y (mgr.25072) 172 : cephadm 1 Reconfiguring daemon mon.b on smithi173 2023-10-01T10:26:54.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:54 smithi078 bash[78274]: cluster 2023-10-01T10:26:53.349489+0000 mgr.y (mgr.25072) 173 : cluster 0 pgmap v127: 89 pgs: 89 active+clean; 439 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 602 op/s 2023-10-01T10:26:54.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:54 smithi173 bash[58816]: cephadm 2023-10-01T10:26:52.523112+0000 mgr.y (mgr.25072) 171 : cephadm 1 Reconfiguring mon.b (monmap changed)... 2023-10-01T10:26:54.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:54 smithi173 bash[58816]: cephadm 2023-10-01T10:26:52.526859+0000 mgr.y (mgr.25072) 172 : cephadm 1 Reconfiguring daemon mon.b on smithi173 2023-10-01T10:26:54.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:54 smithi173 bash[58816]: cluster 2023-10-01T10:26:53.349489+0000 mgr.y (mgr.25072) 173 : cluster 0 pgmap v127: 89 pgs: 89 active+clean; 439 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 602 op/s 2023-10-01T10:26:56.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:56 smithi173 bash[58816]: cluster 2023-10-01T10:26:55.350298+0000 mgr.y (mgr.25072) 174 : cluster 0 pgmap v128: 89 pgs: 89 active+clean; 211 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 906 op/s 2023-10-01T10:26:56.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:56 smithi078 bash[75299]: cluster 2023-10-01T10:26:55.350298+0000 mgr.y (mgr.25072) 174 : cluster 0 pgmap v128: 89 pgs: 89 active+clean; 211 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 906 op/s 2023-10-01T10:26:56.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:56 smithi078 bash[78274]: cluster 2023-10-01T10:26:55.350298+0000 mgr.y (mgr.25072) 174 : cluster 0 pgmap v128: 89 pgs: 89 active+clean; 211 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 906 op/s 2023-10-01T10:26:58.254 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:57 smithi173 bash[58816]: audit 2023-10-01T10:26:56.944802+0000 mon.a (mon.0) 235 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:58.254 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:57 smithi173 bash[58816]: audit 2023-10-01T10:26:56.955418+0000 mon.a (mon.0) 236 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:58.254 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:57 smithi173 bash[58816]: cephadm 2023-10-01T10:26:56.962172+0000 mgr.y (mgr.25072) 175 : cephadm 1 Reconfiguring mgr.x (monmap changed)... 2023-10-01T10:26:58.254 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:57 smithi173 bash[58816]: audit 2023-10-01T10:26:56.962782+0000 mon.a (mon.0) 237 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-01T10:26:58.254 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:57 smithi173 bash[58816]: audit 2023-10-01T10:26:56.964180+0000 mon.a (mon.0) 238 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-01T10:26:58.254 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:57 smithi173 bash[58816]: audit 2023-10-01T10:26:56.965273+0000 mon.a (mon.0) 239 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:26:58.254 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:57 smithi173 bash[58816]: cephadm 2023-10-01T10:26:56.966593+0000 mgr.y (mgr.25072) 176 : cephadm 1 Reconfiguring daemon mgr.x on smithi173 2023-10-01T10:26:58.255 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:26:57 smithi173 bash[58816]: cluster 2023-10-01T10:26:57.350753+0000 mgr.y (mgr.25072) 177 : cluster 0 pgmap v129: 89 pgs: 89 active+clean; 211 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 606 op/s 2023-10-01T10:26:58.324 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Removed 115066 objects 2023-10-01T10:26:58.324 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Clean up completed and total clean up time :156.978 2023-10-01T10:26:58.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:57 smithi078 bash[75299]: audit 2023-10-01T10:26:56.944802+0000 mon.a (mon.0) 235 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:58.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:57 smithi078 bash[75299]: audit 2023-10-01T10:26:56.955418+0000 mon.a (mon.0) 236 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:58.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:57 smithi078 bash[75299]: cephadm 2023-10-01T10:26:56.962172+0000 mgr.y (mgr.25072) 175 : cephadm 1 Reconfiguring mgr.x (monmap changed)... 2023-10-01T10:26:58.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:57 smithi078 bash[75299]: audit 2023-10-01T10:26:56.962782+0000 mon.a (mon.0) 237 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-01T10:26:58.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:57 smithi078 bash[75299]: audit 2023-10-01T10:26:56.964180+0000 mon.a (mon.0) 238 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-01T10:26:58.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:57 smithi078 bash[75299]: audit 2023-10-01T10:26:56.965273+0000 mon.a (mon.0) 239 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:26:58.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:57 smithi078 bash[75299]: cephadm 2023-10-01T10:26:56.966593+0000 mgr.y (mgr.25072) 176 : cephadm 1 Reconfiguring daemon mgr.x on smithi173 2023-10-01T10:26:58.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:26:57 smithi078 bash[75299]: cluster 2023-10-01T10:26:57.350753+0000 mgr.y (mgr.25072) 177 : cluster 0 pgmap v129: 89 pgs: 89 active+clean; 211 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 606 op/s 2023-10-01T10:26:58.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:57 smithi078 bash[78274]: audit 2023-10-01T10:26:56.944802+0000 mon.a (mon.0) 235 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:58.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:57 smithi078 bash[78274]: audit 2023-10-01T10:26:56.955418+0000 mon.a (mon.0) 236 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:26:58.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:57 smithi078 bash[78274]: cephadm 2023-10-01T10:26:56.962172+0000 mgr.y (mgr.25072) 175 : cephadm 1 Reconfiguring mgr.x (monmap changed)... 2023-10-01T10:26:58.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:57 smithi078 bash[78274]: audit 2023-10-01T10:26:56.962782+0000 mon.a (mon.0) 237 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-10-01T10:26:58.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:57 smithi078 bash[78274]: audit 2023-10-01T10:26:56.964180+0000 mon.a (mon.0) 238 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-10-01T10:26:58.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:57 smithi078 bash[78274]: audit 2023-10-01T10:26:56.965273+0000 mon.a (mon.0) 239 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:26:58.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:57 smithi078 bash[78274]: cephadm 2023-10-01T10:26:56.966593+0000 mgr.y (mgr.25072) 176 : cephadm 1 Reconfiguring daemon mgr.x on smithi173 2023-10-01T10:26:58.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:26:57 smithi078 bash[78274]: cluster 2023-10-01T10:26:57.350753+0000 mgr.y (mgr.25072) 177 : cluster 0 pgmap v129: 89 pgs: 89 active+clean; 211 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 606 op/s 2023-10-01T10:27:00.516 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:00 smithi173 bash[58816]: audit 2023-10-01T10:26:59.148118+0000 mon.a (mon.0) 240 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:00.517 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:00 smithi173 bash[58816]: audit 2023-10-01T10:26:59.158357+0000 mon.a (mon.0) 241 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:00.517 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:00 smithi173 bash[58816]: cephadm 2023-10-01T10:26:59.189012+0000 mgr.y (mgr.25072) 178 : cephadm 1 Reconfiguring osd.4 (monmap changed)... 2023-10-01T10:27:00.517 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:00 smithi173 bash[58816]: audit 2023-10-01T10:26:59.189642+0000 mon.a (mon.0) 242 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-01T10:27:00.517 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:00 smithi173 bash[58816]: audit 2023-10-01T10:26:59.191624+0000 mon.a (mon.0) 243 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:27:00.517 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:00 smithi173 bash[58816]: cephadm 2023-10-01T10:26:59.195877+0000 mgr.y (mgr.25072) 179 : cephadm 1 Reconfiguring daemon osd.4 on smithi173 2023-10-01T10:27:00.517 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:00 smithi173 bash[58816]: cluster 2023-10-01T10:26:59.351888+0000 mgr.y (mgr.25072) 180 : cluster 0 pgmap v130: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 886 op/s 2023-10-01T10:27:00.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:27:00 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:27:00] "GET /metrics HTTP/1.1" 200 36065 "" "Prometheus/2.33.4" 2023-10-01T10:27:00.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:00 smithi078 bash[75299]: audit 2023-10-01T10:26:59.148118+0000 mon.a (mon.0) 240 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:00.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:00 smithi078 bash[75299]: audit 2023-10-01T10:26:59.158357+0000 mon.a (mon.0) 241 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:00.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:00 smithi078 bash[75299]: cephadm 2023-10-01T10:26:59.189012+0000 mgr.y (mgr.25072) 178 : cephadm 1 Reconfiguring osd.4 (monmap changed)... 2023-10-01T10:27:00.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:00 smithi078 bash[75299]: audit 2023-10-01T10:26:59.189642+0000 mon.a (mon.0) 242 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-01T10:27:00.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:00 smithi078 bash[75299]: audit 2023-10-01T10:26:59.191624+0000 mon.a (mon.0) 243 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:27:00.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:00 smithi078 bash[75299]: cephadm 2023-10-01T10:26:59.195877+0000 mgr.y (mgr.25072) 179 : cephadm 1 Reconfiguring daemon osd.4 on smithi173 2023-10-01T10:27:00.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:00 smithi078 bash[75299]: cluster 2023-10-01T10:26:59.351888+0000 mgr.y (mgr.25072) 180 : cluster 0 pgmap v130: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 886 op/s 2023-10-01T10:27:00.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:00 smithi078 bash[78274]: audit 2023-10-01T10:26:59.148118+0000 mon.a (mon.0) 240 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:00.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:00 smithi078 bash[78274]: audit 2023-10-01T10:26:59.158357+0000 mon.a (mon.0) 241 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:00.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:00 smithi078 bash[78274]: cephadm 2023-10-01T10:26:59.189012+0000 mgr.y (mgr.25072) 178 : cephadm 1 Reconfiguring osd.4 (monmap changed)... 2023-10-01T10:27:00.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:00 smithi078 bash[78274]: audit 2023-10-01T10:26:59.189642+0000 mon.a (mon.0) 242 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-01T10:27:00.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:00 smithi078 bash[78274]: audit 2023-10-01T10:26:59.191624+0000 mon.a (mon.0) 243 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:27:00.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:00 smithi078 bash[78274]: cephadm 2023-10-01T10:26:59.195877+0000 mgr.y (mgr.25072) 179 : cephadm 1 Reconfiguring daemon osd.4 on smithi173 2023-10-01T10:27:00.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:00 smithi078 bash[78274]: cluster 2023-10-01T10:26:59.351888+0000 mgr.y (mgr.25072) 180 : cluster 0 pgmap v130: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 886 op/s 2023-10-01T10:27:01.436 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_1 2023-10-01T10:27:01.437 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_1 unique_pool_1 --yes-i-really-really-mean-it 2023-10-01T10:27:02.585 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:02 smithi078 bash[75299]: audit 2023-10-01T10:27:01.272983+0000 mon.a (mon.0) 244 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:02.585 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:02 smithi078 bash[75299]: audit 2023-10-01T10:27:01.283109+0000 mon.a (mon.0) 245 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:02.585 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:02 smithi078 bash[75299]: cephadm 2023-10-01T10:27:01.315453+0000 mgr.y (mgr.25072) 181 : cephadm 1 Reconfiguring osd.5 (monmap changed)... 2023-10-01T10:27:02.586 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:02 smithi078 bash[75299]: audit 2023-10-01T10:27:01.316017+0000 mon.a (mon.0) 246 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-01T10:27:02.586 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:02 smithi078 bash[75299]: audit 2023-10-01T10:27:01.317543+0000 mon.a (mon.0) 247 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:27:02.586 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:02 smithi078 bash[75299]: cephadm 2023-10-01T10:27:01.321737+0000 mgr.y (mgr.25072) 182 : cephadm 1 Reconfiguring daemon osd.5 on smithi173 2023-10-01T10:27:02.586 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:02 smithi078 bash[75299]: cluster 2023-10-01T10:27:01.352525+0000 mgr.y (mgr.25072) 183 : cluster 0 pgmap v131: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 583 op/s 2023-10-01T10:27:02.586 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:02 smithi078 bash[78274]: audit 2023-10-01T10:27:01.272983+0000 mon.a (mon.0) 244 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:02.586 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:02 smithi078 bash[78274]: audit 2023-10-01T10:27:01.283109+0000 mon.a (mon.0) 245 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:02.586 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:02 smithi078 bash[78274]: cephadm 2023-10-01T10:27:01.315453+0000 mgr.y (mgr.25072) 181 : cephadm 1 Reconfiguring osd.5 (monmap changed)... 2023-10-01T10:27:02.586 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:02 smithi078 bash[78274]: audit 2023-10-01T10:27:01.316017+0000 mon.a (mon.0) 246 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-01T10:27:02.586 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:02 smithi078 bash[78274]: audit 2023-10-01T10:27:01.317543+0000 mon.a (mon.0) 247 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:27:02.587 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:02 smithi078 bash[78274]: cephadm 2023-10-01T10:27:01.321737+0000 mgr.y (mgr.25072) 182 : cephadm 1 Reconfiguring daemon osd.5 on smithi173 2023-10-01T10:27:02.587 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:02 smithi078 bash[78274]: cluster 2023-10-01T10:27:01.352525+0000 mgr.y (mgr.25072) 183 : cluster 0 pgmap v131: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 583 op/s 2023-10-01T10:27:02.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:02 smithi173 bash[58816]: audit 2023-10-01T10:27:01.272983+0000 mon.a (mon.0) 244 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:02.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:02 smithi173 bash[58816]: audit 2023-10-01T10:27:01.283109+0000 mon.a (mon.0) 245 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:02.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:02 smithi173 bash[58816]: cephadm 2023-10-01T10:27:01.315453+0000 mgr.y (mgr.25072) 181 : cephadm 1 Reconfiguring osd.5 (monmap changed)... 2023-10-01T10:27:02.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:02 smithi173 bash[58816]: audit 2023-10-01T10:27:01.316017+0000 mon.a (mon.0) 246 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-10-01T10:27:02.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:02 smithi173 bash[58816]: audit 2023-10-01T10:27:01.317543+0000 mon.a (mon.0) 247 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:27:02.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:02 smithi173 bash[58816]: cephadm 2023-10-01T10:27:01.321737+0000 mgr.y (mgr.25072) 182 : cephadm 1 Reconfiguring daemon osd.5 on smithi173 2023-10-01T10:27:02.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:02 smithi173 bash[58816]: cluster 2023-10-01T10:27:01.352525+0000 mgr.y (mgr.25072) 183 : cluster 0 pgmap v131: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 583 op/s 2023-10-01T10:27:03.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:03 smithi078 bash[75299]: audit 2023-10-01T10:27:02.386568+0000 mon.a (mon.0) 248 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:27:03.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:03 smithi078 bash[75299]: audit 2023-10-01T10:27:03.255343+0000 mon.a (mon.0) 249 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:03.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:03 smithi078 bash[75299]: audit 2023-10-01T10:27:03.264020+0000 mon.a (mon.0) 250 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:03.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:03 smithi078 bash[78274]: audit 2023-10-01T10:27:02.386568+0000 mon.a (mon.0) 248 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:27:03.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:03 smithi078 bash[78274]: audit 2023-10-01T10:27:03.255343+0000 mon.a (mon.0) 249 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:03.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:03 smithi078 bash[78274]: audit 2023-10-01T10:27:03.264020+0000 mon.a (mon.0) 250 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:03.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:03 smithi173 bash[58816]: audit 2023-10-01T10:27:02.386568+0000 mon.a (mon.0) 248 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:27:03.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:03 smithi173 bash[58816]: audit 2023-10-01T10:27:03.255343+0000 mon.a (mon.0) 249 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:03.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:03 smithi173 bash[58816]: audit 2023-10-01T10:27:03.264020+0000 mon.a (mon.0) 250 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:04.572 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:04 smithi173 bash[58816]: cephadm 2023-10-01T10:27:03.289991+0000 mgr.y (mgr.25072) 184 : cephadm 1 Reconfiguring osd.6 (monmap changed)... 2023-10-01T10:27:04.572 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:04 smithi173 bash[58816]: audit 2023-10-01T10:27:03.292059+0000 mon.a (mon.0) 251 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-01T10:27:04.572 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:04 smithi173 bash[58816]: audit 2023-10-01T10:27:03.293383+0000 mon.a (mon.0) 252 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:27:04.572 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:04 smithi173 bash[58816]: cephadm 2023-10-01T10:27:03.296443+0000 mgr.y (mgr.25072) 185 : cephadm 1 Reconfiguring daemon osd.6 on smithi173 2023-10-01T10:27:04.572 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:04 smithi173 bash[58816]: cluster 2023-10-01T10:27:03.353191+0000 mgr.y (mgr.25072) 186 : cluster 0 pgmap v132: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 583 op/s 2023-10-01T10:27:04.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:04 smithi078 bash[75299]: cephadm 2023-10-01T10:27:03.289991+0000 mgr.y (mgr.25072) 184 : cephadm 1 Reconfiguring osd.6 (monmap changed)... 2023-10-01T10:27:04.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:04 smithi078 bash[75299]: audit 2023-10-01T10:27:03.292059+0000 mon.a (mon.0) 251 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-01T10:27:04.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:04 smithi078 bash[75299]: audit 2023-10-01T10:27:03.293383+0000 mon.a (mon.0) 252 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:27:04.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:04 smithi078 bash[75299]: cephadm 2023-10-01T10:27:03.296443+0000 mgr.y (mgr.25072) 185 : cephadm 1 Reconfiguring daemon osd.6 on smithi173 2023-10-01T10:27:04.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:04 smithi078 bash[75299]: cluster 2023-10-01T10:27:03.353191+0000 mgr.y (mgr.25072) 186 : cluster 0 pgmap v132: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 583 op/s 2023-10-01T10:27:04.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:04 smithi078 bash[78274]: cephadm 2023-10-01T10:27:03.289991+0000 mgr.y (mgr.25072) 184 : cephadm 1 Reconfiguring osd.6 (monmap changed)... 2023-10-01T10:27:04.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:04 smithi078 bash[78274]: audit 2023-10-01T10:27:03.292059+0000 mon.a (mon.0) 251 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-10-01T10:27:04.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:04 smithi078 bash[78274]: audit 2023-10-01T10:27:03.293383+0000 mon.a (mon.0) 252 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:27:04.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:04 smithi078 bash[78274]: cephadm 2023-10-01T10:27:03.296443+0000 mgr.y (mgr.25072) 185 : cephadm 1 Reconfiguring daemon osd.6 on smithi173 2023-10-01T10:27:04.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:04 smithi078 bash[78274]: cluster 2023-10-01T10:27:03.353191+0000 mgr.y (mgr.25072) 186 : cluster 0 pgmap v132: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 583 op/s 2023-10-01T10:27:06.218 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:27:06.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:06 smithi078 bash[75299]: audit 2023-10-01T10:27:05.326088+0000 mon.a (mon.0) 253 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:06.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:06 smithi078 bash[75299]: audit 2023-10-01T10:27:05.333697+0000 mon.a (mon.0) 254 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:06.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:06 smithi078 bash[75299]: cluster 2023-10-01T10:27:05.354430+0000 mgr.y (mgr.25072) 187 : cluster 0 pgmap v133: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 583 op/s 2023-10-01T10:27:06.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:06 smithi078 bash[75299]: cephadm 2023-10-01T10:27:05.364716+0000 mgr.y (mgr.25072) 188 : cephadm 1 Reconfiguring osd.7 (monmap changed)... 2023-10-01T10:27:06.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:06 smithi078 bash[75299]: audit 2023-10-01T10:27:05.365283+0000 mon.a (mon.0) 255 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-01T10:27:06.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:06 smithi078 bash[75299]: audit 2023-10-01T10:27:05.366577+0000 mon.a (mon.0) 256 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:27:06.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:06 smithi078 bash[75299]: cephadm 2023-10-01T10:27:05.369446+0000 mgr.y (mgr.25072) 189 : cephadm 1 Reconfiguring daemon osd.7 on smithi173 2023-10-01T10:27:06.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:06 smithi078 bash[78274]: audit 2023-10-01T10:27:05.326088+0000 mon.a (mon.0) 253 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:06.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:06 smithi078 bash[78274]: audit 2023-10-01T10:27:05.333697+0000 mon.a (mon.0) 254 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:06.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:06 smithi078 bash[78274]: cluster 2023-10-01T10:27:05.354430+0000 mgr.y (mgr.25072) 187 : cluster 0 pgmap v133: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 583 op/s 2023-10-01T10:27:06.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:06 smithi078 bash[78274]: cephadm 2023-10-01T10:27:05.364716+0000 mgr.y (mgr.25072) 188 : cephadm 1 Reconfiguring osd.7 (monmap changed)... 2023-10-01T10:27:06.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:06 smithi078 bash[78274]: audit 2023-10-01T10:27:05.365283+0000 mon.a (mon.0) 255 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-01T10:27:06.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:06 smithi078 bash[78274]: audit 2023-10-01T10:27:05.366577+0000 mon.a (mon.0) 256 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:27:06.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:06 smithi078 bash[78274]: cephadm 2023-10-01T10:27:05.369446+0000 mgr.y (mgr.25072) 189 : cephadm 1 Reconfiguring daemon osd.7 on smithi173 2023-10-01T10:27:06.617 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:06 smithi173 bash[58816]: audit 2023-10-01T10:27:05.326088+0000 mon.a (mon.0) 253 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:06.617 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:06 smithi173 bash[58816]: audit 2023-10-01T10:27:05.333697+0000 mon.a (mon.0) 254 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:06.617 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:06 smithi173 bash[58816]: cluster 2023-10-01T10:27:05.354430+0000 mgr.y (mgr.25072) 187 : cluster 0 pgmap v133: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 583 op/s 2023-10-01T10:27:06.617 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:06 smithi173 bash[58816]: cephadm 2023-10-01T10:27:05.364716+0000 mgr.y (mgr.25072) 188 : cephadm 1 Reconfiguring osd.7 (monmap changed)... 2023-10-01T10:27:06.617 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:06 smithi173 bash[58816]: audit 2023-10-01T10:27:05.365283+0000 mon.a (mon.0) 255 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-10-01T10:27:06.617 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:06 smithi173 bash[58816]: audit 2023-10-01T10:27:05.366577+0000 mon.a (mon.0) 256 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:27:06.617 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:06 smithi173 bash[58816]: cephadm 2023-10-01T10:27:05.369446+0000 mgr.y (mgr.25072) 189 : cephadm 1 Reconfiguring daemon osd.7 on smithi173 2023-10-01T10:27:08.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:08 smithi078 bash[75299]: cluster 2023-10-01T10:27:07.355119+0000 mgr.y (mgr.25072) 190 : cluster 0 pgmap v134: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 280 op/s 2023-10-01T10:27:08.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:08 smithi078 bash[75299]: audit 2023-10-01T10:27:07.368448+0000 mon.a (mon.0) 257 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:08.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:08 smithi078 bash[75299]: audit 2023-10-01T10:27:07.375035+0000 mon.a (mon.0) 258 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:08.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:08 smithi078 bash[75299]: cephadm 2023-10-01T10:27:07.388980+0000 mgr.y (mgr.25072) 191 : cephadm 1 Reconfiguring mds.foo.smithi173.vrqqoc (monmap changed)... 2023-10-01T10:27:08.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:08 smithi078 bash[75299]: audit 2023-10-01T10:27:07.389456+0000 mon.a (mon.0) 259 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi173.vrqqoc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-01T10:27:08.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:08 smithi078 bash[75299]: audit 2023-10-01T10:27:07.390625+0000 mon.a (mon.0) 260 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:27:08.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:08 smithi078 bash[75299]: cephadm 2023-10-01T10:27:07.392075+0000 mgr.y (mgr.25072) 192 : cephadm 1 Reconfiguring daemon mds.foo.smithi173.vrqqoc on smithi173 2023-10-01T10:27:08.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:08 smithi078 bash[78274]: cluster 2023-10-01T10:27:07.355119+0000 mgr.y (mgr.25072) 190 : cluster 0 pgmap v134: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 280 op/s 2023-10-01T10:27:08.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:08 smithi078 bash[78274]: audit 2023-10-01T10:27:07.368448+0000 mon.a (mon.0) 257 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:08.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:08 smithi078 bash[78274]: audit 2023-10-01T10:27:07.375035+0000 mon.a (mon.0) 258 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:08.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:08 smithi078 bash[78274]: cephadm 2023-10-01T10:27:07.388980+0000 mgr.y (mgr.25072) 191 : cephadm 1 Reconfiguring mds.foo.smithi173.vrqqoc (monmap changed)... 2023-10-01T10:27:08.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:08 smithi078 bash[78274]: audit 2023-10-01T10:27:07.389456+0000 mon.a (mon.0) 259 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi173.vrqqoc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-01T10:27:08.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:08 smithi078 bash[78274]: audit 2023-10-01T10:27:07.390625+0000 mon.a (mon.0) 260 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:27:08.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:08 smithi078 bash[78274]: cephadm 2023-10-01T10:27:07.392075+0000 mgr.y (mgr.25072) 192 : cephadm 1 Reconfiguring daemon mds.foo.smithi173.vrqqoc on smithi173 2023-10-01T10:27:08.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:08 smithi173 bash[58816]: cluster 2023-10-01T10:27:07.355119+0000 mgr.y (mgr.25072) 190 : cluster 0 pgmap v134: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 280 op/s 2023-10-01T10:27:08.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:08 smithi173 bash[58816]: audit 2023-10-01T10:27:07.368448+0000 mon.a (mon.0) 257 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:08.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:08 smithi173 bash[58816]: audit 2023-10-01T10:27:07.375035+0000 mon.a (mon.0) 258 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:08.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:08 smithi173 bash[58816]: cephadm 2023-10-01T10:27:07.388980+0000 mgr.y (mgr.25072) 191 : cephadm 1 Reconfiguring mds.foo.smithi173.vrqqoc (monmap changed)... 2023-10-01T10:27:08.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:08 smithi173 bash[58816]: audit 2023-10-01T10:27:07.389456+0000 mon.a (mon.0) 259 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi173.vrqqoc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-10-01T10:27:08.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:08 smithi173 bash[58816]: audit 2023-10-01T10:27:07.390625+0000 mon.a (mon.0) 260 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:27:08.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:08 smithi173 bash[58816]: cephadm 2023-10-01T10:27:07.392075+0000 mgr.y (mgr.25072) 192 : cephadm 1 Reconfiguring daemon mds.foo.smithi173.vrqqoc on smithi173 2023-10-01T10:27:10.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:27:10 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:27:10] "GET /metrics HTTP/1.1" 200 36016 "" "Prometheus/2.33.4" 2023-10-01T10:27:10.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:10 smithi078 bash[75299]: audit 2023-10-01T10:27:09.299383+0000 mon.a (mon.0) 261 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:10.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:10 smithi078 bash[75299]: audit 2023-10-01T10:27:09.307923+0000 mon.a (mon.0) 262 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:10.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:10 smithi078 bash[75299]: audit 2023-10-01T10:27:09.319676+0000 mon.a (mon.0) 263 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:27:10.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:10 smithi078 bash[75299]: cluster 2023-10-01T10:27:09.356137+0000 mgr.y (mgr.25072) 193 : cluster 0 pgmap v135: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 280 op/s 2023-10-01T10:27:10.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:10 smithi078 bash[75299]: audit 2023-10-01T10:27:09.696112+0000 mon.a (mon.0) 264 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:10.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:10 smithi078 bash[75299]: audit 2023-10-01T10:27:09.703972+0000 mon.a (mon.0) 265 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:10.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:10 smithi078 bash[78274]: audit 2023-10-01T10:27:09.299383+0000 mon.a (mon.0) 261 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:10.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:10 smithi078 bash[78274]: audit 2023-10-01T10:27:09.307923+0000 mon.a (mon.0) 262 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:10.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:10 smithi078 bash[78274]: audit 2023-10-01T10:27:09.319676+0000 mon.a (mon.0) 263 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:27:10.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:10 smithi078 bash[78274]: cluster 2023-10-01T10:27:09.356137+0000 mgr.y (mgr.25072) 193 : cluster 0 pgmap v135: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 280 op/s 2023-10-01T10:27:10.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:10 smithi078 bash[78274]: audit 2023-10-01T10:27:09.696112+0000 mon.a (mon.0) 264 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:10.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:10 smithi078 bash[78274]: audit 2023-10-01T10:27:09.703972+0000 mon.a (mon.0) 265 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:10.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:10 smithi173 bash[58816]: audit 2023-10-01T10:27:09.299383+0000 mon.a (mon.0) 261 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:10.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:10 smithi173 bash[58816]: audit 2023-10-01T10:27:09.307923+0000 mon.a (mon.0) 262 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:10.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:10 smithi173 bash[58816]: audit 2023-10-01T10:27:09.319676+0000 mon.a (mon.0) 263 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:27:10.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:10 smithi173 bash[58816]: cluster 2023-10-01T10:27:09.356137+0000 mgr.y (mgr.25072) 193 : cluster 0 pgmap v135: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 280 op/s 2023-10-01T10:27:10.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:10 smithi173 bash[58816]: audit 2023-10-01T10:27:09.696112+0000 mon.a (mon.0) 264 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:10.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:10 smithi173 bash[58816]: audit 2023-10-01T10:27:09.703972+0000 mon.a (mon.0) 265 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:11.333 INFO:teuthology.orchestra.run.smithi078.stderr:pool 'unique_pool_1' removed 2023-10-01T10:27:11.578 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:11 smithi078 bash[78274]: audit 2023-10-01T10:27:10.316543+0000 mon.a (mon.0) 266 : audit 1 from='client.? 172.21.15.78:0/3639025446' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T10:27:11.578 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:11 smithi078 bash[75299]: audit 2023-10-01T10:27:10.316543+0000 mon.a (mon.0) 266 : audit 1 from='client.? 172.21.15.78:0/3639025446' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T10:27:11.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:11 smithi173 bash[58816]: audit 2023-10-01T10:27:10.316543+0000 mon.a (mon.0) 266 : audit 1 from='client.? 172.21.15.78:0/3639025446' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T10:27:12.169 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-01T10:27:12.169 INFO:tasks.radosbench:Beginning radosbench... 2023-10-01T10:27:12.170 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_2 2023-10-01T10:27:12.170 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_2 16 2023-10-01T10:27:12.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:12 smithi078 bash[75299]: cluster 2023-10-01T10:27:11.322560+0000 osd.1 (osd.1) 7 : cluster 3 failed to encode map e78 with expected crc 2023-10-01T10:27:12.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:12 smithi078 bash[75299]: cluster 2023-10-01T10:27:11.325330+0000 osd.7 (osd.7) 4 : cluster 3 failed to encode map e78 with expected crc 2023-10-01T10:27:12.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:12 smithi078 bash[75299]: audit 2023-10-01T10:27:11.325563+0000 mon.a (mon.0) 267 : audit 1 from='client.? 172.21.15.78:0/3639025446' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-10-01T10:27:12.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:12 smithi078 bash[75299]: cluster 2023-10-01T10:27:11.325780+0000 mon.a (mon.0) 268 : cluster 0 osdmap e78: 8 total, 8 up, 8 in 2023-10-01T10:27:12.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:12 smithi078 bash[75299]: cluster 2023-10-01T10:27:11.356637+0000 mgr.y (mgr.25072) 194 : cluster 0 pgmap v137: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:27:12.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:12 smithi078 bash[78274]: cluster 2023-10-01T10:27:11.322560+0000 osd.1 (osd.1) 7 : cluster 3 failed to encode map e78 with expected crc 2023-10-01T10:27:12.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:12 smithi078 bash[78274]: cluster 2023-10-01T10:27:11.325330+0000 osd.7 (osd.7) 4 : cluster 3 failed to encode map e78 with expected crc 2023-10-01T10:27:12.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:12 smithi078 bash[78274]: audit 2023-10-01T10:27:11.325563+0000 mon.a (mon.0) 267 : audit 1 from='client.? 172.21.15.78:0/3639025446' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-10-01T10:27:12.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:12 smithi078 bash[78274]: cluster 2023-10-01T10:27:11.325780+0000 mon.a (mon.0) 268 : cluster 0 osdmap e78: 8 total, 8 up, 8 in 2023-10-01T10:27:12.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:12 smithi078 bash[78274]: cluster 2023-10-01T10:27:11.356637+0000 mgr.y (mgr.25072) 194 : cluster 0 pgmap v137: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:27:12.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:12 smithi173 bash[58816]: cluster 2023-10-01T10:27:11.322560+0000 osd.1 (osd.1) 7 : cluster 3 failed to encode map e78 with expected crc 2023-10-01T10:27:12.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:12 smithi173 bash[58816]: cluster 2023-10-01T10:27:11.325330+0000 osd.7 (osd.7) 4 : cluster 3 failed to encode map e78 with expected crc 2023-10-01T10:27:12.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:12 smithi173 bash[58816]: audit 2023-10-01T10:27:11.325563+0000 mon.a (mon.0) 267 : audit 1 from='client.? 172.21.15.78:0/3639025446' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-10-01T10:27:12.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:12 smithi173 bash[58816]: cluster 2023-10-01T10:27:11.325780+0000 mon.a (mon.0) 268 : cluster 0 osdmap e78: 8 total, 8 up, 8 in 2023-10-01T10:27:12.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:12 smithi173 bash[58816]: cluster 2023-10-01T10:27:11.356637+0000 mgr.y (mgr.25072) 194 : cluster 0 pgmap v137: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:27:13.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:13 smithi173 bash[58816]: cluster 2023-10-01T10:27:11.902228+0000 osd.0 (osd.0) 5 : cluster 3 failed to encode map e78 with expected crc 2023-10-01T10:27:13.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:13 smithi173 bash[58816]: cluster 2023-10-01T10:27:12.079596+0000 osd.6 (osd.6) 4 : cluster 3 failed to encode map e78 with expected crc 2023-10-01T10:27:13.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:13 smithi078 bash[75299]: cluster 2023-10-01T10:27:11.902228+0000 osd.0 (osd.0) 5 : cluster 3 failed to encode map e78 with expected crc 2023-10-01T10:27:13.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:13 smithi078 bash[75299]: cluster 2023-10-01T10:27:12.079596+0000 osd.6 (osd.6) 4 : cluster 3 failed to encode map e78 with expected crc 2023-10-01T10:27:13.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:13 smithi078 bash[78274]: cluster 2023-10-01T10:27:11.902228+0000 osd.0 (osd.0) 5 : cluster 3 failed to encode map e78 with expected crc 2023-10-01T10:27:13.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:13 smithi078 bash[78274]: cluster 2023-10-01T10:27:12.079596+0000 osd.6 (osd.6) 4 : cluster 3 failed to encode map e78 with expected crc 2023-10-01T10:27:14.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:14 smithi173 bash[58816]: cluster 2023-10-01T10:27:13.357243+0000 mgr.y (mgr.25072) 195 : cluster 0 pgmap v138: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:27:14.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:14 smithi078 bash[78274]: cluster 2023-10-01T10:27:13.357243+0000 mgr.y (mgr.25072) 195 : cluster 0 pgmap v138: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:27:14.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:14 smithi078 bash[75299]: cluster 2023-10-01T10:27:13.357243+0000 mgr.y (mgr.25072) 195 : cluster 0 pgmap v138: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:27:16.726 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:27:16.760 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:16 smithi078 bash[78274]: cluster 2023-10-01T10:27:15.358243+0000 mgr.y (mgr.25072) 196 : cluster 0 pgmap v139: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:27:16.760 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:16 smithi078 bash[78274]: audit 2023-10-01T10:27:15.864119+0000 mon.a (mon.0) 269 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:16.760 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:16 smithi078 bash[78274]: audit 2023-10-01T10:27:15.872134+0000 mon.a (mon.0) 270 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:16.760 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:16 smithi078 bash[78274]: audit 2023-10-01T10:27:16.343264+0000 mon.a (mon.0) 271 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:27:16.760 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:16 smithi078 bash[78274]: audit 2023-10-01T10:27:16.344986+0000 mon.a (mon.0) 272 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:27:16.760 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:16 smithi078 bash[78274]: audit 2023-10-01T10:27:16.353761+0000 mon.a (mon.0) 273 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:16.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:16 smithi078 bash[75299]: cluster 2023-10-01T10:27:15.358243+0000 mgr.y (mgr.25072) 196 : cluster 0 pgmap v139: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:27:16.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:16 smithi078 bash[75299]: audit 2023-10-01T10:27:15.864119+0000 mon.a (mon.0) 269 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:16.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:16 smithi078 bash[75299]: audit 2023-10-01T10:27:15.872134+0000 mon.a (mon.0) 270 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:16.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:16 smithi078 bash[75299]: audit 2023-10-01T10:27:16.343264+0000 mon.a (mon.0) 271 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:27:16.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:16 smithi078 bash[75299]: audit 2023-10-01T10:27:16.344986+0000 mon.a (mon.0) 272 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:27:16.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:16 smithi078 bash[75299]: audit 2023-10-01T10:27:16.353761+0000 mon.a (mon.0) 273 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:16.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:16 smithi173 bash[58816]: cluster 2023-10-01T10:27:15.358243+0000 mgr.y (mgr.25072) 196 : cluster 0 pgmap v139: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:27:16.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:16 smithi173 bash[58816]: audit 2023-10-01T10:27:15.864119+0000 mon.a (mon.0) 269 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:16.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:16 smithi173 bash[58816]: audit 2023-10-01T10:27:15.872134+0000 mon.a (mon.0) 270 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:16.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:16 smithi173 bash[58816]: audit 2023-10-01T10:27:16.343264+0000 mon.a (mon.0) 271 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:27:16.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:16 smithi173 bash[58816]: audit 2023-10-01T10:27:16.344986+0000 mon.a (mon.0) 272 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:27:16.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:16 smithi173 bash[58816]: audit 2023-10-01T10:27:16.353761+0000 mon.a (mon.0) 273 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:18.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:17 smithi078 bash[75299]: cluster 2023-10-01T10:27:17.358926+0000 mgr.y (mgr.25072) 197 : cluster 0 pgmap v140: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:27:18.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:17 smithi078 bash[75299]: audit 2023-10-01T10:27:17.386134+0000 mon.a (mon.0) 274 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:27:18.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:17 smithi078 bash[78274]: cluster 2023-10-01T10:27:17.358926+0000 mgr.y (mgr.25072) 197 : cluster 0 pgmap v140: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:27:18.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:17 smithi078 bash[78274]: audit 2023-10-01T10:27:17.386134+0000 mon.a (mon.0) 274 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:27:18.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:17 smithi173 bash[58816]: cluster 2023-10-01T10:27:17.358926+0000 mgr.y (mgr.25072) 197 : cluster 0 pgmap v140: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:27:18.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:17 smithi173 bash[58816]: audit 2023-10-01T10:27:17.386134+0000 mon.a (mon.0) 274 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:27:20.508 INFO:teuthology.orchestra.run.smithi078.stderr:pool 'unique_pool_2' created 2023-10-01T10:27:20.532 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:20 smithi078 bash[78274]: cluster 2023-10-01T10:27:19.360000+0000 mgr.y (mgr.25072) 198 : cluster 0 pgmap v141: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:27:20.532 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:20 smithi078 bash[75299]: cluster 2023-10-01T10:27:19.360000+0000 mgr.y (mgr.25072) 198 : cluster 0 pgmap v141: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:27:20.533 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:27:20 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:27:20] "GET /metrics HTTP/1.1" 200 33725 "" "Prometheus/2.33.4" 2023-10-01T10:27:20.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:20 smithi173 bash[58816]: cluster 2023-10-01T10:27:19.360000+0000 mgr.y (mgr.25072) 198 : cluster 0 pgmap v141: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:27:21.368 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_2 rados --yes-i-really-mean-it || true 2023-10-01T10:27:21.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:21 smithi078 bash[78274]: audit 2023-10-01T10:27:20.008164+0000 mon.c (mon.2) 5 : audit 1 from='client.? 172.21.15.78:0/1449176302' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-01T10:27:21.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:21 smithi078 bash[78274]: audit 2023-10-01T10:27:20.008730+0000 mon.a (mon.0) 275 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-01T10:27:21.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:21 smithi078 bash[78274]: cluster 2023-10-01T10:27:20.495456+0000 osd.4 (osd.4) 8 : cluster 3 failed to encode map e79 with expected crc 2023-10-01T10:27:21.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:21 smithi078 bash[78274]: cluster 2023-10-01T10:27:20.495870+0000 osd.7 (osd.7) 5 : cluster 3 failed to encode map e79 with expected crc 2023-10-01T10:27:21.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:21 smithi078 bash[78274]: audit 2023-10-01T10:27:20.496685+0000 mon.a (mon.0) 276 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-10-01T10:27:21.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:21 smithi078 bash[78274]: cluster 2023-10-01T10:27:20.496863+0000 mon.a (mon.0) 277 : cluster 0 osdmap e79: 8 total, 8 up, 8 in 2023-10-01T10:27:21.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:21 smithi078 bash[75299]: audit 2023-10-01T10:27:20.008164+0000 mon.c (mon.2) 5 : audit 1 from='client.? 172.21.15.78:0/1449176302' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-01T10:27:21.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:21 smithi078 bash[75299]: audit 2023-10-01T10:27:20.008730+0000 mon.a (mon.0) 275 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-01T10:27:21.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:21 smithi078 bash[75299]: cluster 2023-10-01T10:27:20.495456+0000 osd.4 (osd.4) 8 : cluster 3 failed to encode map e79 with expected crc 2023-10-01T10:27:21.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:21 smithi078 bash[75299]: cluster 2023-10-01T10:27:20.495870+0000 osd.7 (osd.7) 5 : cluster 3 failed to encode map e79 with expected crc 2023-10-01T10:27:21.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:21 smithi078 bash[75299]: audit 2023-10-01T10:27:20.496685+0000 mon.a (mon.0) 276 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-10-01T10:27:21.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:21 smithi078 bash[75299]: cluster 2023-10-01T10:27:20.496863+0000 mon.a (mon.0) 277 : cluster 0 osdmap e79: 8 total, 8 up, 8 in 2023-10-01T10:27:21.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:21 smithi173 bash[58816]: audit 2023-10-01T10:27:20.008164+0000 mon.c (mon.2) 5 : audit 1 from='client.? 172.21.15.78:0/1449176302' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-01T10:27:21.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:21 smithi173 bash[58816]: audit 2023-10-01T10:27:20.008730+0000 mon.a (mon.0) 275 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-10-01T10:27:21.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:21 smithi173 bash[58816]: cluster 2023-10-01T10:27:20.495456+0000 osd.4 (osd.4) 8 : cluster 3 failed to encode map e79 with expected crc 2023-10-01T10:27:21.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:21 smithi173 bash[58816]: cluster 2023-10-01T10:27:20.495870+0000 osd.7 (osd.7) 5 : cluster 3 failed to encode map e79 with expected crc 2023-10-01T10:27:21.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:21 smithi173 bash[58816]: audit 2023-10-01T10:27:20.496685+0000 mon.a (mon.0) 276 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-10-01T10:27:21.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:21 smithi173 bash[58816]: cluster 2023-10-01T10:27:20.496863+0000 mon.a (mon.0) 277 : cluster 0 osdmap e79: 8 total, 8 up, 8 in 2023-10-01T10:27:22.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:22 smithi078 bash[75299]: cluster 2023-10-01T10:27:21.360657+0000 mgr.y (mgr.25072) 199 : cluster 0 pgmap v143: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:27:22.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:22 smithi078 bash[75299]: cluster 2023-10-01T10:27:21.504466+0000 osd.7 (osd.7) 6 : cluster 3 failed to encode map e80 with expected crc 2023-10-01T10:27:22.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:22 smithi078 bash[75299]: cluster 2023-10-01T10:27:21.505294+0000 osd.2 (osd.2) 6 : cluster 3 failed to encode map e80 with expected crc 2023-10-01T10:27:22.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:22 smithi078 bash[75299]: cluster 2023-10-01T10:27:21.507465+0000 mon.a (mon.0) 278 : cluster 0 osdmap e80: 8 total, 8 up, 8 in 2023-10-01T10:27:22.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:22 smithi078 bash[75299]: cluster 2023-10-01T10:27:21.507780+0000 osd.2 (osd.2) 7 : cluster 3 failed to encode map e80 with expected crc 2023-10-01T10:27:22.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:22 smithi078 bash[75299]: cluster 2023-10-01T10:27:21.507804+0000 osd.3 (osd.3) 4 : cluster 3 failed to encode map e80 with expected crc 2023-10-01T10:27:22.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:22 smithi078 bash[75299]: cluster 2023-10-01T10:27:21.507922+0000 osd.1 (osd.1) 8 : cluster 3 failed to encode map e80 with expected crc 2023-10-01T10:27:22.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:22 smithi078 bash[75299]: cluster 2023-10-01T10:27:21.507941+0000 osd.0 (osd.0) 6 : cluster 3 failed to encode map e80 with expected crc 2023-10-01T10:27:22.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:22 smithi078 bash[75299]: cluster 2023-10-01T10:27:21.509949+0000 osd.5 (osd.5) 6 : cluster 3 failed to encode map e80 with expected crc 2023-10-01T10:27:22.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:22 smithi078 bash[75299]: cluster 2023-10-01T10:27:21.509952+0000 osd.4 (osd.4) 9 : cluster 3 failed to encode map e80 with expected crc 2023-10-01T10:27:22.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:22 smithi078 bash[75299]: cluster 2023-10-01T10:27:21.510077+0000 osd.6 (osd.6) 5 : cluster 3 failed to encode map e80 with expected crc 2023-10-01T10:27:22.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:22 smithi078 bash[78274]: cluster 2023-10-01T10:27:21.360657+0000 mgr.y (mgr.25072) 199 : cluster 0 pgmap v143: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:27:22.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:22 smithi078 bash[78274]: cluster 2023-10-01T10:27:21.504466+0000 osd.7 (osd.7) 6 : cluster 3 failed to encode map e80 with expected crc 2023-10-01T10:27:22.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:22 smithi078 bash[78274]: cluster 2023-10-01T10:27:21.505294+0000 osd.2 (osd.2) 6 : cluster 3 failed to encode map e80 with expected crc 2023-10-01T10:27:22.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:22 smithi078 bash[78274]: cluster 2023-10-01T10:27:21.507465+0000 mon.a (mon.0) 278 : cluster 0 osdmap e80: 8 total, 8 up, 8 in 2023-10-01T10:27:22.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:22 smithi078 bash[78274]: cluster 2023-10-01T10:27:21.507780+0000 osd.2 (osd.2) 7 : cluster 3 failed to encode map e80 with expected crc 2023-10-01T10:27:22.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:22 smithi078 bash[78274]: cluster 2023-10-01T10:27:21.507804+0000 osd.3 (osd.3) 4 : cluster 3 failed to encode map e80 with expected crc 2023-10-01T10:27:22.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:22 smithi078 bash[78274]: cluster 2023-10-01T10:27:21.507922+0000 osd.1 (osd.1) 8 : cluster 3 failed to encode map e80 with expected crc 2023-10-01T10:27:22.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:22 smithi078 bash[78274]: cluster 2023-10-01T10:27:21.507941+0000 osd.0 (osd.0) 6 : cluster 3 failed to encode map e80 with expected crc 2023-10-01T10:27:22.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:22 smithi078 bash[78274]: cluster 2023-10-01T10:27:21.509949+0000 osd.5 (osd.5) 6 : cluster 3 failed to encode map e80 with expected crc 2023-10-01T10:27:22.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:22 smithi078 bash[78274]: cluster 2023-10-01T10:27:21.509952+0000 osd.4 (osd.4) 9 : cluster 3 failed to encode map e80 with expected crc 2023-10-01T10:27:22.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:22 smithi078 bash[78274]: cluster 2023-10-01T10:27:21.510077+0000 osd.6 (osd.6) 5 : cluster 3 failed to encode map e80 with expected crc 2023-10-01T10:27:22.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:22 smithi173 bash[58816]: cluster 2023-10-01T10:27:21.360657+0000 mgr.y (mgr.25072) 199 : cluster 0 pgmap v143: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:27:22.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:22 smithi173 bash[58816]: cluster 2023-10-01T10:27:21.504466+0000 osd.7 (osd.7) 6 : cluster 3 failed to encode map e80 with expected crc 2023-10-01T10:27:22.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:22 smithi173 bash[58816]: cluster 2023-10-01T10:27:21.505294+0000 osd.2 (osd.2) 6 : cluster 3 failed to encode map e80 with expected crc 2023-10-01T10:27:22.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:22 smithi173 bash[58816]: cluster 2023-10-01T10:27:21.507465+0000 mon.a (mon.0) 278 : cluster 0 osdmap e80: 8 total, 8 up, 8 in 2023-10-01T10:27:22.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:22 smithi173 bash[58816]: cluster 2023-10-01T10:27:21.507780+0000 osd.2 (osd.2) 7 : cluster 3 failed to encode map e80 with expected crc 2023-10-01T10:27:22.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:22 smithi173 bash[58816]: cluster 2023-10-01T10:27:21.507804+0000 osd.3 (osd.3) 4 : cluster 3 failed to encode map e80 with expected crc 2023-10-01T10:27:22.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:22 smithi173 bash[58816]: cluster 2023-10-01T10:27:21.507922+0000 osd.1 (osd.1) 8 : cluster 3 failed to encode map e80 with expected crc 2023-10-01T10:27:22.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:22 smithi173 bash[58816]: cluster 2023-10-01T10:27:21.507941+0000 osd.0 (osd.0) 6 : cluster 3 failed to encode map e80 with expected crc 2023-10-01T10:27:22.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:22 smithi173 bash[58816]: cluster 2023-10-01T10:27:21.509949+0000 osd.5 (osd.5) 6 : cluster 3 failed to encode map e80 with expected crc 2023-10-01T10:27:22.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:22 smithi173 bash[58816]: cluster 2023-10-01T10:27:21.509952+0000 osd.4 (osd.4) 9 : cluster 3 failed to encode map e80 with expected crc 2023-10-01T10:27:22.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:22 smithi173 bash[58816]: cluster 2023-10-01T10:27:21.510077+0000 osd.6 (osd.6) 5 : cluster 3 failed to encode map e80 with expected crc 2023-10-01T10:27:23.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:23 smithi078 bash[75299]: cluster 2023-10-01T10:27:22.507994+0000 osd.5 (osd.5) 7 : cluster 3 failed to encode map e81 with expected crc 2023-10-01T10:27:23.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:23 smithi078 bash[75299]: cluster 2023-10-01T10:27:22.508170+0000 osd.7 (osd.7) 7 : cluster 3 failed to encode map e81 with expected crc 2023-10-01T10:27:23.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:23 smithi078 bash[75299]: cluster 2023-10-01T10:27:22.508561+0000 mon.a (mon.0) 279 : cluster 0 osdmap e81: 8 total, 8 up, 8 in 2023-10-01T10:27:23.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:23 smithi078 bash[75299]: cluster 2023-10-01T10:27:23.361334+0000 mgr.y (mgr.25072) 200 : cluster 0 pgmap v146: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:27:23.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:23 smithi078 bash[78274]: cluster 2023-10-01T10:27:22.507994+0000 osd.5 (osd.5) 7 : cluster 3 failed to encode map e81 with expected crc 2023-10-01T10:27:23.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:23 smithi078 bash[78274]: cluster 2023-10-01T10:27:22.508170+0000 osd.7 (osd.7) 7 : cluster 3 failed to encode map e81 with expected crc 2023-10-01T10:27:23.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:23 smithi078 bash[78274]: cluster 2023-10-01T10:27:22.508561+0000 mon.a (mon.0) 279 : cluster 0 osdmap e81: 8 total, 8 up, 8 in 2023-10-01T10:27:23.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:23 smithi078 bash[78274]: cluster 2023-10-01T10:27:23.361334+0000 mgr.y (mgr.25072) 200 : cluster 0 pgmap v146: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:27:23.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:23 smithi173 bash[58816]: cluster 2023-10-01T10:27:22.507994+0000 osd.5 (osd.5) 7 : cluster 3 failed to encode map e81 with expected crc 2023-10-01T10:27:23.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:23 smithi173 bash[58816]: cluster 2023-10-01T10:27:22.508170+0000 osd.7 (osd.7) 7 : cluster 3 failed to encode map e81 with expected crc 2023-10-01T10:27:23.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:23 smithi173 bash[58816]: cluster 2023-10-01T10:27:22.508561+0000 mon.a (mon.0) 279 : cluster 0 osdmap e81: 8 total, 8 up, 8 in 2023-10-01T10:27:23.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:23 smithi173 bash[58816]: cluster 2023-10-01T10:27:23.361334+0000 mgr.y (mgr.25072) 200 : cluster 0 pgmap v146: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:27:26.161 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:27:26.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:26 smithi078 bash[75299]: cluster 2023-10-01T10:27:25.362349+0000 mgr.y (mgr.25072) 201 : cluster 0 pgmap v147: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:27:26.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:26 smithi078 bash[78274]: cluster 2023-10-01T10:27:25.362349+0000 mgr.y (mgr.25072) 201 : cluster 0 pgmap v147: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:27:26.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:26 smithi173 bash[58816]: cluster 2023-10-01T10:27:25.362349+0000 mgr.y (mgr.25072) 201 : cluster 0 pgmap v147: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:27:28.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:27 smithi078 bash[78274]: cluster 2023-10-01T10:27:27.362939+0000 mgr.y (mgr.25072) 202 : cluster 0 pgmap v148: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:27:28.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:27 smithi078 bash[75299]: cluster 2023-10-01T10:27:27.362939+0000 mgr.y (mgr.25072) 202 : cluster 0 pgmap v148: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:27:28.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:27 smithi173 bash[58816]: cluster 2023-10-01T10:27:27.362939+0000 mgr.y (mgr.25072) 202 : cluster 0 pgmap v148: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:27:29.552 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -e sha1=b56822191ad0416d0c05ed59720df03ad9c2a44e -- bash -c 'ceph orch upgrade resume' 2023-10-01T10:27:30.333 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:27:30 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:27:30] "GET /metrics HTTP/1.1" 200 33725 "" "Prometheus/2.33.4" 2023-10-01T10:27:30.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:30 smithi078 bash[75299]: cluster 2023-10-01T10:27:29.363871+0000 mgr.y (mgr.25072) 203 : cluster 0 pgmap v149: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:27:30.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:30 smithi078 bash[78274]: cluster 2023-10-01T10:27:29.363871+0000 mgr.y (mgr.25072) 203 : cluster 0 pgmap v149: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:27:30.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:30 smithi173 bash[58816]: cluster 2023-10-01T10:27:29.363871+0000 mgr.y (mgr.25072) 203 : cluster 0 pgmap v149: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:27:31.441 INFO:teuthology.orchestra.run.smithi078.stderr:enabled application 'rados' on pool 'unique_pool_2' 2023-10-01T10:27:31.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:31 smithi078 bash[75299]: audit 2023-10-01T10:27:30.489560+0000 mon.c (mon.2) 6 : audit 1 from='client.? 172.21.15.78:0/718707341' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T10:27:31.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:31 smithi078 bash[75299]: audit 2023-10-01T10:27:30.490082+0000 mon.a (mon.0) 280 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T10:27:31.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:31 smithi078 bash[78274]: audit 2023-10-01T10:27:30.489560+0000 mon.c (mon.2) 6 : audit 1 from='client.? 172.21.15.78:0/718707341' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T10:27:31.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:31 smithi078 bash[78274]: audit 2023-10-01T10:27:30.490082+0000 mon.a (mon.0) 280 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T10:27:31.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:31 smithi173 bash[58816]: audit 2023-10-01T10:27:30.489560+0000 mon.c (mon.2) 6 : audit 1 from='client.? 172.21.15.78:0/718707341' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T10:27:31.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:31 smithi173 bash[58816]: audit 2023-10-01T10:27:30.490082+0000 mon.a (mon.0) 280 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T10:27:32.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:32 smithi078 bash[75299]: cluster 2023-10-01T10:27:31.364405+0000 mgr.y (mgr.25072) 204 : cluster 0 pgmap v150: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:27:32.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:32 smithi078 bash[75299]: cluster 2023-10-01T10:27:31.429164+0000 osd.1 (osd.1) 9 : cluster 3 failed to encode map e82 with expected crc 2023-10-01T10:27:32.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:32 smithi078 bash[75299]: audit 2023-10-01T10:27:31.431078+0000 mon.a (mon.0) 281 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-01T10:27:32.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:32 smithi078 bash[75299]: cluster 2023-10-01T10:27:31.431176+0000 mon.a (mon.0) 282 : cluster 0 osdmap e82: 8 total, 8 up, 8 in 2023-10-01T10:27:32.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:32 smithi078 bash[75299]: cluster 2023-10-01T10:27:31.431415+0000 osd.7 (osd.7) 8 : cluster 3 failed to encode map e82 with expected crc 2023-10-01T10:27:32.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:32 smithi078 bash[75299]: cluster 2023-10-01T10:27:31.981462+0000 osd.0 (osd.0) 7 : cluster 3 failed to encode map e82 with expected crc 2023-10-01T10:27:32.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:32 smithi078 bash[75299]: audit 2023-10-01T10:27:32.386903+0000 mon.a (mon.0) 283 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:27:32.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:32 smithi078 bash[78274]: cluster 2023-10-01T10:27:31.364405+0000 mgr.y (mgr.25072) 204 : cluster 0 pgmap v150: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:27:32.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:32 smithi078 bash[78274]: cluster 2023-10-01T10:27:31.429164+0000 osd.1 (osd.1) 9 : cluster 3 failed to encode map e82 with expected crc 2023-10-01T10:27:32.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:32 smithi078 bash[78274]: audit 2023-10-01T10:27:31.431078+0000 mon.a (mon.0) 281 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-01T10:27:32.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:32 smithi078 bash[78274]: cluster 2023-10-01T10:27:31.431176+0000 mon.a (mon.0) 282 : cluster 0 osdmap e82: 8 total, 8 up, 8 in 2023-10-01T10:27:32.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:32 smithi078 bash[78274]: cluster 2023-10-01T10:27:31.431415+0000 osd.7 (osd.7) 8 : cluster 3 failed to encode map e82 with expected crc 2023-10-01T10:27:32.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:32 smithi078 bash[78274]: cluster 2023-10-01T10:27:31.981462+0000 osd.0 (osd.0) 7 : cluster 3 failed to encode map e82 with expected crc 2023-10-01T10:27:32.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:32 smithi078 bash[78274]: audit 2023-10-01T10:27:32.386903+0000 mon.a (mon.0) 283 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:27:32.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:32 smithi173 bash[58816]: cluster 2023-10-01T10:27:31.364405+0000 mgr.y (mgr.25072) 204 : cluster 0 pgmap v150: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:27:32.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:32 smithi173 bash[58816]: cluster 2023-10-01T10:27:31.429164+0000 osd.1 (osd.1) 9 : cluster 3 failed to encode map e82 with expected crc 2023-10-01T10:27:32.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:32 smithi173 bash[58816]: audit 2023-10-01T10:27:31.431078+0000 mon.a (mon.0) 281 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-01T10:27:32.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:32 smithi173 bash[58816]: cluster 2023-10-01T10:27:31.431176+0000 mon.a (mon.0) 282 : cluster 0 osdmap e82: 8 total, 8 up, 8 in 2023-10-01T10:27:32.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:32 smithi173 bash[58816]: cluster 2023-10-01T10:27:31.431415+0000 osd.7 (osd.7) 8 : cluster 3 failed to encode map e82 with expected crc 2023-10-01T10:27:32.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:32 smithi173 bash[58816]: cluster 2023-10-01T10:27:31.981462+0000 osd.0 (osd.0) 7 : cluster 3 failed to encode map e82 with expected crc 2023-10-01T10:27:32.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:32 smithi173 bash[58816]: audit 2023-10-01T10:27:32.386903+0000 mon.a (mon.0) 283 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:27:33.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:33 smithi078 bash[75299]: cluster 2023-10-01T10:27:32.099190+0000 osd.3 (osd.3) 5 : cluster 3 failed to encode map e82 with expected crc 2023-10-01T10:27:33.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:33 smithi078 bash[75299]: audit 2023-10-01T10:27:32.451535+0000 mon.a (mon.0) 284 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:33.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:33 smithi078 bash[75299]: cluster 2023-10-01T10:27:32.735635+0000 osd.5 (osd.5) 8 : cluster 3 failed to encode map e82 with expected crc 2023-10-01T10:27:33.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:33 smithi078 bash[78274]: cluster 2023-10-01T10:27:32.099190+0000 osd.3 (osd.3) 5 : cluster 3 failed to encode map e82 with expected crc 2023-10-01T10:27:33.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:33 smithi078 bash[78274]: audit 2023-10-01T10:27:32.451535+0000 mon.a (mon.0) 284 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:33.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:33 smithi078 bash[78274]: cluster 2023-10-01T10:27:32.735635+0000 osd.5 (osd.5) 8 : cluster 3 failed to encode map e82 with expected crc 2023-10-01T10:27:33.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:33 smithi173 bash[58816]: cluster 2023-10-01T10:27:32.099190+0000 osd.3 (osd.3) 5 : cluster 3 failed to encode map e82 with expected crc 2023-10-01T10:27:33.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:33 smithi173 bash[58816]: audit 2023-10-01T10:27:32.451535+0000 mon.a (mon.0) 284 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:33.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:33 smithi173 bash[58816]: cluster 2023-10-01T10:27:32.735635+0000 osd.5 (osd.5) 8 : cluster 3 failed to encode map e82 with expected crc 2023-10-01T10:27:34.067 INFO:teuthology.orchestra.run.smithi078.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T10:27:34.234 DEBUG:teuthology.orchestra.run.smithi078:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_2 bench 90 write' 2023-10-01T10:27:34.237 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-01T10:27:34.237 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-01T10:27:34.281 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:hints = 1 2023-10-01T10:27:34.281 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-01T10:27:34.281 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Object prefix: benchmark_data_smithi078_83488 2023-10-01T10:27:34.282 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T10:27:34.282 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 0 0 0 0 0 0 - 0 2023-10-01T10:27:34.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:34 smithi078 bash[78274]: cluster 2023-10-01T10:27:33.364871+0000 mgr.y (mgr.25072) 205 : cluster 0 pgmap v152: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:27:34.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:34 smithi078 bash[78274]: audit 2023-10-01T10:27:34.064219+0000 mon.a (mon.0) 285 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:34.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:34 smithi078 bash[78274]: audit 2023-10-01T10:27:34.094137+0000 mon.a (mon.0) 286 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:27:34.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:34 smithi078 bash[75299]: cluster 2023-10-01T10:27:33.364871+0000 mgr.y (mgr.25072) 205 : cluster 0 pgmap v152: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:27:34.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:34 smithi078 bash[75299]: audit 2023-10-01T10:27:34.064219+0000 mon.a (mon.0) 285 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:34.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:34 smithi078 bash[75299]: audit 2023-10-01T10:27:34.094137+0000 mon.a (mon.0) 286 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:27:34.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:34 smithi173 bash[58816]: cluster 2023-10-01T10:27:33.364871+0000 mgr.y (mgr.25072) 205 : cluster 0 pgmap v152: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:27:34.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:34 smithi173 bash[58816]: audit 2023-10-01T10:27:34.064219+0000 mon.a (mon.0) 285 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:34.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:34 smithi173 bash[58816]: audit 2023-10-01T10:27:34.094137+0000 mon.a (mon.0) 286 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:27:35.234 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -e sha1=b56822191ad0416d0c05ed59720df03ad9c2a44e -- bash -c 'echo wait for half of osds to upgrade' 2023-10-01T10:27:35.281 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 1 16 3021 3005 187.802 187.812 0.00588156 0.00529708 2023-10-01T10:27:35.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:35 smithi078 bash[78274]: audit 2023-10-01T10:27:34.058597+0000 mgr.y (mgr.25072) 206 : audit 0 from='client.34266 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:27:35.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:35 smithi078 bash[78274]: cephadm 2023-10-01T10:27:34.059237+0000 mgr.y (mgr.25072) 207 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T10:27:35.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:35 smithi078 bash[75299]: audit 2023-10-01T10:27:34.058597+0000 mgr.y (mgr.25072) 206 : audit 0 from='client.34266 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:27:35.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:35 smithi078 bash[75299]: cephadm 2023-10-01T10:27:34.059237+0000 mgr.y (mgr.25072) 207 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T10:27:35.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:35 smithi173 bash[58816]: audit 2023-10-01T10:27:34.058597+0000 mgr.y (mgr.25072) 206 : audit 0 from='client.34266 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:27:35.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:35 smithi173 bash[58816]: cephadm 2023-10-01T10:27:34.059237+0000 mgr.y (mgr.25072) 207 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T10:27:36.282 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 2 16 5555 5539 173.081 158.375 0.00564082 0.00576655 2023-10-01T10:27:36.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:36 smithi078 bash[78274]: cluster 2023-10-01T10:27:35.365675+0000 mgr.y (mgr.25072) 208 : cluster 0 pgmap v153: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:27:36.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:36 smithi078 bash[75299]: cluster 2023-10-01T10:27:35.365675+0000 mgr.y (mgr.25072) 208 : cluster 0 pgmap v153: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:27:36.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:36 smithi173 bash[58816]: cluster 2023-10-01T10:27:35.365675+0000 mgr.y (mgr.25072) 208 : cluster 0 pgmap v153: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:27:37.282 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 3 16 8163 8147 169.716 163 0.00401367 0.00588169 2023-10-01T10:27:38.026 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:37 smithi078 bash[75299]: cluster 2023-10-01T10:27:37.366247+0000 mgr.y (mgr.25072) 209 : cluster 0 pgmap v154: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:27:38.026 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:37 smithi078 bash[78274]: cluster 2023-10-01T10:27:37.366247+0000 mgr.y (mgr.25072) 209 : cluster 0 pgmap v154: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:27:38.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:37 smithi173 bash[58816]: cluster 2023-10-01T10:27:37.366247+0000 mgr.y (mgr.25072) 209 : cluster 0 pgmap v154: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:27:38.282 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 4 16 10753 10737 167.752 161.875 0.00477573 0.00595153 2023-10-01T10:27:39.282 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 5 16 13151 13135 164.174 149.875 0.00379391 0.00607946 2023-10-01T10:27:40.282 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 6 16 15374 15358 159.966 138.938 0.00413483 0.00624255 2023-10-01T10:27:40.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:40 smithi078 bash[75299]: cluster 2023-10-01T10:27:39.366929+0000 mgr.y (mgr.25072) 210 : cluster 0 pgmap v155: 89 pgs: 89 active+clean; 720 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-01T10:27:40.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:27:40 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:27:40] "GET /metrics HTTP/1.1" 200 36000 "" "Prometheus/2.33.4" 2023-10-01T10:27:40.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:40 smithi078 bash[78274]: cluster 2023-10-01T10:27:39.366929+0000 mgr.y (mgr.25072) 210 : cluster 0 pgmap v155: 89 pgs: 89 active+clean; 720 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-01T10:27:40.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:40 smithi173 bash[58816]: cluster 2023-10-01T10:27:39.366929+0000 mgr.y (mgr.25072) 210 : cluster 0 pgmap v155: 89 pgs: 89 active+clean; 720 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-01T10:27:41.254 INFO:teuthology.orchestra.run.smithi078.stdout:wait for half of osds to upgrade 2023-10-01T10:27:41.282 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 7 15 17370 17355 154.942 124.812 0.00547173 0.00644751 2023-10-01T10:27:41.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:41 smithi078 bash[75299]: audit 2023-10-01T10:27:40.589623+0000 mon.a (mon.0) 287 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:41.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:41 smithi078 bash[75299]: audit 2023-10-01T10:27:40.601589+0000 mon.a (mon.0) 288 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:41.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:41 smithi078 bash[75299]: audit 2023-10-01T10:27:41.222232+0000 mon.a (mon.0) 289 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:27:41.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:41 smithi078 bash[75299]: audit 2023-10-01T10:27:41.224421+0000 mon.a (mon.0) 290 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:27:41.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:41 smithi078 bash[75299]: audit 2023-10-01T10:27:41.236551+0000 mon.a (mon.0) 291 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:41.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:41 smithi078 bash[75299]: audit 2023-10-01T10:27:41.333511+0000 mon.a (mon.0) 292 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:27:41.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:41 smithi078 bash[75299]: audit 2023-10-01T10:27:41.337781+0000 mon.a (mon.0) 293 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:27:41.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:41 smithi078 bash[75299]: audit 2023-10-01T10:27:41.340970+0000 mon.a (mon.0) 294 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:27:41.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:41 smithi078 bash[75299]: cephadm 2023-10-01T10:27:41.343167+0000 mgr.y (mgr.25072) 211 : cephadm 1 Upgrade: Setting container_image for all mon 2023-10-01T10:27:41.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:41 smithi078 bash[75299]: audit 2023-10-01T10:27:41.357507+0000 mon.a (mon.0) 295 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:41.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:41 smithi078 bash[75299]: audit 2023-10-01T10:27:41.359239+0000 mon.a (mon.0) 296 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-10-01T10:27:41.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:41 smithi078 bash[75299]: cluster 2023-10-01T10:27:41.367611+0000 mgr.y (mgr.25072) 212 : cluster 0 pgmap v156: 89 pgs: 89 active+clean; 720 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-01T10:27:41.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:41 smithi078 bash[75299]: audit 2023-10-01T10:27:41.370596+0000 mon.a (mon.0) 297 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-10-01T10:27:41.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:41 smithi078 bash[75299]: audit 2023-10-01T10:27:41.374296+0000 mon.a (mon.0) 298 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-10-01T10:27:41.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:41 smithi078 bash[75299]: audit 2023-10-01T10:27:41.382947+0000 mon.a (mon.0) 299 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-10-01T10:27:41.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:41 smithi078 bash[75299]: audit 2023-10-01T10:27:41.384511+0000 mon.a (mon.0) 300 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-10-01T10:27:41.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:41 smithi078 bash[75299]: audit 2023-10-01T10:27:41.396838+0000 mon.a (mon.0) 301 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-10-01T10:27:41.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:41 smithi078 bash[75299]: audit 2023-10-01T10:27:41.399738+0000 mon.a (mon.0) 302 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:27:41.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:41 smithi078 bash[75299]: cephadm 2023-10-01T10:27:41.400941+0000 mgr.y (mgr.25072) 213 : cephadm 1 Upgrade: Setting container_image for all crash 2023-10-01T10:27:41.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:41 smithi078 bash[75299]: audit 2023-10-01T10:27:41.408374+0000 mon.a (mon.0) 303 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:41.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:41 smithi078 bash[75299]: audit 2023-10-01T10:27:41.411845+0000 mon.a (mon.0) 304 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-01T10:27:41.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:41 smithi078 bash[75299]: audit 2023-10-01T10:27:41.412096+0000 mgr.y (mgr.25072) 214 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-01T10:27:41.836 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:41 smithi078 bash[75299]: cephadm 2023-10-01T10:27:41.413641+0000 mgr.y (mgr.25072) 215 : cephadm 1 Upgrade: osd.0 is safe to restart 2023-10-01T10:27:41.836 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:41 smithi078 bash[78274]: audit 2023-10-01T10:27:40.589623+0000 mon.a (mon.0) 287 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:41.836 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:41 smithi078 bash[78274]: audit 2023-10-01T10:27:40.601589+0000 mon.a (mon.0) 288 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:41.836 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:41 smithi078 bash[78274]: audit 2023-10-01T10:27:41.222232+0000 mon.a (mon.0) 289 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:27:41.836 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:41 smithi078 bash[78274]: audit 2023-10-01T10:27:41.224421+0000 mon.a (mon.0) 290 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:27:41.836 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:41 smithi078 bash[78274]: audit 2023-10-01T10:27:41.236551+0000 mon.a (mon.0) 291 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:41.836 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:41 smithi078 bash[78274]: audit 2023-10-01T10:27:41.333511+0000 mon.a (mon.0) 292 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:27:41.836 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:41 smithi078 bash[78274]: audit 2023-10-01T10:27:41.337781+0000 mon.a (mon.0) 293 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:27:41.836 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:41 smithi078 bash[78274]: audit 2023-10-01T10:27:41.340970+0000 mon.a (mon.0) 294 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:27:41.836 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:41 smithi078 bash[78274]: cephadm 2023-10-01T10:27:41.343167+0000 mgr.y (mgr.25072) 211 : cephadm 1 Upgrade: Setting container_image for all mon 2023-10-01T10:27:41.836 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:41 smithi078 bash[78274]: audit 2023-10-01T10:27:41.357507+0000 mon.a (mon.0) 295 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:41.837 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:41 smithi078 bash[78274]: audit 2023-10-01T10:27:41.359239+0000 mon.a (mon.0) 296 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-10-01T10:27:41.837 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:41 smithi078 bash[78274]: cluster 2023-10-01T10:27:41.367611+0000 mgr.y (mgr.25072) 212 : cluster 0 pgmap v156: 89 pgs: 89 active+clean; 720 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-01T10:27:41.837 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:41 smithi078 bash[78274]: audit 2023-10-01T10:27:41.370596+0000 mon.a (mon.0) 297 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-10-01T10:27:41.837 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:41 smithi078 bash[78274]: audit 2023-10-01T10:27:41.374296+0000 mon.a (mon.0) 298 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-10-01T10:27:41.837 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:41 smithi078 bash[78274]: audit 2023-10-01T10:27:41.382947+0000 mon.a (mon.0) 299 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-10-01T10:27:41.837 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:41 smithi078 bash[78274]: audit 2023-10-01T10:27:41.384511+0000 mon.a (mon.0) 300 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-10-01T10:27:41.837 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:41 smithi078 bash[78274]: audit 2023-10-01T10:27:41.396838+0000 mon.a (mon.0) 301 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-10-01T10:27:41.837 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:41 smithi078 bash[78274]: audit 2023-10-01T10:27:41.399738+0000 mon.a (mon.0) 302 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:27:41.837 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:41 smithi078 bash[78274]: cephadm 2023-10-01T10:27:41.400941+0000 mgr.y (mgr.25072) 213 : cephadm 1 Upgrade: Setting container_image for all crash 2023-10-01T10:27:41.837 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:41 smithi078 bash[78274]: audit 2023-10-01T10:27:41.408374+0000 mon.a (mon.0) 303 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:41.837 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:41 smithi078 bash[78274]: audit 2023-10-01T10:27:41.411845+0000 mon.a (mon.0) 304 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-01T10:27:41.837 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:41 smithi078 bash[78274]: audit 2023-10-01T10:27:41.412096+0000 mgr.y (mgr.25072) 214 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-01T10:27:41.838 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:41 smithi078 bash[78274]: cephadm 2023-10-01T10:27:41.413641+0000 mgr.y (mgr.25072) 215 : cephadm 1 Upgrade: osd.0 is safe to restart 2023-10-01T10:27:41.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:41 smithi173 bash[58816]: audit 2023-10-01T10:27:40.589623+0000 mon.a (mon.0) 287 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:41.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:41 smithi173 bash[58816]: audit 2023-10-01T10:27:40.601589+0000 mon.a (mon.0) 288 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:41.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:41 smithi173 bash[58816]: audit 2023-10-01T10:27:41.222232+0000 mon.a (mon.0) 289 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:27:41.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:41 smithi173 bash[58816]: audit 2023-10-01T10:27:41.224421+0000 mon.a (mon.0) 290 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:27:41.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:41 smithi173 bash[58816]: audit 2023-10-01T10:27:41.236551+0000 mon.a (mon.0) 291 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:41.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:41 smithi173 bash[58816]: audit 2023-10-01T10:27:41.333511+0000 mon.a (mon.0) 292 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:27:41.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:41 smithi173 bash[58816]: audit 2023-10-01T10:27:41.337781+0000 mon.a (mon.0) 293 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:27:41.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:41 smithi173 bash[58816]: audit 2023-10-01T10:27:41.340970+0000 mon.a (mon.0) 294 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:27:41.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:41 smithi173 bash[58816]: cephadm 2023-10-01T10:27:41.343167+0000 mgr.y (mgr.25072) 211 : cephadm 1 Upgrade: Setting container_image for all mon 2023-10-01T10:27:41.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:41 smithi173 bash[58816]: audit 2023-10-01T10:27:41.357507+0000 mon.a (mon.0) 295 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:41.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:41 smithi173 bash[58816]: audit 2023-10-01T10:27:41.359239+0000 mon.a (mon.0) 296 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-10-01T10:27:41.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:41 smithi173 bash[58816]: cluster 2023-10-01T10:27:41.367611+0000 mgr.y (mgr.25072) 212 : cluster 0 pgmap v156: 89 pgs: 89 active+clean; 720 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-01T10:27:41.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:41 smithi173 bash[58816]: audit 2023-10-01T10:27:41.370596+0000 mon.a (mon.0) 297 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-10-01T10:27:41.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:41 smithi173 bash[58816]: audit 2023-10-01T10:27:41.374296+0000 mon.a (mon.0) 298 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-10-01T10:27:41.872 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:41 smithi173 bash[58816]: audit 2023-10-01T10:27:41.382947+0000 mon.a (mon.0) 299 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-10-01T10:27:41.872 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:41 smithi173 bash[58816]: audit 2023-10-01T10:27:41.384511+0000 mon.a (mon.0) 300 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-10-01T10:27:41.872 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:41 smithi173 bash[58816]: audit 2023-10-01T10:27:41.396838+0000 mon.a (mon.0) 301 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-10-01T10:27:41.872 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:41 smithi173 bash[58816]: audit 2023-10-01T10:27:41.399738+0000 mon.a (mon.0) 302 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:27:41.872 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:41 smithi173 bash[58816]: cephadm 2023-10-01T10:27:41.400941+0000 mgr.y (mgr.25072) 213 : cephadm 1 Upgrade: Setting container_image for all crash 2023-10-01T10:27:41.872 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:41 smithi173 bash[58816]: audit 2023-10-01T10:27:41.408374+0000 mon.a (mon.0) 303 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:41.872 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:41 smithi173 bash[58816]: audit 2023-10-01T10:27:41.411845+0000 mon.a (mon.0) 304 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-01T10:27:41.872 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:41 smithi173 bash[58816]: audit 2023-10-01T10:27:41.412096+0000 mgr.y (mgr.25072) 214 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-10-01T10:27:41.872 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:41 smithi173 bash[58816]: cephadm 2023-10-01T10:27:41.413641+0000 mgr.y (mgr.25072) 215 : cephadm 1 Upgrade: osd.0 is safe to restart 2023-10-01T10:27:42.282 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 8 16 19249 19233 150.245 117.375 0.00679582 0.00664765 2023-10-01T10:27:43.282 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 9 15 21256 21241 147.494 125.5 0.0137578 0.00677304 2023-10-01T10:27:43.554 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -e sha1=b56822191ad0416d0c05ed59720df03ad9c2a44e -- bash -c 'while ! ceph osd versions | grep $sha1 | egrep '"'"': [45678]'"'"'; do sleep 2 ; done' 2023-10-01T10:27:44.282 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 10 16 23054 23038 143.975 112.312 0.00338121 0.00693899 2023-10-01T10:27:44.602 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:44 smithi078 bash[75299]: cluster 2023-10-01T10:27:43.368045+0000 mgr.y (mgr.25072) 216 : cluster 0 pgmap v157: 89 pgs: 89 active+clean; 720 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 60 MiB/s wr, 964 op/s 2023-10-01T10:27:44.602 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:44 smithi078 bash[78274]: cluster 2023-10-01T10:27:43.368045+0000 mgr.y (mgr.25072) 216 : cluster 0 pgmap v157: 89 pgs: 89 active+clean; 720 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 60 MiB/s wr, 964 op/s 2023-10-01T10:27:44.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:44 smithi173 bash[58816]: cluster 2023-10-01T10:27:43.368045+0000 mgr.y (mgr.25072) 216 : cluster 0 pgmap v157: 89 pgs: 89 active+clean; 720 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 60 MiB/s wr, 964 op/s 2023-10-01T10:27:45.283 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 11 16 24696 24680 140.215 102.625 0.00415233 0.00712457 2023-10-01T10:27:46.283 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 12 16 25896 25880 134.78 75 0.0089391 0.00741289 2023-10-01T10:27:46.739 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:46 smithi078 bash[75299]: cluster 2023-10-01T10:27:45.369103+0000 mgr.y (mgr.25072) 217 : cluster 0 pgmap v158: 89 pgs: 89 active+clean; 1.3 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-10-01T10:27:46.740 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:46 smithi078 bash[78274]: cluster 2023-10-01T10:27:45.369103+0000 mgr.y (mgr.25072) 217 : cluster 0 pgmap v158: 89 pgs: 89 active+clean; 1.3 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-10-01T10:27:46.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:46 smithi173 bash[58816]: cluster 2023-10-01T10:27:45.369103+0000 mgr.y (mgr.25072) 217 : cluster 0 pgmap v158: 89 pgs: 89 active+clean; 1.3 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-10-01T10:27:47.283 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 13 16 27555 27539 132.387 103.688 0.00615266 0.00754725 2023-10-01T10:27:48.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:47 smithi078 bash[75299]: cephadm 2023-10-01T10:27:46.694699+0000 mgr.y (mgr.25072) 218 : cephadm 1 Upgrade: Updating osd.0 2023-10-01T10:27:48.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:47 smithi078 bash[75299]: audit 2023-10-01T10:27:46.707638+0000 mon.a (mon.0) 305 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:48.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:47 smithi078 bash[75299]: audit 2023-10-01T10:27:46.708974+0000 mon.a (mon.0) 306 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-01T10:27:48.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:47 smithi078 bash[75299]: audit 2023-10-01T10:27:46.710184+0000 mon.a (mon.0) 307 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:27:48.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:47 smithi078 bash[75299]: cephadm 2023-10-01T10:27:46.713548+0000 mgr.y (mgr.25072) 219 : cephadm 1 Deploying daemon osd.0 on smithi078 2023-10-01T10:27:48.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:47 smithi078 bash[75299]: cluster 2023-10-01T10:27:47.369641+0000 mgr.y (mgr.25072) 220 : cluster 0 pgmap v159: 89 pgs: 89 active+clean; 1.3 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-10-01T10:27:48.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:47 smithi078 bash[75299]: audit 2023-10-01T10:27:47.387124+0000 mon.a (mon.0) 308 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:27:48.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:47 smithi078 bash[78274]: cephadm 2023-10-01T10:27:46.694699+0000 mgr.y (mgr.25072) 218 : cephadm 1 Upgrade: Updating osd.0 2023-10-01T10:27:48.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:47 smithi078 bash[78274]: audit 2023-10-01T10:27:46.707638+0000 mon.a (mon.0) 305 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:48.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:47 smithi078 bash[78274]: audit 2023-10-01T10:27:46.708974+0000 mon.a (mon.0) 306 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-01T10:27:48.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:47 smithi078 bash[78274]: audit 2023-10-01T10:27:46.710184+0000 mon.a (mon.0) 307 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:27:48.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:47 smithi078 bash[78274]: cephadm 2023-10-01T10:27:46.713548+0000 mgr.y (mgr.25072) 219 : cephadm 1 Deploying daemon osd.0 on smithi078 2023-10-01T10:27:48.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:47 smithi078 bash[78274]: cluster 2023-10-01T10:27:47.369641+0000 mgr.y (mgr.25072) 220 : cluster 0 pgmap v159: 89 pgs: 89 active+clean; 1.3 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-10-01T10:27:48.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:47 smithi078 bash[78274]: audit 2023-10-01T10:27:47.387124+0000 mon.a (mon.0) 308 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:27:48.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:47 smithi173 bash[58816]: cephadm 2023-10-01T10:27:46.694699+0000 mgr.y (mgr.25072) 218 : cephadm 1 Upgrade: Updating osd.0 2023-10-01T10:27:48.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:47 smithi173 bash[58816]: audit 2023-10-01T10:27:46.707638+0000 mon.a (mon.0) 305 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:27:48.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:47 smithi173 bash[58816]: audit 2023-10-01T10:27:46.708974+0000 mon.a (mon.0) 306 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-10-01T10:27:48.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:47 smithi173 bash[58816]: audit 2023-10-01T10:27:46.710184+0000 mon.a (mon.0) 307 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:27:48.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:47 smithi173 bash[58816]: cephadm 2023-10-01T10:27:46.713548+0000 mgr.y (mgr.25072) 219 : cephadm 1 Deploying daemon osd.0 on smithi078 2023-10-01T10:27:48.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:47 smithi173 bash[58816]: cluster 2023-10-01T10:27:47.369641+0000 mgr.y (mgr.25072) 220 : cluster 0 pgmap v159: 89 pgs: 89 active+clean; 1.3 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-10-01T10:27:48.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:47 smithi173 bash[58816]: audit 2023-10-01T10:27:47.387124+0000 mon.a (mon.0) 308 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:27:48.285 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 14 15 29373 29358 131.051 113.688 0.0066442 0.00762413 2023-10-01T10:27:49.283 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 15 16 31157 31141 129.743 111.438 0.0022859 0.0076909 2023-10-01T10:27:50.283 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 16 16 32629 32613 127.384 92 0.0056552 0.00784448 2023-10-01T10:27:50.557 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:50 smithi078 bash[75299]: cluster 2023-10-01T10:27:49.370429+0000 mgr.y (mgr.25072) 221 : cluster 0 pgmap v160: 89 pgs: 89 active+clean; 1.8 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 156 MiB/s wr, 2.50k op/s 2023-10-01T10:27:50.558 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:27:50 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:27:50] "GET /metrics HTTP/1.1" 200 36059 "" "Prometheus/2.33.4" 2023-10-01T10:27:50.558 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:50 smithi078 bash[78274]: cluster 2023-10-01T10:27:49.370429+0000 mgr.y (mgr.25072) 221 : cluster 0 pgmap v160: 89 pgs: 89 active+clean; 1.8 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 156 MiB/s wr, 2.50k op/s 2023-10-01T10:27:50.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:50 smithi173 bash[58816]: cluster 2023-10-01T10:27:49.370429+0000 mgr.y (mgr.25072) 221 : cluster 0 pgmap v160: 89 pgs: 89 active+clean; 1.8 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 156 MiB/s wr, 2.50k op/s 2023-10-01T10:27:51.283 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 17 16 34145 34129 125.463 94.75 0.00934784 0.00796482 2023-10-01T10:27:52.283 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 18 16 35469 35453 123.09 82.75 0.011544 0.00811751 2023-10-01T10:27:52.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:52 smithi078 bash[75299]: cluster 2023-10-01T10:27:51.370825+0000 mgr.y (mgr.25072) 222 : cluster 0 pgmap v161: 89 pgs: 89 active+clean; 1.8 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s 2023-10-01T10:27:52.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:52 smithi078 bash[75299]: audit 2023-10-01T10:27:51.648127+0000 mon.c (mon.2) 7 : audit 0 from='client.? 172.21.15.78:0/1134620719' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:27:52.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:52 smithi078 bash[78274]: cluster 2023-10-01T10:27:51.370825+0000 mgr.y (mgr.25072) 222 : cluster 0 pgmap v161: 89 pgs: 89 active+clean; 1.8 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s 2023-10-01T10:27:52.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:52 smithi078 bash[78274]: audit 2023-10-01T10:27:51.648127+0000 mon.c (mon.2) 7 : audit 0 from='client.? 172.21.15.78:0/1134620719' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:27:52.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:52 smithi173 bash[58816]: cluster 2023-10-01T10:27:51.370825+0000 mgr.y (mgr.25072) 222 : cluster 0 pgmap v161: 89 pgs: 89 active+clean; 1.8 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s 2023-10-01T10:27:52.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:52 smithi173 bash[58816]: audit 2023-10-01T10:27:51.648127+0000 mon.c (mon.2) 7 : audit 0 from='client.? 172.21.15.78:0/1134620719' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:27:53.283 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 19 16 37106 37090 121.996 102.312 0.00536842 0.00819138 2023-10-01T10:27:54.283 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:2023-10-01T10:27:54.281655+0000 min lat: 0.00149611 max lat: 0.210688 avg lat: 0.00831206 2023-10-01T10:27:54.283 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T10:27:54.284 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 20 16 38488 38472 120.214 86.375 0.0119555 0.00831206 2023-10-01T10:27:54.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:54 smithi078 bash[75299]: cluster 2023-10-01T10:27:53.371233+0000 mgr.y (mgr.25072) 223 : cluster 0 pgmap v162: 89 pgs: 89 active+clean; 1.8 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s 2023-10-01T10:27:54.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:54 smithi078 bash[75299]: audit 2023-10-01T10:27:54.233669+0000 mon.c (mon.2) 8 : audit 0 from='client.? 172.21.15.78:0/3651060246' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:27:54.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:54 smithi078 bash[78274]: cluster 2023-10-01T10:27:53.371233+0000 mgr.y (mgr.25072) 223 : cluster 0 pgmap v162: 89 pgs: 89 active+clean; 1.8 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s 2023-10-01T10:27:54.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:54 smithi078 bash[78274]: audit 2023-10-01T10:27:54.233669+0000 mon.c (mon.2) 8 : audit 0 from='client.? 172.21.15.78:0/3651060246' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:27:54.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:54 smithi173 bash[58816]: cluster 2023-10-01T10:27:53.371233+0000 mgr.y (mgr.25072) 223 : cluster 0 pgmap v162: 89 pgs: 89 active+clean; 1.8 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s 2023-10-01T10:27:54.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:54 smithi173 bash[58816]: audit 2023-10-01T10:27:54.233669+0000 mon.c (mon.2) 8 : audit 0 from='client.? 172.21.15.78:0/3651060246' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:27:55.283 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 21 16 39962 39946 118.876 92.125 0.0127821 0.00840512 2023-10-01T10:27:56.284 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 22 16 41297 41281 117.265 83.4375 0.00888738 0.00851962 2023-10-01T10:27:56.333 INFO:journalctl@ceph.osd.0.smithi078.stdout:Oct 01 10:27:56 smithi078 systemd[1]: Stopping Ceph osd.0 for d8016480-6041-11ee-8db5-212e2dc638e7... 2023-10-01T10:27:56.333 INFO:journalctl@ceph.osd.0.smithi078.stdout:Oct 01 10:27:56 smithi078 bash[36963]: debug 2023-10-01T10:27:56.224+0000 7fc73ba77700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-01T10:27:56.333 INFO:journalctl@ceph.osd.0.smithi078.stdout:Oct 01 10:27:56 smithi078 bash[36963]: debug 2023-10-01T10:27:56.224+0000 7fc73ba77700 -1 osd.0 82 *** Got signal Terminated *** 2023-10-01T10:27:56.333 INFO:journalctl@ceph.osd.0.smithi078.stdout:Oct 01 10:27:56 smithi078 bash[36963]: debug 2023-10-01T10:27:56.224+0000 7fc73ba77700 -1 osd.0 82 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-01T10:27:56.771 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:56 smithi078 bash[75299]: cluster 2023-10-01T10:27:55.372008+0000 mgr.y (mgr.25072) 224 : cluster 0 pgmap v163: 89 pgs: 89 active+clean; 2.3 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 136 MiB/s wr, 2.17k op/s 2023-10-01T10:27:56.772 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:56 smithi078 bash[75299]: cluster 2023-10-01T10:27:56.233892+0000 mon.a (mon.0) 309 : cluster 1 osd.0 marked itself down and dead 2023-10-01T10:27:56.772 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:56 smithi078 bash[78274]: cluster 2023-10-01T10:27:55.372008+0000 mgr.y (mgr.25072) 224 : cluster 0 pgmap v163: 89 pgs: 89 active+clean; 2.3 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 136 MiB/s wr, 2.17k op/s 2023-10-01T10:27:56.772 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:56 smithi078 bash[78274]: cluster 2023-10-01T10:27:56.233892+0000 mon.a (mon.0) 309 : cluster 1 osd.0 marked itself down and dead 2023-10-01T10:27:56.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:56 smithi173 bash[58816]: cluster 2023-10-01T10:27:55.372008+0000 mgr.y (mgr.25072) 224 : cluster 0 pgmap v163: 89 pgs: 89 active+clean; 2.3 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 136 MiB/s wr, 2.17k op/s 2023-10-01T10:27:56.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:56 smithi173 bash[58816]: cluster 2023-10-01T10:27:56.233892+0000 mon.a (mon.0) 309 : cluster 1 osd.0 marked itself down and dead 2023-10-01T10:27:57.284 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 23 16 41534 41518 112.81 14.8125 0.0314562 0.00854664 2023-10-01T10:27:57.705 INFO:journalctl@ceph.osd.0.smithi078.stdout:Oct 01 10:27:57 smithi078 bash[84165]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd-0 2023-10-01T10:27:57.705 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:57 smithi078 bash[78274]: cluster 2023-10-01T10:27:56.436150+0000 mon.a (mon.0) 310 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-01T10:27:57.705 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:57 smithi078 bash[78274]: cluster 2023-10-01T10:27:56.456176+0000 osd.3 (osd.3) 6 : cluster 3 failed to encode map e83 with expected crc 2023-10-01T10:27:57.705 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:57 smithi078 bash[78274]: cluster 2023-10-01T10:27:56.459736+0000 osd.7 (osd.7) 9 : cluster 3 failed to encode map e83 with expected crc 2023-10-01T10:27:57.705 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:57 smithi078 bash[78274]: cluster 2023-10-01T10:27:56.463917+0000 mon.a (mon.0) 311 : cluster 0 osdmap e83: 8 total, 7 up, 8 in 2023-10-01T10:27:57.706 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:57 smithi078 bash[78274]: audit 2023-10-01T10:27:56.688320+0000 mon.b (mon.1) 3 : audit 0 from='client.? 172.21.15.78:0/1867848860' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:27:57.706 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:57 smithi078 bash[75299]: cluster 2023-10-01T10:27:56.436150+0000 mon.a (mon.0) 310 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-01T10:27:57.706 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:57 smithi078 bash[75299]: cluster 2023-10-01T10:27:56.456176+0000 osd.3 (osd.3) 6 : cluster 3 failed to encode map e83 with expected crc 2023-10-01T10:27:57.706 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:57 smithi078 bash[75299]: cluster 2023-10-01T10:27:56.459736+0000 osd.7 (osd.7) 9 : cluster 3 failed to encode map e83 with expected crc 2023-10-01T10:27:57.706 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:57 smithi078 bash[75299]: cluster 2023-10-01T10:27:56.463917+0000 mon.a (mon.0) 311 : cluster 0 osdmap e83: 8 total, 7 up, 8 in 2023-10-01T10:27:57.707 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:57 smithi078 bash[75299]: audit 2023-10-01T10:27:56.688320+0000 mon.b (mon.1) 3 : audit 0 from='client.? 172.21.15.78:0/1867848860' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:27:57.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:57 smithi173 bash[58816]: cluster 2023-10-01T10:27:56.436150+0000 mon.a (mon.0) 310 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-01T10:27:57.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:57 smithi173 bash[58816]: cluster 2023-10-01T10:27:56.456176+0000 osd.3 (osd.3) 6 : cluster 3 failed to encode map e83 with expected crc 2023-10-01T10:27:57.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:57 smithi173 bash[58816]: cluster 2023-10-01T10:27:56.459736+0000 osd.7 (osd.7) 9 : cluster 3 failed to encode map e83 with expected crc 2023-10-01T10:27:57.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:57 smithi173 bash[58816]: cluster 2023-10-01T10:27:56.463917+0000 mon.a (mon.0) 311 : cluster 0 osdmap e83: 8 total, 7 up, 8 in 2023-10-01T10:27:57.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:57 smithi173 bash[58816]: audit 2023-10-01T10:27:56.688320+0000 mon.b (mon.1) 3 : audit 0 from='client.? 172.21.15.78:0/1867848860' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:27:58.083 INFO:journalctl@ceph.osd.0.smithi078.stdout:Oct 01 10:27:57 smithi078 bash[84263]: Error response from daemon: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd-0 2023-10-01T10:27:58.284 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 24 16 42956 42940 111.812 88.875 0.0123945 0.00893793 2023-10-01T10:27:58.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:58 smithi078 bash[75299]: cluster 2023-10-01T10:27:57.372581+0000 mgr.y (mgr.25072) 225 : cluster 0 pgmap v165: 89 pgs: 15 stale+active+clean, 74 active+clean; 2.3 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2023-10-01T10:27:58.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:58 smithi078 bash[75299]: cluster 2023-10-01T10:27:57.453714+0000 osd.3 (osd.3) 7 : cluster 3 failed to encode map e84 with expected crc 2023-10-01T10:27:58.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:58 smithi078 bash[75299]: cluster 2023-10-01T10:27:57.456033+0000 osd.7 (osd.7) 10 : cluster 3 failed to encode map e84 with expected crc 2023-10-01T10:27:58.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:58 smithi078 bash[75299]: cluster 2023-10-01T10:27:57.456734+0000 mon.a (mon.0) 312 : cluster 0 osdmap e84: 8 total, 7 up, 8 in 2023-10-01T10:27:58.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:58 smithi078 bash[75299]: cluster 2023-10-01T10:27:57.456883+0000 osd.3 (osd.3) 8 : cluster 3 failed to encode map e84 with expected crc 2023-10-01T10:27:58.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:58 smithi078 bash[75299]: cluster 2023-10-01T10:27:57.456999+0000 osd.2 (osd.2) 8 : cluster 3 failed to encode map e84 with expected crc 2023-10-01T10:27:58.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:58 smithi078 bash[75299]: cluster 2023-10-01T10:27:57.457894+0000 osd.1 (osd.1) 10 : cluster 3 failed to encode map e84 with expected crc 2023-10-01T10:27:58.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:58 smithi078 bash[75299]: cluster 2023-10-01T10:27:57.458970+0000 osd.5 (osd.5) 9 : cluster 3 failed to encode map e84 with expected crc 2023-10-01T10:27:58.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:58 smithi078 bash[75299]: cluster 2023-10-01T10:27:57.459249+0000 osd.7 (osd.7) 11 : cluster 3 failed to encode map e84 with expected crc 2023-10-01T10:27:58.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:58 smithi078 bash[75299]: cluster 2023-10-01T10:27:57.459339+0000 osd.6 (osd.6) 6 : cluster 3 failed to encode map e84 with expected crc 2023-10-01T10:27:58.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:58 smithi078 bash[75299]: cluster 2023-10-01T10:27:57.461482+0000 osd.4 (osd.4) 10 : cluster 3 failed to encode map e84 with expected crc 2023-10-01T10:27:58.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:58 smithi078 bash[78274]: cluster 2023-10-01T10:27:57.372581+0000 mgr.y (mgr.25072) 225 : cluster 0 pgmap v165: 89 pgs: 15 stale+active+clean, 74 active+clean; 2.3 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2023-10-01T10:27:58.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:58 smithi078 bash[78274]: cluster 2023-10-01T10:27:57.453714+0000 osd.3 (osd.3) 7 : cluster 3 failed to encode map e84 with expected crc 2023-10-01T10:27:58.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:58 smithi078 bash[78274]: cluster 2023-10-01T10:27:57.456033+0000 osd.7 (osd.7) 10 : cluster 3 failed to encode map e84 with expected crc 2023-10-01T10:27:58.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:58 smithi078 bash[78274]: cluster 2023-10-01T10:27:57.456734+0000 mon.a (mon.0) 312 : cluster 0 osdmap e84: 8 total, 7 up, 8 in 2023-10-01T10:27:58.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:58 smithi078 bash[78274]: cluster 2023-10-01T10:27:57.456883+0000 osd.3 (osd.3) 8 : cluster 3 failed to encode map e84 with expected crc 2023-10-01T10:27:58.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:58 smithi078 bash[78274]: cluster 2023-10-01T10:27:57.456999+0000 osd.2 (osd.2) 8 : cluster 3 failed to encode map e84 with expected crc 2023-10-01T10:27:58.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:58 smithi078 bash[78274]: cluster 2023-10-01T10:27:57.457894+0000 osd.1 (osd.1) 10 : cluster 3 failed to encode map e84 with expected crc 2023-10-01T10:27:58.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:58 smithi078 bash[78274]: cluster 2023-10-01T10:27:57.458970+0000 osd.5 (osd.5) 9 : cluster 3 failed to encode map e84 with expected crc 2023-10-01T10:27:58.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:58 smithi078 bash[78274]: cluster 2023-10-01T10:27:57.459249+0000 osd.7 (osd.7) 11 : cluster 3 failed to encode map e84 with expected crc 2023-10-01T10:27:58.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:58 smithi078 bash[78274]: cluster 2023-10-01T10:27:57.459339+0000 osd.6 (osd.6) 6 : cluster 3 failed to encode map e84 with expected crc 2023-10-01T10:27:58.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:58 smithi078 bash[78274]: cluster 2023-10-01T10:27:57.461482+0000 osd.4 (osd.4) 10 : cluster 3 failed to encode map e84 with expected crc 2023-10-01T10:27:58.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:58 smithi173 bash[58816]: cluster 2023-10-01T10:27:57.372581+0000 mgr.y (mgr.25072) 225 : cluster 0 pgmap v165: 89 pgs: 15 stale+active+clean, 74 active+clean; 2.3 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2023-10-01T10:27:58.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:58 smithi173 bash[58816]: cluster 2023-10-01T10:27:57.453714+0000 osd.3 (osd.3) 7 : cluster 3 failed to encode map e84 with expected crc 2023-10-01T10:27:58.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:58 smithi173 bash[58816]: cluster 2023-10-01T10:27:57.456033+0000 osd.7 (osd.7) 10 : cluster 3 failed to encode map e84 with expected crc 2023-10-01T10:27:58.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:58 smithi173 bash[58816]: cluster 2023-10-01T10:27:57.456734+0000 mon.a (mon.0) 312 : cluster 0 osdmap e84: 8 total, 7 up, 8 in 2023-10-01T10:27:58.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:58 smithi173 bash[58816]: cluster 2023-10-01T10:27:57.456883+0000 osd.3 (osd.3) 8 : cluster 3 failed to encode map e84 with expected crc 2023-10-01T10:27:58.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:58 smithi173 bash[58816]: cluster 2023-10-01T10:27:57.456999+0000 osd.2 (osd.2) 8 : cluster 3 failed to encode map e84 with expected crc 2023-10-01T10:27:58.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:58 smithi173 bash[58816]: cluster 2023-10-01T10:27:57.457894+0000 osd.1 (osd.1) 10 : cluster 3 failed to encode map e84 with expected crc 2023-10-01T10:27:58.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:58 smithi173 bash[58816]: cluster 2023-10-01T10:27:57.458970+0000 osd.5 (osd.5) 9 : cluster 3 failed to encode map e84 with expected crc 2023-10-01T10:27:58.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:58 smithi173 bash[58816]: cluster 2023-10-01T10:27:57.459249+0000 osd.7 (osd.7) 11 : cluster 3 failed to encode map e84 with expected crc 2023-10-01T10:27:58.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:58 smithi173 bash[58816]: cluster 2023-10-01T10:27:57.459339+0000 osd.6 (osd.6) 6 : cluster 3 failed to encode map e84 with expected crc 2023-10-01T10:27:58.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:58 smithi173 bash[58816]: cluster 2023-10-01T10:27:57.461482+0000 osd.4 (osd.4) 10 : cluster 3 failed to encode map e84 with expected crc 2023-10-01T10:27:59.284 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 25 16 44658 44642 111.595 106.375 0.00736186 0.00895527 2023-10-01T10:27:59.471 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:27:59 smithi078 bash[75299]: audit 2023-10-01T10:27:59.225149+0000 mon.b (mon.1) 4 : audit 0 from='client.? 172.21.15.78:0/4117105059' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:27:59.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:27:59 smithi078 bash[78274]: audit 2023-10-01T10:27:59.225149+0000 mon.b (mon.1) 4 : audit 0 from='client.? 172.21.15.78:0/4117105059' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:27:59.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:27:59 smithi173 bash[58816]: audit 2023-10-01T10:27:59.225149+0000 mon.b (mon.1) 4 : audit 0 from='client.? 172.21.15.78:0/4117105059' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:00.270 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:28:00 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:28:00] "GET /metrics HTTP/1.1" 200 36059 "" "Prometheus/2.33.4" 2023-10-01T10:28:00.284 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 26 16 46342 46326 111.35 105.25 0.00824429 0.00897457 2023-10-01T10:28:00.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:00 smithi078 bash[75299]: cluster 2023-10-01T10:27:59.373524+0000 mgr.y (mgr.25072) 226 : cluster 0 pgmap v167: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s; 21715/130821 objects degraded (16.599%); 8.0 KiB/s, 0 objects/s recovering 2023-10-01T10:28:00.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:00 smithi078 bash[75299]: cluster 2023-10-01T10:27:59.457485+0000 mon.a (mon.0) 313 : cluster 3 Health check failed: Degraded data redundancy: 21715/130821 objects degraded (16.599%), 15 pgs degraded (PG_DEGRADED) 2023-10-01T10:28:00.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:00 smithi078 bash[78274]: cluster 2023-10-01T10:27:59.373524+0000 mgr.y (mgr.25072) 226 : cluster 0 pgmap v167: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s; 21715/130821 objects degraded (16.599%); 8.0 KiB/s, 0 objects/s recovering 2023-10-01T10:28:00.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:00 smithi078 bash[78274]: cluster 2023-10-01T10:27:59.457485+0000 mon.a (mon.0) 313 : cluster 3 Health check failed: Degraded data redundancy: 21715/130821 objects degraded (16.599%), 15 pgs degraded (PG_DEGRADED) 2023-10-01T10:28:00.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:00 smithi173 bash[58816]: cluster 2023-10-01T10:27:59.373524+0000 mgr.y (mgr.25072) 226 : cluster 0 pgmap v167: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s; 21715/130821 objects degraded (16.599%); 8.0 KiB/s, 0 objects/s recovering 2023-10-01T10:28:00.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:00 smithi173 bash[58816]: cluster 2023-10-01T10:27:59.457485+0000 mon.a (mon.0) 313 : cluster 3 Health check failed: Degraded data redundancy: 21715/130821 objects degraded (16.599%), 15 pgs degraded (PG_DEGRADED) 2023-10-01T10:28:01.284 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 27 16 48011 47995 111.089 104.312 0.00625678 0.00899425 2023-10-01T10:28:01.333 INFO:journalctl@ceph.osd.0.smithi078.stdout:Oct 01 10:28:01 smithi078 systemd[1]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7@osd.0.service: Succeeded. 2023-10-01T10:28:01.334 INFO:journalctl@ceph.osd.0.smithi078.stdout:Oct 01 10:28:01 smithi078 systemd[1]: Stopped Ceph osd.0 for d8016480-6041-11ee-8db5-212e2dc638e7. 2023-10-01T10:28:01.833 INFO:journalctl@ceph.osd.0.smithi078.stdout:Oct 01 10:28:01 smithi078 systemd[1]: Started Ceph osd.0 for d8016480-6041-11ee-8db5-212e2dc638e7. 2023-10-01T10:28:02.284 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 28 16 49412 49396 110.249 87.5625 0.00450477 0.00906481 2023-10-01T10:28:02.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:02 smithi078 bash[78274]: cluster 2023-10-01T10:28:01.374100+0000 mgr.y (mgr.25072) 227 : cluster 0 pgmap v168: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s; 21715/130821 objects degraded (16.599%); 8.0 KiB/s, 0 objects/s recovering 2023-10-01T10:28:02.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:02 smithi078 bash[78274]: audit 2023-10-01T10:28:01.425980+0000 mon.a (mon.0) 314 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:28:02.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:02 smithi078 bash[78274]: audit 2023-10-01T10:28:01.441320+0000 mon.a (mon.0) 315 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:28:02.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:02 smithi078 bash[78274]: audit 2023-10-01T10:28:01.445825+0000 mon.a (mon.0) 316 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:28:02.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:02 smithi078 bash[78274]: audit 2023-10-01T10:28:01.843930+0000 mon.b (mon.1) 5 : audit 0 from='client.? 172.21.15.78:0/2562258185' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:02.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:02 smithi078 bash[78274]: audit 2023-10-01T10:28:02.391760+0000 mon.a (mon.0) 317 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:28:02.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:02 smithi078 bash[78274]: audit 2023-10-01T10:28:02.393536+0000 mon.a (mon.0) 318 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:28:02.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:02 smithi078 bash[75299]: cluster 2023-10-01T10:28:01.374100+0000 mgr.y (mgr.25072) 227 : cluster 0 pgmap v168: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s; 21715/130821 objects degraded (16.599%); 8.0 KiB/s, 0 objects/s recovering 2023-10-01T10:28:02.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:02 smithi078 bash[75299]: audit 2023-10-01T10:28:01.425980+0000 mon.a (mon.0) 314 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:28:02.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:02 smithi078 bash[75299]: audit 2023-10-01T10:28:01.441320+0000 mon.a (mon.0) 315 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:28:02.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:02 smithi078 bash[75299]: audit 2023-10-01T10:28:01.445825+0000 mon.a (mon.0) 316 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:28:02.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:02 smithi078 bash[75299]: audit 2023-10-01T10:28:01.843930+0000 mon.b (mon.1) 5 : audit 0 from='client.? 172.21.15.78:0/2562258185' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:02.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:02 smithi078 bash[75299]: audit 2023-10-01T10:28:02.391760+0000 mon.a (mon.0) 317 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:28:02.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:02 smithi078 bash[75299]: audit 2023-10-01T10:28:02.393536+0000 mon.a (mon.0) 318 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:28:02.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:02 smithi173 bash[58816]: cluster 2023-10-01T10:28:01.374100+0000 mgr.y (mgr.25072) 227 : cluster 0 pgmap v168: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s; 21715/130821 objects degraded (16.599%); 8.0 KiB/s, 0 objects/s recovering 2023-10-01T10:28:02.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:02 smithi173 bash[58816]: audit 2023-10-01T10:28:01.425980+0000 mon.a (mon.0) 314 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:28:02.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:02 smithi173 bash[58816]: audit 2023-10-01T10:28:01.441320+0000 mon.a (mon.0) 315 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:28:02.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:02 smithi173 bash[58816]: audit 2023-10-01T10:28:01.445825+0000 mon.a (mon.0) 316 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:28:02.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:02 smithi173 bash[58816]: audit 2023-10-01T10:28:01.843930+0000 mon.b (mon.1) 5 : audit 0 from='client.? 172.21.15.78:0/2562258185' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:02.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:02 smithi173 bash[58816]: audit 2023-10-01T10:28:02.391760+0000 mon.a (mon.0) 317 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:28:02.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:02 smithi173 bash[58816]: audit 2023-10-01T10:28:02.393536+0000 mon.a (mon.0) 318 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:28:03.284 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 29 16 50884 50868 109.619 92 0.00681832 0.00911666 2023-10-01T10:28:04.020 INFO:journalctl@ceph.osd.0.smithi078.stdout:Oct 01 10:28:03 smithi078 bash[84470]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-01T10:28:04.020 INFO:journalctl@ceph.osd.0.smithi078.stdout:Oct 01 10:28:03 smithi078 bash[84470]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-10-01T10:28:04.020 INFO:journalctl@ceph.osd.0.smithi078.stdout:Oct 01 10:28:03 smithi078 bash[84470]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2023-10-01T10:28:04.021 INFO:journalctl@ceph.osd.0.smithi078.stdout:Oct 01 10:28:03 smithi078 bash[84470]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2023-10-01T10:28:04.021 INFO:journalctl@ceph.osd.0.smithi078.stdout:Oct 01 10:28:03 smithi078 bash[84470]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2023-10-01T10:28:04.021 INFO:journalctl@ceph.osd.0.smithi078.stdout:Oct 01 10:28:03 smithi078 bash[84470]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2023-10-01T10:28:04.021 INFO:journalctl@ceph.osd.0.smithi078.stdout:Oct 01 10:28:03 smithi078 bash[84470]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-10-01T10:28:04.021 INFO:journalctl@ceph.osd.0.smithi078.stdout:Oct 01 10:28:03 smithi078 bash[84470]: --> ceph-volume lvm activate successful for osd ID: 0 2023-10-01T10:28:04.285 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 30 16 52360 52344 109.04 92.25 0.00699605 0.00916534 2023-10-01T10:28:04.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:04 smithi078 bash[78274]: cluster 2023-10-01T10:28:03.374640+0000 mgr.y (mgr.25072) 228 : cluster 0 pgmap v169: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 47 MiB/s wr, 749 op/s; 21715/130821 objects degraded (16.599%); 8.0 KiB/s, 0 objects/s recovering 2023-10-01T10:28:04.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:04 smithi078 bash[78274]: audit 2023-10-01T10:28:04.431832+0000 mon.a (mon.0) 319 : audit 0 from='client.? 172.21.15.78:0/4036436591' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:04.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:04 smithi078 bash[75299]: cluster 2023-10-01T10:28:03.374640+0000 mgr.y (mgr.25072) 228 : cluster 0 pgmap v169: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 47 MiB/s wr, 749 op/s; 21715/130821 objects degraded (16.599%); 8.0 KiB/s, 0 objects/s recovering 2023-10-01T10:28:04.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:04 smithi078 bash[75299]: audit 2023-10-01T10:28:04.431832+0000 mon.a (mon.0) 319 : audit 0 from='client.? 172.21.15.78:0/4036436591' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:04.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:04 smithi173 bash[58816]: cluster 2023-10-01T10:28:03.374640+0000 mgr.y (mgr.25072) 228 : cluster 0 pgmap v169: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 47 MiB/s wr, 749 op/s; 21715/130821 objects degraded (16.599%); 8.0 KiB/s, 0 objects/s recovering 2023-10-01T10:28:04.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:04 smithi173 bash[58816]: audit 2023-10-01T10:28:04.431832+0000 mon.a (mon.0) 319 : audit 0 from='client.? 172.21.15.78:0/4036436591' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:05.285 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 31 16 53888 53872 108.603 95.5 0.0217019 0.00920146 2023-10-01T10:28:05.494 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:05 smithi078 bash[75299]: cluster 2023-10-01T10:28:05.437829+0000 mon.a (mon.0) 320 : cluster 3 Health check update: Degraded data redundancy: 25567/154254 objects degraded (16.575%), 15 pgs degraded (PG_DEGRADED) 2023-10-01T10:28:05.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:05 smithi078 bash[78274]: cluster 2023-10-01T10:28:05.437829+0000 mon.a (mon.0) 320 : cluster 3 Health check update: Degraded data redundancy: 25567/154254 objects degraded (16.575%), 15 pgs degraded (PG_DEGRADED) 2023-10-01T10:28:05.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:05 smithi173 bash[58816]: cluster 2023-10-01T10:28:05.437829+0000 mon.a (mon.0) 320 : cluster 3 Health check update: Degraded data redundancy: 25567/154254 objects degraded (16.575%), 15 pgs degraded (PG_DEGRADED) 2023-10-01T10:28:06.285 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 32 16 55459 55443 108.277 98.1875 0.0123904 0.00922967 2023-10-01T10:28:06.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:06 smithi078 bash[78274]: cluster 2023-10-01T10:28:05.375513+0000 mgr.y (mgr.25072) 229 : cluster 0 pgmap v170: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s; 25567/154254 objects degraded (16.575%); 7.2 KiB/s, 0 objects/s recovering 2023-10-01T10:28:06.833 INFO:journalctl@ceph.osd.0.smithi078.stdout:Oct 01 10:28:06 smithi078 bash[84784]: debug 2023-10-01T10:28:06.516+0000 7f3482436700 1 -- 172.21.15.78:0/1498886884 <== mon.0 v2:172.21.15.78:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55cfdf2204e0 con 0x55cfdf27b000 2023-10-01T10:28:06.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:06 smithi078 bash[75299]: cluster 2023-10-01T10:28:05.375513+0000 mgr.y (mgr.25072) 229 : cluster 0 pgmap v170: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s; 25567/154254 objects degraded (16.575%); 7.2 KiB/s, 0 objects/s recovering 2023-10-01T10:28:06.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:06 smithi173 bash[58816]: cluster 2023-10-01T10:28:05.375513+0000 mgr.y (mgr.25072) 229 : cluster 0 pgmap v170: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s; 25567/154254 objects degraded (16.575%); 7.2 KiB/s, 0 objects/s recovering 2023-10-01T10:28:07.285 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 33 16 56838 56822 107.607 86.1875 0.00948521 0.00928706 2023-10-01T10:28:07.582 INFO:journalctl@ceph.osd.0.smithi078.stdout:Oct 01 10:28:07 smithi078 bash[84784]: debug 2023-10-01T10:28:07.076+0000 7f348b5bc640 -1 Falling back to public interface 2023-10-01T10:28:08.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:07 smithi078 bash[78274]: audit 2023-10-01T10:28:07.002831+0000 mon.a (mon.0) 321 : audit 0 from='client.? 172.21.15.78:0/3066386544' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:08.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:07 smithi078 bash[78274]: cluster 2023-10-01T10:28:07.376268+0000 mgr.y (mgr.25072) 230 : cluster 0 pgmap v171: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s; 25567/154254 objects degraded (16.575%); 6.4 KiB/s, 0 objects/s recovering 2023-10-01T10:28:08.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:07 smithi078 bash[75299]: audit 2023-10-01T10:28:07.002831+0000 mon.a (mon.0) 321 : audit 0 from='client.? 172.21.15.78:0/3066386544' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:08.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:07 smithi078 bash[75299]: cluster 2023-10-01T10:28:07.376268+0000 mgr.y (mgr.25072) 230 : cluster 0 pgmap v171: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s; 25567/154254 objects degraded (16.575%); 6.4 KiB/s, 0 objects/s recovering 2023-10-01T10:28:08.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:07 smithi173 bash[58816]: audit 2023-10-01T10:28:07.002831+0000 mon.a (mon.0) 321 : audit 0 from='client.? 172.21.15.78:0/3066386544' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:08.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:07 smithi173 bash[58816]: cluster 2023-10-01T10:28:07.376268+0000 mgr.y (mgr.25072) 230 : cluster 0 pgmap v171: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s; 25567/154254 objects degraded (16.575%); 6.4 KiB/s, 0 objects/s recovering 2023-10-01T10:28:08.285 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 34 16 58382 58366 107.28 96.5 0.0119169 0.00931607 2023-10-01T10:28:09.285 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 35 16 59661 59645 106.499 79.9375 0.0116868 0.00938432 2023-10-01T10:28:10.270 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:28:10 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:28:10] "GET /metrics HTTP/1.1" 200 36208 "" "Prometheus/2.33.4" 2023-10-01T10:28:10.285 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 36 16 60779 60763 105.481 69.875 0.00859459 0.00947425 2023-10-01T10:28:10.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:10 smithi078 bash[75299]: cluster 2023-10-01T10:28:09.377181+0000 mgr.y (mgr.25072) 231 : cluster 0 pgmap v172: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s; 29358/176775 objects degraded (16.608%); 5.4 KiB/s, 0 objects/s recovering 2023-10-01T10:28:10.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:10 smithi078 bash[75299]: audit 2023-10-01T10:28:09.632243+0000 mon.b (mon.1) 6 : audit 0 from='client.? 172.21.15.78:0/604900477' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:10.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:10 smithi078 bash[75299]: audit 2023-10-01T10:28:10.067407+0000 mon.a (mon.0) 322 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:28:10.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:10 smithi078 bash[75299]: audit 2023-10-01T10:28:10.078403+0000 mon.a (mon.0) 323 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:28:10.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:10 smithi078 bash[78274]: cluster 2023-10-01T10:28:09.377181+0000 mgr.y (mgr.25072) 231 : cluster 0 pgmap v172: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s; 29358/176775 objects degraded (16.608%); 5.4 KiB/s, 0 objects/s recovering 2023-10-01T10:28:10.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:10 smithi078 bash[78274]: audit 2023-10-01T10:28:09.632243+0000 mon.b (mon.1) 6 : audit 0 from='client.? 172.21.15.78:0/604900477' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:10.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:10 smithi078 bash[78274]: audit 2023-10-01T10:28:10.067407+0000 mon.a (mon.0) 322 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:28:10.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:10 smithi078 bash[78274]: audit 2023-10-01T10:28:10.078403+0000 mon.a (mon.0) 323 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:28:10.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:10 smithi173 bash[58816]: cluster 2023-10-01T10:28:09.377181+0000 mgr.y (mgr.25072) 231 : cluster 0 pgmap v172: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s; 29358/176775 objects degraded (16.608%); 5.4 KiB/s, 0 objects/s recovering 2023-10-01T10:28:10.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:10 smithi173 bash[58816]: audit 2023-10-01T10:28:09.632243+0000 mon.b (mon.1) 6 : audit 0 from='client.? 172.21.15.78:0/604900477' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:10.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:10 smithi173 bash[58816]: audit 2023-10-01T10:28:10.067407+0000 mon.a (mon.0) 322 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:28:10.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:10 smithi173 bash[58816]: audit 2023-10-01T10:28:10.078403+0000 mon.a (mon.0) 323 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:28:11.285 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 37 16 61990 61974 104.676 75.6875 0.0197132 0.00954692 2023-10-01T10:28:12.285 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 38 16 63142 63126 103.816 72 0.00929211 0.009627 2023-10-01T10:28:12.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:12 smithi078 bash[75299]: audit 2023-10-01T10:28:11.125070+0000 mon.a (mon.0) 324 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:28:12.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:12 smithi078 bash[75299]: audit 2023-10-01T10:28:11.135194+0000 mon.a (mon.0) 325 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:28:12.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:12 smithi078 bash[75299]: cluster 2023-10-01T10:28:11.377717+0000 mgr.y (mgr.25072) 232 : cluster 0 pgmap v173: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s; 29358/176775 objects degraded (16.608%) 2023-10-01T10:28:12.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:12 smithi078 bash[75299]: cluster 2023-10-01T10:28:11.752987+0000 mon.a (mon.0) 326 : cluster 3 Health check update: Degraded data redundancy: 29358/176775 objects degraded (16.608%), 15 pgs degraded (PG_DEGRADED) 2023-10-01T10:28:12.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:12 smithi078 bash[78274]: audit 2023-10-01T10:28:11.125070+0000 mon.a (mon.0) 324 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:28:12.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:12 smithi078 bash[78274]: audit 2023-10-01T10:28:11.135194+0000 mon.a (mon.0) 325 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:28:12.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:12 smithi078 bash[78274]: cluster 2023-10-01T10:28:11.377717+0000 mgr.y (mgr.25072) 232 : cluster 0 pgmap v173: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s; 29358/176775 objects degraded (16.608%) 2023-10-01T10:28:12.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:12 smithi078 bash[78274]: cluster 2023-10-01T10:28:11.752987+0000 mon.a (mon.0) 326 : cluster 3 Health check update: Degraded data redundancy: 29358/176775 objects degraded (16.608%), 15 pgs degraded (PG_DEGRADED) 2023-10-01T10:28:12.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:12 smithi173 bash[58816]: audit 2023-10-01T10:28:11.125070+0000 mon.a (mon.0) 324 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:28:12.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:12 smithi173 bash[58816]: audit 2023-10-01T10:28:11.135194+0000 mon.a (mon.0) 325 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:28:12.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:12 smithi173 bash[58816]: cluster 2023-10-01T10:28:11.377717+0000 mgr.y (mgr.25072) 232 : cluster 0 pgmap v173: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s; 29358/176775 objects degraded (16.608%) 2023-10-01T10:28:12.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:12 smithi173 bash[58816]: cluster 2023-10-01T10:28:11.752987+0000 mon.a (mon.0) 326 : cluster 3 Health check update: Degraded data redundancy: 29358/176775 objects degraded (16.608%), 15 pgs degraded (PG_DEGRADED) 2023-10-01T10:28:13.285 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 39 16 64446 64430 103.243 81.5 0.00792046 0.00967899 2023-10-01T10:28:13.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:13 smithi078 bash[75299]: audit 2023-10-01T10:28:12.238249+0000 mon.c (mon.2) 9 : audit 0 from='client.? 172.21.15.78:0/1106856575' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:13.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:13 smithi078 bash[78274]: audit 2023-10-01T10:28:12.238249+0000 mon.c (mon.2) 9 : audit 0 from='client.? 172.21.15.78:0/1106856575' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:13.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:13 smithi173 bash[58816]: audit 2023-10-01T10:28:12.238249+0000 mon.c (mon.2) 9 : audit 0 from='client.? 172.21.15.78:0/1106856575' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:14.285 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:2023-10-01T10:28:14.283685+0000 min lat: 0.00149611 max lat: 1.03372 avg lat: 0.00973406 2023-10-01T10:28:14.286 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T10:28:14.286 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 40 16 65731 65715 102.67 80.3125 0.0118116 0.00973406 2023-10-01T10:28:14.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:14 smithi078 bash[75299]: cluster 2023-10-01T10:28:13.378296+0000 mgr.y (mgr.25072) 233 : cluster 0 pgmap v174: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s; 29358/176775 objects degraded (16.608%) 2023-10-01T10:28:14.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:14 smithi078 bash[78274]: cluster 2023-10-01T10:28:13.378296+0000 mgr.y (mgr.25072) 233 : cluster 0 pgmap v174: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s; 29358/176775 objects degraded (16.608%) 2023-10-01T10:28:14.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:14 smithi173 bash[58816]: cluster 2023-10-01T10:28:13.378296+0000 mgr.y (mgr.25072) 233 : cluster 0 pgmap v174: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s; 29358/176775 objects degraded (16.608%) 2023-10-01T10:28:15.286 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 41 16 66937 66921 102.004 75.375 0.00676568 0.00979845 2023-10-01T10:28:15.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:15 smithi078 bash[75299]: audit 2023-10-01T10:28:14.843389+0000 mon.b (mon.1) 7 : audit 0 from='client.? 172.21.15.78:0/1266410741' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:15.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:15 smithi078 bash[78274]: audit 2023-10-01T10:28:14.843389+0000 mon.b (mon.1) 7 : audit 0 from='client.? 172.21.15.78:0/1266410741' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:15.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:15 smithi173 bash[58816]: audit 2023-10-01T10:28:14.843389+0000 mon.b (mon.1) 7 : audit 0 from='client.? 172.21.15.78:0/1266410741' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:16.286 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 42 16 68232 68216 101.502 80.9375 0.00808568 0.00984614 2023-10-01T10:28:16.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:16 smithi078 bash[78274]: cluster 2023-10-01T10:28:15.379129+0000 mgr.y (mgr.25072) 234 : cluster 0 pgmap v175: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s; 32343/194829 objects degraded (16.601%) 2023-10-01T10:28:16.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:16 smithi078 bash[75299]: cluster 2023-10-01T10:28:15.379129+0000 mgr.y (mgr.25072) 234 : cluster 0 pgmap v175: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s; 32343/194829 objects degraded (16.601%) 2023-10-01T10:28:16.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:16 smithi173 bash[58816]: cluster 2023-10-01T10:28:15.379129+0000 mgr.y (mgr.25072) 234 : cluster 0 pgmap v175: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s; 32343/194829 objects degraded (16.601%) 2023-10-01T10:28:17.286 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 43 16 69403 69387 100.844 73.1875 0.0131724 0.00990837 2023-10-01T10:28:17.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:17 smithi078 bash[75299]: cluster 2023-10-01T10:28:16.754277+0000 mon.a (mon.0) 327 : cluster 3 Health check update: Degraded data redundancy: 32343/194829 objects degraded (16.601%), 15 pgs degraded (PG_DEGRADED) 2023-10-01T10:28:17.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:17 smithi078 bash[78274]: cluster 2023-10-01T10:28:16.754277+0000 mon.a (mon.0) 327 : cluster 3 Health check update: Degraded data redundancy: 32343/194829 objects degraded (16.601%), 15 pgs degraded (PG_DEGRADED) 2023-10-01T10:28:17.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:17 smithi173 bash[58816]: cluster 2023-10-01T10:28:16.754277+0000 mon.a (mon.0) 327 : cluster 3 Health check update: Degraded data redundancy: 32343/194829 objects degraded (16.601%), 15 pgs degraded (PG_DEGRADED) 2023-10-01T10:28:18.177 INFO:journalctl@ceph.osd.0.smithi078.stdout:Oct 01 10:28:17 smithi078 bash[84784]: debug 2023-10-01T10:28:17.860+0000 7f348b5bc640 -1 osd.0 82 log_to_monitors true 2023-10-01T10:28:18.286 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 44 16 70256 70240 99.7631 53.3125 0.00365321 0.00995825 2023-10-01T10:28:18.582 INFO:journalctl@ceph.osd.0.smithi078.stdout:Oct 01 10:28:18 smithi078 bash[84784]: debug 2023-10-01T10:28:18.216+0000 7f347a223700 -1 osd.0 82 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-01T10:28:18.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:18 smithi078 bash[75299]: cluster 2023-10-01T10:28:17.379673+0000 mgr.y (mgr.25072) 235 : cluster 0 pgmap v176: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 70 MiB/s wr, 1.13k op/s; 32343/194829 objects degraded (16.601%) 2023-10-01T10:28:18.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:18 smithi078 bash[75299]: audit 2023-10-01T10:28:17.387706+0000 mon.a (mon.0) 328 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:28:18.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:18 smithi078 bash[75299]: audit 2023-10-01T10:28:17.436072+0000 mon.c (mon.2) 10 : audit 0 from='client.? 172.21.15.78:0/3598895946' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:18.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:18 smithi078 bash[75299]: audit 2023-10-01T10:28:17.881125+0000 mon.c (mon.2) 11 : audit 1 from='osd.0 [v2:172.21.15.78:6802/2878175461,v1:172.21.15.78:6803/2878175461]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-01T10:28:18.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:18 smithi078 bash[75299]: audit 2023-10-01T10:28:17.881939+0000 mon.a (mon.0) 329 : audit 1 from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-01T10:28:18.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:18 smithi078 bash[78274]: cluster 2023-10-01T10:28:17.379673+0000 mgr.y (mgr.25072) 235 : cluster 0 pgmap v176: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 70 MiB/s wr, 1.13k op/s; 32343/194829 objects degraded (16.601%) 2023-10-01T10:28:18.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:18 smithi078 bash[78274]: audit 2023-10-01T10:28:17.387706+0000 mon.a (mon.0) 328 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:28:18.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:18 smithi078 bash[78274]: audit 2023-10-01T10:28:17.436072+0000 mon.c (mon.2) 10 : audit 0 from='client.? 172.21.15.78:0/3598895946' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:18.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:18 smithi078 bash[78274]: audit 2023-10-01T10:28:17.881125+0000 mon.c (mon.2) 11 : audit 1 from='osd.0 [v2:172.21.15.78:6802/2878175461,v1:172.21.15.78:6803/2878175461]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-01T10:28:18.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:18 smithi078 bash[78274]: audit 2023-10-01T10:28:17.881939+0000 mon.a (mon.0) 329 : audit 1 from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-01T10:28:18.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:18 smithi173 bash[58816]: cluster 2023-10-01T10:28:17.379673+0000 mgr.y (mgr.25072) 235 : cluster 0 pgmap v176: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 70 MiB/s wr, 1.13k op/s; 32343/194829 objects degraded (16.601%) 2023-10-01T10:28:18.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:18 smithi173 bash[58816]: audit 2023-10-01T10:28:17.387706+0000 mon.a (mon.0) 328 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:28:18.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:18 smithi173 bash[58816]: audit 2023-10-01T10:28:17.436072+0000 mon.c (mon.2) 10 : audit 0 from='client.? 172.21.15.78:0/3598895946' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:18.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:18 smithi173 bash[58816]: audit 2023-10-01T10:28:17.881125+0000 mon.c (mon.2) 11 : audit 1 from='osd.0 [v2:172.21.15.78:6802/2878175461,v1:172.21.15.78:6803/2878175461]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-01T10:28:18.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:18 smithi173 bash[58816]: audit 2023-10-01T10:28:17.881939+0000 mon.a (mon.0) 329 : audit 1 from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-10-01T10:28:19.286 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 45 16 71493 71477 99.2639 77.3125 0.00378206 0.0100506 2023-10-01T10:28:19.482 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:19 smithi078 bash[78274]: cluster 2023-10-01T10:28:18.169322+0000 osd.2 (osd.2) 9 : cluster 3 failed to encode map e85 with expected crc 2023-10-01T10:28:19.482 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:19 smithi078 bash[78274]: audit 2023-10-01T10:28:18.170842+0000 mon.a (mon.0) 330 : audit 1 from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-01T10:28:19.482 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:19 smithi078 bash[78274]: cluster 2023-10-01T10:28:18.170918+0000 mon.a (mon.0) 331 : cluster 0 osdmap e85: 8 total, 7 up, 8 in 2023-10-01T10:28:19.482 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:19 smithi078 bash[78274]: cluster 2023-10-01T10:28:18.172340+0000 osd.7 (osd.7) 12 : cluster 3 failed to encode map e85 with expected crc 2023-10-01T10:28:19.483 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:19 smithi078 bash[78274]: audit 2023-10-01T10:28:18.174366+0000 mon.c (mon.2) 12 : audit 1 from='osd.0 [v2:172.21.15.78:6802/2878175461,v1:172.21.15.78:6803/2878175461]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2023-10-01T10:28:19.483 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:19 smithi078 bash[78274]: audit 2023-10-01T10:28:18.178619+0000 mon.a (mon.0) 332 : audit 1 from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2023-10-01T10:28:19.483 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:19 smithi078 bash[75299]: cluster 2023-10-01T10:28:18.169322+0000 osd.2 (osd.2) 9 : cluster 3 failed to encode map e85 with expected crc 2023-10-01T10:28:19.483 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:19 smithi078 bash[75299]: audit 2023-10-01T10:28:18.170842+0000 mon.a (mon.0) 330 : audit 1 from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-01T10:28:19.483 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:19 smithi078 bash[75299]: cluster 2023-10-01T10:28:18.170918+0000 mon.a (mon.0) 331 : cluster 0 osdmap e85: 8 total, 7 up, 8 in 2023-10-01T10:28:19.483 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:19 smithi078 bash[75299]: cluster 2023-10-01T10:28:18.172340+0000 osd.7 (osd.7) 12 : cluster 3 failed to encode map e85 with expected crc 2023-10-01T10:28:19.483 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:19 smithi078 bash[75299]: audit 2023-10-01T10:28:18.174366+0000 mon.c (mon.2) 12 : audit 1 from='osd.0 [v2:172.21.15.78:6802/2878175461,v1:172.21.15.78:6803/2878175461]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2023-10-01T10:28:19.483 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:19 smithi078 bash[75299]: audit 2023-10-01T10:28:18.178619+0000 mon.a (mon.0) 332 : audit 1 from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2023-10-01T10:28:19.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:19 smithi173 bash[58816]: cluster 2023-10-01T10:28:18.169322+0000 osd.2 (osd.2) 9 : cluster 3 failed to encode map e85 with expected crc 2023-10-01T10:28:19.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:19 smithi173 bash[58816]: audit 2023-10-01T10:28:18.170842+0000 mon.a (mon.0) 330 : audit 1 from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-10-01T10:28:19.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:19 smithi173 bash[58816]: cluster 2023-10-01T10:28:18.170918+0000 mon.a (mon.0) 331 : cluster 0 osdmap e85: 8 total, 7 up, 8 in 2023-10-01T10:28:19.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:19 smithi173 bash[58816]: cluster 2023-10-01T10:28:18.172340+0000 osd.7 (osd.7) 12 : cluster 3 failed to encode map e85 with expected crc 2023-10-01T10:28:19.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:19 smithi173 bash[58816]: audit 2023-10-01T10:28:18.174366+0000 mon.c (mon.2) 12 : audit 1 from='osd.0 [v2:172.21.15.78:6802/2878175461,v1:172.21.15.78:6803/2878175461]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2023-10-01T10:28:19.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:19 smithi173 bash[58816]: audit 2023-10-01T10:28:18.178619+0000 mon.a (mon.0) 332 : audit 1 from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2023-10-01T10:28:20.286 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 46 16 71493 71477 97.1058 0 - 0.0100506 2023-10-01T10:28:20.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:28:20 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:28:20] "GET /metrics HTTP/1.1" 200 36199 "" "Prometheus/2.33.4" 2023-10-01T10:28:20.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:20 smithi078 bash[78274]: cluster 2023-10-01T10:28:19.175485+0000 mon.a (mon.0) 333 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-01T10:28:20.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:20 smithi078 bash[78274]: cluster 2023-10-01T10:28:19.178884+0000 osd.1 (osd.1) 11 : cluster 3 failed to encode map e86 with expected crc 2023-10-01T10:28:20.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:20 smithi078 bash[78274]: cluster 2023-10-01T10:28:19.180368+0000 osd.7 (osd.7) 13 : cluster 3 failed to encode map e86 with expected crc 2023-10-01T10:28:20.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:20 smithi078 bash[78274]: cluster 2023-10-01T10:28:19.180736+0000 mon.a (mon.0) 334 : cluster 1 osd.0 [v2:172.21.15.78:6802/2878175461,v1:172.21.15.78:6803/2878175461] boot 2023-10-01T10:28:20.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:20 smithi078 bash[78274]: cluster 2023-10-01T10:28:19.180808+0000 mon.a (mon.0) 335 : cluster 0 osdmap e86: 8 total, 8 up, 8 in 2023-10-01T10:28:20.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:20 smithi078 bash[78274]: audit 2023-10-01T10:28:19.181357+0000 mon.a (mon.0) 336 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T10:28:20.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:20 smithi078 bash[78274]: cluster 2023-10-01T10:28:19.380451+0000 mgr.y (mgr.25072) 236 : cluster 0 pgmap v179: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 92 MiB/s wr, 1.48k op/s; 35177/212274 objects degraded (16.572%) 2023-10-01T10:28:20.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:20 smithi078 bash[78274]: audit 2023-10-01T10:28:19.805387+0000 mon.a (mon.0) 337 : audit 0 from='client.? 172.21.15.78:0/2607614818' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:20.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:20 smithi078 bash[75299]: cluster 2023-10-01T10:28:19.175485+0000 mon.a (mon.0) 333 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-01T10:28:20.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:20 smithi078 bash[75299]: cluster 2023-10-01T10:28:19.178884+0000 osd.1 (osd.1) 11 : cluster 3 failed to encode map e86 with expected crc 2023-10-01T10:28:20.585 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:20 smithi078 bash[75299]: cluster 2023-10-01T10:28:19.180368+0000 osd.7 (osd.7) 13 : cluster 3 failed to encode map e86 with expected crc 2023-10-01T10:28:20.585 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:20 smithi078 bash[75299]: cluster 2023-10-01T10:28:19.180736+0000 mon.a (mon.0) 334 : cluster 1 osd.0 [v2:172.21.15.78:6802/2878175461,v1:172.21.15.78:6803/2878175461] boot 2023-10-01T10:28:20.585 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:20 smithi078 bash[75299]: cluster 2023-10-01T10:28:19.180808+0000 mon.a (mon.0) 335 : cluster 0 osdmap e86: 8 total, 8 up, 8 in 2023-10-01T10:28:20.585 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:20 smithi078 bash[75299]: audit 2023-10-01T10:28:19.181357+0000 mon.a (mon.0) 336 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T10:28:20.585 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:20 smithi078 bash[75299]: cluster 2023-10-01T10:28:19.380451+0000 mgr.y (mgr.25072) 236 : cluster 0 pgmap v179: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 92 MiB/s wr, 1.48k op/s; 35177/212274 objects degraded (16.572%) 2023-10-01T10:28:20.585 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:20 smithi078 bash[75299]: audit 2023-10-01T10:28:19.805387+0000 mon.a (mon.0) 337 : audit 0 from='client.? 172.21.15.78:0/2607614818' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:20.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:20 smithi173 bash[58816]: cluster 2023-10-01T10:28:19.175485+0000 mon.a (mon.0) 333 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-01T10:28:20.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:20 smithi173 bash[58816]: cluster 2023-10-01T10:28:19.178884+0000 osd.1 (osd.1) 11 : cluster 3 failed to encode map e86 with expected crc 2023-10-01T10:28:20.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:20 smithi173 bash[58816]: cluster 2023-10-01T10:28:19.180368+0000 osd.7 (osd.7) 13 : cluster 3 failed to encode map e86 with expected crc 2023-10-01T10:28:20.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:20 smithi173 bash[58816]: cluster 2023-10-01T10:28:19.180736+0000 mon.a (mon.0) 334 : cluster 1 osd.0 [v2:172.21.15.78:6802/2878175461,v1:172.21.15.78:6803/2878175461] boot 2023-10-01T10:28:20.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:20 smithi173 bash[58816]: cluster 2023-10-01T10:28:19.180808+0000 mon.a (mon.0) 335 : cluster 0 osdmap e86: 8 total, 8 up, 8 in 2023-10-01T10:28:20.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:20 smithi173 bash[58816]: audit 2023-10-01T10:28:19.181357+0000 mon.a (mon.0) 336 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-10-01T10:28:20.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:20 smithi173 bash[58816]: cluster 2023-10-01T10:28:19.380451+0000 mgr.y (mgr.25072) 236 : cluster 0 pgmap v179: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 92 MiB/s wr, 1.48k op/s; 35177/212274 objects degraded (16.572%) 2023-10-01T10:28:20.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:20 smithi173 bash[58816]: audit 2023-10-01T10:28:19.805387+0000 mon.a (mon.0) 337 : audit 0 from='client.? 172.21.15.78:0/2607614818' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:21.286 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 47 16 71493 71477 95.0396 0 - 0.0100506 2023-10-01T10:28:21.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:21 smithi078 bash[75299]: cluster 2023-10-01T10:28:20.187377+0000 osd.3 (osd.3) 9 : cluster 3 failed to encode map e87 with expected crc 2023-10-01T10:28:21.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:21 smithi078 bash[75299]: cluster 2023-10-01T10:28:20.191974+0000 osd.1 (osd.1) 12 : cluster 3 failed to encode map e87 with expected crc 2023-10-01T10:28:21.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:21 smithi078 bash[75299]: cluster 2023-10-01T10:28:20.192016+0000 mon.a (mon.0) 338 : cluster 0 osdmap e87: 8 total, 8 up, 8 in 2023-10-01T10:28:21.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:21 smithi078 bash[75299]: cluster 2023-10-01T10:28:20.192213+0000 osd.7 (osd.7) 14 : cluster 3 failed to encode map e87 with expected crc 2023-10-01T10:28:21.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:21 smithi078 bash[75299]: cluster 2023-10-01T10:28:20.192263+0000 osd.3 (osd.3) 10 : cluster 3 failed to encode map e87 with expected crc 2023-10-01T10:28:21.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:21 smithi078 bash[75299]: cluster 2023-10-01T10:28:20.192298+0000 osd.2 (osd.2) 10 : cluster 3 failed to encode map e87 with expected crc 2023-10-01T10:28:21.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:21 smithi078 bash[75299]: cluster 2023-10-01T10:28:20.194123+0000 osd.6 (osd.6) 7 : cluster 3 failed to encode map e87 with expected crc 2023-10-01T10:28:21.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:21 smithi078 bash[75299]: cluster 2023-10-01T10:28:20.194312+0000 osd.4 (osd.4) 11 : cluster 3 failed to encode map e87 with expected crc 2023-10-01T10:28:21.585 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:21 smithi078 bash[75299]: cluster 2023-10-01T10:28:20.195125+0000 osd.3 (osd.3) 11 : cluster 3 failed to encode map e87 with expected crc 2023-10-01T10:28:21.585 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:21 smithi078 bash[75299]: cluster 2023-10-01T10:28:20.195223+0000 osd.7 (osd.7) 15 : cluster 3 failed to encode map e87 with expected crc 2023-10-01T10:28:21.585 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:21 smithi078 bash[75299]: cluster 2023-10-01T10:28:20.197929+0000 osd.6 (osd.6) 8 : cluster 3 failed to encode map e87 with expected crc 2023-10-01T10:28:21.585 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:21 smithi078 bash[75299]: cluster 2023-10-01T10:28:20.198234+0000 osd.7 (osd.7) 16 : cluster 3 failed to encode map e87 with expected crc 2023-10-01T10:28:21.585 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:21 smithi078 bash[75299]: cluster 2023-10-01T10:28:20.200878+0000 osd.4 (osd.4) 12 : cluster 3 failed to encode map e87 with expected crc 2023-10-01T10:28:21.585 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:21 smithi078 bash[75299]: cluster 2023-10-01T10:28:21.196367+0000 mon.a (mon.0) 339 : cluster 0 osdmap e88: 8 total, 8 up, 8 in 2023-10-01T10:28:21.586 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:21 smithi078 bash[78274]: cluster 2023-10-01T10:28:20.187377+0000 osd.3 (osd.3) 9 : cluster 3 failed to encode map e87 with expected crc 2023-10-01T10:28:21.586 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:21 smithi078 bash[78274]: cluster 2023-10-01T10:28:20.191974+0000 osd.1 (osd.1) 12 : cluster 3 failed to encode map e87 with expected crc 2023-10-01T10:28:21.586 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:21 smithi078 bash[78274]: cluster 2023-10-01T10:28:20.192016+0000 mon.a (mon.0) 338 : cluster 0 osdmap e87: 8 total, 8 up, 8 in 2023-10-01T10:28:21.586 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:21 smithi078 bash[78274]: cluster 2023-10-01T10:28:20.192213+0000 osd.7 (osd.7) 14 : cluster 3 failed to encode map e87 with expected crc 2023-10-01T10:28:21.586 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:21 smithi078 bash[78274]: cluster 2023-10-01T10:28:20.192263+0000 osd.3 (osd.3) 10 : cluster 3 failed to encode map e87 with expected crc 2023-10-01T10:28:21.587 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:21 smithi078 bash[78274]: cluster 2023-10-01T10:28:20.192298+0000 osd.2 (osd.2) 10 : cluster 3 failed to encode map e87 with expected crc 2023-10-01T10:28:21.587 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:21 smithi078 bash[78274]: cluster 2023-10-01T10:28:20.194123+0000 osd.6 (osd.6) 7 : cluster 3 failed to encode map e87 with expected crc 2023-10-01T10:28:21.587 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:21 smithi078 bash[78274]: cluster 2023-10-01T10:28:20.194312+0000 osd.4 (osd.4) 11 : cluster 3 failed to encode map e87 with expected crc 2023-10-01T10:28:21.587 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:21 smithi078 bash[78274]: cluster 2023-10-01T10:28:20.195125+0000 osd.3 (osd.3) 11 : cluster 3 failed to encode map e87 with expected crc 2023-10-01T10:28:21.587 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:21 smithi078 bash[78274]: cluster 2023-10-01T10:28:20.195223+0000 osd.7 (osd.7) 15 : cluster 3 failed to encode map e87 with expected crc 2023-10-01T10:28:21.587 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:21 smithi078 bash[78274]: cluster 2023-10-01T10:28:20.197929+0000 osd.6 (osd.6) 8 : cluster 3 failed to encode map e87 with expected crc 2023-10-01T10:28:21.588 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:21 smithi078 bash[78274]: cluster 2023-10-01T10:28:20.198234+0000 osd.7 (osd.7) 16 : cluster 3 failed to encode map e87 with expected crc 2023-10-01T10:28:21.588 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:21 smithi078 bash[78274]: cluster 2023-10-01T10:28:20.200878+0000 osd.4 (osd.4) 12 : cluster 3 failed to encode map e87 with expected crc 2023-10-01T10:28:21.588 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:21 smithi078 bash[78274]: cluster 2023-10-01T10:28:21.196367+0000 mon.a (mon.0) 339 : cluster 0 osdmap e88: 8 total, 8 up, 8 in 2023-10-01T10:28:21.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:21 smithi173 bash[58816]: cluster 2023-10-01T10:28:20.187377+0000 osd.3 (osd.3) 9 : cluster 3 failed to encode map e87 with expected crc 2023-10-01T10:28:21.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:21 smithi173 bash[58816]: cluster 2023-10-01T10:28:20.191974+0000 osd.1 (osd.1) 12 : cluster 3 failed to encode map e87 with expected crc 2023-10-01T10:28:21.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:21 smithi173 bash[58816]: cluster 2023-10-01T10:28:20.192016+0000 mon.a (mon.0) 338 : cluster 0 osdmap e87: 8 total, 8 up, 8 in 2023-10-01T10:28:21.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:21 smithi173 bash[58816]: cluster 2023-10-01T10:28:20.192213+0000 osd.7 (osd.7) 14 : cluster 3 failed to encode map e87 with expected crc 2023-10-01T10:28:21.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:21 smithi173 bash[58816]: cluster 2023-10-01T10:28:20.192263+0000 osd.3 (osd.3) 10 : cluster 3 failed to encode map e87 with expected crc 2023-10-01T10:28:21.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:21 smithi173 bash[58816]: cluster 2023-10-01T10:28:20.192298+0000 osd.2 (osd.2) 10 : cluster 3 failed to encode map e87 with expected crc 2023-10-01T10:28:21.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:21 smithi173 bash[58816]: cluster 2023-10-01T10:28:20.194123+0000 osd.6 (osd.6) 7 : cluster 3 failed to encode map e87 with expected crc 2023-10-01T10:28:21.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:21 smithi173 bash[58816]: cluster 2023-10-01T10:28:20.194312+0000 osd.4 (osd.4) 11 : cluster 3 failed to encode map e87 with expected crc 2023-10-01T10:28:21.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:21 smithi173 bash[58816]: cluster 2023-10-01T10:28:20.195125+0000 osd.3 (osd.3) 11 : cluster 3 failed to encode map e87 with expected crc 2023-10-01T10:28:21.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:21 smithi173 bash[58816]: cluster 2023-10-01T10:28:20.195223+0000 osd.7 (osd.7) 15 : cluster 3 failed to encode map e87 with expected crc 2023-10-01T10:28:21.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:21 smithi173 bash[58816]: cluster 2023-10-01T10:28:20.197929+0000 osd.6 (osd.6) 8 : cluster 3 failed to encode map e87 with expected crc 2023-10-01T10:28:21.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:21 smithi173 bash[58816]: cluster 2023-10-01T10:28:20.198234+0000 osd.7 (osd.7) 16 : cluster 3 failed to encode map e87 with expected crc 2023-10-01T10:28:21.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:21 smithi173 bash[58816]: cluster 2023-10-01T10:28:20.200878+0000 osd.4 (osd.4) 12 : cluster 3 failed to encode map e87 with expected crc 2023-10-01T10:28:21.622 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:21 smithi173 bash[58816]: cluster 2023-10-01T10:28:21.196367+0000 mon.a (mon.0) 339 : cluster 0 osdmap e88: 8 total, 8 up, 8 in 2023-10-01T10:28:22.287 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 48 16 71606 71590 93.2067 2.35417 0.020198 0.0107208 2023-10-01T10:28:22.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:22 smithi078 bash[75299]: cluster 2023-10-01T10:28:21.192012+0000 osd.2 (osd.2) 11 : cluster 3 failed to encode map e88 with expected crc 2023-10-01T10:28:22.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:22 smithi078 bash[75299]: cluster 2023-10-01T10:28:21.194734+0000 osd.7 (osd.7) 17 : cluster 3 failed to encode map e88 with expected crc 2023-10-01T10:28:22.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:22 smithi078 bash[75299]: cluster 2023-10-01T10:28:21.196668+0000 osd.2 (osd.2) 12 : cluster 3 failed to encode map e88 with expected crc 2023-10-01T10:28:22.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:22 smithi078 bash[75299]: cluster 2023-10-01T10:28:21.196669+0000 osd.1 (osd.1) 13 : cluster 3 failed to encode map e88 with expected crc 2023-10-01T10:28:22.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:22 smithi078 bash[75299]: cluster 2023-10-01T10:28:21.197117+0000 osd.3 (osd.3) 12 : cluster 3 failed to encode map e88 with expected crc 2023-10-01T10:28:22.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:22 smithi078 bash[75299]: cluster 2023-10-01T10:28:21.198757+0000 osd.7 (osd.7) 18 : cluster 3 failed to encode map e88 with expected crc 2023-10-01T10:28:22.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:22 smithi078 bash[75299]: cluster 2023-10-01T10:28:21.198864+0000 osd.6 (osd.6) 9 : cluster 3 failed to encode map e88 with expected crc 2023-10-01T10:28:22.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:22 smithi078 bash[75299]: cluster 2023-10-01T10:28:21.381359+0000 mgr.y (mgr.25072) 237 : cluster 0 pgmap v182: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s; 35177/212274 objects degraded (16.572%) 2023-10-01T10:28:22.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:22 smithi078 bash[75299]: cluster 2023-10-01T10:28:21.755339+0000 mon.a (mon.0) 340 : cluster 3 Health check update: Degraded data redundancy: 35177/212274 objects degraded (16.572%), 15 pgs degraded (PG_DEGRADED) 2023-10-01T10:28:22.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:22 smithi078 bash[78274]: cluster 2023-10-01T10:28:21.192012+0000 osd.2 (osd.2) 11 : cluster 3 failed to encode map e88 with expected crc 2023-10-01T10:28:22.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:22 smithi078 bash[78274]: cluster 2023-10-01T10:28:21.194734+0000 osd.7 (osd.7) 17 : cluster 3 failed to encode map e88 with expected crc 2023-10-01T10:28:22.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:22 smithi078 bash[78274]: cluster 2023-10-01T10:28:21.196668+0000 osd.2 (osd.2) 12 : cluster 3 failed to encode map e88 with expected crc 2023-10-01T10:28:22.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:22 smithi078 bash[78274]: cluster 2023-10-01T10:28:21.196669+0000 osd.1 (osd.1) 13 : cluster 3 failed to encode map e88 with expected crc 2023-10-01T10:28:22.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:22 smithi078 bash[78274]: cluster 2023-10-01T10:28:21.197117+0000 osd.3 (osd.3) 12 : cluster 3 failed to encode map e88 with expected crc 2023-10-01T10:28:22.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:22 smithi078 bash[78274]: cluster 2023-10-01T10:28:21.198757+0000 osd.7 (osd.7) 18 : cluster 3 failed to encode map e88 with expected crc 2023-10-01T10:28:22.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:22 smithi078 bash[78274]: cluster 2023-10-01T10:28:21.198864+0000 osd.6 (osd.6) 9 : cluster 3 failed to encode map e88 with expected crc 2023-10-01T10:28:22.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:22 smithi078 bash[78274]: cluster 2023-10-01T10:28:21.381359+0000 mgr.y (mgr.25072) 237 : cluster 0 pgmap v182: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s; 35177/212274 objects degraded (16.572%) 2023-10-01T10:28:22.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:22 smithi078 bash[78274]: cluster 2023-10-01T10:28:21.755339+0000 mon.a (mon.0) 340 : cluster 3 Health check update: Degraded data redundancy: 35177/212274 objects degraded (16.572%), 15 pgs degraded (PG_DEGRADED) 2023-10-01T10:28:22.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:22 smithi173 bash[58816]: cluster 2023-10-01T10:28:21.192012+0000 osd.2 (osd.2) 11 : cluster 3 failed to encode map e88 with expected crc 2023-10-01T10:28:22.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:22 smithi173 bash[58816]: cluster 2023-10-01T10:28:21.194734+0000 osd.7 (osd.7) 17 : cluster 3 failed to encode map e88 with expected crc 2023-10-01T10:28:22.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:22 smithi173 bash[58816]: cluster 2023-10-01T10:28:21.196668+0000 osd.2 (osd.2) 12 : cluster 3 failed to encode map e88 with expected crc 2023-10-01T10:28:22.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:22 smithi173 bash[58816]: cluster 2023-10-01T10:28:21.196669+0000 osd.1 (osd.1) 13 : cluster 3 failed to encode map e88 with expected crc 2023-10-01T10:28:22.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:22 smithi173 bash[58816]: cluster 2023-10-01T10:28:21.197117+0000 osd.3 (osd.3) 12 : cluster 3 failed to encode map e88 with expected crc 2023-10-01T10:28:22.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:22 smithi173 bash[58816]: cluster 2023-10-01T10:28:21.198757+0000 osd.7 (osd.7) 18 : cluster 3 failed to encode map e88 with expected crc 2023-10-01T10:28:22.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:22 smithi173 bash[58816]: cluster 2023-10-01T10:28:21.198864+0000 osd.6 (osd.6) 9 : cluster 3 failed to encode map e88 with expected crc 2023-10-01T10:28:22.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:22 smithi173 bash[58816]: cluster 2023-10-01T10:28:21.381359+0000 mgr.y (mgr.25072) 237 : cluster 0 pgmap v182: 89 pgs: 15 active+undersized+degraded, 19 active+undersized, 55 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s; 35177/212274 objects degraded (16.572%) 2023-10-01T10:28:22.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:22 smithi173 bash[58816]: cluster 2023-10-01T10:28:21.755339+0000 mon.a (mon.0) 340 : cluster 3 Health check update: Degraded data redundancy: 35177/212274 objects degraded (16.572%), 15 pgs degraded (PG_DEGRADED) 2023-10-01T10:28:23.287 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 49 16 72418 72402 92.3402 50.75 0.0131993 0.0108226 2023-10-01T10:28:23.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:23 smithi078 bash[75299]: audit 2023-10-01T10:28:22.314541+0000 mon.a (mon.0) 341 : audit 0 from='client.? 172.21.15.78:0/3339779140' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:23.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:23 smithi078 bash[78274]: audit 2023-10-01T10:28:22.314541+0000 mon.a (mon.0) 341 : audit 0 from='client.? 172.21.15.78:0/3339779140' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:23.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:23 smithi173 bash[58816]: audit 2023-10-01T10:28:22.314541+0000 mon.a (mon.0) 341 : audit 0 from='client.? 172.21.15.78:0/3339779140' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:24.287 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 50 16 73298 73282 91.5933 55 0.0111102 0.0109112 2023-10-01T10:28:24.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:24 smithi078 bash[75299]: cluster 2023-10-01T10:28:23.382022+0000 mgr.y (mgr.25072) 238 : cluster 0 pgmap v183: 89 pgs: 13 active+undersized+degraded, 10 active+undersized, 66 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s; 35174/212274 objects degraded (16.570%) 2023-10-01T10:28:24.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:24 smithi078 bash[75299]: audit 2023-10-01T10:28:23.964565+0000 mon.a (mon.0) 342 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:28:24.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:24 smithi078 bash[75299]: audit 2023-10-01T10:28:23.973196+0000 mon.a (mon.0) 343 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:28:24.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:24 smithi078 bash[75299]: audit 2023-10-01T10:28:23.978914+0000 mon.a (mon.0) 344 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:28:24.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:24 smithi078 bash[75299]: audit 2023-10-01T10:28:23.980026+0000 mon.a (mon.0) 345 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:28:24.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:24 smithi078 bash[75299]: audit 2023-10-01T10:28:23.993294+0000 mon.a (mon.0) 346 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:28:24.585 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:24 smithi078 bash[75299]: audit 2023-10-01T10:28:24.081425+0000 mon.a (mon.0) 347 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:28:24.585 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:24 smithi078 bash[75299]: audit 2023-10-01T10:28:24.086221+0000 mon.a (mon.0) 348 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:28:24.585 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:24 smithi078 bash[75299]: audit 2023-10-01T10:28:24.089721+0000 mon.a (mon.0) 349 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:28:24.585 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:24 smithi078 bash[75299]: audit 2023-10-01T10:28:24.091988+0000 mon.a (mon.0) 350 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:28:24.585 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:24 smithi078 bash[75299]: audit 2023-10-01T10:28:24.094907+0000 mon.a (mon.0) 351 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:28:24.586 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:28:24 smithi078 bash[71214]: debug 2023-10-01T10:28:24.092+0000 7fd9c245e700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2023-10-01T10:28:24.586 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:24 smithi078 bash[78274]: cluster 2023-10-01T10:28:23.382022+0000 mgr.y (mgr.25072) 238 : cluster 0 pgmap v183: 89 pgs: 13 active+undersized+degraded, 10 active+undersized, 66 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s; 35174/212274 objects degraded (16.570%) 2023-10-01T10:28:24.587 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:24 smithi078 bash[78274]: audit 2023-10-01T10:28:23.964565+0000 mon.a (mon.0) 342 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:28:24.587 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:24 smithi078 bash[78274]: audit 2023-10-01T10:28:23.973196+0000 mon.a (mon.0) 343 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:28:24.587 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:24 smithi078 bash[78274]: audit 2023-10-01T10:28:23.978914+0000 mon.a (mon.0) 344 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:28:24.587 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:24 smithi078 bash[78274]: audit 2023-10-01T10:28:23.980026+0000 mon.a (mon.0) 345 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:28:24.587 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:24 smithi078 bash[78274]: audit 2023-10-01T10:28:23.993294+0000 mon.a (mon.0) 346 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:28:24.587 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:24 smithi078 bash[78274]: audit 2023-10-01T10:28:24.081425+0000 mon.a (mon.0) 347 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:28:24.587 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:24 smithi078 bash[78274]: audit 2023-10-01T10:28:24.086221+0000 mon.a (mon.0) 348 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:28:24.587 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:24 smithi078 bash[78274]: audit 2023-10-01T10:28:24.089721+0000 mon.a (mon.0) 349 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:28:24.588 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:24 smithi078 bash[78274]: audit 2023-10-01T10:28:24.091988+0000 mon.a (mon.0) 350 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:28:24.588 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:24 smithi078 bash[78274]: audit 2023-10-01T10:28:24.094907+0000 mon.a (mon.0) 351 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:28:24.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:24 smithi173 bash[58816]: cluster 2023-10-01T10:28:23.382022+0000 mgr.y (mgr.25072) 238 : cluster 0 pgmap v183: 89 pgs: 13 active+undersized+degraded, 10 active+undersized, 66 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s; 35174/212274 objects degraded (16.570%) 2023-10-01T10:28:24.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:24 smithi173 bash[58816]: audit 2023-10-01T10:28:23.964565+0000 mon.a (mon.0) 342 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:28:24.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:24 smithi173 bash[58816]: audit 2023-10-01T10:28:23.973196+0000 mon.a (mon.0) 343 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:28:24.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:24 smithi173 bash[58816]: audit 2023-10-01T10:28:23.978914+0000 mon.a (mon.0) 344 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:28:24.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:24 smithi173 bash[58816]: audit 2023-10-01T10:28:23.980026+0000 mon.a (mon.0) 345 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:28:24.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:24 smithi173 bash[58816]: audit 2023-10-01T10:28:23.993294+0000 mon.a (mon.0) 346 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:28:24.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:24 smithi173 bash[58816]: audit 2023-10-01T10:28:24.081425+0000 mon.a (mon.0) 347 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:28:24.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:24 smithi173 bash[58816]: audit 2023-10-01T10:28:24.086221+0000 mon.a (mon.0) 348 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:28:24.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:24 smithi173 bash[58816]: audit 2023-10-01T10:28:24.089721+0000 mon.a (mon.0) 349 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:28:24.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:24 smithi173 bash[58816]: audit 2023-10-01T10:28:24.091988+0000 mon.a (mon.0) 350 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:28:24.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:24 smithi173 bash[58816]: audit 2023-10-01T10:28:24.094907+0000 mon.a (mon.0) 351 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:28:25.287 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 51 16 74107 74091 90.7886 50.5625 0.0154747 0.0110078 2023-10-01T10:28:25.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:25 smithi078 bash[78274]: audit 2023-10-01T10:28:24.095161+0000 mgr.y (mgr.25072) 239 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:28:25.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:25 smithi078 bash[78274]: cephadm 2023-10-01T10:28:24.096681+0000 mgr.y (mgr.25072) 240 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2023-10-01T10:28:25.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:25 smithi078 bash[78274]: audit 2023-10-01T10:28:24.880273+0000 mon.c (mon.2) 13 : audit 0 from='client.? 172.21.15.78:0/336411518' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:25.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:25 smithi078 bash[75299]: audit 2023-10-01T10:28:24.095161+0000 mgr.y (mgr.25072) 239 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:28:25.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:25 smithi078 bash[75299]: cephadm 2023-10-01T10:28:24.096681+0000 mgr.y (mgr.25072) 240 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2023-10-01T10:28:25.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:25 smithi078 bash[75299]: audit 2023-10-01T10:28:24.880273+0000 mon.c (mon.2) 13 : audit 0 from='client.? 172.21.15.78:0/336411518' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:25.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:25 smithi173 bash[58816]: audit 2023-10-01T10:28:24.095161+0000 mgr.y (mgr.25072) 239 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:28:25.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:25 smithi173 bash[58816]: cephadm 2023-10-01T10:28:24.096681+0000 mgr.y (mgr.25072) 240 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2023-10-01T10:28:25.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:25 smithi173 bash[58816]: audit 2023-10-01T10:28:24.880273+0000 mon.c (mon.2) 13 : audit 0 from='client.? 172.21.15.78:0/336411518' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:26.287 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 52 16 74926 74910 90.027 51.1875 0.0103709 0.0111009 2023-10-01T10:28:26.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:26 smithi078 bash[75299]: cluster 2023-10-01T10:28:25.382970+0000 mgr.y (mgr.25072) 241 : cluster 0 pgmap v184: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 21 MiB/s wr, 332 op/s; 14813/218460 objects degraded (6.781%); 609 KiB/s, 9 objects/s recovering 2023-10-01T10:28:26.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:26 smithi078 bash[78274]: cluster 2023-10-01T10:28:25.382970+0000 mgr.y (mgr.25072) 241 : cluster 0 pgmap v184: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 21 MiB/s wr, 332 op/s; 14813/218460 objects degraded (6.781%); 609 KiB/s, 9 objects/s recovering 2023-10-01T10:28:26.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:26 smithi173 bash[58816]: cluster 2023-10-01T10:28:25.382970+0000 mgr.y (mgr.25072) 241 : cluster 0 pgmap v184: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 21 MiB/s wr, 332 op/s; 14813/218460 objects degraded (6.781%); 609 KiB/s, 9 objects/s recovering 2023-10-01T10:28:27.287 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 53 16 75755 75739 89.3058 51.8125 0.0167035 0.0111908 2023-10-01T10:28:27.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:27 smithi078 bash[75299]: cluster 2023-10-01T10:28:26.756690+0000 mon.a (mon.0) 352 : cluster 3 Health check update: Degraded data redundancy: 14813/218460 objects degraded (6.781%), 8 pgs degraded (PG_DEGRADED) 2023-10-01T10:28:27.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:27 smithi078 bash[78274]: cluster 2023-10-01T10:28:26.756690+0000 mon.a (mon.0) 352 : cluster 3 Health check update: Degraded data redundancy: 14813/218460 objects degraded (6.781%), 8 pgs degraded (PG_DEGRADED) 2023-10-01T10:28:27.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:27 smithi173 bash[58816]: cluster 2023-10-01T10:28:26.756690+0000 mon.a (mon.0) 352 : cluster 3 Health check update: Degraded data redundancy: 14813/218460 objects degraded (6.781%), 8 pgs degraded (PG_DEGRADED) 2023-10-01T10:28:28.287 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 54 16 76544 76528 88.5651 49.3125 0.0268937 0.0112852 2023-10-01T10:28:28.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:28 smithi078 bash[75299]: cluster 2023-10-01T10:28:27.383488+0000 mgr.y (mgr.25072) 242 : cluster 0 pgmap v185: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 16 MiB/s wr, 257 op/s; 14813/218460 objects degraded (6.781%); 472 KiB/s, 7 objects/s recovering 2023-10-01T10:28:28.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:28 smithi078 bash[75299]: audit 2023-10-01T10:28:27.544520+0000 mon.a (mon.0) 353 : audit 0 from='client.? 172.21.15.78:0/2699247333' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:28.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:28 smithi078 bash[78274]: cluster 2023-10-01T10:28:27.383488+0000 mgr.y (mgr.25072) 242 : cluster 0 pgmap v185: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 16 MiB/s wr, 257 op/s; 14813/218460 objects degraded (6.781%); 472 KiB/s, 7 objects/s recovering 2023-10-01T10:28:28.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:28 smithi078 bash[78274]: audit 2023-10-01T10:28:27.544520+0000 mon.a (mon.0) 353 : audit 0 from='client.? 172.21.15.78:0/2699247333' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:28.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:28 smithi173 bash[58816]: cluster 2023-10-01T10:28:27.383488+0000 mgr.y (mgr.25072) 242 : cluster 0 pgmap v185: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 16 MiB/s wr, 257 op/s; 14813/218460 objects degraded (6.781%); 472 KiB/s, 7 objects/s recovering 2023-10-01T10:28:28.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:28 smithi173 bash[58816]: audit 2023-10-01T10:28:27.544520+0000 mon.a (mon.0) 353 : audit 0 from='client.? 172.21.15.78:0/2699247333' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:29.287 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 55 16 77377 77361 87.9013 52.0625 0.0156321 0.0113695 2023-10-01T10:28:30.287 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 56 16 78143 78127 87.1865 47.875 0.0141554 0.0114636 2023-10-01T10:28:30.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:30 smithi078 bash[75299]: cluster 2023-10-01T10:28:29.384345+0000 mgr.y (mgr.25072) 243 : cluster 0 pgmap v186: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 42 MiB/s wr, 670 op/s; 14606/230763 objects degraded (6.329%); 1.8 MiB/s, 28 objects/s recovering 2023-10-01T10:28:30.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:30 smithi078 bash[75299]: audit 2023-10-01T10:28:30.190573+0000 mon.c (mon.2) 14 : audit 0 from='client.? 172.21.15.78:0/201210165' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:30.584 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:28:30 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:28:30] "GET /metrics HTTP/1.1" 200 36199 "" "Prometheus/2.33.4" 2023-10-01T10:28:30.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:30 smithi078 bash[78274]: cluster 2023-10-01T10:28:29.384345+0000 mgr.y (mgr.25072) 243 : cluster 0 pgmap v186: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 42 MiB/s wr, 670 op/s; 14606/230763 objects degraded (6.329%); 1.8 MiB/s, 28 objects/s recovering 2023-10-01T10:28:30.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:30 smithi078 bash[78274]: audit 2023-10-01T10:28:30.190573+0000 mon.c (mon.2) 14 : audit 0 from='client.? 172.21.15.78:0/201210165' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:30.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:30 smithi173 bash[58816]: cluster 2023-10-01T10:28:29.384345+0000 mgr.y (mgr.25072) 243 : cluster 0 pgmap v186: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 42 MiB/s wr, 670 op/s; 14606/230763 objects degraded (6.329%); 1.8 MiB/s, 28 objects/s recovering 2023-10-01T10:28:30.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:30 smithi173 bash[58816]: audit 2023-10-01T10:28:30.190573+0000 mon.c (mon.2) 14 : audit 0 from='client.? 172.21.15.78:0/201210165' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:31.287 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 57 16 78944 78928 86.5351 50.0625 0.0285351 0.0115501 2023-10-01T10:28:32.288 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 58 16 79760 79744 85.9224 51 0.0186016 0.0116329 2023-10-01T10:28:32.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:32 smithi078 bash[78274]: cluster 2023-10-01T10:28:31.384872+0000 mgr.y (mgr.25072) 244 : cluster 0 pgmap v187: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 38 MiB/s wr, 604 op/s; 14606/230763 objects degraded (6.329%); 1.6 MiB/s, 26 objects/s recovering 2023-10-01T10:28:32.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:32 smithi078 bash[78274]: cluster 2023-10-01T10:28:31.759691+0000 mon.a (mon.0) 354 : cluster 3 Health check update: Degraded data redundancy: 14606/230763 objects degraded (6.329%), 8 pgs degraded (PG_DEGRADED) 2023-10-01T10:28:32.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:32 smithi078 bash[78274]: audit 2023-10-01T10:28:32.395581+0000 mon.a (mon.0) 355 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:28:32.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:32 smithi078 bash[78274]: audit 2023-10-01T10:28:32.396997+0000 mon.a (mon.0) 356 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:28:32.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:32 smithi078 bash[75299]: cluster 2023-10-01T10:28:31.384872+0000 mgr.y (mgr.25072) 244 : cluster 0 pgmap v187: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 38 MiB/s wr, 604 op/s; 14606/230763 objects degraded (6.329%); 1.6 MiB/s, 26 objects/s recovering 2023-10-01T10:28:32.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:32 smithi078 bash[75299]: cluster 2023-10-01T10:28:31.759691+0000 mon.a (mon.0) 354 : cluster 3 Health check update: Degraded data redundancy: 14606/230763 objects degraded (6.329%), 8 pgs degraded (PG_DEGRADED) 2023-10-01T10:28:32.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:32 smithi078 bash[75299]: audit 2023-10-01T10:28:32.395581+0000 mon.a (mon.0) 355 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:28:32.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:32 smithi078 bash[75299]: audit 2023-10-01T10:28:32.396997+0000 mon.a (mon.0) 356 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:28:32.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:32 smithi173 bash[58816]: cluster 2023-10-01T10:28:31.384872+0000 mgr.y (mgr.25072) 244 : cluster 0 pgmap v187: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 38 MiB/s wr, 604 op/s; 14606/230763 objects degraded (6.329%); 1.6 MiB/s, 26 objects/s recovering 2023-10-01T10:28:32.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:32 smithi173 bash[58816]: cluster 2023-10-01T10:28:31.759691+0000 mon.a (mon.0) 354 : cluster 3 Health check update: Degraded data redundancy: 14606/230763 objects degraded (6.329%), 8 pgs degraded (PG_DEGRADED) 2023-10-01T10:28:32.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:32 smithi173 bash[58816]: audit 2023-10-01T10:28:32.395581+0000 mon.a (mon.0) 355 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:28:32.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:32 smithi173 bash[58816]: audit 2023-10-01T10:28:32.396997+0000 mon.a (mon.0) 356 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:28:33.288 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 59 16 80523 80507 85.2742 47.6875 0.0249783 0.0117208 2023-10-01T10:28:33.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:33 smithi078 bash[75299]: audit 2023-10-01T10:28:32.819380+0000 mon.a (mon.0) 357 : audit 0 from='client.? 172.21.15.78:0/4028983250' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:33.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:33 smithi078 bash[78274]: audit 2023-10-01T10:28:32.819380+0000 mon.a (mon.0) 357 : audit 0 from='client.? 172.21.15.78:0/4028983250' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:33.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:33 smithi173 bash[58816]: audit 2023-10-01T10:28:32.819380+0000 mon.a (mon.0) 357 : audit 0 from='client.? 172.21.15.78:0/4028983250' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:34.288 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:2023-10-01T10:28:34.285955+0000 min lat: 0.00149611 max lat: 2.99842 avg lat: 0.0118022 2023-10-01T10:28:34.288 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T10:28:34.288 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 60 16 81309 81293 84.6717 49.125 0.0523774 0.0118022 2023-10-01T10:28:34.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:34 smithi078 bash[75299]: cluster 2023-10-01T10:28:33.385513+0000 mgr.y (mgr.25072) 245 : cluster 0 pgmap v188: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 32 MiB/s wr, 513 op/s; 14606/230763 objects degraded (6.329%); 1.4 MiB/s, 22 objects/s recovering 2023-10-01T10:28:34.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:34 smithi078 bash[78274]: cluster 2023-10-01T10:28:33.385513+0000 mgr.y (mgr.25072) 245 : cluster 0 pgmap v188: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 32 MiB/s wr, 513 op/s; 14606/230763 objects degraded (6.329%); 1.4 MiB/s, 22 objects/s recovering 2023-10-01T10:28:34.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:34 smithi173 bash[58816]: cluster 2023-10-01T10:28:33.385513+0000 mgr.y (mgr.25072) 245 : cluster 0 pgmap v188: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 32 MiB/s wr, 513 op/s; 14606/230763 objects degraded (6.329%); 1.4 MiB/s, 22 objects/s recovering 2023-10-01T10:28:35.288 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 61 16 82065 82049 84.0581 47.25 0.0162118 0.0118901 2023-10-01T10:28:35.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:35 smithi078 bash[75299]: audit 2023-10-01T10:28:35.452832+0000 mon.b (mon.1) 8 : audit 0 from='client.? 172.21.15.78:0/36072386' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:35.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:35 smithi078 bash[78274]: audit 2023-10-01T10:28:35.452832+0000 mon.b (mon.1) 8 : audit 0 from='client.? 172.21.15.78:0/36072386' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:35.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:35 smithi173 bash[58816]: audit 2023-10-01T10:28:35.452832+0000 mon.b (mon.1) 8 : audit 0 from='client.? 172.21.15.78:0/36072386' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:36.288 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 62 16 82826 82810 83.4694 47.5625 0.0430418 0.0119737 2023-10-01T10:28:36.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:36 smithi078 bash[75299]: cluster 2023-10-01T10:28:35.386380+0000 mgr.y (mgr.25072) 246 : cluster 0 pgmap v189: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 53 MiB/s wr, 843 op/s; 14403/242664 objects degraded (5.935%); 2.4 MiB/s, 39 objects/s recovering 2023-10-01T10:28:36.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:36 smithi078 bash[78274]: cluster 2023-10-01T10:28:35.386380+0000 mgr.y (mgr.25072) 246 : cluster 0 pgmap v189: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 53 MiB/s wr, 843 op/s; 14403/242664 objects degraded (5.935%); 2.4 MiB/s, 39 objects/s recovering 2023-10-01T10:28:36.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:36 smithi173 bash[58816]: cluster 2023-10-01T10:28:35.386380+0000 mgr.y (mgr.25072) 246 : cluster 0 pgmap v189: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 53 MiB/s wr, 843 op/s; 14403/242664 objects degraded (5.935%); 2.4 MiB/s, 39 objects/s recovering 2023-10-01T10:28:37.288 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 63 16 83625 83609 82.9371 49.9375 0.0128836 0.0120515 2023-10-01T10:28:37.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:37 smithi078 bash[75299]: cluster 2023-10-01T10:28:36.763796+0000 mon.a (mon.0) 358 : cluster 3 Health check update: Degraded data redundancy: 14403/242664 objects degraded (5.935%), 8 pgs degraded (PG_DEGRADED) 2023-10-01T10:28:37.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:37 smithi078 bash[78274]: cluster 2023-10-01T10:28:36.763796+0000 mon.a (mon.0) 358 : cluster 3 Health check update: Degraded data redundancy: 14403/242664 objects degraded (5.935%), 8 pgs degraded (PG_DEGRADED) 2023-10-01T10:28:37.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:37 smithi173 bash[58816]: cluster 2023-10-01T10:28:36.763796+0000 mon.a (mon.0) 358 : cluster 3 Health check update: Degraded data redundancy: 14403/242664 objects degraded (5.935%), 8 pgs degraded (PG_DEGRADED) 2023-10-01T10:28:38.288 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 64 16 84370 84354 82.3686 46.5625 0.00481391 0.0121344 2023-10-01T10:28:38.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:38 smithi078 bash[75299]: cluster 2023-10-01T10:28:37.387149+0000 mgr.y (mgr.25072) 247 : cluster 0 pgmap v190: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 42 MiB/s wr, 672 op/s; 14403/242664 objects degraded (5.935%); 2.1 MiB/s, 34 objects/s recovering 2023-10-01T10:28:38.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:38 smithi078 bash[75299]: audit 2023-10-01T10:28:38.107905+0000 mon.c (mon.2) 15 : audit 0 from='client.? 172.21.15.78:0/447274639' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:38.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:38 smithi078 bash[78274]: cluster 2023-10-01T10:28:37.387149+0000 mgr.y (mgr.25072) 247 : cluster 0 pgmap v190: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 42 MiB/s wr, 672 op/s; 14403/242664 objects degraded (5.935%); 2.1 MiB/s, 34 objects/s recovering 2023-10-01T10:28:38.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:38 smithi078 bash[78274]: audit 2023-10-01T10:28:38.107905+0000 mon.c (mon.2) 15 : audit 0 from='client.? 172.21.15.78:0/447274639' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:38.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:38 smithi173 bash[58816]: cluster 2023-10-01T10:28:37.387149+0000 mgr.y (mgr.25072) 247 : cluster 0 pgmap v190: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 42 MiB/s wr, 672 op/s; 14403/242664 objects degraded (5.935%); 2.1 MiB/s, 34 objects/s recovering 2023-10-01T10:28:38.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:38 smithi173 bash[58816]: audit 2023-10-01T10:28:38.107905+0000 mon.c (mon.2) 15 : audit 0 from='client.? 172.21.15.78:0/447274639' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:39.288 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 65 16 85155 85139 81.8562 49.0625 0.00829601 0.0122111 2023-10-01T10:28:39.490 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:39 smithi078 bash[78274]: audit 2023-10-01T10:28:39.110550+0000 mon.a (mon.0) 359 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:28:39.490 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:39 smithi078 bash[75299]: audit 2023-10-01T10:28:39.110550+0000 mon.a (mon.0) 359 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:28:39.490 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:28:39 smithi078 bash[71214]: debug 2023-10-01T10:28:39.108+0000 7fd9c245e700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-01T10:28:39.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:39 smithi173 bash[58816]: audit 2023-10-01T10:28:39.110550+0000 mon.a (mon.0) 359 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:28:40.288 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 66 16 85923 85907 81.3431 48 0.0472438 0.012286 2023-10-01T10:28:40.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:40 smithi078 bash[78274]: audit 2023-10-01T10:28:39.110960+0000 mgr.y (mgr.25072) 248 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:28:40.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:40 smithi078 bash[78274]: cephadm 2023-10-01T10:28:39.112286+0000 mgr.y (mgr.25072) 249 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-01T10:28:40.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:40 smithi078 bash[78274]: cluster 2023-10-01T10:28:39.390546+0000 mgr.y (mgr.25072) 250 : cluster 0 pgmap v191: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 62 MiB/s wr, 993 op/s; 14192/254247 objects degraded (5.582%); 3.2 MiB/s, 51 objects/s recovering 2023-10-01T10:28:40.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:40 smithi078 bash[75299]: audit 2023-10-01T10:28:39.110960+0000 mgr.y (mgr.25072) 248 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:28:40.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:40 smithi078 bash[75299]: cephadm 2023-10-01T10:28:39.112286+0000 mgr.y (mgr.25072) 249 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-01T10:28:40.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:40 smithi078 bash[75299]: cluster 2023-10-01T10:28:39.390546+0000 mgr.y (mgr.25072) 250 : cluster 0 pgmap v191: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 62 MiB/s wr, 993 op/s; 14192/254247 objects degraded (5.582%); 3.2 MiB/s, 51 objects/s recovering 2023-10-01T10:28:40.584 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:28:40 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:28:40] "GET /metrics HTTP/1.1" 200 36213 "" "Prometheus/2.33.4" 2023-10-01T10:28:40.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:40 smithi173 bash[58816]: audit 2023-10-01T10:28:39.110960+0000 mgr.y (mgr.25072) 248 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:28:40.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:40 smithi173 bash[58816]: cephadm 2023-10-01T10:28:39.112286+0000 mgr.y (mgr.25072) 249 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-01T10:28:40.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:40 smithi173 bash[58816]: cluster 2023-10-01T10:28:39.390546+0000 mgr.y (mgr.25072) 250 : cluster 0 pgmap v191: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 62 MiB/s wr, 993 op/s; 14192/254247 objects degraded (5.582%); 3.2 MiB/s, 51 objects/s recovering 2023-10-01T10:28:41.289 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 67 16 86668 86652 80.8239 46.5625 0.0160071 0.012366 2023-10-01T10:28:41.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:41 smithi078 bash[75299]: audit 2023-10-01T10:28:40.703968+0000 mon.c (mon.2) 16 : audit 0 from='client.? 172.21.15.78:0/2760065714' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:41.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:41 smithi078 bash[78274]: audit 2023-10-01T10:28:40.703968+0000 mon.c (mon.2) 16 : audit 0 from='client.? 172.21.15.78:0/2760065714' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:41.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:41 smithi173 bash[58816]: audit 2023-10-01T10:28:40.703968+0000 mon.c (mon.2) 16 : audit 0 from='client.? 172.21.15.78:0/2760065714' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:42.289 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 68 16 87467 87451 80.3696 49.9375 0.047221 0.0124364 2023-10-01T10:28:42.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:42 smithi078 bash[78274]: cluster 2023-10-01T10:28:41.391103+0000 mgr.y (mgr.25072) 251 : cluster 0 pgmap v192: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 41 MiB/s wr, 651 op/s; 14192/254247 objects degraded (5.582%); 2.2 MiB/s, 34 objects/s recovering 2023-10-01T10:28:42.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:42 smithi078 bash[78274]: cluster 2023-10-01T10:28:41.765031+0000 mon.a (mon.0) 360 : cluster 3 Health check update: Degraded data redundancy: 14192/254247 objects degraded (5.582%), 8 pgs degraded (PG_DEGRADED) 2023-10-01T10:28:42.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:42 smithi078 bash[75299]: cluster 2023-10-01T10:28:41.391103+0000 mgr.y (mgr.25072) 251 : cluster 0 pgmap v192: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 41 MiB/s wr, 651 op/s; 14192/254247 objects degraded (5.582%); 2.2 MiB/s, 34 objects/s recovering 2023-10-01T10:28:42.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:42 smithi078 bash[75299]: cluster 2023-10-01T10:28:41.765031+0000 mon.a (mon.0) 360 : cluster 3 Health check update: Degraded data redundancy: 14192/254247 objects degraded (5.582%), 8 pgs degraded (PG_DEGRADED) 2023-10-01T10:28:42.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:42 smithi173 bash[58816]: cluster 2023-10-01T10:28:41.391103+0000 mgr.y (mgr.25072) 251 : cluster 0 pgmap v192: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 41 MiB/s wr, 651 op/s; 14192/254247 objects degraded (5.582%); 2.2 MiB/s, 34 objects/s recovering 2023-10-01T10:28:42.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:42 smithi173 bash[58816]: cluster 2023-10-01T10:28:41.765031+0000 mon.a (mon.0) 360 : cluster 3 Health check update: Degraded data redundancy: 14192/254247 objects degraded (5.582%), 8 pgs degraded (PG_DEGRADED) 2023-10-01T10:28:43.289 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 69 16 88197 88181 79.866 45.625 0.016413 0.012515 2023-10-01T10:28:43.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:43 smithi078 bash[78274]: audit 2023-10-01T10:28:43.326055+0000 mon.c (mon.2) 17 : audit 0 from='client.? 172.21.15.78:0/2335916167' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:43.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:43 smithi078 bash[75299]: audit 2023-10-01T10:28:43.326055+0000 mon.c (mon.2) 17 : audit 0 from='client.? 172.21.15.78:0/2335916167' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:43.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:43 smithi173 bash[58816]: audit 2023-10-01T10:28:43.326055+0000 mon.c (mon.2) 17 : audit 0 from='client.? 172.21.15.78:0/2335916167' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:44.289 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 70 16 88950 88934 79.3973 47.0625 0.00826296 0.0125885 2023-10-01T10:28:44.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:44 smithi078 bash[78274]: cluster 2023-10-01T10:28:43.391744+0000 mgr.y (mgr.25072) 252 : cluster 0 pgmap v193: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 41 MiB/s wr, 651 op/s; 14192/254247 objects degraded (5.582%); 2.2 MiB/s, 34 objects/s recovering 2023-10-01T10:28:44.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:44 smithi078 bash[75299]: cluster 2023-10-01T10:28:43.391744+0000 mgr.y (mgr.25072) 252 : cluster 0 pgmap v193: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 41 MiB/s wr, 651 op/s; 14192/254247 objects degraded (5.582%); 2.2 MiB/s, 34 objects/s recovering 2023-10-01T10:28:44.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:44 smithi173 bash[58816]: cluster 2023-10-01T10:28:43.391744+0000 mgr.y (mgr.25072) 252 : cluster 0 pgmap v193: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 41 MiB/s wr, 651 op/s; 14192/254247 objects degraded (5.582%); 2.2 MiB/s, 34 objects/s recovering 2023-10-01T10:28:45.289 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 71 16 89715 89699 78.9524 47.8125 0.00805103 0.0126572 2023-10-01T10:28:46.289 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 72 16 90459 90443 78.5016 46.5 0.0571858 0.0127287 2023-10-01T10:28:46.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:46 smithi078 bash[78274]: cluster 2023-10-01T10:28:45.392526+0000 mgr.y (mgr.25072) 253 : cluster 0 pgmap v194: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 61 MiB/s wr, 969 op/s; 13978/265680 objects degraded (5.261%); 3.3 MiB/s, 52 objects/s recovering 2023-10-01T10:28:46.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:46 smithi078 bash[78274]: audit 2023-10-01T10:28:45.943487+0000 mon.c (mon.2) 18 : audit 0 from='client.? 172.21.15.78:0/2138287046' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:46.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:46 smithi078 bash[75299]: cluster 2023-10-01T10:28:45.392526+0000 mgr.y (mgr.25072) 253 : cluster 0 pgmap v194: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 61 MiB/s wr, 969 op/s; 13978/265680 objects degraded (5.261%); 3.3 MiB/s, 52 objects/s recovering 2023-10-01T10:28:46.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:46 smithi078 bash[75299]: audit 2023-10-01T10:28:45.943487+0000 mon.c (mon.2) 18 : audit 0 from='client.? 172.21.15.78:0/2138287046' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:46.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:46 smithi173 bash[58816]: cluster 2023-10-01T10:28:45.392526+0000 mgr.y (mgr.25072) 253 : cluster 0 pgmap v194: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 61 MiB/s wr, 969 op/s; 13978/265680 objects degraded (5.261%); 3.3 MiB/s, 52 objects/s recovering 2023-10-01T10:28:46.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:46 smithi173 bash[58816]: audit 2023-10-01T10:28:45.943487+0000 mon.c (mon.2) 18 : audit 0 from='client.? 172.21.15.78:0/2138287046' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:47.289 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 73 16 91270 91254 78.1205 50.6875 0.0103352 0.0127952 2023-10-01T10:28:47.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:47 smithi078 bash[75299]: cluster 2023-10-01T10:28:46.766262+0000 mon.a (mon.0) 361 : cluster 3 Health check update: Degraded data redundancy: 13978/265680 objects degraded (5.261%), 8 pgs degraded (PG_DEGRADED) 2023-10-01T10:28:47.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:47 smithi078 bash[75299]: audit 2023-10-01T10:28:47.388245+0000 mon.a (mon.0) 362 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:28:47.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:47 smithi078 bash[78274]: cluster 2023-10-01T10:28:46.766262+0000 mon.a (mon.0) 361 : cluster 3 Health check update: Degraded data redundancy: 13978/265680 objects degraded (5.261%), 8 pgs degraded (PG_DEGRADED) 2023-10-01T10:28:47.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:47 smithi078 bash[78274]: audit 2023-10-01T10:28:47.388245+0000 mon.a (mon.0) 362 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:28:47.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:47 smithi173 bash[58816]: cluster 2023-10-01T10:28:46.766262+0000 mon.a (mon.0) 361 : cluster 3 Health check update: Degraded data redundancy: 13978/265680 objects degraded (5.261%), 8 pgs degraded (PG_DEGRADED) 2023-10-01T10:28:47.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:47 smithi173 bash[58816]: audit 2023-10-01T10:28:47.388245+0000 mon.a (mon.0) 362 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:28:48.289 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 74 15 92045 92030 77.7202 48.5 0.0220023 0.0128596 2023-10-01T10:28:48.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:48 smithi078 bash[78274]: cluster 2023-10-01T10:28:47.394820+0000 mgr.y (mgr.25072) 254 : cluster 0 pgmap v195: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 40 MiB/s wr, 638 op/s; 13978/265680 objects degraded (5.261%); 2.2 MiB/s, 35 objects/s recovering 2023-10-01T10:28:48.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:48 smithi078 bash[75299]: cluster 2023-10-01T10:28:47.394820+0000 mgr.y (mgr.25072) 254 : cluster 0 pgmap v195: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 40 MiB/s wr, 638 op/s; 13978/265680 objects degraded (5.261%); 2.2 MiB/s, 35 objects/s recovering 2023-10-01T10:28:48.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:48 smithi173 bash[58816]: cluster 2023-10-01T10:28:47.394820+0000 mgr.y (mgr.25072) 254 : cluster 0 pgmap v195: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 40 MiB/s wr, 638 op/s; 13978/265680 objects degraded (5.261%); 2.2 MiB/s, 35 objects/s recovering 2023-10-01T10:28:49.289 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 75 16 92831 92815 77.338 49.0625 0.017573 0.0129245 2023-10-01T10:28:49.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:49 smithi078 bash[75299]: audit 2023-10-01T10:28:48.537692+0000 mon.a (mon.0) 363 : audit 0 from='client.? 172.21.15.78:0/3460850870' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:49.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:49 smithi078 bash[75299]: cluster 2023-10-01T10:28:49.395726+0000 mgr.y (mgr.25072) 255 : cluster 0 pgmap v196: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 958 op/s; 13702/277197 objects degraded (4.943%); 3.6 MiB/s, 58 objects/s recovering 2023-10-01T10:28:49.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:49 smithi078 bash[78274]: audit 2023-10-01T10:28:48.537692+0000 mon.a (mon.0) 363 : audit 0 from='client.? 172.21.15.78:0/3460850870' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:49.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:49 smithi078 bash[78274]: cluster 2023-10-01T10:28:49.395726+0000 mgr.y (mgr.25072) 255 : cluster 0 pgmap v196: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 958 op/s; 13702/277197 objects degraded (4.943%); 3.6 MiB/s, 58 objects/s recovering 2023-10-01T10:28:49.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:49 smithi173 bash[58816]: audit 2023-10-01T10:28:48.537692+0000 mon.a (mon.0) 363 : audit 0 from='client.? 172.21.15.78:0/3460850870' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:49.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:49 smithi173 bash[58816]: cluster 2023-10-01T10:28:49.395726+0000 mgr.y (mgr.25072) 255 : cluster 0 pgmap v196: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 60 MiB/s wr, 958 op/s; 13702/277197 objects degraded (4.943%); 3.6 MiB/s, 58 objects/s recovering 2023-10-01T10:28:50.289 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 76 16 93614 93598 76.9643 48.9375 0.0133668 0.0129847 2023-10-01T10:28:50.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:28:50 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:28:50] "GET /metrics HTTP/1.1" 200 36214 "" "Prometheus/2.33.4" 2023-10-01T10:28:51.289 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 77 16 94375 94359 76.5824 47.5625 0.0166801 0.0130504 2023-10-01T10:28:51.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:51 smithi078 bash[78274]: audit 2023-10-01T10:28:51.119396+0000 mon.c (mon.2) 19 : audit 0 from='client.? 172.21.15.78:0/973824387' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:51.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:51 smithi078 bash[75299]: audit 2023-10-01T10:28:51.119396+0000 mon.c (mon.2) 19 : audit 0 from='client.? 172.21.15.78:0/973824387' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:51.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:51 smithi173 bash[58816]: audit 2023-10-01T10:28:51.119396+0000 mon.c (mon.2) 19 : audit 0 from='client.? 172.21.15.78:0/973824387' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:52.290 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 78 16 95167 95151 76.2351 49.5 0.0183535 0.0131106 2023-10-01T10:28:52.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:52 smithi078 bash[78274]: cluster 2023-10-01T10:28:51.396243+0000 mgr.y (mgr.25072) 256 : cluster 0 pgmap v197: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 40 MiB/s wr, 637 op/s; 13702/277197 objects degraded (4.943%); 2.6 MiB/s, 40 objects/s recovering 2023-10-01T10:28:52.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:52 smithi078 bash[78274]: cluster 2023-10-01T10:28:51.767401+0000 mon.a (mon.0) 364 : cluster 3 Health check update: Degraded data redundancy: 13702/277197 objects degraded (4.943%), 8 pgs degraded (PG_DEGRADED) 2023-10-01T10:28:52.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:52 smithi078 bash[75299]: cluster 2023-10-01T10:28:51.396243+0000 mgr.y (mgr.25072) 256 : cluster 0 pgmap v197: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 40 MiB/s wr, 637 op/s; 13702/277197 objects degraded (4.943%); 2.6 MiB/s, 40 objects/s recovering 2023-10-01T10:28:52.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:52 smithi078 bash[75299]: cluster 2023-10-01T10:28:51.767401+0000 mon.a (mon.0) 364 : cluster 3 Health check update: Degraded data redundancy: 13702/277197 objects degraded (4.943%), 8 pgs degraded (PG_DEGRADED) 2023-10-01T10:28:52.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:52 smithi173 bash[58816]: cluster 2023-10-01T10:28:51.396243+0000 mgr.y (mgr.25072) 256 : cluster 0 pgmap v197: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 40 MiB/s wr, 637 op/s; 13702/277197 objects degraded (4.943%); 2.6 MiB/s, 40 objects/s recovering 2023-10-01T10:28:52.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:52 smithi173 bash[58816]: cluster 2023-10-01T10:28:51.767401+0000 mon.a (mon.0) 364 : cluster 3 Health check update: Degraded data redundancy: 13702/277197 objects degraded (4.943%), 8 pgs degraded (PG_DEGRADED) 2023-10-01T10:28:53.290 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 79 16 95891 95875 75.8429 45.25 0.03184 0.0131789 2023-10-01T10:28:54.290 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:2023-10-01T10:28:54.287931+0000 min lat: 0.00149611 max lat: 2.99842 avg lat: 0.0132489 2023-10-01T10:28:54.290 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T10:28:54.290 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 80 16 96593 96577 75.4432 43.875 0.0336052 0.0132489 2023-10-01T10:28:54.462 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:54 smithi078 bash[75299]: cluster 2023-10-01T10:28:53.396880+0000 mgr.y (mgr.25072) 257 : cluster 0 pgmap v198: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 40 MiB/s wr, 637 op/s; 13702/277197 objects degraded (4.943%); 2.6 MiB/s, 40 objects/s recovering 2023-10-01T10:28:54.462 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:54 smithi078 bash[75299]: audit 2023-10-01T10:28:53.689090+0000 mon.a (mon.0) 365 : audit 0 from='client.? 172.21.15.78:0/411670' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:54.463 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:54 smithi078 bash[75299]: audit 2023-10-01T10:28:54.126967+0000 mon.a (mon.0) 366 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:28:54.463 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:28:54 smithi078 bash[71214]: debug 2023-10-01T10:28:54.127+0000 7fd9c245e700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-01T10:28:54.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:54 smithi078 bash[78274]: cluster 2023-10-01T10:28:53.396880+0000 mgr.y (mgr.25072) 257 : cluster 0 pgmap v198: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 40 MiB/s wr, 637 op/s; 13702/277197 objects degraded (4.943%); 2.6 MiB/s, 40 objects/s recovering 2023-10-01T10:28:54.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:54 smithi078 bash[78274]: audit 2023-10-01T10:28:53.689090+0000 mon.a (mon.0) 365 : audit 0 from='client.? 172.21.15.78:0/411670' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:54.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:54 smithi078 bash[78274]: audit 2023-10-01T10:28:54.126967+0000 mon.a (mon.0) 366 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:28:54.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:54 smithi173 bash[58816]: cluster 2023-10-01T10:28:53.396880+0000 mgr.y (mgr.25072) 257 : cluster 0 pgmap v198: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 40 MiB/s wr, 637 op/s; 13702/277197 objects degraded (4.943%); 2.6 MiB/s, 40 objects/s recovering 2023-10-01T10:28:54.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:54 smithi173 bash[58816]: audit 2023-10-01T10:28:53.689090+0000 mon.a (mon.0) 365 : audit 0 from='client.? 172.21.15.78:0/411670' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:54.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:54 smithi173 bash[58816]: audit 2023-10-01T10:28:54.126967+0000 mon.a (mon.0) 366 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:28:55.290 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 81 16 97344 97328 75.0912 46.9375 0.0170372 0.0133102 2023-10-01T10:28:55.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:55 smithi078 bash[75299]: audit 2023-10-01T10:28:54.129972+0000 mgr.y (mgr.25072) 258 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:28:55.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:55 smithi078 bash[75299]: cephadm 2023-10-01T10:28:54.131565+0000 mgr.y (mgr.25072) 259 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-01T10:28:55.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:55 smithi078 bash[78274]: audit 2023-10-01T10:28:54.129972+0000 mgr.y (mgr.25072) 258 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:28:55.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:55 smithi078 bash[78274]: cephadm 2023-10-01T10:28:54.131565+0000 mgr.y (mgr.25072) 259 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-01T10:28:55.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:55 smithi173 bash[58816]: audit 2023-10-01T10:28:54.129972+0000 mgr.y (mgr.25072) 258 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:28:55.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:55 smithi173 bash[58816]: cephadm 2023-10-01T10:28:54.131565+0000 mgr.y (mgr.25072) 259 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-01T10:28:56.290 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 82 16 98022 98006 74.6922 42.375 0.0304362 0.0133814 2023-10-01T10:28:56.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:56 smithi078 bash[75299]: cluster 2023-10-01T10:28:55.397704+0000 mgr.y (mgr.25072) 260 : cluster 0 pgmap v199: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 957 op/s; 13443/288735 objects degraded (4.656%); 3.9 MiB/s, 62 objects/s recovering 2023-10-01T10:28:56.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:56 smithi078 bash[75299]: audit 2023-10-01T10:28:56.287496+0000 mon.a (mon.0) 367 : audit 0 from='client.? 172.21.15.78:0/980550793' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:56.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:56 smithi078 bash[78274]: cluster 2023-10-01T10:28:55.397704+0000 mgr.y (mgr.25072) 260 : cluster 0 pgmap v199: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 957 op/s; 13443/288735 objects degraded (4.656%); 3.9 MiB/s, 62 objects/s recovering 2023-10-01T10:28:56.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:56 smithi078 bash[78274]: audit 2023-10-01T10:28:56.287496+0000 mon.a (mon.0) 367 : audit 0 from='client.? 172.21.15.78:0/980550793' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:56.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:56 smithi173 bash[58816]: cluster 2023-10-01T10:28:55.397704+0000 mgr.y (mgr.25072) 260 : cluster 0 pgmap v199: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 957 op/s; 13443/288735 objects degraded (4.656%); 3.9 MiB/s, 62 objects/s recovering 2023-10-01T10:28:56.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:56 smithi173 bash[58816]: audit 2023-10-01T10:28:56.287496+0000 mon.a (mon.0) 367 : audit 0 from='client.? 172.21.15.78:0/980550793' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:57.290 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 83 16 98753 98737 74.3427 45.6875 0.0111599 0.0134443 2023-10-01T10:28:57.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:57 smithi078 bash[75299]: cluster 2023-10-01T10:28:56.768690+0000 mon.a (mon.0) 368 : cluster 3 Health check update: Degraded data redundancy: 13443/288735 objects degraded (4.656%), 8 pgs degraded (PG_DEGRADED) 2023-10-01T10:28:57.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:57 smithi078 bash[78274]: cluster 2023-10-01T10:28:56.768690+0000 mon.a (mon.0) 368 : cluster 3 Health check update: Degraded data redundancy: 13443/288735 objects degraded (4.656%), 8 pgs degraded (PG_DEGRADED) 2023-10-01T10:28:57.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:57 smithi173 bash[58816]: cluster 2023-10-01T10:28:56.768690+0000 mon.a (mon.0) 368 : cluster 3 Health check update: Degraded data redundancy: 13443/288735 objects degraded (4.656%), 8 pgs degraded (PG_DEGRADED) 2023-10-01T10:28:58.290 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 84 16 99486 99470 74.003 45.8125 0.00874137 0.0135072 2023-10-01T10:28:58.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:58 smithi078 bash[78274]: cluster 2023-10-01T10:28:57.398276+0000 mgr.y (mgr.25072) 261 : cluster 0 pgmap v200: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 40 MiB/s wr, 640 op/s; 13443/288735 objects degraded (4.656%); 2.8 MiB/s, 44 objects/s recovering 2023-10-01T10:28:58.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:58 smithi078 bash[75299]: cluster 2023-10-01T10:28:57.398276+0000 mgr.y (mgr.25072) 261 : cluster 0 pgmap v200: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 40 MiB/s wr, 640 op/s; 13443/288735 objects degraded (4.656%); 2.8 MiB/s, 44 objects/s recovering 2023-10-01T10:28:58.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:58 smithi173 bash[58816]: cluster 2023-10-01T10:28:57.398276+0000 mgr.y (mgr.25072) 261 : cluster 0 pgmap v200: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 40 MiB/s wr, 640 op/s; 13443/288735 objects degraded (4.656%); 2.8 MiB/s, 44 objects/s recovering 2023-10-01T10:28:59.290 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 85 16 100154 100138 73.6235 41.75 0.0162788 0.013576 2023-10-01T10:28:59.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:28:59 smithi078 bash[78274]: audit 2023-10-01T10:28:58.914778+0000 mon.a (mon.0) 369 : audit 0 from='client.? 172.21.15.78:0/459337152' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:59.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:28:59 smithi078 bash[75299]: audit 2023-10-01T10:28:58.914778+0000 mon.a (mon.0) 369 : audit 0 from='client.? 172.21.15.78:0/459337152' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:28:59.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:28:59 smithi173 bash[58816]: audit 2023-10-01T10:28:58.914778+0000 mon.a (mon.0) 369 : audit 0 from='client.? 172.21.15.78:0/459337152' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:00.290 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 86 16 100866 100850 73.2848 44.5 0.0111929 0.0136389 2023-10-01T10:29:00.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:00 smithi078 bash[78274]: cluster 2023-10-01T10:28:59.399128+0000 mgr.y (mgr.25072) 262 : cluster 0 pgmap v201: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 58 MiB/s wr, 935 op/s; 13152/299367 objects degraded (4.393%); 4.3 MiB/s, 68 objects/s recovering 2023-10-01T10:29:00.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:00 smithi078 bash[75299]: cluster 2023-10-01T10:28:59.399128+0000 mgr.y (mgr.25072) 262 : cluster 0 pgmap v201: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 58 MiB/s wr, 935 op/s; 13152/299367 objects degraded (4.393%); 4.3 MiB/s, 68 objects/s recovering 2023-10-01T10:29:00.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:29:00 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:29:00] "GET /metrics HTTP/1.1" 200 36214 "" "Prometheus/2.33.4" 2023-10-01T10:29:00.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:00 smithi173 bash[58816]: cluster 2023-10-01T10:28:59.399128+0000 mgr.y (mgr.25072) 262 : cluster 0 pgmap v201: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 58 MiB/s wr, 935 op/s; 13152/299367 objects degraded (4.393%); 4.3 MiB/s, 68 objects/s recovering 2023-10-01T10:29:01.291 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 87 16 101573 101557 72.9503 44.1875 0.0202451 0.0137021 2023-10-01T10:29:01.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:01 smithi078 bash[75299]: audit 2023-10-01T10:29:01.484284+0000 mon.c (mon.2) 20 : audit 0 from='client.? 172.21.15.78:0/2570196289' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:01.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:01 smithi078 bash[78274]: audit 2023-10-01T10:29:01.484284+0000 mon.c (mon.2) 20 : audit 0 from='client.? 172.21.15.78:0/2570196289' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:01.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:01 smithi173 bash[58816]: audit 2023-10-01T10:29:01.484284+0000 mon.c (mon.2) 20 : audit 0 from='client.? 172.21.15.78:0/2570196289' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:02.291 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 88 16 102003 101987 72.4266 26.875 0.00656288 0.0137256 2023-10-01T10:29:02.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:02 smithi078 bash[75299]: cluster 2023-10-01T10:29:01.399582+0000 mgr.y (mgr.25072) 263 : cluster 0 pgmap v202: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 38 MiB/s wr, 615 op/s; 13152/299367 objects degraded (4.393%); 2.9 MiB/s, 45 objects/s recovering 2023-10-01T10:29:02.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:02 smithi078 bash[75299]: cluster 2023-10-01T10:29:01.525035+0000 osd.7 (osd.7) 19 : cluster 3 failed to encode map e89 with expected crc 2023-10-01T10:29:02.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:02 smithi078 bash[75299]: cluster 2023-10-01T10:29:01.525990+0000 osd.5 (osd.5) 10 : cluster 3 failed to encode map e89 with expected crc 2023-10-01T10:29:02.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:02 smithi078 bash[75299]: cluster 2023-10-01T10:29:01.529912+0000 mon.a (mon.0) 370 : cluster 0 osdmap e89: 8 total, 8 up, 8 in 2023-10-01T10:29:02.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:02 smithi078 bash[75299]: cluster 2023-10-01T10:29:01.534812+0000 osd.1 (osd.1) 14 : cluster 3 failed to encode map e89 with expected crc 2023-10-01T10:29:02.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:02 smithi078 bash[75299]: cluster 2023-10-01T10:29:01.770023+0000 mon.a (mon.0) 371 : cluster 3 Health check update: Degraded data redundancy: 13152/299367 objects degraded (4.393%), 8 pgs degraded (PG_DEGRADED) 2023-10-01T10:29:02.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:02 smithi078 bash[75299]: audit 2023-10-01T10:29:02.389207+0000 mon.a (mon.0) 372 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:29:02.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:02 smithi078 bash[78274]: cluster 2023-10-01T10:29:01.399582+0000 mgr.y (mgr.25072) 263 : cluster 0 pgmap v202: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 38 MiB/s wr, 615 op/s; 13152/299367 objects degraded (4.393%); 2.9 MiB/s, 45 objects/s recovering 2023-10-01T10:29:02.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:02 smithi078 bash[78274]: cluster 2023-10-01T10:29:01.525035+0000 osd.7 (osd.7) 19 : cluster 3 failed to encode map e89 with expected crc 2023-10-01T10:29:02.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:02 smithi078 bash[78274]: cluster 2023-10-01T10:29:01.525990+0000 osd.5 (osd.5) 10 : cluster 3 failed to encode map e89 with expected crc 2023-10-01T10:29:02.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:02 smithi078 bash[78274]: cluster 2023-10-01T10:29:01.529912+0000 mon.a (mon.0) 370 : cluster 0 osdmap e89: 8 total, 8 up, 8 in 2023-10-01T10:29:02.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:02 smithi078 bash[78274]: cluster 2023-10-01T10:29:01.534812+0000 osd.1 (osd.1) 14 : cluster 3 failed to encode map e89 with expected crc 2023-10-01T10:29:02.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:02 smithi078 bash[78274]: cluster 2023-10-01T10:29:01.770023+0000 mon.a (mon.0) 371 : cluster 3 Health check update: Degraded data redundancy: 13152/299367 objects degraded (4.393%), 8 pgs degraded (PG_DEGRADED) 2023-10-01T10:29:02.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:02 smithi078 bash[78274]: audit 2023-10-01T10:29:02.389207+0000 mon.a (mon.0) 372 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:29:02.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:02 smithi173 bash[58816]: cluster 2023-10-01T10:29:01.399582+0000 mgr.y (mgr.25072) 263 : cluster 0 pgmap v202: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 81 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 38 MiB/s wr, 615 op/s; 13152/299367 objects degraded (4.393%); 2.9 MiB/s, 45 objects/s recovering 2023-10-01T10:29:02.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:02 smithi173 bash[58816]: cluster 2023-10-01T10:29:01.525035+0000 osd.7 (osd.7) 19 : cluster 3 failed to encode map e89 with expected crc 2023-10-01T10:29:02.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:02 smithi173 bash[58816]: cluster 2023-10-01T10:29:01.525990+0000 osd.5 (osd.5) 10 : cluster 3 failed to encode map e89 with expected crc 2023-10-01T10:29:02.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:02 smithi173 bash[58816]: cluster 2023-10-01T10:29:01.529912+0000 mon.a (mon.0) 370 : cluster 0 osdmap e89: 8 total, 8 up, 8 in 2023-10-01T10:29:02.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:02 smithi173 bash[58816]: cluster 2023-10-01T10:29:01.534812+0000 osd.1 (osd.1) 14 : cluster 3 failed to encode map e89 with expected crc 2023-10-01T10:29:02.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:02 smithi173 bash[58816]: cluster 2023-10-01T10:29:01.770023+0000 mon.a (mon.0) 371 : cluster 3 Health check update: Degraded data redundancy: 13152/299367 objects degraded (4.393%), 8 pgs degraded (PG_DEGRADED) 2023-10-01T10:29:02.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:02 smithi173 bash[58816]: audit 2023-10-01T10:29:02.389207+0000 mon.a (mon.0) 372 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:29:03.291 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 89 16 102517 102501 71.9738 32.125 0.0137818 0.0138877 2023-10-01T10:29:03.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:03 smithi078 bash[75299]: cluster 2023-10-01T10:29:02.526905+0000 osd.6 (osd.6) 10 : cluster 3 failed to encode map e90 with expected crc 2023-10-01T10:29:03.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:03 smithi078 bash[75299]: cluster 2023-10-01T10:29:02.527090+0000 osd.7 (osd.7) 20 : cluster 3 failed to encode map e90 with expected crc 2023-10-01T10:29:03.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:03 smithi078 bash[75299]: cluster 2023-10-01T10:29:02.527592+0000 mon.a (mon.0) 373 : cluster 0 osdmap e90: 8 total, 8 up, 8 in 2023-10-01T10:29:03.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:03 smithi078 bash[75299]: cluster 2023-10-01T10:29:02.530903+0000 osd.1 (osd.1) 15 : cluster 3 failed to encode map e90 with expected crc 2023-10-01T10:29:03.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:03 smithi078 bash[75299]: cluster 2023-10-01T10:29:02.533642+0000 osd.4 (osd.4) 13 : cluster 3 failed to encode map e90 with expected crc 2023-10-01T10:29:03.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:03 smithi078 bash[75299]: cluster 2023-10-01T10:29:02.558176+0000 osd.3 (osd.3) 13 : cluster 3 failed to encode map e90 with expected crc 2023-10-01T10:29:03.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:03 smithi078 bash[75299]: cluster 2023-10-01T10:29:02.558218+0000 osd.2 (osd.2) 13 : cluster 3 failed to encode map e90 with expected crc 2023-10-01T10:29:03.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:03 smithi078 bash[75299]: cluster 2023-10-01T10:29:02.581887+0000 osd.5 (osd.5) 11 : cluster 3 failed to encode map e90 with expected crc 2023-10-01T10:29:03.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:03 smithi078 bash[75299]: cluster 2023-10-01T10:29:03.400209+0000 mgr.y (mgr.25072) 264 : cluster 0 pgmap v205: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 29 MiB/s wr, 465 op/s; 13037/299916 objects degraded (4.347%); 3.2 MiB/s, 50 objects/s recovering 2023-10-01T10:29:03.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:03 smithi078 bash[78274]: cluster 2023-10-01T10:29:02.526905+0000 osd.6 (osd.6) 10 : cluster 3 failed to encode map e90 with expected crc 2023-10-01T10:29:03.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:03 smithi078 bash[78274]: cluster 2023-10-01T10:29:02.527090+0000 osd.7 (osd.7) 20 : cluster 3 failed to encode map e90 with expected crc 2023-10-01T10:29:03.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:03 smithi078 bash[78274]: cluster 2023-10-01T10:29:02.527592+0000 mon.a (mon.0) 373 : cluster 0 osdmap e90: 8 total, 8 up, 8 in 2023-10-01T10:29:03.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:03 smithi078 bash[78274]: cluster 2023-10-01T10:29:02.530903+0000 osd.1 (osd.1) 15 : cluster 3 failed to encode map e90 with expected crc 2023-10-01T10:29:03.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:03 smithi078 bash[78274]: cluster 2023-10-01T10:29:02.533642+0000 osd.4 (osd.4) 13 : cluster 3 failed to encode map e90 with expected crc 2023-10-01T10:29:03.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:03 smithi078 bash[78274]: cluster 2023-10-01T10:29:02.558176+0000 osd.3 (osd.3) 13 : cluster 3 failed to encode map e90 with expected crc 2023-10-01T10:29:03.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:03 smithi078 bash[78274]: cluster 2023-10-01T10:29:02.558218+0000 osd.2 (osd.2) 13 : cluster 3 failed to encode map e90 with expected crc 2023-10-01T10:29:03.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:03 smithi078 bash[78274]: cluster 2023-10-01T10:29:02.581887+0000 osd.5 (osd.5) 11 : cluster 3 failed to encode map e90 with expected crc 2023-10-01T10:29:03.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:03 smithi078 bash[78274]: cluster 2023-10-01T10:29:03.400209+0000 mgr.y (mgr.25072) 264 : cluster 0 pgmap v205: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 29 MiB/s wr, 465 op/s; 13037/299916 objects degraded (4.347%); 3.2 MiB/s, 50 objects/s recovering 2023-10-01T10:29:03.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:03 smithi173 bash[58816]: cluster 2023-10-01T10:29:02.526905+0000 osd.6 (osd.6) 10 : cluster 3 failed to encode map e90 with expected crc 2023-10-01T10:29:03.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:03 smithi173 bash[58816]: cluster 2023-10-01T10:29:02.527090+0000 osd.7 (osd.7) 20 : cluster 3 failed to encode map e90 with expected crc 2023-10-01T10:29:03.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:03 smithi173 bash[58816]: cluster 2023-10-01T10:29:02.527592+0000 mon.a (mon.0) 373 : cluster 0 osdmap e90: 8 total, 8 up, 8 in 2023-10-01T10:29:03.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:03 smithi173 bash[58816]: cluster 2023-10-01T10:29:02.530903+0000 osd.1 (osd.1) 15 : cluster 3 failed to encode map e90 with expected crc 2023-10-01T10:29:03.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:03 smithi173 bash[58816]: cluster 2023-10-01T10:29:02.533642+0000 osd.4 (osd.4) 13 : cluster 3 failed to encode map e90 with expected crc 2023-10-01T10:29:03.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:03 smithi173 bash[58816]: cluster 2023-10-01T10:29:02.558176+0000 osd.3 (osd.3) 13 : cluster 3 failed to encode map e90 with expected crc 2023-10-01T10:29:03.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:03 smithi173 bash[58816]: cluster 2023-10-01T10:29:02.558218+0000 osd.2 (osd.2) 13 : cluster 3 failed to encode map e90 with expected crc 2023-10-01T10:29:03.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:03 smithi173 bash[58816]: cluster 2023-10-01T10:29:02.581887+0000 osd.5 (osd.5) 11 : cluster 3 failed to encode map e90 with expected crc 2023-10-01T10:29:03.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:03 smithi173 bash[58816]: cluster 2023-10-01T10:29:03.400209+0000 mgr.y (mgr.25072) 264 : cluster 0 pgmap v205: 89 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 29 MiB/s wr, 465 op/s; 13037/299916 objects degraded (4.347%); 3.2 MiB/s, 50 objects/s recovering 2023-10-01T10:29:04.291 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 90 12 103155 103143 71.6198 40.125 0.00934456 0.0139552 2023-10-01T10:29:04.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:04 smithi078 bash[75299]: audit 2023-10-01T10:29:04.160271+0000 mon.b (mon.1) 9 : audit 0 from='client.? 172.21.15.78:0/127081838' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:04.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:04 smithi078 bash[78274]: audit 2023-10-01T10:29:04.160271+0000 mon.b (mon.1) 9 : audit 0 from='client.? 172.21.15.78:0/127081838' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:04.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:04 smithi173 bash[58816]: audit 2023-10-01T10:29:04.160271+0000 mon.b (mon.1) 9 : audit 0 from='client.? 172.21.15.78:0/127081838' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:05.291 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Total time run: 90.0218 2023-10-01T10:29:05.292 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Total writes made: 103155 2023-10-01T10:29:05.292 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Write size: 65536 2023-10-01T10:29:05.292 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Object size: 65536 2023-10-01T10:29:05.292 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Bandwidth (MB/sec): 71.6181 2023-10-01T10:29:05.292 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Stddev Bandwidth: 36.4571 2023-10-01T10:29:05.292 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Max bandwidth (MB/sec): 187.812 2023-10-01T10:29:05.292 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Min bandwidth (MB/sec): 0 2023-10-01T10:29:05.292 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Average IOPS: 1145 2023-10-01T10:29:05.292 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Stddev IOPS: 583.327 2023-10-01T10:29:05.292 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Max IOPS: 3005 2023-10-01T10:29:05.292 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Min IOPS: 0 2023-10-01T10:29:05.293 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Average Latency(s): 0.0139571 2023-10-01T10:29:05.293 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Stddev Latency(s): 0.041326 2023-10-01T10:29:05.293 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Max latency(s): 2.99842 2023-10-01T10:29:05.293 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Min latency(s): 0.00149611 2023-10-01T10:29:05.302 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Cleaning up (deleting benchmark objects) 2023-10-01T10:29:05.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:05 smithi078 bash[75299]: cluster 2023-10-01T10:29:05.401095+0000 mgr.y (mgr.25072) 265 : cluster 0 pgmap v206: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 824 op/s; 12980/308541 objects degraded (4.207%); 3.6 MiB/s, 57 objects/s recovering 2023-10-01T10:29:05.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:05 smithi078 bash[78274]: cluster 2023-10-01T10:29:05.401095+0000 mgr.y (mgr.25072) 265 : cluster 0 pgmap v206: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 824 op/s; 12980/308541 objects degraded (4.207%); 3.6 MiB/s, 57 objects/s recovering 2023-10-01T10:29:05.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:05 smithi173 bash[58816]: cluster 2023-10-01T10:29:05.401095+0000 mgr.y (mgr.25072) 265 : cluster 0 pgmap v206: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 824 op/s; 12980/308541 objects degraded (4.207%); 3.6 MiB/s, 57 objects/s recovering 2023-10-01T10:29:07.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:06 smithi078 bash[75299]: audit 2023-10-01T10:29:06.755200+0000 mon.a (mon.0) 374 : audit 0 from='client.? 172.21.15.78:0/1250655460' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:07.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:06 smithi078 bash[75299]: cluster 2023-10-01T10:29:06.771978+0000 mon.a (mon.0) 375 : cluster 3 Health check update: Degraded data redundancy: 12980/308541 objects degraded (4.207%), 7 pgs degraded (PG_DEGRADED) 2023-10-01T10:29:07.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:06 smithi078 bash[78274]: audit 2023-10-01T10:29:06.755200+0000 mon.a (mon.0) 374 : audit 0 from='client.? 172.21.15.78:0/1250655460' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:07.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:06 smithi078 bash[78274]: cluster 2023-10-01T10:29:06.771978+0000 mon.a (mon.0) 375 : cluster 3 Health check update: Degraded data redundancy: 12980/308541 objects degraded (4.207%), 7 pgs degraded (PG_DEGRADED) 2023-10-01T10:29:07.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:06 smithi173 bash[58816]: audit 2023-10-01T10:29:06.755200+0000 mon.a (mon.0) 374 : audit 0 from='client.? 172.21.15.78:0/1250655460' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:07.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:06 smithi173 bash[58816]: cluster 2023-10-01T10:29:06.771978+0000 mon.a (mon.0) 375 : cluster 3 Health check update: Degraded data redundancy: 12980/308541 objects degraded (4.207%), 7 pgs degraded (PG_DEGRADED) 2023-10-01T10:29:08.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:07 smithi078 bash[75299]: cluster 2023-10-01T10:29:07.401619+0000 mgr.y (mgr.25072) 266 : cluster 0 pgmap v207: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 24 MiB/s wr, 382 op/s; 12980/308541 objects degraded (4.207%); 1.3 MiB/s, 21 objects/s recovering 2023-10-01T10:29:08.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:07 smithi078 bash[78274]: cluster 2023-10-01T10:29:07.401619+0000 mgr.y (mgr.25072) 266 : cluster 0 pgmap v207: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 24 MiB/s wr, 382 op/s; 12980/308541 objects degraded (4.207%); 1.3 MiB/s, 21 objects/s recovering 2023-10-01T10:29:08.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:07 smithi173 bash[58816]: cluster 2023-10-01T10:29:07.401619+0000 mgr.y (mgr.25072) 266 : cluster 0 pgmap v207: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 24 MiB/s wr, 382 op/s; 12980/308541 objects degraded (4.207%); 1.3 MiB/s, 21 objects/s recovering 2023-10-01T10:29:09.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:09 smithi078 bash[75299]: audit 2023-10-01T10:29:09.145771+0000 mon.a (mon.0) 376 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:29:09.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:29:09 smithi078 bash[71214]: debug 2023-10-01T10:29:09.143+0000 7fd9c245e700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:29:09.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:09 smithi078 bash[78274]: audit 2023-10-01T10:29:09.145771+0000 mon.a (mon.0) 376 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:29:09.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:09 smithi173 bash[58816]: audit 2023-10-01T10:29:09.145771+0000 mon.a (mon.0) 376 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:29:10.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:10 smithi078 bash[75299]: audit 2023-10-01T10:29:09.146332+0000 mgr.y (mgr.25072) 267 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:29:10.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:10 smithi078 bash[75299]: cephadm 2023-10-01T10:29:09.148067+0000 mgr.y (mgr.25072) 268 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:29:10.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:10 smithi078 bash[75299]: audit 2023-10-01T10:29:09.303296+0000 mon.c (mon.2) 21 : audit 0 from='client.? 172.21.15.78:0/254258377' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:10.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:10 smithi078 bash[75299]: cluster 2023-10-01T10:29:09.402563+0000 mgr.y (mgr.25072) 269 : cluster 0 pgmap v208: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 127 B/s rd, 26 MiB/s wr, 752 op/s; 12905/301650 objects degraded (4.278%); 1.9 MiB/s, 30 objects/s recovering 2023-10-01T10:29:10.584 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:29:10 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:29:10] "GET /metrics HTTP/1.1" 200 36213 "" "Prometheus/2.33.4" 2023-10-01T10:29:10.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:10 smithi078 bash[78274]: audit 2023-10-01T10:29:09.146332+0000 mgr.y (mgr.25072) 267 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:29:10.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:10 smithi078 bash[78274]: cephadm 2023-10-01T10:29:09.148067+0000 mgr.y (mgr.25072) 268 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:29:10.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:10 smithi078 bash[78274]: audit 2023-10-01T10:29:09.303296+0000 mon.c (mon.2) 21 : audit 0 from='client.? 172.21.15.78:0/254258377' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:10.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:10 smithi078 bash[78274]: cluster 2023-10-01T10:29:09.402563+0000 mgr.y (mgr.25072) 269 : cluster 0 pgmap v208: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 127 B/s rd, 26 MiB/s wr, 752 op/s; 12905/301650 objects degraded (4.278%); 1.9 MiB/s, 30 objects/s recovering 2023-10-01T10:29:10.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:10 smithi173 bash[58816]: audit 2023-10-01T10:29:09.146332+0000 mgr.y (mgr.25072) 267 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:29:10.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:10 smithi173 bash[58816]: cephadm 2023-10-01T10:29:09.148067+0000 mgr.y (mgr.25072) 268 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:29:10.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:10 smithi173 bash[58816]: audit 2023-10-01T10:29:09.303296+0000 mon.c (mon.2) 21 : audit 0 from='client.? 172.21.15.78:0/254258377' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:10.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:10 smithi173 bash[58816]: cluster 2023-10-01T10:29:09.402563+0000 mgr.y (mgr.25072) 269 : cluster 0 pgmap v208: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 127 B/s rd, 26 MiB/s wr, 752 op/s; 12905/301650 objects degraded (4.278%); 1.9 MiB/s, 30 objects/s recovering 2023-10-01T10:29:12.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:12 smithi078 bash[78274]: cluster 2023-10-01T10:29:11.403087+0000 mgr.y (mgr.25072) 270 : cluster 0 pgmap v209: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 103 B/s rd, 21 MiB/s wr, 609 op/s; 12905/301650 objects degraded (4.278%); 1.6 MiB/s, 25 objects/s recovering 2023-10-01T10:29:12.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:12 smithi078 bash[78274]: cluster 2023-10-01T10:29:11.773179+0000 mon.a (mon.0) 377 : cluster 3 Health check update: Degraded data redundancy: 12905/301650 objects degraded (4.278%), 7 pgs degraded (PG_DEGRADED) 2023-10-01T10:29:12.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:12 smithi078 bash[78274]: audit 2023-10-01T10:29:11.884654+0000 mon.c (mon.2) 22 : audit 0 from='client.? 172.21.15.78:0/346590635' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:12.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:12 smithi078 bash[75299]: cluster 2023-10-01T10:29:11.403087+0000 mgr.y (mgr.25072) 270 : cluster 0 pgmap v209: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 103 B/s rd, 21 MiB/s wr, 609 op/s; 12905/301650 objects degraded (4.278%); 1.6 MiB/s, 25 objects/s recovering 2023-10-01T10:29:12.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:12 smithi078 bash[75299]: cluster 2023-10-01T10:29:11.773179+0000 mon.a (mon.0) 377 : cluster 3 Health check update: Degraded data redundancy: 12905/301650 objects degraded (4.278%), 7 pgs degraded (PG_DEGRADED) 2023-10-01T10:29:12.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:12 smithi078 bash[75299]: audit 2023-10-01T10:29:11.884654+0000 mon.c (mon.2) 22 : audit 0 from='client.? 172.21.15.78:0/346590635' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:12.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:12 smithi173 bash[58816]: cluster 2023-10-01T10:29:11.403087+0000 mgr.y (mgr.25072) 270 : cluster 0 pgmap v209: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 103 B/s rd, 21 MiB/s wr, 609 op/s; 12905/301650 objects degraded (4.278%); 1.6 MiB/s, 25 objects/s recovering 2023-10-01T10:29:12.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:12 smithi173 bash[58816]: cluster 2023-10-01T10:29:11.773179+0000 mon.a (mon.0) 377 : cluster 3 Health check update: Degraded data redundancy: 12905/301650 objects degraded (4.278%), 7 pgs degraded (PG_DEGRADED) 2023-10-01T10:29:12.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:12 smithi173 bash[58816]: audit 2023-10-01T10:29:11.884654+0000 mon.c (mon.2) 22 : audit 0 from='client.? 172.21.15.78:0/346590635' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:14.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:14 smithi078 bash[75299]: cluster 2023-10-01T10:29:13.403626+0000 mgr.y (mgr.25072) 271 : cluster 0 pgmap v210: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 94 B/s rd, 19 MiB/s wr, 573 op/s; 12905/300999 objects degraded (4.287%); 1.4 MiB/s, 22 objects/s recovering 2023-10-01T10:29:14.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:14 smithi078 bash[78274]: cluster 2023-10-01T10:29:13.403626+0000 mgr.y (mgr.25072) 271 : cluster 0 pgmap v210: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 94 B/s rd, 19 MiB/s wr, 573 op/s; 12905/300999 objects degraded (4.287%); 1.4 MiB/s, 22 objects/s recovering 2023-10-01T10:29:14.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:14 smithi173 bash[58816]: cluster 2023-10-01T10:29:13.403626+0000 mgr.y (mgr.25072) 271 : cluster 0 pgmap v210: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 94 B/s rd, 19 MiB/s wr, 573 op/s; 12905/300999 objects degraded (4.287%); 1.4 MiB/s, 22 objects/s recovering 2023-10-01T10:29:15.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:15 smithi078 bash[75299]: audit 2023-10-01T10:29:14.496913+0000 mon.a (mon.0) 378 : audit 0 from='client.? 172.21.15.78:0/427266892' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:15.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:15 smithi078 bash[78274]: audit 2023-10-01T10:29:14.496913+0000 mon.a (mon.0) 378 : audit 0 from='client.? 172.21.15.78:0/427266892' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:15.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:15 smithi173 bash[58816]: audit 2023-10-01T10:29:14.496913+0000 mon.a (mon.0) 378 : audit 0 from='client.? 172.21.15.78:0/427266892' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:16.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:16 smithi078 bash[75299]: cluster 2023-10-01T10:29:15.406104+0000 mgr.y (mgr.25072) 272 : cluster 0 pgmap v211: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 85 B/s rd, 17 MiB/s wr, 802 op/s; 12836/290262 objects degraded (4.422%); 1.0 MiB/s, 16 objects/s recovering 2023-10-01T10:29:16.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:16 smithi078 bash[78274]: cluster 2023-10-01T10:29:15.406104+0000 mgr.y (mgr.25072) 272 : cluster 0 pgmap v211: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 85 B/s rd, 17 MiB/s wr, 802 op/s; 12836/290262 objects degraded (4.422%); 1.0 MiB/s, 16 objects/s recovering 2023-10-01T10:29:16.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:16 smithi173 bash[58816]: cluster 2023-10-01T10:29:15.406104+0000 mgr.y (mgr.25072) 272 : cluster 0 pgmap v211: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 85 B/s rd, 17 MiB/s wr, 802 op/s; 12836/290262 objects degraded (4.422%); 1.0 MiB/s, 16 objects/s recovering 2023-10-01T10:29:17.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:17 smithi078 bash[75299]: cluster 2023-10-01T10:29:16.781664+0000 mon.a (mon.0) 379 : cluster 3 Health check update: Degraded data redundancy: 12836/290262 objects degraded (4.422%), 7 pgs degraded (PG_DEGRADED) 2023-10-01T10:29:17.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:17 smithi078 bash[75299]: audit 2023-10-01T10:29:17.100597+0000 mon.a (mon.0) 380 : audit 0 from='client.? 172.21.15.78:0/1309249647' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:17.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:17 smithi078 bash[75299]: audit 2023-10-01T10:29:17.388746+0000 mon.a (mon.0) 381 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:29:17.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:17 smithi078 bash[78274]: cluster 2023-10-01T10:29:16.781664+0000 mon.a (mon.0) 379 : cluster 3 Health check update: Degraded data redundancy: 12836/290262 objects degraded (4.422%), 7 pgs degraded (PG_DEGRADED) 2023-10-01T10:29:17.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:17 smithi078 bash[78274]: audit 2023-10-01T10:29:17.100597+0000 mon.a (mon.0) 380 : audit 0 from='client.? 172.21.15.78:0/1309249647' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:17.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:17 smithi078 bash[78274]: audit 2023-10-01T10:29:17.388746+0000 mon.a (mon.0) 381 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:29:17.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:17 smithi173 bash[58816]: cluster 2023-10-01T10:29:16.781664+0000 mon.a (mon.0) 379 : cluster 3 Health check update: Degraded data redundancy: 12836/290262 objects degraded (4.422%), 7 pgs degraded (PG_DEGRADED) 2023-10-01T10:29:17.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:17 smithi173 bash[58816]: audit 2023-10-01T10:29:17.100597+0000 mon.a (mon.0) 380 : audit 0 from='client.? 172.21.15.78:0/1309249647' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:17.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:17 smithi173 bash[58816]: audit 2023-10-01T10:29:17.388746+0000 mon.a (mon.0) 381 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:29:18.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:18 smithi078 bash[75299]: cluster 2023-10-01T10:29:17.412443+0000 mgr.y (mgr.25072) 273 : cluster 0 pgmap v212: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 85 B/s rd, 1.7 MiB/s wr, 562 op/s; 12836/290262 objects degraded (4.422%); 767 KiB/s, 11 objects/s recovering 2023-10-01T10:29:18.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:18 smithi078 bash[78274]: cluster 2023-10-01T10:29:17.412443+0000 mgr.y (mgr.25072) 273 : cluster 0 pgmap v212: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 85 B/s rd, 1.7 MiB/s wr, 562 op/s; 12836/290262 objects degraded (4.422%); 767 KiB/s, 11 objects/s recovering 2023-10-01T10:29:18.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:18 smithi173 bash[58816]: cluster 2023-10-01T10:29:17.412443+0000 mgr.y (mgr.25072) 273 : cluster 0 pgmap v212: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 85 B/s rd, 1.7 MiB/s wr, 562 op/s; 12836/290262 objects degraded (4.422%); 767 KiB/s, 11 objects/s recovering 2023-10-01T10:29:20.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:20 smithi078 bash[75299]: cluster 2023-10-01T10:29:19.413295+0000 mgr.y (mgr.25072) 274 : cluster 0 pgmap v213: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 85 B/s rd, 1.7 MiB/s wr, 869 op/s; 12770/279207 objects degraded (4.574%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T10:29:20.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:20 smithi078 bash[75299]: audit 2023-10-01T10:29:19.721105+0000 mon.a (mon.0) 382 : audit 0 from='client.? 172.21.15.78:0/2533748998' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:20.584 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:29:20 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:29:20] "GET /metrics HTTP/1.1" 200 36216 "" "Prometheus/2.33.4" 2023-10-01T10:29:20.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:20 smithi078 bash[78274]: cluster 2023-10-01T10:29:19.413295+0000 mgr.y (mgr.25072) 274 : cluster 0 pgmap v213: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 85 B/s rd, 1.7 MiB/s wr, 869 op/s; 12770/279207 objects degraded (4.574%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T10:29:20.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:20 smithi078 bash[78274]: audit 2023-10-01T10:29:19.721105+0000 mon.a (mon.0) 382 : audit 0 from='client.? 172.21.15.78:0/2533748998' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:20.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:20 smithi173 bash[58816]: cluster 2023-10-01T10:29:19.413295+0000 mgr.y (mgr.25072) 274 : cluster 0 pgmap v213: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 85 B/s rd, 1.7 MiB/s wr, 869 op/s; 12770/279207 objects degraded (4.574%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T10:29:20.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:20 smithi173 bash[58816]: audit 2023-10-01T10:29:19.721105+0000 mon.a (mon.0) 382 : audit 0 from='client.? 172.21.15.78:0/2533748998' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:22.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:22 smithi078 bash[75299]: cluster 2023-10-01T10:29:21.413859+0000 mgr.y (mgr.25072) 275 : cluster 0 pgmap v214: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 12770/279207 objects degraded (4.574%); 719 KiB/s, 11 objects/s recovering 2023-10-01T10:29:22.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:22 smithi078 bash[75299]: cluster 2023-10-01T10:29:21.782848+0000 mon.a (mon.0) 383 : cluster 3 Health check update: Degraded data redundancy: 12770/279207 objects degraded (4.574%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-01T10:29:22.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:22 smithi078 bash[75299]: audit 2023-10-01T10:29:22.313607+0000 mon.a (mon.0) 384 : audit 0 from='client.? 172.21.15.78:0/1688873621' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:22.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:22 smithi078 bash[78274]: cluster 2023-10-01T10:29:21.413859+0000 mgr.y (mgr.25072) 275 : cluster 0 pgmap v214: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 12770/279207 objects degraded (4.574%); 719 KiB/s, 11 objects/s recovering 2023-10-01T10:29:22.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:22 smithi078 bash[78274]: cluster 2023-10-01T10:29:21.782848+0000 mon.a (mon.0) 383 : cluster 3 Health check update: Degraded data redundancy: 12770/279207 objects degraded (4.574%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-01T10:29:22.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:22 smithi078 bash[78274]: audit 2023-10-01T10:29:22.313607+0000 mon.a (mon.0) 384 : audit 0 from='client.? 172.21.15.78:0/1688873621' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:22.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:22 smithi173 bash[58816]: cluster 2023-10-01T10:29:21.413859+0000 mgr.y (mgr.25072) 275 : cluster 0 pgmap v214: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 12770/279207 objects degraded (4.574%); 719 KiB/s, 11 objects/s recovering 2023-10-01T10:29:22.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:22 smithi173 bash[58816]: cluster 2023-10-01T10:29:21.782848+0000 mon.a (mon.0) 383 : cluster 3 Health check update: Degraded data redundancy: 12770/279207 objects degraded (4.574%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-01T10:29:22.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:22 smithi173 bash[58816]: audit 2023-10-01T10:29:22.313607+0000 mon.a (mon.0) 384 : audit 0 from='client.? 172.21.15.78:0/1688873621' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:24.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:24 smithi078 bash[75299]: cluster 2023-10-01T10:29:23.414442+0000 mgr.y (mgr.25072) 276 : cluster 0 pgmap v215: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 641 op/s; 12770/278517 objects degraded (4.585%); 719 KiB/s, 11 objects/s recovering 2023-10-01T10:29:24.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:24 smithi078 bash[75299]: audit 2023-10-01T10:29:24.162241+0000 mon.a (mon.0) 385 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:29:24.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:24 smithi078 bash[78274]: cluster 2023-10-01T10:29:23.414442+0000 mgr.y (mgr.25072) 276 : cluster 0 pgmap v215: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 641 op/s; 12770/278517 objects degraded (4.585%); 719 KiB/s, 11 objects/s recovering 2023-10-01T10:29:24.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:24 smithi078 bash[78274]: audit 2023-10-01T10:29:24.162241+0000 mon.a (mon.0) 385 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:29:24.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:24 smithi173 bash[58816]: cluster 2023-10-01T10:29:23.414442+0000 mgr.y (mgr.25072) 276 : cluster 0 pgmap v215: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 641 op/s; 12770/278517 objects degraded (4.585%); 719 KiB/s, 11 objects/s recovering 2023-10-01T10:29:24.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:24 smithi173 bash[58816]: audit 2023-10-01T10:29:24.162241+0000 mon.a (mon.0) 385 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:29:25.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:25 smithi078 bash[75299]: audit 2023-10-01T10:29:24.941069+0000 mon.a (mon.0) 386 : audit 0 from='client.? 172.21.15.78:0/3539771767' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:25.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:25 smithi078 bash[78274]: audit 2023-10-01T10:29:24.941069+0000 mon.a (mon.0) 386 : audit 0 from='client.? 172.21.15.78:0/3539771767' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:25.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:25 smithi173 bash[58816]: audit 2023-10-01T10:29:24.941069+0000 mon.a (mon.0) 386 : audit 0 from='client.? 172.21.15.78:0/3539771767' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:26.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:26 smithi078 bash[78274]: cluster 2023-10-01T10:29:25.417979+0000 mgr.y (mgr.25072) 277 : cluster 0 pgmap v216: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 902 op/s; 12703/268482 objects degraded (4.731%); 1.1 MiB/s, 16 objects/s recovering 2023-10-01T10:29:26.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:26 smithi078 bash[75299]: cluster 2023-10-01T10:29:25.417979+0000 mgr.y (mgr.25072) 277 : cluster 0 pgmap v216: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 902 op/s; 12703/268482 objects degraded (4.731%); 1.1 MiB/s, 16 objects/s recovering 2023-10-01T10:29:26.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:26 smithi173 bash[58816]: cluster 2023-10-01T10:29:25.417979+0000 mgr.y (mgr.25072) 277 : cluster 0 pgmap v216: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 902 op/s; 12703/268482 objects degraded (4.731%); 1.1 MiB/s, 16 objects/s recovering 2023-10-01T10:29:27.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:27 smithi078 bash[75299]: cluster 2023-10-01T10:29:26.783949+0000 mon.a (mon.0) 387 : cluster 3 Health check update: Degraded data redundancy: 12703/268482 objects degraded (4.731%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-01T10:29:27.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:27 smithi078 bash[78274]: cluster 2023-10-01T10:29:26.783949+0000 mon.a (mon.0) 387 : cluster 3 Health check update: Degraded data redundancy: 12703/268482 objects degraded (4.731%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-01T10:29:27.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:27 smithi173 bash[58816]: cluster 2023-10-01T10:29:26.783949+0000 mon.a (mon.0) 387 : cluster 3 Health check update: Degraded data redundancy: 12703/268482 objects degraded (4.731%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-01T10:29:28.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:28 smithi078 bash[78274]: cluster 2023-10-01T10:29:27.418522+0000 mgr.y (mgr.25072) 278 : cluster 0 pgmap v217: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 12703/268482 objects degraded (4.731%); 709 KiB/s, 11 objects/s recovering 2023-10-01T10:29:28.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:28 smithi078 bash[78274]: audit 2023-10-01T10:29:27.565907+0000 mon.a (mon.0) 388 : audit 0 from='client.? 172.21.15.78:0/4178181840' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:28.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:28 smithi078 bash[75299]: cluster 2023-10-01T10:29:27.418522+0000 mgr.y (mgr.25072) 278 : cluster 0 pgmap v217: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 12703/268482 objects degraded (4.731%); 709 KiB/s, 11 objects/s recovering 2023-10-01T10:29:28.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:28 smithi078 bash[75299]: audit 2023-10-01T10:29:27.565907+0000 mon.a (mon.0) 388 : audit 0 from='client.? 172.21.15.78:0/4178181840' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:28.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:28 smithi173 bash[58816]: cluster 2023-10-01T10:29:27.418522+0000 mgr.y (mgr.25072) 278 : cluster 0 pgmap v217: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 12703/268482 objects degraded (4.731%); 709 KiB/s, 11 objects/s recovering 2023-10-01T10:29:28.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:28 smithi173 bash[58816]: audit 2023-10-01T10:29:27.565907+0000 mon.a (mon.0) 388 : audit 0 from='client.? 172.21.15.78:0/4178181840' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:30.532 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:30 smithi078 bash[75299]: cluster 2023-10-01T10:29:29.419399+0000 mgr.y (mgr.25072) 279 : cluster 0 pgmap v218: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 889 op/s; 12636/258240 objects degraded (4.893%); 1.0 MiB/s, 16 objects/s recovering 2023-10-01T10:29:30.533 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:30 smithi078 bash[75299]: audit 2023-10-01T10:29:30.200459+0000 mon.a (mon.0) 389 : audit 0 from='client.? 172.21.15.78:0/3925980020' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:30.533 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:29:30 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:29:30] "GET /metrics HTTP/1.1" 200 36216 "" "Prometheus/2.33.4" 2023-10-01T10:29:30.533 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:30 smithi078 bash[78274]: cluster 2023-10-01T10:29:29.419399+0000 mgr.y (mgr.25072) 279 : cluster 0 pgmap v218: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 889 op/s; 12636/258240 objects degraded (4.893%); 1.0 MiB/s, 16 objects/s recovering 2023-10-01T10:29:30.533 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:30 smithi078 bash[78274]: audit 2023-10-01T10:29:30.200459+0000 mon.a (mon.0) 389 : audit 0 from='client.? 172.21.15.78:0/3925980020' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:30.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:30 smithi173 bash[58816]: cluster 2023-10-01T10:29:29.419399+0000 mgr.y (mgr.25072) 279 : cluster 0 pgmap v218: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 889 op/s; 12636/258240 objects degraded (4.893%); 1.0 MiB/s, 16 objects/s recovering 2023-10-01T10:29:30.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:30 smithi173 bash[58816]: audit 2023-10-01T10:29:30.200459+0000 mon.a (mon.0) 389 : audit 0 from='client.? 172.21.15.78:0/3925980020' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:31.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:31 smithi078 bash[75299]: audit 2023-10-01T10:29:30.521457+0000 mon.a (mon.0) 390 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:29:31.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:31 smithi078 bash[75299]: audit 2023-10-01T10:29:30.541911+0000 mon.a (mon.0) 391 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:29:31.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:31 smithi078 bash[75299]: audit 2023-10-01T10:29:30.773505+0000 mon.a (mon.0) 392 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:29:31.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:31 smithi078 bash[75299]: audit 2023-10-01T10:29:30.783121+0000 mon.a (mon.0) 393 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:29:31.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:31 smithi078 bash[75299]: audit 2023-10-01T10:29:31.433034+0000 mon.a (mon.0) 394 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:29:31.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:31 smithi078 bash[75299]: audit 2023-10-01T10:29:31.434372+0000 mon.a (mon.0) 395 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:29:31.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:31 smithi078 bash[75299]: audit 2023-10-01T10:29:31.445053+0000 mon.a (mon.0) 396 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:29:31.834 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:29:31 smithi078 bash[71214]: debug 2023-10-01T10:29:31.551+0000 7fd9c245e700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:29:31.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:31 smithi078 bash[78274]: audit 2023-10-01T10:29:30.521457+0000 mon.a (mon.0) 390 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:29:31.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:31 smithi078 bash[78274]: audit 2023-10-01T10:29:30.541911+0000 mon.a (mon.0) 391 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:29:31.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:31 smithi078 bash[78274]: audit 2023-10-01T10:29:30.773505+0000 mon.a (mon.0) 392 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:29:31.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:31 smithi078 bash[78274]: audit 2023-10-01T10:29:30.783121+0000 mon.a (mon.0) 393 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:29:31.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:31 smithi078 bash[78274]: audit 2023-10-01T10:29:31.433034+0000 mon.a (mon.0) 394 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:29:31.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:31 smithi078 bash[78274]: audit 2023-10-01T10:29:31.434372+0000 mon.a (mon.0) 395 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:29:31.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:31 smithi078 bash[78274]: audit 2023-10-01T10:29:31.445053+0000 mon.a (mon.0) 396 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:29:31.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:31 smithi173 bash[58816]: audit 2023-10-01T10:29:30.521457+0000 mon.a (mon.0) 390 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:29:31.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:31 smithi173 bash[58816]: audit 2023-10-01T10:29:30.541911+0000 mon.a (mon.0) 391 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:29:31.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:31 smithi173 bash[58816]: audit 2023-10-01T10:29:30.773505+0000 mon.a (mon.0) 392 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:29:31.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:31 smithi173 bash[58816]: audit 2023-10-01T10:29:30.783121+0000 mon.a (mon.0) 393 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:29:31.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:31 smithi173 bash[58816]: audit 2023-10-01T10:29:31.433034+0000 mon.a (mon.0) 394 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:29:31.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:31 smithi173 bash[58816]: audit 2023-10-01T10:29:31.434372+0000 mon.a (mon.0) 395 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:29:31.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:31 smithi173 bash[58816]: audit 2023-10-01T10:29:31.445053+0000 mon.a (mon.0) 396 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:29:32.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:32 smithi078 bash[75299]: cluster 2023-10-01T10:29:31.420007+0000 mgr.y (mgr.25072) 280 : cluster 0 pgmap v219: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 582 op/s; 12636/258240 objects degraded (4.893%); 714 KiB/s, 11 objects/s recovering 2023-10-01T10:29:32.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:32 smithi078 bash[75299]: audit 2023-10-01T10:29:31.528565+0000 mon.a (mon.0) 397 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:29:32.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:32 smithi078 bash[75299]: audit 2023-10-01T10:29:31.541604+0000 mon.a (mon.0) 398 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:29:32.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:32 smithi078 bash[75299]: audit 2023-10-01T10:29:31.543796+0000 mon.a (mon.0) 399 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:29:32.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:32 smithi078 bash[75299]: audit 2023-10-01T10:29:31.548055+0000 mon.a (mon.0) 400 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:29:32.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:32 smithi078 bash[75299]: audit 2023-10-01T10:29:31.551170+0000 mon.a (mon.0) 401 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:29:32.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:32 smithi078 bash[75299]: cluster 2023-10-01T10:29:31.785529+0000 mon.a (mon.0) 402 : cluster 3 Health check update: Degraded data redundancy: 12636/258240 objects degraded (4.893%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-01T10:29:32.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:32 smithi078 bash[75299]: audit 2023-10-01T10:29:32.389391+0000 mon.a (mon.0) 403 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:29:32.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:32 smithi078 bash[78274]: cluster 2023-10-01T10:29:31.420007+0000 mgr.y (mgr.25072) 280 : cluster 0 pgmap v219: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 582 op/s; 12636/258240 objects degraded (4.893%); 714 KiB/s, 11 objects/s recovering 2023-10-01T10:29:32.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:32 smithi078 bash[78274]: audit 2023-10-01T10:29:31.528565+0000 mon.a (mon.0) 397 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:29:32.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:32 smithi078 bash[78274]: audit 2023-10-01T10:29:31.541604+0000 mon.a (mon.0) 398 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:29:32.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:32 smithi078 bash[78274]: audit 2023-10-01T10:29:31.543796+0000 mon.a (mon.0) 399 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:29:32.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:32 smithi078 bash[78274]: audit 2023-10-01T10:29:31.548055+0000 mon.a (mon.0) 400 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:29:32.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:32 smithi078 bash[78274]: audit 2023-10-01T10:29:31.551170+0000 mon.a (mon.0) 401 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:29:32.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:32 smithi078 bash[78274]: cluster 2023-10-01T10:29:31.785529+0000 mon.a (mon.0) 402 : cluster 3 Health check update: Degraded data redundancy: 12636/258240 objects degraded (4.893%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-01T10:29:32.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:32 smithi078 bash[78274]: audit 2023-10-01T10:29:32.389391+0000 mon.a (mon.0) 403 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:29:32.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:32 smithi173 bash[58816]: cluster 2023-10-01T10:29:31.420007+0000 mgr.y (mgr.25072) 280 : cluster 0 pgmap v219: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 582 op/s; 12636/258240 objects degraded (4.893%); 714 KiB/s, 11 objects/s recovering 2023-10-01T10:29:32.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:32 smithi173 bash[58816]: audit 2023-10-01T10:29:31.528565+0000 mon.a (mon.0) 397 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:29:32.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:32 smithi173 bash[58816]: audit 2023-10-01T10:29:31.541604+0000 mon.a (mon.0) 398 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:29:32.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:32 smithi173 bash[58816]: audit 2023-10-01T10:29:31.543796+0000 mon.a (mon.0) 399 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:29:32.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:32 smithi173 bash[58816]: audit 2023-10-01T10:29:31.548055+0000 mon.a (mon.0) 400 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:29:32.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:32 smithi173 bash[58816]: audit 2023-10-01T10:29:31.551170+0000 mon.a (mon.0) 401 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:29:32.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:32 smithi173 bash[58816]: cluster 2023-10-01T10:29:31.785529+0000 mon.a (mon.0) 402 : cluster 3 Health check update: Degraded data redundancy: 12636/258240 objects degraded (4.893%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-01T10:29:32.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:32 smithi173 bash[58816]: audit 2023-10-01T10:29:32.389391+0000 mon.a (mon.0) 403 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:29:33.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:33 smithi078 bash[75299]: audit 2023-10-01T10:29:31.555131+0000 mgr.y (mgr.25072) 281 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:29:33.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:33 smithi078 bash[75299]: cephadm 2023-10-01T10:29:31.560557+0000 mgr.y (mgr.25072) 282 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:29:33.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:33 smithi078 bash[75299]: audit 2023-10-01T10:29:32.788205+0000 mon.c (mon.2) 23 : audit 0 from='client.? 172.21.15.78:0/1651987410' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:33.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:33 smithi078 bash[78274]: audit 2023-10-01T10:29:31.555131+0000 mgr.y (mgr.25072) 281 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:29:33.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:33 smithi078 bash[78274]: cephadm 2023-10-01T10:29:31.560557+0000 mgr.y (mgr.25072) 282 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:29:33.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:33 smithi078 bash[78274]: audit 2023-10-01T10:29:32.788205+0000 mon.c (mon.2) 23 : audit 0 from='client.? 172.21.15.78:0/1651987410' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:33.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:33 smithi173 bash[58816]: audit 2023-10-01T10:29:31.555131+0000 mgr.y (mgr.25072) 281 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:29:33.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:33 smithi173 bash[58816]: cephadm 2023-10-01T10:29:31.560557+0000 mgr.y (mgr.25072) 282 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:29:33.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:33 smithi173 bash[58816]: audit 2023-10-01T10:29:32.788205+0000 mon.c (mon.2) 23 : audit 0 from='client.? 172.21.15.78:0/1651987410' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:34.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:34 smithi078 bash[75299]: cluster 2023-10-01T10:29:33.420577+0000 mgr.y (mgr.25072) 283 : cluster 0 pgmap v220: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 12636/256455 objects degraded (4.927%); 714 KiB/s, 11 objects/s recovering 2023-10-01T10:29:34.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:34 smithi078 bash[78274]: cluster 2023-10-01T10:29:33.420577+0000 mgr.y (mgr.25072) 283 : cluster 0 pgmap v220: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 12636/256455 objects degraded (4.927%); 714 KiB/s, 11 objects/s recovering 2023-10-01T10:29:34.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:34 smithi173 bash[58816]: cluster 2023-10-01T10:29:33.420577+0000 mgr.y (mgr.25072) 283 : cluster 0 pgmap v220: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 12636/256455 objects degraded (4.927%); 714 KiB/s, 11 objects/s recovering 2023-10-01T10:29:35.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:35 smithi078 bash[75299]: audit 2023-10-01T10:29:35.411063+0000 mon.b (mon.1) 10 : audit 0 from='client.? 172.21.15.78:0/1586908674' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:35.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:35 smithi078 bash[75299]: cluster 2023-10-01T10:29:35.421352+0000 mgr.y (mgr.25072) 284 : cluster 0 pgmap v221: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 828 op/s; 12566/248661 objects degraded (5.053%); 1.1 MiB/s, 16 objects/s recovering 2023-10-01T10:29:35.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:35 smithi078 bash[78274]: audit 2023-10-01T10:29:35.411063+0000 mon.b (mon.1) 10 : audit 0 from='client.? 172.21.15.78:0/1586908674' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:35.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:35 smithi078 bash[78274]: cluster 2023-10-01T10:29:35.421352+0000 mgr.y (mgr.25072) 284 : cluster 0 pgmap v221: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 828 op/s; 12566/248661 objects degraded (5.053%); 1.1 MiB/s, 16 objects/s recovering 2023-10-01T10:29:35.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:35 smithi173 bash[58816]: audit 2023-10-01T10:29:35.411063+0000 mon.b (mon.1) 10 : audit 0 from='client.? 172.21.15.78:0/1586908674' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:35.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:35 smithi173 bash[58816]: cluster 2023-10-01T10:29:35.421352+0000 mgr.y (mgr.25072) 284 : cluster 0 pgmap v221: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 828 op/s; 12566/248661 objects degraded (5.053%); 1.1 MiB/s, 16 objects/s recovering 2023-10-01T10:29:37.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:36 smithi173 bash[58816]: cluster 2023-10-01T10:29:36.786658+0000 mon.a (mon.0) 404 : cluster 3 Health check update: Degraded data redundancy: 12566/248661 objects degraded (5.053%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-01T10:29:37.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:36 smithi078 bash[78274]: cluster 2023-10-01T10:29:36.786658+0000 mon.a (mon.0) 404 : cluster 3 Health check update: Degraded data redundancy: 12566/248661 objects degraded (5.053%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-01T10:29:37.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:36 smithi078 bash[75299]: cluster 2023-10-01T10:29:36.786658+0000 mon.a (mon.0) 404 : cluster 3 Health check update: Degraded data redundancy: 12566/248661 objects degraded (5.053%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-01T10:29:38.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:37 smithi173 bash[58816]: cluster 2023-10-01T10:29:37.421944+0000 mgr.y (mgr.25072) 285 : cluster 0 pgmap v222: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 550 op/s; 12566/248661 objects degraded (5.053%); 730 KiB/s, 11 objects/s recovering 2023-10-01T10:29:38.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:37 smithi078 bash[78274]: cluster 2023-10-01T10:29:37.421944+0000 mgr.y (mgr.25072) 285 : cluster 0 pgmap v222: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 550 op/s; 12566/248661 objects degraded (5.053%); 730 KiB/s, 11 objects/s recovering 2023-10-01T10:29:38.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:37 smithi078 bash[75299]: cluster 2023-10-01T10:29:37.421944+0000 mgr.y (mgr.25072) 285 : cluster 0 pgmap v222: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 550 op/s; 12566/248661 objects degraded (5.053%); 730 KiB/s, 11 objects/s recovering 2023-10-01T10:29:39.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:38 smithi173 bash[58816]: audit 2023-10-01T10:29:38.016138+0000 mon.a (mon.0) 405 : audit 0 from='client.? 172.21.15.78:0/3045635338' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:39.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:38 smithi078 bash[78274]: audit 2023-10-01T10:29:38.016138+0000 mon.a (mon.0) 405 : audit 0 from='client.? 172.21.15.78:0/3045635338' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:39.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:38 smithi078 bash[75299]: audit 2023-10-01T10:29:38.016138+0000 mon.a (mon.0) 405 : audit 0 from='client.? 172.21.15.78:0/3045635338' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:40.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:39 smithi173 bash[58816]: cluster 2023-10-01T10:29:39.422911+0000 mgr.y (mgr.25072) 286 : cluster 0 pgmap v223: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 825 op/s; 12500/238749 objects degraded (5.236%); 1.1 MiB/s, 16 objects/s recovering 2023-10-01T10:29:40.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:39 smithi078 bash[78274]: cluster 2023-10-01T10:29:39.422911+0000 mgr.y (mgr.25072) 286 : cluster 0 pgmap v223: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 825 op/s; 12500/238749 objects degraded (5.236%); 1.1 MiB/s, 16 objects/s recovering 2023-10-01T10:29:40.271 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:39 smithi078 bash[75299]: cluster 2023-10-01T10:29:39.422911+0000 mgr.y (mgr.25072) 286 : cluster 0 pgmap v223: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 825 op/s; 12500/238749 objects degraded (5.236%); 1.1 MiB/s, 16 objects/s recovering 2023-10-01T10:29:40.271 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:29:40 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:29:40] "GET /metrics HTTP/1.1" 200 36218 "" "Prometheus/2.33.4" 2023-10-01T10:29:41.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:40 smithi173 bash[58816]: audit 2023-10-01T10:29:40.649953+0000 mon.a (mon.0) 406 : audit 0 from='client.? 172.21.15.78:0/2052005918' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:41.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:40 smithi078 bash[75299]: audit 2023-10-01T10:29:40.649953+0000 mon.a (mon.0) 406 : audit 0 from='client.? 172.21.15.78:0/2052005918' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:41.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:40 smithi078 bash[78274]: audit 2023-10-01T10:29:40.649953+0000 mon.a (mon.0) 406 : audit 0 from='client.? 172.21.15.78:0/2052005918' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:42.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:41 smithi078 bash[75299]: cluster 2023-10-01T10:29:41.423509+0000 mgr.y (mgr.25072) 287 : cluster 0 pgmap v224: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 12500/238749 objects degraded (5.236%); 725 KiB/s, 11 objects/s recovering 2023-10-01T10:29:42.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:41 smithi078 bash[75299]: cluster 2023-10-01T10:29:41.787940+0000 mon.a (mon.0) 407 : cluster 3 Health check update: Degraded data redundancy: 12500/238749 objects degraded (5.236%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-01T10:29:42.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:41 smithi078 bash[78274]: cluster 2023-10-01T10:29:41.423509+0000 mgr.y (mgr.25072) 287 : cluster 0 pgmap v224: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 12500/238749 objects degraded (5.236%); 725 KiB/s, 11 objects/s recovering 2023-10-01T10:29:42.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:41 smithi078 bash[78274]: cluster 2023-10-01T10:29:41.787940+0000 mon.a (mon.0) 407 : cluster 3 Health check update: Degraded data redundancy: 12500/238749 objects degraded (5.236%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-01T10:29:42.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:41 smithi173 bash[58816]: cluster 2023-10-01T10:29:41.423509+0000 mgr.y (mgr.25072) 287 : cluster 0 pgmap v224: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 12500/238749 objects degraded (5.236%); 725 KiB/s, 11 objects/s recovering 2023-10-01T10:29:42.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:41 smithi173 bash[58816]: cluster 2023-10-01T10:29:41.787940+0000 mon.a (mon.0) 407 : cluster 3 Health check update: Degraded data redundancy: 12500/238749 objects degraded (5.236%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-01T10:29:43.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:43 smithi173 bash[58816]: audit 2023-10-01T10:29:43.284847+0000 mon.a (mon.0) 408 : audit 0 from='client.? 172.21.15.78:0/827418495' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:43.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:43 smithi078 bash[75299]: audit 2023-10-01T10:29:43.284847+0000 mon.a (mon.0) 408 : audit 0 from='client.? 172.21.15.78:0/827418495' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:43.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:43 smithi078 bash[78274]: audit 2023-10-01T10:29:43.284847+0000 mon.a (mon.0) 408 : audit 0 from='client.? 172.21.15.78:0/827418495' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:44.622 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:44 smithi173 bash[58816]: cluster 2023-10-01T10:29:43.424060+0000 mgr.y (mgr.25072) 288 : cluster 0 pgmap v225: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 12500/238749 objects degraded (5.236%); 725 KiB/s, 11 objects/s recovering 2023-10-01T10:29:44.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:44 smithi078 bash[78274]: cluster 2023-10-01T10:29:43.424060+0000 mgr.y (mgr.25072) 288 : cluster 0 pgmap v225: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 12500/238749 objects degraded (5.236%); 725 KiB/s, 11 objects/s recovering 2023-10-01T10:29:44.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:44 smithi078 bash[75299]: cluster 2023-10-01T10:29:43.424060+0000 mgr.y (mgr.25072) 288 : cluster 0 pgmap v225: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 12500/238749 objects degraded (5.236%); 725 KiB/s, 11 objects/s recovering 2023-10-01T10:29:46.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:46 smithi078 bash[78274]: cluster 2023-10-01T10:29:45.424879+0000 mgr.y (mgr.25072) 289 : cluster 0 pgmap v226: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 12434/229158 objects degraded (5.426%); 1.1 MiB/s, 16 objects/s recovering 2023-10-01T10:29:46.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:46 smithi078 bash[78274]: audit 2023-10-01T10:29:45.920600+0000 mon.b (mon.1) 11 : audit 0 from='client.? 172.21.15.78:0/4255709562' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:46.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:46 smithi078 bash[75299]: cluster 2023-10-01T10:29:45.424879+0000 mgr.y (mgr.25072) 289 : cluster 0 pgmap v226: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 12434/229158 objects degraded (5.426%); 1.1 MiB/s, 16 objects/s recovering 2023-10-01T10:29:46.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:46 smithi078 bash[75299]: audit 2023-10-01T10:29:45.920600+0000 mon.b (mon.1) 11 : audit 0 from='client.? 172.21.15.78:0/4255709562' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:46.834 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:29:46 smithi078 bash[71214]: debug 2023-10-01T10:29:46.570+0000 7fd9c245e700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:29:46.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:46 smithi173 bash[58816]: cluster 2023-10-01T10:29:45.424879+0000 mgr.y (mgr.25072) 289 : cluster 0 pgmap v226: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 12434/229158 objects degraded (5.426%); 1.1 MiB/s, 16 objects/s recovering 2023-10-01T10:29:46.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:46 smithi173 bash[58816]: audit 2023-10-01T10:29:45.920600+0000 mon.b (mon.1) 11 : audit 0 from='client.? 172.21.15.78:0/4255709562' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:47.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:47 smithi078 bash[75299]: audit 2023-10-01T10:29:46.573772+0000 mon.a (mon.0) 409 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:29:47.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:47 smithi078 bash[75299]: cluster 2023-10-01T10:29:46.789208+0000 mon.a (mon.0) 410 : cluster 3 Health check update: Degraded data redundancy: 12434/229158 objects degraded (5.426%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-01T10:29:47.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:47 smithi078 bash[75299]: audit 2023-10-01T10:29:47.389691+0000 mon.a (mon.0) 411 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:29:47.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:47 smithi078 bash[78274]: audit 2023-10-01T10:29:46.573772+0000 mon.a (mon.0) 409 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:29:47.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:47 smithi078 bash[78274]: cluster 2023-10-01T10:29:46.789208+0000 mon.a (mon.0) 410 : cluster 3 Health check update: Degraded data redundancy: 12434/229158 objects degraded (5.426%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-01T10:29:47.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:47 smithi078 bash[78274]: audit 2023-10-01T10:29:47.389691+0000 mon.a (mon.0) 411 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:29:47.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:47 smithi173 bash[58816]: audit 2023-10-01T10:29:46.573772+0000 mon.a (mon.0) 409 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:29:47.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:47 smithi173 bash[58816]: cluster 2023-10-01T10:29:46.789208+0000 mon.a (mon.0) 410 : cluster 3 Health check update: Degraded data redundancy: 12434/229158 objects degraded (5.426%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-01T10:29:47.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:47 smithi173 bash[58816]: audit 2023-10-01T10:29:47.389691+0000 mon.a (mon.0) 411 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:29:48.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:48 smithi078 bash[75299]: audit 2023-10-01T10:29:46.573997+0000 mgr.y (mgr.25072) 290 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:29:48.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:48 smithi078 bash[75299]: cephadm 2023-10-01T10:29:46.575109+0000 mgr.y (mgr.25072) 291 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:29:48.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:48 smithi078 bash[75299]: cluster 2023-10-01T10:29:47.425416+0000 mgr.y (mgr.25072) 292 : cluster 0 pgmap v227: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 12434/229158 objects degraded (5.426%); 704 KiB/s, 10 objects/s recovering 2023-10-01T10:29:48.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:48 smithi078 bash[75299]: audit 2023-10-01T10:29:48.485313+0000 mon.c (mon.2) 24 : audit 0 from='client.? 172.21.15.78:0/3887667420' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:48.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:48 smithi078 bash[78274]: audit 2023-10-01T10:29:46.573997+0000 mgr.y (mgr.25072) 290 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:29:48.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:48 smithi078 bash[78274]: cephadm 2023-10-01T10:29:46.575109+0000 mgr.y (mgr.25072) 291 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:29:48.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:48 smithi078 bash[78274]: cluster 2023-10-01T10:29:47.425416+0000 mgr.y (mgr.25072) 292 : cluster 0 pgmap v227: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 12434/229158 objects degraded (5.426%); 704 KiB/s, 10 objects/s recovering 2023-10-01T10:29:48.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:48 smithi078 bash[78274]: audit 2023-10-01T10:29:48.485313+0000 mon.c (mon.2) 24 : audit 0 from='client.? 172.21.15.78:0/3887667420' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:48.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:48 smithi173 bash[58816]: audit 2023-10-01T10:29:46.573997+0000 mgr.y (mgr.25072) 290 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:29:48.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:48 smithi173 bash[58816]: cephadm 2023-10-01T10:29:46.575109+0000 mgr.y (mgr.25072) 291 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:29:48.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:48 smithi173 bash[58816]: cluster 2023-10-01T10:29:47.425416+0000 mgr.y (mgr.25072) 292 : cluster 0 pgmap v227: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 12434/229158 objects degraded (5.426%); 704 KiB/s, 10 objects/s recovering 2023-10-01T10:29:48.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:48 smithi173 bash[58816]: audit 2023-10-01T10:29:48.485313+0000 mon.c (mon.2) 24 : audit 0 from='client.? 172.21.15.78:0/3887667420' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:50.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:50 smithi078 bash[78274]: cluster 2023-10-01T10:29:49.426261+0000 mgr.y (mgr.25072) 293 : cluster 0 pgmap v228: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 12434/222069 objects degraded (5.599%); 704 KiB/s, 10 objects/s recovering 2023-10-01T10:29:50.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:50 smithi078 bash[75299]: cluster 2023-10-01T10:29:49.426261+0000 mgr.y (mgr.25072) 293 : cluster 0 pgmap v228: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 12434/222069 objects degraded (5.599%); 704 KiB/s, 10 objects/s recovering 2023-10-01T10:29:50.584 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:29:50 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:29:50] "GET /metrics HTTP/1.1" 200 36217 "" "Prometheus/2.33.4" 2023-10-01T10:29:50.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:50 smithi173 bash[58816]: cluster 2023-10-01T10:29:49.426261+0000 mgr.y (mgr.25072) 293 : cluster 0 pgmap v228: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 12434/222069 objects degraded (5.599%); 704 KiB/s, 10 objects/s recovering 2023-10-01T10:29:51.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:51 smithi078 bash[75299]: audit 2023-10-01T10:29:51.105980+0000 mon.c (mon.2) 25 : audit 0 from='client.? 172.21.15.78:0/3301700336' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:51.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:51 smithi078 bash[78274]: audit 2023-10-01T10:29:51.105980+0000 mon.c (mon.2) 25 : audit 0 from='client.? 172.21.15.78:0/3301700336' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:51.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:51 smithi173 bash[58816]: audit 2023-10-01T10:29:51.105980+0000 mon.c (mon.2) 25 : audit 0 from='client.? 172.21.15.78:0/3301700336' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:52.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:52 smithi078 bash[75299]: cluster 2023-10-01T10:29:51.426892+0000 mgr.y (mgr.25072) 294 : cluster 0 pgmap v229: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 549 op/s; 12366/218946 objects degraded (5.648%); 714 KiB/s, 11 objects/s recovering 2023-10-01T10:29:52.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:52 smithi078 bash[75299]: cluster 2023-10-01T10:29:51.790602+0000 mon.a (mon.0) 412 : cluster 3 Health check update: Degraded data redundancy: 12366/218946 objects degraded (5.648%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-01T10:29:52.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:52 smithi078 bash[78274]: cluster 2023-10-01T10:29:51.426892+0000 mgr.y (mgr.25072) 294 : cluster 0 pgmap v229: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 549 op/s; 12366/218946 objects degraded (5.648%); 714 KiB/s, 11 objects/s recovering 2023-10-01T10:29:52.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:52 smithi078 bash[78274]: cluster 2023-10-01T10:29:51.790602+0000 mon.a (mon.0) 412 : cluster 3 Health check update: Degraded data redundancy: 12366/218946 objects degraded (5.648%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-01T10:29:52.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:52 smithi173 bash[58816]: cluster 2023-10-01T10:29:51.426892+0000 mgr.y (mgr.25072) 294 : cluster 0 pgmap v229: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 549 op/s; 12366/218946 objects degraded (5.648%); 714 KiB/s, 11 objects/s recovering 2023-10-01T10:29:52.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:52 smithi173 bash[58816]: cluster 2023-10-01T10:29:51.790602+0000 mon.a (mon.0) 412 : cluster 3 Health check update: Degraded data redundancy: 12366/218946 objects degraded (5.648%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-01T10:29:54.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:54 smithi078 bash[75299]: cluster 2023-10-01T10:29:53.427627+0000 mgr.y (mgr.25072) 295 : cluster 0 pgmap v230: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 549 op/s; 12366/218946 objects degraded (5.648%); 714 KiB/s, 11 objects/s recovering 2023-10-01T10:29:54.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:54 smithi078 bash[75299]: audit 2023-10-01T10:29:53.742736+0000 mon.c (mon.2) 26 : audit 0 from='client.? 172.21.15.78:0/2296026448' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:54.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:54 smithi078 bash[78274]: cluster 2023-10-01T10:29:53.427627+0000 mgr.y (mgr.25072) 295 : cluster 0 pgmap v230: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 549 op/s; 12366/218946 objects degraded (5.648%); 714 KiB/s, 11 objects/s recovering 2023-10-01T10:29:54.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:54 smithi078 bash[78274]: audit 2023-10-01T10:29:53.742736+0000 mon.c (mon.2) 26 : audit 0 from='client.? 172.21.15.78:0/2296026448' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:54.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:54 smithi173 bash[58816]: cluster 2023-10-01T10:29:53.427627+0000 mgr.y (mgr.25072) 295 : cluster 0 pgmap v230: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 549 op/s; 12366/218946 objects degraded (5.648%); 714 KiB/s, 11 objects/s recovering 2023-10-01T10:29:54.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:54 smithi173 bash[58816]: audit 2023-10-01T10:29:53.742736+0000 mon.c (mon.2) 26 : audit 0 from='client.? 172.21.15.78:0/2296026448' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:56.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:56 smithi078 bash[75299]: cluster 2023-10-01T10:29:55.428461+0000 mgr.y (mgr.25072) 296 : cluster 0 pgmap v231: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 830 op/s; 12298/208857 objects degraded (5.888%); 1.1 MiB/s, 16 objects/s recovering 2023-10-01T10:29:56.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:56 smithi078 bash[75299]: audit 2023-10-01T10:29:56.351982+0000 mon.c (mon.2) 27 : audit 0 from='client.? 172.21.15.78:0/2345830667' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:56.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:56 smithi078 bash[78274]: cluster 2023-10-01T10:29:55.428461+0000 mgr.y (mgr.25072) 296 : cluster 0 pgmap v231: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 830 op/s; 12298/208857 objects degraded (5.888%); 1.1 MiB/s, 16 objects/s recovering 2023-10-01T10:29:56.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:56 smithi078 bash[78274]: audit 2023-10-01T10:29:56.351982+0000 mon.c (mon.2) 27 : audit 0 from='client.? 172.21.15.78:0/2345830667' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:56.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:56 smithi173 bash[58816]: cluster 2023-10-01T10:29:55.428461+0000 mgr.y (mgr.25072) 296 : cluster 0 pgmap v231: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 830 op/s; 12298/208857 objects degraded (5.888%); 1.1 MiB/s, 16 objects/s recovering 2023-10-01T10:29:56.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:56 smithi173 bash[58816]: audit 2023-10-01T10:29:56.351982+0000 mon.c (mon.2) 27 : audit 0 from='client.? 172.21.15.78:0/2345830667' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:57.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:57 smithi078 bash[75299]: cluster 2023-10-01T10:29:56.791918+0000 mon.a (mon.0) 413 : cluster 3 Health check update: Degraded data redundancy: 12298/208857 objects degraded (5.888%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-01T10:29:57.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:57 smithi078 bash[78274]: cluster 2023-10-01T10:29:56.791918+0000 mon.a (mon.0) 413 : cluster 3 Health check update: Degraded data redundancy: 12298/208857 objects degraded (5.888%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-01T10:29:57.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:57 smithi173 bash[58816]: cluster 2023-10-01T10:29:56.791918+0000 mon.a (mon.0) 413 : cluster 3 Health check update: Degraded data redundancy: 12298/208857 objects degraded (5.888%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-01T10:29:58.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:58 smithi078 bash[75299]: cluster 2023-10-01T10:29:57.428875+0000 mgr.y (mgr.25072) 297 : cluster 0 pgmap v232: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 563 op/s; 12298/208857 objects degraded (5.888%); 725 KiB/s, 11 objects/s recovering 2023-10-01T10:29:58.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:58 smithi078 bash[78274]: cluster 2023-10-01T10:29:57.428875+0000 mgr.y (mgr.25072) 297 : cluster 0 pgmap v232: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 563 op/s; 12298/208857 objects degraded (5.888%); 725 KiB/s, 11 objects/s recovering 2023-10-01T10:29:58.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:58 smithi173 bash[58816]: cluster 2023-10-01T10:29:57.428875+0000 mgr.y (mgr.25072) 297 : cluster 0 pgmap v232: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 563 op/s; 12298/208857 objects degraded (5.888%); 725 KiB/s, 11 objects/s recovering 2023-10-01T10:29:59.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:29:59 smithi078 bash[75299]: audit 2023-10-01T10:29:58.914538+0000 mon.a (mon.0) 414 : audit 0 from='client.? 172.21.15.78:0/1871806583' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:59.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:29:59 smithi078 bash[78274]: audit 2023-10-01T10:29:58.914538+0000 mon.a (mon.0) 414 : audit 0 from='client.? 172.21.15.78:0/1871806583' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:29:59.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:29:59 smithi173 bash[58816]: audit 2023-10-01T10:29:58.914538+0000 mon.a (mon.0) 414 : audit 0 from='client.? 172.21.15.78:0/1871806583' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:00.540 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:00 smithi078 bash[75299]: cluster 2023-10-01T10:29:59.429687+0000 mgr.y (mgr.25072) 298 : cluster 0 pgmap v233: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 12298/201984 objects degraded (6.089%); 725 KiB/s, 11 objects/s recovering 2023-10-01T10:30:00.540 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:00 smithi078 bash[75299]: cluster 2023-10-01T10:30:00.000125+0000 mon.a (mon.0) 415 : cluster 3 Health detail: HEALTH_WARN Degraded data redundancy: 12298/201984 objects degraded (6.089%), 7 pgs degraded, 7 pgs undersized 2023-10-01T10:30:00.541 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:00 smithi078 bash[75299]: cluster 2023-10-01T10:30:00.000157+0000 mon.a (mon.0) 416 : cluster 3 [WRN] PG_DEGRADED: Degraded data redundancy: 12298/201984 objects degraded (6.089%), 7 pgs degraded, 7 pgs undersized 2023-10-01T10:30:00.541 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:00 smithi078 bash[75299]: cluster 2023-10-01T10:30:00.000169+0000 mon.a (mon.0) 417 : cluster 3 pg 7.5 is stuck undersized for 98s, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,2] 2023-10-01T10:30:00.541 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:00 smithi078 bash[75299]: cluster 2023-10-01T10:30:00.000183+0000 mon.a (mon.0) 418 : cluster 3 pg 7.6 is stuck undersized for 98s, current state active+recovering+undersized+degraded+remapped, last acting [3,7] 2023-10-01T10:30:00.541 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:00 smithi078 bash[75299]: cluster 2023-10-01T10:30:00.000196+0000 mon.a (mon.0) 419 : cluster 3 pg 7.7 is stuck undersized for 98s, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,3] 2023-10-01T10:30:00.541 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:30:00 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:30:00] "GET /metrics HTTP/1.1" 200 36217 "" "Prometheus/2.33.4" 2023-10-01T10:30:00.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:00 smithi078 bash[75299]: cluster 2023-10-01T10:30:00.000210+0000 mon.a (mon.0) 420 : cluster 3 pg 7.9 is stuck undersized for 98s, current state active+recovery_wait+undersized+degraded+remapped, last acting [1,4] 2023-10-01T10:30:00.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:00 smithi078 bash[75299]: cluster 2023-10-01T10:30:00.000226+0000 mon.a (mon.0) 421 : cluster 3 pg 7.d is stuck undersized for 98s, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,1] 2023-10-01T10:30:00.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:00 smithi078 bash[75299]: cluster 2023-10-01T10:30:00.000239+0000 mon.a (mon.0) 422 : cluster 3 pg 7.e is stuck undersized for 98s, current state active+recovery_wait+undersized+degraded+remapped, last acting [2,5] 2023-10-01T10:30:00.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:00 smithi078 bash[75299]: cluster 2023-10-01T10:30:00.000252+0000 mon.a (mon.0) 423 : cluster 3 pg 7.f is stuck undersized for 98s, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,7] 2023-10-01T10:30:00.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:00 smithi078 bash[78274]: cluster 2023-10-01T10:29:59.429687+0000 mgr.y (mgr.25072) 298 : cluster 0 pgmap v233: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 12298/201984 objects degraded (6.089%); 725 KiB/s, 11 objects/s recovering 2023-10-01T10:30:00.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:00 smithi078 bash[78274]: cluster 2023-10-01T10:30:00.000125+0000 mon.a (mon.0) 415 : cluster 3 Health detail: HEALTH_WARN Degraded data redundancy: 12298/201984 objects degraded (6.089%), 7 pgs degraded, 7 pgs undersized 2023-10-01T10:30:00.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:00 smithi078 bash[78274]: cluster 2023-10-01T10:30:00.000157+0000 mon.a (mon.0) 416 : cluster 3 [WRN] PG_DEGRADED: Degraded data redundancy: 12298/201984 objects degraded (6.089%), 7 pgs degraded, 7 pgs undersized 2023-10-01T10:30:00.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:00 smithi078 bash[78274]: cluster 2023-10-01T10:30:00.000169+0000 mon.a (mon.0) 417 : cluster 3 pg 7.5 is stuck undersized for 98s, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,2] 2023-10-01T10:30:00.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:00 smithi078 bash[78274]: cluster 2023-10-01T10:30:00.000183+0000 mon.a (mon.0) 418 : cluster 3 pg 7.6 is stuck undersized for 98s, current state active+recovering+undersized+degraded+remapped, last acting [3,7] 2023-10-01T10:30:00.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:00 smithi078 bash[78274]: cluster 2023-10-01T10:30:00.000196+0000 mon.a (mon.0) 419 : cluster 3 pg 7.7 is stuck undersized for 98s, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,3] 2023-10-01T10:30:00.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:00 smithi078 bash[78274]: cluster 2023-10-01T10:30:00.000210+0000 mon.a (mon.0) 420 : cluster 3 pg 7.9 is stuck undersized for 98s, current state active+recovery_wait+undersized+degraded+remapped, last acting [1,4] 2023-10-01T10:30:00.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:00 smithi078 bash[78274]: cluster 2023-10-01T10:30:00.000226+0000 mon.a (mon.0) 421 : cluster 3 pg 7.d is stuck undersized for 98s, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,1] 2023-10-01T10:30:00.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:00 smithi078 bash[78274]: cluster 2023-10-01T10:30:00.000239+0000 mon.a (mon.0) 422 : cluster 3 pg 7.e is stuck undersized for 98s, current state active+recovery_wait+undersized+degraded+remapped, last acting [2,5] 2023-10-01T10:30:00.836 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:00 smithi078 bash[78274]: cluster 2023-10-01T10:30:00.000252+0000 mon.a (mon.0) 423 : cluster 3 pg 7.f is stuck undersized for 98s, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,7] 2023-10-01T10:30:00.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:00 smithi173 bash[58816]: cluster 2023-10-01T10:29:59.429687+0000 mgr.y (mgr.25072) 298 : cluster 0 pgmap v233: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 12298/201984 objects degraded (6.089%); 725 KiB/s, 11 objects/s recovering 2023-10-01T10:30:00.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:00 smithi173 bash[58816]: cluster 2023-10-01T10:30:00.000125+0000 mon.a (mon.0) 415 : cluster 3 Health detail: HEALTH_WARN Degraded data redundancy: 12298/201984 objects degraded (6.089%), 7 pgs degraded, 7 pgs undersized 2023-10-01T10:30:00.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:00 smithi173 bash[58816]: cluster 2023-10-01T10:30:00.000157+0000 mon.a (mon.0) 416 : cluster 3 [WRN] PG_DEGRADED: Degraded data redundancy: 12298/201984 objects degraded (6.089%), 7 pgs degraded, 7 pgs undersized 2023-10-01T10:30:00.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:00 smithi173 bash[58816]: cluster 2023-10-01T10:30:00.000169+0000 mon.a (mon.0) 417 : cluster 3 pg 7.5 is stuck undersized for 98s, current state active+recovery_wait+undersized+degraded+remapped, last acting [7,2] 2023-10-01T10:30:00.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:00 smithi173 bash[58816]: cluster 2023-10-01T10:30:00.000183+0000 mon.a (mon.0) 418 : cluster 3 pg 7.6 is stuck undersized for 98s, current state active+recovering+undersized+degraded+remapped, last acting [3,7] 2023-10-01T10:30:00.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:00 smithi173 bash[58816]: cluster 2023-10-01T10:30:00.000196+0000 mon.a (mon.0) 419 : cluster 3 pg 7.7 is stuck undersized for 98s, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,3] 2023-10-01T10:30:00.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:00 smithi173 bash[58816]: cluster 2023-10-01T10:30:00.000210+0000 mon.a (mon.0) 420 : cluster 3 pg 7.9 is stuck undersized for 98s, current state active+recovery_wait+undersized+degraded+remapped, last acting [1,4] 2023-10-01T10:30:00.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:00 smithi173 bash[58816]: cluster 2023-10-01T10:30:00.000226+0000 mon.a (mon.0) 421 : cluster 3 pg 7.d is stuck undersized for 98s, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,1] 2023-10-01T10:30:00.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:00 smithi173 bash[58816]: cluster 2023-10-01T10:30:00.000239+0000 mon.a (mon.0) 422 : cluster 3 pg 7.e is stuck undersized for 98s, current state active+recovery_wait+undersized+degraded+remapped, last acting [2,5] 2023-10-01T10:30:00.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:00 smithi173 bash[58816]: cluster 2023-10-01T10:30:00.000252+0000 mon.a (mon.0) 423 : cluster 3 pg 7.f is stuck undersized for 98s, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,7] 2023-10-01T10:30:02.083 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:30:01 smithi078 bash[71214]: debug 2023-10-01T10:30:01.586+0000 7fd9c245e700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:30:02.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:02 smithi078 bash[75299]: cluster 2023-10-01T10:30:01.430323+0000 mgr.y (mgr.25072) 299 : cluster 0 pgmap v234: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 12231/198936 objects degraded (6.148%); 1.1 MiB/s, 16 objects/s recovering 2023-10-01T10:30:02.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:02 smithi078 bash[75299]: audit 2023-10-01T10:30:01.554734+0000 mon.a (mon.0) 424 : audit 0 from='client.? 172.21.15.78:0/68292512' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:02.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:02 smithi078 bash[75299]: audit 2023-10-01T10:30:01.592738+0000 mon.a (mon.0) 425 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:30:02.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:02 smithi078 bash[75299]: cluster 2023-10-01T10:30:01.793209+0000 mon.a (mon.0) 426 : cluster 3 Health check update: Degraded data redundancy: 12231/198936 objects degraded (6.148%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-01T10:30:02.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:02 smithi078 bash[75299]: audit 2023-10-01T10:30:02.390001+0000 mon.a (mon.0) 427 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:30:02.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:02 smithi078 bash[78274]: cluster 2023-10-01T10:30:01.430323+0000 mgr.y (mgr.25072) 299 : cluster 0 pgmap v234: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 12231/198936 objects degraded (6.148%); 1.1 MiB/s, 16 objects/s recovering 2023-10-01T10:30:02.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:02 smithi078 bash[78274]: audit 2023-10-01T10:30:01.554734+0000 mon.a (mon.0) 424 : audit 0 from='client.? 172.21.15.78:0/68292512' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:02.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:02 smithi078 bash[78274]: audit 2023-10-01T10:30:01.592738+0000 mon.a (mon.0) 425 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:30:02.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:02 smithi078 bash[78274]: cluster 2023-10-01T10:30:01.793209+0000 mon.a (mon.0) 426 : cluster 3 Health check update: Degraded data redundancy: 12231/198936 objects degraded (6.148%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-01T10:30:02.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:02 smithi078 bash[78274]: audit 2023-10-01T10:30:02.390001+0000 mon.a (mon.0) 427 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:30:02.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:02 smithi173 bash[58816]: cluster 2023-10-01T10:30:01.430323+0000 mgr.y (mgr.25072) 299 : cluster 0 pgmap v234: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 12231/198936 objects degraded (6.148%); 1.1 MiB/s, 16 objects/s recovering 2023-10-01T10:30:02.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:02 smithi173 bash[58816]: audit 2023-10-01T10:30:01.554734+0000 mon.a (mon.0) 424 : audit 0 from='client.? 172.21.15.78:0/68292512' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:02.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:02 smithi173 bash[58816]: audit 2023-10-01T10:30:01.592738+0000 mon.a (mon.0) 425 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:30:02.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:02 smithi173 bash[58816]: cluster 2023-10-01T10:30:01.793209+0000 mon.a (mon.0) 426 : cluster 3 Health check update: Degraded data redundancy: 12231/198936 objects degraded (6.148%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-01T10:30:02.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:02 smithi173 bash[58816]: audit 2023-10-01T10:30:02.390001+0000 mon.a (mon.0) 427 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:30:03.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:03 smithi078 bash[78274]: audit 2023-10-01T10:30:01.593098+0000 mgr.y (mgr.25072) 300 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:30:03.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:03 smithi078 bash[78274]: cephadm 2023-10-01T10:30:01.598020+0000 mgr.y (mgr.25072) 301 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:30:03.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:03 smithi078 bash[75299]: audit 2023-10-01T10:30:01.593098+0000 mgr.y (mgr.25072) 300 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:30:03.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:03 smithi078 bash[75299]: cephadm 2023-10-01T10:30:01.598020+0000 mgr.y (mgr.25072) 301 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:30:03.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:03 smithi173 bash[58816]: audit 2023-10-01T10:30:01.593098+0000 mgr.y (mgr.25072) 300 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:30:03.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:03 smithi173 bash[58816]: cephadm 2023-10-01T10:30:01.598020+0000 mgr.y (mgr.25072) 301 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:30:04.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:04 smithi078 bash[78274]: cluster 2023-10-01T10:30:03.430896+0000 mgr.y (mgr.25072) 302 : cluster 0 pgmap v235: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 555 op/s; 12231/198936 objects degraded (6.148%); 720 KiB/s, 11 objects/s recovering 2023-10-01T10:30:04.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:04 smithi078 bash[78274]: audit 2023-10-01T10:30:04.134989+0000 mon.c (mon.2) 28 : audit 0 from='client.? 172.21.15.78:0/3365892599' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:04.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:04 smithi078 bash[75299]: cluster 2023-10-01T10:30:03.430896+0000 mgr.y (mgr.25072) 302 : cluster 0 pgmap v235: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 555 op/s; 12231/198936 objects degraded (6.148%); 720 KiB/s, 11 objects/s recovering 2023-10-01T10:30:04.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:04 smithi078 bash[75299]: audit 2023-10-01T10:30:04.134989+0000 mon.c (mon.2) 28 : audit 0 from='client.? 172.21.15.78:0/3365892599' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:04.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:04 smithi173 bash[58816]: cluster 2023-10-01T10:30:03.430896+0000 mgr.y (mgr.25072) 302 : cluster 0 pgmap v235: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 555 op/s; 12231/198936 objects degraded (6.148%); 720 KiB/s, 11 objects/s recovering 2023-10-01T10:30:04.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:04 smithi173 bash[58816]: audit 2023-10-01T10:30:04.134989+0000 mon.c (mon.2) 28 : audit 0 from='client.? 172.21.15.78:0/3365892599' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:06.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:06 smithi078 bash[75299]: cluster 2023-10-01T10:30:05.431710+0000 mgr.y (mgr.25072) 303 : cluster 0 pgmap v236: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 825 op/s; 12162/189216 objects degraded (6.428%); 1.1 MiB/s, 16 objects/s recovering 2023-10-01T10:30:06.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:06 smithi078 bash[78274]: cluster 2023-10-01T10:30:05.431710+0000 mgr.y (mgr.25072) 303 : cluster 0 pgmap v236: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 825 op/s; 12162/189216 objects degraded (6.428%); 1.1 MiB/s, 16 objects/s recovering 2023-10-01T10:30:06.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:06 smithi173 bash[58816]: cluster 2023-10-01T10:30:05.431710+0000 mgr.y (mgr.25072) 303 : cluster 0 pgmap v236: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 825 op/s; 12162/189216 objects degraded (6.428%); 1.1 MiB/s, 16 objects/s recovering 2023-10-01T10:30:07.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:07 smithi078 bash[75299]: audit 2023-10-01T10:30:06.712342+0000 mon.b (mon.1) 12 : audit 0 from='client.? 172.21.15.78:0/1850247395' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:07.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:07 smithi078 bash[75299]: cluster 2023-10-01T10:30:06.794293+0000 mon.a (mon.0) 428 : cluster 3 Health check update: Degraded data redundancy: 12162/189216 objects degraded (6.428%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-01T10:30:07.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:07 smithi078 bash[75299]: cluster 2023-10-01T10:30:07.432234+0000 mgr.y (mgr.25072) 304 : cluster 0 pgmap v237: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 545 op/s; 12162/189216 objects degraded (6.428%); 725 KiB/s, 11 objects/s recovering 2023-10-01T10:30:07.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:07 smithi078 bash[78274]: audit 2023-10-01T10:30:06.712342+0000 mon.b (mon.1) 12 : audit 0 from='client.? 172.21.15.78:0/1850247395' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:07.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:07 smithi078 bash[78274]: cluster 2023-10-01T10:30:06.794293+0000 mon.a (mon.0) 428 : cluster 3 Health check update: Degraded data redundancy: 12162/189216 objects degraded (6.428%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-01T10:30:07.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:07 smithi078 bash[78274]: cluster 2023-10-01T10:30:07.432234+0000 mgr.y (mgr.25072) 304 : cluster 0 pgmap v237: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 545 op/s; 12162/189216 objects degraded (6.428%); 725 KiB/s, 11 objects/s recovering 2023-10-01T10:30:07.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:07 smithi173 bash[58816]: audit 2023-10-01T10:30:06.712342+0000 mon.b (mon.1) 12 : audit 0 from='client.? 172.21.15.78:0/1850247395' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:07.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:07 smithi173 bash[58816]: cluster 2023-10-01T10:30:06.794293+0000 mon.a (mon.0) 428 : cluster 3 Health check update: Degraded data redundancy: 12162/189216 objects degraded (6.428%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-01T10:30:07.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:07 smithi173 bash[58816]: cluster 2023-10-01T10:30:07.432234+0000 mgr.y (mgr.25072) 304 : cluster 0 pgmap v237: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 545 op/s; 12162/189216 objects degraded (6.428%); 725 KiB/s, 11 objects/s recovering 2023-10-01T10:30:09.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:09 smithi173 bash[58816]: audit 2023-10-01T10:30:09.308001+0000 mon.c (mon.2) 29 : audit 0 from='client.? 172.21.15.78:0/2755059000' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:09.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:09 smithi078 bash[75299]: audit 2023-10-01T10:30:09.308001+0000 mon.c (mon.2) 29 : audit 0 from='client.? 172.21.15.78:0/2755059000' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:09.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:09 smithi078 bash[78274]: audit 2023-10-01T10:30:09.308001+0000 mon.c (mon.2) 29 : audit 0 from='client.? 172.21.15.78:0/2755059000' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:10.586 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:10 smithi078 bash[75299]: cluster 2023-10-01T10:30:09.433030+0000 mgr.y (mgr.25072) 305 : cluster 0 pgmap v238: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 12162/182337 objects degraded (6.670%); 725 KiB/s, 11 objects/s recovering 2023-10-01T10:30:10.586 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:30:10 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:30:10] "GET /metrics HTTP/1.1" 200 36214 "" "Prometheus/2.33.4" 2023-10-01T10:30:10.586 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:10 smithi078 bash[78274]: cluster 2023-10-01T10:30:09.433030+0000 mgr.y (mgr.25072) 305 : cluster 0 pgmap v238: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 12162/182337 objects degraded (6.670%); 725 KiB/s, 11 objects/s recovering 2023-10-01T10:30:10.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:10 smithi173 bash[58816]: cluster 2023-10-01T10:30:09.433030+0000 mgr.y (mgr.25072) 305 : cluster 0 pgmap v238: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 12162/182337 objects degraded (6.670%); 725 KiB/s, 11 objects/s recovering 2023-10-01T10:30:12.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:12 smithi078 bash[75299]: cluster 2023-10-01T10:30:11.436671+0000 mgr.y (mgr.25072) 306 : cluster 0 pgmap v239: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 626 op/s; 12094/179433 objects degraded (6.740%); 1.1 MiB/s, 16 objects/s recovering 2023-10-01T10:30:12.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:12 smithi078 bash[75299]: cluster 2023-10-01T10:30:11.795405+0000 mon.a (mon.0) 429 : cluster 3 Health check update: Degraded data redundancy: 12094/179433 objects degraded (6.740%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-01T10:30:12.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:12 smithi078 bash[75299]: audit 2023-10-01T10:30:11.942373+0000 mon.a (mon.0) 430 : audit 0 from='client.? 172.21.15.78:0/4275155281' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:12.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:12 smithi078 bash[78274]: cluster 2023-10-01T10:30:11.436671+0000 mgr.y (mgr.25072) 306 : cluster 0 pgmap v239: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 626 op/s; 12094/179433 objects degraded (6.740%); 1.1 MiB/s, 16 objects/s recovering 2023-10-01T10:30:12.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:12 smithi078 bash[78274]: cluster 2023-10-01T10:30:11.795405+0000 mon.a (mon.0) 429 : cluster 3 Health check update: Degraded data redundancy: 12094/179433 objects degraded (6.740%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-01T10:30:12.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:12 smithi078 bash[78274]: audit 2023-10-01T10:30:11.942373+0000 mon.a (mon.0) 430 : audit 0 from='client.? 172.21.15.78:0/4275155281' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:12.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:12 smithi173 bash[58816]: cluster 2023-10-01T10:30:11.436671+0000 mgr.y (mgr.25072) 306 : cluster 0 pgmap v239: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 626 op/s; 12094/179433 objects degraded (6.740%); 1.1 MiB/s, 16 objects/s recovering 2023-10-01T10:30:12.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:12 smithi173 bash[58816]: cluster 2023-10-01T10:30:11.795405+0000 mon.a (mon.0) 429 : cluster 3 Health check update: Degraded data redundancy: 12094/179433 objects degraded (6.740%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-01T10:30:12.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:12 smithi173 bash[58816]: audit 2023-10-01T10:30:11.942373+0000 mon.a (mon.0) 430 : audit 0 from='client.? 172.21.15.78:0/4275155281' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:14.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:14 smithi078 bash[75299]: cluster 2023-10-01T10:30:13.437209+0000 mgr.y (mgr.25072) 307 : cluster 0 pgmap v240: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 12094/179433 objects degraded (6.740%); 730 KiB/s, 11 objects/s recovering 2023-10-01T10:30:14.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:14 smithi078 bash[78274]: cluster 2023-10-01T10:30:13.437209+0000 mgr.y (mgr.25072) 307 : cluster 0 pgmap v240: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 12094/179433 objects degraded (6.740%); 730 KiB/s, 11 objects/s recovering 2023-10-01T10:30:14.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:14 smithi173 bash[58816]: cluster 2023-10-01T10:30:13.437209+0000 mgr.y (mgr.25072) 307 : cluster 0 pgmap v240: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 12094/179433 objects degraded (6.740%); 730 KiB/s, 11 objects/s recovering 2023-10-01T10:30:15.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:15 smithi078 bash[75299]: audit 2023-10-01T10:30:14.574751+0000 mon.c (mon.2) 30 : audit 0 from='client.? 172.21.15.78:0/1429724487' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:15.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:15 smithi078 bash[78274]: audit 2023-10-01T10:30:14.574751+0000 mon.c (mon.2) 30 : audit 0 from='client.? 172.21.15.78:0/1429724487' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:15.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:15 smithi173 bash[58816]: audit 2023-10-01T10:30:14.574751+0000 mon.c (mon.2) 30 : audit 0 from='client.? 172.21.15.78:0/1429724487' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:16.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:16 smithi078 bash[75299]: cluster 2023-10-01T10:30:15.437876+0000 mgr.y (mgr.25072) 308 : cluster 0 pgmap v241: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 12023/169830 objects degraded (7.079%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T10:30:16.834 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:30:16 smithi078 bash[71214]: debug 2023-10-01T10:30:16.610+0000 7fd9c245e700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:30:16.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:16 smithi078 bash[78274]: cluster 2023-10-01T10:30:15.437876+0000 mgr.y (mgr.25072) 308 : cluster 0 pgmap v241: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 12023/169830 objects degraded (7.079%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T10:30:16.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:16 smithi173 bash[58816]: cluster 2023-10-01T10:30:15.437876+0000 mgr.y (mgr.25072) 308 : cluster 0 pgmap v241: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 12023/169830 objects degraded (7.079%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T10:30:17.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:17 smithi078 bash[75299]: audit 2023-10-01T10:30:16.613374+0000 mon.a (mon.0) 431 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:30:17.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:17 smithi078 bash[75299]: cluster 2023-10-01T10:30:16.796626+0000 mon.a (mon.0) 432 : cluster 3 Health check update: Degraded data redundancy: 12023/169830 objects degraded (7.079%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-01T10:30:17.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:17 smithi078 bash[75299]: audit 2023-10-01T10:30:17.127291+0000 mon.c (mon.2) 31 : audit 0 from='client.? 172.21.15.78:0/4121668308' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:17.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:17 smithi078 bash[75299]: audit 2023-10-01T10:30:17.390295+0000 mon.a (mon.0) 433 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:30:17.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:17 smithi078 bash[78274]: audit 2023-10-01T10:30:16.613374+0000 mon.a (mon.0) 431 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:30:17.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:17 smithi078 bash[78274]: cluster 2023-10-01T10:30:16.796626+0000 mon.a (mon.0) 432 : cluster 3 Health check update: Degraded data redundancy: 12023/169830 objects degraded (7.079%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-01T10:30:17.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:17 smithi078 bash[78274]: audit 2023-10-01T10:30:17.127291+0000 mon.c (mon.2) 31 : audit 0 from='client.? 172.21.15.78:0/4121668308' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:17.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:17 smithi078 bash[78274]: audit 2023-10-01T10:30:17.390295+0000 mon.a (mon.0) 433 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:30:17.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:17 smithi173 bash[58816]: audit 2023-10-01T10:30:16.613374+0000 mon.a (mon.0) 431 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:30:17.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:17 smithi173 bash[58816]: cluster 2023-10-01T10:30:16.796626+0000 mon.a (mon.0) 432 : cluster 3 Health check update: Degraded data redundancy: 12023/169830 objects degraded (7.079%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-01T10:30:17.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:17 smithi173 bash[58816]: audit 2023-10-01T10:30:17.127291+0000 mon.c (mon.2) 31 : audit 0 from='client.? 172.21.15.78:0/4121668308' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:17.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:17 smithi173 bash[58816]: audit 2023-10-01T10:30:17.390295+0000 mon.a (mon.0) 433 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:30:18.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:18 smithi078 bash[75299]: audit 2023-10-01T10:30:16.613630+0000 mgr.y (mgr.25072) 309 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:30:18.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:18 smithi078 bash[75299]: cephadm 2023-10-01T10:30:16.615479+0000 mgr.y (mgr.25072) 310 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:30:18.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:18 smithi078 bash[75299]: cluster 2023-10-01T10:30:17.440671+0000 mgr.y (mgr.25072) 311 : cluster 0 pgmap v242: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 538 op/s; 12023/169830 objects degraded (7.079%); 741 KiB/s, 11 objects/s recovering 2023-10-01T10:30:18.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:18 smithi078 bash[78274]: audit 2023-10-01T10:30:16.613630+0000 mgr.y (mgr.25072) 309 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:30:18.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:18 smithi078 bash[78274]: cephadm 2023-10-01T10:30:16.615479+0000 mgr.y (mgr.25072) 310 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:30:18.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:18 smithi078 bash[78274]: cluster 2023-10-01T10:30:17.440671+0000 mgr.y (mgr.25072) 311 : cluster 0 pgmap v242: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 538 op/s; 12023/169830 objects degraded (7.079%); 741 KiB/s, 11 objects/s recovering 2023-10-01T10:30:18.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:18 smithi173 bash[58816]: audit 2023-10-01T10:30:16.613630+0000 mgr.y (mgr.25072) 309 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:30:18.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:18 smithi173 bash[58816]: cephadm 2023-10-01T10:30:16.615479+0000 mgr.y (mgr.25072) 310 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:30:18.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:18 smithi173 bash[58816]: cluster 2023-10-01T10:30:17.440671+0000 mgr.y (mgr.25072) 311 : cluster 0 pgmap v242: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 538 op/s; 12023/169830 objects degraded (7.079%); 741 KiB/s, 11 objects/s recovering 2023-10-01T10:30:20.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:20 smithi078 bash[75299]: cluster 2023-10-01T10:30:19.445414+0000 mgr.y (mgr.25072) 312 : cluster 0 pgmap v243: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 12023/163290 objects degraded (7.363%); 741 KiB/s, 11 objects/s recovering 2023-10-01T10:30:20.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:20 smithi078 bash[75299]: audit 2023-10-01T10:30:19.697946+0000 mon.c (mon.2) 32 : audit 0 from='client.? 172.21.15.78:0/4177524908' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:20.584 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:30:20 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:30:20] "GET /metrics HTTP/1.1" 200 36218 "" "Prometheus/2.33.4" 2023-10-01T10:30:20.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:20 smithi078 bash[78274]: cluster 2023-10-01T10:30:19.445414+0000 mgr.y (mgr.25072) 312 : cluster 0 pgmap v243: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 12023/163290 objects degraded (7.363%); 741 KiB/s, 11 objects/s recovering 2023-10-01T10:30:20.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:20 smithi078 bash[78274]: audit 2023-10-01T10:30:19.697946+0000 mon.c (mon.2) 32 : audit 0 from='client.? 172.21.15.78:0/4177524908' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:20.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:20 smithi173 bash[58816]: cluster 2023-10-01T10:30:19.445414+0000 mgr.y (mgr.25072) 312 : cluster 0 pgmap v243: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 12023/163290 objects degraded (7.363%); 741 KiB/s, 11 objects/s recovering 2023-10-01T10:30:20.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:20 smithi173 bash[58816]: audit 2023-10-01T10:30:19.697946+0000 mon.c (mon.2) 32 : audit 0 from='client.? 172.21.15.78:0/4177524908' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:22.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:22 smithi078 bash[78274]: cluster 2023-10-01T10:30:21.446128+0000 mgr.y (mgr.25072) 313 : cluster 0 pgmap v244: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 11952/160107 objects degraded (7.465%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T10:30:22.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:22 smithi078 bash[78274]: cluster 2023-10-01T10:30:21.797814+0000 mon.a (mon.0) 434 : cluster 3 Health check update: Degraded data redundancy: 11952/160107 objects degraded (7.465%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-01T10:30:22.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:22 smithi078 bash[78274]: audit 2023-10-01T10:30:22.296297+0000 mon.b (mon.1) 13 : audit 0 from='client.? 172.21.15.78:0/1654176378' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:22.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:22 smithi078 bash[75299]: cluster 2023-10-01T10:30:21.446128+0000 mgr.y (mgr.25072) 313 : cluster 0 pgmap v244: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 11952/160107 objects degraded (7.465%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T10:30:22.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:22 smithi078 bash[75299]: cluster 2023-10-01T10:30:21.797814+0000 mon.a (mon.0) 434 : cluster 3 Health check update: Degraded data redundancy: 11952/160107 objects degraded (7.465%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-01T10:30:22.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:22 smithi078 bash[75299]: audit 2023-10-01T10:30:22.296297+0000 mon.b (mon.1) 13 : audit 0 from='client.? 172.21.15.78:0/1654176378' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:22.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:22 smithi173 bash[58816]: cluster 2023-10-01T10:30:21.446128+0000 mgr.y (mgr.25072) 313 : cluster 0 pgmap v244: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 11952/160107 objects degraded (7.465%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T10:30:22.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:22 smithi173 bash[58816]: cluster 2023-10-01T10:30:21.797814+0000 mon.a (mon.0) 434 : cluster 3 Health check update: Degraded data redundancy: 11952/160107 objects degraded (7.465%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-01T10:30:22.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:22 smithi173 bash[58816]: audit 2023-10-01T10:30:22.296297+0000 mon.b (mon.1) 13 : audit 0 from='client.? 172.21.15.78:0/1654176378' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:24.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:24 smithi078 bash[75299]: cluster 2023-10-01T10:30:23.449418+0000 mgr.y (mgr.25072) 314 : cluster 0 pgmap v245: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 536 op/s; 11952/160107 objects degraded (7.465%); 757 KiB/s, 11 objects/s recovering 2023-10-01T10:30:24.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:24 smithi078 bash[78274]: cluster 2023-10-01T10:30:23.449418+0000 mgr.y (mgr.25072) 314 : cluster 0 pgmap v245: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 536 op/s; 11952/160107 objects degraded (7.465%); 757 KiB/s, 11 objects/s recovering 2023-10-01T10:30:24.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:24 smithi173 bash[58816]: cluster 2023-10-01T10:30:23.449418+0000 mgr.y (mgr.25072) 314 : cluster 0 pgmap v245: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 536 op/s; 11952/160107 objects degraded (7.465%); 757 KiB/s, 11 objects/s recovering 2023-10-01T10:30:25.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:25 smithi078 bash[75299]: audit 2023-10-01T10:30:24.890260+0000 mon.c (mon.2) 33 : audit 0 from='client.? 172.21.15.78:0/3693186158' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:25.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:25 smithi078 bash[78274]: audit 2023-10-01T10:30:24.890260+0000 mon.c (mon.2) 33 : audit 0 from='client.? 172.21.15.78:0/3693186158' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:25.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:25 smithi173 bash[58816]: audit 2023-10-01T10:30:24.890260+0000 mon.c (mon.2) 33 : audit 0 from='client.? 172.21.15.78:0/3693186158' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:26.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:26 smithi078 bash[75299]: cluster 2023-10-01T10:30:25.450240+0000 mgr.y (mgr.25072) 315 : cluster 0 pgmap v246: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 807 op/s; 11881/150348 objects degraded (7.902%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T10:30:26.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:26 smithi078 bash[78274]: cluster 2023-10-01T10:30:25.450240+0000 mgr.y (mgr.25072) 315 : cluster 0 pgmap v246: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 807 op/s; 11881/150348 objects degraded (7.902%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T10:30:26.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:26 smithi173 bash[58816]: cluster 2023-10-01T10:30:25.450240+0000 mgr.y (mgr.25072) 315 : cluster 0 pgmap v246: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 807 op/s; 11881/150348 objects degraded (7.902%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T10:30:27.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:27 smithi078 bash[75299]: cluster 2023-10-01T10:30:26.798981+0000 mon.a (mon.0) 435 : cluster 3 Health check update: Degraded data redundancy: 11881/150348 objects degraded (7.902%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-01T10:30:27.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:27 smithi078 bash[75299]: audit 2023-10-01T10:30:27.501070+0000 mon.b (mon.1) 14 : audit 0 from='client.? 172.21.15.78:0/3536932488' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:27.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:27 smithi078 bash[78274]: cluster 2023-10-01T10:30:26.798981+0000 mon.a (mon.0) 435 : cluster 3 Health check update: Degraded data redundancy: 11881/150348 objects degraded (7.902%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-01T10:30:27.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:27 smithi078 bash[78274]: audit 2023-10-01T10:30:27.501070+0000 mon.b (mon.1) 14 : audit 0 from='client.? 172.21.15.78:0/3536932488' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:27.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:27 smithi173 bash[58816]: cluster 2023-10-01T10:30:26.798981+0000 mon.a (mon.0) 435 : cluster 3 Health check update: Degraded data redundancy: 11881/150348 objects degraded (7.902%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-01T10:30:27.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:27 smithi173 bash[58816]: audit 2023-10-01T10:30:27.501070+0000 mon.b (mon.1) 14 : audit 0 from='client.? 172.21.15.78:0/3536932488' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:28.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:28 smithi078 bash[75299]: cluster 2023-10-01T10:30:27.450775+0000 mgr.y (mgr.25072) 316 : cluster 0 pgmap v247: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 11881/150348 objects degraded (7.902%); 757 KiB/s, 11 objects/s recovering 2023-10-01T10:30:28.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:28 smithi078 bash[78274]: cluster 2023-10-01T10:30:27.450775+0000 mgr.y (mgr.25072) 316 : cluster 0 pgmap v247: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 11881/150348 objects degraded (7.902%); 757 KiB/s, 11 objects/s recovering 2023-10-01T10:30:28.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:28 smithi173 bash[58816]: cluster 2023-10-01T10:30:27.450775+0000 mgr.y (mgr.25072) 316 : cluster 0 pgmap v247: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 11881/150348 objects degraded (7.902%); 757 KiB/s, 11 objects/s recovering 2023-10-01T10:30:29.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:29 smithi173 bash[58816]: cluster 2023-10-01T10:30:29.451592+0000 mgr.y (mgr.25072) 317 : cluster 0 pgmap v248: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 11881/143706 objects degraded (8.268%); 757 KiB/s, 11 objects/s recovering 2023-10-01T10:30:30.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:29 smithi078 bash[78274]: cluster 2023-10-01T10:30:29.451592+0000 mgr.y (mgr.25072) 317 : cluster 0 pgmap v248: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 11881/143706 objects degraded (8.268%); 757 KiB/s, 11 objects/s recovering 2023-10-01T10:30:30.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:29 smithi078 bash[75299]: cluster 2023-10-01T10:30:29.451592+0000 mgr.y (mgr.25072) 317 : cluster 0 pgmap v248: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 11881/143706 objects degraded (8.268%); 757 KiB/s, 11 objects/s recovering 2023-10-01T10:30:30.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:30:30 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:30:30] "GET /metrics HTTP/1.1" 200 36218 "" "Prometheus/2.33.4" 2023-10-01T10:30:30.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:30 smithi173 bash[58816]: audit 2023-10-01T10:30:30.120900+0000 mon.c (mon.2) 34 : audit 0 from='client.? 172.21.15.78:0/2299777733' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:31.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:30 smithi078 bash[75299]: audit 2023-10-01T10:30:30.120900+0000 mon.c (mon.2) 34 : audit 0 from='client.? 172.21.15.78:0/2299777733' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:31.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:30 smithi078 bash[78274]: audit 2023-10-01T10:30:30.120900+0000 mon.c (mon.2) 34 : audit 0 from='client.? 172.21.15.78:0/2299777733' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:32.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:31 smithi078 bash[75299]: cluster 2023-10-01T10:30:31.452214+0000 mgr.y (mgr.25072) 318 : cluster 0 pgmap v249: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 11809/140640 objects degraded (8.397%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T10:30:32.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:31 smithi078 bash[78274]: cluster 2023-10-01T10:30:31.452214+0000 mgr.y (mgr.25072) 318 : cluster 0 pgmap v249: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 11809/140640 objects degraded (8.397%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T10:30:32.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:31 smithi173 bash[58816]: cluster 2023-10-01T10:30:31.452214+0000 mgr.y (mgr.25072) 318 : cluster 0 pgmap v249: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 11809/140640 objects degraded (8.397%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T10:30:33.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:32 smithi078 bash[75299]: audit 2023-10-01T10:30:31.632958+0000 mon.a (mon.0) 436 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:30:33.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:32 smithi078 bash[75299]: cluster 2023-10-01T10:30:31.799986+0000 mon.a (mon.0) 437 : cluster 3 Health check update: Degraded data redundancy: 11809/140640 objects degraded (8.397%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-01T10:30:33.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:32 smithi078 bash[75299]: audit 2023-10-01T10:30:32.390784+0000 mon.a (mon.0) 438 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:30:33.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:32 smithi078 bash[78274]: audit 2023-10-01T10:30:31.632958+0000 mon.a (mon.0) 436 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:30:33.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:32 smithi078 bash[78274]: cluster 2023-10-01T10:30:31.799986+0000 mon.a (mon.0) 437 : cluster 3 Health check update: Degraded data redundancy: 11809/140640 objects degraded (8.397%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-01T10:30:33.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:32 smithi078 bash[78274]: audit 2023-10-01T10:30:32.390784+0000 mon.a (mon.0) 438 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:30:33.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:32 smithi173 bash[58816]: audit 2023-10-01T10:30:31.632958+0000 mon.a (mon.0) 436 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:30:33.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:32 smithi173 bash[58816]: cluster 2023-10-01T10:30:31.799986+0000 mon.a (mon.0) 437 : cluster 3 Health check update: Degraded data redundancy: 11809/140640 objects degraded (8.397%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-01T10:30:33.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:32 smithi173 bash[58816]: audit 2023-10-01T10:30:32.390784+0000 mon.a (mon.0) 438 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:30:34.085 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:33 smithi078 bash[75299]: audit 2023-10-01T10:30:32.738746+0000 mon.c (mon.2) 35 : audit 0 from='client.? 172.21.15.78:0/358277845' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:34.085 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:33 smithi078 bash[75299]: cluster 2023-10-01T10:30:33.452751+0000 mgr.y (mgr.25072) 319 : cluster 0 pgmap v250: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 11809/140640 objects degraded (8.397%); 762 KiB/s, 11 objects/s recovering 2023-10-01T10:30:34.085 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:33 smithi078 bash[78274]: audit 2023-10-01T10:30:32.738746+0000 mon.c (mon.2) 35 : audit 0 from='client.? 172.21.15.78:0/358277845' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:34.085 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:33 smithi078 bash[78274]: cluster 2023-10-01T10:30:33.452751+0000 mgr.y (mgr.25072) 319 : cluster 0 pgmap v250: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 11809/140640 objects degraded (8.397%); 762 KiB/s, 11 objects/s recovering 2023-10-01T10:30:34.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:33 smithi173 bash[58816]: audit 2023-10-01T10:30:32.738746+0000 mon.c (mon.2) 35 : audit 0 from='client.? 172.21.15.78:0/358277845' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:34.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:33 smithi173 bash[58816]: cluster 2023-10-01T10:30:33.452751+0000 mgr.y (mgr.25072) 319 : cluster 0 pgmap v250: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 11809/140640 objects degraded (8.397%); 762 KiB/s, 11 objects/s recovering 2023-10-01T10:30:35.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:35 smithi078 bash[75299]: audit 2023-10-01T10:30:35.352770+0000 mon.a (mon.0) 439 : audit 0 from='client.? 172.21.15.78:0/1134111603' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:35.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:35 smithi078 bash[78274]: audit 2023-10-01T10:30:35.352770+0000 mon.a (mon.0) 439 : audit 0 from='client.? 172.21.15.78:0/1134111603' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:35.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:35 smithi173 bash[58816]: audit 2023-10-01T10:30:35.352770+0000 mon.a (mon.0) 439 : audit 0 from='client.? 172.21.15.78:0/1134111603' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:36.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:36 smithi078 bash[75299]: cluster 2023-10-01T10:30:35.453599+0000 mgr.y (mgr.25072) 320 : cluster 0 pgmap v251: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 801 op/s; 11739/131247 objects degraded (8.944%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T10:30:36.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:36 smithi078 bash[78274]: cluster 2023-10-01T10:30:35.453599+0000 mgr.y (mgr.25072) 320 : cluster 0 pgmap v251: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 801 op/s; 11739/131247 objects degraded (8.944%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T10:30:36.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:36 smithi173 bash[58816]: cluster 2023-10-01T10:30:35.453599+0000 mgr.y (mgr.25072) 320 : cluster 0 pgmap v251: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 801 op/s; 11739/131247 objects degraded (8.944%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T10:30:37.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:37 smithi078 bash[75299]: cluster 2023-10-01T10:30:36.804075+0000 mon.a (mon.0) 440 : cluster 3 Health check update: Degraded data redundancy: 11739/131247 objects degraded (8.944%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-01T10:30:37.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:37 smithi078 bash[78274]: cluster 2023-10-01T10:30:36.804075+0000 mon.a (mon.0) 440 : cluster 3 Health check update: Degraded data redundancy: 11739/131247 objects degraded (8.944%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-01T10:30:37.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:37 smithi173 bash[58816]: cluster 2023-10-01T10:30:36.804075+0000 mon.a (mon.0) 440 : cluster 3 Health check update: Degraded data redundancy: 11739/131247 objects degraded (8.944%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-01T10:30:38.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:38 smithi078 bash[78274]: cluster 2023-10-01T10:30:37.454136+0000 mgr.y (mgr.25072) 321 : cluster 0 pgmap v252: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 530 op/s; 11739/131247 objects degraded (8.944%); 757 KiB/s, 11 objects/s recovering 2023-10-01T10:30:38.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:38 smithi078 bash[78274]: audit 2023-10-01T10:30:37.985918+0000 mon.a (mon.0) 441 : audit 0 from='client.? 172.21.15.78:0/965145260' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:38.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:38 smithi078 bash[78274]: audit 2023-10-01T10:30:38.001316+0000 mon.a (mon.0) 442 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:30:38.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:38 smithi078 bash[78274]: audit 2023-10-01T10:30:38.011710+0000 mon.a (mon.0) 443 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:30:38.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:38 smithi078 bash[78274]: audit 2023-10-01T10:30:38.192092+0000 mon.a (mon.0) 444 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:30:38.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:38 smithi078 bash[78274]: audit 2023-10-01T10:30:38.201152+0000 mon.a (mon.0) 445 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:30:38.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:38 smithi078 bash[75299]: cluster 2023-10-01T10:30:37.454136+0000 mgr.y (mgr.25072) 321 : cluster 0 pgmap v252: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 530 op/s; 11739/131247 objects degraded (8.944%); 757 KiB/s, 11 objects/s recovering 2023-10-01T10:30:38.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:38 smithi078 bash[75299]: audit 2023-10-01T10:30:37.985918+0000 mon.a (mon.0) 441 : audit 0 from='client.? 172.21.15.78:0/965145260' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:38.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:38 smithi078 bash[75299]: audit 2023-10-01T10:30:38.001316+0000 mon.a (mon.0) 442 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:30:38.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:38 smithi078 bash[75299]: audit 2023-10-01T10:30:38.011710+0000 mon.a (mon.0) 443 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:30:38.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:38 smithi078 bash[75299]: audit 2023-10-01T10:30:38.192092+0000 mon.a (mon.0) 444 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:30:38.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:38 smithi078 bash[75299]: audit 2023-10-01T10:30:38.201152+0000 mon.a (mon.0) 445 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:30:38.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:38 smithi173 bash[58816]: cluster 2023-10-01T10:30:37.454136+0000 mgr.y (mgr.25072) 321 : cluster 0 pgmap v252: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 530 op/s; 11739/131247 objects degraded (8.944%); 757 KiB/s, 11 objects/s recovering 2023-10-01T10:30:38.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:38 smithi173 bash[58816]: audit 2023-10-01T10:30:37.985918+0000 mon.a (mon.0) 441 : audit 0 from='client.? 172.21.15.78:0/965145260' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:38.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:38 smithi173 bash[58816]: audit 2023-10-01T10:30:38.001316+0000 mon.a (mon.0) 442 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:30:38.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:38 smithi173 bash[58816]: audit 2023-10-01T10:30:38.011710+0000 mon.a (mon.0) 443 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:30:38.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:38 smithi173 bash[58816]: audit 2023-10-01T10:30:38.192092+0000 mon.a (mon.0) 444 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:30:38.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:38 smithi173 bash[58816]: audit 2023-10-01T10:30:38.201152+0000 mon.a (mon.0) 445 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:30:39.333 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:30:38 smithi078 bash[71214]: debug 2023-10-01T10:30:38.945+0000 7fd9c245e700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:30:39.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:39 smithi078 bash[78274]: audit 2023-10-01T10:30:38.836076+0000 mon.a (mon.0) 446 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:30:39.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:39 smithi078 bash[78274]: audit 2023-10-01T10:30:38.837217+0000 mon.a (mon.0) 447 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:30:39.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:39 smithi078 bash[78274]: audit 2023-10-01T10:30:38.845105+0000 mon.a (mon.0) 448 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:30:39.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:39 smithi078 bash[78274]: audit 2023-10-01T10:30:38.937746+0000 mon.a (mon.0) 449 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:30:39.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:39 smithi078 bash[78274]: audit 2023-10-01T10:30:38.942574+0000 mon.a (mon.0) 450 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:30:39.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:39 smithi078 bash[78274]: audit 2023-10-01T10:30:38.944969+0000 mon.a (mon.0) 451 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:30:39.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:39 smithi078 bash[78274]: audit 2023-10-01T10:30:38.947031+0000 mon.a (mon.0) 452 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:30:39.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:39 smithi078 bash[78274]: audit 2023-10-01T10:30:38.949970+0000 mon.a (mon.0) 453 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:30:39.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:39 smithi078 bash[75299]: audit 2023-10-01T10:30:38.836076+0000 mon.a (mon.0) 446 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:30:39.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:39 smithi078 bash[75299]: audit 2023-10-01T10:30:38.837217+0000 mon.a (mon.0) 447 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:30:39.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:39 smithi078 bash[75299]: audit 2023-10-01T10:30:38.845105+0000 mon.a (mon.0) 448 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:30:39.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:39 smithi078 bash[75299]: audit 2023-10-01T10:30:38.937746+0000 mon.a (mon.0) 449 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:30:39.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:39 smithi078 bash[75299]: audit 2023-10-01T10:30:38.942574+0000 mon.a (mon.0) 450 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:30:39.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:39 smithi078 bash[75299]: audit 2023-10-01T10:30:38.944969+0000 mon.a (mon.0) 451 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:30:39.836 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:39 smithi078 bash[75299]: audit 2023-10-01T10:30:38.947031+0000 mon.a (mon.0) 452 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:30:39.836 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:39 smithi078 bash[75299]: audit 2023-10-01T10:30:38.949970+0000 mon.a (mon.0) 453 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:30:39.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:39 smithi173 bash[58816]: audit 2023-10-01T10:30:38.836076+0000 mon.a (mon.0) 446 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:30:39.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:39 smithi173 bash[58816]: audit 2023-10-01T10:30:38.837217+0000 mon.a (mon.0) 447 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:30:39.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:39 smithi173 bash[58816]: audit 2023-10-01T10:30:38.845105+0000 mon.a (mon.0) 448 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:30:39.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:39 smithi173 bash[58816]: audit 2023-10-01T10:30:38.937746+0000 mon.a (mon.0) 449 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:30:39.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:39 smithi173 bash[58816]: audit 2023-10-01T10:30:38.942574+0000 mon.a (mon.0) 450 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:30:39.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:39 smithi173 bash[58816]: audit 2023-10-01T10:30:38.944969+0000 mon.a (mon.0) 451 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:30:39.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:39 smithi173 bash[58816]: audit 2023-10-01T10:30:38.947031+0000 mon.a (mon.0) 452 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:30:39.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:39 smithi173 bash[58816]: audit 2023-10-01T10:30:38.949970+0000 mon.a (mon.0) 453 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:30:40.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:40 smithi078 bash[78274]: audit 2023-10-01T10:30:38.950189+0000 mgr.y (mgr.25072) 322 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:30:40.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:40 smithi078 bash[78274]: cephadm 2023-10-01T10:30:38.952343+0000 mgr.y (mgr.25072) 323 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:30:40.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:40 smithi078 bash[78274]: cluster 2023-10-01T10:30:39.454995+0000 mgr.y (mgr.25072) 324 : cluster 0 pgmap v253: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 11739/125070 objects degraded (9.386%); 757 KiB/s, 11 objects/s recovering 2023-10-01T10:30:40.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:40 smithi078 bash[75299]: audit 2023-10-01T10:30:38.950189+0000 mgr.y (mgr.25072) 322 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:30:40.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:40 smithi078 bash[75299]: cephadm 2023-10-01T10:30:38.952343+0000 mgr.y (mgr.25072) 323 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:30:40.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:40 smithi078 bash[75299]: cluster 2023-10-01T10:30:39.454995+0000 mgr.y (mgr.25072) 324 : cluster 0 pgmap v253: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 11739/125070 objects degraded (9.386%); 757 KiB/s, 11 objects/s recovering 2023-10-01T10:30:40.584 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:30:40 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:30:40] "GET /metrics HTTP/1.1" 200 36212 "" "Prometheus/2.33.4" 2023-10-01T10:30:40.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:40 smithi173 bash[58816]: audit 2023-10-01T10:30:38.950189+0000 mgr.y (mgr.25072) 322 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:30:40.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:40 smithi173 bash[58816]: cephadm 2023-10-01T10:30:38.952343+0000 mgr.y (mgr.25072) 323 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:30:40.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:40 smithi173 bash[58816]: cluster 2023-10-01T10:30:39.454995+0000 mgr.y (mgr.25072) 324 : cluster 0 pgmap v253: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 11739/125070 objects degraded (9.386%); 757 KiB/s, 11 objects/s recovering 2023-10-01T10:30:41.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:41 smithi078 bash[75299]: audit 2023-10-01T10:30:40.570355+0000 mon.a (mon.0) 454 : audit 0 from='client.? 172.21.15.78:0/1055892487' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:41.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:41 smithi078 bash[78274]: audit 2023-10-01T10:30:40.570355+0000 mon.a (mon.0) 454 : audit 0 from='client.? 172.21.15.78:0/1055892487' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:41.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:41 smithi173 bash[58816]: audit 2023-10-01T10:30:40.570355+0000 mon.a (mon.0) 454 : audit 0 from='client.? 172.21.15.78:0/1055892487' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:42.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:42 smithi078 bash[78274]: cluster 2023-10-01T10:30:41.455627+0000 mgr.y (mgr.25072) 325 : cluster 0 pgmap v254: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.5 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 11668/122295 objects degraded (9.541%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T10:30:42.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:42 smithi078 bash[78274]: cluster 2023-10-01T10:30:41.809260+0000 mon.a (mon.0) 455 : cluster 3 Health check update: Degraded data redundancy: 11668/122295 objects degraded (9.541%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-01T10:30:42.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:42 smithi078 bash[75299]: cluster 2023-10-01T10:30:41.455627+0000 mgr.y (mgr.25072) 325 : cluster 0 pgmap v254: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.5 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 11668/122295 objects degraded (9.541%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T10:30:42.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:42 smithi078 bash[75299]: cluster 2023-10-01T10:30:41.809260+0000 mon.a (mon.0) 455 : cluster 3 Health check update: Degraded data redundancy: 11668/122295 objects degraded (9.541%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-01T10:30:42.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:42 smithi173 bash[58816]: cluster 2023-10-01T10:30:41.455627+0000 mgr.y (mgr.25072) 325 : cluster 0 pgmap v254: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.5 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 11668/122295 objects degraded (9.541%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T10:30:42.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:42 smithi173 bash[58816]: cluster 2023-10-01T10:30:41.809260+0000 mon.a (mon.0) 455 : cluster 3 Health check update: Degraded data redundancy: 11668/122295 objects degraded (9.541%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-01T10:30:43.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:43 smithi078 bash[75299]: audit 2023-10-01T10:30:43.145316+0000 mon.b (mon.1) 15 : audit 0 from='client.? 172.21.15.78:0/787826860' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:43.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:43 smithi078 bash[78274]: audit 2023-10-01T10:30:43.145316+0000 mon.b (mon.1) 15 : audit 0 from='client.? 172.21.15.78:0/787826860' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:43.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:43 smithi173 bash[58816]: audit 2023-10-01T10:30:43.145316+0000 mon.b (mon.1) 15 : audit 0 from='client.? 172.21.15.78:0/787826860' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:44.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:44 smithi078 bash[75299]: cluster 2023-10-01T10:30:43.456213+0000 mgr.y (mgr.25072) 326 : cluster 0 pgmap v255: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.5 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 509 op/s; 11668/122295 objects degraded (9.541%); 752 KiB/s, 11 objects/s recovering 2023-10-01T10:30:44.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:44 smithi078 bash[78274]: cluster 2023-10-01T10:30:43.456213+0000 mgr.y (mgr.25072) 326 : cluster 0 pgmap v255: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.5 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 509 op/s; 11668/122295 objects degraded (9.541%); 752 KiB/s, 11 objects/s recovering 2023-10-01T10:30:44.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:44 smithi173 bash[58816]: cluster 2023-10-01T10:30:43.456213+0000 mgr.y (mgr.25072) 326 : cluster 0 pgmap v255: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.5 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 509 op/s; 11668/122295 objects degraded (9.541%); 752 KiB/s, 11 objects/s recovering 2023-10-01T10:30:46.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:46 smithi078 bash[75299]: cluster 2023-10-01T10:30:45.457063+0000 mgr.y (mgr.25072) 327 : cluster 0 pgmap v256: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.3 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 776 op/s; 11598/112677 objects degraded (10.293%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T10:30:46.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:46 smithi078 bash[75299]: audit 2023-10-01T10:30:45.737982+0000 mon.c (mon.2) 36 : audit 0 from='client.? 172.21.15.78:0/1652423999' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:46.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:46 smithi078 bash[78274]: cluster 2023-10-01T10:30:45.457063+0000 mgr.y (mgr.25072) 327 : cluster 0 pgmap v256: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.3 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 776 op/s; 11598/112677 objects degraded (10.293%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T10:30:46.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:46 smithi078 bash[78274]: audit 2023-10-01T10:30:45.737982+0000 mon.c (mon.2) 36 : audit 0 from='client.? 172.21.15.78:0/1652423999' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:46.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:46 smithi173 bash[58816]: cluster 2023-10-01T10:30:45.457063+0000 mgr.y (mgr.25072) 327 : cluster 0 pgmap v256: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.3 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 776 op/s; 11598/112677 objects degraded (10.293%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T10:30:46.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:46 smithi173 bash[58816]: audit 2023-10-01T10:30:45.737982+0000 mon.c (mon.2) 36 : audit 0 from='client.? 172.21.15.78:0/1652423999' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:47.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:47 smithi078 bash[75299]: cluster 2023-10-01T10:30:46.810766+0000 mon.a (mon.0) 456 : cluster 3 Health check update: Degraded data redundancy: 11598/112677 objects degraded (10.293%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-01T10:30:47.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:47 smithi078 bash[75299]: audit 2023-10-01T10:30:47.393665+0000 mon.a (mon.0) 457 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:30:47.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:47 smithi078 bash[78274]: cluster 2023-10-01T10:30:46.810766+0000 mon.a (mon.0) 456 : cluster 3 Health check update: Degraded data redundancy: 11598/112677 objects degraded (10.293%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-01T10:30:47.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:47 smithi078 bash[78274]: audit 2023-10-01T10:30:47.393665+0000 mon.a (mon.0) 457 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:30:47.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:47 smithi173 bash[58816]: cluster 2023-10-01T10:30:46.810766+0000 mon.a (mon.0) 456 : cluster 3 Health check update: Degraded data redundancy: 11598/112677 objects degraded (10.293%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-01T10:30:47.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:47 smithi173 bash[58816]: audit 2023-10-01T10:30:47.393665+0000 mon.a (mon.0) 457 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:30:48.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:48 smithi078 bash[75299]: cluster 2023-10-01T10:30:47.457623+0000 mgr.y (mgr.25072) 328 : cluster 0 pgmap v257: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.3 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 515 op/s; 11598/112677 objects degraded (10.293%); 752 KiB/s, 11 objects/s recovering 2023-10-01T10:30:48.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:48 smithi078 bash[75299]: audit 2023-10-01T10:30:48.344755+0000 mon.c (mon.2) 37 : audit 0 from='client.? 172.21.15.78:0/3486621458' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:48.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:48 smithi078 bash[78274]: cluster 2023-10-01T10:30:47.457623+0000 mgr.y (mgr.25072) 328 : cluster 0 pgmap v257: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.3 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 515 op/s; 11598/112677 objects degraded (10.293%); 752 KiB/s, 11 objects/s recovering 2023-10-01T10:30:48.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:48 smithi078 bash[78274]: audit 2023-10-01T10:30:48.344755+0000 mon.c (mon.2) 37 : audit 0 from='client.? 172.21.15.78:0/3486621458' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:48.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:48 smithi173 bash[58816]: cluster 2023-10-01T10:30:47.457623+0000 mgr.y (mgr.25072) 328 : cluster 0 pgmap v257: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.3 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 515 op/s; 11598/112677 objects degraded (10.293%); 752 KiB/s, 11 objects/s recovering 2023-10-01T10:30:48.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:48 smithi173 bash[58816]: audit 2023-10-01T10:30:48.344755+0000 mon.c (mon.2) 37 : audit 0 from='client.? 172.21.15.78:0/3486621458' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:50.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:50 smithi078 bash[75299]: cluster 2023-10-01T10:30:49.458436+0000 mgr.y (mgr.25072) 329 : cluster 0 pgmap v258: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.2 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 698 op/s; 11598/106098 objects degraded (10.931%); 752 KiB/s, 11 objects/s recovering 2023-10-01T10:30:50.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:30:50 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:30:50] "GET /metrics HTTP/1.1" 200 36212 "" "Prometheus/2.33.4" 2023-10-01T10:30:50.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:50 smithi078 bash[78274]: cluster 2023-10-01T10:30:49.458436+0000 mgr.y (mgr.25072) 329 : cluster 0 pgmap v258: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.2 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 698 op/s; 11598/106098 objects degraded (10.931%); 752 KiB/s, 11 objects/s recovering 2023-10-01T10:30:50.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:50 smithi173 bash[58816]: cluster 2023-10-01T10:30:49.458436+0000 mgr.y (mgr.25072) 329 : cluster 0 pgmap v258: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.2 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 698 op/s; 11598/106098 objects degraded (10.931%); 752 KiB/s, 11 objects/s recovering 2023-10-01T10:30:51.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:51 smithi078 bash[75299]: audit 2023-10-01T10:30:50.957562+0000 mon.b (mon.1) 16 : audit 0 from='client.? 172.21.15.78:0/361908901' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:51.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:51 smithi078 bash[78274]: audit 2023-10-01T10:30:50.957562+0000 mon.b (mon.1) 16 : audit 0 from='client.? 172.21.15.78:0/361908901' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:51.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:51 smithi173 bash[58816]: audit 2023-10-01T10:30:50.957562+0000 mon.b (mon.1) 16 : audit 0 from='client.? 172.21.15.78:0/361908901' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:52.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:52 smithi078 bash[75299]: cluster 2023-10-01T10:30:51.459035+0000 mgr.y (mgr.25072) 330 : cluster 0 pgmap v259: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.1 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 609 op/s; 11523/103110 objects degraded (11.175%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T10:30:52.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:52 smithi078 bash[75299]: cluster 2023-10-01T10:30:51.811957+0000 mon.a (mon.0) 458 : cluster 3 Health check update: Degraded data redundancy: 11523/103110 objects degraded (11.175%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-01T10:30:52.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:52 smithi078 bash[78274]: cluster 2023-10-01T10:30:51.459035+0000 mgr.y (mgr.25072) 330 : cluster 0 pgmap v259: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.1 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 609 op/s; 11523/103110 objects degraded (11.175%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T10:30:52.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:52 smithi078 bash[78274]: cluster 2023-10-01T10:30:51.811957+0000 mon.a (mon.0) 458 : cluster 3 Health check update: Degraded data redundancy: 11523/103110 objects degraded (11.175%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-01T10:30:52.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:52 smithi173 bash[58816]: cluster 2023-10-01T10:30:51.459035+0000 mgr.y (mgr.25072) 330 : cluster 0 pgmap v259: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.1 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 609 op/s; 11523/103110 objects degraded (11.175%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T10:30:52.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:52 smithi173 bash[58816]: cluster 2023-10-01T10:30:51.811957+0000 mon.a (mon.0) 458 : cluster 3 Health check update: Degraded data redundancy: 11523/103110 objects degraded (11.175%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-01T10:30:54.333 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:30:53 smithi078 bash[71214]: debug 2023-10-01T10:30:53.965+0000 7fd9c245e700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:30:54.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:54 smithi078 bash[75299]: cluster 2023-10-01T10:30:53.459562+0000 mgr.y (mgr.25072) 331 : cluster 0 pgmap v260: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.1 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 532 op/s; 11523/103110 objects degraded (11.175%); 773 KiB/s, 12 objects/s recovering 2023-10-01T10:30:54.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:54 smithi078 bash[75299]: audit 2023-10-01T10:30:53.572542+0000 mon.c (mon.2) 38 : audit 0 from='client.? 172.21.15.78:0/861763552' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:54.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:54 smithi078 bash[75299]: audit 2023-10-01T10:30:53.969002+0000 mon.a (mon.0) 459 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:30:54.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:54 smithi078 bash[78274]: cluster 2023-10-01T10:30:53.459562+0000 mgr.y (mgr.25072) 331 : cluster 0 pgmap v260: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.1 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 532 op/s; 11523/103110 objects degraded (11.175%); 773 KiB/s, 12 objects/s recovering 2023-10-01T10:30:54.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:54 smithi078 bash[78274]: audit 2023-10-01T10:30:53.572542+0000 mon.c (mon.2) 38 : audit 0 from='client.? 172.21.15.78:0/861763552' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:54.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:54 smithi078 bash[78274]: audit 2023-10-01T10:30:53.969002+0000 mon.a (mon.0) 459 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:30:54.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:54 smithi173 bash[58816]: cluster 2023-10-01T10:30:53.459562+0000 mgr.y (mgr.25072) 331 : cluster 0 pgmap v260: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.1 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 532 op/s; 11523/103110 objects degraded (11.175%); 773 KiB/s, 12 objects/s recovering 2023-10-01T10:30:54.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:54 smithi173 bash[58816]: audit 2023-10-01T10:30:53.572542+0000 mon.c (mon.2) 38 : audit 0 from='client.? 172.21.15.78:0/861763552' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:54.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:54 smithi173 bash[58816]: audit 2023-10-01T10:30:53.969002+0000 mon.a (mon.0) 459 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:30:55.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:55 smithi078 bash[75299]: audit 2023-10-01T10:30:53.969315+0000 mgr.y (mgr.25072) 332 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:30:55.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:55 smithi078 bash[75299]: cephadm 2023-10-01T10:30:53.970776+0000 mgr.y (mgr.25072) 333 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:30:55.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:55 smithi078 bash[78274]: audit 2023-10-01T10:30:53.969315+0000 mgr.y (mgr.25072) 332 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:30:55.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:55 smithi078 bash[78274]: cephadm 2023-10-01T10:30:53.970776+0000 mgr.y (mgr.25072) 333 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:30:55.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:55 smithi173 bash[58816]: audit 2023-10-01T10:30:53.969315+0000 mgr.y (mgr.25072) 332 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:30:55.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:55 smithi173 bash[58816]: cephadm 2023-10-01T10:30:53.970776+0000 mgr.y (mgr.25072) 333 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:30:56.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:56 smithi078 bash[75299]: cluster 2023-10-01T10:30:55.460407+0000 mgr.y (mgr.25072) 334 : cluster 0 pgmap v261: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.9 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 799 op/s; 11436/93486 objects degraded (12.233%); 1.0 MiB/s, 19 objects/s recovering 2023-10-01T10:30:56.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:56 smithi078 bash[75299]: audit 2023-10-01T10:30:56.174521+0000 mon.c (mon.2) 39 : audit 0 from='client.? 172.21.15.78:0/4045022718' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:56.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:56 smithi078 bash[78274]: cluster 2023-10-01T10:30:55.460407+0000 mgr.y (mgr.25072) 334 : cluster 0 pgmap v261: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.9 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 799 op/s; 11436/93486 objects degraded (12.233%); 1.0 MiB/s, 19 objects/s recovering 2023-10-01T10:30:56.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:56 smithi078 bash[78274]: audit 2023-10-01T10:30:56.174521+0000 mon.c (mon.2) 39 : audit 0 from='client.? 172.21.15.78:0/4045022718' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:56.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:56 smithi173 bash[58816]: cluster 2023-10-01T10:30:55.460407+0000 mgr.y (mgr.25072) 334 : cluster 0 pgmap v261: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.9 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 799 op/s; 11436/93486 objects degraded (12.233%); 1.0 MiB/s, 19 objects/s recovering 2023-10-01T10:30:56.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:56 smithi173 bash[58816]: audit 2023-10-01T10:30:56.174521+0000 mon.c (mon.2) 39 : audit 0 from='client.? 172.21.15.78:0/4045022718' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:57.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:57 smithi078 bash[75299]: cluster 2023-10-01T10:30:56.815881+0000 mon.a (mon.0) 460 : cluster 3 Health check update: Degraded data redundancy: 11436/93486 objects degraded (12.233%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-01T10:30:57.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:57 smithi078 bash[78274]: cluster 2023-10-01T10:30:56.815881+0000 mon.a (mon.0) 460 : cluster 3 Health check update: Degraded data redundancy: 11436/93486 objects degraded (12.233%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-01T10:30:57.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:57 smithi173 bash[58816]: cluster 2023-10-01T10:30:56.815881+0000 mon.a (mon.0) 460 : cluster 3 Health check update: Degraded data redundancy: 11436/93486 objects degraded (12.233%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-01T10:30:58.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:58 smithi078 bash[75299]: cluster 2023-10-01T10:30:57.460973+0000 mgr.y (mgr.25072) 335 : cluster 0 pgmap v262: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.9 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 532 op/s; 11436/93486 objects degraded (12.233%); 677 KiB/s, 13 objects/s recovering 2023-10-01T10:30:58.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:58 smithi078 bash[78274]: cluster 2023-10-01T10:30:57.460973+0000 mgr.y (mgr.25072) 335 : cluster 0 pgmap v262: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.9 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 532 op/s; 11436/93486 objects degraded (12.233%); 677 KiB/s, 13 objects/s recovering 2023-10-01T10:30:58.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:58 smithi173 bash[58816]: cluster 2023-10-01T10:30:57.460973+0000 mgr.y (mgr.25072) 335 : cluster 0 pgmap v262: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.9 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 532 op/s; 11436/93486 objects degraded (12.233%); 677 KiB/s, 13 objects/s recovering 2023-10-01T10:30:59.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:30:59 smithi078 bash[75299]: audit 2023-10-01T10:30:58.763050+0000 mon.a (mon.0) 461 : audit 0 from='client.? 172.21.15.78:0/3101432750' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:59.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:30:59 smithi078 bash[78274]: audit 2023-10-01T10:30:58.763050+0000 mon.a (mon.0) 461 : audit 0 from='client.? 172.21.15.78:0/3101432750' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:30:59.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:30:59 smithi173 bash[58816]: audit 2023-10-01T10:30:58.763050+0000 mon.a (mon.0) 461 : audit 0 from='client.? 172.21.15.78:0/3101432750' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:00.562 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:31:00 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:31:00] "GET /metrics HTTP/1.1" 200 36212 "" "Prometheus/2.33.4" 2023-10-01T10:31:00.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:00 smithi078 bash[75299]: cluster 2023-10-01T10:30:59.462043+0000 mgr.y (mgr.25072) 336 : cluster 0 pgmap v263: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.8 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 712 op/s; 11436/87009 objects degraded (13.143%); 677 KiB/s, 13 objects/s recovering 2023-10-01T10:31:00.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:00 smithi078 bash[78274]: cluster 2023-10-01T10:30:59.462043+0000 mgr.y (mgr.25072) 336 : cluster 0 pgmap v263: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.8 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 712 op/s; 11436/87009 objects degraded (13.143%); 677 KiB/s, 13 objects/s recovering 2023-10-01T10:31:00.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:00 smithi173 bash[58816]: cluster 2023-10-01T10:30:59.462043+0000 mgr.y (mgr.25072) 336 : cluster 0 pgmap v263: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.8 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 712 op/s; 11436/87009 objects degraded (13.143%); 677 KiB/s, 13 objects/s recovering 2023-10-01T10:31:01.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:01 smithi078 bash[75299]: audit 2023-10-01T10:31:01.387922+0000 mon.c (mon.2) 40 : audit 0 from='client.? 172.21.15.78:0/2772492337' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:01.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:01 smithi078 bash[78274]: audit 2023-10-01T10:31:01.387922+0000 mon.c (mon.2) 40 : audit 0 from='client.? 172.21.15.78:0/2772492337' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:01.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:01 smithi173 bash[58816]: audit 2023-10-01T10:31:01.387922+0000 mon.c (mon.2) 40 : audit 0 from='client.? 172.21.15.78:0/2772492337' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:02.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:02 smithi078 bash[75299]: cluster 2023-10-01T10:31:01.462684+0000 mgr.y (mgr.25072) 337 : cluster 0 pgmap v264: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 11324/83904 objects degraded (13.496%); 677 KiB/s, 22 objects/s recovering 2023-10-01T10:31:02.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:02 smithi078 bash[75299]: cluster 2023-10-01T10:31:01.817083+0000 mon.a (mon.0) 462 : cluster 3 Health check update: Degraded data redundancy: 11324/83904 objects degraded (13.496%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-01T10:31:02.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:02 smithi078 bash[75299]: audit 2023-10-01T10:31:02.393984+0000 mon.a (mon.0) 463 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:31:02.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:02 smithi078 bash[78274]: cluster 2023-10-01T10:31:01.462684+0000 mgr.y (mgr.25072) 337 : cluster 0 pgmap v264: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 11324/83904 objects degraded (13.496%); 677 KiB/s, 22 objects/s recovering 2023-10-01T10:31:02.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:02 smithi078 bash[78274]: cluster 2023-10-01T10:31:01.817083+0000 mon.a (mon.0) 462 : cluster 3 Health check update: Degraded data redundancy: 11324/83904 objects degraded (13.496%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-01T10:31:02.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:02 smithi078 bash[78274]: audit 2023-10-01T10:31:02.393984+0000 mon.a (mon.0) 463 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:31:02.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:02 smithi173 bash[58816]: cluster 2023-10-01T10:31:01.462684+0000 mgr.y (mgr.25072) 337 : cluster 0 pgmap v264: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 11324/83904 objects degraded (13.496%); 677 KiB/s, 22 objects/s recovering 2023-10-01T10:31:02.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:02 smithi173 bash[58816]: cluster 2023-10-01T10:31:01.817083+0000 mon.a (mon.0) 462 : cluster 3 Health check update: Degraded data redundancy: 11324/83904 objects degraded (13.496%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-10-01T10:31:02.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:02 smithi173 bash[58816]: audit 2023-10-01T10:31:02.393984+0000 mon.a (mon.0) 463 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:31:03.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:03 smithi173 bash[58816]: cluster 2023-10-01T10:31:03.466331+0000 mgr.y (mgr.25072) 338 : cluster 0 pgmap v265: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 533 op/s; 11324/83904 objects degraded (13.496%); 277 KiB/s, 16 objects/s recovering 2023-10-01T10:31:04.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:03 smithi078 bash[78274]: cluster 2023-10-01T10:31:03.466331+0000 mgr.y (mgr.25072) 338 : cluster 0 pgmap v265: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 533 op/s; 11324/83904 objects degraded (13.496%); 277 KiB/s, 16 objects/s recovering 2023-10-01T10:31:04.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:03 smithi078 bash[75299]: cluster 2023-10-01T10:31:03.466331+0000 mgr.y (mgr.25072) 338 : cluster 0 pgmap v265: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 533 op/s; 11324/83904 objects degraded (13.496%); 277 KiB/s, 16 objects/s recovering 2023-10-01T10:31:04.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:04 smithi173 bash[58816]: cluster 2023-10-01T10:31:03.586151+0000 osd.4 (osd.4) 14 : cluster 3 failed to encode map e91 with expected crc 2023-10-01T10:31:04.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:04 smithi173 bash[58816]: cluster 2023-10-01T10:31:03.589174+0000 mon.a (mon.0) 464 : cluster 0 osdmap e91: 8 total, 8 up, 8 in 2023-10-01T10:31:04.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:04 smithi173 bash[58816]: cluster 2023-10-01T10:31:03.592719+0000 osd.7 (osd.7) 21 : cluster 3 failed to encode map e91 with expected crc 2023-10-01T10:31:04.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:04 smithi173 bash[58816]: cluster 2023-10-01T10:31:03.595690+0000 osd.3 (osd.3) 14 : cluster 3 failed to encode map e91 with expected crc 2023-10-01T10:31:04.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:04 smithi173 bash[58816]: audit 2023-10-01T10:31:03.952172+0000 mon.c (mon.2) 41 : audit 0 from='client.? 172.21.15.78:0/1342687459' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:05.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:04 smithi078 bash[75299]: cluster 2023-10-01T10:31:03.586151+0000 osd.4 (osd.4) 14 : cluster 3 failed to encode map e91 with expected crc 2023-10-01T10:31:05.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:04 smithi078 bash[75299]: cluster 2023-10-01T10:31:03.589174+0000 mon.a (mon.0) 464 : cluster 0 osdmap e91: 8 total, 8 up, 8 in 2023-10-01T10:31:05.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:04 smithi078 bash[75299]: cluster 2023-10-01T10:31:03.592719+0000 osd.7 (osd.7) 21 : cluster 3 failed to encode map e91 with expected crc 2023-10-01T10:31:05.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:04 smithi078 bash[75299]: cluster 2023-10-01T10:31:03.595690+0000 osd.3 (osd.3) 14 : cluster 3 failed to encode map e91 with expected crc 2023-10-01T10:31:05.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:04 smithi078 bash[75299]: audit 2023-10-01T10:31:03.952172+0000 mon.c (mon.2) 41 : audit 0 from='client.? 172.21.15.78:0/1342687459' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:05.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:04 smithi078 bash[78274]: cluster 2023-10-01T10:31:03.586151+0000 osd.4 (osd.4) 14 : cluster 3 failed to encode map e91 with expected crc 2023-10-01T10:31:05.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:04 smithi078 bash[78274]: cluster 2023-10-01T10:31:03.589174+0000 mon.a (mon.0) 464 : cluster 0 osdmap e91: 8 total, 8 up, 8 in 2023-10-01T10:31:05.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:04 smithi078 bash[78274]: cluster 2023-10-01T10:31:03.592719+0000 osd.7 (osd.7) 21 : cluster 3 failed to encode map e91 with expected crc 2023-10-01T10:31:05.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:04 smithi078 bash[78274]: cluster 2023-10-01T10:31:03.595690+0000 osd.3 (osd.3) 14 : cluster 3 failed to encode map e91 with expected crc 2023-10-01T10:31:05.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:04 smithi078 bash[78274]: audit 2023-10-01T10:31:03.952172+0000 mon.c (mon.2) 41 : audit 0 from='client.? 172.21.15.78:0/1342687459' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:05.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:05 smithi173 bash[58816]: cluster 2023-10-01T10:31:04.594393+0000 osd.6 (osd.6) 11 : cluster 3 failed to encode map e92 with expected crc 2023-10-01T10:31:05.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:05 smithi173 bash[58816]: cluster 2023-10-01T10:31:04.595012+0000 mon.a (mon.0) 465 : cluster 0 osdmap e92: 8 total, 8 up, 8 in 2023-10-01T10:31:05.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:05 smithi173 bash[58816]: cluster 2023-10-01T10:31:04.595097+0000 osd.7 (osd.7) 22 : cluster 3 failed to encode map e92 with expected crc 2023-10-01T10:31:05.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:05 smithi173 bash[58816]: cluster 2023-10-01T10:31:04.603973+0000 osd.3 (osd.3) 15 : cluster 3 failed to encode map e92 with expected crc 2023-10-01T10:31:05.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:05 smithi173 bash[58816]: cluster 2023-10-01T10:31:04.681024+0000 osd.1 (osd.1) 16 : cluster 3 failed to encode map e92 with expected crc 2023-10-01T10:31:05.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:05 smithi173 bash[58816]: cluster 2023-10-01T10:31:05.466969+0000 mgr.y (mgr.25072) 339 : cluster 0 pgmap v268: 89 pgs: 1 peering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 11242/74712 objects degraded (15.047%); 0 B/s, 24 objects/s recovering 2023-10-01T10:31:06.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:05 smithi078 bash[75299]: cluster 2023-10-01T10:31:04.594393+0000 osd.6 (osd.6) 11 : cluster 3 failed to encode map e92 with expected crc 2023-10-01T10:31:06.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:05 smithi078 bash[75299]: cluster 2023-10-01T10:31:04.595012+0000 mon.a (mon.0) 465 : cluster 0 osdmap e92: 8 total, 8 up, 8 in 2023-10-01T10:31:06.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:05 smithi078 bash[75299]: cluster 2023-10-01T10:31:04.595097+0000 osd.7 (osd.7) 22 : cluster 3 failed to encode map e92 with expected crc 2023-10-01T10:31:06.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:05 smithi078 bash[75299]: cluster 2023-10-01T10:31:04.603973+0000 osd.3 (osd.3) 15 : cluster 3 failed to encode map e92 with expected crc 2023-10-01T10:31:06.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:05 smithi078 bash[75299]: cluster 2023-10-01T10:31:04.681024+0000 osd.1 (osd.1) 16 : cluster 3 failed to encode map e92 with expected crc 2023-10-01T10:31:06.085 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:05 smithi078 bash[75299]: cluster 2023-10-01T10:31:05.466969+0000 mgr.y (mgr.25072) 339 : cluster 0 pgmap v268: 89 pgs: 1 peering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 11242/74712 objects degraded (15.047%); 0 B/s, 24 objects/s recovering 2023-10-01T10:31:06.085 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:05 smithi078 bash[78274]: cluster 2023-10-01T10:31:04.594393+0000 osd.6 (osd.6) 11 : cluster 3 failed to encode map e92 with expected crc 2023-10-01T10:31:06.085 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:05 smithi078 bash[78274]: cluster 2023-10-01T10:31:04.595012+0000 mon.a (mon.0) 465 : cluster 0 osdmap e92: 8 total, 8 up, 8 in 2023-10-01T10:31:06.085 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:05 smithi078 bash[78274]: cluster 2023-10-01T10:31:04.595097+0000 osd.7 (osd.7) 22 : cluster 3 failed to encode map e92 with expected crc 2023-10-01T10:31:06.085 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:05 smithi078 bash[78274]: cluster 2023-10-01T10:31:04.603973+0000 osd.3 (osd.3) 15 : cluster 3 failed to encode map e92 with expected crc 2023-10-01T10:31:06.085 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:05 smithi078 bash[78274]: cluster 2023-10-01T10:31:04.681024+0000 osd.1 (osd.1) 16 : cluster 3 failed to encode map e92 with expected crc 2023-10-01T10:31:06.085 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:05 smithi078 bash[78274]: cluster 2023-10-01T10:31:05.466969+0000 mgr.y (mgr.25072) 339 : cluster 0 pgmap v268: 89 pgs: 1 peering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 11242/74712 objects degraded (15.047%); 0 B/s, 24 objects/s recovering 2023-10-01T10:31:06.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:06 smithi173 bash[58816]: audit 2023-10-01T10:31:06.573797+0000 mon.b (mon.1) 17 : audit 0 from='client.? 172.21.15.78:0/4083722169' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:07.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:06 smithi078 bash[75299]: audit 2023-10-01T10:31:06.573797+0000 mon.b (mon.1) 17 : audit 0 from='client.? 172.21.15.78:0/4083722169' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:07.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:06 smithi078 bash[78274]: audit 2023-10-01T10:31:06.573797+0000 mon.b (mon.1) 17 : audit 0 from='client.? 172.21.15.78:0/4083722169' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:07.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:07 smithi173 bash[58816]: cluster 2023-10-01T10:31:06.827582+0000 mon.a (mon.0) 466 : cluster 3 Health check update: Degraded data redundancy: 11242/74712 objects degraded (15.047%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-01T10:31:07.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:07 smithi173 bash[58816]: cluster 2023-10-01T10:31:07.467516+0000 mgr.y (mgr.25072) 340 : cluster 0 pgmap v269: 89 pgs: 1 peering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 512 op/s; 11242/74712 objects degraded (15.047%); 0 B/s, 24 objects/s recovering 2023-10-01T10:31:08.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:07 smithi078 bash[75299]: cluster 2023-10-01T10:31:06.827582+0000 mon.a (mon.0) 466 : cluster 3 Health check update: Degraded data redundancy: 11242/74712 objects degraded (15.047%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-01T10:31:08.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:07 smithi078 bash[75299]: cluster 2023-10-01T10:31:07.467516+0000 mgr.y (mgr.25072) 340 : cluster 0 pgmap v269: 89 pgs: 1 peering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 512 op/s; 11242/74712 objects degraded (15.047%); 0 B/s, 24 objects/s recovering 2023-10-01T10:31:08.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:07 smithi078 bash[78274]: cluster 2023-10-01T10:31:06.827582+0000 mon.a (mon.0) 466 : cluster 3 Health check update: Degraded data redundancy: 11242/74712 objects degraded (15.047%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-01T10:31:08.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:07 smithi078 bash[78274]: cluster 2023-10-01T10:31:07.467516+0000 mgr.y (mgr.25072) 340 : cluster 0 pgmap v269: 89 pgs: 1 peering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.5 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 512 op/s; 11242/74712 objects degraded (15.047%); 0 B/s, 24 objects/s recovering 2023-10-01T10:31:09.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:09 smithi078 bash[75299]: audit 2023-10-01T10:31:08.989476+0000 mon.a (mon.0) 467 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:31:09.333 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:31:08 smithi078 bash[71214]: debug 2023-10-01T10:31:08.985+0000 7fd9c245e700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:31:09.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:09 smithi078 bash[78274]: audit 2023-10-01T10:31:08.989476+0000 mon.a (mon.0) 467 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:31:09.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:09 smithi173 bash[58816]: audit 2023-10-01T10:31:08.989476+0000 mon.a (mon.0) 467 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:31:10.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:10 smithi078 bash[75299]: audit 2023-10-01T10:31:08.989770+0000 mgr.y (mgr.25072) 341 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:31:10.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:10 smithi078 bash[75299]: cephadm 2023-10-01T10:31:08.991004+0000 mgr.y (mgr.25072) 342 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:31:10.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:10 smithi078 bash[75299]: audit 2023-10-01T10:31:09.183819+0000 mon.b (mon.1) 18 : audit 0 from='client.? 172.21.15.78:0/574958533' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:10.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:10 smithi078 bash[75299]: cluster 2023-10-01T10:31:09.468296+0000 mgr.y (mgr.25072) 343 : cluster 0 pgmap v270: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.4 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 11137/68625 objects degraded (16.229%); 0 B/s, 23 objects/s recovering 2023-10-01T10:31:10.334 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:31:10 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:31:10] "GET /metrics HTTP/1.1" 200 36208 "" "Prometheus/2.33.4" 2023-10-01T10:31:10.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:10 smithi078 bash[78274]: audit 2023-10-01T10:31:08.989770+0000 mgr.y (mgr.25072) 341 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:31:10.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:10 smithi078 bash[78274]: cephadm 2023-10-01T10:31:08.991004+0000 mgr.y (mgr.25072) 342 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:31:10.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:10 smithi078 bash[78274]: audit 2023-10-01T10:31:09.183819+0000 mon.b (mon.1) 18 : audit 0 from='client.? 172.21.15.78:0/574958533' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:10.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:10 smithi078 bash[78274]: cluster 2023-10-01T10:31:09.468296+0000 mgr.y (mgr.25072) 343 : cluster 0 pgmap v270: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.4 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 11137/68625 objects degraded (16.229%); 0 B/s, 23 objects/s recovering 2023-10-01T10:31:10.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:10 smithi173 bash[58816]: audit 2023-10-01T10:31:08.989770+0000 mgr.y (mgr.25072) 341 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:31:10.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:10 smithi173 bash[58816]: cephadm 2023-10-01T10:31:08.991004+0000 mgr.y (mgr.25072) 342 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:31:10.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:10 smithi173 bash[58816]: audit 2023-10-01T10:31:09.183819+0000 mon.b (mon.1) 18 : audit 0 from='client.? 172.21.15.78:0/574958533' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:10.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:10 smithi173 bash[58816]: cluster 2023-10-01T10:31:09.468296+0000 mgr.y (mgr.25072) 343 : cluster 0 pgmap v270: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.4 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 11137/68625 objects degraded (16.229%); 0 B/s, 23 objects/s recovering 2023-10-01T10:31:12.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:12 smithi078 bash[75299]: cluster 2023-10-01T10:31:11.473428+0000 mgr.y (mgr.25072) 344 : cluster 0 pgmap v271: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.3 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 11137/66201 objects degraded (16.823%); 0 B/s, 23 objects/s recovering 2023-10-01T10:31:12.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:12 smithi078 bash[75299]: audit 2023-10-01T10:31:11.792124+0000 mon.c (mon.2) 42 : audit 0 from='client.? 172.21.15.78:0/689299182' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:12.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:12 smithi078 bash[75299]: cluster 2023-10-01T10:31:11.828852+0000 mon.a (mon.0) 468 : cluster 3 Health check update: Degraded data redundancy: 11137/66201 objects degraded (16.823%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-01T10:31:12.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:12 smithi078 bash[78274]: cluster 2023-10-01T10:31:11.473428+0000 mgr.y (mgr.25072) 344 : cluster 0 pgmap v271: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.3 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 11137/66201 objects degraded (16.823%); 0 B/s, 23 objects/s recovering 2023-10-01T10:31:12.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:12 smithi078 bash[78274]: audit 2023-10-01T10:31:11.792124+0000 mon.c (mon.2) 42 : audit 0 from='client.? 172.21.15.78:0/689299182' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:12.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:12 smithi078 bash[78274]: cluster 2023-10-01T10:31:11.828852+0000 mon.a (mon.0) 468 : cluster 3 Health check update: Degraded data redundancy: 11137/66201 objects degraded (16.823%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-01T10:31:12.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:12 smithi173 bash[58816]: cluster 2023-10-01T10:31:11.473428+0000 mgr.y (mgr.25072) 344 : cluster 0 pgmap v271: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.3 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 11137/66201 objects degraded (16.823%); 0 B/s, 23 objects/s recovering 2023-10-01T10:31:12.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:12 smithi173 bash[58816]: audit 2023-10-01T10:31:11.792124+0000 mon.c (mon.2) 42 : audit 0 from='client.? 172.21.15.78:0/689299182' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:12.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:12 smithi173 bash[58816]: cluster 2023-10-01T10:31:11.828852+0000 mon.a (mon.0) 468 : cluster 3 Health check update: Degraded data redundancy: 11137/66201 objects degraded (16.823%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-01T10:31:14.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:14 smithi078 bash[75299]: cluster 2023-10-01T10:31:13.473999+0000 mgr.y (mgr.25072) 345 : cluster 0 pgmap v272: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.3 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 537 op/s; 11137/64470 objects degraded (17.275%); 0 B/s, 17 objects/s recovering 2023-10-01T10:31:14.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:14 smithi078 bash[75299]: audit 2023-10-01T10:31:14.350296+0000 mon.a (mon.0) 469 : audit 0 from='client.? 172.21.15.78:0/2200346319' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:14.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:14 smithi078 bash[78274]: cluster 2023-10-01T10:31:13.473999+0000 mgr.y (mgr.25072) 345 : cluster 0 pgmap v272: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.3 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 537 op/s; 11137/64470 objects degraded (17.275%); 0 B/s, 17 objects/s recovering 2023-10-01T10:31:14.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:14 smithi078 bash[78274]: audit 2023-10-01T10:31:14.350296+0000 mon.a (mon.0) 469 : audit 0 from='client.? 172.21.15.78:0/2200346319' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:14.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:14 smithi173 bash[58816]: cluster 2023-10-01T10:31:13.473999+0000 mgr.y (mgr.25072) 345 : cluster 0 pgmap v272: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.3 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 537 op/s; 11137/64470 objects degraded (17.275%); 0 B/s, 17 objects/s recovering 2023-10-01T10:31:14.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:14 smithi173 bash[58816]: audit 2023-10-01T10:31:14.350296+0000 mon.a (mon.0) 469 : audit 0 from='client.? 172.21.15.78:0/2200346319' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:16.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:16 smithi078 bash[75299]: cluster 2023-10-01T10:31:15.474780+0000 mgr.y (mgr.25072) 346 : cluster 0 pgmap v273: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.2 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 11034/56757 objects degraded (19.441%); 0 B/s, 19 objects/s recovering 2023-10-01T10:31:16.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:16 smithi078 bash[78274]: cluster 2023-10-01T10:31:15.474780+0000 mgr.y (mgr.25072) 346 : cluster 0 pgmap v273: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.2 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 11034/56757 objects degraded (19.441%); 0 B/s, 19 objects/s recovering 2023-10-01T10:31:16.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:16 smithi173 bash[58816]: cluster 2023-10-01T10:31:15.474780+0000 mgr.y (mgr.25072) 346 : cluster 0 pgmap v273: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.2 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 11034/56757 objects degraded (19.441%); 0 B/s, 19 objects/s recovering 2023-10-01T10:31:17.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:17 smithi078 bash[75299]: cluster 2023-10-01T10:31:16.830078+0000 mon.a (mon.0) 470 : cluster 3 Health check update: Degraded data redundancy: 11034/56757 objects degraded (19.441%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-01T10:31:17.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:17 smithi078 bash[75299]: audit 2023-10-01T10:31:16.976274+0000 mon.c (mon.2) 43 : audit 0 from='client.? 172.21.15.78:0/1574647271' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:17.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:17 smithi078 bash[75299]: audit 2023-10-01T10:31:17.394360+0000 mon.a (mon.0) 471 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:31:17.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:17 smithi078 bash[78274]: cluster 2023-10-01T10:31:16.830078+0000 mon.a (mon.0) 470 : cluster 3 Health check update: Degraded data redundancy: 11034/56757 objects degraded (19.441%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-01T10:31:17.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:17 smithi078 bash[78274]: audit 2023-10-01T10:31:16.976274+0000 mon.c (mon.2) 43 : audit 0 from='client.? 172.21.15.78:0/1574647271' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:17.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:17 smithi078 bash[78274]: audit 2023-10-01T10:31:17.394360+0000 mon.a (mon.0) 471 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:31:17.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:17 smithi173 bash[58816]: cluster 2023-10-01T10:31:16.830078+0000 mon.a (mon.0) 470 : cluster 3 Health check update: Degraded data redundancy: 11034/56757 objects degraded (19.441%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-01T10:31:17.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:17 smithi173 bash[58816]: audit 2023-10-01T10:31:16.976274+0000 mon.c (mon.2) 43 : audit 0 from='client.? 172.21.15.78:0/1574647271' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:17.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:17 smithi173 bash[58816]: audit 2023-10-01T10:31:17.394360+0000 mon.a (mon.0) 471 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:31:18.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:18 smithi078 bash[75299]: cluster 2023-10-01T10:31:17.475333+0000 mgr.y (mgr.25072) 347 : cluster 0 pgmap v274: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.2 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 498 op/s; 11034/56757 objects degraded (19.441%); 0 B/s, 17 objects/s recovering 2023-10-01T10:31:18.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:18 smithi078 bash[78274]: cluster 2023-10-01T10:31:17.475333+0000 mgr.y (mgr.25072) 347 : cluster 0 pgmap v274: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.2 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 498 op/s; 11034/56757 objects degraded (19.441%); 0 B/s, 17 objects/s recovering 2023-10-01T10:31:18.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:18 smithi173 bash[58816]: cluster 2023-10-01T10:31:17.475333+0000 mgr.y (mgr.25072) 347 : cluster 0 pgmap v274: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.2 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 498 op/s; 11034/56757 objects degraded (19.441%); 0 B/s, 17 objects/s recovering 2023-10-01T10:31:20.548 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:20 smithi078 bash[75299]: cluster 2023-10-01T10:31:19.476143+0000 mgr.y (mgr.25072) 348 : cluster 0 pgmap v275: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.0 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 10928/49341 objects degraded (22.148%); 0 B/s, 26 objects/s recovering 2023-10-01T10:31:20.548 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:20 smithi078 bash[75299]: audit 2023-10-01T10:31:19.595501+0000 mon.a (mon.0) 472 : audit 0 from='client.? 172.21.15.78:0/4118333430' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:20.548 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:31:20 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:31:20] "GET /metrics HTTP/1.1" 200 36199 "" "Prometheus/2.33.4" 2023-10-01T10:31:20.548 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:20 smithi078 bash[78274]: cluster 2023-10-01T10:31:19.476143+0000 mgr.y (mgr.25072) 348 : cluster 0 pgmap v275: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.0 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 10928/49341 objects degraded (22.148%); 0 B/s, 26 objects/s recovering 2023-10-01T10:31:20.548 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:20 smithi078 bash[78274]: audit 2023-10-01T10:31:19.595501+0000 mon.a (mon.0) 472 : audit 0 from='client.? 172.21.15.78:0/4118333430' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:20.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:20 smithi173 bash[58816]: cluster 2023-10-01T10:31:19.476143+0000 mgr.y (mgr.25072) 348 : cluster 0 pgmap v275: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.0 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 10928/49341 objects degraded (22.148%); 0 B/s, 26 objects/s recovering 2023-10-01T10:31:20.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:20 smithi173 bash[58816]: audit 2023-10-01T10:31:19.595501+0000 mon.a (mon.0) 472 : audit 0 from='client.? 172.21.15.78:0/4118333430' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:22.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:22 smithi078 bash[75299]: cluster 2023-10-01T10:31:21.476758+0000 mgr.y (mgr.25072) 349 : cluster 0 pgmap v276: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 979 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 10928/47022 objects degraded (23.240%); 0 B/s, 17 objects/s recovering 2023-10-01T10:31:22.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:22 smithi078 bash[75299]: cluster 2023-10-01T10:31:21.831333+0000 mon.a (mon.0) 473 : cluster 3 Health check update: Degraded data redundancy: 10928/47022 objects degraded (23.240%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-01T10:31:22.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:22 smithi078 bash[75299]: audit 2023-10-01T10:31:22.193077+0000 mon.c (mon.2) 44 : audit 0 from='client.? 172.21.15.78:0/4082100955' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:22.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:22 smithi078 bash[78274]: cluster 2023-10-01T10:31:21.476758+0000 mgr.y (mgr.25072) 349 : cluster 0 pgmap v276: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 979 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 10928/47022 objects degraded (23.240%); 0 B/s, 17 objects/s recovering 2023-10-01T10:31:22.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:22 smithi078 bash[78274]: cluster 2023-10-01T10:31:21.831333+0000 mon.a (mon.0) 473 : cluster 3 Health check update: Degraded data redundancy: 10928/47022 objects degraded (23.240%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-01T10:31:22.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:22 smithi078 bash[78274]: audit 2023-10-01T10:31:22.193077+0000 mon.c (mon.2) 44 : audit 0 from='client.? 172.21.15.78:0/4082100955' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:22.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:22 smithi173 bash[58816]: cluster 2023-10-01T10:31:21.476758+0000 mgr.y (mgr.25072) 349 : cluster 0 pgmap v276: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 979 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 10928/47022 objects degraded (23.240%); 0 B/s, 17 objects/s recovering 2023-10-01T10:31:22.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:22 smithi173 bash[58816]: cluster 2023-10-01T10:31:21.831333+0000 mon.a (mon.0) 473 : cluster 3 Health check update: Degraded data redundancy: 10928/47022 objects degraded (23.240%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-01T10:31:22.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:22 smithi173 bash[58816]: audit 2023-10-01T10:31:22.193077+0000 mon.c (mon.2) 44 : audit 0 from='client.? 172.21.15.78:0/4082100955' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:24.333 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:31:24 smithi078 bash[71214]: debug 2023-10-01T10:31:24.001+0000 7fd9c245e700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:31:24.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:24 smithi078 bash[75299]: cluster 2023-10-01T10:31:23.477338+0000 mgr.y (mgr.25072) 350 : cluster 0 pgmap v277: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 979 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 532 op/s; 10928/47022 objects degraded (23.240%); 0 B/s, 17 objects/s recovering 2023-10-01T10:31:24.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:24 smithi078 bash[75299]: audit 2023-10-01T10:31:24.005707+0000 mon.a (mon.0) 474 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:31:24.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:24 smithi078 bash[78274]: cluster 2023-10-01T10:31:23.477338+0000 mgr.y (mgr.25072) 350 : cluster 0 pgmap v277: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 979 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 532 op/s; 10928/47022 objects degraded (23.240%); 0 B/s, 17 objects/s recovering 2023-10-01T10:31:24.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:24 smithi078 bash[78274]: audit 2023-10-01T10:31:24.005707+0000 mon.a (mon.0) 474 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:31:24.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:24 smithi173 bash[58816]: cluster 2023-10-01T10:31:23.477338+0000 mgr.y (mgr.25072) 350 : cluster 0 pgmap v277: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 979 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 532 op/s; 10928/47022 objects degraded (23.240%); 0 B/s, 17 objects/s recovering 2023-10-01T10:31:24.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:24 smithi173 bash[58816]: audit 2023-10-01T10:31:24.005707+0000 mon.a (mon.0) 474 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:31:25.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:25 smithi078 bash[75299]: audit 2023-10-01T10:31:24.005881+0000 mgr.y (mgr.25072) 351 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:31:25.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:25 smithi078 bash[75299]: cephadm 2023-10-01T10:31:24.007036+0000 mgr.y (mgr.25072) 352 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:31:25.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:25 smithi078 bash[75299]: audit 2023-10-01T10:31:24.766073+0000 mon.c (mon.2) 45 : audit 0 from='client.? 172.21.15.78:0/3007241525' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:25.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:25 smithi078 bash[78274]: audit 2023-10-01T10:31:24.005881+0000 mgr.y (mgr.25072) 351 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:31:25.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:25 smithi078 bash[78274]: cephadm 2023-10-01T10:31:24.007036+0000 mgr.y (mgr.25072) 352 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:31:25.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:25 smithi078 bash[78274]: audit 2023-10-01T10:31:24.766073+0000 mon.c (mon.2) 45 : audit 0 from='client.? 172.21.15.78:0/3007241525' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:25.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:25 smithi173 bash[58816]: audit 2023-10-01T10:31:24.005881+0000 mgr.y (mgr.25072) 351 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:31:25.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:25 smithi173 bash[58816]: cephadm 2023-10-01T10:31:24.007036+0000 mgr.y (mgr.25072) 352 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:31:25.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:25 smithi173 bash[58816]: audit 2023-10-01T10:31:24.766073+0000 mon.c (mon.2) 45 : audit 0 from='client.? 172.21.15.78:0/3007241525' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:26.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:26 smithi078 bash[75299]: cluster 2023-10-01T10:31:25.478150+0000 mgr.y (mgr.25072) 353 : cluster 0 pgmap v278: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 773 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 10818/37137 objects degraded (29.130%); 0 B/s, 26 objects/s recovering 2023-10-01T10:31:26.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:26 smithi078 bash[78274]: cluster 2023-10-01T10:31:25.478150+0000 mgr.y (mgr.25072) 353 : cluster 0 pgmap v278: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 773 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 10818/37137 objects degraded (29.130%); 0 B/s, 26 objects/s recovering 2023-10-01T10:31:26.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:26 smithi173 bash[58816]: cluster 2023-10-01T10:31:25.478150+0000 mgr.y (mgr.25072) 353 : cluster 0 pgmap v278: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 773 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 10818/37137 objects degraded (29.130%); 0 B/s, 26 objects/s recovering 2023-10-01T10:31:27.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:27 smithi078 bash[75299]: cluster 2023-10-01T10:31:26.832424+0000 mon.a (mon.0) 475 : cluster 3 Health check update: Degraded data redundancy: 10818/37137 objects degraded (29.130%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-01T10:31:27.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:27 smithi078 bash[75299]: audit 2023-10-01T10:31:27.356267+0000 mon.a (mon.0) 476 : audit 0 from='client.? 172.21.15.78:0/267422195' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:27.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:27 smithi078 bash[78274]: cluster 2023-10-01T10:31:26.832424+0000 mon.a (mon.0) 475 : cluster 3 Health check update: Degraded data redundancy: 10818/37137 objects degraded (29.130%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-01T10:31:27.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:27 smithi078 bash[78274]: audit 2023-10-01T10:31:27.356267+0000 mon.a (mon.0) 476 : audit 0 from='client.? 172.21.15.78:0/267422195' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:27.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:27 smithi173 bash[58816]: cluster 2023-10-01T10:31:26.832424+0000 mon.a (mon.0) 475 : cluster 3 Health check update: Degraded data redundancy: 10818/37137 objects degraded (29.130%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-01T10:31:27.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:27 smithi173 bash[58816]: audit 2023-10-01T10:31:27.356267+0000 mon.a (mon.0) 476 : audit 0 from='client.? 172.21.15.78:0/267422195' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:28.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:28 smithi078 bash[75299]: cluster 2023-10-01T10:31:27.478757+0000 mgr.y (mgr.25072) 354 : cluster 0 pgmap v279: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 773 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 544 op/s; 10818/37137 objects degraded (29.130%); 0 B/s, 17 objects/s recovering 2023-10-01T10:31:28.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:28 smithi078 bash[78274]: cluster 2023-10-01T10:31:27.478757+0000 mgr.y (mgr.25072) 354 : cluster 0 pgmap v279: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 773 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 544 op/s; 10818/37137 objects degraded (29.130%); 0 B/s, 17 objects/s recovering 2023-10-01T10:31:28.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:28 smithi173 bash[58816]: cluster 2023-10-01T10:31:27.478757+0000 mgr.y (mgr.25072) 354 : cluster 0 pgmap v279: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 773 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 544 op/s; 10818/37137 objects degraded (29.130%); 0 B/s, 17 objects/s recovering 2023-10-01T10:31:30.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:29 smithi078 bash[75299]: cluster 2023-10-01T10:31:29.479635+0000 mgr.y (mgr.25072) 355 : cluster 0 pgmap v280: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 622 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 10710/29883 objects degraded (35.840%); 0 B/s, 26 objects/s recovering 2023-10-01T10:31:30.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:29 smithi078 bash[78274]: cluster 2023-10-01T10:31:29.479635+0000 mgr.y (mgr.25072) 355 : cluster 0 pgmap v280: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 622 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 10710/29883 objects degraded (35.840%); 0 B/s, 26 objects/s recovering 2023-10-01T10:31:30.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:29 smithi173 bash[58816]: cluster 2023-10-01T10:31:29.479635+0000 mgr.y (mgr.25072) 355 : cluster 0 pgmap v280: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 622 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 10710/29883 objects degraded (35.840%); 0 B/s, 26 objects/s recovering 2023-10-01T10:31:30.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:31:30 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:31:30] "GET /metrics HTTP/1.1" 200 36199 "" "Prometheus/2.33.4" 2023-10-01T10:31:31.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:30 smithi078 bash[75299]: audit 2023-10-01T10:31:29.967373+0000 mon.a (mon.0) 477 : audit 0 from='client.? 172.21.15.78:0/3695968996' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:31.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:30 smithi078 bash[78274]: audit 2023-10-01T10:31:29.967373+0000 mon.a (mon.0) 477 : audit 0 from='client.? 172.21.15.78:0/3695968996' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:31.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:30 smithi173 bash[58816]: audit 2023-10-01T10:31:29.967373+0000 mon.a (mon.0) 477 : audit 0 from='client.? 172.21.15.78:0/3695968996' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:32.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:31 smithi078 bash[75299]: cluster 2023-10-01T10:31:31.480274+0000 mgr.y (mgr.25072) 356 : cluster 0 pgmap v281: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 573 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 10710/27555 objects degraded (38.868%); 0 B/s, 18 objects/s recovering 2023-10-01T10:31:32.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:31 smithi078 bash[78274]: cluster 2023-10-01T10:31:31.480274+0000 mgr.y (mgr.25072) 356 : cluster 0 pgmap v281: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 573 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 10710/27555 objects degraded (38.868%); 0 B/s, 18 objects/s recovering 2023-10-01T10:31:32.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:31 smithi173 bash[58816]: cluster 2023-10-01T10:31:31.480274+0000 mgr.y (mgr.25072) 356 : cluster 0 pgmap v281: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 573 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 10710/27555 objects degraded (38.868%); 0 B/s, 18 objects/s recovering 2023-10-01T10:31:33.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:32 smithi078 bash[75299]: cluster 2023-10-01T10:31:31.833329+0000 mon.a (mon.0) 478 : cluster 3 Health check update: Degraded data redundancy: 10710/27555 objects degraded (38.868%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-01T10:31:33.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:32 smithi078 bash[75299]: audit 2023-10-01T10:31:32.395754+0000 mon.a (mon.0) 479 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:31:33.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:32 smithi078 bash[75299]: audit 2023-10-01T10:31:32.593252+0000 mon.a (mon.0) 480 : audit 0 from='client.? 172.21.15.78:0/1524574604' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:33.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:32 smithi078 bash[78274]: cluster 2023-10-01T10:31:31.833329+0000 mon.a (mon.0) 478 : cluster 3 Health check update: Degraded data redundancy: 10710/27555 objects degraded (38.868%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-01T10:31:33.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:32 smithi078 bash[78274]: audit 2023-10-01T10:31:32.395754+0000 mon.a (mon.0) 479 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:31:33.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:32 smithi078 bash[78274]: audit 2023-10-01T10:31:32.593252+0000 mon.a (mon.0) 480 : audit 0 from='client.? 172.21.15.78:0/1524574604' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:33.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:32 smithi173 bash[58816]: cluster 2023-10-01T10:31:31.833329+0000 mon.a (mon.0) 478 : cluster 3 Health check update: Degraded data redundancy: 10710/27555 objects degraded (38.868%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-01T10:31:33.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:32 smithi173 bash[58816]: audit 2023-10-01T10:31:32.395754+0000 mon.a (mon.0) 479 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:31:33.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:32 smithi173 bash[58816]: audit 2023-10-01T10:31:32.593252+0000 mon.a (mon.0) 480 : audit 0 from='client.? 172.21.15.78:0/1524574604' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:34.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:33 smithi078 bash[78274]: cluster 2023-10-01T10:31:33.480798+0000 mgr.y (mgr.25072) 357 : cluster 0 pgmap v282: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 573 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 10710/27555 objects degraded (38.868%); 0 B/s, 18 objects/s recovering 2023-10-01T10:31:34.128 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:33 smithi078 bash[75299]: cluster 2023-10-01T10:31:33.480798+0000 mgr.y (mgr.25072) 357 : cluster 0 pgmap v282: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 573 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 10710/27555 objects degraded (38.868%); 0 B/s, 18 objects/s recovering 2023-10-01T10:31:34.129 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:33 smithi173 bash[58816]: cluster 2023-10-01T10:31:33.480798+0000 mgr.y (mgr.25072) 357 : cluster 0 pgmap v282: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 573 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 10710/27555 objects degraded (38.868%); 0 B/s, 18 objects/s recovering 2023-10-01T10:31:35.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:35 smithi078 bash[78274]: audit 2023-10-01T10:31:35.188387+0000 mon.c (mon.2) 46 : audit 0 from='client.? 172.21.15.78:0/204256388' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:35.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:35 smithi078 bash[75299]: audit 2023-10-01T10:31:35.188387+0000 mon.c (mon.2) 46 : audit 0 from='client.? 172.21.15.78:0/204256388' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:35.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:35 smithi173 bash[58816]: audit 2023-10-01T10:31:35.188387+0000 mon.c (mon.2) 46 : audit 0 from='client.? 172.21.15.78:0/204256388' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:36.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:36 smithi078 bash[75299]: cluster 2023-10-01T10:31:35.481689+0000 mgr.y (mgr.25072) 358 : cluster 0 pgmap v283: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 371 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 810 op/s; 10601/17844 objects degraded (59.409%); 0 B/s, 27 objects/s recovering 2023-10-01T10:31:36.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:36 smithi078 bash[78274]: cluster 2023-10-01T10:31:35.481689+0000 mgr.y (mgr.25072) 358 : cluster 0 pgmap v283: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 371 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 810 op/s; 10601/17844 objects degraded (59.409%); 0 B/s, 27 objects/s recovering 2023-10-01T10:31:36.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:36 smithi173 bash[58816]: cluster 2023-10-01T10:31:35.481689+0000 mgr.y (mgr.25072) 358 : cluster 0 pgmap v283: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 371 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 810 op/s; 10601/17844 objects degraded (59.409%); 0 B/s, 27 objects/s recovering 2023-10-01T10:31:37.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:37 smithi078 bash[75299]: cluster 2023-10-01T10:31:36.834593+0000 mon.a (mon.0) 481 : cluster 3 Health check update: Degraded data redundancy: 10601/17844 objects degraded (59.409%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-01T10:31:37.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:37 smithi078 bash[78274]: cluster 2023-10-01T10:31:36.834593+0000 mon.a (mon.0) 481 : cluster 3 Health check update: Degraded data redundancy: 10601/17844 objects degraded (59.409%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-01T10:31:37.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:37 smithi173 bash[58816]: cluster 2023-10-01T10:31:36.834593+0000 mon.a (mon.0) 481 : cluster 3 Health check update: Degraded data redundancy: 10601/17844 objects degraded (59.409%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-01T10:31:38.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:38 smithi078 bash[75299]: cluster 2023-10-01T10:31:37.482191+0000 mgr.y (mgr.25072) 359 : cluster 0 pgmap v284: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 371 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 535 op/s; 10601/17844 objects degraded (59.409%); 0 B/s, 18 objects/s recovering 2023-10-01T10:31:38.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:38 smithi078 bash[75299]: audit 2023-10-01T10:31:37.773143+0000 mon.a (mon.0) 482 : audit 0 from='client.? 172.21.15.78:0/2911116320' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:38.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:38 smithi078 bash[78274]: cluster 2023-10-01T10:31:37.482191+0000 mgr.y (mgr.25072) 359 : cluster 0 pgmap v284: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 371 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 535 op/s; 10601/17844 objects degraded (59.409%); 0 B/s, 18 objects/s recovering 2023-10-01T10:31:38.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:38 smithi078 bash[78274]: audit 2023-10-01T10:31:37.773143+0000 mon.a (mon.0) 482 : audit 0 from='client.? 172.21.15.78:0/2911116320' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:38.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:38 smithi173 bash[58816]: cluster 2023-10-01T10:31:37.482191+0000 mgr.y (mgr.25072) 359 : cluster 0 pgmap v284: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 371 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 535 op/s; 10601/17844 objects degraded (59.409%); 0 B/s, 18 objects/s recovering 2023-10-01T10:31:38.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:38 smithi173 bash[58816]: audit 2023-10-01T10:31:37.773143+0000 mon.a (mon.0) 482 : audit 0 from='client.? 172.21.15.78:0/2911116320' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:39.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:39 smithi078 bash[75299]: audit 2023-10-01T10:31:39.022232+0000 mon.a (mon.0) 483 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:31:39.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:39 smithi078 bash[78274]: audit 2023-10-01T10:31:39.022232+0000 mon.a (mon.0) 483 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:31:39.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:39 smithi173 bash[58816]: audit 2023-10-01T10:31:39.022232+0000 mon.a (mon.0) 483 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:31:40.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:40 smithi078 bash[75299]: cluster 2023-10-01T10:31:39.483013+0000 mgr.y (mgr.25072) 360 : cluster 0 pgmap v285: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 221 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 10498/10650 objects degraded (98.573%); 0 B/s, 26 objects/s recovering 2023-10-01T10:31:40.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:31:40 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:31:40] "GET /metrics HTTP/1.1" 200 36196 "" "Prometheus/2.33.4" 2023-10-01T10:31:40.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:40 smithi078 bash[78274]: cluster 2023-10-01T10:31:39.483013+0000 mgr.y (mgr.25072) 360 : cluster 0 pgmap v285: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 221 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 10498/10650 objects degraded (98.573%); 0 B/s, 26 objects/s recovering 2023-10-01T10:31:40.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:40 smithi173 bash[58816]: cluster 2023-10-01T10:31:39.483013+0000 mgr.y (mgr.25072) 360 : cluster 0 pgmap v285: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 221 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 10498/10650 objects degraded (98.573%); 0 B/s, 26 objects/s recovering 2023-10-01T10:31:41.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:41 smithi078 bash[75299]: audit 2023-10-01T10:31:40.394666+0000 mon.b (mon.1) 19 : audit 0 from='client.? 172.21.15.78:0/386584191' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:41.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:41 smithi078 bash[78274]: audit 2023-10-01T10:31:40.394666+0000 mon.b (mon.1) 19 : audit 0 from='client.? 172.21.15.78:0/386584191' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:41.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:41 smithi173 bash[58816]: audit 2023-10-01T10:31:40.394666+0000 mon.b (mon.1) 19 : audit 0 from='client.? 172.21.15.78:0/386584191' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:42.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:42 smithi078 bash[75299]: cluster 2023-10-01T10:31:41.483808+0000 mgr.y (mgr.25072) 361 : cluster 0 pgmap v286: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 173 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 10498/8349 objects degraded (125.740%); 0 B/s, 17 objects/s recovering 2023-10-01T10:31:42.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:42 smithi078 bash[75299]: cluster 2023-10-01T10:31:41.835838+0000 mon.a (mon.0) 484 : cluster 3 Health check update: Degraded data redundancy: 10498/10650 objects degraded (98.573%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-01T10:31:42.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:42 smithi078 bash[78274]: cluster 2023-10-01T10:31:41.483808+0000 mgr.y (mgr.25072) 361 : cluster 0 pgmap v286: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 173 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 10498/8349 objects degraded (125.740%); 0 B/s, 17 objects/s recovering 2023-10-01T10:31:42.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:42 smithi078 bash[78274]: cluster 2023-10-01T10:31:41.835838+0000 mon.a (mon.0) 484 : cluster 3 Health check update: Degraded data redundancy: 10498/10650 objects degraded (98.573%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-01T10:31:42.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:42 smithi173 bash[58816]: cluster 2023-10-01T10:31:41.483808+0000 mgr.y (mgr.25072) 361 : cluster 0 pgmap v286: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 173 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 10498/8349 objects degraded (125.740%); 0 B/s, 17 objects/s recovering 2023-10-01T10:31:42.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:42 smithi173 bash[58816]: cluster 2023-10-01T10:31:41.835838+0000 mon.a (mon.0) 484 : cluster 3 Health check update: Degraded data redundancy: 10498/10650 objects degraded (98.573%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-01T10:31:43.542 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Removed 103155 objects 2023-10-01T10:31:43.542 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Clean up completed and total clean up time :158.239 2023-10-01T10:31:43.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:43 smithi078 bash[78274]: audit 2023-10-01T10:31:43.049069+0000 mon.b (mon.1) 20 : audit 0 from='client.? 172.21.15.78:0/541566269' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:43.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:43 smithi078 bash[75299]: audit 2023-10-01T10:31:43.049069+0000 mon.b (mon.1) 20 : audit 0 from='client.? 172.21.15.78:0/541566269' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:43.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:43 smithi173 bash[58816]: audit 2023-10-01T10:31:43.049069+0000 mon.b (mon.1) 20 : audit 0 from='client.? 172.21.15.78:0/541566269' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:44.392 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_2 2023-10-01T10:31:44.393 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_2 unique_pool_2 --yes-i-really-really-mean-it 2023-10-01T10:31:44.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:44 smithi078 bash[75299]: cluster 2023-10-01T10:31:43.484342+0000 mgr.y (mgr.25072) 362 : cluster 0 pgmap v287: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 173 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 533 op/s; 10498/8349 objects degraded (125.740%); 0 B/s, 17 objects/s recovering 2023-10-01T10:31:44.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:44 smithi078 bash[78274]: cluster 2023-10-01T10:31:43.484342+0000 mgr.y (mgr.25072) 362 : cluster 0 pgmap v287: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 173 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 533 op/s; 10498/8349 objects degraded (125.740%); 0 B/s, 17 objects/s recovering 2023-10-01T10:31:44.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:44 smithi173 bash[58816]: cluster 2023-10-01T10:31:43.484342+0000 mgr.y (mgr.25072) 362 : cluster 0 pgmap v287: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 173 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 533 op/s; 10498/8349 objects degraded (125.740%); 0 B/s, 17 objects/s recovering 2023-10-01T10:31:46.074 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:31:45 smithi078 bash[71214]: debug 2023-10-01T10:31:45.813+0000 7fd9c245e700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:31:46.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:46 smithi078 bash[75299]: audit 2023-10-01T10:31:45.073921+0000 mon.a (mon.0) 485 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:31:46.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:46 smithi078 bash[75299]: audit 2023-10-01T10:31:45.082390+0000 mon.a (mon.0) 486 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:31:46.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:46 smithi078 bash[75299]: audit 2023-10-01T10:31:45.221853+0000 mon.a (mon.0) 487 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:31:46.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:46 smithi078 bash[75299]: audit 2023-10-01T10:31:45.229361+0000 mon.a (mon.0) 488 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:31:46.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:46 smithi078 bash[75299]: audit 2023-10-01T10:31:45.424345+0000 mon.c (mon.2) 47 : audit 0 from='client.? 172.21.15.78:0/3869706077' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:46.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:46 smithi078 bash[75299]: cluster 2023-10-01T10:31:45.485071+0000 mgr.y (mgr.25072) 363 : cluster 0 pgmap v288: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 762 op/s; 10391/96 objects degraded (10823.958%); 0 B/s, 26 objects/s recovering 2023-10-01T10:31:46.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:46 smithi078 bash[75299]: audit 2023-10-01T10:31:45.705130+0000 mon.a (mon.0) 489 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:31:46.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:46 smithi078 bash[75299]: audit 2023-10-01T10:31:45.706866+0000 mon.a (mon.0) 490 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:31:46.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:46 smithi078 bash[75299]: audit 2023-10-01T10:31:45.715371+0000 mon.a (mon.0) 491 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:31:46.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:46 smithi078 bash[75299]: audit 2023-10-01T10:31:45.802322+0000 mon.a (mon.0) 492 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:31:46.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:46 smithi078 bash[75299]: audit 2023-10-01T10:31:45.809040+0000 mon.a (mon.0) 493 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:31:46.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:46 smithi078 bash[75299]: audit 2023-10-01T10:31:45.812106+0000 mon.a (mon.0) 494 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:31:46.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:46 smithi078 bash[75299]: audit 2023-10-01T10:31:45.814145+0000 mon.a (mon.0) 495 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:31:46.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:46 smithi078 bash[75299]: audit 2023-10-01T10:31:45.816772+0000 mon.a (mon.0) 496 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:31:46.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:46 smithi078 bash[78274]: audit 2023-10-01T10:31:45.073921+0000 mon.a (mon.0) 485 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:31:46.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:46 smithi078 bash[78274]: audit 2023-10-01T10:31:45.082390+0000 mon.a (mon.0) 486 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:31:46.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:46 smithi078 bash[78274]: audit 2023-10-01T10:31:45.221853+0000 mon.a (mon.0) 487 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:31:46.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:46 smithi078 bash[78274]: audit 2023-10-01T10:31:45.229361+0000 mon.a (mon.0) 488 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:31:46.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:46 smithi078 bash[78274]: audit 2023-10-01T10:31:45.424345+0000 mon.c (mon.2) 47 : audit 0 from='client.? 172.21.15.78:0/3869706077' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:46.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:46 smithi078 bash[78274]: cluster 2023-10-01T10:31:45.485071+0000 mgr.y (mgr.25072) 363 : cluster 0 pgmap v288: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 762 op/s; 10391/96 objects degraded (10823.958%); 0 B/s, 26 objects/s recovering 2023-10-01T10:31:46.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:46 smithi078 bash[78274]: audit 2023-10-01T10:31:45.705130+0000 mon.a (mon.0) 489 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:31:46.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:46 smithi078 bash[78274]: audit 2023-10-01T10:31:45.706866+0000 mon.a (mon.0) 490 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:31:46.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:46 smithi078 bash[78274]: audit 2023-10-01T10:31:45.715371+0000 mon.a (mon.0) 491 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:31:46.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:46 smithi078 bash[78274]: audit 2023-10-01T10:31:45.802322+0000 mon.a (mon.0) 492 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:31:46.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:46 smithi078 bash[78274]: audit 2023-10-01T10:31:45.809040+0000 mon.a (mon.0) 493 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:31:46.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:46 smithi078 bash[78274]: audit 2023-10-01T10:31:45.812106+0000 mon.a (mon.0) 494 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:31:46.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:46 smithi078 bash[78274]: audit 2023-10-01T10:31:45.814145+0000 mon.a (mon.0) 495 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:31:46.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:46 smithi078 bash[78274]: audit 2023-10-01T10:31:45.816772+0000 mon.a (mon.0) 496 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:31:46.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:46 smithi173 bash[58816]: audit 2023-10-01T10:31:45.073921+0000 mon.a (mon.0) 485 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:31:46.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:46 smithi173 bash[58816]: audit 2023-10-01T10:31:45.082390+0000 mon.a (mon.0) 486 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:31:46.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:46 smithi173 bash[58816]: audit 2023-10-01T10:31:45.221853+0000 mon.a (mon.0) 487 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:31:46.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:46 smithi173 bash[58816]: audit 2023-10-01T10:31:45.229361+0000 mon.a (mon.0) 488 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:31:46.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:46 smithi173 bash[58816]: audit 2023-10-01T10:31:45.424345+0000 mon.c (mon.2) 47 : audit 0 from='client.? 172.21.15.78:0/3869706077' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:46.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:46 smithi173 bash[58816]: cluster 2023-10-01T10:31:45.485071+0000 mgr.y (mgr.25072) 363 : cluster 0 pgmap v288: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 762 op/s; 10391/96 objects degraded (10823.958%); 0 B/s, 26 objects/s recovering 2023-10-01T10:31:46.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:46 smithi173 bash[58816]: audit 2023-10-01T10:31:45.705130+0000 mon.a (mon.0) 489 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:31:46.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:46 smithi173 bash[58816]: audit 2023-10-01T10:31:45.706866+0000 mon.a (mon.0) 490 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:31:46.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:46 smithi173 bash[58816]: audit 2023-10-01T10:31:45.715371+0000 mon.a (mon.0) 491 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:31:46.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:46 smithi173 bash[58816]: audit 2023-10-01T10:31:45.802322+0000 mon.a (mon.0) 492 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:31:46.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:46 smithi173 bash[58816]: audit 2023-10-01T10:31:45.809040+0000 mon.a (mon.0) 493 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:31:46.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:46 smithi173 bash[58816]: audit 2023-10-01T10:31:45.812106+0000 mon.a (mon.0) 494 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:31:46.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:46 smithi173 bash[58816]: audit 2023-10-01T10:31:45.814145+0000 mon.a (mon.0) 495 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:31:46.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:46 smithi173 bash[58816]: audit 2023-10-01T10:31:45.816772+0000 mon.a (mon.0) 496 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:31:47.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:47 smithi078 bash[78274]: audit 2023-10-01T10:31:45.817071+0000 mgr.y (mgr.25072) 364 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:31:47.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:47 smithi078 bash[78274]: cephadm 2023-10-01T10:31:45.818335+0000 mgr.y (mgr.25072) 365 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:31:47.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:47 smithi078 bash[78274]: cluster 2023-10-01T10:31:46.837133+0000 mon.a (mon.0) 497 : cluster 3 Health check update: Degraded data redundancy: 10391/96 objects degraded (10823.958%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-01T10:31:47.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:47 smithi078 bash[75299]: audit 2023-10-01T10:31:45.817071+0000 mgr.y (mgr.25072) 364 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:31:47.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:47 smithi078 bash[75299]: cephadm 2023-10-01T10:31:45.818335+0000 mgr.y (mgr.25072) 365 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:31:47.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:47 smithi078 bash[75299]: cluster 2023-10-01T10:31:46.837133+0000 mon.a (mon.0) 497 : cluster 3 Health check update: Degraded data redundancy: 10391/96 objects degraded (10823.958%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-01T10:31:47.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:47 smithi173 bash[58816]: audit 2023-10-01T10:31:45.817071+0000 mgr.y (mgr.25072) 364 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:31:47.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:47 smithi173 bash[58816]: cephadm 2023-10-01T10:31:45.818335+0000 mgr.y (mgr.25072) 365 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:31:47.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:47 smithi173 bash[58816]: cluster 2023-10-01T10:31:46.837133+0000 mon.a (mon.0) 497 : cluster 3 Health check update: Degraded data redundancy: 10391/96 objects degraded (10823.958%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-01T10:31:48.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:48 smithi078 bash[78274]: audit 2023-10-01T10:31:47.396546+0000 mon.a (mon.0) 498 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:31:48.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:48 smithi078 bash[78274]: cluster 2023-10-01T10:31:47.485831+0000 mgr.y (mgr.25072) 366 : cluster 0 pgmap v289: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 492 op/s; 10391/96 objects degraded (10823.958%); 0 B/s, 17 objects/s recovering 2023-10-01T10:31:48.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:48 smithi078 bash[78274]: audit 2023-10-01T10:31:47.785792+0000 mon.a (mon.0) 499 : audit 0 from='client.? 172.21.15.78:0/1393268712' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:48.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:48 smithi078 bash[75299]: audit 2023-10-01T10:31:47.396546+0000 mon.a (mon.0) 498 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:31:48.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:48 smithi078 bash[75299]: cluster 2023-10-01T10:31:47.485831+0000 mgr.y (mgr.25072) 366 : cluster 0 pgmap v289: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 492 op/s; 10391/96 objects degraded (10823.958%); 0 B/s, 17 objects/s recovering 2023-10-01T10:31:48.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:48 smithi078 bash[75299]: audit 2023-10-01T10:31:47.785792+0000 mon.a (mon.0) 499 : audit 0 from='client.? 172.21.15.78:0/1393268712' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:48.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:48 smithi173 bash[58816]: audit 2023-10-01T10:31:47.396546+0000 mon.a (mon.0) 498 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:31:48.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:48 smithi173 bash[58816]: cluster 2023-10-01T10:31:47.485831+0000 mgr.y (mgr.25072) 366 : cluster 0 pgmap v289: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.0 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 492 op/s; 10391/96 objects degraded (10823.958%); 0 B/s, 17 objects/s recovering 2023-10-01T10:31:48.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:48 smithi173 bash[58816]: audit 2023-10-01T10:31:47.785792+0000 mon.a (mon.0) 499 : audit 0 from='client.? 172.21.15.78:0/1393268712' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:49.169 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:31:50.544 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:31:50 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:31:50] "GET /metrics HTTP/1.1" 200 36203 "" "Prometheus/2.33.4" 2023-10-01T10:31:50.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:50 smithi078 bash[75299]: cluster 2023-10-01T10:31:49.487191+0000 mgr.y (mgr.25072) 367 : cluster 0 pgmap v290: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 493 op/s; 10275/72 objects degraded (14270.833%); 0 B/s, 27 objects/s recovering 2023-10-01T10:31:50.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:50 smithi078 bash[75299]: audit 2023-10-01T10:31:50.144733+0000 mon.c (mon.2) 48 : audit 0 from='client.? 172.21.15.78:0/3603049782' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:50.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:50 smithi078 bash[78274]: cluster 2023-10-01T10:31:49.487191+0000 mgr.y (mgr.25072) 367 : cluster 0 pgmap v290: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 493 op/s; 10275/72 objects degraded (14270.833%); 0 B/s, 27 objects/s recovering 2023-10-01T10:31:50.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:50 smithi078 bash[78274]: audit 2023-10-01T10:31:50.144733+0000 mon.c (mon.2) 48 : audit 0 from='client.? 172.21.15.78:0/3603049782' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:50.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:50 smithi173 bash[58816]: cluster 2023-10-01T10:31:49.487191+0000 mgr.y (mgr.25072) 367 : cluster 0 pgmap v290: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 493 op/s; 10275/72 objects degraded (14270.833%); 0 B/s, 27 objects/s recovering 2023-10-01T10:31:50.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:50 smithi173 bash[58816]: audit 2023-10-01T10:31:50.144733+0000 mon.c (mon.2) 48 : audit 0 from='client.? 172.21.15.78:0/3603049782' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:52.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:52 smithi078 bash[78274]: cluster 2023-10-01T10:31:51.488181+0000 mgr.y (mgr.25072) 368 : cluster 0 pgmap v291: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 293 op/s; 10275/72 objects degraded (14270.833%); 0 B/s, 18 objects/s recovering 2023-10-01T10:31:52.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:52 smithi078 bash[78274]: cluster 2023-10-01T10:31:51.838930+0000 mon.a (mon.0) 500 : cluster 3 Health check update: Degraded data redundancy: 10275/72 objects degraded (14270.833%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-01T10:31:52.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:52 smithi078 bash[78274]: audit 2023-10-01T10:31:52.513869+0000 mon.c (mon.2) 49 : audit 0 from='client.? 172.21.15.78:0/4200008686' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:52.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:52 smithi078 bash[75299]: cluster 2023-10-01T10:31:51.488181+0000 mgr.y (mgr.25072) 368 : cluster 0 pgmap v291: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 293 op/s; 10275/72 objects degraded (14270.833%); 0 B/s, 18 objects/s recovering 2023-10-01T10:31:52.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:52 smithi078 bash[75299]: cluster 2023-10-01T10:31:51.838930+0000 mon.a (mon.0) 500 : cluster 3 Health check update: Degraded data redundancy: 10275/72 objects degraded (14270.833%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-01T10:31:52.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:52 smithi078 bash[75299]: audit 2023-10-01T10:31:52.513869+0000 mon.c (mon.2) 49 : audit 0 from='client.? 172.21.15.78:0/4200008686' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:52.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:52 smithi173 bash[58816]: cluster 2023-10-01T10:31:51.488181+0000 mgr.y (mgr.25072) 368 : cluster 0 pgmap v291: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 293 op/s; 10275/72 objects degraded (14270.833%); 0 B/s, 18 objects/s recovering 2023-10-01T10:31:52.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:52 smithi173 bash[58816]: cluster 2023-10-01T10:31:51.838930+0000 mon.a (mon.0) 500 : cluster 3 Health check update: Degraded data redundancy: 10275/72 objects degraded (14270.833%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-01T10:31:52.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:52 smithi173 bash[58816]: audit 2023-10-01T10:31:52.513869+0000 mon.c (mon.2) 49 : audit 0 from='client.? 172.21.15.78:0/4200008686' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:53.581 INFO:teuthology.orchestra.run.smithi078.stderr:pool 'unique_pool_2' removed 2023-10-01T10:31:53.807 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:53 smithi078 bash[75299]: audit 2023-10-01T10:31:52.709103+0000 mon.c (mon.2) 50 : audit 1 from='client.? 172.21.15.78:0/212409393' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T10:31:53.807 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:53 smithi078 bash[75299]: audit 2023-10-01T10:31:52.709703+0000 mon.a (mon.0) 501 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T10:31:53.807 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:53 smithi078 bash[78274]: audit 2023-10-01T10:31:52.709103+0000 mon.c (mon.2) 50 : audit 1 from='client.? 172.21.15.78:0/212409393' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T10:31:53.808 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:53 smithi078 bash[78274]: audit 2023-10-01T10:31:52.709703+0000 mon.a (mon.0) 501 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T10:31:53.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:53 smithi173 bash[58816]: audit 2023-10-01T10:31:52.709103+0000 mon.c (mon.2) 50 : audit 1 from='client.? 172.21.15.78:0/212409393' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T10:31:53.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:53 smithi173 bash[58816]: audit 2023-10-01T10:31:52.709703+0000 mon.a (mon.0) 501 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T10:31:54.413 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-01T10:31:54.415 INFO:tasks.radosbench:Beginning radosbench... 2023-10-01T10:31:54.415 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_3 2023-10-01T10:31:54.415 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_3 16 2023-10-01T10:31:54.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:54 smithi078 bash[75299]: cluster 2023-10-01T10:31:53.488988+0000 mgr.y (mgr.25072) 369 : cluster 0 pgmap v292: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 229 op/s; 10275/72 objects degraded (14270.833%); 0 B/s, 18 objects/s recovering 2023-10-01T10:31:54.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:54 smithi078 bash[75299]: cluster 2023-10-01T10:31:53.557105+0000 osd.1 (osd.1) 17 : cluster 3 failed to encode map e93 with expected crc 2023-10-01T10:31:54.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:54 smithi078 bash[75299]: audit 2023-10-01T10:31:53.559359+0000 mon.a (mon.0) 502 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-10-01T10:31:54.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:54 smithi078 bash[75299]: cluster 2023-10-01T10:31:53.559486+0000 mon.a (mon.0) 503 : cluster 0 osdmap e93: 8 total, 8 up, 8 in 2023-10-01T10:31:54.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:54 smithi078 bash[75299]: cluster 2023-10-01T10:31:53.561869+0000 osd.7 (osd.7) 23 : cluster 3 failed to encode map e93 with expected crc 2023-10-01T10:31:54.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:54 smithi078 bash[78274]: cluster 2023-10-01T10:31:53.488988+0000 mgr.y (mgr.25072) 369 : cluster 0 pgmap v292: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 229 op/s; 10275/72 objects degraded (14270.833%); 0 B/s, 18 objects/s recovering 2023-10-01T10:31:54.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:54 smithi078 bash[78274]: cluster 2023-10-01T10:31:53.557105+0000 osd.1 (osd.1) 17 : cluster 3 failed to encode map e93 with expected crc 2023-10-01T10:31:54.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:54 smithi078 bash[78274]: audit 2023-10-01T10:31:53.559359+0000 mon.a (mon.0) 502 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-10-01T10:31:54.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:54 smithi078 bash[78274]: cluster 2023-10-01T10:31:53.559486+0000 mon.a (mon.0) 503 : cluster 0 osdmap e93: 8 total, 8 up, 8 in 2023-10-01T10:31:54.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:54 smithi078 bash[78274]: cluster 2023-10-01T10:31:53.561869+0000 osd.7 (osd.7) 23 : cluster 3 failed to encode map e93 with expected crc 2023-10-01T10:31:54.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:54 smithi173 bash[58816]: cluster 2023-10-01T10:31:53.488988+0000 mgr.y (mgr.25072) 369 : cluster 0 pgmap v292: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 229 op/s; 10275/72 objects degraded (14270.833%); 0 B/s, 18 objects/s recovering 2023-10-01T10:31:54.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:54 smithi173 bash[58816]: cluster 2023-10-01T10:31:53.557105+0000 osd.1 (osd.1) 17 : cluster 3 failed to encode map e93 with expected crc 2023-10-01T10:31:54.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:54 smithi173 bash[58816]: audit 2023-10-01T10:31:53.559359+0000 mon.a (mon.0) 502 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-10-01T10:31:54.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:54 smithi173 bash[58816]: cluster 2023-10-01T10:31:53.559486+0000 mon.a (mon.0) 503 : cluster 0 osdmap e93: 8 total, 8 up, 8 in 2023-10-01T10:31:54.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:54 smithi173 bash[58816]: cluster 2023-10-01T10:31:53.561869+0000 osd.7 (osd.7) 23 : cluster 3 failed to encode map e93 with expected crc 2023-10-01T10:31:55.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:55 smithi078 bash[78274]: cluster 2023-10-01T10:31:54.798066+0000 osd.3 (osd.3) 16 : cluster 3 failed to encode map e93 with expected crc 2023-10-01T10:31:55.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:55 smithi078 bash[78274]: audit 2023-10-01T10:31:54.905695+0000 mon.c (mon.2) 51 : audit 0 from='client.? 172.21.15.78:0/1083994361' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:55.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:55 smithi078 bash[75299]: cluster 2023-10-01T10:31:54.798066+0000 osd.3 (osd.3) 16 : cluster 3 failed to encode map e93 with expected crc 2023-10-01T10:31:55.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:55 smithi078 bash[75299]: audit 2023-10-01T10:31:54.905695+0000 mon.c (mon.2) 51 : audit 0 from='client.? 172.21.15.78:0/1083994361' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:55.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:55 smithi173 bash[58816]: cluster 2023-10-01T10:31:54.798066+0000 osd.3 (osd.3) 16 : cluster 3 failed to encode map e93 with expected crc 2023-10-01T10:31:55.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:55 smithi173 bash[58816]: audit 2023-10-01T10:31:54.905695+0000 mon.c (mon.2) 51 : audit 0 from='client.? 172.21.15.78:0/1083994361' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:56.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:56 smithi078 bash[75299]: cluster 2023-10-01T10:31:54.621458+0000 osd.2 (osd.2) 14 : cluster 3 failed to encode map e93 with expected crc 2023-10-01T10:31:56.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:56 smithi078 bash[75299]: cluster 2023-10-01T10:31:55.490087+0000 mgr.y (mgr.25072) 370 : cluster 0 pgmap v294: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:31:56.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:56 smithi078 bash[75299]: cluster 2023-10-01T10:31:55.559490+0000 mon.a (mon.0) 504 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 10275/72 objects degraded (14270.833%), 6 pgs degraded, 6 pgs undersized) 2023-10-01T10:31:56.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:56 smithi078 bash[75299]: cluster 2023-10-01T10:31:55.559576+0000 mon.a (mon.0) 505 : cluster 1 Cluster is now healthy 2023-10-01T10:31:56.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:56 smithi078 bash[78274]: cluster 2023-10-01T10:31:54.621458+0000 osd.2 (osd.2) 14 : cluster 3 failed to encode map e93 with expected crc 2023-10-01T10:31:56.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:56 smithi078 bash[78274]: cluster 2023-10-01T10:31:55.490087+0000 mgr.y (mgr.25072) 370 : cluster 0 pgmap v294: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:31:56.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:56 smithi078 bash[78274]: cluster 2023-10-01T10:31:55.559490+0000 mon.a (mon.0) 504 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 10275/72 objects degraded (14270.833%), 6 pgs degraded, 6 pgs undersized) 2023-10-01T10:31:56.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:56 smithi078 bash[78274]: cluster 2023-10-01T10:31:55.559576+0000 mon.a (mon.0) 505 : cluster 1 Cluster is now healthy 2023-10-01T10:31:56.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:56 smithi173 bash[58816]: cluster 2023-10-01T10:31:54.621458+0000 osd.2 (osd.2) 14 : cluster 3 failed to encode map e93 with expected crc 2023-10-01T10:31:56.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:56 smithi173 bash[58816]: cluster 2023-10-01T10:31:55.490087+0000 mgr.y (mgr.25072) 370 : cluster 0 pgmap v294: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:31:56.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:56 smithi173 bash[58816]: cluster 2023-10-01T10:31:55.559490+0000 mon.a (mon.0) 504 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 10275/72 objects degraded (14270.833%), 6 pgs degraded, 6 pgs undersized) 2023-10-01T10:31:56.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:56 smithi173 bash[58816]: cluster 2023-10-01T10:31:55.559576+0000 mon.a (mon.0) 505 : cluster 1 Cluster is now healthy 2023-10-01T10:31:58.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:57 smithi173 bash[58816]: audit 2023-10-01T10:31:57.275776+0000 mon.c (mon.2) 52 : audit 0 from='client.? 172.21.15.78:0/144060116' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:58.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:31:57 smithi173 bash[58816]: cluster 2023-10-01T10:31:57.490744+0000 mgr.y (mgr.25072) 371 : cluster 0 pgmap v295: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:31:58.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:57 smithi078 bash[75299]: audit 2023-10-01T10:31:57.275776+0000 mon.c (mon.2) 52 : audit 0 from='client.? 172.21.15.78:0/144060116' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:58.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:31:57 smithi078 bash[75299]: cluster 2023-10-01T10:31:57.490744+0000 mgr.y (mgr.25072) 371 : cluster 0 pgmap v295: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:31:58.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:57 smithi078 bash[78274]: audit 2023-10-01T10:31:57.275776+0000 mon.c (mon.2) 52 : audit 0 from='client.? 172.21.15.78:0/144060116' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:31:58.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:31:57 smithi078 bash[78274]: cluster 2023-10-01T10:31:57.490744+0000 mgr.y (mgr.25072) 371 : cluster 0 pgmap v295: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:31:59.209 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:32:00.497 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:32:00 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:32:00] "GET /metrics HTTP/1.1" 200 36203 "" "Prometheus/2.33.4" 2023-10-01T10:32:00.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:00 smithi078 bash[78274]: cluster 2023-10-01T10:31:59.491784+0000 mgr.y (mgr.25072) 372 : cluster 0 pgmap v296: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:32:00.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:00 smithi078 bash[78274]: audit 2023-10-01T10:31:59.636405+0000 mon.a (mon.0) 506 : audit 0 from='client.? 172.21.15.78:0/401067243' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:00.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:00 smithi078 bash[75299]: cluster 2023-10-01T10:31:59.491784+0000 mgr.y (mgr.25072) 372 : cluster 0 pgmap v296: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:32:00.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:00 smithi078 bash[75299]: audit 2023-10-01T10:31:59.636405+0000 mon.a (mon.0) 506 : audit 0 from='client.? 172.21.15.78:0/401067243' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:00.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:00 smithi173 bash[58816]: cluster 2023-10-01T10:31:59.491784+0000 mgr.y (mgr.25072) 372 : cluster 0 pgmap v296: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:32:00.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:00 smithi173 bash[58816]: audit 2023-10-01T10:31:59.636405+0000 mon.a (mon.0) 506 : audit 0 from='client.? 172.21.15.78:0/401067243' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:01.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:01 smithi078 bash[78274]: audit 2023-10-01T10:32:00.833943+0000 mon.a (mon.0) 507 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:32:01.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:01 smithi078 bash[75299]: audit 2023-10-01T10:32:00.833943+0000 mon.a (mon.0) 507 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:32:01.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:01 smithi173 bash[58816]: audit 2023-10-01T10:32:00.833943+0000 mon.a (mon.0) 507 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:32:02.824 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:02 smithi078 bash[78274]: audit 2023-10-01T10:32:00.834337+0000 mgr.y (mgr.25072) 373 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:32:02.824 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:02 smithi078 bash[78274]: cephadm 2023-10-01T10:32:00.836347+0000 mgr.y (mgr.25072) 374 : cephadm 1 Upgrade: osd.1 is safe to restart 2023-10-01T10:32:02.824 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:02 smithi078 bash[78274]: cluster 2023-10-01T10:32:01.492477+0000 mgr.y (mgr.25072) 375 : cluster 0 pgmap v297: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:32:02.824 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:02 smithi078 bash[78274]: audit 2023-10-01T10:32:02.004824+0000 mon.a (mon.0) 508 : audit 0 from='client.? 172.21.15.78:0/2702623735' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:02.824 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:02 smithi078 bash[78274]: audit 2023-10-01T10:32:02.400845+0000 mon.a (mon.0) 509 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:32:02.824 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:02 smithi078 bash[78274]: audit 2023-10-01T10:32:02.402400+0000 mon.a (mon.0) 510 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:32:02.825 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:02 smithi078 bash[78274]: audit 2023-10-01T10:32:02.527421+0000 mon.a (mon.0) 511 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:32:02.825 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:02 smithi078 bash[75299]: audit 2023-10-01T10:32:00.834337+0000 mgr.y (mgr.25072) 373 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:32:02.825 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:02 smithi078 bash[75299]: cephadm 2023-10-01T10:32:00.836347+0000 mgr.y (mgr.25072) 374 : cephadm 1 Upgrade: osd.1 is safe to restart 2023-10-01T10:32:02.825 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:02 smithi078 bash[75299]: cluster 2023-10-01T10:32:01.492477+0000 mgr.y (mgr.25072) 375 : cluster 0 pgmap v297: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:32:02.825 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:02 smithi078 bash[75299]: audit 2023-10-01T10:32:02.004824+0000 mon.a (mon.0) 508 : audit 0 from='client.? 172.21.15.78:0/2702623735' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:02.825 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:02 smithi078 bash[75299]: audit 2023-10-01T10:32:02.400845+0000 mon.a (mon.0) 509 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:32:02.825 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:02 smithi078 bash[75299]: audit 2023-10-01T10:32:02.402400+0000 mon.a (mon.0) 510 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:32:02.825 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:02 smithi078 bash[75299]: audit 2023-10-01T10:32:02.527421+0000 mon.a (mon.0) 511 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:32:02.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:02 smithi173 bash[58816]: audit 2023-10-01T10:32:00.834337+0000 mgr.y (mgr.25072) 373 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-10-01T10:32:02.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:02 smithi173 bash[58816]: cephadm 2023-10-01T10:32:00.836347+0000 mgr.y (mgr.25072) 374 : cephadm 1 Upgrade: osd.1 is safe to restart 2023-10-01T10:32:02.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:02 smithi173 bash[58816]: cluster 2023-10-01T10:32:01.492477+0000 mgr.y (mgr.25072) 375 : cluster 0 pgmap v297: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:32:02.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:02 smithi173 bash[58816]: audit 2023-10-01T10:32:02.004824+0000 mon.a (mon.0) 508 : audit 0 from='client.? 172.21.15.78:0/2702623735' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:02.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:02 smithi173 bash[58816]: audit 2023-10-01T10:32:02.400845+0000 mon.a (mon.0) 509 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:32:02.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:02 smithi173 bash[58816]: audit 2023-10-01T10:32:02.402400+0000 mon.a (mon.0) 510 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:32:02.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:02 smithi173 bash[58816]: audit 2023-10-01T10:32:02.527421+0000 mon.a (mon.0) 511 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:32:03.694 INFO:teuthology.orchestra.run.smithi078.stderr:pool 'unique_pool_3' created 2023-10-01T10:32:03.980 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:03 smithi078 bash[78274]: audit 2023-10-01T10:32:03.203610+0000 mon.a (mon.0) 512 : audit 1 from='client.? 172.21.15.78:0/3204120227' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-10-01T10:32:03.980 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:03 smithi078 bash[78274]: cluster 2023-10-01T10:32:03.492991+0000 mgr.y (mgr.25072) 376 : cluster 0 pgmap v298: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:32:03.980 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:03 smithi078 bash[75299]: audit 2023-10-01T10:32:03.203610+0000 mon.a (mon.0) 512 : audit 1 from='client.? 172.21.15.78:0/3204120227' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-10-01T10:32:03.980 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:03 smithi078 bash[75299]: cluster 2023-10-01T10:32:03.492991+0000 mgr.y (mgr.25072) 376 : cluster 0 pgmap v298: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:32:04.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:03 smithi173 bash[58816]: audit 2023-10-01T10:32:03.203610+0000 mon.a (mon.0) 512 : audit 1 from='client.? 172.21.15.78:0/3204120227' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-10-01T10:32:04.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:03 smithi173 bash[58816]: cluster 2023-10-01T10:32:03.492991+0000 mgr.y (mgr.25072) 376 : cluster 0 pgmap v298: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:32:05.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:04 smithi078 bash[75299]: cluster 2023-10-01T10:32:03.684556+0000 osd.3 (osd.3) 17 : cluster 3 failed to encode map e94 with expected crc 2023-10-01T10:32:05.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:04 smithi078 bash[75299]: audit 2023-10-01T10:32:03.686389+0000 mon.a (mon.0) 513 : audit 1 from='client.? 172.21.15.78:0/3204120227' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-10-01T10:32:05.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:04 smithi078 bash[75299]: cluster 2023-10-01T10:32:03.686448+0000 mon.a (mon.0) 514 : cluster 0 osdmap e94: 8 total, 8 up, 8 in 2023-10-01T10:32:05.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:04 smithi078 bash[75299]: cluster 2023-10-01T10:32:03.690215+0000 osd.7 (osd.7) 24 : cluster 3 failed to encode map e94 with expected crc 2023-10-01T10:32:05.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:04 smithi078 bash[75299]: cephadm 2023-10-01T10:32:04.005955+0000 mgr.y (mgr.25072) 377 : cephadm 1 Upgrade: Updating osd.1 2023-10-01T10:32:05.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:04 smithi078 bash[75299]: audit 2023-10-01T10:32:04.014515+0000 mon.a (mon.0) 515 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:32:05.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:04 smithi078 bash[75299]: audit 2023-10-01T10:32:04.015651+0000 mon.a (mon.0) 516 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-01T10:32:05.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:04 smithi078 bash[75299]: audit 2023-10-01T10:32:04.016557+0000 mon.a (mon.0) 517 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:32:05.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:04 smithi078 bash[75299]: cephadm 2023-10-01T10:32:04.019244+0000 mgr.y (mgr.25072) 378 : cephadm 1 Deploying daemon osd.1 on smithi078 2023-10-01T10:32:05.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:04 smithi078 bash[75299]: audit 2023-10-01T10:32:04.347265+0000 mon.a (mon.0) 518 : audit 0 from='client.? 172.21.15.78:0/1941993893' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:05.085 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:04 smithi078 bash[75299]: cluster 2023-10-01T10:32:04.692726+0000 mon.a (mon.0) 519 : cluster 0 osdmap e95: 8 total, 8 up, 8 in 2023-10-01T10:32:05.085 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:04 smithi078 bash[78274]: cluster 2023-10-01T10:32:03.684556+0000 osd.3 (osd.3) 17 : cluster 3 failed to encode map e94 with expected crc 2023-10-01T10:32:05.085 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:04 smithi078 bash[78274]: audit 2023-10-01T10:32:03.686389+0000 mon.a (mon.0) 513 : audit 1 from='client.? 172.21.15.78:0/3204120227' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-10-01T10:32:05.085 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:04 smithi078 bash[78274]: cluster 2023-10-01T10:32:03.686448+0000 mon.a (mon.0) 514 : cluster 0 osdmap e94: 8 total, 8 up, 8 in 2023-10-01T10:32:05.085 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:04 smithi078 bash[78274]: cluster 2023-10-01T10:32:03.690215+0000 osd.7 (osd.7) 24 : cluster 3 failed to encode map e94 with expected crc 2023-10-01T10:32:05.085 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:04 smithi078 bash[78274]: cephadm 2023-10-01T10:32:04.005955+0000 mgr.y (mgr.25072) 377 : cephadm 1 Upgrade: Updating osd.1 2023-10-01T10:32:05.086 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:04 smithi078 bash[78274]: audit 2023-10-01T10:32:04.014515+0000 mon.a (mon.0) 515 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:32:05.086 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:04 smithi078 bash[78274]: audit 2023-10-01T10:32:04.015651+0000 mon.a (mon.0) 516 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-01T10:32:05.086 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:04 smithi078 bash[78274]: audit 2023-10-01T10:32:04.016557+0000 mon.a (mon.0) 517 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:32:05.086 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:04 smithi078 bash[78274]: cephadm 2023-10-01T10:32:04.019244+0000 mgr.y (mgr.25072) 378 : cephadm 1 Deploying daemon osd.1 on smithi078 2023-10-01T10:32:05.086 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:04 smithi078 bash[78274]: audit 2023-10-01T10:32:04.347265+0000 mon.a (mon.0) 518 : audit 0 from='client.? 172.21.15.78:0/1941993893' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:05.086 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:04 smithi078 bash[78274]: cluster 2023-10-01T10:32:04.692726+0000 mon.a (mon.0) 519 : cluster 0 osdmap e95: 8 total, 8 up, 8 in 2023-10-01T10:32:05.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:04 smithi173 bash[58816]: cluster 2023-10-01T10:32:03.684556+0000 osd.3 (osd.3) 17 : cluster 3 failed to encode map e94 with expected crc 2023-10-01T10:32:05.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:04 smithi173 bash[58816]: audit 2023-10-01T10:32:03.686389+0000 mon.a (mon.0) 513 : audit 1 from='client.? 172.21.15.78:0/3204120227' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-10-01T10:32:05.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:04 smithi173 bash[58816]: cluster 2023-10-01T10:32:03.686448+0000 mon.a (mon.0) 514 : cluster 0 osdmap e94: 8 total, 8 up, 8 in 2023-10-01T10:32:05.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:04 smithi173 bash[58816]: cluster 2023-10-01T10:32:03.690215+0000 osd.7 (osd.7) 24 : cluster 3 failed to encode map e94 with expected crc 2023-10-01T10:32:05.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:04 smithi173 bash[58816]: cephadm 2023-10-01T10:32:04.005955+0000 mgr.y (mgr.25072) 377 : cephadm 1 Upgrade: Updating osd.1 2023-10-01T10:32:05.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:04 smithi173 bash[58816]: audit 2023-10-01T10:32:04.014515+0000 mon.a (mon.0) 515 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:32:05.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:04 smithi173 bash[58816]: audit 2023-10-01T10:32:04.015651+0000 mon.a (mon.0) 516 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-10-01T10:32:05.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:04 smithi173 bash[58816]: audit 2023-10-01T10:32:04.016557+0000 mon.a (mon.0) 517 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:32:05.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:04 smithi173 bash[58816]: cephadm 2023-10-01T10:32:04.019244+0000 mgr.y (mgr.25072) 378 : cephadm 1 Deploying daemon osd.1 on smithi078 2023-10-01T10:32:05.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:04 smithi173 bash[58816]: audit 2023-10-01T10:32:04.347265+0000 mon.a (mon.0) 518 : audit 0 from='client.? 172.21.15.78:0/1941993893' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:05.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:04 smithi173 bash[58816]: cluster 2023-10-01T10:32:04.692726+0000 mon.a (mon.0) 519 : cluster 0 osdmap e95: 8 total, 8 up, 8 in 2023-10-01T10:32:05.207 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_3 rados --yes-i-really-mean-it || true 2023-10-01T10:32:06.006 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:05 smithi078 bash[75299]: cluster 2023-10-01T10:32:04.687370+0000 osd.1 (osd.1) 18 : cluster 3 failed to encode map e95 with expected crc 2023-10-01T10:32:06.006 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:05 smithi078 bash[75299]: cluster 2023-10-01T10:32:04.690003+0000 osd.7 (osd.7) 25 : cluster 3 failed to encode map e95 with expected crc 2023-10-01T10:32:06.006 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:05 smithi078 bash[75299]: cluster 2023-10-01T10:32:04.692891+0000 osd.3 (osd.3) 18 : cluster 3 failed to encode map e95 with expected crc 2023-10-01T10:32:06.006 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:05 smithi078 bash[75299]: cluster 2023-10-01T10:32:04.693232+0000 osd.1 (osd.1) 19 : cluster 3 failed to encode map e95 with expected crc 2023-10-01T10:32:06.007 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:05 smithi078 bash[75299]: cluster 2023-10-01T10:32:04.695113+0000 osd.5 (osd.5) 12 : cluster 3 failed to encode map e95 with expected crc 2023-10-01T10:32:06.007 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:05 smithi078 bash[75299]: cluster 2023-10-01T10:32:04.695287+0000 osd.4 (osd.4) 15 : cluster 3 failed to encode map e95 with expected crc 2023-10-01T10:32:06.007 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:05 smithi078 bash[75299]: cluster 2023-10-01T10:32:04.695448+0000 osd.6 (osd.6) 12 : cluster 3 failed to encode map e95 with expected crc 2023-10-01T10:32:06.007 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:05 smithi078 bash[75299]: cluster 2023-10-01T10:32:05.493592+0000 mgr.y (mgr.25072) 379 : cluster 0 pgmap v301: 89 pgs: 7 creating+peering, 6 unknown, 76 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:32:06.007 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:05 smithi078 bash[78274]: cluster 2023-10-01T10:32:04.687370+0000 osd.1 (osd.1) 18 : cluster 3 failed to encode map e95 with expected crc 2023-10-01T10:32:06.008 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:05 smithi078 bash[78274]: cluster 2023-10-01T10:32:04.690003+0000 osd.7 (osd.7) 25 : cluster 3 failed to encode map e95 with expected crc 2023-10-01T10:32:06.008 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:05 smithi078 bash[78274]: cluster 2023-10-01T10:32:04.692891+0000 osd.3 (osd.3) 18 : cluster 3 failed to encode map e95 with expected crc 2023-10-01T10:32:06.008 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:05 smithi078 bash[78274]: cluster 2023-10-01T10:32:04.693232+0000 osd.1 (osd.1) 19 : cluster 3 failed to encode map e95 with expected crc 2023-10-01T10:32:06.008 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:05 smithi078 bash[78274]: cluster 2023-10-01T10:32:04.695113+0000 osd.5 (osd.5) 12 : cluster 3 failed to encode map e95 with expected crc 2023-10-01T10:32:06.009 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:05 smithi078 bash[78274]: cluster 2023-10-01T10:32:04.695287+0000 osd.4 (osd.4) 15 : cluster 3 failed to encode map e95 with expected crc 2023-10-01T10:32:06.009 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:05 smithi078 bash[78274]: cluster 2023-10-01T10:32:04.695448+0000 osd.6 (osd.6) 12 : cluster 3 failed to encode map e95 with expected crc 2023-10-01T10:32:06.009 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:05 smithi078 bash[78274]: cluster 2023-10-01T10:32:05.493592+0000 mgr.y (mgr.25072) 379 : cluster 0 pgmap v301: 89 pgs: 7 creating+peering, 6 unknown, 76 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:32:06.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:05 smithi173 bash[58816]: cluster 2023-10-01T10:32:04.687370+0000 osd.1 (osd.1) 18 : cluster 3 failed to encode map e95 with expected crc 2023-10-01T10:32:06.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:05 smithi173 bash[58816]: cluster 2023-10-01T10:32:04.690003+0000 osd.7 (osd.7) 25 : cluster 3 failed to encode map e95 with expected crc 2023-10-01T10:32:06.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:05 smithi173 bash[58816]: cluster 2023-10-01T10:32:04.692891+0000 osd.3 (osd.3) 18 : cluster 3 failed to encode map e95 with expected crc 2023-10-01T10:32:06.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:05 smithi173 bash[58816]: cluster 2023-10-01T10:32:04.693232+0000 osd.1 (osd.1) 19 : cluster 3 failed to encode map e95 with expected crc 2023-10-01T10:32:06.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:05 smithi173 bash[58816]: cluster 2023-10-01T10:32:04.695113+0000 osd.5 (osd.5) 12 : cluster 3 failed to encode map e95 with expected crc 2023-10-01T10:32:06.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:05 smithi173 bash[58816]: cluster 2023-10-01T10:32:04.695287+0000 osd.4 (osd.4) 15 : cluster 3 failed to encode map e95 with expected crc 2023-10-01T10:32:06.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:05 smithi173 bash[58816]: cluster 2023-10-01T10:32:04.695448+0000 osd.6 (osd.6) 12 : cluster 3 failed to encode map e95 with expected crc 2023-10-01T10:32:06.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:05 smithi173 bash[58816]: cluster 2023-10-01T10:32:05.493592+0000 mgr.y (mgr.25072) 379 : cluster 0 pgmap v301: 89 pgs: 7 creating+peering, 6 unknown, 76 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:32:07.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:06 smithi078 bash[75299]: cluster 2023-10-01T10:32:05.705385+0000 osd.2 (osd.2) 15 : cluster 3 failed to encode map e96 with expected crc 2023-10-01T10:32:07.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:06 smithi078 bash[75299]: cluster 2023-10-01T10:32:05.708521+0000 osd.7 (osd.7) 26 : cluster 3 failed to encode map e96 with expected crc 2023-10-01T10:32:07.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:06 smithi078 bash[75299]: cluster 2023-10-01T10:32:05.709759+0000 mon.a (mon.0) 520 : cluster 0 osdmap e96: 8 total, 8 up, 8 in 2023-10-01T10:32:07.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:06 smithi078 bash[78274]: cluster 2023-10-01T10:32:05.705385+0000 osd.2 (osd.2) 15 : cluster 3 failed to encode map e96 with expected crc 2023-10-01T10:32:07.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:06 smithi078 bash[78274]: cluster 2023-10-01T10:32:05.708521+0000 osd.7 (osd.7) 26 : cluster 3 failed to encode map e96 with expected crc 2023-10-01T10:32:07.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:06 smithi078 bash[78274]: cluster 2023-10-01T10:32:05.709759+0000 mon.a (mon.0) 520 : cluster 0 osdmap e96: 8 total, 8 up, 8 in 2023-10-01T10:32:07.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:06 smithi173 bash[58816]: cluster 2023-10-01T10:32:05.705385+0000 osd.2 (osd.2) 15 : cluster 3 failed to encode map e96 with expected crc 2023-10-01T10:32:07.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:06 smithi173 bash[58816]: cluster 2023-10-01T10:32:05.708521+0000 osd.7 (osd.7) 26 : cluster 3 failed to encode map e96 with expected crc 2023-10-01T10:32:07.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:06 smithi173 bash[58816]: cluster 2023-10-01T10:32:05.709759+0000 mon.a (mon.0) 520 : cluster 0 osdmap e96: 8 total, 8 up, 8 in 2023-10-01T10:32:08.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:07 smithi078 bash[78274]: audit 2023-10-01T10:32:06.709379+0000 mon.c (mon.2) 53 : audit 0 from='client.? 172.21.15.78:0/321003741' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:08.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:07 smithi078 bash[78274]: cluster 2023-10-01T10:32:07.494102+0000 mgr.y (mgr.25072) 380 : cluster 0 pgmap v303: 89 pgs: 7 creating+peering, 6 unknown, 76 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:32:08.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:07 smithi078 bash[75299]: audit 2023-10-01T10:32:06.709379+0000 mon.c (mon.2) 53 : audit 0 from='client.? 172.21.15.78:0/321003741' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:08.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:07 smithi078 bash[75299]: cluster 2023-10-01T10:32:07.494102+0000 mgr.y (mgr.25072) 380 : cluster 0 pgmap v303: 89 pgs: 7 creating+peering, 6 unknown, 76 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:32:08.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:07 smithi173 bash[58816]: audit 2023-10-01T10:32:06.709379+0000 mon.c (mon.2) 53 : audit 0 from='client.? 172.21.15.78:0/321003741' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:08.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:07 smithi173 bash[58816]: cluster 2023-10-01T10:32:07.494102+0000 mgr.y (mgr.25072) 380 : cluster 0 pgmap v303: 89 pgs: 7 creating+peering, 6 unknown, 76 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:32:09.469 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:09 smithi078 bash[75299]: audit 2023-10-01T10:32:09.083083+0000 mon.c (mon.2) 54 : audit 0 from='client.? 172.21.15.78:0/1834711543' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:09.469 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:09 smithi078 bash[78274]: audit 2023-10-01T10:32:09.083083+0000 mon.c (mon.2) 54 : audit 0 from='client.? 172.21.15.78:0/1834711543' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:09.470 INFO:journalctl@ceph.osd.1.smithi078.stdout:Oct 01 10:32:09 smithi078 systemd[1]: Stopping Ceph osd.1 for d8016480-6041-11ee-8db5-212e2dc638e7... 2023-10-01T10:32:09.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:09 smithi173 bash[58816]: audit 2023-10-01T10:32:09.083083+0000 mon.c (mon.2) 54 : audit 0 from='client.? 172.21.15.78:0/1834711543' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:09.791 INFO:journalctl@ceph.osd.1.smithi078.stdout:Oct 01 10:32:09 smithi078 bash[41042]: debug 2023-10-01T10:32:09.464+0000 7f9cedcff700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-01T10:32:09.791 INFO:journalctl@ceph.osd.1.smithi078.stdout:Oct 01 10:32:09 smithi078 bash[41042]: debug 2023-10-01T10:32:09.464+0000 7f9cedcff700 -1 osd.1 96 *** Got signal Terminated *** 2023-10-01T10:32:09.792 INFO:journalctl@ceph.osd.1.smithi078.stdout:Oct 01 10:32:09 smithi078 bash[41042]: debug 2023-10-01T10:32:09.464+0000 7f9cedcff700 -1 osd.1 96 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-01T10:32:10.428 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:32:10.460 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:10 smithi078 bash[78274]: cluster 2023-10-01T10:32:09.466695+0000 mon.a (mon.0) 521 : cluster 1 osd.1 marked itself down and dead 2023-10-01T10:32:10.460 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:10 smithi078 bash[78274]: cluster 2023-10-01T10:32:09.495186+0000 mgr.y (mgr.25072) 381 : cluster 0 pgmap v304: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:32:10.460 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:32:10 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:32:10] "GET /metrics HTTP/1.1" 200 33846 "" "Prometheus/2.33.4" 2023-10-01T10:32:10.461 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:10 smithi078 bash[75299]: cluster 2023-10-01T10:32:09.466695+0000 mon.a (mon.0) 521 : cluster 1 osd.1 marked itself down and dead 2023-10-01T10:32:10.461 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:10 smithi078 bash[75299]: cluster 2023-10-01T10:32:09.495186+0000 mgr.y (mgr.25072) 381 : cluster 0 pgmap v304: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:32:10.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:10 smithi173 bash[58816]: cluster 2023-10-01T10:32:09.466695+0000 mon.a (mon.0) 521 : cluster 1 osd.1 marked itself down and dead 2023-10-01T10:32:10.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:10 smithi173 bash[58816]: cluster 2023-10-01T10:32:09.495186+0000 mgr.y (mgr.25072) 381 : cluster 0 pgmap v304: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:32:11.583 INFO:journalctl@ceph.osd.1.smithi078.stdout:Oct 01 10:32:11 smithi078 bash[90745]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd-1 2023-10-01T10:32:11.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:11 smithi078 bash[75299]: cluster 2023-10-01T10:32:10.137352+0000 mon.a (mon.0) 522 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-01T10:32:11.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:11 smithi078 bash[75299]: cluster 2023-10-01T10:32:10.150746+0000 osd.2 (osd.2) 16 : cluster 3 failed to encode map e97 with expected crc 2023-10-01T10:32:11.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:11 smithi078 bash[75299]: cluster 2023-10-01T10:32:10.152979+0000 mon.a (mon.0) 523 : cluster 0 osdmap e97: 8 total, 7 up, 8 in 2023-10-01T10:32:11.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:11 smithi078 bash[75299]: cluster 2023-10-01T10:32:10.160917+0000 osd.7 (osd.7) 27 : cluster 3 failed to encode map e97 with expected crc 2023-10-01T10:32:11.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:11 smithi078 bash[78274]: cluster 2023-10-01T10:32:10.137352+0000 mon.a (mon.0) 522 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-01T10:32:11.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:11 smithi078 bash[78274]: cluster 2023-10-01T10:32:10.150746+0000 osd.2 (osd.2) 16 : cluster 3 failed to encode map e97 with expected crc 2023-10-01T10:32:11.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:11 smithi078 bash[78274]: cluster 2023-10-01T10:32:10.152979+0000 mon.a (mon.0) 523 : cluster 0 osdmap e97: 8 total, 7 up, 8 in 2023-10-01T10:32:11.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:11 smithi078 bash[78274]: cluster 2023-10-01T10:32:10.160917+0000 osd.7 (osd.7) 27 : cluster 3 failed to encode map e97 with expected crc 2023-10-01T10:32:11.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:11 smithi173 bash[58816]: cluster 2023-10-01T10:32:10.137352+0000 mon.a (mon.0) 522 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-01T10:32:11.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:11 smithi173 bash[58816]: cluster 2023-10-01T10:32:10.150746+0000 osd.2 (osd.2) 16 : cluster 3 failed to encode map e97 with expected crc 2023-10-01T10:32:11.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:11 smithi173 bash[58816]: cluster 2023-10-01T10:32:10.152979+0000 mon.a (mon.0) 523 : cluster 0 osdmap e97: 8 total, 7 up, 8 in 2023-10-01T10:32:11.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:11 smithi173 bash[58816]: cluster 2023-10-01T10:32:10.160917+0000 osd.7 (osd.7) 27 : cluster 3 failed to encode map e97 with expected crc 2023-10-01T10:32:12.333 INFO:journalctl@ceph.osd.1.smithi078.stdout:Oct 01 10:32:11 smithi078 bash[90884]: Error response from daemon: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd-1 2023-10-01T10:32:12.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:12 smithi078 bash[78274]: cluster 2023-10-01T10:32:11.154414+0000 osd.3 (osd.3) 19 : cluster 3 failed to encode map e98 with expected crc 2023-10-01T10:32:12.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:12 smithi078 bash[78274]: cluster 2023-10-01T10:32:11.156698+0000 osd.7 (osd.7) 28 : cluster 3 failed to encode map e98 with expected crc 2023-10-01T10:32:12.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:12 smithi078 bash[78274]: cluster 2023-10-01T10:32:11.157444+0000 mon.a (mon.0) 524 : cluster 0 osdmap e98: 8 total, 7 up, 8 in 2023-10-01T10:32:12.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:12 smithi078 bash[78274]: cluster 2023-10-01T10:32:11.157771+0000 osd.2 (osd.2) 17 : cluster 3 failed to encode map e98 with expected crc 2023-10-01T10:32:12.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:12 smithi078 bash[78274]: cluster 2023-10-01T10:32:11.157880+0000 osd.3 (osd.3) 20 : cluster 3 failed to encode map e98 with expected crc 2023-10-01T10:32:12.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:12 smithi078 bash[78274]: cluster 2023-10-01T10:32:11.159818+0000 osd.6 (osd.6) 13 : cluster 3 failed to encode map e98 with expected crc 2023-10-01T10:32:12.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:12 smithi078 bash[78274]: cluster 2023-10-01T10:32:11.159869+0000 osd.5 (osd.5) 13 : cluster 3 failed to encode map e98 with expected crc 2023-10-01T10:32:12.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:12 smithi078 bash[78274]: cluster 2023-10-01T10:32:11.160092+0000 osd.4 (osd.4) 16 : cluster 3 failed to encode map e98 with expected crc 2023-10-01T10:32:12.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:12 smithi078 bash[78274]: audit 2023-10-01T10:32:11.450779+0000 mon.a (mon.0) 525 : audit 0 from='client.? 172.21.15.78:0/2369312917' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:12.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:12 smithi078 bash[78274]: cluster 2023-10-01T10:32:11.495860+0000 mgr.y (mgr.25072) 382 : cluster 0 pgmap v307: 89 pgs: 13 stale+active+clean, 76 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:32:12.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:12 smithi078 bash[75299]: cluster 2023-10-01T10:32:11.154414+0000 osd.3 (osd.3) 19 : cluster 3 failed to encode map e98 with expected crc 2023-10-01T10:32:12.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:12 smithi078 bash[75299]: cluster 2023-10-01T10:32:11.156698+0000 osd.7 (osd.7) 28 : cluster 3 failed to encode map e98 with expected crc 2023-10-01T10:32:12.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:12 smithi078 bash[75299]: cluster 2023-10-01T10:32:11.157444+0000 mon.a (mon.0) 524 : cluster 0 osdmap e98: 8 total, 7 up, 8 in 2023-10-01T10:32:12.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:12 smithi078 bash[75299]: cluster 2023-10-01T10:32:11.157771+0000 osd.2 (osd.2) 17 : cluster 3 failed to encode map e98 with expected crc 2023-10-01T10:32:12.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:12 smithi078 bash[75299]: cluster 2023-10-01T10:32:11.157880+0000 osd.3 (osd.3) 20 : cluster 3 failed to encode map e98 with expected crc 2023-10-01T10:32:12.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:12 smithi078 bash[75299]: cluster 2023-10-01T10:32:11.159818+0000 osd.6 (osd.6) 13 : cluster 3 failed to encode map e98 with expected crc 2023-10-01T10:32:12.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:12 smithi078 bash[75299]: cluster 2023-10-01T10:32:11.159869+0000 osd.5 (osd.5) 13 : cluster 3 failed to encode map e98 with expected crc 2023-10-01T10:32:12.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:12 smithi078 bash[75299]: cluster 2023-10-01T10:32:11.160092+0000 osd.4 (osd.4) 16 : cluster 3 failed to encode map e98 with expected crc 2023-10-01T10:32:12.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:12 smithi078 bash[75299]: audit 2023-10-01T10:32:11.450779+0000 mon.a (mon.0) 525 : audit 0 from='client.? 172.21.15.78:0/2369312917' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:12.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:12 smithi078 bash[75299]: cluster 2023-10-01T10:32:11.495860+0000 mgr.y (mgr.25072) 382 : cluster 0 pgmap v307: 89 pgs: 13 stale+active+clean, 76 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:32:12.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:12 smithi173 bash[58816]: cluster 2023-10-01T10:32:11.154414+0000 osd.3 (osd.3) 19 : cluster 3 failed to encode map e98 with expected crc 2023-10-01T10:32:12.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:12 smithi173 bash[58816]: cluster 2023-10-01T10:32:11.156698+0000 osd.7 (osd.7) 28 : cluster 3 failed to encode map e98 with expected crc 2023-10-01T10:32:12.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:12 smithi173 bash[58816]: cluster 2023-10-01T10:32:11.157444+0000 mon.a (mon.0) 524 : cluster 0 osdmap e98: 8 total, 7 up, 8 in 2023-10-01T10:32:12.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:12 smithi173 bash[58816]: cluster 2023-10-01T10:32:11.157771+0000 osd.2 (osd.2) 17 : cluster 3 failed to encode map e98 with expected crc 2023-10-01T10:32:12.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:12 smithi173 bash[58816]: cluster 2023-10-01T10:32:11.157880+0000 osd.3 (osd.3) 20 : cluster 3 failed to encode map e98 with expected crc 2023-10-01T10:32:12.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:12 smithi173 bash[58816]: cluster 2023-10-01T10:32:11.159818+0000 osd.6 (osd.6) 13 : cluster 3 failed to encode map e98 with expected crc 2023-10-01T10:32:12.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:12 smithi173 bash[58816]: cluster 2023-10-01T10:32:11.159869+0000 osd.5 (osd.5) 13 : cluster 3 failed to encode map e98 with expected crc 2023-10-01T10:32:12.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:12 smithi173 bash[58816]: cluster 2023-10-01T10:32:11.160092+0000 osd.4 (osd.4) 16 : cluster 3 failed to encode map e98 with expected crc 2023-10-01T10:32:12.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:12 smithi173 bash[58816]: audit 2023-10-01T10:32:11.450779+0000 mon.a (mon.0) 525 : audit 0 from='client.? 172.21.15.78:0/2369312917' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:12.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:12 smithi173 bash[58816]: cluster 2023-10-01T10:32:11.495860+0000 mgr.y (mgr.25072) 382 : cluster 0 pgmap v307: 89 pgs: 13 stale+active+clean, 76 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:32:14.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:14 smithi078 bash[75299]: cluster 2023-10-01T10:32:13.496543+0000 mgr.y (mgr.25072) 383 : cluster 0 pgmap v308: 89 pgs: 13 stale+active+clean, 76 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:32:14.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:14 smithi078 bash[75299]: audit 2023-10-01T10:32:13.817816+0000 mon.c (mon.2) 55 : audit 0 from='client.? 172.21.15.78:0/2409904483' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:14.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:14 smithi078 bash[78274]: cluster 2023-10-01T10:32:13.496543+0000 mgr.y (mgr.25072) 383 : cluster 0 pgmap v308: 89 pgs: 13 stale+active+clean, 76 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:32:14.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:14 smithi078 bash[78274]: audit 2023-10-01T10:32:13.817816+0000 mon.c (mon.2) 55 : audit 0 from='client.? 172.21.15.78:0/2409904483' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:14.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:14 smithi173 bash[58816]: cluster 2023-10-01T10:32:13.496543+0000 mgr.y (mgr.25072) 383 : cluster 0 pgmap v308: 89 pgs: 13 stale+active+clean, 76 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:32:14.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:14 smithi173 bash[58816]: audit 2023-10-01T10:32:13.817816+0000 mon.c (mon.2) 55 : audit 0 from='client.? 172.21.15.78:0/2409904483' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:15.597 INFO:journalctl@ceph.osd.1.smithi078.stdout:Oct 01 10:32:15 smithi078 systemd[1]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7@osd.1.service: Succeeded. 2023-10-01T10:32:15.597 INFO:journalctl@ceph.osd.1.smithi078.stdout:Oct 01 10:32:15 smithi078 systemd[1]: Stopped Ceph osd.1 for d8016480-6041-11ee-8db5-212e2dc638e7. 2023-10-01T10:32:15.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:15 smithi173 bash[58816]: cluster 2023-10-01T10:32:15.497907+0000 mgr.y (mgr.25072) 384 : cluster 0 pgmap v309: 89 pgs: 3 active+undersized+degraded, 36 active+undersized, 50 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-01T10:32:15.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:15 smithi173 bash[58816]: cluster 2023-10-01T10:32:15.551064+0000 mon.a (mon.0) 526 : cluster 3 Health check failed: Degraded data redundancy: 3/72 objects degraded (4.167%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T10:32:16.083 INFO:journalctl@ceph.osd.1.smithi078.stdout:Oct 01 10:32:15 smithi078 systemd[1]: Started Ceph osd.1 for d8016480-6041-11ee-8db5-212e2dc638e7. 2023-10-01T10:32:16.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:15 smithi078 bash[75299]: cluster 2023-10-01T10:32:15.497907+0000 mgr.y (mgr.25072) 384 : cluster 0 pgmap v309: 89 pgs: 3 active+undersized+degraded, 36 active+undersized, 50 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-01T10:32:16.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:15 smithi078 bash[75299]: cluster 2023-10-01T10:32:15.551064+0000 mon.a (mon.0) 526 : cluster 3 Health check failed: Degraded data redundancy: 3/72 objects degraded (4.167%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T10:32:16.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:15 smithi078 bash[78274]: cluster 2023-10-01T10:32:15.497907+0000 mgr.y (mgr.25072) 384 : cluster 0 pgmap v309: 89 pgs: 3 active+undersized+degraded, 36 active+undersized, 50 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-01T10:32:16.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:15 smithi078 bash[78274]: cluster 2023-10-01T10:32:15.551064+0000 mon.a (mon.0) 526 : cluster 3 Health check failed: Degraded data redundancy: 3/72 objects degraded (4.167%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T10:32:16.651 INFO:teuthology.orchestra.run.smithi078.stderr:enabled application 'rados' on pool 'unique_pool_3' 2023-10-01T10:32:16.888 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:16 smithi078 bash[78274]: audit 2023-10-01T10:32:15.629972+0000 mon.a (mon.0) 527 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:32:16.888 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:16 smithi078 bash[78274]: audit 2023-10-01T10:32:15.636376+0000 mon.a (mon.0) 528 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:32:16.889 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:16 smithi078 bash[78274]: audit 2023-10-01T10:32:15.657856+0000 mon.a (mon.0) 529 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:32:16.889 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:16 smithi078 bash[78274]: audit 2023-10-01T10:32:16.071103+0000 mon.a (mon.0) 530 : audit 1 from='client.? 172.21.15.78:0/2733659517' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T10:32:16.889 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:16 smithi078 bash[78274]: audit 2023-10-01T10:32:16.159988+0000 mon.c (mon.2) 56 : audit 0 from='client.? 172.21.15.78:0/2450664005' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:16.889 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:16 smithi078 bash[75299]: audit 2023-10-01T10:32:15.629972+0000 mon.a (mon.0) 527 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:32:16.889 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:16 smithi078 bash[75299]: audit 2023-10-01T10:32:15.636376+0000 mon.a (mon.0) 528 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:32:16.889 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:16 smithi078 bash[75299]: audit 2023-10-01T10:32:15.657856+0000 mon.a (mon.0) 529 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:32:16.889 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:16 smithi078 bash[75299]: audit 2023-10-01T10:32:16.071103+0000 mon.a (mon.0) 530 : audit 1 from='client.? 172.21.15.78:0/2733659517' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T10:32:16.890 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:16 smithi078 bash[75299]: audit 2023-10-01T10:32:16.159988+0000 mon.c (mon.2) 56 : audit 0 from='client.? 172.21.15.78:0/2450664005' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:17.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:16 smithi173 bash[58816]: audit 2023-10-01T10:32:15.629972+0000 mon.a (mon.0) 527 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:32:17.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:16 smithi173 bash[58816]: audit 2023-10-01T10:32:15.636376+0000 mon.a (mon.0) 528 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:32:17.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:16 smithi173 bash[58816]: audit 2023-10-01T10:32:15.657856+0000 mon.a (mon.0) 529 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:32:17.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:16 smithi173 bash[58816]: audit 2023-10-01T10:32:16.071103+0000 mon.a (mon.0) 530 : audit 1 from='client.? 172.21.15.78:0/2733659517' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T10:32:17.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:16 smithi173 bash[58816]: audit 2023-10-01T10:32:16.159988+0000 mon.c (mon.2) 56 : audit 0 from='client.? 172.21.15.78:0/2450664005' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:17.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:17 smithi078 bash[78274]: cluster 2023-10-01T10:32:16.640099+0000 osd.2 (osd.2) 18 : cluster 3 failed to encode map e99 with expected crc 2023-10-01T10:32:17.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:17 smithi078 bash[78274]: audit 2023-10-01T10:32:16.643434+0000 mon.a (mon.0) 531 : audit 1 from='client.? 172.21.15.78:0/2733659517' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-01T10:32:17.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:17 smithi078 bash[78274]: cluster 2023-10-01T10:32:16.643542+0000 mon.a (mon.0) 532 : cluster 0 osdmap e99: 8 total, 7 up, 8 in 2023-10-01T10:32:17.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:17 smithi078 bash[78274]: audit 2023-10-01T10:32:17.402433+0000 mon.a (mon.0) 533 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:32:17.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:17 smithi078 bash[78274]: audit 2023-10-01T10:32:17.403990+0000 mon.a (mon.0) 534 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:32:17.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:17 smithi078 bash[78274]: cluster 2023-10-01T10:32:17.498623+0000 mgr.y (mgr.25072) 385 : cluster 0 pgmap v311: 89 pgs: 3 active+undersized+degraded, 36 active+undersized, 50 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-01T10:32:17.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:17 smithi078 bash[75299]: cluster 2023-10-01T10:32:16.640099+0000 osd.2 (osd.2) 18 : cluster 3 failed to encode map e99 with expected crc 2023-10-01T10:32:17.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:17 smithi078 bash[75299]: audit 2023-10-01T10:32:16.643434+0000 mon.a (mon.0) 531 : audit 1 from='client.? 172.21.15.78:0/2733659517' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-01T10:32:17.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:17 smithi078 bash[75299]: cluster 2023-10-01T10:32:16.643542+0000 mon.a (mon.0) 532 : cluster 0 osdmap e99: 8 total, 7 up, 8 in 2023-10-01T10:32:17.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:17 smithi078 bash[75299]: audit 2023-10-01T10:32:17.402433+0000 mon.a (mon.0) 533 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:32:17.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:17 smithi078 bash[75299]: audit 2023-10-01T10:32:17.403990+0000 mon.a (mon.0) 534 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:32:17.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:17 smithi078 bash[75299]: cluster 2023-10-01T10:32:17.498623+0000 mgr.y (mgr.25072) 385 : cluster 0 pgmap v311: 89 pgs: 3 active+undersized+degraded, 36 active+undersized, 50 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-01T10:32:17.835 INFO:journalctl@ceph.osd.1.smithi078.stdout:Oct 01 10:32:17 smithi078 bash[91166]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-01T10:32:17.835 INFO:journalctl@ceph.osd.1.smithi078.stdout:Oct 01 10:32:17 smithi078 bash[91166]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-10-01T10:32:17.835 INFO:journalctl@ceph.osd.1.smithi078.stdout:Oct 01 10:32:17 smithi078 bash[91166]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2023-10-01T10:32:17.835 INFO:journalctl@ceph.osd.1.smithi078.stdout:Oct 01 10:32:17 smithi078 bash[91166]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2023-10-01T10:32:17.835 INFO:journalctl@ceph.osd.1.smithi078.stdout:Oct 01 10:32:17 smithi078 bash[91166]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2023-10-01T10:32:17.835 INFO:journalctl@ceph.osd.1.smithi078.stdout:Oct 01 10:32:17 smithi078 bash[91166]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2023-10-01T10:32:17.836 INFO:journalctl@ceph.osd.1.smithi078.stdout:Oct 01 10:32:17 smithi078 bash[91166]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-10-01T10:32:17.836 INFO:journalctl@ceph.osd.1.smithi078.stdout:Oct 01 10:32:17 smithi078 bash[91166]: --> ceph-volume lvm activate successful for osd ID: 1 2023-10-01T10:32:18.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:17 smithi173 bash[58816]: cluster 2023-10-01T10:32:16.640099+0000 osd.2 (osd.2) 18 : cluster 3 failed to encode map e99 with expected crc 2023-10-01T10:32:18.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:17 smithi173 bash[58816]: audit 2023-10-01T10:32:16.643434+0000 mon.a (mon.0) 531 : audit 1 from='client.? 172.21.15.78:0/2733659517' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-01T10:32:18.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:17 smithi173 bash[58816]: cluster 2023-10-01T10:32:16.643542+0000 mon.a (mon.0) 532 : cluster 0 osdmap e99: 8 total, 7 up, 8 in 2023-10-01T10:32:18.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:17 smithi173 bash[58816]: audit 2023-10-01T10:32:17.402433+0000 mon.a (mon.0) 533 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:32:18.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:17 smithi173 bash[58816]: audit 2023-10-01T10:32:17.403990+0000 mon.a (mon.0) 534 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:32:18.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:17 smithi173 bash[58816]: cluster 2023-10-01T10:32:17.498623+0000 mgr.y (mgr.25072) 385 : cluster 0 pgmap v311: 89 pgs: 3 active+undersized+degraded, 36 active+undersized, 50 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-01T10:32:18.566 DEBUG:teuthology.orchestra.run.smithi078:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_3 bench 90 write' 2023-10-01T10:32:18.568 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-01T10:32:18.568 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-01T10:32:18.608 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:hints = 1 2023-10-01T10:32:18.608 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-01T10:32:18.608 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Object prefix: benchmark_data_smithi078_91566 2023-10-01T10:32:18.608 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T10:32:18.608 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 0 0 0 0 0 0 - 0 2023-10-01T10:32:19.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:18 smithi078 bash[75299]: cluster 2023-10-01T10:32:17.073945+0000 osd.7 (osd.7) 29 : cluster 3 failed to encode map e99 with expected crc 2023-10-01T10:32:19.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:18 smithi078 bash[75299]: cluster 2023-10-01T10:32:17.411714+0000 osd.5 (osd.5) 14 : cluster 3 failed to encode map e99 with expected crc 2023-10-01T10:32:19.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:18 smithi078 bash[75299]: audit 2023-10-01T10:32:18.519251+0000 mon.a (mon.0) 535 : audit 0 from='client.? 172.21.15.78:0/2634539376' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:19.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:18 smithi078 bash[78274]: cluster 2023-10-01T10:32:17.073945+0000 osd.7 (osd.7) 29 : cluster 3 failed to encode map e99 with expected crc 2023-10-01T10:32:19.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:18 smithi078 bash[78274]: cluster 2023-10-01T10:32:17.411714+0000 osd.5 (osd.5) 14 : cluster 3 failed to encode map e99 with expected crc 2023-10-01T10:32:19.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:18 smithi078 bash[78274]: audit 2023-10-01T10:32:18.519251+0000 mon.a (mon.0) 535 : audit 0 from='client.? 172.21.15.78:0/2634539376' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:19.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:18 smithi173 bash[58816]: cluster 2023-10-01T10:32:17.073945+0000 osd.7 (osd.7) 29 : cluster 3 failed to encode map e99 with expected crc 2023-10-01T10:32:19.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:18 smithi173 bash[58816]: cluster 2023-10-01T10:32:17.411714+0000 osd.5 (osd.5) 14 : cluster 3 failed to encode map e99 with expected crc 2023-10-01T10:32:19.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:18 smithi173 bash[58816]: audit 2023-10-01T10:32:18.519251+0000 mon.a (mon.0) 535 : audit 0 from='client.? 172.21.15.78:0/2634539376' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:19.583 INFO:journalctl@ceph.osd.1.smithi078.stdout:Oct 01 10:32:19 smithi078 bash[91532]: debug 2023-10-01T10:32:19.216+0000 7fe94168e700 1 -- 172.21.15.78:0/1933578472 <== mon.0 v2:172.21.15.78:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5618515a44e0 con 0x5618515fec00 2023-10-01T10:32:19.608 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 1 16 3470 3454 215.861 215.875 0.0030571 0.00461522 2023-10-01T10:32:20.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:19 smithi078 bash[75299]: cluster 2023-10-01T10:32:19.499520+0000 mgr.y (mgr.25072) 386 : cluster 0 pgmap v312: 89 pgs: 5 active+undersized+degraded, 34 active+undersized, 50 active+clean; 25 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.0 MiB/s wr, 47 op/s; 166/1257 objects degraded (13.206%) 2023-10-01T10:32:20.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:19 smithi078 bash[78274]: cluster 2023-10-01T10:32:19.499520+0000 mgr.y (mgr.25072) 386 : cluster 0 pgmap v312: 89 pgs: 5 active+undersized+degraded, 34 active+undersized, 50 active+clean; 25 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.0 MiB/s wr, 47 op/s; 166/1257 objects degraded (13.206%) 2023-10-01T10:32:20.084 INFO:journalctl@ceph.osd.1.smithi078.stdout:Oct 01 10:32:19 smithi078 bash[91532]: debug 2023-10-01T10:32:19.768+0000 7fe94a814640 -1 Falling back to public interface 2023-10-01T10:32:20.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:19 smithi173 bash[58816]: cluster 2023-10-01T10:32:19.499520+0000 mgr.y (mgr.25072) 386 : cluster 0 pgmap v312: 89 pgs: 5 active+undersized+degraded, 34 active+undersized, 50 active+clean; 25 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.0 MiB/s wr, 47 op/s; 166/1257 objects degraded (13.206%) 2023-10-01T10:32:20.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:32:20 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:32:20] "GET /metrics HTTP/1.1" 200 36126 "" "Prometheus/2.33.4" 2023-10-01T10:32:20.608 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 2 16 6712 6696 209.233 202.625 0.00269864 0.00476674 2023-10-01T10:32:21.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:21 smithi078 bash[78274]: audit 2023-10-01T10:32:21.144731+0000 mon.c (mon.2) 57 : audit 0 from='client.? 172.21.15.78:0/788397874' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:21.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:21 smithi078 bash[75299]: audit 2023-10-01T10:32:21.144731+0000 mon.c (mon.2) 57 : audit 0 from='client.? 172.21.15.78:0/788397874' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:21.610 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 3 16 9431 9415 196.029 169.938 0.008195 0.00509244 2023-10-01T10:32:21.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:21 smithi173 bash[58816]: audit 2023-10-01T10:32:21.144731+0000 mon.c (mon.2) 57 : audit 0 from='client.? 172.21.15.78:0/788397874' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:22.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:22 smithi078 bash[78274]: cluster 2023-10-01T10:32:21.500226+0000 mgr.y (mgr.25072) 387 : cluster 0 pgmap v313: 89 pgs: 9 active+undersized+degraded, 30 active+undersized, 50 active+clean; 96 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.5 MiB/s wr, 152 op/s; 1295/4644 objects degraded (27.885%) 2023-10-01T10:32:22.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:22 smithi078 bash[78274]: cluster 2023-10-01T10:32:21.854082+0000 mon.a (mon.0) 536 : cluster 3 Health check update: Degraded data redundancy: 166/1257 objects degraded (13.206%), 5 pgs degraded (PG_DEGRADED) 2023-10-01T10:32:22.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:22 smithi078 bash[75299]: cluster 2023-10-01T10:32:21.500226+0000 mgr.y (mgr.25072) 387 : cluster 0 pgmap v313: 89 pgs: 9 active+undersized+degraded, 30 active+undersized, 50 active+clean; 96 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.5 MiB/s wr, 152 op/s; 1295/4644 objects degraded (27.885%) 2023-10-01T10:32:22.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:22 smithi078 bash[75299]: cluster 2023-10-01T10:32:21.854082+0000 mon.a (mon.0) 536 : cluster 3 Health check update: Degraded data redundancy: 166/1257 objects degraded (13.206%), 5 pgs degraded (PG_DEGRADED) 2023-10-01T10:32:22.610 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 4 16 11889 11873 185.428 153.625 0.00967514 0.00538416 2023-10-01T10:32:22.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:22 smithi173 bash[58816]: cluster 2023-10-01T10:32:21.500226+0000 mgr.y (mgr.25072) 387 : cluster 0 pgmap v313: 89 pgs: 9 active+undersized+degraded, 30 active+undersized, 50 active+clean; 96 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.5 MiB/s wr, 152 op/s; 1295/4644 objects degraded (27.885%) 2023-10-01T10:32:22.622 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:22 smithi173 bash[58816]: cluster 2023-10-01T10:32:21.854082+0000 mon.a (mon.0) 536 : cluster 3 Health check update: Degraded data redundancy: 166/1257 objects degraded (13.206%), 5 pgs degraded (PG_DEGRADED) 2023-10-01T10:32:23.610 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 5 16 14123 14107 176.267 139.625 0.0105788 0.00566203 2023-10-01T10:32:23.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:23 smithi078 bash[78274]: audit 2023-10-01T10:32:22.400081+0000 mon.a (mon.0) 537 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:32:23.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:23 smithi078 bash[78274]: audit 2023-10-01T10:32:22.410116+0000 mon.a (mon.0) 538 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:32:23.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:23 smithi078 bash[75299]: audit 2023-10-01T10:32:22.400081+0000 mon.a (mon.0) 537 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:32:23.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:23 smithi078 bash[75299]: audit 2023-10-01T10:32:22.410116+0000 mon.a (mon.0) 538 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:32:23.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:23 smithi173 bash[58816]: audit 2023-10-01T10:32:22.400081+0000 mon.a (mon.0) 537 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:32:23.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:23 smithi173 bash[58816]: audit 2023-10-01T10:32:22.410116+0000 mon.a (mon.0) 538 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:32:24.610 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 6 16 16468 16452 171.315 146.562 0.00549459 0.00583205 2023-10-01T10:32:24.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:24 smithi078 bash[78274]: cluster 2023-10-01T10:32:23.500959+0000 mgr.y (mgr.25072) 388 : cluster 0 pgmap v314: 89 pgs: 9 active+undersized+degraded, 30 active+undersized, 50 active+clean; 96 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.5 MiB/s wr, 152 op/s; 1295/4644 objects degraded (27.885%) 2023-10-01T10:32:24.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:24 smithi078 bash[78274]: audit 2023-10-01T10:32:23.522741+0000 mon.a (mon.0) 539 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:32:24.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:24 smithi078 bash[78274]: audit 2023-10-01T10:32:23.534317+0000 mon.a (mon.0) 540 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:32:24.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:24 smithi078 bash[78274]: audit 2023-10-01T10:32:23.835000+0000 mon.c (mon.2) 58 : audit 0 from='client.? 172.21.15.78:0/3726352015' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:24.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:24 smithi078 bash[75299]: cluster 2023-10-01T10:32:23.500959+0000 mgr.y (mgr.25072) 388 : cluster 0 pgmap v314: 89 pgs: 9 active+undersized+degraded, 30 active+undersized, 50 active+clean; 96 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.5 MiB/s wr, 152 op/s; 1295/4644 objects degraded (27.885%) 2023-10-01T10:32:24.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:24 smithi078 bash[75299]: audit 2023-10-01T10:32:23.522741+0000 mon.a (mon.0) 539 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:32:24.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:24 smithi078 bash[75299]: audit 2023-10-01T10:32:23.534317+0000 mon.a (mon.0) 540 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:32:24.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:24 smithi078 bash[75299]: audit 2023-10-01T10:32:23.835000+0000 mon.c (mon.2) 58 : audit 0 from='client.? 172.21.15.78:0/3726352015' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:24.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:24 smithi173 bash[58816]: cluster 2023-10-01T10:32:23.500959+0000 mgr.y (mgr.25072) 388 : cluster 0 pgmap v314: 89 pgs: 9 active+undersized+degraded, 30 active+undersized, 50 active+clean; 96 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.5 MiB/s wr, 152 op/s; 1295/4644 objects degraded (27.885%) 2023-10-01T10:32:24.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:24 smithi173 bash[58816]: audit 2023-10-01T10:32:23.522741+0000 mon.a (mon.0) 539 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:32:24.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:24 smithi173 bash[58816]: audit 2023-10-01T10:32:23.534317+0000 mon.a (mon.0) 540 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:32:24.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:24 smithi173 bash[58816]: audit 2023-10-01T10:32:23.835000+0000 mon.c (mon.2) 58 : audit 0 from='client.? 172.21.15.78:0/3726352015' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:25.610 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 7 16 18912 18896 168.662 152.75 0.00390091 0.00592327 2023-10-01T10:32:26.611 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 8 16 21089 21073 164.584 136.062 0.00549715 0.00606988 2023-10-01T10:32:26.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:26 smithi078 bash[78274]: cluster 2023-10-01T10:32:25.501949+0000 mgr.y (mgr.25072) 389 : cluster 0 pgmap v315: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 942 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s; 6993/45243 objects degraded (15.457%) 2023-10-01T10:32:26.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:26 smithi078 bash[78274]: audit 2023-10-01T10:32:26.489689+0000 mon.a (mon.0) 541 : audit 0 from='client.? 172.21.15.78:0/580110372' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:26.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:26 smithi078 bash[75299]: cluster 2023-10-01T10:32:25.501949+0000 mgr.y (mgr.25072) 389 : cluster 0 pgmap v315: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 942 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s; 6993/45243 objects degraded (15.457%) 2023-10-01T10:32:26.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:26 smithi078 bash[75299]: audit 2023-10-01T10:32:26.489689+0000 mon.a (mon.0) 541 : audit 0 from='client.? 172.21.15.78:0/580110372' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:26.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:26 smithi173 bash[58816]: cluster 2023-10-01T10:32:25.501949+0000 mgr.y (mgr.25072) 389 : cluster 0 pgmap v315: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 942 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s; 6993/45243 objects degraded (15.457%) 2023-10-01T10:32:26.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:26 smithi173 bash[58816]: audit 2023-10-01T10:32:26.489689+0000 mon.a (mon.0) 541 : audit 0 from='client.? 172.21.15.78:0/580110372' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:27.611 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 9 16 23391 23375 162.282 143.875 0.0105699 0.006157 2023-10-01T10:32:27.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:27 smithi078 bash[75299]: cluster 2023-10-01T10:32:26.855507+0000 mon.a (mon.0) 542 : cluster 3 Health check update: Degraded data redundancy: 6993/45243 objects degraded (15.457%), 10 pgs degraded (PG_DEGRADED) 2023-10-01T10:32:27.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:27 smithi078 bash[78274]: cluster 2023-10-01T10:32:26.855507+0000 mon.a (mon.0) 542 : cluster 3 Health check update: Degraded data redundancy: 6993/45243 objects degraded (15.457%), 10 pgs degraded (PG_DEGRADED) 2023-10-01T10:32:27.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:27 smithi173 bash[58816]: cluster 2023-10-01T10:32:26.855507+0000 mon.a (mon.0) 542 : cluster 3 Health check update: Degraded data redundancy: 6993/45243 objects degraded (15.457%), 10 pgs degraded (PG_DEGRADED) 2023-10-01T10:32:28.611 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 10 16 25514 25498 159.322 132.688 0.00924055 0.00626751 2023-10-01T10:32:28.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:28 smithi078 bash[78274]: cluster 2023-10-01T10:32:27.502531+0000 mgr.y (mgr.25072) 390 : cluster 0 pgmap v316: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 942 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s; 6993/45243 objects degraded (15.457%) 2023-10-01T10:32:28.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:28 smithi078 bash[75299]: cluster 2023-10-01T10:32:27.502531+0000 mgr.y (mgr.25072) 390 : cluster 0 pgmap v316: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 942 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s; 6993/45243 objects degraded (15.457%) 2023-10-01T10:32:28.833 INFO:journalctl@ceph.osd.1.smithi078.stdout:Oct 01 10:32:28 smithi078 bash[91532]: debug 2023-10-01T10:32:28.572+0000 7fe94a814640 -1 osd.1 96 log_to_monitors true 2023-10-01T10:32:28.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:28 smithi173 bash[58816]: cluster 2023-10-01T10:32:27.502531+0000 mgr.y (mgr.25072) 390 : cluster 0 pgmap v316: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 942 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s; 6993/45243 objects degraded (15.457%) 2023-10-01T10:32:29.611 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 11 16 27661 27645 157.036 134.188 0.004338 0.0063626 2023-10-01T10:32:29.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:29 smithi078 bash[78274]: audit 2023-10-01T10:32:28.582835+0000 mon.a (mon.0) 543 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-01T10:32:29.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:29 smithi078 bash[78274]: audit 2023-10-01T10:32:28.583764+0000 mon.b (mon.1) 21 : audit 1 from='osd.1 [v2:172.21.15.78:6810/1115426690,v1:172.21.15.78:6811/1115426690]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-01T10:32:29.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:29 smithi078 bash[78274]: audit 2023-10-01T10:32:29.039928+0000 mon.a (mon.0) 544 : audit 0 from='client.? 172.21.15.78:0/1834122732' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:29.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:29 smithi078 bash[75299]: audit 2023-10-01T10:32:28.582835+0000 mon.a (mon.0) 543 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-01T10:32:29.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:29 smithi078 bash[75299]: audit 2023-10-01T10:32:28.583764+0000 mon.b (mon.1) 21 : audit 1 from='osd.1 [v2:172.21.15.78:6810/1115426690,v1:172.21.15.78:6811/1115426690]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-01T10:32:29.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:29 smithi078 bash[75299]: audit 2023-10-01T10:32:29.039928+0000 mon.a (mon.0) 544 : audit 0 from='client.? 172.21.15.78:0/1834122732' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:29.834 INFO:journalctl@ceph.osd.1.smithi078.stdout:Oct 01 10:32:29 smithi078 bash[91532]: debug 2023-10-01T10:32:29.600+0000 7fe93947b700 -1 osd.1 96 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-01T10:32:29.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:29 smithi173 bash[58816]: audit 2023-10-01T10:32:28.582835+0000 mon.a (mon.0) 543 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-01T10:32:29.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:29 smithi173 bash[58816]: audit 2023-10-01T10:32:28.583764+0000 mon.b (mon.1) 21 : audit 1 from='osd.1 [v2:172.21.15.78:6810/1115426690,v1:172.21.15.78:6811/1115426690]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-10-01T10:32:29.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:29 smithi173 bash[58816]: audit 2023-10-01T10:32:29.039928+0000 mon.a (mon.0) 544 : audit 0 from='client.? 172.21.15.78:0/1834122732' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:30.563 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:32:30 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:32:30] "GET /metrics HTTP/1.1" 200 36126 "" "Prometheus/2.33.4" 2023-10-01T10:32:30.611 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 12 16 29729 29713 154.72 129.25 0.00224531 0.00643947 2023-10-01T10:32:30.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:30 smithi078 bash[75299]: cluster 2023-10-01T10:32:29.503959+0000 mgr.y (mgr.25072) 391 : cluster 0 pgmap v317: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 1.4 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s; 9140/71160 objects degraded (12.844%) 2023-10-01T10:32:30.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:30 smithi078 bash[75299]: cluster 2023-10-01T10:32:29.539982+0000 osd.5 (osd.5) 15 : cluster 3 failed to encode map e100 with expected crc 2023-10-01T10:32:30.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:30 smithi078 bash[75299]: cluster 2023-10-01T10:32:29.540893+0000 osd.7 (osd.7) 30 : cluster 3 failed to encode map e100 with expected crc 2023-10-01T10:32:30.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:30 smithi078 bash[75299]: audit 2023-10-01T10:32:29.542141+0000 mon.a (mon.0) 545 : audit 1 from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-01T10:32:30.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:30 smithi078 bash[75299]: cluster 2023-10-01T10:32:29.542245+0000 mon.a (mon.0) 546 : cluster 0 osdmap e100: 8 total, 7 up, 8 in 2023-10-01T10:32:30.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:30 smithi078 bash[75299]: audit 2023-10-01T10:32:29.544664+0000 mon.a (mon.0) 547 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2023-10-01T10:32:30.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:30 smithi078 bash[75299]: audit 2023-10-01T10:32:29.546006+0000 mon.b (mon.1) 22 : audit 1 from='osd.1 [v2:172.21.15.78:6810/1115426690,v1:172.21.15.78:6811/1115426690]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2023-10-01T10:32:30.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:30 smithi078 bash[75299]: cluster 2023-10-01T10:32:29.550337+0000 osd.6 (osd.6) 14 : cluster 3 failed to encode map e100 with expected crc 2023-10-01T10:32:30.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:30 smithi078 bash[78274]: cluster 2023-10-01T10:32:29.503959+0000 mgr.y (mgr.25072) 391 : cluster 0 pgmap v317: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 1.4 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s; 9140/71160 objects degraded (12.844%) 2023-10-01T10:32:30.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:30 smithi078 bash[78274]: cluster 2023-10-01T10:32:29.539982+0000 osd.5 (osd.5) 15 : cluster 3 failed to encode map e100 with expected crc 2023-10-01T10:32:30.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:30 smithi078 bash[78274]: cluster 2023-10-01T10:32:29.540893+0000 osd.7 (osd.7) 30 : cluster 3 failed to encode map e100 with expected crc 2023-10-01T10:32:30.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:30 smithi078 bash[78274]: audit 2023-10-01T10:32:29.542141+0000 mon.a (mon.0) 545 : audit 1 from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-01T10:32:30.836 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:30 smithi078 bash[78274]: cluster 2023-10-01T10:32:29.542245+0000 mon.a (mon.0) 546 : cluster 0 osdmap e100: 8 total, 7 up, 8 in 2023-10-01T10:32:30.836 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:30 smithi078 bash[78274]: audit 2023-10-01T10:32:29.544664+0000 mon.a (mon.0) 547 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2023-10-01T10:32:30.836 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:30 smithi078 bash[78274]: audit 2023-10-01T10:32:29.546006+0000 mon.b (mon.1) 22 : audit 1 from='osd.1 [v2:172.21.15.78:6810/1115426690,v1:172.21.15.78:6811/1115426690]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2023-10-01T10:32:30.836 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:30 smithi078 bash[78274]: cluster 2023-10-01T10:32:29.550337+0000 osd.6 (osd.6) 14 : cluster 3 failed to encode map e100 with expected crc 2023-10-01T10:32:30.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:30 smithi173 bash[58816]: cluster 2023-10-01T10:32:29.503959+0000 mgr.y (mgr.25072) 391 : cluster 0 pgmap v317: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 1.4 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s; 9140/71160 objects degraded (12.844%) 2023-10-01T10:32:30.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:30 smithi173 bash[58816]: cluster 2023-10-01T10:32:29.539982+0000 osd.5 (osd.5) 15 : cluster 3 failed to encode map e100 with expected crc 2023-10-01T10:32:30.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:30 smithi173 bash[58816]: cluster 2023-10-01T10:32:29.540893+0000 osd.7 (osd.7) 30 : cluster 3 failed to encode map e100 with expected crc 2023-10-01T10:32:30.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:30 smithi173 bash[58816]: audit 2023-10-01T10:32:29.542141+0000 mon.a (mon.0) 545 : audit 1 from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-10-01T10:32:30.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:30 smithi173 bash[58816]: cluster 2023-10-01T10:32:29.542245+0000 mon.a (mon.0) 546 : cluster 0 osdmap e100: 8 total, 7 up, 8 in 2023-10-01T10:32:30.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:30 smithi173 bash[58816]: audit 2023-10-01T10:32:29.544664+0000 mon.a (mon.0) 547 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2023-10-01T10:32:30.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:30 smithi173 bash[58816]: audit 2023-10-01T10:32:29.546006+0000 mon.b (mon.1) 22 : audit 1 from='osd.1 [v2:172.21.15.78:6810/1115426690,v1:172.21.15.78:6811/1115426690]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2023-10-01T10:32:30.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:30 smithi173 bash[58816]: cluster 2023-10-01T10:32:29.550337+0000 osd.6 (osd.6) 14 : cluster 3 failed to encode map e100 with expected crc 2023-10-01T10:32:31.611 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 13 16 29729 29713 142.819 0 - 0.00643947 2023-10-01T10:32:31.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:31 smithi078 bash[75299]: cluster 2023-10-01T10:32:30.538238+0000 mon.a (mon.0) 548 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-01T10:32:31.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:31 smithi078 bash[75299]: cluster 2023-10-01T10:32:30.548765+0000 mon.a (mon.0) 549 : cluster 1 osd.1 [v2:172.21.15.78:6810/1115426690,v1:172.21.15.78:6811/1115426690] boot 2023-10-01T10:32:31.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:31 smithi078 bash[75299]: cluster 2023-10-01T10:32:30.550314+0000 osd.7 (osd.7) 31 : cluster 3 failed to encode map e101 with expected crc 2023-10-01T10:32:31.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:31 smithi078 bash[75299]: cluster 2023-10-01T10:32:30.553407+0000 mon.a (mon.0) 550 : cluster 0 osdmap e101: 8 total, 8 up, 8 in 2023-10-01T10:32:31.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:31 smithi078 bash[75299]: audit 2023-10-01T10:32:30.555202+0000 mon.a (mon.0) 551 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T10:32:31.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:31 smithi078 bash[75299]: cluster 2023-10-01T10:32:30.556718+0000 osd.2 (osd.2) 19 : cluster 3 failed to encode map e101 with expected crc 2023-10-01T10:32:31.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:31 smithi078 bash[75299]: audit 2023-10-01T10:32:31.411966+0000 mon.a (mon.0) 552 : audit 0 from='client.? 172.21.15.78:0/885684405' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:31.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:31 smithi078 bash[75299]: cluster 2023-10-01T10:32:31.549200+0000 mon.a (mon.0) 553 : cluster 0 osdmap e102: 8 total, 8 up, 8 in 2023-10-01T10:32:31.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:31 smithi078 bash[78274]: cluster 2023-10-01T10:32:30.538238+0000 mon.a (mon.0) 548 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-01T10:32:31.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:31 smithi078 bash[78274]: cluster 2023-10-01T10:32:30.548765+0000 mon.a (mon.0) 549 : cluster 1 osd.1 [v2:172.21.15.78:6810/1115426690,v1:172.21.15.78:6811/1115426690] boot 2023-10-01T10:32:31.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:31 smithi078 bash[78274]: cluster 2023-10-01T10:32:30.550314+0000 osd.7 (osd.7) 31 : cluster 3 failed to encode map e101 with expected crc 2023-10-01T10:32:31.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:31 smithi078 bash[78274]: cluster 2023-10-01T10:32:30.553407+0000 mon.a (mon.0) 550 : cluster 0 osdmap e101: 8 total, 8 up, 8 in 2023-10-01T10:32:31.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:31 smithi078 bash[78274]: audit 2023-10-01T10:32:30.555202+0000 mon.a (mon.0) 551 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T10:32:31.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:31 smithi078 bash[78274]: cluster 2023-10-01T10:32:30.556718+0000 osd.2 (osd.2) 19 : cluster 3 failed to encode map e101 with expected crc 2023-10-01T10:32:31.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:31 smithi078 bash[78274]: audit 2023-10-01T10:32:31.411966+0000 mon.a (mon.0) 552 : audit 0 from='client.? 172.21.15.78:0/885684405' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:31.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:31 smithi078 bash[78274]: cluster 2023-10-01T10:32:31.549200+0000 mon.a (mon.0) 553 : cluster 0 osdmap e102: 8 total, 8 up, 8 in 2023-10-01T10:32:31.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:31 smithi173 bash[58816]: cluster 2023-10-01T10:32:30.538238+0000 mon.a (mon.0) 548 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-01T10:32:31.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:31 smithi173 bash[58816]: cluster 2023-10-01T10:32:30.548765+0000 mon.a (mon.0) 549 : cluster 1 osd.1 [v2:172.21.15.78:6810/1115426690,v1:172.21.15.78:6811/1115426690] boot 2023-10-01T10:32:31.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:31 smithi173 bash[58816]: cluster 2023-10-01T10:32:30.550314+0000 osd.7 (osd.7) 31 : cluster 3 failed to encode map e101 with expected crc 2023-10-01T10:32:31.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:31 smithi173 bash[58816]: cluster 2023-10-01T10:32:30.553407+0000 mon.a (mon.0) 550 : cluster 0 osdmap e101: 8 total, 8 up, 8 in 2023-10-01T10:32:31.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:31 smithi173 bash[58816]: audit 2023-10-01T10:32:30.555202+0000 mon.a (mon.0) 551 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-10-01T10:32:31.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:31 smithi173 bash[58816]: cluster 2023-10-01T10:32:30.556718+0000 osd.2 (osd.2) 19 : cluster 3 failed to encode map e101 with expected crc 2023-10-01T10:32:31.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:31 smithi173 bash[58816]: audit 2023-10-01T10:32:31.411966+0000 mon.a (mon.0) 552 : audit 0 from='client.? 172.21.15.78:0/885684405' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:31.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:31 smithi173 bash[58816]: cluster 2023-10-01T10:32:31.549200+0000 mon.a (mon.0) 553 : cluster 0 osdmap e102: 8 total, 8 up, 8 in 2023-10-01T10:32:32.611 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 14 16 29729 29713 132.619 0 - 0.00643947 2023-10-01T10:32:32.838 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:32 smithi078 bash[75299]: cluster 2023-10-01T10:32:31.504793+0000 mgr.y (mgr.25072) 392 : cluster 0 pgmap v320: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 1.6 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 195 MiB/s wr, 3.11k op/s; 11893/79419 objects degraded (14.975%) 2023-10-01T10:32:32.838 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:32 smithi078 bash[75299]: cluster 2023-10-01T10:32:31.543224+0000 osd.2 (osd.2) 20 : cluster 3 failed to encode map e102 with expected crc 2023-10-01T10:32:32.838 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:32 smithi078 bash[75299]: cluster 2023-10-01T10:32:31.546018+0000 osd.7 (osd.7) 32 : cluster 3 failed to encode map e102 with expected crc 2023-10-01T10:32:32.838 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:32 smithi078 bash[75299]: cluster 2023-10-01T10:32:31.549194+0000 osd.7 (osd.7) 33 : cluster 3 failed to encode map e102 with expected crc 2023-10-01T10:32:32.838 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:32 smithi078 bash[75299]: cluster 2023-10-01T10:32:31.549402+0000 osd.5 (osd.5) 16 : cluster 3 failed to encode map e102 with expected crc 2023-10-01T10:32:32.838 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:32 smithi078 bash[75299]: cluster 2023-10-01T10:32:31.549449+0000 osd.4 (osd.4) 17 : cluster 3 failed to encode map e102 with expected crc 2023-10-01T10:32:32.838 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:32 smithi078 bash[75299]: cluster 2023-10-01T10:32:31.549652+0000 osd.6 (osd.6) 15 : cluster 3 failed to encode map e102 with expected crc 2023-10-01T10:32:32.838 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:32 smithi078 bash[75299]: cluster 2023-10-01T10:32:31.549992+0000 osd.3 (osd.3) 21 : cluster 3 failed to encode map e102 with expected crc 2023-10-01T10:32:32.839 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:32 smithi078 bash[75299]: cluster 2023-10-01T10:32:31.550008+0000 osd.2 (osd.2) 21 : cluster 3 failed to encode map e102 with expected crc 2023-10-01T10:32:32.839 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:32 smithi078 bash[75299]: cluster 2023-10-01T10:32:31.551039+0000 osd.5 (osd.5) 17 : cluster 3 failed to encode map e102 with expected crc 2023-10-01T10:32:32.839 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:32 smithi078 bash[75299]: cluster 2023-10-01T10:32:31.551956+0000 osd.4 (osd.4) 18 : cluster 3 failed to encode map e102 with expected crc 2023-10-01T10:32:32.839 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:32 smithi078 bash[75299]: cluster 2023-10-01T10:32:31.551973+0000 osd.3 (osd.3) 22 : cluster 3 failed to encode map e102 with expected crc 2023-10-01T10:32:32.840 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:32 smithi078 bash[75299]: cluster 2023-10-01T10:32:31.552499+0000 osd.2 (osd.2) 22 : cluster 3 failed to encode map e102 with expected crc 2023-10-01T10:32:32.840 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:32 smithi078 bash[75299]: cluster 2023-10-01T10:32:31.554150+0000 osd.3 (osd.3) 23 : cluster 3 failed to encode map e102 with expected crc 2023-10-01T10:32:32.840 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:32 smithi078 bash[75299]: cluster 2023-10-01T10:32:31.554254+0000 osd.6 (osd.6) 16 : cluster 3 failed to encode map e102 with expected crc 2023-10-01T10:32:32.840 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:32 smithi078 bash[75299]: cluster 2023-10-01T10:32:31.856945+0000 mon.a (mon.0) 554 : cluster 3 Health check update: Degraded data redundancy: 11893/79419 objects degraded (14.975%), 10 pgs degraded (PG_DEGRADED) 2023-10-01T10:32:32.841 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:32 smithi078 bash[75299]: audit 2023-10-01T10:32:32.403547+0000 mon.a (mon.0) 555 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:32:32.841 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:32 smithi078 bash[75299]: audit 2023-10-01T10:32:32.405028+0000 mon.a (mon.0) 556 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:32:32.841 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:32 smithi078 bash[75299]: cluster 2023-10-01T10:32:32.551900+0000 mon.a (mon.0) 557 : cluster 0 osdmap e103: 8 total, 8 up, 8 in 2023-10-01T10:32:32.841 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:32 smithi078 bash[78274]: cluster 2023-10-01T10:32:31.504793+0000 mgr.y (mgr.25072) 392 : cluster 0 pgmap v320: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 1.6 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 195 MiB/s wr, 3.11k op/s; 11893/79419 objects degraded (14.975%) 2023-10-01T10:32:32.841 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:32 smithi078 bash[78274]: cluster 2023-10-01T10:32:31.543224+0000 osd.2 (osd.2) 20 : cluster 3 failed to encode map e102 with expected crc 2023-10-01T10:32:32.841 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:32 smithi078 bash[78274]: cluster 2023-10-01T10:32:31.546018+0000 osd.7 (osd.7) 32 : cluster 3 failed to encode map e102 with expected crc 2023-10-01T10:32:32.841 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:32 smithi078 bash[78274]: cluster 2023-10-01T10:32:31.549194+0000 osd.7 (osd.7) 33 : cluster 3 failed to encode map e102 with expected crc 2023-10-01T10:32:32.842 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:32 smithi078 bash[78274]: cluster 2023-10-01T10:32:31.549402+0000 osd.5 (osd.5) 16 : cluster 3 failed to encode map e102 with expected crc 2023-10-01T10:32:32.842 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:32 smithi078 bash[78274]: cluster 2023-10-01T10:32:31.549449+0000 osd.4 (osd.4) 17 : cluster 3 failed to encode map e102 with expected crc 2023-10-01T10:32:32.842 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:32 smithi078 bash[78274]: cluster 2023-10-01T10:32:31.549652+0000 osd.6 (osd.6) 15 : cluster 3 failed to encode map e102 with expected crc 2023-10-01T10:32:32.842 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:32 smithi078 bash[78274]: cluster 2023-10-01T10:32:31.549992+0000 osd.3 (osd.3) 21 : cluster 3 failed to encode map e102 with expected crc 2023-10-01T10:32:32.842 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:32 smithi078 bash[78274]: cluster 2023-10-01T10:32:31.550008+0000 osd.2 (osd.2) 21 : cluster 3 failed to encode map e102 with expected crc 2023-10-01T10:32:32.842 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:32 smithi078 bash[78274]: cluster 2023-10-01T10:32:31.551039+0000 osd.5 (osd.5) 17 : cluster 3 failed to encode map e102 with expected crc 2023-10-01T10:32:32.842 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:32 smithi078 bash[78274]: cluster 2023-10-01T10:32:31.551956+0000 osd.4 (osd.4) 18 : cluster 3 failed to encode map e102 with expected crc 2023-10-01T10:32:32.842 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:32 smithi078 bash[78274]: cluster 2023-10-01T10:32:31.551973+0000 osd.3 (osd.3) 22 : cluster 3 failed to encode map e102 with expected crc 2023-10-01T10:32:32.842 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:32 smithi078 bash[78274]: cluster 2023-10-01T10:32:31.552499+0000 osd.2 (osd.2) 22 : cluster 3 failed to encode map e102 with expected crc 2023-10-01T10:32:32.842 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:32 smithi078 bash[78274]: cluster 2023-10-01T10:32:31.554150+0000 osd.3 (osd.3) 23 : cluster 3 failed to encode map e102 with expected crc 2023-10-01T10:32:32.843 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:32 smithi078 bash[78274]: cluster 2023-10-01T10:32:31.554254+0000 osd.6 (osd.6) 16 : cluster 3 failed to encode map e102 with expected crc 2023-10-01T10:32:32.843 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:32 smithi078 bash[78274]: cluster 2023-10-01T10:32:31.856945+0000 mon.a (mon.0) 554 : cluster 3 Health check update: Degraded data redundancy: 11893/79419 objects degraded (14.975%), 10 pgs degraded (PG_DEGRADED) 2023-10-01T10:32:32.843 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:32 smithi078 bash[78274]: audit 2023-10-01T10:32:32.403547+0000 mon.a (mon.0) 555 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:32:32.843 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:32 smithi078 bash[78274]: audit 2023-10-01T10:32:32.405028+0000 mon.a (mon.0) 556 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:32:32.843 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:32 smithi078 bash[78274]: cluster 2023-10-01T10:32:32.551900+0000 mon.a (mon.0) 557 : cluster 0 osdmap e103: 8 total, 8 up, 8 in 2023-10-01T10:32:32.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:32 smithi173 bash[58816]: cluster 2023-10-01T10:32:31.504793+0000 mgr.y (mgr.25072) 392 : cluster 0 pgmap v320: 89 pgs: 10 active+undersized+degraded, 29 active+undersized, 50 active+clean; 1.6 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 195 MiB/s wr, 3.11k op/s; 11893/79419 objects degraded (14.975%) 2023-10-01T10:32:32.872 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:32 smithi173 bash[58816]: cluster 2023-10-01T10:32:31.543224+0000 osd.2 (osd.2) 20 : cluster 3 failed to encode map e102 with expected crc 2023-10-01T10:32:32.872 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:32 smithi173 bash[58816]: cluster 2023-10-01T10:32:31.546018+0000 osd.7 (osd.7) 32 : cluster 3 failed to encode map e102 with expected crc 2023-10-01T10:32:32.872 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:32 smithi173 bash[58816]: cluster 2023-10-01T10:32:31.549194+0000 osd.7 (osd.7) 33 : cluster 3 failed to encode map e102 with expected crc 2023-10-01T10:32:32.872 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:32 smithi173 bash[58816]: cluster 2023-10-01T10:32:31.549402+0000 osd.5 (osd.5) 16 : cluster 3 failed to encode map e102 with expected crc 2023-10-01T10:32:32.872 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:32 smithi173 bash[58816]: cluster 2023-10-01T10:32:31.549449+0000 osd.4 (osd.4) 17 : cluster 3 failed to encode map e102 with expected crc 2023-10-01T10:32:32.872 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:32 smithi173 bash[58816]: cluster 2023-10-01T10:32:31.549652+0000 osd.6 (osd.6) 15 : cluster 3 failed to encode map e102 with expected crc 2023-10-01T10:32:32.872 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:32 smithi173 bash[58816]: cluster 2023-10-01T10:32:31.549992+0000 osd.3 (osd.3) 21 : cluster 3 failed to encode map e102 with expected crc 2023-10-01T10:32:32.872 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:32 smithi173 bash[58816]: cluster 2023-10-01T10:32:31.550008+0000 osd.2 (osd.2) 21 : cluster 3 failed to encode map e102 with expected crc 2023-10-01T10:32:32.872 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:32 smithi173 bash[58816]: cluster 2023-10-01T10:32:31.551039+0000 osd.5 (osd.5) 17 : cluster 3 failed to encode map e102 with expected crc 2023-10-01T10:32:32.872 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:32 smithi173 bash[58816]: cluster 2023-10-01T10:32:31.551956+0000 osd.4 (osd.4) 18 : cluster 3 failed to encode map e102 with expected crc 2023-10-01T10:32:32.872 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:32 smithi173 bash[58816]: cluster 2023-10-01T10:32:31.551973+0000 osd.3 (osd.3) 22 : cluster 3 failed to encode map e102 with expected crc 2023-10-01T10:32:32.872 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:32 smithi173 bash[58816]: cluster 2023-10-01T10:32:31.552499+0000 osd.2 (osd.2) 22 : cluster 3 failed to encode map e102 with expected crc 2023-10-01T10:32:32.873 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:32 smithi173 bash[58816]: cluster 2023-10-01T10:32:31.554150+0000 osd.3 (osd.3) 23 : cluster 3 failed to encode map e102 with expected crc 2023-10-01T10:32:32.873 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:32 smithi173 bash[58816]: cluster 2023-10-01T10:32:31.554254+0000 osd.6 (osd.6) 16 : cluster 3 failed to encode map e102 with expected crc 2023-10-01T10:32:32.873 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:32 smithi173 bash[58816]: cluster 2023-10-01T10:32:31.856945+0000 mon.a (mon.0) 554 : cluster 3 Health check update: Degraded data redundancy: 11893/79419 objects degraded (14.975%), 10 pgs degraded (PG_DEGRADED) 2023-10-01T10:32:32.873 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:32 smithi173 bash[58816]: audit 2023-10-01T10:32:32.403547+0000 mon.a (mon.0) 555 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:32:32.873 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:32 smithi173 bash[58816]: audit 2023-10-01T10:32:32.405028+0000 mon.a (mon.0) 556 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:32:32.873 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:32 smithi173 bash[58816]: cluster 2023-10-01T10:32:32.551900+0000 mon.a (mon.0) 557 : cluster 0 osdmap e103: 8 total, 8 up, 8 in 2023-10-01T10:32:33.611 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 15 16 30830 30814 128.365 22.9375 0.00955999 0.00777992 2023-10-01T10:32:33.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:33 smithi173 bash[58816]: cluster 2023-10-01T10:32:32.548814+0000 osd.5 (osd.5) 18 : cluster 3 failed to encode map e103 with expected crc 2023-10-01T10:32:33.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:33 smithi173 bash[58816]: cluster 2023-10-01T10:32:32.550980+0000 osd.2 (osd.2) 23 : cluster 3 failed to encode map e103 with expected crc 2023-10-01T10:32:33.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:33 smithi173 bash[58816]: cluster 2023-10-01T10:32:32.551149+0000 osd.7 (osd.7) 34 : cluster 3 failed to encode map e103 with expected crc 2023-10-01T10:32:33.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:33 smithi173 bash[58816]: cluster 2023-10-01T10:32:32.551812+0000 osd.5 (osd.5) 19 : cluster 3 failed to encode map e103 with expected crc 2023-10-01T10:32:33.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:33 smithi173 bash[58816]: cluster 2023-10-01T10:32:32.553036+0000 osd.3 (osd.3) 24 : cluster 3 failed to encode map e103 with expected crc 2023-10-01T10:32:33.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:33 smithi173 bash[58816]: cluster 2023-10-01T10:32:32.553560+0000 osd.2 (osd.2) 24 : cluster 3 failed to encode map e103 with expected crc 2023-10-01T10:32:33.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:33 smithi173 bash[58816]: cluster 2023-10-01T10:32:32.554479+0000 osd.3 (osd.3) 25 : cluster 3 failed to encode map e103 with expected crc 2023-10-01T10:32:33.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:33 smithi173 bash[58816]: cluster 2023-10-01T10:32:32.554733+0000 osd.5 (osd.5) 20 : cluster 3 failed to encode map e103 with expected crc 2023-10-01T10:32:34.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:33 smithi078 bash[75299]: cluster 2023-10-01T10:32:32.548814+0000 osd.5 (osd.5) 18 : cluster 3 failed to encode map e103 with expected crc 2023-10-01T10:32:34.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:33 smithi078 bash[75299]: cluster 2023-10-01T10:32:32.550980+0000 osd.2 (osd.2) 23 : cluster 3 failed to encode map e103 with expected crc 2023-10-01T10:32:34.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:33 smithi078 bash[75299]: cluster 2023-10-01T10:32:32.551149+0000 osd.7 (osd.7) 34 : cluster 3 failed to encode map e103 with expected crc 2023-10-01T10:32:34.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:33 smithi078 bash[75299]: cluster 2023-10-01T10:32:32.551812+0000 osd.5 (osd.5) 19 : cluster 3 failed to encode map e103 with expected crc 2023-10-01T10:32:34.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:33 smithi078 bash[75299]: cluster 2023-10-01T10:32:32.553036+0000 osd.3 (osd.3) 24 : cluster 3 failed to encode map e103 with expected crc 2023-10-01T10:32:34.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:33 smithi078 bash[75299]: cluster 2023-10-01T10:32:32.553560+0000 osd.2 (osd.2) 24 : cluster 3 failed to encode map e103 with expected crc 2023-10-01T10:32:34.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:33 smithi078 bash[75299]: cluster 2023-10-01T10:32:32.554479+0000 osd.3 (osd.3) 25 : cluster 3 failed to encode map e103 with expected crc 2023-10-01T10:32:34.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:33 smithi078 bash[75299]: cluster 2023-10-01T10:32:32.554733+0000 osd.5 (osd.5) 20 : cluster 3 failed to encode map e103 with expected crc 2023-10-01T10:32:34.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:33 smithi078 bash[78274]: cluster 2023-10-01T10:32:32.548814+0000 osd.5 (osd.5) 18 : cluster 3 failed to encode map e103 with expected crc 2023-10-01T10:32:34.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:33 smithi078 bash[78274]: cluster 2023-10-01T10:32:32.550980+0000 osd.2 (osd.2) 23 : cluster 3 failed to encode map e103 with expected crc 2023-10-01T10:32:34.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:33 smithi078 bash[78274]: cluster 2023-10-01T10:32:32.551149+0000 osd.7 (osd.7) 34 : cluster 3 failed to encode map e103 with expected crc 2023-10-01T10:32:34.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:33 smithi078 bash[78274]: cluster 2023-10-01T10:32:32.551812+0000 osd.5 (osd.5) 19 : cluster 3 failed to encode map e103 with expected crc 2023-10-01T10:32:34.085 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:33 smithi078 bash[78274]: cluster 2023-10-01T10:32:32.553036+0000 osd.3 (osd.3) 24 : cluster 3 failed to encode map e103 with expected crc 2023-10-01T10:32:34.085 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:33 smithi078 bash[78274]: cluster 2023-10-01T10:32:32.553560+0000 osd.2 (osd.2) 24 : cluster 3 failed to encode map e103 with expected crc 2023-10-01T10:32:34.085 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:33 smithi078 bash[78274]: cluster 2023-10-01T10:32:32.554479+0000 osd.3 (osd.3) 25 : cluster 3 failed to encode map e103 with expected crc 2023-10-01T10:32:34.085 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:33 smithi078 bash[78274]: cluster 2023-10-01T10:32:32.554733+0000 osd.5 (osd.5) 20 : cluster 3 failed to encode map e103 with expected crc 2023-10-01T10:32:34.611 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 16 16 32020 32004 124.99 74.375 0.00401242 0.00798616 2023-10-01T10:32:34.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:34 smithi173 bash[58816]: cluster 2023-10-01T10:32:33.505521+0000 mgr.y (mgr.25072) 393 : cluster 0 pgmap v323: 89 pgs: 10 active+undersized+degraded, 26 active+undersized, 53 active+clean; 1.7 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 53 MiB/s wr, 855 op/s; 11893/81429 objects degraded (14.605%) 2023-10-01T10:32:34.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:34 smithi173 bash[58816]: audit 2023-10-01T10:32:33.997650+0000 mon.b (mon.1) 23 : audit 0 from='client.? 172.21.15.78:0/39415914' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:35.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:34 smithi078 bash[78274]: cluster 2023-10-01T10:32:33.505521+0000 mgr.y (mgr.25072) 393 : cluster 0 pgmap v323: 89 pgs: 10 active+undersized+degraded, 26 active+undersized, 53 active+clean; 1.7 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 53 MiB/s wr, 855 op/s; 11893/81429 objects degraded (14.605%) 2023-10-01T10:32:35.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:34 smithi078 bash[78274]: audit 2023-10-01T10:32:33.997650+0000 mon.b (mon.1) 23 : audit 0 from='client.? 172.21.15.78:0/39415914' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:35.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:34 smithi078 bash[75299]: cluster 2023-10-01T10:32:33.505521+0000 mgr.y (mgr.25072) 393 : cluster 0 pgmap v323: 89 pgs: 10 active+undersized+degraded, 26 active+undersized, 53 active+clean; 1.7 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 53 MiB/s wr, 855 op/s; 11893/81429 objects degraded (14.605%) 2023-10-01T10:32:35.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:34 smithi078 bash[75299]: audit 2023-10-01T10:32:33.997650+0000 mon.b (mon.1) 23 : audit 0 from='client.? 172.21.15.78:0/39415914' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:35.612 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 17 16 33197 33181 121.965 73.5625 0.0275694 0.00818943 2023-10-01T10:32:35.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:35 smithi078 bash[75299]: cluster 2023-10-01T10:32:35.506427+0000 mgr.y (mgr.25072) 394 : cluster 0 pgmap v324: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.9 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s; 12800/94125 objects degraded (13.599%); 761 KiB/s, 11 objects/s recovering 2023-10-01T10:32:35.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:35 smithi078 bash[78274]: cluster 2023-10-01T10:32:35.506427+0000 mgr.y (mgr.25072) 394 : cluster 0 pgmap v324: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.9 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s; 12800/94125 objects degraded (13.599%); 761 KiB/s, 11 objects/s recovering 2023-10-01T10:32:35.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:35 smithi173 bash[58816]: cluster 2023-10-01T10:32:35.506427+0000 mgr.y (mgr.25072) 394 : cluster 0 pgmap v324: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.9 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s; 12800/94125 objects degraded (13.599%); 761 KiB/s, 11 objects/s recovering 2023-10-01T10:32:36.611 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 18 16 34340 34324 119.158 71.4375 0.0163098 0.00838201 2023-10-01T10:32:36.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:36 smithi173 bash[58816]: audit 2023-10-01T10:32:36.575601+0000 mon.b (mon.1) 24 : audit 0 from='client.? 172.21.15.78:0/2282170111' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:37.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:36 smithi078 bash[78274]: audit 2023-10-01T10:32:36.575601+0000 mon.b (mon.1) 24 : audit 0 from='client.? 172.21.15.78:0/2282170111' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:37.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:36 smithi078 bash[75299]: audit 2023-10-01T10:32:36.575601+0000 mon.b (mon.1) 24 : audit 0 from='client.? 172.21.15.78:0/2282170111' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:37.613 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 19 16 35573 35557 116.942 77.0625 0.021189 0.00854391 2023-10-01T10:32:37.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:37 smithi078 bash[75299]: cluster 2023-10-01T10:32:36.858585+0000 mon.a (mon.0) 558 : cluster 3 Health check update: Degraded data redundancy: 12800/94125 objects degraded (13.599%), 7 pgs degraded (PG_DEGRADED) 2023-10-01T10:32:37.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:37 smithi078 bash[75299]: cluster 2023-10-01T10:32:37.506998+0000 mgr.y (mgr.25072) 395 : cluster 0 pgmap v325: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.9 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 44 MiB/s wr, 704 op/s; 12800/94125 objects degraded (13.599%); 653 KiB/s, 10 objects/s recovering 2023-10-01T10:32:37.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:37 smithi078 bash[78274]: cluster 2023-10-01T10:32:36.858585+0000 mon.a (mon.0) 558 : cluster 3 Health check update: Degraded data redundancy: 12800/94125 objects degraded (13.599%), 7 pgs degraded (PG_DEGRADED) 2023-10-01T10:32:37.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:37 smithi078 bash[78274]: cluster 2023-10-01T10:32:37.506998+0000 mgr.y (mgr.25072) 395 : cluster 0 pgmap v325: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.9 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 44 MiB/s wr, 704 op/s; 12800/94125 objects degraded (13.599%); 653 KiB/s, 10 objects/s recovering 2023-10-01T10:32:37.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:37 smithi173 bash[58816]: cluster 2023-10-01T10:32:36.858585+0000 mon.a (mon.0) 558 : cluster 3 Health check update: Degraded data redundancy: 12800/94125 objects degraded (13.599%), 7 pgs degraded (PG_DEGRADED) 2023-10-01T10:32:37.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:37 smithi173 bash[58816]: cluster 2023-10-01T10:32:37.506998+0000 mgr.y (mgr.25072) 395 : cluster 0 pgmap v325: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 1.9 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 44 MiB/s wr, 704 op/s; 12800/94125 objects degraded (13.599%); 653 KiB/s, 10 objects/s recovering 2023-10-01T10:32:38.612 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:2023-10-01T10:32:38.609738+0000 min lat: 0.00124552 max lat: 2.17561 avg lat: 0.00869765 2023-10-01T10:32:38.612 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T10:32:38.612 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 20 16 36787 36771 114.888 75.875 0.0176307 0.00869765 2023-10-01T10:32:39.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:32:39 smithi078 bash[71214]: debug 2023-10-01T10:32:39.208+0000 7fd9c245e700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-01T10:32:39.612 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 21 16 37913 37897 112.769 70.375 0.0356121 0.00885979 2023-10-01T10:32:40.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:40 smithi078 bash[75299]: audit 2023-10-01T10:32:39.073397+0000 mon.a (mon.0) 559 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:32:40.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:40 smithi078 bash[75299]: audit 2023-10-01T10:32:39.083395+0000 mon.a (mon.0) 560 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:32:40.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:40 smithi078 bash[75299]: audit 2023-10-01T10:32:39.085623+0000 mon.a (mon.0) 561 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:32:40.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:40 smithi078 bash[75299]: audit 2023-10-01T10:32:39.087874+0000 mon.a (mon.0) 562 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:32:40.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:40 smithi078 bash[75299]: audit 2023-10-01T10:32:39.100896+0000 mon.a (mon.0) 563 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:32:40.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:40 smithi078 bash[75299]: audit 2023-10-01T10:32:39.187069+0000 mon.b (mon.1) 25 : audit 0 from='client.? 172.21.15.78:0/2529959763' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:40.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:40 smithi078 bash[75299]: audit 2023-10-01T10:32:39.188547+0000 mon.a (mon.0) 564 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:32:40.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:40 smithi078 bash[75299]: audit 2023-10-01T10:32:39.194853+0000 mon.a (mon.0) 565 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:32:40.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:40 smithi078 bash[75299]: audit 2023-10-01T10:32:39.202494+0000 mon.a (mon.0) 566 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:32:40.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:40 smithi078 bash[75299]: audit 2023-10-01T10:32:39.206700+0000 mon.a (mon.0) 567 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:32:40.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:40 smithi078 bash[75299]: audit 2023-10-01T10:32:39.212688+0000 mon.a (mon.0) 568 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:32:40.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:40 smithi078 bash[75299]: audit 2023-10-01T10:32:39.213016+0000 mgr.y (mgr.25072) 396 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:32:40.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:40 smithi078 bash[75299]: cephadm 2023-10-01T10:32:39.215212+0000 mgr.y (mgr.25072) 397 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-01T10:32:40.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:40 smithi078 bash[75299]: cluster 2023-10-01T10:32:39.507747+0000 mgr.y (mgr.25072) 398 : cluster 0 pgmap v326: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.2 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s; 12533/107412 objects degraded (11.668%); 2.6 MiB/s, 42 objects/s recovering 2023-10-01T10:32:40.335 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:32:40 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:32:40] "GET /metrics HTTP/1.1" 200 36041 "" "Prometheus/2.33.4" 2023-10-01T10:32:40.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:40 smithi078 bash[78274]: audit 2023-10-01T10:32:39.073397+0000 mon.a (mon.0) 559 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:32:40.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:40 smithi078 bash[78274]: audit 2023-10-01T10:32:39.083395+0000 mon.a (mon.0) 560 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:32:40.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:40 smithi078 bash[78274]: audit 2023-10-01T10:32:39.085623+0000 mon.a (mon.0) 561 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:32:40.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:40 smithi078 bash[78274]: audit 2023-10-01T10:32:39.087874+0000 mon.a (mon.0) 562 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:32:40.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:40 smithi078 bash[78274]: audit 2023-10-01T10:32:39.100896+0000 mon.a (mon.0) 563 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:32:40.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:40 smithi078 bash[78274]: audit 2023-10-01T10:32:39.187069+0000 mon.b (mon.1) 25 : audit 0 from='client.? 172.21.15.78:0/2529959763' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:40.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:40 smithi078 bash[78274]: audit 2023-10-01T10:32:39.188547+0000 mon.a (mon.0) 564 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:32:40.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:40 smithi078 bash[78274]: audit 2023-10-01T10:32:39.194853+0000 mon.a (mon.0) 565 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:32:40.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:40 smithi078 bash[78274]: audit 2023-10-01T10:32:39.202494+0000 mon.a (mon.0) 566 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:32:40.337 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:40 smithi078 bash[78274]: audit 2023-10-01T10:32:39.206700+0000 mon.a (mon.0) 567 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:32:40.337 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:40 smithi078 bash[78274]: audit 2023-10-01T10:32:39.212688+0000 mon.a (mon.0) 568 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:32:40.337 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:40 smithi078 bash[78274]: audit 2023-10-01T10:32:39.213016+0000 mgr.y (mgr.25072) 396 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:32:40.337 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:40 smithi078 bash[78274]: cephadm 2023-10-01T10:32:39.215212+0000 mgr.y (mgr.25072) 397 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-01T10:32:40.337 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:40 smithi078 bash[78274]: cluster 2023-10-01T10:32:39.507747+0000 mgr.y (mgr.25072) 398 : cluster 0 pgmap v326: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.2 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s; 12533/107412 objects degraded (11.668%); 2.6 MiB/s, 42 objects/s recovering 2023-10-01T10:32:40.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:40 smithi173 bash[58816]: audit 2023-10-01T10:32:39.073397+0000 mon.a (mon.0) 559 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:32:40.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:40 smithi173 bash[58816]: audit 2023-10-01T10:32:39.083395+0000 mon.a (mon.0) 560 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:32:40.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:40 smithi173 bash[58816]: audit 2023-10-01T10:32:39.085623+0000 mon.a (mon.0) 561 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:32:40.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:40 smithi173 bash[58816]: audit 2023-10-01T10:32:39.087874+0000 mon.a (mon.0) 562 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:32:40.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:40 smithi173 bash[58816]: audit 2023-10-01T10:32:39.100896+0000 mon.a (mon.0) 563 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:32:40.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:40 smithi173 bash[58816]: audit 2023-10-01T10:32:39.187069+0000 mon.b (mon.1) 25 : audit 0 from='client.? 172.21.15.78:0/2529959763' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:40.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:40 smithi173 bash[58816]: audit 2023-10-01T10:32:39.188547+0000 mon.a (mon.0) 564 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:32:40.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:40 smithi173 bash[58816]: audit 2023-10-01T10:32:39.194853+0000 mon.a (mon.0) 565 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:32:40.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:40 smithi173 bash[58816]: audit 2023-10-01T10:32:39.202494+0000 mon.a (mon.0) 566 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:32:40.372 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:40 smithi173 bash[58816]: audit 2023-10-01T10:32:39.206700+0000 mon.a (mon.0) 567 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:32:40.372 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:40 smithi173 bash[58816]: audit 2023-10-01T10:32:39.212688+0000 mon.a (mon.0) 568 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:32:40.372 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:40 smithi173 bash[58816]: audit 2023-10-01T10:32:39.213016+0000 mgr.y (mgr.25072) 396 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:32:40.372 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:40 smithi173 bash[58816]: cephadm 2023-10-01T10:32:39.215212+0000 mgr.y (mgr.25072) 397 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-01T10:32:40.372 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:40 smithi173 bash[58816]: cluster 2023-10-01T10:32:39.507747+0000 mgr.y (mgr.25072) 398 : cluster 0 pgmap v326: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.2 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s; 12533/107412 objects degraded (11.668%); 2.6 MiB/s, 42 objects/s recovering 2023-10-01T10:32:40.612 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 22 16 39130 39114 111.1 76.0625 0.0313498 0.0089925 2023-10-01T10:32:41.612 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 23 16 40292 40276 109.427 72.625 0.00399482 0.00912315 2023-10-01T10:32:42.612 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 24 16 41440 41424 107.857 71.75 0.0224384 0.00926376 2023-10-01T10:32:42.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:42 smithi078 bash[75299]: cluster 2023-10-01T10:32:41.508414+0000 mgr.y (mgr.25072) 399 : cluster 0 pgmap v327: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.3 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s; 12533/111765 objects degraded (11.214%); 2.1 MiB/s, 33 objects/s recovering 2023-10-01T10:32:42.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:42 smithi078 bash[75299]: audit 2023-10-01T10:32:41.849318+0000 mon.c (mon.2) 59 : audit 0 from='client.? 172.21.15.78:0/4216154463' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:42.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:42 smithi078 bash[75299]: cluster 2023-10-01T10:32:41.860156+0000 mon.a (mon.0) 569 : cluster 3 Health check update: Degraded data redundancy: 12533/111765 objects degraded (11.214%), 7 pgs degraded (PG_DEGRADED) 2023-10-01T10:32:42.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:42 smithi078 bash[78274]: cluster 2023-10-01T10:32:41.508414+0000 mgr.y (mgr.25072) 399 : cluster 0 pgmap v327: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.3 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s; 12533/111765 objects degraded (11.214%); 2.1 MiB/s, 33 objects/s recovering 2023-10-01T10:32:42.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:42 smithi078 bash[78274]: audit 2023-10-01T10:32:41.849318+0000 mon.c (mon.2) 59 : audit 0 from='client.? 172.21.15.78:0/4216154463' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:42.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:42 smithi078 bash[78274]: cluster 2023-10-01T10:32:41.860156+0000 mon.a (mon.0) 569 : cluster 3 Health check update: Degraded data redundancy: 12533/111765 objects degraded (11.214%), 7 pgs degraded (PG_DEGRADED) 2023-10-01T10:32:42.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:42 smithi173 bash[58816]: cluster 2023-10-01T10:32:41.508414+0000 mgr.y (mgr.25072) 399 : cluster 0 pgmap v327: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.3 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s; 12533/111765 objects degraded (11.214%); 2.1 MiB/s, 33 objects/s recovering 2023-10-01T10:32:42.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:42 smithi173 bash[58816]: audit 2023-10-01T10:32:41.849318+0000 mon.c (mon.2) 59 : audit 0 from='client.? 172.21.15.78:0/4216154463' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:42.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:42 smithi173 bash[58816]: cluster 2023-10-01T10:32:41.860156+0000 mon.a (mon.0) 569 : cluster 3 Health check update: Degraded data redundancy: 12533/111765 objects degraded (11.214%), 7 pgs degraded (PG_DEGRADED) 2023-10-01T10:32:43.612 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 25 16 42626 42610 106.507 74.125 0.0146013 0.00938365 2023-10-01T10:32:44.612 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 26 16 43674 43658 104.93 65.5 0.0169114 0.00952359 2023-10-01T10:32:44.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:44 smithi078 bash[75299]: cluster 2023-10-01T10:32:43.509044+0000 mgr.y (mgr.25072) 400 : cluster 0 pgmap v328: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.3 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s; 12533/114063 objects degraded (10.988%); 1.9 MiB/s, 30 objects/s recovering 2023-10-01T10:32:44.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:44 smithi078 bash[75299]: audit 2023-10-01T10:32:44.473766+0000 mon.c (mon.2) 60 : audit 0 from='client.? 172.21.15.78:0/2296376470' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:44.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:44 smithi078 bash[78274]: cluster 2023-10-01T10:32:43.509044+0000 mgr.y (mgr.25072) 400 : cluster 0 pgmap v328: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.3 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s; 12533/114063 objects degraded (10.988%); 1.9 MiB/s, 30 objects/s recovering 2023-10-01T10:32:44.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:44 smithi078 bash[78274]: audit 2023-10-01T10:32:44.473766+0000 mon.c (mon.2) 60 : audit 0 from='client.? 172.21.15.78:0/2296376470' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:44.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:44 smithi173 bash[58816]: cluster 2023-10-01T10:32:43.509044+0000 mgr.y (mgr.25072) 400 : cluster 0 pgmap v328: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.3 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s; 12533/114063 objects degraded (10.988%); 1.9 MiB/s, 30 objects/s recovering 2023-10-01T10:32:44.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:44 smithi173 bash[58816]: audit 2023-10-01T10:32:44.473766+0000 mon.c (mon.2) 60 : audit 0 from='client.? 172.21.15.78:0/2296376470' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:45.612 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 27 16 44877 44861 103.828 75.1875 0.00736141 0.00962259 2023-10-01T10:32:46.612 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 28 16 46021 46005 102.673 71.5 0.010107 0.00973238 2023-10-01T10:32:46.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:46 smithi078 bash[75299]: cluster 2023-10-01T10:32:45.510088+0000 mgr.y (mgr.25072) 401 : cluster 0 pgmap v329: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.6 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s; 12257/129294 objects degraded (9.480%); 3.2 MiB/s, 51 objects/s recovering 2023-10-01T10:32:46.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:46 smithi078 bash[78274]: cluster 2023-10-01T10:32:45.510088+0000 mgr.y (mgr.25072) 401 : cluster 0 pgmap v329: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.6 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s; 12257/129294 objects degraded (9.480%); 3.2 MiB/s, 51 objects/s recovering 2023-10-01T10:32:46.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:46 smithi173 bash[58816]: cluster 2023-10-01T10:32:45.510088+0000 mgr.y (mgr.25072) 401 : cluster 0 pgmap v329: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.6 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s; 12257/129294 objects degraded (9.480%); 3.2 MiB/s, 51 objects/s recovering 2023-10-01T10:32:47.613 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 29 16 47067 47051 101.387 65.375 0.0118211 0.00985584 2023-10-01T10:32:47.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:47 smithi078 bash[78274]: cluster 2023-10-01T10:32:46.861443+0000 mon.a (mon.0) 570 : cluster 3 Health check update: Degraded data redundancy: 12257/129294 objects degraded (9.480%), 7 pgs degraded (PG_DEGRADED) 2023-10-01T10:32:47.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:47 smithi078 bash[78274]: audit 2023-10-01T10:32:47.090486+0000 mon.c (mon.2) 61 : audit 0 from='client.? 172.21.15.78:0/2002586224' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:47.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:47 smithi078 bash[78274]: audit 2023-10-01T10:32:47.397783+0000 mon.a (mon.0) 571 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:32:47.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:47 smithi078 bash[75299]: cluster 2023-10-01T10:32:46.861443+0000 mon.a (mon.0) 570 : cluster 3 Health check update: Degraded data redundancy: 12257/129294 objects degraded (9.480%), 7 pgs degraded (PG_DEGRADED) 2023-10-01T10:32:47.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:47 smithi078 bash[75299]: audit 2023-10-01T10:32:47.090486+0000 mon.c (mon.2) 61 : audit 0 from='client.? 172.21.15.78:0/2002586224' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:47.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:47 smithi078 bash[75299]: audit 2023-10-01T10:32:47.397783+0000 mon.a (mon.0) 571 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:32:47.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:47 smithi173 bash[58816]: cluster 2023-10-01T10:32:46.861443+0000 mon.a (mon.0) 570 : cluster 3 Health check update: Degraded data redundancy: 12257/129294 objects degraded (9.480%), 7 pgs degraded (PG_DEGRADED) 2023-10-01T10:32:47.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:47 smithi173 bash[58816]: audit 2023-10-01T10:32:47.090486+0000 mon.c (mon.2) 61 : audit 0 from='client.? 172.21.15.78:0/2002586224' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:47.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:47 smithi173 bash[58816]: audit 2023-10-01T10:32:47.397783+0000 mon.a (mon.0) 571 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:32:48.613 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 30 16 48145 48129 100.253 67.375 0.0133497 0.00996787 2023-10-01T10:32:48.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:48 smithi173 bash[58816]: cluster 2023-10-01T10:32:47.510660+0000 mgr.y (mgr.25072) 402 : cluster 0 pgmap v330: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.6 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 61 MiB/s wr, 976 op/s; 12257/129294 objects degraded (9.480%); 2.8 MiB/s, 45 objects/s recovering 2023-10-01T10:32:49.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:48 smithi078 bash[78274]: cluster 2023-10-01T10:32:47.510660+0000 mgr.y (mgr.25072) 402 : cluster 0 pgmap v330: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.6 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 61 MiB/s wr, 976 op/s; 12257/129294 objects degraded (9.480%); 2.8 MiB/s, 45 objects/s recovering 2023-10-01T10:32:49.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:48 smithi078 bash[75299]: cluster 2023-10-01T10:32:47.510660+0000 mgr.y (mgr.25072) 402 : cluster 0 pgmap v330: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.6 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 61 MiB/s wr, 976 op/s; 12257/129294 objects degraded (9.480%); 2.8 MiB/s, 45 objects/s recovering 2023-10-01T10:32:49.613 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 31 16 49233 49217 99.2127 68 0.0062613 0.0100724 2023-10-01T10:32:50.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:32:50 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:32:50] "GET /metrics HTTP/1.1" 200 36209 "" "Prometheus/2.33.4" 2023-10-01T10:32:50.613 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 32 16 50308 50292 98.2117 67.1875 0.0167423 0.0101759 2023-10-01T10:32:50.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:50 smithi173 bash[58816]: cluster 2023-10-01T10:32:49.511517+0000 mgr.y (mgr.25072) 403 : cluster 0 pgmap v331: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 82 MiB/s wr, 1.32k op/s; 11956/141654 objects degraded (8.440%); 4.4 MiB/s, 70 objects/s recovering 2023-10-01T10:32:50.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:50 smithi173 bash[58816]: audit 2023-10-01T10:32:49.695779+0000 mon.c (mon.2) 62 : audit 0 from='client.? 172.21.15.78:0/1963195103' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:51.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:50 smithi078 bash[75299]: cluster 2023-10-01T10:32:49.511517+0000 mgr.y (mgr.25072) 403 : cluster 0 pgmap v331: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 82 MiB/s wr, 1.32k op/s; 11956/141654 objects degraded (8.440%); 4.4 MiB/s, 70 objects/s recovering 2023-10-01T10:32:51.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:50 smithi078 bash[75299]: audit 2023-10-01T10:32:49.695779+0000 mon.c (mon.2) 62 : audit 0 from='client.? 172.21.15.78:0/1963195103' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:51.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:50 smithi078 bash[78274]: cluster 2023-10-01T10:32:49.511517+0000 mgr.y (mgr.25072) 403 : cluster 0 pgmap v331: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 82 MiB/s wr, 1.32k op/s; 11956/141654 objects degraded (8.440%); 4.4 MiB/s, 70 objects/s recovering 2023-10-01T10:32:51.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:50 smithi078 bash[78274]: audit 2023-10-01T10:32:49.695779+0000 mon.c (mon.2) 62 : audit 0 from='client.? 172.21.15.78:0/1963195103' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:51.613 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 33 16 51404 51388 97.3112 68.5 0.00870646 0.0102672 2023-10-01T10:32:52.613 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 34 16 52388 52372 96.2579 61.5 0.0350642 0.0103804 2023-10-01T10:32:52.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:52 smithi173 bash[58816]: cluster 2023-10-01T10:32:51.512177+0000 mgr.y (mgr.25072) 404 : cluster 0 pgmap v332: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.0 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 11956/145890 objects degraded (8.195%); 3.0 MiB/s, 48 objects/s recovering 2023-10-01T10:32:52.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:52 smithi173 bash[58816]: cluster 2023-10-01T10:32:51.862686+0000 mon.a (mon.0) 572 : cluster 3 Health check update: Degraded data redundancy: 11956/145890 objects degraded (8.195%), 7 pgs degraded (PG_DEGRADED) 2023-10-01T10:32:52.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:52 smithi173 bash[58816]: audit 2023-10-01T10:32:52.333107+0000 mon.c (mon.2) 63 : audit 0 from='client.? 172.21.15.78:0/3430787198' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:53.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:52 smithi078 bash[78274]: cluster 2023-10-01T10:32:51.512177+0000 mgr.y (mgr.25072) 404 : cluster 0 pgmap v332: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.0 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 11956/145890 objects degraded (8.195%); 3.0 MiB/s, 48 objects/s recovering 2023-10-01T10:32:53.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:52 smithi078 bash[78274]: cluster 2023-10-01T10:32:51.862686+0000 mon.a (mon.0) 572 : cluster 3 Health check update: Degraded data redundancy: 11956/145890 objects degraded (8.195%), 7 pgs degraded (PG_DEGRADED) 2023-10-01T10:32:53.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:52 smithi078 bash[78274]: audit 2023-10-01T10:32:52.333107+0000 mon.c (mon.2) 63 : audit 0 from='client.? 172.21.15.78:0/3430787198' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:53.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:52 smithi078 bash[75299]: cluster 2023-10-01T10:32:51.512177+0000 mgr.y (mgr.25072) 404 : cluster 0 pgmap v332: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.0 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 11956/145890 objects degraded (8.195%); 3.0 MiB/s, 48 objects/s recovering 2023-10-01T10:32:53.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:52 smithi078 bash[75299]: cluster 2023-10-01T10:32:51.862686+0000 mon.a (mon.0) 572 : cluster 3 Health check update: Degraded data redundancy: 11956/145890 objects degraded (8.195%), 7 pgs degraded (PG_DEGRADED) 2023-10-01T10:32:53.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:52 smithi078 bash[75299]: audit 2023-10-01T10:32:52.333107+0000 mon.c (mon.2) 63 : audit 0 from='client.? 172.21.15.78:0/3430787198' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:53.613 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 35 16 53446 53430 95.3968 66.125 0.0199107 0.0104751 2023-10-01T10:32:53.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:53 smithi173 bash[58816]: cluster 2023-10-01T10:32:53.512781+0000 mgr.y (mgr.25072) 405 : cluster 0 pgmap v333: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.0 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 11956/147876 objects degraded (8.085%); 3.0 MiB/s, 48 objects/s recovering 2023-10-01T10:32:54.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:53 smithi078 bash[75299]: cluster 2023-10-01T10:32:53.512781+0000 mgr.y (mgr.25072) 405 : cluster 0 pgmap v333: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.0 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 11956/147876 objects degraded (8.085%); 3.0 MiB/s, 48 objects/s recovering 2023-10-01T10:32:54.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:53 smithi078 bash[78274]: cluster 2023-10-01T10:32:53.512781+0000 mgr.y (mgr.25072) 405 : cluster 0 pgmap v333: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.0 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 11956/147876 objects degraded (8.085%); 3.0 MiB/s, 48 objects/s recovering 2023-10-01T10:32:54.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:32:54 smithi078 bash[71214]: debug 2023-10-01T10:32:54.228+0000 7fd9c245e700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-01T10:32:54.613 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 36 16 54491 54475 94.561 65.3125 0.00842545 0.0105669 2023-10-01T10:32:54.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:54 smithi173 bash[58816]: audit 2023-10-01T10:32:54.229851+0000 mon.a (mon.0) 573 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:32:54.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:54 smithi173 bash[58816]: audit 2023-10-01T10:32:54.230276+0000 mgr.y (mgr.25072) 406 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:32:54.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:54 smithi173 bash[58816]: cephadm 2023-10-01T10:32:54.231588+0000 mgr.y (mgr.25072) 407 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-01T10:32:55.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:54 smithi078 bash[75299]: audit 2023-10-01T10:32:54.229851+0000 mon.a (mon.0) 573 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:32:55.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:54 smithi078 bash[75299]: audit 2023-10-01T10:32:54.230276+0000 mgr.y (mgr.25072) 406 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:32:55.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:54 smithi078 bash[75299]: cephadm 2023-10-01T10:32:54.231588+0000 mgr.y (mgr.25072) 407 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-01T10:32:55.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:54 smithi078 bash[78274]: audit 2023-10-01T10:32:54.229851+0000 mon.a (mon.0) 573 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:32:55.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:54 smithi078 bash[78274]: audit 2023-10-01T10:32:54.230276+0000 mgr.y (mgr.25072) 406 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:32:55.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:54 smithi078 bash[78274]: cephadm 2023-10-01T10:32:54.231588+0000 mgr.y (mgr.25072) 407 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-10-01T10:32:55.613 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 37 16 55495 55479 93.7011 62.75 0.00864478 0.0106652 2023-10-01T10:32:55.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:55 smithi173 bash[58816]: audit 2023-10-01T10:32:54.978070+0000 mon.c (mon.2) 64 : audit 0 from='client.? 172.21.15.78:0/605195195' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:55.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:55 smithi173 bash[58816]: cluster 2023-10-01T10:32:55.513653+0000 mgr.y (mgr.25072) 408 : cluster 0 pgmap v334: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s; 11589/161730 objects degraded (7.166%); 4.9 MiB/s, 78 objects/s recovering 2023-10-01T10:32:56.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:55 smithi078 bash[75299]: audit 2023-10-01T10:32:54.978070+0000 mon.c (mon.2) 64 : audit 0 from='client.? 172.21.15.78:0/605195195' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:56.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:55 smithi078 bash[75299]: cluster 2023-10-01T10:32:55.513653+0000 mgr.y (mgr.25072) 408 : cluster 0 pgmap v334: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s; 11589/161730 objects degraded (7.166%); 4.9 MiB/s, 78 objects/s recovering 2023-10-01T10:32:56.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:55 smithi078 bash[78274]: audit 2023-10-01T10:32:54.978070+0000 mon.c (mon.2) 64 : audit 0 from='client.? 172.21.15.78:0/605195195' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:56.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:55 smithi078 bash[78274]: cluster 2023-10-01T10:32:55.513653+0000 mgr.y (mgr.25072) 408 : cluster 0 pgmap v334: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s; 11589/161730 objects degraded (7.166%); 4.9 MiB/s, 78 objects/s recovering 2023-10-01T10:32:56.614 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 38 16 56538 56522 92.9506 65.1875 0.0245756 0.0107507 2023-10-01T10:32:57.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:56 smithi078 bash[75299]: cluster 2023-10-01T10:32:56.863935+0000 mon.a (mon.0) 574 : cluster 3 Health check update: Degraded data redundancy: 11589/161730 objects degraded (7.166%), 7 pgs degraded (PG_DEGRADED) 2023-10-01T10:32:57.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:56 smithi078 bash[78274]: cluster 2023-10-01T10:32:56.863935+0000 mon.a (mon.0) 574 : cluster 3 Health check update: Degraded data redundancy: 11589/161730 objects degraded (7.166%), 7 pgs degraded (PG_DEGRADED) 2023-10-01T10:32:57.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:56 smithi173 bash[58816]: cluster 2023-10-01T10:32:56.863935+0000 mon.a (mon.0) 574 : cluster 3 Health check update: Degraded data redundancy: 11589/161730 objects degraded (7.166%), 7 pgs degraded (PG_DEGRADED) 2023-10-01T10:32:57.613 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 39 16 57481 57465 92.0783 58.9375 0.00361924 0.0108521 2023-10-01T10:32:58.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:57 smithi078 bash[75299]: cluster 2023-10-01T10:32:57.514133+0000 mgr.y (mgr.25072) 409 : cluster 0 pgmap v335: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 56 MiB/s wr, 900 op/s; 11589/161730 objects degraded (7.166%); 3.5 MiB/s, 55 objects/s recovering 2023-10-01T10:32:58.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:32:57 smithi078 bash[75299]: audit 2023-10-01T10:32:57.575097+0000 mon.c (mon.2) 65 : audit 0 from='client.? 172.21.15.78:0/995023184' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:58.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:57 smithi078 bash[78274]: cluster 2023-10-01T10:32:57.514133+0000 mgr.y (mgr.25072) 409 : cluster 0 pgmap v335: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 56 MiB/s wr, 900 op/s; 11589/161730 objects degraded (7.166%); 3.5 MiB/s, 55 objects/s recovering 2023-10-01T10:32:58.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:32:57 smithi078 bash[78274]: audit 2023-10-01T10:32:57.575097+0000 mon.c (mon.2) 65 : audit 0 from='client.? 172.21.15.78:0/995023184' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:58.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:57 smithi173 bash[58816]: cluster 2023-10-01T10:32:57.514133+0000 mgr.y (mgr.25072) 409 : cluster 0 pgmap v335: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 56 MiB/s wr, 900 op/s; 11589/161730 objects degraded (7.166%); 3.5 MiB/s, 55 objects/s recovering 2023-10-01T10:32:58.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:32:57 smithi173 bash[58816]: audit 2023-10-01T10:32:57.575097+0000 mon.c (mon.2) 65 : audit 0 from='client.? 172.21.15.78:0/995023184' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:32:58.614 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:2023-10-01T10:32:58.611693+0000 min lat: 0.00124552 max lat: 2.17561 avg lat: 0.0109294 2023-10-01T10:32:58.614 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T10:32:58.614 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 40 16 58548 58532 91.4434 66.6875 0.0108801 0.0109294 2023-10-01T10:32:59.614 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 41 16 59565 59549 90.7633 63.5625 0.0192978 0.0110116 2023-10-01T10:33:00.574 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:00 smithi078 bash[75299]: cluster 2023-10-01T10:32:59.514930+0000 mgr.y (mgr.25072) 410 : cluster 0 pgmap v336: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 11194/173172 objects degraded (6.464%); 5.5 MiB/s, 88 objects/s recovering 2023-10-01T10:33:00.574 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:00 smithi078 bash[75299]: audit 2023-10-01T10:33:00.195154+0000 mon.c (mon.2) 66 : audit 0 from='client.? 172.21.15.78:0/2879629921' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:00.575 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:33:00 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:33:00] "GET /metrics HTTP/1.1" 200 36209 "" "Prometheus/2.33.4" 2023-10-01T10:33:00.614 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 42 16 60538 60522 90.0501 60.8125 0.0181695 0.0110984 2023-10-01T10:33:00.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:00 smithi078 bash[78274]: cluster 2023-10-01T10:32:59.514930+0000 mgr.y (mgr.25072) 410 : cluster 0 pgmap v336: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 11194/173172 objects degraded (6.464%); 5.5 MiB/s, 88 objects/s recovering 2023-10-01T10:33:00.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:00 smithi078 bash[78274]: audit 2023-10-01T10:33:00.195154+0000 mon.c (mon.2) 66 : audit 0 from='client.? 172.21.15.78:0/2879629921' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:00.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:00 smithi173 bash[58816]: cluster 2023-10-01T10:32:59.514930+0000 mgr.y (mgr.25072) 410 : cluster 0 pgmap v336: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 11194/173172 objects degraded (6.464%); 5.5 MiB/s, 88 objects/s recovering 2023-10-01T10:33:00.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:00 smithi173 bash[58816]: audit 2023-10-01T10:33:00.195154+0000 mon.c (mon.2) 66 : audit 0 from='client.? 172.21.15.78:0/2879629921' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:01.614 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 43 16 61644 61628 89.5633 69.125 0.00859332 0.011159 2023-10-01T10:33:02.614 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 44 16 61971 61955 87.9923 20.4375 1.00747 0.0111781 2023-10-01T10:33:02.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:02 smithi173 bash[58816]: cluster 2023-10-01T10:33:01.515592+0000 mgr.y (mgr.25072) 411 : cluster 0 pgmap v337: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 61 MiB/s wr, 978 op/s; 11194/176907 objects degraded (6.328%); 4.0 MiB/s, 63 objects/s recovering 2023-10-01T10:33:02.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:02 smithi173 bash[58816]: cluster 2023-10-01T10:33:01.585880+0000 osd.5 (osd.5) 21 : cluster 3 failed to encode map e104 with expected crc 2023-10-01T10:33:02.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:02 smithi173 bash[58816]: cluster 2023-10-01T10:33:01.587084+0000 osd.7 (osd.7) 35 : cluster 3 failed to encode map e104 with expected crc 2023-10-01T10:33:02.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:02 smithi173 bash[58816]: cluster 2023-10-01T10:33:01.588579+0000 mon.a (mon.0) 575 : cluster 0 osdmap e104: 8 total, 8 up, 8 in 2023-10-01T10:33:02.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:02 smithi173 bash[58816]: cluster 2023-10-01T10:33:01.590841+0000 osd.2 (osd.2) 25 : cluster 3 failed to encode map e104 with expected crc 2023-10-01T10:33:02.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:02 smithi173 bash[58816]: cluster 2023-10-01T10:33:01.595396+0000 osd.3 (osd.3) 26 : cluster 3 failed to encode map e104 with expected crc 2023-10-01T10:33:02.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:02 smithi173 bash[58816]: cluster 2023-10-01T10:33:01.605109+0000 osd.6 (osd.6) 17 : cluster 3 failed to encode map e104 with expected crc 2023-10-01T10:33:02.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:02 smithi173 bash[58816]: cluster 2023-10-01T10:33:01.865380+0000 mon.a (mon.0) 576 : cluster 3 Health check update: Degraded data redundancy: 11194/176907 objects degraded (6.328%), 7 pgs degraded (PG_DEGRADED) 2023-10-01T10:33:02.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:02 smithi173 bash[58816]: audit 2023-10-01T10:33:02.398218+0000 mon.a (mon.0) 577 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:33:03.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:02 smithi078 bash[78274]: cluster 2023-10-01T10:33:01.515592+0000 mgr.y (mgr.25072) 411 : cluster 0 pgmap v337: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 61 MiB/s wr, 978 op/s; 11194/176907 objects degraded (6.328%); 4.0 MiB/s, 63 objects/s recovering 2023-10-01T10:33:03.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:02 smithi078 bash[78274]: cluster 2023-10-01T10:33:01.585880+0000 osd.5 (osd.5) 21 : cluster 3 failed to encode map e104 with expected crc 2023-10-01T10:33:03.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:02 smithi078 bash[78274]: cluster 2023-10-01T10:33:01.587084+0000 osd.7 (osd.7) 35 : cluster 3 failed to encode map e104 with expected crc 2023-10-01T10:33:03.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:02 smithi078 bash[78274]: cluster 2023-10-01T10:33:01.588579+0000 mon.a (mon.0) 575 : cluster 0 osdmap e104: 8 total, 8 up, 8 in 2023-10-01T10:33:03.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:02 smithi078 bash[78274]: cluster 2023-10-01T10:33:01.590841+0000 osd.2 (osd.2) 25 : cluster 3 failed to encode map e104 with expected crc 2023-10-01T10:33:03.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:02 smithi078 bash[78274]: cluster 2023-10-01T10:33:01.595396+0000 osd.3 (osd.3) 26 : cluster 3 failed to encode map e104 with expected crc 2023-10-01T10:33:03.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:02 smithi078 bash[78274]: cluster 2023-10-01T10:33:01.605109+0000 osd.6 (osd.6) 17 : cluster 3 failed to encode map e104 with expected crc 2023-10-01T10:33:03.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:02 smithi078 bash[78274]: cluster 2023-10-01T10:33:01.865380+0000 mon.a (mon.0) 576 : cluster 3 Health check update: Degraded data redundancy: 11194/176907 objects degraded (6.328%), 7 pgs degraded (PG_DEGRADED) 2023-10-01T10:33:03.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:02 smithi078 bash[78274]: audit 2023-10-01T10:33:02.398218+0000 mon.a (mon.0) 577 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:33:03.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:02 smithi078 bash[75299]: cluster 2023-10-01T10:33:01.515592+0000 mgr.y (mgr.25072) 411 : cluster 0 pgmap v337: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 61 MiB/s wr, 978 op/s; 11194/176907 objects degraded (6.328%); 4.0 MiB/s, 63 objects/s recovering 2023-10-01T10:33:03.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:02 smithi078 bash[75299]: cluster 2023-10-01T10:33:01.585880+0000 osd.5 (osd.5) 21 : cluster 3 failed to encode map e104 with expected crc 2023-10-01T10:33:03.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:02 smithi078 bash[75299]: cluster 2023-10-01T10:33:01.587084+0000 osd.7 (osd.7) 35 : cluster 3 failed to encode map e104 with expected crc 2023-10-01T10:33:03.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:02 smithi078 bash[75299]: cluster 2023-10-01T10:33:01.588579+0000 mon.a (mon.0) 575 : cluster 0 osdmap e104: 8 total, 8 up, 8 in 2023-10-01T10:33:03.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:02 smithi078 bash[75299]: cluster 2023-10-01T10:33:01.590841+0000 osd.2 (osd.2) 25 : cluster 3 failed to encode map e104 with expected crc 2023-10-01T10:33:03.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:02 smithi078 bash[75299]: cluster 2023-10-01T10:33:01.595396+0000 osd.3 (osd.3) 26 : cluster 3 failed to encode map e104 with expected crc 2023-10-01T10:33:03.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:02 smithi078 bash[75299]: cluster 2023-10-01T10:33:01.605109+0000 osd.6 (osd.6) 17 : cluster 3 failed to encode map e104 with expected crc 2023-10-01T10:33:03.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:02 smithi078 bash[75299]: cluster 2023-10-01T10:33:01.865380+0000 mon.a (mon.0) 576 : cluster 3 Health check update: Degraded data redundancy: 11194/176907 objects degraded (6.328%), 7 pgs degraded (PG_DEGRADED) 2023-10-01T10:33:03.085 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:02 smithi078 bash[75299]: audit 2023-10-01T10:33:02.398218+0000 mon.a (mon.0) 577 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:33:03.614 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 45 16 62984 62968 87.4437 63.3125 0.0337254 0.0114285 2023-10-01T10:33:03.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:03 smithi173 bash[58816]: cluster 2023-10-01T10:33:02.594224+0000 osd.6 (osd.6) 18 : cluster 3 failed to encode map e105 with expected crc 2023-10-01T10:33:03.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:03 smithi173 bash[58816]: cluster 2023-10-01T10:33:02.594244+0000 osd.7 (osd.7) 36 : cluster 3 failed to encode map e105 with expected crc 2023-10-01T10:33:03.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:03 smithi173 bash[58816]: audit 2023-10-01T10:33:02.594516+0000 mon.c (mon.2) 67 : audit 0 from='client.? 172.21.15.78:0/979316372' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:03.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:03 smithi173 bash[58816]: cluster 2023-10-01T10:33:02.594593+0000 osd.2 (osd.2) 26 : cluster 3 failed to encode map e105 with expected crc 2023-10-01T10:33:03.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:03 smithi173 bash[58816]: cluster 2023-10-01T10:33:02.595144+0000 mon.a (mon.0) 578 : cluster 0 osdmap e105: 8 total, 8 up, 8 in 2023-10-01T10:33:03.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:03 smithi173 bash[58816]: cluster 2023-10-01T10:33:02.595936+0000 osd.5 (osd.5) 22 : cluster 3 failed to encode map e105 with expected crc 2023-10-01T10:33:03.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:03 smithi173 bash[58816]: cluster 2023-10-01T10:33:02.613148+0000 osd.3 (osd.3) 27 : cluster 3 failed to encode map e105 with expected crc 2023-10-01T10:33:03.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:03 smithi173 bash[58816]: cluster 2023-10-01T10:33:03.516148+0000 mgr.y (mgr.25072) 412 : cluster 0 pgmap v340: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 44 MiB/s wr, 704 op/s; 11194/178647 objects degraded (6.266%); 3.1 MiB/s, 49 objects/s recovering 2023-10-01T10:33:04.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:03 smithi078 bash[75299]: cluster 2023-10-01T10:33:02.594224+0000 osd.6 (osd.6) 18 : cluster 3 failed to encode map e105 with expected crc 2023-10-01T10:33:04.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:03 smithi078 bash[75299]: cluster 2023-10-01T10:33:02.594244+0000 osd.7 (osd.7) 36 : cluster 3 failed to encode map e105 with expected crc 2023-10-01T10:33:04.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:03 smithi078 bash[75299]: audit 2023-10-01T10:33:02.594516+0000 mon.c (mon.2) 67 : audit 0 from='client.? 172.21.15.78:0/979316372' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:04.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:03 smithi078 bash[75299]: cluster 2023-10-01T10:33:02.594593+0000 osd.2 (osd.2) 26 : cluster 3 failed to encode map e105 with expected crc 2023-10-01T10:33:04.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:03 smithi078 bash[75299]: cluster 2023-10-01T10:33:02.595144+0000 mon.a (mon.0) 578 : cluster 0 osdmap e105: 8 total, 8 up, 8 in 2023-10-01T10:33:04.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:03 smithi078 bash[75299]: cluster 2023-10-01T10:33:02.595936+0000 osd.5 (osd.5) 22 : cluster 3 failed to encode map e105 with expected crc 2023-10-01T10:33:04.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:03 smithi078 bash[75299]: cluster 2023-10-01T10:33:02.613148+0000 osd.3 (osd.3) 27 : cluster 3 failed to encode map e105 with expected crc 2023-10-01T10:33:04.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:03 smithi078 bash[75299]: cluster 2023-10-01T10:33:03.516148+0000 mgr.y (mgr.25072) 412 : cluster 0 pgmap v340: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 44 MiB/s wr, 704 op/s; 11194/178647 objects degraded (6.266%); 3.1 MiB/s, 49 objects/s recovering 2023-10-01T10:33:04.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:03 smithi078 bash[78274]: cluster 2023-10-01T10:33:02.594224+0000 osd.6 (osd.6) 18 : cluster 3 failed to encode map e105 with expected crc 2023-10-01T10:33:04.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:03 smithi078 bash[78274]: cluster 2023-10-01T10:33:02.594244+0000 osd.7 (osd.7) 36 : cluster 3 failed to encode map e105 with expected crc 2023-10-01T10:33:04.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:03 smithi078 bash[78274]: audit 2023-10-01T10:33:02.594516+0000 mon.c (mon.2) 67 : audit 0 from='client.? 172.21.15.78:0/979316372' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:04.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:03 smithi078 bash[78274]: cluster 2023-10-01T10:33:02.594593+0000 osd.2 (osd.2) 26 : cluster 3 failed to encode map e105 with expected crc 2023-10-01T10:33:04.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:03 smithi078 bash[78274]: cluster 2023-10-01T10:33:02.595144+0000 mon.a (mon.0) 578 : cluster 0 osdmap e105: 8 total, 8 up, 8 in 2023-10-01T10:33:04.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:03 smithi078 bash[78274]: cluster 2023-10-01T10:33:02.595936+0000 osd.5 (osd.5) 22 : cluster 3 failed to encode map e105 with expected crc 2023-10-01T10:33:04.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:03 smithi078 bash[78274]: cluster 2023-10-01T10:33:02.613148+0000 osd.3 (osd.3) 27 : cluster 3 failed to encode map e105 with expected crc 2023-10-01T10:33:04.085 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:03 smithi078 bash[78274]: cluster 2023-10-01T10:33:03.516148+0000 mgr.y (mgr.25072) 412 : cluster 0 pgmap v340: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 44 MiB/s wr, 704 op/s; 11194/178647 objects degraded (6.266%); 3.1 MiB/s, 49 objects/s recovering 2023-10-01T10:33:04.614 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 46 16 63994 63978 86.9149 63.125 0.00648957 0.0114982 2023-10-01T10:33:05.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:05 smithi078 bash[75299]: audit 2023-10-01T10:33:05.167179+0000 mon.c (mon.2) 68 : audit 0 from='client.? 172.21.15.78:0/1658568663' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:05.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:05 smithi078 bash[78274]: audit 2023-10-01T10:33:05.167179+0000 mon.c (mon.2) 68 : audit 0 from='client.? 172.21.15.78:0/1658568663' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:05.614 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 47 16 64978 64962 86.3741 61.5 0.0135537 0.0115712 2023-10-01T10:33:05.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:05 smithi173 bash[58816]: audit 2023-10-01T10:33:05.167179+0000 mon.c (mon.2) 68 : audit 0 from='client.? 172.21.15.78:0/1658568663' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:06.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:06 smithi078 bash[75299]: cluster 2023-10-01T10:33:05.516964+0000 mgr.y (mgr.25072) 413 : cluster 0 pgmap v341: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s; 10832/190623 objects degraded (5.682%); 5.9 MiB/s, 94 objects/s recovering 2023-10-01T10:33:06.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:06 smithi078 bash[78274]: cluster 2023-10-01T10:33:05.516964+0000 mgr.y (mgr.25072) 413 : cluster 0 pgmap v341: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s; 10832/190623 objects degraded (5.682%); 5.9 MiB/s, 94 objects/s recovering 2023-10-01T10:33:06.614 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 48 16 66004 65988 85.9105 64.125 0.0149752 0.0116339 2023-10-01T10:33:06.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:06 smithi173 bash[58816]: cluster 2023-10-01T10:33:05.516964+0000 mgr.y (mgr.25072) 413 : cluster 0 pgmap v341: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s; 10832/190623 objects degraded (5.682%); 5.9 MiB/s, 94 objects/s recovering 2023-10-01T10:33:07.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:07 smithi078 bash[75299]: cluster 2023-10-01T10:33:06.866895+0000 mon.a (mon.0) 579 : cluster 3 Health check update: Degraded data redundancy: 10832/190623 objects degraded (5.682%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T10:33:07.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:07 smithi078 bash[78274]: cluster 2023-10-01T10:33:06.866895+0000 mon.a (mon.0) 579 : cluster 3 Health check update: Degraded data redundancy: 10832/190623 objects degraded (5.682%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T10:33:07.614 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 49 16 66961 66945 85.3778 59.8125 0.00873182 0.0117052 2023-10-01T10:33:07.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:07 smithi173 bash[58816]: cluster 2023-10-01T10:33:06.866895+0000 mon.a (mon.0) 579 : cluster 3 Health check update: Degraded data redundancy: 10832/190623 objects degraded (5.682%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T10:33:08.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:08 smithi078 bash[78274]: cluster 2023-10-01T10:33:07.517671+0000 mgr.y (mgr.25072) 414 : cluster 0 pgmap v342: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 45 MiB/s wr, 726 op/s; 10832/190623 objects degraded (5.682%); 2.8 MiB/s, 45 objects/s recovering 2023-10-01T10:33:08.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:08 smithi078 bash[78274]: audit 2023-10-01T10:33:07.777930+0000 mon.b (mon.1) 26 : audit 0 from='client.? 172.21.15.78:0/2663666985' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:08.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:08 smithi078 bash[75299]: cluster 2023-10-01T10:33:07.517671+0000 mgr.y (mgr.25072) 414 : cluster 0 pgmap v342: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 45 MiB/s wr, 726 op/s; 10832/190623 objects degraded (5.682%); 2.8 MiB/s, 45 objects/s recovering 2023-10-01T10:33:08.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:08 smithi078 bash[75299]: audit 2023-10-01T10:33:07.777930+0000 mon.b (mon.1) 26 : audit 0 from='client.? 172.21.15.78:0/2663666985' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:08.615 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 50 16 67943 67927 84.8976 61.375 0.0282482 0.0117729 2023-10-01T10:33:08.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:08 smithi173 bash[58816]: cluster 2023-10-01T10:33:07.517671+0000 mgr.y (mgr.25072) 414 : cluster 0 pgmap v342: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 45 MiB/s wr, 726 op/s; 10832/190623 objects degraded (5.682%); 2.8 MiB/s, 45 objects/s recovering 2023-10-01T10:33:08.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:08 smithi173 bash[58816]: audit 2023-10-01T10:33:07.777930+0000 mon.b (mon.1) 26 : audit 0 from='client.? 172.21.15.78:0/2663666985' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:09.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:09 smithi078 bash[75299]: audit 2023-10-01T10:33:09.245866+0000 mon.a (mon.0) 580 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:33:09.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:33:09 smithi078 bash[71214]: debug 2023-10-01T10:33:09.243+0000 7fd9c245e700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:33:09.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:09 smithi078 bash[78274]: audit 2023-10-01T10:33:09.245866+0000 mon.a (mon.0) 580 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:33:09.615 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 51 16 68925 68909 84.4363 61.375 0.00832575 0.0118361 2023-10-01T10:33:09.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:09 smithi173 bash[58816]: audit 2023-10-01T10:33:09.245866+0000 mon.a (mon.0) 580 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:33:10.268 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:10 smithi173 bash[58816]: audit 2023-10-01T10:33:09.246108+0000 mgr.y (mgr.25072) 415 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:33:10.269 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:10 smithi173 bash[58816]: cephadm 2023-10-01T10:33:09.247881+0000 mgr.y (mgr.25072) 416 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:33:10.269 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:10 smithi173 bash[58816]: cluster 2023-10-01T10:33:09.518493+0000 mgr.y (mgr.25072) 417 : cluster 0 pgmap v343: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 62 MiB/s wr, 996 op/s; 10596/200829 objects degraded (5.276%); 4.7 MiB/s, 74 objects/s recovering 2023-10-01T10:33:10.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:10 smithi078 bash[75299]: audit 2023-10-01T10:33:09.246108+0000 mgr.y (mgr.25072) 415 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:33:10.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:10 smithi078 bash[75299]: cephadm 2023-10-01T10:33:09.247881+0000 mgr.y (mgr.25072) 416 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:33:10.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:10 smithi078 bash[75299]: cluster 2023-10-01T10:33:09.518493+0000 mgr.y (mgr.25072) 417 : cluster 0 pgmap v343: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 62 MiB/s wr, 996 op/s; 10596/200829 objects degraded (5.276%); 4.7 MiB/s, 74 objects/s recovering 2023-10-01T10:33:10.584 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:33:10 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:33:10] "GET /metrics HTTP/1.1" 200 36208 "" "Prometheus/2.33.4" 2023-10-01T10:33:10.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:10 smithi078 bash[78274]: audit 2023-10-01T10:33:09.246108+0000 mgr.y (mgr.25072) 415 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:33:10.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:10 smithi078 bash[78274]: cephadm 2023-10-01T10:33:09.247881+0000 mgr.y (mgr.25072) 416 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:33:10.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:10 smithi078 bash[78274]: cluster 2023-10-01T10:33:09.518493+0000 mgr.y (mgr.25072) 417 : cluster 0 pgmap v343: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 62 MiB/s wr, 996 op/s; 10596/200829 objects degraded (5.276%); 4.7 MiB/s, 74 objects/s recovering 2023-10-01T10:33:10.615 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 52 16 69843 69827 83.9158 57.375 0.00419627 0.011909 2023-10-01T10:33:11.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:11 smithi078 bash[75299]: audit 2023-10-01T10:33:10.418144+0000 mon.a (mon.0) 581 : audit 0 from='client.? 172.21.15.78:0/1468748468' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:11.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:11 smithi078 bash[78274]: audit 2023-10-01T10:33:10.418144+0000 mon.a (mon.0) 581 : audit 0 from='client.? 172.21.15.78:0/1468748468' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:11.615 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 53 16 70784 70768 83.442 58.8125 0.0495014 0.0119775 2023-10-01T10:33:11.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:11 smithi173 bash[58816]: audit 2023-10-01T10:33:10.418144+0000 mon.a (mon.0) 581 : audit 0 from='client.? 172.21.15.78:0/1468748468' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:12.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:12 smithi078 bash[75299]: cluster 2023-10-01T10:33:11.519198+0000 mgr.y (mgr.25072) 418 : cluster 0 pgmap v344: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 60 MiB/s wr, 952 op/s; 10596/205296 objects degraded (5.161%); 3.8 MiB/s, 60 objects/s recovering 2023-10-01T10:33:12.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:12 smithi078 bash[75299]: cluster 2023-10-01T10:33:11.868652+0000 mon.a (mon.0) 582 : cluster 3 Health check update: Degraded data redundancy: 10596/200829 objects degraded (5.276%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T10:33:12.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:12 smithi078 bash[78274]: cluster 2023-10-01T10:33:11.519198+0000 mgr.y (mgr.25072) 418 : cluster 0 pgmap v344: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 60 MiB/s wr, 952 op/s; 10596/205296 objects degraded (5.161%); 3.8 MiB/s, 60 objects/s recovering 2023-10-01T10:33:12.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:12 smithi078 bash[78274]: cluster 2023-10-01T10:33:11.868652+0000 mon.a (mon.0) 582 : cluster 3 Health check update: Degraded data redundancy: 10596/200829 objects degraded (5.276%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T10:33:12.615 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 54 16 71731 71715 82.9928 59.1875 0.0167277 0.0120408 2023-10-01T10:33:12.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:12 smithi173 bash[58816]: cluster 2023-10-01T10:33:11.519198+0000 mgr.y (mgr.25072) 418 : cluster 0 pgmap v344: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 60 MiB/s wr, 952 op/s; 10596/205296 objects degraded (5.161%); 3.8 MiB/s, 60 objects/s recovering 2023-10-01T10:33:12.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:12 smithi173 bash[58816]: cluster 2023-10-01T10:33:11.868652+0000 mon.a (mon.0) 582 : cluster 3 Health check update: Degraded data redundancy: 10596/200829 objects degraded (5.276%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T10:33:13.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:13 smithi078 bash[75299]: audit 2023-10-01T10:33:13.021218+0000 mon.b (mon.1) 27 : audit 0 from='client.? 172.21.15.78:0/3277914774' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:13.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:13 smithi078 bash[78274]: audit 2023-10-01T10:33:13.021218+0000 mon.b (mon.1) 27 : audit 0 from='client.? 172.21.15.78:0/3277914774' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:13.615 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 55 16 72661 72645 82.5406 58.125 0.00803598 0.0121089 2023-10-01T10:33:13.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:13 smithi173 bash[58816]: audit 2023-10-01T10:33:13.021218+0000 mon.b (mon.1) 27 : audit 0 from='client.? 172.21.15.78:0/3277914774' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:14.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:14 smithi078 bash[75299]: cluster 2023-10-01T10:33:13.519809+0000 mgr.y (mgr.25072) 419 : cluster 0 pgmap v345: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 58 MiB/s wr, 921 op/s; 10596/207126 objects degraded (5.116%); 3.4 MiB/s, 54 objects/s recovering 2023-10-01T10:33:14.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:14 smithi078 bash[78274]: cluster 2023-10-01T10:33:13.519809+0000 mgr.y (mgr.25072) 419 : cluster 0 pgmap v345: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 58 MiB/s wr, 921 op/s; 10596/207126 objects degraded (5.116%); 3.4 MiB/s, 54 objects/s recovering 2023-10-01T10:33:14.615 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 56 16 73599 73583 82.1134 58.625 0.0451623 0.0121717 2023-10-01T10:33:14.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:14 smithi173 bash[58816]: cluster 2023-10-01T10:33:13.519809+0000 mgr.y (mgr.25072) 419 : cluster 0 pgmap v345: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 58 MiB/s wr, 921 op/s; 10596/207126 objects degraded (5.116%); 3.4 MiB/s, 54 objects/s recovering 2023-10-01T10:33:15.615 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 57 16 74530 74514 81.6936 58.1875 0.0130764 0.0122349 2023-10-01T10:33:16.615 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 58 16 75469 75453 81.2968 58.6875 0.0322899 0.0122941 2023-10-01T10:33:16.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:16 smithi078 bash[75299]: cluster 2023-10-01T10:33:15.520640+0000 mgr.y (mgr.25072) 420 : cluster 0 pgmap v346: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s; 10364/219405 objects degraded (4.724%); 4.3 MiB/s, 69 objects/s recovering 2023-10-01T10:33:16.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:16 smithi078 bash[75299]: audit 2023-10-01T10:33:15.645390+0000 mon.c (mon.2) 69 : audit 0 from='client.? 172.21.15.78:0/2324251343' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:16.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:16 smithi078 bash[78274]: cluster 2023-10-01T10:33:15.520640+0000 mgr.y (mgr.25072) 420 : cluster 0 pgmap v346: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s; 10364/219405 objects degraded (4.724%); 4.3 MiB/s, 69 objects/s recovering 2023-10-01T10:33:16.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:16 smithi078 bash[78274]: audit 2023-10-01T10:33:15.645390+0000 mon.c (mon.2) 69 : audit 0 from='client.? 172.21.15.78:0/2324251343' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:16.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:16 smithi173 bash[58816]: cluster 2023-10-01T10:33:15.520640+0000 mgr.y (mgr.25072) 420 : cluster 0 pgmap v346: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s; 10364/219405 objects degraded (4.724%); 4.3 MiB/s, 69 objects/s recovering 2023-10-01T10:33:16.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:16 smithi173 bash[58816]: audit 2023-10-01T10:33:15.645390+0000 mon.c (mon.2) 69 : audit 0 from='client.? 172.21.15.78:0/2324251343' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:17.615 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 59 16 76366 76350 80.869 56.0625 0.0113885 0.0123589 2023-10-01T10:33:17.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:17 smithi078 bash[75299]: cluster 2023-10-01T10:33:16.869876+0000 mon.a (mon.0) 583 : cluster 3 Health check update: Degraded data redundancy: 10364/219405 objects degraded (4.724%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T10:33:17.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:17 smithi078 bash[75299]: audit 2023-10-01T10:33:17.398369+0000 mon.a (mon.0) 584 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:33:17.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:17 smithi078 bash[78274]: cluster 2023-10-01T10:33:16.869876+0000 mon.a (mon.0) 583 : cluster 3 Health check update: Degraded data redundancy: 10364/219405 objects degraded (4.724%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T10:33:17.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:17 smithi078 bash[78274]: audit 2023-10-01T10:33:17.398369+0000 mon.a (mon.0) 584 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:33:17.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:17 smithi173 bash[58816]: cluster 2023-10-01T10:33:16.869876+0000 mon.a (mon.0) 583 : cluster 3 Health check update: Degraded data redundancy: 10364/219405 objects degraded (4.724%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T10:33:17.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:17 smithi173 bash[58816]: audit 2023-10-01T10:33:17.398369+0000 mon.a (mon.0) 584 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:33:18.615 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:2023-10-01T10:33:18.613626+0000 min lat: 0.00124552 max lat: 2.17561 avg lat: 0.0124284 2023-10-01T10:33:18.616 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T10:33:18.616 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 60 16 77235 77219 80.4264 54.3125 0.0219004 0.0124284 2023-10-01T10:33:18.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:18 smithi173 bash[58816]: cluster 2023-10-01T10:33:17.521230+0000 mgr.y (mgr.25072) 421 : cluster 0 pgmap v347: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 50 MiB/s wr, 799 op/s; 10364/219405 objects degraded (4.724%); 2.4 MiB/s, 38 objects/s recovering 2023-10-01T10:33:18.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:18 smithi173 bash[58816]: audit 2023-10-01T10:33:18.238266+0000 mon.a (mon.0) 585 : audit 0 from='client.? 172.21.15.78:0/4186874737' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:19.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:18 smithi078 bash[75299]: cluster 2023-10-01T10:33:17.521230+0000 mgr.y (mgr.25072) 421 : cluster 0 pgmap v347: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 50 MiB/s wr, 799 op/s; 10364/219405 objects degraded (4.724%); 2.4 MiB/s, 38 objects/s recovering 2023-10-01T10:33:19.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:18 smithi078 bash[75299]: audit 2023-10-01T10:33:18.238266+0000 mon.a (mon.0) 585 : audit 0 from='client.? 172.21.15.78:0/4186874737' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:19.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:18 smithi078 bash[78274]: cluster 2023-10-01T10:33:17.521230+0000 mgr.y (mgr.25072) 421 : cluster 0 pgmap v347: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 50 MiB/s wr, 799 op/s; 10364/219405 objects degraded (4.724%); 2.4 MiB/s, 38 objects/s recovering 2023-10-01T10:33:19.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:18 smithi078 bash[78274]: audit 2023-10-01T10:33:18.238266+0000 mon.a (mon.0) 585 : audit 0 from='client.? 172.21.15.78:0/4186874737' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:19.616 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 61 16 78183 78167 80.0791 59.25 0.0394086 0.0124807 2023-10-01T10:33:20.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:33:20 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:33:20] "GET /metrics HTTP/1.1" 200 36207 "" "Prometheus/2.33.4" 2023-10-01T10:33:20.616 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 62 16 79069 79053 79.6806 55.375 0.00925592 0.0125421 2023-10-01T10:33:20.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:20 smithi173 bash[58816]: cluster 2023-10-01T10:33:19.522003+0000 mgr.y (mgr.25072) 422 : cluster 0 pgmap v348: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 67 MiB/s wr, 1.06k op/s; 10118/228960 objects degraded (4.419%); 3.7 MiB/s, 59 objects/s recovering 2023-10-01T10:33:21.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:20 smithi078 bash[78274]: cluster 2023-10-01T10:33:19.522003+0000 mgr.y (mgr.25072) 422 : cluster 0 pgmap v348: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 67 MiB/s wr, 1.06k op/s; 10118/228960 objects degraded (4.419%); 3.7 MiB/s, 59 objects/s recovering 2023-10-01T10:33:21.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:20 smithi078 bash[75299]: cluster 2023-10-01T10:33:19.522003+0000 mgr.y (mgr.25072) 422 : cluster 0 pgmap v348: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 67 MiB/s wr, 1.06k op/s; 10118/228960 objects degraded (4.419%); 3.7 MiB/s, 59 objects/s recovering 2023-10-01T10:33:21.616 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 63 16 79955 79939 79.2947 55.375 0.0283391 0.0126052 2023-10-01T10:33:21.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:21 smithi173 bash[58816]: audit 2023-10-01T10:33:20.838002+0000 mon.c (mon.2) 70 : audit 0 from='client.? 172.21.15.78:0/499776144' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:22.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:21 smithi078 bash[78274]: audit 2023-10-01T10:33:20.838002+0000 mon.c (mon.2) 70 : audit 0 from='client.? 172.21.15.78:0/499776144' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:22.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:21 smithi078 bash[75299]: audit 2023-10-01T10:33:20.838002+0000 mon.c (mon.2) 70 : audit 0 from='client.? 172.21.15.78:0/499776144' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:22.616 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 64 16 80865 80849 78.9443 56.875 0.0165306 0.0126586 2023-10-01T10:33:22.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:22 smithi173 bash[58816]: cluster 2023-10-01T10:33:21.522628+0000 mgr.y (mgr.25072) 423 : cluster 0 pgmap v349: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 56 MiB/s wr, 897 op/s; 10118/233163 objects degraded (4.339%); 2.5 MiB/s, 39 objects/s recovering 2023-10-01T10:33:22.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:22 smithi173 bash[58816]: cluster 2023-10-01T10:33:21.871024+0000 mon.a (mon.0) 586 : cluster 3 Health check update: Degraded data redundancy: 10118/233163 objects degraded (4.339%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T10:33:23.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:22 smithi078 bash[75299]: cluster 2023-10-01T10:33:21.522628+0000 mgr.y (mgr.25072) 423 : cluster 0 pgmap v349: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 56 MiB/s wr, 897 op/s; 10118/233163 objects degraded (4.339%); 2.5 MiB/s, 39 objects/s recovering 2023-10-01T10:33:23.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:22 smithi078 bash[75299]: cluster 2023-10-01T10:33:21.871024+0000 mon.a (mon.0) 586 : cluster 3 Health check update: Degraded data redundancy: 10118/233163 objects degraded (4.339%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T10:33:23.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:22 smithi078 bash[78274]: cluster 2023-10-01T10:33:21.522628+0000 mgr.y (mgr.25072) 423 : cluster 0 pgmap v349: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 56 MiB/s wr, 897 op/s; 10118/233163 objects degraded (4.339%); 2.5 MiB/s, 39 objects/s recovering 2023-10-01T10:33:23.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:22 smithi078 bash[78274]: cluster 2023-10-01T10:33:21.871024+0000 mon.a (mon.0) 586 : cluster 3 Health check update: Degraded data redundancy: 10118/233163 objects degraded (4.339%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T10:33:23.616 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 65 16 81685 81669 78.5182 51.25 0.0108723 0.0127284 2023-10-01T10:33:23.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:23 smithi173 bash[58816]: audit 2023-10-01T10:33:23.451811+0000 mon.a (mon.0) 587 : audit 0 from='client.? 172.21.15.78:0/3196149484' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:23.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:23 smithi173 bash[58816]: cluster 2023-10-01T10:33:23.523479+0000 mgr.y (mgr.25072) 424 : cluster 0 pgmap v350: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 51 MiB/s wr, 823 op/s; 10118/234951 objects degraded (4.306%); 2.5 MiB/s, 39 objects/s recovering 2023-10-01T10:33:24.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:23 smithi078 bash[75299]: audit 2023-10-01T10:33:23.451811+0000 mon.a (mon.0) 587 : audit 0 from='client.? 172.21.15.78:0/3196149484' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:24.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:23 smithi078 bash[75299]: cluster 2023-10-01T10:33:23.523479+0000 mgr.y (mgr.25072) 424 : cluster 0 pgmap v350: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 51 MiB/s wr, 823 op/s; 10118/234951 objects degraded (4.306%); 2.5 MiB/s, 39 objects/s recovering 2023-10-01T10:33:24.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:23 smithi078 bash[78274]: audit 2023-10-01T10:33:23.451811+0000 mon.a (mon.0) 587 : audit 0 from='client.? 172.21.15.78:0/3196149484' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:24.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:23 smithi078 bash[78274]: cluster 2023-10-01T10:33:23.523479+0000 mgr.y (mgr.25072) 424 : cluster 0 pgmap v350: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 51 MiB/s wr, 823 op/s; 10118/234951 objects degraded (4.306%); 2.5 MiB/s, 39 objects/s recovering 2023-10-01T10:33:24.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:33:24 smithi078 bash[71214]: debug 2023-10-01T10:33:24.259+0000 7fd9c245e700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:33:24.616 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 66 16 82593 82577 78.1883 56.75 0.018011 0.0127835 2023-10-01T10:33:25.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:24 smithi078 bash[75299]: audit 2023-10-01T10:33:24.261867+0000 mon.a (mon.0) 588 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:33:25.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:24 smithi078 bash[75299]: audit 2023-10-01T10:33:24.262184+0000 mgr.y (mgr.25072) 425 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:33:25.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:24 smithi078 bash[75299]: cephadm 2023-10-01T10:33:24.263278+0000 mgr.y (mgr.25072) 426 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:33:25.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:24 smithi078 bash[78274]: audit 2023-10-01T10:33:24.261867+0000 mon.a (mon.0) 588 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:33:25.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:24 smithi078 bash[78274]: audit 2023-10-01T10:33:24.262184+0000 mgr.y (mgr.25072) 425 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:33:25.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:24 smithi078 bash[78274]: cephadm 2023-10-01T10:33:24.263278+0000 mgr.y (mgr.25072) 426 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:33:25.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:24 smithi173 bash[58816]: audit 2023-10-01T10:33:24.261867+0000 mon.a (mon.0) 588 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:33:25.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:24 smithi173 bash[58816]: audit 2023-10-01T10:33:24.262184+0000 mgr.y (mgr.25072) 425 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:33:25.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:24 smithi173 bash[58816]: cephadm 2023-10-01T10:33:24.263278+0000 mgr.y (mgr.25072) 426 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:33:25.616 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 67 16 83465 83449 77.8347 54.5 0.0335931 0.0128409 2023-10-01T10:33:26.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:25 smithi078 bash[78274]: cluster 2023-10-01T10:33:25.524296+0000 mgr.y (mgr.25072) 427 : cluster 0 pgmap v351: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s; 9848/246429 objects degraded (3.996%); 3.9 MiB/s, 62 objects/s recovering 2023-10-01T10:33:26.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:25 smithi078 bash[75299]: cluster 2023-10-01T10:33:25.524296+0000 mgr.y (mgr.25072) 427 : cluster 0 pgmap v351: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s; 9848/246429 objects degraded (3.996%); 3.9 MiB/s, 62 objects/s recovering 2023-10-01T10:33:26.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:25 smithi173 bash[58816]: cluster 2023-10-01T10:33:25.524296+0000 mgr.y (mgr.25072) 427 : cluster 0 pgmap v351: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s; 9848/246429 objects degraded (3.996%); 3.9 MiB/s, 62 objects/s recovering 2023-10-01T10:33:26.616 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 68 16 84299 84283 77.4565 52.125 0.0298025 0.0129035 2023-10-01T10:33:27.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:26 smithi078 bash[75299]: audit 2023-10-01T10:33:26.043046+0000 mon.c (mon.2) 71 : audit 0 from='client.? 172.21.15.78:0/543173898' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:27.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:26 smithi078 bash[78274]: audit 2023-10-01T10:33:26.043046+0000 mon.c (mon.2) 71 : audit 0 from='client.? 172.21.15.78:0/543173898' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:27.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:26 smithi173 bash[58816]: audit 2023-10-01T10:33:26.043046+0000 mon.c (mon.2) 71 : audit 0 from='client.? 172.21.15.78:0/543173898' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:27.616 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 69 16 85165 85149 77.1183 54.125 0.0151538 0.0129602 2023-10-01T10:33:28.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:27 smithi078 bash[75299]: cluster 2023-10-01T10:33:26.872317+0000 mon.a (mon.0) 589 : cluster 3 Health check update: Degraded data redundancy: 9848/246429 objects degraded (3.996%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T10:33:28.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:27 smithi078 bash[75299]: cluster 2023-10-01T10:33:27.524818+0000 mgr.y (mgr.25072) 428 : cluster 0 pgmap v352: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 47 MiB/s wr, 750 op/s; 9848/246429 objects degraded (3.996%); 2.7 MiB/s, 42 objects/s recovering 2023-10-01T10:33:28.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:27 smithi078 bash[78274]: cluster 2023-10-01T10:33:26.872317+0000 mon.a (mon.0) 589 : cluster 3 Health check update: Degraded data redundancy: 9848/246429 objects degraded (3.996%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T10:33:28.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:27 smithi078 bash[78274]: cluster 2023-10-01T10:33:27.524818+0000 mgr.y (mgr.25072) 428 : cluster 0 pgmap v352: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 47 MiB/s wr, 750 op/s; 9848/246429 objects degraded (3.996%); 2.7 MiB/s, 42 objects/s recovering 2023-10-01T10:33:28.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:27 smithi173 bash[58816]: cluster 2023-10-01T10:33:26.872317+0000 mon.a (mon.0) 589 : cluster 3 Health check update: Degraded data redundancy: 9848/246429 objects degraded (3.996%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T10:33:28.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:27 smithi173 bash[58816]: cluster 2023-10-01T10:33:27.524818+0000 mgr.y (mgr.25072) 428 : cluster 0 pgmap v352: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 47 MiB/s wr, 750 op/s; 9848/246429 objects degraded (3.996%); 2.7 MiB/s, 42 objects/s recovering 2023-10-01T10:33:28.616 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 70 16 86021 86005 76.7809 53.5 0.00916655 0.0130182 2023-10-01T10:33:29.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:28 smithi078 bash[75299]: audit 2023-10-01T10:33:28.621953+0000 mon.a (mon.0) 590 : audit 0 from='client.? 172.21.15.78:0/1525291281' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:29.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:28 smithi078 bash[78274]: audit 2023-10-01T10:33:28.621953+0000 mon.a (mon.0) 590 : audit 0 from='client.? 172.21.15.78:0/1525291281' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:29.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:28 smithi173 bash[58816]: audit 2023-10-01T10:33:28.621953+0000 mon.a (mon.0) 590 : audit 0 from='client.? 172.21.15.78:0/1525291281' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:29.617 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 71 16 86891 86875 76.4652 54.375 0.0198065 0.0130711 2023-10-01T10:33:30.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:29 smithi078 bash[78274]: cluster 2023-10-01T10:33:29.525574+0000 mgr.y (mgr.25072) 429 : cluster 0 pgmap v353: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 62 MiB/s wr, 999 op/s; 9589/255390 objects degraded (3.755%); 4.0 MiB/s, 64 objects/s recovering 2023-10-01T10:33:30.271 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:29 smithi078 bash[75299]: cluster 2023-10-01T10:33:29.525574+0000 mgr.y (mgr.25072) 429 : cluster 0 pgmap v353: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 62 MiB/s wr, 999 op/s; 9589/255390 objects degraded (3.755%); 4.0 MiB/s, 64 objects/s recovering 2023-10-01T10:33:30.271 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:33:30 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:33:30] "GET /metrics HTTP/1.1" 200 36207 "" "Prometheus/2.33.4" 2023-10-01T10:33:30.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:29 smithi173 bash[58816]: cluster 2023-10-01T10:33:29.525574+0000 mgr.y (mgr.25072) 429 : cluster 0 pgmap v353: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 62 MiB/s wr, 999 op/s; 9589/255390 objects degraded (3.755%); 4.0 MiB/s, 64 objects/s recovering 2023-10-01T10:33:30.617 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 72 16 87770 87754 76.1661 54.9375 0.00657623 0.0131225 2023-10-01T10:33:31.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:31 smithi078 bash[75299]: audit 2023-10-01T10:33:31.242982+0000 mon.a (mon.0) 591 : audit 0 from='client.? 172.21.15.78:0/3806614928' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:31.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:31 smithi078 bash[78274]: audit 2023-10-01T10:33:31.242982+0000 mon.a (mon.0) 591 : audit 0 from='client.? 172.21.15.78:0/3806614928' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:31.617 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 73 16 88612 88596 75.8436 52.625 0.0180958 0.0131782 2023-10-01T10:33:31.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:31 smithi173 bash[58816]: audit 2023-10-01T10:33:31.242982+0000 mon.a (mon.0) 591 : audit 0 from='client.? 172.21.15.78:0/3806614928' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:32.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:32 smithi078 bash[75299]: cluster 2023-10-01T10:33:31.526245+0000 mgr.y (mgr.25072) 430 : cluster 0 pgmap v354: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 53 MiB/s wr, 846 op/s; 9589/259428 objects degraded (3.696%); 2.8 MiB/s, 44 objects/s recovering 2023-10-01T10:33:32.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:32 smithi078 bash[75299]: cluster 2023-10-01T10:33:31.874006+0000 mon.a (mon.0) 592 : cluster 3 Health check update: Degraded data redundancy: 9589/255390 objects degraded (3.755%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T10:33:32.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:32 smithi078 bash[78274]: cluster 2023-10-01T10:33:31.526245+0000 mgr.y (mgr.25072) 430 : cluster 0 pgmap v354: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 53 MiB/s wr, 846 op/s; 9589/259428 objects degraded (3.696%); 2.8 MiB/s, 44 objects/s recovering 2023-10-01T10:33:32.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:32 smithi078 bash[78274]: cluster 2023-10-01T10:33:31.874006+0000 mon.a (mon.0) 592 : cluster 3 Health check update: Degraded data redundancy: 9589/255390 objects degraded (3.755%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T10:33:32.617 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 74 16 89477 89461 75.5492 54.0625 0.0133736 0.0132297 2023-10-01T10:33:32.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:32 smithi173 bash[58816]: cluster 2023-10-01T10:33:31.526245+0000 mgr.y (mgr.25072) 430 : cluster 0 pgmap v354: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 53 MiB/s wr, 846 op/s; 9589/259428 objects degraded (3.696%); 2.8 MiB/s, 44 objects/s recovering 2023-10-01T10:33:32.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:32 smithi173 bash[58816]: cluster 2023-10-01T10:33:31.874006+0000 mon.a (mon.0) 592 : cluster 3 Health check update: Degraded data redundancy: 9589/255390 objects degraded (3.755%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T10:33:33.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:33 smithi078 bash[78274]: audit 2023-10-01T10:33:32.398489+0000 mon.a (mon.0) 593 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:33:33.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:33 smithi078 bash[75299]: audit 2023-10-01T10:33:32.398489+0000 mon.a (mon.0) 593 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:33:33.617 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 75 16 90320 90304 75.2443 52.6875 0.016299 0.0132817 2023-10-01T10:33:33.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:33 smithi173 bash[58816]: audit 2023-10-01T10:33:32.398489+0000 mon.a (mon.0) 593 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:33:34.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:34 smithi078 bash[78274]: cluster 2023-10-01T10:33:33.526892+0000 mgr.y (mgr.25072) 431 : cluster 0 pgmap v355: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 49 MiB/s wr, 778 op/s; 9589/261186 objects degraded (3.671%); 2.8 MiB/s, 44 objects/s recovering 2023-10-01T10:33:34.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:34 smithi078 bash[78274]: audit 2023-10-01T10:33:33.844605+0000 mon.c (mon.2) 72 : audit 0 from='client.? 172.21.15.78:0/917804762' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:34.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:34 smithi078 bash[75299]: cluster 2023-10-01T10:33:33.526892+0000 mgr.y (mgr.25072) 431 : cluster 0 pgmap v355: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 49 MiB/s wr, 778 op/s; 9589/261186 objects degraded (3.671%); 2.8 MiB/s, 44 objects/s recovering 2023-10-01T10:33:34.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:34 smithi078 bash[75299]: audit 2023-10-01T10:33:33.844605+0000 mon.c (mon.2) 72 : audit 0 from='client.? 172.21.15.78:0/917804762' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:34.617 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 76 16 91159 91143 74.9442 52.4375 0.0141043 0.013337 2023-10-01T10:33:34.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:34 smithi173 bash[58816]: cluster 2023-10-01T10:33:33.526892+0000 mgr.y (mgr.25072) 431 : cluster 0 pgmap v355: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 49 MiB/s wr, 778 op/s; 9589/261186 objects degraded (3.671%); 2.8 MiB/s, 44 objects/s recovering 2023-10-01T10:33:34.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:34 smithi173 bash[58816]: audit 2023-10-01T10:33:33.844605+0000 mon.c (mon.2) 72 : audit 0 from='client.? 172.21.15.78:0/917804762' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:35.617 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 77 16 92011 91995 74.6624 53.25 0.0204685 0.0133847 2023-10-01T10:33:36.617 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 78 16 92279 92263 73.9199 16.75 0.921672 0.0134112 2023-10-01T10:33:36.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:36 smithi078 bash[78274]: cluster 2023-10-01T10:33:35.527763+0000 mgr.y (mgr.25072) 432 : cluster 0 pgmap v356: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 9316/272325 objects degraded (3.421%); 4.2 MiB/s, 66 objects/s recovering 2023-10-01T10:33:36.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:36 smithi078 bash[78274]: cluster 2023-10-01T10:33:35.574750+0000 osd.7 (osd.7) 37 : cluster 3 failed to encode map e106 with expected crc 2023-10-01T10:33:36.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:36 smithi078 bash[78274]: cluster 2023-10-01T10:33:35.575070+0000 mon.a (mon.0) 594 : cluster 0 osdmap e106: 8 total, 8 up, 8 in 2023-10-01T10:33:36.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:36 smithi078 bash[78274]: cluster 2023-10-01T10:33:35.577441+0000 osd.2 (osd.2) 27 : cluster 3 failed to encode map e106 with expected crc 2023-10-01T10:33:36.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:36 smithi078 bash[78274]: cluster 2023-10-01T10:33:35.577642+0000 osd.5 (osd.5) 23 : cluster 3 failed to encode map e106 with expected crc 2023-10-01T10:33:36.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:36 smithi078 bash[78274]: cluster 2023-10-01T10:33:35.579083+0000 osd.5 (osd.5) 24 : cluster 3 failed to encode map e106 with expected crc 2023-10-01T10:33:36.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:36 smithi078 bash[78274]: cluster 2023-10-01T10:33:35.583171+0000 osd.6 (osd.6) 19 : cluster 3 failed to encode map e106 with expected crc 2023-10-01T10:33:36.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:36 smithi078 bash[78274]: cluster 2023-10-01T10:33:35.583382+0000 osd.3 (osd.3) 28 : cluster 3 failed to encode map e106 with expected crc 2023-10-01T10:33:36.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:36 smithi078 bash[78274]: audit 2023-10-01T10:33:36.270539+0000 mon.c (mon.2) 73 : audit 0 from='client.? 172.21.15.78:0/386652292' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:36.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:36 smithi078 bash[75299]: cluster 2023-10-01T10:33:35.527763+0000 mgr.y (mgr.25072) 432 : cluster 0 pgmap v356: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 9316/272325 objects degraded (3.421%); 4.2 MiB/s, 66 objects/s recovering 2023-10-01T10:33:36.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:36 smithi078 bash[75299]: cluster 2023-10-01T10:33:35.574750+0000 osd.7 (osd.7) 37 : cluster 3 failed to encode map e106 with expected crc 2023-10-01T10:33:36.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:36 smithi078 bash[75299]: cluster 2023-10-01T10:33:35.575070+0000 mon.a (mon.0) 594 : cluster 0 osdmap e106: 8 total, 8 up, 8 in 2023-10-01T10:33:36.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:36 smithi078 bash[75299]: cluster 2023-10-01T10:33:35.577441+0000 osd.2 (osd.2) 27 : cluster 3 failed to encode map e106 with expected crc 2023-10-01T10:33:36.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:36 smithi078 bash[75299]: cluster 2023-10-01T10:33:35.577642+0000 osd.5 (osd.5) 23 : cluster 3 failed to encode map e106 with expected crc 2023-10-01T10:33:36.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:36 smithi078 bash[75299]: cluster 2023-10-01T10:33:35.579083+0000 osd.5 (osd.5) 24 : cluster 3 failed to encode map e106 with expected crc 2023-10-01T10:33:36.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:36 smithi078 bash[75299]: cluster 2023-10-01T10:33:35.583171+0000 osd.6 (osd.6) 19 : cluster 3 failed to encode map e106 with expected crc 2023-10-01T10:33:36.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:36 smithi078 bash[75299]: cluster 2023-10-01T10:33:35.583382+0000 osd.3 (osd.3) 28 : cluster 3 failed to encode map e106 with expected crc 2023-10-01T10:33:36.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:36 smithi078 bash[75299]: audit 2023-10-01T10:33:36.270539+0000 mon.c (mon.2) 73 : audit 0 from='client.? 172.21.15.78:0/386652292' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:36.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:36 smithi173 bash[58816]: cluster 2023-10-01T10:33:35.527763+0000 mgr.y (mgr.25072) 432 : cluster 0 pgmap v356: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 9316/272325 objects degraded (3.421%); 4.2 MiB/s, 66 objects/s recovering 2023-10-01T10:33:36.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:36 smithi173 bash[58816]: cluster 2023-10-01T10:33:35.574750+0000 osd.7 (osd.7) 37 : cluster 3 failed to encode map e106 with expected crc 2023-10-01T10:33:36.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:36 smithi173 bash[58816]: cluster 2023-10-01T10:33:35.575070+0000 mon.a (mon.0) 594 : cluster 0 osdmap e106: 8 total, 8 up, 8 in 2023-10-01T10:33:36.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:36 smithi173 bash[58816]: cluster 2023-10-01T10:33:35.577441+0000 osd.2 (osd.2) 27 : cluster 3 failed to encode map e106 with expected crc 2023-10-01T10:33:36.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:36 smithi173 bash[58816]: cluster 2023-10-01T10:33:35.577642+0000 osd.5 (osd.5) 23 : cluster 3 failed to encode map e106 with expected crc 2023-10-01T10:33:36.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:36 smithi173 bash[58816]: cluster 2023-10-01T10:33:35.579083+0000 osd.5 (osd.5) 24 : cluster 3 failed to encode map e106 with expected crc 2023-10-01T10:33:36.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:36 smithi173 bash[58816]: cluster 2023-10-01T10:33:35.583171+0000 osd.6 (osd.6) 19 : cluster 3 failed to encode map e106 with expected crc 2023-10-01T10:33:36.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:36 smithi173 bash[58816]: cluster 2023-10-01T10:33:35.583382+0000 osd.3 (osd.3) 28 : cluster 3 failed to encode map e106 with expected crc 2023-10-01T10:33:36.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:36 smithi173 bash[58816]: audit 2023-10-01T10:33:36.270539+0000 mon.c (mon.2) 73 : audit 0 from='client.? 172.21.15.78:0/386652292' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:37.617 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 79 16 93101 93085 73.6344 51.375 0.0197045 0.0135744 2023-10-01T10:33:37.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:37 smithi173 bash[58816]: cluster 2023-10-01T10:33:36.588531+0000 osd.7 (osd.7) 38 : cluster 3 failed to encode map e107 with expected crc 2023-10-01T10:33:37.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:37 smithi173 bash[58816]: cluster 2023-10-01T10:33:36.588750+0000 osd.5 (osd.5) 25 : cluster 3 failed to encode map e107 with expected crc 2023-10-01T10:33:37.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:37 smithi173 bash[58816]: cluster 2023-10-01T10:33:36.588865+0000 osd.3 (osd.3) 29 : cluster 3 failed to encode map e107 with expected crc 2023-10-01T10:33:37.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:37 smithi173 bash[58816]: cluster 2023-10-01T10:33:36.589772+0000 mon.a (mon.0) 595 : cluster 0 osdmap e107: 8 total, 8 up, 8 in 2023-10-01T10:33:37.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:37 smithi173 bash[58816]: cluster 2023-10-01T10:33:36.590575+0000 osd.6 (osd.6) 20 : cluster 3 failed to encode map e107 with expected crc 2023-10-01T10:33:37.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:37 smithi173 bash[58816]: cluster 2023-10-01T10:33:36.590978+0000 osd.5 (osd.5) 26 : cluster 3 failed to encode map e107 with expected crc 2023-10-01T10:33:37.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:37 smithi173 bash[58816]: cluster 2023-10-01T10:33:36.614425+0000 osd.2 (osd.2) 28 : cluster 3 failed to encode map e107 with expected crc 2023-10-01T10:33:37.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:37 smithi173 bash[58816]: cluster 2023-10-01T10:33:36.875206+0000 mon.a (mon.0) 596 : cluster 3 Health check update: Degraded data redundancy: 9316/272325 objects degraded (3.421%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-01T10:33:38.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:37 smithi078 bash[78274]: cluster 2023-10-01T10:33:36.588531+0000 osd.7 (osd.7) 38 : cluster 3 failed to encode map e107 with expected crc 2023-10-01T10:33:38.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:37 smithi078 bash[78274]: cluster 2023-10-01T10:33:36.588750+0000 osd.5 (osd.5) 25 : cluster 3 failed to encode map e107 with expected crc 2023-10-01T10:33:38.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:37 smithi078 bash[78274]: cluster 2023-10-01T10:33:36.588865+0000 osd.3 (osd.3) 29 : cluster 3 failed to encode map e107 with expected crc 2023-10-01T10:33:38.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:37 smithi078 bash[78274]: cluster 2023-10-01T10:33:36.589772+0000 mon.a (mon.0) 595 : cluster 0 osdmap e107: 8 total, 8 up, 8 in 2023-10-01T10:33:38.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:37 smithi078 bash[78274]: cluster 2023-10-01T10:33:36.590575+0000 osd.6 (osd.6) 20 : cluster 3 failed to encode map e107 with expected crc 2023-10-01T10:33:38.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:37 smithi078 bash[78274]: cluster 2023-10-01T10:33:36.590978+0000 osd.5 (osd.5) 26 : cluster 3 failed to encode map e107 with expected crc 2023-10-01T10:33:38.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:37 smithi078 bash[78274]: cluster 2023-10-01T10:33:36.614425+0000 osd.2 (osd.2) 28 : cluster 3 failed to encode map e107 with expected crc 2023-10-01T10:33:38.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:37 smithi078 bash[78274]: cluster 2023-10-01T10:33:36.875206+0000 mon.a (mon.0) 596 : cluster 3 Health check update: Degraded data redundancy: 9316/272325 objects degraded (3.421%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-01T10:33:38.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:37 smithi078 bash[75299]: cluster 2023-10-01T10:33:36.588531+0000 osd.7 (osd.7) 38 : cluster 3 failed to encode map e107 with expected crc 2023-10-01T10:33:38.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:37 smithi078 bash[75299]: cluster 2023-10-01T10:33:36.588750+0000 osd.5 (osd.5) 25 : cluster 3 failed to encode map e107 with expected crc 2023-10-01T10:33:38.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:37 smithi078 bash[75299]: cluster 2023-10-01T10:33:36.588865+0000 osd.3 (osd.3) 29 : cluster 3 failed to encode map e107 with expected crc 2023-10-01T10:33:38.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:37 smithi078 bash[75299]: cluster 2023-10-01T10:33:36.589772+0000 mon.a (mon.0) 595 : cluster 0 osdmap e107: 8 total, 8 up, 8 in 2023-10-01T10:33:38.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:37 smithi078 bash[75299]: cluster 2023-10-01T10:33:36.590575+0000 osd.6 (osd.6) 20 : cluster 3 failed to encode map e107 with expected crc 2023-10-01T10:33:38.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:37 smithi078 bash[75299]: cluster 2023-10-01T10:33:36.590978+0000 osd.5 (osd.5) 26 : cluster 3 failed to encode map e107 with expected crc 2023-10-01T10:33:38.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:37 smithi078 bash[75299]: cluster 2023-10-01T10:33:36.614425+0000 osd.2 (osd.2) 28 : cluster 3 failed to encode map e107 with expected crc 2023-10-01T10:33:38.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:37 smithi078 bash[75299]: cluster 2023-10-01T10:33:36.875206+0000 mon.a (mon.0) 596 : cluster 3 Health check update: Degraded data redundancy: 9316/272325 objects degraded (3.421%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-10-01T10:33:38.618 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:2023-10-01T10:33:38.615572+0000 min lat: 0.00124552 max lat: 2.17561 avg lat: 0.0136261 2023-10-01T10:33:38.618 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T10:33:38.618 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 80 16 93909 93893 73.3452 50.5 0.0768389 0.0136261 2023-10-01T10:33:38.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:38 smithi173 bash[58816]: cluster 2023-10-01T10:33:37.528398+0000 mgr.y (mgr.25072) 433 : cluster 0 pgmap v359: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 44 MiB/s wr, 705 op/s; 9316/272325 objects degraded (3.421%); 2.1 MiB/s, 34 objects/s recovering 2023-10-01T10:33:39.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:38 smithi078 bash[75299]: cluster 2023-10-01T10:33:37.528398+0000 mgr.y (mgr.25072) 433 : cluster 0 pgmap v359: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 44 MiB/s wr, 705 op/s; 9316/272325 objects degraded (3.421%); 2.1 MiB/s, 34 objects/s recovering 2023-10-01T10:33:39.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:38 smithi078 bash[78274]: cluster 2023-10-01T10:33:37.528398+0000 mgr.y (mgr.25072) 433 : cluster 0 pgmap v359: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 44 MiB/s wr, 705 op/s; 9316/272325 objects degraded (3.421%); 2.1 MiB/s, 34 objects/s recovering 2023-10-01T10:33:39.618 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 81 16 94661 94645 73.0199 47 0.0270545 0.0136875 2023-10-01T10:33:39.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:39 smithi173 bash[58816]: audit 2023-10-01T10:33:38.836974+0000 mon.a (mon.0) 597 : audit 0 from='client.? 172.21.15.78:0/1907528505' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:39.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:39 smithi173 bash[58816]: audit 2023-10-01T10:33:39.278013+0000 mon.a (mon.0) 598 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:33:40.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:39 smithi078 bash[75299]: audit 2023-10-01T10:33:38.836974+0000 mon.a (mon.0) 597 : audit 0 from='client.? 172.21.15.78:0/1907528505' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:40.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:39 smithi078 bash[75299]: audit 2023-10-01T10:33:39.278013+0000 mon.a (mon.0) 598 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:33:40.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:39 smithi078 bash[78274]: audit 2023-10-01T10:33:38.836974+0000 mon.a (mon.0) 597 : audit 0 from='client.? 172.21.15.78:0/1907528505' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:40.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:39 smithi078 bash[78274]: audit 2023-10-01T10:33:39.278013+0000 mon.a (mon.0) 598 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:33:40.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:33:40 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:33:40] "GET /metrics HTTP/1.1" 200 36206 "" "Prometheus/2.33.4" 2023-10-01T10:33:40.618 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 82 16 95453 95437 72.733 49.5 0.0253308 0.0137427 2023-10-01T10:33:40.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:40 smithi173 bash[58816]: cluster 2023-10-01T10:33:39.529275+0000 mgr.y (mgr.25072) 434 : cluster 0 pgmap v360: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 819 op/s; 9316/279102 objects degraded (3.338%); 2.1 MiB/s, 34 objects/s recovering 2023-10-01T10:33:41.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:40 smithi078 bash[78274]: cluster 2023-10-01T10:33:39.529275+0000 mgr.y (mgr.25072) 434 : cluster 0 pgmap v360: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 819 op/s; 9316/279102 objects degraded (3.338%); 2.1 MiB/s, 34 objects/s recovering 2023-10-01T10:33:41.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:40 smithi078 bash[75299]: cluster 2023-10-01T10:33:39.529275+0000 mgr.y (mgr.25072) 434 : cluster 0 pgmap v360: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 819 op/s; 9316/279102 objects degraded (3.338%); 2.1 MiB/s, 34 objects/s recovering 2023-10-01T10:33:41.618 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 83 16 96210 96194 72.4267 47.3125 0.0219 0.0138016 2023-10-01T10:33:41.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:41 smithi173 bash[58816]: audit 2023-10-01T10:33:41.443019+0000 mon.b (mon.1) 28 : audit 0 from='client.? 172.21.15.78:0/4138991633' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:41.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:41 smithi173 bash[58816]: cluster 2023-10-01T10:33:41.529983+0000 mgr.y (mgr.25072) 435 : cluster 0 pgmap v361: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 57 MiB/s wr, 915 op/s; 9017/283167 objects degraded (3.184%); 4.5 MiB/s, 71 objects/s recovering 2023-10-01T10:33:42.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:41 smithi078 bash[78274]: audit 2023-10-01T10:33:41.443019+0000 mon.b (mon.1) 28 : audit 0 from='client.? 172.21.15.78:0/4138991633' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:42.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:41 smithi078 bash[78274]: cluster 2023-10-01T10:33:41.529983+0000 mgr.y (mgr.25072) 435 : cluster 0 pgmap v361: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 57 MiB/s wr, 915 op/s; 9017/283167 objects degraded (3.184%); 4.5 MiB/s, 71 objects/s recovering 2023-10-01T10:33:42.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:41 smithi078 bash[75299]: audit 2023-10-01T10:33:41.443019+0000 mon.b (mon.1) 28 : audit 0 from='client.? 172.21.15.78:0/4138991633' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:42.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:41 smithi078 bash[75299]: cluster 2023-10-01T10:33:41.529983+0000 mgr.y (mgr.25072) 435 : cluster 0 pgmap v361: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 57 MiB/s wr, 915 op/s; 9017/283167 objects degraded (3.184%); 4.5 MiB/s, 71 objects/s recovering 2023-10-01T10:33:42.618 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 84 16 97006 96990 72.1567 49.75 0.0395573 0.0138524 2023-10-01T10:33:42.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:42 smithi173 bash[58816]: cluster 2023-10-01T10:33:41.876641+0000 mon.a (mon.0) 599 : cluster 3 Health check update: Degraded data redundancy: 9017/283167 objects degraded (3.184%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T10:33:43.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:42 smithi078 bash[75299]: cluster 2023-10-01T10:33:41.876641+0000 mon.a (mon.0) 599 : cluster 3 Health check update: Degraded data redundancy: 9017/283167 objects degraded (3.184%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T10:33:43.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:42 smithi078 bash[78274]: cluster 2023-10-01T10:33:41.876641+0000 mon.a (mon.0) 599 : cluster 3 Health check update: Degraded data redundancy: 9017/283167 objects degraded (3.184%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T10:33:43.618 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 85 16 97807 97791 71.8967 50.0625 0.0446134 0.0139015 2023-10-01T10:33:44.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:43 smithi078 bash[78274]: cluster 2023-10-01T10:33:43.530586+0000 mgr.y (mgr.25072) 436 : cluster 0 pgmap v362: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 32 MiB/s wr, 515 op/s; 9017/284706 objects degraded (3.167%); 2.3 MiB/s, 37 objects/s recovering 2023-10-01T10:33:44.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:43 smithi078 bash[75299]: cluster 2023-10-01T10:33:43.530586+0000 mgr.y (mgr.25072) 436 : cluster 0 pgmap v362: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 32 MiB/s wr, 515 op/s; 9017/284706 objects degraded (3.167%); 2.3 MiB/s, 37 objects/s recovering 2023-10-01T10:33:44.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:43 smithi173 bash[58816]: cluster 2023-10-01T10:33:43.530586+0000 mgr.y (mgr.25072) 436 : cluster 0 pgmap v362: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 32 MiB/s wr, 515 op/s; 9017/284706 objects degraded (3.167%); 2.3 MiB/s, 37 objects/s recovering 2023-10-01T10:33:44.618 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 86 16 98527 98511 71.5839 45 0.0103653 0.0139615 2023-10-01T10:33:45.086 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:44 smithi078 bash[78274]: audit 2023-10-01T10:33:44.020189+0000 mon.a (mon.0) 600 : audit 0 from='client.? 172.21.15.78:0/2843711716' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:45.086 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:44 smithi078 bash[75299]: audit 2023-10-01T10:33:44.020189+0000 mon.a (mon.0) 600 : audit 0 from='client.? 172.21.15.78:0/2843711716' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:45.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:44 smithi173 bash[58816]: audit 2023-10-01T10:33:44.020189+0000 mon.a (mon.0) 600 : audit 0 from='client.? 172.21.15.78:0/2843711716' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:45.620 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 87 16 99189 99173 71.2366 41.375 0.00950838 0.0140296 2023-10-01T10:33:45.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:45 smithi173 bash[58816]: cluster 2023-10-01T10:33:45.531417+0000 mgr.y (mgr.25072) 437 : cluster 0 pgmap v363: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 47 MiB/s wr, 749 op/s; 8890/294723 objects degraded (3.016%); 2.7 MiB/s, 42 objects/s recovering 2023-10-01T10:33:46.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:45 smithi078 bash[75299]: cluster 2023-10-01T10:33:45.531417+0000 mgr.y (mgr.25072) 437 : cluster 0 pgmap v363: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 47 MiB/s wr, 749 op/s; 8890/294723 objects degraded (3.016%); 2.7 MiB/s, 42 objects/s recovering 2023-10-01T10:33:46.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:45 smithi078 bash[78274]: cluster 2023-10-01T10:33:45.531417+0000 mgr.y (mgr.25072) 437 : cluster 0 pgmap v363: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 47 MiB/s wr, 749 op/s; 8890/294723 objects degraded (3.016%); 2.7 MiB/s, 42 objects/s recovering 2023-10-01T10:33:46.618 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 88 16 99838 99822 70.888 40.5625 0.0703394 0.0141001 2023-10-01T10:33:46.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:46 smithi078 bash[75299]: audit 2023-10-01T10:33:45.666869+0000 mon.a (mon.0) 601 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:33:46.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:46 smithi078 bash[75299]: audit 2023-10-01T10:33:45.677572+0000 mon.a (mon.0) 602 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:33:46.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:46 smithi078 bash[75299]: audit 2023-10-01T10:33:45.756910+0000 mon.a (mon.0) 603 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:33:46.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:46 smithi078 bash[75299]: audit 2023-10-01T10:33:45.769429+0000 mon.a (mon.0) 604 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:33:46.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:46 smithi078 bash[75299]: audit 2023-10-01T10:33:46.376475+0000 mon.a (mon.0) 605 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:33:46.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:46 smithi078 bash[75299]: audit 2023-10-01T10:33:46.377885+0000 mon.a (mon.0) 606 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:33:46.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:46 smithi078 bash[75299]: audit 2023-10-01T10:33:46.392671+0000 mon.a (mon.0) 607 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:33:46.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:46 smithi078 bash[75299]: audit 2023-10-01T10:33:46.483355+0000 mon.a (mon.0) 608 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:33:46.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:46 smithi078 bash[75299]: audit 2023-10-01T10:33:46.489049+0000 mon.a (mon.0) 609 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:33:46.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:46 smithi078 bash[75299]: audit 2023-10-01T10:33:46.492942+0000 mon.a (mon.0) 610 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:33:46.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:46 smithi078 bash[75299]: audit 2023-10-01T10:33:46.495546+0000 mon.a (mon.0) 611 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:33:46.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:46 smithi078 bash[75299]: audit 2023-10-01T10:33:46.502879+0000 mon.a (mon.0) 612 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:33:46.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:46 smithi078 bash[75299]: audit 2023-10-01T10:33:46.504241+0000 mgr.y (mgr.25072) 438 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:33:46.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:46 smithi078 bash[75299]: cephadm 2023-10-01T10:33:46.505620+0000 mgr.y (mgr.25072) 439 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:33:46.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:46 smithi078 bash[75299]: audit 2023-10-01T10:33:46.646859+0000 mon.c (mon.2) 74 : audit 0 from='client.? 172.21.15.78:0/2504578545' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:46.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:46 smithi078 bash[78274]: audit 2023-10-01T10:33:45.666869+0000 mon.a (mon.0) 601 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:33:46.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:46 smithi078 bash[78274]: audit 2023-10-01T10:33:45.677572+0000 mon.a (mon.0) 602 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:33:46.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:46 smithi078 bash[78274]: audit 2023-10-01T10:33:45.756910+0000 mon.a (mon.0) 603 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:33:46.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:46 smithi078 bash[78274]: audit 2023-10-01T10:33:45.769429+0000 mon.a (mon.0) 604 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:33:46.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:46 smithi078 bash[78274]: audit 2023-10-01T10:33:46.376475+0000 mon.a (mon.0) 605 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:33:46.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:46 smithi078 bash[78274]: audit 2023-10-01T10:33:46.377885+0000 mon.a (mon.0) 606 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:33:46.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:46 smithi078 bash[78274]: audit 2023-10-01T10:33:46.392671+0000 mon.a (mon.0) 607 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:33:46.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:46 smithi078 bash[78274]: audit 2023-10-01T10:33:46.483355+0000 mon.a (mon.0) 608 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:33:46.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:46 smithi078 bash[78274]: audit 2023-10-01T10:33:46.489049+0000 mon.a (mon.0) 609 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:33:46.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:46 smithi078 bash[78274]: audit 2023-10-01T10:33:46.492942+0000 mon.a (mon.0) 610 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:33:46.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:46 smithi078 bash[78274]: audit 2023-10-01T10:33:46.495546+0000 mon.a (mon.0) 611 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:33:46.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:46 smithi078 bash[78274]: audit 2023-10-01T10:33:46.502879+0000 mon.a (mon.0) 612 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:33:46.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:46 smithi078 bash[78274]: audit 2023-10-01T10:33:46.504241+0000 mgr.y (mgr.25072) 438 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:33:46.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:46 smithi078 bash[78274]: cephadm 2023-10-01T10:33:46.505620+0000 mgr.y (mgr.25072) 439 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:33:46.836 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:46 smithi078 bash[78274]: audit 2023-10-01T10:33:46.646859+0000 mon.c (mon.2) 74 : audit 0 from='client.? 172.21.15.78:0/2504578545' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:46.836 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:33:46 smithi078 bash[71214]: debug 2023-10-01T10:33:46.499+0000 7fd9c245e700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:33:47.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:46 smithi173 bash[58816]: audit 2023-10-01T10:33:45.666869+0000 mon.a (mon.0) 601 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:33:47.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:46 smithi173 bash[58816]: audit 2023-10-01T10:33:45.677572+0000 mon.a (mon.0) 602 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:33:47.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:46 smithi173 bash[58816]: audit 2023-10-01T10:33:45.756910+0000 mon.a (mon.0) 603 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:33:47.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:46 smithi173 bash[58816]: audit 2023-10-01T10:33:45.769429+0000 mon.a (mon.0) 604 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:33:47.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:46 smithi173 bash[58816]: audit 2023-10-01T10:33:46.376475+0000 mon.a (mon.0) 605 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:33:47.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:46 smithi173 bash[58816]: audit 2023-10-01T10:33:46.377885+0000 mon.a (mon.0) 606 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:33:47.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:46 smithi173 bash[58816]: audit 2023-10-01T10:33:46.392671+0000 mon.a (mon.0) 607 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:33:47.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:46 smithi173 bash[58816]: audit 2023-10-01T10:33:46.483355+0000 mon.a (mon.0) 608 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:33:47.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:46 smithi173 bash[58816]: audit 2023-10-01T10:33:46.489049+0000 mon.a (mon.0) 609 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:33:47.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:46 smithi173 bash[58816]: audit 2023-10-01T10:33:46.492942+0000 mon.a (mon.0) 610 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:33:47.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:46 smithi173 bash[58816]: audit 2023-10-01T10:33:46.495546+0000 mon.a (mon.0) 611 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:33:47.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:46 smithi173 bash[58816]: audit 2023-10-01T10:33:46.502879+0000 mon.a (mon.0) 612 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:33:47.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:46 smithi173 bash[58816]: audit 2023-10-01T10:33:46.504241+0000 mgr.y (mgr.25072) 438 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:33:47.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:46 smithi173 bash[58816]: cephadm 2023-10-01T10:33:46.505620+0000 mgr.y (mgr.25072) 439 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:33:47.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:46 smithi173 bash[58816]: audit 2023-10-01T10:33:46.646859+0000 mon.c (mon.2) 74 : audit 0 from='client.? 172.21.15.78:0/2504578545' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:47.618 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 89 16 100663 100647 70.6708 51.5625 0.0105072 0.0141436 2023-10-01T10:33:48.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:47 smithi078 bash[75299]: cluster 2023-10-01T10:33:46.878079+0000 mon.a (mon.0) 613 : cluster 3 Health check update: Degraded data redundancy: 8890/294723 objects degraded (3.016%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T10:33:48.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:47 smithi078 bash[75299]: audit 2023-10-01T10:33:47.398843+0000 mon.a (mon.0) 614 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:33:48.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:47 smithi078 bash[75299]: cluster 2023-10-01T10:33:47.531915+0000 mgr.y (mgr.25072) 440 : cluster 0 pgmap v364: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 43 MiB/s wr, 682 op/s; 8890/294723 objects degraded (3.016%); 2.4 MiB/s, 38 objects/s recovering 2023-10-01T10:33:48.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:47 smithi078 bash[78274]: cluster 2023-10-01T10:33:46.878079+0000 mon.a (mon.0) 613 : cluster 3 Health check update: Degraded data redundancy: 8890/294723 objects degraded (3.016%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T10:33:48.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:47 smithi078 bash[78274]: audit 2023-10-01T10:33:47.398843+0000 mon.a (mon.0) 614 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:33:48.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:47 smithi078 bash[78274]: cluster 2023-10-01T10:33:47.531915+0000 mgr.y (mgr.25072) 440 : cluster 0 pgmap v364: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 43 MiB/s wr, 682 op/s; 8890/294723 objects degraded (3.016%); 2.4 MiB/s, 38 objects/s recovering 2023-10-01T10:33:48.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:47 smithi173 bash[58816]: cluster 2023-10-01T10:33:46.878079+0000 mon.a (mon.0) 613 : cluster 3 Health check update: Degraded data redundancy: 8890/294723 objects degraded (3.016%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T10:33:48.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:47 smithi173 bash[58816]: audit 2023-10-01T10:33:47.398843+0000 mon.a (mon.0) 614 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:33:48.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:47 smithi173 bash[58816]: cluster 2023-10-01T10:33:47.531915+0000 mgr.y (mgr.25072) 440 : cluster 0 pgmap v364: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 43 MiB/s wr, 682 op/s; 8890/294723 objects degraded (3.016%); 2.4 MiB/s, 38 objects/s recovering 2023-10-01T10:33:48.618 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 90 9 101447 101438 70.4349 49.4375 0.0123591 0.0141911 2023-10-01T10:33:49.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:49 smithi173 bash[58816]: audit 2023-10-01T10:33:49.030746+0000 mon.c (mon.2) 75 : audit 0 from='client.? 172.21.15.78:0/2927446387' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:49.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:49 smithi078 bash[75299]: audit 2023-10-01T10:33:49.030746+0000 mon.c (mon.2) 75 : audit 0 from='client.? 172.21.15.78:0/2927446387' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:49.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:49 smithi078 bash[78274]: audit 2023-10-01T10:33:49.030746+0000 mon.c (mon.2) 75 : audit 0 from='client.? 172.21.15.78:0/2927446387' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:49.619 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Total time run: 90.0185 2023-10-01T10:33:49.619 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Total writes made: 101447 2023-10-01T10:33:49.619 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Write size: 65536 2023-10-01T10:33:49.619 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Object size: 65536 2023-10-01T10:33:49.619 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Bandwidth (MB/sec): 70.4348 2023-10-01T10:33:49.619 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Stddev Bandwidth: 37.4706 2023-10-01T10:33:49.620 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Max bandwidth (MB/sec): 215.875 2023-10-01T10:33:49.620 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Min bandwidth (MB/sec): 0 2023-10-01T10:33:49.620 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Average IOPS: 1126 2023-10-01T10:33:49.620 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Stddev IOPS: 599.529 2023-10-01T10:33:49.620 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Max IOPS: 3454 2023-10-01T10:33:49.620 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Min IOPS: 0 2023-10-01T10:33:49.620 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Average Latency(s): 0.0141918 2023-10-01T10:33:49.620 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Stddev Latency(s): 0.0321045 2023-10-01T10:33:49.620 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Max latency(s): 2.17561 2023-10-01T10:33:49.620 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Min latency(s): 0.00124552 2023-10-01T10:33:49.630 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Cleaning up (deleting benchmark objects) 2023-10-01T10:33:50.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:50 smithi173 bash[58816]: cluster 2023-10-01T10:33:49.532875+0000 mgr.y (mgr.25072) 441 : cluster 0 pgmap v365: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 802 op/s; 8890/301212 objects degraded (2.951%); 2.2 MiB/s, 35 objects/s recovering 2023-10-01T10:33:50.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:50 smithi078 bash[75299]: cluster 2023-10-01T10:33:49.532875+0000 mgr.y (mgr.25072) 441 : cluster 0 pgmap v365: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 802 op/s; 8890/301212 objects degraded (2.951%); 2.2 MiB/s, 35 objects/s recovering 2023-10-01T10:33:50.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:50 smithi078 bash[78274]: cluster 2023-10-01T10:33:49.532875+0000 mgr.y (mgr.25072) 441 : cluster 0 pgmap v365: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 802 op/s; 8890/301212 objects degraded (2.951%); 2.2 MiB/s, 35 objects/s recovering 2023-10-01T10:33:50.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:33:50 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:33:50] "GET /metrics HTTP/1.1" 200 36207 "" "Prometheus/2.33.4" 2023-10-01T10:33:52.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:52 smithi173 bash[58816]: cluster 2023-10-01T10:33:51.533715+0000 mgr.y (mgr.25072) 442 : cluster 0 pgmap v366: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 44 MiB/s wr, 710 op/s; 8689/304056 objects degraded (2.858%); 3.3 MiB/s, 52 objects/s recovering 2023-10-01T10:33:52.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:52 smithi173 bash[58816]: audit 2023-10-01T10:33:51.621649+0000 mon.c (mon.2) 76 : audit 0 from='client.? 172.21.15.78:0/414688306' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:52.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:52 smithi173 bash[58816]: cluster 2023-10-01T10:33:51.880695+0000 mon.a (mon.0) 615 : cluster 3 Health check update: Degraded data redundancy: 8689/304056 objects degraded (2.858%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T10:33:53.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:52 smithi078 bash[75299]: cluster 2023-10-01T10:33:51.533715+0000 mgr.y (mgr.25072) 442 : cluster 0 pgmap v366: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 44 MiB/s wr, 710 op/s; 8689/304056 objects degraded (2.858%); 3.3 MiB/s, 52 objects/s recovering 2023-10-01T10:33:53.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:52 smithi078 bash[75299]: audit 2023-10-01T10:33:51.621649+0000 mon.c (mon.2) 76 : audit 0 from='client.? 172.21.15.78:0/414688306' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:53.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:52 smithi078 bash[75299]: cluster 2023-10-01T10:33:51.880695+0000 mon.a (mon.0) 615 : cluster 3 Health check update: Degraded data redundancy: 8689/304056 objects degraded (2.858%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T10:33:53.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:52 smithi078 bash[78274]: cluster 2023-10-01T10:33:51.533715+0000 mgr.y (mgr.25072) 442 : cluster 0 pgmap v366: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 44 MiB/s wr, 710 op/s; 8689/304056 objects degraded (2.858%); 3.3 MiB/s, 52 objects/s recovering 2023-10-01T10:33:53.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:52 smithi078 bash[78274]: audit 2023-10-01T10:33:51.621649+0000 mon.c (mon.2) 76 : audit 0 from='client.? 172.21.15.78:0/414688306' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:53.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:52 smithi078 bash[78274]: cluster 2023-10-01T10:33:51.880695+0000 mon.a (mon.0) 615 : cluster 3 Health check update: Degraded data redundancy: 8689/304056 objects degraded (2.858%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T10:33:54.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:54 smithi173 bash[58816]: cluster 2023-10-01T10:33:53.534329+0000 mgr.y (mgr.25072) 443 : cluster 0 pgmap v367: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 37 MiB/s wr, 597 op/s; 8689/304056 objects degraded (2.858%); 1.7 MiB/s, 27 objects/s recovering 2023-10-01T10:33:54.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:54 smithi173 bash[58816]: audit 2023-10-01T10:33:54.208592+0000 mon.a (mon.0) 616 : audit 0 from='client.? 172.21.15.78:0/4057202199' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:55.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:54 smithi078 bash[75299]: cluster 2023-10-01T10:33:53.534329+0000 mgr.y (mgr.25072) 443 : cluster 0 pgmap v367: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 37 MiB/s wr, 597 op/s; 8689/304056 objects degraded (2.858%); 1.7 MiB/s, 27 objects/s recovering 2023-10-01T10:33:55.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:54 smithi078 bash[75299]: audit 2023-10-01T10:33:54.208592+0000 mon.a (mon.0) 616 : audit 0 from='client.? 172.21.15.78:0/4057202199' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:55.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:54 smithi078 bash[78274]: cluster 2023-10-01T10:33:53.534329+0000 mgr.y (mgr.25072) 443 : cluster 0 pgmap v367: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 37 MiB/s wr, 597 op/s; 8689/304056 objects degraded (2.858%); 1.7 MiB/s, 27 objects/s recovering 2023-10-01T10:33:55.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:54 smithi078 bash[78274]: audit 2023-10-01T10:33:54.208592+0000 mon.a (mon.0) 616 : audit 0 from='client.? 172.21.15.78:0/4057202199' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:56.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:56 smithi173 bash[58816]: cluster 2023-10-01T10:33:55.535254+0000 mgr.y (mgr.25072) 444 : cluster 0 pgmap v368: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 34 MiB/s wr, 861 op/s; 8560/293121 objects degraded (2.920%); 2.4 MiB/s, 38 objects/s recovering 2023-10-01T10:33:57.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:56 smithi078 bash[75299]: cluster 2023-10-01T10:33:55.535254+0000 mgr.y (mgr.25072) 444 : cluster 0 pgmap v368: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 34 MiB/s wr, 861 op/s; 8560/293121 objects degraded (2.920%); 2.4 MiB/s, 38 objects/s recovering 2023-10-01T10:33:57.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:56 smithi078 bash[78274]: cluster 2023-10-01T10:33:55.535254+0000 mgr.y (mgr.25072) 444 : cluster 0 pgmap v368: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 34 MiB/s wr, 861 op/s; 8560/293121 objects degraded (2.920%); 2.4 MiB/s, 38 objects/s recovering 2023-10-01T10:33:57.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:57 smithi173 bash[58816]: audit 2023-10-01T10:33:56.812081+0000 mon.b (mon.1) 29 : audit 0 from='client.? 172.21.15.78:0/648938679' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:57.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:57 smithi173 bash[58816]: cluster 2023-10-01T10:33:56.882196+0000 mon.a (mon.0) 617 : cluster 3 Health check update: Degraded data redundancy: 8560/293121 objects degraded (2.920%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T10:33:58.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:57 smithi078 bash[75299]: audit 2023-10-01T10:33:56.812081+0000 mon.b (mon.1) 29 : audit 0 from='client.? 172.21.15.78:0/648938679' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:58.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:57 smithi078 bash[75299]: cluster 2023-10-01T10:33:56.882196+0000 mon.a (mon.0) 617 : cluster 3 Health check update: Degraded data redundancy: 8560/293121 objects degraded (2.920%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T10:33:58.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:57 smithi078 bash[78274]: audit 2023-10-01T10:33:56.812081+0000 mon.b (mon.1) 29 : audit 0 from='client.? 172.21.15.78:0/648938679' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:58.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:57 smithi078 bash[78274]: cluster 2023-10-01T10:33:56.882196+0000 mon.a (mon.0) 617 : cluster 3 Health check update: Degraded data redundancy: 8560/293121 objects degraded (2.920%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T10:33:58.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:58 smithi173 bash[58816]: cluster 2023-10-01T10:33:57.535846+0000 mgr.y (mgr.25072) 445 : cluster 0 pgmap v369: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 17 MiB/s wr, 582 op/s; 8560/293121 objects degraded (2.920%); 1.7 MiB/s, 27 objects/s recovering 2023-10-01T10:33:59.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:58 smithi078 bash[75299]: cluster 2023-10-01T10:33:57.535846+0000 mgr.y (mgr.25072) 445 : cluster 0 pgmap v369: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 17 MiB/s wr, 582 op/s; 8560/293121 objects degraded (2.920%); 1.7 MiB/s, 27 objects/s recovering 2023-10-01T10:33:59.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:58 smithi078 bash[78274]: cluster 2023-10-01T10:33:57.535846+0000 mgr.y (mgr.25072) 445 : cluster 0 pgmap v369: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 17 MiB/s wr, 582 op/s; 8560/293121 objects degraded (2.920%); 1.7 MiB/s, 27 objects/s recovering 2023-10-01T10:33:59.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:59 smithi173 bash[58816]: audit 2023-10-01T10:33:59.433495+0000 mon.c (mon.2) 77 : audit 0 from='client.? 172.21.15.78:0/777720518' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:33:59.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:33:59 smithi173 bash[58816]: cluster 2023-10-01T10:33:59.536712+0000 mgr.y (mgr.25072) 446 : cluster 0 pgmap v370: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 17 MiB/s wr, 788 op/s; 8560/285720 objects degraded (2.996%); 1.7 MiB/s, 27 objects/s recovering 2023-10-01T10:34:00.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:59 smithi078 bash[75299]: audit 2023-10-01T10:33:59.433495+0000 mon.c (mon.2) 77 : audit 0 from='client.? 172.21.15.78:0/777720518' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:00.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:33:59 smithi078 bash[75299]: cluster 2023-10-01T10:33:59.536712+0000 mgr.y (mgr.25072) 446 : cluster 0 pgmap v370: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 17 MiB/s wr, 788 op/s; 8560/285720 objects degraded (2.996%); 1.7 MiB/s, 27 objects/s recovering 2023-10-01T10:34:00.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:59 smithi078 bash[78274]: audit 2023-10-01T10:33:59.433495+0000 mon.c (mon.2) 77 : audit 0 from='client.? 172.21.15.78:0/777720518' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:00.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:33:59 smithi078 bash[78274]: cluster 2023-10-01T10:33:59.536712+0000 mgr.y (mgr.25072) 446 : cluster 0 pgmap v370: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 17 MiB/s wr, 788 op/s; 8560/285720 objects degraded (2.996%); 1.7 MiB/s, 27 objects/s recovering 2023-10-01T10:34:00.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:34:00 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:34:00] "GET /metrics HTTP/1.1" 200 36207 "" "Prometheus/2.33.4" 2023-10-01T10:34:01.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:01 smithi078 bash[75299]: audit 2023-10-01T10:34:01.519046+0000 mon.a (mon.0) 618 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:34:01.833 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:34:01 smithi078 bash[71214]: debug 2023-10-01T10:34:01.515+0000 7fd9c245e700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:34:01.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:01 smithi078 bash[78274]: audit 2023-10-01T10:34:01.519046+0000 mon.a (mon.0) 618 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:34:01.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:01 smithi173 bash[58816]: audit 2023-10-01T10:34:01.519046+0000 mon.a (mon.0) 618 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:34:02.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:02 smithi173 bash[58816]: audit 2023-10-01T10:34:01.519285+0000 mgr.y (mgr.25072) 447 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:34:02.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:02 smithi173 bash[58816]: cephadm 2023-10-01T10:34:01.520990+0000 mgr.y (mgr.25072) 448 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:34:02.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:02 smithi173 bash[58816]: cluster 2023-10-01T10:34:01.537498+0000 mgr.y (mgr.25072) 449 : cluster 0 pgmap v371: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 5.6 MiB/s wr, 728 op/s; 8429/281394 objects degraded (2.995%); 2.4 MiB/s, 38 objects/s recovering 2023-10-01T10:34:02.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:02 smithi173 bash[58816]: cluster 2023-10-01T10:34:01.883666+0000 mon.a (mon.0) 619 : cluster 3 Health check update: Degraded data redundancy: 8429/281394 objects degraded (2.995%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T10:34:02.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:02 smithi173 bash[58816]: audit 2023-10-01T10:34:02.020175+0000 mon.c (mon.2) 78 : audit 0 from='client.? 172.21.15.78:0/281962013' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:02.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:02 smithi173 bash[58816]: audit 2023-10-01T10:34:02.399478+0000 mon.a (mon.0) 620 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:34:03.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:02 smithi078 bash[75299]: audit 2023-10-01T10:34:01.519285+0000 mgr.y (mgr.25072) 447 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:34:03.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:02 smithi078 bash[75299]: cephadm 2023-10-01T10:34:01.520990+0000 mgr.y (mgr.25072) 448 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:34:03.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:02 smithi078 bash[75299]: cluster 2023-10-01T10:34:01.537498+0000 mgr.y (mgr.25072) 449 : cluster 0 pgmap v371: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 5.6 MiB/s wr, 728 op/s; 8429/281394 objects degraded (2.995%); 2.4 MiB/s, 38 objects/s recovering 2023-10-01T10:34:03.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:02 smithi078 bash[75299]: cluster 2023-10-01T10:34:01.883666+0000 mon.a (mon.0) 619 : cluster 3 Health check update: Degraded data redundancy: 8429/281394 objects degraded (2.995%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T10:34:03.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:02 smithi078 bash[75299]: audit 2023-10-01T10:34:02.020175+0000 mon.c (mon.2) 78 : audit 0 from='client.? 172.21.15.78:0/281962013' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:03.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:02 smithi078 bash[75299]: audit 2023-10-01T10:34:02.399478+0000 mon.a (mon.0) 620 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:34:03.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:02 smithi078 bash[78274]: audit 2023-10-01T10:34:01.519285+0000 mgr.y (mgr.25072) 447 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:34:03.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:02 smithi078 bash[78274]: cephadm 2023-10-01T10:34:01.520990+0000 mgr.y (mgr.25072) 448 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:34:03.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:02 smithi078 bash[78274]: cluster 2023-10-01T10:34:01.537498+0000 mgr.y (mgr.25072) 449 : cluster 0 pgmap v371: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 5.6 MiB/s wr, 728 op/s; 8429/281394 objects degraded (2.995%); 2.4 MiB/s, 38 objects/s recovering 2023-10-01T10:34:03.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:02 smithi078 bash[78274]: cluster 2023-10-01T10:34:01.883666+0000 mon.a (mon.0) 619 : cluster 3 Health check update: Degraded data redundancy: 8429/281394 objects degraded (2.995%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T10:34:03.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:02 smithi078 bash[78274]: audit 2023-10-01T10:34:02.020175+0000 mon.c (mon.2) 78 : audit 0 from='client.? 172.21.15.78:0/281962013' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:03.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:02 smithi078 bash[78274]: audit 2023-10-01T10:34:02.399478+0000 mon.a (mon.0) 620 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:34:04.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:04 smithi173 bash[58816]: cluster 2023-10-01T10:34:03.538076+0000 mgr.y (mgr.25072) 450 : cluster 0 pgmap v372: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 69 KiB/s wr, 631 op/s; 8429/281394 objects degraded (2.995%); 1.4 MiB/s, 21 objects/s recovering 2023-10-01T10:34:05.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:04 smithi078 bash[78274]: cluster 2023-10-01T10:34:03.538076+0000 mgr.y (mgr.25072) 450 : cluster 0 pgmap v372: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 69 KiB/s wr, 631 op/s; 8429/281394 objects degraded (2.995%); 1.4 MiB/s, 21 objects/s recovering 2023-10-01T10:34:05.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:04 smithi078 bash[75299]: cluster 2023-10-01T10:34:03.538076+0000 mgr.y (mgr.25072) 450 : cluster 0 pgmap v372: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 85 B/s rd, 69 KiB/s wr, 631 op/s; 8429/281394 objects degraded (2.995%); 1.4 MiB/s, 21 objects/s recovering 2023-10-01T10:34:06.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:05 smithi078 bash[75299]: audit 2023-10-01T10:34:04.611687+0000 mon.a (mon.0) 621 : audit 0 from='client.? 172.21.15.78:0/868267017' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:06.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:05 smithi078 bash[75299]: cluster 2023-10-01T10:34:05.538941+0000 mgr.y (mgr.25072) 451 : cluster 0 pgmap v373: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 85 B/s rd, 69 KiB/s wr, 951 op/s; 8295/269868 objects degraded (3.074%); 2.1 MiB/s, 32 objects/s recovering 2023-10-01T10:34:06.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:05 smithi078 bash[78274]: audit 2023-10-01T10:34:04.611687+0000 mon.a (mon.0) 621 : audit 0 from='client.? 172.21.15.78:0/868267017' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:06.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:05 smithi078 bash[78274]: cluster 2023-10-01T10:34:05.538941+0000 mgr.y (mgr.25072) 451 : cluster 0 pgmap v373: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 85 B/s rd, 69 KiB/s wr, 951 op/s; 8295/269868 objects degraded (3.074%); 2.1 MiB/s, 32 objects/s recovering 2023-10-01T10:34:06.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:05 smithi173 bash[58816]: audit 2023-10-01T10:34:04.611687+0000 mon.a (mon.0) 621 : audit 0 from='client.? 172.21.15.78:0/868267017' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:06.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:05 smithi173 bash[58816]: cluster 2023-10-01T10:34:05.538941+0000 mgr.y (mgr.25072) 451 : cluster 0 pgmap v373: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 85 B/s rd, 69 KiB/s wr, 951 op/s; 8295/269868 objects degraded (3.074%); 2.1 MiB/s, 32 objects/s recovering 2023-10-01T10:34:07.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:06 smithi078 bash[75299]: cluster 2023-10-01T10:34:06.888211+0000 mon.a (mon.0) 622 : cluster 3 Health check update: Degraded data redundancy: 8295/269868 objects degraded (3.074%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T10:34:07.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:06 smithi078 bash[78274]: cluster 2023-10-01T10:34:06.888211+0000 mon.a (mon.0) 622 : cluster 3 Health check update: Degraded data redundancy: 8295/269868 objects degraded (3.074%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T10:34:07.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:06 smithi173 bash[58816]: cluster 2023-10-01T10:34:06.888211+0000 mon.a (mon.0) 622 : cluster 3 Health check update: Degraded data redundancy: 8295/269868 objects degraded (3.074%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T10:34:08.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:07 smithi078 bash[75299]: audit 2023-10-01T10:34:07.204126+0000 mon.a (mon.0) 623 : audit 0 from='client.? 172.21.15.78:0/1315850004' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:08.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:07 smithi078 bash[75299]: cluster 2023-10-01T10:34:07.539396+0000 mgr.y (mgr.25072) 452 : cluster 0 pgmap v374: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 8295/269868 objects degraded (3.074%); 1.4 MiB/s, 22 objects/s recovering 2023-10-01T10:34:08.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:07 smithi078 bash[78274]: audit 2023-10-01T10:34:07.204126+0000 mon.a (mon.0) 623 : audit 0 from='client.? 172.21.15.78:0/1315850004' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:08.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:07 smithi078 bash[78274]: cluster 2023-10-01T10:34:07.539396+0000 mgr.y (mgr.25072) 452 : cluster 0 pgmap v374: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 8295/269868 objects degraded (3.074%); 1.4 MiB/s, 22 objects/s recovering 2023-10-01T10:34:08.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:07 smithi173 bash[58816]: audit 2023-10-01T10:34:07.204126+0000 mon.a (mon.0) 623 : audit 0 from='client.? 172.21.15.78:0/1315850004' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:08.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:07 smithi173 bash[58816]: cluster 2023-10-01T10:34:07.539396+0000 mgr.y (mgr.25072) 452 : cluster 0 pgmap v374: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 8295/269868 objects degraded (3.074%); 1.4 MiB/s, 22 objects/s recovering 2023-10-01T10:34:10.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:34:10 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:34:10] "GET /metrics HTTP/1.1" 200 36214 "" "Prometheus/2.33.4" 2023-10-01T10:34:10.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:10 smithi173 bash[58816]: cluster 2023-10-01T10:34:09.540152+0000 mgr.y (mgr.25072) 453 : cluster 0 pgmap v375: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 8295/262632 objects degraded (3.158%); 1.4 MiB/s, 22 objects/s recovering 2023-10-01T10:34:10.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:10 smithi173 bash[58816]: audit 2023-10-01T10:34:09.783808+0000 mon.c (mon.2) 79 : audit 0 from='client.? 172.21.15.78:0/654759085' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:11.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:10 smithi078 bash[75299]: cluster 2023-10-01T10:34:09.540152+0000 mgr.y (mgr.25072) 453 : cluster 0 pgmap v375: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 8295/262632 objects degraded (3.158%); 1.4 MiB/s, 22 objects/s recovering 2023-10-01T10:34:11.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:10 smithi078 bash[75299]: audit 2023-10-01T10:34:09.783808+0000 mon.c (mon.2) 79 : audit 0 from='client.? 172.21.15.78:0/654759085' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:11.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:10 smithi078 bash[78274]: cluster 2023-10-01T10:34:09.540152+0000 mgr.y (mgr.25072) 453 : cluster 0 pgmap v375: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 8295/262632 objects degraded (3.158%); 1.4 MiB/s, 22 objects/s recovering 2023-10-01T10:34:11.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:10 smithi078 bash[78274]: audit 2023-10-01T10:34:09.783808+0000 mon.c (mon.2) 79 : audit 0 from='client.? 172.21.15.78:0/654759085' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:12.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:12 smithi173 bash[58816]: cluster 2023-10-01T10:34:11.540825+0000 mgr.y (mgr.25072) 454 : cluster 0 pgmap v376: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 8147/258375 objects degraded (3.153%); 2.2 MiB/s, 34 objects/s recovering 2023-10-01T10:34:12.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:12 smithi173 bash[58816]: cluster 2023-10-01T10:34:11.889459+0000 mon.a (mon.0) 624 : cluster 3 Health check update: Degraded data redundancy: 8147/258375 objects degraded (3.153%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T10:34:12.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:12 smithi173 bash[58816]: audit 2023-10-01T10:34:12.364273+0000 mon.a (mon.0) 625 : audit 0 from='client.? 172.21.15.78:0/2205091186' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:13.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:12 smithi078 bash[75299]: cluster 2023-10-01T10:34:11.540825+0000 mgr.y (mgr.25072) 454 : cluster 0 pgmap v376: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 8147/258375 objects degraded (3.153%); 2.2 MiB/s, 34 objects/s recovering 2023-10-01T10:34:13.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:12 smithi078 bash[75299]: cluster 2023-10-01T10:34:11.889459+0000 mon.a (mon.0) 624 : cluster 3 Health check update: Degraded data redundancy: 8147/258375 objects degraded (3.153%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T10:34:13.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:12 smithi078 bash[75299]: audit 2023-10-01T10:34:12.364273+0000 mon.a (mon.0) 625 : audit 0 from='client.? 172.21.15.78:0/2205091186' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:13.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:12 smithi078 bash[78274]: cluster 2023-10-01T10:34:11.540825+0000 mgr.y (mgr.25072) 454 : cluster 0 pgmap v376: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 8147/258375 objects degraded (3.153%); 2.2 MiB/s, 34 objects/s recovering 2023-10-01T10:34:13.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:12 smithi078 bash[78274]: cluster 2023-10-01T10:34:11.889459+0000 mon.a (mon.0) 624 : cluster 3 Health check update: Degraded data redundancy: 8147/258375 objects degraded (3.153%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T10:34:13.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:12 smithi078 bash[78274]: audit 2023-10-01T10:34:12.364273+0000 mon.a (mon.0) 625 : audit 0 from='client.? 172.21.15.78:0/2205091186' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:14.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:14 smithi173 bash[58816]: cluster 2023-10-01T10:34:13.541389+0000 mgr.y (mgr.25072) 455 : cluster 0 pgmap v377: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 639 op/s; 8147/258375 objects degraded (3.153%); 1.5 MiB/s, 23 objects/s recovering 2023-10-01T10:34:15.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:14 smithi078 bash[75299]: cluster 2023-10-01T10:34:13.541389+0000 mgr.y (mgr.25072) 455 : cluster 0 pgmap v377: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 639 op/s; 8147/258375 objects degraded (3.153%); 1.5 MiB/s, 23 objects/s recovering 2023-10-01T10:34:15.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:14 smithi078 bash[78274]: cluster 2023-10-01T10:34:13.541389+0000 mgr.y (mgr.25072) 455 : cluster 0 pgmap v377: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 639 op/s; 8147/258375 objects degraded (3.153%); 1.5 MiB/s, 23 objects/s recovering 2023-10-01T10:34:15.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:15 smithi173 bash[58816]: audit 2023-10-01T10:34:14.963916+0000 mon.a (mon.0) 626 : audit 0 from='client.? 172.21.15.78:0/718221117' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:16.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:15 smithi078 bash[75299]: audit 2023-10-01T10:34:14.963916+0000 mon.a (mon.0) 626 : audit 0 from='client.? 172.21.15.78:0/718221117' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:16.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:15 smithi078 bash[78274]: audit 2023-10-01T10:34:14.963916+0000 mon.a (mon.0) 626 : audit 0 from='client.? 172.21.15.78:0/718221117' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:16.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:16 smithi078 bash[75299]: cluster 2023-10-01T10:34:15.542362+0000 mgr.y (mgr.25072) 456 : cluster 0 pgmap v378: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 959 op/s; 7996/246828 objects degraded (3.240%); 2.3 MiB/s, 36 objects/s recovering 2023-10-01T10:34:16.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:16 smithi078 bash[75299]: audit 2023-10-01T10:34:16.537212+0000 mon.a (mon.0) 627 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:34:16.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:16 smithi078 bash[75299]: audit 2023-10-01T10:34:16.537475+0000 mgr.y (mgr.25072) 457 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:34:16.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:16 smithi078 bash[75299]: cephadm 2023-10-01T10:34:16.539263+0000 mgr.y (mgr.25072) 458 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:34:16.834 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:34:16 smithi078 bash[71214]: debug 2023-10-01T10:34:16.534+0000 7fd9c245e700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:34:16.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:16 smithi078 bash[78274]: cluster 2023-10-01T10:34:15.542362+0000 mgr.y (mgr.25072) 456 : cluster 0 pgmap v378: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 959 op/s; 7996/246828 objects degraded (3.240%); 2.3 MiB/s, 36 objects/s recovering 2023-10-01T10:34:16.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:16 smithi078 bash[78274]: audit 2023-10-01T10:34:16.537212+0000 mon.a (mon.0) 627 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:34:16.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:16 smithi078 bash[78274]: audit 2023-10-01T10:34:16.537475+0000 mgr.y (mgr.25072) 457 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:34:16.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:16 smithi078 bash[78274]: cephadm 2023-10-01T10:34:16.539263+0000 mgr.y (mgr.25072) 458 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:34:16.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:16 smithi173 bash[58816]: cluster 2023-10-01T10:34:15.542362+0000 mgr.y (mgr.25072) 456 : cluster 0 pgmap v378: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 959 op/s; 7996/246828 objects degraded (3.240%); 2.3 MiB/s, 36 objects/s recovering 2023-10-01T10:34:16.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:16 smithi173 bash[58816]: audit 2023-10-01T10:34:16.537212+0000 mon.a (mon.0) 627 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:34:16.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:16 smithi173 bash[58816]: audit 2023-10-01T10:34:16.537475+0000 mgr.y (mgr.25072) 457 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:34:16.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:16 smithi173 bash[58816]: cephadm 2023-10-01T10:34:16.539263+0000 mgr.y (mgr.25072) 458 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:34:17.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:17 smithi173 bash[58816]: cluster 2023-10-01T10:34:16.890868+0000 mon.a (mon.0) 628 : cluster 3 Health check update: Degraded data redundancy: 7996/246828 objects degraded (3.240%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T10:34:17.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:17 smithi173 bash[58816]: audit 2023-10-01T10:34:17.399805+0000 mon.a (mon.0) 629 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:34:17.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:17 smithi173 bash[58816]: cluster 2023-10-01T10:34:17.542905+0000 mgr.y (mgr.25072) 459 : cluster 0 pgmap v379: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 639 op/s; 7996/246828 objects degraded (3.240%); 1.6 MiB/s, 24 objects/s recovering 2023-10-01T10:34:17.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:17 smithi173 bash[58816]: audit 2023-10-01T10:34:17.578597+0000 mon.c (mon.2) 80 : audit 0 from='client.? 172.21.15.78:0/3296419827' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:18.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:17 smithi078 bash[75299]: cluster 2023-10-01T10:34:16.890868+0000 mon.a (mon.0) 628 : cluster 3 Health check update: Degraded data redundancy: 7996/246828 objects degraded (3.240%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T10:34:18.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:17 smithi078 bash[75299]: audit 2023-10-01T10:34:17.399805+0000 mon.a (mon.0) 629 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:34:18.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:17 smithi078 bash[75299]: cluster 2023-10-01T10:34:17.542905+0000 mgr.y (mgr.25072) 459 : cluster 0 pgmap v379: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 639 op/s; 7996/246828 objects degraded (3.240%); 1.6 MiB/s, 24 objects/s recovering 2023-10-01T10:34:18.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:17 smithi078 bash[75299]: audit 2023-10-01T10:34:17.578597+0000 mon.c (mon.2) 80 : audit 0 from='client.? 172.21.15.78:0/3296419827' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:18.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:17 smithi078 bash[78274]: cluster 2023-10-01T10:34:16.890868+0000 mon.a (mon.0) 628 : cluster 3 Health check update: Degraded data redundancy: 7996/246828 objects degraded (3.240%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T10:34:18.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:17 smithi078 bash[78274]: audit 2023-10-01T10:34:17.399805+0000 mon.a (mon.0) 629 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:34:18.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:17 smithi078 bash[78274]: cluster 2023-10-01T10:34:17.542905+0000 mgr.y (mgr.25072) 459 : cluster 0 pgmap v379: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 639 op/s; 7996/246828 objects degraded (3.240%); 1.6 MiB/s, 24 objects/s recovering 2023-10-01T10:34:18.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:17 smithi078 bash[78274]: audit 2023-10-01T10:34:17.578597+0000 mon.c (mon.2) 80 : audit 0 from='client.? 172.21.15.78:0/3296419827' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:20.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:34:20 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:34:20] "GET /metrics HTTP/1.1" 200 36213 "" "Prometheus/2.33.4" 2023-10-01T10:34:20.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:20 smithi173 bash[58816]: cluster 2023-10-01T10:34:19.543697+0000 mgr.y (mgr.25072) 460 : cluster 0 pgmap v380: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.9 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 830 op/s; 7996/239964 objects degraded (3.332%); 1.6 MiB/s, 24 objects/s recovering 2023-10-01T10:34:20.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:20 smithi173 bash[58816]: audit 2023-10-01T10:34:20.169424+0000 mon.b (mon.1) 30 : audit 0 from='client.? 172.21.15.78:0/3525207539' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:21.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:20 smithi078 bash[75299]: cluster 2023-10-01T10:34:19.543697+0000 mgr.y (mgr.25072) 460 : cluster 0 pgmap v380: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.9 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 830 op/s; 7996/239964 objects degraded (3.332%); 1.6 MiB/s, 24 objects/s recovering 2023-10-01T10:34:21.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:20 smithi078 bash[75299]: audit 2023-10-01T10:34:20.169424+0000 mon.b (mon.1) 30 : audit 0 from='client.? 172.21.15.78:0/3525207539' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:21.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:20 smithi078 bash[78274]: cluster 2023-10-01T10:34:19.543697+0000 mgr.y (mgr.25072) 460 : cluster 0 pgmap v380: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.9 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 830 op/s; 7996/239964 objects degraded (3.332%); 1.6 MiB/s, 24 objects/s recovering 2023-10-01T10:34:21.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:20 smithi078 bash[78274]: audit 2023-10-01T10:34:20.169424+0000 mon.b (mon.1) 30 : audit 0 from='client.? 172.21.15.78:0/3525207539' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:22.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:22 smithi173 bash[58816]: cluster 2023-10-01T10:34:21.544261+0000 mgr.y (mgr.25072) 461 : cluster 0 pgmap v381: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 745 op/s; 7842/235785 objects degraded (3.326%); 2.4 MiB/s, 37 objects/s recovering 2023-10-01T10:34:22.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:22 smithi173 bash[58816]: cluster 2023-10-01T10:34:21.892134+0000 mon.a (mon.0) 630 : cluster 3 Health check update: Degraded data redundancy: 7842/235785 objects degraded (3.326%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T10:34:23.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:22 smithi078 bash[75299]: cluster 2023-10-01T10:34:21.544261+0000 mgr.y (mgr.25072) 461 : cluster 0 pgmap v381: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 745 op/s; 7842/235785 objects degraded (3.326%); 2.4 MiB/s, 37 objects/s recovering 2023-10-01T10:34:23.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:22 smithi078 bash[75299]: cluster 2023-10-01T10:34:21.892134+0000 mon.a (mon.0) 630 : cluster 3 Health check update: Degraded data redundancy: 7842/235785 objects degraded (3.326%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T10:34:23.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:22 smithi078 bash[78274]: cluster 2023-10-01T10:34:21.544261+0000 mgr.y (mgr.25072) 461 : cluster 0 pgmap v381: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 745 op/s; 7842/235785 objects degraded (3.326%); 2.4 MiB/s, 37 objects/s recovering 2023-10-01T10:34:23.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:22 smithi078 bash[78274]: cluster 2023-10-01T10:34:21.892134+0000 mon.a (mon.0) 630 : cluster 3 Health check update: Degraded data redundancy: 7842/235785 objects degraded (3.326%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T10:34:23.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:23 smithi173 bash[58816]: audit 2023-10-01T10:34:22.745594+0000 mon.c (mon.2) 81 : audit 0 from='client.? 172.21.15.78:0/1462833472' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:24.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:23 smithi078 bash[78274]: audit 2023-10-01T10:34:22.745594+0000 mon.c (mon.2) 81 : audit 0 from='client.? 172.21.15.78:0/1462833472' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:24.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:23 smithi078 bash[75299]: audit 2023-10-01T10:34:22.745594+0000 mon.c (mon.2) 81 : audit 0 from='client.? 172.21.15.78:0/1462833472' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:24.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:24 smithi173 bash[58816]: cluster 2023-10-01T10:34:23.544801+0000 mgr.y (mgr.25072) 462 : cluster 0 pgmap v382: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 627 op/s; 7842/235785 objects degraded (3.326%); 1.6 MiB/s, 25 objects/s recovering 2023-10-01T10:34:25.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:24 smithi078 bash[78274]: cluster 2023-10-01T10:34:23.544801+0000 mgr.y (mgr.25072) 462 : cluster 0 pgmap v382: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 627 op/s; 7842/235785 objects degraded (3.326%); 1.6 MiB/s, 25 objects/s recovering 2023-10-01T10:34:25.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:24 smithi078 bash[75299]: cluster 2023-10-01T10:34:23.544801+0000 mgr.y (mgr.25072) 462 : cluster 0 pgmap v382: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 627 op/s; 7842/235785 objects degraded (3.326%); 1.6 MiB/s, 25 objects/s recovering 2023-10-01T10:34:26.087 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:25 smithi078 bash[78274]: audit 2023-10-01T10:34:25.373515+0000 mon.c (mon.2) 82 : audit 0 from='client.? 172.21.15.78:0/796466150' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:26.087 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:25 smithi078 bash[78274]: cluster 2023-10-01T10:34:25.545820+0000 mgr.y (mgr.25072) 463 : cluster 0 pgmap v383: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 925 op/s; 7695/225057 objects degraded (3.419%); 2.4 MiB/s, 37 objects/s recovering 2023-10-01T10:34:26.087 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:25 smithi078 bash[75299]: audit 2023-10-01T10:34:25.373515+0000 mon.c (mon.2) 82 : audit 0 from='client.? 172.21.15.78:0/796466150' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:26.087 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:25 smithi078 bash[75299]: cluster 2023-10-01T10:34:25.545820+0000 mgr.y (mgr.25072) 463 : cluster 0 pgmap v383: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 925 op/s; 7695/225057 objects degraded (3.419%); 2.4 MiB/s, 37 objects/s recovering 2023-10-01T10:34:26.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:25 smithi173 bash[58816]: audit 2023-10-01T10:34:25.373515+0000 mon.c (mon.2) 82 : audit 0 from='client.? 172.21.15.78:0/796466150' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:26.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:25 smithi173 bash[58816]: cluster 2023-10-01T10:34:25.545820+0000 mgr.y (mgr.25072) 463 : cluster 0 pgmap v383: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 925 op/s; 7695/225057 objects degraded (3.419%); 2.4 MiB/s, 37 objects/s recovering 2023-10-01T10:34:27.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:26 smithi078 bash[78274]: cluster 2023-10-01T10:34:26.893289+0000 mon.a (mon.0) 631 : cluster 3 Health check update: Degraded data redundancy: 7695/225057 objects degraded (3.419%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T10:34:27.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:26 smithi078 bash[75299]: cluster 2023-10-01T10:34:26.893289+0000 mon.a (mon.0) 631 : cluster 3 Health check update: Degraded data redundancy: 7695/225057 objects degraded (3.419%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T10:34:27.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:26 smithi173 bash[58816]: cluster 2023-10-01T10:34:26.893289+0000 mon.a (mon.0) 631 : cluster 3 Health check update: Degraded data redundancy: 7695/225057 objects degraded (3.419%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T10:34:28.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:27 smithi078 bash[78274]: cluster 2023-10-01T10:34:27.546404+0000 mgr.y (mgr.25072) 464 : cluster 0 pgmap v384: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 7695/225057 objects degraded (3.419%); 1.6 MiB/s, 25 objects/s recovering 2023-10-01T10:34:28.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:27 smithi078 bash[78274]: audit 2023-10-01T10:34:27.949705+0000 mon.c (mon.2) 83 : audit 0 from='client.? 172.21.15.78:0/1695871624' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:28.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:27 smithi078 bash[75299]: cluster 2023-10-01T10:34:27.546404+0000 mgr.y (mgr.25072) 464 : cluster 0 pgmap v384: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 7695/225057 objects degraded (3.419%); 1.6 MiB/s, 25 objects/s recovering 2023-10-01T10:34:28.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:27 smithi078 bash[75299]: audit 2023-10-01T10:34:27.949705+0000 mon.c (mon.2) 83 : audit 0 from='client.? 172.21.15.78:0/1695871624' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:28.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:27 smithi173 bash[58816]: cluster 2023-10-01T10:34:27.546404+0000 mgr.y (mgr.25072) 464 : cluster 0 pgmap v384: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 7695/225057 objects degraded (3.419%); 1.6 MiB/s, 25 objects/s recovering 2023-10-01T10:34:28.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:27 smithi173 bash[58816]: audit 2023-10-01T10:34:27.949705+0000 mon.c (mon.2) 83 : audit 0 from='client.? 172.21.15.78:0/1695871624' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:30.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:34:30 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:34:30] "GET /metrics HTTP/1.1" 200 36213 "" "Prometheus/2.33.4" 2023-10-01T10:34:30.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:30 smithi173 bash[58816]: cluster 2023-10-01T10:34:29.547218+0000 mgr.y (mgr.25072) 465 : cluster 0 pgmap v385: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 787 op/s; 7695/218475 objects degraded (3.522%); 1.6 MiB/s, 25 objects/s recovering 2023-10-01T10:34:30.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:30 smithi173 bash[58816]: audit 2023-10-01T10:34:30.529446+0000 mon.c (mon.2) 84 : audit 0 from='client.? 172.21.15.78:0/2943196418' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:31.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:30 smithi078 bash[75299]: cluster 2023-10-01T10:34:29.547218+0000 mgr.y (mgr.25072) 465 : cluster 0 pgmap v385: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 787 op/s; 7695/218475 objects degraded (3.522%); 1.6 MiB/s, 25 objects/s recovering 2023-10-01T10:34:31.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:30 smithi078 bash[75299]: audit 2023-10-01T10:34:30.529446+0000 mon.c (mon.2) 84 : audit 0 from='client.? 172.21.15.78:0/2943196418' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:31.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:30 smithi078 bash[78274]: cluster 2023-10-01T10:34:29.547218+0000 mgr.y (mgr.25072) 465 : cluster 0 pgmap v385: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 787 op/s; 7695/218475 objects degraded (3.522%); 1.6 MiB/s, 25 objects/s recovering 2023-10-01T10:34:31.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:30 smithi078 bash[78274]: audit 2023-10-01T10:34:30.529446+0000 mon.c (mon.2) 84 : audit 0 from='client.? 172.21.15.78:0/2943196418' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:31.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:31 smithi078 bash[75299]: audit 2023-10-01T10:34:31.553924+0000 mon.a (mon.0) 632 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:34:31.833 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:34:31 smithi078 bash[71214]: debug 2023-10-01T10:34:31.550+0000 7fd9c245e700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:34:31.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:31 smithi078 bash[78274]: audit 2023-10-01T10:34:31.553924+0000 mon.a (mon.0) 632 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:34:31.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:31 smithi173 bash[58816]: audit 2023-10-01T10:34:31.553924+0000 mon.a (mon.0) 632 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:34:33.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:32 smithi078 bash[78274]: cluster 2023-10-01T10:34:31.548096+0000 mgr.y (mgr.25072) 466 : cluster 0 pgmap v386: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 7538/214392 objects degraded (3.516%); 2.2 MiB/s, 38 objects/s recovering 2023-10-01T10:34:33.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:32 smithi078 bash[78274]: audit 2023-10-01T10:34:31.554347+0000 mgr.y (mgr.25072) 467 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:34:33.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:32 smithi078 bash[78274]: cephadm 2023-10-01T10:34:31.556266+0000 mgr.y (mgr.25072) 468 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:34:33.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:32 smithi078 bash[78274]: cluster 2023-10-01T10:34:31.898128+0000 mon.a (mon.0) 633 : cluster 3 Health check update: Degraded data redundancy: 7538/214392 objects degraded (3.516%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T10:34:33.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:32 smithi078 bash[78274]: audit 2023-10-01T10:34:32.399954+0000 mon.a (mon.0) 634 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:34:33.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:32 smithi078 bash[75299]: cluster 2023-10-01T10:34:31.548096+0000 mgr.y (mgr.25072) 466 : cluster 0 pgmap v386: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 7538/214392 objects degraded (3.516%); 2.2 MiB/s, 38 objects/s recovering 2023-10-01T10:34:33.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:32 smithi078 bash[75299]: audit 2023-10-01T10:34:31.554347+0000 mgr.y (mgr.25072) 467 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:34:33.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:32 smithi078 bash[75299]: cephadm 2023-10-01T10:34:31.556266+0000 mgr.y (mgr.25072) 468 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:34:33.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:32 smithi078 bash[75299]: cluster 2023-10-01T10:34:31.898128+0000 mon.a (mon.0) 633 : cluster 3 Health check update: Degraded data redundancy: 7538/214392 objects degraded (3.516%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T10:34:33.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:32 smithi078 bash[75299]: audit 2023-10-01T10:34:32.399954+0000 mon.a (mon.0) 634 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:34:33.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:32 smithi173 bash[58816]: cluster 2023-10-01T10:34:31.548096+0000 mgr.y (mgr.25072) 466 : cluster 0 pgmap v386: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 7538/214392 objects degraded (3.516%); 2.2 MiB/s, 38 objects/s recovering 2023-10-01T10:34:33.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:32 smithi173 bash[58816]: audit 2023-10-01T10:34:31.554347+0000 mgr.y (mgr.25072) 467 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:34:33.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:32 smithi173 bash[58816]: cephadm 2023-10-01T10:34:31.556266+0000 mgr.y (mgr.25072) 468 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:34:33.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:32 smithi173 bash[58816]: cluster 2023-10-01T10:34:31.898128+0000 mon.a (mon.0) 633 : cluster 3 Health check update: Degraded data redundancy: 7538/214392 objects degraded (3.516%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T10:34:33.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:32 smithi173 bash[58816]: audit 2023-10-01T10:34:32.399954+0000 mon.a (mon.0) 634 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:34:34.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:33 smithi078 bash[78274]: audit 2023-10-01T10:34:33.125909+0000 mon.c (mon.2) 85 : audit 0 from='client.? 172.21.15.78:0/2524791860' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:34.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:33 smithi078 bash[78274]: cluster 2023-10-01T10:34:33.548686+0000 mgr.y (mgr.25072) 469 : cluster 0 pgmap v387: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 7538/214392 objects degraded (3.516%); 1.4 MiB/s, 25 objects/s recovering 2023-10-01T10:34:34.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:33 smithi078 bash[75299]: audit 2023-10-01T10:34:33.125909+0000 mon.c (mon.2) 85 : audit 0 from='client.? 172.21.15.78:0/2524791860' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:34.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:33 smithi078 bash[75299]: cluster 2023-10-01T10:34:33.548686+0000 mgr.y (mgr.25072) 469 : cluster 0 pgmap v387: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 7538/214392 objects degraded (3.516%); 1.4 MiB/s, 25 objects/s recovering 2023-10-01T10:34:34.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:33 smithi173 bash[58816]: audit 2023-10-01T10:34:33.125909+0000 mon.c (mon.2) 85 : audit 0 from='client.? 172.21.15.78:0/2524791860' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:34.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:33 smithi173 bash[58816]: cluster 2023-10-01T10:34:33.548686+0000 mgr.y (mgr.25072) 469 : cluster 0 pgmap v387: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 7538/214392 objects degraded (3.516%); 1.4 MiB/s, 25 objects/s recovering 2023-10-01T10:34:36.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:35 smithi078 bash[78274]: cluster 2023-10-01T10:34:34.917856+0000 osd.5 (osd.5) 27 : cluster 3 failed to encode map e108 with expected crc 2023-10-01T10:34:36.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:35 smithi078 bash[78274]: cluster 2023-10-01T10:34:34.919245+0000 mon.a (mon.0) 635 : cluster 0 osdmap e108: 8 total, 8 up, 8 in 2023-10-01T10:34:36.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:35 smithi078 bash[78274]: cluster 2023-10-01T10:34:34.919500+0000 osd.3 (osd.3) 30 : cluster 3 failed to encode map e108 with expected crc 2023-10-01T10:34:36.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:35 smithi078 bash[78274]: cluster 2023-10-01T10:34:34.920139+0000 osd.7 (osd.7) 39 : cluster 3 failed to encode map e108 with expected crc 2023-10-01T10:34:36.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:35 smithi078 bash[78274]: audit 2023-10-01T10:34:35.532435+0000 mon.a (mon.0) 636 : audit 0 from='client.? 172.21.15.78:0/1328599531' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:36.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:35 smithi078 bash[78274]: cluster 2023-10-01T10:34:35.549417+0000 mgr.y (mgr.25072) 470 : cluster 0 pgmap v389: 89 pgs: 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 7396/204006 objects degraded (3.625%); 768 KiB/s, 29 objects/s recovering 2023-10-01T10:34:36.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:35 smithi078 bash[75299]: cluster 2023-10-01T10:34:34.917856+0000 osd.5 (osd.5) 27 : cluster 3 failed to encode map e108 with expected crc 2023-10-01T10:34:36.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:35 smithi078 bash[75299]: cluster 2023-10-01T10:34:34.919245+0000 mon.a (mon.0) 635 : cluster 0 osdmap e108: 8 total, 8 up, 8 in 2023-10-01T10:34:36.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:35 smithi078 bash[75299]: cluster 2023-10-01T10:34:34.919500+0000 osd.3 (osd.3) 30 : cluster 3 failed to encode map e108 with expected crc 2023-10-01T10:34:36.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:35 smithi078 bash[75299]: cluster 2023-10-01T10:34:34.920139+0000 osd.7 (osd.7) 39 : cluster 3 failed to encode map e108 with expected crc 2023-10-01T10:34:36.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:35 smithi078 bash[75299]: audit 2023-10-01T10:34:35.532435+0000 mon.a (mon.0) 636 : audit 0 from='client.? 172.21.15.78:0/1328599531' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:36.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:35 smithi078 bash[75299]: cluster 2023-10-01T10:34:35.549417+0000 mgr.y (mgr.25072) 470 : cluster 0 pgmap v389: 89 pgs: 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 7396/204006 objects degraded (3.625%); 768 KiB/s, 29 objects/s recovering 2023-10-01T10:34:36.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:35 smithi173 bash[58816]: cluster 2023-10-01T10:34:34.917856+0000 osd.5 (osd.5) 27 : cluster 3 failed to encode map e108 with expected crc 2023-10-01T10:34:36.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:35 smithi173 bash[58816]: cluster 2023-10-01T10:34:34.919245+0000 mon.a (mon.0) 635 : cluster 0 osdmap e108: 8 total, 8 up, 8 in 2023-10-01T10:34:36.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:35 smithi173 bash[58816]: cluster 2023-10-01T10:34:34.919500+0000 osd.3 (osd.3) 30 : cluster 3 failed to encode map e108 with expected crc 2023-10-01T10:34:36.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:35 smithi173 bash[58816]: cluster 2023-10-01T10:34:34.920139+0000 osd.7 (osd.7) 39 : cluster 3 failed to encode map e108 with expected crc 2023-10-01T10:34:36.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:35 smithi173 bash[58816]: audit 2023-10-01T10:34:35.532435+0000 mon.a (mon.0) 636 : audit 0 from='client.? 172.21.15.78:0/1328599531' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:36.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:35 smithi173 bash[58816]: cluster 2023-10-01T10:34:35.549417+0000 mgr.y (mgr.25072) 470 : cluster 0 pgmap v389: 89 pgs: 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 7396/204006 objects degraded (3.625%); 768 KiB/s, 29 objects/s recovering 2023-10-01T10:34:37.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:36 smithi078 bash[75299]: cluster 2023-10-01T10:34:35.924503+0000 osd.2 (osd.2) 29 : cluster 3 failed to encode map e109 with expected crc 2023-10-01T10:34:37.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:36 smithi078 bash[75299]: cluster 2023-10-01T10:34:35.928723+0000 osd.7 (osd.7) 40 : cluster 3 failed to encode map e109 with expected crc 2023-10-01T10:34:37.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:36 smithi078 bash[75299]: cluster 2023-10-01T10:34:35.930880+0000 mon.a (mon.0) 637 : cluster 0 osdmap e109: 8 total, 8 up, 8 in 2023-10-01T10:34:37.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:36 smithi078 bash[75299]: cluster 2023-10-01T10:34:35.936618+0000 osd.3 (osd.3) 31 : cluster 3 failed to encode map e109 with expected crc 2023-10-01T10:34:37.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:36 smithi078 bash[75299]: cluster 2023-10-01T10:34:35.972023+0000 osd.6 (osd.6) 21 : cluster 3 failed to encode map e109 with expected crc 2023-10-01T10:34:37.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:36 smithi078 bash[75299]: cluster 2023-10-01T10:34:36.899338+0000 mon.a (mon.0) 638 : cluster 3 Health check update: Degraded data redundancy: 7396/204006 objects degraded (3.625%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T10:34:37.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:36 smithi078 bash[78274]: cluster 2023-10-01T10:34:35.924503+0000 osd.2 (osd.2) 29 : cluster 3 failed to encode map e109 with expected crc 2023-10-01T10:34:37.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:36 smithi078 bash[78274]: cluster 2023-10-01T10:34:35.928723+0000 osd.7 (osd.7) 40 : cluster 3 failed to encode map e109 with expected crc 2023-10-01T10:34:37.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:36 smithi078 bash[78274]: cluster 2023-10-01T10:34:35.930880+0000 mon.a (mon.0) 637 : cluster 0 osdmap e109: 8 total, 8 up, 8 in 2023-10-01T10:34:37.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:36 smithi078 bash[78274]: cluster 2023-10-01T10:34:35.936618+0000 osd.3 (osd.3) 31 : cluster 3 failed to encode map e109 with expected crc 2023-10-01T10:34:37.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:36 smithi078 bash[78274]: cluster 2023-10-01T10:34:35.972023+0000 osd.6 (osd.6) 21 : cluster 3 failed to encode map e109 with expected crc 2023-10-01T10:34:37.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:36 smithi078 bash[78274]: cluster 2023-10-01T10:34:36.899338+0000 mon.a (mon.0) 638 : cluster 3 Health check update: Degraded data redundancy: 7396/204006 objects degraded (3.625%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T10:34:37.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:36 smithi173 bash[58816]: cluster 2023-10-01T10:34:35.924503+0000 osd.2 (osd.2) 29 : cluster 3 failed to encode map e109 with expected crc 2023-10-01T10:34:37.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:36 smithi173 bash[58816]: cluster 2023-10-01T10:34:35.928723+0000 osd.7 (osd.7) 40 : cluster 3 failed to encode map e109 with expected crc 2023-10-01T10:34:37.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:36 smithi173 bash[58816]: cluster 2023-10-01T10:34:35.930880+0000 mon.a (mon.0) 637 : cluster 0 osdmap e109: 8 total, 8 up, 8 in 2023-10-01T10:34:37.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:36 smithi173 bash[58816]: cluster 2023-10-01T10:34:35.936618+0000 osd.3 (osd.3) 31 : cluster 3 failed to encode map e109 with expected crc 2023-10-01T10:34:37.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:36 smithi173 bash[58816]: cluster 2023-10-01T10:34:35.972023+0000 osd.6 (osd.6) 21 : cluster 3 failed to encode map e109 with expected crc 2023-10-01T10:34:37.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:36 smithi173 bash[58816]: cluster 2023-10-01T10:34:36.899338+0000 mon.a (mon.0) 638 : cluster 3 Health check update: Degraded data redundancy: 7396/204006 objects degraded (3.625%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T10:34:38.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:37 smithi078 bash[78274]: cluster 2023-10-01T10:34:37.549951+0000 mgr.y (mgr.25072) 471 : cluster 0 pgmap v391: 89 pgs: 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 7396/204006 objects degraded (3.625%); 960 KiB/s, 37 objects/s recovering 2023-10-01T10:34:38.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:37 smithi078 bash[75299]: cluster 2023-10-01T10:34:37.549951+0000 mgr.y (mgr.25072) 471 : cluster 0 pgmap v391: 89 pgs: 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 7396/204006 objects degraded (3.625%); 960 KiB/s, 37 objects/s recovering 2023-10-01T10:34:38.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:37 smithi173 bash[58816]: cluster 2023-10-01T10:34:37.549951+0000 mgr.y (mgr.25072) 471 : cluster 0 pgmap v391: 89 pgs: 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 7396/204006 objects degraded (3.625%); 960 KiB/s, 37 objects/s recovering 2023-10-01T10:34:39.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:38 smithi078 bash[78274]: audit 2023-10-01T10:34:38.121405+0000 mon.a (mon.0) 639 : audit 0 from='client.? 172.21.15.78:0/2966540668' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:39.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:38 smithi078 bash[75299]: audit 2023-10-01T10:34:38.121405+0000 mon.a (mon.0) 639 : audit 0 from='client.? 172.21.15.78:0/2966540668' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:39.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:38 smithi173 bash[58816]: audit 2023-10-01T10:34:38.121405+0000 mon.a (mon.0) 639 : audit 0 from='client.? 172.21.15.78:0/2966540668' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:40.270 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:39 smithi078 bash[75299]: cluster 2023-10-01T10:34:39.550713+0000 mgr.y (mgr.25072) 472 : cluster 0 pgmap v392: 89 pgs: 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 7149/198537 objects degraded (3.601%); 0 B/s, 48 objects/s recovering 2023-10-01T10:34:40.271 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:34:40 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:34:40] "GET /metrics HTTP/1.1" 200 36210 "" "Prometheus/2.33.4" 2023-10-01T10:34:40.271 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:39 smithi078 bash[78274]: cluster 2023-10-01T10:34:39.550713+0000 mgr.y (mgr.25072) 472 : cluster 0 pgmap v392: 89 pgs: 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 7149/198537 objects degraded (3.601%); 0 B/s, 48 objects/s recovering 2023-10-01T10:34:40.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:39 smithi173 bash[58816]: cluster 2023-10-01T10:34:39.550713+0000 mgr.y (mgr.25072) 472 : cluster 0 pgmap v392: 89 pgs: 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 7149/198537 objects degraded (3.601%); 0 B/s, 48 objects/s recovering 2023-10-01T10:34:41.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:40 smithi078 bash[75299]: audit 2023-10-01T10:34:40.700014+0000 mon.c (mon.2) 86 : audit 0 from='client.? 172.21.15.78:0/1005515557' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:41.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:40 smithi078 bash[78274]: audit 2023-10-01T10:34:40.700014+0000 mon.c (mon.2) 86 : audit 0 from='client.? 172.21.15.78:0/1005515557' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:41.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:40 smithi173 bash[58816]: audit 2023-10-01T10:34:40.700014+0000 mon.c (mon.2) 86 : audit 0 from='client.? 172.21.15.78:0/1005515557' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:42.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:41 smithi078 bash[75299]: cluster 2023-10-01T10:34:41.551287+0000 mgr.y (mgr.25072) 473 : cluster 0 pgmap v393: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 796 op/s; 7149/195276 objects degraded (3.661%); 0 B/s, 48 objects/s recovering 2023-10-01T10:34:42.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:41 smithi078 bash[75299]: cluster 2023-10-01T10:34:41.900813+0000 mon.a (mon.0) 640 : cluster 3 Health check update: Degraded data redundancy: 7149/198537 objects degraded (3.601%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T10:34:42.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:41 smithi078 bash[78274]: cluster 2023-10-01T10:34:41.551287+0000 mgr.y (mgr.25072) 473 : cluster 0 pgmap v393: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 796 op/s; 7149/195276 objects degraded (3.661%); 0 B/s, 48 objects/s recovering 2023-10-01T10:34:42.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:41 smithi078 bash[78274]: cluster 2023-10-01T10:34:41.900813+0000 mon.a (mon.0) 640 : cluster 3 Health check update: Degraded data redundancy: 7149/198537 objects degraded (3.601%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T10:34:42.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:41 smithi173 bash[58816]: cluster 2023-10-01T10:34:41.551287+0000 mgr.y (mgr.25072) 473 : cluster 0 pgmap v393: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 796 op/s; 7149/195276 objects degraded (3.661%); 0 B/s, 48 objects/s recovering 2023-10-01T10:34:42.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:41 smithi173 bash[58816]: cluster 2023-10-01T10:34:41.900813+0000 mon.a (mon.0) 640 : cluster 3 Health check update: Degraded data redundancy: 7149/198537 objects degraded (3.601%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T10:34:43.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:43 smithi173 bash[58816]: audit 2023-10-01T10:34:43.297883+0000 mon.a (mon.0) 641 : audit 0 from='client.? 172.21.15.78:0/2268032110' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:43.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:43 smithi078 bash[75299]: audit 2023-10-01T10:34:43.297883+0000 mon.a (mon.0) 641 : audit 0 from='client.? 172.21.15.78:0/2268032110' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:43.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:43 smithi078 bash[78274]: audit 2023-10-01T10:34:43.297883+0000 mon.a (mon.0) 641 : audit 0 from='client.? 172.21.15.78:0/2268032110' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:44.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:44 smithi078 bash[75299]: cluster 2023-10-01T10:34:43.551775+0000 mgr.y (mgr.25072) 474 : cluster 0 pgmap v394: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 491 op/s; 7149/195276 objects degraded (3.661%); 0 B/s, 45 objects/s recovering 2023-10-01T10:34:44.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:44 smithi078 bash[78274]: cluster 2023-10-01T10:34:43.551775+0000 mgr.y (mgr.25072) 474 : cluster 0 pgmap v394: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 491 op/s; 7149/195276 objects degraded (3.661%); 0 B/s, 45 objects/s recovering 2023-10-01T10:34:44.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:44 smithi173 bash[58816]: cluster 2023-10-01T10:34:43.551775+0000 mgr.y (mgr.25072) 474 : cluster 0 pgmap v394: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 491 op/s; 7149/195276 objects degraded (3.661%); 0 B/s, 45 objects/s recovering 2023-10-01T10:34:46.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:46 smithi173 bash[58816]: cluster 2023-10-01T10:34:45.557284+0000 mgr.y (mgr.25072) 475 : cluster 0 pgmap v395: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 639 op/s; 6974/184806 objects degraded (3.774%); 0 B/s, 42 objects/s recovering 2023-10-01T10:34:46.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:46 smithi173 bash[58816]: audit 2023-10-01T10:34:45.862009+0000 mon.c (mon.2) 87 : audit 0 from='client.? 172.21.15.78:0/2753609213' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:46.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:46 smithi173 bash[58816]: audit 2023-10-01T10:34:46.570048+0000 mon.a (mon.0) 642 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:34:47.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:46 smithi078 bash[75299]: cluster 2023-10-01T10:34:45.557284+0000 mgr.y (mgr.25072) 475 : cluster 0 pgmap v395: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 639 op/s; 6974/184806 objects degraded (3.774%); 0 B/s, 42 objects/s recovering 2023-10-01T10:34:47.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:46 smithi078 bash[75299]: audit 2023-10-01T10:34:45.862009+0000 mon.c (mon.2) 87 : audit 0 from='client.? 172.21.15.78:0/2753609213' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:47.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:46 smithi078 bash[75299]: audit 2023-10-01T10:34:46.570048+0000 mon.a (mon.0) 642 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:34:47.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:46 smithi078 bash[78274]: cluster 2023-10-01T10:34:45.557284+0000 mgr.y (mgr.25072) 475 : cluster 0 pgmap v395: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 639 op/s; 6974/184806 objects degraded (3.774%); 0 B/s, 42 objects/s recovering 2023-10-01T10:34:47.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:46 smithi078 bash[78274]: audit 2023-10-01T10:34:45.862009+0000 mon.c (mon.2) 87 : audit 0 from='client.? 172.21.15.78:0/2753609213' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:47.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:46 smithi078 bash[78274]: audit 2023-10-01T10:34:46.570048+0000 mon.a (mon.0) 642 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:34:48.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:47 smithi078 bash[78274]: cluster 2023-10-01T10:34:46.902187+0000 mon.a (mon.0) 643 : cluster 3 Health check update: Degraded data redundancy: 6974/184806 objects degraded (3.774%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T10:34:48.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:47 smithi078 bash[78274]: audit 2023-10-01T10:34:47.400099+0000 mon.a (mon.0) 644 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:34:48.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:47 smithi078 bash[75299]: cluster 2023-10-01T10:34:46.902187+0000 mon.a (mon.0) 643 : cluster 3 Health check update: Degraded data redundancy: 6974/184806 objects degraded (3.774%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T10:34:48.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:47 smithi078 bash[75299]: audit 2023-10-01T10:34:47.400099+0000 mon.a (mon.0) 644 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:34:48.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:47 smithi173 bash[58816]: cluster 2023-10-01T10:34:46.902187+0000 mon.a (mon.0) 643 : cluster 3 Health check update: Degraded data redundancy: 6974/184806 objects degraded (3.774%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T10:34:48.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:47 smithi173 bash[58816]: audit 2023-10-01T10:34:47.400099+0000 mon.a (mon.0) 644 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:34:49.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:48 smithi078 bash[78274]: cluster 2023-10-01T10:34:47.562319+0000 mgr.y (mgr.25072) 476 : cluster 0 pgmap v396: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 549 op/s; 6974/184806 objects degraded (3.774%); 0 B/s, 36 objects/s recovering 2023-10-01T10:34:49.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:48 smithi078 bash[78274]: audit 2023-10-01T10:34:48.406953+0000 mon.a (mon.0) 645 : audit 0 from='client.? 172.21.15.78:0/3509728509' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:49.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:48 smithi078 bash[75299]: cluster 2023-10-01T10:34:47.562319+0000 mgr.y (mgr.25072) 476 : cluster 0 pgmap v396: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 549 op/s; 6974/184806 objects degraded (3.774%); 0 B/s, 36 objects/s recovering 2023-10-01T10:34:49.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:48 smithi078 bash[75299]: audit 2023-10-01T10:34:48.406953+0000 mon.a (mon.0) 645 : audit 0 from='client.? 172.21.15.78:0/3509728509' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:49.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:48 smithi173 bash[58816]: cluster 2023-10-01T10:34:47.562319+0000 mgr.y (mgr.25072) 476 : cluster 0 pgmap v396: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 549 op/s; 6974/184806 objects degraded (3.774%); 0 B/s, 36 objects/s recovering 2023-10-01T10:34:49.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:48 smithi173 bash[58816]: audit 2023-10-01T10:34:48.406953+0000 mon.a (mon.0) 645 : audit 0 from='client.? 172.21.15.78:0/3509728509' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:50.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:49 smithi078 bash[75299]: cluster 2023-10-01T10:34:49.563074+0000 mgr.y (mgr.25072) 477 : cluster 0 pgmap v397: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 6770/178410 objects degraded (3.795%); 0 B/s, 52 objects/s recovering 2023-10-01T10:34:50.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:49 smithi078 bash[78274]: cluster 2023-10-01T10:34:49.563074+0000 mgr.y (mgr.25072) 477 : cluster 0 pgmap v397: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 6770/178410 objects degraded (3.795%); 0 B/s, 52 objects/s recovering 2023-10-01T10:34:50.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:49 smithi173 bash[58816]: cluster 2023-10-01T10:34:49.563074+0000 mgr.y (mgr.25072) 477 : cluster 0 pgmap v397: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 6770/178410 objects degraded (3.795%); 0 B/s, 52 objects/s recovering 2023-10-01T10:34:50.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:34:50 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:34:50] "GET /metrics HTTP/1.1" 200 36199 "" "Prometheus/2.33.4" 2023-10-01T10:34:51.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:51 smithi078 bash[75299]: audit 2023-10-01T10:34:50.982782+0000 mon.a (mon.0) 646 : audit 0 from='client.? 172.21.15.78:0/29238087' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:51.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:51 smithi078 bash[78274]: audit 2023-10-01T10:34:50.982782+0000 mon.a (mon.0) 646 : audit 0 from='client.? 172.21.15.78:0/29238087' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:51.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:51 smithi173 bash[58816]: audit 2023-10-01T10:34:50.982782+0000 mon.a (mon.0) 646 : audit 0 from='client.? 172.21.15.78:0/29238087' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:52.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:52 smithi078 bash[78274]: cluster 2023-10-01T10:34:51.563730+0000 mgr.y (mgr.25072) 478 : cluster 0 pgmap v398: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 6770/174441 objects degraded (3.881%); 0 B/s, 31 objects/s recovering 2023-10-01T10:34:52.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:52 smithi078 bash[78274]: cluster 2023-10-01T10:34:51.905225+0000 mon.a (mon.0) 647 : cluster 3 Health check update: Degraded data redundancy: 6770/178410 objects degraded (3.795%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T10:34:52.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:52 smithi078 bash[75299]: cluster 2023-10-01T10:34:51.563730+0000 mgr.y (mgr.25072) 478 : cluster 0 pgmap v398: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 6770/174441 objects degraded (3.881%); 0 B/s, 31 objects/s recovering 2023-10-01T10:34:52.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:52 smithi078 bash[75299]: cluster 2023-10-01T10:34:51.905225+0000 mon.a (mon.0) 647 : cluster 3 Health check update: Degraded data redundancy: 6770/178410 objects degraded (3.795%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T10:34:52.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:52 smithi173 bash[58816]: cluster 2023-10-01T10:34:51.563730+0000 mgr.y (mgr.25072) 478 : cluster 0 pgmap v398: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 6770/174441 objects degraded (3.881%); 0 B/s, 31 objects/s recovering 2023-10-01T10:34:52.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:52 smithi173 bash[58816]: cluster 2023-10-01T10:34:51.905225+0000 mon.a (mon.0) 647 : cluster 3 Health check update: Degraded data redundancy: 6770/178410 objects degraded (3.795%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T10:34:54.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:53 smithi078 bash[78274]: audit 2023-10-01T10:34:52.934252+0000 mon.a (mon.0) 648 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:34:54.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:53 smithi078 bash[78274]: audit 2023-10-01T10:34:52.951892+0000 mon.a (mon.0) 649 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:34:54.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:53 smithi078 bash[78274]: audit 2023-10-01T10:34:53.028002+0000 mon.a (mon.0) 650 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:34:54.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:53 smithi078 bash[78274]: audit 2023-10-01T10:34:53.044541+0000 mon.a (mon.0) 651 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:34:54.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:53 smithi078 bash[78274]: cluster 2023-10-01T10:34:53.564349+0000 mgr.y (mgr.25072) 479 : cluster 0 pgmap v399: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 6770/174441 objects degraded (3.881%); 0 B/s, 31 objects/s recovering 2023-10-01T10:34:54.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:53 smithi078 bash[78274]: audit 2023-10-01T10:34:53.635932+0000 mon.c (mon.2) 88 : audit 0 from='client.? 172.21.15.78:0/4158413978' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:54.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:53 smithi078 bash[78274]: audit 2023-10-01T10:34:53.741756+0000 mon.a (mon.0) 652 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:34:54.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:53 smithi078 bash[78274]: audit 2023-10-01T10:34:53.744465+0000 mon.a (mon.0) 653 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:34:54.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:53 smithi078 bash[78274]: audit 2023-10-01T10:34:53.755861+0000 mon.a (mon.0) 654 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:34:54.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:53 smithi078 bash[78274]: audit 2023-10-01T10:34:53.840122+0000 mon.a (mon.0) 655 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:34:54.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:53 smithi078 bash[78274]: audit 2023-10-01T10:34:53.844109+0000 mon.a (mon.0) 656 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:34:54.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:53 smithi078 bash[78274]: audit 2023-10-01T10:34:53.846886+0000 mon.a (mon.0) 657 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:34:54.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:53 smithi078 bash[78274]: audit 2023-10-01T10:34:53.848645+0000 mon.a (mon.0) 658 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:34:54.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:53 smithi078 bash[78274]: audit 2023-10-01T10:34:53.851741+0000 mon.a (mon.0) 659 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:34:54.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:53 smithi078 bash[75299]: audit 2023-10-01T10:34:52.934252+0000 mon.a (mon.0) 648 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:34:54.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:53 smithi078 bash[75299]: audit 2023-10-01T10:34:52.951892+0000 mon.a (mon.0) 649 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:34:54.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:53 smithi078 bash[75299]: audit 2023-10-01T10:34:53.028002+0000 mon.a (mon.0) 650 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:34:54.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:53 smithi078 bash[75299]: audit 2023-10-01T10:34:53.044541+0000 mon.a (mon.0) 651 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:34:54.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:53 smithi078 bash[75299]: cluster 2023-10-01T10:34:53.564349+0000 mgr.y (mgr.25072) 479 : cluster 0 pgmap v399: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 6770/174441 objects degraded (3.881%); 0 B/s, 31 objects/s recovering 2023-10-01T10:34:54.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:53 smithi078 bash[75299]: audit 2023-10-01T10:34:53.635932+0000 mon.c (mon.2) 88 : audit 0 from='client.? 172.21.15.78:0/4158413978' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:54.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:53 smithi078 bash[75299]: audit 2023-10-01T10:34:53.741756+0000 mon.a (mon.0) 652 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:34:54.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:53 smithi078 bash[75299]: audit 2023-10-01T10:34:53.744465+0000 mon.a (mon.0) 653 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:34:54.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:53 smithi078 bash[75299]: audit 2023-10-01T10:34:53.755861+0000 mon.a (mon.0) 654 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:34:54.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:53 smithi078 bash[75299]: audit 2023-10-01T10:34:53.840122+0000 mon.a (mon.0) 655 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:34:54.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:53 smithi078 bash[75299]: audit 2023-10-01T10:34:53.844109+0000 mon.a (mon.0) 656 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:34:54.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:53 smithi078 bash[75299]: audit 2023-10-01T10:34:53.846886+0000 mon.a (mon.0) 657 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:34:54.336 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:53 smithi078 bash[75299]: audit 2023-10-01T10:34:53.848645+0000 mon.a (mon.0) 658 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:34:54.336 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:53 smithi078 bash[75299]: audit 2023-10-01T10:34:53.851741+0000 mon.a (mon.0) 659 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:34:54.336 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:34:53 smithi078 bash[71214]: debug 2023-10-01T10:34:53.846+0000 7fd9c245e700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:34:54.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:53 smithi173 bash[58816]: audit 2023-10-01T10:34:52.934252+0000 mon.a (mon.0) 648 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:34:54.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:53 smithi173 bash[58816]: audit 2023-10-01T10:34:52.951892+0000 mon.a (mon.0) 649 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:34:54.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:53 smithi173 bash[58816]: audit 2023-10-01T10:34:53.028002+0000 mon.a (mon.0) 650 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:34:54.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:53 smithi173 bash[58816]: audit 2023-10-01T10:34:53.044541+0000 mon.a (mon.0) 651 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:34:54.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:53 smithi173 bash[58816]: cluster 2023-10-01T10:34:53.564349+0000 mgr.y (mgr.25072) 479 : cluster 0 pgmap v399: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 6770/174441 objects degraded (3.881%); 0 B/s, 31 objects/s recovering 2023-10-01T10:34:54.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:53 smithi173 bash[58816]: audit 2023-10-01T10:34:53.635932+0000 mon.c (mon.2) 88 : audit 0 from='client.? 172.21.15.78:0/4158413978' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:54.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:53 smithi173 bash[58816]: audit 2023-10-01T10:34:53.741756+0000 mon.a (mon.0) 652 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:34:54.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:53 smithi173 bash[58816]: audit 2023-10-01T10:34:53.744465+0000 mon.a (mon.0) 653 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:34:54.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:53 smithi173 bash[58816]: audit 2023-10-01T10:34:53.755861+0000 mon.a (mon.0) 654 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:34:54.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:53 smithi173 bash[58816]: audit 2023-10-01T10:34:53.840122+0000 mon.a (mon.0) 655 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:34:54.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:53 smithi173 bash[58816]: audit 2023-10-01T10:34:53.844109+0000 mon.a (mon.0) 656 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:34:54.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:53 smithi173 bash[58816]: audit 2023-10-01T10:34:53.846886+0000 mon.a (mon.0) 657 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:34:54.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:53 smithi173 bash[58816]: audit 2023-10-01T10:34:53.848645+0000 mon.a (mon.0) 658 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:34:54.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:53 smithi173 bash[58816]: audit 2023-10-01T10:34:53.851741+0000 mon.a (mon.0) 659 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:34:55.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:55 smithi078 bash[78274]: audit 2023-10-01T10:34:53.852028+0000 mgr.y (mgr.25072) 480 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:34:55.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:55 smithi078 bash[78274]: cephadm 2023-10-01T10:34:53.854492+0000 mgr.y (mgr.25072) 481 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:34:55.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:55 smithi078 bash[75299]: audit 2023-10-01T10:34:53.852028+0000 mgr.y (mgr.25072) 480 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:34:55.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:55 smithi078 bash[75299]: cephadm 2023-10-01T10:34:53.854492+0000 mgr.y (mgr.25072) 481 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:34:55.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:55 smithi173 bash[58816]: audit 2023-10-01T10:34:53.852028+0000 mgr.y (mgr.25072) 480 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:34:55.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:55 smithi173 bash[58816]: cephadm 2023-10-01T10:34:53.854492+0000 mgr.y (mgr.25072) 481 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:34:56.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:56 smithi078 bash[78274]: cluster 2023-10-01T10:34:55.565226+0000 mgr.y (mgr.25072) 482 : cluster 0 pgmap v400: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 847 op/s; 6580/164721 objects degraded (3.995%); 0 B/s, 47 objects/s recovering 2023-10-01T10:34:56.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:56 smithi078 bash[75299]: cluster 2023-10-01T10:34:55.565226+0000 mgr.y (mgr.25072) 482 : cluster 0 pgmap v400: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 847 op/s; 6580/164721 objects degraded (3.995%); 0 B/s, 47 objects/s recovering 2023-10-01T10:34:56.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:56 smithi173 bash[58816]: cluster 2023-10-01T10:34:55.565226+0000 mgr.y (mgr.25072) 482 : cluster 0 pgmap v400: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 847 op/s; 6580/164721 objects degraded (3.995%); 0 B/s, 47 objects/s recovering 2023-10-01T10:34:57.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:57 smithi078 bash[78274]: audit 2023-10-01T10:34:56.233638+0000 mon.c (mon.2) 89 : audit 0 from='client.? 172.21.15.78:0/1239069187' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:57.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:57 smithi078 bash[78274]: cluster 2023-10-01T10:34:56.906346+0000 mon.a (mon.0) 660 : cluster 3 Health check update: Degraded data redundancy: 6580/164721 objects degraded (3.995%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T10:34:57.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:57 smithi078 bash[75299]: audit 2023-10-01T10:34:56.233638+0000 mon.c (mon.2) 89 : audit 0 from='client.? 172.21.15.78:0/1239069187' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:57.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:57 smithi078 bash[75299]: cluster 2023-10-01T10:34:56.906346+0000 mon.a (mon.0) 660 : cluster 3 Health check update: Degraded data redundancy: 6580/164721 objects degraded (3.995%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T10:34:57.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:57 smithi173 bash[58816]: audit 2023-10-01T10:34:56.233638+0000 mon.c (mon.2) 89 : audit 0 from='client.? 172.21.15.78:0/1239069187' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:57.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:57 smithi173 bash[58816]: cluster 2023-10-01T10:34:56.906346+0000 mon.a (mon.0) 660 : cluster 3 Health check update: Degraded data redundancy: 6580/164721 objects degraded (3.995%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T10:34:58.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:58 smithi078 bash[78274]: cluster 2023-10-01T10:34:57.565754+0000 mgr.y (mgr.25072) 483 : cluster 0 pgmap v401: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 557 op/s; 6580/164721 objects degraded (3.995%); 0 B/s, 32 objects/s recovering 2023-10-01T10:34:58.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:58 smithi078 bash[75299]: cluster 2023-10-01T10:34:57.565754+0000 mgr.y (mgr.25072) 483 : cluster 0 pgmap v401: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 557 op/s; 6580/164721 objects degraded (3.995%); 0 B/s, 32 objects/s recovering 2023-10-01T10:34:58.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:58 smithi173 bash[58816]: cluster 2023-10-01T10:34:57.565754+0000 mgr.y (mgr.25072) 483 : cluster 0 pgmap v401: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 557 op/s; 6580/164721 objects degraded (3.995%); 0 B/s, 32 objects/s recovering 2023-10-01T10:34:59.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:34:59 smithi078 bash[78274]: audit 2023-10-01T10:34:58.829450+0000 mon.c (mon.2) 90 : audit 0 from='client.? 172.21.15.78:0/4193780631' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:59.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:34:59 smithi078 bash[75299]: audit 2023-10-01T10:34:58.829450+0000 mon.c (mon.2) 90 : audit 0 from='client.? 172.21.15.78:0/4193780631' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:34:59.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:34:59 smithi173 bash[58816]: audit 2023-10-01T10:34:58.829450+0000 mon.c (mon.2) 90 : audit 0 from='client.? 172.21.15.78:0/4193780631' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:00.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:00 smithi078 bash[78274]: cluster 2023-10-01T10:34:59.566496+0000 mgr.y (mgr.25072) 484 : cluster 0 pgmap v402: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 730 op/s; 6402/158493 objects degraded (4.039%); 0 B/s, 47 objects/s recovering 2023-10-01T10:35:00.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:00 smithi078 bash[75299]: cluster 2023-10-01T10:34:59.566496+0000 mgr.y (mgr.25072) 484 : cluster 0 pgmap v402: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 730 op/s; 6402/158493 objects degraded (4.039%); 0 B/s, 47 objects/s recovering 2023-10-01T10:35:00.333 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:35:00 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:35:00] "GET /metrics HTTP/1.1" 200 36199 "" "Prometheus/2.33.4" 2023-10-01T10:35:00.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:00 smithi173 bash[58816]: cluster 2023-10-01T10:34:59.566496+0000 mgr.y (mgr.25072) 484 : cluster 0 pgmap v402: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 730 op/s; 6402/158493 objects degraded (4.039%); 0 B/s, 47 objects/s recovering 2023-10-01T10:35:01.483 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:01 smithi173 bash[58816]: audit 2023-10-01T10:35:01.429089+0000 mon.b (mon.1) 31 : audit 0 from='client.? 172.21.15.78:0/3226475741' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:01.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:01 smithi078 bash[78274]: audit 2023-10-01T10:35:01.429089+0000 mon.b (mon.1) 31 : audit 0 from='client.? 172.21.15.78:0/3226475741' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:01.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:01 smithi078 bash[75299]: audit 2023-10-01T10:35:01.429089+0000 mon.b (mon.1) 31 : audit 0 from='client.? 172.21.15.78:0/3226475741' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:02.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:02 smithi078 bash[75299]: cluster 2023-10-01T10:35:01.567038+0000 mgr.y (mgr.25072) 485 : cluster 0 pgmap v403: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 6402/154839 objects degraded (4.135%); 0 B/s, 30 objects/s recovering 2023-10-01T10:35:02.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:02 smithi078 bash[75299]: cluster 2023-10-01T10:35:01.907637+0000 mon.a (mon.0) 661 : cluster 3 Health check update: Degraded data redundancy: 6402/158493 objects degraded (4.039%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T10:35:02.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:02 smithi078 bash[75299]: audit 2023-10-01T10:35:02.400641+0000 mon.a (mon.0) 662 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:35:02.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:02 smithi078 bash[78274]: cluster 2023-10-01T10:35:01.567038+0000 mgr.y (mgr.25072) 485 : cluster 0 pgmap v403: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 6402/154839 objects degraded (4.135%); 0 B/s, 30 objects/s recovering 2023-10-01T10:35:02.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:02 smithi078 bash[78274]: cluster 2023-10-01T10:35:01.907637+0000 mon.a (mon.0) 661 : cluster 3 Health check update: Degraded data redundancy: 6402/158493 objects degraded (4.039%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T10:35:02.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:02 smithi078 bash[78274]: audit 2023-10-01T10:35:02.400641+0000 mon.a (mon.0) 662 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:35:02.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:02 smithi173 bash[58816]: cluster 2023-10-01T10:35:01.567038+0000 mgr.y (mgr.25072) 485 : cluster 0 pgmap v403: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 6402/154839 objects degraded (4.135%); 0 B/s, 30 objects/s recovering 2023-10-01T10:35:02.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:02 smithi173 bash[58816]: cluster 2023-10-01T10:35:01.907637+0000 mon.a (mon.0) 661 : cluster 3 Health check update: Degraded data redundancy: 6402/158493 objects degraded (4.039%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T10:35:02.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:02 smithi173 bash[58816]: audit 2023-10-01T10:35:02.400641+0000 mon.a (mon.0) 662 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:35:04.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:03 smithi078 bash[78274]: cluster 2023-10-01T10:35:03.567727+0000 mgr.y (mgr.25072) 486 : cluster 0 pgmap v404: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 544 op/s; 6402/154839 objects degraded (4.135%); 0 B/s, 30 objects/s recovering 2023-10-01T10:35:04.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:03 smithi078 bash[75299]: cluster 2023-10-01T10:35:03.567727+0000 mgr.y (mgr.25072) 486 : cluster 0 pgmap v404: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 544 op/s; 6402/154839 objects degraded (4.135%); 0 B/s, 30 objects/s recovering 2023-10-01T10:35:04.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:03 smithi173 bash[58816]: cluster 2023-10-01T10:35:03.567727+0000 mgr.y (mgr.25072) 486 : cluster 0 pgmap v404: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 544 op/s; 6402/154839 objects degraded (4.135%); 0 B/s, 30 objects/s recovering 2023-10-01T10:35:05.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:04 smithi078 bash[78274]: audit 2023-10-01T10:35:04.001731+0000 mon.a (mon.0) 663 : audit 0 from='client.? 172.21.15.78:0/569379027' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:05.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:04 smithi078 bash[75299]: audit 2023-10-01T10:35:04.001731+0000 mon.a (mon.0) 663 : audit 0 from='client.? 172.21.15.78:0/569379027' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:05.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:04 smithi173 bash[58816]: audit 2023-10-01T10:35:04.001731+0000 mon.a (mon.0) 663 : audit 0 from='client.? 172.21.15.78:0/569379027' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:06.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:05 smithi078 bash[75299]: cluster 2023-10-01T10:35:05.568554+0000 mgr.y (mgr.25072) 487 : cluster 0 pgmap v405: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 822 op/s; 6220/144810 objects degraded (4.295%); 0 B/s, 45 objects/s recovering 2023-10-01T10:35:06.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:05 smithi078 bash[78274]: cluster 2023-10-01T10:35:05.568554+0000 mgr.y (mgr.25072) 487 : cluster 0 pgmap v405: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 822 op/s; 6220/144810 objects degraded (4.295%); 0 B/s, 45 objects/s recovering 2023-10-01T10:35:06.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:05 smithi173 bash[58816]: cluster 2023-10-01T10:35:05.568554+0000 mgr.y (mgr.25072) 487 : cluster 0 pgmap v405: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 822 op/s; 6220/144810 objects degraded (4.295%); 0 B/s, 45 objects/s recovering 2023-10-01T10:35:07.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:06 smithi078 bash[75299]: audit 2023-10-01T10:35:06.605384+0000 mon.a (mon.0) 664 : audit 0 from='client.? 172.21.15.78:0/4222353789' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:07.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:06 smithi078 bash[78274]: audit 2023-10-01T10:35:06.605384+0000 mon.a (mon.0) 664 : audit 0 from='client.? 172.21.15.78:0/4222353789' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:07.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:06 smithi173 bash[58816]: audit 2023-10-01T10:35:06.605384+0000 mon.a (mon.0) 664 : audit 0 from='client.? 172.21.15.78:0/4222353789' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:08.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:07 smithi078 bash[75299]: cluster 2023-10-01T10:35:06.908856+0000 mon.a (mon.0) 665 : cluster 3 Health check update: Degraded data redundancy: 6220/144810 objects degraded (4.295%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T10:35:08.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:07 smithi078 bash[75299]: cluster 2023-10-01T10:35:07.569086+0000 mgr.y (mgr.25072) 488 : cluster 0 pgmap v406: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 552 op/s; 6220/144810 objects degraded (4.295%); 0 B/s, 29 objects/s recovering 2023-10-01T10:35:08.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:07 smithi078 bash[78274]: cluster 2023-10-01T10:35:06.908856+0000 mon.a (mon.0) 665 : cluster 3 Health check update: Degraded data redundancy: 6220/144810 objects degraded (4.295%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T10:35:08.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:07 smithi078 bash[78274]: cluster 2023-10-01T10:35:07.569086+0000 mgr.y (mgr.25072) 488 : cluster 0 pgmap v406: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 552 op/s; 6220/144810 objects degraded (4.295%); 0 B/s, 29 objects/s recovering 2023-10-01T10:35:08.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:07 smithi173 bash[58816]: cluster 2023-10-01T10:35:06.908856+0000 mon.a (mon.0) 665 : cluster 3 Health check update: Degraded data redundancy: 6220/144810 objects degraded (4.295%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T10:35:08.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:07 smithi173 bash[58816]: cluster 2023-10-01T10:35:07.569086+0000 mgr.y (mgr.25072) 488 : cluster 0 pgmap v406: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 552 op/s; 6220/144810 objects degraded (4.295%); 0 B/s, 29 objects/s recovering 2023-10-01T10:35:09.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:08 smithi078 bash[75299]: audit 2023-10-01T10:35:08.869882+0000 mon.a (mon.0) 666 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:35:09.333 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:35:08 smithi078 bash[71214]: debug 2023-10-01T10:35:08.866+0000 7fd9c245e700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:35:09.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:08 smithi078 bash[78274]: audit 2023-10-01T10:35:08.869882+0000 mon.a (mon.0) 666 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:35:09.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:08 smithi173 bash[58816]: audit 2023-10-01T10:35:08.869882+0000 mon.a (mon.0) 666 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:35:10.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:09 smithi078 bash[78274]: audit 2023-10-01T10:35:08.870141+0000 mgr.y (mgr.25072) 489 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:35:10.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:09 smithi078 bash[78274]: cephadm 2023-10-01T10:35:08.872515+0000 mgr.y (mgr.25072) 490 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:35:10.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:09 smithi078 bash[78274]: audit 2023-10-01T10:35:09.179240+0000 mon.b (mon.1) 32 : audit 0 from='client.? 172.21.15.78:0/1273026649' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:10.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:09 smithi078 bash[78274]: cluster 2023-10-01T10:35:09.569922+0000 mgr.y (mgr.25072) 491 : cluster 0 pgmap v407: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 723 op/s; 6026/138657 objects degraded (4.346%); 0 B/s, 46 objects/s recovering 2023-10-01T10:35:10.271 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:09 smithi078 bash[75299]: audit 2023-10-01T10:35:08.870141+0000 mgr.y (mgr.25072) 489 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:35:10.271 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:09 smithi078 bash[75299]: cephadm 2023-10-01T10:35:08.872515+0000 mgr.y (mgr.25072) 490 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:35:10.271 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:09 smithi078 bash[75299]: audit 2023-10-01T10:35:09.179240+0000 mon.b (mon.1) 32 : audit 0 from='client.? 172.21.15.78:0/1273026649' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:10.271 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:09 smithi078 bash[75299]: cluster 2023-10-01T10:35:09.569922+0000 mgr.y (mgr.25072) 491 : cluster 0 pgmap v407: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 723 op/s; 6026/138657 objects degraded (4.346%); 0 B/s, 46 objects/s recovering 2023-10-01T10:35:10.271 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:35:10 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:35:10] "GET /metrics HTTP/1.1" 200 36199 "" "Prometheus/2.33.4" 2023-10-01T10:35:10.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:09 smithi173 bash[58816]: audit 2023-10-01T10:35:08.870141+0000 mgr.y (mgr.25072) 489 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:35:10.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:09 smithi173 bash[58816]: cephadm 2023-10-01T10:35:08.872515+0000 mgr.y (mgr.25072) 490 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:35:10.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:09 smithi173 bash[58816]: audit 2023-10-01T10:35:09.179240+0000 mon.b (mon.1) 32 : audit 0 from='client.? 172.21.15.78:0/1273026649' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:10.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:09 smithi173 bash[58816]: cluster 2023-10-01T10:35:09.569922+0000 mgr.y (mgr.25072) 491 : cluster 0 pgmap v407: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 723 op/s; 6026/138657 objects degraded (4.346%); 0 B/s, 46 objects/s recovering 2023-10-01T10:35:13.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:12 smithi078 bash[78274]: cluster 2023-10-01T10:35:11.570767+0000 mgr.y (mgr.25072) 492 : cluster 0 pgmap v408: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 6026/134823 objects degraded (4.470%); 0 B/s, 31 objects/s recovering 2023-10-01T10:35:13.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:12 smithi078 bash[78274]: audit 2023-10-01T10:35:11.798606+0000 mon.c (mon.2) 91 : audit 0 from='client.? 172.21.15.78:0/3291657229' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:13.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:12 smithi078 bash[78274]: cluster 2023-10-01T10:35:11.910184+0000 mon.a (mon.0) 667 : cluster 3 Health check update: Degraded data redundancy: 6026/134823 objects degraded (4.470%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T10:35:13.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:12 smithi078 bash[75299]: cluster 2023-10-01T10:35:11.570767+0000 mgr.y (mgr.25072) 492 : cluster 0 pgmap v408: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 6026/134823 objects degraded (4.470%); 0 B/s, 31 objects/s recovering 2023-10-01T10:35:13.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:12 smithi078 bash[75299]: audit 2023-10-01T10:35:11.798606+0000 mon.c (mon.2) 91 : audit 0 from='client.? 172.21.15.78:0/3291657229' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:13.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:12 smithi078 bash[75299]: cluster 2023-10-01T10:35:11.910184+0000 mon.a (mon.0) 667 : cluster 3 Health check update: Degraded data redundancy: 6026/134823 objects degraded (4.470%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T10:35:13.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:12 smithi173 bash[58816]: cluster 2023-10-01T10:35:11.570767+0000 mgr.y (mgr.25072) 492 : cluster 0 pgmap v408: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 6026/134823 objects degraded (4.470%); 0 B/s, 31 objects/s recovering 2023-10-01T10:35:13.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:12 smithi173 bash[58816]: audit 2023-10-01T10:35:11.798606+0000 mon.c (mon.2) 91 : audit 0 from='client.? 172.21.15.78:0/3291657229' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:13.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:12 smithi173 bash[58816]: cluster 2023-10-01T10:35:11.910184+0000 mon.a (mon.0) 667 : cluster 3 Health check update: Degraded data redundancy: 6026/134823 objects degraded (4.470%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T10:35:14.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:13 smithi078 bash[75299]: cluster 2023-10-01T10:35:13.571270+0000 mgr.y (mgr.25072) 493 : cluster 0 pgmap v409: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 555 op/s; 6026/134823 objects degraded (4.470%); 0 B/s, 31 objects/s recovering 2023-10-01T10:35:14.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:13 smithi078 bash[78274]: cluster 2023-10-01T10:35:13.571270+0000 mgr.y (mgr.25072) 493 : cluster 0 pgmap v409: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 555 op/s; 6026/134823 objects degraded (4.470%); 0 B/s, 31 objects/s recovering 2023-10-01T10:35:14.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:13 smithi173 bash[58816]: cluster 2023-10-01T10:35:13.571270+0000 mgr.y (mgr.25072) 493 : cluster 0 pgmap v409: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 555 op/s; 6026/134823 objects degraded (4.470%); 0 B/s, 31 objects/s recovering 2023-10-01T10:35:15.087 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:14 smithi078 bash[78274]: audit 2023-10-01T10:35:14.371672+0000 mon.a (mon.0) 668 : audit 0 from='client.? 172.21.15.78:0/592970124' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:15.087 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:14 smithi078 bash[75299]: audit 2023-10-01T10:35:14.371672+0000 mon.a (mon.0) 668 : audit 0 from='client.? 172.21.15.78:0/592970124' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:15.122 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:14 smithi173 bash[58816]: audit 2023-10-01T10:35:14.371672+0000 mon.a (mon.0) 668 : audit 0 from='client.? 172.21.15.78:0/592970124' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:16.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:15 smithi078 bash[75299]: cluster 2023-10-01T10:35:15.572124+0000 mgr.y (mgr.25072) 494 : cluster 0 pgmap v410: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 835 op/s; 5833/124758 objects degraded (4.675%); 0 B/s, 47 objects/s recovering 2023-10-01T10:35:16.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:15 smithi078 bash[78274]: cluster 2023-10-01T10:35:15.572124+0000 mgr.y (mgr.25072) 494 : cluster 0 pgmap v410: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 835 op/s; 5833/124758 objects degraded (4.675%); 0 B/s, 47 objects/s recovering 2023-10-01T10:35:16.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:15 smithi173 bash[58816]: cluster 2023-10-01T10:35:15.572124+0000 mgr.y (mgr.25072) 494 : cluster 0 pgmap v410: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 835 op/s; 5833/124758 objects degraded (4.675%); 0 B/s, 47 objects/s recovering 2023-10-01T10:35:17.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:16 smithi078 bash[75299]: cluster 2023-10-01T10:35:16.911193+0000 mon.a (mon.0) 669 : cluster 3 Health check update: Degraded data redundancy: 5833/124758 objects degraded (4.675%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T10:35:17.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:16 smithi078 bash[75299]: audit 2023-10-01T10:35:16.957912+0000 mon.a (mon.0) 670 : audit 0 from='client.? 172.21.15.78:0/3145575591' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:17.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:16 smithi078 bash[78274]: cluster 2023-10-01T10:35:16.911193+0000 mon.a (mon.0) 669 : cluster 3 Health check update: Degraded data redundancy: 5833/124758 objects degraded (4.675%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T10:35:17.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:16 smithi078 bash[78274]: audit 2023-10-01T10:35:16.957912+0000 mon.a (mon.0) 670 : audit 0 from='client.? 172.21.15.78:0/3145575591' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:17.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:16 smithi173 bash[58816]: cluster 2023-10-01T10:35:16.911193+0000 mon.a (mon.0) 669 : cluster 3 Health check update: Degraded data redundancy: 5833/124758 objects degraded (4.675%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T10:35:17.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:16 smithi173 bash[58816]: audit 2023-10-01T10:35:16.957912+0000 mon.a (mon.0) 670 : audit 0 from='client.? 172.21.15.78:0/3145575591' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:18.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:17 smithi078 bash[75299]: audit 2023-10-01T10:35:17.404078+0000 mon.a (mon.0) 671 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:35:18.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:17 smithi078 bash[75299]: cluster 2023-10-01T10:35:17.572696+0000 mgr.y (mgr.25072) 495 : cluster 0 pgmap v411: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 556 op/s; 5833/124758 objects degraded (4.675%); 0 B/s, 32 objects/s recovering 2023-10-01T10:35:18.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:17 smithi078 bash[78274]: audit 2023-10-01T10:35:17.404078+0000 mon.a (mon.0) 671 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:35:18.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:17 smithi078 bash[78274]: cluster 2023-10-01T10:35:17.572696+0000 mgr.y (mgr.25072) 495 : cluster 0 pgmap v411: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 556 op/s; 5833/124758 objects degraded (4.675%); 0 B/s, 32 objects/s recovering 2023-10-01T10:35:18.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:17 smithi173 bash[58816]: audit 2023-10-01T10:35:17.404078+0000 mon.a (mon.0) 671 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:35:18.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:17 smithi173 bash[58816]: cluster 2023-10-01T10:35:17.572696+0000 mgr.y (mgr.25072) 495 : cluster 0 pgmap v411: 89 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 556 op/s; 5833/124758 objects degraded (4.675%); 0 B/s, 32 objects/s recovering 2023-10-01T10:35:20.544 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:35:20 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:35:20] "GET /metrics HTTP/1.1" 200 36192 "" "Prometheus/2.33.4" 2023-10-01T10:35:20.546 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:20 smithi078 bash[78274]: cluster 2023-10-01T10:35:19.537571+0000 osd.3 (osd.3) 32 : cluster 3 failed to encode map e110 with expected crc 2023-10-01T10:35:20.546 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:20 smithi078 bash[78274]: cluster 2023-10-01T10:35:19.540225+0000 mon.a (mon.0) 672 : cluster 0 osdmap e110: 8 total, 8 up, 8 in 2023-10-01T10:35:20.546 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:20 smithi078 bash[78274]: cluster 2023-10-01T10:35:19.542782+0000 osd.7 (osd.7) 41 : cluster 3 failed to encode map e110 with expected crc 2023-10-01T10:35:20.546 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:20 smithi078 bash[78274]: cluster 2023-10-01T10:35:19.553033+0000 osd.6 (osd.6) 22 : cluster 3 failed to encode map e110 with expected crc 2023-10-01T10:35:20.546 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:20 smithi078 bash[78274]: cluster 2023-10-01T10:35:19.554472+0000 osd.2 (osd.2) 30 : cluster 3 failed to encode map e110 with expected crc 2023-10-01T10:35:20.546 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:20 smithi078 bash[78274]: cluster 2023-10-01T10:35:19.573211+0000 mgr.y (mgr.25072) 496 : cluster 0 pgmap v413: 89 pgs: 1 active+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 5624/118353 objects degraded (4.752%); 0 B/s, 40 objects/s recovering 2023-10-01T10:35:20.546 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:20 smithi078 bash[78274]: audit 2023-10-01T10:35:19.574879+0000 mon.a (mon.0) 673 : audit 0 from='client.? 172.21.15.78:0/3124999261' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:20.547 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:20 smithi078 bash[75299]: cluster 2023-10-01T10:35:19.537571+0000 osd.3 (osd.3) 32 : cluster 3 failed to encode map e110 with expected crc 2023-10-01T10:35:20.547 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:20 smithi078 bash[75299]: cluster 2023-10-01T10:35:19.540225+0000 mon.a (mon.0) 672 : cluster 0 osdmap e110: 8 total, 8 up, 8 in 2023-10-01T10:35:20.547 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:20 smithi078 bash[75299]: cluster 2023-10-01T10:35:19.542782+0000 osd.7 (osd.7) 41 : cluster 3 failed to encode map e110 with expected crc 2023-10-01T10:35:20.547 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:20 smithi078 bash[75299]: cluster 2023-10-01T10:35:19.553033+0000 osd.6 (osd.6) 22 : cluster 3 failed to encode map e110 with expected crc 2023-10-01T10:35:20.547 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:20 smithi078 bash[75299]: cluster 2023-10-01T10:35:19.554472+0000 osd.2 (osd.2) 30 : cluster 3 failed to encode map e110 with expected crc 2023-10-01T10:35:20.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:20 smithi078 bash[75299]: cluster 2023-10-01T10:35:19.573211+0000 mgr.y (mgr.25072) 496 : cluster 0 pgmap v413: 89 pgs: 1 active+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 5624/118353 objects degraded (4.752%); 0 B/s, 40 objects/s recovering 2023-10-01T10:35:20.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:20 smithi078 bash[75299]: audit 2023-10-01T10:35:19.574879+0000 mon.a (mon.0) 673 : audit 0 from='client.? 172.21.15.78:0/3124999261' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:20.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:20 smithi173 bash[58816]: cluster 2023-10-01T10:35:19.537571+0000 osd.3 (osd.3) 32 : cluster 3 failed to encode map e110 with expected crc 2023-10-01T10:35:20.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:20 smithi173 bash[58816]: cluster 2023-10-01T10:35:19.540225+0000 mon.a (mon.0) 672 : cluster 0 osdmap e110: 8 total, 8 up, 8 in 2023-10-01T10:35:20.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:20 smithi173 bash[58816]: cluster 2023-10-01T10:35:19.542782+0000 osd.7 (osd.7) 41 : cluster 3 failed to encode map e110 with expected crc 2023-10-01T10:35:20.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:20 smithi173 bash[58816]: cluster 2023-10-01T10:35:19.553033+0000 osd.6 (osd.6) 22 : cluster 3 failed to encode map e110 with expected crc 2023-10-01T10:35:20.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:20 smithi173 bash[58816]: cluster 2023-10-01T10:35:19.554472+0000 osd.2 (osd.2) 30 : cluster 3 failed to encode map e110 with expected crc 2023-10-01T10:35:20.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:20 smithi173 bash[58816]: cluster 2023-10-01T10:35:19.573211+0000 mgr.y (mgr.25072) 496 : cluster 0 pgmap v413: 89 pgs: 1 active+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 5624/118353 objects degraded (4.752%); 0 B/s, 40 objects/s recovering 2023-10-01T10:35:20.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:20 smithi173 bash[58816]: audit 2023-10-01T10:35:19.574879+0000 mon.a (mon.0) 673 : audit 0 from='client.? 172.21.15.78:0/3124999261' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:21.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:21 smithi078 bash[75299]: cluster 2023-10-01T10:35:20.544272+0000 osd.5 (osd.5) 28 : cluster 3 failed to encode map e111 with expected crc 2023-10-01T10:35:21.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:21 smithi078 bash[75299]: cluster 2023-10-01T10:35:20.544642+0000 osd.7 (osd.7) 42 : cluster 3 failed to encode map e111 with expected crc 2023-10-01T10:35:21.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:21 smithi078 bash[75299]: cluster 2023-10-01T10:35:20.546885+0000 osd.2 (osd.2) 31 : cluster 3 failed to encode map e111 with expected crc 2023-10-01T10:35:21.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:21 smithi078 bash[75299]: cluster 2023-10-01T10:35:20.547107+0000 mon.a (mon.0) 674 : cluster 0 osdmap e111: 8 total, 8 up, 8 in 2023-10-01T10:35:21.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:21 smithi078 bash[75299]: cluster 2023-10-01T10:35:20.584739+0000 osd.3 (osd.3) 33 : cluster 3 failed to encode map e111 with expected crc 2023-10-01T10:35:21.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:21 smithi078 bash[75299]: cluster 2023-10-01T10:35:20.607173+0000 osd.6 (osd.6) 23 : cluster 3 failed to encode map e111 with expected crc 2023-10-01T10:35:21.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:21 smithi078 bash[78274]: cluster 2023-10-01T10:35:20.544272+0000 osd.5 (osd.5) 28 : cluster 3 failed to encode map e111 with expected crc 2023-10-01T10:35:21.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:21 smithi078 bash[78274]: cluster 2023-10-01T10:35:20.544642+0000 osd.7 (osd.7) 42 : cluster 3 failed to encode map e111 with expected crc 2023-10-01T10:35:21.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:21 smithi078 bash[78274]: cluster 2023-10-01T10:35:20.546885+0000 osd.2 (osd.2) 31 : cluster 3 failed to encode map e111 with expected crc 2023-10-01T10:35:21.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:21 smithi078 bash[78274]: cluster 2023-10-01T10:35:20.547107+0000 mon.a (mon.0) 674 : cluster 0 osdmap e111: 8 total, 8 up, 8 in 2023-10-01T10:35:21.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:21 smithi078 bash[78274]: cluster 2023-10-01T10:35:20.584739+0000 osd.3 (osd.3) 33 : cluster 3 failed to encode map e111 with expected crc 2023-10-01T10:35:21.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:21 smithi078 bash[78274]: cluster 2023-10-01T10:35:20.607173+0000 osd.6 (osd.6) 23 : cluster 3 failed to encode map e111 with expected crc 2023-10-01T10:35:21.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:21 smithi173 bash[58816]: cluster 2023-10-01T10:35:20.544272+0000 osd.5 (osd.5) 28 : cluster 3 failed to encode map e111 with expected crc 2023-10-01T10:35:21.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:21 smithi173 bash[58816]: cluster 2023-10-01T10:35:20.544642+0000 osd.7 (osd.7) 42 : cluster 3 failed to encode map e111 with expected crc 2023-10-01T10:35:21.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:21 smithi173 bash[58816]: cluster 2023-10-01T10:35:20.546885+0000 osd.2 (osd.2) 31 : cluster 3 failed to encode map e111 with expected crc 2023-10-01T10:35:21.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:21 smithi173 bash[58816]: cluster 2023-10-01T10:35:20.547107+0000 mon.a (mon.0) 674 : cluster 0 osdmap e111: 8 total, 8 up, 8 in 2023-10-01T10:35:21.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:21 smithi173 bash[58816]: cluster 2023-10-01T10:35:20.584739+0000 osd.3 (osd.3) 33 : cluster 3 failed to encode map e111 with expected crc 2023-10-01T10:35:21.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:21 smithi173 bash[58816]: cluster 2023-10-01T10:35:20.607173+0000 osd.6 (osd.6) 23 : cluster 3 failed to encode map e111 with expected crc 2023-10-01T10:35:22.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:22 smithi078 bash[78274]: cluster 2023-10-01T10:35:21.573716+0000 mgr.y (mgr.25072) 497 : cluster 0 pgmap v415: 89 pgs: 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 838 op/s; 5607/114690 objects degraded (4.889%); 0 B/s, 52 objects/s recovering 2023-10-01T10:35:22.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:22 smithi078 bash[78274]: cluster 2023-10-01T10:35:21.912398+0000 mon.a (mon.0) 675 : cluster 3 Health check update: Degraded data redundancy: 5624/118353 objects degraded (4.752%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T10:35:22.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:22 smithi078 bash[78274]: audit 2023-10-01T10:35:22.135409+0000 mon.c (mon.2) 92 : audit 0 from='client.? 172.21.15.78:0/4248998453' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:22.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:22 smithi078 bash[75299]: cluster 2023-10-01T10:35:21.573716+0000 mgr.y (mgr.25072) 497 : cluster 0 pgmap v415: 89 pgs: 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 838 op/s; 5607/114690 objects degraded (4.889%); 0 B/s, 52 objects/s recovering 2023-10-01T10:35:22.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:22 smithi078 bash[75299]: cluster 2023-10-01T10:35:21.912398+0000 mon.a (mon.0) 675 : cluster 3 Health check update: Degraded data redundancy: 5624/118353 objects degraded (4.752%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T10:35:22.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:22 smithi078 bash[75299]: audit 2023-10-01T10:35:22.135409+0000 mon.c (mon.2) 92 : audit 0 from='client.? 172.21.15.78:0/4248998453' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:22.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:22 smithi173 bash[58816]: cluster 2023-10-01T10:35:21.573716+0000 mgr.y (mgr.25072) 497 : cluster 0 pgmap v415: 89 pgs: 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 838 op/s; 5607/114690 objects degraded (4.889%); 0 B/s, 52 objects/s recovering 2023-10-01T10:35:22.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:22 smithi173 bash[58816]: cluster 2023-10-01T10:35:21.912398+0000 mon.a (mon.0) 675 : cluster 3 Health check update: Degraded data redundancy: 5624/118353 objects degraded (4.752%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T10:35:22.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:22 smithi173 bash[58816]: audit 2023-10-01T10:35:22.135409+0000 mon.c (mon.2) 92 : audit 0 from='client.? 172.21.15.78:0/4248998453' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:24.333 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:35:23 smithi078 bash[71214]: debug 2023-10-01T10:35:23.881+0000 7fd9c245e700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:35:25.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:24 smithi078 bash[75299]: cluster 2023-10-01T10:35:23.574281+0000 mgr.y (mgr.25072) 498 : cluster 0 pgmap v416: 89 pgs: 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 419 op/s; 5607/114690 objects degraded (4.889%); 0 B/s, 28 objects/s recovering 2023-10-01T10:35:25.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:24 smithi078 bash[75299]: audit 2023-10-01T10:35:23.885857+0000 mon.a (mon.0) 676 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:35:25.085 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:24 smithi078 bash[78274]: cluster 2023-10-01T10:35:23.574281+0000 mgr.y (mgr.25072) 498 : cluster 0 pgmap v416: 89 pgs: 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 419 op/s; 5607/114690 objects degraded (4.889%); 0 B/s, 28 objects/s recovering 2023-10-01T10:35:25.085 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:24 smithi078 bash[78274]: audit 2023-10-01T10:35:23.885857+0000 mon.a (mon.0) 676 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:35:25.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:24 smithi173 bash[58816]: cluster 2023-10-01T10:35:23.574281+0000 mgr.y (mgr.25072) 498 : cluster 0 pgmap v416: 89 pgs: 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 419 op/s; 5607/114690 objects degraded (4.889%); 0 B/s, 28 objects/s recovering 2023-10-01T10:35:25.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:24 smithi173 bash[58816]: audit 2023-10-01T10:35:23.885857+0000 mon.a (mon.0) 676 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:35:26.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:25 smithi078 bash[75299]: audit 2023-10-01T10:35:23.886113+0000 mgr.y (mgr.25072) 499 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:35:26.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:25 smithi078 bash[75299]: cephadm 2023-10-01T10:35:23.887712+0000 mgr.y (mgr.25072) 500 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:35:26.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:25 smithi078 bash[75299]: audit 2023-10-01T10:35:24.723514+0000 mon.a (mon.0) 677 : audit 0 from='client.? 172.21.15.78:0/1044092075' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:26.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:25 smithi078 bash[78274]: audit 2023-10-01T10:35:23.886113+0000 mgr.y (mgr.25072) 499 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:35:26.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:25 smithi078 bash[78274]: cephadm 2023-10-01T10:35:23.887712+0000 mgr.y (mgr.25072) 500 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:35:26.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:25 smithi078 bash[78274]: audit 2023-10-01T10:35:24.723514+0000 mon.a (mon.0) 677 : audit 0 from='client.? 172.21.15.78:0/1044092075' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:26.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:25 smithi173 bash[58816]: audit 2023-10-01T10:35:23.886113+0000 mgr.y (mgr.25072) 499 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:35:26.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:25 smithi173 bash[58816]: cephadm 2023-10-01T10:35:23.887712+0000 mgr.y (mgr.25072) 500 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2023-10-01T10:35:26.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:25 smithi173 bash[58816]: audit 2023-10-01T10:35:24.723514+0000 mon.a (mon.0) 677 : audit 0 from='client.? 172.21.15.78:0/1044092075' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:27.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:26 smithi078 bash[75299]: cluster 2023-10-01T10:35:25.575158+0000 mgr.y (mgr.25072) 501 : cluster 0 pgmap v417: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.2 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 5501/105741 objects degraded (5.202%); 0 B/s, 41 objects/s recovering 2023-10-01T10:35:27.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:26 smithi078 bash[78274]: cluster 2023-10-01T10:35:25.575158+0000 mgr.y (mgr.25072) 501 : cluster 0 pgmap v417: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.2 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 5501/105741 objects degraded (5.202%); 0 B/s, 41 objects/s recovering 2023-10-01T10:35:27.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:26 smithi173 bash[58816]: cluster 2023-10-01T10:35:25.575158+0000 mgr.y (mgr.25072) 501 : cluster 0 pgmap v417: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.2 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 5501/105741 objects degraded (5.202%); 0 B/s, 41 objects/s recovering 2023-10-01T10:35:28.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:27 smithi078 bash[75299]: cluster 2023-10-01T10:35:26.913610+0000 mon.a (mon.0) 678 : cluster 3 Health check update: Degraded data redundancy: 5501/105741 objects degraded (5.202%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T10:35:28.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:27 smithi078 bash[75299]: audit 2023-10-01T10:35:27.316603+0000 mon.c (mon.2) 93 : audit 0 from='client.? 172.21.15.78:0/3306301092' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:28.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:27 smithi078 bash[78274]: cluster 2023-10-01T10:35:26.913610+0000 mon.a (mon.0) 678 : cluster 3 Health check update: Degraded data redundancy: 5501/105741 objects degraded (5.202%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T10:35:28.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:27 smithi078 bash[78274]: audit 2023-10-01T10:35:27.316603+0000 mon.c (mon.2) 93 : audit 0 from='client.? 172.21.15.78:0/3306301092' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:28.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:27 smithi173 bash[58816]: cluster 2023-10-01T10:35:26.913610+0000 mon.a (mon.0) 678 : cluster 3 Health check update: Degraded data redundancy: 5501/105741 objects degraded (5.202%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T10:35:28.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:27 smithi173 bash[58816]: audit 2023-10-01T10:35:27.316603+0000 mon.c (mon.2) 93 : audit 0 from='client.? 172.21.15.78:0/3306301092' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:29.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:28 smithi078 bash[75299]: cluster 2023-10-01T10:35:27.575677+0000 mgr.y (mgr.25072) 502 : cluster 0 pgmap v418: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.2 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 523 op/s; 5501/105741 objects degraded (5.202%); 0 B/s, 15 objects/s recovering 2023-10-01T10:35:29.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:28 smithi078 bash[78274]: cluster 2023-10-01T10:35:27.575677+0000 mgr.y (mgr.25072) 502 : cluster 0 pgmap v418: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.2 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 523 op/s; 5501/105741 objects degraded (5.202%); 0 B/s, 15 objects/s recovering 2023-10-01T10:35:29.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:28 smithi173 bash[58816]: cluster 2023-10-01T10:35:27.575677+0000 mgr.y (mgr.25072) 502 : cluster 0 pgmap v418: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.2 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 523 op/s; 5501/105741 objects degraded (5.202%); 0 B/s, 15 objects/s recovering 2023-10-01T10:35:30.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:29 smithi078 bash[78274]: cluster 2023-10-01T10:35:29.576388+0000 mgr.y (mgr.25072) 503 : cluster 0 pgmap v419: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.1 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 545 op/s; 5501/101979 objects degraded (5.394%); 0 B/s, 12 objects/s recovering 2023-10-01T10:35:30.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:29 smithi078 bash[75299]: cluster 2023-10-01T10:35:29.576388+0000 mgr.y (mgr.25072) 503 : cluster 0 pgmap v419: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.1 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 545 op/s; 5501/101979 objects degraded (5.394%); 0 B/s, 12 objects/s recovering 2023-10-01T10:35:30.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:29 smithi173 bash[58816]: cluster 2023-10-01T10:35:29.576388+0000 mgr.y (mgr.25072) 503 : cluster 0 pgmap v419: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 2.1 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 545 op/s; 5501/101979 objects degraded (5.394%); 0 B/s, 12 objects/s recovering 2023-10-01T10:35:30.582 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:35:30 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:35:30] "GET /metrics HTTP/1.1" 200 36192 "" "Prometheus/2.33.4" 2023-10-01T10:35:31.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:30 smithi078 bash[75299]: audit 2023-10-01T10:35:29.955813+0000 mon.a (mon.0) 679 : audit 0 from='client.? 172.21.15.78:0/821665827' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:31.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:30 smithi078 bash[78274]: audit 2023-10-01T10:35:29.955813+0000 mon.a (mon.0) 679 : audit 0 from='client.? 172.21.15.78:0/821665827' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:31.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:30 smithi173 bash[58816]: audit 2023-10-01T10:35:29.955813+0000 mon.a (mon.0) 679 : audit 0 from='client.? 172.21.15.78:0/821665827' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:32.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:31 smithi078 bash[75299]: cluster 2023-10-01T10:35:31.577087+0000 mgr.y (mgr.25072) 504 : cluster 0 pgmap v420: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.9 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 574 op/s; 5394/95685 objects degraded (5.637%); 0 B/s, 19 objects/s recovering 2023-10-01T10:35:32.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:31 smithi078 bash[78274]: cluster 2023-10-01T10:35:31.577087+0000 mgr.y (mgr.25072) 504 : cluster 0 pgmap v420: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.9 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 574 op/s; 5394/95685 objects degraded (5.637%); 0 B/s, 19 objects/s recovering 2023-10-01T10:35:32.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:31 smithi173 bash[58816]: cluster 2023-10-01T10:35:31.577087+0000 mgr.y (mgr.25072) 504 : cluster 0 pgmap v420: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.9 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 574 op/s; 5394/95685 objects degraded (5.637%); 0 B/s, 19 objects/s recovering 2023-10-01T10:35:33.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:32 smithi078 bash[75299]: cluster 2023-10-01T10:35:31.914733+0000 mon.a (mon.0) 680 : cluster 3 Health check update: Degraded data redundancy: 5394/95685 objects degraded (5.637%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T10:35:33.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:32 smithi078 bash[75299]: audit 2023-10-01T10:35:32.401612+0000 mon.a (mon.0) 681 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:35:33.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:32 smithi078 bash[75299]: audit 2023-10-01T10:35:32.566334+0000 mon.c (mon.2) 94 : audit 0 from='client.? 172.21.15.78:0/461672204' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:33.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:32 smithi078 bash[78274]: cluster 2023-10-01T10:35:31.914733+0000 mon.a (mon.0) 680 : cluster 3 Health check update: Degraded data redundancy: 5394/95685 objects degraded (5.637%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T10:35:33.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:32 smithi078 bash[78274]: audit 2023-10-01T10:35:32.401612+0000 mon.a (mon.0) 681 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:35:33.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:32 smithi078 bash[78274]: audit 2023-10-01T10:35:32.566334+0000 mon.c (mon.2) 94 : audit 0 from='client.? 172.21.15.78:0/461672204' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:33.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:32 smithi173 bash[58816]: cluster 2023-10-01T10:35:31.914733+0000 mon.a (mon.0) 680 : cluster 3 Health check update: Degraded data redundancy: 5394/95685 objects degraded (5.637%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T10:35:33.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:32 smithi173 bash[58816]: audit 2023-10-01T10:35:32.401612+0000 mon.a (mon.0) 681 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:35:33.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:32 smithi173 bash[58816]: audit 2023-10-01T10:35:32.566334+0000 mon.c (mon.2) 94 : audit 0 from='client.? 172.21.15.78:0/461672204' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:34.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:33 smithi078 bash[75299]: cluster 2023-10-01T10:35:33.577682+0000 mgr.y (mgr.25072) 505 : cluster 0 pgmap v421: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.9 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 527 op/s; 5394/95685 objects degraded (5.637%); 0 B/s, 17 objects/s recovering 2023-10-01T10:35:34.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:33 smithi078 bash[78274]: cluster 2023-10-01T10:35:33.577682+0000 mgr.y (mgr.25072) 505 : cluster 0 pgmap v421: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.9 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 527 op/s; 5394/95685 objects degraded (5.637%); 0 B/s, 17 objects/s recovering 2023-10-01T10:35:34.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:33 smithi173 bash[58816]: cluster 2023-10-01T10:35:33.577682+0000 mgr.y (mgr.25072) 505 : cluster 0 pgmap v421: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.9 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 527 op/s; 5394/95685 objects degraded (5.637%); 0 B/s, 17 objects/s recovering 2023-10-01T10:35:35.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:35 smithi078 bash[75299]: audit 2023-10-01T10:35:35.172914+0000 mon.a (mon.0) 682 : audit 0 from='client.? 172.21.15.78:0/3779138080' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:35.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:35 smithi078 bash[78274]: audit 2023-10-01T10:35:35.172914+0000 mon.a (mon.0) 682 : audit 0 from='client.? 172.21.15.78:0/3779138080' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:35.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:35 smithi173 bash[58816]: audit 2023-10-01T10:35:35.172914+0000 mon.a (mon.0) 682 : audit 0 from='client.? 172.21.15.78:0/3779138080' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:36.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:36 smithi078 bash[75299]: cluster 2023-10-01T10:35:35.578361+0000 mgr.y (mgr.25072) 506 : cluster 0 pgmap v422: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.7 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 807 op/s; 5291/85608 objects degraded (6.180%); 0 B/s, 26 objects/s recovering 2023-10-01T10:35:36.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:36 smithi078 bash[78274]: cluster 2023-10-01T10:35:35.578361+0000 mgr.y (mgr.25072) 506 : cluster 0 pgmap v422: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.7 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 807 op/s; 5291/85608 objects degraded (6.180%); 0 B/s, 26 objects/s recovering 2023-10-01T10:35:36.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:36 smithi173 bash[58816]: cluster 2023-10-01T10:35:35.578361+0000 mgr.y (mgr.25072) 506 : cluster 0 pgmap v422: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.7 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 807 op/s; 5291/85608 objects degraded (6.180%); 0 B/s, 26 objects/s recovering 2023-10-01T10:35:37.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:37 smithi078 bash[75299]: cluster 2023-10-01T10:35:36.917585+0000 mon.a (mon.0) 683 : cluster 3 Health check update: Degraded data redundancy: 5291/85608 objects degraded (6.180%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T10:35:37.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:37 smithi078 bash[78274]: cluster 2023-10-01T10:35:36.917585+0000 mon.a (mon.0) 683 : cluster 3 Health check update: Degraded data redundancy: 5291/85608 objects degraded (6.180%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T10:35:37.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:37 smithi173 bash[58816]: cluster 2023-10-01T10:35:36.917585+0000 mon.a (mon.0) 683 : cluster 3 Health check update: Degraded data redundancy: 5291/85608 objects degraded (6.180%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T10:35:38.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:38 smithi078 bash[75299]: cluster 2023-10-01T10:35:37.578864+0000 mgr.y (mgr.25072) 507 : cluster 0 pgmap v423: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.7 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 559 op/s; 5291/85608 objects degraded (6.180%); 0 B/s, 17 objects/s recovering 2023-10-01T10:35:38.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:38 smithi078 bash[75299]: audit 2023-10-01T10:35:38.128626+0000 mon.b (mon.1) 33 : audit 0 from='client.? 172.21.15.78:0/1093733903' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:38.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:38 smithi078 bash[78274]: cluster 2023-10-01T10:35:37.578864+0000 mgr.y (mgr.25072) 507 : cluster 0 pgmap v423: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.7 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 559 op/s; 5291/85608 objects degraded (6.180%); 0 B/s, 17 objects/s recovering 2023-10-01T10:35:38.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:38 smithi078 bash[78274]: audit 2023-10-01T10:35:38.128626+0000 mon.b (mon.1) 33 : audit 0 from='client.? 172.21.15.78:0/1093733903' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:38.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:38 smithi173 bash[58816]: cluster 2023-10-01T10:35:37.578864+0000 mgr.y (mgr.25072) 507 : cluster 0 pgmap v423: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.7 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 559 op/s; 5291/85608 objects degraded (6.180%); 0 B/s, 17 objects/s recovering 2023-10-01T10:35:38.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:38 smithi173 bash[58816]: audit 2023-10-01T10:35:38.128626+0000 mon.b (mon.1) 33 : audit 0 from='client.? 172.21.15.78:0/1093733903' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:39.269 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:39 smithi078 bash[75299]: audit 2023-10-01T10:35:38.901827+0000 mon.a (mon.0) 684 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:35:39.270 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:35:38 smithi078 bash[71214]: debug 2023-10-01T10:35:38.897+0000 7fd9c245e700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-01T10:35:39.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:39 smithi078 bash[78274]: audit 2023-10-01T10:35:38.901827+0000 mon.a (mon.0) 684 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:35:39.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:39 smithi173 bash[58816]: audit 2023-10-01T10:35:38.901827+0000 mon.a (mon.0) 684 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:35:40.605 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:40 smithi078 bash[78274]: audit 2023-10-01T10:35:38.902078+0000 mgr.y (mgr.25072) 508 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:35:40.605 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:40 smithi078 bash[78274]: cephadm 2023-10-01T10:35:38.903427+0000 mgr.y (mgr.25072) 509 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-01T10:35:40.605 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:40 smithi078 bash[78274]: cluster 2023-10-01T10:35:39.579564+0000 mgr.y (mgr.25072) 510 : cluster 0 pgmap v424: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.7 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 5291/81756 objects degraded (6.472%); 0 B/s, 17 objects/s recovering 2023-10-01T10:35:40.606 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:40 smithi078 bash[75299]: audit 2023-10-01T10:35:38.902078+0000 mgr.y (mgr.25072) 508 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:35:40.606 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:40 smithi078 bash[75299]: cephadm 2023-10-01T10:35:38.903427+0000 mgr.y (mgr.25072) 509 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-01T10:35:40.606 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:40 smithi078 bash[75299]: cluster 2023-10-01T10:35:39.579564+0000 mgr.y (mgr.25072) 510 : cluster 0 pgmap v424: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.7 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 5291/81756 objects degraded (6.472%); 0 B/s, 17 objects/s recovering 2023-10-01T10:35:40.606 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:35:40 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:35:40] "GET /metrics HTTP/1.1" 200 36190 "" "Prometheus/2.33.4" 2023-10-01T10:35:40.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:40 smithi173 bash[58816]: audit 2023-10-01T10:35:38.902078+0000 mgr.y (mgr.25072) 508 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:35:40.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:40 smithi173 bash[58816]: cephadm 2023-10-01T10:35:38.903427+0000 mgr.y (mgr.25072) 509 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-01T10:35:40.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:40 smithi173 bash[58816]: cluster 2023-10-01T10:35:39.579564+0000 mgr.y (mgr.25072) 510 : cluster 0 pgmap v424: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.7 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 5291/81756 objects degraded (6.472%); 0 B/s, 17 objects/s recovering 2023-10-01T10:35:41.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:41 smithi078 bash[75299]: audit 2023-10-01T10:35:40.770824+0000 mon.b (mon.1) 34 : audit 0 from='client.? 172.21.15.78:0/3856401060' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:41.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:41 smithi078 bash[78274]: audit 2023-10-01T10:35:40.770824+0000 mon.b (mon.1) 34 : audit 0 from='client.? 172.21.15.78:0/3856401060' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:41.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:41 smithi173 bash[58816]: audit 2023-10-01T10:35:40.770824+0000 mon.b (mon.1) 34 : audit 0 from='client.? 172.21.15.78:0/3856401060' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:42.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:42 smithi078 bash[75299]: cluster 2023-10-01T10:35:41.580301+0000 mgr.y (mgr.25072) 511 : cluster 0 pgmap v425: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.5 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 5181/75498 objects degraded (6.862%); 0 B/s, 26 objects/s recovering 2023-10-01T10:35:42.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:42 smithi078 bash[75299]: cluster 2023-10-01T10:35:41.918768+0000 mon.a (mon.0) 685 : cluster 3 Health check update: Degraded data redundancy: 5291/81756 objects degraded (6.472%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T10:35:42.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:42 smithi078 bash[78274]: cluster 2023-10-01T10:35:41.580301+0000 mgr.y (mgr.25072) 511 : cluster 0 pgmap v425: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.5 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 5181/75498 objects degraded (6.862%); 0 B/s, 26 objects/s recovering 2023-10-01T10:35:42.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:42 smithi078 bash[78274]: cluster 2023-10-01T10:35:41.918768+0000 mon.a (mon.0) 685 : cluster 3 Health check update: Degraded data redundancy: 5291/81756 objects degraded (6.472%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T10:35:42.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:42 smithi173 bash[58816]: cluster 2023-10-01T10:35:41.580301+0000 mgr.y (mgr.25072) 511 : cluster 0 pgmap v425: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.5 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 5181/75498 objects degraded (6.862%); 0 B/s, 26 objects/s recovering 2023-10-01T10:35:42.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:42 smithi173 bash[58816]: cluster 2023-10-01T10:35:41.918768+0000 mon.a (mon.0) 685 : cluster 3 Health check update: Degraded data redundancy: 5291/81756 objects degraded (6.472%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T10:35:43.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:43 smithi078 bash[78274]: audit 2023-10-01T10:35:43.398957+0000 mon.c (mon.2) 95 : audit 0 from='client.? 172.21.15.78:0/2677017065' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:43.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:43 smithi078 bash[75299]: audit 2023-10-01T10:35:43.398957+0000 mon.c (mon.2) 95 : audit 0 from='client.? 172.21.15.78:0/2677017065' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:43.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:43 smithi173 bash[58816]: audit 2023-10-01T10:35:43.398957+0000 mon.c (mon.2) 95 : audit 0 from='client.? 172.21.15.78:0/2677017065' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:44.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:44 smithi078 bash[78274]: cluster 2023-10-01T10:35:43.580840+0000 mgr.y (mgr.25072) 512 : cluster 0 pgmap v426: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.5 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 5181/75498 objects degraded (6.862%); 0 B/s, 17 objects/s recovering 2023-10-01T10:35:44.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:44 smithi078 bash[75299]: cluster 2023-10-01T10:35:43.580840+0000 mgr.y (mgr.25072) 512 : cluster 0 pgmap v426: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.5 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 5181/75498 objects degraded (6.862%); 0 B/s, 17 objects/s recovering 2023-10-01T10:35:44.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:44 smithi173 bash[58816]: cluster 2023-10-01T10:35:43.580840+0000 mgr.y (mgr.25072) 512 : cluster 0 pgmap v426: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.5 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 5181/75498 objects degraded (6.862%); 0 B/s, 17 objects/s recovering 2023-10-01T10:35:46.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:45 smithi078 bash[75299]: cluster 2023-10-01T10:35:45.586065+0000 mgr.y (mgr.25072) 513 : cluster 0 pgmap v427: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.3 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 833 op/s; 5072/65670 objects degraded (7.723%); 0 B/s, 26 objects/s recovering 2023-10-01T10:35:46.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:45 smithi078 bash[78274]: cluster 2023-10-01T10:35:45.586065+0000 mgr.y (mgr.25072) 513 : cluster 0 pgmap v427: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.3 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 833 op/s; 5072/65670 objects degraded (7.723%); 0 B/s, 26 objects/s recovering 2023-10-01T10:35:46.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:45 smithi173 bash[58816]: cluster 2023-10-01T10:35:45.586065+0000 mgr.y (mgr.25072) 513 : cluster 0 pgmap v427: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.3 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 833 op/s; 5072/65670 objects degraded (7.723%); 0 B/s, 26 objects/s recovering 2023-10-01T10:35:47.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:46 smithi078 bash[75299]: audit 2023-10-01T10:35:46.014232+0000 mon.c (mon.2) 96 : audit 0 from='client.? 172.21.15.78:0/3612157214' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:47.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:46 smithi078 bash[78274]: audit 2023-10-01T10:35:46.014232+0000 mon.c (mon.2) 96 : audit 0 from='client.? 172.21.15.78:0/3612157214' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:47.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:46 smithi173 bash[58816]: audit 2023-10-01T10:35:46.014232+0000 mon.c (mon.2) 96 : audit 0 from='client.? 172.21.15.78:0/3612157214' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:48.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:47 smithi078 bash[78274]: cluster 2023-10-01T10:35:46.919896+0000 mon.a (mon.0) 686 : cluster 3 Health check update: Degraded data redundancy: 5072/65670 objects degraded (7.723%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T10:35:48.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:47 smithi078 bash[78274]: audit 2023-10-01T10:35:47.401646+0000 mon.a (mon.0) 687 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:35:48.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:47 smithi078 bash[78274]: cluster 2023-10-01T10:35:47.586644+0000 mgr.y (mgr.25072) 514 : cluster 0 pgmap v428: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.3 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 553 op/s; 5072/65670 objects degraded (7.723%); 0 B/s, 18 objects/s recovering 2023-10-01T10:35:48.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:47 smithi078 bash[75299]: cluster 2023-10-01T10:35:46.919896+0000 mon.a (mon.0) 686 : cluster 3 Health check update: Degraded data redundancy: 5072/65670 objects degraded (7.723%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T10:35:48.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:47 smithi078 bash[75299]: audit 2023-10-01T10:35:47.401646+0000 mon.a (mon.0) 687 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:35:48.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:47 smithi078 bash[75299]: cluster 2023-10-01T10:35:47.586644+0000 mgr.y (mgr.25072) 514 : cluster 0 pgmap v428: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.3 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 553 op/s; 5072/65670 objects degraded (7.723%); 0 B/s, 18 objects/s recovering 2023-10-01T10:35:48.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:47 smithi173 bash[58816]: cluster 2023-10-01T10:35:46.919896+0000 mon.a (mon.0) 686 : cluster 3 Health check update: Degraded data redundancy: 5072/65670 objects degraded (7.723%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T10:35:48.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:47 smithi173 bash[58816]: audit 2023-10-01T10:35:47.401646+0000 mon.a (mon.0) 687 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:35:48.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:47 smithi173 bash[58816]: cluster 2023-10-01T10:35:47.586644+0000 mgr.y (mgr.25072) 514 : cluster 0 pgmap v428: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.3 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 553 op/s; 5072/65670 objects degraded (7.723%); 0 B/s, 18 objects/s recovering 2023-10-01T10:35:49.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:48 smithi078 bash[75299]: audit 2023-10-01T10:35:48.624731+0000 mon.b (mon.1) 35 : audit 0 from='client.? 172.21.15.78:0/2978771795' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:49.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:48 smithi078 bash[78274]: audit 2023-10-01T10:35:48.624731+0000 mon.b (mon.1) 35 : audit 0 from='client.? 172.21.15.78:0/2978771795' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:49.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:48 smithi173 bash[58816]: audit 2023-10-01T10:35:48.624731+0000 mon.b (mon.1) 35 : audit 0 from='client.? 172.21.15.78:0/2978771795' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:50.272 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:49 smithi078 bash[75299]: cluster 2023-10-01T10:35:49.587371+0000 mgr.y (mgr.25072) 515 : cluster 0 pgmap v429: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.3 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 5072/61839 objects degraded (8.202%); 0 B/s, 18 objects/s recovering 2023-10-01T10:35:50.272 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:49 smithi078 bash[78274]: cluster 2023-10-01T10:35:49.587371+0000 mgr.y (mgr.25072) 515 : cluster 0 pgmap v429: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.3 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 5072/61839 objects degraded (8.202%); 0 B/s, 18 objects/s recovering 2023-10-01T10:35:50.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:49 smithi173 bash[58816]: cluster 2023-10-01T10:35:49.587371+0000 mgr.y (mgr.25072) 515 : cluster 0 pgmap v429: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.3 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 5072/61839 objects degraded (8.202%); 0 B/s, 18 objects/s recovering 2023-10-01T10:35:50.582 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:35:50 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:35:50] "GET /metrics HTTP/1.1" 200 36190 "" "Prometheus/2.33.4" 2023-10-01T10:35:51.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:51 smithi078 bash[78274]: audit 2023-10-01T10:35:51.229110+0000 mon.c (mon.2) 97 : audit 0 from='client.? 172.21.15.78:0/90205172' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:51.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:51 smithi078 bash[75299]: audit 2023-10-01T10:35:51.229110+0000 mon.c (mon.2) 97 : audit 0 from='client.? 172.21.15.78:0/90205172' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:51.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:51 smithi173 bash[58816]: audit 2023-10-01T10:35:51.229110+0000 mon.c (mon.2) 97 : audit 0 from='client.? 172.21.15.78:0/90205172' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:52.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:52 smithi078 bash[78274]: cluster 2023-10-01T10:35:51.588095+0000 mgr.y (mgr.25072) 516 : cluster 0 pgmap v430: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.1 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 4961/55548 objects degraded (8.931%); 0 B/s, 27 objects/s recovering 2023-10-01T10:35:52.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:52 smithi078 bash[78274]: cluster 2023-10-01T10:35:51.920791+0000 mon.a (mon.0) 688 : cluster 3 Health check update: Degraded data redundancy: 5072/61839 objects degraded (8.202%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T10:35:52.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:52 smithi078 bash[75299]: cluster 2023-10-01T10:35:51.588095+0000 mgr.y (mgr.25072) 516 : cluster 0 pgmap v430: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.1 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 4961/55548 objects degraded (8.931%); 0 B/s, 27 objects/s recovering 2023-10-01T10:35:52.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:52 smithi078 bash[75299]: cluster 2023-10-01T10:35:51.920791+0000 mon.a (mon.0) 688 : cluster 3 Health check update: Degraded data redundancy: 5072/61839 objects degraded (8.202%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T10:35:52.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:52 smithi173 bash[58816]: cluster 2023-10-01T10:35:51.588095+0000 mgr.y (mgr.25072) 516 : cluster 0 pgmap v430: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.1 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 4961/55548 objects degraded (8.931%); 0 B/s, 27 objects/s recovering 2023-10-01T10:35:52.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:52 smithi173 bash[58816]: cluster 2023-10-01T10:35:51.920791+0000 mon.a (mon.0) 688 : cluster 3 Health check update: Degraded data redundancy: 5072/61839 objects degraded (8.202%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T10:35:54.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:53 smithi078 bash[75299]: cluster 2023-10-01T10:35:53.588640+0000 mgr.y (mgr.25072) 517 : cluster 0 pgmap v431: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.1 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 553 op/s; 4961/55548 objects degraded (8.931%); 0 B/s, 18 objects/s recovering 2023-10-01T10:35:54.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:53 smithi078 bash[78274]: cluster 2023-10-01T10:35:53.588640+0000 mgr.y (mgr.25072) 517 : cluster 0 pgmap v431: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.1 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 553 op/s; 4961/55548 objects degraded (8.931%); 0 B/s, 18 objects/s recovering 2023-10-01T10:35:54.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:53 smithi173 bash[58816]: cluster 2023-10-01T10:35:53.588640+0000 mgr.y (mgr.25072) 517 : cluster 0 pgmap v431: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 1.1 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 553 op/s; 4961/55548 objects degraded (8.931%); 0 B/s, 18 objects/s recovering 2023-10-01T10:35:55.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:54 smithi078 bash[75299]: audit 2023-10-01T10:35:53.826457+0000 mon.c (mon.2) 98 : audit 0 from='client.? 172.21.15.78:0/4016548752' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:55.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:54 smithi078 bash[75299]: audit 2023-10-01T10:35:53.922009+0000 mon.a (mon.0) 689 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:35:55.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:54 smithi078 bash[78274]: audit 2023-10-01T10:35:53.826457+0000 mon.c (mon.2) 98 : audit 0 from='client.? 172.21.15.78:0/4016548752' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:55.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:54 smithi078 bash[78274]: audit 2023-10-01T10:35:53.922009+0000 mon.a (mon.0) 689 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:35:55.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:54 smithi173 bash[58816]: audit 2023-10-01T10:35:53.826457+0000 mon.c (mon.2) 98 : audit 0 from='client.? 172.21.15.78:0/4016548752' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:55.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:54 smithi173 bash[58816]: audit 2023-10-01T10:35:53.922009+0000 mon.a (mon.0) 689 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:35:56.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:55 smithi078 bash[75299]: audit 2023-10-01T10:35:55.000898+0000 mon.a (mon.0) 690 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:35:56.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:55 smithi078 bash[75299]: audit 2023-10-01T10:35:55.010093+0000 mon.a (mon.0) 691 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:35:56.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:55 smithi078 bash[75299]: cluster 2023-10-01T10:35:55.589471+0000 mgr.y (mgr.25072) 518 : cluster 0 pgmap v432: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 950 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 828 op/s; 4855/45642 objects degraded (10.637%); 0 B/s, 27 objects/s recovering 2023-10-01T10:35:56.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:55 smithi078 bash[78274]: audit 2023-10-01T10:35:55.000898+0000 mon.a (mon.0) 690 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:35:56.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:55 smithi078 bash[78274]: audit 2023-10-01T10:35:55.010093+0000 mon.a (mon.0) 691 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:35:56.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:55 smithi078 bash[78274]: cluster 2023-10-01T10:35:55.589471+0000 mgr.y (mgr.25072) 518 : cluster 0 pgmap v432: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 950 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 828 op/s; 4855/45642 objects degraded (10.637%); 0 B/s, 27 objects/s recovering 2023-10-01T10:35:56.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:55 smithi173 bash[58816]: audit 2023-10-01T10:35:55.000898+0000 mon.a (mon.0) 690 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:35:56.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:55 smithi173 bash[58816]: audit 2023-10-01T10:35:55.010093+0000 mon.a (mon.0) 691 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:35:56.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:55 smithi173 bash[58816]: cluster 2023-10-01T10:35:55.589471+0000 mgr.y (mgr.25072) 518 : cluster 0 pgmap v432: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 950 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 828 op/s; 4855/45642 objects degraded (10.637%); 0 B/s, 27 objects/s recovering 2023-10-01T10:35:57.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:57 smithi078 bash[75299]: audit 2023-10-01T10:35:56.460433+0000 mon.a (mon.0) 692 : audit 0 from='client.? 172.21.15.78:0/2470909603' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:57.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:57 smithi078 bash[75299]: cluster 2023-10-01T10:35:56.921861+0000 mon.a (mon.0) 693 : cluster 3 Health check update: Degraded data redundancy: 4855/45642 objects degraded (10.637%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T10:35:57.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:57 smithi078 bash[78274]: audit 2023-10-01T10:35:56.460433+0000 mon.a (mon.0) 692 : audit 0 from='client.? 172.21.15.78:0/2470909603' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:57.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:57 smithi078 bash[78274]: cluster 2023-10-01T10:35:56.921861+0000 mon.a (mon.0) 693 : cluster 3 Health check update: Degraded data redundancy: 4855/45642 objects degraded (10.637%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T10:35:57.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:57 smithi173 bash[58816]: audit 2023-10-01T10:35:56.460433+0000 mon.a (mon.0) 692 : audit 0 from='client.? 172.21.15.78:0/2470909603' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:57.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:57 smithi173 bash[58816]: cluster 2023-10-01T10:35:56.921861+0000 mon.a (mon.0) 693 : cluster 3 Health check update: Degraded data redundancy: 4855/45642 objects degraded (10.637%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T10:35:58.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:58 smithi078 bash[75299]: cluster 2023-10-01T10:35:57.590021+0000 mgr.y (mgr.25072) 519 : cluster 0 pgmap v433: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 950 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 556 op/s; 4855/45642 objects degraded (10.637%); 0 B/s, 18 objects/s recovering 2023-10-01T10:35:58.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:58 smithi078 bash[78274]: cluster 2023-10-01T10:35:57.590021+0000 mgr.y (mgr.25072) 519 : cluster 0 pgmap v433: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 950 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 556 op/s; 4855/45642 objects degraded (10.637%); 0 B/s, 18 objects/s recovering 2023-10-01T10:35:58.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:58 smithi173 bash[58816]: cluster 2023-10-01T10:35:57.590021+0000 mgr.y (mgr.25072) 519 : cluster 0 pgmap v433: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 950 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 556 op/s; 4855/45642 objects degraded (10.637%); 0 B/s, 18 objects/s recovering 2023-10-01T10:35:59.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:35:59 smithi078 bash[75299]: audit 2023-10-01T10:35:59.079512+0000 mon.b (mon.1) 36 : audit 0 from='client.? 172.21.15.78:0/3027955789' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:59.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:35:59 smithi078 bash[78274]: audit 2023-10-01T10:35:59.079512+0000 mon.b (mon.1) 36 : audit 0 from='client.? 172.21.15.78:0/3027955789' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:35:59.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:35:59 smithi173 bash[58816]: audit 2023-10-01T10:35:59.079512+0000 mon.b (mon.1) 36 : audit 0 from='client.? 172.21.15.78:0/3027955789' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:00.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:00 smithi078 bash[75299]: cluster 2023-10-01T10:35:59.590889+0000 mgr.y (mgr.25072) 520 : cluster 0 pgmap v434: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 870 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 4855/41787 objects degraded (11.618%); 0 B/s, 18 objects/s recovering 2023-10-01T10:36:00.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:36:00 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:36:00] "GET /metrics HTTP/1.1" 200 36190 "" "Prometheus/2.33.4" 2023-10-01T10:36:00.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:00 smithi078 bash[78274]: cluster 2023-10-01T10:35:59.590889+0000 mgr.y (mgr.25072) 520 : cluster 0 pgmap v434: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 870 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 4855/41787 objects degraded (11.618%); 0 B/s, 18 objects/s recovering 2023-10-01T10:36:00.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:00 smithi173 bash[58816]: cluster 2023-10-01T10:35:59.590889+0000 mgr.y (mgr.25072) 520 : cluster 0 pgmap v434: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 870 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 4855/41787 objects degraded (11.618%); 0 B/s, 18 objects/s recovering 2023-10-01T10:36:01.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:01 smithi078 bash[75299]: audit 2023-10-01T10:36:00.431170+0000 mon.a (mon.0) 694 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:36:01.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:01 smithi078 bash[75299]: audit 2023-10-01T10:36:00.440094+0000 mon.a (mon.0) 695 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:36:01.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:01 smithi078 bash[78274]: audit 2023-10-01T10:36:00.431170+0000 mon.a (mon.0) 694 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:36:01.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:01 smithi078 bash[78274]: audit 2023-10-01T10:36:00.440094+0000 mon.a (mon.0) 695 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:36:01.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:01 smithi173 bash[58816]: audit 2023-10-01T10:36:00.431170+0000 mon.a (mon.0) 694 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:36:01.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:01 smithi173 bash[58816]: audit 2023-10-01T10:36:00.440094+0000 mon.a (mon.0) 695 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:36:02.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:02 smithi078 bash[75299]: cluster 2023-10-01T10:36:01.591690+0000 mgr.y (mgr.25072) 521 : cluster 0 pgmap v435: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 736 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 4743/35358 objects degraded (13.414%); 0 B/s, 27 objects/s recovering 2023-10-01T10:36:02.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:02 smithi078 bash[75299]: audit 2023-10-01T10:36:01.677018+0000 mon.c (mon.2) 99 : audit 0 from='client.? 172.21.15.78:0/2825075397' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:02.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:02 smithi078 bash[75299]: cluster 2023-10-01T10:36:01.922849+0000 mon.a (mon.0) 696 : cluster 3 Health check update: Degraded data redundancy: 4855/41787 objects degraded (11.618%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T10:36:02.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:02 smithi078 bash[75299]: audit 2023-10-01T10:36:02.401960+0000 mon.a (mon.0) 697 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:36:02.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:02 smithi078 bash[78274]: cluster 2023-10-01T10:36:01.591690+0000 mgr.y (mgr.25072) 521 : cluster 0 pgmap v435: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 736 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 4743/35358 objects degraded (13.414%); 0 B/s, 27 objects/s recovering 2023-10-01T10:36:02.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:02 smithi078 bash[78274]: audit 2023-10-01T10:36:01.677018+0000 mon.c (mon.2) 99 : audit 0 from='client.? 172.21.15.78:0/2825075397' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:02.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:02 smithi078 bash[78274]: cluster 2023-10-01T10:36:01.922849+0000 mon.a (mon.0) 696 : cluster 3 Health check update: Degraded data redundancy: 4855/41787 objects degraded (11.618%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T10:36:02.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:02 smithi078 bash[78274]: audit 2023-10-01T10:36:02.401960+0000 mon.a (mon.0) 697 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:36:02.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:02 smithi173 bash[58816]: cluster 2023-10-01T10:36:01.591690+0000 mgr.y (mgr.25072) 521 : cluster 0 pgmap v435: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 736 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 4743/35358 objects degraded (13.414%); 0 B/s, 27 objects/s recovering 2023-10-01T10:36:02.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:02 smithi173 bash[58816]: audit 2023-10-01T10:36:01.677018+0000 mon.c (mon.2) 99 : audit 0 from='client.? 172.21.15.78:0/2825075397' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:02.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:02 smithi173 bash[58816]: cluster 2023-10-01T10:36:01.922849+0000 mon.a (mon.0) 696 : cluster 3 Health check update: Degraded data redundancy: 4855/41787 objects degraded (11.618%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T10:36:02.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:02 smithi173 bash[58816]: audit 2023-10-01T10:36:02.401960+0000 mon.a (mon.0) 697 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:36:03.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:03 smithi078 bash[75299]: audit 2023-10-01T10:36:02.793133+0000 mon.a (mon.0) 698 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:36:03.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:03 smithi078 bash[75299]: audit 2023-10-01T10:36:02.800154+0000 mon.a (mon.0) 699 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:36:03.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:03 smithi078 bash[75299]: audit 2023-10-01T10:36:03.422519+0000 mon.a (mon.0) 700 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:36:03.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:03 smithi078 bash[75299]: audit 2023-10-01T10:36:03.426218+0000 mon.a (mon.0) 701 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:36:03.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:03 smithi078 bash[75299]: audit 2023-10-01T10:36:03.438725+0000 mon.a (mon.0) 702 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:36:03.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:03 smithi078 bash[75299]: audit 2023-10-01T10:36:03.557891+0000 mon.a (mon.0) 703 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:36:03.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:03 smithi078 bash[75299]: audit 2023-10-01T10:36:03.562977+0000 mon.a (mon.0) 704 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:36:03.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:03 smithi078 bash[75299]: audit 2023-10-01T10:36:03.565894+0000 mon.a (mon.0) 705 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:36:03.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:03 smithi078 bash[75299]: audit 2023-10-01T10:36:03.569807+0000 mon.a (mon.0) 706 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:36:03.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:03 smithi078 bash[75299]: audit 2023-10-01T10:36:03.571785+0000 mon.a (mon.0) 707 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:36:03.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:03 smithi078 bash[75299]: audit 2023-10-01T10:36:03.572042+0000 mgr.y (mgr.25072) 522 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:36:03.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:03 smithi078 bash[75299]: cephadm 2023-10-01T10:36:03.573331+0000 mgr.y (mgr.25072) 523 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-01T10:36:03.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:03 smithi078 bash[75299]: cluster 2023-10-01T10:36:03.592118+0000 mgr.y (mgr.25072) 524 : cluster 0 pgmap v436: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 736 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 4743/35358 objects degraded (13.414%); 0 B/s, 18 objects/s recovering 2023-10-01T10:36:03.835 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:36:03 smithi078 bash[71214]: debug 2023-10-01T10:36:03.565+0000 7fd9c245e700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-01T10:36:03.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:03 smithi078 bash[78274]: audit 2023-10-01T10:36:02.793133+0000 mon.a (mon.0) 698 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:36:03.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:03 smithi078 bash[78274]: audit 2023-10-01T10:36:02.800154+0000 mon.a (mon.0) 699 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:36:03.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:03 smithi078 bash[78274]: audit 2023-10-01T10:36:03.422519+0000 mon.a (mon.0) 700 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:36:03.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:03 smithi078 bash[78274]: audit 2023-10-01T10:36:03.426218+0000 mon.a (mon.0) 701 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:36:03.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:03 smithi078 bash[78274]: audit 2023-10-01T10:36:03.438725+0000 mon.a (mon.0) 702 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:36:03.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:03 smithi078 bash[78274]: audit 2023-10-01T10:36:03.557891+0000 mon.a (mon.0) 703 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:36:03.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:03 smithi078 bash[78274]: audit 2023-10-01T10:36:03.562977+0000 mon.a (mon.0) 704 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:36:03.836 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:03 smithi078 bash[78274]: audit 2023-10-01T10:36:03.565894+0000 mon.a (mon.0) 705 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:36:03.836 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:03 smithi078 bash[78274]: audit 2023-10-01T10:36:03.569807+0000 mon.a (mon.0) 706 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:36:03.836 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:03 smithi078 bash[78274]: audit 2023-10-01T10:36:03.571785+0000 mon.a (mon.0) 707 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:36:03.836 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:03 smithi078 bash[78274]: audit 2023-10-01T10:36:03.572042+0000 mgr.y (mgr.25072) 522 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:36:03.836 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:03 smithi078 bash[78274]: cephadm 2023-10-01T10:36:03.573331+0000 mgr.y (mgr.25072) 523 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-01T10:36:03.836 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:03 smithi078 bash[78274]: cluster 2023-10-01T10:36:03.592118+0000 mgr.y (mgr.25072) 524 : cluster 0 pgmap v436: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 736 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 4743/35358 objects degraded (13.414%); 0 B/s, 18 objects/s recovering 2023-10-01T10:36:04.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:03 smithi173 bash[58816]: audit 2023-10-01T10:36:02.793133+0000 mon.a (mon.0) 698 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:36:04.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:03 smithi173 bash[58816]: audit 2023-10-01T10:36:02.800154+0000 mon.a (mon.0) 699 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:36:04.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:03 smithi173 bash[58816]: audit 2023-10-01T10:36:03.422519+0000 mon.a (mon.0) 700 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:36:04.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:03 smithi173 bash[58816]: audit 2023-10-01T10:36:03.426218+0000 mon.a (mon.0) 701 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:36:04.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:03 smithi173 bash[58816]: audit 2023-10-01T10:36:03.438725+0000 mon.a (mon.0) 702 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:36:04.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:03 smithi173 bash[58816]: audit 2023-10-01T10:36:03.557891+0000 mon.a (mon.0) 703 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:36:04.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:03 smithi173 bash[58816]: audit 2023-10-01T10:36:03.562977+0000 mon.a (mon.0) 704 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:36:04.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:03 smithi173 bash[58816]: audit 2023-10-01T10:36:03.565894+0000 mon.a (mon.0) 705 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:36:04.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:03 smithi173 bash[58816]: audit 2023-10-01T10:36:03.569807+0000 mon.a (mon.0) 706 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:36:04.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:03 smithi173 bash[58816]: audit 2023-10-01T10:36:03.571785+0000 mon.a (mon.0) 707 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:36:04.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:03 smithi173 bash[58816]: audit 2023-10-01T10:36:03.572042+0000 mgr.y (mgr.25072) 522 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:36:04.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:03 smithi173 bash[58816]: cephadm 2023-10-01T10:36:03.573331+0000 mgr.y (mgr.25072) 523 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-01T10:36:04.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:03 smithi173 bash[58816]: cluster 2023-10-01T10:36:03.592118+0000 mgr.y (mgr.25072) 524 : cluster 0 pgmap v436: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 736 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 4743/35358 objects degraded (13.414%); 0 B/s, 18 objects/s recovering 2023-10-01T10:36:05.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:04 smithi078 bash[75299]: audit 2023-10-01T10:36:04.264577+0000 mon.c (mon.2) 100 : audit 0 from='client.? 172.21.15.78:0/618105780' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:05.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:04 smithi078 bash[78274]: audit 2023-10-01T10:36:04.264577+0000 mon.c (mon.2) 100 : audit 0 from='client.? 172.21.15.78:0/618105780' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:05.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:04 smithi173 bash[58816]: audit 2023-10-01T10:36:04.264577+0000 mon.c (mon.2) 100 : audit 0 from='client.? 172.21.15.78:0/618105780' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:06.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:05 smithi078 bash[75299]: cluster 2023-10-01T10:36:05.592978+0000 mgr.y (mgr.25072) 525 : cluster 0 pgmap v437: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 541 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 819 op/s; 4635/26025 objects degraded (17.810%); 0 B/s, 27 objects/s recovering 2023-10-01T10:36:06.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:05 smithi078 bash[78274]: cluster 2023-10-01T10:36:05.592978+0000 mgr.y (mgr.25072) 525 : cluster 0 pgmap v437: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 541 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 819 op/s; 4635/26025 objects degraded (17.810%); 0 B/s, 27 objects/s recovering 2023-10-01T10:36:06.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:05 smithi173 bash[58816]: cluster 2023-10-01T10:36:05.592978+0000 mgr.y (mgr.25072) 525 : cluster 0 pgmap v437: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 541 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 819 op/s; 4635/26025 objects degraded (17.810%); 0 B/s, 27 objects/s recovering 2023-10-01T10:36:07.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:06 smithi078 bash[75299]: audit 2023-10-01T10:36:06.885599+0000 mon.c (mon.2) 101 : audit 0 from='client.? 172.21.15.78:0/2520459934' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:07.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:06 smithi078 bash[75299]: cluster 2023-10-01T10:36:06.923802+0000 mon.a (mon.0) 708 : cluster 3 Health check update: Degraded data redundancy: 4635/26025 objects degraded (17.810%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T10:36:07.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:06 smithi078 bash[78274]: audit 2023-10-01T10:36:06.885599+0000 mon.c (mon.2) 101 : audit 0 from='client.? 172.21.15.78:0/2520459934' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:07.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:06 smithi078 bash[78274]: cluster 2023-10-01T10:36:06.923802+0000 mon.a (mon.0) 708 : cluster 3 Health check update: Degraded data redundancy: 4635/26025 objects degraded (17.810%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T10:36:07.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:06 smithi173 bash[58816]: audit 2023-10-01T10:36:06.885599+0000 mon.c (mon.2) 101 : audit 0 from='client.? 172.21.15.78:0/2520459934' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:07.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:06 smithi173 bash[58816]: cluster 2023-10-01T10:36:06.923802+0000 mon.a (mon.0) 708 : cluster 3 Health check update: Degraded data redundancy: 4635/26025 objects degraded (17.810%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T10:36:08.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:07 smithi078 bash[75299]: cluster 2023-10-01T10:36:07.593548+0000 mgr.y (mgr.25072) 526 : cluster 0 pgmap v438: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 541 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 544 op/s; 4635/26025 objects degraded (17.810%); 0 B/s, 18 objects/s recovering 2023-10-01T10:36:08.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:07 smithi078 bash[78274]: cluster 2023-10-01T10:36:07.593548+0000 mgr.y (mgr.25072) 526 : cluster 0 pgmap v438: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 541 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 544 op/s; 4635/26025 objects degraded (17.810%); 0 B/s, 18 objects/s recovering 2023-10-01T10:36:08.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:07 smithi173 bash[58816]: cluster 2023-10-01T10:36:07.593548+0000 mgr.y (mgr.25072) 526 : cluster 0 pgmap v438: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 541 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 544 op/s; 4635/26025 objects degraded (17.810%); 0 B/s, 18 objects/s recovering 2023-10-01T10:36:09.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:09 smithi173 bash[58816]: audit 2023-10-01T10:36:09.526934+0000 mon.c (mon.2) 102 : audit 0 from='client.? 172.21.15.78:0/3769076104' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:10.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:09 smithi078 bash[75299]: audit 2023-10-01T10:36:09.526934+0000 mon.c (mon.2) 102 : audit 0 from='client.? 172.21.15.78:0/3769076104' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:10.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:09 smithi078 bash[78274]: audit 2023-10-01T10:36:09.526934+0000 mon.c (mon.2) 102 : audit 0 from='client.? 172.21.15.78:0/3769076104' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:10.582 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:36:10 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:36:10] "GET /metrics HTTP/1.1" 200 36191 "" "Prometheus/2.33.4" 2023-10-01T10:36:10.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:10 smithi173 bash[58816]: cluster 2023-10-01T10:36:09.594279+0000 mgr.y (mgr.25072) 527 : cluster 0 pgmap v439: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 461 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 651 op/s; 4635/22170 objects degraded (20.907%); 0 B/s, 18 objects/s recovering 2023-10-01T10:36:11.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:10 smithi078 bash[75299]: cluster 2023-10-01T10:36:09.594279+0000 mgr.y (mgr.25072) 527 : cluster 0 pgmap v439: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 461 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 651 op/s; 4635/22170 objects degraded (20.907%); 0 B/s, 18 objects/s recovering 2023-10-01T10:36:11.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:10 smithi078 bash[78274]: cluster 2023-10-01T10:36:09.594279+0000 mgr.y (mgr.25072) 527 : cluster 0 pgmap v439: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 461 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 651 op/s; 4635/22170 objects degraded (20.907%); 0 B/s, 18 objects/s recovering 2023-10-01T10:36:12.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:11 smithi078 bash[75299]: cluster 2023-10-01T10:36:11.594830+0000 mgr.y (mgr.25072) 528 : cluster 0 pgmap v440: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 330 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 718 op/s; 4525/15906 objects degraded (28.448%); 0 B/s, 27 objects/s recovering 2023-10-01T10:36:12.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:11 smithi078 bash[78274]: cluster 2023-10-01T10:36:11.594830+0000 mgr.y (mgr.25072) 528 : cluster 0 pgmap v440: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 330 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 718 op/s; 4525/15906 objects degraded (28.448%); 0 B/s, 27 objects/s recovering 2023-10-01T10:36:12.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:11 smithi173 bash[58816]: cluster 2023-10-01T10:36:11.594830+0000 mgr.y (mgr.25072) 528 : cluster 0 pgmap v440: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 330 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 718 op/s; 4525/15906 objects degraded (28.448%); 0 B/s, 27 objects/s recovering 2023-10-01T10:36:13.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:12 smithi078 bash[75299]: cluster 2023-10-01T10:36:11.924852+0000 mon.a (mon.0) 709 : cluster 3 Health check update: Degraded data redundancy: 4525/15906 objects degraded (28.448%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T10:36:13.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:12 smithi078 bash[75299]: audit 2023-10-01T10:36:12.104809+0000 mon.c (mon.2) 103 : audit 0 from='client.? 172.21.15.78:0/1792717265' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:13.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:12 smithi078 bash[78274]: cluster 2023-10-01T10:36:11.924852+0000 mon.a (mon.0) 709 : cluster 3 Health check update: Degraded data redundancy: 4525/15906 objects degraded (28.448%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T10:36:13.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:12 smithi078 bash[78274]: audit 2023-10-01T10:36:12.104809+0000 mon.c (mon.2) 103 : audit 0 from='client.? 172.21.15.78:0/1792717265' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:13.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:12 smithi173 bash[58816]: cluster 2023-10-01T10:36:11.924852+0000 mon.a (mon.0) 709 : cluster 3 Health check update: Degraded data redundancy: 4525/15906 objects degraded (28.448%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T10:36:13.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:12 smithi173 bash[58816]: audit 2023-10-01T10:36:12.104809+0000 mon.c (mon.2) 103 : audit 0 from='client.? 172.21.15.78:0/1792717265' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:14.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:13 smithi078 bash[78274]: cluster 2023-10-01T10:36:13.597706+0000 mgr.y (mgr.25072) 529 : cluster 0 pgmap v441: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 330 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 4525/15906 objects degraded (28.448%); 0 B/s, 18 objects/s recovering 2023-10-01T10:36:14.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:13 smithi078 bash[75299]: cluster 2023-10-01T10:36:13.597706+0000 mgr.y (mgr.25072) 529 : cluster 0 pgmap v441: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 330 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 4525/15906 objects degraded (28.448%); 0 B/s, 18 objects/s recovering 2023-10-01T10:36:14.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:13 smithi173 bash[58816]: cluster 2023-10-01T10:36:13.597706+0000 mgr.y (mgr.25072) 529 : cluster 0 pgmap v441: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 330 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 4525/15906 objects degraded (28.448%); 0 B/s, 18 objects/s recovering 2023-10-01T10:36:15.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:14 smithi078 bash[78274]: audit 2023-10-01T10:36:14.721781+0000 mon.b (mon.1) 37 : audit 0 from='client.? 172.21.15.78:0/1800747944' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:15.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:14 smithi078 bash[75299]: audit 2023-10-01T10:36:14.721781+0000 mon.b (mon.1) 37 : audit 0 from='client.? 172.21.15.78:0/1800747944' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:15.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:14 smithi173 bash[58816]: audit 2023-10-01T10:36:14.721781+0000 mon.b (mon.1) 37 : audit 0 from='client.? 172.21.15.78:0/1800747944' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:16.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:15 smithi078 bash[75299]: cluster 2023-10-01T10:36:15.598637+0000 mgr.y (mgr.25072) 530 : cluster 0 pgmap v442: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 118 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 822 op/s; 4413/5724 objects degraded (77.096%); 0 B/s, 27 objects/s recovering 2023-10-01T10:36:16.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:15 smithi078 bash[78274]: cluster 2023-10-01T10:36:15.598637+0000 mgr.y (mgr.25072) 530 : cluster 0 pgmap v442: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 118 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 822 op/s; 4413/5724 objects degraded (77.096%); 0 B/s, 27 objects/s recovering 2023-10-01T10:36:16.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:15 smithi173 bash[58816]: cluster 2023-10-01T10:36:15.598637+0000 mgr.y (mgr.25072) 530 : cluster 0 pgmap v442: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 118 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 822 op/s; 4413/5724 objects degraded (77.096%); 0 B/s, 27 objects/s recovering 2023-10-01T10:36:17.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:17 smithi078 bash[78274]: cluster 2023-10-01T10:36:16.925831+0000 mon.a (mon.0) 710 : cluster 3 Health check update: Degraded data redundancy: 4413/5724 objects degraded (77.096%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T10:36:17.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:17 smithi078 bash[75299]: cluster 2023-10-01T10:36:16.925831+0000 mon.a (mon.0) 710 : cluster 3 Health check update: Degraded data redundancy: 4413/5724 objects degraded (77.096%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T10:36:17.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:17 smithi173 bash[58816]: cluster 2023-10-01T10:36:16.925831+0000 mon.a (mon.0) 710 : cluster 3 Health check update: Degraded data redundancy: 4413/5724 objects degraded (77.096%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T10:36:17.512 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Removed 101447 objects 2023-10-01T10:36:17.519 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Clean up completed and total clean up time :147.883 2023-10-01T10:36:18.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:18 smithi078 bash[78274]: audit 2023-10-01T10:36:17.346284+0000 mon.a (mon.0) 711 : audit 0 from='client.? 172.21.15.78:0/2250227234' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:18.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:18 smithi078 bash[78274]: audit 2023-10-01T10:36:17.404427+0000 mon.a (mon.0) 712 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:36:18.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:18 smithi078 bash[78274]: cluster 2023-10-01T10:36:17.599454+0000 mgr.y (mgr.25072) 531 : cluster 0 pgmap v443: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 118 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 563 op/s; 4413/5724 objects degraded (77.096%); 0 B/s, 18 objects/s recovering 2023-10-01T10:36:18.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:18 smithi078 bash[75299]: audit 2023-10-01T10:36:17.346284+0000 mon.a (mon.0) 711 : audit 0 from='client.? 172.21.15.78:0/2250227234' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:18.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:18 smithi078 bash[75299]: audit 2023-10-01T10:36:17.404427+0000 mon.a (mon.0) 712 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:36:18.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:18 smithi078 bash[75299]: cluster 2023-10-01T10:36:17.599454+0000 mgr.y (mgr.25072) 531 : cluster 0 pgmap v443: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 118 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 563 op/s; 4413/5724 objects degraded (77.096%); 0 B/s, 18 objects/s recovering 2023-10-01T10:36:18.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:18 smithi173 bash[58816]: audit 2023-10-01T10:36:17.346284+0000 mon.a (mon.0) 711 : audit 0 from='client.? 172.21.15.78:0/2250227234' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:18.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:18 smithi173 bash[58816]: audit 2023-10-01T10:36:17.404427+0000 mon.a (mon.0) 712 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:36:18.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:18 smithi173 bash[58816]: cluster 2023-10-01T10:36:17.599454+0000 mgr.y (mgr.25072) 531 : cluster 0 pgmap v443: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 118 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 563 op/s; 4413/5724 objects degraded (77.096%); 0 B/s, 18 objects/s recovering 2023-10-01T10:36:19.021 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:36:18 smithi078 bash[71214]: debug 2023-10-01T10:36:18.581+0000 7fd9c245e700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-01T10:36:19.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:19 smithi078 bash[78274]: audit 2023-10-01T10:36:18.588156+0000 mon.a (mon.0) 713 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:36:19.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:19 smithi078 bash[78274]: audit 2023-10-01T10:36:18.588511+0000 mgr.y (mgr.25072) 532 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:36:19.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:19 smithi078 bash[78274]: cephadm 2023-10-01T10:36:18.590298+0000 mgr.y (mgr.25072) 533 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-01T10:36:19.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:19 smithi078 bash[75299]: audit 2023-10-01T10:36:18.588156+0000 mon.a (mon.0) 713 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:36:19.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:19 smithi078 bash[75299]: audit 2023-10-01T10:36:18.588511+0000 mgr.y (mgr.25072) 532 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:36:19.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:19 smithi078 bash[75299]: cephadm 2023-10-01T10:36:18.590298+0000 mgr.y (mgr.25072) 533 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-01T10:36:19.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:19 smithi173 bash[58816]: audit 2023-10-01T10:36:18.588156+0000 mon.a (mon.0) 713 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:36:19.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:19 smithi173 bash[58816]: audit 2023-10-01T10:36:18.588511+0000 mgr.y (mgr.25072) 532 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:36:19.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:19 smithi173 bash[58816]: cephadm 2023-10-01T10:36:18.590298+0000 mgr.y (mgr.25072) 533 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-10-01T10:36:20.332 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:36:20 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:36:20] "GET /metrics HTTP/1.1" 200 36188 "" "Prometheus/2.33.4" 2023-10-01T10:36:20.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:20 smithi078 bash[75299]: cluster 2023-10-01T10:36:19.600369+0000 mgr.y (mgr.25072) 534 : cluster 0 pgmap v444: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 62 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 638 op/s; 4413/3027 objects degraded (145.788%); 0 B/s, 18 objects/s recovering 2023-10-01T10:36:20.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:20 smithi078 bash[75299]: audit 2023-10-01T10:36:19.721545+0000 mon.c (mon.2) 104 : audit 0 from='client.? 172.21.15.78:0/1566356122' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:20.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:20 smithi078 bash[78274]: cluster 2023-10-01T10:36:19.600369+0000 mgr.y (mgr.25072) 534 : cluster 0 pgmap v444: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 62 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 638 op/s; 4413/3027 objects degraded (145.788%); 0 B/s, 18 objects/s recovering 2023-10-01T10:36:20.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:20 smithi078 bash[78274]: audit 2023-10-01T10:36:19.721545+0000 mon.c (mon.2) 104 : audit 0 from='client.? 172.21.15.78:0/1566356122' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:20.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:20 smithi173 bash[58816]: cluster 2023-10-01T10:36:19.600369+0000 mgr.y (mgr.25072) 534 : cluster 0 pgmap v444: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 62 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 638 op/s; 4413/3027 objects degraded (145.788%); 0 B/s, 18 objects/s recovering 2023-10-01T10:36:20.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:20 smithi173 bash[58816]: audit 2023-10-01T10:36:19.721545+0000 mon.c (mon.2) 104 : audit 0 from='client.? 172.21.15.78:0/1566356122' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:22.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:21 smithi078 bash[75299]: cluster 2023-10-01T10:36:21.601463+0000 mgr.y (mgr.25072) 535 : cluster 0 pgmap v445: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 4300/72 objects degraded (5972.222%); 0 B/s, 27 objects/s recovering 2023-10-01T10:36:22.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:21 smithi078 bash[78274]: cluster 2023-10-01T10:36:21.601463+0000 mgr.y (mgr.25072) 535 : cluster 0 pgmap v445: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 4300/72 objects degraded (5972.222%); 0 B/s, 27 objects/s recovering 2023-10-01T10:36:22.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:21 smithi173 bash[58816]: cluster 2023-10-01T10:36:21.601463+0000 mgr.y (mgr.25072) 535 : cluster 0 pgmap v445: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 4300/72 objects degraded (5972.222%); 0 B/s, 27 objects/s recovering 2023-10-01T10:36:22.620 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_3 2023-10-01T10:36:22.621 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_3 unique_pool_3 --yes-i-really-really-mean-it 2023-10-01T10:36:23.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:22 smithi078 bash[75299]: cluster 2023-10-01T10:36:21.946304+0000 mon.a (mon.0) 714 : cluster 3 Health check update: Degraded data redundancy: 4300/72 objects degraded (5972.222%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T10:36:23.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:22 smithi078 bash[75299]: audit 2023-10-01T10:36:22.081105+0000 mon.a (mon.0) 715 : audit 0 from='client.? 172.21.15.78:0/3699853049' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:23.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:22 smithi078 bash[78274]: cluster 2023-10-01T10:36:21.946304+0000 mon.a (mon.0) 714 : cluster 3 Health check update: Degraded data redundancy: 4300/72 objects degraded (5972.222%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T10:36:23.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:22 smithi078 bash[78274]: audit 2023-10-01T10:36:22.081105+0000 mon.a (mon.0) 715 : audit 0 from='client.? 172.21.15.78:0/3699853049' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:23.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:22 smithi173 bash[58816]: cluster 2023-10-01T10:36:21.946304+0000 mon.a (mon.0) 714 : cluster 3 Health check update: Degraded data redundancy: 4300/72 objects degraded (5972.222%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T10:36:23.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:22 smithi173 bash[58816]: audit 2023-10-01T10:36:22.081105+0000 mon.a (mon.0) 715 : audit 0 from='client.? 172.21.15.78:0/3699853049' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:24.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:23 smithi078 bash[78274]: cluster 2023-10-01T10:36:23.602094+0000 mgr.y (mgr.25072) 536 : cluster 0 pgmap v446: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 439 op/s; 4300/72 objects degraded (5972.222%); 0 B/s, 18 objects/s recovering 2023-10-01T10:36:24.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:23 smithi078 bash[75299]: cluster 2023-10-01T10:36:23.602094+0000 mgr.y (mgr.25072) 536 : cluster 0 pgmap v446: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 439 op/s; 4300/72 objects degraded (5972.222%); 0 B/s, 18 objects/s recovering 2023-10-01T10:36:24.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:23 smithi173 bash[58816]: cluster 2023-10-01T10:36:23.602094+0000 mgr.y (mgr.25072) 536 : cluster 0 pgmap v446: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 439 op/s; 4300/72 objects degraded (5972.222%); 0 B/s, 18 objects/s recovering 2023-10-01T10:36:25.007 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:24 smithi078 bash[75299]: audit 2023-10-01T10:36:24.420995+0000 mon.c (mon.2) 105 : audit 0 from='client.? 172.21.15.78:0/937401642' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:25.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:24 smithi173 bash[58816]: audit 2023-10-01T10:36:24.420995+0000 mon.c (mon.2) 105 : audit 0 from='client.? 172.21.15.78:0/937401642' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:25.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:25 smithi078 bash[78274]: audit 2023-10-01T10:36:24.420995+0000 mon.c (mon.2) 105 : audit 0 from='client.? 172.21.15.78:0/937401642' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:26.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:25 smithi078 bash[75299]: cluster 2023-10-01T10:36:25.603349+0000 mgr.y (mgr.25072) 537 : cluster 0 pgmap v447: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 439 op/s; 4183/72 objects degraded (5809.722%); 0 B/s, 28 objects/s recovering 2023-10-01T10:36:26.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:25 smithi078 bash[78274]: cluster 2023-10-01T10:36:25.603349+0000 mgr.y (mgr.25072) 537 : cluster 0 pgmap v447: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 439 op/s; 4183/72 objects degraded (5809.722%); 0 B/s, 28 objects/s recovering 2023-10-01T10:36:26.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:25 smithi173 bash[58816]: cluster 2023-10-01T10:36:25.603349+0000 mgr.y (mgr.25072) 537 : cluster 0 pgmap v447: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 439 op/s; 4183/72 objects degraded (5809.722%); 0 B/s, 28 objects/s recovering 2023-10-01T10:36:27.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:26 smithi173 bash[58816]: audit 2023-10-01T10:36:26.781198+0000 mon.c (mon.2) 106 : audit 0 from='client.? 172.21.15.78:0/2117545048' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:27.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:26 smithi078 bash[78274]: audit 2023-10-01T10:36:26.781198+0000 mon.c (mon.2) 106 : audit 0 from='client.? 172.21.15.78:0/2117545048' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:27.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:26 smithi078 bash[75299]: audit 2023-10-01T10:36:26.781198+0000 mon.c (mon.2) 106 : audit 0 from='client.? 172.21.15.78:0/2117545048' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:27.501 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:36:28.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:27 smithi078 bash[78274]: cluster 2023-10-01T10:36:26.948049+0000 mon.a (mon.0) 716 : cluster 3 Health check update: Degraded data redundancy: 4183/72 objects degraded (5809.722%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T10:36:28.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:27 smithi078 bash[78274]: cluster 2023-10-01T10:36:27.603778+0000 mgr.y (mgr.25072) 538 : cluster 0 pgmap v448: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 156 op/s; 4183/72 objects degraded (5809.722%); 0 B/s, 19 objects/s recovering 2023-10-01T10:36:28.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:27 smithi078 bash[75299]: cluster 2023-10-01T10:36:26.948049+0000 mon.a (mon.0) 716 : cluster 3 Health check update: Degraded data redundancy: 4183/72 objects degraded (5809.722%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T10:36:28.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:27 smithi078 bash[75299]: cluster 2023-10-01T10:36:27.603778+0000 mgr.y (mgr.25072) 538 : cluster 0 pgmap v448: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 156 op/s; 4183/72 objects degraded (5809.722%); 0 B/s, 19 objects/s recovering 2023-10-01T10:36:28.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:27 smithi173 bash[58816]: cluster 2023-10-01T10:36:26.948049+0000 mon.a (mon.0) 716 : cluster 3 Health check update: Degraded data redundancy: 4183/72 objects degraded (5809.722%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T10:36:28.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:27 smithi173 bash[58816]: cluster 2023-10-01T10:36:27.603778+0000 mgr.y (mgr.25072) 538 : cluster 0 pgmap v448: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 156 op/s; 4183/72 objects degraded (5809.722%); 0 B/s, 19 objects/s recovering 2023-10-01T10:36:29.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:29 smithi078 bash[75299]: audit 2023-10-01T10:36:29.164941+0000 mon.c (mon.2) 107 : audit 0 from='client.? 172.21.15.78:0/1011303682' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:29.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:29 smithi078 bash[78274]: audit 2023-10-01T10:36:29.164941+0000 mon.c (mon.2) 107 : audit 0 from='client.? 172.21.15.78:0/1011303682' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:29.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:29 smithi173 bash[58816]: audit 2023-10-01T10:36:29.164941+0000 mon.c (mon.2) 107 : audit 0 from='client.? 172.21.15.78:0/1011303682' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:30.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:30 smithi078 bash[75299]: cluster 2023-10-01T10:36:29.604439+0000 mgr.y (mgr.25072) 539 : cluster 0 pgmap v449: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 156 op/s; 4183/72 objects degraded (5809.722%); 0 B/s, 19 objects/s recovering 2023-10-01T10:36:30.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:30 smithi078 bash[78274]: cluster 2023-10-01T10:36:29.604439+0000 mgr.y (mgr.25072) 539 : cluster 0 pgmap v449: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 156 op/s; 4183/72 objects degraded (5809.722%); 0 B/s, 19 objects/s recovering 2023-10-01T10:36:30.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:36:30 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:36:30] "GET /metrics HTTP/1.1" 200 36188 "" "Prometheus/2.33.4" 2023-10-01T10:36:30.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:30 smithi173 bash[58816]: cluster 2023-10-01T10:36:29.604439+0000 mgr.y (mgr.25072) 539 : cluster 0 pgmap v449: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 156 op/s; 4183/72 objects degraded (5809.722%); 0 B/s, 19 objects/s recovering 2023-10-01T10:36:31.257 INFO:teuthology.orchestra.run.smithi078.stderr:pool 'unique_pool_3' removed 2023-10-01T10:36:31.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:31 smithi078 bash[75299]: audit 2023-10-01T10:36:31.099874+0000 mon.c (mon.2) 108 : audit 1 from='client.? 172.21.15.78:0/3809933279' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T10:36:31.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:31 smithi078 bash[75299]: audit 2023-10-01T10:36:31.100410+0000 mon.a (mon.0) 717 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T10:36:31.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:31 smithi078 bash[78274]: audit 2023-10-01T10:36:31.099874+0000 mon.c (mon.2) 108 : audit 1 from='client.? 172.21.15.78:0/3809933279' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T10:36:31.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:31 smithi078 bash[78274]: audit 2023-10-01T10:36:31.100410+0000 mon.a (mon.0) 717 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T10:36:31.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:31 smithi173 bash[58816]: audit 2023-10-01T10:36:31.099874+0000 mon.c (mon.2) 108 : audit 1 from='client.? 172.21.15.78:0/3809933279' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T10:36:31.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:31 smithi173 bash[58816]: audit 2023-10-01T10:36:31.100410+0000 mon.a (mon.0) 717 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T10:36:32.276 INFO:teuthology.task.sequential:In sequential, running task print... 2023-10-01T10:36:32.277 INFO:teuthology.task.print:**** done end radosbench.yaml 2023-10-01T10:36:32.277 DEBUG:teuthology.parallel:result is None 2023-10-01T10:36:32.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:32 smithi078 bash[78274]: cluster 2023-10-01T10:36:31.231300+0000 osd.3 (osd.3) 34 : cluster 3 failed to encode map e112 with expected crc 2023-10-01T10:36:32.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:32 smithi078 bash[78274]: cluster 2023-10-01T10:36:31.232871+0000 osd.7 (osd.7) 43 : cluster 3 failed to encode map e112 with expected crc 2023-10-01T10:36:32.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:32 smithi078 bash[78274]: audit 2023-10-01T10:36:31.235579+0000 mon.a (mon.0) 718 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-10-01T10:36:32.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:32 smithi078 bash[78274]: cluster 2023-10-01T10:36:31.235783+0000 mon.a (mon.0) 719 : cluster 0 osdmap e112: 8 total, 8 up, 8 in 2023-10-01T10:36:32.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:32 smithi078 bash[78274]: cluster 2023-10-01T10:36:31.297422+0000 osd.5 (osd.5) 29 : cluster 3 failed to encode map e112 with expected crc 2023-10-01T10:36:32.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:32 smithi078 bash[78274]: cluster 2023-10-01T10:36:31.499329+0000 osd.6 (osd.6) 24 : cluster 3 failed to encode map e112 with expected crc 2023-10-01T10:36:32.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:32 smithi078 bash[78274]: audit 2023-10-01T10:36:31.560131+0000 mon.c (mon.2) 109 : audit 0 from='client.? 172.21.15.78:0/27343841' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:32.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:32 smithi078 bash[78274]: cluster 2023-10-01T10:36:31.605077+0000 mgr.y (mgr.25072) 540 : cluster 0 pgmap v451: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:36:32.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:32 smithi078 bash[75299]: cluster 2023-10-01T10:36:31.231300+0000 osd.3 (osd.3) 34 : cluster 3 failed to encode map e112 with expected crc 2023-10-01T10:36:32.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:32 smithi078 bash[75299]: cluster 2023-10-01T10:36:31.232871+0000 osd.7 (osd.7) 43 : cluster 3 failed to encode map e112 with expected crc 2023-10-01T10:36:32.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:32 smithi078 bash[75299]: audit 2023-10-01T10:36:31.235579+0000 mon.a (mon.0) 718 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-10-01T10:36:32.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:32 smithi078 bash[75299]: cluster 2023-10-01T10:36:31.235783+0000 mon.a (mon.0) 719 : cluster 0 osdmap e112: 8 total, 8 up, 8 in 2023-10-01T10:36:32.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:32 smithi078 bash[75299]: cluster 2023-10-01T10:36:31.297422+0000 osd.5 (osd.5) 29 : cluster 3 failed to encode map e112 with expected crc 2023-10-01T10:36:32.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:32 smithi078 bash[75299]: cluster 2023-10-01T10:36:31.499329+0000 osd.6 (osd.6) 24 : cluster 3 failed to encode map e112 with expected crc 2023-10-01T10:36:32.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:32 smithi078 bash[75299]: audit 2023-10-01T10:36:31.560131+0000 mon.c (mon.2) 109 : audit 0 from='client.? 172.21.15.78:0/27343841' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:32.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:32 smithi078 bash[75299]: cluster 2023-10-01T10:36:31.605077+0000 mgr.y (mgr.25072) 540 : cluster 0 pgmap v451: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:36:32.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:32 smithi173 bash[58816]: cluster 2023-10-01T10:36:31.231300+0000 osd.3 (osd.3) 34 : cluster 3 failed to encode map e112 with expected crc 2023-10-01T10:36:32.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:32 smithi173 bash[58816]: cluster 2023-10-01T10:36:31.232871+0000 osd.7 (osd.7) 43 : cluster 3 failed to encode map e112 with expected crc 2023-10-01T10:36:32.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:32 smithi173 bash[58816]: audit 2023-10-01T10:36:31.235579+0000 mon.a (mon.0) 718 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-10-01T10:36:32.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:32 smithi173 bash[58816]: cluster 2023-10-01T10:36:31.235783+0000 mon.a (mon.0) 719 : cluster 0 osdmap e112: 8 total, 8 up, 8 in 2023-10-01T10:36:32.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:32 smithi173 bash[58816]: cluster 2023-10-01T10:36:31.297422+0000 osd.5 (osd.5) 29 : cluster 3 failed to encode map e112 with expected crc 2023-10-01T10:36:32.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:32 smithi173 bash[58816]: cluster 2023-10-01T10:36:31.499329+0000 osd.6 (osd.6) 24 : cluster 3 failed to encode map e112 with expected crc 2023-10-01T10:36:32.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:32 smithi173 bash[58816]: audit 2023-10-01T10:36:31.560131+0000 mon.c (mon.2) 109 : audit 0 from='client.? 172.21.15.78:0/27343841' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:32.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:32 smithi173 bash[58816]: cluster 2023-10-01T10:36:31.605077+0000 mgr.y (mgr.25072) 540 : cluster 0 pgmap v451: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:36:33.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:33 smithi078 bash[78274]: cluster 2023-10-01T10:36:32.241326+0000 mon.a (mon.0) 720 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4183/72 objects degraded (5809.722%), 3 pgs degraded, 3 pgs undersized) 2023-10-01T10:36:33.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:33 smithi078 bash[78274]: cluster 2023-10-01T10:36:32.241396+0000 mon.a (mon.0) 721 : cluster 1 Cluster is now healthy 2023-10-01T10:36:33.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:33 smithi078 bash[78274]: audit 2023-10-01T10:36:32.412760+0000 mon.a (mon.0) 722 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:36:33.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:33 smithi078 bash[78274]: audit 2023-10-01T10:36:32.414344+0000 mon.a (mon.0) 723 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:36:33.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:33 smithi078 bash[75299]: cluster 2023-10-01T10:36:32.241326+0000 mon.a (mon.0) 720 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4183/72 objects degraded (5809.722%), 3 pgs degraded, 3 pgs undersized) 2023-10-01T10:36:33.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:33 smithi078 bash[75299]: cluster 2023-10-01T10:36:32.241396+0000 mon.a (mon.0) 721 : cluster 1 Cluster is now healthy 2023-10-01T10:36:33.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:33 smithi078 bash[75299]: audit 2023-10-01T10:36:32.412760+0000 mon.a (mon.0) 722 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:36:33.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:33 smithi078 bash[75299]: audit 2023-10-01T10:36:32.414344+0000 mon.a (mon.0) 723 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:36:33.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:33 smithi173 bash[58816]: cluster 2023-10-01T10:36:32.241326+0000 mon.a (mon.0) 720 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4183/72 objects degraded (5809.722%), 3 pgs degraded, 3 pgs undersized) 2023-10-01T10:36:33.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:33 smithi173 bash[58816]: cluster 2023-10-01T10:36:32.241396+0000 mon.a (mon.0) 721 : cluster 1 Cluster is now healthy 2023-10-01T10:36:33.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:33 smithi173 bash[58816]: audit 2023-10-01T10:36:32.412760+0000 mon.a (mon.0) 722 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:36:33.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:33 smithi173 bash[58816]: audit 2023-10-01T10:36:32.414344+0000 mon.a (mon.0) 723 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:36:34.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:34 smithi078 bash[75299]: cluster 2023-10-01T10:36:33.605786+0000 mgr.y (mgr.25072) 541 : cluster 0 pgmap v452: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:36:34.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:34 smithi078 bash[75299]: audit 2023-10-01T10:36:33.606065+0000 mon.a (mon.0) 724 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:36:34.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:34 smithi078 bash[75299]: audit 2023-10-01T10:36:33.606419+0000 mgr.y (mgr.25072) 542 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:36:34.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:34 smithi078 bash[75299]: cephadm 2023-10-01T10:36:33.609075+0000 mgr.y (mgr.25072) 543 : cephadm 1 Upgrade: osd.2 is safe to restart 2023-10-01T10:36:34.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:34 smithi078 bash[75299]: audit 2023-10-01T10:36:33.924896+0000 mon.a (mon.0) 725 : audit 0 from='client.? 172.21.15.78:0/4004961252' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:34.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:34 smithi078 bash[78274]: cluster 2023-10-01T10:36:33.605786+0000 mgr.y (mgr.25072) 541 : cluster 0 pgmap v452: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:36:34.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:34 smithi078 bash[78274]: audit 2023-10-01T10:36:33.606065+0000 mon.a (mon.0) 724 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:36:34.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:34 smithi078 bash[78274]: audit 2023-10-01T10:36:33.606419+0000 mgr.y (mgr.25072) 542 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:36:34.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:34 smithi078 bash[78274]: cephadm 2023-10-01T10:36:33.609075+0000 mgr.y (mgr.25072) 543 : cephadm 1 Upgrade: osd.2 is safe to restart 2023-10-01T10:36:34.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:34 smithi078 bash[78274]: audit 2023-10-01T10:36:33.924896+0000 mon.a (mon.0) 725 : audit 0 from='client.? 172.21.15.78:0/4004961252' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:34.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:34 smithi173 bash[58816]: cluster 2023-10-01T10:36:33.605786+0000 mgr.y (mgr.25072) 541 : cluster 0 pgmap v452: 73 pgs: 73 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:36:34.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:34 smithi173 bash[58816]: audit 2023-10-01T10:36:33.606065+0000 mon.a (mon.0) 724 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:36:34.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:34 smithi173 bash[58816]: audit 2023-10-01T10:36:33.606419+0000 mgr.y (mgr.25072) 542 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-10-01T10:36:34.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:34 smithi173 bash[58816]: cephadm 2023-10-01T10:36:33.609075+0000 mgr.y (mgr.25072) 543 : cephadm 1 Upgrade: osd.2 is safe to restart 2023-10-01T10:36:34.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:34 smithi173 bash[58816]: audit 2023-10-01T10:36:33.924896+0000 mon.a (mon.0) 725 : audit 0 from='client.? 172.21.15.78:0/4004961252' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:36.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:35 smithi078 bash[78274]: cluster 2023-10-01T10:36:35.606696+0000 mgr.y (mgr.25072) 544 : cluster 0 pgmap v453: 73 pgs: 73 active+clean; 582 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:36:36.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:35 smithi078 bash[75299]: cluster 2023-10-01T10:36:35.606696+0000 mgr.y (mgr.25072) 544 : cluster 0 pgmap v453: 73 pgs: 73 active+clean; 582 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:36:36.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:35 smithi173 bash[58816]: cluster 2023-10-01T10:36:35.606696+0000 mgr.y (mgr.25072) 544 : cluster 0 pgmap v453: 73 pgs: 73 active+clean; 582 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:36:37.313 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:37 smithi078 bash[78274]: cephadm 2023-10-01T10:36:36.028551+0000 mgr.y (mgr.25072) 545 : cephadm 1 Upgrade: Updating osd.2 2023-10-01T10:36:37.314 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:37 smithi078 bash[78274]: audit 2023-10-01T10:36:36.037066+0000 mon.a (mon.0) 726 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:36:37.314 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:37 smithi078 bash[78274]: audit 2023-10-01T10:36:36.038433+0000 mon.a (mon.0) 727 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-01T10:36:37.314 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:37 smithi078 bash[78274]: audit 2023-10-01T10:36:36.039330+0000 mon.a (mon.0) 728 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:36:37.314 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:37 smithi078 bash[78274]: cephadm 2023-10-01T10:36:36.042081+0000 mgr.y (mgr.25072) 546 : cephadm 1 Deploying daemon osd.2 on smithi078 2023-10-01T10:36:37.314 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:37 smithi078 bash[78274]: audit 2023-10-01T10:36:36.301703+0000 mon.c (mon.2) 110 : audit 0 from='client.? 172.21.15.78:0/1541090015' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:37.314 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:37 smithi078 bash[75299]: cephadm 2023-10-01T10:36:36.028551+0000 mgr.y (mgr.25072) 545 : cephadm 1 Upgrade: Updating osd.2 2023-10-01T10:36:37.314 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:37 smithi078 bash[75299]: audit 2023-10-01T10:36:36.037066+0000 mon.a (mon.0) 726 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:36:37.315 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:37 smithi078 bash[75299]: audit 2023-10-01T10:36:36.038433+0000 mon.a (mon.0) 727 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-01T10:36:37.315 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:37 smithi078 bash[75299]: audit 2023-10-01T10:36:36.039330+0000 mon.a (mon.0) 728 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:36:37.315 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:37 smithi078 bash[75299]: cephadm 2023-10-01T10:36:36.042081+0000 mgr.y (mgr.25072) 546 : cephadm 1 Deploying daemon osd.2 on smithi078 2023-10-01T10:36:37.315 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:37 smithi078 bash[75299]: audit 2023-10-01T10:36:36.301703+0000 mon.c (mon.2) 110 : audit 0 from='client.? 172.21.15.78:0/1541090015' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:37.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:37 smithi173 bash[58816]: cephadm 2023-10-01T10:36:36.028551+0000 mgr.y (mgr.25072) 545 : cephadm 1 Upgrade: Updating osd.2 2023-10-01T10:36:37.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:37 smithi173 bash[58816]: audit 2023-10-01T10:36:36.037066+0000 mon.a (mon.0) 726 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:36:37.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:37 smithi173 bash[58816]: audit 2023-10-01T10:36:36.038433+0000 mon.a (mon.0) 727 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-10-01T10:36:37.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:37 smithi173 bash[58816]: audit 2023-10-01T10:36:36.039330+0000 mon.a (mon.0) 728 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:36:37.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:37 smithi173 bash[58816]: cephadm 2023-10-01T10:36:36.042081+0000 mgr.y (mgr.25072) 546 : cephadm 1 Deploying daemon osd.2 on smithi078 2023-10-01T10:36:37.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:37 smithi173 bash[58816]: audit 2023-10-01T10:36:36.301703+0000 mon.c (mon.2) 110 : audit 0 from='client.? 172.21.15.78:0/1541090015' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:38.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:38 smithi173 bash[58816]: audit 2023-10-01T10:36:37.600588+0000 mon.a (mon.0) 729 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:36:38.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:38 smithi173 bash[58816]: cluster 2023-10-01T10:36:37.607150+0000 mgr.y (mgr.25072) 547 : cluster 0 pgmap v454: 73 pgs: 73 active+clean; 582 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:36:38.930 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:38 smithi078 bash[75299]: audit 2023-10-01T10:36:37.600588+0000 mon.a (mon.0) 729 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:36:38.930 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:38 smithi078 bash[75299]: cluster 2023-10-01T10:36:37.607150+0000 mgr.y (mgr.25072) 547 : cluster 0 pgmap v454: 73 pgs: 73 active+clean; 582 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:36:38.930 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:38 smithi078 bash[78274]: audit 2023-10-01T10:36:37.600588+0000 mon.a (mon.0) 729 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:36:38.931 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:38 smithi078 bash[78274]: cluster 2023-10-01T10:36:37.607150+0000 mgr.y (mgr.25072) 547 : cluster 0 pgmap v454: 73 pgs: 73 active+clean; 582 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:36:39.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:39 smithi078 bash[75299]: audit 2023-10-01T10:36:38.659503+0000 mon.a (mon.0) 730 : audit 0 from='client.? 172.21.15.78:0/1101535042' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:39.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:39 smithi078 bash[78274]: audit 2023-10-01T10:36:38.659503+0000 mon.a (mon.0) 730 : audit 0 from='client.? 172.21.15.78:0/1101535042' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:39.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:39 smithi173 bash[58816]: audit 2023-10-01T10:36:38.659503+0000 mon.a (mon.0) 730 : audit 0 from='client.? 172.21.15.78:0/1101535042' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:40.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:36:40 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:36:40] "GET /metrics HTTP/1.1" 200 33828 "" "Prometheus/2.33.4" 2023-10-01T10:36:40.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:40 smithi078 bash[75299]: cluster 2023-10-01T10:36:39.607682+0000 mgr.y (mgr.25072) 548 : cluster 0 pgmap v455: 73 pgs: 73 active+clean; 582 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:36:40.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:40 smithi078 bash[78274]: cluster 2023-10-01T10:36:39.607682+0000 mgr.y (mgr.25072) 548 : cluster 0 pgmap v455: 73 pgs: 73 active+clean; 582 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:36:40.833 INFO:journalctl@ceph.osd.2.smithi078.stdout:Oct 01 10:36:40 smithi078 systemd[1]: Stopping Ceph osd.2 for d8016480-6041-11ee-8db5-212e2dc638e7... 2023-10-01T10:36:40.833 INFO:journalctl@ceph.osd.2.smithi078.stdout:Oct 01 10:36:40 smithi078 bash[45186]: debug 2023-10-01T10:36:40.761+0000 7fefac3f0700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-01T10:36:40.833 INFO:journalctl@ceph.osd.2.smithi078.stdout:Oct 01 10:36:40 smithi078 bash[45186]: debug 2023-10-01T10:36:40.761+0000 7fefac3f0700 -1 osd.2 112 *** Got signal Terminated *** 2023-10-01T10:36:40.833 INFO:journalctl@ceph.osd.2.smithi078.stdout:Oct 01 10:36:40 smithi078 bash[45186]: debug 2023-10-01T10:36:40.761+0000 7fefac3f0700 -1 osd.2 112 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-01T10:36:40.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:40 smithi173 bash[58816]: cluster 2023-10-01T10:36:39.607682+0000 mgr.y (mgr.25072) 548 : cluster 0 pgmap v455: 73 pgs: 73 active+clean; 582 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:36:41.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:41 smithi173 bash[58816]: cluster 2023-10-01T10:36:40.763658+0000 mon.a (mon.0) 731 : cluster 1 osd.2 marked itself down and dead 2023-10-01T10:36:41.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:41 smithi173 bash[58816]: audit 2023-10-01T10:36:41.019471+0000 mon.c (mon.2) 111 : audit 0 from='client.? 172.21.15.78:0/588229742' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:41.875 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:41 smithi078 bash[78274]: cluster 2023-10-01T10:36:40.763658+0000 mon.a (mon.0) 731 : cluster 1 osd.2 marked itself down and dead 2023-10-01T10:36:41.876 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:41 smithi078 bash[78274]: audit 2023-10-01T10:36:41.019471+0000 mon.c (mon.2) 111 : audit 0 from='client.? 172.21.15.78:0/588229742' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:41.876 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:41 smithi078 bash[75299]: cluster 2023-10-01T10:36:40.763658+0000 mon.a (mon.0) 731 : cluster 1 osd.2 marked itself down and dead 2023-10-01T10:36:41.876 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:41 smithi078 bash[75299]: audit 2023-10-01T10:36:41.019471+0000 mon.c (mon.2) 111 : audit 0 from='client.? 172.21.15.78:0/588229742' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:42.728 INFO:journalctl@ceph.osd.2.smithi078.stdout:Oct 01 10:36:42 smithi078 bash[97411]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd-2 2023-10-01T10:36:42.728 INFO:journalctl@ceph.osd.2.smithi078.stdout:Oct 01 10:36:42 smithi078 bash[97505]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd-2 2023-10-01T10:36:42.728 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:42 smithi078 bash[75299]: cluster 2023-10-01T10:36:41.608660+0000 mgr.y (mgr.25072) 549 : cluster 0 pgmap v456: 73 pgs: 73 active+clean; 582 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:36:42.728 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:42 smithi078 bash[75299]: cluster 2023-10-01T10:36:41.614723+0000 mon.a (mon.0) 732 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-01T10:36:42.728 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:42 smithi078 bash[75299]: cluster 2023-10-01T10:36:41.632593+0000 osd.3 (osd.3) 35 : cluster 3 failed to encode map e113 with expected crc 2023-10-01T10:36:42.729 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:42 smithi078 bash[75299]: cluster 2023-10-01T10:36:41.633746+0000 osd.7 (osd.7) 44 : cluster 3 failed to encode map e113 with expected crc 2023-10-01T10:36:42.729 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:42 smithi078 bash[75299]: cluster 2023-10-01T10:36:41.636212+0000 mon.a (mon.0) 733 : cluster 0 osdmap e113: 8 total, 7 up, 8 in 2023-10-01T10:36:42.729 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:42 smithi078 bash[75299]: cluster 2023-10-01T10:36:41.636890+0000 osd.5 (osd.5) 30 : cluster 3 failed to encode map e113 with expected crc 2023-10-01T10:36:42.729 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:42 smithi078 bash[75299]: cluster 2023-10-01T10:36:41.637071+0000 osd.3 (osd.3) 36 : cluster 3 failed to encode map e113 with expected crc 2023-10-01T10:36:42.729 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:42 smithi078 bash[75299]: cluster 2023-10-01T10:36:41.637083+0000 osd.6 (osd.6) 25 : cluster 3 failed to encode map e113 with expected crc 2023-10-01T10:36:42.729 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:42 smithi078 bash[75299]: cluster 2023-10-01T10:36:41.637153+0000 osd.4 (osd.4) 19 : cluster 3 failed to encode map e113 with expected crc 2023-10-01T10:36:42.729 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:42 smithi078 bash[78274]: cluster 2023-10-01T10:36:41.608660+0000 mgr.y (mgr.25072) 549 : cluster 0 pgmap v456: 73 pgs: 73 active+clean; 582 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:36:42.729 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:42 smithi078 bash[78274]: cluster 2023-10-01T10:36:41.614723+0000 mon.a (mon.0) 732 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-01T10:36:42.729 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:42 smithi078 bash[78274]: cluster 2023-10-01T10:36:41.632593+0000 osd.3 (osd.3) 35 : cluster 3 failed to encode map e113 with expected crc 2023-10-01T10:36:42.730 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:42 smithi078 bash[78274]: cluster 2023-10-01T10:36:41.633746+0000 osd.7 (osd.7) 44 : cluster 3 failed to encode map e113 with expected crc 2023-10-01T10:36:42.730 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:42 smithi078 bash[78274]: cluster 2023-10-01T10:36:41.636212+0000 mon.a (mon.0) 733 : cluster 0 osdmap e113: 8 total, 7 up, 8 in 2023-10-01T10:36:42.730 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:42 smithi078 bash[78274]: cluster 2023-10-01T10:36:41.636890+0000 osd.5 (osd.5) 30 : cluster 3 failed to encode map e113 with expected crc 2023-10-01T10:36:42.730 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:42 smithi078 bash[78274]: cluster 2023-10-01T10:36:41.637071+0000 osd.3 (osd.3) 36 : cluster 3 failed to encode map e113 with expected crc 2023-10-01T10:36:42.730 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:42 smithi078 bash[78274]: cluster 2023-10-01T10:36:41.637083+0000 osd.6 (osd.6) 25 : cluster 3 failed to encode map e113 with expected crc 2023-10-01T10:36:42.730 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:42 smithi078 bash[78274]: cluster 2023-10-01T10:36:41.637153+0000 osd.4 (osd.4) 19 : cluster 3 failed to encode map e113 with expected crc 2023-10-01T10:36:43.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:42 smithi173 bash[58816]: cluster 2023-10-01T10:36:41.608660+0000 mgr.y (mgr.25072) 549 : cluster 0 pgmap v456: 73 pgs: 73 active+clean; 582 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:36:43.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:42 smithi173 bash[58816]: cluster 2023-10-01T10:36:41.614723+0000 mon.a (mon.0) 732 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-01T10:36:43.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:42 smithi173 bash[58816]: cluster 2023-10-01T10:36:41.632593+0000 osd.3 (osd.3) 35 : cluster 3 failed to encode map e113 with expected crc 2023-10-01T10:36:43.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:42 smithi173 bash[58816]: cluster 2023-10-01T10:36:41.633746+0000 osd.7 (osd.7) 44 : cluster 3 failed to encode map e113 with expected crc 2023-10-01T10:36:43.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:42 smithi173 bash[58816]: cluster 2023-10-01T10:36:41.636212+0000 mon.a (mon.0) 733 : cluster 0 osdmap e113: 8 total, 7 up, 8 in 2023-10-01T10:36:43.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:42 smithi173 bash[58816]: cluster 2023-10-01T10:36:41.636890+0000 osd.5 (osd.5) 30 : cluster 3 failed to encode map e113 with expected crc 2023-10-01T10:36:43.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:42 smithi173 bash[58816]: cluster 2023-10-01T10:36:41.637071+0000 osd.3 (osd.3) 36 : cluster 3 failed to encode map e113 with expected crc 2023-10-01T10:36:43.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:42 smithi173 bash[58816]: cluster 2023-10-01T10:36:41.637083+0000 osd.6 (osd.6) 25 : cluster 3 failed to encode map e113 with expected crc 2023-10-01T10:36:43.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:42 smithi173 bash[58816]: cluster 2023-10-01T10:36:41.637153+0000 osd.4 (osd.4) 19 : cluster 3 failed to encode map e113 with expected crc 2023-10-01T10:36:44.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:43 smithi078 bash[75299]: cluster 2023-10-01T10:36:42.629149+0000 osd.3 (osd.3) 37 : cluster 3 failed to encode map e114 with expected crc 2023-10-01T10:36:44.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:43 smithi078 bash[75299]: cluster 2023-10-01T10:36:42.631202+0000 mon.a (mon.0) 734 : cluster 0 osdmap e114: 8 total, 7 up, 8 in 2023-10-01T10:36:44.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:43 smithi078 bash[75299]: cluster 2023-10-01T10:36:42.631438+0000 osd.3 (osd.3) 38 : cluster 3 failed to encode map e114 with expected crc 2023-10-01T10:36:44.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:43 smithi078 bash[75299]: cluster 2023-10-01T10:36:42.632987+0000 osd.5 (osd.5) 31 : cluster 3 failed to encode map e114 with expected crc 2023-10-01T10:36:44.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:43 smithi078 bash[75299]: cluster 2023-10-01T10:36:42.633076+0000 osd.6 (osd.6) 26 : cluster 3 failed to encode map e114 with expected crc 2023-10-01T10:36:44.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:43 smithi078 bash[75299]: cluster 2023-10-01T10:36:42.633076+0000 osd.4 (osd.4) 20 : cluster 3 failed to encode map e114 with expected crc 2023-10-01T10:36:44.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:43 smithi078 bash[75299]: cluster 2023-10-01T10:36:42.634608+0000 osd.7 (osd.7) 45 : cluster 3 failed to encode map e114 with expected crc 2023-10-01T10:36:44.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:43 smithi078 bash[75299]: cluster 2023-10-01T10:36:42.639916+0000 osd.7 (osd.7) 46 : cluster 3 failed to encode map e114 with expected crc 2023-10-01T10:36:44.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:43 smithi078 bash[75299]: audit 2023-10-01T10:36:43.385129+0000 mon.a (mon.0) 735 : audit 0 from='client.? 172.21.15.78:0/3965025181' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:44.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:43 smithi078 bash[78274]: cluster 2023-10-01T10:36:42.629149+0000 osd.3 (osd.3) 37 : cluster 3 failed to encode map e114 with expected crc 2023-10-01T10:36:44.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:43 smithi078 bash[78274]: cluster 2023-10-01T10:36:42.631202+0000 mon.a (mon.0) 734 : cluster 0 osdmap e114: 8 total, 7 up, 8 in 2023-10-01T10:36:44.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:43 smithi078 bash[78274]: cluster 2023-10-01T10:36:42.631438+0000 osd.3 (osd.3) 38 : cluster 3 failed to encode map e114 with expected crc 2023-10-01T10:36:44.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:43 smithi078 bash[78274]: cluster 2023-10-01T10:36:42.632987+0000 osd.5 (osd.5) 31 : cluster 3 failed to encode map e114 with expected crc 2023-10-01T10:36:44.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:43 smithi078 bash[78274]: cluster 2023-10-01T10:36:42.633076+0000 osd.6 (osd.6) 26 : cluster 3 failed to encode map e114 with expected crc 2023-10-01T10:36:44.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:43 smithi078 bash[78274]: cluster 2023-10-01T10:36:42.633076+0000 osd.4 (osd.4) 20 : cluster 3 failed to encode map e114 with expected crc 2023-10-01T10:36:44.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:43 smithi078 bash[78274]: cluster 2023-10-01T10:36:42.634608+0000 osd.7 (osd.7) 45 : cluster 3 failed to encode map e114 with expected crc 2023-10-01T10:36:44.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:43 smithi078 bash[78274]: cluster 2023-10-01T10:36:42.639916+0000 osd.7 (osd.7) 46 : cluster 3 failed to encode map e114 with expected crc 2023-10-01T10:36:44.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:43 smithi078 bash[78274]: audit 2023-10-01T10:36:43.385129+0000 mon.a (mon.0) 735 : audit 0 from='client.? 172.21.15.78:0/3965025181' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:44.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:43 smithi173 bash[58816]: cluster 2023-10-01T10:36:42.629149+0000 osd.3 (osd.3) 37 : cluster 3 failed to encode map e114 with expected crc 2023-10-01T10:36:44.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:43 smithi173 bash[58816]: cluster 2023-10-01T10:36:42.631202+0000 mon.a (mon.0) 734 : cluster 0 osdmap e114: 8 total, 7 up, 8 in 2023-10-01T10:36:44.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:43 smithi173 bash[58816]: cluster 2023-10-01T10:36:42.631438+0000 osd.3 (osd.3) 38 : cluster 3 failed to encode map e114 with expected crc 2023-10-01T10:36:44.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:43 smithi173 bash[58816]: cluster 2023-10-01T10:36:42.632987+0000 osd.5 (osd.5) 31 : cluster 3 failed to encode map e114 with expected crc 2023-10-01T10:36:44.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:43 smithi173 bash[58816]: cluster 2023-10-01T10:36:42.633076+0000 osd.6 (osd.6) 26 : cluster 3 failed to encode map e114 with expected crc 2023-10-01T10:36:44.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:43 smithi173 bash[58816]: cluster 2023-10-01T10:36:42.633076+0000 osd.4 (osd.4) 20 : cluster 3 failed to encode map e114 with expected crc 2023-10-01T10:36:44.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:43 smithi173 bash[58816]: cluster 2023-10-01T10:36:42.634608+0000 osd.7 (osd.7) 45 : cluster 3 failed to encode map e114 with expected crc 2023-10-01T10:36:44.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:43 smithi173 bash[58816]: cluster 2023-10-01T10:36:42.639916+0000 osd.7 (osd.7) 46 : cluster 3 failed to encode map e114 with expected crc 2023-10-01T10:36:44.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:43 smithi173 bash[58816]: audit 2023-10-01T10:36:43.385129+0000 mon.a (mon.0) 735 : audit 0 from='client.? 172.21.15.78:0/3965025181' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:45.075 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:44 smithi078 bash[78274]: cluster 2023-10-01T10:36:43.609332+0000 mgr.y (mgr.25072) 550 : cluster 0 pgmap v459: 73 pgs: 3 stale+active+clean, 70 active+clean; 582 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:36:45.075 INFO:journalctl@ceph.osd.2.smithi078.stdout:Oct 01 10:36:44 smithi078 systemd[1]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7@osd.2.service: Succeeded. 2023-10-01T10:36:45.076 INFO:journalctl@ceph.osd.2.smithi078.stdout:Oct 01 10:36:44 smithi078 systemd[1]: Stopped Ceph osd.2 for d8016480-6041-11ee-8db5-212e2dc638e7. 2023-10-01T10:36:45.076 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:44 smithi078 bash[75299]: cluster 2023-10-01T10:36:43.609332+0000 mgr.y (mgr.25072) 550 : cluster 0 pgmap v459: 73 pgs: 3 stale+active+clean, 70 active+clean; 582 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:36:45.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:44 smithi173 bash[58816]: cluster 2023-10-01T10:36:43.609332+0000 mgr.y (mgr.25072) 550 : cluster 0 pgmap v459: 73 pgs: 3 stale+active+clean, 70 active+clean; 582 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:36:45.332 INFO:journalctl@ceph.osd.2.smithi078.stdout:Oct 01 10:36:45 smithi078 systemd[1]: Started Ceph osd.2 for d8016480-6041-11ee-8db5-212e2dc638e7. 2023-10-01T10:36:46.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:46 smithi173 bash[58816]: audit 2023-10-01T10:36:45.107819+0000 mon.a (mon.0) 736 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:36:46.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:46 smithi173 bash[58816]: audit 2023-10-01T10:36:45.115273+0000 mon.a (mon.0) 737 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:36:46.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:46 smithi173 bash[58816]: audit 2023-10-01T10:36:45.126645+0000 mon.a (mon.0) 738 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:36:46.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:46 smithi173 bash[58816]: cluster 2023-10-01T10:36:45.610422+0000 mgr.y (mgr.25072) 551 : cluster 0 pgmap v460: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-01T10:36:46.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:46 smithi173 bash[58816]: audit 2023-10-01T10:36:45.728879+0000 mon.c (mon.2) 112 : audit 0 from='client.? 172.21.15.78:0/593062341' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:46.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:46 smithi078 bash[75299]: audit 2023-10-01T10:36:45.107819+0000 mon.a (mon.0) 736 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:36:46.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:46 smithi078 bash[75299]: audit 2023-10-01T10:36:45.115273+0000 mon.a (mon.0) 737 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:36:46.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:46 smithi078 bash[75299]: audit 2023-10-01T10:36:45.126645+0000 mon.a (mon.0) 738 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:36:46.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:46 smithi078 bash[75299]: cluster 2023-10-01T10:36:45.610422+0000 mgr.y (mgr.25072) 551 : cluster 0 pgmap v460: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-01T10:36:46.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:46 smithi078 bash[75299]: audit 2023-10-01T10:36:45.728879+0000 mon.c (mon.2) 112 : audit 0 from='client.? 172.21.15.78:0/593062341' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:46.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:46 smithi078 bash[78274]: audit 2023-10-01T10:36:45.107819+0000 mon.a (mon.0) 736 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:36:46.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:46 smithi078 bash[78274]: audit 2023-10-01T10:36:45.115273+0000 mon.a (mon.0) 737 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:36:46.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:46 smithi078 bash[78274]: audit 2023-10-01T10:36:45.126645+0000 mon.a (mon.0) 738 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:36:46.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:46 smithi078 bash[78274]: cluster 2023-10-01T10:36:45.610422+0000 mgr.y (mgr.25072) 551 : cluster 0 pgmap v460: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-01T10:36:46.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:46 smithi078 bash[78274]: audit 2023-10-01T10:36:45.728879+0000 mon.c (mon.2) 112 : audit 0 from='client.? 172.21.15.78:0/593062341' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:47.082 INFO:journalctl@ceph.osd.2.smithi078.stdout:Oct 01 10:36:46 smithi078 bash[97687]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-01T10:36:47.082 INFO:journalctl@ceph.osd.2.smithi078.stdout:Oct 01 10:36:46 smithi078 bash[97687]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-10-01T10:36:47.082 INFO:journalctl@ceph.osd.2.smithi078.stdout:Oct 01 10:36:46 smithi078 bash[97687]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2023-10-01T10:36:47.083 INFO:journalctl@ceph.osd.2.smithi078.stdout:Oct 01 10:36:46 smithi078 bash[97687]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2023-10-01T10:36:47.083 INFO:journalctl@ceph.osd.2.smithi078.stdout:Oct 01 10:36:46 smithi078 bash[97687]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2023-10-01T10:36:47.083 INFO:journalctl@ceph.osd.2.smithi078.stdout:Oct 01 10:36:46 smithi078 bash[97687]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2023-10-01T10:36:47.083 INFO:journalctl@ceph.osd.2.smithi078.stdout:Oct 01 10:36:46 smithi078 bash[97687]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-10-01T10:36:47.083 INFO:journalctl@ceph.osd.2.smithi078.stdout:Oct 01 10:36:46 smithi078 bash[97687]: --> ceph-volume lvm activate successful for osd ID: 2 2023-10-01T10:36:47.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:47 smithi078 bash[75299]: cluster 2023-10-01T10:36:46.112185+0000 mon.a (mon.0) 739 : cluster 3 Health check failed: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded (PG_DEGRADED) 2023-10-01T10:36:47.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:47 smithi078 bash[78274]: cluster 2023-10-01T10:36:46.112185+0000 mon.a (mon.0) 739 : cluster 3 Health check failed: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded (PG_DEGRADED) 2023-10-01T10:36:47.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:47 smithi173 bash[58816]: cluster 2023-10-01T10:36:46.112185+0000 mon.a (mon.0) 739 : cluster 3 Health check failed: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded (PG_DEGRADED) 2023-10-01T10:36:48.716 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:48 smithi078 bash[75299]: audit 2023-10-01T10:36:47.410837+0000 mon.a (mon.0) 740 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:36:48.716 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:48 smithi078 bash[75299]: audit 2023-10-01T10:36:47.412062+0000 mon.a (mon.0) 741 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:36:48.716 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:48 smithi078 bash[75299]: cluster 2023-10-01T10:36:47.610963+0000 mgr.y (mgr.25072) 552 : cluster 0 pgmap v461: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-01T10:36:48.717 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:48 smithi078 bash[75299]: audit 2023-10-01T10:36:48.076996+0000 mon.c (mon.2) 113 : audit 0 from='client.? 172.21.15.78:0/87045762' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:48.717 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:48 smithi078 bash[78274]: audit 2023-10-01T10:36:47.410837+0000 mon.a (mon.0) 740 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:36:48.717 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:48 smithi078 bash[78274]: audit 2023-10-01T10:36:47.412062+0000 mon.a (mon.0) 741 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:36:48.717 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:48 smithi078 bash[78274]: cluster 2023-10-01T10:36:47.610963+0000 mgr.y (mgr.25072) 552 : cluster 0 pgmap v461: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-01T10:36:48.717 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:48 smithi078 bash[78274]: audit 2023-10-01T10:36:48.076996+0000 mon.c (mon.2) 113 : audit 0 from='client.? 172.21.15.78:0/87045762' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:48.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:48 smithi173 bash[58816]: audit 2023-10-01T10:36:47.410837+0000 mon.a (mon.0) 740 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:36:48.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:48 smithi173 bash[58816]: audit 2023-10-01T10:36:47.412062+0000 mon.a (mon.0) 741 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:36:48.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:48 smithi173 bash[58816]: cluster 2023-10-01T10:36:47.610963+0000 mgr.y (mgr.25072) 552 : cluster 0 pgmap v461: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-01T10:36:48.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:48 smithi173 bash[58816]: audit 2023-10-01T10:36:48.076996+0000 mon.c (mon.2) 113 : audit 0 from='client.? 172.21.15.78:0/87045762' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:49.082 INFO:journalctl@ceph.osd.2.smithi078.stdout:Oct 01 10:36:48 smithi078 bash[97994]: debug 2023-10-01T10:36:48.757+0000 7f9c932d2700 1 -- 172.21.15.78:0/3154967840 <== mon.2 v2:172.21.15.78:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5569bc8f84e0 con 0x5569bc952c00 2023-10-01T10:36:49.582 INFO:journalctl@ceph.osd.2.smithi078.stdout:Oct 01 10:36:49 smithi078 bash[97994]: debug 2023-10-01T10:36:49.309+0000 7f9c9c458640 -1 Falling back to public interface 2023-10-01T10:36:50.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:49 smithi078 bash[75299]: cluster 2023-10-01T10:36:49.611996+0000 mgr.y (mgr.25072) 553 : cluster 0 pgmap v462: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-01T10:36:50.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:49 smithi078 bash[78274]: cluster 2023-10-01T10:36:49.611996+0000 mgr.y (mgr.25072) 553 : cluster 0 pgmap v462: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-01T10:36:50.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:49 smithi173 bash[58816]: cluster 2023-10-01T10:36:49.611996+0000 mgr.y (mgr.25072) 553 : cluster 0 pgmap v462: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-01T10:36:50.582 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:36:50 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:36:50] "GET /metrics HTTP/1.1" 200 33827 "" "Prometheus/2.33.4" 2023-10-01T10:36:51.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:50 smithi078 bash[75299]: audit 2023-10-01T10:36:50.447305+0000 mon.a (mon.0) 742 : audit 0 from='client.? 172.21.15.78:0/3059458039' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:51.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:50 smithi078 bash[78274]: audit 2023-10-01T10:36:50.447305+0000 mon.a (mon.0) 742 : audit 0 from='client.? 172.21.15.78:0/3059458039' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:51.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:50 smithi173 bash[58816]: audit 2023-10-01T10:36:50.447305+0000 mon.a (mon.0) 742 : audit 0 from='client.? 172.21.15.78:0/3059458039' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:52.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:51 smithi078 bash[78274]: cluster 2023-10-01T10:36:51.612864+0000 mgr.y (mgr.25072) 554 : cluster 0 pgmap v463: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-01T10:36:52.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:51 smithi078 bash[75299]: cluster 2023-10-01T10:36:51.612864+0000 mgr.y (mgr.25072) 554 : cluster 0 pgmap v463: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-01T10:36:52.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:51 smithi173 bash[58816]: cluster 2023-10-01T10:36:51.612864+0000 mgr.y (mgr.25072) 554 : cluster 0 pgmap v463: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-01T10:36:53.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:53 smithi078 bash[78274]: audit 2023-10-01T10:36:52.317184+0000 mon.a (mon.0) 743 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:36:53.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:53 smithi078 bash[78274]: audit 2023-10-01T10:36:52.324629+0000 mon.a (mon.0) 744 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:36:53.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:53 smithi078 bash[78274]: audit 2023-10-01T10:36:52.850991+0000 mon.a (mon.0) 745 : audit 0 from='client.? 172.21.15.78:0/1817425343' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:53.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:53 smithi078 bash[78274]: audit 2023-10-01T10:36:53.168279+0000 mon.a (mon.0) 746 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:36:53.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:53 smithi078 bash[78274]: audit 2023-10-01T10:36:53.175699+0000 mon.a (mon.0) 747 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:36:53.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:53 smithi078 bash[75299]: audit 2023-10-01T10:36:52.317184+0000 mon.a (mon.0) 743 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:36:53.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:53 smithi078 bash[75299]: audit 2023-10-01T10:36:52.324629+0000 mon.a (mon.0) 744 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:36:53.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:53 smithi078 bash[75299]: audit 2023-10-01T10:36:52.850991+0000 mon.a (mon.0) 745 : audit 0 from='client.? 172.21.15.78:0/1817425343' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:53.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:53 smithi078 bash[75299]: audit 2023-10-01T10:36:53.168279+0000 mon.a (mon.0) 746 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:36:53.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:53 smithi078 bash[75299]: audit 2023-10-01T10:36:53.175699+0000 mon.a (mon.0) 747 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:36:53.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:53 smithi173 bash[58816]: audit 2023-10-01T10:36:52.317184+0000 mon.a (mon.0) 743 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:36:53.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:53 smithi173 bash[58816]: audit 2023-10-01T10:36:52.324629+0000 mon.a (mon.0) 744 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:36:53.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:53 smithi173 bash[58816]: audit 2023-10-01T10:36:52.850991+0000 mon.a (mon.0) 745 : audit 0 from='client.? 172.21.15.78:0/1817425343' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:53.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:53 smithi173 bash[58816]: audit 2023-10-01T10:36:53.168279+0000 mon.a (mon.0) 746 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:36:53.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:53 smithi173 bash[58816]: audit 2023-10-01T10:36:53.175699+0000 mon.a (mon.0) 747 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:36:54.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:54 smithi078 bash[75299]: cluster 2023-10-01T10:36:53.613498+0000 mgr.y (mgr.25072) 555 : cluster 0 pgmap v464: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-01T10:36:54.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:54 smithi078 bash[78274]: cluster 2023-10-01T10:36:53.613498+0000 mgr.y (mgr.25072) 555 : cluster 0 pgmap v464: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-01T10:36:54.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:54 smithi173 bash[58816]: cluster 2023-10-01T10:36:53.613498+0000 mgr.y (mgr.25072) 555 : cluster 0 pgmap v464: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-01T10:36:55.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:55 smithi078 bash[75299]: audit 2023-10-01T10:36:55.207980+0000 mon.c (mon.2) 114 : audit 0 from='client.? 172.21.15.78:0/3883552067' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:55.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:55 smithi078 bash[78274]: audit 2023-10-01T10:36:55.207980+0000 mon.c (mon.2) 114 : audit 0 from='client.? 172.21.15.78:0/3883552067' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:55.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:55 smithi173 bash[58816]: audit 2023-10-01T10:36:55.207980+0000 mon.c (mon.2) 114 : audit 0 from='client.? 172.21.15.78:0/3883552067' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:56.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:56 smithi173 bash[58816]: cluster 2023-10-01T10:36:55.614583+0000 mgr.y (mgr.25072) 556 : cluster 0 pgmap v465: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-01T10:36:56.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:56 smithi078 bash[75299]: cluster 2023-10-01T10:36:55.614583+0000 mgr.y (mgr.25072) 556 : cluster 0 pgmap v465: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-01T10:36:56.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:56 smithi078 bash[78274]: cluster 2023-10-01T10:36:55.614583+0000 mgr.y (mgr.25072) 556 : cluster 0 pgmap v465: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-01T10:36:58.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:57 smithi078 bash[78274]: audit 2023-10-01T10:36:57.575785+0000 mon.a (mon.0) 748 : audit 0 from='client.? 172.21.15.78:0/2054933248' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:58.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:57 smithi078 bash[78274]: cluster 2023-10-01T10:36:57.614990+0000 mgr.y (mgr.25072) 557 : cluster 0 pgmap v466: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-01T10:36:58.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:57 smithi078 bash[75299]: audit 2023-10-01T10:36:57.575785+0000 mon.a (mon.0) 748 : audit 0 from='client.? 172.21.15.78:0/2054933248' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:58.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:57 smithi078 bash[75299]: cluster 2023-10-01T10:36:57.614990+0000 mgr.y (mgr.25072) 557 : cluster 0 pgmap v466: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-01T10:36:58.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:57 smithi173 bash[58816]: audit 2023-10-01T10:36:57.575785+0000 mon.a (mon.0) 748 : audit 0 from='client.? 172.21.15.78:0/2054933248' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:36:58.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:57 smithi173 bash[58816]: cluster 2023-10-01T10:36:57.614990+0000 mgr.y (mgr.25072) 557 : cluster 0 pgmap v466: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-01T10:37:00.033 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:36:59 smithi078 bash[75299]: cluster 2023-10-01T10:36:59.615597+0000 mgr.y (mgr.25072) 558 : cluster 0 pgmap v467: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-01T10:37:00.034 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:36:59 smithi078 bash[78274]: cluster 2023-10-01T10:36:59.615597+0000 mgr.y (mgr.25072) 558 : cluster 0 pgmap v467: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-01T10:37:00.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:36:59 smithi173 bash[58816]: cluster 2023-10-01T10:36:59.615597+0000 mgr.y (mgr.25072) 558 : cluster 0 pgmap v467: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-01T10:37:00.332 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:37:00 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:37:00] "GET /metrics HTTP/1.1" 200 33827 "" "Prometheus/2.33.4" 2023-10-01T10:37:01.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:00 smithi078 bash[78274]: audit 2023-10-01T10:36:59.923427+0000 mon.c (mon.2) 115 : audit 0 from='client.? 172.21.15.78:0/1304235297' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:37:01.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:00 smithi078 bash[75299]: audit 2023-10-01T10:36:59.923427+0000 mon.c (mon.2) 115 : audit 0 from='client.? 172.21.15.78:0/1304235297' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:37:01.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:00 smithi173 bash[58816]: audit 2023-10-01T10:36:59.923427+0000 mon.c (mon.2) 115 : audit 0 from='client.? 172.21.15.78:0/1304235297' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:37:02.085 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:01 smithi078 bash[75299]: cluster 2023-10-01T10:37:01.616382+0000 mgr.y (mgr.25072) 559 : cluster 0 pgmap v468: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-01T10:37:02.085 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:01 smithi078 bash[78274]: cluster 2023-10-01T10:37:01.616382+0000 mgr.y (mgr.25072) 559 : cluster 0 pgmap v468: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-01T10:37:02.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:01 smithi173 bash[58816]: cluster 2023-10-01T10:37:01.616382+0000 mgr.y (mgr.25072) 559 : cluster 0 pgmap v468: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-01T10:37:02.582 INFO:journalctl@ceph.osd.2.smithi078.stdout:Oct 01 10:37:02 smithi078 bash[97994]: debug 2023-10-01T10:37:02.104+0000 7f9c9c458640 -1 osd.2 112 log_to_monitors true 2023-10-01T10:37:03.332 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:37:03 smithi078 bash[71214]: debug 2023-10-01T10:37:03.024+0000 7fd9c245e700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2023-10-01T10:37:03.333 INFO:journalctl@ceph.osd.2.smithi078.stdout:Oct 01 10:37:03 smithi078 bash[97994]: debug 2023-10-01T10:37:03.024+0000 7f9c8b8c0700 -1 osd.2 112 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-01T10:37:03.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:02 smithi078 bash[75299]: audit 2023-10-01T10:37:02.112374+0000 mon.c (mon.2) 116 : audit 1 from='osd.2 [v2:172.21.15.78:6818/4228246677,v1:172.21.15.78:6819/4228246677]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-01T10:37:03.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:02 smithi078 bash[75299]: audit 2023-10-01T10:37:02.112807+0000 mon.a (mon.0) 749 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-01T10:37:03.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:02 smithi078 bash[75299]: audit 2023-10-01T10:37:02.305456+0000 mon.a (mon.0) 750 : audit 0 from='client.? 172.21.15.78:0/399323455' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:37:03.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:02 smithi078 bash[75299]: audit 2023-10-01T10:37:02.406069+0000 mon.a (mon.0) 751 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:37:03.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:02 smithi078 bash[75299]: audit 2023-10-01T10:37:02.919267+0000 mon.a (mon.0) 752 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:03.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:02 smithi078 bash[75299]: audit 2023-10-01T10:37:02.926581+0000 mon.a (mon.0) 753 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:03.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:02 smithi078 bash[75299]: audit 2023-10-01T10:37:02.928090+0000 mon.a (mon.0) 754 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:37:03.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:02 smithi078 bash[75299]: audit 2023-10-01T10:37:02.929339+0000 mon.a (mon.0) 755 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:37:03.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:02 smithi078 bash[75299]: audit 2023-10-01T10:37:02.936317+0000 mon.a (mon.0) 756 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:03.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:02 smithi078 bash[78274]: audit 2023-10-01T10:37:02.112374+0000 mon.c (mon.2) 116 : audit 1 from='osd.2 [v2:172.21.15.78:6818/4228246677,v1:172.21.15.78:6819/4228246677]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-01T10:37:03.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:02 smithi078 bash[78274]: audit 2023-10-01T10:37:02.112807+0000 mon.a (mon.0) 749 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-01T10:37:03.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:02 smithi078 bash[78274]: audit 2023-10-01T10:37:02.305456+0000 mon.a (mon.0) 750 : audit 0 from='client.? 172.21.15.78:0/399323455' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:37:03.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:02 smithi078 bash[78274]: audit 2023-10-01T10:37:02.406069+0000 mon.a (mon.0) 751 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:37:03.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:02 smithi078 bash[78274]: audit 2023-10-01T10:37:02.919267+0000 mon.a (mon.0) 752 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:03.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:02 smithi078 bash[78274]: audit 2023-10-01T10:37:02.926581+0000 mon.a (mon.0) 753 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:03.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:02 smithi078 bash[78274]: audit 2023-10-01T10:37:02.928090+0000 mon.a (mon.0) 754 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:37:03.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:02 smithi078 bash[78274]: audit 2023-10-01T10:37:02.929339+0000 mon.a (mon.0) 755 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:37:03.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:02 smithi078 bash[78274]: audit 2023-10-01T10:37:02.936317+0000 mon.a (mon.0) 756 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:03.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:02 smithi173 bash[58816]: audit 2023-10-01T10:37:02.112374+0000 mon.c (mon.2) 116 : audit 1 from='osd.2 [v2:172.21.15.78:6818/4228246677,v1:172.21.15.78:6819/4228246677]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-01T10:37:03.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:02 smithi173 bash[58816]: audit 2023-10-01T10:37:02.112807+0000 mon.a (mon.0) 749 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-10-01T10:37:03.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:02 smithi173 bash[58816]: audit 2023-10-01T10:37:02.305456+0000 mon.a (mon.0) 750 : audit 0 from='client.? 172.21.15.78:0/399323455' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:37:03.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:02 smithi173 bash[58816]: audit 2023-10-01T10:37:02.406069+0000 mon.a (mon.0) 751 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:37:03.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:02 smithi173 bash[58816]: audit 2023-10-01T10:37:02.919267+0000 mon.a (mon.0) 752 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:03.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:02 smithi173 bash[58816]: audit 2023-10-01T10:37:02.926581+0000 mon.a (mon.0) 753 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:03.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:02 smithi173 bash[58816]: audit 2023-10-01T10:37:02.928090+0000 mon.a (mon.0) 754 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:37:03.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:02 smithi173 bash[58816]: audit 2023-10-01T10:37:02.929339+0000 mon.a (mon.0) 755 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:37:03.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:02 smithi173 bash[58816]: audit 2023-10-01T10:37:02.936317+0000 mon.a (mon.0) 756 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:04.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:03 smithi078 bash[78274]: cluster 2023-10-01T10:37:02.978370+0000 osd.3 (osd.3) 39 : cluster 3 failed to encode map e115 with expected crc 2023-10-01T10:37:04.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:03 smithi078 bash[78274]: audit 2023-10-01T10:37:02.981087+0000 mon.a (mon.0) 757 : audit 1 from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-01T10:37:04.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:03 smithi078 bash[78274]: cluster 2023-10-01T10:37:02.981207+0000 mon.a (mon.0) 758 : cluster 0 osdmap e115: 8 total, 7 up, 8 in 2023-10-01T10:37:04.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:03 smithi078 bash[78274]: audit 2023-10-01T10:37:02.987432+0000 mon.c (mon.2) 117 : audit 1 from='osd.2 [v2:172.21.15.78:6818/4228246677,v1:172.21.15.78:6819/4228246677]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2023-10-01T10:37:04.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:03 smithi078 bash[78274]: audit 2023-10-01T10:37:02.987923+0000 mon.a (mon.0) 759 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2023-10-01T10:37:04.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:03 smithi078 bash[78274]: audit 2023-10-01T10:37:03.018717+0000 mon.a (mon.0) 760 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:37:04.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:03 smithi078 bash[78274]: audit 2023-10-01T10:37:03.022352+0000 mon.a (mon.0) 761 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:37:04.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:03 smithi078 bash[78274]: audit 2023-10-01T10:37:03.024565+0000 mon.a (mon.0) 762 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:37:04.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:03 smithi078 bash[78274]: audit 2023-10-01T10:37:03.026212+0000 mon.a (mon.0) 763 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:37:04.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:03 smithi078 bash[78274]: audit 2023-10-01T10:37:03.028862+0000 mon.a (mon.0) 764 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-01T10:37:04.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:03 smithi078 bash[78274]: audit 2023-10-01T10:37:03.029093+0000 mgr.y (mgr.25072) 560 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-01T10:37:04.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:03 smithi078 bash[78274]: cephadm 2023-10-01T10:37:03.030186+0000 mgr.y (mgr.25072) 561 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2023-10-01T10:37:04.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:03 smithi078 bash[78274]: cluster 2023-10-01T10:37:03.616976+0000 mgr.y (mgr.25072) 562 : cluster 0 pgmap v470: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-01T10:37:04.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:03 smithi078 bash[75299]: cluster 2023-10-01T10:37:02.978370+0000 osd.3 (osd.3) 39 : cluster 3 failed to encode map e115 with expected crc 2023-10-01T10:37:04.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:03 smithi078 bash[75299]: audit 2023-10-01T10:37:02.981087+0000 mon.a (mon.0) 757 : audit 1 from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-01T10:37:04.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:03 smithi078 bash[75299]: cluster 2023-10-01T10:37:02.981207+0000 mon.a (mon.0) 758 : cluster 0 osdmap e115: 8 total, 7 up, 8 in 2023-10-01T10:37:04.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:03 smithi078 bash[75299]: audit 2023-10-01T10:37:02.987432+0000 mon.c (mon.2) 117 : audit 1 from='osd.2 [v2:172.21.15.78:6818/4228246677,v1:172.21.15.78:6819/4228246677]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2023-10-01T10:37:04.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:03 smithi078 bash[75299]: audit 2023-10-01T10:37:02.987923+0000 mon.a (mon.0) 759 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2023-10-01T10:37:04.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:03 smithi078 bash[75299]: audit 2023-10-01T10:37:03.018717+0000 mon.a (mon.0) 760 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:37:04.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:03 smithi078 bash[75299]: audit 2023-10-01T10:37:03.022352+0000 mon.a (mon.0) 761 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:37:04.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:03 smithi078 bash[75299]: audit 2023-10-01T10:37:03.024565+0000 mon.a (mon.0) 762 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:37:04.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:03 smithi078 bash[75299]: audit 2023-10-01T10:37:03.026212+0000 mon.a (mon.0) 763 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:37:04.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:03 smithi078 bash[75299]: audit 2023-10-01T10:37:03.028862+0000 mon.a (mon.0) 764 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-01T10:37:04.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:03 smithi078 bash[75299]: audit 2023-10-01T10:37:03.029093+0000 mgr.y (mgr.25072) 560 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-01T10:37:04.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:03 smithi078 bash[75299]: cephadm 2023-10-01T10:37:03.030186+0000 mgr.y (mgr.25072) 561 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2023-10-01T10:37:04.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:03 smithi078 bash[75299]: cluster 2023-10-01T10:37:03.616976+0000 mgr.y (mgr.25072) 562 : cluster 0 pgmap v470: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-01T10:37:04.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:03 smithi173 bash[58816]: cluster 2023-10-01T10:37:02.978370+0000 osd.3 (osd.3) 39 : cluster 3 failed to encode map e115 with expected crc 2023-10-01T10:37:04.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:03 smithi173 bash[58816]: audit 2023-10-01T10:37:02.981087+0000 mon.a (mon.0) 757 : audit 1 from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-10-01T10:37:04.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:03 smithi173 bash[58816]: cluster 2023-10-01T10:37:02.981207+0000 mon.a (mon.0) 758 : cluster 0 osdmap e115: 8 total, 7 up, 8 in 2023-10-01T10:37:04.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:03 smithi173 bash[58816]: audit 2023-10-01T10:37:02.987432+0000 mon.c (mon.2) 117 : audit 1 from='osd.2 [v2:172.21.15.78:6818/4228246677,v1:172.21.15.78:6819/4228246677]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2023-10-01T10:37:04.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:03 smithi173 bash[58816]: audit 2023-10-01T10:37:02.987923+0000 mon.a (mon.0) 759 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2023-10-01T10:37:04.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:03 smithi173 bash[58816]: audit 2023-10-01T10:37:03.018717+0000 mon.a (mon.0) 760 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:37:04.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:03 smithi173 bash[58816]: audit 2023-10-01T10:37:03.022352+0000 mon.a (mon.0) 761 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:37:04.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:03 smithi173 bash[58816]: audit 2023-10-01T10:37:03.024565+0000 mon.a (mon.0) 762 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:37:04.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:03 smithi173 bash[58816]: audit 2023-10-01T10:37:03.026212+0000 mon.a (mon.0) 763 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:37:04.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:03 smithi173 bash[58816]: audit 2023-10-01T10:37:03.028862+0000 mon.a (mon.0) 764 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-01T10:37:04.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:03 smithi173 bash[58816]: audit 2023-10-01T10:37:03.029093+0000 mgr.y (mgr.25072) 560 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-01T10:37:04.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:03 smithi173 bash[58816]: cephadm 2023-10-01T10:37:03.030186+0000 mgr.y (mgr.25072) 561 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2023-10-01T10:37:04.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:03 smithi173 bash[58816]: cluster 2023-10-01T10:37:03.616976+0000 mgr.y (mgr.25072) 562 : cluster 0 pgmap v470: 73 pgs: 2 active+undersized+degraded, 17 active+undersized, 54 active+clean; 582 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-01T10:37:05.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:04 smithi078 bash[75299]: cluster 2023-10-01T10:37:03.979054+0000 mon.a (mon.0) 765 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-01T10:37:05.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:04 smithi078 bash[75299]: cluster 2023-10-01T10:37:03.991348+0000 osd.3 (osd.3) 40 : cluster 3 failed to encode map e116 with expected crc 2023-10-01T10:37:05.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:04 smithi078 bash[75299]: cluster 2023-10-01T10:37:03.994019+0000 osd.7 (osd.7) 47 : cluster 3 failed to encode map e116 with expected crc 2023-10-01T10:37:05.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:04 smithi078 bash[75299]: cluster 2023-10-01T10:37:03.996820+0000 osd.3 (osd.3) 41 : cluster 3 failed to encode map e116 with expected crc 2023-10-01T10:37:05.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:04 smithi078 bash[75299]: cluster 2023-10-01T10:37:03.997525+0000 osd.4 (osd.4) 21 : cluster 3 failed to encode map e116 with expected crc 2023-10-01T10:37:05.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:04 smithi078 bash[75299]: cluster 2023-10-01T10:37:03.997573+0000 osd.6 (osd.6) 27 : cluster 3 failed to encode map e116 with expected crc 2023-10-01T10:37:05.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:04 smithi078 bash[75299]: cluster 2023-10-01T10:37:03.997774+0000 mon.a (mon.0) 766 : cluster 1 osd.2 [v2:172.21.15.78:6818/4228246677,v1:172.21.15.78:6819/4228246677] boot 2023-10-01T10:37:05.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:04 smithi078 bash[75299]: cluster 2023-10-01T10:37:03.997864+0000 mon.a (mon.0) 767 : cluster 0 osdmap e116: 8 total, 8 up, 8 in 2023-10-01T10:37:05.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:04 smithi078 bash[75299]: cluster 2023-10-01T10:37:03.998394+0000 osd.5 (osd.5) 32 : cluster 3 failed to encode map e116 with expected crc 2023-10-01T10:37:05.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:04 smithi078 bash[75299]: audit 2023-10-01T10:37:03.998652+0000 mon.a (mon.0) 768 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T10:37:05.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:04 smithi078 bash[75299]: cluster 2023-10-01T10:37:04.000826+0000 osd.6 (osd.6) 28 : cluster 3 failed to encode map e116 with expected crc 2023-10-01T10:37:05.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:04 smithi078 bash[75299]: audit 2023-10-01T10:37:04.663645+0000 mon.c (mon.2) 118 : audit 0 from='client.? 172.21.15.78:0/893747497' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:37:05.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:04 smithi078 bash[78274]: cluster 2023-10-01T10:37:03.979054+0000 mon.a (mon.0) 765 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-01T10:37:05.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:04 smithi078 bash[78274]: cluster 2023-10-01T10:37:03.991348+0000 osd.3 (osd.3) 40 : cluster 3 failed to encode map e116 with expected crc 2023-10-01T10:37:05.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:04 smithi078 bash[78274]: cluster 2023-10-01T10:37:03.994019+0000 osd.7 (osd.7) 47 : cluster 3 failed to encode map e116 with expected crc 2023-10-01T10:37:05.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:04 smithi078 bash[78274]: cluster 2023-10-01T10:37:03.996820+0000 osd.3 (osd.3) 41 : cluster 3 failed to encode map e116 with expected crc 2023-10-01T10:37:05.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:04 smithi078 bash[78274]: cluster 2023-10-01T10:37:03.997525+0000 osd.4 (osd.4) 21 : cluster 3 failed to encode map e116 with expected crc 2023-10-01T10:37:05.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:04 smithi078 bash[78274]: cluster 2023-10-01T10:37:03.997573+0000 osd.6 (osd.6) 27 : cluster 3 failed to encode map e116 with expected crc 2023-10-01T10:37:05.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:04 smithi078 bash[78274]: cluster 2023-10-01T10:37:03.997774+0000 mon.a (mon.0) 766 : cluster 1 osd.2 [v2:172.21.15.78:6818/4228246677,v1:172.21.15.78:6819/4228246677] boot 2023-10-01T10:37:05.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:04 smithi078 bash[78274]: cluster 2023-10-01T10:37:03.997864+0000 mon.a (mon.0) 767 : cluster 0 osdmap e116: 8 total, 8 up, 8 in 2023-10-01T10:37:05.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:04 smithi078 bash[78274]: cluster 2023-10-01T10:37:03.998394+0000 osd.5 (osd.5) 32 : cluster 3 failed to encode map e116 with expected crc 2023-10-01T10:37:05.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:04 smithi078 bash[78274]: audit 2023-10-01T10:37:03.998652+0000 mon.a (mon.0) 768 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T10:37:05.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:04 smithi078 bash[78274]: cluster 2023-10-01T10:37:04.000826+0000 osd.6 (osd.6) 28 : cluster 3 failed to encode map e116 with expected crc 2023-10-01T10:37:05.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:04 smithi078 bash[78274]: audit 2023-10-01T10:37:04.663645+0000 mon.c (mon.2) 118 : audit 0 from='client.? 172.21.15.78:0/893747497' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:37:05.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:04 smithi173 bash[58816]: cluster 2023-10-01T10:37:03.979054+0000 mon.a (mon.0) 765 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-01T10:37:05.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:04 smithi173 bash[58816]: cluster 2023-10-01T10:37:03.991348+0000 osd.3 (osd.3) 40 : cluster 3 failed to encode map e116 with expected crc 2023-10-01T10:37:05.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:04 smithi173 bash[58816]: cluster 2023-10-01T10:37:03.994019+0000 osd.7 (osd.7) 47 : cluster 3 failed to encode map e116 with expected crc 2023-10-01T10:37:05.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:04 smithi173 bash[58816]: cluster 2023-10-01T10:37:03.996820+0000 osd.3 (osd.3) 41 : cluster 3 failed to encode map e116 with expected crc 2023-10-01T10:37:05.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:04 smithi173 bash[58816]: cluster 2023-10-01T10:37:03.997525+0000 osd.4 (osd.4) 21 : cluster 3 failed to encode map e116 with expected crc 2023-10-01T10:37:05.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:04 smithi173 bash[58816]: cluster 2023-10-01T10:37:03.997573+0000 osd.6 (osd.6) 27 : cluster 3 failed to encode map e116 with expected crc 2023-10-01T10:37:05.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:04 smithi173 bash[58816]: cluster 2023-10-01T10:37:03.997774+0000 mon.a (mon.0) 766 : cluster 1 osd.2 [v2:172.21.15.78:6818/4228246677,v1:172.21.15.78:6819/4228246677] boot 2023-10-01T10:37:05.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:04 smithi173 bash[58816]: cluster 2023-10-01T10:37:03.997864+0000 mon.a (mon.0) 767 : cluster 0 osdmap e116: 8 total, 8 up, 8 in 2023-10-01T10:37:05.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:04 smithi173 bash[58816]: cluster 2023-10-01T10:37:03.998394+0000 osd.5 (osd.5) 32 : cluster 3 failed to encode map e116 with expected crc 2023-10-01T10:37:05.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:04 smithi173 bash[58816]: audit 2023-10-01T10:37:03.998652+0000 mon.a (mon.0) 768 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-10-01T10:37:05.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:04 smithi173 bash[58816]: cluster 2023-10-01T10:37:04.000826+0000 osd.6 (osd.6) 28 : cluster 3 failed to encode map e116 with expected crc 2023-10-01T10:37:05.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:04 smithi173 bash[58816]: audit 2023-10-01T10:37:04.663645+0000 mon.c (mon.2) 118 : audit 0 from='client.? 172.21.15.78:0/893747497' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:37:06.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:06 smithi078 bash[75299]: cluster 2023-10-01T10:37:05.003311+0000 osd.3 (osd.3) 42 : cluster 3 failed to encode map e117 with expected crc 2023-10-01T10:37:06.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:06 smithi078 bash[75299]: cluster 2023-10-01T10:37:05.006323+0000 mon.a (mon.0) 769 : cluster 0 osdmap e117: 8 total, 8 up, 8 in 2023-10-01T10:37:06.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:06 smithi078 bash[75299]: cluster 2023-10-01T10:37:05.006635+0000 osd.3 (osd.3) 43 : cluster 3 failed to encode map e117 with expected crc 2023-10-01T10:37:06.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:06 smithi078 bash[75299]: cluster 2023-10-01T10:37:05.007800+0000 osd.4 (osd.4) 22 : cluster 3 failed to encode map e117 with expected crc 2023-10-01T10:37:06.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:06 smithi078 bash[75299]: cluster 2023-10-01T10:37:05.009150+0000 osd.4 (osd.4) 23 : cluster 3 failed to encode map e117 with expected crc 2023-10-01T10:37:06.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:06 smithi078 bash[75299]: cluster 2023-10-01T10:37:05.009773+0000 osd.5 (osd.5) 33 : cluster 3 failed to encode map e117 with expected crc 2023-10-01T10:37:06.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:06 smithi078 bash[75299]: cluster 2023-10-01T10:37:05.010020+0000 osd.6 (osd.6) 29 : cluster 3 failed to encode map e117 with expected crc 2023-10-01T10:37:06.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:06 smithi078 bash[75299]: cluster 2023-10-01T10:37:05.010041+0000 osd.7 (osd.7) 48 : cluster 3 failed to encode map e117 with expected crc 2023-10-01T10:37:06.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:06 smithi078 bash[75299]: cluster 2023-10-01T10:37:05.012932+0000 osd.6 (osd.6) 30 : cluster 3 failed to encode map e117 with expected crc 2023-10-01T10:37:06.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:06 smithi078 bash[75299]: cluster 2023-10-01T10:37:05.013704+0000 osd.5 (osd.5) 34 : cluster 3 failed to encode map e117 with expected crc 2023-10-01T10:37:06.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:06 smithi078 bash[75299]: cluster 2023-10-01T10:37:05.013751+0000 osd.7 (osd.7) 49 : cluster 3 failed to encode map e117 with expected crc 2023-10-01T10:37:06.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:06 smithi078 bash[75299]: cluster 2023-10-01T10:37:05.016219+0000 osd.5 (osd.5) 35 : cluster 3 failed to encode map e117 with expected crc 2023-10-01T10:37:06.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:06 smithi078 bash[75299]: cluster 2023-10-01T10:37:05.617858+0000 mgr.y (mgr.25072) 563 : cluster 0 pgmap v473: 73 pgs: 2 peering, 1 active+undersized+degraded, 9 active+undersized, 61 active+clean; 582 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-01T10:37:06.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:06 smithi078 bash[78274]: cluster 2023-10-01T10:37:05.003311+0000 osd.3 (osd.3) 42 : cluster 3 failed to encode map e117 with expected crc 2023-10-01T10:37:06.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:06 smithi078 bash[78274]: cluster 2023-10-01T10:37:05.006323+0000 mon.a (mon.0) 769 : cluster 0 osdmap e117: 8 total, 8 up, 8 in 2023-10-01T10:37:06.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:06 smithi078 bash[78274]: cluster 2023-10-01T10:37:05.006635+0000 osd.3 (osd.3) 43 : cluster 3 failed to encode map e117 with expected crc 2023-10-01T10:37:06.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:06 smithi078 bash[78274]: cluster 2023-10-01T10:37:05.007800+0000 osd.4 (osd.4) 22 : cluster 3 failed to encode map e117 with expected crc 2023-10-01T10:37:06.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:06 smithi078 bash[78274]: cluster 2023-10-01T10:37:05.009150+0000 osd.4 (osd.4) 23 : cluster 3 failed to encode map e117 with expected crc 2023-10-01T10:37:06.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:06 smithi078 bash[78274]: cluster 2023-10-01T10:37:05.009773+0000 osd.5 (osd.5) 33 : cluster 3 failed to encode map e117 with expected crc 2023-10-01T10:37:06.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:06 smithi078 bash[78274]: cluster 2023-10-01T10:37:05.010020+0000 osd.6 (osd.6) 29 : cluster 3 failed to encode map e117 with expected crc 2023-10-01T10:37:06.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:06 smithi078 bash[78274]: cluster 2023-10-01T10:37:05.010041+0000 osd.7 (osd.7) 48 : cluster 3 failed to encode map e117 with expected crc 2023-10-01T10:37:06.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:06 smithi078 bash[78274]: cluster 2023-10-01T10:37:05.012932+0000 osd.6 (osd.6) 30 : cluster 3 failed to encode map e117 with expected crc 2023-10-01T10:37:06.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:06 smithi078 bash[78274]: cluster 2023-10-01T10:37:05.013704+0000 osd.5 (osd.5) 34 : cluster 3 failed to encode map e117 with expected crc 2023-10-01T10:37:06.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:06 smithi078 bash[78274]: cluster 2023-10-01T10:37:05.013751+0000 osd.7 (osd.7) 49 : cluster 3 failed to encode map e117 with expected crc 2023-10-01T10:37:06.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:06 smithi078 bash[78274]: cluster 2023-10-01T10:37:05.016219+0000 osd.5 (osd.5) 35 : cluster 3 failed to encode map e117 with expected crc 2023-10-01T10:37:06.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:06 smithi078 bash[78274]: cluster 2023-10-01T10:37:05.617858+0000 mgr.y (mgr.25072) 563 : cluster 0 pgmap v473: 73 pgs: 2 peering, 1 active+undersized+degraded, 9 active+undersized, 61 active+clean; 582 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-01T10:37:06.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:06 smithi173 bash[58816]: cluster 2023-10-01T10:37:05.003311+0000 osd.3 (osd.3) 42 : cluster 3 failed to encode map e117 with expected crc 2023-10-01T10:37:06.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:06 smithi173 bash[58816]: cluster 2023-10-01T10:37:05.006323+0000 mon.a (mon.0) 769 : cluster 0 osdmap e117: 8 total, 8 up, 8 in 2023-10-01T10:37:06.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:06 smithi173 bash[58816]: cluster 2023-10-01T10:37:05.006635+0000 osd.3 (osd.3) 43 : cluster 3 failed to encode map e117 with expected crc 2023-10-01T10:37:06.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:06 smithi173 bash[58816]: cluster 2023-10-01T10:37:05.007800+0000 osd.4 (osd.4) 22 : cluster 3 failed to encode map e117 with expected crc 2023-10-01T10:37:06.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:06 smithi173 bash[58816]: cluster 2023-10-01T10:37:05.009150+0000 osd.4 (osd.4) 23 : cluster 3 failed to encode map e117 with expected crc 2023-10-01T10:37:06.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:06 smithi173 bash[58816]: cluster 2023-10-01T10:37:05.009773+0000 osd.5 (osd.5) 33 : cluster 3 failed to encode map e117 with expected crc 2023-10-01T10:37:06.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:06 smithi173 bash[58816]: cluster 2023-10-01T10:37:05.010020+0000 osd.6 (osd.6) 29 : cluster 3 failed to encode map e117 with expected crc 2023-10-01T10:37:06.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:06 smithi173 bash[58816]: cluster 2023-10-01T10:37:05.010041+0000 osd.7 (osd.7) 48 : cluster 3 failed to encode map e117 with expected crc 2023-10-01T10:37:06.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:06 smithi173 bash[58816]: cluster 2023-10-01T10:37:05.012932+0000 osd.6 (osd.6) 30 : cluster 3 failed to encode map e117 with expected crc 2023-10-01T10:37:06.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:06 smithi173 bash[58816]: cluster 2023-10-01T10:37:05.013704+0000 osd.5 (osd.5) 34 : cluster 3 failed to encode map e117 with expected crc 2023-10-01T10:37:06.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:06 smithi173 bash[58816]: cluster 2023-10-01T10:37:05.013751+0000 osd.7 (osd.7) 49 : cluster 3 failed to encode map e117 with expected crc 2023-10-01T10:37:06.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:06 smithi173 bash[58816]: cluster 2023-10-01T10:37:05.016219+0000 osd.5 (osd.5) 35 : cluster 3 failed to encode map e117 with expected crc 2023-10-01T10:37:06.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:06 smithi173 bash[58816]: cluster 2023-10-01T10:37:05.617858+0000 mgr.y (mgr.25072) 563 : cluster 0 pgmap v473: 73 pgs: 2 peering, 1 active+undersized+degraded, 9 active+undersized, 61 active+clean; 582 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-01T10:37:07.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:07 smithi078 bash[78274]: cluster 2023-10-01T10:37:06.002658+0000 mon.a (mon.0) 770 : cluster 3 Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2023-10-01T10:37:07.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:07 smithi078 bash[75299]: cluster 2023-10-01T10:37:06.002658+0000 mon.a (mon.0) 770 : cluster 3 Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2023-10-01T10:37:07.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:07 smithi173 bash[58816]: cluster 2023-10-01T10:37:06.002658+0000 mon.a (mon.0) 770 : cluster 3 Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2023-10-01T10:37:08.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:08 smithi078 bash[75299]: audit 2023-10-01T10:37:07.028156+0000 mon.a (mon.0) 771 : audit 0 from='client.? 172.21.15.78:0/375029269' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:37:08.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:08 smithi078 bash[75299]: cluster 2023-10-01T10:37:07.618502+0000 mgr.y (mgr.25072) 564 : cluster 0 pgmap v474: 73 pgs: 2 peering, 1 active+undersized+degraded, 9 active+undersized, 61 active+clean; 582 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-01T10:37:08.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:08 smithi078 bash[78274]: audit 2023-10-01T10:37:07.028156+0000 mon.a (mon.0) 771 : audit 0 from='client.? 172.21.15.78:0/375029269' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:37:08.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:08 smithi078 bash[78274]: cluster 2023-10-01T10:37:07.618502+0000 mgr.y (mgr.25072) 564 : cluster 0 pgmap v474: 73 pgs: 2 peering, 1 active+undersized+degraded, 9 active+undersized, 61 active+clean; 582 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-01T10:37:08.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:08 smithi173 bash[58816]: audit 2023-10-01T10:37:07.028156+0000 mon.a (mon.0) 771 : audit 0 from='client.? 172.21.15.78:0/375029269' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:37:08.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:08 smithi173 bash[58816]: cluster 2023-10-01T10:37:07.618502+0000 mgr.y (mgr.25072) 564 : cluster 0 pgmap v474: 73 pgs: 2 peering, 1 active+undersized+degraded, 9 active+undersized, 61 active+clean; 582 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-01T10:37:09.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:09 smithi078 bash[75299]: audit 2023-10-01T10:37:09.396976+0000 mon.c (mon.2) 119 : audit 0 from='client.? 172.21.15.78:0/2422114621' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:37:09.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:09 smithi078 bash[78274]: audit 2023-10-01T10:37:09.396976+0000 mon.c (mon.2) 119 : audit 0 from='client.? 172.21.15.78:0/2422114621' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:37:09.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:09 smithi173 bash[58816]: audit 2023-10-01T10:37:09.396976+0000 mon.c (mon.2) 119 : audit 0 from='client.? 172.21.15.78:0/2422114621' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:37:10.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:10 smithi078 bash[78274]: cluster 2023-10-01T10:37:09.619455+0000 mgr.y (mgr.25072) 565 : cluster 0 pgmap v475: 73 pgs: 2 peering, 71 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:37:10.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:10 smithi078 bash[75299]: cluster 2023-10-01T10:37:09.619455+0000 mgr.y (mgr.25072) 565 : cluster 0 pgmap v475: 73 pgs: 2 peering, 71 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:37:10.583 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:37:10 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:37:10] "GET /metrics HTTP/1.1" 200 33831 "" "Prometheus/2.33.4" 2023-10-01T10:37:10.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:10 smithi173 bash[58816]: cluster 2023-10-01T10:37:09.619455+0000 mgr.y (mgr.25072) 565 : cluster 0 pgmap v475: 73 pgs: 2 peering, 71 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:37:11.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:11 smithi078 bash[75299]: cluster 2023-10-01T10:37:10.450380+0000 mon.a (mon.0) 772 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded) 2023-10-01T10:37:11.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:11 smithi078 bash[75299]: cluster 2023-10-01T10:37:10.450439+0000 mon.a (mon.0) 773 : cluster 1 Cluster is now healthy 2023-10-01T10:37:11.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:11 smithi078 bash[78274]: cluster 2023-10-01T10:37:10.450380+0000 mon.a (mon.0) 772 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded) 2023-10-01T10:37:11.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:11 smithi078 bash[78274]: cluster 2023-10-01T10:37:10.450439+0000 mon.a (mon.0) 773 : cluster 1 Cluster is now healthy 2023-10-01T10:37:11.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:11 smithi173 bash[58816]: cluster 2023-10-01T10:37:10.450380+0000 mon.a (mon.0) 772 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded) 2023-10-01T10:37:11.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:11 smithi173 bash[58816]: cluster 2023-10-01T10:37:10.450439+0000 mon.a (mon.0) 773 : cluster 1 Cluster is now healthy 2023-10-01T10:37:12.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:12 smithi078 bash[75299]: cluster 2023-10-01T10:37:11.620297+0000 mgr.y (mgr.25072) 566 : cluster 0 pgmap v476: 73 pgs: 73 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:37:12.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:12 smithi078 bash[75299]: audit 2023-10-01T10:37:11.762442+0000 mon.c (mon.2) 120 : audit 0 from='client.? 172.21.15.78:0/3845758334' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:37:12.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:12 smithi078 bash[78274]: cluster 2023-10-01T10:37:11.620297+0000 mgr.y (mgr.25072) 566 : cluster 0 pgmap v476: 73 pgs: 73 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:37:12.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:12 smithi078 bash[78274]: audit 2023-10-01T10:37:11.762442+0000 mon.c (mon.2) 120 : audit 0 from='client.? 172.21.15.78:0/3845758334' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:37:12.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:12 smithi173 bash[58816]: cluster 2023-10-01T10:37:11.620297+0000 mgr.y (mgr.25072) 566 : cluster 0 pgmap v476: 73 pgs: 73 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:37:12.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:12 smithi173 bash[58816]: audit 2023-10-01T10:37:11.762442+0000 mon.c (mon.2) 120 : audit 0 from='client.? 172.21.15.78:0/3845758334' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:37:14.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:13 smithi078 bash[75299]: cluster 2023-10-01T10:37:13.621078+0000 mgr.y (mgr.25072) 567 : cluster 0 pgmap v477: 73 pgs: 73 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:37:14.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:13 smithi078 bash[78274]: cluster 2023-10-01T10:37:13.621078+0000 mgr.y (mgr.25072) 567 : cluster 0 pgmap v477: 73 pgs: 73 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:37:14.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:13 smithi173 bash[58816]: cluster 2023-10-01T10:37:13.621078+0000 mgr.y (mgr.25072) 567 : cluster 0 pgmap v477: 73 pgs: 73 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:37:15.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:14 smithi078 bash[78274]: audit 2023-10-01T10:37:14.134188+0000 mon.a (mon.0) 774 : audit 0 from='client.? 172.21.15.78:0/3037308097' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:37:15.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:14 smithi078 bash[75299]: audit 2023-10-01T10:37:14.134188+0000 mon.a (mon.0) 774 : audit 0 from='client.? 172.21.15.78:0/3037308097' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:37:15.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:14 smithi173 bash[58816]: audit 2023-10-01T10:37:14.134188+0000 mon.a (mon.0) 774 : audit 0 from='client.? 172.21.15.78:0/3037308097' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:37:16.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:15 smithi078 bash[78274]: cluster 2023-10-01T10:37:15.622171+0000 mgr.y (mgr.25072) 568 : cluster 0 pgmap v478: 73 pgs: 73 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:37:16.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:15 smithi078 bash[75299]: cluster 2023-10-01T10:37:15.622171+0000 mgr.y (mgr.25072) 568 : cluster 0 pgmap v478: 73 pgs: 73 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:37:16.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:15 smithi173 bash[58816]: cluster 2023-10-01T10:37:15.622171+0000 mgr.y (mgr.25072) 568 : cluster 0 pgmap v478: 73 pgs: 73 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:37:17.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:16 smithi078 bash[78274]: audit 2023-10-01T10:37:16.499177+0000 mon.c (mon.2) 121 : audit 0 from='client.? 172.21.15.78:0/898587458' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:37:17.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:16 smithi078 bash[75299]: audit 2023-10-01T10:37:16.499177+0000 mon.c (mon.2) 121 : audit 0 from='client.? 172.21.15.78:0/898587458' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:37:17.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:16 smithi173 bash[58816]: audit 2023-10-01T10:37:16.499177+0000 mon.c (mon.2) 121 : audit 0 from='client.? 172.21.15.78:0/898587458' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:37:18.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:18 smithi078 bash[75299]: audit 2023-10-01T10:37:17.412854+0000 mon.a (mon.0) 775 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:18.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:18 smithi078 bash[75299]: audit 2023-10-01T10:37:17.414604+0000 mon.a (mon.0) 776 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:37:18.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:18 smithi078 bash[75299]: cluster 2023-10-01T10:37:17.622801+0000 mgr.y (mgr.25072) 569 : cluster 0 pgmap v479: 73 pgs: 73 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:37:18.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:18 smithi078 bash[75299]: audit 2023-10-01T10:37:17.625913+0000 mon.a (mon.0) 777 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:18.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:18 smithi078 bash[75299]: audit 2023-10-01T10:37:18.044922+0000 mon.a (mon.0) 778 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-01T10:37:18.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:18 smithi078 bash[78274]: audit 2023-10-01T10:37:17.412854+0000 mon.a (mon.0) 775 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:18.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:18 smithi078 bash[78274]: audit 2023-10-01T10:37:17.414604+0000 mon.a (mon.0) 776 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:37:18.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:18 smithi078 bash[78274]: cluster 2023-10-01T10:37:17.622801+0000 mgr.y (mgr.25072) 569 : cluster 0 pgmap v479: 73 pgs: 73 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:37:18.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:18 smithi078 bash[78274]: audit 2023-10-01T10:37:17.625913+0000 mon.a (mon.0) 777 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:18.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:18 smithi078 bash[78274]: audit 2023-10-01T10:37:18.044922+0000 mon.a (mon.0) 778 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-01T10:37:18.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:18 smithi173 bash[58816]: audit 2023-10-01T10:37:17.412854+0000 mon.a (mon.0) 775 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:18.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:18 smithi173 bash[58816]: audit 2023-10-01T10:37:17.414604+0000 mon.a (mon.0) 776 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:37:18.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:18 smithi173 bash[58816]: cluster 2023-10-01T10:37:17.622801+0000 mgr.y (mgr.25072) 569 : cluster 0 pgmap v479: 73 pgs: 73 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:37:18.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:18 smithi173 bash[58816]: audit 2023-10-01T10:37:17.625913+0000 mon.a (mon.0) 777 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:18.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:18 smithi173 bash[58816]: audit 2023-10-01T10:37:18.044922+0000 mon.a (mon.0) 778 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-01T10:37:19.764 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:19 smithi078 bash[78274]: audit 2023-10-01T10:37:18.045381+0000 mgr.y (mgr.25072) 570 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-01T10:37:19.764 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:19 smithi078 bash[78274]: cephadm 2023-10-01T10:37:18.047754+0000 mgr.y (mgr.25072) 571 : cephadm 1 Upgrade: osd.3 is safe to restart 2023-10-01T10:37:19.764 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:19 smithi078 bash[78274]: audit 2023-10-01T10:37:18.851293+0000 mon.c (mon.2) 122 : audit 0 from='client.? 172.21.15.78:0/3136249772' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:37:19.764 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:19 smithi078 bash[75299]: audit 2023-10-01T10:37:18.045381+0000 mgr.y (mgr.25072) 570 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-01T10:37:19.764 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:19 smithi078 bash[75299]: cephadm 2023-10-01T10:37:18.047754+0000 mgr.y (mgr.25072) 571 : cephadm 1 Upgrade: osd.3 is safe to restart 2023-10-01T10:37:19.764 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:19 smithi078 bash[75299]: audit 2023-10-01T10:37:18.851293+0000 mon.c (mon.2) 122 : audit 0 from='client.? 172.21.15.78:0/3136249772' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:37:20.122 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:19 smithi173 bash[58816]: audit 2023-10-01T10:37:18.045381+0000 mgr.y (mgr.25072) 570 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-10-01T10:37:20.122 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:19 smithi173 bash[58816]: cephadm 2023-10-01T10:37:18.047754+0000 mgr.y (mgr.25072) 571 : cephadm 1 Upgrade: osd.3 is safe to restart 2023-10-01T10:37:20.123 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:19 smithi173 bash[58816]: audit 2023-10-01T10:37:18.851293+0000 mon.c (mon.2) 122 : audit 0 from='client.? 172.21.15.78:0/3136249772' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:37:20.582 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:37:20 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:37:20] "GET /metrics HTTP/1.1" 200 33828 "" "Prometheus/2.33.4" 2023-10-01T10:37:20.879 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:20 smithi078 bash[78274]: cluster 2023-10-01T10:37:19.623510+0000 mgr.y (mgr.25072) 572 : cluster 0 pgmap v480: 73 pgs: 73 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:37:20.880 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:20 smithi078 bash[78274]: audit 2023-10-01T10:37:20.576755+0000 mon.a (mon.0) 779 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:20.880 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:20 smithi078 bash[78274]: audit 2023-10-01T10:37:20.578500+0000 mon.a (mon.0) 780 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-01T10:37:20.880 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:20 smithi078 bash[78274]: audit 2023-10-01T10:37:20.579823+0000 mon.a (mon.0) 781 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:37:20.880 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:20 smithi078 bash[75299]: cluster 2023-10-01T10:37:19.623510+0000 mgr.y (mgr.25072) 572 : cluster 0 pgmap v480: 73 pgs: 73 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:37:20.880 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:20 smithi078 bash[75299]: audit 2023-10-01T10:37:20.576755+0000 mon.a (mon.0) 779 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:20.880 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:20 smithi078 bash[75299]: audit 2023-10-01T10:37:20.578500+0000 mon.a (mon.0) 780 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-01T10:37:20.880 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:20 smithi078 bash[75299]: audit 2023-10-01T10:37:20.579823+0000 mon.a (mon.0) 781 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:37:21.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:20 smithi173 bash[58816]: cluster 2023-10-01T10:37:19.623510+0000 mgr.y (mgr.25072) 572 : cluster 0 pgmap v480: 73 pgs: 73 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:37:21.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:20 smithi173 bash[58816]: audit 2023-10-01T10:37:20.576755+0000 mon.a (mon.0) 779 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:21.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:20 smithi173 bash[58816]: audit 2023-10-01T10:37:20.578500+0000 mon.a (mon.0) 780 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-10-01T10:37:21.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:20 smithi173 bash[58816]: audit 2023-10-01T10:37:20.579823+0000 mon.a (mon.0) 781 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:37:21.970 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:21 smithi078 bash[78274]: cephadm 2023-10-01T10:37:20.568529+0000 mgr.y (mgr.25072) 573 : cephadm 1 Upgrade: Updating osd.3 2023-10-01T10:37:21.970 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:21 smithi078 bash[78274]: cephadm 2023-10-01T10:37:20.583582+0000 mgr.y (mgr.25072) 574 : cephadm 1 Deploying daemon osd.3 on smithi078 2023-10-01T10:37:21.970 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:21 smithi078 bash[78274]: audit 2023-10-01T10:37:21.217472+0000 mon.a (mon.0) 782 : audit 0 from='client.? 172.21.15.78:0/1573400645' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:37:21.970 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:21 smithi078 bash[75299]: cephadm 2023-10-01T10:37:20.568529+0000 mgr.y (mgr.25072) 573 : cephadm 1 Upgrade: Updating osd.3 2023-10-01T10:37:21.970 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:21 smithi078 bash[75299]: cephadm 2023-10-01T10:37:20.583582+0000 mgr.y (mgr.25072) 574 : cephadm 1 Deploying daemon osd.3 on smithi078 2023-10-01T10:37:21.970 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:21 smithi078 bash[75299]: audit 2023-10-01T10:37:21.217472+0000 mon.a (mon.0) 782 : audit 0 from='client.? 172.21.15.78:0/1573400645' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:37:22.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:21 smithi173 bash[58816]: cephadm 2023-10-01T10:37:20.568529+0000 mgr.y (mgr.25072) 573 : cephadm 1 Upgrade: Updating osd.3 2023-10-01T10:37:22.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:21 smithi173 bash[58816]: cephadm 2023-10-01T10:37:20.583582+0000 mgr.y (mgr.25072) 574 : cephadm 1 Deploying daemon osd.3 on smithi078 2023-10-01T10:37:22.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:21 smithi173 bash[58816]: audit 2023-10-01T10:37:21.217472+0000 mon.a (mon.0) 782 : audit 0 from='client.? 172.21.15.78:0/1573400645' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:37:22.950 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:22 smithi078 bash[75299]: cluster 2023-10-01T10:37:21.624185+0000 mgr.y (mgr.25072) 575 : cluster 0 pgmap v481: 73 pgs: 73 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:37:22.950 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:22 smithi078 bash[78274]: cluster 2023-10-01T10:37:21.624185+0000 mgr.y (mgr.25072) 575 : cluster 0 pgmap v481: 73 pgs: 73 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:37:23.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:22 smithi173 bash[58816]: cluster 2023-10-01T10:37:21.624185+0000 mgr.y (mgr.25072) 575 : cluster 0 pgmap v481: 73 pgs: 73 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:37:24.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:23 smithi078 bash[75299]: audit 2023-10-01T10:37:23.578726+0000 mon.a (mon.0) 783 : audit 0 from='client.? 172.21.15.78:0/4039713985' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:37:24.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:23 smithi078 bash[78274]: audit 2023-10-01T10:37:23.578726+0000 mon.a (mon.0) 783 : audit 0 from='client.? 172.21.15.78:0/4039713985' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:37:24.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:23 smithi173 bash[58816]: audit 2023-10-01T10:37:23.578726+0000 mon.a (mon.0) 783 : audit 0 from='client.? 172.21.15.78:0/4039713985' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:37:24.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:24 smithi078 bash[75299]: cluster 2023-10-01T10:37:23.624841+0000 mgr.y (mgr.25072) 576 : cluster 0 pgmap v482: 73 pgs: 73 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:37:24.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:24 smithi078 bash[78274]: cluster 2023-10-01T10:37:23.624841+0000 mgr.y (mgr.25072) 576 : cluster 0 pgmap v482: 73 pgs: 73 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:37:25.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:24 smithi173 bash[58816]: cluster 2023-10-01T10:37:23.624841+0000 mgr.y (mgr.25072) 576 : cluster 0 pgmap v482: 73 pgs: 73 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:37:25.582 INFO:journalctl@ceph.osd.3.smithi078.stdout:Oct 01 10:37:25 smithi078 systemd[1]: Stopping Ceph osd.3 for d8016480-6041-11ee-8db5-212e2dc638e7... 2023-10-01T10:37:25.884 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:25 smithi078 bash[78274]: cluster 2023-10-01T10:37:25.618539+0000 mon.a (mon.0) 784 : cluster 1 osd.3 marked itself down and dead 2023-10-01T10:37:25.884 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:25 smithi078 bash[75299]: cluster 2023-10-01T10:37:25.618539+0000 mon.a (mon.0) 784 : cluster 1 osd.3 marked itself down and dead 2023-10-01T10:37:25.885 INFO:journalctl@ceph.osd.3.smithi078.stdout:Oct 01 10:37:25 smithi078 bash[49409]: debug 2023-10-01T10:37:25.616+0000 7f7502652700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-01T10:37:25.885 INFO:journalctl@ceph.osd.3.smithi078.stdout:Oct 01 10:37:25 smithi078 bash[49409]: debug 2023-10-01T10:37:25.616+0000 7f7502652700 -1 osd.3 117 *** Got signal Terminated *** 2023-10-01T10:37:25.885 INFO:journalctl@ceph.osd.3.smithi078.stdout:Oct 01 10:37:25 smithi078 bash[49409]: debug 2023-10-01T10:37:25.616+0000 7f7502652700 -1 osd.3 117 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-01T10:37:26.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:25 smithi173 bash[58816]: cluster 2023-10-01T10:37:25.618539+0000 mon.a (mon.0) 784 : cluster 1 osd.3 marked itself down and dead 2023-10-01T10:37:26.728 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:26 smithi078 bash[78274]: cluster 2023-10-01T10:37:25.625748+0000 mgr.y (mgr.25072) 577 : cluster 0 pgmap v483: 73 pgs: 73 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:37:26.728 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:26 smithi078 bash[78274]: cluster 2023-10-01T10:37:25.653243+0000 mon.a (mon.0) 785 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-01T10:37:26.728 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:26 smithi078 bash[78274]: cluster 2023-10-01T10:37:25.665970+0000 osd.4 (osd.4) 24 : cluster 3 failed to encode map e118 with expected crc 2023-10-01T10:37:26.728 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:26 smithi078 bash[78274]: cluster 2023-10-01T10:37:25.668531+0000 osd.7 (osd.7) 50 : cluster 3 failed to encode map e118 with expected crc 2023-10-01T10:37:26.728 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:26 smithi078 bash[78274]: cluster 2023-10-01T10:37:25.669141+0000 osd.4 (osd.4) 25 : cluster 3 failed to encode map e118 with expected crc 2023-10-01T10:37:26.728 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:26 smithi078 bash[78274]: cluster 2023-10-01T10:37:25.670275+0000 osd.6 (osd.6) 31 : cluster 3 failed to encode map e118 with expected crc 2023-10-01T10:37:26.728 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:26 smithi078 bash[78274]: cluster 2023-10-01T10:37:25.670465+0000 osd.7 (osd.7) 51 : cluster 3 failed to encode map e118 with expected crc 2023-10-01T10:37:26.728 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:26 smithi078 bash[78274]: cluster 2023-10-01T10:37:25.671246+0000 mon.a (mon.0) 786 : cluster 0 osdmap e118: 8 total, 7 up, 8 in 2023-10-01T10:37:26.729 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:26 smithi078 bash[78274]: cluster 2023-10-01T10:37:25.673687+0000 osd.5 (osd.5) 36 : cluster 3 failed to encode map e118 with expected crc 2023-10-01T10:37:26.729 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:26 smithi078 bash[78274]: cluster 2023-10-01T10:37:25.677861+0000 osd.5 (osd.5) 37 : cluster 3 failed to encode map e118 with expected crc 2023-10-01T10:37:26.729 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:26 smithi078 bash[78274]: audit 2023-10-01T10:37:25.957877+0000 mon.a (mon.0) 787 : audit 0 from='client.? 172.21.15.78:0/4229726993' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:37:26.729 INFO:journalctl@ceph.osd.3.smithi078.stdout:Oct 01 10:37:26 smithi078 bash[100359]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd-3 2023-10-01T10:37:26.729 INFO:journalctl@ceph.osd.3.smithi078.stdout:Oct 01 10:37:26 smithi078 bash[100451]: Error response from daemon: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd-3 2023-10-01T10:37:26.729 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:26 smithi078 bash[75299]: cluster 2023-10-01T10:37:25.625748+0000 mgr.y (mgr.25072) 577 : cluster 0 pgmap v483: 73 pgs: 73 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:37:26.729 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:26 smithi078 bash[75299]: cluster 2023-10-01T10:37:25.653243+0000 mon.a (mon.0) 785 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-01T10:37:26.730 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:26 smithi078 bash[75299]: cluster 2023-10-01T10:37:25.665970+0000 osd.4 (osd.4) 24 : cluster 3 failed to encode map e118 with expected crc 2023-10-01T10:37:26.730 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:26 smithi078 bash[75299]: cluster 2023-10-01T10:37:25.668531+0000 osd.7 (osd.7) 50 : cluster 3 failed to encode map e118 with expected crc 2023-10-01T10:37:26.730 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:26 smithi078 bash[75299]: cluster 2023-10-01T10:37:25.669141+0000 osd.4 (osd.4) 25 : cluster 3 failed to encode map e118 with expected crc 2023-10-01T10:37:26.730 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:26 smithi078 bash[75299]: cluster 2023-10-01T10:37:25.670275+0000 osd.6 (osd.6) 31 : cluster 3 failed to encode map e118 with expected crc 2023-10-01T10:37:26.730 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:26 smithi078 bash[75299]: cluster 2023-10-01T10:37:25.670465+0000 osd.7 (osd.7) 51 : cluster 3 failed to encode map e118 with expected crc 2023-10-01T10:37:26.730 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:26 smithi078 bash[75299]: cluster 2023-10-01T10:37:25.671246+0000 mon.a (mon.0) 786 : cluster 0 osdmap e118: 8 total, 7 up, 8 in 2023-10-01T10:37:26.730 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:26 smithi078 bash[75299]: cluster 2023-10-01T10:37:25.673687+0000 osd.5 (osd.5) 36 : cluster 3 failed to encode map e118 with expected crc 2023-10-01T10:37:26.730 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:26 smithi078 bash[75299]: cluster 2023-10-01T10:37:25.677861+0000 osd.5 (osd.5) 37 : cluster 3 failed to encode map e118 with expected crc 2023-10-01T10:37:26.730 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:26 smithi078 bash[75299]: audit 2023-10-01T10:37:25.957877+0000 mon.a (mon.0) 787 : audit 0 from='client.? 172.21.15.78:0/4229726993' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:37:27.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:26 smithi173 bash[58816]: cluster 2023-10-01T10:37:25.625748+0000 mgr.y (mgr.25072) 577 : cluster 0 pgmap v483: 73 pgs: 73 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:37:27.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:26 smithi173 bash[58816]: cluster 2023-10-01T10:37:25.653243+0000 mon.a (mon.0) 785 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-01T10:37:27.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:26 smithi173 bash[58816]: cluster 2023-10-01T10:37:25.665970+0000 osd.4 (osd.4) 24 : cluster 3 failed to encode map e118 with expected crc 2023-10-01T10:37:27.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:26 smithi173 bash[58816]: cluster 2023-10-01T10:37:25.668531+0000 osd.7 (osd.7) 50 : cluster 3 failed to encode map e118 with expected crc 2023-10-01T10:37:27.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:26 smithi173 bash[58816]: cluster 2023-10-01T10:37:25.669141+0000 osd.4 (osd.4) 25 : cluster 3 failed to encode map e118 with expected crc 2023-10-01T10:37:27.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:26 smithi173 bash[58816]: cluster 2023-10-01T10:37:25.670275+0000 osd.6 (osd.6) 31 : cluster 3 failed to encode map e118 with expected crc 2023-10-01T10:37:27.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:26 smithi173 bash[58816]: cluster 2023-10-01T10:37:25.670465+0000 osd.7 (osd.7) 51 : cluster 3 failed to encode map e118 with expected crc 2023-10-01T10:37:27.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:26 smithi173 bash[58816]: cluster 2023-10-01T10:37:25.671246+0000 mon.a (mon.0) 786 : cluster 0 osdmap e118: 8 total, 7 up, 8 in 2023-10-01T10:37:27.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:26 smithi173 bash[58816]: cluster 2023-10-01T10:37:25.673687+0000 osd.5 (osd.5) 36 : cluster 3 failed to encode map e118 with expected crc 2023-10-01T10:37:27.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:26 smithi173 bash[58816]: cluster 2023-10-01T10:37:25.677861+0000 osd.5 (osd.5) 37 : cluster 3 failed to encode map e118 with expected crc 2023-10-01T10:37:27.122 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:26 smithi173 bash[58816]: audit 2023-10-01T10:37:25.957877+0000 mon.a (mon.0) 787 : audit 0 from='client.? 172.21.15.78:0/4229726993' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:37:27.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:27 smithi078 bash[78274]: cluster 2023-10-01T10:37:26.666094+0000 osd.4 (osd.4) 26 : cluster 3 failed to encode map e119 with expected crc 2023-10-01T10:37:27.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:27 smithi078 bash[78274]: cluster 2023-10-01T10:37:26.666389+0000 osd.7 (osd.7) 52 : cluster 3 failed to encode map e119 with expected crc 2023-10-01T10:37:27.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:27 smithi078 bash[78274]: cluster 2023-10-01T10:37:26.669602+0000 osd.4 (osd.4) 27 : cluster 3 failed to encode map e119 with expected crc 2023-10-01T10:37:27.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:27 smithi078 bash[78274]: cluster 2023-10-01T10:37:26.671023+0000 osd.7 (osd.7) 53 : cluster 3 failed to encode map e119 with expected crc 2023-10-01T10:37:27.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:27 smithi078 bash[78274]: cluster 2023-10-01T10:37:26.671410+0000 mon.a (mon.0) 788 : cluster 0 osdmap e119: 8 total, 7 up, 8 in 2023-10-01T10:37:27.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:27 smithi078 bash[78274]: cluster 2023-10-01T10:37:26.673489+0000 osd.4 (osd.4) 28 : cluster 3 failed to encode map e119 with expected crc 2023-10-01T10:37:27.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:27 smithi078 bash[78274]: cluster 2023-10-01T10:37:26.673527+0000 osd.5 (osd.5) 38 : cluster 3 failed to encode map e119 with expected crc 2023-10-01T10:37:27.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:27 smithi078 bash[78274]: cluster 2023-10-01T10:37:26.675461+0000 osd.4 (osd.4) 29 : cluster 3 failed to encode map e119 with expected crc 2023-10-01T10:37:27.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:27 smithi078 bash[78274]: cluster 2023-10-01T10:37:26.676811+0000 osd.5 (osd.5) 39 : cluster 3 failed to encode map e119 with expected crc 2023-10-01T10:37:27.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:27 smithi078 bash[78274]: cluster 2023-10-01T10:37:26.677416+0000 osd.4 (osd.4) 30 : cluster 3 failed to encode map e119 with expected crc 2023-10-01T10:37:27.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:27 smithi078 bash[78274]: cluster 2023-10-01T10:37:26.679488+0000 osd.6 (osd.6) 32 : cluster 3 failed to encode map e119 with expected crc 2023-10-01T10:37:27.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:27 smithi078 bash[78274]: cluster 2023-10-01T10:37:26.681019+0000 osd.6 (osd.6) 33 : cluster 3 failed to encode map e119 with expected crc 2023-10-01T10:37:27.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:27 smithi078 bash[78274]: cluster 2023-10-01T10:37:27.626146+0000 mgr.y (mgr.25072) 578 : cluster 0 pgmap v486: 73 pgs: 13 stale+active+clean, 60 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:37:27.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:27 smithi078 bash[75299]: cluster 2023-10-01T10:37:26.666094+0000 osd.4 (osd.4) 26 : cluster 3 failed to encode map e119 with expected crc 2023-10-01T10:37:27.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:27 smithi078 bash[75299]: cluster 2023-10-01T10:37:26.666389+0000 osd.7 (osd.7) 52 : cluster 3 failed to encode map e119 with expected crc 2023-10-01T10:37:27.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:27 smithi078 bash[75299]: cluster 2023-10-01T10:37:26.669602+0000 osd.4 (osd.4) 27 : cluster 3 failed to encode map e119 with expected crc 2023-10-01T10:37:27.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:27 smithi078 bash[75299]: cluster 2023-10-01T10:37:26.671023+0000 osd.7 (osd.7) 53 : cluster 3 failed to encode map e119 with expected crc 2023-10-01T10:37:27.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:27 smithi078 bash[75299]: cluster 2023-10-01T10:37:26.671410+0000 mon.a (mon.0) 788 : cluster 0 osdmap e119: 8 total, 7 up, 8 in 2023-10-01T10:37:27.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:27 smithi078 bash[75299]: cluster 2023-10-01T10:37:26.673489+0000 osd.4 (osd.4) 28 : cluster 3 failed to encode map e119 with expected crc 2023-10-01T10:37:27.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:27 smithi078 bash[75299]: cluster 2023-10-01T10:37:26.673527+0000 osd.5 (osd.5) 38 : cluster 3 failed to encode map e119 with expected crc 2023-10-01T10:37:27.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:27 smithi078 bash[75299]: cluster 2023-10-01T10:37:26.675461+0000 osd.4 (osd.4) 29 : cluster 3 failed to encode map e119 with expected crc 2023-10-01T10:37:27.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:27 smithi078 bash[75299]: cluster 2023-10-01T10:37:26.676811+0000 osd.5 (osd.5) 39 : cluster 3 failed to encode map e119 with expected crc 2023-10-01T10:37:27.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:27 smithi078 bash[75299]: cluster 2023-10-01T10:37:26.677416+0000 osd.4 (osd.4) 30 : cluster 3 failed to encode map e119 with expected crc 2023-10-01T10:37:27.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:27 smithi078 bash[75299]: cluster 2023-10-01T10:37:26.679488+0000 osd.6 (osd.6) 32 : cluster 3 failed to encode map e119 with expected crc 2023-10-01T10:37:27.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:27 smithi078 bash[75299]: cluster 2023-10-01T10:37:26.681019+0000 osd.6 (osd.6) 33 : cluster 3 failed to encode map e119 with expected crc 2023-10-01T10:37:27.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:27 smithi078 bash[75299]: cluster 2023-10-01T10:37:27.626146+0000 mgr.y (mgr.25072) 578 : cluster 0 pgmap v486: 73 pgs: 13 stale+active+clean, 60 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:37:28.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:27 smithi173 bash[58816]: cluster 2023-10-01T10:37:26.666094+0000 osd.4 (osd.4) 26 : cluster 3 failed to encode map e119 with expected crc 2023-10-01T10:37:28.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:27 smithi173 bash[58816]: cluster 2023-10-01T10:37:26.666389+0000 osd.7 (osd.7) 52 : cluster 3 failed to encode map e119 with expected crc 2023-10-01T10:37:28.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:27 smithi173 bash[58816]: cluster 2023-10-01T10:37:26.669602+0000 osd.4 (osd.4) 27 : cluster 3 failed to encode map e119 with expected crc 2023-10-01T10:37:28.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:27 smithi173 bash[58816]: cluster 2023-10-01T10:37:26.671023+0000 osd.7 (osd.7) 53 : cluster 3 failed to encode map e119 with expected crc 2023-10-01T10:37:28.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:27 smithi173 bash[58816]: cluster 2023-10-01T10:37:26.671410+0000 mon.a (mon.0) 788 : cluster 0 osdmap e119: 8 total, 7 up, 8 in 2023-10-01T10:37:28.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:27 smithi173 bash[58816]: cluster 2023-10-01T10:37:26.673489+0000 osd.4 (osd.4) 28 : cluster 3 failed to encode map e119 with expected crc 2023-10-01T10:37:28.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:27 smithi173 bash[58816]: cluster 2023-10-01T10:37:26.673527+0000 osd.5 (osd.5) 38 : cluster 3 failed to encode map e119 with expected crc 2023-10-01T10:37:28.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:27 smithi173 bash[58816]: cluster 2023-10-01T10:37:26.675461+0000 osd.4 (osd.4) 29 : cluster 3 failed to encode map e119 with expected crc 2023-10-01T10:37:28.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:27 smithi173 bash[58816]: cluster 2023-10-01T10:37:26.676811+0000 osd.5 (osd.5) 39 : cluster 3 failed to encode map e119 with expected crc 2023-10-01T10:37:28.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:27 smithi173 bash[58816]: cluster 2023-10-01T10:37:26.677416+0000 osd.4 (osd.4) 30 : cluster 3 failed to encode map e119 with expected crc 2023-10-01T10:37:28.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:27 smithi173 bash[58816]: cluster 2023-10-01T10:37:26.679488+0000 osd.6 (osd.6) 32 : cluster 3 failed to encode map e119 with expected crc 2023-10-01T10:37:28.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:27 smithi173 bash[58816]: cluster 2023-10-01T10:37:26.681019+0000 osd.6 (osd.6) 33 : cluster 3 failed to encode map e119 with expected crc 2023-10-01T10:37:28.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:27 smithi173 bash[58816]: cluster 2023-10-01T10:37:27.626146+0000 mgr.y (mgr.25072) 578 : cluster 0 pgmap v486: 73 pgs: 13 stale+active+clean, 60 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:37:29.157 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:28 smithi078 bash[78274]: audit 2023-10-01T10:37:28.319501+0000 mon.c (mon.2) 123 : audit 0 from='client.? 172.21.15.78:0/772975251' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:37:29.158 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:28 smithi078 bash[75299]: audit 2023-10-01T10:37:28.319501+0000 mon.c (mon.2) 123 : audit 0 from='client.? 172.21.15.78:0/772975251' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:37:29.158 INFO:journalctl@ceph.osd.3.smithi078.stdout:Oct 01 10:37:28 smithi078 systemd[1]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7@osd.3.service: Succeeded. 2023-10-01T10:37:29.158 INFO:journalctl@ceph.osd.3.smithi078.stdout:Oct 01 10:37:28 smithi078 systemd[1]: Stopped Ceph osd.3 for d8016480-6041-11ee-8db5-212e2dc638e7. 2023-10-01T10:37:29.158 INFO:journalctl@ceph.osd.3.smithi078.stdout:Oct 01 10:37:29 smithi078 systemd[1]: Started Ceph osd.3 for d8016480-6041-11ee-8db5-212e2dc638e7. 2023-10-01T10:37:29.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:28 smithi173 bash[58816]: audit 2023-10-01T10:37:28.319501+0000 mon.c (mon.2) 123 : audit 0 from='client.? 172.21.15.78:0/772975251' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:37:30.582 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:37:30 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:37:30] "GET /metrics HTTP/1.1" 200 33828 "" "Prometheus/2.33.4" 2023-10-01T10:37:30.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:30 smithi078 bash[75299]: audit 2023-10-01T10:37:29.133524+0000 mon.a (mon.0) 789 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:30.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:30 smithi078 bash[75299]: audit 2023-10-01T10:37:29.141591+0000 mon.a (mon.0) 790 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:30.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:30 smithi078 bash[75299]: audit 2023-10-01T10:37:29.165456+0000 mon.a (mon.0) 791 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:37:30.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:30 smithi078 bash[75299]: audit 2023-10-01T10:37:29.541318+0000 mon.a (mon.0) 792 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:30.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:30 smithi078 bash[75299]: audit 2023-10-01T10:37:29.548976+0000 mon.a (mon.0) 793 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:30.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:30 smithi078 bash[75299]: cluster 2023-10-01T10:37:29.626744+0000 mgr.y (mgr.25072) 579 : cluster 0 pgmap v487: 73 pgs: 2 active+undersized+degraded, 5 stale+active+clean, 17 active+undersized, 49 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-01T10:37:30.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:30 smithi078 bash[78274]: audit 2023-10-01T10:37:29.133524+0000 mon.a (mon.0) 789 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:30.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:30 smithi078 bash[78274]: audit 2023-10-01T10:37:29.141591+0000 mon.a (mon.0) 790 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:30.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:30 smithi078 bash[78274]: audit 2023-10-01T10:37:29.165456+0000 mon.a (mon.0) 791 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:37:30.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:30 smithi078 bash[78274]: audit 2023-10-01T10:37:29.541318+0000 mon.a (mon.0) 792 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:30.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:30 smithi078 bash[78274]: audit 2023-10-01T10:37:29.548976+0000 mon.a (mon.0) 793 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:30.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:30 smithi078 bash[78274]: cluster 2023-10-01T10:37:29.626744+0000 mgr.y (mgr.25072) 579 : cluster 0 pgmap v487: 73 pgs: 2 active+undersized+degraded, 5 stale+active+clean, 17 active+undersized, 49 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-01T10:37:30.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:30 smithi173 bash[58816]: audit 2023-10-01T10:37:29.133524+0000 mon.a (mon.0) 789 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:30.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:30 smithi173 bash[58816]: audit 2023-10-01T10:37:29.141591+0000 mon.a (mon.0) 790 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:30.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:30 smithi173 bash[58816]: audit 2023-10-01T10:37:29.165456+0000 mon.a (mon.0) 791 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:37:30.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:30 smithi173 bash[58816]: audit 2023-10-01T10:37:29.541318+0000 mon.a (mon.0) 792 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:30.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:30 smithi173 bash[58816]: audit 2023-10-01T10:37:29.548976+0000 mon.a (mon.0) 793 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:30.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:30 smithi173 bash[58816]: cluster 2023-10-01T10:37:29.626744+0000 mgr.y (mgr.25072) 579 : cluster 0 pgmap v487: 73 pgs: 2 active+undersized+degraded, 5 stale+active+clean, 17 active+undersized, 49 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2023-10-01T10:37:31.082 INFO:journalctl@ceph.osd.3.smithi078.stdout:Oct 01 10:37:30 smithi078 bash[100631]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-01T10:37:31.082 INFO:journalctl@ceph.osd.3.smithi078.stdout:Oct 01 10:37:30 smithi078 bash[100631]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-10-01T10:37:31.082 INFO:journalctl@ceph.osd.3.smithi078.stdout:Oct 01 10:37:30 smithi078 bash[100631]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2023-10-01T10:37:31.082 INFO:journalctl@ceph.osd.3.smithi078.stdout:Oct 01 10:37:30 smithi078 bash[100631]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2023-10-01T10:37:31.082 INFO:journalctl@ceph.osd.3.smithi078.stdout:Oct 01 10:37:30 smithi078 bash[100631]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2023-10-01T10:37:31.082 INFO:journalctl@ceph.osd.3.smithi078.stdout:Oct 01 10:37:30 smithi078 bash[100631]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2023-10-01T10:37:31.083 INFO:journalctl@ceph.osd.3.smithi078.stdout:Oct 01 10:37:30 smithi078 bash[100631]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-10-01T10:37:31.083 INFO:journalctl@ceph.osd.3.smithi078.stdout:Oct 01 10:37:30 smithi078 bash[100631]: --> ceph-volume lvm activate successful for osd ID: 3 2023-10-01T10:37:31.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:31 smithi078 bash[78274]: cluster 2023-10-01T10:37:30.545631+0000 mon.a (mon.0) 794 : cluster 3 Health check failed: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2023-10-01T10:37:31.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:31 smithi078 bash[78274]: audit 2023-10-01T10:37:30.692604+0000 mon.c (mon.2) 124 : audit 0 from='client.? 172.21.15.78:0/2896339182' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:37:31.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:31 smithi078 bash[75299]: cluster 2023-10-01T10:37:30.545631+0000 mon.a (mon.0) 794 : cluster 3 Health check failed: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2023-10-01T10:37:31.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:31 smithi078 bash[75299]: audit 2023-10-01T10:37:30.692604+0000 mon.c (mon.2) 124 : audit 0 from='client.? 172.21.15.78:0/2896339182' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:37:31.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:31 smithi173 bash[58816]: cluster 2023-10-01T10:37:30.545631+0000 mon.a (mon.0) 794 : cluster 3 Health check failed: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2023-10-01T10:37:31.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:31 smithi173 bash[58816]: audit 2023-10-01T10:37:30.692604+0000 mon.c (mon.2) 124 : audit 0 from='client.? 172.21.15.78:0/2896339182' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:37:32.446 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:32 smithi078 bash[75299]: cluster 2023-10-01T10:37:31.627641+0000 mgr.y (mgr.25072) 580 : cluster 0 pgmap v488: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-01T10:37:32.446 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:32 smithi078 bash[78274]: cluster 2023-10-01T10:37:31.627641+0000 mgr.y (mgr.25072) 580 : cluster 0 pgmap v488: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-01T10:37:32.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:32 smithi173 bash[58816]: cluster 2023-10-01T10:37:31.627641+0000 mgr.y (mgr.25072) 580 : cluster 0 pgmap v488: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-01T10:37:32.832 INFO:journalctl@ceph.osd.3.smithi078.stdout:Oct 01 10:37:32 smithi078 bash[100923]: debug 2023-10-01T10:37:32.568+0000 7f2e7a0bc700 1 -- 172.21.15.78:0/1604074814 <== mon.2 v2:172.21.15.78:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55b8dbd044e0 con 0x55b8dbd5e800 2023-10-01T10:37:33.414 INFO:journalctl@ceph.osd.3.smithi078.stdout:Oct 01 10:37:33 smithi078 bash[100923]: debug 2023-10-01T10:37:33.124+0000 7f2e83242640 -1 Falling back to public interface 2023-10-01T10:37:33.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:33 smithi078 bash[75299]: audit 2023-10-01T10:37:32.414609+0000 mon.a (mon.0) 795 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:33.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:33 smithi078 bash[75299]: audit 2023-10-01T10:37:32.415885+0000 mon.a (mon.0) 796 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:37:33.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:33 smithi078 bash[75299]: audit 2023-10-01T10:37:33.054553+0000 mon.a (mon.0) 797 : audit 0 from='client.? 172.21.15.78:0/449892903' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:37:33.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:33 smithi078 bash[78274]: audit 2023-10-01T10:37:32.414609+0000 mon.a (mon.0) 795 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:33.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:33 smithi078 bash[78274]: audit 2023-10-01T10:37:32.415885+0000 mon.a (mon.0) 796 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:37:33.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:33 smithi078 bash[78274]: audit 2023-10-01T10:37:33.054553+0000 mon.a (mon.0) 797 : audit 0 from='client.? 172.21.15.78:0/449892903' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:37:33.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:33 smithi173 bash[58816]: audit 2023-10-01T10:37:32.414609+0000 mon.a (mon.0) 795 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:33.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:33 smithi173 bash[58816]: audit 2023-10-01T10:37:32.415885+0000 mon.a (mon.0) 796 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:37:33.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:33 smithi173 bash[58816]: audit 2023-10-01T10:37:33.054553+0000 mon.a (mon.0) 797 : audit 0 from='client.? 172.21.15.78:0/449892903' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:37:34.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:34 smithi078 bash[75299]: cluster 2023-10-01T10:37:33.628440+0000 mgr.y (mgr.25072) 581 : cluster 0 pgmap v489: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-01T10:37:34.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:34 smithi078 bash[78274]: cluster 2023-10-01T10:37:33.628440+0000 mgr.y (mgr.25072) 581 : cluster 0 pgmap v489: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-01T10:37:34.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:34 smithi173 bash[58816]: cluster 2023-10-01T10:37:33.628440+0000 mgr.y (mgr.25072) 581 : cluster 0 pgmap v489: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-01T10:37:35.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:35 smithi078 bash[75299]: audit 2023-10-01T10:37:35.470401+0000 mon.c (mon.2) 125 : audit 0 from='client.? 172.21.15.78:0/1284990388' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:37:35.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:35 smithi078 bash[78274]: audit 2023-10-01T10:37:35.470401+0000 mon.c (mon.2) 125 : audit 0 from='client.? 172.21.15.78:0/1284990388' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:37:35.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:35 smithi173 bash[58816]: audit 2023-10-01T10:37:35.470401+0000 mon.c (mon.2) 125 : audit 0 from='client.? 172.21.15.78:0/1284990388' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:37:36.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:36 smithi078 bash[78274]: cluster 2023-10-01T10:37:35.629150+0000 mgr.y (mgr.25072) 582 : cluster 0 pgmap v490: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-01T10:37:36.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:36 smithi078 bash[78274]: audit 2023-10-01T10:37:35.792584+0000 mon.a (mon.0) 798 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:36.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:36 smithi078 bash[78274]: audit 2023-10-01T10:37:35.801607+0000 mon.a (mon.0) 799 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:36.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:36 smithi078 bash[78274]: audit 2023-10-01T10:37:36.310552+0000 mon.a (mon.0) 800 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:36.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:36 smithi078 bash[78274]: audit 2023-10-01T10:37:36.318820+0000 mon.a (mon.0) 801 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:36.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:36 smithi078 bash[75299]: cluster 2023-10-01T10:37:35.629150+0000 mgr.y (mgr.25072) 582 : cluster 0 pgmap v490: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-01T10:37:36.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:36 smithi078 bash[75299]: audit 2023-10-01T10:37:35.792584+0000 mon.a (mon.0) 798 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:36.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:36 smithi078 bash[75299]: audit 2023-10-01T10:37:35.801607+0000 mon.a (mon.0) 799 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:36.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:36 smithi078 bash[75299]: audit 2023-10-01T10:37:36.310552+0000 mon.a (mon.0) 800 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:36.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:36 smithi078 bash[75299]: audit 2023-10-01T10:37:36.318820+0000 mon.a (mon.0) 801 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:36.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:36 smithi173 bash[58816]: cluster 2023-10-01T10:37:35.629150+0000 mgr.y (mgr.25072) 582 : cluster 0 pgmap v490: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-01T10:37:36.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:36 smithi173 bash[58816]: audit 2023-10-01T10:37:35.792584+0000 mon.a (mon.0) 798 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:36.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:36 smithi173 bash[58816]: audit 2023-10-01T10:37:35.801607+0000 mon.a (mon.0) 799 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:36.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:36 smithi173 bash[58816]: audit 2023-10-01T10:37:36.310552+0000 mon.a (mon.0) 800 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:36.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:36 smithi173 bash[58816]: audit 2023-10-01T10:37:36.318820+0000 mon.a (mon.0) 801 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:37.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:37 smithi078 bash[75299]: cluster 2023-10-01T10:37:36.971469+0000 mon.a (mon.0) 802 : cluster 3 Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T10:37:37.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:37 smithi078 bash[75299]: audit 2023-10-01T10:37:37.112531+0000 mon.a (mon.0) 803 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:37.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:37 smithi078 bash[75299]: audit 2023-10-01T10:37:37.121968+0000 mon.a (mon.0) 804 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:37.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:37 smithi078 bash[78274]: cluster 2023-10-01T10:37:36.971469+0000 mon.a (mon.0) 802 : cluster 3 Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T10:37:37.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:37 smithi078 bash[78274]: audit 2023-10-01T10:37:37.112531+0000 mon.a (mon.0) 803 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:37.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:37 smithi078 bash[78274]: audit 2023-10-01T10:37:37.121968+0000 mon.a (mon.0) 804 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:37.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:37 smithi173 bash[58816]: cluster 2023-10-01T10:37:36.971469+0000 mon.a (mon.0) 802 : cluster 3 Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T10:37:37.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:37 smithi173 bash[58816]: audit 2023-10-01T10:37:37.112531+0000 mon.a (mon.0) 803 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:37.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:37 smithi173 bash[58816]: audit 2023-10-01T10:37:37.121968+0000 mon.a (mon.0) 804 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:38.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:38 smithi078 bash[75299]: cluster 2023-10-01T10:37:37.629680+0000 mgr.y (mgr.25072) 583 : cluster 0 pgmap v491: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-01T10:37:38.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:38 smithi078 bash[75299]: audit 2023-10-01T10:37:37.823040+0000 mon.a (mon.0) 805 : audit 0 from='client.? 172.21.15.78:0/3711173522' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:37:38.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:38 smithi078 bash[78274]: cluster 2023-10-01T10:37:37.629680+0000 mgr.y (mgr.25072) 583 : cluster 0 pgmap v491: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-01T10:37:38.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:38 smithi078 bash[78274]: audit 2023-10-01T10:37:37.823040+0000 mon.a (mon.0) 805 : audit 0 from='client.? 172.21.15.78:0/3711173522' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:37:38.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:38 smithi173 bash[58816]: cluster 2023-10-01T10:37:37.629680+0000 mgr.y (mgr.25072) 583 : cluster 0 pgmap v491: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-01T10:37:38.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:38 smithi173 bash[58816]: audit 2023-10-01T10:37:37.823040+0000 mon.a (mon.0) 805 : audit 0 from='client.? 172.21.15.78:0/3711173522' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:37:40.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:39 smithi078 bash[75299]: cluster 2023-10-01T10:37:39.630628+0000 mgr.y (mgr.25072) 584 : cluster 0 pgmap v492: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-01T10:37:40.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:39 smithi078 bash[78274]: cluster 2023-10-01T10:37:39.630628+0000 mgr.y (mgr.25072) 584 : cluster 0 pgmap v492: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-01T10:37:40.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:39 smithi173 bash[58816]: cluster 2023-10-01T10:37:39.630628+0000 mgr.y (mgr.25072) 584 : cluster 0 pgmap v492: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-01T10:37:40.540 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:37:40 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:37:40] "GET /metrics HTTP/1.1" 200 33832 "" "Prometheus/2.33.4" 2023-10-01T10:37:40.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:40 smithi078 bash[78274]: audit 2023-10-01T10:37:40.160883+0000 mon.a (mon.0) 806 : audit 0 from='client.? 172.21.15.78:0/3953828723' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:37:40.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:40 smithi078 bash[78274]: audit 2023-10-01T10:37:40.541967+0000 mon.a (mon.0) 807 : audit 1 from='osd.3 [v2:172.21.15.78:6826/2365141372,v1:172.21.15.78:6827/2365141372]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-01T10:37:40.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:40 smithi078 bash[75299]: audit 2023-10-01T10:37:40.160883+0000 mon.a (mon.0) 806 : audit 0 from='client.? 172.21.15.78:0/3953828723' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:37:40.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:40 smithi078 bash[75299]: audit 2023-10-01T10:37:40.541967+0000 mon.a (mon.0) 807 : audit 1 from='osd.3 [v2:172.21.15.78:6826/2365141372,v1:172.21.15.78:6827/2365141372]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-01T10:37:40.833 INFO:journalctl@ceph.osd.3.smithi078.stdout:Oct 01 10:37:40 smithi078 bash[100923]: debug 2023-10-01T10:37:40.536+0000 7f2e83242640 -1 osd.3 117 log_to_monitors true 2023-10-01T10:37:40.833 INFO:journalctl@ceph.osd.3.smithi078.stdout:Oct 01 10:37:40 smithi078 bash[100923]: debug 2023-10-01T10:37:40.760+0000 7f2e73ead700 -1 osd.3 117 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-01T10:37:41.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:40 smithi173 bash[58816]: audit 2023-10-01T10:37:40.160883+0000 mon.a (mon.0) 806 : audit 0 from='client.? 172.21.15.78:0/3953828723' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:37:41.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:40 smithi173 bash[58816]: audit 2023-10-01T10:37:40.541967+0000 mon.a (mon.0) 807 : audit 1 from='osd.3 [v2:172.21.15.78:6826/2365141372,v1:172.21.15.78:6827/2365141372]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-10-01T10:37:41.999 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:41 smithi078 bash[75299]: cluster 2023-10-01T10:37:40.720418+0000 osd.5 (osd.5) 40 : cluster 3 failed to encode map e120 with expected crc 2023-10-01T10:37:41.999 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:41 smithi078 bash[75299]: cluster 2023-10-01T10:37:40.723159+0000 osd.7 (osd.7) 54 : cluster 3 failed to encode map e120 with expected crc 2023-10-01T10:37:42.000 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:41 smithi078 bash[75299]: audit 2023-10-01T10:37:40.723308+0000 mon.a (mon.0) 808 : audit 1 from='osd.3 [v2:172.21.15.78:6826/2365141372,v1:172.21.15.78:6827/2365141372]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-01T10:37:42.000 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:41 smithi078 bash[75299]: cluster 2023-10-01T10:37:40.723429+0000 mon.a (mon.0) 809 : cluster 0 osdmap e120: 8 total, 7 up, 8 in 2023-10-01T10:37:42.000 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:41 smithi078 bash[75299]: audit 2023-10-01T10:37:40.724417+0000 mon.a (mon.0) 810 : audit 1 from='osd.3 [v2:172.21.15.78:6826/2365141372,v1:172.21.15.78:6827/2365141372]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2023-10-01T10:37:42.000 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:41 smithi078 bash[75299]: cluster 2023-10-01T10:37:41.631396+0000 mgr.y (mgr.25072) 585 : cluster 0 pgmap v494: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-01T10:37:42.000 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:41 smithi078 bash[78274]: cluster 2023-10-01T10:37:40.720418+0000 osd.5 (osd.5) 40 : cluster 3 failed to encode map e120 with expected crc 2023-10-01T10:37:42.000 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:41 smithi078 bash[78274]: cluster 2023-10-01T10:37:40.723159+0000 osd.7 (osd.7) 54 : cluster 3 failed to encode map e120 with expected crc 2023-10-01T10:37:42.000 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:41 smithi078 bash[78274]: audit 2023-10-01T10:37:40.723308+0000 mon.a (mon.0) 808 : audit 1 from='osd.3 [v2:172.21.15.78:6826/2365141372,v1:172.21.15.78:6827/2365141372]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-01T10:37:42.000 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:41 smithi078 bash[78274]: cluster 2023-10-01T10:37:40.723429+0000 mon.a (mon.0) 809 : cluster 0 osdmap e120: 8 total, 7 up, 8 in 2023-10-01T10:37:42.000 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:41 smithi078 bash[78274]: audit 2023-10-01T10:37:40.724417+0000 mon.a (mon.0) 810 : audit 1 from='osd.3 [v2:172.21.15.78:6826/2365141372,v1:172.21.15.78:6827/2365141372]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2023-10-01T10:37:42.001 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:41 smithi078 bash[78274]: cluster 2023-10-01T10:37:41.631396+0000 mgr.y (mgr.25072) 585 : cluster 0 pgmap v494: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-01T10:37:42.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:41 smithi173 bash[58816]: cluster 2023-10-01T10:37:40.720418+0000 osd.5 (osd.5) 40 : cluster 3 failed to encode map e120 with expected crc 2023-10-01T10:37:42.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:41 smithi173 bash[58816]: cluster 2023-10-01T10:37:40.723159+0000 osd.7 (osd.7) 54 : cluster 3 failed to encode map e120 with expected crc 2023-10-01T10:37:42.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:41 smithi173 bash[58816]: audit 2023-10-01T10:37:40.723308+0000 mon.a (mon.0) 808 : audit 1 from='osd.3 [v2:172.21.15.78:6826/2365141372,v1:172.21.15.78:6827/2365141372]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-10-01T10:37:42.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:41 smithi173 bash[58816]: cluster 2023-10-01T10:37:40.723429+0000 mon.a (mon.0) 809 : cluster 0 osdmap e120: 8 total, 7 up, 8 in 2023-10-01T10:37:42.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:41 smithi173 bash[58816]: audit 2023-10-01T10:37:40.724417+0000 mon.a (mon.0) 810 : audit 1 from='osd.3 [v2:172.21.15.78:6826/2365141372,v1:172.21.15.78:6827/2365141372]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2023-10-01T10:37:42.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:41 smithi173 bash[58816]: cluster 2023-10-01T10:37:41.631396+0000 mgr.y (mgr.25072) 585 : cluster 0 pgmap v494: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-01T10:37:42.515 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 18.2.0-497-gb5682219 (b56822191ad0416d0c05ed59720df03ad9c2a44e) reef (stable)": 4 2023-10-01T10:37:42.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:42 smithi078 bash[75299]: cluster 2023-10-01T10:37:41.730127+0000 mon.a (mon.0) 811 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-01T10:37:42.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:42 smithi078 bash[75299]: cluster 2023-10-01T10:37:41.739963+0000 osd.7 (osd.7) 55 : cluster 3 failed to encode map e121 with expected crc 2023-10-01T10:37:42.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:42 smithi078 bash[75299]: cluster 2023-10-01T10:37:41.741797+0000 mon.a (mon.0) 812 : cluster 1 osd.3 [v2:172.21.15.78:6826/2365141372,v1:172.21.15.78:6827/2365141372] boot 2023-10-01T10:37:42.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:42 smithi078 bash[75299]: cluster 2023-10-01T10:37:41.741868+0000 mon.a (mon.0) 813 : cluster 0 osdmap e121: 8 total, 8 up, 8 in 2023-10-01T10:37:42.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:42 smithi078 bash[75299]: audit 2023-10-01T10:37:41.742506+0000 mon.a (mon.0) 814 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T10:37:42.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:42 smithi078 bash[75299]: cluster 2023-10-01T10:37:41.748091+0000 osd.4 (osd.4) 31 : cluster 3 failed to encode map e121 with expected crc 2023-10-01T10:37:42.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:42 smithi078 bash[75299]: audit 2023-10-01T10:37:42.499655+0000 mon.c (mon.2) 126 : audit 0 from='client.? 172.21.15.78:0/4221320776' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:37:42.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:42 smithi078 bash[78274]: cluster 2023-10-01T10:37:41.730127+0000 mon.a (mon.0) 811 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-01T10:37:42.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:42 smithi078 bash[78274]: cluster 2023-10-01T10:37:41.739963+0000 osd.7 (osd.7) 55 : cluster 3 failed to encode map e121 with expected crc 2023-10-01T10:37:42.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:42 smithi078 bash[78274]: cluster 2023-10-01T10:37:41.741797+0000 mon.a (mon.0) 812 : cluster 1 osd.3 [v2:172.21.15.78:6826/2365141372,v1:172.21.15.78:6827/2365141372] boot 2023-10-01T10:37:42.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:42 smithi078 bash[78274]: cluster 2023-10-01T10:37:41.741868+0000 mon.a (mon.0) 813 : cluster 0 osdmap e121: 8 total, 8 up, 8 in 2023-10-01T10:37:42.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:42 smithi078 bash[78274]: audit 2023-10-01T10:37:41.742506+0000 mon.a (mon.0) 814 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T10:37:42.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:42 smithi078 bash[78274]: cluster 2023-10-01T10:37:41.748091+0000 osd.4 (osd.4) 31 : cluster 3 failed to encode map e121 with expected crc 2023-10-01T10:37:42.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:42 smithi078 bash[78274]: audit 2023-10-01T10:37:42.499655+0000 mon.c (mon.2) 126 : audit 0 from='client.? 172.21.15.78:0/4221320776' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:37:43.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:42 smithi173 bash[58816]: cluster 2023-10-01T10:37:41.730127+0000 mon.a (mon.0) 811 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-01T10:37:43.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:42 smithi173 bash[58816]: cluster 2023-10-01T10:37:41.739963+0000 osd.7 (osd.7) 55 : cluster 3 failed to encode map e121 with expected crc 2023-10-01T10:37:43.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:42 smithi173 bash[58816]: cluster 2023-10-01T10:37:41.741797+0000 mon.a (mon.0) 812 : cluster 1 osd.3 [v2:172.21.15.78:6826/2365141372,v1:172.21.15.78:6827/2365141372] boot 2023-10-01T10:37:43.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:42 smithi173 bash[58816]: cluster 2023-10-01T10:37:41.741868+0000 mon.a (mon.0) 813 : cluster 0 osdmap e121: 8 total, 8 up, 8 in 2023-10-01T10:37:43.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:42 smithi173 bash[58816]: audit 2023-10-01T10:37:41.742506+0000 mon.a (mon.0) 814 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-10-01T10:37:43.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:42 smithi173 bash[58816]: cluster 2023-10-01T10:37:41.748091+0000 osd.4 (osd.4) 31 : cluster 3 failed to encode map e121 with expected crc 2023-10-01T10:37:43.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:42 smithi173 bash[58816]: audit 2023-10-01T10:37:42.499655+0000 mon.c (mon.2) 126 : audit 0 from='client.? 172.21.15.78:0/4221320776' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-10-01T10:37:43.805 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -e sha1=b56822191ad0416d0c05ed59720df03ad9c2a44e -- bash -c 'ceph orch upgrade pause' 2023-10-01T10:37:44.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:43 smithi078 bash[75299]: cluster 2023-10-01T10:37:42.750384+0000 osd.4 (osd.4) 32 : cluster 3 failed to encode map e122 with expected crc 2023-10-01T10:37:44.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:43 smithi078 bash[75299]: cluster 2023-10-01T10:37:42.752164+0000 osd.6 (osd.6) 34 : cluster 3 failed to encode map e122 with expected crc 2023-10-01T10:37:44.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:43 smithi078 bash[75299]: cluster 2023-10-01T10:37:42.753026+0000 osd.5 (osd.5) 41 : cluster 3 failed to encode map e122 with expected crc 2023-10-01T10:37:44.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:43 smithi078 bash[75299]: cluster 2023-10-01T10:37:42.753930+0000 mon.a (mon.0) 815 : cluster 0 osdmap e122: 8 total, 8 up, 8 in 2023-10-01T10:37:44.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:43 smithi078 bash[75299]: cluster 2023-10-01T10:37:42.754394+0000 osd.4 (osd.4) 33 : cluster 3 failed to encode map e122 with expected crc 2023-10-01T10:37:44.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:43 smithi078 bash[75299]: cluster 2023-10-01T10:37:42.755378+0000 osd.5 (osd.5) 42 : cluster 3 failed to encode map e122 with expected crc 2023-10-01T10:37:44.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:43 smithi078 bash[75299]: cluster 2023-10-01T10:37:42.755888+0000 osd.7 (osd.7) 56 : cluster 3 failed to encode map e122 with expected crc 2023-10-01T10:37:44.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:43 smithi078 bash[75299]: cluster 2023-10-01T10:37:42.756097+0000 osd.4 (osd.4) 34 : cluster 3 failed to encode map e122 with expected crc 2023-10-01T10:37:44.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:43 smithi078 bash[75299]: cluster 2023-10-01T10:37:42.757061+0000 osd.5 (osd.5) 43 : cluster 3 failed to encode map e122 with expected crc 2023-10-01T10:37:44.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:43 smithi078 bash[75299]: cluster 2023-10-01T10:37:42.758768+0000 osd.7 (osd.7) 57 : cluster 3 failed to encode map e122 with expected crc 2023-10-01T10:37:44.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:43 smithi078 bash[75299]: cluster 2023-10-01T10:37:42.759654+0000 osd.4 (osd.4) 35 : cluster 3 failed to encode map e122 with expected crc 2023-10-01T10:37:44.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:43 smithi078 bash[75299]: cluster 2023-10-01T10:37:43.631943+0000 mgr.y (mgr.25072) 586 : cluster 0 pgmap v497: 73 pgs: 3 active+undersized+degraded, 25 active+undersized, 45 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-01T10:37:44.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:43 smithi078 bash[78274]: cluster 2023-10-01T10:37:42.750384+0000 osd.4 (osd.4) 32 : cluster 3 failed to encode map e122 with expected crc 2023-10-01T10:37:44.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:43 smithi078 bash[78274]: cluster 2023-10-01T10:37:42.752164+0000 osd.6 (osd.6) 34 : cluster 3 failed to encode map e122 with expected crc 2023-10-01T10:37:44.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:43 smithi078 bash[78274]: cluster 2023-10-01T10:37:42.753026+0000 osd.5 (osd.5) 41 : cluster 3 failed to encode map e122 with expected crc 2023-10-01T10:37:44.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:43 smithi078 bash[78274]: cluster 2023-10-01T10:37:42.753930+0000 mon.a (mon.0) 815 : cluster 0 osdmap e122: 8 total, 8 up, 8 in 2023-10-01T10:37:44.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:43 smithi078 bash[78274]: cluster 2023-10-01T10:37:42.754394+0000 osd.4 (osd.4) 33 : cluster 3 failed to encode map e122 with expected crc 2023-10-01T10:37:44.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:43 smithi078 bash[78274]: cluster 2023-10-01T10:37:42.755378+0000 osd.5 (osd.5) 42 : cluster 3 failed to encode map e122 with expected crc 2023-10-01T10:37:44.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:43 smithi078 bash[78274]: cluster 2023-10-01T10:37:42.755888+0000 osd.7 (osd.7) 56 : cluster 3 failed to encode map e122 with expected crc 2023-10-01T10:37:44.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:43 smithi078 bash[78274]: cluster 2023-10-01T10:37:42.756097+0000 osd.4 (osd.4) 34 : cluster 3 failed to encode map e122 with expected crc 2023-10-01T10:37:44.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:43 smithi078 bash[78274]: cluster 2023-10-01T10:37:42.757061+0000 osd.5 (osd.5) 43 : cluster 3 failed to encode map e122 with expected crc 2023-10-01T10:37:44.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:43 smithi078 bash[78274]: cluster 2023-10-01T10:37:42.758768+0000 osd.7 (osd.7) 57 : cluster 3 failed to encode map e122 with expected crc 2023-10-01T10:37:44.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:43 smithi078 bash[78274]: cluster 2023-10-01T10:37:42.759654+0000 osd.4 (osd.4) 35 : cluster 3 failed to encode map e122 with expected crc 2023-10-01T10:37:44.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:43 smithi078 bash[78274]: cluster 2023-10-01T10:37:43.631943+0000 mgr.y (mgr.25072) 586 : cluster 0 pgmap v497: 73 pgs: 3 active+undersized+degraded, 25 active+undersized, 45 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-01T10:37:44.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:43 smithi173 bash[58816]: cluster 2023-10-01T10:37:42.750384+0000 osd.4 (osd.4) 32 : cluster 3 failed to encode map e122 with expected crc 2023-10-01T10:37:44.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:43 smithi173 bash[58816]: cluster 2023-10-01T10:37:42.752164+0000 osd.6 (osd.6) 34 : cluster 3 failed to encode map e122 with expected crc 2023-10-01T10:37:44.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:43 smithi173 bash[58816]: cluster 2023-10-01T10:37:42.753026+0000 osd.5 (osd.5) 41 : cluster 3 failed to encode map e122 with expected crc 2023-10-01T10:37:44.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:43 smithi173 bash[58816]: cluster 2023-10-01T10:37:42.753930+0000 mon.a (mon.0) 815 : cluster 0 osdmap e122: 8 total, 8 up, 8 in 2023-10-01T10:37:44.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:43 smithi173 bash[58816]: cluster 2023-10-01T10:37:42.754394+0000 osd.4 (osd.4) 33 : cluster 3 failed to encode map e122 with expected crc 2023-10-01T10:37:44.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:43 smithi173 bash[58816]: cluster 2023-10-01T10:37:42.755378+0000 osd.5 (osd.5) 42 : cluster 3 failed to encode map e122 with expected crc 2023-10-01T10:37:44.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:43 smithi173 bash[58816]: cluster 2023-10-01T10:37:42.755888+0000 osd.7 (osd.7) 56 : cluster 3 failed to encode map e122 with expected crc 2023-10-01T10:37:44.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:43 smithi173 bash[58816]: cluster 2023-10-01T10:37:42.756097+0000 osd.4 (osd.4) 34 : cluster 3 failed to encode map e122 with expected crc 2023-10-01T10:37:44.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:43 smithi173 bash[58816]: cluster 2023-10-01T10:37:42.757061+0000 osd.5 (osd.5) 43 : cluster 3 failed to encode map e122 with expected crc 2023-10-01T10:37:44.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:43 smithi173 bash[58816]: cluster 2023-10-01T10:37:42.758768+0000 osd.7 (osd.7) 57 : cluster 3 failed to encode map e122 with expected crc 2023-10-01T10:37:44.122 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:43 smithi173 bash[58816]: cluster 2023-10-01T10:37:42.759654+0000 osd.4 (osd.4) 35 : cluster 3 failed to encode map e122 with expected crc 2023-10-01T10:37:44.122 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:43 smithi173 bash[58816]: cluster 2023-10-01T10:37:43.631943+0000 mgr.y (mgr.25072) 586 : cluster 0 pgmap v497: 73 pgs: 3 active+undersized+degraded, 25 active+undersized, 45 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2023-10-01T10:37:46.006 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:45 smithi078 bash[75299]: cluster 2023-10-01T10:37:45.633013+0000 mgr.y (mgr.25072) 587 : cluster 0 pgmap v498: 73 pgs: 2 active+undersized+degraded, 11 active+undersized, 60 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-01T10:37:46.006 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:45 smithi078 bash[75299]: cluster 2023-10-01T10:37:45.684591+0000 mon.a (mon.0) 816 : cluster 3 Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded (PG_DEGRADED) 2023-10-01T10:37:46.006 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:45 smithi078 bash[78274]: cluster 2023-10-01T10:37:45.633013+0000 mgr.y (mgr.25072) 587 : cluster 0 pgmap v498: 73 pgs: 2 active+undersized+degraded, 11 active+undersized, 60 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-01T10:37:46.006 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:45 smithi078 bash[78274]: cluster 2023-10-01T10:37:45.684591+0000 mon.a (mon.0) 816 : cluster 3 Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded (PG_DEGRADED) 2023-10-01T10:37:46.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:45 smithi173 bash[58816]: cluster 2023-10-01T10:37:45.633013+0000 mgr.y (mgr.25072) 587 : cluster 0 pgmap v498: 73 pgs: 2 active+undersized+degraded, 11 active+undersized, 60 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2023-10-01T10:37:46.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:45 smithi173 bash[58816]: cluster 2023-10-01T10:37:45.684591+0000 mon.a (mon.0) 816 : cluster 3 Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded (PG_DEGRADED) 2023-10-01T10:37:48.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:48 smithi078 bash[75299]: audit 2023-10-01T10:37:47.413029+0000 mon.a (mon.0) 817 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:48.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:48 smithi078 bash[75299]: audit 2023-10-01T10:37:47.414402+0000 mon.a (mon.0) 818 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:37:48.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:48 smithi078 bash[75299]: cluster 2023-10-01T10:37:47.633447+0000 mgr.y (mgr.25072) 588 : cluster 0 pgmap v499: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:37:48.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:48 smithi078 bash[78274]: audit 2023-10-01T10:37:47.413029+0000 mon.a (mon.0) 817 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:48.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:48 smithi078 bash[78274]: audit 2023-10-01T10:37:47.414402+0000 mon.a (mon.0) 818 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:37:48.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:48 smithi078 bash[78274]: cluster 2023-10-01T10:37:47.633447+0000 mgr.y (mgr.25072) 588 : cluster 0 pgmap v499: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:37:48.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:48 smithi173 bash[58816]: audit 2023-10-01T10:37:47.413029+0000 mon.a (mon.0) 817 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:48.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:48 smithi173 bash[58816]: audit 2023-10-01T10:37:47.414402+0000 mon.a (mon.0) 818 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:37:48.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:48 smithi173 bash[58816]: cluster 2023-10-01T10:37:47.633447+0000 mgr.y (mgr.25072) 588 : cluster 0 pgmap v499: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:37:49.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:49 smithi078 bash[75299]: cluster 2023-10-01T10:37:48.410826+0000 mon.a (mon.0) 819 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded) 2023-10-01T10:37:49.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:49 smithi078 bash[75299]: cluster 2023-10-01T10:37:48.410865+0000 mon.a (mon.0) 820 : cluster 1 Cluster is now healthy 2023-10-01T10:37:49.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:49 smithi078 bash[75299]: audit 2023-10-01T10:37:49.323378+0000 mon.a (mon.0) 821 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:49.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:49 smithi078 bash[75299]: audit 2023-10-01T10:37:49.330664+0000 mon.a (mon.0) 822 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:49.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:49 smithi078 bash[75299]: audit 2023-10-01T10:37:49.332273+0000 mon.a (mon.0) 823 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:37:49.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:49 smithi078 bash[75299]: audit 2023-10-01T10:37:49.333514+0000 mon.a (mon.0) 824 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:37:49.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:49 smithi078 bash[75299]: audit 2023-10-01T10:37:49.340557+0000 mon.a (mon.0) 825 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:49.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:49 smithi078 bash[78274]: cluster 2023-10-01T10:37:48.410826+0000 mon.a (mon.0) 819 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded) 2023-10-01T10:37:49.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:49 smithi078 bash[78274]: cluster 2023-10-01T10:37:48.410865+0000 mon.a (mon.0) 820 : cluster 1 Cluster is now healthy 2023-10-01T10:37:49.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:49 smithi078 bash[78274]: audit 2023-10-01T10:37:49.323378+0000 mon.a (mon.0) 821 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:49.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:49 smithi078 bash[78274]: audit 2023-10-01T10:37:49.330664+0000 mon.a (mon.0) 822 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:49.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:49 smithi078 bash[78274]: audit 2023-10-01T10:37:49.332273+0000 mon.a (mon.0) 823 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:37:49.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:49 smithi078 bash[78274]: audit 2023-10-01T10:37:49.333514+0000 mon.a (mon.0) 824 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:37:49.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:49 smithi078 bash[78274]: audit 2023-10-01T10:37:49.340557+0000 mon.a (mon.0) 825 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:49.677 INFO:teuthology.orchestra.run.smithi078.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T10:37:49.732 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:49 smithi173 bash[58816]: cluster 2023-10-01T10:37:48.410826+0000 mon.a (mon.0) 819 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 2 pgs degraded) 2023-10-01T10:37:49.732 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:49 smithi173 bash[58816]: cluster 2023-10-01T10:37:48.410865+0000 mon.a (mon.0) 820 : cluster 1 Cluster is now healthy 2023-10-01T10:37:49.733 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:49 smithi173 bash[58816]: audit 2023-10-01T10:37:49.323378+0000 mon.a (mon.0) 821 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:49.733 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:49 smithi173 bash[58816]: audit 2023-10-01T10:37:49.330664+0000 mon.a (mon.0) 822 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:49.733 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:49 smithi173 bash[58816]: audit 2023-10-01T10:37:49.332273+0000 mon.a (mon.0) 823 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:37:49.733 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:49 smithi173 bash[58816]: audit 2023-10-01T10:37:49.333514+0000 mon.a (mon.0) 824 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:37:49.733 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:49 smithi173 bash[58816]: audit 2023-10-01T10:37:49.340557+0000 mon.a (mon.0) 825 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:50.562 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:50 smithi078 bash[75299]: audit 2023-10-01T10:37:49.423140+0000 mon.a (mon.0) 826 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:37:50.562 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:50 smithi078 bash[75299]: audit 2023-10-01T10:37:49.426631+0000 mon.a (mon.0) 827 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:37:50.562 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:50 smithi078 bash[75299]: audit 2023-10-01T10:37:49.429309+0000 mon.a (mon.0) 828 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:37:50.562 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:50 smithi078 bash[75299]: audit 2023-10-01T10:37:49.431166+0000 mon.a (mon.0) 829 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:37:50.562 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:50 smithi078 bash[75299]: audit 2023-10-01T10:37:49.433444+0000 mon.a (mon.0) 830 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-10-01T10:37:50.563 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:50 smithi078 bash[75299]: audit 2023-10-01T10:37:49.433628+0000 mgr.y (mgr.25072) 589 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-10-01T10:37:50.563 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:50 smithi078 bash[75299]: cephadm 2023-10-01T10:37:49.434771+0000 mgr.y (mgr.25072) 590 : cephadm 1 Upgrade: osd.4 is safe to restart 2023-10-01T10:37:50.563 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:50 smithi078 bash[75299]: cluster 2023-10-01T10:37:49.634241+0000 mgr.y (mgr.25072) 591 : cluster 0 pgmap v500: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:37:50.563 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:50 smithi078 bash[75299]: audit 2023-10-01T10:37:49.675622+0000 mon.a (mon.0) 831 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:50.563 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:37:50 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:37:50] "GET /metrics HTTP/1.1" 200 33822 "" "Prometheus/2.33.4" 2023-10-01T10:37:50.563 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:50 smithi078 bash[78274]: audit 2023-10-01T10:37:49.423140+0000 mon.a (mon.0) 826 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:37:50.564 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:50 smithi078 bash[78274]: audit 2023-10-01T10:37:49.426631+0000 mon.a (mon.0) 827 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:37:50.564 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:50 smithi078 bash[78274]: audit 2023-10-01T10:37:49.429309+0000 mon.a (mon.0) 828 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:37:50.564 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:50 smithi078 bash[78274]: audit 2023-10-01T10:37:49.431166+0000 mon.a (mon.0) 829 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:37:50.564 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:50 smithi078 bash[78274]: audit 2023-10-01T10:37:49.433444+0000 mon.a (mon.0) 830 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-10-01T10:37:50.564 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:50 smithi078 bash[78274]: audit 2023-10-01T10:37:49.433628+0000 mgr.y (mgr.25072) 589 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-10-01T10:37:50.564 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:50 smithi078 bash[78274]: cephadm 2023-10-01T10:37:49.434771+0000 mgr.y (mgr.25072) 590 : cephadm 1 Upgrade: osd.4 is safe to restart 2023-10-01T10:37:50.564 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:50 smithi078 bash[78274]: cluster 2023-10-01T10:37:49.634241+0000 mgr.y (mgr.25072) 591 : cluster 0 pgmap v500: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:37:50.564 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:50 smithi078 bash[78274]: audit 2023-10-01T10:37:49.675622+0000 mon.a (mon.0) 831 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:50.565 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -e sha1=b56822191ad0416d0c05ed59720df03ad9c2a44e -- bash -c 'ceph orch ps' 2023-10-01T10:37:50.721 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:50 smithi173 bash[58816]: audit 2023-10-01T10:37:49.423140+0000 mon.a (mon.0) 826 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:37:50.721 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:50 smithi173 bash[58816]: audit 2023-10-01T10:37:49.426631+0000 mon.a (mon.0) 827 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:37:50.721 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:50 smithi173 bash[58816]: audit 2023-10-01T10:37:49.429309+0000 mon.a (mon.0) 828 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:37:50.721 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:50 smithi173 bash[58816]: audit 2023-10-01T10:37:49.431166+0000 mon.a (mon.0) 829 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:37:50.721 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:50 smithi173 bash[58816]: audit 2023-10-01T10:37:49.433444+0000 mon.a (mon.0) 830 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-10-01T10:37:50.721 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:50 smithi173 bash[58816]: audit 2023-10-01T10:37:49.433628+0000 mgr.y (mgr.25072) 589 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-10-01T10:37:50.721 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:50 smithi173 bash[58816]: cephadm 2023-10-01T10:37:49.434771+0000 mgr.y (mgr.25072) 590 : cephadm 1 Upgrade: osd.4 is safe to restart 2023-10-01T10:37:50.722 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:50 smithi173 bash[58816]: cluster 2023-10-01T10:37:49.634241+0000 mgr.y (mgr.25072) 591 : cluster 0 pgmap v500: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:37:50.722 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:50 smithi173 bash[58816]: audit 2023-10-01T10:37:49.675622+0000 mon.a (mon.0) 831 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:51.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:51 smithi173 bash[58816]: audit 2023-10-01T10:37:49.664838+0000 mgr.y (mgr.25072) 592 : audit 0 from='client.45365 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:37:51.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:51 smithi173 bash[58816]: cephadm 2023-10-01T10:37:49.665552+0000 mgr.y (mgr.25072) 593 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T10:37:51.793 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:51 smithi078 bash[78274]: audit 2023-10-01T10:37:49.664838+0000 mgr.y (mgr.25072) 592 : audit 0 from='client.45365 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:37:51.793 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:51 smithi078 bash[78274]: cephadm 2023-10-01T10:37:49.665552+0000 mgr.y (mgr.25072) 593 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T10:37:51.793 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:51 smithi078 bash[75299]: audit 2023-10-01T10:37:49.664838+0000 mgr.y (mgr.25072) 592 : audit 0 from='client.45365 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:37:51.793 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:51 smithi078 bash[75299]: cephadm 2023-10-01T10:37:49.665552+0000 mgr.y (mgr.25072) 593 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf 2023-10-01T10:37:52.779 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:52 smithi078 bash[78274]: cluster 2023-10-01T10:37:51.635112+0000 mgr.y (mgr.25072) 594 : cluster 0 pgmap v501: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:37:52.779 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:52 smithi078 bash[78274]: audit 2023-10-01T10:37:51.761680+0000 mon.a (mon.0) 832 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:52.779 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:52 smithi078 bash[78274]: audit 2023-10-01T10:37:51.763112+0000 mon.a (mon.0) 833 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-01T10:37:52.779 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:52 smithi078 bash[78274]: audit 2023-10-01T10:37:51.764339+0000 mon.a (mon.0) 834 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:37:52.779 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:52 smithi078 bash[75299]: cluster 2023-10-01T10:37:51.635112+0000 mgr.y (mgr.25072) 594 : cluster 0 pgmap v501: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:37:52.779 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:52 smithi078 bash[75299]: audit 2023-10-01T10:37:51.761680+0000 mon.a (mon.0) 832 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:52.780 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:52 smithi078 bash[75299]: audit 2023-10-01T10:37:51.763112+0000 mon.a (mon.0) 833 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-01T10:37:52.780 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:52 smithi078 bash[75299]: audit 2023-10-01T10:37:51.764339+0000 mon.a (mon.0) 834 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:37:52.865 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:52 smithi173 bash[58816]: cluster 2023-10-01T10:37:51.635112+0000 mgr.y (mgr.25072) 594 : cluster 0 pgmap v501: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:37:52.865 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:52 smithi173 bash[58816]: audit 2023-10-01T10:37:51.761680+0000 mon.a (mon.0) 832 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:52.865 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:52 smithi173 bash[58816]: audit 2023-10-01T10:37:51.763112+0000 mon.a (mon.0) 833 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-10-01T10:37:52.865 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:52 smithi173 bash[58816]: audit 2023-10-01T10:37:51.764339+0000 mon.a (mon.0) 834 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:37:54.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:53 smithi078 bash[78274]: cephadm 2023-10-01T10:37:51.753992+0000 mgr.y (mgr.25072) 595 : cephadm 1 Upgrade: Updating osd.4 2023-10-01T10:37:54.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:53 smithi078 bash[78274]: cephadm 2023-10-01T10:37:51.766715+0000 mgr.y (mgr.25072) 596 : cephadm 1 Deploying daemon osd.4 on smithi173 2023-10-01T10:37:54.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:53 smithi078 bash[78274]: audit 2023-10-01T10:37:52.648379+0000 mon.a (mon.0) 835 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:54.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:53 smithi078 bash[75299]: cephadm 2023-10-01T10:37:51.753992+0000 mgr.y (mgr.25072) 595 : cephadm 1 Upgrade: Updating osd.4 2023-10-01T10:37:54.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:53 smithi078 bash[75299]: cephadm 2023-10-01T10:37:51.766715+0000 mgr.y (mgr.25072) 596 : cephadm 1 Deploying daemon osd.4 on smithi173 2023-10-01T10:37:54.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:53 smithi078 bash[75299]: audit 2023-10-01T10:37:52.648379+0000 mon.a (mon.0) 835 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:54.124 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:53 smithi173 bash[58816]: cephadm 2023-10-01T10:37:51.753992+0000 mgr.y (mgr.25072) 595 : cephadm 1 Upgrade: Updating osd.4 2023-10-01T10:37:54.124 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:53 smithi173 bash[58816]: cephadm 2023-10-01T10:37:51.766715+0000 mgr.y (mgr.25072) 596 : cephadm 1 Deploying daemon osd.4 on smithi173 2023-10-01T10:37:54.124 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:53 smithi173 bash[58816]: audit 2023-10-01T10:37:52.648379+0000 mon.a (mon.0) 835 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:37:54.199 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-01T10:37:54.200 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.a smithi078 *:9093,9094 running (17m) - 24m 14.5M - 0.23.0 ba2b418f427c 90745d6cf2c8 2023-10-01T10:37:54.200 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.a smithi173 *:3000 running (16m) - 23m 41.5M - 8.3.5 dad864ee21e9 4874aca3a861 2023-10-01T10:37:54.200 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foo.smithi078.vjqrth smithi078 running (22m) - 22m 20.2M - 16.2.14-44-geeb13ac9 94a6c4498547 aedf3352e433 2023-10-01T10:37:54.200 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foo.smithi173.vrqqoc smithi173 running (22m) - 22m 16.1M - 16.2.14-44-geeb13ac9 94a6c4498547 3815ca08717b 2023-10-01T10:37:54.200 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.x smithi173 *:8443,9283,8765 running (15m) - 30m 432M - 18.2.0-497-gb5682219 a3fc96fed5ac 0e4d4ec3a75c 2023-10-01T10:37:54.200 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.y smithi078 *:8443,9283,8765 running (16m) - 32m 487M - 18.2.0-497-gb5682219 a3fc96fed5ac bdb8227ce5b1 2023-10-01T10:37:54.200 INFO:teuthology.orchestra.run.smithi078.stdout:mon.a smithi078 running (15m) - 32m 106M 2048M 18.2.0-497-gb5682219 a3fc96fed5ac 62398b61255b 2023-10-01T10:37:54.200 INFO:teuthology.orchestra.run.smithi078.stdout:mon.b smithi173 running (11m) - 31m 50.1M 2048M 18.2.0-497-gb5682219 a3fc96fed5ac a513d9d49797 2023-10-01T10:37:54.200 INFO:teuthology.orchestra.run.smithi078.stdout:mon.c smithi078 running (13m) - 31m 58.1M 2048M 18.2.0-497-gb5682219 a3fc96fed5ac 5f02c5b8aecd 2023-10-01T10:37:54.201 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.a smithi078 *:9100 running (17m) - 24m 9.97M - 1.3.1 1dbe0e931976 3c271e279ae2 2023-10-01T10:37:54.201 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.b smithi173 *:9100 running (17m) - 24m 9480k - 1.3.1 1dbe0e931976 cf13fd39e3de 2023-10-01T10:37:54.201 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (9m) - 30m 764M 2524M 18.2.0-497-gb5682219 a3fc96fed5ac 58892d6f5063 2023-10-01T10:37:54.201 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (5m) - 29m 544M 2524M 18.2.0-497-gb5682219 a3fc96fed5ac 0eede59aa6a4 2023-10-01T10:37:54.201 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (66s) - 28m 433M 2524M 18.2.0-497-gb5682219 a3fc96fed5ac 90dfed39ddb9 2023-10-01T10:37:54.201 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (22s) - 28m - 2524M 18.2.0-497-gb5682219 a3fc96fed5ac e9299b97fd93 2023-10-01T10:37:54.201 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi173 running (27m) - 27m 626M 2524M 16.2.14-44-geeb13ac9 94a6c4498547 f6a3e2fc7289 2023-10-01T10:37:54.201 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi173 running (26m) - 26m 962M 2524M 16.2.14-44-geeb13ac9 94a6c4498547 b4ab64b3b760 2023-10-01T10:37:54.201 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi173 running (25m) - 26m 584M 2524M 16.2.14-44-geeb13ac9 94a6c4498547 67631580752d 2023-10-01T10:37:54.201 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi173 running (25m) - 25m 779M 2524M 16.2.14-44-geeb13ac9 94a6c4498547 86157c8ed4b2 2023-10-01T10:37:54.201 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.a smithi173 *:9095 running (15m) - 24m 65.7M - 2.33.4 514e6a882f6e 94c2cb9de105 2023-10-01T10:37:55.053 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:54 smithi078 bash[75299]: cluster 2023-10-01T10:37:53.635991+0000 mgr.y (mgr.25072) 597 : cluster 0 pgmap v502: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:37:55.053 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:54 smithi078 bash[78274]: cluster 2023-10-01T10:37:53.635991+0000 mgr.y (mgr.25072) 597 : cluster 0 pgmap v502: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:37:55.054 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -e sha1=b56822191ad0416d0c05ed59720df03ad9c2a44e -- bash -c 'ceph orch ps' 2023-10-01T10:37:55.060 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:54 smithi173 bash[58816]: cluster 2023-10-01T10:37:53.635991+0000 mgr.y (mgr.25072) 597 : cluster 0 pgmap v502: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:37:55.998 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:55 smithi173 bash[58816]: audit 2023-10-01T10:37:54.179802+0000 mgr.y (mgr.25072) 598 : audit 0 from='client.45371 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:37:56.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:55 smithi078 bash[78274]: audit 2023-10-01T10:37:54.179802+0000 mgr.y (mgr.25072) 598 : audit 0 from='client.45371 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:37:56.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:55 smithi078 bash[75299]: audit 2023-10-01T10:37:54.179802+0000 mgr.y (mgr.25072) 598 : audit 0 from='client.45371 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:37:56.370 INFO:journalctl@ceph.osd.4.smithi173.stdout:Oct 01 10:37:56 smithi173 systemd[1]: Stopping Ceph osd.4 for d8016480-6041-11ee-8db5-212e2dc638e7... 2023-10-01T10:37:56.666 INFO:journalctl@ceph.osd.4.smithi173.stdout:Oct 01 10:37:56 smithi173 bash[31216]: debug 2023-10-01T10:37:56.380+0000 7fe11805e700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-01T10:37:56.666 INFO:journalctl@ceph.osd.4.smithi173.stdout:Oct 01 10:37:56 smithi173 bash[31216]: debug 2023-10-01T10:37:56.380+0000 7fe11805e700 -1 osd.4 122 *** Got signal Terminated *** 2023-10-01T10:37:56.666 INFO:journalctl@ceph.osd.4.smithi173.stdout:Oct 01 10:37:56 smithi173 bash[31216]: debug 2023-10-01T10:37:56.380+0000 7fe11805e700 -1 osd.4 122 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-01T10:37:57.038 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:56 smithi173 bash[58816]: cluster 2023-10-01T10:37:55.636815+0000 mgr.y (mgr.25072) 599 : cluster 0 pgmap v503: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:37:57.038 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:56 smithi173 bash[58816]: cluster 2023-10-01T10:37:56.381328+0000 mon.a (mon.0) 836 : cluster 1 osd.4 marked itself down and dead 2023-10-01T10:37:57.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:56 smithi078 bash[78274]: cluster 2023-10-01T10:37:55.636815+0000 mgr.y (mgr.25072) 599 : cluster 0 pgmap v503: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:37:57.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:56 smithi078 bash[78274]: cluster 2023-10-01T10:37:56.381328+0000 mon.a (mon.0) 836 : cluster 1 osd.4 marked itself down and dead 2023-10-01T10:37:57.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:56 smithi078 bash[75299]: cluster 2023-10-01T10:37:55.636815+0000 mgr.y (mgr.25072) 599 : cluster 0 pgmap v503: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:37:57.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:56 smithi078 bash[75299]: cluster 2023-10-01T10:37:56.381328+0000 mon.a (mon.0) 836 : cluster 1 osd.4 marked itself down and dead 2023-10-01T10:37:57.722 INFO:journalctl@ceph.osd.4.smithi173.stdout:Oct 01 10:37:57 smithi173 bash[64036]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd-4 2023-10-01T10:37:57.722 INFO:journalctl@ceph.osd.4.smithi173.stdout:Oct 01 10:37:57 smithi173 bash[64101]: Error response from daemon: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd-4 2023-10-01T10:37:57.722 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:57 smithi173 bash[58816]: cluster 2023-10-01T10:37:56.662866+0000 mon.a (mon.0) 837 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-01T10:37:57.723 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:57 smithi173 bash[58816]: cluster 2023-10-01T10:37:56.674817+0000 mon.a (mon.0) 838 : cluster 0 osdmap e123: 8 total, 7 up, 8 in 2023-10-01T10:37:57.723 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:57 smithi173 bash[58816]: cluster 2023-10-01T10:37:56.677734+0000 osd.6 (osd.6) 35 : cluster 3 failed to encode map e123 with expected crc 2023-10-01T10:37:57.723 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:57 smithi173 bash[58816]: cluster 2023-10-01T10:37:56.679083+0000 osd.7 (osd.7) 58 : cluster 3 failed to encode map e123 with expected crc 2023-10-01T10:37:57.723 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:57 smithi173 bash[58816]: cluster 2023-10-01T10:37:56.682148+0000 osd.7 (osd.7) 59 : cluster 3 failed to encode map e123 with expected crc 2023-10-01T10:37:57.723 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:57 smithi173 bash[58816]: cluster 2023-10-01T10:37:56.683738+0000 osd.5 (osd.5) 44 : cluster 3 failed to encode map e123 with expected crc 2023-10-01T10:37:58.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:57 smithi078 bash[75299]: cluster 2023-10-01T10:37:56.662866+0000 mon.a (mon.0) 837 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-01T10:37:58.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:57 smithi078 bash[75299]: cluster 2023-10-01T10:37:56.674817+0000 mon.a (mon.0) 838 : cluster 0 osdmap e123: 8 total, 7 up, 8 in 2023-10-01T10:37:58.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:57 smithi078 bash[75299]: cluster 2023-10-01T10:37:56.677734+0000 osd.6 (osd.6) 35 : cluster 3 failed to encode map e123 with expected crc 2023-10-01T10:37:58.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:57 smithi078 bash[75299]: cluster 2023-10-01T10:37:56.679083+0000 osd.7 (osd.7) 58 : cluster 3 failed to encode map e123 with expected crc 2023-10-01T10:37:58.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:57 smithi078 bash[75299]: cluster 2023-10-01T10:37:56.682148+0000 osd.7 (osd.7) 59 : cluster 3 failed to encode map e123 with expected crc 2023-10-01T10:37:58.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:57 smithi078 bash[75299]: cluster 2023-10-01T10:37:56.683738+0000 osd.5 (osd.5) 44 : cluster 3 failed to encode map e123 with expected crc 2023-10-01T10:37:58.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:57 smithi078 bash[78274]: cluster 2023-10-01T10:37:56.662866+0000 mon.a (mon.0) 837 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2023-10-01T10:37:58.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:57 smithi078 bash[78274]: cluster 2023-10-01T10:37:56.674817+0000 mon.a (mon.0) 838 : cluster 0 osdmap e123: 8 total, 7 up, 8 in 2023-10-01T10:37:58.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:57 smithi078 bash[78274]: cluster 2023-10-01T10:37:56.677734+0000 osd.6 (osd.6) 35 : cluster 3 failed to encode map e123 with expected crc 2023-10-01T10:37:58.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:57 smithi078 bash[78274]: cluster 2023-10-01T10:37:56.679083+0000 osd.7 (osd.7) 58 : cluster 3 failed to encode map e123 with expected crc 2023-10-01T10:37:58.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:57 smithi078 bash[78274]: cluster 2023-10-01T10:37:56.682148+0000 osd.7 (osd.7) 59 : cluster 3 failed to encode map e123 with expected crc 2023-10-01T10:37:58.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:57 smithi078 bash[78274]: cluster 2023-10-01T10:37:56.683738+0000 osd.5 (osd.5) 44 : cluster 3 failed to encode map e123 with expected crc 2023-10-01T10:37:58.643 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-10-01T10:37:58.643 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.a smithi078 *:9093,9094 running (17m) - 24m 14.5M - 0.23.0 ba2b418f427c 90745d6cf2c8 2023-10-01T10:37:58.643 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.a smithi173 *:3000 running (16m) - 23m 41.5M - 8.3.5 dad864ee21e9 4874aca3a861 2023-10-01T10:37:58.643 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foo.smithi078.vjqrth smithi078 running (22m) - 22m 20.2M - 16.2.14-44-geeb13ac9 94a6c4498547 aedf3352e433 2023-10-01T10:37:58.643 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foo.smithi173.vrqqoc smithi173 running (22m) - 22m 16.1M - 16.2.14-44-geeb13ac9 94a6c4498547 3815ca08717b 2023-10-01T10:37:58.643 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.x smithi173 *:8443,9283,8765 running (15m) - 30m 432M - 18.2.0-497-gb5682219 a3fc96fed5ac 0e4d4ec3a75c 2023-10-01T10:37:58.643 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.y smithi078 *:8443,9283,8765 running (16m) - 32m 487M - 18.2.0-497-gb5682219 a3fc96fed5ac bdb8227ce5b1 2023-10-01T10:37:58.643 INFO:teuthology.orchestra.run.smithi078.stdout:mon.a smithi078 running (15m) - 33m 106M 2048M 18.2.0-497-gb5682219 a3fc96fed5ac 62398b61255b 2023-10-01T10:37:58.643 INFO:teuthology.orchestra.run.smithi078.stdout:mon.b smithi173 running (11m) - 31m 50.1M 2048M 18.2.0-497-gb5682219 a3fc96fed5ac a513d9d49797 2023-10-01T10:37:58.643 INFO:teuthology.orchestra.run.smithi078.stdout:mon.c smithi078 running (13m) - 31m 58.1M 2048M 18.2.0-497-gb5682219 a3fc96fed5ac 5f02c5b8aecd 2023-10-01T10:37:58.644 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.a smithi078 *:9100 running (18m) - 24m 9.97M - 1.3.1 1dbe0e931976 3c271e279ae2 2023-10-01T10:37:58.644 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.b smithi173 *:9100 running (17m) - 24m 9480k - 1.3.1 1dbe0e931976 cf13fd39e3de 2023-10-01T10:37:58.644 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (9m) - 30m 764M 2524M 18.2.0-497-gb5682219 a3fc96fed5ac 58892d6f5063 2023-10-01T10:37:58.644 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (5m) - 29m 544M 2524M 18.2.0-497-gb5682219 a3fc96fed5ac 0eede59aa6a4 2023-10-01T10:37:58.644 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (70s) - 28m 433M 2524M 18.2.0-497-gb5682219 a3fc96fed5ac 90dfed39ddb9 2023-10-01T10:37:58.644 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (27s) - 28m - 2524M 18.2.0-497-gb5682219 a3fc96fed5ac e9299b97fd93 2023-10-01T10:37:58.644 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi173 running (27m) - 27m 626M 2524M 16.2.14-44-geeb13ac9 94a6c4498547 f6a3e2fc7289 2023-10-01T10:37:58.644 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi173 running (26m) - 26m 962M 2524M 16.2.14-44-geeb13ac9 94a6c4498547 b4ab64b3b760 2023-10-01T10:37:58.644 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi173 running (26m) - 26m 584M 2524M 16.2.14-44-geeb13ac9 94a6c4498547 67631580752d 2023-10-01T10:37:58.644 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi173 running (25m) - 25m 779M 2524M 16.2.14-44-geeb13ac9 94a6c4498547 86157c8ed4b2 2023-10-01T10:37:58.644 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.a smithi173 *:9095 running (15m) - 24m 65.7M - 2.33.4 514e6a882f6e 94c2cb9de105 2023-10-01T10:37:59.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:58 smithi078 bash[75299]: cluster 2023-10-01T10:37:57.637607+0000 mgr.y (mgr.25072) 600 : cluster 0 pgmap v505: 73 pgs: 13 stale+active+clean, 60 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:37:59.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:58 smithi078 bash[75299]: cluster 2023-10-01T10:37:57.677443+0000 osd.6 (osd.6) 36 : cluster 3 failed to encode map e124 with expected crc 2023-10-01T10:37:59.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:58 smithi078 bash[75299]: cluster 2023-10-01T10:37:57.677816+0000 osd.7 (osd.7) 60 : cluster 3 failed to encode map e124 with expected crc 2023-10-01T10:37:59.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:58 smithi078 bash[75299]: cluster 2023-10-01T10:37:57.679009+0000 osd.6 (osd.6) 37 : cluster 3 failed to encode map e124 with expected crc 2023-10-01T10:37:59.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:58 smithi078 bash[75299]: cluster 2023-10-01T10:37:57.679506+0000 mon.a (mon.0) 839 : cluster 0 osdmap e124: 8 total, 7 up, 8 in 2023-10-01T10:37:59.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:58 smithi078 bash[75299]: cluster 2023-10-01T10:37:57.679525+0000 osd.7 (osd.7) 61 : cluster 3 failed to encode map e124 with expected crc 2023-10-01T10:37:59.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:58 smithi078 bash[75299]: cluster 2023-10-01T10:37:57.679540+0000 osd.5 (osd.5) 45 : cluster 3 failed to encode map e124 with expected crc 2023-10-01T10:37:59.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:58 smithi078 bash[75299]: cluster 2023-10-01T10:37:57.682337+0000 osd.6 (osd.6) 38 : cluster 3 failed to encode map e124 with expected crc 2023-10-01T10:37:59.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:58 smithi078 bash[75299]: cluster 2023-10-01T10:37:57.682760+0000 osd.5 (osd.5) 46 : cluster 3 failed to encode map e124 with expected crc 2023-10-01T10:37:59.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:58 smithi078 bash[75299]: cluster 2023-10-01T10:37:57.682832+0000 osd.7 (osd.7) 62 : cluster 3 failed to encode map e124 with expected crc 2023-10-01T10:37:59.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:58 smithi078 bash[75299]: cluster 2023-10-01T10:37:57.684169+0000 osd.7 (osd.7) 63 : cluster 3 failed to encode map e124 with expected crc 2023-10-01T10:37:59.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:58 smithi078 bash[75299]: audit 2023-10-01T10:37:58.621526+0000 mgr.y (mgr.25072) 601 : audit 0 from='client.45377 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:37:59.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:58 smithi078 bash[78274]: cluster 2023-10-01T10:37:57.637607+0000 mgr.y (mgr.25072) 600 : cluster 0 pgmap v505: 73 pgs: 13 stale+active+clean, 60 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:37:59.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:58 smithi078 bash[78274]: cluster 2023-10-01T10:37:57.677443+0000 osd.6 (osd.6) 36 : cluster 3 failed to encode map e124 with expected crc 2023-10-01T10:37:59.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:58 smithi078 bash[78274]: cluster 2023-10-01T10:37:57.677816+0000 osd.7 (osd.7) 60 : cluster 3 failed to encode map e124 with expected crc 2023-10-01T10:37:59.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:58 smithi078 bash[78274]: cluster 2023-10-01T10:37:57.679009+0000 osd.6 (osd.6) 37 : cluster 3 failed to encode map e124 with expected crc 2023-10-01T10:37:59.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:58 smithi078 bash[78274]: cluster 2023-10-01T10:37:57.679506+0000 mon.a (mon.0) 839 : cluster 0 osdmap e124: 8 total, 7 up, 8 in 2023-10-01T10:37:59.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:58 smithi078 bash[78274]: cluster 2023-10-01T10:37:57.679525+0000 osd.7 (osd.7) 61 : cluster 3 failed to encode map e124 with expected crc 2023-10-01T10:37:59.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:58 smithi078 bash[78274]: cluster 2023-10-01T10:37:57.679540+0000 osd.5 (osd.5) 45 : cluster 3 failed to encode map e124 with expected crc 2023-10-01T10:37:59.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:58 smithi078 bash[78274]: cluster 2023-10-01T10:37:57.682337+0000 osd.6 (osd.6) 38 : cluster 3 failed to encode map e124 with expected crc 2023-10-01T10:37:59.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:58 smithi078 bash[78274]: cluster 2023-10-01T10:37:57.682760+0000 osd.5 (osd.5) 46 : cluster 3 failed to encode map e124 with expected crc 2023-10-01T10:37:59.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:58 smithi078 bash[78274]: cluster 2023-10-01T10:37:57.682832+0000 osd.7 (osd.7) 62 : cluster 3 failed to encode map e124 with expected crc 2023-10-01T10:37:59.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:58 smithi078 bash[78274]: cluster 2023-10-01T10:37:57.684169+0000 osd.7 (osd.7) 63 : cluster 3 failed to encode map e124 with expected crc 2023-10-01T10:37:59.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:58 smithi078 bash[78274]: audit 2023-10-01T10:37:58.621526+0000 mgr.y (mgr.25072) 601 : audit 0 from='client.45377 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:37:59.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:58 smithi173 bash[58816]: cluster 2023-10-01T10:37:57.637607+0000 mgr.y (mgr.25072) 600 : cluster 0 pgmap v505: 73 pgs: 13 stale+active+clean, 60 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:37:59.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:58 smithi173 bash[58816]: cluster 2023-10-01T10:37:57.677443+0000 osd.6 (osd.6) 36 : cluster 3 failed to encode map e124 with expected crc 2023-10-01T10:37:59.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:58 smithi173 bash[58816]: cluster 2023-10-01T10:37:57.677816+0000 osd.7 (osd.7) 60 : cluster 3 failed to encode map e124 with expected crc 2023-10-01T10:37:59.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:58 smithi173 bash[58816]: cluster 2023-10-01T10:37:57.679009+0000 osd.6 (osd.6) 37 : cluster 3 failed to encode map e124 with expected crc 2023-10-01T10:37:59.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:58 smithi173 bash[58816]: cluster 2023-10-01T10:37:57.679506+0000 mon.a (mon.0) 839 : cluster 0 osdmap e124: 8 total, 7 up, 8 in 2023-10-01T10:37:59.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:58 smithi173 bash[58816]: cluster 2023-10-01T10:37:57.679525+0000 osd.7 (osd.7) 61 : cluster 3 failed to encode map e124 with expected crc 2023-10-01T10:37:59.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:58 smithi173 bash[58816]: cluster 2023-10-01T10:37:57.679540+0000 osd.5 (osd.5) 45 : cluster 3 failed to encode map e124 with expected crc 2023-10-01T10:37:59.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:58 smithi173 bash[58816]: cluster 2023-10-01T10:37:57.682337+0000 osd.6 (osd.6) 38 : cluster 3 failed to encode map e124 with expected crc 2023-10-01T10:37:59.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:58 smithi173 bash[58816]: cluster 2023-10-01T10:37:57.682760+0000 osd.5 (osd.5) 46 : cluster 3 failed to encode map e124 with expected crc 2023-10-01T10:37:59.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:58 smithi173 bash[58816]: cluster 2023-10-01T10:37:57.682832+0000 osd.7 (osd.7) 62 : cluster 3 failed to encode map e124 with expected crc 2023-10-01T10:37:59.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:58 smithi173 bash[58816]: cluster 2023-10-01T10:37:57.684169+0000 osd.7 (osd.7) 63 : cluster 3 failed to encode map e124 with expected crc 2023-10-01T10:37:59.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:58 smithi173 bash[58816]: audit 2023-10-01T10:37:58.621526+0000 mgr.y (mgr.25072) 601 : audit 0 from='client.45377 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-10-01T10:37:59.551 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -e sha1=b56822191ad0416d0c05ed59720df03ad9c2a44e -- bash -c 'ceph versions' 2023-10-01T10:37:59.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:37:59 smithi078 bash[78274]: cluster 2023-10-01T10:37:59.638611+0000 mgr.y (mgr.25072) 602 : cluster 0 pgmap v507: 73 pgs: 7 active+undersized+degraded, 6 stale+active+clean, 11 active+undersized, 49 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-01T10:37:59.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:37:59 smithi078 bash[75299]: cluster 2023-10-01T10:37:59.638611+0000 mgr.y (mgr.25072) 602 : cluster 0 pgmap v507: 73 pgs: 7 active+undersized+degraded, 6 stale+active+clean, 11 active+undersized, 49 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-01T10:37:59.998 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:37:59 smithi173 bash[58816]: cluster 2023-10-01T10:37:59.638611+0000 mgr.y (mgr.25072) 602 : cluster 0 pgmap v507: 73 pgs: 7 active+undersized+degraded, 6 stale+active+clean, 11 active+undersized, 49 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12/72 objects degraded (16.667%) 2023-10-01T10:37:59.999 INFO:journalctl@ceph.osd.4.smithi173.stdout:Oct 01 10:37:59 smithi173 systemd[1]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7@osd.4.service: Succeeded. 2023-10-01T10:37:59.999 INFO:journalctl@ceph.osd.4.smithi173.stdout:Oct 01 10:37:59 smithi173 systemd[1]: Stopped Ceph osd.4 for d8016480-6041-11ee-8db5-212e2dc638e7. 2023-10-01T10:38:00.370 INFO:journalctl@ceph.osd.4.smithi173.stdout:Oct 01 10:37:59 smithi173 systemd[1]: Started Ceph osd.4 for d8016480-6041-11ee-8db5-212e2dc638e7. 2023-10-01T10:38:00.582 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:38:00 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:38:00] "GET /metrics HTTP/1.1" 200 33822 "" "Prometheus/2.33.4" 2023-10-01T10:38:00.934 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:00 smithi078 bash[78274]: cluster 2023-10-01T10:37:59.667421+0000 mon.a (mon.0) 840 : cluster 3 Health check failed: Degraded data redundancy: 12/72 objects degraded (16.667%), 7 pgs degraded (PG_DEGRADED) 2023-10-01T10:38:00.934 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:00 smithi078 bash[78274]: audit 2023-10-01T10:38:00.033352+0000 mon.a (mon.0) 841 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:38:00.934 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:00 smithi078 bash[78274]: audit 2023-10-01T10:38:00.040831+0000 mon.a (mon.0) 842 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:38:00.934 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:00 smithi078 bash[78274]: audit 2023-10-01T10:38:00.051111+0000 mon.a (mon.0) 843 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:38:00.935 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:00 smithi078 bash[75299]: cluster 2023-10-01T10:37:59.667421+0000 mon.a (mon.0) 840 : cluster 3 Health check failed: Degraded data redundancy: 12/72 objects degraded (16.667%), 7 pgs degraded (PG_DEGRADED) 2023-10-01T10:38:00.935 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:00 smithi078 bash[75299]: audit 2023-10-01T10:38:00.033352+0000 mon.a (mon.0) 841 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:38:00.935 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:00 smithi078 bash[75299]: audit 2023-10-01T10:38:00.040831+0000 mon.a (mon.0) 842 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:38:00.935 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:00 smithi078 bash[75299]: audit 2023-10-01T10:38:00.051111+0000 mon.a (mon.0) 843 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:38:00.967 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:00 smithi173 bash[58816]: cluster 2023-10-01T10:37:59.667421+0000 mon.a (mon.0) 840 : cluster 3 Health check failed: Degraded data redundancy: 12/72 objects degraded (16.667%), 7 pgs degraded (PG_DEGRADED) 2023-10-01T10:38:00.967 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:00 smithi173 bash[58816]: audit 2023-10-01T10:38:00.033352+0000 mon.a (mon.0) 841 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:38:00.967 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:00 smithi173 bash[58816]: audit 2023-10-01T10:38:00.040831+0000 mon.a (mon.0) 842 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:38:00.967 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:00 smithi173 bash[58816]: audit 2023-10-01T10:38:00.051111+0000 mon.a (mon.0) 843 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:38:01.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:01 smithi173 bash[58816]: cluster 2023-10-01T10:38:01.639652+0000 mgr.y (mgr.25072) 603 : cluster 0 pgmap v508: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2023-10-01T10:38:01.871 INFO:journalctl@ceph.osd.4.smithi173.stdout:Oct 01 10:38:01 smithi173 bash[64256]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-10-01T10:38:01.871 INFO:journalctl@ceph.osd.4.smithi173.stdout:Oct 01 10:38:01 smithi173 bash[64256]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2023-10-01T10:38:01.871 INFO:journalctl@ceph.osd.4.smithi173.stdout:Oct 01 10:38:01 smithi173 bash[64256]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-4 --no-mon-config 2023-10-01T10:38:01.871 INFO:journalctl@ceph.osd.4.smithi173.stdout:Oct 01 10:38:01 smithi173 bash[64256]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-4/block 2023-10-01T10:38:01.871 INFO:journalctl@ceph.osd.4.smithi173.stdout:Oct 01 10:38:01 smithi173 bash[64256]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-4/block 2023-10-01T10:38:01.871 INFO:journalctl@ceph.osd.4.smithi173.stdout:Oct 01 10:38:01 smithi173 bash[64256]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2023-10-01T10:38:01.871 INFO:journalctl@ceph.osd.4.smithi173.stdout:Oct 01 10:38:01 smithi173 bash[64256]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2023-10-01T10:38:01.871 INFO:journalctl@ceph.osd.4.smithi173.stdout:Oct 01 10:38:01 smithi173 bash[64256]: --> ceph-volume lvm activate successful for osd ID: 4 2023-10-01T10:38:02.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:01 smithi078 bash[78274]: cluster 2023-10-01T10:38:01.639652+0000 mgr.y (mgr.25072) 603 : cluster 0 pgmap v508: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2023-10-01T10:38:02.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:01 smithi078 bash[75299]: cluster 2023-10-01T10:38:01.639652+0000 mgr.y (mgr.25072) 603 : cluster 0 pgmap v508: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2023-10-01T10:38:03.114 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2023-10-01T10:38:03.115 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2023-10-01T10:38:03.115 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 18.2.0-497-gb5682219 (b56822191ad0416d0c05ed59720df03ad9c2a44e) reef (stable)": 3 2023-10-01T10:38:03.115 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:38:03.115 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2023-10-01T10:38:03.115 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 18.2.0-497-gb5682219 (b56822191ad0416d0c05ed59720df03ad9c2a44e) reef (stable)": 2 2023-10-01T10:38:03.115 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:38:03.115 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2023-10-01T10:38:03.115 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)": 3, 2023-10-01T10:38:03.115 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 18.2.0-497-gb5682219 (b56822191ad0416d0c05ed59720df03ad9c2a44e) reef (stable)": 4 2023-10-01T10:38:03.115 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:38:03.116 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2023-10-01T10:38:03.116 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)": 2 2023-10-01T10:38:03.116 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:38:03.116 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2023-10-01T10:38:03.116 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 16.2.14-44-geeb13ac9 (eeb13ac974775bd0cce96a2c687e8bcc2faca43d) pacific (stable)": 5, 2023-10-01T10:38:03.116 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 18.2.0-497-gb5682219 (b56822191ad0416d0c05ed59720df03ad9c2a44e) reef (stable)": 9 2023-10-01T10:38:03.116 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:38:03.116 INFO:teuthology.orchestra.run.smithi078.stdout:} 2023-10-01T10:38:03.369 INFO:journalctl@ceph.osd.4.smithi173.stdout:Oct 01 10:38:03 smithi173 bash[64546]: debug 2023-10-01T10:38:03.056+0000 7f33fd84a700 1 -- 172.21.15.173:0/2880275067 <== mon.1 v2:172.21.15.173:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x562ea55304e0 con 0x562ea558a800 2023-10-01T10:38:03.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:03 smithi078 bash[78274]: audit 2023-10-01T10:38:02.413685+0000 mon.a (mon.0) 844 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:38:03.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:03 smithi078 bash[78274]: audit 2023-10-01T10:38:02.415192+0000 mon.a (mon.0) 845 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:38:03.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:03 smithi078 bash[78274]: audit 2023-10-01T10:38:03.112445+0000 mon.c (mon.2) 127 : audit 0 from='client.? 172.21.15.78:0/769697579' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:38:03.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:03 smithi078 bash[75299]: audit 2023-10-01T10:38:02.413685+0000 mon.a (mon.0) 844 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:38:03.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:03 smithi078 bash[75299]: audit 2023-10-01T10:38:02.415192+0000 mon.a (mon.0) 845 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:38:03.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:03 smithi078 bash[75299]: audit 2023-10-01T10:38:03.112445+0000 mon.c (mon.2) 127 : audit 0 from='client.? 172.21.15.78:0/769697579' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:38:03.870 INFO:journalctl@ceph.osd.4.smithi173.stdout:Oct 01 10:38:03 smithi173 bash[64546]: debug 2023-10-01T10:38:03.600+0000 7f34069d0640 -1 Falling back to public interface 2023-10-01T10:38:03.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:03 smithi173 bash[58816]: audit 2023-10-01T10:38:02.413685+0000 mon.a (mon.0) 844 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:38:03.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:03 smithi173 bash[58816]: audit 2023-10-01T10:38:02.415192+0000 mon.a (mon.0) 845 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:38:03.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:03 smithi173 bash[58816]: audit 2023-10-01T10:38:03.112445+0000 mon.c (mon.2) 127 : audit 0 from='client.? 172.21.15.78:0/769697579' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-10-01T10:38:03.908 DEBUG:teuthology.parallel:result is None 2023-10-01T10:38:03.908 INFO:teuthology.run_tasks:Running task print... 2023-10-01T10:38:03.919 INFO:teuthology.task.print:**** done upgrading first half of cluster **** 2023-10-01T10:38:03.919 INFO:teuthology.run_tasks:Running task ceph.healthy... 2023-10-01T10:38:03.928 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-10-01T10:38:03.928 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-10-01T10:38:03.929 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph mgr dump --format=json 2023-10-01T10:38:04.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:04 smithi078 bash[78274]: cluster 2023-10-01T10:38:03.640669+0000 mgr.y (mgr.25072) 604 : cluster 0 pgmap v509: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2023-10-01T10:38:04.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:04 smithi078 bash[75299]: cluster 2023-10-01T10:38:03.640669+0000 mgr.y (mgr.25072) 604 : cluster 0 pgmap v509: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2023-10-01T10:38:04.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:04 smithi173 bash[58816]: cluster 2023-10-01T10:38:03.640669+0000 mgr.y (mgr.25072) 604 : cluster 0 pgmap v509: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2023-10-01T10:38:06.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:05 smithi078 bash[75299]: cluster 2023-10-01T10:38:05.641755+0000 mgr.y (mgr.25072) 605 : cluster 0 pgmap v510: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2023-10-01T10:38:06.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:05 smithi078 bash[78274]: cluster 2023-10-01T10:38:05.641755+0000 mgr.y (mgr.25072) 605 : cluster 0 pgmap v510: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2023-10-01T10:38:06.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:05 smithi173 bash[58816]: cluster 2023-10-01T10:38:05.641755+0000 mgr.y (mgr.25072) 605 : cluster 0 pgmap v510: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2023-10-01T10:38:07.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:07 smithi078 bash[75299]: audit 2023-10-01T10:38:06.299300+0000 mon.a (mon.0) 846 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:38:07.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:07 smithi078 bash[75299]: audit 2023-10-01T10:38:06.306458+0000 mon.a (mon.0) 847 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:38:07.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:07 smithi078 bash[75299]: cluster 2023-10-01T10:38:06.982699+0000 mon.a (mon.0) 848 : cluster 3 Health check update: Degraded data redundancy: 14/72 objects degraded (19.444%), 9 pgs degraded (PG_DEGRADED) 2023-10-01T10:38:07.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:07 smithi078 bash[75299]: audit 2023-10-01T10:38:07.107845+0000 mon.a (mon.0) 849 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:38:07.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:07 smithi078 bash[75299]: audit 2023-10-01T10:38:07.119267+0000 mon.a (mon.0) 850 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:38:07.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:07 smithi078 bash[78274]: audit 2023-10-01T10:38:06.299300+0000 mon.a (mon.0) 846 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:38:07.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:07 smithi078 bash[78274]: audit 2023-10-01T10:38:06.306458+0000 mon.a (mon.0) 847 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:38:07.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:07 smithi078 bash[78274]: cluster 2023-10-01T10:38:06.982699+0000 mon.a (mon.0) 848 : cluster 3 Health check update: Degraded data redundancy: 14/72 objects degraded (19.444%), 9 pgs degraded (PG_DEGRADED) 2023-10-01T10:38:07.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:07 smithi078 bash[78274]: audit 2023-10-01T10:38:07.107845+0000 mon.a (mon.0) 849 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:38:07.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:07 smithi078 bash[78274]: audit 2023-10-01T10:38:07.119267+0000 mon.a (mon.0) 850 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:38:07.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:07 smithi173 bash[58816]: audit 2023-10-01T10:38:06.299300+0000 mon.a (mon.0) 846 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:38:07.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:07 smithi173 bash[58816]: audit 2023-10-01T10:38:06.306458+0000 mon.a (mon.0) 847 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:38:07.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:07 smithi173 bash[58816]: cluster 2023-10-01T10:38:06.982699+0000 mon.a (mon.0) 848 : cluster 3 Health check update: Degraded data redundancy: 14/72 objects degraded (19.444%), 9 pgs degraded (PG_DEGRADED) 2023-10-01T10:38:07.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:07 smithi173 bash[58816]: audit 2023-10-01T10:38:07.107845+0000 mon.a (mon.0) 849 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:38:07.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:07 smithi173 bash[58816]: audit 2023-10-01T10:38:07.119267+0000 mon.a (mon.0) 850 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:38:08.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:08 smithi078 bash[75299]: cluster 2023-10-01T10:38:07.642728+0000 mgr.y (mgr.25072) 606 : cluster 0 pgmap v511: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2023-10-01T10:38:08.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:08 smithi078 bash[78274]: cluster 2023-10-01T10:38:07.642728+0000 mgr.y (mgr.25072) 606 : cluster 0 pgmap v511: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2023-10-01T10:38:08.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:08 smithi173 bash[58816]: cluster 2023-10-01T10:38:07.642728+0000 mgr.y (mgr.25072) 606 : cluster 0 pgmap v511: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2023-10-01T10:38:08.719 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:38:09.978 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:09 smithi078 bash[78274]: cluster 2023-10-01T10:38:09.643806+0000 mgr.y (mgr.25072) 607 : cluster 0 pgmap v512: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2023-10-01T10:38:09.978 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:09 smithi078 bash[75299]: cluster 2023-10-01T10:38:09.643806+0000 mgr.y (mgr.25072) 607 : cluster 0 pgmap v512: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2023-10-01T10:38:10.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:09 smithi173 bash[58816]: cluster 2023-10-01T10:38:09.643806+0000 mgr.y (mgr.25072) 607 : cluster 0 pgmap v512: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2023-10-01T10:38:10.582 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:38:10 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:38:10] "GET /metrics HTTP/1.1" 200 33828 "" "Prometheus/2.33.4" 2023-10-01T10:38:11.989 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:11 smithi173 bash[58816]: cluster 2023-10-01T10:38:11.644898+0000 mgr.y (mgr.25072) 608 : cluster 0 pgmap v513: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2023-10-01T10:38:12.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:11 smithi078 bash[75299]: cluster 2023-10-01T10:38:11.644898+0000 mgr.y (mgr.25072) 608 : cluster 0 pgmap v513: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2023-10-01T10:38:12.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:11 smithi078 bash[78274]: cluster 2023-10-01T10:38:11.644898+0000 mgr.y (mgr.25072) 608 : cluster 0 pgmap v513: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2023-10-01T10:38:12.234 INFO:teuthology.orchestra.run.smithi078.stdout: 2023-10-01T10:38:13.115 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":34,"active_gid":25072,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6800","nonce":974720338},{"type":"v1","addr":"172.21.15.78:6801","nonce":974720338}]},"active_addr":"172.21.15.78:6801/974720338","active_change":"2023-10-01T10:22:47.234337+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[{"gid":25066,"name":"x","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph:v18","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:0.0.2","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph:v18","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:0.0.2","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.78:8443/","prometheus":"http://172.21.15.78:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":76,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.78:0","nonce":1900455377}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.78:0","nonce":1670862790}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.78:0","nonce":1663636325}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.78:0","nonce":3030049856}]}]} 2023-10-01T10:38:13.119 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-10-01T10:38:13.120 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-10-01T10:38:13.120 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-01T10:38:13.234 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:12 smithi173 bash[58816]: audit 2023-10-01T10:38:12.215905+0000 mon.a (mon.0) 851 : audit 0 from='client.? 172.21.15.78:0/4272018701' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-01T10:38:13.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:12 smithi078 bash[75299]: audit 2023-10-01T10:38:12.215905+0000 mon.a (mon.0) 851 : audit 0 from='client.? 172.21.15.78:0/4272018701' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-01T10:38:13.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:12 smithi078 bash[78274]: audit 2023-10-01T10:38:12.215905+0000 mon.a (mon.0) 851 : audit 0 from='client.? 172.21.15.78:0/4272018701' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-10-01T10:38:13.995 INFO:journalctl@ceph.osd.4.smithi173.stdout:Oct 01 10:38:13 smithi173 bash[64546]: debug 2023-10-01T10:38:13.628+0000 7f34069d0640 -1 osd.4 122 log_to_monitors true 2023-10-01T10:38:14.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:13 smithi078 bash[78274]: audit 2023-10-01T10:38:13.634832+0000 mon.a (mon.0) 852 : audit 1 from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-01T10:38:14.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:13 smithi078 bash[78274]: audit 2023-10-01T10:38:13.635169+0000 mon.b (mon.1) 38 : audit 1 from='osd.4 [v2:172.21.15.173:6800/3220463704,v1:172.21.15.173:6801/3220463704]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-01T10:38:14.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:13 smithi078 bash[78274]: cluster 2023-10-01T10:38:13.645843+0000 mgr.y (mgr.25072) 609 : cluster 0 pgmap v514: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2023-10-01T10:38:14.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:13 smithi078 bash[75299]: audit 2023-10-01T10:38:13.634832+0000 mon.a (mon.0) 852 : audit 1 from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-01T10:38:14.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:13 smithi078 bash[75299]: audit 2023-10-01T10:38:13.635169+0000 mon.b (mon.1) 38 : audit 1 from='osd.4 [v2:172.21.15.173:6800/3220463704,v1:172.21.15.173:6801/3220463704]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-01T10:38:14.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:13 smithi078 bash[75299]: cluster 2023-10-01T10:38:13.645843+0000 mgr.y (mgr.25072) 609 : cluster 0 pgmap v514: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2023-10-01T10:38:14.369 INFO:journalctl@ceph.osd.4.smithi173.stdout:Oct 01 10:38:14 smithi173 bash[64546]: debug 2023-10-01T10:38:14.048+0000 7f33f3e34700 -1 osd.4 122 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-10-01T10:38:14.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:13 smithi173 bash[58816]: audit 2023-10-01T10:38:13.634832+0000 mon.a (mon.0) 852 : audit 1 from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-01T10:38:14.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:13 smithi173 bash[58816]: audit 2023-10-01T10:38:13.635169+0000 mon.b (mon.1) 38 : audit 1 from='osd.4 [v2:172.21.15.173:6800/3220463704,v1:172.21.15.173:6801/3220463704]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-10-01T10:38:14.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:13 smithi173 bash[58816]: cluster 2023-10-01T10:38:13.645843+0000 mgr.y (mgr.25072) 609 : cluster 0 pgmap v514: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2023-10-01T10:38:15.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:15 smithi078 bash[75299]: cluster 2023-10-01T10:38:14.003862+0000 osd.7 (osd.7) 64 : cluster 3 failed to encode map e125 with expected crc 2023-10-01T10:38:15.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:15 smithi078 bash[75299]: audit 2023-10-01T10:38:14.004524+0000 mon.a (mon.0) 853 : audit 1 from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-10-01T10:38:15.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:15 smithi078 bash[75299]: cluster 2023-10-01T10:38:14.004634+0000 mon.a (mon.0) 854 : cluster 0 osdmap e125: 8 total, 7 up, 8 in 2023-10-01T10:38:15.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:15 smithi078 bash[75299]: audit 2023-10-01T10:38:14.007144+0000 mon.a (mon.0) 855 : audit 1 from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-10-01T10:38:15.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:15 smithi078 bash[75299]: audit 2023-10-01T10:38:14.007612+0000 mon.b (mon.1) 39 : audit 1 from='osd.4 [v2:172.21.15.173:6800/3220463704,v1:172.21.15.173:6801/3220463704]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-10-01T10:38:15.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:15 smithi078 bash[78274]: cluster 2023-10-01T10:38:14.003862+0000 osd.7 (osd.7) 64 : cluster 3 failed to encode map e125 with expected crc 2023-10-01T10:38:15.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:15 smithi078 bash[78274]: audit 2023-10-01T10:38:14.004524+0000 mon.a (mon.0) 853 : audit 1 from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-10-01T10:38:15.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:15 smithi078 bash[78274]: cluster 2023-10-01T10:38:14.004634+0000 mon.a (mon.0) 854 : cluster 0 osdmap e125: 8 total, 7 up, 8 in 2023-10-01T10:38:15.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:15 smithi078 bash[78274]: audit 2023-10-01T10:38:14.007144+0000 mon.a (mon.0) 855 : audit 1 from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-10-01T10:38:15.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:15 smithi078 bash[78274]: audit 2023-10-01T10:38:14.007612+0000 mon.b (mon.1) 39 : audit 1 from='osd.4 [v2:172.21.15.173:6800/3220463704,v1:172.21.15.173:6801/3220463704]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-10-01T10:38:15.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:15 smithi173 bash[58816]: cluster 2023-10-01T10:38:14.003862+0000 osd.7 (osd.7) 64 : cluster 3 failed to encode map e125 with expected crc 2023-10-01T10:38:15.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:15 smithi173 bash[58816]: audit 2023-10-01T10:38:14.004524+0000 mon.a (mon.0) 853 : audit 1 from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-10-01T10:38:15.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:15 smithi173 bash[58816]: cluster 2023-10-01T10:38:14.004634+0000 mon.a (mon.0) 854 : cluster 0 osdmap e125: 8 total, 7 up, 8 in 2023-10-01T10:38:15.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:15 smithi173 bash[58816]: audit 2023-10-01T10:38:14.007144+0000 mon.a (mon.0) 855 : audit 1 from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-10-01T10:38:15.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:15 smithi173 bash[58816]: audit 2023-10-01T10:38:14.007612+0000 mon.b (mon.1) 39 : audit 1 from='osd.4 [v2:172.21.15.173:6800/3220463704,v1:172.21.15.173:6801/3220463704]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi173", "root=default"]}]: dispatch 2023-10-01T10:38:16.034 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:16 smithi173 bash[58816]: cluster 2023-10-01T10:38:14.923173+0000 osd.6 (osd.6) 39 : cluster 3 failed to encode map e125 with expected crc 2023-10-01T10:38:16.034 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:16 smithi173 bash[58816]: cluster 2023-10-01T10:38:14.925883+0000 osd.6 (osd.6) 40 : cluster 3 failed to encode map e125 with expected crc 2023-10-01T10:38:16.034 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:16 smithi173 bash[58816]: cluster 2023-10-01T10:38:15.002006+0000 mon.a (mon.0) 856 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-01T10:38:16.035 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:16 smithi173 bash[58816]: cluster 2023-10-01T10:38:15.011476+0000 osd.7 (osd.7) 65 : cluster 3 failed to encode map e126 with expected crc 2023-10-01T10:38:16.035 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:16 smithi173 bash[58816]: cluster 2023-10-01T10:38:15.011542+0000 mon.a (mon.0) 857 : cluster 1 osd.4 [v2:172.21.15.173:6800/3220463704,v1:172.21.15.173:6801/3220463704] boot 2023-10-01T10:38:16.035 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:16 smithi173 bash[58816]: cluster 2023-10-01T10:38:15.011604+0000 mon.a (mon.0) 858 : cluster 0 osdmap e126: 8 total, 8 up, 8 in 2023-10-01T10:38:16.035 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:16 smithi173 bash[58816]: audit 2023-10-01T10:38:15.013113+0000 mon.a (mon.0) 859 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T10:38:16.035 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:16 smithi173 bash[58816]: cluster 2023-10-01T10:38:15.015814+0000 osd.7 (osd.7) 66 : cluster 3 failed to encode map e126 with expected crc 2023-10-01T10:38:16.035 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:16 smithi173 bash[58816]: cluster 2023-10-01T10:38:15.025398+0000 osd.5 (osd.5) 47 : cluster 3 failed to encode map e126 with expected crc 2023-10-01T10:38:16.035 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:16 smithi173 bash[58816]: cluster 2023-10-01T10:38:15.646721+0000 mgr.y (mgr.25072) 610 : cluster 0 pgmap v517: 73 pgs: 3 peering, 9 active+undersized+degraded, 21 active+undersized, 40 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2023-10-01T10:38:16.035 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:16 smithi173 bash[58816]: cluster 2023-10-01T10:38:16.014527+0000 mon.a (mon.0) 860 : cluster 0 osdmap e127: 8 total, 8 up, 8 in 2023-10-01T10:38:16.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:16 smithi078 bash[75299]: cluster 2023-10-01T10:38:14.923173+0000 osd.6 (osd.6) 39 : cluster 3 failed to encode map e125 with expected crc 2023-10-01T10:38:16.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:16 smithi078 bash[75299]: cluster 2023-10-01T10:38:14.925883+0000 osd.6 (osd.6) 40 : cluster 3 failed to encode map e125 with expected crc 2023-10-01T10:38:16.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:16 smithi078 bash[75299]: cluster 2023-10-01T10:38:15.002006+0000 mon.a (mon.0) 856 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-01T10:38:16.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:16 smithi078 bash[75299]: cluster 2023-10-01T10:38:15.011476+0000 osd.7 (osd.7) 65 : cluster 3 failed to encode map e126 with expected crc 2023-10-01T10:38:16.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:16 smithi078 bash[75299]: cluster 2023-10-01T10:38:15.011542+0000 mon.a (mon.0) 857 : cluster 1 osd.4 [v2:172.21.15.173:6800/3220463704,v1:172.21.15.173:6801/3220463704] boot 2023-10-01T10:38:16.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:16 smithi078 bash[75299]: cluster 2023-10-01T10:38:15.011604+0000 mon.a (mon.0) 858 : cluster 0 osdmap e126: 8 total, 8 up, 8 in 2023-10-01T10:38:16.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:16 smithi078 bash[75299]: audit 2023-10-01T10:38:15.013113+0000 mon.a (mon.0) 859 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T10:38:16.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:16 smithi078 bash[75299]: cluster 2023-10-01T10:38:15.015814+0000 osd.7 (osd.7) 66 : cluster 3 failed to encode map e126 with expected crc 2023-10-01T10:38:16.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:16 smithi078 bash[75299]: cluster 2023-10-01T10:38:15.025398+0000 osd.5 (osd.5) 47 : cluster 3 failed to encode map e126 with expected crc 2023-10-01T10:38:16.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:16 smithi078 bash[75299]: cluster 2023-10-01T10:38:15.646721+0000 mgr.y (mgr.25072) 610 : cluster 0 pgmap v517: 73 pgs: 3 peering, 9 active+undersized+degraded, 21 active+undersized, 40 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2023-10-01T10:38:16.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:16 smithi078 bash[75299]: cluster 2023-10-01T10:38:16.014527+0000 mon.a (mon.0) 860 : cluster 0 osdmap e127: 8 total, 8 up, 8 in 2023-10-01T10:38:16.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:16 smithi078 bash[78274]: cluster 2023-10-01T10:38:14.923173+0000 osd.6 (osd.6) 39 : cluster 3 failed to encode map e125 with expected crc 2023-10-01T10:38:16.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:16 smithi078 bash[78274]: cluster 2023-10-01T10:38:14.925883+0000 osd.6 (osd.6) 40 : cluster 3 failed to encode map e125 with expected crc 2023-10-01T10:38:16.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:16 smithi078 bash[78274]: cluster 2023-10-01T10:38:15.002006+0000 mon.a (mon.0) 856 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2023-10-01T10:38:16.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:16 smithi078 bash[78274]: cluster 2023-10-01T10:38:15.011476+0000 osd.7 (osd.7) 65 : cluster 3 failed to encode map e126 with expected crc 2023-10-01T10:38:16.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:16 smithi078 bash[78274]: cluster 2023-10-01T10:38:15.011542+0000 mon.a (mon.0) 857 : cluster 1 osd.4 [v2:172.21.15.173:6800/3220463704,v1:172.21.15.173:6801/3220463704] boot 2023-10-01T10:38:16.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:16 smithi078 bash[78274]: cluster 2023-10-01T10:38:15.011604+0000 mon.a (mon.0) 858 : cluster 0 osdmap e126: 8 total, 8 up, 8 in 2023-10-01T10:38:16.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:16 smithi078 bash[78274]: audit 2023-10-01T10:38:15.013113+0000 mon.a (mon.0) 859 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-10-01T10:38:16.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:16 smithi078 bash[78274]: cluster 2023-10-01T10:38:15.015814+0000 osd.7 (osd.7) 66 : cluster 3 failed to encode map e126 with expected crc 2023-10-01T10:38:16.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:16 smithi078 bash[78274]: cluster 2023-10-01T10:38:15.025398+0000 osd.5 (osd.5) 47 : cluster 3 failed to encode map e126 with expected crc 2023-10-01T10:38:16.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:16 smithi078 bash[78274]: cluster 2023-10-01T10:38:15.646721+0000 mgr.y (mgr.25072) 610 : cluster 0 pgmap v517: 73 pgs: 3 peering, 9 active+undersized+degraded, 21 active+undersized, 40 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 14/72 objects degraded (19.444%) 2023-10-01T10:38:16.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:16 smithi078 bash[78274]: cluster 2023-10-01T10:38:16.014527+0000 mon.a (mon.0) 860 : cluster 0 osdmap e127: 8 total, 8 up, 8 in 2023-10-01T10:38:17.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:17 smithi078 bash[75299]: cluster 2023-10-01T10:38:16.013627+0000 osd.5 (osd.5) 48 : cluster 3 failed to encode map e127 with expected crc 2023-10-01T10:38:17.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:17 smithi078 bash[75299]: cluster 2023-10-01T10:38:16.013976+0000 osd.7 (osd.7) 67 : cluster 3 failed to encode map e127 with expected crc 2023-10-01T10:38:17.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:17 smithi078 bash[75299]: cluster 2023-10-01T10:38:16.015188+0000 osd.6 (osd.6) 41 : cluster 3 failed to encode map e127 with expected crc 2023-10-01T10:38:17.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:17 smithi078 bash[75299]: cluster 2023-10-01T10:38:16.015850+0000 osd.7 (osd.7) 68 : cluster 3 failed to encode map e127 with expected crc 2023-10-01T10:38:17.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:17 smithi078 bash[75299]: cluster 2023-10-01T10:38:16.017663+0000 osd.7 (osd.7) 69 : cluster 3 failed to encode map e127 with expected crc 2023-10-01T10:38:17.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:17 smithi078 bash[75299]: cluster 2023-10-01T10:38:16.018072+0000 osd.5 (osd.5) 49 : cluster 3 failed to encode map e127 with expected crc 2023-10-01T10:38:17.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:17 smithi078 bash[75299]: cluster 2023-10-01T10:38:16.018551+0000 osd.6 (osd.6) 42 : cluster 3 failed to encode map e127 with expected crc 2023-10-01T10:38:17.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:17 smithi078 bash[75299]: cluster 2023-10-01T10:38:16.019200+0000 osd.7 (osd.7) 70 : cluster 3 failed to encode map e127 with expected crc 2023-10-01T10:38:17.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:17 smithi078 bash[75299]: audit 2023-10-01T10:38:16.265766+0000 mon.a (mon.0) 861 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:38:17.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:17 smithi078 bash[75299]: audit 2023-10-01T10:38:16.273323+0000 mon.a (mon.0) 862 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:38:17.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:17 smithi078 bash[75299]: audit 2023-10-01T10:38:16.275045+0000 mon.a (mon.0) 863 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:38:17.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:17 smithi078 bash[75299]: audit 2023-10-01T10:38:16.276440+0000 mon.a (mon.0) 864 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:38:17.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:17 smithi078 bash[75299]: audit 2023-10-01T10:38:16.283444+0000 mon.a (mon.0) 865 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:38:17.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:17 smithi078 bash[75299]: audit 2023-10-01T10:38:16.365142+0000 mon.a (mon.0) 866 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:38:17.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:17 smithi078 bash[75299]: audit 2023-10-01T10:38:16.368620+0000 mon.a (mon.0) 867 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:38:17.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:17 smithi078 bash[75299]: audit 2023-10-01T10:38:16.370076+0000 mon.a (mon.0) 868 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:38:17.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:17 smithi078 bash[75299]: audit 2023-10-01T10:38:16.378354+0000 mon.a (mon.0) 869 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:38:17.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:17 smithi078 bash[78274]: cluster 2023-10-01T10:38:16.013627+0000 osd.5 (osd.5) 48 : cluster 3 failed to encode map e127 with expected crc 2023-10-01T10:38:17.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:17 smithi078 bash[78274]: cluster 2023-10-01T10:38:16.013976+0000 osd.7 (osd.7) 67 : cluster 3 failed to encode map e127 with expected crc 2023-10-01T10:38:17.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:17 smithi078 bash[78274]: cluster 2023-10-01T10:38:16.015188+0000 osd.6 (osd.6) 41 : cluster 3 failed to encode map e127 with expected crc 2023-10-01T10:38:17.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:17 smithi078 bash[78274]: cluster 2023-10-01T10:38:16.015850+0000 osd.7 (osd.7) 68 : cluster 3 failed to encode map e127 with expected crc 2023-10-01T10:38:17.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:17 smithi078 bash[78274]: cluster 2023-10-01T10:38:16.017663+0000 osd.7 (osd.7) 69 : cluster 3 failed to encode map e127 with expected crc 2023-10-01T10:38:17.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:17 smithi078 bash[78274]: cluster 2023-10-01T10:38:16.018072+0000 osd.5 (osd.5) 49 : cluster 3 failed to encode map e127 with expected crc 2023-10-01T10:38:17.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:17 smithi078 bash[78274]: cluster 2023-10-01T10:38:16.018551+0000 osd.6 (osd.6) 42 : cluster 3 failed to encode map e127 with expected crc 2023-10-01T10:38:17.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:17 smithi078 bash[78274]: cluster 2023-10-01T10:38:16.019200+0000 osd.7 (osd.7) 70 : cluster 3 failed to encode map e127 with expected crc 2023-10-01T10:38:17.337 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:17 smithi078 bash[78274]: audit 2023-10-01T10:38:16.265766+0000 mon.a (mon.0) 861 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:38:17.337 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:17 smithi078 bash[78274]: audit 2023-10-01T10:38:16.273323+0000 mon.a (mon.0) 862 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:38:17.337 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:17 smithi078 bash[78274]: audit 2023-10-01T10:38:16.275045+0000 mon.a (mon.0) 863 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:38:17.337 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:17 smithi078 bash[78274]: audit 2023-10-01T10:38:16.276440+0000 mon.a (mon.0) 864 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:38:17.337 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:17 smithi078 bash[78274]: audit 2023-10-01T10:38:16.283444+0000 mon.a (mon.0) 865 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:38:17.337 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:17 smithi078 bash[78274]: audit 2023-10-01T10:38:16.365142+0000 mon.a (mon.0) 866 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:38:17.338 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:17 smithi078 bash[78274]: audit 2023-10-01T10:38:16.368620+0000 mon.a (mon.0) 867 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:38:17.338 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:17 smithi078 bash[78274]: audit 2023-10-01T10:38:16.370076+0000 mon.a (mon.0) 868 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:38:17.338 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:17 smithi078 bash[78274]: audit 2023-10-01T10:38:16.378354+0000 mon.a (mon.0) 869 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:38:17.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:17 smithi173 bash[58816]: cluster 2023-10-01T10:38:16.013627+0000 osd.5 (osd.5) 48 : cluster 3 failed to encode map e127 with expected crc 2023-10-01T10:38:17.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:17 smithi173 bash[58816]: cluster 2023-10-01T10:38:16.013976+0000 osd.7 (osd.7) 67 : cluster 3 failed to encode map e127 with expected crc 2023-10-01T10:38:17.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:17 smithi173 bash[58816]: cluster 2023-10-01T10:38:16.015188+0000 osd.6 (osd.6) 41 : cluster 3 failed to encode map e127 with expected crc 2023-10-01T10:38:17.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:17 smithi173 bash[58816]: cluster 2023-10-01T10:38:16.015850+0000 osd.7 (osd.7) 68 : cluster 3 failed to encode map e127 with expected crc 2023-10-01T10:38:17.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:17 smithi173 bash[58816]: cluster 2023-10-01T10:38:16.017663+0000 osd.7 (osd.7) 69 : cluster 3 failed to encode map e127 with expected crc 2023-10-01T10:38:17.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:17 smithi173 bash[58816]: cluster 2023-10-01T10:38:16.018072+0000 osd.5 (osd.5) 49 : cluster 3 failed to encode map e127 with expected crc 2023-10-01T10:38:17.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:17 smithi173 bash[58816]: cluster 2023-10-01T10:38:16.018551+0000 osd.6 (osd.6) 42 : cluster 3 failed to encode map e127 with expected crc 2023-10-01T10:38:17.372 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:17 smithi173 bash[58816]: cluster 2023-10-01T10:38:16.019200+0000 osd.7 (osd.7) 70 : cluster 3 failed to encode map e127 with expected crc 2023-10-01T10:38:17.372 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:17 smithi173 bash[58816]: audit 2023-10-01T10:38:16.265766+0000 mon.a (mon.0) 861 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:38:17.372 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:17 smithi173 bash[58816]: audit 2023-10-01T10:38:16.273323+0000 mon.a (mon.0) 862 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:38:17.372 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:17 smithi173 bash[58816]: audit 2023-10-01T10:38:16.275045+0000 mon.a (mon.0) 863 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:38:17.372 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:17 smithi173 bash[58816]: audit 2023-10-01T10:38:16.276440+0000 mon.a (mon.0) 864 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:38:17.372 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:17 smithi173 bash[58816]: audit 2023-10-01T10:38:16.283444+0000 mon.a (mon.0) 865 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:38:17.373 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:17 smithi173 bash[58816]: audit 2023-10-01T10:38:16.365142+0000 mon.a (mon.0) 866 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:38:17.373 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:17 smithi173 bash[58816]: audit 2023-10-01T10:38:16.368620+0000 mon.a (mon.0) 867 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:38:17.373 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:17 smithi173 bash[58816]: audit 2023-10-01T10:38:16.370076+0000 mon.a (mon.0) 868 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:38:17.373 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:17 smithi173 bash[58816]: audit 2023-10-01T10:38:16.378354+0000 mon.a (mon.0) 869 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:38:17.904 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:38:18.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:18 smithi078 bash[75299]: audit 2023-10-01T10:38:17.415732+0000 mon.a (mon.0) 870 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:38:18.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:18 smithi078 bash[75299]: audit 2023-10-01T10:38:17.417536+0000 mon.a (mon.0) 871 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:38:18.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:18 smithi078 bash[75299]: cluster 2023-10-01T10:38:17.647450+0000 mgr.y (mgr.25072) 611 : cluster 0 pgmap v519: 73 pgs: 5 peering, 8 active+undersized+degraded, 20 active+undersized, 40 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 13/72 objects degraded (18.056%) 2023-10-01T10:38:18.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:18 smithi078 bash[78274]: audit 2023-10-01T10:38:17.415732+0000 mon.a (mon.0) 870 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:38:18.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:18 smithi078 bash[78274]: audit 2023-10-01T10:38:17.417536+0000 mon.a (mon.0) 871 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:38:18.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:18 smithi078 bash[78274]: cluster 2023-10-01T10:38:17.647450+0000 mgr.y (mgr.25072) 611 : cluster 0 pgmap v519: 73 pgs: 5 peering, 8 active+undersized+degraded, 20 active+undersized, 40 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 13/72 objects degraded (18.056%) 2023-10-01T10:38:18.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:18 smithi173 bash[58816]: audit 2023-10-01T10:38:17.415732+0000 mon.a (mon.0) 870 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:38:18.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:18 smithi173 bash[58816]: audit 2023-10-01T10:38:17.417536+0000 mon.a (mon.0) 871 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:38:18.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:18 smithi173 bash[58816]: cluster 2023-10-01T10:38:17.647450+0000 mgr.y (mgr.25072) 611 : cluster 0 pgmap v519: 73 pgs: 5 peering, 8 active+undersized+degraded, 20 active+undersized, 40 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 13/72 objects degraded (18.056%) 2023-10-01T10:38:19.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:19 smithi078 bash[78274]: cluster 2023-10-01T10:38:18.409457+0000 mon.a (mon.0) 872 : cluster 3 Health check update: Degraded data redundancy: 13/72 objects degraded (18.056%), 8 pgs degraded (PG_DEGRADED) 2023-10-01T10:38:19.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:19 smithi078 bash[75299]: cluster 2023-10-01T10:38:18.409457+0000 mon.a (mon.0) 872 : cluster 3 Health check update: Degraded data redundancy: 13/72 objects degraded (18.056%), 8 pgs degraded (PG_DEGRADED) 2023-10-01T10:38:19.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:19 smithi173 bash[58816]: cluster 2023-10-01T10:38:18.409457+0000 mon.a (mon.0) 872 : cluster 3 Health check update: Degraded data redundancy: 13/72 objects degraded (18.056%), 8 pgs degraded (PG_DEGRADED) 2023-10-01T10:38:20.332 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:38:20 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:38:20] "GET /metrics HTTP/1.1" 200 33680 "" "Prometheus/2.33.4" 2023-10-01T10:38:20.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:20 smithi078 bash[75299]: cluster 2023-10-01T10:38:19.648459+0000 mgr.y (mgr.25072) 612 : cluster 0 pgmap v520: 73 pgs: 5 peering, 1 active+undersized+degraded, 3 active+undersized, 64 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2023-10-01T10:38:20.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:20 smithi078 bash[78274]: cluster 2023-10-01T10:38:19.648459+0000 mgr.y (mgr.25072) 612 : cluster 0 pgmap v520: 73 pgs: 5 peering, 1 active+undersized+degraded, 3 active+undersized, 64 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2023-10-01T10:38:20.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:20 smithi173 bash[58816]: cluster 2023-10-01T10:38:19.648459+0000 mgr.y (mgr.25072) 612 : cluster 0 pgmap v520: 73 pgs: 5 peering, 1 active+undersized+degraded, 3 active+undersized, 64 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2023-10-01T10:38:21.322 INFO:teuthology.orchestra.run.smithi078.stdout: 2023-10-01T10:38:21.323 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":127,"fsid":"d8016480-6041-11ee-8db5-212e2dc638e7","created":"2023-10-01T10:04:59.345633+0000","modified":"2023-10-01T10:38:16.004203+0000","last_up_change":"2023-10-01T10:38:15.000941+0000","last_in_change":"2023-10-01T10:12:19.634793+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":23,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":8,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-01T10:09:16.689172+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"70","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{},"mgr_devicehealth":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-01T10:13:43.595790+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-01T10:15:38.225217+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-01T10:15:38.676998+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"3cdf50f0-cae3-4e38-a7ca-c24c4a07b615","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":82,"up_from":86,"up_thru":126,"down_at":83,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6802","nonce":2878175461},{"type":"v1","addr":"172.21.15.78:6803","nonce":2878175461}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6804","nonce":2878175461},{"type":"v1","addr":"172.21.15.78:6805","nonce":2878175461}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6808","nonce":2878175461},{"type":"v1","addr":"172.21.15.78:6809","nonce":2878175461}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6806","nonce":2878175461},{"type":"v1","addr":"172.21.15.78:6807","nonce":2878175461}]},"public_addr":"172.21.15.78:6803/2878175461","cluster_addr":"172.21.15.78:6805/2878175461","heartbeat_back_addr":"172.21.15.78:6809/2878175461","heartbeat_front_addr":"172.21.15.78:6807/2878175461","state":["exists","up"]},{"osd":1,"uuid":"4e6bea65-4109-43d2-b5be-3811fdd2a803","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":96,"up_from":101,"up_thru":126,"down_at":97,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6810","nonce":1115426690},{"type":"v1","addr":"172.21.15.78:6811","nonce":1115426690}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6812","nonce":1115426690},{"type":"v1","addr":"172.21.15.78:6813","nonce":1115426690}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6816","nonce":1115426690},{"type":"v1","addr":"172.21.15.78:6817","nonce":1115426690}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6814","nonce":1115426690},{"type":"v1","addr":"172.21.15.78:6815","nonce":1115426690}]},"public_addr":"172.21.15.78:6811/1115426690","cluster_addr":"172.21.15.78:6813/1115426690","heartbeat_back_addr":"172.21.15.78:6817/1115426690","heartbeat_front_addr":"172.21.15.78:6815/1115426690","state":["exists","up"]},{"osd":2,"uuid":"9f830594-31d6-4994-85fa-0615c280d5c3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":112,"up_from":116,"up_thru":126,"down_at":113,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6818","nonce":4228246677},{"type":"v1","addr":"172.21.15.78:6819","nonce":4228246677}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6820","nonce":4228246677},{"type":"v1","addr":"172.21.15.78:6821","nonce":4228246677}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6824","nonce":4228246677},{"type":"v1","addr":"172.21.15.78:6825","nonce":4228246677}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6822","nonce":4228246677},{"type":"v1","addr":"172.21.15.78:6823","nonce":4228246677}]},"public_addr":"172.21.15.78:6819/4228246677","cluster_addr":"172.21.15.78:6821/4228246677","heartbeat_back_addr":"172.21.15.78:6825/4228246677","heartbeat_front_addr":"172.21.15.78:6823/4228246677","state":["exists","up"]},{"osd":3,"uuid":"2780842b-2f4f-4190-b4a5-279a0444adf2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":117,"up_from":121,"up_thru":126,"down_at":118,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6826","nonce":2365141372},{"type":"v1","addr":"172.21.15.78:6827","nonce":2365141372}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6828","nonce":2365141372},{"type":"v1","addr":"172.21.15.78:6829","nonce":2365141372}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6832","nonce":2365141372},{"type":"v1","addr":"172.21.15.78:6833","nonce":2365141372}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6830","nonce":2365141372},{"type":"v1","addr":"172.21.15.78:6831","nonce":2365141372}]},"public_addr":"172.21.15.78:6827/2365141372","cluster_addr":"172.21.15.78:6829/2365141372","heartbeat_back_addr":"172.21.15.78:6833/2365141372","heartbeat_front_addr":"172.21.15.78:6831/2365141372","state":["exists","up"]},{"osd":4,"uuid":"57c30778-fd7e-4f3e-afd9-389ef377cb62","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":30,"last_clean_end":122,"up_from":126,"up_thru":126,"down_at":123,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6800","nonce":3220463704},{"type":"v1","addr":"172.21.15.173:6801","nonce":3220463704}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6802","nonce":3220463704},{"type":"v1","addr":"172.21.15.173:6803","nonce":3220463704}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6806","nonce":3220463704},{"type":"v1","addr":"172.21.15.173:6807","nonce":3220463704}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6804","nonce":3220463704},{"type":"v1","addr":"172.21.15.173:6805","nonce":3220463704}]},"public_addr":"172.21.15.173:6801/3220463704","cluster_addr":"172.21.15.173:6803/3220463704","heartbeat_back_addr":"172.21.15.173:6807/3220463704","heartbeat_front_addr":"172.21.15.173:6805/3220463704","state":["exists","up"]},{"osd":5,"uuid":"c62aa7c3-2d09-4088-ba36-c84588dc3549","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":126,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6808","nonce":2798643181},{"type":"v1","addr":"172.21.15.173:6809","nonce":2798643181}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6810","nonce":2798643181},{"type":"v1","addr":"172.21.15.173:6811","nonce":2798643181}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6814","nonce":2798643181},{"type":"v1","addr":"172.21.15.173:6815","nonce":2798643181}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6812","nonce":2798643181},{"type":"v1","addr":"172.21.15.173:6813","nonce":2798643181}]},"public_addr":"172.21.15.173:6809/2798643181","cluster_addr":"172.21.15.173:6811/2798643181","heartbeat_back_addr":"172.21.15.173:6815/2798643181","heartbeat_front_addr":"172.21.15.173:6813/2798643181","state":["exists","up"]},{"osd":6,"uuid":"b140330d-dd93-4ce4-9b9a-2ec6321419b3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":126,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6816","nonce":1568190351},{"type":"v1","addr":"172.21.15.173:6817","nonce":1568190351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6818","nonce":1568190351},{"type":"v1","addr":"172.21.15.173:6819","nonce":1568190351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6822","nonce":1568190351},{"type":"v1","addr":"172.21.15.173:6823","nonce":1568190351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6820","nonce":1568190351},{"type":"v1","addr":"172.21.15.173:6821","nonce":1568190351}]},"public_addr":"172.21.15.173:6817/1568190351","cluster_addr":"172.21.15.173:6819/1568190351","heartbeat_back_addr":"172.21.15.173:6823/1568190351","heartbeat_front_addr":"172.21.15.173:6821/1568190351","state":["exists","up"]},{"osd":7,"uuid":"7590027f-50c4-4457-843d-cebaa734d2ea","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":126,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6824","nonce":1958390745},{"type":"v1","addr":"172.21.15.173:6825","nonce":1958390745}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6826","nonce":1958390745},{"type":"v1","addr":"172.21.15.173:6827","nonce":1958390745}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6830","nonce":1958390745},{"type":"v1","addr":"172.21.15.173:6831","nonce":1958390745}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6828","nonce":1958390745},{"type":"v1","addr":"172.21.15.173:6829","nonce":1958390745}]},"public_addr":"172.21.15.173:6825/1958390745","cluster_addr":"172.21.15.173:6827/1958390745","heartbeat_back_addr":"172.21.15.173:6831/1958390745","heartbeat_front_addr":"172.21.15.173:6829/1958390745","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:07:47.950416+0000","dead_epoch":82},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:08:31.368397+0000","dead_epoch":96},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:09:14.866866+0000","dead_epoch":112},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:09:59.445542+0000","dead_epoch":117},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:10:39.550033+0000","dead_epoch":122},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:11:20.934732+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:12:01.654044+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:12:43.447412+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.d","mappings":[{"from":1,"to":5}]},{"pgid":"4.f","mappings":[{"from":1,"to":2}]},{"pgid":"4.14","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.78:0/590730483":"2023-10-02T10:22:47.233952+0000","172.21.15.78:0/3335202251":"2023-10-02T10:22:47.233952+0000","172.21.15.78:0/1585217033":"2023-10-02T10:22:47.233952+0000","172.21.15.78:6801/449971457":"2023-10-02T10:22:47.233952+0000","172.21.15.78:0/3939471097":"2023-10-02T10:05:57.727322+0000","172.21.15.78:6800/1700562157":"2023-10-02T10:05:57.727322+0000","172.21.15.78:6801/1700562157":"2023-10-02T10:05:57.727322+0000","172.21.15.78:0/3635389419":"2023-10-02T10:05:57.727322+0000","172.21.15.78:0/1297753620":"2023-10-02T10:22:47.233952+0000","172.21.15.78:0/158328292":"2023-10-02T10:05:27.007550+0000","172.21.15.78:0/308254153":"2023-10-02T10:05:27.007550+0000","172.21.15.173:0/2382630687":"2023-10-02T10:21:31.381928+0000","172.21.15.78:0/3274439700":"2023-10-02T10:19:23.134044+0000","172.21.15.78:0/2751026363":"2023-10-02T10:22:47.233952+0000","172.21.15.78:0/702157702":"2023-10-02T10:19:23.134044+0000","172.21.15.78:0/3655343641":"2023-10-02T10:13:17.135564+0000","172.21.15.78:6800/3027286102":"2023-10-02T10:05:27.007550+0000","172.21.15.78:6801/516444166":"2023-10-02T10:19:23.134044+0000","172.21.15.78:0/2359078223":"2023-10-02T10:13:17.135564+0000","172.21.15.78:6801/3027286102":"2023-10-02T10:05:27.007550+0000","172.21.15.173:0/4047131805":"2023-10-02T10:21:31.381928+0000","172.21.15.78:0/3611942206":"2023-10-02T10:22:47.233952+0000","172.21.15.173:0/1462478964":"2023-10-02T10:21:31.381928+0000","172.21.15.78:6800/449971457":"2023-10-02T10:22:47.233952+0000","172.21.15.78:6800/1224621968":"2023-10-02T10:13:17.135564+0000","172.21.15.78:6801/1224621968":"2023-10-02T10:13:17.135564+0000","172.21.15.173:6834/1928273846":"2023-10-02T10:21:31.381928+0000","172.21.15.78:6800/516444166":"2023-10-02T10:19:23.134044+0000","172.21.15.173:0/4059345053":"2023-10-02T10:21:31.381928+0000","172.21.15.173:6835/1928273846":"2023-10-02T10:21:31.381928+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-01T10:38:21.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:21 smithi078 bash[78274]: audit 2023-10-01T10:38:21.317530+0000 mon.a (mon.0) 873 : audit 0 from='client.? 172.21.15.78:0/2575787324' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T10:38:21.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:21 smithi078 bash[75299]: audit 2023-10-01T10:38:21.317530+0000 mon.a (mon.0) 873 : audit 0 from='client.? 172.21.15.78:0/2575787324' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T10:38:21.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:21 smithi173 bash[58816]: audit 2023-10-01T10:38:21.317530+0000 mon.a (mon.0) 873 : audit 0 from='client.? 172.21.15.78:0/2575787324' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T10:38:22.186 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-10-01T10:38:22.186 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-01T10:38:22.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:22 smithi078 bash[75299]: cluster 2023-10-01T10:38:21.649416+0000 mgr.y (mgr.25072) 613 : cluster 0 pgmap v521: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:38:22.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:22 smithi078 bash[78274]: cluster 2023-10-01T10:38:21.649416+0000 mgr.y (mgr.25072) 613 : cluster 0 pgmap v521: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:38:22.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:22 smithi173 bash[58816]: cluster 2023-10-01T10:38:21.649416+0000 mgr.y (mgr.25072) 613 : cluster 0 pgmap v521: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:38:23.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:23 smithi078 bash[78274]: cluster 2023-10-01T10:38:22.436059+0000 mon.a (mon.0) 874 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded) 2023-10-01T10:38:23.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:23 smithi078 bash[78274]: cluster 2023-10-01T10:38:22.436119+0000 mon.a (mon.0) 875 : cluster 1 Cluster is now healthy 2023-10-01T10:38:23.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:23 smithi078 bash[75299]: cluster 2023-10-01T10:38:22.436059+0000 mon.a (mon.0) 874 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded) 2023-10-01T10:38:23.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:23 smithi078 bash[75299]: cluster 2023-10-01T10:38:22.436119+0000 mon.a (mon.0) 875 : cluster 1 Cluster is now healthy 2023-10-01T10:38:23.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:23 smithi173 bash[58816]: cluster 2023-10-01T10:38:22.436059+0000 mon.a (mon.0) 874 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded) 2023-10-01T10:38:23.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:23 smithi173 bash[58816]: cluster 2023-10-01T10:38:22.436119+0000 mon.a (mon.0) 875 : cluster 1 Cluster is now healthy 2023-10-01T10:38:24.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:24 smithi078 bash[75299]: cluster 2023-10-01T10:38:23.650196+0000 mgr.y (mgr.25072) 614 : cluster 0 pgmap v522: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:38:24.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:24 smithi078 bash[78274]: cluster 2023-10-01T10:38:23.650196+0000 mgr.y (mgr.25072) 614 : cluster 0 pgmap v522: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:38:24.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:24 smithi173 bash[58816]: cluster 2023-10-01T10:38:23.650196+0000 mgr.y (mgr.25072) 614 : cluster 0 pgmap v522: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:38:26.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:25 smithi078 bash[75299]: cluster 2023-10-01T10:38:25.651147+0000 mgr.y (mgr.25072) 615 : cluster 0 pgmap v523: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:38:26.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:25 smithi078 bash[78274]: cluster 2023-10-01T10:38:25.651147+0000 mgr.y (mgr.25072) 615 : cluster 0 pgmap v523: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:38:26.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:25 smithi173 bash[58816]: cluster 2023-10-01T10:38:25.651147+0000 mgr.y (mgr.25072) 615 : cluster 0 pgmap v523: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:38:26.967 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:38:29.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:28 smithi078 bash[75299]: cluster 2023-10-01T10:38:27.651922+0000 mgr.y (mgr.25072) 616 : cluster 0 pgmap v524: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:38:29.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:28 smithi078 bash[75299]: audit 2023-10-01T10:38:27.672972+0000 mon.a (mon.0) 876 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:38:29.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:28 smithi078 bash[78274]: cluster 2023-10-01T10:38:27.651922+0000 mgr.y (mgr.25072) 616 : cluster 0 pgmap v524: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:38:29.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:28 smithi078 bash[78274]: audit 2023-10-01T10:38:27.672972+0000 mon.a (mon.0) 876 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:38:29.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:28 smithi173 bash[58816]: cluster 2023-10-01T10:38:27.651922+0000 mgr.y (mgr.25072) 616 : cluster 0 pgmap v524: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:38:29.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:28 smithi173 bash[58816]: audit 2023-10-01T10:38:27.672972+0000 mon.a (mon.0) 876 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:38:30.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:29 smithi078 bash[75299]: cluster 2023-10-01T10:38:29.652757+0000 mgr.y (mgr.25072) 617 : cluster 0 pgmap v525: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:38:30.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:29 smithi078 bash[78274]: cluster 2023-10-01T10:38:29.652757+0000 mgr.y (mgr.25072) 617 : cluster 0 pgmap v525: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:38:30.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:29 smithi173 bash[58816]: cluster 2023-10-01T10:38:29.652757+0000 mgr.y (mgr.25072) 617 : cluster 0 pgmap v525: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:38:30.307 INFO:teuthology.orchestra.run.smithi078.stdout: 2023-10-01T10:38:30.307 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":127,"fsid":"d8016480-6041-11ee-8db5-212e2dc638e7","created":"2023-10-01T10:04:59.345633+0000","modified":"2023-10-01T10:38:16.004203+0000","last_up_change":"2023-10-01T10:38:15.000941+0000","last_in_change":"2023-10-01T10:12:19.634793+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":23,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":8,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-01T10:09:16.689172+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"70","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{},"mgr_devicehealth":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-01T10:13:43.595790+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"55","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-01T10:15:38.225217+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-01T10:15:38.676998+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"3cdf50f0-cae3-4e38-a7ca-c24c4a07b615","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":82,"up_from":86,"up_thru":126,"down_at":83,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6802","nonce":2878175461},{"type":"v1","addr":"172.21.15.78:6803","nonce":2878175461}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6804","nonce":2878175461},{"type":"v1","addr":"172.21.15.78:6805","nonce":2878175461}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6808","nonce":2878175461},{"type":"v1","addr":"172.21.15.78:6809","nonce":2878175461}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6806","nonce":2878175461},{"type":"v1","addr":"172.21.15.78:6807","nonce":2878175461}]},"public_addr":"172.21.15.78:6803/2878175461","cluster_addr":"172.21.15.78:6805/2878175461","heartbeat_back_addr":"172.21.15.78:6809/2878175461","heartbeat_front_addr":"172.21.15.78:6807/2878175461","state":["exists","up"]},{"osd":1,"uuid":"4e6bea65-4109-43d2-b5be-3811fdd2a803","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":96,"up_from":101,"up_thru":126,"down_at":97,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6810","nonce":1115426690},{"type":"v1","addr":"172.21.15.78:6811","nonce":1115426690}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6812","nonce":1115426690},{"type":"v1","addr":"172.21.15.78:6813","nonce":1115426690}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6816","nonce":1115426690},{"type":"v1","addr":"172.21.15.78:6817","nonce":1115426690}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6814","nonce":1115426690},{"type":"v1","addr":"172.21.15.78:6815","nonce":1115426690}]},"public_addr":"172.21.15.78:6811/1115426690","cluster_addr":"172.21.15.78:6813/1115426690","heartbeat_back_addr":"172.21.15.78:6817/1115426690","heartbeat_front_addr":"172.21.15.78:6815/1115426690","state":["exists","up"]},{"osd":2,"uuid":"9f830594-31d6-4994-85fa-0615c280d5c3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":112,"up_from":116,"up_thru":126,"down_at":113,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6818","nonce":4228246677},{"type":"v1","addr":"172.21.15.78:6819","nonce":4228246677}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6820","nonce":4228246677},{"type":"v1","addr":"172.21.15.78:6821","nonce":4228246677}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6824","nonce":4228246677},{"type":"v1","addr":"172.21.15.78:6825","nonce":4228246677}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6822","nonce":4228246677},{"type":"v1","addr":"172.21.15.78:6823","nonce":4228246677}]},"public_addr":"172.21.15.78:6819/4228246677","cluster_addr":"172.21.15.78:6821/4228246677","heartbeat_back_addr":"172.21.15.78:6825/4228246677","heartbeat_front_addr":"172.21.15.78:6823/4228246677","state":["exists","up"]},{"osd":3,"uuid":"2780842b-2f4f-4190-b4a5-279a0444adf2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":117,"up_from":121,"up_thru":126,"down_at":118,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6826","nonce":2365141372},{"type":"v1","addr":"172.21.15.78:6827","nonce":2365141372}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6828","nonce":2365141372},{"type":"v1","addr":"172.21.15.78:6829","nonce":2365141372}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6832","nonce":2365141372},{"type":"v1","addr":"172.21.15.78:6833","nonce":2365141372}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6830","nonce":2365141372},{"type":"v1","addr":"172.21.15.78:6831","nonce":2365141372}]},"public_addr":"172.21.15.78:6827/2365141372","cluster_addr":"172.21.15.78:6829/2365141372","heartbeat_back_addr":"172.21.15.78:6833/2365141372","heartbeat_front_addr":"172.21.15.78:6831/2365141372","state":["exists","up"]},{"osd":4,"uuid":"57c30778-fd7e-4f3e-afd9-389ef377cb62","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":30,"last_clean_end":122,"up_from":126,"up_thru":126,"down_at":123,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6800","nonce":3220463704},{"type":"v1","addr":"172.21.15.173:6801","nonce":3220463704}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6802","nonce":3220463704},{"type":"v1","addr":"172.21.15.173:6803","nonce":3220463704}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6806","nonce":3220463704},{"type":"v1","addr":"172.21.15.173:6807","nonce":3220463704}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6804","nonce":3220463704},{"type":"v1","addr":"172.21.15.173:6805","nonce":3220463704}]},"public_addr":"172.21.15.173:6801/3220463704","cluster_addr":"172.21.15.173:6803/3220463704","heartbeat_back_addr":"172.21.15.173:6807/3220463704","heartbeat_front_addr":"172.21.15.173:6805/3220463704","state":["exists","up"]},{"osd":5,"uuid":"c62aa7c3-2d09-4088-ba36-c84588dc3549","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":126,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6808","nonce":2798643181},{"type":"v1","addr":"172.21.15.173:6809","nonce":2798643181}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6810","nonce":2798643181},{"type":"v1","addr":"172.21.15.173:6811","nonce":2798643181}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6814","nonce":2798643181},{"type":"v1","addr":"172.21.15.173:6815","nonce":2798643181}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6812","nonce":2798643181},{"type":"v1","addr":"172.21.15.173:6813","nonce":2798643181}]},"public_addr":"172.21.15.173:6809/2798643181","cluster_addr":"172.21.15.173:6811/2798643181","heartbeat_back_addr":"172.21.15.173:6815/2798643181","heartbeat_front_addr":"172.21.15.173:6813/2798643181","state":["exists","up"]},{"osd":6,"uuid":"b140330d-dd93-4ce4-9b9a-2ec6321419b3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":126,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6816","nonce":1568190351},{"type":"v1","addr":"172.21.15.173:6817","nonce":1568190351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6818","nonce":1568190351},{"type":"v1","addr":"172.21.15.173:6819","nonce":1568190351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6822","nonce":1568190351},{"type":"v1","addr":"172.21.15.173:6823","nonce":1568190351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6820","nonce":1568190351},{"type":"v1","addr":"172.21.15.173:6821","nonce":1568190351}]},"public_addr":"172.21.15.173:6817/1568190351","cluster_addr":"172.21.15.173:6819/1568190351","heartbeat_back_addr":"172.21.15.173:6823/1568190351","heartbeat_front_addr":"172.21.15.173:6821/1568190351","state":["exists","up"]},{"osd":7,"uuid":"7590027f-50c4-4457-843d-cebaa734d2ea","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":126,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6824","nonce":1958390745},{"type":"v1","addr":"172.21.15.173:6825","nonce":1958390745}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6826","nonce":1958390745},{"type":"v1","addr":"172.21.15.173:6827","nonce":1958390745}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6830","nonce":1958390745},{"type":"v1","addr":"172.21.15.173:6831","nonce":1958390745}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6828","nonce":1958390745},{"type":"v1","addr":"172.21.15.173:6829","nonce":1958390745}]},"public_addr":"172.21.15.173:6825/1958390745","cluster_addr":"172.21.15.173:6827/1958390745","heartbeat_back_addr":"172.21.15.173:6831/1958390745","heartbeat_front_addr":"172.21.15.173:6829/1958390745","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:07:47.950416+0000","dead_epoch":82},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:08:31.368397+0000","dead_epoch":96},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:09:14.866866+0000","dead_epoch":112},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:09:59.445542+0000","dead_epoch":117},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:10:39.550033+0000","dead_epoch":122},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:11:20.934732+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:12:01.654044+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:12:43.447412+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.d","mappings":[{"from":1,"to":5}]},{"pgid":"4.f","mappings":[{"from":1,"to":2}]},{"pgid":"4.14","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.78:0/590730483":"2023-10-02T10:22:47.233952+0000","172.21.15.78:0/3335202251":"2023-10-02T10:22:47.233952+0000","172.21.15.78:0/1585217033":"2023-10-02T10:22:47.233952+0000","172.21.15.78:6801/449971457":"2023-10-02T10:22:47.233952+0000","172.21.15.78:0/3939471097":"2023-10-02T10:05:57.727322+0000","172.21.15.78:6800/1700562157":"2023-10-02T10:05:57.727322+0000","172.21.15.78:6801/1700562157":"2023-10-02T10:05:57.727322+0000","172.21.15.78:0/3635389419":"2023-10-02T10:05:57.727322+0000","172.21.15.78:0/1297753620":"2023-10-02T10:22:47.233952+0000","172.21.15.78:0/158328292":"2023-10-02T10:05:27.007550+0000","172.21.15.78:0/308254153":"2023-10-02T10:05:27.007550+0000","172.21.15.173:0/2382630687":"2023-10-02T10:21:31.381928+0000","172.21.15.78:0/3274439700":"2023-10-02T10:19:23.134044+0000","172.21.15.78:0/2751026363":"2023-10-02T10:22:47.233952+0000","172.21.15.78:0/702157702":"2023-10-02T10:19:23.134044+0000","172.21.15.78:0/3655343641":"2023-10-02T10:13:17.135564+0000","172.21.15.78:6800/3027286102":"2023-10-02T10:05:27.007550+0000","172.21.15.78:6801/516444166":"2023-10-02T10:19:23.134044+0000","172.21.15.78:0/2359078223":"2023-10-02T10:13:17.135564+0000","172.21.15.78:6801/3027286102":"2023-10-02T10:05:27.007550+0000","172.21.15.173:0/4047131805":"2023-10-02T10:21:31.381928+0000","172.21.15.78:0/3611942206":"2023-10-02T10:22:47.233952+0000","172.21.15.173:0/1462478964":"2023-10-02T10:21:31.381928+0000","172.21.15.78:6800/449971457":"2023-10-02T10:22:47.233952+0000","172.21.15.78:6800/1224621968":"2023-10-02T10:13:17.135564+0000","172.21.15.78:6801/1224621968":"2023-10-02T10:13:17.135564+0000","172.21.15.173:6834/1928273846":"2023-10-02T10:21:31.381928+0000","172.21.15.78:6800/516444166":"2023-10-02T10:19:23.134044+0000","172.21.15.173:0/4059345053":"2023-10-02T10:21:31.381928+0000","172.21.15.173:6835/1928273846":"2023-10-02T10:21:31.381928+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-01T10:38:30.582 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:38:30 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:38:30] "GET /metrics HTTP/1.1" 200 33680 "" "Prometheus/2.33.4" 2023-10-01T10:38:30.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:30 smithi078 bash[75299]: audit 2023-10-01T10:38:30.302527+0000 mon.a (mon.0) 877 : audit 0 from='client.? 172.21.15.78:0/745429098' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T10:38:30.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:30 smithi078 bash[78274]: audit 2023-10-01T10:38:30.302527+0000 mon.a (mon.0) 877 : audit 0 from='client.? 172.21.15.78:0/745429098' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T10:38:31.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:30 smithi173 bash[58816]: audit 2023-10-01T10:38:30.302527+0000 mon.a (mon.0) 877 : audit 0 from='client.? 172.21.15.78:0/745429098' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T10:38:31.178 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-10-01T10:38:31.179 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-10-01T10:38:31.179 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-10-01T10:38:31.180 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-10-01T10:38:31.180 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-10-01T10:38:31.180 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-10-01T10:38:31.181 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-10-01T10:38:31.181 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-10-01T10:38:32.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:31 smithi078 bash[75299]: cluster 2023-10-01T10:38:31.653670+0000 mgr.y (mgr.25072) 618 : cluster 0 pgmap v526: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:38:32.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:31 smithi078 bash[78274]: cluster 2023-10-01T10:38:31.653670+0000 mgr.y (mgr.25072) 618 : cluster 0 pgmap v526: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:38:32.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:31 smithi173 bash[58816]: cluster 2023-10-01T10:38:31.653670+0000 mgr.y (mgr.25072) 618 : cluster 0 pgmap v526: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:38:33.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:33 smithi078 bash[78274]: audit 2023-10-01T10:38:32.414159+0000 mon.a (mon.0) 878 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:38:33.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:33 smithi078 bash[78274]: audit 2023-10-01T10:38:32.415756+0000 mon.a (mon.0) 879 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:38:33.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:33 smithi078 bash[75299]: audit 2023-10-01T10:38:32.414159+0000 mon.a (mon.0) 878 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:38:33.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:33 smithi078 bash[75299]: audit 2023-10-01T10:38:32.415756+0000 mon.a (mon.0) 879 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:38:33.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:33 smithi173 bash[58816]: audit 2023-10-01T10:38:32.414159+0000 mon.a (mon.0) 878 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:38:33.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:33 smithi173 bash[58816]: audit 2023-10-01T10:38:32.415756+0000 mon.a (mon.0) 879 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:38:34.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:34 smithi078 bash[75299]: cluster 2023-10-01T10:38:33.654419+0000 mgr.y (mgr.25072) 619 : cluster 0 pgmap v527: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:38:34.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:34 smithi078 bash[78274]: cluster 2023-10-01T10:38:33.654419+0000 mgr.y (mgr.25072) 619 : cluster 0 pgmap v527: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:38:34.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:34 smithi173 bash[58816]: cluster 2023-10-01T10:38:33.654419+0000 mgr.y (mgr.25072) 619 : cluster 0 pgmap v527: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:38:36.064 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:38:36.064 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:38:36.065 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:38:36.065 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:38:36.066 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:38:36.066 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:38:36.067 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:38:36.067 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:38:36.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:35 smithi078 bash[75299]: cluster 2023-10-01T10:38:35.655370+0000 mgr.y (mgr.25072) 620 : cluster 0 pgmap v528: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:38:36.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:35 smithi078 bash[78274]: cluster 2023-10-01T10:38:35.655370+0000 mgr.y (mgr.25072) 620 : cluster 0 pgmap v528: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:38:36.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:35 smithi173 bash[58816]: cluster 2023-10-01T10:38:35.655370+0000 mgr.y (mgr.25072) 620 : cluster 0 pgmap v528: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:38:38.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:38 smithi078 bash[78274]: cluster 2023-10-01T10:38:37.656127+0000 mgr.y (mgr.25072) 621 : cluster 0 pgmap v529: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:38:38.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:38 smithi078 bash[75299]: cluster 2023-10-01T10:38:37.656127+0000 mgr.y (mgr.25072) 621 : cluster 0 pgmap v529: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:38:38.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:38 smithi173 bash[58816]: cluster 2023-10-01T10:38:37.656127+0000 mgr.y (mgr.25072) 621 : cluster 0 pgmap v529: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:38:40.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:39 smithi078 bash[75299]: cluster 2023-10-01T10:38:39.657061+0000 mgr.y (mgr.25072) 622 : cluster 0 pgmap v530: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:38:40.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:39 smithi078 bash[78274]: cluster 2023-10-01T10:38:39.657061+0000 mgr.y (mgr.25072) 622 : cluster 0 pgmap v530: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:38:40.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:39 smithi173 bash[58816]: cluster 2023-10-01T10:38:39.657061+0000 mgr.y (mgr.25072) 622 : cluster 0 pgmap v530: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:38:40.540 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:38:40 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:38:40] "GET /metrics HTTP/1.1" 200 33825 "" "Prometheus/2.33.4" 2023-10-01T10:38:42.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:41 smithi078 bash[75299]: cluster 2023-10-01T10:38:41.657906+0000 mgr.y (mgr.25072) 623 : cluster 0 pgmap v531: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:38:42.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:41 smithi078 bash[78274]: cluster 2023-10-01T10:38:41.657906+0000 mgr.y (mgr.25072) 623 : cluster 0 pgmap v531: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:38:42.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:41 smithi173 bash[58816]: cluster 2023-10-01T10:38:41.657906+0000 mgr.y (mgr.25072) 623 : cluster 0 pgmap v531: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:38:44.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:43 smithi078 bash[75299]: cluster 2023-10-01T10:38:43.658705+0000 mgr.y (mgr.25072) 624 : cluster 0 pgmap v532: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:38:44.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:43 smithi078 bash[78274]: cluster 2023-10-01T10:38:43.658705+0000 mgr.y (mgr.25072) 624 : cluster 0 pgmap v532: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:38:44.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:43 smithi173 bash[58816]: cluster 2023-10-01T10:38:43.658705+0000 mgr.y (mgr.25072) 624 : cluster 0 pgmap v532: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:38:46.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:45 smithi078 bash[78274]: cluster 2023-10-01T10:38:45.659451+0000 mgr.y (mgr.25072) 625 : cluster 0 pgmap v533: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:38:46.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:45 smithi078 bash[75299]: cluster 2023-10-01T10:38:45.659451+0000 mgr.y (mgr.25072) 625 : cluster 0 pgmap v533: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:38:46.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:45 smithi173 bash[58816]: cluster 2023-10-01T10:38:45.659451+0000 mgr.y (mgr.25072) 625 : cluster 0 pgmap v533: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:38:48.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:48 smithi078 bash[78274]: audit 2023-10-01T10:38:47.409046+0000 mon.a (mon.0) 880 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:38:48.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:48 smithi078 bash[78274]: cluster 2023-10-01T10:38:47.660130+0000 mgr.y (mgr.25072) 626 : cluster 0 pgmap v534: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:38:48.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:48 smithi078 bash[75299]: audit 2023-10-01T10:38:47.409046+0000 mon.a (mon.0) 880 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:38:48.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:48 smithi078 bash[75299]: cluster 2023-10-01T10:38:47.660130+0000 mgr.y (mgr.25072) 626 : cluster 0 pgmap v534: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:38:48.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:48 smithi173 bash[58816]: audit 2023-10-01T10:38:47.409046+0000 mon.a (mon.0) 880 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:38:48.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:48 smithi173 bash[58816]: cluster 2023-10-01T10:38:47.660130+0000 mgr.y (mgr.25072) 626 : cluster 0 pgmap v534: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:38:49.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:49 smithi078 bash[78274]: cluster 2023-10-01T10:38:49.660799+0000 mgr.y (mgr.25072) 627 : cluster 0 pgmap v535: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:38:49.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:49 smithi078 bash[75299]: cluster 2023-10-01T10:38:49.660799+0000 mgr.y (mgr.25072) 627 : cluster 0 pgmap v535: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:38:49.890 INFO:teuthology.orchestra.run.smithi078.stdout:519691042831 2023-10-01T10:38:49.890 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-10-01T10:38:50.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:49 smithi173 bash[58816]: cluster 2023-10-01T10:38:49.660799+0000 mgr.y (mgr.25072) 627 : cluster 0 pgmap v535: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:38:50.582 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:38:50 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:38:50] "GET /metrics HTTP/1.1" 200 33821 "" "Prometheus/2.33.4" 2023-10-01T10:38:52.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:51 smithi078 bash[75299]: cluster 2023-10-01T10:38:51.661592+0000 mgr.y (mgr.25072) 628 : cluster 0 pgmap v536: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:38:52.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:51 smithi078 bash[78274]: cluster 2023-10-01T10:38:51.661592+0000 mgr.y (mgr.25072) 628 : cluster 0 pgmap v536: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:38:52.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:51 smithi173 bash[58816]: cluster 2023-10-01T10:38:51.661592+0000 mgr.y (mgr.25072) 628 : cluster 0 pgmap v536: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:38:52.320 INFO:teuthology.orchestra.run.smithi078.stdout:498216206358 2023-10-01T10:38:52.320 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-10-01T10:38:52.322 INFO:teuthology.orchestra.run.smithi078.stdout:541165879304 2023-10-01T10:38:52.322 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-10-01T10:38:52.679 INFO:teuthology.orchestra.run.smithi078.stdout:154618822985 2023-10-01T10:38:52.679 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-10-01T10:38:54.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:53 smithi078 bash[78274]: cluster 2023-10-01T10:38:53.662495+0000 mgr.y (mgr.25072) 629 : cluster 0 pgmap v537: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:38:54.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:53 smithi078 bash[75299]: cluster 2023-10-01T10:38:53.662495+0000 mgr.y (mgr.25072) 629 : cluster 0 pgmap v537: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:38:54.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:53 smithi173 bash[58816]: cluster 2023-10-01T10:38:53.662495+0000 mgr.y (mgr.25072) 629 : cluster 0 pgmap v537: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:38:54.155 INFO:teuthology.orchestra.run.smithi078.stdout:180388626756 2023-10-01T10:38:54.155 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-10-01T10:38:54.249 INFO:teuthology.orchestra.run.smithi078.stdout:206158430522 2023-10-01T10:38:54.250 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-10-01T10:38:54.992 INFO:teuthology.orchestra.run.smithi078.stdout:433791696974 2023-10-01T10:38:54.992 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-10-01T10:38:55.006 INFO:teuthology.orchestra.run.smithi078.stdout:369367187584 2023-10-01T10:38:55.006 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-10-01T10:38:56.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:55 smithi078 bash[75299]: cluster 2023-10-01T10:38:55.663457+0000 mgr.y (mgr.25072) 630 : cluster 0 pgmap v538: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:38:56.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:55 smithi078 bash[78274]: cluster 2023-10-01T10:38:55.663457+0000 mgr.y (mgr.25072) 630 : cluster 0 pgmap v538: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:38:56.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:55 smithi173 bash[58816]: cluster 2023-10-01T10:38:55.663457+0000 mgr.y (mgr.25072) 630 : cluster 0 pgmap v538: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:38:57.692 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:38:57.693 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:38:57.694 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:38:57.697 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:38:58.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:58 smithi078 bash[75299]: cluster 2023-10-01T10:38:57.664219+0000 mgr.y (mgr.25072) 631 : cluster 0 pgmap v539: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:38:58.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:58 smithi078 bash[78274]: cluster 2023-10-01T10:38:57.664219+0000 mgr.y (mgr.25072) 631 : cluster 0 pgmap v539: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:38:58.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:58 smithi173 bash[58816]: cluster 2023-10-01T10:38:57.664219+0000 mgr.y (mgr.25072) 631 : cluster 0 pgmap v539: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:38:58.705 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:38:58.706 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:38:59.719 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:38:59.720 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:39:00.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:38:59 smithi078 bash[75299]: cluster 2023-10-01T10:38:59.665000+0000 mgr.y (mgr.25072) 632 : cluster 0 pgmap v540: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:00.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:38:59 smithi078 bash[78274]: cluster 2023-10-01T10:38:59.665000+0000 mgr.y (mgr.25072) 632 : cluster 0 pgmap v540: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:00.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:38:59 smithi173 bash[58816]: cluster 2023-10-01T10:38:59.665000+0000 mgr.y (mgr.25072) 632 : cluster 0 pgmap v540: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:00.604 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:39:00 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:39:00] "GET /metrics HTTP/1.1" 200 33821 "" "Prometheus/2.33.4" 2023-10-01T10:39:01.857 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:39:01 smithi078 bash[78274]: cluster 2023-10-01T10:39:01.665811+0000 mgr.y (mgr.25072) 633 : cluster 0 pgmap v541: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:01.858 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:39:01 smithi078 bash[75299]: cluster 2023-10-01T10:39:01.665811+0000 mgr.y (mgr.25072) 633 : cluster 0 pgmap v541: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:02.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:39:01 smithi173 bash[58816]: cluster 2023-10-01T10:39:01.665811+0000 mgr.y (mgr.25072) 633 : cluster 0 pgmap v541: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:03.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:39:03 smithi078 bash[78274]: audit 2023-10-01T10:39:02.409092+0000 mon.a (mon.0) 881 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:39:03.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:39:03 smithi078 bash[75299]: audit 2023-10-01T10:39:02.409092+0000 mon.a (mon.0) 881 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:39:03.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:39:03 smithi173 bash[58816]: audit 2023-10-01T10:39:02.409092+0000 mon.a (mon.0) 881 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:39:04.369 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:39:04 smithi078 bash[78274]: cluster 2023-10-01T10:39:03.666399+0000 mgr.y (mgr.25072) 634 : cluster 0 pgmap v542: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:04.370 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:39:04 smithi078 bash[75299]: cluster 2023-10-01T10:39:03.666399+0000 mgr.y (mgr.25072) 634 : cluster 0 pgmap v542: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:04.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:39:04 smithi173 bash[58816]: cluster 2023-10-01T10:39:03.666399+0000 mgr.y (mgr.25072) 634 : cluster 0 pgmap v542: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:06.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:39:05 smithi078 bash[75299]: cluster 2023-10-01T10:39:05.667101+0000 mgr.y (mgr.25072) 635 : cluster 0 pgmap v543: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:06.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:39:05 smithi078 bash[78274]: cluster 2023-10-01T10:39:05.667101+0000 mgr.y (mgr.25072) 635 : cluster 0 pgmap v543: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:06.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:39:05 smithi173 bash[58816]: cluster 2023-10-01T10:39:05.667101+0000 mgr.y (mgr.25072) 635 : cluster 0 pgmap v543: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:08.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:39:08 smithi173 bash[58816]: cluster 2023-10-01T10:39:07.667570+0000 mgr.y (mgr.25072) 636 : cluster 0 pgmap v544: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:08.394 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:39:08 smithi078 bash[78274]: cluster 2023-10-01T10:39:07.667570+0000 mgr.y (mgr.25072) 636 : cluster 0 pgmap v544: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:08.394 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:39:08 smithi078 bash[75299]: cluster 2023-10-01T10:39:07.667570+0000 mgr.y (mgr.25072) 636 : cluster 0 pgmap v544: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:09.415 INFO:teuthology.orchestra.run.smithi078.stdout:541165879308 2023-10-01T10:39:09.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:39:09 smithi173 bash[58816]: audit 2023-10-01T10:39:09.414280+0000 mon.a (mon.0) 882 : audit 0 from='client.? 172.21.15.78:0/508620390' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-01T10:39:09.920 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:39:09 smithi078 bash[75299]: audit 2023-10-01T10:39:09.414280+0000 mon.a (mon.0) 882 : audit 0 from='client.? 172.21.15.78:0/508620390' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-01T10:39:09.920 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:39:09 smithi078 bash[78274]: audit 2023-10-01T10:39:09.414280+0000 mon.a (mon.0) 882 : audit 0 from='client.? 172.21.15.78:0/508620390' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-01T10:39:10.608 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:39:10 smithi078 bash[75299]: cluster 2023-10-01T10:39:09.668061+0000 mgr.y (mgr.25072) 637 : cluster 0 pgmap v545: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:10.608 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:39:10 smithi078 bash[78274]: cluster 2023-10-01T10:39:09.668061+0000 mgr.y (mgr.25072) 637 : cluster 0 pgmap v545: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:10.609 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:39:10 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:39:10] "GET /metrics HTTP/1.1" 200 33821 "" "Prometheus/2.33.4" 2023-10-01T10:39:10.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:39:10 smithi173 bash[58816]: cluster 2023-10-01T10:39:09.668061+0000 mgr.y (mgr.25072) 637 : cluster 0 pgmap v545: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:12.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:39:11 smithi078 bash[75299]: cluster 2023-10-01T10:39:11.669026+0000 mgr.y (mgr.25072) 638 : cluster 0 pgmap v546: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:12.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:39:11 smithi078 bash[78274]: cluster 2023-10-01T10:39:11.669026+0000 mgr.y (mgr.25072) 638 : cluster 0 pgmap v546: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:12.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:39:11 smithi173 bash[58816]: cluster 2023-10-01T10:39:11.669026+0000 mgr.y (mgr.25072) 638 : cluster 0 pgmap v546: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:13.806 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:39:13 smithi078 bash[75299]: cluster 2023-10-01T10:39:13.669844+0000 mgr.y (mgr.25072) 639 : cluster 0 pgmap v547: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:13.807 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:39:13 smithi078 bash[78274]: cluster 2023-10-01T10:39:13.669844+0000 mgr.y (mgr.25072) 639 : cluster 0 pgmap v547: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:13.808 INFO:tasks.cephadm.ceph_manager.ceph:need seq 541165879304 got 541165879308 for osd.4 2023-10-01T10:39:13.808 DEBUG:teuthology.parallel:result is None 2023-10-01T10:39:13.964 INFO:teuthology.orchestra.run.smithi078.stdout:519691042836 2023-10-01T10:39:14.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:39:13 smithi173 bash[58816]: cluster 2023-10-01T10:39:13.669844+0000 mgr.y (mgr.25072) 639 : cluster 0 pgmap v547: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:14.289 INFO:teuthology.orchestra.run.smithi078.stdout:154618822990 2023-10-01T10:39:14.641 INFO:teuthology.orchestra.run.smithi078.stdout:498216206364 2023-10-01T10:39:15.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:39:14 smithi078 bash[78274]: audit 2023-10-01T10:39:13.962572+0000 mon.a (mon.0) 883 : audit 0 from='client.? 172.21.15.78:0/606841142' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-01T10:39:15.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:39:14 smithi078 bash[78274]: audit 2023-10-01T10:39:14.287223+0000 mon.c (mon.2) 128 : audit 0 from='client.? 172.21.15.78:0/4080673702' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-01T10:39:15.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:39:14 smithi078 bash[78274]: audit 2023-10-01T10:39:14.639640+0000 mon.c (mon.2) 129 : audit 0 from='client.? 172.21.15.78:0/1029029538' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-01T10:39:15.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:39:14 smithi078 bash[75299]: audit 2023-10-01T10:39:13.962572+0000 mon.a (mon.0) 883 : audit 0 from='client.? 172.21.15.78:0/606841142' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-01T10:39:15.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:39:14 smithi078 bash[75299]: audit 2023-10-01T10:39:14.287223+0000 mon.c (mon.2) 128 : audit 0 from='client.? 172.21.15.78:0/4080673702' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-01T10:39:15.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:39:14 smithi078 bash[75299]: audit 2023-10-01T10:39:14.639640+0000 mon.c (mon.2) 129 : audit 0 from='client.? 172.21.15.78:0/1029029538' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-01T10:39:15.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:39:14 smithi173 bash[58816]: audit 2023-10-01T10:39:13.962572+0000 mon.a (mon.0) 883 : audit 0 from='client.? 172.21.15.78:0/606841142' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-01T10:39:15.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:39:14 smithi173 bash[58816]: audit 2023-10-01T10:39:14.287223+0000 mon.c (mon.2) 128 : audit 0 from='client.? 172.21.15.78:0/4080673702' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-01T10:39:15.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:39:14 smithi173 bash[58816]: audit 2023-10-01T10:39:14.639640+0000 mon.c (mon.2) 129 : audit 0 from='client.? 172.21.15.78:0/1029029538' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-01T10:39:15.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:39:15 smithi078 bash[75299]: cluster 2023-10-01T10:39:15.670462+0000 mgr.y (mgr.25072) 640 : cluster 0 pgmap v548: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:15.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:39:15 smithi078 bash[78274]: cluster 2023-10-01T10:39:15.670462+0000 mgr.y (mgr.25072) 640 : cluster 0 pgmap v548: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:15.985 INFO:teuthology.orchestra.run.smithi078.stdout:369367187589 2023-10-01T10:39:16.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:39:15 smithi173 bash[58816]: cluster 2023-10-01T10:39:15.670462+0000 mgr.y (mgr.25072) 640 : cluster 0 pgmap v548: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:16.148 INFO:teuthology.orchestra.run.smithi078.stdout:433791696979 2023-10-01T10:39:16.436 INFO:teuthology.orchestra.run.smithi078.stdout:180388626761 2023-10-01T10:39:17.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:39:16 smithi078 bash[75299]: audit 2023-10-01T10:39:15.984281+0000 mon.a (mon.0) 884 : audit 0 from='client.? 172.21.15.78:0/3674977585' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-01T10:39:17.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:39:16 smithi078 bash[75299]: audit 2023-10-01T10:39:16.147097+0000 mon.a (mon.0) 885 : audit 0 from='client.? 172.21.15.78:0/3165884701' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-01T10:39:17.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:39:16 smithi078 bash[75299]: audit 2023-10-01T10:39:16.434524+0000 mon.c (mon.2) 130 : audit 0 from='client.? 172.21.15.78:0/4181327094' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-01T10:39:17.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:39:16 smithi078 bash[75299]: audit 2023-10-01T10:39:16.460577+0000 mon.a (mon.0) 886 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:39:17.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:39:16 smithi078 bash[78274]: audit 2023-10-01T10:39:15.984281+0000 mon.a (mon.0) 884 : audit 0 from='client.? 172.21.15.78:0/3674977585' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-01T10:39:17.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:39:16 smithi078 bash[78274]: audit 2023-10-01T10:39:16.147097+0000 mon.a (mon.0) 885 : audit 0 from='client.? 172.21.15.78:0/3165884701' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-01T10:39:17.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:39:16 smithi078 bash[78274]: audit 2023-10-01T10:39:16.434524+0000 mon.c (mon.2) 130 : audit 0 from='client.? 172.21.15.78:0/4181327094' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-01T10:39:17.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:39:16 smithi078 bash[78274]: audit 2023-10-01T10:39:16.460577+0000 mon.a (mon.0) 886 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:39:17.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:39:16 smithi173 bash[58816]: audit 2023-10-01T10:39:15.984281+0000 mon.a (mon.0) 884 : audit 0 from='client.? 172.21.15.78:0/3674977585' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-01T10:39:17.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:39:16 smithi173 bash[58816]: audit 2023-10-01T10:39:16.147097+0000 mon.a (mon.0) 885 : audit 0 from='client.? 172.21.15.78:0/3165884701' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-01T10:39:17.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:39:16 smithi173 bash[58816]: audit 2023-10-01T10:39:16.434524+0000 mon.c (mon.2) 130 : audit 0 from='client.? 172.21.15.78:0/4181327094' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-01T10:39:17.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:39:16 smithi173 bash[58816]: audit 2023-10-01T10:39:16.460577+0000 mon.a (mon.0) 886 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:39:17.273 INFO:tasks.cephadm.ceph_manager.ceph:need seq 519691042831 got 519691042836 for osd.3 2023-10-01T10:39:17.273 DEBUG:teuthology.parallel:result is None 2023-10-01T10:39:17.286 INFO:tasks.cephadm.ceph_manager.ceph:need seq 154618822985 got 154618822990 for osd.5 2023-10-01T10:39:17.286 DEBUG:teuthology.parallel:result is None 2023-10-01T10:39:17.507 INFO:teuthology.orchestra.run.smithi078.stdout:206158430527 2023-10-01T10:39:17.606 INFO:tasks.cephadm.ceph_manager.ceph:need seq 498216206358 got 498216206364 for osd.2 2023-10-01T10:39:17.606 DEBUG:teuthology.parallel:result is None 2023-10-01T10:39:18.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:39:18 smithi078 bash[78274]: audit 2023-10-01T10:39:17.408465+0000 mon.a (mon.0) 887 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:39:18.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:39:18 smithi078 bash[78274]: audit 2023-10-01T10:39:17.506262+0000 mon.a (mon.0) 888 : audit 0 from='client.? 172.21.15.78:0/4057857734' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-01T10:39:18.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:39:18 smithi078 bash[78274]: cluster 2023-10-01T10:39:17.670845+0000 mgr.y (mgr.25072) 641 : cluster 0 pgmap v549: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:18.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:39:18 smithi078 bash[75299]: audit 2023-10-01T10:39:17.408465+0000 mon.a (mon.0) 887 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:39:18.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:39:18 smithi078 bash[75299]: audit 2023-10-01T10:39:17.506262+0000 mon.a (mon.0) 888 : audit 0 from='client.? 172.21.15.78:0/4057857734' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-01T10:39:18.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:39:18 smithi078 bash[75299]: cluster 2023-10-01T10:39:17.670845+0000 mgr.y (mgr.25072) 641 : cluster 0 pgmap v549: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:18.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:39:18 smithi173 bash[58816]: audit 2023-10-01T10:39:17.408465+0000 mon.a (mon.0) 887 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:39:18.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:39:18 smithi173 bash[58816]: audit 2023-10-01T10:39:17.506262+0000 mon.a (mon.0) 888 : audit 0 from='client.? 172.21.15.78:0/4057857734' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-01T10:39:18.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:39:18 smithi173 bash[58816]: cluster 2023-10-01T10:39:17.670845+0000 mgr.y (mgr.25072) 641 : cluster 0 pgmap v549: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:19.057 INFO:tasks.cephadm.ceph_manager.ceph:need seq 433791696974 got 433791696979 for osd.1 2023-10-01T10:39:19.057 DEBUG:teuthology.parallel:result is None 2023-10-01T10:39:19.830 INFO:tasks.cephadm.ceph_manager.ceph:need seq 180388626756 got 180388626761 for osd.6 2023-10-01T10:39:19.831 DEBUG:teuthology.parallel:result is None 2023-10-01T10:39:19.844 INFO:tasks.cephadm.ceph_manager.ceph:need seq 369367187584 got 369367187589 for osd.0 2023-10-01T10:39:19.844 DEBUG:teuthology.parallel:result is None 2023-10-01T10:39:19.918 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430522 got 206158430527 for osd.7 2023-10-01T10:39:19.918 DEBUG:teuthology.parallel:result is None 2023-10-01T10:39:19.919 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-10-01T10:39:19.919 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T10:39:20.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:39:19 smithi078 bash[75299]: cluster 2023-10-01T10:39:19.671838+0000 mgr.y (mgr.25072) 642 : cluster 0 pgmap v550: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:20.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:39:19 smithi078 bash[78274]: cluster 2023-10-01T10:39:19.671838+0000 mgr.y (mgr.25072) 642 : cluster 0 pgmap v550: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:20.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:39:19 smithi173 bash[58816]: cluster 2023-10-01T10:39:19.671838+0000 mgr.y (mgr.25072) 642 : cluster 0 pgmap v550: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:20.582 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:39:20 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:39:20] "GET /metrics HTTP/1.1" 200 33821 "" "Prometheus/2.33.4" 2023-10-01T10:39:22.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:39:21 smithi078 bash[75299]: cluster 2023-10-01T10:39:21.672851+0000 mgr.y (mgr.25072) 643 : cluster 0 pgmap v551: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:22.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:39:21 smithi078 bash[78274]: cluster 2023-10-01T10:39:21.672851+0000 mgr.y (mgr.25072) 643 : cluster 0 pgmap v551: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:22.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:39:21 smithi173 bash[58816]: cluster 2023-10-01T10:39:21.672851+0000 mgr.y (mgr.25072) 643 : cluster 0 pgmap v551: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:24.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:39:23 smithi078 bash[78274]: audit 2023-10-01T10:39:22.703246+0000 mon.a (mon.0) 889 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:39:24.319 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:39:23 smithi078 bash[78274]: audit 2023-10-01T10:39:22.710113+0000 mon.a (mon.0) 890 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:39:24.320 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:39:23 smithi078 bash[78274]: cluster 2023-10-01T10:39:23.673645+0000 mgr.y (mgr.25072) 644 : cluster 0 pgmap v552: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:24.320 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:39:23 smithi078 bash[75299]: audit 2023-10-01T10:39:22.703246+0000 mon.a (mon.0) 889 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:39:24.320 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:39:23 smithi078 bash[75299]: audit 2023-10-01T10:39:22.710113+0000 mon.a (mon.0) 890 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:39:24.320 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:39:23 smithi078 bash[75299]: cluster 2023-10-01T10:39:23.673645+0000 mgr.y (mgr.25072) 644 : cluster 0 pgmap v552: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:24.320 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:39:23 smithi173 bash[58816]: audit 2023-10-01T10:39:22.703246+0000 mon.a (mon.0) 889 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:39:24.321 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:39:23 smithi173 bash[58816]: audit 2023-10-01T10:39:22.710113+0000 mon.a (mon.0) 890 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:39:24.321 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:39:23 smithi173 bash[58816]: cluster 2023-10-01T10:39:23.673645+0000 mgr.y (mgr.25072) 644 : cluster 0 pgmap v552: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:24.469 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:39:25.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:39:25 smithi078 bash[78274]: audit 2023-10-01T10:39:24.621944+0000 mon.a (mon.0) 891 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:39:25.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:39:25 smithi078 bash[78274]: audit 2023-10-01T10:39:24.629247+0000 mon.a (mon.0) 892 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:39:25.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:39:25 smithi078 bash[78274]: audit 2023-10-01T10:39:25.099936+0000 mon.a (mon.0) 893 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:39:25.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:39:25 smithi078 bash[78274]: audit 2023-10-01T10:39:25.101478+0000 mon.a (mon.0) 894 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:39:25.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:39:25 smithi078 bash[78274]: audit 2023-10-01T10:39:25.112845+0000 mon.a (mon.0) 895 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:39:25.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:39:25 smithi078 bash[75299]: audit 2023-10-01T10:39:24.621944+0000 mon.a (mon.0) 891 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:39:25.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:39:25 smithi078 bash[75299]: audit 2023-10-01T10:39:24.629247+0000 mon.a (mon.0) 892 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:39:25.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:39:25 smithi078 bash[75299]: audit 2023-10-01T10:39:25.099936+0000 mon.a (mon.0) 893 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:39:25.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:39:25 smithi078 bash[75299]: audit 2023-10-01T10:39:25.101478+0000 mon.a (mon.0) 894 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:39:25.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:39:25 smithi078 bash[75299]: audit 2023-10-01T10:39:25.112845+0000 mon.a (mon.0) 895 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:39:26.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:39:25 smithi173 bash[58816]: audit 2023-10-01T10:39:24.621944+0000 mon.a (mon.0) 891 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:39:26.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:39:25 smithi173 bash[58816]: audit 2023-10-01T10:39:24.629247+0000 mon.a (mon.0) 892 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:39:26.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:39:25 smithi173 bash[58816]: audit 2023-10-01T10:39:25.099936+0000 mon.a (mon.0) 893 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:39:26.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:39:25 smithi173 bash[58816]: audit 2023-10-01T10:39:25.101478+0000 mon.a (mon.0) 894 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:39:26.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:39:25 smithi173 bash[58816]: audit 2023-10-01T10:39:25.112845+0000 mon.a (mon.0) 895 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:39:26.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:39:26 smithi078 bash[75299]: cluster 2023-10-01T10:39:25.674543+0000 mgr.y (mgr.25072) 645 : cluster 0 pgmap v553: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:26.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:39:26 smithi078 bash[78274]: cluster 2023-10-01T10:39:25.674543+0000 mgr.y (mgr.25072) 645 : cluster 0 pgmap v553: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:27.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:39:26 smithi173 bash[58816]: cluster 2023-10-01T10:39:25.674543+0000 mgr.y (mgr.25072) 645 : cluster 0 pgmap v553: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:27.913 INFO:teuthology.orchestra.run.smithi078.stdout: 2023-10-01T10:39:27.916 INFO:teuthology.orchestra.run.smithi078.stderr:dumped all 2023-10-01T10:39:28.213 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:39:28 smithi078 bash[78274]: cluster 2023-10-01T10:39:27.675065+0000 mgr.y (mgr.25072) 646 : cluster 0 pgmap v554: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:28.214 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:39:28 smithi078 bash[75299]: cluster 2023-10-01T10:39:27.675065+0000 mgr.y (mgr.25072) 646 : cluster 0 pgmap v554: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:28.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:39:28 smithi173 bash[58816]: cluster 2023-10-01T10:39:27.675065+0000 mgr.y (mgr.25072) 646 : cluster 0 pgmap v554: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:28.843 INFO:teuthology.orchestra.run.smithi078.stdout:{"pg_ready":true,"pg_map":{"version":554,"stamp":"2023-10-01T10:39:27.674755+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":261,"num_write_kb":2446,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":195,"ondisk_log_size":195,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2148588,"kb_used_data":11692,"kb_used_omap":30,"kb_used_meta":2136801,"kb_avail":747812628,"statfs":{"total":767960285184,"available":765760131072,"internally_reserved":0,"allocated":11972608,"data_stored":7995856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31502,"internal_metadata":2188084466},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004687"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":113,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.032258+0000","last_change":"2023-10-01T10:38:16.032258+0000","last_active":"2023-10-01T10:38:16.032258+0000","last_peered":"2023-10-01T10:38:16.032258+0000","last_clean":"2023-10-01T10:38:16.032258+0000","last_became_active":"2023-10-01T10:38:16.031633+0000","last_became_peered":"2023-10-01T10:38:16.031633+0000","last_unstale":"2023-10-01T10:38:16.032258+0000","last_undegraded":"2023-10-01T10:38:16.032258+0000","last_fullsized":"2023-10-01T10:38:16.032258+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:17:02.446430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":101,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.025881+0000","last_change":"2023-10-01T10:38:16.025881+0000","last_active":"2023-10-01T10:38:16.025881+0000","last_peered":"2023-10-01T10:38:16.025881+0000","last_clean":"2023-10-01T10:38:16.025881+0000","last_became_active":"2023-10-01T10:38:16.025518+0000","last_became_peered":"2023-10-01T10:38:16.025518+0000","last_unstale":"2023-10-01T10:38:16.025881+0000","last_undegraded":"2023-10-01T10:38:16.025881+0000","last_fullsized":"2023-10-01T10:38:16.025881+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:48:37.826604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":79,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.025075+0000","last_change":"2023-10-01T10:38:16.025075+0000","last_active":"2023-10-01T10:38:16.025075+0000","last_peered":"2023-10-01T10:38:16.025075+0000","last_clean":"2023-10-01T10:38:16.025075+0000","last_became_active":"2023-10-01T10:38:16.024735+0000","last_became_peered":"2023-10-01T10:38:16.024735+0000","last_unstale":"2023-10-01T10:38:16.025075+0000","last_undegraded":"2023-10-01T10:38:16.025075+0000","last_fullsized":"2023-10-01T10:38:16.025075+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:38:01.450741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":83,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.019448+0000","last_change":"2023-10-01T10:37:42.772008+0000","last_active":"2023-10-01T10:38:16.019448+0000","last_peered":"2023-10-01T10:38:16.019448+0000","last_clean":"2023-10-01T10:38:16.019448+0000","last_became_active":"2023-10-01T10:37:42.771244+0000","last_became_peered":"2023-10-01T10:37:42.771244+0000","last_unstale":"2023-10-01T10:38:16.019448+0000","last_undegraded":"2023-10-01T10:38:16.019448+0000","last_fullsized":"2023-10-01T10:38:16.019448+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:38:55.180790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":102,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.010782+0000","last_change":"2023-10-01T10:37:42.766600+0000","last_active":"2023-10-01T10:38:15.010782+0000","last_peered":"2023-10-01T10:38:15.010782+0000","last_clean":"2023-10-01T10:38:15.010782+0000","last_became_active":"2023-10-01T10:37:42.766051+0000","last_became_peered":"2023-10-01T10:37:42.766051+0000","last_unstale":"2023-10-01T10:38:15.010782+0000","last_undegraded":"2023-10-01T10:38:15.010782+0000","last_fullsized":"2023-10-01T10:38:15.010782+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:51:17.707963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":92,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.031873+0000","last_change":"2023-10-01T10:32:31.575109+0000","last_active":"2023-10-01T10:38:15.031873+0000","last_peered":"2023-10-01T10:38:15.031873+0000","last_clean":"2023-10-01T10:38:15.031873+0000","last_became_active":"2023-10-01T10:32:31.574729+0000","last_became_peered":"2023-10-01T10:32:31.574729+0000","last_unstale":"2023-10-01T10:38:15.031873+0000","last_undegraded":"2023-10-01T10:38:15.031873+0000","last_fullsized":"2023-10-01T10:38:15.031873+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":81,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.014432+0000","last_change":"2023-10-01T10:37:42.766667+0000","last_active":"2023-10-01T10:38:15.014432+0000","last_peered":"2023-10-01T10:38:15.014432+0000","last_clean":"2023-10-01T10:38:15.014432+0000","last_became_active":"2023-10-01T10:37:42.766369+0000","last_became_peered":"2023-10-01T10:37:42.766369+0000","last_unstale":"2023-10-01T10:38:15.014432+0000","last_undegraded":"2023-10-01T10:38:15.014432+0000","last_fullsized":"2023-10-01T10:38:15.014432+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:02.708788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":86,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.010584+0000","last_change":"2023-10-01T10:37:42.765925+0000","last_active":"2023-10-01T10:38:15.010584+0000","last_peered":"2023-10-01T10:38:15.010584+0000","last_clean":"2023-10-01T10:38:15.010584+0000","last_became_active":"2023-10-01T10:37:42.765218+0000","last_became_peered":"2023-10-01T10:37:42.765218+0000","last_unstale":"2023-10-01T10:38:15.010584+0000","last_undegraded":"2023-10-01T10:38:15.010584+0000","last_fullsized":"2023-10-01T10:38:15.010584+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T22:10:34.338031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":94,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.034622+0000","last_change":"2023-10-01T10:32:31.579805+0000","last_active":"2023-10-01T10:38:15.034622+0000","last_peered":"2023-10-01T10:38:15.034622+0000","last_clean":"2023-10-01T10:38:15.034622+0000","last_became_active":"2023-10-01T10:32:31.579423+0000","last_became_peered":"2023-10-01T10:32:31.579423+0000","last_unstale":"2023-10-01T10:38:15.034622+0000","last_undegraded":"2023-10-01T10:38:15.034622+0000","last_fullsized":"2023-10-01T10:38:15.034622+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":70,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.014763+0000","last_change":"2023-10-01T10:37:42.772531+0000","last_active":"2023-10-01T10:38:15.014763+0000","last_peered":"2023-10-01T10:38:15.014763+0000","last_clean":"2023-10-01T10:38:15.014763+0000","last_became_active":"2023-10-01T10:37:42.772216+0000","last_became_peered":"2023-10-01T10:37:42.772216+0000","last_unstale":"2023-10-01T10:38:15.014763+0000","last_undegraded":"2023-10-01T10:38:15.014763+0000","last_fullsized":"2023-10-01T10:38:15.014763+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:01:53.373628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":112,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.027085+0000","last_change":"2023-10-01T10:38:16.027085+0000","last_active":"2023-10-01T10:38:16.027085+0000","last_peered":"2023-10-01T10:38:16.027085+0000","last_clean":"2023-10-01T10:38:16.027085+0000","last_became_active":"2023-10-01T10:38:16.026555+0000","last_became_peered":"2023-10-01T10:38:16.026555+0000","last_unstale":"2023-10-01T10:38:16.027085+0000","last_undegraded":"2023-10-01T10:38:16.027085+0000","last_fullsized":"2023-10-01T10:38:16.027085+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":64,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.022465+0000","last_change":"2023-10-01T10:37:05.018755+0000","last_active":"2023-10-01T10:38:15.022465+0000","last_peered":"2023-10-01T10:38:15.022465+0000","last_clean":"2023-10-01T10:38:15.022465+0000","last_became_active":"2023-10-01T10:37:05.018446+0000","last_became_peered":"2023-10-01T10:37:05.018446+0000","last_unstale":"2023-10-01T10:38:15.022465+0000","last_undegraded":"2023-10-01T10:38:15.022465+0000","last_fullsized":"2023-10-01T10:38:15.022465+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:13:53.740503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":107,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.030826+0000","last_change":"2023-10-01T10:38:16.030826+0000","last_active":"2023-10-01T10:38:16.030826+0000","last_peered":"2023-10-01T10:38:16.030826+0000","last_clean":"2023-10-01T10:38:16.030826+0000","last_became_active":"2023-10-01T10:38:16.030277+0000","last_became_peered":"2023-10-01T10:38:16.030277+0000","last_unstale":"2023-10-01T10:38:16.030826+0000","last_undegraded":"2023-10-01T10:38:16.030826+0000","last_fullsized":"2023-10-01T10:38:16.030826+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:31:43.040424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":81,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.034741+0000","last_change":"2023-10-01T10:38:16.034741+0000","last_active":"2023-10-01T10:38:16.034741+0000","last_peered":"2023-10-01T10:38:16.034741+0000","last_clean":"2023-10-01T10:38:16.034741+0000","last_became_active":"2023-10-01T10:38:16.034407+0000","last_became_peered":"2023-10-01T10:38:16.034407+0000","last_unstale":"2023-10-01T10:38:16.034741+0000","last_undegraded":"2023-10-01T10:38:16.034741+0000","last_fullsized":"2023-10-01T10:38:16.034741+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:47:29.911477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":111,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.026777+0000","last_change":"2023-10-01T10:38:16.026777+0000","last_active":"2023-10-01T10:38:16.026777+0000","last_peered":"2023-10-01T10:38:16.026777+0000","last_clean":"2023-10-01T10:38:16.026777+0000","last_became_active":"2023-10-01T10:38:16.026371+0000","last_became_peered":"2023-10-01T10:38:16.026371+0000","last_unstale":"2023-10-01T10:38:16.026777+0000","last_undegraded":"2023-10-01T10:38:16.026777+0000","last_fullsized":"2023-10-01T10:38:16.026777+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":109,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.012172+0000","last_change":"2023-10-01T10:37:42.772319+0000","last_active":"2023-10-01T10:38:15.012172+0000","last_peered":"2023-10-01T10:38:15.012172+0000","last_clean":"2023-10-01T10:38:15.012172+0000","last_became_active":"2023-10-01T10:37:42.771977+0000","last_became_peered":"2023-10-01T10:37:42.771977+0000","last_unstale":"2023-10-01T10:38:15.012172+0000","last_undegraded":"2023-10-01T10:38:15.012172+0000","last_fullsized":"2023-10-01T10:38:15.012172+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:45:12.783214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":81,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.014954+0000","last_change":"2023-10-01T10:37:42.767706+0000","last_active":"2023-10-01T10:38:15.014954+0000","last_peered":"2023-10-01T10:38:15.014954+0000","last_clean":"2023-10-01T10:38:15.014954+0000","last_became_active":"2023-10-01T10:37:42.767211+0000","last_became_peered":"2023-10-01T10:37:42.767211+0000","last_unstale":"2023-10-01T10:38:15.014954+0000","last_undegraded":"2023-10-01T10:38:15.014954+0000","last_fullsized":"2023-10-01T10:38:15.014954+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:37:20.357407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":93,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.034746+0000","last_change":"2023-10-01T10:37:42.768642+0000","last_active":"2023-10-01T10:38:15.034746+0000","last_peered":"2023-10-01T10:38:15.034746+0000","last_clean":"2023-10-01T10:38:15.034746+0000","last_became_active":"2023-10-01T10:37:42.768239+0000","last_became_peered":"2023-10-01T10:37:42.768239+0000","last_unstale":"2023-10-01T10:38:15.034746+0000","last_undegraded":"2023-10-01T10:38:15.034746+0000","last_fullsized":"2023-10-01T10:38:15.034746+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":94,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.028563+0000","last_change":"2023-10-01T10:38:16.028563+0000","last_active":"2023-10-01T10:38:16.028563+0000","last_peered":"2023-10-01T10:38:16.028563+0000","last_clean":"2023-10-01T10:38:16.028563+0000","last_became_active":"2023-10-01T10:38:16.028162+0000","last_became_peered":"2023-10-01T10:38:16.028162+0000","last_unstale":"2023-10-01T10:38:16.028563+0000","last_undegraded":"2023-10-01T10:38:16.028563+0000","last_fullsized":"2023-10-01T10:38:16.028563+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":82,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.014770+0000","last_change":"2023-10-01T10:37:42.772247+0000","last_active":"2023-10-01T10:38:15.014770+0000","last_peered":"2023-10-01T10:38:15.014770+0000","last_clean":"2023-10-01T10:38:15.014770+0000","last_became_active":"2023-10-01T10:37:42.771697+0000","last_became_peered":"2023-10-01T10:37:42.771697+0000","last_unstale":"2023-10-01T10:38:15.014770+0000","last_undegraded":"2023-10-01T10:38:15.014770+0000","last_fullsized":"2023-10-01T10:38:15.014770+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:23:15.018248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":101,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.030438+0000","last_change":"2023-10-01T10:28:20.211798+0000","last_active":"2023-10-01T10:38:15.030438+0000","last_peered":"2023-10-01T10:38:15.030438+0000","last_clean":"2023-10-01T10:38:15.030438+0000","last_became_active":"2023-10-01T10:28:20.210791+0000","last_became_peered":"2023-10-01T10:28:20.210791+0000","last_unstale":"2023-10-01T10:38:15.030438+0000","last_undegraded":"2023-10-01T10:38:15.030438+0000","last_fullsized":"2023-10-01T10:38:15.030438+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":86,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.012486+0000","last_change":"2023-10-01T10:37:42.773127+0000","last_active":"2023-10-01T10:38:15.012486+0000","last_peered":"2023-10-01T10:38:15.012486+0000","last_clean":"2023-10-01T10:38:15.012486+0000","last_became_active":"2023-10-01T10:37:42.772102+0000","last_became_peered":"2023-10-01T10:37:42.772102+0000","last_unstale":"2023-10-01T10:38:15.012486+0000","last_undegraded":"2023-10-01T10:38:15.012486+0000","last_fullsized":"2023-10-01T10:38:15.012486+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:23:26.202295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":110,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.026955+0000","last_change":"2023-10-01T10:38:16.026955+0000","last_active":"2023-10-01T10:38:16.026955+0000","last_peered":"2023-10-01T10:38:16.026955+0000","last_clean":"2023-10-01T10:38:16.026955+0000","last_became_active":"2023-10-01T10:38:16.026552+0000","last_became_peered":"2023-10-01T10:38:16.026552+0000","last_unstale":"2023-10-01T10:38:16.026955+0000","last_undegraded":"2023-10-01T10:38:16.026955+0000","last_fullsized":"2023-10-01T10:38:16.026955+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":78,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.031063+0000","last_change":"2023-10-01T10:15:40.318702+0000","last_active":"2023-10-01T10:38:15.031063+0000","last_peered":"2023-10-01T10:38:15.031063+0000","last_clean":"2023-10-01T10:38:15.031063+0000","last_became_active":"2023-10-01T10:15:40.318333+0000","last_became_peered":"2023-10-01T10:15:40.318333+0000","last_unstale":"2023-10-01T10:38:15.031063+0000","last_undegraded":"2023-10-01T10:38:15.031063+0000","last_fullsized":"2023-10-01T10:38:15.031063+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":111,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.027753+0000","last_change":"2023-10-01T10:38:16.027753+0000","last_active":"2023-10-01T10:38:16.027753+0000","last_peered":"2023-10-01T10:38:16.027753+0000","last_clean":"2023-10-01T10:38:16.027753+0000","last_became_active":"2023-10-01T10:38:16.027384+0000","last_became_peered":"2023-10-01T10:38:16.027384+0000","last_unstale":"2023-10-01T10:38:16.027753+0000","last_undegraded":"2023-10-01T10:38:16.027753+0000","last_fullsized":"2023-10-01T10:38:16.027753+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":115,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.033203+0000","last_change":"2023-10-01T10:38:16.033203+0000","last_active":"2023-10-01T10:38:16.033203+0000","last_peered":"2023-10-01T10:38:16.033203+0000","last_clean":"2023-10-01T10:38:16.033203+0000","last_became_active":"2023-10-01T10:38:16.032681+0000","last_became_peered":"2023-10-01T10:38:16.032681+0000","last_unstale":"2023-10-01T10:38:16.033203+0000","last_undegraded":"2023-10-01T10:38:16.033203+0000","last_fullsized":"2023-10-01T10:38:16.033203+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:42:36.642284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":80,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.023250+0000","last_change":"2023-10-01T10:15:39.293587+0000","last_active":"2023-10-01T10:38:15.023250+0000","last_peered":"2023-10-01T10:38:15.023250+0000","last_clean":"2023-10-01T10:38:15.023250+0000","last_became_active":"2023-10-01T10:15:39.293273+0000","last_became_peered":"2023-10-01T10:15:39.293273+0000","last_unstale":"2023-10-01T10:38:15.023250+0000","last_undegraded":"2023-10-01T10:38:15.023250+0000","last_fullsized":"2023-10-01T10:38:15.023250+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":92,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.031078+0000","last_change":"2023-10-01T10:32:31.571971+0000","last_active":"2023-10-01T10:38:15.031078+0000","last_peered":"2023-10-01T10:38:15.031078+0000","last_clean":"2023-10-01T10:38:15.031078+0000","last_became_active":"2023-10-01T10:32:31.571410+0000","last_became_peered":"2023-10-01T10:32:31.571410+0000","last_unstale":"2023-10-01T10:38:15.031078+0000","last_undegraded":"2023-10-01T10:38:15.031078+0000","last_fullsized":"2023-10-01T10:38:15.031078+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":94,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.034665+0000","last_change":"2023-10-01T10:37:42.765733+0000","last_active":"2023-10-01T10:38:15.034665+0000","last_peered":"2023-10-01T10:38:15.034665+0000","last_clean":"2023-10-01T10:38:15.034665+0000","last_became_active":"2023-10-01T10:37:42.765311+0000","last_became_peered":"2023-10-01T10:37:42.765311+0000","last_unstale":"2023-10-01T10:38:15.034665+0000","last_undegraded":"2023-10-01T10:38:15.034665+0000","last_fullsized":"2023-10-01T10:38:15.034665+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":97,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.026348+0000","last_change":"2023-10-01T10:38:16.026348+0000","last_active":"2023-10-01T10:38:16.026348+0000","last_peered":"2023-10-01T10:38:16.026348+0000","last_clean":"2023-10-01T10:38:16.026348+0000","last_became_active":"2023-10-01T10:38:16.025747+0000","last_became_peered":"2023-10-01T10:38:16.025747+0000","last_unstale":"2023-10-01T10:38:16.026348+0000","last_undegraded":"2023-10-01T10:38:16.026348+0000","last_fullsized":"2023-10-01T10:38:16.026348+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:28:18.934480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":89,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.022493+0000","last_change":"2023-10-01T10:38:16.022493+0000","last_active":"2023-10-01T10:38:16.022493+0000","last_peered":"2023-10-01T10:38:16.022493+0000","last_clean":"2023-10-01T10:38:16.022493+0000","last_became_active":"2023-10-01T10:38:16.022033+0000","last_became_peered":"2023-10-01T10:38:16.022033+0000","last_unstale":"2023-10-01T10:38:16.022493+0000","last_undegraded":"2023-10-01T10:38:16.022493+0000","last_fullsized":"2023-10-01T10:38:16.022493+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:49:05.262763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":100,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.030613+0000","last_change":"2023-10-01T10:38:16.030613+0000","last_active":"2023-10-01T10:38:16.030613+0000","last_peered":"2023-10-01T10:38:16.030613+0000","last_clean":"2023-10-01T10:38:16.030613+0000","last_became_active":"2023-10-01T10:38:16.028975+0000","last_became_peered":"2023-10-01T10:38:16.028975+0000","last_unstale":"2023-10-01T10:38:16.030613+0000","last_undegraded":"2023-10-01T10:38:16.030613+0000","last_fullsized":"2023-10-01T10:38:16.030613+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:46:07.287911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":108,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.030250+0000","last_change":"2023-10-01T10:38:16.030250+0000","last_active":"2023-10-01T10:38:16.030250+0000","last_peered":"2023-10-01T10:38:16.030250+0000","last_clean":"2023-10-01T10:38:16.030250+0000","last_became_active":"2023-10-01T10:38:16.029898+0000","last_became_peered":"2023-10-01T10:38:16.029898+0000","last_unstale":"2023-10-01T10:38:16.030250+0000","last_undegraded":"2023-10-01T10:38:16.030250+0000","last_fullsized":"2023-10-01T10:38:16.030250+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":105,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.035906+0000","last_change":"2023-10-01T10:38:16.035906+0000","last_active":"2023-10-01T10:38:16.035906+0000","last_peered":"2023-10-01T10:38:16.035906+0000","last_clean":"2023-10-01T10:38:16.035906+0000","last_became_active":"2023-10-01T10:38:16.035383+0000","last_became_peered":"2023-10-01T10:38:16.035383+0000","last_unstale":"2023-10-01T10:38:16.035906+0000","last_undegraded":"2023-10-01T10:38:16.035906+0000","last_fullsized":"2023-10-01T10:38:16.035906+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:44:41.500097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":97,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.026611+0000","last_change":"2023-10-01T10:38:16.026611+0000","last_active":"2023-10-01T10:38:16.026611+0000","last_peered":"2023-10-01T10:38:16.026611+0000","last_clean":"2023-10-01T10:38:16.026611+0000","last_became_active":"2023-10-01T10:38:16.026267+0000","last_became_peered":"2023-10-01T10:38:16.026267+0000","last_unstale":"2023-10-01T10:38:16.026611+0000","last_undegraded":"2023-10-01T10:38:16.026611+0000","last_fullsized":"2023-10-01T10:38:16.026611+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:06.135288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":81,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.032162+0000","last_change":"2023-10-01T10:38:16.032162+0000","last_active":"2023-10-01T10:38:16.032162+0000","last_peered":"2023-10-01T10:38:16.032162+0000","last_clean":"2023-10-01T10:38:16.032162+0000","last_became_active":"2023-10-01T10:38:16.031785+0000","last_became_peered":"2023-10-01T10:38:16.031785+0000","last_unstale":"2023-10-01T10:38:16.032162+0000","last_undegraded":"2023-10-01T10:38:16.032162+0000","last_fullsized":"2023-10-01T10:38:16.032162+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:38:43.003626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":87,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.011165+0000","last_change":"2023-10-01T10:37:42.772398+0000","last_active":"2023-10-01T10:38:15.011165+0000","last_peered":"2023-10-01T10:38:15.011165+0000","last_clean":"2023-10-01T10:38:15.011165+0000","last_became_active":"2023-10-01T10:37:42.772084+0000","last_became_peered":"2023-10-01T10:37:42.772084+0000","last_unstale":"2023-10-01T10:38:15.011165+0000","last_undegraded":"2023-10-01T10:38:15.011165+0000","last_fullsized":"2023-10-01T10:38:15.011165+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:32:43.846419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":106,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.021873+0000","last_change":"2023-10-01T10:38:16.019755+0000","last_active":"2023-10-01T10:38:16.021873+0000","last_peered":"2023-10-01T10:38:16.021873+0000","last_clean":"2023-10-01T10:38:16.021873+0000","last_became_active":"2023-10-01T10:38:16.019397+0000","last_became_peered":"2023-10-01T10:38:16.019397+0000","last_unstale":"2023-10-01T10:38:16.021873+0000","last_undegraded":"2023-10-01T10:38:16.021873+0000","last_fullsized":"2023-10-01T10:38:16.021873+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:42:56.623225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":99,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.020037+0000","last_change":"2023-10-01T10:32:31.561108+0000","last_active":"2023-10-01T10:38:16.020037+0000","last_peered":"2023-10-01T10:38:16.020037+0000","last_clean":"2023-10-01T10:38:16.020037+0000","last_became_active":"2023-10-01T10:32:31.560791+0000","last_became_peered":"2023-10-01T10:32:31.560791+0000","last_unstale":"2023-10-01T10:38:16.020037+0000","last_undegraded":"2023-10-01T10:38:16.020037+0000","last_fullsized":"2023-10-01T10:38:16.020037+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:42:47.477671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":109,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.034655+0000","last_change":"2023-10-01T10:37:42.766814+0000","last_active":"2023-10-01T10:38:15.034655+0000","last_peered":"2023-10-01T10:38:15.034655+0000","last_clean":"2023-10-01T10:38:15.034655+0000","last_became_active":"2023-10-01T10:37:42.766451+0000","last_became_peered":"2023-10-01T10:37:42.766451+0000","last_unstale":"2023-10-01T10:38:15.034655+0000","last_undegraded":"2023-10-01T10:38:15.034655+0000","last_fullsized":"2023-10-01T10:38:15.034655+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":65,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.035088+0000","last_change":"2023-10-01T10:38:16.034357+0000","last_active":"2023-10-01T10:38:16.035088+0000","last_peered":"2023-10-01T10:38:16.035088+0000","last_clean":"2023-10-01T10:38:16.035088+0000","last_became_active":"2023-10-01T10:38:16.033905+0000","last_became_peered":"2023-10-01T10:38:16.033905+0000","last_unstale":"2023-10-01T10:38:16.035088+0000","last_undegraded":"2023-10-01T10:38:16.035088+0000","last_fullsized":"2023-10-01T10:38:16.035088+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:47:58.369835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":98,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.034322+0000","last_change":"2023-10-01T10:32:31.572482+0000","last_active":"2023-10-01T10:38:15.034322+0000","last_peered":"2023-10-01T10:38:15.034322+0000","last_clean":"2023-10-01T10:38:15.034322+0000","last_became_active":"2023-10-01T10:32:31.571993+0000","last_became_peered":"2023-10-01T10:32:31.571993+0000","last_unstale":"2023-10-01T10:38:15.034322+0000","last_undegraded":"2023-10-01T10:38:15.034322+0000","last_fullsized":"2023-10-01T10:38:15.034322+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":67,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.030202+0000","last_change":"2023-10-01T10:38:16.030202+0000","last_active":"2023-10-01T10:38:16.030202+0000","last_peered":"2023-10-01T10:38:16.030202+0000","last_clean":"2023-10-01T10:38:16.030202+0000","last_became_active":"2023-10-01T10:38:16.029917+0000","last_became_peered":"2023-10-01T10:38:16.029917+0000","last_unstale":"2023-10-01T10:38:16.030202+0000","last_undegraded":"2023-10-01T10:38:16.030202+0000","last_fullsized":"2023-10-01T10:38:16.030202+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:20:06.763909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":80,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.020177+0000","last_change":"2023-10-01T10:37:42.761153+0000","last_active":"2023-10-01T10:38:16.020177+0000","last_peered":"2023-10-01T10:38:16.020177+0000","last_clean":"2023-10-01T10:38:16.020177+0000","last_became_active":"2023-10-01T10:37:42.760715+0000","last_became_peered":"2023-10-01T10:37:42.760715+0000","last_unstale":"2023-10-01T10:38:16.020177+0000","last_undegraded":"2023-10-01T10:38:16.020177+0000","last_fullsized":"2023-10-01T10:38:16.020177+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:52:50.237174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":97,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.013202+0000","last_change":"2023-10-01T10:37:42.762524+0000","last_active":"2023-10-01T10:38:16.013202+0000","last_peered":"2023-10-01T10:38:16.013202+0000","last_clean":"2023-10-01T10:38:16.013202+0000","last_became_active":"2023-10-01T10:37:42.762209+0000","last_became_peered":"2023-10-01T10:37:42.762209+0000","last_unstale":"2023-10-01T10:38:16.013202+0000","last_undegraded":"2023-10-01T10:38:16.013202+0000","last_fullsized":"2023-10-01T10:38:16.013202+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:45:38.171397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":65,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.020920+0000","last_change":"2023-10-01T10:37:05.019330+0000","last_active":"2023-10-01T10:38:16.020920+0000","last_peered":"2023-10-01T10:38:16.020920+0000","last_clean":"2023-10-01T10:38:16.020920+0000","last_became_active":"2023-10-01T10:37:05.018842+0000","last_became_peered":"2023-10-01T10:37:05.018842+0000","last_unstale":"2023-10-01T10:38:16.020920+0000","last_undegraded":"2023-10-01T10:38:16.020920+0000","last_fullsized":"2023-10-01T10:38:16.020920+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:48:09.551917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":94,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.019778+0000","last_change":"2023-10-01T10:32:31.569162+0000","last_active":"2023-10-01T10:38:16.019778+0000","last_peered":"2023-10-01T10:38:16.019778+0000","last_clean":"2023-10-01T10:38:16.019778+0000","last_became_active":"2023-10-01T10:32:31.568956+0000","last_became_peered":"2023-10-01T10:32:31.568956+0000","last_unstale":"2023-10-01T10:38:16.019778+0000","last_undegraded":"2023-10-01T10:38:16.019778+0000","last_fullsized":"2023-10-01T10:38:16.019778+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:02:08.285304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":98,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.033675+0000","last_change":"2023-10-01T10:37:05.024009+0000","last_active":"2023-10-01T10:38:15.033675+0000","last_peered":"2023-10-01T10:38:15.033675+0000","last_clean":"2023-10-01T10:38:15.033675+0000","last_became_active":"2023-10-01T10:37:05.023458+0000","last_became_peered":"2023-10-01T10:37:05.023458+0000","last_unstale":"2023-10-01T10:38:15.033675+0000","last_undegraded":"2023-10-01T10:38:15.033675+0000","last_fullsized":"2023-10-01T10:38:15.033675+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'139","reported_seq":330,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.019558+0000","last_change":"2023-10-01T10:28:20.217386+0000","last_active":"2023-10-01T10:38:15.019558+0000","last_peered":"2023-10-01T10:38:15.019558+0000","last_clean":"2023-10-01T10:38:15.019558+0000","last_became_active":"2023-10-01T10:28:20.216803+0000","last_became_peered":"2023-10-01T10:28:20.216803+0000","last_unstale":"2023-10-01T10:38:15.019558+0000","last_undegraded":"2023-10-01T10:38:15.019558+0000","last_fullsized":"2023-10-01T10:38:15.019558+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:09:17.221082+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:09:17.221082+0000","last_clean_scrub_stamp":"2023-10-01T10:09:17.221082+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":69,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.012342+0000","last_change":"2023-10-01T10:37:42.765100+0000","last_active":"2023-10-01T10:38:15.012342+0000","last_peered":"2023-10-01T10:38:15.012342+0000","last_clean":"2023-10-01T10:38:15.012342+0000","last_became_active":"2023-10-01T10:37:42.764761+0000","last_became_peered":"2023-10-01T10:37:42.764761+0000","last_unstale":"2023-10-01T10:38:15.012342+0000","last_undegraded":"2023-10-01T10:38:15.012342+0000","last_fullsized":"2023-10-01T10:38:15.012342+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:24:56.907377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":92,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.031255+0000","last_change":"2023-10-01T10:38:16.031255+0000","last_active":"2023-10-01T10:38:16.031255+0000","last_peered":"2023-10-01T10:38:16.031255+0000","last_clean":"2023-10-01T10:38:16.031255+0000","last_became_active":"2023-10-01T10:38:16.030886+0000","last_became_peered":"2023-10-01T10:38:16.030886+0000","last_unstale":"2023-10-01T10:38:16.031255+0000","last_undegraded":"2023-10-01T10:38:16.031255+0000","last_fullsized":"2023-10-01T10:38:16.031255+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":113,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.019451+0000","last_change":"2023-10-01T10:32:31.565868+0000","last_active":"2023-10-01T10:38:15.019451+0000","last_peered":"2023-10-01T10:38:15.019451+0000","last_clean":"2023-10-01T10:38:15.019451+0000","last_became_active":"2023-10-01T10:32:31.565428+0000","last_became_peered":"2023-10-01T10:32:31.565428+0000","last_unstale":"2023-10-01T10:38:15.019451+0000","last_undegraded":"2023-10-01T10:38:15.019451+0000","last_fullsized":"2023-10-01T10:38:15.019451+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":97,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.022511+0000","last_change":"2023-10-01T10:38:16.021968+0000","last_active":"2023-10-01T10:38:16.022511+0000","last_peered":"2023-10-01T10:38:16.022511+0000","last_clean":"2023-10-01T10:38:16.022511+0000","last_became_active":"2023-10-01T10:38:16.021539+0000","last_became_peered":"2023-10-01T10:38:16.021539+0000","last_unstale":"2023-10-01T10:38:16.022511+0000","last_undegraded":"2023-10-01T10:38:16.022511+0000","last_fullsized":"2023-10-01T10:38:16.022511+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:12:53.105747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":98,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.019580+0000","last_change":"2023-10-01T10:37:05.017431+0000","last_active":"2023-10-01T10:38:16.019580+0000","last_peered":"2023-10-01T10:38:16.019580+0000","last_clean":"2023-10-01T10:38:16.019580+0000","last_became_active":"2023-10-01T10:37:05.017094+0000","last_became_peered":"2023-10-01T10:37:05.017094+0000","last_unstale":"2023-10-01T10:38:16.019580+0000","last_undegraded":"2023-10-01T10:38:16.019580+0000","last_fullsized":"2023-10-01T10:38:16.019580+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:10:40.664686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":113,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.032774+0000","last_change":"2023-10-01T10:38:16.032774+0000","last_active":"2023-10-01T10:38:16.032774+0000","last_peered":"2023-10-01T10:38:16.032774+0000","last_clean":"2023-10-01T10:38:16.032774+0000","last_became_active":"2023-10-01T10:38:16.032283+0000","last_became_peered":"2023-10-01T10:38:16.032283+0000","last_unstale":"2023-10-01T10:38:16.032774+0000","last_undegraded":"2023-10-01T10:38:16.032774+0000","last_fullsized":"2023-10-01T10:38:16.032774+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":65,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.020296+0000","last_change":"2023-10-01T10:37:05.021228+0000","last_active":"2023-10-01T10:38:16.020296+0000","last_peered":"2023-10-01T10:38:16.020296+0000","last_clean":"2023-10-01T10:38:16.020296+0000","last_became_active":"2023-10-01T10:37:05.020888+0000","last_became_peered":"2023-10-01T10:37:05.020888+0000","last_unstale":"2023-10-01T10:38:16.020296+0000","last_undegraded":"2023-10-01T10:38:16.020296+0000","last_fullsized":"2023-10-01T10:38:16.020296+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:59:57.641247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":68,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.020664+0000","last_change":"2023-10-01T10:28:20.220812+0000","last_active":"2023-10-01T10:38:16.020664+0000","last_peered":"2023-10-01T10:38:16.020664+0000","last_clean":"2023-10-01T10:38:16.020664+0000","last_became_active":"2023-10-01T10:28:20.215962+0000","last_became_peered":"2023-10-01T10:28:20.215962+0000","last_unstale":"2023-10-01T10:38:16.020664+0000","last_undegraded":"2023-10-01T10:38:16.020664+0000","last_fullsized":"2023-10-01T10:38:16.020664+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:35:15.275097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":74,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.012601+0000","last_change":"2023-10-01T10:37:42.773999+0000","last_active":"2023-10-01T10:38:15.012601+0000","last_peered":"2023-10-01T10:38:15.012601+0000","last_clean":"2023-10-01T10:38:15.012601+0000","last_became_active":"2023-10-01T10:37:42.773355+0000","last_became_peered":"2023-10-01T10:37:42.773355+0000","last_unstale":"2023-10-01T10:38:15.012601+0000","last_undegraded":"2023-10-01T10:38:15.012601+0000","last_fullsized":"2023-10-01T10:38:15.012601+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:46:51.317151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":107,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.035022+0000","last_change":"2023-10-01T10:38:16.034193+0000","last_active":"2023-10-01T10:38:16.035022+0000","last_peered":"2023-10-01T10:38:16.035022+0000","last_clean":"2023-10-01T10:38:16.035022+0000","last_became_active":"2023-10-01T10:38:16.033703+0000","last_became_peered":"2023-10-01T10:38:16.033703+0000","last_unstale":"2023-10-01T10:38:16.035022+0000","last_undegraded":"2023-10-01T10:38:16.035022+0000","last_fullsized":"2023-10-01T10:38:16.035022+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:21:26.061595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":71,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.010953+0000","last_change":"2023-10-01T10:37:42.773538+0000","last_active":"2023-10-01T10:38:15.010953+0000","last_peered":"2023-10-01T10:38:15.010953+0000","last_clean":"2023-10-01T10:38:15.010953+0000","last_became_active":"2023-10-01T10:37:42.771863+0000","last_became_peered":"2023-10-01T10:37:42.771863+0000","last_unstale":"2023-10-01T10:38:15.010953+0000","last_undegraded":"2023-10-01T10:38:15.010953+0000","last_fullsized":"2023-10-01T10:38:15.010953+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:46:46.698627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":97,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.031790+0000","last_change":"2023-10-01T10:37:05.023034+0000","last_active":"2023-10-01T10:38:15.031790+0000","last_peered":"2023-10-01T10:38:15.031790+0000","last_clean":"2023-10-01T10:38:15.031790+0000","last_became_active":"2023-10-01T10:37:05.022670+0000","last_became_peered":"2023-10-01T10:37:05.022670+0000","last_unstale":"2023-10-01T10:38:15.031790+0000","last_undegraded":"2023-10-01T10:38:15.031790+0000","last_fullsized":"2023-10-01T10:38:15.031790+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":101,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.027938+0000","last_change":"2023-10-01T10:38:16.027472+0000","last_active":"2023-10-01T10:38:16.027938+0000","last_peered":"2023-10-01T10:38:16.027938+0000","last_clean":"2023-10-01T10:38:16.027938+0000","last_became_active":"2023-10-01T10:38:16.026810+0000","last_became_peered":"2023-10-01T10:38:16.026810+0000","last_unstale":"2023-10-01T10:38:16.027938+0000","last_undegraded":"2023-10-01T10:38:16.027938+0000","last_fullsized":"2023-10-01T10:38:16.027938+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:30:06.978187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":65,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.035705+0000","last_change":"2023-10-01T10:38:16.035705+0000","last_active":"2023-10-01T10:38:16.035705+0000","last_peered":"2023-10-01T10:38:16.035705+0000","last_clean":"2023-10-01T10:38:16.035705+0000","last_became_active":"2023-10-01T10:38:16.035389+0000","last_became_peered":"2023-10-01T10:38:16.035389+0000","last_unstale":"2023-10-01T10:38:16.035705+0000","last_undegraded":"2023-10-01T10:38:16.035705+0000","last_fullsized":"2023-10-01T10:38:16.035705+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:16:10.201409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":85,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.028785+0000","last_change":"2023-10-01T10:38:16.028491+0000","last_active":"2023-10-01T10:38:16.028785+0000","last_peered":"2023-10-01T10:38:16.028785+0000","last_clean":"2023-10-01T10:38:16.028785+0000","last_became_active":"2023-10-01T10:38:16.028175+0000","last_became_peered":"2023-10-01T10:38:16.028175+0000","last_unstale":"2023-10-01T10:38:16.028785+0000","last_undegraded":"2023-10-01T10:38:16.028785+0000","last_fullsized":"2023-10-01T10:38:16.028785+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:44:29.083129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":106,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.024737+0000","last_change":"2023-10-01T10:37:42.762360+0000","last_active":"2023-10-01T10:38:15.024737+0000","last_peered":"2023-10-01T10:38:15.024737+0000","last_clean":"2023-10-01T10:38:15.024737+0000","last_became_active":"2023-10-01T10:37:42.761936+0000","last_became_peered":"2023-10-01T10:37:42.761936+0000","last_unstale":"2023-10-01T10:38:15.024737+0000","last_undegraded":"2023-10-01T10:38:15.024737+0000","last_fullsized":"2023-10-01T10:38:15.024737+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:50.637582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":111,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.037809+0000","last_change":"2023-10-01T10:38:16.037809+0000","last_active":"2023-10-01T10:38:16.037809+0000","last_peered":"2023-10-01T10:38:16.037809+0000","last_clean":"2023-10-01T10:38:16.037809+0000","last_became_active":"2023-10-01T10:38:16.037318+0000","last_became_peered":"2023-10-01T10:38:16.037318+0000","last_unstale":"2023-10-01T10:38:16.037809+0000","last_undegraded":"2023-10-01T10:38:16.037809+0000","last_fullsized":"2023-10-01T10:38:16.037809+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":115,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.034282+0000","last_change":"2023-10-01T10:38:16.034282+0000","last_active":"2023-10-01T10:38:16.034282+0000","last_peered":"2023-10-01T10:38:16.034282+0000","last_clean":"2023-10-01T10:38:16.034282+0000","last_became_active":"2023-10-01T10:38:16.033820+0000","last_became_peered":"2023-10-01T10:38:16.033820+0000","last_unstale":"2023-10-01T10:38:16.034282+0000","last_undegraded":"2023-10-01T10:38:16.034282+0000","last_fullsized":"2023-10-01T10:38:16.034282+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:48:23.815832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":96,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.038077+0000","last_change":"2023-10-01T10:38:16.038077+0000","last_active":"2023-10-01T10:38:16.038077+0000","last_peered":"2023-10-01T10:38:16.038077+0000","last_clean":"2023-10-01T10:38:16.038077+0000","last_became_active":"2023-10-01T10:38:16.037313+0000","last_became_peered":"2023-10-01T10:38:16.037313+0000","last_unstale":"2023-10-01T10:38:16.038077+0000","last_undegraded":"2023-10-01T10:38:16.038077+0000","last_fullsized":"2023-10-01T10:38:16.038077+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":115,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.025600+0000","last_change":"2023-10-01T10:38:16.025600+0000","last_active":"2023-10-01T10:38:16.025600+0000","last_peered":"2023-10-01T10:38:16.025600+0000","last_clean":"2023-10-01T10:38:16.025600+0000","last_became_active":"2023-10-01T10:38:16.025252+0000","last_became_peered":"2023-10-01T10:38:16.025252+0000","last_unstale":"2023-10-01T10:38:16.025600+0000","last_undegraded":"2023-10-01T10:38:16.025600+0000","last_fullsized":"2023-10-01T10:38:16.025600+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:24:18.871963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":102,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.012215+0000","last_change":"2023-10-01T10:37:42.761243+0000","last_active":"2023-10-01T10:38:15.012215+0000","last_peered":"2023-10-01T10:38:15.012215+0000","last_clean":"2023-10-01T10:38:15.012215+0000","last_became_active":"2023-10-01T10:37:42.760834+0000","last_became_peered":"2023-10-01T10:37:42.760834+0000","last_unstale":"2023-10-01T10:38:15.012215+0000","last_undegraded":"2023-10-01T10:38:15.012215+0000","last_fullsized":"2023-10-01T10:38:15.012215+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:37:21.976095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":115,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.014918+0000","last_change":"2023-10-01T10:37:42.765142+0000","last_active":"2023-10-01T10:38:15.014918+0000","last_peered":"2023-10-01T10:38:15.014918+0000","last_clean":"2023-10-01T10:38:15.014918+0000","last_became_active":"2023-10-01T10:37:42.764770+0000","last_became_peered":"2023-10-01T10:37:42.764770+0000","last_unstale":"2023-10-01T10:38:15.014918+0000","last_undegraded":"2023-10-01T10:38:15.014918+0000","last_fullsized":"2023-10-01T10:38:15.014918+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:23:33.527876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":86,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.019521+0000","last_change":"2023-10-01T10:37:05.021621+0000","last_active":"2023-10-01T10:38:16.019521+0000","last_peered":"2023-10-01T10:38:16.019521+0000","last_clean":"2023-10-01T10:38:16.019521+0000","last_became_active":"2023-10-01T10:37:05.021173+0000","last_became_peered":"2023-10-01T10:37:05.021173+0000","last_unstale":"2023-10-01T10:38:16.019521+0000","last_undegraded":"2023-10-01T10:38:16.019521+0000","last_fullsized":"2023-10-01T10:38:16.019521+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:13:25.364499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":81,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.031427+0000","last_change":"2023-10-01T10:38:16.031427+0000","last_active":"2023-10-01T10:38:16.031427+0000","last_peered":"2023-10-01T10:38:16.031427+0000","last_clean":"2023-10-01T10:38:16.031427+0000","last_became_active":"2023-10-01T10:38:16.029786+0000","last_became_peered":"2023-10-01T10:38:16.029786+0000","last_unstale":"2023-10-01T10:38:16.031427+0000","last_undegraded":"2023-10-01T10:38:16.031427+0000","last_fullsized":"2023-10-01T10:38:16.031427+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:53:22.796191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30319,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430529,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":303116,"kb_used_data":1796,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":93442036,"statfs":{"total":95995035648,"available":95684644864,"internally_reserved":0,"allocated":1839104,"data_stored":1363146,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:38:38 2023","interfaces":[{"interface":"back","average":{"1min":0.97499999999999998,"5min":1.4410000000000001,"15min":1.7450000000000001},"min":{"1min":0.82599999999999996,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":1.2290000000000001,"5min":7.9900000000000002,"15min":14.974},"last":0.69099999999999995},{"interface":"front","average":{"1min":0.96299999999999997,"5min":1.5429999999999999,"15min":1.8169999999999999},"min":{"1min":0.65100000000000002,"5min":0.32800000000000001,"15min":0.29999999999999999},"max":{"1min":1.21,"5min":9.7289999999999992,"15min":15.029},"last":0.71099999999999997}]},{"osd":1,"last update":"Sun Oct 1 10:38:50 2023","interfaces":[{"interface":"back","average":{"1min":0.82199999999999995,"5min":1.454,"15min":1.23},"min":{"1min":0.57899999999999996,"5min":0.35999999999999999,"15min":0.309},"max":{"1min":1.1779999999999999,"5min":8.1180000000000003,"15min":8.1180000000000003},"last":0.92100000000000004},{"interface":"front","average":{"1min":0.94299999999999995,"5min":1.4330000000000001,"15min":1.617},"min":{"1min":0.621,"5min":0.318,"15min":0.318},"max":{"1min":1.2589999999999999,"5min":11.266,"15min":11.266},"last":0.66200000000000003}]},{"osd":2,"last update":"Sun Oct 1 10:39:05 2023","interfaces":[{"interface":"back","average":{"1min":0.96799999999999997,"5min":1.022,"15min":1.0309999999999999},"min":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"max":{"1min":1.1799999999999999,"5min":1.214,"15min":1.214},"last":0.81699999999999995},{"interface":"front","average":{"1min":0.89600000000000002,"5min":0.93600000000000005,"15min":0.94299999999999995},"min":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.167,"5min":1.294,"15min":1.294},"last":0.95699999999999996}]},{"osd":3,"last update":"Sun Oct 1 10:38:44 2023","interfaces":[{"interface":"back","average":{"1min":0.96799999999999997,"5min":0.96799999999999997,"15min":0.96799999999999997},"min":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"max":{"1min":1.3879999999999999,"5min":1.3879999999999999,"15min":1.3879999999999999},"last":0.94199999999999995},{"interface":"front","average":{"1min":0.94699999999999995,"5min":0.94699999999999995,"15min":0.94699999999999995},"min":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"max":{"1min":1.234,"5min":1.234,"15min":1.234},"last":1.0109999999999999}]},{"osd":4,"last update":"Sun Oct 1 10:39:18 2023","interfaces":[{"interface":"back","average":{"1min":0.91700000000000004,"5min":0.91700000000000004,"15min":0.91700000000000004},"min":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"max":{"1min":1.167,"5min":1.167,"15min":1.167},"last":1.028},{"interface":"front","average":{"1min":1.1859999999999999,"5min":1.1859999999999999,"15min":1.1859999999999999},"min":{"1min":0.90500000000000003,"5min":0.90500000000000003,"15min":0.90500000000000003},"max":{"1min":1.333,"5min":1.333,"15min":1.333},"last":0.90800000000000003}]},{"osd":5,"last update":"Sun Oct 1 10:38:50 2023","interfaces":[{"interface":"back","average":{"1min":0.93899999999999995,"5min":1.2310000000000001,"15min":1.2609999999999999},"min":{"1min":0.504,"5min":0.35799999999999998,"15min":0.29699999999999999},"max":{"1min":1.349,"5min":10.106999999999999,"15min":10.106999999999999},"last":0.75},{"interface":"front","average":{"1min":0.93899999999999995,"5min":1.232,"15min":1.21},"min":{"1min":0.48699999999999999,"5min":0.42799999999999999,"15min":0.34799999999999998},"max":{"1min":1.3069999999999999,"5min":6.2050000000000001,"15min":6.2050000000000001},"last":0.877}]},{"osd":6,"last update":"Sun Oct 1 10:38:50 2023","interfaces":[{"interface":"back","average":{"1min":1.052,"5min":1.421,"15min":1.341},"min":{"1min":0.46100000000000002,"5min":0.39300000000000002,"15min":0.32800000000000001},"max":{"1min":1.345,"5min":7.3300000000000001,"15min":9.5500000000000007},"last":0.85499999999999998},{"interface":"front","average":{"1min":0.91900000000000004,"5min":1.4410000000000001,"15min":1.391},"min":{"1min":0.58999999999999997,"5min":0.40100000000000002,"15min":0.33900000000000002},"max":{"1min":1.2709999999999999,"5min":6.984,"15min":12.696},"last":0.83799999999999997}]}]},{"osd":6,"up_from":42,"seq":180388626763,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":307196,"kb_used_data":1780,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":93437956,"statfs":{"total":95995035648,"available":95680466944,"internally_reserved":0,"allocated":1822720,"data_stored":1362685,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:38:33 2023","interfaces":[{"interface":"back","average":{"1min":0.79800000000000004,"5min":1.4339999999999999,"15min":1.4850000000000001},"min":{"1min":0.51700000000000002,"5min":0.29399999999999998,"15min":0.28499999999999998},"max":{"1min":1.0940000000000001,"5min":7.1139999999999999,"15min":9.1300000000000008},"last":1.236},{"interface":"front","average":{"1min":0.91700000000000004,"5min":1.573,"15min":1.4990000000000001},"min":{"1min":0.60599999999999998,"5min":0.29999999999999999,"15min":0.28899999999999998},"max":{"1min":1.2769999999999999,"5min":6.9640000000000004,"15min":7.3109999999999999},"last":0.82999999999999996}]},{"osd":1,"last update":"Sun Oct 1 10:38:41 2023","interfaces":[{"interface":"back","average":{"1min":0.88200000000000001,"5min":1.5920000000000001,"15min":1.1459999999999999},"min":{"1min":0.70599999999999996,"5min":0.37,"15min":0.26600000000000001},"max":{"1min":1.21,"5min":7.9100000000000001,"15min":7.9100000000000001},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.90800000000000003,"5min":1.4930000000000001,"15min":1.492},"min":{"1min":0.55600000000000005,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.2,"5min":7.4199999999999999,"15min":7.4199999999999999},"last":0.63900000000000001}]},{"osd":2,"last update":"Sun Oct 1 10:39:05 2023","interfaces":[{"interface":"back","average":{"1min":0.88700000000000001,"5min":0.87,"15min":0.86699999999999999},"min":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"max":{"1min":1.2450000000000001,"5min":1.2450000000000001,"15min":1.2450000000000001},"last":0.68899999999999995},{"interface":"front","average":{"1min":0.95099999999999996,"5min":0.94099999999999995,"15min":0.93899999999999995},"min":{"1min":0.70899999999999996,"5min":0.64900000000000002,"15min":0.64900000000000002},"max":{"1min":1.1850000000000001,"5min":1.2929999999999999,"15min":1.2929999999999999},"last":0.80100000000000005}]},{"osd":3,"last update":"Sun Oct 1 10:38:44 2023","interfaces":[{"interface":"back","average":{"1min":0.98099999999999998,"5min":0.98099999999999998,"15min":0.98099999999999998},"min":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"max":{"1min":1.2529999999999999,"5min":1.2529999999999999,"15min":1.2529999999999999},"last":1.1160000000000001},{"interface":"front","average":{"1min":1.018,"5min":1.018,"15min":1.018},"min":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"max":{"1min":1.2230000000000001,"5min":1.2230000000000001,"15min":1.2230000000000001},"last":0.89200000000000002}]},{"osd":4,"last update":"Sun Oct 1 10:39:18 2023","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":1.02,"15min":1.02},"min":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"max":{"1min":1.3919999999999999,"5min":1.3919999999999999,"15min":1.3919999999999999},"last":0.90800000000000003},{"interface":"front","average":{"1min":1.103,"5min":1.103,"15min":1.103},"min":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"max":{"1min":1.4179999999999999,"5min":1.4179999999999999,"15min":1.4179999999999999},"last":1.163}]},{"osd":5,"last update":"Sun Oct 1 10:38:49 2023","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":1.3120000000000001,"15min":1.2270000000000001},"min":{"1min":0.46100000000000002,"5min":0.39100000000000001,"15min":0.29699999999999999},"max":{"1min":1.254,"5min":6.085,"15min":8.0150000000000006},"last":1.2989999999999999},{"interface":"front","average":{"1min":0.97899999999999998,"5min":1.284,"15min":1.278},"min":{"1min":0.38,"5min":0.38,"15min":0.34399999999999997},"max":{"1min":1.3779999999999999,"5min":10.842000000000001,"15min":10.842000000000001},"last":1.282}]},{"osd":7,"last update":"Sun Oct 1 10:38:33 2023","interfaces":[{"interface":"back","average":{"1min":0.95199999999999996,"5min":1.1080000000000001,"15min":1.095},"min":{"1min":0.41799999999999998,"5min":0.39700000000000002,"15min":0.36599999999999999},"max":{"1min":1.327,"5min":4.3499999999999996,"15min":7.5960000000000001},"last":1.2090000000000001},{"interface":"front","average":{"1min":1.02,"5min":1.278,"15min":1.3520000000000001},"min":{"1min":0.53000000000000003,"5min":0.42299999999999999,"15min":0.32700000000000001},"max":{"1min":1.405,"5min":5.3470000000000004,"15min":11.031000000000001},"last":1.3460000000000001}]}]},{"osd":1,"up_from":101,"seq":433791696981,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":251816,"kb_used_data":1312,"kb_used_omap":4,"kb_used_meta":250491,"kb_avail":93493336,"statfs":{"total":95995035648,"available":95737176064,"internally_reserved":0,"allocated":1343488,"data_stored":789975,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":256503694},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:38:43 2023","interfaces":[{"interface":"back","average":{"1min":0.68899999999999995,"5min":1.5669999999999999,"15min":2.1429999999999998},"min":{"1min":0.42999999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":1.002,"5min":11.071999999999999,"15min":11.071999999999999},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.64100000000000001,"5min":1.498,"15min":2.0880000000000001},"min":{"1min":0.47499999999999998,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.84799999999999998,"5min":7.9900000000000002,"15min":9.3230000000000004},"last":0.97699999999999998}]},{"osd":2,"last update":"Sun Oct 1 10:39:12 2023","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.94499999999999995,"5min":1.0589999999999999,"15min":1.0589999999999999},"last":0.84099999999999997},{"interface":"front","average":{"1min":0.73399999999999999,"5min":0.76000000000000001,"15min":0.76400000000000001},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.0309999999999999,"5min":1.1319999999999999,"15min":1.1319999999999999},"last":0.88600000000000001}]},{"osd":3,"last update":"Sun Oct 1 10:38:43 2023","interfaces":[{"interface":"back","average":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"min":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"max":{"1min":0.871,"5min":0.871,"15min":0.871},"last":0.629},{"interface":"front","average":{"1min":0.753,"5min":0.753,"15min":0.753},"min":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":0.93500000000000005,"5min":0.93500000000000005,"15min":0.93500000000000005},"last":0.60799999999999998}]},{"osd":4,"last update":"Sun Oct 1 10:39:18 2023","interfaces":[{"interface":"back","average":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88200000000000001},"min":{"1min":0.627,"5min":0.627,"15min":0.627},"max":{"1min":1.165,"5min":1.165,"15min":1.165},"last":0.92600000000000005},{"interface":"front","average":{"1min":1.008,"5min":1.008,"15min":1.008},"min":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"max":{"1min":1.268,"5min":1.268,"15min":1.268},"last":0.75700000000000001}]},{"osd":5,"last update":"Sun Oct 1 10:38:43 2023","interfaces":[{"interface":"back","average":{"1min":1.0089999999999999,"5min":1.482,"15min":1.8140000000000001},"min":{"1min":0.73799999999999999,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":1.3779999999999999,"5min":8.9429999999999996,"15min":8.9429999999999996},"last":0.73199999999999998},{"interface":"front","average":{"1min":1.034,"5min":1.8089999999999999,"15min":2.427},"min":{"1min":0.65200000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":1.399,"5min":9.1300000000000008,"15min":11.353999999999999},"last":1.012}]},{"osd":6,"last update":"Sun Oct 1 10:38:43 2023","interfaces":[{"interface":"back","average":{"1min":1.0369999999999999,"5min":1.542,"15min":2.0619999999999998},"min":{"1min":0.72099999999999997,"5min":0.37,"15min":0.37},"max":{"1min":1.369,"5min":8.6630000000000003,"15min":8.6630000000000003},"last":1.091},{"interface":"front","average":{"1min":1.0229999999999999,"5min":1.5660000000000001,"15min":2.024},"min":{"1min":0.72199999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":1.456,"5min":6,"15min":7.1369999999999996},"last":1.05}]},{"osd":7,"last update":"Sun Oct 1 10:38:43 2023","interfaces":[{"interface":"back","average":{"1min":1.0569999999999999,"5min":1.569,"15min":1.96},"min":{"1min":0.75700000000000001,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":1.363,"5min":6.0510000000000002,"15min":7.1680000000000001},"last":1.113},{"interface":"front","average":{"1min":1.0840000000000001,"5min":1.5940000000000001,"15min":1.7989999999999999},"min":{"1min":0.73399999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":1.286,"5min":9.0749999999999993,"15min":9.0749999999999993},"last":1.1459999999999999}]}]},{"osd":0,"up_from":86,"seq":369367187591,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":294004,"kb_used_data":1900,"kb_used_omap":5,"kb_used_meta":292090,"kb_avail":93451148,"statfs":{"total":95995035648,"available":95693975552,"internally_reserved":0,"allocated":1945600,"data_stored":1378971,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":299100794},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 10:38:43 2023","interfaces":[{"interface":"back","average":{"1min":0.68200000000000005,"5min":1.7649999999999999,"15min":1.7050000000000001},"min":{"1min":0.433,"5min":0.214,"15min":0.214},"max":{"1min":0.95899999999999996,"5min":12.855,"15min":12.855},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.68100000000000005,"5min":1.4359999999999999,"15min":2.5},"min":{"1min":0.35099999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.93100000000000005,"5min":8.6059999999999999,"15min":15.48},"last":0.88200000000000001}]},{"osd":2,"last update":"Sun Oct 1 10:39:07 2023","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.69099999999999995,"15min":0.69799999999999995},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":1.085,"5min":1.085,"15min":1.085},"last":1.141},{"interface":"front","average":{"1min":0.70999999999999996,"5min":0.74399999999999999,"15min":0.75},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":1.149,"5min":1.149,"15min":1.149},"last":0.625}]},{"osd":3,"last update":"Sun Oct 1 10:38:43 2023","interfaces":[{"interface":"back","average":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"min":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":1.3069999999999999,"5min":1.3069999999999999,"15min":1.3069999999999999},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"min":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"max":{"1min":1.1060000000000001,"5min":1.1060000000000001,"15min":1.1060000000000001},"last":0.95299999999999996}]},{"osd":4,"last update":"Sun Oct 1 10:39:18 2023","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":1.02,"15min":1.02},"min":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"max":{"1min":1.3200000000000001,"5min":1.3200000000000001,"15min":1.3200000000000001},"last":1.075},{"interface":"front","average":{"1min":1.0349999999999999,"5min":1.0349999999999999,"15min":1.0349999999999999},"min":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"max":{"1min":1.5029999999999999,"5min":1.5029999999999999,"15min":1.5029999999999999},"last":1.107}]},{"osd":5,"last update":"Sun Oct 1 10:38:43 2023","interfaces":[{"interface":"back","average":{"1min":1.0349999999999999,"5min":1.587,"15min":1.786},"min":{"1min":0.75700000000000001,"5min":0.32000000000000001,"15min":0.31900000000000001},"max":{"1min":1.369,"5min":13.068,"15min":13.068},"last":0.92400000000000004},{"interface":"front","average":{"1min":1.097,"5min":1.7490000000000001,"15min":1.7809999999999999},"min":{"1min":0.79800000000000004,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":1.5649999999999999,"5min":11.747,"15min":11.747},"last":1.25}]},{"osd":6,"last update":"Sun Oct 1 10:38:43 2023","interfaces":[{"interface":"back","average":{"1min":1.0720000000000001,"5min":2.0339999999999998,"15min":1.8859999999999999},"min":{"1min":0.59199999999999997,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":1.4910000000000001,"5min":11.663,"15min":11.663},"last":1.321},{"interface":"front","average":{"1min":1.0609999999999999,"5min":1.8220000000000001,"15min":1.831},"min":{"1min":0.70699999999999996,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":1.653,"5min":15.069000000000001,"15min":15.069000000000001},"last":1.2789999999999999}]},{"osd":7,"last update":"Sun Oct 1 10:38:43 2023","interfaces":[{"interface":"back","average":{"1min":1.1220000000000001,"5min":1.867,"15min":1.952},"min":{"1min":0.76000000000000001,"5min":0.40300000000000002,"15min":0.39000000000000001},"max":{"1min":1.613,"5min":12.686,"15min":12.686},"last":1.171},{"interface":"front","average":{"1min":1.119,"5min":1.625,"15min":1.657},"min":{"1min":0.65600000000000003,"5min":0.42199999999999999,"15min":0.36199999999999999},"max":{"1min":1.732,"5min":8.6829999999999998,"15min":13.654},"last":1.1919999999999999}]}]},{"osd":2,"up_from":116,"seq":498216206366,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":155752,"kb_used_data":1248,"kb_used_omap":7,"kb_used_meta":154488,"kb_avail":93589400,"statfs":{"total":95995035648,"available":95835545600,"internally_reserved":0,"allocated":1277952,"data_stored":778803,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":158195793},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:39:07 2023","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":0.66800000000000004,"15min":0.67400000000000004},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":1.002,"5min":1.0660000000000001,"15min":1.0660000000000001},"last":0.89400000000000002},{"interface":"front","average":{"1min":0.73399999999999999,"5min":0.78200000000000003,"15min":0.79000000000000004},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":1.0760000000000001,"5min":1.1140000000000001,"15min":1.1140000000000001},"last":0.77100000000000002}]},{"osd":1,"last update":"Sun Oct 1 10:39:07 2023","interfaces":[{"interface":"back","average":{"1min":0.73799999999999999,"5min":0.71699999999999997,"15min":0.71299999999999997},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":1.087,"5min":1.087,"15min":1.087},"last":0.84299999999999997},{"interface":"front","average":{"1min":0.67200000000000004,"5min":0.77500000000000002,"15min":0.79200000000000004},"min":{"1min":0.36499999999999999,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.91900000000000004,"5min":1.1479999999999999,"15min":1.1479999999999999},"last":0.72599999999999998}]},{"osd":3,"last update":"Sun Oct 1 10:38:44 2023","interfaces":[{"interface":"back","average":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.94399999999999995,"5min":0.94399999999999995,"15min":0.94399999999999995},"last":0.64200000000000002},{"interface":"front","average":{"1min":0.88700000000000001,"5min":0.88700000000000001,"15min":0.88700000000000001},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":1.2,"5min":1.2,"15min":1.2},"last":1.01}]},{"osd":4,"last update":"Sun Oct 1 10:39:18 2023","interfaces":[{"interface":"back","average":{"1min":0.98099999999999998,"5min":0.98099999999999998,"15min":0.98099999999999998},"min":{"1min":0.745,"5min":0.745,"15min":0.745},"max":{"1min":1.28,"5min":1.28,"15min":1.28},"last":1.238},{"interface":"front","average":{"1min":0.95699999999999996,"5min":0.95699999999999996,"15min":0.95699999999999996},"min":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"max":{"1min":1.254,"5min":1.254,"15min":1.254},"last":1.1679999999999999}]},{"osd":5,"last update":"Sun Oct 1 10:39:07 2023","interfaces":[{"interface":"back","average":{"1min":1.032,"5min":1.1040000000000001,"15min":1.1160000000000001},"min":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"max":{"1min":1.468,"5min":1.6339999999999999,"15min":1.6339999999999999},"last":1.1299999999999999},{"interface":"front","average":{"1min":1.0329999999999999,"5min":1.02,"15min":1.018},"min":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":1.3129999999999999,"5min":1.585,"15min":1.585},"last":1.079}]},{"osd":6,"last update":"Sun Oct 1 10:39:07 2023","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":1.0840000000000001,"15min":1.095},"min":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":1.276,"5min":1.486,"15min":1.486},"last":1.266},{"interface":"front","average":{"1min":1.1279999999999999,"5min":1.1419999999999999,"15min":1.1439999999999999},"min":{"1min":0.84199999999999997,"5min":0.65200000000000002,"15min":0.65200000000000002},"max":{"1min":1.3799999999999999,"5min":1.6639999999999999,"15min":1.6639999999999999},"last":1.2010000000000001}]},{"osd":7,"last update":"Sun Oct 1 10:39:07 2023","interfaces":[{"interface":"back","average":{"1min":1.1020000000000001,"5min":1.1259999999999999,"15min":1.1299999999999999},"min":{"1min":0.79500000000000004,"5min":0.749,"15min":0.749},"max":{"1min":1.444,"5min":1.536,"15min":1.536},"last":1.097},{"interface":"front","average":{"1min":1.129,"5min":1.206,"15min":1.2190000000000001},"min":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"max":{"1min":1.7390000000000001,"5min":1.7390000000000001,"15min":1.7390000000000001},"last":1.3009999999999999}]}]},{"osd":3,"up_from":121,"seq":519691042839,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":103128,"kb_used_data":1232,"kb_used_omap":2,"kb_used_meta":101885,"kb_avail":93642024,"statfs":{"total":95995035648,"available":95889432576,"internally_reserved":0,"allocated":1261568,"data_stored":776896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":104330293},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:38:46 2023","interfaces":[{"interface":"back","average":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.96199999999999997,"5min":0.96199999999999997,"15min":0.96199999999999997},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.63800000000000001}]},{"osd":1,"last update":"Sun Oct 1 10:38:46 2023","interfaces":[{"interface":"back","average":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"min":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"max":{"1min":1.2370000000000001,"5min":1.2370000000000001,"15min":1.2370000000000001},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"min":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"max":{"1min":1.1659999999999999,"5min":1.1659999999999999,"15min":1.1659999999999999},"last":0.76100000000000001}]},{"osd":2,"last update":"Sun Oct 1 10:38:46 2023","interfaces":[{"interface":"back","average":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"min":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"max":{"1min":1.046,"5min":1.046,"15min":1.046},"last":0.70899999999999996}]},{"osd":4,"last update":"Sun Oct 1 10:39:15 2023","interfaces":[{"interface":"back","average":{"1min":0.94099999999999995,"5min":0.94099999999999995,"15min":0.94099999999999995},"min":{"1min":0.623,"5min":0.623,"15min":0.623},"max":{"1min":1.306,"5min":1.306,"15min":1.306},"last":0.91400000000000003},{"interface":"front","average":{"1min":0.92800000000000005,"5min":0.92800000000000005,"15min":0.92800000000000005},"min":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"max":{"1min":1.278,"5min":1.278,"15min":1.278},"last":0.93500000000000005}]},{"osd":5,"last update":"Sun Oct 1 10:38:46 2023","interfaces":[{"interface":"back","average":{"1min":0.94399999999999995,"5min":0.94399999999999995,"15min":0.94399999999999995},"min":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"max":{"1min":1.248,"5min":1.248,"15min":1.248},"last":0.77700000000000002},{"interface":"front","average":{"1min":1.006,"5min":1.006,"15min":1.006},"min":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"max":{"1min":1.4199999999999999,"5min":1.4199999999999999,"15min":1.4199999999999999},"last":0.97199999999999998}]},{"osd":6,"last update":"Sun Oct 1 10:38:46 2023","interfaces":[{"interface":"back","average":{"1min":1.0109999999999999,"5min":1.0109999999999999,"15min":1.0109999999999999},"min":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"max":{"1min":1.325,"5min":1.325,"15min":1.325},"last":1.0089999999999999},{"interface":"front","average":{"1min":1.1060000000000001,"5min":1.1060000000000001,"15min":1.1060000000000001},"min":{"1min":0.89000000000000001,"5min":0.89000000000000001,"15min":0.89000000000000001},"max":{"1min":1.377,"5min":1.377,"15min":1.377},"last":1.095}]},{"osd":7,"last update":"Sun Oct 1 10:38:46 2023","interfaces":[{"interface":"back","average":{"1min":1.0600000000000001,"5min":1.0600000000000001,"15min":1.0600000000000001},"min":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"max":{"1min":1.411,"5min":1.411,"15min":1.411},"last":0.81000000000000005},{"interface":"front","average":{"1min":1.018,"5min":1.018,"15min":1.018},"min":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"max":{"1min":1.3360000000000001,"5min":1.3360000000000001,"15min":1.3360000000000001},"last":1.0609999999999999}]}]},{"osd":4,"up_from":126,"seq":541165879312,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":138584,"kb_used_data":1232,"kb_used_omap":10,"kb_used_meta":137333,"kb_avail":93606568,"statfs":{"total":95995035648,"available":95853125632,"internally_reserved":0,"allocated":1261568,"data_stored":777243,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":140629604},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:39:16 2023","interfaces":[{"interface":"back","average":{"1min":0.90500000000000003,"5min":0.90500000000000003,"15min":0.90500000000000003},"min":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"max":{"1min":1.3009999999999999,"5min":1.3009999999999999,"15min":1.3009999999999999},"last":1.1919999999999999},{"interface":"front","average":{"1min":0.98999999999999999,"5min":0.98999999999999999,"15min":0.98999999999999999},"min":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"max":{"1min":1.4279999999999999,"5min":1.4279999999999999,"15min":1.4279999999999999},"last":0.87}]},{"osd":1,"last update":"Sun Oct 1 10:39:16 2023","interfaces":[{"interface":"back","average":{"1min":0.92700000000000005,"5min":0.92700000000000005,"15min":0.92700000000000005},"min":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"max":{"1min":1.1850000000000001,"5min":1.1850000000000001,"15min":1.1850000000000001},"last":1.05},{"interface":"front","average":{"1min":0.95399999999999996,"5min":0.95399999999999996,"15min":0.95399999999999996},"min":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"max":{"1min":1.2430000000000001,"5min":1.2430000000000001,"15min":1.2430000000000001},"last":0.89900000000000002}]},{"osd":2,"last update":"Sun Oct 1 10:39:16 2023","interfaces":[{"interface":"back","average":{"1min":0.95399999999999996,"5min":0.95399999999999996,"15min":0.95399999999999996},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":1.1970000000000001,"5min":1.1970000000000001,"15min":1.1970000000000001},"last":0.98899999999999999},{"interface":"front","average":{"1min":0.97699999999999998,"5min":0.97699999999999998,"15min":0.97699999999999998},"min":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"max":{"1min":1.222,"5min":1.222,"15min":1.222},"last":0.93400000000000005}]},{"osd":3,"last update":"Sun Oct 1 10:39:16 2023","interfaces":[{"interface":"back","average":{"1min":1.0780000000000001,"5min":1.0780000000000001,"15min":1.0780000000000001},"min":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"max":{"1min":1.3320000000000001,"5min":1.3320000000000001,"15min":1.3320000000000001},"last":1.1060000000000001},{"interface":"front","average":{"1min":0.97899999999999998,"5min":0.97899999999999998,"15min":0.97899999999999998},"min":{"1min":0.628,"5min":0.628,"15min":0.628},"max":{"1min":1.296,"5min":1.296,"15min":1.296},"last":0.95899999999999996}]},{"osd":5,"last update":"Sun Oct 1 10:39:16 2023","interfaces":[{"interface":"back","average":{"1min":1.095,"5min":1.095,"15min":1.095},"min":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"max":{"1min":1.5089999999999999,"5min":1.5089999999999999,"15min":1.5089999999999999},"last":0.77900000000000003},{"interface":"front","average":{"1min":1.0940000000000001,"5min":1.0940000000000001,"15min":1.0940000000000001},"min":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"max":{"1min":1.3779999999999999,"5min":1.3779999999999999,"15min":1.3779999999999999},"last":1.081}]},{"osd":6,"last update":"Sun Oct 1 10:39:16 2023","interfaces":[{"interface":"back","average":{"1min":1.139,"5min":1.139,"15min":1.139},"min":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"max":{"1min":1.579,"5min":1.579,"15min":1.579},"last":1.0229999999999999},{"interface":"front","average":{"1min":1.075,"5min":1.075,"15min":1.075},"min":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"max":{"1min":1.403,"5min":1.403,"15min":1.403},"last":1.1339999999999999}]},{"osd":7,"last update":"Sun Oct 1 10:39:16 2023","interfaces":[{"interface":"back","average":{"1min":1.1819999999999999,"5min":1.1819999999999999,"15min":1.1819999999999999},"min":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"max":{"1min":1.387,"5min":1.387,"15min":1.387},"last":1.2110000000000001},{"interface":"front","average":{"1min":1.169,"5min":1.169,"15min":1.169},"min":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"max":{"1min":1.46,"5min":1.46,"15min":1.46},"last":1.151}]}]},{"osd":5,"up_from":36,"seq":154618822993,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":594992,"kb_used_data":1192,"kb_used_omap":0,"kb_used_meta":593792,"kb_avail":93150160,"statfs":{"total":95995035648,"available":95385763840,"internally_reserved":0,"allocated":1220608,"data_stored":768137,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":608043008},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:38:37 2023","interfaces":[{"interface":"back","average":{"1min":0.86799999999999999,"5min":1.3160000000000001,"15min":1.2949999999999999},"min":{"1min":0.46300000000000002,"5min":0.27300000000000002,"15min":0.26400000000000001},"max":{"1min":1.175,"5min":10.429,"15min":10.429},"last":0.745},{"interface":"front","average":{"1min":0.90900000000000003,"5min":1.54,"15min":1.524},"min":{"1min":0.59399999999999997,"5min":0.29299999999999998,"15min":0.29199999999999998},"max":{"1min":1.2809999999999999,"5min":13.018000000000001,"15min":13.018000000000001},"last":1.117}]},{"osd":1,"last update":"Sun Oct 1 10:38:37 2023","interfaces":[{"interface":"back","average":{"1min":0.874,"5min":1.768,"15min":1.72},"min":{"1min":0.63,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":1.149,"5min":10.858000000000001,"15min":10.858000000000001},"last":0.84899999999999998},{"interface":"front","average":{"1min":0.96499999999999997,"5min":2.0310000000000001,"15min":1.8109999999999999},"min":{"1min":0.623,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.4159999999999999,"5min":12.864000000000001,"15min":12.864000000000001},"last":0.96099999999999997}]},{"osd":2,"last update":"Sun Oct 1 10:39:06 2023","interfaces":[{"interface":"back","average":{"1min":0.88100000000000001,"5min":0.96299999999999997,"15min":0.97699999999999998},"min":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"max":{"1min":1.272,"5min":1.3540000000000001,"15min":1.3540000000000001},"last":1.1739999999999999},{"interface":"front","average":{"1min":0.97399999999999998,"5min":0.96599999999999997,"15min":0.96399999999999997},"min":{"1min":0.751,"5min":0.67600000000000005,"15min":0.67600000000000005},"max":{"1min":1.1879999999999999,"5min":1.272,"15min":1.272},"last":0.77700000000000002}]},{"osd":3,"last update":"Sun Oct 1 10:38:46 2023","interfaces":[{"interface":"back","average":{"1min":0.93300000000000005,"5min":0.93300000000000005,"15min":0.93300000000000005},"min":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"max":{"1min":1.3919999999999999,"5min":1.3919999999999999,"15min":1.3919999999999999},"last":0.67200000000000004},{"interface":"front","average":{"1min":0.93999999999999995,"5min":0.93999999999999995,"15min":0.93999999999999995},"min":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"max":{"1min":1.1879999999999999,"5min":1.1879999999999999,"15min":1.1879999999999999},"last":1.0580000000000001}]},{"osd":4,"last update":"Sun Oct 1 10:39:16 2023","interfaces":[{"interface":"back","average":{"1min":1.077,"5min":1.077,"15min":1.077},"min":{"1min":0.753,"5min":0.753,"15min":0.753},"max":{"1min":1.343,"5min":1.343,"15min":1.343},"last":1.1519999999999999},{"interface":"front","average":{"1min":1.0840000000000001,"5min":1.0840000000000001,"15min":1.0840000000000001},"min":{"1min":0.89700000000000002,"5min":0.89700000000000002,"15min":0.89700000000000002},"max":{"1min":1.2989999999999999,"5min":1.2989999999999999,"15min":1.2989999999999999},"last":0.94199999999999995}]},{"osd":6,"last update":"Sun Oct 1 10:38:52 2023","interfaces":[{"interface":"back","average":{"1min":0.96599999999999997,"5min":1.298,"15min":1.298},"min":{"1min":0.42399999999999999,"5min":0.38500000000000001,"15min":0.33900000000000002},"max":{"1min":1.496,"5min":6.96,"15min":6.96},"last":1.02},{"interface":"front","average":{"1min":0.90000000000000002,"5min":1.325,"15min":1.2330000000000001},"min":{"1min":0.437,"5min":0.312,"15min":0.312},"max":{"1min":1.377,"5min":8.4339999999999993,"15min":8.4339999999999993},"last":1.2729999999999999}]},{"osd":7,"last update":"Sun Oct 1 10:38:52 2023","interfaces":[{"interface":"back","average":{"1min":0.94799999999999995,"5min":1.4199999999999999,"15min":1.292},"min":{"1min":0.39000000000000001,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.3480000000000001,"5min":9.4629999999999992,"15min":9.4629999999999992},"last":1.1990000000000001},{"interface":"front","average":{"1min":1.0109999999999999,"5min":1.1899999999999999,"15min":1.3919999999999999},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":1.4279999999999999,"5min":5.4699999999999998,"15min":12.927},"last":1.1000000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T10:39:28.848 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T10:39:29.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:39:29 smithi078 bash[75299]: audit 2023-10-01T10:39:27.898514+0000 mgr.y (mgr.25072) 647 : audit 0 from='client.35562 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:39:29.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:39:29 smithi078 bash[78274]: audit 2023-10-01T10:39:27.898514+0000 mgr.y (mgr.25072) 647 : audit 0 from='client.35562 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:39:29.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:39:29 smithi173 bash[58816]: audit 2023-10-01T10:39:27.898514+0000 mgr.y (mgr.25072) 647 : audit 0 from='client.35562 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:39:30.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:39:30 smithi078 bash[75299]: cluster 2023-10-01T10:39:29.675959+0000 mgr.y (mgr.25072) 648 : cluster 0 pgmap v555: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:30.332 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:39:30 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:39:30] "GET /metrics HTTP/1.1" 200 33821 "" "Prometheus/2.33.4" 2023-10-01T10:39:30.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:39:30 smithi078 bash[78274]: cluster 2023-10-01T10:39:29.675959+0000 mgr.y (mgr.25072) 648 : cluster 0 pgmap v555: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:30.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:39:30 smithi173 bash[58816]: cluster 2023-10-01T10:39:29.675959+0000 mgr.y (mgr.25072) 648 : cluster 0 pgmap v555: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:32.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:39:31 smithi078 bash[75299]: cluster 2023-10-01T10:39:31.676816+0000 mgr.y (mgr.25072) 649 : cluster 0 pgmap v556: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:32.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:39:31 smithi078 bash[78274]: cluster 2023-10-01T10:39:31.676816+0000 mgr.y (mgr.25072) 649 : cluster 0 pgmap v556: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:32.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:39:31 smithi173 bash[58816]: cluster 2023-10-01T10:39:31.676816+0000 mgr.y (mgr.25072) 649 : cluster 0 pgmap v556: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:33.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:39:33 smithi078 bash[75299]: audit 2023-10-01T10:39:32.409994+0000 mon.a (mon.0) 896 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:39:33.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:39:33 smithi078 bash[78274]: audit 2023-10-01T10:39:32.409994+0000 mon.a (mon.0) 896 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:39:33.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:39:33 smithi173 bash[58816]: audit 2023-10-01T10:39:32.409994+0000 mon.a (mon.0) 896 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:39:33.665 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:39:34.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:39:34 smithi078 bash[78274]: cluster 2023-10-01T10:39:33.677509+0000 mgr.y (mgr.25072) 650 : cluster 0 pgmap v557: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:34.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:39:34 smithi078 bash[75299]: cluster 2023-10-01T10:39:33.677509+0000 mgr.y (mgr.25072) 650 : cluster 0 pgmap v557: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:34.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:39:34 smithi173 bash[58816]: cluster 2023-10-01T10:39:33.677509+0000 mgr.y (mgr.25072) 650 : cluster 0 pgmap v557: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:36.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:39:35 smithi078 bash[75299]: cluster 2023-10-01T10:39:35.678671+0000 mgr.y (mgr.25072) 651 : cluster 0 pgmap v558: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:36.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:39:35 smithi078 bash[78274]: cluster 2023-10-01T10:39:35.678671+0000 mgr.y (mgr.25072) 651 : cluster 0 pgmap v558: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:36.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:39:35 smithi173 bash[58816]: cluster 2023-10-01T10:39:35.678671+0000 mgr.y (mgr.25072) 651 : cluster 0 pgmap v558: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:37.115 INFO:teuthology.orchestra.run.smithi078.stdout: 2023-10-01T10:39:37.120 INFO:teuthology.orchestra.run.smithi078.stderr:dumped all 2023-10-01T10:39:37.917 INFO:teuthology.orchestra.run.smithi078.stdout:{"pg_ready":true,"pg_map":{"version":558,"stamp":"2023-10-01T10:39:35.677794+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":261,"num_write_kb":2446,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":195,"ondisk_log_size":195,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2148588,"kb_used_data":11692,"kb_used_omap":30,"kb_used_meta":2136801,"kb_avail":747812628,"statfs":{"total":767960285184,"available":765760131072,"internally_reserved":0,"allocated":11972608,"data_stored":7995856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31502,"internal_metadata":2188084466},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004623"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":113,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.032258+0000","last_change":"2023-10-01T10:38:16.032258+0000","last_active":"2023-10-01T10:38:16.032258+0000","last_peered":"2023-10-01T10:38:16.032258+0000","last_clean":"2023-10-01T10:38:16.032258+0000","last_became_active":"2023-10-01T10:38:16.031633+0000","last_became_peered":"2023-10-01T10:38:16.031633+0000","last_unstale":"2023-10-01T10:38:16.032258+0000","last_undegraded":"2023-10-01T10:38:16.032258+0000","last_fullsized":"2023-10-01T10:38:16.032258+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:17:02.446430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":101,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.025881+0000","last_change":"2023-10-01T10:38:16.025881+0000","last_active":"2023-10-01T10:38:16.025881+0000","last_peered":"2023-10-01T10:38:16.025881+0000","last_clean":"2023-10-01T10:38:16.025881+0000","last_became_active":"2023-10-01T10:38:16.025518+0000","last_became_peered":"2023-10-01T10:38:16.025518+0000","last_unstale":"2023-10-01T10:38:16.025881+0000","last_undegraded":"2023-10-01T10:38:16.025881+0000","last_fullsized":"2023-10-01T10:38:16.025881+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:48:37.826604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":79,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.025075+0000","last_change":"2023-10-01T10:38:16.025075+0000","last_active":"2023-10-01T10:38:16.025075+0000","last_peered":"2023-10-01T10:38:16.025075+0000","last_clean":"2023-10-01T10:38:16.025075+0000","last_became_active":"2023-10-01T10:38:16.024735+0000","last_became_peered":"2023-10-01T10:38:16.024735+0000","last_unstale":"2023-10-01T10:38:16.025075+0000","last_undegraded":"2023-10-01T10:38:16.025075+0000","last_fullsized":"2023-10-01T10:38:16.025075+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:38:01.450741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":83,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.019448+0000","last_change":"2023-10-01T10:37:42.772008+0000","last_active":"2023-10-01T10:38:16.019448+0000","last_peered":"2023-10-01T10:38:16.019448+0000","last_clean":"2023-10-01T10:38:16.019448+0000","last_became_active":"2023-10-01T10:37:42.771244+0000","last_became_peered":"2023-10-01T10:37:42.771244+0000","last_unstale":"2023-10-01T10:38:16.019448+0000","last_undegraded":"2023-10-01T10:38:16.019448+0000","last_fullsized":"2023-10-01T10:38:16.019448+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:38:55.180790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":102,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.010782+0000","last_change":"2023-10-01T10:37:42.766600+0000","last_active":"2023-10-01T10:38:15.010782+0000","last_peered":"2023-10-01T10:38:15.010782+0000","last_clean":"2023-10-01T10:38:15.010782+0000","last_became_active":"2023-10-01T10:37:42.766051+0000","last_became_peered":"2023-10-01T10:37:42.766051+0000","last_unstale":"2023-10-01T10:38:15.010782+0000","last_undegraded":"2023-10-01T10:38:15.010782+0000","last_fullsized":"2023-10-01T10:38:15.010782+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:51:17.707963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":92,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.031873+0000","last_change":"2023-10-01T10:32:31.575109+0000","last_active":"2023-10-01T10:38:15.031873+0000","last_peered":"2023-10-01T10:38:15.031873+0000","last_clean":"2023-10-01T10:38:15.031873+0000","last_became_active":"2023-10-01T10:32:31.574729+0000","last_became_peered":"2023-10-01T10:32:31.574729+0000","last_unstale":"2023-10-01T10:38:15.031873+0000","last_undegraded":"2023-10-01T10:38:15.031873+0000","last_fullsized":"2023-10-01T10:38:15.031873+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":81,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.014432+0000","last_change":"2023-10-01T10:37:42.766667+0000","last_active":"2023-10-01T10:38:15.014432+0000","last_peered":"2023-10-01T10:38:15.014432+0000","last_clean":"2023-10-01T10:38:15.014432+0000","last_became_active":"2023-10-01T10:37:42.766369+0000","last_became_peered":"2023-10-01T10:37:42.766369+0000","last_unstale":"2023-10-01T10:38:15.014432+0000","last_undegraded":"2023-10-01T10:38:15.014432+0000","last_fullsized":"2023-10-01T10:38:15.014432+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:02.708788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":86,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.010584+0000","last_change":"2023-10-01T10:37:42.765925+0000","last_active":"2023-10-01T10:38:15.010584+0000","last_peered":"2023-10-01T10:38:15.010584+0000","last_clean":"2023-10-01T10:38:15.010584+0000","last_became_active":"2023-10-01T10:37:42.765218+0000","last_became_peered":"2023-10-01T10:37:42.765218+0000","last_unstale":"2023-10-01T10:38:15.010584+0000","last_undegraded":"2023-10-01T10:38:15.010584+0000","last_fullsized":"2023-10-01T10:38:15.010584+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T22:10:34.338031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":94,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.034622+0000","last_change":"2023-10-01T10:32:31.579805+0000","last_active":"2023-10-01T10:38:15.034622+0000","last_peered":"2023-10-01T10:38:15.034622+0000","last_clean":"2023-10-01T10:38:15.034622+0000","last_became_active":"2023-10-01T10:32:31.579423+0000","last_became_peered":"2023-10-01T10:32:31.579423+0000","last_unstale":"2023-10-01T10:38:15.034622+0000","last_undegraded":"2023-10-01T10:38:15.034622+0000","last_fullsized":"2023-10-01T10:38:15.034622+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":70,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.014763+0000","last_change":"2023-10-01T10:37:42.772531+0000","last_active":"2023-10-01T10:38:15.014763+0000","last_peered":"2023-10-01T10:38:15.014763+0000","last_clean":"2023-10-01T10:38:15.014763+0000","last_became_active":"2023-10-01T10:37:42.772216+0000","last_became_peered":"2023-10-01T10:37:42.772216+0000","last_unstale":"2023-10-01T10:38:15.014763+0000","last_undegraded":"2023-10-01T10:38:15.014763+0000","last_fullsized":"2023-10-01T10:38:15.014763+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:01:53.373628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":112,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.027085+0000","last_change":"2023-10-01T10:38:16.027085+0000","last_active":"2023-10-01T10:38:16.027085+0000","last_peered":"2023-10-01T10:38:16.027085+0000","last_clean":"2023-10-01T10:38:16.027085+0000","last_became_active":"2023-10-01T10:38:16.026555+0000","last_became_peered":"2023-10-01T10:38:16.026555+0000","last_unstale":"2023-10-01T10:38:16.027085+0000","last_undegraded":"2023-10-01T10:38:16.027085+0000","last_fullsized":"2023-10-01T10:38:16.027085+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":65,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.020849+0000","last_change":"2023-10-01T10:37:05.018755+0000","last_active":"2023-10-01T10:38:16.020849+0000","last_peered":"2023-10-01T10:38:16.020849+0000","last_clean":"2023-10-01T10:38:16.020849+0000","last_became_active":"2023-10-01T10:37:05.018446+0000","last_became_peered":"2023-10-01T10:37:05.018446+0000","last_unstale":"2023-10-01T10:38:16.020849+0000","last_undegraded":"2023-10-01T10:38:16.020849+0000","last_fullsized":"2023-10-01T10:38:16.020849+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:13:53.740503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":107,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.030826+0000","last_change":"2023-10-01T10:38:16.030826+0000","last_active":"2023-10-01T10:38:16.030826+0000","last_peered":"2023-10-01T10:38:16.030826+0000","last_clean":"2023-10-01T10:38:16.030826+0000","last_became_active":"2023-10-01T10:38:16.030277+0000","last_became_peered":"2023-10-01T10:38:16.030277+0000","last_unstale":"2023-10-01T10:38:16.030826+0000","last_undegraded":"2023-10-01T10:38:16.030826+0000","last_fullsized":"2023-10-01T10:38:16.030826+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:31:43.040424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":81,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.034741+0000","last_change":"2023-10-01T10:38:16.034741+0000","last_active":"2023-10-01T10:38:16.034741+0000","last_peered":"2023-10-01T10:38:16.034741+0000","last_clean":"2023-10-01T10:38:16.034741+0000","last_became_active":"2023-10-01T10:38:16.034407+0000","last_became_peered":"2023-10-01T10:38:16.034407+0000","last_unstale":"2023-10-01T10:38:16.034741+0000","last_undegraded":"2023-10-01T10:38:16.034741+0000","last_fullsized":"2023-10-01T10:38:16.034741+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:47:29.911477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":111,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.026777+0000","last_change":"2023-10-01T10:38:16.026777+0000","last_active":"2023-10-01T10:38:16.026777+0000","last_peered":"2023-10-01T10:38:16.026777+0000","last_clean":"2023-10-01T10:38:16.026777+0000","last_became_active":"2023-10-01T10:38:16.026371+0000","last_became_peered":"2023-10-01T10:38:16.026371+0000","last_unstale":"2023-10-01T10:38:16.026777+0000","last_undegraded":"2023-10-01T10:38:16.026777+0000","last_fullsized":"2023-10-01T10:38:16.026777+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":109,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.012172+0000","last_change":"2023-10-01T10:37:42.772319+0000","last_active":"2023-10-01T10:38:15.012172+0000","last_peered":"2023-10-01T10:38:15.012172+0000","last_clean":"2023-10-01T10:38:15.012172+0000","last_became_active":"2023-10-01T10:37:42.771977+0000","last_became_peered":"2023-10-01T10:37:42.771977+0000","last_unstale":"2023-10-01T10:38:15.012172+0000","last_undegraded":"2023-10-01T10:38:15.012172+0000","last_fullsized":"2023-10-01T10:38:15.012172+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:45:12.783214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":81,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.014954+0000","last_change":"2023-10-01T10:37:42.767706+0000","last_active":"2023-10-01T10:38:15.014954+0000","last_peered":"2023-10-01T10:38:15.014954+0000","last_clean":"2023-10-01T10:38:15.014954+0000","last_became_active":"2023-10-01T10:37:42.767211+0000","last_became_peered":"2023-10-01T10:37:42.767211+0000","last_unstale":"2023-10-01T10:38:15.014954+0000","last_undegraded":"2023-10-01T10:38:15.014954+0000","last_fullsized":"2023-10-01T10:38:15.014954+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:37:20.357407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":93,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.034746+0000","last_change":"2023-10-01T10:37:42.768642+0000","last_active":"2023-10-01T10:38:15.034746+0000","last_peered":"2023-10-01T10:38:15.034746+0000","last_clean":"2023-10-01T10:38:15.034746+0000","last_became_active":"2023-10-01T10:37:42.768239+0000","last_became_peered":"2023-10-01T10:37:42.768239+0000","last_unstale":"2023-10-01T10:38:15.034746+0000","last_undegraded":"2023-10-01T10:38:15.034746+0000","last_fullsized":"2023-10-01T10:38:15.034746+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":94,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.028563+0000","last_change":"2023-10-01T10:38:16.028563+0000","last_active":"2023-10-01T10:38:16.028563+0000","last_peered":"2023-10-01T10:38:16.028563+0000","last_clean":"2023-10-01T10:38:16.028563+0000","last_became_active":"2023-10-01T10:38:16.028162+0000","last_became_peered":"2023-10-01T10:38:16.028162+0000","last_unstale":"2023-10-01T10:38:16.028563+0000","last_undegraded":"2023-10-01T10:38:16.028563+0000","last_fullsized":"2023-10-01T10:38:16.028563+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":82,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.014770+0000","last_change":"2023-10-01T10:37:42.772247+0000","last_active":"2023-10-01T10:38:15.014770+0000","last_peered":"2023-10-01T10:38:15.014770+0000","last_clean":"2023-10-01T10:38:15.014770+0000","last_became_active":"2023-10-01T10:37:42.771697+0000","last_became_peered":"2023-10-01T10:37:42.771697+0000","last_unstale":"2023-10-01T10:38:15.014770+0000","last_undegraded":"2023-10-01T10:38:15.014770+0000","last_fullsized":"2023-10-01T10:38:15.014770+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:23:15.018248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":101,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.030438+0000","last_change":"2023-10-01T10:28:20.211798+0000","last_active":"2023-10-01T10:38:15.030438+0000","last_peered":"2023-10-01T10:38:15.030438+0000","last_clean":"2023-10-01T10:38:15.030438+0000","last_became_active":"2023-10-01T10:28:20.210791+0000","last_became_peered":"2023-10-01T10:28:20.210791+0000","last_unstale":"2023-10-01T10:38:15.030438+0000","last_undegraded":"2023-10-01T10:38:15.030438+0000","last_fullsized":"2023-10-01T10:38:15.030438+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":86,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.012486+0000","last_change":"2023-10-01T10:37:42.773127+0000","last_active":"2023-10-01T10:38:15.012486+0000","last_peered":"2023-10-01T10:38:15.012486+0000","last_clean":"2023-10-01T10:38:15.012486+0000","last_became_active":"2023-10-01T10:37:42.772102+0000","last_became_peered":"2023-10-01T10:37:42.772102+0000","last_unstale":"2023-10-01T10:38:15.012486+0000","last_undegraded":"2023-10-01T10:38:15.012486+0000","last_fullsized":"2023-10-01T10:38:15.012486+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:23:26.202295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":110,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.026955+0000","last_change":"2023-10-01T10:38:16.026955+0000","last_active":"2023-10-01T10:38:16.026955+0000","last_peered":"2023-10-01T10:38:16.026955+0000","last_clean":"2023-10-01T10:38:16.026955+0000","last_became_active":"2023-10-01T10:38:16.026552+0000","last_became_peered":"2023-10-01T10:38:16.026552+0000","last_unstale":"2023-10-01T10:38:16.026955+0000","last_undegraded":"2023-10-01T10:38:16.026955+0000","last_fullsized":"2023-10-01T10:38:16.026955+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":78,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.031063+0000","last_change":"2023-10-01T10:15:40.318702+0000","last_active":"2023-10-01T10:38:15.031063+0000","last_peered":"2023-10-01T10:38:15.031063+0000","last_clean":"2023-10-01T10:38:15.031063+0000","last_became_active":"2023-10-01T10:15:40.318333+0000","last_became_peered":"2023-10-01T10:15:40.318333+0000","last_unstale":"2023-10-01T10:38:15.031063+0000","last_undegraded":"2023-10-01T10:38:15.031063+0000","last_fullsized":"2023-10-01T10:38:15.031063+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":111,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.027753+0000","last_change":"2023-10-01T10:38:16.027753+0000","last_active":"2023-10-01T10:38:16.027753+0000","last_peered":"2023-10-01T10:38:16.027753+0000","last_clean":"2023-10-01T10:38:16.027753+0000","last_became_active":"2023-10-01T10:38:16.027384+0000","last_became_peered":"2023-10-01T10:38:16.027384+0000","last_unstale":"2023-10-01T10:38:16.027753+0000","last_undegraded":"2023-10-01T10:38:16.027753+0000","last_fullsized":"2023-10-01T10:38:16.027753+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":115,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.033203+0000","last_change":"2023-10-01T10:38:16.033203+0000","last_active":"2023-10-01T10:38:16.033203+0000","last_peered":"2023-10-01T10:38:16.033203+0000","last_clean":"2023-10-01T10:38:16.033203+0000","last_became_active":"2023-10-01T10:38:16.032681+0000","last_became_peered":"2023-10-01T10:38:16.032681+0000","last_unstale":"2023-10-01T10:38:16.033203+0000","last_undegraded":"2023-10-01T10:38:16.033203+0000","last_fullsized":"2023-10-01T10:38:16.033203+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:42:36.642284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":80,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.023250+0000","last_change":"2023-10-01T10:15:39.293587+0000","last_active":"2023-10-01T10:38:15.023250+0000","last_peered":"2023-10-01T10:38:15.023250+0000","last_clean":"2023-10-01T10:38:15.023250+0000","last_became_active":"2023-10-01T10:15:39.293273+0000","last_became_peered":"2023-10-01T10:15:39.293273+0000","last_unstale":"2023-10-01T10:38:15.023250+0000","last_undegraded":"2023-10-01T10:38:15.023250+0000","last_fullsized":"2023-10-01T10:38:15.023250+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":92,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.031078+0000","last_change":"2023-10-01T10:32:31.571971+0000","last_active":"2023-10-01T10:38:15.031078+0000","last_peered":"2023-10-01T10:38:15.031078+0000","last_clean":"2023-10-01T10:38:15.031078+0000","last_became_active":"2023-10-01T10:32:31.571410+0000","last_became_peered":"2023-10-01T10:32:31.571410+0000","last_unstale":"2023-10-01T10:38:15.031078+0000","last_undegraded":"2023-10-01T10:38:15.031078+0000","last_fullsized":"2023-10-01T10:38:15.031078+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":94,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.034665+0000","last_change":"2023-10-01T10:37:42.765733+0000","last_active":"2023-10-01T10:38:15.034665+0000","last_peered":"2023-10-01T10:38:15.034665+0000","last_clean":"2023-10-01T10:38:15.034665+0000","last_became_active":"2023-10-01T10:37:42.765311+0000","last_became_peered":"2023-10-01T10:37:42.765311+0000","last_unstale":"2023-10-01T10:38:15.034665+0000","last_undegraded":"2023-10-01T10:38:15.034665+0000","last_fullsized":"2023-10-01T10:38:15.034665+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":98,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.027301+0000","last_change":"2023-10-01T10:38:16.026348+0000","last_active":"2023-10-01T10:38:16.027301+0000","last_peered":"2023-10-01T10:38:16.027301+0000","last_clean":"2023-10-01T10:38:16.027301+0000","last_became_active":"2023-10-01T10:38:16.025747+0000","last_became_peered":"2023-10-01T10:38:16.025747+0000","last_unstale":"2023-10-01T10:38:16.027301+0000","last_undegraded":"2023-10-01T10:38:16.027301+0000","last_fullsized":"2023-10-01T10:38:16.027301+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:28:18.934480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":89,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.022493+0000","last_change":"2023-10-01T10:38:16.022493+0000","last_active":"2023-10-01T10:38:16.022493+0000","last_peered":"2023-10-01T10:38:16.022493+0000","last_clean":"2023-10-01T10:38:16.022493+0000","last_became_active":"2023-10-01T10:38:16.022033+0000","last_became_peered":"2023-10-01T10:38:16.022033+0000","last_unstale":"2023-10-01T10:38:16.022493+0000","last_undegraded":"2023-10-01T10:38:16.022493+0000","last_fullsized":"2023-10-01T10:38:16.022493+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:49:05.262763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":100,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.030613+0000","last_change":"2023-10-01T10:38:16.030613+0000","last_active":"2023-10-01T10:38:16.030613+0000","last_peered":"2023-10-01T10:38:16.030613+0000","last_clean":"2023-10-01T10:38:16.030613+0000","last_became_active":"2023-10-01T10:38:16.028975+0000","last_became_peered":"2023-10-01T10:38:16.028975+0000","last_unstale":"2023-10-01T10:38:16.030613+0000","last_undegraded":"2023-10-01T10:38:16.030613+0000","last_fullsized":"2023-10-01T10:38:16.030613+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:46:07.287911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":108,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.030250+0000","last_change":"2023-10-01T10:38:16.030250+0000","last_active":"2023-10-01T10:38:16.030250+0000","last_peered":"2023-10-01T10:38:16.030250+0000","last_clean":"2023-10-01T10:38:16.030250+0000","last_became_active":"2023-10-01T10:38:16.029898+0000","last_became_peered":"2023-10-01T10:38:16.029898+0000","last_unstale":"2023-10-01T10:38:16.030250+0000","last_undegraded":"2023-10-01T10:38:16.030250+0000","last_fullsized":"2023-10-01T10:38:16.030250+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":105,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.035906+0000","last_change":"2023-10-01T10:38:16.035906+0000","last_active":"2023-10-01T10:38:16.035906+0000","last_peered":"2023-10-01T10:38:16.035906+0000","last_clean":"2023-10-01T10:38:16.035906+0000","last_became_active":"2023-10-01T10:38:16.035383+0000","last_became_peered":"2023-10-01T10:38:16.035383+0000","last_unstale":"2023-10-01T10:38:16.035906+0000","last_undegraded":"2023-10-01T10:38:16.035906+0000","last_fullsized":"2023-10-01T10:38:16.035906+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:44:41.500097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":97,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.026611+0000","last_change":"2023-10-01T10:38:16.026611+0000","last_active":"2023-10-01T10:38:16.026611+0000","last_peered":"2023-10-01T10:38:16.026611+0000","last_clean":"2023-10-01T10:38:16.026611+0000","last_became_active":"2023-10-01T10:38:16.026267+0000","last_became_peered":"2023-10-01T10:38:16.026267+0000","last_unstale":"2023-10-01T10:38:16.026611+0000","last_undegraded":"2023-10-01T10:38:16.026611+0000","last_fullsized":"2023-10-01T10:38:16.026611+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:06.135288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":81,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.032162+0000","last_change":"2023-10-01T10:38:16.032162+0000","last_active":"2023-10-01T10:38:16.032162+0000","last_peered":"2023-10-01T10:38:16.032162+0000","last_clean":"2023-10-01T10:38:16.032162+0000","last_became_active":"2023-10-01T10:38:16.031785+0000","last_became_peered":"2023-10-01T10:38:16.031785+0000","last_unstale":"2023-10-01T10:38:16.032162+0000","last_undegraded":"2023-10-01T10:38:16.032162+0000","last_fullsized":"2023-10-01T10:38:16.032162+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:38:43.003626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":87,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.011165+0000","last_change":"2023-10-01T10:37:42.772398+0000","last_active":"2023-10-01T10:38:15.011165+0000","last_peered":"2023-10-01T10:38:15.011165+0000","last_clean":"2023-10-01T10:38:15.011165+0000","last_became_active":"2023-10-01T10:37:42.772084+0000","last_became_peered":"2023-10-01T10:37:42.772084+0000","last_unstale":"2023-10-01T10:38:15.011165+0000","last_undegraded":"2023-10-01T10:38:15.011165+0000","last_fullsized":"2023-10-01T10:38:15.011165+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:32:43.846419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":106,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.021873+0000","last_change":"2023-10-01T10:38:16.019755+0000","last_active":"2023-10-01T10:38:16.021873+0000","last_peered":"2023-10-01T10:38:16.021873+0000","last_clean":"2023-10-01T10:38:16.021873+0000","last_became_active":"2023-10-01T10:38:16.019397+0000","last_became_peered":"2023-10-01T10:38:16.019397+0000","last_unstale":"2023-10-01T10:38:16.021873+0000","last_undegraded":"2023-10-01T10:38:16.021873+0000","last_fullsized":"2023-10-01T10:38:16.021873+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:42:56.623225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":99,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.020037+0000","last_change":"2023-10-01T10:32:31.561108+0000","last_active":"2023-10-01T10:38:16.020037+0000","last_peered":"2023-10-01T10:38:16.020037+0000","last_clean":"2023-10-01T10:38:16.020037+0000","last_became_active":"2023-10-01T10:32:31.560791+0000","last_became_peered":"2023-10-01T10:32:31.560791+0000","last_unstale":"2023-10-01T10:38:16.020037+0000","last_undegraded":"2023-10-01T10:38:16.020037+0000","last_fullsized":"2023-10-01T10:38:16.020037+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:42:47.477671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":109,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.034655+0000","last_change":"2023-10-01T10:37:42.766814+0000","last_active":"2023-10-01T10:38:15.034655+0000","last_peered":"2023-10-01T10:38:15.034655+0000","last_clean":"2023-10-01T10:38:15.034655+0000","last_became_active":"2023-10-01T10:37:42.766451+0000","last_became_peered":"2023-10-01T10:37:42.766451+0000","last_unstale":"2023-10-01T10:38:15.034655+0000","last_undegraded":"2023-10-01T10:38:15.034655+0000","last_fullsized":"2023-10-01T10:38:15.034655+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":65,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.035088+0000","last_change":"2023-10-01T10:38:16.034357+0000","last_active":"2023-10-01T10:38:16.035088+0000","last_peered":"2023-10-01T10:38:16.035088+0000","last_clean":"2023-10-01T10:38:16.035088+0000","last_became_active":"2023-10-01T10:38:16.033905+0000","last_became_peered":"2023-10-01T10:38:16.033905+0000","last_unstale":"2023-10-01T10:38:16.035088+0000","last_undegraded":"2023-10-01T10:38:16.035088+0000","last_fullsized":"2023-10-01T10:38:16.035088+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:47:58.369835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":98,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.034322+0000","last_change":"2023-10-01T10:32:31.572482+0000","last_active":"2023-10-01T10:38:15.034322+0000","last_peered":"2023-10-01T10:38:15.034322+0000","last_clean":"2023-10-01T10:38:15.034322+0000","last_became_active":"2023-10-01T10:32:31.571993+0000","last_became_peered":"2023-10-01T10:32:31.571993+0000","last_unstale":"2023-10-01T10:38:15.034322+0000","last_undegraded":"2023-10-01T10:38:15.034322+0000","last_fullsized":"2023-10-01T10:38:15.034322+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":67,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.030202+0000","last_change":"2023-10-01T10:38:16.030202+0000","last_active":"2023-10-01T10:38:16.030202+0000","last_peered":"2023-10-01T10:38:16.030202+0000","last_clean":"2023-10-01T10:38:16.030202+0000","last_became_active":"2023-10-01T10:38:16.029917+0000","last_became_peered":"2023-10-01T10:38:16.029917+0000","last_unstale":"2023-10-01T10:38:16.030202+0000","last_undegraded":"2023-10-01T10:38:16.030202+0000","last_fullsized":"2023-10-01T10:38:16.030202+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:20:06.763909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":80,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.020177+0000","last_change":"2023-10-01T10:37:42.761153+0000","last_active":"2023-10-01T10:38:16.020177+0000","last_peered":"2023-10-01T10:38:16.020177+0000","last_clean":"2023-10-01T10:38:16.020177+0000","last_became_active":"2023-10-01T10:37:42.760715+0000","last_became_peered":"2023-10-01T10:37:42.760715+0000","last_unstale":"2023-10-01T10:38:16.020177+0000","last_undegraded":"2023-10-01T10:38:16.020177+0000","last_fullsized":"2023-10-01T10:38:16.020177+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:52:50.237174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":97,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.013202+0000","last_change":"2023-10-01T10:37:42.762524+0000","last_active":"2023-10-01T10:38:16.013202+0000","last_peered":"2023-10-01T10:38:16.013202+0000","last_clean":"2023-10-01T10:38:16.013202+0000","last_became_active":"2023-10-01T10:37:42.762209+0000","last_became_peered":"2023-10-01T10:37:42.762209+0000","last_unstale":"2023-10-01T10:38:16.013202+0000","last_undegraded":"2023-10-01T10:38:16.013202+0000","last_fullsized":"2023-10-01T10:38:16.013202+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:45:38.171397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":65,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.020920+0000","last_change":"2023-10-01T10:37:05.019330+0000","last_active":"2023-10-01T10:38:16.020920+0000","last_peered":"2023-10-01T10:38:16.020920+0000","last_clean":"2023-10-01T10:38:16.020920+0000","last_became_active":"2023-10-01T10:37:05.018842+0000","last_became_peered":"2023-10-01T10:37:05.018842+0000","last_unstale":"2023-10-01T10:38:16.020920+0000","last_undegraded":"2023-10-01T10:38:16.020920+0000","last_fullsized":"2023-10-01T10:38:16.020920+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:48:09.551917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":94,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.019778+0000","last_change":"2023-10-01T10:32:31.569162+0000","last_active":"2023-10-01T10:38:16.019778+0000","last_peered":"2023-10-01T10:38:16.019778+0000","last_clean":"2023-10-01T10:38:16.019778+0000","last_became_active":"2023-10-01T10:32:31.568956+0000","last_became_peered":"2023-10-01T10:32:31.568956+0000","last_unstale":"2023-10-01T10:38:16.019778+0000","last_undegraded":"2023-10-01T10:38:16.019778+0000","last_fullsized":"2023-10-01T10:38:16.019778+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:02:08.285304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":98,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.033675+0000","last_change":"2023-10-01T10:37:05.024009+0000","last_active":"2023-10-01T10:38:15.033675+0000","last_peered":"2023-10-01T10:38:15.033675+0000","last_clean":"2023-10-01T10:38:15.033675+0000","last_became_active":"2023-10-01T10:37:05.023458+0000","last_became_peered":"2023-10-01T10:37:05.023458+0000","last_unstale":"2023-10-01T10:38:15.033675+0000","last_undegraded":"2023-10-01T10:38:15.033675+0000","last_fullsized":"2023-10-01T10:38:15.033675+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'139","reported_seq":330,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.019558+0000","last_change":"2023-10-01T10:28:20.217386+0000","last_active":"2023-10-01T10:38:15.019558+0000","last_peered":"2023-10-01T10:38:15.019558+0000","last_clean":"2023-10-01T10:38:15.019558+0000","last_became_active":"2023-10-01T10:28:20.216803+0000","last_became_peered":"2023-10-01T10:28:20.216803+0000","last_unstale":"2023-10-01T10:38:15.019558+0000","last_undegraded":"2023-10-01T10:38:15.019558+0000","last_fullsized":"2023-10-01T10:38:15.019558+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:09:17.221082+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:09:17.221082+0000","last_clean_scrub_stamp":"2023-10-01T10:09:17.221082+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":69,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.012342+0000","last_change":"2023-10-01T10:37:42.765100+0000","last_active":"2023-10-01T10:38:15.012342+0000","last_peered":"2023-10-01T10:38:15.012342+0000","last_clean":"2023-10-01T10:38:15.012342+0000","last_became_active":"2023-10-01T10:37:42.764761+0000","last_became_peered":"2023-10-01T10:37:42.764761+0000","last_unstale":"2023-10-01T10:38:15.012342+0000","last_undegraded":"2023-10-01T10:38:15.012342+0000","last_fullsized":"2023-10-01T10:38:15.012342+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:24:56.907377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":92,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.031255+0000","last_change":"2023-10-01T10:38:16.031255+0000","last_active":"2023-10-01T10:38:16.031255+0000","last_peered":"2023-10-01T10:38:16.031255+0000","last_clean":"2023-10-01T10:38:16.031255+0000","last_became_active":"2023-10-01T10:38:16.030886+0000","last_became_peered":"2023-10-01T10:38:16.030886+0000","last_unstale":"2023-10-01T10:38:16.031255+0000","last_undegraded":"2023-10-01T10:38:16.031255+0000","last_fullsized":"2023-10-01T10:38:16.031255+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":113,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.019451+0000","last_change":"2023-10-01T10:32:31.565868+0000","last_active":"2023-10-01T10:38:15.019451+0000","last_peered":"2023-10-01T10:38:15.019451+0000","last_clean":"2023-10-01T10:38:15.019451+0000","last_became_active":"2023-10-01T10:32:31.565428+0000","last_became_peered":"2023-10-01T10:32:31.565428+0000","last_unstale":"2023-10-01T10:38:15.019451+0000","last_undegraded":"2023-10-01T10:38:15.019451+0000","last_fullsized":"2023-10-01T10:38:15.019451+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":97,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.022511+0000","last_change":"2023-10-01T10:38:16.021968+0000","last_active":"2023-10-01T10:38:16.022511+0000","last_peered":"2023-10-01T10:38:16.022511+0000","last_clean":"2023-10-01T10:38:16.022511+0000","last_became_active":"2023-10-01T10:38:16.021539+0000","last_became_peered":"2023-10-01T10:38:16.021539+0000","last_unstale":"2023-10-01T10:38:16.022511+0000","last_undegraded":"2023-10-01T10:38:16.022511+0000","last_fullsized":"2023-10-01T10:38:16.022511+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:12:53.105747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":98,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.019580+0000","last_change":"2023-10-01T10:37:05.017431+0000","last_active":"2023-10-01T10:38:16.019580+0000","last_peered":"2023-10-01T10:38:16.019580+0000","last_clean":"2023-10-01T10:38:16.019580+0000","last_became_active":"2023-10-01T10:37:05.017094+0000","last_became_peered":"2023-10-01T10:37:05.017094+0000","last_unstale":"2023-10-01T10:38:16.019580+0000","last_undegraded":"2023-10-01T10:38:16.019580+0000","last_fullsized":"2023-10-01T10:38:16.019580+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:10:40.664686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":113,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.032774+0000","last_change":"2023-10-01T10:38:16.032774+0000","last_active":"2023-10-01T10:38:16.032774+0000","last_peered":"2023-10-01T10:38:16.032774+0000","last_clean":"2023-10-01T10:38:16.032774+0000","last_became_active":"2023-10-01T10:38:16.032283+0000","last_became_peered":"2023-10-01T10:38:16.032283+0000","last_unstale":"2023-10-01T10:38:16.032774+0000","last_undegraded":"2023-10-01T10:38:16.032774+0000","last_fullsized":"2023-10-01T10:38:16.032774+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":65,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.020296+0000","last_change":"2023-10-01T10:37:05.021228+0000","last_active":"2023-10-01T10:38:16.020296+0000","last_peered":"2023-10-01T10:38:16.020296+0000","last_clean":"2023-10-01T10:38:16.020296+0000","last_became_active":"2023-10-01T10:37:05.020888+0000","last_became_peered":"2023-10-01T10:37:05.020888+0000","last_unstale":"2023-10-01T10:38:16.020296+0000","last_undegraded":"2023-10-01T10:38:16.020296+0000","last_fullsized":"2023-10-01T10:38:16.020296+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:59:57.641247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":68,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.020664+0000","last_change":"2023-10-01T10:28:20.220812+0000","last_active":"2023-10-01T10:38:16.020664+0000","last_peered":"2023-10-01T10:38:16.020664+0000","last_clean":"2023-10-01T10:38:16.020664+0000","last_became_active":"2023-10-01T10:28:20.215962+0000","last_became_peered":"2023-10-01T10:28:20.215962+0000","last_unstale":"2023-10-01T10:38:16.020664+0000","last_undegraded":"2023-10-01T10:38:16.020664+0000","last_fullsized":"2023-10-01T10:38:16.020664+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:35:15.275097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":74,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.012601+0000","last_change":"2023-10-01T10:37:42.773999+0000","last_active":"2023-10-01T10:38:15.012601+0000","last_peered":"2023-10-01T10:38:15.012601+0000","last_clean":"2023-10-01T10:38:15.012601+0000","last_became_active":"2023-10-01T10:37:42.773355+0000","last_became_peered":"2023-10-01T10:37:42.773355+0000","last_unstale":"2023-10-01T10:38:15.012601+0000","last_undegraded":"2023-10-01T10:38:15.012601+0000","last_fullsized":"2023-10-01T10:38:15.012601+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:46:51.317151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":107,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.035022+0000","last_change":"2023-10-01T10:38:16.034193+0000","last_active":"2023-10-01T10:38:16.035022+0000","last_peered":"2023-10-01T10:38:16.035022+0000","last_clean":"2023-10-01T10:38:16.035022+0000","last_became_active":"2023-10-01T10:38:16.033703+0000","last_became_peered":"2023-10-01T10:38:16.033703+0000","last_unstale":"2023-10-01T10:38:16.035022+0000","last_undegraded":"2023-10-01T10:38:16.035022+0000","last_fullsized":"2023-10-01T10:38:16.035022+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:21:26.061595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":71,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.010953+0000","last_change":"2023-10-01T10:37:42.773538+0000","last_active":"2023-10-01T10:38:15.010953+0000","last_peered":"2023-10-01T10:38:15.010953+0000","last_clean":"2023-10-01T10:38:15.010953+0000","last_became_active":"2023-10-01T10:37:42.771863+0000","last_became_peered":"2023-10-01T10:37:42.771863+0000","last_unstale":"2023-10-01T10:38:15.010953+0000","last_undegraded":"2023-10-01T10:38:15.010953+0000","last_fullsized":"2023-10-01T10:38:15.010953+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:46:46.698627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":97,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.031790+0000","last_change":"2023-10-01T10:37:05.023034+0000","last_active":"2023-10-01T10:38:15.031790+0000","last_peered":"2023-10-01T10:38:15.031790+0000","last_clean":"2023-10-01T10:38:15.031790+0000","last_became_active":"2023-10-01T10:37:05.022670+0000","last_became_peered":"2023-10-01T10:37:05.022670+0000","last_unstale":"2023-10-01T10:38:15.031790+0000","last_undegraded":"2023-10-01T10:38:15.031790+0000","last_fullsized":"2023-10-01T10:38:15.031790+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":101,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.027938+0000","last_change":"2023-10-01T10:38:16.027472+0000","last_active":"2023-10-01T10:38:16.027938+0000","last_peered":"2023-10-01T10:38:16.027938+0000","last_clean":"2023-10-01T10:38:16.027938+0000","last_became_active":"2023-10-01T10:38:16.026810+0000","last_became_peered":"2023-10-01T10:38:16.026810+0000","last_unstale":"2023-10-01T10:38:16.027938+0000","last_undegraded":"2023-10-01T10:38:16.027938+0000","last_fullsized":"2023-10-01T10:38:16.027938+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:30:06.978187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":65,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.035705+0000","last_change":"2023-10-01T10:38:16.035705+0000","last_active":"2023-10-01T10:38:16.035705+0000","last_peered":"2023-10-01T10:38:16.035705+0000","last_clean":"2023-10-01T10:38:16.035705+0000","last_became_active":"2023-10-01T10:38:16.035389+0000","last_became_peered":"2023-10-01T10:38:16.035389+0000","last_unstale":"2023-10-01T10:38:16.035705+0000","last_undegraded":"2023-10-01T10:38:16.035705+0000","last_fullsized":"2023-10-01T10:38:16.035705+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:16:10.201409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":85,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.028785+0000","last_change":"2023-10-01T10:38:16.028491+0000","last_active":"2023-10-01T10:38:16.028785+0000","last_peered":"2023-10-01T10:38:16.028785+0000","last_clean":"2023-10-01T10:38:16.028785+0000","last_became_active":"2023-10-01T10:38:16.028175+0000","last_became_peered":"2023-10-01T10:38:16.028175+0000","last_unstale":"2023-10-01T10:38:16.028785+0000","last_undegraded":"2023-10-01T10:38:16.028785+0000","last_fullsized":"2023-10-01T10:38:16.028785+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:44:29.083129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":107,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.012738+0000","last_change":"2023-10-01T10:37:42.762360+0000","last_active":"2023-10-01T10:38:16.012738+0000","last_peered":"2023-10-01T10:38:16.012738+0000","last_clean":"2023-10-01T10:38:16.012738+0000","last_became_active":"2023-10-01T10:37:42.761936+0000","last_became_peered":"2023-10-01T10:37:42.761936+0000","last_unstale":"2023-10-01T10:38:16.012738+0000","last_undegraded":"2023-10-01T10:38:16.012738+0000","last_fullsized":"2023-10-01T10:38:16.012738+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:50.637582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":111,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.037809+0000","last_change":"2023-10-01T10:38:16.037809+0000","last_active":"2023-10-01T10:38:16.037809+0000","last_peered":"2023-10-01T10:38:16.037809+0000","last_clean":"2023-10-01T10:38:16.037809+0000","last_became_active":"2023-10-01T10:38:16.037318+0000","last_became_peered":"2023-10-01T10:38:16.037318+0000","last_unstale":"2023-10-01T10:38:16.037809+0000","last_undegraded":"2023-10-01T10:38:16.037809+0000","last_fullsized":"2023-10-01T10:38:16.037809+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":115,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.034282+0000","last_change":"2023-10-01T10:38:16.034282+0000","last_active":"2023-10-01T10:38:16.034282+0000","last_peered":"2023-10-01T10:38:16.034282+0000","last_clean":"2023-10-01T10:38:16.034282+0000","last_became_active":"2023-10-01T10:38:16.033820+0000","last_became_peered":"2023-10-01T10:38:16.033820+0000","last_unstale":"2023-10-01T10:38:16.034282+0000","last_undegraded":"2023-10-01T10:38:16.034282+0000","last_fullsized":"2023-10-01T10:38:16.034282+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:48:23.815832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":96,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.038077+0000","last_change":"2023-10-01T10:38:16.038077+0000","last_active":"2023-10-01T10:38:16.038077+0000","last_peered":"2023-10-01T10:38:16.038077+0000","last_clean":"2023-10-01T10:38:16.038077+0000","last_became_active":"2023-10-01T10:38:16.037313+0000","last_became_peered":"2023-10-01T10:38:16.037313+0000","last_unstale":"2023-10-01T10:38:16.038077+0000","last_undegraded":"2023-10-01T10:38:16.038077+0000","last_fullsized":"2023-10-01T10:38:16.038077+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":115,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.025600+0000","last_change":"2023-10-01T10:38:16.025600+0000","last_active":"2023-10-01T10:38:16.025600+0000","last_peered":"2023-10-01T10:38:16.025600+0000","last_clean":"2023-10-01T10:38:16.025600+0000","last_became_active":"2023-10-01T10:38:16.025252+0000","last_became_peered":"2023-10-01T10:38:16.025252+0000","last_unstale":"2023-10-01T10:38:16.025600+0000","last_undegraded":"2023-10-01T10:38:16.025600+0000","last_fullsized":"2023-10-01T10:38:16.025600+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:24:18.871963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":102,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.012215+0000","last_change":"2023-10-01T10:37:42.761243+0000","last_active":"2023-10-01T10:38:15.012215+0000","last_peered":"2023-10-01T10:38:15.012215+0000","last_clean":"2023-10-01T10:38:15.012215+0000","last_became_active":"2023-10-01T10:37:42.760834+0000","last_became_peered":"2023-10-01T10:37:42.760834+0000","last_unstale":"2023-10-01T10:38:15.012215+0000","last_undegraded":"2023-10-01T10:38:15.012215+0000","last_fullsized":"2023-10-01T10:38:15.012215+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:37:21.976095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":115,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.014918+0000","last_change":"2023-10-01T10:37:42.765142+0000","last_active":"2023-10-01T10:38:15.014918+0000","last_peered":"2023-10-01T10:38:15.014918+0000","last_clean":"2023-10-01T10:38:15.014918+0000","last_became_active":"2023-10-01T10:37:42.764770+0000","last_became_peered":"2023-10-01T10:37:42.764770+0000","last_unstale":"2023-10-01T10:38:15.014918+0000","last_undegraded":"2023-10-01T10:38:15.014918+0000","last_fullsized":"2023-10-01T10:38:15.014918+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:23:33.527876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":86,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.019521+0000","last_change":"2023-10-01T10:37:05.021621+0000","last_active":"2023-10-01T10:38:16.019521+0000","last_peered":"2023-10-01T10:38:16.019521+0000","last_clean":"2023-10-01T10:38:16.019521+0000","last_became_active":"2023-10-01T10:37:05.021173+0000","last_became_peered":"2023-10-01T10:37:05.021173+0000","last_unstale":"2023-10-01T10:38:16.019521+0000","last_undegraded":"2023-10-01T10:38:16.019521+0000","last_fullsized":"2023-10-01T10:38:16.019521+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:13:25.364499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":81,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.031427+0000","last_change":"2023-10-01T10:38:16.031427+0000","last_active":"2023-10-01T10:38:16.031427+0000","last_peered":"2023-10-01T10:38:16.031427+0000","last_clean":"2023-10-01T10:38:16.031427+0000","last_became_active":"2023-10-01T10:38:16.029786+0000","last_became_peered":"2023-10-01T10:38:16.029786+0000","last_unstale":"2023-10-01T10:38:16.031427+0000","last_undegraded":"2023-10-01T10:38:16.031427+0000","last_fullsized":"2023-10-01T10:38:16.031427+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:53:22.796191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30319,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430531,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":303116,"kb_used_data":1796,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":93442036,"statfs":{"total":95995035648,"available":95684644864,"internally_reserved":0,"allocated":1839104,"data_stored":1363146,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:38:38 2023","interfaces":[{"interface":"back","average":{"1min":0.97499999999999998,"5min":1.4410000000000001,"15min":1.7450000000000001},"min":{"1min":0.82599999999999996,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":1.2290000000000001,"5min":7.9900000000000002,"15min":14.974},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.96299999999999997,"5min":1.5429999999999999,"15min":1.8169999999999999},"min":{"1min":0.65100000000000002,"5min":0.32800000000000001,"15min":0.29999999999999999},"max":{"1min":1.21,"5min":9.7289999999999992,"15min":15.029},"last":0.80300000000000005}]},{"osd":1,"last update":"Sun Oct 1 10:38:50 2023","interfaces":[{"interface":"back","average":{"1min":0.82199999999999995,"5min":1.454,"15min":1.23},"min":{"1min":0.57899999999999996,"5min":0.35999999999999999,"15min":0.309},"max":{"1min":1.1779999999999999,"5min":8.1180000000000003,"15min":8.1180000000000003},"last":0.96599999999999997},{"interface":"front","average":{"1min":0.94299999999999995,"5min":1.4330000000000001,"15min":1.617},"min":{"1min":0.621,"5min":0.318,"15min":0.318},"max":{"1min":1.2589999999999999,"5min":11.266,"15min":11.266},"last":0.93300000000000005}]},{"osd":2,"last update":"Sun Oct 1 10:39:05 2023","interfaces":[{"interface":"back","average":{"1min":0.96799999999999997,"5min":1.022,"15min":1.0309999999999999},"min":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"max":{"1min":1.1799999999999999,"5min":1.214,"15min":1.214},"last":1.095},{"interface":"front","average":{"1min":0.89600000000000002,"5min":0.93600000000000005,"15min":0.94299999999999995},"min":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.167,"5min":1.294,"15min":1.294},"last":0.83399999999999996}]},{"osd":3,"last update":"Sun Oct 1 10:38:44 2023","interfaces":[{"interface":"back","average":{"1min":0.96799999999999997,"5min":0.96799999999999997,"15min":0.96799999999999997},"min":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"max":{"1min":1.3879999999999999,"5min":1.3879999999999999,"15min":1.3879999999999999},"last":1.137},{"interface":"front","average":{"1min":0.94699999999999995,"5min":0.94699999999999995,"15min":0.94699999999999995},"min":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"max":{"1min":1.234,"5min":1.234,"15min":1.234},"last":0.748}]},{"osd":4,"last update":"Sun Oct 1 10:39:18 2023","interfaces":[{"interface":"back","average":{"1min":0.91700000000000004,"5min":0.91700000000000004,"15min":0.91700000000000004},"min":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"max":{"1min":1.167,"5min":1.167,"15min":1.167},"last":0.88700000000000001},{"interface":"front","average":{"1min":1.1859999999999999,"5min":1.1859999999999999,"15min":1.1859999999999999},"min":{"1min":0.90500000000000003,"5min":0.90500000000000003,"15min":0.90500000000000003},"max":{"1min":1.333,"5min":1.333,"15min":1.333},"last":1.1779999999999999}]},{"osd":5,"last update":"Sun Oct 1 10:38:50 2023","interfaces":[{"interface":"back","average":{"1min":0.93899999999999995,"5min":1.2310000000000001,"15min":1.2609999999999999},"min":{"1min":0.504,"5min":0.35799999999999998,"15min":0.29699999999999999},"max":{"1min":1.349,"5min":10.106999999999999,"15min":10.106999999999999},"last":0.998},{"interface":"front","average":{"1min":0.93899999999999995,"5min":1.232,"15min":1.21},"min":{"1min":0.48699999999999999,"5min":0.42799999999999999,"15min":0.34799999999999998},"max":{"1min":1.3069999999999999,"5min":6.2050000000000001,"15min":6.2050000000000001},"last":0.90600000000000003}]},{"osd":6,"last update":"Sun Oct 1 10:38:50 2023","interfaces":[{"interface":"back","average":{"1min":1.052,"5min":1.421,"15min":1.341},"min":{"1min":0.46100000000000002,"5min":0.39300000000000002,"15min":0.32800000000000001},"max":{"1min":1.345,"5min":7.3300000000000001,"15min":9.5500000000000007},"last":1.052},{"interface":"front","average":{"1min":0.91900000000000004,"5min":1.4410000000000001,"15min":1.391},"min":{"1min":0.58999999999999997,"5min":0.40100000000000002,"15min":0.33900000000000002},"max":{"1min":1.2709999999999999,"5min":6.984,"15min":12.696},"last":1.0269999999999999}]}]},{"osd":6,"up_from":42,"seq":180388626765,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":307196,"kb_used_data":1780,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":93437956,"statfs":{"total":95995035648,"available":95680466944,"internally_reserved":0,"allocated":1822720,"data_stored":1362685,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:38:33 2023","interfaces":[{"interface":"back","average":{"1min":0.79800000000000004,"5min":1.4339999999999999,"15min":1.4850000000000001},"min":{"1min":0.51700000000000002,"5min":0.29399999999999998,"15min":0.28499999999999998},"max":{"1min":1.0940000000000001,"5min":7.1139999999999999,"15min":9.1300000000000008},"last":0.91100000000000003},{"interface":"front","average":{"1min":0.91700000000000004,"5min":1.573,"15min":1.4990000000000001},"min":{"1min":0.60599999999999998,"5min":0.29999999999999999,"15min":0.28899999999999998},"max":{"1min":1.2769999999999999,"5min":6.9640000000000004,"15min":7.3109999999999999},"last":1.238}]},{"osd":1,"last update":"Sun Oct 1 10:38:41 2023","interfaces":[{"interface":"back","average":{"1min":0.88200000000000001,"5min":1.5920000000000001,"15min":1.1459999999999999},"min":{"1min":0.70599999999999996,"5min":0.37,"15min":0.26600000000000001},"max":{"1min":1.21,"5min":7.9100000000000001,"15min":7.9100000000000001},"last":1.004},{"interface":"front","average":{"1min":0.90800000000000003,"5min":1.4930000000000001,"15min":1.492},"min":{"1min":0.55600000000000005,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.2,"5min":7.4199999999999999,"15min":7.4199999999999999},"last":1.3089999999999999}]},{"osd":2,"last update":"Sun Oct 1 10:39:05 2023","interfaces":[{"interface":"back","average":{"1min":0.88700000000000001,"5min":0.87,"15min":0.86699999999999999},"min":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"max":{"1min":1.2450000000000001,"5min":1.2450000000000001,"15min":1.2450000000000001},"last":0.94799999999999995},{"interface":"front","average":{"1min":0.95099999999999996,"5min":0.94099999999999995,"15min":0.93899999999999995},"min":{"1min":0.70899999999999996,"5min":0.64900000000000002,"15min":0.64900000000000002},"max":{"1min":1.1850000000000001,"5min":1.2929999999999999,"15min":1.2929999999999999},"last":1.214}]},{"osd":3,"last update":"Sun Oct 1 10:38:44 2023","interfaces":[{"interface":"back","average":{"1min":0.98099999999999998,"5min":0.98099999999999998,"15min":0.98099999999999998},"min":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"max":{"1min":1.2529999999999999,"5min":1.2529999999999999,"15min":1.2529999999999999},"last":1.1699999999999999},{"interface":"front","average":{"1min":1.018,"5min":1.018,"15min":1.018},"min":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"max":{"1min":1.2230000000000001,"5min":1.2230000000000001,"15min":1.2230000000000001},"last":1.034}]},{"osd":4,"last update":"Sun Oct 1 10:39:18 2023","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":1.02,"15min":1.02},"min":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"max":{"1min":1.3919999999999999,"5min":1.3919999999999999,"15min":1.3919999999999999},"last":0.83599999999999997},{"interface":"front","average":{"1min":1.103,"5min":1.103,"15min":1.103},"min":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"max":{"1min":1.4179999999999999,"5min":1.4179999999999999,"15min":1.4179999999999999},"last":1.137}]},{"osd":5,"last update":"Sun Oct 1 10:38:49 2023","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":1.3120000000000001,"15min":1.2270000000000001},"min":{"1min":0.46100000000000002,"5min":0.39100000000000001,"15min":0.29699999999999999},"max":{"1min":1.254,"5min":6.085,"15min":8.0150000000000006},"last":1.0820000000000001},{"interface":"front","average":{"1min":0.97899999999999998,"5min":1.284,"15min":1.278},"min":{"1min":0.38,"5min":0.38,"15min":0.34399999999999997},"max":{"1min":1.3779999999999999,"5min":10.842000000000001,"15min":10.842000000000001},"last":1.105}]},{"osd":7,"last update":"Sun Oct 1 10:38:33 2023","interfaces":[{"interface":"back","average":{"1min":0.95199999999999996,"5min":1.1080000000000001,"15min":1.095},"min":{"1min":0.41799999999999998,"5min":0.39700000000000002,"15min":0.36599999999999999},"max":{"1min":1.327,"5min":4.3499999999999996,"15min":7.5960000000000001},"last":0.97699999999999998},{"interface":"front","average":{"1min":1.02,"5min":1.278,"15min":1.3520000000000001},"min":{"1min":0.53000000000000003,"5min":0.42299999999999999,"15min":0.32700000000000001},"max":{"1min":1.405,"5min":5.3470000000000004,"15min":11.031000000000001},"last":1.2609999999999999}]}]},{"osd":1,"up_from":101,"seq":433791696983,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":251816,"kb_used_data":1312,"kb_used_omap":4,"kb_used_meta":250491,"kb_avail":93493336,"statfs":{"total":95995035648,"available":95737176064,"internally_reserved":0,"allocated":1343488,"data_stored":789975,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":256503694},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:38:43 2023","interfaces":[{"interface":"back","average":{"1min":0.68899999999999995,"5min":1.5669999999999999,"15min":2.1429999999999998},"min":{"1min":0.42999999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":1.002,"5min":11.071999999999999,"15min":11.071999999999999},"last":0.85999999999999999},{"interface":"front","average":{"1min":0.64100000000000001,"5min":1.498,"15min":2.0880000000000001},"min":{"1min":0.47499999999999998,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.84799999999999998,"5min":7.9900000000000002,"15min":9.3230000000000004},"last":0.96199999999999997}]},{"osd":2,"last update":"Sun Oct 1 10:39:12 2023","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.94499999999999995,"5min":1.0589999999999999,"15min":1.0589999999999999},"last":0.82499999999999996},{"interface":"front","average":{"1min":0.73399999999999999,"5min":0.76000000000000001,"15min":0.76400000000000001},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.0309999999999999,"5min":1.1319999999999999,"15min":1.1319999999999999},"last":1.0980000000000001}]},{"osd":3,"last update":"Sun Oct 1 10:38:43 2023","interfaces":[{"interface":"back","average":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"min":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"max":{"1min":0.871,"5min":0.871,"15min":0.871},"last":0.76800000000000002},{"interface":"front","average":{"1min":0.753,"5min":0.753,"15min":0.753},"min":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":0.93500000000000005,"5min":0.93500000000000005,"15min":0.93500000000000005},"last":1.022}]},{"osd":4,"last update":"Sun Oct 1 10:39:18 2023","interfaces":[{"interface":"back","average":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88200000000000001},"min":{"1min":0.627,"5min":0.627,"15min":0.627},"max":{"1min":1.165,"5min":1.165,"15min":1.165},"last":1.137},{"interface":"front","average":{"1min":1.008,"5min":1.008,"15min":1.008},"min":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"max":{"1min":1.268,"5min":1.268,"15min":1.268},"last":1.1950000000000001}]},{"osd":5,"last update":"Sun Oct 1 10:38:43 2023","interfaces":[{"interface":"back","average":{"1min":1.0089999999999999,"5min":1.482,"15min":1.8140000000000001},"min":{"1min":0.73799999999999999,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":1.3779999999999999,"5min":8.9429999999999996,"15min":8.9429999999999996},"last":1.161},{"interface":"front","average":{"1min":1.034,"5min":1.8089999999999999,"15min":2.427},"min":{"1min":0.65200000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":1.399,"5min":9.1300000000000008,"15min":11.353999999999999},"last":1.0680000000000001}]},{"osd":6,"last update":"Sun Oct 1 10:38:43 2023","interfaces":[{"interface":"back","average":{"1min":1.0369999999999999,"5min":1.542,"15min":2.0619999999999998},"min":{"1min":0.72099999999999997,"5min":0.37,"15min":0.37},"max":{"1min":1.369,"5min":8.6630000000000003,"15min":8.6630000000000003},"last":1.1819999999999999},{"interface":"front","average":{"1min":1.0229999999999999,"5min":1.5660000000000001,"15min":2.024},"min":{"1min":0.72199999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":1.456,"5min":6,"15min":7.1369999999999996},"last":1.2669999999999999}]},{"osd":7,"last update":"Sun Oct 1 10:38:43 2023","interfaces":[{"interface":"back","average":{"1min":1.0569999999999999,"5min":1.569,"15min":1.96},"min":{"1min":0.75700000000000001,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":1.363,"5min":6.0510000000000002,"15min":7.1680000000000001},"last":1.335},{"interface":"front","average":{"1min":1.0840000000000001,"5min":1.5940000000000001,"15min":1.7989999999999999},"min":{"1min":0.73399999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":1.286,"5min":9.0749999999999993,"15min":9.0749999999999993},"last":1.365}]}]},{"osd":0,"up_from":86,"seq":369367187593,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":294004,"kb_used_data":1900,"kb_used_omap":5,"kb_used_meta":292090,"kb_avail":93451148,"statfs":{"total":95995035648,"available":95693975552,"internally_reserved":0,"allocated":1945600,"data_stored":1378971,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":299100794},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 10:38:43 2023","interfaces":[{"interface":"back","average":{"1min":0.68200000000000005,"5min":1.7649999999999999,"15min":1.7050000000000001},"min":{"1min":0.433,"5min":0.214,"15min":0.214},"max":{"1min":0.95899999999999996,"5min":12.855,"15min":12.855},"last":0.81000000000000005},{"interface":"front","average":{"1min":0.68100000000000005,"5min":1.4359999999999999,"15min":2.5},"min":{"1min":0.35099999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.93100000000000005,"5min":8.6059999999999999,"15min":15.48},"last":1.1180000000000001}]},{"osd":2,"last update":"Sun Oct 1 10:39:07 2023","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.69099999999999995,"15min":0.69799999999999995},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":1.085,"5min":1.085,"15min":1.085},"last":0.623},{"interface":"front","average":{"1min":0.70999999999999996,"5min":0.74399999999999999,"15min":0.75},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":1.149,"5min":1.149,"15min":1.149},"last":0.83899999999999997}]},{"osd":3,"last update":"Sun Oct 1 10:38:43 2023","interfaces":[{"interface":"back","average":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"min":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":1.3069999999999999,"5min":1.3069999999999999,"15min":1.3069999999999999},"last":0.73099999999999998},{"interface":"front","average":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"min":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"max":{"1min":1.1060000000000001,"5min":1.1060000000000001,"15min":1.1060000000000001},"last":0.92500000000000004}]},{"osd":4,"last update":"Sun Oct 1 10:39:18 2023","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":1.02,"15min":1.02},"min":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"max":{"1min":1.3200000000000001,"5min":1.3200000000000001,"15min":1.3200000000000001},"last":1.292},{"interface":"front","average":{"1min":1.0349999999999999,"5min":1.0349999999999999,"15min":1.0349999999999999},"min":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"max":{"1min":1.5029999999999999,"5min":1.5029999999999999,"15min":1.5029999999999999},"last":1.1990000000000001}]},{"osd":5,"last update":"Sun Oct 1 10:38:43 2023","interfaces":[{"interface":"back","average":{"1min":1.0349999999999999,"5min":1.587,"15min":1.786},"min":{"1min":0.75700000000000001,"5min":0.32000000000000001,"15min":0.31900000000000001},"max":{"1min":1.369,"5min":13.068,"15min":13.068},"last":0.96399999999999997},{"interface":"front","average":{"1min":1.097,"5min":1.7490000000000001,"15min":1.7809999999999999},"min":{"1min":0.79800000000000004,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":1.5649999999999999,"5min":11.747,"15min":11.747},"last":1.157}]},{"osd":6,"last update":"Sun Oct 1 10:38:43 2023","interfaces":[{"interface":"back","average":{"1min":1.0720000000000001,"5min":2.0339999999999998,"15min":1.8859999999999999},"min":{"1min":0.59199999999999997,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":1.4910000000000001,"5min":11.663,"15min":11.663},"last":1.0640000000000001},{"interface":"front","average":{"1min":1.0609999999999999,"5min":1.8220000000000001,"15min":1.831},"min":{"1min":0.70699999999999996,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":1.653,"5min":15.069000000000001,"15min":15.069000000000001},"last":0.88200000000000001}]},{"osd":7,"last update":"Sun Oct 1 10:38:43 2023","interfaces":[{"interface":"back","average":{"1min":1.1220000000000001,"5min":1.867,"15min":1.952},"min":{"1min":0.76000000000000001,"5min":0.40300000000000002,"15min":0.39000000000000001},"max":{"1min":1.613,"5min":12.686,"15min":12.686},"last":1.2310000000000001},{"interface":"front","average":{"1min":1.119,"5min":1.625,"15min":1.657},"min":{"1min":0.65600000000000003,"5min":0.42199999999999999,"15min":0.36199999999999999},"max":{"1min":1.732,"5min":8.6829999999999998,"15min":13.654},"last":1.028}]}]},{"osd":2,"up_from":116,"seq":498216206368,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":155752,"kb_used_data":1248,"kb_used_omap":7,"kb_used_meta":154488,"kb_avail":93589400,"statfs":{"total":95995035648,"available":95835545600,"internally_reserved":0,"allocated":1277952,"data_stored":778803,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":158195793},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:39:07 2023","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":0.66800000000000004,"15min":0.67400000000000004},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":1.002,"5min":1.0660000000000001,"15min":1.0660000000000001},"last":0.61499999999999999},{"interface":"front","average":{"1min":0.73399999999999999,"5min":0.78200000000000003,"15min":0.79000000000000004},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":1.0760000000000001,"5min":1.1140000000000001,"15min":1.1140000000000001},"last":0.78100000000000003}]},{"osd":1,"last update":"Sun Oct 1 10:39:07 2023","interfaces":[{"interface":"back","average":{"1min":0.73799999999999999,"5min":0.71699999999999997,"15min":0.71299999999999997},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":1.087,"5min":1.087,"15min":1.087},"last":0.90100000000000002},{"interface":"front","average":{"1min":0.67200000000000004,"5min":0.77500000000000002,"15min":0.79200000000000004},"min":{"1min":0.36499999999999999,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.91900000000000004,"5min":1.1479999999999999,"15min":1.1479999999999999},"last":0.70599999999999996}]},{"osd":3,"last update":"Sun Oct 1 10:38:44 2023","interfaces":[{"interface":"back","average":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.94399999999999995,"5min":0.94399999999999995,"15min":0.94399999999999995},"last":0.753},{"interface":"front","average":{"1min":0.88700000000000001,"5min":0.88700000000000001,"15min":0.88700000000000001},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":1.2,"5min":1.2,"15min":1.2},"last":0.97699999999999998}]},{"osd":4,"last update":"Sun Oct 1 10:39:18 2023","interfaces":[{"interface":"back","average":{"1min":0.98099999999999998,"5min":0.98099999999999998,"15min":0.98099999999999998},"min":{"1min":0.745,"5min":0.745,"15min":0.745},"max":{"1min":1.28,"5min":1.28,"15min":1.28},"last":1.268},{"interface":"front","average":{"1min":0.95699999999999996,"5min":0.95699999999999996,"15min":0.95699999999999996},"min":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"max":{"1min":1.254,"5min":1.254,"15min":1.254},"last":1.196}]},{"osd":5,"last update":"Sun Oct 1 10:39:07 2023","interfaces":[{"interface":"back","average":{"1min":1.032,"5min":1.1040000000000001,"15min":1.1160000000000001},"min":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"max":{"1min":1.468,"5min":1.6339999999999999,"15min":1.6339999999999999},"last":1.22},{"interface":"front","average":{"1min":1.0329999999999999,"5min":1.02,"15min":1.018},"min":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":1.3129999999999999,"5min":1.585,"15min":1.585},"last":1.1060000000000001}]},{"osd":6,"last update":"Sun Oct 1 10:39:07 2023","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":1.0840000000000001,"15min":1.095},"min":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":1.276,"5min":1.486,"15min":1.486},"last":1.02},{"interface":"front","average":{"1min":1.1279999999999999,"5min":1.1419999999999999,"15min":1.1439999999999999},"min":{"1min":0.84199999999999997,"5min":0.65200000000000002,"15min":0.65200000000000002},"max":{"1min":1.3799999999999999,"5min":1.6639999999999999,"15min":1.6639999999999999},"last":1.056}]},{"osd":7,"last update":"Sun Oct 1 10:39:07 2023","interfaces":[{"interface":"back","average":{"1min":1.1020000000000001,"5min":1.1259999999999999,"15min":1.1299999999999999},"min":{"1min":0.79500000000000004,"5min":0.749,"15min":0.749},"max":{"1min":1.444,"5min":1.536,"15min":1.536},"last":0.92800000000000005},{"interface":"front","average":{"1min":1.129,"5min":1.206,"15min":1.2190000000000001},"min":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"max":{"1min":1.7390000000000001,"5min":1.7390000000000001,"15min":1.7390000000000001},"last":1.1399999999999999}]}]},{"osd":3,"up_from":121,"seq":519691042840,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":103128,"kb_used_data":1232,"kb_used_omap":2,"kb_used_meta":101885,"kb_avail":93642024,"statfs":{"total":95995035648,"available":95889432576,"internally_reserved":0,"allocated":1261568,"data_stored":776896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":104330293},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:38:46 2023","interfaces":[{"interface":"back","average":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.96199999999999997,"5min":0.96199999999999997,"15min":0.96199999999999997},"last":0.76300000000000001},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.93300000000000005}]},{"osd":1,"last update":"Sun Oct 1 10:38:46 2023","interfaces":[{"interface":"back","average":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"min":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"max":{"1min":1.2370000000000001,"5min":1.2370000000000001,"15min":1.2370000000000001},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"min":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"max":{"1min":1.1659999999999999,"5min":1.1659999999999999,"15min":1.1659999999999999},"last":0.68899999999999995}]},{"osd":2,"last update":"Sun Oct 1 10:38:46 2023","interfaces":[{"interface":"back","average":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.85399999999999998},{"interface":"front","average":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"min":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"max":{"1min":1.046,"5min":1.046,"15min":1.046},"last":0.89100000000000001}]},{"osd":4,"last update":"Sun Oct 1 10:39:15 2023","interfaces":[{"interface":"back","average":{"1min":0.94099999999999995,"5min":0.94099999999999995,"15min":0.94099999999999995},"min":{"1min":0.623,"5min":0.623,"15min":0.623},"max":{"1min":1.306,"5min":1.306,"15min":1.306},"last":1.0269999999999999},{"interface":"front","average":{"1min":0.92800000000000005,"5min":0.92800000000000005,"15min":0.92800000000000005},"min":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"max":{"1min":1.278,"5min":1.278,"15min":1.278},"last":1.1040000000000001}]},{"osd":5,"last update":"Sun Oct 1 10:38:46 2023","interfaces":[{"interface":"back","average":{"1min":0.94399999999999995,"5min":0.94399999999999995,"15min":0.94399999999999995},"min":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"max":{"1min":1.248,"5min":1.248,"15min":1.248},"last":0.81399999999999995},{"interface":"front","average":{"1min":1.006,"5min":1.006,"15min":1.006},"min":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"max":{"1min":1.4199999999999999,"5min":1.4199999999999999,"15min":1.4199999999999999},"last":1.0649999999999999}]},{"osd":6,"last update":"Sun Oct 1 10:38:46 2023","interfaces":[{"interface":"back","average":{"1min":1.0109999999999999,"5min":1.0109999999999999,"15min":1.0109999999999999},"min":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"max":{"1min":1.325,"5min":1.325,"15min":1.325},"last":1.0449999999999999},{"interface":"front","average":{"1min":1.1060000000000001,"5min":1.1060000000000001,"15min":1.1060000000000001},"min":{"1min":0.89000000000000001,"5min":0.89000000000000001,"15min":0.89000000000000001},"max":{"1min":1.377,"5min":1.377,"15min":1.377},"last":1.131}]},{"osd":7,"last update":"Sun Oct 1 10:38:46 2023","interfaces":[{"interface":"back","average":{"1min":1.0600000000000001,"5min":1.0600000000000001,"15min":1.0600000000000001},"min":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"max":{"1min":1.411,"5min":1.411,"15min":1.411},"last":0.98499999999999999},{"interface":"front","average":{"1min":1.018,"5min":1.018,"15min":1.018},"min":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"max":{"1min":1.3360000000000001,"5min":1.3360000000000001,"15min":1.3360000000000001},"last":0.96899999999999997}]}]},{"osd":4,"up_from":126,"seq":541165879314,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":138584,"kb_used_data":1232,"kb_used_omap":10,"kb_used_meta":137333,"kb_avail":93606568,"statfs":{"total":95995035648,"available":95853125632,"internally_reserved":0,"allocated":1261568,"data_stored":777243,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":140629604},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:39:16 2023","interfaces":[{"interface":"back","average":{"1min":0.90500000000000003,"5min":0.90500000000000003,"15min":0.90500000000000003},"min":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"max":{"1min":1.3009999999999999,"5min":1.3009999999999999,"15min":1.3009999999999999},"last":1.115},{"interface":"front","average":{"1min":0.98999999999999999,"5min":0.98999999999999999,"15min":0.98999999999999999},"min":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"max":{"1min":1.4279999999999999,"5min":1.4279999999999999,"15min":1.4279999999999999},"last":0.77800000000000002}]},{"osd":1,"last update":"Sun Oct 1 10:39:16 2023","interfaces":[{"interface":"back","average":{"1min":0.92700000000000005,"5min":0.92700000000000005,"15min":0.92700000000000005},"min":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"max":{"1min":1.1850000000000001,"5min":1.1850000000000001,"15min":1.1850000000000001},"last":1.1859999999999999},{"interface":"front","average":{"1min":0.95399999999999996,"5min":0.95399999999999996,"15min":0.95399999999999996},"min":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"max":{"1min":1.2430000000000001,"5min":1.2430000000000001,"15min":1.2430000000000001},"last":0.83699999999999997}]},{"osd":2,"last update":"Sun Oct 1 10:39:16 2023","interfaces":[{"interface":"back","average":{"1min":0.95399999999999996,"5min":0.95399999999999996,"15min":0.95399999999999996},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":1.1970000000000001,"5min":1.1970000000000001,"15min":1.1970000000000001},"last":1.25},{"interface":"front","average":{"1min":0.97699999999999998,"5min":0.97699999999999998,"15min":0.97699999999999998},"min":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"max":{"1min":1.222,"5min":1.222,"15min":1.222},"last":1.2310000000000001}]},{"osd":3,"last update":"Sun Oct 1 10:39:16 2023","interfaces":[{"interface":"back","average":{"1min":1.0780000000000001,"5min":1.0780000000000001,"15min":1.0780000000000001},"min":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"max":{"1min":1.3320000000000001,"5min":1.3320000000000001,"15min":1.3320000000000001},"last":0.97399999999999998},{"interface":"front","average":{"1min":0.97899999999999998,"5min":0.97899999999999998,"15min":0.97899999999999998},"min":{"1min":0.628,"5min":0.628,"15min":0.628},"max":{"1min":1.296,"5min":1.296,"15min":1.296},"last":0.93500000000000005}]},{"osd":5,"last update":"Sun Oct 1 10:39:16 2023","interfaces":[{"interface":"back","average":{"1min":1.095,"5min":1.095,"15min":1.095},"min":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"max":{"1min":1.5089999999999999,"5min":1.5089999999999999,"15min":1.5089999999999999},"last":1.3220000000000001},{"interface":"front","average":{"1min":1.0940000000000001,"5min":1.0940000000000001,"15min":1.0940000000000001},"min":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"max":{"1min":1.3779999999999999,"5min":1.3779999999999999,"15min":1.3779999999999999},"last":1.3400000000000001}]},{"osd":6,"last update":"Sun Oct 1 10:39:16 2023","interfaces":[{"interface":"back","average":{"1min":1.139,"5min":1.139,"15min":1.139},"min":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"max":{"1min":1.579,"5min":1.579,"15min":1.579},"last":1.0700000000000001},{"interface":"front","average":{"1min":1.075,"5min":1.075,"15min":1.075},"min":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"max":{"1min":1.403,"5min":1.403,"15min":1.403},"last":1.04}]},{"osd":7,"last update":"Sun Oct 1 10:39:16 2023","interfaces":[{"interface":"back","average":{"1min":1.1819999999999999,"5min":1.1819999999999999,"15min":1.1819999999999999},"min":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"max":{"1min":1.387,"5min":1.387,"15min":1.387},"last":1.2929999999999999},{"interface":"front","average":{"1min":1.169,"5min":1.169,"15min":1.169},"min":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"max":{"1min":1.46,"5min":1.46,"15min":1.46},"last":1.3939999999999999}]}]},{"osd":5,"up_from":36,"seq":154618822995,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":594992,"kb_used_data":1192,"kb_used_omap":0,"kb_used_meta":593792,"kb_avail":93150160,"statfs":{"total":95995035648,"available":95385763840,"internally_reserved":0,"allocated":1220608,"data_stored":768137,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":608043008},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:38:37 2023","interfaces":[{"interface":"back","average":{"1min":0.86799999999999999,"5min":1.3160000000000001,"15min":1.2949999999999999},"min":{"1min":0.46300000000000002,"5min":0.27300000000000002,"15min":0.26400000000000001},"max":{"1min":1.175,"5min":10.429,"15min":10.429},"last":0.99399999999999999},{"interface":"front","average":{"1min":0.90900000000000003,"5min":1.54,"15min":1.524},"min":{"1min":0.59399999999999997,"5min":0.29299999999999998,"15min":0.29199999999999998},"max":{"1min":1.2809999999999999,"5min":13.018000000000001,"15min":13.018000000000001},"last":0.872}]},{"osd":1,"last update":"Sun Oct 1 10:38:37 2023","interfaces":[{"interface":"back","average":{"1min":0.874,"5min":1.768,"15min":1.72},"min":{"1min":0.63,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":1.149,"5min":10.858000000000001,"15min":10.858000000000001},"last":0.71499999999999997},{"interface":"front","average":{"1min":0.96499999999999997,"5min":2.0310000000000001,"15min":1.8109999999999999},"min":{"1min":0.623,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.4159999999999999,"5min":12.864000000000001,"15min":12.864000000000001},"last":1.137}]},{"osd":2,"last update":"Sun Oct 1 10:39:06 2023","interfaces":[{"interface":"back","average":{"1min":0.88100000000000001,"5min":0.96299999999999997,"15min":0.97699999999999998},"min":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"max":{"1min":1.272,"5min":1.3540000000000001,"15min":1.3540000000000001},"last":1.302},{"interface":"front","average":{"1min":0.97399999999999998,"5min":0.96599999999999997,"15min":0.96399999999999997},"min":{"1min":0.751,"5min":0.67600000000000005,"15min":0.67600000000000005},"max":{"1min":1.1879999999999999,"5min":1.272,"15min":1.272},"last":0.96999999999999997}]},{"osd":3,"last update":"Sun Oct 1 10:38:46 2023","interfaces":[{"interface":"back","average":{"1min":0.93300000000000005,"5min":0.93300000000000005,"15min":0.93300000000000005},"min":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"max":{"1min":1.3919999999999999,"5min":1.3919999999999999,"15min":1.3919999999999999},"last":1.0469999999999999},{"interface":"front","average":{"1min":0.93999999999999995,"5min":0.93999999999999995,"15min":0.93999999999999995},"min":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"max":{"1min":1.1879999999999999,"5min":1.1879999999999999,"15min":1.1879999999999999},"last":1.171}]},{"osd":4,"last update":"Sun Oct 1 10:39:16 2023","interfaces":[{"interface":"back","average":{"1min":1.077,"5min":1.077,"15min":1.077},"min":{"1min":0.753,"5min":0.753,"15min":0.753},"max":{"1min":1.343,"5min":1.343,"15min":1.343},"last":0.94299999999999995},{"interface":"front","average":{"1min":1.0840000000000001,"5min":1.0840000000000001,"15min":1.0840000000000001},"min":{"1min":0.89700000000000002,"5min":0.89700000000000002,"15min":0.89700000000000002},"max":{"1min":1.2989999999999999,"5min":1.2989999999999999,"15min":1.2989999999999999},"last":1.2210000000000001}]},{"osd":6,"last update":"Sun Oct 1 10:38:52 2023","interfaces":[{"interface":"back","average":{"1min":0.96599999999999997,"5min":1.298,"15min":1.298},"min":{"1min":0.42399999999999999,"5min":0.38500000000000001,"15min":0.33900000000000002},"max":{"1min":1.496,"5min":6.96,"15min":6.96},"last":0.91000000000000003},{"interface":"front","average":{"1min":0.90000000000000002,"5min":1.325,"15min":1.2330000000000001},"min":{"1min":0.437,"5min":0.312,"15min":0.312},"max":{"1min":1.377,"5min":8.4339999999999993,"15min":8.4339999999999993},"last":1.2769999999999999}]},{"osd":7,"last update":"Sun Oct 1 10:38:52 2023","interfaces":[{"interface":"back","average":{"1min":0.94799999999999995,"5min":1.4199999999999999,"15min":1.292},"min":{"1min":0.39000000000000001,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.3480000000000001,"5min":9.4629999999999992,"15min":9.4629999999999992},"last":1.0669999999999999},{"interface":"front","average":{"1min":1.0109999999999999,"5min":1.1899999999999999,"15min":1.3919999999999999},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":1.4279999999999999,"5min":5.4699999999999998,"15min":12.927},"last":1.109}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T10:39:37.923 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-10-01T10:39:37.923 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-10-01T10:39:37.923 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-10-01T10:39:37.923 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph health --format=json 2023-10-01T10:39:38.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:39:38 smithi078 bash[75299]: audit 2023-10-01T10:39:37.095949+0000 mgr.y (mgr.25072) 652 : audit 0 from='client.35568 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:39:38.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:39:38 smithi078 bash[75299]: cluster 2023-10-01T10:39:37.679497+0000 mgr.y (mgr.25072) 653 : cluster 0 pgmap v559: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:38.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:39:38 smithi078 bash[78274]: audit 2023-10-01T10:39:37.095949+0000 mgr.y (mgr.25072) 652 : audit 0 from='client.35568 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:39:38.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:39:38 smithi078 bash[78274]: cluster 2023-10-01T10:39:37.679497+0000 mgr.y (mgr.25072) 653 : cluster 0 pgmap v559: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:38.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:39:38 smithi173 bash[58816]: audit 2023-10-01T10:39:37.095949+0000 mgr.y (mgr.25072) 652 : audit 0 from='client.35568 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:39:38.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:39:38 smithi173 bash[58816]: cluster 2023-10-01T10:39:37.679497+0000 mgr.y (mgr.25072) 653 : cluster 0 pgmap v559: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:40.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:39:39 smithi078 bash[75299]: cluster 2023-10-01T10:39:39.680535+0000 mgr.y (mgr.25072) 654 : cluster 0 pgmap v560: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:40.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:39:39 smithi078 bash[78274]: cluster 2023-10-01T10:39:39.680535+0000 mgr.y (mgr.25072) 654 : cluster 0 pgmap v560: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:40.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:39:39 smithi173 bash[58816]: cluster 2023-10-01T10:39:39.680535+0000 mgr.y (mgr.25072) 654 : cluster 0 pgmap v560: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:40.582 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:39:40 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:39:40] "GET /metrics HTTP/1.1" 200 33822 "" "Prometheus/2.33.4" 2023-10-01T10:39:42.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:39:41 smithi078 bash[75299]: cluster 2023-10-01T10:39:41.681473+0000 mgr.y (mgr.25072) 655 : cluster 0 pgmap v561: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:42.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:39:41 smithi078 bash[78274]: cluster 2023-10-01T10:39:41.681473+0000 mgr.y (mgr.25072) 655 : cluster 0 pgmap v561: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:42.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:39:41 smithi173 bash[58816]: cluster 2023-10-01T10:39:41.681473+0000 mgr.y (mgr.25072) 655 : cluster 0 pgmap v561: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:42.746 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:39:45.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:39:44 smithi078 bash[75299]: cluster 2023-10-01T10:39:43.682200+0000 mgr.y (mgr.25072) 656 : cluster 0 pgmap v562: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:45.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:39:44 smithi078 bash[78274]: cluster 2023-10-01T10:39:43.682200+0000 mgr.y (mgr.25072) 656 : cluster 0 pgmap v562: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:45.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:39:44 smithi173 bash[58816]: cluster 2023-10-01T10:39:43.682200+0000 mgr.y (mgr.25072) 656 : cluster 0 pgmap v562: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:46.266 INFO:teuthology.orchestra.run.smithi078.stdout: 2023-10-01T10:39:46.267 INFO:teuthology.orchestra.run.smithi078.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-10-01T10:39:46.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:39:46 smithi078 bash[75299]: cluster 2023-10-01T10:39:45.683123+0000 mgr.y (mgr.25072) 657 : cluster 0 pgmap v563: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:46.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:39:46 smithi078 bash[75299]: audit 2023-10-01T10:39:46.264695+0000 mon.a (mon.0) 897 : audit 0 from='client.? 172.21.15.78:0/3582465605' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-01T10:39:46.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:39:46 smithi078 bash[78274]: cluster 2023-10-01T10:39:45.683123+0000 mgr.y (mgr.25072) 657 : cluster 0 pgmap v563: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:46.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:39:46 smithi078 bash[78274]: audit 2023-10-01T10:39:46.264695+0000 mon.a (mon.0) 897 : audit 0 from='client.? 172.21.15.78:0/3582465605' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-01T10:39:47.104 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-10-01T10:39:47.105 INFO:teuthology.run_tasks:Running task print... 2023-10-01T10:39:47.115 INFO:teuthology.task.print:**** applying stress + thrashing to mixed-version cluster **** 2023-10-01T10:39:47.115 INFO:teuthology.run_tasks:Running task parallel... 2023-10-01T10:39:47.124 INFO:teuthology.task.parallel:starting parallel... 2023-10-01T10:39:47.124 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-10-01T10:39:47.124 INFO:teuthology.task.sequential:In sequential, running task thrashosds... 2023-10-01T10:39:47.126 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2023-10-01T10:39:47.126 INFO:tasks.thrashosds:overrides is {'admin_socket': {'branch': 'reef'}, 'ceph': {'conf': {'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'b56822191ad0416d0c05ed59720df03ad9c2a44e'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'b56822191ad0416d0c05ed59720df03ad9c2a44e'}}, 'kernel': {'hwe': True}, 'workunit': {'branch': 'reef', 'sha1': 'fc1e90a395a6a5631142eb70fd9bf3d22740a9da'}} 2023-10-01T10:39:47.126 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2023-10-01T10:39:47.126 INFO:tasks.thrashosds:Beginning thrashosds... 2023-10-01T10:39:47.126 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-10-01T10:39:47.126 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T10:39:47.127 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:39:46 smithi173 bash[58816]: cluster 2023-10-01T10:39:45.683123+0000 mgr.y (mgr.25072) 657 : cluster 0 pgmap v563: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:47.127 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:39:46 smithi173 bash[58816]: audit 2023-10-01T10:39:46.264695+0000 mon.a (mon.0) 897 : audit 0 from='client.? 172.21.15.78:0/3582465605' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-10-01T10:39:48.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:39:48 smithi078 bash[78274]: audit 2023-10-01T10:39:47.410460+0000 mon.a (mon.0) 898 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:39:48.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:39:48 smithi078 bash[75299]: audit 2023-10-01T10:39:47.410460+0000 mon.a (mon.0) 898 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:39:48.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:39:48 smithi173 bash[58816]: audit 2023-10-01T10:39:47.410460+0000 mon.a (mon.0) 898 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:39:49.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:39:49 smithi078 bash[75299]: cluster 2023-10-01T10:39:47.683878+0000 mgr.y (mgr.25072) 658 : cluster 0 pgmap v564: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:49.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:39:49 smithi078 bash[78274]: cluster 2023-10-01T10:39:47.683878+0000 mgr.y (mgr.25072) 658 : cluster 0 pgmap v564: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:49.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:39:49 smithi173 bash[58816]: cluster 2023-10-01T10:39:47.683878+0000 mgr.y (mgr.25072) 658 : cluster 0 pgmap v564: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:50.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:39:50 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:39:50] "GET /metrics HTTP/1.1" 200 33824 "" "Prometheus/2.33.4" 2023-10-01T10:39:51.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:39:51 smithi078 bash[75299]: cluster 2023-10-01T10:39:49.684837+0000 mgr.y (mgr.25072) 659 : cluster 0 pgmap v565: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:51.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:39:51 smithi078 bash[78274]: cluster 2023-10-01T10:39:49.684837+0000 mgr.y (mgr.25072) 659 : cluster 0 pgmap v565: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:51.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:39:51 smithi173 bash[58816]: cluster 2023-10-01T10:39:49.684837+0000 mgr.y (mgr.25072) 659 : cluster 0 pgmap v565: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:51.903 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:39:53.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:39:53 smithi078 bash[75299]: cluster 2023-10-01T10:39:51.685737+0000 mgr.y (mgr.25072) 660 : cluster 0 pgmap v566: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:53.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:39:53 smithi078 bash[78274]: cluster 2023-10-01T10:39:51.685737+0000 mgr.y (mgr.25072) 660 : cluster 0 pgmap v566: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:53.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:39:53 smithi173 bash[58816]: cluster 2023-10-01T10:39:51.685737+0000 mgr.y (mgr.25072) 660 : cluster 0 pgmap v566: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:55.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:39:55 smithi078 bash[75299]: cluster 2023-10-01T10:39:53.686483+0000 mgr.y (mgr.25072) 661 : cluster 0 pgmap v567: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:55.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:39:55 smithi078 bash[78274]: cluster 2023-10-01T10:39:53.686483+0000 mgr.y (mgr.25072) 661 : cluster 0 pgmap v567: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:55.340 INFO:teuthology.orchestra.run.smithi078.stdout: 2023-10-01T10:39:55.342 INFO:teuthology.orchestra.run.smithi078.stderr:dumped all 2023-10-01T10:39:55.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:39:55 smithi173 bash[58816]: cluster 2023-10-01T10:39:53.686483+0000 mgr.y (mgr.25072) 661 : cluster 0 pgmap v567: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:56.156 INFO:teuthology.orchestra.run.smithi078.stdout:{"pg_ready":true,"pg_map":{"version":567,"stamp":"2023-10-01T10:39:53.685997+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":261,"num_write_kb":2446,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":195,"ondisk_log_size":195,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2148588,"kb_used_data":11692,"kb_used_omap":30,"kb_used_meta":2136801,"kb_avail":747812628,"statfs":{"total":767960285184,"available":765760131072,"internally_reserved":0,"allocated":11972608,"data_stored":7995856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31502,"internal_metadata":2188084466},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.005140"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":113,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.032258+0000","last_change":"2023-10-01T10:38:16.032258+0000","last_active":"2023-10-01T10:38:16.032258+0000","last_peered":"2023-10-01T10:38:16.032258+0000","last_clean":"2023-10-01T10:38:16.032258+0000","last_became_active":"2023-10-01T10:38:16.031633+0000","last_became_peered":"2023-10-01T10:38:16.031633+0000","last_unstale":"2023-10-01T10:38:16.032258+0000","last_undegraded":"2023-10-01T10:38:16.032258+0000","last_fullsized":"2023-10-01T10:38:16.032258+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:17:02.446430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":101,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.025881+0000","last_change":"2023-10-01T10:38:16.025881+0000","last_active":"2023-10-01T10:38:16.025881+0000","last_peered":"2023-10-01T10:38:16.025881+0000","last_clean":"2023-10-01T10:38:16.025881+0000","last_became_active":"2023-10-01T10:38:16.025518+0000","last_became_peered":"2023-10-01T10:38:16.025518+0000","last_unstale":"2023-10-01T10:38:16.025881+0000","last_undegraded":"2023-10-01T10:38:16.025881+0000","last_fullsized":"2023-10-01T10:38:16.025881+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:48:37.826604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":80,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.025648+0000","last_change":"2023-10-01T10:38:16.025075+0000","last_active":"2023-10-01T10:38:16.025648+0000","last_peered":"2023-10-01T10:38:16.025648+0000","last_clean":"2023-10-01T10:38:16.025648+0000","last_became_active":"2023-10-01T10:38:16.024735+0000","last_became_peered":"2023-10-01T10:38:16.024735+0000","last_unstale":"2023-10-01T10:38:16.025648+0000","last_undegraded":"2023-10-01T10:38:16.025648+0000","last_fullsized":"2023-10-01T10:38:16.025648+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:38:01.450741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":83,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.019448+0000","last_change":"2023-10-01T10:37:42.772008+0000","last_active":"2023-10-01T10:38:16.019448+0000","last_peered":"2023-10-01T10:38:16.019448+0000","last_clean":"2023-10-01T10:38:16.019448+0000","last_became_active":"2023-10-01T10:37:42.771244+0000","last_became_peered":"2023-10-01T10:37:42.771244+0000","last_unstale":"2023-10-01T10:38:16.019448+0000","last_undegraded":"2023-10-01T10:38:16.019448+0000","last_fullsized":"2023-10-01T10:38:16.019448+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:38:55.180790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":102,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.010782+0000","last_change":"2023-10-01T10:37:42.766600+0000","last_active":"2023-10-01T10:38:15.010782+0000","last_peered":"2023-10-01T10:38:15.010782+0000","last_clean":"2023-10-01T10:38:15.010782+0000","last_became_active":"2023-10-01T10:37:42.766051+0000","last_became_peered":"2023-10-01T10:37:42.766051+0000","last_unstale":"2023-10-01T10:38:15.010782+0000","last_undegraded":"2023-10-01T10:38:15.010782+0000","last_fullsized":"2023-10-01T10:38:15.010782+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:51:17.707963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":92,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.031873+0000","last_change":"2023-10-01T10:32:31.575109+0000","last_active":"2023-10-01T10:38:15.031873+0000","last_peered":"2023-10-01T10:38:15.031873+0000","last_clean":"2023-10-01T10:38:15.031873+0000","last_became_active":"2023-10-01T10:32:31.574729+0000","last_became_peered":"2023-10-01T10:32:31.574729+0000","last_unstale":"2023-10-01T10:38:15.031873+0000","last_undegraded":"2023-10-01T10:38:15.031873+0000","last_fullsized":"2023-10-01T10:38:15.031873+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":81,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.014432+0000","last_change":"2023-10-01T10:37:42.766667+0000","last_active":"2023-10-01T10:38:15.014432+0000","last_peered":"2023-10-01T10:38:15.014432+0000","last_clean":"2023-10-01T10:38:15.014432+0000","last_became_active":"2023-10-01T10:37:42.766369+0000","last_became_peered":"2023-10-01T10:37:42.766369+0000","last_unstale":"2023-10-01T10:38:15.014432+0000","last_undegraded":"2023-10-01T10:38:15.014432+0000","last_fullsized":"2023-10-01T10:38:15.014432+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:02.708788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":86,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.010584+0000","last_change":"2023-10-01T10:37:42.765925+0000","last_active":"2023-10-01T10:38:15.010584+0000","last_peered":"2023-10-01T10:38:15.010584+0000","last_clean":"2023-10-01T10:38:15.010584+0000","last_became_active":"2023-10-01T10:37:42.765218+0000","last_became_peered":"2023-10-01T10:37:42.765218+0000","last_unstale":"2023-10-01T10:38:15.010584+0000","last_undegraded":"2023-10-01T10:38:15.010584+0000","last_fullsized":"2023-10-01T10:38:15.010584+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T22:10:34.338031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":94,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.034622+0000","last_change":"2023-10-01T10:32:31.579805+0000","last_active":"2023-10-01T10:38:15.034622+0000","last_peered":"2023-10-01T10:38:15.034622+0000","last_clean":"2023-10-01T10:38:15.034622+0000","last_became_active":"2023-10-01T10:32:31.579423+0000","last_became_peered":"2023-10-01T10:32:31.579423+0000","last_unstale":"2023-10-01T10:38:15.034622+0000","last_undegraded":"2023-10-01T10:38:15.034622+0000","last_fullsized":"2023-10-01T10:38:15.034622+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":70,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.014763+0000","last_change":"2023-10-01T10:37:42.772531+0000","last_active":"2023-10-01T10:38:15.014763+0000","last_peered":"2023-10-01T10:38:15.014763+0000","last_clean":"2023-10-01T10:38:15.014763+0000","last_became_active":"2023-10-01T10:37:42.772216+0000","last_became_peered":"2023-10-01T10:37:42.772216+0000","last_unstale":"2023-10-01T10:38:15.014763+0000","last_undegraded":"2023-10-01T10:38:15.014763+0000","last_fullsized":"2023-10-01T10:38:15.014763+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:01:53.373628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":112,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.027085+0000","last_change":"2023-10-01T10:38:16.027085+0000","last_active":"2023-10-01T10:38:16.027085+0000","last_peered":"2023-10-01T10:38:16.027085+0000","last_clean":"2023-10-01T10:38:16.027085+0000","last_became_active":"2023-10-01T10:38:16.026555+0000","last_became_peered":"2023-10-01T10:38:16.026555+0000","last_unstale":"2023-10-01T10:38:16.027085+0000","last_undegraded":"2023-10-01T10:38:16.027085+0000","last_fullsized":"2023-10-01T10:38:16.027085+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":65,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.020849+0000","last_change":"2023-10-01T10:37:05.018755+0000","last_active":"2023-10-01T10:38:16.020849+0000","last_peered":"2023-10-01T10:38:16.020849+0000","last_clean":"2023-10-01T10:38:16.020849+0000","last_became_active":"2023-10-01T10:37:05.018446+0000","last_became_peered":"2023-10-01T10:37:05.018446+0000","last_unstale":"2023-10-01T10:38:16.020849+0000","last_undegraded":"2023-10-01T10:38:16.020849+0000","last_fullsized":"2023-10-01T10:38:16.020849+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:13:53.740503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":107,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.030826+0000","last_change":"2023-10-01T10:38:16.030826+0000","last_active":"2023-10-01T10:38:16.030826+0000","last_peered":"2023-10-01T10:38:16.030826+0000","last_clean":"2023-10-01T10:38:16.030826+0000","last_became_active":"2023-10-01T10:38:16.030277+0000","last_became_peered":"2023-10-01T10:38:16.030277+0000","last_unstale":"2023-10-01T10:38:16.030826+0000","last_undegraded":"2023-10-01T10:38:16.030826+0000","last_fullsized":"2023-10-01T10:38:16.030826+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:31:43.040424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":81,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.034741+0000","last_change":"2023-10-01T10:38:16.034741+0000","last_active":"2023-10-01T10:38:16.034741+0000","last_peered":"2023-10-01T10:38:16.034741+0000","last_clean":"2023-10-01T10:38:16.034741+0000","last_became_active":"2023-10-01T10:38:16.034407+0000","last_became_peered":"2023-10-01T10:38:16.034407+0000","last_unstale":"2023-10-01T10:38:16.034741+0000","last_undegraded":"2023-10-01T10:38:16.034741+0000","last_fullsized":"2023-10-01T10:38:16.034741+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:47:29.911477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":111,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.026777+0000","last_change":"2023-10-01T10:38:16.026777+0000","last_active":"2023-10-01T10:38:16.026777+0000","last_peered":"2023-10-01T10:38:16.026777+0000","last_clean":"2023-10-01T10:38:16.026777+0000","last_became_active":"2023-10-01T10:38:16.026371+0000","last_became_peered":"2023-10-01T10:38:16.026371+0000","last_unstale":"2023-10-01T10:38:16.026777+0000","last_undegraded":"2023-10-01T10:38:16.026777+0000","last_fullsized":"2023-10-01T10:38:16.026777+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":109,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.012172+0000","last_change":"2023-10-01T10:37:42.772319+0000","last_active":"2023-10-01T10:38:15.012172+0000","last_peered":"2023-10-01T10:38:15.012172+0000","last_clean":"2023-10-01T10:38:15.012172+0000","last_became_active":"2023-10-01T10:37:42.771977+0000","last_became_peered":"2023-10-01T10:37:42.771977+0000","last_unstale":"2023-10-01T10:38:15.012172+0000","last_undegraded":"2023-10-01T10:38:15.012172+0000","last_fullsized":"2023-10-01T10:38:15.012172+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:45:12.783214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":81,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.014954+0000","last_change":"2023-10-01T10:37:42.767706+0000","last_active":"2023-10-01T10:38:15.014954+0000","last_peered":"2023-10-01T10:38:15.014954+0000","last_clean":"2023-10-01T10:38:15.014954+0000","last_became_active":"2023-10-01T10:37:42.767211+0000","last_became_peered":"2023-10-01T10:37:42.767211+0000","last_unstale":"2023-10-01T10:38:15.014954+0000","last_undegraded":"2023-10-01T10:38:15.014954+0000","last_fullsized":"2023-10-01T10:38:15.014954+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:37:20.357407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":93,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.034746+0000","last_change":"2023-10-01T10:37:42.768642+0000","last_active":"2023-10-01T10:38:15.034746+0000","last_peered":"2023-10-01T10:38:15.034746+0000","last_clean":"2023-10-01T10:38:15.034746+0000","last_became_active":"2023-10-01T10:37:42.768239+0000","last_became_peered":"2023-10-01T10:37:42.768239+0000","last_unstale":"2023-10-01T10:38:15.034746+0000","last_undegraded":"2023-10-01T10:38:15.034746+0000","last_fullsized":"2023-10-01T10:38:15.034746+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":94,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.028563+0000","last_change":"2023-10-01T10:38:16.028563+0000","last_active":"2023-10-01T10:38:16.028563+0000","last_peered":"2023-10-01T10:38:16.028563+0000","last_clean":"2023-10-01T10:38:16.028563+0000","last_became_active":"2023-10-01T10:38:16.028162+0000","last_became_peered":"2023-10-01T10:38:16.028162+0000","last_unstale":"2023-10-01T10:38:16.028563+0000","last_undegraded":"2023-10-01T10:38:16.028563+0000","last_fullsized":"2023-10-01T10:38:16.028563+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":82,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.014770+0000","last_change":"2023-10-01T10:37:42.772247+0000","last_active":"2023-10-01T10:38:15.014770+0000","last_peered":"2023-10-01T10:38:15.014770+0000","last_clean":"2023-10-01T10:38:15.014770+0000","last_became_active":"2023-10-01T10:37:42.771697+0000","last_became_peered":"2023-10-01T10:37:42.771697+0000","last_unstale":"2023-10-01T10:38:15.014770+0000","last_undegraded":"2023-10-01T10:38:15.014770+0000","last_fullsized":"2023-10-01T10:38:15.014770+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:23:15.018248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":101,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.030438+0000","last_change":"2023-10-01T10:28:20.211798+0000","last_active":"2023-10-01T10:38:15.030438+0000","last_peered":"2023-10-01T10:38:15.030438+0000","last_clean":"2023-10-01T10:38:15.030438+0000","last_became_active":"2023-10-01T10:28:20.210791+0000","last_became_peered":"2023-10-01T10:28:20.210791+0000","last_unstale":"2023-10-01T10:38:15.030438+0000","last_undegraded":"2023-10-01T10:38:15.030438+0000","last_fullsized":"2023-10-01T10:38:15.030438+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":86,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.012486+0000","last_change":"2023-10-01T10:37:42.773127+0000","last_active":"2023-10-01T10:38:15.012486+0000","last_peered":"2023-10-01T10:38:15.012486+0000","last_clean":"2023-10-01T10:38:15.012486+0000","last_became_active":"2023-10-01T10:37:42.772102+0000","last_became_peered":"2023-10-01T10:37:42.772102+0000","last_unstale":"2023-10-01T10:38:15.012486+0000","last_undegraded":"2023-10-01T10:38:15.012486+0000","last_fullsized":"2023-10-01T10:38:15.012486+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:23:26.202295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":110,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.026955+0000","last_change":"2023-10-01T10:38:16.026955+0000","last_active":"2023-10-01T10:38:16.026955+0000","last_peered":"2023-10-01T10:38:16.026955+0000","last_clean":"2023-10-01T10:38:16.026955+0000","last_became_active":"2023-10-01T10:38:16.026552+0000","last_became_peered":"2023-10-01T10:38:16.026552+0000","last_unstale":"2023-10-01T10:38:16.026955+0000","last_undegraded":"2023-10-01T10:38:16.026955+0000","last_fullsized":"2023-10-01T10:38:16.026955+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":78,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.031063+0000","last_change":"2023-10-01T10:15:40.318702+0000","last_active":"2023-10-01T10:38:15.031063+0000","last_peered":"2023-10-01T10:38:15.031063+0000","last_clean":"2023-10-01T10:38:15.031063+0000","last_became_active":"2023-10-01T10:15:40.318333+0000","last_became_peered":"2023-10-01T10:15:40.318333+0000","last_unstale":"2023-10-01T10:38:15.031063+0000","last_undegraded":"2023-10-01T10:38:15.031063+0000","last_fullsized":"2023-10-01T10:38:15.031063+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":111,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.027753+0000","last_change":"2023-10-01T10:38:16.027753+0000","last_active":"2023-10-01T10:38:16.027753+0000","last_peered":"2023-10-01T10:38:16.027753+0000","last_clean":"2023-10-01T10:38:16.027753+0000","last_became_active":"2023-10-01T10:38:16.027384+0000","last_became_peered":"2023-10-01T10:38:16.027384+0000","last_unstale":"2023-10-01T10:38:16.027753+0000","last_undegraded":"2023-10-01T10:38:16.027753+0000","last_fullsized":"2023-10-01T10:38:16.027753+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":115,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.033203+0000","last_change":"2023-10-01T10:38:16.033203+0000","last_active":"2023-10-01T10:38:16.033203+0000","last_peered":"2023-10-01T10:38:16.033203+0000","last_clean":"2023-10-01T10:38:16.033203+0000","last_became_active":"2023-10-01T10:38:16.032681+0000","last_became_peered":"2023-10-01T10:38:16.032681+0000","last_unstale":"2023-10-01T10:38:16.033203+0000","last_undegraded":"2023-10-01T10:38:16.033203+0000","last_fullsized":"2023-10-01T10:38:16.033203+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:42:36.642284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":80,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.023250+0000","last_change":"2023-10-01T10:15:39.293587+0000","last_active":"2023-10-01T10:38:15.023250+0000","last_peered":"2023-10-01T10:38:15.023250+0000","last_clean":"2023-10-01T10:38:15.023250+0000","last_became_active":"2023-10-01T10:15:39.293273+0000","last_became_peered":"2023-10-01T10:15:39.293273+0000","last_unstale":"2023-10-01T10:38:15.023250+0000","last_undegraded":"2023-10-01T10:38:15.023250+0000","last_fullsized":"2023-10-01T10:38:15.023250+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":92,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.031078+0000","last_change":"2023-10-01T10:32:31.571971+0000","last_active":"2023-10-01T10:38:15.031078+0000","last_peered":"2023-10-01T10:38:15.031078+0000","last_clean":"2023-10-01T10:38:15.031078+0000","last_became_active":"2023-10-01T10:32:31.571410+0000","last_became_peered":"2023-10-01T10:32:31.571410+0000","last_unstale":"2023-10-01T10:38:15.031078+0000","last_undegraded":"2023-10-01T10:38:15.031078+0000","last_fullsized":"2023-10-01T10:38:15.031078+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":94,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.034665+0000","last_change":"2023-10-01T10:37:42.765733+0000","last_active":"2023-10-01T10:38:15.034665+0000","last_peered":"2023-10-01T10:38:15.034665+0000","last_clean":"2023-10-01T10:38:15.034665+0000","last_became_active":"2023-10-01T10:37:42.765311+0000","last_became_peered":"2023-10-01T10:37:42.765311+0000","last_unstale":"2023-10-01T10:38:15.034665+0000","last_undegraded":"2023-10-01T10:38:15.034665+0000","last_fullsized":"2023-10-01T10:38:15.034665+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":98,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.027301+0000","last_change":"2023-10-01T10:38:16.026348+0000","last_active":"2023-10-01T10:38:16.027301+0000","last_peered":"2023-10-01T10:38:16.027301+0000","last_clean":"2023-10-01T10:38:16.027301+0000","last_became_active":"2023-10-01T10:38:16.025747+0000","last_became_peered":"2023-10-01T10:38:16.025747+0000","last_unstale":"2023-10-01T10:38:16.027301+0000","last_undegraded":"2023-10-01T10:38:16.027301+0000","last_fullsized":"2023-10-01T10:38:16.027301+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:28:18.934480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":89,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.022493+0000","last_change":"2023-10-01T10:38:16.022493+0000","last_active":"2023-10-01T10:38:16.022493+0000","last_peered":"2023-10-01T10:38:16.022493+0000","last_clean":"2023-10-01T10:38:16.022493+0000","last_became_active":"2023-10-01T10:38:16.022033+0000","last_became_peered":"2023-10-01T10:38:16.022033+0000","last_unstale":"2023-10-01T10:38:16.022493+0000","last_undegraded":"2023-10-01T10:38:16.022493+0000","last_fullsized":"2023-10-01T10:38:16.022493+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:49:05.262763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":100,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.030613+0000","last_change":"2023-10-01T10:38:16.030613+0000","last_active":"2023-10-01T10:38:16.030613+0000","last_peered":"2023-10-01T10:38:16.030613+0000","last_clean":"2023-10-01T10:38:16.030613+0000","last_became_active":"2023-10-01T10:38:16.028975+0000","last_became_peered":"2023-10-01T10:38:16.028975+0000","last_unstale":"2023-10-01T10:38:16.030613+0000","last_undegraded":"2023-10-01T10:38:16.030613+0000","last_fullsized":"2023-10-01T10:38:16.030613+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:46:07.287911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":108,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.030250+0000","last_change":"2023-10-01T10:38:16.030250+0000","last_active":"2023-10-01T10:38:16.030250+0000","last_peered":"2023-10-01T10:38:16.030250+0000","last_clean":"2023-10-01T10:38:16.030250+0000","last_became_active":"2023-10-01T10:38:16.029898+0000","last_became_peered":"2023-10-01T10:38:16.029898+0000","last_unstale":"2023-10-01T10:38:16.030250+0000","last_undegraded":"2023-10-01T10:38:16.030250+0000","last_fullsized":"2023-10-01T10:38:16.030250+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":105,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.035906+0000","last_change":"2023-10-01T10:38:16.035906+0000","last_active":"2023-10-01T10:38:16.035906+0000","last_peered":"2023-10-01T10:38:16.035906+0000","last_clean":"2023-10-01T10:38:16.035906+0000","last_became_active":"2023-10-01T10:38:16.035383+0000","last_became_peered":"2023-10-01T10:38:16.035383+0000","last_unstale":"2023-10-01T10:38:16.035906+0000","last_undegraded":"2023-10-01T10:38:16.035906+0000","last_fullsized":"2023-10-01T10:38:16.035906+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:44:41.500097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":97,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.026611+0000","last_change":"2023-10-01T10:38:16.026611+0000","last_active":"2023-10-01T10:38:16.026611+0000","last_peered":"2023-10-01T10:38:16.026611+0000","last_clean":"2023-10-01T10:38:16.026611+0000","last_became_active":"2023-10-01T10:38:16.026267+0000","last_became_peered":"2023-10-01T10:38:16.026267+0000","last_unstale":"2023-10-01T10:38:16.026611+0000","last_undegraded":"2023-10-01T10:38:16.026611+0000","last_fullsized":"2023-10-01T10:38:16.026611+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:06.135288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":81,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.032162+0000","last_change":"2023-10-01T10:38:16.032162+0000","last_active":"2023-10-01T10:38:16.032162+0000","last_peered":"2023-10-01T10:38:16.032162+0000","last_clean":"2023-10-01T10:38:16.032162+0000","last_became_active":"2023-10-01T10:38:16.031785+0000","last_became_peered":"2023-10-01T10:38:16.031785+0000","last_unstale":"2023-10-01T10:38:16.032162+0000","last_undegraded":"2023-10-01T10:38:16.032162+0000","last_fullsized":"2023-10-01T10:38:16.032162+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:38:43.003626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":88,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.012059+0000","last_change":"2023-10-01T10:37:42.772398+0000","last_active":"2023-10-01T10:38:16.012059+0000","last_peered":"2023-10-01T10:38:16.012059+0000","last_clean":"2023-10-01T10:38:16.012059+0000","last_became_active":"2023-10-01T10:37:42.772084+0000","last_became_peered":"2023-10-01T10:37:42.772084+0000","last_unstale":"2023-10-01T10:38:16.012059+0000","last_undegraded":"2023-10-01T10:38:16.012059+0000","last_fullsized":"2023-10-01T10:38:16.012059+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:32:43.846419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":106,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.021873+0000","last_change":"2023-10-01T10:38:16.019755+0000","last_active":"2023-10-01T10:38:16.021873+0000","last_peered":"2023-10-01T10:38:16.021873+0000","last_clean":"2023-10-01T10:38:16.021873+0000","last_became_active":"2023-10-01T10:38:16.019397+0000","last_became_peered":"2023-10-01T10:38:16.019397+0000","last_unstale":"2023-10-01T10:38:16.021873+0000","last_undegraded":"2023-10-01T10:38:16.021873+0000","last_fullsized":"2023-10-01T10:38:16.021873+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:42:56.623225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":99,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.020037+0000","last_change":"2023-10-01T10:32:31.561108+0000","last_active":"2023-10-01T10:38:16.020037+0000","last_peered":"2023-10-01T10:38:16.020037+0000","last_clean":"2023-10-01T10:38:16.020037+0000","last_became_active":"2023-10-01T10:32:31.560791+0000","last_became_peered":"2023-10-01T10:32:31.560791+0000","last_unstale":"2023-10-01T10:38:16.020037+0000","last_undegraded":"2023-10-01T10:38:16.020037+0000","last_fullsized":"2023-10-01T10:38:16.020037+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:42:47.477671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":109,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.034655+0000","last_change":"2023-10-01T10:37:42.766814+0000","last_active":"2023-10-01T10:38:15.034655+0000","last_peered":"2023-10-01T10:38:15.034655+0000","last_clean":"2023-10-01T10:38:15.034655+0000","last_became_active":"2023-10-01T10:37:42.766451+0000","last_became_peered":"2023-10-01T10:37:42.766451+0000","last_unstale":"2023-10-01T10:38:15.034655+0000","last_undegraded":"2023-10-01T10:38:15.034655+0000","last_fullsized":"2023-10-01T10:38:15.034655+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":65,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.035088+0000","last_change":"2023-10-01T10:38:16.034357+0000","last_active":"2023-10-01T10:38:16.035088+0000","last_peered":"2023-10-01T10:38:16.035088+0000","last_clean":"2023-10-01T10:38:16.035088+0000","last_became_active":"2023-10-01T10:38:16.033905+0000","last_became_peered":"2023-10-01T10:38:16.033905+0000","last_unstale":"2023-10-01T10:38:16.035088+0000","last_undegraded":"2023-10-01T10:38:16.035088+0000","last_fullsized":"2023-10-01T10:38:16.035088+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:47:58.369835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":98,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.034322+0000","last_change":"2023-10-01T10:32:31.572482+0000","last_active":"2023-10-01T10:38:15.034322+0000","last_peered":"2023-10-01T10:38:15.034322+0000","last_clean":"2023-10-01T10:38:15.034322+0000","last_became_active":"2023-10-01T10:32:31.571993+0000","last_became_peered":"2023-10-01T10:32:31.571993+0000","last_unstale":"2023-10-01T10:38:15.034322+0000","last_undegraded":"2023-10-01T10:38:15.034322+0000","last_fullsized":"2023-10-01T10:38:15.034322+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":67,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.030202+0000","last_change":"2023-10-01T10:38:16.030202+0000","last_active":"2023-10-01T10:38:16.030202+0000","last_peered":"2023-10-01T10:38:16.030202+0000","last_clean":"2023-10-01T10:38:16.030202+0000","last_became_active":"2023-10-01T10:38:16.029917+0000","last_became_peered":"2023-10-01T10:38:16.029917+0000","last_unstale":"2023-10-01T10:38:16.030202+0000","last_undegraded":"2023-10-01T10:38:16.030202+0000","last_fullsized":"2023-10-01T10:38:16.030202+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:20:06.763909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":80,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.020177+0000","last_change":"2023-10-01T10:37:42.761153+0000","last_active":"2023-10-01T10:38:16.020177+0000","last_peered":"2023-10-01T10:38:16.020177+0000","last_clean":"2023-10-01T10:38:16.020177+0000","last_became_active":"2023-10-01T10:37:42.760715+0000","last_became_peered":"2023-10-01T10:37:42.760715+0000","last_unstale":"2023-10-01T10:38:16.020177+0000","last_undegraded":"2023-10-01T10:38:16.020177+0000","last_fullsized":"2023-10-01T10:38:16.020177+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:52:50.237174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":97,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.013202+0000","last_change":"2023-10-01T10:37:42.762524+0000","last_active":"2023-10-01T10:38:16.013202+0000","last_peered":"2023-10-01T10:38:16.013202+0000","last_clean":"2023-10-01T10:38:16.013202+0000","last_became_active":"2023-10-01T10:37:42.762209+0000","last_became_peered":"2023-10-01T10:37:42.762209+0000","last_unstale":"2023-10-01T10:38:16.013202+0000","last_undegraded":"2023-10-01T10:38:16.013202+0000","last_fullsized":"2023-10-01T10:38:16.013202+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:45:38.171397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":65,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.020920+0000","last_change":"2023-10-01T10:37:05.019330+0000","last_active":"2023-10-01T10:38:16.020920+0000","last_peered":"2023-10-01T10:38:16.020920+0000","last_clean":"2023-10-01T10:38:16.020920+0000","last_became_active":"2023-10-01T10:37:05.018842+0000","last_became_peered":"2023-10-01T10:37:05.018842+0000","last_unstale":"2023-10-01T10:38:16.020920+0000","last_undegraded":"2023-10-01T10:38:16.020920+0000","last_fullsized":"2023-10-01T10:38:16.020920+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:48:09.551917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":94,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.019778+0000","last_change":"2023-10-01T10:32:31.569162+0000","last_active":"2023-10-01T10:38:16.019778+0000","last_peered":"2023-10-01T10:38:16.019778+0000","last_clean":"2023-10-01T10:38:16.019778+0000","last_became_active":"2023-10-01T10:32:31.568956+0000","last_became_peered":"2023-10-01T10:32:31.568956+0000","last_unstale":"2023-10-01T10:38:16.019778+0000","last_undegraded":"2023-10-01T10:38:16.019778+0000","last_fullsized":"2023-10-01T10:38:16.019778+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:02:08.285304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":98,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.033675+0000","last_change":"2023-10-01T10:37:05.024009+0000","last_active":"2023-10-01T10:38:15.033675+0000","last_peered":"2023-10-01T10:38:15.033675+0000","last_clean":"2023-10-01T10:38:15.033675+0000","last_became_active":"2023-10-01T10:37:05.023458+0000","last_became_peered":"2023-10-01T10:37:05.023458+0000","last_unstale":"2023-10-01T10:38:15.033675+0000","last_undegraded":"2023-10-01T10:38:15.033675+0000","last_fullsized":"2023-10-01T10:38:15.033675+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'139","reported_seq":330,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.019558+0000","last_change":"2023-10-01T10:28:20.217386+0000","last_active":"2023-10-01T10:38:15.019558+0000","last_peered":"2023-10-01T10:38:15.019558+0000","last_clean":"2023-10-01T10:38:15.019558+0000","last_became_active":"2023-10-01T10:28:20.216803+0000","last_became_peered":"2023-10-01T10:28:20.216803+0000","last_unstale":"2023-10-01T10:38:15.019558+0000","last_undegraded":"2023-10-01T10:38:15.019558+0000","last_fullsized":"2023-10-01T10:38:15.019558+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:09:17.221082+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:09:17.221082+0000","last_clean_scrub_stamp":"2023-10-01T10:09:17.221082+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":70,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.011905+0000","last_change":"2023-10-01T10:37:42.765100+0000","last_active":"2023-10-01T10:38:16.011905+0000","last_peered":"2023-10-01T10:38:16.011905+0000","last_clean":"2023-10-01T10:38:16.011905+0000","last_became_active":"2023-10-01T10:37:42.764761+0000","last_became_peered":"2023-10-01T10:37:42.764761+0000","last_unstale":"2023-10-01T10:38:16.011905+0000","last_undegraded":"2023-10-01T10:38:16.011905+0000","last_fullsized":"2023-10-01T10:38:16.011905+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:24:56.907377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":92,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.031255+0000","last_change":"2023-10-01T10:38:16.031255+0000","last_active":"2023-10-01T10:38:16.031255+0000","last_peered":"2023-10-01T10:38:16.031255+0000","last_clean":"2023-10-01T10:38:16.031255+0000","last_became_active":"2023-10-01T10:38:16.030886+0000","last_became_peered":"2023-10-01T10:38:16.030886+0000","last_unstale":"2023-10-01T10:38:16.031255+0000","last_undegraded":"2023-10-01T10:38:16.031255+0000","last_fullsized":"2023-10-01T10:38:16.031255+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":113,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.019451+0000","last_change":"2023-10-01T10:32:31.565868+0000","last_active":"2023-10-01T10:38:15.019451+0000","last_peered":"2023-10-01T10:38:15.019451+0000","last_clean":"2023-10-01T10:38:15.019451+0000","last_became_active":"2023-10-01T10:32:31.565428+0000","last_became_peered":"2023-10-01T10:32:31.565428+0000","last_unstale":"2023-10-01T10:38:15.019451+0000","last_undegraded":"2023-10-01T10:38:15.019451+0000","last_fullsized":"2023-10-01T10:38:15.019451+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":97,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.022511+0000","last_change":"2023-10-01T10:38:16.021968+0000","last_active":"2023-10-01T10:38:16.022511+0000","last_peered":"2023-10-01T10:38:16.022511+0000","last_clean":"2023-10-01T10:38:16.022511+0000","last_became_active":"2023-10-01T10:38:16.021539+0000","last_became_peered":"2023-10-01T10:38:16.021539+0000","last_unstale":"2023-10-01T10:38:16.022511+0000","last_undegraded":"2023-10-01T10:38:16.022511+0000","last_fullsized":"2023-10-01T10:38:16.022511+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:12:53.105747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":98,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.019580+0000","last_change":"2023-10-01T10:37:05.017431+0000","last_active":"2023-10-01T10:38:16.019580+0000","last_peered":"2023-10-01T10:38:16.019580+0000","last_clean":"2023-10-01T10:38:16.019580+0000","last_became_active":"2023-10-01T10:37:05.017094+0000","last_became_peered":"2023-10-01T10:37:05.017094+0000","last_unstale":"2023-10-01T10:38:16.019580+0000","last_undegraded":"2023-10-01T10:38:16.019580+0000","last_fullsized":"2023-10-01T10:38:16.019580+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:10:40.664686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":113,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.032774+0000","last_change":"2023-10-01T10:38:16.032774+0000","last_active":"2023-10-01T10:38:16.032774+0000","last_peered":"2023-10-01T10:38:16.032774+0000","last_clean":"2023-10-01T10:38:16.032774+0000","last_became_active":"2023-10-01T10:38:16.032283+0000","last_became_peered":"2023-10-01T10:38:16.032283+0000","last_unstale":"2023-10-01T10:38:16.032774+0000","last_undegraded":"2023-10-01T10:38:16.032774+0000","last_fullsized":"2023-10-01T10:38:16.032774+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":65,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.020296+0000","last_change":"2023-10-01T10:37:05.021228+0000","last_active":"2023-10-01T10:38:16.020296+0000","last_peered":"2023-10-01T10:38:16.020296+0000","last_clean":"2023-10-01T10:38:16.020296+0000","last_became_active":"2023-10-01T10:37:05.020888+0000","last_became_peered":"2023-10-01T10:37:05.020888+0000","last_unstale":"2023-10-01T10:38:16.020296+0000","last_undegraded":"2023-10-01T10:38:16.020296+0000","last_fullsized":"2023-10-01T10:38:16.020296+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:59:57.641247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":68,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.020664+0000","last_change":"2023-10-01T10:28:20.220812+0000","last_active":"2023-10-01T10:38:16.020664+0000","last_peered":"2023-10-01T10:38:16.020664+0000","last_clean":"2023-10-01T10:38:16.020664+0000","last_became_active":"2023-10-01T10:28:20.215962+0000","last_became_peered":"2023-10-01T10:28:20.215962+0000","last_unstale":"2023-10-01T10:38:16.020664+0000","last_undegraded":"2023-10-01T10:38:16.020664+0000","last_fullsized":"2023-10-01T10:38:16.020664+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:35:15.275097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":75,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.014170+0000","last_change":"2023-10-01T10:37:42.773999+0000","last_active":"2023-10-01T10:38:16.014170+0000","last_peered":"2023-10-01T10:38:16.014170+0000","last_clean":"2023-10-01T10:38:16.014170+0000","last_became_active":"2023-10-01T10:37:42.773355+0000","last_became_peered":"2023-10-01T10:37:42.773355+0000","last_unstale":"2023-10-01T10:38:16.014170+0000","last_undegraded":"2023-10-01T10:38:16.014170+0000","last_fullsized":"2023-10-01T10:38:16.014170+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:46:51.317151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":107,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.035022+0000","last_change":"2023-10-01T10:38:16.034193+0000","last_active":"2023-10-01T10:38:16.035022+0000","last_peered":"2023-10-01T10:38:16.035022+0000","last_clean":"2023-10-01T10:38:16.035022+0000","last_became_active":"2023-10-01T10:38:16.033703+0000","last_became_peered":"2023-10-01T10:38:16.033703+0000","last_unstale":"2023-10-01T10:38:16.035022+0000","last_undegraded":"2023-10-01T10:38:16.035022+0000","last_fullsized":"2023-10-01T10:38:16.035022+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:21:26.061595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":72,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.011874+0000","last_change":"2023-10-01T10:37:42.773538+0000","last_active":"2023-10-01T10:38:16.011874+0000","last_peered":"2023-10-01T10:38:16.011874+0000","last_clean":"2023-10-01T10:38:16.011874+0000","last_became_active":"2023-10-01T10:37:42.771863+0000","last_became_peered":"2023-10-01T10:37:42.771863+0000","last_unstale":"2023-10-01T10:38:16.011874+0000","last_undegraded":"2023-10-01T10:38:16.011874+0000","last_fullsized":"2023-10-01T10:38:16.011874+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:46:46.698627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":97,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.031790+0000","last_change":"2023-10-01T10:37:05.023034+0000","last_active":"2023-10-01T10:38:15.031790+0000","last_peered":"2023-10-01T10:38:15.031790+0000","last_clean":"2023-10-01T10:38:15.031790+0000","last_became_active":"2023-10-01T10:37:05.022670+0000","last_became_peered":"2023-10-01T10:37:05.022670+0000","last_unstale":"2023-10-01T10:38:15.031790+0000","last_undegraded":"2023-10-01T10:38:15.031790+0000","last_fullsized":"2023-10-01T10:38:15.031790+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":101,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.027938+0000","last_change":"2023-10-01T10:38:16.027472+0000","last_active":"2023-10-01T10:38:16.027938+0000","last_peered":"2023-10-01T10:38:16.027938+0000","last_clean":"2023-10-01T10:38:16.027938+0000","last_became_active":"2023-10-01T10:38:16.026810+0000","last_became_peered":"2023-10-01T10:38:16.026810+0000","last_unstale":"2023-10-01T10:38:16.027938+0000","last_undegraded":"2023-10-01T10:38:16.027938+0000","last_fullsized":"2023-10-01T10:38:16.027938+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:30:06.978187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":65,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.035705+0000","last_change":"2023-10-01T10:38:16.035705+0000","last_active":"2023-10-01T10:38:16.035705+0000","last_peered":"2023-10-01T10:38:16.035705+0000","last_clean":"2023-10-01T10:38:16.035705+0000","last_became_active":"2023-10-01T10:38:16.035389+0000","last_became_peered":"2023-10-01T10:38:16.035389+0000","last_unstale":"2023-10-01T10:38:16.035705+0000","last_undegraded":"2023-10-01T10:38:16.035705+0000","last_fullsized":"2023-10-01T10:38:16.035705+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:16:10.201409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":85,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.028785+0000","last_change":"2023-10-01T10:38:16.028491+0000","last_active":"2023-10-01T10:38:16.028785+0000","last_peered":"2023-10-01T10:38:16.028785+0000","last_clean":"2023-10-01T10:38:16.028785+0000","last_became_active":"2023-10-01T10:38:16.028175+0000","last_became_peered":"2023-10-01T10:38:16.028175+0000","last_unstale":"2023-10-01T10:38:16.028785+0000","last_undegraded":"2023-10-01T10:38:16.028785+0000","last_fullsized":"2023-10-01T10:38:16.028785+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:44:29.083129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":107,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.012738+0000","last_change":"2023-10-01T10:37:42.762360+0000","last_active":"2023-10-01T10:38:16.012738+0000","last_peered":"2023-10-01T10:38:16.012738+0000","last_clean":"2023-10-01T10:38:16.012738+0000","last_became_active":"2023-10-01T10:37:42.761936+0000","last_became_peered":"2023-10-01T10:37:42.761936+0000","last_unstale":"2023-10-01T10:38:16.012738+0000","last_undegraded":"2023-10-01T10:38:16.012738+0000","last_fullsized":"2023-10-01T10:38:16.012738+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:50.637582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":111,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.037809+0000","last_change":"2023-10-01T10:38:16.037809+0000","last_active":"2023-10-01T10:38:16.037809+0000","last_peered":"2023-10-01T10:38:16.037809+0000","last_clean":"2023-10-01T10:38:16.037809+0000","last_became_active":"2023-10-01T10:38:16.037318+0000","last_became_peered":"2023-10-01T10:38:16.037318+0000","last_unstale":"2023-10-01T10:38:16.037809+0000","last_undegraded":"2023-10-01T10:38:16.037809+0000","last_fullsized":"2023-10-01T10:38:16.037809+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":115,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.034282+0000","last_change":"2023-10-01T10:38:16.034282+0000","last_active":"2023-10-01T10:38:16.034282+0000","last_peered":"2023-10-01T10:38:16.034282+0000","last_clean":"2023-10-01T10:38:16.034282+0000","last_became_active":"2023-10-01T10:38:16.033820+0000","last_became_peered":"2023-10-01T10:38:16.033820+0000","last_unstale":"2023-10-01T10:38:16.034282+0000","last_undegraded":"2023-10-01T10:38:16.034282+0000","last_fullsized":"2023-10-01T10:38:16.034282+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:48:23.815832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":96,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.038077+0000","last_change":"2023-10-01T10:38:16.038077+0000","last_active":"2023-10-01T10:38:16.038077+0000","last_peered":"2023-10-01T10:38:16.038077+0000","last_clean":"2023-10-01T10:38:16.038077+0000","last_became_active":"2023-10-01T10:38:16.037313+0000","last_became_peered":"2023-10-01T10:38:16.037313+0000","last_unstale":"2023-10-01T10:38:16.038077+0000","last_undegraded":"2023-10-01T10:38:16.038077+0000","last_fullsized":"2023-10-01T10:38:16.038077+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":115,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.025600+0000","last_change":"2023-10-01T10:38:16.025600+0000","last_active":"2023-10-01T10:38:16.025600+0000","last_peered":"2023-10-01T10:38:16.025600+0000","last_clean":"2023-10-01T10:38:16.025600+0000","last_became_active":"2023-10-01T10:38:16.025252+0000","last_became_peered":"2023-10-01T10:38:16.025252+0000","last_unstale":"2023-10-01T10:38:16.025600+0000","last_undegraded":"2023-10-01T10:38:16.025600+0000","last_fullsized":"2023-10-01T10:38:16.025600+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:24:18.871963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":103,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.013437+0000","last_change":"2023-10-01T10:37:42.761243+0000","last_active":"2023-10-01T10:38:16.013437+0000","last_peered":"2023-10-01T10:38:16.013437+0000","last_clean":"2023-10-01T10:38:16.013437+0000","last_became_active":"2023-10-01T10:37:42.760834+0000","last_became_peered":"2023-10-01T10:37:42.760834+0000","last_unstale":"2023-10-01T10:38:16.013437+0000","last_undegraded":"2023-10-01T10:38:16.013437+0000","last_fullsized":"2023-10-01T10:38:16.013437+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:37:21.976095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":115,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.014918+0000","last_change":"2023-10-01T10:37:42.765142+0000","last_active":"2023-10-01T10:38:15.014918+0000","last_peered":"2023-10-01T10:38:15.014918+0000","last_clean":"2023-10-01T10:38:15.014918+0000","last_became_active":"2023-10-01T10:37:42.764770+0000","last_became_peered":"2023-10-01T10:37:42.764770+0000","last_unstale":"2023-10-01T10:38:15.014918+0000","last_undegraded":"2023-10-01T10:38:15.014918+0000","last_fullsized":"2023-10-01T10:38:15.014918+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:23:33.527876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":86,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.019521+0000","last_change":"2023-10-01T10:37:05.021621+0000","last_active":"2023-10-01T10:38:16.019521+0000","last_peered":"2023-10-01T10:38:16.019521+0000","last_clean":"2023-10-01T10:38:16.019521+0000","last_became_active":"2023-10-01T10:37:05.021173+0000","last_became_peered":"2023-10-01T10:37:05.021173+0000","last_unstale":"2023-10-01T10:38:16.019521+0000","last_undegraded":"2023-10-01T10:38:16.019521+0000","last_fullsized":"2023-10-01T10:38:16.019521+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:13:25.364499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":81,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.031427+0000","last_change":"2023-10-01T10:38:16.031427+0000","last_active":"2023-10-01T10:38:16.031427+0000","last_peered":"2023-10-01T10:38:16.031427+0000","last_clean":"2023-10-01T10:38:16.031427+0000","last_became_active":"2023-10-01T10:38:16.029786+0000","last_became_peered":"2023-10-01T10:38:16.029786+0000","last_unstale":"2023-10-01T10:38:16.031427+0000","last_undegraded":"2023-10-01T10:38:16.031427+0000","last_fullsized":"2023-10-01T10:38:16.031427+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:53:22.796191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30319,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430535,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":303116,"kb_used_data":1796,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":93442036,"statfs":{"total":95995035648,"available":95684644864,"internally_reserved":0,"allocated":1839104,"data_stored":1363146,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:39:40 2023","interfaces":[{"interface":"back","average":{"1min":0.84199999999999997,"5min":1.3979999999999999,"15min":1.657},"min":{"1min":0.57499999999999996,"5min":0.309,"15min":0.27500000000000002},"max":{"1min":1.1639999999999999,"5min":7.9900000000000002,"15min":14.974},"last":0.73799999999999999},{"interface":"front","average":{"1min":0.84299999999999997,"5min":1.4670000000000001,"15min":1.7250000000000001},"min":{"1min":0.53800000000000003,"5min":0.36499999999999999,"15min":0.29999999999999999},"max":{"1min":1.21,"5min":9.7289999999999992,"15min":15.029},"last":0.79700000000000004}]},{"osd":1,"last update":"Sun Oct 1 10:39:52 2023","interfaces":[{"interface":"back","average":{"1min":0.90400000000000003,"5min":1.1930000000000001,"15min":1.216},"min":{"1min":0.67800000000000005,"5min":0.35999999999999999,"15min":0.309},"max":{"1min":1.1850000000000001,"5min":8.1180000000000003,"15min":8.1180000000000003},"last":1.1850000000000001},{"interface":"front","average":{"1min":0.90100000000000002,"5min":1.159,"15min":1.5629999999999999},"min":{"1min":0.66200000000000003,"5min":0.318,"15min":0.318},"max":{"1min":1.234,"5min":11.266,"15min":11.266},"last":0.93300000000000005}]},{"osd":2,"last update":"Sun Oct 1 10:39:05 2023","interfaces":[{"interface":"back","average":{"1min":0.96799999999999997,"5min":1.022,"15min":1.0309999999999999},"min":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"max":{"1min":1.1799999999999999,"5min":1.214,"15min":1.214},"last":1.1000000000000001},{"interface":"front","average":{"1min":0.89600000000000002,"5min":0.93600000000000005,"15min":0.94299999999999995},"min":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.167,"5min":1.294,"15min":1.294},"last":0.98899999999999999}]},{"osd":3,"last update":"Sun Oct 1 10:39:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0209999999999999,"5min":0.97799999999999998,"15min":0.97099999999999997},"min":{"1min":0.69699999999999995,"5min":0.55700000000000005,"15min":0.55700000000000005},"max":{"1min":1.202,"5min":1.3879999999999999,"15min":1.3879999999999999},"last":1.2230000000000001},{"interface":"front","average":{"1min":0.94599999999999995,"5min":0.94599999999999995,"15min":0.94599999999999995},"min":{"1min":0.748,"5min":0.64400000000000002,"15min":0.64400000000000002},"max":{"1min":1.1359999999999999,"5min":1.234,"15min":1.234},"last":1.073}]},{"osd":4,"last update":"Sun Oct 1 10:39:18 2023","interfaces":[{"interface":"back","average":{"1min":0.91700000000000004,"5min":0.91700000000000004,"15min":0.91700000000000004},"min":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"max":{"1min":1.167,"5min":1.167,"15min":1.167},"last":1.298},{"interface":"front","average":{"1min":1.1859999999999999,"5min":1.1859999999999999,"15min":1.1859999999999999},"min":{"1min":0.90500000000000003,"5min":0.90500000000000003,"15min":0.90500000000000003},"max":{"1min":1.333,"5min":1.333,"15min":1.333},"last":1.3759999999999999}]},{"osd":5,"last update":"Sun Oct 1 10:39:52 2023","interfaces":[{"interface":"back","average":{"1min":1.1240000000000001,"5min":1.1859999999999999,"15min":1.2629999999999999},"min":{"1min":0.74299999999999999,"5min":0.35799999999999998,"15min":0.29699999999999999},"max":{"1min":1.389,"5min":5.8230000000000004,"15min":10.106999999999999},"last":1.3260000000000001},{"interface":"front","average":{"1min":1.054,"5min":1.157,"15min":1.1850000000000001},"min":{"1min":0.877,"5min":0.432,"15min":0.34799999999999998},"max":{"1min":1.26,"5min":5.7800000000000002,"15min":6.2050000000000001},"last":1.26}]},{"osd":6,"last update":"Sun Oct 1 10:39:52 2023","interfaces":[{"interface":"back","average":{"1min":1.123,"5min":1.417,"15min":1.3520000000000001},"min":{"1min":0.82999999999999996,"5min":0.44700000000000001,"15min":0.32800000000000001},"max":{"1min":1.431,"5min":7.3300000000000001,"15min":9.5500000000000007},"last":1.2809999999999999},{"interface":"front","average":{"1min":1.1259999999999999,"5min":1.2929999999999999,"15min":1.3500000000000001},"min":{"1min":0.83799999999999997,"5min":0.40100000000000002,"15min":0.33900000000000002},"max":{"1min":1.355,"5min":6.9370000000000003,"15min":12.696},"last":1.355}]}]},{"osd":6,"up_from":42,"seq":180388626769,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":307196,"kb_used_data":1780,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":93437956,"statfs":{"total":95995035648,"available":95680466944,"internally_reserved":0,"allocated":1822720,"data_stored":1362685,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:39:34 2023","interfaces":[{"interface":"back","average":{"1min":0.878,"5min":1.3200000000000001,"15min":1.4490000000000001},"min":{"1min":0.53800000000000003,"5min":0.29399999999999998,"15min":0.28499999999999998},"max":{"1min":1.236,"5min":7.1139999999999999,"15min":9.1300000000000008},"last":0.83899999999999997},{"interface":"front","average":{"1min":0.85299999999999998,"5min":1.381,"15min":1.46},"min":{"1min":0.60399999999999998,"5min":0.29999999999999999,"15min":0.28899999999999998},"max":{"1min":1.238,"5min":6.9640000000000004,"15min":7.3109999999999999},"last":0.88800000000000001}]},{"osd":1,"last update":"Sun Oct 1 10:39:42 2023","interfaces":[{"interface":"back","average":{"1min":0.94799999999999995,"5min":1.4390000000000001,"15min":1.1479999999999999},"min":{"1min":0.59799999999999998,"5min":0.37,"15min":0.26600000000000001},"max":{"1min":1.2010000000000001,"5min":7.9100000000000001,"15min":7.9100000000000001},"last":1.119},{"interface":"front","average":{"1min":0.89100000000000001,"5min":1.288,"15min":1.452},"min":{"1min":0.63900000000000001,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.3089999999999999,"5min":7.4199999999999999,"15min":7.4199999999999999},"last":1.0009999999999999}]},{"osd":2,"last update":"Sun Oct 1 10:39:05 2023","interfaces":[{"interface":"back","average":{"1min":0.88700000000000001,"5min":0.87,"15min":0.86699999999999999},"min":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"max":{"1min":1.2450000000000001,"5min":1.2450000000000001,"15min":1.2450000000000001},"last":1.034},{"interface":"front","average":{"1min":0.95099999999999996,"5min":0.94099999999999995,"15min":0.93899999999999995},"min":{"1min":0.70899999999999996,"5min":0.64900000000000002,"15min":0.64900000000000002},"max":{"1min":1.1850000000000001,"5min":1.2929999999999999,"15min":1.2929999999999999},"last":1.091}]},{"osd":3,"last update":"Sun Oct 1 10:39:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0329999999999999,"5min":0.99099999999999999,"15min":0.98399999999999999},"min":{"1min":0.82599999999999996,"5min":0.69799999999999995,"15min":0.69799999999999995},"max":{"1min":1.2390000000000001,"5min":1.2529999999999999,"15min":1.2529999999999999},"last":1.2190000000000001},{"interface":"front","average":{"1min":1.024,"5min":1.0189999999999999,"15min":1.018},"min":{"1min":0.89200000000000002,"5min":0.78300000000000003,"15min":0.78300000000000003},"max":{"1min":1.2230000000000001,"5min":1.2230000000000001,"15min":1.2230000000000001},"last":0.79000000000000004}]},{"osd":4,"last update":"Sun Oct 1 10:39:18 2023","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":1.02,"15min":1.02},"min":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"max":{"1min":1.3919999999999999,"5min":1.3919999999999999,"15min":1.3919999999999999},"last":0.74299999999999999},{"interface":"front","average":{"1min":1.103,"5min":1.103,"15min":1.103},"min":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"max":{"1min":1.4179999999999999,"5min":1.4179999999999999,"15min":1.4179999999999999},"last":0.92600000000000005}]},{"osd":5,"last update":"Sun Oct 1 10:39:53 2023","interfaces":[{"interface":"back","average":{"1min":1.115,"5min":1.3240000000000001,"15min":1.214},"min":{"1min":0.873,"5min":0.46100000000000002,"15min":0.373},"max":{"1min":1.369,"5min":6.085,"15min":8.0150000000000006},"last":0.97299999999999998},{"interface":"front","average":{"1min":1.113,"5min":1.1870000000000001,"15min":1.2490000000000001},"min":{"1min":0.72699999999999998,"5min":0.38,"15min":0.34399999999999997},"max":{"1min":1.472,"5min":10.842000000000001,"15min":10.842000000000001},"last":1.1559999999999999}]},{"osd":7,"last update":"Sun Oct 1 10:39:34 2023","interfaces":[{"interface":"back","average":{"1min":1.147,"5min":1.089,"15min":1.077},"min":{"1min":0.94599999999999995,"5min":0.39700000000000002,"15min":0.371},"max":{"1min":1.454,"5min":4.3499999999999996,"15min":4.867},"last":1.0740000000000001},{"interface":"front","average":{"1min":1.196,"5min":1.2589999999999999,"15min":1.331},"min":{"1min":1.0169999999999999,"5min":0.42299999999999999,"15min":0.34799999999999998},"max":{"1min":1.3460000000000001,"5min":5.3470000000000004,"15min":11.031000000000001},"last":1.306}]}]},{"osd":1,"up_from":101,"seq":433791696986,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":251816,"kb_used_data":1312,"kb_used_omap":4,"kb_used_meta":250491,"kb_avail":93493336,"statfs":{"total":95995035648,"available":95737176064,"internally_reserved":0,"allocated":1343488,"data_stored":789975,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":256503694},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:39:47 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":1.236,"15min":2.0209999999999999},"min":{"1min":0.34999999999999998,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.85999999999999999,"5min":11.071999999999999,"15min":11.071999999999999},"last":0.71899999999999997},{"interface":"front","average":{"1min":0.60999999999999999,"5min":1.1739999999999999,"15min":1.97},"min":{"1min":0.31900000000000001,"5min":0.21199999999999999,"15min":0.20499999999999999},"max":{"1min":0.97699999999999998,"5min":7.9900000000000002,"15min":9.3230000000000004},"last":0.60499999999999998}]},{"osd":2,"last update":"Sun Oct 1 10:39:12 2023","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.94499999999999995,"5min":1.0589999999999999,"15min":1.0589999999999999},"last":0.82499999999999996},{"interface":"front","average":{"1min":0.73399999999999999,"5min":0.76000000000000001,"15min":0.76400000000000001},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.0309999999999999,"5min":1.1319999999999999,"15min":1.1319999999999999},"last":0.88800000000000001}]},{"osd":3,"last update":"Sun Oct 1 10:39:47 2023","interfaces":[{"interface":"back","average":{"1min":0.69699999999999995,"5min":0.68100000000000005,"15min":0.67800000000000005},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":1.0820000000000001,"5min":1.0820000000000001,"15min":1.0820000000000001},"last":0.98299999999999998},{"interface":"front","average":{"1min":0.68100000000000005,"5min":0.73799999999999999,"15min":0.748},"min":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":1.022,"5min":1.022,"15min":1.022},"last":0.68400000000000005}]},{"osd":4,"last update":"Sun Oct 1 10:39:18 2023","interfaces":[{"interface":"back","average":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88200000000000001},"min":{"1min":0.627,"5min":0.627,"15min":0.627},"max":{"1min":1.165,"5min":1.165,"15min":1.165},"last":0.86699999999999999},{"interface":"front","average":{"1min":1.008,"5min":1.008,"15min":1.008},"min":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"max":{"1min":1.268,"5min":1.268,"15min":1.268},"last":1.0429999999999999}]},{"osd":5,"last update":"Sun Oct 1 10:39:47 2023","interfaces":[{"interface":"back","average":{"1min":0.93600000000000005,"5min":1.341,"15min":1.7450000000000001},"min":{"1min":0.68400000000000005,"5min":0.372,"15min":0.35799999999999998},"max":{"1min":1.238,"5min":8.9429999999999996,"15min":8.9429999999999996},"last":0.96699999999999997},{"interface":"front","average":{"1min":1.0189999999999999,"5min":1.492,"15min":2.3119999999999998},"min":{"1min":0.68999999999999995,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":1.385,"5min":9.1300000000000008,"15min":11.353999999999999},"last":1.0680000000000001}]},{"osd":6,"last update":"Sun Oct 1 10:39:47 2023","interfaces":[{"interface":"back","average":{"1min":1.046,"5min":1.325,"15min":1.9770000000000001},"min":{"1min":0.68400000000000005,"5min":0.37,"15min":0.37},"max":{"1min":1.387,"5min":6.6849999999999996,"15min":8.6630000000000003},"last":1.159},{"interface":"front","average":{"1min":1.038,"5min":1.4359999999999999,"15min":1.9430000000000001},"min":{"1min":0.69999999999999996,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":1.5529999999999999,"5min":6,"15min":7.1369999999999996},"last":1.1220000000000001}]},{"osd":7,"last update":"Sun Oct 1 10:39:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0800000000000001,"5min":1.452,"15min":1.8879999999999999},"min":{"1min":0.57299999999999995,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":1.474,"5min":6.0510000000000002,"15min":7.1680000000000001},"last":1.2030000000000001},{"interface":"front","average":{"1min":1.032,"5min":1.478,"15min":1.7410000000000001},"min":{"1min":0.71799999999999997,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":1.365,"5min":9.0749999999999993,"15min":9.0749999999999993},"last":1.0169999999999999}]}]},{"osd":0,"up_from":86,"seq":369367187596,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":294004,"kb_used_data":1900,"kb_used_omap":5,"kb_used_meta":292090,"kb_avail":93451148,"statfs":{"total":95995035648,"available":95693975552,"internally_reserved":0,"allocated":1945600,"data_stored":1378971,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":299100794},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 10:39:45 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":1.395,"15min":1.6299999999999999},"min":{"1min":0.28199999999999997,"5min":0.224,"15min":0.214},"max":{"1min":0.92600000000000005,"5min":11.997999999999999,"15min":12.855},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.56899999999999995,"5min":1.1140000000000001,"15min":2.335},"min":{"1min":0.32000000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":1.1180000000000001,"5min":6.8490000000000002,"15min":15.48},"last":0.63800000000000001}]},{"osd":2,"last update":"Sun Oct 1 10:39:07 2023","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.69099999999999995,"15min":0.69799999999999995},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":1.085,"5min":1.085,"15min":1.085},"last":0.997},{"interface":"front","average":{"1min":0.70999999999999996,"5min":0.74399999999999999,"15min":0.75},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":1.149,"5min":1.149,"15min":1.149},"last":0.69399999999999995}]},{"osd":3,"last update":"Sun Oct 1 10:39:45 2023","interfaces":[{"interface":"back","average":{"1min":0.68400000000000005,"5min":0.78800000000000003,"15min":0.80600000000000005},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.96599999999999997,"5min":1.3069999999999999,"15min":1.3069999999999999},"last":0.85899999999999999},{"interface":"front","average":{"1min":0.67000000000000004,"5min":0.79200000000000004,"15min":0.81200000000000006},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.95299999999999996,"5min":1.1060000000000001,"15min":1.1060000000000001},"last":0.73199999999999998}]},{"osd":4,"last update":"Sun Oct 1 10:39:18 2023","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":1.02,"15min":1.02},"min":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"max":{"1min":1.3200000000000001,"5min":1.3200000000000001,"15min":1.3200000000000001},"last":0.93899999999999995},{"interface":"front","average":{"1min":1.0349999999999999,"5min":1.0349999999999999,"15min":1.0349999999999999},"min":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"max":{"1min":1.5029999999999999,"5min":1.5029999999999999,"15min":1.5029999999999999},"last":1.1799999999999999}]},{"osd":5,"last update":"Sun Oct 1 10:39:45 2023","interfaces":[{"interface":"back","average":{"1min":0.97499999999999998,"5min":1.3759999999999999,"15min":1.722},"min":{"1min":0.78300000000000003,"5min":0.32000000000000001,"15min":0.31900000000000001},"max":{"1min":1.3440000000000001,"5min":8.0690000000000008,"15min":13.068},"last":1.2290000000000001},{"interface":"front","average":{"1min":0.96499999999999997,"5min":1.5489999999999999,"15min":1.7190000000000001},"min":{"1min":0.60199999999999998,"5min":0.39300000000000002,"15min":0.34200000000000003},"max":{"1min":1.258,"5min":11.747,"15min":11.747},"last":1.034}]},{"osd":6,"last update":"Sun Oct 1 10:39:45 2023","interfaces":[{"interface":"back","average":{"1min":1.006,"5min":1.8129999999999999,"15min":1.825},"min":{"1min":0.69899999999999995,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":1.321,"5min":11.663,"15min":11.663},"last":1.2030000000000001},{"interface":"front","average":{"1min":0.91300000000000003,"5min":1.5840000000000001,"15min":1.766},"min":{"1min":0.56999999999999995,"5min":0.40400000000000003,"15min":0.38600000000000001},"max":{"1min":1.2789999999999999,"5min":15.069000000000001,"15min":15.069000000000001},"last":1.071}]},{"osd":7,"last update":"Sun Oct 1 10:39:45 2023","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":1.6279999999999999,"15min":1.8680000000000001},"min":{"1min":0.73399999999999999,"5min":0.40300000000000002,"15min":0.39000000000000001},"max":{"1min":1.3819999999999999,"5min":11.941000000000001,"15min":12.686},"last":1.091},{"interface":"front","average":{"1min":0.93400000000000005,"5min":1.4450000000000001,"15min":1.6100000000000001},"min":{"1min":0.59099999999999997,"5min":0.438,"15min":0.36199999999999999},"max":{"1min":1.323,"5min":6.8150000000000004,"15min":13.654},"last":1.1279999999999999}]}]},{"osd":2,"up_from":116,"seq":498216206372,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":155752,"kb_used_data":1248,"kb_used_omap":7,"kb_used_meta":154488,"kb_avail":93589400,"statfs":{"total":95995035648,"available":95835545600,"internally_reserved":0,"allocated":1277952,"data_stored":778803,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":158195793},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:39:07 2023","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":0.66800000000000004,"15min":0.67400000000000004},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":1.002,"5min":1.0660000000000001,"15min":1.0660000000000001},"last":0.874},{"interface":"front","average":{"1min":0.73399999999999999,"5min":0.78200000000000003,"15min":0.79000000000000004},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":1.0760000000000001,"5min":1.1140000000000001,"15min":1.1140000000000001},"last":0.69599999999999995}]},{"osd":1,"last update":"Sun Oct 1 10:39:07 2023","interfaces":[{"interface":"back","average":{"1min":0.73799999999999999,"5min":0.71699999999999997,"15min":0.71299999999999997},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":1.087,"5min":1.087,"15min":1.087},"last":1.036},{"interface":"front","average":{"1min":0.67200000000000004,"5min":0.77500000000000002,"15min":0.79200000000000004},"min":{"1min":0.36499999999999999,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.91900000000000004,"5min":1.1479999999999999,"15min":1.1479999999999999},"last":0.83199999999999996}]},{"osd":3,"last update":"Sun Oct 1 10:39:45 2023","interfaces":[{"interface":"back","average":{"1min":0.67800000000000005,"5min":0.72799999999999998,"15min":0.73599999999999999},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":1.0740000000000001,"5min":1.0740000000000001,"15min":1.0740000000000001},"last":0.76200000000000001},{"interface":"front","average":{"1min":0.77800000000000002,"5min":0.86499999999999999,"15min":0.879},"min":{"1min":0.50600000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":1.1479999999999999,"5min":1.2,"15min":1.2},"last":0.93200000000000005}]},{"osd":4,"last update":"Sun Oct 1 10:39:18 2023","interfaces":[{"interface":"back","average":{"1min":0.98099999999999998,"5min":0.98099999999999998,"15min":0.98099999999999998},"min":{"1min":0.745,"5min":0.745,"15min":0.745},"max":{"1min":1.28,"5min":1.28,"15min":1.28},"last":1.2669999999999999},{"interface":"front","average":{"1min":0.95699999999999996,"5min":0.95699999999999996,"15min":0.95699999999999996},"min":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"max":{"1min":1.254,"5min":1.254,"15min":1.254},"last":1.153}]},{"osd":5,"last update":"Sun Oct 1 10:39:07 2023","interfaces":[{"interface":"back","average":{"1min":1.032,"5min":1.1040000000000001,"15min":1.1160000000000001},"min":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"max":{"1min":1.468,"5min":1.6339999999999999,"15min":1.6339999999999999},"last":1.1799999999999999},{"interface":"front","average":{"1min":1.0329999999999999,"5min":1.02,"15min":1.018},"min":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":1.3129999999999999,"5min":1.585,"15min":1.585},"last":0.98399999999999999}]},{"osd":6,"last update":"Sun Oct 1 10:39:07 2023","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":1.0840000000000001,"15min":1.095},"min":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":1.276,"5min":1.486,"15min":1.486},"last":1.071},{"interface":"front","average":{"1min":1.1279999999999999,"5min":1.1419999999999999,"15min":1.1439999999999999},"min":{"1min":0.84199999999999997,"5min":0.65200000000000002,"15min":0.65200000000000002},"max":{"1min":1.3799999999999999,"5min":1.6639999999999999,"15min":1.6639999999999999},"last":1.018}]},{"osd":7,"last update":"Sun Oct 1 10:39:07 2023","interfaces":[{"interface":"back","average":{"1min":1.1020000000000001,"5min":1.1259999999999999,"15min":1.1299999999999999},"min":{"1min":0.79500000000000004,"5min":0.749,"15min":0.749},"max":{"1min":1.444,"5min":1.536,"15min":1.536},"last":1.101},{"interface":"front","average":{"1min":1.129,"5min":1.206,"15min":1.2190000000000001},"min":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"max":{"1min":1.7390000000000001,"5min":1.7390000000000001,"15min":1.7390000000000001},"last":1.345}]}]},{"osd":3,"up_from":121,"seq":519691042844,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":103128,"kb_used_data":1232,"kb_used_omap":2,"kb_used_meta":101885,"kb_avail":93642024,"statfs":{"total":95995035648,"available":95889432576,"internally_reserved":0,"allocated":1261568,"data_stored":776896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":104330293},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:39:47 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.66000000000000003,"15min":0.66900000000000004},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.93000000000000005,"5min":0.96199999999999997,"15min":0.96199999999999997},"last":1.202},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.59499999999999997,"15min":0.58699999999999997},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.98399999999999999,"5min":0.98399999999999999,"15min":0.98399999999999999},"last":0.63100000000000001}]},{"osd":1,"last update":"Sun Oct 1 10:39:47 2023","interfaces":[{"interface":"back","average":{"1min":0.65900000000000003,"5min":0.69099999999999995,"15min":0.69699999999999995},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":1.2450000000000001,"5min":1.2450000000000001,"15min":1.2450000000000001},"last":0.91100000000000003},{"interface":"front","average":{"1min":0.73599999999999999,"5min":0.77200000000000002,"15min":0.77800000000000002},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":1.3740000000000001,"5min":1.3740000000000001,"15min":1.3740000000000001},"last":1.0289999999999999}]},{"osd":2,"last update":"Sun Oct 1 10:39:47 2023","interfaces":[{"interface":"back","average":{"1min":0.65700000000000003,"5min":0.67700000000000005,"15min":0.68000000000000005},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":1.258,"5min":1.258,"15min":1.258},"last":1.355},{"interface":"front","average":{"1min":0.79500000000000004,"5min":0.78700000000000003,"15min":0.78600000000000003},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":1.256,"5min":1.256,"15min":1.256},"last":1.3160000000000001}]},{"osd":4,"last update":"Sun Oct 1 10:39:15 2023","interfaces":[{"interface":"back","average":{"1min":0.94099999999999995,"5min":0.94099999999999995,"15min":0.94099999999999995},"min":{"1min":0.623,"5min":0.623,"15min":0.623},"max":{"1min":1.306,"5min":1.306,"15min":1.306},"last":1.151},{"interface":"front","average":{"1min":0.92800000000000005,"5min":0.92800000000000005,"15min":0.92800000000000005},"min":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"max":{"1min":1.278,"5min":1.278,"15min":1.278},"last":0.77800000000000002}]},{"osd":5,"last update":"Sun Oct 1 10:39:47 2023","interfaces":[{"interface":"back","average":{"1min":0.95599999999999996,"5min":0.94599999999999995,"15min":0.94399999999999995},"min":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"max":{"1min":1.2909999999999999,"5min":1.2909999999999999,"15min":1.2909999999999999},"last":0.98399999999999999},{"interface":"front","average":{"1min":0.98599999999999999,"5min":1.002,"15min":1.004},"min":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"max":{"1min":1.3100000000000001,"5min":1.4199999999999999,"15min":1.4199999999999999},"last":1.2989999999999999}]},{"osd":6,"last update":"Sun Oct 1 10:39:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0349999999999999,"5min":1.0149999999999999,"15min":1.012},"min":{"1min":0.60899999999999999,"5min":0.59099999999999997,"15min":0.59099999999999997},"max":{"1min":1.417,"5min":1.417,"15min":1.417},"last":0.83299999999999996},{"interface":"front","average":{"1min":1.0369999999999999,"5min":1.0920000000000001,"15min":1.101},"min":{"1min":0.62,"5min":0.62,"15min":0.62},"max":{"1min":1.351,"5min":1.377,"15min":1.377},"last":1.242}]},{"osd":7,"last update":"Sun Oct 1 10:39:47 2023","interfaces":[{"interface":"back","average":{"1min":0.97799999999999998,"5min":1.0429999999999999,"15min":1.054},"min":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.3959999999999999,"5min":1.411,"15min":1.411},"last":1.1739999999999999},{"interface":"front","average":{"1min":1.056,"5min":1.0249999999999999,"15min":1.02},"min":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"max":{"1min":1.5109999999999999,"5min":1.5109999999999999,"15min":1.5109999999999999},"last":1.2669999999999999}]}]},{"osd":4,"up_from":126,"seq":541165879317,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":138584,"kb_used_data":1232,"kb_used_omap":10,"kb_used_meta":137333,"kb_avail":93606568,"statfs":{"total":95995035648,"available":95853125632,"internally_reserved":0,"allocated":1261568,"data_stored":777243,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":140629604},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:39:16 2023","interfaces":[{"interface":"back","average":{"1min":0.90500000000000003,"5min":0.90500000000000003,"15min":0.90500000000000003},"min":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"max":{"1min":1.3009999999999999,"5min":1.3009999999999999,"15min":1.3009999999999999},"last":1.151},{"interface":"front","average":{"1min":0.98999999999999999,"5min":0.98999999999999999,"15min":0.98999999999999999},"min":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"max":{"1min":1.4279999999999999,"5min":1.4279999999999999,"15min":1.4279999999999999},"last":0.79900000000000004}]},{"osd":1,"last update":"Sun Oct 1 10:39:16 2023","interfaces":[{"interface":"back","average":{"1min":0.92700000000000005,"5min":0.92700000000000005,"15min":0.92700000000000005},"min":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"max":{"1min":1.1850000000000001,"5min":1.1850000000000001,"15min":1.1850000000000001},"last":0.97999999999999998},{"interface":"front","average":{"1min":0.95399999999999996,"5min":0.95399999999999996,"15min":0.95399999999999996},"min":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"max":{"1min":1.2430000000000001,"5min":1.2430000000000001,"15min":1.2430000000000001},"last":1.016}]},{"osd":2,"last update":"Sun Oct 1 10:39:16 2023","interfaces":[{"interface":"back","average":{"1min":0.95399999999999996,"5min":0.95399999999999996,"15min":0.95399999999999996},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":1.1970000000000001,"5min":1.1970000000000001,"15min":1.1970000000000001},"last":0.89000000000000001},{"interface":"front","average":{"1min":0.97699999999999998,"5min":0.97699999999999998,"15min":0.97699999999999998},"min":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"max":{"1min":1.222,"5min":1.222,"15min":1.222},"last":1.284}]},{"osd":3,"last update":"Sun Oct 1 10:39:16 2023","interfaces":[{"interface":"back","average":{"1min":1.0780000000000001,"5min":1.0780000000000001,"15min":1.0780000000000001},"min":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"max":{"1min":1.3320000000000001,"5min":1.3320000000000001,"15min":1.3320000000000001},"last":0.95199999999999996},{"interface":"front","average":{"1min":0.97899999999999998,"5min":0.97899999999999998,"15min":0.97899999999999998},"min":{"1min":0.628,"5min":0.628,"15min":0.628},"max":{"1min":1.296,"5min":1.296,"15min":1.296},"last":1.1060000000000001}]},{"osd":5,"last update":"Sun Oct 1 10:39:16 2023","interfaces":[{"interface":"back","average":{"1min":1.095,"5min":1.095,"15min":1.095},"min":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"max":{"1min":1.5089999999999999,"5min":1.5089999999999999,"15min":1.5089999999999999},"last":1.21},{"interface":"front","average":{"1min":1.0940000000000001,"5min":1.0940000000000001,"15min":1.0940000000000001},"min":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"max":{"1min":1.3779999999999999,"5min":1.3779999999999999,"15min":1.3779999999999999},"last":1.319}]},{"osd":6,"last update":"Sun Oct 1 10:39:16 2023","interfaces":[{"interface":"back","average":{"1min":1.139,"5min":1.139,"15min":1.139},"min":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"max":{"1min":1.579,"5min":1.579,"15min":1.579},"last":1.0680000000000001},{"interface":"front","average":{"1min":1.075,"5min":1.075,"15min":1.075},"min":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"max":{"1min":1.403,"5min":1.403,"15min":1.403},"last":1.256}]},{"osd":7,"last update":"Sun Oct 1 10:39:16 2023","interfaces":[{"interface":"back","average":{"1min":1.1819999999999999,"5min":1.1819999999999999,"15min":1.1819999999999999},"min":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"max":{"1min":1.387,"5min":1.387,"15min":1.387},"last":1.1839999999999999},{"interface":"front","average":{"1min":1.169,"5min":1.169,"15min":1.169},"min":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"max":{"1min":1.46,"5min":1.46,"15min":1.46},"last":1.23}]}]},{"osd":5,"up_from":36,"seq":154618822998,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":594992,"kb_used_data":1192,"kb_used_omap":0,"kb_used_meta":593792,"kb_avail":93150160,"statfs":{"total":95995035648,"available":95385763840,"internally_reserved":0,"allocated":1220608,"data_stored":768137,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":608043008},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:39:38 2023","interfaces":[{"interface":"back","average":{"1min":0.83899999999999997,"5min":0.98499999999999999,"15min":1.272},"min":{"1min":0.67900000000000005,"5min":0.27300000000000002,"15min":0.26400000000000001},"max":{"1min":1.2709999999999999,"5min":4.8140000000000001,"15min":10.429},"last":0.82699999999999996},{"interface":"front","average":{"1min":0.92500000000000004,"5min":1.306,"15min":1.4930000000000001},"min":{"1min":0.629,"5min":0.29299999999999998,"15min":0.29199999999999998},"max":{"1min":1.1919999999999999,"5min":13.018000000000001,"15min":13.018000000000001},"last":0.93100000000000005}]},{"osd":1,"last update":"Sun Oct 1 10:39:38 2023","interfaces":[{"interface":"back","average":{"1min":0.85299999999999998,"5min":1.3029999999999999,"15min":1.663},"min":{"1min":0.62,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":1.02,"5min":9.0980000000000008,"15min":10.858000000000001},"last":1.377},{"interface":"front","average":{"1min":0.94199999999999995,"5min":1.7370000000000001,"15min":1.76},"min":{"1min":0.68000000000000005,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.337,"5min":12.864000000000001,"15min":12.864000000000001},"last":1.272}]},{"osd":2,"last update":"Sun Oct 1 10:39:06 2023","interfaces":[{"interface":"back","average":{"1min":0.88100000000000001,"5min":0.96299999999999997,"15min":0.97699999999999998},"min":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"max":{"1min":1.272,"5min":1.3540000000000001,"15min":1.3540000000000001},"last":1.4690000000000001},{"interface":"front","average":{"1min":0.97399999999999998,"5min":0.96599999999999997,"15min":0.96399999999999997},"min":{"1min":0.751,"5min":0.67600000000000005,"15min":0.67600000000000005},"max":{"1min":1.1879999999999999,"5min":1.272,"15min":1.272},"last":1.177}]},{"osd":3,"last update":"Sun Oct 1 10:38:46 2023","interfaces":[{"interface":"back","average":{"1min":0.93300000000000005,"5min":0.93300000000000005,"15min":0.93300000000000005},"min":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"max":{"1min":1.3919999999999999,"5min":1.3919999999999999,"15min":1.3919999999999999},"last":1.079},{"interface":"front","average":{"1min":0.93999999999999995,"5min":0.93999999999999995,"15min":0.93999999999999995},"min":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"max":{"1min":1.1879999999999999,"5min":1.1879999999999999,"15min":1.1879999999999999},"last":0.876}]},{"osd":4,"last update":"Sun Oct 1 10:39:16 2023","interfaces":[{"interface":"back","average":{"1min":1.077,"5min":1.077,"15min":1.077},"min":{"1min":0.753,"5min":0.753,"15min":0.753},"max":{"1min":1.343,"5min":1.343,"15min":1.343},"last":1.0289999999999999},{"interface":"front","average":{"1min":1.0840000000000001,"5min":1.0840000000000001,"15min":1.0840000000000001},"min":{"1min":0.89700000000000002,"5min":0.89700000000000002,"15min":0.89700000000000002},"max":{"1min":1.2989999999999999,"5min":1.2989999999999999,"15min":1.2989999999999999},"last":1.546}]},{"osd":6,"last update":"Sun Oct 1 10:38:52 2023","interfaces":[{"interface":"back","average":{"1min":0.96599999999999997,"5min":1.298,"15min":1.298},"min":{"1min":0.42399999999999999,"5min":0.38500000000000001,"15min":0.33900000000000002},"max":{"1min":1.496,"5min":6.96,"15min":6.96},"last":1.6020000000000001},{"interface":"front","average":{"1min":0.90000000000000002,"5min":1.325,"15min":1.2330000000000001},"min":{"1min":0.437,"5min":0.312,"15min":0.312},"max":{"1min":1.377,"5min":8.4339999999999993,"15min":8.4339999999999993},"last":1.151}]},{"osd":7,"last update":"Sun Oct 1 10:38:52 2023","interfaces":[{"interface":"back","average":{"1min":0.94799999999999995,"5min":1.4199999999999999,"15min":1.292},"min":{"1min":0.39000000000000001,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.3480000000000001,"5min":9.4629999999999992,"15min":9.4629999999999992},"last":1.782},{"interface":"front","average":{"1min":1.0109999999999999,"5min":1.1899999999999999,"15min":1.3919999999999999},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":1.4279999999999999,"5min":5.4699999999999998,"15min":12.927},"last":1.649}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T10:39:56.161 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T10:39:56.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:39:56 smithi078 bash[75299]: audit 2023-10-01T10:39:55.320255+0000 mgr.y (mgr.25072) 662 : audit 0 from='client.45521 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:39:56.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:39:56 smithi078 bash[78274]: audit 2023-10-01T10:39:55.320255+0000 mgr.y (mgr.25072) 662 : audit 0 from='client.45521 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:39:56.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:39:56 smithi173 bash[58816]: audit 2023-10-01T10:39:55.320255+0000 mgr.y (mgr.25072) 662 : audit 0 from='client.45521 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:39:57.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:39:57 smithi078 bash[75299]: cluster 2023-10-01T10:39:55.687017+0000 mgr.y (mgr.25072) 663 : cluster 0 pgmap v568: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:57.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:39:57 smithi078 bash[78274]: cluster 2023-10-01T10:39:55.687017+0000 mgr.y (mgr.25072) 663 : cluster 0 pgmap v568: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:57.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:39:57 smithi173 bash[58816]: cluster 2023-10-01T10:39:55.687017+0000 mgr.y (mgr.25072) 663 : cluster 0 pgmap v568: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:59.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:39:59 smithi078 bash[75299]: cluster 2023-10-01T10:39:57.687677+0000 mgr.y (mgr.25072) 664 : cluster 0 pgmap v569: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:59.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:39:59 smithi078 bash[78274]: cluster 2023-10-01T10:39:57.687677+0000 mgr.y (mgr.25072) 664 : cluster 0 pgmap v569: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:39:59.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:39:59 smithi173 bash[58816]: cluster 2023-10-01T10:39:57.687677+0000 mgr.y (mgr.25072) 664 : cluster 0 pgmap v569: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:00.332 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:40:00 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:40:00] "GET /metrics HTTP/1.1" 200 33824 "" "Prometheus/2.33.4" 2023-10-01T10:40:00.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:00 smithi078 bash[78274]: cluster 2023-10-01T10:40:00.000150+0000 mon.a (mon.0) 899 : cluster 1 overall HEALTH_OK 2023-10-01T10:40:00.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:00 smithi078 bash[75299]: cluster 2023-10-01T10:40:00.000150+0000 mon.a (mon.0) 899 : cluster 1 overall HEALTH_OK 2023-10-01T10:40:00.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:00 smithi173 bash[58816]: cluster 2023-10-01T10:40:00.000150+0000 mon.a (mon.0) 899 : cluster 1 overall HEALTH_OK 2023-10-01T10:40:00.982 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:40:01.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:01 smithi078 bash[75299]: cluster 2023-10-01T10:39:59.688581+0000 mgr.y (mgr.25072) 665 : cluster 0 pgmap v570: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:01.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:01 smithi078 bash[78274]: cluster 2023-10-01T10:39:59.688581+0000 mgr.y (mgr.25072) 665 : cluster 0 pgmap v570: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:01.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:01 smithi173 bash[58816]: cluster 2023-10-01T10:39:59.688581+0000 mgr.y (mgr.25072) 665 : cluster 0 pgmap v570: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:03.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:03 smithi078 bash[75299]: cluster 2023-10-01T10:40:01.689468+0000 mgr.y (mgr.25072) 666 : cluster 0 pgmap v571: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:03.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:03 smithi078 bash[75299]: audit 2023-10-01T10:40:02.410776+0000 mon.a (mon.0) 900 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:40:03.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:03 smithi078 bash[78274]: cluster 2023-10-01T10:40:01.689468+0000 mgr.y (mgr.25072) 666 : cluster 0 pgmap v571: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:03.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:03 smithi078 bash[78274]: audit 2023-10-01T10:40:02.410776+0000 mon.a (mon.0) 900 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:40:03.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:03 smithi173 bash[58816]: cluster 2023-10-01T10:40:01.689468+0000 mgr.y (mgr.25072) 666 : cluster 0 pgmap v571: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:03.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:03 smithi173 bash[58816]: audit 2023-10-01T10:40:02.410776+0000 mon.a (mon.0) 900 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:40:04.511 INFO:teuthology.orchestra.run.smithi078.stdout: 2023-10-01T10:40:04.515 INFO:teuthology.orchestra.run.smithi078.stderr:dumped all 2023-10-01T10:40:05.320 INFO:teuthology.orchestra.run.smithi078.stdout:{"pg_ready":true,"pg_map":{"version":572,"stamp":"2023-10-01T10:40:03.689725+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":261,"num_write_kb":2446,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":195,"ondisk_log_size":195,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2148588,"kb_used_data":11692,"kb_used_omap":30,"kb_used_meta":2136801,"kb_avail":747812628,"statfs":{"total":767960285184,"available":765760131072,"internally_reserved":0,"allocated":11972608,"data_stored":7995856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31502,"internal_metadata":2188084466},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004561"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":113,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.032258+0000","last_change":"2023-10-01T10:38:16.032258+0000","last_active":"2023-10-01T10:38:16.032258+0000","last_peered":"2023-10-01T10:38:16.032258+0000","last_clean":"2023-10-01T10:38:16.032258+0000","last_became_active":"2023-10-01T10:38:16.031633+0000","last_became_peered":"2023-10-01T10:38:16.031633+0000","last_unstale":"2023-10-01T10:38:16.032258+0000","last_undegraded":"2023-10-01T10:38:16.032258+0000","last_fullsized":"2023-10-01T10:38:16.032258+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:17:02.446430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":101,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.025881+0000","last_change":"2023-10-01T10:38:16.025881+0000","last_active":"2023-10-01T10:38:16.025881+0000","last_peered":"2023-10-01T10:38:16.025881+0000","last_clean":"2023-10-01T10:38:16.025881+0000","last_became_active":"2023-10-01T10:38:16.025518+0000","last_became_peered":"2023-10-01T10:38:16.025518+0000","last_unstale":"2023-10-01T10:38:16.025881+0000","last_undegraded":"2023-10-01T10:38:16.025881+0000","last_fullsized":"2023-10-01T10:38:16.025881+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:48:37.826604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":80,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.025648+0000","last_change":"2023-10-01T10:38:16.025075+0000","last_active":"2023-10-01T10:38:16.025648+0000","last_peered":"2023-10-01T10:38:16.025648+0000","last_clean":"2023-10-01T10:38:16.025648+0000","last_became_active":"2023-10-01T10:38:16.024735+0000","last_became_peered":"2023-10-01T10:38:16.024735+0000","last_unstale":"2023-10-01T10:38:16.025648+0000","last_undegraded":"2023-10-01T10:38:16.025648+0000","last_fullsized":"2023-10-01T10:38:16.025648+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:38:01.450741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":83,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.019448+0000","last_change":"2023-10-01T10:37:42.772008+0000","last_active":"2023-10-01T10:38:16.019448+0000","last_peered":"2023-10-01T10:38:16.019448+0000","last_clean":"2023-10-01T10:38:16.019448+0000","last_became_active":"2023-10-01T10:37:42.771244+0000","last_became_peered":"2023-10-01T10:37:42.771244+0000","last_unstale":"2023-10-01T10:38:16.019448+0000","last_undegraded":"2023-10-01T10:38:16.019448+0000","last_fullsized":"2023-10-01T10:38:16.019448+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:38:55.180790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":102,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.010782+0000","last_change":"2023-10-01T10:37:42.766600+0000","last_active":"2023-10-01T10:38:15.010782+0000","last_peered":"2023-10-01T10:38:15.010782+0000","last_clean":"2023-10-01T10:38:15.010782+0000","last_became_active":"2023-10-01T10:37:42.766051+0000","last_became_peered":"2023-10-01T10:37:42.766051+0000","last_unstale":"2023-10-01T10:38:15.010782+0000","last_undegraded":"2023-10-01T10:38:15.010782+0000","last_fullsized":"2023-10-01T10:38:15.010782+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:51:17.707963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":92,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.031873+0000","last_change":"2023-10-01T10:32:31.575109+0000","last_active":"2023-10-01T10:38:15.031873+0000","last_peered":"2023-10-01T10:38:15.031873+0000","last_clean":"2023-10-01T10:38:15.031873+0000","last_became_active":"2023-10-01T10:32:31.574729+0000","last_became_peered":"2023-10-01T10:32:31.574729+0000","last_unstale":"2023-10-01T10:38:15.031873+0000","last_undegraded":"2023-10-01T10:38:15.031873+0000","last_fullsized":"2023-10-01T10:38:15.031873+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":81,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.014432+0000","last_change":"2023-10-01T10:37:42.766667+0000","last_active":"2023-10-01T10:38:15.014432+0000","last_peered":"2023-10-01T10:38:15.014432+0000","last_clean":"2023-10-01T10:38:15.014432+0000","last_became_active":"2023-10-01T10:37:42.766369+0000","last_became_peered":"2023-10-01T10:37:42.766369+0000","last_unstale":"2023-10-01T10:38:15.014432+0000","last_undegraded":"2023-10-01T10:38:15.014432+0000","last_fullsized":"2023-10-01T10:38:15.014432+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:02.708788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":87,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.011572+0000","last_change":"2023-10-01T10:37:42.765925+0000","last_active":"2023-10-01T10:38:16.011572+0000","last_peered":"2023-10-01T10:38:16.011572+0000","last_clean":"2023-10-01T10:38:16.011572+0000","last_became_active":"2023-10-01T10:37:42.765218+0000","last_became_peered":"2023-10-01T10:37:42.765218+0000","last_unstale":"2023-10-01T10:38:16.011572+0000","last_undegraded":"2023-10-01T10:38:16.011572+0000","last_fullsized":"2023-10-01T10:38:16.011572+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T22:10:34.338031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":94,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.034622+0000","last_change":"2023-10-01T10:32:31.579805+0000","last_active":"2023-10-01T10:38:15.034622+0000","last_peered":"2023-10-01T10:38:15.034622+0000","last_clean":"2023-10-01T10:38:15.034622+0000","last_became_active":"2023-10-01T10:32:31.579423+0000","last_became_peered":"2023-10-01T10:32:31.579423+0000","last_unstale":"2023-10-01T10:38:15.034622+0000","last_undegraded":"2023-10-01T10:38:15.034622+0000","last_fullsized":"2023-10-01T10:38:15.034622+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":71,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.016372+0000","last_change":"2023-10-01T10:37:42.772531+0000","last_active":"2023-10-01T10:38:16.016372+0000","last_peered":"2023-10-01T10:38:16.016372+0000","last_clean":"2023-10-01T10:38:16.016372+0000","last_became_active":"2023-10-01T10:37:42.772216+0000","last_became_peered":"2023-10-01T10:37:42.772216+0000","last_unstale":"2023-10-01T10:38:16.016372+0000","last_undegraded":"2023-10-01T10:38:16.016372+0000","last_fullsized":"2023-10-01T10:38:16.016372+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:01:53.373628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":112,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.027085+0000","last_change":"2023-10-01T10:38:16.027085+0000","last_active":"2023-10-01T10:38:16.027085+0000","last_peered":"2023-10-01T10:38:16.027085+0000","last_clean":"2023-10-01T10:38:16.027085+0000","last_became_active":"2023-10-01T10:38:16.026555+0000","last_became_peered":"2023-10-01T10:38:16.026555+0000","last_unstale":"2023-10-01T10:38:16.027085+0000","last_undegraded":"2023-10-01T10:38:16.027085+0000","last_fullsized":"2023-10-01T10:38:16.027085+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":65,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.020849+0000","last_change":"2023-10-01T10:37:05.018755+0000","last_active":"2023-10-01T10:38:16.020849+0000","last_peered":"2023-10-01T10:38:16.020849+0000","last_clean":"2023-10-01T10:38:16.020849+0000","last_became_active":"2023-10-01T10:37:05.018446+0000","last_became_peered":"2023-10-01T10:37:05.018446+0000","last_unstale":"2023-10-01T10:38:16.020849+0000","last_undegraded":"2023-10-01T10:38:16.020849+0000","last_fullsized":"2023-10-01T10:38:16.020849+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:13:53.740503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":107,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.030826+0000","last_change":"2023-10-01T10:38:16.030826+0000","last_active":"2023-10-01T10:38:16.030826+0000","last_peered":"2023-10-01T10:38:16.030826+0000","last_clean":"2023-10-01T10:38:16.030826+0000","last_became_active":"2023-10-01T10:38:16.030277+0000","last_became_peered":"2023-10-01T10:38:16.030277+0000","last_unstale":"2023-10-01T10:38:16.030826+0000","last_undegraded":"2023-10-01T10:38:16.030826+0000","last_fullsized":"2023-10-01T10:38:16.030826+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:31:43.040424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":81,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.034741+0000","last_change":"2023-10-01T10:38:16.034741+0000","last_active":"2023-10-01T10:38:16.034741+0000","last_peered":"2023-10-01T10:38:16.034741+0000","last_clean":"2023-10-01T10:38:16.034741+0000","last_became_active":"2023-10-01T10:38:16.034407+0000","last_became_peered":"2023-10-01T10:38:16.034407+0000","last_unstale":"2023-10-01T10:38:16.034741+0000","last_undegraded":"2023-10-01T10:38:16.034741+0000","last_fullsized":"2023-10-01T10:38:16.034741+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:47:29.911477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":111,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.026777+0000","last_change":"2023-10-01T10:38:16.026777+0000","last_active":"2023-10-01T10:38:16.026777+0000","last_peered":"2023-10-01T10:38:16.026777+0000","last_clean":"2023-10-01T10:38:16.026777+0000","last_became_active":"2023-10-01T10:38:16.026371+0000","last_became_peered":"2023-10-01T10:38:16.026371+0000","last_unstale":"2023-10-01T10:38:16.026777+0000","last_undegraded":"2023-10-01T10:38:16.026777+0000","last_fullsized":"2023-10-01T10:38:16.026777+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":110,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.013942+0000","last_change":"2023-10-01T10:37:42.772319+0000","last_active":"2023-10-01T10:38:16.013942+0000","last_peered":"2023-10-01T10:38:16.013942+0000","last_clean":"2023-10-01T10:38:16.013942+0000","last_became_active":"2023-10-01T10:37:42.771977+0000","last_became_peered":"2023-10-01T10:37:42.771977+0000","last_unstale":"2023-10-01T10:38:16.013942+0000","last_undegraded":"2023-10-01T10:38:16.013942+0000","last_fullsized":"2023-10-01T10:38:16.013942+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:45:12.783214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":82,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.017733+0000","last_change":"2023-10-01T10:37:42.767706+0000","last_active":"2023-10-01T10:38:16.017733+0000","last_peered":"2023-10-01T10:38:16.017733+0000","last_clean":"2023-10-01T10:38:16.017733+0000","last_became_active":"2023-10-01T10:37:42.767211+0000","last_became_peered":"2023-10-01T10:37:42.767211+0000","last_unstale":"2023-10-01T10:38:16.017733+0000","last_undegraded":"2023-10-01T10:38:16.017733+0000","last_fullsized":"2023-10-01T10:38:16.017733+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:37:20.357407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":93,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.034746+0000","last_change":"2023-10-01T10:37:42.768642+0000","last_active":"2023-10-01T10:38:15.034746+0000","last_peered":"2023-10-01T10:38:15.034746+0000","last_clean":"2023-10-01T10:38:15.034746+0000","last_became_active":"2023-10-01T10:37:42.768239+0000","last_became_peered":"2023-10-01T10:37:42.768239+0000","last_unstale":"2023-10-01T10:38:15.034746+0000","last_undegraded":"2023-10-01T10:38:15.034746+0000","last_fullsized":"2023-10-01T10:38:15.034746+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":94,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.028563+0000","last_change":"2023-10-01T10:38:16.028563+0000","last_active":"2023-10-01T10:38:16.028563+0000","last_peered":"2023-10-01T10:38:16.028563+0000","last_clean":"2023-10-01T10:38:16.028563+0000","last_became_active":"2023-10-01T10:38:16.028162+0000","last_became_peered":"2023-10-01T10:38:16.028162+0000","last_unstale":"2023-10-01T10:38:16.028563+0000","last_undegraded":"2023-10-01T10:38:16.028563+0000","last_fullsized":"2023-10-01T10:38:16.028563+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":82,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.014770+0000","last_change":"2023-10-01T10:37:42.772247+0000","last_active":"2023-10-01T10:38:15.014770+0000","last_peered":"2023-10-01T10:38:15.014770+0000","last_clean":"2023-10-01T10:38:15.014770+0000","last_became_active":"2023-10-01T10:37:42.771697+0000","last_became_peered":"2023-10-01T10:37:42.771697+0000","last_unstale":"2023-10-01T10:38:15.014770+0000","last_undegraded":"2023-10-01T10:38:15.014770+0000","last_fullsized":"2023-10-01T10:38:15.014770+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:23:15.018248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":101,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.030438+0000","last_change":"2023-10-01T10:28:20.211798+0000","last_active":"2023-10-01T10:38:15.030438+0000","last_peered":"2023-10-01T10:38:15.030438+0000","last_clean":"2023-10-01T10:38:15.030438+0000","last_became_active":"2023-10-01T10:28:20.210791+0000","last_became_peered":"2023-10-01T10:28:20.210791+0000","last_unstale":"2023-10-01T10:38:15.030438+0000","last_undegraded":"2023-10-01T10:38:15.030438+0000","last_fullsized":"2023-10-01T10:38:15.030438+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":86,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.012486+0000","last_change":"2023-10-01T10:37:42.773127+0000","last_active":"2023-10-01T10:38:15.012486+0000","last_peered":"2023-10-01T10:38:15.012486+0000","last_clean":"2023-10-01T10:38:15.012486+0000","last_became_active":"2023-10-01T10:37:42.772102+0000","last_became_peered":"2023-10-01T10:37:42.772102+0000","last_unstale":"2023-10-01T10:38:15.012486+0000","last_undegraded":"2023-10-01T10:38:15.012486+0000","last_fullsized":"2023-10-01T10:38:15.012486+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:23:26.202295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":110,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.026955+0000","last_change":"2023-10-01T10:38:16.026955+0000","last_active":"2023-10-01T10:38:16.026955+0000","last_peered":"2023-10-01T10:38:16.026955+0000","last_clean":"2023-10-01T10:38:16.026955+0000","last_became_active":"2023-10-01T10:38:16.026552+0000","last_became_peered":"2023-10-01T10:38:16.026552+0000","last_unstale":"2023-10-01T10:38:16.026955+0000","last_undegraded":"2023-10-01T10:38:16.026955+0000","last_fullsized":"2023-10-01T10:38:16.026955+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":78,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.031063+0000","last_change":"2023-10-01T10:15:40.318702+0000","last_active":"2023-10-01T10:38:15.031063+0000","last_peered":"2023-10-01T10:38:15.031063+0000","last_clean":"2023-10-01T10:38:15.031063+0000","last_became_active":"2023-10-01T10:15:40.318333+0000","last_became_peered":"2023-10-01T10:15:40.318333+0000","last_unstale":"2023-10-01T10:38:15.031063+0000","last_undegraded":"2023-10-01T10:38:15.031063+0000","last_fullsized":"2023-10-01T10:38:15.031063+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":111,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.027753+0000","last_change":"2023-10-01T10:38:16.027753+0000","last_active":"2023-10-01T10:38:16.027753+0000","last_peered":"2023-10-01T10:38:16.027753+0000","last_clean":"2023-10-01T10:38:16.027753+0000","last_became_active":"2023-10-01T10:38:16.027384+0000","last_became_peered":"2023-10-01T10:38:16.027384+0000","last_unstale":"2023-10-01T10:38:16.027753+0000","last_undegraded":"2023-10-01T10:38:16.027753+0000","last_fullsized":"2023-10-01T10:38:16.027753+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":115,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.033203+0000","last_change":"2023-10-01T10:38:16.033203+0000","last_active":"2023-10-01T10:38:16.033203+0000","last_peered":"2023-10-01T10:38:16.033203+0000","last_clean":"2023-10-01T10:38:16.033203+0000","last_became_active":"2023-10-01T10:38:16.032681+0000","last_became_peered":"2023-10-01T10:38:16.032681+0000","last_unstale":"2023-10-01T10:38:16.033203+0000","last_undegraded":"2023-10-01T10:38:16.033203+0000","last_fullsized":"2023-10-01T10:38:16.033203+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:42:36.642284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":80,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.023250+0000","last_change":"2023-10-01T10:15:39.293587+0000","last_active":"2023-10-01T10:38:15.023250+0000","last_peered":"2023-10-01T10:38:15.023250+0000","last_clean":"2023-10-01T10:38:15.023250+0000","last_became_active":"2023-10-01T10:15:39.293273+0000","last_became_peered":"2023-10-01T10:15:39.293273+0000","last_unstale":"2023-10-01T10:38:15.023250+0000","last_undegraded":"2023-10-01T10:38:15.023250+0000","last_fullsized":"2023-10-01T10:38:15.023250+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":92,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.031078+0000","last_change":"2023-10-01T10:32:31.571971+0000","last_active":"2023-10-01T10:38:15.031078+0000","last_peered":"2023-10-01T10:38:15.031078+0000","last_clean":"2023-10-01T10:38:15.031078+0000","last_became_active":"2023-10-01T10:32:31.571410+0000","last_became_peered":"2023-10-01T10:32:31.571410+0000","last_unstale":"2023-10-01T10:38:15.031078+0000","last_undegraded":"2023-10-01T10:38:15.031078+0000","last_fullsized":"2023-10-01T10:38:15.031078+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":94,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.034665+0000","last_change":"2023-10-01T10:37:42.765733+0000","last_active":"2023-10-01T10:38:15.034665+0000","last_peered":"2023-10-01T10:38:15.034665+0000","last_clean":"2023-10-01T10:38:15.034665+0000","last_became_active":"2023-10-01T10:37:42.765311+0000","last_became_peered":"2023-10-01T10:37:42.765311+0000","last_unstale":"2023-10-01T10:38:15.034665+0000","last_undegraded":"2023-10-01T10:38:15.034665+0000","last_fullsized":"2023-10-01T10:38:15.034665+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":98,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.027301+0000","last_change":"2023-10-01T10:38:16.026348+0000","last_active":"2023-10-01T10:38:16.027301+0000","last_peered":"2023-10-01T10:38:16.027301+0000","last_clean":"2023-10-01T10:38:16.027301+0000","last_became_active":"2023-10-01T10:38:16.025747+0000","last_became_peered":"2023-10-01T10:38:16.025747+0000","last_unstale":"2023-10-01T10:38:16.027301+0000","last_undegraded":"2023-10-01T10:38:16.027301+0000","last_fullsized":"2023-10-01T10:38:16.027301+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:28:18.934480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":90,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.024148+0000","last_change":"2023-10-01T10:38:16.022493+0000","last_active":"2023-10-01T10:38:16.024148+0000","last_peered":"2023-10-01T10:38:16.024148+0000","last_clean":"2023-10-01T10:38:16.024148+0000","last_became_active":"2023-10-01T10:38:16.022033+0000","last_became_peered":"2023-10-01T10:38:16.022033+0000","last_unstale":"2023-10-01T10:38:16.024148+0000","last_undegraded":"2023-10-01T10:38:16.024148+0000","last_fullsized":"2023-10-01T10:38:16.024148+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:49:05.262763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":100,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.030613+0000","last_change":"2023-10-01T10:38:16.030613+0000","last_active":"2023-10-01T10:38:16.030613+0000","last_peered":"2023-10-01T10:38:16.030613+0000","last_clean":"2023-10-01T10:38:16.030613+0000","last_became_active":"2023-10-01T10:38:16.028975+0000","last_became_peered":"2023-10-01T10:38:16.028975+0000","last_unstale":"2023-10-01T10:38:16.030613+0000","last_undegraded":"2023-10-01T10:38:16.030613+0000","last_fullsized":"2023-10-01T10:38:16.030613+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:46:07.287911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":108,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.030250+0000","last_change":"2023-10-01T10:38:16.030250+0000","last_active":"2023-10-01T10:38:16.030250+0000","last_peered":"2023-10-01T10:38:16.030250+0000","last_clean":"2023-10-01T10:38:16.030250+0000","last_became_active":"2023-10-01T10:38:16.029898+0000","last_became_peered":"2023-10-01T10:38:16.029898+0000","last_unstale":"2023-10-01T10:38:16.030250+0000","last_undegraded":"2023-10-01T10:38:16.030250+0000","last_fullsized":"2023-10-01T10:38:16.030250+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":105,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.035906+0000","last_change":"2023-10-01T10:38:16.035906+0000","last_active":"2023-10-01T10:38:16.035906+0000","last_peered":"2023-10-01T10:38:16.035906+0000","last_clean":"2023-10-01T10:38:16.035906+0000","last_became_active":"2023-10-01T10:38:16.035383+0000","last_became_peered":"2023-10-01T10:38:16.035383+0000","last_unstale":"2023-10-01T10:38:16.035906+0000","last_undegraded":"2023-10-01T10:38:16.035906+0000","last_fullsized":"2023-10-01T10:38:16.035906+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:44:41.500097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":97,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.026611+0000","last_change":"2023-10-01T10:38:16.026611+0000","last_active":"2023-10-01T10:38:16.026611+0000","last_peered":"2023-10-01T10:38:16.026611+0000","last_clean":"2023-10-01T10:38:16.026611+0000","last_became_active":"2023-10-01T10:38:16.026267+0000","last_became_peered":"2023-10-01T10:38:16.026267+0000","last_unstale":"2023-10-01T10:38:16.026611+0000","last_undegraded":"2023-10-01T10:38:16.026611+0000","last_fullsized":"2023-10-01T10:38:16.026611+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:06.135288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":81,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.032162+0000","last_change":"2023-10-01T10:38:16.032162+0000","last_active":"2023-10-01T10:38:16.032162+0000","last_peered":"2023-10-01T10:38:16.032162+0000","last_clean":"2023-10-01T10:38:16.032162+0000","last_became_active":"2023-10-01T10:38:16.031785+0000","last_became_peered":"2023-10-01T10:38:16.031785+0000","last_unstale":"2023-10-01T10:38:16.032162+0000","last_undegraded":"2023-10-01T10:38:16.032162+0000","last_fullsized":"2023-10-01T10:38:16.032162+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:38:43.003626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":88,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.012059+0000","last_change":"2023-10-01T10:37:42.772398+0000","last_active":"2023-10-01T10:38:16.012059+0000","last_peered":"2023-10-01T10:38:16.012059+0000","last_clean":"2023-10-01T10:38:16.012059+0000","last_became_active":"2023-10-01T10:37:42.772084+0000","last_became_peered":"2023-10-01T10:37:42.772084+0000","last_unstale":"2023-10-01T10:38:16.012059+0000","last_undegraded":"2023-10-01T10:38:16.012059+0000","last_fullsized":"2023-10-01T10:38:16.012059+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:32:43.846419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":106,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.021873+0000","last_change":"2023-10-01T10:38:16.019755+0000","last_active":"2023-10-01T10:38:16.021873+0000","last_peered":"2023-10-01T10:38:16.021873+0000","last_clean":"2023-10-01T10:38:16.021873+0000","last_became_active":"2023-10-01T10:38:16.019397+0000","last_became_peered":"2023-10-01T10:38:16.019397+0000","last_unstale":"2023-10-01T10:38:16.021873+0000","last_undegraded":"2023-10-01T10:38:16.021873+0000","last_fullsized":"2023-10-01T10:38:16.021873+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:42:56.623225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":99,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.020037+0000","last_change":"2023-10-01T10:32:31.561108+0000","last_active":"2023-10-01T10:38:16.020037+0000","last_peered":"2023-10-01T10:38:16.020037+0000","last_clean":"2023-10-01T10:38:16.020037+0000","last_became_active":"2023-10-01T10:32:31.560791+0000","last_became_peered":"2023-10-01T10:32:31.560791+0000","last_unstale":"2023-10-01T10:38:16.020037+0000","last_undegraded":"2023-10-01T10:38:16.020037+0000","last_fullsized":"2023-10-01T10:38:16.020037+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:42:47.477671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":109,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.034655+0000","last_change":"2023-10-01T10:37:42.766814+0000","last_active":"2023-10-01T10:38:15.034655+0000","last_peered":"2023-10-01T10:38:15.034655+0000","last_clean":"2023-10-01T10:38:15.034655+0000","last_became_active":"2023-10-01T10:37:42.766451+0000","last_became_peered":"2023-10-01T10:37:42.766451+0000","last_unstale":"2023-10-01T10:38:15.034655+0000","last_undegraded":"2023-10-01T10:38:15.034655+0000","last_fullsized":"2023-10-01T10:38:15.034655+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":65,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.035088+0000","last_change":"2023-10-01T10:38:16.034357+0000","last_active":"2023-10-01T10:38:16.035088+0000","last_peered":"2023-10-01T10:38:16.035088+0000","last_clean":"2023-10-01T10:38:16.035088+0000","last_became_active":"2023-10-01T10:38:16.033905+0000","last_became_peered":"2023-10-01T10:38:16.033905+0000","last_unstale":"2023-10-01T10:38:16.035088+0000","last_undegraded":"2023-10-01T10:38:16.035088+0000","last_fullsized":"2023-10-01T10:38:16.035088+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:47:58.369835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":98,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.034322+0000","last_change":"2023-10-01T10:32:31.572482+0000","last_active":"2023-10-01T10:38:15.034322+0000","last_peered":"2023-10-01T10:38:15.034322+0000","last_clean":"2023-10-01T10:38:15.034322+0000","last_became_active":"2023-10-01T10:32:31.571993+0000","last_became_peered":"2023-10-01T10:32:31.571993+0000","last_unstale":"2023-10-01T10:38:15.034322+0000","last_undegraded":"2023-10-01T10:38:15.034322+0000","last_fullsized":"2023-10-01T10:38:15.034322+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":67,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.030202+0000","last_change":"2023-10-01T10:38:16.030202+0000","last_active":"2023-10-01T10:38:16.030202+0000","last_peered":"2023-10-01T10:38:16.030202+0000","last_clean":"2023-10-01T10:38:16.030202+0000","last_became_active":"2023-10-01T10:38:16.029917+0000","last_became_peered":"2023-10-01T10:38:16.029917+0000","last_unstale":"2023-10-01T10:38:16.030202+0000","last_undegraded":"2023-10-01T10:38:16.030202+0000","last_fullsized":"2023-10-01T10:38:16.030202+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:20:06.763909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":80,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.020177+0000","last_change":"2023-10-01T10:37:42.761153+0000","last_active":"2023-10-01T10:38:16.020177+0000","last_peered":"2023-10-01T10:38:16.020177+0000","last_clean":"2023-10-01T10:38:16.020177+0000","last_became_active":"2023-10-01T10:37:42.760715+0000","last_became_peered":"2023-10-01T10:37:42.760715+0000","last_unstale":"2023-10-01T10:38:16.020177+0000","last_undegraded":"2023-10-01T10:38:16.020177+0000","last_fullsized":"2023-10-01T10:38:16.020177+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:52:50.237174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":97,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.013202+0000","last_change":"2023-10-01T10:37:42.762524+0000","last_active":"2023-10-01T10:38:16.013202+0000","last_peered":"2023-10-01T10:38:16.013202+0000","last_clean":"2023-10-01T10:38:16.013202+0000","last_became_active":"2023-10-01T10:37:42.762209+0000","last_became_peered":"2023-10-01T10:37:42.762209+0000","last_unstale":"2023-10-01T10:38:16.013202+0000","last_undegraded":"2023-10-01T10:38:16.013202+0000","last_fullsized":"2023-10-01T10:38:16.013202+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:45:38.171397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":65,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.020920+0000","last_change":"2023-10-01T10:37:05.019330+0000","last_active":"2023-10-01T10:38:16.020920+0000","last_peered":"2023-10-01T10:38:16.020920+0000","last_clean":"2023-10-01T10:38:16.020920+0000","last_became_active":"2023-10-01T10:37:05.018842+0000","last_became_peered":"2023-10-01T10:37:05.018842+0000","last_unstale":"2023-10-01T10:38:16.020920+0000","last_undegraded":"2023-10-01T10:38:16.020920+0000","last_fullsized":"2023-10-01T10:38:16.020920+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:48:09.551917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":94,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.019778+0000","last_change":"2023-10-01T10:32:31.569162+0000","last_active":"2023-10-01T10:38:16.019778+0000","last_peered":"2023-10-01T10:38:16.019778+0000","last_clean":"2023-10-01T10:38:16.019778+0000","last_became_active":"2023-10-01T10:32:31.568956+0000","last_became_peered":"2023-10-01T10:32:31.568956+0000","last_unstale":"2023-10-01T10:38:16.019778+0000","last_undegraded":"2023-10-01T10:38:16.019778+0000","last_fullsized":"2023-10-01T10:38:16.019778+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:02:08.285304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":98,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.033675+0000","last_change":"2023-10-01T10:37:05.024009+0000","last_active":"2023-10-01T10:38:15.033675+0000","last_peered":"2023-10-01T10:38:15.033675+0000","last_clean":"2023-10-01T10:38:15.033675+0000","last_became_active":"2023-10-01T10:37:05.023458+0000","last_became_peered":"2023-10-01T10:37:05.023458+0000","last_unstale":"2023-10-01T10:38:15.033675+0000","last_undegraded":"2023-10-01T10:38:15.033675+0000","last_fullsized":"2023-10-01T10:38:15.033675+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'139","reported_seq":330,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.019558+0000","last_change":"2023-10-01T10:28:20.217386+0000","last_active":"2023-10-01T10:38:15.019558+0000","last_peered":"2023-10-01T10:38:15.019558+0000","last_clean":"2023-10-01T10:38:15.019558+0000","last_became_active":"2023-10-01T10:28:20.216803+0000","last_became_peered":"2023-10-01T10:28:20.216803+0000","last_unstale":"2023-10-01T10:38:15.019558+0000","last_undegraded":"2023-10-01T10:38:15.019558+0000","last_fullsized":"2023-10-01T10:38:15.019558+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:09:17.221082+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:09:17.221082+0000","last_clean_scrub_stamp":"2023-10-01T10:09:17.221082+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":70,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.011905+0000","last_change":"2023-10-01T10:37:42.765100+0000","last_active":"2023-10-01T10:38:16.011905+0000","last_peered":"2023-10-01T10:38:16.011905+0000","last_clean":"2023-10-01T10:38:16.011905+0000","last_became_active":"2023-10-01T10:37:42.764761+0000","last_became_peered":"2023-10-01T10:37:42.764761+0000","last_unstale":"2023-10-01T10:38:16.011905+0000","last_undegraded":"2023-10-01T10:38:16.011905+0000","last_fullsized":"2023-10-01T10:38:16.011905+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:24:56.907377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":92,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.031255+0000","last_change":"2023-10-01T10:38:16.031255+0000","last_active":"2023-10-01T10:38:16.031255+0000","last_peered":"2023-10-01T10:38:16.031255+0000","last_clean":"2023-10-01T10:38:16.031255+0000","last_became_active":"2023-10-01T10:38:16.030886+0000","last_became_peered":"2023-10-01T10:38:16.030886+0000","last_unstale":"2023-10-01T10:38:16.031255+0000","last_undegraded":"2023-10-01T10:38:16.031255+0000","last_fullsized":"2023-10-01T10:38:16.031255+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":113,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.019451+0000","last_change":"2023-10-01T10:32:31.565868+0000","last_active":"2023-10-01T10:38:15.019451+0000","last_peered":"2023-10-01T10:38:15.019451+0000","last_clean":"2023-10-01T10:38:15.019451+0000","last_became_active":"2023-10-01T10:32:31.565428+0000","last_became_peered":"2023-10-01T10:32:31.565428+0000","last_unstale":"2023-10-01T10:38:15.019451+0000","last_undegraded":"2023-10-01T10:38:15.019451+0000","last_fullsized":"2023-10-01T10:38:15.019451+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":97,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.022511+0000","last_change":"2023-10-01T10:38:16.021968+0000","last_active":"2023-10-01T10:38:16.022511+0000","last_peered":"2023-10-01T10:38:16.022511+0000","last_clean":"2023-10-01T10:38:16.022511+0000","last_became_active":"2023-10-01T10:38:16.021539+0000","last_became_peered":"2023-10-01T10:38:16.021539+0000","last_unstale":"2023-10-01T10:38:16.022511+0000","last_undegraded":"2023-10-01T10:38:16.022511+0000","last_fullsized":"2023-10-01T10:38:16.022511+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:12:53.105747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":98,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.019580+0000","last_change":"2023-10-01T10:37:05.017431+0000","last_active":"2023-10-01T10:38:16.019580+0000","last_peered":"2023-10-01T10:38:16.019580+0000","last_clean":"2023-10-01T10:38:16.019580+0000","last_became_active":"2023-10-01T10:37:05.017094+0000","last_became_peered":"2023-10-01T10:37:05.017094+0000","last_unstale":"2023-10-01T10:38:16.019580+0000","last_undegraded":"2023-10-01T10:38:16.019580+0000","last_fullsized":"2023-10-01T10:38:16.019580+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:10:40.664686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":113,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.032774+0000","last_change":"2023-10-01T10:38:16.032774+0000","last_active":"2023-10-01T10:38:16.032774+0000","last_peered":"2023-10-01T10:38:16.032774+0000","last_clean":"2023-10-01T10:38:16.032774+0000","last_became_active":"2023-10-01T10:38:16.032283+0000","last_became_peered":"2023-10-01T10:38:16.032283+0000","last_unstale":"2023-10-01T10:38:16.032774+0000","last_undegraded":"2023-10-01T10:38:16.032774+0000","last_fullsized":"2023-10-01T10:38:16.032774+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":65,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.020296+0000","last_change":"2023-10-01T10:37:05.021228+0000","last_active":"2023-10-01T10:38:16.020296+0000","last_peered":"2023-10-01T10:38:16.020296+0000","last_clean":"2023-10-01T10:38:16.020296+0000","last_became_active":"2023-10-01T10:37:05.020888+0000","last_became_peered":"2023-10-01T10:37:05.020888+0000","last_unstale":"2023-10-01T10:38:16.020296+0000","last_undegraded":"2023-10-01T10:38:16.020296+0000","last_fullsized":"2023-10-01T10:38:16.020296+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:59:57.641247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":68,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.020664+0000","last_change":"2023-10-01T10:28:20.220812+0000","last_active":"2023-10-01T10:38:16.020664+0000","last_peered":"2023-10-01T10:38:16.020664+0000","last_clean":"2023-10-01T10:38:16.020664+0000","last_became_active":"2023-10-01T10:28:20.215962+0000","last_became_peered":"2023-10-01T10:28:20.215962+0000","last_unstale":"2023-10-01T10:38:16.020664+0000","last_undegraded":"2023-10-01T10:38:16.020664+0000","last_fullsized":"2023-10-01T10:38:16.020664+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:35:15.275097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":75,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.014170+0000","last_change":"2023-10-01T10:37:42.773999+0000","last_active":"2023-10-01T10:38:16.014170+0000","last_peered":"2023-10-01T10:38:16.014170+0000","last_clean":"2023-10-01T10:38:16.014170+0000","last_became_active":"2023-10-01T10:37:42.773355+0000","last_became_peered":"2023-10-01T10:37:42.773355+0000","last_unstale":"2023-10-01T10:38:16.014170+0000","last_undegraded":"2023-10-01T10:38:16.014170+0000","last_fullsized":"2023-10-01T10:38:16.014170+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:46:51.317151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":107,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.035022+0000","last_change":"2023-10-01T10:38:16.034193+0000","last_active":"2023-10-01T10:38:16.035022+0000","last_peered":"2023-10-01T10:38:16.035022+0000","last_clean":"2023-10-01T10:38:16.035022+0000","last_became_active":"2023-10-01T10:38:16.033703+0000","last_became_peered":"2023-10-01T10:38:16.033703+0000","last_unstale":"2023-10-01T10:38:16.035022+0000","last_undegraded":"2023-10-01T10:38:16.035022+0000","last_fullsized":"2023-10-01T10:38:16.035022+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:21:26.061595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":72,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.011874+0000","last_change":"2023-10-01T10:37:42.773538+0000","last_active":"2023-10-01T10:38:16.011874+0000","last_peered":"2023-10-01T10:38:16.011874+0000","last_clean":"2023-10-01T10:38:16.011874+0000","last_became_active":"2023-10-01T10:37:42.771863+0000","last_became_peered":"2023-10-01T10:37:42.771863+0000","last_unstale":"2023-10-01T10:38:16.011874+0000","last_undegraded":"2023-10-01T10:38:16.011874+0000","last_fullsized":"2023-10-01T10:38:16.011874+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:46:46.698627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":97,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.031790+0000","last_change":"2023-10-01T10:37:05.023034+0000","last_active":"2023-10-01T10:38:15.031790+0000","last_peered":"2023-10-01T10:38:15.031790+0000","last_clean":"2023-10-01T10:38:15.031790+0000","last_became_active":"2023-10-01T10:37:05.022670+0000","last_became_peered":"2023-10-01T10:37:05.022670+0000","last_unstale":"2023-10-01T10:38:15.031790+0000","last_undegraded":"2023-10-01T10:38:15.031790+0000","last_fullsized":"2023-10-01T10:38:15.031790+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":101,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.027938+0000","last_change":"2023-10-01T10:38:16.027472+0000","last_active":"2023-10-01T10:38:16.027938+0000","last_peered":"2023-10-01T10:38:16.027938+0000","last_clean":"2023-10-01T10:38:16.027938+0000","last_became_active":"2023-10-01T10:38:16.026810+0000","last_became_peered":"2023-10-01T10:38:16.026810+0000","last_unstale":"2023-10-01T10:38:16.027938+0000","last_undegraded":"2023-10-01T10:38:16.027938+0000","last_fullsized":"2023-10-01T10:38:16.027938+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:30:06.978187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":65,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.035705+0000","last_change":"2023-10-01T10:38:16.035705+0000","last_active":"2023-10-01T10:38:16.035705+0000","last_peered":"2023-10-01T10:38:16.035705+0000","last_clean":"2023-10-01T10:38:16.035705+0000","last_became_active":"2023-10-01T10:38:16.035389+0000","last_became_peered":"2023-10-01T10:38:16.035389+0000","last_unstale":"2023-10-01T10:38:16.035705+0000","last_undegraded":"2023-10-01T10:38:16.035705+0000","last_fullsized":"2023-10-01T10:38:16.035705+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:16:10.201409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":85,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.028785+0000","last_change":"2023-10-01T10:38:16.028491+0000","last_active":"2023-10-01T10:38:16.028785+0000","last_peered":"2023-10-01T10:38:16.028785+0000","last_clean":"2023-10-01T10:38:16.028785+0000","last_became_active":"2023-10-01T10:38:16.028175+0000","last_became_peered":"2023-10-01T10:38:16.028175+0000","last_unstale":"2023-10-01T10:38:16.028785+0000","last_undegraded":"2023-10-01T10:38:16.028785+0000","last_fullsized":"2023-10-01T10:38:16.028785+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:44:29.083129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":107,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.012738+0000","last_change":"2023-10-01T10:37:42.762360+0000","last_active":"2023-10-01T10:38:16.012738+0000","last_peered":"2023-10-01T10:38:16.012738+0000","last_clean":"2023-10-01T10:38:16.012738+0000","last_became_active":"2023-10-01T10:37:42.761936+0000","last_became_peered":"2023-10-01T10:37:42.761936+0000","last_unstale":"2023-10-01T10:38:16.012738+0000","last_undegraded":"2023-10-01T10:38:16.012738+0000","last_fullsized":"2023-10-01T10:38:16.012738+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:50.637582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":111,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.037809+0000","last_change":"2023-10-01T10:38:16.037809+0000","last_active":"2023-10-01T10:38:16.037809+0000","last_peered":"2023-10-01T10:38:16.037809+0000","last_clean":"2023-10-01T10:38:16.037809+0000","last_became_active":"2023-10-01T10:38:16.037318+0000","last_became_peered":"2023-10-01T10:38:16.037318+0000","last_unstale":"2023-10-01T10:38:16.037809+0000","last_undegraded":"2023-10-01T10:38:16.037809+0000","last_fullsized":"2023-10-01T10:38:16.037809+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":115,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.034282+0000","last_change":"2023-10-01T10:38:16.034282+0000","last_active":"2023-10-01T10:38:16.034282+0000","last_peered":"2023-10-01T10:38:16.034282+0000","last_clean":"2023-10-01T10:38:16.034282+0000","last_became_active":"2023-10-01T10:38:16.033820+0000","last_became_peered":"2023-10-01T10:38:16.033820+0000","last_unstale":"2023-10-01T10:38:16.034282+0000","last_undegraded":"2023-10-01T10:38:16.034282+0000","last_fullsized":"2023-10-01T10:38:16.034282+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:48:23.815832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":96,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.038077+0000","last_change":"2023-10-01T10:38:16.038077+0000","last_active":"2023-10-01T10:38:16.038077+0000","last_peered":"2023-10-01T10:38:16.038077+0000","last_clean":"2023-10-01T10:38:16.038077+0000","last_became_active":"2023-10-01T10:38:16.037313+0000","last_became_peered":"2023-10-01T10:38:16.037313+0000","last_unstale":"2023-10-01T10:38:16.038077+0000","last_undegraded":"2023-10-01T10:38:16.038077+0000","last_fullsized":"2023-10-01T10:38:16.038077+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":115,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.025600+0000","last_change":"2023-10-01T10:38:16.025600+0000","last_active":"2023-10-01T10:38:16.025600+0000","last_peered":"2023-10-01T10:38:16.025600+0000","last_clean":"2023-10-01T10:38:16.025600+0000","last_became_active":"2023-10-01T10:38:16.025252+0000","last_became_peered":"2023-10-01T10:38:16.025252+0000","last_unstale":"2023-10-01T10:38:16.025600+0000","last_undegraded":"2023-10-01T10:38:16.025600+0000","last_fullsized":"2023-10-01T10:38:16.025600+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:24:18.871963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":103,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.013437+0000","last_change":"2023-10-01T10:37:42.761243+0000","last_active":"2023-10-01T10:38:16.013437+0000","last_peered":"2023-10-01T10:38:16.013437+0000","last_clean":"2023-10-01T10:38:16.013437+0000","last_became_active":"2023-10-01T10:37:42.760834+0000","last_became_peered":"2023-10-01T10:37:42.760834+0000","last_unstale":"2023-10-01T10:38:16.013437+0000","last_undegraded":"2023-10-01T10:38:16.013437+0000","last_fullsized":"2023-10-01T10:38:16.013437+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:37:21.976095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":115,"reported_epoch":126,"state":"active+clean","last_fresh":"2023-10-01T10:38:15.014918+0000","last_change":"2023-10-01T10:37:42.765142+0000","last_active":"2023-10-01T10:38:15.014918+0000","last_peered":"2023-10-01T10:38:15.014918+0000","last_clean":"2023-10-01T10:38:15.014918+0000","last_became_active":"2023-10-01T10:37:42.764770+0000","last_became_peered":"2023-10-01T10:37:42.764770+0000","last_unstale":"2023-10-01T10:38:15.014918+0000","last_undegraded":"2023-10-01T10:38:15.014918+0000","last_fullsized":"2023-10-01T10:38:15.014918+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:23:33.527876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":86,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.019521+0000","last_change":"2023-10-01T10:37:05.021621+0000","last_active":"2023-10-01T10:38:16.019521+0000","last_peered":"2023-10-01T10:38:16.019521+0000","last_clean":"2023-10-01T10:38:16.019521+0000","last_became_active":"2023-10-01T10:37:05.021173+0000","last_became_peered":"2023-10-01T10:37:05.021173+0000","last_unstale":"2023-10-01T10:38:16.019521+0000","last_undegraded":"2023-10-01T10:38:16.019521+0000","last_fullsized":"2023-10-01T10:38:16.019521+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:13:25.364499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":81,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-10-01T10:38:16.031427+0000","last_change":"2023-10-01T10:38:16.031427+0000","last_active":"2023-10-01T10:38:16.031427+0000","last_peered":"2023-10-01T10:38:16.031427+0000","last_clean":"2023-10-01T10:38:16.031427+0000","last_became_active":"2023-10-01T10:38:16.029786+0000","last_became_peered":"2023-10-01T10:38:16.029786+0000","last_unstale":"2023-10-01T10:38:16.031427+0000","last_undegraded":"2023-10-01T10:38:16.031427+0000","last_fullsized":"2023-10-01T10:38:16.031427+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:53:22.796191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30319,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430537,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":303116,"kb_used_data":1796,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":93442036,"statfs":{"total":95995035648,"available":95684644864,"internally_reserved":0,"allocated":1839104,"data_stored":1363146,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:39:40 2023","interfaces":[{"interface":"back","average":{"1min":0.84199999999999997,"5min":1.3979999999999999,"15min":1.657},"min":{"1min":0.57499999999999996,"5min":0.309,"15min":0.27500000000000002},"max":{"1min":1.1639999999999999,"5min":7.9900000000000002,"15min":14.974},"last":0.995},{"interface":"front","average":{"1min":0.84299999999999997,"5min":1.4670000000000001,"15min":1.7250000000000001},"min":{"1min":0.53800000000000003,"5min":0.36499999999999999,"15min":0.29999999999999999},"max":{"1min":1.21,"5min":9.7289999999999992,"15min":15.029},"last":1.0329999999999999}]},{"osd":1,"last update":"Sun Oct 1 10:39:52 2023","interfaces":[{"interface":"back","average":{"1min":0.90400000000000003,"5min":1.1930000000000001,"15min":1.216},"min":{"1min":0.67800000000000005,"5min":0.35999999999999999,"15min":0.309},"max":{"1min":1.1850000000000001,"5min":8.1180000000000003,"15min":8.1180000000000003},"last":0.90200000000000002},{"interface":"front","average":{"1min":0.90100000000000002,"5min":1.159,"15min":1.5629999999999999},"min":{"1min":0.66200000000000003,"5min":0.318,"15min":0.318},"max":{"1min":1.234,"5min":11.266,"15min":11.266},"last":0.94699999999999995}]},{"osd":2,"last update":"Sun Oct 1 10:39:05 2023","interfaces":[{"interface":"back","average":{"1min":0.96799999999999997,"5min":1.022,"15min":1.0309999999999999},"min":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"max":{"1min":1.1799999999999999,"5min":1.214,"15min":1.214},"last":1.1839999999999999},{"interface":"front","average":{"1min":0.89600000000000002,"5min":0.93600000000000005,"15min":0.94299999999999995},"min":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.167,"5min":1.294,"15min":1.294},"last":1.2170000000000001}]},{"osd":3,"last update":"Sun Oct 1 10:39:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0209999999999999,"5min":0.97799999999999998,"15min":0.97099999999999997},"min":{"1min":0.69699999999999995,"5min":0.55700000000000005,"15min":0.55700000000000005},"max":{"1min":1.202,"5min":1.3879999999999999,"15min":1.3879999999999999},"last":1.292},{"interface":"front","average":{"1min":0.94599999999999995,"5min":0.94599999999999995,"15min":0.94599999999999995},"min":{"1min":0.748,"5min":0.64400000000000002,"15min":0.64400000000000002},"max":{"1min":1.1359999999999999,"5min":1.234,"15min":1.234},"last":1.133}]},{"osd":4,"last update":"Sun Oct 1 10:39:18 2023","interfaces":[{"interface":"back","average":{"1min":0.91700000000000004,"5min":0.91700000000000004,"15min":0.91700000000000004},"min":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"max":{"1min":1.167,"5min":1.167,"15min":1.167},"last":0.749},{"interface":"front","average":{"1min":1.1859999999999999,"5min":1.1859999999999999,"15min":1.1859999999999999},"min":{"1min":0.90500000000000003,"5min":0.90500000000000003,"15min":0.90500000000000003},"max":{"1min":1.333,"5min":1.333,"15min":1.333},"last":1.2430000000000001}]},{"osd":5,"last update":"Sun Oct 1 10:39:52 2023","interfaces":[{"interface":"back","average":{"1min":1.1240000000000001,"5min":1.1859999999999999,"15min":1.2629999999999999},"min":{"1min":0.74299999999999999,"5min":0.35799999999999998,"15min":0.29699999999999999},"max":{"1min":1.389,"5min":5.8230000000000004,"15min":10.106999999999999},"last":1.087},{"interface":"front","average":{"1min":1.054,"5min":1.157,"15min":1.1850000000000001},"min":{"1min":0.877,"5min":0.432,"15min":0.34799999999999998},"max":{"1min":1.26,"5min":5.7800000000000002,"15min":6.2050000000000001},"last":0.82799999999999996}]},{"osd":6,"last update":"Sun Oct 1 10:39:52 2023","interfaces":[{"interface":"back","average":{"1min":1.123,"5min":1.417,"15min":1.3520000000000001},"min":{"1min":0.82999999999999996,"5min":0.44700000000000001,"15min":0.32800000000000001},"max":{"1min":1.431,"5min":7.3300000000000001,"15min":9.5500000000000007},"last":1.3420000000000001},{"interface":"front","average":{"1min":1.1259999999999999,"5min":1.2929999999999999,"15min":1.3500000000000001},"min":{"1min":0.83799999999999997,"5min":0.40100000000000002,"15min":0.33900000000000002},"max":{"1min":1.355,"5min":6.9370000000000003,"15min":12.696},"last":1.1619999999999999}]}]},{"osd":6,"up_from":42,"seq":180388626771,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":307196,"kb_used_data":1780,"kb_used_omap":0,"kb_used_meta":305408,"kb_avail":93437956,"statfs":{"total":95995035648,"available":95680466944,"internally_reserved":0,"allocated":1822720,"data_stored":1362685,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312737792},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:39:34 2023","interfaces":[{"interface":"back","average":{"1min":0.878,"5min":1.3200000000000001,"15min":1.4490000000000001},"min":{"1min":0.53800000000000003,"5min":0.29399999999999998,"15min":0.28499999999999998},"max":{"1min":1.236,"5min":7.1139999999999999,"15min":9.1300000000000008},"last":0.84699999999999998},{"interface":"front","average":{"1min":0.85299999999999998,"5min":1.381,"15min":1.46},"min":{"1min":0.60399999999999998,"5min":0.29999999999999999,"15min":0.28899999999999998},"max":{"1min":1.238,"5min":6.9640000000000004,"15min":7.3109999999999999},"last":1.0009999999999999}]},{"osd":1,"last update":"Sun Oct 1 10:39:42 2023","interfaces":[{"interface":"back","average":{"1min":0.94799999999999995,"5min":1.4390000000000001,"15min":1.1479999999999999},"min":{"1min":0.59799999999999998,"5min":0.37,"15min":0.26600000000000001},"max":{"1min":1.2010000000000001,"5min":7.9100000000000001,"15min":7.9100000000000001},"last":1.306},{"interface":"front","average":{"1min":0.89100000000000001,"5min":1.288,"15min":1.452},"min":{"1min":0.63900000000000001,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.3089999999999999,"5min":7.4199999999999999,"15min":7.4199999999999999},"last":1.258}]},{"osd":2,"last update":"Sun Oct 1 10:39:05 2023","interfaces":[{"interface":"back","average":{"1min":0.88700000000000001,"5min":0.87,"15min":0.86699999999999999},"min":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"max":{"1min":1.2450000000000001,"5min":1.2450000000000001,"15min":1.2450000000000001},"last":1.0900000000000001},{"interface":"front","average":{"1min":0.95099999999999996,"5min":0.94099999999999995,"15min":0.93899999999999995},"min":{"1min":0.70899999999999996,"5min":0.64900000000000002,"15min":0.64900000000000002},"max":{"1min":1.1850000000000001,"5min":1.2929999999999999,"15min":1.2929999999999999},"last":1.361}]},{"osd":3,"last update":"Sun Oct 1 10:39:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0329999999999999,"5min":0.99099999999999999,"15min":0.98399999999999999},"min":{"1min":0.82599999999999996,"5min":0.69799999999999995,"15min":0.69799999999999995},"max":{"1min":1.2390000000000001,"5min":1.2529999999999999,"15min":1.2529999999999999},"last":1.1830000000000001},{"interface":"front","average":{"1min":1.024,"5min":1.0189999999999999,"15min":1.018},"min":{"1min":0.89200000000000002,"5min":0.78300000000000003,"15min":0.78300000000000003},"max":{"1min":1.2230000000000001,"5min":1.2230000000000001,"15min":1.2230000000000001},"last":0.96399999999999997}]},{"osd":4,"last update":"Sun Oct 1 10:39:18 2023","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":1.02,"15min":1.02},"min":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"max":{"1min":1.3919999999999999,"5min":1.3919999999999999,"15min":1.3919999999999999},"last":0.91900000000000004},{"interface":"front","average":{"1min":1.103,"5min":1.103,"15min":1.103},"min":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"max":{"1min":1.4179999999999999,"5min":1.4179999999999999,"15min":1.4179999999999999},"last":1.2330000000000001}]},{"osd":5,"last update":"Sun Oct 1 10:39:53 2023","interfaces":[{"interface":"back","average":{"1min":1.115,"5min":1.3240000000000001,"15min":1.214},"min":{"1min":0.873,"5min":0.46100000000000002,"15min":0.373},"max":{"1min":1.369,"5min":6.085,"15min":8.0150000000000006},"last":1.387},{"interface":"front","average":{"1min":1.113,"5min":1.1870000000000001,"15min":1.2490000000000001},"min":{"1min":0.72699999999999998,"5min":0.38,"15min":0.34399999999999997},"max":{"1min":1.472,"5min":10.842000000000001,"15min":10.842000000000001},"last":1.3340000000000001}]},{"osd":7,"last update":"Sun Oct 1 10:39:34 2023","interfaces":[{"interface":"back","average":{"1min":1.147,"5min":1.089,"15min":1.077},"min":{"1min":0.94599999999999995,"5min":0.39700000000000002,"15min":0.371},"max":{"1min":1.454,"5min":4.3499999999999996,"15min":4.867},"last":1.1519999999999999},{"interface":"front","average":{"1min":1.196,"5min":1.2589999999999999,"15min":1.331},"min":{"1min":1.0169999999999999,"5min":0.42299999999999999,"15min":0.34799999999999998},"max":{"1min":1.3460000000000001,"5min":5.3470000000000004,"15min":11.031000000000001},"last":1.444}]}]},{"osd":1,"up_from":101,"seq":433791696988,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":251816,"kb_used_data":1312,"kb_used_omap":4,"kb_used_meta":250491,"kb_avail":93493336,"statfs":{"total":95995035648,"available":95737176064,"internally_reserved":0,"allocated":1343488,"data_stored":789975,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":256503694},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:39:47 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":1.236,"15min":2.0209999999999999},"min":{"1min":0.34999999999999998,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.85999999999999999,"5min":11.071999999999999,"15min":11.071999999999999},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.60999999999999999,"5min":1.1739999999999999,"15min":1.97},"min":{"1min":0.31900000000000001,"5min":0.21199999999999999,"15min":0.20499999999999999},"max":{"1min":0.97699999999999998,"5min":7.9900000000000002,"15min":9.3230000000000004},"last":0.71699999999999997}]},{"osd":2,"last update":"Sun Oct 1 10:39:12 2023","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.94499999999999995,"5min":1.0589999999999999,"15min":1.0589999999999999},"last":0.873},{"interface":"front","average":{"1min":0.73399999999999999,"5min":0.76000000000000001,"15min":0.76400000000000001},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.0309999999999999,"5min":1.1319999999999999,"15min":1.1319999999999999},"last":0.79600000000000004}]},{"osd":3,"last update":"Sun Oct 1 10:39:47 2023","interfaces":[{"interface":"back","average":{"1min":0.69699999999999995,"5min":0.68100000000000005,"15min":0.67800000000000005},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":1.0820000000000001,"5min":1.0820000000000001,"15min":1.0820000000000001},"last":0.754},{"interface":"front","average":{"1min":0.68100000000000005,"5min":0.73799999999999999,"15min":0.748},"min":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":1.022,"5min":1.022,"15min":1.022},"last":0.95499999999999996}]},{"osd":4,"last update":"Sun Oct 1 10:39:18 2023","interfaces":[{"interface":"back","average":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88200000000000001},"min":{"1min":0.627,"5min":0.627,"15min":0.627},"max":{"1min":1.165,"5min":1.165,"15min":1.165},"last":1.2230000000000001},{"interface":"front","average":{"1min":1.008,"5min":1.008,"15min":1.008},"min":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"max":{"1min":1.268,"5min":1.268,"15min":1.268},"last":0.84399999999999997}]},{"osd":5,"last update":"Sun Oct 1 10:39:47 2023","interfaces":[{"interface":"back","average":{"1min":0.93600000000000005,"5min":1.341,"15min":1.7450000000000001},"min":{"1min":0.68400000000000005,"5min":0.372,"15min":0.35799999999999998},"max":{"1min":1.238,"5min":8.9429999999999996,"15min":8.9429999999999996},"last":1.0529999999999999},{"interface":"front","average":{"1min":1.0189999999999999,"5min":1.492,"15min":2.3119999999999998},"min":{"1min":0.68999999999999995,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":1.385,"5min":9.1300000000000008,"15min":11.353999999999999},"last":1.151}]},{"osd":6,"last update":"Sun Oct 1 10:39:47 2023","interfaces":[{"interface":"back","average":{"1min":1.046,"5min":1.325,"15min":1.9770000000000001},"min":{"1min":0.68400000000000005,"5min":0.37,"15min":0.37},"max":{"1min":1.387,"5min":6.6849999999999996,"15min":8.6630000000000003},"last":1.264},{"interface":"front","average":{"1min":1.038,"5min":1.4359999999999999,"15min":1.9430000000000001},"min":{"1min":0.69999999999999996,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":1.5529999999999999,"5min":6,"15min":7.1369999999999996},"last":1.1279999999999999}]},{"osd":7,"last update":"Sun Oct 1 10:39:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0800000000000001,"5min":1.452,"15min":1.8879999999999999},"min":{"1min":0.57299999999999995,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":1.474,"5min":6.0510000000000002,"15min":7.1680000000000001},"last":1.1850000000000001},{"interface":"front","average":{"1min":1.032,"5min":1.478,"15min":1.7410000000000001},"min":{"1min":0.71799999999999997,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":1.365,"5min":9.0749999999999993,"15min":9.0749999999999993},"last":1.381}]}]},{"osd":0,"up_from":86,"seq":369367187598,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":294004,"kb_used_data":1900,"kb_used_omap":5,"kb_used_meta":292090,"kb_avail":93451148,"statfs":{"total":95995035648,"available":95693975552,"internally_reserved":0,"allocated":1945600,"data_stored":1378971,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":299100794},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 10:39:45 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":1.395,"15min":1.6299999999999999},"min":{"1min":0.28199999999999997,"5min":0.224,"15min":0.214},"max":{"1min":0.92600000000000005,"5min":11.997999999999999,"15min":12.855},"last":0.95599999999999996},{"interface":"front","average":{"1min":0.56899999999999995,"5min":1.1140000000000001,"15min":2.335},"min":{"1min":0.32000000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":1.1180000000000001,"5min":6.8490000000000002,"15min":15.48},"last":0.84999999999999998}]},{"osd":2,"last update":"Sun Oct 1 10:39:07 2023","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.69099999999999995,"15min":0.69799999999999995},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":1.085,"5min":1.085,"15min":1.085},"last":0.67300000000000004},{"interface":"front","average":{"1min":0.70999999999999996,"5min":0.74399999999999999,"15min":0.75},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":1.149,"5min":1.149,"15min":1.149},"last":0.86899999999999999}]},{"osd":3,"last update":"Sun Oct 1 10:39:45 2023","interfaces":[{"interface":"back","average":{"1min":0.68400000000000005,"5min":0.78800000000000003,"15min":0.80600000000000005},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.96599999999999997,"5min":1.3069999999999999,"15min":1.3069999999999999},"last":0.77400000000000002},{"interface":"front","average":{"1min":0.67000000000000004,"5min":0.79200000000000004,"15min":0.81200000000000006},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.95299999999999996,"5min":1.1060000000000001,"15min":1.1060000000000001},"last":0.59599999999999997}]},{"osd":4,"last update":"Sun Oct 1 10:39:18 2023","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":1.02,"15min":1.02},"min":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"max":{"1min":1.3200000000000001,"5min":1.3200000000000001,"15min":1.3200000000000001},"last":1.1060000000000001},{"interface":"front","average":{"1min":1.0349999999999999,"5min":1.0349999999999999,"15min":1.0349999999999999},"min":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"max":{"1min":1.5029999999999999,"5min":1.5029999999999999,"15min":1.5029999999999999},"last":1.149}]},{"osd":5,"last update":"Sun Oct 1 10:39:45 2023","interfaces":[{"interface":"back","average":{"1min":0.97499999999999998,"5min":1.3759999999999999,"15min":1.722},"min":{"1min":0.78300000000000003,"5min":0.32000000000000001,"15min":0.31900000000000001},"max":{"1min":1.3440000000000001,"5min":8.0690000000000008,"15min":13.068},"last":1.0580000000000001},{"interface":"front","average":{"1min":0.96499999999999997,"5min":1.5489999999999999,"15min":1.7190000000000001},"min":{"1min":0.60199999999999998,"5min":0.39300000000000002,"15min":0.34200000000000003},"max":{"1min":1.258,"5min":11.747,"15min":11.747},"last":1.036}]},{"osd":6,"last update":"Sun Oct 1 10:39:45 2023","interfaces":[{"interface":"back","average":{"1min":1.006,"5min":1.8129999999999999,"15min":1.825},"min":{"1min":0.69899999999999995,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":1.321,"5min":11.663,"15min":11.663},"last":0.82499999999999996},{"interface":"front","average":{"1min":0.91300000000000003,"5min":1.5840000000000001,"15min":1.766},"min":{"1min":0.56999999999999995,"5min":0.40400000000000003,"15min":0.38600000000000001},"max":{"1min":1.2789999999999999,"5min":15.069000000000001,"15min":15.069000000000001},"last":0.93200000000000005}]},{"osd":7,"last update":"Sun Oct 1 10:39:45 2023","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":1.6279999999999999,"15min":1.8680000000000001},"min":{"1min":0.73399999999999999,"5min":0.40300000000000002,"15min":0.39000000000000001},"max":{"1min":1.3819999999999999,"5min":11.941000000000001,"15min":12.686},"last":0.90700000000000003},{"interface":"front","average":{"1min":0.93400000000000005,"5min":1.4450000000000001,"15min":1.6100000000000001},"min":{"1min":0.59099999999999997,"5min":0.438,"15min":0.36199999999999999},"max":{"1min":1.323,"5min":6.8150000000000004,"15min":13.654},"last":1.085}]}]},{"osd":2,"up_from":116,"seq":498216206374,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":155752,"kb_used_data":1248,"kb_used_omap":7,"kb_used_meta":154488,"kb_avail":93589400,"statfs":{"total":95995035648,"available":95835545600,"internally_reserved":0,"allocated":1277952,"data_stored":778803,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":158195793},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:39:07 2023","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":0.66800000000000004,"15min":0.67400000000000004},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":1.002,"5min":1.0660000000000001,"15min":1.0660000000000001},"last":0.70099999999999996},{"interface":"front","average":{"1min":0.73399999999999999,"5min":0.78200000000000003,"15min":0.79000000000000004},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":1.0760000000000001,"5min":1.1140000000000001,"15min":1.1140000000000001},"last":0.92700000000000005}]},{"osd":1,"last update":"Sun Oct 1 10:39:07 2023","interfaces":[{"interface":"back","average":{"1min":0.73799999999999999,"5min":0.71699999999999997,"15min":0.71299999999999997},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":1.087,"5min":1.087,"15min":1.087},"last":1.1220000000000001},{"interface":"front","average":{"1min":0.67200000000000004,"5min":0.77500000000000002,"15min":0.79200000000000004},"min":{"1min":0.36499999999999999,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.91900000000000004,"5min":1.1479999999999999,"15min":1.1479999999999999},"last":0.96299999999999997}]},{"osd":3,"last update":"Sun Oct 1 10:39:45 2023","interfaces":[{"interface":"back","average":{"1min":0.67800000000000005,"5min":0.72799999999999998,"15min":0.73599999999999999},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":1.0740000000000001,"5min":1.0740000000000001,"15min":1.0740000000000001},"last":0.84199999999999997},{"interface":"front","average":{"1min":0.77800000000000002,"5min":0.86499999999999999,"15min":0.879},"min":{"1min":0.50600000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":1.1479999999999999,"5min":1.2,"15min":1.2},"last":0.748}]},{"osd":4,"last update":"Sun Oct 1 10:39:18 2023","interfaces":[{"interface":"back","average":{"1min":0.98099999999999998,"5min":0.98099999999999998,"15min":0.98099999999999998},"min":{"1min":0.745,"5min":0.745,"15min":0.745},"max":{"1min":1.28,"5min":1.28,"15min":1.28},"last":1.004},{"interface":"front","average":{"1min":0.95699999999999996,"5min":0.95699999999999996,"15min":0.95699999999999996},"min":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"max":{"1min":1.254,"5min":1.254,"15min":1.254},"last":0.79700000000000004}]},{"osd":5,"last update":"Sun Oct 1 10:39:07 2023","interfaces":[{"interface":"back","average":{"1min":1.032,"5min":1.1040000000000001,"15min":1.1160000000000001},"min":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"max":{"1min":1.468,"5min":1.6339999999999999,"15min":1.6339999999999999},"last":1.1919999999999999},{"interface":"front","average":{"1min":1.0329999999999999,"5min":1.02,"15min":1.018},"min":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":1.3129999999999999,"5min":1.585,"15min":1.585},"last":1.079}]},{"osd":6,"last update":"Sun Oct 1 10:39:07 2023","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":1.0840000000000001,"15min":1.095},"min":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":1.276,"5min":1.486,"15min":1.486},"last":1.02},{"interface":"front","average":{"1min":1.1279999999999999,"5min":1.1419999999999999,"15min":1.1439999999999999},"min":{"1min":0.84199999999999997,"5min":0.65200000000000002,"15min":0.65200000000000002},"max":{"1min":1.3799999999999999,"5min":1.6639999999999999,"15min":1.6639999999999999},"last":1.3089999999999999}]},{"osd":7,"last update":"Sun Oct 1 10:39:07 2023","interfaces":[{"interface":"back","average":{"1min":1.1020000000000001,"5min":1.1259999999999999,"15min":1.1299999999999999},"min":{"1min":0.79500000000000004,"5min":0.749,"15min":0.749},"max":{"1min":1.444,"5min":1.536,"15min":1.536},"last":1.399},{"interface":"front","average":{"1min":1.129,"5min":1.206,"15min":1.2190000000000001},"min":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"max":{"1min":1.7390000000000001,"5min":1.7390000000000001,"15min":1.7390000000000001},"last":1.2669999999999999}]}]},{"osd":3,"up_from":121,"seq":519691042846,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":103128,"kb_used_data":1232,"kb_used_omap":2,"kb_used_meta":101885,"kb_avail":93642024,"statfs":{"total":95995035648,"available":95889432576,"internally_reserved":0,"allocated":1261568,"data_stored":776896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":104330293},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:39:47 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.66000000000000003,"15min":0.66900000000000004},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.93000000000000005,"5min":0.96199999999999997,"15min":0.96199999999999997},"last":0.70599999999999996},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.59499999999999997,"15min":0.58699999999999997},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.98399999999999999,"5min":0.98399999999999999,"15min":0.98399999999999999},"last":0.83699999999999997}]},{"osd":1,"last update":"Sun Oct 1 10:39:47 2023","interfaces":[{"interface":"back","average":{"1min":0.65900000000000003,"5min":0.69099999999999995,"15min":0.69699999999999995},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":1.2450000000000001,"5min":1.2450000000000001,"15min":1.2450000000000001},"last":0.79100000000000004},{"interface":"front","average":{"1min":0.73599999999999999,"5min":0.77200000000000002,"15min":0.77800000000000002},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":1.3740000000000001,"5min":1.3740000000000001,"15min":1.3740000000000001},"last":0.96699999999999997}]},{"osd":2,"last update":"Sun Oct 1 10:39:47 2023","interfaces":[{"interface":"back","average":{"1min":0.65700000000000003,"5min":0.67700000000000005,"15min":0.68000000000000005},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":1.258,"5min":1.258,"15min":1.258},"last":1.0700000000000001},{"interface":"front","average":{"1min":0.79500000000000004,"5min":0.78700000000000003,"15min":0.78600000000000003},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":1.256,"5min":1.256,"15min":1.256},"last":0.89800000000000002}]},{"osd":4,"last update":"Sun Oct 1 10:39:15 2023","interfaces":[{"interface":"back","average":{"1min":0.94099999999999995,"5min":0.94099999999999995,"15min":0.94099999999999995},"min":{"1min":0.623,"5min":0.623,"15min":0.623},"max":{"1min":1.306,"5min":1.306,"15min":1.306},"last":1.1160000000000001},{"interface":"front","average":{"1min":0.92800000000000005,"5min":0.92800000000000005,"15min":0.92800000000000005},"min":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"max":{"1min":1.278,"5min":1.278,"15min":1.278},"last":1.214}]},{"osd":5,"last update":"Sun Oct 1 10:39:47 2023","interfaces":[{"interface":"back","average":{"1min":0.95599999999999996,"5min":0.94599999999999995,"15min":0.94399999999999995},"min":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"max":{"1min":1.2909999999999999,"5min":1.2909999999999999,"15min":1.2909999999999999},"last":1.0089999999999999},{"interface":"front","average":{"1min":0.98599999999999999,"5min":1.002,"15min":1.004},"min":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"max":{"1min":1.3100000000000001,"5min":1.4199999999999999,"15min":1.4199999999999999},"last":1.179}]},{"osd":6,"last update":"Sun Oct 1 10:39:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0349999999999999,"5min":1.0149999999999999,"15min":1.012},"min":{"1min":0.60899999999999999,"5min":0.59099999999999997,"15min":0.59099999999999997},"max":{"1min":1.417,"5min":1.417,"15min":1.417},"last":1.1499999999999999},{"interface":"front","average":{"1min":1.0369999999999999,"5min":1.0920000000000001,"15min":1.101},"min":{"1min":0.62,"5min":0.62,"15min":0.62},"max":{"1min":1.351,"5min":1.377,"15min":1.377},"last":1.3460000000000001}]},{"osd":7,"last update":"Sun Oct 1 10:39:47 2023","interfaces":[{"interface":"back","average":{"1min":0.97799999999999998,"5min":1.0429999999999999,"15min":1.054},"min":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.3959999999999999,"5min":1.411,"15min":1.411},"last":1.284},{"interface":"front","average":{"1min":1.056,"5min":1.0249999999999999,"15min":1.02},"min":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"max":{"1min":1.5109999999999999,"5min":1.5109999999999999,"15min":1.5109999999999999},"last":1.2450000000000001}]}]},{"osd":4,"up_from":126,"seq":541165879319,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":138584,"kb_used_data":1232,"kb_used_omap":10,"kb_used_meta":137333,"kb_avail":93606568,"statfs":{"total":95995035648,"available":95853125632,"internally_reserved":0,"allocated":1261568,"data_stored":777243,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":140629604},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:39:16 2023","interfaces":[{"interface":"back","average":{"1min":0.90500000000000003,"5min":0.90500000000000003,"15min":0.90500000000000003},"min":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"max":{"1min":1.3009999999999999,"5min":1.3009999999999999,"15min":1.3009999999999999},"last":1.0109999999999999},{"interface":"front","average":{"1min":0.98999999999999999,"5min":0.98999999999999999,"15min":0.98999999999999999},"min":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"max":{"1min":1.4279999999999999,"5min":1.4279999999999999,"15min":1.4279999999999999},"last":0.81100000000000005}]},{"osd":1,"last update":"Sun Oct 1 10:39:16 2023","interfaces":[{"interface":"back","average":{"1min":0.92700000000000005,"5min":0.92700000000000005,"15min":0.92700000000000005},"min":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"max":{"1min":1.1850000000000001,"5min":1.1850000000000001,"15min":1.1850000000000001},"last":0.93700000000000006},{"interface":"front","average":{"1min":0.95399999999999996,"5min":0.95399999999999996,"15min":0.95399999999999996},"min":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"max":{"1min":1.2430000000000001,"5min":1.2430000000000001,"15min":1.2430000000000001},"last":1.2290000000000001}]},{"osd":2,"last update":"Sun Oct 1 10:39:16 2023","interfaces":[{"interface":"back","average":{"1min":0.95399999999999996,"5min":0.95399999999999996,"15min":0.95399999999999996},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":1.1970000000000001,"5min":1.1970000000000001,"15min":1.1970000000000001},"last":0.85599999999999998},{"interface":"front","average":{"1min":0.97699999999999998,"5min":0.97699999999999998,"15min":0.97699999999999998},"min":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"max":{"1min":1.222,"5min":1.222,"15min":1.222},"last":1.121}]},{"osd":3,"last update":"Sun Oct 1 10:39:16 2023","interfaces":[{"interface":"back","average":{"1min":1.0780000000000001,"5min":1.0780000000000001,"15min":1.0780000000000001},"min":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"max":{"1min":1.3320000000000001,"5min":1.3320000000000001,"15min":1.3320000000000001},"last":0.90400000000000003},{"interface":"front","average":{"1min":0.97899999999999998,"5min":0.97899999999999998,"15min":0.97899999999999998},"min":{"1min":0.628,"5min":0.628,"15min":0.628},"max":{"1min":1.296,"5min":1.296,"15min":1.296},"last":1.077}]},{"osd":5,"last update":"Sun Oct 1 10:39:16 2023","interfaces":[{"interface":"back","average":{"1min":1.095,"5min":1.095,"15min":1.095},"min":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"max":{"1min":1.5089999999999999,"5min":1.5089999999999999,"15min":1.5089999999999999},"last":0.97499999999999998},{"interface":"front","average":{"1min":1.0940000000000001,"5min":1.0940000000000001,"15min":1.0940000000000001},"min":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"max":{"1min":1.3779999999999999,"5min":1.3779999999999999,"15min":1.3779999999999999},"last":1.2929999999999999}]},{"osd":6,"last update":"Sun Oct 1 10:39:16 2023","interfaces":[{"interface":"back","average":{"1min":1.139,"5min":1.139,"15min":1.139},"min":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"max":{"1min":1.579,"5min":1.579,"15min":1.579},"last":1.202},{"interface":"front","average":{"1min":1.075,"5min":1.075,"15min":1.075},"min":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"max":{"1min":1.403,"5min":1.403,"15min":1.403},"last":1.258}]},{"osd":7,"last update":"Sun Oct 1 10:39:16 2023","interfaces":[{"interface":"back","average":{"1min":1.1819999999999999,"5min":1.1819999999999999,"15min":1.1819999999999999},"min":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"max":{"1min":1.387,"5min":1.387,"15min":1.387},"last":1.175},{"interface":"front","average":{"1min":1.169,"5min":1.169,"15min":1.169},"min":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"max":{"1min":1.46,"5min":1.46,"15min":1.46},"last":1.1559999999999999}]}]},{"osd":5,"up_from":36,"seq":154618823000,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":594992,"kb_used_data":1192,"kb_used_omap":0,"kb_used_meta":593792,"kb_avail":93150160,"statfs":{"total":95995035648,"available":95385763840,"internally_reserved":0,"allocated":1220608,"data_stored":768137,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":608043008},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:39:38 2023","interfaces":[{"interface":"back","average":{"1min":0.83899999999999997,"5min":0.98499999999999999,"15min":1.272},"min":{"1min":0.67900000000000005,"5min":0.27300000000000002,"15min":0.26400000000000001},"max":{"1min":1.2709999999999999,"5min":4.8140000000000001,"15min":10.429},"last":0.88500000000000001},{"interface":"front","average":{"1min":0.92500000000000004,"5min":1.306,"15min":1.4930000000000001},"min":{"1min":0.629,"5min":0.29299999999999998,"15min":0.29199999999999998},"max":{"1min":1.1919999999999999,"5min":13.018000000000001,"15min":13.018000000000001},"last":0.77700000000000002}]},{"osd":1,"last update":"Sun Oct 1 10:39:38 2023","interfaces":[{"interface":"back","average":{"1min":0.85299999999999998,"5min":1.3029999999999999,"15min":1.663},"min":{"1min":0.62,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":1.02,"5min":9.0980000000000008,"15min":10.858000000000001},"last":1.075},{"interface":"front","average":{"1min":0.94199999999999995,"5min":1.7370000000000001,"15min":1.76},"min":{"1min":0.68000000000000005,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.337,"5min":12.864000000000001,"15min":12.864000000000001},"last":0.67600000000000005}]},{"osd":2,"last update":"Sun Oct 1 10:39:06 2023","interfaces":[{"interface":"back","average":{"1min":0.88100000000000001,"5min":0.96299999999999997,"15min":0.97699999999999998},"min":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"max":{"1min":1.272,"5min":1.3540000000000001,"15min":1.3540000000000001},"last":1.0029999999999999},{"interface":"front","average":{"1min":0.97399999999999998,"5min":0.96599999999999997,"15min":0.96399999999999997},"min":{"1min":0.751,"5min":0.67600000000000005,"15min":0.67600000000000005},"max":{"1min":1.1879999999999999,"5min":1.272,"15min":1.272},"last":0.86599999999999999}]},{"osd":3,"last update":"Sun Oct 1 10:39:50 2023","interfaces":[{"interface":"back","average":{"1min":0.98799999999999999,"5min":0.94399999999999995,"15min":0.93600000000000005},"min":{"1min":0.67200000000000004,"5min":0.64900000000000002,"15min":0.64900000000000002},"max":{"1min":1.143,"5min":1.3919999999999999,"15min":1.3919999999999999},"last":0.93300000000000005},{"interface":"front","average":{"1min":0.99099999999999999,"5min":0.94999999999999996,"15min":0.94299999999999995},"min":{"1min":0.76300000000000001,"5min":0.57699999999999996,"15min":0.57699999999999996},"max":{"1min":1.222,"5min":1.222,"15min":1.222},"last":1.0489999999999999}]},{"osd":4,"last update":"Sun Oct 1 10:39:16 2023","interfaces":[{"interface":"back","average":{"1min":1.077,"5min":1.077,"15min":1.077},"min":{"1min":0.753,"5min":0.753,"15min":0.753},"max":{"1min":1.343,"5min":1.343,"15min":1.343},"last":1.1439999999999999},{"interface":"front","average":{"1min":1.0840000000000001,"5min":1.0840000000000001,"15min":1.0840000000000001},"min":{"1min":0.89700000000000002,"5min":0.89700000000000002,"15min":0.89700000000000002},"max":{"1min":1.2989999999999999,"5min":1.2989999999999999,"15min":1.2989999999999999},"last":1.208}]},{"osd":6,"last update":"Sun Oct 1 10:39:52 2023","interfaces":[{"interface":"back","average":{"1min":1.1259999999999999,"5min":1.177,"15min":1.288},"min":{"1min":0.90600000000000003,"5min":0.38500000000000001,"15min":0.33900000000000002},"max":{"1min":1.6020000000000001,"5min":6.96,"15min":6.96},"last":1.1259999999999999},{"interface":"front","average":{"1min":1.083,"5min":1.3100000000000001,"15min":1.2190000000000001},"min":{"1min":0.78600000000000003,"5min":0.312,"15min":0.312},"max":{"1min":1.2769999999999999,"5min":8.4339999999999993,"15min":8.4339999999999993},"last":0.94499999999999995}]},{"osd":7,"last update":"Sun Oct 1 10:39:52 2023","interfaces":[{"interface":"back","average":{"1min":1.1799999999999999,"5min":1.3680000000000001,"15min":1.2370000000000001},"min":{"1min":0.88800000000000001,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.782,"5min":9.4629999999999992,"15min":9.4629999999999992},"last":1.105},{"interface":"front","average":{"1min":1.2090000000000001,"5min":1.1559999999999999,"15min":1.3779999999999999},"min":{"1min":0.86899999999999999,"5min":0.372,"15min":0.372},"max":{"1min":1.649,"5min":5.4699999999999998,"15min":12.927},"last":1.1699999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T10:40:05.326 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-10-01T10:40:05.326 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd dump 2023-10-01T10:40:05.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:05 smithi078 bash[78274]: cluster 2023-10-01T10:40:03.690214+0000 mgr.y (mgr.25072) 667 : cluster 0 pgmap v572: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:05.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:05 smithi078 bash[78274]: audit 2023-10-01T10:40:04.491706+0000 mgr.y (mgr.25072) 668 : audit 0 from='client.35586 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:40:05.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:05 smithi078 bash[75299]: cluster 2023-10-01T10:40:03.690214+0000 mgr.y (mgr.25072) 667 : cluster 0 pgmap v572: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:05.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:05 smithi078 bash[75299]: audit 2023-10-01T10:40:04.491706+0000 mgr.y (mgr.25072) 668 : audit 0 from='client.35586 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:40:05.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:05 smithi173 bash[58816]: cluster 2023-10-01T10:40:03.690214+0000 mgr.y (mgr.25072) 667 : cluster 0 pgmap v572: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:05.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:05 smithi173 bash[58816]: audit 2023-10-01T10:40:04.491706+0000 mgr.y (mgr.25072) 668 : audit 0 from='client.35586 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:40:07.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:07 smithi078 bash[75299]: cluster 2023-10-01T10:40:05.691169+0000 mgr.y (mgr.25072) 669 : cluster 0 pgmap v573: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:07.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:07 smithi078 bash[78274]: cluster 2023-10-01T10:40:05.691169+0000 mgr.y (mgr.25072) 669 : cluster 0 pgmap v573: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:07.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:07 smithi173 bash[58816]: cluster 2023-10-01T10:40:05.691169+0000 mgr.y (mgr.25072) 669 : cluster 0 pgmap v573: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:09.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:09 smithi173 bash[58816]: cluster 2023-10-01T10:40:07.691953+0000 mgr.y (mgr.25072) 670 : cluster 0 pgmap v574: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:09.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:09 smithi078 bash[75299]: cluster 2023-10-01T10:40:07.691953+0000 mgr.y (mgr.25072) 670 : cluster 0 pgmap v574: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:09.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:09 smithi078 bash[78274]: cluster 2023-10-01T10:40:07.691953+0000 mgr.y (mgr.25072) 670 : cluster 0 pgmap v574: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:10.114 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:40:10.582 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:40:10 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:40:10] "GET /metrics HTTP/1.1" 200 33824 "" "Prometheus/2.33.4" 2023-10-01T10:40:11.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:11 smithi173 bash[58816]: cluster 2023-10-01T10:40:09.692894+0000 mgr.y (mgr.25072) 671 : cluster 0 pgmap v575: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:11.389 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:11 smithi078 bash[78274]: cluster 2023-10-01T10:40:09.692894+0000 mgr.y (mgr.25072) 671 : cluster 0 pgmap v575: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:11.389 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:11 smithi078 bash[75299]: cluster 2023-10-01T10:40:09.692894+0000 mgr.y (mgr.25072) 671 : cluster 0 pgmap v575: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:13.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:13 smithi078 bash[78274]: cluster 2023-10-01T10:40:11.693768+0000 mgr.y (mgr.25072) 672 : cluster 0 pgmap v576: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:13.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:13 smithi078 bash[75299]: cluster 2023-10-01T10:40:11.693768+0000 mgr.y (mgr.25072) 672 : cluster 0 pgmap v576: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:13.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:13 smithi173 bash[58816]: cluster 2023-10-01T10:40:11.693768+0000 mgr.y (mgr.25072) 672 : cluster 0 pgmap v576: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:13.481 INFO:teuthology.orchestra.run.smithi078.stdout:epoch 127 2023-10-01T10:40:13.481 INFO:teuthology.orchestra.run.smithi078.stdout:fsid d8016480-6041-11ee-8db5-212e2dc638e7 2023-10-01T10:40:13.481 INFO:teuthology.orchestra.run.smithi078.stdout:created 2023-10-01T10:04:59.345633+0000 2023-10-01T10:40:13.481 INFO:teuthology.orchestra.run.smithi078.stdout:modified 2023-10-01T10:38:16.004203+0000 2023-10-01T10:40:13.481 INFO:teuthology.orchestra.run.smithi078.stdout:flags sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit 2023-10-01T10:40:13.481 INFO:teuthology.orchestra.run.smithi078.stdout:crush_version 23 2023-10-01T10:40:13.481 INFO:teuthology.orchestra.run.smithi078.stdout:full_ratio 0.95 2023-10-01T10:40:13.481 INFO:teuthology.orchestra.run.smithi078.stdout:backfillfull_ratio 0.9 2023-10-01T10:40:13.482 INFO:teuthology.orchestra.run.smithi078.stdout:nearfull_ratio 0.85 2023-10-01T10:40:13.482 INFO:teuthology.orchestra.run.smithi078.stdout:require_min_compat_client luminous 2023-10-01T10:40:13.482 INFO:teuthology.orchestra.run.smithi078.stdout:min_compat_client luminous 2023-10-01T10:40:13.482 INFO:teuthology.orchestra.run.smithi078.stdout:require_osd_release pacific 2023-10-01T10:40:13.482 INFO:teuthology.orchestra.run.smithi078.stdout:stretch_mode_enabled false 2023-10-01T10:40:13.482 INFO:teuthology.orchestra.run.smithi078.stdout:pool 1 '.mgr' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode off last_change 70 flags hashpspool stripe_width 0 pg_num_max 32 pg_num_min 1 application mgr,mgr_devicehealth read_balance_score 7.89 2023-10-01T10:40:13.482 INFO:teuthology.orchestra.run.smithi078.stdout:pool 2 'rbd' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 8 pgp_num 8 autoscale_mode off last_change 55 flags hashpspool stripe_width 0 application rbd read_balance_score 2.00 2023-10-01T10:40:13.482 INFO:teuthology.orchestra.run.smithi078.stdout:pool 3 'cephfs.foo.meta' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 59 flags hashpspool stripe_width 0 pg_autoscale_bias 4 pg_num_min 16 recovery_priority 5 application cephfs read_balance_score 1.50 2023-10-01T10:40:13.482 INFO:teuthology.orchestra.run.smithi078.stdout:pool 4 'cephfs.foo.data' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 60 flags hashpspool,bulk stripe_width 0 application cephfs read_balance_score 2.25 2023-10-01T10:40:13.482 INFO:teuthology.orchestra.run.smithi078.stdout:max_osd 8 2023-10-01T10:40:13.482 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 up in weight 1 up_from 86 up_thru 126 down_at 83 last_clean_interval [8,82) [v2:172.21.15.78:6802/2878175461,v1:172.21.15.78:6803/2878175461] [v2:172.21.15.78:6804/2878175461,v1:172.21.15.78:6805/2878175461] exists,up 3cdf50f0-cae3-4e38-a7ca-c24c4a07b615 2023-10-01T10:40:13.483 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 up in weight 1 up_from 101 up_thru 126 down_at 97 last_clean_interval [13,96) [v2:172.21.15.78:6810/1115426690,v1:172.21.15.78:6811/1115426690] [v2:172.21.15.78:6812/1115426690,v1:172.21.15.78:6813/1115426690] exists,up 4e6bea65-4109-43d2-b5be-3811fdd2a803 2023-10-01T10:40:13.483 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 up in weight 1 up_from 116 up_thru 126 down_at 113 last_clean_interval [18,112) [v2:172.21.15.78:6818/4228246677,v1:172.21.15.78:6819/4228246677] [v2:172.21.15.78:6820/4228246677,v1:172.21.15.78:6821/4228246677] exists,up 9f830594-31d6-4994-85fa-0615c280d5c3 2023-10-01T10:40:13.483 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 up in weight 1 up_from 121 up_thru 126 down_at 118 last_clean_interval [25,117) [v2:172.21.15.78:6826/2365141372,v1:172.21.15.78:6827/2365141372] [v2:172.21.15.78:6828/2365141372,v1:172.21.15.78:6829/2365141372] exists,up 2780842b-2f4f-4190-b4a5-279a0444adf2 2023-10-01T10:40:13.483 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 up in weight 1 up_from 126 up_thru 126 down_at 123 last_clean_interval [30,122) [v2:172.21.15.173:6800/3220463704,v1:172.21.15.173:6801/3220463704] [v2:172.21.15.173:6802/3220463704,v1:172.21.15.173:6803/3220463704] exists,up 57c30778-fd7e-4f3e-afd9-389ef377cb62 2023-10-01T10:40:13.483 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 up in weight 1 up_from 36 up_thru 126 down_at 0 last_clean_interval [0,0) [v2:172.21.15.173:6808/2798643181,v1:172.21.15.173:6809/2798643181] [v2:172.21.15.173:6810/2798643181,v1:172.21.15.173:6811/2798643181] exists,up c62aa7c3-2d09-4088-ba36-c84588dc3549 2023-10-01T10:40:13.483 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 up in weight 1 up_from 42 up_thru 126 down_at 0 last_clean_interval [0,0) [v2:172.21.15.173:6816/1568190351,v1:172.21.15.173:6817/1568190351] [v2:172.21.15.173:6818/1568190351,v1:172.21.15.173:6819/1568190351] exists,up b140330d-dd93-4ce4-9b9a-2ec6321419b3 2023-10-01T10:40:13.483 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 up in weight 1 up_from 48 up_thru 126 down_at 0 last_clean_interval [0,0) [v2:172.21.15.173:6824/1958390745,v1:172.21.15.173:6825/1958390745] [v2:172.21.15.173:6826/1958390745,v1:172.21.15.173:6827/1958390745] exists,up 7590027f-50c4-4457-843d-cebaa734d2ea 2023-10-01T10:40:13.483 INFO:teuthology.orchestra.run.smithi078.stdout:pg_upmap_items 4.d [1,5] 2023-10-01T10:40:13.483 INFO:teuthology.orchestra.run.smithi078.stdout:pg_upmap_items 4.f [1,2] 2023-10-01T10:40:13.483 INFO:teuthology.orchestra.run.smithi078.stdout:pg_upmap_items 4.14 [1,2] 2023-10-01T10:40:13.483 INFO:teuthology.orchestra.run.smithi078.stdout:blocklist 172.21.15.78:0/590730483 expires 2023-10-02T10:22:47.233952+0000 2023-10-01T10:40:13.484 INFO:teuthology.orchestra.run.smithi078.stdout:blocklist 172.21.15.78:0/3335202251 expires 2023-10-02T10:22:47.233952+0000 2023-10-01T10:40:13.484 INFO:teuthology.orchestra.run.smithi078.stdout:blocklist 172.21.15.78:0/1585217033 expires 2023-10-02T10:22:47.233952+0000 2023-10-01T10:40:13.484 INFO:teuthology.orchestra.run.smithi078.stdout:blocklist 172.21.15.78:6801/449971457 expires 2023-10-02T10:22:47.233952+0000 2023-10-01T10:40:13.484 INFO:teuthology.orchestra.run.smithi078.stdout:blocklist 172.21.15.78:0/3939471097 expires 2023-10-02T10:05:57.727322+0000 2023-10-01T10:40:13.484 INFO:teuthology.orchestra.run.smithi078.stdout:blocklist 172.21.15.78:6800/1700562157 expires 2023-10-02T10:05:57.727322+0000 2023-10-01T10:40:13.484 INFO:teuthology.orchestra.run.smithi078.stdout:blocklist 172.21.15.78:6801/1700562157 expires 2023-10-02T10:05:57.727322+0000 2023-10-01T10:40:13.484 INFO:teuthology.orchestra.run.smithi078.stdout:blocklist 172.21.15.78:0/3635389419 expires 2023-10-02T10:05:57.727322+0000 2023-10-01T10:40:13.484 INFO:teuthology.orchestra.run.smithi078.stdout:blocklist 172.21.15.78:0/1297753620 expires 2023-10-02T10:22:47.233952+0000 2023-10-01T10:40:13.484 INFO:teuthology.orchestra.run.smithi078.stdout:blocklist 172.21.15.78:0/158328292 expires 2023-10-02T10:05:27.007550+0000 2023-10-01T10:40:13.484 INFO:teuthology.orchestra.run.smithi078.stdout:blocklist 172.21.15.78:0/308254153 expires 2023-10-02T10:05:27.007550+0000 2023-10-01T10:40:13.484 INFO:teuthology.orchestra.run.smithi078.stdout:blocklist 172.21.15.173:0/2382630687 expires 2023-10-02T10:21:31.381928+0000 2023-10-01T10:40:13.484 INFO:teuthology.orchestra.run.smithi078.stdout:blocklist 172.21.15.78:0/3274439700 expires 2023-10-02T10:19:23.134044+0000 2023-10-01T10:40:13.485 INFO:teuthology.orchestra.run.smithi078.stdout:blocklist 172.21.15.78:0/2751026363 expires 2023-10-02T10:22:47.233952+0000 2023-10-01T10:40:13.485 INFO:teuthology.orchestra.run.smithi078.stdout:blocklist 172.21.15.78:0/702157702 expires 2023-10-02T10:19:23.134044+0000 2023-10-01T10:40:13.485 INFO:teuthology.orchestra.run.smithi078.stdout:blocklist 172.21.15.78:0/3655343641 expires 2023-10-02T10:13:17.135564+0000 2023-10-01T10:40:13.485 INFO:teuthology.orchestra.run.smithi078.stdout:blocklist 172.21.15.78:6800/3027286102 expires 2023-10-02T10:05:27.007550+0000 2023-10-01T10:40:13.485 INFO:teuthology.orchestra.run.smithi078.stdout:blocklist 172.21.15.78:6801/516444166 expires 2023-10-02T10:19:23.134044+0000 2023-10-01T10:40:13.485 INFO:teuthology.orchestra.run.smithi078.stdout:blocklist 172.21.15.78:0/2359078223 expires 2023-10-02T10:13:17.135564+0000 2023-10-01T10:40:13.485 INFO:teuthology.orchestra.run.smithi078.stdout:blocklist 172.21.15.78:6801/3027286102 expires 2023-10-02T10:05:27.007550+0000 2023-10-01T10:40:13.485 INFO:teuthology.orchestra.run.smithi078.stdout:blocklist 172.21.15.173:0/4047131805 expires 2023-10-02T10:21:31.381928+0000 2023-10-01T10:40:13.485 INFO:teuthology.orchestra.run.smithi078.stdout:blocklist 172.21.15.78:0/3611942206 expires 2023-10-02T10:22:47.233952+0000 2023-10-01T10:40:13.485 INFO:teuthology.orchestra.run.smithi078.stdout:blocklist 172.21.15.173:0/1462478964 expires 2023-10-02T10:21:31.381928+0000 2023-10-01T10:40:13.485 INFO:teuthology.orchestra.run.smithi078.stdout:blocklist 172.21.15.78:6800/449971457 expires 2023-10-02T10:22:47.233952+0000 2023-10-01T10:40:13.486 INFO:teuthology.orchestra.run.smithi078.stdout:blocklist 172.21.15.78:6800/1224621968 expires 2023-10-02T10:13:17.135564+0000 2023-10-01T10:40:13.486 INFO:teuthology.orchestra.run.smithi078.stdout:blocklist 172.21.15.78:6801/1224621968 expires 2023-10-02T10:13:17.135564+0000 2023-10-01T10:40:13.486 INFO:teuthology.orchestra.run.smithi078.stdout:blocklist 172.21.15.173:6834/1928273846 expires 2023-10-02T10:21:31.381928+0000 2023-10-01T10:40:13.486 INFO:teuthology.orchestra.run.smithi078.stdout:blocklist 172.21.15.78:6800/516444166 expires 2023-10-02T10:19:23.134044+0000 2023-10-01T10:40:13.486 INFO:teuthology.orchestra.run.smithi078.stdout:blocklist 172.21.15.173:0/4059345053 expires 2023-10-02T10:21:31.381928+0000 2023-10-01T10:40:13.486 INFO:teuthology.orchestra.run.smithi078.stdout:blocklist 172.21.15.173:6835/1928273846 expires 2023-10-02T10:21:31.381928+0000 2023-10-01T10:40:14.308 INFO:tasks.cephadm.ceph_manager.ceph:['osd.0 up in weight 1 up_from 86 up_thru 126 down_at 83 last_clean_interval [8,82) [v2:172.21.15.78:6802/2878175461,v1:172.21.15.78:6803/2878175461] [v2:172.21.15.78:6804/2878175461,v1:172.21.15.78:6805/2878175461] exists,up 3cdf50f0-cae3-4e38-a7ca-c24c4a07b615', 'osd.1 up in weight 1 up_from 101 up_thru 126 down_at 97 last_clean_interval [13,96) [v2:172.21.15.78:6810/1115426690,v1:172.21.15.78:6811/1115426690] [v2:172.21.15.78:6812/1115426690,v1:172.21.15.78:6813/1115426690] exists,up 4e6bea65-4109-43d2-b5be-3811fdd2a803', 'osd.2 up in weight 1 up_from 116 up_thru 126 down_at 113 last_clean_interval [18,112) [v2:172.21.15.78:6818/4228246677,v1:172.21.15.78:6819/4228246677] [v2:172.21.15.78:6820/4228246677,v1:172.21.15.78:6821/4228246677] exists,up 9f830594-31d6-4994-85fa-0615c280d5c3', 'osd.3 up in weight 1 up_from 121 up_thru 126 down_at 118 last_clean_interval [25,117) [v2:172.21.15.78:6826/2365141372,v1:172.21.15.78:6827/2365141372] [v2:172.21.15.78:6828/2365141372,v1:172.21.15.78:6829/2365141372] exists,up 2780842b-2f4f-4190-b4a5-279a0444adf2', 'osd.4 up in weight 1 up_from 126 up_thru 126 down_at 123 last_clean_interval [30,122) [v2:172.21.15.173:6800/3220463704,v1:172.21.15.173:6801/3220463704] [v2:172.21.15.173:6802/3220463704,v1:172.21.15.173:6803/3220463704] exists,up 57c30778-fd7e-4f3e-afd9-389ef377cb62', 'osd.5 up in weight 1 up_from 36 up_thru 126 down_at 0 last_clean_interval [0,0) [v2:172.21.15.173:6808/2798643181,v1:172.21.15.173:6809/2798643181] [v2:172.21.15.173:6810/2798643181,v1:172.21.15.173:6811/2798643181] exists,up c62aa7c3-2d09-4088-ba36-c84588dc3549', 'osd.6 up in weight 1 up_from 42 up_thru 126 down_at 0 last_clean_interval [0,0) [v2:172.21.15.173:6816/1568190351,v1:172.21.15.173:6817/1568190351] [v2:172.21.15.173:6818/1568190351,v1:172.21.15.173:6819/1568190351] exists,up b140330d-dd93-4ce4-9b9a-2ec6321419b3', 'osd.7 up in weight 1 up_from 48 up_thru 126 down_at 0 last_clean_interval [0,0) [v2:172.21.15.173:6824/1958390745,v1:172.21.15.173:6825/1958390745] [v2:172.21.15.173:6826/1958390745,v1:172.21.15.173:6827/1958390745] exists,up 7590027f-50c4-4457-843d-cebaa734d2ea'] 2023-10-01T10:40:14.309 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon mon.a config show 2023-10-01T10:40:14.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:14 smithi173 bash[58816]: audit 2023-10-01T10:40:13.478401+0000 mon.a (mon.0) 901 : audit 0 from='client.? 172.21.15.78:0/2145659530' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-01T10:40:14.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:14 smithi078 bash[75299]: audit 2023-10-01T10:40:13.478401+0000 mon.a (mon.0) 901 : audit 0 from='client.? 172.21.15.78:0/2145659530' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-01T10:40:14.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:14 smithi078 bash[78274]: audit 2023-10-01T10:40:13.478401+0000 mon.a (mon.0) 901 : audit 0 from='client.? 172.21.15.78:0/2145659530' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-01T10:40:15.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:15 smithi173 bash[58816]: cluster 2023-10-01T10:40:13.694260+0000 mgr.y (mgr.25072) 673 : cluster 0 pgmap v577: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:15.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:15 smithi078 bash[78274]: cluster 2023-10-01T10:40:13.694260+0000 mgr.y (mgr.25072) 673 : cluster 0 pgmap v577: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:15.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:15 smithi078 bash[75299]: cluster 2023-10-01T10:40:13.694260+0000 mgr.y (mgr.25072) 673 : cluster 0 pgmap v577: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:17.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:17 smithi173 bash[58816]: cluster 2023-10-01T10:40:15.695114+0000 mgr.y (mgr.25072) 674 : cluster 0 pgmap v578: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:17.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:17 smithi078 bash[75299]: cluster 2023-10-01T10:40:15.695114+0000 mgr.y (mgr.25072) 674 : cluster 0 pgmap v578: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:17.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:17 smithi078 bash[78274]: cluster 2023-10-01T10:40:15.695114+0000 mgr.y (mgr.25072) 674 : cluster 0 pgmap v578: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:18.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:18 smithi173 bash[58816]: audit 2023-10-01T10:40:17.411027+0000 mon.a (mon.0) 902 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:40:18.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:18 smithi078 bash[75299]: audit 2023-10-01T10:40:17.411027+0000 mon.a (mon.0) 902 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:40:18.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:18 smithi078 bash[78274]: audit 2023-10-01T10:40:17.411027+0000 mon.a (mon.0) 902 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:40:19.084 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:40:19.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:19 smithi173 bash[58816]: cluster 2023-10-01T10:40:17.695899+0000 mgr.y (mgr.25072) 675 : cluster 0 pgmap v579: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:19.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:19 smithi078 bash[78274]: cluster 2023-10-01T10:40:17.695899+0000 mgr.y (mgr.25072) 675 : cluster 0 pgmap v579: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:19.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:19 smithi078 bash[75299]: cluster 2023-10-01T10:40:17.695899+0000 mgr.y (mgr.25072) 675 : cluster 0 pgmap v579: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:20.357 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:40:20 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:40:20] "GET /metrics HTTP/1.1" 200 33818 "" "Prometheus/2.33.4" 2023-10-01T10:40:21.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:21 smithi078 bash[75299]: cluster 2023-10-01T10:40:19.696817+0000 mgr.y (mgr.25072) 676 : cluster 0 pgmap v580: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:21.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:21 smithi078 bash[78274]: cluster 2023-10-01T10:40:19.696817+0000 mgr.y (mgr.25072) 676 : cluster 0 pgmap v580: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:21.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:21 smithi173 bash[58816]: cluster 2023-10-01T10:40:19.696817+0000 mgr.y (mgr.25072) 676 : cluster 0 pgmap v580: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:22.211 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2023-10-01T10:40:22.211 INFO:teuthology.orchestra.run.smithi078.stdout: "name": "mon.a", 2023-10-01T10:40:22.211 INFO:teuthology.orchestra.run.smithi078.stdout: "cluster": "ceph", 2023-10-01T10:40:22.211 INFO:teuthology.orchestra.run.smithi078.stdout: "admin_socket": "/var/run/ceph/ceph-mon.a.asok", 2023-10-01T10:40:22.211 INFO:teuthology.orchestra.run.smithi078.stdout: "admin_socket_mode": "", 2023-10-01T10:40:22.211 INFO:teuthology.orchestra.run.smithi078.stdout: "auth_allow_insecure_global_id_reclaim": "false", 2023-10-01T10:40:22.211 INFO:teuthology.orchestra.run.smithi078.stdout: "auth_client_required": "cephx, none", 2023-10-01T10:40:22.211 INFO:teuthology.orchestra.run.smithi078.stdout: "auth_cluster_required": "cephx", 2023-10-01T10:40:22.212 INFO:teuthology.orchestra.run.smithi078.stdout: "auth_debug": "true", 2023-10-01T10:40:22.212 INFO:teuthology.orchestra.run.smithi078.stdout: "auth_expose_insecure_global_id_reclaim": "true", 2023-10-01T10:40:22.212 INFO:teuthology.orchestra.run.smithi078.stdout: "auth_mon_ticket_ttl": "660.000000", 2023-10-01T10:40:22.212 INFO:teuthology.orchestra.run.smithi078.stdout: "auth_service_required": "cephx", 2023-10-01T10:40:22.212 INFO:teuthology.orchestra.run.smithi078.stdout: "auth_service_ticket_ttl": "240.000000", 2023-10-01T10:40:22.212 INFO:teuthology.orchestra.run.smithi078.stdout: "auth_supported": "", 2023-10-01T10:40:22.212 INFO:teuthology.orchestra.run.smithi078.stdout: "bdev_aio": "true", 2023-10-01T10:40:22.212 INFO:teuthology.orchestra.run.smithi078.stdout: "bdev_aio_max_queue_depth": "1024", 2023-10-01T10:40:22.212 INFO:teuthology.orchestra.run.smithi078.stdout: "bdev_aio_poll_ms": "250", 2023-10-01T10:40:22.212 INFO:teuthology.orchestra.run.smithi078.stdout: "bdev_aio_reap_max": "16", 2023-10-01T10:40:22.212 INFO:teuthology.orchestra.run.smithi078.stdout: "bdev_async_discard": "false", 2023-10-01T10:40:22.212 INFO:teuthology.orchestra.run.smithi078.stdout: "bdev_block_size": "4096", 2023-10-01T10:40:22.212 INFO:teuthology.orchestra.run.smithi078.stdout: "bdev_debug_aio": "false", 2023-10-01T10:40:22.213 INFO:teuthology.orchestra.run.smithi078.stdout: "bdev_debug_aio_log_age": "5.000000", 2023-10-01T10:40:22.213 INFO:teuthology.orchestra.run.smithi078.stdout: "bdev_debug_aio_suicide_timeout": "60.000000", 2023-10-01T10:40:22.213 INFO:teuthology.orchestra.run.smithi078.stdout: "bdev_debug_inflight_ios": "false", 2023-10-01T10:40:22.213 INFO:teuthology.orchestra.run.smithi078.stdout: "bdev_enable_discard": "false", 2023-10-01T10:40:22.213 INFO:teuthology.orchestra.run.smithi078.stdout: "bdev_flock_retry": "3", 2023-10-01T10:40:22.213 INFO:teuthology.orchestra.run.smithi078.stdout: "bdev_flock_retry_interval": "0.100000", 2023-10-01T10:40:22.213 INFO:teuthology.orchestra.run.smithi078.stdout: "bdev_inject_crash": "0", 2023-10-01T10:40:22.213 INFO:teuthology.orchestra.run.smithi078.stdout: "bdev_inject_crash_flush_delay": "2", 2023-10-01T10:40:22.213 INFO:teuthology.orchestra.run.smithi078.stdout: "bdev_ioring": "false", 2023-10-01T10:40:22.213 INFO:teuthology.orchestra.run.smithi078.stdout: "bdev_ioring_hipri": "false", 2023-10-01T10:40:22.213 INFO:teuthology.orchestra.run.smithi078.stdout: "bdev_ioring_sqthread_poll": "false", 2023-10-01T10:40:22.213 INFO:teuthology.orchestra.run.smithi078.stdout: "bdev_nvme_unbind_from_kernel": "false", 2023-10-01T10:40:22.214 INFO:teuthology.orchestra.run.smithi078.stdout: "bdev_read_buffer_alignment": "4096", 2023-10-01T10:40:22.214 INFO:teuthology.orchestra.run.smithi078.stdout: "bdev_read_preallocated_huge_buffers": "", 2023-10-01T10:40:22.214 INFO:teuthology.orchestra.run.smithi078.stdout: "bdev_type": "", 2023-10-01T10:40:22.214 INFO:teuthology.orchestra.run.smithi078.stdout: "bluefs_alloc_size": "1048576", 2023-10-01T10:40:22.214 INFO:teuthology.orchestra.run.smithi078.stdout: "bluefs_allocator": "hybrid", 2023-10-01T10:40:22.214 INFO:teuthology.orchestra.run.smithi078.stdout: "bluefs_buffered_io": "true", 2023-10-01T10:40:22.214 INFO:teuthology.orchestra.run.smithi078.stdout: "bluefs_check_for_zeros": "false", 2023-10-01T10:40:22.214 INFO:teuthology.orchestra.run.smithi078.stdout: "bluefs_check_volume_selector_often": "false", 2023-10-01T10:40:22.214 INFO:teuthology.orchestra.run.smithi078.stdout: "bluefs_check_volume_selector_on_umount": "false", 2023-10-01T10:40:22.214 INFO:teuthology.orchestra.run.smithi078.stdout: "bluefs_compact_log_sync": "false", 2023-10-01T10:40:22.214 INFO:teuthology.orchestra.run.smithi078.stdout: "bluefs_failed_shared_alloc_cooldown": "600.000000", 2023-10-01T10:40:22.215 INFO:teuthology.orchestra.run.smithi078.stdout: "bluefs_log_compact_min_ratio": "5.000000", 2023-10-01T10:40:22.215 INFO:teuthology.orchestra.run.smithi078.stdout: "bluefs_log_compact_min_size": "16777216", 2023-10-01T10:40:22.215 INFO:teuthology.orchestra.run.smithi078.stdout: "bluefs_log_replay_check_allocations": "true", 2023-10-01T10:40:22.215 INFO:teuthology.orchestra.run.smithi078.stdout: "bluefs_max_log_runway": "4194304", 2023-10-01T10:40:22.215 INFO:teuthology.orchestra.run.smithi078.stdout: "bluefs_max_prefetch": "1048576", 2023-10-01T10:40:22.215 INFO:teuthology.orchestra.run.smithi078.stdout: "bluefs_min_flush_size": "524288", 2023-10-01T10:40:22.215 INFO:teuthology.orchestra.run.smithi078.stdout: "bluefs_min_log_runway": "1048576", 2023-10-01T10:40:22.215 INFO:teuthology.orchestra.run.smithi078.stdout: "bluefs_replay_recovery": "false", 2023-10-01T10:40:22.215 INFO:teuthology.orchestra.run.smithi078.stdout: "bluefs_replay_recovery_disable_compact": "false", 2023-10-01T10:40:22.215 INFO:teuthology.orchestra.run.smithi078.stdout: "bluefs_shared_alloc_size": "65536", 2023-10-01T10:40:22.215 INFO:teuthology.orchestra.run.smithi078.stdout: "bluefs_sync_write": "false", 2023-10-01T10:40:22.215 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_2q_cache_kin_ratio": "0.500000", 2023-10-01T10:40:22.216 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_2q_cache_kout_ratio": "0.500000", 2023-10-01T10:40:22.216 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_alloc_stats_dump_interval": "86400.000000", 2023-10-01T10:40:22.216 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_allocation_from_file": "true", 2023-10-01T10:40:22.216 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_allocator": "hybrid", 2023-10-01T10:40:22.216 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_avl_alloc_bf_free_pct": "4", 2023-10-01T10:40:22.216 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_avl_alloc_bf_threshold": "131072", 2023-10-01T10:40:22.216 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_avl_alloc_ff_max_search_bytes": "16777216", 2023-10-01T10:40:22.216 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_avl_alloc_ff_max_search_count": "100", 2023-10-01T10:40:22.216 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_bitmapallocator_blocks_per_zone": "1024", 2023-10-01T10:40:22.216 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_bitmapallocator_span_size": "1024", 2023-10-01T10:40:22.216 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_blobid_prealloc": "10240", 2023-10-01T10:40:22.216 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_block_create": "true", 2023-10-01T10:40:22.217 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_block_db_create": "false", 2023-10-01T10:40:22.217 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_block_db_path": "", 2023-10-01T10:40:22.217 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_block_db_size": "0", 2023-10-01T10:40:22.217 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_block_path": "", 2023-10-01T10:40:22.217 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_block_preallocate_file": "false", 2023-10-01T10:40:22.217 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_block_size": "107374182400", 2023-10-01T10:40:22.217 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_block_wal_create": "false", 2023-10-01T10:40:22.217 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_block_wal_path": "", 2023-10-01T10:40:22.217 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_block_wal_size": "100663296", 2023-10-01T10:40:22.217 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_bluefs": "true", 2023-10-01T10:40:22.217 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_bluefs_alloc_failure_dump_interval": "0.000000", 2023-10-01T10:40:22.217 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_bluefs_env_mirror": "false", 2023-10-01T10:40:22.217 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_bluefs_max_free": "10737418240", 2023-10-01T10:40:22.218 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_cache_age_bin_interval": "1.000000", 2023-10-01T10:40:22.218 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_cache_age_bins_data": "1 2 6 24 120 720 0 0 0 0", 2023-10-01T10:40:22.218 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_cache_age_bins_kv": "1 2 6 24 120 720 0 0 0 0", 2023-10-01T10:40:22.218 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_cache_age_bins_kv_onode": "0 0 0 0 0 0 0 0 0 720", 2023-10-01T10:40:22.218 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_cache_age_bins_meta": "1 2 6 24 120 720 0 0 0 0", 2023-10-01T10:40:22.218 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_cache_autotune": "true", 2023-10-01T10:40:22.218 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_cache_autotune_interval": "5.000000", 2023-10-01T10:40:22.218 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_cache_kv_onode_ratio": "0.040000", 2023-10-01T10:40:22.218 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_cache_kv_ratio": "0.450000", 2023-10-01T10:40:22.218 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_cache_meta_ratio": "0.450000", 2023-10-01T10:40:22.218 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_cache_size": "0", 2023-10-01T10:40:22.219 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_cache_size_hdd": "1073741824", 2023-10-01T10:40:22.219 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_cache_size_ssd": "3221225472", 2023-10-01T10:40:22.219 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_cache_trim_interval": "0.050000", 2023-10-01T10:40:22.219 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_cache_trim_max_skip_pinned": "1000", 2023-10-01T10:40:22.219 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_cache_type": "2q", 2023-10-01T10:40:22.219 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_cleaner_sleep_interval": "5.000000", 2023-10-01T10:40:22.219 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_clone_cow": "true", 2023-10-01T10:40:22.219 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_compression_algorithm": "snappy", 2023-10-01T10:40:22.219 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_compression_max_blob_size": "0", 2023-10-01T10:40:22.219 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_compression_max_blob_size_hdd": "65536", 2023-10-01T10:40:22.219 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_compression_max_blob_size_ssd": "65536", 2023-10-01T10:40:22.219 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_compression_min_blob_size": "0", 2023-10-01T10:40:22.220 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_compression_min_blob_size_hdd": "8192", 2023-10-01T10:40:22.220 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_compression_min_blob_size_ssd": "65536", 2023-10-01T10:40:22.220 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_compression_mode": "none", 2023-10-01T10:40:22.220 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_compression_required_ratio": "0.875000", 2023-10-01T10:40:22.220 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_csum_type": "crc32c", 2023-10-01T10:40:22.220 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_debug_enforce_settings": "default", 2023-10-01T10:40:22.220 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_debug_freelist": "false", 2023-10-01T10:40:22.220 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_debug_fsck_abort": "false", 2023-10-01T10:40:22.220 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_debug_inject_allocation_from_file_failure": "0.000000", 2023-10-01T10:40:22.220 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_debug_inject_bug21040": "false", 2023-10-01T10:40:22.220 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_debug_inject_csum_err_probability": "0.000000", 2023-10-01T10:40:22.220 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_debug_inject_read_err": "false", 2023-10-01T10:40:22.221 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_debug_legacy_omap": "false", 2023-10-01T10:40:22.221 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_debug_no_reuse_blocks": "false", 2023-10-01T10:40:22.221 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_debug_omit_block_device_write": "false", 2023-10-01T10:40:22.221 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_debug_omit_kv_commit": "false", 2023-10-01T10:40:22.221 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_debug_permit_any_bdev_label": "false", 2023-10-01T10:40:22.221 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_debug_prefill": "0.000000", 2023-10-01T10:40:22.221 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_debug_prefragment_max": "1048576", 2023-10-01T10:40:22.221 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_debug_random_read_err": "0.000000", 2023-10-01T10:40:22.221 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_debug_randomize_serial_transaction": "0", 2023-10-01T10:40:22.221 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_debug_small_allocations": "0", 2023-10-01T10:40:22.221 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_debug_too_many_blobs_threshold": "24576", 2023-10-01T10:40:22.221 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_default_buffered_read": "true", 2023-10-01T10:40:22.221 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_default_buffered_write": "false", 2023-10-01T10:40:22.222 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_deferred_batch_ops": "0", 2023-10-01T10:40:22.222 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_deferred_batch_ops_hdd": "64", 2023-10-01T10:40:22.222 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_deferred_batch_ops_ssd": "16", 2023-10-01T10:40:22.222 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_extent_map_inline_shard_prealloc_size": "256", 2023-10-01T10:40:22.222 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_extent_map_shard_max_size": "1200", 2023-10-01T10:40:22.222 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_extent_map_shard_min_size": "150", 2023-10-01T10:40:22.222 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_extent_map_shard_target_size": "500", 2023-10-01T10:40:22.222 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_extent_map_shard_target_size_slop": "0.200000", 2023-10-01T10:40:22.222 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_fail_eio": "false", 2023-10-01T10:40:22.222 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_freelist_blocks_per_key": "128", 2023-10-01T10:40:22.222 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_fsck_error_on_no_per_pg_omap": "false", 2023-10-01T10:40:22.222 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_fsck_error_on_no_per_pool_omap": "false", 2023-10-01T10:40:22.223 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_fsck_error_on_no_per_pool_stats": "false", 2023-10-01T10:40:22.223 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_fsck_on_mkfs": "true", 2023-10-01T10:40:22.223 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_fsck_on_mkfs_deep": "false", 2023-10-01T10:40:22.223 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_fsck_on_mount": "false", 2023-10-01T10:40:22.223 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_fsck_on_mount_deep": "false", 2023-10-01T10:40:22.223 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_fsck_on_umount": "false", 2023-10-01T10:40:22.223 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_fsck_on_umount_deep": "false", 2023-10-01T10:40:22.223 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_fsck_quick_fix_on_mount": "false", 2023-10-01T10:40:22.223 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_fsck_quick_fix_threads": "2", 2023-10-01T10:40:22.223 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_fsck_read_bytes_cap": "67108864", 2023-10-01T10:40:22.223 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_fsck_shared_blob_tracker_size": "0.031250", 2023-10-01T10:40:22.223 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_gc_enable_blob_threshold": "0", 2023-10-01T10:40:22.224 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_gc_enable_total_threshold": "0", 2023-10-01T10:40:22.224 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_hybrid_alloc_mem_cap": "67108864", 2023-10-01T10:40:22.224 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_ignore_data_csum": "false", 2023-10-01T10:40:22.224 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_kv_sync_util_logging_s": "10.000000", 2023-10-01T10:40:22.224 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_kvbackend": "rocksdb", 2023-10-01T10:40:22.224 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_log_collection_list_age": "60.000000", 2023-10-01T10:40:22.224 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_log_omap_iterator_age": "5.000000", 2023-10-01T10:40:22.224 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_log_op_age": "5.000000", 2023-10-01T10:40:22.224 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_max_alloc_size": "0", 2023-10-01T10:40:22.224 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_max_blob_size": "0", 2023-10-01T10:40:22.224 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_max_blob_size_hdd": "65536", 2023-10-01T10:40:22.224 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_max_blob_size_ssd": "65536", 2023-10-01T10:40:22.225 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_max_defer_interval": "3.000000", 2023-10-01T10:40:22.225 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_max_deferred_txc": "32", 2023-10-01T10:40:22.225 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_min_alloc_size": "0", 2023-10-01T10:40:22.225 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_min_alloc_size_hdd": "4096", 2023-10-01T10:40:22.225 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_min_alloc_size_ssd": "4096", 2023-10-01T10:40:22.225 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_nid_prealloc": "1024", 2023-10-01T10:40:22.225 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_prefer_deferred_size": "0", 2023-10-01T10:40:22.225 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_prefer_deferred_size_hdd": "65536", 2023-10-01T10:40:22.225 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_prefer_deferred_size_ssd": "0", 2023-10-01T10:40:22.225 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_qfsck_on_mount": "true", 2023-10-01T10:40:22.225 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_retry_disk_reads": "3", 2023-10-01T10:40:22.225 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_rocksdb_cf": "true", 2023-10-01T10:40:22.225 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_rocksdb_cfs": "m(3) p(3,0-12) O(3,0-13)=block_cache={type=binned_lru} L=min_write_buffer_number_to_merge=32 P=min_write_buffer_number_to_merge=32", 2023-10-01T10:40:22.226 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_rocksdb_options": "compression=kNoCompression,max_write_buffer_number=64,min_write_buffer_number_to_merge=6,compaction_style=kCompactionStyleLevel,write_buffer_size=16777216,max_background_jobs=4,level0_file_num_compaction_trigger=8,max_bytes_for_level_base=1073741824,max_bytes_for_level_multiplier=8,compaction_readahead_size=2MB,max_total_wal_size=1073741824,writable_file_max_buffer_size=0", 2023-10-01T10:40:22.226 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_rocksdb_options_annex": "", 2023-10-01T10:40:22.226 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_spdk_coremask": "0x1", 2023-10-01T10:40:22.226 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_spdk_io_sleep": "5", 2023-10-01T10:40:22.226 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_spdk_max_io_completion": "0", 2023-10-01T10:40:22.226 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_spdk_mem": "512", 2023-10-01T10:40:22.226 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_sync_submit_transaction": "false", 2023-10-01T10:40:22.226 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_throttle_bytes": "67108864", 2023-10-01T10:40:22.226 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_throttle_cost_per_io": "0", 2023-10-01T10:40:22.226 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_throttle_cost_per_io_hdd": "670000", 2023-10-01T10:40:22.226 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_throttle_cost_per_io_ssd": "4000", 2023-10-01T10:40:22.226 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_throttle_deferred_bytes": "134217728", 2023-10-01T10:40:22.227 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_throttle_trace_rate": "0.000000", 2023-10-01T10:40:22.227 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_tracing": "false", 2023-10-01T10:40:22.227 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_use_optimal_io_size_for_min_alloc_size": "false", 2023-10-01T10:40:22.227 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_volume_selection_policy": "use_some_extra", 2023-10-01T10:40:22.227 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_volume_selection_reserved": "0", 2023-10-01T10:40:22.227 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_volume_selection_reserved_factor": "2.000000", 2023-10-01T10:40:22.227 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_warn_on_bluefs_spillover": "true", 2023-10-01T10:40:22.227 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_warn_on_legacy_statfs": "true", 2023-10-01T10:40:22.227 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_warn_on_no_per_pg_omap": "false", 2023-10-01T10:40:22.227 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_warn_on_no_per_pool_omap": "true", 2023-10-01T10:40:22.227 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_warn_on_spurious_read_errors": "true", 2023-10-01T10:40:22.228 INFO:teuthology.orchestra.run.smithi078.stdout: "bluestore_zero_block_detection": "false", 2023-10-01T10:40:22.228 INFO:teuthology.orchestra.run.smithi078.stdout: "cephadm_path": "/usr/sbin/cephadm", 2023-10-01T10:40:22.228 INFO:teuthology.orchestra.run.smithi078.stdout: "cephfs_mirror_action_update_interval": "2", 2023-10-01T10:40:22.228 INFO:teuthology.orchestra.run.smithi078.stdout: "cephfs_mirror_directory_scan_interval": "10", 2023-10-01T10:40:22.228 INFO:teuthology.orchestra.run.smithi078.stdout: "cephfs_mirror_max_concurrent_directory_syncs": "3", 2023-10-01T10:40:22.228 INFO:teuthology.orchestra.run.smithi078.stdout: "cephfs_mirror_max_consecutive_failures_per_directory": "10", 2023-10-01T10:40:22.228 INFO:teuthology.orchestra.run.smithi078.stdout: "cephfs_mirror_max_snapshot_sync_per_cycle": "3", 2023-10-01T10:40:22.228 INFO:teuthology.orchestra.run.smithi078.stdout: "cephfs_mirror_mount_timeout": "10", 2023-10-01T10:40:22.228 INFO:teuthology.orchestra.run.smithi078.stdout: "cephfs_mirror_restart_mirror_on_blocklist_interval": "30", 2023-10-01T10:40:22.228 INFO:teuthology.orchestra.run.smithi078.stdout: "cephfs_mirror_restart_mirror_on_failure_interval": "20", 2023-10-01T10:40:22.228 INFO:teuthology.orchestra.run.smithi078.stdout: "cephfs_mirror_retry_failed_directories_interval": "60", 2023-10-01T10:40:22.228 INFO:teuthology.orchestra.run.smithi078.stdout: "cephsqlite_blocklist_dead_locker": "true", 2023-10-01T10:40:22.228 INFO:teuthology.orchestra.run.smithi078.stdout: "cephsqlite_lock_renewal_interval": "2000", 2023-10-01T10:40:22.229 INFO:teuthology.orchestra.run.smithi078.stdout: "cephsqlite_lock_renewal_timeout": "30000", 2023-10-01T10:40:22.229 INFO:teuthology.orchestra.run.smithi078.stdout: "cephx_cluster_require_signatures": "false", 2023-10-01T10:40:22.229 INFO:teuthology.orchestra.run.smithi078.stdout: "cephx_cluster_require_version": "2", 2023-10-01T10:40:22.229 INFO:teuthology.orchestra.run.smithi078.stdout: "cephx_require_signatures": "false", 2023-10-01T10:40:22.229 INFO:teuthology.orchestra.run.smithi078.stdout: "cephx_require_version": "2", 2023-10-01T10:40:22.229 INFO:teuthology.orchestra.run.smithi078.stdout: "cephx_service_require_signatures": "false", 2023-10-01T10:40:22.229 INFO:teuthology.orchestra.run.smithi078.stdout: "cephx_service_require_version": "2", 2023-10-01T10:40:22.229 INFO:teuthology.orchestra.run.smithi078.stdout: "cephx_sign_messages": "true", 2023-10-01T10:40:22.229 INFO:teuthology.orchestra.run.smithi078.stdout: "chdir": "", 2023-10-01T10:40:22.229 INFO:teuthology.orchestra.run.smithi078.stdout: "client_acl_type": "", 2023-10-01T10:40:22.229 INFO:teuthology.orchestra.run.smithi078.stdout: "client_asio_thread_count": "2", 2023-10-01T10:40:22.230 INFO:teuthology.orchestra.run.smithi078.stdout: "client_cache_mid": "0.750000", 2023-10-01T10:40:22.230 INFO:teuthology.orchestra.run.smithi078.stdout: "client_cache_size": "16384", 2023-10-01T10:40:22.230 INFO:teuthology.orchestra.run.smithi078.stdout: "client_caps_release_delay": "5", 2023-10-01T10:40:22.230 INFO:teuthology.orchestra.run.smithi078.stdout: "client_check_pool_perm": "true", 2023-10-01T10:40:22.230 INFO:teuthology.orchestra.run.smithi078.stdout: "client_collect_and_send_global_metrics": "false", 2023-10-01T10:40:22.230 INFO:teuthology.orchestra.run.smithi078.stdout: "client_debug_force_sync_read": "false", 2023-10-01T10:40:22.230 INFO:teuthology.orchestra.run.smithi078.stdout: "client_debug_getattr_caps": "false", 2023-10-01T10:40:22.230 INFO:teuthology.orchestra.run.smithi078.stdout: "client_debug_inject_tick_delay": "0", 2023-10-01T10:40:22.230 INFO:teuthology.orchestra.run.smithi078.stdout: "client_die_on_failed_dentry_invalidate": "true", 2023-10-01T10:40:22.230 INFO:teuthology.orchestra.run.smithi078.stdout: "client_die_on_failed_remount": "false", 2023-10-01T10:40:22.230 INFO:teuthology.orchestra.run.smithi078.stdout: "client_dirsize_rbytes": "true", 2023-10-01T10:40:22.230 INFO:teuthology.orchestra.run.smithi078.stdout: "client_force_lazyio": "false", 2023-10-01T10:40:22.231 INFO:teuthology.orchestra.run.smithi078.stdout: "client_fs": "", 2023-10-01T10:40:22.231 INFO:teuthology.orchestra.run.smithi078.stdout: "client_inject_fixed_oldest_tid": "false", 2023-10-01T10:40:22.231 INFO:teuthology.orchestra.run.smithi078.stdout: "client_inject_release_failure": "false", 2023-10-01T10:40:22.231 INFO:teuthology.orchestra.run.smithi078.stdout: "client_max_inline_size": "4096", 2023-10-01T10:40:22.231 INFO:teuthology.orchestra.run.smithi078.stdout: "client_max_retries_on_remount_failure": "5", 2023-10-01T10:40:22.231 INFO:teuthology.orchestra.run.smithi078.stdout: "client_mds_namespace": "", 2023-10-01T10:40:22.231 INFO:teuthology.orchestra.run.smithi078.stdout: "client_metadata": "", 2023-10-01T10:40:22.231 INFO:teuthology.orchestra.run.smithi078.stdout: "client_mount_gid": "-1", 2023-10-01T10:40:22.231 INFO:teuthology.orchestra.run.smithi078.stdout: "client_mount_timeout": "300", 2023-10-01T10:40:22.231 INFO:teuthology.orchestra.run.smithi078.stdout: "client_mount_uid": "-1", 2023-10-01T10:40:22.231 INFO:teuthology.orchestra.run.smithi078.stdout: "client_mountpoint": "/", 2023-10-01T10:40:22.232 INFO:teuthology.orchestra.run.smithi078.stdout: "client_notify_timeout": "10", 2023-10-01T10:40:22.232 INFO:teuthology.orchestra.run.smithi078.stdout: "client_oc": "true", 2023-10-01T10:40:22.232 INFO:teuthology.orchestra.run.smithi078.stdout: "client_oc_max_dirty": "104857600", 2023-10-01T10:40:22.232 INFO:teuthology.orchestra.run.smithi078.stdout: "client_oc_max_dirty_age": "5.000000", 2023-10-01T10:40:22.232 INFO:teuthology.orchestra.run.smithi078.stdout: "client_oc_max_objects": "1000", 2023-10-01T10:40:22.232 INFO:teuthology.orchestra.run.smithi078.stdout: "client_oc_size": "209715200", 2023-10-01T10:40:22.232 INFO:teuthology.orchestra.run.smithi078.stdout: "client_oc_target_dirty": "8388608", 2023-10-01T10:40:22.232 INFO:teuthology.orchestra.run.smithi078.stdout: "client_permissions": "true", 2023-10-01T10:40:22.232 INFO:teuthology.orchestra.run.smithi078.stdout: "client_quota": "true", 2023-10-01T10:40:22.232 INFO:teuthology.orchestra.run.smithi078.stdout: "client_quota_df": "true", 2023-10-01T10:40:22.232 INFO:teuthology.orchestra.run.smithi078.stdout: "client_readahead_max_bytes": "0", 2023-10-01T10:40:22.232 INFO:teuthology.orchestra.run.smithi078.stdout: "client_readahead_max_periods": "4", 2023-10-01T10:40:22.233 INFO:teuthology.orchestra.run.smithi078.stdout: "client_readahead_min": "131072", 2023-10-01T10:40:22.233 INFO:teuthology.orchestra.run.smithi078.stdout: "client_reconnect_stale": "false", 2023-10-01T10:40:22.233 INFO:teuthology.orchestra.run.smithi078.stdout: "client_shutdown_timeout": "30", 2023-10-01T10:40:22.233 INFO:teuthology.orchestra.run.smithi078.stdout: "client_snapdir": ".snap", 2023-10-01T10:40:22.233 INFO:teuthology.orchestra.run.smithi078.stdout: "client_tick_interval": "1", 2023-10-01T10:40:22.233 INFO:teuthology.orchestra.run.smithi078.stdout: "client_trace": "", 2023-10-01T10:40:22.233 INFO:teuthology.orchestra.run.smithi078.stdout: "client_try_dentry_invalidate": "false", 2023-10-01T10:40:22.233 INFO:teuthology.orchestra.run.smithi078.stdout: "client_use_faked_inos": "false", 2023-10-01T10:40:22.233 INFO:teuthology.orchestra.run.smithi078.stdout: "client_use_random_mds": "false", 2023-10-01T10:40:22.233 INFO:teuthology.orchestra.run.smithi078.stdout: "clog_to_graylog": "false", 2023-10-01T10:40:22.233 INFO:teuthology.orchestra.run.smithi078.stdout: "clog_to_graylog_host": "127.0.0.1", 2023-10-01T10:40:22.233 INFO:teuthology.orchestra.run.smithi078.stdout: "clog_to_graylog_port": "12201", 2023-10-01T10:40:22.233 INFO:teuthology.orchestra.run.smithi078.stdout: "clog_to_monitors": "default=true", 2023-10-01T10:40:22.234 INFO:teuthology.orchestra.run.smithi078.stdout: "clog_to_syslog": "false", 2023-10-01T10:40:22.234 INFO:teuthology.orchestra.run.smithi078.stdout: "clog_to_syslog_facility": "default=daemon audit=local0", 2023-10-01T10:40:22.234 INFO:teuthology.orchestra.run.smithi078.stdout: "clog_to_syslog_level": "info", 2023-10-01T10:40:22.234 INFO:teuthology.orchestra.run.smithi078.stdout: "cluster_addr": "-", 2023-10-01T10:40:22.234 INFO:teuthology.orchestra.run.smithi078.stdout: "cluster_network": "", 2023-10-01T10:40:22.234 INFO:teuthology.orchestra.run.smithi078.stdout: "cluster_network_interface": "", 2023-10-01T10:40:22.234 INFO:teuthology.orchestra.run.smithi078.stdout: "compressor_zlib_isal": "false", 2023-10-01T10:40:22.234 INFO:teuthology.orchestra.run.smithi078.stdout: "compressor_zlib_level": "5", 2023-10-01T10:40:22.234 INFO:teuthology.orchestra.run.smithi078.stdout: "compressor_zlib_winsize": "-15", 2023-10-01T10:40:22.234 INFO:teuthology.orchestra.run.smithi078.stdout: "compressor_zstd_level": "1", 2023-10-01T10:40:22.235 INFO:teuthology.orchestra.run.smithi078.stdout: "container_image": "quay.ceph.io/ceph-ci/ceph@sha256:cf801071e905a244f3d23a5802f8881f8f03a66c69b055bed9db98c99aef9acf", 2023-10-01T10:40:22.235 INFO:teuthology.orchestra.run.smithi078.stdout: "crash_dir": "/var/lib/ceph/crash", 2023-10-01T10:40:22.235 INFO:teuthology.orchestra.run.smithi078.stdout: "crimson_alien_op_num_threads": "6", 2023-10-01T10:40:22.235 INFO:teuthology.orchestra.run.smithi078.stdout: "crimson_alien_thread_cpu_cores": "", 2023-10-01T10:40:22.235 INFO:teuthology.orchestra.run.smithi078.stdout: "crimson_osd_obc_lru_size": "10", 2023-10-01T10:40:22.235 INFO:teuthology.orchestra.run.smithi078.stdout: "crimson_osd_scheduler_concurrency": "0", 2023-10-01T10:40:22.235 INFO:teuthology.orchestra.run.smithi078.stdout: "crimson_seastar_smp": "1", 2023-10-01T10:40:22.235 INFO:teuthology.orchestra.run.smithi078.stdout: "crush_location": "", 2023-10-01T10:40:22.235 INFO:teuthology.orchestra.run.smithi078.stdout: "crush_location_hook": "", 2023-10-01T10:40:22.235 INFO:teuthology.orchestra.run.smithi078.stdout: "crush_location_hook_timeout": "10", 2023-10-01T10:40:22.235 INFO:teuthology.orchestra.run.smithi078.stdout: "daemonize": "false", 2023-10-01T10:40:22.235 INFO:teuthology.orchestra.run.smithi078.stdout: "daos_pool": "tank", 2023-10-01T10:40:22.235 INFO:teuthology.orchestra.run.smithi078.stdout: "dbstore_config_uri": "file:/var/lib/ceph/radosgw/dbstore-config.db", 2023-10-01T10:40:22.236 INFO:teuthology.orchestra.run.smithi078.stdout: "dbstore_db_dir": "/var/lib/ceph/radosgw", 2023-10-01T10:40:22.236 INFO:teuthology.orchestra.run.smithi078.stdout: "dbstore_db_name_prefix": "dbstore", 2023-10-01T10:40:22.236 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_alienstore": "0/5", 2023-10-01T10:40:22.236 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_allow_any_pool_priority": "false", 2023-10-01T10:40:22.236 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_asok": "1/5", 2023-10-01T10:40:22.236 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_asok_assert_abort": "false", 2023-10-01T10:40:22.236 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_asserts_on_shutdown": "true", 2023-10-01T10:40:22.236 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_auth": "1/5", 2023-10-01T10:40:22.236 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_bdev": "1/3", 2023-10-01T10:40:22.236 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_bluefs": "1/5", 2023-10-01T10:40:22.236 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_bluestore": "1/5", 2023-10-01T10:40:22.236 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_buffer": "0/1", 2023-10-01T10:40:22.236 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_ceph_exporter": "1/5", 2023-10-01T10:40:22.237 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_cephfs_mirror": "0/5", 2023-10-01T10:40:22.237 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_cephsqlite": "0/5", 2023-10-01T10:40:22.237 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_client": "0/5", 2023-10-01T10:40:22.237 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_compressor": "1/5", 2023-10-01T10:40:22.237 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_context": "0/1", 2023-10-01T10:40:22.237 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_crush": "1/1", 2023-10-01T10:40:22.237 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_crypto": "1/5", 2023-10-01T10:40:22.237 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_cyanstore": "0/5", 2023-10-01T10:40:22.237 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_deliberately_leak_memory": "false", 2023-10-01T10:40:22.237 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_disable_randomized_ping": "false", 2023-10-01T10:40:22.237 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_dpdk": "1/5", 2023-10-01T10:40:22.237 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_eventtrace": "1/5", 2023-10-01T10:40:22.238 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_filer": "0/1", 2023-10-01T10:40:22.238 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_filestore": "1/3", 2023-10-01T10:40:22.238 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_finisher": "1/1", 2023-10-01T10:40:22.238 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_fuse": "1/5", 2023-10-01T10:40:22.238 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_heartbeat_testing_span": "0", 2023-10-01T10:40:22.238 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_heartbeatmap": "1/5", 2023-10-01T10:40:22.238 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_immutable_obj_cache": "0/5", 2023-10-01T10:40:22.238 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_javaclient": "1/5", 2023-10-01T10:40:22.238 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_journal": "1/3", 2023-10-01T10:40:22.238 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_journaler": "0/5", 2023-10-01T10:40:22.238 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_kstore": "1/5", 2023-10-01T10:40:22.238 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_leveldb": "4/5", 2023-10-01T10:40:22.238 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_lockdep": "0/1", 2023-10-01T10:40:22.239 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_mclock": "1/5", 2023-10-01T10:40:22.239 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_mds": "1/5", 2023-10-01T10:40:22.239 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_mds_balancer": "1/5", 2023-10-01T10:40:22.239 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_mds_locker": "1/5", 2023-10-01T10:40:22.239 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_mds_log": "1/5", 2023-10-01T10:40:22.239 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_mds_log_expire": "1/5", 2023-10-01T10:40:22.239 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_mds_migrator": "1/5", 2023-10-01T10:40:22.239 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_memstore": "1/5", 2023-10-01T10:40:22.239 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_mgr": "2/5", 2023-10-01T10:40:22.239 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_mgrc": "1/5", 2023-10-01T10:40:22.239 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_mon": "20/20", 2023-10-01T10:40:22.239 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_monc": "0/10", 2023-10-01T10:40:22.239 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_ms": "1/1", 2023-10-01T10:40:22.240 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_none": "0/5", 2023-10-01T10:40:22.240 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_objclass": "0/5", 2023-10-01T10:40:22.240 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_objectcacher": "0/5", 2023-10-01T10:40:22.240 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_objecter": "0/1", 2023-10-01T10:40:22.240 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_optracker": "0/5", 2023-10-01T10:40:22.240 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_osd": "1/5", 2023-10-01T10:40:22.240 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_paxos": "20/20", 2023-10-01T10:40:22.240 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_perfcounter": "1/5", 2023-10-01T10:40:22.240 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_prioritycache": "1/5", 2023-10-01T10:40:22.240 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_rados": "0/5", 2023-10-01T10:40:22.240 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_rbd": "0/5", 2023-10-01T10:40:22.240 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_rbd_mirror": "0/5", 2023-10-01T10:40:22.241 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_rbd_pwl": "0/5", 2023-10-01T10:40:22.241 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_rbd_replay": "0/5", 2023-10-01T10:40:22.241 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_refs": "0/0", 2023-10-01T10:40:22.241 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_reserver": "1/1", 2023-10-01T10:40:22.241 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_rgw": "1/5", 2023-10-01T10:40:22.241 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_rgw_access": "1/5", 2023-10-01T10:40:22.241 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_rgw_datacache": "1/5", 2023-10-01T10:40:22.241 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_rgw_dbstore": "1/5", 2023-10-01T10:40:22.241 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_rgw_flight": "1/5", 2023-10-01T10:40:22.241 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_rgw_sync": "1/5", 2023-10-01T10:40:22.241 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_rocksdb": "4/5", 2023-10-01T10:40:22.241 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_seastore": "0/5", 2023-10-01T10:40:22.241 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_seastore_backref": "0/5", 2023-10-01T10:40:22.242 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_seastore_cache": "0/5", 2023-10-01T10:40:22.242 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_seastore_cleaner": "0/5", 2023-10-01T10:40:22.242 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_seastore_device": "0/5", 2023-10-01T10:40:22.242 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_seastore_epm": "0/5", 2023-10-01T10:40:22.242 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_seastore_fixedkv_tree": "0/5", 2023-10-01T10:40:22.242 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_seastore_journal": "0/5", 2023-10-01T10:40:22.242 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_seastore_lba": "0/5", 2023-10-01T10:40:22.242 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_seastore_odata": "0/5", 2023-10-01T10:40:22.242 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_seastore_omap": "0/5", 2023-10-01T10:40:22.242 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_seastore_onode": "0/5", 2023-10-01T10:40:22.242 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_seastore_t": "0/5", 2023-10-01T10:40:22.242 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_seastore_tm": "0/5", 2023-10-01T10:40:22.242 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_striper": "0/1", 2023-10-01T10:40:22.243 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_test": "0/5", 2023-10-01T10:40:22.243 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_throttle": "1/1", 2023-10-01T10:40:22.243 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_timer": "0/1", 2023-10-01T10:40:22.243 INFO:teuthology.orchestra.run.smithi078.stdout: "debug_tp": "0/5", 2023-10-01T10:40:22.243 INFO:teuthology.orchestra.run.smithi078.stdout: "device_failure_prediction_mode": "none", 2023-10-01T10:40:22.243 INFO:teuthology.orchestra.run.smithi078.stdout: "enable_experimental_unrecoverable_data_corrupting_features": "", 2023-10-01T10:40:22.243 INFO:teuthology.orchestra.run.smithi078.stdout: "erasure_code_dir": "/usr/lib64/ceph/erasure-code", 2023-10-01T10:40:22.243 INFO:teuthology.orchestra.run.smithi078.stdout: "err_to_graylog": "false", 2023-10-01T10:40:22.243 INFO:teuthology.orchestra.run.smithi078.stdout: "err_to_journald": "false", 2023-10-01T10:40:22.243 INFO:teuthology.orchestra.run.smithi078.stdout: "err_to_stderr": "true", 2023-10-01T10:40:22.243 INFO:teuthology.orchestra.run.smithi078.stdout: "err_to_syslog": "false", 2023-10-01T10:40:22.243 INFO:teuthology.orchestra.run.smithi078.stdout: "event_tracing": "false", 2023-10-01T10:40:22.244 INFO:teuthology.orchestra.run.smithi078.stdout: "exporter_addr": "0.0.0.0", 2023-10-01T10:40:22.244 INFO:teuthology.orchestra.run.smithi078.stdout: "exporter_http_port": "9926", 2023-10-01T10:40:22.244 INFO:teuthology.orchestra.run.smithi078.stdout: "exporter_prio_limit": "5", 2023-10-01T10:40:22.244 INFO:teuthology.orchestra.run.smithi078.stdout: "exporter_sock_dir": "/var/run/ceph/", 2023-10-01T10:40:22.244 INFO:teuthology.orchestra.run.smithi078.stdout: "exporter_sort_metrics": "true", 2023-10-01T10:40:22.244 INFO:teuthology.orchestra.run.smithi078.stdout: "exporter_stats_period": "5", 2023-10-01T10:40:22.244 INFO:teuthology.orchestra.run.smithi078.stdout: "fake_statfs_for_testing": "0", 2023-10-01T10:40:22.244 INFO:teuthology.orchestra.run.smithi078.stdout: "fatal_signal_handlers": "true", 2023-10-01T10:40:22.244 INFO:teuthology.orchestra.run.smithi078.stdout: "filer_max_purge_ops": "10", 2023-10-01T10:40:22.244 INFO:teuthology.orchestra.run.smithi078.stdout: "filer_max_truncate_ops": "128", 2023-10-01T10:40:22.244 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_apply_finisher_threads": "1", 2023-10-01T10:40:22.245 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_blackhole": "false", 2023-10-01T10:40:22.245 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_btrfs_clone_range": "true", 2023-10-01T10:40:22.245 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_btrfs_snap": "true", 2023-10-01T10:40:22.245 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_caller_concurrency": "10", 2023-10-01T10:40:22.245 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_collect_device_partition_information": "true", 2023-10-01T10:40:22.245 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_commit_timeout": "600.000000", 2023-10-01T10:40:22.245 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_debug_inject_read_err": "false", 2023-10-01T10:40:22.245 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_debug_omap_check": "false", 2023-10-01T10:40:22.245 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_debug_random_read_err": "0.000000", 2023-10-01T10:40:22.245 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_debug_verify_split": "false", 2023-10-01T10:40:22.245 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_dump_file": "", 2023-10-01T10:40:22.245 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_expected_throughput_bytes": "209715200.000000", 2023-10-01T10:40:22.246 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_expected_throughput_ops": "200.000000", 2023-10-01T10:40:22.246 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_fadvise": "true", 2023-10-01T10:40:22.246 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_fail_eio": "true", 2023-10-01T10:40:22.246 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_fd_cache_shards": "16", 2023-10-01T10:40:22.246 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_fd_cache_size": "128", 2023-10-01T10:40:22.246 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_fiemap": "false", 2023-10-01T10:40:22.246 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_fiemap_threshold": "4096", 2023-10-01T10:40:22.246 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_fsync_flushes_journal_data": "false", 2023-10-01T10:40:22.246 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_index_retry_probability": "0.000000", 2023-10-01T10:40:22.246 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_inject_stall": "0", 2023-10-01T10:40:22.246 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_journal_parallel": "false", 2023-10-01T10:40:22.246 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_journal_trailing": "false", 2023-10-01T10:40:22.246 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_journal_writeahead": "false", 2023-10-01T10:40:22.247 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_kill_at": "0", 2023-10-01T10:40:22.247 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_max_alloc_hint_size": "1048576", 2023-10-01T10:40:22.247 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_max_inline_xattr_size": "0", 2023-10-01T10:40:22.247 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_max_inline_xattr_size_btrfs": "2048", 2023-10-01T10:40:22.247 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_max_inline_xattr_size_other": "512", 2023-10-01T10:40:22.247 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_max_inline_xattr_size_xfs": "65536", 2023-10-01T10:40:22.247 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_max_inline_xattrs": "0", 2023-10-01T10:40:22.247 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_max_inline_xattrs_btrfs": "10", 2023-10-01T10:40:22.247 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_max_inline_xattrs_other": "2", 2023-10-01T10:40:22.247 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_max_inline_xattrs_xfs": "10", 2023-10-01T10:40:22.247 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_max_sync_interval": "5.000000", 2023-10-01T10:40:22.247 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_max_xattr_value_size": "0", 2023-10-01T10:40:22.248 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_max_xattr_value_size_btrfs": "65536", 2023-10-01T10:40:22.248 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_max_xattr_value_size_other": "1024", 2023-10-01T10:40:22.248 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_max_xattr_value_size_xfs": "65536", 2023-10-01T10:40:22.248 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_merge_threshold": "-10", 2023-10-01T10:40:22.248 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_min_sync_interval": "0.010000", 2023-10-01T10:40:22.248 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_odsync_write": "false", 2023-10-01T10:40:22.248 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_omap_backend": "rocksdb", 2023-10-01T10:40:22.248 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_omap_backend_path": "", 2023-10-01T10:40:22.248 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_omap_header_cache_size": "1024", 2023-10-01T10:40:22.248 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_ondisk_finisher_threads": "1", 2023-10-01T10:40:22.248 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_op_thread_suicide_timeout": "180", 2023-10-01T10:40:22.248 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_op_thread_timeout": "60", 2023-10-01T10:40:22.248 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_op_threads": "2", 2023-10-01T10:40:22.249 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_punch_hole": "false", 2023-10-01T10:40:22.249 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_queue_high_delay_multiple": "0.000000", 2023-10-01T10:40:22.249 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_queue_high_delay_multiple_bytes": "0.000000", 2023-10-01T10:40:22.249 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_queue_high_delay_multiple_ops": "0.000000", 2023-10-01T10:40:22.249 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_queue_high_threshhold": "0.900000", 2023-10-01T10:40:22.249 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_queue_low_threshhold": "0.300000", 2023-10-01T10:40:22.249 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_queue_max_bytes": "104857600", 2023-10-01T10:40:22.249 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_queue_max_delay_multiple": "0.000000", 2023-10-01T10:40:22.249 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_queue_max_delay_multiple_bytes": "0.000000", 2023-10-01T10:40:22.249 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_queue_max_delay_multiple_ops": "0.000000", 2023-10-01T10:40:22.249 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_queue_max_ops": "50", 2023-10-01T10:40:22.249 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_rocksdb_options": "max_background_jobs=10,compaction_readahead_size=2097152,compression=kNoCompression", 2023-10-01T10:40:22.250 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_seek_data_hole": "false", 2023-10-01T10:40:22.250 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_sloppy_crc": "false", 2023-10-01T10:40:22.250 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_sloppy_crc_block_size": "65536", 2023-10-01T10:40:22.250 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_splice": "false", 2023-10-01T10:40:22.250 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_split_multiple": "2", 2023-10-01T10:40:22.250 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_split_rand_factor": "20", 2023-10-01T10:40:22.250 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_update_to": "1000", 2023-10-01T10:40:22.250 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_wbthrottle_btrfs_bytes_hard_limit": "419430400", 2023-10-01T10:40:22.250 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_wbthrottle_btrfs_bytes_start_flusher": "41943040", 2023-10-01T10:40:22.250 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_wbthrottle_btrfs_inodes_hard_limit": "5000", 2023-10-01T10:40:22.250 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_wbthrottle_btrfs_inodes_start_flusher": "500", 2023-10-01T10:40:22.250 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_wbthrottle_btrfs_ios_hard_limit": "5000", 2023-10-01T10:40:22.250 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_wbthrottle_btrfs_ios_start_flusher": "500", 2023-10-01T10:40:22.251 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_wbthrottle_enable": "true", 2023-10-01T10:40:22.251 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_wbthrottle_xfs_bytes_hard_limit": "419430400", 2023-10-01T10:40:22.251 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_wbthrottle_xfs_bytes_start_flusher": "41943040", 2023-10-01T10:40:22.251 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_wbthrottle_xfs_inodes_hard_limit": "5000", 2023-10-01T10:40:22.251 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_wbthrottle_xfs_inodes_start_flusher": "500", 2023-10-01T10:40:22.251 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_wbthrottle_xfs_ios_hard_limit": "5000", 2023-10-01T10:40:22.251 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_wbthrottle_xfs_ios_start_flusher": "500", 2023-10-01T10:40:22.251 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_xfs_extsize": "false", 2023-10-01T10:40:22.251 INFO:teuthology.orchestra.run.smithi078.stdout: "filestore_zfs_snap": "false", 2023-10-01T10:40:22.251 INFO:teuthology.orchestra.run.smithi078.stdout: "fio_dir": "/tmp/fio", 2023-10-01T10:40:22.251 INFO:teuthology.orchestra.run.smithi078.stdout: "fsid": "d8016480-6041-11ee-8db5-212e2dc638e7", 2023-10-01T10:40:22.251 INFO:teuthology.orchestra.run.smithi078.stdout: "fuse_allow_other": "true", 2023-10-01T10:40:22.252 INFO:teuthology.orchestra.run.smithi078.stdout: "fuse_atomic_o_trunc": "true", 2023-10-01T10:40:22.252 INFO:teuthology.orchestra.run.smithi078.stdout: "fuse_big_writes": "true", 2023-10-01T10:40:22.252 INFO:teuthology.orchestra.run.smithi078.stdout: "fuse_debug": "false", 2023-10-01T10:40:22.252 INFO:teuthology.orchestra.run.smithi078.stdout: "fuse_default_permissions": "false", 2023-10-01T10:40:22.252 INFO:teuthology.orchestra.run.smithi078.stdout: "fuse_disable_pagecache": "false", 2023-10-01T10:40:22.252 INFO:teuthology.orchestra.run.smithi078.stdout: "fuse_max_write": "0", 2023-10-01T10:40:22.252 INFO:teuthology.orchestra.run.smithi078.stdout: "fuse_multithreaded": "true", 2023-10-01T10:40:22.252 INFO:teuthology.orchestra.run.smithi078.stdout: "fuse_require_active_mds": "true", 2023-10-01T10:40:22.252 INFO:teuthology.orchestra.run.smithi078.stdout: "fuse_set_user_groups": "true", 2023-10-01T10:40:22.252 INFO:teuthology.orchestra.run.smithi078.stdout: "fuse_splice_move": "true", 2023-10-01T10:40:22.252 INFO:teuthology.orchestra.run.smithi078.stdout: "fuse_splice_read": "true", 2023-10-01T10:40:22.253 INFO:teuthology.orchestra.run.smithi078.stdout: "fuse_splice_write": "true", 2023-10-01T10:40:22.253 INFO:teuthology.orchestra.run.smithi078.stdout: "fuse_syncfs_on_mksnap": "true", 2023-10-01T10:40:22.253 INFO:teuthology.orchestra.run.smithi078.stdout: "fuse_use_invalidate_cb": "true", 2023-10-01T10:40:22.253 INFO:teuthology.orchestra.run.smithi078.stdout: "gss_ktab_client_file": "/var/lib/ceph/mon.a/gss_client_mon.a.ktab", 2023-10-01T10:40:22.253 INFO:teuthology.orchestra.run.smithi078.stdout: "gss_target_name": "ceph", 2023-10-01T10:40:22.253 INFO:teuthology.orchestra.run.smithi078.stdout: "heartbeat_file": "", 2023-10-01T10:40:22.253 INFO:teuthology.orchestra.run.smithi078.stdout: "heartbeat_inject_failure": "0", 2023-10-01T10:40:22.253 INFO:teuthology.orchestra.run.smithi078.stdout: "heartbeat_interval": "5", 2023-10-01T10:40:22.253 INFO:teuthology.orchestra.run.smithi078.stdout: "host": "smithi078", 2023-10-01T10:40:22.253 INFO:teuthology.orchestra.run.smithi078.stdout: "immutable_object_cache_client_dedicated_thread_num": "2", 2023-10-01T10:40:22.253 INFO:teuthology.orchestra.run.smithi078.stdout: "immutable_object_cache_max_inflight_ops": "128", 2023-10-01T10:40:22.253 INFO:teuthology.orchestra.run.smithi078.stdout: "immutable_object_cache_max_size": "1073741824", 2023-10-01T10:40:22.254 INFO:teuthology.orchestra.run.smithi078.stdout: "immutable_object_cache_path": "/tmp/ceph_immutable_object_cache", 2023-10-01T10:40:22.254 INFO:teuthology.orchestra.run.smithi078.stdout: "immutable_object_cache_qos_bps_burst": "0", 2023-10-01T10:40:22.254 INFO:teuthology.orchestra.run.smithi078.stdout: "immutable_object_cache_qos_bps_burst_seconds": "1", 2023-10-01T10:40:22.254 INFO:teuthology.orchestra.run.smithi078.stdout: "immutable_object_cache_qos_bps_limit": "0", 2023-10-01T10:40:22.254 INFO:teuthology.orchestra.run.smithi078.stdout: "immutable_object_cache_qos_iops_burst": "0", 2023-10-01T10:40:22.254 INFO:teuthology.orchestra.run.smithi078.stdout: "immutable_object_cache_qos_iops_burst_seconds": "1", 2023-10-01T10:40:22.254 INFO:teuthology.orchestra.run.smithi078.stdout: "immutable_object_cache_qos_iops_limit": "0", 2023-10-01T10:40:22.254 INFO:teuthology.orchestra.run.smithi078.stdout: "immutable_object_cache_qos_schedule_tick_min": "50", 2023-10-01T10:40:22.254 INFO:teuthology.orchestra.run.smithi078.stdout: "immutable_object_cache_sock": "/var/run/ceph/immutable_object_cache_sock", 2023-10-01T10:40:22.254 INFO:teuthology.orchestra.run.smithi078.stdout: "immutable_object_cache_watermark": "0.900000", 2023-10-01T10:40:22.254 INFO:teuthology.orchestra.run.smithi078.stdout: "inject_early_sigterm": "false", 2023-10-01T10:40:22.255 INFO:teuthology.orchestra.run.smithi078.stdout: "jaeger_agent_port": "6799", 2023-10-01T10:40:22.255 INFO:teuthology.orchestra.run.smithi078.stdout: "jaeger_tracing_enable": "false", 2023-10-01T10:40:22.255 INFO:teuthology.orchestra.run.smithi078.stdout: "journal_aio": "true", 2023-10-01T10:40:22.255 INFO:teuthology.orchestra.run.smithi078.stdout: "journal_align_min_size": "65536", 2023-10-01T10:40:22.255 INFO:teuthology.orchestra.run.smithi078.stdout: "journal_block_align": "true", 2023-10-01T10:40:22.255 INFO:teuthology.orchestra.run.smithi078.stdout: "journal_block_size": "4096", 2023-10-01T10:40:22.255 INFO:teuthology.orchestra.run.smithi078.stdout: "journal_dio": "true", 2023-10-01T10:40:22.255 INFO:teuthology.orchestra.run.smithi078.stdout: "journal_discard": "false", 2023-10-01T10:40:22.255 INFO:teuthology.orchestra.run.smithi078.stdout: "journal_force_aio": "false", 2023-10-01T10:40:22.255 INFO:teuthology.orchestra.run.smithi078.stdout: "journal_ignore_corruption": "false", 2023-10-01T10:40:22.255 INFO:teuthology.orchestra.run.smithi078.stdout: "journal_max_write_bytes": "10485760", 2023-10-01T10:40:22.255 INFO:teuthology.orchestra.run.smithi078.stdout: "journal_max_write_entries": "100", 2023-10-01T10:40:22.256 INFO:teuthology.orchestra.run.smithi078.stdout: "journal_replay_from": "0", 2023-10-01T10:40:22.256 INFO:teuthology.orchestra.run.smithi078.stdout: "journal_throttle_high_multiple": "0.000000", 2023-10-01T10:40:22.256 INFO:teuthology.orchestra.run.smithi078.stdout: "journal_throttle_high_threshhold": "0.900000", 2023-10-01T10:40:22.256 INFO:teuthology.orchestra.run.smithi078.stdout: "journal_throttle_low_threshhold": "0.600000", 2023-10-01T10:40:22.256 INFO:teuthology.orchestra.run.smithi078.stdout: "journal_throttle_max_multiple": "0.000000", 2023-10-01T10:40:22.256 INFO:teuthology.orchestra.run.smithi078.stdout: "journal_write_header_frequency": "0", 2023-10-01T10:40:22.256 INFO:teuthology.orchestra.run.smithi078.stdout: "journal_zero_on_create": "false", 2023-10-01T10:40:22.256 INFO:teuthology.orchestra.run.smithi078.stdout: "journaler_prefetch_periods": "10", 2023-10-01T10:40:22.256 INFO:teuthology.orchestra.run.smithi078.stdout: "journaler_prezero_periods": "5", 2023-10-01T10:40:22.256 INFO:teuthology.orchestra.run.smithi078.stdout: "journaler_write_head_interval": "15", 2023-10-01T10:40:22.256 INFO:teuthology.orchestra.run.smithi078.stdout: "key": "", 2023-10-01T10:40:22.256 INFO:teuthology.orchestra.run.smithi078.stdout: "keyfile": "", 2023-10-01T10:40:22.257 INFO:teuthology.orchestra.run.smithi078.stdout: "keyring": "/var/lib/ceph/mon/ceph-a/keyring", 2023-10-01T10:40:22.257 INFO:teuthology.orchestra.run.smithi078.stdout: "kstore_backend": "rocksdb", 2023-10-01T10:40:22.257 INFO:teuthology.orchestra.run.smithi078.stdout: "kstore_default_stripe_size": "65536", 2023-10-01T10:40:22.257 INFO:teuthology.orchestra.run.smithi078.stdout: "kstore_fsck_on_mount": "false", 2023-10-01T10:40:22.257 INFO:teuthology.orchestra.run.smithi078.stdout: "kstore_fsck_on_mount_deep": "true", 2023-10-01T10:40:22.257 INFO:teuthology.orchestra.run.smithi078.stdout: "kstore_max_bytes": "67108864", 2023-10-01T10:40:22.257 INFO:teuthology.orchestra.run.smithi078.stdout: "kstore_max_ops": "512", 2023-10-01T10:40:22.257 INFO:teuthology.orchestra.run.smithi078.stdout: "kstore_nid_prealloc": "1024", 2023-10-01T10:40:22.257 INFO:teuthology.orchestra.run.smithi078.stdout: "kstore_onode_map_size": "1024", 2023-10-01T10:40:22.257 INFO:teuthology.orchestra.run.smithi078.stdout: "kstore_rocksdb_options": "compression=kNoCompression", 2023-10-01T10:40:22.257 INFO:teuthology.orchestra.run.smithi078.stdout: "kstore_sync_submit_transaction": "false", 2023-10-01T10:40:22.257 INFO:teuthology.orchestra.run.smithi078.stdout: "kstore_sync_transaction": "false", 2023-10-01T10:40:22.257 INFO:teuthology.orchestra.run.smithi078.stdout: "leveldb_block_size": "65536", 2023-10-01T10:40:22.258 INFO:teuthology.orchestra.run.smithi078.stdout: "leveldb_bloom_size": "0", 2023-10-01T10:40:22.258 INFO:teuthology.orchestra.run.smithi078.stdout: "leveldb_cache_size": "536870912", 2023-10-01T10:40:22.258 INFO:teuthology.orchestra.run.smithi078.stdout: "leveldb_compact_on_mount": "false", 2023-10-01T10:40:22.258 INFO:teuthology.orchestra.run.smithi078.stdout: "leveldb_compression": "false", 2023-10-01T10:40:22.258 INFO:teuthology.orchestra.run.smithi078.stdout: "leveldb_log": "", 2023-10-01T10:40:22.258 INFO:teuthology.orchestra.run.smithi078.stdout: "leveldb_log_to_ceph_log": "true", 2023-10-01T10:40:22.258 INFO:teuthology.orchestra.run.smithi078.stdout: "leveldb_max_open_files": "0", 2023-10-01T10:40:22.258 INFO:teuthology.orchestra.run.smithi078.stdout: "leveldb_paranoid": "false", 2023-10-01T10:40:22.258 INFO:teuthology.orchestra.run.smithi078.stdout: "leveldb_write_buffer_size": "33554432", 2023-10-01T10:40:22.258 INFO:teuthology.orchestra.run.smithi078.stdout: "librados_thread_count": "2", 2023-10-01T10:40:22.258 INFO:teuthology.orchestra.run.smithi078.stdout: "lockdep": "false", 2023-10-01T10:40:22.258 INFO:teuthology.orchestra.run.smithi078.stdout: "lockdep_force_backtrace": "false", 2023-10-01T10:40:22.259 INFO:teuthology.orchestra.run.smithi078.stdout: "log_coarse_timestamps": "true", 2023-10-01T10:40:22.259 INFO:teuthology.orchestra.run.smithi078.stdout: "log_file": "/var/log/ceph/ceph-mon.a.log", 2023-10-01T10:40:22.259 INFO:teuthology.orchestra.run.smithi078.stdout: "log_flush_on_exit": "false", 2023-10-01T10:40:22.259 INFO:teuthology.orchestra.run.smithi078.stdout: "log_graylog_host": "127.0.0.1", 2023-10-01T10:40:22.259 INFO:teuthology.orchestra.run.smithi078.stdout: "log_graylog_port": "12201", 2023-10-01T10:40:22.259 INFO:teuthology.orchestra.run.smithi078.stdout: "log_max_new": "1000", 2023-10-01T10:40:22.259 INFO:teuthology.orchestra.run.smithi078.stdout: "log_max_recent": "10000", 2023-10-01T10:40:22.259 INFO:teuthology.orchestra.run.smithi078.stdout: "log_stderr_prefix": "debug ", 2023-10-01T10:40:22.259 INFO:teuthology.orchestra.run.smithi078.stdout: "log_stop_at_utilization": "0.970000", 2023-10-01T10:40:22.259 INFO:teuthology.orchestra.run.smithi078.stdout: "log_to_file": "true", 2023-10-01T10:40:22.259 INFO:teuthology.orchestra.run.smithi078.stdout: "log_to_graylog": "false", 2023-10-01T10:40:22.260 INFO:teuthology.orchestra.run.smithi078.stdout: "log_to_journald": "false", 2023-10-01T10:40:22.260 INFO:teuthology.orchestra.run.smithi078.stdout: "log_to_stderr": "false", 2023-10-01T10:40:22.260 INFO:teuthology.orchestra.run.smithi078.stdout: "log_to_syslog": "false", 2023-10-01T10:40:22.260 INFO:teuthology.orchestra.run.smithi078.stdout: "max_rotating_auth_attempts": "10", 2023-10-01T10:40:22.260 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_abort_on_newly_corrupt_dentry": "true", 2023-10-01T10:40:22.260 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_action_on_write_error": "1", 2023-10-01T10:40:22.260 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_alternate_name_max": "8192", 2023-10-01T10:40:22.260 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_asio_thread_count": "2", 2023-10-01T10:40:22.260 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_bal_export_pin": "true", 2023-10-01T10:40:22.260 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_bal_fragment_dirs": "true", 2023-10-01T10:40:22.260 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_bal_fragment_fast_factor": "1.500000", 2023-10-01T10:40:22.261 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_bal_fragment_interval": "5", 2023-10-01T10:40:22.261 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_bal_fragment_size_max": "100000", 2023-10-01T10:40:22.261 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_bal_idle_threshold": "0.000000", 2023-10-01T10:40:22.261 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_bal_interval": "10", 2023-10-01T10:40:22.261 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_bal_max": "-1", 2023-10-01T10:40:22.261 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_bal_max_until": "-1", 2023-10-01T10:40:22.261 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_bal_merge_size": "50", 2023-10-01T10:40:22.261 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_bal_midchunk": "0.300000", 2023-10-01T10:40:22.261 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_bal_min_rebalance": "0.100000", 2023-10-01T10:40:22.261 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_bal_min_start": "0.200000", 2023-10-01T10:40:22.261 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_bal_minchunk": "0.001000", 2023-10-01T10:40:22.261 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_bal_mode": "0", 2023-10-01T10:40:22.262 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_bal_need_max": "1.200000", 2023-10-01T10:40:22.262 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_bal_need_min": "0.800000", 2023-10-01T10:40:22.262 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_bal_replicate_threshold": "8000.000000", 2023-10-01T10:40:22.262 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_bal_sample_interval": "3.000000", 2023-10-01T10:40:22.262 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_bal_split_bits": "3", 2023-10-01T10:40:22.262 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_bal_split_rd": "25000.000000", 2023-10-01T10:40:22.262 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_bal_split_size": "10000", 2023-10-01T10:40:22.262 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_bal_split_wr": "10000.000000", 2023-10-01T10:40:22.262 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_bal_target_decay": "10.000000", 2023-10-01T10:40:22.262 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_bal_unreplicate_threshold": "0.000000", 2023-10-01T10:40:22.262 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_beacon_grace": "15.000000", 2023-10-01T10:40:22.263 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_beacon_interval": "4.000000", 2023-10-01T10:40:22.263 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_beacon_mon_down_grace": "60", 2023-10-01T10:40:22.263 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_cache_memory_limit": "4294967296", 2023-10-01T10:40:22.263 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_cache_mid": "0.700000", 2023-10-01T10:40:22.263 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_cache_release_free_interval": "10", 2023-10-01T10:40:22.263 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_cache_reservation": "0.050000", 2023-10-01T10:40:22.263 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_cache_trim_decay_rate": "1.000000", 2023-10-01T10:40:22.263 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_cache_trim_interval": "1", 2023-10-01T10:40:22.263 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_cache_trim_threshold": "262144", 2023-10-01T10:40:22.263 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_cap_acquisition_throttle_retry_request_timeout": "0.500000", 2023-10-01T10:40:22.263 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_cap_revoke_eviction_timeout": "0.000000", 2023-10-01T10:40:22.263 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_client_delegate_inos_pct": "50", 2023-10-01T10:40:22.264 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_client_prealloc_inos": "1000", 2023-10-01T10:40:22.264 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_client_writeable_range_max_inc_objs": "1024", 2023-10-01T10:40:22.264 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_connect_bootstrapping": "false", 2023-10-01T10:40:22.264 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_damage_table_max_entries": "10000", 2023-10-01T10:40:22.264 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_data": "/var/lib/ceph/mds/ceph-a", 2023-10-01T10:40:22.264 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_debug_auth_pins": "false", 2023-10-01T10:40:22.264 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_debug_frag": "false", 2023-10-01T10:40:22.264 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_debug_scatterstat": "false", 2023-10-01T10:40:22.264 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_debug_subtrees": "false", 2023-10-01T10:40:22.264 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_decay_halflife": "5.000000", 2023-10-01T10:40:22.264 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_default_dir_hash": "2", 2023-10-01T10:40:22.265 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_defer_session_stale": "true", 2023-10-01T10:40:22.265 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_deny_all_reconnect": "false", 2023-10-01T10:40:22.265 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_dir_keys_per_op": "16384", 2023-10-01T10:40:22.265 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_dir_max_commit_size": "10", 2023-10-01T10:40:22.265 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_dir_max_entries": "0", 2023-10-01T10:40:22.265 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_dir_prefetch": "true", 2023-10-01T10:40:22.265 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_dirstat_min_interval": "1.000000", 2023-10-01T10:40:22.265 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_dump_cache_after_rejoin": "false", 2023-10-01T10:40:22.265 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_dump_cache_on_map": "false", 2023-10-01T10:40:22.265 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_dump_cache_threshold_file": "0", 2023-10-01T10:40:22.265 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_dump_cache_threshold_formatter": "1073741824", 2023-10-01T10:40:22.265 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_early_reply": "true", 2023-10-01T10:40:22.266 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_enable_op_tracker": "true", 2023-10-01T10:40:22.266 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_enforce_unique_name": "true", 2023-10-01T10:40:22.266 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_export_ephemeral_distributed": "true", 2023-10-01T10:40:22.266 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_export_ephemeral_distributed_factor": "2.000000", 2023-10-01T10:40:22.266 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_export_ephemeral_random": "true", 2023-10-01T10:40:22.266 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_export_ephemeral_random_max": "0.010000", 2023-10-01T10:40:22.266 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_extraordinary_events_dump_interval": "0", 2023-10-01T10:40:22.266 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_forward_all_requests_to_auth": "false", 2023-10-01T10:40:22.266 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_freeze_tree_timeout": "30.000000", 2023-10-01T10:40:22.266 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_fscrypt_last_block_max_size": "4096", 2023-10-01T10:40:22.266 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_go_bad_corrupt_dentry": "true", 2023-10-01T10:40:22.266 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_hack_allow_loading_invalid_metadata": "false", 2023-10-01T10:40:22.267 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_health_cache_threshold": "1.500000", 2023-10-01T10:40:22.267 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_health_summarize_threshold": "10", 2023-10-01T10:40:22.267 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_heartbeat_grace": "15.000000", 2023-10-01T10:40:22.267 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_heartbeat_reset_grace": "1000", 2023-10-01T10:40:22.267 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_inject_health_dummy": "false", 2023-10-01T10:40:22.267 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_inject_journal_corrupt_dentry_first": "0.000000", 2023-10-01T10:40:22.267 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_inject_migrator_session_race": "false", 2023-10-01T10:40:22.267 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_inject_rename_corrupt_dentry_first": "0.000000", 2023-10-01T10:40:22.267 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_inject_skip_replaying_inotable": "false", 2023-10-01T10:40:22.267 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_inject_traceless_reply_probability": "0.000000", 2023-10-01T10:40:22.267 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_join_fs": "", 2023-10-01T10:40:22.267 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_journal_format": "1", 2023-10-01T10:40:22.267 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_kill_create_at": "0", 2023-10-01T10:40:22.268 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_kill_export_at": "0", 2023-10-01T10:40:22.268 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_kill_import_at": "0", 2023-10-01T10:40:22.268 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_kill_journal_at": "0", 2023-10-01T10:40:22.268 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_kill_journal_expire_at": "0", 2023-10-01T10:40:22.268 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_kill_journal_replay_at": "0", 2023-10-01T10:40:22.268 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_kill_link_at": "0", 2023-10-01T10:40:22.268 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_kill_mdstable_at": "0", 2023-10-01T10:40:22.268 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_kill_openc_at": "0", 2023-10-01T10:40:22.268 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_kill_rename_at": "0", 2023-10-01T10:40:22.268 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_kill_skip_replaying_inotable": "false", 2023-10-01T10:40:22.268 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_log_events_per_segment": "1024", 2023-10-01T10:40:22.269 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_log_max_events": "-1", 2023-10-01T10:40:22.269 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_log_max_segments": "128", 2023-10-01T10:40:22.269 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_log_pause": "false", 2023-10-01T10:40:22.269 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_log_segment_size": "0", 2023-10-01T10:40:22.269 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_log_skip_corrupt_events": "false", 2023-10-01T10:40:22.269 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_log_warn_factor": "2.000000", 2023-10-01T10:40:22.269 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_max_caps_per_client": "1048576", 2023-10-01T10:40:22.269 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_max_completed_flushes": "100000", 2023-10-01T10:40:22.269 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_max_completed_requests": "100000", 2023-10-01T10:40:22.269 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_max_export_size": "20971520", 2023-10-01T10:40:22.269 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_max_file_recover": "32", 2023-10-01T10:40:22.269 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_max_purge_files": "64", 2023-10-01T10:40:22.270 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_max_purge_ops": "8192", 2023-10-01T10:40:22.270 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_max_purge_ops_per_pg": "0.500000", 2023-10-01T10:40:22.270 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_max_scrub_ops_in_progress": "5", 2023-10-01T10:40:22.270 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_max_snaps_per_dir": "100", 2023-10-01T10:40:22.270 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_max_xattr_pairs_size": "65536", 2023-10-01T10:40:22.270 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_metrics_update_interval": "2", 2023-10-01T10:40:22.270 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_min_caps_per_client": "100", 2023-10-01T10:40:22.270 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_min_caps_working_set": "10000", 2023-10-01T10:40:22.270 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_mon_shutdown_timeout": "5.000000", 2023-10-01T10:40:22.270 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_numa_node": "-1", 2023-10-01T10:40:22.270 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_oft_prefetch_dirfrags": "false", 2023-10-01T10:40:22.271 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_op_complaint_time": "30.000000", 2023-10-01T10:40:22.271 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_op_history_duration": "600", 2023-10-01T10:40:22.271 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_op_history_size": "20", 2023-10-01T10:40:22.271 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_op_log_threshold": "5", 2023-10-01T10:40:22.271 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_ping_grace": "15", 2023-10-01T10:40:22.271 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_ping_interval": "5", 2023-10-01T10:40:22.271 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_purge_queue_busy_flush_period": "1.000000", 2023-10-01T10:40:22.271 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_recall_global_max_decay_threshold": "131072", 2023-10-01T10:40:22.271 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_recall_max_caps": "30000", 2023-10-01T10:40:22.271 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_recall_max_decay_rate": "1.500000", 2023-10-01T10:40:22.271 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_recall_max_decay_threshold": "131072", 2023-10-01T10:40:22.271 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_recall_warning_decay_rate": "60.000000", 2023-10-01T10:40:22.271 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_recall_warning_threshold": "262144", 2023-10-01T10:40:22.272 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_reconnect_timeout": "45.000000", 2023-10-01T10:40:22.272 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_replay_interval": "1.000000", 2023-10-01T10:40:22.272 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_replay_unsafe_with_closed_session": "false", 2023-10-01T10:40:22.272 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_request_load_average_decay_rate": "60.000000", 2023-10-01T10:40:22.272 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_root_ino_gid": "0", 2023-10-01T10:40:22.272 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_root_ino_uid": "0", 2023-10-01T10:40:22.272 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_scatter_nudge_interval": "5.000000", 2023-10-01T10:40:22.272 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_session_blocklist_on_evict": "true", 2023-10-01T10:40:22.272 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_session_blocklist_on_timeout": "true", 2023-10-01T10:40:22.272 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_session_cache_liveness_decay_rate": "300.000000", 2023-10-01T10:40:22.272 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_session_cache_liveness_magnitude": "10", 2023-10-01T10:40:22.272 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_session_cap_acquisition_decay_rate": "30.000000", 2023-10-01T10:40:22.273 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_session_cap_acquisition_throttle": "100000", 2023-10-01T10:40:22.273 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_session_max_caps_throttle_ratio": "1.100000", 2023-10-01T10:40:22.273 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_sessionmap_keys_per_op": "1024", 2023-10-01T10:40:22.273 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_shutdown_check": "0", 2023-10-01T10:40:22.273 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_skip_ino": "0", 2023-10-01T10:40:22.273 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_sleep_rank_change": "0.000000", 2023-10-01T10:40:22.273 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_snap_max_uid": "4294967294", 2023-10-01T10:40:22.273 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_snap_min_uid": "0", 2023-10-01T10:40:22.273 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_snap_rstat": "false", 2023-10-01T10:40:22.273 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_standby_replay_damaged": "false", 2023-10-01T10:40:22.273 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_symlink_recovery": "true", 2023-10-01T10:40:22.274 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_task_status_update_interval": "2.000000", 2023-10-01T10:40:22.274 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_thrash_exports": "0", 2023-10-01T10:40:22.274 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_thrash_fragments": "0", 2023-10-01T10:40:22.274 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_tick_interval": "5.000000", 2023-10-01T10:40:22.274 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_valgrind_exit": "false", 2023-10-01T10:40:22.274 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_verify_backtrace": "1", 2023-10-01T10:40:22.274 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_verify_scatter": "false", 2023-10-01T10:40:22.274 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_wipe_ino_prealloc": "false", 2023-10-01T10:40:22.274 INFO:teuthology.orchestra.run.smithi078.stdout: "mds_wipe_sessions": "false", 2023-10-01T10:40:22.274 INFO:teuthology.orchestra.run.smithi078.stdout: "mempool_debug": "false", 2023-10-01T10:40:22.275 INFO:teuthology.orchestra.run.smithi078.stdout: "memstore_debug_omit_block_device_write": "false", 2023-10-01T10:40:22.275 INFO:teuthology.orchestra.run.smithi078.stdout: "memstore_device_bytes": "1073741824", 2023-10-01T10:40:22.275 INFO:teuthology.orchestra.run.smithi078.stdout: "memstore_page_set": "false", 2023-10-01T10:40:22.275 INFO:teuthology.orchestra.run.smithi078.stdout: "memstore_page_size": "65536", 2023-10-01T10:40:22.275 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr_client_bytes": "134217728", 2023-10-01T10:40:22.275 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr_client_messages": "512", 2023-10-01T10:40:22.275 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr_client_service_daemon_unregister_timeout": "1.000000", 2023-10-01T10:40:22.275 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr_connect_retry_interval": "1.000000", 2023-10-01T10:40:22.275 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr_data": "/var/lib/ceph/mgr/ceph-a", 2023-10-01T10:40:22.275 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr_debug_aggressive_pg_num_changes": "false", 2023-10-01T10:40:22.275 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr_disabled_modules": "", 2023-10-01T10:40:22.275 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr_initial_modules": "restful iostat nfs", 2023-10-01T10:40:22.275 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr_max_pg_num_change": "128", 2023-10-01T10:40:22.276 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr_mds_bytes": "134217728", 2023-10-01T10:40:22.276 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr_mds_messages": "128", 2023-10-01T10:40:22.276 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr_module_path": "/usr/share/ceph/mgr", 2023-10-01T10:40:22.276 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr_mon_bytes": "134217728", 2023-10-01T10:40:22.276 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr_mon_messages": "128", 2023-10-01T10:40:22.276 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr_osd_bytes": "536870912", 2023-10-01T10:40:22.276 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr_osd_messages": "8192", 2023-10-01T10:40:22.276 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr_pool": "true", 2023-10-01T10:40:22.276 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr_service_beacon_grace": "60.000000", 2023-10-01T10:40:22.276 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr_standby_modules": "true", 2023-10-01T10:40:22.276 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr_stats_period": "5", 2023-10-01T10:40:22.276 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr_stats_threshold": "5", 2023-10-01T10:40:22.277 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr_tick_period": "2", 2023-10-01T10:40:22.277 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr_ttl_cache_expire_seconds": "0", 2023-10-01T10:40:22.277 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_accept_timeout_factor": "2.000000", 2023-10-01T10:40:22.277 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_allow_pool_delete": "true", 2023-10-01T10:40:22.277 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_allow_pool_size_one": "false", 2023-10-01T10:40:22.277 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_auth_validate_all_caps": "true", 2023-10-01T10:40:22.277 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_cache_target_full_warn_ratio": "0.660000", 2023-10-01T10:40:22.277 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_clean_pg_upmaps_per_chunk": "256", 2023-10-01T10:40:22.277 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_client_bytes": "104857600", 2023-10-01T10:40:22.277 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_client_directed_command_retry": "2", 2023-10-01T10:40:22.277 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_client_hunt_interval": "3.000000", 2023-10-01T10:40:22.277 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_client_hunt_interval_backoff": "1.500000", 2023-10-01T10:40:22.278 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_client_hunt_interval_max_multiple": "10.000000", 2023-10-01T10:40:22.278 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_client_hunt_interval_min_multiple": "1.000000", 2023-10-01T10:40:22.278 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_client_hunt_parallel": "3", 2023-10-01T10:40:22.278 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_client_log_interval": "1.000000", 2023-10-01T10:40:22.278 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_client_max_log_entries_per_message": "1000", 2023-10-01T10:40:22.278 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_client_ping_interval": "10.000000", 2023-10-01T10:40:22.278 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_client_ping_timeout": "30.000000", 2023-10-01T10:40:22.278 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_clock_drift_allowed": "1.000000", 2023-10-01T10:40:22.278 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_clock_drift_warn_backoff": "5.000000", 2023-10-01T10:40:22.278 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_cluster_log_file": "default=/var/log/ceph/ceph.$channel.log cluster=/var/log/ceph/ceph.log", 2023-10-01T10:40:22.278 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_cluster_log_file_level": "debug", 2023-10-01T10:40:22.278 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_cluster_log_to_file": "false", 2023-10-01T10:40:22.279 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_cluster_log_to_graylog": "false", 2023-10-01T10:40:22.279 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_cluster_log_to_graylog_host": "127.0.0.1", 2023-10-01T10:40:22.279 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_cluster_log_to_graylog_port": "12201", 2023-10-01T10:40:22.279 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_cluster_log_to_journald": "false", 2023-10-01T10:40:22.279 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_cluster_log_to_stderr": "true", 2023-10-01T10:40:22.279 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_cluster_log_to_syslog": "default=false", 2023-10-01T10:40:22.279 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_cluster_log_to_syslog_facility": "daemon", 2023-10-01T10:40:22.279 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_cluster_log_to_syslog_level": "info", 2023-10-01T10:40:22.279 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_compact_on_bootstrap": "false", 2023-10-01T10:40:22.279 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_compact_on_start": "false", 2023-10-01T10:40:22.279 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_compact_on_trim": "true", 2023-10-01T10:40:22.279 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_con_tracker_persist_interval": "10", 2023-10-01T10:40:22.279 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_con_tracker_score_halflife": "43200", 2023-10-01T10:40:22.280 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_config_key_max_entry_size": "65536", 2023-10-01T10:40:22.280 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_cpu_threads": "4", 2023-10-01T10:40:22.280 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_crush_min_required_version": "hammer", 2023-10-01T10:40:22.280 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_daemon_bytes": "419430400", 2023-10-01T10:40:22.280 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_data": "/var/lib/ceph/mon/ceph-a", 2023-10-01T10:40:22.280 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_data_avail_crit": "5", 2023-10-01T10:40:22.280 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_data_avail_warn": "5", 2023-10-01T10:40:22.280 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_data_size_warn": "16106127360", 2023-10-01T10:40:22.280 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_debug_block_osdmap_trim": "false", 2023-10-01T10:40:22.280 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_debug_deprecated_as_obsolete": "false", 2023-10-01T10:40:22.280 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_debug_dump_json": "false", 2023-10-01T10:40:22.280 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_debug_dump_location": "/var/log/ceph/ceph-mon.a.tdump", 2023-10-01T10:40:22.281 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_debug_dump_transactions": "false", 2023-10-01T10:40:22.281 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_debug_extra_checks": "false", 2023-10-01T10:40:22.281 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_debug_no_initial_persistent_features": "false", 2023-10-01T10:40:22.281 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_debug_no_require_bluestore_for_ec_overwrites": "false", 2023-10-01T10:40:22.281 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_debug_no_require_quincy": "false", 2023-10-01T10:40:22.281 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_debug_no_require_reef": "false", 2023-10-01T10:40:22.281 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_debug_unsafe_allow_tier_with_nonempty_snaps": "false", 2023-10-01T10:40:22.281 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_delta_reset_interval": "10.000000", 2023-10-01T10:40:22.281 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_dns_srv_name": "ceph-mon", 2023-10-01T10:40:22.281 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_down_mkfs_grace": "60", 2023-10-01T10:40:22.281 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_election_default_strategy": "3", 2023-10-01T10:40:22.281 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_election_timeout": "5.000000", 2023-10-01T10:40:22.282 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_elector_ignore_propose_margin": "0.000500", 2023-10-01T10:40:22.282 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_elector_ping_divisor": "2", 2023-10-01T10:40:22.282 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_elector_ping_timeout": "2.000000", 2023-10-01T10:40:22.282 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_enable_op_tracker": "true", 2023-10-01T10:40:22.282 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_fake_pool_delete": "false", 2023-10-01T10:40:22.282 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_force_quorum_join": "false", 2023-10-01T10:40:22.282 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_globalid_prealloc": "10000", 2023-10-01T10:40:22.282 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_health_detail_to_clog": "true", 2023-10-01T10:40:22.282 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_health_log_update_period": "5", 2023-10-01T10:40:22.282 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_health_max_detail": "50", 2023-10-01T10:40:22.282 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_health_to_clog": "true", 2023-10-01T10:40:22.282 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_health_to_clog_interval": "600", 2023-10-01T10:40:22.282 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_health_to_clog_tick_interval": "60.000000", 2023-10-01T10:40:22.283 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_host": "[v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] [v2:172.21.15.173:3300/0,v1:172.21.15.173:6789/0] [v2:172.21.15.78:3301/0,v1:172.21.15.78:6790/0]", 2023-10-01T10:40:22.283 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_host_override": "", 2023-10-01T10:40:22.283 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_initial_members": "", 2023-10-01T10:40:22.283 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_inject_pg_merge_bounce_probability": "0.000000", 2023-10-01T10:40:22.283 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_inject_sync_get_chunk_delay": "0.000000", 2023-10-01T10:40:22.283 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_inject_transaction_delay_max": "10.000000", 2023-10-01T10:40:22.283 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_inject_transaction_delay_probability": "0.000000", 2023-10-01T10:40:22.283 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_keyvaluedb": "rocksdb", 2023-10-01T10:40:22.283 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_lease": "5.000000", 2023-10-01T10:40:22.283 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_lease_ack_timeout_factor": "2.000000", 2023-10-01T10:40:22.283 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_lease_renew_interval_factor": "0.600000", 2023-10-01T10:40:22.283 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_log_full_interval": "50", 2023-10-01T10:40:22.284 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_log_max": "10000", 2023-10-01T10:40:22.284 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_log_max_summary": "50", 2023-10-01T10:40:22.284 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_max_log_entries_per_event": "4096", 2023-10-01T10:40:22.284 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_max_log_epochs": "500", 2023-10-01T10:40:22.284 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_max_mdsmap_epochs": "500", 2023-10-01T10:40:22.284 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_max_mgrmap_epochs": "500", 2023-10-01T10:40:22.284 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_max_osd": "10000", 2023-10-01T10:40:22.284 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_max_pg_per_osd": "10000", 2023-10-01T10:40:22.284 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_max_pool_pg_num": "65536", 2023-10-01T10:40:22.284 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_max_snap_prune_per_epoch": "100", 2023-10-01T10:40:22.284 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_mds_blocklist_interval": "86400.000000", 2023-10-01T10:40:22.285 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_mds_force_trim_to": "0", 2023-10-01T10:40:22.285 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_mds_skip_sanity": "true", 2023-10-01T10:40:22.285 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_memory_autotune": "true", 2023-10-01T10:40:22.285 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_memory_target": "2147483648", 2023-10-01T10:40:22.285 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_mgr_beacon_grace": "30", 2023-10-01T10:40:22.285 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_mgr_blocklist_interval": "86400.000000", 2023-10-01T10:40:22.285 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_mgr_digest_period": "5", 2023-10-01T10:40:22.285 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_mgr_inactive_grace": "60", 2023-10-01T10:40:22.285 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_mgr_mkfs_grace": "240", 2023-10-01T10:40:22.285 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_mgr_proxy_client_bytes_ratio": "0.300000", 2023-10-01T10:40:22.285 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_min_osdmap_epochs": "500", 2023-10-01T10:40:22.285 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_op_complaint_time": "30", 2023-10-01T10:40:22.285 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_op_history_duration": "600", 2023-10-01T10:40:22.286 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_op_history_size": "20", 2023-10-01T10:40:22.286 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_op_history_slow_op_size": "20", 2023-10-01T10:40:22.286 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_op_history_slow_op_threshold": "10", 2023-10-01T10:40:22.286 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_op_log_threshold": "5", 2023-10-01T10:40:22.286 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_osd_adjust_down_out_interval": "true", 2023-10-01T10:40:22.286 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_osd_adjust_heartbeat_grace": "true", 2023-10-01T10:40:22.286 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_osd_auto_mark_auto_out_in": "true", 2023-10-01T10:40:22.286 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_osd_auto_mark_in": "false", 2023-10-01T10:40:22.286 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_osd_auto_mark_new_in": "true", 2023-10-01T10:40:22.286 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_osd_backfillfull_ratio": "0.900000", 2023-10-01T10:40:22.286 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_osd_blocklist_default_expire": "3600.000000", 2023-10-01T10:40:22.286 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_osd_cache_size": "500", 2023-10-01T10:40:22.286 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_osd_cache_size_min": "134217728", 2023-10-01T10:40:22.287 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_osd_crush_smoke_test": "true", 2023-10-01T10:40:22.287 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_osd_destroyed_out_interval": "600", 2023-10-01T10:40:22.287 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_osd_down_out_interval": "600", 2023-10-01T10:40:22.287 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_osd_down_out_subtree_limit": "rack", 2023-10-01T10:40:22.287 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_osd_err_op_age_ratio": "128.000000", 2023-10-01T10:40:22.287 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_osd_force_trim_to": "0", 2023-10-01T10:40:22.287 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_osd_full_ratio": "0.950000", 2023-10-01T10:40:22.287 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_osd_initial_require_min_compat_client": "luminous", 2023-10-01T10:40:22.287 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_osd_laggy_halflife": "3600", 2023-10-01T10:40:22.287 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_osd_laggy_max_interval": "300", 2023-10-01T10:40:22.287 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_osd_laggy_weight": "0.300000", 2023-10-01T10:40:22.288 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_osd_mapping_pgs_per_chunk": "4096", 2023-10-01T10:40:22.288 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_osd_max_creating_pgs": "1024", 2023-10-01T10:40:22.288 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_osd_max_initial_pgs": "1024", 2023-10-01T10:40:22.288 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_osd_min_down_reporters": "2", 2023-10-01T10:40:22.288 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_osd_min_in_ratio": "0.750000", 2023-10-01T10:40:22.288 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_osd_min_up_ratio": "0.300000", 2023-10-01T10:40:22.288 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_osd_nearfull_ratio": "0.850000", 2023-10-01T10:40:22.288 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_osd_prime_pg_temp": "true", 2023-10-01T10:40:22.288 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_osd_prime_pg_temp_max_estimate": "0.250000", 2023-10-01T10:40:22.288 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_osd_prime_pg_temp_max_time": "0.500000", 2023-10-01T10:40:22.288 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_osd_report_timeout": "900", 2023-10-01T10:40:22.288 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_osd_reporter_subtree_level": "osd", 2023-10-01T10:40:22.288 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_osd_snap_trim_queue_warn_on": "32768", 2023-10-01T10:40:22.288 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_osd_warn_num_repaired": "10", 2023-10-01T10:40:22.289 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_osd_warn_op_age": "32.000000", 2023-10-01T10:40:22.289 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_osdmap_full_prune_enabled": "true", 2023-10-01T10:40:22.289 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_osdmap_full_prune_interval": "10", 2023-10-01T10:40:22.289 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_osdmap_full_prune_min": "10000", 2023-10-01T10:40:22.289 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_osdmap_full_prune_txsize": "100", 2023-10-01T10:40:22.289 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_pg_check_down_all_threshold": "0.500000", 2023-10-01T10:40:22.289 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_pg_stuck_threshold": "60", 2023-10-01T10:40:22.289 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_pg_warn_max_object_skew": "0.000000", 2023-10-01T10:40:22.289 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_pg_warn_min_objects": "10000", 2023-10-01T10:40:22.289 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_pg_warn_min_per_osd": "0", 2023-10-01T10:40:22.289 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_pg_warn_min_pool_objects": "1000", 2023-10-01T10:40:22.289 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_pool_quota_crit_threshold": "0", 2023-10-01T10:40:22.290 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_pool_quota_warn_threshold": "0", 2023-10-01T10:40:22.290 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_probe_timeout": "2.000000", 2023-10-01T10:40:22.290 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_reweight_max_change": "0.050000", 2023-10-01T10:40:22.290 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_reweight_max_osds": "4", 2023-10-01T10:40:22.290 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_reweight_min_bytes_per_osd": "10", 2023-10-01T10:40:22.290 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_reweight_min_pgs_per_osd": "4", 2023-10-01T10:40:22.290 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_rocksdb_options": "write_buffer_size=33554432,compression=kNoCompression,level_compaction_dynamic_level_bytes=true", 2023-10-01T10:40:22.290 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_scrub_inject_crc_mismatch": "0.000000", 2023-10-01T10:40:22.290 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_scrub_inject_missing_keys": "0.000000", 2023-10-01T10:40:22.290 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_scrub_interval": "86400", 2023-10-01T10:40:22.290 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_scrub_max_keys": "100", 2023-10-01T10:40:22.290 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_scrub_timeout": "300", 2023-10-01T10:40:22.291 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_session_timeout": "300", 2023-10-01T10:40:22.291 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_smart_report_timeout": "5", 2023-10-01T10:40:22.291 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_stat_smooth_intervals": "6", 2023-10-01T10:40:22.291 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_stretch_cluster_recovery_ratio": "0.600000", 2023-10-01T10:40:22.291 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_stretch_pool_min_size": "2", 2023-10-01T10:40:22.291 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_stretch_pool_size": "4", 2023-10-01T10:40:22.291 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_stretch_recovery_min_wait": "15.000000", 2023-10-01T10:40:22.291 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_subscribe_interval": "86400.000000", 2023-10-01T10:40:22.291 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_sync_debug": "false", 2023-10-01T10:40:22.291 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_sync_max_payload_keys": "2000", 2023-10-01T10:40:22.291 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_sync_max_payload_size": "1048576", 2023-10-01T10:40:22.291 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_sync_provider_kill_at": "0", 2023-10-01T10:40:22.292 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_sync_requester_kill_at": "0", 2023-10-01T10:40:22.292 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_sync_timeout": "60.000000", 2023-10-01T10:40:22.292 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_target_pg_per_osd": "100", 2023-10-01T10:40:22.292 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_tick_interval": "5", 2023-10-01T10:40:22.292 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_timecheck_interval": "300.000000", 2023-10-01T10:40:22.292 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_timecheck_skew_interval": "30.000000", 2023-10-01T10:40:22.292 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_use_min_delay_socket": "false", 2023-10-01T10:40:22.292 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_warn_older_version_delay": "604800", 2023-10-01T10:40:22.292 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_warn_on_cache_pools_without_hit_sets": "true", 2023-10-01T10:40:22.292 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_warn_on_crush_straw_calc_version_zero": "false", 2023-10-01T10:40:22.292 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_warn_on_degraded_stretch_mode": "true", 2023-10-01T10:40:22.292 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_warn_on_filestore_osds": "true", 2023-10-01T10:40:22.293 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_warn_on_insecure_global_id_reclaim": "false", 2023-10-01T10:40:22.293 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed": "false", 2023-10-01T10:40:22.293 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_warn_on_legacy_crush_tunables": "false", 2023-10-01T10:40:22.293 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_warn_on_misplaced": "false", 2023-10-01T10:40:22.293 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_warn_on_msgr2_not_enabled": "true", 2023-10-01T10:40:22.293 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_warn_on_older_version": "true", 2023-10-01T10:40:22.293 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_warn_on_osd_down_out_interval_zero": "false", 2023-10-01T10:40:22.293 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_warn_on_pool_no_app": "true", 2023-10-01T10:40:22.293 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_warn_on_pool_no_redundancy": "true", 2023-10-01T10:40:22.293 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_warn_on_pool_pg_num_not_power_of_two": "false", 2023-10-01T10:40:22.293 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_warn_on_slow_ping_ratio": "0.050000", 2023-10-01T10:40:22.293 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_warn_on_slow_ping_time": "0.000000", 2023-10-01T10:40:22.294 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_warn_on_too_few_osds": "false", 2023-10-01T10:40:22.294 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_warn_pg_not_deep_scrubbed_ratio": "0.750000", 2023-10-01T10:40:22.294 INFO:teuthology.orchestra.run.smithi078.stdout: "mon_warn_pg_not_scrubbed_ratio": "0.500000", 2023-10-01T10:40:22.294 INFO:teuthology.orchestra.run.smithi078.stdout: "monmap": "", 2023-10-01T10:40:22.294 INFO:teuthology.orchestra.run.smithi078.stdout: "motr_admin_endpoint": "192.168.180.182@tcp:12345:4:1", 2023-10-01T10:40:22.294 INFO:teuthology.orchestra.run.smithi078.stdout: "motr_admin_fid": "0x7200000000000001:0x0", 2023-10-01T10:40:22.294 INFO:teuthology.orchestra.run.smithi078.stdout: "motr_ha_endpoint": "192.168.180.182@tcp:12345:1:1", 2023-10-01T10:40:22.294 INFO:teuthology.orchestra.run.smithi078.stdout: "motr_my_endpoint": "192.168.180.182@tcp:12345:4:1", 2023-10-01T10:40:22.294 INFO:teuthology.orchestra.run.smithi078.stdout: "motr_my_fid": "0x7200000000000001:0x0", 2023-10-01T10:40:22.294 INFO:teuthology.orchestra.run.smithi078.stdout: "motr_profile_fid": "0x7000000000000001:0x0", 2023-10-01T10:40:22.294 INFO:teuthology.orchestra.run.smithi078.stdout: "motr_tracing_enabled": "false", 2023-10-01T10:40:22.295 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_async_op_threads": "3", 2023-10-01T10:40:22.295 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_async_rdma_buffer_size": "131072", 2023-10-01T10:40:22.295 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_async_rdma_cm": "false", 2023-10-01T10:40:22.295 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_async_rdma_device_name": "", 2023-10-01T10:40:22.295 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_async_rdma_dscp": "96", 2023-10-01T10:40:22.295 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_async_rdma_enable_hugepage": "false", 2023-10-01T10:40:22.295 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_async_rdma_gid_idx": "0", 2023-10-01T10:40:22.295 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_async_rdma_local_gid": "", 2023-10-01T10:40:22.295 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_async_rdma_polling_us": "1000", 2023-10-01T10:40:22.295 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_async_rdma_port_num": "1", 2023-10-01T10:40:22.295 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_async_rdma_receive_buffers": "32768", 2023-10-01T10:40:22.295 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_async_rdma_receive_queue_len": "4096", 2023-10-01T10:40:22.296 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_async_rdma_roce_ver": "1", 2023-10-01T10:40:22.296 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_async_rdma_send_buffers": "1024", 2023-10-01T10:40:22.296 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_async_rdma_sl": "3", 2023-10-01T10:40:22.296 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_async_rdma_support_srq": "true", 2023-10-01T10:40:22.296 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_async_rdma_type": "ib", 2023-10-01T10:40:22.296 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_async_reap_threshold": "5", 2023-10-01T10:40:22.296 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_bind_before_connect": "false", 2023-10-01T10:40:22.296 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_bind_ipv4": "true", 2023-10-01T10:40:22.296 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_bind_ipv6": "false", 2023-10-01T10:40:22.296 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_bind_msgr1": "true", 2023-10-01T10:40:22.296 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_bind_msgr2": "true", 2023-10-01T10:40:22.296 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_bind_port_max": "7568", 2023-10-01T10:40:22.297 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_bind_port_min": "6800", 2023-10-01T10:40:22.297 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_bind_prefer_ipv4": "false", 2023-10-01T10:40:22.297 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_bind_retry_count": "3", 2023-10-01T10:40:22.297 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_bind_retry_delay": "5", 2023-10-01T10:40:22.297 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_blackhole_client": "false", 2023-10-01T10:40:22.297 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_blackhole_mds": "false", 2023-10-01T10:40:22.297 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_blackhole_mgr": "false", 2023-10-01T10:40:22.297 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_blackhole_mon": "false", 2023-10-01T10:40:22.297 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_blackhole_osd": "false", 2023-10-01T10:40:22.297 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_client_mode": "crc secure", 2023-10-01T10:40:22.297 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_cluster_mode": "crc secure", 2023-10-01T10:40:22.297 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_cluster_type": "", 2023-10-01T10:40:22.298 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_compress_secure": "false", 2023-10-01T10:40:22.298 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_connection_idle_timeout": "900", 2023-10-01T10:40:22.298 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_connection_ready_timeout": "10", 2023-10-01T10:40:22.298 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_crc_data": "true", 2023-10-01T10:40:22.298 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_crc_header": "true", 2023-10-01T10:40:22.298 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_die_on_bad_msg": "false", 2023-10-01T10:40:22.298 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_die_on_bug": "true", 2023-10-01T10:40:22.298 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_die_on_old_message": "true", 2023-10-01T10:40:22.298 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_die_on_skipped_message": "false", 2023-10-01T10:40:22.298 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_die_on_unhandled_msg": "false", 2023-10-01T10:40:22.298 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_dispatch_throttle_bytes": "104857600", 2023-10-01T10:40:22.299 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_dpdk_coremask": "0xF", 2023-10-01T10:40:22.299 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_dpdk_debug_allow_loopback": "false", 2023-10-01T10:40:22.299 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_dpdk_devs_allowlist": "", 2023-10-01T10:40:22.299 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_dpdk_enable_tso": "true", 2023-10-01T10:40:22.299 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_dpdk_gateway_ipv4_addr": "", 2023-10-01T10:40:22.299 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_dpdk_host_ipv4_addr": "", 2023-10-01T10:40:22.299 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_dpdk_hugepages": "", 2023-10-01T10:40:22.299 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_dpdk_hw_flow_control": "true", 2023-10-01T10:40:22.299 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_dpdk_hw_queue_weight": "1.000000", 2023-10-01T10:40:22.299 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_dpdk_lro": "true", 2023-10-01T10:40:22.299 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_dpdk_memory_channel": "4", 2023-10-01T10:40:22.299 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_dpdk_netmask_ipv4_addr": "", 2023-10-01T10:40:22.299 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_dpdk_pmd": "", 2023-10-01T10:40:22.300 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_dpdk_port_id": "0", 2023-10-01T10:40:22.300 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_dpdk_rx_buffer_count_per_core": "8192", 2023-10-01T10:40:22.300 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_dump_corrupt_message_level": "1", 2023-10-01T10:40:22.300 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_dump_on_send": "false", 2023-10-01T10:40:22.300 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_initial_backoff": "0.200000", 2023-10-01T10:40:22.300 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_inject_delay_max": "1.000000", 2023-10-01T10:40:22.300 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_inject_delay_probability": "0.000000", 2023-10-01T10:40:22.300 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_inject_delay_type": "", 2023-10-01T10:40:22.300 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_inject_internal_delays": "0.000000", 2023-10-01T10:40:22.300 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_inject_network_congestion": "0", 2023-10-01T10:40:22.300 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_inject_socket_failures": "0", 2023-10-01T10:40:22.300 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_learn_addr_from_peer": "true", 2023-10-01T10:40:22.300 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_max_accept_failures": "4", 2023-10-01T10:40:22.301 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_max_backoff": "15.000000", 2023-10-01T10:40:22.301 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_mon_client_mode": "secure crc", 2023-10-01T10:40:22.301 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_mon_cluster_mode": "secure crc", 2023-10-01T10:40:22.301 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_mon_service_mode": "secure crc", 2023-10-01T10:40:22.301 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_osd_compress_min_size": "1024", 2023-10-01T10:40:22.301 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_osd_compress_mode": "none", 2023-10-01T10:40:22.301 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_osd_compression_algorithm": "snappy", 2023-10-01T10:40:22.301 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_pq_max_tokens_per_priority": "16777216", 2023-10-01T10:40:22.301 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_pq_min_cost": "65536", 2023-10-01T10:40:22.301 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_public_type": "", 2023-10-01T10:40:22.301 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_service_mode": "crc secure", 2023-10-01T10:40:22.301 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_tcp_listen_backlog": "512", 2023-10-01T10:40:22.302 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_tcp_nodelay": "true", 2023-10-01T10:40:22.302 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_tcp_prefetch_max_size": "4096", 2023-10-01T10:40:22.302 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_tcp_rcvbuf": "0", 2023-10-01T10:40:22.302 INFO:teuthology.orchestra.run.smithi078.stdout: "ms_type": "async+posix", 2023-10-01T10:40:22.302 INFO:teuthology.orchestra.run.smithi078.stdout: "no_config_file": "false", 2023-10-01T10:40:22.302 INFO:teuthology.orchestra.run.smithi078.stdout: "objecter_completion_locks_per_session": "32", 2023-10-01T10:40:22.302 INFO:teuthology.orchestra.run.smithi078.stdout: "objecter_debug_inject_relock_delay": "false", 2023-10-01T10:40:22.302 INFO:teuthology.orchestra.run.smithi078.stdout: "objecter_inflight_op_bytes": "104857600", 2023-10-01T10:40:22.302 INFO:teuthology.orchestra.run.smithi078.stdout: "objecter_inflight_ops": "1024", 2023-10-01T10:40:22.302 INFO:teuthology.orchestra.run.smithi078.stdout: "objecter_inject_no_watch_ping": "false", 2023-10-01T10:40:22.302 INFO:teuthology.orchestra.run.smithi078.stdout: "objecter_retry_writes_after_first_reply": "false", 2023-10-01T10:40:22.303 INFO:teuthology.orchestra.run.smithi078.stdout: "objecter_tick_interval": "5.000000", 2023-10-01T10:40:22.303 INFO:teuthology.orchestra.run.smithi078.stdout: "objecter_timeout": "10.000000", 2023-10-01T10:40:22.303 INFO:teuthology.orchestra.run.smithi078.stdout: "objectstore_blackhole": "false", 2023-10-01T10:40:22.303 INFO:teuthology.orchestra.run.smithi078.stdout: "openssl_engine_opts": "", 2023-10-01T10:40:22.303 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_agent_delay_time": "5.000000", 2023-10-01T10:40:22.303 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_agent_hist_halflife": "1000", 2023-10-01T10:40:22.303 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_agent_max_low_ops": "2", 2023-10-01T10:40:22.303 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_agent_max_ops": "4", 2023-10-01T10:40:22.303 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_agent_min_evict_effort": "0.100000", 2023-10-01T10:40:22.303 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_agent_quantize_effort": "0.100000", 2023-10-01T10:40:22.303 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_agent_slop": "0.020000", 2023-10-01T10:40:22.304 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_aggregated_slow_ops_logging": "true", 2023-10-01T10:40:22.304 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_allow_recovery_below_min_size": "true", 2023-10-01T10:40:22.304 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_asio_thread_count": "2", 2023-10-01T10:40:22.304 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_async_recovery_min_cost": "100", 2023-10-01T10:40:22.304 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_auto_mark_unfound_lost": "false", 2023-10-01T10:40:22.304 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_backfill_retry_interval": "30.000000", 2023-10-01T10:40:22.304 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_backfill_scan_max": "512", 2023-10-01T10:40:22.304 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_backfill_scan_min": "64", 2023-10-01T10:40:22.304 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_backoff_on_degraded": "false", 2023-10-01T10:40:22.304 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_backoff_on_peering": "false", 2023-10-01T10:40:22.304 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_backoff_on_unfound": "true", 2023-10-01T10:40:22.305 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_beacon_report_interval": "300", 2023-10-01T10:40:22.305 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_bench_duration": "30", 2023-10-01T10:40:22.305 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_bench_large_size_max_throughput": "104857600", 2023-10-01T10:40:22.305 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_bench_max_block_size": "67108864", 2023-10-01T10:40:22.305 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_bench_small_size_max_iops": "100", 2023-10-01T10:40:22.305 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_blkin_trace_all": "false", 2023-10-01T10:40:22.305 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_blocked_scrub_grace_period": "120", 2023-10-01T10:40:22.305 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_calc_pg_upmaps_aggressively": "true", 2023-10-01T10:40:22.305 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_calc_pg_upmaps_aggressively_fast": "true", 2023-10-01T10:40:22.305 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_calc_pg_upmaps_local_fallback_retries": "100", 2023-10-01T10:40:22.305 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_check_for_log_corruption": "false", 2023-10-01T10:40:22.305 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_check_max_object_name_len_on_startup": "true", 2023-10-01T10:40:22.306 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_class_default_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2023-10-01T10:40:22.306 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_class_dir": "/usr/lib64/rados-classes", 2023-10-01T10:40:22.306 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_class_load_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2023-10-01T10:40:22.306 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_class_update_on_start": "true", 2023-10-01T10:40:22.306 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_client_message_cap": "256", 2023-10-01T10:40:22.306 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_client_message_size_cap": "524288000", 2023-10-01T10:40:22.306 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_client_op_priority": "63", 2023-10-01T10:40:22.306 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_client_watch_timeout": "30", 2023-10-01T10:40:22.306 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_command_max_records": "256", 2023-10-01T10:40:22.306 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_compact_on_start": "false", 2023-10-01T10:40:22.306 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_copyfrom_max_chunk": "8388608", 2023-10-01T10:40:22.307 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_crush_chooseleaf_type": "1", 2023-10-01T10:40:22.307 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_crush_initial_weight": "-1.000000", 2023-10-01T10:40:22.307 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_crush_update_on_start": "true", 2023-10-01T10:40:22.307 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_crush_update_weight_set": "true", 2023-10-01T10:40:22.307 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_data": "/var/lib/ceph/osd/ceph-a", 2023-10-01T10:40:22.307 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_debug_crash_on_ignored_backoff": "false", 2023-10-01T10:40:22.307 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_debug_deep_scrub_sleep": "0.000000", 2023-10-01T10:40:22.307 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_debug_drop_ping_duration": "0", 2023-10-01T10:40:22.307 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_debug_drop_ping_probability": "0.000000", 2023-10-01T10:40:22.307 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_debug_feed_pullee": "-1", 2023-10-01T10:40:22.307 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_debug_inject_copyfrom_error": "false", 2023-10-01T10:40:22.307 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_debug_inject_dispatch_delay_duration": "0.100000", 2023-10-01T10:40:22.308 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_debug_inject_dispatch_delay_probability": "0.000000", 2023-10-01T10:40:22.308 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_debug_misdirected_ops": "false", 2023-10-01T10:40:22.308 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_debug_no_acting_change": "false", 2023-10-01T10:40:22.308 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_debug_no_purge_strays": "false", 2023-10-01T10:40:22.308 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_debug_op_order": "false", 2023-10-01T10:40:22.308 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_debug_pg_log_writeout": "false", 2023-10-01T10:40:22.308 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_debug_pretend_recovery_active": "false", 2023-10-01T10:40:22.308 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_debug_random_push_read_error": "0.000000", 2023-10-01T10:40:22.308 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_debug_reject_backfill_probability": "0.000000", 2023-10-01T10:40:22.308 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_debug_shutdown": "false", 2023-10-01T10:40:22.308 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_debug_skip_full_check_in_backfill_reservation": "false", 2023-10-01T10:40:22.308 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_debug_skip_full_check_in_recovery": "false", 2023-10-01T10:40:22.309 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_debug_verify_cached_snaps": "false", 2023-10-01T10:40:22.309 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_debug_verify_missing_on_start": "false", 2023-10-01T10:40:22.309 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_debug_verify_snaps": "false", 2023-10-01T10:40:22.309 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_debug_verify_stray_on_activate": "false", 2023-10-01T10:40:22.309 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_deep_scrub_interval": "604800.000000", 2023-10-01T10:40:22.309 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_deep_scrub_keys": "1024", 2023-10-01T10:40:22.309 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_deep_scrub_large_omap_object_key_threshold": "200000", 2023-10-01T10:40:22.309 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_deep_scrub_large_omap_object_value_sum_threshold": "1073741824", 2023-10-01T10:40:22.309 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_deep_scrub_randomize_ratio": "0.150000", 2023-10-01T10:40:22.309 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_deep_scrub_stride": "524288", 2023-10-01T10:40:22.309 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_deep_scrub_update_digest_min_age": "7200", 2023-10-01T10:40:22.309 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_default_data_pool_replay_window": "45", 2023-10-01T10:40:22.309 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_default_notify_timeout": "30", 2023-10-01T10:40:22.310 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_delete_sleep": "0.000000", 2023-10-01T10:40:22.310 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_delete_sleep_hdd": "5.000000", 2023-10-01T10:40:22.310 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_delete_sleep_hybrid": "1.000000", 2023-10-01T10:40:22.310 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_delete_sleep_ssd": "1.000000", 2023-10-01T10:40:22.310 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_discard_disconnected_ops": "true", 2023-10-01T10:40:22.310 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_enable_op_tracker": "true", 2023-10-01T10:40:22.310 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_erasure_code_plugins": "jerasure lrc", 2023-10-01T10:40:22.310 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_extblkdev_plugins": "vdo", 2023-10-01T10:40:22.310 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_failsafe_full_ratio": "0.970000", 2023-10-01T10:40:22.310 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_fast_fail_on_connection_refused": "true", 2023-10-01T10:40:22.310 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_fast_info": "true", 2023-10-01T10:40:22.311 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_fast_shutdown": "true", 2023-10-01T10:40:22.311 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_fast_shutdown_notify_mon": "true", 2023-10-01T10:40:22.311 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_fast_shutdown_timeout": "15", 2023-10-01T10:40:22.311 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_find_best_info_ignore_history_les": "false", 2023-10-01T10:40:22.311 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_force_auth_primary_missing_objects": "100", 2023-10-01T10:40:22.311 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_force_recovery_pg_log_entries_factor": "1.300000", 2023-10-01T10:40:22.311 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_function_tracing": "false", 2023-10-01T10:40:22.311 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_heartbeat_grace": "20", 2023-10-01T10:40:22.311 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_heartbeat_interval": "6", 2023-10-01T10:40:22.311 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_heartbeat_min_healthy_ratio": "0.330000", 2023-10-01T10:40:22.311 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_heartbeat_min_peers": "10", 2023-10-01T10:40:22.311 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_heartbeat_min_size": "2000", 2023-10-01T10:40:22.312 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_heartbeat_stale": "600", 2023-10-01T10:40:22.312 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_heartbeat_use_min_delay_socket": "false", 2023-10-01T10:40:22.312 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_hit_set_max_size": "100000", 2023-10-01T10:40:22.312 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_hit_set_min_size": "1000", 2023-10-01T10:40:22.312 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_hit_set_namespace": ".ceph-internal", 2023-10-01T10:40:22.312 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_ignore_stale_divergent_priors": "false", 2023-10-01T10:40:22.312 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_inject_bad_map_crc_probability": "0.000000", 2023-10-01T10:40:22.312 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_inject_failure_on_pg_removal": "false", 2023-10-01T10:40:22.312 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_journal": "/var/lib/ceph/osd/ceph-a/journal", 2023-10-01T10:40:22.312 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_journal_flush_on_shutdown": "true", 2023-10-01T10:40:22.312 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_journal_size": "5120", 2023-10-01T10:40:22.312 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_kill_backfill_at": "0", 2023-10-01T10:40:22.313 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_loop_before_reset_tphandle": "64", 2023-10-01T10:40:22.313 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_map_cache_size": "50", 2023-10-01T10:40:22.313 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_map_dedup": "true", 2023-10-01T10:40:22.313 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_map_message_max": "40", 2023-10-01T10:40:22.313 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_map_message_max_bytes": "10485760", 2023-10-01T10:40:22.313 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_map_share_max_epochs": "40", 2023-10-01T10:40:22.313 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_max_attr_name_len": "100", 2023-10-01T10:40:22.313 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_max_attr_size": "0", 2023-10-01T10:40:22.313 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_max_backfills": "1", 2023-10-01T10:40:22.313 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_max_markdown_count": "5", 2023-10-01T10:40:22.313 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_max_markdown_period": "600", 2023-10-01T10:40:22.313 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_max_object_name_len": "2048", 2023-10-01T10:40:22.314 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_max_object_namespace_len": "256", 2023-10-01T10:40:22.314 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_max_object_size": "134217728", 2023-10-01T10:40:22.314 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_max_omap_bytes_per_request": "1073741824", 2023-10-01T10:40:22.314 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_max_omap_entries_per_request": "1024", 2023-10-01T10:40:22.314 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_max_pg_blocked_by": "16", 2023-10-01T10:40:22.314 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_max_pg_log_entries": "10000", 2023-10-01T10:40:22.314 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_max_pg_per_osd_hard_ratio": "3.000000", 2023-10-01T10:40:22.314 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_max_pgls": "1024", 2023-10-01T10:40:22.314 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_max_push_cost": "8388608", 2023-10-01T10:40:22.314 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_max_push_objects": "10", 2023-10-01T10:40:22.314 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_max_scrubs": "1", 2023-10-01T10:40:22.314 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_max_snap_prune_intervals_per_epoch": "512", 2023-10-01T10:40:22.315 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_max_trimming_pgs": "2", 2023-10-01T10:40:22.315 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_max_write_op_reply_len": "64", 2023-10-01T10:40:22.315 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_max_write_size": "90", 2023-10-01T10:40:22.315 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_mclock_force_run_benchmark_on_init": "false", 2023-10-01T10:40:22.315 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_mclock_iops_capacity_threshold_hdd": "500.000000", 2023-10-01T10:40:22.315 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_mclock_iops_capacity_threshold_ssd": "80000.000000", 2023-10-01T10:40:22.315 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_mclock_max_capacity_iops_hdd": "315.000000", 2023-10-01T10:40:22.315 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_mclock_max_capacity_iops_ssd": "21500.000000", 2023-10-01T10:40:22.315 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_mclock_max_sequential_bandwidth_hdd": "157286400", 2023-10-01T10:40:22.315 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_mclock_max_sequential_bandwidth_ssd": "1258291200", 2023-10-01T10:40:22.315 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_mclock_override_recovery_settings": "false", 2023-10-01T10:40:22.315 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_mclock_profile": "balanced", 2023-10-01T10:40:22.316 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_mclock_scheduler_anticipation_timeout": "0.000000", 2023-10-01T10:40:22.316 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_mclock_scheduler_background_best_effort_lim": "0.000000", 2023-10-01T10:40:22.316 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_mclock_scheduler_background_best_effort_res": "0.000000", 2023-10-01T10:40:22.316 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_mclock_scheduler_background_best_effort_wgt": "1", 2023-10-01T10:40:22.316 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_mclock_scheduler_background_recovery_lim": "0.000000", 2023-10-01T10:40:22.316 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_mclock_scheduler_background_recovery_res": "0.000000", 2023-10-01T10:40:22.316 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_mclock_scheduler_background_recovery_wgt": "1", 2023-10-01T10:40:22.316 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_mclock_scheduler_client_lim": "0.000000", 2023-10-01T10:40:22.316 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_mclock_scheduler_client_res": "0.000000", 2023-10-01T10:40:22.316 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_mclock_scheduler_client_wgt": "1", 2023-10-01T10:40:22.316 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_mclock_skip_benchmark": "false", 2023-10-01T10:40:22.316 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_memory_base": "805306368", 2023-10-01T10:40:22.317 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_memory_cache_min": "134217728", 2023-10-01T10:40:22.317 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_memory_cache_resize_interval": "1.000000", 2023-10-01T10:40:22.317 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_memory_expected_fragmentation": "0.150000", 2023-10-01T10:40:22.317 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_memory_target": "4294967296", 2023-10-01T10:40:22.317 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_memory_target_autotune": "false", 2023-10-01T10:40:22.317 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_memory_target_cgroup_limit_ratio": "0.800000", 2023-10-01T10:40:22.317 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_min_pg_log_entries": "250", 2023-10-01T10:40:22.317 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_min_recovery_priority": "0", 2023-10-01T10:40:22.317 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_mon_heartbeat_interval": "30", 2023-10-01T10:40:22.317 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_mon_heartbeat_stat_stale": "3600", 2023-10-01T10:40:22.317 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_mon_report_interval": "5", 2023-10-01T10:40:22.318 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_mon_report_max_in_flight": "2", 2023-10-01T10:40:22.318 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_mon_shutdown_timeout": "5.000000", 2023-10-01T10:40:22.318 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_num_cache_shards": "32", 2023-10-01T10:40:22.318 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_num_op_tracker_shard": "32", 2023-10-01T10:40:22.318 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_numa_auto_affinity": "true", 2023-10-01T10:40:22.318 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_numa_node": "-1", 2023-10-01T10:40:22.318 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_numa_prefer_iface": "true", 2023-10-01T10:40:22.318 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_object_clean_region_max_num_intervals": "10", 2023-10-01T10:40:22.318 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_objecter_finishers": "1", 2023-10-01T10:40:22.318 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_objectstore": "bluestore", 2023-10-01T10:40:22.318 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_objectstore_fuse": "false", 2023-10-01T10:40:22.318 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_objectstore_tracing": "false", 2023-10-01T10:40:22.319 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_op_complaint_time": "30.000000", 2023-10-01T10:40:22.319 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_op_history_duration": "600", 2023-10-01T10:40:22.319 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_op_history_size": "20", 2023-10-01T10:40:22.319 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_op_history_slow_op_size": "20", 2023-10-01T10:40:22.319 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_op_history_slow_op_threshold": "10.000000", 2023-10-01T10:40:22.319 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_op_log_threshold": "5", 2023-10-01T10:40:22.319 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_op_num_shards": "0", 2023-10-01T10:40:22.319 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_op_num_shards_hdd": "5", 2023-10-01T10:40:22.319 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_op_num_shards_ssd": "8", 2023-10-01T10:40:22.319 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_op_num_threads_per_shard": "0", 2023-10-01T10:40:22.319 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_op_num_threads_per_shard_hdd": "1", 2023-10-01T10:40:22.319 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_op_num_threads_per_shard_ssd": "2", 2023-10-01T10:40:22.320 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_op_pq_max_tokens_per_priority": "4194304", 2023-10-01T10:40:22.320 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_op_pq_min_cost": "65536", 2023-10-01T10:40:22.320 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_op_queue": "mclock_scheduler", 2023-10-01T10:40:22.320 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_op_queue_cut_off": "high", 2023-10-01T10:40:22.320 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_op_thread_suicide_timeout": "150", 2023-10-01T10:40:22.320 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_op_thread_timeout": "15", 2023-10-01T10:40:22.320 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_open_classes_on_start": "true", 2023-10-01T10:40:22.320 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_os_flags": "0", 2023-10-01T10:40:22.320 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_peering_op_priority": "255", 2023-10-01T10:40:22.320 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_pg_delete_cost": "1048576", 2023-10-01T10:40:22.320 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_pg_delete_priority": "5", 2023-10-01T10:40:22.320 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_pg_epoch_max_lag_factor": "2.000000", 2023-10-01T10:40:22.320 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_pg_epoch_persisted_max_stale": "40", 2023-10-01T10:40:22.321 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_pg_log_dups_tracked": "3000", 2023-10-01T10:40:22.321 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_pg_log_trim_max": "10000", 2023-10-01T10:40:22.321 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_pg_log_trim_min": "100", 2023-10-01T10:40:22.321 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_pg_max_concurrent_snap_trims": "2", 2023-10-01T10:40:22.321 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_pg_object_context_cache_count": "64", 2023-10-01T10:40:22.321 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_pg_stat_report_interval_max": "500", 2023-10-01T10:40:22.321 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_pool_default_cache_max_evict_check_size": "10", 2023-10-01T10:40:22.321 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_pool_default_cache_min_evict_age": "0", 2023-10-01T10:40:22.321 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_pool_default_cache_min_flush_age": "0", 2023-10-01T10:40:22.321 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_pool_default_cache_target_dirty_high_ratio": "0.600000", 2023-10-01T10:40:22.321 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_pool_default_cache_target_dirty_ratio": "0.400000", 2023-10-01T10:40:22.321 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_pool_default_cache_target_full_ratio": "0.800000", 2023-10-01T10:40:22.321 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_pool_default_crimson": "false", 2023-10-01T10:40:22.322 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_pool_default_crush_rule": "-1", 2023-10-01T10:40:22.322 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_pool_default_ec_fast_read": "false", 2023-10-01T10:40:22.322 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_pool_default_erasure_code_profile": "plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd", 2023-10-01T10:40:22.322 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_pool_default_flag_bulk": "false", 2023-10-01T10:40:22.322 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_pool_default_flag_hashpspool": "true", 2023-10-01T10:40:22.322 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_pool_default_flag_nodelete": "false", 2023-10-01T10:40:22.322 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_pool_default_flag_nopgchange": "false", 2023-10-01T10:40:22.322 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_pool_default_flag_nosizechange": "false", 2023-10-01T10:40:22.322 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_pool_default_flags": "0", 2023-10-01T10:40:22.322 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_pool_default_hit_set_bloom_fpp": "0.050000", 2023-10-01T10:40:22.322 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_pool_default_min_size": "0", 2023-10-01T10:40:22.323 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_pool_default_pg_autoscale_mode": "off", 2023-10-01T10:40:22.323 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_pool_default_pg_num": "32", 2023-10-01T10:40:22.323 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_pool_default_pgp_num": "0", 2023-10-01T10:40:22.323 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_pool_default_read_lease_ratio": "0.800000", 2023-10-01T10:40:22.323 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_pool_default_size": "3", 2023-10-01T10:40:22.323 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_pool_default_type": "replicated", 2023-10-01T10:40:22.323 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_pool_erasure_code_stripe_unit": "4096", 2023-10-01T10:40:22.323 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_pool_use_gmt_hitset": "true", 2023-10-01T10:40:22.323 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_push_per_object_cost": "1000", 2023-10-01T10:40:22.323 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_read_ec_check_for_errors": "false", 2023-10-01T10:40:22.323 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_recover_clone_overlap": "true", 2023-10-01T10:40:22.323 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_recover_clone_overlap_limit": "10", 2023-10-01T10:40:22.323 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_recovery_cost": "20971520", 2023-10-01T10:40:22.324 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_recovery_delay_start": "0.000000", 2023-10-01T10:40:22.324 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_recovery_max_active": "0", 2023-10-01T10:40:22.324 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_recovery_max_active_hdd": "3", 2023-10-01T10:40:22.324 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_recovery_max_active_ssd": "10", 2023-10-01T10:40:22.324 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_recovery_max_chunk": "8388608", 2023-10-01T10:40:22.324 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_recovery_max_omap_entries_per_chunk": "8096", 2023-10-01T10:40:22.324 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_recovery_max_single_start": "1", 2023-10-01T10:40:22.324 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_recovery_op_priority": "3", 2023-10-01T10:40:22.324 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_recovery_op_warn_multiple": "16", 2023-10-01T10:40:22.324 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_recovery_priority": "5", 2023-10-01T10:40:22.324 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_recovery_retry_interval": "30.000000", 2023-10-01T10:40:22.325 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_recovery_sleep": "0.000000", 2023-10-01T10:40:22.325 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_recovery_sleep_hdd": "0.100000", 2023-10-01T10:40:22.325 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_recovery_sleep_hybrid": "0.025000", 2023-10-01T10:40:22.325 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_recovery_sleep_ssd": "0.000000", 2023-10-01T10:40:22.325 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_repair_during_recovery": "false", 2023-10-01T10:40:22.325 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_requested_scrub_priority": "120", 2023-10-01T10:40:22.325 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_rocksdb_iterator_bounds_enabled": "true", 2023-10-01T10:40:22.325 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_rollback_to_cluster_snap": "", 2023-10-01T10:40:22.325 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_scrub_auto_repair": "false", 2023-10-01T10:40:22.325 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_scrub_auto_repair_num_errors": "5", 2023-10-01T10:40:22.325 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_scrub_backoff_ratio": "0.660000", 2023-10-01T10:40:22.325 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_scrub_begin_hour": "0", 2023-10-01T10:40:22.326 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_scrub_begin_week_day": "0", 2023-10-01T10:40:22.326 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_scrub_chunk_max": "25", 2023-10-01T10:40:22.326 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_scrub_chunk_min": "5", 2023-10-01T10:40:22.326 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_scrub_cost": "52428800", 2023-10-01T10:40:22.326 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_scrub_during_recovery": "false", 2023-10-01T10:40:22.326 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_scrub_end_hour": "0", 2023-10-01T10:40:22.326 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_scrub_end_week_day": "0", 2023-10-01T10:40:22.326 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_scrub_event_cost": "4096", 2023-10-01T10:40:22.326 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_scrub_extended_sleep": "0.000000", 2023-10-01T10:40:22.326 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_scrub_interval_randomize_ratio": "0.500000", 2023-10-01T10:40:22.326 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_scrub_invalid_stats": "true", 2023-10-01T10:40:22.326 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_scrub_load_threshold": "0.500000", 2023-10-01T10:40:22.327 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_scrub_max_interval": "604800.000000", 2023-10-01T10:40:22.327 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_scrub_max_preemptions": "5", 2023-10-01T10:40:22.327 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_scrub_min_interval": "86400.000000", 2023-10-01T10:40:22.327 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_scrub_priority": "5", 2023-10-01T10:40:22.327 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_scrub_reservation_timeout": "5000", 2023-10-01T10:40:22.327 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_scrub_sleep": "0.000000", 2023-10-01T10:40:22.327 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_scrub_slow_reservation_response": "2200", 2023-10-01T10:40:22.327 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_shallow_scrub_chunk_max": "100", 2023-10-01T10:40:22.327 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_shallow_scrub_chunk_min": "50", 2023-10-01T10:40:22.327 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_shutdown_pgref_assert": "false", 2023-10-01T10:40:22.327 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_skip_data_digest": "false", 2023-10-01T10:40:22.327 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_smart_report_timeout": "5", 2023-10-01T10:40:22.328 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_snap_trim_cost": "1048576", 2023-10-01T10:40:22.328 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_snap_trim_priority": "5", 2023-10-01T10:40:22.328 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_snap_trim_sleep": "0.000000", 2023-10-01T10:40:22.328 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_snap_trim_sleep_hdd": "5.000000", 2023-10-01T10:40:22.328 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_snap_trim_sleep_hybrid": "2.000000", 2023-10-01T10:40:22.328 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_snap_trim_sleep_ssd": "0.000000", 2023-10-01T10:40:22.328 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_stats_update_period_not_scrubbing": "120", 2023-10-01T10:40:22.328 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_stats_update_period_scrubbing": "15", 2023-10-01T10:40:22.328 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_target_pg_log_entries_per_osd": "300000", 2023-10-01T10:40:22.328 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_target_transaction_size": "30", 2023-10-01T10:40:22.328 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_tier_default_cache_hit_set_count": "4", 2023-10-01T10:40:22.328 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_tier_default_cache_hit_set_grade_decay_rate": "20", 2023-10-01T10:40:22.328 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_tier_default_cache_hit_set_period": "1200", 2023-10-01T10:40:22.329 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_tier_default_cache_hit_set_search_last_n": "1", 2023-10-01T10:40:22.329 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_tier_default_cache_hit_set_type": "bloom", 2023-10-01T10:40:22.329 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_tier_default_cache_min_read_recency_for_promote": "1", 2023-10-01T10:40:22.329 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_tier_default_cache_min_write_recency_for_promote": "1", 2023-10-01T10:40:22.329 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_tier_default_cache_mode": "writeback", 2023-10-01T10:40:22.329 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_tier_promote_max_bytes_sec": "5242880", 2023-10-01T10:40:22.329 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_tier_promote_max_objects_sec": "25", 2023-10-01T10:40:22.329 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_tracing": "false", 2023-10-01T10:40:22.329 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_use_stale_snap": "false", 2023-10-01T10:40:22.329 INFO:teuthology.orchestra.run.smithi078.stdout: "osd_uuid": "00000000-0000-0000-0000-000000000000", 2023-10-01T10:40:22.329 INFO:teuthology.orchestra.run.smithi078.stdout: "osdc_blkin_trace_all": "false", 2023-10-01T10:40:22.329 INFO:teuthology.orchestra.run.smithi078.stdout: "paxos_kill_at": "0", 2023-10-01T10:40:22.330 INFO:teuthology.orchestra.run.smithi078.stdout: "paxos_max_join_drift": "10", 2023-10-01T10:40:22.330 INFO:teuthology.orchestra.run.smithi078.stdout: "paxos_min": "500", 2023-10-01T10:40:22.330 INFO:teuthology.orchestra.run.smithi078.stdout: "paxos_min_wait": "0.050000", 2023-10-01T10:40:22.330 INFO:teuthology.orchestra.run.smithi078.stdout: "paxos_propose_interval": "1.000000", 2023-10-01T10:40:22.330 INFO:teuthology.orchestra.run.smithi078.stdout: "paxos_service_trim_max": "500", 2023-10-01T10:40:22.330 INFO:teuthology.orchestra.run.smithi078.stdout: "paxos_service_trim_max_multiplier": "20", 2023-10-01T10:40:22.330 INFO:teuthology.orchestra.run.smithi078.stdout: "paxos_service_trim_min": "250", 2023-10-01T10:40:22.330 INFO:teuthology.orchestra.run.smithi078.stdout: "paxos_stash_full_interval": "25", 2023-10-01T10:40:22.330 INFO:teuthology.orchestra.run.smithi078.stdout: "paxos_trim_max": "500", 2023-10-01T10:40:22.330 INFO:teuthology.orchestra.run.smithi078.stdout: "paxos_trim_min": "250", 2023-10-01T10:40:22.330 INFO:teuthology.orchestra.run.smithi078.stdout: "perf": "true", 2023-10-01T10:40:22.330 INFO:teuthology.orchestra.run.smithi078.stdout: "pid_file": "", 2023-10-01T10:40:22.331 INFO:teuthology.orchestra.run.smithi078.stdout: "plugin_crypto_accelerator": "crypto_isal", 2023-10-01T10:40:22.331 INFO:teuthology.orchestra.run.smithi078.stdout: "plugin_dir": "/usr/lib64/ceph", 2023-10-01T10:40:22.331 INFO:teuthology.orchestra.run.smithi078.stdout: "public_addr": "-", 2023-10-01T10:40:22.331 INFO:teuthology.orchestra.run.smithi078.stdout: "public_addrv": "", 2023-10-01T10:40:22.331 INFO:teuthology.orchestra.run.smithi078.stdout: "public_bind_addr": "-", 2023-10-01T10:40:22.331 INFO:teuthology.orchestra.run.smithi078.stdout: "public_network": "172.21.15.254/32,172.21.0.0/20", 2023-10-01T10:40:22.331 INFO:teuthology.orchestra.run.smithi078.stdout: "public_network_interface": "", 2023-10-01T10:40:22.331 INFO:teuthology.orchestra.run.smithi078.stdout: "qat_compressor_enabled": "false", 2023-10-01T10:40:22.331 INFO:teuthology.orchestra.run.smithi078.stdout: "qat_compressor_session_max_number": "256", 2023-10-01T10:40:22.331 INFO:teuthology.orchestra.run.smithi078.stdout: "rados_mon_op_timeout": "0", 2023-10-01T10:40:22.331 INFO:teuthology.orchestra.run.smithi078.stdout: "rados_osd_op_timeout": "0", 2023-10-01T10:40:22.331 INFO:teuthology.orchestra.run.smithi078.stdout: "rados_tracing": "false", 2023-10-01T10:40:22.332 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_atime_update_interval": "60", 2023-10-01T10:40:22.332 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_auto_exclusive_lock_until_manual_request": "true", 2023-10-01T10:40:22.332 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_balance_parent_reads": "false", 2023-10-01T10:40:22.332 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_balance_snap_reads": "false", 2023-10-01T10:40:22.332 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_blkin_trace_all": "false", 2023-10-01T10:40:22.332 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_blocklist_expire_seconds": "0", 2023-10-01T10:40:22.332 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_blocklist_on_break_lock": "true", 2023-10-01T10:40:22.332 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_cache": "true", 2023-10-01T10:40:22.332 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_cache_block_writes_upfront": "false", 2023-10-01T10:40:22.332 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_cache_max_dirty": "25165824", 2023-10-01T10:40:22.332 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_cache_max_dirty_age": "1.000000", 2023-10-01T10:40:22.332 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_cache_max_dirty_object": "0", 2023-10-01T10:40:22.333 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_cache_policy": "writearound", 2023-10-01T10:40:22.333 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_cache_size": "33554432", 2023-10-01T10:40:22.333 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_cache_target_dirty": "16777216", 2023-10-01T10:40:22.333 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_cache_writethrough_until_flush": "true", 2023-10-01T10:40:22.333 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_clone_copy_on_read": "false", 2023-10-01T10:40:22.333 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_compression_hint": "none", 2023-10-01T10:40:22.333 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_concurrent_management_ops": "10", 2023-10-01T10:40:22.333 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_config_pool_override_update_timestamp": "0", 2023-10-01T10:40:22.333 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_default_clone_format": "auto", 2023-10-01T10:40:22.333 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_default_data_pool": "", 2023-10-01T10:40:22.333 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_default_features": "61", 2023-10-01T10:40:22.334 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_default_format": "2", 2023-10-01T10:40:22.334 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_default_map_options": "", 2023-10-01T10:40:22.334 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_default_order": "22", 2023-10-01T10:40:22.334 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_default_pool": "rbd", 2023-10-01T10:40:22.334 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_default_snapshot_quiesce_mode": "required", 2023-10-01T10:40:22.334 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_default_stripe_count": "0", 2023-10-01T10:40:22.334 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_default_stripe_unit": "0", 2023-10-01T10:40:22.334 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_disable_zero_copy_writes": "true", 2023-10-01T10:40:22.334 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_discard_granularity_bytes": "65536", 2023-10-01T10:40:22.334 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_discard_on_zeroed_write_same": "true", 2023-10-01T10:40:22.334 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_enable_alloc_hint": "true", 2023-10-01T10:40:22.334 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_invalidate_object_map_on_timeout": "true", 2023-10-01T10:40:22.335 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_io_scheduler": "simple", 2023-10-01T10:40:22.335 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_io_scheduler_simple_max_delay": "0", 2023-10-01T10:40:22.335 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_journal_commit_age": "5.000000", 2023-10-01T10:40:22.335 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_journal_max_concurrent_object_sets": "0", 2023-10-01T10:40:22.335 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_journal_max_payload_bytes": "16384", 2023-10-01T10:40:22.335 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_journal_object_flush_age": "0.000000", 2023-10-01T10:40:22.335 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_journal_object_flush_bytes": "1048576", 2023-10-01T10:40:22.335 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_journal_object_flush_interval": "0", 2023-10-01T10:40:22.335 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_journal_object_max_in_flight_appends": "0", 2023-10-01T10:40:22.335 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_journal_object_writethrough_until_flush": "true", 2023-10-01T10:40:22.335 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_journal_order": "24", 2023-10-01T10:40:22.335 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_journal_pool": "", 2023-10-01T10:40:22.335 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_journal_splay_width": "4", 2023-10-01T10:40:22.336 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_localize_parent_reads": "false", 2023-10-01T10:40:22.336 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_localize_snap_reads": "false", 2023-10-01T10:40:22.336 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_mirror_concurrent_image_deletions": "1", 2023-10-01T10:40:22.336 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_mirror_concurrent_image_syncs": "5", 2023-10-01T10:40:22.336 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_mirror_delete_retry_interval": "30.000000", 2023-10-01T10:40:22.336 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_mirror_image_perf_stats_prio": "5", 2023-10-01T10:40:22.336 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_mirror_image_policy_migration_throttle": "300", 2023-10-01T10:40:22.336 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_mirror_image_policy_rebalance_timeout": "0.000000", 2023-10-01T10:40:22.336 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_mirror_image_policy_type": "simple", 2023-10-01T10:40:22.336 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_mirror_image_policy_update_throttle_interval": "1.000000", 2023-10-01T10:40:22.336 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_mirror_image_state_check_interval": "30", 2023-10-01T10:40:22.336 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_mirror_journal_commit_age": "5.000000", 2023-10-01T10:40:22.337 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_mirror_journal_poll_age": "5.000000", 2023-10-01T10:40:22.337 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_mirror_leader_heartbeat_interval": "5", 2023-10-01T10:40:22.337 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_mirror_leader_max_acquire_attempts_before_break": "3", 2023-10-01T10:40:22.337 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_mirror_leader_max_missed_heartbeats": "2", 2023-10-01T10:40:22.337 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_mirror_memory_autotune": "true", 2023-10-01T10:40:22.337 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_mirror_memory_base": "805306368", 2023-10-01T10:40:22.337 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_mirror_memory_cache_autotune_interval": "30.000000", 2023-10-01T10:40:22.337 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_mirror_memory_cache_min": "134217728", 2023-10-01T10:40:22.337 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_mirror_memory_cache_resize_interval": "5.000000", 2023-10-01T10:40:22.337 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_mirror_memory_expected_fragmentation": "0.150000", 2023-10-01T10:40:22.337 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_mirror_memory_target": "4294967296", 2023-10-01T10:40:22.337 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_mirror_perf_stats_prio": "5", 2023-10-01T10:40:22.337 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_mirror_pool_replayers_refresh_interval": "30", 2023-10-01T10:40:22.338 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_mirror_sync_point_update_age": "30.000000", 2023-10-01T10:40:22.338 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_mirroring_delete_delay": "0", 2023-10-01T10:40:22.338 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_mirroring_max_mirroring_snapshots": "5", 2023-10-01T10:40:22.338 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_mirroring_replay_delay": "0", 2023-10-01T10:40:22.338 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_mirroring_resync_after_disconnect": "false", 2023-10-01T10:40:22.338 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_move_parent_to_trash_on_remove": "false", 2023-10-01T10:40:22.338 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_move_to_trash_on_remove": "false", 2023-10-01T10:40:22.338 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_move_to_trash_on_remove_expire_seconds": "0", 2023-10-01T10:40:22.338 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_mtime_update_interval": "60", 2023-10-01T10:40:22.338 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_non_blocking_aio": "true", 2023-10-01T10:40:22.338 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_op_thread_timeout": "60", 2023-10-01T10:40:22.338 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_op_threads": "1", 2023-10-01T10:40:22.339 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_parent_cache_enabled": "false", 2023-10-01T10:40:22.339 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_persistent_cache_mode": "disabled", 2023-10-01T10:40:22.339 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_persistent_cache_path": "/tmp", 2023-10-01T10:40:22.339 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_persistent_cache_size": "1073741824", 2023-10-01T10:40:22.339 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_plugins": "", 2023-10-01T10:40:22.339 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_qos_bps_burst": "0", 2023-10-01T10:40:22.339 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_qos_bps_burst_seconds": "1", 2023-10-01T10:40:22.339 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_qos_bps_limit": "0", 2023-10-01T10:40:22.339 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_qos_exclude_ops": "0", 2023-10-01T10:40:22.339 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_qos_iops_burst": "0", 2023-10-01T10:40:22.339 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_qos_iops_burst_seconds": "1", 2023-10-01T10:40:22.339 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_qos_iops_limit": "0", 2023-10-01T10:40:22.340 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_qos_read_bps_burst": "0", 2023-10-01T10:40:22.340 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_qos_read_bps_burst_seconds": "1", 2023-10-01T10:40:22.340 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_qos_read_bps_limit": "0", 2023-10-01T10:40:22.340 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_qos_read_iops_burst": "0", 2023-10-01T10:40:22.340 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_qos_read_iops_burst_seconds": "1", 2023-10-01T10:40:22.340 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_qos_read_iops_limit": "0", 2023-10-01T10:40:22.340 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_qos_schedule_tick_min": "50", 2023-10-01T10:40:22.340 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_qos_write_bps_burst": "0", 2023-10-01T10:40:22.340 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_qos_write_bps_burst_seconds": "1", 2023-10-01T10:40:22.340 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_qos_write_bps_limit": "0", 2023-10-01T10:40:22.340 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_qos_write_iops_burst": "0", 2023-10-01T10:40:22.340 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_qos_write_iops_burst_seconds": "1", 2023-10-01T10:40:22.341 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_qos_write_iops_limit": "0", 2023-10-01T10:40:22.341 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_quiesce_notification_attempts": "10", 2023-10-01T10:40:22.341 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_read_from_replica_policy": "default", 2023-10-01T10:40:22.341 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_readahead_disable_after_bytes": "52428800", 2023-10-01T10:40:22.341 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_readahead_max_bytes": "524288", 2023-10-01T10:40:22.341 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_readahead_trigger_requests": "10", 2023-10-01T10:40:22.341 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_request_timed_out_seconds": "30", 2023-10-01T10:40:22.341 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_skip_partial_discard": "true", 2023-10-01T10:40:22.341 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_sparse_read_threshold_bytes": "65536", 2023-10-01T10:40:22.341 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_tracing": "false", 2023-10-01T10:40:22.341 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_validate_names": "true", 2023-10-01T10:40:22.342 INFO:teuthology.orchestra.run.smithi078.stdout: "rbd_validate_pool": "true", 2023-10-01T10:40:22.342 INFO:teuthology.orchestra.run.smithi078.stdout: "restapi_base_url": "", 2023-10-01T10:40:22.342 INFO:teuthology.orchestra.run.smithi078.stdout: "restapi_log_level": "", 2023-10-01T10:40:22.342 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_acl_grants_max_num": "100", 2023-10-01T10:40:22.342 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_admin_entry": "admin", 2023-10-01T10:40:22.342 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_allow_notification_secrets_in_cleartext": "false", 2023-10-01T10:40:22.342 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_backend_store": "rados", 2023-10-01T10:40:22.342 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_barbican_url": "", 2023-10-01T10:40:22.342 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_beast_enable_async": "true", 2023-10-01T10:40:22.342 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_bucket_default_quota_max_objects": "-1", 2023-10-01T10:40:22.342 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_bucket_default_quota_max_size": "-1", 2023-10-01T10:40:22.342 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_bucket_index_max_aio": "128", 2023-10-01T10:40:22.343 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_bucket_index_transaction_instrumentation": "false", 2023-10-01T10:40:22.343 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_bucket_quota_cache_size": "10000", 2023-10-01T10:40:22.343 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_bucket_quota_ttl": "600", 2023-10-01T10:40:22.343 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_bucket_sync_spawn_window": "20", 2023-10-01T10:40:22.343 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_cache_enabled": "true", 2023-10-01T10:40:22.343 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_cache_expiry_interval": "900", 2023-10-01T10:40:22.343 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_cache_lru_size": "10000", 2023-10-01T10:40:22.343 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_config_store": "rados", 2023-10-01T10:40:22.343 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_content_length_compat": "false", 2023-10-01T10:40:22.343 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_copy_obj_progress": "true", 2023-10-01T10:40:22.343 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_copy_obj_progress_every_bytes": "1048576", 2023-10-01T10:40:22.344 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_cors_rules_max_num": "100", 2023-10-01T10:40:22.344 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_cross_domain_policy": "", 2023-10-01T10:40:22.344 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_crypt_default_encryption_key": "", 2023-10-01T10:40:22.344 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_crypt_kmip_addr": "", 2023-10-01T10:40:22.344 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_crypt_kmip_ca_path": "", 2023-10-01T10:40:22.344 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_crypt_kmip_client_cert": "", 2023-10-01T10:40:22.344 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_crypt_kmip_client_key": "", 2023-10-01T10:40:22.344 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_crypt_kmip_kms_key_template": "", 2023-10-01T10:40:22.344 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_crypt_kmip_password": "", 2023-10-01T10:40:22.344 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_crypt_kmip_s3_key_template": "$keyid", 2023-10-01T10:40:22.344 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_crypt_kmip_username": "", 2023-10-01T10:40:22.344 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_crypt_require_ssl": "true", 2023-10-01T10:40:22.345 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_crypt_s3_kms_backend": "barbican", 2023-10-01T10:40:22.345 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_crypt_s3_kms_encryption_keys": "", 2023-10-01T10:40:22.345 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_crypt_sse_s3_backend": "vault", 2023-10-01T10:40:22.345 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_crypt_sse_s3_key_template": "%bucket_id", 2023-10-01T10:40:22.345 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_crypt_sse_s3_vault_addr": "", 2023-10-01T10:40:22.345 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_crypt_sse_s3_vault_auth": "token", 2023-10-01T10:40:22.345 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_crypt_sse_s3_vault_namespace": "", 2023-10-01T10:40:22.345 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_crypt_sse_s3_vault_prefix": "", 2023-10-01T10:40:22.345 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_crypt_sse_s3_vault_secret_engine": "transit", 2023-10-01T10:40:22.345 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_crypt_sse_s3_vault_ssl_cacert": "", 2023-10-01T10:40:22.345 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_crypt_sse_s3_vault_ssl_clientcert": "", 2023-10-01T10:40:22.345 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_crypt_sse_s3_vault_ssl_clientkey": "", 2023-10-01T10:40:22.346 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_crypt_sse_s3_vault_token_file": "", 2023-10-01T10:40:22.346 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_crypt_sse_s3_vault_verify_ssl": "true", 2023-10-01T10:40:22.346 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_crypt_suppress_logs": "true", 2023-10-01T10:40:22.346 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_crypt_vault_addr": "", 2023-10-01T10:40:22.346 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_crypt_vault_auth": "token", 2023-10-01T10:40:22.346 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_crypt_vault_namespace": "", 2023-10-01T10:40:22.346 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_crypt_vault_prefix": "", 2023-10-01T10:40:22.346 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_crypt_vault_secret_engine": "transit", 2023-10-01T10:40:22.346 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_crypt_vault_ssl_cacert": "", 2023-10-01T10:40:22.346 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_crypt_vault_ssl_clientcert": "", 2023-10-01T10:40:22.346 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_crypt_vault_ssl_clientkey": "", 2023-10-01T10:40:22.347 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_crypt_vault_token_file": "", 2023-10-01T10:40:22.347 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_crypt_vault_verify_ssl": "true", 2023-10-01T10:40:22.347 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_curl_buffersize": "524288", 2023-10-01T10:40:22.347 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_curl_low_speed_limit": "1024", 2023-10-01T10:40:22.347 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_curl_low_speed_time": "300", 2023-10-01T10:40:22.347 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_curl_tcp_keepalive": "0", 2023-10-01T10:40:22.347 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_curl_wait_timeout_ms": "1000", 2023-10-01T10:40:22.347 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_d3n_l1_datacache_persistent_path": "/tmp/rgw_datacache/", 2023-10-01T10:40:22.347 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_d3n_l1_datacache_size": "1073741824", 2023-10-01T10:40:22.347 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_d3n_l1_evict_cache_on_start": "true", 2023-10-01T10:40:22.347 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_d3n_l1_eviction_policy": "lru", 2023-10-01T10:40:22.347 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_d3n_l1_fadvise": "4", 2023-10-01T10:40:22.348 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_d3n_l1_local_datacache_enabled": "false", 2023-10-01T10:40:22.348 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_d3n_libaio_aio_num": "64", 2023-10-01T10:40:22.348 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_d3n_libaio_aio_threads": "20", 2023-10-01T10:40:22.348 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_data": "/var/lib/ceph/radosgw/ceph-a", 2023-10-01T10:40:22.348 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_data_log_changes_size": "1000", 2023-10-01T10:40:22.348 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_data_log_num_shards": "128", 2023-10-01T10:40:22.348 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_data_log_obj_prefix": "data_log", 2023-10-01T10:40:22.348 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_data_log_window": "30", 2023-10-01T10:40:22.348 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_data_notify_interval_msec": "0", 2023-10-01T10:40:22.348 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_data_sync_poll_interval": "20", 2023-10-01T10:40:22.348 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_data_sync_spawn_window": "20", 2023-10-01T10:40:22.348 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_debug_inject_olh_cancel_modification_err": "false", 2023-10-01T10:40:22.349 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_debug_inject_set_olh_err": "0", 2023-10-01T10:40:22.349 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_default_data_log_backing": "fifo", 2023-10-01T10:40:22.349 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_default_realm_info_oid": "default.realm", 2023-10-01T10:40:22.349 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_default_region_info_oid": "default.region", 2023-10-01T10:40:22.349 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_default_zone_info_oid": "default.zone", 2023-10-01T10:40:22.349 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_default_zonegroup_info_oid": "default.zonegroup", 2023-10-01T10:40:22.349 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_defer_to_bucket_acls": "", 2023-10-01T10:40:22.349 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_delete_multi_obj_max_num": "1000", 2023-10-01T10:40:22.349 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_dmclock_admin_lim": "0.000000", 2023-10-01T10:40:22.349 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_dmclock_admin_res": "100.000000", 2023-10-01T10:40:22.349 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_dmclock_admin_wgt": "100.000000", 2023-10-01T10:40:22.350 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_dmclock_auth_lim": "0.000000", 2023-10-01T10:40:22.350 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_dmclock_auth_res": "200.000000", 2023-10-01T10:40:22.350 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_dmclock_auth_wgt": "100.000000", 2023-10-01T10:40:22.350 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_dmclock_data_lim": "0.000000", 2023-10-01T10:40:22.350 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_dmclock_data_res": "500.000000", 2023-10-01T10:40:22.350 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_dmclock_data_wgt": "500.000000", 2023-10-01T10:40:22.350 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_dmclock_metadata_lim": "0.000000", 2023-10-01T10:40:22.350 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_dmclock_metadata_res": "500.000000", 2023-10-01T10:40:22.350 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_dmclock_metadata_wgt": "500.000000", 2023-10-01T10:40:22.350 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_dns_name": "", 2023-10-01T10:40:22.350 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_dns_s3website_name": "", 2023-10-01T10:40:22.350 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_dynamic_resharding": "true", 2023-10-01T10:40:22.351 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_enable_apis": "s3, s3website, swift, swift_auth, admin, sts, iam, notifications", 2023-10-01T10:40:22.351 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_enable_gc_threads": "true", 2023-10-01T10:40:22.351 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_enable_lc_threads": "true", 2023-10-01T10:40:22.351 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_enable_ops_log": "false", 2023-10-01T10:40:22.351 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_enable_quota_threads": "true", 2023-10-01T10:40:22.351 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_enable_static_website": "false", 2023-10-01T10:40:22.351 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_enable_usage_log": "false", 2023-10-01T10:40:22.351 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_enforce_swift_acls": "true", 2023-10-01T10:40:22.351 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_exit_timeout_secs": "120", 2023-10-01T10:40:22.351 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_expose_bucket": "false", 2023-10-01T10:40:22.351 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_extended_http_attrs": "", 2023-10-01T10:40:22.352 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_filter": "none", 2023-10-01T10:40:22.352 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_frontend_defaults": "beast ssl_certificate=config://rgw/cert/$realm/$zone.crt ssl_private_key=config://rgw/cert/$realm/$zone.key", 2023-10-01T10:40:22.352 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_frontends": "beast port=7480", 2023-10-01T10:40:22.352 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_gc_max_concurrent_io": "10", 2023-10-01T10:40:22.352 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_gc_max_deferred": "50", 2023-10-01T10:40:22.352 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_gc_max_deferred_entries_size": "3072", 2023-10-01T10:40:22.352 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_gc_max_objs": "32", 2023-10-01T10:40:22.352 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_gc_max_queue_size": "134213632", 2023-10-01T10:40:22.352 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_gc_max_trim_chunk": "16", 2023-10-01T10:40:22.352 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_gc_obj_min_wait": "7200", 2023-10-01T10:40:22.352 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_gc_processor_max_time": "3600", 2023-10-01T10:40:22.352 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_gc_processor_period": "3600", 2023-10-01T10:40:22.352 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_get_obj_max_req_size": "4194304", 2023-10-01T10:40:22.353 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_get_obj_window_size": "16777216", 2023-10-01T10:40:22.353 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_healthcheck_disabling_path": "", 2023-10-01T10:40:22.353 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_ignore_get_invalid_range": "false", 2023-10-01T10:40:22.353 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_init_timeout": "300", 2023-10-01T10:40:22.353 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_inject_notify_timeout_probability": "0.000000", 2023-10-01T10:40:22.353 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_json_config": "/var/lib/ceph/radosgw/config.json", 2023-10-01T10:40:22.353 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_keystone_accepted_admin_roles": "", 2023-10-01T10:40:22.353 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_keystone_accepted_roles": "Member, admin", 2023-10-01T10:40:22.353 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_keystone_admin_domain": "", 2023-10-01T10:40:22.353 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_keystone_admin_password": "", 2023-10-01T10:40:22.353 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_keystone_admin_password_path": "", 2023-10-01T10:40:22.354 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_keystone_admin_project": "", 2023-10-01T10:40:22.354 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_keystone_admin_tenant": "", 2023-10-01T10:40:22.354 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_keystone_admin_token": "", 2023-10-01T10:40:22.354 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_keystone_admin_token_path": "", 2023-10-01T10:40:22.354 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_keystone_admin_user": "", 2023-10-01T10:40:22.354 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_keystone_api_version": "2", 2023-10-01T10:40:22.354 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_keystone_barbican_domain": "", 2023-10-01T10:40:22.354 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_keystone_barbican_password": "", 2023-10-01T10:40:22.354 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_keystone_barbican_project": "", 2023-10-01T10:40:22.354 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_keystone_barbican_tenant": "", 2023-10-01T10:40:22.354 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_keystone_barbican_user": "", 2023-10-01T10:40:22.355 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_keystone_expired_token_cache_expiration": "3600", 2023-10-01T10:40:22.355 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_keystone_implicit_tenants": "false", 2023-10-01T10:40:22.355 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_keystone_service_token_accepted_roles": "admin", 2023-10-01T10:40:22.355 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_keystone_service_token_enabled": "false", 2023-10-01T10:40:22.355 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_keystone_token_cache_size": "10000", 2023-10-01T10:40:22.355 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_keystone_url": "", 2023-10-01T10:40:22.355 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_keystone_verify_ssl": "true", 2023-10-01T10:40:22.355 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_lc_debug_interval": "-1", 2023-10-01T10:40:22.355 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_lc_lock_max_time": "90", 2023-10-01T10:40:22.355 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_lc_max_objs": "32", 2023-10-01T10:40:22.355 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_lc_max_rules": "1000", 2023-10-01T10:40:22.355 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_lc_max_worker": "3", 2023-10-01T10:40:22.356 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_lc_max_wp_worker": "3", 2023-10-01T10:40:22.356 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_lc_thread_delay": "0", 2023-10-01T10:40:22.356 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_ldap_binddn": "uid=admin,cn=users,dc=example,dc=com", 2023-10-01T10:40:22.356 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_ldap_dnattr": "uid", 2023-10-01T10:40:22.356 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_ldap_searchdn": "cn=users,cn=accounts,dc=example,dc=com", 2023-10-01T10:40:22.356 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_ldap_searchfilter": "", 2023-10-01T10:40:22.356 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_ldap_secret": "/etc/openldap/secret", 2023-10-01T10:40:22.356 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_ldap_uri": "ldaps://", 2023-10-01T10:40:22.356 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_lifecycle_work_time": "00:00-06:00", 2023-10-01T10:40:22.356 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_list_bucket_min_readahead": "1000", 2023-10-01T10:40:22.356 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_list_buckets_max_chunk": "1000", 2023-10-01T10:40:22.356 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_log_http_headers": "", 2023-10-01T10:40:22.356 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_log_nonexistent_bucket": "false", 2023-10-01T10:40:22.357 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_log_object_name": "%Y-%m-%d-%H-%i-%n", 2023-10-01T10:40:22.357 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_log_object_name_utc": "false", 2023-10-01T10:40:22.357 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_luarocks_location": "/tmp/luarocks", 2023-10-01T10:40:22.357 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_max_attr_name_len": "0", 2023-10-01T10:40:22.357 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_max_attr_size": "0", 2023-10-01T10:40:22.357 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_max_attrs_num_in_req": "0", 2023-10-01T10:40:22.357 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_max_chunk_size": "4194304", 2023-10-01T10:40:22.357 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_max_concurrent_requests": "1024", 2023-10-01T10:40:22.357 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_max_copy_obj_concurrent_io": "10", 2023-10-01T10:40:22.357 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_max_dynamic_shards": "1999", 2023-10-01T10:40:22.357 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_max_listing_results": "1000", 2023-10-01T10:40:22.357 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_max_notify_retries": "10", 2023-10-01T10:40:22.358 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_max_objs_per_shard": "100000", 2023-10-01T10:40:22.358 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_max_put_param_size": "1048576", 2023-10-01T10:40:22.358 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_max_put_size": "5368709120", 2023-10-01T10:40:22.358 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_max_slo_entries": "1000", 2023-10-01T10:40:22.358 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_md_log_max_shards": "64", 2023-10-01T10:40:22.358 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_md_notify_interval_msec": "200", 2023-10-01T10:40:22.358 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_meta_sync_poll_interval": "20", 2023-10-01T10:40:22.358 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_meta_sync_spawn_window": "20", 2023-10-01T10:40:22.358 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_mime_types_file": "/etc/mime.types", 2023-10-01T10:40:22.358 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_mp_lock_max_time": "600", 2023-10-01T10:40:22.358 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_multi_obj_del_max_aio": "16", 2023-10-01T10:40:22.359 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_multipart_min_part_size": "5242880", 2023-10-01T10:40:22.359 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_multipart_part_upload_limit": "10000", 2023-10-01T10:40:22.359 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_nfs_fhcache_partitions": "3", 2023-10-01T10:40:22.359 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_nfs_fhcache_size": "2017", 2023-10-01T10:40:22.359 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_nfs_frontends": "rgw-nfs", 2023-10-01T10:40:22.359 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_nfs_lru_lane_hiwat": "911", 2023-10-01T10:40:22.359 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_nfs_lru_lanes": "5", 2023-10-01T10:40:22.359 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_nfs_max_gc": "300", 2023-10-01T10:40:22.359 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_nfs_namespace_expire_secs": "300", 2023-10-01T10:40:22.359 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_nfs_run_gc_threads": "false", 2023-10-01T10:40:22.359 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_nfs_run_lc_threads": "false", 2023-10-01T10:40:22.360 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_nfs_run_quota_threads": "false", 2023-10-01T10:40:22.360 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_nfs_run_sync_thread": "false", 2023-10-01T10:40:22.360 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_nfs_s3_fast_attrs": "false", 2023-10-01T10:40:22.360 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_nfs_write_completion_interval_s": "10", 2023-10-01T10:40:22.360 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_num_async_rados_threads": "32", 2023-10-01T10:40:22.360 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_num_control_oids": "8", 2023-10-01T10:40:22.360 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_numa_node": "-1", 2023-10-01T10:40:22.360 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_obj_stripe_size": "4194304", 2023-10-01T10:40:22.360 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_obj_tombstone_cache_size": "1000", 2023-10-01T10:40:22.360 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_objexp_chunk_size": "100", 2023-10-01T10:40:22.360 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_objexp_gc_interval": "600", 2023-10-01T10:40:22.360 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_objexp_hints_num_shards": "127", 2023-10-01T10:40:22.361 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_olh_pending_timeout_sec": "3600", 2023-10-01T10:40:22.361 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_op_thread_suicide_timeout": "0", 2023-10-01T10:40:22.361 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_op_thread_timeout": "600", 2023-10-01T10:40:22.361 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_op_tracing": "false", 2023-10-01T10:40:22.361 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_opa_token": "", 2023-10-01T10:40:22.361 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_opa_url": "", 2023-10-01T10:40:22.361 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_opa_verify_ssl": "true", 2023-10-01T10:40:22.361 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_ops_log_data_backlog": "5242880", 2023-10-01T10:40:22.361 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_ops_log_file_path": "/var/log/ceph/ops-log-ceph-mon.a.log", 2023-10-01T10:40:22.361 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_ops_log_rados": "false", 2023-10-01T10:40:22.361 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_ops_log_socket_path": "", 2023-10-01T10:40:22.361 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_override_bucket_index_max_shards": "0", 2023-10-01T10:40:22.362 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_pending_bucket_index_op_expiration": "120", 2023-10-01T10:40:22.362 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_period_latest_epoch_info_oid": ".latest_epoch", 2023-10-01T10:40:22.362 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_period_push_interval": "2.000000", 2023-10-01T10:40:22.362 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_period_push_interval_max": "30.000000", 2023-10-01T10:40:22.362 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_period_root_pool": ".rgw.root", 2023-10-01T10:40:22.362 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_policy_reject_invalid_principals": "true", 2023-10-01T10:40:22.362 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_print_continue": "true", 2023-10-01T10:40:22.362 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_print_prohibited_content_length": "false", 2023-10-01T10:40:22.362 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_put_obj_max_window_size": "67108864", 2023-10-01T10:40:22.362 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_put_obj_min_window_size": "16777216", 2023-10-01T10:40:22.362 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_rados_pool_autoscale_bias": "4.000000", 2023-10-01T10:40:22.362 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_rados_pool_recovery_priority": "5", 2023-10-01T10:40:22.363 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_rados_tracing": "false", 2023-10-01T10:40:22.363 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_realm": "", 2023-10-01T10:40:22.363 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_realm_id": "", 2023-10-01T10:40:22.363 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_realm_root_pool": ".rgw.root", 2023-10-01T10:40:22.363 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_region": "", 2023-10-01T10:40:22.363 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_region_root_pool": ".rgw.root", 2023-10-01T10:40:22.363 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_relaxed_region_enforcement": "false", 2023-10-01T10:40:22.363 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_relaxed_s3_bucket_names": "false", 2023-10-01T10:40:22.363 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_remote_addr_param": "REMOTE_ADDR", 2023-10-01T10:40:22.363 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_request_uri": "", 2023-10-01T10:40:22.363 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_reshard_batch_size": "64", 2023-10-01T10:40:22.363 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_reshard_bucket_lock_duration": "360", 2023-10-01T10:40:22.364 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_reshard_max_aio": "128", 2023-10-01T10:40:22.364 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_reshard_num_logs": "16", 2023-10-01T10:40:22.364 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_reshard_thread_interval": "600", 2023-10-01T10:40:22.364 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_resolve_cname": "false", 2023-10-01T10:40:22.364 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_rest_getusage_op_compat": "false", 2023-10-01T10:40:22.364 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_run_sync_thread": "true", 2023-10-01T10:40:22.364 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_s3_auth_order": "sts, external, local", 2023-10-01T10:40:22.364 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_s3_auth_use_keystone": "false", 2023-10-01T10:40:22.364 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_s3_auth_use_ldap": "false", 2023-10-01T10:40:22.364 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_s3_auth_use_rados": "true", 2023-10-01T10:40:22.364 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_s3_auth_use_sts": "false", 2023-10-01T10:40:22.365 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_s3_client_max_sig_ver": "-1", 2023-10-01T10:40:22.365 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_s3_success_create_obj_status": "0", 2023-10-01T10:40:22.365 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_safe_max_objects_per_shard": "102400", 2023-10-01T10:40:22.365 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_scheduler_type": "throttler", 2023-10-01T10:40:22.365 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_script_uri": "", 2023-10-01T10:40:22.365 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_service_provider_name": "", 2023-10-01T10:40:22.365 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_shard_warning_threshold": "90.000000", 2023-10-01T10:40:22.365 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_sts_client_id": "", 2023-10-01T10:40:22.365 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_sts_client_secret": "", 2023-10-01T10:40:22.365 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_sts_entry": "sts", 2023-10-01T10:40:22.365 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_sts_key": "sts", 2023-10-01T10:40:22.365 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_sts_max_session_duration": "43200", 2023-10-01T10:40:22.366 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_sts_min_session_duration": "900", 2023-10-01T10:40:22.366 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_sts_token_introspection_url": "", 2023-10-01T10:40:22.366 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_swift_account_in_url": "false", 2023-10-01T10:40:22.366 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_swift_auth_entry": "auth", 2023-10-01T10:40:22.366 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_swift_auth_url": "", 2023-10-01T10:40:22.366 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_swift_custom_header": "", 2023-10-01T10:40:22.366 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_swift_enforce_content_length": "false", 2023-10-01T10:40:22.366 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_swift_need_stats": "true", 2023-10-01T10:40:22.366 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_swift_tenant_name": "", 2023-10-01T10:40:22.366 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_swift_token_expiration": "86400", 2023-10-01T10:40:22.366 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_swift_url": "", 2023-10-01T10:40:22.366 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_swift_url_prefix": "swift", 2023-10-01T10:40:22.367 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_swift_versioning_enabled": "false", 2023-10-01T10:40:22.367 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_sync_data_full_inject_err_probability": "0.000000", 2023-10-01T10:40:22.367 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_sync_data_inject_err_probability": "0.000000", 2023-10-01T10:40:22.367 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_sync_lease_period": "120", 2023-10-01T10:40:22.367 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_sync_log_trim_concurrent_buckets": "4", 2023-10-01T10:40:22.367 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_sync_log_trim_interval": "1200", 2023-10-01T10:40:22.367 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_sync_log_trim_max_buckets": "16", 2023-10-01T10:40:22.367 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_sync_log_trim_min_cold_buckets": "4", 2023-10-01T10:40:22.367 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_sync_meta_inject_err_probability": "0.000000", 2023-10-01T10:40:22.367 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_sync_obj_etag_verify": "false", 2023-10-01T10:40:22.367 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_sync_trace_history_size": "4096", 2023-10-01T10:40:22.367 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_sync_trace_per_node_log_size": "32", 2023-10-01T10:40:22.368 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_sync_trace_servicemap_update_interval": "10", 2023-10-01T10:40:22.368 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_thread_pool_size": "512", 2023-10-01T10:40:22.368 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_torrent_comment": "", 2023-10-01T10:40:22.368 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_torrent_createby": "", 2023-10-01T10:40:22.368 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_torrent_encoding": "", 2023-10-01T10:40:22.368 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_torrent_flag": "false", 2023-10-01T10:40:22.368 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_torrent_origin": "", 2023-10-01T10:40:22.368 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_torrent_sha_unit": "524288", 2023-10-01T10:40:22.368 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_torrent_tracker": "", 2023-10-01T10:40:22.368 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_trust_forwarded_https": "false", 2023-10-01T10:40:22.368 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_usage_log_flush_threshold": "1024", 2023-10-01T10:40:22.368 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_usage_log_tick_interval": "30", 2023-10-01T10:40:22.369 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_usage_max_shards": "32", 2023-10-01T10:40:22.369 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_usage_max_user_shards": "1", 2023-10-01T10:40:22.369 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_use_opa_authz": "false", 2023-10-01T10:40:22.369 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_user_default_quota_max_objects": "-1", 2023-10-01T10:40:22.369 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_user_default_quota_max_size": "-1", 2023-10-01T10:40:22.369 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_user_max_buckets": "1000", 2023-10-01T10:40:22.369 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_user_policies_max_num": "100", 2023-10-01T10:40:22.369 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_user_quota_bucket_sync_interval": "180", 2023-10-01T10:40:22.369 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_user_quota_sync_idle_users": "false", 2023-10-01T10:40:22.369 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_user_quota_sync_interval": "86400", 2023-10-01T10:40:22.369 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_user_quota_sync_wait_time": "86400", 2023-10-01T10:40:22.369 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_user_unique_email": "true", 2023-10-01T10:40:22.370 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_verify_ssl": "true", 2023-10-01T10:40:22.370 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_website_routing_rules_max_num": "50", 2023-10-01T10:40:22.370 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_zone": "", 2023-10-01T10:40:22.370 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_zone_id": "", 2023-10-01T10:40:22.370 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_zone_root_pool": ".rgw.root", 2023-10-01T10:40:22.370 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_zonegroup": "", 2023-10-01T10:40:22.370 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_zonegroup_id": "", 2023-10-01T10:40:22.370 INFO:teuthology.orchestra.run.smithi078.stdout: "rgw_zonegroup_root_pool": ".rgw.root", 2023-10-01T10:40:22.370 INFO:teuthology.orchestra.run.smithi078.stdout: "rgwlc_auto_session_clear": "true", 2023-10-01T10:40:22.370 INFO:teuthology.orchestra.run.smithi078.stdout: "rgwlc_skip_bucket_step": "false", 2023-10-01T10:40:22.370 INFO:teuthology.orchestra.run.smithi078.stdout: "rocksdb_block_size": "4096", 2023-10-01T10:40:22.371 INFO:teuthology.orchestra.run.smithi078.stdout: "rocksdb_bloom_bits_per_key": "20", 2023-10-01T10:40:22.371 INFO:teuthology.orchestra.run.smithi078.stdout: "rocksdb_cache_index_and_filter_blocks": "true", 2023-10-01T10:40:22.371 INFO:teuthology.orchestra.run.smithi078.stdout: "rocksdb_cache_index_and_filter_blocks_with_high_priority": "false", 2023-10-01T10:40:22.371 INFO:teuthology.orchestra.run.smithi078.stdout: "rocksdb_cache_row_ratio": "0.000000", 2023-10-01T10:40:22.371 INFO:teuthology.orchestra.run.smithi078.stdout: "rocksdb_cache_shard_bits": "4", 2023-10-01T10:40:22.371 INFO:teuthology.orchestra.run.smithi078.stdout: "rocksdb_cache_size": "536870912", 2023-10-01T10:40:22.371 INFO:teuthology.orchestra.run.smithi078.stdout: "rocksdb_cache_type": "binned_lru", 2023-10-01T10:40:22.371 INFO:teuthology.orchestra.run.smithi078.stdout: "rocksdb_cf_compact_on_deletion": "true", 2023-10-01T10:40:22.371 INFO:teuthology.orchestra.run.smithi078.stdout: "rocksdb_cf_compact_on_deletion_sliding_window": "32768", 2023-10-01T10:40:22.371 INFO:teuthology.orchestra.run.smithi078.stdout: "rocksdb_cf_compact_on_deletion_trigger": "16384", 2023-10-01T10:40:22.371 INFO:teuthology.orchestra.run.smithi078.stdout: "rocksdb_collect_compaction_stats": "false", 2023-10-01T10:40:22.371 INFO:teuthology.orchestra.run.smithi078.stdout: "rocksdb_collect_extended_stats": "false", 2023-10-01T10:40:22.372 INFO:teuthology.orchestra.run.smithi078.stdout: "rocksdb_collect_memory_stats": "false", 2023-10-01T10:40:22.372 INFO:teuthology.orchestra.run.smithi078.stdout: "rocksdb_delete_range_threshold": "1048576", 2023-10-01T10:40:22.372 INFO:teuthology.orchestra.run.smithi078.stdout: "rocksdb_index_type": "binary_search", 2023-10-01T10:40:22.372 INFO:teuthology.orchestra.run.smithi078.stdout: "rocksdb_log_to_ceph_log": "true", 2023-10-01T10:40:22.372 INFO:teuthology.orchestra.run.smithi078.stdout: "rocksdb_metadata_block_size": "4096", 2023-10-01T10:40:22.372 INFO:teuthology.orchestra.run.smithi078.stdout: "rocksdb_partition_filters": "false", 2023-10-01T10:40:22.372 INFO:teuthology.orchestra.run.smithi078.stdout: "rocksdb_perf": "false", 2023-10-01T10:40:22.372 INFO:teuthology.orchestra.run.smithi078.stdout: "rocksdb_pin_l0_filter_and_index_blocks_in_cache": "false", 2023-10-01T10:40:22.372 INFO:teuthology.orchestra.run.smithi078.stdout: "rotating_keys_bootstrap_timeout": "30", 2023-10-01T10:40:22.372 INFO:teuthology.orchestra.run.smithi078.stdout: "rotating_keys_renewal_timeout": "10", 2023-10-01T10:40:22.372 INFO:teuthology.orchestra.run.smithi078.stdout: "run_dir": "/var/run/ceph", 2023-10-01T10:40:22.372 INFO:teuthology.orchestra.run.smithi078.stdout: "seastore_block_create": "true", 2023-10-01T10:40:22.373 INFO:teuthology.orchestra.run.smithi078.stdout: "seastore_cache_lru_size": "67108864", 2023-10-01T10:40:22.373 INFO:teuthology.orchestra.run.smithi078.stdout: "seastore_cbjournal_size": "5368709120", 2023-10-01T10:40:22.373 INFO:teuthology.orchestra.run.smithi078.stdout: "seastore_default_max_object_size": "16777216", 2023-10-01T10:40:22.373 INFO:teuthology.orchestra.run.smithi078.stdout: "seastore_default_object_metadata_reservation": "16777216", 2023-10-01T10:40:22.373 INFO:teuthology.orchestra.run.smithi078.stdout: "seastore_device_size": "53687091200", 2023-10-01T10:40:22.373 INFO:teuthology.orchestra.run.smithi078.stdout: "seastore_journal_batch_capacity": "16", 2023-10-01T10:40:22.373 INFO:teuthology.orchestra.run.smithi078.stdout: "seastore_journal_batch_flush_size": "16777216", 2023-10-01T10:40:22.373 INFO:teuthology.orchestra.run.smithi078.stdout: "seastore_journal_batch_preferred_fullness": "0.950000", 2023-10-01T10:40:22.373 INFO:teuthology.orchestra.run.smithi078.stdout: "seastore_journal_iodepth_limit": "5", 2023-10-01T10:40:22.373 INFO:teuthology.orchestra.run.smithi078.stdout: "seastore_main_device_type": "SSD", 2023-10-01T10:40:22.373 INFO:teuthology.orchestra.run.smithi078.stdout: "seastore_max_concurrent_transactions": "8", 2023-10-01T10:40:22.374 INFO:teuthology.orchestra.run.smithi078.stdout: "seastore_multiple_tiers_default_evict_ratio": "0.600000", 2023-10-01T10:40:22.374 INFO:teuthology.orchestra.run.smithi078.stdout: "seastore_multiple_tiers_fast_evict_ratio": "0.700000", 2023-10-01T10:40:22.374 INFO:teuthology.orchestra.run.smithi078.stdout: "seastore_multiple_tiers_stop_evict_ratio": "0.500000", 2023-10-01T10:40:22.374 INFO:teuthology.orchestra.run.smithi078.stdout: "seastore_obj_data_write_amplification": "1.250000", 2023-10-01T10:40:22.374 INFO:teuthology.orchestra.run.smithi078.stdout: "seastore_segment_size": "67108864", 2023-10-01T10:40:22.374 INFO:teuthology.orchestra.run.smithi078.stdout: "set_keepcaps": "false", 2023-10-01T10:40:22.374 INFO:teuthology.orchestra.run.smithi078.stdout: "setgroup": "ceph", 2023-10-01T10:40:22.374 INFO:teuthology.orchestra.run.smithi078.stdout: "setuser": "ceph", 2023-10-01T10:40:22.374 INFO:teuthology.orchestra.run.smithi078.stdout: "setuser_match_path": "", 2023-10-01T10:40:22.374 INFO:teuthology.orchestra.run.smithi078.stdout: "target_max_misplaced_ratio": "0.050000", 2023-10-01T10:40:22.374 INFO:teuthology.orchestra.run.smithi078.stdout: "thp": "false", 2023-10-01T10:40:22.375 INFO:teuthology.orchestra.run.smithi078.stdout: "threadpool_default_timeout": "60", 2023-10-01T10:40:22.375 INFO:teuthology.orchestra.run.smithi078.stdout: "threadpool_empty_queue_max_wait": "2", 2023-10-01T10:40:22.375 INFO:teuthology.orchestra.run.smithi078.stdout: "throttler_perf_counter": "true" 2023-10-01T10:40:22.375 INFO:teuthology.orchestra.run.smithi078.stdout:} 2023-10-01T10:40:23.039 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=0 2023-10-01T10:40:23.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:23 smithi078 bash[75299]: cluster 2023-10-01T10:40:21.697720+0000 mgr.y (mgr.25072) 677 : cluster 0 pgmap v581: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:23.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:23 smithi078 bash[78274]: cluster 2023-10-01T10:40:21.697720+0000 mgr.y (mgr.25072) 677 : cluster 0 pgmap v581: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:23.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:23 smithi173 bash[58816]: cluster 2023-10-01T10:40:21.697720+0000 mgr.y (mgr.25072) 677 : cluster 0 pgmap v581: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:25.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:25 smithi078 bash[75299]: cluster 2023-10-01T10:40:23.698490+0000 mgr.y (mgr.25072) 678 : cluster 0 pgmap v582: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:25.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:25 smithi078 bash[78274]: cluster 2023-10-01T10:40:23.698490+0000 mgr.y (mgr.25072) 678 : cluster 0 pgmap v582: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:25.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:25 smithi173 bash[58816]: cluster 2023-10-01T10:40:23.698490+0000 mgr.y (mgr.25072) 678 : cluster 0 pgmap v582: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:26.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:26 smithi078 bash[75299]: audit 2023-10-01T10:40:25.193702+0000 mon.a (mon.0) 903 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:40:26.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:26 smithi078 bash[78274]: audit 2023-10-01T10:40:25.193702+0000 mon.a (mon.0) 903 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:40:26.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:26 smithi173 bash[58816]: audit 2023-10-01T10:40:25.193702+0000 mon.a (mon.0) 903 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:40:27.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:27 smithi078 bash[75299]: cluster 2023-10-01T10:40:25.699457+0000 mgr.y (mgr.25072) 679 : cluster 0 pgmap v583: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:27.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:27 smithi078 bash[78274]: cluster 2023-10-01T10:40:25.699457+0000 mgr.y (mgr.25072) 679 : cluster 0 pgmap v583: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:27.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:27 smithi173 bash[58816]: cluster 2023-10-01T10:40:25.699457+0000 mgr.y (mgr.25072) 679 : cluster 0 pgmap v583: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:27.816 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:40:29.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:29 smithi078 bash[75299]: cluster 2023-10-01T10:40:27.700167+0000 mgr.y (mgr.25072) 680 : cluster 0 pgmap v584: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:29.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:29 smithi078 bash[78274]: cluster 2023-10-01T10:40:27.700167+0000 mgr.y (mgr.25072) 680 : cluster 0 pgmap v584: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:29.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:29 smithi173 bash[58816]: cluster 2023-10-01T10:40:27.700167+0000 mgr.y (mgr.25072) 680 : cluster 0 pgmap v584: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:30.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:40:30 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:40:30] "GET /metrics HTTP/1.1" 200 33818 "" "Prometheus/2.33.4" 2023-10-01T10:40:30.903 INFO:teuthology.orchestra.run.smithi078.stdout:mon.a: {} 2023-10-01T10:40:30.925 INFO:teuthology.orchestra.run.smithi078.stdout:mon.b: {} 2023-10-01T10:40:30.944 INFO:teuthology.orchestra.run.smithi078.stdout:mon.c: {} 2023-10-01T10:40:30.946 INFO:teuthology.orchestra.run.smithi078.stderr:mon.a: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-10-01T10:40:30.946 INFO:teuthology.orchestra.run.smithi078.stderr:mon.b: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-10-01T10:40:30.946 INFO:teuthology.orchestra.run.smithi078.stderr:mon.c: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-10-01T10:40:31.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:31 smithi078 bash[75299]: cluster 2023-10-01T10:40:29.701088+0000 mgr.y (mgr.25072) 681 : cluster 0 pgmap v585: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:31.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:31 smithi078 bash[75299]: audit 2023-10-01T10:40:30.885677+0000 mon.a (mon.0) 904 : audit 0 from='client.? 172.21.15.78:0/992684517' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-01T10:40:31.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:31 smithi078 bash[78274]: cluster 2023-10-01T10:40:29.701088+0000 mgr.y (mgr.25072) 681 : cluster 0 pgmap v585: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:31.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:31 smithi078 bash[78274]: audit 2023-10-01T10:40:30.885677+0000 mon.a (mon.0) 904 : audit 0 from='client.? 172.21.15.78:0/992684517' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-01T10:40:31.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:31 smithi173 bash[58816]: cluster 2023-10-01T10:40:29.701088+0000 mgr.y (mgr.25072) 681 : cluster 0 pgmap v585: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:31.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:31 smithi173 bash[58816]: audit 2023-10-01T10:40:30.885677+0000 mon.a (mon.0) 904 : audit 0 from='client.? 172.21.15.78:0/992684517' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-01T10:40:31.864 INFO:tasks.thrashosds.thrasher:Unable to test ceph-objectstore-tool, not available on all OSD nodes 2023-10-01T10:40:31.865 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2023-10-01T10:40:31.865 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-01T10:40:31.865 INFO:tasks.radosbench:Beginning radosbench... 2023-10-01T10:40:31.866 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_4 2023-10-01T10:40:31.866 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_4 16 2023-10-01T10:40:31.866 INFO:tasks.thrashosds.thrasher:starting do_thrash 2023-10-01T10:40:31.866 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T10:40:31.866 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T10:40:31.867 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2023-10-01T10:40:31.867 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-01T10:40:31.867 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-01T10:40:31.867 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 config set filestore_inject_stall 3 2023-10-01T10:40:31.867 INFO:tasks.thrashosds.thrasher:starting do_sighup with a delay of 0.1 2023-10-01T10:40:31.868 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:40:31.868 INFO:tasks.thrashosds.thrasher:starting do_optrack_toggle with a delay of 2.0 2023-10-01T10:40:31.868 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T10:40:31.868 INFO:tasks.thrashosds.thrasher:starting do_dump_ops 2023-10-01T10:40:31.869 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-01T10:40:31.869 INFO:tasks.thrashosds.thrasher:starting do_noscrub_toggle with a delay of 2.0 2023-10-01T10:40:31.869 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T10:40:31.901 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:40:31.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:31.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:40:32.005 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:40:32.035 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:40:32.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:32.039 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:40:32.140 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:40:32.170 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:40:32.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:32.173 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:40:32.273 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:40:32.305 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:40:32.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:32.307 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:40:32.408 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:40:32.442 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:40:32.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:32.445 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:40:32.546 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:40:32.558 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:32 smithi078 bash[78274]: audit 2023-10-01T10:40:31.394272+0000 mon.a (mon.0) 905 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:40:32.558 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:32 smithi078 bash[78274]: audit 2023-10-01T10:40:31.402887+0000 mon.a (mon.0) 906 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:40:32.558 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:32 smithi078 bash[78274]: audit 2023-10-01T10:40:31.440814+0000 mon.a (mon.0) 907 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:40:32.559 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:32 smithi078 bash[78274]: audit 2023-10-01T10:40:31.449851+0000 mon.a (mon.0) 908 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:40:32.559 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:32 smithi078 bash[78274]: audit 2023-10-01T10:40:31.926475+0000 mon.a (mon.0) 909 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:40:32.559 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:32 smithi078 bash[78274]: audit 2023-10-01T10:40:31.927981+0000 mon.a (mon.0) 910 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:40:32.559 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:32 smithi078 bash[78274]: audit 2023-10-01T10:40:31.938180+0000 mon.a (mon.0) 911 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:40:32.559 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:32 smithi078 bash[75299]: audit 2023-10-01T10:40:31.394272+0000 mon.a (mon.0) 905 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:40:32.559 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:32 smithi078 bash[75299]: audit 2023-10-01T10:40:31.402887+0000 mon.a (mon.0) 906 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:40:32.559 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:32 smithi078 bash[75299]: audit 2023-10-01T10:40:31.440814+0000 mon.a (mon.0) 907 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:40:32.559 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:32 smithi078 bash[75299]: audit 2023-10-01T10:40:31.449851+0000 mon.a (mon.0) 908 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:40:32.559 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:32 smithi078 bash[75299]: audit 2023-10-01T10:40:31.926475+0000 mon.a (mon.0) 909 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:40:32.559 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:32 smithi078 bash[75299]: audit 2023-10-01T10:40:31.927981+0000 mon.a (mon.0) 910 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:40:32.560 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:32 smithi078 bash[75299]: audit 2023-10-01T10:40:31.938180+0000 mon.a (mon.0) 911 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:40:32.575 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:40:32.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:32.579 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:40:32.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:32 smithi173 bash[58816]: audit 2023-10-01T10:40:31.394272+0000 mon.a (mon.0) 905 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:40:32.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:32 smithi173 bash[58816]: audit 2023-10-01T10:40:31.402887+0000 mon.a (mon.0) 906 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:40:32.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:32 smithi173 bash[58816]: audit 2023-10-01T10:40:31.440814+0000 mon.a (mon.0) 907 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:40:32.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:32 smithi173 bash[58816]: audit 2023-10-01T10:40:31.449851+0000 mon.a (mon.0) 908 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:40:32.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:32 smithi173 bash[58816]: audit 2023-10-01T10:40:31.926475+0000 mon.a (mon.0) 909 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:40:32.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:32 smithi173 bash[58816]: audit 2023-10-01T10:40:31.927981+0000 mon.a (mon.0) 910 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:40:32.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:32 smithi173 bash[58816]: audit 2023-10-01T10:40:31.938180+0000 mon.a (mon.0) 911 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:40:32.680 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:40:32.715 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:40:32.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:32.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:40:32.819 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:32.850 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:32.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:32.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:40:32.954 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:40:32.989 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:40:32.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:32.992 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:40:33.093 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:40:33.120 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:40:33.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:33.124 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:40:33.224 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:40:33.257 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:40:33.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:33.260 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:40:33.361 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:40:33.393 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:40:33.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:33.396 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:40:33.497 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:40:33.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:33 smithi078 bash[75299]: cluster 2023-10-01T10:40:31.701909+0000 mgr.y (mgr.25072) 682 : cluster 0 pgmap v586: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:33.511 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:33 smithi078 bash[75299]: audit 2023-10-01T10:40:32.411462+0000 mon.a (mon.0) 912 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:40:33.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:33 smithi078 bash[78274]: cluster 2023-10-01T10:40:31.701909+0000 mgr.y (mgr.25072) 682 : cluster 0 pgmap v586: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:33.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:33 smithi078 bash[78274]: audit 2023-10-01T10:40:32.411462+0000 mon.a (mon.0) 912 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:40:33.529 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:40:33.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:33.532 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:40:33.633 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:33.644 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:33 smithi173 bash[58816]: cluster 2023-10-01T10:40:31.701909+0000 mgr.y (mgr.25072) 682 : cluster 0 pgmap v586: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:33.644 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:33 smithi173 bash[58816]: audit 2023-10-01T10:40:32.411462+0000 mon.a (mon.0) 912 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:40:33.663 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:33.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:33.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:40:33.766 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:33.799 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:33.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:33.802 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:40:33.902 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:40:33.933 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:40:33.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:33.936 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:40:34.037 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:40:34.068 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:40:34.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:34.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:40:34.173 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:34.204 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:34.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:34.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:40:34.308 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:34.341 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:34.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:34.344 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:40:34.445 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:40:34.479 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:40:34.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:34.481 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:40:34.582 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:34.612 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:34.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:34.616 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:40:34.716 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:40:34.749 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:40:34.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:34.752 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:40:34.853 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:34.878 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:34.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:34.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:40:34.982 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:40:35.011 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:40:35.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:35.014 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:40:35.115 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:40:35.143 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:40:35.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:35.145 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:40:35.246 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:40:35.276 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:40:35.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:35.278 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:40:35.379 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:40:35.413 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:40:35.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:35.415 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:40:35.516 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:40:35.530 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:35 smithi078 bash[78274]: cluster 2023-10-01T10:40:33.702704+0000 mgr.y (mgr.25072) 683 : cluster 0 pgmap v587: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:35.530 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:35 smithi078 bash[75299]: cluster 2023-10-01T10:40:33.702704+0000 mgr.y (mgr.25072) 683 : cluster 0 pgmap v587: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:35.550 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:40:35.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:35.552 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:40:35.653 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:40:35.684 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:40:35.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:35.687 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:40:35.788 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:40:35.801 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:35 smithi173 bash[58816]: cluster 2023-10-01T10:40:33.702704+0000 mgr.y (mgr.25072) 683 : cluster 0 pgmap v587: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:35.823 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:40:35.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:35.826 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:40:35.927 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:35.960 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:35.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:35.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:40:36.065 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:40:36.097 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:40:36.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:36.101 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:40:36.202 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:40:36.233 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:40:36.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:36.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:40:36.336 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:40:36.368 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:40:36.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:36.371 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:40:36.472 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:40:36.505 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:40:36.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:36.508 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:40:36.609 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:40:36.638 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:40:36.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:36.641 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:40:36.652 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T10:40:36.661 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:40:36.661 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:40:36.662 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:40:36.662 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:40:36.742 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:40:36.766 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:40:36.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:36.769 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:40:36.870 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:40:36.901 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:40:36.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:36.903 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:40:37.004 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:40:37.033 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:40:37.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:37.038 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:40:37.138 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:40:37.171 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:40:37.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:37.174 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:40:37.275 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:40:37.310 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:40:37.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:37.313 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:40:37.411 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:37 smithi173 bash[58816]: cluster 2023-10-01T10:40:35.703647+0000 mgr.y (mgr.25072) 684 : cluster 0 pgmap v588: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:37.413 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:40:37.438 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:40:37.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:37.441 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:40:37.542 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:40:37.573 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:40:37.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:37.576 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:40:37.677 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:40:37.888 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:37 smithi078 bash[75299]: cluster 2023-10-01T10:40:35.703647+0000 mgr.y (mgr.25072) 684 : cluster 0 pgmap v588: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:37.888 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:37 smithi078 bash[78274]: cluster 2023-10-01T10:40:35.703647+0000 mgr.y (mgr.25072) 684 : cluster 0 pgmap v588: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:37.916 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:40:37.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:37.919 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:40:38.020 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:38.043 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:38.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:38.045 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:40:38.146 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:40:38.173 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:40:38.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:38.176 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:40:38.277 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:40:38.303 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:40:38.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:38.307 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:40:38.407 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:38.438 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:38.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:38.441 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:40:38.541 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:40:38.568 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:40:38.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:38.571 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:40:38.672 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:40:38.698 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:40:38.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:38.702 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:40:38.802 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:40:38.829 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:40:38.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:38.832 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:40:38.933 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:40:38.967 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:40:38.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:38.970 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:40:39.071 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:40:39.101 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:40:39.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:39.103 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:40:39.204 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:40:39.230 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:40:39.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:39.232 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:40:39.333 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:39.363 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:39.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:39.367 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:40:39.467 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:40:39.479 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:39 smithi173 bash[58816]: cluster 2023-10-01T10:40:37.704346+0000 mgr.y (mgr.25072) 685 : cluster 0 pgmap v589: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:39.497 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:40:39.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:39.500 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:40:39.601 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:40:39.622 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:40:39.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:39.624 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:40:39.673 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:39 smithi078 bash[75299]: cluster 2023-10-01T10:40:37.704346+0000 mgr.y (mgr.25072) 685 : cluster 0 pgmap v589: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:39.674 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:39 smithi078 bash[78274]: cluster 2023-10-01T10:40:37.704346+0000 mgr.y (mgr.25072) 685 : cluster 0 pgmap v589: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:39.724 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:40:39.746 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:40:39.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:39.749 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:40:39.850 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:40:39.865 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-10-01T10:40:39.866 INFO:teuthology.orchestra.run.smithi173.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-01T10:40:39.866 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-10-01T10:40:39.877 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:40:39.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:39.880 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:40:39.981 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:40:40.006 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:40:40.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:40.009 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:40:40.109 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:40:40.136 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:40:40.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:40.139 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:40:40.239 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:40:40.270 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:40:40.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:40.273 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:40:40.373 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:40.404 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:40.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:40.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:40:40.508 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:40:40.538 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:40:40.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:40.541 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:40:40.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:40:40 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:40:40] "GET /metrics HTTP/1.1" 200 33822 "" "Prometheus/2.33.4" 2023-10-01T10:40:40.642 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:40:40.674 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:40:40.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:40.677 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:40:40.778 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:40:40.803 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:40:40.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:40.806 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:40:40.906 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:40:40.938 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:40:40.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:40.941 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:40:41.041 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:40:41.072 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:40:41.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:41.075 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:40:41.176 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:40:41.198 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:40:41.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:41.200 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:40:41.301 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:40:41.332 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:40:41.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:41.334 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:40:41.435 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:41.464 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:41.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:41.467 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:40:41.568 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:40:41.581 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:41 smithi173 bash[58816]: cluster 2023-10-01T10:40:39.704949+0000 mgr.y (mgr.25072) 686 : cluster 0 pgmap v590: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:41.601 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:40:41.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:41.604 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:40:41.705 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:41.735 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:41.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:41.739 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:40:41.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:41 smithi078 bash[78274]: cluster 2023-10-01T10:40:39.704949+0000 mgr.y (mgr.25072) 686 : cluster 0 pgmap v590: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:41.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:41 smithi078 bash[75299]: cluster 2023-10-01T10:40:39.704949+0000 mgr.y (mgr.25072) 686 : cluster 0 pgmap v590: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:41.839 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:41.872 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:41.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:41.875 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:40:41.976 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:40:42.009 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:40:42.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:42.012 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:40:42.113 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:40:42.147 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:40:42.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:42.150 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:40:42.250 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:40:42.277 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:40:42.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:42.280 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:40:42.381 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:40:42.413 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:40:42.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:42.416 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:40:42.517 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:40:42.541 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:40:42.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:42.543 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:40:42.644 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:40:42.671 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:40:42.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:42.674 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:40:42.775 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:40:42.802 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:40:42.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:42.804 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:40:42.905 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:40:42.937 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:40:42.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:42.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:40:43.041 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:43.071 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:43.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:43.074 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:40:43.175 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:40:43.207 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:40:43.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:43.210 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:40:43.311 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:40:43.343 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:40:43.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:43.347 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:40:43.447 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:40:43.470 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:40:43.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:43.474 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:40:43.574 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:40:43.599 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:40:43.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:43.602 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:40:43.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:43 smithi173 bash[58816]: cluster 2023-10-01T10:40:41.705822+0000 mgr.y (mgr.25072) 687 : cluster 0 pgmap v591: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:43.685 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:43 smithi078 bash[75299]: cluster 2023-10-01T10:40:41.705822+0000 mgr.y (mgr.25072) 687 : cluster 0 pgmap v591: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:43.685 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:43 smithi078 bash[78274]: cluster 2023-10-01T10:40:41.705822+0000 mgr.y (mgr.25072) 687 : cluster 0 pgmap v591: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:43.703 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:43.735 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:43.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:43.738 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:40:43.822 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T10:40:43.839 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:40:43.843 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T10:40:43.865 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T10:40:43.869 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:40:43.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:43.872 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:40:43.888 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T10:40:43.916 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T10:40:43.939 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T10:40:43.961 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T10:40:43.973 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:40:43.991 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T10:40:43.992 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T10:40:43.992 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T10:40:43.993 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T10:40:43.993 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T10:40:43.993 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T10:40:43.993 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T10:40:43.993 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T10:40:43.993 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T10:40:43.997 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:40:43.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:44.000 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:40:44.100 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:40:44.131 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:40:44.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:44.133 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:40:44.234 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:40:44.259 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:40:44.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:44.262 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:40:44.362 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:40:44.385 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:40:44.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:44.388 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:40:44.489 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:40:44.520 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:40:44.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:44.524 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:40:44.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:44 smithi078 bash[75299]: audit 2023-10-01T10:40:43.797857+0000 mon.c (mon.2) 131 : audit 0 from='client.? 172.21.15.78:0/4151017463' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:40:44.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:44 smithi078 bash[78274]: audit 2023-10-01T10:40:43.797857+0000 mon.c (mon.2) 131 : audit 0 from='client.? 172.21.15.78:0/4151017463' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:40:44.624 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:40:44.653 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:40:44.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:44.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:40:44.756 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:40:44.769 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:44 smithi173 bash[58816]: audit 2023-10-01T10:40:43.797857+0000 mon.c (mon.2) 131 : audit 0 from='client.? 172.21.15.78:0/4151017463' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:40:44.790 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:40:44.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:44.793 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:40:44.894 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:40:44.922 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:40:44.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:44.925 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:40:45.026 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:45.053 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:45.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:45.056 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:40:45.157 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:45.189 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:45.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:45.192 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:40:45.293 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:40:45.319 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:40:45.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:45.322 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:40:45.423 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:40:45.448 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:40:45.449 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:45 smithi078 bash[75299]: cluster 2023-10-01T10:40:43.706305+0000 mgr.y (mgr.25072) 688 : cluster 0 pgmap v592: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:45.449 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:45 smithi078 bash[78274]: cluster 2023-10-01T10:40:43.706305+0000 mgr.y (mgr.25072) 688 : cluster 0 pgmap v592: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:45.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:45.450 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:40:45.551 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:40:45.568 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T10:40:45.569 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T10:40:45.578 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:40:45.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:45.582 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:40:45.682 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:40:45.707 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:40:45.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:45.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:40:45.811 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:40:45.833 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:40:45.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:45.835 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:40:45.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:45 smithi173 bash[58816]: cluster 2023-10-01T10:40:43.706305+0000 mgr.y (mgr.25072) 688 : cluster 0 pgmap v592: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:45.936 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:40:45.959 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:40:45.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:45.961 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:40:46.062 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:40:46.097 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:40:46.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:46.100 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:40:46.201 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:40:46.233 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:40:46.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:46.236 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:40:46.337 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:40:46.355 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:40:46.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:46.358 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:40:46.459 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:40:46.482 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:40:46.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:46.484 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:40:46.585 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:40:46.609 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:40:46.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:46.615 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:40:46.716 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:40:46.751 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:40:46.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:46.755 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:40:46.767 INFO:teuthology.orchestra.run.smithi078.stdout:ERROR: (22) Invalid argument 2023-10-01T10:40:46.767 INFO:teuthology.orchestra.run.smithi078.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T10:40:46.855 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:40:46.887 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:40:46.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:46.889 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:40:46.990 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:40:47.017 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:40:47.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:47.020 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:40:47.120 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:40:47.139 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:40:47.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:47.142 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:40:47.242 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:40:47.309 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:40:47.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:47.312 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:40:47.412 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:40:47.448 INFO:teuthology.orchestra.run.smithi078.stderr:pool 'unique_pool_4' created 2023-10-01T10:40:47.450 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:40:47.452 INFO:teuthology.orchestra.run.smithi078.stderr:noscrub is set 2023-10-01T10:40:47.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:47.453 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:40:47.554 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:40:47.589 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:40:47.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:47.592 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:40:47.693 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:40:47.706 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:47 smithi173 bash[58816]: cluster 2023-10-01T10:40:45.706840+0000 mgr.y (mgr.25072) 689 : cluster 0 pgmap v593: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:47.706 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:47 smithi173 bash[58816]: audit 2023-10-01T10:40:46.603495+0000 mon.c (mon.2) 132 : audit 1 from='client.? 172.21.15.78:0/3918716213' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:40:47.706 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:47 smithi173 bash[58816]: audit 2023-10-01T10:40:46.603998+0000 mon.a (mon.0) 913 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:40:47.706 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:47 smithi173 bash[58816]: audit 2023-10-01T10:40:46.856403+0000 mon.a (mon.0) 914 : audit 1 from='client.? 172.21.15.78:0/2491073079' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-10-01T10:40:47.706 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:47 smithi173 bash[58816]: audit 2023-10-01T10:40:47.411718+0000 mon.a (mon.0) 915 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:40:47.727 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:40:47.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:47.730 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:40:47.813 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:47 smithi078 bash[75299]: cluster 2023-10-01T10:40:45.706840+0000 mgr.y (mgr.25072) 689 : cluster 0 pgmap v593: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:47.813 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:47 smithi078 bash[75299]: audit 2023-10-01T10:40:46.603495+0000 mon.c (mon.2) 132 : audit 1 from='client.? 172.21.15.78:0/3918716213' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:40:47.813 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:47 smithi078 bash[75299]: audit 2023-10-01T10:40:46.603998+0000 mon.a (mon.0) 913 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:40:47.813 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:47 smithi078 bash[75299]: audit 2023-10-01T10:40:46.856403+0000 mon.a (mon.0) 914 : audit 1 from='client.? 172.21.15.78:0/2491073079' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-10-01T10:40:47.813 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:47 smithi078 bash[75299]: audit 2023-10-01T10:40:47.411718+0000 mon.a (mon.0) 915 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:40:47.813 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:47 smithi078 bash[78274]: cluster 2023-10-01T10:40:45.706840+0000 mgr.y (mgr.25072) 689 : cluster 0 pgmap v593: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:47.813 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:47 smithi078 bash[78274]: audit 2023-10-01T10:40:46.603495+0000 mon.c (mon.2) 132 : audit 1 from='client.? 172.21.15.78:0/3918716213' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:40:47.814 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:47 smithi078 bash[78274]: audit 2023-10-01T10:40:46.603998+0000 mon.a (mon.0) 913 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:40:47.814 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:47 smithi078 bash[78274]: audit 2023-10-01T10:40:46.856403+0000 mon.a (mon.0) 914 : audit 1 from='client.? 172.21.15.78:0/2491073079' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-10-01T10:40:47.814 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:47 smithi078 bash[78274]: audit 2023-10-01T10:40:47.411718+0000 mon.a (mon.0) 915 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:40:47.815 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-01T10:40:47.830 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:40:47.863 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:40:47.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:47.866 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:40:47.968 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:40:47.985 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:40:47.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:47.987 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:40:48.088 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:40:48.109 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:40:48.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:48.111 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:40:48.212 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:40:48.232 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:40:48.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:48.234 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:40:48.335 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:40:48.354 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:40:48.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:48.356 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:40:48.399 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T10:40:48.457 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:40:48.476 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:40:48.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:48.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:40:48.579 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:40:48.598 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:40:48.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:48.600 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:40:48.700 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:40:48.712 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:48 smithi173 bash[58816]: cluster 2023-10-01T10:40:47.426911+0000 mon.a (mon.0) 916 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:40:48.713 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:48 smithi173 bash[58816]: cluster 2023-10-01T10:40:47.437098+0000 osd.7 (osd.7) 71 : cluster 3 failed to encode map e128 with expected crc 2023-10-01T10:40:48.713 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:48 smithi173 bash[58816]: audit 2023-10-01T10:40:47.439288+0000 mon.a (mon.0) 917 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T10:40:48.713 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:48 smithi173 bash[58816]: audit 2023-10-01T10:40:47.439402+0000 mon.a (mon.0) 918 : audit 1 from='client.? 172.21.15.78:0/2491073079' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-10-01T10:40:48.713 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:48 smithi173 bash[58816]: cluster 2023-10-01T10:40:47.439480+0000 mon.a (mon.0) 919 : cluster 0 osdmap e128: 8 total, 8 up, 8 in 2023-10-01T10:40:48.713 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:48 smithi173 bash[58816]: cluster 2023-10-01T10:40:47.442264+0000 osd.6 (osd.6) 43 : cluster 3 failed to encode map e128 with expected crc 2023-10-01T10:40:48.713 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:48 smithi173 bash[58816]: cluster 2023-10-01T10:40:47.442485+0000 osd.5 (osd.5) 50 : cluster 3 failed to encode map e128 with expected crc 2023-10-01T10:40:48.730 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:40:48.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:48.734 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:40:48.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:48 smithi078 bash[75299]: cluster 2023-10-01T10:40:47.426911+0000 mon.a (mon.0) 916 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:40:48.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:48 smithi078 bash[75299]: cluster 2023-10-01T10:40:47.437098+0000 osd.7 (osd.7) 71 : cluster 3 failed to encode map e128 with expected crc 2023-10-01T10:40:48.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:48 smithi078 bash[75299]: audit 2023-10-01T10:40:47.439288+0000 mon.a (mon.0) 917 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T10:40:48.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:48 smithi078 bash[75299]: audit 2023-10-01T10:40:47.439402+0000 mon.a (mon.0) 918 : audit 1 from='client.? 172.21.15.78:0/2491073079' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-10-01T10:40:48.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:48 smithi078 bash[75299]: cluster 2023-10-01T10:40:47.439480+0000 mon.a (mon.0) 919 : cluster 0 osdmap e128: 8 total, 8 up, 8 in 2023-10-01T10:40:48.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:48 smithi078 bash[75299]: cluster 2023-10-01T10:40:47.442264+0000 osd.6 (osd.6) 43 : cluster 3 failed to encode map e128 with expected crc 2023-10-01T10:40:48.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:48 smithi078 bash[75299]: cluster 2023-10-01T10:40:47.442485+0000 osd.5 (osd.5) 50 : cluster 3 failed to encode map e128 with expected crc 2023-10-01T10:40:48.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:48 smithi078 bash[78274]: cluster 2023-10-01T10:40:47.426911+0000 mon.a (mon.0) 916 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:40:48.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:48 smithi078 bash[78274]: cluster 2023-10-01T10:40:47.437098+0000 osd.7 (osd.7) 71 : cluster 3 failed to encode map e128 with expected crc 2023-10-01T10:40:48.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:48 smithi078 bash[78274]: audit 2023-10-01T10:40:47.439288+0000 mon.a (mon.0) 917 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T10:40:48.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:48 smithi078 bash[78274]: audit 2023-10-01T10:40:47.439402+0000 mon.a (mon.0) 918 : audit 1 from='client.? 172.21.15.78:0/2491073079' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-10-01T10:40:48.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:48 smithi078 bash[78274]: cluster 2023-10-01T10:40:47.439480+0000 mon.a (mon.0) 919 : cluster 0 osdmap e128: 8 total, 8 up, 8 in 2023-10-01T10:40:48.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:48 smithi078 bash[78274]: cluster 2023-10-01T10:40:47.442264+0000 osd.6 (osd.6) 43 : cluster 3 failed to encode map e128 with expected crc 2023-10-01T10:40:48.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:48 smithi078 bash[78274]: cluster 2023-10-01T10:40:47.442485+0000 osd.5 (osd.5) 50 : cluster 3 failed to encode map e128 with expected crc 2023-10-01T10:40:48.834 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:40:48.857 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:40:48.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:48.861 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:40:48.962 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:40:48.996 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:40:48.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:48.998 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:40:49.099 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:40:49.123 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:40:49.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:49.126 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:40:49.226 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:40:49.257 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:40:49.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:49.260 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:40:49.323 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_4 rados --yes-i-really-mean-it || true 2023-10-01T10:40:49.361 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:49.394 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:49.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:49.398 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:40:49.499 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:40:49.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:49.534 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:40:49.534 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:40:49.635 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:49.649 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:49 smithi173 bash[58816]: cluster 2023-10-01T10:40:47.707423+0000 mgr.y (mgr.25072) 690 : cluster 0 pgmap v595: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:49.649 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:49 smithi173 bash[58816]: cluster 2023-10-01T10:40:48.444948+0000 osd.5 (osd.5) 51 : cluster 3 failed to encode map e129 with expected crc 2023-10-01T10:40:49.649 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:49 smithi173 bash[58816]: cluster 2023-10-01T10:40:48.446387+0000 mon.a (mon.0) 920 : cluster 0 osdmap e129: 8 total, 8 up, 8 in 2023-10-01T10:40:49.649 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:49 smithi173 bash[58816]: cluster 2023-10-01T10:40:48.447724+0000 osd.5 (osd.5) 52 : cluster 3 failed to encode map e129 with expected crc 2023-10-01T10:40:49.649 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:49 smithi173 bash[58816]: cluster 2023-10-01T10:40:48.447851+0000 osd.7 (osd.7) 72 : cluster 3 failed to encode map e129 with expected crc 2023-10-01T10:40:49.649 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:49 smithi173 bash[58816]: cluster 2023-10-01T10:40:48.450758+0000 osd.7 (osd.7) 73 : cluster 3 failed to encode map e129 with expected crc 2023-10-01T10:40:49.668 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:49.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:49.672 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:40:49.772 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:40:49.803 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:40:49.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:49.806 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:40:49.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:49 smithi078 bash[75299]: cluster 2023-10-01T10:40:47.707423+0000 mgr.y (mgr.25072) 690 : cluster 0 pgmap v595: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:49.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:49 smithi078 bash[75299]: cluster 2023-10-01T10:40:48.444948+0000 osd.5 (osd.5) 51 : cluster 3 failed to encode map e129 with expected crc 2023-10-01T10:40:49.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:49 smithi078 bash[75299]: cluster 2023-10-01T10:40:48.446387+0000 mon.a (mon.0) 920 : cluster 0 osdmap e129: 8 total, 8 up, 8 in 2023-10-01T10:40:49.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:49 smithi078 bash[75299]: cluster 2023-10-01T10:40:48.447724+0000 osd.5 (osd.5) 52 : cluster 3 failed to encode map e129 with expected crc 2023-10-01T10:40:49.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:49 smithi078 bash[75299]: cluster 2023-10-01T10:40:48.447851+0000 osd.7 (osd.7) 72 : cluster 3 failed to encode map e129 with expected crc 2023-10-01T10:40:49.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:49 smithi078 bash[75299]: cluster 2023-10-01T10:40:48.450758+0000 osd.7 (osd.7) 73 : cluster 3 failed to encode map e129 with expected crc 2023-10-01T10:40:49.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:49 smithi078 bash[78274]: cluster 2023-10-01T10:40:47.707423+0000 mgr.y (mgr.25072) 690 : cluster 0 pgmap v595: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:49.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:49 smithi078 bash[78274]: cluster 2023-10-01T10:40:48.444948+0000 osd.5 (osd.5) 51 : cluster 3 failed to encode map e129 with expected crc 2023-10-01T10:40:49.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:49 smithi078 bash[78274]: cluster 2023-10-01T10:40:48.446387+0000 mon.a (mon.0) 920 : cluster 0 osdmap e129: 8 total, 8 up, 8 in 2023-10-01T10:40:49.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:49 smithi078 bash[78274]: cluster 2023-10-01T10:40:48.447724+0000 osd.5 (osd.5) 52 : cluster 3 failed to encode map e129 with expected crc 2023-10-01T10:40:49.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:49 smithi078 bash[78274]: cluster 2023-10-01T10:40:48.447851+0000 osd.7 (osd.7) 72 : cluster 3 failed to encode map e129 with expected crc 2023-10-01T10:40:49.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:49 smithi078 bash[78274]: cluster 2023-10-01T10:40:48.450758+0000 osd.7 (osd.7) 73 : cluster 3 failed to encode map e129 with expected crc 2023-10-01T10:40:49.906 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:40:49.940 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:40:49.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:49.946 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:40:50.046 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:40:50.075 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:40:50.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:50.078 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:40:50.179 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:40:50.212 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:40:50.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:50.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:40:50.316 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:40:50.346 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:40:50.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:50.349 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:40:50.450 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:40:50.456 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:50 smithi078 bash[75299]: cluster 2023-10-01T10:40:49.452619+0000 osd.7 (osd.7) 74 : cluster 3 failed to encode map e130 with expected crc 2023-10-01T10:40:50.457 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:50 smithi078 bash[75299]: cluster 2023-10-01T10:40:49.453296+0000 mon.a (mon.0) 921 : cluster 0 osdmap e130: 8 total, 8 up, 8 in 2023-10-01T10:40:50.457 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:50 smithi078 bash[75299]: cluster 2023-10-01T10:40:49.654154+0000 osd.6 (osd.6) 44 : cluster 3 failed to encode map e130 with expected crc 2023-10-01T10:40:50.457 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:40:50 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:40:50] "GET /metrics HTTP/1.1" 200 33822 "" "Prometheus/2.33.4" 2023-10-01T10:40:50.480 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:40:50.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:50.483 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:40:50.583 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:40:50.612 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:40:50.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:50.614 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:40:50.714 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:50.728 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:50 smithi173 bash[58816]: cluster 2023-10-01T10:40:49.452619+0000 osd.7 (osd.7) 74 : cluster 3 failed to encode map e130 with expected crc 2023-10-01T10:40:50.728 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:50 smithi173 bash[58816]: cluster 2023-10-01T10:40:49.453296+0000 mon.a (mon.0) 921 : cluster 0 osdmap e130: 8 total, 8 up, 8 in 2023-10-01T10:40:50.728 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:50 smithi173 bash[58816]: cluster 2023-10-01T10:40:49.654154+0000 osd.6 (osd.6) 44 : cluster 3 failed to encode map e130 with expected crc 2023-10-01T10:40:50.746 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:50.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:50.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:40:50.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:50 smithi078 bash[78274]: cluster 2023-10-01T10:40:49.452619+0000 osd.7 (osd.7) 74 : cluster 3 failed to encode map e130 with expected crc 2023-10-01T10:40:50.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:50 smithi078 bash[78274]: cluster 2023-10-01T10:40:49.453296+0000 mon.a (mon.0) 921 : cluster 0 osdmap e130: 8 total, 8 up, 8 in 2023-10-01T10:40:50.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:50 smithi078 bash[78274]: cluster 2023-10-01T10:40:49.654154+0000 osd.6 (osd.6) 44 : cluster 3 failed to encode map e130 with expected crc 2023-10-01T10:40:50.850 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:40:50.884 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:40:50.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:50.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:40:50.988 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:40:51.021 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:40:51.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:51.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:40:51.124 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:40:51.153 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:40:51.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:51.156 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:40:51.257 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:40:51.289 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:40:51.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:51.293 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:40:51.376 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T10:40:51.393 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:40:51.452 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:40:51.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:51.454 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:40:51.555 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:40:51.566 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:51 smithi173 bash[58816]: cluster 2023-10-01T10:40:49.708132+0000 mgr.y (mgr.25072) 691 : cluster 0 pgmap v598: 89 pgs: 3 creating+peering, 9 unknown, 77 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:51.566 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:51 smithi173 bash[58816]: cluster 2023-10-01T10:40:51.287013+0000 osd.5 (osd.5) 53 : cluster 3 failed to encode map e130 with expected crc 2023-10-01T10:40:51.586 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:40:51.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:51.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:40:51.690 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:40:51.702 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:51 smithi078 bash[75299]: cluster 2023-10-01T10:40:49.708132+0000 mgr.y (mgr.25072) 691 : cluster 0 pgmap v598: 89 pgs: 3 creating+peering, 9 unknown, 77 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:51.702 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:51 smithi078 bash[75299]: cluster 2023-10-01T10:40:51.287013+0000 osd.5 (osd.5) 53 : cluster 3 failed to encode map e130 with expected crc 2023-10-01T10:40:51.702 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:51 smithi078 bash[78274]: cluster 2023-10-01T10:40:49.708132+0000 mgr.y (mgr.25072) 691 : cluster 0 pgmap v598: 89 pgs: 3 creating+peering, 9 unknown, 77 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:51.702 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:51 smithi078 bash[78274]: cluster 2023-10-01T10:40:51.287013+0000 osd.5 (osd.5) 53 : cluster 3 failed to encode map e130 with expected crc 2023-10-01T10:40:51.720 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:40:51.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:51.723 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:40:51.823 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:40:51.860 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:40:51.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:51.863 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:40:51.963 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:40:51.995 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:40:51.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:51.998 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:40:52.099 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:40:52.128 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:40:52.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:52.132 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:40:52.233 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:40:52.263 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:40:52.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:52.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:40:52.367 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:40:52.398 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:40:52.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:52.400 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:40:52.501 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:40:52.531 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:40:52.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:52.534 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:40:52.577 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:40:52.579 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:40:52.635 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:40:52.667 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:40:52.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:52.671 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:40:52.771 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:40:52.801 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:40:52.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:52.804 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:40:52.905 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:40:52.941 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:40:52.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:52.943 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:40:53.043 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:40:53.081 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:40:53.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:53.083 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:40:53.184 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:53.219 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:53.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:53.223 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:40:53.324 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:40:53.353 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:40:53.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:53.355 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:40:53.456 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:40:53.483 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:40:53.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:53.486 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:40:53.586 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:40:53.593 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:40:53.620 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:40:53.621 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:53 smithi078 bash[78274]: cluster 2023-10-01T10:40:51.708791+0000 mgr.y (mgr.25072) 692 : cluster 0 pgmap v599: 89 pgs: 3 creating+peering, 86 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:53.621 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:53 smithi078 bash[75299]: cluster 2023-10-01T10:40:51.708791+0000 mgr.y (mgr.25072) 692 : cluster 0 pgmap v599: 89 pgs: 3 creating+peering, 86 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:53.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:53.623 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:40:53.724 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:40:53.749 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:40:53.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:53.752 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:40:53.853 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:40:53.864 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:53 smithi173 bash[58816]: cluster 2023-10-01T10:40:51.708791+0000 mgr.y (mgr.25072) 692 : cluster 0 pgmap v599: 89 pgs: 3 creating+peering, 86 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:53.884 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:40:53.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:53.887 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:40:53.988 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:54.019 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:54.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:54.022 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:40:54.123 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:40:54.150 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:40:54.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:54.152 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:40:54.253 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:40:54.286 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:40:54.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:54.288 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:40:54.389 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:40:54.420 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:40:54.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:54.423 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:40:54.524 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:40:54.552 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:40:54.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:54.555 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:40:54.655 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:54.683 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:54.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:54.686 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:40:54.787 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:40:54.813 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:40:54.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:54.816 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:40:54.916 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:40:54.945 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:40:54.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:54.948 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:40:55.049 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:40:55.080 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:40:55.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:55.083 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:40:55.184 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:40:55.206 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:40:55.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:55.209 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:40:55.310 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:40:55.336 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:40:55.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:55.339 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:40:55.439 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:40:55.471 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:40:55.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:55.474 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:40:55.575 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:40:55.610 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:40:55.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:55.613 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:40:55.619 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:40:55.713 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:40:55.724 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:55 smithi078 bash[78274]: cluster 2023-10-01T10:40:53.709635+0000 mgr.y (mgr.25072) 693 : cluster 0 pgmap v600: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:55.724 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:55 smithi078 bash[75299]: cluster 2023-10-01T10:40:53.709635+0000 mgr.y (mgr.25072) 693 : cluster 0 pgmap v600: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:55.735 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:40:55.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:55.737 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:40:55.837 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:40:55.858 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:40:55.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:55.860 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:40:55.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:55 smithi173 bash[58816]: cluster 2023-10-01T10:40:53.709635+0000 mgr.y (mgr.25072) 693 : cluster 0 pgmap v600: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:55.961 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:40:55.990 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:40:55.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:55.993 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:40:56.094 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:40:56.127 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:40:56.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:56.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:40:56.231 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:40:56.257 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:40:56.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:56.260 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:40:56.361 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:40:56.390 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:40:56.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:56.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:40:56.495 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:40:56.527 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:40:56.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:56.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:40:56.631 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:40:56.663 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:40:56.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:56.666 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:40:56.766 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:40:56.796 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:40:56.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:56.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:40:56.900 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:56.929 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:56.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:56.933 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:40:57.033 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:40:57.060 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:40:57.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:57.062 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:40:57.163 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:40:57.189 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:40:57.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:57.192 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:40:57.292 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:40:57.322 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:40:57.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:57.324 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:40:57.425 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:40:57.450 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:40:57.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:57.453 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:40:57.553 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:40:57.561 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:57 smithi078 bash[78274]: cluster 2023-10-01T10:40:55.710429+0000 mgr.y (mgr.25072) 694 : cluster 0 pgmap v601: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:57.562 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:57 smithi078 bash[75299]: cluster 2023-10-01T10:40:55.710429+0000 mgr.y (mgr.25072) 694 : cluster 0 pgmap v601: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:57.575 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:40:57.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:57.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:40:57.678 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:57.712 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:57.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:57.715 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:40:57.816 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:57.827 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:57 smithi173 bash[58816]: cluster 2023-10-01T10:40:55.710429+0000 mgr.y (mgr.25072) 694 : cluster 0 pgmap v601: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:57.846 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:57.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:57.849 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:40:57.950 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:40:57.971 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:40:57.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:57.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:40:58.075 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:40:58.096 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:40:58.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:58.098 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:40:58.199 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:40:58.223 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:40:58.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:58.226 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:40:58.326 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:40:58.351 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:40:58.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:58.353 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:40:58.454 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:40:58.484 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:40:58.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:58.488 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:40:58.588 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:40:58.614 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:40:58.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:58.616 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:40:58.716 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:40:58.742 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:40:58.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:58.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:40:58.845 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:58.875 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:58.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:58.879 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:40:58.979 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:58.993 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:58 smithi173 bash[58816]: cluster 2023-10-01T10:40:57.711185+0000 mgr.y (mgr.25072) 695 : cluster 0 pgmap v602: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:58.993 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:40:58 smithi173 bash[58816]: audit 2023-10-01T10:40:57.734651+0000 mon.a (mon.0) 922 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:40:59.017 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:59.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:59.019 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:40:59.120 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:40:59.145 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:58 smithi078 bash[78274]: cluster 2023-10-01T10:40:57.711185+0000 mgr.y (mgr.25072) 695 : cluster 0 pgmap v602: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:59.145 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:40:58 smithi078 bash[78274]: audit 2023-10-01T10:40:57.734651+0000 mon.a (mon.0) 922 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:40:59.145 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:58 smithi078 bash[75299]: cluster 2023-10-01T10:40:57.711185+0000 mgr.y (mgr.25072) 695 : cluster 0 pgmap v602: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:40:59.145 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:40:58 smithi078 bash[75299]: audit 2023-10-01T10:40:57.734651+0000 mon.a (mon.0) 922 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:40:59.155 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:40:59.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:59.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:40:59.259 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:40:59.290 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:40:59.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:59.293 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:40:59.394 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:59.426 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:59.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:59.429 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:40:59.529 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:40:59.553 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:40:59.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:59.558 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:40:59.658 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:40:59.683 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:40:59.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:59.685 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:40:59.786 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:59.817 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:40:59.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:59.820 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:40:59.921 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:40:59.948 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:40:59.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:40:59.950 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:41:00.051 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:00.083 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:00.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:00.087 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:41:00.188 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:00.208 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:00.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:00.211 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:41:00.311 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:00.338 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:00.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:00.341 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:41:00.421 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:41:00 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:41:00] "GET /metrics HTTP/1.1" 200 33822 "" "Prometheus/2.33.4" 2023-10-01T10:41:00.441 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:00.465 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:00.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:00.468 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:41:00.569 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:00.597 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:00.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:00.599 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:41:00.700 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:00.732 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:00.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:00.736 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:41:00.836 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:00.863 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:00.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:00.866 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:41:00.967 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:00.978 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:00 smithi173 bash[58816]: cluster 2023-10-01T10:40:59.711872+0000 mgr.y (mgr.25072) 696 : cluster 0 pgmap v603: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:41:00.997 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:01.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:01.000 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:41:01.093 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:00 smithi078 bash[75299]: cluster 2023-10-01T10:40:59.711872+0000 mgr.y (mgr.25072) 696 : cluster 0 pgmap v603: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:41:01.093 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:00 smithi078 bash[78274]: cluster 2023-10-01T10:40:59.711872+0000 mgr.y (mgr.25072) 696 : cluster 0 pgmap v603: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:41:01.101 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:01.133 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:01.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:01.136 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:41:01.237 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:02.286 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:02.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:02.321 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:41:02.422 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:02.455 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:02.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:02.483 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:41:02.584 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:02.682 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:02.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:02.787 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:41:02.888 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:02.979 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:03.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:03.033 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:41:03.133 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:03.169 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:03.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:03.172 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:41:03.273 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:03.293 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:03.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:03.295 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:41:03.362 INFO:teuthology.orchestra.run.smithi078.stdout:ERROR: (22) Invalid argument 2023-10-01T10:41:03.403 INFO:teuthology.orchestra.run.smithi078.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T10:41:03.404 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:03.424 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:03.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:03.427 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:41:03.449 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T10:41:03.473 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T10:41:03.496 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T10:41:03.520 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T10:41:03.528 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:03.544 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:03 smithi078 bash[75299]: cluster 2023-10-01T10:41:01.712683+0000 mgr.y (mgr.25072) 697 : cluster 0 pgmap v604: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:41:03.544 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:03 smithi078 bash[75299]: audit 2023-10-01T10:41:02.415402+0000 mon.a (mon.0) 923 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:41:03.544 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:03 smithi078 bash[75299]: audit 2023-10-01T10:41:02.416775+0000 mon.a (mon.0) 924 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:41:03.544 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:03 smithi078 bash[78274]: cluster 2023-10-01T10:41:01.712683+0000 mgr.y (mgr.25072) 697 : cluster 0 pgmap v604: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:41:03.544 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:03 smithi078 bash[78274]: audit 2023-10-01T10:41:02.415402+0000 mon.a (mon.0) 923 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:41:03.544 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:03 smithi078 bash[78274]: audit 2023-10-01T10:41:02.416775+0000 mon.a (mon.0) 924 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:41:03.548 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T10:41:03.560 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:03.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:03.563 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:41:03.574 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T10:41:03.602 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T10:41:03.633 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T10:41:03.633 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T10:41:03.634 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T10:41:03.634 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T10:41:03.634 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T10:41:03.634 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T10:41:03.634 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T10:41:03.634 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T10:41:03.634 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T10:41:03.664 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:03.685 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:03.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:03.687 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:41:03.788 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:03.801 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:03 smithi173 bash[58816]: cluster 2023-10-01T10:41:01.712683+0000 mgr.y (mgr.25072) 697 : cluster 0 pgmap v604: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:41:03.801 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:03 smithi173 bash[58816]: audit 2023-10-01T10:41:02.415402+0000 mon.a (mon.0) 923 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:41:03.802 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:03 smithi173 bash[58816]: audit 2023-10-01T10:41:02.416775+0000 mon.a (mon.0) 924 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:41:03.819 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:03.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:03.823 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:41:03.923 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:03.954 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:03.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:03.957 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:41:04.057 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:04.092 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:04.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:04.095 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:41:04.196 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:04.221 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:04.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:04.224 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:41:04.325 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:04.354 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:04.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:04.357 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:41:04.438 INFO:teuthology.orchestra.run.smithi078.stderr:enabled application 'rados' on pool 'unique_pool_4' 2023-10-01T10:41:04.438 INFO:teuthology.orchestra.run.smithi078.stderr:nodeep-scrub is set 2023-10-01T10:41:04.458 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:04.480 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:04.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:04.482 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:41:04.583 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:04.612 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:04.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:04.614 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:41:04.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:04 smithi173 bash[58816]: audit 2023-10-01T10:41:03.426315+0000 mon.a (mon.0) 925 : audit 0 from='client.? 172.21.15.78:0/2879323057' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:41:04.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:04 smithi173 bash[58816]: audit 2023-10-01T10:41:03.969140+0000 mon.a (mon.0) 926 : audit 1 from='client.? 172.21.15.78:0/2941564502' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T10:41:04.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:04 smithi173 bash[58816]: audit 2023-10-01T10:41:04.150430+0000 mon.c (mon.2) 133 : audit 1 from='client.? 172.21.15.78:0/3492738682' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:41:04.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:04 smithi173 bash[58816]: audit 2023-10-01T10:41:04.150926+0000 mon.a (mon.0) 927 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:41:04.715 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:04.748 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:04.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:04.751 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:41:04.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:04 smithi078 bash[75299]: audit 2023-10-01T10:41:03.426315+0000 mon.a (mon.0) 925 : audit 0 from='client.? 172.21.15.78:0/2879323057' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:41:04.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:04 smithi078 bash[75299]: audit 2023-10-01T10:41:03.969140+0000 mon.a (mon.0) 926 : audit 1 from='client.? 172.21.15.78:0/2941564502' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T10:41:04.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:04 smithi078 bash[75299]: audit 2023-10-01T10:41:04.150430+0000 mon.c (mon.2) 133 : audit 1 from='client.? 172.21.15.78:0/3492738682' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:41:04.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:04 smithi078 bash[75299]: audit 2023-10-01T10:41:04.150926+0000 mon.a (mon.0) 927 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:41:04.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:04 smithi078 bash[78274]: audit 2023-10-01T10:41:03.426315+0000 mon.a (mon.0) 925 : audit 0 from='client.? 172.21.15.78:0/2879323057' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:41:04.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:04 smithi078 bash[78274]: audit 2023-10-01T10:41:03.969140+0000 mon.a (mon.0) 926 : audit 1 from='client.? 172.21.15.78:0/2941564502' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T10:41:04.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:04 smithi078 bash[78274]: audit 2023-10-01T10:41:04.150430+0000 mon.c (mon.2) 133 : audit 1 from='client.? 172.21.15.78:0/3492738682' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:41:04.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:04 smithi078 bash[78274]: audit 2023-10-01T10:41:04.150926+0000 mon.a (mon.0) 927 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:41:04.852 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:04.886 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:04.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:04.888 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:41:04.989 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:05.013 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:05.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:05.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:41:05.116 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:05.149 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:05.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:05.152 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:41:05.253 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:05.275 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:05.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:05.278 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:41:05.378 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:05.404 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:05.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:05.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:41:05.507 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:05.528 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:05.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:05.531 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:41:05.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:05 smithi078 bash[78274]: cluster 2023-10-01T10:41:03.713131+0000 mgr.y (mgr.25072) 698 : cluster 0 pgmap v605: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:41:05.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:05 smithi078 bash[78274]: cluster 2023-10-01T10:41:04.415422+0000 mon.a (mon.0) 928 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:41:05.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:05 smithi078 bash[78274]: cluster 2023-10-01T10:41:04.430083+0000 osd.7 (osd.7) 75 : cluster 3 failed to encode map e131 with expected crc 2023-10-01T10:41:05.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:05 smithi078 bash[78274]: audit 2023-10-01T10:41:04.431725+0000 mon.a (mon.0) 929 : audit 1 from='client.? 172.21.15.78:0/2941564502' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-01T10:41:05.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:05 smithi078 bash[78274]: audit 2023-10-01T10:41:04.431791+0000 mon.a (mon.0) 930 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T10:41:05.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:05 smithi078 bash[78274]: cluster 2023-10-01T10:41:04.431861+0000 mon.a (mon.0) 931 : cluster 0 osdmap e131: 8 total, 8 up, 8 in 2023-10-01T10:41:05.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:05 smithi078 bash[78274]: cluster 2023-10-01T10:41:04.618342+0000 osd.5 (osd.5) 54 : cluster 3 failed to encode map e131 with expected crc 2023-10-01T10:41:05.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:05 smithi078 bash[78274]: cluster 2023-10-01T10:41:04.757569+0000 osd.6 (osd.6) 45 : cluster 3 failed to encode map e131 with expected crc 2023-10-01T10:41:05.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:05 smithi078 bash[78274]: cluster 2023-10-01T10:41:04.759572+0000 osd.6 (osd.6) 46 : cluster 3 failed to encode map e131 with expected crc 2023-10-01T10:41:05.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:05 smithi078 bash[75299]: cluster 2023-10-01T10:41:03.713131+0000 mgr.y (mgr.25072) 698 : cluster 0 pgmap v605: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:41:05.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:05 smithi078 bash[75299]: cluster 2023-10-01T10:41:04.415422+0000 mon.a (mon.0) 928 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:41:05.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:05 smithi078 bash[75299]: cluster 2023-10-01T10:41:04.430083+0000 osd.7 (osd.7) 75 : cluster 3 failed to encode map e131 with expected crc 2023-10-01T10:41:05.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:05 smithi078 bash[75299]: audit 2023-10-01T10:41:04.431725+0000 mon.a (mon.0) 929 : audit 1 from='client.? 172.21.15.78:0/2941564502' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-01T10:41:05.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:05 smithi078 bash[75299]: audit 2023-10-01T10:41:04.431791+0000 mon.a (mon.0) 930 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T10:41:05.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:05 smithi078 bash[75299]: cluster 2023-10-01T10:41:04.431861+0000 mon.a (mon.0) 931 : cluster 0 osdmap e131: 8 total, 8 up, 8 in 2023-10-01T10:41:05.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:05 smithi078 bash[75299]: cluster 2023-10-01T10:41:04.618342+0000 osd.5 (osd.5) 54 : cluster 3 failed to encode map e131 with expected crc 2023-10-01T10:41:05.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:05 smithi078 bash[75299]: cluster 2023-10-01T10:41:04.757569+0000 osd.6 (osd.6) 45 : cluster 3 failed to encode map e131 with expected crc 2023-10-01T10:41:05.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:05 smithi078 bash[75299]: cluster 2023-10-01T10:41:04.759572+0000 osd.6 (osd.6) 46 : cluster 3 failed to encode map e131 with expected crc 2023-10-01T10:41:05.632 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:05.668 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:05.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:05.672 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:41:05.773 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:05.784 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:05 smithi173 bash[58816]: cluster 2023-10-01T10:41:03.713131+0000 mgr.y (mgr.25072) 698 : cluster 0 pgmap v605: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:41:05.784 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:05 smithi173 bash[58816]: cluster 2023-10-01T10:41:04.415422+0000 mon.a (mon.0) 928 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:41:05.784 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:05 smithi173 bash[58816]: cluster 2023-10-01T10:41:04.430083+0000 osd.7 (osd.7) 75 : cluster 3 failed to encode map e131 with expected crc 2023-10-01T10:41:05.784 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:05 smithi173 bash[58816]: audit 2023-10-01T10:41:04.431725+0000 mon.a (mon.0) 929 : audit 1 from='client.? 172.21.15.78:0/2941564502' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-01T10:41:05.785 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:05 smithi173 bash[58816]: audit 2023-10-01T10:41:04.431791+0000 mon.a (mon.0) 930 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T10:41:05.785 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:05 smithi173 bash[58816]: cluster 2023-10-01T10:41:04.431861+0000 mon.a (mon.0) 931 : cluster 0 osdmap e131: 8 total, 8 up, 8 in 2023-10-01T10:41:05.785 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:05 smithi173 bash[58816]: cluster 2023-10-01T10:41:04.618342+0000 osd.5 (osd.5) 54 : cluster 3 failed to encode map e131 with expected crc 2023-10-01T10:41:05.785 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:05 smithi173 bash[58816]: cluster 2023-10-01T10:41:04.757569+0000 osd.6 (osd.6) 45 : cluster 3 failed to encode map e131 with expected crc 2023-10-01T10:41:05.785 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:05 smithi173 bash[58816]: cluster 2023-10-01T10:41:04.759572+0000 osd.6 (osd.6) 46 : cluster 3 failed to encode map e131 with expected crc 2023-10-01T10:41:05.798 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-01T10:41:05.804 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:05.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:05.806 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:41:05.907 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:05.927 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:05.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:05.930 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:41:06.030 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:06.063 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:06.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:06.067 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:41:06.168 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:06.193 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:06.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:06.196 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:41:06.296 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:06.318 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:06.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:06.321 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:41:06.422 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:06.450 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:06.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:06.453 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:41:06.554 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:06.586 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:06.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:06.590 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:41:06.691 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:06.725 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:06.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:06.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:41:06.828 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:06.857 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:06.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:06.859 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:41:06.960 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:06.988 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:06.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:06.991 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:41:07.092 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:07.119 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:07.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:07.122 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:41:07.223 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:07.251 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:07.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:07.255 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:41:07.356 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:07.389 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:07.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:07.392 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:41:07.493 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:07.524 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:07.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:07.528 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:41:07.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:07 smithi173 bash[58816]: cluster 2023-10-01T10:41:05.713973+0000 mgr.y (mgr.25072) 699 : cluster 0 pgmap v607: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:41:07.628 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:07.659 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:07.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:07.663 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:41:07.763 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:07.798 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:07.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:07.801 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:41:07.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:07 smithi078 bash[75299]: cluster 2023-10-01T10:41:05.713973+0000 mgr.y (mgr.25072) 699 : cluster 0 pgmap v607: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:41:07.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:07 smithi078 bash[78274]: cluster 2023-10-01T10:41:05.713973+0000 mgr.y (mgr.25072) 699 : cluster 0 pgmap v607: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:41:07.902 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:07.933 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:07.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:07.936 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:41:08.037 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:08.067 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:08.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:08.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:41:08.159 DEBUG:teuthology.orchestra.run.smithi078:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_4 bench 90 write' 2023-10-01T10:41:08.160 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_4 2023-10-01T10:41:08.160 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T10:41:08.162 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-01T10:41:08.162 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-01T10:41:08.171 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:08.207 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:08.207 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:hints = 1 2023-10-01T10:41:08.207 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-01T10:41:08.207 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Object prefix: benchmark_data_smithi078_113121 2023-10-01T10:41:08.208 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T10:41:08.208 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 0 0 0 0 0 0 - 0 2023-10-01T10:41:08.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:08.211 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:41:08.312 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:08.339 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:08.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:08.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:41:08.443 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:08.472 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:08.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:08.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:41:08.576 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:08.607 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:08.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:08.612 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:41:08.691 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T10:41:08.713 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:08.742 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:08.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:08.745 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:41:08.846 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:08.873 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:08.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:08.875 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:41:08.976 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:09.040 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:09.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:09.183 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:41:09.208 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 1 16 2836 2820 176.241 176.25 0.00520286 0.00565445 2023-10-01T10:41:09.212 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T10:41:09.284 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:09.308 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:09.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:09.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:41:09.412 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:09.441 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:09.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:09.446 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:41:09.547 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:09.673 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:09 smithi173 bash[58816]: cluster 2023-10-01T10:41:07.714774+0000 mgr.y (mgr.25072) 700 : cluster 0 pgmap v608: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:41:09.703 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:09.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:09.706 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:41:09.807 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:09.814 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:09 smithi078 bash[78274]: cluster 2023-10-01T10:41:07.714774+0000 mgr.y (mgr.25072) 700 : cluster 0 pgmap v608: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:41:09.815 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:09 smithi078 bash[75299]: cluster 2023-10-01T10:41:07.714774+0000 mgr.y (mgr.25072) 700 : cluster 0 pgmap v608: 89 pgs: 89 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2023-10-01T10:41:09.836 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:09.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:09.838 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:41:09.939 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:09.964 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:09.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:09.969 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:41:10.070 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:10.097 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:10.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:10.100 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:41:10.200 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:10.208 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 2 16 5622 5606 175.175 174.125 0.0074219 0.00569348 2023-10-01T10:41:10.227 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:10.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:10.229 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:41:10.330 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:10.331 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:41:10 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:41:10] "GET /metrics HTTP/1.1" 200 36181 "" "Prometheus/2.33.4" 2023-10-01T10:41:10.361 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:10.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:10.364 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:41:10.465 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:10.494 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:10.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:10.496 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:41:10.597 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:10.624 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:10.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:10.627 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:41:10.653 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:41:10.728 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:10.750 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:10.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:10.753 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:41:10.853 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:10.880 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:10.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:10.882 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:41:10.983 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:11.013 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:11.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:11.015 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:41:11.116 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:11.146 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:11.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:11.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:41:11.208 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 3 16 8258 8242 171.694 164.75 0.00403065 0.00581572 2023-10-01T10:41:11.251 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:11.277 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:11.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:11.280 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:41:11.381 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:11.407 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:11.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:11.409 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:41:11.510 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:11.519 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:11 smithi078 bash[75299]: cluster 2023-10-01T10:41:09.715436+0000 mgr.y (mgr.25072) 701 : cluster 0 pgmap v609: 89 pgs: 89 active+clean; 29 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s wr, 45 op/s 2023-10-01T10:41:11.519 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:11 smithi078 bash[78274]: cluster 2023-10-01T10:41:09.715436+0000 mgr.y (mgr.25072) 701 : cluster 0 pgmap v609: 89 pgs: 89 active+clean; 29 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s wr, 45 op/s 2023-10-01T10:41:11.534 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:11.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:11.538 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:41:11.638 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:11.663 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:11.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:11.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:41:11.766 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:11.826 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:11.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:11.829 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:41:11.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:11 smithi173 bash[58816]: cluster 2023-10-01T10:41:09.715436+0000 mgr.y (mgr.25072) 701 : cluster 0 pgmap v609: 89 pgs: 89 active+clean; 29 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s wr, 45 op/s 2023-10-01T10:41:11.929 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:11.960 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:11.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:11.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:41:12.063 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:12.101 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:12.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:12.105 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:41:12.205 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:12.217 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 4 16 10852 10836 169.298 162.125 0.00545972 0.00589781 2023-10-01T10:41:12.243 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:12.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:12.247 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:41:12.348 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:12.451 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:12.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:12.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:41:12.792 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:41:12.792 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:41:12.893 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:12.924 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:12.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:12.926 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:41:13.027 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:13.058 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:13.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:13.061 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:41:13.161 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:13.190 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:13.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:13.193 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:41:13.208 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 5 16 13273 13257 165.698 151.312 0.00726525 0.00602709 2023-10-01T10:41:13.294 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:13.321 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:13.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:13.324 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:41:13.424 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:13.452 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:13.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:13.455 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:41:13.556 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:13.565 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:13 smithi078 bash[75299]: cluster 2023-10-01T10:41:11.715981+0000 mgr.y (mgr.25072) 702 : cluster 0 pgmap v610: 89 pgs: 89 active+clean; 240 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s wr, 383 op/s 2023-10-01T10:41:13.565 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:13 smithi078 bash[78274]: cluster 2023-10-01T10:41:11.715981+0000 mgr.y (mgr.25072) 702 : cluster 0 pgmap v610: 89 pgs: 89 active+clean; 240 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s wr, 383 op/s 2023-10-01T10:41:13.589 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:13.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:13.591 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:41:13.692 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:13.702 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:13 smithi173 bash[58816]: cluster 2023-10-01T10:41:11.715981+0000 mgr.y (mgr.25072) 702 : cluster 0 pgmap v610: 89 pgs: 89 active+clean; 240 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s wr, 383 op/s 2023-10-01T10:41:13.706 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:41:13.723 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:13.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:13.728 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:41:13.829 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:13.857 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:13.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:13.862 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:41:13.963 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:13.988 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:13.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:13.992 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:41:14.093 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:14.120 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:14.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:14.125 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:41:14.209 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 6 16 15596 15580 162.277 145.188 0.0127027 0.006154 2023-10-01T10:41:14.226 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:14.252 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:14.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:14.255 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:41:14.356 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:14.382 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:14.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:14.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:41:14.490 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:14.519 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:14.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:14.522 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:41:14.623 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:14.640 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:14.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:14.642 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:41:14.743 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:14.772 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:14.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:14.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:41:14.875 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:14.897 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:14.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:14.899 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:41:15.000 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:15.027 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:15.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:15.029 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:41:15.130 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:15.158 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:15.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:15.160 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:41:15.209 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 7 16 17803 17787 158.798 137.938 0.00400253 0.00629116 2023-10-01T10:41:15.261 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:15.290 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:15.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:15.294 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:41:15.395 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:15.420 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:15.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:15.422 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:41:15.456 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:15 smithi173 bash[58816]: cluster 2023-10-01T10:41:13.717684+0000 mgr.y (mgr.25072) 703 : cluster 0 pgmap v611: 89 pgs: 89 active+clean; 394 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 39 MiB/s wr, 628 op/s 2023-10-01T10:41:15.523 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:15.551 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:15.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:15.555 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:41:15.655 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:15.694 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:15.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:15.701 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:41:15.802 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:15.829 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:15.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:15 smithi078 bash[78274]: cluster 2023-10-01T10:41:13.717684+0000 mgr.y (mgr.25072) 703 : cluster 0 pgmap v611: 89 pgs: 89 active+clean; 394 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 39 MiB/s wr, 628 op/s 2023-10-01T10:41:15.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:15 smithi078 bash[75299]: cluster 2023-10-01T10:41:13.717684+0000 mgr.y (mgr.25072) 703 : cluster 0 pgmap v611: 89 pgs: 89 active+clean; 394 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 39 MiB/s wr, 628 op/s 2023-10-01T10:41:15.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:15.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:41:15.934 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:15.959 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:15.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:15.961 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:41:16.062 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:16.093 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:16.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:16.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:41:16.197 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:16.209 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 8 16 19871 19855 155.103 129.25 0.00748764 0.00644118 2023-10-01T10:41:16.229 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:16.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:16.232 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:41:16.333 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:16.359 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:16.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:16.362 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:41:16.463 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:16.490 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:16.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:16.494 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:41:16.595 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:16.626 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:16.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:16.629 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:41:16.729 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:16.760 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:16.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:16.763 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:41:16.863 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:16.891 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:16.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:16.893 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:41:16.994 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:17.021 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:17.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:17.025 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:41:17.126 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:17.159 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:17.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:17.163 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:41:17.209 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 9 16 21857 21841 151.66 124.125 0.00542554 0.00658805 2023-10-01T10:41:17.263 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:17.288 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:17.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:17.290 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:41:17.391 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:17.425 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:17.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:17.429 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:41:17.530 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:17.538 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:17 smithi078 bash[75299]: cluster 2023-10-01T10:41:15.718580+0000 mgr.y (mgr.25072) 704 : cluster 0 pgmap v612: 89 pgs: 89 active+clean; 833 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-10-01T10:41:17.538 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:17 smithi078 bash[75299]: audit 2023-10-01T10:41:17.411866+0000 mon.a (mon.0) 932 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:41:17.538 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:17 smithi078 bash[78274]: cluster 2023-10-01T10:41:15.718580+0000 mgr.y (mgr.25072) 704 : cluster 0 pgmap v612: 89 pgs: 89 active+clean; 833 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-10-01T10:41:17.538 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:17 smithi078 bash[78274]: audit 2023-10-01T10:41:17.411866+0000 mon.a (mon.0) 932 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:41:17.557 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:17.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:17.560 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:41:17.660 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:17.686 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:17.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:17.690 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:41:17.791 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:17.800 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:17 smithi173 bash[58816]: cluster 2023-10-01T10:41:15.718580+0000 mgr.y (mgr.25072) 704 : cluster 0 pgmap v612: 89 pgs: 89 active+clean; 833 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-10-01T10:41:17.800 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:17 smithi173 bash[58816]: audit 2023-10-01T10:41:17.411866+0000 mon.a (mon.0) 932 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:41:17.825 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:17.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:17.827 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:41:17.928 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:17.952 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:18.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:18.008 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:41:18.109 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:18.132 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:18.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:18.135 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:41:18.209 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 10 16 23705 23689 148.042 115.5 0.00845435 0.00674842 2023-10-01T10:41:18.235 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:18.260 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:18.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:18.265 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:41:18.366 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:18.394 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:18.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:18.398 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:41:18.499 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:18.524 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:18.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:18.527 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:41:18.628 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:18.652 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:18.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:18.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:41:18.756 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:18.787 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:18.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:18.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:41:18.892 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:18.924 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:18.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:18.926 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:41:19.027 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:19.058 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:19.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:19.061 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:41:19.162 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:19.188 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:19.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:19.191 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:41:19.209 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 11 16 25536 25520 144.987 114.438 0.00936534 0.00689089 2023-10-01T10:41:19.292 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:19.337 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:19.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:19.339 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:41:19.440 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:19.482 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:19.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:19.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:41:19.585 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:19.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:19 smithi173 bash[58816]: cluster 2023-10-01T10:41:17.719082+0000 mgr.y (mgr.25072) 705 : cluster 0 pgmap v613: 89 pgs: 89 active+clean; 1.0 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-01T10:41:19.625 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:19.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:19.629 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:41:19.729 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:19.766 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:19.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:19.770 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:41:19.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:19 smithi078 bash[78274]: cluster 2023-10-01T10:41:17.719082+0000 mgr.y (mgr.25072) 705 : cluster 0 pgmap v613: 89 pgs: 89 active+clean; 1.0 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-01T10:41:19.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:19 smithi078 bash[75299]: cluster 2023-10-01T10:41:17.719082+0000 mgr.y (mgr.25072) 705 : cluster 0 pgmap v613: 89 pgs: 89 active+clean; 1.0 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-01T10:41:19.870 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:19.894 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:19.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:19.898 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:41:19.998 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:20.021 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:20.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:20.024 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:41:20.125 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:20.151 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:20.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:20.158 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:41:20.209 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 12 16 27394 27378 142.581 116.125 0.00628465 0.00700852 2023-10-01T10:41:20.260 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:20.270 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:41:20 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:41:20] "GET /metrics HTTP/1.1" 200 36237 "" "Prometheus/2.33.4" 2023-10-01T10:41:20.290 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:20.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:20.292 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:41:20.393 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:20.418 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:20.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:20.421 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:41:20.522 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:20.551 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:20.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:20.555 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:41:20.656 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:20.684 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:20.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:20.687 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:41:20.788 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:20.819 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:20.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:20.821 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:41:20.922 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:20.956 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:20.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:20.959 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:41:21.060 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:21.089 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:21.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:21.092 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:41:21.193 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:21.210 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 13 15 29267 29252 140.615 117.125 0.0133958 0.00710395 2023-10-01T10:41:21.218 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:21.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:21.221 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:41:21.321 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:21.340 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:21.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:21.342 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:41:21.443 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:21.472 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:21.473 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:21 smithi078 bash[75299]: cluster 2023-10-01T10:41:19.719783+0000 mgr.y (mgr.25072) 706 : cluster 0 pgmap v614: 89 pgs: 89 active+clean; 1.3 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 109 MiB/s wr, 1.74k op/s 2023-10-01T10:41:21.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:21.474 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:41:21.575 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:21.601 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:21.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:21.604 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:41:21.704 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:21.713 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:21 smithi078 bash[78274]: cluster 2023-10-01T10:41:19.719783+0000 mgr.y (mgr.25072) 706 : cluster 0 pgmap v614: 89 pgs: 89 active+clean; 1.3 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 109 MiB/s wr, 1.74k op/s 2023-10-01T10:41:21.732 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:21.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:21.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:41:21.835 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:21.845 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:21 smithi173 bash[58816]: cluster 2023-10-01T10:41:19.719783+0000 mgr.y (mgr.25072) 706 : cluster 0 pgmap v614: 89 pgs: 89 active+clean; 1.3 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 109 MiB/s wr, 1.74k op/s 2023-10-01T10:41:21.867 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:21.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:21.871 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:41:21.972 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:21.997 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:22.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:22.000 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:41:22.101 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:22.127 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:22.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:22.132 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:41:22.210 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 14 16 30889 30873 137.807 101.312 0.0100871 0.00725041 2023-10-01T10:41:22.232 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:22.256 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:22.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:22.259 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:41:22.359 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:22.386 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:22.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:22.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:41:22.491 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:22.519 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:22.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:22.524 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:41:22.624 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:22.657 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:22.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:22.660 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:41:22.761 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:22.791 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:22.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:22.795 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:41:22.896 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:22.929 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:22.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:22.933 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:41:23.034 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:23.065 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:23.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:23.069 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:41:23.170 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:23.198 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:23.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:23.202 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:41:23.210 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 15 15 32644 32629 135.936 109.75 0.010626 0.00735077 2023-10-01T10:41:23.302 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:23.331 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:23.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:23.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:41:23.434 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:23.460 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:23.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:23.463 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:41:23.563 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:23.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:23 smithi078 bash[78274]: cluster 2023-10-01T10:41:21.723664+0000 mgr.y (mgr.25072) 707 : cluster 0 pgmap v615: 89 pgs: 89 active+clean; 1.6 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 134 MiB/s wr, 2.15k op/s 2023-10-01T10:41:23.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:23 smithi078 bash[75299]: cluster 2023-10-01T10:41:21.723664+0000 mgr.y (mgr.25072) 707 : cluster 0 pgmap v615: 89 pgs: 89 active+clean; 1.6 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 134 MiB/s wr, 2.15k op/s 2023-10-01T10:41:23.590 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:23.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:23.594 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:41:23.694 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:23.704 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:23 smithi173 bash[58816]: cluster 2023-10-01T10:41:21.723664+0000 mgr.y (mgr.25072) 707 : cluster 0 pgmap v615: 89 pgs: 89 active+clean; 1.6 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 134 MiB/s wr, 2.15k op/s 2023-10-01T10:41:23.731 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:23.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:23.737 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:41:23.838 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:23.863 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:23.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:23.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:41:23.968 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:23.996 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:24.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:24.000 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:41:24.101 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:24.128 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:24.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:24.132 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:41:24.210 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 16 15 34250 34235 133.713 100.375 0.0139156 0.00747289 2023-10-01T10:41:24.232 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:24.258 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:24.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:24.260 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:41:24.361 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:24.386 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:24.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:24.391 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:41:24.491 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:24.514 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:24.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:24.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:41:24.620 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:24.650 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:24.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:24.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:41:24.753 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:24.783 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:24.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:24.786 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:41:24.887 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:24.914 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:24.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:24.917 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:41:25.018 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:25.045 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:25.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:25.048 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:41:25.148 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:25.176 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:25.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:25.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:41:25.210 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 17 16 35938 35922 132.049 105.438 0.00619418 0.0075677 2023-10-01T10:41:25.280 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:25.313 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:25.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:25.317 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:41:25.417 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:25.446 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:25.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:25.450 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:41:25.482 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:25 smithi173 bash[58816]: cluster 2023-10-01T10:41:23.724183+0000 mgr.y (mgr.25072) 708 : cluster 0 pgmap v616: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 125 MiB/s wr, 2.00k op/s 2023-10-01T10:41:25.551 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:25.581 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:25.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:25.584 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:41:25.685 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:25.715 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:25.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:25.718 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:41:25.819 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:25.848 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:25.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:25.851 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:41:25.952 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:25.977 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:25.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:25.979 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:41:26.080 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:26.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:25 smithi078 bash[78274]: cluster 2023-10-01T10:41:23.724183+0000 mgr.y (mgr.25072) 708 : cluster 0 pgmap v616: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 125 MiB/s wr, 2.00k op/s 2023-10-01T10:41:26.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:25 smithi078 bash[75299]: cluster 2023-10-01T10:41:23.724183+0000 mgr.y (mgr.25072) 708 : cluster 0 pgmap v616: 89 pgs: 89 active+clean; 1.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 125 MiB/s wr, 2.00k op/s 2023-10-01T10:41:26.106 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:26.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:26.110 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:41:26.210 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 18 16 37507 37491 130.16 98.0625 0.00267808 0.00767717 2023-10-01T10:41:26.211 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:26.236 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:26.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:26.239 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:41:26.340 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:26.364 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:26.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:26.367 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:41:26.468 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:26.494 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:26.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:26.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:41:26.600 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:26.623 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:26.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:26.626 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:41:26.727 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:26.748 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:26.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:26.751 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:41:26.852 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:26.888 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:26.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:26.890 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:41:26.991 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:27.017 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:27.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:27.020 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:41:27.121 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:27.150 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:27.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:27.153 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:41:27.210 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 19 16 38808 38792 127.589 81.3125 0.00958327 0.00783284 2023-10-01T10:41:27.254 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:27.278 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:27.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:27.280 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:41:27.381 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:27.407 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:27.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:27.409 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:41:27.510 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:27.535 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:27.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:27.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:41:27.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:27 smithi078 bash[75299]: cluster 2023-10-01T10:41:25.725002+0000 mgr.y (mgr.25072) 709 : cluster 0 pgmap v617: 89 pgs: 89 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 138 MiB/s wr, 2.20k op/s 2023-10-01T10:41:27.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:27 smithi078 bash[78274]: cluster 2023-10-01T10:41:25.725002+0000 mgr.y (mgr.25072) 709 : cluster 0 pgmap v617: 89 pgs: 89 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 138 MiB/s wr, 2.20k op/s 2023-10-01T10:41:27.637 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:27.658 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:27.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:27.660 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:41:27.760 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:27.770 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:27 smithi173 bash[58816]: cluster 2023-10-01T10:41:25.725002+0000 mgr.y (mgr.25072) 709 : cluster 0 pgmap v617: 89 pgs: 89 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 138 MiB/s wr, 2.20k op/s 2023-10-01T10:41:27.795 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:27.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:27.798 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:41:27.899 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:27.929 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:27.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:27.931 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:41:28.033 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:28.064 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:28.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:28.067 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:41:28.167 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:28.190 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:28.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:28.193 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:41:28.211 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:2023-10-01T10:41:28.209956+0000 min lat: 0.00132359 max lat: 0.145466 avg lat: 0.00795209 2023-10-01T10:41:28.211 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T10:41:28.211 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 20 16 40230 40214 125.653 88.875 0.00842884 0.00795209 2023-10-01T10:41:28.294 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:28.321 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:28.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:28.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:41:28.364 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T10:41:28.410 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T10:41:28.428 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:28.450 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T10:41:28.455 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:28.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:28.457 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:41:28.493 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T10:41:28.531 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T10:41:28.558 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:28.574 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T10:41:28.586 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:28.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:28.589 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:41:28.610 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T10:41:28.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:28 smithi173 bash[58816]: audit 2023-10-01T10:41:28.323777+0000 mon.c (mon.2) 134 : audit 0 from='client.? 172.21.15.78:0/2958219136' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:41:28.654 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T10:41:28.658 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T10:41:28.658 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T10:41:28.659 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T10:41:28.659 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T10:41:28.659 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T10:41:28.659 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T10:41:28.659 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T10:41:28.659 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T10:41:28.689 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:28.705 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:28 smithi078 bash[75299]: audit 2023-10-01T10:41:28.323777+0000 mon.c (mon.2) 134 : audit 0 from='client.? 172.21.15.78:0/2958219136' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:41:28.705 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:28 smithi078 bash[78274]: audit 2023-10-01T10:41:28.323777+0000 mon.c (mon.2) 134 : audit 0 from='client.? 172.21.15.78:0/2958219136' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:41:28.722 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:28.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:28.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:41:28.829 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:28.859 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:28.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:28.862 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:41:28.962 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:28.990 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:28.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:28.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:41:29.094 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:29.127 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:29.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:29.131 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:41:29.211 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 21 16 41541 41525 123.571 81.9375 0.0107399 0.00808626 2023-10-01T10:41:29.232 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:29.262 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:29.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:29.265 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:41:29.365 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:29.389 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:29.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:29.392 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:41:29.493 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:29.499 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:29 smithi078 bash[75299]: cluster 2023-10-01T10:41:27.725795+0000 mgr.y (mgr.25072) 710 : cluster 0 pgmap v618: 89 pgs: 89 active+clean; 2.1 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-10-01T10:41:29.519 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:29.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:29.524 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:41:29.625 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:29.652 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:29.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:29.654 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:41:29.755 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:29.765 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:29 smithi173 bash[58816]: cluster 2023-10-01T10:41:27.725795+0000 mgr.y (mgr.25072) 710 : cluster 0 pgmap v618: 89 pgs: 89 active+clean; 2.1 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-10-01T10:41:29.779 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:29.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:29.781 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:41:29.870 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:29 smithi078 bash[78274]: cluster 2023-10-01T10:41:27.725795+0000 mgr.y (mgr.25072) 710 : cluster 0 pgmap v618: 89 pgs: 89 active+clean; 2.1 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-10-01T10:41:29.881 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:29.911 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:29.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:29.914 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:41:30.015 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:30.045 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:30.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:30.049 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:41:30.149 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:30.181 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:30.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:30.185 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:41:30.211 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 22 16 42991 42975 122.073 90.625 0.0114074 0.00818574 2023-10-01T10:41:30.286 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:30.313 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:30.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:30.317 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:41:30.418 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:30.452 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:30.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:30.456 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:41:30.556 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:30.566 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:41:30 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:41:30] "GET /metrics HTTP/1.1" 200 36237 "" "Prometheus/2.33.4" 2023-10-01T10:41:30.586 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:30.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:30.591 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:41:30.692 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:30.723 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:30.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:30.725 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:41:30.826 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:30.850 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:30.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:30.852 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:41:30.953 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:30.981 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:30.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:30.985 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:41:31.086 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:31.119 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:31.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:31.122 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:41:31.211 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 23 16 44290 44274 120.295 81.1875 0.00807106 0.0083071 2023-10-01T10:41:31.223 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:31.260 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:31.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:31.265 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:41:31.366 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:31.391 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:31.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:31.394 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:41:31.495 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:31.520 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:31.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:31.522 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:41:31.622 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:31.637 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:31 smithi078 bash[78274]: cluster 2023-10-01T10:41:29.726487+0000 mgr.y (mgr.25072) 711 : cluster 0 pgmap v619: 89 pgs: 89 active+clean; 2.3 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-10-01T10:41:31.638 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:31 smithi078 bash[75299]: cluster 2023-10-01T10:41:29.726487+0000 mgr.y (mgr.25072) 711 : cluster 0 pgmap v619: 89 pgs: 89 active+clean; 2.3 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-10-01T10:41:31.662 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:31.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:31.666 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:41:31.767 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:31.797 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:31.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:31.800 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:41:31.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:31 smithi173 bash[58816]: cluster 2023-10-01T10:41:29.726487+0000 mgr.y (mgr.25072) 711 : cluster 0 pgmap v619: 89 pgs: 89 active+clean; 2.3 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-10-01T10:41:31.901 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:31.936 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:31.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:31.940 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:41:32.041 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:32.074 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:32.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:32.076 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:41:32.177 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:32.202 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:32.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:32.205 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:41:32.211 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 24 16 45627 45611 118.764 83.5625 0.0122099 0.00841371 2023-10-01T10:41:32.305 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:32.336 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:32.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:32.339 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:41:32.439 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:32.468 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:32.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:32.472 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:41:32.572 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:32.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:32 smithi078 bash[78274]: audit 2023-10-01T10:41:32.021820+0000 mon.a (mon.0) 933 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:41:32.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:32 smithi078 bash[78274]: audit 2023-10-01T10:41:32.412437+0000 mon.a (mon.0) 934 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:41:32.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:32 smithi078 bash[75299]: audit 2023-10-01T10:41:32.021820+0000 mon.a (mon.0) 933 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:41:32.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:32 smithi078 bash[75299]: audit 2023-10-01T10:41:32.412437+0000 mon.a (mon.0) 934 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:41:32.612 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:32.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:32.621 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:41:32.719 INFO:teuthology.orchestra.run.smithi078.stdout:ERROR: (22) Invalid argument 2023-10-01T10:41:32.719 INFO:teuthology.orchestra.run.smithi078.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T10:41:32.722 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:32.750 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:32.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:32.753 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:41:32.853 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:32.863 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:32 smithi173 bash[58816]: audit 2023-10-01T10:41:32.021820+0000 mon.a (mon.0) 933 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:41:32.863 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:32 smithi173 bash[58816]: audit 2023-10-01T10:41:32.412437+0000 mon.a (mon.0) 934 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:41:32.881 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:32.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:32.886 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:41:32.986 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:33.016 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:33.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:33.018 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:41:33.119 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:33.145 INFO:teuthology.orchestra.run.smithi078.stdout: 2023-10-01T10:41:33.148 INFO:teuthology.orchestra.run.smithi078.stderr:dumped all 2023-10-01T10:41:33.154 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:33.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:33.157 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:41:33.211 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 25 15 46817 46802 116.991 74.4375 0.00762792 0.00854223 2023-10-01T10:41:33.258 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:33.287 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:33.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:33.290 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:41:33.391 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:33.424 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:33.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:33.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:41:33.527 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:33.532 INFO:teuthology.orchestra.run.smithi078.stderr:noscrub is unset 2023-10-01T10:41:33.552 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:33.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:33.555 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:41:33.655 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:33.679 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:33.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:33.682 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:41:33.783 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:33.790 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:33 smithi173 bash[58816]: cluster 2023-10-01T10:41:31.727069+0000 mgr.y (mgr.25072) 712 : cluster 0 pgmap v620: 89 pgs: 89 active+clean; 2.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2023-10-01T10:41:33.790 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:33 smithi173 bash[58816]: audit 2023-10-01T10:41:32.850810+0000 mon.c (mon.2) 135 : audit 1 from='client.? 172.21.15.78:0/3636034629' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T10:41:33.791 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:33 smithi173 bash[58816]: audit 2023-10-01T10:41:32.852178+0000 mon.a (mon.0) 935 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T10:41:33.803 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:33.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:33.807 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:41:33.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:33 smithi078 bash[78274]: cluster 2023-10-01T10:41:31.727069+0000 mgr.y (mgr.25072) 712 : cluster 0 pgmap v620: 89 pgs: 89 active+clean; 2.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2023-10-01T10:41:33.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:33 smithi078 bash[78274]: audit 2023-10-01T10:41:32.850810+0000 mon.c (mon.2) 135 : audit 1 from='client.? 172.21.15.78:0/3636034629' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T10:41:33.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:33 smithi078 bash[78274]: audit 2023-10-01T10:41:32.852178+0000 mon.a (mon.0) 935 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T10:41:33.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:33 smithi078 bash[75299]: cluster 2023-10-01T10:41:31.727069+0000 mgr.y (mgr.25072) 712 : cluster 0 pgmap v620: 89 pgs: 89 active+clean; 2.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2023-10-01T10:41:33.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:33 smithi078 bash[75299]: audit 2023-10-01T10:41:32.850810+0000 mon.c (mon.2) 135 : audit 1 from='client.? 172.21.15.78:0/3636034629' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T10:41:33.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:33 smithi078 bash[75299]: audit 2023-10-01T10:41:32.852178+0000 mon.a (mon.0) 935 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T10:41:33.908 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:33.936 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:33.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:33.938 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:41:34.038 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T10:41:34.038 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:34.069 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:34.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:34.073 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:41:34.174 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:34.199 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:34.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:34.201 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:41:34.211 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 26 16 48203 48187 115.821 86.5625 0.00602635 0.00862851 2023-10-01T10:41:34.302 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:34.337 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:34.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:34.341 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:41:34.441 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:34.467 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:34.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:34.472 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:41:34.572 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:34.582 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:34 smithi173 bash[58816]: audit 2023-10-01T10:41:33.121279+0000 mgr.y (mgr.25072) 713 : audit 0 from='client.45608 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:41:34.582 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:34 smithi173 bash[58816]: cluster 2023-10-01T10:41:33.513404+0000 mon.a (mon.0) 936 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:41:34.582 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:34 smithi173 bash[58816]: cluster 2023-10-01T10:41:33.521040+0000 osd.7 (osd.7) 76 : cluster 3 failed to encode map e132 with expected crc 2023-10-01T10:41:34.582 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:34 smithi173 bash[58816]: audit 2023-10-01T10:41:33.521376+0000 mon.a (mon.0) 937 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T10:41:34.582 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:34 smithi173 bash[58816]: cluster 2023-10-01T10:41:33.521564+0000 mon.a (mon.0) 938 : cluster 0 osdmap e132: 8 total, 8 up, 8 in 2023-10-01T10:41:34.582 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:34 smithi173 bash[58816]: cluster 2023-10-01T10:41:33.522246+0000 osd.7 (osd.7) 77 : cluster 3 failed to encode map e132 with expected crc 2023-10-01T10:41:34.582 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:34 smithi173 bash[58816]: cluster 2023-10-01T10:41:33.524017+0000 osd.5 (osd.5) 55 : cluster 3 failed to encode map e132 with expected crc 2023-10-01T10:41:34.582 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:34 smithi173 bash[58816]: cluster 2023-10-01T10:41:33.525957+0000 osd.5 (osd.5) 56 : cluster 3 failed to encode map e132 with expected crc 2023-10-01T10:41:34.583 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:34 smithi173 bash[58816]: cluster 2023-10-01T10:41:33.532578+0000 osd.6 (osd.6) 47 : cluster 3 failed to encode map e132 with expected crc 2023-10-01T10:41:34.599 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:34.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:34.603 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:41:34.703 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:34.743 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:34.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:34.747 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:41:34.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:34 smithi078 bash[75299]: audit 2023-10-01T10:41:33.121279+0000 mgr.y (mgr.25072) 713 : audit 0 from='client.45608 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:41:34.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:34 smithi078 bash[75299]: cluster 2023-10-01T10:41:33.513404+0000 mon.a (mon.0) 936 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:41:34.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:34 smithi078 bash[75299]: cluster 2023-10-01T10:41:33.521040+0000 osd.7 (osd.7) 76 : cluster 3 failed to encode map e132 with expected crc 2023-10-01T10:41:34.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:34 smithi078 bash[75299]: audit 2023-10-01T10:41:33.521376+0000 mon.a (mon.0) 937 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T10:41:34.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:34 smithi078 bash[75299]: cluster 2023-10-01T10:41:33.521564+0000 mon.a (mon.0) 938 : cluster 0 osdmap e132: 8 total, 8 up, 8 in 2023-10-01T10:41:34.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:34 smithi078 bash[75299]: cluster 2023-10-01T10:41:33.522246+0000 osd.7 (osd.7) 77 : cluster 3 failed to encode map e132 with expected crc 2023-10-01T10:41:34.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:34 smithi078 bash[75299]: cluster 2023-10-01T10:41:33.524017+0000 osd.5 (osd.5) 55 : cluster 3 failed to encode map e132 with expected crc 2023-10-01T10:41:34.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:34 smithi078 bash[75299]: cluster 2023-10-01T10:41:33.525957+0000 osd.5 (osd.5) 56 : cluster 3 failed to encode map e132 with expected crc 2023-10-01T10:41:34.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:34 smithi078 bash[75299]: cluster 2023-10-01T10:41:33.532578+0000 osd.6 (osd.6) 47 : cluster 3 failed to encode map e132 with expected crc 2023-10-01T10:41:34.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:34 smithi078 bash[78274]: audit 2023-10-01T10:41:33.121279+0000 mgr.y (mgr.25072) 713 : audit 0 from='client.45608 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:41:34.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:34 smithi078 bash[78274]: cluster 2023-10-01T10:41:33.513404+0000 mon.a (mon.0) 936 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:41:34.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:34 smithi078 bash[78274]: cluster 2023-10-01T10:41:33.521040+0000 osd.7 (osd.7) 76 : cluster 3 failed to encode map e132 with expected crc 2023-10-01T10:41:34.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:34 smithi078 bash[78274]: audit 2023-10-01T10:41:33.521376+0000 mon.a (mon.0) 937 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T10:41:34.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:34 smithi078 bash[78274]: cluster 2023-10-01T10:41:33.521564+0000 mon.a (mon.0) 938 : cluster 0 osdmap e132: 8 total, 8 up, 8 in 2023-10-01T10:41:34.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:34 smithi078 bash[78274]: cluster 2023-10-01T10:41:33.522246+0000 osd.7 (osd.7) 77 : cluster 3 failed to encode map e132 with expected crc 2023-10-01T10:41:34.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:34 smithi078 bash[78274]: cluster 2023-10-01T10:41:33.524017+0000 osd.5 (osd.5) 55 : cluster 3 failed to encode map e132 with expected crc 2023-10-01T10:41:34.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:34 smithi078 bash[78274]: cluster 2023-10-01T10:41:33.525957+0000 osd.5 (osd.5) 56 : cluster 3 failed to encode map e132 with expected crc 2023-10-01T10:41:34.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:34 smithi078 bash[78274]: cluster 2023-10-01T10:41:33.532578+0000 osd.6 (osd.6) 47 : cluster 3 failed to encode map e132 with expected crc 2023-10-01T10:41:34.847 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:34.874 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:34.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:34.882 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:41:34.983 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:35.008 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:35.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:35.013 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:41:35.114 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:35.138 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:35.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:35.141 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:41:35.211 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 27 16 49615 49599 114.799 88.25 0.00498707 0.00870479 2023-10-01T10:41:35.242 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:35.261 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:35.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:35.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:41:35.365 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:35.404 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:35.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:35.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:41:35.508 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:35.534 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:35.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:35.536 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:41:35.637 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:35.656 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:35.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:35.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:41:35.758 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:35.768 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:35 smithi173 bash[58816]: cluster 2023-10-01T10:41:33.730254+0000 mgr.y (mgr.25072) 714 : cluster 0 pgmap v622: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s 2023-10-01T10:41:35.788 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:35.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:35.791 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:41:35.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:35 smithi078 bash[75299]: cluster 2023-10-01T10:41:33.730254+0000 mgr.y (mgr.25072) 714 : cluster 0 pgmap v622: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s 2023-10-01T10:41:35.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:35 smithi078 bash[78274]: cluster 2023-10-01T10:41:33.730254+0000 mgr.y (mgr.25072) 714 : cluster 0 pgmap v622: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s 2023-10-01T10:41:35.892 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:35.914 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:35.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:35.918 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:41:36.019 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:36.047 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:36.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:36.051 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:41:36.134 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-01T10:41:36.152 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:36.175 INFO:teuthology.orchestra.run.smithi078.stdout:{"pg_ready":true,"pg_map":{"version":620,"stamp":"2023-10-01T10:41:31.726688+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2758940576,"num_objects":42113,"num_object_clones":0,"num_object_copies":126339,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":42113,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":42350,"num_write_kb":2696142,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":42284,"ondisk_log_size":42284,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":10376436,"kb_used_data":7908660,"kb_used_omap":30,"kb_used_meta":2467681,"kb_avail":739584780,"statfs":{"total":767960285184,"available":757334814720,"internally_reserved":0,"allocated":8098467840,"data_stored":8094414237,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31502,"internal_metadata":2526905586},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,3,3,3,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1391853568,"num_objects":21238,"num_object_clones":0,"num_object_copies":63714,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":21238,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":21238,"num_write_kb":1359232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.007427"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":118,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:49.454285+0000","last_change":"2023-10-01T10:38:16.032258+0000","last_active":"2023-10-01T10:40:49.454285+0000","last_peered":"2023-10-01T10:40:49.454285+0000","last_clean":"2023-10-01T10:40:49.454285+0000","last_became_active":"2023-10-01T10:38:16.031633+0000","last_became_peered":"2023-10-01T10:38:16.031633+0000","last_unstale":"2023-10-01T10:40:49.454285+0000","last_undegraded":"2023-10-01T10:40:49.454285+0000","last_fullsized":"2023-10-01T10:40:49.454285+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:17:02.446430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":106,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:49.452576+0000","last_change":"2023-10-01T10:38:16.025881+0000","last_active":"2023-10-01T10:40:49.452576+0000","last_peered":"2023-10-01T10:40:49.452576+0000","last_clean":"2023-10-01T10:40:49.452576+0000","last_became_active":"2023-10-01T10:38:16.025518+0000","last_became_peered":"2023-10-01T10:38:16.025518+0000","last_unstale":"2023-10-01T10:40:49.452576+0000","last_undegraded":"2023-10-01T10:40:49.452576+0000","last_fullsized":"2023-10-01T10:40:49.452576+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:48:37.826604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":84,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:49.573353+0000","last_change":"2023-10-01T10:38:16.025075+0000","last_active":"2023-10-01T10:40:49.573353+0000","last_peered":"2023-10-01T10:40:49.573353+0000","last_clean":"2023-10-01T10:40:49.573353+0000","last_became_active":"2023-10-01T10:38:16.024735+0000","last_became_peered":"2023-10-01T10:38:16.024735+0000","last_unstale":"2023-10-01T10:40:49.573353+0000","last_undegraded":"2023-10-01T10:40:49.573353+0000","last_fullsized":"2023-10-01T10:40:49.573353+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:38:01.450741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":88,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:51.288346+0000","last_change":"2023-10-01T10:37:42.772008+0000","last_active":"2023-10-01T10:40:51.288346+0000","last_peered":"2023-10-01T10:40:51.288346+0000","last_clean":"2023-10-01T10:40:51.288346+0000","last_became_active":"2023-10-01T10:37:42.771244+0000","last_became_peered":"2023-10-01T10:37:42.771244+0000","last_unstale":"2023-10-01T10:40:51.288346+0000","last_undegraded":"2023-10-01T10:40:51.288346+0000","last_fullsized":"2023-10-01T10:40:51.288346+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:38:55.180790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":107,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:49.451987+0000","last_change":"2023-10-01T10:37:42.766600+0000","last_active":"2023-10-01T10:40:49.451987+0000","last_peered":"2023-10-01T10:40:49.451987+0000","last_clean":"2023-10-01T10:40:49.451987+0000","last_became_active":"2023-10-01T10:37:42.766051+0000","last_became_peered":"2023-10-01T10:37:42.766051+0000","last_unstale":"2023-10-01T10:40:49.451987+0000","last_undegraded":"2023-10-01T10:40:49.451987+0000","last_fullsized":"2023-10-01T10:40:49.451987+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:51:17.707963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":96,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:49.658595+0000","last_change":"2023-10-01T10:32:31.575109+0000","last_active":"2023-10-01T10:40:49.658595+0000","last_peered":"2023-10-01T10:40:49.658595+0000","last_clean":"2023-10-01T10:40:49.658595+0000","last_became_active":"2023-10-01T10:32:31.574729+0000","last_became_peered":"2023-10-01T10:32:31.574729+0000","last_unstale":"2023-10-01T10:40:49.658595+0000","last_undegraded":"2023-10-01T10:40:49.658595+0000","last_fullsized":"2023-10-01T10:40:49.658595+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":86,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:51.288895+0000","last_change":"2023-10-01T10:37:42.766667+0000","last_active":"2023-10-01T10:40:51.288895+0000","last_peered":"2023-10-01T10:40:51.288895+0000","last_clean":"2023-10-01T10:40:51.288895+0000","last_became_active":"2023-10-01T10:37:42.766369+0000","last_became_peered":"2023-10-01T10:37:42.766369+0000","last_unstale":"2023-10-01T10:40:51.288895+0000","last_undegraded":"2023-10-01T10:40:51.288895+0000","last_fullsized":"2023-10-01T10:40:51.288895+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:02.708788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":91,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:49.451858+0000","last_change":"2023-10-01T10:37:42.765925+0000","last_active":"2023-10-01T10:40:49.451858+0000","last_peered":"2023-10-01T10:40:49.451858+0000","last_clean":"2023-10-01T10:40:49.451858+0000","last_became_active":"2023-10-01T10:37:42.765218+0000","last_became_peered":"2023-10-01T10:37:42.765218+0000","last_unstale":"2023-10-01T10:40:49.451858+0000","last_undegraded":"2023-10-01T10:40:49.451858+0000","last_fullsized":"2023-10-01T10:40:49.451858+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T22:10:34.338031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":98,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:51.292215+0000","last_change":"2023-10-01T10:32:31.579805+0000","last_active":"2023-10-01T10:40:51.292215+0000","last_peered":"2023-10-01T10:40:51.292215+0000","last_clean":"2023-10-01T10:40:51.292215+0000","last_became_active":"2023-10-01T10:32:31.579423+0000","last_became_peered":"2023-10-01T10:32:31.579423+0000","last_unstale":"2023-10-01T10:40:51.292215+0000","last_undegraded":"2023-10-01T10:40:51.292215+0000","last_fullsized":"2023-10-01T10:40:51.292215+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":75,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:49.452058+0000","last_change":"2023-10-01T10:37:42.772531+0000","last_active":"2023-10-01T10:40:49.452058+0000","last_peered":"2023-10-01T10:40:49.452058+0000","last_clean":"2023-10-01T10:40:49.452058+0000","last_became_active":"2023-10-01T10:37:42.772216+0000","last_became_peered":"2023-10-01T10:37:42.772216+0000","last_unstale":"2023-10-01T10:40:49.452058+0000","last_undegraded":"2023-10-01T10:40:49.452058+0000","last_fullsized":"2023-10-01T10:40:49.452058+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:01:53.373628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":116,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:49.458490+0000","last_change":"2023-10-01T10:38:16.027085+0000","last_active":"2023-10-01T10:40:49.458490+0000","last_peered":"2023-10-01T10:40:49.458490+0000","last_clean":"2023-10-01T10:40:49.458490+0000","last_became_active":"2023-10-01T10:38:16.026555+0000","last_became_peered":"2023-10-01T10:38:16.026555+0000","last_unstale":"2023-10-01T10:40:49.458490+0000","last_undegraded":"2023-10-01T10:40:49.458490+0000","last_fullsized":"2023-10-01T10:40:49.458490+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":69,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:49.573724+0000","last_change":"2023-10-01T10:37:05.018755+0000","last_active":"2023-10-01T10:40:49.573724+0000","last_peered":"2023-10-01T10:40:49.573724+0000","last_clean":"2023-10-01T10:40:49.573724+0000","last_became_active":"2023-10-01T10:37:05.018446+0000","last_became_peered":"2023-10-01T10:37:05.018446+0000","last_unstale":"2023-10-01T10:40:49.573724+0000","last_undegraded":"2023-10-01T10:40:49.573724+0000","last_fullsized":"2023-10-01T10:40:49.573724+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:13:53.740503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":112,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:49.455016+0000","last_change":"2023-10-01T10:38:16.030826+0000","last_active":"2023-10-01T10:40:49.455016+0000","last_peered":"2023-10-01T10:40:49.455016+0000","last_clean":"2023-10-01T10:40:49.455016+0000","last_became_active":"2023-10-01T10:38:16.030277+0000","last_became_peered":"2023-10-01T10:38:16.030277+0000","last_unstale":"2023-10-01T10:40:49.455016+0000","last_undegraded":"2023-10-01T10:40:49.455016+0000","last_fullsized":"2023-10-01T10:40:49.455016+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:31:43.040424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":86,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:49.454670+0000","last_change":"2023-10-01T10:38:16.034741+0000","last_active":"2023-10-01T10:40:49.454670+0000","last_peered":"2023-10-01T10:40:49.454670+0000","last_clean":"2023-10-01T10:40:49.454670+0000","last_became_active":"2023-10-01T10:38:16.034407+0000","last_became_peered":"2023-10-01T10:38:16.034407+0000","last_unstale":"2023-10-01T10:40:49.454670+0000","last_undegraded":"2023-10-01T10:40:49.454670+0000","last_fullsized":"2023-10-01T10:40:49.454670+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:47:29.911477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":115,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:49.458221+0000","last_change":"2023-10-01T10:38:16.026777+0000","last_active":"2023-10-01T10:40:49.458221+0000","last_peered":"2023-10-01T10:40:49.458221+0000","last_clean":"2023-10-01T10:40:49.458221+0000","last_became_active":"2023-10-01T10:38:16.026371+0000","last_became_peered":"2023-10-01T10:38:16.026371+0000","last_unstale":"2023-10-01T10:40:49.458221+0000","last_undegraded":"2023-10-01T10:40:49.458221+0000","last_fullsized":"2023-10-01T10:40:49.458221+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":114,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:49.452500+0000","last_change":"2023-10-01T10:37:42.772319+0000","last_active":"2023-10-01T10:40:49.452500+0000","last_peered":"2023-10-01T10:40:49.452500+0000","last_clean":"2023-10-01T10:40:49.452500+0000","last_became_active":"2023-10-01T10:37:42.771977+0000","last_became_peered":"2023-10-01T10:37:42.771977+0000","last_unstale":"2023-10-01T10:40:49.452500+0000","last_undegraded":"2023-10-01T10:40:49.452500+0000","last_fullsized":"2023-10-01T10:40:49.452500+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:45:12.783214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":86,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:51.288043+0000","last_change":"2023-10-01T10:37:42.767706+0000","last_active":"2023-10-01T10:40:51.288043+0000","last_peered":"2023-10-01T10:40:51.288043+0000","last_clean":"2023-10-01T10:40:51.288043+0000","last_became_active":"2023-10-01T10:37:42.767211+0000","last_became_peered":"2023-10-01T10:37:42.767211+0000","last_unstale":"2023-10-01T10:40:51.288043+0000","last_undegraded":"2023-10-01T10:40:51.288043+0000","last_fullsized":"2023-10-01T10:40:51.288043+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:37:20.357407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":97,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:51.292026+0000","last_change":"2023-10-01T10:37:42.768642+0000","last_active":"2023-10-01T10:40:51.292026+0000","last_peered":"2023-10-01T10:40:51.292026+0000","last_clean":"2023-10-01T10:40:51.292026+0000","last_became_active":"2023-10-01T10:37:42.768239+0000","last_became_peered":"2023-10-01T10:37:42.768239+0000","last_unstale":"2023-10-01T10:40:51.292026+0000","last_undegraded":"2023-10-01T10:40:51.292026+0000","last_fullsized":"2023-10-01T10:40:51.292026+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":98,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:49.457905+0000","last_change":"2023-10-01T10:38:16.028563+0000","last_active":"2023-10-01T10:40:49.457905+0000","last_peered":"2023-10-01T10:40:49.457905+0000","last_clean":"2023-10-01T10:40:49.457905+0000","last_became_active":"2023-10-01T10:38:16.028162+0000","last_became_peered":"2023-10-01T10:38:16.028162+0000","last_unstale":"2023-10-01T10:40:49.457905+0000","last_undegraded":"2023-10-01T10:40:49.457905+0000","last_fullsized":"2023-10-01T10:40:49.457905+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":87,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:51.288667+0000","last_change":"2023-10-01T10:37:42.772247+0000","last_active":"2023-10-01T10:40:51.288667+0000","last_peered":"2023-10-01T10:40:51.288667+0000","last_clean":"2023-10-01T10:40:51.288667+0000","last_became_active":"2023-10-01T10:37:42.771697+0000","last_became_peered":"2023-10-01T10:37:42.771697+0000","last_unstale":"2023-10-01T10:40:51.288667+0000","last_undegraded":"2023-10-01T10:40:51.288667+0000","last_fullsized":"2023-10-01T10:40:51.288667+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:23:15.018248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":105,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:49.657718+0000","last_change":"2023-10-01T10:28:20.211798+0000","last_active":"2023-10-01T10:40:49.657718+0000","last_peered":"2023-10-01T10:40:49.657718+0000","last_clean":"2023-10-01T10:40:49.657718+0000","last_became_active":"2023-10-01T10:28:20.210791+0000","last_became_peered":"2023-10-01T10:28:20.210791+0000","last_unstale":"2023-10-01T10:40:49.657718+0000","last_undegraded":"2023-10-01T10:40:49.657718+0000","last_fullsized":"2023-10-01T10:40:49.657718+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":91,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:49.452800+0000","last_change":"2023-10-01T10:37:42.773127+0000","last_active":"2023-10-01T10:40:49.452800+0000","last_peered":"2023-10-01T10:40:49.452800+0000","last_clean":"2023-10-01T10:40:49.452800+0000","last_became_active":"2023-10-01T10:37:42.772102+0000","last_became_peered":"2023-10-01T10:37:42.772102+0000","last_unstale":"2023-10-01T10:40:49.452800+0000","last_undegraded":"2023-10-01T10:40:49.452800+0000","last_fullsized":"2023-10-01T10:40:49.452800+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:23:26.202295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"131'2574","reported_seq":2586,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-10-01T10:41:29.869215+0000","last_change":"2023-10-01T10:40:48.458343+0000","last_active":"2023-10-01T10:41:29.869215+0000","last_peered":"2023-10-01T10:41:29.869215+0000","last_clean":"2023-10-01T10:41:29.869215+0000","last_became_active":"2023-10-01T10:40:48.458009+0000","last_became_peered":"2023-10-01T10:40:48.458009+0000","last_unstale":"2023-10-01T10:41:29.869215+0000","last_undegraded":"2023-10-01T10:41:29.869215+0000","last_fullsized":"2023-10-01T10:41:29.869215+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":128,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_clean_scrub_stamp":"2023-10-01T10:40:47.426361+0000","objects_scrubbed":0,"log_size":2574,"log_dups_size":0,"ondisk_log_size":2574,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:58:33.314892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":168689664,"num_objects":2574,"num_object_clones":0,"num_object_copies":7722,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2574,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2574,"num_write_kb":164736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":114,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:49.458654+0000","last_change":"2023-10-01T10:38:16.026955+0000","last_active":"2023-10-01T10:40:49.458654+0000","last_peered":"2023-10-01T10:40:49.458654+0000","last_clean":"2023-10-01T10:40:49.458654+0000","last_became_active":"2023-10-01T10:38:16.026552+0000","last_became_peered":"2023-10-01T10:38:16.026552+0000","last_unstale":"2023-10-01T10:40:49.458654+0000","last_undegraded":"2023-10-01T10:40:49.458654+0000","last_fullsized":"2023-10-01T10:40:49.458654+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":82,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:51.291467+0000","last_change":"2023-10-01T10:15:40.318702+0000","last_active":"2023-10-01T10:40:51.291467+0000","last_peered":"2023-10-01T10:40:51.291467+0000","last_clean":"2023-10-01T10:40:51.291467+0000","last_became_active":"2023-10-01T10:15:40.318333+0000","last_became_peered":"2023-10-01T10:15:40.318333+0000","last_unstale":"2023-10-01T10:40:51.291467+0000","last_undegraded":"2023-10-01T10:40:51.291467+0000","last_fullsized":"2023-10-01T10:40:51.291467+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"131'2727","reported_seq":2737,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-10-01T10:41:30.406973+0000","last_change":"2023-10-01T10:40:48.488223+0000","last_active":"2023-10-01T10:41:30.406973+0000","last_peered":"2023-10-01T10:41:30.406973+0000","last_clean":"2023-10-01T10:41:30.406973+0000","last_became_active":"2023-10-01T10:40:48.487910+0000","last_became_peered":"2023-10-01T10:40:48.487910+0000","last_unstale":"2023-10-01T10:41:30.406973+0000","last_undegraded":"2023-10-01T10:41:30.406973+0000","last_fullsized":"2023-10-01T10:41:30.406973+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":128,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_clean_scrub_stamp":"2023-10-01T10:40:47.426361+0000","objects_scrubbed":0,"log_size":2727,"log_dups_size":0,"ondisk_log_size":2727,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:18:52.686502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178716672,"num_objects":2727,"num_object_clones":0,"num_object_copies":8181,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2727,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2727,"num_write_kb":174528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":115,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:49.458439+0000","last_change":"2023-10-01T10:38:16.027753+0000","last_active":"2023-10-01T10:40:49.458439+0000","last_peered":"2023-10-01T10:40:49.458439+0000","last_clean":"2023-10-01T10:40:49.458439+0000","last_became_active":"2023-10-01T10:38:16.027384+0000","last_became_peered":"2023-10-01T10:38:16.027384+0000","last_unstale":"2023-10-01T10:40:49.458439+0000","last_undegraded":"2023-10-01T10:40:49.458439+0000","last_fullsized":"2023-10-01T10:40:49.458439+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":120,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:49.454582+0000","last_change":"2023-10-01T10:38:16.033203+0000","last_active":"2023-10-01T10:40:49.454582+0000","last_peered":"2023-10-01T10:40:49.454582+0000","last_clean":"2023-10-01T10:40:49.454582+0000","last_became_active":"2023-10-01T10:38:16.032681+0000","last_became_peered":"2023-10-01T10:38:16.032681+0000","last_unstale":"2023-10-01T10:40:49.454582+0000","last_undegraded":"2023-10-01T10:40:49.454582+0000","last_fullsized":"2023-10-01T10:40:49.454582+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:42:36.642284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"131'2611","reported_seq":2622,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-10-01T10:41:28.819928+0000","last_change":"2023-10-01T10:40:48.459909+0000","last_active":"2023-10-01T10:41:28.819928+0000","last_peered":"2023-10-01T10:41:28.819928+0000","last_clean":"2023-10-01T10:41:28.819928+0000","last_became_active":"2023-10-01T10:40:48.459525+0000","last_became_peered":"2023-10-01T10:40:48.459525+0000","last_unstale":"2023-10-01T10:41:28.819928+0000","last_undegraded":"2023-10-01T10:41:28.819928+0000","last_fullsized":"2023-10-01T10:41:28.819928+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":128,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_clean_scrub_stamp":"2023-10-01T10:40:47.426361+0000","objects_scrubbed":0,"log_size":2611,"log_dups_size":0,"ondisk_log_size":2611,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":171114496,"num_objects":2611,"num_object_clones":0,"num_object_copies":7833,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2611,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2611,"num_write_kb":167104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":84,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:49.458354+0000","last_change":"2023-10-01T10:15:39.293587+0000","last_active":"2023-10-01T10:40:49.458354+0000","last_peered":"2023-10-01T10:40:49.458354+0000","last_clean":"2023-10-01T10:40:49.458354+0000","last_became_active":"2023-10-01T10:15:39.293273+0000","last_became_peered":"2023-10-01T10:15:39.293273+0000","last_unstale":"2023-10-01T10:40:49.458354+0000","last_undegraded":"2023-10-01T10:40:49.458354+0000","last_fullsized":"2023-10-01T10:40:49.458354+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":96,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:49.658119+0000","last_change":"2023-10-01T10:32:31.571971+0000","last_active":"2023-10-01T10:40:49.658119+0000","last_peered":"2023-10-01T10:40:49.658119+0000","last_clean":"2023-10-01T10:40:49.658119+0000","last_became_active":"2023-10-01T10:32:31.571410+0000","last_became_peered":"2023-10-01T10:32:31.571410+0000","last_unstale":"2023-10-01T10:40:49.658119+0000","last_undegraded":"2023-10-01T10:40:49.658119+0000","last_fullsized":"2023-10-01T10:40:49.658119+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"131'2705","reported_seq":2715,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-10-01T10:41:28.796940+0000","last_change":"2023-10-01T10:40:48.487102+0000","last_active":"2023-10-01T10:41:28.796940+0000","last_peered":"2023-10-01T10:41:28.796940+0000","last_clean":"2023-10-01T10:41:28.796940+0000","last_became_active":"2023-10-01T10:40:48.486622+0000","last_became_peered":"2023-10-01T10:40:48.486622+0000","last_unstale":"2023-10-01T10:41:28.796940+0000","last_undegraded":"2023-10-01T10:41:28.796940+0000","last_fullsized":"2023-10-01T10:41:28.796940+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":128,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_clean_scrub_stamp":"2023-10-01T10:40:47.426361+0000","objects_scrubbed":0,"log_size":2705,"log_dups_size":0,"ondisk_log_size":2705,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":177274880,"num_objects":2705,"num_object_clones":0,"num_object_copies":8115,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2705,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2705,"num_write_kb":173120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":98,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:51.291618+0000","last_change":"2023-10-01T10:37:42.765733+0000","last_active":"2023-10-01T10:40:51.291618+0000","last_peered":"2023-10-01T10:40:51.291618+0000","last_clean":"2023-10-01T10:40:51.291618+0000","last_became_active":"2023-10-01T10:37:42.765311+0000","last_became_peered":"2023-10-01T10:37:42.765311+0000","last_unstale":"2023-10-01T10:40:51.291618+0000","last_undegraded":"2023-10-01T10:40:51.291618+0000","last_fullsized":"2023-10-01T10:40:51.291618+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":102,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:51.288495+0000","last_change":"2023-10-01T10:38:16.026348+0000","last_active":"2023-10-01T10:40:51.288495+0000","last_peered":"2023-10-01T10:40:51.288495+0000","last_clean":"2023-10-01T10:40:51.288495+0000","last_became_active":"2023-10-01T10:38:16.025747+0000","last_became_peered":"2023-10-01T10:38:16.025747+0000","last_unstale":"2023-10-01T10:40:51.288495+0000","last_undegraded":"2023-10-01T10:40:51.288495+0000","last_fullsized":"2023-10-01T10:40:51.288495+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:28:18.934480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"131'2775","reported_seq":2787,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-10-01T10:41:30.889562+0000","last_change":"2023-10-01T10:40:48.487554+0000","last_active":"2023-10-01T10:41:30.889562+0000","last_peered":"2023-10-01T10:41:30.889562+0000","last_clean":"2023-10-01T10:41:30.889562+0000","last_became_active":"2023-10-01T10:40:48.487223+0000","last_became_peered":"2023-10-01T10:40:48.487223+0000","last_unstale":"2023-10-01T10:41:30.889562+0000","last_undegraded":"2023-10-01T10:41:30.889562+0000","last_fullsized":"2023-10-01T10:41:30.889562+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":128,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_clean_scrub_stamp":"2023-10-01T10:40:47.426361+0000","objects_scrubbed":0,"log_size":2775,"log_dups_size":0,"ondisk_log_size":2775,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:18:35.800937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181862400,"num_objects":2775,"num_object_clones":0,"num_object_copies":8325,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2775,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2775,"num_write_kb":177600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":94,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:49.452421+0000","last_change":"2023-10-01T10:38:16.022493+0000","last_active":"2023-10-01T10:40:49.452421+0000","last_peered":"2023-10-01T10:40:49.452421+0000","last_clean":"2023-10-01T10:40:49.452421+0000","last_became_active":"2023-10-01T10:38:16.022033+0000","last_became_peered":"2023-10-01T10:38:16.022033+0000","last_unstale":"2023-10-01T10:40:49.452421+0000","last_undegraded":"2023-10-01T10:40:49.452421+0000","last_fullsized":"2023-10-01T10:40:49.452421+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:49:05.262763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":105,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:49.455072+0000","last_change":"2023-10-01T10:38:16.030613+0000","last_active":"2023-10-01T10:40:49.455072+0000","last_peered":"2023-10-01T10:40:49.455072+0000","last_clean":"2023-10-01T10:40:49.455072+0000","last_became_active":"2023-10-01T10:38:16.028975+0000","last_became_peered":"2023-10-01T10:38:16.028975+0000","last_unstale":"2023-10-01T10:40:49.455072+0000","last_undegraded":"2023-10-01T10:40:49.455072+0000","last_fullsized":"2023-10-01T10:40:49.455072+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:46:07.287911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"131'2725","reported_seq":2737,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-10-01T10:41:30.406977+0000","last_change":"2023-10-01T10:40:48.488324+0000","last_active":"2023-10-01T10:41:30.406977+0000","last_peered":"2023-10-01T10:41:30.406977+0000","last_clean":"2023-10-01T10:41:30.406977+0000","last_became_active":"2023-10-01T10:40:48.487379+0000","last_became_peered":"2023-10-01T10:40:48.487379+0000","last_unstale":"2023-10-01T10:41:30.406977+0000","last_undegraded":"2023-10-01T10:41:30.406977+0000","last_fullsized":"2023-10-01T10:41:30.406977+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":128,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_clean_scrub_stamp":"2023-10-01T10:40:47.426361+0000","objects_scrubbed":0,"log_size":2725,"log_dups_size":0,"ondisk_log_size":2725,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:09:55.286907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178585600,"num_objects":2725,"num_object_clones":0,"num_object_copies":8175,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2725,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2725,"num_write_kb":174400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":112,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:49.658391+0000","last_change":"2023-10-01T10:38:16.030250+0000","last_active":"2023-10-01T10:40:49.658391+0000","last_peered":"2023-10-01T10:40:49.658391+0000","last_clean":"2023-10-01T10:40:49.658391+0000","last_became_active":"2023-10-01T10:38:16.029898+0000","last_became_peered":"2023-10-01T10:38:16.029898+0000","last_unstale":"2023-10-01T10:40:49.658391+0000","last_undegraded":"2023-10-01T10:40:49.658391+0000","last_fullsized":"2023-10-01T10:40:49.658391+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":110,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:49.454989+0000","last_change":"2023-10-01T10:38:16.035906+0000","last_active":"2023-10-01T10:40:49.454989+0000","last_peered":"2023-10-01T10:40:49.454989+0000","last_clean":"2023-10-01T10:40:49.454989+0000","last_became_active":"2023-10-01T10:38:16.035383+0000","last_became_peered":"2023-10-01T10:38:16.035383+0000","last_unstale":"2023-10-01T10:40:49.454989+0000","last_undegraded":"2023-10-01T10:40:49.454989+0000","last_fullsized":"2023-10-01T10:40:49.454989+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:44:41.500097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"131'2485","reported_seq":2496,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-10-01T10:41:28.274420+0000","last_change":"2023-10-01T10:40:48.461083+0000","last_active":"2023-10-01T10:41:28.274420+0000","last_peered":"2023-10-01T10:41:28.274420+0000","last_clean":"2023-10-01T10:41:28.274420+0000","last_became_active":"2023-10-01T10:40:48.460671+0000","last_became_peered":"2023-10-01T10:40:48.460671+0000","last_unstale":"2023-10-01T10:41:28.274420+0000","last_undegraded":"2023-10-01T10:41:28.274420+0000","last_fullsized":"2023-10-01T10:41:28.274420+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":128,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_clean_scrub_stamp":"2023-10-01T10:40:47.426361+0000","objects_scrubbed":0,"log_size":2485,"log_dups_size":0,"ondisk_log_size":2485,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162856960,"num_objects":2485,"num_object_clones":0,"num_object_copies":7455,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2485,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2485,"num_write_kb":159040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":102,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:49.454153+0000","last_change":"2023-10-01T10:38:16.026611+0000","last_active":"2023-10-01T10:40:49.454153+0000","last_peered":"2023-10-01T10:40:49.454153+0000","last_clean":"2023-10-01T10:40:49.454153+0000","last_became_active":"2023-10-01T10:38:16.026267+0000","last_became_peered":"2023-10-01T10:38:16.026267+0000","last_unstale":"2023-10-01T10:40:49.454153+0000","last_undegraded":"2023-10-01T10:40:49.454153+0000","last_fullsized":"2023-10-01T10:40:49.454153+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:06.135288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":86,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:49.455322+0000","last_change":"2023-10-01T10:38:16.032162+0000","last_active":"2023-10-01T10:40:49.455322+0000","last_peered":"2023-10-01T10:40:49.455322+0000","last_clean":"2023-10-01T10:40:49.455322+0000","last_became_active":"2023-10-01T10:38:16.031785+0000","last_became_peered":"2023-10-01T10:38:16.031785+0000","last_unstale":"2023-10-01T10:40:49.455322+0000","last_undegraded":"2023-10-01T10:40:49.455322+0000","last_fullsized":"2023-10-01T10:40:49.455322+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:38:43.003626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"131'2836","reported_seq":2846,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-10-01T10:41:30.750089+0000","last_change":"2023-10-01T10:40:48.485646+0000","last_active":"2023-10-01T10:41:30.750089+0000","last_peered":"2023-10-01T10:41:30.750089+0000","last_clean":"2023-10-01T10:41:30.750089+0000","last_became_active":"2023-10-01T10:40:48.485316+0000","last_became_peered":"2023-10-01T10:40:48.485316+0000","last_unstale":"2023-10-01T10:41:30.750089+0000","last_undegraded":"2023-10-01T10:41:30.750089+0000","last_fullsized":"2023-10-01T10:41:30.750089+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":128,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_clean_scrub_stamp":"2023-10-01T10:40:47.426361+0000","objects_scrubbed":0,"log_size":2836,"log_dups_size":0,"ondisk_log_size":2836,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:33:20.144231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185860096,"num_objects":2836,"num_object_clones":0,"num_object_copies":8508,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2836,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2836,"num_write_kb":181504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":92,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:49.452089+0000","last_change":"2023-10-01T10:37:42.772398+0000","last_active":"2023-10-01T10:40:49.452089+0000","last_peered":"2023-10-01T10:40:49.452089+0000","last_clean":"2023-10-01T10:40:49.452089+0000","last_became_active":"2023-10-01T10:37:42.772084+0000","last_became_peered":"2023-10-01T10:37:42.772084+0000","last_unstale":"2023-10-01T10:40:49.452089+0000","last_undegraded":"2023-10-01T10:40:49.452089+0000","last_fullsized":"2023-10-01T10:40:49.452089+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:32:43.846419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":110,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:51.059831+0000","last_change":"2023-10-01T10:38:16.019755+0000","last_active":"2023-10-01T10:40:51.059831+0000","last_peered":"2023-10-01T10:40:51.059831+0000","last_clean":"2023-10-01T10:40:51.059831+0000","last_became_active":"2023-10-01T10:38:16.019397+0000","last_became_peered":"2023-10-01T10:38:16.019397+0000","last_unstale":"2023-10-01T10:40:51.059831+0000","last_undegraded":"2023-10-01T10:40:51.059831+0000","last_fullsized":"2023-10-01T10:40:51.059831+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:42:56.623225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.f","version":"131'2554","reported_seq":2564,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-10-01T10:41:28.817832+0000","last_change":"2023-10-01T10:40:48.455948+0000","last_active":"2023-10-01T10:41:28.817832+0000","last_peered":"2023-10-01T10:41:28.817832+0000","last_clean":"2023-10-01T10:41:28.817832+0000","last_became_active":"2023-10-01T10:40:48.455402+0000","last_became_peered":"2023-10-01T10:40:48.455402+0000","last_unstale":"2023-10-01T10:41:28.817832+0000","last_undegraded":"2023-10-01T10:41:28.817832+0000","last_fullsized":"2023-10-01T10:41:28.817832+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":128,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_clean_scrub_stamp":"2023-10-01T10:40:47.426361+0000","objects_scrubbed":0,"log_size":2554,"log_dups_size":0,"ondisk_log_size":2554,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":167378944,"num_objects":2554,"num_object_clones":0,"num_object_copies":7662,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2554,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2554,"num_write_kb":163456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":104,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:50.369337+0000","last_change":"2023-10-01T10:32:31.561108+0000","last_active":"2023-10-01T10:40:50.369337+0000","last_peered":"2023-10-01T10:40:50.369337+0000","last_clean":"2023-10-01T10:40:50.369337+0000","last_became_active":"2023-10-01T10:32:31.560791+0000","last_became_peered":"2023-10-01T10:32:31.560791+0000","last_unstale":"2023-10-01T10:40:50.369337+0000","last_undegraded":"2023-10-01T10:40:50.369337+0000","last_fullsized":"2023-10-01T10:40:50.369337+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:42:47.477671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":113,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:51.292091+0000","last_change":"2023-10-01T10:37:42.766814+0000","last_active":"2023-10-01T10:40:51.292091+0000","last_peered":"2023-10-01T10:40:51.292091+0000","last_clean":"2023-10-01T10:40:51.292091+0000","last_became_active":"2023-10-01T10:37:42.766451+0000","last_became_peered":"2023-10-01T10:37:42.766451+0000","last_unstale":"2023-10-01T10:40:51.292091+0000","last_undegraded":"2023-10-01T10:40:51.292091+0000","last_fullsized":"2023-10-01T10:40:51.292091+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":70,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:50.369282+0000","last_change":"2023-10-01T10:38:16.034357+0000","last_active":"2023-10-01T10:40:50.369282+0000","last_peered":"2023-10-01T10:40:50.369282+0000","last_clean":"2023-10-01T10:40:50.369282+0000","last_became_active":"2023-10-01T10:38:16.033905+0000","last_became_peered":"2023-10-01T10:38:16.033905+0000","last_unstale":"2023-10-01T10:40:50.369282+0000","last_undegraded":"2023-10-01T10:40:50.369282+0000","last_fullsized":"2023-10-01T10:40:50.369282+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:47:58.369835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"131'2605","reported_seq":2616,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-10-01T10:41:29.744076+0000","last_change":"2023-10-01T10:40:48.456619+0000","last_active":"2023-10-01T10:41:29.744076+0000","last_peered":"2023-10-01T10:41:29.744076+0000","last_clean":"2023-10-01T10:41:29.744076+0000","last_became_active":"2023-10-01T10:40:48.456280+0000","last_became_peered":"2023-10-01T10:40:48.456280+0000","last_unstale":"2023-10-01T10:41:29.744076+0000","last_undegraded":"2023-10-01T10:41:29.744076+0000","last_fullsized":"2023-10-01T10:41:29.744076+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":128,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_clean_scrub_stamp":"2023-10-01T10:40:47.426361+0000","objects_scrubbed":0,"log_size":2605,"log_dups_size":0,"ondisk_log_size":2605,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:34:57.206244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":170721280,"num_objects":2605,"num_object_clones":0,"num_object_copies":7815,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2605,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2605,"num_write_kb":166720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":102,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:51.291641+0000","last_change":"2023-10-01T10:32:31.572482+0000","last_active":"2023-10-01T10:40:51.291641+0000","last_peered":"2023-10-01T10:40:51.291641+0000","last_clean":"2023-10-01T10:40:51.291641+0000","last_became_active":"2023-10-01T10:32:31.571993+0000","last_became_peered":"2023-10-01T10:32:31.571993+0000","last_unstale":"2023-10-01T10:40:51.291641+0000","last_undegraded":"2023-10-01T10:40:51.291641+0000","last_fullsized":"2023-10-01T10:40:51.291641+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":72,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:49.455205+0000","last_change":"2023-10-01T10:38:16.030202+0000","last_active":"2023-10-01T10:40:49.455205+0000","last_peered":"2023-10-01T10:40:49.455205+0000","last_clean":"2023-10-01T10:40:49.455205+0000","last_became_active":"2023-10-01T10:38:16.029917+0000","last_became_peered":"2023-10-01T10:38:16.029917+0000","last_unstale":"2023-10-01T10:40:49.455205+0000","last_undegraded":"2023-10-01T10:40:49.455205+0000","last_fullsized":"2023-10-01T10:40:49.455205+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:20:06.763909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":85,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:50.369359+0000","last_change":"2023-10-01T10:37:42.761153+0000","last_active":"2023-10-01T10:40:50.369359+0000","last_peered":"2023-10-01T10:40:50.369359+0000","last_clean":"2023-10-01T10:40:50.369359+0000","last_became_active":"2023-10-01T10:37:42.760715+0000","last_became_peered":"2023-10-01T10:37:42.760715+0000","last_unstale":"2023-10-01T10:40:50.369359+0000","last_undegraded":"2023-10-01T10:40:50.369359+0000","last_fullsized":"2023-10-01T10:40:50.369359+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:52:50.237174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"131'2522","reported_seq":2533,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-10-01T10:41:28.798783+0000","last_change":"2023-10-01T10:40:48.457495+0000","last_active":"2023-10-01T10:41:28.798783+0000","last_peered":"2023-10-01T10:41:28.798783+0000","last_clean":"2023-10-01T10:41:28.798783+0000","last_became_active":"2023-10-01T10:40:48.457080+0000","last_became_peered":"2023-10-01T10:40:48.457080+0000","last_unstale":"2023-10-01T10:41:28.798783+0000","last_undegraded":"2023-10-01T10:41:28.798783+0000","last_fullsized":"2023-10-01T10:41:28.798783+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":128,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_clean_scrub_stamp":"2023-10-01T10:40:47.426361+0000","objects_scrubbed":0,"log_size":2522,"log_dups_size":0,"ondisk_log_size":2522,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165281792,"num_objects":2522,"num_object_clones":0,"num_object_copies":7566,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2522,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2522,"num_write_kb":161408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":102,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-10-01T10:41:04.743548+0000","last_change":"2023-10-01T10:37:42.762524+0000","last_active":"2023-10-01T10:41:04.743548+0000","last_peered":"2023-10-01T10:41:04.743548+0000","last_clean":"2023-10-01T10:41:04.743548+0000","last_became_active":"2023-10-01T10:37:42.762209+0000","last_became_peered":"2023-10-01T10:37:42.762209+0000","last_unstale":"2023-10-01T10:41:04.743548+0000","last_undegraded":"2023-10-01T10:41:04.743548+0000","last_fullsized":"2023-10-01T10:41:04.743548+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:45:38.171397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":70,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:49.573724+0000","last_change":"2023-10-01T10:37:05.019330+0000","last_active":"2023-10-01T10:40:49.573724+0000","last_peered":"2023-10-01T10:40:49.573724+0000","last_clean":"2023-10-01T10:40:49.573724+0000","last_became_active":"2023-10-01T10:37:05.018842+0000","last_became_peered":"2023-10-01T10:37:05.018842+0000","last_unstale":"2023-10-01T10:40:49.573724+0000","last_undegraded":"2023-10-01T10:40:49.573724+0000","last_fullsized":"2023-10-01T10:40:49.573724+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:48:09.551917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":99,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-10-01T10:41:04.863804+0000","last_change":"2023-10-01T10:32:31.569162+0000","last_active":"2023-10-01T10:41:04.863804+0000","last_peered":"2023-10-01T10:41:04.863804+0000","last_clean":"2023-10-01T10:41:04.863804+0000","last_became_active":"2023-10-01T10:32:31.568956+0000","last_became_peered":"2023-10-01T10:32:31.568956+0000","last_unstale":"2023-10-01T10:41:04.863804+0000","last_undegraded":"2023-10-01T10:41:04.863804+0000","last_fullsized":"2023-10-01T10:41:04.863804+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:02:08.285304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"131'2696","reported_seq":2707,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-10-01T10:41:30.764428+0000","last_change":"2023-10-01T10:40:48.454746+0000","last_active":"2023-10-01T10:41:30.764428+0000","last_peered":"2023-10-01T10:41:30.764428+0000","last_clean":"2023-10-01T10:41:30.764428+0000","last_became_active":"2023-10-01T10:40:48.454409+0000","last_became_peered":"2023-10-01T10:40:48.454409+0000","last_unstale":"2023-10-01T10:41:30.764428+0000","last_undegraded":"2023-10-01T10:41:30.764428+0000","last_fullsized":"2023-10-01T10:41:30.764428+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":128,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_clean_scrub_stamp":"2023-10-01T10:40:47.426361+0000","objects_scrubbed":0,"log_size":2696,"log_dups_size":0,"ondisk_log_size":2696,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:07:17.878925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":176685056,"num_objects":2696,"num_object_clones":0,"num_object_copies":8088,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2696,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2696,"num_write_kb":172544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":102,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:51.291514+0000","last_change":"2023-10-01T10:37:05.024009+0000","last_active":"2023-10-01T10:40:51.291514+0000","last_peered":"2023-10-01T10:40:51.291514+0000","last_clean":"2023-10-01T10:40:51.291514+0000","last_became_active":"2023-10-01T10:37:05.023458+0000","last_became_peered":"2023-10-01T10:37:05.023458+0000","last_unstale":"2023-10-01T10:40:51.291514+0000","last_undegraded":"2023-10-01T10:40:51.291514+0000","last_fullsized":"2023-10-01T10:40:51.291514+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'139","reported_seq":334,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:49.457530+0000","last_change":"2023-10-01T10:28:20.217386+0000","last_active":"2023-10-01T10:40:49.457530+0000","last_peered":"2023-10-01T10:40:49.457530+0000","last_clean":"2023-10-01T10:40:49.457530+0000","last_became_active":"2023-10-01T10:28:20.216803+0000","last_became_peered":"2023-10-01T10:28:20.216803+0000","last_unstale":"2023-10-01T10:40:49.457530+0000","last_undegraded":"2023-10-01T10:40:49.457530+0000","last_fullsized":"2023-10-01T10:40:49.457530+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:09:17.221082+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:09:17.221082+0000","last_clean_scrub_stamp":"2023-10-01T10:09:17.221082+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":74,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:49.452240+0000","last_change":"2023-10-01T10:37:42.765100+0000","last_active":"2023-10-01T10:40:49.452240+0000","last_peered":"2023-10-01T10:40:49.452240+0000","last_clean":"2023-10-01T10:40:49.452240+0000","last_became_active":"2023-10-01T10:37:42.764761+0000","last_became_peered":"2023-10-01T10:37:42.764761+0000","last_unstale":"2023-10-01T10:40:49.452240+0000","last_undegraded":"2023-10-01T10:40:49.452240+0000","last_fullsized":"2023-10-01T10:40:49.452240+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:24:56.907377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":96,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:49.658503+0000","last_change":"2023-10-01T10:38:16.031255+0000","last_active":"2023-10-01T10:40:49.658503+0000","last_peered":"2023-10-01T10:40:49.658503+0000","last_clean":"2023-10-01T10:40:49.658503+0000","last_became_active":"2023-10-01T10:38:16.030886+0000","last_became_peered":"2023-10-01T10:38:16.030886+0000","last_unstale":"2023-10-01T10:40:49.658503+0000","last_undegraded":"2023-10-01T10:40:49.658503+0000","last_fullsized":"2023-10-01T10:40:49.658503+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"131'2579","reported_seq":2591,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-10-01T10:41:29.761973+0000","last_change":"2023-10-01T10:40:48.487771+0000","last_active":"2023-10-01T10:41:29.761973+0000","last_peered":"2023-10-01T10:41:29.761973+0000","last_clean":"2023-10-01T10:41:29.761973+0000","last_became_active":"2023-10-01T10:40:48.487435+0000","last_became_peered":"2023-10-01T10:40:48.487435+0000","last_unstale":"2023-10-01T10:41:29.761973+0000","last_undegraded":"2023-10-01T10:41:29.761973+0000","last_fullsized":"2023-10-01T10:41:29.761973+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":128,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_clean_scrub_stamp":"2023-10-01T10:40:47.426361+0000","objects_scrubbed":0,"log_size":2579,"log_dups_size":0,"ondisk_log_size":2579,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:50:58.475377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169017344,"num_objects":2579,"num_object_clones":0,"num_object_copies":7737,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2579,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2579,"num_write_kb":165056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":117,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:49.457503+0000","last_change":"2023-10-01T10:32:31.565868+0000","last_active":"2023-10-01T10:40:49.457503+0000","last_peered":"2023-10-01T10:40:49.457503+0000","last_clean":"2023-10-01T10:40:49.457503+0000","last_became_active":"2023-10-01T10:32:31.565428+0000","last_became_peered":"2023-10-01T10:32:31.565428+0000","last_unstale":"2023-10-01T10:40:49.457503+0000","last_undegraded":"2023-10-01T10:40:49.457503+0000","last_fullsized":"2023-10-01T10:40:49.457503+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":102,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:51.287878+0000","last_change":"2023-10-01T10:38:16.021968+0000","last_active":"2023-10-01T10:40:51.287878+0000","last_peered":"2023-10-01T10:40:51.287878+0000","last_clean":"2023-10-01T10:40:51.287878+0000","last_became_active":"2023-10-01T10:38:16.021539+0000","last_became_peered":"2023-10-01T10:38:16.021539+0000","last_unstale":"2023-10-01T10:40:51.287878+0000","last_undegraded":"2023-10-01T10:40:51.287878+0000","last_fullsized":"2023-10-01T10:40:51.287878+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:12:53.105747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":103,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-10-01T10:41:04.846763+0000","last_change":"2023-10-01T10:37:05.017431+0000","last_active":"2023-10-01T10:41:04.846763+0000","last_peered":"2023-10-01T10:41:04.846763+0000","last_clean":"2023-10-01T10:41:04.846763+0000","last_became_active":"2023-10-01T10:37:05.017094+0000","last_became_peered":"2023-10-01T10:37:05.017094+0000","last_unstale":"2023-10-01T10:41:04.846763+0000","last_undegraded":"2023-10-01T10:41:04.846763+0000","last_fullsized":"2023-10-01T10:41:04.846763+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:10:40.664686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"131'2575","reported_seq":2586,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-10-01T10:41:28.277589+0000","last_change":"2023-10-01T10:40:48.488899+0000","last_active":"2023-10-01T10:41:28.277589+0000","last_peered":"2023-10-01T10:41:28.277589+0000","last_clean":"2023-10-01T10:41:28.277589+0000","last_became_active":"2023-10-01T10:40:48.488506+0000","last_became_peered":"2023-10-01T10:40:48.488506+0000","last_unstale":"2023-10-01T10:41:28.277589+0000","last_undegraded":"2023-10-01T10:41:28.277589+0000","last_fullsized":"2023-10-01T10:41:28.277589+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":128,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_clean_scrub_stamp":"2023-10-01T10:40:47.426361+0000","objects_scrubbed":0,"log_size":2575,"log_dups_size":0,"ondisk_log_size":2575,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":168755200,"num_objects":2575,"num_object_clones":0,"num_object_copies":7725,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2575,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2575,"num_write_kb":164800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":117,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:49.458331+0000","last_change":"2023-10-01T10:38:16.032774+0000","last_active":"2023-10-01T10:40:49.458331+0000","last_peered":"2023-10-01T10:40:49.458331+0000","last_clean":"2023-10-01T10:40:49.458331+0000","last_became_active":"2023-10-01T10:38:16.032283+0000","last_became_peered":"2023-10-01T10:38:16.032283+0000","last_unstale":"2023-10-01T10:40:49.458331+0000","last_undegraded":"2023-10-01T10:40:49.458331+0000","last_fullsized":"2023-10-01T10:40:49.458331+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":70,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:50.369381+0000","last_change":"2023-10-01T10:37:05.021228+0000","last_active":"2023-10-01T10:40:50.369381+0000","last_peered":"2023-10-01T10:40:50.369381+0000","last_clean":"2023-10-01T10:40:50.369381+0000","last_became_active":"2023-10-01T10:37:05.020888+0000","last_became_peered":"2023-10-01T10:37:05.020888+0000","last_unstale":"2023-10-01T10:40:50.369381+0000","last_undegraded":"2023-10-01T10:40:50.369381+0000","last_fullsized":"2023-10-01T10:40:50.369381+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:59:57.641247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":73,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:51.288266+0000","last_change":"2023-10-01T10:28:20.220812+0000","last_active":"2023-10-01T10:40:51.288266+0000","last_peered":"2023-10-01T10:40:51.288266+0000","last_clean":"2023-10-01T10:40:51.288266+0000","last_became_active":"2023-10-01T10:28:20.215962+0000","last_became_peered":"2023-10-01T10:28:20.215962+0000","last_unstale":"2023-10-01T10:40:51.288266+0000","last_undegraded":"2023-10-01T10:40:51.288266+0000","last_fullsized":"2023-10-01T10:40:51.288266+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:35:15.275097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"131'2447","reported_seq":2458,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-10-01T10:41:28.250891+0000","last_change":"2023-10-01T10:40:48.461167+0000","last_active":"2023-10-01T10:41:28.250891+0000","last_peered":"2023-10-01T10:41:28.250891+0000","last_clean":"2023-10-01T10:41:28.250891+0000","last_became_active":"2023-10-01T10:40:48.460822+0000","last_became_peered":"2023-10-01T10:40:48.460822+0000","last_unstale":"2023-10-01T10:41:28.250891+0000","last_undegraded":"2023-10-01T10:41:28.250891+0000","last_fullsized":"2023-10-01T10:41:28.250891+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":128,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_clean_scrub_stamp":"2023-10-01T10:40:47.426361+0000","objects_scrubbed":0,"log_size":2447,"log_dups_size":0,"ondisk_log_size":2447,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160366592,"num_objects":2447,"num_object_clones":0,"num_object_copies":7341,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2447,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2447,"num_write_kb":156608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":79,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:49.452857+0000","last_change":"2023-10-01T10:37:42.773999+0000","last_active":"2023-10-01T10:40:49.452857+0000","last_peered":"2023-10-01T10:40:49.452857+0000","last_clean":"2023-10-01T10:40:49.452857+0000","last_became_active":"2023-10-01T10:37:42.773355+0000","last_became_peered":"2023-10-01T10:37:42.773355+0000","last_unstale":"2023-10-01T10:40:49.452857+0000","last_undegraded":"2023-10-01T10:40:49.452857+0000","last_fullsized":"2023-10-01T10:40:49.452857+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:46:51.317151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":112,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-10-01T10:41:04.863763+0000","last_change":"2023-10-01T10:38:16.034193+0000","last_active":"2023-10-01T10:41:04.863763+0000","last_peered":"2023-10-01T10:41:04.863763+0000","last_clean":"2023-10-01T10:41:04.863763+0000","last_became_active":"2023-10-01T10:38:16.033703+0000","last_became_peered":"2023-10-01T10:38:16.033703+0000","last_unstale":"2023-10-01T10:41:04.863763+0000","last_undegraded":"2023-10-01T10:41:04.863763+0000","last_fullsized":"2023-10-01T10:41:04.863763+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:21:26.061595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":76,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:49.451989+0000","last_change":"2023-10-01T10:37:42.773538+0000","last_active":"2023-10-01T10:40:49.451989+0000","last_peered":"2023-10-01T10:40:49.451989+0000","last_clean":"2023-10-01T10:40:49.451989+0000","last_became_active":"2023-10-01T10:37:42.771863+0000","last_became_peered":"2023-10-01T10:37:42.771863+0000","last_unstale":"2023-10-01T10:40:49.451989+0000","last_undegraded":"2023-10-01T10:40:49.451989+0000","last_fullsized":"2023-10-01T10:40:49.451989+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:46:46.698627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"131'2673","reported_seq":2685,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-10-01T10:41:30.705944+0000","last_change":"2023-10-01T10:40:48.486926+0000","last_active":"2023-10-01T10:41:30.705944+0000","last_peered":"2023-10-01T10:41:30.705944+0000","last_clean":"2023-10-01T10:41:30.705944+0000","last_became_active":"2023-10-01T10:40:48.486674+0000","last_became_peered":"2023-10-01T10:40:48.486674+0000","last_unstale":"2023-10-01T10:41:30.705944+0000","last_undegraded":"2023-10-01T10:41:30.705944+0000","last_fullsized":"2023-10-01T10:41:30.705944+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":128,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_clean_scrub_stamp":"2023-10-01T10:40:47.426361+0000","objects_scrubbed":0,"log_size":2673,"log_dups_size":0,"ondisk_log_size":2673,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:07:45.799626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":175177728,"num_objects":2673,"num_object_clones":0,"num_object_copies":8019,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2673,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2673,"num_write_kb":171072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":101,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:49.658636+0000","last_change":"2023-10-01T10:37:05.023034+0000","last_active":"2023-10-01T10:40:49.658636+0000","last_peered":"2023-10-01T10:40:49.658636+0000","last_clean":"2023-10-01T10:40:49.658636+0000","last_became_active":"2023-10-01T10:37:05.022670+0000","last_became_peered":"2023-10-01T10:37:05.022670+0000","last_unstale":"2023-10-01T10:40:49.658636+0000","last_undegraded":"2023-10-01T10:40:49.658636+0000","last_fullsized":"2023-10-01T10:40:49.658636+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":106,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-10-01T10:41:04.846686+0000","last_change":"2023-10-01T10:38:16.027472+0000","last_active":"2023-10-01T10:41:04.846686+0000","last_peered":"2023-10-01T10:41:04.846686+0000","last_clean":"2023-10-01T10:41:04.846686+0000","last_became_active":"2023-10-01T10:38:16.026810+0000","last_became_peered":"2023-10-01T10:38:16.026810+0000","last_unstale":"2023-10-01T10:41:04.846686+0000","last_undegraded":"2023-10-01T10:41:04.846686+0000","last_fullsized":"2023-10-01T10:41:04.846686+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:30:06.978187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":70,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:49.454436+0000","last_change":"2023-10-01T10:38:16.035705+0000","last_active":"2023-10-01T10:40:49.454436+0000","last_peered":"2023-10-01T10:40:49.454436+0000","last_clean":"2023-10-01T10:40:49.454436+0000","last_became_active":"2023-10-01T10:38:16.035389+0000","last_became_peered":"2023-10-01T10:38:16.035389+0000","last_unstale":"2023-10-01T10:40:49.454436+0000","last_undegraded":"2023-10-01T10:40:49.454436+0000","last_fullsized":"2023-10-01T10:40:49.454436+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:16:10.201409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":90,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:51.288631+0000","last_change":"2023-10-01T10:38:16.028491+0000","last_active":"2023-10-01T10:40:51.288631+0000","last_peered":"2023-10-01T10:40:51.288631+0000","last_clean":"2023-10-01T10:40:51.288631+0000","last_became_active":"2023-10-01T10:38:16.028175+0000","last_became_peered":"2023-10-01T10:38:16.028175+0000","last_unstale":"2023-10-01T10:40:51.288631+0000","last_undegraded":"2023-10-01T10:40:51.288631+0000","last_fullsized":"2023-10-01T10:40:51.288631+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:44:29.083129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":111,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:51.059560+0000","last_change":"2023-10-01T10:37:42.762360+0000","last_active":"2023-10-01T10:40:51.059560+0000","last_peered":"2023-10-01T10:40:51.059560+0000","last_clean":"2023-10-01T10:40:51.059560+0000","last_became_active":"2023-10-01T10:37:42.761936+0000","last_became_peered":"2023-10-01T10:37:42.761936+0000","last_unstale":"2023-10-01T10:40:51.059560+0000","last_undegraded":"2023-10-01T10:40:51.059560+0000","last_fullsized":"2023-10-01T10:40:51.059560+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:50.637582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":115,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:51.291749+0000","last_change":"2023-10-01T10:38:16.037809+0000","last_active":"2023-10-01T10:40:51.291749+0000","last_peered":"2023-10-01T10:40:51.291749+0000","last_clean":"2023-10-01T10:40:51.291749+0000","last_became_active":"2023-10-01T10:38:16.037318+0000","last_became_peered":"2023-10-01T10:38:16.037318+0000","last_unstale":"2023-10-01T10:40:51.291749+0000","last_undegraded":"2023-10-01T10:40:51.291749+0000","last_fullsized":"2023-10-01T10:40:51.291749+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":120,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:49.454853+0000","last_change":"2023-10-01T10:38:16.034282+0000","last_active":"2023-10-01T10:40:49.454853+0000","last_peered":"2023-10-01T10:40:49.454853+0000","last_clean":"2023-10-01T10:40:49.454853+0000","last_became_active":"2023-10-01T10:38:16.033820+0000","last_became_peered":"2023-10-01T10:38:16.033820+0000","last_unstale":"2023-10-01T10:40:49.454853+0000","last_undegraded":"2023-10-01T10:40:49.454853+0000","last_fullsized":"2023-10-01T10:40:49.454853+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:48:23.815832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":100,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:51.291760+0000","last_change":"2023-10-01T10:38:16.038077+0000","last_active":"2023-10-01T10:40:51.291760+0000","last_peered":"2023-10-01T10:40:51.291760+0000","last_clean":"2023-10-01T10:40:51.291760+0000","last_became_active":"2023-10-01T10:38:16.037313+0000","last_became_peered":"2023-10-01T10:38:16.037313+0000","last_unstale":"2023-10-01T10:40:51.291760+0000","last_undegraded":"2023-10-01T10:40:51.291760+0000","last_fullsized":"2023-10-01T10:40:51.291760+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":120,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:49.454477+0000","last_change":"2023-10-01T10:38:16.025600+0000","last_active":"2023-10-01T10:40:49.454477+0000","last_peered":"2023-10-01T10:40:49.454477+0000","last_clean":"2023-10-01T10:40:49.454477+0000","last_became_active":"2023-10-01T10:38:16.025252+0000","last_became_peered":"2023-10-01T10:38:16.025252+0000","last_unstale":"2023-10-01T10:40:49.454477+0000","last_undegraded":"2023-10-01T10:40:49.454477+0000","last_fullsized":"2023-10-01T10:40:49.454477+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:24:18.871963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":107,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:49.452625+0000","last_change":"2023-10-01T10:37:42.761243+0000","last_active":"2023-10-01T10:40:49.452625+0000","last_peered":"2023-10-01T10:40:49.452625+0000","last_clean":"2023-10-01T10:40:49.452625+0000","last_became_active":"2023-10-01T10:37:42.760834+0000","last_became_peered":"2023-10-01T10:37:42.760834+0000","last_unstale":"2023-10-01T10:40:49.452625+0000","last_undegraded":"2023-10-01T10:40:49.452625+0000","last_fullsized":"2023-10-01T10:40:49.452625+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:37:21.976095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":120,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:49.452031+0000","last_change":"2023-10-01T10:37:42.765142+0000","last_active":"2023-10-01T10:40:49.452031+0000","last_peered":"2023-10-01T10:40:49.452031+0000","last_clean":"2023-10-01T10:40:49.452031+0000","last_became_active":"2023-10-01T10:37:42.764770+0000","last_became_peered":"2023-10-01T10:37:42.764770+0000","last_unstale":"2023-10-01T10:40:49.452031+0000","last_undegraded":"2023-10-01T10:40:49.452031+0000","last_fullsized":"2023-10-01T10:40:49.452031+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:23:33.527876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":91,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:51.288768+0000","last_change":"2023-10-01T10:37:05.021621+0000","last_active":"2023-10-01T10:40:51.288768+0000","last_peered":"2023-10-01T10:40:51.288768+0000","last_clean":"2023-10-01T10:40:51.288768+0000","last_became_active":"2023-10-01T10:37:05.021173+0000","last_became_peered":"2023-10-01T10:37:05.021173+0000","last_unstale":"2023-10-01T10:40:51.288768+0000","last_undegraded":"2023-10-01T10:40:51.288768+0000","last_fullsized":"2023-10-01T10:40:51.288768+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:13:25.364499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":86,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-10-01T10:40:49.454470+0000","last_change":"2023-10-01T10:38:16.031427+0000","last_active":"2023-10-01T10:40:49.454470+0000","last_peered":"2023-10-01T10:40:49.454470+0000","last_clean":"2023-10-01T10:40:49.454470+0000","last_became_active":"2023-10-01T10:38:16.029786+0000","last_became_peered":"2023-10-01T10:38:16.029786+0000","last_unstale":"2023-10-01T10:40:49.454470+0000","last_undegraded":"2023-10-01T10:40:49.454470+0000","last_fullsized":"2023-10-01T10:40:49.454470+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:53:22.796191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30319,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":2758344704,"num_objects":42089,"num_object_clones":0,"num_object_copies":126267,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":42089,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":42089,"num_write_kb":2693696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8224243712,"data_stored":8224243712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":42089,"ondisk_log_size":42089,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430554,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2003692,"kb_used_data":1409892,"kb_used_omap":0,"kb_used_meta":593792,"kb_avail":91741460,"statfs":{"total":95995035648,"available":93943255040,"internally_reserved":0,"allocated":1443729408,"data_stored":1443251663,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":608043008},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,2,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:40:42 2023","interfaces":[{"interface":"back","average":{"1min":0.97299999999999998,"5min":1.169,"15min":1.577},"min":{"1min":0.73799999999999999,"5min":0.34999999999999998,"15min":0.27500000000000002},"max":{"1min":1.181,"5min":7.8470000000000004,"15min":14.974},"last":3.3170000000000002},{"interface":"front","average":{"1min":0.998,"5min":1.153,"15min":1.6439999999999999},"min":{"1min":0.76800000000000002,"5min":0.39800000000000002,"15min":0.29999999999999999},"max":{"1min":1.298,"5min":6.96,"15min":15.029},"last":0.38}]},{"osd":1,"last update":"Sun Oct 1 10:40:53 2023","interfaces":[{"interface":"back","average":{"1min":0.80900000000000005,"5min":0.93899999999999995,"15min":1.1950000000000001},"min":{"1min":0.47099999999999997,"5min":0.38700000000000001,"15min":0.309},"max":{"1min":1.2989999999999999,"5min":6.8739999999999997,"15min":8.1180000000000003},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.88200000000000001,"5min":1.083,"15min":1.508},"min":{"1min":0.55800000000000005,"5min":0.45900000000000002,"15min":0.318},"max":{"1min":1.2090000000000001,"5min":11.266,"15min":11.266},"last":0.48199999999999998}]},{"osd":2,"last update":"Sun Oct 1 10:41:08 2023","interfaces":[{"interface":"back","average":{"1min":0.90900000000000003,"5min":0.98299999999999998,"15min":1.018},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.238,"5min":1.2629999999999999,"15min":1.2629999999999999},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.90800000000000003,"5min":0.92200000000000004,"15min":0.93799999999999994},"min":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":1.841,"5min":1.841,"15min":1.841},"last":3.3610000000000002}]},{"osd":3,"last update":"Sun Oct 1 10:40:48 2023","interfaces":[{"interface":"back","average":{"1min":1.0149999999999999,"5min":0.98799999999999999,"15min":0.97399999999999998},"min":{"1min":0.69099999999999995,"5min":0.55700000000000005,"15min":0.55700000000000005},"max":{"1min":1.387,"5min":1.3879999999999999,"15min":1.3879999999999999},"last":0.89100000000000001},{"interface":"front","average":{"1min":0.97599999999999998,"5min":0.95199999999999996,"15min":0.94799999999999995},"min":{"1min":0.66900000000000004,"5min":0.64400000000000002,"15min":0.64400000000000002},"max":{"1min":1.2689999999999999,"5min":1.2689999999999999,"15min":1.2689999999999999},"last":3.407}]},{"osd":4,"last update":"Sun Oct 1 10:41:22 2023","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":0.96999999999999997,"15min":0.93400000000000005},"min":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"max":{"1min":2.4390000000000001,"5min":2.4390000000000001,"15min":2.4390000000000001},"last":0.67900000000000005},{"interface":"front","average":{"1min":1.0609999999999999,"5min":1.161,"15min":1.177},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":1.5620000000000001,"5min":1.5620000000000001,"15min":1.5620000000000001},"last":0.70999999999999996}]},{"osd":5,"last update":"Sun Oct 1 10:40:53 2023","interfaces":[{"interface":"back","average":{"1min":1.0569999999999999,"5min":1.139,"15min":1.202},"min":{"1min":0.56000000000000005,"5min":0.47299999999999998,"15min":0.29699999999999999},"max":{"1min":1.3680000000000001,"5min":5.8230000000000004,"15min":10.106999999999999},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.999,"5min":1.046,"15min":1.1519999999999999},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.34799999999999998},"max":{"1min":1.478,"5min":4.8970000000000002,"15min":6.2050000000000001},"last":3.3849999999999998}]},{"osd":6,"last update":"Sun Oct 1 10:40:53 2023","interfaces":[{"interface":"back","average":{"1min":1.1339999999999999,"5min":1.2809999999999999,"15min":1.2989999999999999},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.32800000000000001},"max":{"1min":1.444,"5min":7.3300000000000001,"15min":7.3300000000000001},"last":0.75900000000000001},{"interface":"front","average":{"1min":1.1200000000000001,"5min":1.161,"15min":1.2829999999999999},"min":{"1min":0.63900000000000001,"5min":0.58199999999999996,"15min":0.33900000000000002},"max":{"1min":1.3959999999999999,"5min":6.9370000000000003,"15min":9.8339999999999996},"last":0.69699999999999995}]}]},{"osd":6,"up_from":42,"seq":180388626788,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":780636,"kb_used_data":479316,"kb_used_omap":0,"kb_used_meta":301312,"kb_avail":92964516,"statfs":{"total":95995035648,"available":95195664384,"internally_reserved":0,"allocated":490819584,"data_stored":490357762,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308543488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:40:34 2023","interfaces":[{"interface":"back","average":{"1min":0.97899999999999998,"5min":1.014,"15min":1.4199999999999999},"min":{"1min":0.51300000000000001,"5min":0.29399999999999998,"15min":0.28499999999999998},"max":{"1min":1.2689999999999999,"5min":7.1139999999999999,"15min":9.1300000000000008},"last":0.29399999999999998},{"interface":"front","average":{"1min":0.89000000000000001,"5min":1.0169999999999999,"15min":1.423},"min":{"1min":0.57299999999999995,"5min":0.29999999999999999,"15min":0.28899999999999998},"max":{"1min":1.1879999999999999,"5min":4.2119999999999997,"15min":7.3109999999999999},"last":1.0449999999999999}]},{"osd":1,"last update":"Sun Oct 1 10:40:42 2023","interfaces":[{"interface":"back","average":{"1min":1.0169999999999999,"5min":1.0489999999999999,"15min":1.1539999999999999},"min":{"1min":0.78500000000000003,"5min":0.37,"15min":0.26600000000000001},"max":{"1min":1.306,"5min":4.532,"15min":7.9100000000000001},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.94499999999999995,"5min":1.0009999999999999,"15min":1.4159999999999999},"min":{"1min":0.67700000000000005,"5min":0.34399999999999997,"15min":0.33300000000000002},"max":{"1min":1.3580000000000001,"5min":5.1340000000000003,"15min":7.4199999999999999},"last":1.0800000000000001}]},{"osd":2,"last update":"Sun Oct 1 10:41:10 2023","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":0.89200000000000002,"15min":0.874},"min":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"max":{"1min":1.1850000000000001,"5min":1.296,"15min":1.296},"last":2.548},{"interface":"front","average":{"1min":0.97999999999999998,"5min":0.96099999999999997,"15min":0.94599999999999995},"min":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.2569999999999999,"5min":1.361,"15min":1.361},"last":1.1759999999999999}]},{"osd":3,"last update":"Sun Oct 1 10:40:48 2023","interfaces":[{"interface":"back","average":{"1min":0.96799999999999997,"5min":0.98799999999999999,"15min":0.98299999999999998},"min":{"1min":0.74099999999999999,"5min":0.69799999999999995,"15min":0.69799999999999995},"max":{"1min":1.2190000000000001,"5min":1.2529999999999999,"15min":1.2529999999999999},"last":3.0270000000000001},{"interface":"front","average":{"1min":1.04,"5min":1.0229999999999999,"15min":1.0189999999999999},"min":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"max":{"1min":1.3069999999999999,"5min":1.3069999999999999,"15min":1.3069999999999999},"last":1.2190000000000001}]},{"osd":4,"last update":"Sun Oct 1 10:41:23 2023","interfaces":[{"interface":"back","average":{"1min":0.95999999999999996,"5min":0.99399999999999999,"15min":1.0109999999999999},"min":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"max":{"1min":1.8779999999999999,"5min":1.8779999999999999,"15min":1.8779999999999999},"last":2.4340000000000002},{"interface":"front","average":{"1min":0.96999999999999997,"5min":1.0609999999999999,"15min":1.089},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":1.4890000000000001,"5min":1.4890000000000001,"15min":1.4890000000000001},"last":2.5960000000000001}]},{"osd":5,"last update":"Sun Oct 1 10:40:53 2023","interfaces":[{"interface":"back","average":{"1min":1.0669999999999999,"5min":1.167,"15min":1.238},"min":{"1min":0.92700000000000005,"5min":0.46100000000000002,"15min":0.38100000000000001},"max":{"1min":1.387,"5min":5.5880000000000001,"15min":8.0150000000000006},"last":0.53700000000000003},{"interface":"front","average":{"1min":1.149,"5min":1.2330000000000001,"15min":1.2390000000000001},"min":{"1min":0.61499999999999999,"5min":0.38,"15min":0.34399999999999997},"max":{"1min":1.423,"5min":10.842000000000001,"15min":10.842000000000001},"last":2.262}]},{"osd":7,"last update":"Sun Oct 1 10:40:34 2023","interfaces":[{"interface":"back","average":{"1min":1.129,"5min":1.107,"15min":1.0640000000000001},"min":{"1min":0.86799999999999999,"5min":0.39700000000000002,"15min":0.371},"max":{"1min":1.466,"5min":4.3499999999999996,"15min":4.867},"last":1.3919999999999999},{"interface":"front","average":{"1min":1.1699999999999999,"5min":1.2470000000000001,"15min":1.321},"min":{"1min":0.80900000000000005,"5min":0.53000000000000003,"15min":0.34799999999999998},"max":{"1min":1.444,"5min":5.3470000000000004,"15min":11.031000000000001},"last":3.569}]}]},{"osd":1,"up_from":101,"seq":433791697006,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1761624,"kb_used_data":1551440,"kb_used_omap":4,"kb_used_meta":210171,"kb_avail":91983528,"statfs":{"total":95995035648,"available":94191132672,"internally_reserved":0,"allocated":1588674560,"data_stored":1588105259,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":215216014},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:40:49 2023","interfaces":[{"interface":"back","average":{"1min":0.69799999999999995,"5min":0.76200000000000001,"15min":1.905},"min":{"1min":0.48199999999999998,"5min":0.29999999999999999,"15min":0.21299999999999999},"max":{"1min":0.93700000000000006,"5min":3.8330000000000002,"15min":11.071999999999999},"last":0.753},{"interface":"front","average":{"1min":0.73199999999999998,"5min":0.873,"15min":1.8600000000000001},"min":{"1min":0.43099999999999999,"5min":0.21199999999999999,"15min":0.20499999999999999},"max":{"1min":1.1539999999999999,"5min":7.9900000000000002,"15min":9.3230000000000004},"last":2.153}]},{"osd":2,"last update":"Sun Oct 1 10:41:16 2023","interfaces":[{"interface":"back","average":{"1min":0.93100000000000005,"5min":0.75,"15min":0.69499999999999995},"min":{"1min":0.52000000000000002,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":3.2290000000000001,"5min":3.2290000000000001,"15min":3.2290000000000001},"last":0.443},{"interface":"front","average":{"1min":0.90500000000000003,"5min":0.80800000000000005,"15min":0.78000000000000003},"min":{"1min":0.51700000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":3.0899999999999999,"5min":3.0899999999999999,"15min":3.0899999999999999},"last":2.121}]},{"osd":3,"last update":"Sun Oct 1 10:40:49 2023","interfaces":[{"interface":"back","average":{"1min":0.76500000000000001,"5min":0.69799999999999995,"15min":0.68400000000000005},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":1.1259999999999999,"5min":1.1259999999999999,"15min":1.1259999999999999},"last":1.6339999999999999},{"interface":"front","average":{"1min":0.76700000000000002,"5min":0.74099999999999999,"15min":0.749},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":1.1639999999999999,"5min":1.1639999999999999,"15min":1.1639999999999999},"last":0.47199999999999998}]},{"osd":4,"last update":"Sun Oct 1 10:41:19 2023","interfaces":[{"interface":"back","average":{"1min":0.93600000000000005,"5min":0.92700000000000005,"15min":0.89700000000000002},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":3.0640000000000001,"5min":3.0640000000000001,"15min":3.0640000000000001},"last":2.0209999999999999},{"interface":"front","average":{"1min":1.1559999999999999,"5min":1.046,"15min":1.02},"min":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"max":{"1min":3.4550000000000001,"5min":3.4550000000000001,"15min":3.4550000000000001},"last":2.1970000000000001}]},{"osd":5,"last update":"Sun Oct 1 10:40:49 2023","interfaces":[{"interface":"back","average":{"1min":1.0109999999999999,"5min":1.073,"15min":1.6799999999999999},"min":{"1min":0.70399999999999996,"5min":0.48199999999999998,"15min":0.35799999999999998},"max":{"1min":1.272,"5min":5.798,"15min":8.9429999999999996},"last":1.931},{"interface":"front","average":{"1min":1.1160000000000001,"5min":1.089,"15min":2.2040000000000002},"min":{"1min":0.68500000000000005,"5min":0.40699999999999997,"15min":0.33600000000000002},"max":{"1min":1.706,"5min":4.0839999999999996,"15min":11.353999999999999},"last":0.72699999999999998}]},{"osd":6,"last update":"Sun Oct 1 10:40:49 2023","interfaces":[{"interface":"back","average":{"1min":1.0980000000000001,"5min":1.0800000000000001,"15min":1.895},"min":{"1min":0.75800000000000001,"5min":0.46100000000000002,"15min":0.37},"max":{"1min":1.6379999999999999,"5min":4.1079999999999997,"15min":8.6630000000000003},"last":0.69399999999999995},{"interface":"front","average":{"1min":1.1040000000000001,"5min":1.1379999999999999,"15min":1.8660000000000001},"min":{"1min":0.81999999999999995,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":1.738,"5min":6,"15min":7.1369999999999996},"last":2.173}]},{"osd":7,"last update":"Sun Oct 1 10:40:49 2023","interfaces":[{"interface":"back","average":{"1min":1.1779999999999999,"5min":1.214,"15min":1.823},"min":{"1min":0.67300000000000004,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":1.8839999999999999,"5min":6.0510000000000002,"15min":7.1680000000000001},"last":2.3809999999999998},{"interface":"front","average":{"1min":1.107,"5min":1.1519999999999999,"15min":1.6879999999999999},"min":{"1min":0.70599999999999996,"5min":0.52500000000000002,"15min":0.48899999999999999},"max":{"1min":1.609,"5min":5.9109999999999996,"15min":9.0749999999999993},"last":2.5840000000000001}]}]},{"osd":0,"up_from":86,"seq":369367187616,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1000292,"kb_used_data":663900,"kb_used_omap":5,"kb_used_meta":336378,"kb_avail":92744860,"statfs":{"total":95995035648,"available":94970736640,"internally_reserved":0,"allocated":679833600,"data_stored":679251183,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":344451706},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 10:40:49 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.94599999999999995,"15min":1.5580000000000001},"min":{"1min":0.36799999999999999,"5min":0.28199999999999997,"15min":0.214},"max":{"1min":0.97899999999999998,"5min":7.6669999999999998,"15min":12.855},"last":0.86199999999999999},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.83699999999999997,"15min":2.1760000000000002},"min":{"1min":0.34699999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":1.0600000000000001,"5min":6.8490000000000002,"15min":15.48},"last":2.048}]},{"osd":2,"last update":"Sun Oct 1 10:41:12 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.67200000000000004,"15min":0.69199999999999995},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":0.77000000000000002,"5min":1.141,"15min":1.141},"last":4.2910000000000004},{"interface":"front","average":{"1min":0.69699999999999995,"5min":0.73399999999999999,"15min":0.746},"min":{"1min":0.373,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.94399999999999995,"5min":1.149,"15min":1.149},"last":2.3740000000000001}]},{"osd":3,"last update":"Sun Oct 1 10:40:49 2023","interfaces":[{"interface":"back","average":{"1min":0.79500000000000004,"5min":0.78400000000000003,"15min":0.80400000000000005},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":1.2509999999999999,"5min":1.3069999999999999,"15min":1.3069999999999999},"last":2.4809999999999999},{"interface":"front","average":{"1min":0.69199999999999995,"5min":0.76600000000000001,"15min":0.80400000000000005},"min":{"1min":0.42699999999999999,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":1.002,"5min":1.1060000000000001,"15min":1.1060000000000001},"last":0.26400000000000001}]},{"osd":4,"last update":"Sun Oct 1 10:41:24 2023","interfaces":[{"interface":"back","average":{"1min":0.95899999999999996,"5min":1.0029999999999999,"15min":1.014},"min":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.238,"5min":1.3200000000000001,"15min":1.3200000000000001},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.92400000000000004,"5min":1.0109999999999999,"15min":1.0269999999999999},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":1.206,"5min":1.5029999999999999,"15min":1.5029999999999999},"last":0.31900000000000001}]},{"osd":5,"last update":"Sun Oct 1 10:40:49 2023","interfaces":[{"interface":"back","average":{"1min":1.0600000000000001,"5min":1.165,"15min":1.6639999999999999},"min":{"1min":0.71999999999999997,"5min":0.44700000000000001,"15min":0.31900000000000001},"max":{"1min":1.5369999999999999,"5min":8.0690000000000008,"15min":13.068},"last":2.4449999999999998},{"interface":"front","average":{"1min":1.024,"5min":1.2729999999999999,"15min":1.661},"min":{"1min":0.61399999999999999,"5min":0.41699999999999998,"15min":0.34200000000000003},"max":{"1min":1.4379999999999999,"5min":10.666,"15min":11.747},"last":0.36199999999999999}]},{"osd":6,"last update":"Sun Oct 1 10:40:49 2023","interfaces":[{"interface":"back","average":{"1min":0.96499999999999997,"5min":1.3260000000000001,"15min":1.7609999999999999},"min":{"1min":0.57799999999999996,"5min":0.38900000000000001,"15min":0.35899999999999999},"max":{"1min":1.3480000000000001,"5min":9.0410000000000004,"15min":11.663},"last":0.436},{"interface":"front","average":{"1min":0.998,"5min":1.1950000000000001,"15min":1.7070000000000001},"min":{"1min":0.60099999999999998,"5min":0.40400000000000003,"15min":0.38600000000000001},"max":{"1min":1.3260000000000001,"5min":6.8719999999999999,"15min":15.069000000000001},"last":0.38900000000000001}]},{"osd":7,"last update":"Sun Oct 1 10:40:49 2023","interfaces":[{"interface":"back","average":{"1min":1.0229999999999999,"5min":1.321,"15min":1.7849999999999999},"min":{"1min":0.69399999999999995,"5min":0.45300000000000001,"15min":0.39000000000000001},"max":{"1min":1.4219999999999999,"5min":7.5499999999999998,"15min":12.686},"last":0.45700000000000002},{"interface":"front","average":{"1min":1.012,"5min":1.2390000000000001,"15min":1.5669999999999999},"min":{"1min":0.63,"5min":0.47199999999999998,"15min":0.36199999999999999},"max":{"1min":1.393,"5min":6.8150000000000004,"15min":13.654},"last":0.46899999999999997}]}]},{"osd":2,"up_from":116,"seq":498216206391,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":661588,"kb_used_data":494796,"kb_used_omap":7,"kb_used_meta":166776,"kb_avail":93083564,"statfs":{"total":95995035648,"available":95317569536,"internally_reserved":0,"allocated":506671104,"data_stored":506159892,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":170778705},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:41:10 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.64000000000000001,"15min":0.66500000000000004},"min":{"1min":0.31,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.82599999999999996,"5min":1.0660000000000001,"15min":1.0660000000000001},"last":0.442},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.73999999999999999,"15min":0.77600000000000002},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":1.0329999999999999,"5min":1.2769999999999999,"15min":1.2769999999999999},"last":0.71299999999999997}]},{"osd":1,"last update":"Sun Oct 1 10:41:10 2023","interfaces":[{"interface":"back","average":{"1min":0.68400000000000005,"5min":0.73099999999999998,"15min":0.71799999999999997},"min":{"1min":0.45200000000000001,"5min":0.316,"15min":0.316},"max":{"1min":1.2849999999999999,"5min":1.2849999999999999,"15min":1.2849999999999999},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.71899999999999997,"15min":0.77300000000000002},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":1.413,"5min":1.413,"15min":1.413},"last":0.67600000000000005}]},{"osd":3,"last update":"Sun Oct 1 10:40:46 2023","interfaces":[{"interface":"back","average":{"1min":0.75600000000000001,"5min":0.73099999999999998,"15min":0.73699999999999999},"min":{"1min":0.41399999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.98799999999999999,"5min":1.0740000000000001,"15min":1.0740000000000001},"last":1.1000000000000001},{"interface":"front","average":{"1min":0.90600000000000003,"5min":0.86899999999999999,"15min":0.88100000000000001},"min":{"1min":0.49399999999999999,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":1.2709999999999999,"5min":1.2709999999999999,"15min":1.2709999999999999},"last":0.35399999999999998}]},{"osd":4,"last update":"Sun Oct 1 10:41:24 2023","interfaces":[{"interface":"back","average":{"1min":0.81299999999999994,"5min":0.97399999999999998,"15min":0.97799999999999998},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":1.208,"5min":1.391,"15min":1.391},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.77800000000000002,"5min":0.94299999999999995,"15min":0.95199999999999996},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":1.1479999999999999,"5min":1.254,"15min":1.254},"last":0.73199999999999998}]},{"osd":5,"last update":"Sun Oct 1 10:41:10 2023","interfaces":[{"interface":"back","average":{"1min":0.97699999999999998,"5min":1.0780000000000001,"15min":1.107},"min":{"1min":0.621,"5min":0.59899999999999998,"15min":0.59899999999999998},"max":{"1min":1.3009999999999999,"5min":1.6339999999999999,"15min":1.6339999999999999},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.94399999999999995,"5min":1.0209999999999999,"15min":1.018},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":1.264,"5min":1.585,"15min":1.585},"last":0.88600000000000001}]},{"osd":6,"last update":"Sun Oct 1 10:41:10 2023","interfaces":[{"interface":"back","average":{"1min":0.93200000000000005,"5min":1.0489999999999999,"15min":1.083},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":1.2110000000000001,"5min":1.486,"15min":1.486},"last":0.83699999999999997},{"interface":"front","average":{"1min":0.90600000000000003,"5min":1.0900000000000001,"15min":1.127},"min":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"max":{"1min":1.24,"5min":1.6639999999999999,"15min":1.6639999999999999},"last":0.80900000000000005}]},{"osd":7,"last update":"Sun Oct 1 10:41:10 2023","interfaces":[{"interface":"back","average":{"1min":0.96099999999999997,"5min":1.095,"15min":1.1200000000000001},"min":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":1.28,"5min":1.536,"15min":1.536},"last":0.52200000000000002},{"interface":"front","average":{"1min":1.0529999999999999,"5min":1.1699999999999999,"15min":1.2070000000000001},"min":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"max":{"1min":1.3759999999999999,"5min":1.7390000000000001,"15min":1.7390000000000001},"last":0.54400000000000004}]}]},{"osd":3,"up_from":121,"seq":519691042864,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1512072,"kb_used_data":1392128,"kb_used_omap":2,"kb_used_meta":119933,"kb_avail":92233080,"statfs":{"total":95995035648,"available":94446673920,"internally_reserved":0,"allocated":1425539072,"data_stored":1425038612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":122811445},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:40:48 2023","interfaces":[{"interface":"back","average":{"1min":0.66600000000000004,"5min":0.65900000000000003,"15min":0.66900000000000004},"min":{"1min":0.434,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":1.202,"5min":1.202,"15min":1.202},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.67400000000000004,"5min":0.61299999999999999,"15min":0.59299999999999997},"min":{"1min":0.44800000000000001,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.98799999999999999,"5min":0.98799999999999999,"15min":0.98799999999999999},"last":0.40300000000000002}]},{"osd":1,"last update":"Sun Oct 1 10:40:48 2023","interfaces":[{"interface":"back","average":{"1min":0.71599999999999997,"5min":0.69499999999999995,"15min":0.69799999999999995},"min":{"1min":0.42299999999999999,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":1.085,"5min":1.2450000000000001,"15min":1.2450000000000001},"last":2.4129999999999998},{"interface":"front","average":{"1min":0.79000000000000004,"5min":0.77400000000000002,"15min":0.77900000000000003},"min":{"1min":0.376,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":1.0309999999999999,"5min":1.3740000000000001,"15min":1.3740000000000001},"last":2.3820000000000001}]},{"osd":2,"last update":"Sun Oct 1 10:40:48 2023","interfaces":[{"interface":"back","average":{"1min":0.73599999999999999,"5min":0.68700000000000006,"15min":0.68300000000000005},"min":{"1min":0.41899999999999998,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":1.355,"5min":1.355,"15min":1.355},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.84999999999999998,"5min":0.80000000000000004,"15min":0.79000000000000004},"min":{"1min":0.49199999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":1.3160000000000001,"5min":1.3160000000000001,"15min":1.3160000000000001},"last":0.48099999999999998}]},{"osd":4,"last update":"Sun Oct 1 10:41:17 2023","interfaces":[{"interface":"back","average":{"1min":1.054,"5min":0.98099999999999998,"15min":0.95399999999999996},"min":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":2.4620000000000002,"5min":2.4620000000000002,"15min":2.4620000000000002},"last":2.2930000000000001},{"interface":"front","average":{"1min":0.91900000000000004,"5min":0.94999999999999996,"15min":0.93500000000000005},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":1.3069999999999999,"5min":1.355,"15min":1.355},"last":2.488}]},{"osd":5,"last update":"Sun Oct 1 10:40:48 2023","interfaces":[{"interface":"back","average":{"1min":1.0069999999999999,"5min":0.95899999999999996,"15min":0.94899999999999995},"min":{"1min":0.73599999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"max":{"1min":1.4359999999999999,"5min":1.4359999999999999,"15min":1.4359999999999999},"last":2.456},{"interface":"front","average":{"1min":1.026,"5min":1.006,"15min":1.006},"min":{"1min":0.83399999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"max":{"1min":1.2989999999999999,"5min":1.4199999999999999,"15min":1.4199999999999999},"last":1.339}]},{"osd":6,"last update":"Sun Oct 1 10:40:48 2023","interfaces":[{"interface":"back","average":{"1min":1.0680000000000001,"5min":1.0269999999999999,"15min":1.016},"min":{"1min":0.83299999999999996,"5min":0.59099999999999997,"15min":0.59099999999999997},"max":{"1min":1.4870000000000001,"5min":1.4870000000000001,"15min":1.4870000000000001},"last":2.7559999999999998},{"interface":"front","average":{"1min":1.1180000000000001,"5min":1.0940000000000001,"15min":1.1020000000000001},"min":{"1min":0.76100000000000001,"5min":0.62,"15min":0.62},"max":{"1min":1.3959999999999999,"5min":1.3959999999999999,"15min":1.3959999999999999},"last":0.72799999999999998}]},{"osd":7,"last update":"Sun Oct 1 10:40:48 2023","interfaces":[{"interface":"back","average":{"1min":1.036,"5min":1.038,"15min":1.052},"min":{"1min":0.71599999999999997,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.288,"5min":1.411,"15min":1.411},"last":1.768},{"interface":"front","average":{"1min":1.1240000000000001,"5min":1.046,"15min":1.0269999999999999},"min":{"1min":0.78200000000000003,"5min":0.51900000000000002,"15min":0.51900000000000002},"max":{"1min":1.6299999999999999,"5min":1.6299999999999999,"15min":1.6299999999999999},"last":2.3439999999999999}]}]},{"osd":4,"up_from":126,"seq":541165879337,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":632644,"kb_used_data":495292,"kb_used_omap":10,"kb_used_meta":137333,"kb_avail":93112508,"statfs":{"total":95995035648,"available":95347208192,"internally_reserved":0,"allocated":507179008,"data_stored":506682620,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":140629604},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:41:21 2023","interfaces":[{"interface":"back","average":{"1min":0.72299999999999998,"5min":0.89500000000000002,"15min":0.90100000000000002},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":1.1870000000000001,"5min":1.3009999999999999,"15min":1.3009999999999999},"last":0.373},{"interface":"front","average":{"1min":0.86599999999999999,"5min":0.95199999999999996,"15min":0.97699999999999998},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.593,"5min":1.593,"15min":1.593},"last":0.51200000000000001}]},{"osd":1,"last update":"Sun Oct 1 10:41:21 2023","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":0.94099999999999995,"15min":0.93100000000000005},"min":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"max":{"1min":1.9410000000000001,"5min":1.9410000000000001,"15min":1.9410000000000001},"last":0.83899999999999997},{"interface":"front","average":{"1min":1.002,"5min":0.97399999999999998,"15min":0.95999999999999996},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":2.8730000000000002,"5min":2.8730000000000002,"15min":2.8730000000000002},"last":0.47299999999999998}]},{"osd":2,"last update":"Sun Oct 1 10:41:21 2023","interfaces":[{"interface":"back","average":{"1min":0.83699999999999997,"5min":0.94099999999999995,"15min":0.94899999999999995},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":1.3129999999999999,"5min":1.3129999999999999,"15min":1.3129999999999999},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.84699999999999998,"5min":0.96399999999999997,"15min":0.97199999999999998},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":1.337,"5min":1.395,"15min":1.395},"last":0.59799999999999998}]},{"osd":3,"last update":"Sun Oct 1 10:41:21 2023","interfaces":[{"interface":"back","average":{"1min":1.0800000000000001,"5min":1.0629999999999999,"15min":1.073},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":3.1890000000000001,"5min":3.1890000000000001,"15min":3.1890000000000001},"last":0.78700000000000003},{"interface":"front","average":{"1min":0.83699999999999997,"5min":0.94599999999999995,"15min":0.96799999999999997},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":1.143,"5min":1.341,"15min":1.341},"last":0.72499999999999998}]},{"osd":5,"last update":"Sun Oct 1 10:41:21 2023","interfaces":[{"interface":"back","average":{"1min":1.294,"5min":1.119,"15min":1.103},"min":{"1min":0.503,"5min":0.503,"15min":0.503},"max":{"1min":3.9239999999999999,"5min":3.9239999999999999,"15min":3.9239999999999999},"last":0.747},{"interface":"front","average":{"1min":0.95699999999999996,"5min":1.073,"15min":1.087},"min":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":1.415,"5min":1.415,"15min":1.415},"last":0.68899999999999995}]},{"osd":6,"last update":"Sun Oct 1 10:41:21 2023","interfaces":[{"interface":"back","average":{"1min":0.95099999999999996,"5min":1.093,"15min":1.123},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":1.4390000000000001,"5min":1.579,"15min":1.579},"last":0.81200000000000006},{"interface":"front","average":{"1min":0.94499999999999995,"5min":1.0549999999999999,"15min":1.0680000000000001},"min":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":1.633,"5min":1.633,"15min":1.633},"last":0.879}]},{"osd":7,"last update":"Sun Oct 1 10:41:21 2023","interfaces":[{"interface":"back","average":{"1min":1.006,"5min":1.1479999999999999,"15min":1.1699999999999999},"min":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"max":{"1min":1.714,"5min":1.714,"15min":1.714},"last":0.77100000000000002},{"interface":"front","average":{"1min":1.1319999999999999,"5min":1.169,"15min":1.169},"min":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"max":{"1min":3.544,"5min":3.544,"15min":3.544},"last":0.56999999999999995}]}]},{"osd":5,"up_from":36,"seq":154618823018,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2023888,"kb_used_data":1421896,"kb_used_omap":0,"kb_used_meta":601984,"kb_avail":91721264,"statfs":{"total":95995035648,"available":93922574336,"internally_reserved":0,"allocated":1456021504,"data_stored":1455567246,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":616431616},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:40:38 2023","interfaces":[{"interface":"back","average":{"1min":0.93200000000000005,"5min":0.95699999999999996,"15min":1.256},"min":{"1min":0.58499999999999996,"5min":0.27300000000000002,"15min":0.26400000000000001},"max":{"1min":1.3540000000000001,"5min":4.5499999999999998,"15min":10.429},"last":0.82199999999999995},{"interface":"front","average":{"1min":0.94999999999999996,"5min":1.173,"15min":1.464},"min":{"1min":0.63700000000000001,"5min":0.31,"15min":0.29199999999999998},"max":{"1min":1.28,"5min":13.018000000000001,"15min":13.018000000000001},"last":0.52900000000000003}]},{"osd":1,"last update":"Sun Oct 1 10:40:38 2023","interfaces":[{"interface":"back","average":{"1min":1.004,"5min":1.1060000000000001,"15min":1.617},"min":{"1min":0.61199999999999999,"5min":0.29199999999999998,"15min":0.29099999999999998},"max":{"1min":1.377,"5min":8.9359999999999999,"15min":10.858000000000001},"last":0.85399999999999998},{"interface":"front","average":{"1min":1.0089999999999999,"5min":1.4370000000000001,"15min":1.714},"min":{"1min":0.53600000000000003,"5min":0.40300000000000002,"15min":0.32500000000000001},"max":{"1min":1.296,"5min":12.864000000000001,"15min":12.864000000000001},"last":0.65400000000000003}]},{"osd":2,"last update":"Sun Oct 1 10:41:11 2023","interfaces":[{"interface":"back","average":{"1min":0.94499999999999995,"5min":0.96599999999999997,"15min":0.97799999999999998},"min":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"max":{"1min":1.3,"5min":1.4690000000000001,"15min":1.4690000000000001},"last":0.63900000000000001},{"interface":"front","average":{"1min":1.008,"5min":0.98499999999999999,"15min":0.97099999999999997},"min":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"max":{"1min":1.343,"5min":1.343,"15min":1.343},"last":0.56299999999999994}]},{"osd":3,"last update":"Sun Oct 1 10:40:55 2023","interfaces":[{"interface":"back","average":{"1min":0.95099999999999996,"5min":0.94699999999999995,"15min":0.93700000000000006},"min":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"max":{"1min":1.284,"5min":1.3919999999999999,"15min":1.3919999999999999},"last":9.3919999999999995},{"interface":"front","average":{"1min":1.04,"5min":0.96999999999999997,"15min":0.94999999999999996},"min":{"1min":0.84799999999999998,"5min":0.57699999999999996,"15min":0.57699999999999996},"max":{"1min":1.357,"5min":1.357,"15min":1.357},"last":0.54500000000000004}]},{"osd":4,"last update":"Sun Oct 1 10:41:21 2023","interfaces":[{"interface":"back","average":{"1min":1.1359999999999999,"5min":1.0960000000000001,"15min":1.083},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":3.1669999999999998,"5min":3.1669999999999998,"15min":3.1669999999999998},"last":0.38200000000000001},{"interface":"front","average":{"1min":1.204,"5min":1.1240000000000001,"15min":1.097},"min":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"max":{"1min":3.1000000000000001,"5min":3.1000000000000001,"15min":3.1000000000000001},"last":0.56899999999999995}]},{"osd":6,"last update":"Sun Oct 1 10:40:55 2023","interfaces":[{"interface":"back","average":{"1min":1.0900000000000001,"5min":1.141,"15min":1.2589999999999999},"min":{"1min":0.59899999999999998,"5min":0.38500000000000001,"15min":0.35699999999999998},"max":{"1min":1.3420000000000001,"5min":6.96,"15min":6.96},"last":0.79100000000000004},{"interface":"front","average":{"1min":1.105,"5min":1.1879999999999999,"15min":1.242},"min":{"1min":0.72199999999999998,"5min":0.312,"15min":0.312},"max":{"1min":1.3560000000000001,"5min":8.4339999999999993,"15min":8.4339999999999993},"last":0.432}]},{"osd":7,"last update":"Sun Oct 1 10:40:55 2023","interfaces":[{"interface":"back","average":{"1min":1.139,"5min":1.232,"15min":1.2430000000000001},"min":{"1min":0.84299999999999997,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.341,"5min":9.4629999999999992,"15min":9.4629999999999992},"last":0.46800000000000003},{"interface":"front","average":{"1min":1.1779999999999999,"5min":1.1439999999999999,"15min":1.3029999999999999},"min":{"1min":0.54700000000000004,"5min":0.372,"15min":0.372},"max":{"1min":1.365,"5min":4.6959999999999997,"15min":12.927},"last":0.68400000000000005}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":680263680,"data_stored":680263680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1607794688,"data_stored":1607794688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":509935616,"data_stored":509935616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1440874496,"data_stored":1440874496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":510853120,"data_stored":510853120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1505099776,"data_stored":1505099776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":495255552,"data_stored":495255552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1474166784,"data_stored":1474166784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T10:41:36.182 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_4 pgp_num 16 2023-10-01T10:41:36.211 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 28 16 50843 50827 113.44 76.75 0.0202126 0.00880955 2023-10-01T10:41:36.258 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:36.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:36.268 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:41:36.369 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:36.395 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:36.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:36.397 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:41:36.498 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:36.533 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:36.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:36.536 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:41:36.637 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:36.659 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:36.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:36.661 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:41:36.762 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:36.788 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:36.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:36.792 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:41:36.893 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:36.916 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:36.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:36.918 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:41:37.019 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:37.051 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:37.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:37.054 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:41:37.154 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:37.183 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:37.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:37.190 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:41:37.211 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 29 16 52117 52101 112.274 79.625 0.0096055 0.00890104 2023-10-01T10:41:37.291 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:37.318 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:37.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:37.322 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:41:37.423 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:37.461 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:37.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:37.465 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:41:37.566 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:37.576 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:37 smithi078 bash[75299]: cluster 2023-10-01T10:41:35.730961+0000 mgr.y (mgr.25072) 715 : cluster 0 pgmap v623: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-10-01T10:41:37.576 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:37 smithi078 bash[78274]: cluster 2023-10-01T10:41:35.730961+0000 mgr.y (mgr.25072) 715 : cluster 0 pgmap v623: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-10-01T10:41:37.590 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:37.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:37.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:41:37.695 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:37.722 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:37.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:37.727 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:41:37.828 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:37.861 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:37.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:37.866 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:41:37.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:37 smithi173 bash[58816]: cluster 2023-10-01T10:41:35.730961+0000 mgr.y (mgr.25072) 715 : cluster 0 pgmap v623: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-10-01T10:41:37.967 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:38.000 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:38.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:38.005 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:41:38.106 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:38.142 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:38.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:38.146 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:41:38.209 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T10:41:38.212 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 30 16 53316 53300 111.029 74.9375 0.00951032 0.00900049 2023-10-01T10:41:38.246 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:38.275 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:38.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:38.279 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:41:38.379 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:38.435 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:38.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:38.440 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:41:38.541 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:38.585 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:38.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:38.590 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:41:38.691 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:38.718 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:38.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:38.721 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:41:38.822 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:38.858 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:38.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:38.864 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:41:38.964 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:38.994 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:38.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:38.999 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:41:39.100 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:39.131 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:39.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:39.136 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:41:39.212 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 31 15 54493 54478 109.822 73.625 0.0235984 0.00909968 2023-10-01T10:41:39.237 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:39.275 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:39.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:39.278 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:41:39.379 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:39.406 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:39.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:39.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:41:39.512 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:39.536 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:39.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:39.538 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:41:39.639 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:39.669 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:39.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:39.671 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:41:39.772 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:39.781 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:39 smithi173 bash[58816]: cluster 2023-10-01T10:41:37.731703+0000 mgr.y (mgr.25072) 716 : cluster 0 pgmap v624: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s 2023-10-01T10:41:39.782 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:39 smithi173 bash[58816]: audit 2023-10-01T10:41:38.620548+0000 mon.a (mon.0) 939 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:41:39.782 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:39 smithi173 bash[58816]: audit 2023-10-01T10:41:38.628366+0000 mon.a (mon.0) 940 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:41:39.796 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:39.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:39.799 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:41:39.900 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:39.926 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:39.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:39.929 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:41:40.030 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:40.040 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:39 smithi078 bash[75299]: cluster 2023-10-01T10:41:37.731703+0000 mgr.y (mgr.25072) 716 : cluster 0 pgmap v624: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s 2023-10-01T10:41:40.040 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:39 smithi078 bash[75299]: audit 2023-10-01T10:41:38.620548+0000 mon.a (mon.0) 939 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:41:40.040 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:39 smithi078 bash[75299]: audit 2023-10-01T10:41:38.628366+0000 mon.a (mon.0) 940 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:41:40.040 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:39 smithi078 bash[78274]: cluster 2023-10-01T10:41:37.731703+0000 mgr.y (mgr.25072) 716 : cluster 0 pgmap v624: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s 2023-10-01T10:41:40.041 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:39 smithi078 bash[78274]: audit 2023-10-01T10:41:38.620548+0000 mon.a (mon.0) 939 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:41:40.041 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:39 smithi078 bash[78274]: audit 2023-10-01T10:41:38.628366+0000 mon.a (mon.0) 940 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:41:40.061 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:40.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:40.064 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:41:40.165 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:40.194 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:40.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:40.198 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:41:40.213 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 32 16 55741 55725 108.823 77.9375 0.0207793 0.00918361 2023-10-01T10:41:40.298 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:40.328 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:40.331 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:41:40 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:41:40] "GET /metrics HTTP/1.1" 200 36244 "" "Prometheus/2.33.4" 2023-10-01T10:41:40.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:40.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:41:40.433 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:40.458 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:40.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:40.462 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:41:40.563 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:40.587 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:40.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:40.589 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:41:40.690 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:40.730 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:40.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:40.733 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:41:40.750 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:41:40.750 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:41:40.758 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:41:40.834 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:40.866 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:40.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:40.870 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:41:40.971 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:41.002 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:41.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:41.005 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:41:41.105 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:41.137 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:41.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:41.140 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:41:41.213 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 33 16 56872 56856 107.667 70.6875 0.00920851 0.00928282 2023-10-01T10:41:41.241 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:41.266 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:41.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:41.270 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:41:41.371 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:41.395 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:41.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:41.398 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:41:41.498 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:41.533 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:41.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:41.539 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:41:41.640 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:41.666 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:41.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:41.669 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:41:41.770 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:41.805 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:41.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:41.811 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:41:41.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:41 smithi078 bash[75299]: cluster 2023-10-01T10:41:39.732570+0000 mgr.y (mgr.25072) 717 : cluster 0 pgmap v625: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-01T10:41:41.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:41 smithi078 bash[78274]: cluster 2023-10-01T10:41:39.732570+0000 mgr.y (mgr.25072) 717 : cluster 0 pgmap v625: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-01T10:41:41.912 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:42.184 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:41 smithi173 bash[58816]: cluster 2023-10-01T10:41:39.732570+0000 mgr.y (mgr.25072) 717 : cluster 0 pgmap v625: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-01T10:41:42.209 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:42.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:42.227 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:41:42.227 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 34 16 58137 58121 106.826 79.0625 0.0141324 0.0093551 2023-10-01T10:41:42.328 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:42.354 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:42.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:42.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:41:42.457 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:42.494 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:42.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:42.498 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:41:42.600 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:42.632 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:42.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:42.634 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:41:42.735 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:42.762 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:42.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:42.765 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:41:42.787 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:41:42.866 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:42.894 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:42.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:42.899 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:41:43.000 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:43.027 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:43.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:43.029 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:41:43.129 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:43.171 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:43.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:43.175 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:41:43.212 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 35 16 59399 59383 106.027 78.875 0.00941706 0.00942652 2023-10-01T10:41:43.276 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:43.305 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:43.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:43.307 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:41:43.408 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:43.432 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:43.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:43.435 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:41:43.535 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:43.560 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:43.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:43.563 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:41:43.664 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:43.689 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:43.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:43.691 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:41:43.792 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:43.816 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:43.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:43.819 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:41:43.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:43 smithi173 bash[58816]: cluster 2023-10-01T10:41:41.733344+0000 mgr.y (mgr.25072) 718 : cluster 0 pgmap v626: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-01T10:41:43.880 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:43 smithi078 bash[78274]: cluster 2023-10-01T10:41:41.733344+0000 mgr.y (mgr.25072) 718 : cluster 0 pgmap v626: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-01T10:41:43.881 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:43 smithi078 bash[75299]: cluster 2023-10-01T10:41:41.733344+0000 mgr.y (mgr.25072) 718 : cluster 0 pgmap v626: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-01T10:41:43.920 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:43.944 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:43.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:43.947 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:41:44.048 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:44.070 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:44.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:44.079 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:41:44.180 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:44.206 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:44.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:44.210 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:41:44.212 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 36 16 60659 60643 105.269 78.75 0.00890497 0.00949314 2023-10-01T10:41:44.311 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:44.341 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:44.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:44.345 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:41:44.445 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:44.472 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:44.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:44.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:41:44.579 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:44.603 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:44.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:44.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:41:44.707 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:44.744 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:44.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:44.746 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:41:44.847 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:44.884 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:44.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:44.887 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:41:44.988 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:45.020 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:45.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:45.028 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:41:45.129 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:45.171 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:45.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:45.175 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:41:45.213 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 37 15 61868 61853 104.468 75.625 0.00594639 0.00956652 2023-10-01T10:41:45.276 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:45.304 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:45.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:45.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:41:45.407 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:45.436 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:45.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:45.440 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:41:45.540 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:45.576 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:45.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:45.584 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:41:45.684 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:45.694 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:45 smithi173 bash[58816]: cluster 2023-10-01T10:41:43.733884+0000 mgr.y (mgr.25072) 719 : cluster 0 pgmap v627: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-01T10:41:45.694 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:45 smithi173 bash[58816]: audit 2023-10-01T10:41:44.583752+0000 mon.a (mon.0) 941 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:41:45.694 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:45 smithi173 bash[58816]: audit 2023-10-01T10:41:44.594207+0000 mon.a (mon.0) 942 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:41:45.694 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:45 smithi173 bash[58816]: audit 2023-10-01T10:41:45.168992+0000 mon.a (mon.0) 943 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:41:45.694 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:45 smithi173 bash[58816]: audit 2023-10-01T10:41:45.170849+0000 mon.a (mon.0) 944 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:41:45.694 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:45 smithi173 bash[58816]: audit 2023-10-01T10:41:45.178488+0000 mon.a (mon.0) 945 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:41:45.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:45.715 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:45.715 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:41:45.816 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:45.840 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:45.841 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:45 smithi078 bash[75299]: cluster 2023-10-01T10:41:43.733884+0000 mgr.y (mgr.25072) 719 : cluster 0 pgmap v627: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-01T10:41:45.842 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:45 smithi078 bash[75299]: audit 2023-10-01T10:41:44.583752+0000 mon.a (mon.0) 941 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:41:45.842 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:45 smithi078 bash[75299]: audit 2023-10-01T10:41:44.594207+0000 mon.a (mon.0) 942 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:41:45.842 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:45 smithi078 bash[75299]: audit 2023-10-01T10:41:45.168992+0000 mon.a (mon.0) 943 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:41:45.842 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:45 smithi078 bash[75299]: audit 2023-10-01T10:41:45.170849+0000 mon.a (mon.0) 944 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:41:45.842 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:45 smithi078 bash[75299]: audit 2023-10-01T10:41:45.178488+0000 mon.a (mon.0) 945 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:41:45.842 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:45 smithi078 bash[78274]: cluster 2023-10-01T10:41:43.733884+0000 mgr.y (mgr.25072) 719 : cluster 0 pgmap v627: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-01T10:41:45.843 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:45 smithi078 bash[78274]: audit 2023-10-01T10:41:44.583752+0000 mon.a (mon.0) 941 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:41:45.843 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:45 smithi078 bash[78274]: audit 2023-10-01T10:41:44.594207+0000 mon.a (mon.0) 942 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:41:45.843 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:45 smithi078 bash[78274]: audit 2023-10-01T10:41:45.168992+0000 mon.a (mon.0) 943 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:41:45.843 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:45 smithi078 bash[78274]: audit 2023-10-01T10:41:45.170849+0000 mon.a (mon.0) 944 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:41:45.843 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:45 smithi078 bash[78274]: audit 2023-10-01T10:41:45.178488+0000 mon.a (mon.0) 945 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:41:45.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:45.844 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:41:45.944 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:45.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:45.976 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:45.977 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:41:46.077 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:46.101 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:46.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:46.104 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:41:46.205 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:46.213 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 38 16 63143 63127 103.814 79.625 0.0102692 0.00962627 2023-10-01T10:41:46.230 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:46.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:46.233 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:41:46.334 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:46.370 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:46.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:46.375 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:41:46.477 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:46.503 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:46.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:46.505 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:41:46.606 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:46.633 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:46.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:46.636 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:41:46.737 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:46.762 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:46.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:46.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:41:46.872 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:46.902 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:46.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:46.905 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:41:47.006 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:47.034 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:47.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:47.038 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:41:47.139 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:47.165 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:47.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:47.173 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:41:47.213 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 39 16 64333 64317 103.059 74.375 0.013492 0.00969754 2023-10-01T10:41:47.274 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:47.302 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:47.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:47.304 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:41:47.405 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:47.434 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:47.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:47.437 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:41:47.538 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:47.566 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:47.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:47.569 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:41:47.669 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:47.679 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:47 smithi173 bash[58816]: cluster 2023-10-01T10:41:45.734482+0000 mgr.y (mgr.25072) 720 : cluster 0 pgmap v628: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2023-10-01T10:41:47.679 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:47 smithi173 bash[58816]: audit 2023-10-01T10:41:47.412438+0000 mon.a (mon.0) 946 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:41:47.704 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:47.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:47.707 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:41:47.807 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:47.815 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:47 smithi078 bash[78274]: cluster 2023-10-01T10:41:45.734482+0000 mgr.y (mgr.25072) 720 : cluster 0 pgmap v628: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2023-10-01T10:41:47.815 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:47 smithi078 bash[78274]: audit 2023-10-01T10:41:47.412438+0000 mon.a (mon.0) 946 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:41:47.816 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:47 smithi078 bash[75299]: cluster 2023-10-01T10:41:45.734482+0000 mgr.y (mgr.25072) 720 : cluster 0 pgmap v628: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2023-10-01T10:41:47.816 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:47 smithi078 bash[75299]: audit 2023-10-01T10:41:47.412438+0000 mon.a (mon.0) 946 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:41:47.840 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:47.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:47.843 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:41:47.943 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:47.975 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:47.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:47.980 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:41:48.081 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:48.114 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:48.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:48.118 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:41:48.213 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:2023-10-01T10:41:48.212559+0000 min lat: 0.00132359 max lat: 0.151065 avg lat: 0.00976311 2023-10-01T10:41:48.213 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T10:41:48.213 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 40 16 65539 65523 102.367 75.375 0.0136113 0.00976311 2023-10-01T10:41:48.219 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:48.244 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:48.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:48.249 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:41:48.350 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:48.387 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:48.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:48.391 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:41:48.492 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:48.520 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:48.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:48.525 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:41:48.626 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:48.652 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:48.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:48.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:41:48.760 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:48.783 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:48.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:48.785 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:41:48.886 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:48.906 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:48.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:48.907 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:41:49.008 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:49.044 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:49.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:49.046 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:41:49.147 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:49.182 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:49.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:49.187 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:41:49.213 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 41 16 66557 66541 101.422 63.625 0.0136999 0.00985447 2023-10-01T10:41:49.288 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:49.315 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:49.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:49.319 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:41:49.419 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:49.443 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:49.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:49.451 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:41:49.552 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:49.579 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:49.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:49.582 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:41:49.593 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:49 smithi173 bash[58816]: cluster 2023-10-01T10:41:47.734971+0000 mgr.y (mgr.25072) 721 : cluster 0 pgmap v629: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-10-01T10:41:49.683 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:49.692 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:49 smithi078 bash[78274]: cluster 2023-10-01T10:41:47.734971+0000 mgr.y (mgr.25072) 721 : cluster 0 pgmap v629: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-10-01T10:41:49.692 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:49 smithi078 bash[75299]: cluster 2023-10-01T10:41:47.734971+0000 mgr.y (mgr.25072) 721 : cluster 0 pgmap v629: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-10-01T10:41:49.710 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:49.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:49.713 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:41:49.813 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:49.845 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:49.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:49.849 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:41:49.950 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:49.981 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:49.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:49.984 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:41:50.085 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:50.110 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:50.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:50.118 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:41:50.213 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 42 16 67702 67686 100.711 71.5625 0.00645505 0.0099234 2023-10-01T10:41:50.219 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:50.256 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:50.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:50.260 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:41:50.269 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:41:50 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:41:50] "GET /metrics HTTP/1.1" 200 36245 "" "Prometheus/2.33.4" 2023-10-01T10:41:50.360 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:50.400 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:50.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:50.407 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:41:50.508 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:50.539 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:50.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:50.545 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:41:50.646 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:50.667 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:50.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:50.672 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:41:50.773 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:50.804 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:50.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:50.808 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:41:50.909 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:50.934 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:50.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:50.941 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:41:51.042 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:51.072 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:51.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:51.077 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:41:51.178 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:51.213 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:51.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:51.215 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:41:51.215 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 43 16 68852 68836 100.036 71.875 0.0109277 0.00999046 2023-10-01T10:41:51.316 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:51.350 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:51.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:51.352 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:41:51.453 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:51.478 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:51.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:51.483 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:41:51.584 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:51.616 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:51.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:51.621 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:41:51.722 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:51.749 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:51.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:51.752 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:41:51.852 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:51.862 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:51 smithi173 bash[58816]: cluster 2023-10-01T10:41:49.735527+0000 mgr.y (mgr.25072) 722 : cluster 0 pgmap v630: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-01T10:41:51.886 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:51.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:51.890 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:41:51.990 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:52.016 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:52.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:52.021 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:41:52.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:51 smithi078 bash[75299]: cluster 2023-10-01T10:41:49.735527+0000 mgr.y (mgr.25072) 722 : cluster 0 pgmap v630: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-01T10:41:52.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:51 smithi078 bash[78274]: cluster 2023-10-01T10:41:49.735527+0000 mgr.y (mgr.25072) 722 : cluster 0 pgmap v630: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-01T10:41:52.122 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:52.150 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:52.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:52.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:41:52.215 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 44 16 69978 69962 99.3622 70.375 0.0142403 0.0100588 2023-10-01T10:41:52.258 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:52.288 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:52.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:52.290 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:41:52.391 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:52.418 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:52.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:52.420 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:41:52.521 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:52.550 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:52.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:52.554 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:41:52.655 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:52.680 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:52.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:52.684 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:41:52.785 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:52.811 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:52.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:52.813 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:41:52.914 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:52.939 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:52.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:52.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:41:53.045 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:53.069 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:53.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:53.075 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:41:53.176 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:53.205 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:53.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:53.209 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:41:53.215 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 45 16 71101 71085 98.7138 70.1875 0.00915018 0.0101251 2023-10-01T10:41:53.310 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:53.346 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:53.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:53.349 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:41:53.450 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:53.489 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:53.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:53.494 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:41:53.595 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:53.623 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:53.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:53.626 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:41:53.727 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:53.736 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:53 smithi173 bash[58816]: cluster 2023-10-01T10:41:51.735999+0000 mgr.y (mgr.25072) 723 : cluster 0 pgmap v631: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-01T10:41:53.758 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:53.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:53.762 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:41:53.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:53 smithi078 bash[78274]: cluster 2023-10-01T10:41:51.735999+0000 mgr.y (mgr.25072) 723 : cluster 0 pgmap v631: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-01T10:41:53.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:53 smithi078 bash[75299]: cluster 2023-10-01T10:41:51.735999+0000 mgr.y (mgr.25072) 723 : cluster 0 pgmap v631: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-01T10:41:53.862 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:53.899 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:53.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:53.905 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:41:54.005 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:54.035 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:54.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:54.038 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:41:54.139 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:54.177 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:54.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:54.183 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:41:54.218 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 46 16 72198 72182 98.0582 68.5625 0.0158174 0.0101931 2023-10-01T10:41:54.285 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:54.321 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:54.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:54.326 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:41:54.427 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:54.457 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:54.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:54.462 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:41:54.563 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:54.603 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:54.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:54.605 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:41:54.706 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:54.741 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:54.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:54.749 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:41:54.850 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:54.894 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:54.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:54.897 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:41:54.998 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:55.026 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:55.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:55.028 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:41:55.129 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:55.155 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:55.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:55.157 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:41:55.215 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 47 16 73319 73303 97.4624 70.0625 0.0228154 0.0102546 2023-10-01T10:41:55.258 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:55.293 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:55.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:55.298 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:41:55.399 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:55.425 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:55.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:55.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:41:55.528 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:55.553 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:55.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:55.560 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:41:55.661 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:55.671 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:55 smithi173 bash[58816]: cluster 2023-10-01T10:41:53.736516+0000 mgr.y (mgr.25072) 724 : cluster 0 pgmap v632: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-01T10:41:55.693 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:55.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:55.695 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:41:55.796 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:55.805 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:55 smithi078 bash[75299]: cluster 2023-10-01T10:41:53.736516+0000 mgr.y (mgr.25072) 724 : cluster 0 pgmap v632: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-01T10:41:55.805 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:55 smithi078 bash[78274]: cluster 2023-10-01T10:41:53.736516+0000 mgr.y (mgr.25072) 724 : cluster 0 pgmap v632: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-01T10:41:55.824 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:55.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:55.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:41:55.928 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:55.959 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:55.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:55.964 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:41:56.064 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:56.090 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:56.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:56.094 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:41:56.099 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T10:41:56.141 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T10:41:56.189 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T10:41:56.195 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:56.217 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 48 16 74320 74304 96.7313 62.5625 0.0146479 0.0103311 2023-10-01T10:41:56.235 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T10:41:56.238 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:56.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:56.243 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:41:56.273 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T10:41:56.335 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T10:41:56.344 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:56.372 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:56.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:56.378 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:41:56.384 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T10:41:56.425 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T10:41:56.432 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T10:41:56.432 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T10:41:56.432 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T10:41:56.432 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T10:41:56.432 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T10:41:56.433 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T10:41:56.433 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T10:41:56.433 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T10:41:56.478 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:56.512 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:56.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:56.515 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:41:56.585 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2023-10-01T10:41:56.585 INFO:teuthology.orchestra.run.smithi078.stdout: "ops": [ 2023-10-01T10:41:56.586 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:41:56.586 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.35649.0:74546 9.4 9:2466c072:::benchmark_data_smithi078_113121_object74545:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e132)", 2023-10-01T10:41:56.586 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:41:56.534246+0000", 2023-10-01T10:41:56.586 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 0.045306879000000001, 2023-10-01T10:41:56.586 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.045334514999999999, 2023-10-01T10:41:56.586 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:41:56.586 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "waiting for sub ops", 2023-10-01T10:41:56.586 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:41:56.586 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.35649", 2023-10-01T10:41:56.587 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:41:56.587 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 74546 2023-10-01T10:41:56.587 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:41:56.587 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:41:56.587 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:41:56.587 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:41:56.587 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:56.534246+0000", 2023-10-01T10:41:56.587 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:41:56.588 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:41:56.588 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:41:56.588 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:41:56.588 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:56.534246+0000", 2023-10-01T10:41:56.588 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:41:56.588 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:41:56.588 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:41:56.588 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:41:56.588 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:56.534245+0000", 2023-10-01T10:41:56.588 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999986 2023-10-01T10:41:56.589 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:41:56.589 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:41:56.589 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:41:56.589 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:56.534267+0000", 2023-10-01T10:41:56.589 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.1977999999999999e-05 2023-10-01T10:41:56.589 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:41:56.589 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:41:56.589 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:41:56.590 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:56.534278+0000", 2023-10-01T10:41:56.590 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1313e-05 2023-10-01T10:41:56.590 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:41:56.590 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:41:56.590 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:41:56.590 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:56.534286+0000", 2023-10-01T10:41:56.590 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.8059999999999995e-06 2023-10-01T10:41:56.590 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:41:56.590 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:41:56.590 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:41:56.590 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:56.534362+0000", 2023-10-01T10:41:56.591 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.6253000000000003e-05 2023-10-01T10:41:56.591 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:41:56.591 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:41:56.591 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:41:56.591 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:56.534505+0000", 2023-10-01T10:41:56.591 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000142817 2023-10-01T10:41:56.591 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:41:56.591 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:41:56.591 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 3,4", 2023-10-01T10:41:56.591 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:56.534800+0000", 2023-10-01T10:41:56.591 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000295622 2023-10-01T10:41:56.591 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:41:56.591 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:41:56.592 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:41:56.592 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:56.541489+0000", 2023-10-01T10:41:56.592 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0066887789999999997 2023-10-01T10:41:56.592 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:41:56.592 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:41:56.592 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:41:56.592 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:56.545658+0000", 2023-10-01T10:41:56.592 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0041686479999999996 2023-10-01T10:41:56.592 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:41:56.592 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:41:56.593 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:41:56.593 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:41:56.593 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:41:56.593 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.35649.0:74556 9.4 9:20ff93ab:::benchmark_data_smithi078_113121_object74555:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e132)", 2023-10-01T10:41:56.593 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:41:56.545417+0000", 2023-10-01T10:41:56.593 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 0.034135326000000001, 2023-10-01T10:41:56.593 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.035915032999999999, 2023-10-01T10:41:56.594 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:41:56.594 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "waiting for sub ops", 2023-10-01T10:41:56.594 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:41:56.594 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.35649", 2023-10-01T10:41:56.594 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:41:56.594 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 74556 2023-10-01T10:41:56.594 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:41:56.594 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:41:56.594 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:41:56.594 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:41:56.595 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:56.545417+0000", 2023-10-01T10:41:56.595 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:41:56.595 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:41:56.595 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:41:56.595 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:41:56.595 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:56.545417+0000", 2023-10-01T10:41:56.596 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:41:56.596 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:41:56.596 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:41:56.596 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:41:56.596 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:56.545416+0000", 2023-10-01T10:41:56.597 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999986 2023-10-01T10:41:56.597 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:41:56.597 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:41:56.597 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:41:56.597 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:56.545440+0000", 2023-10-01T10:41:56.597 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.3842e-05 2023-10-01T10:41:56.597 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:41:56.598 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:41:56.598 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:41:56.598 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:56.545450+0000", 2023-10-01T10:41:56.598 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.0291e-05 2023-10-01T10:41:56.598 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:41:56.598 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:41:56.599 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:41:56.599 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:56.545457+0000", 2023-10-01T10:41:56.599 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.6780000000000003e-06 2023-10-01T10:41:56.599 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:41:56.599 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:41:56.599 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:41:56.599 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:56.545714+0000", 2023-10-01T10:41:56.600 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00025744199999999998 2023-10-01T10:41:56.600 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:41:56.600 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:41:56.600 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:41:56.600 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:56.545895+0000", 2023-10-01T10:41:56.600 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00018075 2023-10-01T10:41:56.601 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:41:56.601 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:41:56.601 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 3,4", 2023-10-01T10:41:56.601 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:56.546063+0000", 2023-10-01T10:41:56.601 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000168159 2023-10-01T10:41:56.601 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:41:56.601 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:41:56.601 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:41:56.601 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:56.552851+0000", 2023-10-01T10:41:56.601 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0067879560000000004 2023-10-01T10:41:56.601 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:41:56.601 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:41:56.602 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:41:56.602 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:56.552881+0000", 2023-10-01T10:41:56.602 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.0335999999999999e-05 2023-10-01T10:41:56.602 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:41:56.602 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:41:56.602 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:41:56.602 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:41:56.602 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:41:56.602 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.35649.0:74578 9.4 9:225545be:::benchmark_data_smithi078_113121_object74577:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e132)", 2023-10-01T10:41:56.602 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:41:56.571261+0000", 2023-10-01T10:41:56.602 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 0.0082915769999999996, 2023-10-01T10:41:56.603 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.010181355, 2023-10-01T10:41:56.603 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:41:56.603 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "waiting for sub ops", 2023-10-01T10:41:56.603 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:41:56.603 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.35649", 2023-10-01T10:41:56.603 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:41:56.603 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 74578 2023-10-01T10:41:56.603 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:41:56.603 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:41:56.603 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:41:56.603 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:41:56.603 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:56.571261+0000", 2023-10-01T10:41:56.604 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:41:56.604 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:41:56.604 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:41:56.604 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:41:56.604 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:56.571261+0000", 2023-10-01T10:41:56.604 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:41:56.604 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:41:56.604 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:41:56.604 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:41:56.604 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:56.571259+0000", 2023-10-01T10:41:56.604 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999981 2023-10-01T10:41:56.604 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:41:56.605 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:41:56.605 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:41:56.605 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:56.571285+0000", 2023-10-01T10:41:56.605 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.5442000000000001e-05 2023-10-01T10:41:56.605 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:41:56.605 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:41:56.605 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:41:56.605 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:56.571296+0000", 2023-10-01T10:41:56.605 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1605e-05 2023-10-01T10:41:56.605 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:41:56.605 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:41:56.605 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:41:56.606 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:56.571305+0000", 2023-10-01T10:41:56.606 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.7339999999999997e-06 2023-10-01T10:41:56.606 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:41:56.606 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:41:56.606 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:41:56.606 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:56.571511+0000", 2023-10-01T10:41:56.607 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00020638500000000001 2023-10-01T10:41:56.607 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:41:56.607 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:41:56.607 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:41:56.607 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:56.571635+0000", 2023-10-01T10:41:56.607 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00012391999999999999 2023-10-01T10:41:56.607 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:41:56.607 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:41:56.607 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 3,4", 2023-10-01T10:41:56.607 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:56.571773+0000", 2023-10-01T10:41:56.607 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000137322 2023-10-01T10:41:56.608 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:41:56.608 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:41:56.608 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:41:56.608 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:56.577889+0000", 2023-10-01T10:41:56.608 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0061161949999999996 2023-10-01T10:41:56.608 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:41:56.608 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:41:56.608 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:41:56.608 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:41:56.608 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:41:56.608 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.35649.0:74583 9.4 9:2d3dc5ff:::benchmark_data_smithi078_113121_object74582:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e132)", 2023-10-01T10:41:56.608 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:41:56.577207+0000", 2023-10-01T10:41:56.608 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 0.0023460180000000001, 2023-10-01T10:41:56.609 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.004334147, 2023-10-01T10:41:56.609 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:41:56.609 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "waiting for sub ops", 2023-10-01T10:41:56.609 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:41:56.609 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.35649", 2023-10-01T10:41:56.609 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:41:56.609 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 74583 2023-10-01T10:41:56.609 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:41:56.609 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:41:56.609 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:41:56.609 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:41:56.609 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:56.577207+0000", 2023-10-01T10:41:56.610 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:41:56.610 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:41:56.610 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:41:56.610 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:41:56.610 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:56.577207+0000", 2023-10-01T10:41:56.610 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:41:56.610 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:41:56.610 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:41:56.610 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:41:56.610 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:56.577205+0000", 2023-10-01T10:41:56.610 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999981 2023-10-01T10:41:56.610 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:41:56.611 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:41:56.611 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:41:56.611 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:56.577241+0000", 2023-10-01T10:41:56.611 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.6599e-05 2023-10-01T10:41:56.611 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:41:56.611 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:41:56.611 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:41:56.611 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:56.577273+0000", 2023-10-01T10:41:56.611 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.1411999999999997e-05 2023-10-01T10:41:56.611 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:41:56.611 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:41:56.611 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:41:56.611 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:56.577283+0000", 2023-10-01T10:41:56.612 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.0101999999999999e-05 2023-10-01T10:41:56.612 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:41:56.612 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:41:56.612 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:41:56.612 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:56.577915+0000", 2023-10-01T10:41:56.612 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00063231400000000003 2023-10-01T10:41:56.612 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:41:56.612 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:41:56.612 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:41:56.612 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:56.578049+0000", 2023-10-01T10:41:56.612 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00013344499999999999 2023-10-01T10:41:56.612 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:41:56.613 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:41:56.613 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 3,4", 2023-10-01T10:41:56.613 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:56.578195+0000", 2023-10-01T10:41:56.613 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000146002 2023-10-01T10:41:56.613 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:41:56.613 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:41:56.613 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:41:56.613 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:41:56.613 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:41:56.613 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.35649.0:74574 9.3 e132/128 9:cd3ec05a:::benchmark_data_smithi078_113121_object74573:head v 132'4624, mlcod=132'4621)", 2023-10-01T10:41:56.613 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:41:56.577620+0000", 2023-10-01T10:41:56.613 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 0.001932575, 2023-10-01T10:41:56.613 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0040035239999999996, 2023-10-01T10:41:56.614 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:41:56.614 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "started", 2023-10-01T10:41:56.614 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:41:56.614 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:41:56.614 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:41:56.614 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:56.577620+0000", 2023-10-01T10:41:56.614 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:41:56.614 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:41:56.614 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:41:56.615 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:41:56.615 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:56.577620+0000", 2023-10-01T10:41:56.615 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:41:56.615 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:41:56.615 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:41:56.615 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:41:56.615 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:56.577619+0000", 2023-10-01T10:41:56.615 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:41:56.616 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:41:56.616 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:41:56.616 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:41:56.616 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:56.577664+0000", 2023-10-01T10:41:56.616 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.4592000000000003e-05 2023-10-01T10:41:56.616 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:41:56.616 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:41:56.616 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:41:56.617 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:56.577677+0000", 2023-10-01T10:41:56.617 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.3207000000000001e-05 2023-10-01T10:41:56.617 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:41:56.617 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:41:56.617 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:41:56.617 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:56.577685+0000", 2023-10-01T10:41:56.617 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.4319999999999999e-06 2023-10-01T10:41:56.617 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:41:56.617 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:41:56.617 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:41:56.617 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:56.580540+0000", 2023-10-01T10:41:56.617 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0028542049999999998 2023-10-01T10:41:56.618 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:41:56.618 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:41:56.618 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:41:56.618 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:56.580559+0000", 2023-10-01T10:41:56.618 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.9287e-05 2023-10-01T10:41:56.618 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:41:56.618 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:41:56.618 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:41:56.618 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:41:56.618 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:41:56.618 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.35649.0:74581 9.a e132/128 9:5d294249:::benchmark_data_smithi078_113121_object74580:head v 132'4675, mlcod=132'4672)", 2023-10-01T10:41:56.618 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:41:56.580076+0000", 2023-10-01T10:41:56.619 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 4294967295.9994769, 2023-10-01T10:41:56.619 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0016103000000000001, 2023-10-01T10:41:56.619 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:41:56.619 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "started", 2023-10-01T10:41:56.619 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:41:56.619 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:41:56.619 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:41:56.619 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:56.580076+0000", 2023-10-01T10:41:56.619 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:41:56.619 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:41:56.619 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:41:56.619 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:41:56.619 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:56.580076+0000", 2023-10-01T10:41:56.620 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:41:56.620 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:41:56.620 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:41:56.620 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:41:56.620 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:56.580075+0000", 2023-10-01T10:41:56.620 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:41:56.620 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:41:56.620 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:41:56.620 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:41:56.620 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:56.580127+0000", 2023-10-01T10:41:56.620 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.219e-05 2023-10-01T10:41:56.620 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:41:56.621 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:41:56.621 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:41:56.621 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:56.580144+0000", 2023-10-01T10:41:56.621 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.7124000000000001e-05 2023-10-01T10:41:56.621 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:41:56.621 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:41:56.621 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:41:56.621 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:56.580152+0000", 2023-10-01T10:41:56.621 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.9070000000000007e-06 2023-10-01T10:41:56.621 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:41:56.622 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:41:56.622 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:41:56.622 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:56.580232+0000", 2023-10-01T10:41:56.622 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.9835999999999999e-05 2023-10-01T10:41:56.622 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:41:56.622 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:41:56.622 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:41:56.622 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:56.580277+0000", 2023-10-01T10:41:56.622 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.5290999999999997e-05 2023-10-01T10:41:56.623 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:41:56.623 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:41:56.623 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:41:56.623 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:41:56.623 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2023-10-01T10:41:56.623 INFO:teuthology.orchestra.run.smithi078.stdout: "num_ops": 6 2023-10-01T10:41:56.623 INFO:teuthology.orchestra.run.smithi078.stdout:} 2023-10-01T10:41:56.623 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:56.648 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:56.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:56.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:41:56.753 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:56.784 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:56.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:56.788 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:41:56.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:56 smithi173 bash[58816]: audit 2023-10-01T10:41:56.061260+0000 mon.b (mon.1) 40 : audit 0 from='client.? 172.21.15.78:0/351628237' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:41:56.888 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:56.897 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:56 smithi078 bash[78274]: audit 2023-10-01T10:41:56.061260+0000 mon.b (mon.1) 40 : audit 0 from='client.? 172.21.15.78:0/351628237' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:41:56.897 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:56 smithi078 bash[75299]: audit 2023-10-01T10:41:56.061260+0000 mon.b (mon.1) 40 : audit 0 from='client.? 172.21.15.78:0/351628237' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:41:56.917 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:56.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:56.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:41:57.020 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:57.050 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:57.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:57.053 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:41:57.153 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:57.182 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:41:57.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:57.185 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:41:57.217 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 49 16 75208 75192 95.8898 55.5 0.00420749 0.0104233 2023-10-01T10:41:57.286 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:57.325 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:57.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:57.331 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:41:57.432 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:57.451 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:57.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:57.455 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:41:57.555 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:57.579 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:57.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:57.581 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:41:57.641 INFO:teuthology.orchestra.run.smithi078.stderr:nodeep-scrub is unset 2023-10-01T10:41:57.645 INFO:teuthology.orchestra.run.smithi078.stderr:set pool 9 pgp_num to 16 2023-10-01T10:41:57.682 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:57.690 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:57 smithi078 bash[75299]: cluster 2023-10-01T10:41:55.737201+0000 mgr.y (mgr.25072) 725 : cluster 0 pgmap v633: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-10-01T10:41:57.691 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:57 smithi078 bash[75299]: audit 2023-10-01T10:41:56.698033+0000 mon.c (mon.2) 136 : audit 1 from='client.? 172.21.15.78:0/1483052801' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-01T10:41:57.691 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:57 smithi078 bash[75299]: audit 2023-10-01T10:41:56.699332+0000 mon.a (mon.0) 947 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-01T10:41:57.691 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:57 smithi078 bash[75299]: audit 2023-10-01T10:41:56.989588+0000 mon.a (mon.0) 948 : audit 1 from='client.? 172.21.15.78:0/1076031059' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:41:57.691 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:57 smithi078 bash[78274]: cluster 2023-10-01T10:41:55.737201+0000 mgr.y (mgr.25072) 725 : cluster 0 pgmap v633: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-10-01T10:41:57.692 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:57 smithi078 bash[78274]: audit 2023-10-01T10:41:56.698033+0000 mon.c (mon.2) 136 : audit 1 from='client.? 172.21.15.78:0/1483052801' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-01T10:41:57.692 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:57 smithi078 bash[78274]: audit 2023-10-01T10:41:56.699332+0000 mon.a (mon.0) 947 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-01T10:41:57.692 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:57 smithi078 bash[78274]: audit 2023-10-01T10:41:56.989588+0000 mon.a (mon.0) 948 : audit 1 from='client.? 172.21.15.78:0/1076031059' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:41:57.720 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:57.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:57.726 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:41:57.827 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:57.853 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:57.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:57.855 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:41:57.956 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:57.965 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:57 smithi173 bash[58816]: cluster 2023-10-01T10:41:55.737201+0000 mgr.y (mgr.25072) 725 : cluster 0 pgmap v633: 89 pgs: 89 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-10-01T10:41:57.965 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:57 smithi173 bash[58816]: audit 2023-10-01T10:41:56.698033+0000 mon.c (mon.2) 136 : audit 1 from='client.? 172.21.15.78:0/1483052801' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-01T10:41:57.965 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:57 smithi173 bash[58816]: audit 2023-10-01T10:41:56.699332+0000 mon.a (mon.0) 947 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-01T10:41:57.965 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:57 smithi173 bash[58816]: audit 2023-10-01T10:41:56.989588+0000 mon.a (mon.0) 948 : audit 1 from='client.? 172.21.15.78:0/1076031059' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:41:57.986 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:57.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:57.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:41:58.089 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:58.120 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:58.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:58.122 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:41:58.218 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 50 16 76104 76088 95.092 56 0.00824032 0.0105102 2023-10-01T10:41:58.223 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:58.244 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:41:58.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:58.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:41:58.347 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:58.378 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:58.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:58.383 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:41:58.484 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:58.514 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:58.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:58.519 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:41:58.619 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:58.651 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:41:58.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:58.654 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:41:58.755 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:58.763 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:58 smithi173 bash[58816]: cluster 2023-10-01T10:41:57.617380+0000 mon.a (mon.0) 949 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T10:41:58.763 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:58 smithi173 bash[58816]: cluster 2023-10-01T10:41:57.617431+0000 mon.a (mon.0) 950 : cluster 1 Cluster is now healthy 2023-10-01T10:41:58.763 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:58 smithi173 bash[58816]: audit 2023-10-01T10:41:57.629689+0000 mon.a (mon.0) 951 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-10-01T10:41:58.763 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:58 smithi173 bash[58816]: audit 2023-10-01T10:41:57.629778+0000 mon.a (mon.0) 952 : audit 1 from='client.? 172.21.15.78:0/1076031059' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T10:41:58.763 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:58 smithi173 bash[58816]: cluster 2023-10-01T10:41:57.629831+0000 mon.a (mon.0) 953 : cluster 0 osdmap e133: 8 total, 8 up, 8 in 2023-10-01T10:41:58.763 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:58 smithi173 bash[58816]: cluster 2023-10-01T10:41:57.635281+0000 osd.7 (osd.7) 78 : cluster 3 failed to encode map e133 with expected crc 2023-10-01T10:41:58.764 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:58 smithi173 bash[58816]: cluster 2023-10-01T10:41:57.639207+0000 osd.6 (osd.6) 48 : cluster 3 failed to encode map e133 with expected crc 2023-10-01T10:41:58.764 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:58 smithi173 bash[58816]: cluster 2023-10-01T10:41:57.639811+0000 osd.5 (osd.5) 57 : cluster 3 failed to encode map e133 with expected crc 2023-10-01T10:41:58.764 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:58 smithi173 bash[58816]: cluster 2023-10-01T10:41:57.640045+0000 osd.7 (osd.7) 79 : cluster 3 failed to encode map e133 with expected crc 2023-10-01T10:41:58.764 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:58 smithi173 bash[58816]: cluster 2023-10-01T10:41:57.641636+0000 osd.5 (osd.5) 58 : cluster 3 failed to encode map e133 with expected crc 2023-10-01T10:41:58.764 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:58 smithi173 bash[58816]: cluster 2023-10-01T10:41:57.646198+0000 osd.5 (osd.5) 59 : cluster 3 failed to encode map e133 with expected crc 2023-10-01T10:41:58.781 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:41:58.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:58.785 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:41:58.886 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:58.915 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:58.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:58.919 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:41:59.020 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:59.028 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:58 smithi078 bash[75299]: cluster 2023-10-01T10:41:57.617380+0000 mon.a (mon.0) 949 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T10:41:59.029 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:58 smithi078 bash[75299]: cluster 2023-10-01T10:41:57.617431+0000 mon.a (mon.0) 950 : cluster 1 Cluster is now healthy 2023-10-01T10:41:59.029 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:58 smithi078 bash[75299]: audit 2023-10-01T10:41:57.629689+0000 mon.a (mon.0) 951 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-10-01T10:41:59.029 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:58 smithi078 bash[75299]: audit 2023-10-01T10:41:57.629778+0000 mon.a (mon.0) 952 : audit 1 from='client.? 172.21.15.78:0/1076031059' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T10:41:59.029 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:58 smithi078 bash[75299]: cluster 2023-10-01T10:41:57.629831+0000 mon.a (mon.0) 953 : cluster 0 osdmap e133: 8 total, 8 up, 8 in 2023-10-01T10:41:59.029 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:58 smithi078 bash[75299]: cluster 2023-10-01T10:41:57.635281+0000 osd.7 (osd.7) 78 : cluster 3 failed to encode map e133 with expected crc 2023-10-01T10:41:59.029 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:58 smithi078 bash[75299]: cluster 2023-10-01T10:41:57.639207+0000 osd.6 (osd.6) 48 : cluster 3 failed to encode map e133 with expected crc 2023-10-01T10:41:59.029 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:58 smithi078 bash[75299]: cluster 2023-10-01T10:41:57.639811+0000 osd.5 (osd.5) 57 : cluster 3 failed to encode map e133 with expected crc 2023-10-01T10:41:59.029 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:58 smithi078 bash[75299]: cluster 2023-10-01T10:41:57.640045+0000 osd.7 (osd.7) 79 : cluster 3 failed to encode map e133 with expected crc 2023-10-01T10:41:59.029 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:58 smithi078 bash[75299]: cluster 2023-10-01T10:41:57.641636+0000 osd.5 (osd.5) 58 : cluster 3 failed to encode map e133 with expected crc 2023-10-01T10:41:59.029 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:58 smithi078 bash[75299]: cluster 2023-10-01T10:41:57.646198+0000 osd.5 (osd.5) 59 : cluster 3 failed to encode map e133 with expected crc 2023-10-01T10:41:59.030 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:58 smithi078 bash[78274]: cluster 2023-10-01T10:41:57.617380+0000 mon.a (mon.0) 949 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T10:41:59.030 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:58 smithi078 bash[78274]: cluster 2023-10-01T10:41:57.617431+0000 mon.a (mon.0) 950 : cluster 1 Cluster is now healthy 2023-10-01T10:41:59.030 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:58 smithi078 bash[78274]: audit 2023-10-01T10:41:57.629689+0000 mon.a (mon.0) 951 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-10-01T10:41:59.030 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:58 smithi078 bash[78274]: audit 2023-10-01T10:41:57.629778+0000 mon.a (mon.0) 952 : audit 1 from='client.? 172.21.15.78:0/1076031059' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T10:41:59.030 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:58 smithi078 bash[78274]: cluster 2023-10-01T10:41:57.629831+0000 mon.a (mon.0) 953 : cluster 0 osdmap e133: 8 total, 8 up, 8 in 2023-10-01T10:41:59.030 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:58 smithi078 bash[78274]: cluster 2023-10-01T10:41:57.635281+0000 osd.7 (osd.7) 78 : cluster 3 failed to encode map e133 with expected crc 2023-10-01T10:41:59.030 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:58 smithi078 bash[78274]: cluster 2023-10-01T10:41:57.639207+0000 osd.6 (osd.6) 48 : cluster 3 failed to encode map e133 with expected crc 2023-10-01T10:41:59.030 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:58 smithi078 bash[78274]: cluster 2023-10-01T10:41:57.639811+0000 osd.5 (osd.5) 57 : cluster 3 failed to encode map e133 with expected crc 2023-10-01T10:41:59.030 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:58 smithi078 bash[78274]: cluster 2023-10-01T10:41:57.640045+0000 osd.7 (osd.7) 79 : cluster 3 failed to encode map e133 with expected crc 2023-10-01T10:41:59.030 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:58 smithi078 bash[78274]: cluster 2023-10-01T10:41:57.641636+0000 osd.5 (osd.5) 58 : cluster 3 failed to encode map e133 with expected crc 2023-10-01T10:41:59.031 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:58 smithi078 bash[78274]: cluster 2023-10-01T10:41:57.646198+0000 osd.5 (osd.5) 59 : cluster 3 failed to encode map e133 with expected crc 2023-10-01T10:41:59.052 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:59.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:59.056 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:41:59.157 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:59.182 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:59.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:59.184 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:41:59.217 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 51 16 77111 77095 94.4615 62.9375 0.0124664 0.0105807 2023-10-01T10:41:59.285 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:59.309 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:41:59.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:59.312 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:41:59.413 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:59.436 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:41:59.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:59.440 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:41:59.541 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:59.580 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:59.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:59.583 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:41:59.684 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:59.693 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:41:59 smithi078 bash[78274]: cluster 2023-10-01T10:41:57.738021+0000 mgr.y (mgr.25072) 726 : cluster 0 pgmap v635: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-01T10:41:59.693 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:41:59 smithi078 bash[75299]: cluster 2023-10-01T10:41:57.738021+0000 mgr.y (mgr.25072) 726 : cluster 0 pgmap v635: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-01T10:41:59.709 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:59.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:59.711 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:41:59.812 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:59.832 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:41:59.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:59.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:41:59.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:41:59 smithi173 bash[58816]: cluster 2023-10-01T10:41:57.738021+0000 mgr.y (mgr.25072) 726 : cluster 0 pgmap v635: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-10-01T10:41:59.935 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:59.967 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:41:59.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:41:59.971 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:00.046 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-01T10:42:00.071 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:00.101 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:00.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:00.106 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:42:00.206 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:00.217 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 52 16 78146 78130 93.8888 64.6875 0.0110426 0.0106452 2023-10-01T10:42:00.241 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:00.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:00.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:42:00.331 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:42:00 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:42:00] "GET /metrics HTTP/1.1" 200 36245 "" "Prometheus/2.33.4" 2023-10-01T10:42:00.347 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:00.381 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:00.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:00.385 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:42:00.485 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:00.514 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:00.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:00.521 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:42:00.621 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:00.653 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:00.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:00.654 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:42:00.755 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:00.779 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:00.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:00.783 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:00.884 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:00.906 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:00.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:00.910 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:01.011 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:01.046 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:01.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:01.051 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:42:01.152 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:01.211 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:01.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:01.213 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:42:01.218 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 53 16 79188 79172 93.3461 65.125 0.011258 0.0107073 2023-10-01T10:42:01.314 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:01.341 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:01.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:01.348 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:42:01.448 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:01.484 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:01.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:01.488 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:42:01.588 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:01.619 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:01.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:01.620 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:42:01.720 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:01.731 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:01 smithi078 bash[75299]: cluster 2023-10-01T10:41:59.738598+0000 mgr.y (mgr.25072) 727 : cluster 0 pgmap v636: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-01T10:42:01.731 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:01 smithi078 bash[78274]: cluster 2023-10-01T10:41:59.738598+0000 mgr.y (mgr.25072) 727 : cluster 0 pgmap v636: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-01T10:42:01.747 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:01.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:01.749 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:42:01.850 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:01.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:01 smithi173 bash[58816]: cluster 2023-10-01T10:41:59.738598+0000 mgr.y (mgr.25072) 727 : cluster 0 pgmap v636: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-01T10:42:01.886 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:01.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:01.890 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:42:01.941 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T10:42:01.992 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:02.015 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:02.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:02.020 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:02.120 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:02.148 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:02.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:02.150 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:42:02.219 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 54 16 80210 80194 92.7985 63.875 0.011116 0.0107702 2023-10-01T10:42:02.251 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:02.279 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:02.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:02.285 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:42:02.385 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:02.411 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:02.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:02.415 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:42:02.515 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:02.545 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:02.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:02.551 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:42:02.652 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:02.684 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:02.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:02.686 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:42:02.787 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:02.821 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:02.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:02.826 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:42:02.927 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:02.959 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:02.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:02.962 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:03.063 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:03.098 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:03.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:03.105 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:42:03.140 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T10:42:03.205 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:03.220 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 55 15 81226 81211 92.265 63.5625 0.0162857 0.0108323 2023-10-01T10:42:03.229 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:03.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:03.231 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:42:03.332 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:03.365 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:03.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:03.369 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:42:03.419 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:03 smithi078 bash[75299]: cluster 2023-10-01T10:42:01.739080+0000 mgr.y (mgr.25072) 728 : cluster 0 pgmap v637: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-01T10:42:03.419 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:03 smithi078 bash[75299]: audit 2023-10-01T10:42:02.419082+0000 mon.a (mon.0) 954 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:42:03.419 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:03 smithi078 bash[75299]: audit 2023-10-01T10:42:02.420539+0000 mon.a (mon.0) 955 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:42:03.420 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:03 smithi078 bash[78274]: cluster 2023-10-01T10:42:01.739080+0000 mgr.y (mgr.25072) 728 : cluster 0 pgmap v637: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-01T10:42:03.420 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:03 smithi078 bash[78274]: audit 2023-10-01T10:42:02.419082+0000 mon.a (mon.0) 954 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:42:03.420 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:03 smithi078 bash[78274]: audit 2023-10-01T10:42:02.420539+0000 mon.a (mon.0) 955 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:42:03.469 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:03.499 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:03.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:03.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:42:03.604 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:03.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:03 smithi173 bash[58816]: cluster 2023-10-01T10:42:01.739080+0000 mgr.y (mgr.25072) 728 : cluster 0 pgmap v637: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-01T10:42:03.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:03 smithi173 bash[58816]: audit 2023-10-01T10:42:02.419082+0000 mon.a (mon.0) 954 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:42:03.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:03 smithi173 bash[58816]: audit 2023-10-01T10:42:02.420539+0000 mon.a (mon.0) 955 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:42:03.633 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:03.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:03.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:42:03.736 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:03.767 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:03.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:03.770 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:42:03.870 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:03.898 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:03.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:03.902 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:42:04.003 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:04.029 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:04.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:04.032 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:42:04.133 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:04.162 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:04.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:04.165 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:42:04.220 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 56 16 82226 82210 91.7324 62.4375 0.00436124 0.0108954 2023-10-01T10:42:04.266 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:04.297 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:04.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:04.301 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:04.402 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:04.424 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:04.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:04.427 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:42:04.528 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:04.553 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:04.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:04.558 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:42:04.658 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:04.687 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:04.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:04.693 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:42:04.794 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:04.820 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:04.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:04.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:42:04.923 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:04.960 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:04.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:04.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:42:05.064 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:05.088 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:05.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:05.092 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:42:05.193 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:05.219 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:05.221 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 57 16 83237 83221 91.2304 63.1875 0.0214386 0.0109548 2023-10-01T10:42:05.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:05.223 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:05.286 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:42:05.324 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:05.361 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:05.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:05.364 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:42:05.465 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:05.491 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:05.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:05.497 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:42:05.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:05 smithi078 bash[75299]: cluster 2023-10-01T10:42:03.739705+0000 mgr.y (mgr.25072) 729 : cluster 0 pgmap v638: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-01T10:42:05.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:05 smithi078 bash[78274]: cluster 2023-10-01T10:42:03.739705+0000 mgr.y (mgr.25072) 729 : cluster 0 pgmap v638: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-01T10:42:05.598 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:05.624 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:05.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:05.626 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:42:05.727 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:05.761 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:05.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:05.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:42:05.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:05 smithi173 bash[58816]: cluster 2023-10-01T10:42:03.739705+0000 mgr.y (mgr.25072) 729 : cluster 0 pgmap v638: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-01T10:42:05.871 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:05.900 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:05.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:05.903 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:42:06.004 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:06.034 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:06.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:06.040 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:42:06.122 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T10:42:06.122 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T10:42:06.122 INFO:tasks.thrashosds.thrasher:Growing pool device_health_metrics 2023-10-01T10:42:06.123 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T10:42:06.141 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:06.173 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:06.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:06.178 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:42:06.221 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 58 16 84241 84225 90.7394 62.75 0.0184131 0.0110155 2023-10-01T10:42:06.278 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:06.306 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:42:06.308 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:06.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:06.311 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:06.411 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:06.447 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:06.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:06.450 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:06.550 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:06.587 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:06.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:06.591 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:42:06.692 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:06.716 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:06.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:06.719 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:42:06.819 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:06.855 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:06.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:06.859 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:06.960 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:06.989 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:06.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:06.992 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:42:07.092 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:07.120 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:07.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:07.126 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:42:07.221 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 59 16 85224 85208 90.2427 61.4375 0.0296936 0.0110752 2023-10-01T10:42:07.227 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:07.258 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:07.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:07.264 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:07.324 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:42:07.365 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:07.396 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:07.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:07.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:07.499 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:07.542 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:07.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:07.545 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:42:07.646 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:07.655 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:07 smithi078 bash[75299]: cluster 2023-10-01T10:42:05.740575+0000 mgr.y (mgr.25072) 730 : cluster 0 pgmap v639: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-01T10:42:07.655 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:07 smithi078 bash[78274]: cluster 2023-10-01T10:42:05.740575+0000 mgr.y (mgr.25072) 730 : cluster 0 pgmap v639: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-01T10:42:07.670 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:07.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:07.675 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:42:07.776 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:07.785 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:07 smithi173 bash[58816]: cluster 2023-10-01T10:42:05.740575+0000 mgr.y (mgr.25072) 730 : cluster 0 pgmap v639: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-01T10:42:07.812 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:07.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:07.818 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:07.919 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:07.944 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:07.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:07.950 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:42:08.051 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:08.080 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:08.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:08.083 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:42:08.184 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:08.218 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:08.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:08.221 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:42:08.221 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:2023-10-01T10:42:08.220653+0000 min lat: 0.00132359 max lat: 0.210168 avg lat: 0.0111339 2023-10-01T10:42:08.221 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T10:42:08.221 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 60 16 86216 86200 89.772 62 0.0167155 0.0111339 2023-10-01T10:42:08.322 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:08.355 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:08.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:08.359 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:08.460 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:08.488 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:08.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:08.490 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:42:08.591 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:08.618 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:08.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:08.624 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:42:08.724 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:08.762 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:08.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:08.766 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:08.866 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:08.904 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:08.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:08.910 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:09.010 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:09.047 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:09.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:09.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:09.154 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:09.186 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:09.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:09.191 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:42:09.221 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 61 16 87067 87051 89.1722 53.1875 0.0122417 0.0112089 2023-10-01T10:42:09.291 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:09.313 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:09.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:09.316 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:42:09.417 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:09.440 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:09 smithi173 bash[58816]: cluster 2023-10-01T10:42:07.741102+0000 mgr.y (mgr.25072) 731 : cluster 0 pgmap v640: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 62 MiB/s wr, 989 op/s 2023-10-01T10:42:09.449 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:09.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:09.452 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:42:09.553 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:09.561 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:09 smithi078 bash[75299]: cluster 2023-10-01T10:42:07.741102+0000 mgr.y (mgr.25072) 731 : cluster 0 pgmap v640: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 62 MiB/s wr, 989 op/s 2023-10-01T10:42:09.562 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:09 smithi078 bash[78274]: cluster 2023-10-01T10:42:07.741102+0000 mgr.y (mgr.25072) 731 : cluster 0 pgmap v640: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 62 MiB/s wr, 989 op/s 2023-10-01T10:42:09.579 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:09.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:09.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:09.683 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:09.709 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:09.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:09.714 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:42:09.815 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:09.845 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:09.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:09.847 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:09.948 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:09.980 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:09.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:09.986 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:42:10.086 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:10.121 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:10.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:10.126 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:42:10.221 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 62 16 88080 88064 88.7551 63.3125 0.0074879 0.0112615 2023-10-01T10:42:10.227 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:10.258 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:10.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:10.260 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:42:10.361 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:10.378 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:42:10.401 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:10.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:10.406 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:42:10.507 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:10.551 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:10.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:10.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:42:10.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:42:10 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:42:10] "GET /metrics HTTP/1.1" 200 36243 "" "Prometheus/2.33.4" 2023-10-01T10:42:10.656 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:10.696 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:10.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:10.700 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:10.800 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:10.823 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:10.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:10.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:42:10.928 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:10.952 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:10.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:10.955 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:42:11.057 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:11.084 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:11.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:11.090 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:42:11.191 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:11.221 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 63 16 88937 88921 88.1965 53.5625 0.0184993 0.011332 2023-10-01T10:42:11.226 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:11.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:11.230 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:42:11.331 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:11.355 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:11.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:11.359 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:11.460 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:11.488 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:11.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:11.491 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:11.591 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:11.613 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:11.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:11.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:42:11.720 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:11.737 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:11 smithi078 bash[75299]: cluster 2023-10-01T10:42:09.741635+0000 mgr.y (mgr.25072) 732 : cluster 0 pgmap v641: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-01T10:42:11.737 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:11 smithi078 bash[78274]: cluster 2023-10-01T10:42:09.741635+0000 mgr.y (mgr.25072) 732 : cluster 0 pgmap v641: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-01T10:42:11.756 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:11.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:11.760 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:42:11.861 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:11.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:11 smithi173 bash[58816]: cluster 2023-10-01T10:42:09.741635+0000 mgr.y (mgr.25072) 732 : cluster 0 pgmap v641: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-01T10:42:11.885 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:11.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:11.888 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:42:11.989 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:12.018 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:12.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:12.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:42:12.124 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:12.157 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:12.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:12.161 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:12.222 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 64 16 89803 89787 87.6641 54.125 0.0111068 0.0113924 2023-10-01T10:42:12.261 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:12.288 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:12.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:12.293 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:42:12.394 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:12.428 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:12.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:12.434 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:42:12.535 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:12.559 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:12.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:12.563 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:42:12.664 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:12.691 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:12.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:12.696 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:42:12.796 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:12.822 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:12.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:12.829 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:42:12.930 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:12.953 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:12.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:12.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:42:13.057 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:13.088 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:13.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:13.091 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:13.192 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:13.220 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:13.222 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 65 16 90720 90704 87.1971 57.3125 0.00503668 0.0114621 2023-10-01T10:42:13.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:13.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:42:13.325 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:13.356 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:13.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:13.361 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:42:13.461 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:13.484 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:13.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:13.487 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:42:13.587 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:13.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:13 smithi173 bash[58816]: cluster 2023-10-01T10:42:11.742375+0000 mgr.y (mgr.25072) 733 : cluster 0 pgmap v642: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-01T10:42:13.621 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:13.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:13.623 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:13 smithi078 bash[75299]: cluster 2023-10-01T10:42:11.742375+0000 mgr.y (mgr.25072) 733 : cluster 0 pgmap v642: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-01T10:42:13.624 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:13 smithi078 bash[78274]: cluster 2023-10-01T10:42:11.742375+0000 mgr.y (mgr.25072) 733 : cluster 0 pgmap v642: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-01T10:42:13.624 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:42:13.725 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:13.750 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:13.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:13.751 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:13.852 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:13.891 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:13.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:13.894 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:13.995 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:14.017 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:14.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:14.019 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:14.121 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:14.146 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:14.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:14.151 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:42:14.222 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 66 16 91705 91689 86.8087 61.5625 0.0124281 0.0115139 2023-10-01T10:42:14.252 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:14.280 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:14.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:14.287 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:42:14.387 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:14.413 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:14.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:14.416 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:42:14.516 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:14.543 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:14.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:14.547 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:42:14.648 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:14.680 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:14.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:14.683 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:14.783 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:14.819 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:14.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:14.822 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:14.922 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:14.954 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:14.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:14.959 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:42:15.060 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:15.097 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:15.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:15.102 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:42:15.202 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:15.222 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 67 16 92655 92639 86.3992 59.375 0.0137533 0.0115677 2023-10-01T10:42:15.236 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:15.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:15.239 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:42:15.340 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:15.373 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:15.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:15.378 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:42:15.478 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:15.488 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:15 smithi078 bash[75299]: cluster 2023-10-01T10:42:13.742887+0000 mgr.y (mgr.25072) 734 : cluster 0 pgmap v643: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 56 MiB/s wr, 897 op/s 2023-10-01T10:42:15.488 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:15 smithi078 bash[78274]: cluster 2023-10-01T10:42:13.742887+0000 mgr.y (mgr.25072) 734 : cluster 0 pgmap v643: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 56 MiB/s wr, 897 op/s 2023-10-01T10:42:15.512 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:15.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:15.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:42:15.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:15 smithi173 bash[58816]: cluster 2023-10-01T10:42:13.742887+0000 mgr.y (mgr.25072) 734 : cluster 0 pgmap v643: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 56 MiB/s wr, 897 op/s 2023-10-01T10:42:15.712 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:15.730 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:15.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:15.732 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:15.833 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:15.862 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:15.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:15.868 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:42:15.969 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:16.010 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:16.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:16.015 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:16.116 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:16.144 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:16.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:16.151 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:16.222 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 68 16 93481 93465 85.8878 51.625 0.0249863 0.0116375 2023-10-01T10:42:16.252 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:16.291 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:16.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:16.294 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:42:16.394 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:16.424 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:16.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:16.427 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:42:16.527 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:16.556 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:16.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:16.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:16.659 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:16.681 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:16.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:16.684 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:16.784 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:16.810 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:16.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:16.812 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:16.913 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:16.944 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:16.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:16.947 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:42:17.048 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:17.076 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:17.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:17.081 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:42:17.181 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:17.209 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:17.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:17.212 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:42:17.222 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 69 16 94434 94418 85.5062 59.5625 0.0395987 0.0116899 2023-10-01T10:42:17.313 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:17.341 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:17.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:17.346 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:17.446 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:17.472 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:17.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:17.474 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:42:17.575 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:17.593 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:17 smithi173 bash[58816]: cluster 2023-10-01T10:42:15.743470+0000 mgr.y (mgr.25072) 735 : cluster 0 pgmap v644: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-10-01T10:42:17.594 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:17 smithi173 bash[58816]: audit 2023-10-01T10:42:17.413281+0000 mon.a (mon.0) 956 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:42:17.620 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:17.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:17.624 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:42:17.724 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:17.752 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:17.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:17.756 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:42:17.838 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:17 smithi078 bash[75299]: cluster 2023-10-01T10:42:15.743470+0000 mgr.y (mgr.25072) 735 : cluster 0 pgmap v644: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-10-01T10:42:17.838 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:17 smithi078 bash[75299]: audit 2023-10-01T10:42:17.413281+0000 mon.a (mon.0) 956 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:42:17.838 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:17 smithi078 bash[78274]: cluster 2023-10-01T10:42:15.743470+0000 mgr.y (mgr.25072) 735 : cluster 0 pgmap v644: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-10-01T10:42:17.838 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:17 smithi078 bash[78274]: audit 2023-10-01T10:42:17.413281+0000 mon.a (mon.0) 956 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:42:17.856 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:17.883 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:17.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:17.886 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:42:17.986 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:18.008 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:18.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:18.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:18.111 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:18.154 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:18.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:18.158 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:42:18.222 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 70 16 95084 95068 84.865 40.625 0.0145654 0.0117783 2023-10-01T10:42:18.259 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:18.294 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:18.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:18.296 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:42:18.397 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:18.423 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:18.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:18.427 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:42:18.528 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:18.553 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:18.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:18.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:42:18.659 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:18.685 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:18.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:18.688 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:18.789 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:18.813 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:18.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:18.815 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:42:18.916 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:18.942 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:18.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:18.944 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:42:19.045 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:19.073 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:19.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:19.076 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:42:19.177 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:19.211 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:19.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:19.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:42:19.222 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 71 16 96012 95996 84.4866 58 0.0128234 0.0118307 2023-10-01T10:42:19.317 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:19.361 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:19.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:19.367 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:19.465 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:19 smithi078 bash[78274]: cluster 2023-10-01T10:42:17.743884+0000 mgr.y (mgr.25072) 736 : cluster 0 pgmap v645: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 56 MiB/s wr, 892 op/s 2023-10-01T10:42:19.467 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:19.494 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:19.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:19.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:19.602 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:19.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:19 smithi173 bash[58816]: cluster 2023-10-01T10:42:17.743884+0000 mgr.y (mgr.25072) 736 : cluster 0 pgmap v645: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 56 MiB/s wr, 892 op/s 2023-10-01T10:42:19.638 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:19.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:19.641 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:19.742 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:19.782 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:19.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:19.786 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:42:19.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:19 smithi078 bash[75299]: cluster 2023-10-01T10:42:17.743884+0000 mgr.y (mgr.25072) 736 : cluster 0 pgmap v645: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 56 MiB/s wr, 892 op/s 2023-10-01T10:42:19.887 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:19.914 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:19.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:19.916 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:42:20.017 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:20.049 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:20.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:20.054 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:20.155 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:20.190 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:20.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:20.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:42:20.223 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 72 16 96830 96814 84.0232 51.125 0.0100329 0.011896 2023-10-01T10:42:20.295 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:20.324 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:20.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:20.328 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:42:20.331 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:42:20 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:42:20] "GET /metrics HTTP/1.1" 200 36240 "" "Prometheus/2.33.4" 2023-10-01T10:42:20.429 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:20.459 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:20.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:20.465 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:42:20.566 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:20.590 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:20.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:20.592 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:20.693 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:20.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:20.718 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:20.719 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:42:20.819 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:20.841 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:20.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:20.847 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:42:20.948 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:20.971 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:20.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:20.975 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:42:21.006 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2023-10-01T10:42:21.006 INFO:teuthology.orchestra.run.smithi078.stdout: "ops": [], 2023-10-01T10:42:21.006 INFO:teuthology.orchestra.run.smithi078.stdout: "complaint_time": 30, 2023-10-01T10:42:21.006 INFO:teuthology.orchestra.run.smithi078.stdout: "num_blocked_ops": 0 2023-10-01T10:42:21.006 INFO:teuthology.orchestra.run.smithi078.stdout:} 2023-10-01T10:42:21.076 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:21.108 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:21.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:21.109 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:42:21.210 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:21.222 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 73 16 97747 97731 83.6573 57.3125 0.0147148 0.0119476 2023-10-01T10:42:21.245 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:21.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:21.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:42:21.352 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:21.379 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:21.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:21.382 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:42:21.471 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:21 smithi173 bash[58816]: cluster 2023-10-01T10:42:19.744567+0000 mgr.y (mgr.25072) 737 : cluster 0 pgmap v646: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 59 MiB/s wr, 946 op/s 2023-10-01T10:42:21.482 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:21.510 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:21.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:21.515 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:21.616 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:21.623 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:21 smithi078 bash[75299]: cluster 2023-10-01T10:42:19.744567+0000 mgr.y (mgr.25072) 737 : cluster 0 pgmap v646: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 59 MiB/s wr, 946 op/s 2023-10-01T10:42:21.623 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:21 smithi078 bash[78274]: cluster 2023-10-01T10:42:19.744567+0000 mgr.y (mgr.25072) 737 : cluster 0 pgmap v646: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 59 MiB/s wr, 946 op/s 2023-10-01T10:42:21.640 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:21.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:21.642 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:21.743 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:21.770 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:21.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:21.773 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:21.873 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:21.906 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:21.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:21.910 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:42:22.010 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:22.041 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:22.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:22.044 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:42:22.145 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:22.178 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:22.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:22.182 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:42:22.222 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 74 16 98550 98534 83.205 50.1875 0.00559945 0.0120113 2023-10-01T10:42:22.260 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T10:42:22.283 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:22.298 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T10:42:22.307 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:22.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:22.312 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:42:22.330 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T10:42:22.382 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T10:42:22.413 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T10:42:22.413 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:22.439 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:22.444 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T10:42:22.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:22.445 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:42:22.484 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T10:42:22.510 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T10:42:22.512 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T10:42:22.512 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T10:42:22.512 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T10:42:22.513 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T10:42:22.513 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T10:42:22.513 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T10:42:22.513 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T10:42:22.513 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T10:42:22.545 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:22.574 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:22.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:22.580 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:42:22.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:22 smithi173 bash[58816]: audit 2023-10-01T10:42:22.220552+0000 mon.a (mon.0) 957 : audit 0 from='client.? 172.21.15.78:0/2961724890' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:42:22.681 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:22.710 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:22.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:22.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:42:22.817 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:22.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:22 smithi078 bash[75299]: audit 2023-10-01T10:42:22.220552+0000 mon.a (mon.0) 957 : audit 0 from='client.? 172.21.15.78:0/2961724890' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:42:22.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:22 smithi078 bash[78274]: audit 2023-10-01T10:42:22.220552+0000 mon.a (mon.0) 957 : audit 0 from='client.? 172.21.15.78:0/2961724890' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:42:22.849 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:22.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:22.854 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:42:22.955 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:22.989 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:22.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:22.994 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:42:23.095 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:23.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:23.131 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:23.131 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:42:23.223 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 75 16 99347 99331 82.7598 49.8125 0.0111639 0.0120771 2023-10-01T10:42:23.232 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:23.256 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:23.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:23.265 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:42:23.365 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:23.391 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:23.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:23.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:42:23.495 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:23.498 INFO:teuthology.orchestra.run.smithi078.stderr:noscrub is set 2023-10-01T10:42:23.505 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:23 smithi078 bash[75299]: cluster 2023-10-01T10:42:21.745172+0000 mgr.y (mgr.25072) 738 : cluster 0 pgmap v647: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 60 MiB/s wr, 964 op/s 2023-10-01T10:42:23.505 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:23 smithi078 bash[75299]: audit 2023-10-01T10:42:22.649400+0000 mon.c (mon.2) 137 : audit 1 from='client.? 172.21.15.78:0/4279415151' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:42:23.506 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:23 smithi078 bash[75299]: audit 2023-10-01T10:42:22.649979+0000 mon.a (mon.0) 958 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:42:23.506 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:23 smithi078 bash[78274]: cluster 2023-10-01T10:42:21.745172+0000 mgr.y (mgr.25072) 738 : cluster 0 pgmap v647: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 60 MiB/s wr, 964 op/s 2023-10-01T10:42:23.507 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:23 smithi078 bash[78274]: audit 2023-10-01T10:42:22.649400+0000 mon.c (mon.2) 137 : audit 1 from='client.? 172.21.15.78:0/4279415151' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:42:23.507 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:23 smithi078 bash[78274]: audit 2023-10-01T10:42:22.649979+0000 mon.a (mon.0) 958 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:42:23.522 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:23.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:23.530 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:42:23.631 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:23.640 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:23 smithi173 bash[58816]: cluster 2023-10-01T10:42:21.745172+0000 mgr.y (mgr.25072) 738 : cluster 0 pgmap v647: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 60 MiB/s wr, 964 op/s 2023-10-01T10:42:23.640 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:23 smithi173 bash[58816]: audit 2023-10-01T10:42:22.649400+0000 mon.c (mon.2) 137 : audit 1 from='client.? 172.21.15.78:0/4279415151' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:42:23.640 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:23 smithi173 bash[58816]: audit 2023-10-01T10:42:22.649979+0000 mon.a (mon.0) 958 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:42:23.662 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:23.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:23.665 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:23.766 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:23.792 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:23.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:23.794 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:42:23.895 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:23.928 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:23.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:23.930 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:24.033 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:24.066 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:24.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:24.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:42:24.172 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:24.205 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:24.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:24.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:24.223 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 76 16 100133 100117 82.3172 49.125 0.0122567 0.0121429 2023-10-01T10:42:24.307 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:24.334 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:24.335 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-01T10:42:24.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:24.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:42:24.440 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:24.469 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:24.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:24.473 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:42:24.495 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:24 smithi173 bash[58816]: cluster 2023-10-01T10:42:23.485527+0000 mon.a (mon.0) 959 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:42:24.495 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:24 smithi173 bash[58816]: cluster 2023-10-01T10:42:23.493664+0000 osd.5 (osd.5) 60 : cluster 3 failed to encode map e134 with expected crc 2023-10-01T10:42:24.495 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:24 smithi173 bash[58816]: cluster 2023-10-01T10:42:23.494229+0000 osd.7 (osd.7) 80 : cluster 3 failed to encode map e134 with expected crc 2023-10-01T10:42:24.495 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:24 smithi173 bash[58816]: audit 2023-10-01T10:42:23.494230+0000 mon.a (mon.0) 960 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T10:42:24.495 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:24 smithi173 bash[58816]: cluster 2023-10-01T10:42:23.494279+0000 mon.a (mon.0) 961 : cluster 0 osdmap e134: 8 total, 8 up, 8 in 2023-10-01T10:42:24.541 INFO:teuthology.orchestra.run.smithi078.stdout: 2023-10-01T10:42:24.546 INFO:teuthology.orchestra.run.smithi078.stderr:dumped all 2023-10-01T10:42:24.574 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:24.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:24 smithi078 bash[78274]: cluster 2023-10-01T10:42:23.485527+0000 mon.a (mon.0) 959 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:42:24.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:24 smithi078 bash[78274]: cluster 2023-10-01T10:42:23.493664+0000 osd.5 (osd.5) 60 : cluster 3 failed to encode map e134 with expected crc 2023-10-01T10:42:24.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:24 smithi078 bash[78274]: cluster 2023-10-01T10:42:23.494229+0000 osd.7 (osd.7) 80 : cluster 3 failed to encode map e134 with expected crc 2023-10-01T10:42:24.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:24 smithi078 bash[78274]: audit 2023-10-01T10:42:23.494230+0000 mon.a (mon.0) 960 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T10:42:24.585 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:24 smithi078 bash[78274]: cluster 2023-10-01T10:42:23.494279+0000 mon.a (mon.0) 961 : cluster 0 osdmap e134: 8 total, 8 up, 8 in 2023-10-01T10:42:24.586 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:24 smithi078 bash[75299]: cluster 2023-10-01T10:42:23.485527+0000 mon.a (mon.0) 959 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:42:24.586 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:24 smithi078 bash[75299]: cluster 2023-10-01T10:42:23.493664+0000 osd.5 (osd.5) 60 : cluster 3 failed to encode map e134 with expected crc 2023-10-01T10:42:24.586 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:24 smithi078 bash[75299]: cluster 2023-10-01T10:42:23.494229+0000 osd.7 (osd.7) 80 : cluster 3 failed to encode map e134 with expected crc 2023-10-01T10:42:24.586 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:24 smithi078 bash[75299]: audit 2023-10-01T10:42:23.494230+0000 mon.a (mon.0) 960 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T10:42:24.586 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:24 smithi078 bash[75299]: cluster 2023-10-01T10:42:23.494279+0000 mon.a (mon.0) 961 : cluster 0 osdmap e134: 8 total, 8 up, 8 in 2023-10-01T10:42:24.604 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:24.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:24.608 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:42:24.709 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:24.742 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:24.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:24.744 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:24.845 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:24.869 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:24.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:24.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:24.972 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:25.000 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:25.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:25.002 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:42:25.103 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:25.133 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:25.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:25.139 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:42:25.223 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 77 16 100991 100975 81.9445 53.625 0.0103461 0.0121975 2023-10-01T10:42:25.240 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:25.262 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:25.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:25.264 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:42:25.364 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:25.403 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:25.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:25.409 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:42:25.510 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:25.539 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:25.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:25.543 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:42:25.644 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:25.672 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:25.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:25.675 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:25.776 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:25.787 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:25 smithi078 bash[75299]: cluster 2023-10-01T10:42:23.745565+0000 mgr.y (mgr.25072) 739 : cluster 0 pgmap v649: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 55 MiB/s wr, 885 op/s 2023-10-01T10:42:25.787 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:25 smithi078 bash[75299]: audit 2023-10-01T10:42:24.519497+0000 mgr.y (mgr.25072) 740 : audit 0 from='client.35691 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:42:25.787 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:25 smithi078 bash[78274]: cluster 2023-10-01T10:42:23.745565+0000 mgr.y (mgr.25072) 739 : cluster 0 pgmap v649: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 55 MiB/s wr, 885 op/s 2023-10-01T10:42:25.787 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:25 smithi078 bash[78274]: audit 2023-10-01T10:42:24.519497+0000 mgr.y (mgr.25072) 740 : audit 0 from='client.35691 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:42:25.812 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:25.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:25.815 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:25.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:25 smithi173 bash[58816]: cluster 2023-10-01T10:42:23.745565+0000 mgr.y (mgr.25072) 739 : cluster 0 pgmap v649: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 55 MiB/s wr, 885 op/s 2023-10-01T10:42:25.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:25 smithi173 bash[58816]: audit 2023-10-01T10:42:24.519497+0000 mgr.y (mgr.25072) 740 : audit 0 from='client.35691 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:42:25.916 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:25.943 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:25.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:25.953 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:42:26.054 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:26.083 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:26.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:26.085 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:42:26.186 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:26.211 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:26.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:26.216 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:26.223 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 78 16 101818 101802 81.5566 51.6875 0.00621106 0.0122558 2023-10-01T10:42:26.317 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:26.356 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:26.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:26.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:42:26.461 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:26.490 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:26.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:26.492 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:42:26.593 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:26.618 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:26.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:26.623 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:42:26.724 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:26.753 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:26.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:26.755 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:42:26.856 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:26.878 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:26.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:26.881 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:26.981 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:27.013 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:27.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:27.018 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:27.119 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:27.133 INFO:teuthology.orchestra.run.smithi078.stdout:{"pg_ready":true,"pg_map":{"version":649,"stamp":"2023-10-01T10:42:23.745335+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6372464544,"num_objects":97251,"num_object_clones":0,"num_object_copies":291753,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":97251,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":97488,"num_write_kb":6224974,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97422,"ondisk_log_size":97422,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":21302428,"kb_used_data":18652508,"kb_used_omap":30,"kb_used_meta":2649825,"kb_avail":728658788,"statfs":{"total":767960285184,"available":746146598912,"internally_reserved":0,"allocated":19100168192,"data_stored":19096071567,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31502,"internal_metadata":2713421042},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,5,0,2,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":580255744,"num_objects":8854,"num_object_clones":0,"num_object_copies":26562,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8854,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8854,"num_write_kb":566656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002778"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":120,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-01T10:41:33.519602+0000","last_change":"2023-10-01T10:38:16.032258+0000","last_active":"2023-10-01T10:41:33.519602+0000","last_peered":"2023-10-01T10:41:33.519602+0000","last_clean":"2023-10-01T10:41:33.519602+0000","last_became_active":"2023-10-01T10:38:16.031633+0000","last_became_peered":"2023-10-01T10:38:16.031633+0000","last_unstale":"2023-10-01T10:41:33.519602+0000","last_undegraded":"2023-10-01T10:41:33.519602+0000","last_fullsized":"2023-10-01T10:41:33.519602+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:17:02.446430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":108,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-01T10:41:33.527134+0000","last_change":"2023-10-01T10:38:16.025881+0000","last_active":"2023-10-01T10:41:33.527134+0000","last_peered":"2023-10-01T10:41:33.527134+0000","last_clean":"2023-10-01T10:41:33.527134+0000","last_became_active":"2023-10-01T10:38:16.025518+0000","last_became_peered":"2023-10-01T10:38:16.025518+0000","last_unstale":"2023-10-01T10:41:33.527134+0000","last_undegraded":"2023-10-01T10:41:33.527134+0000","last_fullsized":"2023-10-01T10:41:33.527134+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:48:37.826604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":87,"reported_epoch":133,"state":"active+clean","last_fresh":"2023-10-01T10:41:57.640119+0000","last_change":"2023-10-01T10:38:16.025075+0000","last_active":"2023-10-01T10:41:57.640119+0000","last_peered":"2023-10-01T10:41:57.640119+0000","last_clean":"2023-10-01T10:41:57.640119+0000","last_became_active":"2023-10-01T10:38:16.024735+0000","last_became_peered":"2023-10-01T10:38:16.024735+0000","last_unstale":"2023-10-01T10:41:57.640119+0000","last_undegraded":"2023-10-01T10:41:57.640119+0000","last_fullsized":"2023-10-01T10:41:57.640119+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:38:01.450741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":91,"reported_epoch":133,"state":"active+clean","last_fresh":"2023-10-01T10:41:57.637358+0000","last_change":"2023-10-01T10:37:42.772008+0000","last_active":"2023-10-01T10:41:57.637358+0000","last_peered":"2023-10-01T10:41:57.637358+0000","last_clean":"2023-10-01T10:41:57.637358+0000","last_became_active":"2023-10-01T10:37:42.771244+0000","last_became_peered":"2023-10-01T10:37:42.771244+0000","last_unstale":"2023-10-01T10:41:57.637358+0000","last_undegraded":"2023-10-01T10:41:57.637358+0000","last_fullsized":"2023-10-01T10:41:57.637358+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:38:55.180790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":109,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-01T10:41:33.530375+0000","last_change":"2023-10-01T10:37:42.766600+0000","last_active":"2023-10-01T10:41:33.530375+0000","last_peered":"2023-10-01T10:41:33.530375+0000","last_clean":"2023-10-01T10:41:33.530375+0000","last_became_active":"2023-10-01T10:37:42.766051+0000","last_became_peered":"2023-10-01T10:37:42.766051+0000","last_unstale":"2023-10-01T10:41:33.530375+0000","last_undegraded":"2023-10-01T10:41:33.530375+0000","last_fullsized":"2023-10-01T10:41:33.530375+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:51:17.707963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":98,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-01T10:41:33.536424+0000","last_change":"2023-10-01T10:32:31.575109+0000","last_active":"2023-10-01T10:41:33.536424+0000","last_peered":"2023-10-01T10:41:33.536424+0000","last_clean":"2023-10-01T10:41:33.536424+0000","last_became_active":"2023-10-01T10:32:31.574729+0000","last_became_peered":"2023-10-01T10:32:31.574729+0000","last_unstale":"2023-10-01T10:41:33.536424+0000","last_undegraded":"2023-10-01T10:41:33.536424+0000","last_fullsized":"2023-10-01T10:41:33.536424+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":88,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-01T10:41:33.523983+0000","last_change":"2023-10-01T10:37:42.766667+0000","last_active":"2023-10-01T10:41:33.523983+0000","last_peered":"2023-10-01T10:41:33.523983+0000","last_clean":"2023-10-01T10:41:33.523983+0000","last_became_active":"2023-10-01T10:37:42.766369+0000","last_became_peered":"2023-10-01T10:37:42.766369+0000","last_unstale":"2023-10-01T10:41:33.523983+0000","last_undegraded":"2023-10-01T10:41:33.523983+0000","last_fullsized":"2023-10-01T10:41:33.523983+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:02.708788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":94,"reported_epoch":133,"state":"active+clean","last_fresh":"2023-10-01T10:41:57.633281+0000","last_change":"2023-10-01T10:37:42.765925+0000","last_active":"2023-10-01T10:41:57.633281+0000","last_peered":"2023-10-01T10:41:57.633281+0000","last_clean":"2023-10-01T10:41:57.633281+0000","last_became_active":"2023-10-01T10:37:42.765218+0000","last_became_peered":"2023-10-01T10:37:42.765218+0000","last_unstale":"2023-10-01T10:41:57.633281+0000","last_undegraded":"2023-10-01T10:41:57.633281+0000","last_fullsized":"2023-10-01T10:41:57.633281+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T22:10:34.338031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":100,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-01T10:41:33.530761+0000","last_change":"2023-10-01T10:32:31.579805+0000","last_active":"2023-10-01T10:41:33.530761+0000","last_peered":"2023-10-01T10:41:33.530761+0000","last_clean":"2023-10-01T10:41:33.530761+0000","last_became_active":"2023-10-01T10:32:31.579423+0000","last_became_peered":"2023-10-01T10:32:31.579423+0000","last_unstale":"2023-10-01T10:41:33.530761+0000","last_undegraded":"2023-10-01T10:41:33.530761+0000","last_fullsized":"2023-10-01T10:41:33.530761+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":78,"reported_epoch":133,"state":"active+clean","last_fresh":"2023-10-01T10:41:57.640653+0000","last_change":"2023-10-01T10:37:42.772531+0000","last_active":"2023-10-01T10:41:57.640653+0000","last_peered":"2023-10-01T10:41:57.640653+0000","last_clean":"2023-10-01T10:41:57.640653+0000","last_became_active":"2023-10-01T10:37:42.772216+0000","last_became_peered":"2023-10-01T10:37:42.772216+0000","last_unstale":"2023-10-01T10:41:57.640653+0000","last_undegraded":"2023-10-01T10:41:57.640653+0000","last_fullsized":"2023-10-01T10:41:57.640653+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:01:53.373628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":118,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-01T10:41:33.527682+0000","last_change":"2023-10-01T10:38:16.027085+0000","last_active":"2023-10-01T10:41:33.527682+0000","last_peered":"2023-10-01T10:41:33.527682+0000","last_clean":"2023-10-01T10:41:33.527682+0000","last_became_active":"2023-10-01T10:38:16.026555+0000","last_became_peered":"2023-10-01T10:38:16.026555+0000","last_unstale":"2023-10-01T10:41:33.527682+0000","last_undegraded":"2023-10-01T10:41:33.527682+0000","last_fullsized":"2023-10-01T10:41:33.527682+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":72,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-01T10:41:50.425764+0000","last_change":"2023-10-01T10:37:05.018755+0000","last_active":"2023-10-01T10:41:50.425764+0000","last_peered":"2023-10-01T10:41:50.425764+0000","last_clean":"2023-10-01T10:41:50.425764+0000","last_became_active":"2023-10-01T10:37:05.018446+0000","last_became_peered":"2023-10-01T10:37:05.018446+0000","last_unstale":"2023-10-01T10:41:50.425764+0000","last_undegraded":"2023-10-01T10:41:50.425764+0000","last_fullsized":"2023-10-01T10:41:50.425764+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:13:53.740503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":114,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-01T10:41:33.520552+0000","last_change":"2023-10-01T10:38:16.030826+0000","last_active":"2023-10-01T10:41:33.520552+0000","last_peered":"2023-10-01T10:41:33.520552+0000","last_clean":"2023-10-01T10:41:33.520552+0000","last_became_active":"2023-10-01T10:38:16.030277+0000","last_became_peered":"2023-10-01T10:38:16.030277+0000","last_unstale":"2023-10-01T10:41:33.520552+0000","last_undegraded":"2023-10-01T10:41:33.520552+0000","last_fullsized":"2023-10-01T10:41:33.520552+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:31:43.040424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":88,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-01T10:41:33.520150+0000","last_change":"2023-10-01T10:38:16.034741+0000","last_active":"2023-10-01T10:41:33.520150+0000","last_peered":"2023-10-01T10:41:33.520150+0000","last_clean":"2023-10-01T10:41:33.520150+0000","last_became_active":"2023-10-01T10:38:16.034407+0000","last_became_peered":"2023-10-01T10:38:16.034407+0000","last_unstale":"2023-10-01T10:41:33.520150+0000","last_undegraded":"2023-10-01T10:41:33.520150+0000","last_fullsized":"2023-10-01T10:41:33.520150+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:47:29.911477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":117,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-01T10:41:33.527279+0000","last_change":"2023-10-01T10:38:16.026777+0000","last_active":"2023-10-01T10:41:33.527279+0000","last_peered":"2023-10-01T10:41:33.527279+0000","last_clean":"2023-10-01T10:41:33.527279+0000","last_became_active":"2023-10-01T10:38:16.026371+0000","last_became_peered":"2023-10-01T10:38:16.026371+0000","last_unstale":"2023-10-01T10:41:33.527279+0000","last_undegraded":"2023-10-01T10:41:33.527279+0000","last_fullsized":"2023-10-01T10:41:33.527279+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":117,"reported_epoch":133,"state":"active+clean","last_fresh":"2023-10-01T10:41:57.633467+0000","last_change":"2023-10-01T10:37:42.772319+0000","last_active":"2023-10-01T10:41:57.633467+0000","last_peered":"2023-10-01T10:41:57.633467+0000","last_clean":"2023-10-01T10:41:57.633467+0000","last_became_active":"2023-10-01T10:37:42.771977+0000","last_became_peered":"2023-10-01T10:37:42.771977+0000","last_unstale":"2023-10-01T10:41:57.633467+0000","last_undegraded":"2023-10-01T10:41:57.633467+0000","last_fullsized":"2023-10-01T10:41:57.633467+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:45:12.783214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":89,"reported_epoch":133,"state":"active+clean","last_fresh":"2023-10-01T10:41:57.636382+0000","last_change":"2023-10-01T10:37:42.767706+0000","last_active":"2023-10-01T10:41:57.636382+0000","last_peered":"2023-10-01T10:41:57.636382+0000","last_clean":"2023-10-01T10:41:57.636382+0000","last_became_active":"2023-10-01T10:37:42.767211+0000","last_became_peered":"2023-10-01T10:37:42.767211+0000","last_unstale":"2023-10-01T10:41:57.636382+0000","last_undegraded":"2023-10-01T10:41:57.636382+0000","last_fullsized":"2023-10-01T10:41:57.636382+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:37:20.357407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":99,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-01T10:41:33.530689+0000","last_change":"2023-10-01T10:37:42.768642+0000","last_active":"2023-10-01T10:41:33.530689+0000","last_peered":"2023-10-01T10:41:33.530689+0000","last_clean":"2023-10-01T10:41:33.530689+0000","last_became_active":"2023-10-01T10:37:42.768239+0000","last_became_peered":"2023-10-01T10:37:42.768239+0000","last_unstale":"2023-10-01T10:41:33.530689+0000","last_undegraded":"2023-10-01T10:41:33.530689+0000","last_fullsized":"2023-10-01T10:41:33.530689+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":100,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-01T10:41:33.527128+0000","last_change":"2023-10-01T10:38:16.028563+0000","last_active":"2023-10-01T10:41:33.527128+0000","last_peered":"2023-10-01T10:41:33.527128+0000","last_clean":"2023-10-01T10:41:33.527128+0000","last_became_active":"2023-10-01T10:38:16.028162+0000","last_became_peered":"2023-10-01T10:38:16.028162+0000","last_unstale":"2023-10-01T10:41:33.527128+0000","last_undegraded":"2023-10-01T10:41:33.527128+0000","last_fullsized":"2023-10-01T10:41:33.527128+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":89,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-01T10:41:33.524147+0000","last_change":"2023-10-01T10:37:42.772247+0000","last_active":"2023-10-01T10:41:33.524147+0000","last_peered":"2023-10-01T10:41:33.524147+0000","last_clean":"2023-10-01T10:41:33.524147+0000","last_became_active":"2023-10-01T10:37:42.771697+0000","last_became_peered":"2023-10-01T10:37:42.771697+0000","last_unstale":"2023-10-01T10:41:33.524147+0000","last_undegraded":"2023-10-01T10:41:33.524147+0000","last_fullsized":"2023-10-01T10:41:33.524147+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:23:15.018248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":107,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-01T10:41:33.535485+0000","last_change":"2023-10-01T10:28:20.211798+0000","last_active":"2023-10-01T10:41:33.535485+0000","last_peered":"2023-10-01T10:41:33.535485+0000","last_clean":"2023-10-01T10:41:33.535485+0000","last_became_active":"2023-10-01T10:28:20.210791+0000","last_became_peered":"2023-10-01T10:28:20.210791+0000","last_unstale":"2023-10-01T10:41:33.535485+0000","last_undegraded":"2023-10-01T10:41:33.535485+0000","last_fullsized":"2023-10-01T10:41:33.535485+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":93,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-01T10:41:33.524505+0000","last_change":"2023-10-01T10:37:42.773127+0000","last_active":"2023-10-01T10:41:33.524505+0000","last_peered":"2023-10-01T10:41:33.524505+0000","last_clean":"2023-10-01T10:41:33.524505+0000","last_became_active":"2023-10-01T10:37:42.772102+0000","last_became_peered":"2023-10-01T10:37:42.772102+0000","last_unstale":"2023-10-01T10:41:33.524505+0000","last_undegraded":"2023-10-01T10:41:33.524505+0000","last_fullsized":"2023-10-01T10:41:33.524505+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:23:26.202295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"133'5932","reported_seq":5946,"reported_epoch":133,"state":"active+clean","last_fresh":"2023-10-01T10:42:19.833934+0000","last_change":"2023-10-01T10:40:48.458343+0000","last_active":"2023-10-01T10:42:19.833934+0000","last_peered":"2023-10-01T10:42:19.833934+0000","last_clean":"2023-10-01T10:42:19.833934+0000","last_became_active":"2023-10-01T10:40:48.458009+0000","last_became_peered":"2023-10-01T10:40:48.458009+0000","last_unstale":"2023-10-01T10:42:19.833934+0000","last_undegraded":"2023-10-01T10:42:19.833934+0000","last_fullsized":"2023-10-01T10:42:19.833934+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":128,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_clean_scrub_stamp":"2023-10-01T10:40:47.426361+0000","objects_scrubbed":0,"log_size":5932,"log_dups_size":0,"ondisk_log_size":5932,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:58:33.314892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":388759552,"num_objects":5932,"num_object_clones":0,"num_object_copies":17796,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5932,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5932,"num_write_kb":379648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":116,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-01T10:41:33.527183+0000","last_change":"2023-10-01T10:38:16.026955+0000","last_active":"2023-10-01T10:41:33.527183+0000","last_peered":"2023-10-01T10:41:33.527183+0000","last_clean":"2023-10-01T10:41:33.527183+0000","last_became_active":"2023-10-01T10:38:16.026552+0000","last_became_peered":"2023-10-01T10:38:16.026552+0000","last_unstale":"2023-10-01T10:41:33.527183+0000","last_undegraded":"2023-10-01T10:41:33.527183+0000","last_fullsized":"2023-10-01T10:41:33.527183+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":84,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-01T10:41:33.529854+0000","last_change":"2023-10-01T10:15:40.318702+0000","last_active":"2023-10-01T10:41:33.529854+0000","last_peered":"2023-10-01T10:41:33.529854+0000","last_clean":"2023-10-01T10:41:33.529854+0000","last_became_active":"2023-10-01T10:15:40.318333+0000","last_became_peered":"2023-10-01T10:15:40.318333+0000","last_unstale":"2023-10-01T10:41:33.529854+0000","last_undegraded":"2023-10-01T10:41:33.529854+0000","last_fullsized":"2023-10-01T10:41:33.529854+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"133'6054","reported_seq":6068,"reported_epoch":133,"state":"active+clean","last_fresh":"2023-10-01T10:42:20.480193+0000","last_change":"2023-10-01T10:40:48.488223+0000","last_active":"2023-10-01T10:42:20.480193+0000","last_peered":"2023-10-01T10:42:20.480193+0000","last_clean":"2023-10-01T10:42:20.480193+0000","last_became_active":"2023-10-01T10:40:48.487910+0000","last_became_peered":"2023-10-01T10:40:48.487910+0000","last_unstale":"2023-10-01T10:42:20.480193+0000","last_undegraded":"2023-10-01T10:42:20.480193+0000","last_fullsized":"2023-10-01T10:42:20.480193+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":128,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_clean_scrub_stamp":"2023-10-01T10:40:47.426361+0000","objects_scrubbed":0,"log_size":6054,"log_dups_size":0,"ondisk_log_size":6054,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:18:52.686502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":396754944,"num_objects":6054,"num_object_clones":0,"num_object_copies":18162,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6054,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6054,"num_write_kb":387456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":117,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-01T10:41:33.527252+0000","last_change":"2023-10-01T10:38:16.027753+0000","last_active":"2023-10-01T10:41:33.527252+0000","last_peered":"2023-10-01T10:41:33.527252+0000","last_clean":"2023-10-01T10:41:33.527252+0000","last_became_active":"2023-10-01T10:38:16.027384+0000","last_became_peered":"2023-10-01T10:38:16.027384+0000","last_unstale":"2023-10-01T10:41:33.527252+0000","last_undegraded":"2023-10-01T10:41:33.527252+0000","last_fullsized":"2023-10-01T10:41:33.527252+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":122,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-01T10:41:33.519845+0000","last_change":"2023-10-01T10:38:16.033203+0000","last_active":"2023-10-01T10:41:33.519845+0000","last_peered":"2023-10-01T10:41:33.519845+0000","last_clean":"2023-10-01T10:41:33.519845+0000","last_became_active":"2023-10-01T10:38:16.032681+0000","last_became_peered":"2023-10-01T10:38:16.032681+0000","last_unstale":"2023-10-01T10:41:33.519845+0000","last_undegraded":"2023-10-01T10:41:33.519845+0000","last_fullsized":"2023-10-01T10:41:33.519845+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:42:36.642284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"133'6068","reported_seq":6081,"reported_epoch":133,"state":"active+clean","last_fresh":"2023-10-01T10:42:18.879007+0000","last_change":"2023-10-01T10:40:48.459909+0000","last_active":"2023-10-01T10:42:18.879007+0000","last_peered":"2023-10-01T10:42:18.879007+0000","last_clean":"2023-10-01T10:42:18.879007+0000","last_became_active":"2023-10-01T10:40:48.459525+0000","last_became_peered":"2023-10-01T10:40:48.459525+0000","last_unstale":"2023-10-01T10:42:18.879007+0000","last_undegraded":"2023-10-01T10:42:18.879007+0000","last_fullsized":"2023-10-01T10:42:18.879007+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":128,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_clean_scrub_stamp":"2023-10-01T10:40:47.426361+0000","objects_scrubbed":0,"log_size":6068,"log_dups_size":0,"ondisk_log_size":6068,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":397672448,"num_objects":6068,"num_object_clones":0,"num_object_copies":18204,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6068,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6068,"num_write_kb":388352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":86,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-01T10:41:33.527580+0000","last_change":"2023-10-01T10:15:39.293587+0000","last_active":"2023-10-01T10:41:33.527580+0000","last_peered":"2023-10-01T10:41:33.527580+0000","last_clean":"2023-10-01T10:41:33.527580+0000","last_became_active":"2023-10-01T10:15:39.293273+0000","last_became_peered":"2023-10-01T10:15:39.293273+0000","last_unstale":"2023-10-01T10:41:33.527580+0000","last_undegraded":"2023-10-01T10:41:33.527580+0000","last_fullsized":"2023-10-01T10:41:33.527580+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":98,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-01T10:41:33.535808+0000","last_change":"2023-10-01T10:32:31.571971+0000","last_active":"2023-10-01T10:41:33.535808+0000","last_peered":"2023-10-01T10:41:33.535808+0000","last_clean":"2023-10-01T10:41:33.535808+0000","last_became_active":"2023-10-01T10:32:31.571410+0000","last_became_peered":"2023-10-01T10:32:31.571410+0000","last_unstale":"2023-10-01T10:41:33.535808+0000","last_undegraded":"2023-10-01T10:41:33.535808+0000","last_fullsized":"2023-10-01T10:41:33.535808+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"133'6106","reported_seq":6119,"reported_epoch":133,"state":"active+clean","last_fresh":"2023-10-01T10:42:18.846880+0000","last_change":"2023-10-01T10:40:48.487102+0000","last_active":"2023-10-01T10:42:18.846880+0000","last_peered":"2023-10-01T10:42:18.846880+0000","last_clean":"2023-10-01T10:42:18.846880+0000","last_became_active":"2023-10-01T10:40:48.486622+0000","last_became_peered":"2023-10-01T10:40:48.486622+0000","last_unstale":"2023-10-01T10:42:18.846880+0000","last_undegraded":"2023-10-01T10:42:18.846880+0000","last_fullsized":"2023-10-01T10:42:18.846880+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":128,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_clean_scrub_stamp":"2023-10-01T10:40:47.426361+0000","objects_scrubbed":0,"log_size":6106,"log_dups_size":0,"ondisk_log_size":6106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":400162816,"num_objects":6106,"num_object_clones":0,"num_object_copies":18318,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6106,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6106,"num_write_kb":390784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":100,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-01T10:41:33.530316+0000","last_change":"2023-10-01T10:37:42.765733+0000","last_active":"2023-10-01T10:41:33.530316+0000","last_peered":"2023-10-01T10:41:33.530316+0000","last_clean":"2023-10-01T10:41:33.530316+0000","last_became_active":"2023-10-01T10:37:42.765311+0000","last_became_peered":"2023-10-01T10:37:42.765311+0000","last_unstale":"2023-10-01T10:41:33.530316+0000","last_undegraded":"2023-10-01T10:41:33.530316+0000","last_fullsized":"2023-10-01T10:41:33.530316+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":105,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-01T10:41:49.874693+0000","last_change":"2023-10-01T10:38:16.026348+0000","last_active":"2023-10-01T10:41:49.874693+0000","last_peered":"2023-10-01T10:41:49.874693+0000","last_clean":"2023-10-01T10:41:49.874693+0000","last_became_active":"2023-10-01T10:38:16.025747+0000","last_became_peered":"2023-10-01T10:38:16.025747+0000","last_unstale":"2023-10-01T10:41:49.874693+0000","last_undegraded":"2023-10-01T10:41:49.874693+0000","last_fullsized":"2023-10-01T10:41:49.874693+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:28:18.934480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"133'6148","reported_seq":6162,"reported_epoch":133,"state":"active+clean","last_fresh":"2023-10-01T10:42:20.919819+0000","last_change":"2023-10-01T10:40:48.487554+0000","last_active":"2023-10-01T10:42:20.919819+0000","last_peered":"2023-10-01T10:42:20.919819+0000","last_clean":"2023-10-01T10:42:20.919819+0000","last_became_active":"2023-10-01T10:40:48.487223+0000","last_became_peered":"2023-10-01T10:40:48.487223+0000","last_unstale":"2023-10-01T10:42:20.919819+0000","last_undegraded":"2023-10-01T10:42:20.919819+0000","last_fullsized":"2023-10-01T10:42:20.919819+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":128,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_clean_scrub_stamp":"2023-10-01T10:40:47.426361+0000","objects_scrubbed":0,"log_size":6148,"log_dups_size":0,"ondisk_log_size":6148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:18:35.800937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":402915328,"num_objects":6148,"num_object_clones":0,"num_object_copies":18444,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6148,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6148,"num_write_kb":393472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":97,"reported_epoch":133,"state":"active+clean","last_fresh":"2023-10-01T10:41:57.630990+0000","last_change":"2023-10-01T10:38:16.022493+0000","last_active":"2023-10-01T10:41:57.630990+0000","last_peered":"2023-10-01T10:41:57.630990+0000","last_clean":"2023-10-01T10:41:57.630990+0000","last_became_active":"2023-10-01T10:38:16.022033+0000","last_became_peered":"2023-10-01T10:38:16.022033+0000","last_unstale":"2023-10-01T10:41:57.630990+0000","last_undegraded":"2023-10-01T10:41:57.630990+0000","last_fullsized":"2023-10-01T10:41:57.630990+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:49:05.262763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":107,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-01T10:41:33.520621+0000","last_change":"2023-10-01T10:38:16.030613+0000","last_active":"2023-10-01T10:41:33.520621+0000","last_peered":"2023-10-01T10:41:33.520621+0000","last_clean":"2023-10-01T10:41:33.520621+0000","last_became_active":"2023-10-01T10:38:16.028975+0000","last_became_peered":"2023-10-01T10:38:16.028975+0000","last_unstale":"2023-10-01T10:41:33.520621+0000","last_undegraded":"2023-10-01T10:41:33.520621+0000","last_fullsized":"2023-10-01T10:41:33.520621+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:46:07.287911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"133'6025","reported_seq":6039,"reported_epoch":133,"state":"active+clean","last_fresh":"2023-10-01T10:42:20.467866+0000","last_change":"2023-10-01T10:40:48.488324+0000","last_active":"2023-10-01T10:42:20.467866+0000","last_peered":"2023-10-01T10:42:20.467866+0000","last_clean":"2023-10-01T10:42:20.467866+0000","last_became_active":"2023-10-01T10:40:48.487379+0000","last_became_peered":"2023-10-01T10:40:48.487379+0000","last_unstale":"2023-10-01T10:42:20.467866+0000","last_undegraded":"2023-10-01T10:42:20.467866+0000","last_fullsized":"2023-10-01T10:42:20.467866+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":128,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_clean_scrub_stamp":"2023-10-01T10:40:47.426361+0000","objects_scrubbed":0,"log_size":6025,"log_dups_size":0,"ondisk_log_size":6025,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:09:55.286907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":394854400,"num_objects":6025,"num_object_clones":0,"num_object_copies":18075,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6025,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6025,"num_write_kb":385600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":114,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-01T10:41:33.536025+0000","last_change":"2023-10-01T10:38:16.030250+0000","last_active":"2023-10-01T10:41:33.536025+0000","last_peered":"2023-10-01T10:41:33.536025+0000","last_clean":"2023-10-01T10:41:33.536025+0000","last_became_active":"2023-10-01T10:38:16.029898+0000","last_became_peered":"2023-10-01T10:38:16.029898+0000","last_unstale":"2023-10-01T10:41:33.536025+0000","last_undegraded":"2023-10-01T10:41:33.536025+0000","last_fullsized":"2023-10-01T10:41:33.536025+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":112,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-01T10:41:33.520214+0000","last_change":"2023-10-01T10:38:16.035906+0000","last_active":"2023-10-01T10:41:33.520214+0000","last_peered":"2023-10-01T10:41:33.520214+0000","last_clean":"2023-10-01T10:41:33.520214+0000","last_became_active":"2023-10-01T10:38:16.035383+0000","last_became_peered":"2023-10-01T10:38:16.035383+0000","last_unstale":"2023-10-01T10:41:33.520214+0000","last_undegraded":"2023-10-01T10:41:33.520214+0000","last_fullsized":"2023-10-01T10:41:33.520214+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:44:41.500097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"133'6102","reported_seq":6115,"reported_epoch":133,"state":"active+clean","last_fresh":"2023-10-01T10:42:23.329238+0000","last_change":"2023-10-01T10:40:48.461083+0000","last_active":"2023-10-01T10:42:23.329238+0000","last_peered":"2023-10-01T10:42:23.329238+0000","last_clean":"2023-10-01T10:42:23.329238+0000","last_became_active":"2023-10-01T10:40:48.460671+0000","last_became_peered":"2023-10-01T10:40:48.460671+0000","last_unstale":"2023-10-01T10:42:23.329238+0000","last_undegraded":"2023-10-01T10:42:23.329238+0000","last_fullsized":"2023-10-01T10:42:23.329238+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":128,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_clean_scrub_stamp":"2023-10-01T10:40:47.426361+0000","objects_scrubbed":0,"log_size":6102,"log_dups_size":0,"ondisk_log_size":6102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":399900672,"num_objects":6102,"num_object_clones":0,"num_object_copies":18306,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6102,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6102,"num_write_kb":390528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":104,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-01T10:41:33.519590+0000","last_change":"2023-10-01T10:38:16.026611+0000","last_active":"2023-10-01T10:41:33.519590+0000","last_peered":"2023-10-01T10:41:33.519590+0000","last_clean":"2023-10-01T10:41:33.519590+0000","last_became_active":"2023-10-01T10:38:16.026267+0000","last_became_peered":"2023-10-01T10:38:16.026267+0000","last_unstale":"2023-10-01T10:41:33.519590+0000","last_undegraded":"2023-10-01T10:41:33.519590+0000","last_fullsized":"2023-10-01T10:41:33.519590+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:06.135288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":88,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-01T10:41:33.520775+0000","last_change":"2023-10-01T10:38:16.032162+0000","last_active":"2023-10-01T10:41:33.520775+0000","last_peered":"2023-10-01T10:41:33.520775+0000","last_clean":"2023-10-01T10:41:33.520775+0000","last_became_active":"2023-10-01T10:38:16.031785+0000","last_became_peered":"2023-10-01T10:38:16.031785+0000","last_unstale":"2023-10-01T10:41:33.520775+0000","last_undegraded":"2023-10-01T10:41:33.520775+0000","last_fullsized":"2023-10-01T10:41:33.520775+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:38:43.003626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"133'6189","reported_seq":6203,"reported_epoch":133,"state":"active+clean","last_fresh":"2023-10-01T10:42:20.974804+0000","last_change":"2023-10-01T10:40:48.485646+0000","last_active":"2023-10-01T10:42:20.974804+0000","last_peered":"2023-10-01T10:42:20.974804+0000","last_clean":"2023-10-01T10:42:20.974804+0000","last_became_active":"2023-10-01T10:40:48.485316+0000","last_became_peered":"2023-10-01T10:40:48.485316+0000","last_unstale":"2023-10-01T10:42:20.974804+0000","last_undegraded":"2023-10-01T10:42:20.974804+0000","last_fullsized":"2023-10-01T10:42:20.974804+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":128,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_clean_scrub_stamp":"2023-10-01T10:40:47.426361+0000","objects_scrubbed":0,"log_size":6189,"log_dups_size":0,"ondisk_log_size":6189,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:33:20.144231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":405602304,"num_objects":6189,"num_object_clones":0,"num_object_copies":18567,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6189,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6189,"num_write_kb":396096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":95,"reported_epoch":133,"state":"active+clean","last_fresh":"2023-10-01T10:41:57.633803+0000","last_change":"2023-10-01T10:37:42.772398+0000","last_active":"2023-10-01T10:41:57.633803+0000","last_peered":"2023-10-01T10:41:57.633803+0000","last_clean":"2023-10-01T10:41:57.633803+0000","last_became_active":"2023-10-01T10:37:42.772084+0000","last_became_peered":"2023-10-01T10:37:42.772084+0000","last_unstale":"2023-10-01T10:41:57.633803+0000","last_undegraded":"2023-10-01T10:41:57.633803+0000","last_fullsized":"2023-10-01T10:41:57.633803+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:32:43.846419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":113,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-01T10:41:33.524374+0000","last_change":"2023-10-01T10:38:16.019755+0000","last_active":"2023-10-01T10:41:33.524374+0000","last_peered":"2023-10-01T10:41:33.524374+0000","last_clean":"2023-10-01T10:41:33.524374+0000","last_became_active":"2023-10-01T10:38:16.019397+0000","last_became_peered":"2023-10-01T10:38:16.019397+0000","last_unstale":"2023-10-01T10:41:33.524374+0000","last_undegraded":"2023-10-01T10:41:33.524374+0000","last_fullsized":"2023-10-01T10:41:33.524374+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:42:56.623225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.f","version":"133'5942","reported_seq":5954,"reported_epoch":133,"state":"active+clean","last_fresh":"2023-10-01T10:42:18.879163+0000","last_change":"2023-10-01T10:40:48.455948+0000","last_active":"2023-10-01T10:42:18.879163+0000","last_peered":"2023-10-01T10:42:18.879163+0000","last_clean":"2023-10-01T10:42:18.879163+0000","last_became_active":"2023-10-01T10:40:48.455402+0000","last_became_peered":"2023-10-01T10:40:48.455402+0000","last_unstale":"2023-10-01T10:42:18.879163+0000","last_undegraded":"2023-10-01T10:42:18.879163+0000","last_fullsized":"2023-10-01T10:42:18.879163+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":128,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_clean_scrub_stamp":"2023-10-01T10:40:47.426361+0000","objects_scrubbed":0,"log_size":5942,"log_dups_size":0,"ondisk_log_size":5942,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":389414912,"num_objects":5942,"num_object_clones":0,"num_object_copies":17826,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5942,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5942,"num_write_kb":380288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":106,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-01T10:41:33.535539+0000","last_change":"2023-10-01T10:32:31.561108+0000","last_active":"2023-10-01T10:41:33.535539+0000","last_peered":"2023-10-01T10:41:33.535539+0000","last_clean":"2023-10-01T10:41:33.535539+0000","last_became_active":"2023-10-01T10:32:31.560791+0000","last_became_peered":"2023-10-01T10:32:31.560791+0000","last_unstale":"2023-10-01T10:41:33.535539+0000","last_undegraded":"2023-10-01T10:41:33.535539+0000","last_fullsized":"2023-10-01T10:41:33.535539+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:42:47.477671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":115,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-01T10:41:33.530634+0000","last_change":"2023-10-01T10:37:42.766814+0000","last_active":"2023-10-01T10:41:33.530634+0000","last_peered":"2023-10-01T10:41:33.530634+0000","last_clean":"2023-10-01T10:41:33.530634+0000","last_became_active":"2023-10-01T10:37:42.766451+0000","last_became_peered":"2023-10-01T10:37:42.766451+0000","last_unstale":"2023-10-01T10:41:33.530634+0000","last_undegraded":"2023-10-01T10:41:33.530634+0000","last_fullsized":"2023-10-01T10:41:33.530634+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":72,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-01T10:41:33.537348+0000","last_change":"2023-10-01T10:38:16.034357+0000","last_active":"2023-10-01T10:41:33.537348+0000","last_peered":"2023-10-01T10:41:33.537348+0000","last_clean":"2023-10-01T10:41:33.537348+0000","last_became_active":"2023-10-01T10:38:16.033905+0000","last_became_peered":"2023-10-01T10:38:16.033905+0000","last_unstale":"2023-10-01T10:41:33.537348+0000","last_undegraded":"2023-10-01T10:41:33.537348+0000","last_fullsized":"2023-10-01T10:41:33.537348+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:47:58.369835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"133'6003","reported_seq":6017,"reported_epoch":133,"state":"active+clean","last_fresh":"2023-10-01T10:42:19.823062+0000","last_change":"2023-10-01T10:40:48.456619+0000","last_active":"2023-10-01T10:42:19.823062+0000","last_peered":"2023-10-01T10:42:19.823062+0000","last_clean":"2023-10-01T10:42:19.823062+0000","last_became_active":"2023-10-01T10:40:48.456280+0000","last_became_peered":"2023-10-01T10:40:48.456280+0000","last_unstale":"2023-10-01T10:42:19.823062+0000","last_undegraded":"2023-10-01T10:42:19.823062+0000","last_fullsized":"2023-10-01T10:42:19.823062+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":128,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_clean_scrub_stamp":"2023-10-01T10:40:47.426361+0000","objects_scrubbed":0,"log_size":6003,"log_dups_size":0,"ondisk_log_size":6003,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:34:57.206244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":393412608,"num_objects":6003,"num_object_clones":0,"num_object_copies":18009,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6003,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6003,"num_write_kb":384192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":104,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-01T10:41:33.529945+0000","last_change":"2023-10-01T10:32:31.572482+0000","last_active":"2023-10-01T10:41:33.529945+0000","last_peered":"2023-10-01T10:41:33.529945+0000","last_clean":"2023-10-01T10:41:33.529945+0000","last_became_active":"2023-10-01T10:32:31.571993+0000","last_became_peered":"2023-10-01T10:32:31.571993+0000","last_unstale":"2023-10-01T10:41:33.529945+0000","last_undegraded":"2023-10-01T10:41:33.529945+0000","last_fullsized":"2023-10-01T10:41:33.529945+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":74,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-01T10:41:33.520371+0000","last_change":"2023-10-01T10:38:16.030202+0000","last_active":"2023-10-01T10:41:33.520371+0000","last_peered":"2023-10-01T10:41:33.520371+0000","last_clean":"2023-10-01T10:41:33.520371+0000","last_became_active":"2023-10-01T10:38:16.029917+0000","last_became_peered":"2023-10-01T10:38:16.029917+0000","last_unstale":"2023-10-01T10:41:33.520371+0000","last_undegraded":"2023-10-01T10:41:33.520371+0000","last_fullsized":"2023-10-01T10:41:33.520371+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:20:06.763909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":87,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-01T10:41:33.535619+0000","last_change":"2023-10-01T10:37:42.761153+0000","last_active":"2023-10-01T10:41:33.535619+0000","last_peered":"2023-10-01T10:41:33.535619+0000","last_clean":"2023-10-01T10:41:33.535619+0000","last_became_active":"2023-10-01T10:37:42.760715+0000","last_became_peered":"2023-10-01T10:37:42.760715+0000","last_unstale":"2023-10-01T10:41:33.535619+0000","last_undegraded":"2023-10-01T10:41:33.535619+0000","last_fullsized":"2023-10-01T10:41:33.535619+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:52:50.237174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"133'6011","reported_seq":6023,"reported_epoch":133,"state":"active+clean","last_fresh":"2023-10-01T10:42:18.897393+0000","last_change":"2023-10-01T10:40:48.457495+0000","last_active":"2023-10-01T10:42:18.897393+0000","last_peered":"2023-10-01T10:42:18.897393+0000","last_clean":"2023-10-01T10:42:18.897393+0000","last_became_active":"2023-10-01T10:40:48.457080+0000","last_became_peered":"2023-10-01T10:40:48.457080+0000","last_unstale":"2023-10-01T10:42:18.897393+0000","last_undegraded":"2023-10-01T10:42:18.897393+0000","last_fullsized":"2023-10-01T10:42:18.897393+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":128,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_clean_scrub_stamp":"2023-10-01T10:40:47.426361+0000","objects_scrubbed":0,"log_size":6011,"log_dups_size":0,"ondisk_log_size":6011,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":393936896,"num_objects":6011,"num_object_clones":0,"num_object_copies":18033,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6011,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6011,"num_write_kb":384704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":104,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-01T10:41:33.523192+0000","last_change":"2023-10-01T10:37:42.762524+0000","last_active":"2023-10-01T10:41:33.523192+0000","last_peered":"2023-10-01T10:41:33.523192+0000","last_clean":"2023-10-01T10:41:33.523192+0000","last_became_active":"2023-10-01T10:37:42.762209+0000","last_became_peered":"2023-10-01T10:37:42.762209+0000","last_unstale":"2023-10-01T10:41:33.523192+0000","last_undegraded":"2023-10-01T10:41:33.523192+0000","last_fullsized":"2023-10-01T10:41:33.523192+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:45:38.171397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":72,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-01T10:41:33.535621+0000","last_change":"2023-10-01T10:37:05.019330+0000","last_active":"2023-10-01T10:41:33.535621+0000","last_peered":"2023-10-01T10:41:33.535621+0000","last_clean":"2023-10-01T10:41:33.535621+0000","last_became_active":"2023-10-01T10:37:05.018842+0000","last_became_peered":"2023-10-01T10:37:05.018842+0000","last_unstale":"2023-10-01T10:41:33.535621+0000","last_undegraded":"2023-10-01T10:41:33.535621+0000","last_fullsized":"2023-10-01T10:41:33.535621+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:48:09.551917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":101,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-01T10:41:33.536041+0000","last_change":"2023-10-01T10:32:31.569162+0000","last_active":"2023-10-01T10:41:33.536041+0000","last_peered":"2023-10-01T10:41:33.536041+0000","last_clean":"2023-10-01T10:41:33.536041+0000","last_became_active":"2023-10-01T10:32:31.568956+0000","last_became_peered":"2023-10-01T10:32:31.568956+0000","last_unstale":"2023-10-01T10:41:33.536041+0000","last_undegraded":"2023-10-01T10:41:33.536041+0000","last_fullsized":"2023-10-01T10:41:33.536041+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:02:08.285304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"133'6065","reported_seq":6079,"reported_epoch":133,"state":"active+clean","last_fresh":"2023-10-01T10:42:20.974367+0000","last_change":"2023-10-01T10:40:48.454746+0000","last_active":"2023-10-01T10:42:20.974367+0000","last_peered":"2023-10-01T10:42:20.974367+0000","last_clean":"2023-10-01T10:42:20.974367+0000","last_became_active":"2023-10-01T10:40:48.454409+0000","last_became_peered":"2023-10-01T10:40:48.454409+0000","last_unstale":"2023-10-01T10:42:20.974367+0000","last_undegraded":"2023-10-01T10:42:20.974367+0000","last_fullsized":"2023-10-01T10:42:20.974367+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":128,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_clean_scrub_stamp":"2023-10-01T10:40:47.426361+0000","objects_scrubbed":0,"log_size":6065,"log_dups_size":0,"ondisk_log_size":6065,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:07:17.878925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":397475840,"num_objects":6065,"num_object_clones":0,"num_object_copies":18195,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6065,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6065,"num_write_kb":388160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":104,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-01T10:41:33.530079+0000","last_change":"2023-10-01T10:37:05.024009+0000","last_active":"2023-10-01T10:41:33.530079+0000","last_peered":"2023-10-01T10:41:33.530079+0000","last_clean":"2023-10-01T10:41:33.530079+0000","last_became_active":"2023-10-01T10:37:05.023458+0000","last_became_peered":"2023-10-01T10:37:05.023458+0000","last_unstale":"2023-10-01T10:41:33.530079+0000","last_undegraded":"2023-10-01T10:41:33.530079+0000","last_fullsized":"2023-10-01T10:41:33.530079+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'139","reported_seq":336,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-01T10:41:33.526927+0000","last_change":"2023-10-01T10:28:20.217386+0000","last_active":"2023-10-01T10:41:33.526927+0000","last_peered":"2023-10-01T10:41:33.526927+0000","last_clean":"2023-10-01T10:41:33.526927+0000","last_became_active":"2023-10-01T10:28:20.216803+0000","last_became_peered":"2023-10-01T10:28:20.216803+0000","last_unstale":"2023-10-01T10:41:33.526927+0000","last_undegraded":"2023-10-01T10:41:33.526927+0000","last_fullsized":"2023-10-01T10:41:33.526927+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:09:17.221082+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:09:17.221082+0000","last_clean_scrub_stamp":"2023-10-01T10:09:17.221082+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":77,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-01T10:41:50.923251+0000","last_change":"2023-10-01T10:37:42.765100+0000","last_active":"2023-10-01T10:41:50.923251+0000","last_peered":"2023-10-01T10:41:50.923251+0000","last_clean":"2023-10-01T10:41:50.923251+0000","last_became_active":"2023-10-01T10:37:42.764761+0000","last_became_peered":"2023-10-01T10:37:42.764761+0000","last_unstale":"2023-10-01T10:41:50.923251+0000","last_undegraded":"2023-10-01T10:41:50.923251+0000","last_fullsized":"2023-10-01T10:41:50.923251+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:24:56.907377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":98,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-01T10:41:33.536224+0000","last_change":"2023-10-01T10:38:16.031255+0000","last_active":"2023-10-01T10:41:33.536224+0000","last_peered":"2023-10-01T10:41:33.536224+0000","last_clean":"2023-10-01T10:41:33.536224+0000","last_became_active":"2023-10-01T10:38:16.030886+0000","last_became_peered":"2023-10-01T10:38:16.030886+0000","last_unstale":"2023-10-01T10:41:33.536224+0000","last_undegraded":"2023-10-01T10:41:33.536224+0000","last_fullsized":"2023-10-01T10:41:33.536224+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"133'6057","reported_seq":6071,"reported_epoch":133,"state":"active+clean","last_fresh":"2023-10-01T10:42:19.839976+0000","last_change":"2023-10-01T10:40:48.487771+0000","last_active":"2023-10-01T10:42:19.839976+0000","last_peered":"2023-10-01T10:42:19.839976+0000","last_clean":"2023-10-01T10:42:19.839976+0000","last_became_active":"2023-10-01T10:40:48.487435+0000","last_became_peered":"2023-10-01T10:40:48.487435+0000","last_unstale":"2023-10-01T10:42:19.839976+0000","last_undegraded":"2023-10-01T10:42:19.839976+0000","last_fullsized":"2023-10-01T10:42:19.839976+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":128,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_clean_scrub_stamp":"2023-10-01T10:40:47.426361+0000","objects_scrubbed":0,"log_size":6057,"log_dups_size":0,"ondisk_log_size":6057,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:50:58.475377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":396951552,"num_objects":6057,"num_object_clones":0,"num_object_copies":18171,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6057,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6057,"num_write_kb":387648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":119,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-01T10:41:33.526696+0000","last_change":"2023-10-01T10:32:31.565868+0000","last_active":"2023-10-01T10:41:33.526696+0000","last_peered":"2023-10-01T10:41:33.526696+0000","last_clean":"2023-10-01T10:41:33.526696+0000","last_became_active":"2023-10-01T10:32:31.565428+0000","last_became_peered":"2023-10-01T10:32:31.565428+0000","last_unstale":"2023-10-01T10:41:33.526696+0000","last_undegraded":"2023-10-01T10:41:33.526696+0000","last_fullsized":"2023-10-01T10:41:33.526696+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":104,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-01T10:41:33.523470+0000","last_change":"2023-10-01T10:38:16.021968+0000","last_active":"2023-10-01T10:41:33.523470+0000","last_peered":"2023-10-01T10:41:33.523470+0000","last_clean":"2023-10-01T10:41:33.523470+0000","last_became_active":"2023-10-01T10:38:16.021539+0000","last_became_peered":"2023-10-01T10:38:16.021539+0000","last_unstale":"2023-10-01T10:41:33.523470+0000","last_undegraded":"2023-10-01T10:41:33.523470+0000","last_fullsized":"2023-10-01T10:41:33.523470+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:12:53.105747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":105,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-01T10:41:33.524076+0000","last_change":"2023-10-01T10:37:05.017431+0000","last_active":"2023-10-01T10:41:33.524076+0000","last_peered":"2023-10-01T10:41:33.524076+0000","last_clean":"2023-10-01T10:41:33.524076+0000","last_became_active":"2023-10-01T10:37:05.017094+0000","last_became_peered":"2023-10-01T10:37:05.017094+0000","last_unstale":"2023-10-01T10:41:33.524076+0000","last_undegraded":"2023-10-01T10:41:33.524076+0000","last_fullsized":"2023-10-01T10:41:33.524076+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:10:40.664686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"133'6307","reported_seq":6320,"reported_epoch":133,"state":"active+clean","last_fresh":"2023-10-01T10:42:23.339644+0000","last_change":"2023-10-01T10:40:48.488899+0000","last_active":"2023-10-01T10:42:23.339644+0000","last_peered":"2023-10-01T10:42:23.339644+0000","last_clean":"2023-10-01T10:42:23.339644+0000","last_became_active":"2023-10-01T10:40:48.488506+0000","last_became_peered":"2023-10-01T10:40:48.488506+0000","last_unstale":"2023-10-01T10:42:23.339644+0000","last_undegraded":"2023-10-01T10:42:23.339644+0000","last_fullsized":"2023-10-01T10:42:23.339644+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":128,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_clean_scrub_stamp":"2023-10-01T10:40:47.426361+0000","objects_scrubbed":0,"log_size":6307,"log_dups_size":0,"ondisk_log_size":6307,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":413335552,"num_objects":6307,"num_object_clones":0,"num_object_copies":18921,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6307,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6307,"num_write_kb":403648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":119,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-01T10:41:33.527624+0000","last_change":"2023-10-01T10:38:16.032774+0000","last_active":"2023-10-01T10:41:33.527624+0000","last_peered":"2023-10-01T10:41:33.527624+0000","last_clean":"2023-10-01T10:41:33.527624+0000","last_became_active":"2023-10-01T10:38:16.032283+0000","last_became_peered":"2023-10-01T10:38:16.032283+0000","last_unstale":"2023-10-01T10:41:33.527624+0000","last_undegraded":"2023-10-01T10:41:33.527624+0000","last_fullsized":"2023-10-01T10:41:33.527624+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":72,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-01T10:41:33.535700+0000","last_change":"2023-10-01T10:37:05.021228+0000","last_active":"2023-10-01T10:41:33.535700+0000","last_peered":"2023-10-01T10:41:33.535700+0000","last_clean":"2023-10-01T10:41:33.535700+0000","last_became_active":"2023-10-01T10:37:05.020888+0000","last_became_peered":"2023-10-01T10:37:05.020888+0000","last_unstale":"2023-10-01T10:41:33.535700+0000","last_undegraded":"2023-10-01T10:41:33.535700+0000","last_fullsized":"2023-10-01T10:41:33.535700+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:59:57.641247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":75,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-01T10:41:33.523349+0000","last_change":"2023-10-01T10:28:20.220812+0000","last_active":"2023-10-01T10:41:33.523349+0000","last_peered":"2023-10-01T10:41:33.523349+0000","last_clean":"2023-10-01T10:41:33.523349+0000","last_became_active":"2023-10-01T10:28:20.215962+0000","last_became_peered":"2023-10-01T10:28:20.215962+0000","last_unstale":"2023-10-01T10:41:33.523349+0000","last_undegraded":"2023-10-01T10:41:33.523349+0000","last_fullsized":"2023-10-01T10:41:33.523349+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:35:15.275097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"133'6089","reported_seq":6102,"reported_epoch":133,"state":"active+clean","last_fresh":"2023-10-01T10:42:23.323392+0000","last_change":"2023-10-01T10:40:48.461167+0000","last_active":"2023-10-01T10:42:23.323392+0000","last_peered":"2023-10-01T10:42:23.323392+0000","last_clean":"2023-10-01T10:42:23.323392+0000","last_became_active":"2023-10-01T10:40:48.460822+0000","last_became_peered":"2023-10-01T10:40:48.460822+0000","last_unstale":"2023-10-01T10:42:23.323392+0000","last_undegraded":"2023-10-01T10:42:23.323392+0000","last_fullsized":"2023-10-01T10:42:23.323392+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":128,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_clean_scrub_stamp":"2023-10-01T10:40:47.426361+0000","objects_scrubbed":0,"log_size":6089,"log_dups_size":0,"ondisk_log_size":6089,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":399048704,"num_objects":6089,"num_object_clones":0,"num_object_copies":18267,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6089,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6089,"num_write_kb":389696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":82,"reported_epoch":133,"state":"active+clean","last_fresh":"2023-10-01T10:41:57.632462+0000","last_change":"2023-10-01T10:37:42.773999+0000","last_active":"2023-10-01T10:41:57.632462+0000","last_peered":"2023-10-01T10:41:57.632462+0000","last_clean":"2023-10-01T10:41:57.632462+0000","last_became_active":"2023-10-01T10:37:42.773355+0000","last_became_peered":"2023-10-01T10:37:42.773355+0000","last_unstale":"2023-10-01T10:41:57.632462+0000","last_undegraded":"2023-10-01T10:41:57.632462+0000","last_fullsized":"2023-10-01T10:41:57.632462+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:46:51.317151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":114,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-01T10:41:33.535230+0000","last_change":"2023-10-01T10:38:16.034193+0000","last_active":"2023-10-01T10:41:33.535230+0000","last_peered":"2023-10-01T10:41:33.535230+0000","last_clean":"2023-10-01T10:41:33.535230+0000","last_became_active":"2023-10-01T10:38:16.033703+0000","last_became_peered":"2023-10-01T10:38:16.033703+0000","last_unstale":"2023-10-01T10:41:33.535230+0000","last_undegraded":"2023-10-01T10:41:33.535230+0000","last_fullsized":"2023-10-01T10:41:33.535230+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:21:26.061595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":79,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-01T10:41:40.918901+0000","last_change":"2023-10-01T10:37:42.773538+0000","last_active":"2023-10-01T10:41:40.918901+0000","last_peered":"2023-10-01T10:41:40.918901+0000","last_clean":"2023-10-01T10:41:40.918901+0000","last_became_active":"2023-10-01T10:37:42.771863+0000","last_became_peered":"2023-10-01T10:37:42.771863+0000","last_unstale":"2023-10-01T10:41:40.918901+0000","last_undegraded":"2023-10-01T10:41:40.918901+0000","last_fullsized":"2023-10-01T10:41:40.918901+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:46:46.698627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"133'6129","reported_seq":6143,"reported_epoch":133,"state":"active+clean","last_fresh":"2023-10-01T10:42:20.945112+0000","last_change":"2023-10-01T10:40:48.486926+0000","last_active":"2023-10-01T10:42:20.945112+0000","last_peered":"2023-10-01T10:42:20.945112+0000","last_clean":"2023-10-01T10:42:20.945112+0000","last_became_active":"2023-10-01T10:40:48.486674+0000","last_became_peered":"2023-10-01T10:40:48.486674+0000","last_unstale":"2023-10-01T10:42:20.945112+0000","last_undegraded":"2023-10-01T10:42:20.945112+0000","last_fullsized":"2023-10-01T10:42:20.945112+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":128,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:40:47.426361+0000","last_clean_scrub_stamp":"2023-10-01T10:40:47.426361+0000","objects_scrubbed":0,"log_size":6129,"log_dups_size":0,"ondisk_log_size":6129,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:07:45.799626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":401670144,"num_objects":6129,"num_object_clones":0,"num_object_copies":18387,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6129,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6129,"num_write_kb":392256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":103,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-01T10:41:33.536232+0000","last_change":"2023-10-01T10:37:05.023034+0000","last_active":"2023-10-01T10:41:33.536232+0000","last_peered":"2023-10-01T10:41:33.536232+0000","last_clean":"2023-10-01T10:41:33.536232+0000","last_became_active":"2023-10-01T10:37:05.022670+0000","last_became_peered":"2023-10-01T10:37:05.022670+0000","last_unstale":"2023-10-01T10:41:33.536232+0000","last_undegraded":"2023-10-01T10:41:33.536232+0000","last_fullsized":"2023-10-01T10:41:33.536232+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":108,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-01T10:41:33.523437+0000","last_change":"2023-10-01T10:38:16.027472+0000","last_active":"2023-10-01T10:41:33.523437+0000","last_peered":"2023-10-01T10:41:33.523437+0000","last_clean":"2023-10-01T10:41:33.523437+0000","last_became_active":"2023-10-01T10:38:16.026810+0000","last_became_peered":"2023-10-01T10:38:16.026810+0000","last_unstale":"2023-10-01T10:41:33.523437+0000","last_undegraded":"2023-10-01T10:41:33.523437+0000","last_fullsized":"2023-10-01T10:41:33.523437+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:30:06.978187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":72,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-01T10:41:33.519762+0000","last_change":"2023-10-01T10:38:16.035705+0000","last_active":"2023-10-01T10:41:33.519762+0000","last_peered":"2023-10-01T10:41:33.519762+0000","last_clean":"2023-10-01T10:41:33.519762+0000","last_became_active":"2023-10-01T10:38:16.035389+0000","last_became_peered":"2023-10-01T10:38:16.035389+0000","last_unstale":"2023-10-01T10:41:33.519762+0000","last_undegraded":"2023-10-01T10:41:33.519762+0000","last_fullsized":"2023-10-01T10:41:33.519762+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:16:10.201409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":92,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-01T10:41:33.523620+0000","last_change":"2023-10-01T10:38:16.028491+0000","last_active":"2023-10-01T10:41:33.523620+0000","last_peered":"2023-10-01T10:41:33.523620+0000","last_clean":"2023-10-01T10:41:33.523620+0000","last_became_active":"2023-10-01T10:38:16.028175+0000","last_became_peered":"2023-10-01T10:38:16.028175+0000","last_unstale":"2023-10-01T10:41:33.523620+0000","last_undegraded":"2023-10-01T10:41:33.523620+0000","last_fullsized":"2023-10-01T10:41:33.523620+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:44:29.083129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":114,"reported_epoch":133,"state":"active+clean","last_fresh":"2023-10-01T10:41:57.634357+0000","last_change":"2023-10-01T10:37:42.762360+0000","last_active":"2023-10-01T10:41:57.634357+0000","last_peered":"2023-10-01T10:41:57.634357+0000","last_clean":"2023-10-01T10:41:57.634357+0000","last_became_active":"2023-10-01T10:37:42.761936+0000","last_became_peered":"2023-10-01T10:37:42.761936+0000","last_unstale":"2023-10-01T10:41:57.634357+0000","last_undegraded":"2023-10-01T10:41:57.634357+0000","last_fullsized":"2023-10-01T10:41:57.634357+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:50.637582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":117,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-01T10:41:33.530391+0000","last_change":"2023-10-01T10:38:16.037809+0000","last_active":"2023-10-01T10:41:33.530391+0000","last_peered":"2023-10-01T10:41:33.530391+0000","last_clean":"2023-10-01T10:41:33.530391+0000","last_became_active":"2023-10-01T10:38:16.037318+0000","last_became_peered":"2023-10-01T10:38:16.037318+0000","last_unstale":"2023-10-01T10:41:33.530391+0000","last_undegraded":"2023-10-01T10:41:33.530391+0000","last_fullsized":"2023-10-01T10:41:33.530391+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":122,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-01T10:41:33.520283+0000","last_change":"2023-10-01T10:38:16.034282+0000","last_active":"2023-10-01T10:41:33.520283+0000","last_peered":"2023-10-01T10:41:33.520283+0000","last_clean":"2023-10-01T10:41:33.520283+0000","last_became_active":"2023-10-01T10:38:16.033820+0000","last_became_peered":"2023-10-01T10:38:16.033820+0000","last_unstale":"2023-10-01T10:41:33.520283+0000","last_undegraded":"2023-10-01T10:41:33.520283+0000","last_fullsized":"2023-10-01T10:41:33.520283+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:48:23.815832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":102,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-01T10:41:33.530570+0000","last_change":"2023-10-01T10:38:16.038077+0000","last_active":"2023-10-01T10:41:33.530570+0000","last_peered":"2023-10-01T10:41:33.530570+0000","last_clean":"2023-10-01T10:41:33.530570+0000","last_became_active":"2023-10-01T10:38:16.037313+0000","last_became_peered":"2023-10-01T10:38:16.037313+0000","last_unstale":"2023-10-01T10:41:33.530570+0000","last_undegraded":"2023-10-01T10:41:33.530570+0000","last_fullsized":"2023-10-01T10:41:33.530570+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":122,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-01T10:41:33.520056+0000","last_change":"2023-10-01T10:38:16.025600+0000","last_active":"2023-10-01T10:41:33.520056+0000","last_peered":"2023-10-01T10:41:33.520056+0000","last_clean":"2023-10-01T10:41:33.520056+0000","last_became_active":"2023-10-01T10:38:16.025252+0000","last_became_peered":"2023-10-01T10:38:16.025252+0000","last_unstale":"2023-10-01T10:41:33.520056+0000","last_undegraded":"2023-10-01T10:41:33.520056+0000","last_fullsized":"2023-10-01T10:41:33.520056+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:24:18.871963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":110,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-01T10:41:40.920066+0000","last_change":"2023-10-01T10:37:42.761243+0000","last_active":"2023-10-01T10:41:40.920066+0000","last_peered":"2023-10-01T10:41:40.920066+0000","last_clean":"2023-10-01T10:41:40.920066+0000","last_became_active":"2023-10-01T10:37:42.760834+0000","last_became_peered":"2023-10-01T10:37:42.760834+0000","last_unstale":"2023-10-01T10:41:40.920066+0000","last_undegraded":"2023-10-01T10:41:40.920066+0000","last_fullsized":"2023-10-01T10:41:40.920066+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:37:21.976095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":122,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-01T10:41:33.523708+0000","last_change":"2023-10-01T10:37:42.765142+0000","last_active":"2023-10-01T10:41:33.523708+0000","last_peered":"2023-10-01T10:41:33.523708+0000","last_clean":"2023-10-01T10:41:33.523708+0000","last_became_active":"2023-10-01T10:37:42.764770+0000","last_became_peered":"2023-10-01T10:37:42.764770+0000","last_unstale":"2023-10-01T10:41:33.523708+0000","last_undegraded":"2023-10-01T10:41:33.523708+0000","last_fullsized":"2023-10-01T10:41:33.523708+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:23:33.527876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":93,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-01T10:41:33.523850+0000","last_change":"2023-10-01T10:37:05.021621+0000","last_active":"2023-10-01T10:41:33.523850+0000","last_peered":"2023-10-01T10:41:33.523850+0000","last_clean":"2023-10-01T10:41:33.523850+0000","last_became_active":"2023-10-01T10:37:05.021173+0000","last_became_peered":"2023-10-01T10:37:05.021173+0000","last_unstale":"2023-10-01T10:41:33.523850+0000","last_undegraded":"2023-10-01T10:41:33.523850+0000","last_fullsized":"2023-10-01T10:41:33.523850+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:13:25.364499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":88,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-10-01T10:41:33.519580+0000","last_change":"2023-10-01T10:38:16.031427+0000","last_active":"2023-10-01T10:41:33.519580+0000","last_peered":"2023-10-01T10:41:33.519580+0000","last_clean":"2023-10-01T10:41:33.519580+0000","last_became_active":"2023-10-01T10:38:16.029786+0000","last_became_peered":"2023-10-01T10:38:16.029786+0000","last_unstale":"2023-10-01T10:41:33.519580+0000","last_undegraded":"2023-10-01T10:41:33.519580+0000","last_fullsized":"2023-10-01T10:41:33.519580+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:53:22.796191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30319,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":6371868672,"num_objects":97227,"num_object_clones":0,"num_object_copies":291681,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":97227,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":97227,"num_write_kb":6222528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":19167641600,"data_stored":19167641600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97227,"ondisk_log_size":97227,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430565,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4144700,"kb_used_data":3550900,"kb_used_omap":0,"kb_used_meta":593792,"kb_avail":89600452,"statfs":{"total":95995035648,"available":91750862848,"internally_reserved":0,"allocated":3636121600,"data_stored":3635642971,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":608043008},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:41:46 2023","interfaces":[{"interface":"back","average":{"1min":2.1539999999999999,"5min":1.169,"15min":1.577},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.27500000000000002},"max":{"1min":10.695,"5min":10.695,"15min":14.974},"last":0.498},{"interface":"front","average":{"1min":1.2430000000000001,"5min":1.0109999999999999,"15min":1.579},"min":{"1min":0.314,"5min":0.314,"15min":0.29999999999999999},"max":{"1min":3.7469999999999999,"5min":3.7469999999999999,"15min":15.029},"last":0.54200000000000004}]},{"osd":1,"last update":"Sun Oct 1 10:41:55 2023","interfaces":[{"interface":"back","average":{"1min":2.9460000000000002,"5min":1.282,"15min":1.3169999999999999},"min":{"1min":0.311,"5min":0.311,"15min":0.309},"max":{"1min":24.308,"5min":24.308,"15min":24.308},"last":2.2930000000000001},{"interface":"front","average":{"1min":2.7869999999999999,"5min":1.278,"15min":1.5800000000000001},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.318},"max":{"1min":24.032,"5min":24.032,"15min":24.032},"last":0.434}]},{"osd":2,"last update":"Sun Oct 1 10:42:13 2023","interfaces":[{"interface":"back","average":{"1min":0.89900000000000002,"5min":0.95499999999999996,"15min":1.0089999999999999},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":2.8500000000000001,"5min":2.8500000000000001,"15min":2.8500000000000001},"last":2.8479999999999999},{"interface":"front","average":{"1min":2.1360000000000001,"5min":1.1599999999999999,"15min":1.018},"min":{"1min":0.502,"5min":0.502,"15min":0.502},"max":{"1min":10.766,"5min":10.766,"15min":10.766},"last":0.64600000000000002}]},{"osd":3,"last update":"Sun Oct 1 10:41:48 2023","interfaces":[{"interface":"back","average":{"1min":2.9380000000000002,"5min":1.3819999999999999,"15min":1.1060000000000001},"min":{"1min":0.55900000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"max":{"1min":21.585000000000001,"5min":21.585000000000001,"15min":21.585000000000001},"last":1.927},{"interface":"front","average":{"1min":3.7160000000000002,"5min":1.506,"15min":1.133},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":21.57,"5min":21.57,"15min":21.57},"last":1.8069999999999999}]},{"osd":4,"last update":"Sun Oct 1 10:42:23 2023","interfaces":[{"interface":"back","average":{"1min":1.2370000000000001,"5min":1.034,"15min":0.95599999999999996},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":4.0439999999999996,"5min":4.0439999999999996,"15min":4.0439999999999996},"last":0.59599999999999997},{"interface":"front","average":{"1min":1.3220000000000001,"5min":1.1879999999999999,"15min":1.1859999999999999},"min":{"1min":0.33900000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":6.6189999999999998,"5min":6.6189999999999998,"15min":6.6189999999999998},"last":0.47599999999999998}]},{"osd":5,"last update":"Sun Oct 1 10:41:55 2023","interfaces":[{"interface":"back","average":{"1min":1.577,"5min":1.135,"15min":1.1990000000000001},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.29699999999999999},"max":{"1min":5.7350000000000003,"5min":5.7350000000000003,"15min":10.106999999999999},"last":2.8170000000000002},{"interface":"front","average":{"1min":2.278,"5min":1.2529999999999999,"15min":1.2090000000000001},"min":{"1min":0.47399999999999998,"5min":0.40400000000000003,"15min":0.34899999999999998},"max":{"1min":10.787000000000001,"5min":10.787000000000001,"15min":10.787000000000001},"last":0.52900000000000003}]},{"osd":6,"last update":"Sun Oct 1 10:41:55 2023","interfaces":[{"interface":"back","average":{"1min":1.157,"5min":1.087,"15min":1.288},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.32800000000000001},"max":{"1min":5.7880000000000003,"5min":5.7880000000000003,"15min":7.3300000000000001},"last":0.55700000000000005},{"interface":"front","average":{"1min":1.5109999999999999,"5min":1.1479999999999999,"15min":1.2669999999999999},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.33900000000000002},"max":{"1min":5.734,"5min":5.734,"15min":6.984},"last":0.77100000000000002}]}]},{"osd":6,"up_from":42,"seq":180388626799,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1492012,"kb_used_data":1194788,"kb_used_omap":0,"kb_used_meta":297216,"kb_avail":92253140,"statfs":{"total":95995035648,"available":94467215360,"internally_reserved":0,"allocated":1223462912,"data_stored":1223000206,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":304349184},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:41:40 2023","interfaces":[{"interface":"back","average":{"1min":0.90200000000000002,"5min":0.89500000000000002,"15min":1.3859999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.28499999999999998},"max":{"1min":4.5069999999999997,"5min":4.5069999999999997,"15min":9.1300000000000008},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.88200000000000001,"5min":0.88800000000000001,"15min":1.3859999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.28899999999999998},"max":{"1min":2.3599999999999999,"5min":2.3599999999999999,"15min":7.3109999999999999},"last":0.54700000000000004}]},{"osd":1,"last update":"Sun Oct 1 10:41:45 2023","interfaces":[{"interface":"back","average":{"1min":1.175,"5min":0.97399999999999998,"15min":1.171},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.26600000000000001},"max":{"1min":4.5410000000000004,"5min":4.5410000000000004,"15min":7.9100000000000001},"last":2.5270000000000001},{"interface":"front","average":{"1min":1.0109999999999999,"5min":0.92500000000000004,"15min":1.3839999999999999},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.33300000000000002},"max":{"1min":3.601,"5min":3.601,"15min":7.4199999999999999},"last":0.68799999999999994}]},{"osd":2,"last update":"Sun Oct 1 10:42:14 2023","interfaces":[{"interface":"back","average":{"1min":1.1319999999999999,"5min":0.94499999999999995,"15min":0.89200000000000002},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":4.6150000000000002,"5min":4.6150000000000002,"15min":4.6150000000000002},"last":0.51200000000000001},{"interface":"front","average":{"1min":1.052,"5min":0.98399999999999999,"15min":0.95399999999999996},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":4.6420000000000003,"5min":4.6420000000000003,"15min":4.6420000000000003},"last":0.56899999999999995}]},{"osd":3,"last update":"Sun Oct 1 10:41:53 2023","interfaces":[{"interface":"back","average":{"1min":1.6000000000000001,"5min":1.1120000000000001,"15min":1.024},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":4.8300000000000001,"5min":4.8300000000000001,"15min":4.8300000000000001},"last":1.823},{"interface":"front","average":{"1min":1.1200000000000001,"5min":1.044,"15min":1.026},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":4.8719999999999999,"5min":4.8719999999999999,"15min":4.8719999999999999},"last":1.615}]},{"osd":4,"last update":"Sun Oct 1 10:41:23 2023","interfaces":[{"interface":"back","average":{"1min":0.95999999999999996,"5min":0.99399999999999999,"15min":1.0109999999999999},"min":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"max":{"1min":1.8779999999999999,"5min":1.8779999999999999,"15min":1.8779999999999999},"last":1.0209999999999999},{"interface":"front","average":{"1min":0.96999999999999997,"5min":1.0609999999999999,"15min":1.089},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":1.4890000000000001,"5min":1.4890000000000001,"15min":1.4890000000000001},"last":0.879}]},{"osd":5,"last update":"Sun Oct 1 10:41:55 2023","interfaces":[{"interface":"back","average":{"1min":1.073,"5min":1.032,"15min":1.2190000000000001},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":4.4450000000000003,"5min":4.4450000000000003,"15min":8.0150000000000006},"last":1.3029999999999999},{"interface":"front","average":{"1min":1.1499999999999999,"5min":1.0840000000000001,"15min":1.222},"min":{"1min":0.45900000000000002,"5min":0.38,"15min":0.34399999999999997},"max":{"1min":2.262,"5min":2.262,"15min":10.842000000000001},"last":4.7380000000000004}]},{"osd":7,"last update":"Sun Oct 1 10:41:40 2023","interfaces":[{"interface":"back","average":{"1min":1.1539999999999999,"5min":1.1000000000000001,"15min":1.085},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.371},"max":{"1min":2.177,"5min":2.177,"15min":4.867},"last":0.80600000000000005},{"interface":"front","average":{"1min":1.4810000000000001,"5min":1.1910000000000001,"15min":1.2969999999999999},"min":{"1min":0.60499999999999998,"5min":0.53000000000000003,"15min":0.34799999999999998},"max":{"1min":5.508,"5min":5.508,"15min":8.077},"last":0.76400000000000001}]}]},{"osd":1,"up_from":101,"seq":433791697016,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3787824,"kb_used_data":3483304,"kb_used_omap":4,"kb_used_meta":304507,"kb_avail":89957328,"statfs":{"total":95995035648,"available":92116303872,"internally_reserved":0,"allocated":3566903296,"data_stored":3566325921,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":311816078},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:41:51 2023","interfaces":[{"interface":"back","average":{"1min":1.0860000000000001,"5min":0.72799999999999998,"15min":1.8160000000000001},"min":{"1min":0.189,"5min":0.189,"15min":0.189},"max":{"1min":3.6459999999999999,"5min":3.6459999999999999,"15min":11.071999999999999},"last":2.4700000000000002},{"interface":"front","average":{"1min":0.89300000000000002,"5min":0.69299999999999995,"15min":1.7609999999999999},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.20499999999999999},"max":{"1min":2.9980000000000002,"5min":2.9980000000000002,"15min":9.3230000000000004},"last":2.0049999999999999}]},{"osd":2,"last update":"Sun Oct 1 10:42:17 2023","interfaces":[{"interface":"back","average":{"1min":1.2230000000000001,"5min":0.86099999999999999,"15min":0.73199999999999998},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":8.2539999999999996,"5min":8.2539999999999996,"15min":8.2539999999999996},"last":0.59099999999999997},{"interface":"front","average":{"1min":1.343,"5min":0.92400000000000004,"15min":0.81899999999999995},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":8.4239999999999995,"5min":8.4239999999999995,"15min":8.4239999999999995},"last":2.0449999999999999}]},{"osd":3,"last update":"Sun Oct 1 10:41:51 2023","interfaces":[{"interface":"back","average":{"1min":1.306,"5min":0.82399999999999995,"15min":0.72599999999999998},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":4.6340000000000003,"5min":4.6340000000000003,"15min":4.6340000000000003},"last":2.6110000000000002},{"interface":"front","average":{"1min":1.6499999999999999,"5min":0.92000000000000004,"15min":0.80800000000000005},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":4.3700000000000001,"5min":4.3700000000000001,"15min":4.3700000000000001},"last":0.80000000000000004}]},{"osd":4,"last update":"Sun Oct 1 10:41:19 2023","interfaces":[{"interface":"back","average":{"1min":0.93600000000000005,"5min":0.92700000000000005,"15min":0.89700000000000002},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":3.0640000000000001,"5min":3.0640000000000001,"15min":3.0640000000000001},"last":2.6859999999999999},{"interface":"front","average":{"1min":1.1559999999999999,"5min":1.046,"15min":1.02},"min":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"max":{"1min":3.4550000000000001,"5min":3.4550000000000001,"15min":3.4550000000000001},"last":2.3780000000000001}]},{"osd":5,"last update":"Sun Oct 1 10:41:51 2023","interfaces":[{"interface":"back","average":{"1min":1.8420000000000001,"5min":1.1459999999999999,"15min":1.671},"min":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.35799999999999998},"max":{"1min":7.2610000000000001,"5min":7.2610000000000001,"15min":8.9429999999999996},"last":0.82099999999999995},{"interface":"front","average":{"1min":1.3460000000000001,"5min":1.0820000000000001,"15min":2.1120000000000001},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.33600000000000002},"max":{"1min":4.5380000000000003,"5min":4.5380000000000003,"15min":11.353999999999999},"last":2.77}]},{"osd":6,"last update":"Sun Oct 1 10:41:51 2023","interfaces":[{"interface":"back","average":{"1min":1.1799999999999999,"5min":1.0569999999999999,"15min":1.819},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.37},"max":{"1min":4.4939999999999998,"5min":4.4939999999999998,"15min":8.6630000000000003},"last":2.738},{"interface":"front","average":{"1min":1.3520000000000001,"5min":1.1000000000000001,"15min":1.806},"min":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.41999999999999998},"max":{"1min":3.6349999999999998,"5min":3.6349999999999998,"15min":7.1369999999999996},"last":2.3519999999999999}]},{"osd":7,"last update":"Sun Oct 1 10:41:51 2023","interfaces":[{"interface":"back","average":{"1min":1.762,"5min":1.2230000000000001,"15min":1.7969999999999999},"min":{"1min":0.66200000000000003,"5min":0.57299999999999995,"15min":0.40500000000000003},"max":{"1min":5.2050000000000001,"5min":5.2050000000000001,"15min":7.1680000000000001},"last":2.3170000000000002},{"interface":"front","average":{"1min":1.704,"5min":1.1910000000000001,"15min":1.675},"min":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.48899999999999999},"max":{"1min":4.5149999999999997,"5min":4.5149999999999997,"15min":9.0749999999999993},"last":0.77000000000000002}]}]},{"osd":0,"up_from":86,"seq":369367187626,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1922364,"kb_used_data":1531828,"kb_used_omap":5,"kb_used_meta":390522,"kb_avail":91822788,"statfs":{"total":95995035648,"available":94026534912,"internally_reserved":0,"allocated":1568591872,"data_stored":1568001381,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":399895162},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 10:41:50 2023","interfaces":[{"interface":"back","average":{"1min":1.236,"5min":0.748,"15min":1.5289999999999999},"min":{"1min":0.224,"5min":0.224,"15min":0.214},"max":{"1min":3.7749999999999999,"5min":3.7749999999999999,"15min":12.855},"last":0.60799999999999998},{"interface":"front","average":{"1min":1.145,"5min":0.747,"15min":2.0499999999999998},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":5.242,"5min":5.242,"15min":15.48},"last":2.8769999999999998}]},{"osd":2,"last update":"Sun Oct 1 10:42:16 2023","interfaces":[{"interface":"back","average":{"1min":1.746,"5min":0.88100000000000001,"15min":0.76100000000000001},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":9.2420000000000009,"5min":9.2420000000000009,"15min":9.2420000000000009},"last":3.3889999999999998},{"interface":"front","average":{"1min":0.95899999999999996,"5min":0.77500000000000002,"15min":0.76000000000000001},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":3.605,"5min":3.605,"15min":3.605},"last":0.28799999999999998}]},{"osd":3,"last update":"Sun Oct 1 10:41:50 2023","interfaces":[{"interface":"back","average":{"1min":1.9630000000000001,"5min":1.014,"15min":0.88100000000000001},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":7.4889999999999999,"5min":7.4889999999999999,"15min":7.4889999999999999},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.93400000000000005,"5min":0.78800000000000003,"15min":0.81100000000000005},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":2.899,"5min":2.899,"15min":2.899},"last":4.0679999999999996}]},{"osd":4,"last update":"Sun Oct 1 10:41:24 2023","interfaces":[{"interface":"back","average":{"1min":0.95899999999999996,"5min":1.0029999999999999,"15min":1.014},"min":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.238,"5min":1.3200000000000001,"15min":1.3200000000000001},"last":2.1669999999999998},{"interface":"front","average":{"1min":0.92400000000000004,"5min":1.0109999999999999,"15min":1.0269999999999999},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":1.206,"5min":1.5029999999999999,"15min":1.5029999999999999},"last":6.0019999999999998}]},{"osd":5,"last update":"Sun Oct 1 10:41:50 2023","interfaces":[{"interface":"back","average":{"1min":1.113,"5min":1.0369999999999999,"15min":1.609},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.31900000000000001},"max":{"1min":3.6469999999999998,"5min":3.6469999999999998,"15min":13.068},"last":0.76900000000000002},{"interface":"front","average":{"1min":1.0409999999999999,"5min":1.022,"15min":1.603},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.34200000000000003},"max":{"1min":3.0459999999999998,"5min":3.0459999999999998,"15min":11.747},"last":0.67900000000000005}]},{"osd":6,"last update":"Sun Oct 1 10:41:50 2023","interfaces":[{"interface":"back","average":{"1min":1.1890000000000001,"5min":1.04,"15min":1.712},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.35899999999999999},"max":{"1min":3.9980000000000002,"5min":3.9980000000000002,"15min":11.663},"last":0.64700000000000002},{"interface":"front","average":{"1min":0.97099999999999997,"5min":0.99299999999999999,"15min":1.6459999999999999},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38600000000000001},"max":{"1min":2.9380000000000002,"5min":2.9380000000000002,"15min":15.069000000000001},"last":2.9409999999999998}]},{"osd":7,"last update":"Sun Oct 1 10:41:50 2023","interfaces":[{"interface":"back","average":{"1min":1.456,"5min":1.133,"15min":1.7310000000000001},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":3.2250000000000001,"5min":3.2250000000000001,"15min":12.686},"last":4.2009999999999996},{"interface":"front","average":{"1min":1.5600000000000001,"5min":1.133,"15min":1.5609999999999999},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.36199999999999999},"max":{"1min":5.0430000000000001,"5min":5.0430000000000001,"15min":13.654},"last":3.6019999999999999}]}]},{"osd":2,"up_from":116,"seq":498216206402,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1376620,"kb_used_data":1195876,"kb_used_omap":7,"kb_used_meta":180728,"kb_avail":92368532,"statfs":{"total":95995035648,"available":94585376768,"internally_reserved":0,"allocated":1224577024,"data_stored":1224057738,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":185065553},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:42:11 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.623,"15min":0.65900000000000003},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":2.4089999999999998,"5min":2.4089999999999998,"15min":2.4089999999999998},"last":0.49399999999999999},{"interface":"front","average":{"1min":1.075,"5min":0.79600000000000004,"15min":0.79400000000000004},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":6.1829999999999998,"5min":6.1829999999999998,"15min":6.1829999999999998},"last":0.246}]},{"osd":1,"last update":"Sun Oct 1 10:42:11 2023","interfaces":[{"interface":"back","average":{"1min":1.6499999999999999,"5min":0.91900000000000004,"15min":0.78100000000000003},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":5.556,"5min":5.556,"15min":5.556},"last":0.29399999999999998},{"interface":"front","average":{"1min":1.0189999999999999,"5min":0.76300000000000001,"15min":0.78800000000000003},"min":{"1min":0.318,"5min":0.252,"15min":0.252},"max":{"1min":4.8170000000000002,"5min":4.8170000000000002,"15min":4.8170000000000002},"last":0.436}]},{"osd":3,"last update":"Sun Oct 1 10:41:46 2023","interfaces":[{"interface":"back","average":{"1min":0.83299999999999996,"5min":0.749,"15min":0.74299999999999999},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":2.4449999999999998,"5min":2.4449999999999998,"15min":2.4449999999999998},"last":1.2789999999999999},{"interface":"front","average":{"1min":0.93200000000000005,"5min":0.878,"15min":0.88400000000000001},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":3.181,"5min":3.181,"15min":3.181},"last":0.33000000000000002}]},{"osd":4,"last update":"Sun Oct 1 10:41:24 2023","interfaces":[{"interface":"back","average":{"1min":0.81299999999999994,"5min":0.97399999999999998,"15min":0.97799999999999998},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":1.208,"5min":1.391,"15min":1.391},"last":4.266},{"interface":"front","average":{"1min":0.77800000000000002,"5min":0.94299999999999995,"15min":0.95199999999999996},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":1.1479999999999999,"5min":1.254,"15min":1.254},"last":0.37}]},{"osd":5,"last update":"Sun Oct 1 10:42:11 2023","interfaces":[{"interface":"back","average":{"1min":1.262,"5min":1.1060000000000001,"15min":1.1160000000000001},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":3.5710000000000002,"5min":3.5710000000000002,"15min":3.5710000000000002},"last":0.48299999999999998},{"interface":"front","average":{"1min":1.161,"5min":1.05,"15min":1.028},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":3.5030000000000001,"5min":3.5030000000000001,"15min":3.5030000000000001},"last":0.60799999999999998}]},{"osd":6,"last update":"Sun Oct 1 10:42:11 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.94699999999999995,"15min":1.0489999999999999},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":1.05,"5min":1.486,"15min":1.486},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.86599999999999999,"5min":1.034,"15min":1.1080000000000001},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":2.6869999999999998,"5min":2.6869999999999998,"15min":2.6869999999999998},"last":0.57099999999999995}]},{"osd":7,"last update":"Sun Oct 1 10:42:11 2023","interfaces":[{"interface":"back","average":{"1min":0.95199999999999996,"5min":1.0580000000000001,"15min":1.1080000000000001},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":2.363,"5min":2.363,"15min":2.363},"last":0.53800000000000003},{"interface":"front","average":{"1min":1.992,"5min":1.3240000000000001,"15min":1.258},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":6.7460000000000004,"5min":6.7460000000000004,"15min":6.7460000000000004},"last":4.2210000000000001}]}]},{"osd":3,"up_from":121,"seq":519691042874,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3258528,"kb_used_data":3124632,"kb_used_omap":2,"kb_used_meta":133885,"kb_avail":90486624,"statfs":{"total":95995035648,"available":92658302976,"internally_reserved":0,"allocated":3199623168,"data_stored":3199114634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":137098293},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,0,1,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:41:50 2023","interfaces":[{"interface":"back","average":{"1min":3.4390000000000001,"5min":1.212,"15min":0.85299999999999998},"min":{"1min":0.318,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":51.396999999999998,"5min":51.396999999999998,"15min":51.396999999999998},"last":0.52300000000000002},{"interface":"front","average":{"1min":3.6920000000000002,"5min":1.2350000000000001,"15min":0.80000000000000004},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":53.191000000000003,"5min":53.191000000000003,"15min":53.191000000000003},"last":2.5350000000000001}]},{"osd":1,"last update":"Sun Oct 1 10:41:50 2023","interfaces":[{"interface":"back","average":{"1min":1.0089999999999999,"5min":0.75600000000000001,"15min":0.71799999999999997},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":2.5640000000000001,"5min":2.5640000000000001,"15min":2.5640000000000001},"last":0.49399999999999999},{"interface":"front","average":{"1min":1.0780000000000001,"5min":0.83299999999999996,"15min":0.79900000000000004},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":2.524,"5min":2.524,"15min":2.524},"last":1.256}]},{"osd":2,"last update":"Sun Oct 1 10:41:50 2023","interfaces":[{"interface":"back","average":{"1min":0.70699999999999996,"5min":0.69199999999999995,"15min":0.68500000000000005},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":1.9430000000000001,"5min":1.9430000000000001,"15min":1.9430000000000001},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.85199999999999998,"5min":0.81299999999999994,"15min":0.79500000000000004},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":2.4340000000000002,"5min":2.4340000000000002,"15min":2.4340000000000002},"last":0.70999999999999996}]},{"osd":4,"last update":"Sun Oct 1 10:41:17 2023","interfaces":[{"interface":"back","average":{"1min":1.054,"5min":0.98099999999999998,"15min":0.95399999999999996},"min":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":2.4620000000000002,"5min":2.4620000000000002,"15min":2.4620000000000002},"last":0.92500000000000004},{"interface":"front","average":{"1min":0.91900000000000004,"5min":0.94999999999999996,"15min":0.93500000000000005},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":1.3069999999999999,"5min":1.355,"15min":1.355},"last":2.6309999999999998}]},{"osd":5,"last update":"Sun Oct 1 10:41:50 2023","interfaces":[{"interface":"back","average":{"1min":1.6899999999999999,"5min":1.1080000000000001,"15min":0.998},"min":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"max":{"1min":5.399,"5min":5.399,"15min":5.399},"last":4.6760000000000002},{"interface":"front","average":{"1min":1.321,"5min":1.069,"15min":1.0269999999999999},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":5.7359999999999998,"5min":5.7359999999999998,"15min":5.7359999999999998},"last":7.0910000000000002}]},{"osd":6,"last update":"Sun Oct 1 10:41:50 2023","interfaces":[{"interface":"back","average":{"1min":1.4970000000000001,"5min":1.1240000000000001,"15min":1.048},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":4.7809999999999997,"5min":4.7809999999999997,"15min":4.7809999999999997},"last":2.6579999999999999},{"interface":"front","average":{"1min":0.97299999999999998,"5min":1.0680000000000001,"15min":1.093},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":2.4449999999999998,"5min":2.4449999999999998,"15min":2.4449999999999998},"last":0.90700000000000003}]},{"osd":7,"last update":"Sun Oct 1 10:41:50 2023","interfaces":[{"interface":"back","average":{"1min":1.054,"5min":1.0369999999999999,"15min":1.052},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":2.7189999999999999,"5min":2.7189999999999999,"15min":2.7189999999999999},"last":1.1060000000000001},{"interface":"front","average":{"1min":1.214,"5min":1.0860000000000001,"15min":1.04},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":2.4940000000000002,"5min":2.4940000000000002,"15min":2.4940000000000002},"last":3.7810000000000001}]}]},{"osd":4,"up_from":126,"seq":541165879347,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1314140,"kb_used_data":1154644,"kb_used_omap":10,"kb_used_meta":159477,"kb_avail":92431012,"statfs":{"total":95995035648,"available":94649356288,"internally_reserved":0,"allocated":1182355456,"data_stored":1181850994,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":163305060},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:41:21 2023","interfaces":[{"interface":"back","average":{"1min":0.72299999999999998,"5min":0.89500000000000002,"15min":0.90100000000000002},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":1.1870000000000001,"5min":1.3009999999999999,"15min":1.3009999999999999},"last":0.28999999999999998},{"interface":"front","average":{"1min":0.86599999999999999,"5min":0.95199999999999996,"15min":0.97699999999999998},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.593,"5min":1.593,"15min":1.593},"last":0.35199999999999998}]},{"osd":1,"last update":"Sun Oct 1 10:41:21 2023","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":0.94099999999999995,"15min":0.93100000000000005},"min":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"max":{"1min":1.9410000000000001,"5min":1.9410000000000001,"15min":1.9410000000000001},"last":0.56399999999999995},{"interface":"front","average":{"1min":1.002,"5min":0.97399999999999998,"15min":0.95999999999999996},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":2.8730000000000002,"5min":2.8730000000000002,"15min":2.8730000000000002},"last":0.38100000000000001}]},{"osd":2,"last update":"Sun Oct 1 10:41:21 2023","interfaces":[{"interface":"back","average":{"1min":0.83699999999999997,"5min":0.94099999999999995,"15min":0.94899999999999995},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":1.3129999999999999,"5min":1.3129999999999999,"15min":1.3129999999999999},"last":0.41099999999999998},{"interface":"front","average":{"1min":0.84699999999999998,"5min":0.96399999999999997,"15min":0.97199999999999998},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":1.337,"5min":1.395,"15min":1.395},"last":0.47299999999999998}]},{"osd":3,"last update":"Sun Oct 1 10:41:21 2023","interfaces":[{"interface":"back","average":{"1min":1.0800000000000001,"5min":1.0629999999999999,"15min":1.073},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":3.1890000000000001,"5min":3.1890000000000001,"15min":3.1890000000000001},"last":0.435},{"interface":"front","average":{"1min":0.83699999999999997,"5min":0.94599999999999995,"15min":0.96799999999999997},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":1.143,"5min":1.341,"15min":1.341},"last":0.58699999999999997}]},{"osd":5,"last update":"Sun Oct 1 10:41:21 2023","interfaces":[{"interface":"back","average":{"1min":1.294,"5min":1.119,"15min":1.103},"min":{"1min":0.503,"5min":0.503,"15min":0.503},"max":{"1min":3.9239999999999999,"5min":3.9239999999999999,"15min":3.9239999999999999},"last":0.70599999999999996},{"interface":"front","average":{"1min":0.95699999999999996,"5min":1.073,"15min":1.087},"min":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":1.415,"5min":1.415,"15min":1.415},"last":0.76500000000000001}]},{"osd":6,"last update":"Sun Oct 1 10:41:21 2023","interfaces":[{"interface":"back","average":{"1min":0.95099999999999996,"5min":1.093,"15min":1.123},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":1.4390000000000001,"5min":1.579,"15min":1.579},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.94499999999999995,"5min":1.0549999999999999,"15min":1.0680000000000001},"min":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"max":{"1min":1.633,"5min":1.633,"15min":1.633},"last":0.52500000000000002}]},{"osd":7,"last update":"Sun Oct 1 10:41:21 2023","interfaces":[{"interface":"back","average":{"1min":1.006,"5min":1.1479999999999999,"15min":1.1699999999999999},"min":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"max":{"1min":1.714,"5min":1.714,"15min":1.714},"last":1.002},{"interface":"front","average":{"1min":1.1319999999999999,"5min":1.169,"15min":1.169},"min":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"max":{"1min":3.544,"5min":3.544,"15min":3.544},"last":0.55400000000000005}]}]},{"osd":5,"up_from":36,"seq":154618823028,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4006240,"kb_used_data":3416536,"kb_used_omap":0,"kb_used_meta":589696,"kb_avail":89738912,"statfs":{"total":95995035648,"available":91892645888,"internally_reserved":0,"allocated":3498532864,"data_stored":3498077722,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":603848704},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:41:41 2023","interfaces":[{"interface":"back","average":{"1min":1.1990000000000001,"5min":0.97399999999999998,"15min":1.2569999999999999},"min":{"1min":0.59199999999999997,"5min":0.46300000000000002,"15min":0.26400000000000001},"max":{"1min":2.883,"5min":4.1900000000000004,"15min":10.429},"last":0.63500000000000001},{"interface":"front","average":{"1min":1.1439999999999999,"5min":1.0149999999999999,"15min":1.448},"min":{"1min":0.317,"5min":0.317,"15min":0.29199999999999998},"max":{"1min":4.7249999999999996,"5min":5.3920000000000003,"15min":13.018000000000001},"last":0.60199999999999998}]},{"osd":1,"last update":"Sun Oct 1 10:41:41 2023","interfaces":[{"interface":"back","average":{"1min":1.2589999999999999,"5min":1.022,"15min":1.5880000000000001},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.29099999999999998},"max":{"1min":3.1480000000000001,"5min":3.8769999999999998,"15min":10.858000000000001},"last":0.73499999999999999},{"interface":"front","average":{"1min":1.5740000000000001,"5min":1.1279999999999999,"15min":1.706},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.32500000000000001},"max":{"1min":9.8469999999999995,"5min":9.8469999999999995,"15min":12.864000000000001},"last":0.504}]},{"osd":2,"last update":"Sun Oct 1 10:42:16 2023","interfaces":[{"interface":"back","average":{"1min":1.042,"5min":0.97699999999999998,"15min":0.98099999999999998},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":3.0350000000000001,"5min":3.0350000000000001,"15min":3.0350000000000001},"last":0.86699999999999999},{"interface":"front","average":{"1min":1.5529999999999999,"5min":1.103,"15min":1.01},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":5.0960000000000001,"5min":5.0960000000000001,"15min":5.0960000000000001},"last":0.69099999999999995}]},{"osd":3,"last update":"Sun Oct 1 10:41:56 2023","interfaces":[{"interface":"back","average":{"1min":2.758,"5min":1.3120000000000001,"15min":1.0589999999999999},"min":{"1min":0.71599999999999997,"5min":0.58399999999999996,"15min":0.58399999999999996},"max":{"1min":9.8059999999999992,"5min":9.8059999999999992,"15min":9.8059999999999992},"last":65.061000000000007},{"interface":"front","average":{"1min":1.3029999999999999,"5min":1.042,"15min":0.97399999999999998},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":3.2250000000000001,"5min":3.2250000000000001,"15min":3.2250000000000001},"last":65.111999999999995}]},{"osd":4,"last update":"Sun Oct 1 10:41:21 2023","interfaces":[{"interface":"back","average":{"1min":1.1359999999999999,"5min":1.0960000000000001,"15min":1.083},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":3.1669999999999998,"5min":3.1669999999999998,"15min":3.1669999999999998},"last":0.79600000000000004},{"interface":"front","average":{"1min":1.204,"5min":1.1240000000000001,"15min":1.097},"min":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"max":{"1min":3.1000000000000001,"5min":3.1000000000000001,"15min":3.1000000000000001},"last":0.81799999999999995}]},{"osd":6,"last update":"Sun Oct 1 10:41:56 2023","interfaces":[{"interface":"back","average":{"1min":1.361,"5min":1.1859999999999999,"15min":1.2549999999999999},"min":{"1min":0.47899999999999998,"5min":0.42399999999999999,"15min":0.36199999999999999},"max":{"1min":3.464,"5min":6.96,"15min":6.96},"last":0.91400000000000003},{"interface":"front","average":{"1min":1.6399999999999999,"5min":1.214,"15min":1.2649999999999999},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.312},"max":{"1min":5.7160000000000002,"5min":8.1790000000000003,"15min":8.4339999999999993},"last":0.83999999999999997}]},{"osd":7,"last update":"Sun Oct 1 10:41:56 2023","interfaces":[{"interface":"back","average":{"1min":1.968,"5min":1.286,"15min":1.3069999999999999},"min":{"1min":0.46800000000000003,"5min":0.39000000000000001,"15min":0.34699999999999998},"max":{"1min":7.0099999999999998,"5min":7.0099999999999998,"15min":9.4629999999999992},"last":0.89000000000000001},{"interface":"front","average":{"1min":1.7110000000000001,"5min":1.276,"15min":1.2569999999999999},"min":{"1min":0.64100000000000001,"5min":0.372,"15min":0.372},"max":{"1min":4.625,"5min":4.625,"15min":10.656000000000001},"last":0.95399999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1572536320,"data_stored":1572536320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3577806848,"data_stored":3577806848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1228537856,"data_stored":1228537856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3200843776,"data_stored":3200843776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1183776768,"data_stored":1183776768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3522494464,"data_stored":3522494464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1221656576,"data_stored":1221656576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3659988992,"data_stored":3659988992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T10:42:27.139 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-01T10:42:27.140 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd pool set device_health_metrics pg_num 11 2023-10-01T10:42:27.151 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:27.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:27.154 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:42:27.223 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 79 16 102731 102715 81.2465 57.0625 0.0185526 0.012303 2023-10-01T10:42:27.255 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:27.294 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:27.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:27.299 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:42:27.400 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:27.434 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:27.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:27.436 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:27.519 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:27 smithi078 bash[75299]: cluster 2023-10-01T10:42:25.746266+0000 mgr.y (mgr.25072) 741 : cluster 0 pgmap v650: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 864 op/s 2023-10-01T10:42:27.537 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:27.571 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:27.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:27.575 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:42:27.675 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:27.708 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:27.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:27.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:42:27.811 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:27.820 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:27 smithi078 bash[78274]: cluster 2023-10-01T10:42:25.746266+0000 mgr.y (mgr.25072) 741 : cluster 0 pgmap v650: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 864 op/s 2023-10-01T10:42:27.841 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:27.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:27.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:42:27.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:27 smithi173 bash[58816]: cluster 2023-10-01T10:42:25.746266+0000 mgr.y (mgr.25072) 741 : cluster 0 pgmap v650: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 864 op/s 2023-10-01T10:42:27.946 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:27.976 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:27.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:27.981 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:42:28.082 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:28.115 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:28.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:28.117 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:42:28.218 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:28.225 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:2023-10-01T10:42:28.222509+0000 min lat: 0.00132359 max lat: 0.210168 avg lat: 0.0123532 2023-10-01T10:42:28.225 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T10:42:28.225 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 80 16 103602 103586 80.9114 54.4375 0.00909239 0.0123532 2023-10-01T10:42:28.246 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:28.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:28.254 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:42:28.354 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:28.375 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:28.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:28.377 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:42:28.478 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:28.508 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:28.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:28.516 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:42:28.616 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:28.642 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:28.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:28.648 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:42:28.748 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:28.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:28.782 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:28.782 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:42:28.883 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:28.906 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:28.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:28.911 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:42:29.012 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:29.041 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:29.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:29.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:42:29.053 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T10:42:29.067 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T10:42:29.146 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:29.173 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:29.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:29.180 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:42:29.187 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:42:29.225 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 81 16 104446 104430 80.5615 52.75 0.00824839 0.0124053 2023-10-01T10:42:29.281 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:29.322 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:29.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:29.326 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:42:29.426 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:29.456 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:29.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:29.458 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:29.559 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:29.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:29 smithi078 bash[75299]: cluster 2023-10-01T10:42:27.746748+0000 mgr.y (mgr.25072) 742 : cluster 0 pgmap v651: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 53 MiB/s wr, 842 op/s 2023-10-01T10:42:29.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:29 smithi078 bash[78274]: cluster 2023-10-01T10:42:27.746748+0000 mgr.y (mgr.25072) 742 : cluster 0 pgmap v651: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 53 MiB/s wr, 842 op/s 2023-10-01T10:42:29.586 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:29.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:29.592 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:42:29.693 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:29.704 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:29 smithi173 bash[58816]: cluster 2023-10-01T10:42:27.746748+0000 mgr.y (mgr.25072) 742 : cluster 0 pgmap v651: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 53 MiB/s wr, 842 op/s 2023-10-01T10:42:29.727 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:29.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:29.730 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:29.830 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:29.857 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:29.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:29.862 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:42:29.963 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:30.007 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:30.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:30.012 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:42:30.113 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:30.143 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:30.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:30.147 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:42:30.225 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 82 16 105270 105254 80.207 51.5 0.0152938 0.0124629 2023-10-01T10:42:30.248 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:30.270 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:42:30 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:42:30] "GET /metrics HTTP/1.1" 200 36240 "" "Prometheus/2.33.4" 2023-10-01T10:42:30.280 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:30.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:30.283 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:42:30.383 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:30.409 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:30.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:30.413 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:42:30.513 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:30.542 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:30.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:30.545 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:42:30.646 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:30.681 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:30.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:30.686 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:42:30.787 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:30.824 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:30.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:30.829 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:42:30.930 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:30.955 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:30.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:30.958 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:42:31.059 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:31.085 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:31.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:31.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:42:31.190 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:31.223 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:31.225 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 83 16 106109 106093 79.8724 52.4375 0.0210594 0.0125142 2023-10-01T10:42:31.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:31.229 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:42:31.238 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:42:31.330 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:31.360 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:31.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:31.363 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:31.464 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:31.504 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:31.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:31.506 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:42:31.607 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:31.641 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:31.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:31.648 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:42:31.749 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:31.758 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:31 smithi173 bash[58816]: cluster 2023-10-01T10:42:29.747361+0000 mgr.y (mgr.25072) 743 : cluster 0 pgmap v652: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 52 MiB/s wr, 836 op/s 2023-10-01T10:42:31.780 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:31.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:31.786 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:42:31.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:31 smithi078 bash[75299]: cluster 2023-10-01T10:42:29.747361+0000 mgr.y (mgr.25072) 743 : cluster 0 pgmap v652: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 52 MiB/s wr, 836 op/s 2023-10-01T10:42:31.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:31 smithi078 bash[78274]: cluster 2023-10-01T10:42:29.747361+0000 mgr.y (mgr.25072) 743 : cluster 0 pgmap v652: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 52 MiB/s wr, 836 op/s 2023-10-01T10:42:31.886 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:31.915 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:31.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:31.919 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:42:32.020 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:32.057 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:32.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:32.066 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:42:32.167 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:32.207 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:32.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:32.214 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:42:32.226 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 84 16 106934 106918 79.5354 51.5625 0.0122969 0.0125675 2023-10-01T10:42:32.314 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:32.358 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:32.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:32.360 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:42:32.461 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:32.492 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:32.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:32.497 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:42:32.598 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:32.633 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:32.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:32.639 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:32.740 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:32.767 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:32.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:32.769 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:42:32.870 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:32.896 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:32.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:32.898 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:42:32.999 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:33.036 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:33.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:33.039 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:42:33.140 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:33.175 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:33.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:33.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:42:33.226 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 85 16 107769 107753 79.2136 52.1875 0.0305233 0.0126182 2023-10-01T10:42:33.279 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:33.307 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:33.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:33.309 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:33.410 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:33.447 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:33.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:33.449 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:33.550 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:33.559 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:33 smithi173 bash[58816]: cluster 2023-10-01T10:42:31.748013+0000 mgr.y (mgr.25072) 744 : cluster 0 pgmap v653: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 52 MiB/s wr, 838 op/s 2023-10-01T10:42:33.559 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:33 smithi173 bash[58816]: audit 2023-10-01T10:42:32.419627+0000 mon.a (mon.0) 962 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:42:33.559 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:33 smithi173 bash[58816]: audit 2023-10-01T10:42:32.420767+0000 mon.a (mon.0) 963 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:42:33.584 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:33.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:33.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:42:33.691 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:33.730 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:33.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:33.735 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:33.808 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:42:33.813 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:42:33.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:33 smithi078 bash[78274]: cluster 2023-10-01T10:42:31.748013+0000 mgr.y (mgr.25072) 744 : cluster 0 pgmap v653: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 52 MiB/s wr, 838 op/s 2023-10-01T10:42:33.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:33 smithi078 bash[78274]: audit 2023-10-01T10:42:32.419627+0000 mon.a (mon.0) 962 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:42:33.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:33 smithi078 bash[78274]: audit 2023-10-01T10:42:32.420767+0000 mon.a (mon.0) 963 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:42:33.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:33 smithi078 bash[75299]: cluster 2023-10-01T10:42:31.748013+0000 mgr.y (mgr.25072) 744 : cluster 0 pgmap v653: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 52 MiB/s wr, 838 op/s 2023-10-01T10:42:33.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:33 smithi078 bash[75299]: audit 2023-10-01T10:42:32.419627+0000 mon.a (mon.0) 962 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:42:33.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:33 smithi078 bash[75299]: audit 2023-10-01T10:42:32.420767+0000 mon.a (mon.0) 963 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:42:33.836 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:33.863 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:33.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:33.870 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:42:33.971 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:33.999 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:34.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:34.006 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:42:34.107 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:34.146 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:34.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:34.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:34.226 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 86 16 108611 108595 78.9045 52.625 0.0183201 0.0126679 2023-10-01T10:42:34.251 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:34.282 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:34.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:34.286 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:42:34.386 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:34.412 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:34.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:34.415 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:34.516 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:34.546 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:34.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:34.550 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:42:34.651 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:34.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:34.685 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:34.685 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:34.785 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:34.818 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:34.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:34.823 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:34.924 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:34.956 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:34.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:34.958 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:42:35.059 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:35.092 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:35.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:35.094 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:35.195 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:35.221 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:35.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:35.225 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:35.226 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 87 16 109445 109429 78.5966 52.125 0.0292153 0.0127178 2023-10-01T10:42:35.326 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:35.357 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:35.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:35.360 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:35.460 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:35.490 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:35.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:35.494 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:35.513 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:35 smithi078 bash[75299]: cluster 2023-10-01T10:42:33.752372+0000 mgr.y (mgr.25072) 745 : cluster 0 pgmap v654: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 51 MiB/s wr, 814 op/s 2023-10-01T10:42:35.594 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:35.643 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:35.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:35.646 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:42:35.747 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:35.767 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:35.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:35.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:35.780 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:35 smithi078 bash[78274]: cluster 2023-10-01T10:42:33.752372+0000 mgr.y (mgr.25072) 745 : cluster 0 pgmap v654: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 51 MiB/s wr, 814 op/s 2023-10-01T10:42:35.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:35 smithi173 bash[58816]: cluster 2023-10-01T10:42:33.752372+0000 mgr.y (mgr.25072) 745 : cluster 0 pgmap v654: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 51 MiB/s wr, 814 op/s 2023-10-01T10:42:35.872 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:35.902 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:35.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:35.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:42:36.004 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:36.037 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:36.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:36.039 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:42:36.140 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:36.166 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:36.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:36.172 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:36.226 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 88 16 110202 110186 78.2411 47.3125 0.0055792 0.0127707 2023-10-01T10:42:36.274 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:36.304 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:36.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:36.308 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:42:36.409 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:36.437 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:36.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:36.439 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:42:36.540 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:36.559 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:36.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:36.561 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:42:36.661 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:36.690 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:36.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:36.693 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:42:36.794 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:36.819 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:36.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:36.825 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:36.926 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:36.945 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:36.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:36.950 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:42:37.050 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:37.090 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:37.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:37.095 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:42:37.195 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:37.225 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:37.226 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 89 16 110981 110965 77.909 48.6875 0.0285227 0.0128301 2023-10-01T10:42:37.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:37.228 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:42:37.328 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:37.355 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:37.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:37.359 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:37.460 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:37.468 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:37 smithi078 bash[78274]: cluster 2023-10-01T10:42:35.753071+0000 mgr.y (mgr.25072) 746 : cluster 0 pgmap v655: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 56 MiB/s wr, 894 op/s 2023-10-01T10:42:37.468 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:37 smithi078 bash[75299]: cluster 2023-10-01T10:42:35.753071+0000 mgr.y (mgr.25072) 746 : cluster 0 pgmap v655: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 56 MiB/s wr, 894 op/s 2023-10-01T10:42:37.484 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:37.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:37.488 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:42:37.589 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:37.639 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:37.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:37.642 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:42:37.743 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:37.751 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:37 smithi173 bash[58816]: cluster 2023-10-01T10:42:35.753071+0000 mgr.y (mgr.25072) 746 : cluster 0 pgmap v655: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 56 MiB/s wr, 894 op/s 2023-10-01T10:42:37.776 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:37.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:37.778 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:42:37.878 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:37.904 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:37.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:37.911 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:42:38.012 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:38.041 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:38.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:38.048 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:38.148 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:38.180 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:38.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:38.187 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:42:38.226 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Total time run: 90.0102 2023-10-01T10:42:38.227 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Total writes made: 111812 2023-10-01T10:42:38.227 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Write size: 65536 2023-10-01T10:42:38.227 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Object size: 65536 2023-10-01T10:42:38.227 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Bandwidth (MB/sec): 77.6384 2023-10-01T10:42:38.227 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Stddev Bandwidth: 30.7996 2023-10-01T10:42:38.227 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Max bandwidth (MB/sec): 176.25 2023-10-01T10:42:38.227 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Min bandwidth (MB/sec): 40.625 2023-10-01T10:42:38.227 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Average IOPS: 1242 2023-10-01T10:42:38.227 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Stddev IOPS: 492.794 2023-10-01T10:42:38.227 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Max IOPS: 2820 2023-10-01T10:42:38.228 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Min IOPS: 650 2023-10-01T10:42:38.228 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Average Latency(s): 0.0128756 2023-10-01T10:42:38.228 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Stddev Latency(s): 0.0104119 2023-10-01T10:42:38.228 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Max latency(s): 0.210168 2023-10-01T10:42:38.228 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Min latency(s): 0.00132359 2023-10-01T10:42:38.234 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Cleaning up (deleting benchmark objects) 2023-10-01T10:42:38.287 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:38.308 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:38.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:38.311 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:38.412 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:38.448 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:38.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:38.453 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:42:38.554 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:38.581 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:38.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:38.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:38.684 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:38.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:38.716 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:38.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:42:38.817 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:38.844 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:38.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:38.847 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:38.947 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:38.975 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:38.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:38.981 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:42:39.082 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:39.118 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:39.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:39.120 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:42:39.221 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:39.251 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:39.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:39.253 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:39.354 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:39.357 INFO:teuthology.orchestra.run.smithi078.stdout:ERROR: (22) Invalid argument 2023-10-01T10:42:39.358 INFO:teuthology.orchestra.run.smithi078.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T10:42:39.385 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:39.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:39.390 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:39.491 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:39.516 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:39.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:39.517 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:42:39.542 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:39 smithi078 bash[75299]: cluster 2023-10-01T10:42:37.753624+0000 mgr.y (mgr.25072) 747 : cluster 0 pgmap v656: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 49 MiB/s wr, 787 op/s 2023-10-01T10:42:39.543 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:39 smithi078 bash[78274]: cluster 2023-10-01T10:42:37.753624+0000 mgr.y (mgr.25072) 747 : cluster 0 pgmap v656: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 49 MiB/s wr, 787 op/s 2023-10-01T10:42:39.618 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:39.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:39 smithi173 bash[58816]: cluster 2023-10-01T10:42:37.753624+0000 mgr.y (mgr.25072) 747 : cluster 0 pgmap v656: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 49 MiB/s wr, 787 op/s 2023-10-01T10:42:39.654 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:39.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:39.659 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:42:39.760 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:39.798 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:39.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:39.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:42:39.903 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:39.929 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:39.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:39.931 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:42:40.032 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:40.066 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:40.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:40.071 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:42:40.172 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:40.195 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:40.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:40.200 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:42:40.301 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:40.328 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:40.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:40.331 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:42:40.431 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:40.438 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:42:40 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:42:40] "GET /metrics HTTP/1.1" 200 36246 "" "Prometheus/2.33.4" 2023-10-01T10:42:40.449 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:40.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:40.452 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:42:40.552 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:40.592 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:40.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:40.598 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:42:40.699 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:40.731 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:40.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:40.736 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:42:40.837 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:40.891 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:40.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:40.894 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:42:40.995 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:41.027 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:41.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:41.032 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:42:41.132 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:41.169 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:41.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:41.172 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:42:41.272 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:41.301 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:41.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:41.308 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:42:41.409 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:41.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:41.442 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:41.442 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:41.543 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:41.559 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:41 smithi078 bash[75299]: cluster 2023-10-01T10:42:39.754229+0000 mgr.y (mgr.25072) 748 : cluster 0 pgmap v657: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 52 MiB/s wr, 852 op/s 2023-10-01T10:42:41.581 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:41.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:41.588 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:42:41.689 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:41.698 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:41 smithi173 bash[58816]: cluster 2023-10-01T10:42:39.754229+0000 mgr.y (mgr.25072) 748 : cluster 0 pgmap v657: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 52 MiB/s wr, 852 op/s 2023-10-01T10:42:41.716 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:41.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:41.718 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:42:41.819 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:41.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:41 smithi078 bash[78274]: cluster 2023-10-01T10:42:39.754229+0000 mgr.y (mgr.25072) 748 : cluster 0 pgmap v657: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 52 MiB/s wr, 852 op/s 2023-10-01T10:42:41.851 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:41.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:41.855 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:42:41.956 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:41.975 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:41.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:41.976 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:42.077 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:42.114 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:42.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:42.119 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:42:42.220 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:42.250 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:42.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:42.252 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:42:42.353 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:42.382 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:42.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:42.385 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:42:42.397 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-01T10:42:42.486 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:42.516 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:42.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:42.524 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:42.625 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:42.664 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:42.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:42.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:42:42.769 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:42.803 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:42.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:42.810 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:42:42.910 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:42.932 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:42.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:42.938 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:42:43.039 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:43.079 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:43.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:43.085 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:43.185 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:43.214 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:43.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:43.216 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:43.317 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:43.343 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:43.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:43.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:42:43.447 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:43.456 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:43 smithi078 bash[78274]: cluster 2023-10-01T10:42:41.755008+0000 mgr.y (mgr.25072) 749 : cluster 0 pgmap v658: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 49 MiB/s wr, 891 op/s 2023-10-01T10:42:43.456 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:43 smithi078 bash[75299]: cluster 2023-10-01T10:42:41.755008+0000 mgr.y (mgr.25072) 749 : cluster 0 pgmap v658: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 49 MiB/s wr, 891 op/s 2023-10-01T10:42:43.474 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:43.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:43.480 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:43.581 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:43.606 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:43.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:43.612 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:42:43.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:43 smithi173 bash[58816]: cluster 2023-10-01T10:42:41.755008+0000 mgr.y (mgr.25072) 749 : cluster 0 pgmap v658: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 49 MiB/s wr, 891 op/s 2023-10-01T10:42:43.713 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:43.733 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:43.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:43.737 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:43.837 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:43.866 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:43.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:43.871 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:43.971 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:43.998 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:44.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:44.003 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:44.104 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:44.129 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:44.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:44.136 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:42:44.236 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:44.264 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:44.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:44.269 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:42:44.369 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:44.394 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:44.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:44.397 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:42:44.497 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:44.527 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:44.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:44.532 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:42:44.632 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:44.660 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:44.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:44.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:44.763 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:44.808 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:44.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:44.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:42:44.913 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:44.941 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:44.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:44.947 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:42:45.048 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:45.078 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:45.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:45.087 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:42:45.188 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:45.218 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:45.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:45.224 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:45.324 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:45.363 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:45.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:45.365 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:42:45.433 INFO:teuthology.orchestra.run.smithi078.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-10-01T10:42:45.466 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:45.549 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:45.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:45.553 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:42:45.654 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:45.669 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:45 smithi078 bash[75299]: cluster 2023-10-01T10:42:43.755521+0000 mgr.y (mgr.25072) 750 : cluster 0 pgmap v659: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 36 MiB/s wr, 751 op/s 2023-10-01T10:42:45.670 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:45 smithi078 bash[75299]: audit 2023-10-01T10:42:45.260738+0000 mon.a (mon.0) 964 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:42:45.670 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:45 smithi078 bash[75299]: audit 2023-10-01T10:42:45.428278+0000 mon.a (mon.0) 965 : audit 1 from='client.? 172.21.15.78:0/2753593049' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-10-01T10:42:45.670 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:45 smithi078 bash[78274]: cluster 2023-10-01T10:42:43.755521+0000 mgr.y (mgr.25072) 750 : cluster 0 pgmap v659: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 36 MiB/s wr, 751 op/s 2023-10-01T10:42:45.670 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:45 smithi078 bash[78274]: audit 2023-10-01T10:42:45.260738+0000 mon.a (mon.0) 964 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:42:45.670 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:45 smithi078 bash[78274]: audit 2023-10-01T10:42:45.428278+0000 mon.a (mon.0) 965 : audit 1 from='client.? 172.21.15.78:0/2753593049' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-10-01T10:42:45.695 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:45.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:45.698 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:42:45.798 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:45.814 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:45 smithi173 bash[58816]: cluster 2023-10-01T10:42:43.755521+0000 mgr.y (mgr.25072) 750 : cluster 0 pgmap v659: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 36 MiB/s wr, 751 op/s 2023-10-01T10:42:45.815 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:45 smithi173 bash[58816]: audit 2023-10-01T10:42:45.260738+0000 mon.a (mon.0) 964 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:42:45.815 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:45 smithi173 bash[58816]: audit 2023-10-01T10:42:45.428278+0000 mon.a (mon.0) 965 : audit 1 from='client.? 172.21.15.78:0/2753593049' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "11"}]: dispatch 2023-10-01T10:42:45.833 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:45.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:45.835 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:45.936 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:45.970 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:45.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:45.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:42:46.074 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:46.099 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:46.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:46.100 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:42:46.201 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:46.234 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:46.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:46.238 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:46.339 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:46.391 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:46.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:46.396 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:42:46.497 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:46.525 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:46.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:46.528 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:42:46.629 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:46.658 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:46.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:46.663 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:46.764 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:46.797 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:46.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:46.801 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:42:46.902 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:46.928 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:46.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:46.931 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:47.031 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:47.069 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:47.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:47.076 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:42:47.177 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:47.210 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:47.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:47.216 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:42:47.317 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:47.343 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:47.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:47.348 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:47.449 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:47.481 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:47.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:47.483 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:42:47.584 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:47.616 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:47.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:47.622 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:47.625 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-10-01T10:42:47.723 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:47.731 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:47 smithi173 bash[58816]: cluster 2023-10-01T10:42:45.756058+0000 mgr.y (mgr.25072) 751 : cluster 0 pgmap v660: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 32 MiB/s wr, 885 op/s 2023-10-01T10:42:47.731 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:47 smithi173 bash[58816]: audit 2023-10-01T10:42:47.413591+0000 mon.a (mon.0) 966 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:42:47.748 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:47.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:47.751 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:42:47.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:47 smithi078 bash[78274]: cluster 2023-10-01T10:42:45.756058+0000 mgr.y (mgr.25072) 751 : cluster 0 pgmap v660: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 32 MiB/s wr, 885 op/s 2023-10-01T10:42:47.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:47 smithi078 bash[78274]: audit 2023-10-01T10:42:47.413591+0000 mon.a (mon.0) 966 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:42:47.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:47 smithi078 bash[75299]: cluster 2023-10-01T10:42:45.756058+0000 mgr.y (mgr.25072) 751 : cluster 0 pgmap v660: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 32 MiB/s wr, 885 op/s 2023-10-01T10:42:47.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:47 smithi078 bash[75299]: audit 2023-10-01T10:42:47.413591+0000 mon.a (mon.0) 966 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:42:47.851 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:47.874 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:47.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:47.882 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:42:47.983 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:48.009 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:48.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:48.011 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:48.112 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:48.154 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:48.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:48.160 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:42:48.260 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:48.298 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:48.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:48.304 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:42:48.304 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T10:42:48.347 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T10:42:48.394 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T10:42:48.405 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:48.431 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:48.434 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T10:42:48.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:48.435 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:42:48.454 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:48 smithi078 bash[75299]: audit 2023-10-01T10:42:48.246159+0000 mon.a (mon.0) 967 : audit 0 from='client.? 172.21.15.78:0/2003464719' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:42:48.476 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T10:42:48.518 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T10:42:48.535 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:48.573 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T10:42:48.579 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:48.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:48.583 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:42:48.609 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T10:42:48.615 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T10:42:48.615 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T10:42:48.615 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T10:42:48.615 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T10:42:48.615 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T10:42:48.615 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T10:42:48.615 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T10:42:48.615 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T10:42:48.684 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:48.694 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:48 smithi173 bash[58816]: audit 2023-10-01T10:42:48.246159+0000 mon.a (mon.0) 967 : audit 0 from='client.? 172.21.15.78:0/2003464719' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:42:48.711 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:48.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:48.722 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:48.811 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:48 smithi078 bash[78274]: audit 2023-10-01T10:42:48.246159+0000 mon.a (mon.0) 967 : audit 0 from='client.? 172.21.15.78:0/2003464719' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:42:48.824 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:48.863 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:48.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:48.869 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:42:48.969 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:49.004 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:49.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:49.007 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:49.108 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:49.134 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:49.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:49.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:42:49.239 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:49.278 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:49.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:49.281 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:42:49.382 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:49.406 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:49.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:49.410 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:42:49.480 INFO:teuthology.orchestra.run.smithi078.stderr:nodeep-scrub is set 2023-10-01T10:42:49.511 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:49.537 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:49.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:49.543 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:49.644 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:49.684 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:49.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:49.688 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:42:49.789 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:49.798 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:49 smithi173 bash[58816]: cluster 2023-10-01T10:42:47.756562+0000 mgr.y (mgr.25072) 752 : cluster 0 pgmap v661: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 20 MiB/s wr, 768 op/s 2023-10-01T10:42:49.798 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:49 smithi173 bash[58816]: audit 2023-10-01T10:42:48.643768+0000 mon.b (mon.1) 41 : audit 1 from='client.? 172.21.15.78:0/3839861263' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:42:49.798 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:49 smithi173 bash[58816]: audit 2023-10-01T10:42:48.644714+0000 mon.a (mon.0) 968 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:42:49.817 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:49.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:49.821 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:49.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:49 smithi078 bash[75299]: cluster 2023-10-01T10:42:47.756562+0000 mgr.y (mgr.25072) 752 : cluster 0 pgmap v661: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 20 MiB/s wr, 768 op/s 2023-10-01T10:42:49.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:49 smithi078 bash[75299]: audit 2023-10-01T10:42:48.643768+0000 mon.b (mon.1) 41 : audit 1 from='client.? 172.21.15.78:0/3839861263' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:42:49.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:49 smithi078 bash[75299]: audit 2023-10-01T10:42:48.644714+0000 mon.a (mon.0) 968 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:42:49.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:49 smithi078 bash[78274]: cluster 2023-10-01T10:42:47.756562+0000 mgr.y (mgr.25072) 752 : cluster 0 pgmap v661: 89 pgs: 89 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 20 MiB/s wr, 768 op/s 2023-10-01T10:42:49.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:49 smithi078 bash[78274]: audit 2023-10-01T10:42:48.643768+0000 mon.b (mon.1) 41 : audit 1 from='client.? 172.21.15.78:0/3839861263' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:42:49.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:49 smithi078 bash[78274]: audit 2023-10-01T10:42:48.644714+0000 mon.a (mon.0) 968 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:42:49.922 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:49.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:49.958 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:49.959 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:42:50.060 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:50.088 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:50.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:50.092 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:50.193 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:50.223 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:50.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:50.229 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:42:50.269 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:42:50 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:42:50] "GET /metrics HTTP/1.1" 200 36249 "" "Prometheus/2.33.4" 2023-10-01T10:42:50.329 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:50.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:50.354 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:50.354 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:42:50.455 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:50.491 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:50.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:50.498 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:42:50.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:50 smithi078 bash[78274]: cluster 2023-10-01T10:42:49.464313+0000 mon.a (mon.0) 969 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:42:50.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:50 smithi078 bash[78274]: cluster 2023-10-01T10:42:49.470474+0000 osd.5 (osd.5) 61 : cluster 3 failed to encode map e135 with expected crc 2023-10-01T10:42:50.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:50 smithi078 bash[78274]: audit 2023-10-01T10:42:49.472414+0000 mon.a (mon.0) 970 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T10:42:50.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:50 smithi078 bash[78274]: cluster 2023-10-01T10:42:49.472519+0000 mon.a (mon.0) 971 : cluster 0 osdmap e135: 8 total, 8 up, 8 in 2023-10-01T10:42:50.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:50 smithi078 bash[78274]: cluster 2023-10-01T10:42:49.479130+0000 osd.5 (osd.5) 62 : cluster 3 failed to encode map e135 with expected crc 2023-10-01T10:42:50.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:50 smithi078 bash[78274]: cluster 2023-10-01T10:42:49.480562+0000 osd.7 (osd.7) 81 : cluster 3 failed to encode map e135 with expected crc 2023-10-01T10:42:50.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:50 smithi078 bash[78274]: cluster 2023-10-01T10:42:49.492356+0000 osd.6 (osd.6) 49 : cluster 3 failed to encode map e135 with expected crc 2023-10-01T10:42:50.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:50 smithi078 bash[75299]: cluster 2023-10-01T10:42:49.464313+0000 mon.a (mon.0) 969 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:42:50.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:50 smithi078 bash[75299]: cluster 2023-10-01T10:42:49.470474+0000 osd.5 (osd.5) 61 : cluster 3 failed to encode map e135 with expected crc 2023-10-01T10:42:50.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:50 smithi078 bash[75299]: audit 2023-10-01T10:42:49.472414+0000 mon.a (mon.0) 970 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T10:42:50.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:50 smithi078 bash[75299]: cluster 2023-10-01T10:42:49.472519+0000 mon.a (mon.0) 971 : cluster 0 osdmap e135: 8 total, 8 up, 8 in 2023-10-01T10:42:50.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:50 smithi078 bash[75299]: cluster 2023-10-01T10:42:49.479130+0000 osd.5 (osd.5) 62 : cluster 3 failed to encode map e135 with expected crc 2023-10-01T10:42:50.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:50 smithi078 bash[75299]: cluster 2023-10-01T10:42:49.480562+0000 osd.7 (osd.7) 81 : cluster 3 failed to encode map e135 with expected crc 2023-10-01T10:42:50.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:50 smithi078 bash[75299]: cluster 2023-10-01T10:42:49.492356+0000 osd.6 (osd.6) 49 : cluster 3 failed to encode map e135 with expected crc 2023-10-01T10:42:50.598 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:50.619 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:50.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:50.624 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:50.724 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:50.749 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:50.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:50.754 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:42:50.855 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:50.864 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:50 smithi173 bash[58816]: cluster 2023-10-01T10:42:49.464313+0000 mon.a (mon.0) 969 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:42:50.864 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:50 smithi173 bash[58816]: cluster 2023-10-01T10:42:49.470474+0000 osd.5 (osd.5) 61 : cluster 3 failed to encode map e135 with expected crc 2023-10-01T10:42:50.864 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:50 smithi173 bash[58816]: audit 2023-10-01T10:42:49.472414+0000 mon.a (mon.0) 970 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T10:42:50.864 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:50 smithi173 bash[58816]: cluster 2023-10-01T10:42:49.472519+0000 mon.a (mon.0) 971 : cluster 0 osdmap e135: 8 total, 8 up, 8 in 2023-10-01T10:42:50.864 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:50 smithi173 bash[58816]: cluster 2023-10-01T10:42:49.479130+0000 osd.5 (osd.5) 62 : cluster 3 failed to encode map e135 with expected crc 2023-10-01T10:42:50.864 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:50 smithi173 bash[58816]: cluster 2023-10-01T10:42:49.480562+0000 osd.7 (osd.7) 81 : cluster 3 failed to encode map e135 with expected crc 2023-10-01T10:42:50.864 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:50 smithi173 bash[58816]: cluster 2023-10-01T10:42:49.492356+0000 osd.6 (osd.6) 49 : cluster 3 failed to encode map e135 with expected crc 2023-10-01T10:42:50.887 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:50.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:51.022 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:42:51.122 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:51.183 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:51.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:51.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:51.287 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:51.314 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:42:51.323 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:51.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:51.326 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:42:51.427 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:51.461 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:51.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:51.464 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:51.565 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:51.573 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:51 smithi078 bash[78274]: cluster 2023-10-01T10:42:49.757000+0000 mgr.y (mgr.25072) 753 : cluster 0 pgmap v663: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 7.7 MiB/s wr, 813 op/s 2023-10-01T10:42:51.574 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:51 smithi078 bash[75299]: cluster 2023-10-01T10:42:49.757000+0000 mgr.y (mgr.25072) 753 : cluster 0 pgmap v663: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 7.7 MiB/s wr, 813 op/s 2023-10-01T10:42:51.588 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:51.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:51.591 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:51.692 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:51.716 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:51.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:51.724 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:51.825 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:51.836 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:51 smithi173 bash[58816]: cluster 2023-10-01T10:42:49.757000+0000 mgr.y (mgr.25072) 753 : cluster 0 pgmap v663: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 7.7 MiB/s wr, 813 op/s 2023-10-01T10:42:51.859 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:51.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:51.861 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:42:51.962 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:52.017 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:52.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:52.020 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:42:52.046 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T10:42:52.121 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:52.161 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:52.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:52.164 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:42:52.265 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:52.298 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:52.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:52.303 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:52.404 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:52.429 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:52.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:52.432 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:42:52.533 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:52.564 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:52.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:52.567 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:42:52.627 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T10:42:52.627 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-01T10:42:52.668 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:52.693 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:52.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:52.699 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:42:52.799 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:52.842 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:52.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:52.844 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:42:52.888 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T10:42:52.945 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:52.965 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:52.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:52.968 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:42:53.069 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:53.093 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:53.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:53.096 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:53.197 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:53.204 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:53 smithi173 bash[58816]: cluster 2023-10-01T10:42:51.757666+0000 mgr.y (mgr.25072) 754 : cluster 0 pgmap v664: 89 pgs: 89 active+clean; 6.3 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 804 op/s 2023-10-01T10:42:53.204 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:53 smithi173 bash[58816]: audit 2023-10-01T10:42:52.192588+0000 mon.a (mon.0) 972 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:42:53.205 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:53 smithi173 bash[58816]: audit 2023-10-01T10:42:52.203794+0000 mon.a (mon.0) 973 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:42:53.219 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:53 smithi078 bash[75299]: cluster 2023-10-01T10:42:51.757666+0000 mgr.y (mgr.25072) 754 : cluster 0 pgmap v664: 89 pgs: 89 active+clean; 6.3 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 804 op/s 2023-10-01T10:42:53.219 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:53 smithi078 bash[75299]: audit 2023-10-01T10:42:52.192588+0000 mon.a (mon.0) 972 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:42:53.220 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:53 smithi078 bash[75299]: audit 2023-10-01T10:42:52.203794+0000 mon.a (mon.0) 973 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:42:53.220 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:53 smithi078 bash[78274]: cluster 2023-10-01T10:42:51.757666+0000 mgr.y (mgr.25072) 754 : cluster 0 pgmap v664: 89 pgs: 89 active+clean; 6.3 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 804 op/s 2023-10-01T10:42:53.220 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:53 smithi078 bash[78274]: audit 2023-10-01T10:42:52.192588+0000 mon.a (mon.0) 972 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:42:53.220 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:53 smithi078 bash[78274]: audit 2023-10-01T10:42:52.203794+0000 mon.a (mon.0) 973 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:42:53.230 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:53.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:53.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:42:53.338 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:53.360 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:53.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:53.362 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:53.463 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:53.499 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:53.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:53.505 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:42:53.606 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:53.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:53.637 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:53.637 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:53.738 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:53.769 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:53.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:53.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:42:53.875 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:53.924 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:53.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:53.929 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:54.029 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:54.067 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:54.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:54.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:42:54.175 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:54.197 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:54.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:54.200 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:54.300 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:54.327 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:54.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:54.336 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:42:54.437 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:54.473 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:54.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:54.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:54.584 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:54.610 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:54.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:54.614 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:54.715 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:54.741 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:54.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:54.745 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:54.846 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:54.876 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:54.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:54.883 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:42:54.983 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:55.007 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:55.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:55.009 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:42:55.110 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:55.137 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:55.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:55.142 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:55.243 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:55.277 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:55.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:55.283 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:55.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:55 smithi078 bash[75299]: cluster 2023-10-01T10:42:53.758276+0000 mgr.y (mgr.25072) 755 : cluster 0 pgmap v665: 89 pgs: 89 active+clean; 6.2 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 798 op/s 2023-10-01T10:42:55.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:55 smithi078 bash[78274]: cluster 2023-10-01T10:42:53.758276+0000 mgr.y (mgr.25072) 755 : cluster 0 pgmap v665: 89 pgs: 89 active+clean; 6.2 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 798 op/s 2023-10-01T10:42:55.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:55 smithi173 bash[58816]: cluster 2023-10-01T10:42:53.758276+0000 mgr.y (mgr.25072) 755 : cluster 0 pgmap v665: 89 pgs: 89 active+clean; 6.2 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 798 op/s 2023-10-01T10:42:55.384 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:55.420 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:55.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:55.423 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:42:55.523 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:55.574 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:55.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:55.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:42:55.679 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2023-10-01T10:42:55.680 INFO:teuthology.orchestra.run.smithi078.stdout: "ops": [ 2023-10-01T10:42:55.680 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:42:55.680 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.35649.0:125462 9.2 e135/128 9:42d1f54c:::benchmark_data_smithi078_113121_object13647:head v 135'7971, mlcod=135'7969)", 2023-10-01T10:42:55.680 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:42:55.675473+0000", 2023-10-01T10:42:55.680 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 0.0026516550000000002, 2023-10-01T10:42:55.680 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0026850870000000001, 2023-10-01T10:42:55.680 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:42:55.680 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "started", 2023-10-01T10:42:55.680 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:42:55.680 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:42:55.680 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:42:55.681 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:55.675473+0000", 2023-10-01T10:42:55.681 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:42:55.681 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:42:55.681 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:42:55.681 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:42:55.681 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:55.675473+0000", 2023-10-01T10:42:55.681 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:42:55.681 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:42:55.681 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:42:55.681 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:42:55.681 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:55.675471+0000", 2023-10-01T10:42:55.681 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:42:55.682 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:42:55.682 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:42:55.682 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:42:55.682 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:55.675481+0000", 2023-10-01T10:42:55.682 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.2280000000000004e-06 2023-10-01T10:42:55.682 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:42:55.682 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:42:55.682 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:42:55.682 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:55.675496+0000", 2023-10-01T10:42:55.682 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.5596000000000001e-05 2023-10-01T10:42:55.682 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:42:55.682 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:42:55.683 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:42:55.683 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:55.675505+0000", 2023-10-01T10:42:55.683 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.8810000000000002e-06 2023-10-01T10:42:55.683 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:42:55.683 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:42:55.683 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:42:55.683 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:55.675551+0000", 2023-10-01T10:42:55.683 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.5772000000000002e-05 2023-10-01T10:42:55.683 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:42:55.683 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:42:55.683 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:42:55.683 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:55.675574+0000", 2023-10-01T10:42:55.684 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.3051999999999999e-05 2023-10-01T10:42:55.684 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:42:55.684 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:42:55.684 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:42:55.684 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:42:55.684 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2023-10-01T10:42:55.684 INFO:teuthology.orchestra.run.smithi078.stdout: "num_ops": 1 2023-10-01T10:42:55.684 INFO:teuthology.orchestra.run.smithi078.stdout:} 2023-10-01T10:42:55.684 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:55.721 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:55.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:55.729 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:42:55.830 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:55.867 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:55.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:55.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:42:55.972 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:56.000 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:56.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:56.005 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:56.105 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:56.134 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:56.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:56.137 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:56.237 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:56.260 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:56 smithi173 bash[58816]: audit 2023-10-01T10:42:55.256722+0000 mon.a (mon.0) 974 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:42:56.264 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:42:56.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:56.269 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:42:56.369 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:56.378 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:56 smithi078 bash[78274]: audit 2023-10-01T10:42:55.256722+0000 mon.a (mon.0) 974 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:42:56.378 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:56 smithi078 bash[78274]: audit 2023-10-01T10:42:55.265011+0000 mon.a (mon.0) 975 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:42:56.378 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:56 smithi078 bash[78274]: audit 2023-10-01T10:42:55.942915+0000 mon.a (mon.0) 976 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:42:56.378 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:56 smithi078 bash[78274]: audit 2023-10-01T10:42:55.943884+0000 mon.a (mon.0) 977 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:42:56.378 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:56 smithi078 bash[78274]: audit 2023-10-01T10:42:55.952861+0000 mon.a (mon.0) 978 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:42:56.379 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:56 smithi078 bash[75299]: audit 2023-10-01T10:42:55.256722+0000 mon.a (mon.0) 974 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:42:56.379 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:56 smithi078 bash[75299]: audit 2023-10-01T10:42:55.265011+0000 mon.a (mon.0) 975 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:42:56.379 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:56 smithi078 bash[75299]: audit 2023-10-01T10:42:55.942915+0000 mon.a (mon.0) 976 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:42:56.379 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:56 smithi078 bash[75299]: audit 2023-10-01T10:42:55.943884+0000 mon.a (mon.0) 977 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:42:56.379 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:56 smithi078 bash[75299]: audit 2023-10-01T10:42:55.952861+0000 mon.a (mon.0) 978 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:42:56.393 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:56.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:56.397 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:42:56.498 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:56.519 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:56.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:56.525 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:56.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:56 smithi173 bash[58816]: audit 2023-10-01T10:42:55.265011+0000 mon.a (mon.0) 975 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:42:56.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:56 smithi173 bash[58816]: audit 2023-10-01T10:42:55.942915+0000 mon.a (mon.0) 976 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:42:56.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:56 smithi173 bash[58816]: audit 2023-10-01T10:42:55.943884+0000 mon.a (mon.0) 977 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:42:56.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:56 smithi173 bash[58816]: audit 2023-10-01T10:42:55.952861+0000 mon.a (mon.0) 978 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:42:56.626 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:56.654 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:56.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:56.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:42:56.710 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-01T10:42:56.760 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:56.788 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:56.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:56.791 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:56.892 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:56.922 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:56.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:56.926 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:42:57.027 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:57.058 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:57.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:57.063 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:42:57.164 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:57.187 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:57.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:57.196 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:42:57.208 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:42:57.296 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:57.305 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:57 smithi173 bash[58816]: cluster 2023-10-01T10:42:55.762556+0000 mgr.y (mgr.25072) 756 : cluster 0 pgmap v666: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 765 op/s 2023-10-01T10:42:57.329 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:57.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:57.330 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:42:57.431 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:57.461 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:57.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:57.468 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:42:57.569 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:57.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:57 smithi078 bash[78274]: cluster 2023-10-01T10:42:55.762556+0000 mgr.y (mgr.25072) 756 : cluster 0 pgmap v666: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 765 op/s 2023-10-01T10:42:57.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:57 smithi078 bash[75299]: cluster 2023-10-01T10:42:55.762556+0000 mgr.y (mgr.25072) 756 : cluster 0 pgmap v666: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 765 op/s 2023-10-01T10:42:57.599 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:57.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:57.602 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:57.702 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:57.732 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:57.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:57.739 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:57.839 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:57.870 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:57.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:57.876 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:57.977 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:58.027 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:42:58.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:58.031 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:42:58.132 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:58.159 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:58.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:58.165 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:42:58.225 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:42:58.225 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:42:58.266 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:58.309 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:58.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:58.317 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:58.418 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:58.444 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:42:58.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:58.449 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:42:58.549 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:58.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:58.576 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:58.576 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:42:58.677 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:58.709 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:58.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:58.715 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:42:58.816 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:58.840 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:58.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:58.843 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:42:58.944 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:58.971 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:42:58.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:58.974 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:42:59.075 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:59.104 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:59.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:59.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:42:59.208 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:59.232 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:42:59.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:59.234 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:42:59.335 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:59.351 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:42:59 smithi078 bash[75299]: cluster 2023-10-01T10:42:57.763143+0000 mgr.y (mgr.25072) 757 : cluster 0 pgmap v667: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 761 op/s 2023-10-01T10:42:59.362 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:59.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:59.367 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:42:59.468 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:59.488 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:42:59.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:59.491 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:42:59.591 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:59.605 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:42:59 smithi173 bash[58816]: cluster 2023-10-01T10:42:57.763143+0000 mgr.y (mgr.25072) 757 : cluster 0 pgmap v667: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 761 op/s 2023-10-01T10:42:59.621 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:42:59 smithi078 bash[78274]: cluster 2023-10-01T10:42:57.763143+0000 mgr.y (mgr.25072) 757 : cluster 0 pgmap v667: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 761 op/s 2023-10-01T10:42:59.623 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:59.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:59.633 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:42:59.734 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:59.766 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:42:59.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:59.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:42:59.872 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:59.905 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:42:59.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:42:59.908 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:43:00.009 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:00.038 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:00.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:00.043 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:43:00.144 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:00.178 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:00.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:00.184 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:43:00.284 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:00.327 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:00.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:00.330 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:43:00.431 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:00.450 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:00.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:00.452 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:00.553 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:00.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:43:00 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:43:00] "GET /metrics HTTP/1.1" 200 36249 "" "Prometheus/2.33.4" 2023-10-01T10:43:00.584 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:00.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:00.586 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:43:00.687 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:00.720 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:00.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:00.724 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:43:00.825 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:00.858 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:00.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:00.862 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:43:00.962 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:00.989 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:00.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:00.991 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:43:01.092 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:01.128 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:01.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:01.135 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:43:01.236 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:01.262 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:01.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:01.265 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:43:01.278 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:43:01.366 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:01.393 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:01.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:01.402 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:43:01.503 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:01.539 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:01.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:01.543 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:43:01.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:01 smithi078 bash[75299]: cluster 2023-10-01T10:42:59.763807+0000 mgr.y (mgr.25072) 758 : cluster 0 pgmap v668: 89 pgs: 89 active+clean; 5.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 742 op/s 2023-10-01T10:43:01.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:01 smithi078 bash[78274]: cluster 2023-10-01T10:42:59.763807+0000 mgr.y (mgr.25072) 758 : cluster 0 pgmap v668: 89 pgs: 89 active+clean; 5.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 742 op/s 2023-10-01T10:43:01.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:01 smithi173 bash[58816]: cluster 2023-10-01T10:42:59.763807+0000 mgr.y (mgr.25072) 758 : cluster 0 pgmap v668: 89 pgs: 89 active+clean; 5.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 742 op/s 2023-10-01T10:43:01.644 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:01.683 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:01.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:01.689 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:01.790 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:01.812 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:01.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:01.815 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:43:01.916 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:01.938 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:01.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:01.944 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:02.044 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:02.074 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:02.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:02.081 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:43:02.181 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:02.222 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:02.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:02.223 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:43:02.324 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:02.351 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:02.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:02.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:43:02.455 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:02.488 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:02.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:02.494 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:43:02.595 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:02.618 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:02.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:02.622 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:43:02.723 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:02.751 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:02.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:02.754 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:02.855 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:02.880 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:02.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:02.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:43:02.984 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:03.011 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:03.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:03.014 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:43:03.114 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:03.137 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:03.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:03.141 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:43:03.242 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:03.276 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:03.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:03.283 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:43:03.383 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:03.424 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:03.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:03.430 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:43:03.531 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:03.540 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:03 smithi078 bash[75299]: cluster 2023-10-01T10:43:01.764492+0000 mgr.y (mgr.25072) 759 : cluster 0 pgmap v669: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 809 op/s 2023-10-01T10:43:03.540 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:03 smithi078 bash[75299]: audit 2023-10-01T10:43:02.414217+0000 mon.a (mon.0) 979 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:43:03.540 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:03 smithi078 bash[78274]: cluster 2023-10-01T10:43:01.764492+0000 mgr.y (mgr.25072) 759 : cluster 0 pgmap v669: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 809 op/s 2023-10-01T10:43:03.540 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:03 smithi078 bash[78274]: audit 2023-10-01T10:43:02.414217+0000 mon.a (mon.0) 979 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:43:03.564 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:03.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:03.566 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:03.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:03 smithi173 bash[58816]: cluster 2023-10-01T10:43:01.764492+0000 mgr.y (mgr.25072) 759 : cluster 0 pgmap v669: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 809 op/s 2023-10-01T10:43:03.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:03 smithi173 bash[58816]: audit 2023-10-01T10:43:02.414217+0000 mon.a (mon.0) 979 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:43:03.667 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:03.700 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:03.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:03.704 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:43:03.805 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:03.839 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:03.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:03.844 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:43:03.945 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:03.973 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:03.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:03.980 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:43:04.081 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:04.113 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:04.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:04.118 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:43:04.218 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:04.253 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:04.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:04.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:43:04.360 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:04.391 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:04.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:04.397 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:43:04.498 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:04.521 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:04.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:04.524 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:43:04.625 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:04.660 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:04.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:04.675 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:43:04.775 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:04.957 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:04.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:04.960 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:43:05.061 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:05.089 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:05.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:05.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:05.192 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:05.220 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:05.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:05.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:43:05.325 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:05.351 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:05.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:05.356 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:05.457 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:05.466 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:05 smithi173 bash[58816]: cluster 2023-10-01T10:43:03.765087+0000 mgr.y (mgr.25072) 760 : cluster 0 pgmap v670: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 675 op/s 2023-10-01T10:43:05.488 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:05.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:05.491 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:43:05.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:05 smithi078 bash[75299]: cluster 2023-10-01T10:43:03.765087+0000 mgr.y (mgr.25072) 760 : cluster 0 pgmap v670: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 675 op/s 2023-10-01T10:43:05.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:05 smithi078 bash[78274]: cluster 2023-10-01T10:43:03.765087+0000 mgr.y (mgr.25072) 760 : cluster 0 pgmap v670: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 675 op/s 2023-10-01T10:43:05.591 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:05.618 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:05.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:05.623 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:43:05.724 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:05.773 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:05.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:05.776 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:43:05.876 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:05.908 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:05.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:05.913 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:43:06.014 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:06.047 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:06.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:06.051 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:43:06.151 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:06.179 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:06.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:06.183 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:43:06.284 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:06.315 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:06.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:06.319 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:06.419 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:06.448 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:06.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:06.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:43:06.551 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:06.588 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:06.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:06.591 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:43:06.693 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:06.735 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:06.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:06.737 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:43:06.838 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:06.878 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:06.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:06.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:43:06.982 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:07.008 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:07.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:07.013 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:43:07.113 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:07.138 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:07.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:07.140 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:43:07.241 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:07.277 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:07.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:07.280 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:07.381 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:07.389 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:07 smithi173 bash[58816]: cluster 2023-10-01T10:43:05.765725+0000 mgr.y (mgr.25072) 761 : cluster 0 pgmap v671: 89 pgs: 89 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 792 op/s 2023-10-01T10:43:07.415 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:07.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:07.418 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:43:07.519 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:07.527 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:07 smithi078 bash[78274]: cluster 2023-10-01T10:43:05.765725+0000 mgr.y (mgr.25072) 761 : cluster 0 pgmap v671: 89 pgs: 89 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 792 op/s 2023-10-01T10:43:07.528 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:07 smithi078 bash[75299]: cluster 2023-10-01T10:43:05.765725+0000 mgr.y (mgr.25072) 761 : cluster 0 pgmap v671: 89 pgs: 89 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 792 op/s 2023-10-01T10:43:07.540 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:07.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:07.542 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:43:07.643 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:07.776 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:07.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:07.779 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:43:07.880 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:07.913 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:07.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:07.916 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:43:08.016 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:08.048 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:08.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:08.051 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:43:08.152 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:08.178 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:08.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:08.182 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:43:08.283 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:08.312 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:08.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:08.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:43:08.419 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:08.445 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:08.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:08.448 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:43:08.549 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:08.581 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:08.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:08.584 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:08.685 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:08.716 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:08.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:08.718 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:43:08.819 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:08.860 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:08.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:08.863 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:43:08.964 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:08.988 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:08.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:08.992 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:43:09.093 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:09.121 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:09.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:09.124 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:09.224 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:09.250 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:09.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:09.256 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:43:09.356 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:09.376 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:09 smithi078 bash[75299]: cluster 2023-10-01T10:43:07.766239+0000 mgr.y (mgr.25072) 762 : cluster 0 pgmap v672: 89 pgs: 89 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 706 op/s 2023-10-01T10:43:09.387 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:09.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:09.392 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:43:09.493 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:09.519 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:09.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:09.524 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:43:09.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:09 smithi173 bash[58816]: cluster 2023-10-01T10:43:07.766239+0000 mgr.y (mgr.25072) 762 : cluster 0 pgmap v672: 89 pgs: 89 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 706 op/s 2023-10-01T10:43:09.625 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:09.633 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:09 smithi078 bash[78274]: cluster 2023-10-01T10:43:07.766239+0000 mgr.y (mgr.25072) 762 : cluster 0 pgmap v672: 89 pgs: 89 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 706 op/s 2023-10-01T10:43:09.654 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:09.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:09.660 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:43:09.761 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:09.792 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:09.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:09.794 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:43:09.895 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:09.928 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:09.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:09.933 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:43:10.033 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:10.096 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:10.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:10.098 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:43:10.198 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:10.233 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:10.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:10.236 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:43:10.337 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:10.369 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:10.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:10.375 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:43:10.476 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:10.501 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:10.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:10.507 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:43:10.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:43:10 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:43:10] "GET /metrics HTTP/1.1" 200 36251 "" "Prometheus/2.33.4" 2023-10-01T10:43:10.608 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:10.637 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:10.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:10.646 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:43:10.746 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:10.770 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:10.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:10.775 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:43:10.876 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:10.902 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:10.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:10.905 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:11.006 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:11.041 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:11.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:11.049 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:43:11.150 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:11.182 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:11.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:11.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:43:11.287 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:11.328 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:11.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:11.332 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:43:11.433 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:11.459 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:11.463 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:11 smithi078 bash[78274]: cluster 2023-10-01T10:43:09.769432+0000 mgr.y (mgr.25072) 763 : cluster 0 pgmap v673: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 772 op/s 2023-10-01T10:43:11.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:11.466 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:11 smithi078 bash[75299]: cluster 2023-10-01T10:43:09.769432+0000 mgr.y (mgr.25072) 763 : cluster 0 pgmap v673: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 772 op/s 2023-10-01T10:43:11.467 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:43:11.494 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T10:43:11.537 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T10:43:11.568 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:11.586 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T10:43:11.595 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:11.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:11.600 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:11.622 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T10:43:11.656 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T10:43:11.690 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T10:43:11.702 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:11.723 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T10:43:11.733 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:11.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:11.736 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:11.754 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T10:43:11.757 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T10:43:11.757 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T10:43:11.757 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T10:43:11.757 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T10:43:11.757 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T10:43:11.758 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T10:43:11.758 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T10:43:11.758 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T10:43:11.836 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:11.864 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:11.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:11.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:43:11.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:11 smithi173 bash[58816]: cluster 2023-10-01T10:43:09.769432+0000 mgr.y (mgr.25072) 763 : cluster 0 pgmap v673: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 772 op/s 2023-10-01T10:43:11.968 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:11.992 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:11.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:11.998 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:43:12.099 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:12.135 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:12.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:12.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:12.239 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:12.387 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:12 smithi078 bash[75299]: audit 2023-10-01T10:43:11.448053+0000 mon.a (mon.0) 980 : audit 0 from='client.? 172.21.15.78:0/595728736' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:43:12.539 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:12.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:12.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:43:12.643 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:12.652 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:12 smithi173 bash[58816]: audit 2023-10-01T10:43:11.448053+0000 mon.a (mon.0) 980 : audit 0 from='client.? 172.21.15.78:0/595728736' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:43:12.677 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:12.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:12.684 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:43:12.717 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:12 smithi078 bash[78274]: audit 2023-10-01T10:43:11.448053+0000 mon.a (mon.0) 980 : audit 0 from='client.? 172.21.15.78:0/595728736' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:43:12.785 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:12.814 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:12.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:12.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:43:12.920 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:12.949 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:12.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:12.954 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:43:13.055 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:13.077 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:13.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:13.079 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:43:13.179 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:13.212 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:13.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:13.215 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:43:13.315 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:13.345 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:13.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:13.350 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:43:13.406 INFO:teuthology.orchestra.run.smithi078.stderr:reweighted osd.0 to 1 (10000) 2023-10-01T10:43:13.446 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:13 smithi078 bash[75299]: cluster 2023-10-01T10:43:11.771271+0000 mgr.y (mgr.25072) 764 : cluster 0 pgmap v674: 89 pgs: 89 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 802 op/s 2023-10-01T10:43:13.446 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:13 smithi078 bash[75299]: audit 2023-10-01T10:43:13.254478+0000 mon.c (mon.2) 138 : audit 1 from='client.? 172.21.15.78:0/2209016055' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T10:43:13.446 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:13 smithi078 bash[75299]: audit 2023-10-01T10:43:13.254787+0000 mon.a (mon.0) 981 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T10:43:13.447 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:13 smithi078 bash[78274]: cluster 2023-10-01T10:43:11.771271+0000 mgr.y (mgr.25072) 764 : cluster 0 pgmap v674: 89 pgs: 89 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 802 op/s 2023-10-01T10:43:13.447 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:13 smithi078 bash[78274]: audit 2023-10-01T10:43:13.254478+0000 mon.c (mon.2) 138 : audit 1 from='client.? 172.21.15.78:0/2209016055' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T10:43:13.447 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:13 smithi078 bash[78274]: audit 2023-10-01T10:43:13.254787+0000 mon.a (mon.0) 981 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T10:43:13.451 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:13.477 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:13.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:13.479 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:43:13.580 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:13.609 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:13.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:13.614 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:43:13.714 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:13.737 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:13.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:13.741 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:13.842 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:13.850 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:13 smithi173 bash[58816]: cluster 2023-10-01T10:43:11.771271+0000 mgr.y (mgr.25072) 764 : cluster 0 pgmap v674: 89 pgs: 89 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 802 op/s 2023-10-01T10:43:13.850 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:13 smithi173 bash[58816]: audit 2023-10-01T10:43:13.254478+0000 mon.c (mon.2) 138 : audit 1 from='client.? 172.21.15.78:0/2209016055' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T10:43:13.850 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:13 smithi173 bash[58816]: audit 2023-10-01T10:43:13.254787+0000 mon.a (mon.0) 981 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T10:43:13.880 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:13.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:13.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:43:13.984 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:14.007 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:14.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:14.017 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:43:14.118 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:14.145 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:14.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:14.149 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:43:14.250 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:14.274 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:14.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:14.279 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:43:14.380 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:14.404 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:14.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:14.406 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:43:14.422 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:14 smithi078 bash[75299]: cluster 2023-10-01T10:43:13.396106+0000 osd.7 (osd.7) 82 : cluster 3 failed to encode map e136 with expected crc 2023-10-01T10:43:14.422 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:14 smithi078 bash[75299]: audit 2023-10-01T10:43:13.399124+0000 mon.a (mon.0) 982 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-01T10:43:14.422 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:14 smithi078 bash[75299]: cluster 2023-10-01T10:43:13.399407+0000 mon.a (mon.0) 983 : cluster 0 osdmap e136: 8 total, 8 up, 8 in 2023-10-01T10:43:14.422 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:14 smithi078 bash[75299]: cluster 2023-10-01T10:43:13.402009+0000 osd.6 (osd.6) 50 : cluster 3 failed to encode map e136 with expected crc 2023-10-01T10:43:14.422 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:14 smithi078 bash[75299]: cluster 2023-10-01T10:43:13.407587+0000 osd.5 (osd.5) 63 : cluster 3 failed to encode map e136 with expected crc 2023-10-01T10:43:14.422 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:14 smithi078 bash[75299]: audit 2023-10-01T10:43:13.880368+0000 mon.c (mon.2) 139 : audit 1 from='client.? 172.21.15.78:0/2827486589' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T10:43:14.422 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:14 smithi078 bash[75299]: audit 2023-10-01T10:43:13.880783+0000 mon.a (mon.0) 984 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T10:43:14.423 INFO:teuthology.orchestra.run.smithi078.stderr:noscrub is unset 2023-10-01T10:43:14.507 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:14.518 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:14 smithi173 bash[58816]: cluster 2023-10-01T10:43:13.396106+0000 osd.7 (osd.7) 82 : cluster 3 failed to encode map e136 with expected crc 2023-10-01T10:43:14.519 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:14 smithi173 bash[58816]: audit 2023-10-01T10:43:13.399124+0000 mon.a (mon.0) 982 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-01T10:43:14.519 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:14 smithi173 bash[58816]: cluster 2023-10-01T10:43:13.399407+0000 mon.a (mon.0) 983 : cluster 0 osdmap e136: 8 total, 8 up, 8 in 2023-10-01T10:43:14.519 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:14 smithi173 bash[58816]: cluster 2023-10-01T10:43:13.402009+0000 osd.6 (osd.6) 50 : cluster 3 failed to encode map e136 with expected crc 2023-10-01T10:43:14.519 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:14 smithi173 bash[58816]: cluster 2023-10-01T10:43:13.407587+0000 osd.5 (osd.5) 63 : cluster 3 failed to encode map e136 with expected crc 2023-10-01T10:43:14.519 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:14 smithi173 bash[58816]: audit 2023-10-01T10:43:13.880368+0000 mon.c (mon.2) 139 : audit 1 from='client.? 172.21.15.78:0/2827486589' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T10:43:14.519 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:14 smithi173 bash[58816]: audit 2023-10-01T10:43:13.880783+0000 mon.a (mon.0) 984 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T10:43:14.538 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:14.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:14.547 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:43:14.648 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:14.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:14.681 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:14 smithi078 bash[78274]: cluster 2023-10-01T10:43:13.396106+0000 osd.7 (osd.7) 82 : cluster 3 failed to encode map e136 with expected crc 2023-10-01T10:43:14.681 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:14 smithi078 bash[78274]: audit 2023-10-01T10:43:13.399124+0000 mon.a (mon.0) 982 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-01T10:43:14.681 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:14 smithi078 bash[78274]: cluster 2023-10-01T10:43:13.399407+0000 mon.a (mon.0) 983 : cluster 0 osdmap e136: 8 total, 8 up, 8 in 2023-10-01T10:43:14.682 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:14 smithi078 bash[78274]: cluster 2023-10-01T10:43:13.402009+0000 osd.6 (osd.6) 50 : cluster 3 failed to encode map e136 with expected crc 2023-10-01T10:43:14.682 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:14 smithi078 bash[78274]: cluster 2023-10-01T10:43:13.407587+0000 osd.5 (osd.5) 63 : cluster 3 failed to encode map e136 with expected crc 2023-10-01T10:43:14.682 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:14 smithi078 bash[78274]: audit 2023-10-01T10:43:13.880368+0000 mon.c (mon.2) 139 : audit 1 from='client.? 172.21.15.78:0/2827486589' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T10:43:14.682 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:14 smithi078 bash[78274]: audit 2023-10-01T10:43:13.880783+0000 mon.a (mon.0) 984 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T10:43:14.682 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:14.683 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:43:14.783 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:14.826 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:14.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:14.828 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:43:14.929 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:14.957 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:14.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:14.961 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:15.062 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:15.087 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:15.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:15.095 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:43:15.195 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:15.227 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:15.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:15.229 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:43:15.329 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:15.350 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:15.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:15.353 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:15.454 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:15.481 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:15.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:15.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:43:15.584 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:15.593 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:15 smithi078 bash[78274]: cluster 2023-10-01T10:43:13.771816+0000 mgr.y (mgr.25072) 765 : cluster 0 pgmap v676: 89 pgs: 89 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 762 op/s 2023-10-01T10:43:15.593 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:15 smithi078 bash[78274]: cluster 2023-10-01T10:43:14.404828+0000 mon.a (mon.0) 985 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:43:15.593 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:15 smithi078 bash[78274]: audit 2023-10-01T10:43:14.409937+0000 mon.a (mon.0) 986 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T10:43:15.594 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:15 smithi078 bash[78274]: cluster 2023-10-01T10:43:14.410143+0000 mon.a (mon.0) 987 : cluster 0 osdmap e137: 8 total, 8 up, 8 in 2023-10-01T10:43:15.594 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:15 smithi078 bash[78274]: cluster 2023-10-01T10:43:14.413189+0000 osd.6 (osd.6) 51 : cluster 3 failed to encode map e137 with expected crc 2023-10-01T10:43:15.594 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:15 smithi078 bash[78274]: cluster 2023-10-01T10:43:14.419229+0000 osd.7 (osd.7) 83 : cluster 3 failed to encode map e137 with expected crc 2023-10-01T10:43:15.594 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:15 smithi078 bash[78274]: cluster 2023-10-01T10:43:14.419645+0000 osd.5 (osd.5) 64 : cluster 3 failed to encode map e137 with expected crc 2023-10-01T10:43:15.594 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:15 smithi078 bash[78274]: cluster 2023-10-01T10:43:14.420986+0000 osd.7 (osd.7) 84 : cluster 3 failed to encode map e137 with expected crc 2023-10-01T10:43:15.594 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:15 smithi078 bash[78274]: cluster 2023-10-01T10:43:14.422065+0000 osd.5 (osd.5) 65 : cluster 3 failed to encode map e137 with expected crc 2023-10-01T10:43:15.594 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:15 smithi078 bash[75299]: cluster 2023-10-01T10:43:13.771816+0000 mgr.y (mgr.25072) 765 : cluster 0 pgmap v676: 89 pgs: 89 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 762 op/s 2023-10-01T10:43:15.595 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:15 smithi078 bash[75299]: cluster 2023-10-01T10:43:14.404828+0000 mon.a (mon.0) 985 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:43:15.595 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:15 smithi078 bash[75299]: audit 2023-10-01T10:43:14.409937+0000 mon.a (mon.0) 986 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T10:43:15.595 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:15 smithi078 bash[75299]: cluster 2023-10-01T10:43:14.410143+0000 mon.a (mon.0) 987 : cluster 0 osdmap e137: 8 total, 8 up, 8 in 2023-10-01T10:43:15.595 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:15 smithi078 bash[75299]: cluster 2023-10-01T10:43:14.413189+0000 osd.6 (osd.6) 51 : cluster 3 failed to encode map e137 with expected crc 2023-10-01T10:43:15.595 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:15 smithi078 bash[75299]: cluster 2023-10-01T10:43:14.419229+0000 osd.7 (osd.7) 83 : cluster 3 failed to encode map e137 with expected crc 2023-10-01T10:43:15.595 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:15 smithi078 bash[75299]: cluster 2023-10-01T10:43:14.419645+0000 osd.5 (osd.5) 64 : cluster 3 failed to encode map e137 with expected crc 2023-10-01T10:43:15.595 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:15 smithi078 bash[75299]: cluster 2023-10-01T10:43:14.420986+0000 osd.7 (osd.7) 84 : cluster 3 failed to encode map e137 with expected crc 2023-10-01T10:43:15.595 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:15 smithi078 bash[75299]: cluster 2023-10-01T10:43:14.422065+0000 osd.5 (osd.5) 65 : cluster 3 failed to encode map e137 with expected crc 2023-10-01T10:43:15.615 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:15.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:15.618 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:15.721 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:15.761 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:15.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:15.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:43:15.864 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:15.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:15 smithi173 bash[58816]: cluster 2023-10-01T10:43:13.771816+0000 mgr.y (mgr.25072) 765 : cluster 0 pgmap v676: 89 pgs: 89 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 762 op/s 2023-10-01T10:43:15.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:15 smithi173 bash[58816]: cluster 2023-10-01T10:43:14.404828+0000 mon.a (mon.0) 985 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:43:15.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:15 smithi173 bash[58816]: audit 2023-10-01T10:43:14.409937+0000 mon.a (mon.0) 986 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T10:43:15.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:15 smithi173 bash[58816]: cluster 2023-10-01T10:43:14.410143+0000 mon.a (mon.0) 987 : cluster 0 osdmap e137: 8 total, 8 up, 8 in 2023-10-01T10:43:15.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:15 smithi173 bash[58816]: cluster 2023-10-01T10:43:14.413189+0000 osd.6 (osd.6) 51 : cluster 3 failed to encode map e137 with expected crc 2023-10-01T10:43:15.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:15 smithi173 bash[58816]: cluster 2023-10-01T10:43:14.419229+0000 osd.7 (osd.7) 83 : cluster 3 failed to encode map e137 with expected crc 2023-10-01T10:43:15.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:15 smithi173 bash[58816]: cluster 2023-10-01T10:43:14.419645+0000 osd.5 (osd.5) 64 : cluster 3 failed to encode map e137 with expected crc 2023-10-01T10:43:15.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:15 smithi173 bash[58816]: cluster 2023-10-01T10:43:14.420986+0000 osd.7 (osd.7) 84 : cluster 3 failed to encode map e137 with expected crc 2023-10-01T10:43:15.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:15 smithi173 bash[58816]: cluster 2023-10-01T10:43:14.422065+0000 osd.5 (osd.5) 65 : cluster 3 failed to encode map e137 with expected crc 2023-10-01T10:43:15.914 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:15.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:15.920 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:43:16.021 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:16.047 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:16.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:16.049 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:16.150 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:16.181 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:16.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:16.186 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:43:16.287 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:16.312 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:16.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:16.316 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:43:16.417 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:16.445 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:16.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:16.451 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:43:16.551 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:16.585 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:16.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:16.589 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:43:16.690 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:16.716 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:16.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:16.719 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:43:16.820 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:16.850 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:16.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:16.852 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:43:16.952 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:16.980 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:16.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:16.983 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:43:17.084 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:17.120 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:17.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:17.126 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:43:17.226 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:17.261 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:17.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:17.263 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:43:17.364 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:17.392 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:17.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:17.397 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:43:17.497 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:17.506 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:17 smithi078 bash[75299]: cluster 2023-10-01T10:43:15.772414+0000 mgr.y (mgr.25072) 766 : cluster 0 pgmap v678: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 821 op/s 2023-10-01T10:43:17.506 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:17 smithi078 bash[75299]: audit 2023-10-01T10:43:17.414149+0000 mon.a (mon.0) 988 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:43:17.506 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:17 smithi078 bash[78274]: cluster 2023-10-01T10:43:15.772414+0000 mgr.y (mgr.25072) 766 : cluster 0 pgmap v678: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 821 op/s 2023-10-01T10:43:17.507 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:17 smithi078 bash[78274]: audit 2023-10-01T10:43:17.414149+0000 mon.a (mon.0) 988 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:43:17.526 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:17.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:17.532 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:43:17.633 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:17.659 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:17.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:17.662 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:17.763 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:17.771 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:17 smithi173 bash[58816]: cluster 2023-10-01T10:43:15.772414+0000 mgr.y (mgr.25072) 766 : cluster 0 pgmap v678: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 821 op/s 2023-10-01T10:43:17.771 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:17 smithi173 bash[58816]: audit 2023-10-01T10:43:17.414149+0000 mon.a (mon.0) 988 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:43:17.791 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:17.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:17.793 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:43:17.839 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-01T10:43:17.894 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:17.939 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:17.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:17.944 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:17.947 INFO:teuthology.orchestra.run.smithi078.stdout:ERROR: (22) Invalid argument 2023-10-01T10:43:17.948 INFO:teuthology.orchestra.run.smithi078.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T10:43:18.045 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:18.074 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:18.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:18.082 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:18.183 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:18.196 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T10:43:18.221 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:18.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:18.227 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:43:18.327 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:18.360 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:18.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:18.364 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:18.465 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:18.506 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:18.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:18.514 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:43:18.615 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:18.649 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:18.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:18.655 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:43:18.756 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:18.777 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:18.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:18.779 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:43:18.880 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:18.921 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:18.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:18.927 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:43:19.027 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:19.065 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:19.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:19.068 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:19.168 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-01T10:43:19.168 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:19.559 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:19.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:19.563 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:43:19.664 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:19.704 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:19.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:19.709 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:43:19.709 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:19 smithi173 bash[58816]: cluster 2023-10-01T10:43:17.772957+0000 mgr.y (mgr.25072) 767 : cluster 0 pgmap v679: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 721 op/s 2023-10-01T10:43:19.810 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:19.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:19 smithi078 bash[75299]: cluster 2023-10-01T10:43:17.772957+0000 mgr.y (mgr.25072) 767 : cluster 0 pgmap v679: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 721 op/s 2023-10-01T10:43:19.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:19 smithi078 bash[78274]: cluster 2023-10-01T10:43:17.772957+0000 mgr.y (mgr.25072) 767 : cluster 0 pgmap v679: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 721 op/s 2023-10-01T10:43:19.837 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:19.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:19.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:43:19.943 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:19.949 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T10:43:19.975 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:19.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:19.980 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:20.081 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:20.112 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:20.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:20.116 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:43:20.217 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:20.251 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:20.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:20.254 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:43:20.355 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:20.384 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:20.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:20.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:43:20.488 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:20.497 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:43:20 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:43:20] "GET /metrics HTTP/1.1" 200 36250 "" "Prometheus/2.33.4" 2023-10-01T10:43:20.519 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:20.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:20.526 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:20.627 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:20.673 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:20.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:20.678 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:20.778 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:20.812 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:20.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:20.815 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:43:20.915 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:20.946 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:20.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:20.948 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:43:21.049 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:21.078 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:21.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:21.081 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:43:21.182 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:21.212 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:21.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:21.218 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:43:21.319 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:21.344 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:21.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:21.346 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:43:21.447 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:21.448 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:21 smithi173 bash[58816]: cluster 2023-10-01T10:43:19.773648+0000 mgr.y (mgr.25072) 768 : cluster 0 pgmap v680: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 718 op/s 2023-10-01T10:43:21.473 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:21.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:21.476 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:21.577 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:21.609 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:21.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:21.616 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:43:21.717 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:21.760 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:21.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:21.763 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:43:21.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:21 smithi078 bash[75299]: cluster 2023-10-01T10:43:19.773648+0000 mgr.y (mgr.25072) 768 : cluster 0 pgmap v680: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 718 op/s 2023-10-01T10:43:21.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:21 smithi078 bash[78274]: cluster 2023-10-01T10:43:19.773648+0000 mgr.y (mgr.25072) 768 : cluster 0 pgmap v680: 89 pgs: 89 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 718 op/s 2023-10-01T10:43:21.863 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:21.922 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:21.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:21.928 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:43:22.029 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:22.063 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:22.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:22.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:43:22.174 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:22.204 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:22.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:22.207 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:22.307 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:22.338 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:22.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:22.347 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:43:22.448 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:22.478 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:22.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:22.481 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:43:22.582 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:22.656 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:22.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:22.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:43:22.682 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:43:22.690 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:43:22.759 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:22.802 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:22.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:22.806 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:43:22.906 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:22.934 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:22.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:22.940 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:43:23.041 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:23.073 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:23.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:23.080 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:43:23.181 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:23.207 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:23.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:23.211 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:43:23.312 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:23.342 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:23.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:23.345 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:43:23.446 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:23.479 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:23.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:23.483 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:43:23.583 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:23.611 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:23.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:23.613 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:43:23.701 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:43:23.714 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:23.721 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:23 smithi173 bash[58816]: cluster 2023-10-01T10:43:21.774462+0000 mgr.y (mgr.25072) 769 : cluster 0 pgmap v681: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 861 op/s 2023-10-01T10:43:23.741 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:23.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:23.746 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:43:23.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:23 smithi078 bash[75299]: cluster 2023-10-01T10:43:21.774462+0000 mgr.y (mgr.25072) 769 : cluster 0 pgmap v681: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 861 op/s 2023-10-01T10:43:23.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:23 smithi078 bash[78274]: cluster 2023-10-01T10:43:21.774462+0000 mgr.y (mgr.25072) 769 : cluster 0 pgmap v681: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 861 op/s 2023-10-01T10:43:23.847 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:23.875 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:23.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:23.882 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:23.983 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:24.010 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:24.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:24.017 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:43:24.117 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:24.149 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:24.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:24.154 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:43:24.255 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:24.281 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:24.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:24.287 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:43:24.388 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:24.423 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:24.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:24.426 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:43:24.527 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:24.555 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:24.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:24.559 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:43:24.660 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:24.685 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:24.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:24.690 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:43:24.719 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:43:24.791 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:24.822 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:24.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:24.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:43:24.928 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:24.952 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:24.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:24.955 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:43:25.056 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:25.078 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:25.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:25.084 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:43:25.185 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:25.210 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:25.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:25.218 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:43:25.319 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:25.353 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:25.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:25.359 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:43:25.460 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:25.464 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:25 smithi173 bash[58816]: cluster 2023-10-01T10:43:23.775054+0000 mgr.y (mgr.25072) 770 : cluster 0 pgmap v682: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 791 op/s 2023-10-01T10:43:25.489 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:25.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:25.492 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:43:25.593 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:25.620 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:25.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:25.623 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:43:25.724 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:25.754 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:25.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:25.758 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:43:25.781 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:25 smithi078 bash[78274]: cluster 2023-10-01T10:43:23.775054+0000 mgr.y (mgr.25072) 770 : cluster 0 pgmap v682: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 791 op/s 2023-10-01T10:43:25.782 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:25 smithi078 bash[75299]: cluster 2023-10-01T10:43:23.775054+0000 mgr.y (mgr.25072) 770 : cluster 0 pgmap v682: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 791 op/s 2023-10-01T10:43:25.859 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:25.881 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:25.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:25.884 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:43:25.985 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:26.030 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:26.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:26.033 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:43:26.134 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:26.157 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:26.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:26.160 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:43:26.260 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:26.286 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:26.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:26.288 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:43:26.389 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:26.414 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:26.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:26.418 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:43:26.519 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:26.546 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:26.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:26.552 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:26.653 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:26.675 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:26.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:26.680 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:26.780 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:26.816 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:26.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:26.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:43:26.920 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:26.942 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:26.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:26.944 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:43:27.045 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:27.067 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:27.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:27.071 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:43:27.172 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:27.205 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:27.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:27.208 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:43:27.309 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:27.344 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:27.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:27.348 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:43:27.448 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:27.467 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:27 smithi173 bash[58816]: cluster 2023-10-01T10:43:25.775608+0000 mgr.y (mgr.25072) 771 : cluster 0 pgmap v683: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 742 op/s 2023-10-01T10:43:27.473 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:27.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:27.478 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:43:27.579 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:27.604 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:27.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:27.608 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:27.709 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:27.716 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:27 smithi078 bash[78274]: cluster 2023-10-01T10:43:25.775608+0000 mgr.y (mgr.25072) 771 : cluster 0 pgmap v683: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 742 op/s 2023-10-01T10:43:27.716 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:27 smithi078 bash[75299]: cluster 2023-10-01T10:43:25.775608+0000 mgr.y (mgr.25072) 771 : cluster 0 pgmap v683: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 742 op/s 2023-10-01T10:43:27.731 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:27.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:27.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:43:27.835 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:27.899 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:27.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:27.906 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:43:28.007 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:28.031 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:28.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:28.034 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:43:28.135 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:28.162 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:28.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:28.166 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:43:28.266 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:28.290 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:28.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:28.292 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:43:28.392 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:28.420 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:28.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:28.423 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:28.523 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:28.554 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:28.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:28.559 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:43:28.660 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:28.689 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:28.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:28.692 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:43:28.792 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:28.826 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:28.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:28.832 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:43:28.932 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:28.972 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:28.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:28.975 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:43:29.076 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:29.099 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:29.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:29.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:43:29.208 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:29.232 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:29.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:29.236 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:29.337 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:29.359 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:29.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:29.364 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:43:29.465 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:29.492 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:29.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:29.497 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:43:29.598 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:29.627 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:29.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:29.631 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:43:29.663 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:29 smithi078 bash[78274]: cluster 2023-10-01T10:43:27.776096+0000 mgr.y (mgr.25072) 772 : cluster 0 pgmap v684: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 688 op/s 2023-10-01T10:43:29.663 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:29 smithi078 bash[75299]: cluster 2023-10-01T10:43:27.776096+0000 mgr.y (mgr.25072) 772 : cluster 0 pgmap v684: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 688 op/s 2023-10-01T10:43:29.732 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:29.741 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:29 smithi173 bash[58816]: cluster 2023-10-01T10:43:27.776096+0000 mgr.y (mgr.25072) 772 : cluster 0 pgmap v684: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 688 op/s 2023-10-01T10:43:29.767 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:29.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:29.769 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:43:29.870 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:29.893 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:29.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:29.900 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:43:30.001 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:30.032 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:30.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:30.035 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:43:30.136 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:30.181 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:30.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:30.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:43:30.269 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:43:30 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:43:30] "GET /metrics HTTP/1.1" 200 36250 "" "Prometheus/2.33.4" 2023-10-01T10:43:30.285 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:30.319 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:30.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:30.321 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:43:30.422 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:30.448 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:30.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:30.450 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:43:30.551 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:30.580 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:30.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:30.586 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:43:30.687 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:30.714 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:30.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:30.720 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:43:30.821 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:30.848 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:30.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:30.851 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:30.951 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:30.981 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:30.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:30.983 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:43:31.084 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:31.125 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:31.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:31.128 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:43:31.229 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:31.274 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:31.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:31.278 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:43:31.379 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:31.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:31.415 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:31.415 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:31.516 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:31.525 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:31 smithi173 bash[58816]: cluster 2023-10-01T10:43:29.776767+0000 mgr.y (mgr.25072) 773 : cluster 0 pgmap v685: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 813 op/s 2023-10-01T10:43:31.543 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:31.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:31.545 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:43:31.646 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:31.679 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:31.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:31.683 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:43:31.784 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:31.792 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:31 smithi078 bash[78274]: cluster 2023-10-01T10:43:29.776767+0000 mgr.y (mgr.25072) 773 : cluster 0 pgmap v685: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 813 op/s 2023-10-01T10:43:31.792 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:31 smithi078 bash[75299]: cluster 2023-10-01T10:43:29.776767+0000 mgr.y (mgr.25072) 773 : cluster 0 pgmap v685: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 813 op/s 2023-10-01T10:43:31.827 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:31.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:31.832 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:43:31.933 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:31.960 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:31.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:31.964 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:32.065 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:32.096 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:32.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:32.102 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:43:32.204 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:32.244 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:32.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:32.249 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:43:32.350 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:32.394 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:32.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:32.396 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:43:32.497 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:32.533 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:32.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:32.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:43:32.638 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:32.663 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:32.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:32.668 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:43:32.769 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:32.803 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:32.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:32.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:43:32.909 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:32.933 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:32.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:32.939 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:43:33.040 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:33.077 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:33.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:33.079 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:43:33.179 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:33.209 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:33.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:33.212 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:43:33.312 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:33.339 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:33.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:33.345 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:33.363 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:33 smithi173 bash[58816]: cluster 2023-10-01T10:43:31.777298+0000 mgr.y (mgr.25072) 774 : cluster 0 pgmap v686: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 753 op/s 2023-10-01T10:43:33.363 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:33 smithi173 bash[58816]: audit 2023-10-01T10:43:32.417019+0000 mon.a (mon.0) 989 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:43:33.445 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:33.478 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:33.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:33.481 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:43:33.581 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:33.592 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:33 smithi078 bash[75299]: cluster 2023-10-01T10:43:31.777298+0000 mgr.y (mgr.25072) 774 : cluster 0 pgmap v686: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 753 op/s 2023-10-01T10:43:33.592 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:33 smithi078 bash[75299]: audit 2023-10-01T10:43:32.417019+0000 mon.a (mon.0) 989 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:43:33.592 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:33 smithi078 bash[78274]: cluster 2023-10-01T10:43:31.777298+0000 mgr.y (mgr.25072) 774 : cluster 0 pgmap v686: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 753 op/s 2023-10-01T10:43:33.592 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:33 smithi078 bash[78274]: audit 2023-10-01T10:43:32.417019+0000 mon.a (mon.0) 989 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:43:33.604 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:33.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:33.607 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:33.708 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:33.733 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:33.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:33.736 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:43:33.837 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:33.881 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:33.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:33.886 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:43:33.987 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:34.275 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:34.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:34.279 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:43:34.379 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:34.417 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:34.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:34.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:43:34.520 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:34.552 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:34.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:34.554 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:43:34.655 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:34.682 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:34.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:34.690 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:43:34.791 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:34.821 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:34.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:34.823 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:43:34.924 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:34.951 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:34.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:34.955 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:43:35.055 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:35.082 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:35.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:35.086 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:43:35.187 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:35.216 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:35.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:35.219 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:43:35.319 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:35.352 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:35.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:35.357 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:43:35.458 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:35.490 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:35.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:35.501 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:35.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:35 smithi078 bash[75299]: cluster 2023-10-01T10:43:33.777862+0000 mgr.y (mgr.25072) 775 : cluster 0 pgmap v687: 89 pgs: 89 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 748 op/s 2023-10-01T10:43:35.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:35 smithi078 bash[78274]: cluster 2023-10-01T10:43:33.777862+0000 mgr.y (mgr.25072) 775 : cluster 0 pgmap v687: 89 pgs: 89 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 748 op/s 2023-10-01T10:43:35.602 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:35.611 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:35 smithi173 bash[58816]: cluster 2023-10-01T10:43:33.777862+0000 mgr.y (mgr.25072) 775 : cluster 0 pgmap v687: 89 pgs: 89 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 748 op/s 2023-10-01T10:43:35.644 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:35.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:35.650 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:43:35.751 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:35.774 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:35.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:35.776 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:35.877 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:35.900 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:35.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:35.906 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:36.006 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:36.041 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:36.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:36.046 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:36.147 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:36.188 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:36.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:36.195 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:43:36.295 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:36.322 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:36.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:36.329 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:43:36.430 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:36.447 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:36.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:36.450 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:36.550 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:36.590 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:36.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:36.595 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:43:36.696 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:36.726 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:36.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:36.728 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:43:36.828 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:36.861 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:36.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:36.865 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:36.965 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:36.994 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:36.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:36.999 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:43:37.100 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:37.133 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:37.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:37.139 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:43:37.239 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:37.273 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:37.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:37.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:43:37.379 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:37.415 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:37.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:37.418 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:43:37.519 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:37.555 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:37.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:37.560 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:43:37.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:37 smithi078 bash[75299]: cluster 2023-10-01T10:43:35.778415+0000 mgr.y (mgr.25072) 776 : cluster 0 pgmap v688: 89 pgs: 89 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 745 op/s 2023-10-01T10:43:37.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:37 smithi078 bash[78274]: cluster 2023-10-01T10:43:35.778415+0000 mgr.y (mgr.25072) 776 : cluster 0 pgmap v688: 89 pgs: 89 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 745 op/s 2023-10-01T10:43:37.661 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:37.689 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:37.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:37.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:43:37.792 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:37.816 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:37.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:37.819 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:43:37.898 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:37 smithi173 bash[58816]: cluster 2023-10-01T10:43:35.778415+0000 mgr.y (mgr.25072) 776 : cluster 0 pgmap v688: 89 pgs: 89 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 745 op/s 2023-10-01T10:43:37.920 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:37.964 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:37.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:37.973 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:43:38.073 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:38.105 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:38.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:38.107 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:43:38.132 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T10:43:38.175 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T10:43:38.208 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:38.215 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T10:43:38.240 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:38.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:38.243 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:43:38.259 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T10:43:38.293 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T10:43:38.336 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T10:43:38.344 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:38.372 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T10:43:38.376 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:38.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:38.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:43:38.402 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T10:43:38.405 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T10:43:38.405 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T10:43:38.405 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T10:43:38.405 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T10:43:38.405 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T10:43:38.405 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T10:43:38.406 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T10:43:38.406 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T10:43:38.483 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:38.492 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:38 smithi173 bash[58816]: audit 2023-10-01T10:43:38.082035+0000 mon.c (mon.2) 140 : audit 0 from='client.? 172.21.15.78:0/4273290650' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:43:38.508 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:38.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:38.514 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:43:38.615 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:38.642 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:38.645 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:38 smithi078 bash[75299]: audit 2023-10-01T10:43:38.082035+0000 mon.c (mon.2) 140 : audit 0 from='client.? 172.21.15.78:0/4273290650' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:43:38.645 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:38 smithi078 bash[78274]: audit 2023-10-01T10:43:38.082035+0000 mon.c (mon.2) 140 : audit 0 from='client.? 172.21.15.78:0/4273290650' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:43:38.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:38.646 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:43:38.747 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:38.782 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:38.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:38.786 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:43:38.887 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:38.917 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:38.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:38.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:43:39.020 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:39.054 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:39.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:39.061 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:43:39.162 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:39.193 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:39.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:39.199 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:43:39.214 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2023-10-01T10:43:39.214 INFO:teuthology.orchestra.run.smithi078.stdout: "size": 20, 2023-10-01T10:43:39.215 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 600, 2023-10-01T10:43:39.215 INFO:teuthology.orchestra.run.smithi078.stdout: "ops": [ 2023-10-01T10:43:39.215 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.215 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.35649.0:35594 9.6 e131/128 9:630495e2:::benchmark_data_smithi078_113121_object35593:head v 131'2349, mlcod=131'2346)", 2023-10-01T10:43:39.215 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:41:25.013005+0000", 2023-10-01T10:43:39.215 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 134.19811555699999, 2023-10-01T10:43:39.215 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.019516789999999999, 2023-10-01T10:43:39.215 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:43:39.215 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:43:39.215 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:43:39.215 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.215 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:43:39.215 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:25.013005+0000", 2023-10-01T10:43:39.215 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:43:39.216 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.216 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.216 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:43:39.216 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:25.013005+0000", 2023-10-01T10:43:39.216 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:43:39.216 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.216 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.216 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:43:39.216 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:25.013004+0000", 2023-10-01T10:43:39.216 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:43:39.216 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.216 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.216 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:43:39.217 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:25.013067+0000", 2023-10-01T10:43:39.217 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.2527999999999995e-05 2023-10-01T10:43:39.217 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.217 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.217 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:43:39.217 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:25.013079+0000", 2023-10-01T10:43:39.217 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1855000000000001e-05 2023-10-01T10:43:39.217 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.217 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.217 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:43:39.217 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:25.013087+0000", 2023-10-01T10:43:39.217 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.7090000000000002e-06 2023-10-01T10:43:39.217 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.218 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.218 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:43:39.218 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:25.013145+0000", 2023-10-01T10:43:39.218 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.8087999999999997e-05 2023-10-01T10:43:39.218 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.218 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.218 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:43:39.218 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:25.013158+0000", 2023-10-01T10:43:39.218 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.3154e-05 2023-10-01T10:43:39.218 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.218 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.219 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:43:39.219 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:25.032448+0000", 2023-10-01T10:43:39.219 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.019290726000000001 2023-10-01T10:43:39.219 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.219 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.219 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:43:39.219 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:25.032522+0000", 2023-10-01T10:43:39.219 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.3669000000000002e-05 2023-10-01T10:43:39.219 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:43:39.219 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:43:39.219 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:43:39.219 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.220 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.220 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.35649.0:35597 9.0 e131/128 9:0b06f2eb:::benchmark_data_smithi078_113121_object35596:head v 131'2267, mlcod=131'2264)", 2023-10-01T10:43:39.220 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:41:25.014428+0000", 2023-10-01T10:43:39.220 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 134.19669263700001, 2023-10-01T10:43:39.220 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.022943949000000002, 2023-10-01T10:43:39.220 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:43:39.220 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:43:39.220 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:43:39.220 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.220 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:43:39.220 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:25.014428+0000", 2023-10-01T10:43:39.220 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:43:39.221 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.221 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.221 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:43:39.221 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:25.014428+0000", 2023-10-01T10:43:39.221 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:43:39.221 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.221 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.221 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:43:39.221 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:25.014428+0000", 2023-10-01T10:43:39.221 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:43:39.221 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.221 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.222 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:43:39.222 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:25.014451+0000", 2023-10-01T10:43:39.222 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.3646999999999999e-05 2023-10-01T10:43:39.222 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.222 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.222 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:43:39.222 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:25.029937+0000", 2023-10-01T10:43:39.223 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.015485314999999999 2023-10-01T10:43:39.223 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.223 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.223 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:43:39.224 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:25.029951+0000", 2023-10-01T10:43:39.224 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.4508000000000001e-05 2023-10-01T10:43:39.224 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.224 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.224 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:43:39.224 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:25.032796+0000", 2023-10-01T10:43:39.224 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0028449510000000001 2023-10-01T10:43:39.224 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.224 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.224 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:43:39.224 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:25.032811+0000", 2023-10-01T10:43:39.225 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.4946999999999999e-05 2023-10-01T10:43:39.225 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.225 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.225 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:43:39.225 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:25.037248+0000", 2023-10-01T10:43:39.225 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0044370549999999996 2023-10-01T10:43:39.225 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.225 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.225 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:43:39.225 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:25.037372+0000", 2023-10-01T10:43:39.225 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00012424800000000001 2023-10-01T10:43:39.225 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:43:39.226 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:43:39.226 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:43:39.226 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.226 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.226 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.35649.0:78541 9.2 e133/128 9:462158ea:::benchmark_data_smithi078_113121_object78540:head v 133'4991, mlcod=133'4989)", 2023-10-01T10:43:39.226 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:42:00.592631+0000", 2023-10-01T10:43:39.226 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 98.618489486000001, 2023-10-01T10:43:39.226 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.026710879999999999, 2023-10-01T10:43:39.226 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:43:39.226 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:43:39.226 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:43:39.226 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.227 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:43:39.227 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:00.592631+0000", 2023-10-01T10:43:39.227 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:43:39.227 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.227 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.227 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:43:39.227 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:00.592631+0000", 2023-10-01T10:43:39.227 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:43:39.227 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.227 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.227 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:43:39.228 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:00.592631+0000", 2023-10-01T10:43:39.228 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999995 2023-10-01T10:43:39.228 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.228 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.228 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:43:39.228 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:00.592652+0000", 2023-10-01T10:43:39.228 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.0767e-05 2023-10-01T10:43:39.228 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.228 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.228 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:43:39.228 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:00.592659+0000", 2023-10-01T10:43:39.228 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.1289999999999998e-06 2023-10-01T10:43:39.229 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.229 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.229 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:43:39.229 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:00.592664+0000", 2023-10-01T10:43:39.229 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.057e-06 2023-10-01T10:43:39.229 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.229 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.229 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:43:39.229 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:00.592711+0000", 2023-10-01T10:43:39.229 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.7314999999999997e-05 2023-10-01T10:43:39.229 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.229 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.230 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:43:39.230 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:00.592732+0000", 2023-10-01T10:43:39.230 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.0375000000000001e-05 2023-10-01T10:43:39.230 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.230 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.230 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:43:39.230 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:00.619299+0000", 2023-10-01T10:43:39.230 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.026566950999999998 2023-10-01T10:43:39.230 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.230 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.230 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:43:39.231 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:00.619342+0000", 2023-10-01T10:43:39.231 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.3779999999999998e-05 2023-10-01T10:43:39.231 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:43:39.231 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:43:39.231 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:43:39.231 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.231 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.231 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.35649.0:81258 9.0 e133/128 9:0a846cd5:::benchmark_data_smithi078_113121_object81257:head v 133'5067, mlcod=133'5062)", 2023-10-01T10:43:39.231 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:42:03.260119+0000", 2023-10-01T10:43:39.231 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 95.951001946999995, 2023-10-01T10:43:39.231 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.022828317000000001, 2023-10-01T10:43:39.231 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:43:39.232 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:43:39.232 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:43:39.232 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.232 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:43:39.232 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:03.260119+0000", 2023-10-01T10:43:39.232 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:43:39.232 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.232 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.232 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:43:39.232 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:03.260119+0000", 2023-10-01T10:43:39.232 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:43:39.232 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.233 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.233 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:43:39.233 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:03.260118+0000", 2023-10-01T10:43:39.233 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:43:39.233 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.233 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.233 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:43:39.233 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:03.260169+0000", 2023-10-01T10:43:39.233 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.0714999999999998e-05 2023-10-01T10:43:39.233 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.233 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.233 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:43:39.234 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:03.260181+0000", 2023-10-01T10:43:39.234 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.2556e-05 2023-10-01T10:43:39.234 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.234 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.234 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:43:39.234 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:03.260187+0000", 2023-10-01T10:43:39.234 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.1449999999999996e-06 2023-10-01T10:43:39.234 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.234 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.234 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:43:39.234 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:03.264272+0000", 2023-10-01T10:43:39.235 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0040846320000000004 2023-10-01T10:43:39.235 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.235 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.235 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:43:39.235 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:03.264335+0000", 2023-10-01T10:43:39.235 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.3571000000000005e-05 2023-10-01T10:43:39.235 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.235 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.235 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:43:39.235 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:03.282843+0000", 2023-10-01T10:43:39.235 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.018507820000000001 2023-10-01T10:43:39.235 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.236 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.236 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:43:39.236 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:03.282947+0000", 2023-10-01T10:43:39.236 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000104021 2023-10-01T10:43:39.236 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:43:39.236 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:43:39.236 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:43:39.236 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.236 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.236 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.35649.0:81265 9.0 e133/128 9:058c3f7c:::benchmark_data_smithi078_113121_object81264:head v 133'5068, mlcod=133'5063)", 2023-10-01T10:43:39.236 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:42:03.263276+0000", 2023-10-01T10:43:39.236 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 95.947844762000003, 2023-10-01T10:43:39.236 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.019743862000000001, 2023-10-01T10:43:39.237 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:43:39.237 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:43:39.237 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:43:39.237 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.237 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:43:39.237 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:03.263276+0000", 2023-10-01T10:43:39.237 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:43:39.237 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.237 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.237 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:43:39.237 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:03.263276+0000", 2023-10-01T10:43:39.237 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:43:39.237 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.238 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.238 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:43:39.238 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:03.263275+0000", 2023-10-01T10:43:39.238 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999986 2023-10-01T10:43:39.238 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.238 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.238 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:43:39.238 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:03.263328+0000", 2023-10-01T10:43:39.238 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.3369999999999999e-05 2023-10-01T10:43:39.238 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.238 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.239 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:43:39.239 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:03.263348+0000", 2023-10-01T10:43:39.239 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.9894999999999998e-05 2023-10-01T10:43:39.239 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.239 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.239 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:43:39.239 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:03.263357+0000", 2023-10-01T10:43:39.239 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.2320000000000007e-06 2023-10-01T10:43:39.239 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.239 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.239 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:43:39.239 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:03.267371+0000", 2023-10-01T10:43:39.239 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0040137430000000002 2023-10-01T10:43:39.240 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.240 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.240 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:43:39.240 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:03.267432+0000", 2023-10-01T10:43:39.240 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.0432e-05 2023-10-01T10:43:39.240 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.240 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.240 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:43:39.240 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:03.282951+0000", 2023-10-01T10:43:39.240 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.015518936000000001 2023-10-01T10:43:39.240 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.241 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.241 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:43:39.241 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:03.283020+0000", 2023-10-01T10:43:39.241 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.9457999999999998e-05 2023-10-01T10:43:39.241 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:43:39.241 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:43:39.241 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:43:39.241 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.241 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.241 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.35649.0:93667 9.2 e133/128 9:4c5b1188:::benchmark_data_smithi078_113121_object93666:head v 133'5951, mlcod=133'5949)", 2023-10-01T10:43:39.241 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:42:16.427855+0000", 2023-10-01T10:43:39.241 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 82.783266136999998, 2023-10-01T10:43:39.241 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.019434882000000001, 2023-10-01T10:43:39.241 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:43:39.242 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:43:39.242 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:43:39.242 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.242 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:43:39.242 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:16.427855+0000", 2023-10-01T10:43:39.242 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:43:39.242 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.242 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.242 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:43:39.242 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:16.427855+0000", 2023-10-01T10:43:39.242 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:43:39.242 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.243 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.243 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:43:39.243 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:16.427854+0000", 2023-10-01T10:43:39.243 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:43:39.243 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.243 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.243 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:43:39.243 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:16.427882+0000", 2023-10-01T10:43:39.243 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.8458000000000001e-05 2023-10-01T10:43:39.243 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.243 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.244 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:43:39.244 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:16.427897+0000", 2023-10-01T10:43:39.244 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.4229999999999999e-05 2023-10-01T10:43:39.244 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.244 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.244 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:43:39.244 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:16.427905+0000", 2023-10-01T10:43:39.244 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.4819999999999994e-06 2023-10-01T10:43:39.244 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.244 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.244 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:43:39.244 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:16.428103+0000", 2023-10-01T10:43:39.245 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000197834 2023-10-01T10:43:39.245 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.245 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.245 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:43:39.245 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:16.428155+0000", 2023-10-01T10:43:39.245 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.2524000000000001e-05 2023-10-01T10:43:39.245 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.245 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.245 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:43:39.245 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:16.447093+0000", 2023-10-01T10:43:39.245 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.018937236 2023-10-01T10:43:39.245 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.246 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.246 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:43:39.246 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:16.447290+0000", 2023-10-01T10:43:39.246 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00019700100000000001 2023-10-01T10:43:39.246 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:43:39.246 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:43:39.246 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:43:39.246 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.246 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.246 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.35649.0:123429 9.2 e135/128 9:4f4710a8:::benchmark_data_smithi078_113121_object11614:head v 135'7839, mlcod=135'7837)", 2023-10-01T10:43:39.246 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:42:52.843311+0000", 2023-10-01T10:43:39.246 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 46.367810345000002, 2023-10-01T10:43:39.247 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.031075759000000001, 2023-10-01T10:43:39.247 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:43:39.247 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:43:39.247 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:43:39.247 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.247 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:43:39.247 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:52.843311+0000", 2023-10-01T10:43:39.247 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:43:39.247 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.247 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.247 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:43:39.247 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:52.843311+0000", 2023-10-01T10:43:39.247 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:43:39.248 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.248 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.248 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:43:39.248 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:52.843310+0000", 2023-10-01T10:43:39.248 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:43:39.248 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.248 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.248 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:43:39.248 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:52.843318+0000", 2023-10-01T10:43:39.248 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.4600000000000003e-06 2023-10-01T10:43:39.248 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.248 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.248 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:43:39.249 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:52.843337+0000", 2023-10-01T10:43:39.249 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.8583000000000001e-05 2023-10-01T10:43:39.249 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.249 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.249 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:43:39.249 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:52.843345+0000", 2023-10-01T10:43:39.249 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.7560000000000005e-06 2023-10-01T10:43:39.249 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.249 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.249 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:43:39.249 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:52.843554+0000", 2023-10-01T10:43:39.249 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000208493 2023-10-01T10:43:39.249 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.250 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.250 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:43:39.250 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:52.843624+0000", 2023-10-01T10:43:39.250 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.9653999999999994e-05 2023-10-01T10:43:39.250 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.250 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.250 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:43:39.250 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:52.874200+0000", 2023-10-01T10:43:39.250 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.030576124999999999 2023-10-01T10:43:39.250 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.250 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.250 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:43:39.251 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:52.874386+0000", 2023-10-01T10:43:39.251 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00018657499999999999 2023-10-01T10:43:39.251 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:43:39.251 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:43:39.251 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:43:39.251 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.251 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.251 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.35649.0:123434 9.2 e135/128 9:47729478:::benchmark_data_smithi078_113121_object11619:head v 135'7840, mlcod=135'7837)", 2023-10-01T10:43:39.251 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:42:52.846876+0000", 2023-10-01T10:43:39.251 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 46.364244943000003, 2023-10-01T10:43:39.251 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.027650352, 2023-10-01T10:43:39.251 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:43:39.252 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:43:39.252 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:43:39.252 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.252 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:43:39.252 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:52.846876+0000", 2023-10-01T10:43:39.252 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:43:39.252 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.252 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.252 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:43:39.252 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:52.846876+0000", 2023-10-01T10:43:39.252 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:43:39.252 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.253 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.253 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:43:39.253 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:52.846875+0000", 2023-10-01T10:43:39.253 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:43:39.253 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.253 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.253 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:43:39.253 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:52.846884+0000", 2023-10-01T10:43:39.253 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.2189999999999993e-06 2023-10-01T10:43:39.253 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.253 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.253 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:43:39.254 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:52.846896+0000", 2023-10-01T10:43:39.254 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1719000000000001e-05 2023-10-01T10:43:39.254 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.254 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.254 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:43:39.254 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:52.846913+0000", 2023-10-01T10:43:39.254 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.6864999999999999e-05 2023-10-01T10:43:39.254 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.254 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.254 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:43:39.254 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:52.848217+0000", 2023-10-01T10:43:39.255 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0013041439999999999 2023-10-01T10:43:39.255 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.255 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.255 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:43:39.255 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:52.848270+0000", 2023-10-01T10:43:39.255 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.2614000000000001e-05 2023-10-01T10:43:39.255 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.255 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.255 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:43:39.255 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:52.874390+0000", 2023-10-01T10:43:39.255 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.026120455000000001 2023-10-01T10:43:39.255 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.256 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.256 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:43:39.256 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:52.874526+0000", 2023-10-01T10:43:39.256 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00013636599999999999 2023-10-01T10:43:39.256 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:43:39.256 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:43:39.256 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:43:39.256 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.256 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.256 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.35649.0:124359 9.6 e135/128 9:6f6776a8:::benchmark_data_smithi078_113121_object12544:head v 135'7891, mlcod=135'7888)", 2023-10-01T10:43:39.256 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:42:54.168129+0000", 2023-10-01T10:43:39.256 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 45.04299176, 2023-10-01T10:43:39.256 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.028998243999999999, 2023-10-01T10:43:39.257 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:43:39.257 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:43:39.257 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:43:39.257 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.257 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:43:39.257 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:54.168129+0000", 2023-10-01T10:43:39.257 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:43:39.257 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.257 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.257 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:43:39.257 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:54.168129+0000", 2023-10-01T10:43:39.257 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:43:39.257 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.258 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.258 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:43:39.258 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:54.168128+0000", 2023-10-01T10:43:39.258 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:43:39.258 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.258 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.258 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:43:39.258 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:54.168138+0000", 2023-10-01T10:43:39.258 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.8109999999999997e-06 2023-10-01T10:43:39.258 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.258 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.258 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:43:39.258 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:54.168149+0000", 2023-10-01T10:43:39.259 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1212e-05 2023-10-01T10:43:39.259 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.259 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.259 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:43:39.259 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:54.168156+0000", 2023-10-01T10:43:39.259 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.9120000000000001e-06 2023-10-01T10:43:39.259 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.259 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.259 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:43:39.259 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:54.168220+0000", 2023-10-01T10:43:39.259 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.4153999999999996e-05 2023-10-01T10:43:39.259 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.259 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.259 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:43:39.260 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:54.168240+0000", 2023-10-01T10:43:39.260 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.9881000000000002e-05 2023-10-01T10:43:39.260 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.260 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.260 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:43:39.260 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:54.197092+0000", 2023-10-01T10:43:39.260 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.028852019999999999 2023-10-01T10:43:39.260 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.260 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.260 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:43:39.260 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:54.197127+0000", 2023-10-01T10:43:39.260 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.5040000000000003e-05 2023-10-01T10:43:39.260 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:43:39.261 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:43:39.261 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:43:39.261 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.261 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.261 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.35649.0:124363 9.6 e135/128 9:66319909:::benchmark_data_smithi078_113121_object12548:head v 135'7892, mlcod=135'7888)", 2023-10-01T10:43:39.261 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:42:54.181156+0000", 2023-10-01T10:43:39.261 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 45.029965271000002, 2023-10-01T10:43:39.261 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.023620130999999999, 2023-10-01T10:43:39.261 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:43:39.261 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:43:39.261 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:43:39.261 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.261 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:43:39.262 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:54.181156+0000", 2023-10-01T10:43:39.262 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:43:39.262 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.262 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.262 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:43:39.262 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:54.181156+0000", 2023-10-01T10:43:39.262 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:43:39.262 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.262 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.262 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:43:39.262 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:54.181154+0000", 2023-10-01T10:43:39.262 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999986 2023-10-01T10:43:39.263 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.263 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.263 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:43:39.263 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:54.181167+0000", 2023-10-01T10:43:39.263 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.3039e-05 2023-10-01T10:43:39.263 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.263 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.263 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:43:39.263 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:54.181194+0000", 2023-10-01T10:43:39.263 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.6625e-05 2023-10-01T10:43:39.263 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.263 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.264 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:43:39.264 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:54.181205+0000", 2023-10-01T10:43:39.264 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1562999999999999e-05 2023-10-01T10:43:39.264 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.264 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.264 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:43:39.264 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:54.181450+0000", 2023-10-01T10:43:39.264 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000245214 2023-10-01T10:43:39.264 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.264 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.264 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:43:39.264 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:54.181476+0000", 2023-10-01T10:43:39.265 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.5746e-05 2023-10-01T10:43:39.265 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.265 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.265 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:43:39.265 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:54.204727+0000", 2023-10-01T10:43:39.265 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.023250606 2023-10-01T10:43:39.265 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.265 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.265 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:43:39.265 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:54.204776+0000", 2023-10-01T10:43:39.265 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.8962999999999999e-05 2023-10-01T10:43:39.265 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:43:39.266 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:43:39.266 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:43:39.266 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.266 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.266 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.35649.0:124366 9.6 e135/128 9:6b7937a2:::benchmark_data_smithi078_113121_object12551:head v 135'7893, mlcod=135'7888)", 2023-10-01T10:43:39.266 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:42:54.181344+0000", 2023-10-01T10:43:39.266 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 45.029776939999998, 2023-10-01T10:43:39.266 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.023461617000000001, 2023-10-01T10:43:39.266 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:43:39.266 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:43:39.266 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:43:39.266 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.267 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:43:39.267 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:54.181344+0000", 2023-10-01T10:43:39.267 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:43:39.267 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.267 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.267 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:43:39.267 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:54.181344+0000", 2023-10-01T10:43:39.267 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:43:39.267 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.267 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.267 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:43:39.267 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:54.181343+0000", 2023-10-01T10:43:39.267 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999995 2023-10-01T10:43:39.268 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.268 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.268 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:43:39.268 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:54.181349+0000", 2023-10-01T10:43:39.268 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.0299999999999999e-06 2023-10-01T10:43:39.268 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.268 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.268 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:43:39.268 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:54.181361+0000", 2023-10-01T10:43:39.268 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1783000000000001e-05 2023-10-01T10:43:39.268 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.268 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.269 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:43:39.269 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:54.181368+0000", 2023-10-01T10:43:39.269 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.3439999999999997e-06 2023-10-01T10:43:39.269 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.269 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.269 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:43:39.269 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:54.197149+0000", 2023-10-01T10:43:39.269 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.015781261000000001 2023-10-01T10:43:39.269 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.269 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.269 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:43:39.269 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:54.197164+0000", 2023-10-01T10:43:39.269 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.5081e-05 2023-10-01T10:43:39.269 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.270 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.270 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:43:39.270 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:54.204780+0000", 2023-10-01T10:43:39.270 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.007616121 2023-10-01T10:43:39.270 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.270 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.270 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:43:39.270 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:54.204806+0000", 2023-10-01T10:43:39.270 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.5626000000000002e-05 2023-10-01T10:43:39.270 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:43:39.270 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:43:39.270 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:43:39.271 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.271 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.271 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.35649.0:124371 9.6 e135/128 9:647381e4:::benchmark_data_smithi078_113121_object12556:head v 135'7894, mlcod=135'7888)", 2023-10-01T10:43:39.271 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:42:54.185744+0000", 2023-10-01T10:43:39.271 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 45.025376520000002, 2023-10-01T10:43:39.271 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.019229217, 2023-10-01T10:43:39.271 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:43:39.271 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:43:39.271 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:43:39.271 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.271 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:43:39.271 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:54.185744+0000", 2023-10-01T10:43:39.272 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:43:39.272 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.272 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.272 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:43:39.272 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:54.185744+0000", 2023-10-01T10:43:39.272 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:43:39.272 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.272 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.272 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:43:39.272 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:54.185743+0000", 2023-10-01T10:43:39.272 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:43:39.273 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.273 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.273 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:43:39.273 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:54.185755+0000", 2023-10-01T10:43:39.273 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1304000000000001e-05 2023-10-01T10:43:39.273 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.273 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.273 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:43:39.273 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:54.185767+0000", 2023-10-01T10:43:39.273 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.2044e-05 2023-10-01T10:43:39.273 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.273 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.273 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:43:39.273 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:54.185773+0000", 2023-10-01T10:43:39.274 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.8860000000000001e-06 2023-10-01T10:43:39.274 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.274 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.274 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:43:39.274 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:54.200666+0000", 2023-10-01T10:43:39.274 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.014893725 2023-10-01T10:43:39.274 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.274 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.274 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:43:39.274 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:54.200692+0000", 2023-10-01T10:43:39.274 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.5108000000000001e-05 2023-10-01T10:43:39.275 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.275 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.275 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:43:39.275 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:54.204948+0000", 2023-10-01T10:43:39.275 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0042569219999999998 2023-10-01T10:43:39.275 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.275 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.275 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:43:39.275 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:54.204974+0000", 2023-10-01T10:43:39.275 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.5145999999999999e-05 2023-10-01T10:43:39.275 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:43:39.275 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:43:39.275 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:43:39.276 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.276 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.276 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.35649.0:125022 9.6 e135/128 9:65002ff5:::benchmark_data_smithi078_113121_object13207:head v 135'7942, mlcod=135'7940)", 2023-10-01T10:43:39.276 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:42:55.049186+0000", 2023-10-01T10:43:39.276 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 44.161934801999998, 2023-10-01T10:43:39.276 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.023410357999999999, 2023-10-01T10:43:39.276 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:43:39.276 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:43:39.276 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:43:39.276 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.276 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:43:39.276 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:55.049186+0000", 2023-10-01T10:43:39.276 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:43:39.277 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.277 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.277 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:43:39.277 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:55.049186+0000", 2023-10-01T10:43:39.277 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:43:39.277 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.277 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.277 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:43:39.277 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:55.049185+0000", 2023-10-01T10:43:39.277 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:43:39.277 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.277 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.277 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:43:39.278 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:55.049194+0000", 2023-10-01T10:43:39.278 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.3270000000000001e-06 2023-10-01T10:43:39.278 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.278 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.278 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:43:39.278 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:55.049208+0000", 2023-10-01T10:43:39.278 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.4222999999999999e-05 2023-10-01T10:43:39.278 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.278 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.278 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:43:39.278 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:55.049215+0000", 2023-10-01T10:43:39.279 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.1620000000000002e-06 2023-10-01T10:43:39.279 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.279 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.279 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:43:39.279 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:55.049276+0000", 2023-10-01T10:43:39.279 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.1231999999999999e-05 2023-10-01T10:43:39.279 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.279 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.279 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:43:39.279 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:55.049324+0000", 2023-10-01T10:43:39.279 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.7876e-05 2023-10-01T10:43:39.279 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.280 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.280 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:43:39.280 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:55.072476+0000", 2023-10-01T10:43:39.280 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.023151830000000002 2023-10-01T10:43:39.280 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.280 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.280 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:43:39.280 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:55.072596+0000", 2023-10-01T10:43:39.280 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000120641 2023-10-01T10:43:39.280 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:43:39.280 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:43:39.280 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:43:39.281 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.281 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.281 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.35649.0:125027 9.6 e135/128 9:6ecef1b2:::benchmark_data_smithi078_113121_object13212:head v 135'7943, mlcod=135'7940)", 2023-10-01T10:43:39.281 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:42:55.049837+0000", 2023-10-01T10:43:39.281 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 44.161283611000002, 2023-10-01T10:43:39.281 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.022820435, 2023-10-01T10:43:39.281 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:43:39.281 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:43:39.281 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:43:39.281 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.281 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:43:39.281 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:55.049837+0000", 2023-10-01T10:43:39.282 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:43:39.282 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.282 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.282 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:43:39.282 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:55.049837+0000", 2023-10-01T10:43:39.282 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:43:39.282 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.282 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.282 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:43:39.282 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:55.049837+0000", 2023-10-01T10:43:39.282 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:43:39.282 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.283 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.283 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:43:39.283 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:55.049846+0000", 2023-10-01T10:43:39.283 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.7659999999999993e-06 2023-10-01T10:43:39.283 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.283 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.283 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:43:39.283 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:55.049861+0000", 2023-10-01T10:43:39.283 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.4524e-05 2023-10-01T10:43:39.283 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.283 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.283 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:43:39.284 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:55.049868+0000", 2023-10-01T10:43:39.284 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.2860000000000002e-06 2023-10-01T10:43:39.284 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.284 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.284 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:43:39.284 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:55.053696+0000", 2023-10-01T10:43:39.284 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0038275570000000001 2023-10-01T10:43:39.284 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.284 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.284 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:43:39.284 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:55.053730+0000", 2023-10-01T10:43:39.284 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.3807999999999999e-05 2023-10-01T10:43:39.285 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.285 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.285 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:43:39.285 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:55.072602+0000", 2023-10-01T10:43:39.285 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.018872177 2023-10-01T10:43:39.285 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.285 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.285 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:43:39.285 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:55.072658+0000", 2023-10-01T10:43:39.285 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.6036999999999999e-05 2023-10-01T10:43:39.285 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:43:39.285 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:43:39.286 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:43:39.286 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.286 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.286 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.35649.0:126072 9.6 e135/128 9:6fac3e4c:::benchmark_data_smithi078_113121_object14257:head v 135'8012, mlcod=135'8009)", 2023-10-01T10:43:39.286 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:42:56.450665+0000", 2023-10-01T10:43:39.286 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 42.760455706000002, 2023-10-01T10:43:39.286 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.021825977, 2023-10-01T10:43:39.286 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:43:39.286 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:43:39.286 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:43:39.286 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.286 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:43:39.287 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:56.450665+0000", 2023-10-01T10:43:39.287 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:43:39.287 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.287 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.287 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:43:39.287 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:56.450665+0000", 2023-10-01T10:43:39.287 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:43:39.287 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.287 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.287 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:43:39.287 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:56.450664+0000", 2023-10-01T10:43:39.287 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:43:39.288 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.288 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.288 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:43:39.288 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:56.450679+0000", 2023-10-01T10:43:39.288 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.5216e-05 2023-10-01T10:43:39.288 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.288 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.288 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:43:39.288 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:56.450695+0000", 2023-10-01T10:43:39.288 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.5704000000000001e-05 2023-10-01T10:43:39.288 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.288 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.288 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:43:39.289 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:56.450701+0000", 2023-10-01T10:43:39.289 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.3720000000000001e-06 2023-10-01T10:43:39.289 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.289 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.289 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:43:39.289 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:56.450829+0000", 2023-10-01T10:43:39.289 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000127274 2023-10-01T10:43:39.289 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.289 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.289 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:43:39.289 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:56.450873+0000", 2023-10-01T10:43:39.289 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.4323000000000002e-05 2023-10-01T10:43:39.289 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.290 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.290 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:43:39.290 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:56.472403+0000", 2023-10-01T10:43:39.290 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.021530455 2023-10-01T10:43:39.290 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.290 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.290 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:43:39.290 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:56.472491+0000", 2023-10-01T10:43:39.290 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.7782999999999996e-05 2023-10-01T10:43:39.290 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:43:39.290 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:43:39.291 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:43:39.291 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.291 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.291 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.35649.0:126073 9.6 e135/128 9:6ca2dd44:::benchmark_data_smithi078_113121_object14258:head v 135'8013, mlcod=135'8009)", 2023-10-01T10:43:39.291 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:42:56.450977+0000", 2023-10-01T10:43:39.293 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 42.760143485999997, 2023-10-01T10:43:39.293 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.021606976, 2023-10-01T10:43:39.293 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:43:39.293 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:43:39.293 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:43:39.293 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.293 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:43:39.293 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:56.450977+0000", 2023-10-01T10:43:39.294 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:43:39.294 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.294 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.294 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:43:39.294 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:56.450977+0000", 2023-10-01T10:43:39.294 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:43:39.294 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.294 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.294 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:43:39.294 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:56.450977+0000", 2023-10-01T10:43:39.294 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:43:39.295 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.295 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.295 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:43:39.295 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:56.450987+0000", 2023-10-01T10:43:39.295 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.9809999999999998e-06 2023-10-01T10:43:39.295 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.295 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.295 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:43:39.295 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:56.451021+0000", 2023-10-01T10:43:39.295 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.4190000000000003e-05 2023-10-01T10:43:39.295 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.295 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.295 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:43:39.296 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:56.451029+0000", 2023-10-01T10:43:39.296 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.7540000000000006e-06 2023-10-01T10:43:39.296 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.296 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.296 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:43:39.296 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:56.461654+0000", 2023-10-01T10:43:39.296 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.010625848 2023-10-01T10:43:39.296 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.296 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.296 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:43:39.296 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:56.461744+0000", 2023-10-01T10:43:39.296 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.9090000000000003e-05 2023-10-01T10:43:39.297 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.297 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.297 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:43:39.297 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:56.472494+0000", 2023-10-01T10:43:39.297 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.010750796999999999 2023-10-01T10:43:39.297 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.297 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.297 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:43:39.297 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:56.472584+0000", 2023-10-01T10:43:39.297 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.0039999999999999e-05 2023-10-01T10:43:39.297 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:43:39.297 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:43:39.298 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:43:39.298 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.298 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.298 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.35649.0:130136 9.6 e135/128 9:68f92e58:::benchmark_data_smithi078_113121_object18321:head v 135'8276, mlcod=135'8273)", 2023-10-01T10:43:39.298 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:43:01.916090+0000", 2023-10-01T10:43:39.298 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 37.295030654999998, 2023-10-01T10:43:39.298 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.020314187000000001, 2023-10-01T10:43:39.298 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:43:39.298 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:43:39.298 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:43:39.298 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.298 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:43:39.299 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:01.916090+0000", 2023-10-01T10:43:39.299 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:43:39.299 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.299 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.299 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:43:39.299 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:01.916090+0000", 2023-10-01T10:43:39.299 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:43:39.299 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.299 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.299 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:43:39.299 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:01.916090+0000", 2023-10-01T10:43:39.299 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999995 2023-10-01T10:43:39.299 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.300 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.300 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:43:39.300 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:01.916096+0000", 2023-10-01T10:43:39.300 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.5300000000000002e-06 2023-10-01T10:43:39.300 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.300 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.300 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:43:39.300 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:01.916103+0000", 2023-10-01T10:43:39.300 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.8759999999999999e-06 2023-10-01T10:43:39.300 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.300 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.300 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:43:39.300 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:01.916108+0000", 2023-10-01T10:43:39.300 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.2170000000000002e-06 2023-10-01T10:43:39.301 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.301 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.301 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:43:39.301 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:01.916355+0000", 2023-10-01T10:43:39.301 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000247063 2023-10-01T10:43:39.301 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.301 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.301 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:43:39.301 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:01.916413+0000", 2023-10-01T10:43:39.301 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.7630000000000002e-05 2023-10-01T10:43:39.301 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.301 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.302 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:43:39.302 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:01.936295+0000", 2023-10-01T10:43:39.302 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.019882207999999998 2023-10-01T10:43:39.302 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.302 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.302 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:43:39.302 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:01.936404+0000", 2023-10-01T10:43:39.302 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00010912399999999999 2023-10-01T10:43:39.302 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:43:39.302 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:43:39.302 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:43:39.302 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.303 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.303 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.35649.0:130143 9.6 e135/128 9:6efe31b1:::benchmark_data_smithi078_113121_object18328:head v 135'8277, mlcod=135'8274)", 2023-10-01T10:43:39.303 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:43:01.916127+0000", 2023-10-01T10:43:39.303 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 37.294994406999997, 2023-10-01T10:43:39.303 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.020398197, 2023-10-01T10:43:39.303 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:43:39.303 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:43:39.303 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:43:39.303 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.303 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:43:39.303 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:01.916127+0000", 2023-10-01T10:43:39.303 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:43:39.304 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.304 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.304 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:43:39.304 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:01.916127+0000", 2023-10-01T10:43:39.304 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:43:39.304 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.304 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.304 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:43:39.304 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:01.916126+0000", 2023-10-01T10:43:39.304 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999995 2023-10-01T10:43:39.304 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.304 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.305 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:43:39.305 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:01.916130+0000", 2023-10-01T10:43:39.305 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.8689999999999997e-06 2023-10-01T10:43:39.305 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.305 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.305 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:43:39.305 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:01.916136+0000", 2023-10-01T10:43:39.305 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.8200000000000002e-06 2023-10-01T10:43:39.305 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.305 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.305 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:43:39.306 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:01.916141+0000", 2023-10-01T10:43:39.306 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.2299999999999999e-06 2023-10-01T10:43:39.306 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.306 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.306 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:43:39.306 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:01.921300+0000", 2023-10-01T10:43:39.306 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0051588780000000004 2023-10-01T10:43:39.306 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.306 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.306 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:43:39.306 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:01.921329+0000", 2023-10-01T10:43:39.307 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.8764000000000001e-05 2023-10-01T10:43:39.307 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.307 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.307 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:43:39.307 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:01.936411+0000", 2023-10-01T10:43:39.307 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.015082257999999999 2023-10-01T10:43:39.307 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.307 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.307 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:43:39.307 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:01.936525+0000", 2023-10-01T10:43:39.307 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000113671 2023-10-01T10:43:39.307 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:43:39.307 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:43:39.308 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:43:39.308 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.308 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.308 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.35649.0:137250 9.6 e135/128 9:60fbcc3c:::benchmark_data_smithi078_113121_object25435:head v 135'8741, mlcod=135'8739)", 2023-10-01T10:43:39.308 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:43:11.188634+0000", 2023-10-01T10:43:39.308 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 28.022486601000001, 2023-10-01T10:43:39.308 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.020130631, 2023-10-01T10:43:39.308 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:43:39.308 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:43:39.308 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:43:39.308 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.308 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:43:39.309 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:11.188634+0000", 2023-10-01T10:43:39.309 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:43:39.309 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.309 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.309 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:43:39.309 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:11.188634+0000", 2023-10-01T10:43:39.309 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:43:39.309 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.309 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.309 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:43:39.309 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:11.188633+0000", 2023-10-01T10:43:39.309 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999986 2023-10-01T10:43:39.310 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.310 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.310 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:43:39.310 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:11.188649+0000", 2023-10-01T10:43:39.310 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.5585000000000001e-05 2023-10-01T10:43:39.310 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.310 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.310 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:43:39.310 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:11.188672+0000", 2023-10-01T10:43:39.310 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.3870999999999999e-05 2023-10-01T10:43:39.310 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.311 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.311 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:43:39.311 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:11.188685+0000", 2023-10-01T10:43:39.311 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.2119e-05 2023-10-01T10:43:39.311 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.311 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.311 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:43:39.311 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:11.191131+0000", 2023-10-01T10:43:39.311 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0024463010000000001 2023-10-01T10:43:39.311 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.311 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.311 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:43:39.312 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:11.191162+0000", 2023-10-01T10:43:39.312 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.1655000000000001e-05 2023-10-01T10:43:39.312 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.312 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.312 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:43:39.312 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:11.208687+0000", 2023-10-01T10:43:39.312 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.017524794999999999 2023-10-01T10:43:39.312 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.312 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.312 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:43:39.312 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:11.208765+0000", 2023-10-01T10:43:39.312 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.7668000000000002e-05 2023-10-01T10:43:39.313 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:43:39.313 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:43:39.313 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:43:39.313 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.313 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.313 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.35649.0:157327 9.0 e137/128 9:0733c66e:::benchmark_data_smithi078_113121_object45512:head v 137'9797, mlcod=137'9794)", 2023-10-01T10:43:39.313 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:43:38.287341+0000", 2023-10-01T10:43:39.313 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 0.92377951199999997, 2023-10-01T10:43:39.313 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.025467046, 2023-10-01T10:43:39.313 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:43:39.313 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:43:39.313 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:43:39.314 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.314 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:43:39.315 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:38.287341+0000", 2023-10-01T10:43:39.315 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:43:39.315 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.315 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.315 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:43:39.315 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:38.287341+0000", 2023-10-01T10:43:39.315 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:43:39.315 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.315 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.315 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:43:39.316 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:38.287340+0000", 2023-10-01T10:43:39.316 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:43:39.316 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.316 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.316 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:43:39.316 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:38.287350+0000", 2023-10-01T10:43:39.316 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.6870000000000006e-06 2023-10-01T10:43:39.316 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.316 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.316 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:43:39.316 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:38.287368+0000", 2023-10-01T10:43:39.316 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.8247e-05 2023-10-01T10:43:39.316 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.317 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.317 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:43:39.317 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:38.287378+0000", 2023-10-01T10:43:39.317 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.006e-05 2023-10-01T10:43:39.317 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.317 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.317 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:43:39.317 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:38.287438+0000", 2023-10-01T10:43:39.317 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.9369e-05 2023-10-01T10:43:39.317 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.317 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.317 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:43:39.318 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:38.287465+0000", 2023-10-01T10:43:39.318 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.7756999999999999e-05 2023-10-01T10:43:39.318 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.318 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.318 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:43:39.318 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:38.312687+0000", 2023-10-01T10:43:39.318 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.025221337 2023-10-01T10:43:39.318 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:43:39.318 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:43:39.318 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:43:39.318 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:38.312808+0000", 2023-10-01T10:43:39.318 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000121746 2023-10-01T10:43:39.319 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:43:39.319 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:43:39.319 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:43:39.319 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:43:39.319 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:43:39.319 INFO:teuthology.orchestra.run.smithi078.stdout:} 2023-10-01T10:43:39.319 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:39.351 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:39.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:39.358 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:43:39.407 INFO:teuthology.orchestra.run.smithi078.stderr:reweighted osd.1 to 1 (10000) 2023-10-01T10:43:39.459 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:39.479 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:39.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:39.481 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:39.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:39 smithi078 bash[75299]: cluster 2023-10-01T10:43:37.778895+0000 mgr.y (mgr.25072) 777 : cluster 0 pgmap v689: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 698 op/s 2023-10-01T10:43:39.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:39 smithi078 bash[75299]: audit 2023-10-01T10:43:39.075261+0000 mon.a (mon.0) 990 : audit 1 from='client.? 172.21.15.78:0/2572033226' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T10:43:39.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:39 smithi078 bash[78274]: cluster 2023-10-01T10:43:37.778895+0000 mgr.y (mgr.25072) 777 : cluster 0 pgmap v689: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 698 op/s 2023-10-01T10:43:39.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:39 smithi078 bash[78274]: audit 2023-10-01T10:43:39.075261+0000 mon.a (mon.0) 990 : audit 1 from='client.? 172.21.15.78:0/2572033226' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T10:43:39.582 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:39.601 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:39.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:39.604 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:39.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:39 smithi173 bash[58816]: cluster 2023-10-01T10:43:37.778895+0000 mgr.y (mgr.25072) 777 : cluster 0 pgmap v689: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 698 op/s 2023-10-01T10:43:39.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:39 smithi173 bash[58816]: audit 2023-10-01T10:43:39.075261+0000 mon.a (mon.0) 990 : audit 1 from='client.? 172.21.15.78:0/2572033226' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T10:43:39.705 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:39.745 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:39.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:39.751 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:43:39.852 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:39.886 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:39.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:39.891 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:43:39.992 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:40.025 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:40.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:40.028 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:43:40.129 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:40.155 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:40.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:40.163 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:40.264 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:40.297 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:40.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:40.306 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:43:40.407 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:40.412 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:40 smithi078 bash[75299]: audit 2023-10-01T10:43:39.394640+0000 mon.a (mon.0) 991 : audit 1 from='client.? 172.21.15.78:0/2572033226' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-01T10:43:40.412 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:40 smithi078 bash[75299]: cluster 2023-10-01T10:43:39.394798+0000 mon.a (mon.0) 992 : cluster 0 osdmap e138: 8 total, 8 up, 8 in 2023-10-01T10:43:40.413 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:43:40 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:43:40] "GET /metrics HTTP/1.1" 200 36249 "" "Prometheus/2.33.4" 2023-10-01T10:43:40.418 INFO:teuthology.orchestra.run.smithi078.stderr:nodeep-scrub is unset 2023-10-01T10:43:40.429 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:40.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:40.432 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:43:40.533 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:40.570 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:40.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:40.575 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:43:40.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:40 smithi173 bash[58816]: audit 2023-10-01T10:43:39.394640+0000 mon.a (mon.0) 991 : audit 1 from='client.? 172.21.15.78:0/2572033226' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-01T10:43:40.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:40 smithi173 bash[58816]: cluster 2023-10-01T10:43:39.394798+0000 mon.a (mon.0) 992 : cluster 0 osdmap e138: 8 total, 8 up, 8 in 2023-10-01T10:43:40.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:40 smithi173 bash[58816]: cluster 2023-10-01T10:43:39.400718+0000 osd.7 (osd.7) 85 : cluster 3 failed to encode map e138 with expected crc 2023-10-01T10:43:40.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:40 smithi173 bash[58816]: cluster 2023-10-01T10:43:39.400978+0000 osd.5 (osd.5) 66 : cluster 3 failed to encode map e138 with expected crc 2023-10-01T10:43:40.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:40 smithi173 bash[58816]: audit 2023-10-01T10:43:39.406239+0000 mon.a (mon.0) 993 : audit 1 from='client.? 172.21.15.78:0/3000649013' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:43:40.676 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:40.700 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:40.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:40.702 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:43:40.784 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:40 smithi078 bash[78274]: audit 2023-10-01T10:43:39.394640+0000 mon.a (mon.0) 991 : audit 1 from='client.? 172.21.15.78:0/2572033226' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-01T10:43:40.784 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:40 smithi078 bash[78274]: cluster 2023-10-01T10:43:39.394798+0000 mon.a (mon.0) 992 : cluster 0 osdmap e138: 8 total, 8 up, 8 in 2023-10-01T10:43:40.784 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:40 smithi078 bash[78274]: cluster 2023-10-01T10:43:39.400718+0000 osd.7 (osd.7) 85 : cluster 3 failed to encode map e138 with expected crc 2023-10-01T10:43:40.784 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:40 smithi078 bash[78274]: cluster 2023-10-01T10:43:39.400978+0000 osd.5 (osd.5) 66 : cluster 3 failed to encode map e138 with expected crc 2023-10-01T10:43:40.784 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:40 smithi078 bash[78274]: audit 2023-10-01T10:43:39.406239+0000 mon.a (mon.0) 993 : audit 1 from='client.? 172.21.15.78:0/3000649013' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:43:40.784 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:40 smithi078 bash[75299]: cluster 2023-10-01T10:43:39.400718+0000 osd.7 (osd.7) 85 : cluster 3 failed to encode map e138 with expected crc 2023-10-01T10:43:40.785 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:40 smithi078 bash[75299]: cluster 2023-10-01T10:43:39.400978+0000 osd.5 (osd.5) 66 : cluster 3 failed to encode map e138 with expected crc 2023-10-01T10:43:40.785 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:40 smithi078 bash[75299]: audit 2023-10-01T10:43:39.406239+0000 mon.a (mon.0) 993 : audit 1 from='client.? 172.21.15.78:0/3000649013' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:43:40.803 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:40.838 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:40.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:40.841 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:43:40.941 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:40.986 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:40.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:40.991 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:43:41.092 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:41.130 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:41.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:41.133 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:43:41.234 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:41.261 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:41.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:41.267 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:43:41.368 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:41.395 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:41.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:41.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:43:41.415 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:41 smithi078 bash[75299]: cluster 2023-10-01T10:43:39.779334+0000 mgr.y (mgr.25072) 778 : cluster 0 pgmap v691: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 724 op/s 2023-10-01T10:43:41.415 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:41 smithi078 bash[75299]: cluster 2023-10-01T10:43:40.398788+0000 mon.a (mon.0) 994 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T10:43:41.416 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:41 smithi078 bash[75299]: cluster 2023-10-01T10:43:40.398826+0000 mon.a (mon.0) 995 : cluster 1 Cluster is now healthy 2023-10-01T10:43:41.416 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:41 smithi078 bash[75299]: cluster 2023-10-01T10:43:40.405669+0000 osd.6 (osd.6) 52 : cluster 3 failed to encode map e139 with expected crc 2023-10-01T10:43:41.416 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:41 smithi078 bash[75299]: audit 2023-10-01T10:43:40.406260+0000 mon.a (mon.0) 996 : audit 1 from='client.? 172.21.15.78:0/3000649013' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T10:43:41.416 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:41 smithi078 bash[75299]: cluster 2023-10-01T10:43:40.406443+0000 mon.a (mon.0) 997 : cluster 0 osdmap e139: 8 total, 8 up, 8 in 2023-10-01T10:43:41.416 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:41 smithi078 bash[75299]: cluster 2023-10-01T10:43:40.407091+0000 osd.7 (osd.7) 86 : cluster 3 failed to encode map e139 with expected crc 2023-10-01T10:43:41.498 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:41.530 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:41.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:41.535 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:43:41.636 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:41.665 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:41.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:41.669 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:41 smithi078 bash[78274]: cluster 2023-10-01T10:43:39.779334+0000 mgr.y (mgr.25072) 778 : cluster 0 pgmap v691: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 724 op/s 2023-10-01T10:43:41.669 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:41 smithi078 bash[78274]: cluster 2023-10-01T10:43:40.398788+0000 mon.a (mon.0) 994 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T10:43:41.669 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:41 smithi078 bash[78274]: cluster 2023-10-01T10:43:40.398826+0000 mon.a (mon.0) 995 : cluster 1 Cluster is now healthy 2023-10-01T10:43:41.669 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:41 smithi078 bash[78274]: cluster 2023-10-01T10:43:40.405669+0000 osd.6 (osd.6) 52 : cluster 3 failed to encode map e139 with expected crc 2023-10-01T10:43:41.669 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:41 smithi078 bash[78274]: audit 2023-10-01T10:43:40.406260+0000 mon.a (mon.0) 996 : audit 1 from='client.? 172.21.15.78:0/3000649013' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T10:43:41.669 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:41 smithi078 bash[78274]: cluster 2023-10-01T10:43:40.406443+0000 mon.a (mon.0) 997 : cluster 0 osdmap e139: 8 total, 8 up, 8 in 2023-10-01T10:43:41.669 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:41 smithi078 bash[78274]: cluster 2023-10-01T10:43:40.407091+0000 osd.7 (osd.7) 86 : cluster 3 failed to encode map e139 with expected crc 2023-10-01T10:43:41.670 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:41.770 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:41.795 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:41.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:41.798 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:43:41.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:41 smithi173 bash[58816]: cluster 2023-10-01T10:43:39.779334+0000 mgr.y (mgr.25072) 778 : cluster 0 pgmap v691: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 724 op/s 2023-10-01T10:43:41.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:41 smithi173 bash[58816]: cluster 2023-10-01T10:43:40.398788+0000 mon.a (mon.0) 994 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T10:43:41.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:41 smithi173 bash[58816]: cluster 2023-10-01T10:43:40.398826+0000 mon.a (mon.0) 995 : cluster 1 Cluster is now healthy 2023-10-01T10:43:41.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:41 smithi173 bash[58816]: cluster 2023-10-01T10:43:40.405669+0000 osd.6 (osd.6) 52 : cluster 3 failed to encode map e139 with expected crc 2023-10-01T10:43:41.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:41 smithi173 bash[58816]: audit 2023-10-01T10:43:40.406260+0000 mon.a (mon.0) 996 : audit 1 from='client.? 172.21.15.78:0/3000649013' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T10:43:41.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:41 smithi173 bash[58816]: cluster 2023-10-01T10:43:40.406443+0000 mon.a (mon.0) 997 : cluster 0 osdmap e139: 8 total, 8 up, 8 in 2023-10-01T10:43:41.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:41 smithi173 bash[58816]: cluster 2023-10-01T10:43:40.407091+0000 osd.7 (osd.7) 86 : cluster 3 failed to encode map e139 with expected crc 2023-10-01T10:43:41.898 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:41.925 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:41.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:41.931 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:43:42.032 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:42.063 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:42.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:42.065 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:43:42.166 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:42.202 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:42.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:42.207 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:43:42.307 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:42.332 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:42.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:42.334 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:43:42.435 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:42.469 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:42.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:42.472 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:43:42.573 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:42.595 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:42.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:42.600 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:43:42.701 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:42.733 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:42.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:42.736 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:43:42.821 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-01T10:43:42.838 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:42.850 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-01T10:43:42.884 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:42.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:42.889 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:43:42.989 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:43.025 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:43.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:43.030 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:43:43.084 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T10:43:43.131 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:43.166 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:43.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:43.168 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:43:43.269 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:43.318 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:43.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:43.325 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:43.425 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:43.456 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:43.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:43.464 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:43:43.564 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:43.573 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:43 smithi078 bash[78274]: cluster 2023-10-01T10:43:41.780100+0000 mgr.y (mgr.25072) 779 : cluster 0 pgmap v693: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 720 op/s 2023-10-01T10:43:43.573 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:43 smithi078 bash[75299]: cluster 2023-10-01T10:43:41.780100+0000 mgr.y (mgr.25072) 779 : cluster 0 pgmap v693: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 720 op/s 2023-10-01T10:43:43.594 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:43.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:43.596 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:43.697 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:43.720 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:43 smithi173 bash[58816]: cluster 2023-10-01T10:43:41.780100+0000 mgr.y (mgr.25072) 779 : cluster 0 pgmap v693: 89 pgs: 89 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 720 op/s 2023-10-01T10:43:43.746 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:43.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:43.751 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:43:43.852 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:43.878 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:43.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:43.883 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:43:43.984 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:44.017 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:44.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:44.023 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:43:44.123 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:44.155 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:44.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:44.159 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:43:44.260 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:44.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:44.296 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:44.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:44.397 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:44.420 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:44.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:44.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:43:44.524 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:44.556 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:44.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:44.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:43:44.661 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:44.686 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:44.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:44.694 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:43:44.795 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:44.823 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:44.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:44.826 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:43:44.853 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T10:43:44.927 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:44.959 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:44.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:44.963 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:43:45.064 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:45.095 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:45.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:45.104 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:43:45.204 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:45.231 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:45.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:45.233 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:43:45.334 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:45.364 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:45.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:45.371 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:43:45.472 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:45.496 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:45.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:45.499 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:43:45.599 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:45.607 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:45 smithi078 bash[78274]: cluster 2023-10-01T10:43:43.780608+0000 mgr.y (mgr.25072) 780 : cluster 0 pgmap v694: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 702 op/s 2023-10-01T10:43:45.607 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:45 smithi078 bash[75299]: cluster 2023-10-01T10:43:43.780608+0000 mgr.y (mgr.25072) 780 : cluster 0 pgmap v694: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 702 op/s 2023-10-01T10:43:45.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:45 smithi173 bash[58816]: cluster 2023-10-01T10:43:43.780608+0000 mgr.y (mgr.25072) 780 : cluster 0 pgmap v694: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 702 op/s 2023-10-01T10:43:45.632 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:45.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:45.636 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:43:45.737 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:45.757 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:45.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:45.762 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:43:45.865 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:45.898 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:45.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:45.902 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:46.003 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:46.041 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:46.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:46.044 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:43:46.145 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:46.184 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:46.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:46.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:43:46.288 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:46.318 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:46.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:46.322 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:43:46.423 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:46.455 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:46.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:46.459 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:43:46.560 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:46.594 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:46.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:46.598 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:43:46.698 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:46.725 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:46.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:46.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:46.828 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:46.870 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:46.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:46.872 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:46.973 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:47.014 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:47.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:47.018 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:43:47.119 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:47.141 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:47.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:47.144 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:43:47.245 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:47.282 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:47.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:47.284 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:43:47.385 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:47.408 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:47.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:47.411 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:43:47.511 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:47.570 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:47.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:47.575 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:43:47.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:47 smithi078 bash[75299]: cluster 2023-10-01T10:43:45.781186+0000 mgr.y (mgr.25072) 781 : cluster 0 pgmap v695: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 745 op/s 2023-10-01T10:43:47.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:47 smithi078 bash[75299]: audit 2023-10-01T10:43:47.426456+0000 mon.a (mon.0) 998 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:43:47.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:47 smithi078 bash[78274]: cluster 2023-10-01T10:43:45.781186+0000 mgr.y (mgr.25072) 781 : cluster 0 pgmap v695: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 745 op/s 2023-10-01T10:43:47.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:47 smithi078 bash[78274]: audit 2023-10-01T10:43:47.426456+0000 mon.a (mon.0) 998 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:43:47.676 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:47.684 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:43:47.685 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:43:47.686 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:43:47.713 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:47.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:47.719 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:43:47.819 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:47.828 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:47 smithi173 bash[58816]: cluster 2023-10-01T10:43:45.781186+0000 mgr.y (mgr.25072) 781 : cluster 0 pgmap v695: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 745 op/s 2023-10-01T10:43:47.829 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:47 smithi173 bash[58816]: audit 2023-10-01T10:43:47.426456+0000 mon.a (mon.0) 998 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:43:47.846 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:47.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:47.848 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:43:47.949 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:47.974 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:47.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:47.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:43:48.080 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:48.112 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:48.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:48.118 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:43:48.219 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:48.249 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:48.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:48.252 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:43:48.353 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:48.384 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:48.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:48.391 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:43:48.491 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:48.532 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:48.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:48.541 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:43:48.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:48 smithi078 bash[78274]: audit 2023-10-01T10:43:47.428978+0000 mon.a (mon.0) 999 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:43:48.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:48 smithi078 bash[75299]: audit 2023-10-01T10:43:47.428978+0000 mon.a (mon.0) 999 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:43:48.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:48 smithi173 bash[58816]: audit 2023-10-01T10:43:47.428978+0000 mon.a (mon.0) 999 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:43:48.641 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:48.672 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:48.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:48.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:43:48.701 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:43:48.776 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:48.815 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:48.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:48.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:43:48.918 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:48.947 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:48.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:48.950 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:43:49.050 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:49.093 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:49.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:49.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:43:49.200 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:49.225 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:49.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:49.230 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:43:49.331 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:49.366 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:49.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:49.372 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:43:49.472 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:49.507 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:49.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:49.511 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:43:49.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:49 smithi078 bash[78274]: cluster 2023-10-01T10:43:47.781763+0000 mgr.y (mgr.25072) 782 : cluster 0 pgmap v696: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 553 op/s 2023-10-01T10:43:49.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:49 smithi078 bash[75299]: cluster 2023-10-01T10:43:47.781763+0000 mgr.y (mgr.25072) 782 : cluster 0 pgmap v696: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 553 op/s 2023-10-01T10:43:49.612 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:49.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:49 smithi173 bash[58816]: cluster 2023-10-01T10:43:47.781763+0000 mgr.y (mgr.25072) 782 : cluster 0 pgmap v696: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 553 op/s 2023-10-01T10:43:49.642 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:49.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:49.646 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:43:49.747 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:49.773 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:49.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:49.775 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:43:49.876 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:49.901 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:49.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:49.903 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:43:50.004 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:50.052 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:50.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:50.058 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:43:50.161 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:50.196 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:50.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:50.199 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:43:50.300 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:50.324 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:50.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:50.329 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:43:50.430 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:50.458 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:50.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:50.461 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:43:50.561 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:50.582 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:43:50 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:43:50] "GET /metrics HTTP/1.1" 200 36251 "" "Prometheus/2.33.4" 2023-10-01T10:43:50.590 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:50.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:50.597 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:43:50.697 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:50.727 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:50.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:50.736 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:43:50.837 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:50.872 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:50.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:50.876 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:43:50.977 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:51.005 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:51.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:51.008 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:43:51.109 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:51.154 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:51.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:51.159 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:43:51.260 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:51.285 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:51.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:51.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:43:51.390 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:51.416 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:51.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:51.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:43:51.519 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:51.536 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:51.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:51.538 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:43:51.560 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:51 smithi078 bash[75299]: cluster 2023-10-01T10:43:49.782451+0000 mgr.y (mgr.25072) 783 : cluster 0 pgmap v697: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 691 op/s 2023-10-01T10:43:51.639 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:51.666 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:51.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:51.672 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:43:51.773 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:51.784 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:51 smithi173 bash[58816]: cluster 2023-10-01T10:43:49.782451+0000 mgr.y (mgr.25072) 783 : cluster 0 pgmap v697: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 691 op/s 2023-10-01T10:43:51.801 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:51.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:51.804 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:43:51.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:51 smithi078 bash[78274]: cluster 2023-10-01T10:43:49.782451+0000 mgr.y (mgr.25072) 783 : cluster 0 pgmap v697: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 691 op/s 2023-10-01T10:43:51.904 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:51.951 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:51.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:51.955 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:43:52.056 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:52.090 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:52.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:52.096 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:43:52.197 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:52.223 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:52.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:52.226 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:43:52.327 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:52.354 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:52.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:52.359 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:43:52.459 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:52.486 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:52.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:52.488 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:43:52.589 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:52.618 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:52.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:52.623 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:43:52.724 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:52.753 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:52.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:52.757 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:52.858 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:52.896 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:52.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:52.899 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:43:53.000 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:53.031 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:53.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:53.035 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:43:53.136 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:53.164 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:53.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:53.168 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:43:53.269 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:53.306 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:53.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:53.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:43:53.412 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:53.441 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:53.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:53.444 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:43:53.544 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:53.555 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:53 smithi173 bash[58816]: cluster 2023-10-01T10:43:51.783212+0000 mgr.y (mgr.25072) 784 : cluster 0 pgmap v698: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 720 op/s 2023-10-01T10:43:53.577 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:53.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:53.579 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:43:53.680 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:53.711 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:53.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:53.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:43:53.814 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:53.822 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:53 smithi078 bash[75299]: cluster 2023-10-01T10:43:51.783212+0000 mgr.y (mgr.25072) 784 : cluster 0 pgmap v698: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 720 op/s 2023-10-01T10:43:53.823 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:53 smithi078 bash[78274]: cluster 2023-10-01T10:43:51.783212+0000 mgr.y (mgr.25072) 784 : cluster 0 pgmap v698: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 720 op/s 2023-10-01T10:43:53.838 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:53.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:53.843 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:43:53.944 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:53.978 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:53.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:53.982 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:43:54.082 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:54.115 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:54.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:54.122 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:43:54.223 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:54.249 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:54.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:54.254 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:43:54.354 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:54.385 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:54.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:54.388 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:43:54.489 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:54.518 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:54.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:54.523 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:43:54.623 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:54.654 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:54.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:54.659 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:43:54.760 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:54.785 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:54.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:54.787 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:43:54.888 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:54.922 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:54.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:54.926 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:43:55.027 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:55.062 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:55.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:55.070 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:43:55.171 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:55.211 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:55.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:55.216 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:43:55.317 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:55.348 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:55.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:55.353 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:43:55.454 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:55.483 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:55.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:55.488 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:43:55.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:55 smithi078 bash[78274]: cluster 2023-10-01T10:43:53.783842+0000 mgr.y (mgr.25072) 785 : cluster 0 pgmap v699: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2023-10-01T10:43:55.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:55 smithi078 bash[75299]: cluster 2023-10-01T10:43:53.783842+0000 mgr.y (mgr.25072) 785 : cluster 0 pgmap v699: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2023-10-01T10:43:55.589 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:55.618 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:55.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:55 smithi173 bash[58816]: cluster 2023-10-01T10:43:53.783842+0000 mgr.y (mgr.25072) 785 : cluster 0 pgmap v699: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2023-10-01T10:43:55.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:55.627 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:43:55.728 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:55.757 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:43:55.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:55.764 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:43:55.865 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:55.899 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:55.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:55.909 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:43:56.009 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:56.038 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:56.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:56.041 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:43:56.142 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:56.171 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:56.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:56.174 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:43:56.275 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:56.300 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:56.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:56.307 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:43:56.408 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:56.438 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:56.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:56.444 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:56.545 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:56.569 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:56.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:56.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:56.673 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:56.717 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:56.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:56.720 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:43:56.720 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:56 smithi173 bash[58816]: audit 2023-10-01T10:43:56.077638+0000 mon.a (mon.0) 1000 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:43:56.821 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:56.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:56 smithi078 bash[75299]: audit 2023-10-01T10:43:56.077638+0000 mon.a (mon.0) 1000 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:43:56.836 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:56 smithi078 bash[78274]: audit 2023-10-01T10:43:56.077638+0000 mon.a (mon.0) 1000 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:43:56.853 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:56.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:56.860 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:43:56.961 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:56.999 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:57.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:57.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:43:57.104 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:57.135 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:57.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:57.139 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:57.240 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:57.273 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:57.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:57.276 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:43:57.377 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:57.411 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:57.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:57.413 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:57.514 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:57.545 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:43:57.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:57.548 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:43:57.649 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:57.672 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:57.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:57.677 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:43:57.778 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:57.788 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:57 smithi173 bash[58816]: cluster 2023-10-01T10:43:55.784365+0000 mgr.y (mgr.25072) 786 : cluster 0 pgmap v700: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 683 op/s 2023-10-01T10:43:57.819 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:57.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:57.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:43:57.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:57 smithi078 bash[78274]: cluster 2023-10-01T10:43:55.784365+0000 mgr.y (mgr.25072) 786 : cluster 0 pgmap v700: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 683 op/s 2023-10-01T10:43:57.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:57 smithi078 bash[75299]: cluster 2023-10-01T10:43:55.784365+0000 mgr.y (mgr.25072) 786 : cluster 0 pgmap v700: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 683 op/s 2023-10-01T10:43:57.923 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:57.952 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:43:57.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:57.955 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:43:58.056 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:58.091 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:58.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:58.096 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:43:58.196 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:58.229 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:58.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:58.232 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:43:58.332 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:58.352 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:58.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:58.354 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:43:58.455 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:58.489 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:58.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:58.491 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:43:58.592 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:58.621 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:58.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:58.624 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:58.725 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:58.763 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:58.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:58.765 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:43:58.866 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:58.893 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:58.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:58.895 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:58.996 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:59.019 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:59.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:59.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:43:59.125 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:59.176 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:59.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:59.178 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:43:59.279 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:59.312 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:43:59.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:59.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:43:59.416 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:59.451 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:43:59.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:59.456 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:43:59.557 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:59.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:59.591 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:43:59.591 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:43:59.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:43:59 smithi173 bash[58816]: cluster 2023-10-01T10:43:57.784855+0000 mgr.y (mgr.25072) 787 : cluster 0 pgmap v701: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 647 op/s 2023-10-01T10:43:59.692 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:59.729 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:43:59.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:59.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:43:59.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:43:59 smithi078 bash[75299]: cluster 2023-10-01T10:43:57.784855+0000 mgr.y (mgr.25072) 787 : cluster 0 pgmap v701: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 647 op/s 2023-10-01T10:43:59.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:43:59 smithi078 bash[78274]: cluster 2023-10-01T10:43:57.784855+0000 mgr.y (mgr.25072) 787 : cluster 0 pgmap v701: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 647 op/s 2023-10-01T10:43:59.835 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:59.864 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:43:59.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:43:59.867 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:43:59.968 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:00.007 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:00.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:00.013 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:44:00.114 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:00.138 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:00.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:00.142 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:44:00.242 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:00.273 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:00.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:00.276 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:00.376 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:00.409 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:00.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:00.412 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:44:00.513 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:00.526 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:44:00 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:44:00] "GET /metrics HTTP/1.1" 200 36251 "" "Prometheus/2.33.4" 2023-10-01T10:44:00.533 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:00.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:00.538 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:00.639 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:00.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:00.678 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:00.678 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:44:00.779 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:00.814 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:00.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:00.817 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:44:00.917 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:00.945 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:00.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:00.950 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:44:01.051 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:01.088 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:01.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:01.097 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:44:01.198 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:01.224 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:01.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:01.226 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:01.327 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:01.358 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:01.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:01.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:01.464 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:01.495 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:01.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:01.500 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:44:01.600 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:01.640 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:01.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:01.642 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:44:01.743 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:01.752 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:01 smithi173 bash[58816]: cluster 2023-10-01T10:43:59.785546+0000 mgr.y (mgr.25072) 788 : cluster 0 pgmap v702: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 759 op/s 2023-10-01T10:44:01.767 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:01.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:01.770 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:44:01.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:01 smithi078 bash[75299]: cluster 2023-10-01T10:43:59.785546+0000 mgr.y (mgr.25072) 788 : cluster 0 pgmap v702: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 759 op/s 2023-10-01T10:44:01.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:01 smithi078 bash[78274]: cluster 2023-10-01T10:43:59.785546+0000 mgr.y (mgr.25072) 788 : cluster 0 pgmap v702: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 759 op/s 2023-10-01T10:44:01.870 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:01.933 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:01.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:01.937 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:44:02.037 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:02.072 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:02.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:02.074 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:44:02.175 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:02.199 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:02.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:02.202 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:02.302 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:02.355 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:02.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:02.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:44:02.488 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:02.537 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:02.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:02.540 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:02.641 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:02.723 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:02.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:02.731 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:44:02.831 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:02.837 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T10:44:02.882 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:02.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:02.885 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:44:02.914 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T10:44:02.978 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T10:44:02.985 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:03.020 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:03.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:03.028 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:44:03.037 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T10:44:03.080 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T10:44:03.119 INFO:teuthology.orchestra.run.smithi078.stdout:ERROR: (22) Invalid argument 2023-10-01T10:44:03.119 INFO:teuthology.orchestra.run.smithi078.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T10:44:03.129 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:03.133 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T10:44:03.166 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:03.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:03.169 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:44:03.182 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T10:44:03.210 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T10:44:03.223 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T10:44:03.224 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T10:44:03.224 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T10:44:03.224 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T10:44:03.224 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T10:44:03.224 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T10:44:03.224 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T10:44:03.224 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T10:44:03.270 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:03.299 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:03.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:03.302 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:44:03.403 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:03.434 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:03.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:03.439 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:44:03.540 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:03.551 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:03 smithi078 bash[75299]: cluster 2023-10-01T10:44:01.786074+0000 mgr.y (mgr.25072) 789 : cluster 0 pgmap v703: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 683 op/s 2023-10-01T10:44:03.551 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:03 smithi078 bash[75299]: audit 2023-10-01T10:44:02.337185+0000 mon.a (mon.0) 1001 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:44:03.551 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:03 smithi078 bash[75299]: audit 2023-10-01T10:44:02.343956+0000 mon.a (mon.0) 1002 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:44:03.551 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:03 smithi078 bash[75299]: audit 2023-10-01T10:44:02.417765+0000 mon.a (mon.0) 1003 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:44:03.551 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:03 smithi078 bash[75299]: audit 2023-10-01T10:44:02.760650+0000 mon.a (mon.0) 1004 : audit 0 from='client.? 172.21.15.78:0/3490892987' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:44:03.551 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:03 smithi078 bash[75299]: audit 2023-10-01T10:44:03.174616+0000 mon.a (mon.0) 1005 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:44:03.552 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:03 smithi078 bash[75299]: audit 2023-10-01T10:44:03.194060+0000 mon.a (mon.0) 1006 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:44:03.552 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:03 smithi078 bash[75299]: audit 2023-10-01T10:44:03.231556+0000 mon.a (mon.0) 1007 : audit 1 from='client.? 172.21.15.78:0/3677681680' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-01T10:44:03.552 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:03 smithi078 bash[78274]: cluster 2023-10-01T10:44:01.786074+0000 mgr.y (mgr.25072) 789 : cluster 0 pgmap v703: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 683 op/s 2023-10-01T10:44:03.552 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:03 smithi078 bash[78274]: audit 2023-10-01T10:44:02.337185+0000 mon.a (mon.0) 1001 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:44:03.552 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:03 smithi078 bash[78274]: audit 2023-10-01T10:44:02.343956+0000 mon.a (mon.0) 1002 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:44:03.552 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:03 smithi078 bash[78274]: audit 2023-10-01T10:44:02.417765+0000 mon.a (mon.0) 1003 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:44:03.552 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:03 smithi078 bash[78274]: audit 2023-10-01T10:44:02.760650+0000 mon.a (mon.0) 1004 : audit 0 from='client.? 172.21.15.78:0/3490892987' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:44:03.552 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:03 smithi078 bash[78274]: audit 2023-10-01T10:44:03.174616+0000 mon.a (mon.0) 1005 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:44:03.552 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:03 smithi078 bash[78274]: audit 2023-10-01T10:44:03.194060+0000 mon.a (mon.0) 1006 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:44:03.553 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:03 smithi078 bash[78274]: audit 2023-10-01T10:44:03.231556+0000 mon.a (mon.0) 1007 : audit 1 from='client.? 172.21.15.78:0/3677681680' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-01T10:44:03.580 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:03.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:03.583 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:44:03.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:03 smithi173 bash[58816]: cluster 2023-10-01T10:44:01.786074+0000 mgr.y (mgr.25072) 789 : cluster 0 pgmap v703: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 683 op/s 2023-10-01T10:44:03.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:03 smithi173 bash[58816]: audit 2023-10-01T10:44:02.337185+0000 mon.a (mon.0) 1001 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:44:03.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:03 smithi173 bash[58816]: audit 2023-10-01T10:44:02.343956+0000 mon.a (mon.0) 1002 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:44:03.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:03 smithi173 bash[58816]: audit 2023-10-01T10:44:02.417765+0000 mon.a (mon.0) 1003 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:44:03.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:03 smithi173 bash[58816]: audit 2023-10-01T10:44:02.760650+0000 mon.a (mon.0) 1004 : audit 0 from='client.? 172.21.15.78:0/3490892987' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:44:03.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:03 smithi173 bash[58816]: audit 2023-10-01T10:44:03.174616+0000 mon.a (mon.0) 1005 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:44:03.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:03 smithi173 bash[58816]: audit 2023-10-01T10:44:03.194060+0000 mon.a (mon.0) 1006 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:44:03.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:03 smithi173 bash[58816]: audit 2023-10-01T10:44:03.231556+0000 mon.a (mon.0) 1007 : audit 1 from='client.? 172.21.15.78:0/3677681680' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-01T10:44:03.683 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:03.715 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:03.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:03.721 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:44:03.821 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:03.866 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:03.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:03.871 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:44:03.972 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:04.016 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:04.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:04.021 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:44:04.122 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:04.157 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:04.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:04.163 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:44:04.198 INFO:teuthology.orchestra.run.smithi078.stderr:reweighted osd.2 to 1 (10000) 2023-10-01T10:44:04.206 INFO:teuthology.orchestra.run.smithi078.stderr:noscrub is set 2023-10-01T10:44:04.264 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:04.291 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:04.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:04.298 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:44:04.399 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:04.408 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:04 smithi173 bash[58816]: audit 2023-10-01T10:44:03.524207+0000 mon.c (mon.2) 141 : audit 1 from='client.? 172.21.15.78:0/1893082301' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:44:04.408 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:04 smithi173 bash[58816]: audit 2023-10-01T10:44:03.524771+0000 mon.a (mon.0) 1008 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:44:04.408 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:04 smithi173 bash[58816]: audit 2023-10-01T10:44:03.828024+0000 mon.a (mon.0) 1009 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:44:04.408 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:04 smithi173 bash[58816]: audit 2023-10-01T10:44:03.828853+0000 mon.a (mon.0) 1010 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:44:04.408 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:04 smithi173 bash[58816]: audit 2023-10-01T10:44:03.837005+0000 mon.a (mon.0) 1011 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:44:04.408 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:04 smithi173 bash[58816]: cluster 2023-10-01T10:44:04.179928+0000 mon.a (mon.0) 1012 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:44:04.408 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:04 smithi173 bash[58816]: audit 2023-10-01T10:44:04.188429+0000 mon.a (mon.0) 1013 : audit 1 from='client.? 172.21.15.78:0/3677681680' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-01T10:44:04.408 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:04 smithi173 bash[58816]: audit 2023-10-01T10:44:04.188603+0000 mon.a (mon.0) 1014 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T10:44:04.409 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:04 smithi173 bash[58816]: cluster 2023-10-01T10:44:04.191564+0000 mon.a (mon.0) 1015 : cluster 0 osdmap e140: 8 total, 8 up, 8 in 2023-10-01T10:44:04.433 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:04.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:04.435 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:44:04.536 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:04.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:04 smithi078 bash[78274]: audit 2023-10-01T10:44:03.524207+0000 mon.c (mon.2) 141 : audit 1 from='client.? 172.21.15.78:0/1893082301' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:44:04.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:04 smithi078 bash[78274]: audit 2023-10-01T10:44:03.524771+0000 mon.a (mon.0) 1008 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:44:04.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:04 smithi078 bash[78274]: audit 2023-10-01T10:44:03.828024+0000 mon.a (mon.0) 1009 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:44:04.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:04 smithi078 bash[78274]: audit 2023-10-01T10:44:03.828853+0000 mon.a (mon.0) 1010 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:44:04.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:04 smithi078 bash[78274]: audit 2023-10-01T10:44:03.837005+0000 mon.a (mon.0) 1011 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:44:04.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:04 smithi078 bash[78274]: cluster 2023-10-01T10:44:04.179928+0000 mon.a (mon.0) 1012 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:44:04.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:04 smithi078 bash[78274]: audit 2023-10-01T10:44:04.188429+0000 mon.a (mon.0) 1013 : audit 1 from='client.? 172.21.15.78:0/3677681680' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-01T10:44:04.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:04 smithi078 bash[78274]: audit 2023-10-01T10:44:04.188603+0000 mon.a (mon.0) 1014 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T10:44:04.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:04 smithi078 bash[78274]: cluster 2023-10-01T10:44:04.191564+0000 mon.a (mon.0) 1015 : cluster 0 osdmap e140: 8 total, 8 up, 8 in 2023-10-01T10:44:04.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:04 smithi078 bash[75299]: audit 2023-10-01T10:44:03.524207+0000 mon.c (mon.2) 141 : audit 1 from='client.? 172.21.15.78:0/1893082301' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:44:04.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:04 smithi078 bash[75299]: audit 2023-10-01T10:44:03.524771+0000 mon.a (mon.0) 1008 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:44:04.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:04 smithi078 bash[75299]: audit 2023-10-01T10:44:03.828024+0000 mon.a (mon.0) 1009 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:44:04.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:04 smithi078 bash[75299]: audit 2023-10-01T10:44:03.828853+0000 mon.a (mon.0) 1010 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:44:04.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:04 smithi078 bash[75299]: audit 2023-10-01T10:44:03.837005+0000 mon.a (mon.0) 1011 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:44:04.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:04 smithi078 bash[75299]: cluster 2023-10-01T10:44:04.179928+0000 mon.a (mon.0) 1012 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:44:04.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:04 smithi078 bash[75299]: audit 2023-10-01T10:44:04.188429+0000 mon.a (mon.0) 1013 : audit 1 from='client.? 172.21.15.78:0/3677681680' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-01T10:44:04.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:04 smithi078 bash[75299]: audit 2023-10-01T10:44:04.188603+0000 mon.a (mon.0) 1014 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T10:44:04.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:04 smithi078 bash[75299]: cluster 2023-10-01T10:44:04.191564+0000 mon.a (mon.0) 1015 : cluster 0 osdmap e140: 8 total, 8 up, 8 in 2023-10-01T10:44:04.584 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:04.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:04.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:44:04.688 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:04.713 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:04.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:04.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:44:04.815 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:04.857 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:04.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:04.861 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:44:04.961 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:04.987 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:04.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:04.989 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:44:05.090 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:05.120 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:05.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:05.124 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:44:05.225 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:05.252 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:05.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:05.255 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:44:05.355 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:05.356 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:05 smithi078 bash[75299]: cluster 2023-10-01T10:44:03.789731+0000 mgr.y (mgr.25072) 790 : cluster 0 pgmap v704: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 680 op/s 2023-10-01T10:44:05.356 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:05 smithi078 bash[75299]: cluster 2023-10-01T10:44:04.187555+0000 osd.5 (osd.5) 67 : cluster 3 failed to encode map e140 with expected crc 2023-10-01T10:44:05.357 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:05 smithi078 bash[75299]: cluster 2023-10-01T10:44:04.188119+0000 osd.7 (osd.7) 87 : cluster 3 failed to encode map e140 with expected crc 2023-10-01T10:44:05.381 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:05.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:05.384 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:44:05.485 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:05.494 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:05 smithi173 bash[58816]: cluster 2023-10-01T10:44:03.789731+0000 mgr.y (mgr.25072) 790 : cluster 0 pgmap v704: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 680 op/s 2023-10-01T10:44:05.494 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:05 smithi173 bash[58816]: cluster 2023-10-01T10:44:04.187555+0000 osd.5 (osd.5) 67 : cluster 3 failed to encode map e140 with expected crc 2023-10-01T10:44:05.494 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:05 smithi173 bash[58816]: cluster 2023-10-01T10:44:04.188119+0000 osd.7 (osd.7) 87 : cluster 3 failed to encode map e140 with expected crc 2023-10-01T10:44:05.518 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:05.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:05.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:44:05.627 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:05.635 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:05 smithi078 bash[78274]: cluster 2023-10-01T10:44:03.789731+0000 mgr.y (mgr.25072) 790 : cluster 0 pgmap v704: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 680 op/s 2023-10-01T10:44:05.635 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:05 smithi078 bash[78274]: cluster 2023-10-01T10:44:04.187555+0000 osd.5 (osd.5) 67 : cluster 3 failed to encode map e140 with expected crc 2023-10-01T10:44:05.635 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:05 smithi078 bash[78274]: cluster 2023-10-01T10:44:04.188119+0000 osd.7 (osd.7) 87 : cluster 3 failed to encode map e140 with expected crc 2023-10-01T10:44:05.650 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:05.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:05.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:05.754 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:05.788 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:05.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:05.793 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:44:05.894 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:05.929 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:05.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:05.935 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:44:06.036 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:06.062 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:06.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:06.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:44:06.158 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-01T10:44:06.168 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:06.198 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:06.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:06.200 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:44:06.301 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:06.339 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:06.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:06.342 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:44:06.443 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:06.468 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:06.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:06.471 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:44:06.572 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:06.604 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:06.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:06.607 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:44:06.707 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:06.743 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:06.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:06.749 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:44:06.849 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:06.880 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:06.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:06.882 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:44:06.983 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:07.011 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:07.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:07.014 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:07.114 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:07.147 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:07.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:07.152 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:44:07.252 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:07.261 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-01T10:44:07.285 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:07.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:07.291 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:07.392 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:07.431 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:07.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:07.436 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:44:07.537 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:07.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:07 smithi078 bash[75299]: cluster 2023-10-01T10:44:05.790135+0000 mgr.y (mgr.25072) 791 : cluster 0 pgmap v706: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 688 op/s 2023-10-01T10:44:07.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:07 smithi078 bash[78274]: cluster 2023-10-01T10:44:05.790135+0000 mgr.y (mgr.25072) 791 : cluster 0 pgmap v706: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 688 op/s 2023-10-01T10:44:07.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:07 smithi173 bash[58816]: cluster 2023-10-01T10:44:05.790135+0000 mgr.y (mgr.25072) 791 : cluster 0 pgmap v706: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 688 op/s 2023-10-01T10:44:07.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:07.789 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:07.789 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:44:07.890 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:07.924 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:07.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:07.929 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:44:08.029 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:08.066 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:08.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:08.071 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:44:08.172 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:08.200 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:08.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:08.202 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:44:08.303 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:08.317 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T10:44:08.341 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:08.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:08.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:08.448 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:08.481 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:08.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:08.487 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:44:08.587 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:08.632 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:08.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:08.639 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:44:08.739 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:08.761 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:08.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:08.767 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:44:08.867 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:08.903 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:08.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:08.908 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:09.008 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:09.041 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:09.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:09.047 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:44:09.148 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:09.174 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:09.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:09.177 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:44:09.278 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:09.280 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T10:44:09.311 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:09.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:09.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:44:09.416 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:09.450 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:09.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:09.455 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:44:09.555 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:09.564 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:09 smithi078 bash[78274]: cluster 2023-10-01T10:44:07.790644+0000 mgr.y (mgr.25072) 792 : cluster 0 pgmap v707: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 687 op/s 2023-10-01T10:44:09.564 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:09 smithi078 bash[75299]: cluster 2023-10-01T10:44:07.790644+0000 mgr.y (mgr.25072) 792 : cluster 0 pgmap v707: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 687 op/s 2023-10-01T10:44:09.597 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:09.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:09.600 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:44:09.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:09 smithi173 bash[58816]: cluster 2023-10-01T10:44:07.790644+0000 mgr.y (mgr.25072) 792 : cluster 0 pgmap v707: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 687 op/s 2023-10-01T10:44:09.700 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:09.748 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:09.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:09.753 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:44:09.854 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:09.883 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:09.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:09.888 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:44:09.989 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:10.013 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:10.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:10.017 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:44:10.117 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:10.144 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:10.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:10.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:10.252 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:10.290 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:10.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:10.297 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:44:10.398 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:10.424 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:10.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:10.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:44:10.527 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:10.537 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:44:10 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:44:10] "GET /metrics HTTP/1.1" 200 36246 "" "Prometheus/2.33.4" 2023-10-01T10:44:10.559 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:10.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:10.564 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:44:10.665 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:10.687 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:10.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:10.690 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:44:10.791 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:10.817 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:10.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:10.821 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:44:10.921 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:10.948 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:10.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:10.952 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:44:10.980 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:44:11.053 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:11.094 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:11.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:11.100 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:44:11.201 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:11.230 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:11.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:11.232 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:44:11.333 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:11.363 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:11.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:11.367 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:44:11.467 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:11.499 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:11.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:11.502 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:44:11.603 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:11.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:11 smithi173 bash[58816]: cluster 2023-10-01T10:44:09.791294+0000 mgr.y (mgr.25072) 793 : cluster 0 pgmap v708: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 678 op/s 2023-10-01T10:44:11.626 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:11.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:11.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:44:11.729 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:11.751 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:11.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:11.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:44:11.846 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:11 smithi078 bash[75299]: cluster 2023-10-01T10:44:09.791294+0000 mgr.y (mgr.25072) 793 : cluster 0 pgmap v708: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 678 op/s 2023-10-01T10:44:11.846 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:11 smithi078 bash[78274]: cluster 2023-10-01T10:44:09.791294+0000 mgr.y (mgr.25072) 793 : cluster 0 pgmap v708: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 678 op/s 2023-10-01T10:44:11.856 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:11.880 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:11.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:11.883 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:11.983 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:12.001 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:44:12.021 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:12.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:12.022 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:44:12.123 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:12.163 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:12.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:12.169 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:44:12.269 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:12.296 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:12.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:12.300 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:44:12.401 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:12.429 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:12.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:12.432 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:12.533 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:12.566 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:12.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:12.569 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:44:12.670 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:12.694 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:12.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:12.700 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:44:12.800 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:12.830 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:12.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:12.835 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:44:12.936 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:12.960 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:12.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:12.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:13.015 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:44:13.062 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:13.097 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:13.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:13.103 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:44:13.203 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:13.231 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:13.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:13.233 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:44:13.334 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:13.360 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:13.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:13.363 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:44:13.463 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:13.473 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:13 smithi078 bash[75299]: cluster 2023-10-01T10:44:11.791948+0000 mgr.y (mgr.25072) 794 : cluster 0 pgmap v709: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 676 op/s 2023-10-01T10:44:13.474 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:13 smithi078 bash[78274]: cluster 2023-10-01T10:44:11.791948+0000 mgr.y (mgr.25072) 794 : cluster 0 pgmap v709: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 676 op/s 2023-10-01T10:44:13.493 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:13.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:13.498 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:44:13.599 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:13.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:13 smithi173 bash[58816]: cluster 2023-10-01T10:44:11.791948+0000 mgr.y (mgr.25072) 794 : cluster 0 pgmap v709: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 676 op/s 2023-10-01T10:44:13.639 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:13.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:13.642 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:44:13.742 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:13.781 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:13.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:13.785 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:44:13.886 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:13.921 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:13.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:13.923 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:44:14.023 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:14.043 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:44:14.059 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:14.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:14.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:14.165 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:14.193 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:14.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:14.199 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:44:14.300 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:14.333 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:14.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:14.336 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:14.437 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:14.466 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:14.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:14.471 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:44:14.572 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:14.603 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:14.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:14.606 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:44:14.706 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:14.750 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:14.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:14.753 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:44:14.853 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:14.881 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:14.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:14.890 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:44:14.991 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:15.018 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:15.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:15.020 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:15.121 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:15.146 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:15.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:15.149 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:44:15.250 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:15.278 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:15.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:15.288 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:44:15.389 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:15.397 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:15 smithi173 bash[58816]: cluster 2023-10-01T10:44:13.792577+0000 mgr.y (mgr.25072) 795 : cluster 0 pgmap v710: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 687 op/s 2023-10-01T10:44:15.417 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:15.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:15.420 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:44:15.521 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:15.543 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:15.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:15.544 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:44:15.645 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:15.652 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:15 smithi078 bash[75299]: cluster 2023-10-01T10:44:13.792577+0000 mgr.y (mgr.25072) 795 : cluster 0 pgmap v710: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 687 op/s 2023-10-01T10:44:15.653 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:15 smithi078 bash[78274]: cluster 2023-10-01T10:44:13.792577+0000 mgr.y (mgr.25072) 795 : cluster 0 pgmap v710: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 687 op/s 2023-10-01T10:44:15.672 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:15.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:15.676 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:44:15.777 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:15.801 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:15.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:15.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:44:15.905 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:15.932 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:15.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:15.939 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:44:16.039 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:16.075 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:16.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:16.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:16.180 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:16.214 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:16.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:16.216 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:44:16.317 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:16.361 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:16.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:16.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:44:16.468 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:16.498 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:16.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:16.503 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:44:16.605 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:16.631 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:16.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:16.634 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:44:16.734 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:16.766 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:16.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:16.769 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:16.869 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:16.900 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:16.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:16.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:44:17.005 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:17.046 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:17.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:17.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:44:17.152 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:17.178 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:17.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:17.181 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:44:17.282 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:17.307 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:17.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:17.310 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:44:17.399 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:17 smithi078 bash[75299]: cluster 2023-10-01T10:44:15.793279+0000 mgr.y (mgr.25072) 796 : cluster 0 pgmap v711: 89 pgs: 89 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 637 op/s 2023-10-01T10:44:17.411 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:17.437 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:17.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:17.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:44:17.545 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:17.571 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:17.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:17.575 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:44:17.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:17 smithi173 bash[58816]: cluster 2023-10-01T10:44:15.793279+0000 mgr.y (mgr.25072) 796 : cluster 0 pgmap v711: 89 pgs: 89 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 637 op/s 2023-10-01T10:44:17.676 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:17.695 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:17.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:17.697 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:44:17.707 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:17 smithi078 bash[78274]: cluster 2023-10-01T10:44:15.793279+0000 mgr.y (mgr.25072) 796 : cluster 0 pgmap v711: 89 pgs: 89 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 637 op/s 2023-10-01T10:44:17.797 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:17.828 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:17.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:17.835 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:44:17.935 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:17.963 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:17.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:17.976 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:44:18.076 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:18.098 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:18.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:18.100 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:44:18.201 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:18.227 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:18.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:18.229 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:44:18.330 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:18.353 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:18.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:18.356 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:18.456 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:18.478 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:18.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:18.481 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:44:18.582 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:18.592 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:18 smithi078 bash[75299]: audit 2023-10-01T10:44:17.427404+0000 mon.a (mon.0) 1016 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:44:18.592 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:18 smithi078 bash[75299]: audit 2023-10-01T10:44:17.428759+0000 mon.a (mon.0) 1017 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:44:18.592 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:18 smithi078 bash[78274]: audit 2023-10-01T10:44:17.427404+0000 mon.a (mon.0) 1016 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:44:18.593 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:18 smithi078 bash[78274]: audit 2023-10-01T10:44:17.428759+0000 mon.a (mon.0) 1017 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:44:18.623 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:18.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:18.624 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:44:18.725 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:18.761 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:18.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:18.763 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:44:18.863 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:18.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:18 smithi173 bash[58816]: audit 2023-10-01T10:44:17.427404+0000 mon.a (mon.0) 1016 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:44:18.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:18 smithi173 bash[58816]: audit 2023-10-01T10:44:17.428759+0000 mon.a (mon.0) 1017 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:44:18.901 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:18.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:18.905 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:44:19.006 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:19.028 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:19.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:19.035 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:44:19.136 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:19.161 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:19.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:19.163 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:19.264 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:19.295 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:19.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:19.299 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:44:19.399 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:19.428 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:19.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:19.435 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:19.536 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:19.539 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:19 smithi078 bash[75299]: cluster 2023-10-01T10:44:17.793821+0000 mgr.y (mgr.25072) 797 : cluster 0 pgmap v712: 89 pgs: 89 active+clean; 2.5 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 658 op/s 2023-10-01T10:44:19.559 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:19.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:19.563 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:19.664 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:19.704 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:19.710 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:19 smithi173 bash[58816]: cluster 2023-10-01T10:44:17.793821+0000 mgr.y (mgr.25072) 797 : cluster 0 pgmap v712: 89 pgs: 89 active+clean; 2.5 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 658 op/s 2023-10-01T10:44:19.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:19.710 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:44:19.811 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:19.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:19 smithi078 bash[78274]: cluster 2023-10-01T10:44:17.793821+0000 mgr.y (mgr.25072) 797 : cluster 0 pgmap v712: 89 pgs: 89 active+clean; 2.5 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 658 op/s 2023-10-01T10:44:19.842 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:19.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:19.847 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:44:19.948 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:19.978 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:19.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:19.980 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:44:20.081 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:20.119 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:20.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:20.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:44:20.224 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:20.266 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:20.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:20.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:44:20.372 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:20.417 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:20.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:20.423 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:44:20.524 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:20.550 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:20.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:20.553 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:44:20.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:44:20 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:44:20] "GET /metrics HTTP/1.1" 200 36247 "" "Prometheus/2.33.4" 2023-10-01T10:44:20.653 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:20.674 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:20.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:20.677 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:44:20.777 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:20.807 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:20.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:20.809 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:44:20.909 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:20.934 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:20.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:20.939 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:21.040 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:21.079 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:21.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:21.082 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:44:21.183 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:21.223 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:21.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:21.226 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:44:21.326 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:21.363 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:21.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:21.366 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:44:21.443 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:21 smithi078 bash[75299]: cluster 2023-10-01T10:44:19.794589+0000 mgr.y (mgr.25072) 798 : cluster 0 pgmap v713: 89 pgs: 89 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 770 op/s 2023-10-01T10:44:21.443 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:21 smithi078 bash[78274]: cluster 2023-10-01T10:44:19.794589+0000 mgr.y (mgr.25072) 798 : cluster 0 pgmap v713: 89 pgs: 89 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 770 op/s 2023-10-01T10:44:21.467 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:21.496 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:21.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:21.498 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:44:21.599 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:21.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:21 smithi173 bash[58816]: cluster 2023-10-01T10:44:19.794589+0000 mgr.y (mgr.25072) 798 : cluster 0 pgmap v713: 89 pgs: 89 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 770 op/s 2023-10-01T10:44:21.632 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:21.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:21.639 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:44:21.740 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:21.764 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:21.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:21.767 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:44:21.867 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:21.890 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:21.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:21.892 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:44:21.993 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:22.016 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:22.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:22.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:44:22.120 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:22.162 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:22.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:22.166 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:44:22.266 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:22.293 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:22.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:22.298 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:22.399 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:22.424 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:22.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:22.428 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:44:22.528 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:22.561 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:22.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:22.564 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:22.665 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:22.697 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:22.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:22.703 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:44:22.803 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:22.822 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:22.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:22.823 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:44:22.924 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:22.950 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:22.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:22.958 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:44:23.059 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:23.084 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:23.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:23.088 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:44:23.189 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:23.260 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:23.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:23.262 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:44:23.365 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:23.396 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:23.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:23.399 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:23.500 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:23.526 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:23.531 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:23 smithi078 bash[78274]: cluster 2023-10-01T10:44:21.795032+0000 mgr.y (mgr.25072) 799 : cluster 0 pgmap v714: 89 pgs: 89 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 694 op/s 2023-10-01T10:44:23.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:23.532 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:44:23.533 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:23 smithi078 bash[75299]: cluster 2023-10-01T10:44:21.795032+0000 mgr.y (mgr.25072) 799 : cluster 0 pgmap v714: 89 pgs: 89 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 694 op/s 2023-10-01T10:44:23.633 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:23.659 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:23.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:23.663 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:44:23.764 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:23.788 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:23.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:23.793 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:44:23.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:23 smithi173 bash[58816]: cluster 2023-10-01T10:44:21.795032+0000 mgr.y (mgr.25072) 799 : cluster 0 pgmap v714: 89 pgs: 89 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 694 op/s 2023-10-01T10:44:23.894 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:23.941 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:23.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:23.948 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:44:24.049 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:24.084 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:24.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:24.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:44:24.191 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:24.222 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:24.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:24.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:24.325 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:24.357 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:24.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:24.359 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:44:24.460 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:24.487 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:24.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:24.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:44:24.593 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:24.618 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:24.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:24.621 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:44:24.722 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:24.751 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:24.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:24.758 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:44:24.859 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:24.880 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:24.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:24.887 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:24.987 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:25.018 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:25.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:25.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:44:25.122 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:25.151 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:25.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:25.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:44:25.255 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:25.284 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:25.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:25.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:44:25.387 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:25.424 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:25.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:25.430 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:44:25.531 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:25.540 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:25 smithi173 bash[58816]: cluster 2023-10-01T10:44:23.795480+0000 mgr.y (mgr.25072) 800 : cluster 0 pgmap v715: 89 pgs: 89 active+clean; 2.2 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2023-10-01T10:44:25.563 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:25.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:25.567 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:44:25.668 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:25.697 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:25.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:25.700 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:44:25.801 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:25.814 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:25 smithi078 bash[78274]: cluster 2023-10-01T10:44:23.795480+0000 mgr.y (mgr.25072) 800 : cluster 0 pgmap v715: 89 pgs: 89 active+clean; 2.2 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2023-10-01T10:44:25.814 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:25 smithi078 bash[75299]: cluster 2023-10-01T10:44:23.795480+0000 mgr.y (mgr.25072) 800 : cluster 0 pgmap v715: 89 pgs: 89 active+clean; 2.2 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2023-10-01T10:44:25.835 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:25.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:25.840 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:44:25.941 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:25.984 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:25.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:25.989 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:26.090 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:26.118 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:26.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:26.127 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:44:26.227 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:26.253 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:26.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:26.256 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:44:26.357 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:26.387 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:26.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:26.391 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:44:26.487 INFO:teuthology.orchestra.run.smithi078.stdout:ERROR: (22) Invalid argument 2023-10-01T10:44:26.487 INFO:teuthology.orchestra.run.smithi078.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T10:44:26.492 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:26.533 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:26.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:26.537 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:26.637 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:26.667 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:26.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:26.670 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:44:26.771 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:26.804 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:26.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:26.811 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:44:26.911 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:26.937 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:26.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:26.945 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:44:27.046 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:27.085 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:27.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:27.090 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:44:27.191 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:27.219 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:27.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:27.222 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:44:27.322 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:27.346 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:27.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:27.349 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:44:27.450 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:27.451 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:27 smithi173 bash[58816]: cluster 2023-10-01T10:44:25.796086+0000 mgr.y (mgr.25072) 801 : cluster 0 pgmap v716: 89 pgs: 89 active+clean; 2.2 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 683 op/s 2023-10-01T10:44:27.492 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:27.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:27.495 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:44:27.596 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:27.613 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:27.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:27.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:44:27.715 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:27.722 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:27 smithi078 bash[78274]: cluster 2023-10-01T10:44:25.796086+0000 mgr.y (mgr.25072) 801 : cluster 0 pgmap v716: 89 pgs: 89 active+clean; 2.2 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 683 op/s 2023-10-01T10:44:27.722 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:27 smithi078 bash[75299]: cluster 2023-10-01T10:44:25.796086+0000 mgr.y (mgr.25072) 801 : cluster 0 pgmap v716: 89 pgs: 89 active+clean; 2.2 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 683 op/s 2023-10-01T10:44:27.748 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:27.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:27.751 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:44:27.851 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:27.879 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:27.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:27.883 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:44:27.984 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:28.021 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:28.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:28.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:44:28.128 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:28.165 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:28.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:28.171 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:44:28.271 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:28.295 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:28.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:28.298 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:28.398 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:28.433 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:28.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:28.440 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:44:28.474 INFO:teuthology.orchestra.run.smithi078.stderr:reweighted osd.3 to 1 (10000) 2023-10-01T10:44:28.541 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:28.568 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:28.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:28.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:44:28.680 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:28.706 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:28 smithi078 bash[78274]: audit 2023-10-01T10:44:28.287205+0000 mon.a (mon.0) 1018 : audit 1 from='client.? 172.21.15.78:0/252455981' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-01T10:44:28.707 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:28 smithi078 bash[75299]: audit 2023-10-01T10:44:28.287205+0000 mon.a (mon.0) 1018 : audit 1 from='client.? 172.21.15.78:0/252455981' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-01T10:44:28.711 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:28.713 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:28 smithi173 bash[58816]: audit 2023-10-01T10:44:28.287205+0000 mon.a (mon.0) 1018 : audit 1 from='client.? 172.21.15.78:0/252455981' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-01T10:44:28.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:28.714 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:44:28.815 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:28.851 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:28.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:28.853 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:44:28.954 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:28.979 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:28.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:28.987 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:29.087 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:29.114 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:29.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:29.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:44:29.219 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:29.249 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:29.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:29.253 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:44:29.354 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:29.378 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:29.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:29.381 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:44:29.481 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:29.512 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:29.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:29.516 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:44:29.617 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:29.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:29 smithi173 bash[58816]: cluster 2023-10-01T10:44:27.796617+0000 mgr.y (mgr.25072) 802 : cluster 0 pgmap v717: 89 pgs: 89 active+clean; 2.1 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 644 op/s 2023-10-01T10:44:29.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:29 smithi173 bash[58816]: cluster 2023-10-01T10:44:28.461375+0000 osd.5 (osd.5) 68 : cluster 3 failed to encode map e141 with expected crc 2023-10-01T10:44:29.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:29 smithi173 bash[58816]: audit 2023-10-01T10:44:28.462799+0000 mon.a (mon.0) 1019 : audit 1 from='client.? 172.21.15.78:0/252455981' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-01T10:44:29.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:29 smithi173 bash[58816]: cluster 2023-10-01T10:44:28.462938+0000 mon.a (mon.0) 1020 : cluster 0 osdmap e141: 8 total, 8 up, 8 in 2023-10-01T10:44:29.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:29 smithi173 bash[58816]: cluster 2023-10-01T10:44:28.468809+0000 osd.7 (osd.7) 88 : cluster 3 failed to encode map e141 with expected crc 2023-10-01T10:44:29.639 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:29.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:29.644 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:44:29.745 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:29.780 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:29.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:29.785 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:44:29.794 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-01T10:44:29.795 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:29 smithi078 bash[75299]: cluster 2023-10-01T10:44:27.796617+0000 mgr.y (mgr.25072) 802 : cluster 0 pgmap v717: 89 pgs: 89 active+clean; 2.1 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 644 op/s 2023-10-01T10:44:29.795 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:29 smithi078 bash[75299]: cluster 2023-10-01T10:44:28.461375+0000 osd.5 (osd.5) 68 : cluster 3 failed to encode map e141 with expected crc 2023-10-01T10:44:29.795 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:29 smithi078 bash[75299]: audit 2023-10-01T10:44:28.462799+0000 mon.a (mon.0) 1019 : audit 1 from='client.? 172.21.15.78:0/252455981' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-01T10:44:29.795 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:29 smithi078 bash[75299]: cluster 2023-10-01T10:44:28.462938+0000 mon.a (mon.0) 1020 : cluster 0 osdmap e141: 8 total, 8 up, 8 in 2023-10-01T10:44:29.796 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:29 smithi078 bash[75299]: cluster 2023-10-01T10:44:28.468809+0000 osd.7 (osd.7) 88 : cluster 3 failed to encode map e141 with expected crc 2023-10-01T10:44:29.796 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:29 smithi078 bash[78274]: cluster 2023-10-01T10:44:27.796617+0000 mgr.y (mgr.25072) 802 : cluster 0 pgmap v717: 89 pgs: 89 active+clean; 2.1 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 644 op/s 2023-10-01T10:44:29.796 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:29 smithi078 bash[78274]: cluster 2023-10-01T10:44:28.461375+0000 osd.5 (osd.5) 68 : cluster 3 failed to encode map e141 with expected crc 2023-10-01T10:44:29.796 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:29 smithi078 bash[78274]: audit 2023-10-01T10:44:28.462799+0000 mon.a (mon.0) 1019 : audit 1 from='client.? 172.21.15.78:0/252455981' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-01T10:44:29.796 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:29 smithi078 bash[78274]: cluster 2023-10-01T10:44:28.462938+0000 mon.a (mon.0) 1020 : cluster 0 osdmap e141: 8 total, 8 up, 8 in 2023-10-01T10:44:29.796 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:29 smithi078 bash[78274]: cluster 2023-10-01T10:44:28.468809+0000 osd.7 (osd.7) 88 : cluster 3 failed to encode map e141 with expected crc 2023-10-01T10:44:29.886 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:29.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:29.918 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:29.918 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:44:30.019 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:30.064 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:30.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:30.067 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:44:30.167 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:30.201 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:30.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:30.208 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:44:30.308 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:30.331 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:44:30 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:44:30] "GET /metrics HTTP/1.1" 200 36247 "" "Prometheus/2.33.4" 2023-10-01T10:44:30.335 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:30.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:30.338 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:44:30.439 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:30.479 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:30.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:30.483 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:44:30.583 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:30.601 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:30.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:30.602 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:30.703 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:30.729 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:30.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:30.735 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:44:30.835 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:30.872 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:30.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:30.877 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:44:30.977 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:31.006 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:31.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:31.011 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:44:31.024 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-01T10:44:31.112 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:31.145 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:31.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:31.148 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:31.249 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:31.275 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:31.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:31.283 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:44:31.384 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:31.429 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:31.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:31.433 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:44:31.533 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:31.542 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:31 smithi173 bash[58816]: cluster 2023-10-01T10:44:29.797174+0000 mgr.y (mgr.25072) 803 : cluster 0 pgmap v719: 89 pgs: 89 active+clean; 2.0 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2023-10-01T10:44:31.549 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T10:44:31.565 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:31.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:31.570 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:44:31.594 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T10:44:31.629 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T10:44:31.666 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T10:44:31.671 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:31.703 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:31.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:31.707 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:44:31.722 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T10:44:31.765 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T10:44:31.800 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T10:44:31.808 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:31.817 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:31 smithi078 bash[78274]: cluster 2023-10-01T10:44:29.797174+0000 mgr.y (mgr.25072) 803 : cluster 0 pgmap v719: 89 pgs: 89 active+clean; 2.0 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2023-10-01T10:44:31.818 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:31 smithi078 bash[75299]: cluster 2023-10-01T10:44:29.797174+0000 mgr.y (mgr.25072) 803 : cluster 0 pgmap v719: 89 pgs: 89 active+clean; 2.0 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2023-10-01T10:44:31.838 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T10:44:31.840 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T10:44:31.840 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T10:44:31.840 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T10:44:31.840 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T10:44:31.841 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T10:44:31.841 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T10:44:31.841 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T10:44:31.841 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T10:44:31.844 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:31.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:31.851 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:44:31.952 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:31.993 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:32.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:32.002 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:44:32.102 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:32.127 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:32.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:32.133 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:44:32.234 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:32.258 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:32.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:32.266 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:44:32.366 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:32.404 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:32.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:32.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:44:32.496 INFO:teuthology.orchestra.run.smithi078.stderr:nodeep-scrub is set 2023-10-01T10:44:32.507 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:32.533 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:32.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:32.543 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:44:32.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:32 smithi173 bash[58816]: audit 2023-10-01T10:44:31.486078+0000 mon.a (mon.0) 1021 : audit 0 from='client.? 172.21.15.78:0/839479673' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:44:32.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:32 smithi173 bash[58816]: audit 2023-10-01T10:44:31.841744+0000 mon.c (mon.2) 142 : audit 1 from='client.? 172.21.15.78:0/3981220542' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:44:32.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:32 smithi173 bash[58816]: audit 2023-10-01T10:44:31.842445+0000 mon.a (mon.0) 1022 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:44:32.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:32 smithi173 bash[58816]: audit 2023-10-01T10:44:32.421620+0000 mon.a (mon.0) 1023 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:44:32.645 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:32.690 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:32.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:32.696 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:44:32.797 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:32.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:32 smithi078 bash[75299]: audit 2023-10-01T10:44:31.486078+0000 mon.a (mon.0) 1021 : audit 0 from='client.? 172.21.15.78:0/839479673' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:44:32.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:32 smithi078 bash[75299]: audit 2023-10-01T10:44:31.841744+0000 mon.c (mon.2) 142 : audit 1 from='client.? 172.21.15.78:0/3981220542' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:44:32.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:32 smithi078 bash[75299]: audit 2023-10-01T10:44:31.842445+0000 mon.a (mon.0) 1022 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:44:32.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:32 smithi078 bash[75299]: audit 2023-10-01T10:44:32.421620+0000 mon.a (mon.0) 1023 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:44:32.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:32 smithi078 bash[78274]: audit 2023-10-01T10:44:31.486078+0000 mon.a (mon.0) 1021 : audit 0 from='client.? 172.21.15.78:0/839479673' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:44:32.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:32 smithi078 bash[78274]: audit 2023-10-01T10:44:31.841744+0000 mon.c (mon.2) 142 : audit 1 from='client.? 172.21.15.78:0/3981220542' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:44:32.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:32 smithi078 bash[78274]: audit 2023-10-01T10:44:31.842445+0000 mon.a (mon.0) 1022 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:44:32.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:32 smithi078 bash[78274]: audit 2023-10-01T10:44:32.421620+0000 mon.a (mon.0) 1023 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:44:32.833 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:32.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:32.838 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:44:32.938 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:32.961 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:32.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:32.965 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:44:33.066 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:33.093 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:33.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:33.095 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:44:33.196 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:33.226 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:33.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:33.230 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:44:33.331 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:33.368 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:33.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:33.370 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:44:33.470 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:33.505 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:33.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:33.509 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:44:33.609 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:33.637 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:33.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:33.643 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:44:33.744 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:33.753 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:33 smithi078 bash[75299]: cluster 2023-10-01T10:44:31.797711+0000 mgr.y (mgr.25072) 804 : cluster 0 pgmap v720: 89 pgs: 89 active+clean; 1.9 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 695 op/s 2023-10-01T10:44:33.753 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:33 smithi078 bash[75299]: cluster 2023-10-01T10:44:32.470481+0000 mon.a (mon.0) 1024 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:44:33.753 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:33 smithi078 bash[75299]: cluster 2023-10-01T10:44:32.486263+0000 osd.7 (osd.7) 89 : cluster 3 failed to encode map e142 with expected crc 2023-10-01T10:44:33.753 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:33 smithi078 bash[75299]: audit 2023-10-01T10:44:32.488190+0000 mon.a (mon.0) 1025 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T10:44:33.753 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:33 smithi078 bash[75299]: cluster 2023-10-01T10:44:32.488648+0000 mon.a (mon.0) 1026 : cluster 0 osdmap e142: 8 total, 8 up, 8 in 2023-10-01T10:44:33.753 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:33 smithi078 bash[75299]: cluster 2023-10-01T10:44:32.488657+0000 osd.5 (osd.5) 69 : cluster 3 failed to encode map e142 with expected crc 2023-10-01T10:44:33.754 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:33 smithi078 bash[75299]: cluster 2023-10-01T10:44:32.490222+0000 osd.5 (osd.5) 70 : cluster 3 failed to encode map e142 with expected crc 2023-10-01T10:44:33.754 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:33 smithi078 bash[75299]: cluster 2023-10-01T10:44:32.490679+0000 osd.6 (osd.6) 53 : cluster 3 failed to encode map e142 with expected crc 2023-10-01T10:44:33.754 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:33 smithi078 bash[75299]: cluster 2023-10-01T10:44:32.491844+0000 osd.6 (osd.6) 54 : cluster 3 failed to encode map e142 with expected crc 2023-10-01T10:44:33.754 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:33 smithi078 bash[75299]: cluster 2023-10-01T10:44:32.492976+0000 osd.5 (osd.5) 71 : cluster 3 failed to encode map e142 with expected crc 2023-10-01T10:44:33.754 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:33 smithi078 bash[78274]: cluster 2023-10-01T10:44:31.797711+0000 mgr.y (mgr.25072) 804 : cluster 0 pgmap v720: 89 pgs: 89 active+clean; 1.9 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 695 op/s 2023-10-01T10:44:33.754 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:33 smithi078 bash[78274]: cluster 2023-10-01T10:44:32.470481+0000 mon.a (mon.0) 1024 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:44:33.754 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:33 smithi078 bash[78274]: cluster 2023-10-01T10:44:32.486263+0000 osd.7 (osd.7) 89 : cluster 3 failed to encode map e142 with expected crc 2023-10-01T10:44:33.754 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:33 smithi078 bash[78274]: audit 2023-10-01T10:44:32.488190+0000 mon.a (mon.0) 1025 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T10:44:33.755 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:33 smithi078 bash[78274]: cluster 2023-10-01T10:44:32.488648+0000 mon.a (mon.0) 1026 : cluster 0 osdmap e142: 8 total, 8 up, 8 in 2023-10-01T10:44:33.755 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:33 smithi078 bash[78274]: cluster 2023-10-01T10:44:32.488657+0000 osd.5 (osd.5) 69 : cluster 3 failed to encode map e142 with expected crc 2023-10-01T10:44:33.755 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:33 smithi078 bash[78274]: cluster 2023-10-01T10:44:32.490222+0000 osd.5 (osd.5) 70 : cluster 3 failed to encode map e142 with expected crc 2023-10-01T10:44:33.755 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:33 smithi078 bash[78274]: cluster 2023-10-01T10:44:32.490679+0000 osd.6 (osd.6) 53 : cluster 3 failed to encode map e142 with expected crc 2023-10-01T10:44:33.755 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:33 smithi078 bash[78274]: cluster 2023-10-01T10:44:32.491844+0000 osd.6 (osd.6) 54 : cluster 3 failed to encode map e142 with expected crc 2023-10-01T10:44:33.755 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:33 smithi078 bash[78274]: cluster 2023-10-01T10:44:32.492976+0000 osd.5 (osd.5) 71 : cluster 3 failed to encode map e142 with expected crc 2023-10-01T10:44:33.779 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:33.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:33.785 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:44:33.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:33 smithi173 bash[58816]: cluster 2023-10-01T10:44:31.797711+0000 mgr.y (mgr.25072) 804 : cluster 0 pgmap v720: 89 pgs: 89 active+clean; 1.9 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 695 op/s 2023-10-01T10:44:33.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:33 smithi173 bash[58816]: cluster 2023-10-01T10:44:32.470481+0000 mon.a (mon.0) 1024 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:44:33.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:33 smithi173 bash[58816]: cluster 2023-10-01T10:44:32.486263+0000 osd.7 (osd.7) 89 : cluster 3 failed to encode map e142 with expected crc 2023-10-01T10:44:33.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:33 smithi173 bash[58816]: audit 2023-10-01T10:44:32.488190+0000 mon.a (mon.0) 1025 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T10:44:33.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:33 smithi173 bash[58816]: cluster 2023-10-01T10:44:32.488648+0000 mon.a (mon.0) 1026 : cluster 0 osdmap e142: 8 total, 8 up, 8 in 2023-10-01T10:44:33.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:33 smithi173 bash[58816]: cluster 2023-10-01T10:44:32.488657+0000 osd.5 (osd.5) 69 : cluster 3 failed to encode map e142 with expected crc 2023-10-01T10:44:33.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:33 smithi173 bash[58816]: cluster 2023-10-01T10:44:32.490222+0000 osd.5 (osd.5) 70 : cluster 3 failed to encode map e142 with expected crc 2023-10-01T10:44:33.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:33 smithi173 bash[58816]: cluster 2023-10-01T10:44:32.490679+0000 osd.6 (osd.6) 53 : cluster 3 failed to encode map e142 with expected crc 2023-10-01T10:44:33.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:33 smithi173 bash[58816]: cluster 2023-10-01T10:44:32.491844+0000 osd.6 (osd.6) 54 : cluster 3 failed to encode map e142 with expected crc 2023-10-01T10:44:33.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:33 smithi173 bash[58816]: cluster 2023-10-01T10:44:32.492976+0000 osd.5 (osd.5) 71 : cluster 3 failed to encode map e142 with expected crc 2023-10-01T10:44:33.886 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:33.923 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:33.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:33.925 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:44:34.026 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:34.067 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:34.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:34.075 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:44:34.176 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:34.214 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:34.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:34.219 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:44:34.321 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:34.358 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:34.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:34.364 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:34.465 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:34.498 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:34.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:34.509 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:44:34.610 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:34.652 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:34.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:34.655 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:44:34.756 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:34.792 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:34.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:34.797 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:44:34.897 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:34.925 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:34.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:34.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:44:35.029 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:35.057 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:35.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:35.064 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:44:35.109 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T10:44:35.166 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:35.205 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:35.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:35.208 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:35.309 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:35.333 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:35.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:35.336 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:44:35.437 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:35.464 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:35.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:35.466 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:44:35.567 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:35.595 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:35.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:35.597 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:44:35.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:35 smithi173 bash[58816]: cluster 2023-10-01T10:44:33.798312+0000 mgr.y (mgr.25072) 805 : cluster 0 pgmap v722: 89 pgs: 89 active+clean; 1.8 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 712 op/s 2023-10-01T10:44:35.699 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:35.735 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:35.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:35.742 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:44:35.805 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T10:44:35.814 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:35 smithi078 bash[75299]: cluster 2023-10-01T10:44:33.798312+0000 mgr.y (mgr.25072) 805 : cluster 0 pgmap v722: 89 pgs: 89 active+clean; 1.8 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 712 op/s 2023-10-01T10:44:35.814 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:35 smithi078 bash[78274]: cluster 2023-10-01T10:44:33.798312+0000 mgr.y (mgr.25072) 805 : cluster 0 pgmap v722: 89 pgs: 89 active+clean; 1.8 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 712 op/s 2023-10-01T10:44:35.843 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:35.871 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:35.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:35.877 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:44:35.978 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:36.009 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:36.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:36.014 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:44:36.115 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:36.141 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:36.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:36.145 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:44:36.246 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:36.273 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:36.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:36.278 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:44:36.378 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:36.405 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:36.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:36.411 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:44:36.512 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:36.540 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:36.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:36.542 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:44:36.637 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:44:36.637 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:44:36.643 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:36.673 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:36.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:36.678 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:44:36.779 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:36.817 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:36.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:36.820 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:44:36.921 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:36.943 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:36.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:36.947 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:44:37.047 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:37.108 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:37.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:37.113 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:44:37.213 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:37.248 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:37.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:37.258 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:44:37.359 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:37.387 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:37.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:37.392 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:37.493 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:37.526 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:37.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:37.530 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:44:37.630 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:37.638 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:37 smithi078 bash[78274]: cluster 2023-10-01T10:44:35.798908+0000 mgr.y (mgr.25072) 806 : cluster 0 pgmap v723: 89 pgs: 89 active+clean; 1.7 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 761 op/s 2023-10-01T10:44:37.639 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:37 smithi078 bash[75299]: cluster 2023-10-01T10:44:35.798908+0000 mgr.y (mgr.25072) 806 : cluster 0 pgmap v723: 89 pgs: 89 active+clean; 1.7 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 761 op/s 2023-10-01T10:44:37.669 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:37.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:37.672 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:44:37.773 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:37.781 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:37 smithi173 bash[58816]: cluster 2023-10-01T10:44:35.798908+0000 mgr.y (mgr.25072) 806 : cluster 0 pgmap v723: 89 pgs: 89 active+clean; 1.7 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 761 op/s 2023-10-01T10:44:37.798 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:37.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:37.802 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:44:37.903 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:37.932 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:37.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:37.940 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:44:38.041 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:38.065 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:38.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:38.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:44:38.168 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:38.195 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:38.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:38.202 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:44:38.303 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:38.336 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:38.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:38.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:44:38.440 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:38.463 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:38.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:38.468 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:44:38.569 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:38.599 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:38.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:38.605 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:44:38.705 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:38.728 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:38.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:38.730 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:44:38.831 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:38.871 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:38.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:38.874 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:44:38.974 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:39.011 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:39.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:39.014 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:44:39.115 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:39.143 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:39.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:39.145 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:39.246 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:39.284 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:39.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:39.287 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:44:39.387 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:39.419 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:39.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:39.425 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:44:39.526 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:39.556 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:39.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:39.558 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:44:39.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:39 smithi078 bash[75299]: cluster 2023-10-01T10:44:37.799393+0000 mgr.y (mgr.25072) 807 : cluster 0 pgmap v724: 89 pgs: 89 active+clean; 1.7 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2023-10-01T10:44:39.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:39 smithi078 bash[78274]: cluster 2023-10-01T10:44:37.799393+0000 mgr.y (mgr.25072) 807 : cluster 0 pgmap v724: 89 pgs: 89 active+clean; 1.7 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2023-10-01T10:44:39.659 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:39.689 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:39.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:39.693 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:44:39.695 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:44:39.696 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:44:39.793 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:39.802 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:39 smithi173 bash[58816]: cluster 2023-10-01T10:44:37.799393+0000 mgr.y (mgr.25072) 807 : cluster 0 pgmap v724: 89 pgs: 89 active+clean; 1.7 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 684 op/s 2023-10-01T10:44:39.828 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:39.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:39.833 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:44:39.934 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:39.974 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:39.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:39.976 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:44:40.077 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:40.108 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:40.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:40.112 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:40.213 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:40.241 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:40.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:40.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:44:40.288 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:44:40 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:44:40] "GET /metrics HTTP/1.1" 200 36236 "" "Prometheus/2.33.4" 2023-10-01T10:44:40.347 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:40.371 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:40.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:40.375 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:44:40.475 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:40.502 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:40.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:40.505 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:44:40.606 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:40.632 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:40.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:40.637 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:44:40.738 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:40.769 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:40.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:40.773 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:40.874 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:40.908 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:40.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:40.913 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:44:41.013 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:41.041 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:41.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:41.045 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:44:41.146 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:41.180 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:41.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:41.184 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:44:41.285 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:41.320 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:41.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:41.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:44:41.427 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:41.453 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:41.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:41.456 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:44:41.504 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:41 smithi173 bash[58816]: cluster 2023-10-01T10:44:39.800096+0000 mgr.y (mgr.25072) 808 : cluster 0 pgmap v725: 89 pgs: 89 active+clean; 1.5 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 710 op/s 2023-10-01T10:44:41.556 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:41.591 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:41.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:41.594 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:44:41.694 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:41.701 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:41 smithi078 bash[78274]: cluster 2023-10-01T10:44:39.800096+0000 mgr.y (mgr.25072) 808 : cluster 0 pgmap v725: 89 pgs: 89 active+clean; 1.5 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 710 op/s 2023-10-01T10:44:41.702 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:41 smithi078 bash[75299]: cluster 2023-10-01T10:44:39.800096+0000 mgr.y (mgr.25072) 808 : cluster 0 pgmap v725: 89 pgs: 89 active+clean; 1.5 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 710 op/s 2023-10-01T10:44:41.721 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:41.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:41.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:41.828 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:41.862 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:41.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:41.867 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:44:41.967 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:41.996 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:41.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:41.998 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:44:42.099 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:42.125 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:42.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:42.128 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:42.229 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:42.254 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:42.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:42.260 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:42.361 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:42.384 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:42.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:42.386 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:44:42.487 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:42.519 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:42.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:42.525 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:44:42.626 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:42.657 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:42.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:42.659 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:44:42.760 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:42.788 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:42.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:42.795 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:44:42.896 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:42.939 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:42.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:42.944 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:44:43.044 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:43.067 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:43.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:43.069 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:44:43.170 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:43.196 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:43.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:43.203 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:43.304 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:43.336 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:43.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:43.342 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:44:43.443 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:43.475 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:43.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:43.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:44:43.579 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:43.593 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:43 smithi173 bash[58816]: cluster 2023-10-01T10:44:41.800677+0000 mgr.y (mgr.25072) 809 : cluster 0 pgmap v726: 89 pgs: 89 active+clean; 1.5 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 712 op/s 2023-10-01T10:44:43.609 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:43.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:43.615 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:44:43.715 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:43.723 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:43 smithi078 bash[75299]: cluster 2023-10-01T10:44:41.800677+0000 mgr.y (mgr.25072) 809 : cluster 0 pgmap v726: 89 pgs: 89 active+clean; 1.5 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 712 op/s 2023-10-01T10:44:43.724 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:43 smithi078 bash[78274]: cluster 2023-10-01T10:44:41.800677+0000 mgr.y (mgr.25072) 809 : cluster 0 pgmap v726: 89 pgs: 89 active+clean; 1.5 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 712 op/s 2023-10-01T10:44:43.742 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:43.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:43.748 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:44:43.849 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:43.871 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:43.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:43.881 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:43.982 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:44.009 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:44.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:44.014 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:44:44.115 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:44.145 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:44.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:44.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:44:44.247 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:44.288 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:44.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:44.291 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:44:44.391 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:44.417 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:44.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:44.422 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:44:44.523 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:44.553 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:44.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:44.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:44:44.658 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:44.678 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:44.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:44.680 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:44.781 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:44.808 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:44.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:44.811 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:44:44.911 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:44.954 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:44.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:44.957 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:44:45.057 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:45.086 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:45.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:45.090 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:44:45.191 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:45.218 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:45.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:45.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:44:45.326 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:45.366 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:45.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:45.372 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:44:45.472 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:45.506 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:45.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:45.511 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:44:45.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:45 smithi078 bash[78274]: cluster 2023-10-01T10:44:43.801245+0000 mgr.y (mgr.25072) 810 : cluster 0 pgmap v727: 89 pgs: 89 active+clean; 1.4 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 746 op/s 2023-10-01T10:44:45.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:45 smithi078 bash[75299]: cluster 2023-10-01T10:44:43.801245+0000 mgr.y (mgr.25072) 810 : cluster 0 pgmap v727: 89 pgs: 89 active+clean; 1.4 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 746 op/s 2023-10-01T10:44:45.612 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:45.637 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:45.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:45.638 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:44:45.738 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:45.768 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:45.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:45.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:45 smithi173 bash[58816]: cluster 2023-10-01T10:44:43.801245+0000 mgr.y (mgr.25072) 810 : cluster 0 pgmap v727: 89 pgs: 89 active+clean; 1.4 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 746 op/s 2023-10-01T10:44:45.780 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:44:45.880 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:45.908 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:45.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:45.911 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:46.012 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:46.052 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:46.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:46.055 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:44:46.156 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:46.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:46.186 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:46.186 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:44:46.287 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:46.324 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:46.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:46.329 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:44:46.430 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:46.461 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:46.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:46.463 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:44:46.564 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:46.594 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:46.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:46.601 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:44:46.702 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:46.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:46.734 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:46.734 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:44:46.835 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:46.862 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:46.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:46.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:44:46.968 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:46.991 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:46.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:46.996 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:47.097 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:47.121 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:47.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:47.128 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:44:47.229 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:47.260 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:47.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:47.264 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:47.365 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:47.410 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:47.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:47.412 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:44:47.513 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:47.553 INFO:teuthology.orchestra.run.smithi078.stderr:reweighted osd.4 to 1 (10000) 2023-10-01T10:44:47.553 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:47.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:47.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:44:47.660 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:47.705 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:47.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:47.710 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:47.810 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:47.818 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:47 smithi078 bash[78274]: cluster 2023-10-01T10:44:45.801849+0000 mgr.y (mgr.25072) 811 : cluster 0 pgmap v728: 89 pgs: 89 active+clean; 1.3 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2023-10-01T10:44:47.818 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:47 smithi078 bash[78274]: audit 2023-10-01T10:44:47.025444+0000 mon.a (mon.0) 1027 : audit 1 from='client.? 172.21.15.78:0/1500198906' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-01T10:44:47.819 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:47 smithi078 bash[78274]: audit 2023-10-01T10:44:47.421697+0000 mon.a (mon.0) 1028 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:44:47.819 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:47 smithi078 bash[75299]: cluster 2023-10-01T10:44:45.801849+0000 mgr.y (mgr.25072) 811 : cluster 0 pgmap v728: 89 pgs: 89 active+clean; 1.3 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2023-10-01T10:44:47.819 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:47 smithi078 bash[75299]: audit 2023-10-01T10:44:47.025444+0000 mon.a (mon.0) 1027 : audit 1 from='client.? 172.21.15.78:0/1500198906' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-01T10:44:47.819 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:47 smithi078 bash[75299]: audit 2023-10-01T10:44:47.421697+0000 mon.a (mon.0) 1028 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:44:47.832 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:47.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:47.834 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:44:47.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:47 smithi173 bash[58816]: cluster 2023-10-01T10:44:45.801849+0000 mgr.y (mgr.25072) 811 : cluster 0 pgmap v728: 89 pgs: 89 active+clean; 1.3 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2023-10-01T10:44:47.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:47 smithi173 bash[58816]: audit 2023-10-01T10:44:47.025444+0000 mon.a (mon.0) 1027 : audit 1 from='client.? 172.21.15.78:0/1500198906' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-01T10:44:47.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:47 smithi173 bash[58816]: audit 2023-10-01T10:44:47.421697+0000 mon.a (mon.0) 1028 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:44:47.934 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:47.960 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:47.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:47.963 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:48.063 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:48.084 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:48.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:48.087 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:44:48.188 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:48.211 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:48.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:48.214 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:48.314 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:48.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:48.348 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:48.348 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:44:48.449 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:48.477 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:48.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:48.480 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:44:48.580 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:48.589 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:48 smithi173 bash[58816]: cluster 2023-10-01T10:44:47.534274+0000 osd.5 (osd.5) 72 : cluster 3 failed to encode map e143 with expected crc 2023-10-01T10:44:48.589 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:48 smithi173 bash[58816]: audit 2023-10-01T10:44:47.534972+0000 mon.a (mon.0) 1029 : audit 1 from='client.? 172.21.15.78:0/1500198906' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-01T10:44:48.589 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:48 smithi173 bash[58816]: cluster 2023-10-01T10:44:47.535046+0000 mon.a (mon.0) 1030 : cluster 0 osdmap e143: 8 total, 8 up, 8 in 2023-10-01T10:44:48.590 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:48 smithi173 bash[58816]: cluster 2023-10-01T10:44:47.549361+0000 osd.7 (osd.7) 90 : cluster 3 failed to encode map e143 with expected crc 2023-10-01T10:44:48.590 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:48 smithi173 bash[58816]: cluster 2023-10-01T10:44:47.556467+0000 osd.6 (osd.6) 55 : cluster 3 failed to encode map e143 with expected crc 2023-10-01T10:44:48.627 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:48.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:48.633 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:44:48.734 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:48.760 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:48.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:48.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:48.778 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2023-10-01T10:44:48.778 INFO:teuthology.orchestra.run.smithi078.stdout: "size": 20, 2023-10-01T10:44:48.778 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 600, 2023-10-01T10:44:48.778 INFO:teuthology.orchestra.run.smithi078.stdout: "ops": [ 2023-10-01T10:44:48.778 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.778 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.35649.0:75435 9.c 9:35a3dedc:::benchmark_data_smithi078_113121_object75434:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e132)", 2023-10-01T10:44:48.778 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:41:57.427643+0000", 2023-10-01T10:44:48.779 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 171.34618406499999, 2023-10-01T10:44:48.779 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.20691067399999999, 2023-10-01T10:44:48.779 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:44:48.779 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:44:48.779 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:44:48.779 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.35649", 2023-10-01T10:44:48.779 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:44:48.779 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 75435 2023-10-01T10:44:48.779 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.779 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:44:48.779 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.780 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:44:48.780 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.427643+0000", 2023-10-01T10:44:48.780 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:44:48.781 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.781 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.781 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:44:48.781 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.427643+0000", 2023-10-01T10:44:48.781 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:44:48.782 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.782 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.782 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:44:48.782 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.427642+0000", 2023-10-01T10:44:48.782 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999986 2023-10-01T10:44:48.782 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.782 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.782 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:44:48.782 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.427665+0000", 2023-10-01T10:44:48.782 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.3399000000000001e-05 2023-10-01T10:44:48.782 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.783 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.783 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:44:48.783 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.427677+0000", 2023-10-01T10:44:48.783 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1813e-05 2023-10-01T10:44:48.783 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.783 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.783 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:44:48.783 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.427685+0000", 2023-10-01T10:44:48.783 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.7759999999999997e-06 2023-10-01T10:44:48.783 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.783 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.783 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:44:48.784 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.427721+0000", 2023-10-01T10:44:48.784 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.6562e-05 2023-10-01T10:44:48.784 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.784 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.784 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:44:48.784 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.427806+0000", 2023-10-01T10:44:48.784 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.5396999999999996e-05 2023-10-01T10:44:48.784 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.784 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.784 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 4,7", 2023-10-01T10:44:48.784 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.427892+0000", 2023-10-01T10:44:48.785 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.5191000000000005e-05 2023-10-01T10:44:48.785 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.785 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.785 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:44:48.785 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.430024+0000", 2023-10-01T10:44:48.785 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.002132802 2023-10-01T10:44:48.785 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.785 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.786 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:44:48.786 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.633364+0000", 2023-10-01T10:44:48.786 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.20333915699999999 2023-10-01T10:44:48.786 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.786 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.786 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:44:48.786 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.633825+0000", 2023-10-01T10:44:48.786 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00046099999999999998 2023-10-01T10:44:48.786 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.786 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.786 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:44:48.787 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.634505+0000", 2023-10-01T10:44:48.787 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00068042099999999996 2023-10-01T10:44:48.787 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.787 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.787 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:44:48.787 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.634553+0000", 2023-10-01T10:44:48.787 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.8429e-05 2023-10-01T10:44:48.787 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:44:48.787 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:44:48.788 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:44:48.788 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.788 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.788 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.35649.0:75440 9.c 9:363a79ad:::benchmark_data_smithi078_113121_object75439:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e132)", 2023-10-01T10:44:48.788 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:41:57.429993+0000", 2023-10-01T10:44:48.790 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 171.34383356800001, 2023-10-01T10:44:48.790 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.204878119, 2023-10-01T10:44:48.790 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:44:48.790 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:44:48.791 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:44:48.791 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.35649", 2023-10-01T10:44:48.791 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:44:48.791 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 75440 2023-10-01T10:44:48.791 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.791 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:44:48.791 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.791 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:44:48.791 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.429993+0000", 2023-10-01T10:44:48.792 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:44:48.792 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.792 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.792 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:44:48.792 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.429993+0000", 2023-10-01T10:44:48.792 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:44:48.792 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.792 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.792 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:44:48.793 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.429992+0000", 2023-10-01T10:44:48.793 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:44:48.793 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.793 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.793 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:44:48.793 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.430013+0000", 2023-10-01T10:44:48.793 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.0625999999999999e-05 2023-10-01T10:44:48.793 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.793 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.794 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:44:48.794 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.430023+0000", 2023-10-01T10:44:48.794 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.0319999999999999e-05 2023-10-01T10:44:48.794 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.794 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.794 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:44:48.794 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.430030+0000", 2023-10-01T10:44:48.794 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.1380000000000001e-06 2023-10-01T10:44:48.795 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.795 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.795 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:44:48.795 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.430067+0000", 2023-10-01T10:44:48.795 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.6430000000000002e-05 2023-10-01T10:44:48.795 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.795 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.795 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:44:48.796 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.430158+0000", 2023-10-01T10:44:48.796 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.0628e-05 2023-10-01T10:44:48.796 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.796 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.796 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 4,7", 2023-10-01T10:44:48.796 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.430297+0000", 2023-10-01T10:44:48.796 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000139777 2023-10-01T10:44:48.796 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.796 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.797 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:44:48.797 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.432629+0000", 2023-10-01T10:44:48.797 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0023313280000000001 2023-10-01T10:44:48.797 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.797 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.797 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:44:48.797 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.634616+0000", 2023-10-01T10:44:48.797 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.20198776700000001 2023-10-01T10:44:48.797 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.797 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.797 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:44:48.798 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.634648+0000", 2023-10-01T10:44:48.798 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.1571e-05 2023-10-01T10:44:48.798 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.798 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.798 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:44:48.798 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.634828+0000", 2023-10-01T10:44:48.798 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000180187 2023-10-01T10:44:48.798 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.798 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.798 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:44:48.798 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.634871+0000", 2023-10-01T10:44:48.798 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.3279999999999999e-05 2023-10-01T10:44:48.799 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:44:48.799 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:44:48.799 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:44:48.799 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.799 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.799 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.35649.0:75442 9.1 9:818b3f9a:::benchmark_data_smithi078_113121_object75441:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e132)", 2023-10-01T10:44:48.799 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:41:57.430836+0000", 2023-10-01T10:44:48.799 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 171.34299043499999, 2023-10-01T10:44:48.799 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.21003887700000001, 2023-10-01T10:44:48.799 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:44:48.799 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:44:48.799 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:44:48.799 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.35649", 2023-10-01T10:44:48.800 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:44:48.800 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 75442 2023-10-01T10:44:48.800 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.800 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:44:48.800 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.800 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:44:48.800 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.430836+0000", 2023-10-01T10:44:48.800 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:44:48.801 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.801 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.801 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:44:48.801 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.430836+0000", 2023-10-01T10:44:48.801 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:44:48.801 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.801 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.801 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:44:48.801 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.430835+0000", 2023-10-01T10:44:48.801 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:44:48.801 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.802 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.802 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:44:48.802 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.430859+0000", 2023-10-01T10:44:48.802 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.3326e-05 2023-10-01T10:44:48.802 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.802 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.802 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:44:48.802 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.430868+0000", 2023-10-01T10:44:48.802 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.6269999999999995e-06 2023-10-01T10:44:48.802 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.802 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.803 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:44:48.803 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.430874+0000", 2023-10-01T10:44:48.803 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.3990000000000003e-06 2023-10-01T10:44:48.803 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.803 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.803 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:44:48.803 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.430920+0000", 2023-10-01T10:44:48.803 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.6486000000000002e-05 2023-10-01T10:44:48.803 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.803 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.803 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:44:48.804 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.431034+0000", 2023-10-01T10:44:48.804 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000113694 2023-10-01T10:44:48.804 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.804 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.804 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 6,7", 2023-10-01T10:44:48.804 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.431156+0000", 2023-10-01T10:44:48.804 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000121891 2023-10-01T10:44:48.804 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.804 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.804 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:44:48.804 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.624620+0000", 2023-10-01T10:44:48.804 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.19346456400000001 2023-10-01T10:44:48.805 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.805 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.805 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:44:48.805 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.624752+0000", 2023-10-01T10:44:48.805 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00013177100000000001 2023-10-01T10:44:48.805 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.805 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.805 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:44:48.805 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.640802+0000", 2023-10-01T10:44:48.805 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.016049752 2023-10-01T10:44:48.805 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.806 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.806 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:44:48.806 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.640839+0000", 2023-10-01T10:44:48.806 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.7577999999999998e-05 2023-10-01T10:44:48.806 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.806 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.806 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:44:48.806 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.640875+0000", 2023-10-01T10:44:48.806 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.5861000000000001e-05 2023-10-01T10:44:48.806 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:44:48.806 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:44:48.807 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:44:48.807 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.807 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.807 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.35649.0:75436 9.3 e132/128 9:c2f783d2:::benchmark_data_smithi078_113121_object75435:head v 132'4665, mlcod=132'4663)", 2023-10-01T10:44:48.807 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:41:57.430995+0000", 2023-10-01T10:44:48.807 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 171.34283165400001, 2023-10-01T10:44:48.807 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.194232663, 2023-10-01T10:44:48.807 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:44:48.807 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:44:48.807 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:44:48.807 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.807 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:44:48.808 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.430995+0000", 2023-10-01T10:44:48.808 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:44:48.808 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.808 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.808 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:44:48.808 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.430995+0000", 2023-10-01T10:44:48.808 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:44:48.808 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.808 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.808 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:44:48.808 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.430995+0000", 2023-10-01T10:44:48.808 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999995 2023-10-01T10:44:48.809 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.809 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.809 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:44:48.809 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.431077+0000", 2023-10-01T10:44:48.809 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.1965999999999994e-05 2023-10-01T10:44:48.809 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.809 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.809 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:44:48.809 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.431086+0000", 2023-10-01T10:44:48.809 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.9429999999999995e-06 2023-10-01T10:44:48.809 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.810 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.810 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:44:48.810 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.431093+0000", 2023-10-01T10:44:48.810 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.4740000000000002e-06 2023-10-01T10:44:48.810 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.810 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.810 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:44:48.810 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.431140+0000", 2023-10-01T10:44:48.810 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.6739000000000001e-05 2023-10-01T10:44:48.810 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.810 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.810 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:44:48.811 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.431155+0000", 2023-10-01T10:44:48.811 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.5498e-05 2023-10-01T10:44:48.811 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.811 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.811 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:44:48.811 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.435610+0000", 2023-10-01T10:44:48.811 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0044546689999999996 2023-10-01T10:44:48.811 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.811 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.811 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:44:48.811 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.625228+0000", 2023-10-01T10:44:48.811 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.18961800300000001 2023-10-01T10:44:48.812 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:44:48.812 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:44:48.812 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:44:48.812 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.812 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.812 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.35649.0:75437 9.7 e132/128 9:eee78d3d:::benchmark_data_smithi078_113121_object75436:head v 132'4767, mlcod=132'4765)", 2023-10-01T10:44:48.812 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:41:57.432106+0000", 2023-10-01T10:44:48.812 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 171.341721017, 2023-10-01T10:44:48.812 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.19307002300000001, 2023-10-01T10:44:48.812 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:44:48.813 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:44:48.813 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:44:48.813 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.813 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:44:48.813 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.432106+0000", 2023-10-01T10:44:48.813 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:44:48.813 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.813 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.813 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:44:48.813 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.432106+0000", 2023-10-01T10:44:48.813 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:44:48.814 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.814 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.814 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:44:48.814 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.432105+0000", 2023-10-01T10:44:48.814 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:44:48.814 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.814 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.814 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:44:48.814 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.432169+0000", 2023-10-01T10:44:48.814 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.3794000000000006e-05 2023-10-01T10:44:48.814 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.814 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.815 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:44:48.815 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.432181+0000", 2023-10-01T10:44:48.815 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.2415000000000001e-05 2023-10-01T10:44:48.815 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.815 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.815 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:44:48.815 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.432189+0000", 2023-10-01T10:44:48.815 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.7249999999999997e-06 2023-10-01T10:44:48.815 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.815 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.815 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:44:48.815 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.432237+0000", 2023-10-01T10:44:48.816 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.8260000000000002e-05 2023-10-01T10:44:48.816 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.816 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.816 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:44:48.816 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.432261+0000", 2023-10-01T10:44:48.816 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.3819e-05 2023-10-01T10:44:48.816 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.816 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.816 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:44:48.816 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.625119+0000", 2023-10-01T10:44:48.816 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.192858376 2023-10-01T10:44:48.816 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.817 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.817 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:44:48.817 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.625176+0000", 2023-10-01T10:44:48.817 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.6455000000000001e-05 2023-10-01T10:44:48.817 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:44:48.817 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:44:48.817 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:44:48.817 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.817 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.817 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.35649.0:75445 9.c 9:3c7722e5:::benchmark_data_smithi078_113121_object75444:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e132)", 2023-10-01T10:44:48.817 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:41:57.433553+0000", 2023-10-01T10:44:48.818 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 171.34027339900001, 2023-10-01T10:44:48.818 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.20628380700000001, 2023-10-01T10:44:48.818 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:44:48.818 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:44:48.818 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:44:48.818 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.35649", 2023-10-01T10:44:48.818 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:44:48.818 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 75445 2023-10-01T10:44:48.818 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.818 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:44:48.818 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.818 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:44:48.819 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.433553+0000", 2023-10-01T10:44:48.819 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:44:48.819 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.819 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.819 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:44:48.819 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.433553+0000", 2023-10-01T10:44:48.819 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:44:48.819 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.819 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.819 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:44:48.819 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.433552+0000", 2023-10-01T10:44:48.820 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999986 2023-10-01T10:44:48.820 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.820 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.820 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:44:48.820 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.433606+0000", 2023-10-01T10:44:48.820 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.4082999999999997e-05 2023-10-01T10:44:48.820 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.820 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.820 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:44:48.820 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.433632+0000", 2023-10-01T10:44:48.820 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.5792e-05 2023-10-01T10:44:48.820 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.821 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.821 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:44:48.821 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.433641+0000", 2023-10-01T10:44:48.821 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.2299999999999997e-06 2023-10-01T10:44:48.821 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.821 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.821 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:44:48.822 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.433752+0000", 2023-10-01T10:44:48.822 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00011108299999999999 2023-10-01T10:44:48.822 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.822 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.822 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:44:48.822 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.433900+0000", 2023-10-01T10:44:48.822 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000147823 2023-10-01T10:44:48.822 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.822 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.822 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 4,7", 2023-10-01T10:44:48.822 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.624952+0000", 2023-10-01T10:44:48.823 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.19105241100000001 2023-10-01T10:44:48.823 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.823 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.823 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:44:48.823 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.633606+0000", 2023-10-01T10:44:48.823 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0086534890000000003 2023-10-01T10:44:48.823 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.823 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.823 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:44:48.824 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.634928+0000", 2023-10-01T10:44:48.824 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0013220669999999999 2023-10-01T10:44:48.824 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.824 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.824 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:44:48.824 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.639652+0000", 2023-10-01T10:44:48.824 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0047245330000000004 2023-10-01T10:44:48.824 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.824 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.824 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:44:48.825 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.639798+0000", 2023-10-01T10:44:48.825 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000145532 2023-10-01T10:44:48.825 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.825 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.825 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:44:48.825 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.639837+0000", 2023-10-01T10:44:48.825 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.9390000000000001e-05 2023-10-01T10:44:48.825 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:44:48.825 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:44:48.825 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:44:48.825 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.826 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.826 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.35649.0:75446 9.c 9:3afa93b1:::benchmark_data_smithi078_113121_object75445:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e132)", 2023-10-01T10:44:48.826 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:41:57.433659+0000", 2023-10-01T10:44:48.826 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 171.340167501, 2023-10-01T10:44:48.826 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.207073217, 2023-10-01T10:44:48.826 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:44:48.826 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:44:48.826 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:44:48.826 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.35649", 2023-10-01T10:44:48.827 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:44:48.827 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 75446 2023-10-01T10:44:48.827 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.827 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:44:48.827 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.827 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:44:48.827 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.433659+0000", 2023-10-01T10:44:48.827 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:44:48.827 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.828 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.828 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:44:48.828 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.433659+0000", 2023-10-01T10:44:48.828 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:44:48.828 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.828 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.828 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:44:48.828 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.433659+0000", 2023-10-01T10:44:48.828 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999995 2023-10-01T10:44:48.828 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.829 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.829 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:44:48.829 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.433679+0000", 2023-10-01T10:44:48.829 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.0443999999999999e-05 2023-10-01T10:44:48.829 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.829 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.829 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:44:48.829 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.433702+0000", 2023-10-01T10:44:48.830 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.3264e-05 2023-10-01T10:44:48.830 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.830 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.830 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:44:48.830 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.433710+0000", 2023-10-01T10:44:48.830 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.4139999999999997e-06 2023-10-01T10:44:48.830 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.831 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.831 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:44:48.831 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.628458+0000", 2023-10-01T10:44:48.831 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.19474844999999999 2023-10-01T10:44:48.831 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.831 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.831 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:44:48.831 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.628657+0000", 2023-10-01T10:44:48.831 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000198507 2023-10-01T10:44:48.832 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.832 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.832 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 4,7", 2023-10-01T10:44:48.832 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.628932+0000", 2023-10-01T10:44:48.832 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000275684 2023-10-01T10:44:48.832 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.832 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.833 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:44:48.833 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.639589+0000", 2023-10-01T10:44:48.833 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.010656677999999999 2023-10-01T10:44:48.833 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.833 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.833 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:44:48.833 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.639874+0000", 2023-10-01T10:44:48.833 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00028527499999999998 2023-10-01T10:44:48.834 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.834 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.834 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:44:48.834 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.639900+0000", 2023-10-01T10:44:48.834 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.5545e-05 2023-10-01T10:44:48.834 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.834 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.834 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:44:48.834 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.640691+0000", 2023-10-01T10:44:48.834 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00079071899999999995 2023-10-01T10:44:48.834 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.835 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.835 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:44:48.835 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.640733+0000", 2023-10-01T10:44:48.835 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.1896e-05 2023-10-01T10:44:48.835 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:44:48.835 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:44:48.835 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:44:48.835 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.835 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.835 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop_reply(client.35649.0:75442 9.1 e132/128 ondisk, result = 0)", 2023-10-01T10:44:48.835 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:41:57.434538+0000", 2023-10-01T10:44:48.835 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 171.33928903399999, 2023-10-01T10:44:48.836 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.19021684599999999, 2023-10-01T10:44:48.836 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:44:48.836 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "started", 2023-10-01T10:44:48.836 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:44:48.836 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.836 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:44:48.836 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.434538+0000", 2023-10-01T10:44:48.836 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:44:48.836 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.836 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.836 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:44:48.836 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.434538+0000", 2023-10-01T10:44:48.837 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:44:48.837 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.837 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.837 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:44:48.837 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.434537+0000", 2023-10-01T10:44:48.837 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:44:48.837 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.837 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.837 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:44:48.837 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.434546+0000", 2023-10-01T10:44:48.837 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.149e-06 2023-10-01T10:44:48.837 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.838 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.838 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:44:48.838 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.624667+0000", 2023-10-01T10:44:48.838 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.19012103399999999 2023-10-01T10:44:48.838 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.838 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.838 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:44:48.838 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.624690+0000", 2023-10-01T10:44:48.838 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.3223000000000001e-05 2023-10-01T10:44:48.838 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.838 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.839 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:44:48.839 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.624740+0000", 2023-10-01T10:44:48.839 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.9660999999999998e-05 2023-10-01T10:44:48.839 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.839 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.839 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:44:48.839 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.624748+0000", 2023-10-01T10:44:48.839 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.6240000000000002e-06 2023-10-01T10:44:48.839 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.839 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.839 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:44:48.840 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.624755+0000", 2023-10-01T10:44:48.840 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.0489999999999998e-06 2023-10-01T10:44:48.840 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:44:48.840 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:44:48.840 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:44:48.840 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.840 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.840 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop_reply(client.35649.0:75435 9.c e132/128 ondisk, result = 0)", 2023-10-01T10:44:48.840 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:41:57.436699+0000", 2023-10-01T10:44:48.840 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 171.337128248, 2023-10-01T10:44:48.840 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.19666953300000001, 2023-10-01T10:44:48.840 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:44:48.841 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "started", 2023-10-01T10:44:48.841 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:44:48.841 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.841 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:44:48.841 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.436699+0000", 2023-10-01T10:44:48.841 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:44:48.841 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.841 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.841 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:44:48.841 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.436699+0000", 2023-10-01T10:44:48.841 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:44:48.841 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.841 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.842 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:44:48.842 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.436698+0000", 2023-10-01T10:44:48.842 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999995 2023-10-01T10:44:48.842 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.842 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.842 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:44:48.842 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.436707+0000", 2023-10-01T10:44:48.842 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.2620000000000004e-06 2023-10-01T10:44:48.842 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.842 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.843 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:44:48.843 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.624621+0000", 2023-10-01T10:44:48.843 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.18791355200000001 2023-10-01T10:44:48.843 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.843 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.843 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:44:48.843 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.624793+0000", 2023-10-01T10:44:48.843 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000172607 2023-10-01T10:44:48.843 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.843 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.843 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:44:48.843 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.633345+0000", 2023-10-01T10:44:48.843 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0085519399999999992 2023-10-01T10:44:48.844 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.844 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.844 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:44:48.844 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.633355+0000", 2023-10-01T10:44:48.844 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.9320000000000008e-06 2023-10-01T10:44:48.844 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.844 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.844 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:44:48.844 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.633368+0000", 2023-10-01T10:44:48.844 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.2901999999999999e-05 2023-10-01T10:44:48.844 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:44:48.845 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:44:48.845 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:44:48.845 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.845 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.845 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.35649.0:75453 9.c 9:34e2c81b:::benchmark_data_smithi078_113121_object75452:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e132)", 2023-10-01T10:44:48.845 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:41:57.445798+0000", 2023-10-01T10:44:48.845 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 171.32802871800001, 2023-10-01T10:44:48.845 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.20244446599999999, 2023-10-01T10:44:48.845 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:44:48.845 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:44:48.845 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:44:48.845 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.35649", 2023-10-01T10:44:48.846 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:44:48.846 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 75453 2023-10-01T10:44:48.846 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.846 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:44:48.846 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.846 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:44:48.846 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.445798+0000", 2023-10-01T10:44:48.846 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:44:48.846 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.846 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.846 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:44:48.847 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.445798+0000", 2023-10-01T10:44:48.847 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:44:48.847 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.847 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.847 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:44:48.847 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.445796+0000", 2023-10-01T10:44:48.847 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999976 2023-10-01T10:44:48.847 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.847 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.847 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:44:48.847 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.445897+0000", 2023-10-01T10:44:48.847 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000100864 2023-10-01T10:44:48.847 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.848 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.848 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:44:48.848 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.624650+0000", 2023-10-01T10:44:48.848 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.178753207 2023-10-01T10:44:48.848 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.848 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.848 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:44:48.848 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.624664+0000", 2023-10-01T10:44:48.848 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.3658e-05 2023-10-01T10:44:48.848 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.848 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.849 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:44:48.849 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.635264+0000", 2023-10-01T10:44:48.849 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.010600547 2023-10-01T10:44:48.849 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.849 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.849 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:44:48.849 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.635409+0000", 2023-10-01T10:44:48.849 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00014442599999999999 2023-10-01T10:44:48.849 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.849 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.849 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 4,7", 2023-10-01T10:44:48.849 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.635555+0000", 2023-10-01T10:44:48.850 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000146592 2023-10-01T10:44:48.850 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.850 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.850 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:44:48.850 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.640775+0000", 2023-10-01T10:44:48.850 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0052196389999999999 2023-10-01T10:44:48.850 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.850 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.850 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:44:48.850 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.640783+0000", 2023-10-01T10:44:48.850 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.1380000000000007e-06 2023-10-01T10:44:48.850 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.851 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.851 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:44:48.851 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.648149+0000", 2023-10-01T10:44:48.851 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0073663519999999996 2023-10-01T10:44:48.851 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.851 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.851 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:44:48.851 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.648199+0000", 2023-10-01T10:44:48.851 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.9561999999999997e-05 2023-10-01T10:44:48.851 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.851 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.852 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:44:48.852 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.648243+0000", 2023-10-01T10:44:48.852 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.3642e-05 2023-10-01T10:44:48.852 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:44:48.852 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:44:48.852 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:44:48.852 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.852 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.852 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.35649.0:128351 9.1 9:837bca8d:::benchmark_data_smithi078_113121_object16536:head [delete] snapc 0=[] ondisk+write+known_if_redirected e135)", 2023-10-01T10:44:48.852 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:42:59.450138+0000", 2023-10-01T10:44:48.852 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 109.323688988, 2023-10-01T10:44:48.852 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.169955685, 2023-10-01T10:44:48.853 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:44:48.853 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:44:48.853 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:44:48.853 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.35649", 2023-10-01T10:44:48.853 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:44:48.853 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 128351 2023-10-01T10:44:48.853 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.853 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:44:48.853 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.853 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:44:48.854 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:59.450138+0000", 2023-10-01T10:44:48.854 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:44:48.854 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.854 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.854 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:44:48.854 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:59.450138+0000", 2023-10-01T10:44:48.854 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:44:48.854 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.854 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.854 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:44:48.855 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:59.450136+0000", 2023-10-01T10:44:48.855 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999981 2023-10-01T10:44:48.855 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.855 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.855 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:44:48.855 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:59.450145+0000", 2023-10-01T10:44:48.855 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.6119999999999999e-06 2023-10-01T10:44:48.855 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.855 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.855 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:44:48.855 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:59.450159+0000", 2023-10-01T10:44:48.855 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.4717e-05 2023-10-01T10:44:48.856 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.856 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.856 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:44:48.856 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:59.450168+0000", 2023-10-01T10:44:48.856 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.8899999999999996e-06 2023-10-01T10:44:48.856 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.856 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.857 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:44:48.857 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:59.450215+0000", 2023-10-01T10:44:48.857 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.6683e-05 2023-10-01T10:44:48.857 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.857 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.857 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:44:48.858 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:59.450333+0000", 2023-10-01T10:44:48.858 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000118194 2023-10-01T10:44:48.858 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.858 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.858 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 6,7", 2023-10-01T10:44:48.858 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:59.450446+0000", 2023-10-01T10:44:48.858 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00011256600000000001 2023-10-01T10:44:48.859 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.859 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.859 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:44:48.859 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:59.464667+0000", 2023-10-01T10:44:48.859 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.014221134 2023-10-01T10:44:48.859 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.860 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.860 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:44:48.860 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:59.473280+0000", 2023-10-01T10:44:48.860 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0086130600000000005 2023-10-01T10:44:48.860 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.860 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.860 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:44:48.861 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:59.619973+0000", 2023-10-01T10:44:48.861 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.14669332199999999 2023-10-01T10:44:48.861 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.861 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.861 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:44:48.861 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:59.620034+0000", 2023-10-01T10:44:48.862 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.0727000000000003e-05 2023-10-01T10:44:48.862 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.862 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.862 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:44:48.862 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:59.620094+0000", 2023-10-01T10:44:48.862 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.9698999999999999e-05 2023-10-01T10:44:48.862 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:44:48.862 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:44:48.863 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:44:48.863 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.863 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.863 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.35649.0:128375 9.2 9:4b8172c8:::benchmark_data_smithi078_113121_object16560:head [delete] snapc 0=[] ondisk+write+known_if_redirected e135)", 2023-10-01T10:44:48.863 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:42:59.476366+0000", 2023-10-01T10:44:48.863 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 109.29746063, 2023-10-01T10:44:48.863 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.16866862399999999, 2023-10-01T10:44:48.863 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:44:48.864 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:44:48.864 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:44:48.864 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.35649", 2023-10-01T10:44:48.864 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:44:48.864 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 128375 2023-10-01T10:44:48.864 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.864 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:44:48.864 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.864 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:44:48.865 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:59.476366+0000", 2023-10-01T10:44:48.865 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:44:48.865 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.865 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.865 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:44:48.865 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:59.476366+0000", 2023-10-01T10:44:48.865 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:44:48.865 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.866 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.866 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:44:48.866 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:59.476365+0000", 2023-10-01T10:44:48.866 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999986 2023-10-01T10:44:48.866 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.866 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.866 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:44:48.867 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:59.476373+0000", 2023-10-01T10:44:48.867 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.6799999999999993e-06 2023-10-01T10:44:48.867 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.867 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.867 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:44:48.867 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:59.620134+0000", 2023-10-01T10:44:48.867 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.14376102900000001 2023-10-01T10:44:48.867 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.867 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.867 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:44:48.867 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:59.620147+0000", 2023-10-01T10:44:48.868 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.3054000000000001e-05 2023-10-01T10:44:48.868 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.868 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.868 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:44:48.868 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:59.630794+0000", 2023-10-01T10:44:48.868 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.01064726 2023-10-01T10:44:48.868 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.868 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.868 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:44:48.868 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:59.630931+0000", 2023-10-01T10:44:48.868 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00013696100000000001 2023-10-01T10:44:48.868 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.869 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.869 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 1,2", 2023-10-01T10:44:48.869 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:59.631071+0000", 2023-10-01T10:44:48.869 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000139837 2023-10-01T10:44:48.869 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.869 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.869 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:44:48.869 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:59.639409+0000", 2023-10-01T10:44:48.869 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0083385390000000007 2023-10-01T10:44:48.869 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.869 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.870 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:44:48.870 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:59.639514+0000", 2023-10-01T10:44:48.870 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000104684 2023-10-01T10:44:48.870 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.870 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.870 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:44:48.870 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:59.644811+0000", 2023-10-01T10:44:48.870 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0052969380000000002 2023-10-01T10:44:48.870 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.870 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.870 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:44:48.870 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:59.644982+0000", 2023-10-01T10:44:48.871 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00017147899999999999 2023-10-01T10:44:48.871 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.871 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.871 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:44:48.871 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:42:59.645035+0000", 2023-10-01T10:44:48.871 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.2441999999999997e-05 2023-10-01T10:44:48.871 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:44:48.871 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:44:48.871 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:44:48.871 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.871 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.871 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.35649.0:158065 9.2 9:4fdf2531:::benchmark_data_smithi078_113121_object46250:head [delete] snapc 0=[] ondisk+write+known_if_redirected e137)", 2023-10-01T10:44:48.872 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:43:39.400238+0000", 2023-10-01T10:44:48.872 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 69.373589265999996, 2023-10-01T10:44:48.872 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.224930456, 2023-10-01T10:44:48.872 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:44:48.872 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:44:48.872 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:44:48.872 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.35649", 2023-10-01T10:44:48.872 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:44:48.872 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 158065 2023-10-01T10:44:48.872 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.872 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:44:48.872 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.873 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:44:48.873 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.400238+0000", 2023-10-01T10:44:48.873 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:44:48.873 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.873 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.873 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:44:48.873 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.400238+0000", 2023-10-01T10:44:48.873 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:44:48.873 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.873 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.873 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:44:48.873 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.400237+0000", 2023-10-01T10:44:48.874 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:44:48.874 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.874 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.874 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:44:48.874 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.400242+0000", 2023-10-01T10:44:48.874 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.8790000000000002e-06 2023-10-01T10:44:48.874 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.874 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.874 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:44:48.874 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.400251+0000", 2023-10-01T10:44:48.874 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.8529999999999998e-06 2023-10-01T10:44:48.874 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.875 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.875 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:44:48.875 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.400258+0000", 2023-10-01T10:44:48.875 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.3699999999999997e-06 2023-10-01T10:44:48.875 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.875 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.875 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:44:48.875 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.406170+0000", 2023-10-01T10:44:48.875 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0059117900000000001 2023-10-01T10:44:48.875 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.875 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.875 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:44:48.876 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.406348+0000", 2023-10-01T10:44:48.876 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000177851 2023-10-01T10:44:48.876 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.876 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.876 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 1,2", 2023-10-01T10:44:48.876 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.406506+0000", 2023-10-01T10:44:48.876 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000158252 2023-10-01T10:44:48.876 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.876 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.876 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:44:48.876 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.409929+0000", 2023-10-01T10:44:48.876 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.003422876 2023-10-01T10:44:48.877 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.877 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.877 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:44:48.877 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.412218+0000", 2023-10-01T10:44:48.877 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0022896090000000002 2023-10-01T10:44:48.877 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.877 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.877 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:44:48.877 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.625077+0000", 2023-10-01T10:44:48.877 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.21285906399999999 2023-10-01T10:44:48.877 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.877 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.878 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:44:48.878 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.625127+0000", 2023-10-01T10:44:48.878 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.9465000000000002e-05 2023-10-01T10:44:48.878 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.878 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.878 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:44:48.878 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.625168+0000", 2023-10-01T10:44:48.878 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.1186999999999997e-05 2023-10-01T10:44:48.878 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:44:48.878 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:44:48.878 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:44:48.878 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.879 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.879 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.35649.0:158810 9.2 9:48008bee:::benchmark_data_smithi078_113121_object46995:head [delete] snapc 0=[] ondisk+write+known_if_redirected e139)", 2023-10-01T10:44:48.879 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:43:40.577229+0000", 2023-10-01T10:44:48.879 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 68.196597917000005, 2023-10-01T10:44:48.879 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.21258954699999999, 2023-10-01T10:44:48.879 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:44:48.879 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:44:48.879 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:44:48.879 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.35649", 2023-10-01T10:44:48.879 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:44:48.879 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 158810 2023-10-01T10:44:48.879 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.880 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:44:48.880 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.880 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:44:48.880 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.577229+0000", 2023-10-01T10:44:48.880 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:44:48.880 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.880 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.880 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:44:48.880 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.577229+0000", 2023-10-01T10:44:48.880 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:44:48.880 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.880 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.881 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:44:48.881 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.577228+0000", 2023-10-01T10:44:48.881 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999995 2023-10-01T10:44:48.881 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.881 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.881 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:44:48.881 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.577232+0000", 2023-10-01T10:44:48.881 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.3830000000000001e-06 2023-10-01T10:44:48.881 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.881 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.881 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:44:48.881 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.577238+0000", 2023-10-01T10:44:48.882 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.4919999999999998e-06 2023-10-01T10:44:48.882 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.882 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.882 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:44:48.882 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.577242+0000", 2023-10-01T10:44:48.882 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.41e-06 2023-10-01T10:44:48.882 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.882 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.882 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:44:48.882 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.672105+0000", 2023-10-01T10:44:48.882 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.094863399000000001 2023-10-01T10:44:48.883 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.883 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.883 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:44:48.883 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.672507+0000", 2023-10-01T10:44:48.883 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00040220000000000002 2023-10-01T10:44:48.883 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.883 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.883 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 1,2", 2023-10-01T10:44:48.883 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.672669+0000", 2023-10-01T10:44:48.883 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000161953 2023-10-01T10:44:48.883 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.884 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.884 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:44:48.884 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.686715+0000", 2023-10-01T10:44:48.884 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.014046018 2023-10-01T10:44:48.884 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.884 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.884 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:44:48.884 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.686723+0000", 2023-10-01T10:44:48.884 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.5689999999999999e-06 2023-10-01T10:44:48.884 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.884 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.884 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:44:48.885 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.789080+0000", 2023-10-01T10:44:48.885 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.10235736300000001 2023-10-01T10:44:48.885 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.885 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.885 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:44:48.885 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.789131+0000", 2023-10-01T10:44:48.885 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.0568000000000001e-05 2023-10-01T10:44:48.885 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.885 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.885 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:44:48.885 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.789818+0000", 2023-10-01T10:44:48.885 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00068768799999999999 2023-10-01T10:44:48.886 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:44:48.886 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:44:48.886 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:44:48.886 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.886 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.886 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.35649.0:158811 9.2 9:43e6941c:::benchmark_data_smithi078_113121_object46996:head [delete] snapc 0=[] ondisk+write+known_if_redirected e139)", 2023-10-01T10:44:48.886 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:43:40.577279+0000", 2023-10-01T10:44:48.886 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 68.196547643000002, 2023-10-01T10:44:48.886 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.212757111, 2023-10-01T10:44:48.886 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:44:48.886 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:44:48.886 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:44:48.887 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.35649", 2023-10-01T10:44:48.887 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:44:48.887 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 158811 2023-10-01T10:44:48.887 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.887 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:44:48.887 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.887 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:44:48.887 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.577279+0000", 2023-10-01T10:44:48.887 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:44:48.887 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.887 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.887 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:44:48.888 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.577279+0000", 2023-10-01T10:44:48.888 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:44:48.888 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.888 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.888 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:44:48.888 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.577279+0000", 2023-10-01T10:44:48.888 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999995 2023-10-01T10:44:48.888 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.888 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.888 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:44:48.888 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.577283+0000", 2023-10-01T10:44:48.888 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.0579999999999999e-06 2023-10-01T10:44:48.889 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.889 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.889 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:44:48.889 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.577292+0000", 2023-10-01T10:44:48.889 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.5289999999999997e-06 2023-10-01T10:44:48.889 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.889 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.889 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:44:48.889 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.577299+0000", 2023-10-01T10:44:48.889 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.7349999999999999e-06 2023-10-01T10:44:48.889 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.889 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.890 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:44:48.890 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.681513+0000", 2023-10-01T10:44:48.890 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.10421393299999999 2023-10-01T10:44:48.890 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.890 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.890 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:44:48.890 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.681619+0000", 2023-10-01T10:44:48.890 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000106477 2023-10-01T10:44:48.890 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.890 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.890 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 1,2", 2023-10-01T10:44:48.890 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.681807+0000", 2023-10-01T10:44:48.890 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00018774300000000001 2023-10-01T10:44:48.891 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.891 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.891 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:44:48.891 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.686728+0000", 2023-10-01T10:44:48.891 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0049204100000000001 2023-10-01T10:44:48.891 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.891 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.891 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:44:48.892 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.686817+0000", 2023-10-01T10:44:48.892 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.9351999999999994e-05 2023-10-01T10:44:48.892 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.892 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.892 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:44:48.892 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.789872+0000", 2023-10-01T10:44:48.892 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.10305476700000001 2023-10-01T10:44:48.892 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.892 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.892 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:44:48.892 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.789959+0000", 2023-10-01T10:44:48.892 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.7702999999999997e-05 2023-10-01T10:44:48.893 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.893 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.893 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:44:48.893 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.790036+0000", 2023-10-01T10:44:48.893 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.6994000000000002e-05 2023-10-01T10:44:48.893 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:44:48.893 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:44:48.893 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:44:48.893 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.894 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.894 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.35649.0:158815 9.1 9:81709652:::benchmark_data_smithi078_113121_object47000:head [delete] snapc 0=[] ondisk+write+known_if_redirected e139)", 2023-10-01T10:44:48.894 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:43:40.583378+0000", 2023-10-01T10:44:48.894 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 68.190449231000002, 2023-10-01T10:44:48.894 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.20006031099999999, 2023-10-01T10:44:48.894 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:44:48.894 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:44:48.894 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:44:48.894 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.35649", 2023-10-01T10:44:48.894 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:44:48.895 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 158815 2023-10-01T10:44:48.895 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.895 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:44:48.895 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.895 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:44:48.895 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.583378+0000", 2023-10-01T10:44:48.895 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:44:48.895 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.895 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.895 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:44:48.896 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.583378+0000", 2023-10-01T10:44:48.896 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:44:48.896 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.896 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.896 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:44:48.896 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.583376+0000", 2023-10-01T10:44:48.896 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999981 2023-10-01T10:44:48.896 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.896 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.896 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:44:48.897 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.583384+0000", 2023-10-01T10:44:48.897 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.1270000000000003e-06 2023-10-01T10:44:48.897 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.897 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.897 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:44:48.897 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.672010+0000", 2023-10-01T10:44:48.897 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.088626446999999997 2023-10-01T10:44:48.897 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.897 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.897 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:44:48.898 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.672027+0000", 2023-10-01T10:44:48.898 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.6361000000000001e-05 2023-10-01T10:44:48.898 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.898 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.898 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:44:48.898 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.676860+0000", 2023-10-01T10:44:48.898 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00483279 2023-10-01T10:44:48.898 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.898 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.898 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:44:48.899 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.676962+0000", 2023-10-01T10:44:48.899 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000102567 2023-10-01T10:44:48.899 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.899 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.899 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 6,7", 2023-10-01T10:44:48.899 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.677052+0000", 2023-10-01T10:44:48.899 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.0008999999999998e-05 2023-10-01T10:44:48.899 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.900 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.900 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:44:48.900 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.681732+0000", 2023-10-01T10:44:48.900 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0046803000000000001 2023-10-01T10:44:48.900 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.900 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.900 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:44:48.900 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.691375+0000", 2023-10-01T10:44:48.900 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0096430100000000005 2023-10-01T10:44:48.901 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.901 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.901 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:44:48.901 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.783275+0000", 2023-10-01T10:44:48.901 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.091899397999999993 2023-10-01T10:44:48.901 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.901 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.901 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:44:48.901 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.783369+0000", 2023-10-01T10:44:48.902 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.3764999999999994e-05 2023-10-01T10:44:48.902 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.902 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.902 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:44:48.902 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.783438+0000", 2023-10-01T10:44:48.902 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.9298e-05 2023-10-01T10:44:48.902 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:44:48.903 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:44:48.903 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:44:48.903 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.903 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.903 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.35649.0:159544 9.1 9:83c5b7cd:::benchmark_data_smithi078_113121_object47729:head [delete] snapc 0=[] ondisk+write+known_if_redirected e139)", 2023-10-01T10:44:48.903 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:43:41.686189+0000", 2023-10-01T10:44:48.903 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 67.087637494999996, 2023-10-01T10:44:48.903 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.19142947499999999, 2023-10-01T10:44:48.904 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:44:48.904 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:44:48.904 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:44:48.904 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.35649", 2023-10-01T10:44:48.904 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:44:48.904 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 159544 2023-10-01T10:44:48.904 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.904 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:44:48.904 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.904 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:44:48.904 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.686189+0000", 2023-10-01T10:44:48.904 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:44:48.904 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.905 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.905 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:44:48.905 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.686189+0000", 2023-10-01T10:44:48.905 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:44:48.905 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.905 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.905 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:44:48.905 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.686188+0000", 2023-10-01T10:44:48.905 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:44:48.905 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.905 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.905 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:44:48.905 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.686194+0000", 2023-10-01T10:44:48.906 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.8039999999999998e-06 2023-10-01T10:44:48.906 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.906 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.906 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:44:48.906 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.686204+0000", 2023-10-01T10:44:48.906 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.0305e-05 2023-10-01T10:44:48.906 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.906 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.906 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:44:48.906 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.686212+0000", 2023-10-01T10:44:48.906 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.0020000000000006e-06 2023-10-01T10:44:48.906 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.907 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.907 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:44:48.907 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.686254+0000", 2023-10-01T10:44:48.907 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.1561999999999999e-05 2023-10-01T10:44:48.907 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.907 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.907 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:44:48.907 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.686360+0000", 2023-10-01T10:44:48.907 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000105676 2023-10-01T10:44:48.907 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.907 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.907 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 6,7", 2023-10-01T10:44:48.907 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.686464+0000", 2023-10-01T10:44:48.908 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000104209 2023-10-01T10:44:48.908 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.908 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.908 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:44:48.908 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.690429+0000", 2023-10-01T10:44:48.908 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0039646619999999999 2023-10-01T10:44:48.908 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.908 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.908 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:44:48.908 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.691141+0000", 2023-10-01T10:44:48.908 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00071247699999999999 2023-10-01T10:44:48.908 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.908 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.909 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:44:48.909 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.877553+0000", 2023-10-01T10:44:48.909 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.18641245300000001 2023-10-01T10:44:48.909 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.909 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.909 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:44:48.909 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.877584+0000", 2023-10-01T10:44:48.909 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.0642000000000002e-05 2023-10-01T10:44:48.909 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.909 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.909 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:44:48.909 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.877619+0000", 2023-10-01T10:44:48.909 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.4727999999999997e-05 2023-10-01T10:44:48.910 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:44:48.910 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:44:48.910 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:44:48.910 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.910 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.910 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.35649.0:159554 9.1 9:866e6940:::benchmark_data_smithi078_113121_object47739:head [delete] snapc 0=[] ondisk+write+known_if_redirected e139)", 2023-10-01T10:44:48.910 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:43:41.702172+0000", 2023-10-01T10:44:48.910 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 67.071655192999998, 2023-10-01T10:44:48.910 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.17563100000000001, 2023-10-01T10:44:48.910 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:44:48.910 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:44:48.910 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:44:48.910 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.35649", 2023-10-01T10:44:48.911 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:44:48.911 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 159554 2023-10-01T10:44:48.911 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.911 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:44:48.911 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.911 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:44:48.911 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.702172+0000", 2023-10-01T10:44:48.911 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:44:48.911 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.911 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.911 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:44:48.911 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.702172+0000", 2023-10-01T10:44:48.911 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:44:48.912 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.912 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.912 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:44:48.912 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.702171+0000", 2023-10-01T10:44:48.912 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:44:48.912 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.912 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.912 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:44:48.912 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.702177+0000", 2023-10-01T10:44:48.912 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.8699999999999997e-06 2023-10-01T10:44:48.912 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.913 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.913 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:44:48.913 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.702188+0000", 2023-10-01T10:44:48.913 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1494e-05 2023-10-01T10:44:48.913 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.913 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.913 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:44:48.913 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.702197+0000", 2023-10-01T10:44:48.913 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.986e-06 2023-10-01T10:44:48.913 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.913 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.913 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:44:48.913 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.702238+0000", 2023-10-01T10:44:48.914 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.1057999999999999e-05 2023-10-01T10:44:48.914 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.914 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.914 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:44:48.914 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.702373+0000", 2023-10-01T10:44:48.914 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00013459400000000001 2023-10-01T10:44:48.914 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.914 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.914 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 6,7", 2023-10-01T10:44:48.914 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.702458+0000", 2023-10-01T10:44:48.914 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.5320000000000003e-05 2023-10-01T10:44:48.915 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.915 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.915 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:44:48.915 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.705539+0000", 2023-10-01T10:44:48.915 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0030812299999999999 2023-10-01T10:44:48.915 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.915 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.915 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:44:48.915 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.707125+0000", 2023-10-01T10:44:48.915 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0015853270000000001 2023-10-01T10:44:48.915 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.915 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.915 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:44:48.916 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.877694+0000", 2023-10-01T10:44:48.916 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.17056949900000001 2023-10-01T10:44:48.916 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.916 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.916 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:44:48.916 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.877737+0000", 2023-10-01T10:44:48.916 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.2760999999999999e-05 2023-10-01T10:44:48.916 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.916 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.916 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:44:48.916 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.877803+0000", 2023-10-01T10:44:48.916 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.5870999999999993e-05 2023-10-01T10:44:48.917 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:44:48.917 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:44:48.917 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:44:48.917 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.917 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.917 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.35649.0:159558 9.1 9:86d77d86:::benchmark_data_smithi078_113121_object47743:head [delete] snapc 0=[] ondisk+write+known_if_redirected e139)", 2023-10-01T10:44:48.917 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:43:41.707304+0000", 2023-10-01T10:44:48.917 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 67.066522978999998, 2023-10-01T10:44:48.917 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.17061336999999999, 2023-10-01T10:44:48.917 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:44:48.917 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:44:48.917 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:44:48.918 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.35649", 2023-10-01T10:44:48.918 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:44:48.918 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 159558 2023-10-01T10:44:48.918 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.918 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:44:48.918 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.918 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:44:48.918 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.707304+0000", 2023-10-01T10:44:48.918 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:44:48.918 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.918 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.918 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:44:48.919 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.707304+0000", 2023-10-01T10:44:48.919 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:44:48.919 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.919 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.919 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:44:48.919 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.707303+0000", 2023-10-01T10:44:48.919 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:44:48.919 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.919 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.919 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:44:48.919 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.707309+0000", 2023-10-01T10:44:48.919 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.7429999999999999e-06 2023-10-01T10:44:48.919 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.920 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.920 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:44:48.920 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.707317+0000", 2023-10-01T10:44:48.920 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.4929999999999998e-06 2023-10-01T10:44:48.920 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.920 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.920 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:44:48.920 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.707322+0000", 2023-10-01T10:44:48.920 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.2959999999999998e-06 2023-10-01T10:44:48.920 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.920 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.920 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:44:48.921 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.707359+0000", 2023-10-01T10:44:48.921 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.6362000000000002e-05 2023-10-01T10:44:48.921 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.921 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.921 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:44:48.921 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.707431+0000", 2023-10-01T10:44:48.921 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.2000000000000002e-05 2023-10-01T10:44:48.921 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.921 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.921 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 6,7", 2023-10-01T10:44:48.921 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.707540+0000", 2023-10-01T10:44:48.921 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000109561 2023-10-01T10:44:48.922 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.922 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.922 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:44:48.922 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.712274+0000", 2023-10-01T10:44:48.922 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0047335429999999998 2023-10-01T10:44:48.922 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.922 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.922 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:44:48.922 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.712346+0000", 2023-10-01T10:44:48.922 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.2365000000000003e-05 2023-10-01T10:44:48.922 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.922 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.923 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:44:48.923 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.877841+0000", 2023-10-01T10:44:48.923 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.16549518799999999 2023-10-01T10:44:48.923 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.923 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.923 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:44:48.923 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.877884+0000", 2023-10-01T10:44:48.923 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.2599000000000003e-05 2023-10-01T10:44:48.923 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.923 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.923 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:44:48.923 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.877917+0000", 2023-10-01T10:44:48.923 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.3265000000000001e-05 2023-10-01T10:44:48.924 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:44:48.924 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:44:48.924 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:44:48.924 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.924 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.924 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.35649.0:159579 9.1 9:823bd484:::benchmark_data_smithi078_113121_object47764:head [delete] snapc 0=[] ondisk+write+known_if_redirected e139)", 2023-10-01T10:44:48.924 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:43:41.742298+0000", 2023-10-01T10:44:48.924 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 67.031529039000006, 2023-10-01T10:44:48.925 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.168909644, 2023-10-01T10:44:48.925 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:44:48.925 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:44:48.925 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:44:48.925 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.35649", 2023-10-01T10:44:48.925 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:44:48.925 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 159579 2023-10-01T10:44:48.925 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.925 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:44:48.925 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.925 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:44:48.925 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.742298+0000", 2023-10-01T10:44:48.925 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:44:48.926 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.926 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.926 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:44:48.926 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.742298+0000", 2023-10-01T10:44:48.926 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:44:48.926 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.926 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.926 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:44:48.926 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.742296+0000", 2023-10-01T10:44:48.926 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999981 2023-10-01T10:44:48.927 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.927 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.927 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:44:48.927 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.742305+0000", 2023-10-01T10:44:48.927 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.5639999999999997e-06 2023-10-01T10:44:48.927 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.927 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.927 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:44:48.927 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.867797+0000", 2023-10-01T10:44:48.927 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.12549238500000001 2023-10-01T10:44:48.928 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.928 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.928 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:44:48.928 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.867811+0000", 2023-10-01T10:44:48.928 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.3604999999999999e-05 2023-10-01T10:44:48.928 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.928 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.928 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:44:48.928 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.878121+0000", 2023-10-01T10:44:48.928 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.010310191999999999 2023-10-01T10:44:48.928 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.929 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.929 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:44:48.929 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.878235+0000", 2023-10-01T10:44:48.929 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000114453 2023-10-01T10:44:48.929 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.929 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.929 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 6,7", 2023-10-01T10:44:48.929 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.878350+0000", 2023-10-01T10:44:48.929 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00011483 2023-10-01T10:44:48.929 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.929 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.930 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:44:48.930 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.893299+0000", 2023-10-01T10:44:48.930 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.014948398999999999 2023-10-01T10:44:48.930 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.930 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.930 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:44:48.930 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.909554+0000", 2023-10-01T10:44:48.930 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.016255576000000001 2023-10-01T10:44:48.930 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.931 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.931 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:44:48.931 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.911116+0000", 2023-10-01T10:44:48.931 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0015614649999999999 2023-10-01T10:44:48.931 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.931 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.931 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:44:48.931 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.911163+0000", 2023-10-01T10:44:48.931 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.7888e-05 2023-10-01T10:44:48.931 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:44:48.931 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:44:48.931 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:44:48.932 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.911207+0000", 2023-10-01T10:44:48.932 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.3999999999999999e-05 2023-10-01T10:44:48.932 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:44:48.932 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:44:48.932 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:44:48.932 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:44:48.932 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:44:48.932 INFO:teuthology.orchestra.run.smithi078.stdout:} 2023-10-01T10:44:48.933 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:48.934 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:48 smithi078 bash[78274]: cluster 2023-10-01T10:44:47.534274+0000 osd.5 (osd.5) 72 : cluster 3 failed to encode map e143 with expected crc 2023-10-01T10:44:48.934 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:48 smithi078 bash[78274]: audit 2023-10-01T10:44:47.534972+0000 mon.a (mon.0) 1029 : audit 1 from='client.? 172.21.15.78:0/1500198906' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-01T10:44:48.934 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:48 smithi078 bash[78274]: cluster 2023-10-01T10:44:47.535046+0000 mon.a (mon.0) 1030 : cluster 0 osdmap e143: 8 total, 8 up, 8 in 2023-10-01T10:44:48.934 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:48 smithi078 bash[78274]: cluster 2023-10-01T10:44:47.549361+0000 osd.7 (osd.7) 90 : cluster 3 failed to encode map e143 with expected crc 2023-10-01T10:44:48.935 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:48 smithi078 bash[78274]: cluster 2023-10-01T10:44:47.556467+0000 osd.6 (osd.6) 55 : cluster 3 failed to encode map e143 with expected crc 2023-10-01T10:44:48.935 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:48 smithi078 bash[75299]: cluster 2023-10-01T10:44:47.534274+0000 osd.5 (osd.5) 72 : cluster 3 failed to encode map e143 with expected crc 2023-10-01T10:44:48.935 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:48 smithi078 bash[75299]: audit 2023-10-01T10:44:47.534972+0000 mon.a (mon.0) 1029 : audit 1 from='client.? 172.21.15.78:0/1500198906' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-01T10:44:48.935 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:48 smithi078 bash[75299]: cluster 2023-10-01T10:44:47.535046+0000 mon.a (mon.0) 1030 : cluster 0 osdmap e143: 8 total, 8 up, 8 in 2023-10-01T10:44:48.935 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:48 smithi078 bash[75299]: cluster 2023-10-01T10:44:47.549361+0000 osd.7 (osd.7) 90 : cluster 3 failed to encode map e143 with expected crc 2023-10-01T10:44:48.935 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:48 smithi078 bash[75299]: cluster 2023-10-01T10:44:47.556467+0000 osd.6 (osd.6) 55 : cluster 3 failed to encode map e143 with expected crc 2023-10-01T10:44:48.966 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:48.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:48.972 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:44:49.073 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:49.106 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:49.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:49.113 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:44:49.213 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:49.262 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:49.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:49.264 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:44:49.365 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:49.413 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:49.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:49.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:49.548 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:49 smithi173 bash[58816]: cluster 2023-10-01T10:44:47.802282+0000 mgr.y (mgr.25072) 812 : cluster 0 pgmap v730: 89 pgs: 89 active+clean; 1.2 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 712 op/s 2023-10-01T10:44:49.549 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:49 smithi078 bash[75299]: cluster 2023-10-01T10:44:47.802282+0000 mgr.y (mgr.25072) 812 : cluster 0 pgmap v730: 89 pgs: 89 active+clean; 1.2 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 712 op/s 2023-10-01T10:44:49.549 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:49 smithi078 bash[78274]: cluster 2023-10-01T10:44:47.802282+0000 mgr.y (mgr.25072) 812 : cluster 0 pgmap v730: 89 pgs: 89 active+clean; 1.2 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 712 op/s 2023-10-01T10:44:49.580 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:49.606 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:49.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:49.610 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:49.711 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:49.734 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:49.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:49.742 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:44:49.843 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:49.878 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:49.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:49.879 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:44:49.980 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:50.019 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:50.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:50.023 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:44:50.124 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:50.148 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:50.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:50.149 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:44:50.250 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:50.278 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:50.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:50.281 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:44:50.381 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:50.416 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:50.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:50.422 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:44:50.523 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:50.557 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:50.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:50.562 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:44:50.582 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:44:50 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:44:50] "GET /metrics HTTP/1.1" 200 36244 "" "Prometheus/2.33.4" 2023-10-01T10:44:50.662 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:50.693 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:50.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:50.699 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:44:50.799 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:50.832 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:50.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:50.842 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:44:50.942 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:50.978 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:50.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:50.980 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:51.081 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:51.117 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:51.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:51.119 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:44:51.219 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:51.242 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:51.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:51.248 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:44:51.343 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-01T10:44:51.347 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-01T10:44:51.349 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:51.385 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:51.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:51.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:44:51.490 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:51.516 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:51.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:51.522 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:44:51.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:51 smithi173 bash[58816]: cluster 2023-10-01T10:44:49.802948+0000 mgr.y (mgr.25072) 813 : cluster 0 pgmap v731: 89 pgs: 89 active+clean; 1.1 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 704 op/s 2023-10-01T10:44:51.622 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:51.653 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:51.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:51.658 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:44:51.759 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:51.767 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:51 smithi078 bash[75299]: cluster 2023-10-01T10:44:49.802948+0000 mgr.y (mgr.25072) 813 : cluster 0 pgmap v731: 89 pgs: 89 active+clean; 1.1 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 704 op/s 2023-10-01T10:44:51.767 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:51 smithi078 bash[78274]: cluster 2023-10-01T10:44:49.802948+0000 mgr.y (mgr.25072) 813 : cluster 0 pgmap v731: 89 pgs: 89 active+clean; 1.1 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 704 op/s 2023-10-01T10:44:51.788 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:51.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:51.793 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:44:51.893 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:51.924 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:51.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:51.928 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:44:52.029 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:52.053 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:52.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:52.055 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:44:52.120 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T10:44:52.152 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T10:44:52.156 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:52.198 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T10:44:52.201 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:52.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:52.203 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:44:52.242 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T10:44:52.284 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T10:44:52.304 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:52.319 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T10:44:52.336 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:52.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:52.341 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:44:52.362 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T10:44:52.402 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T10:44:52.408 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T10:44:52.408 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T10:44:52.408 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T10:44:52.408 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T10:44:52.408 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T10:44:52.408 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T10:44:52.408 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T10:44:52.408 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T10:44:52.442 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:52.470 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:52.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:52.478 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:52.567 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:52 smithi173 bash[58816]: audit 2023-10-01T10:44:52.080465+0000 mon.c (mon.2) 143 : audit 0 from='client.? 172.21.15.78:0/1099530193' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:44:52.578 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:52.604 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:52.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:52.607 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:44:52.708 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:52.717 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:52 smithi078 bash[75299]: audit 2023-10-01T10:44:52.080465+0000 mon.c (mon.2) 143 : audit 0 from='client.? 172.21.15.78:0/1099530193' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:44:52.717 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:52 smithi078 bash[78274]: audit 2023-10-01T10:44:52.080465+0000 mon.c (mon.2) 143 : audit 0 from='client.? 172.21.15.78:0/1099530193' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:44:52.737 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:52.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:52.739 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:44:52.840 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:52.865 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:52.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:52.875 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:52.975 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:53.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:53.009 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:53.009 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:44:53.110 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:53.140 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:53.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:53.145 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:44:53.246 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:53.279 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:53.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:53.286 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:44:53.387 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:53.411 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:53.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:53.414 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:44:53.514 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:53.556 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:53.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:53.557 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:44:53.580 INFO:teuthology.orchestra.run.smithi078.stderr:noscrub is unset 2023-10-01T10:44:53.649 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:53 smithi078 bash[75299]: cluster 2023-10-01T10:44:51.805151+0000 mgr.y (mgr.25072) 814 : cluster 0 pgmap v732: 89 pgs: 89 active+clean; 1.0 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 696 op/s 2023-10-01T10:44:53.650 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:53 smithi078 bash[75299]: audit 2023-10-01T10:44:52.591407+0000 mon.c (mon.2) 144 : audit 1 from='client.? 172.21.15.78:0/862718612' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T10:44:53.650 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:53 smithi078 bash[75299]: audit 2023-10-01T10:44:52.592592+0000 mon.a (mon.0) 1031 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T10:44:53.650 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:53 smithi078 bash[78274]: cluster 2023-10-01T10:44:51.805151+0000 mgr.y (mgr.25072) 814 : cluster 0 pgmap v732: 89 pgs: 89 active+clean; 1.0 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 696 op/s 2023-10-01T10:44:53.650 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:53 smithi078 bash[78274]: audit 2023-10-01T10:44:52.591407+0000 mon.c (mon.2) 144 : audit 1 from='client.? 172.21.15.78:0/862718612' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T10:44:53.650 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:53 smithi078 bash[78274]: audit 2023-10-01T10:44:52.592592+0000 mon.a (mon.0) 1031 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T10:44:53.658 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:53.692 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:53.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:53.700 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:44:53.800 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:53.836 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:53.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:53.839 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:44:53.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:53 smithi173 bash[58816]: cluster 2023-10-01T10:44:51.805151+0000 mgr.y (mgr.25072) 814 : cluster 0 pgmap v732: 89 pgs: 89 active+clean; 1.0 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 696 op/s 2023-10-01T10:44:53.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:53 smithi173 bash[58816]: audit 2023-10-01T10:44:52.591407+0000 mon.c (mon.2) 144 : audit 1 from='client.? 172.21.15.78:0/862718612' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T10:44:53.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:53 smithi173 bash[58816]: audit 2023-10-01T10:44:52.592592+0000 mon.a (mon.0) 1031 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T10:44:53.940 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:53.967 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:53.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:53.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:44:54.072 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:54.091 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:54.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:54.095 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:44:54.196 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:54.224 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:54.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:54.227 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:44:54.328 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:54.351 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:54.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:54.357 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:54.458 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:54.498 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:54.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:54.503 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:44:54.594 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:54 smithi078 bash[75299]: cluster 2023-10-01T10:44:53.564793+0000 mon.a (mon.0) 1032 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:44:54.594 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:54 smithi078 bash[75299]: cluster 2023-10-01T10:44:53.571406+0000 osd.5 (osd.5) 73 : cluster 3 failed to encode map e144 with expected crc 2023-10-01T10:44:54.594 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:54 smithi078 bash[75299]: audit 2023-10-01T10:44:53.571557+0000 mon.a (mon.0) 1033 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T10:44:54.594 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:54 smithi078 bash[75299]: cluster 2023-10-01T10:44:53.571719+0000 mon.a (mon.0) 1034 : cluster 0 osdmap e144: 8 total, 8 up, 8 in 2023-10-01T10:44:54.595 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:54 smithi078 bash[75299]: cluster 2023-10-01T10:44:53.574679+0000 osd.7 (osd.7) 91 : cluster 3 failed to encode map e144 with expected crc 2023-10-01T10:44:54.604 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:54.631 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:54.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:54.633 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:44:54.734 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:54.743 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:54 smithi173 bash[58816]: cluster 2023-10-01T10:44:53.564793+0000 mon.a (mon.0) 1032 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:44:54.743 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:54 smithi173 bash[58816]: cluster 2023-10-01T10:44:53.571406+0000 osd.5 (osd.5) 73 : cluster 3 failed to encode map e144 with expected crc 2023-10-01T10:44:54.744 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:54 smithi173 bash[58816]: audit 2023-10-01T10:44:53.571557+0000 mon.a (mon.0) 1033 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T10:44:54.744 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:54 smithi173 bash[58816]: cluster 2023-10-01T10:44:53.571719+0000 mon.a (mon.0) 1034 : cluster 0 osdmap e144: 8 total, 8 up, 8 in 2023-10-01T10:44:54.744 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:54 smithi173 bash[58816]: cluster 2023-10-01T10:44:53.574679+0000 osd.7 (osd.7) 91 : cluster 3 failed to encode map e144 with expected crc 2023-10-01T10:44:54.771 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:54.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:54.777 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:44:54.878 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:54.888 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:54 smithi078 bash[78274]: cluster 2023-10-01T10:44:53.564793+0000 mon.a (mon.0) 1032 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:44:54.888 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:54 smithi078 bash[78274]: cluster 2023-10-01T10:44:53.571406+0000 osd.5 (osd.5) 73 : cluster 3 failed to encode map e144 with expected crc 2023-10-01T10:44:54.888 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:54 smithi078 bash[78274]: audit 2023-10-01T10:44:53.571557+0000 mon.a (mon.0) 1033 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T10:44:54.888 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:54 smithi078 bash[78274]: cluster 2023-10-01T10:44:53.571719+0000 mon.a (mon.0) 1034 : cluster 0 osdmap e144: 8 total, 8 up, 8 in 2023-10-01T10:44:54.888 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:54 smithi078 bash[78274]: cluster 2023-10-01T10:44:53.574679+0000 osd.7 (osd.7) 91 : cluster 3 failed to encode map e144 with expected crc 2023-10-01T10:44:54.902 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:54.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:54.908 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:44:55.008 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:55.044 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:55.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:55.047 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:44:55.148 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:55.180 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:55.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:55.182 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:44:55.283 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:55.322 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:55.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:55.327 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:44:55.428 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:55.457 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:55.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:55.462 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:44:55.563 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:55.589 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:55.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:55.598 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:44:55.699 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:55.724 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:44:55.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:55.733 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:44:55.775 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T10:44:55.834 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:55.843 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:55 smithi173 bash[58816]: cluster 2023-10-01T10:44:53.805669+0000 mgr.y (mgr.25072) 815 : cluster 0 pgmap v734: 89 pgs: 89 active+clean; 973 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 705 op/s 2023-10-01T10:44:55.867 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:55.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:55.869 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:44:55.970 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:56.005 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:56.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:56.010 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:44:56.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:55 smithi078 bash[78274]: cluster 2023-10-01T10:44:53.805669+0000 mgr.y (mgr.25072) 815 : cluster 0 pgmap v734: 89 pgs: 89 active+clean; 973 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 705 op/s 2023-10-01T10:44:56.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:55 smithi078 bash[75299]: cluster 2023-10-01T10:44:53.805669+0000 mgr.y (mgr.25072) 815 : cluster 0 pgmap v734: 89 pgs: 89 active+clean; 973 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 705 op/s 2023-10-01T10:44:56.111 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:56.137 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:56.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:56.139 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:44:56.240 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:56.269 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:56.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:56.276 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:44:56.377 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:56.407 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:56.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:56.409 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:44:56.488 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T10:44:56.510 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:56.543 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:56.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:56.550 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:44:56.651 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:56.690 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:56.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:56.692 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:44:56.792 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:56.825 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:56.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:56.827 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:44:56.837 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T10:44:56.928 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:56.951 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:56.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:56.953 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:44:57.054 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:57.087 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:57.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:57.089 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:44:57.190 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:57.220 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:57.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:57.228 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:57.328 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:57.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:57.366 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:57.366 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:44:57.467 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:57.502 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:57.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:57.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:57.603 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:57 smithi173 bash[58816]: cluster 2023-10-01T10:44:55.806272+0000 mgr.y (mgr.25072) 816 : cluster 0 pgmap v735: 89 pgs: 89 active+clean; 888 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 735 op/s 2023-10-01T10:44:57.604 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:57.647 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:44:57.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:57.651 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:44:57.751 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:57.783 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:57.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:57.787 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:44:57.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:57 smithi078 bash[78274]: cluster 2023-10-01T10:44:55.806272+0000 mgr.y (mgr.25072) 816 : cluster 0 pgmap v735: 89 pgs: 89 active+clean; 888 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 735 op/s 2023-10-01T10:44:57.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:57 smithi078 bash[75299]: cluster 2023-10-01T10:44:55.806272+0000 mgr.y (mgr.25072) 816 : cluster 0 pgmap v735: 89 pgs: 89 active+clean; 888 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 735 op/s 2023-10-01T10:44:57.887 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:57.918 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:57.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:57.923 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:58.022 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:44:58.023 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:58.058 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:58.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:58.061 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:44:58.162 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:58.212 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:58.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:58.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:44:58.315 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:58.343 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:58.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:58.352 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:44:58.452 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:58.477 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:58.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:58.480 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:44:58.581 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:58.609 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:44:58.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:58.611 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:44:58.712 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:58.742 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:58.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:58.750 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:44:58.851 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:58.873 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:58.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:58.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:44:58.976 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:59.008 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:44:59.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:59.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:44:59.111 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:59.138 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:59.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:59.140 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:44:59.241 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:59.273 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:59.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:59.280 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:44:59.381 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:59.411 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:44:59.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:59.414 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:44:59.515 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:59.541 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:59.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:59.544 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:44:59.645 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:59.676 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:44:59.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:59.679 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:44:59.767 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:44:59 smithi173 bash[58816]: cluster 2023-10-01T10:44:57.806764+0000 mgr.y (mgr.25072) 817 : cluster 0 pgmap v736: 89 pgs: 89 active+clean; 831 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2023-10-01T10:44:59.780 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:59.808 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:44:59.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:59.810 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:44:59.867 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:44:59 smithi078 bash[78274]: cluster 2023-10-01T10:44:57.806764+0000 mgr.y (mgr.25072) 817 : cluster 0 pgmap v736: 89 pgs: 89 active+clean; 831 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2023-10-01T10:44:59.868 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:44:59 smithi078 bash[75299]: cluster 2023-10-01T10:44:57.806764+0000 mgr.y (mgr.25072) 817 : cluster 0 pgmap v736: 89 pgs: 89 active+clean; 831 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2023-10-01T10:44:59.911 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:59.945 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:44:59.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:44:59.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:00.052 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:00.058 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:45:00.095 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:00.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:00.098 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:45:00.198 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:00.239 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:00.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:00.245 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:45:00.269 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:45:00 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:45:00] "GET /metrics HTTP/1.1" 200 36244 "" "Prometheus/2.33.4" 2023-10-01T10:45:00.346 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:00.380 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:00.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:00.383 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:45:00.483 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:00.515 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:00.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:00.522 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:45:00.622 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:00.656 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:00.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:00.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:45:00.759 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:00.784 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:00.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:00.789 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:45:00.889 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:00.925 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:00.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:00.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:45:01.036 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:01.064 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:01.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:01.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:45:01.170 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:01.194 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:01.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:01.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:45:01.202 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-10-01T10:45:01.202 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T10:45:01.300 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:01.329 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:01.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:01.334 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:01.435 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:01.467 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:01.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:01.472 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:45:01.573 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:01.599 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:01.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:01.601 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:45:01.611 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:01 smithi173 bash[58816]: cluster 2023-10-01T10:44:59.807418+0000 mgr.y (mgr.25072) 818 : cluster 0 pgmap v737: 89 pgs: 89 active+clean; 688 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 712 op/s 2023-10-01T10:45:01.702 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:01.712 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:01 smithi078 bash[78274]: cluster 2023-10-01T10:44:59.807418+0000 mgr.y (mgr.25072) 818 : cluster 0 pgmap v737: 89 pgs: 89 active+clean; 688 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 712 op/s 2023-10-01T10:45:01.713 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:01 smithi078 bash[75299]: cluster 2023-10-01T10:44:59.807418+0000 mgr.y (mgr.25072) 818 : cluster 0 pgmap v737: 89 pgs: 89 active+clean; 688 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 712 op/s 2023-10-01T10:45:01.736 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:01.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:01.738 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:45:01.839 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:01.864 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:01.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:01.868 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:45:01.969 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:02.003 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:02.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:02.007 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:45:02.038 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-01T10:45:02.108 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:02.136 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:02.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:02.138 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:45:02.239 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:02.279 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:02.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:02.283 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:45:02.384 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:02.418 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:02.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:02.424 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:45:02.524 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:02.548 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:02.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:02.551 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:45:02.652 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:02.678 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:02.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:02.680 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:45:02.782 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:02.801 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:02.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:02.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:45:02.843 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:45:02.904 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:02.934 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:02.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:02.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:45:03.037 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:03.064 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:03.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:03.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:45:03.170 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:03.187 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:03.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:03.189 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:45:03.289 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:03.327 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:03.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:03.332 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:45:03.433 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:03.470 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:03.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:03.473 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:03.566 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:03 smithi078 bash[78274]: cluster 2023-10-01T10:45:01.808094+0000 mgr.y (mgr.25072) 819 : cluster 0 pgmap v738: 89 pgs: 89 active+clean; 607 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 711 op/s 2023-10-01T10:45:03.567 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:03 smithi078 bash[78274]: audit 2023-10-01T10:45:02.418968+0000 mon.a (mon.0) 1035 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:45:03.567 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:03 smithi078 bash[75299]: cluster 2023-10-01T10:45:01.808094+0000 mgr.y (mgr.25072) 819 : cluster 0 pgmap v738: 89 pgs: 89 active+clean; 607 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 711 op/s 2023-10-01T10:45:03.567 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:03 smithi078 bash[75299]: audit 2023-10-01T10:45:02.418968+0000 mon.a (mon.0) 1035 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:45:03.574 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:03.607 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:03.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:03.614 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:45:03.715 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:03.724 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:03 smithi173 bash[58816]: cluster 2023-10-01T10:45:01.808094+0000 mgr.y (mgr.25072) 819 : cluster 0 pgmap v738: 89 pgs: 89 active+clean; 607 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 711 op/s 2023-10-01T10:45:03.724 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:03 smithi173 bash[58816]: audit 2023-10-01T10:45:02.418968+0000 mon.a (mon.0) 1035 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:45:03.740 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:03.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:03.744 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:45:03.845 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:03.878 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:03.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:03.884 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:45:03.985 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:04.018 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:04.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:04.021 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:45:04.121 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:04.158 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:04.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:04.167 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:04.267 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:04.306 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:04.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:04.309 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:45:04.399 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:04 smithi173 bash[58816]: audit 2023-10-01T10:45:03.918890+0000 mon.a (mon.0) 1036 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:45:04.409 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:04.434 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:04.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:04.439 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:45:04.540 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:04.555 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:04 smithi078 bash[75299]: audit 2023-10-01T10:45:03.918890+0000 mon.a (mon.0) 1036 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:45:04.556 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:04 smithi078 bash[78274]: audit 2023-10-01T10:45:03.918890+0000 mon.a (mon.0) 1036 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:45:04.575 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:04.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:04.578 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:45:04.678 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:04.706 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:04.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:04.708 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:45:04.808 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:04.831 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:04.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:04.837 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:45:04.937 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:04.980 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:04.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:04.981 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:45:05.082 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:05.106 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:05.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:05.110 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:05.211 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:05.247 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:05.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:05.252 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:45:05.353 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:05.381 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:05.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:05.385 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:45:05.486 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:05.523 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:05.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:05.529 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:05.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:05 smithi078 bash[78274]: cluster 2023-10-01T10:45:03.808570+0000 mgr.y (mgr.25072) 820 : cluster 0 pgmap v739: 89 pgs: 89 active+clean; 527 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 758 op/s 2023-10-01T10:45:05.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:05 smithi078 bash[75299]: cluster 2023-10-01T10:45:03.808570+0000 mgr.y (mgr.25072) 820 : cluster 0 pgmap v739: 89 pgs: 89 active+clean; 527 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 758 op/s 2023-10-01T10:45:05.631 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:05.662 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:05.665 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:05 smithi173 bash[58816]: cluster 2023-10-01T10:45:03.808570+0000 mgr.y (mgr.25072) 820 : cluster 0 pgmap v739: 89 pgs: 89 active+clean; 527 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 758 op/s 2023-10-01T10:45:05.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:05.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:05.767 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:05.788 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:05.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:05.792 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:45:05.892 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:05.925 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:05.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:05.932 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:06.033 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:06.073 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:06.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:06.078 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:06.179 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:06.210 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:06.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:06.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:45:06.313 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:06.340 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:06.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:06.343 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:06.443 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:06.469 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:06.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:06.472 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:45:06.573 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:06.612 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:06.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:06.615 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:45:06.716 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:06.747 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:06.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:06.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:06.855 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:06.879 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T10:45:06.882 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:06.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:06.887 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:45:06.988 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:07.017 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:07.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:07.028 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:07.129 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:07.221 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:07.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:07.227 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:45:07.328 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:07.356 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:07.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:07.359 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:45:07.423 INFO:teuthology.orchestra.run.smithi078.stderr:reweighted osd.5 to 1 (10000) 2023-10-01T10:45:07.460 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:07.504 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:07.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:07.507 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:45:07.608 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:07.615 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:07 smithi078 bash[75299]: cluster 2023-10-01T10:45:05.809030+0000 mgr.y (mgr.25072) 821 : cluster 0 pgmap v740: 89 pgs: 89 active+clean; 474 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 664 op/s 2023-10-01T10:45:07.616 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:07 smithi078 bash[75299]: audit 2023-10-01T10:45:06.683918+0000 mon.c (mon.2) 145 : audit 1 from='client.? 172.21.15.78:0/2710218989' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-01T10:45:07.616 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:07 smithi078 bash[75299]: audit 2023-10-01T10:45:06.684525+0000 mon.a (mon.0) 1037 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-01T10:45:07.616 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:07 smithi078 bash[78274]: cluster 2023-10-01T10:45:05.809030+0000 mgr.y (mgr.25072) 821 : cluster 0 pgmap v740: 89 pgs: 89 active+clean; 474 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 664 op/s 2023-10-01T10:45:07.616 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:07 smithi078 bash[78274]: audit 2023-10-01T10:45:06.683918+0000 mon.c (mon.2) 145 : audit 1 from='client.? 172.21.15.78:0/2710218989' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-01T10:45:07.616 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:07 smithi078 bash[78274]: audit 2023-10-01T10:45:06.684525+0000 mon.a (mon.0) 1037 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-01T10:45:07.627 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:07.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:07.635 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:45:07.736 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:07.746 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:07 smithi173 bash[58816]: cluster 2023-10-01T10:45:05.809030+0000 mgr.y (mgr.25072) 821 : cluster 0 pgmap v740: 89 pgs: 89 active+clean; 474 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 664 op/s 2023-10-01T10:45:07.747 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:07 smithi173 bash[58816]: audit 2023-10-01T10:45:06.683918+0000 mon.c (mon.2) 145 : audit 1 from='client.? 172.21.15.78:0/2710218989' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-01T10:45:07.747 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:07 smithi173 bash[58816]: audit 2023-10-01T10:45:06.684525+0000 mon.a (mon.0) 1037 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-01T10:45:07.765 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:07.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:07.770 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:45:07.871 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:07.897 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:07.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:07.903 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:08.004 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:08.039 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:08.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:08.043 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:08.143 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:08.180 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:08.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:08.185 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:45:08.286 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:08.317 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:08.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:08.323 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:08.424 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:08.451 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:08.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:08.456 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:45:08.557 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:08.565 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:08 smithi173 bash[58816]: cluster 2023-10-01T10:45:07.407764+0000 osd.5 (osd.5) 74 : cluster 3 failed to encode map e145 with expected crc 2023-10-01T10:45:08.565 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:08 smithi173 bash[58816]: audit 2023-10-01T10:45:07.409238+0000 mon.a (mon.0) 1038 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-01T10:45:08.565 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:08 smithi173 bash[58816]: cluster 2023-10-01T10:45:07.409369+0000 mon.a (mon.0) 1039 : cluster 0 osdmap e145: 8 total, 8 up, 8 in 2023-10-01T10:45:08.565 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:08 smithi173 bash[58816]: cluster 2023-10-01T10:45:07.424670+0000 osd.7 (osd.7) 92 : cluster 3 failed to encode map e145 with expected crc 2023-10-01T10:45:08.565 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:08 smithi173 bash[58816]: cluster 2023-10-01T10:45:07.426621+0000 osd.7 (osd.7) 93 : cluster 3 failed to encode map e145 with expected crc 2023-10-01T10:45:08.585 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:08.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:08.588 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:45:08.689 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:08.724 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:08.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:08.729 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:08.830 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:08.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:08 smithi078 bash[78274]: cluster 2023-10-01T10:45:07.407764+0000 osd.5 (osd.5) 74 : cluster 3 failed to encode map e145 with expected crc 2023-10-01T10:45:08.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:08 smithi078 bash[78274]: audit 2023-10-01T10:45:07.409238+0000 mon.a (mon.0) 1038 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-01T10:45:08.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:08 smithi078 bash[78274]: cluster 2023-10-01T10:45:07.409369+0000 mon.a (mon.0) 1039 : cluster 0 osdmap e145: 8 total, 8 up, 8 in 2023-10-01T10:45:08.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:08 smithi078 bash[78274]: cluster 2023-10-01T10:45:07.424670+0000 osd.7 (osd.7) 92 : cluster 3 failed to encode map e145 with expected crc 2023-10-01T10:45:08.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:08 smithi078 bash[78274]: cluster 2023-10-01T10:45:07.426621+0000 osd.7 (osd.7) 93 : cluster 3 failed to encode map e145 with expected crc 2023-10-01T10:45:08.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:08 smithi078 bash[75299]: cluster 2023-10-01T10:45:07.407764+0000 osd.5 (osd.5) 74 : cluster 3 failed to encode map e145 with expected crc 2023-10-01T10:45:08.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:08 smithi078 bash[75299]: audit 2023-10-01T10:45:07.409238+0000 mon.a (mon.0) 1038 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-01T10:45:08.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:08 smithi078 bash[75299]: cluster 2023-10-01T10:45:07.409369+0000 mon.a (mon.0) 1039 : cluster 0 osdmap e145: 8 total, 8 up, 8 in 2023-10-01T10:45:08.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:08 smithi078 bash[75299]: cluster 2023-10-01T10:45:07.424670+0000 osd.7 (osd.7) 92 : cluster 3 failed to encode map e145 with expected crc 2023-10-01T10:45:08.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:08 smithi078 bash[75299]: cluster 2023-10-01T10:45:07.426621+0000 osd.7 (osd.7) 93 : cluster 3 failed to encode map e145 with expected crc 2023-10-01T10:45:08.865 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:08.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:08.875 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:45:08.976 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:09.011 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:09.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:09.013 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:09.114 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:09.145 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:09.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:09.148 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:45:09.248 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:09.274 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:09.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:09.280 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:45:09.381 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:09.408 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:09.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:09.413 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:45:09.423 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:09 smithi173 bash[58816]: cluster 2023-10-01T10:45:07.809599+0000 mgr.y (mgr.25072) 822 : cluster 0 pgmap v742: 89 pgs: 89 active+clean; 394 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2023-10-01T10:45:09.514 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:09.536 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:09.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:09.541 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:45:09.642 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:09.681 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:09.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:09.687 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:09.787 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:09.816 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:09.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:09.820 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:45:09.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:09 smithi078 bash[75299]: cluster 2023-10-01T10:45:07.809599+0000 mgr.y (mgr.25072) 822 : cluster 0 pgmap v742: 89 pgs: 89 active+clean; 394 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2023-10-01T10:45:09.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:09 smithi078 bash[78274]: cluster 2023-10-01T10:45:07.809599+0000 mgr.y (mgr.25072) 822 : cluster 0 pgmap v742: 89 pgs: 89 active+clean; 394 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2023-10-01T10:45:09.920 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:09.937 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-01T10:45:09.953 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:09.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:09.961 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:45:10.062 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:10.095 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:10.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:10.097 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:10.198 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:10.232 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:10.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:10.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:45:10.336 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:10.368 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:10.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:10.371 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:45:10.471 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:10.496 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:10.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:10.500 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:45:10.531 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:45:10 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:45:10] "GET /metrics HTTP/1.1" 200 36237 "" "Prometheus/2.33.4" 2023-10-01T10:45:10.601 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:10.624 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:10.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:10.630 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:45:10.730 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:10.764 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:10.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:10.771 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:45:10.872 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:10.893 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:10.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:10.895 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:10.996 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:11.030 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:11.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:11.033 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:45:11.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:11 smithi078 bash[78274]: cluster 2023-10-01T10:45:09.810650+0000 mgr.y (mgr.25072) 823 : cluster 0 pgmap v743: 89 pgs: 89 active+clean; 255 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2023-10-01T10:45:11.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:11 smithi078 bash[78274]: audit 2023-10-01T10:45:10.035465+0000 mon.a (mon.0) 1040 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:45:11.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:11 smithi078 bash[78274]: audit 2023-10-01T10:45:10.050035+0000 mon.a (mon.0) 1041 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:45:11.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:11 smithi078 bash[75299]: cluster 2023-10-01T10:45:09.810650+0000 mgr.y (mgr.25072) 823 : cluster 0 pgmap v743: 89 pgs: 89 active+clean; 255 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2023-10-01T10:45:11.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:11 smithi078 bash[75299]: audit 2023-10-01T10:45:10.035465+0000 mon.a (mon.0) 1040 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:45:11.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:11 smithi078 bash[75299]: audit 2023-10-01T10:45:10.050035+0000 mon.a (mon.0) 1041 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:45:11.135 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:11.174 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:11.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:11.179 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:45:11.211 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-10-01T10:45:11.211 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T10:45:11.244 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:11 smithi173 bash[58816]: cluster 2023-10-01T10:45:09.810650+0000 mgr.y (mgr.25072) 823 : cluster 0 pgmap v743: 89 pgs: 89 active+clean; 255 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2023-10-01T10:45:11.244 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:11 smithi173 bash[58816]: audit 2023-10-01T10:45:10.035465+0000 mon.a (mon.0) 1040 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:45:11.244 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:11 smithi173 bash[58816]: audit 2023-10-01T10:45:10.050035+0000 mon.a (mon.0) 1041 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:45:11.259 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T10:45:11.280 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:11.303 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T10:45:11.305 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:11.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:11.313 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:45:11.339 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T10:45:11.385 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T10:45:11.414 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:11.429 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T10:45:11.446 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:11.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:11.453 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:45:11.472 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T10:45:11.510 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T10:45:11.545 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T10:45:11.549 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T10:45:11.550 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T10:45:11.550 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T10:45:11.550 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T10:45:11.550 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T10:45:11.550 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T10:45:11.550 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T10:45:11.550 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T10:45:11.554 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:11.581 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:11.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:11.583 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:11.683 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:11.718 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:11.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:11.722 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:45:11.822 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:11.846 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:11.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:11.851 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:45:11.952 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:11.981 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:11.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:11.984 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:45:12.084 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:12.116 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:12.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:12.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:45:12.131 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-01T10:45:12.224 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:12.233 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:12 smithi078 bash[78274]: audit 2023-10-01T10:45:11.208165+0000 mon.c (mon.2) 146 : audit 0 from='client.? 172.21.15.78:0/84604446' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:45:12.234 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:12 smithi078 bash[75299]: audit 2023-10-01T10:45:11.208165+0000 mon.c (mon.2) 146 : audit 0 from='client.? 172.21.15.78:0/84604446' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:45:12.271 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:12.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:12.273 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:45:12.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:12 smithi173 bash[58816]: audit 2023-10-01T10:45:11.208165+0000 mon.c (mon.2) 146 : audit 0 from='client.? 172.21.15.78:0/84604446' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:45:12.374 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:12.406 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:12.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:12.411 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:45:12.512 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:12.547 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:12.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:12.555 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:45:12.655 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:12.700 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:12.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:12.706 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:12.807 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:12.834 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:12.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:12.837 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:12.938 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:12.971 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:12.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:12.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:45:13.074 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:13.104 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:13.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:13.111 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:45:13.212 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:13.242 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:13.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:13.244 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:45:13.345 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:13.380 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:13.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:13.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:45:13.488 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:13.514 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:13.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:13.522 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:13.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:13 smithi173 bash[58816]: cluster 2023-10-01T10:45:11.811396+0000 mgr.y (mgr.25072) 824 : cluster 0 pgmap v744: 89 pgs: 89 active+clean; 174 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2023-10-01T10:45:13.623 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:13.649 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:13.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:13.656 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:45:13.757 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:13.767 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:13 smithi078 bash[75299]: cluster 2023-10-01T10:45:11.811396+0000 mgr.y (mgr.25072) 824 : cluster 0 pgmap v744: 89 pgs: 89 active+clean; 174 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2023-10-01T10:45:13.767 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:13 smithi078 bash[78274]: cluster 2023-10-01T10:45:11.811396+0000 mgr.y (mgr.25072) 824 : cluster 0 pgmap v744: 89 pgs: 89 active+clean; 174 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2023-10-01T10:45:13.793 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:13.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:13.797 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:45:13.898 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:13.937 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:13.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:13.942 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:45:13.959 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Removed 111812 objects 2023-10-01T10:45:13.959 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Clean up completed and total clean up time :155.725 2023-10-01T10:45:14.043 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:14.073 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:14.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:14.076 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:14.177 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:14.211 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:14.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:14.214 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:45:14.315 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:14.349 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:14.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:14.352 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:14.402 INFO:teuthology.orchestra.run.smithi078.stderr:nodeep-scrub is unset 2023-10-01T10:45:14.453 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:14.479 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:14.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:14.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:45:14.583 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:14.597 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:14 smithi173 bash[58816]: audit 2023-10-01T10:45:14.023535+0000 mon.c (mon.2) 147 : audit 1 from='client.? 172.21.15.78:0/646994864' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:45:14.597 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:14 smithi173 bash[58816]: audit 2023-10-01T10:45:14.023898+0000 mon.a (mon.0) 1042 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:45:14.615 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:14.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:14.618 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:14.719 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:14.751 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:14.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:14.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:14.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:14 smithi078 bash[75299]: audit 2023-10-01T10:45:14.023535+0000 mon.c (mon.2) 147 : audit 1 from='client.? 172.21.15.78:0/646994864' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:45:14.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:14 smithi078 bash[75299]: audit 2023-10-01T10:45:14.023898+0000 mon.a (mon.0) 1042 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:45:14.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:14 smithi078 bash[78274]: audit 2023-10-01T10:45:14.023535+0000 mon.c (mon.2) 147 : audit 1 from='client.? 172.21.15.78:0/646994864' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:45:14.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:14 smithi078 bash[78274]: audit 2023-10-01T10:45:14.023898+0000 mon.a (mon.0) 1042 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:45:14.856 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:14.887 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:14.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:14.890 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:14.991 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:15.017 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:15.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:15.020 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:45:15.120 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:15.151 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:15.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:15.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:15.255 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:15.282 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:15.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:15.284 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:15.385 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:15.413 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:15.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:15.416 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:45:15.517 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:15.528 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T10:45:15.546 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:15.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:15.549 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:15.650 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:15.662 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:15 smithi173 bash[58816]: cluster 2023-10-01T10:45:13.811981+0000 mgr.y (mgr.25072) 825 : cluster 0 pgmap v745: 89 pgs: 89 active+clean; 98 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 686 op/s 2023-10-01T10:45:15.662 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:15 smithi173 bash[58816]: cluster 2023-10-01T10:45:14.381530+0000 mon.a (mon.0) 1043 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T10:45:15.662 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:15 smithi173 bash[58816]: cluster 2023-10-01T10:45:14.381592+0000 mon.a (mon.0) 1044 : cluster 1 Cluster is now healthy 2023-10-01T10:45:15.663 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:15 smithi173 bash[58816]: audit 2023-10-01T10:45:14.396496+0000 mon.a (mon.0) 1045 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T10:45:15.663 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:15 smithi173 bash[58816]: cluster 2023-10-01T10:45:14.396620+0000 mon.a (mon.0) 1046 : cluster 0 osdmap e146: 8 total, 8 up, 8 in 2023-10-01T10:45:15.663 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:15 smithi173 bash[58816]: cluster 2023-10-01T10:45:14.749979+0000 osd.7 (osd.7) 94 : cluster 3 failed to encode map e146 with expected crc 2023-10-01T10:45:15.663 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:15 smithi173 bash[58816]: cluster 2023-10-01T10:45:14.751461+0000 osd.7 (osd.7) 95 : cluster 3 failed to encode map e146 with expected crc 2023-10-01T10:45:15.681 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:15.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:15.683 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:15.784 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:15.797 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:15 smithi078 bash[78274]: cluster 2023-10-01T10:45:13.811981+0000 mgr.y (mgr.25072) 825 : cluster 0 pgmap v745: 89 pgs: 89 active+clean; 98 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 686 op/s 2023-10-01T10:45:15.797 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:15 smithi078 bash[78274]: cluster 2023-10-01T10:45:14.381530+0000 mon.a (mon.0) 1043 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T10:45:15.797 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:15 smithi078 bash[78274]: cluster 2023-10-01T10:45:14.381592+0000 mon.a (mon.0) 1044 : cluster 1 Cluster is now healthy 2023-10-01T10:45:15.797 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:15 smithi078 bash[78274]: audit 2023-10-01T10:45:14.396496+0000 mon.a (mon.0) 1045 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T10:45:15.797 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:15 smithi078 bash[78274]: cluster 2023-10-01T10:45:14.396620+0000 mon.a (mon.0) 1046 : cluster 0 osdmap e146: 8 total, 8 up, 8 in 2023-10-01T10:45:15.797 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:15 smithi078 bash[78274]: cluster 2023-10-01T10:45:14.749979+0000 osd.7 (osd.7) 94 : cluster 3 failed to encode map e146 with expected crc 2023-10-01T10:45:15.797 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:15 smithi078 bash[78274]: cluster 2023-10-01T10:45:14.751461+0000 osd.7 (osd.7) 95 : cluster 3 failed to encode map e146 with expected crc 2023-10-01T10:45:15.798 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:15 smithi078 bash[75299]: cluster 2023-10-01T10:45:13.811981+0000 mgr.y (mgr.25072) 825 : cluster 0 pgmap v745: 89 pgs: 89 active+clean; 98 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 686 op/s 2023-10-01T10:45:15.798 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:15 smithi078 bash[75299]: cluster 2023-10-01T10:45:14.381530+0000 mon.a (mon.0) 1043 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T10:45:15.798 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:15 smithi078 bash[75299]: cluster 2023-10-01T10:45:14.381592+0000 mon.a (mon.0) 1044 : cluster 1 Cluster is now healthy 2023-10-01T10:45:15.798 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:15 smithi078 bash[75299]: audit 2023-10-01T10:45:14.396496+0000 mon.a (mon.0) 1045 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T10:45:15.798 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:15 smithi078 bash[75299]: cluster 2023-10-01T10:45:14.396620+0000 mon.a (mon.0) 1046 : cluster 0 osdmap e146: 8 total, 8 up, 8 in 2023-10-01T10:45:15.798 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:15 smithi078 bash[75299]: cluster 2023-10-01T10:45:14.749979+0000 osd.7 (osd.7) 94 : cluster 3 failed to encode map e146 with expected crc 2023-10-01T10:45:15.798 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:15 smithi078 bash[75299]: cluster 2023-10-01T10:45:14.751461+0000 osd.7 (osd.7) 95 : cluster 3 failed to encode map e146 with expected crc 2023-10-01T10:45:15.812 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:15.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:15.814 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:45:15.915 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:15.946 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:15.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:15.949 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:45:16.050 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:16.079 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:16.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:16.082 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:45:16.182 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:16.214 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:16.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:16.216 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:45:16.317 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:16.348 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:16.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:16.352 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:45:16.452 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:16.464 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:16 smithi173 bash[58816]: audit 2023-10-01T10:45:15.405580+0000 mon.a (mon.0) 1047 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:45:16.464 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:16 smithi173 bash[58816]: audit 2023-10-01T10:45:15.412480+0000 mon.a (mon.0) 1048 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:45:16.465 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:16 smithi173 bash[58816]: audit 2023-10-01T10:45:15.880336+0000 mon.a (mon.0) 1049 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:45:16.465 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:16 smithi173 bash[58816]: audit 2023-10-01T10:45:15.881923+0000 mon.a (mon.0) 1050 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:45:16.465 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:16 smithi173 bash[58816]: audit 2023-10-01T10:45:15.890643+0000 mon.a (mon.0) 1051 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:45:16.488 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:16.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:16.492 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:45:16.592 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:16.626 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:16.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:16.629 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:45:16.730 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:16.762 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:16.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:16.766 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:45:16.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:16 smithi078 bash[75299]: audit 2023-10-01T10:45:15.405580+0000 mon.a (mon.0) 1047 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:45:16.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:16 smithi078 bash[75299]: audit 2023-10-01T10:45:15.412480+0000 mon.a (mon.0) 1048 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:45:16.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:16 smithi078 bash[75299]: audit 2023-10-01T10:45:15.880336+0000 mon.a (mon.0) 1049 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:45:16.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:16 smithi078 bash[75299]: audit 2023-10-01T10:45:15.881923+0000 mon.a (mon.0) 1050 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:45:16.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:16 smithi078 bash[75299]: audit 2023-10-01T10:45:15.890643+0000 mon.a (mon.0) 1051 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:45:16.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:16 smithi078 bash[78274]: audit 2023-10-01T10:45:15.405580+0000 mon.a (mon.0) 1047 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:45:16.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:16 smithi078 bash[78274]: audit 2023-10-01T10:45:15.412480+0000 mon.a (mon.0) 1048 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:45:16.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:16 smithi078 bash[78274]: audit 2023-10-01T10:45:15.880336+0000 mon.a (mon.0) 1049 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:45:16.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:16 smithi078 bash[78274]: audit 2023-10-01T10:45:15.881923+0000 mon.a (mon.0) 1050 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:45:16.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:16 smithi078 bash[78274]: audit 2023-10-01T10:45:15.890643+0000 mon.a (mon.0) 1051 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:45:16.866 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:16.899 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:16.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:16.902 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:45:16.921 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:45:17.001 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T10:45:17.002 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:17.034 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:17.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:17.036 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:45:17.137 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:17.159 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:17.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:17.162 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:45:17.263 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:17.289 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:17.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:17.291 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:45:17.392 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:17.419 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:17.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:17.422 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:17.424 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T10:45:17.523 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:17.536 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:17 smithi173 bash[58816]: cluster 2023-10-01T10:45:15.812411+0000 mgr.y (mgr.25072) 826 : cluster 0 pgmap v747: 89 pgs: 89 active+clean; 49 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 656 op/s 2023-10-01T10:45:17.556 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:17.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:17.559 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:45:17.659 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:17.693 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:17.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:17.695 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:45:17.796 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:17.829 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:17.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:17.832 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:17.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:17 smithi078 bash[75299]: cluster 2023-10-01T10:45:15.812411+0000 mgr.y (mgr.25072) 826 : cluster 0 pgmap v747: 89 pgs: 89 active+clean; 49 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 656 op/s 2023-10-01T10:45:17.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:17 smithi078 bash[78274]: cluster 2023-10-01T10:45:15.812411+0000 mgr.y (mgr.25072) 826 : cluster 0 pgmap v747: 89 pgs: 89 active+clean; 49 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 656 op/s 2023-10-01T10:45:17.933 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:17.961 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:17.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:17.963 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:45:18.064 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:18.097 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:18.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:18.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:45:18.200 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:18.231 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:18.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:18.234 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:45:18.313 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_4 2023-10-01T10:45:18.313 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_4 unique_pool_4 --yes-i-really-really-mean-it 2023-10-01T10:45:18.335 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:18.385 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:18.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:18.388 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:45:18.488 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:18.499 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:18 smithi078 bash[78274]: audit 2023-10-01T10:45:17.424265+0000 mon.a (mon.0) 1052 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:45:18.499 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:18 smithi078 bash[78274]: audit 2023-10-01T10:45:17.426509+0000 mon.a (mon.0) 1053 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:45:18.499 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:18 smithi078 bash[75299]: audit 2023-10-01T10:45:17.424265+0000 mon.a (mon.0) 1052 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:45:18.499 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:18 smithi078 bash[75299]: audit 2023-10-01T10:45:17.426509+0000 mon.a (mon.0) 1053 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:45:18.514 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:18.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:18.517 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:45:18.618 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:18.643 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:18.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:18.647 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:45:18.747 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:18.759 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:18 smithi173 bash[58816]: audit 2023-10-01T10:45:17.424265+0000 mon.a (mon.0) 1052 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:45:18.759 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:18 smithi173 bash[58816]: audit 2023-10-01T10:45:17.426509+0000 mon.a (mon.0) 1053 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:45:18.775 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:18.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:18.777 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:18.878 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:18.903 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:18.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:18.906 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:45:19.007 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:19.033 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:19.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:19.035 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:19.136 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:19.165 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:19.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:19.168 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:45:19.269 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:19.300 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:19.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:19.302 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:19.403 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:19.429 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:19.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:19.432 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:19.533 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:19.540 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:19 smithi173 bash[58816]: cluster 2023-10-01T10:45:17.812886+0000 mgr.y (mgr.25072) 827 : cluster 0 pgmap v748: 89 pgs: 89 active+clean; 5.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 620 op/s 2023-10-01T10:45:19.558 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:19.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:19.560 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:19.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:19 smithi078 bash[75299]: cluster 2023-10-01T10:45:17.812886+0000 mgr.y (mgr.25072) 827 : cluster 0 pgmap v748: 89 pgs: 89 active+clean; 5.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 620 op/s 2023-10-01T10:45:19.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:19 smithi078 bash[78274]: cluster 2023-10-01T10:45:17.812886+0000 mgr.y (mgr.25072) 827 : cluster 0 pgmap v748: 89 pgs: 89 active+clean; 5.6 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 620 op/s 2023-10-01T10:45:19.661 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:19.689 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:19.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:19.692 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:45:19.793 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:19.821 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:19.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:19.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:19.924 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:19.955 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:19.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:19.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:45:19.969 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:45:20.058 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:20.083 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:20.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:20.086 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:45:20.187 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:20.211 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:20.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:20.213 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:20.314 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:20.340 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:20.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:20.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:20.445 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:20.476 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:20.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:20.479 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:20.580 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:20.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:45:20 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:45:20] "GET /metrics HTTP/1.1" 200 36236 "" "Prometheus/2.33.4" 2023-10-01T10:45:20.612 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:20.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:20.614 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:20.715 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:20.740 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:20.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:20.742 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:45:20.789 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-10-01T10:45:20.789 INFO:teuthology.orchestra.run.smithi173.stdout: "size": 20, 2023-10-01T10:45:20.789 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 600, 2023-10-01T10:45:20.789 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [ 2023-10-01T10:45:20.789 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.790 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.35649.0:125703 9.4 e135/128 9:21321cc7:::benchmark_data_smithi078_113121_object13888:head v 135'7824, mlcod=135'7821)", 2023-10-01T10:45:20.790 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:42:55.995809+0000", 2023-10-01T10:45:20.790 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 144.78929183400001, 2023-10-01T10:45:20.790 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.024788925999999999, 2023-10-01T10:45:20.790 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:45:20.790 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:45:20.790 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:45:20.790 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.790 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:45:20.790 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:55.995809+0000", 2023-10-01T10:45:20.791 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:45:20.791 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.791 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.791 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:45:20.791 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:55.995809+0000", 2023-10-01T10:45:20.791 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:45:20.791 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.791 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.791 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:45:20.791 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:55.995801+0000", 2023-10-01T10:45:20.791 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999919 2023-10-01T10:45:20.791 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.792 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.792 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:45:20.792 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:55.995830+0000", 2023-10-01T10:45:20.792 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.8935e-05 2023-10-01T10:45:20.792 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.792 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.792 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:45:20.792 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:55.995852+0000", 2023-10-01T10:45:20.792 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.2234000000000001e-05 2023-10-01T10:45:20.792 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.792 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.792 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:45:20.793 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:55.995864+0000", 2023-10-01T10:45:20.793 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1714e-05 2023-10-01T10:45:20.793 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.793 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.793 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:45:20.793 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:55.996712+0000", 2023-10-01T10:45:20.793 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00084793800000000001 2023-10-01T10:45:20.793 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.793 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.793 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:45:20.793 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:55.996759+0000", 2023-10-01T10:45:20.793 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.7383999999999999e-05 2023-10-01T10:45:20.794 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.794 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.794 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:45:20.794 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:56.020527+0000", 2023-10-01T10:45:20.794 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.023767903 2023-10-01T10:45:20.794 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.794 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.794 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:45:20.794 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:56.020598+0000", 2023-10-01T10:45:20.794 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.0794e-05 2023-10-01T10:45:20.794 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:20.794 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:45:20.794 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:20.795 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.795 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.795 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.35649.0:134171 9.4 e135/128 9:2b615339:::benchmark_data_smithi078_113121_object22356:head v 135'8395, mlcod=135'8392)", 2023-10-01T10:45:20.795 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:43:07.119713+0000", 2023-10-01T10:45:20.795 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 133.66538789500001, 2023-10-01T10:45:20.795 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.025575820999999999, 2023-10-01T10:45:20.795 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:45:20.795 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:45:20.795 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:45:20.795 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.795 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:45:20.796 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:07.119713+0000", 2023-10-01T10:45:20.796 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:45:20.796 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.796 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.796 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:45:20.796 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:07.119713+0000", 2023-10-01T10:45:20.796 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:45:20.796 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.796 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.796 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:45:20.796 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:07.119711+0000", 2023-10-01T10:45:20.796 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-10-01T10:45:20.796 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.797 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.797 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:45:20.797 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:07.119722+0000", 2023-10-01T10:45:20.797 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0293e-05 2023-10-01T10:45:20.797 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.797 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.797 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:45:20.797 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:07.119738+0000", 2023-10-01T10:45:20.797 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.6756999999999999e-05 2023-10-01T10:45:20.797 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.797 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.797 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:45:20.798 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:07.119750+0000", 2023-10-01T10:45:20.798 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1229e-05 2023-10-01T10:45:20.798 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.798 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.798 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:45:20.798 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:07.119846+0000", 2023-10-01T10:45:20.798 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.6199999999999994e-05 2023-10-01T10:45:20.798 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.798 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.798 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:45:20.798 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:07.119875+0000", 2023-10-01T10:45:20.798 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.9354000000000001e-05 2023-10-01T10:45:20.799 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.799 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.799 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:45:20.799 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:07.145237+0000", 2023-10-01T10:45:20.799 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.025362246000000001 2023-10-01T10:45:20.799 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.799 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.799 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:45:20.799 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:07.145289+0000", 2023-10-01T10:45:20.799 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.1208999999999997e-05 2023-10-01T10:45:20.799 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:20.800 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:45:20.800 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:20.800 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.800 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.800 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.35649.0:134583 9.c e135/128 9:3415305d:::benchmark_data_smithi078_113121_object22768:head v 135'8428, mlcod=135'8425)", 2023-10-01T10:45:20.800 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:43:07.645374+0000", 2023-10-01T10:45:20.800 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 133.139726892, 2023-10-01T10:45:20.800 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.026436998, 2023-10-01T10:45:20.800 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:45:20.800 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:45:20.800 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:45:20.800 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.800 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:45:20.801 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:07.645374+0000", 2023-10-01T10:45:20.801 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:45:20.801 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.801 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.801 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:45:20.801 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:07.645374+0000", 2023-10-01T10:45:20.801 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:45:20.801 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.801 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.801 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:45:20.801 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:07.645373+0000", 2023-10-01T10:45:20.801 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999995 2023-10-01T10:45:20.802 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.802 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.802 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:45:20.802 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:07.645379+0000", 2023-10-01T10:45:20.802 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.2129999999999996e-06 2023-10-01T10:45:20.802 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.802 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.802 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:45:20.802 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:07.645387+0000", 2023-10-01T10:45:20.802 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.9869999999999999e-06 2023-10-01T10:45:20.802 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.802 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.803 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:45:20.803 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:07.645394+0000", 2023-10-01T10:45:20.803 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.178e-06 2023-10-01T10:45:20.803 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.803 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.803 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:45:20.803 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:07.653066+0000", 2023-10-01T10:45:20.803 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0076721289999999998 2023-10-01T10:45:20.803 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.803 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.803 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:45:20.803 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:07.653095+0000", 2023-10-01T10:45:20.803 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.8843000000000002e-05 2023-10-01T10:45:20.804 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.804 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.804 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:45:20.804 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:07.671720+0000", 2023-10-01T10:45:20.804 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.018625584000000001 2023-10-01T10:45:20.804 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.804 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.804 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:45:20.804 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:07.671811+0000", 2023-10-01T10:45:20.804 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.0692000000000005e-05 2023-10-01T10:45:20.804 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:20.805 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:45:20.805 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:20.805 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.805 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.805 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.35649.0:134595 9.4 e135/128 9:2028ba16:::benchmark_data_smithi078_113121_object22780:head v 135'8429, mlcod=135'8427)", 2023-10-01T10:45:20.805 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:43:07.649374+0000", 2023-10-01T10:45:20.805 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 133.135726586, 2023-10-01T10:45:20.805 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.027201581999999998, 2023-10-01T10:45:20.805 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:45:20.805 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:45:20.805 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:45:20.805 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.806 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:45:20.806 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:07.649374+0000", 2023-10-01T10:45:20.806 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:45:20.806 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.806 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.806 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:45:20.806 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:07.649374+0000", 2023-10-01T10:45:20.806 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:45:20.806 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.806 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.806 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:45:20.806 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:07.649372+0000", 2023-10-01T10:45:20.807 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-10-01T10:45:20.807 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.807 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.807 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:45:20.807 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:07.649389+0000", 2023-10-01T10:45:20.807 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.6566999999999999e-05 2023-10-01T10:45:20.807 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.807 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.807 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:45:20.807 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:07.649416+0000", 2023-10-01T10:45:20.807 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.658e-05 2023-10-01T10:45:20.808 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.808 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.808 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:45:20.808 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:07.649428+0000", 2023-10-01T10:45:20.808 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.221e-05 2023-10-01T10:45:20.808 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.808 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.808 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:45:20.808 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:07.650207+0000", 2023-10-01T10:45:20.808 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000778961 2023-10-01T10:45:20.808 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.809 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.809 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:45:20.809 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:07.650230+0000", 2023-10-01T10:45:20.809 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.3691000000000001e-05 2023-10-01T10:45:20.809 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.809 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.809 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:45:20.809 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:07.676516+0000", 2023-10-01T10:45:20.809 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.026285408 2023-10-01T10:45:20.809 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.809 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.810 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:45:20.810 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:07.676576+0000", 2023-10-01T10:45:20.810 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.9885e-05 2023-10-01T10:45:20.810 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:20.810 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:45:20.810 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:20.810 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.810 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.810 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.35649.0:134584 9.c e135/128 9:34cbecf3:::benchmark_data_smithi078_113121_object22769:head v 135'8429, mlcod=135'8425)", 2023-10-01T10:45:20.810 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:43:07.652650+0000", 2023-10-01T10:45:20.810 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 133.132450608, 2023-10-01T10:45:20.810 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.023961521999999999, 2023-10-01T10:45:20.811 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:45:20.811 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:45:20.811 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:45:20.811 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.811 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:45:20.811 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:07.652650+0000", 2023-10-01T10:45:20.811 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:45:20.811 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.811 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.811 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:45:20.811 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:07.652650+0000", 2023-10-01T10:45:20.811 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:45:20.812 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.812 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.812 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:45:20.812 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:07.652649+0000", 2023-10-01T10:45:20.812 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-10-01T10:45:20.812 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.812 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.812 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:45:20.812 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:07.652663+0000", 2023-10-01T10:45:20.812 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4253999999999999e-05 2023-10-01T10:45:20.812 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.812 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.813 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:45:20.813 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:07.652691+0000", 2023-10-01T10:45:20.813 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.8600000000000001e-05 2023-10-01T10:45:20.813 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.813 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.813 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:45:20.813 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:07.652704+0000", 2023-10-01T10:45:20.813 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2454e-05 2023-10-01T10:45:20.813 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.813 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.813 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:45:20.813 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:07.656023+0000", 2023-10-01T10:45:20.814 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0033187099999999999 2023-10-01T10:45:20.814 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.814 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.814 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:45:20.814 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:07.656048+0000", 2023-10-01T10:45:20.814 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.5712999999999999e-05 2023-10-01T10:45:20.814 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.814 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.814 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:45:20.814 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:07.676579+0000", 2023-10-01T10:45:20.814 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.020531159 2023-10-01T10:45:20.814 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.815 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.815 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:45:20.815 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:07.676612+0000", 2023-10-01T10:45:20.815 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.2240000000000003e-05 2023-10-01T10:45:20.815 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:20.815 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:45:20.815 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:20.815 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.815 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.815 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.35649.0:158149 9.c e138/128 9:3e9f63a3:::benchmark_data_smithi078_113121_object46334:head v 138'9841, mlcod=138'9839)", 2023-10-01T10:45:20.815 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:43:39.710975+0000", 2023-10-01T10:45:20.815 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 101.074126108, 2023-10-01T10:45:20.816 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.031357723999999997, 2023-10-01T10:45:20.816 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:45:20.816 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:45:20.816 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:45:20.816 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.816 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:45:20.816 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.710975+0000", 2023-10-01T10:45:20.816 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:45:20.816 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.816 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.816 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:45:20.816 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.710975+0000", 2023-10-01T10:45:20.817 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:45:20.817 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.817 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.817 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:45:20.817 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.710972+0000", 2023-10-01T10:45:20.817 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999976 2023-10-01T10:45:20.817 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.817 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.817 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:45:20.817 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.710993+0000", 2023-10-01T10:45:20.817 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.1211000000000001e-05 2023-10-01T10:45:20.817 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.817 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.818 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:45:20.818 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.711024+0000", 2023-10-01T10:45:20.818 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.0791999999999999e-05 2023-10-01T10:45:20.818 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.818 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.818 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:45:20.818 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.711040+0000", 2023-10-01T10:45:20.818 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.5783000000000001e-05 2023-10-01T10:45:20.818 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.818 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.818 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:45:20.818 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.711114+0000", 2023-10-01T10:45:20.819 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.4554999999999993e-05 2023-10-01T10:45:20.819 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.819 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.819 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:45:20.819 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.711153+0000", 2023-10-01T10:45:20.819 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.8473000000000002e-05 2023-10-01T10:45:20.819 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.819 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.819 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:45:20.819 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.742221+0000", 2023-10-01T10:45:20.819 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.031067689999999999 2023-10-01T10:45:20.819 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.820 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.820 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:45:20.820 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.742332+0000", 2023-10-01T10:45:20.820 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000111736 2023-10-01T10:45:20.820 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:20.820 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:45:20.820 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:20.820 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.820 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.820 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.35649.0:158152 9.c e138/128 9:3da3e31a:::benchmark_data_smithi078_113121_object46337:head v 138'9842, mlcod=138'9839)", 2023-10-01T10:45:20.820 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:43:39.718535+0000", 2023-10-01T10:45:20.820 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 101.066565379, 2023-10-01T10:45:20.821 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.024046822999999998, 2023-10-01T10:45:20.821 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:45:20.821 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:45:20.821 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:45:20.821 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.821 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:45:20.821 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.718535+0000", 2023-10-01T10:45:20.821 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:45:20.821 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.821 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.821 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:45:20.821 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.718535+0000", 2023-10-01T10:45:20.822 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:45:20.822 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.822 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.822 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:45:20.822 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.718533+0000", 2023-10-01T10:45:20.822 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999976 2023-10-01T10:45:20.822 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.822 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.822 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:45:20.822 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.718554+0000", 2023-10-01T10:45:20.822 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.0958000000000002e-05 2023-10-01T10:45:20.823 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.823 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.823 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:45:20.823 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.718589+0000", 2023-10-01T10:45:20.823 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.5139999999999999e-05 2023-10-01T10:45:20.823 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.823 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.823 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:45:20.823 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.718620+0000", 2023-10-01T10:45:20.823 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.1461999999999998e-05 2023-10-01T10:45:20.823 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.823 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.823 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:45:20.824 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.722956+0000", 2023-10-01T10:45:20.824 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0043353860000000001 2023-10-01T10:45:20.824 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.824 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.824 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:45:20.824 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.722977+0000", 2023-10-01T10:45:20.824 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.0716e-05 2023-10-01T10:45:20.824 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.824 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.824 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:45:20.824 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.742489+0000", 2023-10-01T10:45:20.824 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.019512403000000001 2023-10-01T10:45:20.825 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.825 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.825 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:45:20.825 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.742582+0000", 2023-10-01T10:45:20.825 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.3215999999999997e-05 2023-10-01T10:45:20.825 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:20.825 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:45:20.825 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:20.825 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.825 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.825 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.35649.0:160694 9.c e139/128 9:3772dfcf:::benchmark_data_smithi078_113121_object48879:head v 139'10002, mlcod=139'9998)", 2023-10-01T10:45:20.825 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:43:43.458368+0000", 2023-10-01T10:45:20.826 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 97.326733073, 2023-10-01T10:45:20.826 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.02828106, 2023-10-01T10:45:20.826 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:45:20.826 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:45:20.826 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:45:20.826 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.826 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:45:20.826 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:43.458368+0000", 2023-10-01T10:45:20.826 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:45:20.826 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.826 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.826 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:45:20.827 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:43.458368+0000", 2023-10-01T10:45:20.827 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:45:20.827 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.827 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.827 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:45:20.827 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:43.458366+0000", 2023-10-01T10:45:20.827 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-10-01T10:45:20.827 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.827 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.827 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:45:20.827 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:43.458376+0000", 2023-10-01T10:45:20.827 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.3370000000000006e-06 2023-10-01T10:45:20.828 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.828 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.828 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:45:20.828 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:43.458393+0000", 2023-10-01T10:45:20.828 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.7258999999999999e-05 2023-10-01T10:45:20.828 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.828 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.828 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:45:20.828 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:43.458402+0000", 2023-10-01T10:45:20.828 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.9360000000000005e-06 2023-10-01T10:45:20.828 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.829 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.829 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:45:20.829 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:43.461270+0000", 2023-10-01T10:45:20.829 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.002867957 2023-10-01T10:45:20.829 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.829 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.829 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:45:20.829 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:43.461339+0000", 2023-10-01T10:45:20.829 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.9051e-05 2023-10-01T10:45:20.829 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.829 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.829 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:45:20.830 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:43.486594+0000", 2023-10-01T10:45:20.830 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.025255225999999999 2023-10-01T10:45:20.830 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.830 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.830 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:45:20.830 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:43.486649+0000", 2023-10-01T10:45:20.830 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.4500999999999999e-05 2023-10-01T10:45:20.830 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:20.830 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:45:20.830 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:20.830 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.831 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.831 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.35649.0:160695 9.c e139/128 9:3714e29f:::benchmark_data_smithi078_113121_object48880:head v 139'10003, mlcod=139'9998)", 2023-10-01T10:45:20.831 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:43:43.462978+0000", 2023-10-01T10:45:20.831 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 97.322122841999999, 2023-10-01T10:45:20.831 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.023704921, 2023-10-01T10:45:20.831 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:45:20.831 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:45:20.831 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:45:20.831 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.831 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:45:20.832 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:43.462978+0000", 2023-10-01T10:45:20.832 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:45:20.832 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.832 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.832 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:45:20.832 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:43.462978+0000", 2023-10-01T10:45:20.832 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:45:20.832 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.832 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.832 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:45:20.832 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:43.462976+0000", 2023-10-01T10:45:20.832 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-10-01T10:45:20.833 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.833 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.833 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:45:20.833 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:43.462989+0000", 2023-10-01T10:45:20.833 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2863e-05 2023-10-01T10:45:20.833 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.833 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.833 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:45:20.833 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:43.463007+0000", 2023-10-01T10:45:20.833 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.8162999999999999e-05 2023-10-01T10:45:20.833 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.833 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.833 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:45:20.834 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:43.463018+0000", 2023-10-01T10:45:20.834 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0458e-05 2023-10-01T10:45:20.834 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.834 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.834 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:45:20.834 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:43.465717+0000", 2023-10-01T10:45:20.834 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0026994649999999999 2023-10-01T10:45:20.834 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.834 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.834 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:45:20.834 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:43.465784+0000", 2023-10-01T10:45:20.835 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.6791999999999993e-05 2023-10-01T10:45:20.835 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.835 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.835 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:45:20.835 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:43.486656+0000", 2023-10-01T10:45:20.835 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.020872315999999998 2023-10-01T10:45:20.835 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.835 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.835 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:45:20.835 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:43.486683+0000", 2023-10-01T10:45:20.835 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.6526999999999999e-05 2023-10-01T10:45:20.835 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:20.835 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:45:20.836 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:20.836 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.836 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.836 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.35649.0:169895 9.4 e139/128 9:21e60bfb:::benchmark_data_smithi078_113121_object58080:head v 139'10666, mlcod=139'10664)", 2023-10-01T10:45:20.836 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:43:56.701991+0000", 2023-10-01T10:45:20.836 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 84.083109335000003, 2023-10-01T10:45:20.836 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.035147035, 2023-10-01T10:45:20.836 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:45:20.836 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:45:20.836 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:45:20.836 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.836 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:45:20.837 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:56.701991+0000", 2023-10-01T10:45:20.837 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:45:20.837 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.837 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.837 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:45:20.837 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:56.701991+0000", 2023-10-01T10:45:20.837 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:45:20.837 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.837 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.837 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:45:20.837 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:56.701990+0000", 2023-10-01T10:45:20.837 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-10-01T10:45:20.838 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.838 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.838 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:45:20.838 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:56.702000+0000", 2023-10-01T10:45:20.838 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.5759999999999995e-06 2023-10-01T10:45:20.838 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.838 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.838 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:45:20.838 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:56.702020+0000", 2023-10-01T10:45:20.838 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.0024999999999999e-05 2023-10-01T10:45:20.838 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.838 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.839 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:45:20.839 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:56.702028+0000", 2023-10-01T10:45:20.839 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.5879999999999998e-06 2023-10-01T10:45:20.839 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.839 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.839 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:45:20.839 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:56.718330+0000", 2023-10-01T10:45:20.839 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.016301703000000001 2023-10-01T10:45:20.839 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.839 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.839 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:45:20.839 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:56.718364+0000", 2023-10-01T10:45:20.840 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.3686000000000002e-05 2023-10-01T10:45:20.840 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.840 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.840 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:45:20.840 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:56.729979+0000", 2023-10-01T10:45:20.840 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.011615476 2023-10-01T10:45:20.840 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.840 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.840 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:45:20.840 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:56.737138+0000", 2023-10-01T10:45:20.840 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0071593710000000003 2023-10-01T10:45:20.841 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:20.841 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:45:20.841 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:20.841 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.841 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.841 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.35649.0:200282 9.4 e142/128 9:269dc145:::benchmark_data_smithi078_113121_object88467:head v 142'12519, mlcod=142'12515)", 2023-10-01T10:45:20.841 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:44:40.361117+0000", 2023-10-01T10:45:20.841 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 40.423983307, 2023-10-01T10:45:20.841 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.024199664999999999, 2023-10-01T10:45:20.841 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:45:20.841 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:45:20.841 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:45:20.842 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.842 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:45:20.842 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:40.361117+0000", 2023-10-01T10:45:20.842 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:45:20.842 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.842 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.842 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:45:20.842 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:40.361117+0000", 2023-10-01T10:45:20.842 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:45:20.842 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.842 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.842 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:45:20.843 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:40.361115+0000", 2023-10-01T10:45:20.843 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999976 2023-10-01T10:45:20.843 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.843 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.843 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:45:20.843 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:40.361135+0000", 2023-10-01T10:45:20.843 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.9623000000000002e-05 2023-10-01T10:45:20.843 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.843 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.843 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:45:20.843 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:40.361169+0000", 2023-10-01T10:45:20.843 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.4242000000000002e-05 2023-10-01T10:45:20.844 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.844 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.844 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:45:20.844 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:40.361183+0000", 2023-10-01T10:45:20.844 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4447000000000001e-05 2023-10-01T10:45:20.844 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.844 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.844 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:45:20.844 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:40.367130+0000", 2023-10-01T10:45:20.844 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0059463470000000003 2023-10-01T10:45:20.844 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.844 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.845 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:45:20.845 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:40.367165+0000", 2023-10-01T10:45:20.845 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.5839e-05 2023-10-01T10:45:20.845 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.845 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.845 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:45:20.845 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:40.385058+0000", 2023-10-01T10:45:20.845 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.017892594000000001 2023-10-01T10:45:20.845 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.845 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.845 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:45:20.845 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:40.385317+0000", 2023-10-01T10:45:20.846 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00025911599999999999 2023-10-01T10:45:20.846 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:20.846 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:45:20.846 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:20.846 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.846 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.846 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.35649.0:200583 9.c e142/128 9:307d9a2f:::benchmark_data_smithi078_113121_object88768:head v 142'12563, mlcod=142'12559)", 2023-10-01T10:45:20.846 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:44:40.783768+0000", 2023-10-01T10:45:20.846 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 40.001332292999997, 2023-10-01T10:45:20.846 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.025421353000000001, 2023-10-01T10:45:20.846 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:45:20.846 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:45:20.847 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:45:20.847 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.847 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:45:20.847 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:40.783768+0000", 2023-10-01T10:45:20.847 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:45:20.847 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.847 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.847 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:45:20.847 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:40.783768+0000", 2023-10-01T10:45:20.847 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:45:20.847 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.847 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.848 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:45:20.848 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:40.783767+0000", 2023-10-01T10:45:20.848 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-10-01T10:45:20.848 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.848 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.848 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:45:20.848 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:40.783778+0000", 2023-10-01T10:45:20.848 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1131999999999999e-05 2023-10-01T10:45:20.848 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.848 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.848 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:45:20.848 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:40.783796+0000", 2023-10-01T10:45:20.849 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.7666999999999999e-05 2023-10-01T10:45:20.849 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.849 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.849 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:45:20.849 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:40.783808+0000", 2023-10-01T10:45:20.849 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1729000000000001e-05 2023-10-01T10:45:20.849 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.849 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.849 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:45:20.849 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:40.788250+0000", 2023-10-01T10:45:20.849 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0044426609999999997 2023-10-01T10:45:20.849 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.850 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.850 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:45:20.850 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:40.788288+0000", 2023-10-01T10:45:20.850 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.8180999999999999e-05 2023-10-01T10:45:20.850 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.850 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.850 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:45:20.850 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:40.809124+0000", 2023-10-01T10:45:20.850 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.020835078 2023-10-01T10:45:20.850 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.850 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.850 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:45:20.851 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:40.809190+0000", 2023-10-01T10:45:20.851 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.6235999999999994e-05 2023-10-01T10:45:20.851 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:20.851 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:45:20.851 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:20.851 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.851 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.851 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.35649.0:202469 9.c e142/128 9:3098a9c3:::benchmark_data_smithi078_113121_object90654:head v 142'12692, mlcod=142'12690)", 2023-10-01T10:45:20.851 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:44:43.482924+0000", 2023-10-01T10:45:20.851 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 37.302177194999999, 2023-10-01T10:45:20.851 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.028394910999999998, 2023-10-01T10:45:20.851 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:45:20.852 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:45:20.852 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:45:20.852 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.852 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:45:20.852 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:43.482924+0000", 2023-10-01T10:45:20.852 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:45:20.852 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.852 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.852 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:45:20.852 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:43.482924+0000", 2023-10-01T10:45:20.852 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:45:20.852 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.853 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.853 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:45:20.853 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:43.482922+0000", 2023-10-01T10:45:20.853 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-10-01T10:45:20.853 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.853 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.853 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:45:20.853 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:43.482935+0000", 2023-10-01T10:45:20.853 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2109999999999999e-05 2023-10-01T10:45:20.853 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.853 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.853 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:45:20.854 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:43.482952+0000", 2023-10-01T10:45:20.854 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.7753000000000001e-05 2023-10-01T10:45:20.854 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.854 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.854 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:45:20.854 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:43.482961+0000", 2023-10-01T10:45:20.854 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.0520000000000007e-06 2023-10-01T10:45:20.854 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.854 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.854 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:45:20.854 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:43.486691+0000", 2023-10-01T10:45:20.854 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0037294339999999998 2023-10-01T10:45:20.855 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.855 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.855 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:45:20.855 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:43.486724+0000", 2023-10-01T10:45:20.855 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.3067e-05 2023-10-01T10:45:20.855 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.855 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.855 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:45:20.855 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:43.511174+0000", 2023-10-01T10:45:20.855 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.024449800000000001 2023-10-01T10:45:20.855 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.855 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.856 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:45:20.856 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:43.511318+0000", 2023-10-01T10:45:20.856 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00014482300000000001 2023-10-01T10:45:20.856 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:20.856 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:45:20.856 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:20.856 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.856 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.856 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.35649.0:202470 9.c e142/128 9:30bd6011:::benchmark_data_smithi078_113121_object90655:head v 142'12693, mlcod=142'12690)", 2023-10-01T10:45:20.856 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:44:43.486960+0000", 2023-10-01T10:45:20.856 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 37.298140705000002, 2023-10-01T10:45:20.856 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.029010039000000001, 2023-10-01T10:45:20.857 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:45:20.857 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:45:20.857 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:45:20.857 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.857 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:45:20.857 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:43.486960+0000", 2023-10-01T10:45:20.857 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:45:20.857 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.857 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.857 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:45:20.857 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:43.486960+0000", 2023-10-01T10:45:20.857 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:45:20.858 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.858 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.858 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:45:20.858 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:43.486959+0000", 2023-10-01T10:45:20.858 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-10-01T10:45:20.858 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.858 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.858 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:45:20.858 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:43.486972+0000", 2023-10-01T10:45:20.858 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.3788000000000001e-05 2023-10-01T10:45:20.858 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.859 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.859 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:45:20.859 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:43.486994+0000", 2023-10-01T10:45:20.859 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.143e-05 2023-10-01T10:45:20.859 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.859 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.859 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:45:20.859 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:43.487005+0000", 2023-10-01T10:45:20.859 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1593999999999999e-05 2023-10-01T10:45:20.859 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.859 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.859 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:45:20.860 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:43.493667+0000", 2023-10-01T10:45:20.860 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0066615880000000004 2023-10-01T10:45:20.860 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.860 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.860 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:45:20.860 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:43.493704+0000", 2023-10-01T10:45:20.860 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.7202000000000001e-05 2023-10-01T10:45:20.860 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.860 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.860 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:45:20.860 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:43.515841+0000", 2023-10-01T10:45:20.860 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.022136800000000002 2023-10-01T10:45:20.860 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.861 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.861 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:45:20.861 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:43.515970+0000", 2023-10-01T10:45:20.861 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00012911800000000001 2023-10-01T10:45:20.861 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:20.861 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:45:20.861 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:20.861 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.861 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.861 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.35649.0:204372 9.e e142/128 9:785ab86f:::benchmark_data_smithi078_113121_object92557:head v 142'12924, mlcod=142'12921)", 2023-10-01T10:45:20.861 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:44:46.162337+0000", 2023-10-01T10:45:20.861 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 34.622763294000002, 2023-10-01T10:45:20.862 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.027926494999999999, 2023-10-01T10:45:20.862 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:45:20.862 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:45:20.862 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:45:20.862 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.862 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:45:20.862 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:46.162337+0000", 2023-10-01T10:45:20.862 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:45:20.862 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.862 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.862 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:45:20.862 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:46.162337+0000", 2023-10-01T10:45:20.863 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:45:20.863 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.863 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.863 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:45:20.863 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:46.162335+0000", 2023-10-01T10:45:20.863 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999976 2023-10-01T10:45:20.863 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.863 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.863 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:45:20.863 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:46.162348+0000", 2023-10-01T10:45:20.863 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2624e-05 2023-10-01T10:45:20.863 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.864 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.864 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:45:20.864 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:46.162370+0000", 2023-10-01T10:45:20.864 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.2532999999999999e-05 2023-10-01T10:45:20.864 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.864 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.864 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:45:20.864 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:46.162382+0000", 2023-10-01T10:45:20.864 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1491999999999999e-05 2023-10-01T10:45:20.864 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.864 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.864 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:45:20.865 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:46.162427+0000", 2023-10-01T10:45:20.865 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.5226999999999999e-05 2023-10-01T10:45:20.865 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.865 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.865 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:45:20.865 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:46.162454+0000", 2023-10-01T10:45:20.865 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.6832999999999999e-05 2023-10-01T10:45:20.865 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.865 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.865 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:45:20.865 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:46.190222+0000", 2023-10-01T10:45:20.865 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.027767869000000001 2023-10-01T10:45:20.866 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.866 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.866 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:45:20.866 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:46.190264+0000", 2023-10-01T10:45:20.866 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.2187000000000001e-05 2023-10-01T10:45:20.866 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:20.866 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:45:20.866 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:20.866 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.866 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.866 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.35649.0:204382 9.4 e142/128 9:2d8570ea:::benchmark_data_smithi078_113121_object92567:head v 142'12768, mlcod=142'12766)", 2023-10-01T10:45:20.866 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:44:46.170555+0000", 2023-10-01T10:45:20.867 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 34.614545808000003, 2023-10-01T10:45:20.867 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.026221266, 2023-10-01T10:45:20.867 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:45:20.867 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:45:20.867 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:45:20.867 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.867 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:45:20.867 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:46.170555+0000", 2023-10-01T10:45:20.867 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:45:20.867 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.867 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.867 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:45:20.868 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:46.170555+0000", 2023-10-01T10:45:20.868 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:45:20.868 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.868 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.868 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:45:20.868 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:46.170554+0000", 2023-10-01T10:45:20.868 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-10-01T10:45:20.868 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.868 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.868 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:45:20.868 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:46.170569+0000", 2023-10-01T10:45:20.868 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.5126000000000001e-05 2023-10-01T10:45:20.869 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.869 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.869 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:45:20.869 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:46.170586+0000", 2023-10-01T10:45:20.869 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.7261e-05 2023-10-01T10:45:20.869 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.869 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.869 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:45:20.869 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:46.170594+0000", 2023-10-01T10:45:20.869 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.1040000000000007e-06 2023-10-01T10:45:20.869 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.869 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.869 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:45:20.870 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:46.174302+0000", 2023-10-01T10:45:20.870 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.003707962 2023-10-01T10:45:20.870 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.870 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.870 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:45:20.870 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:46.174359+0000", 2023-10-01T10:45:20.870 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.6478999999999999e-05 2023-10-01T10:45:20.870 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.870 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.870 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:45:20.870 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:46.196696+0000", 2023-10-01T10:45:20.871 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.022336846 2023-10-01T10:45:20.871 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.871 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.871 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:45:20.871 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:46.196776+0000", 2023-10-01T10:45:20.871 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.0556000000000006e-05 2023-10-01T10:45:20.871 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:20.871 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:45:20.871 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:20.871 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.871 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.871 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.35649.0:205184 9.e e142/128 9:7713014b:::benchmark_data_smithi078_113121_object93369:head v 142'12986, mlcod=142'12983)", 2023-10-01T10:45:20.872 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:44:47.354405+0000", 2023-10-01T10:45:20.872 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 33.430696062000003, 2023-10-01T10:45:20.872 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.025872582000000002, 2023-10-01T10:45:20.872 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:45:20.872 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:45:20.872 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:45:20.872 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.872 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:45:20.872 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:47.354405+0000", 2023-10-01T10:45:20.872 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:45:20.872 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.872 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.872 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:45:20.873 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:47.354405+0000", 2023-10-01T10:45:20.873 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:45:20.873 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.873 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.873 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:45:20.873 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:47.354403+0000", 2023-10-01T10:45:20.873 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-10-01T10:45:20.873 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.873 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.873 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:45:20.873 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:47.354422+0000", 2023-10-01T10:45:20.873 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.9366999999999999e-05 2023-10-01T10:45:20.874 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.874 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.874 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:45:20.874 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:47.354468+0000", 2023-10-01T10:45:20.874 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.5754000000000003e-05 2023-10-01T10:45:20.874 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.874 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.874 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:45:20.874 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:47.354486+0000", 2023-10-01T10:45:20.874 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.7838000000000001e-05 2023-10-01T10:45:20.874 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.875 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.875 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:45:20.875 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:47.363061+0000", 2023-10-01T10:45:20.875 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0085753949999999995 2023-10-01T10:45:20.875 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.875 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.875 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:45:20.875 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:47.363088+0000", 2023-10-01T10:45:20.875 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.7305999999999999e-05 2023-10-01T10:45:20.875 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.875 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.875 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:45:20.875 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:47.380228+0000", 2023-10-01T10:45:20.876 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.017139135999999999 2023-10-01T10:45:20.876 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.876 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.876 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:45:20.876 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:47.380277+0000", 2023-10-01T10:45:20.876 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.9771999999999997e-05 2023-10-01T10:45:20.876 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:20.876 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:45:20.876 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:20.876 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.876 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.876 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.35649.0:208070 9.c e143/128 9:39d188c9:::benchmark_data_smithi078_113121_object96255:head v 143'13054, mlcod=143'13051)", 2023-10-01T10:45:20.877 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:44:51.515875+0000", 2023-10-01T10:45:20.877 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 29.269225665, 2023-10-01T10:45:20.877 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.025059846, 2023-10-01T10:45:20.877 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:45:20.877 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:45:20.877 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:45:20.877 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.877 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:45:20.877 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:51.515875+0000", 2023-10-01T10:45:20.877 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:45:20.877 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.877 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.878 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:45:20.878 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:51.515875+0000", 2023-10-01T10:45:20.878 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:45:20.878 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.878 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.878 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:45:20.878 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:51.515869+0000", 2023-10-01T10:45:20.878 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999933 2023-10-01T10:45:20.878 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.878 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.878 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:45:20.878 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:51.515892+0000", 2023-10-01T10:45:20.878 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.3263000000000001e-05 2023-10-01T10:45:20.879 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.879 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.879 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:45:20.879 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:51.515927+0000", 2023-10-01T10:45:20.879 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.5210999999999999e-05 2023-10-01T10:45:20.879 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.879 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.879 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:45:20.879 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:51.515942+0000", 2023-10-01T10:45:20.879 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.5455999999999999e-05 2023-10-01T10:45:20.879 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.879 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.880 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:45:20.880 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:51.516072+0000", 2023-10-01T10:45:20.880 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00012989300000000001 2023-10-01T10:45:20.880 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.880 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.880 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:45:20.880 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:51.516095+0000", 2023-10-01T10:45:20.880 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.2555e-05 2023-10-01T10:45:20.880 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.880 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.880 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:45:20.880 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:51.540868+0000", 2023-10-01T10:45:20.881 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.024772830999999999 2023-10-01T10:45:20.881 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.881 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.881 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:45:20.881 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:51.540935+0000", 2023-10-01T10:45:20.881 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.7166e-05 2023-10-01T10:45:20.881 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:20.881 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:45:20.881 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:20.881 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.881 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.881 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.35649.0:208074 9.c e143/128 9:36e3c68d:::benchmark_data_smithi078_113121_object96259:head v 143'13055, mlcod=143'13052)", 2023-10-01T10:45:20.882 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:44:51.515968+0000", 2023-10-01T10:45:20.882 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 29.269132430999999, 2023-10-01T10:45:20.882 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.024989376000000001, 2023-10-01T10:45:20.882 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:45:20.882 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:45:20.882 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:45:20.882 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.882 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:45:20.882 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:51.515968+0000", 2023-10-01T10:45:20.882 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:45:20.882 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.882 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.883 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:45:20.883 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:51.515968+0000", 2023-10-01T10:45:20.883 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:45:20.883 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.883 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.883 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:45:20.883 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:51.515968+0000", 2023-10-01T10:45:20.883 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999995 2023-10-01T10:45:20.883 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.883 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.883 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:45:20.884 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:51.515974+0000", 2023-10-01T10:45:20.884 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.161e-06 2023-10-01T10:45:20.884 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.884 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.884 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:45:20.884 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:51.515985+0000", 2023-10-01T10:45:20.884 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1413999999999999e-05 2023-10-01T10:45:20.884 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.884 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.884 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:45:20.884 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:51.515994+0000", 2023-10-01T10:45:20.884 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.7010000000000002e-06 2023-10-01T10:45:20.885 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.885 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.885 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:45:20.885 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:51.523618+0000", 2023-10-01T10:45:20.885 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0076239020000000001 2023-10-01T10:45:20.885 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.885 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.885 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:45:20.885 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:51.523645+0000", 2023-10-01T10:45:20.885 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.6876000000000002e-05 2023-10-01T10:45:20.885 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.885 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.885 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:45:20.886 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:51.540938+0000", 2023-10-01T10:45:20.886 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.017293464000000001 2023-10-01T10:45:20.886 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.886 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.886 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:45:20.886 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:51.540958+0000", 2023-10-01T10:45:20.886 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.9358e-05 2023-10-01T10:45:20.886 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:20.886 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:45:20.886 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:20.886 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.886 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.887 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.35649.0:208076 9.c e143/128 9:3c280f9f:::benchmark_data_smithi078_113121_object96261:head v 143'13056, mlcod=143'13052)", 2023-10-01T10:45:20.887 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:44:51.517557+0000", 2023-10-01T10:45:20.887 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 29.267543507999999, 2023-10-01T10:45:20.887 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.023418491999999999, 2023-10-01T10:45:20.887 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:45:20.887 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:45:20.887 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:45:20.887 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.887 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:45:20.887 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:51.517557+0000", 2023-10-01T10:45:20.887 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:45:20.887 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.888 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.888 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:45:20.888 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:51.517557+0000", 2023-10-01T10:45:20.888 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:45:20.888 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.888 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.888 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:45:20.888 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:51.517556+0000", 2023-10-01T10:45:20.888 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-10-01T10:45:20.888 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.888 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.888 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:45:20.889 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:51.517568+0000", 2023-10-01T10:45:20.889 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.152e-05 2023-10-01T10:45:20.889 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.889 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.889 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:45:20.889 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:51.517588+0000", 2023-10-01T10:45:20.889 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.9945e-05 2023-10-01T10:45:20.889 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.889 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.889 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:45:20.889 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:51.517596+0000", 2023-10-01T10:45:20.889 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.0050000000000004e-06 2023-10-01T10:45:20.890 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.890 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.890 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:45:20.890 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:51.527373+0000", 2023-10-01T10:45:20.890 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0097776440000000003 2023-10-01T10:45:20.890 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.890 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.890 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:45:20.890 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:51.527409+0000", 2023-10-01T10:45:20.890 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.5361000000000002e-05 2023-10-01T10:45:20.890 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.890 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.891 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:45:20.891 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:51.540960+0000", 2023-10-01T10:45:20.891 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.013550850999999999 2023-10-01T10:45:20.891 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:20.891 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:20.891 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:45:20.891 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:44:51.540976+0000", 2023-10-01T10:45:20.891 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.6056e-05 2023-10-01T10:45:20.891 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:20.891 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:45:20.891 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:20.891 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:20.892 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:45:20.892 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-10-01T10:45:20.892 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:20.920 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:20.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:20.922 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:21.023 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:21.050 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:21.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:21.053 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:21.154 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:21.180 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:21.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:21.182 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:45:21.283 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:21.313 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:21.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:21.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:45:21.416 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:21.451 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:21.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:21.454 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:21.544 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-01T10:45:21.554 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:21.616 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:21.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:21.619 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:45:21.719 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:21.731 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:21 smithi078 bash[78274]: cluster 2023-10-01T10:45:19.813561+0000 mgr.y (mgr.25072) 828 : cluster 0 pgmap v749: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 406 op/s 2023-10-01T10:45:21.731 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:21 smithi078 bash[75299]: cluster 2023-10-01T10:45:19.813561+0000 mgr.y (mgr.25072) 828 : cluster 0 pgmap v749: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 406 op/s 2023-10-01T10:45:21.743 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:21.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:21.746 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:45:21.846 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:21.858 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:21 smithi173 bash[58816]: cluster 2023-10-01T10:45:19.813561+0000 mgr.y (mgr.25072) 828 : cluster 0 pgmap v749: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 406 op/s 2023-10-01T10:45:21.875 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:21.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:21.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:45:21.978 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:22.000 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:45:22.010 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:22.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:22.012 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:45:22.113 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:22.146 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:22.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:22.149 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:22.250 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:22.277 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:22.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:22.280 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:45:22.380 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:22.408 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:22.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:22.410 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:45:22.455 INFO:teuthology.orchestra.run.smithi078.stderr:reweighted osd.6 to 1 (10000) 2023-10-01T10:45:22.511 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:22.521 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:22 smithi078 bash[75299]: audit 2023-10-01T10:45:22.058332+0000 mon.a (mon.0) 1054 : audit 1 from='client.? 172.21.15.78:0/3550831813' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-01T10:45:22.521 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:22 smithi078 bash[78274]: audit 2023-10-01T10:45:22.058332+0000 mon.a (mon.0) 1054 : audit 1 from='client.? 172.21.15.78:0/3550831813' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-01T10:45:22.534 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:22.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:22.537 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:45:22.638 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:22.670 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:22.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:22.673 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:22.774 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:22.786 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:22 smithi173 bash[58816]: audit 2023-10-01T10:45:22.058332+0000 mon.a (mon.0) 1054 : audit 1 from='client.? 172.21.15.78:0/3550831813' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-01T10:45:22.806 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:22.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:22.809 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:22.909 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:22.941 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:22.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:22.944 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:23.045 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:23.072 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:23.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:23.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:45:23.176 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:23.200 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:23.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:23.203 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:45:23.304 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:23.336 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:23.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:23.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:45:23.440 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:23.464 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:23.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:23.466 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:45:23.567 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:23.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:23 smithi078 bash[78274]: cluster 2023-10-01T10:45:21.814144+0000 mgr.y (mgr.25072) 829 : cluster 0 pgmap v750: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 277 op/s 2023-10-01T10:45:23.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:23 smithi078 bash[78274]: audit 2023-10-01T10:45:22.449924+0000 mon.a (mon.0) 1055 : audit 1 from='client.? 172.21.15.78:0/3550831813' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-01T10:45:23.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:23 smithi078 bash[78274]: cluster 2023-10-01T10:45:22.450016+0000 mon.a (mon.0) 1056 : cluster 0 osdmap e147: 8 total, 8 up, 8 in 2023-10-01T10:45:23.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:23 smithi078 bash[78274]: cluster 2023-10-01T10:45:22.717147+0000 osd.7 (osd.7) 96 : cluster 3 failed to encode map e147 with expected crc 2023-10-01T10:45:23.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:23 smithi078 bash[78274]: cluster 2023-10-01T10:45:22.717176+0000 osd.6 (osd.6) 56 : cluster 3 failed to encode map e147 with expected crc 2023-10-01T10:45:23.582 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:23 smithi173 bash[58816]: cluster 2023-10-01T10:45:21.814144+0000 mgr.y (mgr.25072) 829 : cluster 0 pgmap v750: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 277 op/s 2023-10-01T10:45:23.582 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:23 smithi173 bash[58816]: audit 2023-10-01T10:45:22.449924+0000 mon.a (mon.0) 1055 : audit 1 from='client.? 172.21.15.78:0/3550831813' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-01T10:45:23.583 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:23 smithi173 bash[58816]: cluster 2023-10-01T10:45:22.450016+0000 mon.a (mon.0) 1056 : cluster 0 osdmap e147: 8 total, 8 up, 8 in 2023-10-01T10:45:23.583 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:23 smithi173 bash[58816]: cluster 2023-10-01T10:45:22.717147+0000 osd.7 (osd.7) 96 : cluster 3 failed to encode map e147 with expected crc 2023-10-01T10:45:23.583 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:23 smithi173 bash[58816]: cluster 2023-10-01T10:45:22.717176+0000 osd.6 (osd.6) 56 : cluster 3 failed to encode map e147 with expected crc 2023-10-01T10:45:23.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:23 smithi078 bash[75299]: cluster 2023-10-01T10:45:21.814144+0000 mgr.y (mgr.25072) 829 : cluster 0 pgmap v750: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 277 op/s 2023-10-01T10:45:23.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:23 smithi078 bash[75299]: audit 2023-10-01T10:45:22.449924+0000 mon.a (mon.0) 1055 : audit 1 from='client.? 172.21.15.78:0/3550831813' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-01T10:45:23.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:23 smithi078 bash[75299]: cluster 2023-10-01T10:45:22.450016+0000 mon.a (mon.0) 1056 : cluster 0 osdmap e147: 8 total, 8 up, 8 in 2023-10-01T10:45:23.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:23 smithi078 bash[75299]: cluster 2023-10-01T10:45:22.717147+0000 osd.7 (osd.7) 96 : cluster 3 failed to encode map e147 with expected crc 2023-10-01T10:45:23.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:23 smithi078 bash[75299]: cluster 2023-10-01T10:45:22.717176+0000 osd.6 (osd.6) 56 : cluster 3 failed to encode map e147 with expected crc 2023-10-01T10:45:23.603 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:23.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:23.606 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:23.706 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:23.744 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:23.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:23.746 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:23.847 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:23.876 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:23.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:23.879 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:45:23.979 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:24.004 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:24.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:24.007 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:45:24.108 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:24.133 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:24.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:24.135 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:45:24.236 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:24.265 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:24.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:24.268 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:45:24.369 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:24.404 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:24.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:24.406 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:24.507 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:24.515 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:24 smithi078 bash[78274]: cluster 2023-10-01T10:45:22.539865+0000 osd.5 (osd.5) 75 : cluster 3 failed to encode map e147 with expected crc 2023-10-01T10:45:24.515 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:24 smithi078 bash[78274]: cluster 2023-10-01T10:45:22.542379+0000 osd.5 (osd.5) 76 : cluster 3 failed to encode map e147 with expected crc 2023-10-01T10:45:24.515 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:24 smithi078 bash[78274]: cluster 2023-10-01T10:45:22.544849+0000 osd.5 (osd.5) 77 : cluster 3 failed to encode map e147 with expected crc 2023-10-01T10:45:24.516 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:24 smithi078 bash[75299]: cluster 2023-10-01T10:45:22.539865+0000 osd.5 (osd.5) 75 : cluster 3 failed to encode map e147 with expected crc 2023-10-01T10:45:24.516 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:24 smithi078 bash[75299]: cluster 2023-10-01T10:45:22.542379+0000 osd.5 (osd.5) 76 : cluster 3 failed to encode map e147 with expected crc 2023-10-01T10:45:24.516 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:24 smithi078 bash[75299]: cluster 2023-10-01T10:45:22.544849+0000 osd.5 (osd.5) 77 : cluster 3 failed to encode map e147 with expected crc 2023-10-01T10:45:24.530 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:24.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:24.532 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:45:24.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:24 smithi173 bash[58816]: cluster 2023-10-01T10:45:22.539865+0000 osd.5 (osd.5) 75 : cluster 3 failed to encode map e147 with expected crc 2023-10-01T10:45:24.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:24 smithi173 bash[58816]: cluster 2023-10-01T10:45:22.542379+0000 osd.5 (osd.5) 76 : cluster 3 failed to encode map e147 with expected crc 2023-10-01T10:45:24.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:24 smithi173 bash[58816]: cluster 2023-10-01T10:45:22.544849+0000 osd.5 (osd.5) 77 : cluster 3 failed to encode map e147 with expected crc 2023-10-01T10:45:24.633 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:24.663 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:24.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:24.666 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:45:24.767 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:24.800 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:24.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:24.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:24.904 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:24.909 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-01T10:45:24.925 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:24.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:24.927 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:45:25.028 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:25.058 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:25.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:25.060 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:45:25.120 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:45:25.161 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:25.188 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:25.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:25.192 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:45:25.292 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:25.321 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:25.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:25.324 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:45:25.425 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:25.450 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:25.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:25.453 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:45:25.553 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:25.583 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:25.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:25.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:45:25.687 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:25.699 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:25 smithi078 bash[75299]: cluster 2023-10-01T10:45:23.814660+0000 mgr.y (mgr.25072) 830 : cluster 0 pgmap v752: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 81 op/s 2023-10-01T10:45:25.699 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:25 smithi078 bash[78274]: cluster 2023-10-01T10:45:23.814660+0000 mgr.y (mgr.25072) 830 : cluster 0 pgmap v752: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 81 op/s 2023-10-01T10:45:25.713 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:25.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:25.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:45:25.816 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:25.826 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:25 smithi173 bash[58816]: cluster 2023-10-01T10:45:23.814660+0000 mgr.y (mgr.25072) 830 : cluster 0 pgmap v752: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 81 op/s 2023-10-01T10:45:25.847 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:25.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:25.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:45:25.951 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:25.979 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:25.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:25.981 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:45:26.082 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:26.113 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:26.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:26.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:45:26.219 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:26.252 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:26.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:26.256 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:26.323 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T10:45:26.357 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:26.385 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:26.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:26.388 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:26.488 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:26.521 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:26.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:26.524 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:26.625 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:26.654 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:26.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:26.656 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:45:26.757 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:26.781 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:26.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:26.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:45:26.884 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:26.917 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:26.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:26.920 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:27.021 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:27.049 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:27.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:27.052 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:45:27.153 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:27.179 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:27.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:27.181 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:27.282 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:27.312 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:27.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:27.315 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:27.417 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:27.442 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:27.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:27.446 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:45:27.547 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:27.558 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:27 smithi173 bash[58816]: cluster 2023-10-01T10:45:25.815338+0000 mgr.y (mgr.25072) 831 : cluster 0 pgmap v753: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 77 op/s 2023-10-01T10:45:27.578 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:27.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:27.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:27.682 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:27.695 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:27 smithi078 bash[75299]: cluster 2023-10-01T10:45:25.815338+0000 mgr.y (mgr.25072) 831 : cluster 0 pgmap v753: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 77 op/s 2023-10-01T10:45:27.695 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:27 smithi078 bash[78274]: cluster 2023-10-01T10:45:25.815338+0000 mgr.y (mgr.25072) 831 : cluster 0 pgmap v753: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 77 op/s 2023-10-01T10:45:27.710 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:27.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:27.713 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:45:27.814 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:27.845 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:27.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:27.848 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:45:27.948 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:27.980 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:27.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:27.982 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:45:28.083 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:28.104 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:28.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:28.107 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:28.207 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:28.236 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:28.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:28.239 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:45:28.339 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:28.367 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:28.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:28.370 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:45:28.471 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:28.504 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:28.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:28.507 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:45:28.608 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:28.633 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:28.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:28.636 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:45:28.736 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:28.762 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:28.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:28.764 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:45:28.851 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T10:45:28.865 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:28.871 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T10:45:28.892 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T10:45:28.897 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:28.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:28.900 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:45:28.921 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T10:45:28.951 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T10:45:28.978 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T10:45:29.001 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T10:45:29.001 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:29.023 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T10:45:29.024 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:29.027 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T10:45:29.027 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T10:45:29.027 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T10:45:29.027 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T10:45:29.027 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T10:45:29.027 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T10:45:29.027 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T10:45:29.027 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T10:45:29.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:29.029 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:45:29.129 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:29.156 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:29.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:29.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:45:29.259 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:29.292 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:29.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:29.295 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:45:29.396 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:29.423 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:29.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:29.425 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:45:29.441 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:45:29.527 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:29.534 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:29 smithi078 bash[78274]: cluster 2023-10-01T10:45:27.816190+0000 mgr.y (mgr.25072) 832 : cluster 0 pgmap v754: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 8 op/s 2023-10-01T10:45:29.535 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:29 smithi078 bash[78274]: audit 2023-10-01T10:45:28.825790+0000 mon.c (mon.2) 148 : audit 0 from='client.? 172.21.15.78:0/2541668971' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:45:29.535 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:29 smithi078 bash[75299]: cluster 2023-10-01T10:45:27.816190+0000 mgr.y (mgr.25072) 832 : cluster 0 pgmap v754: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 8 op/s 2023-10-01T10:45:29.535 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:29 smithi078 bash[75299]: audit 2023-10-01T10:45:28.825790+0000 mon.c (mon.2) 148 : audit 0 from='client.? 172.21.15.78:0/2541668971' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:45:29.548 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:29.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:29.550 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:45:29.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:29 smithi173 bash[58816]: cluster 2023-10-01T10:45:27.816190+0000 mgr.y (mgr.25072) 832 : cluster 0 pgmap v754: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 8 op/s 2023-10-01T10:45:29.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:29 smithi173 bash[58816]: audit 2023-10-01T10:45:28.825790+0000 mon.c (mon.2) 148 : audit 0 from='client.? 172.21.15.78:0/2541668971' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:45:29.651 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:29.679 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:29.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:29.681 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:45:29.782 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:29.810 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:29.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:29.814 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:45:29.896 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-10-01T10:45:29.896 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T10:45:29.915 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:29.943 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:29.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:29.945 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:45:30.046 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:30.071 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:30.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:30.073 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:45:30.174 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:30.197 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:30.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:30.200 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:45:30.301 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:30.327 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:30.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:30.329 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:45:30.430 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:30.462 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:30.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:30.465 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:45:30.565 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:30.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:45:30 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:45:30] "GET /metrics HTTP/1.1" 200 36236 "" "Prometheus/2.33.4" 2023-10-01T10:45:30.590 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:30.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:30.593 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:30.694 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:30.723 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:30.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:30.726 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:45:30.741 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-01T10:45:30.826 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:30.854 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:30.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:30.856 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:45:30.957 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:30.984 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:30.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:30.987 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:45:31.088 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:31.113 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:31.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:31.116 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:45:31.217 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:31.249 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:31.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:31.252 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:45:31.353 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:31.384 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:31.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:31.388 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:45:31.489 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:31.514 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:31.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:31.516 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:45:31.617 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:31.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:31 smithi173 bash[58816]: cluster 2023-10-01T10:45:29.816977+0000 mgr.y (mgr.25072) 833 : cluster 0 pgmap v755: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:45:31.649 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:31.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:31.652 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:45:31.753 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:31.782 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:31.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:31.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:45:31.884 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:31.917 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:31.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:31.921 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:45:32.021 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:32.051 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:32.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:32.055 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:32.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:31 smithi078 bash[75299]: cluster 2023-10-01T10:45:29.816977+0000 mgr.y (mgr.25072) 833 : cluster 0 pgmap v755: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:45:32.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:31 smithi078 bash[78274]: cluster 2023-10-01T10:45:29.816977+0000 mgr.y (mgr.25072) 833 : cluster 0 pgmap v755: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:45:32.155 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:32.189 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:32.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:32.192 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:32.293 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:32.320 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:32.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:32.323 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:45:32.424 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:32.456 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:32.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:32.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:32.559 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:32.580 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:32.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:32.583 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:45:32.683 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:32.712 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:32.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:32.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:45:32.815 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:32.843 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:32.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:32.846 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:45:32.947 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:32.975 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:32.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:32.977 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:45:33.078 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:33.108 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:33.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:33.111 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:45:33.212 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:33.233 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:33.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:33.235 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:45:33.336 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:33.357 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:33.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:33.358 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:45:33.459 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:33.489 INFO:teuthology.orchestra.run.smithi078.stderr:noscrub is set 2023-10-01T10:45:33.495 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:33.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:33.498 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:33.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:33 smithi078 bash[78274]: cluster 2023-10-01T10:45:31.817823+0000 mgr.y (mgr.25072) 834 : cluster 0 pgmap v756: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:45:33.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:33 smithi078 bash[78274]: audit 2023-10-01T10:45:32.420580+0000 mon.a (mon.0) 1057 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:45:33.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:33 smithi078 bash[78274]: audit 2023-10-01T10:45:33.365339+0000 mon.a (mon.0) 1058 : audit 1 from='client.? 172.21.15.78:0/4288733926' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:45:33.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:33 smithi078 bash[75299]: cluster 2023-10-01T10:45:31.817823+0000 mgr.y (mgr.25072) 834 : cluster 0 pgmap v756: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:45:33.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:33 smithi078 bash[75299]: audit 2023-10-01T10:45:32.420580+0000 mon.a (mon.0) 1057 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:45:33.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:33 smithi078 bash[75299]: audit 2023-10-01T10:45:33.365339+0000 mon.a (mon.0) 1058 : audit 1 from='client.? 172.21.15.78:0/4288733926' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:45:33.599 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:33.622 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:33.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:33.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:45:33.725 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:33.750 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:33.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:33.753 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:45:33.854 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:33.866 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:33 smithi173 bash[58816]: cluster 2023-10-01T10:45:31.817823+0000 mgr.y (mgr.25072) 834 : cluster 0 pgmap v756: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:45:33.866 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:33 smithi173 bash[58816]: audit 2023-10-01T10:45:32.420580+0000 mon.a (mon.0) 1057 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:45:33.866 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:33 smithi173 bash[58816]: audit 2023-10-01T10:45:33.365339+0000 mon.a (mon.0) 1058 : audit 1 from='client.? 172.21.15.78:0/4288733926' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:45:33.886 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:33.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:33.890 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:45:33.990 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:34.022 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:34.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:34.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:45:34.126 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:34.151 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:34.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:34.154 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:45:34.254 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:34.275 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:34.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:34.278 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:45:34.379 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:34.414 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:34.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:34.417 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:45:34.518 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:34.522 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T10:45:34.531 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:34 smithi078 bash[75299]: cluster 2023-10-01T10:45:33.476393+0000 mon.a (mon.0) 1059 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:45:34.531 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:34 smithi078 bash[75299]: cluster 2023-10-01T10:45:33.480737+0000 osd.5 (osd.5) 78 : cluster 3 failed to encode map e148 with expected crc 2023-10-01T10:45:34.531 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:34 smithi078 bash[75299]: cluster 2023-10-01T10:45:33.480865+0000 osd.7 (osd.7) 97 : cluster 3 failed to encode map e148 with expected crc 2023-10-01T10:45:34.532 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:34 smithi078 bash[75299]: audit 2023-10-01T10:45:33.483701+0000 mon.a (mon.0) 1060 : audit 1 from='client.? 172.21.15.78:0/4288733926' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T10:45:34.532 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:34 smithi078 bash[75299]: cluster 2023-10-01T10:45:33.483895+0000 mon.a (mon.0) 1061 : cluster 0 osdmap e148: 8 total, 8 up, 8 in 2023-10-01T10:45:34.532 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:34 smithi078 bash[75299]: cluster 2023-10-01T10:45:33.596674+0000 osd.6 (osd.6) 57 : cluster 3 failed to encode map e148 with expected crc 2023-10-01T10:45:34.532 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:34 smithi078 bash[78274]: cluster 2023-10-01T10:45:33.476393+0000 mon.a (mon.0) 1059 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:45:34.532 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:34 smithi078 bash[78274]: cluster 2023-10-01T10:45:33.480737+0000 osd.5 (osd.5) 78 : cluster 3 failed to encode map e148 with expected crc 2023-10-01T10:45:34.532 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:34 smithi078 bash[78274]: cluster 2023-10-01T10:45:33.480865+0000 osd.7 (osd.7) 97 : cluster 3 failed to encode map e148 with expected crc 2023-10-01T10:45:34.532 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:34 smithi078 bash[78274]: audit 2023-10-01T10:45:33.483701+0000 mon.a (mon.0) 1060 : audit 1 from='client.? 172.21.15.78:0/4288733926' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T10:45:34.532 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:34 smithi078 bash[78274]: cluster 2023-10-01T10:45:33.483895+0000 mon.a (mon.0) 1061 : cluster 0 osdmap e148: 8 total, 8 up, 8 in 2023-10-01T10:45:34.533 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:34 smithi078 bash[78274]: cluster 2023-10-01T10:45:33.596674+0000 osd.6 (osd.6) 57 : cluster 3 failed to encode map e148 with expected crc 2023-10-01T10:45:34.545 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:34.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:34.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:45:34.648 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:34.672 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:34.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:34.675 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:45:34.775 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:34.788 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:34 smithi173 bash[58816]: cluster 2023-10-01T10:45:33.476393+0000 mon.a (mon.0) 1059 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:45:34.788 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:34 smithi173 bash[58816]: cluster 2023-10-01T10:45:33.480737+0000 osd.5 (osd.5) 78 : cluster 3 failed to encode map e148 with expected crc 2023-10-01T10:45:34.788 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:34 smithi173 bash[58816]: cluster 2023-10-01T10:45:33.480865+0000 osd.7 (osd.7) 97 : cluster 3 failed to encode map e148 with expected crc 2023-10-01T10:45:34.788 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:34 smithi173 bash[58816]: audit 2023-10-01T10:45:33.483701+0000 mon.a (mon.0) 1060 : audit 1 from='client.? 172.21.15.78:0/4288733926' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T10:45:34.788 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:34 smithi173 bash[58816]: cluster 2023-10-01T10:45:33.483895+0000 mon.a (mon.0) 1061 : cluster 0 osdmap e148: 8 total, 8 up, 8 in 2023-10-01T10:45:34.788 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:34 smithi173 bash[58816]: cluster 2023-10-01T10:45:33.596674+0000 osd.6 (osd.6) 57 : cluster 3 failed to encode map e148 with expected crc 2023-10-01T10:45:34.810 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:34.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:34.813 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:45:34.914 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:34.942 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:34.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:34.944 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:45:35.045 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:35.077 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:35.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:35.080 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:45:35.180 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:35.208 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:35.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:35.211 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:45:35.312 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:35.337 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:35.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:35.340 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:45:35.440 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:35.471 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:35.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:35.476 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:35.517 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T10:45:35.577 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:35.605 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:35.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:35.608 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:35.654 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:35 smithi078 bash[78274]: cluster 2023-10-01T10:45:33.818409+0000 mgr.y (mgr.25072) 835 : cluster 0 pgmap v758: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:45:35.655 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:35 smithi078 bash[75299]: cluster 2023-10-01T10:45:33.818409+0000 mgr.y (mgr.25072) 835 : cluster 0 pgmap v758: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:45:35.708 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:35.738 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:35.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:35.741 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:45:35.841 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:35.866 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:35.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:35.868 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:45:35.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:35 smithi173 bash[58816]: cluster 2023-10-01T10:45:33.818409+0000 mgr.y (mgr.25072) 835 : cluster 0 pgmap v758: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:45:35.969 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:35.991 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:35.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:35.993 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:45:36.094 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:36.126 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:36.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:36.129 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:36.230 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:36.255 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:36.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:36.257 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:45:36.358 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:36.389 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:36.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:36.391 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:36.492 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:36.521 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:36.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:36.524 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:36.625 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:36.656 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:36.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:36.659 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:45:36.760 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:36.786 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:36.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:36.789 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:36.890 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:36.911 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:36.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:36.914 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:45:37.015 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:37.034 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:37.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:37.036 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:45:37.137 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:37.160 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:37.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:37.163 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:45:37.263 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:37.291 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:37.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:37.294 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:37.394 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:37.424 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:37.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:37.427 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:45:37.528 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:37.531 INFO:teuthology.orchestra.run.smithi078.stderr:pool 'unique_pool_4' removed 2023-10-01T10:45:37.538 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:37 smithi173 bash[58816]: cluster 2023-10-01T10:45:35.819153+0000 mgr.y (mgr.25072) 836 : cluster 0 pgmap v759: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:45:37.538 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:37 smithi173 bash[58816]: audit 2023-10-01T10:45:37.277156+0000 mon.a (mon.0) 1062 : audit 1 from='client.? 172.21.15.78:0/1398156615' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T10:45:37.550 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:37.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:37.553 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:37.653 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:37.686 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:37.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:37.692 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:37.793 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:37.816 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:37.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:37.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:37.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:37 smithi078 bash[75299]: cluster 2023-10-01T10:45:35.819153+0000 mgr.y (mgr.25072) 836 : cluster 0 pgmap v759: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:45:37.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:37 smithi078 bash[75299]: audit 2023-10-01T10:45:37.277156+0000 mon.a (mon.0) 1062 : audit 1 from='client.? 172.21.15.78:0/1398156615' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T10:45:37.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:37 smithi078 bash[78274]: cluster 2023-10-01T10:45:35.819153+0000 mgr.y (mgr.25072) 836 : cluster 0 pgmap v759: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:45:37.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:37 smithi078 bash[78274]: audit 2023-10-01T10:45:37.277156+0000 mon.a (mon.0) 1062 : audit 1 from='client.? 172.21.15.78:0/1398156615' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T10:45:37.919 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:37.940 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:37.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:37.943 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:45:38.043 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:38.065 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:38.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:38.067 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:45:38.167 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:38.197 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:38.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:38.199 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:45:38.300 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:38.327 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:38.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:38.329 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:45:38.430 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:38.463 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:38.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:38.466 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:38.539 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T10:45:38.550 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:38 smithi078 bash[75299]: audit 2023-10-01T10:45:37.501719+0000 mon.a (mon.0) 1063 : audit 1 from='client.? 172.21.15.78:0/1398156615' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-10-01T10:45:38.550 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:38 smithi078 bash[75299]: cluster 2023-10-01T10:45:37.501813+0000 mon.a (mon.0) 1064 : cluster 0 osdmap e149: 8 total, 8 up, 8 in 2023-10-01T10:45:38.551 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:38 smithi078 bash[75299]: cluster 2023-10-01T10:45:37.504151+0000 osd.7 (osd.7) 98 : cluster 3 failed to encode map e149 with expected crc 2023-10-01T10:45:38.551 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:38 smithi078 bash[75299]: cluster 2023-10-01T10:45:37.564768+0000 osd.6 (osd.6) 58 : cluster 3 failed to encode map e149 with expected crc 2023-10-01T10:45:38.551 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:38 smithi078 bash[78274]: audit 2023-10-01T10:45:37.501719+0000 mon.a (mon.0) 1063 : audit 1 from='client.? 172.21.15.78:0/1398156615' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-10-01T10:45:38.551 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:38 smithi078 bash[78274]: cluster 2023-10-01T10:45:37.501813+0000 mon.a (mon.0) 1064 : cluster 0 osdmap e149: 8 total, 8 up, 8 in 2023-10-01T10:45:38.551 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:38 smithi078 bash[78274]: cluster 2023-10-01T10:45:37.504151+0000 osd.7 (osd.7) 98 : cluster 3 failed to encode map e149 with expected crc 2023-10-01T10:45:38.551 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:38 smithi078 bash[78274]: cluster 2023-10-01T10:45:37.564768+0000 osd.6 (osd.6) 58 : cluster 3 failed to encode map e149 with expected crc 2023-10-01T10:45:38.567 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:38.579 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:38 smithi173 bash[58816]: audit 2023-10-01T10:45:37.501719+0000 mon.a (mon.0) 1063 : audit 1 from='client.? 172.21.15.78:0/1398156615' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-10-01T10:45:38.579 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:38 smithi173 bash[58816]: cluster 2023-10-01T10:45:37.501813+0000 mon.a (mon.0) 1064 : cluster 0 osdmap e149: 8 total, 8 up, 8 in 2023-10-01T10:45:38.579 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:38 smithi173 bash[58816]: cluster 2023-10-01T10:45:37.504151+0000 osd.7 (osd.7) 98 : cluster 3 failed to encode map e149 with expected crc 2023-10-01T10:45:38.579 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:38 smithi173 bash[58816]: cluster 2023-10-01T10:45:37.564768+0000 osd.6 (osd.6) 58 : cluster 3 failed to encode map e149 with expected crc 2023-10-01T10:45:38.594 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:38.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:38.596 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:38.697 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:38.729 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:38.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:38.732 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:45:38.833 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:38.861 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:38.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:38.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:45:38.964 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:38.966 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-10-01T10:45:38.966 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T10:45:38.996 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:38.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:38.999 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:45:39.100 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:39.125 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:39.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:39.128 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:45:39.229 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:39.256 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:39.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:39.259 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:45:39.354 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-01T10:45:39.355 INFO:tasks.radosbench:Beginning radosbench... 2023-10-01T10:45:39.355 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_5 2023-10-01T10:45:39.355 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_5 16 2023-10-01T10:45:39.360 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:39.429 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:39.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:39.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:45:39.533 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:39.555 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:39.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:39.558 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:45:39.659 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:39.688 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:39.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:39.692 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:39.753 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-01T10:45:39.766 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:39 smithi173 bash[58816]: cluster 2023-10-01T10:45:37.566287+0000 osd.5 (osd.5) 79 : cluster 3 failed to encode map e149 with expected crc 2023-10-01T10:45:39.766 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:39 smithi173 bash[58816]: cluster 2023-10-01T10:45:37.819706+0000 mgr.y (mgr.25072) 837 : cluster 0 pgmap v761: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:45:39.793 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:39.805 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:39 smithi078 bash[78274]: cluster 2023-10-01T10:45:37.566287+0000 osd.5 (osd.5) 79 : cluster 3 failed to encode map e149 with expected crc 2023-10-01T10:45:39.805 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:39 smithi078 bash[78274]: cluster 2023-10-01T10:45:37.819706+0000 mgr.y (mgr.25072) 837 : cluster 0 pgmap v761: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:45:39.806 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:39 smithi078 bash[75299]: cluster 2023-10-01T10:45:37.566287+0000 osd.5 (osd.5) 79 : cluster 3 failed to encode map e149 with expected crc 2023-10-01T10:45:39.806 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:39 smithi078 bash[75299]: cluster 2023-10-01T10:45:37.819706+0000 mgr.y (mgr.25072) 837 : cluster 0 pgmap v761: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:45:39.825 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:39.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:39.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:45:39.928 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:39.955 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:39.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:39.958 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:45:40.059 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:40.092 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:40.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:40.094 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:45:40.195 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:40.229 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:40.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:40.232 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:40.333 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:40.359 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:40.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:40.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:45:40.463 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:40.496 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:40.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:40.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:40.529 INFO:teuthology.orchestra.run.smithi078.stderr:reweighted osd.7 to 1 (10000) 2023-10-01T10:45:40.577 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:40 smithi078 bash[78274]: audit 2023-10-01T10:45:39.789545+0000 mon.a (mon.0) 1065 : audit 1 from='client.? 172.21.15.78:0/3545074114' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T10:45:40.578 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:45:40 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:45:40] "GET /metrics HTTP/1.1" 200 36192 "" "Prometheus/2.33.4" 2023-10-01T10:45:40.578 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:40 smithi078 bash[75299]: audit 2023-10-01T10:45:39.789545+0000 mon.a (mon.0) 1065 : audit 1 from='client.? 172.21.15.78:0/3545074114' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T10:45:40.599 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:40.633 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:40.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:40.636 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:40.737 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:40.771 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:40.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:40.774 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:45:40.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:40 smithi173 bash[58816]: audit 2023-10-01T10:45:39.789545+0000 mon.a (mon.0) 1065 : audit 1 from='client.? 172.21.15.78:0/3545074114' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T10:45:40.875 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:40.900 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:40.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:40.903 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:45:41.004 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:41.034 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:41.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:41.036 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:45:41.137 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:41.171 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:41.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:41.174 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:41.274 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:41.306 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:41.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:41.308 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:41.409 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:41.441 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:41.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:41.444 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:41.545 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:41.557 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:41 smithi173 bash[58816]: cluster 2023-10-01T10:45:39.820791+0000 mgr.y (mgr.25072) 838 : cluster 0 pgmap v762: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:45:41.557 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:41 smithi173 bash[58816]: cluster 2023-10-01T10:45:40.521322+0000 osd.7 (osd.7) 99 : cluster 3 failed to encode map e150 with expected crc 2023-10-01T10:45:41.558 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:41 smithi173 bash[58816]: audit 2023-10-01T10:45:40.521364+0000 mon.a (mon.0) 1066 : audit 1 from='client.? 172.21.15.78:0/3545074114' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-01T10:45:41.558 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:41 smithi173 bash[58816]: cluster 2023-10-01T10:45:40.521445+0000 mon.a (mon.0) 1067 : cluster 0 osdmap e150: 8 total, 8 up, 8 in 2023-10-01T10:45:41.577 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:41.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:41.579 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:45:41.610 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-01T10:45:41.610 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T10:45:41.680 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:41.711 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:41.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:41.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:45:41.814 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:41.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:41 smithi078 bash[75299]: cluster 2023-10-01T10:45:39.820791+0000 mgr.y (mgr.25072) 838 : cluster 0 pgmap v762: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:45:41.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:41 smithi078 bash[75299]: cluster 2023-10-01T10:45:40.521322+0000 osd.7 (osd.7) 99 : cluster 3 failed to encode map e150 with expected crc 2023-10-01T10:45:41.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:41 smithi078 bash[75299]: audit 2023-10-01T10:45:40.521364+0000 mon.a (mon.0) 1066 : audit 1 from='client.? 172.21.15.78:0/3545074114' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-01T10:45:41.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:41 smithi078 bash[75299]: cluster 2023-10-01T10:45:40.521445+0000 mon.a (mon.0) 1067 : cluster 0 osdmap e150: 8 total, 8 up, 8 in 2023-10-01T10:45:41.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:41 smithi078 bash[78274]: cluster 2023-10-01T10:45:39.820791+0000 mgr.y (mgr.25072) 838 : cluster 0 pgmap v762: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:45:41.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:41 smithi078 bash[78274]: cluster 2023-10-01T10:45:40.521322+0000 osd.7 (osd.7) 99 : cluster 3 failed to encode map e150 with expected crc 2023-10-01T10:45:41.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:41 smithi078 bash[78274]: audit 2023-10-01T10:45:40.521364+0000 mon.a (mon.0) 1066 : audit 1 from='client.? 172.21.15.78:0/3545074114' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-01T10:45:41.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:41 smithi078 bash[78274]: cluster 2023-10-01T10:45:40.521445+0000 mon.a (mon.0) 1067 : cluster 0 osdmap e150: 8 total, 8 up, 8 in 2023-10-01T10:45:41.845 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:41.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:41.850 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:45:41.951 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:41.984 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:41.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:41.987 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:45:42.028 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:45:42.088 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:42.111 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:42.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:42.114 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:45:42.215 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:42.246 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:42.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:42.249 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:42.350 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:42.380 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:42.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:42.382 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:45:42.483 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:42.513 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:42.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:42.515 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:45:42.616 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:42.643 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:42.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:42.646 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:45:42.747 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:42.776 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:42.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:42.779 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:42.880 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:42.907 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:42.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:42.911 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:45:43.012 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:43.038 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:43.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:43.041 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:45:43.141 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:43.171 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:43.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:43.174 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:45:43.275 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:43.305 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:43.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:43.309 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:43.410 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:43.438 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:43.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:43.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:45:43.543 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:43.572 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:43.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:43.575 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:45:43.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:43 smithi173 bash[58816]: cluster 2023-10-01T10:45:41.821662+0000 mgr.y (mgr.25072) 839 : cluster 0 pgmap v764: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:45:43.676 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:43.705 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:43.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:43.708 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:45:43.809 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:43.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:43 smithi078 bash[75299]: cluster 2023-10-01T10:45:41.821662+0000 mgr.y (mgr.25072) 839 : cluster 0 pgmap v764: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:45:43.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:43 smithi078 bash[78274]: cluster 2023-10-01T10:45:41.821662+0000 mgr.y (mgr.25072) 839 : cluster 0 pgmap v764: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:45:43.839 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:43.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:43.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:43.943 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:43.973 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:43.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:43.977 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:45:44.077 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:44.098 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:44.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:44.101 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:45:44.202 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:44.236 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:44.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:44.239 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:45:44.339 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:44.372 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:44.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:44.376 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:45:44.476 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:44.500 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:44.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:44.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:45:44.542 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T10:45:44.604 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:44.634 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:44.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:44.637 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:44.737 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:44.739 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:45:44.739 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:45:44.763 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:44.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:44.766 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:45:44.867 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:44.891 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:44.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:44.893 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:45:44.994 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:45.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:45.020 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:45.020 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:45:45.121 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:45.145 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:45.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:45.147 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:45:45.248 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:45.280 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:45.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:45.284 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:45:45.384 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:45.409 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:45.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:45.411 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:45:45.512 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:45.542 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:45.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:45.545 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:45.646 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:45.675 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:45.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:45.677 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:45.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:45 smithi173 bash[58816]: cluster 2023-10-01T10:45:43.822399+0000 mgr.y (mgr.25072) 840 : cluster 0 pgmap v765: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:45:45.778 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:45.788 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:45 smithi078 bash[78274]: cluster 2023-10-01T10:45:43.822399+0000 mgr.y (mgr.25072) 840 : cluster 0 pgmap v765: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:45:45.789 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:45 smithi078 bash[75299]: cluster 2023-10-01T10:45:43.822399+0000 mgr.y (mgr.25072) 840 : cluster 0 pgmap v765: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:45:45.803 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:45.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:45.805 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:45:45.906 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:45.932 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:45.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:45.935 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:45:46.035 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:46.067 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:46.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:46.069 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:46.170 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:46.193 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:46.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:46.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:45:46.295 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:46.323 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:46.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:46.327 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:45:46.428 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:46.460 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:46.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:46.463 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:45:46.564 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:46.588 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:46.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:46.590 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:46.691 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:46.717 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:46.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:46.719 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:46.820 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:46.841 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:46.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:46.844 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:45:46.945 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:46.977 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:46.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:46.980 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:47.081 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:47.113 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:47.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:47.116 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:45:47.217 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:47.236 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T10:45:47.251 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:47.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:47.254 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:47.261 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T10:45:47.284 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T10:45:47.310 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T10:45:47.310 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:45:47.342 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T10:45:47.354 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:47.366 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T10:45:47.378 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:47.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:47.380 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:45:47.387 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T10:45:47.410 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T10:45:47.412 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T10:45:47.412 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T10:45:47.412 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T10:45:47.413 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T10:45:47.413 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T10:45:47.413 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T10:45:47.413 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T10:45:47.413 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T10:45:47.481 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:47.511 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:47.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:47.515 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:45:47.615 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:47.624 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:47 smithi078 bash[78274]: cluster 2023-10-01T10:45:45.823123+0000 mgr.y (mgr.25072) 841 : cluster 0 pgmap v766: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:45:47.624 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:47 smithi078 bash[78274]: audit 2023-10-01T10:45:47.209551+0000 mon.c (mon.2) 149 : audit 0 from='client.? 172.21.15.78:0/2289777786' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:45:47.624 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:47 smithi078 bash[78274]: audit 2023-10-01T10:45:47.425881+0000 mon.a (mon.0) 1068 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:45:47.624 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:47 smithi078 bash[78274]: audit 2023-10-01T10:45:47.427454+0000 mon.a (mon.0) 1069 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:45:47.624 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:47 smithi078 bash[75299]: cluster 2023-10-01T10:45:45.823123+0000 mgr.y (mgr.25072) 841 : cluster 0 pgmap v766: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:45:47.624 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:47 smithi078 bash[75299]: audit 2023-10-01T10:45:47.209551+0000 mon.c (mon.2) 149 : audit 0 from='client.? 172.21.15.78:0/2289777786' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:45:47.624 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:47 smithi078 bash[75299]: audit 2023-10-01T10:45:47.425881+0000 mon.a (mon.0) 1068 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:45:47.624 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:47 smithi078 bash[75299]: audit 2023-10-01T10:45:47.427454+0000 mon.a (mon.0) 1069 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:45:47.640 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:47.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:47.642 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:45:47.730 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-10-01T10:45:47.730 INFO:teuthology.orchestra.run.smithi173.stdout: "size": 20, 2023-10-01T10:45:47.730 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 600, 2023-10-01T10:45:47.730 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [ 2023-10-01T10:45:47.730 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.730 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.35649.0:75437 9.7 9:eee78d3d:::benchmark_data_smithi078_113121_object75436:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e132)", 2023-10-01T10:45:47.730 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:41:57.432042+0000", 2023-10-01T10:45:47.731 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 230.29170085800001, 2023-10-01T10:45:47.731 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.19556342600000001, 2023-10-01T10:45:47.731 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:45:47.731 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:45:47.731 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:45:47.731 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.35649", 2023-10-01T10:45:47.731 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:45:47.731 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 75437 2023-10-01T10:45:47.731 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.731 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:45:47.731 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.732 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:45:47.732 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:41:57.432042+0000", 2023-10-01T10:45:47.732 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:45:47.732 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.732 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.732 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:45:47.732 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:41:57.432042+0000", 2023-10-01T10:45:47.732 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-10-01T10:45:47.732 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.732 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.733 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:45:47.733 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:41:57.432040+0000", 2023-10-01T10:45:47.733 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.5297000000000003e-05 2023-10-01T10:45:47.733 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.733 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.733 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:45:47.733 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:41:57.432095+0000", 2023-10-01T10:45:47.733 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.849e-05 2023-10-01T10:45:47.733 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.733 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.733 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:45:47.734 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:41:57.432124+0000", 2023-10-01T10:45:47.734 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.7589999999999997e-06 2023-10-01T10:45:47.734 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.734 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.734 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:45:47.734 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:41:57.432132+0000", 2023-10-01T10:45:47.734 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.3866000000000001e-05 2023-10-01T10:45:47.734 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.734 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.734 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:45:47.735 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:41:57.432195+0000", 2023-10-01T10:45:47.735 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00013779800000000001 2023-10-01T10:45:47.735 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.735 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.735 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:45:47.735 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:41:57.432333+0000", 2023-10-01T10:45:47.735 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000153521 2023-10-01T10:45:47.735 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.735 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.735 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 3,7", 2023-10-01T10:45:47.735 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:41:57.432487+0000", 2023-10-01T10:45:47.735 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.007768357 2023-10-01T10:45:47.735 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.736 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.736 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:45:47.736 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:41:57.440255+0000", 2023-10-01T10:45:47.736 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.001740644 2023-10-01T10:45:47.736 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.736 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.736 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:45:47.736 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:41:57.441996+0000", 2023-10-01T10:45:47.736 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.185329206 2023-10-01T10:45:47.736 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.736 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.737 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:45:47.737 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:41:57.627325+0000", 2023-10-01T10:45:47.737 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00017787099999999999 2023-10-01T10:45:47.737 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.737 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.737 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:45:47.737 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:41:57.627503+0000", 2023-10-01T10:45:47.737 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000102398 2023-10-01T10:45:47.737 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.737 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.738 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:45:47.738 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:41:57.627605+0000", 2023-10-01T10:45:47.738 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.19556342600000001 2023-10-01T10:45:47.738 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:47.738 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:45:47.738 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:47.738 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.738 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.738 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.35649.0:75444 9.f 9:f4f35a40:::benchmark_data_smithi078_113121_object75443:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e132)", 2023-10-01T10:45:47.738 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:41:57.438106+0000", 2023-10-01T10:45:47.739 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 230.28563631700001, 2023-10-01T10:45:47.739 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.20050689099999999, 2023-10-01T10:45:47.739 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:45:47.739 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:45:47.739 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:45:47.739 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.35649", 2023-10-01T10:45:47.739 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:45:47.739 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 75444 2023-10-01T10:45:47.739 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.739 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:45:47.739 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.739 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:45:47.740 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:41:57.438106+0000", 2023-10-01T10:45:47.740 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:45:47.740 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.740 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.740 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:45:47.740 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:41:57.438106+0000", 2023-10-01T10:45:47.740 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-10-01T10:45:47.740 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.740 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.741 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:45:47.741 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:41:57.438104+0000", 2023-10-01T10:45:47.741 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.5148000000000002e-05 2023-10-01T10:45:47.741 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.741 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.741 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:45:47.741 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:41:57.438149+0000", 2023-10-01T10:45:47.741 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.5713999999999999e-05 2023-10-01T10:45:47.741 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.741 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.742 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:45:47.742 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:41:57.438165+0000", 2023-10-01T10:45:47.742 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.7649999999999993e-06 2023-10-01T10:45:47.742 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.742 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.742 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:45:47.743 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:41:57.438173+0000", 2023-10-01T10:45:47.743 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00083788600000000003 2023-10-01T10:45:47.743 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.743 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.743 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:45:47.744 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:41:57.439011+0000", 2023-10-01T10:45:47.744 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000119517 2023-10-01T10:45:47.744 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.744 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.744 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:45:47.744 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:41:57.439130+0000", 2023-10-01T10:45:47.744 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000142132 2023-10-01T10:45:47.744 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.744 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.744 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1,3", 2023-10-01T10:45:47.744 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:41:57.439272+0000", 2023-10-01T10:45:47.744 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.003671648 2023-10-01T10:45:47.745 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.745 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.745 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:45:47.745 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:41:57.442944+0000", 2023-10-01T10:45:47.745 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00054012999999999999 2023-10-01T10:45:47.745 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.745 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.745 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:45:47.745 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:41:57.443484+0000", 2023-10-01T10:45:47.745 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.19502510100000001 2023-10-01T10:45:47.745 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.746 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.746 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:45:47.746 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:41:57.638509+0000", 2023-10-01T10:45:47.746 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.4530000000000001e-05 2023-10-01T10:45:47.746 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.746 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.746 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:45:47.746 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:41:57.638564+0000", 2023-10-01T10:45:47.746 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.9406000000000001e-05 2023-10-01T10:45:47.746 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.746 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.746 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:45:47.747 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:41:57.638613+0000", 2023-10-01T10:45:47.747 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.20050689099999999 2023-10-01T10:45:47.747 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:47.747 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:45:47.747 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:47.747 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.747 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.747 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.35649.0:75462 9.f 9:f232a5e4:::benchmark_data_smithi078_113121_object75461:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e132)", 2023-10-01T10:45:47.747 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:41:57.455581+0000", 2023-10-01T10:45:47.747 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 230.26816144700001, 2023-10-01T10:45:47.747 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.18330308200000001, 2023-10-01T10:45:47.747 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:45:47.748 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:45:47.748 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:45:47.748 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.35649", 2023-10-01T10:45:47.748 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:45:47.748 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 75462 2023-10-01T10:45:47.748 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.748 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:45:47.748 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.748 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:45:47.748 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:41:57.455581+0000", 2023-10-01T10:45:47.748 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:45:47.748 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.749 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.749 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:45:47.749 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:41:57.455581+0000", 2023-10-01T10:45:47.749 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-10-01T10:45:47.749 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.749 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.749 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:45:47.749 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:41:57.455579+0000", 2023-10-01T10:45:47.749 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.4064000000000003e-05 2023-10-01T10:45:47.749 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.749 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.749 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:45:47.750 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:41:57.455633+0000", 2023-10-01T10:45:47.750 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.6701000000000002e-05 2023-10-01T10:45:47.750 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.750 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.750 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:45:47.750 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:41:57.455650+0000", 2023-10-01T10:45:47.750 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.4170000000000003e-06 2023-10-01T10:45:47.750 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.750 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.750 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:45:47.750 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:41:57.455658+0000", 2023-10-01T10:45:47.751 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.5665000000000002e-05 2023-10-01T10:45:47.751 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.751 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.751 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:45:47.751 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:41:57.455723+0000", 2023-10-01T10:45:47.751 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000184026 2023-10-01T10:45:47.751 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.751 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.751 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:45:47.751 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:41:57.455907+0000", 2023-10-01T10:45:47.751 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00024078399999999999 2023-10-01T10:45:47.751 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.752 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.752 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1,3", 2023-10-01T10:45:47.752 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:41:57.456148+0000", 2023-10-01T10:45:47.752 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0034636699999999999 2023-10-01T10:45:47.752 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.752 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.752 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:45:47.752 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:41:57.459612+0000", 2023-10-01T10:45:47.752 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000101338 2023-10-01T10:45:47.752 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.752 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.752 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:45:47.753 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:41:57.459713+0000", 2023-10-01T10:45:47.753 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.17907108999999999 2023-10-01T10:45:47.753 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.753 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.753 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:45:47.753 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:41:57.638784+0000", 2023-10-01T10:45:47.753 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.8457e-05 2023-10-01T10:45:47.753 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.753 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.753 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:45:47.753 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:41:57.638843+0000", 2023-10-01T10:45:47.753 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.1637999999999997e-05 2023-10-01T10:45:47.754 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.754 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.754 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:45:47.754 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:41:57.638884+0000", 2023-10-01T10:45:47.754 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.18330308200000001 2023-10-01T10:45:47.754 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:47.754 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:45:47.754 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:47.754 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.754 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.754 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.35649.0:75467 9.f 9:fdc00e4b:::benchmark_data_smithi078_113121_object75466:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e132)", 2023-10-01T10:45:47.754 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:41:57.463762+0000", 2023-10-01T10:45:47.755 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 230.259980461, 2023-10-01T10:45:47.755 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.182304942, 2023-10-01T10:45:47.755 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:45:47.755 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:45:47.755 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:45:47.755 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.35649", 2023-10-01T10:45:47.755 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:45:47.755 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 75467 2023-10-01T10:45:47.755 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.755 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:45:47.755 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.755 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:45:47.756 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:41:57.463762+0000", 2023-10-01T10:45:47.756 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:45:47.756 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.756 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.756 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:45:47.756 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:41:57.463762+0000", 2023-10-01T10:45:47.756 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-10-01T10:45:47.756 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.756 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.756 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:45:47.756 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:41:57.463761+0000", 2023-10-01T10:45:47.756 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.6884e-05 2023-10-01T10:45:47.756 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.757 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.757 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:45:47.757 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:41:57.463818+0000", 2023-10-01T10:45:47.757 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.8649999999999996e-06 2023-10-01T10:45:47.757 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.757 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.757 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:45:47.757 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:41:57.463827+0000", 2023-10-01T10:45:47.757 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.6650000000000002e-06 2023-10-01T10:45:47.757 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.757 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.758 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:45:47.758 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:41:57.463832+0000", 2023-10-01T10:45:47.758 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.1201e-05 2023-10-01T10:45:47.758 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.758 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.758 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:45:47.758 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:41:57.463883+0000", 2023-10-01T10:45:47.758 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00014306900000000001 2023-10-01T10:45:47.758 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.758 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.758 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:45:47.758 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:41:57.464026+0000", 2023-10-01T10:45:47.759 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000186665 2023-10-01T10:45:47.759 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.759 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.759 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1,3", 2023-10-01T10:45:47.759 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:41:57.464213+0000", 2023-10-01T10:45:47.759 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0022879889999999998 2023-10-01T10:45:47.759 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.759 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.759 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:45:47.759 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:41:57.466501+0000", 2023-10-01T10:45:47.759 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00011879699999999999 2023-10-01T10:45:47.759 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.760 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.760 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:45:47.760 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:41:57.466620+0000", 2023-10-01T10:45:47.760 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.17911406699999999 2023-10-01T10:45:47.760 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.760 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.760 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:45:47.760 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:41:57.645734+0000", 2023-10-01T10:45:47.760 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000284839 2023-10-01T10:45:47.760 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.760 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.760 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:45:47.761 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:41:57.646019+0000", 2023-10-01T10:45:47.761 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.8421999999999998e-05 2023-10-01T10:45:47.761 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.761 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.761 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:45:47.761 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:41:57.646067+0000", 2023-10-01T10:45:47.761 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.182304942 2023-10-01T10:45:47.761 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:47.761 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:45:47.761 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:47.761 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.762 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.762 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.35649.0:93061 9.7 9:eb2aac41:::benchmark_data_smithi078_113121_object93060:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e133)", 2023-10-01T10:45:47.762 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:42:15.639582+0000", 2023-10-01T10:45:47.762 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 212.08416072, 2023-10-01T10:45:47.762 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.15776612600000001, 2023-10-01T10:45:47.762 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:45:47.762 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:45:47.762 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:45:47.762 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.35649", 2023-10-01T10:45:47.762 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:45:47.762 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 93061 2023-10-01T10:45:47.763 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.763 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:45:47.763 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.763 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:45:47.763 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:15.639582+0000", 2023-10-01T10:45:47.763 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:45:47.763 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.763 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.763 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:45:47.763 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:15.639582+0000", 2023-10-01T10:45:47.763 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999995 2023-10-01T10:45:47.763 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.764 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.764 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:45:47.764 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:15.639581+0000", 2023-10-01T10:45:47.764 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.3912999999999999e-05 2023-10-01T10:45:47.764 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.764 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.764 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:45:47.764 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:15.639615+0000", 2023-10-01T10:45:47.764 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.2599999999999996e-06 2023-10-01T10:45:47.764 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.764 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.765 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:45:47.765 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:15.639621+0000", 2023-10-01T10:45:47.765 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.022e-06 2023-10-01T10:45:47.765 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.765 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.765 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:45:47.765 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:15.639624+0000", 2023-10-01T10:45:47.765 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.012969279 2023-10-01T10:45:47.765 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.765 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.765 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:45:47.766 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:15.652593+0000", 2023-10-01T10:45:47.766 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00015688599999999999 2023-10-01T10:45:47.766 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.766 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.766 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:45:47.766 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:15.652750+0000", 2023-10-01T10:45:47.766 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00017989699999999999 2023-10-01T10:45:47.766 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.766 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.766 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 3,7", 2023-10-01T10:45:47.766 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:15.652930+0000", 2023-10-01T10:45:47.767 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0093396480000000007 2023-10-01T10:45:47.767 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.767 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.767 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:45:47.767 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:15.662269+0000", 2023-10-01T10:45:47.767 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.2760000000000006e-06 2023-10-01T10:45:47.767 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.767 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.767 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:45:47.767 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:15.662279+0000", 2023-10-01T10:45:47.767 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.13484352599999999 2023-10-01T10:45:47.767 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.768 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.768 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:45:47.768 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:15.797122+0000", 2023-10-01T10:45:47.768 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000114444 2023-10-01T10:45:47.768 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.768 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.768 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:45:47.768 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:15.797237+0000", 2023-10-01T10:45:47.768 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000111488 2023-10-01T10:45:47.768 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.768 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.768 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:45:47.769 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:15.797348+0000", 2023-10-01T10:45:47.769 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.15776612600000001 2023-10-01T10:45:47.769 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:47.769 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:45:47.769 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:47.769 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.769 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.769 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.35649.0:93063 9.7 9:e440e98e:::benchmark_data_smithi078_113121_object93062:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e133)", 2023-10-01T10:45:47.769 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:42:15.640193+0000", 2023-10-01T10:45:47.769 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 212.083549175, 2023-10-01T10:45:47.769 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.17225057499999999, 2023-10-01T10:45:47.769 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:45:47.770 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:45:47.770 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:45:47.770 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.35649", 2023-10-01T10:45:47.770 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:45:47.770 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 93063 2023-10-01T10:45:47.770 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.770 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:45:47.770 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.770 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:45:47.770 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:15.640193+0000", 2023-10-01T10:45:47.771 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:45:47.771 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.771 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.771 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:45:47.771 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:15.640193+0000", 2023-10-01T10:45:47.771 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-10-01T10:45:47.771 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.771 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.771 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:45:47.771 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:15.640193+0000", 2023-10-01T10:45:47.771 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.1755999999999996e-05 2023-10-01T10:45:47.771 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.772 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.772 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:45:47.772 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:15.640254+0000", 2023-10-01T10:45:47.772 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1326e-05 2023-10-01T10:45:47.772 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.772 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.772 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:45:47.772 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:15.640266+0000", 2023-10-01T10:45:47.772 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.9050000000000002e-06 2023-10-01T10:45:47.772 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.772 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.773 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:45:47.773 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:15.640271+0000", 2023-10-01T10:45:47.773 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.018255782000000002 2023-10-01T10:45:47.773 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.773 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.773 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:45:47.773 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:15.658527+0000", 2023-10-01T10:45:47.773 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000151528 2023-10-01T10:45:47.773 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.773 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.773 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:45:47.773 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:15.658679+0000", 2023-10-01T10:45:47.774 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000178887 2023-10-01T10:45:47.774 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.774 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.774 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 3,7", 2023-10-01T10:45:47.774 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:15.658858+0000", 2023-10-01T10:45:47.774 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.007018425 2023-10-01T10:45:47.774 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.774 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.774 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:45:47.774 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:15.665876+0000", 2023-10-01T10:45:47.774 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000539111 2023-10-01T10:45:47.775 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.775 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.775 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:45:47.775 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:15.666415+0000", 2023-10-01T10:45:47.775 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.14593177800000001 2023-10-01T10:45:47.775 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.775 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.775 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:45:47.775 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:15.812347+0000", 2023-10-01T10:45:47.775 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.0695999999999997e-05 2023-10-01T10:45:47.775 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.775 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.776 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:45:47.776 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:15.812398+0000", 2023-10-01T10:45:47.776 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.6323999999999999e-05 2023-10-01T10:45:47.776 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.776 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.776 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:45:47.776 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:15.812444+0000", 2023-10-01T10:45:47.776 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.17225057499999999 2023-10-01T10:45:47.776 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:47.776 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:45:47.776 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:47.776 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.777 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.777 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.35649.0:128209 9.f 9:fa8140d7:::benchmark_data_smithi078_113121_object16394:head [delete] snapc 0=[] ondisk+write+known_if_redirected e135)", 2023-10-01T10:45:47.777 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:42:59.143573+0000", 2023-10-01T10:45:47.777 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 168.58017007199999, 2023-10-01T10:45:47.777 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.16214827300000001, 2023-10-01T10:45:47.777 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:45:47.777 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:45:47.777 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:45:47.777 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.35649", 2023-10-01T10:45:47.777 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:45:47.777 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 128209 2023-10-01T10:45:47.777 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.778 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:45:47.778 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.778 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:45:47.778 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:59.143573+0000", 2023-10-01T10:45:47.778 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:45:47.778 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.778 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.778 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:45:47.778 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:59.143573+0000", 2023-10-01T10:45:47.778 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999995 2023-10-01T10:45:47.778 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.779 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.779 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:45:47.779 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:59.143572+0000", 2023-10-01T10:45:47.779 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.6820000000000001e-06 2023-10-01T10:45:47.779 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.779 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.779 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:45:47.779 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:59.143576+0000", 2023-10-01T10:45:47.779 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.8969999999999996e-06 2023-10-01T10:45:47.779 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.779 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.779 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:45:47.779 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:59.143582+0000", 2023-10-01T10:45:47.780 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.8240000000000001e-06 2023-10-01T10:45:47.780 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.780 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.780 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:45:47.780 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:59.143585+0000", 2023-10-01T10:45:47.780 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0045280040000000004 2023-10-01T10:45:47.780 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.780 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.780 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:45:47.780 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:59.148113+0000", 2023-10-01T10:45:47.780 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.6457000000000002e-05 2023-10-01T10:45:47.781 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.781 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.781 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:45:47.781 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:59.148200+0000", 2023-10-01T10:45:47.781 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.8331999999999995e-05 2023-10-01T10:45:47.781 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.781 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.781 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1,3", 2023-10-01T10:45:47.781 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:59.148278+0000", 2023-10-01T10:45:47.781 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0078642060000000003 2023-10-01T10:45:47.781 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.782 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.782 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:45:47.782 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:59.156142+0000", 2023-10-01T10:45:47.782 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.140514206 2023-10-01T10:45:47.782 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.782 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.782 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:45:47.782 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:59.296657+0000", 2023-10-01T10:45:47.782 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0089669199999999998 2023-10-01T10:45:47.782 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.782 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.782 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:45:47.783 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:59.305624+0000", 2023-10-01T10:45:47.783 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.3078999999999997e-05 2023-10-01T10:45:47.783 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.783 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.783 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:45:47.783 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:59.305687+0000", 2023-10-01T10:45:47.783 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.4248000000000001e-05 2023-10-01T10:45:47.783 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.783 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.783 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:45:47.783 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:59.305721+0000", 2023-10-01T10:45:47.784 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.16214827300000001 2023-10-01T10:45:47.784 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:47.784 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:45:47.784 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:47.784 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.784 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.784 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.35649.0:128210 9.f 9:f0e1d0a0:::benchmark_data_smithi078_113121_object16395:head [delete] snapc 0=[] ondisk+write+known_if_redirected e135)", 2023-10-01T10:45:47.784 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:42:59.149060+0000", 2023-10-01T10:45:47.785 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 168.57468306199999, 2023-10-01T10:45:47.785 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.15677439200000001, 2023-10-01T10:45:47.785 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:45:47.785 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:45:47.786 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:45:47.786 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.35649", 2023-10-01T10:45:47.786 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:45:47.786 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 128210 2023-10-01T10:45:47.786 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.786 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:45:47.786 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.786 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:45:47.786 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:59.149060+0000", 2023-10-01T10:45:47.786 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:45:47.786 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.787 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.787 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:45:47.787 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:59.149060+0000", 2023-10-01T10:45:47.787 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-10-01T10:45:47.787 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.787 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.787 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:45:47.787 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:59.149058+0000", 2023-10-01T10:45:47.787 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1285e-05 2023-10-01T10:45:47.787 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.787 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.787 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:45:47.788 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:59.149069+0000", 2023-10-01T10:45:47.788 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.7546000000000001e-05 2023-10-01T10:45:47.788 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.788 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.788 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:45:47.788 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:59.149087+0000", 2023-10-01T10:45:47.788 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.4209999999999995e-06 2023-10-01T10:45:47.788 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.788 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.788 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:45:47.788 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:59.149095+0000", 2023-10-01T10:45:47.788 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0037841989999999998 2023-10-01T10:45:47.789 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.789 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.789 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:45:47.789 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:59.152879+0000", 2023-10-01T10:45:47.789 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00016067300000000001 2023-10-01T10:45:47.789 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.789 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.789 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:45:47.789 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:59.153040+0000", 2023-10-01T10:45:47.789 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000107396 2023-10-01T10:45:47.789 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.790 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.790 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1,3", 2023-10-01T10:45:47.790 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:59.153147+0000", 2023-10-01T10:45:47.790 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0031130929999999999 2023-10-01T10:45:47.790 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.790 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.790 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:45:47.790 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:59.156260+0000", 2023-10-01T10:45:47.790 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.14350358599999999 2023-10-01T10:45:47.790 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.790 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.790 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:45:47.791 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:59.299764+0000", 2023-10-01T10:45:47.791 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0059918250000000001 2023-10-01T10:45:47.791 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.791 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.791 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:45:47.791 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:59.305756+0000", 2023-10-01T10:45:47.791 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.9641000000000002e-05 2023-10-01T10:45:47.791 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.791 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.791 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:45:47.791 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:59.305795+0000", 2023-10-01T10:45:47.791 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.8560999999999999e-05 2023-10-01T10:45:47.792 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.792 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.792 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:45:47.792 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:59.305834+0000", 2023-10-01T10:45:47.792 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.15677439200000001 2023-10-01T10:45:47.792 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:47.792 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:45:47.792 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:47.792 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.792 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.792 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.35649.0:128373 9.7 9:e4f059a2:::benchmark_data_smithi078_113121_object16558:head [delete] snapc 0=[] ondisk+write+known_if_redirected e135)", 2023-10-01T10:45:47.793 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:42:59.473399+0000", 2023-10-01T10:45:47.793 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 168.250343566, 2023-10-01T10:45:47.793 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.15808470299999999, 2023-10-01T10:45:47.793 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:45:47.793 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:45:47.793 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:45:47.793 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.35649", 2023-10-01T10:45:47.793 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:45:47.793 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 128373 2023-10-01T10:45:47.793 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.793 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:45:47.793 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.794 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:45:47.794 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:59.473399+0000", 2023-10-01T10:45:47.794 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:45:47.794 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.794 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.794 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:45:47.794 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:59.473399+0000", 2023-10-01T10:45:47.794 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-10-01T10:45:47.794 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.794 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.794 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:45:47.795 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:59.473398+0000", 2023-10-01T10:45:47.795 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.302e-06 2023-10-01T10:45:47.795 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.795 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.795 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:45:47.795 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:59.473404+0000", 2023-10-01T10:45:47.795 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.3870000000000001e-06 2023-10-01T10:45:47.795 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.795 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.795 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:45:47.795 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:59.473414+0000", 2023-10-01T10:45:47.795 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.7679999999999999e-06 2023-10-01T10:45:47.796 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.796 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.796 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:45:47.796 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:59.473418+0000", 2023-10-01T10:45:47.796 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0019375589999999999 2023-10-01T10:45:47.796 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.796 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.796 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:45:47.796 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:59.475356+0000", 2023-10-01T10:45:47.796 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000109547 2023-10-01T10:45:47.797 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.797 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.797 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:45:47.797 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:59.475465+0000", 2023-10-01T10:45:47.797 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00011694 2023-10-01T10:45:47.797 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.797 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.797 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 3,7", 2023-10-01T10:45:47.797 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:59.475582+0000", 2023-10-01T10:45:47.797 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00546942 2023-10-01T10:45:47.797 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.798 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.798 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:45:47.798 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:59.481052+0000", 2023-10-01T10:45:47.798 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0022581860000000001 2023-10-01T10:45:47.798 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.798 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.798 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:45:47.798 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:59.483310+0000", 2023-10-01T10:45:47.798 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.14803249499999999 2023-10-01T10:45:47.798 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.799 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.799 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:45:47.799 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:59.631342+0000", 2023-10-01T10:45:47.799 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.8361000000000004e-05 2023-10-01T10:45:47.799 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.799 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.799 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:45:47.799 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:59.631421+0000", 2023-10-01T10:45:47.799 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.2961999999999998e-05 2023-10-01T10:45:47.799 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.799 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.799 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:45:47.799 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:42:59.631484+0000", 2023-10-01T10:45:47.800 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.15808470299999999 2023-10-01T10:45:47.800 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:47.800 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:45:47.800 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:47.800 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.800 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.800 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.35649.0:134689 9.f 9:f2e55769:::benchmark_data_smithi078_113121_object22874:head [delete] snapc 0=[] ondisk+write+known_if_redirected e135)", 2023-10-01T10:45:47.800 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:43:07.762823+0000", 2023-10-01T10:45:47.800 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 159.96091961499999, 2023-10-01T10:45:47.800 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.15597934999999999, 2023-10-01T10:45:47.800 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:45:47.800 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:45:47.801 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:45:47.801 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.35649", 2023-10-01T10:45:47.801 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:45:47.801 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 134689 2023-10-01T10:45:47.801 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.801 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:45:47.801 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.801 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:45:47.801 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:07.762823+0000", 2023-10-01T10:45:47.801 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:45:47.801 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.802 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.802 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:45:47.802 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:07.762823+0000", 2023-10-01T10:45:47.802 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-10-01T10:45:47.802 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.802 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.802 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:45:47.802 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:07.762822+0000", 2023-10-01T10:45:47.802 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.6830000000000002e-06 2023-10-01T10:45:47.802 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.802 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.802 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:45:47.802 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:07.762829+0000", 2023-10-01T10:45:47.803 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0832999999999999e-05 2023-10-01T10:45:47.803 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.803 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.803 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:45:47.803 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:07.762840+0000", 2023-10-01T10:45:47.803 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.9150000000000001e-06 2023-10-01T10:45:47.803 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.803 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.803 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:45:47.803 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:07.762845+0000", 2023-10-01T10:45:47.803 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.016720097999999999 2023-10-01T10:45:47.803 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.804 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.804 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:45:47.804 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:07.779566+0000", 2023-10-01T10:45:47.804 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000139402 2023-10-01T10:45:47.804 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.804 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.804 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:45:47.804 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:07.779705+0000", 2023-10-01T10:45:47.804 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00013796500000000001 2023-10-01T10:45:47.804 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.804 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.804 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1,3", 2023-10-01T10:45:47.805 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:07.779843+0000", 2023-10-01T10:45:47.805 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0048130079999999997 2023-10-01T10:45:47.805 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.805 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.805 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:45:47.805 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:07.784656+0000", 2023-10-01T10:45:47.805 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0023711940000000001 2023-10-01T10:45:47.805 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.805 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.805 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:45:47.805 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:07.787027+0000", 2023-10-01T10:45:47.806 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.13167738500000001 2023-10-01T10:45:47.806 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.806 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.806 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:45:47.806 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:07.918704+0000", 2023-10-01T10:45:47.806 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.4215999999999997e-05 2023-10-01T10:45:47.806 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.806 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.806 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:45:47.806 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:07.918759+0000", 2023-10-01T10:45:47.806 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.3652999999999997e-05 2023-10-01T10:45:47.807 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.807 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.807 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:45:47.807 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:07.918802+0000", 2023-10-01T10:45:47.807 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.15597934999999999 2023-10-01T10:45:47.807 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:47.807 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:45:47.807 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:47.807 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.807 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.807 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.35649.0:158051 9.6 9:6c542c4a:::benchmark_data_smithi078_113121_object46236:head [delete] snapc 0=[] ondisk+write+known_if_redirected e137)", 2023-10-01T10:45:47.807 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:43:39.378615+0000", 2023-10-01T10:45:47.808 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 128.345127598, 2023-10-01T10:45:47.808 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.23057741400000001, 2023-10-01T10:45:47.808 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:45:47.808 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:45:47.808 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:45:47.808 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.35649", 2023-10-01T10:45:47.808 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:45:47.808 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 158051 2023-10-01T10:45:47.808 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.808 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:45:47.808 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.808 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:45:47.809 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.378615+0000", 2023-10-01T10:45:47.809 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:45:47.809 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.809 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.809 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:45:47.809 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.378615+0000", 2023-10-01T10:45:47.809 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-10-01T10:45:47.809 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.809 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.809 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:45:47.809 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.378613+0000", 2023-10-01T10:45:47.809 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.3070000000000003e-06 2023-10-01T10:45:47.810 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.810 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.810 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:45:47.810 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.378621+0000", 2023-10-01T10:45:47.810 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.9786999999999999e-05 2023-10-01T10:45:47.810 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.810 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.810 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:45:47.810 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.378641+0000", 2023-10-01T10:45:47.810 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.8250000000000005e-06 2023-10-01T10:45:47.810 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.810 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.811 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:45:47.811 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.378649+0000", 2023-10-01T10:45:47.811 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.6515999999999999e-05 2023-10-01T10:45:47.811 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.811 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.811 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:45:47.811 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.378705+0000", 2023-10-01T10:45:47.811 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000144108 2023-10-01T10:45:47.811 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.811 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.811 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:45:47.811 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.378850+0000", 2023-10-01T10:45:47.812 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00015646099999999999 2023-10-01T10:45:47.812 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.812 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.812 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1,2", 2023-10-01T10:45:47.812 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.379006+0000", 2023-10-01T10:45:47.812 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.016587009 2023-10-01T10:45:47.812 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.812 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.812 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:45:47.812 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.395593+0000", 2023-10-01T10:45:47.812 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.3212000000000003e-05 2023-10-01T10:45:47.813 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.813 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.813 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:45:47.813 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.395666+0000", 2023-10-01T10:45:47.813 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.213244132 2023-10-01T10:45:47.813 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.813 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.813 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:45:47.813 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.608910+0000", 2023-10-01T10:45:47.813 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00016685699999999999 2023-10-01T10:45:47.813 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.814 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.814 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:45:47.814 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.609077+0000", 2023-10-01T10:45:47.814 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000115271 2023-10-01T10:45:47.814 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.814 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.814 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:45:47.814 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.609192+0000", 2023-10-01T10:45:47.814 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.23057741400000001 2023-10-01T10:45:47.814 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:47.814 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:45:47.814 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:47.815 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.815 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.815 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.35649.0:158056 9.6 9:68800f44:::benchmark_data_smithi078_113121_object46241:head [delete] snapc 0=[] ondisk+write+known_if_redirected e137)", 2023-10-01T10:45:47.815 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:43:39.383099+0000", 2023-10-01T10:45:47.815 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 128.340643978, 2023-10-01T10:45:47.815 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.24195572700000001, 2023-10-01T10:45:47.815 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:45:47.815 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:45:47.815 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:45:47.815 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.35649", 2023-10-01T10:45:47.815 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:45:47.815 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 158056 2023-10-01T10:45:47.816 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.816 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:45:47.816 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.816 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:45:47.816 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.383099+0000", 2023-10-01T10:45:47.816 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:45:47.816 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.816 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.816 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:45:47.816 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.383099+0000", 2023-10-01T10:45:47.816 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-10-01T10:45:47.817 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.817 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.817 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:45:47.817 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.383097+0000", 2023-10-01T10:45:47.817 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.044e-05 2023-10-01T10:45:47.817 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.817 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.817 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:45:47.817 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.383108+0000", 2023-10-01T10:45:47.817 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1627999999999999e-05 2023-10-01T10:45:47.817 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.817 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.818 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:45:47.818 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.383119+0000", 2023-10-01T10:45:47.818 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.5639999999999996e-06 2023-10-01T10:45:47.818 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.818 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.818 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:45:47.818 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.383125+0000", 2023-10-01T10:45:47.818 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.001306923 2023-10-01T10:45:47.818 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.818 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.818 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:45:47.818 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.384432+0000", 2023-10-01T10:45:47.819 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00010113 2023-10-01T10:45:47.819 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.819 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.819 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:45:47.819 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.384533+0000", 2023-10-01T10:45:47.819 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.4908999999999995e-05 2023-10-01T10:45:47.819 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.819 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.819 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1,2", 2023-10-01T10:45:47.819 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.384628+0000", 2023-10-01T10:45:47.819 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.010997521999999999 2023-10-01T10:45:47.820 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.820 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.820 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:45:47.820 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.395625+0000", 2023-10-01T10:45:47.820 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.6918e-05 2023-10-01T10:45:47.820 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.820 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.820 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:45:47.820 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.395672+0000", 2023-10-01T10:45:47.820 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.22925029099999999 2023-10-01T10:45:47.820 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.820 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.821 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:45:47.821 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.624923+0000", 2023-10-01T10:45:47.821 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.3591999999999997e-05 2023-10-01T10:45:47.821 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.821 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.821 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:45:47.821 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.624986+0000", 2023-10-01T10:45:47.821 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.8189999999999996e-05 2023-10-01T10:45:47.821 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.821 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.821 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:45:47.821 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.625054+0000", 2023-10-01T10:45:47.822 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.24195572700000001 2023-10-01T10:45:47.822 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:47.822 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:45:47.822 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:47.822 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.822 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.822 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.35649.0:158057 9.a 9:5726c16d:::benchmark_data_smithi078_113121_object46242:head [delete] snapc 0=[] ondisk+write+known_if_redirected e137)", 2023-10-01T10:45:47.822 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:43:39.386088+0000", 2023-10-01T10:45:47.822 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 128.33765442800001, 2023-10-01T10:45:47.822 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.23333553600000001, 2023-10-01T10:45:47.822 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:45:47.823 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:45:47.823 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:45:47.823 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.35649", 2023-10-01T10:45:47.823 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:45:47.823 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 158057 2023-10-01T10:45:47.823 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.823 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:45:47.823 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.823 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:45:47.823 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.386088+0000", 2023-10-01T10:45:47.823 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:45:47.823 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.823 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.824 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:45:47.824 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.386088+0000", 2023-10-01T10:45:47.824 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-10-01T10:45:47.824 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.824 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.824 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:45:47.824 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.386086+0000", 2023-10-01T10:45:47.824 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.9789999999999993e-06 2023-10-01T10:45:47.824 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.824 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.824 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:45:47.824 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.386094+0000", 2023-10-01T10:45:47.825 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4647999999999999e-05 2023-10-01T10:45:47.825 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.825 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.825 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:45:47.825 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.386109+0000", 2023-10-01T10:45:47.825 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.0670000000000003e-06 2023-10-01T10:45:47.825 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.825 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.825 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:45:47.825 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.386116+0000", 2023-10-01T10:45:47.825 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.1812999999999999e-05 2023-10-01T10:45:47.826 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.826 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.826 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:45:47.826 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.386178+0000", 2023-10-01T10:45:47.826 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000114607 2023-10-01T10:45:47.826 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.826 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.826 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:45:47.826 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.386292+0000", 2023-10-01T10:45:47.826 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000136711 2023-10-01T10:45:47.827 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.827 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.827 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1,7", 2023-10-01T10:45:47.827 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.386429+0000", 2023-10-01T10:45:47.827 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.010457914 2023-10-01T10:45:47.828 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.828 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.828 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:45:47.828 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.396887+0000", 2023-10-01T10:45:47.828 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0020678089999999999 2023-10-01T10:45:47.828 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.828 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.828 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:45:47.828 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.398955+0000", 2023-10-01T10:45:47.828 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.220305955 2023-10-01T10:45:47.828 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.828 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.829 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:45:47.829 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.619261+0000", 2023-10-01T10:45:47.829 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.9376999999999995e-05 2023-10-01T10:45:47.829 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.829 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.829 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:45:47.829 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.619350+0000", 2023-10-01T10:45:47.829 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.3505999999999997e-05 2023-10-01T10:45:47.829 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.829 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.829 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:45:47.829 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.619424+0000", 2023-10-01T10:45:47.830 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.23333553600000001 2023-10-01T10:45:47.830 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:47.830 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:45:47.830 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:47.830 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.830 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.830 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.35649.0:158058 9.6 9:687242d5:::benchmark_data_smithi078_113121_object46243:head [delete] snapc 0=[] ondisk+write+known_if_redirected e137)", 2023-10-01T10:45:47.830 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:43:39.386184+0000", 2023-10-01T10:45:47.830 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 128.33755892600001, 2023-10-01T10:45:47.830 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.239007572, 2023-10-01T10:45:47.830 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:45:47.830 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:45:47.831 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:45:47.831 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.35649", 2023-10-01T10:45:47.831 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:45:47.831 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 158058 2023-10-01T10:45:47.831 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.831 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:45:47.831 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.831 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:45:47.831 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.386184+0000", 2023-10-01T10:45:47.831 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:45:47.831 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.832 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.832 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:45:47.832 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.386184+0000", 2023-10-01T10:45:47.832 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-10-01T10:45:47.832 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.832 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.832 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:45:47.832 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.386183+0000", 2023-10-01T10:45:47.832 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.6150000000000003e-06 2023-10-01T10:45:47.832 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.832 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.832 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:45:47.833 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.386192+0000", 2023-10-01T10:45:47.833 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1348999999999999e-05 2023-10-01T10:45:47.833 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.833 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.833 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:45:47.833 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.386204+0000", 2023-10-01T10:45:47.833 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.1600000000000003e-06 2023-10-01T10:45:47.833 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.833 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.833 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:45:47.833 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.386210+0000", 2023-10-01T10:45:47.833 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0031139729999999999 2023-10-01T10:45:47.834 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.834 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.834 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:45:47.834 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.389324+0000", 2023-10-01T10:45:47.834 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000119659 2023-10-01T10:45:47.834 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.834 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.834 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:45:47.834 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.389444+0000", 2023-10-01T10:45:47.834 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00013454600000000001 2023-10-01T10:45:47.834 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.834 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.835 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1,2", 2023-10-01T10:45:47.835 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.389578+0000", 2023-10-01T10:45:47.835 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0060741800000000002 2023-10-01T10:45:47.835 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.835 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.835 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:45:47.835 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.395652+0000", 2023-10-01T10:45:47.835 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0031828770000000002 2023-10-01T10:45:47.835 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.835 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.835 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:45:47.835 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.398835+0000", 2023-10-01T10:45:47.836 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.22626552999999999 2023-10-01T10:45:47.836 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.836 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.836 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:45:47.836 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.625101+0000", 2023-10-01T10:45:47.836 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.9758e-05 2023-10-01T10:45:47.836 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.836 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.836 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:45:47.836 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.625150+0000", 2023-10-01T10:45:47.836 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.0877999999999997e-05 2023-10-01T10:45:47.836 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.837 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.837 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:45:47.837 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.625191+0000", 2023-10-01T10:45:47.837 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.239007572 2023-10-01T10:45:47.837 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:47.837 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:45:47.837 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:47.837 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.837 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.837 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.35649.0:158071 9.a 9:59beab16:::benchmark_data_smithi078_113121_object46256:head [delete] snapc 0=[] ondisk+write+known_if_redirected e138)", 2023-10-01T10:45:47.837 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:43:39.419012+0000", 2023-10-01T10:45:47.837 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 128.30473096, 2023-10-01T10:45:47.838 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.200749969, 2023-10-01T10:45:47.838 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:45:47.838 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:45:47.838 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:45:47.838 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.35649", 2023-10-01T10:45:47.838 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:45:47.838 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 158071 2023-10-01T10:45:47.838 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.838 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:45:47.838 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.838 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:45:47.838 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.419012+0000", 2023-10-01T10:45:47.839 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:45:47.839 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.839 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.839 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:45:47.839 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.419012+0000", 2023-10-01T10:45:47.839 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-10-01T10:45:47.839 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.839 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.839 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:45:47.839 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.419010+0000", 2023-10-01T10:45:47.839 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.2259999999999994e-06 2023-10-01T10:45:47.839 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.840 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.840 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:45:47.840 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.419019+0000", 2023-10-01T10:45:47.840 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.5914999999999999e-05 2023-10-01T10:45:47.840 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.840 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.840 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:45:47.840 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.419035+0000", 2023-10-01T10:45:47.840 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.477e-06 2023-10-01T10:45:47.840 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.840 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.840 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:45:47.841 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.419041+0000", 2023-10-01T10:45:47.841 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.0578999999999997e-05 2023-10-01T10:45:47.841 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.841 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.841 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:45:47.841 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.419102+0000", 2023-10-01T10:45:47.841 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00012139899999999999 2023-10-01T10:45:47.841 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.841 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.841 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:45:47.841 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.419223+0000", 2023-10-01T10:45:47.841 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000121236 2023-10-01T10:45:47.842 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.842 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.842 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1,7", 2023-10-01T10:45:47.842 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.419345+0000", 2023-10-01T10:45:47.842 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0047990990000000002 2023-10-01T10:45:47.842 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.842 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.842 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:45:47.842 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.424144+0000", 2023-10-01T10:45:47.842 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00019378900000000001 2023-10-01T10:45:47.842 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.842 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.843 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:45:47.843 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.424337+0000", 2023-10-01T10:45:47.843 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.195223853 2023-10-01T10:45:47.843 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.843 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.843 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:45:47.843 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.619561+0000", 2023-10-01T10:45:47.843 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000106698 2023-10-01T10:45:47.843 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.843 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.843 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:45:47.843 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.619668+0000", 2023-10-01T10:45:47.844 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.3640999999999993e-05 2023-10-01T10:45:47.844 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.844 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.844 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:45:47.844 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.619762+0000", 2023-10-01T10:45:47.844 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.200749969 2023-10-01T10:45:47.844 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:47.844 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:45:47.844 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:47.844 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.844 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.844 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.35649.0:166281 9.f 9:fd064da1:::benchmark_data_smithi078_113121_object54466:head [delete] snapc 0=[] ondisk+write+known_if_redirected e139)", 2023-10-01T10:45:47.845 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:43:51.304959+0000", 2023-10-01T10:45:47.845 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 116.418783788, 2023-10-01T10:45:47.845 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.25730191099999999, 2023-10-01T10:45:47.845 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:45:47.845 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:45:47.845 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:45:47.845 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.35649", 2023-10-01T10:45:47.845 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:45:47.845 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 166281 2023-10-01T10:45:47.845 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.845 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:45:47.845 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.846 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:45:47.846 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.304959+0000", 2023-10-01T10:45:47.846 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:45:47.846 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.846 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.846 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:45:47.846 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.304959+0000", 2023-10-01T10:45:47.846 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-10-01T10:45:47.846 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.846 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.846 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:45:47.846 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.304958+0000", 2023-10-01T10:45:47.847 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.9640000000000002e-06 2023-10-01T10:45:47.847 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.847 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.847 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:45:47.847 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.304963+0000", 2023-10-01T10:45:47.847 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.8380000000000002e-06 2023-10-01T10:45:47.847 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.847 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.847 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:45:47.847 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.304971+0000", 2023-10-01T10:45:47.847 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.5754000000000001e-05 2023-10-01T10:45:47.848 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.848 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.848 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:45:47.848 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.304997+0000", 2023-10-01T10:45:47.848 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.1581999999999993e-05 2023-10-01T10:45:47.848 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.848 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.848 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:45:47.848 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.305068+0000", 2023-10-01T10:45:47.848 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000140288 2023-10-01T10:45:47.848 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.848 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.849 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:45:47.849 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.305209+0000", 2023-10-01T10:45:47.849 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00014844200000000001 2023-10-01T10:45:47.849 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.849 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.849 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1,3", 2023-10-01T10:45:47.849 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.305357+0000", 2023-10-01T10:45:47.849 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.022858590000000002 2023-10-01T10:45:47.849 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.849 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.849 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:45:47.849 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.328216+0000", 2023-10-01T10:45:47.850 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.6810000000000005e-05 2023-10-01T10:45:47.850 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.850 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.850 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:45:47.850 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.328292+0000", 2023-10-01T10:45:47.850 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.23366745899999999 2023-10-01T10:45:47.850 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.850 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.850 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:45:47.850 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.561960+0000", 2023-10-01T10:45:47.850 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00015784999999999999 2023-10-01T10:45:47.850 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.851 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.851 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:45:47.851 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.562118+0000", 2023-10-01T10:45:47.851 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00014305000000000001 2023-10-01T10:45:47.851 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.851 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.851 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:45:47.851 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.562261+0000", 2023-10-01T10:45:47.851 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.25730191099999999 2023-10-01T10:45:47.851 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:47.851 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:45:47.851 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:47.852 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.852 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.852 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.35649.0:166289 9.f 9:f0eeae49:::benchmark_data_smithi078_113121_object54474:head [delete] snapc 0=[] ondisk+write+known_if_redirected e139)", 2023-10-01T10:45:47.852 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:43:51.311529+0000", 2023-10-01T10:45:47.852 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 116.412213998, 2023-10-01T10:45:47.852 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.26435192099999999, 2023-10-01T10:45:47.852 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:45:47.852 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:45:47.852 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:45:47.852 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.35649", 2023-10-01T10:45:47.852 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:45:47.852 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 166289 2023-10-01T10:45:47.852 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.853 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:45:47.853 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.853 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:45:47.853 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.311529+0000", 2023-10-01T10:45:47.853 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:45:47.853 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.853 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.853 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:45:47.853 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.311529+0000", 2023-10-01T10:45:47.853 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-10-01T10:45:47.853 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.853 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.854 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:45:47.854 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.311527+0000", 2023-10-01T10:45:47.854 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.8909999999999998e-06 2023-10-01T10:45:47.854 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.854 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.854 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:45:47.854 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.311533+0000", 2023-10-01T10:45:47.854 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.6220999999999999e-05 2023-10-01T10:45:47.854 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.854 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.854 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:45:47.854 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.311550+0000", 2023-10-01T10:45:47.855 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.5630000000000003e-06 2023-10-01T10:45:47.855 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.855 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.855 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:45:47.855 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.311557+0000", 2023-10-01T10:45:47.855 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0022710740000000001 2023-10-01T10:45:47.855 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.855 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.855 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:45:47.855 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.313828+0000", 2023-10-01T10:45:47.855 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000110944 2023-10-01T10:45:47.855 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.856 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.856 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:45:47.856 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.313939+0000", 2023-10-01T10:45:47.856 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00013319499999999999 2023-10-01T10:45:47.856 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.856 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.856 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1,3", 2023-10-01T10:45:47.856 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.314072+0000", 2023-10-01T10:45:47.856 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.01415136 2023-10-01T10:45:47.856 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.856 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.856 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:45:47.857 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.328224+0000", 2023-10-01T10:45:47.857 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.001166952 2023-10-01T10:45:47.857 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.857 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.857 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:45:47.857 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.329391+0000", 2023-10-01T10:45:47.857 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.24640379800000001 2023-10-01T10:45:47.857 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.857 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.857 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:45:47.857 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.575795+0000", 2023-10-01T10:45:47.857 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.6008999999999999e-05 2023-10-01T10:45:47.858 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.858 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.858 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:45:47.858 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.575841+0000", 2023-10-01T10:45:47.858 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.9950000000000002e-05 2023-10-01T10:45:47.858 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.858 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.858 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:45:47.858 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.575881+0000", 2023-10-01T10:45:47.858 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.26435192099999999 2023-10-01T10:45:47.858 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:47.859 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:45:47.859 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:47.859 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.859 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.859 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.35649.0:166292 9.f 9:f0255b7f:::benchmark_data_smithi078_113121_object54477:head [delete] snapc 0=[] ondisk+write+known_if_redirected e139)", 2023-10-01T10:45:47.859 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:43:51.316157+0000", 2023-10-01T10:45:47.859 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 116.40758527200001, 2023-10-01T10:45:47.859 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.25986541400000002, 2023-10-01T10:45:47.859 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:45:47.859 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:45:47.859 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:45:47.859 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.35649", 2023-10-01T10:45:47.860 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:45:47.860 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 166292 2023-10-01T10:45:47.860 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.861 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:45:47.861 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.861 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:45:47.861 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.316157+0000", 2023-10-01T10:45:47.861 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:45:47.861 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.861 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.861 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:45:47.861 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.316157+0000", 2023-10-01T10:45:47.862 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-10-01T10:45:47.862 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.862 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.862 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:45:47.862 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.316156+0000", 2023-10-01T10:45:47.862 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.4810000000000003e-06 2023-10-01T10:45:47.862 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.862 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.862 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:45:47.862 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.316162+0000", 2023-10-01T10:45:47.862 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.6629999999999998e-05 2023-10-01T10:45:47.862 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.863 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.863 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:45:47.863 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.316179+0000", 2023-10-01T10:45:47.863 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.2309999999999999e-06 2023-10-01T10:45:47.863 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.863 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.863 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:45:47.863 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.316186+0000", 2023-10-01T10:45:47.863 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0022799230000000001 2023-10-01T10:45:47.863 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.864 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.864 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:45:47.864 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.318466+0000", 2023-10-01T10:45:47.864 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.4634000000000002e-05 2023-10-01T10:45:47.864 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.864 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.864 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:45:47.864 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.318560+0000", 2023-10-01T10:45:47.864 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.6883000000000001e-05 2023-10-01T10:45:47.864 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.864 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.865 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1,3", 2023-10-01T10:45:47.865 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.318657+0000", 2023-10-01T10:45:47.865 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0095721520000000004 2023-10-01T10:45:47.865 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.865 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.865 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:45:47.865 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.328229+0000", 2023-10-01T10:45:47.865 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.001190913 2023-10-01T10:45:47.865 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.865 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.865 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:45:47.865 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.329420+0000", 2023-10-01T10:45:47.866 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.246501003 2023-10-01T10:45:47.866 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.866 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.866 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:45:47.866 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.575921+0000", 2023-10-01T10:45:47.866 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.1399999999999997e-05 2023-10-01T10:45:47.866 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.866 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.866 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:45:47.866 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.575963+0000", 2023-10-01T10:45:47.866 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.0003999999999997e-05 2023-10-01T10:45:47.867 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.867 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.867 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:45:47.867 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.576023+0000", 2023-10-01T10:45:47.867 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.25986541400000002 2023-10-01T10:45:47.867 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:47.867 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:45:47.867 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:47.867 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.867 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.867 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.35649.0:166294 9.f 9:f325518b:::benchmark_data_smithi078_113121_object54479:head [delete] snapc 0=[] ondisk+write+known_if_redirected e139)", 2023-10-01T10:45:47.868 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:43:51.316515+0000", 2023-10-01T10:45:47.868 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 116.407227299, 2023-10-01T10:45:47.868 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.25958063999999997, 2023-10-01T10:45:47.868 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:45:47.868 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:45:47.868 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:45:47.868 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.35649", 2023-10-01T10:45:47.868 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:45:47.868 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 166294 2023-10-01T10:45:47.868 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.868 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:45:47.868 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.869 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:45:47.869 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.316515+0000", 2023-10-01T10:45:47.869 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:45:47.869 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.869 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.869 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:45:47.869 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.316515+0000", 2023-10-01T10:45:47.869 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-10-01T10:45:47.869 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.869 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.869 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:45:47.869 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.316514+0000", 2023-10-01T10:45:47.870 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.8499999999999999e-06 2023-10-01T10:45:47.870 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.870 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.870 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:45:47.870 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.316520+0000", 2023-10-01T10:45:47.870 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.7333000000000002e-05 2023-10-01T10:45:47.870 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.870 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.870 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:45:47.870 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.316538+0000", 2023-10-01T10:45:47.870 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.3350000000000003e-06 2023-10-01T10:45:47.870 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.871 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.871 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:45:47.871 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.316544+0000", 2023-10-01T10:45:47.871 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0063673200000000001 2023-10-01T10:45:47.871 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.871 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.871 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:45:47.871 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.322911+0000", 2023-10-01T10:45:47.871 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000111931 2023-10-01T10:45:47.871 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.871 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.871 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:45:47.872 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.323023+0000", 2023-10-01T10:45:47.872 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00013561399999999999 2023-10-01T10:45:47.872 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.872 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.872 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1,3", 2023-10-01T10:45:47.872 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.323159+0000", 2023-10-01T10:45:47.872 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0052132979999999999 2023-10-01T10:45:47.872 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.872 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.872 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:45:47.872 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.328372+0000", 2023-10-01T10:45:47.872 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0011226299999999999 2023-10-01T10:45:47.873 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.873 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.873 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:45:47.873 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.329495+0000", 2023-10-01T10:45:47.873 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.24655533399999999 2023-10-01T10:45:47.873 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.873 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.873 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:45:47.873 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.576050+0000", 2023-10-01T10:45:47.873 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.4501000000000001e-05 2023-10-01T10:45:47.873 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.873 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.874 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:45:47.874 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.576074+0000", 2023-10-01T10:45:47.874 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.1484e-05 2023-10-01T10:45:47.874 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.874 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.874 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:45:47.874 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.576096+0000", 2023-10-01T10:45:47.874 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.25958063999999997 2023-10-01T10:45:47.874 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:47.874 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:45:47.875 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:47.875 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.875 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.875 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.35649.0:166304 9.f 9:f52a2f72:::benchmark_data_smithi078_113121_object54489:head [delete] snapc 0=[] ondisk+write+known_if_redirected e139)", 2023-10-01T10:45:47.875 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:43:51.343789+0000", 2023-10-01T10:45:47.875 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 116.379953598, 2023-10-01T10:45:47.875 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.24844228900000001, 2023-10-01T10:45:47.875 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:45:47.875 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:45:47.875 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:45:47.875 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.35649", 2023-10-01T10:45:47.875 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:45:47.876 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 166304 2023-10-01T10:45:47.876 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.876 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:45:47.876 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.876 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:45:47.876 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.343789+0000", 2023-10-01T10:45:47.876 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:45:47.876 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.876 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.876 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:45:47.876 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.343789+0000", 2023-10-01T10:45:47.877 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-10-01T10:45:47.877 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.877 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.877 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:45:47.877 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.343788+0000", 2023-10-01T10:45:47.877 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.1620000000000001e-06 2023-10-01T10:45:47.877 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.877 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.877 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:45:47.877 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.343792+0000", 2023-10-01T10:45:47.877 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.5560000000000008e-06 2023-10-01T10:45:47.877 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.877 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.878 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:45:47.878 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.343801+0000", 2023-10-01T10:45:47.878 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.4750000000000004e-06 2023-10-01T10:45:47.878 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.878 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.878 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:45:47.878 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.343805+0000", 2023-10-01T10:45:47.878 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.1152000000000001e-05 2023-10-01T10:45:47.878 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.878 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.878 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:45:47.879 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.343856+0000", 2023-10-01T10:45:47.879 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000121626 2023-10-01T10:45:47.879 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.879 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.879 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:45:47.879 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.343978+0000", 2023-10-01T10:45:47.879 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000125766 2023-10-01T10:45:47.879 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.879 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.879 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1,3", 2023-10-01T10:45:47.879 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.344104+0000", 2023-10-01T10:45:47.879 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0049757450000000002 2023-10-01T10:45:47.880 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.880 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.880 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:45:47.880 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.349079+0000", 2023-10-01T10:45:47.880 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000178254 2023-10-01T10:45:47.880 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.880 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.880 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:45:47.880 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.349258+0000", 2023-10-01T10:45:47.880 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.24281037499999999 2023-10-01T10:45:47.880 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.880 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.881 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:45:47.881 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.592068+0000", 2023-10-01T10:45:47.881 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00011524 2023-10-01T10:45:47.881 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.881 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.881 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:45:47.881 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.592183+0000", 2023-10-01T10:45:47.881 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.8003000000000001e-05 2023-10-01T10:45:47.881 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:45:47.881 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:45:47.881 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:45:47.882 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.592231+0000", 2023-10-01T10:45:47.882 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.24844228900000001 2023-10-01T10:45:47.882 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:47.882 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:45:47.882 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:47.882 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:45:47.882 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:45:47.882 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-10-01T10:45:47.882 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:47.883 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:47 smithi173 bash[58816]: cluster 2023-10-01T10:45:45.823123+0000 mgr.y (mgr.25072) 841 : cluster 0 pgmap v766: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:45:47.883 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:47 smithi173 bash[58816]: audit 2023-10-01T10:45:47.209551+0000 mon.c (mon.2) 149 : audit 0 from='client.? 172.21.15.78:0/2289777786' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:45:47.883 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:47 smithi173 bash[58816]: audit 2023-10-01T10:45:47.425881+0000 mon.a (mon.0) 1068 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:45:47.884 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:47 smithi173 bash[58816]: audit 2023-10-01T10:45:47.427454+0000 mon.a (mon.0) 1069 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:45:47.912 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:47.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:47.915 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:45:48.016 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:48.042 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:48.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:48.045 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:45:48.146 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:48.172 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:48.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:48.175 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:45:48.276 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:48.308 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:48.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:48.311 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:45:48.411 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:48.442 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:48.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:48.444 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:48.477 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-01T10:45:48.546 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:48.593 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:48.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:48.595 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:48.696 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:48.716 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:48.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:48.719 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:45:48.819 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:48.854 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:48.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:48.858 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:45:48.959 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:48.989 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:48.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:48.992 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:49.093 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:49.123 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:49.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:49.127 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:49.227 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:49.249 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:49.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:49.251 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:45:49.352 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:49.384 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:49.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:49.387 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:49.488 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:49.516 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:49.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:49.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:45:49.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:49 smithi078 bash[75299]: cluster 2023-10-01T10:45:47.823893+0000 mgr.y (mgr.25072) 842 : cluster 0 pgmap v767: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:45:49.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:49 smithi078 bash[78274]: cluster 2023-10-01T10:45:47.823893+0000 mgr.y (mgr.25072) 842 : cluster 0 pgmap v767: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:45:49.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:49 smithi173 bash[58816]: cluster 2023-10-01T10:45:47.823893+0000 mgr.y (mgr.25072) 842 : cluster 0 pgmap v767: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:45:49.620 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:49.643 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:49.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:49.646 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:45:49.747 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:49.774 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:49.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:49.777 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:45:49.878 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:49.907 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:49.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:49.910 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:50.010 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:50.038 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:50.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:50.041 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:50.142 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:50.171 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:50.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:50.173 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:45:50.275 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:50.310 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:50.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:50.313 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:50.413 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:50.435 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:50.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:50.438 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:45:50.538 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:50.550 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:45:50 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:45:50] "GET /metrics HTTP/1.1" 200 33891 "" "Prometheus/2.33.4" 2023-10-01T10:45:50.565 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:50.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:50.567 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:45:50.668 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:50.699 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:50.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:50.702 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:50.803 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:50.838 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:50.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:50.841 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:45:50.942 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:50.968 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:50.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:50.971 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:45:51.071 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:51.106 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:51.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:51.110 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:51.211 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:51.246 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:51.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:51.250 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:51.350 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:51.384 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:51.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:51.619 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:51.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:51 smithi173 bash[58816]: cluster 2023-10-01T10:45:49.824872+0000 mgr.y (mgr.25072) 843 : cluster 0 pgmap v768: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:45:51.720 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:51.761 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:51.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:51.776 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:51.780 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:51 smithi078 bash[75299]: cluster 2023-10-01T10:45:49.824872+0000 mgr.y (mgr.25072) 843 : cluster 0 pgmap v768: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:45:51.780 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:51 smithi078 bash[78274]: cluster 2023-10-01T10:45:49.824872+0000 mgr.y (mgr.25072) 843 : cluster 0 pgmap v768: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:45:51.877 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:51.903 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:51.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:51.906 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:45:52.006 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:52.040 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:52.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:52.043 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:45:52.144 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:52.173 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:52.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:52.175 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:45:52.210 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T10:45:52.276 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:52.306 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:52.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:52.309 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:52.410 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:52.433 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:52.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:52.436 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:45:52.536 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:52.565 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:52.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:52.569 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:45:52.670 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:52.695 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:52.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:52.697 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:45:52.798 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:52.826 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:52.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:52.828 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:45:52.929 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:52.954 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:52.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:52.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:45:53.057 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:53.083 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:53.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:53.085 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:45:53.186 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:53.219 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:53.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:53.222 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:45:53.304 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T10:45:53.322 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:53.347 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:53.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:53.351 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:45:53.451 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:53.476 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:53.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:53.479 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:53.580 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:53.591 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:53 smithi078 bash[78274]: cluster 2023-10-01T10:45:51.825683+0000 mgr.y (mgr.25072) 844 : cluster 0 pgmap v769: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:45:53.592 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:53 smithi078 bash[75299]: cluster 2023-10-01T10:45:51.825683+0000 mgr.y (mgr.25072) 844 : cluster 0 pgmap v769: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:45:53.606 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:53.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:53.608 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:45:53.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:53 smithi173 bash[58816]: cluster 2023-10-01T10:45:51.825683+0000 mgr.y (mgr.25072) 844 : cluster 0 pgmap v769: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:45:53.709 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:53.737 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:53.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:53.740 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:45:53.840 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:53.875 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:53.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:53.878 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:53.979 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:54.000 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:54.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:54.003 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:45:54.104 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:54.135 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:54.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:54.138 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:54.239 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:54.266 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:54.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:54.269 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:54.370 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:54.399 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:54.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:54.402 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:45:54.503 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:54.527 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:54.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:54.530 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:45:54.631 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:54.661 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:54.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:54.664 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:45:54.765 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:54.791 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:54.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:54.793 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:45:54.894 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:54.920 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:54.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:54.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:45:55.024 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:55.048 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:55.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:55.052 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:55.152 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:55.185 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:55.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:55.189 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:55.289 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:55.318 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:55.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:55.321 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:45:55.422 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:55.452 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:55.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:55.455 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:55.555 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:55.582 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:55.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:55.585 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:55.685 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:55.717 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:55.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:55.718 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:45:55.819 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:55.850 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:55.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:55.854 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:55.954 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:55.980 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:55.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:55.983 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:45:56.041 INFO:teuthology.orchestra.run.smithi078.stderr:pool 'unique_pool_5' created 2023-10-01T10:45:56.084 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:56.097 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:55 smithi173 bash[58816]: cluster 2023-10-01T10:45:53.826435+0000 mgr.y (mgr.25072) 845 : cluster 0 pgmap v770: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:45:56.115 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:56.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:56.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:45:56.219 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:56.253 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:56.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:56.255 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:56.328 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:55 smithi078 bash[75299]: cluster 2023-10-01T10:45:53.826435+0000 mgr.y (mgr.25072) 845 : cluster 0 pgmap v770: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:45:56.328 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:55 smithi078 bash[78274]: cluster 2023-10-01T10:45:53.826435+0000 mgr.y (mgr.25072) 845 : cluster 0 pgmap v770: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:45:56.355 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:56.388 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:56.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:56.391 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:56.492 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:56.520 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:56.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:56.522 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:45:56.613 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-10-01T10:45:56.614 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [], 2023-10-01T10:45:56.614 INFO:teuthology.orchestra.run.smithi173.stdout: "num_ops": 0 2023-10-01T10:45:56.614 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-10-01T10:45:56.622 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:56.645 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:56.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:56.647 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:45:56.748 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:56.775 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:56.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:56.778 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:56.878 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:56.910 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:56.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:56.913 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:57.014 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:57.025 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:56 smithi173 bash[58816]: audit 2023-10-01T10:45:55.533715+0000 mon.c (mon.2) 150 : audit 1 from='client.? 172.21.15.78:0/1083877963' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-10-01T10:45:57.025 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:56 smithi173 bash[58816]: audit 2023-10-01T10:45:55.534064+0000 mon.a (mon.0) 1070 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-10-01T10:45:57.025 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:56 smithi173 bash[58816]: cluster 2023-10-01T10:45:55.826973+0000 mgr.y (mgr.25072) 846 : cluster 0 pgmap v771: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:45:57.025 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:56 smithi173 bash[58816]: cluster 2023-10-01T10:45:56.033782+0000 osd.6 (osd.6) 59 : cluster 3 failed to encode map e151 with expected crc 2023-10-01T10:45:57.025 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:56 smithi173 bash[58816]: cluster 2023-10-01T10:45:56.033851+0000 osd.7 (osd.7) 100 : cluster 3 failed to encode map e151 with expected crc 2023-10-01T10:45:57.025 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:56 smithi173 bash[58816]: audit 2023-10-01T10:45:56.035137+0000 mon.a (mon.0) 1071 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-10-01T10:45:57.025 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:56 smithi173 bash[58816]: cluster 2023-10-01T10:45:56.035206+0000 mon.a (mon.0) 1072 : cluster 0 osdmap e151: 8 total, 8 up, 8 in 2023-10-01T10:45:57.025 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:56 smithi173 bash[58816]: cluster 2023-10-01T10:45:56.036421+0000 osd.5 (osd.5) 80 : cluster 3 failed to encode map e151 with expected crc 2023-10-01T10:45:57.025 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:56 smithi173 bash[58816]: audit 2023-10-01T10:45:56.211991+0000 mon.a (mon.0) 1073 : audit 1 from='client.? 172.21.15.78:0/2956496484' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:45:57.048 INFO:teuthology.orchestra.run.smithi078.stderr:nodeep-scrub is set 2023-10-01T10:45:57.050 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:57.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:57.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:45:57.154 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:57.164 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:56 smithi078 bash[78274]: audit 2023-10-01T10:45:55.533715+0000 mon.c (mon.2) 150 : audit 1 from='client.? 172.21.15.78:0/1083877963' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-10-01T10:45:57.164 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:56 smithi078 bash[78274]: audit 2023-10-01T10:45:55.534064+0000 mon.a (mon.0) 1070 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-10-01T10:45:57.164 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:56 smithi078 bash[78274]: cluster 2023-10-01T10:45:55.826973+0000 mgr.y (mgr.25072) 846 : cluster 0 pgmap v771: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:45:57.164 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:56 smithi078 bash[78274]: cluster 2023-10-01T10:45:56.033782+0000 osd.6 (osd.6) 59 : cluster 3 failed to encode map e151 with expected crc 2023-10-01T10:45:57.165 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:56 smithi078 bash[78274]: cluster 2023-10-01T10:45:56.033851+0000 osd.7 (osd.7) 100 : cluster 3 failed to encode map e151 with expected crc 2023-10-01T10:45:57.165 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:56 smithi078 bash[78274]: audit 2023-10-01T10:45:56.035137+0000 mon.a (mon.0) 1071 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-10-01T10:45:57.165 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:56 smithi078 bash[78274]: cluster 2023-10-01T10:45:56.035206+0000 mon.a (mon.0) 1072 : cluster 0 osdmap e151: 8 total, 8 up, 8 in 2023-10-01T10:45:57.165 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:56 smithi078 bash[78274]: cluster 2023-10-01T10:45:56.036421+0000 osd.5 (osd.5) 80 : cluster 3 failed to encode map e151 with expected crc 2023-10-01T10:45:57.165 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:56 smithi078 bash[78274]: audit 2023-10-01T10:45:56.211991+0000 mon.a (mon.0) 1073 : audit 1 from='client.? 172.21.15.78:0/2956496484' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:45:57.165 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:56 smithi078 bash[75299]: audit 2023-10-01T10:45:55.533715+0000 mon.c (mon.2) 150 : audit 1 from='client.? 172.21.15.78:0/1083877963' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-10-01T10:45:57.165 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:56 smithi078 bash[75299]: audit 2023-10-01T10:45:55.534064+0000 mon.a (mon.0) 1070 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-10-01T10:45:57.165 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:56 smithi078 bash[75299]: cluster 2023-10-01T10:45:55.826973+0000 mgr.y (mgr.25072) 846 : cluster 0 pgmap v771: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:45:57.166 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:56 smithi078 bash[75299]: cluster 2023-10-01T10:45:56.033782+0000 osd.6 (osd.6) 59 : cluster 3 failed to encode map e151 with expected crc 2023-10-01T10:45:57.166 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:56 smithi078 bash[75299]: cluster 2023-10-01T10:45:56.033851+0000 osd.7 (osd.7) 100 : cluster 3 failed to encode map e151 with expected crc 2023-10-01T10:45:57.166 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:56 smithi078 bash[75299]: audit 2023-10-01T10:45:56.035137+0000 mon.a (mon.0) 1071 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-10-01T10:45:57.166 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:56 smithi078 bash[75299]: cluster 2023-10-01T10:45:56.035206+0000 mon.a (mon.0) 1072 : cluster 0 osdmap e151: 8 total, 8 up, 8 in 2023-10-01T10:45:57.166 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:56 smithi078 bash[75299]: cluster 2023-10-01T10:45:56.036421+0000 osd.5 (osd.5) 80 : cluster 3 failed to encode map e151 with expected crc 2023-10-01T10:45:57.166 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:56 smithi078 bash[75299]: audit 2023-10-01T10:45:56.211991+0000 mon.a (mon.0) 1073 : audit 1 from='client.? 172.21.15.78:0/2956496484' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:45:57.180 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:57.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:57.182 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:45:57.283 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:57.314 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:57.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:57.317 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:45:57.345 INFO:teuthology.orchestra.run.smithi078.stdout: 2023-10-01T10:45:57.347 INFO:teuthology.orchestra.run.smithi078.stderr:dumped all 2023-10-01T10:45:57.418 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:57.422 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-01T10:45:57.449 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:57.451 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_5 rados --yes-i-really-mean-it || true 2023-10-01T10:45:57.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:57.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:57.553 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:57.580 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:57.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:57.583 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:45:57.684 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:57.714 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:45:57.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:57.717 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:45:57.818 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:57.844 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:57.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:57.847 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:45:57.947 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:57.981 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:57.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:57.985 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:58.086 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:58.119 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:58.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:58.122 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:58.223 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:58.233 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:57 smithi078 bash[78274]: cluster 2023-10-01T10:45:57.026693+0000 mon.a (mon.0) 1074 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:45:58.233 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:57 smithi078 bash[78274]: cluster 2023-10-01T10:45:57.035328+0000 osd.7 (osd.7) 101 : cluster 3 failed to encode map e152 with expected crc 2023-10-01T10:45:58.233 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:57 smithi078 bash[78274]: audit 2023-10-01T10:45:57.036154+0000 mon.a (mon.0) 1075 : audit 1 from='client.? 172.21.15.78:0/2956496484' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T10:45:58.233 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:57 smithi078 bash[78274]: cluster 2023-10-01T10:45:57.036253+0000 mon.a (mon.0) 1076 : cluster 0 osdmap e152: 8 total, 8 up, 8 in 2023-10-01T10:45:58.234 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:57 smithi078 bash[78274]: cluster 2023-10-01T10:45:57.037734+0000 osd.6 (osd.6) 60 : cluster 3 failed to encode map e152 with expected crc 2023-10-01T10:45:58.234 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:57 smithi078 bash[78274]: cluster 2023-10-01T10:45:57.037753+0000 osd.5 (osd.5) 81 : cluster 3 failed to encode map e152 with expected crc 2023-10-01T10:45:58.234 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:57 smithi078 bash[78274]: cluster 2023-10-01T10:45:57.041842+0000 osd.7 (osd.7) 102 : cluster 3 failed to encode map e152 with expected crc 2023-10-01T10:45:58.234 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:57 smithi078 bash[78274]: audit 2023-10-01T10:45:57.330383+0000 mgr.y (mgr.25072) 847 : audit 0 from='client.35838 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:45:58.234 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:57 smithi078 bash[75299]: cluster 2023-10-01T10:45:57.026693+0000 mon.a (mon.0) 1074 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:45:58.234 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:57 smithi078 bash[75299]: cluster 2023-10-01T10:45:57.035328+0000 osd.7 (osd.7) 101 : cluster 3 failed to encode map e152 with expected crc 2023-10-01T10:45:58.234 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:57 smithi078 bash[75299]: audit 2023-10-01T10:45:57.036154+0000 mon.a (mon.0) 1075 : audit 1 from='client.? 172.21.15.78:0/2956496484' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T10:45:58.234 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:57 smithi078 bash[75299]: cluster 2023-10-01T10:45:57.036253+0000 mon.a (mon.0) 1076 : cluster 0 osdmap e152: 8 total, 8 up, 8 in 2023-10-01T10:45:58.234 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:57 smithi078 bash[75299]: cluster 2023-10-01T10:45:57.037734+0000 osd.6 (osd.6) 60 : cluster 3 failed to encode map e152 with expected crc 2023-10-01T10:45:58.234 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:57 smithi078 bash[75299]: cluster 2023-10-01T10:45:57.037753+0000 osd.5 (osd.5) 81 : cluster 3 failed to encode map e152 with expected crc 2023-10-01T10:45:58.235 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:57 smithi078 bash[75299]: cluster 2023-10-01T10:45:57.041842+0000 osd.7 (osd.7) 102 : cluster 3 failed to encode map e152 with expected crc 2023-10-01T10:45:58.235 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:57 smithi078 bash[75299]: audit 2023-10-01T10:45:57.330383+0000 mgr.y (mgr.25072) 847 : audit 0 from='client.35838 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:45:58.246 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:58.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:58.248 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:45:58.349 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:58.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:57 smithi173 bash[58816]: cluster 2023-10-01T10:45:57.026693+0000 mon.a (mon.0) 1074 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:45:58.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:57 smithi173 bash[58816]: cluster 2023-10-01T10:45:57.035328+0000 osd.7 (osd.7) 101 : cluster 3 failed to encode map e152 with expected crc 2023-10-01T10:45:58.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:57 smithi173 bash[58816]: audit 2023-10-01T10:45:57.036154+0000 mon.a (mon.0) 1075 : audit 1 from='client.? 172.21.15.78:0/2956496484' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T10:45:58.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:57 smithi173 bash[58816]: cluster 2023-10-01T10:45:57.036253+0000 mon.a (mon.0) 1076 : cluster 0 osdmap e152: 8 total, 8 up, 8 in 2023-10-01T10:45:58.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:57 smithi173 bash[58816]: cluster 2023-10-01T10:45:57.037734+0000 osd.6 (osd.6) 60 : cluster 3 failed to encode map e152 with expected crc 2023-10-01T10:45:58.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:57 smithi173 bash[58816]: cluster 2023-10-01T10:45:57.037753+0000 osd.5 (osd.5) 81 : cluster 3 failed to encode map e152 with expected crc 2023-10-01T10:45:58.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:57 smithi173 bash[58816]: cluster 2023-10-01T10:45:57.041842+0000 osd.7 (osd.7) 102 : cluster 3 failed to encode map e152 with expected crc 2023-10-01T10:45:58.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:57 smithi173 bash[58816]: audit 2023-10-01T10:45:57.330383+0000 mgr.y (mgr.25072) 847 : audit 0 from='client.35838 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:45:58.374 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:45:58.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:58.376 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:45:58.477 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:58.507 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:45:58.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:58.510 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:45:58.611 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:58.641 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:58.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:58.644 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:45:58.745 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:58.777 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:58.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:58.780 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:45:58.880 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:58.909 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:45:58.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:58.912 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:45:59.012 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:59.038 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:59.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:59.042 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:45:59.111 INFO:teuthology.orchestra.run.smithi078.stdout:{"pg_ready":true,"pg_map":{"version":773,"stamp":"2023-10-01T10:45:57.031161+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595872,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":261,"num_write_kb":2446,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":195,"ondisk_log_size":195,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":3170796,"kb_used_data":13548,"kb_used_omap":30,"kb_used_meta":3157153,"kb_avail":746790420,"statfs":{"total":767960285184,"available":764713390080,"internally_reserved":0,"allocated":13873152,"data_stored":9455075,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31502,"internal_metadata":3232924914},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.207762"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":139,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-01T10:45:40.523129+0000","last_change":"2023-10-01T10:38:16.032258+0000","last_active":"2023-10-01T10:45:40.523129+0000","last_peered":"2023-10-01T10:45:40.523129+0000","last_clean":"2023-10-01T10:45:40.523129+0000","last_became_active":"2023-10-01T10:38:16.031633+0000","last_became_peered":"2023-10-01T10:38:16.031633+0000","last_unstale":"2023-10-01T10:45:40.523129+0000","last_undegraded":"2023-10-01T10:45:40.523129+0000","last_fullsized":"2023-10-01T10:45:40.523129+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:17:02.446430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":128,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-01T10:45:40.521824+0000","last_change":"2023-10-01T10:38:16.025881+0000","last_active":"2023-10-01T10:45:40.521824+0000","last_peered":"2023-10-01T10:45:40.521824+0000","last_clean":"2023-10-01T10:45:40.521824+0000","last_became_active":"2023-10-01T10:38:16.025518+0000","last_became_peered":"2023-10-01T10:38:16.025518+0000","last_unstale":"2023-10-01T10:45:40.521824+0000","last_undegraded":"2023-10-01T10:45:40.521824+0000","last_fullsized":"2023-10-01T10:45:40.521824+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:48:37.826604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":105,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-01T10:45:37.503943+0000","last_change":"2023-10-01T10:38:16.025075+0000","last_active":"2023-10-01T10:45:37.503943+0000","last_peered":"2023-10-01T10:45:37.503943+0000","last_clean":"2023-10-01T10:45:37.503943+0000","last_became_active":"2023-10-01T10:38:16.024735+0000","last_became_peered":"2023-10-01T10:38:16.024735+0000","last_unstale":"2023-10-01T10:45:37.503943+0000","last_undegraded":"2023-10-01T10:45:37.503943+0000","last_fullsized":"2023-10-01T10:45:37.503943+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:38:01.450741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":109,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-01T10:45:37.571264+0000","last_change":"2023-10-01T10:37:42.772008+0000","last_active":"2023-10-01T10:45:37.571264+0000","last_peered":"2023-10-01T10:45:37.571264+0000","last_clean":"2023-10-01T10:45:37.571264+0000","last_became_active":"2023-10-01T10:37:42.771244+0000","last_became_peered":"2023-10-01T10:37:42.771244+0000","last_unstale":"2023-10-01T10:45:37.571264+0000","last_undegraded":"2023-10-01T10:45:37.571264+0000","last_fullsized":"2023-10-01T10:45:37.571264+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:38:55.180790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":129,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-01T10:45:40.520281+0000","last_change":"2023-10-01T10:37:42.766600+0000","last_active":"2023-10-01T10:45:40.520281+0000","last_peered":"2023-10-01T10:45:40.520281+0000","last_clean":"2023-10-01T10:45:40.520281+0000","last_became_active":"2023-10-01T10:37:42.766051+0000","last_became_peered":"2023-10-01T10:37:42.766051+0000","last_unstale":"2023-10-01T10:45:40.520281+0000","last_undegraded":"2023-10-01T10:45:40.520281+0000","last_fullsized":"2023-10-01T10:45:40.520281+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:51:17.707963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":115,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-01T10:45:37.568528+0000","last_change":"2023-10-01T10:32:31.575109+0000","last_active":"2023-10-01T10:45:37.568528+0000","last_peered":"2023-10-01T10:45:37.568528+0000","last_clean":"2023-10-01T10:45:37.568528+0000","last_became_active":"2023-10-01T10:32:31.574729+0000","last_became_peered":"2023-10-01T10:32:31.574729+0000","last_unstale":"2023-10-01T10:45:37.568528+0000","last_undegraded":"2023-10-01T10:45:37.568528+0000","last_fullsized":"2023-10-01T10:45:37.568528+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":107,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-01T10:45:37.572696+0000","last_change":"2023-10-01T10:37:42.766667+0000","last_active":"2023-10-01T10:45:37.572696+0000","last_peered":"2023-10-01T10:45:37.572696+0000","last_clean":"2023-10-01T10:45:37.572696+0000","last_became_active":"2023-10-01T10:37:42.766369+0000","last_became_peered":"2023-10-01T10:37:42.766369+0000","last_unstale":"2023-10-01T10:45:37.572696+0000","last_undegraded":"2023-10-01T10:45:37.572696+0000","last_fullsized":"2023-10-01T10:45:37.572696+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:02.708788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":113,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-01T10:45:40.520255+0000","last_change":"2023-10-01T10:37:42.765925+0000","last_active":"2023-10-01T10:45:40.520255+0000","last_peered":"2023-10-01T10:45:40.520255+0000","last_clean":"2023-10-01T10:45:40.520255+0000","last_became_active":"2023-10-01T10:37:42.765218+0000","last_became_peered":"2023-10-01T10:37:42.765218+0000","last_unstale":"2023-10-01T10:45:40.520255+0000","last_undegraded":"2023-10-01T10:45:40.520255+0000","last_fullsized":"2023-10-01T10:45:40.520255+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T22:10:34.338031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":117,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-01T10:45:37.579632+0000","last_change":"2023-10-01T10:32:31.579805+0000","last_active":"2023-10-01T10:45:37.579632+0000","last_peered":"2023-10-01T10:45:37.579632+0000","last_clean":"2023-10-01T10:45:37.579632+0000","last_became_active":"2023-10-01T10:32:31.579423+0000","last_became_peered":"2023-10-01T10:32:31.579423+0000","last_unstale":"2023-10-01T10:45:37.579632+0000","last_undegraded":"2023-10-01T10:45:37.579632+0000","last_fullsized":"2023-10-01T10:45:37.579632+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":97,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-01T10:45:40.520568+0000","last_change":"2023-10-01T10:37:42.772531+0000","last_active":"2023-10-01T10:45:40.520568+0000","last_peered":"2023-10-01T10:45:40.520568+0000","last_clean":"2023-10-01T10:45:40.520568+0000","last_became_active":"2023-10-01T10:37:42.772216+0000","last_became_peered":"2023-10-01T10:37:42.772216+0000","last_unstale":"2023-10-01T10:45:40.520568+0000","last_undegraded":"2023-10-01T10:45:40.520568+0000","last_fullsized":"2023-10-01T10:45:40.520568+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:01:53.373628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":135,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-01T10:45:37.511386+0000","last_change":"2023-10-01T10:38:16.027085+0000","last_active":"2023-10-01T10:45:37.511386+0000","last_peered":"2023-10-01T10:45:37.511386+0000","last_clean":"2023-10-01T10:45:37.511386+0000","last_became_active":"2023-10-01T10:38:16.026555+0000","last_became_peered":"2023-10-01T10:38:16.026555+0000","last_unstale":"2023-10-01T10:45:37.511386+0000","last_undegraded":"2023-10-01T10:45:37.511386+0000","last_fullsized":"2023-10-01T10:45:37.511386+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":90,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-01T10:45:37.508264+0000","last_change":"2023-10-01T10:37:05.018755+0000","last_active":"2023-10-01T10:45:37.508264+0000","last_peered":"2023-10-01T10:45:37.508264+0000","last_clean":"2023-10-01T10:45:37.508264+0000","last_became_active":"2023-10-01T10:37:05.018446+0000","last_became_peered":"2023-10-01T10:37:05.018446+0000","last_unstale":"2023-10-01T10:45:37.508264+0000","last_undegraded":"2023-10-01T10:45:37.508264+0000","last_fullsized":"2023-10-01T10:45:37.508264+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:13:53.740503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":133,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-01T10:45:37.568059+0000","last_change":"2023-10-01T10:38:16.030826+0000","last_active":"2023-10-01T10:45:37.568059+0000","last_peered":"2023-10-01T10:45:37.568059+0000","last_clean":"2023-10-01T10:45:37.568059+0000","last_became_active":"2023-10-01T10:38:16.030277+0000","last_became_peered":"2023-10-01T10:38:16.030277+0000","last_unstale":"2023-10-01T10:45:37.568059+0000","last_undegraded":"2023-10-01T10:45:37.568059+0000","last_fullsized":"2023-10-01T10:45:37.568059+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:31:43.040424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":107,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-01T10:45:37.566872+0000","last_change":"2023-10-01T10:38:16.034741+0000","last_active":"2023-10-01T10:45:37.566872+0000","last_peered":"2023-10-01T10:45:37.566872+0000","last_clean":"2023-10-01T10:45:37.566872+0000","last_became_active":"2023-10-01T10:38:16.034407+0000","last_became_peered":"2023-10-01T10:38:16.034407+0000","last_unstale":"2023-10-01T10:45:37.566872+0000","last_undegraded":"2023-10-01T10:45:37.566872+0000","last_fullsized":"2023-10-01T10:45:37.566872+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:47:29.911477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":134,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-01T10:45:37.509607+0000","last_change":"2023-10-01T10:38:16.026777+0000","last_active":"2023-10-01T10:45:37.509607+0000","last_peered":"2023-10-01T10:45:37.509607+0000","last_clean":"2023-10-01T10:45:37.509607+0000","last_became_active":"2023-10-01T10:38:16.026371+0000","last_became_peered":"2023-10-01T10:38:16.026371+0000","last_unstale":"2023-10-01T10:45:37.509607+0000","last_undegraded":"2023-10-01T10:45:37.509607+0000","last_fullsized":"2023-10-01T10:45:37.509607+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":136,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-01T10:45:40.520643+0000","last_change":"2023-10-01T10:37:42.772319+0000","last_active":"2023-10-01T10:45:40.520643+0000","last_peered":"2023-10-01T10:45:40.520643+0000","last_clean":"2023-10-01T10:45:40.520643+0000","last_became_active":"2023-10-01T10:37:42.771977+0000","last_became_peered":"2023-10-01T10:37:42.771977+0000","last_unstale":"2023-10-01T10:45:40.520643+0000","last_undegraded":"2023-10-01T10:45:40.520643+0000","last_fullsized":"2023-10-01T10:45:40.520643+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:45:12.783214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":107,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-01T10:45:37.566892+0000","last_change":"2023-10-01T10:37:42.767706+0000","last_active":"2023-10-01T10:45:37.566892+0000","last_peered":"2023-10-01T10:45:37.566892+0000","last_clean":"2023-10-01T10:45:37.566892+0000","last_became_active":"2023-10-01T10:37:42.767211+0000","last_became_peered":"2023-10-01T10:37:42.767211+0000","last_unstale":"2023-10-01T10:45:37.566892+0000","last_undegraded":"2023-10-01T10:45:37.566892+0000","last_fullsized":"2023-10-01T10:45:37.566892+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:37:20.357407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":116,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-01T10:45:37.581379+0000","last_change":"2023-10-01T10:37:42.768642+0000","last_active":"2023-10-01T10:45:37.581379+0000","last_peered":"2023-10-01T10:45:37.581379+0000","last_clean":"2023-10-01T10:45:37.581379+0000","last_became_active":"2023-10-01T10:37:42.768239+0000","last_became_peered":"2023-10-01T10:37:42.768239+0000","last_unstale":"2023-10-01T10:45:37.581379+0000","last_undegraded":"2023-10-01T10:45:37.581379+0000","last_fullsized":"2023-10-01T10:45:37.581379+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":117,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-01T10:45:37.509989+0000","last_change":"2023-10-01T10:38:16.028563+0000","last_active":"2023-10-01T10:45:37.509989+0000","last_peered":"2023-10-01T10:45:37.509989+0000","last_clean":"2023-10-01T10:45:37.509989+0000","last_became_active":"2023-10-01T10:38:16.028162+0000","last_became_peered":"2023-10-01T10:38:16.028162+0000","last_unstale":"2023-10-01T10:45:37.509989+0000","last_undegraded":"2023-10-01T10:45:37.509989+0000","last_fullsized":"2023-10-01T10:45:37.509989+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":108,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-01T10:45:37.571397+0000","last_change":"2023-10-01T10:37:42.772247+0000","last_active":"2023-10-01T10:45:37.571397+0000","last_peered":"2023-10-01T10:45:37.571397+0000","last_clean":"2023-10-01T10:45:37.571397+0000","last_became_active":"2023-10-01T10:37:42.771697+0000","last_became_peered":"2023-10-01T10:37:42.771697+0000","last_unstale":"2023-10-01T10:45:37.571397+0000","last_undegraded":"2023-10-01T10:45:37.571397+0000","last_fullsized":"2023-10-01T10:45:37.571397+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:23:15.018248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":124,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-01T10:45:37.568335+0000","last_change":"2023-10-01T10:28:20.211798+0000","last_active":"2023-10-01T10:45:37.568335+0000","last_peered":"2023-10-01T10:45:37.568335+0000","last_clean":"2023-10-01T10:45:37.568335+0000","last_became_active":"2023-10-01T10:28:20.210791+0000","last_became_peered":"2023-10-01T10:28:20.210791+0000","last_unstale":"2023-10-01T10:45:37.568335+0000","last_undegraded":"2023-10-01T10:45:37.568335+0000","last_fullsized":"2023-10-01T10:45:37.568335+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":113,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-01T10:45:40.521886+0000","last_change":"2023-10-01T10:37:42.773127+0000","last_active":"2023-10-01T10:45:40.521886+0000","last_peered":"2023-10-01T10:45:40.521886+0000","last_clean":"2023-10-01T10:45:40.521886+0000","last_became_active":"2023-10-01T10:37:42.772102+0000","last_became_peered":"2023-10-01T10:37:42.772102+0000","last_unstale":"2023-10-01T10:45:40.521886+0000","last_undegraded":"2023-10-01T10:45:40.521886+0000","last_fullsized":"2023-10-01T10:45:40.521886+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:23:26.202295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-01T10:45:56.023726+0000","last_change":"2023-10-01T10:45:56.023726+0000","last_active":"2023-10-01T10:45:56.023726+0000","last_peered":"2023-10-01T10:45:56.023726+0000","last_clean":"2023-10-01T10:45:56.023726+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-01T10:45:56.023726+0000","last_undegraded":"2023-10-01T10:45:56.023726+0000","last_fullsized":"2023-10-01T10:45:56.023726+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_clean_scrub_stamp":"2023-10-01T10:45:56.023726+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":133,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-01T10:45:37.511305+0000","last_change":"2023-10-01T10:38:16.026955+0000","last_active":"2023-10-01T10:45:37.511305+0000","last_peered":"2023-10-01T10:45:37.511305+0000","last_clean":"2023-10-01T10:45:37.511305+0000","last_became_active":"2023-10-01T10:38:16.026552+0000","last_became_peered":"2023-10-01T10:38:16.026552+0000","last_unstale":"2023-10-01T10:45:37.511305+0000","last_undegraded":"2023-10-01T10:45:37.511305+0000","last_fullsized":"2023-10-01T10:45:37.511305+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":101,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-01T10:45:37.574088+0000","last_change":"2023-10-01T10:15:40.318702+0000","last_active":"2023-10-01T10:45:37.574088+0000","last_peered":"2023-10-01T10:45:37.574088+0000","last_clean":"2023-10-01T10:45:37.574088+0000","last_became_active":"2023-10-01T10:15:40.318333+0000","last_became_peered":"2023-10-01T10:15:40.318333+0000","last_unstale":"2023-10-01T10:45:37.574088+0000","last_undegraded":"2023-10-01T10:45:37.574088+0000","last_fullsized":"2023-10-01T10:45:37.574088+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-01T10:45:56.023726+0000","last_change":"2023-10-01T10:45:56.023726+0000","last_active":"2023-10-01T10:45:56.023726+0000","last_peered":"2023-10-01T10:45:56.023726+0000","last_clean":"2023-10-01T10:45:56.023726+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-01T10:45:56.023726+0000","last_undegraded":"2023-10-01T10:45:56.023726+0000","last_fullsized":"2023-10-01T10:45:56.023726+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_clean_scrub_stamp":"2023-10-01T10:45:56.023726+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":134,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-01T10:45:37.510154+0000","last_change":"2023-10-01T10:38:16.027753+0000","last_active":"2023-10-01T10:45:37.510154+0000","last_peered":"2023-10-01T10:45:37.510154+0000","last_clean":"2023-10-01T10:45:37.510154+0000","last_became_active":"2023-10-01T10:38:16.027384+0000","last_became_peered":"2023-10-01T10:38:16.027384+0000","last_unstale":"2023-10-01T10:45:37.510154+0000","last_undegraded":"2023-10-01T10:45:37.510154+0000","last_fullsized":"2023-10-01T10:45:37.510154+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":141,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-01T10:45:37.566891+0000","last_change":"2023-10-01T10:38:16.033203+0000","last_active":"2023-10-01T10:45:37.566891+0000","last_peered":"2023-10-01T10:45:37.566891+0000","last_clean":"2023-10-01T10:45:37.566891+0000","last_became_active":"2023-10-01T10:38:16.032681+0000","last_became_peered":"2023-10-01T10:38:16.032681+0000","last_unstale":"2023-10-01T10:45:37.566891+0000","last_undegraded":"2023-10-01T10:45:37.566891+0000","last_fullsized":"2023-10-01T10:45:37.566891+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:42:36.642284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-01T10:45:56.023726+0000","last_change":"2023-10-01T10:45:56.023726+0000","last_active":"2023-10-01T10:45:56.023726+0000","last_peered":"2023-10-01T10:45:56.023726+0000","last_clean":"2023-10-01T10:45:56.023726+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-01T10:45:56.023726+0000","last_undegraded":"2023-10-01T10:45:56.023726+0000","last_fullsized":"2023-10-01T10:45:56.023726+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_clean_scrub_stamp":"2023-10-01T10:45:56.023726+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":103,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-01T10:45:37.511092+0000","last_change":"2023-10-01T10:15:39.293587+0000","last_active":"2023-10-01T10:45:37.511092+0000","last_peered":"2023-10-01T10:45:37.511092+0000","last_clean":"2023-10-01T10:45:37.511092+0000","last_became_active":"2023-10-01T10:15:39.293273+0000","last_became_peered":"2023-10-01T10:15:39.293273+0000","last_unstale":"2023-10-01T10:45:37.511092+0000","last_undegraded":"2023-10-01T10:45:37.511092+0000","last_fullsized":"2023-10-01T10:45:37.511092+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":115,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-01T10:45:37.567715+0000","last_change":"2023-10-01T10:32:31.571971+0000","last_active":"2023-10-01T10:45:37.567715+0000","last_peered":"2023-10-01T10:45:37.567715+0000","last_clean":"2023-10-01T10:45:37.567715+0000","last_became_active":"2023-10-01T10:32:31.571410+0000","last_became_peered":"2023-10-01T10:32:31.571410+0000","last_unstale":"2023-10-01T10:45:37.567715+0000","last_undegraded":"2023-10-01T10:45:37.567715+0000","last_fullsized":"2023-10-01T10:45:37.567715+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-01T10:45:56.023726+0000","last_change":"2023-10-01T10:45:56.023726+0000","last_active":"2023-10-01T10:45:56.023726+0000","last_peered":"2023-10-01T10:45:56.023726+0000","last_clean":"2023-10-01T10:45:56.023726+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-01T10:45:56.023726+0000","last_undegraded":"2023-10-01T10:45:56.023726+0000","last_fullsized":"2023-10-01T10:45:56.023726+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_clean_scrub_stamp":"2023-10-01T10:45:56.023726+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":117,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-01T10:45:37.574407+0000","last_change":"2023-10-01T10:37:42.765733+0000","last_active":"2023-10-01T10:45:37.574407+0000","last_peered":"2023-10-01T10:45:37.574407+0000","last_clean":"2023-10-01T10:45:37.574407+0000","last_became_active":"2023-10-01T10:37:42.765311+0000","last_became_peered":"2023-10-01T10:37:42.765311+0000","last_unstale":"2023-10-01T10:45:37.574407+0000","last_undegraded":"2023-10-01T10:45:37.574407+0000","last_fullsized":"2023-10-01T10:45:37.574407+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":123,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-01T10:45:37.576830+0000","last_change":"2023-10-01T10:38:16.026348+0000","last_active":"2023-10-01T10:45:37.576830+0000","last_peered":"2023-10-01T10:45:37.576830+0000","last_clean":"2023-10-01T10:45:37.576830+0000","last_became_active":"2023-10-01T10:38:16.025747+0000","last_became_peered":"2023-10-01T10:38:16.025747+0000","last_unstale":"2023-10-01T10:45:37.576830+0000","last_undegraded":"2023-10-01T10:45:37.576830+0000","last_fullsized":"2023-10-01T10:45:37.576830+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:28:18.934480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-01T10:45:56.023726+0000","last_change":"2023-10-01T10:45:56.023726+0000","last_active":"2023-10-01T10:45:56.023726+0000","last_peered":"2023-10-01T10:45:56.023726+0000","last_clean":"2023-10-01T10:45:56.023726+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-01T10:45:56.023726+0000","last_undegraded":"2023-10-01T10:45:56.023726+0000","last_fullsized":"2023-10-01T10:45:56.023726+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_clean_scrub_stamp":"2023-10-01T10:45:56.023726+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":116,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-01T10:45:40.521133+0000","last_change":"2023-10-01T10:38:16.022493+0000","last_active":"2023-10-01T10:45:40.521133+0000","last_peered":"2023-10-01T10:45:40.521133+0000","last_clean":"2023-10-01T10:45:40.521133+0000","last_became_active":"2023-10-01T10:38:16.022033+0000","last_became_peered":"2023-10-01T10:38:16.022033+0000","last_unstale":"2023-10-01T10:45:40.521133+0000","last_undegraded":"2023-10-01T10:45:40.521133+0000","last_fullsized":"2023-10-01T10:45:40.521133+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:49:05.262763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":126,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-01T10:45:37.568145+0000","last_change":"2023-10-01T10:38:16.030613+0000","last_active":"2023-10-01T10:45:37.568145+0000","last_peered":"2023-10-01T10:45:37.568145+0000","last_clean":"2023-10-01T10:45:37.568145+0000","last_became_active":"2023-10-01T10:38:16.028975+0000","last_became_peered":"2023-10-01T10:38:16.028975+0000","last_unstale":"2023-10-01T10:45:37.568145+0000","last_undegraded":"2023-10-01T10:45:37.568145+0000","last_fullsized":"2023-10-01T10:45:37.568145+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:46:07.287911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-01T10:45:56.023726+0000","last_change":"2023-10-01T10:45:56.023726+0000","last_active":"2023-10-01T10:45:56.023726+0000","last_peered":"2023-10-01T10:45:56.023726+0000","last_clean":"2023-10-01T10:45:56.023726+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-01T10:45:56.023726+0000","last_undegraded":"2023-10-01T10:45:56.023726+0000","last_fullsized":"2023-10-01T10:45:56.023726+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_clean_scrub_stamp":"2023-10-01T10:45:56.023726+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":131,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-01T10:45:37.567866+0000","last_change":"2023-10-01T10:38:16.030250+0000","last_active":"2023-10-01T10:45:37.567866+0000","last_peered":"2023-10-01T10:45:37.567866+0000","last_clean":"2023-10-01T10:45:37.567866+0000","last_became_active":"2023-10-01T10:38:16.029898+0000","last_became_peered":"2023-10-01T10:38:16.029898+0000","last_unstale":"2023-10-01T10:45:37.567866+0000","last_undegraded":"2023-10-01T10:45:37.567866+0000","last_fullsized":"2023-10-01T10:45:37.567866+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":131,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-01T10:45:37.567481+0000","last_change":"2023-10-01T10:38:16.035906+0000","last_active":"2023-10-01T10:45:37.567481+0000","last_peered":"2023-10-01T10:45:37.567481+0000","last_clean":"2023-10-01T10:45:37.567481+0000","last_became_active":"2023-10-01T10:38:16.035383+0000","last_became_peered":"2023-10-01T10:38:16.035383+0000","last_unstale":"2023-10-01T10:45:37.567481+0000","last_undegraded":"2023-10-01T10:45:37.567481+0000","last_fullsized":"2023-10-01T10:45:37.567481+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:44:41.500097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-01T10:45:56.023726+0000","last_change":"2023-10-01T10:45:56.023726+0000","last_active":"2023-10-01T10:45:56.023726+0000","last_peered":"2023-10-01T10:45:56.023726+0000","last_clean":"2023-10-01T10:45:56.023726+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-01T10:45:56.023726+0000","last_undegraded":"2023-10-01T10:45:56.023726+0000","last_fullsized":"2023-10-01T10:45:56.023726+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_clean_scrub_stamp":"2023-10-01T10:45:56.023726+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":123,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-01T10:45:37.566605+0000","last_change":"2023-10-01T10:38:16.026611+0000","last_active":"2023-10-01T10:45:37.566605+0000","last_peered":"2023-10-01T10:45:37.566605+0000","last_clean":"2023-10-01T10:45:37.566605+0000","last_became_active":"2023-10-01T10:38:16.026267+0000","last_became_peered":"2023-10-01T10:38:16.026267+0000","last_unstale":"2023-10-01T10:45:37.566605+0000","last_undegraded":"2023-10-01T10:45:37.566605+0000","last_fullsized":"2023-10-01T10:45:37.566605+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:06.135288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":107,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-01T10:45:37.568570+0000","last_change":"2023-10-01T10:38:16.032162+0000","last_active":"2023-10-01T10:45:37.568570+0000","last_peered":"2023-10-01T10:45:37.568570+0000","last_clean":"2023-10-01T10:45:37.568570+0000","last_became_active":"2023-10-01T10:38:16.031785+0000","last_became_peered":"2023-10-01T10:38:16.031785+0000","last_unstale":"2023-10-01T10:45:37.568570+0000","last_undegraded":"2023-10-01T10:45:37.568570+0000","last_fullsized":"2023-10-01T10:45:37.568570+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:38:43.003626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-01T10:45:56.023726+0000","last_change":"2023-10-01T10:45:56.023726+0000","last_active":"2023-10-01T10:45:56.023726+0000","last_peered":"2023-10-01T10:45:56.023726+0000","last_clean":"2023-10-01T10:45:56.023726+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-01T10:45:56.023726+0000","last_undegraded":"2023-10-01T10:45:56.023726+0000","last_fullsized":"2023-10-01T10:45:56.023726+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_clean_scrub_stamp":"2023-10-01T10:45:56.023726+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":114,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-01T10:45:40.520473+0000","last_change":"2023-10-01T10:37:42.772398+0000","last_active":"2023-10-01T10:45:40.520473+0000","last_peered":"2023-10-01T10:45:40.520473+0000","last_clean":"2023-10-01T10:45:40.520473+0000","last_became_active":"2023-10-01T10:37:42.772084+0000","last_became_peered":"2023-10-01T10:37:42.772084+0000","last_unstale":"2023-10-01T10:45:40.520473+0000","last_undegraded":"2023-10-01T10:45:40.520473+0000","last_fullsized":"2023-10-01T10:45:40.520473+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:32:43.846419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":131,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-01T10:45:37.565690+0000","last_change":"2023-10-01T10:38:16.019755+0000","last_active":"2023-10-01T10:45:37.565690+0000","last_peered":"2023-10-01T10:45:37.565690+0000","last_clean":"2023-10-01T10:45:37.565690+0000","last_became_active":"2023-10-01T10:38:16.019397+0000","last_became_peered":"2023-10-01T10:38:16.019397+0000","last_unstale":"2023-10-01T10:45:37.565690+0000","last_undegraded":"2023-10-01T10:45:37.565690+0000","last_fullsized":"2023-10-01T10:45:37.565690+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:42:56.623225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-01T10:45:56.023726+0000","last_change":"2023-10-01T10:45:56.023726+0000","last_active":"2023-10-01T10:45:56.023726+0000","last_peered":"2023-10-01T10:45:56.023726+0000","last_clean":"2023-10-01T10:45:56.023726+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-01T10:45:56.023726+0000","last_undegraded":"2023-10-01T10:45:56.023726+0000","last_fullsized":"2023-10-01T10:45:56.023726+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_clean_scrub_stamp":"2023-10-01T10:45:56.023726+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":125,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-01T10:45:37.508677+0000","last_change":"2023-10-01T10:32:31.561108+0000","last_active":"2023-10-01T10:45:37.508677+0000","last_peered":"2023-10-01T10:45:37.508677+0000","last_clean":"2023-10-01T10:45:37.508677+0000","last_became_active":"2023-10-01T10:32:31.560791+0000","last_became_peered":"2023-10-01T10:32:31.560791+0000","last_unstale":"2023-10-01T10:45:37.508677+0000","last_undegraded":"2023-10-01T10:45:37.508677+0000","last_fullsized":"2023-10-01T10:45:37.508677+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:42:47.477671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":132,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-01T10:45:37.581375+0000","last_change":"2023-10-01T10:37:42.766814+0000","last_active":"2023-10-01T10:45:37.581375+0000","last_peered":"2023-10-01T10:45:37.581375+0000","last_clean":"2023-10-01T10:45:37.581375+0000","last_became_active":"2023-10-01T10:37:42.766451+0000","last_became_peered":"2023-10-01T10:37:42.766451+0000","last_unstale":"2023-10-01T10:45:37.581375+0000","last_undegraded":"2023-10-01T10:45:37.581375+0000","last_fullsized":"2023-10-01T10:45:37.581375+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":91,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-01T10:45:37.508351+0000","last_change":"2023-10-01T10:38:16.034357+0000","last_active":"2023-10-01T10:45:37.508351+0000","last_peered":"2023-10-01T10:45:37.508351+0000","last_clean":"2023-10-01T10:45:37.508351+0000","last_became_active":"2023-10-01T10:38:16.033905+0000","last_became_peered":"2023-10-01T10:38:16.033905+0000","last_unstale":"2023-10-01T10:45:37.508351+0000","last_undegraded":"2023-10-01T10:45:37.508351+0000","last_fullsized":"2023-10-01T10:45:37.508351+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:47:58.369835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-01T10:45:56.023726+0000","last_change":"2023-10-01T10:45:56.023726+0000","last_active":"2023-10-01T10:45:56.023726+0000","last_peered":"2023-10-01T10:45:56.023726+0000","last_clean":"2023-10-01T10:45:56.023726+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-01T10:45:56.023726+0000","last_undegraded":"2023-10-01T10:45:56.023726+0000","last_fullsized":"2023-10-01T10:45:56.023726+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_clean_scrub_stamp":"2023-10-01T10:45:56.023726+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":121,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-01T10:45:37.575175+0000","last_change":"2023-10-01T10:32:31.572482+0000","last_active":"2023-10-01T10:45:37.575175+0000","last_peered":"2023-10-01T10:45:37.575175+0000","last_clean":"2023-10-01T10:45:37.575175+0000","last_became_active":"2023-10-01T10:32:31.571993+0000","last_became_peered":"2023-10-01T10:32:31.571993+0000","last_unstale":"2023-10-01T10:45:37.575175+0000","last_undegraded":"2023-10-01T10:45:37.575175+0000","last_fullsized":"2023-10-01T10:45:37.575175+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":93,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-01T10:45:37.567436+0000","last_change":"2023-10-01T10:38:16.030202+0000","last_active":"2023-10-01T10:45:37.567436+0000","last_peered":"2023-10-01T10:45:37.567436+0000","last_clean":"2023-10-01T10:45:37.567436+0000","last_became_active":"2023-10-01T10:38:16.029917+0000","last_became_peered":"2023-10-01T10:38:16.029917+0000","last_unstale":"2023-10-01T10:45:37.567436+0000","last_undegraded":"2023-10-01T10:45:37.567436+0000","last_fullsized":"2023-10-01T10:45:37.567436+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:20:06.763909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":106,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-01T10:45:37.508765+0000","last_change":"2023-10-01T10:37:42.761153+0000","last_active":"2023-10-01T10:45:37.508765+0000","last_peered":"2023-10-01T10:45:37.508765+0000","last_clean":"2023-10-01T10:45:37.508765+0000","last_became_active":"2023-10-01T10:37:42.760715+0000","last_became_peered":"2023-10-01T10:37:42.760715+0000","last_unstale":"2023-10-01T10:45:37.508765+0000","last_undegraded":"2023-10-01T10:45:37.508765+0000","last_fullsized":"2023-10-01T10:45:37.508765+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:52:50.237174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-01T10:45:56.023726+0000","last_change":"2023-10-01T10:45:56.023726+0000","last_active":"2023-10-01T10:45:56.023726+0000","last_peered":"2023-10-01T10:45:56.023726+0000","last_clean":"2023-10-01T10:45:56.023726+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-01T10:45:56.023726+0000","last_undegraded":"2023-10-01T10:45:56.023726+0000","last_fullsized":"2023-10-01T10:45:56.023726+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_clean_scrub_stamp":"2023-10-01T10:45:56.023726+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":123,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-01T10:45:37.564549+0000","last_change":"2023-10-01T10:37:42.762524+0000","last_active":"2023-10-01T10:45:37.564549+0000","last_peered":"2023-10-01T10:45:37.564549+0000","last_clean":"2023-10-01T10:45:37.564549+0000","last_became_active":"2023-10-01T10:37:42.762209+0000","last_became_peered":"2023-10-01T10:37:42.762209+0000","last_unstale":"2023-10-01T10:45:37.564549+0000","last_undegraded":"2023-10-01T10:45:37.564549+0000","last_fullsized":"2023-10-01T10:45:37.564549+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:45:38.171397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":91,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-01T10:45:37.507838+0000","last_change":"2023-10-01T10:37:05.019330+0000","last_active":"2023-10-01T10:45:37.507838+0000","last_peered":"2023-10-01T10:45:37.507838+0000","last_clean":"2023-10-01T10:45:37.507838+0000","last_became_active":"2023-10-01T10:37:05.018842+0000","last_became_peered":"2023-10-01T10:37:05.018842+0000","last_unstale":"2023-10-01T10:45:37.507838+0000","last_undegraded":"2023-10-01T10:45:37.507838+0000","last_fullsized":"2023-10-01T10:45:37.507838+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:48:09.551917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":120,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-01T10:45:37.505007+0000","last_change":"2023-10-01T10:32:31.569162+0000","last_active":"2023-10-01T10:45:37.505007+0000","last_peered":"2023-10-01T10:45:37.505007+0000","last_clean":"2023-10-01T10:45:37.505007+0000","last_became_active":"2023-10-01T10:32:31.568956+0000","last_became_peered":"2023-10-01T10:32:31.568956+0000","last_unstale":"2023-10-01T10:45:37.505007+0000","last_undegraded":"2023-10-01T10:45:37.505007+0000","last_fullsized":"2023-10-01T10:45:37.505007+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:02:08.285304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-01T10:45:56.023726+0000","last_change":"2023-10-01T10:45:56.023726+0000","last_active":"2023-10-01T10:45:56.023726+0000","last_peered":"2023-10-01T10:45:56.023726+0000","last_clean":"2023-10-01T10:45:56.023726+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-01T10:45:56.023726+0000","last_undegraded":"2023-10-01T10:45:56.023726+0000","last_fullsized":"2023-10-01T10:45:56.023726+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_clean_scrub_stamp":"2023-10-01T10:45:56.023726+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":121,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-01T10:45:37.575807+0000","last_change":"2023-10-01T10:37:05.024009+0000","last_active":"2023-10-01T10:45:37.575807+0000","last_peered":"2023-10-01T10:45:37.575807+0000","last_clean":"2023-10-01T10:45:37.575807+0000","last_became_active":"2023-10-01T10:37:05.023458+0000","last_became_peered":"2023-10-01T10:37:05.023458+0000","last_unstale":"2023-10-01T10:45:37.575807+0000","last_undegraded":"2023-10-01T10:45:37.575807+0000","last_fullsized":"2023-10-01T10:45:37.575807+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'139","reported_seq":353,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-01T10:45:37.508854+0000","last_change":"2023-10-01T10:28:20.217386+0000","last_active":"2023-10-01T10:45:37.508854+0000","last_peered":"2023-10-01T10:45:37.508854+0000","last_clean":"2023-10-01T10:45:37.508854+0000","last_became_active":"2023-10-01T10:28:20.216803+0000","last_became_peered":"2023-10-01T10:28:20.216803+0000","last_unstale":"2023-10-01T10:45:37.508854+0000","last_undegraded":"2023-10-01T10:45:37.508854+0000","last_fullsized":"2023-10-01T10:45:37.508854+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:09:17.221082+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:09:17.221082+0000","last_clean_scrub_stamp":"2023-10-01T10:09:17.221082+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":96,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-01T10:45:40.520682+0000","last_change":"2023-10-01T10:37:42.765100+0000","last_active":"2023-10-01T10:45:40.520682+0000","last_peered":"2023-10-01T10:45:40.520682+0000","last_clean":"2023-10-01T10:45:40.520682+0000","last_became_active":"2023-10-01T10:37:42.764761+0000","last_became_peered":"2023-10-01T10:37:42.764761+0000","last_unstale":"2023-10-01T10:45:40.520682+0000","last_undegraded":"2023-10-01T10:45:40.520682+0000","last_fullsized":"2023-10-01T10:45:40.520682+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:24:56.907377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":115,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-01T10:45:37.568337+0000","last_change":"2023-10-01T10:38:16.031255+0000","last_active":"2023-10-01T10:45:37.568337+0000","last_peered":"2023-10-01T10:45:37.568337+0000","last_clean":"2023-10-01T10:45:37.568337+0000","last_became_active":"2023-10-01T10:38:16.030886+0000","last_became_peered":"2023-10-01T10:38:16.030886+0000","last_unstale":"2023-10-01T10:45:37.568337+0000","last_undegraded":"2023-10-01T10:45:37.568337+0000","last_fullsized":"2023-10-01T10:45:37.568337+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-01T10:45:56.023726+0000","last_change":"2023-10-01T10:45:56.023726+0000","last_active":"2023-10-01T10:45:56.023726+0000","last_peered":"2023-10-01T10:45:56.023726+0000","last_clean":"2023-10-01T10:45:56.023726+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-01T10:45:56.023726+0000","last_undegraded":"2023-10-01T10:45:56.023726+0000","last_fullsized":"2023-10-01T10:45:56.023726+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_clean_scrub_stamp":"2023-10-01T10:45:56.023726+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":136,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-01T10:45:37.508848+0000","last_change":"2023-10-01T10:32:31.565868+0000","last_active":"2023-10-01T10:45:37.508848+0000","last_peered":"2023-10-01T10:45:37.508848+0000","last_clean":"2023-10-01T10:45:37.508848+0000","last_became_active":"2023-10-01T10:32:31.565428+0000","last_became_peered":"2023-10-01T10:32:31.565428+0000","last_unstale":"2023-10-01T10:45:37.508848+0000","last_undegraded":"2023-10-01T10:45:37.508848+0000","last_fullsized":"2023-10-01T10:45:37.508848+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":123,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-01T10:45:37.576477+0000","last_change":"2023-10-01T10:38:16.021968+0000","last_active":"2023-10-01T10:45:37.576477+0000","last_peered":"2023-10-01T10:45:37.576477+0000","last_clean":"2023-10-01T10:45:37.576477+0000","last_became_active":"2023-10-01T10:38:16.021539+0000","last_became_peered":"2023-10-01T10:38:16.021539+0000","last_unstale":"2023-10-01T10:45:37.576477+0000","last_undegraded":"2023-10-01T10:45:37.576477+0000","last_fullsized":"2023-10-01T10:45:37.576477+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:12:53.105747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":124,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-01T10:45:38.378868+0000","last_change":"2023-10-01T10:37:05.017431+0000","last_active":"2023-10-01T10:45:38.378868+0000","last_peered":"2023-10-01T10:45:38.378868+0000","last_clean":"2023-10-01T10:45:38.378868+0000","last_became_active":"2023-10-01T10:37:05.017094+0000","last_became_peered":"2023-10-01T10:37:05.017094+0000","last_unstale":"2023-10-01T10:45:38.378868+0000","last_undegraded":"2023-10-01T10:45:38.378868+0000","last_fullsized":"2023-10-01T10:45:38.378868+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:10:40.664686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-01T10:45:56.023726+0000","last_change":"2023-10-01T10:45:56.023726+0000","last_active":"2023-10-01T10:45:56.023726+0000","last_peered":"2023-10-01T10:45:56.023726+0000","last_clean":"2023-10-01T10:45:56.023726+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-01T10:45:56.023726+0000","last_undegraded":"2023-10-01T10:45:56.023726+0000","last_fullsized":"2023-10-01T10:45:56.023726+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_clean_scrub_stamp":"2023-10-01T10:45:56.023726+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":136,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-01T10:45:37.511234+0000","last_change":"2023-10-01T10:38:16.032774+0000","last_active":"2023-10-01T10:45:37.511234+0000","last_peered":"2023-10-01T10:45:37.511234+0000","last_clean":"2023-10-01T10:45:37.511234+0000","last_became_active":"2023-10-01T10:38:16.032283+0000","last_became_peered":"2023-10-01T10:38:16.032283+0000","last_unstale":"2023-10-01T10:45:37.511234+0000","last_undegraded":"2023-10-01T10:45:37.511234+0000","last_fullsized":"2023-10-01T10:45:37.511234+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":91,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-01T10:45:37.508857+0000","last_change":"2023-10-01T10:37:05.021228+0000","last_active":"2023-10-01T10:45:37.508857+0000","last_peered":"2023-10-01T10:45:37.508857+0000","last_clean":"2023-10-01T10:45:37.508857+0000","last_became_active":"2023-10-01T10:37:05.020888+0000","last_became_peered":"2023-10-01T10:37:05.020888+0000","last_unstale":"2023-10-01T10:45:37.508857+0000","last_undegraded":"2023-10-01T10:45:37.508857+0000","last_fullsized":"2023-10-01T10:45:37.508857+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:59:57.641247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":94,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-01T10:45:37.576713+0000","last_change":"2023-10-01T10:28:20.220812+0000","last_active":"2023-10-01T10:45:37.576713+0000","last_peered":"2023-10-01T10:45:37.576713+0000","last_clean":"2023-10-01T10:45:37.576713+0000","last_became_active":"2023-10-01T10:28:20.215962+0000","last_became_peered":"2023-10-01T10:28:20.215962+0000","last_unstale":"2023-10-01T10:45:37.576713+0000","last_undegraded":"2023-10-01T10:45:37.576713+0000","last_fullsized":"2023-10-01T10:45:37.576713+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:35:15.275097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-01T10:45:56.023726+0000","last_change":"2023-10-01T10:45:56.023726+0000","last_active":"2023-10-01T10:45:56.023726+0000","last_peered":"2023-10-01T10:45:56.023726+0000","last_clean":"2023-10-01T10:45:56.023726+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-01T10:45:56.023726+0000","last_undegraded":"2023-10-01T10:45:56.023726+0000","last_fullsized":"2023-10-01T10:45:56.023726+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_clean_scrub_stamp":"2023-10-01T10:45:56.023726+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":101,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-01T10:45:40.522018+0000","last_change":"2023-10-01T10:37:42.773999+0000","last_active":"2023-10-01T10:45:40.522018+0000","last_peered":"2023-10-01T10:45:40.522018+0000","last_clean":"2023-10-01T10:45:40.522018+0000","last_became_active":"2023-10-01T10:37:42.773355+0000","last_became_peered":"2023-10-01T10:37:42.773355+0000","last_unstale":"2023-10-01T10:45:40.522018+0000","last_undegraded":"2023-10-01T10:45:40.522018+0000","last_fullsized":"2023-10-01T10:45:40.522018+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:46:51.317151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":133,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-01T10:45:37.513404+0000","last_change":"2023-10-01T10:38:16.034193+0000","last_active":"2023-10-01T10:45:37.513404+0000","last_peered":"2023-10-01T10:45:37.513404+0000","last_clean":"2023-10-01T10:45:37.513404+0000","last_became_active":"2023-10-01T10:38:16.033703+0000","last_became_peered":"2023-10-01T10:38:16.033703+0000","last_unstale":"2023-10-01T10:45:37.513404+0000","last_undegraded":"2023-10-01T10:45:37.513404+0000","last_fullsized":"2023-10-01T10:45:37.513404+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:21:26.061595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":99,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-01T10:45:41.487281+0000","last_change":"2023-10-01T10:37:42.773538+0000","last_active":"2023-10-01T10:45:41.487281+0000","last_peered":"2023-10-01T10:45:41.487281+0000","last_clean":"2023-10-01T10:45:41.487281+0000","last_became_active":"2023-10-01T10:37:42.771863+0000","last_became_peered":"2023-10-01T10:37:42.771863+0000","last_unstale":"2023-10-01T10:45:41.487281+0000","last_undegraded":"2023-10-01T10:45:41.487281+0000","last_fullsized":"2023-10-01T10:45:41.487281+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:46:46.698627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-10-01T10:45:56.023726+0000","last_change":"2023-10-01T10:45:56.023726+0000","last_active":"2023-10-01T10:45:56.023726+0000","last_peered":"2023-10-01T10:45:56.023726+0000","last_clean":"2023-10-01T10:45:56.023726+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-10-01T10:45:56.023726+0000","last_undegraded":"2023-10-01T10:45:56.023726+0000","last_fullsized":"2023-10-01T10:45:56.023726+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_clean_scrub_stamp":"2023-10-01T10:45:56.023726+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":120,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-01T10:45:37.568457+0000","last_change":"2023-10-01T10:37:05.023034+0000","last_active":"2023-10-01T10:45:37.568457+0000","last_peered":"2023-10-01T10:45:37.568457+0000","last_clean":"2023-10-01T10:45:37.568457+0000","last_became_active":"2023-10-01T10:37:05.022670+0000","last_became_peered":"2023-10-01T10:37:05.022670+0000","last_unstale":"2023-10-01T10:45:37.568457+0000","last_undegraded":"2023-10-01T10:45:37.568457+0000","last_fullsized":"2023-10-01T10:45:37.568457+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":127,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-01T10:45:38.378786+0000","last_change":"2023-10-01T10:38:16.027472+0000","last_active":"2023-10-01T10:45:38.378786+0000","last_peered":"2023-10-01T10:45:38.378786+0000","last_clean":"2023-10-01T10:45:38.378786+0000","last_became_active":"2023-10-01T10:38:16.026810+0000","last_became_peered":"2023-10-01T10:38:16.026810+0000","last_unstale":"2023-10-01T10:45:38.378786+0000","last_undegraded":"2023-10-01T10:45:38.378786+0000","last_fullsized":"2023-10-01T10:45:38.378786+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:30:06.978187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":91,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-01T10:45:37.566802+0000","last_change":"2023-10-01T10:38:16.035705+0000","last_active":"2023-10-01T10:45:37.566802+0000","last_peered":"2023-10-01T10:45:37.566802+0000","last_clean":"2023-10-01T10:45:37.566802+0000","last_became_active":"2023-10-01T10:38:16.035389+0000","last_became_peered":"2023-10-01T10:38:16.035389+0000","last_unstale":"2023-10-01T10:45:37.566802+0000","last_undegraded":"2023-10-01T10:45:37.566802+0000","last_fullsized":"2023-10-01T10:45:37.566802+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:16:10.201409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":111,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-01T10:45:40.759815+0000","last_change":"2023-10-01T10:38:16.028491+0000","last_active":"2023-10-01T10:45:40.759815+0000","last_peered":"2023-10-01T10:45:40.759815+0000","last_clean":"2023-10-01T10:45:40.759815+0000","last_became_active":"2023-10-01T10:38:16.028175+0000","last_became_peered":"2023-10-01T10:38:16.028175+0000","last_unstale":"2023-10-01T10:45:40.759815+0000","last_undegraded":"2023-10-01T10:45:40.759815+0000","last_fullsized":"2023-10-01T10:45:40.759815+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:44:29.083129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":132,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-01T10:45:37.564679+0000","last_change":"2023-10-01T10:37:42.762360+0000","last_active":"2023-10-01T10:45:37.564679+0000","last_peered":"2023-10-01T10:45:37.564679+0000","last_clean":"2023-10-01T10:45:37.564679+0000","last_became_active":"2023-10-01T10:37:42.761936+0000","last_became_peered":"2023-10-01T10:37:42.761936+0000","last_unstale":"2023-10-01T10:45:37.564679+0000","last_undegraded":"2023-10-01T10:45:37.564679+0000","last_fullsized":"2023-10-01T10:45:37.564679+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:50.637582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":134,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-01T10:45:37.574492+0000","last_change":"2023-10-01T10:38:16.037809+0000","last_active":"2023-10-01T10:45:37.574492+0000","last_peered":"2023-10-01T10:45:37.574492+0000","last_clean":"2023-10-01T10:45:37.574492+0000","last_became_active":"2023-10-01T10:38:16.037318+0000","last_became_peered":"2023-10-01T10:38:16.037318+0000","last_unstale":"2023-10-01T10:45:37.574492+0000","last_undegraded":"2023-10-01T10:45:37.574492+0000","last_fullsized":"2023-10-01T10:45:37.574492+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":141,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-01T10:45:40.523629+0000","last_change":"2023-10-01T10:38:16.034282+0000","last_active":"2023-10-01T10:45:40.523629+0000","last_peered":"2023-10-01T10:45:40.523629+0000","last_clean":"2023-10-01T10:45:40.523629+0000","last_became_active":"2023-10-01T10:38:16.033820+0000","last_became_peered":"2023-10-01T10:38:16.033820+0000","last_unstale":"2023-10-01T10:45:40.523629+0000","last_undegraded":"2023-10-01T10:45:40.523629+0000","last_fullsized":"2023-10-01T10:45:40.523629+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:48:23.815832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":119,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-01T10:45:37.581213+0000","last_change":"2023-10-01T10:38:16.038077+0000","last_active":"2023-10-01T10:45:37.581213+0000","last_peered":"2023-10-01T10:45:37.581213+0000","last_clean":"2023-10-01T10:45:37.581213+0000","last_became_active":"2023-10-01T10:38:16.037313+0000","last_became_peered":"2023-10-01T10:38:16.037313+0000","last_unstale":"2023-10-01T10:45:37.581213+0000","last_undegraded":"2023-10-01T10:45:37.581213+0000","last_fullsized":"2023-10-01T10:45:37.581213+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":141,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-01T10:45:40.523312+0000","last_change":"2023-10-01T10:38:16.025600+0000","last_active":"2023-10-01T10:45:40.523312+0000","last_peered":"2023-10-01T10:45:40.523312+0000","last_clean":"2023-10-01T10:45:40.523312+0000","last_became_active":"2023-10-01T10:38:16.025252+0000","last_became_peered":"2023-10-01T10:38:16.025252+0000","last_unstale":"2023-10-01T10:45:40.523312+0000","last_undegraded":"2023-10-01T10:45:40.523312+0000","last_fullsized":"2023-10-01T10:45:40.523312+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:24:18.871963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":130,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-01T10:45:41.487463+0000","last_change":"2023-10-01T10:37:42.761243+0000","last_active":"2023-10-01T10:45:41.487463+0000","last_peered":"2023-10-01T10:45:41.487463+0000","last_clean":"2023-10-01T10:45:41.487463+0000","last_became_active":"2023-10-01T10:37:42.760834+0000","last_became_peered":"2023-10-01T10:37:42.760834+0000","last_unstale":"2023-10-01T10:45:41.487463+0000","last_undegraded":"2023-10-01T10:45:41.487463+0000","last_fullsized":"2023-10-01T10:45:41.487463+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:37:21.976095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":142,"reported_epoch":150,"state":"active+clean","last_fresh":"2023-10-01T10:45:40.520471+0000","last_change":"2023-10-01T10:37:42.765142+0000","last_active":"2023-10-01T10:45:40.520471+0000","last_peered":"2023-10-01T10:45:40.520471+0000","last_clean":"2023-10-01T10:45:40.520471+0000","last_became_active":"2023-10-01T10:37:42.764770+0000","last_became_peered":"2023-10-01T10:37:42.764770+0000","last_unstale":"2023-10-01T10:45:40.520471+0000","last_undegraded":"2023-10-01T10:45:40.520471+0000","last_fullsized":"2023-10-01T10:45:40.520471+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:23:33.527876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":112,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-01T10:45:37.566930+0000","last_change":"2023-10-01T10:37:05.021621+0000","last_active":"2023-10-01T10:45:37.566930+0000","last_peered":"2023-10-01T10:45:37.566930+0000","last_clean":"2023-10-01T10:45:37.566930+0000","last_became_active":"2023-10-01T10:37:05.021173+0000","last_became_peered":"2023-10-01T10:37:05.021173+0000","last_unstale":"2023-10-01T10:45:37.566930+0000","last_undegraded":"2023-10-01T10:45:37.566930+0000","last_fullsized":"2023-10-01T10:45:37.566930+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:13:25.364499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":107,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-10-01T10:45:37.566598+0000","last_change":"2023-10-01T10:38:16.031427+0000","last_active":"2023-10-01T10:45:37.566598+0000","last_peered":"2023-10-01T10:45:37.566598+0000","last_clean":"2023-10-01T10:45:37.566598+0000","last_became_active":"2023-10-01T10:38:16.029786+0000","last_became_peered":"2023-10-01T10:38:16.029786+0000","last_unstale":"2023-10-01T10:45:37.566598+0000","last_undegraded":"2023-10-01T10:45:37.566598+0000","last_fullsized":"2023-10-01T10:45:37.566598+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:53:22.796191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30319,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430607,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":595784,"kb_used_data":1984,"kb_used_omap":0,"kb_used_meta":593792,"kb_avail":93149368,"statfs":{"total":95995035648,"available":95384952832,"internally_reserved":0,"allocated":2031616,"data_stored":1541012,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":608043008},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:44:57 2023","interfaces":[{"interface":"back","average":{"1min":6.1180000000000003,"5min":2.5219999999999998,"15min":1.776},"min":{"1min":0.36899999999999999,"5min":0.28499999999999998,"15min":0.27500000000000002},"max":{"1min":51.07,"5min":51.07,"15min":51.07},"last":0.63600000000000001},{"interface":"front","average":{"1min":4.9960000000000004,"5min":1.887,"15min":1.6160000000000001},"min":{"1min":0.307,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":51.024000000000001,"5min":51.024000000000001,"15min":51.024000000000001},"last":0.87}]},{"osd":1,"last update":"Sun Oct 1 10:45:03 2023","interfaces":[{"interface":"back","average":{"1min":5.8200000000000003,"5min":2.746,"15min":1.758},"min":{"1min":0.35299999999999998,"5min":0.311,"15min":0.309},"max":{"1min":56.110999999999997,"5min":56.110999999999997,"15min":56.110999999999997},"last":0.81100000000000005},{"interface":"front","average":{"1min":5.7720000000000002,"5min":2.7879999999999998,"15min":1.923},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":51.258000000000003,"5min":51.258000000000003,"15min":51.258000000000003},"last":0.89600000000000002}]},{"osd":2,"last update":"Sun Oct 1 10:45:19 2023","interfaces":[{"interface":"back","average":{"1min":1.6459999999999999,"5min":1.3140000000000001,"15min":1.119},"min":{"1min":0.36899999999999999,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":5.4820000000000002,"5min":6.5979999999999999,"15min":6.5979999999999999},"last":1},{"interface":"front","average":{"1min":1.8220000000000001,"5min":1.8320000000000001,"15min":1.236},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":6.3419999999999996,"5min":10.766,"15min":10.766},"last":0.83899999999999997}]},{"osd":3,"last update":"Sun Oct 1 10:44:57 2023","interfaces":[{"interface":"back","average":{"1min":5.3380000000000001,"5min":2.5760000000000001,"15min":1.5069999999999999},"min":{"1min":0.34699999999999998,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":51.334000000000003,"5min":51.334000000000003,"15min":51.334000000000003},"last":1.1180000000000001},{"interface":"front","average":{"1min":6.0750000000000002,"5min":3.0819999999999999,"15min":1.6579999999999999},"min":{"1min":0.496,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":51.404000000000003,"5min":51.404000000000003,"15min":51.404000000000003},"last":0.72699999999999998}]},{"osd":4,"last update":"Sun Oct 1 10:45:30 2023","interfaces":[{"interface":"back","average":{"1min":1.468,"5min":1.4019999999999999,"15min":1.087},"min":{"1min":0.53600000000000003,"5min":0.317,"15min":0.317},"max":{"1min":5.085,"5min":8.6340000000000003,"15min":8.6340000000000003},"last":0.74199999999999999},{"interface":"front","average":{"1min":1.661,"5min":1.353,"15min":1.2410000000000001},"min":{"1min":0.41599999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":5.6699999999999999,"5min":7.54,"15min":7.54},"last":1.0389999999999999}]},{"osd":5,"last update":"Sun Oct 1 10:45:03 2023","interfaces":[{"interface":"back","average":{"1min":2.3380000000000001,"5min":2.0750000000000002,"15min":1.526},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.29699999999999999},"max":{"1min":6.758,"5min":10.381,"15min":10.381},"last":0.95699999999999996},{"interface":"front","average":{"1min":3.5249999999999999,"5min":2.657,"15min":1.6639999999999999},"min":{"1min":0.40300000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":9.7279999999999998,"5min":17.536999999999999,"15min":17.536999999999999},"last":0.77600000000000002}]},{"osd":6,"last update":"Sun Oct 1 10:45:03 2023","interfaces":[{"interface":"back","average":{"1min":1.573,"5min":1.4359999999999999,"15min":1.389},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.32800000000000001},"max":{"1min":5.6379999999999999,"5min":6.5789999999999997,"15min":7.3300000000000001},"last":1.0780000000000001},{"interface":"front","average":{"1min":1.6699999999999999,"5min":1.3080000000000001,"15min":1.3029999999999999},"min":{"1min":0.51200000000000001,"5min":0.36599999999999999,"15min":0.33900000000000002},"max":{"1min":5.1210000000000004,"5min":5.734,"15min":6.984},"last":0.92500000000000004}]}]},{"osd":6,"up_from":42,"seq":180388626841,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":311480,"kb_used_data":1968,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":93433672,"statfs":{"total":95995035648,"available":95676080128,"internally_reserved":0,"allocated":2015232,"data_stored":1540551,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:45:51 2023","interfaces":[{"interface":"back","average":{"1min":0.85699999999999998,"5min":2.4740000000000002,"15min":1.764},"min":{"1min":0.30299999999999999,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":3.3420000000000001,"5min":140.524,"15min":140.524},"last":0.69999999999999996},{"interface":"front","average":{"1min":0.96199999999999997,"5min":2.3540000000000001,"15min":1.714},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.28899999999999998},"max":{"1min":3.0859999999999999,"5min":140.44,"15min":140.44},"last":0.83099999999999996}]},{"osd":1,"last update":"Sun Oct 1 10:44:51 2023","interfaces":[{"interface":"back","average":{"1min":1.4590000000000001,"5min":1.3140000000000001,"15min":1.278},"min":{"1min":0.30299999999999999,"5min":0.29999999999999999,"15min":0.26600000000000001},"max":{"1min":4.8719999999999999,"5min":6.8010000000000002,"15min":7.9100000000000001},"last":0.61699999999999999},{"interface":"front","average":{"1min":1.526,"5min":1.3140000000000001,"15min":1.393},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":4.4500000000000002,"5min":6.3710000000000004,"15min":7.4199999999999999},"last":0.71899999999999997}]},{"osd":2,"last update":"Sun Oct 1 10:45:18 2023","interfaces":[{"interface":"back","average":{"1min":1.2609999999999999,"5min":1.177,"15min":0.97499999999999998},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":4.2450000000000001,"5min":4.8810000000000002,"15min":4.8810000000000002},"last":0.89300000000000002},{"interface":"front","average":{"1min":1.3340000000000001,"5min":1.2250000000000001,"15min":1.0389999999999999},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":7.9619999999999997,"5min":7.9619999999999997,"15min":7.9619999999999997},"last":1.0549999999999999}]},{"osd":3,"last update":"Sun Oct 1 10:45:01 2023","interfaces":[{"interface":"back","average":{"1min":3.3700000000000001,"5min":2.7829999999999999,"15min":1.585},"min":{"1min":0.39000000000000001,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":25.556000000000001,"5min":61.817999999999998,"15min":61.817999999999998},"last":0.99099999999999999},{"interface":"front","average":{"1min":2.734,"5min":2.3399999999999999,"15min":1.4590000000000001},"min":{"1min":0.371,"5min":0.315,"15min":0.315},"max":{"1min":24.879000000000001,"5min":61.219000000000001,"15min":61.219000000000001},"last":0.96799999999999997}]},{"osd":4,"last update":"Sun Oct 1 10:45:34 2023","interfaces":[{"interface":"back","average":{"1min":1.1659999999999999,"5min":0.92400000000000004,"15min":0.98299999999999998},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":4.5570000000000004,"5min":4.5570000000000004,"15min":4.5570000000000004},"last":0.91900000000000004},{"interface":"front","average":{"1min":1.1830000000000001,"5min":1.113,"15min":1.101},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":3.7799999999999998,"5min":7.1589999999999998,"15min":7.1589999999999998},"last":1.1499999999999999}]},{"osd":5,"last update":"Sun Oct 1 10:45:01 2023","interfaces":[{"interface":"back","average":{"1min":1.9399999999999999,"5min":1.4239999999999999,"15min":1.3089999999999999},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":12.398999999999999,"5min":12.398999999999999,"15min":12.398999999999999},"last":1.083},{"interface":"front","average":{"1min":2.6139999999999999,"5min":1.641,"15min":1.381},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":14.459,"5min":14.459,"15min":14.459},"last":1.105}]},{"osd":7,"last update":"Sun Oct 1 10:45:51 2023","interfaces":[{"interface":"back","average":{"1min":1.018,"5min":1.2070000000000001,"15min":1.129},"min":{"1min":0.42299999999999999,"5min":0.371,"15min":0.371},"max":{"1min":3.1349999999999998,"5min":8.8360000000000003,"15min":8.8360000000000003},"last":1.03},{"interface":"front","average":{"1min":0.98599999999999999,"5min":1.504,"15min":1.3200000000000001},"min":{"1min":0.40899999999999997,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":3.649,"5min":8.1720000000000006,"15min":8.1720000000000006},"last":1.129}]}]},{"osd":1,"up_from":101,"seq":433791697059,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":368424,"kb_used_data":1568,"kb_used_omap":4,"kb_used_meta":366843,"kb_avail":93376728,"statfs":{"total":95995035648,"available":95617769472,"internally_reserved":0,"allocated":1605632,"data_stored":974907,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":375648142},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:45:01 2023","interfaces":[{"interface":"back","average":{"1min":0.89700000000000002,"5min":1.1379999999999999,"15min":1.5900000000000001},"min":{"1min":0.19900000000000001,"5min":0.189,"15min":0.189},"max":{"1min":3.1429999999999998,"5min":8.3520000000000003,"15min":11.071999999999999},"last":0.42299999999999999},{"interface":"front","average":{"1min":2.1890000000000001,"5min":2.081,"15min":1.869},"min":{"1min":0.22500000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":13.218,"5min":61.207999999999998,"15min":61.207999999999998},"last":0.39000000000000001}]},{"osd":2,"last update":"Sun Oct 1 10:45:22 2023","interfaces":[{"interface":"back","average":{"1min":1.5860000000000001,"5min":1.8420000000000001,"15min":1.069},"min":{"1min":0.30399999999999999,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":4.9550000000000001,"5min":61.491,"15min":61.491},"last":0.76900000000000002},{"interface":"front","average":{"1min":1.0669999999999999,"5min":2.0129999999999999,"15min":1.1870000000000001},"min":{"1min":0.317,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":3.9390000000000001,"5min":61.371000000000002,"15min":61.371000000000002},"last":0.74099999999999999}]},{"osd":3,"last update":"Sun Oct 1 10:45:01 2023","interfaces":[{"interface":"back","average":{"1min":2.177,"5min":2.4169999999999998,"15min":1.258},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":13.756,"5min":62.020000000000003,"15min":62.020000000000003},"last":0.66500000000000004},{"interface":"front","average":{"1min":2.718,"5min":2.6040000000000001,"15min":1.365},"min":{"1min":0.32900000000000001,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":13.669,"5min":62.338999999999999,"15min":62.338999999999999},"last":0.70599999999999996}]},{"osd":4,"last update":"Sun Oct 1 10:45:22 2023","interfaces":[{"interface":"back","average":{"1min":1.292,"5min":1.8240000000000001,"15min":1.2070000000000001},"min":{"1min":0.30199999999999999,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":4.5259999999999998,"5min":61.588000000000001,"15min":61.588000000000001},"last":0.78800000000000003},{"interface":"front","average":{"1min":1.206,"5min":2.2330000000000001,"15min":1.419},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":4.0519999999999996,"5min":61.411000000000001,"15min":61.411000000000001},"last":0.68700000000000006}]},{"osd":5,"last update":"Sun Oct 1 10:45:01 2023","interfaces":[{"interface":"back","average":{"1min":2.5350000000000001,"5min":2.3919999999999999,"15min":1.8819999999999999},"min":{"1min":0.42899999999999999,"5min":0.38900000000000001,"15min":0.35799999999999998},"max":{"1min":7.6210000000000004,"5min":61.560000000000002,"15min":61.560000000000002},"last":1.109},{"interface":"front","average":{"1min":1.496,"5min":2.0590000000000002,"15min":2.0870000000000002},"min":{"1min":0.37,"5min":0.34699999999999998,"15min":0.33600000000000002},"max":{"1min":9.2680000000000007,"5min":61.667999999999999,"15min":61.667999999999999},"last":1.044}]},{"osd":6,"last update":"Sun Oct 1 10:45:01 2023","interfaces":[{"interface":"back","average":{"1min":1.143,"5min":1.8859999999999999,"15min":1.8320000000000001},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.37},"max":{"1min":3.4249999999999998,"5min":62.219999999999999,"15min":62.219999999999999},"last":0.99099999999999999},{"interface":"front","average":{"1min":2.1030000000000002,"5min":2.1440000000000001,"15min":1.907},"min":{"1min":0.47099999999999997,"5min":0.439,"15min":0.41999999999999998},"max":{"1min":13.195,"5min":61.462000000000003,"15min":61.462000000000003},"last":1.012}]},{"osd":7,"last update":"Sun Oct 1 10:45:01 2023","interfaces":[{"interface":"back","average":{"1min":3.3999999999999999,"5min":2.6960000000000002,"15min":2.0680000000000001},"min":{"1min":0.57399999999999995,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":13.164999999999999,"5min":62.646999999999998,"15min":62.646999999999998},"last":0.96499999999999997},{"interface":"front","average":{"1min":2.4260000000000002,"5min":2.3559999999999999,"15min":1.8919999999999999},"min":{"1min":0.48699999999999999,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":6.7560000000000002,"5min":62.048000000000002,"15min":62.048000000000002},"last":1.071}]}]},{"osd":0,"up_from":86,"seq":369367187669,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":402532,"kb_used_data":2140,"kb_used_omap":5,"kb_used_meta":400378,"kb_avail":93342620,"statfs":{"total":95995035648,"available":95582842880,"internally_reserved":0,"allocated":2191360,"data_stored":1562579,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":409987706},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 10:44:56 2023","interfaces":[{"interface":"back","average":{"1min":2.27,"5min":1.6399999999999999,"15min":1.6180000000000001},"min":{"1min":0.24399999999999999,"5min":0.184,"15min":0.184},"max":{"1min":15.074,"5min":15.074,"15min":15.074},"last":0.55000000000000004},{"interface":"front","average":{"1min":2.3450000000000002,"5min":1.837,"15min":1.9370000000000001},"min":{"1min":0.26900000000000002,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":8.2110000000000003,"5min":11.815,"15min":15.48},"last":0.57299999999999995}]},{"osd":2,"last update":"Sun Oct 1 10:45:25 2023","interfaces":[{"interface":"back","average":{"1min":0.45900000000000002,"5min":1.022,"15min":0.80600000000000005},"min":{"1min":0.26600000000000001,"5min":0.223,"15min":0.223},"max":{"1min":0.64900000000000002,"5min":9.2420000000000009,"15min":9.2420000000000009},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.91700000000000004,"15min":0.80500000000000005},"min":{"1min":0.312,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":1.268,"5min":4.274,"15min":4.274},"last":0.60999999999999999}]},{"osd":3,"last update":"Sun Oct 1 10:44:56 2023","interfaces":[{"interface":"back","average":{"1min":2.3439999999999999,"5min":1.7030000000000001,"15min":1.1020000000000001},"min":{"1min":0.35799999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":9.4789999999999992,"5min":9.4789999999999992,"15min":9.4789999999999992},"last":0.70299999999999996},{"interface":"front","average":{"1min":2.0659999999999998,"5min":1.49,"15min":1.0349999999999999},"min":{"1min":0.33600000000000002,"5min":0.245,"15min":0.245},"max":{"1min":10.984999999999999,"5min":10.984999999999999,"15min":10.984999999999999},"last":0.68400000000000005}]},{"osd":4,"last update":"Sun Oct 1 10:45:36 2023","interfaces":[{"interface":"back","average":{"1min":1.1839999999999999,"5min":1.2210000000000001,"15min":1.085},"min":{"1min":0.45100000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":3.4529999999999998,"5min":9.0980000000000008,"15min":9.0980000000000008},"last":1.1739999999999999},{"interface":"front","average":{"1min":1.486,"5min":1.274,"15min":1.1140000000000001},"min":{"1min":0.45800000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":8.1479999999999997,"5min":8.1479999999999997,"15min":8.1479999999999997},"last":1.2090000000000001}]},{"osd":5,"last update":"Sun Oct 1 10:44:56 2023","interfaces":[{"interface":"back","average":{"1min":1.3169999999999999,"5min":1.5780000000000001,"15min":1.603},"min":{"1min":0.37,"5min":0.35799999999999998,"15min":0.31900000000000001},"max":{"1min":8.9209999999999994,"5min":10.672000000000001,"15min":13.068},"last":0.91400000000000003},{"interface":"front","average":{"1min":1.823,"5min":1.5069999999999999,"15min":1.589},"min":{"1min":0.44500000000000001,"5min":0.34699999999999998,"15min":0.34200000000000003},"max":{"1min":7.758,"5min":9.1440000000000001,"15min":11.747},"last":1.127}]},{"osd":6,"last update":"Sun Oct 1 10:44:56 2023","interfaces":[{"interface":"back","average":{"1min":1.1859999999999999,"5min":1.2110000000000001,"15min":1.587},"min":{"1min":0.373,"5min":0.36399999999999999,"15min":0.35899999999999999},"max":{"1min":5.4340000000000002,"5min":9.1780000000000008,"15min":11.663},"last":1.2290000000000001},{"interface":"front","average":{"1min":1.3620000000000001,"5min":1.169,"15min":1.528},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":8.2569999999999997,"5min":8.2569999999999997,"15min":15.069000000000001},"last":1.141}]},{"osd":7,"last update":"Sun Oct 1 10:44:56 2023","interfaces":[{"interface":"back","average":{"1min":1.3069999999999999,"5min":1.5880000000000001,"15min":1.6419999999999999},"min":{"1min":0.45200000000000001,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":5.4589999999999996,"5min":9.2059999999999995,"15min":12.686},"last":1.0760000000000001},{"interface":"front","average":{"1min":1.887,"5min":1.587,"15min":1.589},"min":{"1min":0.434,"5min":0.434,"15min":0.36199999999999999},"max":{"1min":8.3100000000000005,"5min":8.3100000000000005,"15min":13.654},"last":1.2509999999999999}]}]},{"osd":2,"up_from":116,"seq":498216206444,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":192108,"kb_used_data":1508,"kb_used_omap":7,"kb_used_meta":190584,"kb_avail":93553044,"statfs":{"total":95995035648,"available":95798317056,"internally_reserved":0,"allocated":1544192,"data_stored":964034,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":195158097},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:45:12 2023","interfaces":[{"interface":"back","average":{"1min":1.272,"5min":0.97699999999999998,"15min":0.77400000000000002},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":4.1219999999999999,"5min":14.851000000000001,"15min":14.851000000000001},"last":0.47999999999999998},{"interface":"front","average":{"1min":1.2330000000000001,"5min":1.026,"15min":0.86299999999999999},"min":{"1min":0.29199999999999998,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":3.903,"5min":6.1829999999999998,"15min":6.1829999999999998},"last":0.75}]},{"osd":1,"last update":"Sun Oct 1 10:45:12 2023","interfaces":[{"interface":"back","average":{"1min":1.7949999999999999,"5min":1.3879999999999999,"15min":0.94499999999999995},"min":{"1min":0.29299999999999998,"5min":0.219,"15min":0.219},"max":{"1min":10.565,"5min":10.565,"15min":10.565},"last":0.89400000000000002},{"interface":"front","average":{"1min":1.325,"5min":1.1759999999999999,"15min":0.91100000000000003},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":5.2759999999999998,"5min":8.9209999999999994,"15min":8.9209999999999994},"last":0.54300000000000004}]},{"osd":3,"last update":"Sun Oct 1 10:44:51 2023","interfaces":[{"interface":"back","average":{"1min":1.113,"5min":1.671,"15min":1.046},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":4.4649999999999999,"5min":64.266000000000005,"15min":64.266000000000005},"last":0.60499999999999998},{"interface":"front","average":{"1min":1.1839999999999999,"5min":1.6599999999999999,"15min":1.137},"min":{"1min":0.28199999999999997,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":4.3380000000000001,"5min":67.620999999999995,"15min":67.620999999999995},"last":0.96799999999999997}]},{"osd":4,"last update":"Sun Oct 1 10:45:32 2023","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":1.0529999999999999,"15min":1.014},"min":{"1min":0.40300000000000002,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":3.6269999999999998,"5min":7.9580000000000002,"15min":7.9580000000000002},"last":1.145},{"interface":"front","average":{"1min":0.97399999999999998,"5min":0.89800000000000002,"15min":0.94399999999999995},"min":{"1min":0.40100000000000002,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":3.181,"5min":4.3559999999999999,"15min":4.3559999999999999},"last":0.80400000000000005}]},{"osd":5,"last update":"Sun Oct 1 10:45:12 2023","interfaces":[{"interface":"back","average":{"1min":1.637,"5min":1.383,"15min":1.204},"min":{"1min":0.375,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":5.9960000000000004,"5min":5.9960000000000004,"15min":5.9960000000000004},"last":1.292},{"interface":"front","average":{"1min":1.665,"5min":1.2789999999999999,"15min":1.1100000000000001},"min":{"1min":0.39200000000000002,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":6.0640000000000001,"5min":6.0640000000000001,"15min":6.0640000000000001},"last":0.86799999999999999}]},{"osd":6,"last update":"Sun Oct 1 10:45:12 2023","interfaces":[{"interface":"back","average":{"1min":1.411,"5min":0.95099999999999996,"15min":1.0449999999999999},"min":{"1min":0.41699999999999998,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":6.2240000000000002,"5min":6.2240000000000002,"15min":6.2240000000000002},"last":0.996},{"interface":"front","average":{"1min":0.95499999999999996,"5min":0.875,"15min":1.0529999999999999},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":3.2930000000000001,"5min":3.2930000000000001,"15min":3.2930000000000001},"last":0.88100000000000001}]},{"osd":7,"last update":"Sun Oct 1 10:45:12 2023","interfaces":[{"interface":"back","average":{"1min":1.137,"5min":1.1100000000000001,"15min":1.1240000000000001},"min":{"1min":0.44,"5min":0.373,"15min":0.373},"max":{"1min":6.0460000000000003,"5min":6.7869999999999999,"15min":6.7869999999999999},"last":0.92000000000000004},{"interface":"front","average":{"1min":2.0659999999999998,"5min":1.6359999999999999,"15min":1.355},"min":{"1min":0.501,"5min":0.376,"15min":0.376},"max":{"1min":4.4980000000000002,"5min":8.9269999999999996,"15min":8.9269999999999996},"last":1.3819999999999999}]}]},{"osd":3,"up_from":121,"seq":519691042917,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":223968,"kb_used_data":1496,"kb_used_omap":2,"kb_used_meta":222461,"kb_avail":93521184,"statfs":{"total":95995035648,"available":95765692416,"internally_reserved":0,"allocated":1531904,"data_stored":962506,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":227800117},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:45:55 2023","interfaces":[{"interface":"back","average":{"1min":0.97199999999999998,"5min":2.5710000000000002,"15min":1.3009999999999999},"min":{"1min":0.30499999999999999,"5min":0.187,"15min":0.187},"max":{"1min":5.8499999999999996,"5min":77.981999999999999,"15min":77.981999999999999},"last":0.64000000000000001},{"interface":"front","average":{"1min":1.304,"5min":2.8159999999999998,"15min":1.337},"min":{"1min":0.375,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":5.6120000000000001,"5min":77.281000000000006,"15min":77.281000000000006},"last":0.33500000000000002}]},{"osd":1,"last update":"Sun Oct 1 10:45:55 2023","interfaces":[{"interface":"back","average":{"1min":1.1299999999999999,"5min":2.8090000000000002,"15min":1.401},"min":{"1min":0.34899999999999998,"5min":0.222,"15min":0.222},"max":{"1min":6.1159999999999997,"5min":77.361000000000004,"15min":77.361000000000004},"last":0.54900000000000004},{"interface":"front","average":{"1min":1.0820000000000001,"5min":3.0529999999999999,"15min":1.536},"min":{"1min":0.40400000000000003,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":6.0970000000000004,"5min":77.867000000000004,"15min":77.867000000000004},"last":0.43099999999999999}]},{"osd":2,"last update":"Sun Oct 1 10:45:55 2023","interfaces":[{"interface":"back","average":{"1min":1.2649999999999999,"5min":1.3220000000000001,"15min":0.89700000000000002},"min":{"1min":0.41399999999999998,"5min":0.222,"15min":0.222},"max":{"1min":10.028,"5min":10.028,"15min":10.028},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.84799999999999998,"5min":1.9359999999999999,"15min":1.1739999999999999},"min":{"1min":0.39000000000000001,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":2.6429999999999998,"5min":75.299999999999997,"15min":75.299999999999997},"last":0.48799999999999999}]},{"osd":4,"last update":"Sun Oct 1 10:45:25 2023","interfaces":[{"interface":"back","average":{"1min":1.413,"5min":1.968,"15min":1.2889999999999999},"min":{"1min":0.373,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":6.1390000000000002,"5min":75.388999999999996,"15min":75.388999999999996},"last":0.83799999999999997},{"interface":"front","average":{"1min":1.556,"5min":2.1699999999999999,"15min":1.3500000000000001},"min":{"1min":0.40799999999999997,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":4.7699999999999996,"5min":75.165999999999997,"15min":75.165999999999997},"last":0.73399999999999999}]},{"osd":5,"last update":"Sun Oct 1 10:45:55 2023","interfaces":[{"interface":"back","average":{"1min":1.6890000000000001,"5min":2.722,"15min":1.5409999999999999},"min":{"1min":0.51600000000000001,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":9.5540000000000003,"5min":75.221000000000004,"15min":75.221000000000004},"last":0.65600000000000003},{"interface":"front","average":{"1min":1.5089999999999999,"5min":2.9510000000000001,"15min":1.6539999999999999},"min":{"1min":0.64000000000000001,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":10.43,"5min":75.231999999999999,"15min":75.231999999999999},"last":0.92600000000000005}]},{"osd":6,"last update":"Sun Oct 1 10:45:55 2023","interfaces":[{"interface":"back","average":{"1min":1.49,"5min":2.403,"15min":1.48},"min":{"1min":0.44900000000000001,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":4.8090000000000002,"5min":75.257000000000005,"15min":75.257000000000005},"last":0.80700000000000005},{"interface":"front","average":{"1min":1.677,"5min":2.488,"15min":1.5629999999999999},"min":{"1min":0.72899999999999998,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":10.066000000000001,"5min":75.194000000000003,"15min":75.194000000000003},"last":0.81899999999999995}]},{"osd":7,"last update":"Sun Oct 1 10:45:55 2023","interfaces":[{"interface":"back","average":{"1min":1.579,"5min":2.73,"15min":1.609},"min":{"1min":0.68400000000000005,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":11.782999999999999,"5min":75.138999999999996,"15min":75.138999999999996},"last":0.67100000000000004},{"interface":"front","average":{"1min":1.3029999999999999,"5min":2.4199999999999999,"15min":1.494},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":6.069,"5min":75.355000000000004,"15min":75.355000000000004},"last":0.77400000000000002}]}]},{"osd":4,"up_from":126,"seq":541165879390,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":179048,"kb_used_data":1504,"kb_used_omap":10,"kb_used_meta":177525,"kb_avail":93566104,"statfs":{"total":95995035648,"available":95811690496,"internally_reserved":0,"allocated":1540096,"data_stored":963483,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":181786212},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:45:25 2023","interfaces":[{"interface":"back","average":{"1min":1.0629999999999999,"5min":1.2330000000000001,"15min":1.0229999999999999},"min":{"1min":0.30599999999999999,"5min":0.251,"15min":0.251},"max":{"1min":3.8860000000000001,"5min":8.1329999999999991,"15min":8.1329999999999991},"last":0.58599999999999997},{"interface":"front","average":{"1min":1.157,"5min":1.143,"15min":1.036},"min":{"1min":0.32800000000000001,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":3.9449999999999998,"5min":6.7789999999999999,"15min":6.7789999999999999},"last":0.70099999999999996}]},{"osd":1,"last update":"Sun Oct 1 10:45:25 2023","interfaces":[{"interface":"back","average":{"1min":0.871,"5min":2.4060000000000001,"15min":1.423},"min":{"1min":0.33700000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":3.948,"5min":97.622,"15min":97.622},"last":0.96099999999999997},{"interface":"front","average":{"1min":0.96099999999999997,"5min":2.504,"15min":1.474},"min":{"1min":0.32900000000000001,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":4.101,"5min":97.346999999999994,"15min":97.346999999999994},"last":1.0980000000000001}]},{"osd":2,"last update":"Sun Oct 1 10:45:25 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.82999999999999996,"15min":0.91600000000000004},"min":{"1min":0.36499999999999999,"5min":0.31,"15min":0.31},"max":{"1min":1.278,"5min":5.5049999999999999,"15min":5.5049999999999999},"last":1.1970000000000001},{"interface":"front","average":{"1min":0.88800000000000001,"5min":0.97199999999999998,"15min":0.97899999999999998},"min":{"1min":0.36499999999999999,"5min":0.308,"15min":0.308},"max":{"1min":3.9769999999999999,"5min":6.8339999999999996,"15min":6.8339999999999996},"last":0.999}]},{"osd":3,"last update":"Sun Oct 1 10:45:25 2023","interfaces":[{"interface":"back","average":{"1min":1.272,"5min":3.0779999999999998,"15min":1.7390000000000001},"min":{"1min":0.438,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":4.0629999999999997,"5min":61.398000000000003,"15min":61.398000000000003},"last":0.90900000000000003},{"interface":"front","average":{"1min":1.042,"5min":2.8300000000000001,"15min":1.5940000000000001},"min":{"1min":0.38800000000000001,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":4.6070000000000002,"5min":61.463000000000001,"15min":61.463000000000001},"last":0.83899999999999997}]},{"osd":5,"last update":"Sun Oct 1 10:45:25 2023","interfaces":[{"interface":"back","average":{"1min":1.643,"5min":1.2849999999999999,"15min":1.153},"min":{"1min":0.377,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":5.0519999999999996,"5min":6.9340000000000002,"15min":6.9340000000000002},"last":1.0720000000000001},{"interface":"front","average":{"1min":1.0920000000000001,"5min":1.393,"15min":1.1950000000000001},"min":{"1min":0.46999999999999997,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":5.2530000000000001,"5min":16.779,"15min":16.779},"last":1.2609999999999999}]},{"osd":6,"last update":"Sun Oct 1 10:45:25 2023","interfaces":[{"interface":"back","average":{"1min":1.129,"5min":1.0109999999999999,"15min":1.093},"min":{"1min":0.42899999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":4.0220000000000002,"5min":6.9009999999999998,"15min":6.9009999999999998},"last":1.222},{"interface":"front","average":{"1min":1.3320000000000001,"5min":1.1619999999999999,"15min":1.1060000000000001},"min":{"1min":0.47299999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":4.5670000000000002,"5min":6.7930000000000001,"15min":6.7930000000000001},"last":1.0449999999999999}]},{"osd":7,"last update":"Sun Oct 1 10:45:25 2023","interfaces":[{"interface":"back","average":{"1min":1.006,"5min":1.163,"15min":1.1759999999999999},"min":{"1min":0.42699999999999999,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":4.7930000000000001,"5min":6.8360000000000003,"15min":6.8360000000000003},"last":1.1639999999999999},{"interface":"front","average":{"1min":1.5580000000000001,"5min":1.8999999999999999,"15min":1.415},"min":{"1min":0.442,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":3.9169999999999998,"5min":7.726,"15min":7.726},"last":1.3120000000000001}]}]},{"osd":5,"up_from":36,"seq":154618823071,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":897452,"kb_used_data":1380,"kb_used_omap":0,"kb_used_meta":896064,"kb_avail":92847700,"statfs":{"total":95995035648,"available":95076044800,"internally_reserved":0,"allocated":1413120,"data_stored":946003,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":917569536},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:45:48 2023","interfaces":[{"interface":"back","average":{"1min":1.1579999999999999,"5min":1.5940000000000001,"15min":1.409},"min":{"1min":0.35799999999999998,"5min":0.28299999999999997,"15min":0.27300000000000002},"max":{"1min":5.8049999999999997,"5min":52.094999999999999,"15min":52.094999999999999},"last":0.90400000000000003},{"interface":"front","average":{"1min":0.93999999999999995,"5min":1.9179999999999999,"15min":1.631},"min":{"1min":0.314,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":3.7490000000000001,"5min":52.048000000000002,"15min":52.048000000000002},"last":0.77400000000000002}]},{"osd":1,"last update":"Sun Oct 1 10:45:48 2023","interfaces":[{"interface":"back","average":{"1min":1.5,"5min":4.1900000000000004,"15min":2.4489999999999998},"min":{"1min":0.42699999999999999,"5min":0.29799999999999999,"15min":0.29099999999999998},"max":{"1min":6.0049999999999999,"5min":202.96600000000001,"15min":202.96600000000001},"last":0.72899999999999998},{"interface":"front","average":{"1min":1.6619999999999999,"5min":4.9379999999999997,"15min":2.7930000000000001},"min":{"1min":0.503,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":7.2859999999999996,"5min":203.05099999999999,"15min":203.05099999999999},"last":0.874}]},{"osd":2,"last update":"Sun Oct 1 10:45:22 2023","interfaces":[{"interface":"back","average":{"1min":1.0620000000000001,"5min":1.0600000000000001,"15min":1.006},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":5.7519999999999998,"5min":8.7110000000000003,"15min":8.7110000000000003},"last":0.98099999999999998},{"interface":"front","average":{"1min":2.3239999999999998,"5min":1.6579999999999999,"15min":1.1990000000000001},"min":{"1min":0.38200000000000001,"5min":0.317,"15min":0.317},"max":{"1min":11.266,"5min":11.266,"15min":11.266},"last":1.1140000000000001}]},{"osd":3,"last update":"Sun Oct 1 10:45:02 2023","interfaces":[{"interface":"back","average":{"1min":5.5190000000000001,"5min":4.2590000000000003,"15min":2.0449999999999999},"min":{"1min":0.44500000000000001,"5min":0.316,"15min":0.316},"max":{"1min":52.482999999999997,"5min":70.506,"15min":70.506},"last":0.995},{"interface":"front","average":{"1min":5.3220000000000001,"5min":3.831,"15min":1.907},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":53.152000000000001,"5min":70.346999999999994,"15min":70.346999999999994},"last":0.80200000000000005}]},{"osd":4,"last update":"Sun Oct 1 10:45:27 2023","interfaces":[{"interface":"back","average":{"1min":1.986,"5min":1.466,"15min":1.2090000000000001},"min":{"1min":0.39500000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":15.170999999999999,"5min":15.170999999999999,"15min":15.170999999999999},"last":0.94899999999999995},{"interface":"front","average":{"1min":1.165,"5min":1.089,"15min":1.091},"min":{"1min":0.45900000000000002,"5min":0.376,"15min":0.376},"max":{"1min":5.6970000000000001,"5min":8.6329999999999991,"15min":8.6329999999999991},"last":1.0600000000000001}]},{"osd":6,"last update":"Sun Oct 1 10:45:02 2023","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":1.129,"15min":1.2110000000000001},"min":{"1min":0.41899999999999998,"5min":0.39200000000000002,"15min":0.38500000000000001},"max":{"1min":3.7570000000000001,"5min":8.7420000000000009,"15min":8.7420000000000009},"last":1.141},{"interface":"front","average":{"1min":1.6060000000000001,"5min":1.54,"15min":1.3600000000000001},"min":{"1min":0.39300000000000002,"5min":0.38300000000000001,"15min":0.312},"max":{"1min":6.7690000000000001,"5min":7.8010000000000002,"15min":8.4339999999999993},"last":1.1870000000000001}]},{"osd":7,"last update":"Sun Oct 1 10:45:02 2023","interfaces":[{"interface":"back","average":{"1min":2.585,"5min":2.129,"15min":1.6060000000000001},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.34699999999999998},"max":{"1min":12.757999999999999,"5min":12.757999999999999,"15min":12.757999999999999},"last":1.079},{"interface":"front","average":{"1min":2.4300000000000002,"5min":1.859,"15min":1.444},"min":{"1min":0.55100000000000005,"5min":0.53200000000000003,"15min":0.372},"max":{"1min":11.635,"5min":11.635,"15min":11.635},"last":1.2290000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T10:45:59.117 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T10:45:59.143 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:59.190 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:45:59.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:59.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:45:59.294 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:59.302 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:59 smithi078 bash[75299]: cluster 2023-10-01T10:45:57.827576+0000 mgr.y (mgr.25072) 848 : cluster 0 pgmap v774: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:45:59.302 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:59 smithi078 bash[75299]: cluster 2023-10-01T10:45:58.035253+0000 osd.7 (osd.7) 103 : cluster 3 failed to encode map e153 with expected crc 2023-10-01T10:45:59.302 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:45:59 smithi078 bash[75299]: cluster 2023-10-01T10:45:58.038230+0000 mon.a (mon.0) 1077 : cluster 0 osdmap e153: 8 total, 8 up, 8 in 2023-10-01T10:45:59.303 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:59 smithi078 bash[78274]: cluster 2023-10-01T10:45:57.827576+0000 mgr.y (mgr.25072) 848 : cluster 0 pgmap v774: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:45:59.303 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:59 smithi078 bash[78274]: cluster 2023-10-01T10:45:58.035253+0000 osd.7 (osd.7) 103 : cluster 3 failed to encode map e153 with expected crc 2023-10-01T10:45:59.303 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:45:59 smithi078 bash[78274]: cluster 2023-10-01T10:45:58.038230+0000 mon.a (mon.0) 1077 : cluster 0 osdmap e153: 8 total, 8 up, 8 in 2023-10-01T10:45:59.317 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:59.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:59.320 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:45:59.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:59 smithi173 bash[58816]: cluster 2023-10-01T10:45:57.827576+0000 mgr.y (mgr.25072) 848 : cluster 0 pgmap v774: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:45:59.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:59 smithi173 bash[58816]: cluster 2023-10-01T10:45:58.035253+0000 osd.7 (osd.7) 103 : cluster 3 failed to encode map e153 with expected crc 2023-10-01T10:45:59.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:45:59 smithi173 bash[58816]: cluster 2023-10-01T10:45:58.038230+0000 mon.a (mon.0) 1077 : cluster 0 osdmap e153: 8 total, 8 up, 8 in 2023-10-01T10:45:59.421 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:59.454 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:45:59.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:59.457 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:45:59.534 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:45:59.557 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:59.588 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:59.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:59.592 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:59.692 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:59.725 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:45:59.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:59.729 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:45:59.830 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:59.863 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:45:59.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:45:59.867 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:45:59.968 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:00.002 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:00.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:00.005 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:46:00.106 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:00.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:00 smithi173 bash[58816]: cluster 2023-10-01T10:45:58.549145+0000 osd.5 (osd.5) 82 : cluster 3 failed to encode map e153 with expected crc 2023-10-01T10:46:00.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:00 smithi173 bash[58816]: cluster 2023-10-01T10:45:58.552190+0000 osd.5 (osd.5) 83 : cluster 3 failed to encode map e153 with expected crc 2023-10-01T10:46:00.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:00 smithi173 bash[58816]: cluster 2023-10-01T10:45:58.554296+0000 osd.5 (osd.5) 84 : cluster 3 failed to encode map e153 with expected crc 2023-10-01T10:46:00.131 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:00.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:00.134 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:00.234 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:00.265 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:00.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:00.269 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:46:00.269 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:00 smithi078 bash[78274]: cluster 2023-10-01T10:45:58.549145+0000 osd.5 (osd.5) 82 : cluster 3 failed to encode map e153 with expected crc 2023-10-01T10:46:00.269 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:00 smithi078 bash[78274]: cluster 2023-10-01T10:45:58.552190+0000 osd.5 (osd.5) 83 : cluster 3 failed to encode map e153 with expected crc 2023-10-01T10:46:00.269 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:00 smithi078 bash[78274]: cluster 2023-10-01T10:45:58.554296+0000 osd.5 (osd.5) 84 : cluster 3 failed to encode map e153 with expected crc 2023-10-01T10:46:00.270 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:00 smithi078 bash[75299]: cluster 2023-10-01T10:45:58.549145+0000 osd.5 (osd.5) 82 : cluster 3 failed to encode map e153 with expected crc 2023-10-01T10:46:00.270 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:00 smithi078 bash[75299]: cluster 2023-10-01T10:45:58.552190+0000 osd.5 (osd.5) 83 : cluster 3 failed to encode map e153 with expected crc 2023-10-01T10:46:00.270 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:00 smithi078 bash[75299]: cluster 2023-10-01T10:45:58.554296+0000 osd.5 (osd.5) 84 : cluster 3 failed to encode map e153 with expected crc 2023-10-01T10:46:00.369 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:00.401 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:00.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:00.406 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:46:00.507 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:00.519 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:46:00 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:46:00] "GET /metrics HTTP/1.1" 200 33891 "" "Prometheus/2.33.4" 2023-10-01T10:46:00.540 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:00.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:00.543 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:00.644 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:00.671 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:00.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:00.674 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:46:00.775 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:00.803 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:00.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:00.806 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:00.907 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:00.935 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:00.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:00.938 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:46:01.016 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T10:46:01.039 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:01.070 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:01.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:01.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:46:01.173 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:01.201 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:01.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:01.203 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:46:01.304 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:01.316 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:01 smithi078 bash[75299]: cluster 2023-10-01T10:45:59.304759+0000 osd.6 (osd.6) 61 : cluster 3 failed to encode map e153 with expected crc 2023-10-01T10:46:01.316 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:01 smithi078 bash[75299]: cluster 2023-10-01T10:45:59.828234+0000 mgr.y (mgr.25072) 849 : cluster 0 pgmap v776: 89 pgs: 1 unknown, 88 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:46:01.316 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:01 smithi078 bash[78274]: cluster 2023-10-01T10:45:59.304759+0000 osd.6 (osd.6) 61 : cluster 3 failed to encode map e153 with expected crc 2023-10-01T10:46:01.316 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:01 smithi078 bash[78274]: cluster 2023-10-01T10:45:59.828234+0000 mgr.y (mgr.25072) 849 : cluster 0 pgmap v776: 89 pgs: 1 unknown, 88 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:46:01.332 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:01.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:01.334 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:46:01.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:01 smithi173 bash[58816]: cluster 2023-10-01T10:45:59.304759+0000 osd.6 (osd.6) 61 : cluster 3 failed to encode map e153 with expected crc 2023-10-01T10:46:01.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:01 smithi173 bash[58816]: cluster 2023-10-01T10:45:59.828234+0000 mgr.y (mgr.25072) 849 : cluster 0 pgmap v776: 89 pgs: 1 unknown, 88 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:46:01.434 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:01.469 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:01.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:01.471 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:46:01.572 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:01.608 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:01.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:01.610 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:46:01.712 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:01.738 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:01.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:01.741 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:46:01.841 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:01.868 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:01.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:01.870 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:46:01.971 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:01.999 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:02.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:02.002 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:46:02.103 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:02.134 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:02.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:02.136 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:46:02.225 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T10:46:02.237 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:02.264 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:02.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:02.267 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:02.368 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:02.396 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:02.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:02.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:46:02.499 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:02.526 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:02.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:02.529 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:46:02.630 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:02.636 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:46:02.655 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:02.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:02.658 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:46:02.759 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:02.788 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:02.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:02.791 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:46:02.892 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:02.917 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:02.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:02.920 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:46:03.020 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:03.049 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:03.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:03.052 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:46:03.153 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:03.186 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:03.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:03.188 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:46:03.288 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:03.316 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:03.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:03.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:46:03.420 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:03.443 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:03.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:03.445 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:03.545 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:03.572 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:03.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:03.574 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:46:03.675 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:03.689 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:03 smithi173 bash[58816]: cluster 2023-10-01T10:46:01.828893+0000 mgr.y (mgr.25072) 850 : cluster 0 pgmap v777: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:46:03.689 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:03 smithi173 bash[58816]: audit 2023-10-01T10:46:02.421225+0000 mon.a (mon.0) 1078 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:46:03.712 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:03.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:03.715 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:46:03.816 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:03.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:03 smithi078 bash[75299]: cluster 2023-10-01T10:46:01.828893+0000 mgr.y (mgr.25072) 850 : cluster 0 pgmap v777: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:46:03.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:03 smithi078 bash[75299]: audit 2023-10-01T10:46:02.421225+0000 mon.a (mon.0) 1078 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:46:03.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:03 smithi078 bash[78274]: cluster 2023-10-01T10:46:01.828893+0000 mgr.y (mgr.25072) 850 : cluster 0 pgmap v777: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:46:03.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:03 smithi078 bash[78274]: audit 2023-10-01T10:46:02.421225+0000 mon.a (mon.0) 1078 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:46:03.848 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:03.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:03.851 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:46:03.952 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:03.983 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:03.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:03.985 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:46:04.086 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:04.118 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:04.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:04.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:46:04.222 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:04.253 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:04.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:04.256 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:46:04.356 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:04.384 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:04.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:04.387 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:46:04.411 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T10:46:04.438 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T10:46:04.459 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T10:46:04.487 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T10:46:04.488 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:04.513 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:04.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:04.516 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:46:04.517 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T10:46:04.543 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T10:46:04.570 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T10:46:04.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:04 smithi078 bash[78274]: audit 2023-10-01T10:46:04.382931+0000 mon.c (mon.2) 151 : audit 0 from='client.? 172.21.15.78:0/1167073158' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:46:04.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:04 smithi078 bash[75299]: audit 2023-10-01T10:46:04.382931+0000 mon.c (mon.2) 151 : audit 0 from='client.? 172.21.15.78:0/1167073158' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:46:04.598 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T10:46:04.600 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T10:46:04.600 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T10:46:04.600 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T10:46:04.600 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T10:46:04.600 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T10:46:04.600 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T10:46:04.601 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T10:46:04.601 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T10:46:04.617 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:04.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:04 smithi173 bash[58816]: audit 2023-10-01T10:46:04.382931+0000 mon.c (mon.2) 151 : audit 0 from='client.? 172.21.15.78:0/1167073158' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:46:04.637 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:04.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:04.639 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:46:04.664 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:46:04.740 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:04.762 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:04.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:04.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:46:04.866 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:04.891 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:04.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:04.893 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:04.994 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:05.022 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:05.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:05.025 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:46:05.126 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:05.155 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:05.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:05.158 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:46:05.259 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:05.280 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:05.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:05.283 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:05.383 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:05.411 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:05.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:05.414 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:46:05.515 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:05.527 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:05 smithi078 bash[78274]: cluster 2023-10-01T10:46:03.829819+0000 mgr.y (mgr.25072) 851 : cluster 0 pgmap v778: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:46:05.527 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:05 smithi078 bash[75299]: cluster 2023-10-01T10:46:03.829819+0000 mgr.y (mgr.25072) 851 : cluster 0 pgmap v778: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:46:05.546 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:05.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:05.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:05.650 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:05.678 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:05.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:05.680 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:46:05.681 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-10-01T10:46:05.681 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T10:46:05.704 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:05 smithi173 bash[58816]: cluster 2023-10-01T10:46:03.829819+0000 mgr.y (mgr.25072) 851 : cluster 0 pgmap v778: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:46:05.781 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:05.811 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:05.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:05.815 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:46:05.915 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:05.946 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:05.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:05.949 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:46:06.049 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:06.077 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:06.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:06.079 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:46:06.180 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:06.211 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:06.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:06.213 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:46:06.314 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:06.337 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:06.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:06.339 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:06.439 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:06.466 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:06.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:06.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:06.569 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:06.584 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-01T10:46:06.595 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:06.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:06.598 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:46:06.698 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:06.721 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:06.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:06.723 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:46:06.824 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:06.852 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:06.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:06.856 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:46:06.956 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:06.986 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:06.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:06.989 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:46:07.090 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:07.120 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:07.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:07.123 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:07.224 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:07.258 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:07.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:07.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:46:07.362 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:07.388 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:07.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:07.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:07.491 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:07.503 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:07 smithi173 bash[58816]: cluster 2023-10-01T10:46:05.830794+0000 mgr.y (mgr.25072) 852 : cluster 0 pgmap v779: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:46:07.523 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:07.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:07.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:46:07.627 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:07.637 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:07 smithi078 bash[75299]: cluster 2023-10-01T10:46:05.830794+0000 mgr.y (mgr.25072) 852 : cluster 0 pgmap v779: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:46:07.637 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:07 smithi078 bash[78274]: cluster 2023-10-01T10:46:05.830794+0000 mgr.y (mgr.25072) 852 : cluster 0 pgmap v779: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:46:07.653 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:07.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:07.656 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:46:07.757 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:07.790 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:07.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:07.793 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:46:07.894 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:07.915 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:07.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:07.917 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:46:08.018 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:08.039 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:08.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:08.042 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:08.142 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:08.178 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:08.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:08.181 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:46:08.282 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:08.315 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:08.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:08.318 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:46:08.418 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:08.448 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:08.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:08.450 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:46:08.551 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:08.578 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:08.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:08.581 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:46:08.681 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:08.703 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:08.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:08.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:46:08.778 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:46:08.806 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:08.819 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:08 smithi078 bash[75299]: audit 2023-10-01T10:46:07.794782+0000 mon.a (mon.0) 1079 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:46:08.819 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:08 smithi078 bash[78274]: audit 2023-10-01T10:46:07.794782+0000 mon.a (mon.0) 1079 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:46:08.835 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:08.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:08.838 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:46:08.938 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:08.964 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:08.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:08.966 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:46:09.066 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:09.091 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:09.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:09.093 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:46:09.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:08 smithi173 bash[58816]: audit 2023-10-01T10:46:07.794782+0000 mon.a (mon.0) 1079 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:46:09.193 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:09.218 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:09.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:09.220 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:09.321 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:09.350 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:09.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:09.352 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:46:09.453 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:09.472 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:09.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:09.475 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:46:09.575 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:09.606 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:09.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:09.609 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:46:09.710 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:09.744 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:09.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:09.747 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:46:09.817 INFO:teuthology.orchestra.run.smithi078.stderr:enabled application 'rados' on pool 'unique_pool_5' 2023-10-01T10:46:09.848 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:09.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:09 smithi173 bash[58816]: cluster 2023-10-01T10:46:07.831450+0000 mgr.y (mgr.25072) 853 : cluster 0 pgmap v780: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:46:09.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:09 smithi173 bash[58816]: audit 2023-10-01T10:46:09.424881+0000 mon.c (mon.2) 152 : audit 1 from='client.? 172.21.15.78:0/3660524297' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T10:46:09.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:09 smithi173 bash[58816]: audit 2023-10-01T10:46:09.425369+0000 mon.a (mon.0) 1080 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T10:46:09.872 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:09.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:09.875 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:46:09.975 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:10.008 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:10.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:10.011 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:46:10.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:09 smithi078 bash[75299]: cluster 2023-10-01T10:46:07.831450+0000 mgr.y (mgr.25072) 853 : cluster 0 pgmap v780: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:46:10.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:09 smithi078 bash[75299]: audit 2023-10-01T10:46:09.424881+0000 mon.c (mon.2) 152 : audit 1 from='client.? 172.21.15.78:0/3660524297' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T10:46:10.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:09 smithi078 bash[75299]: audit 2023-10-01T10:46:09.425369+0000 mon.a (mon.0) 1080 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T10:46:10.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:09 smithi078 bash[78274]: cluster 2023-10-01T10:46:07.831450+0000 mgr.y (mgr.25072) 853 : cluster 0 pgmap v780: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:46:10.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:09 smithi078 bash[78274]: audit 2023-10-01T10:46:09.424881+0000 mon.c (mon.2) 152 : audit 1 from='client.? 172.21.15.78:0/3660524297' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T10:46:10.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:09 smithi078 bash[78274]: audit 2023-10-01T10:46:09.425369+0000 mon.a (mon.0) 1080 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T10:46:10.112 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:10.145 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:10.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:10.147 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:46:10.248 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:10.282 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:10.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:10.285 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:46:10.295 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T10:46:10.386 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:10.422 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:10.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:10.426 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:46:10.527 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:10.534 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:46:10 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:46:10] "GET /metrics HTTP/1.1" 200 36230 "" "Prometheus/2.33.4" 2023-10-01T10:46:10.547 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:10.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:10.549 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:46:10.650 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:10.684 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:10.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:10.688 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:46:10.789 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:10.820 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:10.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:10.823 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:46:10.924 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:10.932 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:10 smithi078 bash[78274]: cluster 2023-10-01T10:46:09.806620+0000 osd.5 (osd.5) 85 : cluster 3 failed to encode map e154 with expected crc 2023-10-01T10:46:10.933 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:10 smithi078 bash[78274]: audit 2023-10-01T10:46:09.810640+0000 mon.a (mon.0) 1081 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-01T10:46:10.933 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:10 smithi078 bash[78274]: cluster 2023-10-01T10:46:09.810858+0000 mon.a (mon.0) 1082 : cluster 0 osdmap e154: 8 total, 8 up, 8 in 2023-10-01T10:46:10.933 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:10 smithi078 bash[78274]: cluster 2023-10-01T10:46:09.832077+0000 mgr.y (mgr.25072) 854 : cluster 0 pgmap v782: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:46:10.933 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:10 smithi078 bash[75299]: cluster 2023-10-01T10:46:09.806620+0000 osd.5 (osd.5) 85 : cluster 3 failed to encode map e154 with expected crc 2023-10-01T10:46:10.933 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:10 smithi078 bash[75299]: audit 2023-10-01T10:46:09.810640+0000 mon.a (mon.0) 1081 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-01T10:46:10.933 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:10 smithi078 bash[75299]: cluster 2023-10-01T10:46:09.810858+0000 mon.a (mon.0) 1082 : cluster 0 osdmap e154: 8 total, 8 up, 8 in 2023-10-01T10:46:10.933 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:10 smithi078 bash[75299]: cluster 2023-10-01T10:46:09.832077+0000 mgr.y (mgr.25072) 854 : cluster 0 pgmap v782: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:46:10.948 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:10.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:10.951 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:46:11.052 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:11.081 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:11.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:11.084 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:46:11.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:10 smithi173 bash[58816]: cluster 2023-10-01T10:46:09.806620+0000 osd.5 (osd.5) 85 : cluster 3 failed to encode map e154 with expected crc 2023-10-01T10:46:11.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:10 smithi173 bash[58816]: audit 2023-10-01T10:46:09.810640+0000 mon.a (mon.0) 1081 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-01T10:46:11.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:10 smithi173 bash[58816]: cluster 2023-10-01T10:46:09.810858+0000 mon.a (mon.0) 1082 : cluster 0 osdmap e154: 8 total, 8 up, 8 in 2023-10-01T10:46:11.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:10 smithi173 bash[58816]: cluster 2023-10-01T10:46:09.832077+0000 mgr.y (mgr.25072) 854 : cluster 0 pgmap v782: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:46:11.184 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:11.215 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:11.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:11.218 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:46:11.319 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:11.350 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:11.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:11.352 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:46:11.358 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T10:46:11.453 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:11.481 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:11.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:11.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:46:11.584 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:11.609 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:11.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:11.612 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:11.712 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:11.745 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:11.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:11.747 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:11.848 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:11.878 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:11.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:11.882 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:46:11.983 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:12.012 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:12.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:12.015 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:46:12.115 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:12.143 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:12.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:12.145 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:46:12.246 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:12.270 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:12.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:12.273 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:46:12.373 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:12.400 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:12.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:12.402 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:12.503 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:12.529 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:12.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:12.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:46:12.632 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:12.654 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:12.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:12.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:46:12.757 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:12.784 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:12.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:12.786 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:46:12.887 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:12.915 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:12.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:12.917 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:13.004 DEBUG:teuthology.orchestra.run.smithi078:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_5 bench 90 write' 2023-10-01T10:46:13.006 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-01T10:46:13.006 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-01T10:46:13.017 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:13.046 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:13.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:13.049 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:13.149 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:13.173 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:13.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:13.176 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:46:13.277 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:13.290 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:hints = 1 2023-10-01T10:46:13.290 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-01T10:46:13.290 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Object prefix: benchmark_data_smithi078_135100 2023-10-01T10:46:13.290 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T10:46:13.291 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 0 0 0 0 0 0 - 0 2023-10-01T10:46:13.315 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:13.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:13.317 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:46:13.418 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:13.448 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:13.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:13.451 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:46:13.552 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:13.579 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:13.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:13.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:46:13.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:13 smithi173 bash[58816]: cluster 2023-10-01T10:46:11.832773+0000 mgr.y (mgr.25072) 855 : cluster 0 pgmap v783: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:46:13.684 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:13.713 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:13.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:13.715 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:46:13.816 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:13.824 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:13 smithi078 bash[75299]: cluster 2023-10-01T10:46:11.832773+0000 mgr.y (mgr.25072) 855 : cluster 0 pgmap v783: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:46:13.824 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:13 smithi078 bash[78274]: cluster 2023-10-01T10:46:11.832773+0000 mgr.y (mgr.25072) 855 : cluster 0 pgmap v783: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-10-01T10:46:13.849 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:13.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:13.852 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:46:13.953 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:13.981 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:13.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:13.984 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:14.085 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:14.110 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:14.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:14.113 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:46:14.213 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:14.240 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:14.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:14.244 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:46:14.290 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 1 16 3038 3022 188.865 188.875 0.00366966 0.00527431 2023-10-01T10:46:14.345 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:14.372 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:14.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:14.376 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:46:14.477 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:14.503 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:14.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:14.506 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:46:14.606 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:14.639 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:14.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:14.642 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:46:14.743 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:14.768 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:14.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:14.775 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:14.877 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:14.905 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:14.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:14.908 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:46:15.009 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:15.042 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-10-01T10:46:15.042 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T10:46:15.045 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:15.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:15.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:46:15.078 INFO:teuthology.orchestra.run.smithi078.stdout: 2023-10-01T10:46:15.080 INFO:teuthology.orchestra.run.smithi078.stderr:dumped all 2023-10-01T10:46:15.148 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:15.178 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:15.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:15.180 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:15.280 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:15.291 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 2 16 5651 5635 176.051 163.312 0.00848767 0.00566355 2023-10-01T10:46:15.308 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:15.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:15.310 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:46:15.411 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:15.440 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:15.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:15.442 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:46:15.543 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:15.552 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:15 smithi173 bash[58816]: cluster 2023-10-01T10:46:13.833392+0000 mgr.y (mgr.25072) 856 : cluster 0 pgmap v784: 89 pgs: 89 active+clean; 14 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 1.4 MiB/s wr, 21 op/s 2023-10-01T10:46:15.576 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:15.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:15.579 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:46:15.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:15 smithi078 bash[78274]: cluster 2023-10-01T10:46:13.833392+0000 mgr.y (mgr.25072) 856 : cluster 0 pgmap v784: 89 pgs: 89 active+clean; 14 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 1.4 MiB/s wr, 21 op/s 2023-10-01T10:46:15.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:15 smithi078 bash[75299]: cluster 2023-10-01T10:46:13.833392+0000 mgr.y (mgr.25072) 856 : cluster 0 pgmap v784: 89 pgs: 89 active+clean; 14 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 1.4 MiB/s wr, 21 op/s 2023-10-01T10:46:15.680 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:15.704 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:15.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:15.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:46:15.808 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:15.840 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:15.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:15.843 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:46:15.943 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:15.977 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:15.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:15.980 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:46:16.081 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:16.107 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:16.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:16.110 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:46:16.211 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:16.238 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:16.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:16.240 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:16.291 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 3 16 8153 8137 169.489 156.375 0.00475641 0.00589037 2023-10-01T10:46:16.341 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:16.362 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:16.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:16.364 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:46:16.465 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:16.497 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:16.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:16.500 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:16.570 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:16 smithi078 bash[75299]: audit 2023-10-01T10:46:15.050809+0000 mgr.y (mgr.25072) 857 : audit 0 from='client.35865 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:46:16.571 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:16 smithi078 bash[75299]: audit 2023-10-01T10:46:15.970167+0000 mon.a (mon.0) 1083 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:46:16.601 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:16.631 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:16.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:16.635 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:46:16.736 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:16.742 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:16 smithi173 bash[58816]: audit 2023-10-01T10:46:15.050809+0000 mgr.y (mgr.25072) 857 : audit 0 from='client.35865 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:46:16.742 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:16 smithi173 bash[58816]: audit 2023-10-01T10:46:15.970167+0000 mon.a (mon.0) 1083 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:46:16.759 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:16.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:16.762 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:46:16.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:16 smithi078 bash[78274]: audit 2023-10-01T10:46:15.050809+0000 mgr.y (mgr.25072) 857 : audit 0 from='client.35865 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:46:16.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:16 smithi078 bash[78274]: audit 2023-10-01T10:46:15.970167+0000 mon.a (mon.0) 1083 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:46:16.863 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:16.885 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:16.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:16.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:46:16.988 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:17.019 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:17.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:17.021 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:46:17.121 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:17.149 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:17.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:17.152 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:17.253 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:17.288 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:17.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:17.290 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:46:17.291 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 4 16 10326 10310 161.067 135.812 0.00723967 0.00619905 2023-10-01T10:46:17.391 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:17.416 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:17.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:17.418 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:46:17.519 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:17.548 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:17.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:17.551 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:46:17.652 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:17.661 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:17 smithi173 bash[58816]: cluster 2023-10-01T10:46:15.833919+0000 mgr.y (mgr.25072) 858 : cluster 0 pgmap v785: 89 pgs: 89 active+clean; 58 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.8 MiB/s wr, 92 op/s 2023-10-01T10:46:17.662 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:17 smithi173 bash[58816]: audit 2023-10-01T10:46:16.597623+0000 mon.a (mon.0) 1084 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:46:17.662 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:17 smithi173 bash[58816]: audit 2023-10-01T10:46:16.607224+0000 mon.a (mon.0) 1085 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:46:17.662 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:17 smithi173 bash[58816]: audit 2023-10-01T10:46:17.421168+0000 mon.a (mon.0) 1086 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:46:17.681 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:17.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:17.683 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:46:17.743 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-01T10:46:17.784 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:17.794 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:17 smithi078 bash[75299]: cluster 2023-10-01T10:46:15.833919+0000 mgr.y (mgr.25072) 858 : cluster 0 pgmap v785: 89 pgs: 89 active+clean; 58 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.8 MiB/s wr, 92 op/s 2023-10-01T10:46:17.794 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:17 smithi078 bash[75299]: audit 2023-10-01T10:46:16.597623+0000 mon.a (mon.0) 1084 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:46:17.794 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:17 smithi078 bash[75299]: audit 2023-10-01T10:46:16.607224+0000 mon.a (mon.0) 1085 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:46:17.794 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:17 smithi078 bash[75299]: audit 2023-10-01T10:46:17.421168+0000 mon.a (mon.0) 1086 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:46:17.794 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:17 smithi078 bash[78274]: cluster 2023-10-01T10:46:15.833919+0000 mgr.y (mgr.25072) 858 : cluster 0 pgmap v785: 89 pgs: 89 active+clean; 58 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.8 MiB/s wr, 92 op/s 2023-10-01T10:46:17.794 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:17 smithi078 bash[78274]: audit 2023-10-01T10:46:16.597623+0000 mon.a (mon.0) 1084 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:46:17.794 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:17 smithi078 bash[78274]: audit 2023-10-01T10:46:16.607224+0000 mon.a (mon.0) 1085 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:46:17.795 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:17 smithi078 bash[78274]: audit 2023-10-01T10:46:17.421168+0000 mon.a (mon.0) 1086 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:46:17.808 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:17.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:17.810 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:46:17.911 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:17.940 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:17.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:17.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:46:18.044 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:18.067 INFO:teuthology.orchestra.run.smithi078.stdout:{"pg_ready":true,"pg_map":{"version":784,"stamp":"2023-10-01T10:46:13.832948+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":14817184,"num_objects":241,"num_object_clones":0,"num_object_copies":723,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":241,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":478,"num_write_kb":16334,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":412,"ondisk_log_size":412,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":3184344,"kb_used_data":27096,"kb_used_omap":30,"kb_used_meta":3157153,"kb_avail":746776872,"statfs":{"total":767960285184,"available":764699516928,"internally_reserved":0,"allocated":27746304,"data_stored":23255607,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31502,"internal_metadata":3232924914},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":14221312,"num_objects":217,"num_object_clones":0,"num_object_copies":651,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":217,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":217,"num_write_kb":13888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003800"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":142,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-01T10:45:57.040079+0000","last_change":"2023-10-01T10:38:16.032258+0000","last_active":"2023-10-01T10:45:57.040079+0000","last_peered":"2023-10-01T10:45:57.040079+0000","last_clean":"2023-10-01T10:45:57.040079+0000","last_became_active":"2023-10-01T10:38:16.031633+0000","last_became_peered":"2023-10-01T10:38:16.031633+0000","last_unstale":"2023-10-01T10:45:57.040079+0000","last_undegraded":"2023-10-01T10:45:57.040079+0000","last_fullsized":"2023-10-01T10:45:57.040079+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:17:02.446430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":130,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-01T10:45:57.037340+0000","last_change":"2023-10-01T10:38:16.025881+0000","last_active":"2023-10-01T10:45:57.037340+0000","last_peered":"2023-10-01T10:45:57.037340+0000","last_clean":"2023-10-01T10:45:57.037340+0000","last_became_active":"2023-10-01T10:38:16.025518+0000","last_became_peered":"2023-10-01T10:38:16.025518+0000","last_unstale":"2023-10-01T10:45:57.037340+0000","last_undegraded":"2023-10-01T10:45:57.037340+0000","last_fullsized":"2023-10-01T10:45:57.037340+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:48:37.826604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":109,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-01T10:45:58.036183+0000","last_change":"2023-10-01T10:38:16.025075+0000","last_active":"2023-10-01T10:45:58.036183+0000","last_peered":"2023-10-01T10:45:58.036183+0000","last_clean":"2023-10-01T10:45:58.036183+0000","last_became_active":"2023-10-01T10:38:16.024735+0000","last_became_peered":"2023-10-01T10:38:16.024735+0000","last_unstale":"2023-10-01T10:45:58.036183+0000","last_undegraded":"2023-10-01T10:45:58.036183+0000","last_fullsized":"2023-10-01T10:45:58.036183+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:38:01.450741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":113,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-01T10:45:58.191526+0000","last_change":"2023-10-01T10:37:42.772008+0000","last_active":"2023-10-01T10:45:58.191526+0000","last_peered":"2023-10-01T10:45:58.191526+0000","last_clean":"2023-10-01T10:45:58.191526+0000","last_became_active":"2023-10-01T10:37:42.771244+0000","last_became_peered":"2023-10-01T10:37:42.771244+0000","last_unstale":"2023-10-01T10:45:58.191526+0000","last_undegraded":"2023-10-01T10:45:58.191526+0000","last_fullsized":"2023-10-01T10:45:58.191526+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:38:55.180790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":131,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-01T10:45:57.036953+0000","last_change":"2023-10-01T10:37:42.766600+0000","last_active":"2023-10-01T10:45:57.036953+0000","last_peered":"2023-10-01T10:45:57.036953+0000","last_clean":"2023-10-01T10:45:57.036953+0000","last_became_active":"2023-10-01T10:37:42.766051+0000","last_became_peered":"2023-10-01T10:37:42.766051+0000","last_unstale":"2023-10-01T10:45:57.036953+0000","last_undegraded":"2023-10-01T10:45:57.036953+0000","last_fullsized":"2023-10-01T10:45:57.036953+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:51:17.707963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":119,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-01T10:45:59.309836+0000","last_change":"2023-10-01T10:32:31.575109+0000","last_active":"2023-10-01T10:45:59.309836+0000","last_peered":"2023-10-01T10:45:59.309836+0000","last_clean":"2023-10-01T10:45:59.309836+0000","last_became_active":"2023-10-01T10:32:31.574729+0000","last_became_peered":"2023-10-01T10:32:31.574729+0000","last_unstale":"2023-10-01T10:45:59.309836+0000","last_undegraded":"2023-10-01T10:45:59.309836+0000","last_fullsized":"2023-10-01T10:45:59.309836+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":111,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-01T10:45:58.191618+0000","last_change":"2023-10-01T10:37:42.766667+0000","last_active":"2023-10-01T10:45:58.191618+0000","last_peered":"2023-10-01T10:45:58.191618+0000","last_clean":"2023-10-01T10:45:58.191618+0000","last_became_active":"2023-10-01T10:37:42.766369+0000","last_became_peered":"2023-10-01T10:37:42.766369+0000","last_unstale":"2023-10-01T10:45:58.191618+0000","last_undegraded":"2023-10-01T10:45:58.191618+0000","last_fullsized":"2023-10-01T10:45:58.191618+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:02.708788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":115,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-01T10:45:57.036951+0000","last_change":"2023-10-01T10:37:42.765925+0000","last_active":"2023-10-01T10:45:57.036951+0000","last_peered":"2023-10-01T10:45:57.036951+0000","last_clean":"2023-10-01T10:45:57.036951+0000","last_became_active":"2023-10-01T10:37:42.765218+0000","last_became_peered":"2023-10-01T10:37:42.765218+0000","last_unstale":"2023-10-01T10:45:57.036951+0000","last_undegraded":"2023-10-01T10:45:57.036951+0000","last_fullsized":"2023-10-01T10:45:57.036951+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T22:10:34.338031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":120,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-01T10:45:57.044006+0000","last_change":"2023-10-01T10:32:31.579805+0000","last_active":"2023-10-01T10:45:57.044006+0000","last_peered":"2023-10-01T10:45:57.044006+0000","last_clean":"2023-10-01T10:45:57.044006+0000","last_became_active":"2023-10-01T10:32:31.579423+0000","last_became_peered":"2023-10-01T10:32:31.579423+0000","last_unstale":"2023-10-01T10:45:57.044006+0000","last_undegraded":"2023-10-01T10:45:57.044006+0000","last_fullsized":"2023-10-01T10:45:57.044006+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":99,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-01T10:45:57.037190+0000","last_change":"2023-10-01T10:37:42.772531+0000","last_active":"2023-10-01T10:45:57.037190+0000","last_peered":"2023-10-01T10:45:57.037190+0000","last_clean":"2023-10-01T10:45:57.037190+0000","last_became_active":"2023-10-01T10:37:42.772216+0000","last_became_peered":"2023-10-01T10:37:42.772216+0000","last_unstale":"2023-10-01T10:45:57.037190+0000","last_undegraded":"2023-10-01T10:45:57.037190+0000","last_fullsized":"2023-10-01T10:45:57.037190+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:01:53.373628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":139,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-01T10:45:58.043399+0000","last_change":"2023-10-01T10:38:16.027085+0000","last_active":"2023-10-01T10:45:58.043399+0000","last_peered":"2023-10-01T10:45:58.043399+0000","last_clean":"2023-10-01T10:45:58.043399+0000","last_became_active":"2023-10-01T10:38:16.026555+0000","last_became_peered":"2023-10-01T10:38:16.026555+0000","last_unstale":"2023-10-01T10:45:58.043399+0000","last_undegraded":"2023-10-01T10:45:58.043399+0000","last_fullsized":"2023-10-01T10:45:58.043399+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":94,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-01T10:45:58.038134+0000","last_change":"2023-10-01T10:37:05.018755+0000","last_active":"2023-10-01T10:45:58.038134+0000","last_peered":"2023-10-01T10:45:58.038134+0000","last_clean":"2023-10-01T10:45:58.038134+0000","last_became_active":"2023-10-01T10:37:05.018446+0000","last_became_peered":"2023-10-01T10:37:05.018446+0000","last_unstale":"2023-10-01T10:45:58.038134+0000","last_undegraded":"2023-10-01T10:45:58.038134+0000","last_fullsized":"2023-10-01T10:45:58.038134+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:13:53.740503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":136,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-01T10:45:57.040640+0000","last_change":"2023-10-01T10:38:16.030826+0000","last_active":"2023-10-01T10:45:57.040640+0000","last_peered":"2023-10-01T10:45:57.040640+0000","last_clean":"2023-10-01T10:45:57.040640+0000","last_became_active":"2023-10-01T10:38:16.030277+0000","last_became_peered":"2023-10-01T10:38:16.030277+0000","last_unstale":"2023-10-01T10:45:57.040640+0000","last_undegraded":"2023-10-01T10:45:57.040640+0000","last_fullsized":"2023-10-01T10:45:57.040640+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:31:43.040424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":110,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-01T10:45:57.041186+0000","last_change":"2023-10-01T10:38:16.034741+0000","last_active":"2023-10-01T10:45:57.041186+0000","last_peered":"2023-10-01T10:45:57.041186+0000","last_clean":"2023-10-01T10:45:57.041186+0000","last_became_active":"2023-10-01T10:38:16.034407+0000","last_became_peered":"2023-10-01T10:38:16.034407+0000","last_unstale":"2023-10-01T10:45:57.041186+0000","last_undegraded":"2023-10-01T10:45:57.041186+0000","last_fullsized":"2023-10-01T10:45:57.041186+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:47:29.911477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":138,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-01T10:45:58.043018+0000","last_change":"2023-10-01T10:38:16.026777+0000","last_active":"2023-10-01T10:45:58.043018+0000","last_peered":"2023-10-01T10:45:58.043018+0000","last_clean":"2023-10-01T10:45:58.043018+0000","last_became_active":"2023-10-01T10:38:16.026371+0000","last_became_peered":"2023-10-01T10:38:16.026371+0000","last_unstale":"2023-10-01T10:45:58.043018+0000","last_undegraded":"2023-10-01T10:45:58.043018+0000","last_fullsized":"2023-10-01T10:45:58.043018+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":138,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-01T10:45:57.037394+0000","last_change":"2023-10-01T10:37:42.772319+0000","last_active":"2023-10-01T10:45:57.037394+0000","last_peered":"2023-10-01T10:45:57.037394+0000","last_clean":"2023-10-01T10:45:57.037394+0000","last_became_active":"2023-10-01T10:37:42.771977+0000","last_became_peered":"2023-10-01T10:37:42.771977+0000","last_unstale":"2023-10-01T10:45:57.037394+0000","last_undegraded":"2023-10-01T10:45:57.037394+0000","last_fullsized":"2023-10-01T10:45:57.037394+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:45:12.783214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":111,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-01T10:45:58.191293+0000","last_change":"2023-10-01T10:37:42.767706+0000","last_active":"2023-10-01T10:45:58.191293+0000","last_peered":"2023-10-01T10:45:58.191293+0000","last_clean":"2023-10-01T10:45:58.191293+0000","last_became_active":"2023-10-01T10:37:42.767211+0000","last_became_peered":"2023-10-01T10:37:42.767211+0000","last_unstale":"2023-10-01T10:45:58.191293+0000","last_undegraded":"2023-10-01T10:45:58.191293+0000","last_fullsized":"2023-10-01T10:45:58.191293+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:37:20.357407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":119,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-01T10:45:57.043778+0000","last_change":"2023-10-01T10:37:42.768642+0000","last_active":"2023-10-01T10:45:57.043778+0000","last_peered":"2023-10-01T10:45:57.043778+0000","last_clean":"2023-10-01T10:45:57.043778+0000","last_became_active":"2023-10-01T10:37:42.768239+0000","last_became_peered":"2023-10-01T10:37:42.768239+0000","last_unstale":"2023-10-01T10:45:57.043778+0000","last_undegraded":"2023-10-01T10:45:57.043778+0000","last_fullsized":"2023-10-01T10:45:57.043778+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":121,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-01T10:45:58.042729+0000","last_change":"2023-10-01T10:38:16.028563+0000","last_active":"2023-10-01T10:45:58.042729+0000","last_peered":"2023-10-01T10:45:58.042729+0000","last_clean":"2023-10-01T10:45:58.042729+0000","last_became_active":"2023-10-01T10:38:16.028162+0000","last_became_peered":"2023-10-01T10:38:16.028162+0000","last_unstale":"2023-10-01T10:45:58.042729+0000","last_undegraded":"2023-10-01T10:45:58.042729+0000","last_fullsized":"2023-10-01T10:45:58.042729+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":112,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-01T10:45:58.191637+0000","last_change":"2023-10-01T10:37:42.772247+0000","last_active":"2023-10-01T10:45:58.191637+0000","last_peered":"2023-10-01T10:45:58.191637+0000","last_clean":"2023-10-01T10:45:58.191637+0000","last_became_active":"2023-10-01T10:37:42.771697+0000","last_became_peered":"2023-10-01T10:37:42.771697+0000","last_unstale":"2023-10-01T10:45:58.191637+0000","last_undegraded":"2023-10-01T10:45:58.191637+0000","last_fullsized":"2023-10-01T10:45:58.191637+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:23:15.018248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":128,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-01T10:45:59.308777+0000","last_change":"2023-10-01T10:28:20.211798+0000","last_active":"2023-10-01T10:45:59.308777+0000","last_peered":"2023-10-01T10:45:59.308777+0000","last_clean":"2023-10-01T10:45:59.308777+0000","last_became_active":"2023-10-01T10:28:20.210791+0000","last_became_peered":"2023-10-01T10:28:20.210791+0000","last_unstale":"2023-10-01T10:45:59.308777+0000","last_undegraded":"2023-10-01T10:45:59.308777+0000","last_fullsized":"2023-10-01T10:45:59.308777+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":115,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-01T10:45:57.037580+0000","last_change":"2023-10-01T10:37:42.773127+0000","last_active":"2023-10-01T10:45:57.037580+0000","last_peered":"2023-10-01T10:45:57.037580+0000","last_clean":"2023-10-01T10:45:57.037580+0000","last_became_active":"2023-10-01T10:37:42.772102+0000","last_became_peered":"2023-10-01T10:37:42.772102+0000","last_unstale":"2023-10-01T10:45:57.037580+0000","last_undegraded":"2023-10-01T10:45:57.037580+0000","last_fullsized":"2023-10-01T10:45:57.037580+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:23:26.202295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"0'0","reported_seq":10,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-01T10:45:57.054209+0000","last_change":"2023-10-01T10:45:57.053617+0000","last_active":"2023-10-01T10:45:57.054209+0000","last_peered":"2023-10-01T10:45:57.054209+0000","last_clean":"2023-10-01T10:45:57.054209+0000","last_became_active":"2023-10-01T10:45:57.053257+0000","last_became_peered":"2023-10-01T10:45:57.053257+0000","last_unstale":"2023-10-01T10:45:57.054209+0000","last_undegraded":"2023-10-01T10:45:57.054209+0000","last_fullsized":"2023-10-01T10:45:57.054209+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":151,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_clean_scrub_stamp":"2023-10-01T10:45:56.023726+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":137,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-01T10:45:58.043450+0000","last_change":"2023-10-01T10:38:16.026955+0000","last_active":"2023-10-01T10:45:58.043450+0000","last_peered":"2023-10-01T10:45:58.043450+0000","last_clean":"2023-10-01T10:45:58.043450+0000","last_became_active":"2023-10-01T10:38:16.026552+0000","last_became_peered":"2023-10-01T10:38:16.026552+0000","last_unstale":"2023-10-01T10:45:58.043450+0000","last_undegraded":"2023-10-01T10:45:58.043450+0000","last_fullsized":"2023-10-01T10:45:58.043450+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":104,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-01T10:45:57.043414+0000","last_change":"2023-10-01T10:15:40.318702+0000","last_active":"2023-10-01T10:45:57.043414+0000","last_peered":"2023-10-01T10:45:57.043414+0000","last_clean":"2023-10-01T10:45:57.043414+0000","last_became_active":"2023-10-01T10:15:40.318333+0000","last_became_peered":"2023-10-01T10:15:40.318333+0000","last_unstale":"2023-10-01T10:45:57.043414+0000","last_undegraded":"2023-10-01T10:45:57.043414+0000","last_fullsized":"2023-10-01T10:45:57.043414+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"154'25","reported_seq":34,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:13.377354+0000","last_change":"2023-10-01T10:45:57.056390+0000","last_active":"2023-10-01T10:46:13.377354+0000","last_peered":"2023-10-01T10:46:13.377354+0000","last_clean":"2023-10-01T10:46:13.377354+0000","last_became_active":"2023-10-01T10:45:57.055649+0000","last_became_peered":"2023-10-01T10:45:57.055649+0000","last_unstale":"2023-10-01T10:46:13.377354+0000","last_undegraded":"2023-10-01T10:46:13.377354+0000","last_fullsized":"2023-10-01T10:46:13.377354+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":151,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_clean_scrub_stamp":"2023-10-01T10:45:56.023726+0000","objects_scrubbed":0,"log_size":25,"log_dups_size":0,"ondisk_log_size":25,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1638400,"num_objects":25,"num_object_clones":0,"num_object_copies":75,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":25,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":25,"num_write_kb":1600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":138,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-01T10:45:58.043367+0000","last_change":"2023-10-01T10:38:16.027753+0000","last_active":"2023-10-01T10:45:58.043367+0000","last_peered":"2023-10-01T10:45:58.043367+0000","last_clean":"2023-10-01T10:45:58.043367+0000","last_became_active":"2023-10-01T10:38:16.027384+0000","last_became_peered":"2023-10-01T10:38:16.027384+0000","last_unstale":"2023-10-01T10:45:58.043367+0000","last_undegraded":"2023-10-01T10:45:58.043367+0000","last_fullsized":"2023-10-01T10:45:58.043367+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":144,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-01T10:45:57.040116+0000","last_change":"2023-10-01T10:38:16.033203+0000","last_active":"2023-10-01T10:45:57.040116+0000","last_peered":"2023-10-01T10:45:57.040116+0000","last_clean":"2023-10-01T10:45:57.040116+0000","last_became_active":"2023-10-01T10:38:16.032681+0000","last_became_peered":"2023-10-01T10:38:16.032681+0000","last_unstale":"2023-10-01T10:45:57.040116+0000","last_undegraded":"2023-10-01T10:45:57.040116+0000","last_fullsized":"2023-10-01T10:45:57.040116+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:42:36.642284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"154'23","reported_seq":35,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:13.371025+0000","last_change":"2023-10-01T10:45:57.044458+0000","last_active":"2023-10-01T10:46:13.371025+0000","last_peered":"2023-10-01T10:46:13.371025+0000","last_clean":"2023-10-01T10:46:13.371025+0000","last_became_active":"2023-10-01T10:45:57.043330+0000","last_became_peered":"2023-10-01T10:45:57.043330+0000","last_unstale":"2023-10-01T10:46:13.371025+0000","last_undegraded":"2023-10-01T10:46:13.371025+0000","last_fullsized":"2023-10-01T10:46:13.371025+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":151,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_clean_scrub_stamp":"2023-10-01T10:45:56.023726+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:19:44.671386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1507328,"num_objects":23,"num_object_clones":0,"num_object_copies":69,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":23,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":1472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":107,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-01T10:45:58.043229+0000","last_change":"2023-10-01T10:15:39.293587+0000","last_active":"2023-10-01T10:45:58.043229+0000","last_peered":"2023-10-01T10:45:58.043229+0000","last_clean":"2023-10-01T10:45:58.043229+0000","last_became_active":"2023-10-01T10:15:39.293273+0000","last_became_peered":"2023-10-01T10:15:39.293273+0000","last_unstale":"2023-10-01T10:45:58.043229+0000","last_undegraded":"2023-10-01T10:45:58.043229+0000","last_fullsized":"2023-10-01T10:45:58.043229+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":119,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-01T10:45:59.309325+0000","last_change":"2023-10-01T10:32:31.571971+0000","last_active":"2023-10-01T10:45:59.309325+0000","last_peered":"2023-10-01T10:45:59.309325+0000","last_clean":"2023-10-01T10:45:59.309325+0000","last_became_active":"2023-10-01T10:32:31.571410+0000","last_became_peered":"2023-10-01T10:32:31.571410+0000","last_unstale":"2023-10-01T10:45:59.309325+0000","last_undegraded":"2023-10-01T10:45:59.309325+0000","last_fullsized":"2023-10-01T10:45:59.309325+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"0'0","reported_seq":12,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-01T10:45:58.127836+0000","last_change":"2023-10-01T10:45:57.046146+0000","last_active":"2023-10-01T10:45:58.127836+0000","last_peered":"2023-10-01T10:45:58.127836+0000","last_clean":"2023-10-01T10:45:58.127836+0000","last_became_active":"2023-10-01T10:45:57.045852+0000","last_became_peered":"2023-10-01T10:45:57.045852+0000","last_unstale":"2023-10-01T10:45:58.127836+0000","last_undegraded":"2023-10-01T10:45:58.127836+0000","last_fullsized":"2023-10-01T10:45:58.127836+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":151,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_clean_scrub_stamp":"2023-10-01T10:45:56.023726+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:55:55.424584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":120,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-01T10:45:57.043550+0000","last_change":"2023-10-01T10:37:42.765733+0000","last_active":"2023-10-01T10:45:57.043550+0000","last_peered":"2023-10-01T10:45:57.043550+0000","last_clean":"2023-10-01T10:45:57.043550+0000","last_became_active":"2023-10-01T10:37:42.765311+0000","last_became_peered":"2023-10-01T10:37:42.765311+0000","last_unstale":"2023-10-01T10:45:57.043550+0000","last_undegraded":"2023-10-01T10:45:57.043550+0000","last_fullsized":"2023-10-01T10:45:57.043550+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":127,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-01T10:45:58.191132+0000","last_change":"2023-10-01T10:38:16.026348+0000","last_active":"2023-10-01T10:45:58.191132+0000","last_peered":"2023-10-01T10:45:58.191132+0000","last_clean":"2023-10-01T10:45:58.191132+0000","last_became_active":"2023-10-01T10:38:16.025747+0000","last_became_peered":"2023-10-01T10:38:16.025747+0000","last_unstale":"2023-10-01T10:45:58.191132+0000","last_undegraded":"2023-10-01T10:45:58.191132+0000","last_fullsized":"2023-10-01T10:45:58.191132+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:28:18.934480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"0'0","reported_seq":12,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-01T10:46:00.861031+0000","last_change":"2023-10-01T10:45:57.043636+0000","last_active":"2023-10-01T10:46:00.861031+0000","last_peered":"2023-10-01T10:46:00.861031+0000","last_clean":"2023-10-01T10:46:00.861031+0000","last_became_active":"2023-10-01T10:45:57.043317+0000","last_became_peered":"2023-10-01T10:45:57.043317+0000","last_unstale":"2023-10-01T10:46:00.861031+0000","last_undegraded":"2023-10-01T10:46:00.861031+0000","last_fullsized":"2023-10-01T10:46:00.861031+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":151,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_clean_scrub_stamp":"2023-10-01T10:45:56.023726+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:52:18.949427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":118,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-01T10:45:57.037336+0000","last_change":"2023-10-01T10:38:16.022493+0000","last_active":"2023-10-01T10:45:57.037336+0000","last_peered":"2023-10-01T10:45:57.037336+0000","last_clean":"2023-10-01T10:45:57.037336+0000","last_became_active":"2023-10-01T10:38:16.022033+0000","last_became_peered":"2023-10-01T10:38:16.022033+0000","last_unstale":"2023-10-01T10:45:57.037336+0000","last_undegraded":"2023-10-01T10:45:57.037336+0000","last_fullsized":"2023-10-01T10:45:57.037336+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:49:05.262763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":129,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-01T10:45:57.040810+0000","last_change":"2023-10-01T10:38:16.030613+0000","last_active":"2023-10-01T10:45:57.040810+0000","last_peered":"2023-10-01T10:45:57.040810+0000","last_clean":"2023-10-01T10:45:57.040810+0000","last_became_active":"2023-10-01T10:38:16.028975+0000","last_became_peered":"2023-10-01T10:38:16.028975+0000","last_unstale":"2023-10-01T10:45:57.040810+0000","last_undegraded":"2023-10-01T10:45:57.040810+0000","last_fullsized":"2023-10-01T10:45:57.040810+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:46:07.287911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"0'0","reported_seq":12,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-01T10:45:58.127812+0000","last_change":"2023-10-01T10:45:57.050946+0000","last_active":"2023-10-01T10:45:58.127812+0000","last_peered":"2023-10-01T10:45:58.127812+0000","last_clean":"2023-10-01T10:45:58.127812+0000","last_became_active":"2023-10-01T10:45:57.050609+0000","last_became_peered":"2023-10-01T10:45:57.050609+0000","last_unstale":"2023-10-01T10:45:58.127812+0000","last_undegraded":"2023-10-01T10:45:58.127812+0000","last_fullsized":"2023-10-01T10:45:58.127812+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":151,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_clean_scrub_stamp":"2023-10-01T10:45:56.023726+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:35:24.694811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":135,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-01T10:45:59.309575+0000","last_change":"2023-10-01T10:38:16.030250+0000","last_active":"2023-10-01T10:45:59.309575+0000","last_peered":"2023-10-01T10:45:59.309575+0000","last_clean":"2023-10-01T10:45:59.309575+0000","last_became_active":"2023-10-01T10:38:16.029898+0000","last_became_peered":"2023-10-01T10:38:16.029898+0000","last_unstale":"2023-10-01T10:45:59.309575+0000","last_undegraded":"2023-10-01T10:45:59.309575+0000","last_fullsized":"2023-10-01T10:45:59.309575+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":134,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-01T10:45:57.040923+0000","last_change":"2023-10-01T10:38:16.035906+0000","last_active":"2023-10-01T10:45:57.040923+0000","last_peered":"2023-10-01T10:45:57.040923+0000","last_clean":"2023-10-01T10:45:57.040923+0000","last_became_active":"2023-10-01T10:38:16.035383+0000","last_became_peered":"2023-10-01T10:38:16.035383+0000","last_unstale":"2023-10-01T10:45:57.040923+0000","last_undegraded":"2023-10-01T10:45:57.040923+0000","last_fullsized":"2023-10-01T10:45:57.040923+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:44:41.500097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"154'17","reported_seq":28,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:13.372572+0000","last_change":"2023-10-01T10:45:57.055262+0000","last_active":"2023-10-01T10:46:13.372572+0000","last_peered":"2023-10-01T10:46:13.372572+0000","last_clean":"2023-10-01T10:46:13.372572+0000","last_became_active":"2023-10-01T10:45:57.054388+0000","last_became_peered":"2023-10-01T10:45:57.054388+0000","last_unstale":"2023-10-01T10:46:13.372572+0000","last_undegraded":"2023-10-01T10:46:13.372572+0000","last_fullsized":"2023-10-01T10:46:13.372572+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":151,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_clean_scrub_stamp":"2023-10-01T10:45:56.023726+0000","objects_scrubbed":0,"log_size":17,"log_dups_size":0,"ondisk_log_size":17,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1114112,"num_objects":17,"num_object_clones":0,"num_object_copies":51,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":17,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":17,"num_write_kb":1088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":126,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-01T10:45:57.040440+0000","last_change":"2023-10-01T10:38:16.026611+0000","last_active":"2023-10-01T10:45:57.040440+0000","last_peered":"2023-10-01T10:45:57.040440+0000","last_clean":"2023-10-01T10:45:57.040440+0000","last_became_active":"2023-10-01T10:38:16.026267+0000","last_became_peered":"2023-10-01T10:38:16.026267+0000","last_unstale":"2023-10-01T10:45:57.040440+0000","last_undegraded":"2023-10-01T10:45:57.040440+0000","last_fullsized":"2023-10-01T10:45:57.040440+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:06.135288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":110,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-01T10:45:57.041365+0000","last_change":"2023-10-01T10:38:16.032162+0000","last_active":"2023-10-01T10:45:57.041365+0000","last_peered":"2023-10-01T10:45:57.041365+0000","last_clean":"2023-10-01T10:45:57.041365+0000","last_became_active":"2023-10-01T10:38:16.031785+0000","last_became_peered":"2023-10-01T10:38:16.031785+0000","last_unstale":"2023-10-01T10:45:57.041365+0000","last_undegraded":"2023-10-01T10:45:57.041365+0000","last_fullsized":"2023-10-01T10:45:57.041365+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:38:43.003626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"0'0","reported_seq":11,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-01T10:45:58.842556+0000","last_change":"2023-10-01T10:45:57.049550+0000","last_active":"2023-10-01T10:45:58.842556+0000","last_peered":"2023-10-01T10:45:58.842556+0000","last_clean":"2023-10-01T10:45:58.842556+0000","last_became_active":"2023-10-01T10:45:57.049239+0000","last_became_peered":"2023-10-01T10:45:57.049239+0000","last_unstale":"2023-10-01T10:45:58.842556+0000","last_undegraded":"2023-10-01T10:45:58.842556+0000","last_fullsized":"2023-10-01T10:45:58.842556+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":151,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_clean_scrub_stamp":"2023-10-01T10:45:56.023726+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:09:14.295778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":116,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-01T10:45:57.037091+0000","last_change":"2023-10-01T10:37:42.772398+0000","last_active":"2023-10-01T10:45:57.037091+0000","last_peered":"2023-10-01T10:45:57.037091+0000","last_clean":"2023-10-01T10:45:57.037091+0000","last_became_active":"2023-10-01T10:37:42.772084+0000","last_became_peered":"2023-10-01T10:37:42.772084+0000","last_unstale":"2023-10-01T10:45:57.037091+0000","last_undegraded":"2023-10-01T10:45:57.037091+0000","last_fullsized":"2023-10-01T10:45:57.037091+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:32:43.846419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":136,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-01T10:46:03.128564+0000","last_change":"2023-10-01T10:38:16.019755+0000","last_active":"2023-10-01T10:46:03.128564+0000","last_peered":"2023-10-01T10:46:03.128564+0000","last_clean":"2023-10-01T10:46:03.128564+0000","last_became_active":"2023-10-01T10:38:16.019397+0000","last_became_peered":"2023-10-01T10:38:16.019397+0000","last_unstale":"2023-10-01T10:46:03.128564+0000","last_undegraded":"2023-10-01T10:46:03.128564+0000","last_fullsized":"2023-10-01T10:46:03.128564+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:42:56.623225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.c","version":"154'75","reported_seq":86,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:13.758830+0000","last_change":"2023-10-01T10:45:57.052410+0000","last_active":"2023-10-01T10:46:13.758830+0000","last_peered":"2023-10-01T10:46:13.758830+0000","last_clean":"2023-10-01T10:46:13.758830+0000","last_became_active":"2023-10-01T10:45:57.052058+0000","last_became_peered":"2023-10-01T10:45:57.052058+0000","last_unstale":"2023-10-01T10:46:13.758830+0000","last_undegraded":"2023-10-01T10:46:13.758830+0000","last_fullsized":"2023-10-01T10:46:13.758830+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":151,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_clean_scrub_stamp":"2023-10-01T10:45:56.023726+0000","objects_scrubbed":0,"log_size":75,"log_dups_size":0,"ondisk_log_size":75,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4915200,"num_objects":75,"num_object_clones":0,"num_object_copies":225,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":75,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":75,"num_write_kb":4800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":129,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-01T10:45:58.037168+0000","last_change":"2023-10-01T10:32:31.561108+0000","last_active":"2023-10-01T10:45:58.037168+0000","last_peered":"2023-10-01T10:45:58.037168+0000","last_clean":"2023-10-01T10:45:58.037168+0000","last_became_active":"2023-10-01T10:32:31.560791+0000","last_became_peered":"2023-10-01T10:32:31.560791+0000","last_unstale":"2023-10-01T10:45:58.037168+0000","last_undegraded":"2023-10-01T10:45:58.037168+0000","last_fullsized":"2023-10-01T10:45:58.037168+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:42:47.477671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":135,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-01T10:45:57.043682+0000","last_change":"2023-10-01T10:37:42.766814+0000","last_active":"2023-10-01T10:45:57.043682+0000","last_peered":"2023-10-01T10:45:57.043682+0000","last_clean":"2023-10-01T10:45:57.043682+0000","last_became_active":"2023-10-01T10:37:42.766451+0000","last_became_peered":"2023-10-01T10:37:42.766451+0000","last_unstale":"2023-10-01T10:45:57.043682+0000","last_undegraded":"2023-10-01T10:45:57.043682+0000","last_fullsized":"2023-10-01T10:45:57.043682+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":95,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-01T10:45:58.038303+0000","last_change":"2023-10-01T10:38:16.034357+0000","last_active":"2023-10-01T10:45:58.038303+0000","last_peered":"2023-10-01T10:45:58.038303+0000","last_clean":"2023-10-01T10:45:58.038303+0000","last_became_active":"2023-10-01T10:38:16.033905+0000","last_became_peered":"2023-10-01T10:38:16.033905+0000","last_unstale":"2023-10-01T10:45:58.038303+0000","last_undegraded":"2023-10-01T10:45:58.038303+0000","last_fullsized":"2023-10-01T10:45:58.038303+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:47:58.369835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"154'14","reported_seq":25,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:13.374041+0000","last_change":"2023-10-01T10:45:57.055125+0000","last_active":"2023-10-01T10:46:13.374041+0000","last_peered":"2023-10-01T10:46:13.374041+0000","last_clean":"2023-10-01T10:46:13.374041+0000","last_became_active":"2023-10-01T10:45:57.054944+0000","last_became_peered":"2023-10-01T10:45:57.054944+0000","last_unstale":"2023-10-01T10:46:13.374041+0000","last_undegraded":"2023-10-01T10:46:13.374041+0000","last_fullsized":"2023-10-01T10:46:13.374041+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":151,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_clean_scrub_stamp":"2023-10-01T10:45:56.023726+0000","objects_scrubbed":0,"log_size":14,"log_dups_size":0,"ondisk_log_size":14,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":917504,"num_objects":14,"num_object_clones":0,"num_object_copies":42,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":14,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14,"num_write_kb":896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":124,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-01T10:45:57.047049+0000","last_change":"2023-10-01T10:32:31.572482+0000","last_active":"2023-10-01T10:45:57.047049+0000","last_peered":"2023-10-01T10:45:57.047049+0000","last_clean":"2023-10-01T10:45:57.047049+0000","last_became_active":"2023-10-01T10:32:31.571993+0000","last_became_peered":"2023-10-01T10:32:31.571993+0000","last_unstale":"2023-10-01T10:45:57.047049+0000","last_undegraded":"2023-10-01T10:45:57.047049+0000","last_fullsized":"2023-10-01T10:45:57.047049+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":96,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-01T10:45:57.041610+0000","last_change":"2023-10-01T10:38:16.030202+0000","last_active":"2023-10-01T10:45:57.041610+0000","last_peered":"2023-10-01T10:45:57.041610+0000","last_clean":"2023-10-01T10:45:57.041610+0000","last_became_active":"2023-10-01T10:38:16.029917+0000","last_became_peered":"2023-10-01T10:38:16.029917+0000","last_unstale":"2023-10-01T10:45:57.041610+0000","last_undegraded":"2023-10-01T10:45:57.041610+0000","last_fullsized":"2023-10-01T10:45:57.041610+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:20:06.763909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":110,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-01T10:45:58.037360+0000","last_change":"2023-10-01T10:37:42.761153+0000","last_active":"2023-10-01T10:45:58.037360+0000","last_peered":"2023-10-01T10:45:58.037360+0000","last_clean":"2023-10-01T10:45:58.037360+0000","last_became_active":"2023-10-01T10:37:42.760715+0000","last_became_peered":"2023-10-01T10:37:42.760715+0000","last_unstale":"2023-10-01T10:45:58.037360+0000","last_undegraded":"2023-10-01T10:45:58.037360+0000","last_fullsized":"2023-10-01T10:45:58.037360+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:52:50.237174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"0'0","reported_seq":10,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-01T10:45:57.054180+0000","last_change":"2023-10-01T10:45:57.053546+0000","last_active":"2023-10-01T10:45:57.054180+0000","last_peered":"2023-10-01T10:45:57.054180+0000","last_clean":"2023-10-01T10:45:57.054180+0000","last_became_active":"2023-10-01T10:45:57.053064+0000","last_became_peered":"2023-10-01T10:45:57.053064+0000","last_unstale":"2023-10-01T10:45:57.054180+0000","last_undegraded":"2023-10-01T10:45:57.054180+0000","last_fullsized":"2023-10-01T10:45:57.054180+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":151,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_clean_scrub_stamp":"2023-10-01T10:45:56.023726+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":127,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-01T10:45:58.123378+0000","last_change":"2023-10-01T10:37:42.762524+0000","last_active":"2023-10-01T10:45:58.123378+0000","last_peered":"2023-10-01T10:45:58.123378+0000","last_clean":"2023-10-01T10:45:58.123378+0000","last_became_active":"2023-10-01T10:37:42.762209+0000","last_became_peered":"2023-10-01T10:37:42.762209+0000","last_unstale":"2023-10-01T10:45:58.123378+0000","last_undegraded":"2023-10-01T10:45:58.123378+0000","last_fullsized":"2023-10-01T10:45:58.123378+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:45:38.171397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":95,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-01T10:45:58.037023+0000","last_change":"2023-10-01T10:37:05.019330+0000","last_active":"2023-10-01T10:45:58.037023+0000","last_peered":"2023-10-01T10:45:58.037023+0000","last_clean":"2023-10-01T10:45:58.037023+0000","last_became_active":"2023-10-01T10:37:05.018842+0000","last_became_peered":"2023-10-01T10:37:05.018842+0000","last_unstale":"2023-10-01T10:45:58.037023+0000","last_undegraded":"2023-10-01T10:45:58.037023+0000","last_fullsized":"2023-10-01T10:45:58.037023+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:48:09.551917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":124,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-01T10:45:58.037169+0000","last_change":"2023-10-01T10:32:31.569162+0000","last_active":"2023-10-01T10:45:58.037169+0000","last_peered":"2023-10-01T10:45:58.037169+0000","last_clean":"2023-10-01T10:45:58.037169+0000","last_became_active":"2023-10-01T10:32:31.568956+0000","last_became_peered":"2023-10-01T10:32:31.568956+0000","last_unstale":"2023-10-01T10:45:58.037169+0000","last_undegraded":"2023-10-01T10:45:58.037169+0000","last_fullsized":"2023-10-01T10:45:58.037169+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:02:08.285304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"154'21","reported_seq":31,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:13.378516+0000","last_change":"2023-10-01T10:45:57.054040+0000","last_active":"2023-10-01T10:46:13.378516+0000","last_peered":"2023-10-01T10:46:13.378516+0000","last_clean":"2023-10-01T10:46:13.378516+0000","last_became_active":"2023-10-01T10:45:57.052762+0000","last_became_peered":"2023-10-01T10:45:57.052762+0000","last_unstale":"2023-10-01T10:46:13.378516+0000","last_undegraded":"2023-10-01T10:46:13.378516+0000","last_fullsized":"2023-10-01T10:46:13.378516+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":151,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_clean_scrub_stamp":"2023-10-01T10:45:56.023726+0000","objects_scrubbed":0,"log_size":21,"log_dups_size":0,"ondisk_log_size":21,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1376256,"num_objects":21,"num_object_clones":0,"num_object_copies":63,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":21,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":21,"num_write_kb":1344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":124,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-01T10:45:57.046360+0000","last_change":"2023-10-01T10:37:05.024009+0000","last_active":"2023-10-01T10:45:57.046360+0000","last_peered":"2023-10-01T10:45:57.046360+0000","last_clean":"2023-10-01T10:45:57.046360+0000","last_became_active":"2023-10-01T10:37:05.023458+0000","last_became_peered":"2023-10-01T10:37:05.023458+0000","last_unstale":"2023-10-01T10:45:57.046360+0000","last_undegraded":"2023-10-01T10:45:57.046360+0000","last_fullsized":"2023-10-01T10:45:57.046360+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'139","reported_seq":357,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-01T10:45:58.042612+0000","last_change":"2023-10-01T10:28:20.217386+0000","last_active":"2023-10-01T10:45:58.042612+0000","last_peered":"2023-10-01T10:45:58.042612+0000","last_clean":"2023-10-01T10:45:58.042612+0000","last_became_active":"2023-10-01T10:28:20.216803+0000","last_became_peered":"2023-10-01T10:28:20.216803+0000","last_unstale":"2023-10-01T10:45:58.042612+0000","last_undegraded":"2023-10-01T10:45:58.042612+0000","last_fullsized":"2023-10-01T10:45:58.042612+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:09:17.221082+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:09:17.221082+0000","last_clean_scrub_stamp":"2023-10-01T10:09:17.221082+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":99,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-01T10:45:58.035918+0000","last_change":"2023-10-01T10:37:42.765100+0000","last_active":"2023-10-01T10:45:58.035918+0000","last_peered":"2023-10-01T10:45:58.035918+0000","last_clean":"2023-10-01T10:45:58.035918+0000","last_became_active":"2023-10-01T10:37:42.764761+0000","last_became_peered":"2023-10-01T10:37:42.764761+0000","last_unstale":"2023-10-01T10:45:58.035918+0000","last_undegraded":"2023-10-01T10:45:58.035918+0000","last_fullsized":"2023-10-01T10:45:58.035918+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:24:56.907377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":119,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-01T10:45:59.309664+0000","last_change":"2023-10-01T10:38:16.031255+0000","last_active":"2023-10-01T10:45:59.309664+0000","last_peered":"2023-10-01T10:45:59.309664+0000","last_clean":"2023-10-01T10:45:59.309664+0000","last_became_active":"2023-10-01T10:38:16.030886+0000","last_became_peered":"2023-10-01T10:38:16.030886+0000","last_unstale":"2023-10-01T10:45:59.309664+0000","last_undegraded":"2023-10-01T10:45:59.309664+0000","last_fullsized":"2023-10-01T10:45:59.309664+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"154'15","reported_seq":27,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:13.371816+0000","last_change":"2023-10-01T10:45:57.048663+0000","last_active":"2023-10-01T10:46:13.371816+0000","last_peered":"2023-10-01T10:46:13.371816+0000","last_clean":"2023-10-01T10:46:13.371816+0000","last_became_active":"2023-10-01T10:45:57.048378+0000","last_became_peered":"2023-10-01T10:45:57.048378+0000","last_unstale":"2023-10-01T10:46:13.371816+0000","last_undegraded":"2023-10-01T10:46:13.371816+0000","last_fullsized":"2023-10-01T10:46:13.371816+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":151,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_clean_scrub_stamp":"2023-10-01T10:45:56.023726+0000","objects_scrubbed":0,"log_size":15,"log_dups_size":0,"ondisk_log_size":15,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:00:59.519343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":983040,"num_objects":15,"num_object_clones":0,"num_object_copies":45,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":15,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15,"num_write_kb":960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":140,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-01T10:45:58.042330+0000","last_change":"2023-10-01T10:32:31.565868+0000","last_active":"2023-10-01T10:45:58.042330+0000","last_peered":"2023-10-01T10:45:58.042330+0000","last_clean":"2023-10-01T10:45:58.042330+0000","last_became_active":"2023-10-01T10:32:31.565428+0000","last_became_peered":"2023-10-01T10:32:31.565428+0000","last_unstale":"2023-10-01T10:45:58.042330+0000","last_undegraded":"2023-10-01T10:45:58.042330+0000","last_fullsized":"2023-10-01T10:45:58.042330+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":127,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-01T10:45:58.190773+0000","last_change":"2023-10-01T10:38:16.021968+0000","last_active":"2023-10-01T10:45:58.190773+0000","last_peered":"2023-10-01T10:45:58.190773+0000","last_clean":"2023-10-01T10:45:58.190773+0000","last_became_active":"2023-10-01T10:38:16.021539+0000","last_became_peered":"2023-10-01T10:38:16.021539+0000","last_unstale":"2023-10-01T10:45:58.190773+0000","last_undegraded":"2023-10-01T10:45:58.190773+0000","last_fullsized":"2023-10-01T10:45:58.190773+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:12:53.105747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":128,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-01T10:45:58.191582+0000","last_change":"2023-10-01T10:37:05.017431+0000","last_active":"2023-10-01T10:45:58.191582+0000","last_peered":"2023-10-01T10:45:58.191582+0000","last_clean":"2023-10-01T10:45:58.191582+0000","last_became_active":"2023-10-01T10:37:05.017094+0000","last_became_peered":"2023-10-01T10:37:05.017094+0000","last_unstale":"2023-10-01T10:45:58.191582+0000","last_undegraded":"2023-10-01T10:45:58.191582+0000","last_fullsized":"2023-10-01T10:45:58.191582+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:10:40.664686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"154'19","reported_seq":30,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:13.377982+0000","last_change":"2023-10-01T10:45:57.052327+0000","last_active":"2023-10-01T10:46:13.377982+0000","last_peered":"2023-10-01T10:46:13.377982+0000","last_clean":"2023-10-01T10:46:13.377982+0000","last_became_active":"2023-10-01T10:45:57.051994+0000","last_became_peered":"2023-10-01T10:45:57.051994+0000","last_unstale":"2023-10-01T10:46:13.377982+0000","last_undegraded":"2023-10-01T10:46:13.377982+0000","last_fullsized":"2023-10-01T10:46:13.377982+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":151,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_clean_scrub_stamp":"2023-10-01T10:45:56.023726+0000","objects_scrubbed":0,"log_size":19,"log_dups_size":0,"ondisk_log_size":19,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1245184,"num_objects":19,"num_object_clones":0,"num_object_copies":57,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":19,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":19,"num_write_kb":1216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":140,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-01T10:45:58.043190+0000","last_change":"2023-10-01T10:38:16.032774+0000","last_active":"2023-10-01T10:45:58.043190+0000","last_peered":"2023-10-01T10:45:58.043190+0000","last_clean":"2023-10-01T10:45:58.043190+0000","last_became_active":"2023-10-01T10:38:16.032283+0000","last_became_peered":"2023-10-01T10:38:16.032283+0000","last_unstale":"2023-10-01T10:45:58.043190+0000","last_undegraded":"2023-10-01T10:45:58.043190+0000","last_fullsized":"2023-10-01T10:45:58.043190+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":95,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-01T10:45:58.037532+0000","last_change":"2023-10-01T10:37:05.021228+0000","last_active":"2023-10-01T10:45:58.037532+0000","last_peered":"2023-10-01T10:45:58.037532+0000","last_clean":"2023-10-01T10:45:58.037532+0000","last_became_active":"2023-10-01T10:37:05.020888+0000","last_became_peered":"2023-10-01T10:37:05.020888+0000","last_unstale":"2023-10-01T10:45:58.037532+0000","last_undegraded":"2023-10-01T10:45:58.037532+0000","last_fullsized":"2023-10-01T10:45:58.037532+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:59:57.641247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":98,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-01T10:45:58.190908+0000","last_change":"2023-10-01T10:28:20.220812+0000","last_active":"2023-10-01T10:45:58.190908+0000","last_peered":"2023-10-01T10:45:58.190908+0000","last_clean":"2023-10-01T10:45:58.190908+0000","last_became_active":"2023-10-01T10:28:20.215962+0000","last_became_peered":"2023-10-01T10:28:20.215962+0000","last_unstale":"2023-10-01T10:45:58.190908+0000","last_undegraded":"2023-10-01T10:45:58.190908+0000","last_fullsized":"2023-10-01T10:45:58.190908+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:35:15.275097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"0'0","reported_seq":11,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-01T10:45:58.843725+0000","last_change":"2023-10-01T10:45:57.050504+0000","last_active":"2023-10-01T10:45:58.843725+0000","last_peered":"2023-10-01T10:45:58.843725+0000","last_clean":"2023-10-01T10:45:58.843725+0000","last_became_active":"2023-10-01T10:45:57.048979+0000","last_became_peered":"2023-10-01T10:45:57.048979+0000","last_unstale":"2023-10-01T10:45:58.843725+0000","last_undegraded":"2023-10-01T10:45:58.843725+0000","last_fullsized":"2023-10-01T10:45:58.843725+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":151,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_clean_scrub_stamp":"2023-10-01T10:45:56.023726+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:54:06.967857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":103,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-01T10:45:57.037630+0000","last_change":"2023-10-01T10:37:42.773999+0000","last_active":"2023-10-01T10:45:57.037630+0000","last_peered":"2023-10-01T10:45:57.037630+0000","last_clean":"2023-10-01T10:45:57.037630+0000","last_became_active":"2023-10-01T10:37:42.773355+0000","last_became_peered":"2023-10-01T10:37:42.773355+0000","last_unstale":"2023-10-01T10:45:57.037630+0000","last_undegraded":"2023-10-01T10:45:57.037630+0000","last_fullsized":"2023-10-01T10:45:57.037630+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:46:51.317151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":137,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-01T10:45:58.037124+0000","last_change":"2023-10-01T10:38:16.034193+0000","last_active":"2023-10-01T10:45:58.037124+0000","last_peered":"2023-10-01T10:45:58.037124+0000","last_clean":"2023-10-01T10:45:58.037124+0000","last_became_active":"2023-10-01T10:38:16.033703+0000","last_became_peered":"2023-10-01T10:38:16.033703+0000","last_unstale":"2023-10-01T10:45:58.037124+0000","last_undegraded":"2023-10-01T10:45:58.037124+0000","last_fullsized":"2023-10-01T10:45:58.037124+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:21:26.061595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":101,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-01T10:45:57.037084+0000","last_change":"2023-10-01T10:37:42.773538+0000","last_active":"2023-10-01T10:45:57.037084+0000","last_peered":"2023-10-01T10:45:57.037084+0000","last_clean":"2023-10-01T10:45:57.037084+0000","last_became_active":"2023-10-01T10:37:42.771863+0000","last_became_peered":"2023-10-01T10:37:42.771863+0000","last_unstale":"2023-10-01T10:45:57.037084+0000","last_undegraded":"2023-10-01T10:45:57.037084+0000","last_fullsized":"2023-10-01T10:45:57.037084+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:46:46.698627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"154'8","reported_seq":19,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:13.372142+0000","last_change":"2023-10-01T10:45:57.056633+0000","last_active":"2023-10-01T10:46:13.372142+0000","last_peered":"2023-10-01T10:46:13.372142+0000","last_clean":"2023-10-01T10:46:13.372142+0000","last_became_active":"2023-10-01T10:45:57.056070+0000","last_became_peered":"2023-10-01T10:45:57.056070+0000","last_unstale":"2023-10-01T10:46:13.372142+0000","last_undegraded":"2023-10-01T10:46:13.372142+0000","last_fullsized":"2023-10-01T10:46:13.372142+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":151,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_clean_scrub_stamp":"2023-10-01T10:45:56.023726+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":524288,"num_objects":8,"num_object_clones":0,"num_object_copies":24,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8,"num_write_kb":512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":124,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-01T10:45:59.309738+0000","last_change":"2023-10-01T10:37:05.023034+0000","last_active":"2023-10-01T10:45:59.309738+0000","last_peered":"2023-10-01T10:45:59.309738+0000","last_clean":"2023-10-01T10:45:59.309738+0000","last_became_active":"2023-10-01T10:37:05.022670+0000","last_became_peered":"2023-10-01T10:37:05.022670+0000","last_unstale":"2023-10-01T10:45:59.309738+0000","last_undegraded":"2023-10-01T10:45:59.309738+0000","last_fullsized":"2023-10-01T10:45:59.309738+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":131,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-01T10:45:58.191445+0000","last_change":"2023-10-01T10:38:16.027472+0000","last_active":"2023-10-01T10:45:58.191445+0000","last_peered":"2023-10-01T10:45:58.191445+0000","last_clean":"2023-10-01T10:45:58.191445+0000","last_became_active":"2023-10-01T10:38:16.026810+0000","last_became_peered":"2023-10-01T10:38:16.026810+0000","last_unstale":"2023-10-01T10:45:58.191445+0000","last_undegraded":"2023-10-01T10:45:58.191445+0000","last_fullsized":"2023-10-01T10:45:58.191445+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:30:06.978187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":94,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-01T10:45:57.040201+0000","last_change":"2023-10-01T10:38:16.035705+0000","last_active":"2023-10-01T10:45:57.040201+0000","last_peered":"2023-10-01T10:45:57.040201+0000","last_clean":"2023-10-01T10:45:57.040201+0000","last_became_active":"2023-10-01T10:38:16.035389+0000","last_became_peered":"2023-10-01T10:38:16.035389+0000","last_unstale":"2023-10-01T10:45:57.040201+0000","last_undegraded":"2023-10-01T10:45:57.040201+0000","last_fullsized":"2023-10-01T10:45:57.040201+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:16:10.201409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":115,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-01T10:45:58.191225+0000","last_change":"2023-10-01T10:38:16.028491+0000","last_active":"2023-10-01T10:45:58.191225+0000","last_peered":"2023-10-01T10:45:58.191225+0000","last_clean":"2023-10-01T10:45:58.191225+0000","last_became_active":"2023-10-01T10:38:16.028175+0000","last_became_peered":"2023-10-01T10:38:16.028175+0000","last_unstale":"2023-10-01T10:45:58.191225+0000","last_undegraded":"2023-10-01T10:45:58.191225+0000","last_fullsized":"2023-10-01T10:45:58.191225+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:44:29.083129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":136,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-01T10:45:58.123629+0000","last_change":"2023-10-01T10:37:42.762360+0000","last_active":"2023-10-01T10:45:58.123629+0000","last_peered":"2023-10-01T10:45:58.123629+0000","last_clean":"2023-10-01T10:45:58.123629+0000","last_became_active":"2023-10-01T10:37:42.761936+0000","last_became_peered":"2023-10-01T10:37:42.761936+0000","last_unstale":"2023-10-01T10:45:58.123629+0000","last_undegraded":"2023-10-01T10:45:58.123629+0000","last_fullsized":"2023-10-01T10:45:58.123629+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:50.637582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":137,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-01T10:45:57.043634+0000","last_change":"2023-10-01T10:38:16.037809+0000","last_active":"2023-10-01T10:45:57.043634+0000","last_peered":"2023-10-01T10:45:57.043634+0000","last_clean":"2023-10-01T10:45:57.043634+0000","last_became_active":"2023-10-01T10:38:16.037318+0000","last_became_peered":"2023-10-01T10:38:16.037318+0000","last_unstale":"2023-10-01T10:45:57.043634+0000","last_undegraded":"2023-10-01T10:45:57.043634+0000","last_fullsized":"2023-10-01T10:45:57.043634+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":144,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-01T10:45:57.041434+0000","last_change":"2023-10-01T10:38:16.034282+0000","last_active":"2023-10-01T10:45:57.041434+0000","last_peered":"2023-10-01T10:45:57.041434+0000","last_clean":"2023-10-01T10:45:57.041434+0000","last_became_active":"2023-10-01T10:38:16.033820+0000","last_became_peered":"2023-10-01T10:38:16.033820+0000","last_unstale":"2023-10-01T10:45:57.041434+0000","last_undegraded":"2023-10-01T10:45:57.041434+0000","last_fullsized":"2023-10-01T10:45:57.041434+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:48:23.815832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":122,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-01T10:45:57.043543+0000","last_change":"2023-10-01T10:38:16.038077+0000","last_active":"2023-10-01T10:45:57.043543+0000","last_peered":"2023-10-01T10:45:57.043543+0000","last_clean":"2023-10-01T10:45:57.043543+0000","last_became_active":"2023-10-01T10:38:16.037313+0000","last_became_peered":"2023-10-01T10:38:16.037313+0000","last_unstale":"2023-10-01T10:45:57.043543+0000","last_undegraded":"2023-10-01T10:45:57.043543+0000","last_fullsized":"2023-10-01T10:45:57.043543+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":144,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-01T10:45:57.040381+0000","last_change":"2023-10-01T10:38:16.025600+0000","last_active":"2023-10-01T10:45:57.040381+0000","last_peered":"2023-10-01T10:45:57.040381+0000","last_clean":"2023-10-01T10:45:57.040381+0000","last_became_active":"2023-10-01T10:38:16.025252+0000","last_became_peered":"2023-10-01T10:38:16.025252+0000","last_unstale":"2023-10-01T10:45:57.040381+0000","last_undegraded":"2023-10-01T10:45:57.040381+0000","last_fullsized":"2023-10-01T10:45:57.040381+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:24:18.871963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":132,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-01T10:45:57.037564+0000","last_change":"2023-10-01T10:37:42.761243+0000","last_active":"2023-10-01T10:45:57.037564+0000","last_peered":"2023-10-01T10:45:57.037564+0000","last_clean":"2023-10-01T10:45:57.037564+0000","last_became_active":"2023-10-01T10:37:42.760834+0000","last_became_peered":"2023-10-01T10:37:42.760834+0000","last_unstale":"2023-10-01T10:45:57.037564+0000","last_undegraded":"2023-10-01T10:45:57.037564+0000","last_fullsized":"2023-10-01T10:45:57.037564+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:37:21.976095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":144,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-01T10:45:57.037068+0000","last_change":"2023-10-01T10:37:42.765142+0000","last_active":"2023-10-01T10:45:57.037068+0000","last_peered":"2023-10-01T10:45:57.037068+0000","last_clean":"2023-10-01T10:45:57.037068+0000","last_became_active":"2023-10-01T10:37:42.764770+0000","last_became_peered":"2023-10-01T10:37:42.764770+0000","last_unstale":"2023-10-01T10:45:57.037068+0000","last_undegraded":"2023-10-01T10:45:57.037068+0000","last_fullsized":"2023-10-01T10:45:57.037068+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:23:33.527876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":116,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-10-01T10:45:58.191540+0000","last_change":"2023-10-01T10:37:05.021621+0000","last_active":"2023-10-01T10:45:58.191540+0000","last_peered":"2023-10-01T10:45:58.191540+0000","last_clean":"2023-10-01T10:45:58.191540+0000","last_became_active":"2023-10-01T10:37:05.021173+0000","last_became_peered":"2023-10-01T10:37:05.021173+0000","last_unstale":"2023-10-01T10:45:58.191540+0000","last_undegraded":"2023-10-01T10:45:58.191540+0000","last_fullsized":"2023-10-01T10:45:58.191540+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:13:25.364499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":110,"reported_epoch":152,"state":"active+clean","last_fresh":"2023-10-01T10:45:57.039908+0000","last_change":"2023-10-01T10:38:16.031427+0000","last_active":"2023-10-01T10:45:57.039908+0000","last_peered":"2023-10-01T10:45:57.039908+0000","last_clean":"2023-10-01T10:45:57.039908+0000","last_became_active":"2023-10-01T10:38:16.029786+0000","last_became_peered":"2023-10-01T10:38:16.029786+0000","last_unstale":"2023-10-01T10:45:57.039908+0000","last_undegraded":"2023-10-01T10:45:57.039908+0000","last_fullsized":"2023-10-01T10:45:57.039908+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:53:22.796191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":14221312,"num_objects":217,"num_object_clones":0,"num_object_copies":651,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":217,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":217,"num_write_kb":13888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":40828928,"data_stored":40828928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":217,"ondisk_log_size":217,"up":48,"acting":48,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30319,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430611,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":599976,"kb_used_data":6176,"kb_used_omap":0,"kb_used_meta":593792,"kb_avail":93145176,"statfs":{"total":95995035648,"available":95380660224,"internally_reserved":0,"allocated":6324224,"data_stored":5831833,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":608043008},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:45:59 2023","interfaces":[{"interface":"back","average":{"1min":0.78700000000000003,"5min":2.484,"15min":1.6899999999999999},"min":{"1min":0.38800000000000001,"5min":0.28499999999999998,"15min":0.27500000000000002},"max":{"1min":1.1919999999999999,"5min":51.07,"15min":51.07},"last":0.98299999999999998},{"interface":"front","average":{"1min":1.4710000000000001,"5min":1.982,"15min":1.5740000000000001},"min":{"1min":0.63400000000000001,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":7.3209999999999997,"5min":51.024000000000001,"15min":51.024000000000001},"last":0.96299999999999997}]},{"osd":1,"last update":"Sun Oct 1 10:46:05 2023","interfaces":[{"interface":"back","average":{"1min":1.157,"5min":2.8149999999999999,"15min":1.7609999999999999},"min":{"1min":0.58399999999999996,"5min":0.311,"15min":0.309},"max":{"1min":5.0510000000000002,"5min":56.110999999999997,"15min":56.110999999999997},"last":1.071},{"interface":"front","average":{"1min":1.4059999999999999,"5min":2.8929999999999998,"15min":1.903},"min":{"1min":0.59599999999999997,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":6.6459999999999999,"5min":51.258000000000003,"15min":51.258000000000003},"last":0.90900000000000003}]},{"osd":2,"last update":"Sun Oct 1 10:45:19 2023","interfaces":[{"interface":"back","average":{"1min":1.6459999999999999,"5min":1.3140000000000001,"15min":1.119},"min":{"1min":0.36899999999999999,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":5.4820000000000002,"5min":6.5979999999999999,"15min":6.5979999999999999},"last":1.2929999999999999},{"interface":"front","average":{"1min":1.8220000000000001,"5min":1.8320000000000001,"15min":1.236},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":6.3419999999999996,"5min":10.766,"15min":10.766},"last":0.88}]},{"osd":3,"last update":"Sun Oct 1 10:45:59 2023","interfaces":[{"interface":"back","average":{"1min":1.7490000000000001,"5min":2.7229999999999999,"15min":1.5589999999999999},"min":{"1min":0.76000000000000001,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":6.7709999999999999,"5min":51.334000000000003,"15min":51.334000000000003},"last":1.036},{"interface":"front","average":{"1min":1.2909999999999999,"5min":3.145,"15min":1.681},"min":{"1min":0.38600000000000001,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":8.6110000000000007,"5min":51.404000000000003,"15min":51.404000000000003},"last":0.70999999999999996}]},{"osd":4,"last update":"Sun Oct 1 10:45:30 2023","interfaces":[{"interface":"back","average":{"1min":1.468,"5min":1.4019999999999999,"15min":1.087},"min":{"1min":0.53600000000000003,"5min":0.317,"15min":0.317},"max":{"1min":5.085,"5min":8.6340000000000003,"15min":8.6340000000000003},"last":1.1719999999999999},{"interface":"front","average":{"1min":1.661,"5min":1.353,"15min":1.2410000000000001},"min":{"1min":0.41599999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":5.6699999999999999,"5min":7.54,"15min":7.54},"last":1.117}]},{"osd":5,"last update":"Sun Oct 1 10:46:05 2023","interfaces":[{"interface":"back","average":{"1min":1.875,"5min":2.2389999999999999,"15min":1.593},"min":{"1min":0.73999999999999999,"5min":0.36699999999999999,"15min":0.35399999999999998},"max":{"1min":12.555,"5min":12.555,"15min":12.555},"last":1.1910000000000001},{"interface":"front","average":{"1min":1.0149999999999999,"5min":2.6600000000000001,"15min":1.665},"min":{"1min":0.67500000000000004,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":1.427,"5min":17.536999999999999,"15min":17.536999999999999},"last":0.59499999999999997}]},{"osd":6,"last update":"Sun Oct 1 10:46:05 2023","interfaces":[{"interface":"back","average":{"1min":1.5409999999999999,"5min":1.518,"15min":1.4099999999999999},"min":{"1min":0.873,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":5.024,"5min":6.5789999999999997,"15min":7.3300000000000001},"last":0.76700000000000002},{"interface":"front","average":{"1min":1.3120000000000001,"5min":1.347,"15min":1.3109999999999999},"min":{"1min":0.79000000000000004,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":5.1360000000000001,"5min":5.734,"15min":6.984},"last":0.83899999999999997}]}]},{"osd":6,"up_from":42,"seq":180388626845,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":311512,"kb_used_data":2000,"kb_used_omap":0,"kb_used_meta":309504,"kb_avail":93433640,"statfs":{"total":95995035648,"available":95676047360,"internally_reserved":0,"allocated":2048000,"data_stored":1571532,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316932096},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:45:51 2023","interfaces":[{"interface":"back","average":{"1min":0.85699999999999998,"5min":2.4740000000000002,"15min":1.764},"min":{"1min":0.30299999999999999,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":3.3420000000000001,"5min":140.524,"15min":140.524},"last":0.85599999999999998},{"interface":"front","average":{"1min":0.96199999999999997,"5min":2.3540000000000001,"15min":1.714},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.28899999999999998},"max":{"1min":3.0859999999999999,"5min":140.44,"15min":140.44},"last":0.874}]},{"osd":1,"last update":"Sun Oct 1 10:45:55 2023","interfaces":[{"interface":"back","average":{"1min":1.123,"5min":1.335,"15min":1.2909999999999999},"min":{"1min":0.316,"5min":0.29999999999999999,"15min":0.26600000000000001},"max":{"1min":9.673,"5min":9.673,"15min":9.673},"last":0.89900000000000002},{"interface":"front","average":{"1min":1.02,"5min":1.329,"15min":1.361},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":6.2770000000000001,"5min":6.3710000000000004,"15min":7.4199999999999999},"last":0.98799999999999999}]},{"osd":2,"last update":"Sun Oct 1 10:45:18 2023","interfaces":[{"interface":"back","average":{"1min":1.2609999999999999,"5min":1.177,"15min":0.97499999999999998},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":4.2450000000000001,"5min":4.8810000000000002,"15min":4.8810000000000002},"last":0.88900000000000001},{"interface":"front","average":{"1min":1.3340000000000001,"5min":1.2250000000000001,"15min":1.0389999999999999},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":7.9619999999999997,"5min":7.9619999999999997,"15min":7.9619999999999997},"last":0.73299999999999998}]},{"osd":3,"last update":"Sun Oct 1 10:46:03 2023","interfaces":[{"interface":"back","average":{"1min":1.042,"5min":2.7970000000000002,"15min":1.589},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":4.0350000000000001,"5min":61.817999999999998,"15min":61.817999999999998},"last":0.80200000000000005},{"interface":"front","average":{"1min":1.5369999999999999,"5min":2.4390000000000001,"15min":1.4930000000000001},"min":{"1min":0.61199999999999999,"5min":0.315,"15min":0.315},"max":{"1min":8.0229999999999997,"5min":61.219000000000001,"15min":61.219000000000001},"last":0.83799999999999997}]},{"osd":4,"last update":"Sun Oct 1 10:45:34 2023","interfaces":[{"interface":"back","average":{"1min":1.1659999999999999,"5min":0.92400000000000004,"15min":0.98299999999999998},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":4.5570000000000004,"5min":4.5570000000000004,"15min":4.5570000000000004},"last":0.82599999999999996},{"interface":"front","average":{"1min":1.1830000000000001,"5min":1.113,"15min":1.101},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":3.7799999999999998,"5min":7.1589999999999998,"15min":7.1589999999999998},"last":0.92800000000000005}]},{"osd":5,"last update":"Sun Oct 1 10:46:03 2023","interfaces":[{"interface":"back","average":{"1min":1.0669999999999999,"5min":1.4239999999999999,"15min":1.3080000000000001},"min":{"1min":0.47999999999999998,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":3.9119999999999999,"5min":12.398999999999999,"15min":12.398999999999999},"last":0.70399999999999996},{"interface":"front","average":{"1min":0.94999999999999996,"5min":1.601,"15min":1.3720000000000001},"min":{"1min":0.45100000000000001,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":1.341,"5min":14.459,"15min":14.459},"last":0.75600000000000001}]},{"osd":7,"last update":"Sun Oct 1 10:45:51 2023","interfaces":[{"interface":"back","average":{"1min":1.018,"5min":1.2070000000000001,"15min":1.129},"min":{"1min":0.42299999999999999,"5min":0.371,"15min":0.371},"max":{"1min":3.1349999999999998,"5min":8.8360000000000003,"15min":8.8360000000000003},"last":0.95799999999999996},{"interface":"front","average":{"1min":0.98599999999999999,"5min":1.504,"15min":1.3200000000000001},"min":{"1min":0.40899999999999997,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":3.649,"5min":8.1720000000000006,"15min":8.1720000000000006},"last":1.1319999999999999}]}]},{"osd":1,"up_from":101,"seq":433791697062,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":368472,"kb_used_data":1616,"kb_used_omap":4,"kb_used_meta":366843,"kb_avail":93376680,"statfs":{"total":95995035648,"available":95617720320,"internally_reserved":0,"allocated":1654784,"data_stored":1008287,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":375648142},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:46:03 2023","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":1.1240000000000001,"15min":1.4690000000000001},"min":{"1min":0.224,"5min":0.189,"15min":0.189},"max":{"1min":1.1040000000000001,"5min":8.3520000000000003,"15min":11.071999999999999},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.70899999999999996,"5min":2.0760000000000001,"15min":1.758},"min":{"1min":0.30399999999999999,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":2.137,"5min":61.207999999999998,"15min":61.207999999999998},"last":0.70999999999999996}]},{"osd":2,"last update":"Sun Oct 1 10:45:22 2023","interfaces":[{"interface":"back","average":{"1min":1.5860000000000001,"5min":1.8420000000000001,"15min":1.069},"min":{"1min":0.30399999999999999,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":4.9550000000000001,"5min":61.491,"15min":61.491},"last":0.80900000000000005},{"interface":"front","average":{"1min":1.0669999999999999,"5min":2.0129999999999999,"15min":1.1870000000000001},"min":{"1min":0.317,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":3.9390000000000001,"5min":61.371000000000002,"15min":61.371000000000002},"last":0.88400000000000001}]},{"osd":3,"last update":"Sun Oct 1 10:46:03 2023","interfaces":[{"interface":"back","average":{"1min":0.64700000000000002,"5min":2.3929999999999998,"15min":1.256},"min":{"1min":0.29299999999999998,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":0.82399999999999995,"5min":62.020000000000003,"15min":62.020000000000003},"last":0.68600000000000005},{"interface":"front","average":{"1min":0.88800000000000001,"5min":2.6280000000000001,"15min":1.3740000000000001},"min":{"1min":0.502,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":2.3039999999999998,"5min":62.338999999999999,"15min":62.338999999999999},"last":0.61699999999999999}]},{"osd":4,"last update":"Sun Oct 1 10:45:22 2023","interfaces":[{"interface":"back","average":{"1min":1.292,"5min":1.8240000000000001,"15min":1.2070000000000001},"min":{"1min":0.30199999999999999,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":4.5259999999999998,"5min":61.588000000000001,"15min":61.588000000000001},"last":1.0669999999999999},{"interface":"front","average":{"1min":1.206,"5min":2.2330000000000001,"15min":1.419},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":4.0519999999999996,"5min":61.411000000000001,"15min":61.411000000000001},"last":1.1990000000000001}]},{"osd":5,"last update":"Sun Oct 1 10:46:03 2023","interfaces":[{"interface":"back","average":{"1min":1.1970000000000001,"5min":2.4289999999999998,"15min":1.8300000000000001},"min":{"1min":0.81899999999999995,"5min":0.38900000000000001,"15min":0.35799999999999998},"max":{"1min":3.1869999999999998,"5min":61.560000000000002,"15min":61.560000000000002},"last":1.083},{"interface":"front","average":{"1min":1.0129999999999999,"5min":2.0379999999999998,"15min":1.972},"min":{"1min":0.53900000000000003,"5min":0.34699999999999998,"15min":0.33600000000000002},"max":{"1min":1.456,"5min":61.667999999999999,"15min":61.667999999999999},"last":1.181}]},{"osd":6,"last update":"Sun Oct 1 10:46:03 2023","interfaces":[{"interface":"back","average":{"1min":1.036,"5min":1.873,"15min":1.746},"min":{"1min":0.433,"5min":0.39100000000000001,"15min":0.37},"max":{"1min":1.5349999999999999,"5min":62.219999999999999,"15min":62.219999999999999},"last":1.373},{"interface":"front","average":{"1min":1.3089999999999999,"5min":2.1850000000000001,"15min":1.8440000000000001},"min":{"1min":0.52500000000000002,"5min":0.439,"15min":0.41999999999999998},"max":{"1min":4.4660000000000002,"5min":61.462000000000003,"15min":61.462000000000003},"last":1.29}]},{"osd":7,"last update":"Sun Oct 1 10:46:03 2023","interfaces":[{"interface":"back","average":{"1min":1.1539999999999999,"5min":2.6909999999999998,"15min":2.0019999999999998},"min":{"1min":0.83299999999999996,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":1.7470000000000001,"5min":62.646999999999998,"15min":62.646999999999998},"last":1.248},{"interface":"front","average":{"1min":1.2190000000000001,"5min":2.3780000000000001,"15min":1.847},"min":{"1min":0.94599999999999995,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":2.5009999999999999,"5min":62.048000000000002,"15min":62.048000000000002},"last":1.1539999999999999}]}]},{"osd":0,"up_from":86,"seq":369367187673,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":411664,"kb_used_data":11272,"kb_used_omap":5,"kb_used_meta":400378,"kb_avail":93333488,"statfs":{"total":95995035648,"available":95573491712,"internally_reserved":0,"allocated":11542528,"data_stored":10901400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":409987706},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 10:45:58 2023","interfaces":[{"interface":"back","average":{"1min":0.69999999999999996,"5min":1.6599999999999999,"15min":1.5529999999999999},"min":{"1min":0.20699999999999999,"5min":0.184,"15min":0.184},"max":{"1min":1.718,"5min":15.074,"15min":15.074},"last":0.28199999999999997},{"interface":"front","average":{"1min":1.492,"5min":2.008,"15min":1.835},"min":{"1min":0.34699999999999998,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":13.601000000000001,"5min":13.601000000000001,"15min":15.48},"last":0.38100000000000001}]},{"osd":2,"last update":"Sun Oct 1 10:45:25 2023","interfaces":[{"interface":"back","average":{"1min":0.45900000000000002,"5min":1.022,"15min":0.80600000000000005},"min":{"1min":0.26600000000000001,"5min":0.223,"15min":0.223},"max":{"1min":0.64900000000000002,"5min":9.2420000000000009,"15min":9.2420000000000009},"last":0.315},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.91700000000000004,"15min":0.80500000000000005},"min":{"1min":0.312,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":1.268,"5min":4.274,"15min":4.274},"last":0.442}]},{"osd":3,"last update":"Sun Oct 1 10:45:58 2023","interfaces":[{"interface":"back","average":{"1min":0.80000000000000004,"5min":1.704,"15min":1.101},"min":{"1min":0.47299999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":2.2210000000000001,"5min":9.4789999999999992,"15min":9.4789999999999992},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.90100000000000002,"5min":1.5309999999999999,"15min":1.04},"min":{"1min":0.25700000000000001,"5min":0.245,"15min":0.245},"max":{"1min":3.754,"5min":10.984999999999999,"15min":10.984999999999999},"last":0.39600000000000002}]},{"osd":4,"last update":"Sun Oct 1 10:45:36 2023","interfaces":[{"interface":"back","average":{"1min":1.1839999999999999,"5min":1.2210000000000001,"15min":1.085},"min":{"1min":0.45100000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":3.4529999999999998,"5min":9.0980000000000008,"15min":9.0980000000000008},"last":0.93500000000000005},{"interface":"front","average":{"1min":1.486,"5min":1.274,"15min":1.1140000000000001},"min":{"1min":0.45800000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":8.1479999999999997,"5min":8.1479999999999997,"15min":8.1479999999999997},"last":0.94799999999999995}]},{"osd":5,"last update":"Sun Oct 1 10:45:58 2023","interfaces":[{"interface":"back","average":{"1min":1.022,"5min":1.5700000000000001,"15min":1.5780000000000001},"min":{"1min":0.56399999999999995,"5min":0.35799999999999998,"15min":0.31900000000000001},"max":{"1min":2.6000000000000001,"5min":10.672000000000001,"15min":13.068},"last":0.89100000000000001},{"interface":"front","average":{"1min":1.1799999999999999,"5min":1.5389999999999999,"15min":1.5680000000000001},"min":{"1min":0.35099999999999998,"5min":0.34699999999999998,"15min":0.34200000000000003},"max":{"1min":3.504,"5min":9.1440000000000001,"15min":11.747},"last":0.71399999999999997}]},{"osd":6,"last update":"Sun Oct 1 10:45:58 2023","interfaces":[{"interface":"back","average":{"1min":1.119,"5min":1.242,"15min":1.536},"min":{"1min":0.46100000000000002,"5min":0.36399999999999999,"15min":0.35899999999999999},"max":{"1min":3.4199999999999999,"5min":9.1780000000000008,"15min":11.663},"last":0.80200000000000005},{"interface":"front","average":{"1min":1.5449999999999999,"5min":1.278,"15min":1.5349999999999999},"min":{"1min":0.62,"5min":0.374,"15min":0.374},"max":{"1min":7.306,"5min":8.2569999999999997,"15min":15.069000000000001},"last":0.68100000000000005}]},{"osd":7,"last update":"Sun Oct 1 10:45:58 2023","interfaces":[{"interface":"back","average":{"1min":1.587,"5min":1.7010000000000001,"15min":1.6180000000000001},"min":{"1min":0.75900000000000001,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":6.819,"5min":9.2059999999999995,"15min":12.686},"last":0.84399999999999997},{"interface":"front","average":{"1min":1.915,"5min":1.768,"15min":1.629},"min":{"1min":0.65600000000000003,"5min":0.434,"15min":0.36199999999999999},"max":{"1min":8.0099999999999998,"5min":8.3100000000000005,"15min":13.654},"last":0.70899999999999996}]}]},{"osd":2,"up_from":116,"seq":498216206448,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":192156,"kb_used_data":1556,"kb_used_omap":7,"kb_used_meta":190584,"kb_avail":93552996,"statfs":{"total":95995035648,"available":95798267904,"internally_reserved":0,"allocated":1593344,"data_stored":997414,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":195158097},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:45:12 2023","interfaces":[{"interface":"back","average":{"1min":1.272,"5min":0.97699999999999998,"15min":0.77400000000000002},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":4.1219999999999999,"5min":14.851000000000001,"15min":14.851000000000001},"last":0.626},{"interface":"front","average":{"1min":1.2330000000000001,"5min":1.026,"15min":0.86299999999999999},"min":{"1min":0.29199999999999998,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":3.903,"5min":6.1829999999999998,"15min":6.1829999999999998},"last":0.56799999999999995}]},{"osd":1,"last update":"Sun Oct 1 10:45:12 2023","interfaces":[{"interface":"back","average":{"1min":1.7949999999999999,"5min":1.3879999999999999,"15min":0.94499999999999995},"min":{"1min":0.29299999999999998,"5min":0.219,"15min":0.219},"max":{"1min":10.565,"5min":10.565,"15min":10.565},"last":0.67000000000000004},{"interface":"front","average":{"1min":1.325,"5min":1.1759999999999999,"15min":0.91100000000000003},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":5.2759999999999998,"5min":8.9209999999999994,"15min":8.9209999999999994},"last":0.78700000000000003}]},{"osd":3,"last update":"Sun Oct 1 10:45:53 2023","interfaces":[{"interface":"back","average":{"1min":1.125,"5min":1.744,"15min":1.0720000000000001},"min":{"1min":0.29499999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":3.8340000000000001,"5min":64.266000000000005,"15min":64.266000000000005},"last":0.71799999999999997},{"interface":"front","average":{"1min":1.4570000000000001,"5min":1.77,"15min":1.175},"min":{"1min":0.47699999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":6.0339999999999998,"5min":67.620999999999995,"15min":67.620999999999995},"last":0.82199999999999995}]},{"osd":4,"last update":"Sun Oct 1 10:45:32 2023","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":1.0529999999999999,"15min":1.014},"min":{"1min":0.40300000000000002,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":3.6269999999999998,"5min":7.9580000000000002,"15min":7.9580000000000002},"last":0.751},{"interface":"front","average":{"1min":0.97399999999999998,"5min":0.89800000000000002,"15min":0.94399999999999995},"min":{"1min":0.40100000000000002,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":3.181,"5min":4.3559999999999999,"15min":4.3559999999999999},"last":0.94999999999999996}]},{"osd":5,"last update":"Sun Oct 1 10:45:12 2023","interfaces":[{"interface":"back","average":{"1min":1.637,"5min":1.383,"15min":1.204},"min":{"1min":0.375,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":5.9960000000000004,"5min":5.9960000000000004,"15min":5.9960000000000004},"last":0.871},{"interface":"front","average":{"1min":1.665,"5min":1.2789999999999999,"15min":1.1100000000000001},"min":{"1min":0.39200000000000002,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":6.0640000000000001,"5min":6.0640000000000001,"15min":6.0640000000000001},"last":1.0609999999999999}]},{"osd":6,"last update":"Sun Oct 1 10:45:12 2023","interfaces":[{"interface":"back","average":{"1min":1.411,"5min":0.95099999999999996,"15min":1.0449999999999999},"min":{"1min":0.41699999999999998,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":6.2240000000000002,"5min":6.2240000000000002,"15min":6.2240000000000002},"last":0.89700000000000002},{"interface":"front","average":{"1min":0.95499999999999996,"5min":0.875,"15min":1.0529999999999999},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":3.2930000000000001,"5min":3.2930000000000001,"15min":3.2930000000000001},"last":1.0269999999999999}]},{"osd":7,"last update":"Sun Oct 1 10:45:12 2023","interfaces":[{"interface":"back","average":{"1min":1.137,"5min":1.1100000000000001,"15min":1.1240000000000001},"min":{"1min":0.44,"5min":0.373,"15min":0.373},"max":{"1min":6.0460000000000003,"5min":6.7869999999999999,"15min":6.7869999999999999},"last":1.1080000000000001},{"interface":"front","average":{"1min":2.0659999999999998,"5min":1.6359999999999999,"15min":1.355},"min":{"1min":0.501,"5min":0.376,"15min":0.376},"max":{"1min":4.4980000000000002,"5min":8.9269999999999996,"15min":8.9269999999999996},"last":0.92200000000000004}]}]},{"osd":3,"up_from":121,"seq":519691042920,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":224004,"kb_used_data":1532,"kb_used_omap":2,"kb_used_meta":222461,"kb_avail":93521148,"statfs":{"total":95995035648,"available":95765655552,"internally_reserved":0,"allocated":1568768,"data_stored":987465,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":227800117},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:45:55 2023","interfaces":[{"interface":"back","average":{"1min":0.97199999999999998,"5min":2.5710000000000002,"15min":1.3009999999999999},"min":{"1min":0.30499999999999999,"5min":0.187,"15min":0.187},"max":{"1min":5.8499999999999996,"5min":77.981999999999999,"15min":77.981999999999999},"last":0.69699999999999995},{"interface":"front","average":{"1min":1.304,"5min":2.8159999999999998,"15min":1.337},"min":{"1min":0.375,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":5.6120000000000001,"5min":77.281000000000006,"15min":77.281000000000006},"last":0.58299999999999996}]},{"osd":1,"last update":"Sun Oct 1 10:45:55 2023","interfaces":[{"interface":"back","average":{"1min":1.1299999999999999,"5min":2.8090000000000002,"15min":1.401},"min":{"1min":0.34899999999999998,"5min":0.222,"15min":0.222},"max":{"1min":6.1159999999999997,"5min":77.361000000000004,"15min":77.361000000000004},"last":0.496},{"interface":"front","average":{"1min":1.0820000000000001,"5min":3.0529999999999999,"15min":1.536},"min":{"1min":0.40400000000000003,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":6.0970000000000004,"5min":77.867000000000004,"15min":77.867000000000004},"last":0.61199999999999999}]},{"osd":2,"last update":"Sun Oct 1 10:45:55 2023","interfaces":[{"interface":"back","average":{"1min":1.2649999999999999,"5min":1.3220000000000001,"15min":0.89700000000000002},"min":{"1min":0.41399999999999998,"5min":0.222,"15min":0.222},"max":{"1min":10.028,"5min":10.028,"15min":10.028},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.84799999999999998,"5min":1.9359999999999999,"15min":1.1739999999999999},"min":{"1min":0.39000000000000001,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":2.6429999999999998,"5min":75.299999999999997,"15min":75.299999999999997},"last":0.64700000000000002}]},{"osd":4,"last update":"Sun Oct 1 10:45:25 2023","interfaces":[{"interface":"back","average":{"1min":1.413,"5min":1.968,"15min":1.2889999999999999},"min":{"1min":0.373,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":6.1390000000000002,"5min":75.388999999999996,"15min":75.388999999999996},"last":0.748},{"interface":"front","average":{"1min":1.556,"5min":2.1699999999999999,"15min":1.3500000000000001},"min":{"1min":0.40799999999999997,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":4.7699999999999996,"5min":75.165999999999997,"15min":75.165999999999997},"last":0.80500000000000005}]},{"osd":5,"last update":"Sun Oct 1 10:45:55 2023","interfaces":[{"interface":"back","average":{"1min":1.6890000000000001,"5min":2.722,"15min":1.5409999999999999},"min":{"1min":0.51600000000000001,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":9.5540000000000003,"5min":75.221000000000004,"15min":75.221000000000004},"last":0.879},{"interface":"front","average":{"1min":1.5089999999999999,"5min":2.9510000000000001,"15min":1.6539999999999999},"min":{"1min":0.64000000000000001,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":10.43,"5min":75.231999999999999,"15min":75.231999999999999},"last":0.73699999999999999}]},{"osd":6,"last update":"Sun Oct 1 10:45:55 2023","interfaces":[{"interface":"back","average":{"1min":1.49,"5min":2.403,"15min":1.48},"min":{"1min":0.44900000000000001,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":4.8090000000000002,"5min":75.257000000000005,"15min":75.257000000000005},"last":0.72599999999999998},{"interface":"front","average":{"1min":1.677,"5min":2.488,"15min":1.5629999999999999},"min":{"1min":0.72899999999999998,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":10.066000000000001,"5min":75.194000000000003,"15min":75.194000000000003},"last":0.92000000000000004}]},{"osd":7,"last update":"Sun Oct 1 10:45:55 2023","interfaces":[{"interface":"back","average":{"1min":1.579,"5min":2.73,"15min":1.609},"min":{"1min":0.68400000000000005,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":11.782999999999999,"5min":75.138999999999996,"15min":75.138999999999996},"last":1.028},{"interface":"front","average":{"1min":1.3029999999999999,"5min":2.4199999999999999,"15min":1.494},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":6.069,"5min":75.355000000000004,"15min":75.355000000000004},"last":0.97499999999999998}]}]},{"osd":4,"up_from":126,"seq":541165879393,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":179084,"kb_used_data":1540,"kb_used_omap":10,"kb_used_meta":177525,"kb_avail":93566068,"statfs":{"total":95995035648,"available":95811653632,"internally_reserved":0,"allocated":1576960,"data_stored":988442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":181786212},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:45:25 2023","interfaces":[{"interface":"back","average":{"1min":1.0629999999999999,"5min":1.2330000000000001,"15min":1.0229999999999999},"min":{"1min":0.30599999999999999,"5min":0.251,"15min":0.251},"max":{"1min":3.8860000000000001,"5min":8.1329999999999991,"15min":8.1329999999999991},"last":1.1439999999999999},{"interface":"front","average":{"1min":1.157,"5min":1.143,"15min":1.036},"min":{"1min":0.32800000000000001,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":3.9449999999999998,"5min":6.7789999999999999,"15min":6.7789999999999999},"last":0.998}]},{"osd":1,"last update":"Sun Oct 1 10:45:25 2023","interfaces":[{"interface":"back","average":{"1min":0.871,"5min":2.4060000000000001,"15min":1.423},"min":{"1min":0.33700000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":3.948,"5min":97.622,"15min":97.622},"last":1.0289999999999999},{"interface":"front","average":{"1min":0.96099999999999997,"5min":2.504,"15min":1.474},"min":{"1min":0.32900000000000001,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":4.101,"5min":97.346999999999994,"15min":97.346999999999994},"last":1.018}]},{"osd":2,"last update":"Sun Oct 1 10:45:25 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.82999999999999996,"15min":0.91600000000000004},"min":{"1min":0.36499999999999999,"5min":0.31,"15min":0.31},"max":{"1min":1.278,"5min":5.5049999999999999,"15min":5.5049999999999999},"last":0.76500000000000001},{"interface":"front","average":{"1min":0.88800000000000001,"5min":0.97199999999999998,"15min":0.97899999999999998},"min":{"1min":0.36499999999999999,"5min":0.308,"15min":0.308},"max":{"1min":3.9769999999999999,"5min":6.8339999999999996,"15min":6.8339999999999996},"last":1.0600000000000001}]},{"osd":3,"last update":"Sun Oct 1 10:45:25 2023","interfaces":[{"interface":"back","average":{"1min":1.272,"5min":3.0779999999999998,"15min":1.7390000000000001},"min":{"1min":0.438,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":4.0629999999999997,"5min":61.398000000000003,"15min":61.398000000000003},"last":1.1160000000000001},{"interface":"front","average":{"1min":1.042,"5min":2.8300000000000001,"15min":1.5940000000000001},"min":{"1min":0.38800000000000001,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":4.6070000000000002,"5min":61.463000000000001,"15min":61.463000000000001},"last":0.97899999999999998}]},{"osd":5,"last update":"Sun Oct 1 10:45:25 2023","interfaces":[{"interface":"back","average":{"1min":1.643,"5min":1.2849999999999999,"15min":1.153},"min":{"1min":0.377,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":5.0519999999999996,"5min":6.9340000000000002,"15min":6.9340000000000002},"last":0.86899999999999999},{"interface":"front","average":{"1min":1.0920000000000001,"5min":1.393,"15min":1.1950000000000001},"min":{"1min":0.46999999999999997,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":5.2530000000000001,"5min":16.779,"15min":16.779},"last":1.0900000000000001}]},{"osd":6,"last update":"Sun Oct 1 10:45:25 2023","interfaces":[{"interface":"back","average":{"1min":1.129,"5min":1.0109999999999999,"15min":1.093},"min":{"1min":0.42899999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":4.0220000000000002,"5min":6.9009999999999998,"15min":6.9009999999999998},"last":1.1599999999999999},{"interface":"front","average":{"1min":1.3320000000000001,"5min":1.1619999999999999,"15min":1.1060000000000001},"min":{"1min":0.47299999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":4.5670000000000002,"5min":6.7930000000000001,"15min":6.7930000000000001},"last":0.81699999999999995}]},{"osd":7,"last update":"Sun Oct 1 10:45:25 2023","interfaces":[{"interface":"back","average":{"1min":1.006,"5min":1.163,"15min":1.1759999999999999},"min":{"1min":0.42699999999999999,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":4.7930000000000001,"5min":6.8360000000000003,"15min":6.8360000000000003},"last":0.92500000000000004},{"interface":"front","average":{"1min":1.5580000000000001,"5min":1.8999999999999999,"15min":1.415},"min":{"1min":0.442,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":3.9169999999999998,"5min":7.726,"15min":7.726},"last":0.95099999999999996}]}]},{"osd":5,"up_from":36,"seq":154618823074,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":897476,"kb_used_data":1404,"kb_used_omap":0,"kb_used_meta":896064,"kb_avail":92847676,"statfs":{"total":95995035648,"available":95076020224,"internally_reserved":0,"allocated":1437696,"data_stored":969234,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":917569536},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:45:48 2023","interfaces":[{"interface":"back","average":{"1min":1.1579999999999999,"5min":1.5940000000000001,"15min":1.409},"min":{"1min":0.35799999999999998,"5min":0.28299999999999997,"15min":0.27300000000000002},"max":{"1min":5.8049999999999997,"5min":52.094999999999999,"15min":52.094999999999999},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.93999999999999995,"5min":1.9179999999999999,"15min":1.631},"min":{"1min":0.314,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":3.7490000000000001,"5min":52.048000000000002,"15min":52.048000000000002},"last":0.90600000000000003}]},{"osd":1,"last update":"Sun Oct 1 10:45:48 2023","interfaces":[{"interface":"back","average":{"1min":1.5,"5min":4.1900000000000004,"15min":2.4489999999999998},"min":{"1min":0.42699999999999999,"5min":0.29799999999999999,"15min":0.29099999999999998},"max":{"1min":6.0049999999999999,"5min":202.96600000000001,"15min":202.96600000000001},"last":0.54000000000000004},{"interface":"front","average":{"1min":1.6619999999999999,"5min":4.9379999999999997,"15min":2.7930000000000001},"min":{"1min":0.503,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":7.2859999999999996,"5min":203.05099999999999,"15min":203.05099999999999},"last":0.81999999999999995}]},{"osd":2,"last update":"Sun Oct 1 10:45:22 2023","interfaces":[{"interface":"back","average":{"1min":1.0620000000000001,"5min":1.0600000000000001,"15min":1.006},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":5.7519999999999998,"5min":8.7110000000000003,"15min":8.7110000000000003},"last":0.78800000000000003},{"interface":"front","average":{"1min":2.3239999999999998,"5min":1.6579999999999999,"15min":1.1990000000000001},"min":{"1min":0.38200000000000001,"5min":0.317,"15min":0.317},"max":{"1min":11.266,"5min":11.266,"15min":11.266},"last":0.70599999999999996}]},{"osd":3,"last update":"Sun Oct 1 10:46:02 2023","interfaces":[{"interface":"back","average":{"1min":0.96299999999999997,"5min":4.2610000000000001,"15min":2.0470000000000002},"min":{"1min":0.46500000000000002,"5min":0.316,"15min":0.316},"max":{"1min":1.798,"5min":70.506,"15min":70.506},"last":0.999},{"interface":"front","average":{"1min":0.81699999999999995,"5min":3.7869999999999999,"15min":1.899},"min":{"1min":0.43099999999999999,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":1.2869999999999999,"5min":70.346999999999994,"15min":70.346999999999994},"last":0.85999999999999999}]},{"osd":4,"last update":"Sun Oct 1 10:45:27 2023","interfaces":[{"interface":"back","average":{"1min":1.986,"5min":1.466,"15min":1.2090000000000001},"min":{"1min":0.39500000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":15.170999999999999,"5min":15.170999999999999,"15min":15.170999999999999},"last":0.94599999999999995},{"interface":"front","average":{"1min":1.165,"5min":1.089,"15min":1.091},"min":{"1min":0.45900000000000002,"5min":0.376,"15min":0.376},"max":{"1min":5.6970000000000001,"5min":8.6329999999999991,"15min":8.6329999999999991},"last":0.88100000000000001}]},{"osd":6,"last update":"Sun Oct 1 10:46:02 2023","interfaces":[{"interface":"back","average":{"1min":1.2290000000000001,"5min":1.157,"15min":1.2310000000000001},"min":{"1min":0.42599999999999999,"5min":0.39200000000000002,"15min":0.38500000000000001},"max":{"1min":5.6219999999999999,"5min":8.7420000000000009,"15min":8.7420000000000009},"last":0.96999999999999997},{"interface":"front","average":{"1min":0.94399999999999995,"5min":1.508,"15min":1.335},"min":{"1min":0.57899999999999996,"5min":0.38300000000000001,"15min":0.312},"max":{"1min":1.528,"5min":7.8010000000000002,"15min":8.4339999999999993},"last":1.0169999999999999}]},{"osd":7,"last update":"Sun Oct 1 10:46:02 2023","interfaces":[{"interface":"back","average":{"1min":1.018,"5min":2.1040000000000001,"15min":1.5609999999999999},"min":{"1min":0.48399999999999999,"5min":0.42899999999999999,"15min":0.34699999999999998},"max":{"1min":1.833,"5min":12.757999999999999,"15min":12.757999999999999},"last":1.0800000000000001},{"interface":"front","average":{"1min":1.3819999999999999,"5min":1.8999999999999999,"15min":1.4530000000000001},"min":{"1min":0.70099999999999996,"5min":0.53200000000000003,"15min":0.372},"max":{"1min":5.6669999999999998,"5min":11.635,"15min":11.635},"last":1.0449999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":10289152,"data_stored":10289152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":8126464,"data_stored":8126464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":22413312,"data_stored":22413312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T10:46:18.073 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-01T10:46:18.073 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T10:46:18.073 INFO:tasks.thrashosds.thrasher:fixing pg num pool device_health_metrics 2023-10-01T10:46:18.074 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T10:46:18.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:18.075 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:18.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:18.176 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:18.201 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:18.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:18.205 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:46:18.293 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 5 16 12633 12617 157.644 144.188 0.0100608 0.00633477 2023-10-01T10:46:18.305 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:18.343 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:18.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:18.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:46:18.418 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:46:18.449 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:18.474 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:18.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:18.479 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:46:18.579 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:18.607 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:18.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:18.611 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:46:18.711 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:18.753 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:18.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:18.756 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:46:18.858 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:18.883 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:18.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:18.884 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:46:18.985 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:19.012 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:19.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:19.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:19.117 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:19.145 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:19.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:19.149 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:19.249 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:19.277 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:19.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:19.279 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:19.293 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 6 16 14725 14709 153.161 130.75 0.00959327 0.00652227 2023-10-01T10:46:19.380 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:19.412 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:19.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:19.415 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:46:19.515 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:19.538 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:19.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:19.542 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:46:19.615 INFO:teuthology.orchestra.run.smithi078.stderr:noscrub is unset 2023-10-01T10:46:19.643 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:19.668 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:19.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:19.671 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:46:19.771 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:19.796 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:19.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:19.799 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:46:19.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:19 smithi173 bash[58816]: cluster 2023-10-01T10:46:17.834507+0000 mgr.y (mgr.25072) 859 : cluster 0 pgmap v786: 89 pgs: 89 active+clean; 88 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.7 MiB/s wr, 139 op/s 2023-10-01T10:46:19.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:19 smithi173 bash[58816]: audit 2023-10-01T10:46:19.049424+0000 mon.c (mon.2) 153 : audit 1 from='client.? 172.21.15.78:0/20333987' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T10:46:19.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:19 smithi173 bash[58816]: audit 2023-10-01T10:46:19.050492+0000 mon.a (mon.0) 1087 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T10:46:19.900 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:19.929 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:19.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:19.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:46:20.034 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:20.060 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:20.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:20.062 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:46:20.106 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:19 smithi078 bash[75299]: cluster 2023-10-01T10:46:17.834507+0000 mgr.y (mgr.25072) 859 : cluster 0 pgmap v786: 89 pgs: 89 active+clean; 88 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.7 MiB/s wr, 139 op/s 2023-10-01T10:46:20.106 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:19 smithi078 bash[75299]: audit 2023-10-01T10:46:19.049424+0000 mon.c (mon.2) 153 : audit 1 from='client.? 172.21.15.78:0/20333987' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T10:46:20.106 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:19 smithi078 bash[75299]: audit 2023-10-01T10:46:19.050492+0000 mon.a (mon.0) 1087 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T10:46:20.107 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:19 smithi078 bash[78274]: cluster 2023-10-01T10:46:17.834507+0000 mgr.y (mgr.25072) 859 : cluster 0 pgmap v786: 89 pgs: 89 active+clean; 88 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.7 MiB/s wr, 139 op/s 2023-10-01T10:46:20.107 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:19 smithi078 bash[78274]: audit 2023-10-01T10:46:19.049424+0000 mon.c (mon.2) 153 : audit 1 from='client.? 172.21.15.78:0/20333987' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T10:46:20.107 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:19 smithi078 bash[78274]: audit 2023-10-01T10:46:19.050492+0000 mon.a (mon.0) 1087 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T10:46:20.163 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:20.201 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:20.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:20.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:46:20.293 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 7 16 16768 16752 149.521 127.688 0.0030778 0.00667787 2023-10-01T10:46:20.305 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:20.323 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:20.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:20.325 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:20.426 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:20.435 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:46:20 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:46:20] "GET /metrics HTTP/1.1" 200 36285 "" "Prometheus/2.33.4" 2023-10-01T10:46:20.451 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:20.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:20.455 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:46:20.556 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:20.584 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:20.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:20.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:46:20.687 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:20.722 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:20.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:20.726 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:46:20.827 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:20.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:20 smithi078 bash[75299]: cluster 2023-10-01T10:46:19.597493+0000 mon.a (mon.0) 1088 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:46:20.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:20 smithi078 bash[75299]: cluster 2023-10-01T10:46:19.602270+0000 osd.7 (osd.7) 104 : cluster 3 failed to encode map e155 with expected crc 2023-10-01T10:46:20.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:20 smithi078 bash[75299]: audit 2023-10-01T10:46:19.604768+0000 mon.a (mon.0) 1089 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T10:46:20.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:20 smithi078 bash[75299]: cluster 2023-10-01T10:46:19.605013+0000 mon.a (mon.0) 1090 : cluster 0 osdmap e155: 8 total, 8 up, 8 in 2023-10-01T10:46:20.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:20 smithi078 bash[75299]: cluster 2023-10-01T10:46:19.609810+0000 osd.6 (osd.6) 62 : cluster 3 failed to encode map e155 with expected crc 2023-10-01T10:46:20.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:20 smithi078 bash[75299]: cluster 2023-10-01T10:46:19.612592+0000 osd.6 (osd.6) 63 : cluster 3 failed to encode map e155 with expected crc 2023-10-01T10:46:20.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:20 smithi078 bash[75299]: cluster 2023-10-01T10:46:19.615078+0000 osd.6 (osd.6) 64 : cluster 3 failed to encode map e155 with expected crc 2023-10-01T10:46:20.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:20 smithi078 bash[75299]: cluster 2023-10-01T10:46:19.616298+0000 osd.6 (osd.6) 65 : cluster 3 failed to encode map e155 with expected crc 2023-10-01T10:46:20.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:20 smithi078 bash[75299]: cluster 2023-10-01T10:46:19.618647+0000 osd.5 (osd.5) 86 : cluster 3 failed to encode map e155 with expected crc 2023-10-01T10:46:20.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:20 smithi078 bash[78274]: cluster 2023-10-01T10:46:19.597493+0000 mon.a (mon.0) 1088 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:46:20.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:20 smithi078 bash[78274]: cluster 2023-10-01T10:46:19.602270+0000 osd.7 (osd.7) 104 : cluster 3 failed to encode map e155 with expected crc 2023-10-01T10:46:20.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:20 smithi078 bash[78274]: audit 2023-10-01T10:46:19.604768+0000 mon.a (mon.0) 1089 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T10:46:20.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:20 smithi078 bash[78274]: cluster 2023-10-01T10:46:19.605013+0000 mon.a (mon.0) 1090 : cluster 0 osdmap e155: 8 total, 8 up, 8 in 2023-10-01T10:46:20.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:20 smithi078 bash[78274]: cluster 2023-10-01T10:46:19.609810+0000 osd.6 (osd.6) 62 : cluster 3 failed to encode map e155 with expected crc 2023-10-01T10:46:20.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:20 smithi078 bash[78274]: cluster 2023-10-01T10:46:19.612592+0000 osd.6 (osd.6) 63 : cluster 3 failed to encode map e155 with expected crc 2023-10-01T10:46:20.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:20 smithi078 bash[78274]: cluster 2023-10-01T10:46:19.615078+0000 osd.6 (osd.6) 64 : cluster 3 failed to encode map e155 with expected crc 2023-10-01T10:46:20.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:20 smithi078 bash[78274]: cluster 2023-10-01T10:46:19.616298+0000 osd.6 (osd.6) 65 : cluster 3 failed to encode map e155 with expected crc 2023-10-01T10:46:20.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:20 smithi078 bash[78274]: cluster 2023-10-01T10:46:19.618647+0000 osd.5 (osd.5) 86 : cluster 3 failed to encode map e155 with expected crc 2023-10-01T10:46:20.837 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:20 smithi173 bash[58816]: cluster 2023-10-01T10:46:19.597493+0000 mon.a (mon.0) 1088 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:46:20.837 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:20 smithi173 bash[58816]: cluster 2023-10-01T10:46:19.602270+0000 osd.7 (osd.7) 104 : cluster 3 failed to encode map e155 with expected crc 2023-10-01T10:46:20.837 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:20 smithi173 bash[58816]: audit 2023-10-01T10:46:19.604768+0000 mon.a (mon.0) 1089 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T10:46:20.837 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:20 smithi173 bash[58816]: cluster 2023-10-01T10:46:19.605013+0000 mon.a (mon.0) 1090 : cluster 0 osdmap e155: 8 total, 8 up, 8 in 2023-10-01T10:46:20.837 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:20 smithi173 bash[58816]: cluster 2023-10-01T10:46:19.609810+0000 osd.6 (osd.6) 62 : cluster 3 failed to encode map e155 with expected crc 2023-10-01T10:46:20.837 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:20 smithi173 bash[58816]: cluster 2023-10-01T10:46:19.612592+0000 osd.6 (osd.6) 63 : cluster 3 failed to encode map e155 with expected crc 2023-10-01T10:46:20.837 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:20 smithi173 bash[58816]: cluster 2023-10-01T10:46:19.615078+0000 osd.6 (osd.6) 64 : cluster 3 failed to encode map e155 with expected crc 2023-10-01T10:46:20.837 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:20 smithi173 bash[58816]: cluster 2023-10-01T10:46:19.616298+0000 osd.6 (osd.6) 65 : cluster 3 failed to encode map e155 with expected crc 2023-10-01T10:46:20.837 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:20 smithi173 bash[58816]: cluster 2023-10-01T10:46:19.618647+0000 osd.5 (osd.5) 86 : cluster 3 failed to encode map e155 with expected crc 2023-10-01T10:46:20.852 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:20.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:20.855 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:46:20.956 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:20.982 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:20.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:20.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:21.086 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:21.115 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:21.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:21.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:46:21.218 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:21.245 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:21.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:21.249 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:46:21.293 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 8 16 18813 18797 146.806 127.812 0.0110199 0.00680287 2023-10-01T10:46:21.350 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:21.375 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:21.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:21.377 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:46:21.478 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:21.509 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:21.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:21.516 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:46:21.617 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:21.643 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:21.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:21.646 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:46:21.747 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:21.758 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:21 smithi173 bash[58816]: cluster 2023-10-01T10:46:19.835345+0000 mgr.y (mgr.25072) 860 : cluster 0 pgmap v788: 89 pgs: 89 active+clean; 801 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-10-01T10:46:21.774 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:21.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:21.779 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:46:21.880 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:21.904 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:21.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:21.906 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:46:22.007 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:22.041 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:22.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:22.046 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:46:22.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:21 smithi078 bash[78274]: cluster 2023-10-01T10:46:19.835345+0000 mgr.y (mgr.25072) 860 : cluster 0 pgmap v788: 89 pgs: 89 active+clean; 801 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-10-01T10:46:22.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:21 smithi078 bash[75299]: cluster 2023-10-01T10:46:19.835345+0000 mgr.y (mgr.25072) 860 : cluster 0 pgmap v788: 89 pgs: 89 active+clean; 801 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-10-01T10:46:22.115 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T10:46:22.147 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:22.177 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:22.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:22.180 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:46:22.280 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:22.293 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 9 16 20376 20360 141.348 97.6875 0.0111414 0.0070655 2023-10-01T10:46:22.307 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:22.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:22.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:46:22.411 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:22.437 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:22.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:22.440 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:46:22.541 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:22.572 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:22.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:22.576 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:46:22.677 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:22.710 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:22.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:22.715 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:46:22.815 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:22.842 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:22.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:22.847 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:46:22.948 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:22.974 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:22.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:22.978 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:46:23.079 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:23.109 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:23.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:23.117 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:46:23.218 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:23.254 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:23.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:23.256 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:23.293 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 10 16 22092 22076 137.938 107.25 0.00644439 0.00724271 2023-10-01T10:46:23.357 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:23.384 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:23.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:23.387 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:46:23.488 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:23.516 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:23.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:23.520 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:46:23.621 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:23.649 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:23.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:23.652 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:23.752 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:23.775 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:23.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:23.777 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:46:23.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:23 smithi078 bash[78274]: cluster 2023-10-01T10:46:21.835770+0000 mgr.y (mgr.25072) 861 : cluster 0 pgmap v789: 89 pgs: 89 active+clean; 847 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2023-10-01T10:46:23.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:23 smithi078 bash[78274]: audit 2023-10-01T10:46:22.664526+0000 mon.a (mon.0) 1091 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:46:23.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:23 smithi078 bash[78274]: audit 2023-10-01T10:46:22.679656+0000 mon.a (mon.0) 1092 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:46:23.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:23 smithi078 bash[75299]: cluster 2023-10-01T10:46:21.835770+0000 mgr.y (mgr.25072) 861 : cluster 0 pgmap v789: 89 pgs: 89 active+clean; 847 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2023-10-01T10:46:23.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:23 smithi078 bash[75299]: audit 2023-10-01T10:46:22.664526+0000 mon.a (mon.0) 1091 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:46:23.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:23 smithi078 bash[75299]: audit 2023-10-01T10:46:22.679656+0000 mon.a (mon.0) 1092 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:46:23.878 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:23.904 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:23.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:23.908 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:46:24.009 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:24.037 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:24.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:24.042 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:46:24.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:23 smithi173 bash[58816]: cluster 2023-10-01T10:46:21.835770+0000 mgr.y (mgr.25072) 861 : cluster 0 pgmap v789: 89 pgs: 89 active+clean; 847 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2023-10-01T10:46:24.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:23 smithi173 bash[58816]: audit 2023-10-01T10:46:22.664526+0000 mon.a (mon.0) 1091 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:46:24.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:23 smithi173 bash[58816]: audit 2023-10-01T10:46:22.679656+0000 mon.a (mon.0) 1092 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:46:24.142 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:24.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:24.172 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:24.172 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:46:24.224 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T10:46:24.273 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:24.294 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 11 16 23985 23969 136.15 118.312 0.00434957 0.00733807 2023-10-01T10:46:24.308 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:24.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:24.311 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:24.411 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:24.449 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:24.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:24.452 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:46:24.552 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:24.588 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:24.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:24.590 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:24.691 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:24.722 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:24.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:24.724 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:46:24.730 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:46:24.825 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:24.859 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:24.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:24.861 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:46:24.962 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:24.994 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:24.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:24.997 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:46:25.098 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:25.121 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:25.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:25.124 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:46:25.225 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:25.272 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:25.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:25.275 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:46:25.294 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 12 16 25745 25729 133.968 110 0.0100976 0.00745749 2023-10-01T10:46:25.376 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:25.413 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:25.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:25.417 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:46:25.518 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:25.537 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:25.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:25.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:46:25.640 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:25.665 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:25.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:25.669 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:46:25.681 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:25 smithi173 bash[58816]: cluster 2023-10-01T10:46:23.836334+0000 mgr.y (mgr.25072) 862 : cluster 0 pgmap v790: 89 pgs: 89 active+clean; 1.2 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 124 MiB/s wr, 1.98k op/s 2023-10-01T10:46:25.769 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:25.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:25 smithi078 bash[78274]: cluster 2023-10-01T10:46:23.836334+0000 mgr.y (mgr.25072) 862 : cluster 0 pgmap v790: 89 pgs: 89 active+clean; 1.2 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 124 MiB/s wr, 1.98k op/s 2023-10-01T10:46:25.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:25 smithi078 bash[75299]: cluster 2023-10-01T10:46:23.836334+0000 mgr.y (mgr.25072) 862 : cluster 0 pgmap v790: 89 pgs: 89 active+clean; 1.2 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 124 MiB/s wr, 1.98k op/s 2023-10-01T10:46:25.832 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:25.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:25.835 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:46:25.935 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:25.962 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:25.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:25.965 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:46:26.066 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:26.101 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:26.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:26.104 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:26.205 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:26.228 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:26.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:26.230 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:46:26.294 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 13 16 27411 27395 131.672 104.125 0.00728636 0.00758871 2023-10-01T10:46:26.331 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:26.359 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:26.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:26.363 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:46:26.464 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:26.492 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:26.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:26.494 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:46:26.595 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:26.619 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:26.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:26.622 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:46:26.723 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:26.750 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:26.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:26.754 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:46:26.854 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:26.881 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:26.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:26.885 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:46:26.985 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:27.011 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:27.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:27.012 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:46:27.113 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:27.145 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:27.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:27.150 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:46:27.250 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:27.281 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:27.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:27.286 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:27.294 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 14 16 29066 29050 129.655 103.438 0.00799088 0.00770527 2023-10-01T10:46:27.387 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:27.419 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:27.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:27.420 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:46:27.432 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:27 smithi078 bash[78274]: cluster 2023-10-01T10:46:25.836791+0000 mgr.y (mgr.25072) 863 : cluster 0 pgmap v791: 89 pgs: 89 active+clean; 1.4 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 134 MiB/s wr, 2.15k op/s 2023-10-01T10:46:27.432 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:27 smithi078 bash[78274]: audit 2023-10-01T10:46:26.430449+0000 mon.a (mon.0) 1093 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:46:27.432 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:27 smithi078 bash[78274]: audit 2023-10-01T10:46:26.438274+0000 mon.a (mon.0) 1094 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:46:27.433 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:27 smithi078 bash[78274]: audit 2023-10-01T10:46:27.063815+0000 mon.a (mon.0) 1095 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:46:27.433 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:27 smithi078 bash[78274]: audit 2023-10-01T10:46:27.065085+0000 mon.a (mon.0) 1096 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:46:27.433 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:27 smithi078 bash[78274]: audit 2023-10-01T10:46:27.073692+0000 mon.a (mon.0) 1097 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:46:27.433 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:27 smithi078 bash[78274]: audit 2023-10-01T10:46:27.423336+0000 mon.b (mon.1) 42 : audit 0 from='client.? 172.21.15.78:0/296949990' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:46:27.433 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:27 smithi078 bash[75299]: cluster 2023-10-01T10:46:25.836791+0000 mgr.y (mgr.25072) 863 : cluster 0 pgmap v791: 89 pgs: 89 active+clean; 1.4 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 134 MiB/s wr, 2.15k op/s 2023-10-01T10:46:27.433 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:27 smithi078 bash[75299]: audit 2023-10-01T10:46:26.430449+0000 mon.a (mon.0) 1093 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:46:27.433 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:27 smithi078 bash[75299]: audit 2023-10-01T10:46:26.438274+0000 mon.a (mon.0) 1094 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:46:27.433 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:27 smithi078 bash[75299]: audit 2023-10-01T10:46:27.063815+0000 mon.a (mon.0) 1095 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:46:27.433 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:27 smithi078 bash[75299]: audit 2023-10-01T10:46:27.065085+0000 mon.a (mon.0) 1096 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:46:27.434 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:27 smithi078 bash[75299]: audit 2023-10-01T10:46:27.073692+0000 mon.a (mon.0) 1097 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:46:27.434 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:27 smithi078 bash[75299]: audit 2023-10-01T10:46:27.423336+0000 mon.b (mon.1) 42 : audit 0 from='client.? 172.21.15.78:0/296949990' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:46:27.465 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T10:46:27.502 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T10:46:27.521 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:27.543 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T10:46:27.579 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:27.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:27.585 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:46:27.591 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T10:46:27.636 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T10:46:27.671 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T10:46:27.685 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:27.696 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:27 smithi173 bash[58816]: cluster 2023-10-01T10:46:25.836791+0000 mgr.y (mgr.25072) 863 : cluster 0 pgmap v791: 89 pgs: 89 active+clean; 1.4 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 134 MiB/s wr, 2.15k op/s 2023-10-01T10:46:27.696 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:27 smithi173 bash[58816]: audit 2023-10-01T10:46:26.430449+0000 mon.a (mon.0) 1093 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:46:27.696 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:27 smithi173 bash[58816]: audit 2023-10-01T10:46:26.438274+0000 mon.a (mon.0) 1094 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:46:27.696 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:27 smithi173 bash[58816]: audit 2023-10-01T10:46:27.063815+0000 mon.a (mon.0) 1095 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:46:27.696 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:27 smithi173 bash[58816]: audit 2023-10-01T10:46:27.065085+0000 mon.a (mon.0) 1096 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:46:27.696 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:27 smithi173 bash[58816]: audit 2023-10-01T10:46:27.073692+0000 mon.a (mon.0) 1097 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:46:27.696 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:27 smithi173 bash[58816]: audit 2023-10-01T10:46:27.423336+0000 mon.b (mon.1) 42 : audit 0 from='client.? 172.21.15.78:0/296949990' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:46:27.716 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:27.717 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T10:46:27.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:27.718 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:46:27.759 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T10:46:27.761 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T10:46:27.761 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T10:46:27.761 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T10:46:27.761 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T10:46:27.761 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T10:46:27.761 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T10:46:27.761 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T10:46:27.761 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T10:46:27.819 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:27.852 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:27.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:27.857 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:46:27.958 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:27.983 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:27.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:27.987 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:28.088 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:28.116 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:28.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:28.120 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:46:28.221 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:28.252 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:28.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:28.256 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:46:28.295 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 15 16 30728 30712 127.932 103.875 0.0124238 0.00780956 2023-10-01T10:46:28.356 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:28.389 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:28.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:28.392 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:28.493 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:28.519 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:28.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:28.521 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:46:28.622 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:28.648 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:28.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:28.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:46:28.754 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:28.777 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:28.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:28.781 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:46:28.881 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:28.909 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:28.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:28.912 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:46:29.013 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:29.040 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:29.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:29.043 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:46:29.144 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:29.175 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:29.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:29.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:46:29.280 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:29.295 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 16 16 32419 32403 126.535 105.688 0.0151344 0.00789605 2023-10-01T10:46:29.303 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:29.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:29.307 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:46:29.408 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:29.442 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:29.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:29.447 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:46:29.547 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:29.577 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:29.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:29.581 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:46:29.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:29 smithi078 bash[78274]: cluster 2023-10-01T10:46:27.837938+0000 mgr.y (mgr.25072) 864 : cluster 0 pgmap v792: 89 pgs: 89 active+clean; 1.4 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 135 MiB/s wr, 2.15k op/s 2023-10-01T10:46:29.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:29 smithi078 bash[75299]: cluster 2023-10-01T10:46:27.837938+0000 mgr.y (mgr.25072) 864 : cluster 0 pgmap v792: 89 pgs: 89 active+clean; 1.4 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 135 MiB/s wr, 2.15k op/s 2023-10-01T10:46:29.682 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:29.690 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:29 smithi173 bash[58816]: cluster 2023-10-01T10:46:27.837938+0000 mgr.y (mgr.25072) 864 : cluster 0 pgmap v792: 89 pgs: 89 active+clean; 1.4 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 135 MiB/s wr, 2.15k op/s 2023-10-01T10:46:29.706 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:29.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:29.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:46:29.812 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:29.843 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:29.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:29.847 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:46:29.948 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:29.964 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-10-01T10:46:29.964 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [], 2023-10-01T10:46:29.964 INFO:teuthology.orchestra.run.smithi173.stdout: "num_ops": 0 2023-10-01T10:46:29.964 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-10-01T10:46:29.978 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:29.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:29.980 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:46:30.080 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:30.109 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:30.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:30.112 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:30.213 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:30.243 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:30.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:30.245 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:46:30.296 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 17 16 34062 34046 125.132 102.688 0.00477842 0.00798483 2023-10-01T10:46:30.346 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:30.368 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:46:30 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:46:30] "GET /metrics HTTP/1.1" 200 36285 "" "Prometheus/2.33.4" 2023-10-01T10:46:30.375 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:30.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:30.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:46:30.430 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:46:30.480 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:30.513 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:30.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:30.516 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:46:30.617 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:30.647 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:30.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:30.653 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:46:30.754 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:30.782 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:30.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:30.786 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:46:30.887 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:30.914 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:30.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:30.916 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:46:31.017 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:31.046 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:31.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:31.049 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:31.150 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:31.176 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:31.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:31.180 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:46:31.280 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:31.282 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-01T10:46:31.295 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 18 16 35655 35639 123.711 99.5625 0.00673955 0.00807755 2023-10-01T10:46:31.351 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:31.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:31.355 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:46:31.456 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:31.482 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:31.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:31.485 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:46:31.586 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:31.611 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:31.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:31.614 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:46:31.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:31 smithi173 bash[58816]: cluster 2023-10-01T10:46:29.838685+0000 mgr.y (mgr.25072) 865 : cluster 0 pgmap v793: 89 pgs: 89 active+clean; 1.9 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2023-10-01T10:46:31.715 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:31.728 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:31 smithi078 bash[75299]: cluster 2023-10-01T10:46:29.838685+0000 mgr.y (mgr.25072) 865 : cluster 0 pgmap v793: 89 pgs: 89 active+clean; 1.9 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2023-10-01T10:46:31.728 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:31 smithi078 bash[78274]: cluster 2023-10-01T10:46:29.838685+0000 mgr.y (mgr.25072) 865 : cluster 0 pgmap v793: 89 pgs: 89 active+clean; 1.9 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2023-10-01T10:46:31.742 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:31.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:31.745 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:46:31.846 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:31.871 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:31.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:31.874 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:46:31.975 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:31.977 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T10:46:31.998 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:32.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:32.001 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:32.102 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:32.137 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:32.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:32.140 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:46:32.241 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:32.267 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:32.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:32.270 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:46:32.296 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 19 16 37210 37194 122.315 97.1875 0.0203495 0.00816926 2023-10-01T10:46:32.371 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:32.395 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:32.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:32.400 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:46:32.500 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:32.520 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:32.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:32.521 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:32.622 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:32.649 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:32.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:32.651 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:46:32.752 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:32.779 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:32.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:32.783 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:46:32.883 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:32.908 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:32.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:32.910 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:46:33.011 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:33.034 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:33.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:33.037 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:46:33.138 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:33.165 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:33.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:33.168 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:33.269 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:33.296 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:2023-10-01T10:46:33.295293+0000 min lat: 0.00137887 max lat: 0.128804 avg lat: 0.00826592 2023-10-01T10:46:33.296 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T10:46:33.296 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 20 16 38706 38690 120.874 93.5 0.0154509 0.00826592 2023-10-01T10:46:33.298 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:33.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:33.303 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:46:33.404 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:33.437 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:33.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:33.440 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:33.541 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:33.566 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:33.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:33.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:46:33.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:33 smithi173 bash[58816]: cluster 2023-10-01T10:46:31.839667+0000 mgr.y (mgr.25072) 866 : cluster 0 pgmap v794: 89 pgs: 89 active+clean; 1.9 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s 2023-10-01T10:46:33.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:33 smithi173 bash[58816]: audit 2023-10-01T10:46:32.421532+0000 mon.a (mon.0) 1098 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:46:33.673 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:33.682 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:33 smithi078 bash[78274]: cluster 2023-10-01T10:46:31.839667+0000 mgr.y (mgr.25072) 866 : cluster 0 pgmap v794: 89 pgs: 89 active+clean; 1.9 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s 2023-10-01T10:46:33.682 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:33 smithi078 bash[78274]: audit 2023-10-01T10:46:32.421532+0000 mon.a (mon.0) 1098 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:46:33.682 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:33 smithi078 bash[75299]: cluster 2023-10-01T10:46:31.839667+0000 mgr.y (mgr.25072) 866 : cluster 0 pgmap v794: 89 pgs: 89 active+clean; 1.9 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s 2023-10-01T10:46:33.683 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:33 smithi078 bash[75299]: audit 2023-10-01T10:46:32.421532+0000 mon.a (mon.0) 1098 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:46:33.701 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:33.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:33.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:46:33.752 INFO:teuthology.orchestra.run.smithi078.stdout: 2023-10-01T10:46:33.764 INFO:teuthology.orchestra.run.smithi078.stderr:dumped all 2023-10-01T10:46:33.806 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:33.833 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:33.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:33.836 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:46:33.937 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:33.969 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:34.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:34.263 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:46:34.296 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 21 16 40092 40076 119.243 86.625 0.0117845 0.00838045 2023-10-01T10:46:34.364 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:34.402 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:34.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:34.437 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:46:34.538 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:34.646 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:34.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:34.664 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:46:34.765 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:34.904 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:34 smithi173 bash[58816]: audit 2023-10-01T10:46:33.735557+0000 mgr.y (mgr.25072) 867 : audit 0 from='client.45836 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:46:34.905 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:34 smithi078 bash[75299]: audit 2023-10-01T10:46:33.735557+0000 mgr.y (mgr.25072) 867 : audit 0 from='client.45836 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:46:34.905 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:34 smithi078 bash[78274]: audit 2023-10-01T10:46:33.735557+0000 mgr.y (mgr.25072) 867 : audit 0 from='client.45836 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:46:34.930 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:34.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:34.934 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:46:35.035 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:35.068 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:35.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:35.072 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:46:35.173 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:35.202 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:35.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:35.205 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:46:35.296 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 22 16 41327 41311 117.331 77.1875 0.00868188 0.00851732 2023-10-01T10:46:35.535 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:35.577 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:35.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:35.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:46:35.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:35 smithi173 bash[58816]: cluster 2023-10-01T10:46:33.840786+0000 mgr.y (mgr.25072) 868 : cluster 0 pgmap v795: 89 pgs: 89 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 122 MiB/s wr, 1.95k op/s 2023-10-01T10:46:35.684 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:35.693 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:35 smithi078 bash[78274]: cluster 2023-10-01T10:46:33.840786+0000 mgr.y (mgr.25072) 868 : cluster 0 pgmap v795: 89 pgs: 89 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 122 MiB/s wr, 1.95k op/s 2023-10-01T10:46:35.694 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:35 smithi078 bash[75299]: cluster 2023-10-01T10:46:33.840786+0000 mgr.y (mgr.25072) 868 : cluster 0 pgmap v795: 89 pgs: 89 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 122 MiB/s wr, 1.95k op/s 2023-10-01T10:46:35.717 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:35.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:35.722 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:46:35.823 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:35.873 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:35.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:35.876 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:46:35.976 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:36.007 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:36.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:36.013 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:46:36.114 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:36.144 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:36.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:36.147 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:46:36.185 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T10:46:36.247 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:36.265 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:36.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:36.267 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:46:36.296 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 23 16 42760 42744 116.124 89.5625 0.0579417 0.00858702 2023-10-01T10:46:36.367 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:36.387 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:36.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:36.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:36.490 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:36.520 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:36.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:36.522 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:46:36.623 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:36.649 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:36.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:36.653 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:46:36.754 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:36.779 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:36.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:36.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:46:36.884 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:36.915 INFO:teuthology.orchestra.run.smithi078.stdout:{"pg_ready":true,"pg_map":{"version":794,"stamp":"2023-10-01T10:46:31.839342+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2064717728,"num_objects":31520,"num_object_clones":0,"num_object_copies":94560,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":31520,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":31757,"num_write_kb":2018190,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":31691,"ondisk_log_size":31691,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":9867072,"kb_used_data":6051136,"kb_used_omap":30,"kb_used_meta":3815841,"kb_avail":740094144,"statfs":{"total":767960285184,"available":757856403456,"internally_reserved":0,"allocated":6196363264,"data_stored":6191852645,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31502,"internal_metadata":3907421426},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,3,4,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1224474624,"num_objects":18684,"num_object_clones":0,"num_object_copies":56052,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":18684,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":18684,"num_write_kb":1195776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004194"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":144,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:11.744629+0000","last_change":"2023-10-01T10:38:16.032258+0000","last_active":"2023-10-01T10:46:11.744629+0000","last_peered":"2023-10-01T10:46:11.744629+0000","last_clean":"2023-10-01T10:46:11.744629+0000","last_became_active":"2023-10-01T10:38:16.031633+0000","last_became_peered":"2023-10-01T10:38:16.031633+0000","last_unstale":"2023-10-01T10:46:11.744629+0000","last_undegraded":"2023-10-01T10:46:11.744629+0000","last_fullsized":"2023-10-01T10:46:11.744629+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:17:02.446430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":132,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:12.244060+0000","last_change":"2023-10-01T10:38:16.025881+0000","last_active":"2023-10-01T10:46:12.244060+0000","last_peered":"2023-10-01T10:46:12.244060+0000","last_clean":"2023-10-01T10:46:12.244060+0000","last_became_active":"2023-10-01T10:38:16.025518+0000","last_became_peered":"2023-10-01T10:38:16.025518+0000","last_unstale":"2023-10-01T10:46:12.244060+0000","last_undegraded":"2023-10-01T10:46:12.244060+0000","last_fullsized":"2023-10-01T10:46:12.244060+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:48:37.826604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":110,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:09.808290+0000","last_change":"2023-10-01T10:38:16.025075+0000","last_active":"2023-10-01T10:46:09.808290+0000","last_peered":"2023-10-01T10:46:09.808290+0000","last_clean":"2023-10-01T10:46:09.808290+0000","last_became_active":"2023-10-01T10:38:16.024735+0000","last_became_peered":"2023-10-01T10:38:16.024735+0000","last_unstale":"2023-10-01T10:46:09.808290+0000","last_undegraded":"2023-10-01T10:46:09.808290+0000","last_fullsized":"2023-10-01T10:46:09.808290+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:38:01.450741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":115,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-01T10:46:19.615578+0000","last_change":"2023-10-01T10:37:42.772008+0000","last_active":"2023-10-01T10:46:19.615578+0000","last_peered":"2023-10-01T10:46:19.615578+0000","last_clean":"2023-10-01T10:46:19.615578+0000","last_became_active":"2023-10-01T10:37:42.771244+0000","last_became_peered":"2023-10-01T10:37:42.771244+0000","last_unstale":"2023-10-01T10:46:19.615578+0000","last_undegraded":"2023-10-01T10:46:19.615578+0000","last_fullsized":"2023-10-01T10:46:19.615578+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:38:55.180790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":133,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:12.243441+0000","last_change":"2023-10-01T10:37:42.766600+0000","last_active":"2023-10-01T10:46:12.243441+0000","last_peered":"2023-10-01T10:46:12.243441+0000","last_clean":"2023-10-01T10:46:12.243441+0000","last_became_active":"2023-10-01T10:37:42.766051+0000","last_became_peered":"2023-10-01T10:37:42.766051+0000","last_unstale":"2023-10-01T10:46:12.243441+0000","last_undegraded":"2023-10-01T10:46:12.243441+0000","last_fullsized":"2023-10-01T10:46:12.243441+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:51:17.707963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":120,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:12.246612+0000","last_change":"2023-10-01T10:32:31.575109+0000","last_active":"2023-10-01T10:46:12.246612+0000","last_peered":"2023-10-01T10:46:12.246612+0000","last_clean":"2023-10-01T10:46:12.246612+0000","last_became_active":"2023-10-01T10:32:31.574729+0000","last_became_peered":"2023-10-01T10:32:31.574729+0000","last_unstale":"2023-10-01T10:46:12.246612+0000","last_undegraded":"2023-10-01T10:46:12.246612+0000","last_fullsized":"2023-10-01T10:46:12.246612+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":112,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:12.244475+0000","last_change":"2023-10-01T10:37:42.766667+0000","last_active":"2023-10-01T10:46:12.244475+0000","last_peered":"2023-10-01T10:46:12.244475+0000","last_clean":"2023-10-01T10:46:12.244475+0000","last_became_active":"2023-10-01T10:37:42.766369+0000","last_became_peered":"2023-10-01T10:37:42.766369+0000","last_unstale":"2023-10-01T10:46:12.244475+0000","last_undegraded":"2023-10-01T10:46:12.244475+0000","last_fullsized":"2023-10-01T10:46:12.244475+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:02.708788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":117,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:12.243430+0000","last_change":"2023-10-01T10:37:42.765925+0000","last_active":"2023-10-01T10:46:12.243430+0000","last_peered":"2023-10-01T10:46:12.243430+0000","last_clean":"2023-10-01T10:46:12.243430+0000","last_became_active":"2023-10-01T10:37:42.765218+0000","last_became_peered":"2023-10-01T10:37:42.765218+0000","last_unstale":"2023-10-01T10:46:12.243430+0000","last_undegraded":"2023-10-01T10:46:12.243430+0000","last_fullsized":"2023-10-01T10:46:12.243430+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T22:10:34.338031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":122,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:09.815564+0000","last_change":"2023-10-01T10:32:31.579805+0000","last_active":"2023-10-01T10:46:09.815564+0000","last_peered":"2023-10-01T10:46:09.815564+0000","last_clean":"2023-10-01T10:46:09.815564+0000","last_became_active":"2023-10-01T10:32:31.579423+0000","last_became_peered":"2023-10-01T10:32:31.579423+0000","last_unstale":"2023-10-01T10:46:09.815564+0000","last_undegraded":"2023-10-01T10:46:09.815564+0000","last_fullsized":"2023-10-01T10:46:09.815564+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":101,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:12.244290+0000","last_change":"2023-10-01T10:37:42.772531+0000","last_active":"2023-10-01T10:46:12.244290+0000","last_peered":"2023-10-01T10:46:12.244290+0000","last_clean":"2023-10-01T10:46:12.244290+0000","last_became_active":"2023-10-01T10:37:42.772216+0000","last_became_peered":"2023-10-01T10:37:42.772216+0000","last_unstale":"2023-10-01T10:46:12.244290+0000","last_undegraded":"2023-10-01T10:46:12.244290+0000","last_fullsized":"2023-10-01T10:46:12.244290+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:01:53.373628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":140,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:12.166134+0000","last_change":"2023-10-01T10:38:16.027085+0000","last_active":"2023-10-01T10:46:12.166134+0000","last_peered":"2023-10-01T10:46:12.166134+0000","last_clean":"2023-10-01T10:46:12.166134+0000","last_became_active":"2023-10-01T10:38:16.026555+0000","last_became_peered":"2023-10-01T10:38:16.026555+0000","last_unstale":"2023-10-01T10:46:12.166134+0000","last_undegraded":"2023-10-01T10:46:12.166134+0000","last_fullsized":"2023-10-01T10:46:12.166134+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":96,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-01T10:46:19.610691+0000","last_change":"2023-10-01T10:37:05.018755+0000","last_active":"2023-10-01T10:46:19.610691+0000","last_peered":"2023-10-01T10:46:19.610691+0000","last_clean":"2023-10-01T10:46:19.610691+0000","last_became_active":"2023-10-01T10:37:05.018446+0000","last_became_peered":"2023-10-01T10:37:05.018446+0000","last_unstale":"2023-10-01T10:46:19.610691+0000","last_undegraded":"2023-10-01T10:46:19.610691+0000","last_fullsized":"2023-10-01T10:46:19.610691+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:13:53.740503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":138,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:11.744834+0000","last_change":"2023-10-01T10:38:16.030826+0000","last_active":"2023-10-01T10:46:11.744834+0000","last_peered":"2023-10-01T10:46:11.744834+0000","last_clean":"2023-10-01T10:46:11.744834+0000","last_became_active":"2023-10-01T10:38:16.030277+0000","last_became_peered":"2023-10-01T10:38:16.030277+0000","last_unstale":"2023-10-01T10:46:11.744834+0000","last_undegraded":"2023-10-01T10:46:11.744834+0000","last_fullsized":"2023-10-01T10:46:11.744834+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:31:43.040424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":112,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:11.744469+0000","last_change":"2023-10-01T10:38:16.034741+0000","last_active":"2023-10-01T10:46:11.744469+0000","last_peered":"2023-10-01T10:46:11.744469+0000","last_clean":"2023-10-01T10:46:11.744469+0000","last_became_active":"2023-10-01T10:38:16.034407+0000","last_became_peered":"2023-10-01T10:38:16.034407+0000","last_unstale":"2023-10-01T10:46:11.744469+0000","last_undegraded":"2023-10-01T10:46:11.744469+0000","last_fullsized":"2023-10-01T10:46:11.744469+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:47:29.911477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":139,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:12.165642+0000","last_change":"2023-10-01T10:38:16.026777+0000","last_active":"2023-10-01T10:46:12.165642+0000","last_peered":"2023-10-01T10:46:12.165642+0000","last_clean":"2023-10-01T10:46:12.165642+0000","last_became_active":"2023-10-01T10:38:16.026371+0000","last_became_peered":"2023-10-01T10:38:16.026371+0000","last_unstale":"2023-10-01T10:46:12.165642+0000","last_undegraded":"2023-10-01T10:46:12.165642+0000","last_fullsized":"2023-10-01T10:46:12.165642+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":140,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:12.243870+0000","last_change":"2023-10-01T10:37:42.772319+0000","last_active":"2023-10-01T10:46:12.243870+0000","last_peered":"2023-10-01T10:46:12.243870+0000","last_clean":"2023-10-01T10:46:12.243870+0000","last_became_active":"2023-10-01T10:37:42.771977+0000","last_became_peered":"2023-10-01T10:37:42.771977+0000","last_unstale":"2023-10-01T10:46:12.243870+0000","last_undegraded":"2023-10-01T10:46:12.243870+0000","last_fullsized":"2023-10-01T10:46:12.243870+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:45:12.783214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":112,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:12.244434+0000","last_change":"2023-10-01T10:37:42.767706+0000","last_active":"2023-10-01T10:46:12.244434+0000","last_peered":"2023-10-01T10:46:12.244434+0000","last_clean":"2023-10-01T10:46:12.244434+0000","last_became_active":"2023-10-01T10:37:42.767211+0000","last_became_peered":"2023-10-01T10:37:42.767211+0000","last_unstale":"2023-10-01T10:46:12.244434+0000","last_undegraded":"2023-10-01T10:46:12.244434+0000","last_fullsized":"2023-10-01T10:46:12.244434+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:37:20.357407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":121,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:09.815514+0000","last_change":"2023-10-01T10:37:42.768642+0000","last_active":"2023-10-01T10:46:09.815514+0000","last_peered":"2023-10-01T10:46:09.815514+0000","last_clean":"2023-10-01T10:46:09.815514+0000","last_became_active":"2023-10-01T10:37:42.768239+0000","last_became_peered":"2023-10-01T10:37:42.768239+0000","last_unstale":"2023-10-01T10:46:09.815514+0000","last_undegraded":"2023-10-01T10:46:09.815514+0000","last_fullsized":"2023-10-01T10:46:09.815514+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":122,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:12.165871+0000","last_change":"2023-10-01T10:38:16.028563+0000","last_active":"2023-10-01T10:46:12.165871+0000","last_peered":"2023-10-01T10:46:12.165871+0000","last_clean":"2023-10-01T10:46:12.165871+0000","last_became_active":"2023-10-01T10:38:16.028162+0000","last_became_peered":"2023-10-01T10:38:16.028162+0000","last_unstale":"2023-10-01T10:46:12.165871+0000","last_undegraded":"2023-10-01T10:46:12.165871+0000","last_fullsized":"2023-10-01T10:46:12.165871+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":113,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:12.244351+0000","last_change":"2023-10-01T10:37:42.772247+0000","last_active":"2023-10-01T10:46:12.244351+0000","last_peered":"2023-10-01T10:46:12.244351+0000","last_clean":"2023-10-01T10:46:12.244351+0000","last_became_active":"2023-10-01T10:37:42.771697+0000","last_became_peered":"2023-10-01T10:37:42.771697+0000","last_unstale":"2023-10-01T10:46:12.244351+0000","last_undegraded":"2023-10-01T10:46:12.244351+0000","last_fullsized":"2023-10-01T10:46:12.244351+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:23:15.018248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":129,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:12.245448+0000","last_change":"2023-10-01T10:28:20.211798+0000","last_active":"2023-10-01T10:46:12.245448+0000","last_peered":"2023-10-01T10:46:12.245448+0000","last_clean":"2023-10-01T10:46:12.245448+0000","last_became_active":"2023-10-01T10:28:20.210791+0000","last_became_peered":"2023-10-01T10:28:20.210791+0000","last_unstale":"2023-10-01T10:46:12.245448+0000","last_undegraded":"2023-10-01T10:46:12.245448+0000","last_fullsized":"2023-10-01T10:46:12.245448+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":117,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:12.245040+0000","last_change":"2023-10-01T10:37:42.773127+0000","last_active":"2023-10-01T10:46:12.245040+0000","last_peered":"2023-10-01T10:46:12.245040+0000","last_clean":"2023-10-01T10:46:12.245040+0000","last_became_active":"2023-10-01T10:37:42.772102+0000","last_became_peered":"2023-10-01T10:37:42.772102+0000","last_unstale":"2023-10-01T10:46:12.245040+0000","last_undegraded":"2023-10-01T10:46:12.245040+0000","last_fullsized":"2023-10-01T10:46:12.245040+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:23:26.202295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"155'1966","reported_seq":1978,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-01T10:46:29.295389+0000","last_change":"2023-10-01T10:45:57.053617+0000","last_active":"2023-10-01T10:46:29.295389+0000","last_peered":"2023-10-01T10:46:29.295389+0000","last_clean":"2023-10-01T10:46:29.295389+0000","last_became_active":"2023-10-01T10:45:57.053257+0000","last_became_peered":"2023-10-01T10:45:57.053257+0000","last_unstale":"2023-10-01T10:46:29.295389+0000","last_undegraded":"2023-10-01T10:46:29.295389+0000","last_fullsized":"2023-10-01T10:46:29.295389+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":151,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_clean_scrub_stamp":"2023-10-01T10:45:56.023726+0000","objects_scrubbed":0,"log_size":1966,"log_dups_size":0,"ondisk_log_size":1966,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":128843776,"num_objects":1966,"num_object_clones":0,"num_object_copies":5898,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1966,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1966,"num_write_kb":125824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":138,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:12.166357+0000","last_change":"2023-10-01T10:38:16.026955+0000","last_active":"2023-10-01T10:46:12.166357+0000","last_peered":"2023-10-01T10:46:12.166357+0000","last_clean":"2023-10-01T10:46:12.166357+0000","last_became_active":"2023-10-01T10:38:16.026552+0000","last_became_peered":"2023-10-01T10:38:16.026552+0000","last_unstale":"2023-10-01T10:46:12.166357+0000","last_undegraded":"2023-10-01T10:46:12.166357+0000","last_fullsized":"2023-10-01T10:46:12.166357+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":106,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:09.814543+0000","last_change":"2023-10-01T10:15:40.318702+0000","last_active":"2023-10-01T10:46:09.814543+0000","last_peered":"2023-10-01T10:46:09.814543+0000","last_clean":"2023-10-01T10:46:09.814543+0000","last_became_active":"2023-10-01T10:15:40.318333+0000","last_became_peered":"2023-10-01T10:15:40.318333+0000","last_unstale":"2023-10-01T10:46:09.814543+0000","last_undegraded":"2023-10-01T10:46:09.814543+0000","last_fullsized":"2023-10-01T10:46:09.814543+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"155'1874","reported_seq":1886,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-01T10:46:28.379661+0000","last_change":"2023-10-01T10:45:57.056390+0000","last_active":"2023-10-01T10:46:28.379661+0000","last_peered":"2023-10-01T10:46:28.379661+0000","last_clean":"2023-10-01T10:46:28.379661+0000","last_became_active":"2023-10-01T10:45:57.055649+0000","last_became_peered":"2023-10-01T10:45:57.055649+0000","last_unstale":"2023-10-01T10:46:28.379661+0000","last_undegraded":"2023-10-01T10:46:28.379661+0000","last_fullsized":"2023-10-01T10:46:28.379661+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":151,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_clean_scrub_stamp":"2023-10-01T10:45:56.023726+0000","objects_scrubbed":0,"log_size":1874,"log_dups_size":0,"ondisk_log_size":1874,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122814464,"num_objects":1874,"num_object_clones":0,"num_object_copies":5622,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1874,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1874,"num_write_kb":119936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":139,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:12.166435+0000","last_change":"2023-10-01T10:38:16.027753+0000","last_active":"2023-10-01T10:46:12.166435+0000","last_peered":"2023-10-01T10:46:12.166435+0000","last_clean":"2023-10-01T10:46:12.166435+0000","last_became_active":"2023-10-01T10:38:16.027384+0000","last_became_peered":"2023-10-01T10:38:16.027384+0000","last_unstale":"2023-10-01T10:46:12.166435+0000","last_undegraded":"2023-10-01T10:46:12.166435+0000","last_fullsized":"2023-10-01T10:46:12.166435+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":146,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:11.744221+0000","last_change":"2023-10-01T10:38:16.033203+0000","last_active":"2023-10-01T10:46:11.744221+0000","last_peered":"2023-10-01T10:46:11.744221+0000","last_clean":"2023-10-01T10:46:11.744221+0000","last_became_active":"2023-10-01T10:38:16.032681+0000","last_became_peered":"2023-10-01T10:38:16.032681+0000","last_unstale":"2023-10-01T10:46:11.744221+0000","last_undegraded":"2023-10-01T10:46:11.744221+0000","last_fullsized":"2023-10-01T10:46:11.744221+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:42:36.642284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"155'1918","reported_seq":1931,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-01T10:46:28.391665+0000","last_change":"2023-10-01T10:45:57.044458+0000","last_active":"2023-10-01T10:46:28.391665+0000","last_peered":"2023-10-01T10:46:28.391665+0000","last_clean":"2023-10-01T10:46:28.391665+0000","last_became_active":"2023-10-01T10:45:57.043330+0000","last_became_peered":"2023-10-01T10:45:57.043330+0000","last_unstale":"2023-10-01T10:46:28.391665+0000","last_undegraded":"2023-10-01T10:46:28.391665+0000","last_fullsized":"2023-10-01T10:46:28.391665+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":151,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_clean_scrub_stamp":"2023-10-01T10:45:56.023726+0000","objects_scrubbed":0,"log_size":1918,"log_dups_size":0,"ondisk_log_size":1918,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:19:44.671386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":125698048,"num_objects":1918,"num_object_clones":0,"num_object_copies":5754,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1918,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1918,"num_write_kb":122752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":108,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:12.165504+0000","last_change":"2023-10-01T10:15:39.293587+0000","last_active":"2023-10-01T10:46:12.165504+0000","last_peered":"2023-10-01T10:46:12.165504+0000","last_clean":"2023-10-01T10:46:12.165504+0000","last_became_active":"2023-10-01T10:15:39.293273+0000","last_became_peered":"2023-10-01T10:15:39.293273+0000","last_unstale":"2023-10-01T10:46:12.165504+0000","last_undegraded":"2023-10-01T10:46:12.165504+0000","last_fullsized":"2023-10-01T10:46:12.165504+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":120,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:12.245741+0000","last_change":"2023-10-01T10:32:31.571971+0000","last_active":"2023-10-01T10:46:12.245741+0000","last_peered":"2023-10-01T10:46:12.245741+0000","last_clean":"2023-10-01T10:46:12.245741+0000","last_became_active":"2023-10-01T10:32:31.571410+0000","last_became_peered":"2023-10-01T10:32:31.571410+0000","last_unstale":"2023-10-01T10:46:12.245741+0000","last_undegraded":"2023-10-01T10:46:12.245741+0000","last_fullsized":"2023-10-01T10:46:12.245741+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"155'1932","reported_seq":1945,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-01T10:46:28.103053+0000","last_change":"2023-10-01T10:45:57.046146+0000","last_active":"2023-10-01T10:46:28.103053+0000","last_peered":"2023-10-01T10:46:28.103053+0000","last_clean":"2023-10-01T10:46:28.103053+0000","last_became_active":"2023-10-01T10:45:57.045852+0000","last_became_peered":"2023-10-01T10:45:57.045852+0000","last_unstale":"2023-10-01T10:46:28.103053+0000","last_undegraded":"2023-10-01T10:46:28.103053+0000","last_fullsized":"2023-10-01T10:46:28.103053+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":151,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_clean_scrub_stamp":"2023-10-01T10:45:56.023726+0000","objects_scrubbed":0,"log_size":1932,"log_dups_size":0,"ondisk_log_size":1932,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:55:55.424584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":126615552,"num_objects":1932,"num_object_clones":0,"num_object_copies":5796,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1932,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1932,"num_write_kb":123648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":122,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:09.815224+0000","last_change":"2023-10-01T10:37:42.765733+0000","last_active":"2023-10-01T10:46:09.815224+0000","last_peered":"2023-10-01T10:46:09.815224+0000","last_clean":"2023-10-01T10:46:09.815224+0000","last_became_active":"2023-10-01T10:37:42.765311+0000","last_became_peered":"2023-10-01T10:37:42.765311+0000","last_unstale":"2023-10-01T10:46:09.815224+0000","last_undegraded":"2023-10-01T10:46:09.815224+0000","last_fullsized":"2023-10-01T10:46:09.815224+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":129,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:18.386174+0000","last_change":"2023-10-01T10:38:16.026348+0000","last_active":"2023-10-01T10:46:18.386174+0000","last_peered":"2023-10-01T10:46:18.386174+0000","last_clean":"2023-10-01T10:46:18.386174+0000","last_became_active":"2023-10-01T10:38:16.025747+0000","last_became_peered":"2023-10-01T10:38:16.025747+0000","last_unstale":"2023-10-01T10:46:18.386174+0000","last_undegraded":"2023-10-01T10:46:18.386174+0000","last_fullsized":"2023-10-01T10:46:18.386174+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:28:18.934480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"155'2235","reported_seq":2248,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-01T10:46:30.857776+0000","last_change":"2023-10-01T10:45:57.043636+0000","last_active":"2023-10-01T10:46:30.857776+0000","last_peered":"2023-10-01T10:46:30.857776+0000","last_clean":"2023-10-01T10:46:30.857776+0000","last_became_active":"2023-10-01T10:45:57.043317+0000","last_became_peered":"2023-10-01T10:45:57.043317+0000","last_unstale":"2023-10-01T10:46:30.857776+0000","last_undegraded":"2023-10-01T10:46:30.857776+0000","last_fullsized":"2023-10-01T10:46:30.857776+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":151,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_clean_scrub_stamp":"2023-10-01T10:45:56.023726+0000","objects_scrubbed":0,"log_size":2235,"log_dups_size":0,"ondisk_log_size":2235,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:52:18.949427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146472960,"num_objects":2235,"num_object_clones":0,"num_object_copies":6705,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2235,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2235,"num_write_kb":143040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":120,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:12.243859+0000","last_change":"2023-10-01T10:38:16.022493+0000","last_active":"2023-10-01T10:46:12.243859+0000","last_peered":"2023-10-01T10:46:12.243859+0000","last_clean":"2023-10-01T10:46:12.243859+0000","last_became_active":"2023-10-01T10:38:16.022033+0000","last_became_peered":"2023-10-01T10:38:16.022033+0000","last_unstale":"2023-10-01T10:46:12.243859+0000","last_undegraded":"2023-10-01T10:46:12.243859+0000","last_fullsized":"2023-10-01T10:46:12.243859+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:49:05.262763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":131,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:11.744937+0000","last_change":"2023-10-01T10:38:16.030613+0000","last_active":"2023-10-01T10:46:11.744937+0000","last_peered":"2023-10-01T10:46:11.744937+0000","last_clean":"2023-10-01T10:46:11.744937+0000","last_became_active":"2023-10-01T10:38:16.028975+0000","last_became_peered":"2023-10-01T10:38:16.028975+0000","last_unstale":"2023-10-01T10:46:11.744937+0000","last_undegraded":"2023-10-01T10:46:11.744937+0000","last_fullsized":"2023-10-01T10:46:11.744937+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:46:07.287911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"155'1890","reported_seq":1903,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-01T10:46:28.108645+0000","last_change":"2023-10-01T10:45:57.050946+0000","last_active":"2023-10-01T10:46:28.108645+0000","last_peered":"2023-10-01T10:46:28.108645+0000","last_clean":"2023-10-01T10:46:28.108645+0000","last_became_active":"2023-10-01T10:45:57.050609+0000","last_became_peered":"2023-10-01T10:45:57.050609+0000","last_unstale":"2023-10-01T10:46:28.108645+0000","last_undegraded":"2023-10-01T10:46:28.108645+0000","last_fullsized":"2023-10-01T10:46:28.108645+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":151,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_clean_scrub_stamp":"2023-10-01T10:45:56.023726+0000","objects_scrubbed":0,"log_size":1890,"log_dups_size":0,"ondisk_log_size":1890,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:35:24.694811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123863040,"num_objects":1890,"num_object_clones":0,"num_object_copies":5670,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1890,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1890,"num_write_kb":120960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":136,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:12.245844+0000","last_change":"2023-10-01T10:38:16.030250+0000","last_active":"2023-10-01T10:46:12.245844+0000","last_peered":"2023-10-01T10:46:12.245844+0000","last_clean":"2023-10-01T10:46:12.245844+0000","last_became_active":"2023-10-01T10:38:16.029898+0000","last_became_peered":"2023-10-01T10:38:16.029898+0000","last_unstale":"2023-10-01T10:46:12.245844+0000","last_undegraded":"2023-10-01T10:46:12.245844+0000","last_fullsized":"2023-10-01T10:46:12.245844+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":136,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:11.745130+0000","last_change":"2023-10-01T10:38:16.035906+0000","last_active":"2023-10-01T10:46:11.745130+0000","last_peered":"2023-10-01T10:46:11.745130+0000","last_clean":"2023-10-01T10:46:11.745130+0000","last_became_active":"2023-10-01T10:38:16.035383+0000","last_became_peered":"2023-10-01T10:38:16.035383+0000","last_unstale":"2023-10-01T10:46:11.745130+0000","last_undegraded":"2023-10-01T10:46:11.745130+0000","last_fullsized":"2023-10-01T10:46:11.745130+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:44:41.500097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"155'1995","reported_seq":2007,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-01T10:46:28.383630+0000","last_change":"2023-10-01T10:45:57.055262+0000","last_active":"2023-10-01T10:46:28.383630+0000","last_peered":"2023-10-01T10:46:28.383630+0000","last_clean":"2023-10-01T10:46:28.383630+0000","last_became_active":"2023-10-01T10:45:57.054388+0000","last_became_peered":"2023-10-01T10:45:57.054388+0000","last_unstale":"2023-10-01T10:46:28.383630+0000","last_undegraded":"2023-10-01T10:46:28.383630+0000","last_fullsized":"2023-10-01T10:46:28.383630+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":151,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_clean_scrub_stamp":"2023-10-01T10:45:56.023726+0000","objects_scrubbed":0,"log_size":1995,"log_dups_size":0,"ondisk_log_size":1995,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":130744320,"num_objects":1995,"num_object_clones":0,"num_object_copies":5985,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1995,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1995,"num_write_kb":127680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":128,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:11.744447+0000","last_change":"2023-10-01T10:38:16.026611+0000","last_active":"2023-10-01T10:46:11.744447+0000","last_peered":"2023-10-01T10:46:11.744447+0000","last_clean":"2023-10-01T10:46:11.744447+0000","last_became_active":"2023-10-01T10:38:16.026267+0000","last_became_peered":"2023-10-01T10:38:16.026267+0000","last_unstale":"2023-10-01T10:46:11.744447+0000","last_undegraded":"2023-10-01T10:46:11.744447+0000","last_fullsized":"2023-10-01T10:46:11.744447+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:06.135288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":112,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:11.745266+0000","last_change":"2023-10-01T10:38:16.032162+0000","last_active":"2023-10-01T10:46:11.745266+0000","last_peered":"2023-10-01T10:46:11.745266+0000","last_clean":"2023-10-01T10:46:11.745266+0000","last_became_active":"2023-10-01T10:38:16.031785+0000","last_became_peered":"2023-10-01T10:38:16.031785+0000","last_unstale":"2023-10-01T10:46:11.745266+0000","last_undegraded":"2023-10-01T10:46:11.745266+0000","last_fullsized":"2023-10-01T10:46:11.745266+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:38:43.003626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"155'1952","reported_seq":1965,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-01T10:46:29.148985+0000","last_change":"2023-10-01T10:45:57.049550+0000","last_active":"2023-10-01T10:46:29.148985+0000","last_peered":"2023-10-01T10:46:29.148985+0000","last_clean":"2023-10-01T10:46:29.148985+0000","last_became_active":"2023-10-01T10:45:57.049239+0000","last_became_peered":"2023-10-01T10:45:57.049239+0000","last_unstale":"2023-10-01T10:46:29.148985+0000","last_undegraded":"2023-10-01T10:46:29.148985+0000","last_fullsized":"2023-10-01T10:46:29.148985+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":151,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_clean_scrub_stamp":"2023-10-01T10:45:56.023726+0000","objects_scrubbed":0,"log_size":1952,"log_dups_size":0,"ondisk_log_size":1952,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:09:14.295778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":127926272,"num_objects":1952,"num_object_clones":0,"num_object_copies":5856,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1952,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1952,"num_write_kb":124928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":119,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-01T10:46:19.602928+0000","last_change":"2023-10-01T10:37:42.772398+0000","last_active":"2023-10-01T10:46:19.602928+0000","last_peered":"2023-10-01T10:46:19.602928+0000","last_clean":"2023-10-01T10:46:19.602928+0000","last_became_active":"2023-10-01T10:37:42.772084+0000","last_became_peered":"2023-10-01T10:37:42.772084+0000","last_unstale":"2023-10-01T10:46:19.602928+0000","last_undegraded":"2023-10-01T10:46:19.602928+0000","last_fullsized":"2023-10-01T10:46:19.602928+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:32:43.846419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":137,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:09.811237+0000","last_change":"2023-10-01T10:38:16.019755+0000","last_active":"2023-10-01T10:46:09.811237+0000","last_peered":"2023-10-01T10:46:09.811237+0000","last_clean":"2023-10-01T10:46:09.811237+0000","last_became_active":"2023-10-01T10:38:16.019397+0000","last_became_peered":"2023-10-01T10:38:16.019397+0000","last_unstale":"2023-10-01T10:46:09.811237+0000","last_undegraded":"2023-10-01T10:46:09.811237+0000","last_fullsized":"2023-10-01T10:46:09.811237+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:42:56.623225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.c","version":"155'1949","reported_seq":1961,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-01T10:46:28.750230+0000","last_change":"2023-10-01T10:45:57.052410+0000","last_active":"2023-10-01T10:46:28.750230+0000","last_peered":"2023-10-01T10:46:28.750230+0000","last_clean":"2023-10-01T10:46:28.750230+0000","last_became_active":"2023-10-01T10:45:57.052058+0000","last_became_peered":"2023-10-01T10:45:57.052058+0000","last_unstale":"2023-10-01T10:46:28.750230+0000","last_undegraded":"2023-10-01T10:46:28.750230+0000","last_fullsized":"2023-10-01T10:46:28.750230+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":151,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_clean_scrub_stamp":"2023-10-01T10:45:56.023726+0000","objects_scrubbed":0,"log_size":1949,"log_dups_size":0,"ondisk_log_size":1949,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":127729664,"num_objects":1949,"num_object_clones":0,"num_object_copies":5847,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1949,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1949,"num_write_kb":124736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":130,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:09.808628+0000","last_change":"2023-10-01T10:32:31.561108+0000","last_active":"2023-10-01T10:46:09.808628+0000","last_peered":"2023-10-01T10:46:09.808628+0000","last_clean":"2023-10-01T10:46:09.808628+0000","last_became_active":"2023-10-01T10:32:31.560791+0000","last_became_peered":"2023-10-01T10:32:31.560791+0000","last_unstale":"2023-10-01T10:46:09.808628+0000","last_undegraded":"2023-10-01T10:46:09.808628+0000","last_fullsized":"2023-10-01T10:46:09.808628+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:42:47.477671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":137,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:09.815271+0000","last_change":"2023-10-01T10:37:42.766814+0000","last_active":"2023-10-01T10:46:09.815271+0000","last_peered":"2023-10-01T10:46:09.815271+0000","last_clean":"2023-10-01T10:46:09.815271+0000","last_became_active":"2023-10-01T10:37:42.766451+0000","last_became_peered":"2023-10-01T10:37:42.766451+0000","last_unstale":"2023-10-01T10:46:09.815271+0000","last_undegraded":"2023-10-01T10:46:09.815271+0000","last_fullsized":"2023-10-01T10:46:09.815271+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":96,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:09.809645+0000","last_change":"2023-10-01T10:38:16.034357+0000","last_active":"2023-10-01T10:46:09.809645+0000","last_peered":"2023-10-01T10:46:09.809645+0000","last_clean":"2023-10-01T10:46:09.809645+0000","last_became_active":"2023-10-01T10:38:16.033905+0000","last_became_peered":"2023-10-01T10:38:16.033905+0000","last_unstale":"2023-10-01T10:46:09.809645+0000","last_undegraded":"2023-10-01T10:46:09.809645+0000","last_fullsized":"2023-10-01T10:46:09.809645+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:47:58.369835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"155'1923","reported_seq":1935,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-01T10:46:28.363175+0000","last_change":"2023-10-01T10:45:57.055125+0000","last_active":"2023-10-01T10:46:28.363175+0000","last_peered":"2023-10-01T10:46:28.363175+0000","last_clean":"2023-10-01T10:46:28.363175+0000","last_became_active":"2023-10-01T10:45:57.054944+0000","last_became_peered":"2023-10-01T10:45:57.054944+0000","last_unstale":"2023-10-01T10:46:28.363175+0000","last_undegraded":"2023-10-01T10:46:28.363175+0000","last_fullsized":"2023-10-01T10:46:28.363175+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":151,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_clean_scrub_stamp":"2023-10-01T10:45:56.023726+0000","objects_scrubbed":0,"log_size":1923,"log_dups_size":0,"ondisk_log_size":1923,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":126025728,"num_objects":1923,"num_object_clones":0,"num_object_copies":5769,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1923,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1923,"num_write_kb":123072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":126,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:09.814981+0000","last_change":"2023-10-01T10:32:31.572482+0000","last_active":"2023-10-01T10:46:09.814981+0000","last_peered":"2023-10-01T10:46:09.814981+0000","last_clean":"2023-10-01T10:46:09.814981+0000","last_became_active":"2023-10-01T10:32:31.571993+0000","last_became_peered":"2023-10-01T10:32:31.571993+0000","last_unstale":"2023-10-01T10:46:09.814981+0000","last_undegraded":"2023-10-01T10:46:09.814981+0000","last_fullsized":"2023-10-01T10:46:09.814981+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":98,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:11.744869+0000","last_change":"2023-10-01T10:38:16.030202+0000","last_active":"2023-10-01T10:46:11.744869+0000","last_peered":"2023-10-01T10:46:11.744869+0000","last_clean":"2023-10-01T10:46:11.744869+0000","last_became_active":"2023-10-01T10:38:16.029917+0000","last_became_peered":"2023-10-01T10:38:16.029917+0000","last_unstale":"2023-10-01T10:46:11.744869+0000","last_undegraded":"2023-10-01T10:46:11.744869+0000","last_fullsized":"2023-10-01T10:46:11.744869+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:20:06.763909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":111,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:09.809138+0000","last_change":"2023-10-01T10:37:42.761153+0000","last_active":"2023-10-01T10:46:09.809138+0000","last_peered":"2023-10-01T10:46:09.809138+0000","last_clean":"2023-10-01T10:46:09.809138+0000","last_became_active":"2023-10-01T10:37:42.760715+0000","last_became_peered":"2023-10-01T10:37:42.760715+0000","last_unstale":"2023-10-01T10:46:09.809138+0000","last_undegraded":"2023-10-01T10:46:09.809138+0000","last_fullsized":"2023-10-01T10:46:09.809138+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:52:50.237174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"155'2078","reported_seq":2090,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-01T10:46:29.291147+0000","last_change":"2023-10-01T10:45:57.053546+0000","last_active":"2023-10-01T10:46:29.291147+0000","last_peered":"2023-10-01T10:46:29.291147+0000","last_clean":"2023-10-01T10:46:29.291147+0000","last_became_active":"2023-10-01T10:45:57.053064+0000","last_became_peered":"2023-10-01T10:45:57.053064+0000","last_unstale":"2023-10-01T10:46:29.291147+0000","last_undegraded":"2023-10-01T10:46:29.291147+0000","last_fullsized":"2023-10-01T10:46:29.291147+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":151,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_clean_scrub_stamp":"2023-10-01T10:45:56.023726+0000","objects_scrubbed":0,"log_size":2078,"log_dups_size":0,"ondisk_log_size":2078,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136183808,"num_objects":2078,"num_object_clones":0,"num_object_copies":6234,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2078,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2078,"num_write_kb":132992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":128,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:09.808852+0000","last_change":"2023-10-01T10:37:42.762524+0000","last_active":"2023-10-01T10:46:09.808852+0000","last_peered":"2023-10-01T10:46:09.808852+0000","last_clean":"2023-10-01T10:46:09.808852+0000","last_became_active":"2023-10-01T10:37:42.762209+0000","last_became_peered":"2023-10-01T10:37:42.762209+0000","last_unstale":"2023-10-01T10:46:09.808852+0000","last_undegraded":"2023-10-01T10:46:09.808852+0000","last_fullsized":"2023-10-01T10:46:09.808852+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:45:38.171397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":97,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-01T10:46:19.611485+0000","last_change":"2023-10-01T10:37:05.019330+0000","last_active":"2023-10-01T10:46:19.611485+0000","last_peered":"2023-10-01T10:46:19.611485+0000","last_clean":"2023-10-01T10:46:19.611485+0000","last_became_active":"2023-10-01T10:37:05.018842+0000","last_became_peered":"2023-10-01T10:37:05.018842+0000","last_unstale":"2023-10-01T10:46:19.611485+0000","last_undegraded":"2023-10-01T10:46:19.611485+0000","last_fullsized":"2023-10-01T10:46:19.611485+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:48:09.551917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":125,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:09.809882+0000","last_change":"2023-10-01T10:32:31.569162+0000","last_active":"2023-10-01T10:46:09.809882+0000","last_peered":"2023-10-01T10:46:09.809882+0000","last_clean":"2023-10-01T10:46:09.809882+0000","last_became_active":"2023-10-01T10:32:31.568956+0000","last_became_peered":"2023-10-01T10:32:31.568956+0000","last_unstale":"2023-10-01T10:46:09.809882+0000","last_undegraded":"2023-10-01T10:46:09.809882+0000","last_fullsized":"2023-10-01T10:46:09.809882+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:02:08.285304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"155'1911","reported_seq":1923,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-01T10:46:28.372446+0000","last_change":"2023-10-01T10:45:57.054040+0000","last_active":"2023-10-01T10:46:28.372446+0000","last_peered":"2023-10-01T10:46:28.372446+0000","last_clean":"2023-10-01T10:46:28.372446+0000","last_became_active":"2023-10-01T10:45:57.052762+0000","last_became_peered":"2023-10-01T10:45:57.052762+0000","last_unstale":"2023-10-01T10:46:28.372446+0000","last_undegraded":"2023-10-01T10:46:28.372446+0000","last_fullsized":"2023-10-01T10:46:28.372446+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":151,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_clean_scrub_stamp":"2023-10-01T10:45:56.023726+0000","objects_scrubbed":0,"log_size":1911,"log_dups_size":0,"ondisk_log_size":1911,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":125239296,"num_objects":1911,"num_object_clones":0,"num_object_copies":5733,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1911,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1911,"num_write_kb":122304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":126,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:09.815082+0000","last_change":"2023-10-01T10:37:05.024009+0000","last_active":"2023-10-01T10:46:09.815082+0000","last_peered":"2023-10-01T10:46:09.815082+0000","last_clean":"2023-10-01T10:46:09.815082+0000","last_became_active":"2023-10-01T10:37:05.023458+0000","last_became_peered":"2023-10-01T10:37:05.023458+0000","last_unstale":"2023-10-01T10:46:09.815082+0000","last_undegraded":"2023-10-01T10:46:09.815082+0000","last_fullsized":"2023-10-01T10:46:09.815082+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'139","reported_seq":358,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:12.165960+0000","last_change":"2023-10-01T10:28:20.217386+0000","last_active":"2023-10-01T10:46:12.165960+0000","last_peered":"2023-10-01T10:46:12.165960+0000","last_clean":"2023-10-01T10:46:12.165960+0000","last_became_active":"2023-10-01T10:28:20.216803+0000","last_became_peered":"2023-10-01T10:28:20.216803+0000","last_unstale":"2023-10-01T10:46:12.165960+0000","last_undegraded":"2023-10-01T10:46:12.165960+0000","last_fullsized":"2023-10-01T10:46:12.165960+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:09:17.221082+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:09:17.221082+0000","last_clean_scrub_stamp":"2023-10-01T10:09:17.221082+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":101,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:12.244541+0000","last_change":"2023-10-01T10:37:42.765100+0000","last_active":"2023-10-01T10:46:12.244541+0000","last_peered":"2023-10-01T10:46:12.244541+0000","last_clean":"2023-10-01T10:46:12.244541+0000","last_became_active":"2023-10-01T10:37:42.764761+0000","last_became_peered":"2023-10-01T10:37:42.764761+0000","last_unstale":"2023-10-01T10:46:12.244541+0000","last_undegraded":"2023-10-01T10:46:12.244541+0000","last_fullsized":"2023-10-01T10:46:12.244541+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:24:56.907377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":120,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:12.246514+0000","last_change":"2023-10-01T10:38:16.031255+0000","last_active":"2023-10-01T10:46:12.246514+0000","last_peered":"2023-10-01T10:46:12.246514+0000","last_clean":"2023-10-01T10:46:12.246514+0000","last_became_active":"2023-10-01T10:38:16.030886+0000","last_became_peered":"2023-10-01T10:38:16.030886+0000","last_unstale":"2023-10-01T10:46:12.246514+0000","last_undegraded":"2023-10-01T10:46:12.246514+0000","last_fullsized":"2023-10-01T10:46:12.246514+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"155'1949","reported_seq":1962,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-01T10:46:28.387840+0000","last_change":"2023-10-01T10:45:57.048663+0000","last_active":"2023-10-01T10:46:28.387840+0000","last_peered":"2023-10-01T10:46:28.387840+0000","last_clean":"2023-10-01T10:46:28.387840+0000","last_became_active":"2023-10-01T10:45:57.048378+0000","last_became_peered":"2023-10-01T10:45:57.048378+0000","last_unstale":"2023-10-01T10:46:28.387840+0000","last_undegraded":"2023-10-01T10:46:28.387840+0000","last_fullsized":"2023-10-01T10:46:28.387840+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":151,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_clean_scrub_stamp":"2023-10-01T10:45:56.023726+0000","objects_scrubbed":0,"log_size":1949,"log_dups_size":0,"ondisk_log_size":1949,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:00:59.519343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":127729664,"num_objects":1949,"num_object_clones":0,"num_object_copies":5847,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1949,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1949,"num_write_kb":124736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":141,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:12.165672+0000","last_change":"2023-10-01T10:32:31.565868+0000","last_active":"2023-10-01T10:46:12.165672+0000","last_peered":"2023-10-01T10:46:12.165672+0000","last_clean":"2023-10-01T10:46:12.165672+0000","last_became_active":"2023-10-01T10:32:31.565428+0000","last_became_peered":"2023-10-01T10:32:31.565428+0000","last_unstale":"2023-10-01T10:46:12.165672+0000","last_undegraded":"2023-10-01T10:46:12.165672+0000","last_fullsized":"2023-10-01T10:46:12.165672+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":128,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:12.244292+0000","last_change":"2023-10-01T10:38:16.021968+0000","last_active":"2023-10-01T10:46:12.244292+0000","last_peered":"2023-10-01T10:46:12.244292+0000","last_clean":"2023-10-01T10:46:12.244292+0000","last_became_active":"2023-10-01T10:38:16.021539+0000","last_became_peered":"2023-10-01T10:38:16.021539+0000","last_unstale":"2023-10-01T10:46:12.244292+0000","last_undegraded":"2023-10-01T10:46:12.244292+0000","last_fullsized":"2023-10-01T10:46:12.244292+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:12:53.105747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":129,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:12.244242+0000","last_change":"2023-10-01T10:37:05.017431+0000","last_active":"2023-10-01T10:46:12.244242+0000","last_peered":"2023-10-01T10:46:12.244242+0000","last_clean":"2023-10-01T10:46:12.244242+0000","last_became_active":"2023-10-01T10:37:05.017094+0000","last_became_peered":"2023-10-01T10:37:05.017094+0000","last_unstale":"2023-10-01T10:46:12.244242+0000","last_undegraded":"2023-10-01T10:46:12.244242+0000","last_fullsized":"2023-10-01T10:46:12.244242+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:10:40.664686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"155'1931","reported_seq":1943,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-01T10:46:28.363375+0000","last_change":"2023-10-01T10:45:57.052327+0000","last_active":"2023-10-01T10:46:28.363375+0000","last_peered":"2023-10-01T10:46:28.363375+0000","last_clean":"2023-10-01T10:46:28.363375+0000","last_became_active":"2023-10-01T10:45:57.051994+0000","last_became_peered":"2023-10-01T10:45:57.051994+0000","last_unstale":"2023-10-01T10:46:28.363375+0000","last_undegraded":"2023-10-01T10:46:28.363375+0000","last_fullsized":"2023-10-01T10:46:28.363375+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":151,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_clean_scrub_stamp":"2023-10-01T10:45:56.023726+0000","objects_scrubbed":0,"log_size":1931,"log_dups_size":0,"ondisk_log_size":1931,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":126550016,"num_objects":1931,"num_object_clones":0,"num_object_copies":5793,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1931,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1931,"num_write_kb":123584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":141,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:12.165950+0000","last_change":"2023-10-01T10:38:16.032774+0000","last_active":"2023-10-01T10:46:12.165950+0000","last_peered":"2023-10-01T10:46:12.165950+0000","last_clean":"2023-10-01T10:46:12.165950+0000","last_became_active":"2023-10-01T10:38:16.032283+0000","last_became_peered":"2023-10-01T10:38:16.032283+0000","last_unstale":"2023-10-01T10:46:12.165950+0000","last_undegraded":"2023-10-01T10:46:12.165950+0000","last_fullsized":"2023-10-01T10:46:12.165950+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":96,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:09.809223+0000","last_change":"2023-10-01T10:37:05.021228+0000","last_active":"2023-10-01T10:46:09.809223+0000","last_peered":"2023-10-01T10:46:09.809223+0000","last_clean":"2023-10-01T10:46:09.809223+0000","last_became_active":"2023-10-01T10:37:05.020888+0000","last_became_peered":"2023-10-01T10:37:05.020888+0000","last_unstale":"2023-10-01T10:46:09.809223+0000","last_undegraded":"2023-10-01T10:46:09.809223+0000","last_fullsized":"2023-10-01T10:46:09.809223+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:59:57.641247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":99,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:12.243788+0000","last_change":"2023-10-01T10:28:20.220812+0000","last_active":"2023-10-01T10:46:12.243788+0000","last_peered":"2023-10-01T10:46:12.243788+0000","last_clean":"2023-10-01T10:46:12.243788+0000","last_became_active":"2023-10-01T10:28:20.215962+0000","last_became_peered":"2023-10-01T10:28:20.215962+0000","last_unstale":"2023-10-01T10:46:12.243788+0000","last_undegraded":"2023-10-01T10:46:12.243788+0000","last_fullsized":"2023-10-01T10:46:12.243788+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:35:15.275097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"155'2065","reported_seq":2078,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-01T10:46:29.159006+0000","last_change":"2023-10-01T10:45:57.050504+0000","last_active":"2023-10-01T10:46:29.159006+0000","last_peered":"2023-10-01T10:46:29.159006+0000","last_clean":"2023-10-01T10:46:29.159006+0000","last_became_active":"2023-10-01T10:45:57.048979+0000","last_became_peered":"2023-10-01T10:45:57.048979+0000","last_unstale":"2023-10-01T10:46:29.159006+0000","last_undegraded":"2023-10-01T10:46:29.159006+0000","last_fullsized":"2023-10-01T10:46:29.159006+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":151,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_clean_scrub_stamp":"2023-10-01T10:45:56.023726+0000","objects_scrubbed":0,"log_size":2065,"log_dups_size":0,"ondisk_log_size":2065,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:54:06.967857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135331840,"num_objects":2065,"num_object_clones":0,"num_object_copies":6195,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2065,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2065,"num_write_kb":132160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":106,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-01T10:46:19.605446+0000","last_change":"2023-10-01T10:37:42.773999+0000","last_active":"2023-10-01T10:46:19.605446+0000","last_peered":"2023-10-01T10:46:19.605446+0000","last_clean":"2023-10-01T10:46:19.605446+0000","last_became_active":"2023-10-01T10:37:42.773355+0000","last_became_peered":"2023-10-01T10:37:42.773355+0000","last_unstale":"2023-10-01T10:46:19.605446+0000","last_undegraded":"2023-10-01T10:46:19.605446+0000","last_fullsized":"2023-10-01T10:46:19.605446+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:46:51.317151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":138,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:09.809466+0000","last_change":"2023-10-01T10:38:16.034193+0000","last_active":"2023-10-01T10:46:09.809466+0000","last_peered":"2023-10-01T10:46:09.809466+0000","last_clean":"2023-10-01T10:46:09.809466+0000","last_became_active":"2023-10-01T10:38:16.033703+0000","last_became_peered":"2023-10-01T10:38:16.033703+0000","last_unstale":"2023-10-01T10:46:09.809466+0000","last_undegraded":"2023-10-01T10:46:09.809466+0000","last_fullsized":"2023-10-01T10:46:09.809466+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:21:26.061595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":103,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:12.243571+0000","last_change":"2023-10-01T10:37:42.773538+0000","last_active":"2023-10-01T10:46:12.243571+0000","last_peered":"2023-10-01T10:46:12.243571+0000","last_clean":"2023-10-01T10:46:12.243571+0000","last_became_active":"2023-10-01T10:37:42.771863+0000","last_became_peered":"2023-10-01T10:37:42.771863+0000","last_unstale":"2023-10-01T10:46:12.243571+0000","last_undegraded":"2023-10-01T10:46:12.243571+0000","last_fullsized":"2023-10-01T10:46:12.243571+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:46:46.698627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"155'1928","reported_seq":1940,"reported_epoch":155,"state":"active+clean","last_fresh":"2023-10-01T10:46:28.379819+0000","last_change":"2023-10-01T10:45:57.056633+0000","last_active":"2023-10-01T10:46:28.379819+0000","last_peered":"2023-10-01T10:46:28.379819+0000","last_clean":"2023-10-01T10:46:28.379819+0000","last_became_active":"2023-10-01T10:45:57.056070+0000","last_became_peered":"2023-10-01T10:45:57.056070+0000","last_unstale":"2023-10-01T10:46:28.379819+0000","last_undegraded":"2023-10-01T10:46:28.379819+0000","last_fullsized":"2023-10-01T10:46:28.379819+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":151,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:45:56.023726+0000","last_clean_scrub_stamp":"2023-10-01T10:45:56.023726+0000","objects_scrubbed":0,"log_size":1928,"log_dups_size":0,"ondisk_log_size":1928,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":126353408,"num_objects":1928,"num_object_clones":0,"num_object_copies":5784,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1928,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1928,"num_write_kb":123392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":125,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:12.246472+0000","last_change":"2023-10-01T10:37:05.023034+0000","last_active":"2023-10-01T10:46:12.246472+0000","last_peered":"2023-10-01T10:46:12.246472+0000","last_clean":"2023-10-01T10:46:12.246472+0000","last_became_active":"2023-10-01T10:37:05.022670+0000","last_became_peered":"2023-10-01T10:37:05.022670+0000","last_unstale":"2023-10-01T10:46:12.246472+0000","last_undegraded":"2023-10-01T10:46:12.246472+0000","last_fullsized":"2023-10-01T10:46:12.246472+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":132,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:12.243998+0000","last_change":"2023-10-01T10:38:16.027472+0000","last_active":"2023-10-01T10:46:12.243998+0000","last_peered":"2023-10-01T10:46:12.243998+0000","last_clean":"2023-10-01T10:46:12.243998+0000","last_became_active":"2023-10-01T10:38:16.026810+0000","last_became_peered":"2023-10-01T10:38:16.026810+0000","last_unstale":"2023-10-01T10:46:12.243998+0000","last_undegraded":"2023-10-01T10:46:12.243998+0000","last_fullsized":"2023-10-01T10:46:12.243998+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:30:06.978187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":96,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:11.744280+0000","last_change":"2023-10-01T10:38:16.035705+0000","last_active":"2023-10-01T10:46:11.744280+0000","last_peered":"2023-10-01T10:46:11.744280+0000","last_clean":"2023-10-01T10:46:11.744280+0000","last_became_active":"2023-10-01T10:38:16.035389+0000","last_became_peered":"2023-10-01T10:38:16.035389+0000","last_unstale":"2023-10-01T10:46:11.744280+0000","last_undegraded":"2023-10-01T10:46:11.744280+0000","last_fullsized":"2023-10-01T10:46:11.744280+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:16:10.201409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":116,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:12.243896+0000","last_change":"2023-10-01T10:38:16.028491+0000","last_active":"2023-10-01T10:46:12.243896+0000","last_peered":"2023-10-01T10:46:12.243896+0000","last_clean":"2023-10-01T10:46:12.243896+0000","last_became_active":"2023-10-01T10:38:16.028175+0000","last_became_peered":"2023-10-01T10:38:16.028175+0000","last_unstale":"2023-10-01T10:46:12.243896+0000","last_undegraded":"2023-10-01T10:46:12.243896+0000","last_fullsized":"2023-10-01T10:46:12.243896+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:44:29.083129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":137,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:09.807847+0000","last_change":"2023-10-01T10:37:42.762360+0000","last_active":"2023-10-01T10:46:09.807847+0000","last_peered":"2023-10-01T10:46:09.807847+0000","last_clean":"2023-10-01T10:46:09.807847+0000","last_became_active":"2023-10-01T10:37:42.761936+0000","last_became_peered":"2023-10-01T10:37:42.761936+0000","last_unstale":"2023-10-01T10:46:09.807847+0000","last_undegraded":"2023-10-01T10:46:09.807847+0000","last_fullsized":"2023-10-01T10:46:09.807847+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:50.637582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":139,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:09.815595+0000","last_change":"2023-10-01T10:38:16.037809+0000","last_active":"2023-10-01T10:46:09.815595+0000","last_peered":"2023-10-01T10:46:09.815595+0000","last_clean":"2023-10-01T10:46:09.815595+0000","last_became_active":"2023-10-01T10:38:16.037318+0000","last_became_peered":"2023-10-01T10:38:16.037318+0000","last_unstale":"2023-10-01T10:46:09.815595+0000","last_undegraded":"2023-10-01T10:46:09.815595+0000","last_fullsized":"2023-10-01T10:46:09.815595+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":146,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:11.744726+0000","last_change":"2023-10-01T10:38:16.034282+0000","last_active":"2023-10-01T10:46:11.744726+0000","last_peered":"2023-10-01T10:46:11.744726+0000","last_clean":"2023-10-01T10:46:11.744726+0000","last_became_active":"2023-10-01T10:38:16.033820+0000","last_became_peered":"2023-10-01T10:38:16.033820+0000","last_unstale":"2023-10-01T10:46:11.744726+0000","last_undegraded":"2023-10-01T10:46:11.744726+0000","last_fullsized":"2023-10-01T10:46:11.744726+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:48:23.815832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":124,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:09.815280+0000","last_change":"2023-10-01T10:38:16.038077+0000","last_active":"2023-10-01T10:46:09.815280+0000","last_peered":"2023-10-01T10:46:09.815280+0000","last_clean":"2023-10-01T10:46:09.815280+0000","last_became_active":"2023-10-01T10:38:16.037313+0000","last_became_peered":"2023-10-01T10:38:16.037313+0000","last_unstale":"2023-10-01T10:46:09.815280+0000","last_undegraded":"2023-10-01T10:46:09.815280+0000","last_fullsized":"2023-10-01T10:46:09.815280+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":146,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:11.744718+0000","last_change":"2023-10-01T10:38:16.025600+0000","last_active":"2023-10-01T10:46:11.744718+0000","last_peered":"2023-10-01T10:46:11.744718+0000","last_clean":"2023-10-01T10:46:11.744718+0000","last_became_active":"2023-10-01T10:38:16.025252+0000","last_became_peered":"2023-10-01T10:38:16.025252+0000","last_unstale":"2023-10-01T10:46:11.744718+0000","last_undegraded":"2023-10-01T10:46:11.744718+0000","last_fullsized":"2023-10-01T10:46:11.744718+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:24:18.871963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":134,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:12.244611+0000","last_change":"2023-10-01T10:37:42.761243+0000","last_active":"2023-10-01T10:46:12.244611+0000","last_peered":"2023-10-01T10:46:12.244611+0000","last_clean":"2023-10-01T10:46:12.244611+0000","last_became_active":"2023-10-01T10:37:42.760834+0000","last_became_peered":"2023-10-01T10:37:42.760834+0000","last_unstale":"2023-10-01T10:46:12.244611+0000","last_undegraded":"2023-10-01T10:46:12.244611+0000","last_fullsized":"2023-10-01T10:46:12.244611+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:37:21.976095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":146,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:12.244509+0000","last_change":"2023-10-01T10:37:42.765142+0000","last_active":"2023-10-01T10:46:12.244509+0000","last_peered":"2023-10-01T10:46:12.244509+0000","last_clean":"2023-10-01T10:46:12.244509+0000","last_became_active":"2023-10-01T10:37:42.764770+0000","last_became_peered":"2023-10-01T10:37:42.764770+0000","last_unstale":"2023-10-01T10:46:12.244509+0000","last_undegraded":"2023-10-01T10:46:12.244509+0000","last_fullsized":"2023-10-01T10:46:12.244509+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:23:33.527876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":117,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:12.244416+0000","last_change":"2023-10-01T10:37:05.021621+0000","last_active":"2023-10-01T10:46:12.244416+0000","last_peered":"2023-10-01T10:46:12.244416+0000","last_clean":"2023-10-01T10:46:12.244416+0000","last_became_active":"2023-10-01T10:37:05.021173+0000","last_became_peered":"2023-10-01T10:37:05.021173+0000","last_unstale":"2023-10-01T10:46:12.244416+0000","last_undegraded":"2023-10-01T10:46:12.244416+0000","last_fullsized":"2023-10-01T10:46:12.244416+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:13:25.364499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":112,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-10-01T10:46:11.744189+0000","last_change":"2023-10-01T10:38:16.031427+0000","last_active":"2023-10-01T10:46:11.744189+0000","last_peered":"2023-10-01T10:46:11.744189+0000","last_clean":"2023-10-01T10:46:11.744189+0000","last_became_active":"2023-10-01T10:38:16.029786+0000","last_became_peered":"2023-10-01T10:38:16.029786+0000","last_unstale":"2023-10-01T10:46:11.744189+0000","last_undegraded":"2023-10-01T10:46:11.744189+0000","last_fullsized":"2023-10-01T10:46:11.744189+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:53:22.796191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":2064121856,"num_objects":31496,"num_object_clones":0,"num_object_copies":94488,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":31496,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":31496,"num_write_kb":2015744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6301876224,"data_stored":6301876224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":31496,"ondisk_log_size":31496,"up":48,"acting":48,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30319,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430614,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1388720,"kb_used_data":498472,"kb_used_omap":0,"kb_used_meta":890240,"kb_avail":92356432,"statfs":{"total":95995035648,"available":94572986368,"internally_reserved":0,"allocated":510435328,"data_stored":509942495,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":911605760},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:45:59 2023","interfaces":[{"interface":"back","average":{"1min":0.78700000000000003,"5min":2.484,"15min":1.6899999999999999},"min":{"1min":0.38800000000000001,"5min":0.28499999999999998,"15min":0.27500000000000002},"max":{"1min":1.1919999999999999,"5min":51.07,"15min":51.07},"last":5.516},{"interface":"front","average":{"1min":1.4710000000000001,"5min":1.982,"15min":1.5740000000000001},"min":{"1min":0.63400000000000001,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":7.3209999999999997,"5min":51.024000000000001,"15min":51.024000000000001},"last":5.4950000000000001}]},{"osd":1,"last update":"Sun Oct 1 10:46:05 2023","interfaces":[{"interface":"back","average":{"1min":1.157,"5min":2.8149999999999999,"15min":1.7609999999999999},"min":{"1min":0.58399999999999996,"5min":0.311,"15min":0.309},"max":{"1min":5.0510000000000002,"5min":56.110999999999997,"15min":56.110999999999997},"last":0.48699999999999999},{"interface":"front","average":{"1min":1.4059999999999999,"5min":2.8929999999999998,"15min":1.903},"min":{"1min":0.59599999999999997,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":6.6459999999999999,"5min":51.258000000000003,"15min":51.258000000000003},"last":0.33100000000000002}]},{"osd":2,"last update":"Sun Oct 1 10:46:20 2023","interfaces":[{"interface":"back","average":{"1min":1.028,"5min":1.3380000000000001,"15min":1.119},"min":{"1min":0.73299999999999998,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":1.389,"5min":6.5979999999999999,"15min":6.5979999999999999},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.82099999999999995,"5min":1.8140000000000001,"15min":1.228},"min":{"1min":0.58899999999999997,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":1.1399999999999999,"5min":10.766,"15min":10.766},"last":0.35099999999999998}]},{"osd":3,"last update":"Sun Oct 1 10:45:59 2023","interfaces":[{"interface":"back","average":{"1min":1.7490000000000001,"5min":2.7229999999999999,"15min":1.5589999999999999},"min":{"1min":0.76000000000000001,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":6.7709999999999999,"5min":51.334000000000003,"15min":51.334000000000003},"last":0.40300000000000002},{"interface":"front","average":{"1min":1.2909999999999999,"5min":3.145,"15min":1.681},"min":{"1min":0.38600000000000001,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":8.6110000000000007,"5min":51.404000000000003,"15min":51.404000000000003},"last":0.61299999999999999}]},{"osd":4,"last update":"Sun Oct 1 10:45:30 2023","interfaces":[{"interface":"back","average":{"1min":1.468,"5min":1.4019999999999999,"15min":1.087},"min":{"1min":0.53600000000000003,"5min":0.317,"15min":0.317},"max":{"1min":5.085,"5min":8.6340000000000003,"15min":8.6340000000000003},"last":0.53500000000000003},{"interface":"front","average":{"1min":1.661,"5min":1.353,"15min":1.2410000000000001},"min":{"1min":0.41599999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":5.6699999999999999,"5min":7.54,"15min":7.54},"last":0.54500000000000004}]},{"osd":5,"last update":"Sun Oct 1 10:46:05 2023","interfaces":[{"interface":"back","average":{"1min":1.875,"5min":2.2389999999999999,"15min":1.593},"min":{"1min":0.73999999999999999,"5min":0.36699999999999999,"15min":0.35399999999999998},"max":{"1min":12.555,"5min":12.555,"15min":12.555},"last":0.59199999999999997},{"interface":"front","average":{"1min":1.0149999999999999,"5min":2.6600000000000001,"15min":1.665},"min":{"1min":0.67500000000000004,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":1.427,"5min":17.536999999999999,"15min":17.536999999999999},"last":0.58299999999999996}]},{"osd":6,"last update":"Sun Oct 1 10:46:05 2023","interfaces":[{"interface":"back","average":{"1min":1.5409999999999999,"5min":1.518,"15min":1.4099999999999999},"min":{"1min":0.873,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":5.024,"5min":6.5789999999999997,"15min":7.3300000000000001},"last":0.51700000000000002},{"interface":"front","average":{"1min":1.3120000000000001,"5min":1.347,"15min":1.3109999999999999},"min":{"1min":0.79000000000000004,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":5.1360000000000001,"5min":5.734,"15min":6.984},"last":0.55900000000000005}]}]},{"osd":6,"up_from":42,"seq":180388626848,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1447072,"kb_used_data":857368,"kb_used_omap":0,"kb_used_meta":589696,"kb_avail":92298080,"statfs":{"total":95995035648,"available":94513233920,"internally_reserved":0,"allocated":877944832,"data_stored":877467922,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":603848704},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,3,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:45:51 2023","interfaces":[{"interface":"back","average":{"1min":0.85699999999999998,"5min":2.4740000000000002,"15min":1.764},"min":{"1min":0.30299999999999999,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":3.3420000000000001,"5min":140.524,"15min":140.524},"last":0.61499999999999999},{"interface":"front","average":{"1min":0.96199999999999997,"5min":2.3540000000000001,"15min":1.714},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.28899999999999998},"max":{"1min":3.0859999999999999,"5min":140.44,"15min":140.44},"last":1.661}]},{"osd":1,"last update":"Sun Oct 1 10:45:55 2023","interfaces":[{"interface":"back","average":{"1min":1.123,"5min":1.335,"15min":1.2909999999999999},"min":{"1min":0.316,"5min":0.29999999999999999,"15min":0.26600000000000001},"max":{"1min":9.673,"5min":9.673,"15min":9.673},"last":0.53200000000000003},{"interface":"front","average":{"1min":1.02,"5min":1.329,"15min":1.361},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":6.2770000000000001,"5min":6.3710000000000004,"15min":7.4199999999999999},"last":1.623}]},{"osd":2,"last update":"Sun Oct 1 10:46:18 2023","interfaces":[{"interface":"back","average":{"1min":1.175,"5min":1.23,"15min":0.996},"min":{"1min":0.61299999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":6.4000000000000004,"5min":6.4000000000000004,"15min":6.4000000000000004},"last":0.82799999999999996},{"interface":"front","average":{"1min":1.2030000000000001,"5min":1.2689999999999999,"15min":1.056},"min":{"1min":0.68000000000000005,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":6.4829999999999997,"5min":7.9619999999999997,"15min":7.9619999999999997},"last":0.873}]},{"osd":3,"last update":"Sun Oct 1 10:46:03 2023","interfaces":[{"interface":"back","average":{"1min":1.042,"5min":2.7970000000000002,"15min":1.589},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":4.0350000000000001,"5min":61.817999999999998,"15min":61.817999999999998},"last":0.50900000000000001},{"interface":"front","average":{"1min":1.5369999999999999,"5min":2.4390000000000001,"15min":1.4930000000000001},"min":{"1min":0.61199999999999999,"5min":0.315,"15min":0.315},"max":{"1min":8.0229999999999997,"5min":61.219000000000001,"15min":61.219000000000001},"last":0.91600000000000004}]},{"osd":4,"last update":"Sun Oct 1 10:45:34 2023","interfaces":[{"interface":"back","average":{"1min":1.1659999999999999,"5min":0.92400000000000004,"15min":0.98299999999999998},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":4.5570000000000004,"5min":4.5570000000000004,"15min":4.5570000000000004},"last":1.679},{"interface":"front","average":{"1min":1.1830000000000001,"5min":1.113,"15min":1.101},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":3.7799999999999998,"5min":7.1589999999999998,"15min":7.1589999999999998},"last":1.024}]},{"osd":5,"last update":"Sun Oct 1 10:46:03 2023","interfaces":[{"interface":"back","average":{"1min":1.0669999999999999,"5min":1.4239999999999999,"15min":1.3080000000000001},"min":{"1min":0.47999999999999998,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":3.9119999999999999,"5min":12.398999999999999,"15min":12.398999999999999},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.94999999999999996,"5min":1.601,"15min":1.3720000000000001},"min":{"1min":0.45100000000000001,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":1.341,"5min":14.459,"15min":14.459},"last":2.3540000000000001}]},{"osd":7,"last update":"Sun Oct 1 10:45:51 2023","interfaces":[{"interface":"back","average":{"1min":1.018,"5min":1.2070000000000001,"15min":1.129},"min":{"1min":0.42299999999999999,"5min":0.371,"15min":0.371},"max":{"1min":3.1349999999999998,"5min":8.8360000000000003,"15min":8.8360000000000003},"last":1.589},{"interface":"front","average":{"1min":0.98599999999999999,"5min":1.504,"15min":1.3200000000000001},"min":{"1min":0.40899999999999997,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":3.649,"5min":8.1720000000000006,"15min":8.1720000000000006},"last":0.48299999999999998}]}]},{"osd":1,"up_from":101,"seq":433791697066,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":776164,"kb_used_data":417500,"kb_used_omap":4,"kb_used_meta":358651,"kb_avail":92968988,"statfs":{"total":95995035648,"available":95200243712,"internally_reserved":0,"allocated":427520000,"data_stored":426869296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":367259534},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:46:03 2023","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":1.1240000000000001,"15min":1.4690000000000001},"min":{"1min":0.224,"5min":0.189,"15min":0.189},"max":{"1min":1.1040000000000001,"5min":8.3520000000000003,"15min":11.071999999999999},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.70899999999999996,"5min":2.0760000000000001,"15min":1.758},"min":{"1min":0.30399999999999999,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":2.137,"5min":61.207999999999998,"15min":61.207999999999998},"last":0.496}]},{"osd":2,"last update":"Sun Oct 1 10:46:23 2023","interfaces":[{"interface":"back","average":{"1min":0.755,"5min":1.8069999999999999,"15min":1.075},"min":{"1min":0.41099999999999998,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":1.0569999999999999,"5min":61.491,"15min":61.491},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.749,"5min":1.982,"15min":1.1859999999999999},"min":{"1min":0.47799999999999998,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":1.0549999999999999,"5min":61.371000000000002,"15min":61.371000000000002},"last":0.437}]},{"osd":3,"last update":"Sun Oct 1 10:46:03 2023","interfaces":[{"interface":"back","average":{"1min":0.64700000000000002,"5min":2.3929999999999998,"15min":1.256},"min":{"1min":0.29299999999999998,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":0.82399999999999995,"5min":62.020000000000003,"15min":62.020000000000003},"last":0.252},{"interface":"front","average":{"1min":0.88800000000000001,"5min":2.6280000000000001,"15min":1.3740000000000001},"min":{"1min":0.502,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":2.3039999999999998,"5min":62.338999999999999,"15min":62.338999999999999},"last":0.67200000000000004}]},{"osd":4,"last update":"Sun Oct 1 10:46:23 2023","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":1.819,"15min":1.2090000000000001},"min":{"1min":0.442,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":1.546,"5min":61.588000000000001,"15min":61.588000000000001},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.94399999999999995,"5min":2.1909999999999998,"15min":1.415},"min":{"1min":0.48699999999999999,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":1.2110000000000001,"5min":61.411000000000001,"15min":61.411000000000001},"last":2.089}]},{"osd":5,"last update":"Sun Oct 1 10:46:03 2023","interfaces":[{"interface":"back","average":{"1min":1.1970000000000001,"5min":2.4289999999999998,"15min":1.8300000000000001},"min":{"1min":0.81899999999999995,"5min":0.38900000000000001,"15min":0.35799999999999998},"max":{"1min":3.1869999999999998,"5min":61.560000000000002,"15min":61.560000000000002},"last":0.77200000000000002},{"interface":"front","average":{"1min":1.0129999999999999,"5min":2.0379999999999998,"15min":1.972},"min":{"1min":0.53900000000000003,"5min":0.34699999999999998,"15min":0.33600000000000002},"max":{"1min":1.456,"5min":61.667999999999999,"15min":61.667999999999999},"last":1.502}]},{"osd":6,"last update":"Sun Oct 1 10:46:03 2023","interfaces":[{"interface":"back","average":{"1min":1.036,"5min":1.873,"15min":1.746},"min":{"1min":0.433,"5min":0.39100000000000001,"15min":0.37},"max":{"1min":1.5349999999999999,"5min":62.219999999999999,"15min":62.219999999999999},"last":1.012},{"interface":"front","average":{"1min":1.3089999999999999,"5min":2.1850000000000001,"15min":1.8440000000000001},"min":{"1min":0.52500000000000002,"5min":0.439,"15min":0.41999999999999998},"max":{"1min":4.4660000000000002,"5min":61.462000000000003,"15min":61.462000000000003},"last":0.92100000000000004}]},{"osd":7,"last update":"Sun Oct 1 10:46:03 2023","interfaces":[{"interface":"back","average":{"1min":1.1539999999999999,"5min":2.6909999999999998,"15min":2.0019999999999998},"min":{"1min":0.83299999999999996,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":1.7470000000000001,"5min":62.646999999999998,"15min":62.646999999999998},"last":0.56000000000000005},{"interface":"front","average":{"1min":1.2190000000000001,"5min":2.3780000000000001,"15min":1.847},"min":{"1min":0.94599999999999995,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":2.5009999999999999,"5min":62.048000000000002,"15min":62.048000000000002},"last":0.73599999999999999}]}]},{"osd":0,"up_from":86,"seq":369367187676,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1675032,"kb_used_data":1230352,"kb_used_omap":5,"kb_used_meta":444666,"kb_avail":92070120,"statfs":{"total":95995035648,"available":94279802880,"internally_reserved":0,"allocated":1259880448,"data_stored":1259238878,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":455338618},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 10:45:58 2023","interfaces":[{"interface":"back","average":{"1min":0.69999999999999996,"5min":1.6599999999999999,"15min":1.5529999999999999},"min":{"1min":0.20699999999999999,"5min":0.184,"15min":0.184},"max":{"1min":1.718,"5min":15.074,"15min":15.074},"last":0.35999999999999999},{"interface":"front","average":{"1min":1.492,"5min":2.008,"15min":1.835},"min":{"1min":0.34699999999999998,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":13.601000000000001,"5min":13.601000000000001,"15min":15.48},"last":0.49399999999999999}]},{"osd":2,"last update":"Sun Oct 1 10:46:25 2023","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":1.0329999999999999,"15min":0.80200000000000005},"min":{"1min":0.311,"5min":0.223,"15min":0.223},"max":{"1min":1.3799999999999999,"5min":9.2420000000000009,"15min":9.2420000000000009},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.83999999999999997,"5min":0.94499999999999995,"15min":0.81000000000000005},"min":{"1min":0.376,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":1.5169999999999999,"5min":4.274,"15min":4.274},"last":1.5169999999999999}]},{"osd":3,"last update":"Sun Oct 1 10:45:58 2023","interfaces":[{"interface":"back","average":{"1min":0.80000000000000004,"5min":1.704,"15min":1.101},"min":{"1min":0.47299999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":2.2210000000000001,"5min":9.4789999999999992,"15min":9.4789999999999992},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.90100000000000002,"5min":1.5309999999999999,"15min":1.04},"min":{"1min":0.25700000000000001,"5min":0.245,"15min":0.245},"max":{"1min":3.754,"5min":10.984999999999999,"15min":10.984999999999999},"last":0.39300000000000002}]},{"osd":4,"last update":"Sun Oct 1 10:45:36 2023","interfaces":[{"interface":"back","average":{"1min":1.1839999999999999,"5min":1.2210000000000001,"15min":1.085},"min":{"1min":0.45100000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":3.4529999999999998,"5min":9.0980000000000008,"15min":9.0980000000000008},"last":1.915},{"interface":"front","average":{"1min":1.486,"5min":1.274,"15min":1.1140000000000001},"min":{"1min":0.45800000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":8.1479999999999997,"5min":8.1479999999999997,"15min":8.1479999999999997},"last":1.292}]},{"osd":5,"last update":"Sun Oct 1 10:45:58 2023","interfaces":[{"interface":"back","average":{"1min":1.022,"5min":1.5700000000000001,"15min":1.5780000000000001},"min":{"1min":0.56399999999999995,"5min":0.35799999999999998,"15min":0.31900000000000001},"max":{"1min":2.6000000000000001,"5min":10.672000000000001,"15min":13.068},"last":0.66600000000000004},{"interface":"front","average":{"1min":1.1799999999999999,"5min":1.5389999999999999,"15min":1.5680000000000001},"min":{"1min":0.35099999999999998,"5min":0.34699999999999998,"15min":0.34200000000000003},"max":{"1min":3.504,"5min":9.1440000000000001,"15min":11.747},"last":0.63600000000000001}]},{"osd":6,"last update":"Sun Oct 1 10:45:58 2023","interfaces":[{"interface":"back","average":{"1min":1.119,"5min":1.242,"15min":1.536},"min":{"1min":0.46100000000000002,"5min":0.36399999999999999,"15min":0.35899999999999999},"max":{"1min":3.4199999999999999,"5min":9.1780000000000008,"15min":11.663},"last":2.1429999999999998},{"interface":"front","average":{"1min":1.5449999999999999,"5min":1.278,"15min":1.5349999999999999},"min":{"1min":0.62,"5min":0.374,"15min":0.374},"max":{"1min":7.306,"5min":8.2569999999999997,"15min":15.069000000000001},"last":0.67500000000000004}]},{"osd":7,"last update":"Sun Oct 1 10:45:58 2023","interfaces":[{"interface":"back","average":{"1min":1.587,"5min":1.7010000000000001,"15min":1.6180000000000001},"min":{"1min":0.75900000000000001,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":6.819,"5min":9.2059999999999995,"15min":12.686},"last":0.68700000000000006},{"interface":"front","average":{"1min":1.915,"5min":1.768,"15min":1.629},"min":{"1min":0.65600000000000003,"5min":0.434,"15min":0.36199999999999999},"max":{"1min":8.0099999999999998,"5min":8.3100000000000005,"15min":13.654},"last":0.77900000000000003}]}]},{"osd":2,"up_from":116,"seq":498216206451,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":932328,"kb_used_data":719584,"kb_used_omap":7,"kb_used_meta":212728,"kb_avail":92812824,"statfs":{"total":95995035648,"available":95040331776,"internally_reserved":0,"allocated":736854016,"data_stored":736253879,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":217833553},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:46:16 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.98999999999999999,"15min":0.76900000000000002},"min":{"1min":0.36799999999999999,"5min":0.215,"15min":0.215},"max":{"1min":1.1519999999999999,"5min":14.851000000000001,"15min":14.851000000000001},"last":2.8399999999999999},{"interface":"front","average":{"1min":0.68300000000000005,"5min":1.0329999999999999,"15min":0.85499999999999998},"min":{"1min":0.33300000000000002,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":1.4019999999999999,"5min":6.1829999999999998,"15min":6.1829999999999998},"last":2.8730000000000002}]},{"osd":1,"last update":"Sun Oct 1 10:46:16 2023","interfaces":[{"interface":"back","average":{"1min":0.90100000000000002,"5min":1.431,"15min":0.95799999999999996},"min":{"1min":0.377,"5min":0.219,"15min":0.219},"max":{"1min":4.0640000000000001,"5min":10.565,"15min":10.565},"last":3.7989999999999999},{"interface":"front","average":{"1min":0.93600000000000005,"5min":1.242,"15min":0.92000000000000004},"min":{"1min":0.34799999999999998,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":3.8889999999999998,"5min":8.9209999999999994,"15min":8.9209999999999994},"last":3.1219999999999999}]},{"osd":3,"last update":"Sun Oct 1 10:45:53 2023","interfaces":[{"interface":"back","average":{"1min":1.125,"5min":1.744,"15min":1.0720000000000001},"min":{"1min":0.29499999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":3.8340000000000001,"5min":64.266000000000005,"15min":64.266000000000005},"last":2.964},{"interface":"front","average":{"1min":1.4570000000000001,"5min":1.77,"15min":1.175},"min":{"1min":0.47699999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":6.0339999999999998,"5min":67.620999999999995,"15min":67.620999999999995},"last":4.9039999999999999}]},{"osd":4,"last update":"Sun Oct 1 10:45:32 2023","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":1.0529999999999999,"15min":1.014},"min":{"1min":0.40300000000000002,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":3.6269999999999998,"5min":7.9580000000000002,"15min":7.9580000000000002},"last":3.7749999999999999},{"interface":"front","average":{"1min":0.97399999999999998,"5min":0.89800000000000002,"15min":0.94399999999999995},"min":{"1min":0.40100000000000002,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":3.181,"5min":4.3559999999999999,"15min":4.3559999999999999},"last":3.2010000000000001}]},{"osd":5,"last update":"Sun Oct 1 10:46:16 2023","interfaces":[{"interface":"back","average":{"1min":1.0409999999999999,"5min":1.3959999999999999,"15min":1.198},"min":{"1min":0.495,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.573,"5min":5.9960000000000004,"15min":5.9960000000000004},"last":3.7149999999999999},{"interface":"front","average":{"1min":1.0740000000000001,"5min":1.3049999999999999,"15min":1.1140000000000001},"min":{"1min":0.46400000000000002,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":4.1310000000000002,"5min":6.0640000000000001,"15min":6.0640000000000001},"last":3.1659999999999999}]},{"osd":6,"last update":"Sun Oct 1 10:46:16 2023","interfaces":[{"interface":"back","average":{"1min":1.0880000000000001,"5min":0.98199999999999998,"15min":1.044},"min":{"1min":0.433,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":4.2220000000000004,"5min":6.2240000000000002,"15min":6.2240000000000002},"last":3.3149999999999999},{"interface":"front","average":{"1min":1.2789999999999999,"5min":0.94999999999999996,"15min":1.0620000000000001},"min":{"1min":0.55600000000000005,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":6.0599999999999996,"5min":6.0599999999999996,"15min":6.0599999999999996},"last":2.0190000000000001}]},{"osd":7,"last update":"Sun Oct 1 10:46:16 2023","interfaces":[{"interface":"back","average":{"1min":0.95599999999999996,"5min":1.109,"15min":1.1120000000000001},"min":{"1min":0.44700000000000001,"5min":0.373,"15min":0.373},"max":{"1min":1.361,"5min":6.7869999999999999,"15min":6.7869999999999999},"last":2.1869999999999998},{"interface":"front","average":{"1min":1.105,"5min":1.6459999999999999,"15min":1.347},"min":{"1min":0.58399999999999996,"5min":0.376,"15min":0.376},"max":{"1min":2.2629999999999999,"5min":8.9269999999999996,"15min":8.9269999999999996},"last":3.7480000000000002}]}]},{"osd":3,"up_from":121,"seq":519691042924,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":904408,"kb_used_data":694224,"kb_used_omap":2,"kb_used_meta":210173,"kb_avail":92840744,"statfs":{"total":95995035648,"available":95068921856,"internally_reserved":0,"allocated":710885376,"data_stored":710299424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":215217205},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:45:55 2023","interfaces":[{"interface":"back","average":{"1min":0.97199999999999998,"5min":2.5710000000000002,"15min":1.3009999999999999},"min":{"1min":0.30499999999999999,"5min":0.187,"15min":0.187},"max":{"1min":5.8499999999999996,"5min":77.981999999999999,"15min":77.981999999999999},"last":1.2070000000000001},{"interface":"front","average":{"1min":1.304,"5min":2.8159999999999998,"15min":1.337},"min":{"1min":0.375,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":5.6120000000000001,"5min":77.281000000000006,"15min":77.281000000000006},"last":1.304}]},{"osd":1,"last update":"Sun Oct 1 10:45:55 2023","interfaces":[{"interface":"back","average":{"1min":1.1299999999999999,"5min":2.8090000000000002,"15min":1.401},"min":{"1min":0.34899999999999998,"5min":0.222,"15min":0.222},"max":{"1min":6.1159999999999997,"5min":77.361000000000004,"15min":77.361000000000004},"last":0.95299999999999996},{"interface":"front","average":{"1min":1.0820000000000001,"5min":3.0529999999999999,"15min":1.536},"min":{"1min":0.40400000000000003,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":6.0970000000000004,"5min":77.867000000000004,"15min":77.867000000000004},"last":1.028}]},{"osd":2,"last update":"Sun Oct 1 10:45:55 2023","interfaces":[{"interface":"back","average":{"1min":1.2649999999999999,"5min":1.3220000000000001,"15min":0.89700000000000002},"min":{"1min":0.41399999999999998,"5min":0.222,"15min":0.222},"max":{"1min":10.028,"5min":10.028,"15min":10.028},"last":4.1390000000000002},{"interface":"front","average":{"1min":0.84799999999999998,"5min":1.9359999999999999,"15min":1.1739999999999999},"min":{"1min":0.39000000000000001,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":2.6429999999999998,"5min":75.299999999999997,"15min":75.299999999999997},"last":5.5129999999999999}]},{"osd":4,"last update":"Sun Oct 1 10:45:25 2023","interfaces":[{"interface":"back","average":{"1min":1.413,"5min":1.968,"15min":1.2889999999999999},"min":{"1min":0.373,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":6.1390000000000002,"5min":75.388999999999996,"15min":75.388999999999996},"last":0.97099999999999997},{"interface":"front","average":{"1min":1.556,"5min":2.1699999999999999,"15min":1.3500000000000001},"min":{"1min":0.40799999999999997,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":4.7699999999999996,"5min":75.165999999999997,"15min":75.165999999999997},"last":1.7889999999999999}]},{"osd":5,"last update":"Sun Oct 1 10:45:55 2023","interfaces":[{"interface":"back","average":{"1min":1.6890000000000001,"5min":2.722,"15min":1.5409999999999999},"min":{"1min":0.51600000000000001,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":9.5540000000000003,"5min":75.221000000000004,"15min":75.221000000000004},"last":1.266},{"interface":"front","average":{"1min":1.5089999999999999,"5min":2.9510000000000001,"15min":1.6539999999999999},"min":{"1min":0.64000000000000001,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":10.43,"5min":75.231999999999999,"15min":75.231999999999999},"last":0.505}]},{"osd":6,"last update":"Sun Oct 1 10:45:55 2023","interfaces":[{"interface":"back","average":{"1min":1.49,"5min":2.403,"15min":1.48},"min":{"1min":0.44900000000000001,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":4.8090000000000002,"5min":75.257000000000005,"15min":75.257000000000005},"last":1.4530000000000001},{"interface":"front","average":{"1min":1.677,"5min":2.488,"15min":1.5629999999999999},"min":{"1min":0.72899999999999998,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":10.066000000000001,"5min":75.194000000000003,"15min":75.194000000000003},"last":0.53600000000000003}]},{"osd":7,"last update":"Sun Oct 1 10:45:55 2023","interfaces":[{"interface":"back","average":{"1min":1.579,"5min":2.73,"15min":1.609},"min":{"1min":0.68400000000000005,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":11.782999999999999,"5min":75.138999999999996,"15min":75.138999999999996},"last":0.432},{"interface":"front","average":{"1min":1.3029999999999999,"5min":2.4199999999999999,"15min":1.494},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":6.069,"5min":75.355000000000004,"15min":75.355000000000004},"last":0.98499999999999999}]}]},{"osd":4,"up_from":126,"seq":541165879397,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1213280,"kb_used_data":999640,"kb_used_omap":10,"kb_used_meta":213621,"kb_avail":92531872,"statfs":{"total":95995035648,"available":94752636928,"internally_reserved":0,"allocated":1023631360,"data_stored":1023038193,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":218748516},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:46:27 2023","interfaces":[{"interface":"back","average":{"1min":0.78400000000000003,"5min":1.2450000000000001,"15min":1.0149999999999999},"min":{"1min":0.33100000000000002,"5min":0.251,"15min":0.251},"max":{"1min":2.0960000000000001,"5min":8.1329999999999991,"15min":8.1329999999999991},"last":0.76000000000000001},{"interface":"front","average":{"1min":1.2090000000000001,"5min":1.2110000000000001,"15min":1.0509999999999999},"min":{"1min":0.441,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":3.5830000000000002,"5min":6.7789999999999999,"15min":6.7789999999999999},"last":1.5229999999999999}]},{"osd":1,"last update":"Sun Oct 1 10:46:27 2023","interfaces":[{"interface":"back","average":{"1min":1.167,"5min":2.4489999999999998,"15min":1.4390000000000001},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":5.6980000000000004,"5min":97.622,"15min":97.622},"last":5.6980000000000004},{"interface":"front","average":{"1min":0.89200000000000002,"5min":2.4820000000000002,"15min":1.47},"min":{"1min":0.38200000000000001,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":2.0350000000000001,"5min":97.346999999999994,"15min":97.346999999999994},"last":0.82199999999999995}]},{"osd":2,"last update":"Sun Oct 1 10:46:27 2023","interfaces":[{"interface":"back","average":{"1min":1.246,"5min":0.91200000000000003,"15min":0.93500000000000005},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":6.806,"5min":6.806,"15min":6.806},"last":6.806},{"interface":"front","average":{"1min":1.1080000000000001,"5min":1.024,"15min":0.98799999999999999},"min":{"1min":0.47799999999999998,"5min":0.308,"15min":0.308},"max":{"1min":2.6299999999999999,"5min":6.8339999999999996,"15min":6.8339999999999996},"last":1.599}]},{"osd":3,"last update":"Sun Oct 1 10:46:27 2023","interfaces":[{"interface":"back","average":{"1min":0.98999999999999999,"5min":3.0600000000000001,"15min":1.7330000000000001},"min":{"1min":0.50700000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":1.7709999999999999,"5min":61.398000000000003,"15min":61.398000000000003},"last":1.4810000000000001},{"interface":"front","average":{"1min":1.0580000000000001,"5min":2.8740000000000001,"15min":1.599},"min":{"1min":0.40000000000000002,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":2.512,"5min":61.463000000000001,"15min":61.463000000000001},"last":1.395}]},{"osd":5,"last update":"Sun Oct 1 10:46:27 2023","interfaces":[{"interface":"back","average":{"1min":1.0349999999999999,"5min":1.2330000000000001,"15min":1.149},"min":{"1min":0.38200000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":1.8460000000000001,"5min":6.9340000000000002,"15min":6.9340000000000002},"last":1.8460000000000001},{"interface":"front","average":{"1min":1.159,"5min":1.4330000000000001,"15min":1.2},"min":{"1min":0.35699999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":4.2039999999999997,"5min":16.779,"15min":16.779},"last":1.2310000000000001}]},{"osd":6,"last update":"Sun Oct 1 10:46:27 2023","interfaces":[{"interface":"back","average":{"1min":1.641,"5min":1.149,"15min":1.127},"min":{"1min":0.53000000000000003,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":13.646000000000001,"5min":13.646000000000001,"15min":13.646000000000001},"last":1.7350000000000001},{"interface":"front","average":{"1min":1.8089999999999999,"5min":1.335,"15min":1.155},"min":{"1min":0.48399999999999999,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":14.446999999999999,"5min":14.446999999999999,"15min":14.446999999999999},"last":1.4279999999999999}]},{"osd":7,"last update":"Sun Oct 1 10:46:27 2023","interfaces":[{"interface":"back","average":{"1min":1.175,"5min":1.1970000000000001,"15min":1.175},"min":{"1min":0.45900000000000002,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":2.4710000000000001,"5min":6.8360000000000003,"15min":6.8360000000000003},"last":1.351},{"interface":"front","average":{"1min":1.0580000000000001,"5min":1.885,"15min":1.4079999999999999},"min":{"1min":0.44500000000000001,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":2.0609999999999999,"5min":7.726,"15min":7.726},"last":0.79300000000000004}]}]},{"osd":5,"up_from":36,"seq":154618823078,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1530068,"kb_used_data":633996,"kb_used_omap":0,"kb_used_meta":896064,"kb_avail":92215084,"statfs":{"total":95995035648,"available":94428246016,"internally_reserved":0,"allocated":649211904,"data_stored":648742558,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":917569536},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:45:48 2023","interfaces":[{"interface":"back","average":{"1min":1.1579999999999999,"5min":1.5940000000000001,"15min":1.409},"min":{"1min":0.35799999999999998,"5min":0.28299999999999997,"15min":0.27300000000000002},"max":{"1min":5.8049999999999997,"5min":52.094999999999999,"15min":52.094999999999999},"last":2.8260000000000001},{"interface":"front","average":{"1min":0.93999999999999995,"5min":1.9179999999999999,"15min":1.631},"min":{"1min":0.314,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":3.7490000000000001,"5min":52.048000000000002,"15min":52.048000000000002},"last":1.9570000000000001}]},{"osd":1,"last update":"Sun Oct 1 10:45:48 2023","interfaces":[{"interface":"back","average":{"1min":1.5,"5min":4.1900000000000004,"15min":2.4489999999999998},"min":{"1min":0.42699999999999999,"5min":0.29799999999999999,"15min":0.29099999999999998},"max":{"1min":6.0049999999999999,"5min":202.96600000000001,"15min":202.96600000000001},"last":1.611},{"interface":"front","average":{"1min":1.6619999999999999,"5min":4.9379999999999997,"15min":2.7930000000000001},"min":{"1min":0.503,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":7.2859999999999996,"5min":203.05099999999999,"15min":203.05099999999999},"last":2.0070000000000001}]},{"osd":2,"last update":"Sun Oct 1 10:46:28 2023","interfaces":[{"interface":"back","average":{"1min":1.056,"5min":1.0820000000000001,"15min":1.01},"min":{"1min":0.54200000000000004,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":2.7559999999999998,"5min":8.7110000000000003,"15min":8.7110000000000003},"last":1.534},{"interface":"front","average":{"1min":1.103,"5min":1.677,"15min":1.2090000000000001},"min":{"1min":0.57299999999999995,"5min":0.317,"15min":0.317},"max":{"1min":4.4290000000000003,"5min":11.266,"15min":11.266},"last":1.9790000000000001}]},{"osd":3,"last update":"Sun Oct 1 10:46:02 2023","interfaces":[{"interface":"back","average":{"1min":0.96299999999999997,"5min":4.2610000000000001,"15min":2.0470000000000002},"min":{"1min":0.46500000000000002,"5min":0.316,"15min":0.316},"max":{"1min":1.798,"5min":70.506,"15min":70.506},"last":2.0430000000000001},{"interface":"front","average":{"1min":0.81699999999999995,"5min":3.7869999999999999,"15min":1.899},"min":{"1min":0.43099999999999999,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":1.2869999999999999,"5min":70.346999999999994,"15min":70.346999999999994},"last":1.1970000000000001}]},{"osd":4,"last update":"Sun Oct 1 10:46:28 2023","interfaces":[{"interface":"back","average":{"1min":0.997,"5min":1.4390000000000001,"15min":1.204},"min":{"1min":0.51900000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":2.6059999999999999,"5min":15.170999999999999,"15min":15.170999999999999},"last":0.96599999999999997},{"interface":"front","average":{"1min":1.175,"5min":1.083,"15min":1.097},"min":{"1min":0.41399999999999998,"5min":0.376,"15min":0.376},"max":{"1min":2.7759999999999998,"5min":8.6329999999999991,"15min":8.6329999999999991},"last":1.5720000000000001}]},{"osd":6,"last update":"Sun Oct 1 10:46:02 2023","interfaces":[{"interface":"back","average":{"1min":1.2290000000000001,"5min":1.157,"15min":1.2310000000000001},"min":{"1min":0.42599999999999999,"5min":0.39200000000000002,"15min":0.38500000000000001},"max":{"1min":5.6219999999999999,"5min":8.7420000000000009,"15min":8.7420000000000009},"last":1.6419999999999999},{"interface":"front","average":{"1min":0.94399999999999995,"5min":1.508,"15min":1.335},"min":{"1min":0.57899999999999996,"5min":0.38300000000000001,"15min":0.312},"max":{"1min":1.528,"5min":7.8010000000000002,"15min":8.4339999999999993},"last":3.3620000000000001}]},{"osd":7,"last update":"Sun Oct 1 10:46:02 2023","interfaces":[{"interface":"back","average":{"1min":1.018,"5min":2.1040000000000001,"15min":1.5609999999999999},"min":{"1min":0.48399999999999999,"5min":0.42899999999999999,"15min":0.34699999999999998},"max":{"1min":1.833,"5min":12.757999999999999,"15min":12.757999999999999},"last":1.1539999999999999},{"interface":"front","average":{"1min":1.3819999999999999,"5min":1.8999999999999999,"15min":1.4530000000000001},"min":{"1min":0.70099999999999996,"5min":0.53200000000000003,"15min":0.372},"max":{"1min":5.6669999999999998,"5min":11.635,"15min":11.635},"last":0.52000000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1260650496,"data_stored":1260650496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":435421184,"data_stored":435421184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":754319360,"data_stored":754319360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":732102656,"data_stored":732102656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1055719424,"data_stored":1055719424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":659619840,"data_stored":659619840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":887554048,"data_stored":887554048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":516489216,"data_stored":516489216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T10:46:36.951 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd pool set device_health_metrics pgp_num 11 2023-10-01T10:46:36.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:36.952 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:36.952 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:46:37.053 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:37.108 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:37.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:37.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:46:37.214 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:37.241 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:37.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:37.244 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:46:37.296 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 24 16 44019 44003 114.564 78.6875 0.00721843 0.00872345 2023-10-01T10:46:37.345 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:37.379 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:37.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:37.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:46:37.484 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:37.509 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:37.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:37.514 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:46:37.615 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:37.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:37 smithi173 bash[58816]: cluster 2023-10-01T10:46:35.841400+0000 mgr.y (mgr.25072) 869 : cluster 0 pgmap v796: 89 pgs: 89 active+clean; 2.4 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2023-10-01T10:46:37.645 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:37.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:37.649 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:46:37.749 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:37.758 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:37 smithi078 bash[78274]: cluster 2023-10-01T10:46:35.841400+0000 mgr.y (mgr.25072) 869 : cluster 0 pgmap v796: 89 pgs: 89 active+clean; 2.4 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2023-10-01T10:46:37.759 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:37 smithi078 bash[75299]: cluster 2023-10-01T10:46:35.841400+0000 mgr.y (mgr.25072) 869 : cluster 0 pgmap v796: 89 pgs: 89 active+clean; 2.4 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2023-10-01T10:46:37.778 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:37.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:37.784 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:46:37.884 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:37.909 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:37.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:37.912 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:46:38.013 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:38.037 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:38.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:38.041 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:46:38.142 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:38.179 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:38.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:38.182 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:46:38.282 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:38.298 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 25 16 45402 45386 113.432 86.4375 0.00947238 0.00880691 2023-10-01T10:46:38.310 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:38.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:38.316 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:46:38.417 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:38.443 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:38.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:38.447 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:46:38.548 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:38.573 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:38.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:38.579 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:46:38.680 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:38.709 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:38.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:38.712 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:46:38.812 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:38.841 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:38.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:38.889 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:38.990 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:41.147 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 26 16 46784 46768 112.391 86.375 0.0107093 0.00889055 2023-10-01T10:46:41.351 INFO:teuthology.orchestra.run.smithi078.stderr:nodeep-scrub is unset 2023-10-01T10:46:41.352 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:39 smithi078 bash[78274]: cluster 2023-10-01T10:46:37.842033+0000 mgr.y (mgr.25072) 870 : cluster 0 pgmap v797: 89 pgs: 89 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-10-01T10:46:41.352 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:39 smithi078 bash[78274]: audit 2023-10-01T10:46:38.727665+0000 mon.a (mon.0) 1099 : audit 1 from='client.? 172.21.15.78:0/3696465538' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:46:41.358 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:39 smithi078 bash[75299]: cluster 2023-10-01T10:46:37.842033+0000 mgr.y (mgr.25072) 870 : cluster 0 pgmap v797: 89 pgs: 89 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-10-01T10:46:41.358 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:39 smithi078 bash[75299]: audit 2023-10-01T10:46:38.727665+0000 mon.a (mon.0) 1099 : audit 1 from='client.? 172.21.15.78:0/3696465538' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:46:41.359 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 27 16 48006 47990 111.057 76.375 0.0108759 0.00899825 2023-10-01T10:46:41.360 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:40 smithi078 bash[78274]: cluster 2023-10-01T10:46:39.420359+0000 mon.a (mon.0) 1100 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T10:46:41.360 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:40 smithi078 bash[78274]: cluster 2023-10-01T10:46:39.420400+0000 mon.a (mon.0) 1101 : cluster 1 Cluster is now healthy 2023-10-01T10:46:41.360 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:40 smithi078 bash[78274]: cluster 2023-10-01T10:46:39.438855+0000 osd.7 (osd.7) 105 : cluster 3 failed to encode map e156 with expected crc 2023-10-01T10:46:41.360 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:40 smithi078 bash[78274]: audit 2023-10-01T10:46:39.439753+0000 mon.a (mon.0) 1102 : audit 1 from='client.? 172.21.15.78:0/3696465538' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T10:46:41.361 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:40 smithi078 bash[78274]: cluster 2023-10-01T10:46:39.439911+0000 mon.a (mon.0) 1103 : cluster 0 osdmap e156: 8 total, 8 up, 8 in 2023-10-01T10:46:41.361 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:40 smithi078 bash[78274]: cluster 2023-10-01T10:46:39.441549+0000 osd.5 (osd.5) 87 : cluster 3 failed to encode map e156 with expected crc 2023-10-01T10:46:41.361 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:40 smithi078 bash[78274]: cluster 2023-10-01T10:46:39.444722+0000 osd.5 (osd.5) 88 : cluster 3 failed to encode map e156 with expected crc 2023-10-01T10:46:41.361 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:40 smithi078 bash[78274]: cluster 2023-10-01T10:46:39.450020+0000 osd.6 (osd.6) 66 : cluster 3 failed to encode map e156 with expected crc 2023-10-01T10:46:41.362 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:40 smithi078 bash[75299]: cluster 2023-10-01T10:46:39.420359+0000 mon.a (mon.0) 1100 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T10:46:41.364 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:40 smithi078 bash[75299]: cluster 2023-10-01T10:46:39.420400+0000 mon.a (mon.0) 1101 : cluster 1 Cluster is now healthy 2023-10-01T10:46:41.963 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:40 smithi078 bash[75299]: cluster 2023-10-01T10:46:39.438855+0000 osd.7 (osd.7) 105 : cluster 3 failed to encode map e156 with expected crc 2023-10-01T10:46:41.964 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:40 smithi078 bash[75299]: audit 2023-10-01T10:46:39.439753+0000 mon.a (mon.0) 1102 : audit 1 from='client.? 172.21.15.78:0/3696465538' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T10:46:41.964 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:40 smithi078 bash[75299]: cluster 2023-10-01T10:46:39.439911+0000 mon.a (mon.0) 1103 : cluster 0 osdmap e156: 8 total, 8 up, 8 in 2023-10-01T10:46:41.964 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:40 smithi078 bash[75299]: cluster 2023-10-01T10:46:39.441549+0000 osd.5 (osd.5) 87 : cluster 3 failed to encode map e156 with expected crc 2023-10-01T10:46:41.964 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:40 smithi078 bash[75299]: cluster 2023-10-01T10:46:39.444722+0000 osd.5 (osd.5) 88 : cluster 3 failed to encode map e156 with expected crc 2023-10-01T10:46:41.964 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:40 smithi078 bash[75299]: cluster 2023-10-01T10:46:39.450020+0000 osd.6 (osd.6) 66 : cluster 3 failed to encode map e156 with expected crc 2023-10-01T10:46:41.964 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:46:40 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:46:40] "GET /metrics HTTP/1.1" 200 36297 "" "Prometheus/2.33.4" 2023-10-01T10:46:42.150 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:46:42.155 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 28 16 49463 49447 110.34 91.0625 0.0117168 0.00905712 2023-10-01T10:46:42.158 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:41 smithi078 bash[78274]: cluster 2023-10-01T10:46:39.842447+0000 mgr.y (mgr.25072) 871 : cluster 0 pgmap v799: 89 pgs: 89 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-10-01T10:46:42.570 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:41 smithi078 bash[75299]: cluster 2023-10-01T10:46:39.842447+0000 mgr.y (mgr.25072) 871 : cluster 0 pgmap v799: 89 pgs: 89 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-10-01T10:46:42.570 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 29 16 50829 50813 109.479 85.375 0.0162513 0.00912754 2023-10-01T10:46:42.571 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:39 smithi173 bash[58816]: cluster 2023-10-01T10:46:37.842033+0000 mgr.y (mgr.25072) 870 : cluster 0 pgmap v797: 89 pgs: 89 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-10-01T10:46:42.572 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:39 smithi173 bash[58816]: audit 2023-10-01T10:46:38.727665+0000 mon.a (mon.0) 1099 : audit 1 from='client.? 172.21.15.78:0/3696465538' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:46:42.572 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:40 smithi173 bash[58816]: cluster 2023-10-01T10:46:39.420359+0000 mon.a (mon.0) 1100 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T10:46:42.572 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:40 smithi173 bash[58816]: cluster 2023-10-01T10:46:39.420400+0000 mon.a (mon.0) 1101 : cluster 1 Cluster is now healthy 2023-10-01T10:46:42.572 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:40 smithi173 bash[58816]: cluster 2023-10-01T10:46:39.438855+0000 osd.7 (osd.7) 105 : cluster 3 failed to encode map e156 with expected crc 2023-10-01T10:46:42.572 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:40 smithi173 bash[58816]: audit 2023-10-01T10:46:39.439753+0000 mon.a (mon.0) 1102 : audit 1 from='client.? 172.21.15.78:0/3696465538' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T10:46:42.572 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:40 smithi173 bash[58816]: cluster 2023-10-01T10:46:39.439911+0000 mon.a (mon.0) 1103 : cluster 0 osdmap e156: 8 total, 8 up, 8 in 2023-10-01T10:46:42.572 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:40 smithi173 bash[58816]: cluster 2023-10-01T10:46:39.441549+0000 osd.5 (osd.5) 87 : cluster 3 failed to encode map e156 with expected crc 2023-10-01T10:46:42.572 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:40 smithi173 bash[58816]: cluster 2023-10-01T10:46:39.444722+0000 osd.5 (osd.5) 88 : cluster 3 failed to encode map e156 with expected crc 2023-10-01T10:46:42.572 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:40 smithi173 bash[58816]: cluster 2023-10-01T10:46:39.450020+0000 osd.6 (osd.6) 66 : cluster 3 failed to encode map e156 with expected crc 2023-10-01T10:46:42.572 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:41 smithi173 bash[58816]: cluster 2023-10-01T10:46:39.842447+0000 mgr.y (mgr.25072) 871 : cluster 0 pgmap v799: 89 pgs: 89 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-10-01T10:46:42.573 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-10-01T10:46:42.573 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [], 2023-10-01T10:46:42.573 INFO:teuthology.orchestra.run.smithi173.stdout: "complaint_time": 30, 2023-10-01T10:46:42.573 INFO:teuthology.orchestra.run.smithi173.stdout: "num_blocked_ops": 0 2023-10-01T10:46:42.573 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-10-01T10:46:42.605 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:42.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:42.732 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:46:42.833 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:42.924 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:43.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:43.051 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:46:43.151 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:43.243 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:43.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:43.250 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:46:43.299 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 30 16 52125 52109 108.53 81 0.00978093 0.00920812 2023-10-01T10:46:43.362 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:43.387 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:43.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:43.523 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:46:43.623 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:43.684 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-01T10:46:43.738 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:43 smithi173 bash[58816]: cluster 2023-10-01T10:46:41.843096+0000 mgr.y (mgr.25072) 872 : cluster 0 pgmap v800: 89 pgs: 89 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2023-10-01T10:46:44.023 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:43 smithi078 bash[78274]: cluster 2023-10-01T10:46:41.843096+0000 mgr.y (mgr.25072) 872 : cluster 0 pgmap v800: 89 pgs: 89 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2023-10-01T10:46:44.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:44.024 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:43 smithi078 bash[75299]: cluster 2023-10-01T10:46:41.843096+0000 mgr.y (mgr.25072) 872 : cluster 0 pgmap v800: 89 pgs: 89 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2023-10-01T10:46:44.024 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:44.025 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:46:44.125 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:44.157 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T10:46:44.188 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:44.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:44.194 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:44.232 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:46:44.295 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:44.317 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 31 16 53369 53353 107.537 77.75 0.0158649 0.00929192 2023-10-01T10:46:44.344 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:44.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:44.349 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:44.449 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:44.510 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:44.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:44.517 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:46:44.618 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:44.674 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:44.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:44.676 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:46:44.777 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:44.831 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:44.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:44.835 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:46:44.935 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:45.165 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:45.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:45.275 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:46:45.301 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 32 16 54496 54480 106.377 70.4375 0.0115744 0.00939503 2023-10-01T10:46:45.377 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:45.460 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:45.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:45.462 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:46:45.563 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:45.624 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:45.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:45.750 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:46:45.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:45 smithi078 bash[75299]: cluster 2023-10-01T10:46:43.843802+0000 mgr.y (mgr.25072) 873 : cluster 0 pgmap v801: 89 pgs: 89 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-01T10:46:45.906 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:45 smithi078 bash[78274]: cluster 2023-10-01T10:46:43.843802+0000 mgr.y (mgr.25072) 873 : cluster 0 pgmap v801: 89 pgs: 89 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-01T10:46:45.908 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:45.909 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:45 smithi173 bash[58816]: cluster 2023-10-01T10:46:43.843802+0000 mgr.y (mgr.25072) 873 : cluster 0 pgmap v801: 89 pgs: 89 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-01T10:46:45.940 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:45.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:45.944 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:46:46.164 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:46.255 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:46.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:46.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:46:46.299 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 33 16 55661 55645 105.36 72.8125 0.00969176 0.00948545 2023-10-01T10:46:46.360 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:46.391 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:46.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:46.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:46:46.496 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:46.559 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:46.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:46.561 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:46:46.662 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:46.724 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:46.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:46.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:46:46.827 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:46.893 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:46.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:46.895 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:46:46.988 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T10:46:47.024 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:47.032 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T10:46:47.061 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T10:46:47.064 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:47.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:47.069 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:46:47.091 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T10:46:47.180 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:47.181 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T10:46:47.181 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T10:46:47.207 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:47.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:47.211 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:46:47.220 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T10:46:47.256 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T10:46:47.259 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T10:46:47.260 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T10:46:47.260 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T10:46:47.260 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T10:46:47.260 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T10:46:47.260 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T10:46:47.260 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T10:46:47.260 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T10:46:47.299 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 34 16 56844 56828 104.436 73.9375 0.0134215 0.00956901 2023-10-01T10:46:47.334 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:47.365 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:47.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:47.368 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:46:47.453 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:47 smithi078 bash[75299]: cluster 2023-10-01T10:46:45.844258+0000 mgr.y (mgr.25072) 874 : cluster 0 pgmap v802: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-01T10:46:47.477 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:47 smithi078 bash[75299]: audit 2023-10-01T10:46:46.936276+0000 mon.c (mon.2) 154 : audit 0 from='client.? 172.21.15.78:0/2213034245' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:46:47.477 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:47 smithi078 bash[75299]: audit 2023-10-01T10:46:47.431989+0000 mon.a (mon.0) 1104 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:46:47.477 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:47 smithi078 bash[75299]: audit 2023-10-01T10:46:47.434377+0000 mon.a (mon.0) 1105 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:46:47.478 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:47 smithi078 bash[78274]: cluster 2023-10-01T10:46:45.844258+0000 mgr.y (mgr.25072) 874 : cluster 0 pgmap v802: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-01T10:46:47.478 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:47 smithi078 bash[78274]: audit 2023-10-01T10:46:46.936276+0000 mon.c (mon.2) 154 : audit 0 from='client.? 172.21.15.78:0/2213034245' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:46:47.478 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:47 smithi078 bash[78274]: audit 2023-10-01T10:46:47.431989+0000 mon.a (mon.0) 1104 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:46:47.478 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:47 smithi078 bash[78274]: audit 2023-10-01T10:46:47.434377+0000 mon.a (mon.0) 1105 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:46:47.479 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:47.509 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:47.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:47.514 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:46:47.615 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:47.652 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:47.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:47.654 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:46:47.755 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:47.796 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:47 smithi173 bash[58816]: cluster 2023-10-01T10:46:45.844258+0000 mgr.y (mgr.25072) 874 : cluster 0 pgmap v802: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-01T10:46:47.797 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:47 smithi173 bash[58816]: audit 2023-10-01T10:46:46.936276+0000 mon.c (mon.2) 154 : audit 0 from='client.? 172.21.15.78:0/2213034245' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:46:47.797 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:47 smithi173 bash[58816]: audit 2023-10-01T10:46:47.431989+0000 mon.a (mon.0) 1104 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:46:47.797 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:47 smithi173 bash[58816]: audit 2023-10-01T10:46:47.434377+0000 mon.a (mon.0) 1105 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:46:47.808 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:47.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:47.810 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:46:47.911 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:47.984 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:47.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:47.987 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:46:48.088 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:48.112 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:48.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:48.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:48.217 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:48.244 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:48.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:48.269 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:48.299 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 35 16 57783 57767 103.129 58.6875 0.00389559 0.00967762 2023-10-01T10:46:48.370 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:48.448 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:48.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:48.450 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:46:48.551 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:48.579 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T10:46:48.621 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:48.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:48.625 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:46:48.725 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:48.760 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:48.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:48.767 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:48.868 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:48.913 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:48.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:48.915 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:46:49.016 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:49.071 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:49.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:49.077 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:49.177 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:49.232 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:49.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:49.239 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:46:49.300 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 36 16 58994 58978 102.366 75.6875 0.0154467 0.00976303 2023-10-01T10:46:49.359 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:49.387 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:49.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:49.391 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:46:49.492 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:49.522 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:49 smithi173 bash[58816]: cluster 2023-10-01T10:46:47.844949+0000 mgr.y (mgr.25072) 875 : cluster 0 pgmap v803: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-01T10:46:49.540 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:49.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:49.542 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:46:49.643 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:49.679 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:49.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:49.681 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:46:49.782 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:49.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:49 smithi078 bash[78274]: cluster 2023-10-01T10:46:47.844949+0000 mgr.y (mgr.25072) 875 : cluster 0 pgmap v803: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-01T10:46:49.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:49 smithi078 bash[75299]: cluster 2023-10-01T10:46:47.844949+0000 mgr.y (mgr.25072) 875 : cluster 0 pgmap v803: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-01T10:46:49.848 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:49.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:49.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:46:49.951 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:50.003 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:50.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:50.008 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:46:50.108 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:50.135 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:50.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:50.141 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:50.242 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:50.300 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 37 16 60246 60230 101.715 78.25 0.0181468 0.00982519 2023-10-01T10:46:50.316 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:50.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:50.320 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:50.420 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:50.479 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:50.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:50.483 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:46:50.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:46:50 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:46:50] "GET /metrics HTTP/1.1" 200 36298 "" "Prometheus/2.33.4" 2023-10-01T10:46:50.589 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:50.592 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:46:50.612 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:50.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:50.615 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:46:50.716 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:50.778 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:50.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:50.780 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:46:50.881 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:50.927 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:50.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:50.929 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:46:51.029 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:51.088 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:51.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:51.092 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:46:51.107 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T10:46:51.193 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:51.241 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:51.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:51.245 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:46:51.300 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 38 16 61419 61403 100.967 73.3125 0.00750339 0.00989754 2023-10-01T10:46:51.345 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:51.371 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:51.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:51.376 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:46:51.477 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:51.509 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:51.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:51.512 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:46:51.612 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:51.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:51 smithi173 bash[58816]: cluster 2023-10-01T10:46:49.845792+0000 mgr.y (mgr.25072) 876 : cluster 0 pgmap v804: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-10-01T10:46:51.640 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:51.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:51.645 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:46:51.746 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:51.773 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:51.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:51.778 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:46:51.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:51 smithi078 bash[75299]: cluster 2023-10-01T10:46:49.845792+0000 mgr.y (mgr.25072) 876 : cluster 0 pgmap v804: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-10-01T10:46:51.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:51 smithi078 bash[78274]: cluster 2023-10-01T10:46:49.845792+0000 mgr.y (mgr.25072) 876 : cluster 0 pgmap v804: 89 pgs: 89 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-10-01T10:46:51.878 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:51.902 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:51.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:51.903 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:46:52.004 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:52.032 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:52.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:52.037 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:52.138 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:52.171 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:52.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:52.175 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:46:52.276 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:52.300 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 39 16 62481 62465 100.08 66.375 0.00794434 0.00998613 2023-10-01T10:46:52.300 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:52.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:52.303 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:52.404 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:52.431 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:52.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:52.434 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:46:52.535 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:52.560 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:52.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:52.563 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:52.664 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:52.687 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:52.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:52.690 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:46:52.791 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:52.826 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:52.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:52.829 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:46:52.930 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:52.963 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:52.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:52.966 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:46:53.067 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:53.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:53.099 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:53.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:46:53.200 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:53.225 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:53.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:53.240 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:46:53.300 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:2023-10-01T10:46:53.299472+0000 min lat: 0.00137887 max lat: 0.206481 avg lat: 0.010069 2023-10-01T10:46:53.300 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T10:46:53.300 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 40 16 63553 63537 99.253 67 0.0165939 0.010069 2023-10-01T10:46:53.342 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:53.385 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:53.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:53.389 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:53.468 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:53 smithi078 bash[75299]: cluster 2023-10-01T10:46:51.846200+0000 mgr.y (mgr.25072) 877 : cluster 0 pgmap v805: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-01T10:46:53.489 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:53.516 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:53.517 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:53 smithi078 bash[78274]: cluster 2023-10-01T10:46:51.846200+0000 mgr.y (mgr.25072) 877 : cluster 0 pgmap v805: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-01T10:46:53.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:53.518 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:53.543 INFO:teuthology.orchestra.run.smithi078.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-10-01T10:46:53.620 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:53.806 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:53.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:53.811 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:53.850 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-10-01T10:46:53.850 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T10:46:53.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:53 smithi173 bash[58816]: cluster 2023-10-01T10:46:51.846200+0000 mgr.y (mgr.25072) 877 : cluster 0 pgmap v805: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-01T10:46:53.912 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:53.941 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:53.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:53.945 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:46:54.046 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:54.072 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:54.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:54.080 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:46:54.180 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:54.208 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:54.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:54.213 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:46:54.300 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 41 16 64691 64675 98.5669 71.125 0.0115707 0.0101399 2023-10-01T10:46:54.314 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:54.349 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:54.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:54.356 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:46:54.456 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:54.478 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:54 smithi173 bash[58816]: audit 2023-10-01T10:46:53.537728+0000 mon.a (mon.0) 1106 : audit 1 from='client.? 172.21.15.78:0/639695179' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-01T10:46:54.489 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:54.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:54.492 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:54.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:54 smithi078 bash[78274]: audit 2023-10-01T10:46:53.537728+0000 mon.a (mon.0) 1106 : audit 1 from='client.? 172.21.15.78:0/639695179' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-01T10:46:54.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:54 smithi078 bash[75299]: audit 2023-10-01T10:46:53.537728+0000 mon.a (mon.0) 1106 : audit 1 from='client.? 172.21.15.78:0/639695179' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-01T10:46:54.593 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:54.624 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:54.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:54.628 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:46:54.729 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:54.761 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:54.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:54.766 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:46:54.867 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:54.893 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:54.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:54.901 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:46:54.974 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-01T10:46:55.002 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:55.034 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:55.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:55.040 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:46:55.141 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:55.180 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:55.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:55.183 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:46:55.284 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:55.302 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 42 16 65796 65780 97.8606 69.0625 0.00866027 0.0102118 2023-10-01T10:46:55.310 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:55.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:55.312 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:46:55.413 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:55.449 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:55.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:55.452 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:46:55.553 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:55.571 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:55 smithi078 bash[75299]: cluster 2023-10-01T10:46:53.846744+0000 mgr.y (mgr.25072) 878 : cluster 0 pgmap v806: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s 2023-10-01T10:46:55.572 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:55 smithi078 bash[78274]: cluster 2023-10-01T10:46:53.846744+0000 mgr.y (mgr.25072) 878 : cluster 0 pgmap v806: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s 2023-10-01T10:46:55.592 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:55.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:55.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:55.752 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-10-01T10:46:55.766 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:55.790 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:55.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:55.794 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:46:55.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:55 smithi173 bash[58816]: cluster 2023-10-01T10:46:53.846744+0000 mgr.y (mgr.25072) 878 : cluster 0 pgmap v806: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s 2023-10-01T10:46:55.895 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:55.927 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:55.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:55.931 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:46:56.032 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:56.060 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:56.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:56.063 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:56.163 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:56.191 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:56.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:56.202 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:46:56.302 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 43 16 66931 66915 97.2344 70.9375 0.0136312 0.0102787 2023-10-01T10:46:56.303 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:56.332 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:56.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:56.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:46:56.436 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:56.459 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:56.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:56.463 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:46:56.564 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:56.592 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:56.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:56.667 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:46:56.767 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:56.804 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:56.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:56.826 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:46:56.927 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:56.961 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:56.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:56.966 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:46:57.067 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:57.098 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:57.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:57.101 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:57.202 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:57.234 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:57.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:57.241 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:46:57.302 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 44 16 68072 68056 96.6452 71.3125 0.00992382 0.0103414 2023-10-01T10:46:57.342 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:57.366 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:57.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:57.371 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:46:57.471 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:57.498 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:57.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:57.501 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:46:57.602 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:57.611 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:57 smithi078 bash[75299]: cluster 2023-10-01T10:46:55.847189+0000 mgr.y (mgr.25072) 879 : cluster 0 pgmap v807: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-01T10:46:57.611 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:57 smithi078 bash[78274]: cluster 2023-10-01T10:46:55.847189+0000 mgr.y (mgr.25072) 879 : cluster 0 pgmap v807: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-01T10:46:57.634 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:57.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:57.637 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:57.737 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:57.746 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:57 smithi173 bash[58816]: cluster 2023-10-01T10:46:55.847189+0000 mgr.y (mgr.25072) 879 : cluster 0 pgmap v807: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-01T10:46:57.763 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:46:57.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:57.767 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:46:57.868 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:57.893 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:57.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:57.898 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:46:57.999 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:58.035 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:58.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:58.041 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:46:58.142 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:58.168 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:58.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:58.172 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:46:58.273 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:58.301 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:58.302 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 45 16 69171 69155 96.0239 68.6875 0.017439 0.0104087 2023-10-01T10:46:58.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:58.306 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:46:58.407 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:58.435 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:58.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:58.439 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:46:58.511 INFO:teuthology.orchestra.run.smithi078.stderr:noscrub is set 2023-10-01T10:46:58.539 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:58.549 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:58 smithi173 bash[58816]: audit 2023-10-01T10:46:57.750889+0000 mon.c (mon.2) 155 : audit 1 from='client.? 172.21.15.78:0/3287560959' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:46:58.549 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:58 smithi173 bash[58816]: audit 2023-10-01T10:46:57.751977+0000 mon.a (mon.0) 1107 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:46:58.567 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:58.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:58.571 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:46:58.672 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:58.680 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:58 smithi078 bash[78274]: audit 2023-10-01T10:46:57.750889+0000 mon.c (mon.2) 155 : audit 1 from='client.? 172.21.15.78:0/3287560959' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:46:58.681 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:58 smithi078 bash[78274]: audit 2023-10-01T10:46:57.751977+0000 mon.a (mon.0) 1107 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:46:58.681 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:58 smithi078 bash[75299]: audit 2023-10-01T10:46:57.750889+0000 mon.c (mon.2) 155 : audit 1 from='client.? 172.21.15.78:0/3287560959' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:46:58.681 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:58 smithi078 bash[75299]: audit 2023-10-01T10:46:57.751977+0000 mon.a (mon.0) 1107 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:46:58.704 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:58.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:58.707 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:46:58.808 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:58.835 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:46:58.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:58.840 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:46:58.940 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:58.967 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:58.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:58.977 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:46:59.078 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:59.104 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:59.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:59.109 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:46:59.209 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:59.230 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:46:59.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:59.231 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:46:59.302 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 46 16 70206 70190 95.3426 64.6875 0.003206 0.0104615 2023-10-01T10:46:59.332 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:59.357 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:46:59.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:59.360 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:46:59.461 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:59.493 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:46:59.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:59.495 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:46:59.596 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:59.606 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:59 smithi173 bash[58816]: cluster 2023-10-01T10:46:57.847811+0000 mgr.y (mgr.25072) 880 : cluster 0 pgmap v808: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 61 MiB/s wr, 980 op/s 2023-10-01T10:46:59.606 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:59 smithi173 bash[58816]: cluster 2023-10-01T10:46:58.482872+0000 mon.a (mon.0) 1108 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:46:59.606 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:59 smithi173 bash[58816]: audit 2023-10-01T10:46:58.502293+0000 mon.a (mon.0) 1109 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T10:46:59.607 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:59 smithi173 bash[58816]: cluster 2023-10-01T10:46:58.502432+0000 mon.a (mon.0) 1110 : cluster 0 osdmap e157: 8 total, 8 up, 8 in 2023-10-01T10:46:59.607 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:59 smithi173 bash[58816]: cluster 2023-10-01T10:46:58.503626+0000 osd.7 (osd.7) 106 : cluster 3 failed to encode map e157 with expected crc 2023-10-01T10:46:59.607 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:59 smithi173 bash[58816]: cluster 2023-10-01T10:46:58.508438+0000 osd.6 (osd.6) 67 : cluster 3 failed to encode map e157 with expected crc 2023-10-01T10:46:59.607 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:59 smithi173 bash[58816]: cluster 2023-10-01T10:46:58.510198+0000 osd.6 (osd.6) 68 : cluster 3 failed to encode map e157 with expected crc 2023-10-01T10:46:59.607 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:59 smithi173 bash[58816]: cluster 2023-10-01T10:46:58.511165+0000 osd.5 (osd.5) 89 : cluster 3 failed to encode map e157 with expected crc 2023-10-01T10:46:59.607 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:46:59 smithi173 bash[58816]: cluster 2023-10-01T10:46:58.512921+0000 osd.6 (osd.6) 69 : cluster 3 failed to encode map e157 with expected crc 2023-10-01T10:46:59.629 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:46:59.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:59.638 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:46:59.739 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:59.780 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:46:59.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:59.785 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:46:59.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:59 smithi078 bash[75299]: cluster 2023-10-01T10:46:57.847811+0000 mgr.y (mgr.25072) 880 : cluster 0 pgmap v808: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 61 MiB/s wr, 980 op/s 2023-10-01T10:46:59.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:59 smithi078 bash[75299]: cluster 2023-10-01T10:46:58.482872+0000 mon.a (mon.0) 1108 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:46:59.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:59 smithi078 bash[75299]: audit 2023-10-01T10:46:58.502293+0000 mon.a (mon.0) 1109 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T10:46:59.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:59 smithi078 bash[75299]: cluster 2023-10-01T10:46:58.502432+0000 mon.a (mon.0) 1110 : cluster 0 osdmap e157: 8 total, 8 up, 8 in 2023-10-01T10:46:59.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:59 smithi078 bash[75299]: cluster 2023-10-01T10:46:58.503626+0000 osd.7 (osd.7) 106 : cluster 3 failed to encode map e157 with expected crc 2023-10-01T10:46:59.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:59 smithi078 bash[75299]: cluster 2023-10-01T10:46:58.508438+0000 osd.6 (osd.6) 67 : cluster 3 failed to encode map e157 with expected crc 2023-10-01T10:46:59.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:59 smithi078 bash[75299]: cluster 2023-10-01T10:46:58.510198+0000 osd.6 (osd.6) 68 : cluster 3 failed to encode map e157 with expected crc 2023-10-01T10:46:59.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:59 smithi078 bash[75299]: cluster 2023-10-01T10:46:58.511165+0000 osd.5 (osd.5) 89 : cluster 3 failed to encode map e157 with expected crc 2023-10-01T10:46:59.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:46:59 smithi078 bash[75299]: cluster 2023-10-01T10:46:58.512921+0000 osd.6 (osd.6) 69 : cluster 3 failed to encode map e157 with expected crc 2023-10-01T10:46:59.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:59 smithi078 bash[78274]: cluster 2023-10-01T10:46:57.847811+0000 mgr.y (mgr.25072) 880 : cluster 0 pgmap v808: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 61 MiB/s wr, 980 op/s 2023-10-01T10:46:59.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:59 smithi078 bash[78274]: cluster 2023-10-01T10:46:58.482872+0000 mon.a (mon.0) 1108 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:46:59.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:59 smithi078 bash[78274]: audit 2023-10-01T10:46:58.502293+0000 mon.a (mon.0) 1109 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T10:46:59.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:59 smithi078 bash[78274]: cluster 2023-10-01T10:46:58.502432+0000 mon.a (mon.0) 1110 : cluster 0 osdmap e157: 8 total, 8 up, 8 in 2023-10-01T10:46:59.835 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:59 smithi078 bash[78274]: cluster 2023-10-01T10:46:58.503626+0000 osd.7 (osd.7) 106 : cluster 3 failed to encode map e157 with expected crc 2023-10-01T10:46:59.836 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:59 smithi078 bash[78274]: cluster 2023-10-01T10:46:58.508438+0000 osd.6 (osd.6) 67 : cluster 3 failed to encode map e157 with expected crc 2023-10-01T10:46:59.836 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:59 smithi078 bash[78274]: cluster 2023-10-01T10:46:58.510198+0000 osd.6 (osd.6) 68 : cluster 3 failed to encode map e157 with expected crc 2023-10-01T10:46:59.836 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:59 smithi078 bash[78274]: cluster 2023-10-01T10:46:58.511165+0000 osd.5 (osd.5) 89 : cluster 3 failed to encode map e157 with expected crc 2023-10-01T10:46:59.836 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:46:59 smithi078 bash[78274]: cluster 2023-10-01T10:46:58.512921+0000 osd.6 (osd.6) 69 : cluster 3 failed to encode map e157 with expected crc 2023-10-01T10:46:59.886 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:59.912 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:46:59.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:46:59.916 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:47:00.017 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:00.051 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:00.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:00.058 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:00.159 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:00.190 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:00.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:00.193 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:47:00.293 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:00.302 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 47 16 71137 71121 94.5521 58.1875 0.0189021 0.010571 2023-10-01T10:47:00.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:00.327 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:00.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:47:00.361 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:47:00.428 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:00.462 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:00.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:00.467 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:00.568 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:00.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:47:00 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:47:00] "GET /metrics HTTP/1.1" 200 36298 "" "Prometheus/2.33.4" 2023-10-01T10:47:00.611 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:00.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:00.613 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:47:00.714 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:00.742 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:00.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:00.745 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:47:00.754 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T10:47:00.754 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T10:47:00.754 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2023-10-01T10:47:00.754 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-01T10:47:00.754 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-01T10:47:00.754 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 config set filestore_inject_stall 3 2023-10-01T10:47:00.847 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:00.877 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:00.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:00.880 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:47:00.981 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:01.015 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:01.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:01.018 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:47:01.119 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:01.146 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:01.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:01.151 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:47:01.252 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:01.276 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:01.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:01.280 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:47:01.302 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 48 16 72188 72172 93.9507 65.6875 0.0160437 0.0106385 2023-10-01T10:47:01.381 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:01.424 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:01.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:01.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:47:01.528 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:01.554 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:01.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:01.557 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:47:01.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:01 smithi173 bash[58816]: cluster 2023-10-01T10:46:59.848800+0000 mgr.y (mgr.25072) 881 : cluster 0 pgmap v810: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-10-01T10:47:01.658 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:01.697 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:01.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:01.702 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:01.803 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:01.811 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:01 smithi078 bash[78274]: cluster 2023-10-01T10:46:59.848800+0000 mgr.y (mgr.25072) 881 : cluster 0 pgmap v810: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-10-01T10:47:01.812 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:01 smithi078 bash[75299]: cluster 2023-10-01T10:46:59.848800+0000 mgr.y (mgr.25072) 881 : cluster 0 pgmap v810: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-10-01T10:47:01.831 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:01.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:01.834 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:47:01.935 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:01.951 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T10:47:01.962 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:01.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:01.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:47:02.070 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:02.097 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:02.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:02.102 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:47:02.202 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:02.243 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:02.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:02.248 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:47:02.302 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 49 16 73234 73218 93.3675 65.375 0.00984611 0.0107042 2023-10-01T10:47:02.349 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:02.384 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:02.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:02.386 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:47:02.455 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:47:02.488 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:02.516 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:02.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:02.519 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:47:02.620 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:02.658 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:02.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:02.663 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:47:02.764 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:02.789 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:02.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:02.793 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:47:02.894 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:02.924 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:02.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:02.928 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:47:03.029 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:03.070 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:03.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:03.075 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:47:03.175 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:03.201 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:03.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:03.203 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:47:03.303 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 50 16 74313 74297 92.8489 67.4375 0.0124773 0.010765 2023-10-01T10:47:03.304 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:03.326 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:03.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:03.328 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:47:03.428 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:03.464 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:03.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:03.470 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:47:03.571 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:03.580 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:03 smithi173 bash[58816]: cluster 2023-10-01T10:47:01.849457+0000 mgr.y (mgr.25072) 882 : cluster 0 pgmap v811: 89 pgs: 89 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-10-01T10:47:03.580 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:03 smithi173 bash[58816]: audit 2023-10-01T10:47:02.428082+0000 mon.a (mon.0) 1111 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:47:03.580 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:03 smithi173 bash[58816]: audit 2023-10-01T10:47:02.430487+0000 mon.a (mon.0) 1112 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:47:03.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:03 smithi078 bash[78274]: cluster 2023-10-01T10:47:01.849457+0000 mgr.y (mgr.25072) 882 : cluster 0 pgmap v811: 89 pgs: 89 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-10-01T10:47:03.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:03 smithi078 bash[78274]: audit 2023-10-01T10:47:02.428082+0000 mon.a (mon.0) 1111 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:47:03.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:03 smithi078 bash[78274]: audit 2023-10-01T10:47:02.430487+0000 mon.a (mon.0) 1112 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:47:03.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:03 smithi078 bash[75299]: cluster 2023-10-01T10:47:01.849457+0000 mgr.y (mgr.25072) 882 : cluster 0 pgmap v811: 89 pgs: 89 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-10-01T10:47:03.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:03 smithi078 bash[75299]: audit 2023-10-01T10:47:02.428082+0000 mon.a (mon.0) 1111 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:47:03.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:03 smithi078 bash[75299]: audit 2023-10-01T10:47:02.430487+0000 mon.a (mon.0) 1112 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:47:03.612 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:03.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:03.615 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:47:03.717 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:03.750 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:03.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:03.752 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:03.853 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:03.879 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:03.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:03.884 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:47:03.985 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:04.013 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:04.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:04.017 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:47:04.120 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:04.148 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:04.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:04.154 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:47:04.254 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:04.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:04.285 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:04.285 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:04.302 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 51 16 75288 75272 92.2231 60.9375 0.00755186 0.0108377 2023-10-01T10:47:04.386 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:04.412 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:04.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:04.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:47:04.520 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:04.560 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:04.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:04.562 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:47:04.663 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:04.692 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:04.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:04.697 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:47:04.798 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:04.825 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:04.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:04.830 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:47:04.930 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:04.967 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:04.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:04.970 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:05.071 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:05.104 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:05.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:05.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:47:05.208 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:05.241 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:05.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:05.246 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:05.303 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 52 16 76378 76362 91.7596 68.125 0.0134215 0.0108934 2023-10-01T10:47:05.347 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:05.385 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:05.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:05.388 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:47:05.489 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:05.498 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:05 smithi173 bash[58816]: cluster 2023-10-01T10:47:03.851141+0000 mgr.y (mgr.25072) 883 : cluster 0 pgmap v812: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-01T10:47:05.520 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:05.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:05.524 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:47:05.625 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:05.632 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T10:47:05.632 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:05 smithi078 bash[75299]: cluster 2023-10-01T10:47:03.851141+0000 mgr.y (mgr.25072) 883 : cluster 0 pgmap v812: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-01T10:47:05.633 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:05 smithi078 bash[78274]: cluster 2023-10-01T10:47:03.851141+0000 mgr.y (mgr.25072) 883 : cluster 0 pgmap v812: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-01T10:47:05.650 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:05.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:05.653 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:47:05.754 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:05.789 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:05.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:05.791 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:47:05.892 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:05.920 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:05.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:05.922 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:47:06.023 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:06.047 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:06.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:06.050 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:06.150 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:06.181 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:06.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:06.184 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:47:06.285 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:06.304 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 53 16 77441 77425 91.2793 66.4375 0.0311541 0.0109502 2023-10-01T10:47:06.313 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:06.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:06.318 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:47:06.418 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:06.448 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:06.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:06.450 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:47:06.528 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:47:06.551 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:06.577 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:06.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:06.585 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:47:06.686 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:06.698 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T10:47:06.716 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:06.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:06.721 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:47:06.738 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T10:47:06.782 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T10:47:06.822 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:06.823 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T10:47:06.845 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:06.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:06.849 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:47:06.870 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T10:47:06.910 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T10:47:06.947 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T10:47:06.949 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:06.981 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:06.983 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T10:47:06.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:06.987 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:47:06.990 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T10:47:06.991 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T10:47:06.991 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T10:47:06.991 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T10:47:06.991 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T10:47:06.991 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T10:47:06.991 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T10:47:06.991 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T10:47:07.088 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:07.122 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:07.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:07.126 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:47:07.227 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:07.252 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:07.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:07.255 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:47:07.305 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 54 16 78346 78330 90.6364 56.5625 0.00970884 0.0110176 2023-10-01T10:47:07.357 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:07.385 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:07.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:07.390 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:47:07.490 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:07.512 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:07.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:07.515 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:47:07.616 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:07.656 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:07.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:07.662 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:47:07.763 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:07.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:07 smithi173 bash[58816]: cluster 2023-10-01T10:47:05.851655+0000 mgr.y (mgr.25072) 884 : cluster 0 pgmap v813: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-01T10:47:07.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:07 smithi173 bash[58816]: audit 2023-10-01T10:47:06.635946+0000 mon.b (mon.1) 43 : audit 0 from='client.? 172.21.15.78:0/4096719473' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:47:07.797 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:07.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:07.800 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:07.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:07 smithi078 bash[75299]: cluster 2023-10-01T10:47:05.851655+0000 mgr.y (mgr.25072) 884 : cluster 0 pgmap v813: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-01T10:47:08.804 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:07 smithi078 bash[75299]: audit 2023-10-01T10:47:06.635946+0000 mon.b (mon.1) 43 : audit 0 from='client.? 172.21.15.78:0/4096719473' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:47:08.805 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:07 smithi078 bash[78274]: cluster 2023-10-01T10:47:05.851655+0000 mgr.y (mgr.25072) 884 : cluster 0 pgmap v813: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-01T10:47:08.805 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:07 smithi078 bash[78274]: audit 2023-10-01T10:47:06.635946+0000 mon.b (mon.1) 43 : audit 0 from='client.? 172.21.15.78:0/4096719473' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:47:08.805 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 55 16 79198 79182 89.9566 53.25 0.00597822 0.0111084 2023-10-01T10:47:08.806 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:08.844 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:08.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:08.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:47:08.949 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:08.987 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:08.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:08.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:47:09.089 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:09.119 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:09.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:09.122 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:09.223 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:09.274 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:09.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:09.277 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:47:09.314 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 56 16 80171 80155 89.4362 60.8125 0.0128157 0.0111751 2023-10-01T10:47:09.378 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:09.412 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:09.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:09.416 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:09.517 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:09.552 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:09.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:09.554 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:47:09.655 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:09.689 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:09.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:09.695 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:47:09.795 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:09.807 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:09 smithi173 bash[58816]: cluster 2023-10-01T10:47:07.852187+0000 mgr.y (mgr.25072) 885 : cluster 0 pgmap v814: 89 pgs: 89 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-01T10:47:09.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:09.830 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:09.830 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:47:09.867 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:09 smithi078 bash[75299]: cluster 2023-10-01T10:47:07.852187+0000 mgr.y (mgr.25072) 885 : cluster 0 pgmap v814: 89 pgs: 89 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-01T10:47:09.868 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:09 smithi078 bash[78274]: cluster 2023-10-01T10:47:07.852187+0000 mgr.y (mgr.25072) 885 : cluster 0 pgmap v814: 89 pgs: 89 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-01T10:47:09.931 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:09.956 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:09.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:09.962 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:10.062 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:10.095 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:10.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:10.099 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:47:10.200 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:10.231 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:10.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:10.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:47:10.305 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 57 16 81193 81177 88.9877 63.875 0.0130213 0.0112317 2023-10-01T10:47:10.336 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:10.351 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:47:10 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:47:10] "GET /metrics HTTP/1.1" 200 36303 "" "Prometheus/2.33.4" 2023-10-01T10:47:10.367 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:10.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:10.370 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:47:10.470 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:10.498 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:10.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:10.502 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:10.603 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:10.637 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:10.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:10.641 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:47:10.742 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:10.776 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:10.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:10.781 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:10.875 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-10-01T10:47:10.875 INFO:teuthology.orchestra.run.smithi173.stdout: "success": "" 2023-10-01T10:47:10.875 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-10-01T10:47:10.883 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:10.913 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:10.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:10.916 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:47:11.017 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:11.052 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:11.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:11.054 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:47:11.155 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:11.179 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:11.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:11.185 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:47:11.286 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:11.305 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 58 16 82151 82135 88.4857 59.875 0.00588007 0.0112949 2023-10-01T10:47:11.315 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:11.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:11.319 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:47:11.420 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:11.448 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:11.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:11.451 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:47:11.552 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:11.568 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T10:47:11.578 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:11.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:11.581 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:47:11.650 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:11 smithi078 bash[75299]: cluster 2023-10-01T10:47:09.852966+0000 mgr.y (mgr.25072) 886 : cluster 0 pgmap v815: 89 pgs: 89 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-01T10:47:11.650 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:11 smithi078 bash[78274]: cluster 2023-10-01T10:47:09.852966+0000 mgr.y (mgr.25072) 886 : cluster 0 pgmap v815: 89 pgs: 89 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-01T10:47:11.682 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:11.716 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:11.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:11.720 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:47:11.821 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:11.830 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:11 smithi173 bash[58816]: cluster 2023-10-01T10:47:09.852966+0000 mgr.y (mgr.25072) 886 : cluster 0 pgmap v815: 89 pgs: 89 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2023-10-01T10:47:11.851 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:11.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:11.856 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:11.956 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:11.966 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2023-10-01T10:47:11.966 INFO:teuthology.orchestra.run.smithi078.stdout: "ops": [ 2023-10-01T10:47:11.966 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:11.966 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.45818.0:82717 10.d e157/151 10:bdbd2966:::benchmark_data_smithi078_135100_object82716:head v 157'5210, mlcod=157'5208)", 2023-10-01T10:47:11.966 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:47:11.952427+0000", 2023-10-01T10:47:11.966 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 0.010570054000000001, 2023-10-01T10:47:11.966 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.010602297, 2023-10-01T10:47:11.966 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:47:11.967 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "started", 2023-10-01T10:47:11.967 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:47:11.967 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:11.967 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:47:11.967 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:11.952427+0000", 2023-10-01T10:47:11.967 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:47:11.967 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:11.967 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:11.967 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:47:11.967 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:11.952427+0000", 2023-10-01T10:47:11.967 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:47:11.967 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:11.968 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:11.968 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:47:11.968 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:11.952426+0000", 2023-10-01T10:47:11.968 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:47:11.968 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:11.968 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:11.968 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:47:11.968 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:11.952466+0000", 2023-10-01T10:47:11.968 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.0759000000000001e-05 2023-10-01T10:47:11.968 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:11.968 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:11.968 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:47:11.968 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:11.952479+0000", 2023-10-01T10:47:11.969 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.2594e-05 2023-10-01T10:47:11.969 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:11.969 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:11.969 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:47:11.969 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:11.952487+0000", 2023-10-01T10:47:11.969 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.0870000000000007e-06 2023-10-01T10:47:11.969 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:11.969 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:11.969 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:47:11.969 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:11.952582+0000", 2023-10-01T10:47:11.969 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.5428999999999997e-05 2023-10-01T10:47:11.969 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:11.970 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:11.970 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:47:11.970 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:11.952603+0000", 2023-10-01T10:47:11.970 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.0347e-05 2023-10-01T10:47:11.970 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:47:11.970 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:47:11.970 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:47:11.970 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:11.970 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:11.970 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.45818.0:82721 10.d e157/151 10:b11f95b7:::benchmark_data_smithi078_135100_object82720:head v 157'5211, mlcod=157'5208)", 2023-10-01T10:47:11.970 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:47:11.956514+0000", 2023-10-01T10:47:11.970 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 0.0064829409999999999, 2023-10-01T10:47:11.971 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0065843339999999998, 2023-10-01T10:47:11.971 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:47:11.971 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "started", 2023-10-01T10:47:11.971 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:47:11.971 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:11.971 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:47:11.971 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:11.956514+0000", 2023-10-01T10:47:11.971 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:47:11.971 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:11.971 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:11.971 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:47:11.971 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:11.956514+0000", 2023-10-01T10:47:11.971 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:47:11.972 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:11.972 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:11.972 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:47:11.972 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:11.956513+0000", 2023-10-01T10:47:11.972 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:47:11.972 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:11.972 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:11.972 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:47:11.972 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:11.956553+0000", 2023-10-01T10:47:11.972 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.9805000000000003e-05 2023-10-01T10:47:11.972 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:11.972 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:11.972 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:47:11.973 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:11.956571+0000", 2023-10-01T10:47:11.973 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.7934000000000001e-05 2023-10-01T10:47:11.973 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:11.973 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:11.973 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:47:11.973 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:11.956579+0000", 2023-10-01T10:47:11.973 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.9240000000000007e-06 2023-10-01T10:47:11.973 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:11.973 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:11.973 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:47:11.973 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:11.956649+0000", 2023-10-01T10:47:11.973 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.0708000000000001e-05 2023-10-01T10:47:11.974 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:11.974 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:11.974 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:47:11.974 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:11.956674+0000", 2023-10-01T10:47:11.974 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.4564999999999999e-05 2023-10-01T10:47:11.974 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:47:11.974 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:47:11.974 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:47:11.974 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:11.974 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:11.974 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.45818.0:82722 10.c e157/151 10:390eacc3:::benchmark_data_smithi078_135100_object82721:head v 157'5183, mlcod=157'5180)", 2023-10-01T10:47:11.974 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:47:11.958345+0000", 2023-10-01T10:47:11.975 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 0.0046514449999999997, 2023-10-01T10:47:11.975 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0048140359999999998, 2023-10-01T10:47:11.975 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:47:11.975 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "started", 2023-10-01T10:47:11.975 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:47:11.975 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:11.975 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:47:11.975 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:11.958345+0000", 2023-10-01T10:47:11.975 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:47:11.975 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:11.975 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:11.975 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:47:11.976 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:11.958345+0000", 2023-10-01T10:47:11.976 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:47:11.976 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:11.976 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:11.976 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:47:11.976 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:11.958344+0000", 2023-10-01T10:47:11.976 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:47:11.976 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:11.976 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:11.976 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:47:11.976 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:11.958394+0000", 2023-10-01T10:47:11.976 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.0009999999999997e-05 2023-10-01T10:47:11.976 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:11.977 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:11.977 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:47:11.977 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:11.958412+0000", 2023-10-01T10:47:11.977 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.7261999999999998e-05 2023-10-01T10:47:11.977 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:11.977 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:11.977 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:47:11.977 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:11.958419+0000", 2023-10-01T10:47:11.977 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.3939999999999998e-06 2023-10-01T10:47:11.977 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:11.977 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:11.977 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:47:11.978 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:11.958566+0000", 2023-10-01T10:47:11.978 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000147329 2023-10-01T10:47:11.978 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:11.978 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:11.978 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:47:11.978 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:11.958636+0000", 2023-10-01T10:47:11.978 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.9344999999999994e-05 2023-10-01T10:47:11.978 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:47:11.978 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:47:11.979 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:47:11.979 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:11.979 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:11.979 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.45818.0:82726 10.2 e157/151 10:4e70dd3b:::benchmark_data_smithi078_135100_object82725:head v 157'5197, mlcod=157'5195)", 2023-10-01T10:47:11.979 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:47:11.961576+0000", 2023-10-01T10:47:11.979 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 0.0014208389999999999, 2023-10-01T10:47:11.979 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0016435779999999999, 2023-10-01T10:47:11.979 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:47:11.979 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "started", 2023-10-01T10:47:11.979 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:47:11.979 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:11.979 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:47:11.980 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:11.961576+0000", 2023-10-01T10:47:11.980 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:47:11.980 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:11.980 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:11.980 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:47:11.980 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:11.961576+0000", 2023-10-01T10:47:11.980 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:47:11.980 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:11.980 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:11.980 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:47:11.980 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:11.961575+0000", 2023-10-01T10:47:11.980 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:47:11.981 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:11.981 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:11.981 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:47:11.981 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:11.961604+0000", 2023-10-01T10:47:11.981 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.9419000000000001e-05 2023-10-01T10:47:11.981 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:11.981 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:11.981 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:47:11.981 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:11.961624+0000", 2023-10-01T10:47:11.981 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.9335999999999999e-05 2023-10-01T10:47:11.981 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:11.981 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:11.982 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:47:11.982 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:11.961632+0000", 2023-10-01T10:47:11.982 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.6279999999999994e-06 2023-10-01T10:47:11.982 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:11.982 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:11.982 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:47:11.982 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:11.961737+0000", 2023-10-01T10:47:11.982 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00010492000000000001 2023-10-01T10:47:11.982 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:11.982 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:11.982 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:47:11.982 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:11.961761+0000", 2023-10-01T10:47:11.983 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.3515000000000001e-05 2023-10-01T10:47:11.983 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:47:11.983 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:47:11.983 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:47:11.983 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:11.983 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:11.983 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.45818.0:82728 10.b e157/151 10:d8f58662:::benchmark_data_smithi078_135100_object82727:head v 157'5025, mlcod=157'5022)", 2023-10-01T10:47:11.983 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:47:11.962661+0000", 2023-10-01T10:47:11.983 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 0.00033610900000000002, 2023-10-01T10:47:11.983 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00061988799999999997, 2023-10-01T10:47:11.983 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:47:11.983 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "started", 2023-10-01T10:47:11.984 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:47:11.984 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:11.984 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:47:11.984 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:11.962661+0000", 2023-10-01T10:47:11.984 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:47:11.984 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:11.984 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:11.984 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:47:11.984 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:11.962661+0000", 2023-10-01T10:47:11.984 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:47:11.984 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:11.984 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:11.985 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:47:11.985 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:11.962660+0000", 2023-10-01T10:47:11.985 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:47:11.985 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:11.985 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:11.985 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:47:11.985 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:11.962848+0000", 2023-10-01T10:47:11.985 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000188204 2023-10-01T10:47:11.985 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:11.985 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:11.985 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:47:11.985 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:11.962860+0000", 2023-10-01T10:47:11.985 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1841999999999999e-05 2023-10-01T10:47:11.986 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:11.986 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:11.986 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:47:11.986 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:11.962867+0000", 2023-10-01T10:47:11.986 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.5939999999999996e-06 2023-10-01T10:47:11.986 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:11.986 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:11.986 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:47:11.986 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:11.962919+0000", 2023-10-01T10:47:11.986 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.1078000000000001e-05 2023-10-01T10:47:11.986 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:11.986 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:11.986 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:47:11.987 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:11.962944+0000", 2023-10-01T10:47:11.987 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.5758999999999999e-05 2023-10-01T10:47:11.987 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:47:11.987 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:47:11.987 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:47:11.987 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:47:11.987 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2023-10-01T10:47:11.987 INFO:teuthology.orchestra.run.smithi078.stdout: "num_ops": 5 2023-10-01T10:47:11.987 INFO:teuthology.orchestra.run.smithi078.stdout:} 2023-10-01T10:47:11.988 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:11.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:11.992 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:47:12.092 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:12.120 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:12.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:12.123 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:47:12.224 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:12.270 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:12.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:12.275 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:47:12.305 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 59 16 83058 83042 87.9468 56.6875 0.0154209 0.0113647 2023-10-01T10:47:12.376 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:12.409 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:12.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:12.414 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:47:12.514 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:12.538 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:12.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:12.542 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:47:12.642 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:12.684 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:12.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:12.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:47:12.792 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:12.815 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:12.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:12.819 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:47:12.920 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:12.950 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:12.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:12.954 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:47:13.054 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:13.085 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:13.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:13.085 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:47:13.186 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:13.229 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:13.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:13.235 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:47:13.305 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:2023-10-01T10:47:13.304470+0000 min lat: 0.00137887 max lat: 0.26908 avg lat: 0.0114201 2023-10-01T10:47:13.305 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T10:47:13.305 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 60 16 84058 84042 87.5226 62.5 0.0146127 0.0114201 2023-10-01T10:47:13.336 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:13.358 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:13.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:13.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:47:13.462 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:13.501 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:13.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:13.505 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:47:13.605 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:13.631 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:13.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:13.633 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:13.734 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:13.743 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:13 smithi078 bash[78274]: cluster 2023-10-01T10:47:11.853551+0000 mgr.y (mgr.25072) 887 : cluster 0 pgmap v816: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 54 MiB/s wr, 859 op/s 2023-10-01T10:47:13.743 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:13 smithi078 bash[75299]: cluster 2023-10-01T10:47:11.853551+0000 mgr.y (mgr.25072) 887 : cluster 0 pgmap v816: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 54 MiB/s wr, 859 op/s 2023-10-01T10:47:13.760 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:13.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:13.763 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:47:13.864 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:13.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:13 smithi173 bash[58816]: cluster 2023-10-01T10:47:11.853551+0000 mgr.y (mgr.25072) 887 : cluster 0 pgmap v816: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 54 MiB/s wr, 859 op/s 2023-10-01T10:47:13.893 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:13.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:13.897 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:47:13.997 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:14.023 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:14.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:14.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:47:14.128 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:14.161 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:14.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:14.166 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:47:14.266 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:14.300 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:14.305 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 61 16 85045 85029 87.099 61.6875 0.00979531 0.0114749 2023-10-01T10:47:14.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:14.306 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:47:14.407 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:14.443 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:14.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:14.447 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:14.547 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:14.575 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:14.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:14.578 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:47:14.636 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-01T10:47:14.681 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:14.707 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:14.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:14.716 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:47:14.817 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:14.845 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:14.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:14.849 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:47:14.950 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:14.986 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:14.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:14.990 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:47:15.091 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:15.121 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:15.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:15.124 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:15.224 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:15.256 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:15.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:15.260 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:47:15.305 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 62 16 85949 85933 86.6055 56.5 0.0284853 0.0115402 2023-10-01T10:47:15.361 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:15.386 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:15.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:15.391 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:47:15.492 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:15.501 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:15 smithi078 bash[78274]: cluster 2023-10-01T10:47:13.854162+0000 mgr.y (mgr.25072) 888 : cluster 0 pgmap v817: 89 pgs: 89 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2023-10-01T10:47:15.502 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:15 smithi078 bash[75299]: cluster 2023-10-01T10:47:13.854162+0000 mgr.y (mgr.25072) 888 : cluster 0 pgmap v817: 89 pgs: 89 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2023-10-01T10:47:15.525 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:15.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:15.529 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:47:15.630 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:15.658 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:15.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:15.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:47:15.764 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:15.796 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:15.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:15.801 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:47:15.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:15 smithi173 bash[58816]: cluster 2023-10-01T10:47:13.854162+0000 mgr.y (mgr.25072) 888 : cluster 0 pgmap v817: 89 pgs: 89 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2023-10-01T10:47:15.901 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:15.927 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:15.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:15.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:16.033 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:16.059 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:16.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:16.064 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:47:16.165 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:16.190 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:16.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:16.193 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:47:16.294 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:16.305 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 63 16 86922 86906 86.196 60.8125 0.0111264 0.0115958 2023-10-01T10:47:16.327 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:16.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:16.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:16.429 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:16.453 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:16.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:16.458 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:47:16.488 INFO:teuthology.orchestra.run.smithi078.stderr:nodeep-scrub is set 2023-10-01T10:47:16.559 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:16.591 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:16.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:16.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:47:16.697 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:16.707 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:16 smithi173 bash[58816]: audit 2023-10-01T10:47:15.567688+0000 mon.a (mon.0) 1113 : audit 1 from='client.? 172.21.15.78:0/328402523' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:47:16.731 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:16.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:16.735 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:16.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:16 smithi078 bash[78274]: audit 2023-10-01T10:47:15.567688+0000 mon.a (mon.0) 1113 : audit 1 from='client.? 172.21.15.78:0/328402523' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:47:16.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:16 smithi078 bash[75299]: audit 2023-10-01T10:47:15.567688+0000 mon.a (mon.0) 1113 : audit 1 from='client.? 172.21.15.78:0/328402523' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:47:16.835 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:16.875 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:16.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:16.879 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:47:16.979 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:17.016 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:17.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:17.018 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:47:17.069 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T10:47:17.069 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T10:47:17.069 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2023-10-01T10:47:17.069 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-01T10:47:17.069 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-01T10:47:17.070 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 config set filestore_inject_stall 3 2023-10-01T10:47:17.119 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:17.137 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:17.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:17.139 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:47:17.240 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:17.281 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:17.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:17.286 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:47:17.305 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 64 16 87811 87795 85.7174 55.5625 0.0102229 0.0116607 2023-10-01T10:47:17.388 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:17.420 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:17.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:17.423 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:47:17.524 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:17.549 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:17.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:17.551 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:47:17.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:17 smithi173 bash[58816]: cluster 2023-10-01T10:47:15.854692+0000 mgr.y (mgr.25072) 889 : cluster 0 pgmap v818: 89 pgs: 89 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 58 MiB/s wr, 924 op/s 2023-10-01T10:47:17.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:17 smithi173 bash[58816]: cluster 2023-10-01T10:47:16.457732+0000 mon.a (mon.0) 1114 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:47:17.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:17 smithi173 bash[58816]: cluster 2023-10-01T10:47:16.477487+0000 osd.7 (osd.7) 107 : cluster 3 failed to encode map e158 with expected crc 2023-10-01T10:47:17.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:17 smithi173 bash[58816]: audit 2023-10-01T10:47:16.478401+0000 mon.a (mon.0) 1115 : audit 1 from='client.? 172.21.15.78:0/328402523' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T10:47:17.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:17 smithi173 bash[58816]: cluster 2023-10-01T10:47:16.478538+0000 mon.a (mon.0) 1116 : cluster 0 osdmap e158: 8 total, 8 up, 8 in 2023-10-01T10:47:17.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:17 smithi173 bash[58816]: cluster 2023-10-01T10:47:16.484666+0000 osd.6 (osd.6) 70 : cluster 3 failed to encode map e158 with expected crc 2023-10-01T10:47:17.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:17 smithi173 bash[58816]: audit 2023-10-01T10:47:17.424678+0000 mon.a (mon.0) 1117 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:47:17.652 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:17.661 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:17 smithi078 bash[75299]: cluster 2023-10-01T10:47:15.854692+0000 mgr.y (mgr.25072) 889 : cluster 0 pgmap v818: 89 pgs: 89 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 58 MiB/s wr, 924 op/s 2023-10-01T10:47:17.661 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:17 smithi078 bash[75299]: cluster 2023-10-01T10:47:16.457732+0000 mon.a (mon.0) 1114 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:47:17.661 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:17 smithi078 bash[75299]: cluster 2023-10-01T10:47:16.477487+0000 osd.7 (osd.7) 107 : cluster 3 failed to encode map e158 with expected crc 2023-10-01T10:47:17.661 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:17 smithi078 bash[75299]: audit 2023-10-01T10:47:16.478401+0000 mon.a (mon.0) 1115 : audit 1 from='client.? 172.21.15.78:0/328402523' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T10:47:17.661 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:17 smithi078 bash[75299]: cluster 2023-10-01T10:47:16.478538+0000 mon.a (mon.0) 1116 : cluster 0 osdmap e158: 8 total, 8 up, 8 in 2023-10-01T10:47:17.661 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:17 smithi078 bash[75299]: cluster 2023-10-01T10:47:16.484666+0000 osd.6 (osd.6) 70 : cluster 3 failed to encode map e158 with expected crc 2023-10-01T10:47:17.661 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:17 smithi078 bash[75299]: audit 2023-10-01T10:47:17.424678+0000 mon.a (mon.0) 1117 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:47:17.662 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:17 smithi078 bash[78274]: cluster 2023-10-01T10:47:15.854692+0000 mgr.y (mgr.25072) 889 : cluster 0 pgmap v818: 89 pgs: 89 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 58 MiB/s wr, 924 op/s 2023-10-01T10:47:17.662 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:17 smithi078 bash[78274]: cluster 2023-10-01T10:47:16.457732+0000 mon.a (mon.0) 1114 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:47:17.662 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:17 smithi078 bash[78274]: cluster 2023-10-01T10:47:16.477487+0000 osd.7 (osd.7) 107 : cluster 3 failed to encode map e158 with expected crc 2023-10-01T10:47:17.662 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:17 smithi078 bash[78274]: audit 2023-10-01T10:47:16.478401+0000 mon.a (mon.0) 1115 : audit 1 from='client.? 172.21.15.78:0/328402523' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T10:47:17.662 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:17 smithi078 bash[78274]: cluster 2023-10-01T10:47:16.478538+0000 mon.a (mon.0) 1116 : cluster 0 osdmap e158: 8 total, 8 up, 8 in 2023-10-01T10:47:17.662 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:17 smithi078 bash[78274]: cluster 2023-10-01T10:47:16.484666+0000 osd.6 (osd.6) 70 : cluster 3 failed to encode map e158 with expected crc 2023-10-01T10:47:17.662 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:17 smithi078 bash[78274]: audit 2023-10-01T10:47:17.424678+0000 mon.a (mon.0) 1117 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:47:17.680 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:17.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:17.682 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:47:17.783 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:17.816 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:17.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:17.823 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:17.924 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:17.955 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:17.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:17.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:47:18.059 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:18.087 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:18.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:18.090 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:47:18.191 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:18.221 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:18.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:18.225 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:47:18.305 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 65 16 88790 88774 85.34 61.1875 0.0118136 0.0117119 2023-10-01T10:47:18.326 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:18.351 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:18.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:18.358 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:18.442 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:47:18.458 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:18.504 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:18.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:18.509 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:47:18.610 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:18.637 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:18.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:18.640 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:47:18.741 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:18.776 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:18.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:18.780 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:47:18.881 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:18.917 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:18.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:18.921 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:19.022 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:19.047 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:19.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:19.051 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:47:19.152 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:19.179 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:19.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:19.182 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:19.282 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:19.306 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 66 16 89734 89718 84.9405 59 0.0123634 0.0117662 2023-10-01T10:47:19.308 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:19.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:19.311 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:47:19.412 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:19.439 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:19.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:19.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:47:19.470 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:19 smithi173 bash[58816]: cluster 2023-10-01T10:47:17.855200+0000 mgr.y (mgr.25072) 890 : cluster 0 pgmap v820: 89 pgs: 89 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 61 MiB/s wr, 981 op/s 2023-10-01T10:47:19.543 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:19.553 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:19 smithi078 bash[78274]: cluster 2023-10-01T10:47:17.855200+0000 mgr.y (mgr.25072) 890 : cluster 0 pgmap v820: 89 pgs: 89 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 61 MiB/s wr, 981 op/s 2023-10-01T10:47:19.553 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:19 smithi078 bash[75299]: cluster 2023-10-01T10:47:17.855200+0000 mgr.y (mgr.25072) 890 : cluster 0 pgmap v820: 89 pgs: 89 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 61 MiB/s wr, 981 op/s 2023-10-01T10:47:19.577 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:19.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:19.581 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:47:19.681 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:19.708 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:19.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:19.712 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:47:19.813 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:19.844 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:19.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:19.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:47:19.949 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:19.972 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T10:47:19.976 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:19.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:19.979 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:20.079 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:20.130 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:20.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:20.133 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:47:20.234 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:20.276 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:20.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:20.279 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:47:20.307 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 67 16 90637 90621 84.5143 56.4375 0.0101407 0.0118266 2023-10-01T10:47:20.380 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:20.409 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:20.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:20.414 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:20.488 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:47:20.514 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:20.518 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:47:20 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:47:20] "GET /metrics HTTP/1.1" 200 36300 "" "Prometheus/2.33.4" 2023-10-01T10:47:20.543 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:20.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:20.549 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:20.650 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:20.671 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:20.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:20.673 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:47:20.774 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:20.801 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:20.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:20.804 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:47:20.904 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:20.944 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:20.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:20.947 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:47:21.048 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:21.102 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:21.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:21.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:47:21.207 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:21.236 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:21.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:21.239 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:47:21.307 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 68 16 91583 91567 84.1409 59.125 0.0120587 0.0118797 2023-10-01T10:47:21.340 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:21.369 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:21.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:21.372 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:47:21.473 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:21.477 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:21 smithi173 bash[58816]: cluster 2023-10-01T10:47:19.856091+0000 mgr.y (mgr.25072) 891 : cluster 0 pgmap v821: 89 pgs: 89 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 60 MiB/s wr, 956 op/s 2023-10-01T10:47:21.500 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:21.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:21.503 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:21.603 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:21.624 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:21 smithi078 bash[78274]: cluster 2023-10-01T10:47:19.856091+0000 mgr.y (mgr.25072) 891 : cluster 0 pgmap v821: 89 pgs: 89 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 60 MiB/s wr, 956 op/s 2023-10-01T10:47:21.624 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:21 smithi078 bash[75299]: cluster 2023-10-01T10:47:19.856091+0000 mgr.y (mgr.25072) 891 : cluster 0 pgmap v821: 89 pgs: 89 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 60 MiB/s wr, 956 op/s 2023-10-01T10:47:21.626 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:21.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:21.632 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:47:21.733 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:21.770 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:21.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:21.776 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:47:21.877 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:21.898 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T10:47:21.925 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:21.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:21.933 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:47:22.034 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:22.072 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:22.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:22.077 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:47:22.178 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:22.221 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:22.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:22.228 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:47:22.307 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 69 16 92478 92462 83.7322 55.9375 0.0198952 0.0119379 2023-10-01T10:47:22.328 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:22.365 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:22.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:22.368 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:47:22.469 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:22.493 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:22.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:22.495 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:47:22.596 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:22.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:22.642 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:22.643 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:47:22.743 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:22.773 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:22.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:22.777 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:22.878 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:22.910 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:22.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:22.912 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:47:23.013 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:23.050 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:23.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:23.053 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:47:23.153 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:23.184 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:23.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:23.186 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:47:23.287 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:23.307 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 70 16 93276 93260 83.2485 49.875 0.0256554 0.0120065 2023-10-01T10:47:23.327 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:23.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:23.333 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:47:23.433 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:23.457 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:23.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:23.460 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:47:23.482 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:23 smithi173 bash[58816]: cluster 2023-10-01T10:47:21.856670+0000 mgr.y (mgr.25072) 892 : cluster 0 pgmap v822: 89 pgs: 89 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 60 MiB/s wr, 954 op/s 2023-10-01T10:47:23.561 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:23.588 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:23.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:23.591 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:47:23.692 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:23.722 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:23.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:23.727 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:47:23.828 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:23.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:23 smithi078 bash[78274]: cluster 2023-10-01T10:47:21.856670+0000 mgr.y (mgr.25072) 892 : cluster 0 pgmap v822: 89 pgs: 89 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 60 MiB/s wr, 954 op/s 2023-10-01T10:47:23.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:23 smithi078 bash[75299]: cluster 2023-10-01T10:47:21.856670+0000 mgr.y (mgr.25072) 892 : cluster 0 pgmap v822: 89 pgs: 89 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 60 MiB/s wr, 954 op/s 2023-10-01T10:47:23.859 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:23.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:23.866 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:47:23.967 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:23.993 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:23.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:23.996 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:47:24.097 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:24.135 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:24.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:24.139 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:47:24.240 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:24.267 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:24.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:24.273 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:47:24.307 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 71 16 94150 94134 82.8453 54.625 0.0121089 0.0120649 2023-10-01T10:47:24.373 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:24.406 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:24.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:24.412 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:47:24.492 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:24 smithi078 bash[75299]: audit 2023-10-01T10:47:24.444648+0000 mon.a (mon.0) 1118 : audit 0 from='client.? 172.21.15.78:0/843738528' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:47:24.500 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T10:47:24.513 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:24.522 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:24 smithi173 bash[58816]: audit 2023-10-01T10:47:24.444648+0000 mon.a (mon.0) 1118 : audit 0 from='client.? 172.21.15.78:0/843738528' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:47:24.529 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T10:47:24.544 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:24.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:24.547 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:24.557 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:47:24.573 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T10:47:24.615 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T10:47:24.647 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:24.658 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T10:47:24.673 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:24.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:24.676 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:47:24.702 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T10:47:24.737 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T10:47:24.774 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T10:47:24.777 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:24.778 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T10:47:24.778 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T10:47:24.779 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T10:47:24.779 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T10:47:24.779 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T10:47:24.779 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T10:47:24.779 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T10:47:24.779 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T10:47:24.803 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:24.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:24.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:47:24.908 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:24.915 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:24 smithi078 bash[78274]: audit 2023-10-01T10:47:24.444648+0000 mon.a (mon.0) 1118 : audit 0 from='client.? 172.21.15.78:0/843738528' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:47:24.931 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:24.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:24.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:47:25.035 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:25.059 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:25.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:25.062 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:47:25.163 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:25.204 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:25.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:25.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:47:25.309 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 72 16 94972 94956 82.4082 51.375 0.0171623 0.0121289 2023-10-01T10:47:25.312 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:25.341 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:25.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:25.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:47:25.448 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:25.482 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:25.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:25.484 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:47:25.585 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:25.592 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:25 smithi173 bash[58816]: cluster 2023-10-01T10:47:23.857344+0000 mgr.y (mgr.25072) 893 : cluster 0 pgmap v823: 89 pgs: 89 active+clean; 5.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 58 MiB/s wr, 930 op/s 2023-10-01T10:47:25.605 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:25.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:25.609 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:47:25.710 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:25.740 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:25.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:25.746 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:47:25.779 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:25 smithi078 bash[78274]: cluster 2023-10-01T10:47:23.857344+0000 mgr.y (mgr.25072) 893 : cluster 0 pgmap v823: 89 pgs: 89 active+clean; 5.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 58 MiB/s wr, 930 op/s 2023-10-01T10:47:25.780 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:25 smithi078 bash[75299]: cluster 2023-10-01T10:47:23.857344+0000 mgr.y (mgr.25072) 893 : cluster 0 pgmap v823: 89 pgs: 89 active+clean; 5.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 58 MiB/s wr, 930 op/s 2023-10-01T10:47:25.847 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:25.877 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:25.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:25.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:47:25.984 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:26.014 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:26.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:26.020 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:47:26.120 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:26.140 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:26.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:26.142 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:47:26.243 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:26.271 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:26.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:26.274 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:47:26.307 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 73 16 95842 95826 82.0242 54.375 0.00637532 0.0121863 2023-10-01T10:47:26.374 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:26.402 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:26.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:26.406 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:47:26.506 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:26.538 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:26.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:26.540 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:47:26.640 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:26.672 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:26.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:26.677 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:47:26.777 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:26.813 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:26.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:26.815 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:47:26.916 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:26.940 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:26.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:26.944 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:47:27.044 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:27.073 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:27.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:27.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:47:27.177 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:27.220 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:27.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:27.222 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:47:27.307 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 74 16 96728 96712 81.664 55.375 0.0119809 0.0122397 2023-10-01T10:47:27.323 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:27.331 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-10-01T10:47:27.331 INFO:teuthology.orchestra.run.smithi173.stdout: "success": "" 2023-10-01T10:47:27.331 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-10-01T10:47:27.358 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:27.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:27.366 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:47:27.467 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:27.509 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:27.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:27.515 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:27.615 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:27.649 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:27.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:27.651 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:47:27.752 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:27.760 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:27 smithi078 bash[78274]: cluster 2023-10-01T10:47:25.857908+0000 mgr.y (mgr.25072) 894 : cluster 0 pgmap v824: 89 pgs: 89 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 58 MiB/s wr, 924 op/s 2023-10-01T10:47:27.760 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:27 smithi078 bash[78274]: audit 2023-10-01T10:47:27.160161+0000 mon.a (mon.0) 1119 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:47:27.760 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:27 smithi078 bash[75299]: cluster 2023-10-01T10:47:25.857908+0000 mgr.y (mgr.25072) 894 : cluster 0 pgmap v824: 89 pgs: 89 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 58 MiB/s wr, 924 op/s 2023-10-01T10:47:27.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:27 smithi078 bash[75299]: audit 2023-10-01T10:47:27.160161+0000 mon.a (mon.0) 1119 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:47:27.772 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:27.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:27.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:47:27.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:27 smithi173 bash[58816]: cluster 2023-10-01T10:47:25.857908+0000 mgr.y (mgr.25072) 894 : cluster 0 pgmap v824: 89 pgs: 89 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 58 MiB/s wr, 924 op/s 2023-10-01T10:47:27.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:27 smithi173 bash[58816]: audit 2023-10-01T10:47:27.160161+0000 mon.a (mon.0) 1119 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:47:27.875 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:27.905 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:27.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:27.911 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:47:28.011 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:28.037 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:28.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:28.040 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:47:28.141 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:28.170 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:28.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:28.178 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:47:28.279 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:28.307 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 75 16 97575 97559 81.281 52.9375 0.0165924 0.0122973 2023-10-01T10:47:28.321 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:28.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:28.325 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:47:28.426 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:28.460 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:28.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:28.464 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:47:28.565 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:28.611 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:28.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:28.616 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:47:28.689 INFO:teuthology.orchestra.run.smithi078.stdout:ERROR: (22) Invalid argument 2023-10-01T10:47:28.689 INFO:teuthology.orchestra.run.smithi078.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T10:47:28.717 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:28.755 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:28.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:28.758 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:28.858 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:28.885 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:28.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:28.888 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:47:28.988 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:29.012 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:29.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:29.015 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:47:29.116 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:29.149 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:29.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:29.152 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:47:29.253 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:29.294 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:29.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:29.297 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:47:29.307 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 76 16 98421 98405 80.9072 52.875 0.0230684 0.0123541 2023-10-01T10:47:29.398 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:29.422 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:29.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:29.426 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:29.527 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:29.536 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:29 smithi078 bash[75299]: cluster 2023-10-01T10:47:27.860296+0000 mgr.y (mgr.25072) 895 : cluster 0 pgmap v825: 89 pgs: 89 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 806 op/s 2023-10-01T10:47:29.536 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:29 smithi078 bash[78274]: cluster 2023-10-01T10:47:27.860296+0000 mgr.y (mgr.25072) 895 : cluster 0 pgmap v825: 89 pgs: 89 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 806 op/s 2023-10-01T10:47:29.556 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:29.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:29.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:47:29.661 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:29.670 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:29 smithi173 bash[58816]: cluster 2023-10-01T10:47:27.860296+0000 mgr.y (mgr.25072) 895 : cluster 0 pgmap v825: 89 pgs: 89 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 806 op/s 2023-10-01T10:47:29.686 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:29.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:29.690 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:29.791 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:29.816 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:29.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:29.818 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:29.919 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:29.940 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T10:47:29.946 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:29.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:29.953 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:47:30.054 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:30.082 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:30.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:30.086 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:47:30.187 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:30.217 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:30.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:30.220 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:47:30.307 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 77 16 99297 99281 80.5675 54.75 0.0265358 0.0124062 2023-10-01T10:47:30.321 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:30.353 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:30.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:30.358 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:47:30.459 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:30.468 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:47:30 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:47:30] "GET /metrics HTTP/1.1" 200 36300 "" "Prometheus/2.33.4" 2023-10-01T10:47:30.486 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:30.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:30.492 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:47:30.593 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:30.623 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:30.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:30.626 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:47:30.727 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:30.757 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:30.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:30.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:47:30.862 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:30.892 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:30.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:30.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:30.998 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:31.022 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:31.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:31.025 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:47:31.126 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:31.158 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:31.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:31.164 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:47:31.177 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-01T10:47:31.265 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:31.296 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:31.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:31.299 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:47:31.307 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 78 16 100177 100161 80.2397 55 0.0097902 0.012457 2023-10-01T10:47:31.400 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:31.433 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:31.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:31.436 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:47:31.537 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:31.546 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:31 smithi173 bash[58816]: cluster 2023-10-01T10:47:29.861007+0000 mgr.y (mgr.25072) 896 : cluster 0 pgmap v826: 89 pgs: 89 active+clean; 6.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-01T10:47:31.564 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:31.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:31.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:47:31.667 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:31.699 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:31.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:31.702 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:47:31.802 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:31.813 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:31 smithi078 bash[78274]: cluster 2023-10-01T10:47:29.861007+0000 mgr.y (mgr.25072) 896 : cluster 0 pgmap v826: 89 pgs: 89 active+clean; 6.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-01T10:47:31.813 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:31 smithi078 bash[75299]: cluster 2023-10-01T10:47:29.861007+0000 mgr.y (mgr.25072) 896 : cluster 0 pgmap v826: 89 pgs: 89 active+clean; 6.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-01T10:47:31.832 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:31.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:31.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:47:31.935 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:31.969 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:31.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:31.974 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:47:32.075 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:32.111 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:32.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:32.115 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:47:32.216 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:32.251 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:32.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:32.255 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:47:32.308 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 79 16 101016 101000 79.8877 52.4375 0.0248439 0.012511 2023-10-01T10:47:32.356 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:32.383 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:32.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:32.387 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:47:32.487 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:32.513 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:32.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:32.516 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:47:32.616 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:32.644 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:32.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:32.647 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:47:32.747 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:32.789 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:32.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:32.795 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:47:32.896 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:32.924 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:32.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:32.928 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:47:33.029 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:33.056 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:33.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:33.062 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:47:33.163 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:33.214 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:33.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:33.218 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:33.308 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:2023-10-01T10:47:33.307210+0000 min lat: 0.00137887 max lat: 0.26908 avg lat: 0.0125728 2023-10-01T10:47:33.308 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T10:47:33.308 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 80 16 101799 101783 79.5008 48.9375 0.0333766 0.0125728 2023-10-01T10:47:33.319 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:33.341 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:33.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:33.344 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:47:33.402 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T10:47:33.402 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T10:47:33.402 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-01T10:47:33.402 INFO:tasks.thrashosds.thrasher:Setting osd 6 primary_affinity to 0.455630 2023-10-01T10:47:33.403 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 6 0.4556301060760598 2023-10-01T10:47:33.428 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:33 smithi173 bash[58816]: cluster 2023-10-01T10:47:31.861618+0000 mgr.y (mgr.25072) 897 : cluster 0 pgmap v827: 89 pgs: 89 active+clean; 6.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 47 MiB/s wr, 756 op/s 2023-10-01T10:47:33.428 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:33 smithi173 bash[58816]: audit 2023-10-01T10:47:32.426158+0000 mon.a (mon.0) 1120 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:47:33.445 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:33.481 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:33.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:33.509 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:47:33.532 INFO:teuthology.orchestra.run.smithi078.stderr:noscrub is unset 2023-10-01T10:47:33.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:33 smithi078 bash[78274]: cluster 2023-10-01T10:47:31.861618+0000 mgr.y (mgr.25072) 897 : cluster 0 pgmap v827: 89 pgs: 89 active+clean; 6.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 47 MiB/s wr, 756 op/s 2023-10-01T10:47:33.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:33 smithi078 bash[78274]: audit 2023-10-01T10:47:32.426158+0000 mon.a (mon.0) 1120 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:47:33.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:33 smithi078 bash[78274]: audit 2023-10-01T10:47:32.871516+0000 mon.c (mon.2) 156 : audit 1 from='client.? 172.21.15.78:0/2871069162' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T10:47:33.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:33 smithi078 bash[78274]: audit 2023-10-01T10:47:32.872353+0000 mon.a (mon.0) 1121 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T10:47:33.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:33 smithi078 bash[75299]: cluster 2023-10-01T10:47:31.861618+0000 mgr.y (mgr.25072) 897 : cluster 0 pgmap v827: 89 pgs: 89 active+clean; 6.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 47 MiB/s wr, 756 op/s 2023-10-01T10:47:33.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:33 smithi078 bash[75299]: audit 2023-10-01T10:47:32.426158+0000 mon.a (mon.0) 1120 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:47:33.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:33 smithi078 bash[75299]: audit 2023-10-01T10:47:32.871516+0000 mon.c (mon.2) 156 : audit 1 from='client.? 172.21.15.78:0/2871069162' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T10:47:33.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:33 smithi078 bash[75299]: audit 2023-10-01T10:47:32.872353+0000 mon.a (mon.0) 1121 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T10:47:33.610 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:33.637 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:33.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:33.641 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:47:33.741 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:33.770 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:33.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:33.778 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:47:33.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:33 smithi173 bash[58816]: audit 2023-10-01T10:47:32.871516+0000 mon.c (mon.2) 156 : audit 1 from='client.? 172.21.15.78:0/2871069162' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T10:47:33.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:33 smithi173 bash[58816]: audit 2023-10-01T10:47:32.872353+0000 mon.a (mon.0) 1121 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T10:47:33.879 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:33.916 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:33.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:33.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:47:34.024 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:34.058 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:34.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:34.066 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:47:34.166 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:34.193 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:34.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:34.199 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:47:34.299 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:34.308 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 81 16 102555 102539 79.1027 47.25 0.0139658 0.0126355 2023-10-01T10:47:34.338 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:34.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:34.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:34.428 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:34 smithi078 bash[78274]: cluster 2023-10-01T10:47:33.507730+0000 mon.a (mon.0) 1122 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:47:34.428 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:34 smithi078 bash[78274]: audit 2023-10-01T10:47:33.513477+0000 mon.a (mon.0) 1123 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T10:47:34.428 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:34 smithi078 bash[78274]: cluster 2023-10-01T10:47:33.513560+0000 mon.a (mon.0) 1124 : cluster 0 osdmap e159: 8 total, 8 up, 8 in 2023-10-01T10:47:34.428 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:34 smithi078 bash[78274]: cluster 2023-10-01T10:47:33.515825+0000 osd.5 (osd.5) 90 : cluster 3 failed to encode map e159 with expected crc 2023-10-01T10:47:34.428 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:34 smithi078 bash[78274]: cluster 2023-10-01T10:47:33.516584+0000 osd.7 (osd.7) 108 : cluster 3 failed to encode map e159 with expected crc 2023-10-01T10:47:34.428 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:34 smithi078 bash[78274]: cluster 2023-10-01T10:47:33.522452+0000 osd.6 (osd.6) 71 : cluster 3 failed to encode map e159 with expected crc 2023-10-01T10:47:34.428 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:34 smithi078 bash[78274]: cluster 2023-10-01T10:47:33.524227+0000 osd.6 (osd.6) 72 : cluster 3 failed to encode map e159 with expected crc 2023-10-01T10:47:34.429 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:34 smithi078 bash[78274]: cluster 2023-10-01T10:47:33.530248+0000 osd.6 (osd.6) 73 : cluster 3 failed to encode map e159 with expected crc 2023-10-01T10:47:34.429 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:34 smithi078 bash[78274]: cluster 2023-10-01T10:47:33.532029+0000 osd.6 (osd.6) 74 : cluster 3 failed to encode map e159 with expected crc 2023-10-01T10:47:34.429 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:34 smithi078 bash[78274]: audit 2023-10-01T10:47:33.880898+0000 mon.a (mon.0) 1125 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:47:34.429 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:34 smithi078 bash[78274]: audit 2023-10-01T10:47:33.892535+0000 mon.a (mon.0) 1126 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:47:34.429 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:34 smithi078 bash[75299]: cluster 2023-10-01T10:47:33.507730+0000 mon.a (mon.0) 1122 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:47:34.429 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:34 smithi078 bash[75299]: audit 2023-10-01T10:47:33.513477+0000 mon.a (mon.0) 1123 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T10:47:34.429 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:34 smithi078 bash[75299]: cluster 2023-10-01T10:47:33.513560+0000 mon.a (mon.0) 1124 : cluster 0 osdmap e159: 8 total, 8 up, 8 in 2023-10-01T10:47:34.429 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:34 smithi078 bash[75299]: cluster 2023-10-01T10:47:33.515825+0000 osd.5 (osd.5) 90 : cluster 3 failed to encode map e159 with expected crc 2023-10-01T10:47:34.429 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:34 smithi078 bash[75299]: cluster 2023-10-01T10:47:33.516584+0000 osd.7 (osd.7) 108 : cluster 3 failed to encode map e159 with expected crc 2023-10-01T10:47:34.429 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:34 smithi078 bash[75299]: cluster 2023-10-01T10:47:33.522452+0000 osd.6 (osd.6) 71 : cluster 3 failed to encode map e159 with expected crc 2023-10-01T10:47:34.430 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:34 smithi078 bash[75299]: cluster 2023-10-01T10:47:33.524227+0000 osd.6 (osd.6) 72 : cluster 3 failed to encode map e159 with expected crc 2023-10-01T10:47:34.430 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:34 smithi078 bash[75299]: cluster 2023-10-01T10:47:33.530248+0000 osd.6 (osd.6) 73 : cluster 3 failed to encode map e159 with expected crc 2023-10-01T10:47:34.430 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:34 smithi078 bash[75299]: cluster 2023-10-01T10:47:33.532029+0000 osd.6 (osd.6) 74 : cluster 3 failed to encode map e159 with expected crc 2023-10-01T10:47:34.430 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:34 smithi078 bash[75299]: audit 2023-10-01T10:47:33.880898+0000 mon.a (mon.0) 1125 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:47:34.430 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:34 smithi078 bash[75299]: audit 2023-10-01T10:47:33.892535+0000 mon.a (mon.0) 1126 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:47:34.445 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:34.475 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:34.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:34.484 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:47:34.585 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:34.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:34 smithi173 bash[58816]: cluster 2023-10-01T10:47:33.507730+0000 mon.a (mon.0) 1122 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:47:34.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:34 smithi173 bash[58816]: audit 2023-10-01T10:47:33.513477+0000 mon.a (mon.0) 1123 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T10:47:34.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:34 smithi173 bash[58816]: cluster 2023-10-01T10:47:33.513560+0000 mon.a (mon.0) 1124 : cluster 0 osdmap e159: 8 total, 8 up, 8 in 2023-10-01T10:47:34.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:34 smithi173 bash[58816]: cluster 2023-10-01T10:47:33.515825+0000 osd.5 (osd.5) 90 : cluster 3 failed to encode map e159 with expected crc 2023-10-01T10:47:34.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:34 smithi173 bash[58816]: cluster 2023-10-01T10:47:33.516584+0000 osd.7 (osd.7) 108 : cluster 3 failed to encode map e159 with expected crc 2023-10-01T10:47:34.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:34 smithi173 bash[58816]: cluster 2023-10-01T10:47:33.522452+0000 osd.6 (osd.6) 71 : cluster 3 failed to encode map e159 with expected crc 2023-10-01T10:47:34.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:34 smithi173 bash[58816]: cluster 2023-10-01T10:47:33.524227+0000 osd.6 (osd.6) 72 : cluster 3 failed to encode map e159 with expected crc 2023-10-01T10:47:34.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:34 smithi173 bash[58816]: cluster 2023-10-01T10:47:33.530248+0000 osd.6 (osd.6) 73 : cluster 3 failed to encode map e159 with expected crc 2023-10-01T10:47:34.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:34 smithi173 bash[58816]: cluster 2023-10-01T10:47:33.532029+0000 osd.6 (osd.6) 74 : cluster 3 failed to encode map e159 with expected crc 2023-10-01T10:47:34.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:34 smithi173 bash[58816]: audit 2023-10-01T10:47:33.880898+0000 mon.a (mon.0) 1125 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:47:34.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:34 smithi173 bash[58816]: audit 2023-10-01T10:47:33.892535+0000 mon.a (mon.0) 1126 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:47:34.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:34.628 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:34.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:47:34.729 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:34.756 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:34.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:34.758 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:47:34.859 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:34.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:34.890 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:34.890 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:47:34.991 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:35.020 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:35.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:35.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:35.126 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:35.167 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:35.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:35.171 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:47:35.271 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:35.302 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:35.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:35.308 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:47:35.308 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 82 16 103414 103398 78.7927 53.6875 0.00997699 0.0126858 2023-10-01T10:47:35.409 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:35.449 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:35.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:35.452 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:47:35.552 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:35.564 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:35 smithi078 bash[78274]: cluster 2023-10-01T10:47:33.862287+0000 mgr.y (mgr.25072) 898 : cluster 0 pgmap v829: 89 pgs: 89 active+clean; 6.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 54 MiB/s wr, 863 op/s 2023-10-01T10:47:35.566 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:35 smithi078 bash[75299]: cluster 2023-10-01T10:47:33.862287+0000 mgr.y (mgr.25072) 898 : cluster 0 pgmap v829: 89 pgs: 89 active+clean; 6.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 54 MiB/s wr, 863 op/s 2023-10-01T10:47:35.582 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:35.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:35.585 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:47:35.686 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:35.696 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:35 smithi173 bash[58816]: cluster 2023-10-01T10:47:33.862287+0000 mgr.y (mgr.25072) 898 : cluster 0 pgmap v829: 89 pgs: 89 active+clean; 6.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 54 MiB/s wr, 863 op/s 2023-10-01T10:47:35.719 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:35.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:35.726 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:35.827 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:35.856 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:35.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:35.860 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:47:35.961 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:35.992 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:35.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:35.999 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:36.100 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:36.126 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:36.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:36.137 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:47:36.237 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:36.298 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:36.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:36.304 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:47:36.308 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 83 16 104206 104190 78.4394 49.5 0.0240293 0.0127426 2023-10-01T10:47:36.405 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:36.445 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:36.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:36.450 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:47:36.551 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:36.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:36.579 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:36.579 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:47:36.681 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:36.708 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:36.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:36.714 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:36.814 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:36.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:36.862 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:36.863 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:47:36.963 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:36.998 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:37.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:37.000 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:47:37.051 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T10:47:37.101 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:37.133 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:37.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:37.136 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:47:37.237 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:37.272 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:37.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:37.278 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:47:37.308 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 84 16 104961 104945 78.0674 47.1875 0.0113912 0.0128027 2023-10-01T10:47:37.379 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:37.408 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:37.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:37.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:47:37.516 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:37.527 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:37 smithi078 bash[75299]: cluster 2023-10-01T10:47:35.865557+0000 mgr.y (mgr.25072) 899 : cluster 0 pgmap v830: 89 pgs: 89 active+clean; 6.2 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 53 MiB/s wr, 847 op/s 2023-10-01T10:47:37.545 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:37.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:37.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:47:37.575 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:47:37.580 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:47:37.651 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:37.681 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:37.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:37.684 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:47:37.785 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:37.794 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:37 smithi078 bash[75299]: audit 2023-10-01T10:47:36.981065+0000 mon.a (mon.0) 1127 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:47:37.794 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:37 smithi078 bash[75299]: audit 2023-10-01T10:47:36.993581+0000 mon.a (mon.0) 1128 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:47:37.794 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:37 smithi078 bash[78274]: cluster 2023-10-01T10:47:35.865557+0000 mgr.y (mgr.25072) 899 : cluster 0 pgmap v830: 89 pgs: 89 active+clean; 6.2 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 53 MiB/s wr, 847 op/s 2023-10-01T10:47:37.794 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:37 smithi078 bash[78274]: audit 2023-10-01T10:47:36.981065+0000 mon.a (mon.0) 1127 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:47:37.795 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:37 smithi078 bash[78274]: audit 2023-10-01T10:47:36.993581+0000 mon.a (mon.0) 1128 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:47:37.816 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:37.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:37.819 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:47:37.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:37 smithi173 bash[58816]: cluster 2023-10-01T10:47:35.865557+0000 mgr.y (mgr.25072) 899 : cluster 0 pgmap v830: 89 pgs: 89 active+clean; 6.2 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 53 MiB/s wr, 847 op/s 2023-10-01T10:47:37.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:37 smithi173 bash[58816]: audit 2023-10-01T10:47:36.981065+0000 mon.a (mon.0) 1127 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:47:37.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:37 smithi173 bash[58816]: audit 2023-10-01T10:47:36.993581+0000 mon.a (mon.0) 1128 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:47:37.920 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:37.946 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:37.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:37.949 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:47:38.050 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:38.082 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:38.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:38.088 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:47:38.188 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:38.223 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:38.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:38.230 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:47:38.309 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 85 16 105786 105770 77.7555 51.5625 0.0205906 0.0128554 2023-10-01T10:47:38.331 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:38.363 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:38.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:38.371 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:47:38.471 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:38.501 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:38.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:38.508 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:47:38.596 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:47:38.608 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:38.634 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:38.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:38.638 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:47:38.738 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:38.748 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:38 smithi173 bash[58816]: audit 2023-10-01T10:47:37.678679+0000 mon.a (mon.0) 1129 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:47:38.748 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:38 smithi173 bash[58816]: audit 2023-10-01T10:47:37.679975+0000 mon.a (mon.0) 1130 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:47:38.748 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:38 smithi173 bash[58816]: audit 2023-10-01T10:47:37.686497+0000 mon.a (mon.0) 1131 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:47:38.769 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:38.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:38.775 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:47:38.876 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:38.921 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:38.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:38.978 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:47:38.978 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:38 smithi078 bash[75299]: audit 2023-10-01T10:47:37.678679+0000 mon.a (mon.0) 1129 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:47:38.979 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:38 smithi078 bash[75299]: audit 2023-10-01T10:47:37.679975+0000 mon.a (mon.0) 1130 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:47:38.979 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:38 smithi078 bash[75299]: audit 2023-10-01T10:47:37.686497+0000 mon.a (mon.0) 1131 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:47:38.979 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:38 smithi078 bash[78274]: audit 2023-10-01T10:47:37.678679+0000 mon.a (mon.0) 1129 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:47:38.979 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:38 smithi078 bash[78274]: audit 2023-10-01T10:47:37.679975+0000 mon.a (mon.0) 1130 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:47:38.979 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:38 smithi078 bash[78274]: audit 2023-10-01T10:47:37.686497+0000 mon.a (mon.0) 1131 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:47:39.079 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:39.109 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:39.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:39.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:47:39.213 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:39.262 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:39.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:39.269 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:47:39.309 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 86 16 106660 106644 77.4866 54.625 0.0199495 0.0128998 2023-10-01T10:47:39.369 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:39.412 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:39.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:39.414 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:47:39.515 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:39.545 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:39.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:39.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:47:39.652 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:39.681 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:39.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:39.689 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:47:39.690 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:39 smithi173 bash[58816]: cluster 2023-10-01T10:47:37.867357+0000 mgr.y (mgr.25072) 900 : cluster 0 pgmap v831: 89 pgs: 89 active+clean; 6.2 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 53 MiB/s wr, 846 op/s 2023-10-01T10:47:39.790 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:39.824 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:39.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:39.828 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:39.929 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:39.940 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:39 smithi078 bash[78274]: cluster 2023-10-01T10:47:37.867357+0000 mgr.y (mgr.25072) 900 : cluster 0 pgmap v831: 89 pgs: 89 active+clean; 6.2 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 53 MiB/s wr, 846 op/s 2023-10-01T10:47:39.941 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:39 smithi078 bash[75299]: cluster 2023-10-01T10:47:37.867357+0000 mgr.y (mgr.25072) 900 : cluster 0 pgmap v831: 89 pgs: 89 active+clean; 6.2 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 53 MiB/s wr, 846 op/s 2023-10-01T10:47:39.960 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:39.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:39.965 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:47:40.066 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:40.101 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:40.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:40.105 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:47:40.206 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:40.233 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:40.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:40.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:47:40.309 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 87 16 107490 107474 77.1922 51.875 0.0174578 0.0129489 2023-10-01T10:47:40.337 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:40.384 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:40.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:40.386 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:47:40.487 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:40.516 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:40.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:40.519 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:47:40.582 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:47:40 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:47:40] "GET /metrics HTTP/1.1" 200 36310 "" "Prometheus/2.33.4" 2023-10-01T10:47:40.619 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:40.648 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:40.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:40.653 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:47:40.754 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:40.782 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:40.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:40.789 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:47:40.890 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:40.919 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:40.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:40.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:47:41.024 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:41.048 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:41.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:41.053 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:47:41.154 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:41.182 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:41.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:41.185 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:47:41.286 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:41.313 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 88 16 108332 108316 76.909 52.625 0.0270107 0.0129967 2023-10-01T10:47:41.332 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:41.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:41.336 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:47:41.437 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:41.468 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:41.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:41.472 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:47:41.573 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:41.603 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:41.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:41.607 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:41.651 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:47:41.709 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:41.737 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:41.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:41.741 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:47:41.842 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:41.864 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:41.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:41.869 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:47:41.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:41 smithi173 bash[58816]: cluster 2023-10-01T10:47:39.868077+0000 mgr.y (mgr.25072) 901 : cluster 0 pgmap v832: 89 pgs: 89 active+clean; 6.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 51 MiB/s wr, 820 op/s 2023-10-01T10:47:41.970 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:41.992 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:41.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:41.998 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:47:42.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:41 smithi078 bash[75299]: cluster 2023-10-01T10:47:39.868077+0000 mgr.y (mgr.25072) 901 : cluster 0 pgmap v832: 89 pgs: 89 active+clean; 6.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 51 MiB/s wr, 820 op/s 2023-10-01T10:47:42.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:41 smithi078 bash[78274]: cluster 2023-10-01T10:47:39.868077+0000 mgr.y (mgr.25072) 901 : cluster 0 pgmap v832: 89 pgs: 89 active+clean; 6.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 51 MiB/s wr, 820 op/s 2023-10-01T10:47:42.098 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:42.128 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:42.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:42.133 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:47:42.234 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:42.278 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:42.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:42.281 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:47:42.313 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 89 16 109153 109137 76.6214 51.3125 0.0215363 0.0130448 2023-10-01T10:47:42.382 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:42.408 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:42.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:42.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:47:42.516 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:42.549 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:42.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:42.555 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:42.655 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:42.680 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:42.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:42.685 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:47:42.786 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:42.819 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:42.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:42.823 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:42.923 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:42.959 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:42.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:42.963 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:47:43.064 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:43.087 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:43.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:43.092 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:47:43.192 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:43.219 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:43.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:43.222 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:47:43.314 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Total time run: 90.0081 2023-10-01T10:47:43.314 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Total writes made: 109910 2023-10-01T10:47:43.314 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Write size: 65536 2023-10-01T10:47:43.314 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Object size: 65536 2023-10-01T10:47:43.314 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Bandwidth (MB/sec): 76.3195 2023-10-01T10:47:43.314 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Stddev Bandwidth: 28.6254 2023-10-01T10:47:43.314 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Max bandwidth (MB/sec): 188.875 2023-10-01T10:47:43.315 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Min bandwidth (MB/sec): 47.1875 2023-10-01T10:47:43.315 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Average IOPS: 1221 2023-10-01T10:47:43.315 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Stddev IOPS: 458.006 2023-10-01T10:47:43.315 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Max IOPS: 3022 2023-10-01T10:47:43.315 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Min IOPS: 755 2023-10-01T10:47:43.315 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Average Latency(s): 0.0130982 2023-10-01T10:47:43.315 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Stddev Latency(s): 0.0103828 2023-10-01T10:47:43.315 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Max latency(s): 0.26908 2023-10-01T10:47:43.315 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Min latency(s): 0.00137887 2023-10-01T10:47:43.322 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Cleaning up (deleting benchmark objects) 2023-10-01T10:47:43.323 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:43.347 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:43.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:43.353 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:47:43.454 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:43.482 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:43.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:43.489 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:47:43.590 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:43.616 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:43.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:43.618 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:47:43.719 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:43.728 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:43 smithi173 bash[58816]: cluster 2023-10-01T10:47:41.868641+0000 mgr.y (mgr.25072) 902 : cluster 0 pgmap v833: 89 pgs: 89 active+clean; 6.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 51 MiB/s wr, 821 op/s 2023-10-01T10:47:43.755 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:43.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:43.759 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:47:43.859 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:43.889 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:43.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:43.895 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:47:43.995 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:44.013 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:44.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:44.015 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:47:44.037 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:43 smithi078 bash[78274]: cluster 2023-10-01T10:47:41.868641+0000 mgr.y (mgr.25072) 902 : cluster 0 pgmap v833: 89 pgs: 89 active+clean; 6.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 51 MiB/s wr, 821 op/s 2023-10-01T10:47:44.037 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:43 smithi078 bash[75299]: cluster 2023-10-01T10:47:41.868641+0000 mgr.y (mgr.25072) 902 : cluster 0 pgmap v833: 89 pgs: 89 active+clean; 6.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 51 MiB/s wr, 821 op/s 2023-10-01T10:47:44.116 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:44.139 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:44.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:44.145 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:47:44.245 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:44.281 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:44.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:44.287 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:47:44.388 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:44.412 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:44.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:44.415 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:47:44.516 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:44.545 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:44.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:44.551 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:47:44.651 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:44.679 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:44.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:44.681 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:44.783 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:44.811 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:44.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:44.814 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:47:44.915 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:44.941 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:44.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:44.946 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:47:45.047 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:45.073 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:45.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:45.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:47:45.176 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:45.208 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:45.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:45.219 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:47:45.319 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:45.348 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:45.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:45.351 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:47:45.452 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:45.483 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:45.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:45.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:45.588 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:45.613 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:45.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:45.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:47:45.716 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:45.741 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:45.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:45.742 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:47:45.843 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:45.851 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:45 smithi078 bash[75299]: cluster 2023-10-01T10:47:43.869139+0000 mgr.y (mgr.25072) 903 : cluster 0 pgmap v834: 89 pgs: 89 active+clean; 6.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 98 B/s rd, 53 MiB/s wr, 859 op/s 2023-10-01T10:47:45.852 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:45 smithi078 bash[78274]: cluster 2023-10-01T10:47:43.869139+0000 mgr.y (mgr.25072) 903 : cluster 0 pgmap v834: 89 pgs: 89 active+clean; 6.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 98 B/s rd, 53 MiB/s wr, 859 op/s 2023-10-01T10:47:45.867 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:45.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:45.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:45 smithi173 bash[58816]: cluster 2023-10-01T10:47:43.869139+0000 mgr.y (mgr.25072) 903 : cluster 0 pgmap v834: 89 pgs: 89 active+clean; 6.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 98 B/s rd, 53 MiB/s wr, 859 op/s 2023-10-01T10:47:45.870 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:47:45.971 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:46.006 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:46.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:46.010 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:47:46.110 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:46.132 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:46.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:46.134 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:47:46.235 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:46.260 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:46.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:46.262 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:47:46.363 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:46.387 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:46.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:46.394 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:47:46.494 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:46.520 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:46.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:46.522 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:47:46.623 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:46.680 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:46.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:46.682 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:47:46.783 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:46.820 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:46.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:46.829 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:46.930 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:46.958 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:46.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:46.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:47.063 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:47.099 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:47.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:47.105 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:47.206 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:47.229 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:47.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:47.231 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:47:47.332 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:47.365 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:47.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:47.367 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:47:47.468 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:47.505 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:47.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:47.509 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:47:47.610 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:47.638 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:47.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:47.641 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:47:47.741 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:47.751 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:47 smithi173 bash[58816]: cluster 2023-10-01T10:47:45.869688+0000 mgr.y (mgr.25072) 904 : cluster 0 pgmap v835: 89 pgs: 89 active+clean; 6.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 46 MiB/s wr, 761 op/s 2023-10-01T10:47:47.751 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:47 smithi173 bash[58816]: audit 2023-10-01T10:47:47.426388+0000 mon.a (mon.0) 1132 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:47:47.768 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:47.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:47.772 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:47.872 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:47.898 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:47.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:47.901 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:47:48.001 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:48.009 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:47 smithi078 bash[75299]: cluster 2023-10-01T10:47:45.869688+0000 mgr.y (mgr.25072) 904 : cluster 0 pgmap v835: 89 pgs: 89 active+clean; 6.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 46 MiB/s wr, 761 op/s 2023-10-01T10:47:48.010 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:47 smithi078 bash[75299]: audit 2023-10-01T10:47:47.426388+0000 mon.a (mon.0) 1132 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:47:48.010 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:47 smithi078 bash[78274]: cluster 2023-10-01T10:47:45.869688+0000 mgr.y (mgr.25072) 904 : cluster 0 pgmap v835: 89 pgs: 89 active+clean; 6.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 46 MiB/s wr, 761 op/s 2023-10-01T10:47:48.010 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:47 smithi078 bash[78274]: audit 2023-10-01T10:47:47.426388+0000 mon.a (mon.0) 1132 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:47:48.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:48.031 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:48.032 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:47:48.132 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:48.163 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:48.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:48.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:47:48.267 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:48.308 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:48.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:48.311 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:47:48.412 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:48.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:48.445 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:48.445 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:48.546 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:48.569 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:48.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:48.571 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:47:48.672 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:48.707 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:48.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:48.711 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:47:48.812 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:48.842 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:48.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:48.844 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:47:48.945 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:48.971 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:48.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:48.976 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:47:49.077 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:49.109 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:49.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:49.112 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:47:49.213 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:49.236 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:49.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:49.239 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:47:49.339 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:49.376 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:49.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:49.379 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:47:49.480 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:49.501 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:49.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:49.503 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:49.604 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:49.630 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:49.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:49.636 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:47:49.737 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:49.747 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:49 smithi173 bash[58816]: cluster 2023-10-01T10:47:47.870151+0000 mgr.y (mgr.25072) 905 : cluster 0 pgmap v836: 89 pgs: 89 active+clean; 6.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 41 MiB/s wr, 702 op/s 2023-10-01T10:47:49.779 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:49.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:49.781 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:49.882 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:49.911 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:49.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:49.918 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:47:50.018 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:50.026 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:49 smithi078 bash[78274]: cluster 2023-10-01T10:47:47.870151+0000 mgr.y (mgr.25072) 905 : cluster 0 pgmap v836: 89 pgs: 89 active+clean; 6.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 41 MiB/s wr, 702 op/s 2023-10-01T10:47:50.027 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:49 smithi078 bash[75299]: cluster 2023-10-01T10:47:47.870151+0000 mgr.y (mgr.25072) 905 : cluster 0 pgmap v836: 89 pgs: 89 active+clean; 6.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 41 MiB/s wr, 702 op/s 2023-10-01T10:47:50.041 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:50.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:50.044 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:47:50.145 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:50.176 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:50.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:50.179 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:47:50.280 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:50.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:50.323 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:50.323 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:50.331 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:47:50 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:47:50] "GET /metrics HTTP/1.1" 200 36311 "" "Prometheus/2.33.4" 2023-10-01T10:47:50.424 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:50.474 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:50.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:50.476 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:50.577 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:50.610 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:50.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:50.615 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:47:50.715 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:50.740 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:50.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:50.745 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:47:50.846 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:50.873 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:50.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:50.876 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:47:50.977 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:51.007 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:51.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:51.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:47:51.110 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:51.134 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:51.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:51.139 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:47:51.163 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T10:47:51.202 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T10:47:51.239 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:51.245 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T10:47:51.271 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:51.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:51.278 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:51.280 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T10:47:51.325 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T10:47:51.357 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T10:47:51.379 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:51.392 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T10:47:51.407 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:51.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:51.411 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:47:51.426 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T10:47:51.437 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T10:47:51.437 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T10:47:51.437 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T10:47:51.437 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T10:47:51.437 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T10:47:51.437 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T10:47:51.437 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T10:47:51.438 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T10:47:51.512 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:51.540 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:51.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:51.543 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:47:51.645 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:51.677 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:51.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:51.683 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:47:51.783 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:51.808 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:51.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:51.811 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:51.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:51 smithi078 bash[75299]: cluster 2023-10-01T10:47:49.870701+0000 mgr.y (mgr.25072) 906 : cluster 0 pgmap v837: 89 pgs: 89 active+clean; 6.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 40 MiB/s wr, 999 op/s 2023-10-01T10:47:51.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:51 smithi078 bash[75299]: audit 2023-10-01T10:47:51.112010+0000 mon.c (mon.2) 157 : audit 0 from='client.? 172.21.15.78:0/1296164947' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:47:51.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:51 smithi078 bash[78274]: cluster 2023-10-01T10:47:49.870701+0000 mgr.y (mgr.25072) 906 : cluster 0 pgmap v837: 89 pgs: 89 active+clean; 6.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 40 MiB/s wr, 999 op/s 2023-10-01T10:47:51.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:51 smithi078 bash[78274]: audit 2023-10-01T10:47:51.112010+0000 mon.c (mon.2) 157 : audit 0 from='client.? 172.21.15.78:0/1296164947' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:47:51.911 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:51.939 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:51.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:51.946 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:47:51.997 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2023-10-01T10:47:51.997 INFO:teuthology.orchestra.run.smithi078.stdout: "size": 20, 2023-10-01T10:47:51.997 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 600, 2023-10-01T10:47:51.997 INFO:teuthology.orchestra.run.smithi078.stdout: "ops": [ 2023-10-01T10:47:51.997 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:51.997 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.35649.0:165324 9.b 9:d324b897:::benchmark_data_smithi078_113121_object53509:head [delete] snapc 0=[] ondisk+write+known_if_redirected e139)", 2023-10-01T10:47:51.997 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:43:49.845675+0000", 2023-10-01T10:47:51.997 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 242.14680065600001, 2023-10-01T10:47:51.997 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.18518485100000001, 2023-10-01T10:47:51.998 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:47:51.998 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:47:51.998 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:47:51.998 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.35649", 2023-10-01T10:47:51.998 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:47:51.998 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 165324 2023-10-01T10:47:51.998 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:51.998 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:47:51.998 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:51.998 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:47:51.998 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.845675+0000", 2023-10-01T10:47:51.999 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:47:51.999 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:51.999 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:51.999 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:47:51.999 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.845675+0000", 2023-10-01T10:47:51.999 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:47:51.999 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:51.999 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:51.999 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:47:51.999 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.845674+0000", 2023-10-01T10:47:51.999 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:47:51.999 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:51.999 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.000 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:47:52.000 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.845680+0000", 2023-10-01T10:47:52.000 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.8479999999999997e-06 2023-10-01T10:47:52.000 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.000 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.000 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:47:52.000 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.845689+0000", 2023-10-01T10:47:52.000 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.3170000000000008e-06 2023-10-01T10:47:52.000 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.000 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.000 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:47:52.000 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.845697+0000", 2023-10-01T10:47:52.001 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.6029999999999999e-06 2023-10-01T10:47:52.001 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.001 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.001 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:47:52.001 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.845732+0000", 2023-10-01T10:47:52.001 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.4755999999999997e-05 2023-10-01T10:47:52.001 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.001 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.001 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:47:52.001 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.845880+0000", 2023-10-01T10:47:52.001 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00014844500000000001 2023-10-01T10:47:52.001 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.001 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.002 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 5,6", 2023-10-01T10:47:52.002 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.846013+0000", 2023-10-01T10:47:52.002 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00013311200000000001 2023-10-01T10:47:52.002 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.002 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.002 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:47:52.002 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.856881+0000", 2023-10-01T10:47:52.002 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.010867488 2023-10-01T10:47:52.002 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.002 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.002 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:47:52.003 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.856889+0000", 2023-10-01T10:47:52.003 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.0469999999999994e-06 2023-10-01T10:47:52.003 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.003 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.003 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:47:52.003 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:50.030766+0000", 2023-10-01T10:47:52.003 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.173877796 2023-10-01T10:47:52.003 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.003 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.003 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:47:52.003 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:50.030818+0000", 2023-10-01T10:47:52.003 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.1505000000000002e-05 2023-10-01T10:47:52.003 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.004 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.004 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:47:52.004 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:50.030860+0000", 2023-10-01T10:47:52.004 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.2073999999999997e-05 2023-10-01T10:47:52.004 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:47:52.004 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:47:52.004 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:47:52.004 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.004 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.004 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.35649.0:165327 9.5 9:a0b9b19b:::benchmark_data_smithi078_113121_object53512:head [delete] snapc 0=[] ondisk+write+known_if_redirected e139)", 2023-10-01T10:47:52.004 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:43:49.846851+0000", 2023-10-01T10:47:52.004 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 242.145624424, 2023-10-01T10:47:52.005 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.18776417500000001, 2023-10-01T10:47:52.006 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:47:52.006 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:47:52.006 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:47:52.006 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.35649", 2023-10-01T10:47:52.006 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:47:52.006 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 165327 2023-10-01T10:47:52.006 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.007 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:47:52.007 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.007 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:47:52.007 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.846851+0000", 2023-10-01T10:47:52.007 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:47:52.007 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.007 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.007 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:47:52.007 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.846851+0000", 2023-10-01T10:47:52.007 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:47:52.007 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.007 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.008 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:47:52.008 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.846851+0000", 2023-10-01T10:47:52.008 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:47:52.008 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.008 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.008 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:47:52.008 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.846856+0000", 2023-10-01T10:47:52.008 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.2639999999999999e-06 2023-10-01T10:47:52.008 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.008 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.008 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:47:52.008 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.846867+0000", 2023-10-01T10:47:52.008 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1072e-05 2023-10-01T10:47:52.009 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.009 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.009 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:47:52.009 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.846874+0000", 2023-10-01T10:47:52.009 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.3499999999999999e-06 2023-10-01T10:47:52.009 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.009 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.009 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:47:52.009 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.846919+0000", 2023-10-01T10:47:52.009 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.4827000000000003e-05 2023-10-01T10:47:52.009 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.009 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.010 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:47:52.010 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.847088+0000", 2023-10-01T10:47:52.010 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00016869300000000001 2023-10-01T10:47:52.010 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.010 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.010 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 5,7", 2023-10-01T10:47:52.010 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.847194+0000", 2023-10-01T10:47:52.010 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000106282 2023-10-01T10:47:52.010 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.010 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.010 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:47:52.010 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.863676+0000", 2023-10-01T10:47:52.011 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.016482055999999998 2023-10-01T10:47:52.011 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.011 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.011 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:47:52.011 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:50.034413+0000", 2023-10-01T10:47:52.011 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.170736782 2023-10-01T10:47:52.011 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.011 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.011 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:47:52.011 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:50.034521+0000", 2023-10-01T10:47:52.011 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00010861099999999999 2023-10-01T10:47:52.011 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.012 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.012 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:47:52.012 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:50.034574+0000", 2023-10-01T10:47:52.012 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.2485000000000003e-05 2023-10-01T10:47:52.012 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.012 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.012 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:47:52.012 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:50.034616+0000", 2023-10-01T10:47:52.012 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.159e-05 2023-10-01T10:47:52.012 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:47:52.012 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:47:52.012 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:47:52.013 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.013 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.013 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.35649.0:165329 9.b 9:d607af82:::benchmark_data_smithi078_113121_object53514:head [delete] snapc 0=[] ondisk+write+known_if_redirected e139)", 2023-10-01T10:47:52.013 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:43:49.847003+0000", 2023-10-01T10:47:52.013 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 242.14547259, 2023-10-01T10:47:52.013 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.18398266199999999, 2023-10-01T10:47:52.013 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:47:52.013 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:47:52.013 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:47:52.013 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.35649", 2023-10-01T10:47:52.013 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:47:52.013 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 165329 2023-10-01T10:47:52.014 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.014 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:47:52.014 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.014 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:47:52.014 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.847003+0000", 2023-10-01T10:47:52.014 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:47:52.014 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.014 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.014 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:47:52.014 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.847003+0000", 2023-10-01T10:47:52.014 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:47:52.015 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.015 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.015 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:47:52.015 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.847002+0000", 2023-10-01T10:47:52.015 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:47:52.015 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.015 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.015 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:47:52.015 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.847007+0000", 2023-10-01T10:47:52.015 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.1810000000000003e-06 2023-10-01T10:47:52.015 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.015 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.016 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:47:52.016 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.847015+0000", 2023-10-01T10:47:52.016 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.7180000000000002e-06 2023-10-01T10:47:52.016 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.016 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.016 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:47:52.016 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.847022+0000", 2023-10-01T10:47:52.016 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.6420000000000001e-06 2023-10-01T10:47:52.016 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.016 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.016 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:47:52.016 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.851102+0000", 2023-10-01T10:47:52.017 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0040797539999999997 2023-10-01T10:47:52.017 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.017 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.017 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:47:52.017 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.851217+0000", 2023-10-01T10:47:52.017 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00011499599999999999 2023-10-01T10:47:52.017 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.017 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.017 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 5,6", 2023-10-01T10:47:52.017 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.851330+0000", 2023-10-01T10:47:52.017 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000113595 2023-10-01T10:47:52.017 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.017 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.018 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:47:52.018 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.857901+0000", 2023-10-01T10:47:52.018 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.006570729 2023-10-01T10:47:52.018 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.018 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.018 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:47:52.018 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.862758+0000", 2023-10-01T10:47:52.018 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0048568229999999997 2023-10-01T10:47:52.018 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.018 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.018 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:47:52.018 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:50.030904+0000", 2023-10-01T10:47:52.019 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.168146252 2023-10-01T10:47:52.019 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.019 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.019 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:47:52.019 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:50.030950+0000", 2023-10-01T10:47:52.019 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.5374999999999998e-05 2023-10-01T10:47:52.019 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.019 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.019 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:47:52.019 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:50.030986+0000", 2023-10-01T10:47:52.019 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.6356000000000002e-05 2023-10-01T10:47:52.019 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:47:52.019 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:47:52.020 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:47:52.020 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.020 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.020 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.35649.0:165330 9.5 9:a504be94:::benchmark_data_smithi078_113121_object53515:head [delete] snapc 0=[] ondisk+write+known_if_redirected e139)", 2023-10-01T10:47:52.020 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:43:49.853347+0000", 2023-10-01T10:47:52.020 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 242.13912920199999, 2023-10-01T10:47:52.020 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.18145360199999999, 2023-10-01T10:47:52.020 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:47:52.020 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:47:52.020 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:47:52.020 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.35649", 2023-10-01T10:47:52.020 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:47:52.021 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 165330 2023-10-01T10:47:52.021 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.021 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:47:52.021 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.021 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:47:52.021 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.853347+0000", 2023-10-01T10:47:52.021 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:47:52.021 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.021 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.021 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:47:52.021 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.853347+0000", 2023-10-01T10:47:52.021 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:47:52.022 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.022 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.022 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:47:52.022 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.853345+0000", 2023-10-01T10:47:52.022 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999986 2023-10-01T10:47:52.022 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.022 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.022 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:47:52.022 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.853353+0000", 2023-10-01T10:47:52.022 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.6199999999999999e-06 2023-10-01T10:47:52.022 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.023 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.023 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:47:52.023 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.853366+0000", 2023-10-01T10:47:52.023 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.293e-05 2023-10-01T10:47:52.023 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.023 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.023 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:47:52.023 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.853374+0000", 2023-10-01T10:47:52.023 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.5310000000000001e-06 2023-10-01T10:47:52.023 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.023 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.023 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:47:52.024 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.858491+0000", 2023-10-01T10:47:52.024 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0051165439999999998 2023-10-01T10:47:52.024 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.024 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.024 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:47:52.024 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.858635+0000", 2023-10-01T10:47:52.024 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00014448099999999999 2023-10-01T10:47:52.024 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.024 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.024 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 5,7", 2023-10-01T10:47:52.024 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.858776+0000", 2023-10-01T10:47:52.024 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00014127000000000001 2023-10-01T10:47:52.025 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.025 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.025 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:47:52.025 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:50.034433+0000", 2023-10-01T10:47:52.025 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.17565662000000001 2023-10-01T10:47:52.025 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.025 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.025 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:47:52.025 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:50.034475+0000", 2023-10-01T10:47:52.025 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.2148999999999999e-05 2023-10-01T10:47:52.025 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.025 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.026 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:47:52.026 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:50.034685+0000", 2023-10-01T10:47:52.026 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00021024999999999999 2023-10-01T10:47:52.026 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.026 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.026 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:47:52.026 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:50.034741+0000", 2023-10-01T10:47:52.026 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.5853000000000002e-05 2023-10-01T10:47:52.026 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.026 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.026 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:47:52.026 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:50.034800+0000", 2023-10-01T10:47:52.027 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.8974999999999997e-05 2023-10-01T10:47:52.027 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:47:52.027 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:47:52.027 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:47:52.027 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.027 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.027 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.35649.0:165335 9.9 9:994d35a6:::benchmark_data_smithi078_113121_object53520:head [delete] snapc 0=[] ondisk+write+known_if_redirected e139)", 2023-10-01T10:47:52.027 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:43:49.858745+0000", 2023-10-01T10:47:52.027 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 242.13373132000001, 2023-10-01T10:47:52.027 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.16675427500000001, 2023-10-01T10:47:52.027 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:47:52.027 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:47:52.027 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:47:52.028 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.35649", 2023-10-01T10:47:52.028 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:47:52.028 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 165335 2023-10-01T10:47:52.028 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.028 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:47:52.028 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.028 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:47:52.028 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.858745+0000", 2023-10-01T10:47:52.028 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:47:52.028 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.028 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.028 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:47:52.028 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.858745+0000", 2023-10-01T10:47:52.029 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:47:52.029 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.029 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.029 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:47:52.029 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.858743+0000", 2023-10-01T10:47:52.029 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:47:52.029 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.029 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.029 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:47:52.029 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.858750+0000", 2023-10-01T10:47:52.029 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.5880000000000003e-06 2023-10-01T10:47:52.030 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.030 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.030 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:47:52.030 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.858764+0000", 2023-10-01T10:47:52.030 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.399e-05 2023-10-01T10:47:52.030 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.030 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.030 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:47:52.030 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.858773+0000", 2023-10-01T10:47:52.030 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.0450000000000006e-06 2023-10-01T10:47:52.030 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.031 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.031 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:47:52.031 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.858824+0000", 2023-10-01T10:47:52.031 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.1566000000000001e-05 2023-10-01T10:47:52.031 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.031 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.031 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:47:52.031 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.858946+0000", 2023-10-01T10:47:52.031 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00012193699999999999 2023-10-01T10:47:52.031 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.031 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.031 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 5,7", 2023-10-01T10:47:52.032 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.859037+0000", 2023-10-01T10:47:52.032 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.0455e-05 2023-10-01T10:47:52.032 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.032 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.032 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:47:52.032 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:50.025130+0000", 2023-10-01T10:47:52.032 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.166093466 2023-10-01T10:47:52.032 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.032 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.032 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:47:52.032 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:50.025353+0000", 2023-10-01T10:47:52.032 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00022297500000000001 2023-10-01T10:47:52.032 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.033 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.033 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:47:52.033 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:50.025412+0000", 2023-10-01T10:47:52.033 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.8675999999999999e-05 2023-10-01T10:47:52.033 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.033 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.033 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:47:52.033 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:50.025458+0000", 2023-10-01T10:47:52.033 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.6335999999999998e-05 2023-10-01T10:47:52.033 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.033 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.033 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:47:52.034 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:50.025499+0000", 2023-10-01T10:47:52.034 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.0432000000000002e-05 2023-10-01T10:47:52.034 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:47:52.034 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:47:52.034 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:47:52.034 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.034 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.034 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.35649.0:165341 9.5 9:ab37ed78:::benchmark_data_smithi078_113121_object53526:head [delete] snapc 0=[] ondisk+write+known_if_redirected e139)", 2023-10-01T10:47:52.034 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:43:49.864344+0000", 2023-10-01T10:47:52.034 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 242.128131669, 2023-10-01T10:47:52.034 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.22500295400000001, 2023-10-01T10:47:52.034 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:47:52.035 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:47:52.035 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:47:52.035 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.35649", 2023-10-01T10:47:52.035 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:47:52.035 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 165341 2023-10-01T10:47:52.035 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.035 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:47:52.035 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.035 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:47:52.035 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.864344+0000", 2023-10-01T10:47:52.035 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:47:52.035 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.036 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.036 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:47:52.036 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.864344+0000", 2023-10-01T10:47:52.036 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:47:52.036 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.036 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.036 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:47:52.036 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.864343+0000", 2023-10-01T10:47:52.036 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999986 2023-10-01T10:47:52.036 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.036 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.036 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:47:52.036 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.864350+0000", 2023-10-01T10:47:52.037 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.627e-06 2023-10-01T10:47:52.037 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.037 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.037 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:47:52.037 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.864362+0000", 2023-10-01T10:47:52.037 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.2159e-05 2023-10-01T10:47:52.037 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.037 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.037 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:47:52.037 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.864371+0000", 2023-10-01T10:47:52.037 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.4160000000000004e-06 2023-10-01T10:47:52.037 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.038 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.038 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:47:52.038 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.864412+0000", 2023-10-01T10:47:52.038 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.1554000000000002e-05 2023-10-01T10:47:52.038 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.038 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.038 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:47:52.038 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.864498+0000", 2023-10-01T10:47:52.038 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.5133e-05 2023-10-01T10:47:52.039 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.039 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.039 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 5,7", 2023-10-01T10:47:52.039 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:50.025249+0000", 2023-10-01T10:47:52.039 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.16075122999999999 2023-10-01T10:47:52.039 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.039 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.039 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:47:52.039 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:50.046572+0000", 2023-10-01T10:47:52.039 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.021323420999999999 2023-10-01T10:47:52.039 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.039 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.040 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:47:52.040 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:50.082928+0000", 2023-10-01T10:47:52.040 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.036355910999999998 2023-10-01T10:47:52.040 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.040 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.040 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:47:52.040 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:50.089201+0000", 2023-10-01T10:47:52.040 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0062731849999999997 2023-10-01T10:47:52.040 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.040 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.040 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:47:52.040 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:50.089298+0000", 2023-10-01T10:47:52.040 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.6272000000000003e-05 2023-10-01T10:47:52.041 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.041 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.041 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:47:52.041 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:50.089347+0000", 2023-10-01T10:47:52.041 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.9555000000000002e-05 2023-10-01T10:47:52.041 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:47:52.041 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:47:52.041 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:47:52.041 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.041 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.041 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.35649.0:165343 9.b 9:d277ee49:::benchmark_data_smithi078_113121_object53528:head [delete] snapc 0=[] ondisk+write+known_if_redirected e139)", 2023-10-01T10:47:52.041 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:43:49.866662+0000", 2023-10-01T10:47:52.042 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 242.125813491, 2023-10-01T10:47:52.042 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.18554657999999999, 2023-10-01T10:47:52.042 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:47:52.042 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:47:52.042 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:47:52.042 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.35649", 2023-10-01T10:47:52.042 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:47:52.042 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 165343 2023-10-01T10:47:52.042 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.043 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:47:52.043 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.043 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:47:52.043 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.866662+0000", 2023-10-01T10:47:52.043 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:47:52.043 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.043 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.043 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:47:52.043 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.866662+0000", 2023-10-01T10:47:52.043 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:47:52.043 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.043 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.043 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:47:52.044 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.866661+0000", 2023-10-01T10:47:52.044 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999986 2023-10-01T10:47:52.044 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.044 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.044 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:47:52.044 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.866667+0000", 2023-10-01T10:47:52.044 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.3289999999999999e-06 2023-10-01T10:47:52.044 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.044 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.044 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:47:52.044 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:50.025061+0000", 2023-10-01T10:47:52.044 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.15839352000000001 2023-10-01T10:47:52.044 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.045 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.045 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:47:52.045 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:50.025088+0000", 2023-10-01T10:47:52.045 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.7636000000000001e-05 2023-10-01T10:47:52.045 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.045 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.045 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:47:52.045 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:50.025190+0000", 2023-10-01T10:47:52.045 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000101469 2023-10-01T10:47:52.045 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.045 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.045 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:47:52.046 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:50.025349+0000", 2023-10-01T10:47:52.046 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00015940999999999999 2023-10-01T10:47:52.046 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.046 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.046 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 5,6", 2023-10-01T10:47:52.046 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:50.025480+0000", 2023-10-01T10:47:52.046 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00013033899999999999 2023-10-01T10:47:52.046 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.046 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.046 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:47:52.046 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:50.039706+0000", 2023-10-01T10:47:52.046 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.014226383 2023-10-01T10:47:52.047 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.047 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.047 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:47:52.047 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:50.044150+0000", 2023-10-01T10:47:52.047 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0044442279999999997 2023-10-01T10:47:52.047 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.047 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.047 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:47:52.047 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:50.052152+0000", 2023-10-01T10:47:52.047 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0080016989999999993 2023-10-01T10:47:52.047 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.047 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.047 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:47:52.048 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:50.052180+0000", 2023-10-01T10:47:52.048 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.7898999999999998e-05 2023-10-01T10:47:52.048 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.048 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.048 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:47:52.048 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:50.052209+0000", 2023-10-01T10:47:52.048 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.9022000000000001e-05 2023-10-01T10:47:52.048 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:47:52.048 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:47:52.048 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:47:52.049 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.049 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.049 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop_reply(client.35649.0:165324 9.b e139/128 ondisk, result = 0)", 2023-10-01T10:47:52.049 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:43:49.866963+0000", 2023-10-01T10:47:52.049 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 242.12551263200001, 2023-10-01T10:47:52.049 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.163905575, 2023-10-01T10:47:52.049 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:47:52.049 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "started", 2023-10-01T10:47:52.049 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:47:52.049 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.049 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:47:52.049 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.866963+0000", 2023-10-01T10:47:52.049 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:47:52.050 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.050 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.050 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:47:52.050 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.866963+0000", 2023-10-01T10:47:52.050 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:47:52.050 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.050 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.050 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:47:52.050 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.866962+0000", 2023-10-01T10:47:52.050 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:47:52.050 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.050 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.051 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:47:52.051 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:49.866968+0000", 2023-10-01T10:47:52.051 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.7250000000000002e-06 2023-10-01T10:47:52.051 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.051 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.051 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:47:52.051 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:50.025099+0000", 2023-10-01T10:47:52.051 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.158130716 2023-10-01T10:47:52.051 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.051 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.051 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:47:52.051 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:50.025121+0000", 2023-10-01T10:47:52.052 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.2405999999999999e-05 2023-10-01T10:47:52.052 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.052 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.052 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:47:52.052 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:50.030749+0000", 2023-10-01T10:47:52.052 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0056272800000000001 2023-10-01T10:47:52.052 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.052 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.052 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:47:52.052 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:50.030760+0000", 2023-10-01T10:47:52.052 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1045e-05 2023-10-01T10:47:52.052 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.052 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.053 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:47:52.053 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:50.030869+0000", 2023-10-01T10:47:52.053 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000109145 2023-10-01T10:47:52.053 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:47:52.053 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:47:52.053 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:47:52.053 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.053 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.053 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.45818.0:40400 10.4 10:28c05415:::benchmark_data_smithi078_135100_object40399:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e155)", 2023-10-01T10:47:52.053 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:46:34.566749+0000", 2023-10-01T10:47:52.053 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 77.425727319000003, 2023-10-01T10:47:52.053 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.16419449, 2023-10-01T10:47:52.054 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:47:52.054 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:47:52.054 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:47:52.054 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.45818", 2023-10-01T10:47:52.054 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1151403832", 2023-10-01T10:47:52.054 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 40400 2023-10-01T10:47:52.054 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.054 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:47:52.054 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.054 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:47:52.054 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:34.566749+0000", 2023-10-01T10:47:52.055 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:47:52.055 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.055 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.055 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:47:52.055 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:34.566749+0000", 2023-10-01T10:47:52.055 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:47:52.055 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.055 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.055 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:47:52.055 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:34.566748+0000", 2023-10-01T10:47:52.055 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:47:52.055 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.056 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.056 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:47:52.056 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:34.566767+0000", 2023-10-01T10:47:52.056 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.9862000000000001e-05 2023-10-01T10:47:52.056 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.056 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.056 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:47:52.056 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:34.566774+0000", 2023-10-01T10:47:52.056 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.3579999999999999e-06 2023-10-01T10:47:52.056 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.056 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.056 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:47:52.057 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:34.566778+0000", 2023-10-01T10:47:52.057 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.1910000000000002e-06 2023-10-01T10:47:52.057 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.057 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.057 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:47:52.057 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:34.721092+0000", 2023-10-01T10:47:52.057 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.154314127 2023-10-01T10:47:52.057 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.057 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.057 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:47:52.057 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:34.721221+0000", 2023-10-01T10:47:52.057 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000128567 2023-10-01T10:47:52.057 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.058 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.058 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 3,4", 2023-10-01T10:47:52.058 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:34.721472+0000", 2023-10-01T10:47:52.058 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00025120299999999997 2023-10-01T10:47:52.058 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.058 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.058 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:47:52.058 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:34.726258+0000", 2023-10-01T10:47:52.058 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0047862440000000003 2023-10-01T10:47:52.058 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.058 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.058 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:47:52.059 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:34.726502+0000", 2023-10-01T10:47:52.059 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00024361900000000001 2023-10-01T10:47:52.059 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.059 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.059 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:47:52.059 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:34.730783+0000", 2023-10-01T10:47:52.059 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0042814460000000004 2023-10-01T10:47:52.059 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.059 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.059 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:47:52.059 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:34.730898+0000", 2023-10-01T10:47:52.059 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000114401 2023-10-01T10:47:52.060 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.060 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.060 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:47:52.060 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:34.730943+0000", 2023-10-01T10:47:52.060 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.5343999999999998e-05 2023-10-01T10:47:52.060 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:47:52.060 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:47:52.060 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:47:52.061 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.061 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.061 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.45818.0:40401 10.4 10:272d91db:::benchmark_data_smithi078_135100_object40400:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e155)", 2023-10-01T10:47:52.061 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:46:34.566939+0000", 2023-10-01T10:47:52.061 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 77.425537301999995, 2023-10-01T10:47:52.061 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.164125928, 2023-10-01T10:47:52.061 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:47:52.061 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:47:52.061 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:47:52.061 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.45818", 2023-10-01T10:47:52.061 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1151403832", 2023-10-01T10:47:52.061 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 40401 2023-10-01T10:47:52.061 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.062 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:47:52.062 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.062 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:47:52.062 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:34.566939+0000", 2023-10-01T10:47:52.062 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:47:52.062 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.062 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.062 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:47:52.062 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:34.566939+0000", 2023-10-01T10:47:52.062 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:47:52.062 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.062 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.063 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:47:52.063 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:34.566938+0000", 2023-10-01T10:47:52.063 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:47:52.063 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.063 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.063 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:47:52.063 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:34.566961+0000", 2023-10-01T10:47:52.063 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.3481000000000001e-05 2023-10-01T10:47:52.063 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.063 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.063 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:47:52.064 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:34.720591+0000", 2023-10-01T10:47:52.064 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.15362958099999999 2023-10-01T10:47:52.064 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.064 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.064 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:47:52.064 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:34.720603+0000", 2023-10-01T10:47:52.064 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1901e-05 2023-10-01T10:47:52.064 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.064 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.064 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:47:52.064 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:34.723872+0000", 2023-10-01T10:47:52.064 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0032695950000000001 2023-10-01T10:47:52.064 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.065 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.065 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:47:52.065 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:34.723996+0000", 2023-10-01T10:47:52.065 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000124244 2023-10-01T10:47:52.065 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.065 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.065 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 3,4", 2023-10-01T10:47:52.065 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:34.724395+0000", 2023-10-01T10:47:52.065 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00039870500000000001 2023-10-01T10:47:52.065 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.065 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.065 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:47:52.065 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:34.726989+0000", 2023-10-01T10:47:52.066 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0025938910000000001 2023-10-01T10:47:52.066 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.066 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.066 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:47:52.066 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:34.727179+0000", 2023-10-01T10:47:52.066 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00018950400000000001 2023-10-01T10:47:52.066 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.066 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.066 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:47:52.066 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:34.730989+0000", 2023-10-01T10:47:52.066 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0038105140000000001 2023-10-01T10:47:52.066 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.067 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.067 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:47:52.067 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:34.731031+0000", 2023-10-01T10:47:52.067 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.2011999999999997e-05 2023-10-01T10:47:52.067 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.067 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.067 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:47:52.067 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:34.731064+0000", 2023-10-01T10:47:52.067 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.3362999999999999e-05 2023-10-01T10:47:52.067 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:47:52.067 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:47:52.067 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:47:52.067 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.068 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.068 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.45818.0:42745 10.8 10:19bd7cbb:::benchmark_data_smithi078_135100_object42744:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e155)", 2023-10-01T10:47:52.068 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:46:36.231461+0000", 2023-10-01T10:47:52.068 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 75.761015317000002, 2023-10-01T10:47:52.068 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.205847792, 2023-10-01T10:47:52.068 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:47:52.068 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:47:52.068 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:47:52.068 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.45818", 2023-10-01T10:47:52.068 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1151403832", 2023-10-01T10:47:52.068 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 42745 2023-10-01T10:47:52.068 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.069 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:47:52.069 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.069 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:47:52.069 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.231461+0000", 2023-10-01T10:47:52.069 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:47:52.069 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.069 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.069 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:47:52.069 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.231461+0000", 2023-10-01T10:47:52.069 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:47:52.069 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.069 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.069 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:47:52.070 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.231459+0000", 2023-10-01T10:47:52.070 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999986 2023-10-01T10:47:52.070 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.070 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.070 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:47:52.070 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.231481+0000", 2023-10-01T10:47:52.070 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.2208000000000002e-05 2023-10-01T10:47:52.070 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.070 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.071 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:47:52.071 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.231492+0000", 2023-10-01T10:47:52.071 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1042e-05 2023-10-01T10:47:52.071 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.071 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.071 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:47:52.071 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.231498+0000", 2023-10-01T10:47:52.071 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.9050000000000002e-06 2023-10-01T10:47:52.071 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.071 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.071 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:47:52.071 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.231548+0000", 2023-10-01T10:47:52.071 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.0059000000000003e-05 2023-10-01T10:47:52.072 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.072 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.072 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:47:52.072 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.231697+0000", 2023-10-01T10:47:52.072 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00014874899999999999 2023-10-01T10:47:52.072 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.072 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.072 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 2,7", 2023-10-01T10:47:52.072 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.231822+0000", 2023-10-01T10:47:52.072 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000124559 2023-10-01T10:47:52.072 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.072 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.072 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:47:52.073 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.233086+0000", 2023-10-01T10:47:52.073 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.001264041 2023-10-01T10:47:52.073 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.073 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.073 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:47:52.073 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.234282+0000", 2023-10-01T10:47:52.073 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.001195907 2023-10-01T10:47:52.073 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.073 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.073 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:47:52.073 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.437224+0000", 2023-10-01T10:47:52.073 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.202942768 2023-10-01T10:47:52.074 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.074 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.074 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:47:52.074 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.437284+0000", 2023-10-01T10:47:52.074 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.9472999999999998e-05 2023-10-01T10:47:52.074 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.074 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.074 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:47:52.074 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.437308+0000", 2023-10-01T10:47:52.074 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.4477e-05 2023-10-01T10:47:52.074 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:47:52.074 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:47:52.075 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:47:52.075 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.075 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.075 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.45818.0:42751 10.8 10:105edb8c:::benchmark_data_smithi078_135100_object42750:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e155)", 2023-10-01T10:47:52.075 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:46:36.234798+0000", 2023-10-01T10:47:52.075 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 75.757677380999993, 2023-10-01T10:47:52.075 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.20421113599999999, 2023-10-01T10:47:52.075 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:47:52.075 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:47:52.075 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:47:52.075 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.45818", 2023-10-01T10:47:52.075 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1151403832", 2023-10-01T10:47:52.075 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 42751 2023-10-01T10:47:52.076 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.076 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:47:52.076 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.076 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:47:52.076 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.234798+0000", 2023-10-01T10:47:52.076 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:47:52.076 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.076 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.076 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:47:52.076 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.234798+0000", 2023-10-01T10:47:52.076 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:47:52.076 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.077 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.077 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:47:52.077 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.234797+0000", 2023-10-01T10:47:52.077 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:47:52.077 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.077 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.077 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:47:52.077 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.234817+0000", 2023-10-01T10:47:52.077 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.9562e-05 2023-10-01T10:47:52.077 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.077 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.077 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:47:52.078 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.234827+0000", 2023-10-01T10:47:52.078 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.0122000000000001e-05 2023-10-01T10:47:52.078 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.078 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.078 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:47:52.078 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.234833+0000", 2023-10-01T10:47:52.078 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.806e-06 2023-10-01T10:47:52.078 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.078 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.078 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:47:52.078 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.234868+0000", 2023-10-01T10:47:52.078 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.4915e-05 2023-10-01T10:47:52.078 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.079 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.079 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:47:52.079 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.234965+0000", 2023-10-01T10:47:52.079 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.7515000000000004e-05 2023-10-01T10:47:52.079 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.079 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.079 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 2,7", 2023-10-01T10:47:52.079 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.235057+0000", 2023-10-01T10:47:52.079 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.1446000000000005e-05 2023-10-01T10:47:52.079 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.079 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.079 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:47:52.080 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.236999+0000", 2023-10-01T10:47:52.080 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0019417169999999999 2023-10-01T10:47:52.080 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.080 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.080 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:47:52.080 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.238601+0000", 2023-10-01T10:47:52.080 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0016025670000000001 2023-10-01T10:47:52.080 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.080 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.080 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:47:52.080 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.438955+0000", 2023-10-01T10:47:52.080 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.20035378500000001 2023-10-01T10:47:52.080 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.081 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.081 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:47:52.081 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.438986+0000", 2023-10-01T10:47:52.081 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.0976000000000003e-05 2023-10-01T10:47:52.081 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.081 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.081 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:47:52.081 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.439010+0000", 2023-10-01T10:47:52.081 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.3685000000000001e-05 2023-10-01T10:47:52.081 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:47:52.081 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:47:52.081 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:47:52.082 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.082 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.082 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.45818.0:42752 10.8 10:17219ec3:::benchmark_data_smithi078_135100_object42751:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e155)", 2023-10-01T10:47:52.082 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:46:36.236052+0000", 2023-10-01T10:47:52.082 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 75.756423796000007, 2023-10-01T10:47:52.082 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.20305263000000001, 2023-10-01T10:47:52.082 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:47:52.082 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:47:52.082 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:47:52.082 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.45818", 2023-10-01T10:47:52.082 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1151403832", 2023-10-01T10:47:52.082 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 42752 2023-10-01T10:47:52.082 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.083 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:47:52.083 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.083 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:47:52.083 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.236052+0000", 2023-10-01T10:47:52.083 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:47:52.083 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.083 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.083 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:47:52.083 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.236052+0000", 2023-10-01T10:47:52.083 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:47:52.083 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.083 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.084 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:47:52.084 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.236051+0000", 2023-10-01T10:47:52.084 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:47:52.084 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.084 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.084 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:47:52.084 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.236073+0000", 2023-10-01T10:47:52.084 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.1943e-05 2023-10-01T10:47:52.084 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.084 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.084 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:47:52.084 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.236086+0000", 2023-10-01T10:47:52.084 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.2495e-05 2023-10-01T10:47:52.085 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.085 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.085 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:47:52.085 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.236095+0000", 2023-10-01T10:47:52.085 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.8920000000000006e-06 2023-10-01T10:47:52.085 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.085 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.085 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:47:52.085 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.236128+0000", 2023-10-01T10:47:52.085 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.3485000000000002e-05 2023-10-01T10:47:52.085 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.085 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.085 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:47:52.086 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.236211+0000", 2023-10-01T10:47:52.086 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.2773000000000002e-05 2023-10-01T10:47:52.086 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.086 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.086 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 2,7", 2023-10-01T10:47:52.086 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.236300+0000", 2023-10-01T10:47:52.086 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.9381000000000004e-05 2023-10-01T10:47:52.086 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.086 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.086 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:47:52.086 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.237130+0000", 2023-10-01T10:47:52.087 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00082942300000000003 2023-10-01T10:47:52.087 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.087 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.087 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:47:52.087 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.239639+0000", 2023-10-01T10:47:52.087 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0025098410000000001 2023-10-01T10:47:52.087 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.087 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.087 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:47:52.087 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.439043+0000", 2023-10-01T10:47:52.087 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.19940406799999999 2023-10-01T10:47:52.087 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.087 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.088 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:47:52.088 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.439079+0000", 2023-10-01T10:47:52.088 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.5794000000000003e-05 2023-10-01T10:47:52.088 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.088 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.088 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:47:52.088 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.439105+0000", 2023-10-01T10:47:52.088 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.5378e-05 2023-10-01T10:47:52.088 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:47:52.088 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:47:52.088 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:47:52.088 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.089 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.089 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.45818.0:42753 10.8 10:1df96728:::benchmark_data_smithi078_135100_object42752:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e155)", 2023-10-01T10:47:52.089 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:46:36.237240+0000", 2023-10-01T10:47:52.089 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 75.755236217000004, 2023-10-01T10:47:52.089 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.20196054099999999, 2023-10-01T10:47:52.089 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:47:52.089 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:47:52.089 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:47:52.089 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.45818", 2023-10-01T10:47:52.089 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1151403832", 2023-10-01T10:47:52.089 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 42753 2023-10-01T10:47:52.089 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.090 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:47:52.090 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.090 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:47:52.090 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.237240+0000", 2023-10-01T10:47:52.090 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:47:52.090 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.090 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.090 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:47:52.090 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.237240+0000", 2023-10-01T10:47:52.090 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:47:52.091 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.091 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.091 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:47:52.091 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.237239+0000", 2023-10-01T10:47:52.091 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:47:52.091 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.091 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.091 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:47:52.091 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.237279+0000", 2023-10-01T10:47:52.091 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.0454999999999998e-05 2023-10-01T10:47:52.091 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.091 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.092 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:47:52.092 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.237295+0000", 2023-10-01T10:47:52.092 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.5495999999999999e-05 2023-10-01T10:47:52.092 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.092 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.092 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:47:52.092 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.237304+0000", 2023-10-01T10:47:52.092 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.6309999999999998e-06 2023-10-01T10:47:52.092 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.092 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.092 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:47:52.092 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.237353+0000", 2023-10-01T10:47:52.093 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.9175999999999998e-05 2023-10-01T10:47:52.093 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.093 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.093 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:47:52.093 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.237464+0000", 2023-10-01T10:47:52.093 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000110742 2023-10-01T10:47:52.093 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.093 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.093 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 2,7", 2023-10-01T10:47:52.093 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.237588+0000", 2023-10-01T10:47:52.093 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000124065 2023-10-01T10:47:52.093 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.093 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.094 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:47:52.094 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.238744+0000", 2023-10-01T10:47:52.094 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0011562619999999999 2023-10-01T10:47:52.094 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.094 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.094 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:47:52.094 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.239781+0000", 2023-10-01T10:47:52.094 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.001036854 2023-10-01T10:47:52.094 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.094 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.094 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:47:52.095 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.439142+0000", 2023-10-01T10:47:52.095 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.19936047800000001 2023-10-01T10:47:52.095 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.095 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.095 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:47:52.095 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.439175+0000", 2023-10-01T10:47:52.095 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.3507999999999998e-05 2023-10-01T10:47:52.095 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.095 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.095 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:47:52.095 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.439200+0000", 2023-10-01T10:47:52.095 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.4808e-05 2023-10-01T10:47:52.096 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:47:52.096 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:47:52.096 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:47:52.096 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.096 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.096 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.45818.0:42757 10.8 10:11807276:::benchmark_data_smithi078_135100_object42756:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e155)", 2023-10-01T10:47:52.096 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:46:36.240015+0000", 2023-10-01T10:47:52.096 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 75.752461201000003, 2023-10-01T10:47:52.096 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.199250345, 2023-10-01T10:47:52.096 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:47:52.096 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:47:52.096 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:47:52.097 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.45818", 2023-10-01T10:47:52.097 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1151403832", 2023-10-01T10:47:52.097 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 42757 2023-10-01T10:47:52.097 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.097 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:47:52.097 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.097 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:47:52.097 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.240015+0000", 2023-10-01T10:47:52.097 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:47:52.097 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.097 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.097 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:47:52.097 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.240015+0000", 2023-10-01T10:47:52.098 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:47:52.098 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.098 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.098 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:47:52.098 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.240013+0000", 2023-10-01T10:47:52.098 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:47:52.098 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.098 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.098 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:47:52.098 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.240037+0000", 2023-10-01T10:47:52.098 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.3091999999999998e-05 2023-10-01T10:47:52.098 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.099 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.099 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:47:52.099 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.240046+0000", 2023-10-01T10:47:52.099 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.1759999999999999e-06 2023-10-01T10:47:52.099 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.099 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.099 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:47:52.099 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.240051+0000", 2023-10-01T10:47:52.099 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.242e-06 2023-10-01T10:47:52.099 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.099 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.099 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:47:52.100 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.240084+0000", 2023-10-01T10:47:52.100 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.2706000000000002e-05 2023-10-01T10:47:52.100 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.100 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.100 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:47:52.100 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.240174+0000", 2023-10-01T10:47:52.100 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.0367999999999999e-05 2023-10-01T10:47:52.100 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.100 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.100 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 2,7", 2023-10-01T10:47:52.100 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.240260+0000", 2023-10-01T10:47:52.100 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.6346000000000003e-05 2023-10-01T10:47:52.100 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.101 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.101 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:47:52.101 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.241355+0000", 2023-10-01T10:47:52.101 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0010945480000000001 2023-10-01T10:47:52.101 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.101 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.101 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:47:52.101 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.241891+0000", 2023-10-01T10:47:52.101 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00053648400000000005 2023-10-01T10:47:52.101 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.101 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.101 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:47:52.102 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.439223+0000", 2023-10-01T10:47:52.102 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.197331125 2023-10-01T10:47:52.102 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.102 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.102 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:47:52.102 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.439247+0000", 2023-10-01T10:47:52.102 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.4085000000000001e-05 2023-10-01T10:47:52.102 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.102 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.102 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:47:52.102 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.439265+0000", 2023-10-01T10:47:52.102 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.8326999999999999e-05 2023-10-01T10:47:52.103 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:47:52.103 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:47:52.103 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:47:52.103 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.103 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.103 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.45818.0:54053 10.4 10:289aa393:::benchmark_data_smithi078_135100_object54052:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e156)", 2023-10-01T10:47:52.103 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:46:44.823112+0000", 2023-10-01T10:47:52.103 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 67.169364246000001, 2023-10-01T10:47:52.103 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.166384852, 2023-10-01T10:47:52.103 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:47:52.103 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:47:52.103 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:47:52.104 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.45818", 2023-10-01T10:47:52.104 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1151403832", 2023-10-01T10:47:52.104 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 54053 2023-10-01T10:47:52.104 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.104 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:47:52.104 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.104 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:47:52.104 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.823112+0000", 2023-10-01T10:47:52.104 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:47:52.104 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.104 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.105 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:47:52.105 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.823112+0000", 2023-10-01T10:47:52.105 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:47:52.105 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.105 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.105 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:47:52.105 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.823110+0000", 2023-10-01T10:47:52.105 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999981 2023-10-01T10:47:52.105 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.105 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.105 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:47:52.105 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.823134+0000", 2023-10-01T10:47:52.106 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.4151e-05 2023-10-01T10:47:52.106 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.106 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.106 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:47:52.106 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.823146+0000", 2023-10-01T10:47:52.106 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1897e-05 2023-10-01T10:47:52.106 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.106 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.106 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:47:52.106 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.823154+0000", 2023-10-01T10:47:52.106 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.8059999999999995e-06 2023-10-01T10:47:52.107 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.107 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.107 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:47:52.107 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.823918+0000", 2023-10-01T10:47:52.107 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00076451000000000004 2023-10-01T10:47:52.107 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.107 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.107 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:47:52.107 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.824053+0000", 2023-10-01T10:47:52.107 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00013454999999999999 2023-10-01T10:47:52.107 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.107 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.107 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 3,4", 2023-10-01T10:47:52.108 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.824196+0000", 2023-10-01T10:47:52.108 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00014300900000000001 2023-10-01T10:47:52.108 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.108 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.108 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:47:52.108 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.986730+0000", 2023-10-01T10:47:52.108 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.16253409899999999 2023-10-01T10:47:52.108 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.108 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.108 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:47:52.108 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.986777+0000", 2023-10-01T10:47:52.108 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.7141999999999997e-05 2023-10-01T10:47:52.109 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.109 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.109 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:47:52.109 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.989415+0000", 2023-10-01T10:47:52.109 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0026378539999999998 2023-10-01T10:47:52.109 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.109 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.109 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:47:52.109 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.989459+0000", 2023-10-01T10:47:52.109 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.4356e-05 2023-10-01T10:47:52.109 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.109 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.109 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:47:52.110 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.989496+0000", 2023-10-01T10:47:52.110 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.7177e-05 2023-10-01T10:47:52.110 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:47:52.110 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:47:52.110 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:47:52.110 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.110 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.110 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.45818.0:54054 10.4 10:20463acc:::benchmark_data_smithi078_135100_object54053:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e156)", 2023-10-01T10:47:52.110 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:46:44.823896+0000", 2023-10-01T10:47:52.110 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 67.168579960000002, 2023-10-01T10:47:52.110 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.17607024800000001, 2023-10-01T10:47:52.110 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:47:52.111 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:47:52.111 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:47:52.111 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.45818", 2023-10-01T10:47:52.111 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1151403832", 2023-10-01T10:47:52.111 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 54054 2023-10-01T10:47:52.111 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.111 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:47:52.111 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.111 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:47:52.111 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.823896+0000", 2023-10-01T10:47:52.111 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:47:52.111 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.111 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.112 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:47:52.112 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.823896+0000", 2023-10-01T10:47:52.112 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:47:52.112 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.112 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.112 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:47:52.112 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.823895+0000", 2023-10-01T10:47:52.112 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:47:52.112 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.112 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.112 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:47:52.112 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.823918+0000", 2023-10-01T10:47:52.113 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.2527e-05 2023-10-01T10:47:52.113 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.113 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.113 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:47:52.113 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.823929+0000", 2023-10-01T10:47:52.113 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1566000000000001e-05 2023-10-01T10:47:52.113 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.113 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.113 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:47:52.113 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.823983+0000", 2023-10-01T10:47:52.113 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.3429e-05 2023-10-01T10:47:52.113 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.114 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.114 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:47:52.114 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.983693+0000", 2023-10-01T10:47:52.114 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.159710037 2023-10-01T10:47:52.114 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.114 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.114 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:47:52.114 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.983916+0000", 2023-10-01T10:47:52.114 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000223253 2023-10-01T10:47:52.114 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.114 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.114 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 3,4", 2023-10-01T10:47:52.114 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.984235+0000", 2023-10-01T10:47:52.115 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00031897099999999998 2023-10-01T10:47:52.115 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.115 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.115 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:47:52.115 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.989508+0000", 2023-10-01T10:47:52.115 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0052732079999999997 2023-10-01T10:47:52.115 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.115 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.115 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:47:52.115 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.989643+0000", 2023-10-01T10:47:52.115 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000135088 2023-10-01T10:47:52.115 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.116 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.116 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:47:52.116 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.999901+0000", 2023-10-01T10:47:52.116 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.010257547000000001 2023-10-01T10:47:52.116 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.116 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.116 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:47:52.116 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.999935+0000", 2023-10-01T10:47:52.116 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.4712999999999998e-05 2023-10-01T10:47:52.116 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.116 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.116 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:47:52.117 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.999966+0000", 2023-10-01T10:47:52.117 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.0772999999999998e-05 2023-10-01T10:47:52.117 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:47:52.117 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:47:52.117 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:47:52.117 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.117 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.117 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.45818.0:92524 10.8 10:19f5325b:::benchmark_data_smithi078_135100_object92523:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-10-01T10:47:52.117 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:47:22.357629+0000", 2023-10-01T10:47:52.117 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 29.634847245, 2023-10-01T10:47:52.117 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.193231981, 2023-10-01T10:47:52.117 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:47:52.117 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:47:52.118 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:47:52.118 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.45818", 2023-10-01T10:47:52.118 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1151403832", 2023-10-01T10:47:52.118 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 92524 2023-10-01T10:47:52.118 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.118 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:47:52.118 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.118 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:47:52.118 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.357629+0000", 2023-10-01T10:47:52.118 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:47:52.118 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.118 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.119 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:47:52.119 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.357629+0000", 2023-10-01T10:47:52.119 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:47:52.119 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.119 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.119 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:47:52.119 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.357626+0000", 2023-10-01T10:47:52.119 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999971 2023-10-01T10:47:52.119 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.119 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.119 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:47:52.119 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.357654+0000", 2023-10-01T10:47:52.120 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.7929e-05 2023-10-01T10:47:52.120 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.120 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.120 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:47:52.120 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.357668+0000", 2023-10-01T10:47:52.120 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.4324999999999999e-05 2023-10-01T10:47:52.120 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.120 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.120 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:47:52.120 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.357678+0000", 2023-10-01T10:47:52.120 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.0059e-05 2023-10-01T10:47:52.120 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.120 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.121 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:47:52.121 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.357756+0000", 2023-10-01T10:47:52.121 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.8045000000000002e-05 2023-10-01T10:47:52.121 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.121 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.121 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:47:52.121 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.357925+0000", 2023-10-01T10:47:52.121 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000168474 2023-10-01T10:47:52.121 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.121 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.121 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 2,7", 2023-10-01T10:47:52.121 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.360941+0000", 2023-10-01T10:47:52.122 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0030160410000000001 2023-10-01T10:47:52.122 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.122 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.122 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:47:52.122 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.366151+0000", 2023-10-01T10:47:52.122 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0052104849999999999 2023-10-01T10:47:52.122 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.122 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.122 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:47:52.122 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.371651+0000", 2023-10-01T10:47:52.122 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0055003150000000004 2023-10-01T10:47:52.122 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.123 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.123 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:47:52.123 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.550728+0000", 2023-10-01T10:47:52.123 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.17907624799999999 2023-10-01T10:47:52.123 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.123 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.123 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:47:52.123 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.550806+0000", 2023-10-01T10:47:52.123 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.8757000000000005e-05 2023-10-01T10:47:52.123 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.123 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.123 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:47:52.123 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.550861+0000", 2023-10-01T10:47:52.123 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.4184000000000002e-05 2023-10-01T10:47:52.124 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:47:52.124 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:47:52.124 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:47:52.124 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.124 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.124 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.45818.0:92546 10.8 10:14e40b42:::benchmark_data_smithi078_135100_object92545:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-10-01T10:47:52.124 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:47:22.379228+0000", 2023-10-01T10:47:52.124 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 29.613248255999999, 2023-10-01T10:47:52.124 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.17459656600000001, 2023-10-01T10:47:52.124 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:47:52.124 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:47:52.124 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:47:52.124 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.45818", 2023-10-01T10:47:52.124 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1151403832", 2023-10-01T10:47:52.124 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 92546 2023-10-01T10:47:52.125 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.125 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:47:52.125 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.125 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:47:52.125 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.379228+0000", 2023-10-01T10:47:52.125 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:47:52.125 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.125 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.125 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:47:52.125 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.379228+0000", 2023-10-01T10:47:52.125 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:47:52.125 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.126 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.126 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:47:52.126 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.379226+0000", 2023-10-01T10:47:52.126 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999986 2023-10-01T10:47:52.126 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.126 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.126 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:47:52.126 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.379251+0000", 2023-10-01T10:47:52.126 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.4451999999999999e-05 2023-10-01T10:47:52.126 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.126 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.126 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:47:52.126 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.379263+0000", 2023-10-01T10:47:52.127 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.2632999999999999e-05 2023-10-01T10:47:52.127 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.127 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.127 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:47:52.127 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.379271+0000", 2023-10-01T10:47:52.127 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.3760000000000002e-06 2023-10-01T10:47:52.127 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.127 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.127 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:47:52.127 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.379318+0000", 2023-10-01T10:47:52.127 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.7802000000000001e-05 2023-10-01T10:47:52.127 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.128 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.128 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:47:52.128 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.379456+0000", 2023-10-01T10:47:52.128 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000137832 2023-10-01T10:47:52.128 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.128 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.128 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 2,7", 2023-10-01T10:47:52.128 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.379603+0000", 2023-10-01T10:47:52.128 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00014640000000000001 2023-10-01T10:47:52.128 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.128 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.128 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:47:52.129 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.383268+0000", 2023-10-01T10:47:52.129 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.003665354 2023-10-01T10:47:52.129 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.129 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.129 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:47:52.129 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.385098+0000", 2023-10-01T10:47:52.129 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0018296689999999999 2023-10-01T10:47:52.129 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.129 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.129 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:47:52.129 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.553673+0000", 2023-10-01T10:47:52.129 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.168575055 2023-10-01T10:47:52.130 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.130 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.130 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:47:52.130 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.553777+0000", 2023-10-01T10:47:52.130 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00010442000000000001 2023-10-01T10:47:52.130 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.130 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.130 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:47:52.130 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.553824+0000", 2023-10-01T10:47:52.130 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.6996999999999997e-05 2023-10-01T10:47:52.130 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:47:52.130 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:47:52.131 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:47:52.131 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.131 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.131 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.45818.0:92553 10.8 10:14da9059:::benchmark_data_smithi078_135100_object92552:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-10-01T10:47:52.131 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:47:22.395779+0000", 2023-10-01T10:47:52.131 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 29.596696972, 2023-10-01T10:47:52.131 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.16324920200000001, 2023-10-01T10:47:52.131 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:47:52.131 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:47:52.131 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:47:52.131 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.45818", 2023-10-01T10:47:52.131 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1151403832", 2023-10-01T10:47:52.132 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 92553 2023-10-01T10:47:52.132 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.132 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:47:52.132 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.132 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:47:52.132 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.395779+0000", 2023-10-01T10:47:52.132 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:47:52.132 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.132 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.132 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:47:52.132 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.395779+0000", 2023-10-01T10:47:52.132 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:47:52.133 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.133 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.133 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:47:52.133 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.395777+0000", 2023-10-01T10:47:52.133 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999986 2023-10-01T10:47:52.133 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.133 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.133 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:47:52.133 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.395816+0000", 2023-10-01T10:47:52.133 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.8374999999999998e-05 2023-10-01T10:47:52.133 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.133 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.134 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:47:52.134 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.395831+0000", 2023-10-01T10:47:52.134 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.5356999999999999e-05 2023-10-01T10:47:52.134 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.134 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.134 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:47:52.134 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.395839+0000", 2023-10-01T10:47:52.134 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.5610000000000001e-06 2023-10-01T10:47:52.134 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.134 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.134 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:47:52.134 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.395890+0000", 2023-10-01T10:47:52.135 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.1159000000000003e-05 2023-10-01T10:47:52.135 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.135 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.135 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:47:52.135 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.396022+0000", 2023-10-01T10:47:52.135 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00013213799999999999 2023-10-01T10:47:52.135 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.135 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.135 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 2,7", 2023-10-01T10:47:52.135 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.396169+0000", 2023-10-01T10:47:52.135 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000146701 2023-10-01T10:47:52.135 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.135 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.136 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:47:52.136 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.398929+0000", 2023-10-01T10:47:52.136 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0027600699999999999 2023-10-01T10:47:52.136 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.136 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.136 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:47:52.136 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.403790+0000", 2023-10-01T10:47:52.136 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0048615569999999999 2023-10-01T10:47:52.136 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.136 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.136 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:47:52.137 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.558899+0000", 2023-10-01T10:47:52.137 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.155108738 2023-10-01T10:47:52.137 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.137 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.137 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:47:52.137 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.558980+0000", 2023-10-01T10:47:52.137 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.1069999999999995e-05 2023-10-01T10:47:52.137 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:47:52.137 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:47:52.137 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:47:52.137 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.559028+0000", 2023-10-01T10:47:52.137 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.8024e-05 2023-10-01T10:47:52.138 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:47:52.138 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:47:52.138 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:47:52.138 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:47:52.138 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:47:52.138 INFO:teuthology.orchestra.run.smithi078.stdout:} 2023-10-01T10:47:52.138 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:52.139 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:51 smithi173 bash[58816]: cluster 2023-10-01T10:47:49.870701+0000 mgr.y (mgr.25072) 906 : cluster 0 pgmap v837: 89 pgs: 89 active+clean; 6.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 40 MiB/s wr, 999 op/s 2023-10-01T10:47:52.139 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:51 smithi173 bash[58816]: audit 2023-10-01T10:47:51.112010+0000 mon.c (mon.2) 157 : audit 0 from='client.? 172.21.15.78:0/1296164947' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:47:52.173 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:52.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:52.175 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:47:52.275 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:52.316 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:52.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:52.321 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:47:52.421 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:52.463 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:52.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:52.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:47:52.569 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:52.596 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:52.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:52.602 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:52.703 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:52.735 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:52.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:52.740 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:47:52.841 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:52.869 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:52.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:52.874 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:47:52.974 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:52.993 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:52.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:52.995 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:47:53.095 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:53.128 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:53.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:53.133 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:47:53.234 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:53.268 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:53.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:53.271 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:53.372 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:53.399 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:53.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:53.405 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:47:53.505 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:53.535 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:53.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:53.543 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:47:53.643 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:53.680 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:53.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:53.683 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:47:53.730 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:53 smithi078 bash[75299]: cluster 2023-10-01T10:47:51.871235+0000 mgr.y (mgr.25072) 907 : cluster 0 pgmap v838: 89 pgs: 89 active+clean; 6.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 21 MiB/s wr, 704 op/s 2023-10-01T10:47:53.730 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:53 smithi078 bash[75299]: audit 2023-10-01T10:47:53.599206+0000 mon.a (mon.0) 1133 : audit 1 from='client.? 172.21.15.78:0/1171498637' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.4556301060760598}]: dispatch 2023-10-01T10:47:53.737 INFO:teuthology.orchestra.run.smithi078.stderr:set osd.6 primary-affinity to 0.45563 (8298602) 2023-10-01T10:47:53.784 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:53.792 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:53 smithi173 bash[58816]: cluster 2023-10-01T10:47:51.871235+0000 mgr.y (mgr.25072) 907 : cluster 0 pgmap v838: 89 pgs: 89 active+clean; 6.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 21 MiB/s wr, 704 op/s 2023-10-01T10:47:53.792 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:53 smithi173 bash[58816]: audit 2023-10-01T10:47:53.599206+0000 mon.a (mon.0) 1133 : audit 1 from='client.? 172.21.15.78:0/1171498637' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.4556301060760598}]: dispatch 2023-10-01T10:47:53.813 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:53.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:53.815 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:47:53.916 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:53.949 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:53.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:53.952 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:54.053 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:54.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:53 smithi078 bash[78274]: cluster 2023-10-01T10:47:51.871235+0000 mgr.y (mgr.25072) 907 : cluster 0 pgmap v838: 89 pgs: 89 active+clean; 6.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 21 MiB/s wr, 704 op/s 2023-10-01T10:47:54.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:53 smithi078 bash[78274]: audit 2023-10-01T10:47:53.599206+0000 mon.a (mon.0) 1133 : audit 1 from='client.? 172.21.15.78:0/1171498637' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.4556301060760598}]: dispatch 2023-10-01T10:47:54.085 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:54.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:54.089 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:47:54.189 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:54.214 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:54.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:54.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:47:54.317 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:54.350 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:54.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:54.353 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:47:54.453 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:54.478 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:54.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:54.480 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:47:54.581 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:54.602 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:54.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:54.605 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:47:54.705 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:54.737 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:54.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:54.740 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:47:54.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:54 smithi078 bash[75299]: cluster 2023-10-01T10:47:53.722005+0000 osd.5 (osd.5) 91 : cluster 3 failed to encode map e160 with expected crc 2023-10-01T10:47:54.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:54 smithi078 bash[75299]: audit 2023-10-01T10:47:53.722981+0000 mon.a (mon.0) 1134 : audit 1 from='client.? 172.21.15.78:0/1171498637' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.4556301060760598}]': finished 2023-10-01T10:47:54.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:54 smithi078 bash[75299]: cluster 2023-10-01T10:47:53.723049+0000 mon.a (mon.0) 1135 : cluster 0 osdmap e160: 8 total, 8 up, 8 in 2023-10-01T10:47:54.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:54 smithi078 bash[75299]: cluster 2023-10-01T10:47:53.723926+0000 osd.7 (osd.7) 109 : cluster 3 failed to encode map e160 with expected crc 2023-10-01T10:47:54.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:54 smithi078 bash[78274]: cluster 2023-10-01T10:47:53.722005+0000 osd.5 (osd.5) 91 : cluster 3 failed to encode map e160 with expected crc 2023-10-01T10:47:54.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:54 smithi078 bash[78274]: audit 2023-10-01T10:47:53.722981+0000 mon.a (mon.0) 1134 : audit 1 from='client.? 172.21.15.78:0/1171498637' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.4556301060760598}]': finished 2023-10-01T10:47:54.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:54 smithi078 bash[78274]: cluster 2023-10-01T10:47:53.723049+0000 mon.a (mon.0) 1135 : cluster 0 osdmap e160: 8 total, 8 up, 8 in 2023-10-01T10:47:54.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:54 smithi078 bash[78274]: cluster 2023-10-01T10:47:53.723926+0000 osd.7 (osd.7) 109 : cluster 3 failed to encode map e160 with expected crc 2023-10-01T10:47:54.841 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:54.867 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:54.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:54.870 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:47:54.970 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:55.009 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:55.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:55.012 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:47:55.113 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:55.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:54 smithi173 bash[58816]: cluster 2023-10-01T10:47:53.722005+0000 osd.5 (osd.5) 91 : cluster 3 failed to encode map e160 with expected crc 2023-10-01T10:47:55.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:54 smithi173 bash[58816]: audit 2023-10-01T10:47:53.722981+0000 mon.a (mon.0) 1134 : audit 1 from='client.? 172.21.15.78:0/1171498637' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 0.4556301060760598}]': finished 2023-10-01T10:47:55.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:54 smithi173 bash[58816]: cluster 2023-10-01T10:47:53.723049+0000 mon.a (mon.0) 1135 : cluster 0 osdmap e160: 8 total, 8 up, 8 in 2023-10-01T10:47:55.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:54 smithi173 bash[58816]: cluster 2023-10-01T10:47:53.723926+0000 osd.7 (osd.7) 109 : cluster 3 failed to encode map e160 with expected crc 2023-10-01T10:47:55.141 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:55.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:55.147 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:55.247 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:55.274 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:55.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:55.280 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:47:55.381 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:55.407 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:55.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:55.412 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:47:55.513 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:55.544 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:55.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:55.548 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:47:55.649 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:55.676 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:55.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:55.680 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:47:55.780 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:55.810 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:55.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:55.814 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:55.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:55 smithi078 bash[78274]: cluster 2023-10-01T10:47:53.871932+0000 mgr.y (mgr.25072) 908 : cluster 0 pgmap v840: 89 pgs: 89 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 5.9 MiB/s wr, 798 op/s 2023-10-01T10:47:55.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:55 smithi078 bash[78274]: cluster 2023-10-01T10:47:54.741700+0000 mon.a (mon.0) 1136 : cluster 0 osdmap e161: 8 total, 8 up, 8 in 2023-10-01T10:47:55.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:55 smithi078 bash[78274]: cluster 2023-10-01T10:47:54.741905+0000 osd.6 (osd.6) 75 : cluster 3 failed to encode map e161 with expected crc 2023-10-01T10:47:55.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:55 smithi078 bash[78274]: cluster 2023-10-01T10:47:54.744916+0000 osd.6 (osd.6) 76 : cluster 3 failed to encode map e161 with expected crc 2023-10-01T10:47:55.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:55 smithi078 bash[78274]: cluster 2023-10-01T10:47:54.745179+0000 osd.5 (osd.5) 92 : cluster 3 failed to encode map e161 with expected crc 2023-10-01T10:47:55.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:55 smithi078 bash[78274]: cluster 2023-10-01T10:47:54.764699+0000 osd.7 (osd.7) 110 : cluster 3 failed to encode map e161 with expected crc 2023-10-01T10:47:55.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:55 smithi078 bash[75299]: cluster 2023-10-01T10:47:53.871932+0000 mgr.y (mgr.25072) 908 : cluster 0 pgmap v840: 89 pgs: 89 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 5.9 MiB/s wr, 798 op/s 2023-10-01T10:47:55.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:55 smithi078 bash[75299]: cluster 2023-10-01T10:47:54.741700+0000 mon.a (mon.0) 1136 : cluster 0 osdmap e161: 8 total, 8 up, 8 in 2023-10-01T10:47:55.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:55 smithi078 bash[75299]: cluster 2023-10-01T10:47:54.741905+0000 osd.6 (osd.6) 75 : cluster 3 failed to encode map e161 with expected crc 2023-10-01T10:47:55.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:55 smithi078 bash[75299]: cluster 2023-10-01T10:47:54.744916+0000 osd.6 (osd.6) 76 : cluster 3 failed to encode map e161 with expected crc 2023-10-01T10:47:55.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:55 smithi078 bash[75299]: cluster 2023-10-01T10:47:54.745179+0000 osd.5 (osd.5) 92 : cluster 3 failed to encode map e161 with expected crc 2023-10-01T10:47:55.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:55 smithi078 bash[75299]: cluster 2023-10-01T10:47:54.764699+0000 osd.7 (osd.7) 110 : cluster 3 failed to encode map e161 with expected crc 2023-10-01T10:47:55.915 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:55.939 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:55.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:55.941 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:47:56.042 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:56.066 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:56.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:56.070 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:47:56.084 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-01T10:47:56.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:55 smithi173 bash[58816]: cluster 2023-10-01T10:47:53.871932+0000 mgr.y (mgr.25072) 908 : cluster 0 pgmap v840: 89 pgs: 89 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 5.9 MiB/s wr, 798 op/s 2023-10-01T10:47:56.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:55 smithi173 bash[58816]: cluster 2023-10-01T10:47:54.741700+0000 mon.a (mon.0) 1136 : cluster 0 osdmap e161: 8 total, 8 up, 8 in 2023-10-01T10:47:56.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:55 smithi173 bash[58816]: cluster 2023-10-01T10:47:54.741905+0000 osd.6 (osd.6) 75 : cluster 3 failed to encode map e161 with expected crc 2023-10-01T10:47:56.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:55 smithi173 bash[58816]: cluster 2023-10-01T10:47:54.744916+0000 osd.6 (osd.6) 76 : cluster 3 failed to encode map e161 with expected crc 2023-10-01T10:47:56.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:55 smithi173 bash[58816]: cluster 2023-10-01T10:47:54.745179+0000 osd.5 (osd.5) 92 : cluster 3 failed to encode map e161 with expected crc 2023-10-01T10:47:56.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:55 smithi173 bash[58816]: cluster 2023-10-01T10:47:54.764699+0000 osd.7 (osd.7) 110 : cluster 3 failed to encode map e161 with expected crc 2023-10-01T10:47:56.171 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:56.206 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:56.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:56.212 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:47:56.313 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:56.344 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:56.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:56.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:56.452 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:56.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:56.501 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:56.502 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:47:56.602 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:56.632 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:56.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:56.639 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:47:56.739 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:56.774 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:56.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:56.779 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:56.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:56 smithi078 bash[75299]: cluster 2023-10-01T10:47:56.737319+0000 mon.a (mon.0) 1137 : cluster 3 Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-01T10:47:56.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:56 smithi078 bash[78274]: cluster 2023-10-01T10:47:56.737319+0000 mon.a (mon.0) 1137 : cluster 3 Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-01T10:47:56.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:56 smithi173 bash[58816]: cluster 2023-10-01T10:47:56.737319+0000 mon.a (mon.0) 1137 : cluster 3 Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-01T10:47:56.880 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:56.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:56.933 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:56.933 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:57.033 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:57.062 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:57.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:57.063 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:47:57.164 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:57.195 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:57.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:57.199 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:47:57.300 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:57.331 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:57.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:57.340 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:47:57.441 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:57.468 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:57.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:57.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:47:57.571 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:57.608 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:57.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:57.612 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:57.713 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:57.742 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:57.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:57.746 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:47:57.811 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:57 smithi173 bash[58816]: cluster 2023-10-01T10:47:55.872364+0000 mgr.y (mgr.25072) 909 : cluster 0 pgmap v842: 89 pgs: 3 peering, 86 active+clean; 6.2 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 80 KiB/s wr, 949 op/s 2023-10-01T10:47:57.812 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:57 smithi173 bash[58816]: audit 2023-10-01T10:47:57.255355+0000 mon.c (mon.2) 158 : audit 1 from='client.? 172.21.15.78:0/1484567829' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:47:57.812 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:57 smithi173 bash[58816]: audit 2023-10-01T10:47:57.255752+0000 mon.a (mon.0) 1138 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:47:57.813 INFO:teuthology.orchestra.run.smithi078.stderr:nodeep-scrub is unset 2023-10-01T10:47:57.847 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:57.855 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T10:47:57.870 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:57.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:57.872 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:47:57.973 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:57.997 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:57.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:58.000 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:58.061 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:57 smithi078 bash[78274]: cluster 2023-10-01T10:47:55.872364+0000 mgr.y (mgr.25072) 909 : cluster 0 pgmap v842: 89 pgs: 3 peering, 86 active+clean; 6.2 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 80 KiB/s wr, 949 op/s 2023-10-01T10:47:58.061 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:57 smithi078 bash[78274]: audit 2023-10-01T10:47:57.255355+0000 mon.c (mon.2) 158 : audit 1 from='client.? 172.21.15.78:0/1484567829' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:47:58.061 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:57 smithi078 bash[78274]: audit 2023-10-01T10:47:57.255752+0000 mon.a (mon.0) 1138 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:47:58.062 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:57 smithi078 bash[75299]: cluster 2023-10-01T10:47:55.872364+0000 mgr.y (mgr.25072) 909 : cluster 0 pgmap v842: 89 pgs: 3 peering, 86 active+clean; 6.2 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 80 KiB/s wr, 949 op/s 2023-10-01T10:47:58.062 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:57 smithi078 bash[75299]: audit 2023-10-01T10:47:57.255355+0000 mon.c (mon.2) 158 : audit 1 from='client.? 172.21.15.78:0/1484567829' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:47:58.062 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:57 smithi078 bash[75299]: audit 2023-10-01T10:47:57.255752+0000 mon.a (mon.0) 1138 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:47:58.100 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:58.155 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:47:58.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:58.162 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:47:58.263 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:58.295 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:58.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:58.305 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:47:58.406 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:58.431 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:58.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:58.434 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:47:58.535 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:58.565 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:47:58.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:58.570 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:47:58.671 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:58.700 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:58.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:58.702 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:58.803 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:58.835 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:58.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:58.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:47:58.939 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:58.963 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:47:58.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:58.968 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:47:59.069 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:59.077 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:58 smithi173 bash[58816]: cluster 2023-10-01T10:47:57.789977+0000 mon.a (mon.0) 1139 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T10:47:59.077 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:58 smithi173 bash[58816]: cluster 2023-10-01T10:47:57.797485+0000 osd.5 (osd.5) 93 : cluster 3 failed to encode map e162 with expected crc 2023-10-01T10:47:59.077 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:58 smithi173 bash[58816]: cluster 2023-10-01T10:47:57.800035+0000 osd.7 (osd.7) 111 : cluster 3 failed to encode map e162 with expected crc 2023-10-01T10:47:59.078 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:58 smithi173 bash[58816]: audit 2023-10-01T10:47:57.800449+0000 mon.a (mon.0) 1140 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T10:47:59.078 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:58 smithi173 bash[58816]: cluster 2023-10-01T10:47:57.800583+0000 mon.a (mon.0) 1141 : cluster 0 osdmap e162: 8 total, 8 up, 8 in 2023-10-01T10:47:59.078 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:58 smithi173 bash[58816]: cluster 2023-10-01T10:47:57.818304+0000 osd.6 (osd.6) 77 : cluster 3 failed to encode map e162 with expected crc 2023-10-01T10:47:59.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:58 smithi078 bash[75299]: cluster 2023-10-01T10:47:57.789977+0000 mon.a (mon.0) 1139 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T10:47:59.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:58 smithi078 bash[75299]: cluster 2023-10-01T10:47:57.797485+0000 osd.5 (osd.5) 93 : cluster 3 failed to encode map e162 with expected crc 2023-10-01T10:47:59.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:58 smithi078 bash[75299]: cluster 2023-10-01T10:47:57.800035+0000 osd.7 (osd.7) 111 : cluster 3 failed to encode map e162 with expected crc 2023-10-01T10:47:59.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:58 smithi078 bash[75299]: audit 2023-10-01T10:47:57.800449+0000 mon.a (mon.0) 1140 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T10:47:59.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:58 smithi078 bash[75299]: cluster 2023-10-01T10:47:57.800583+0000 mon.a (mon.0) 1141 : cluster 0 osdmap e162: 8 total, 8 up, 8 in 2023-10-01T10:47:59.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:58 smithi078 bash[75299]: cluster 2023-10-01T10:47:57.818304+0000 osd.6 (osd.6) 77 : cluster 3 failed to encode map e162 with expected crc 2023-10-01T10:47:59.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:58 smithi078 bash[78274]: cluster 2023-10-01T10:47:57.789977+0000 mon.a (mon.0) 1139 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T10:47:59.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:58 smithi078 bash[78274]: cluster 2023-10-01T10:47:57.797485+0000 osd.5 (osd.5) 93 : cluster 3 failed to encode map e162 with expected crc 2023-10-01T10:47:59.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:58 smithi078 bash[78274]: cluster 2023-10-01T10:47:57.800035+0000 osd.7 (osd.7) 111 : cluster 3 failed to encode map e162 with expected crc 2023-10-01T10:47:59.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:58 smithi078 bash[78274]: audit 2023-10-01T10:47:57.800449+0000 mon.a (mon.0) 1140 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T10:47:59.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:58 smithi078 bash[78274]: cluster 2023-10-01T10:47:57.800583+0000 mon.a (mon.0) 1141 : cluster 0 osdmap e162: 8 total, 8 up, 8 in 2023-10-01T10:47:59.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:58 smithi078 bash[78274]: cluster 2023-10-01T10:47:57.818304+0000 osd.6 (osd.6) 77 : cluster 3 failed to encode map e162 with expected crc 2023-10-01T10:47:59.097 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:59.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:59.102 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:47:59.202 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:59.228 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:59.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:59.231 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:47:59.332 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:59.374 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:47:59.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:59.379 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:47:59.479 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:59.510 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:47:59.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:59.514 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:47:59.615 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:59.646 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:47:59.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:59.658 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:47:59.759 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:59.785 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:47:59.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:59.793 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:47:59.894 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:59.922 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:47:59.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:47:59.929 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:48:00.030 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:00.039 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:47:59 smithi173 bash[58816]: cluster 2023-10-01T10:47:57.872829+0000 mgr.y (mgr.25072) 910 : cluster 0 pgmap v844: 89 pgs: 3 peering, 86 active+clean; 6.2 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 657 op/s 2023-10-01T10:48:00.059 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:00.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:00.062 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:48:00.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:47:59 smithi078 bash[75299]: cluster 2023-10-01T10:47:57.872829+0000 mgr.y (mgr.25072) 910 : cluster 0 pgmap v844: 89 pgs: 3 peering, 86 active+clean; 6.2 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 657 op/s 2023-10-01T10:48:00.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:47:59 smithi078 bash[78274]: cluster 2023-10-01T10:47:57.872829+0000 mgr.y (mgr.25072) 910 : cluster 0 pgmap v844: 89 pgs: 3 peering, 86 active+clean; 6.2 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 657 op/s 2023-10-01T10:48:00.162 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:00.201 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:00.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:00.207 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:48:00.309 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:00.346 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:00.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:00.349 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:48:00.449 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:00.477 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:00.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:00.480 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:48:00.580 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:00.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:48:00 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:48:00] "GET /metrics HTTP/1.1" 200 36311 "" "Prometheus/2.33.4" 2023-10-01T10:48:00.610 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:00.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:00.616 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:48:00.717 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:00.752 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:00.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:00.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:48:00.861 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:00.871 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:00 smithi078 bash[75299]: cluster 2023-10-01T10:47:59.873494+0000 mgr.y (mgr.25072) 911 : cluster 0 pgmap v845: 89 pgs: 89 active+clean; 6.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 693 op/s 2023-10-01T10:48:00.871 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:00 smithi078 bash[78274]: cluster 2023-10-01T10:47:59.873494+0000 mgr.y (mgr.25072) 911 : cluster 0 pgmap v845: 89 pgs: 89 active+clean; 6.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 693 op/s 2023-10-01T10:48:00.889 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:00.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:00.891 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:48:00.992 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:01.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:01.032 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:01.032 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:48:01.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:00 smithi173 bash[58816]: cluster 2023-10-01T10:47:59.873494+0000 mgr.y (mgr.25072) 911 : cluster 0 pgmap v845: 89 pgs: 89 active+clean; 6.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 693 op/s 2023-10-01T10:48:01.132 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:01.144 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T10:48:01.158 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:01.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:01.161 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:48:01.262 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:01.295 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:01.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:01.303 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:48:01.403 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:01.428 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:01.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:01.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:48:01.537 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:01.570 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T10:48:01.571 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-01T10:48:01.572 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:01.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:01.578 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:48:01.667 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:48:01.679 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:01.702 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:01.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:01.705 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:48:01.806 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:01.849 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:01.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:01.852 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:01.953 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:01.961 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:01 smithi173 bash[58816]: cluster 2023-10-01T10:48:00.821034+0000 mon.a (mon.0) 1142 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-10-01T10:48:01.961 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:01 smithi173 bash[58816]: cluster 2023-10-01T10:48:00.821082+0000 mon.a (mon.0) 1143 : cluster 1 Cluster is now healthy 2023-10-01T10:48:01.976 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:01.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:01.981 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:48:02.082 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:02.113 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:02.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:02.117 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:48:02.218 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:02.251 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:02.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:02.257 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:48:02.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:01 smithi078 bash[78274]: cluster 2023-10-01T10:48:00.821034+0000 mon.a (mon.0) 1142 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-10-01T10:48:02.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:01 smithi078 bash[78274]: cluster 2023-10-01T10:48:00.821082+0000 mon.a (mon.0) 1143 : cluster 1 Cluster is now healthy 2023-10-01T10:48:02.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:01 smithi078 bash[75299]: cluster 2023-10-01T10:48:00.821034+0000 mon.a (mon.0) 1142 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-10-01T10:48:02.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:01 smithi078 bash[75299]: cluster 2023-10-01T10:48:00.821082+0000 mon.a (mon.0) 1143 : cluster 1 Cluster is now healthy 2023-10-01T10:48:02.359 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:02.383 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:02.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:02.387 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:02.488 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:02.517 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:02.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:02.522 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:02.623 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:02.657 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:02.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:02.662 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:48:02.686 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:48:02.763 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:02.795 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:02.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:02.796 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:48:02.897 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:02.927 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:02.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:02.931 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:48:03.032 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:03.055 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:03.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:03.060 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:48:03.160 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:03.184 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:03.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:03.188 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:48:03.289 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:03.329 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:03.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:03.335 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:48:03.434 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:03 smithi078 bash[75299]: cluster 2023-10-01T10:48:01.874060+0000 mgr.y (mgr.25072) 912 : cluster 0 pgmap v846: 89 pgs: 89 active+clean; 6.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 605 op/s 2023-10-01T10:48:03.434 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:03 smithi078 bash[75299]: audit 2023-10-01T10:48:02.432629+0000 mon.a (mon.0) 1144 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:48:03.434 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:03 smithi078 bash[75299]: audit 2023-10-01T10:48:02.433959+0000 mon.a (mon.0) 1145 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:48:03.437 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:03.475 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:03.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:03.479 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:48:03.579 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:03.589 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:03 smithi173 bash[58816]: cluster 2023-10-01T10:48:01.874060+0000 mgr.y (mgr.25072) 912 : cluster 0 pgmap v846: 89 pgs: 89 active+clean; 6.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 605 op/s 2023-10-01T10:48:03.589 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:03 smithi173 bash[58816]: audit 2023-10-01T10:48:02.432629+0000 mon.a (mon.0) 1144 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:48:03.589 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:03 smithi173 bash[58816]: audit 2023-10-01T10:48:02.433959+0000 mon.a (mon.0) 1145 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:48:03.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:03.608 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:03.608 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:48:03.708 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:03.733 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:03 smithi078 bash[78274]: cluster 2023-10-01T10:48:01.874060+0000 mgr.y (mgr.25072) 912 : cluster 0 pgmap v846: 89 pgs: 89 active+clean; 6.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 605 op/s 2023-10-01T10:48:03.733 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:03 smithi078 bash[78274]: audit 2023-10-01T10:48:02.432629+0000 mon.a (mon.0) 1144 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:48:03.733 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:03 smithi078 bash[78274]: audit 2023-10-01T10:48:02.433959+0000 mon.a (mon.0) 1145 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:48:03.768 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:03.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:03.776 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:48:03.877 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:03.906 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:03.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:03.913 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:48:04.014 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:04.051 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:04.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:04.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:48:04.154 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:04.179 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:04.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:04.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:48:04.283 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:04.307 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:04.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:04.310 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:48:04.410 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:04.442 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:04.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:04.447 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:48:04.547 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:04.574 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:04.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:04.576 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:48:04.677 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:04.702 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:04.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:04.707 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:48:04.808 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:04.849 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:04.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:04.854 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:48:04.955 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:04.991 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:04.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:04.997 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:48:05.098 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:05.126 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:05.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:05.133 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:48:05.234 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:05.255 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:05.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:05.260 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:05.360 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:05.384 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:05.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:05.388 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:48:05.438 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:05 smithi078 bash[75299]: cluster 2023-10-01T10:48:03.874571+0000 mgr.y (mgr.25072) 913 : cluster 0 pgmap v847: 89 pgs: 89 active+clean; 5.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 648 op/s 2023-10-01T10:48:05.488 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:05.513 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:05.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:05.515 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:48:05.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:05 smithi078 bash[78274]: cluster 2023-10-01T10:48:03.874571+0000 mgr.y (mgr.25072) 913 : cluster 0 pgmap v847: 89 pgs: 89 active+clean; 5.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 648 op/s 2023-10-01T10:48:05.616 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:05.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:05 smithi173 bash[58816]: cluster 2023-10-01T10:48:03.874571+0000 mgr.y (mgr.25072) 913 : cluster 0 pgmap v847: 89 pgs: 89 active+clean; 5.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 648 op/s 2023-10-01T10:48:05.652 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:05.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:05.654 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:05.755 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:05.783 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:05.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:05.785 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:48:05.886 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:05.917 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:05.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:05.920 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:48:06.021 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:06.051 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:06.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:06.055 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:48:06.155 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:06.194 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:06.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:06.199 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:48:06.299 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:06.324 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:06.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:06.328 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:48:06.429 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:06.454 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:06.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:06.458 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:48:06.558 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:06.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:06.589 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:06.589 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:06.690 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:06.722 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:06.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:06.724 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:48:06.825 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:06.858 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:06.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:06.861 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:48:06.961 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:06.990 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:06.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:06.996 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:48:07.097 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:07.130 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:07.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:07.132 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:48:07.232 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:07.261 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:07.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:07.263 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:48:07.364 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:07.399 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:07.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:07.403 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:48:07.435 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:48:07.435 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:48:07.504 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:07.538 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:07.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:07.540 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:48:07.641 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:07.650 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:07 smithi173 bash[58816]: cluster 2023-10-01T10:48:05.875098+0000 mgr.y (mgr.25072) 914 : cluster 0 pgmap v848: 89 pgs: 89 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 710 op/s 2023-10-01T10:48:07.672 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:07.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:07.675 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:48:07.776 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:07.802 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:07.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:07.807 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:48:07.843 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:07 smithi078 bash[75299]: cluster 2023-10-01T10:48:05.875098+0000 mgr.y (mgr.25072) 914 : cluster 0 pgmap v848: 89 pgs: 89 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 710 op/s 2023-10-01T10:48:07.843 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:07 smithi078 bash[78274]: cluster 2023-10-01T10:48:05.875098+0000 mgr.y (mgr.25072) 914 : cluster 0 pgmap v848: 89 pgs: 89 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 710 op/s 2023-10-01T10:48:07.908 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:07.942 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:07.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:07.945 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:48:08.046 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:08.070 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:08.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:08.073 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:08.174 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:08.209 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:08.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:08.212 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:48:08.313 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:08.337 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:08.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:08.340 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:48:08.440 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:08.476 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:08.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:08.482 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:48:08.583 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:08.607 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:08.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:08.609 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:48:08.710 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:08.740 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:08.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:08.748 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:48:08.844 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:08 smithi173 bash[58816]: audit 2023-10-01T10:48:07.833422+0000 mon.a (mon.0) 1146 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:48:08.844 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:08 smithi173 bash[58816]: cluster 2023-10-01T10:48:07.875582+0000 mgr.y (mgr.25072) 915 : cluster 0 pgmap v849: 89 pgs: 89 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 739 op/s 2023-10-01T10:48:08.849 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:08.878 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:08.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:08.880 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:48:08.981 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:08.988 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:08 smithi078 bash[75299]: audit 2023-10-01T10:48:07.833422+0000 mon.a (mon.0) 1146 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:48:08.988 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:08 smithi078 bash[75299]: cluster 2023-10-01T10:48:07.875582+0000 mgr.y (mgr.25072) 915 : cluster 0 pgmap v849: 89 pgs: 89 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 739 op/s 2023-10-01T10:48:08.989 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:08 smithi078 bash[78274]: audit 2023-10-01T10:48:07.833422+0000 mon.a (mon.0) 1146 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:48:08.989 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:08 smithi078 bash[78274]: cluster 2023-10-01T10:48:07.875582+0000 mgr.y (mgr.25072) 915 : cluster 0 pgmap v849: 89 pgs: 89 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 739 op/s 2023-10-01T10:48:09.010 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:09.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:09.011 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:48:09.112 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:09.150 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:09.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:09.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:48:09.256 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:09.290 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:09.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:09.296 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:48:09.397 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:09.422 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:09.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:09.431 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:48:09.533 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:09.566 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:09.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:09.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:48:09.673 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:09.699 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:09.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:09.704 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:09.805 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:09.844 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:09.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:09.846 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:48:09.947 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:09.969 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:09.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:09.972 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:48:10.073 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:10.106 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:10.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:10.196 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:48:10.297 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:10.330 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:10.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:10.336 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:10.437 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:10.480 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:10.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:10.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:48:10.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:48:10 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:48:10] "GET /metrics HTTP/1.1" 200 36319 "" "Prometheus/2.33.4" 2023-10-01T10:48:10.586 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:10.632 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:10.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:10.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:48:10.737 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:10.778 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:10.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:10.781 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:10.881 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:10.904 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:10.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:10.911 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:48:11.012 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:11.041 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:11.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:11.049 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:48:11.150 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:11.160 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:10 smithi173 bash[58816]: cluster 2023-10-01T10:48:09.876273+0000 mgr.y (mgr.25072) 916 : cluster 0 pgmap v850: 89 pgs: 89 active+clean; 5.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 893 op/s 2023-10-01T10:48:11.175 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:11.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:11.179 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:48:11.279 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:11.289 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:10 smithi078 bash[75299]: cluster 2023-10-01T10:48:09.876273+0000 mgr.y (mgr.25072) 916 : cluster 0 pgmap v850: 89 pgs: 89 active+clean; 5.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 893 op/s 2023-10-01T10:48:11.289 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:10 smithi078 bash[78274]: cluster 2023-10-01T10:48:09.876273+0000 mgr.y (mgr.25072) 916 : cluster 0 pgmap v850: 89 pgs: 89 active+clean; 5.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 893 op/s 2023-10-01T10:48:11.306 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:11.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:11.315 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:11.415 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:11.444 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:11.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:11.448 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:11.515 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T10:48:11.549 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:11.553 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2023-10-01T10:48:11.553 INFO:teuthology.orchestra.run.smithi078.stdout: "ops": [ 2023-10-01T10:48:11.553 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:11.553 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.45818.0:131450 10.2 10:49f7efbc:::benchmark_data_smithi078_135100_object21537:head [delete] snapc 0=[] ondisk+write+known_if_redirected e162)", 2023-10-01T10:48:11.553 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:48:11.549326+0000", 2023-10-01T10:48:11.553 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 0.0024275519999999999, 2023-10-01T10:48:11.553 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.002460556, 2023-10-01T10:48:11.554 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:48:11.554 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "waiting for sub ops", 2023-10-01T10:48:11.554 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:48:11.554 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.45818", 2023-10-01T10:48:11.554 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1151403832", 2023-10-01T10:48:11.554 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 131450 2023-10-01T10:48:11.554 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:11.554 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:48:11.554 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:11.554 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:48:11.555 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:11.549326+0000", 2023-10-01T10:48:11.555 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:48:11.555 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:11.555 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:11.555 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:48:11.555 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:11.549326+0000", 2023-10-01T10:48:11.555 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:48:11.555 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:11.555 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:11.555 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:48:11.555 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:11.549324+0000", 2023-10-01T10:48:11.556 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999981 2023-10-01T10:48:11.556 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:11.556 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:11.556 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:48:11.556 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:11.549334+0000", 2023-10-01T10:48:11.556 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.0149999999999999e-05 2023-10-01T10:48:11.556 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:11.556 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:11.556 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:48:11.556 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:11.549351+0000", 2023-10-01T10:48:11.557 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.7251000000000001e-05 2023-10-01T10:48:11.557 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:11.557 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:11.557 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:48:11.557 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:11.549361+0000", 2023-10-01T10:48:11.557 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.0115000000000001e-05 2023-10-01T10:48:11.557 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:11.557 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:11.557 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:48:11.557 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:11.549422+0000", 2023-10-01T10:48:11.558 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.0451000000000001e-05 2023-10-01T10:48:11.558 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:11.558 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:11.558 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:48:11.558 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:11.549575+0000", 2023-10-01T10:48:11.558 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00015355999999999999 2023-10-01T10:48:11.558 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:11.558 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:11.558 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 0,4", 2023-10-01T10:48:11.558 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:11.549703+0000", 2023-10-01T10:48:11.558 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00012835099999999999 2023-10-01T10:48:11.559 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:48:11.559 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:48:11.559 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:48:11.559 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:48:11.559 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2023-10-01T10:48:11.559 INFO:teuthology.orchestra.run.smithi078.stdout: "num_ops": 1 2023-10-01T10:48:11.559 INFO:teuthology.orchestra.run.smithi078.stdout:} 2023-10-01T10:48:11.575 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T10:48:11.581 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:11.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:11.584 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:48:11.616 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T10:48:11.652 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T10:48:11.684 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T10:48:11.685 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:11.711 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:11.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:11.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:48:11.720 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T10:48:11.757 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T10:48:11.798 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T10:48:11.805 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T10:48:11.806 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T10:48:11.806 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T10:48:11.806 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T10:48:11.806 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T10:48:11.806 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T10:48:11.806 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T10:48:11.806 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T10:48:11.815 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:11.849 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:11.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:11.855 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:48:11.955 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:12.000 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:12.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:12.006 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:48:12.107 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:12.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:11 smithi173 bash[58816]: audit 2023-10-01T10:48:11.464517+0000 mon.a (mon.0) 1147 : audit 0 from='client.? 172.21.15.78:0/1415718284' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:48:12.130 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:12.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:12.136 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:12.152 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:11 smithi078 bash[75299]: audit 2023-10-01T10:48:11.464517+0000 mon.a (mon.0) 1147 : audit 0 from='client.? 172.21.15.78:0/1415718284' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:48:12.152 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:11 smithi078 bash[78274]: audit 2023-10-01T10:48:11.464517+0000 mon.a (mon.0) 1147 : audit 0 from='client.? 172.21.15.78:0/1415718284' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:48:12.237 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:12.269 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:12.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:12.273 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:48:12.373 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:12.395 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:12.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:12.399 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:48:12.500 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:12.531 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:12.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:12.537 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:48:12.637 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:12.663 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:12.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:12.668 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:48:12.769 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:12.805 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:12.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:12.811 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:48:12.912 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:12.945 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:12.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:12.948 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:48:13.049 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:13.084 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:13.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:13.087 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:48:13.188 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:13.218 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:13.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:13.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:48:13.325 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:13.343 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:13.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:13.346 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:48:13.447 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:13.472 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:13.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:13.481 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:48:13.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:13 smithi078 bash[78274]: cluster 2023-10-01T10:48:11.876835+0000 mgr.y (mgr.25072) 917 : cluster 0 pgmap v851: 89 pgs: 89 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 720 op/s 2023-10-01T10:48:13.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:13 smithi078 bash[75299]: cluster 2023-10-01T10:48:11.876835+0000 mgr.y (mgr.25072) 917 : cluster 0 pgmap v851: 89 pgs: 89 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 720 op/s 2023-10-01T10:48:13.582 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:13.617 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:13.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:13.620 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:48:13.720 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:13.729 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:13 smithi173 bash[58816]: cluster 2023-10-01T10:48:11.876835+0000 mgr.y (mgr.25072) 917 : cluster 0 pgmap v851: 89 pgs: 89 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 720 op/s 2023-10-01T10:48:13.761 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:13.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:13.766 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:48:13.867 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:13.890 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:13.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:13.898 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:13.999 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:14.022 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:14.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:14.027 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:48:14.128 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:14.149 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:14.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:14.151 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:14.252 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:14.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:14.301 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:14.301 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:48:14.402 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:14.438 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:14.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:14.444 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:14.545 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:14.569 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:14.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:14.572 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:14.673 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:14.697 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:14.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:14.701 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:48:14.801 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:14.831 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:14.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:14.835 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:48:14.936 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:14.960 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:14.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:14.965 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:15.065 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:15.091 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:15.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:15.096 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:48:15.197 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:15.239 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:15.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:15.242 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:48:15.343 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:15.371 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:15.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:15.374 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:48:15.475 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:15.503 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:15.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:15.508 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:15.609 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:15.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:15 smithi173 bash[58816]: cluster 2023-10-01T10:48:13.877421+0000 mgr.y (mgr.25072) 918 : cluster 0 pgmap v852: 89 pgs: 89 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 804 op/s 2023-10-01T10:48:15.636 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:15.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:15.641 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:48:15.742 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:15.787 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:15.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:15.789 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:48:15.807 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:15 smithi078 bash[75299]: cluster 2023-10-01T10:48:13.877421+0000 mgr.y (mgr.25072) 918 : cluster 0 pgmap v852: 89 pgs: 89 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 804 op/s 2023-10-01T10:48:15.807 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:15 smithi078 bash[78274]: cluster 2023-10-01T10:48:13.877421+0000 mgr.y (mgr.25072) 918 : cluster 0 pgmap v852: 89 pgs: 89 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 804 op/s 2023-10-01T10:48:15.808 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-01T10:48:15.890 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:15.928 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:15.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:15.933 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:48:16.034 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:16.067 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:16.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:16.079 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:48:16.180 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:16.210 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:16.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:16.214 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:48:16.315 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:16.341 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:16.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:16.343 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:16.444 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:16.480 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:16.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:16.486 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:48:16.586 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:16.621 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:16.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:16.626 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:16.727 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:16.764 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:16.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:16.771 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:48:16.872 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:16.899 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:16.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:16.901 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:17.002 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:17.032 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:17.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:17.035 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:17.135 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:17.159 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:17.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:17.162 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:17.262 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:17.286 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:17.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:17.289 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:48:17.390 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:17.415 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:17.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:17.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:48:17.519 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:17.553 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:17.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:17.555 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:17.656 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:17.664 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:17 smithi078 bash[75299]: cluster 2023-10-01T10:48:15.877954+0000 mgr.y (mgr.25072) 919 : cluster 0 pgmap v853: 89 pgs: 89 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 741 op/s 2023-10-01T10:48:17.665 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:17 smithi078 bash[75299]: audit 2023-10-01T10:48:17.429968+0000 mon.a (mon.0) 1148 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:48:17.665 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:17 smithi078 bash[78274]: cluster 2023-10-01T10:48:15.877954+0000 mgr.y (mgr.25072) 919 : cluster 0 pgmap v853: 89 pgs: 89 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 741 op/s 2023-10-01T10:48:17.665 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:17 smithi078 bash[78274]: audit 2023-10-01T10:48:17.429968+0000 mon.a (mon.0) 1148 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:48:17.688 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:17.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:17.692 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:48:17.793 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:17.802 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:17 smithi173 bash[58816]: cluster 2023-10-01T10:48:15.877954+0000 mgr.y (mgr.25072) 919 : cluster 0 pgmap v853: 89 pgs: 89 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 741 op/s 2023-10-01T10:48:17.802 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:17 smithi173 bash[58816]: audit 2023-10-01T10:48:17.429968+0000 mon.a (mon.0) 1148 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:48:17.821 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:17.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:17.825 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:48:17.926 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:17.949 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:17.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:17.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:48:18.050 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T10:48:18.058 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:18.134 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:18.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:18.136 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:48:18.237 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:18.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:18.266 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:18.266 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:48:18.367 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:18.401 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:18.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:18.404 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:48:18.505 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:18.540 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:18.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:18.542 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:48:18.643 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:18.671 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:18.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:18.675 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:48:18.776 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:18.801 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:18.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:18.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:48:18.908 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:18.932 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:18.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:18.937 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:48:19.038 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:19.066 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:19.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:19.071 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:48:19.172 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:19.205 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:19.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:19.210 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:48:19.311 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:19.344 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:19.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:19.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:48:19.449 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:19.482 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:19.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:19.490 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:48:19.590 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:19.620 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:19.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:19.627 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:48:19.727 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:19.737 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:19 smithi078 bash[75299]: cluster 2023-10-01T10:48:17.878460+0000 mgr.y (mgr.25072) 920 : cluster 0 pgmap v854: 89 pgs: 89 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2023-10-01T10:48:19.737 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:19 smithi078 bash[78274]: cluster 2023-10-01T10:48:17.878460+0000 mgr.y (mgr.25072) 920 : cluster 0 pgmap v854: 89 pgs: 89 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2023-10-01T10:48:19.763 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:19.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:19.768 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:19.870 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:19.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:19 smithi173 bash[58816]: cluster 2023-10-01T10:48:17.878460+0000 mgr.y (mgr.25072) 920 : cluster 0 pgmap v854: 89 pgs: 89 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2023-10-01T10:48:19.895 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:19.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:19.899 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:20.000 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:20.047 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:20.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:20.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:48:20.152 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:20.179 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:20.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:20.185 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:48:20.285 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:20.316 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:20.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:20.319 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:48:20.420 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:20.428 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:48:20 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:48:20] "GET /metrics HTTP/1.1" 200 36319 "" "Prometheus/2.33.4" 2023-10-01T10:48:20.453 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:20.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:20.460 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:48:20.476 INFO:teuthology.orchestra.run.smithi078.stderr:reweighted osd.0 to 1 (10000) 2023-10-01T10:48:20.479 INFO:teuthology.orchestra.run.smithi078.stderr:noscrub is set 2023-10-01T10:48:20.561 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:20.590 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:20.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:20.593 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:48:20.694 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:20.702 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:20 smithi078 bash[78274]: audit 2023-10-01T10:48:19.771893+0000 mon.c (mon.2) 159 : audit 1 from='client.? 172.21.15.78:0/3587428465' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:48:20.702 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:20 smithi078 bash[78274]: audit 2023-10-01T10:48:19.773125+0000 mon.a (mon.0) 1149 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:48:20.702 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:20 smithi078 bash[78274]: audit 2023-10-01T10:48:19.806051+0000 mon.a (mon.0) 1150 : audit 1 from='client.? 172.21.15.78:0/2616515176' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T10:48:20.703 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:20 smithi078 bash[75299]: audit 2023-10-01T10:48:19.771893+0000 mon.c (mon.2) 159 : audit 1 from='client.? 172.21.15.78:0/3587428465' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:48:20.703 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:20 smithi078 bash[75299]: audit 2023-10-01T10:48:19.773125+0000 mon.a (mon.0) 1149 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:48:20.703 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:20 smithi078 bash[75299]: audit 2023-10-01T10:48:19.806051+0000 mon.a (mon.0) 1150 : audit 1 from='client.? 172.21.15.78:0/2616515176' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T10:48:20.725 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:20.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:20.733 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:48:20.833 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:20.863 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:20.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:20 smithi173 bash[58816]: audit 2023-10-01T10:48:19.771893+0000 mon.c (mon.2) 159 : audit 1 from='client.? 172.21.15.78:0/3587428465' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:48:20.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:20 smithi173 bash[58816]: audit 2023-10-01T10:48:19.773125+0000 mon.a (mon.0) 1149 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:48:20.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:20 smithi173 bash[58816]: audit 2023-10-01T10:48:19.806051+0000 mon.a (mon.0) 1150 : audit 1 from='client.? 172.21.15.78:0/2616515176' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T10:48:20.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:20.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:48:20.972 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:21.009 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:21.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:21.012 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:48:21.113 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:21.144 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:21.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:21.147 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:48:21.248 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:21.272 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:21.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:21.274 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:48:21.375 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:21.399 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:21.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:21.404 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:21.453 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:21 smithi173 bash[58816]: cluster 2023-10-01T10:48:19.879092+0000 mgr.y (mgr.25072) 921 : cluster 0 pgmap v855: 89 pgs: 89 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 896 op/s 2023-10-01T10:48:21.454 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:21 smithi173 bash[58816]: cluster 2023-10-01T10:48:20.445642+0000 mon.a (mon.0) 1151 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:48:21.454 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:21 smithi173 bash[58816]: cluster 2023-10-01T10:48:20.464458+0000 osd.5 (osd.5) 94 : cluster 3 failed to encode map e163 with expected crc 2023-10-01T10:48:21.454 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:21 smithi173 bash[58816]: audit 2023-10-01T10:48:20.466066+0000 mon.a (mon.0) 1152 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T10:48:21.454 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:21 smithi173 bash[58816]: audit 2023-10-01T10:48:20.466196+0000 mon.a (mon.0) 1153 : audit 1 from='client.? 172.21.15.78:0/2616515176' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-01T10:48:21.454 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:21 smithi173 bash[58816]: cluster 2023-10-01T10:48:20.466324+0000 mon.a (mon.0) 1154 : cluster 0 osdmap e163: 8 total, 8 up, 8 in 2023-10-01T10:48:21.454 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:21 smithi173 bash[58816]: cluster 2023-10-01T10:48:20.472292+0000 osd.7 (osd.7) 112 : cluster 3 failed to encode map e163 with expected crc 2023-10-01T10:48:21.454 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:21 smithi173 bash[58816]: cluster 2023-10-01T10:48:20.476123+0000 osd.6 (osd.6) 78 : cluster 3 failed to encode map e163 with expected crc 2023-10-01T10:48:21.454 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:21 smithi173 bash[58816]: cluster 2023-10-01T10:48:20.477405+0000 osd.7 (osd.7) 113 : cluster 3 failed to encode map e163 with expected crc 2023-10-01T10:48:21.466 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:21 smithi078 bash[78274]: cluster 2023-10-01T10:48:19.879092+0000 mgr.y (mgr.25072) 921 : cluster 0 pgmap v855: 89 pgs: 89 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 896 op/s 2023-10-01T10:48:21.466 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:21 smithi078 bash[78274]: cluster 2023-10-01T10:48:20.445642+0000 mon.a (mon.0) 1151 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:48:21.466 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:21 smithi078 bash[78274]: cluster 2023-10-01T10:48:20.464458+0000 osd.5 (osd.5) 94 : cluster 3 failed to encode map e163 with expected crc 2023-10-01T10:48:21.466 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:21 smithi078 bash[78274]: audit 2023-10-01T10:48:20.466066+0000 mon.a (mon.0) 1152 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T10:48:21.467 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:21 smithi078 bash[78274]: audit 2023-10-01T10:48:20.466196+0000 mon.a (mon.0) 1153 : audit 1 from='client.? 172.21.15.78:0/2616515176' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-01T10:48:21.467 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:21 smithi078 bash[78274]: cluster 2023-10-01T10:48:20.466324+0000 mon.a (mon.0) 1154 : cluster 0 osdmap e163: 8 total, 8 up, 8 in 2023-10-01T10:48:21.467 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:21 smithi078 bash[78274]: cluster 2023-10-01T10:48:20.472292+0000 osd.7 (osd.7) 112 : cluster 3 failed to encode map e163 with expected crc 2023-10-01T10:48:21.467 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:21 smithi078 bash[78274]: cluster 2023-10-01T10:48:20.476123+0000 osd.6 (osd.6) 78 : cluster 3 failed to encode map e163 with expected crc 2023-10-01T10:48:21.467 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:21 smithi078 bash[78274]: cluster 2023-10-01T10:48:20.477405+0000 osd.7 (osd.7) 113 : cluster 3 failed to encode map e163 with expected crc 2023-10-01T10:48:21.467 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:21 smithi078 bash[75299]: cluster 2023-10-01T10:48:19.879092+0000 mgr.y (mgr.25072) 921 : cluster 0 pgmap v855: 89 pgs: 89 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 896 op/s 2023-10-01T10:48:21.467 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:21 smithi078 bash[75299]: cluster 2023-10-01T10:48:20.445642+0000 mon.a (mon.0) 1151 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:48:21.467 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:21 smithi078 bash[75299]: cluster 2023-10-01T10:48:20.464458+0000 osd.5 (osd.5) 94 : cluster 3 failed to encode map e163 with expected crc 2023-10-01T10:48:21.467 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:21 smithi078 bash[75299]: audit 2023-10-01T10:48:20.466066+0000 mon.a (mon.0) 1152 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T10:48:21.467 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:21 smithi078 bash[75299]: audit 2023-10-01T10:48:20.466196+0000 mon.a (mon.0) 1153 : audit 1 from='client.? 172.21.15.78:0/2616515176' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-01T10:48:21.467 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:21 smithi078 bash[75299]: cluster 2023-10-01T10:48:20.466324+0000 mon.a (mon.0) 1154 : cluster 0 osdmap e163: 8 total, 8 up, 8 in 2023-10-01T10:48:21.468 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:21 smithi078 bash[75299]: cluster 2023-10-01T10:48:20.472292+0000 osd.7 (osd.7) 112 : cluster 3 failed to encode map e163 with expected crc 2023-10-01T10:48:21.468 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:21 smithi078 bash[75299]: cluster 2023-10-01T10:48:20.476123+0000 osd.6 (osd.6) 78 : cluster 3 failed to encode map e163 with expected crc 2023-10-01T10:48:21.468 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:21 smithi078 bash[75299]: cluster 2023-10-01T10:48:20.477405+0000 osd.7 (osd.7) 113 : cluster 3 failed to encode map e163 with expected crc 2023-10-01T10:48:21.505 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:21.536 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:21.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:21.540 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:48:21.641 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:21.679 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:21.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:21.682 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:48:21.782 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:21.813 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:21.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:21.819 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:48:21.920 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:21.947 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:21.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:21.954 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:22.055 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:22.085 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:22.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:22.091 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:48:22.192 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:22.219 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:22.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:22.222 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:48:22.322 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:22.353 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:22.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:22.354 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:48:22.455 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:22.499 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:22.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:22.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:22.569 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-01T10:48:22.604 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:22.640 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:22.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:22.647 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:48:22.748 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:22.777 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:22.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:22.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:48:22.884 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:22.928 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:22.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:22.934 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:48:22.966 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:48:23.035 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:23.062 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:23.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:23.066 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:48:23.167 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:23.201 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:23.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:23.207 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:48:23.307 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:23.335 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:23.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:23.339 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:48:23.440 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:23.490 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:23.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:23.497 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:48:23.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:23 smithi078 bash[75299]: cluster 2023-10-01T10:48:21.879634+0000 mgr.y (mgr.25072) 922 : cluster 0 pgmap v857: 89 pgs: 89 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 750 op/s 2023-10-01T10:48:23.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:23 smithi078 bash[78274]: cluster 2023-10-01T10:48:21.879634+0000 mgr.y (mgr.25072) 922 : cluster 0 pgmap v857: 89 pgs: 89 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 750 op/s 2023-10-01T10:48:23.598 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:23.641 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:23.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:23.647 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:23.748 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:23.784 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:23.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:23.788 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:23.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:23 smithi173 bash[58816]: cluster 2023-10-01T10:48:21.879634+0000 mgr.y (mgr.25072) 922 : cluster 0 pgmap v857: 89 pgs: 89 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 750 op/s 2023-10-01T10:48:23.889 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:23.912 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:23.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:23.918 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:48:24.019 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:24.041 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:24.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:24.044 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:48:24.145 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:24.179 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:24.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:24.181 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:48:24.281 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:24.308 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:24.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:24.311 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:24.412 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:24.444 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:24.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:24.447 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:48:24.547 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:24.587 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:24.588 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T10:48:24.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:24.591 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:48:24.692 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:24.716 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:24.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:24.725 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:48:24.826 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:24.855 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:24.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:24.863 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:48:24.964 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:24.996 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:25.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:25.004 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:48:25.091 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:48:25.105 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:25.139 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:25.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:25.145 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:48:25.245 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:25.285 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:25.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:25.288 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:48:25.389 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:25.423 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:25.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:25.426 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:48:25.527 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:25.536 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:25 smithi078 bash[75299]: cluster 2023-10-01T10:48:23.880215+0000 mgr.y (mgr.25072) 923 : cluster 0 pgmap v858: 89 pgs: 89 active+clean; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 761 op/s 2023-10-01T10:48:25.536 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:25 smithi078 bash[78274]: cluster 2023-10-01T10:48:23.880215+0000 mgr.y (mgr.25072) 923 : cluster 0 pgmap v858: 89 pgs: 89 active+clean; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 761 op/s 2023-10-01T10:48:25.559 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:25.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:25.564 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:48:25.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:25 smithi173 bash[58816]: cluster 2023-10-01T10:48:23.880215+0000 mgr.y (mgr.25072) 923 : cluster 0 pgmap v858: 89 pgs: 89 active+clean; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 761 op/s 2023-10-01T10:48:25.665 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:25.692 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:25.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:25.695 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:48:25.795 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:25.823 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:25.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:25.827 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:48:25.927 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:25.952 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:25.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:25.956 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:48:26.056 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:26.090 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:26.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:26.093 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:48:26.194 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:26.220 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:26.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:26.223 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:48:26.324 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:26.360 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:26.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:26.362 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:48:26.463 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:26.505 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:26.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:26.510 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:48:26.611 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:26.636 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:26.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:26.638 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:48:26.739 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:26.772 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:26.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:26.774 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:26.875 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:26.904 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:26.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:26.911 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:48:27.011 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:27.050 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:27.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:27.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:48:27.153 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:48:27.155 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:27.209 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:27.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:27.217 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:48:27.318 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:27.345 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:27.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:27.349 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:48:27.449 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:27.476 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:27 smithi078 bash[75299]: cluster 2023-10-01T10:48:25.881721+0000 mgr.y (mgr.25072) 924 : cluster 0 pgmap v859: 89 pgs: 89 active+clean; 4.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 754 op/s 2023-10-01T10:48:27.478 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:27.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:27.484 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:48:27.585 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:27.618 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:27.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:27.620 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:27.720 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:27.732 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:27 smithi173 bash[58816]: cluster 2023-10-01T10:48:25.881721+0000 mgr.y (mgr.25072) 924 : cluster 0 pgmap v859: 89 pgs: 89 active+clean; 4.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 754 op/s 2023-10-01T10:48:27.743 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:27.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:27.746 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:48:27.847 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:27.880 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:27.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:27.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:48:27.885 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:27 smithi078 bash[78274]: cluster 2023-10-01T10:48:25.881721+0000 mgr.y (mgr.25072) 924 : cluster 0 pgmap v859: 89 pgs: 89 active+clean; 4.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 754 op/s 2023-10-01T10:48:27.984 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:28.006 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:28.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:28.009 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:48:28.109 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:28.134 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:28.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:28.139 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:48:28.240 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:28.267 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:28.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:28.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:48:28.371 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:28.398 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:28.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:28.402 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:48:28.502 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:28.536 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:28.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:28.541 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:28.641 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:28.683 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:28.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:28.686 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:48:28.787 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:28.832 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:28.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:28.835 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:48:28.935 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:28.960 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:28.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:28.966 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:48:29.067 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:29.101 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:29.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:29.104 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:48:29.205 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:29.243 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:29.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:29.246 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:48:29.347 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:29.378 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:29.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:29.383 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:48:29.481 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:29 smithi173 bash[58816]: cluster 2023-10-01T10:48:27.882183+0000 mgr.y (mgr.25072) 925 : cluster 0 pgmap v860: 89 pgs: 89 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 754 op/s 2023-10-01T10:48:29.484 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:29.516 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:29.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:29.522 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:48:29.623 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:29.648 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:29.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:29.651 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:48:29.752 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:29.787 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:29.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:29.790 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:48:29.820 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:29 smithi078 bash[75299]: cluster 2023-10-01T10:48:27.882183+0000 mgr.y (mgr.25072) 925 : cluster 0 pgmap v860: 89 pgs: 89 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 754 op/s 2023-10-01T10:48:29.822 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:29 smithi078 bash[78274]: cluster 2023-10-01T10:48:27.882183+0000 mgr.y (mgr.25072) 925 : cluster 0 pgmap v860: 89 pgs: 89 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 754 op/s 2023-10-01T10:48:29.891 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:29.928 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:29.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:29.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:48:30.032 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:30.059 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:30.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:30.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:48:30.165 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:30.188 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:30.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:30.199 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:30.299 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:30.336 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:30.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:30.339 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:48:30.439 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:30.470 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:30.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:30.473 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:48:30.573 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:30.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:48:30 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:48:30] "GET /metrics HTTP/1.1" 200 36319 "" "Prometheus/2.33.4" 2023-10-01T10:48:30.600 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:30.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:30.603 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:48:30.704 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:30.738 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:30.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:30.744 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:30.844 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:30.873 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:30.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:30.878 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:48:30.979 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:31.000 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:31.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:31.004 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:48:31.104 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:31.151 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:31.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:31.154 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:31.255 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:31.279 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:31.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:31.281 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:48:31.382 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:31.411 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:31.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:31.413 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:48:31.514 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:31.543 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:31.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:31.548 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:48:31.649 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:31.657 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:31 smithi173 bash[58816]: cluster 2023-10-01T10:48:29.882721+0000 mgr.y (mgr.25072) 926 : cluster 0 pgmap v861: 89 pgs: 89 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 738 op/s 2023-10-01T10:48:31.676 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:31.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:31.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:48:31.779 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:31.805 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:31.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:31.807 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:48:31.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:31 smithi078 bash[75299]: cluster 2023-10-01T10:48:29.882721+0000 mgr.y (mgr.25072) 926 : cluster 0 pgmap v861: 89 pgs: 89 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 738 op/s 2023-10-01T10:48:31.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:31 smithi078 bash[78274]: cluster 2023-10-01T10:48:29.882721+0000 mgr.y (mgr.25072) 926 : cluster 0 pgmap v861: 89 pgs: 89 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 738 op/s 2023-10-01T10:48:31.868 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:48:31.908 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:31.934 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:31.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:31.940 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:48:32.041 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:32.063 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:32.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:32.066 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:48:32.166 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:32.198 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:32.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:32.201 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:48:32.301 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:32.327 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:32.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:32.334 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:32.434 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:32.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:32.462 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:32.463 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:32.563 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:32.594 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:32.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:32.602 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:48:32.703 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:32.730 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:32.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:32.733 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:48:32.833 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:32.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:32.875 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:32.875 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:48:32.976 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:33.012 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:33.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:33.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:48:33.119 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:33.153 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:33.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:33.156 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:48:33.256 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:33.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:33.289 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:33.290 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:48:33.390 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:33.392 INFO:teuthology.orchestra.run.smithi078.stdout:ERROR: (22) Invalid argument 2023-10-01T10:48:33.392 INFO:teuthology.orchestra.run.smithi078.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T10:48:33.420 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:33.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:33.424 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:48:33.525 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:33.557 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:33.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:33.563 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:48:33.664 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:33.673 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:33 smithi078 bash[78274]: cluster 2023-10-01T10:48:31.883314+0000 mgr.y (mgr.25072) 927 : cluster 0 pgmap v862: 89 pgs: 89 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 711 op/s 2023-10-01T10:48:33.673 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:33 smithi078 bash[78274]: audit 2023-10-01T10:48:32.441479+0000 mon.a (mon.0) 1155 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:48:33.673 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:33 smithi078 bash[78274]: audit 2023-10-01T10:48:32.445717+0000 mon.a (mon.0) 1156 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:48:33.673 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:33 smithi078 bash[75299]: cluster 2023-10-01T10:48:31.883314+0000 mgr.y (mgr.25072) 927 : cluster 0 pgmap v862: 89 pgs: 89 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 711 op/s 2023-10-01T10:48:33.673 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:33 smithi078 bash[75299]: audit 2023-10-01T10:48:32.441479+0000 mon.a (mon.0) 1155 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:48:33.673 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:33 smithi078 bash[75299]: audit 2023-10-01T10:48:32.445717+0000 mon.a (mon.0) 1156 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:48:33.686 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:33.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:33.688 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:48:33.789 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:33.797 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:33 smithi173 bash[58816]: cluster 2023-10-01T10:48:31.883314+0000 mgr.y (mgr.25072) 927 : cluster 0 pgmap v862: 89 pgs: 89 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 711 op/s 2023-10-01T10:48:33.797 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:33 smithi173 bash[58816]: audit 2023-10-01T10:48:32.441479+0000 mon.a (mon.0) 1155 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:48:33.797 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:33 smithi173 bash[58816]: audit 2023-10-01T10:48:32.445717+0000 mon.a (mon.0) 1156 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:48:33.820 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:33.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:33.823 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:48:33.923 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:33.951 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:33.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:33.953 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:34.054 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:34.085 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:34.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:34.087 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:48:34.188 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:34.211 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:34.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:34.215 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:48:34.315 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:34.341 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:34.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:34.349 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:48:34.449 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:34.489 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:34.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:34.494 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:48:34.594 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:34.630 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:34.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:34.634 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:48:34.735 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:34.779 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:34.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:34.781 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:48:34.882 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:34.906 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:34.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:34.914 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:48:35.015 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:35.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:35.046 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:35.046 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:35.147 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:35.178 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:35.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:35.184 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:48:35.284 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:35.334 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:35.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:35.336 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:48:35.437 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:35.470 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:35.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:35.474 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:48:35.575 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:35.605 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:35.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:35.611 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:48:35.711 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:35.722 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:35 smithi173 bash[58816]: cluster 2023-10-01T10:48:33.883939+0000 mgr.y (mgr.25072) 928 : cluster 0 pgmap v863: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 770 op/s 2023-10-01T10:48:35.746 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:35.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:35.753 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:48:35.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:35 smithi078 bash[75299]: cluster 2023-10-01T10:48:33.883939+0000 mgr.y (mgr.25072) 928 : cluster 0 pgmap v863: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 770 op/s 2023-10-01T10:48:35.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:35 smithi078 bash[78274]: cluster 2023-10-01T10:48:33.883939+0000 mgr.y (mgr.25072) 928 : cluster 0 pgmap v863: 89 pgs: 89 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 770 op/s 2023-10-01T10:48:35.854 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:35.882 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:35.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:35.884 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:35.985 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:36.019 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:36.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:36.024 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:48:36.125 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:36.151 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:36.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:36.158 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:48:36.259 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:36.288 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:36.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:36.295 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:48:36.396 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:36.433 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:36.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:36.435 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:36.536 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:36.568 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:36.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:36.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:48:36.672 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:36.704 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:36.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:36.706 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:48:36.807 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:36.842 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:36.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:36.844 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:48:36.944 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:36.976 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:36.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:36.978 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:48:36.995 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-01T10:48:37.079 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:37.111 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:37.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:37.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:48:37.214 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:37.264 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:37.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:37.266 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:48:37.367 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:37.402 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:37.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:37.405 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:48:37.505 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:37.514 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:37 smithi078 bash[75299]: cluster 2023-10-01T10:48:35.884472+0000 mgr.y (mgr.25072) 929 : cluster 0 pgmap v864: 89 pgs: 89 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2023-10-01T10:48:37.514 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:37 smithi078 bash[78274]: cluster 2023-10-01T10:48:35.884472+0000 mgr.y (mgr.25072) 929 : cluster 0 pgmap v864: 89 pgs: 89 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2023-10-01T10:48:37.528 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:37.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:37.530 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:48:37.630 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:37.660 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:37.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:37.664 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:48:37.764 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:37.791 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:37.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:37.796 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:48:37.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:37 smithi173 bash[58816]: cluster 2023-10-01T10:48:35.884472+0000 mgr.y (mgr.25072) 929 : cluster 0 pgmap v864: 89 pgs: 89 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2023-10-01T10:48:37.898 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:37.924 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:37.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:37.933 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:48:38.033 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:38.051 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:38.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:38.053 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:48:38.154 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:38.178 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:38.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:38.186 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:48:38.286 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:38.339 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:38.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:38.344 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:48:38.445 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:38.463 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:38 smithi173 bash[58816]: audit 2023-10-01T10:48:37.768973+0000 mon.a (mon.0) 1157 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:48:38.481 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:38.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:38.486 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:48:38.587 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:38.621 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:38.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:38.627 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:48:38.721 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:38 smithi078 bash[75299]: audit 2023-10-01T10:48:37.768973+0000 mon.a (mon.0) 1157 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:48:38.722 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:38 smithi078 bash[75299]: audit 2023-10-01T10:48:38.262350+0000 mon.a (mon.0) 1158 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:48:38.722 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:38 smithi078 bash[75299]: audit 2023-10-01T10:48:38.276544+0000 mon.a (mon.0) 1159 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:48:38.722 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:38 smithi078 bash[78274]: audit 2023-10-01T10:48:37.768973+0000 mon.a (mon.0) 1157 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:48:38.722 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:38 smithi078 bash[78274]: audit 2023-10-01T10:48:38.262350+0000 mon.a (mon.0) 1158 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:48:38.722 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:38 smithi078 bash[78274]: audit 2023-10-01T10:48:38.276544+0000 mon.a (mon.0) 1159 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:48:38.727 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:38.737 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:38 smithi173 bash[58816]: audit 2023-10-01T10:48:38.262350+0000 mon.a (mon.0) 1158 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:48:38.737 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:38 smithi173 bash[58816]: audit 2023-10-01T10:48:38.276544+0000 mon.a (mon.0) 1159 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:48:38.760 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:38.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:38.764 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:48:38.864 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:38.892 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:38.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:38.895 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:48:38.996 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:39.027 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:39.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:39.034 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:48:39.134 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:39.157 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:39.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:39.163 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:48:39.236 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T10:48:39.264 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:39.266 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T10:48:39.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:39.292 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:39.292 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:48:39.310 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T10:48:39.350 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T10:48:39.389 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T10:48:39.393 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:39.414 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T10:48:39.427 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:39.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:39.431 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:48:39.439 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T10:48:39.479 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T10:48:39.489 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T10:48:39.489 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T10:48:39.489 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T10:48:39.489 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T10:48:39.489 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T10:48:39.489 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T10:48:39.489 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T10:48:39.489 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T10:48:39.532 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:39.555 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:39.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:39.562 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:48:39.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:39 smithi173 bash[58816]: cluster 2023-10-01T10:48:37.885726+0000 mgr.y (mgr.25072) 930 : cluster 0 pgmap v865: 89 pgs: 89 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 661 op/s 2023-10-01T10:48:39.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:39 smithi173 bash[58816]: audit 2023-10-01T10:48:39.192820+0000 mon.a (mon.0) 1160 : audit 0 from='client.? 172.21.15.78:0/1918320145' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:48:39.663 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:39.696 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:39.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:39.703 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:48:39.776 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:39 smithi078 bash[75299]: cluster 2023-10-01T10:48:37.885726+0000 mgr.y (mgr.25072) 930 : cluster 0 pgmap v865: 89 pgs: 89 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 661 op/s 2023-10-01T10:48:39.776 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:39 smithi078 bash[75299]: audit 2023-10-01T10:48:39.192820+0000 mon.a (mon.0) 1160 : audit 0 from='client.? 172.21.15.78:0/1918320145' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:48:39.776 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:39 smithi078 bash[78274]: cluster 2023-10-01T10:48:37.885726+0000 mgr.y (mgr.25072) 930 : cluster 0 pgmap v865: 89 pgs: 89 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 661 op/s 2023-10-01T10:48:39.776 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:39 smithi078 bash[78274]: audit 2023-10-01T10:48:39.192820+0000 mon.a (mon.0) 1160 : audit 0 from='client.? 172.21.15.78:0/1918320145' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:48:39.804 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:39.828 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:39.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:39.830 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:48:39.931 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:39.970 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:39.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:39.975 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:48:40.075 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:40.103 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:40.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:40.110 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:48:40.210 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:40.251 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:40.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:40.256 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:48:40.357 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:40.389 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:40.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:40.394 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:48:40.482 INFO:teuthology.orchestra.run.smithi078.stderr:reweighted osd.1 to 1 (10000) 2023-10-01T10:48:40.495 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:40.522 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:40 smithi078 bash[75299]: audit 2023-10-01T10:48:40.269455+0000 mon.a (mon.0) 1161 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T10:48:40.523 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:40 smithi078 bash[75299]: audit 2023-10-01T10:48:40.269852+0000 mon.b (mon.1) 44 : audit 1 from='client.? 172.21.15.78:0/2050254894' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T10:48:40.523 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:48:40 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:48:40] "GET /metrics HTTP/1.1" 200 36317 "" "Prometheus/2.33.4" 2023-10-01T10:48:40.523 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:40 smithi078 bash[78274]: audit 2023-10-01T10:48:40.269455+0000 mon.a (mon.0) 1161 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T10:48:40.523 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:40 smithi078 bash[78274]: audit 2023-10-01T10:48:40.269852+0000 mon.b (mon.1) 44 : audit 1 from='client.? 172.21.15.78:0/2050254894' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T10:48:40.524 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:40.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:40.532 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:40.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:40 smithi173 bash[58816]: audit 2023-10-01T10:48:40.269455+0000 mon.a (mon.0) 1161 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T10:48:40.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:40 smithi173 bash[58816]: audit 2023-10-01T10:48:40.269852+0000 mon.b (mon.1) 44 : audit 1 from='client.? 172.21.15.78:0/2050254894' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T10:48:40.633 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:40.661 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:40.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:40.664 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:48:40.765 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:40.793 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:40.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:40.797 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:48:40.898 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:40.926 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:40.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:40.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:48:41.029 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:41.061 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:41.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:41.062 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:48:41.163 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:41.196 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:41.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:41.203 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:48:41.304 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:41.347 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:41.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:41.352 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:48:41.453 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:41.486 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:41.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:41.490 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:48:41.571 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:41 smithi078 bash[75299]: cluster 2023-10-01T10:48:39.886381+0000 mgr.y (mgr.25072) 931 : cluster 0 pgmap v866: 89 pgs: 89 active+clean; 4.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 839 op/s 2023-10-01T10:48:41.571 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:41 smithi078 bash[75299]: cluster 2023-10-01T10:48:40.466403+0000 osd.7 (osd.7) 114 : cluster 3 failed to encode map e164 with expected crc 2023-10-01T10:48:41.571 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:41 smithi078 bash[75299]: audit 2023-10-01T10:48:40.467164+0000 mon.a (mon.0) 1162 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-01T10:48:41.571 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:41 smithi078 bash[75299]: cluster 2023-10-01T10:48:40.467229+0000 mon.a (mon.0) 1163 : cluster 0 osdmap e164: 8 total, 8 up, 8 in 2023-10-01T10:48:41.571 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:41 smithi078 bash[75299]: cluster 2023-10-01T10:48:40.471428+0000 osd.6 (osd.6) 79 : cluster 3 failed to encode map e164 with expected crc 2023-10-01T10:48:41.571 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:41 smithi078 bash[75299]: cluster 2023-10-01T10:48:40.478708+0000 osd.6 (osd.6) 80 : cluster 3 failed to encode map e164 with expected crc 2023-10-01T10:48:41.572 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:41 smithi078 bash[78274]: cluster 2023-10-01T10:48:39.886381+0000 mgr.y (mgr.25072) 931 : cluster 0 pgmap v866: 89 pgs: 89 active+clean; 4.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 839 op/s 2023-10-01T10:48:41.572 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:41 smithi078 bash[78274]: cluster 2023-10-01T10:48:40.466403+0000 osd.7 (osd.7) 114 : cluster 3 failed to encode map e164 with expected crc 2023-10-01T10:48:41.572 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:41 smithi078 bash[78274]: audit 2023-10-01T10:48:40.467164+0000 mon.a (mon.0) 1162 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-01T10:48:41.572 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:41 smithi078 bash[78274]: cluster 2023-10-01T10:48:40.467229+0000 mon.a (mon.0) 1163 : cluster 0 osdmap e164: 8 total, 8 up, 8 in 2023-10-01T10:48:41.572 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:41 smithi078 bash[78274]: cluster 2023-10-01T10:48:40.471428+0000 osd.6 (osd.6) 79 : cluster 3 failed to encode map e164 with expected crc 2023-10-01T10:48:41.572 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:41 smithi078 bash[78274]: cluster 2023-10-01T10:48:40.478708+0000 osd.6 (osd.6) 80 : cluster 3 failed to encode map e164 with expected crc 2023-10-01T10:48:41.591 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:41.616 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:41.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:41.618 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:48:41.719 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:41.778 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:41.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:41.780 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:48:41.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:41 smithi173 bash[58816]: cluster 2023-10-01T10:48:39.886381+0000 mgr.y (mgr.25072) 931 : cluster 0 pgmap v866: 89 pgs: 89 active+clean; 4.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 839 op/s 2023-10-01T10:48:41.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:41 smithi173 bash[58816]: cluster 2023-10-01T10:48:40.466403+0000 osd.7 (osd.7) 114 : cluster 3 failed to encode map e164 with expected crc 2023-10-01T10:48:41.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:41 smithi173 bash[58816]: audit 2023-10-01T10:48:40.467164+0000 mon.a (mon.0) 1162 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-01T10:48:41.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:41 smithi173 bash[58816]: cluster 2023-10-01T10:48:40.467229+0000 mon.a (mon.0) 1163 : cluster 0 osdmap e164: 8 total, 8 up, 8 in 2023-10-01T10:48:41.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:41 smithi173 bash[58816]: cluster 2023-10-01T10:48:40.471428+0000 osd.6 (osd.6) 79 : cluster 3 failed to encode map e164 with expected crc 2023-10-01T10:48:41.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:41 smithi173 bash[58816]: cluster 2023-10-01T10:48:40.478708+0000 osd.6 (osd.6) 80 : cluster 3 failed to encode map e164 with expected crc 2023-10-01T10:48:41.881 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:41.906 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:41.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:41.913 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:48:42.014 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:42.046 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:42.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:42.054 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:48:42.154 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:42.184 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:42.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:42.190 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:42.291 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:42.333 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:42.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:42.340 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:48:42.441 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:42.489 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:42.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:42.492 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:48:42.592 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:42.621 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:42.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:42.626 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:48:42.726 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:42.759 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:42.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:42.763 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:48:42.864 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:42.895 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:42.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:42.902 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:48:43.003 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:43.027 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:43.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:43.029 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:48:43.129 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:43.155 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:43.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:43.160 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:48:43.261 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:43.307 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:43.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:43.310 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:43.411 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:43.447 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:43.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:43.449 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:48:43.480 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:43 smithi173 bash[58816]: cluster 2023-10-01T10:48:41.886904+0000 mgr.y (mgr.25072) 932 : cluster 0 pgmap v868: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2023-10-01T10:48:43.550 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:43.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:43 smithi078 bash[75299]: cluster 2023-10-01T10:48:41.886904+0000 mgr.y (mgr.25072) 932 : cluster 0 pgmap v868: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2023-10-01T10:48:43.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:43 smithi078 bash[78274]: cluster 2023-10-01T10:48:41.886904+0000 mgr.y (mgr.25072) 932 : cluster 0 pgmap v868: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2023-10-01T10:48:43.592 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:43.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:43.596 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:48:43.696 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:43.748 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:43.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:43.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:48:43.850 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:43.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:43.889 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:43.889 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:48:43.990 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:44.005 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-01T10:48:44.026 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:44.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:44.028 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:44.129 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:44.169 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:44.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:44.175 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:48:44.276 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:44.313 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:44.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:44.316 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:48:44.417 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:44.449 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:44.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:44.452 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:44.553 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:44.603 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:44.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:44.612 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:48:44.713 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:44.742 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:44.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:44.747 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:48:44.848 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:44.873 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:44.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:44.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:48:44.980 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:45.011 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:45.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:45.016 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:48:45.117 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:45.183 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:45.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:45.187 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:48:45.288 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:45.317 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:45.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:45.322 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:48:45.423 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:45.446 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:45.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:45.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:48:45.489 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:45 smithi173 bash[58816]: cluster 2023-10-01T10:48:43.887476+0000 mgr.y (mgr.25072) 933 : cluster 0 pgmap v869: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2023-10-01T10:48:45.489 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:45 smithi173 bash[58816]: audit 2023-10-01T10:48:44.933756+0000 mon.a (mon.0) 1164 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:48:45.489 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:45 smithi173 bash[58816]: audit 2023-10-01T10:48:44.944671+0000 mon.a (mon.0) 1165 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:48:45.489 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:45 smithi173 bash[58816]: audit 2023-10-01T10:48:45.423381+0000 mon.c (mon.2) 160 : audit 1 from='client.? 172.21.15.78:0/1868732750' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:48:45.489 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:45 smithi173 bash[58816]: audit 2023-10-01T10:48:45.423778+0000 mon.a (mon.0) 1166 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:48:45.556 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:45.564 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:45 smithi078 bash[75299]: cluster 2023-10-01T10:48:43.887476+0000 mgr.y (mgr.25072) 933 : cluster 0 pgmap v869: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2023-10-01T10:48:45.564 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:45 smithi078 bash[75299]: audit 2023-10-01T10:48:44.933756+0000 mon.a (mon.0) 1164 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:48:45.565 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:45 smithi078 bash[75299]: audit 2023-10-01T10:48:44.944671+0000 mon.a (mon.0) 1165 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:48:45.565 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:45 smithi078 bash[75299]: audit 2023-10-01T10:48:45.423381+0000 mon.c (mon.2) 160 : audit 1 from='client.? 172.21.15.78:0/1868732750' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:48:45.565 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:45 smithi078 bash[75299]: audit 2023-10-01T10:48:45.423778+0000 mon.a (mon.0) 1166 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:48:45.565 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:45 smithi078 bash[78274]: cluster 2023-10-01T10:48:43.887476+0000 mgr.y (mgr.25072) 933 : cluster 0 pgmap v869: 89 pgs: 89 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2023-10-01T10:48:45.565 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:45 smithi078 bash[78274]: audit 2023-10-01T10:48:44.933756+0000 mon.a (mon.0) 1164 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:48:45.565 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:45 smithi078 bash[78274]: audit 2023-10-01T10:48:44.944671+0000 mon.a (mon.0) 1165 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:48:45.565 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:45 smithi078 bash[78274]: audit 2023-10-01T10:48:45.423381+0000 mon.c (mon.2) 160 : audit 1 from='client.? 172.21.15.78:0/1868732750' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:48:45.565 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:45 smithi078 bash[78274]: audit 2023-10-01T10:48:45.423778+0000 mon.a (mon.0) 1166 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:48:45.585 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:45.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:45.592 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:45.692 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:45.725 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:45.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:45.728 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:48:45.828 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:45.851 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T10:48:45.866 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:45.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:45.870 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:48:45.960 INFO:teuthology.orchestra.run.smithi078.stderr:nodeep-scrub is set 2023-10-01T10:48:45.971 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:46.014 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:46.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:46.015 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:46.117 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:46.159 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:46.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:46.165 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:46.266 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:46.315 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:46.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:46.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:46.418 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:46.450 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:46.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:46.454 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:48:46.555 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:46.586 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:46.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:46.592 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:48:46.692 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:46.729 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:46.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:46.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:48:46.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:46 smithi078 bash[78274]: cluster 2023-10-01T10:48:45.940115+0000 mon.a (mon.0) 1167 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:48:46.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:46 smithi078 bash[78274]: audit 2023-10-01T10:48:45.946870+0000 mon.a (mon.0) 1168 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T10:48:46.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:46 smithi078 bash[78274]: cluster 2023-10-01T10:48:45.947433+0000 mon.a (mon.0) 1169 : cluster 0 osdmap e165: 8 total, 8 up, 8 in 2023-10-01T10:48:46.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:46 smithi078 bash[78274]: cluster 2023-10-01T10:48:45.949252+0000 osd.7 (osd.7) 115 : cluster 3 failed to encode map e165 with expected crc 2023-10-01T10:48:46.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:46 smithi078 bash[78274]: cluster 2023-10-01T10:48:45.955734+0000 osd.6 (osd.6) 81 : cluster 3 failed to encode map e165 with expected crc 2023-10-01T10:48:46.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:46 smithi078 bash[75299]: cluster 2023-10-01T10:48:45.940115+0000 mon.a (mon.0) 1167 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:48:46.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:46 smithi078 bash[75299]: audit 2023-10-01T10:48:45.946870+0000 mon.a (mon.0) 1168 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T10:48:46.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:46 smithi078 bash[75299]: cluster 2023-10-01T10:48:45.947433+0000 mon.a (mon.0) 1169 : cluster 0 osdmap e165: 8 total, 8 up, 8 in 2023-10-01T10:48:46.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:46 smithi078 bash[75299]: cluster 2023-10-01T10:48:45.949252+0000 osd.7 (osd.7) 115 : cluster 3 failed to encode map e165 with expected crc 2023-10-01T10:48:46.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:46 smithi078 bash[75299]: cluster 2023-10-01T10:48:45.955734+0000 osd.6 (osd.6) 81 : cluster 3 failed to encode map e165 with expected crc 2023-10-01T10:48:46.836 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:46.846 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:46 smithi173 bash[58816]: cluster 2023-10-01T10:48:45.940115+0000 mon.a (mon.0) 1167 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:48:46.846 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:46 smithi173 bash[58816]: audit 2023-10-01T10:48:45.946870+0000 mon.a (mon.0) 1168 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T10:48:46.846 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:46 smithi173 bash[58816]: cluster 2023-10-01T10:48:45.947433+0000 mon.a (mon.0) 1169 : cluster 0 osdmap e165: 8 total, 8 up, 8 in 2023-10-01T10:48:46.846 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:46 smithi173 bash[58816]: cluster 2023-10-01T10:48:45.949252+0000 osd.7 (osd.7) 115 : cluster 3 failed to encode map e165 with expected crc 2023-10-01T10:48:46.846 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:46 smithi173 bash[58816]: cluster 2023-10-01T10:48:45.955734+0000 osd.6 (osd.6) 81 : cluster 3 failed to encode map e165 with expected crc 2023-10-01T10:48:46.871 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:46.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:46.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:48:46.980 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:47.012 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:47.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:47.019 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:48:47.119 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:47.148 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:47.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:47.153 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:48:47.254 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:47.277 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:47.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:47.281 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:48:47.382 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:47.406 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:47.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:47.411 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:48:47.512 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:47.546 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:47.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:47.555 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:48:47.655 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:47.664 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:47 smithi078 bash[75299]: cluster 2023-10-01T10:48:45.888031+0000 mgr.y (mgr.25072) 934 : cluster 0 pgmap v870: 89 pgs: 89 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 694 op/s 2023-10-01T10:48:47.664 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:47 smithi078 bash[75299]: cluster 2023-10-01T10:48:45.958374+0000 osd.5 (osd.5) 95 : cluster 3 failed to encode map e165 with expected crc 2023-10-01T10:48:47.664 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:47 smithi078 bash[75299]: audit 2023-10-01T10:48:47.428802+0000 mon.a (mon.0) 1170 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:48:47.664 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:47 smithi078 bash[78274]: cluster 2023-10-01T10:48:45.888031+0000 mgr.y (mgr.25072) 934 : cluster 0 pgmap v870: 89 pgs: 89 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 694 op/s 2023-10-01T10:48:47.664 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:47 smithi078 bash[78274]: cluster 2023-10-01T10:48:45.958374+0000 osd.5 (osd.5) 95 : cluster 3 failed to encode map e165 with expected crc 2023-10-01T10:48:47.665 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:47 smithi078 bash[78274]: audit 2023-10-01T10:48:47.428802+0000 mon.a (mon.0) 1170 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:48:47.692 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:47.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:47.696 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:47.796 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:47.806 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:47 smithi173 bash[58816]: cluster 2023-10-01T10:48:45.888031+0000 mgr.y (mgr.25072) 934 : cluster 0 pgmap v870: 89 pgs: 89 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 694 op/s 2023-10-01T10:48:47.806 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:47 smithi173 bash[58816]: cluster 2023-10-01T10:48:45.958374+0000 osd.5 (osd.5) 95 : cluster 3 failed to encode map e165 with expected crc 2023-10-01T10:48:47.806 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:47 smithi173 bash[58816]: audit 2023-10-01T10:48:47.428802+0000 mon.a (mon.0) 1170 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:48:47.821 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:47.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:47.823 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:48:47.886 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:48:47.924 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:47.950 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:47.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:47.953 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:48:48.053 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:48.098 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:48.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:48.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:48:48.202 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:48.227 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:48.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:48.230 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:48.331 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:48.359 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:48.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:48.366 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:48:48.467 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:48.507 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:48.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:48.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:48:48.615 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:48.647 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:48.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:48.653 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:48:48.753 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:48.796 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:48.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:48.800 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:48:48.901 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:48.944 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:48.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:48.956 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:49.057 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:49.098 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:49.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:49.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:48:49.203 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:49.231 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:49.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:49.238 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:48:49.338 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:49.390 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:49.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:49.397 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:48:49.402 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T10:48:49.499 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:49.538 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:49.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:49.542 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:48:49.643 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:49.656 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:49 smithi078 bash[75299]: cluster 2023-10-01T10:48:47.890622+0000 mgr.y (mgr.25072) 935 : cluster 0 pgmap v872: 89 pgs: 89 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 513 op/s 2023-10-01T10:48:49.656 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:49 smithi078 bash[75299]: audit 2023-10-01T10:48:49.450128+0000 mon.a (mon.0) 1171 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:48:49.656 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:49 smithi078 bash[75299]: audit 2023-10-01T10:48:49.464181+0000 mon.a (mon.0) 1172 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:48:49.656 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:49 smithi078 bash[78274]: cluster 2023-10-01T10:48:47.890622+0000 mgr.y (mgr.25072) 935 : cluster 0 pgmap v872: 89 pgs: 89 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 513 op/s 2023-10-01T10:48:49.656 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:49 smithi078 bash[78274]: audit 2023-10-01T10:48:49.450128+0000 mon.a (mon.0) 1171 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:48:49.656 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:49 smithi078 bash[78274]: audit 2023-10-01T10:48:49.464181+0000 mon.a (mon.0) 1172 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:48:49.691 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:49.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:49.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:49.799 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:49.807 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:49 smithi173 bash[58816]: cluster 2023-10-01T10:48:47.890622+0000 mgr.y (mgr.25072) 935 : cluster 0 pgmap v872: 89 pgs: 89 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 513 op/s 2023-10-01T10:48:49.807 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:49 smithi173 bash[58816]: audit 2023-10-01T10:48:49.450128+0000 mon.a (mon.0) 1171 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:48:49.807 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:49 smithi173 bash[58816]: audit 2023-10-01T10:48:49.464181+0000 mon.a (mon.0) 1172 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:48:49.829 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:49.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:49.834 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:48:49.935 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:49.937 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:48:49.988 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:49.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:49.992 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:48:50.093 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:50.128 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:50.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:50.135 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:48:50.236 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:50.269 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:48:50 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:48:50] "GET /metrics HTTP/1.1" 200 36314 "" "Prometheus/2.33.4" 2023-10-01T10:48:50.276 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:50.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:50.281 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:48:50.382 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:50.401 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:50.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:50.404 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:50.504 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:50.528 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:50.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:50.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:48:50.631 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:50.655 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:50.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:50.660 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:48:50.661 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:50 smithi078 bash[75299]: audit 2023-10-01T10:48:50.194559+0000 mon.a (mon.0) 1173 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:48:50.661 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:50 smithi078 bash[75299]: audit 2023-10-01T10:48:50.196269+0000 mon.a (mon.0) 1174 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:48:50.661 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:50 smithi078 bash[75299]: audit 2023-10-01T10:48:50.208580+0000 mon.a (mon.0) 1175 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:48:50.661 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:50 smithi078 bash[78274]: audit 2023-10-01T10:48:50.194559+0000 mon.a (mon.0) 1173 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:48:50.662 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:50 smithi078 bash[78274]: audit 2023-10-01T10:48:50.196269+0000 mon.a (mon.0) 1174 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:48:50.662 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:50 smithi078 bash[78274]: audit 2023-10-01T10:48:50.208580+0000 mon.a (mon.0) 1175 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:48:50.762 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:50.806 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:50.808 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:50 smithi173 bash[58816]: audit 2023-10-01T10:48:50.194559+0000 mon.a (mon.0) 1173 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:48:50.808 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:50 smithi173 bash[58816]: audit 2023-10-01T10:48:50.196269+0000 mon.a (mon.0) 1174 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:48:50.808 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:50 smithi173 bash[58816]: audit 2023-10-01T10:48:50.208580+0000 mon.a (mon.0) 1175 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:48:50.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:50.809 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:48:50.909 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:50.949 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:50.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:50.952 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:48:51.052 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:51.082 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:51.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:51.084 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:48:51.184 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:51.211 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:51.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:51.215 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:51.315 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:51.343 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:51.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:51.348 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:48:51.449 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:51.503 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:51.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:51.509 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:48:51.610 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:51.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:51 smithi173 bash[58816]: cluster 2023-10-01T10:48:49.891277+0000 mgr.y (mgr.25072) 936 : cluster 0 pgmap v873: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 732 op/s 2023-10-01T10:48:51.634 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:51.637 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:51 smithi078 bash[75299]: cluster 2023-10-01T10:48:49.891277+0000 mgr.y (mgr.25072) 936 : cluster 0 pgmap v873: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 732 op/s 2023-10-01T10:48:51.637 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:51 smithi078 bash[78274]: cluster 2023-10-01T10:48:49.891277+0000 mgr.y (mgr.25072) 936 : cluster 0 pgmap v873: 89 pgs: 89 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 732 op/s 2023-10-01T10:48:51.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:51.638 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:48:51.739 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:51.784 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:51.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:51.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:48:51.887 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:51.915 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:51.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:51.918 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:48:51.976 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:48:52.019 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:52.046 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:52.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:52.056 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:48:52.157 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:52.191 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:52.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:52.198 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:48:52.299 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:52.327 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:52.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:52.330 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:48:52.430 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:52.481 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:52.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:52.488 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:48:52.589 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:52.618 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:52.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:52.620 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:48:52.721 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:52.753 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:52.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:52.757 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:48:52.858 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:52.887 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:52.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:52.896 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:48:52.996 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:53.023 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:53.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:53.025 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:48:53.126 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:53.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:53.153 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:53.153 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:48:53.254 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:53.290 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:53.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:53.294 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:53.395 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:53.419 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:53.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:53.421 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:48:53.522 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:53.563 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:53.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:53.568 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:48:53.668 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:53.692 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:53.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:53.696 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:48:53.797 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:53.805 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:53 smithi078 bash[78274]: cluster 2023-10-01T10:48:51.891783+0000 mgr.y (mgr.25072) 937 : cluster 0 pgmap v874: 89 pgs: 89 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 691 op/s 2023-10-01T10:48:53.806 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:53 smithi078 bash[75299]: cluster 2023-10-01T10:48:51.891783+0000 mgr.y (mgr.25072) 937 : cluster 0 pgmap v874: 89 pgs: 89 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 691 op/s 2023-10-01T10:48:53.821 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:53.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:53.824 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:53.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:53 smithi173 bash[58816]: cluster 2023-10-01T10:48:51.891783+0000 mgr.y (mgr.25072) 937 : cluster 0 pgmap v874: 89 pgs: 89 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 691 op/s 2023-10-01T10:48:53.925 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:53.952 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:53.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:53.958 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:54.008 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:48:54.059 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:54.092 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:54.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:54.095 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:48:54.196 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:54.225 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:54.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:54.230 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:48:54.330 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:54.360 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:54.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:54.362 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:48:54.463 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:54.497 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:54.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:54.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:48:54.600 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:54.627 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:54.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:54.632 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:48:54.732 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:54.764 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:54.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:54.767 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:48:54.868 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:54.895 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:54.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:54.900 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:48:55.001 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:55.028 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:55.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:55.033 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:48:55.134 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:55.159 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:55.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:55.163 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:55.263 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:55.296 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:55.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:55.299 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:48:55.399 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:55.431 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:55.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:55.434 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:48:55.534 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:55.552 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:55 smithi173 bash[58816]: cluster 2023-10-01T10:48:53.892352+0000 mgr.y (mgr.25072) 938 : cluster 0 pgmap v875: 89 pgs: 89 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 686 op/s 2023-10-01T10:48:55.555 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:55.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:55.558 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:48:55.658 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:55.712 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:55.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:55.715 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:48:55.816 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:55.824 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:55 smithi078 bash[78274]: cluster 2023-10-01T10:48:53.892352+0000 mgr.y (mgr.25072) 938 : cluster 0 pgmap v875: 89 pgs: 89 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 686 op/s 2023-10-01T10:48:55.825 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:55 smithi078 bash[75299]: cluster 2023-10-01T10:48:53.892352+0000 mgr.y (mgr.25072) 938 : cluster 0 pgmap v875: 89 pgs: 89 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 686 op/s 2023-10-01T10:48:55.851 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:55.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:55.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:48:55.955 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:55.984 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:55.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:55.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:48:56.089 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:56.113 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:56.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:56.116 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:48:56.217 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:56.245 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:56.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:56.248 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:48:56.349 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:56.374 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:56.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:56.380 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:48:56.481 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:56.510 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:56.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:56.514 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:48:56.615 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:56.645 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:56.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:56.650 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:56.656 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2023-10-01T10:48:56.656 INFO:teuthology.orchestra.run.smithi078.stdout: "size": 20, 2023-10-01T10:48:56.656 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 600, 2023-10-01T10:48:56.656 INFO:teuthology.orchestra.run.smithi078.stdout: "ops": [ 2023-10-01T10:48:56.657 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.657 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.35649.0:158043 9.d e137/128 9:bccdffc8:::benchmark_data_smithi078_113121_object46228:head v 137'9692, mlcod=137'9686)", 2023-10-01T10:48:56.657 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:43:39.371656+0000", 2023-10-01T10:48:56.657 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 317.28118435900001, 2023-10-01T10:48:56.657 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.249114108, 2023-10-01T10:48:56.657 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:48:56.657 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:48:56.657 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:48:56.657 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.657 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:48:56.657 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.371656+0000", 2023-10-01T10:48:56.657 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:48:56.658 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.658 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.658 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:48:56.658 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.371656+0000", 2023-10-01T10:48:56.658 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:48:56.658 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.658 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.658 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:48:56.658 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.371655+0000", 2023-10-01T10:48:56.658 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:48:56.658 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.659 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.659 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:48:56.659 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.371665+0000", 2023-10-01T10:48:56.659 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.0125999999999999e-05 2023-10-01T10:48:56.659 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.659 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.659 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:48:56.659 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.371678+0000", 2023-10-01T10:48:56.659 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.2787e-05 2023-10-01T10:48:56.659 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.659 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.659 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:48:56.659 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.371685+0000", 2023-10-01T10:48:56.660 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.4800000000000004e-06 2023-10-01T10:48:56.660 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.660 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.660 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:48:56.660 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.375346+0000", 2023-10-01T10:48:56.660 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0036604530000000001 2023-10-01T10:48:56.660 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.660 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.660 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:48:56.660 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.375365+0000", 2023-10-01T10:48:56.660 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.9598000000000001e-05 2023-10-01T10:48:56.661 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.661 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.661 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:48:56.661 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.620684+0000", 2023-10-01T10:48:56.661 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.24531868200000001 2023-10-01T10:48:56.661 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.661 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.661 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:48:56.661 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.620770+0000", 2023-10-01T10:48:56.662 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.5748999999999995e-05 2023-10-01T10:48:56.662 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:48:56.662 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:48:56.662 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:48:56.662 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.662 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.662 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.35649.0:158048 9.4 9:211513e2:::benchmark_data_smithi078_113121_object46233:head [delete] snapc 0=[] ondisk+write+known_if_redirected e137)", 2023-10-01T10:48:56.662 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:43:39.372308+0000", 2023-10-01T10:48:56.662 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 317.28053230299997, 2023-10-01T10:48:56.662 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.23505704699999999, 2023-10-01T10:48:56.662 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:48:56.662 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:48:56.663 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:48:56.663 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.35649", 2023-10-01T10:48:56.663 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:48:56.663 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 158048 2023-10-01T10:48:56.663 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.663 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:48:56.663 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.663 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:48:56.663 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.372308+0000", 2023-10-01T10:48:56.663 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:48:56.663 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.663 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.664 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:48:56.664 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.372308+0000", 2023-10-01T10:48:56.664 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:48:56.664 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.664 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.664 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:48:56.664 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.372306+0000", 2023-10-01T10:48:56.664 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999986 2023-10-01T10:48:56.664 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.664 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.664 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:48:56.664 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.372311+0000", 2023-10-01T10:48:56.665 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.2140000000000004e-06 2023-10-01T10:48:56.666 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.666 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.666 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:48:56.666 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.372323+0000", 2023-10-01T10:48:56.667 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.2055000000000001e-05 2023-10-01T10:48:56.667 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.667 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.667 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:48:56.667 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.372331+0000", 2023-10-01T10:48:56.667 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.17e-06 2023-10-01T10:48:56.667 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.667 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.667 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:48:56.667 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.372378+0000", 2023-10-01T10:48:56.667 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.7311000000000002e-05 2023-10-01T10:48:56.667 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.668 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.668 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:48:56.668 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.372493+0000", 2023-10-01T10:48:56.668 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000115018 2023-10-01T10:48:56.668 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.668 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.668 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 3,4", 2023-10-01T10:48:56.668 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.372602+0000", 2023-10-01T10:48:56.668 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000109385 2023-10-01T10:48:56.668 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.668 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.668 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:48:56.669 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.377307+0000", 2023-10-01T10:48:56.669 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.004704677 2023-10-01T10:48:56.669 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.669 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.669 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:48:56.669 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.379657+0000", 2023-10-01T10:48:56.669 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0023498149999999999 2023-10-01T10:48:56.669 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.669 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.669 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:48:56.669 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.607277+0000", 2023-10-01T10:48:56.669 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.22761999499999999 2023-10-01T10:48:56.669 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.670 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.670 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:48:56.670 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.607320+0000", 2023-10-01T10:48:56.670 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.3744e-05 2023-10-01T10:48:56.670 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.670 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.670 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:48:56.670 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.607365+0000", 2023-10-01T10:48:56.670 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.4153000000000002e-05 2023-10-01T10:48:56.670 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:48:56.670 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:48:56.670 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:48:56.671 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.671 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.671 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.35649.0:158049 9.d e137/128 9:b922e85f:::benchmark_data_smithi078_113121_object46234:head v 137'9693, mlcod=137'9686)", 2023-10-01T10:48:56.671 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:43:39.377291+0000", 2023-10-01T10:48:56.671 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 317.27554883699997, 2023-10-01T10:48:56.671 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.24749200399999999, 2023-10-01T10:48:56.671 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:48:56.671 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:48:56.671 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:48:56.671 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.671 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:48:56.671 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.377291+0000", 2023-10-01T10:48:56.672 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:48:56.672 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.672 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.672 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:48:56.672 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.377291+0000", 2023-10-01T10:48:56.672 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:48:56.672 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.672 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.672 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:48:56.672 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.377290+0000", 2023-10-01T10:48:56.672 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:48:56.673 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.673 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.673 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:48:56.673 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.377317+0000", 2023-10-01T10:48:56.673 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.7120000000000001e-05 2023-10-01T10:48:56.673 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.673 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.673 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:48:56.673 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.377329+0000", 2023-10-01T10:48:56.673 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1518000000000001e-05 2023-10-01T10:48:56.673 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.673 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.673 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:48:56.674 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.377338+0000", 2023-10-01T10:48:56.674 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.0650000000000005e-06 2023-10-01T10:48:56.674 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.674 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.674 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:48:56.674 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.380748+0000", 2023-10-01T10:48:56.674 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.003409895 2023-10-01T10:48:56.674 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.674 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.674 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:48:56.675 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.380766+0000", 2023-10-01T10:48:56.675 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.8756000000000001e-05 2023-10-01T10:48:56.675 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.675 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.675 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:48:56.675 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.624754+0000", 2023-10-01T10:48:56.675 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.24398745799999999 2023-10-01T10:48:56.675 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.675 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.675 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:48:56.675 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.624783+0000", 2023-10-01T10:48:56.675 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.9271000000000001e-05 2023-10-01T10:48:56.675 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:48:56.676 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:48:56.676 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:48:56.676 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.676 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.676 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.35649.0:158051 9.6 e137/128 9:6c542c4a:::benchmark_data_smithi078_113121_object46236:head v 137'10105, mlcod=137'10103)", 2023-10-01T10:48:56.676 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:43:39.377901+0000", 2023-10-01T10:48:56.676 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 317.27493921799999, 2023-10-01T10:48:56.676 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.22936497, 2023-10-01T10:48:56.676 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:48:56.676 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:48:56.676 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:48:56.676 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.677 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:48:56.677 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.377901+0000", 2023-10-01T10:48:56.677 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:48:56.677 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.677 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.677 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:48:56.677 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.377901+0000", 2023-10-01T10:48:56.677 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:48:56.677 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.677 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.677 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:48:56.677 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.377900+0000", 2023-10-01T10:48:56.677 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:48:56.678 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.678 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.678 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:48:56.678 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.377909+0000", 2023-10-01T10:48:56.678 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.3740000000000004e-06 2023-10-01T10:48:56.678 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.678 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.678 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:48:56.678 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.377920+0000", 2023-10-01T10:48:56.678 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.0798000000000001e-05 2023-10-01T10:48:56.678 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.679 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.679 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:48:56.679 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.377927+0000", 2023-10-01T10:48:56.679 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.9929999999999998e-06 2023-10-01T10:48:56.679 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.679 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.679 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:48:56.679 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.377965+0000", 2023-10-01T10:48:56.679 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.7774000000000001e-05 2023-10-01T10:48:56.679 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.679 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.679 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:48:56.680 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.377984+0000", 2023-10-01T10:48:56.680 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.8921000000000001e-05 2023-10-01T10:48:56.680 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.680 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.680 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:48:56.680 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.607222+0000", 2023-10-01T10:48:56.680 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.22923817099999999 2023-10-01T10:48:56.680 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.680 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.680 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:48:56.680 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.607266+0000", 2023-10-01T10:48:56.680 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.3655999999999997e-05 2023-10-01T10:48:56.680 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:48:56.681 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:48:56.681 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:48:56.681 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.681 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.681 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.35649.0:158054 9.4 9:2dd5ac1d:::benchmark_data_smithi078_113121_object46239:head [delete] snapc 0=[] ondisk+write+known_if_redirected e137)", 2023-10-01T10:48:56.681 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:43:39.380125+0000", 2023-10-01T10:48:56.681 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 317.27271453499998, 2023-10-01T10:48:56.681 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.23519099299999999, 2023-10-01T10:48:56.681 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:48:56.681 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:48:56.681 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:48:56.682 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.35649", 2023-10-01T10:48:56.682 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:48:56.682 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 158054 2023-10-01T10:48:56.682 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.682 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:48:56.682 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.682 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:48:56.682 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.380125+0000", 2023-10-01T10:48:56.682 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:48:56.682 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.682 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.682 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:48:56.683 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.380125+0000", 2023-10-01T10:48:56.683 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:48:56.683 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.683 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.683 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:48:56.683 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.380124+0000", 2023-10-01T10:48:56.683 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999986 2023-10-01T10:48:56.683 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.683 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.683 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:48:56.683 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.380130+0000", 2023-10-01T10:48:56.684 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.3450000000000002e-06 2023-10-01T10:48:56.684 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.684 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.684 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:48:56.684 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.380142+0000", 2023-10-01T10:48:56.684 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.2074e-05 2023-10-01T10:48:56.684 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.684 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.684 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:48:56.684 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.380150+0000", 2023-10-01T10:48:56.684 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.1629999999999996e-06 2023-10-01T10:48:56.684 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.685 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.685 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:48:56.685 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.380197+0000", 2023-10-01T10:48:56.685 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.6882999999999998e-05 2023-10-01T10:48:56.685 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.685 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.685 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:48:56.685 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.380312+0000", 2023-10-01T10:48:56.685 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000114335 2023-10-01T10:48:56.685 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.685 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.685 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 3,4", 2023-10-01T10:48:56.686 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.380421+0000", 2023-10-01T10:48:56.686 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000109941 2023-10-01T10:48:56.686 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.686 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.686 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:48:56.686 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.607441+0000", 2023-10-01T10:48:56.686 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.22701929400000001 2023-10-01T10:48:56.686 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.686 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.686 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:48:56.686 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.607460+0000", 2023-10-01T10:48:56.686 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.8989000000000001e-05 2023-10-01T10:48:56.687 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.687 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.687 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:48:56.687 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.615215+0000", 2023-10-01T10:48:56.687 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0077555790000000003 2023-10-01T10:48:56.687 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.687 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.687 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:48:56.687 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.615270+0000", 2023-10-01T10:48:56.687 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.4392999999999999e-05 2023-10-01T10:48:56.687 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.687 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.688 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:48:56.688 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.615316+0000", 2023-10-01T10:48:56.688 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.6637000000000001e-05 2023-10-01T10:48:56.688 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:48:56.688 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:48:56.688 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:48:56.688 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.688 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.688 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.35649.0:158056 9.6 e137/128 9:68800f44:::benchmark_data_smithi078_113121_object46241:head v 137'10106, mlcod=137'10103)", 2023-10-01T10:48:56.688 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:43:39.383516+0000", 2023-10-01T10:48:56.688 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 317.26932355399998, 2023-10-01T10:48:56.688 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.23990089000000001, 2023-10-01T10:48:56.689 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:48:56.689 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:48:56.689 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:48:56.689 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.689 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:48:56.689 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.383516+0000", 2023-10-01T10:48:56.689 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:48:56.689 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.689 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.689 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:48:56.689 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.383516+0000", 2023-10-01T10:48:56.689 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:48:56.690 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.690 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.690 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:48:56.690 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.383515+0000", 2023-10-01T10:48:56.690 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:48:56.690 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.690 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.690 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:48:56.690 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.383528+0000", 2023-10-01T10:48:56.690 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.2992e-05 2023-10-01T10:48:56.690 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.690 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.691 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:48:56.691 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.607221+0000", 2023-10-01T10:48:56.691 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.223692314 2023-10-01T10:48:56.691 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.691 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.691 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:48:56.691 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.607230+0000", 2023-10-01T10:48:56.691 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.1290000000000002e-06 2023-10-01T10:48:56.691 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.691 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.691 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:48:56.691 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.607294+0000", 2023-10-01T10:48:56.691 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.4319000000000005e-05 2023-10-01T10:48:56.692 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.692 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.692 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:48:56.692 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.607317+0000", 2023-10-01T10:48:56.692 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.3332999999999998e-05 2023-10-01T10:48:56.692 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.692 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.692 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:48:56.692 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.623388+0000", 2023-10-01T10:48:56.692 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.016070261999999998 2023-10-01T10:48:56.692 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.692 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.693 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:48:56.693 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.623417+0000", 2023-10-01T10:48:56.693 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.9612e-05 2023-10-01T10:48:56.693 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:48:56.693 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:48:56.693 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:48:56.693 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.693 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.693 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.35649.0:158061 9.d e137/128 9:b93f3ebf:::benchmark_data_smithi078_113121_object46246:head v 137'9694, mlcod=137'9689)", 2023-10-01T10:48:56.693 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:43:39.393875+0000", 2023-10-01T10:48:56.693 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 317.25896521200002, 2023-10-01T10:48:56.693 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.23092847, 2023-10-01T10:48:56.694 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:48:56.694 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:48:56.694 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:48:56.694 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.694 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:48:56.694 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.393875+0000", 2023-10-01T10:48:56.694 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:48:56.694 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.694 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.694 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:48:56.694 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.393875+0000", 2023-10-01T10:48:56.694 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:48:56.695 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.695 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.695 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:48:56.695 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.393873+0000", 2023-10-01T10:48:56.695 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999986 2023-10-01T10:48:56.695 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.695 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.695 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:48:56.695 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.393888+0000", 2023-10-01T10:48:56.695 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.4620000000000001e-05 2023-10-01T10:48:56.695 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.695 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.695 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:48:56.696 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.607189+0000", 2023-10-01T10:48:56.696 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.21330076000000001 2023-10-01T10:48:56.696 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.696 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.696 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:48:56.696 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.607208+0000", 2023-10-01T10:48:56.696 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.8913999999999999e-05 2023-10-01T10:48:56.696 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.696 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.696 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:48:56.696 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.612039+0000", 2023-10-01T10:48:56.697 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0048318199999999997 2023-10-01T10:48:56.697 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.697 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.697 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:48:56.697 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.612080+0000", 2023-10-01T10:48:56.697 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.0868999999999998e-05 2023-10-01T10:48:56.697 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.697 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.697 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:48:56.697 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.624786+0000", 2023-10-01T10:48:56.697 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.012705615 2023-10-01T10:48:56.697 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.697 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.698 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:48:56.698 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.624803+0000", 2023-10-01T10:48:56.698 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.7297999999999999e-05 2023-10-01T10:48:56.698 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:48:56.698 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:48:56.698 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:48:56.698 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.698 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.698 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.45818.0:54050 10.2 10:45a8f827:::benchmark_data_smithi078_135100_object54049:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e156)", 2023-10-01T10:48:56.698 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:46:44.821302+0000", 2023-10-01T10:48:56.698 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 131.83153815399999, 2023-10-01T10:48:56.699 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.15738976800000001, 2023-10-01T10:48:56.699 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:48:56.699 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:48:56.699 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:48:56.699 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.45818", 2023-10-01T10:48:56.699 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1151403832", 2023-10-01T10:48:56.699 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 54050 2023-10-01T10:48:56.699 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.699 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:48:56.699 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.699 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:48:56.699 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.821302+0000", 2023-10-01T10:48:56.699 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:48:56.700 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.700 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.700 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:48:56.700 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.821302+0000", 2023-10-01T10:48:56.700 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:48:56.700 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.700 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.700 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:48:56.700 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.821300+0000", 2023-10-01T10:48:56.700 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999981 2023-10-01T10:48:56.700 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.700 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.701 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:48:56.701 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.821336+0000", 2023-10-01T10:48:56.701 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.5834000000000003e-05 2023-10-01T10:48:56.701 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.701 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.701 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:48:56.701 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.821347+0000", 2023-10-01T10:48:56.701 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1396000000000001e-05 2023-10-01T10:48:56.701 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.701 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.701 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:48:56.701 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.821354+0000", 2023-10-01T10:48:56.702 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.5259999999999999e-06 2023-10-01T10:48:56.702 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.702 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.702 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:48:56.702 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.821397+0000", 2023-10-01T10:48:56.702 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.3189000000000003e-05 2023-10-01T10:48:56.702 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.702 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.702 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:48:56.702 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.821518+0000", 2023-10-01T10:48:56.702 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00012084299999999999 2023-10-01T10:48:56.702 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.703 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.703 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 0,4", 2023-10-01T10:48:56.703 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.821621+0000", 2023-10-01T10:48:56.703 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00010375800000000001 2023-10-01T10:48:56.703 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.703 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.703 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:48:56.703 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.823694+0000", 2023-10-01T10:48:56.703 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0020727620000000001 2023-10-01T10:48:56.703 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.703 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.703 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:48:56.703 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.823733+0000", 2023-10-01T10:48:56.704 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.8686000000000002e-05 2023-10-01T10:48:56.704 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.704 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.704 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:48:56.704 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.978579+0000", 2023-10-01T10:48:56.704 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.15484651799999999 2023-10-01T10:48:56.704 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.704 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.704 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:48:56.704 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.978641+0000", 2023-10-01T10:48:56.704 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.1952000000000006e-05 2023-10-01T10:48:56.704 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.705 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.705 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:48:56.705 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.978692+0000", 2023-10-01T10:48:56.705 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.0124000000000003e-05 2023-10-01T10:48:56.705 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:48:56.705 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:48:56.705 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:48:56.705 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.705 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.705 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.45818.0:54058 10.2 10:4fafcc28:::benchmark_data_smithi078_135100_object54057:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e156)", 2023-10-01T10:48:56.705 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:46:44.826645+0000", 2023-10-01T10:48:56.705 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 131.82619521500001, 2023-10-01T10:48:56.705 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.157624084, 2023-10-01T10:48:56.706 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:48:56.706 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:48:56.706 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:48:56.706 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.45818", 2023-10-01T10:48:56.706 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1151403832", 2023-10-01T10:48:56.706 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 54058 2023-10-01T10:48:56.706 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.706 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:48:56.706 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.706 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:48:56.706 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.826645+0000", 2023-10-01T10:48:56.706 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:48:56.707 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.707 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.707 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:48:56.707 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.826645+0000", 2023-10-01T10:48:56.707 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:48:56.707 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.707 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.707 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:48:56.707 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.826643+0000", 2023-10-01T10:48:56.707 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999986 2023-10-01T10:48:56.707 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.708 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.708 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:48:56.708 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.826675+0000", 2023-10-01T10:48:56.708 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.1878999999999998e-05 2023-10-01T10:48:56.708 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.708 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.708 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:48:56.708 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.826687+0000", 2023-10-01T10:48:56.708 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1701e-05 2023-10-01T10:48:56.708 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.708 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.708 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:48:56.709 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.826694+0000", 2023-10-01T10:48:56.709 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.0609999999999998e-06 2023-10-01T10:48:56.709 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.709 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.709 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:48:56.709 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.826724+0000", 2023-10-01T10:48:56.709 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.0159999999999999e-05 2023-10-01T10:48:56.709 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.709 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.709 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:48:56.709 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.826827+0000", 2023-10-01T10:48:56.709 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000102357 2023-10-01T10:48:56.709 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.710 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.710 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 0,4", 2023-10-01T10:48:56.710 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.826927+0000", 2023-10-01T10:48:56.710 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000100509 2023-10-01T10:48:56.710 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.710 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.710 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:48:56.710 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.828932+0000", 2023-10-01T10:48:56.710 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.002005476 2023-10-01T10:48:56.710 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.710 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.711 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:48:56.711 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.829489+0000", 2023-10-01T10:48:56.711 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000556147 2023-10-01T10:48:56.711 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.711 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.711 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:48:56.711 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.983360+0000", 2023-10-01T10:48:56.711 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.153871491 2023-10-01T10:48:56.711 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.711 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.711 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:48:56.711 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.984214+0000", 2023-10-01T10:48:56.712 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00085380500000000002 2023-10-01T10:48:56.712 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.712 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.712 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:48:56.712 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.984269+0000", 2023-10-01T10:48:56.712 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.4833000000000002e-05 2023-10-01T10:48:56.712 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:48:56.712 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:48:56.712 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:48:56.712 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.712 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.712 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.45818.0:78729 10.2 10:49346079:::benchmark_data_smithi078_135100_object78728:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e157)", 2023-10-01T10:48:56.713 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:47:07.662877+0000", 2023-10-01T10:48:56.713 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 108.98996285600001, 2023-10-01T10:48:56.713 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.158240191, 2023-10-01T10:48:56.713 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:48:56.713 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:48:56.713 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:48:56.713 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.45818", 2023-10-01T10:48:56.713 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1151403832", 2023-10-01T10:48:56.713 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 78729 2023-10-01T10:48:56.713 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.713 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:48:56.713 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.713 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:48:56.714 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:07.662877+0000", 2023-10-01T10:48:56.714 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:48:56.714 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.714 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.714 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:48:56.714 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:07.662877+0000", 2023-10-01T10:48:56.714 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:48:56.714 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.714 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.714 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:48:56.715 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:07.662876+0000", 2023-10-01T10:48:56.715 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:48:56.715 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.715 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.715 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:48:56.715 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:07.662899+0000", 2023-10-01T10:48:56.715 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.3572999999999999e-05 2023-10-01T10:48:56.715 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.715 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.715 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:48:56.715 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:07.662912+0000", 2023-10-01T10:48:56.715 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.2174999999999999e-05 2023-10-01T10:48:56.715 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.716 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.716 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:48:56.716 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:07.662919+0000", 2023-10-01T10:48:56.716 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.3490000000000003e-06 2023-10-01T10:48:56.716 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.716 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.716 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:48:56.716 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:07.666743+0000", 2023-10-01T10:48:56.716 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0038235529999999999 2023-10-01T10:48:56.716 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.716 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.717 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:48:56.717 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:07.666947+0000", 2023-10-01T10:48:56.717 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00020447799999999999 2023-10-01T10:48:56.717 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.717 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.717 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 0,4", 2023-10-01T10:48:56.717 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:07.667086+0000", 2023-10-01T10:48:56.717 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00013851 2023-10-01T10:48:56.717 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.717 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.717 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:48:56.717 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:07.669525+0000", 2023-10-01T10:48:56.718 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0024390089999999998 2023-10-01T10:48:56.718 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.718 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.718 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:48:56.718 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:07.674874+0000", 2023-10-01T10:48:56.718 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0053494170000000004 2023-10-01T10:48:56.718 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.718 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.718 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:48:56.718 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:07.820889+0000", 2023-10-01T10:48:56.718 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.14601512999999999 2023-10-01T10:48:56.718 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.719 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.719 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:48:56.719 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:07.821002+0000", 2023-10-01T10:48:56.719 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00011325299999999999 2023-10-01T10:48:56.719 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.719 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.719 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:48:56.719 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:07.821117+0000", 2023-10-01T10:48:56.719 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00011488 2023-10-01T10:48:56.719 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:48:56.719 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:48:56.719 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:48:56.720 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.720 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.720 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.45818.0:78735 10.2 10:440039f9:::benchmark_data_smithi078_135100_object78734:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e157)", 2023-10-01T10:48:56.720 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:47:07.668622+0000", 2023-10-01T10:48:56.720 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 108.98421820199999, 2023-10-01T10:48:56.720 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.15278265899999999, 2023-10-01T10:48:56.720 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:48:56.720 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:48:56.720 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:48:56.720 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.45818", 2023-10-01T10:48:56.720 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1151403832", 2023-10-01T10:48:56.720 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 78735 2023-10-01T10:48:56.721 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.721 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:48:56.721 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.721 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:48:56.721 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:07.668622+0000", 2023-10-01T10:48:56.721 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:48:56.721 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.721 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.721 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:48:56.721 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:07.668622+0000", 2023-10-01T10:48:56.721 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:48:56.721 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.722 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.722 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:48:56.722 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:07.668620+0000", 2023-10-01T10:48:56.722 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999986 2023-10-01T10:48:56.722 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.722 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.722 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:48:56.722 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:07.668670+0000", 2023-10-01T10:48:56.722 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.0033e-05 2023-10-01T10:48:56.722 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.722 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.722 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:48:56.723 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:07.668682+0000", 2023-10-01T10:48:56.723 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.2152e-05 2023-10-01T10:48:56.723 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.723 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.723 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:48:56.723 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:07.668690+0000", 2023-10-01T10:48:56.723 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.695e-06 2023-10-01T10:48:56.723 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.723 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.723 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:48:56.723 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:07.669548+0000", 2023-10-01T10:48:56.724 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00085787700000000005 2023-10-01T10:48:56.724 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.724 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.724 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:48:56.724 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:07.669666+0000", 2023-10-01T10:48:56.724 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000117488 2023-10-01T10:48:56.724 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.724 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.724 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 0,4", 2023-10-01T10:48:56.724 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:07.669759+0000", 2023-10-01T10:48:56.724 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.3902999999999999e-05 2023-10-01T10:48:56.724 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.725 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.725 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:48:56.725 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:07.674881+0000", 2023-10-01T10:48:56.725 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0051212549999999999 2023-10-01T10:48:56.725 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.725 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.725 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:48:56.725 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:07.674907+0000", 2023-10-01T10:48:56.725 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.6126999999999999e-05 2023-10-01T10:48:56.725 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.725 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.725 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:48:56.726 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:07.821310+0000", 2023-10-01T10:48:56.726 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.1464029 2023-10-01T10:48:56.726 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.726 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.726 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:48:56.726 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:07.821365+0000", 2023-10-01T10:48:56.726 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.5316999999999999e-05 2023-10-01T10:48:56.726 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.726 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.726 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:48:56.726 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:07.821404+0000", 2023-10-01T10:48:56.726 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.9357000000000003e-05 2023-10-01T10:48:56.727 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:48:56.727 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:48:56.727 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:48:56.727 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.727 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.727 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.45818.0:78738 10.2 10:4f5064ff:::benchmark_data_smithi078_135100_object78737:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e157)", 2023-10-01T10:48:56.727 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:47:07.672758+0000", 2023-10-01T10:48:56.727 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 108.980081441, 2023-10-01T10:48:56.727 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.16005508700000001, 2023-10-01T10:48:56.727 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:48:56.727 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:48:56.728 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:48:56.728 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.45818", 2023-10-01T10:48:56.728 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1151403832", 2023-10-01T10:48:56.728 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 78738 2023-10-01T10:48:56.728 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.728 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:48:56.728 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.728 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:48:56.728 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:07.672758+0000", 2023-10-01T10:48:56.728 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:48:56.728 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.729 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.729 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:48:56.729 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:07.672758+0000", 2023-10-01T10:48:56.729 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:48:56.729 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.729 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.729 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:48:56.729 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:07.672757+0000", 2023-10-01T10:48:56.729 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999986 2023-10-01T10:48:56.729 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.729 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.729 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:48:56.730 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:07.672813+0000", 2023-10-01T10:48:56.730 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.6158e-05 2023-10-01T10:48:56.730 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.730 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.730 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:48:56.730 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:07.672827+0000", 2023-10-01T10:48:56.730 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.3623e-05 2023-10-01T10:48:56.730 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.730 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.730 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:48:56.730 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:07.672835+0000", 2023-10-01T10:48:56.731 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.5560000000000008e-06 2023-10-01T10:48:56.731 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.731 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.731 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:48:56.731 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:07.674927+0000", 2023-10-01T10:48:56.731 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.002091444 2023-10-01T10:48:56.731 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.731 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.731 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:48:56.731 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:07.675089+0000", 2023-10-01T10:48:56.731 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00016278500000000001 2023-10-01T10:48:56.731 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.732 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.732 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 0,4", 2023-10-01T10:48:56.732 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:07.675234+0000", 2023-10-01T10:48:56.732 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00014487500000000001 2023-10-01T10:48:56.732 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.732 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.732 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:48:56.732 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:07.677449+0000", 2023-10-01T10:48:56.732 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.002214733 2023-10-01T10:48:56.732 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.734 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.735 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:48:56.735 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:07.678399+0000", 2023-10-01T10:48:56.735 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00095040900000000004 2023-10-01T10:48:56.735 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.735 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.735 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:48:56.735 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:07.832714+0000", 2023-10-01T10:48:56.735 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.15431445699999999 2023-10-01T10:48:56.735 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.735 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.735 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:48:56.736 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:07.832771+0000", 2023-10-01T10:48:56.736 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.7262000000000002e-05 2023-10-01T10:48:56.736 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.736 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.736 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:48:56.736 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:07.832814+0000", 2023-10-01T10:48:56.736 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.2354999999999997e-05 2023-10-01T10:48:56.736 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:48:56.736 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:48:56.736 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:48:56.736 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.736 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.736 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.45818.0:78751 10.2 10:4c53d406:::benchmark_data_smithi078_135100_object78750:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e157)", 2023-10-01T10:48:56.737 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:47:07.682961+0000", 2023-10-01T10:48:56.737 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 108.96987875799999, 2023-10-01T10:48:56.737 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.15006907899999999, 2023-10-01T10:48:56.737 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:48:56.737 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:48:56.737 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:48:56.737 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.45818", 2023-10-01T10:48:56.737 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1151403832", 2023-10-01T10:48:56.737 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 78751 2023-10-01T10:48:56.737 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.737 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:48:56.737 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.737 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:48:56.738 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:07.682961+0000", 2023-10-01T10:48:56.738 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:48:56.738 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.738 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.738 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:48:56.738 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:07.682961+0000", 2023-10-01T10:48:56.738 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:48:56.738 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.738 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.738 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:48:56.738 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:07.682960+0000", 2023-10-01T10:48:56.739 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999986 2023-10-01T10:48:56.739 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.739 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.739 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:48:56.739 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:07.683006+0000", 2023-10-01T10:48:56.739 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.6684999999999998e-05 2023-10-01T10:48:56.739 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.739 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.739 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:48:56.739 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:07.683017+0000", 2023-10-01T10:48:56.739 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.0933e-05 2023-10-01T10:48:56.739 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.739 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.740 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:48:56.740 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:07.683024+0000", 2023-10-01T10:48:56.740 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.9450000000000004e-06 2023-10-01T10:48:56.740 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.740 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.740 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:48:56.740 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:07.683059+0000", 2023-10-01T10:48:56.740 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.5157000000000002e-05 2023-10-01T10:48:56.740 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.740 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.740 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:48:56.740 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:07.683175+0000", 2023-10-01T10:48:56.740 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000115188 2023-10-01T10:48:56.741 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.741 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.741 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 0,4", 2023-10-01T10:48:56.741 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:07.683269+0000", 2023-10-01T10:48:56.741 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.4423e-05 2023-10-01T10:48:56.741 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.741 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.741 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:48:56.741 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:07.685403+0000", 2023-10-01T10:48:56.741 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.002134202 2023-10-01T10:48:56.741 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.741 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.742 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:48:56.742 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:07.820265+0000", 2023-10-01T10:48:56.742 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.13486202899999999 2023-10-01T10:48:56.742 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.742 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.742 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:48:56.742 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:07.832957+0000", 2023-10-01T10:48:56.742 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.012691354 2023-10-01T10:48:56.742 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.742 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.742 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:48:56.742 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:07.832999+0000", 2023-10-01T10:48:56.743 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.2082000000000001e-05 2023-10-01T10:48:56.743 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.743 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.743 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:48:56.743 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:07.833030+0000", 2023-10-01T10:48:56.743 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.1545999999999999e-05 2023-10-01T10:48:56.743 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:48:56.743 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:48:56.743 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:48:56.743 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.743 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.743 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.45818.0:118223 10.b 10:d395dcb0:::benchmark_data_smithi078_135100_object8310:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e160)", 2023-10-01T10:48:56.744 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:47:53.735416+0000", 2023-10-01T10:48:56.744 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 62.917423464000002, 2023-10-01T10:48:56.744 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.0362317210000001, 2023-10-01T10:48:56.744 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:48:56.744 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:48:56.744 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:48:56.744 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.45818", 2023-10-01T10:48:56.744 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1151403832", 2023-10-01T10:48:56.744 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 118223 2023-10-01T10:48:56.744 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.744 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:48:56.744 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.745 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:48:56.745 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:53.735416+0000", 2023-10-01T10:48:56.745 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:48:56.745 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.745 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.745 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:48:56.745 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:53.735416+0000", 2023-10-01T10:48:56.745 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:48:56.745 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.745 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.745 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:48:56.745 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:53.735414+0000", 2023-10-01T10:48:56.746 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999981 2023-10-01T10:48:56.746 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.746 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.746 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:48:56.746 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:53.735423+0000", 2023-10-01T10:48:56.746 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.507e-06 2023-10-01T10:48:56.746 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.746 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.746 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:48:56.746 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:53.735442+0000", 2023-10-01T10:48:56.746 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.9018e-05 2023-10-01T10:48:56.746 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.747 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.747 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:48:56.747 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:53.735452+0000", 2023-10-01T10:48:56.747 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.0489000000000001e-05 2023-10-01T10:48:56.747 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.747 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.747 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:48:56.747 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:53.735508+0000", 2023-10-01T10:48:56.747 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.5203e-05 2023-10-01T10:48:56.747 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.747 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.747 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T10:48:56.747 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:53.735518+0000", 2023-10-01T10:48:56.747 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.0387000000000001e-05 2023-10-01T10:48:56.748 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.748 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.748 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:48:56.748 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:54.756973+0000", 2023-10-01T10:48:56.748 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.02145496 2023-10-01T10:48:56.748 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.748 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.748 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:48:56.748 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:54.759690+0000", 2023-10-01T10:48:56.748 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.002716605 2023-10-01T10:48:56.748 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.748 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.748 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 0,6", 2023-10-01T10:48:56.749 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:54.759830+0000", 2023-10-01T10:48:56.749 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00014060900000000001 2023-10-01T10:48:56.749 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.749 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.749 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:48:56.749 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:54.768040+0000", 2023-10-01T10:48:56.749 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0082101229999999997 2023-10-01T10:48:56.749 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.749 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.749 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:48:56.749 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:54.768050+0000", 2023-10-01T10:48:56.749 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.8660000000000001e-06 2023-10-01T10:48:56.750 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.750 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.750 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:48:56.750 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:54.771575+0000", 2023-10-01T10:48:56.750 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0035249669999999999 2023-10-01T10:48:56.750 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.750 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.750 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:48:56.750 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:54.771621+0000", 2023-10-01T10:48:56.750 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.5657e-05 2023-10-01T10:48:56.750 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.750 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.751 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:48:56.751 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:54.771648+0000", 2023-10-01T10:48:56.751 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.7322999999999999e-05 2023-10-01T10:48:56.751 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:48:56.751 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:48:56.751 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:48:56.751 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.751 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.751 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.45818.0:118284 10.b 10:d81fece7:::benchmark_data_smithi078_135100_object8371:head [delete] snapc 0=[] ondisk+write+known_if_redirected e160)", 2023-10-01T10:48:56.751 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:47:53.826422+0000", 2023-10-01T10:48:56.751 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 62.826418224999998, 2023-10-01T10:48:56.751 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.94535291600000004, 2023-10-01T10:48:56.751 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:48:56.752 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:48:56.752 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:48:56.752 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.45818", 2023-10-01T10:48:56.752 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1151403832", 2023-10-01T10:48:56.752 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 118284 2023-10-01T10:48:56.752 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.752 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:48:56.752 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.752 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:48:56.752 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:53.826422+0000", 2023-10-01T10:48:56.752 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:48:56.752 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.753 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.753 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:48:56.753 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:53.826422+0000", 2023-10-01T10:48:56.753 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:48:56.753 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.753 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.753 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:48:56.753 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:53.826420+0000", 2023-10-01T10:48:56.753 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999986 2023-10-01T10:48:56.753 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.753 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.753 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:48:56.753 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:53.826428+0000", 2023-10-01T10:48:56.754 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.6199999999999999e-06 2023-10-01T10:48:56.754 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.754 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.754 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:48:56.754 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:53.826440+0000", 2023-10-01T10:48:56.754 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.2723e-05 2023-10-01T10:48:56.754 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.754 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.754 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:48:56.754 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:53.826447+0000", 2023-10-01T10:48:56.754 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.246e-06 2023-10-01T10:48:56.755 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.755 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.755 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:48:56.755 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:53.826501+0000", 2023-10-01T10:48:56.755 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.4039999999999998e-05 2023-10-01T10:48:56.755 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.755 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.755 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T10:48:56.755 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:53.826511+0000", 2023-10-01T10:48:56.755 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.9350000000000006e-06 2023-10-01T10:48:56.755 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.755 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.756 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:48:56.756 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:54.763997+0000", 2023-10-01T10:48:56.756 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.93748644699999995 2023-10-01T10:48:56.756 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.756 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.756 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:48:56.756 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:54.764126+0000", 2023-10-01T10:48:56.756 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00012912399999999999 2023-10-01T10:48:56.756 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.756 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.756 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 0,6", 2023-10-01T10:48:56.756 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:54.764249+0000", 2023-10-01T10:48:56.757 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000122853 2023-10-01T10:48:56.757 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.757 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.757 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:48:56.757 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:54.768147+0000", 2023-10-01T10:48:56.757 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0038984250000000001 2023-10-01T10:48:56.757 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.757 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.757 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:48:56.757 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:54.769785+0000", 2023-10-01T10:48:56.757 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.001637714 2023-10-01T10:48:56.757 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.757 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.758 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:48:56.758 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:54.771694+0000", 2023-10-01T10:48:56.758 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.001908996 2023-10-01T10:48:56.758 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.758 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.758 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:48:56.758 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:54.771739+0000", 2023-10-01T10:48:56.758 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.4503999999999999e-05 2023-10-01T10:48:56.758 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.758 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.758 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:48:56.759 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:54.771775+0000", 2023-10-01T10:48:56.759 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.5913e-05 2023-10-01T10:48:56.759 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:48:56.759 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:48:56.759 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:48:56.759 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.759 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.759 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.45818.0:153201 10.b 10:db449973:::benchmark_data_smithi078_135100_object43288:head [delete] snapc 0=[] ondisk+write+known_if_redirected e164)", 2023-10-01T10:48:56.759 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:48:41.352950+0000", 2023-10-01T10:48:56.759 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 15.299889426, 2023-10-01T10:48:56.759 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.15791414100000001, 2023-10-01T10:48:56.759 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:48:56.759 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:48:56.760 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:48:56.760 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.45818", 2023-10-01T10:48:56.760 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1151403832", 2023-10-01T10:48:56.760 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 153201 2023-10-01T10:48:56.760 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.760 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:48:56.760 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.760 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:48:56.760 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.352950+0000", 2023-10-01T10:48:56.760 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:48:56.760 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.760 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.760 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:48:56.761 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.352950+0000", 2023-10-01T10:48:56.761 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:48:56.761 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.761 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.761 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:48:56.761 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.352950+0000", 2023-10-01T10:48:56.761 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:48:56.761 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.761 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.761 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:48:56.761 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.352955+0000", 2023-10-01T10:48:56.761 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.0329999999999999e-06 2023-10-01T10:48:56.762 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.762 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.762 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:48:56.762 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.352964+0000", 2023-10-01T10:48:56.762 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.0729999999999994e-06 2023-10-01T10:48:56.762 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.762 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.762 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:48:56.762 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.352970+0000", 2023-10-01T10:48:56.762 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.3080000000000004e-06 2023-10-01T10:48:56.762 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.762 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.763 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:48:56.763 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.357580+0000", 2023-10-01T10:48:56.763 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0046096210000000004 2023-10-01T10:48:56.763 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.763 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.763 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:48:56.763 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.357691+0000", 2023-10-01T10:48:56.763 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000111603 2023-10-01T10:48:56.763 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.763 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.763 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 0,6", 2023-10-01T10:48:56.763 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.357799+0000", 2023-10-01T10:48:56.764 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00010783 2023-10-01T10:48:56.764 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.764 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.764 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:48:56.764 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.364982+0000", 2023-10-01T10:48:56.764 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0071828530000000003 2023-10-01T10:48:56.764 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.764 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.764 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:48:56.764 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.370066+0000", 2023-10-01T10:48:56.765 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0050839129999999998 2023-10-01T10:48:56.765 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.765 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.765 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:48:56.765 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.510711+0000", 2023-10-01T10:48:56.765 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.14064460200000001 2023-10-01T10:48:56.765 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.765 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.765 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:48:56.765 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.510819+0000", 2023-10-01T10:48:56.765 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000108942 2023-10-01T10:48:56.765 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.766 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.766 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:48:56.766 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.510865+0000", 2023-10-01T10:48:56.766 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.5160999999999997e-05 2023-10-01T10:48:56.766 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:48:56.766 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:48:56.766 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:48:56.766 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.766 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.766 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.45818.0:153209 10.2 10:45500809:::benchmark_data_smithi078_135100_object43296:head [delete] snapc 0=[] ondisk+write+known_if_redirected e164)", 2023-10-01T10:48:56.766 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:48:41.364877+0000", 2023-10-01T10:48:56.767 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 15.287963174, 2023-10-01T10:48:56.767 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.14093081099999999, 2023-10-01T10:48:56.767 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:48:56.767 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:48:56.767 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:48:56.767 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.45818", 2023-10-01T10:48:56.767 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1151403832", 2023-10-01T10:48:56.767 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 153209 2023-10-01T10:48:56.767 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.767 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:48:56.767 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.767 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:48:56.768 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.364877+0000", 2023-10-01T10:48:56.768 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:48:56.768 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.768 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.768 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:48:56.768 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.364877+0000", 2023-10-01T10:48:56.768 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:48:56.768 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.768 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.768 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:48:56.768 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.364875+0000", 2023-10-01T10:48:56.768 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999986 2023-10-01T10:48:56.768 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.769 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.769 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:48:56.769 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.364884+0000", 2023-10-01T10:48:56.769 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.3340000000000008e-06 2023-10-01T10:48:56.769 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.769 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.769 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:48:56.769 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.364901+0000", 2023-10-01T10:48:56.769 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.6413e-05 2023-10-01T10:48:56.769 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.769 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.769 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:48:56.769 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.364909+0000", 2023-10-01T10:48:56.770 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.3350000000000007e-06 2023-10-01T10:48:56.770 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.770 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.770 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:48:56.770 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.365376+0000", 2023-10-01T10:48:56.770 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000467274 2023-10-01T10:48:56.770 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.770 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.770 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:48:56.770 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.365527+0000", 2023-10-01T10:48:56.770 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000150841 2023-10-01T10:48:56.770 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.771 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.771 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 0,4", 2023-10-01T10:48:56.771 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.365657+0000", 2023-10-01T10:48:56.771 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00012948899999999999 2023-10-01T10:48:56.771 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.771 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.771 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:48:56.771 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.369390+0000", 2023-10-01T10:48:56.771 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0037327630000000001 2023-10-01T10:48:56.771 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.771 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.771 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:48:56.771 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.377325+0000", 2023-10-01T10:48:56.772 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.007935503 2023-10-01T10:48:56.772 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.772 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.772 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:48:56.772 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.505704+0000", 2023-10-01T10:48:56.772 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.128379245 2023-10-01T10:48:56.772 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.772 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.772 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:48:56.772 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.505766+0000", 2023-10-01T10:48:56.772 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.1982000000000004e-05 2023-10-01T10:48:56.773 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.773 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.773 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:48:56.773 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.505808+0000", 2023-10-01T10:48:56.773 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.1230000000000003e-05 2023-10-01T10:48:56.773 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:48:56.773 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:48:56.773 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:48:56.773 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.773 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.773 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.45818.0:153214 10.b 10:da87ff1a:::benchmark_data_smithi078_135100_object43301:head [delete] snapc 0=[] ondisk+write+known_if_redirected e164)", 2023-10-01T10:48:56.773 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:48:41.365405+0000", 2023-10-01T10:48:56.774 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 15.287435111000001, 2023-10-01T10:48:56.774 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.14560463800000001, 2023-10-01T10:48:56.774 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:48:56.774 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:48:56.774 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:48:56.774 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.45818", 2023-10-01T10:48:56.774 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1151403832", 2023-10-01T10:48:56.774 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 153214 2023-10-01T10:48:56.774 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.774 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:48:56.774 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.774 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:48:56.775 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.365405+0000", 2023-10-01T10:48:56.775 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:48:56.775 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.775 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.775 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:48:56.775 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.365405+0000", 2023-10-01T10:48:56.775 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:48:56.775 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.775 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.775 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:48:56.775 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.365404+0000", 2023-10-01T10:48:56.775 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:48:56.775 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.776 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.776 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:48:56.776 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.365409+0000", 2023-10-01T10:48:56.776 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.4879999999999998e-06 2023-10-01T10:48:56.776 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.776 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.776 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:48:56.776 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.365420+0000", 2023-10-01T10:48:56.776 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.058e-05 2023-10-01T10:48:56.776 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.776 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.776 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:48:56.776 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.365427+0000", 2023-10-01T10:48:56.777 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.3830000000000003e-06 2023-10-01T10:48:56.777 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.777 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.777 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:48:56.777 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.365486+0000", 2023-10-01T10:48:56.777 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.8323999999999999e-05 2023-10-01T10:48:56.777 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.777 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.777 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:48:56.777 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.365618+0000", 2023-10-01T10:48:56.777 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00013231899999999999 2023-10-01T10:48:56.777 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.778 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.778 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 0,6", 2023-10-01T10:48:56.778 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.365734+0000", 2023-10-01T10:48:56.778 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00011576799999999999 2023-10-01T10:48:56.778 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.778 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.778 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:48:56.778 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.373367+0000", 2023-10-01T10:48:56.778 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0076336820000000001 2023-10-01T10:48:56.778 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.778 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.778 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:48:56.779 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.373377+0000", 2023-10-01T10:48:56.779 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.4769999999999992e-06 2023-10-01T10:48:56.779 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.779 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.779 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:48:56.779 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.510900+0000", 2023-10-01T10:48:56.779 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.137523276 2023-10-01T10:48:56.779 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.779 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.779 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:48:56.779 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.510975+0000", 2023-10-01T10:48:56.779 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.4435000000000001e-05 2023-10-01T10:48:56.779 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.779 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.780 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:48:56.780 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.511009+0000", 2023-10-01T10:48:56.780 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.4758000000000002e-05 2023-10-01T10:48:56.780 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:48:56.780 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:48:56.780 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:48:56.780 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.780 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.780 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.45818.0:153215 10.b 10:d2fe9821:::benchmark_data_smithi078_135100_object43302:head [delete] snapc 0=[] ondisk+write+known_if_redirected e164)", 2023-10-01T10:48:56.780 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:48:41.375729+0000", 2023-10-01T10:48:56.780 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 15.277111176, 2023-10-01T10:48:56.780 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.136099622, 2023-10-01T10:48:56.780 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:48:56.781 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:48:56.781 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:48:56.781 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.45818", 2023-10-01T10:48:56.781 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1151403832", 2023-10-01T10:48:56.781 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 153215 2023-10-01T10:48:56.781 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.781 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:48:56.781 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.781 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:48:56.781 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.375729+0000", 2023-10-01T10:48:56.781 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:48:56.781 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.781 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.782 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:48:56.782 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.375729+0000", 2023-10-01T10:48:56.782 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:48:56.782 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.782 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.782 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:48:56.782 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.375727+0000", 2023-10-01T10:48:56.782 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999986 2023-10-01T10:48:56.782 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.782 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.782 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:48:56.783 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.375734+0000", 2023-10-01T10:48:56.783 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.1810000000000004e-06 2023-10-01T10:48:56.783 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.783 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.783 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:48:56.783 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.375747+0000", 2023-10-01T10:48:56.783 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.2926e-05 2023-10-01T10:48:56.783 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.783 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.783 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:48:56.783 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.375753+0000", 2023-10-01T10:48:56.783 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.1659999999999999e-06 2023-10-01T10:48:56.784 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.784 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.784 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:48:56.784 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.375803+0000", 2023-10-01T10:48:56.784 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.9659e-05 2023-10-01T10:48:56.784 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.784 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.784 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:48:56.784 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.375913+0000", 2023-10-01T10:48:56.784 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000109917 2023-10-01T10:48:56.784 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.784 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.785 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 0,6", 2023-10-01T10:48:56.785 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.375998+0000", 2023-10-01T10:48:56.785 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.5068000000000006e-05 2023-10-01T10:48:56.785 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.785 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.785 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:48:56.785 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.379456+0000", 2023-10-01T10:48:56.785 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.003458371 2023-10-01T10:48:56.785 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.785 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.785 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:48:56.785 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.380611+0000", 2023-10-01T10:48:56.785 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.001154055 2023-10-01T10:48:56.786 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.786 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.786 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:48:56.786 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.511707+0000", 2023-10-01T10:48:56.786 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.13109678 2023-10-01T10:48:56.786 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.786 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.786 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:48:56.786 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.511792+0000", 2023-10-01T10:48:56.786 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.4467000000000003e-05 2023-10-01T10:48:56.786 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.786 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.787 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:48:56.787 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.511828+0000", 2023-10-01T10:48:56.787 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.6559e-05 2023-10-01T10:48:56.787 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:48:56.787 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:48:56.787 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:48:56.787 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.787 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.787 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.45818.0:153216 10.2 10:4b6261f5:::benchmark_data_smithi078_135100_object43303:head [delete] snapc 0=[] ondisk+write+known_if_redirected e164)", 2023-10-01T10:48:56.787 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:48:41.378558+0000", 2023-10-01T10:48:56.787 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 15.274282201, 2023-10-01T10:48:56.787 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.133377302, 2023-10-01T10:48:56.788 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:48:56.788 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:48:56.788 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:48:56.788 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.45818", 2023-10-01T10:48:56.788 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1151403832", 2023-10-01T10:48:56.788 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 153216 2023-10-01T10:48:56.788 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.788 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:48:56.788 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.788 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:48:56.788 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.378558+0000", 2023-10-01T10:48:56.788 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:48:56.789 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.789 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.789 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:48:56.789 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.378558+0000", 2023-10-01T10:48:56.789 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:48:56.789 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.789 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.789 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:48:56.789 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.378557+0000", 2023-10-01T10:48:56.789 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:48:56.789 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.789 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.789 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:48:56.790 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.378562+0000", 2023-10-01T10:48:56.790 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.7060000000000001e-06 2023-10-01T10:48:56.790 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.790 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.790 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:48:56.790 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.378574+0000", 2023-10-01T10:48:56.790 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1487e-05 2023-10-01T10:48:56.790 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.790 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.790 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:48:56.790 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.378581+0000", 2023-10-01T10:48:56.790 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.8369999999999999e-06 2023-10-01T10:48:56.791 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.791 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.791 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:48:56.791 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.378622+0000", 2023-10-01T10:48:56.791 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.1755000000000003e-05 2023-10-01T10:48:56.791 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.791 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.791 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:48:56.791 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.378707+0000", 2023-10-01T10:48:56.791 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.4458999999999999e-05 2023-10-01T10:48:56.791 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.791 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.791 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 0,4", 2023-10-01T10:48:56.792 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.378780+0000", 2023-10-01T10:48:56.792 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.3449999999999996e-05 2023-10-01T10:48:56.792 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.792 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.792 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:48:56.792 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.381922+0000", 2023-10-01T10:48:56.792 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0031411410000000001 2023-10-01T10:48:56.792 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.792 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.792 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:48:56.792 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.382605+0000", 2023-10-01T10:48:56.792 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00068317499999999995 2023-10-01T10:48:56.792 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.793 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.793 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:48:56.793 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.511853+0000", 2023-10-01T10:48:56.793 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.12924854299999999 2023-10-01T10:48:56.793 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.793 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.793 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:48:56.793 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.511899+0000", 2023-10-01T10:48:56.793 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.5856000000000003e-05 2023-10-01T10:48:56.793 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:48:56.793 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:48:56.793 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:48:56.794 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:48:41.511935+0000", 2023-10-01T10:48:56.794 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.5921999999999999e-05 2023-10-01T10:48:56.794 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:48:56.794 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:48:56.794 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:48:56.794 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:48:56.794 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:48:56.794 INFO:teuthology.orchestra.run.smithi078.stdout:} 2023-10-01T10:48:56.794 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:56.830 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:56.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:56.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:48:56.937 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:56.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:56.969 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:56.969 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:48:57.071 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:57.106 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:57.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:57.113 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:57.214 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:57.249 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:48:57.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:57.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:48:57.353 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:57.382 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:57.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:57.386 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:48:57.487 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:57.513 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:57.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:57.516 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:48:57.617 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:57.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:57 smithi173 bash[58816]: cluster 2023-10-01T10:48:55.892884+0000 mgr.y (mgr.25072) 939 : cluster 0 pgmap v876: 89 pgs: 89 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 688 op/s 2023-10-01T10:48:57.646 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:57.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:57.651 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:48:57.752 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:57.776 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:57.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:57.779 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:57.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:57 smithi078 bash[75299]: cluster 2023-10-01T10:48:55.892884+0000 mgr.y (mgr.25072) 939 : cluster 0 pgmap v876: 89 pgs: 89 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 688 op/s 2023-10-01T10:48:57.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:57 smithi078 bash[78274]: cluster 2023-10-01T10:48:55.892884+0000 mgr.y (mgr.25072) 939 : cluster 0 pgmap v876: 89 pgs: 89 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 688 op/s 2023-10-01T10:48:57.880 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:57.914 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:57.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:57.918 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:58.019 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:58.046 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:58.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:58.052 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:48:58.152 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:58.188 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:58.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:58.192 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:48:58.293 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:58.322 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:58.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:58.325 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:48:58.426 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:58.448 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:58.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:58.453 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:48:58.554 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:58.581 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:48:58.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:58.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:48:58.684 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:58.717 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:58.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:58.723 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:48:58.824 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:58.857 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:48:58.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:58.866 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:48:58.966 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:58.995 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:58.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:58.997 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:48:59.098 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:59.135 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:48:59.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:59.141 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:48:59.242 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:59.278 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:59.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:59.282 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:48:59.382 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:59.422 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:48:59.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:59.427 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:48:59.527 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:59.554 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:59.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:59.555 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:48:59.574 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:48:59 smithi078 bash[75299]: cluster 2023-10-01T10:48:57.895924+0000 mgr.y (mgr.25072) 940 : cluster 0 pgmap v877: 89 pgs: 89 active+clean; 3.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 634 op/s 2023-10-01T10:48:59.575 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:48:59 smithi078 bash[78274]: cluster 2023-10-01T10:48:57.895924+0000 mgr.y (mgr.25072) 940 : cluster 0 pgmap v877: 89 pgs: 89 active+clean; 3.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 634 op/s 2023-10-01T10:48:59.656 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:59.678 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:48:59.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:59.688 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:48:59.788 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:59.824 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:48:59.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:59.828 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:48:59.828 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:48:59 smithi173 bash[58816]: cluster 2023-10-01T10:48:57.895924+0000 mgr.y (mgr.25072) 940 : cluster 0 pgmap v877: 89 pgs: 89 active+clean; 3.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 634 op/s 2023-10-01T10:48:59.929 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:59.962 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:48:59.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:48:59.967 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:00.067 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:00.102 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:00.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:00.104 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:49:00.205 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:00.235 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:00.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:00.240 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:49:00.331 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:49:00 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:49:00] "GET /metrics HTTP/1.1" 200 36314 "" "Prometheus/2.33.4" 2023-10-01T10:49:00.334 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-01T10:49:00.341 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:00.372 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:00.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:00.375 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:49:00.475 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:00.510 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:00.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:00.513 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:49:00.614 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:00.663 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:00.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:00.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:49:00.766 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:00.792 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:00.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:00.794 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:49:00.895 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:00.922 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:00.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:00.927 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:49:01.028 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:01.062 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:01.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:01.064 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:49:01.165 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:01.199 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:01.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:01.204 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:49:01.305 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:01.340 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:01.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:01.342 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:01.443 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:01.468 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:01.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:01.472 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:49:01.571 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:01 smithi173 bash[58816]: cluster 2023-10-01T10:48:59.896535+0000 mgr.y (mgr.25072) 941 : cluster 0 pgmap v878: 89 pgs: 89 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 820 op/s 2023-10-01T10:49:01.573 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:01.613 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:01.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:01.617 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:49:01.718 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:01.739 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:01.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:01.741 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:49:01.841 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:01.860 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:01.861 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:01 smithi078 bash[78274]: cluster 2023-10-01T10:48:59.896535+0000 mgr.y (mgr.25072) 941 : cluster 0 pgmap v878: 89 pgs: 89 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 820 op/s 2023-10-01T10:49:01.861 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:01 smithi078 bash[75299]: cluster 2023-10-01T10:48:59.896535+0000 mgr.y (mgr.25072) 941 : cluster 0 pgmap v878: 89 pgs: 89 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 820 op/s 2023-10-01T10:49:01.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:01.869 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:49:01.969 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:01.994 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:02.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:02.000 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:49:02.101 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:02.134 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:02.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:02.136 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:49:02.237 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:02.267 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:02.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:02.270 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:49:02.371 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:02.402 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:02.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:02.405 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:49:02.505 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:02.533 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:02.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:02.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:02.639 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:02.668 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:02.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:02.671 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:49:02.771 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:02.780 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:02 smithi173 bash[58816]: audit 2023-10-01T10:49:02.429157+0000 mon.a (mon.0) 1176 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:49:02.805 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:02.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:02.810 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:49:02.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:02 smithi078 bash[75299]: audit 2023-10-01T10:49:02.429157+0000 mon.a (mon.0) 1176 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:49:02.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:02 smithi078 bash[78274]: audit 2023-10-01T10:49:02.429157+0000 mon.a (mon.0) 1176 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:49:02.911 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:02.954 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:02.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:02.960 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:03.061 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:03.102 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:03.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:03.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:03.207 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:03.241 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:03.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:03.242 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:03.343 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:03.365 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:03.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:03.368 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:49:03.469 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:03.508 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:03.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:03.515 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:03.615 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:03.623 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:03 smithi078 bash[75299]: cluster 2023-10-01T10:49:01.897098+0000 mgr.y (mgr.25072) 942 : cluster 0 pgmap v879: 89 pgs: 89 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 636 op/s 2023-10-01T10:49:03.623 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:03 smithi078 bash[78274]: cluster 2023-10-01T10:49:01.897098+0000 mgr.y (mgr.25072) 942 : cluster 0 pgmap v879: 89 pgs: 89 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 636 op/s 2023-10-01T10:49:03.639 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:03.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:03.644 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:49:03.745 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:03.783 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:03.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:03.785 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:49:03.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:03 smithi173 bash[58816]: cluster 2023-10-01T10:49:01.897098+0000 mgr.y (mgr.25072) 942 : cluster 0 pgmap v879: 89 pgs: 89 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 636 op/s 2023-10-01T10:49:03.886 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:03.913 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:03.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:03.916 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:49:04.016 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:04.051 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:04.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:04.053 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:49:04.154 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:04.192 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:04.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:04.196 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:49:04.296 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:04.326 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:04.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:04.330 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:49:04.431 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:04.467 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:04.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:04.471 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:04.572 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:04.608 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:04.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:04.612 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:49:04.712 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:04.740 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:04.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:04.746 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:04.847 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:04.877 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:04.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:04.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:04.983 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:05.021 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:05.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:05.027 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:49:05.127 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:05.160 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:05.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:05.165 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:49:05.266 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:05.297 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:05.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:05.300 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:49:05.401 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:05.425 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:05.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:05.428 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:05.528 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:05.583 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:05 smithi173 bash[58816]: cluster 2023-10-01T10:49:03.897622+0000 mgr.y (mgr.25072) 943 : cluster 0 pgmap v880: 89 pgs: 89 active+clean; 3.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 732 op/s 2023-10-01T10:49:05.588 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:05.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:05.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:49:05.694 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:05.728 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:05.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:05.731 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:49:05.774 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:05 smithi078 bash[78274]: cluster 2023-10-01T10:49:03.897622+0000 mgr.y (mgr.25072) 943 : cluster 0 pgmap v880: 89 pgs: 89 active+clean; 3.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 732 op/s 2023-10-01T10:49:05.774 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:05 smithi078 bash[75299]: cluster 2023-10-01T10:49:03.897622+0000 mgr.y (mgr.25072) 943 : cluster 0 pgmap v880: 89 pgs: 89 active+clean; 3.2 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 732 op/s 2023-10-01T10:49:05.832 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:05.859 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:05.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:05.861 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:49:05.962 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:05.991 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:05.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:05.998 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:06.021 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T10:49:06.086 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T10:49:06.099 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:06.123 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:06.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:06.126 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:06.148 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T10:49:06.198 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T10:49:06.226 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:06.239 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T10:49:06.253 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:06.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:06.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:49:06.281 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T10:49:06.330 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T10:49:06.362 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:06.377 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T10:49:06.390 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T10:49:06.390 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T10:49:06.391 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T10:49:06.391 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T10:49:06.391 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T10:49:06.391 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T10:49:06.391 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T10:49:06.391 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T10:49:06.406 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:06.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:06.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:49:06.509 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:06.540 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:06.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:06.547 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:06.604 INFO:teuthology.orchestra.run.smithi078.stderr:reweighted osd.2 to 1 (10000) 2023-10-01T10:49:06.605 INFO:teuthology.orchestra.run.smithi078.stderr:noscrub is unset 2023-10-01T10:49:06.640 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:06 smithi078 bash[78274]: audit 2023-10-01T10:49:05.965745+0000 mon.c (mon.2) 161 : audit 0 from='client.? 172.21.15.78:0/3537068860' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:49:06.640 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:06 smithi078 bash[78274]: audit 2023-10-01T10:49:06.412469+0000 mon.c (mon.2) 162 : audit 1 from='client.? 172.21.15.78:0/2152499712' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-01T10:49:06.640 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:06 smithi078 bash[78274]: audit 2023-10-01T10:49:06.413405+0000 mon.a (mon.0) 1177 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-01T10:49:06.640 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:06 smithi078 bash[78274]: audit 2023-10-01T10:49:06.465178+0000 mon.c (mon.2) 163 : audit 1 from='client.? 172.21.15.78:0/86836974' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T10:49:06.640 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:06 smithi078 bash[78274]: audit 2023-10-01T10:49:06.466170+0000 mon.a (mon.0) 1178 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T10:49:06.641 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:06 smithi078 bash[75299]: audit 2023-10-01T10:49:05.965745+0000 mon.c (mon.2) 161 : audit 0 from='client.? 172.21.15.78:0/3537068860' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:49:06.641 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:06 smithi078 bash[75299]: audit 2023-10-01T10:49:06.412469+0000 mon.c (mon.2) 162 : audit 1 from='client.? 172.21.15.78:0/2152499712' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-01T10:49:06.641 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:06 smithi078 bash[75299]: audit 2023-10-01T10:49:06.413405+0000 mon.a (mon.0) 1177 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-01T10:49:06.641 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:06 smithi078 bash[75299]: audit 2023-10-01T10:49:06.465178+0000 mon.c (mon.2) 163 : audit 1 from='client.? 172.21.15.78:0/86836974' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T10:49:06.641 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:06 smithi078 bash[75299]: audit 2023-10-01T10:49:06.466170+0000 mon.a (mon.0) 1178 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T10:49:06.647 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:06.674 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:06.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:06.676 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:49:06.777 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:06.807 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:06.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:06.810 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:49:06.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:06 smithi173 bash[58816]: audit 2023-10-01T10:49:05.965745+0000 mon.c (mon.2) 161 : audit 0 from='client.? 172.21.15.78:0/3537068860' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:49:06.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:06 smithi173 bash[58816]: audit 2023-10-01T10:49:06.412469+0000 mon.c (mon.2) 162 : audit 1 from='client.? 172.21.15.78:0/2152499712' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-01T10:49:06.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:06 smithi173 bash[58816]: audit 2023-10-01T10:49:06.413405+0000 mon.a (mon.0) 1177 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-01T10:49:06.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:06 smithi173 bash[58816]: audit 2023-10-01T10:49:06.465178+0000 mon.c (mon.2) 163 : audit 1 from='client.? 172.21.15.78:0/86836974' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T10:49:06.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:06 smithi173 bash[58816]: audit 2023-10-01T10:49:06.466170+0000 mon.a (mon.0) 1178 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T10:49:06.910 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:06.944 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:06.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:06.946 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:49:07.047 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:07.080 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:07.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:07.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:49:07.183 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:07.233 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:07.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:07.241 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:49:07.343 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:07.374 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:07.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:07.379 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:49:07.480 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:07.499 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:07.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:07.501 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:49:07.602 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:07.621 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:07.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:07.622 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:49:07.677 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:07 smithi078 bash[75299]: cluster 2023-10-01T10:49:05.898155+0000 mgr.y (mgr.25072) 944 : cluster 0 pgmap v881: 89 pgs: 89 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2023-10-01T10:49:07.677 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:07 smithi078 bash[75299]: cluster 2023-10-01T10:49:06.581311+0000 mon.a (mon.0) 1179 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:49:07.677 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:07 smithi078 bash[75299]: audit 2023-10-01T10:49:06.592159+0000 mon.a (mon.0) 1180 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-01T10:49:07.677 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:07 smithi078 bash[75299]: audit 2023-10-01T10:49:06.592496+0000 mon.a (mon.0) 1181 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T10:49:07.677 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:07 smithi078 bash[75299]: cluster 2023-10-01T10:49:06.592675+0000 osd.5 (osd.5) 96 : cluster 3 failed to encode map e166 with expected crc 2023-10-01T10:49:07.677 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:07 smithi078 bash[75299]: cluster 2023-10-01T10:49:06.592848+0000 mon.a (mon.0) 1182 : cluster 0 osdmap e166: 8 total, 8 up, 8 in 2023-10-01T10:49:07.723 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:07.749 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:07.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:07.752 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:49:07.853 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:07.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:07 smithi173 bash[58816]: cluster 2023-10-01T10:49:05.898155+0000 mgr.y (mgr.25072) 944 : cluster 0 pgmap v881: 89 pgs: 89 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2023-10-01T10:49:07.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:07 smithi173 bash[58816]: cluster 2023-10-01T10:49:06.581311+0000 mon.a (mon.0) 1179 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:49:07.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:07 smithi173 bash[58816]: audit 2023-10-01T10:49:06.592159+0000 mon.a (mon.0) 1180 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-01T10:49:07.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:07 smithi173 bash[58816]: audit 2023-10-01T10:49:06.592496+0000 mon.a (mon.0) 1181 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T10:49:07.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:07 smithi173 bash[58816]: cluster 2023-10-01T10:49:06.592675+0000 osd.5 (osd.5) 96 : cluster 3 failed to encode map e166 with expected crc 2023-10-01T10:49:07.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:07 smithi173 bash[58816]: cluster 2023-10-01T10:49:06.592848+0000 mon.a (mon.0) 1182 : cluster 0 osdmap e166: 8 total, 8 up, 8 in 2023-10-01T10:49:07.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:07 smithi173 bash[58816]: cluster 2023-10-01T10:49:06.595076+0000 osd.6 (osd.6) 82 : cluster 3 failed to encode map e166 with expected crc 2023-10-01T10:49:07.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:07 smithi173 bash[58816]: cluster 2023-10-01T10:49:06.596966+0000 osd.6 (osd.6) 83 : cluster 3 failed to encode map e166 with expected crc 2023-10-01T10:49:07.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:07 smithi173 bash[58816]: cluster 2023-10-01T10:49:06.597043+0000 osd.7 (osd.7) 116 : cluster 3 failed to encode map e166 with expected crc 2023-10-01T10:49:07.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:07 smithi173 bash[58816]: cluster 2023-10-01T10:49:06.606346+0000 osd.7 (osd.7) 117 : cluster 3 failed to encode map e166 with expected crc 2023-10-01T10:49:07.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:07 smithi173 bash[58816]: cluster 2023-10-01T10:49:06.612264+0000 osd.7 (osd.7) 118 : cluster 3 failed to encode map e166 with expected crc 2023-10-01T10:49:07.885 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:07.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:07.888 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:49:07.989 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:08.019 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:08.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:08.023 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:08.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:07 smithi078 bash[78274]: cluster 2023-10-01T10:49:05.898155+0000 mgr.y (mgr.25072) 944 : cluster 0 pgmap v881: 89 pgs: 89 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2023-10-01T10:49:08.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:07 smithi078 bash[78274]: cluster 2023-10-01T10:49:06.581311+0000 mon.a (mon.0) 1179 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:49:08.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:07 smithi078 bash[78274]: audit 2023-10-01T10:49:06.592159+0000 mon.a (mon.0) 1180 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-01T10:49:08.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:07 smithi078 bash[78274]: audit 2023-10-01T10:49:06.592496+0000 mon.a (mon.0) 1181 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T10:49:08.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:07 smithi078 bash[78274]: cluster 2023-10-01T10:49:06.592675+0000 osd.5 (osd.5) 96 : cluster 3 failed to encode map e166 with expected crc 2023-10-01T10:49:08.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:07 smithi078 bash[78274]: cluster 2023-10-01T10:49:06.592848+0000 mon.a (mon.0) 1182 : cluster 0 osdmap e166: 8 total, 8 up, 8 in 2023-10-01T10:49:08.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:07 smithi078 bash[78274]: cluster 2023-10-01T10:49:06.595076+0000 osd.6 (osd.6) 82 : cluster 3 failed to encode map e166 with expected crc 2023-10-01T10:49:08.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:07 smithi078 bash[78274]: cluster 2023-10-01T10:49:06.596966+0000 osd.6 (osd.6) 83 : cluster 3 failed to encode map e166 with expected crc 2023-10-01T10:49:08.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:07 smithi078 bash[78274]: cluster 2023-10-01T10:49:06.597043+0000 osd.7 (osd.7) 116 : cluster 3 failed to encode map e166 with expected crc 2023-10-01T10:49:08.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:07 smithi078 bash[78274]: cluster 2023-10-01T10:49:06.606346+0000 osd.7 (osd.7) 117 : cluster 3 failed to encode map e166 with expected crc 2023-10-01T10:49:08.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:07 smithi078 bash[78274]: cluster 2023-10-01T10:49:06.612264+0000 osd.7 (osd.7) 118 : cluster 3 failed to encode map e166 with expected crc 2023-10-01T10:49:08.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:07 smithi078 bash[75299]: cluster 2023-10-01T10:49:06.595076+0000 osd.6 (osd.6) 82 : cluster 3 failed to encode map e166 with expected crc 2023-10-01T10:49:08.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:07 smithi078 bash[75299]: cluster 2023-10-01T10:49:06.596966+0000 osd.6 (osd.6) 83 : cluster 3 failed to encode map e166 with expected crc 2023-10-01T10:49:08.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:07 smithi078 bash[75299]: cluster 2023-10-01T10:49:06.597043+0000 osd.7 (osd.7) 116 : cluster 3 failed to encode map e166 with expected crc 2023-10-01T10:49:08.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:07 smithi078 bash[75299]: cluster 2023-10-01T10:49:06.606346+0000 osd.7 (osd.7) 117 : cluster 3 failed to encode map e166 with expected crc 2023-10-01T10:49:08.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:07 smithi078 bash[75299]: cluster 2023-10-01T10:49:06.612264+0000 osd.7 (osd.7) 118 : cluster 3 failed to encode map e166 with expected crc 2023-10-01T10:49:08.123 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:08.147 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:08.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:08.149 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:08.250 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:08.278 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:08.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:08.283 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:49:08.384 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:08.410 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:08.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:08.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:08.517 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:08.551 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:08.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:08.555 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:49:08.656 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:08.683 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:08.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:08.686 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:49:08.786 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:08.818 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:08.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:08.823 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:49:08.923 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:08.947 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:08.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:08.951 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:09.052 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:09.079 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:09.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:09.083 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:09.183 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:09.222 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:09.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:09.229 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:09.330 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:09.378 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-01T10:49:09.392 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:09.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:09.397 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:49:09.498 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:09.524 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:09.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:09.527 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:49:09.608 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:09 smithi078 bash[75299]: cluster 2023-10-01T10:49:07.898628+0000 mgr.y (mgr.25072) 945 : cluster 0 pgmap v883: 89 pgs: 89 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 702 op/s 2023-10-01T10:49:09.608 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:09 smithi078 bash[78274]: cluster 2023-10-01T10:49:07.898628+0000 mgr.y (mgr.25072) 945 : cluster 0 pgmap v883: 89 pgs: 89 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 702 op/s 2023-10-01T10:49:09.628 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:09.657 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:09.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:09.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:49:09.744 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:49:09.759 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:09.768 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:09 smithi173 bash[58816]: cluster 2023-10-01T10:49:07.898628+0000 mgr.y (mgr.25072) 945 : cluster 0 pgmap v883: 89 pgs: 89 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 702 op/s 2023-10-01T10:49:09.800 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:09.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:09.803 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:49:09.904 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:09.937 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:09.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:09.943 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:49:10.043 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:10.076 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:10.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:10.081 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:49:10.182 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:10.208 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:10.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:10.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:10.315 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:10.331 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:49:10 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:49:10] "GET /metrics HTTP/1.1" 200 36318 "" "Prometheus/2.33.4" 2023-10-01T10:49:10.338 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:10.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:10.340 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:10.409 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T10:49:10.441 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:10.480 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:10.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:10.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:49:10.587 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:10.613 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:10.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:10.624 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:49:10.725 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:10.765 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:10.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:10.767 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:49:10.868 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:10.898 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:10.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:10.902 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:11.003 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:11.035 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:11.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:11.041 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:49:11.141 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:11.175 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:11.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:11.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:49:11.279 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:11.316 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:11.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:11.323 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:49:11.359 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T10:49:11.423 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:11.466 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:11.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:11.473 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:49:11.574 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:11.611 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:11.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:11.615 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:11.716 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:11.725 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:11 smithi078 bash[78274]: cluster 2023-10-01T10:49:09.899292+0000 mgr.y (mgr.25072) 946 : cluster 0 pgmap v884: 89 pgs: 89 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 685 op/s 2023-10-01T10:49:11.726 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:11 smithi078 bash[75299]: cluster 2023-10-01T10:49:09.899292+0000 mgr.y (mgr.25072) 946 : cluster 0 pgmap v884: 89 pgs: 89 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 685 op/s 2023-10-01T10:49:11.745 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:11.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:11.748 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:49:11.849 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:11.858 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:11 smithi173 bash[58816]: cluster 2023-10-01T10:49:09.899292+0000 mgr.y (mgr.25072) 946 : cluster 0 pgmap v884: 89 pgs: 89 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 685 op/s 2023-10-01T10:49:11.882 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:11.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:11.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:49:11.987 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:12.013 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:12.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:12.018 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:49:12.118 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:12.149 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:12.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:12.155 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:49:12.256 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:12.286 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:12.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:12.287 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:49:12.388 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:12.412 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:12.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:12.414 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:49:12.515 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:12.555 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:12.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:12.557 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:49:12.658 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:12.685 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:12.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:12.699 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:12.800 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:12.831 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:12.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:12.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:12.935 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:12.963 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:12.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:12.967 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:49:13.068 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:13.090 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:13.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:13.096 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:49:13.196 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:13.243 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:13.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:13.250 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:13.351 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:13.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:13.378 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:13.378 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:49:13.479 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:13.502 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:13.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:13.504 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:49:13.605 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:13.629 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:13.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:13.636 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:49:13.737 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:13.769 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:13.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:13.772 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:13.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:13 smithi173 bash[58816]: cluster 2023-10-01T10:49:11.899908+0000 mgr.y (mgr.25072) 947 : cluster 0 pgmap v885: 89 pgs: 89 active+clean; 2.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 688 op/s 2023-10-01T10:49:13.873 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:13.880 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:13 smithi078 bash[78274]: cluster 2023-10-01T10:49:11.899908+0000 mgr.y (mgr.25072) 947 : cluster 0 pgmap v885: 89 pgs: 89 active+clean; 2.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 688 op/s 2023-10-01T10:49:13.881 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:13 smithi078 bash[75299]: cluster 2023-10-01T10:49:11.899908+0000 mgr.y (mgr.25072) 947 : cluster 0 pgmap v885: 89 pgs: 89 active+clean; 2.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 688 op/s 2023-10-01T10:49:13.909 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:13.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:13.911 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:49:13.965 INFO:teuthology.orchestra.run.smithi078.stdout:ERROR: (22) Invalid argument 2023-10-01T10:49:13.965 INFO:teuthology.orchestra.run.smithi078.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T10:49:14.013 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:14.051 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:14.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:14.055 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:14.156 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:14.190 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:14.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:14.193 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:14.294 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:14.323 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:14.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:14.330 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:14.431 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:14.465 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:14.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:14.472 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:49:14.572 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:14.605 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:14.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:14.612 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:49:14.713 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:14.737 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:14.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:14.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:49:14.841 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:14.871 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:14.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:14.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:49:14.977 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:15.005 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:15.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:15.009 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:49:15.110 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:15.140 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:15.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:15.146 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:49:15.247 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:15.290 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:15.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:15.296 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:15.397 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:15.426 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:15.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:15.430 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:15.470 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-01T10:49:15.530 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:15.573 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:15.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:15.579 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:15.640 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:49:15.640 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:49:15.679 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:15.702 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:15.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:15.706 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:49:15.807 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:15.819 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:15 smithi173 bash[58816]: cluster 2023-10-01T10:49:13.900523+0000 mgr.y (mgr.25072) 948 : cluster 0 pgmap v886: 89 pgs: 89 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2023-10-01T10:49:15.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:15 smithi078 bash[78274]: cluster 2023-10-01T10:49:13.900523+0000 mgr.y (mgr.25072) 948 : cluster 0 pgmap v886: 89 pgs: 89 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2023-10-01T10:49:15.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:15 smithi078 bash[75299]: cluster 2023-10-01T10:49:13.900523+0000 mgr.y (mgr.25072) 948 : cluster 0 pgmap v886: 89 pgs: 89 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2023-10-01T10:49:15.844 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:15.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:15.853 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:49:15.954 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:15.984 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:15.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:15.989 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:49:16.089 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:16.118 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:16.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:16.120 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:49:16.221 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:16.255 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:16.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:16.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:49:16.359 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:16.390 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:16.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:16.395 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:49:16.496 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:16.524 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:16.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:16.528 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:49:16.629 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:16.651 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:16.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:16.656 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:49:16.757 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:16.783 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:16.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:16.787 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:49:16.887 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:16.929 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:16.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:16.935 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:49:16.962 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:49:17.036 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:17.078 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:17.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:17.082 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:49:17.182 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:17.225 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:17.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:17.230 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:17.331 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:17.365 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:17.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:17.368 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:49:17.468 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:17.501 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:17.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:17.504 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:49:17.605 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:17.635 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:17.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:17.643 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:17.744 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:17.780 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:17.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:17.784 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:17.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:17 smithi078 bash[75299]: cluster 2023-10-01T10:49:15.901112+0000 mgr.y (mgr.25072) 949 : cluster 0 pgmap v887: 89 pgs: 89 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 686 op/s 2023-10-01T10:49:17.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:17 smithi078 bash[75299]: audit 2023-10-01T10:49:17.429201+0000 mon.a (mon.0) 1183 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:49:17.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:17 smithi078 bash[78274]: cluster 2023-10-01T10:49:15.901112+0000 mgr.y (mgr.25072) 949 : cluster 0 pgmap v887: 89 pgs: 89 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 686 op/s 2023-10-01T10:49:17.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:17 smithi078 bash[78274]: audit 2023-10-01T10:49:17.429201+0000 mon.a (mon.0) 1183 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:49:17.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:17 smithi173 bash[58816]: cluster 2023-10-01T10:49:15.901112+0000 mgr.y (mgr.25072) 949 : cluster 0 pgmap v887: 89 pgs: 89 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 686 op/s 2023-10-01T10:49:17.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:17 smithi173 bash[58816]: audit 2023-10-01T10:49:17.429201+0000 mon.a (mon.0) 1183 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:49:17.885 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:17.915 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:17.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:17.919 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:18.020 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:18.057 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:18.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:18.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:18.164 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:18.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:18.204 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:18.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:18.304 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:18.338 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:18.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:18.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:18.444 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:18.486 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:18.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:18.491 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:18.592 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:18.640 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:18.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:18.647 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:18.748 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:18.780 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:18.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:18.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:18.884 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:18.906 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:18.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:18.909 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:49:19.010 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:19.036 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:19.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:19.040 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:49:19.141 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:19.171 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:19.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:19.175 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:19.276 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:19.314 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:19.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:19.319 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:19.420 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:19.460 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:19.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:19.463 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:49:19.564 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:19.592 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:19.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:19.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:49:19.694 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:19.703 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:19 smithi173 bash[58816]: cluster 2023-10-01T10:49:17.901697+0000 mgr.y (mgr.25072) 950 : cluster 0 pgmap v888: 89 pgs: 89 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 626 op/s 2023-10-01T10:49:19.728 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:19.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:19.731 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:19.832 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:19.857 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:19.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:19.862 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:19.963 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:20.001 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:20.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:20.007 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:49:20.010 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:19 smithi078 bash[75299]: cluster 2023-10-01T10:49:17.901697+0000 mgr.y (mgr.25072) 950 : cluster 0 pgmap v888: 89 pgs: 89 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 626 op/s 2023-10-01T10:49:20.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:19 smithi078 bash[78274]: cluster 2023-10-01T10:49:17.901697+0000 mgr.y (mgr.25072) 950 : cluster 0 pgmap v888: 89 pgs: 89 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 626 op/s 2023-10-01T10:49:20.107 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:20.136 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:20.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:20.143 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:49:20.243 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:20.266 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:20.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:20.268 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:49:20.369 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:20.403 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:20.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:20.408 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:20.508 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:20.539 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:20.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:20.543 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:20.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:49:20 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:49:20] "GET /metrics HTTP/1.1" 200 36315 "" "Prometheus/2.33.4" 2023-10-01T10:49:20.643 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:20.669 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:20.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:20.675 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:20.775 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:20.798 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:20.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:20.802 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:20.903 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:20.926 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:20.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:20.928 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:49:21.029 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:21.062 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:21.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:21.064 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:49:21.165 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:21.190 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:21.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:21.191 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:49:21.292 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:21.324 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:21.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:21.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:49:21.428 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:21.465 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:21.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:21.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:49:21.570 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:21.635 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:21.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:21.638 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:49:21.739 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:21.769 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:21.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:21.771 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:21.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:21 smithi078 bash[75299]: cluster 2023-10-01T10:49:19.902695+0000 mgr.y (mgr.25072) 951 : cluster 0 pgmap v889: 89 pgs: 89 active+clean; 2.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 818 op/s 2023-10-01T10:49:21.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:21 smithi078 bash[78274]: cluster 2023-10-01T10:49:19.902695+0000 mgr.y (mgr.25072) 951 : cluster 0 pgmap v889: 89 pgs: 89 active+clean; 2.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 818 op/s 2023-10-01T10:49:21.872 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:21.898 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:21.906 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:21 smithi173 bash[58816]: cluster 2023-10-01T10:49:19.902695+0000 mgr.y (mgr.25072) 951 : cluster 0 pgmap v889: 89 pgs: 89 active+clean; 2.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 818 op/s 2023-10-01T10:49:21.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:21.907 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:49:22.007 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:22.041 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:22.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:22.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:49:22.093 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:49:22.146 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:22.179 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:22.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:22.183 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:49:22.284 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:22.309 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:22.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:22.313 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:49:22.414 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:22.439 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:22.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:22.445 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:22.546 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:22.574 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:22.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:22.577 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:49:22.678 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:22.715 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:22.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:22.720 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:49:22.821 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:22.849 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:22.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:22.852 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:49:22.953 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:22.977 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:22.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:22.980 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:49:23.081 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:23.115 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:23.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:23.121 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:23.222 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:23.259 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:23.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:23.262 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:49:23.363 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:23.394 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:23.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:23.401 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:49:23.501 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:23.527 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:23.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:23.530 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:49:23.631 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:23.656 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:23.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:23.660 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:23.760 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:23.786 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:23.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:23.789 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:49:23.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:23 smithi173 bash[58816]: cluster 2023-10-01T10:49:21.903206+0000 mgr.y (mgr.25072) 952 : cluster 0 pgmap v890: 89 pgs: 89 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 641 op/s 2023-10-01T10:49:23.890 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:23.898 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:23 smithi078 bash[75299]: cluster 2023-10-01T10:49:21.903206+0000 mgr.y (mgr.25072) 952 : cluster 0 pgmap v890: 89 pgs: 89 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 641 op/s 2023-10-01T10:49:23.898 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:23 smithi078 bash[78274]: cluster 2023-10-01T10:49:21.903206+0000 mgr.y (mgr.25072) 952 : cluster 0 pgmap v890: 89 pgs: 89 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 641 op/s 2023-10-01T10:49:23.913 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:23.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:23.923 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:49:24.024 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:24.068 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:24.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:24.071 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:49:24.171 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:24.194 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:24.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:24.203 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:49:24.304 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:24.344 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:24.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:24.347 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:24.448 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:24.477 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:24.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:24.481 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:49:24.582 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:24.607 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:24.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:24.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:49:24.712 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:24.751 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:24.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:24.759 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:49:24.859 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:24.897 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:24.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:24.902 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:49:25.003 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:25.032 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:25.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:25.036 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:49:25.136 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:25.173 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:25.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:25.177 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:49:25.278 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:25.315 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:25.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:25.326 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:49:25.340 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T10:49:25.373 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T10:49:25.423 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T10:49:25.427 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:25.455 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:25.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:25.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:25.482 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T10:49:25.527 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T10:49:25.558 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:25.562 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T10:49:25.580 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:25.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:25.583 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:25.614 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T10:49:25.652 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T10:49:25.658 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T10:49:25.658 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T10:49:25.658 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T10:49:25.658 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T10:49:25.658 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T10:49:25.658 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T10:49:25.659 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T10:49:25.659 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T10:49:25.684 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:25.707 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:25.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:25.710 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:49:25.811 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:25.819 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:25 smithi078 bash[78274]: cluster 2023-10-01T10:49:23.903756+0000 mgr.y (mgr.25072) 953 : cluster 0 pgmap v891: 89 pgs: 89 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 735 op/s 2023-10-01T10:49:25.819 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:25 smithi078 bash[78274]: audit 2023-10-01T10:49:25.273572+0000 mon.c (mon.2) 164 : audit 0 from='client.? 172.21.15.78:0/3908341744' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:49:25.819 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:25 smithi078 bash[75299]: cluster 2023-10-01T10:49:23.903756+0000 mgr.y (mgr.25072) 953 : cluster 0 pgmap v891: 89 pgs: 89 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 735 op/s 2023-10-01T10:49:25.820 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:25 smithi078 bash[75299]: audit 2023-10-01T10:49:25.273572+0000 mon.c (mon.2) 164 : audit 0 from='client.? 172.21.15.78:0/3908341744' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:49:25.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:25.844 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:25.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:49:25.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:25 smithi173 bash[58816]: cluster 2023-10-01T10:49:23.903756+0000 mgr.y (mgr.25072) 953 : cluster 0 pgmap v891: 89 pgs: 89 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 735 op/s 2023-10-01T10:49:25.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:25 smithi173 bash[58816]: audit 2023-10-01T10:49:25.273572+0000 mon.c (mon.2) 164 : audit 0 from='client.? 172.21.15.78:0/3908341744' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:49:25.946 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:25.971 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:25.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:25.977 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:26.078 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:26.109 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:26.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:26.115 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:49:26.216 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:26.242 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:26.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:26.247 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:49:26.347 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:26.377 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:26.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:26.380 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:49:26.480 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:26.518 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:26.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:26.522 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:49:26.623 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:26.655 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:26.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:26.658 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:49:26.671 INFO:teuthology.orchestra.run.smithi078.stderr:reweighted osd.3 to 1 (10000) 2023-10-01T10:49:26.672 INFO:teuthology.orchestra.run.smithi078.stderr:nodeep-scrub is unset 2023-10-01T10:49:26.759 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:26.768 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:26 smithi078 bash[78274]: audit 2023-10-01T10:49:25.808660+0000 mon.a (mon.0) 1184 : audit 1 from='client.? 172.21.15.78:0/1107117279' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:49:26.768 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:26 smithi078 bash[78274]: audit 2023-10-01T10:49:26.092794+0000 mon.c (mon.2) 165 : audit 1 from='client.? 172.21.15.78:0/3756891355' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-01T10:49:26.768 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:26 smithi078 bash[78274]: audit 2023-10-01T10:49:26.093689+0000 mon.a (mon.0) 1185 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-01T10:49:26.768 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:26 smithi078 bash[75299]: audit 2023-10-01T10:49:25.808660+0000 mon.a (mon.0) 1184 : audit 1 from='client.? 172.21.15.78:0/1107117279' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:49:26.768 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:26 smithi078 bash[75299]: audit 2023-10-01T10:49:26.092794+0000 mon.c (mon.2) 165 : audit 1 from='client.? 172.21.15.78:0/3756891355' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-01T10:49:26.768 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:26 smithi078 bash[75299]: audit 2023-10-01T10:49:26.093689+0000 mon.a (mon.0) 1185 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-01T10:49:26.790 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:26.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:26.793 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:49:26.893 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:26.902 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:26 smithi173 bash[58816]: audit 2023-10-01T10:49:25.808660+0000 mon.a (mon.0) 1184 : audit 1 from='client.? 172.21.15.78:0/1107117279' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:49:26.902 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:26 smithi173 bash[58816]: audit 2023-10-01T10:49:26.092794+0000 mon.c (mon.2) 165 : audit 1 from='client.? 172.21.15.78:0/3756891355' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-01T10:49:26.902 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:26 smithi173 bash[58816]: audit 2023-10-01T10:49:26.093689+0000 mon.a (mon.0) 1185 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-01T10:49:26.940 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:26.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:26.943 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:27.044 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:27.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:27.078 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:27.078 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:49:27.179 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:27.207 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:27.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:27.209 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:49:27.310 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:27.335 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:27.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:27.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:49:27.440 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:27.463 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:27.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:27.465 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:49:27.565 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:27.600 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:27.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:27.610 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:27.711 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:27.741 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:27.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:27.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:49:27.843 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:27.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:27 smithi173 bash[58816]: cluster 2023-10-01T10:49:25.904292+0000 mgr.y (mgr.25072) 954 : cluster 0 pgmap v892: 89 pgs: 89 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 678 op/s 2023-10-01T10:49:27.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:27 smithi173 bash[58816]: cluster 2023-10-01T10:49:26.645822+0000 mon.a (mon.0) 1186 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T10:49:27.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:27 smithi173 bash[58816]: cluster 2023-10-01T10:49:26.645855+0000 mon.a (mon.0) 1187 : cluster 1 Cluster is now healthy 2023-10-01T10:49:27.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:27 smithi173 bash[58816]: cluster 2023-10-01T10:49:26.655642+0000 osd.7 (osd.7) 119 : cluster 3 failed to encode map e167 with expected crc 2023-10-01T10:49:27.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:27 smithi173 bash[58816]: audit 2023-10-01T10:49:26.662827+0000 mon.a (mon.0) 1188 : audit 1 from='client.? 172.21.15.78:0/1107117279' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T10:49:27.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:27 smithi173 bash[58816]: audit 2023-10-01T10:49:26.662973+0000 mon.a (mon.0) 1189 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-01T10:49:27.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:27 smithi173 bash[58816]: cluster 2023-10-01T10:49:26.663393+0000 mon.a (mon.0) 1190 : cluster 0 osdmap e167: 8 total, 8 up, 8 in 2023-10-01T10:49:27.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:27 smithi173 bash[58816]: cluster 2023-10-01T10:49:26.666082+0000 osd.6 (osd.6) 84 : cluster 3 failed to encode map e167 with expected crc 2023-10-01T10:49:27.871 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:27.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:27.876 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:49:27.977 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:27.985 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:27 smithi078 bash[75299]: cluster 2023-10-01T10:49:25.904292+0000 mgr.y (mgr.25072) 954 : cluster 0 pgmap v892: 89 pgs: 89 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 678 op/s 2023-10-01T10:49:27.985 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:27 smithi078 bash[75299]: cluster 2023-10-01T10:49:26.645822+0000 mon.a (mon.0) 1186 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T10:49:27.986 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:27 smithi078 bash[75299]: cluster 2023-10-01T10:49:26.645855+0000 mon.a (mon.0) 1187 : cluster 1 Cluster is now healthy 2023-10-01T10:49:27.986 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:27 smithi078 bash[75299]: cluster 2023-10-01T10:49:26.655642+0000 osd.7 (osd.7) 119 : cluster 3 failed to encode map e167 with expected crc 2023-10-01T10:49:27.986 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:27 smithi078 bash[75299]: audit 2023-10-01T10:49:26.662827+0000 mon.a (mon.0) 1188 : audit 1 from='client.? 172.21.15.78:0/1107117279' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T10:49:27.986 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:27 smithi078 bash[75299]: audit 2023-10-01T10:49:26.662973+0000 mon.a (mon.0) 1189 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-01T10:49:27.986 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:27 smithi078 bash[75299]: cluster 2023-10-01T10:49:26.663393+0000 mon.a (mon.0) 1190 : cluster 0 osdmap e167: 8 total, 8 up, 8 in 2023-10-01T10:49:27.986 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:27 smithi078 bash[75299]: cluster 2023-10-01T10:49:26.666082+0000 osd.6 (osd.6) 84 : cluster 3 failed to encode map e167 with expected crc 2023-10-01T10:49:27.987 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:27 smithi078 bash[78274]: cluster 2023-10-01T10:49:25.904292+0000 mgr.y (mgr.25072) 954 : cluster 0 pgmap v892: 89 pgs: 89 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 678 op/s 2023-10-01T10:49:27.987 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:27 smithi078 bash[78274]: cluster 2023-10-01T10:49:26.645822+0000 mon.a (mon.0) 1186 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T10:49:27.987 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:27 smithi078 bash[78274]: cluster 2023-10-01T10:49:26.645855+0000 mon.a (mon.0) 1187 : cluster 1 Cluster is now healthy 2023-10-01T10:49:27.987 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:27 smithi078 bash[78274]: cluster 2023-10-01T10:49:26.655642+0000 osd.7 (osd.7) 119 : cluster 3 failed to encode map e167 with expected crc 2023-10-01T10:49:27.987 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:27 smithi078 bash[78274]: audit 2023-10-01T10:49:26.662827+0000 mon.a (mon.0) 1188 : audit 1 from='client.? 172.21.15.78:0/1107117279' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T10:49:27.988 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:27 smithi078 bash[78274]: audit 2023-10-01T10:49:26.662973+0000 mon.a (mon.0) 1189 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-01T10:49:27.988 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:27 smithi078 bash[78274]: cluster 2023-10-01T10:49:26.663393+0000 mon.a (mon.0) 1190 : cluster 0 osdmap e167: 8 total, 8 up, 8 in 2023-10-01T10:49:27.988 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:27 smithi078 bash[78274]: cluster 2023-10-01T10:49:26.666082+0000 osd.6 (osd.6) 84 : cluster 3 failed to encode map e167 with expected crc 2023-10-01T10:49:28.008 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:28.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:28.013 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:49:28.113 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:28.145 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:28.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:28.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:49:28.248 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:28.280 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:28.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:28.284 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:28.385 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:28.410 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:28.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:28.414 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:49:28.514 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:28.544 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:28.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:28.551 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:49:28.652 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:28.684 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:28.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:28.688 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:49:28.788 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:28.818 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:28.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:28.821 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:49:28.921 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:28.949 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:28.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:28.951 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:49:29.052 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:29.088 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:29.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:29.095 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:49:29.196 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:29.215 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:29.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:29.220 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:49:29.321 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:29.370 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:29.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:29.374 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:49:29.475 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:29.512 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:29.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:29.516 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:29.617 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:29.648 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:29.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:29.651 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:49:29.751 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:29.780 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:29.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:29.784 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:49:29.885 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:29.911 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:29.917 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:29 smithi173 bash[58816]: cluster 2023-10-01T10:49:27.904863+0000 mgr.y (mgr.25072) 955 : cluster 0 pgmap v894: 89 pgs: 89 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 703 op/s 2023-10-01T10:49:29.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:29.918 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:49:30.018 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:30.058 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:30.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:30.069 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:49:30.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:29 smithi078 bash[75299]: cluster 2023-10-01T10:49:27.904863+0000 mgr.y (mgr.25072) 955 : cluster 0 pgmap v894: 89 pgs: 89 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 703 op/s 2023-10-01T10:49:30.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:29 smithi078 bash[78274]: cluster 2023-10-01T10:49:27.904863+0000 mgr.y (mgr.25072) 955 : cluster 0 pgmap v894: 89 pgs: 89 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 703 op/s 2023-10-01T10:49:30.170 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:30.214 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:30.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:30.219 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:30.319 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:30.389 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:30.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:30.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:49:30.496 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:30.546 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:30.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:30.550 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:30.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:49:30 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:49:30] "GET /metrics HTTP/1.1" 200 36315 "" "Prometheus/2.33.4" 2023-10-01T10:49:30.651 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:30.691 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:30.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:30.693 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:49:30.794 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:30.835 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:30.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:30.838 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:49:30.939 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:30.978 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:30.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:30.981 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:49:30.997 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-01T10:49:31.082 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:31.112 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:31.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:31.119 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:49:31.220 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:31.256 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:31.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:31.261 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:49:31.362 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:31.392 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:31.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:31.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:49:31.500 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:31.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:31.528 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:31.529 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:49:31.629 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:31.653 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:31.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:31.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:49:31.658 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:31 smithi173 bash[58816]: cluster 2023-10-01T10:49:29.905539+0000 mgr.y (mgr.25072) 956 : cluster 0 pgmap v895: 89 pgs: 89 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2023-10-01T10:49:31.756 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:31.780 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:31.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:31.784 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:49:31.826 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T10:49:31.885 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:31.902 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:31 smithi078 bash[75299]: cluster 2023-10-01T10:49:29.905539+0000 mgr.y (mgr.25072) 956 : cluster 0 pgmap v895: 89 pgs: 89 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2023-10-01T10:49:31.903 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:31 smithi078 bash[78274]: cluster 2023-10-01T10:49:29.905539+0000 mgr.y (mgr.25072) 956 : cluster 0 pgmap v895: 89 pgs: 89 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2023-10-01T10:49:31.927 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:31.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:31.931 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:49:32.031 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:32.077 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:32.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:32.082 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:49:32.157 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2023-10-01T10:49:32.157 INFO:teuthology.orchestra.run.smithi078.stdout: "ops": [], 2023-10-01T10:49:32.158 INFO:teuthology.orchestra.run.smithi078.stdout: "complaint_time": 30, 2023-10-01T10:49:32.158 INFO:teuthology.orchestra.run.smithi078.stdout: "num_blocked_ops": 0 2023-10-01T10:49:32.158 INFO:teuthology.orchestra.run.smithi078.stdout:} 2023-10-01T10:49:32.183 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:32.225 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:32.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:32.227 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:49:32.328 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:32.363 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:32.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:32.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:49:32.468 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:32.495 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:32.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:32.500 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:49:32.600 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:32.630 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:32.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:32.635 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:32.735 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:32.764 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:32.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:32.767 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:49:32.799 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T10:49:32.867 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:32.897 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:32.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:32.900 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:33.000 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:33.032 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:33.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:33.034 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:33.135 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:33.166 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:33.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:33.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:49:33.272 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:33.325 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:33.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:33.326 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:49:33.427 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:33.434 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:33 smithi078 bash[78274]: cluster 2023-10-01T10:49:31.909546+0000 mgr.y (mgr.25072) 957 : cluster 0 pgmap v896: 89 pgs: 89 active+clean; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2023-10-01T10:49:33.434 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:33 smithi078 bash[78274]: audit 2023-10-01T10:49:32.439177+0000 mon.a (mon.0) 1191 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:49:33.435 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:33 smithi078 bash[78274]: audit 2023-10-01T10:49:32.440282+0000 mon.a (mon.0) 1192 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:49:33.456 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:33.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:33.461 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:49:33.512 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-01T10:49:33.561 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:33.597 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:33.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:33.601 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:33.702 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:33.710 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:33 smithi173 bash[58816]: cluster 2023-10-01T10:49:31.909546+0000 mgr.y (mgr.25072) 957 : cluster 0 pgmap v896: 89 pgs: 89 active+clean; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2023-10-01T10:49:33.710 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:33 smithi173 bash[58816]: audit 2023-10-01T10:49:32.439177+0000 mon.a (mon.0) 1191 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:49:33.711 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:33 smithi173 bash[58816]: audit 2023-10-01T10:49:32.440282+0000 mon.a (mon.0) 1192 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:49:33.727 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:33.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:33.730 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:49:33.831 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:33.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:33 smithi078 bash[75299]: cluster 2023-10-01T10:49:31.909546+0000 mgr.y (mgr.25072) 957 : cluster 0 pgmap v896: 89 pgs: 89 active+clean; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2023-10-01T10:49:33.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:33 smithi078 bash[75299]: audit 2023-10-01T10:49:32.439177+0000 mon.a (mon.0) 1191 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:49:33.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:33 smithi078 bash[75299]: audit 2023-10-01T10:49:32.440282+0000 mon.a (mon.0) 1192 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:49:33.855 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:33.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:33.862 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:33.962 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:33.998 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:34.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:34.000 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:49:34.101 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:34.136 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:34.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:34.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:49:34.243 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:34.274 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:34.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:34.279 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:49:34.380 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:34.408 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:34.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:34.415 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:49:34.515 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:34.546 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:34.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:34.550 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:34.650 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:34.677 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:34.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:34.678 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:34.779 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:34.807 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:34.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:34.809 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:34.910 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:34.942 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:34.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:34.945 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:35.046 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:35.079 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:35.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:35.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:49:35.184 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:35.217 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:35.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:35.220 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:49:35.321 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:35.347 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:35.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:35.352 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:49:35.453 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:35.490 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:35.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:35.494 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:49:35.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:35 smithi078 bash[75299]: cluster 2023-10-01T10:49:33.910162+0000 mgr.y (mgr.25072) 958 : cluster 0 pgmap v897: 89 pgs: 89 active+clean; 1.9 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 654 op/s 2023-10-01T10:49:35.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:35 smithi078 bash[78274]: cluster 2023-10-01T10:49:33.910162+0000 mgr.y (mgr.25072) 958 : cluster 0 pgmap v897: 89 pgs: 89 active+clean; 1.9 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 654 op/s 2023-10-01T10:49:35.594 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:35.633 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:35.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:35.638 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:49:35.739 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:35.769 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:35.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:35.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:49:35.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:35 smithi173 bash[58816]: cluster 2023-10-01T10:49:33.910162+0000 mgr.y (mgr.25072) 958 : cluster 0 pgmap v897: 89 pgs: 89 active+clean; 1.9 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 654 op/s 2023-10-01T10:49:35.873 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:35.899 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:35.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:35.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:36.006 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:36.038 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:36.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:36.041 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:49:36.142 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:36.181 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:36.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:36.188 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:36.288 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:36.321 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:36.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:36.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:49:36.425 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:36.453 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:36.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:36.459 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:36.560 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:36.592 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:36.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:36.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:49:36.696 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:36.721 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:36.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:36.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:49:36.827 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:36.855 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:36.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:36.857 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:49:36.958 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:36.985 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:49:36.985 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:49:36.995 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:36.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:36.998 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:49:37.099 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:37.128 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:37.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:37.130 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:37.231 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:37.262 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:37.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:37.268 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:37.369 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:37.399 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:37.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:37.403 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:49:37.504 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:37.525 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:37.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:37.530 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:49:37.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:37 smithi173 bash[58816]: cluster 2023-10-01T10:49:35.913664+0000 mgr.y (mgr.25072) 959 : cluster 0 pgmap v898: 89 pgs: 89 active+clean; 1.9 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2023-10-01T10:49:37.631 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:37.640 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:37 smithi078 bash[75299]: cluster 2023-10-01T10:49:35.913664+0000 mgr.y (mgr.25072) 959 : cluster 0 pgmap v898: 89 pgs: 89 active+clean; 1.9 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2023-10-01T10:49:37.640 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:37 smithi078 bash[78274]: cluster 2023-10-01T10:49:35.913664+0000 mgr.y (mgr.25072) 959 : cluster 0 pgmap v898: 89 pgs: 89 active+clean; 1.9 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2023-10-01T10:49:37.656 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:37.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:37.668 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:49:37.769 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:37.797 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:37.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:37.800 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:49:37.900 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:37.933 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:37.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:37.939 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:49:38.003 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:49:38.004 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:49:38.040 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:38.098 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:38.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:38.102 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:38.205 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:38.235 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:38.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:38.244 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:49:38.344 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:38.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:38.375 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:38.375 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:49:38.476 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:38.509 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:38.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:38.516 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:49:38.617 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:38.644 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:38.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:38.649 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:49:38.749 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:38.785 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:38.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:38.791 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:49:38.892 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:38.913 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:38.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:38.915 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:49:39.016 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:39.046 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:39.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:39.052 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:49:39.153 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:39.185 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:39.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:39.190 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:39.291 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:39.318 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:39.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:39.321 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:49:39.422 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:39.447 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:39.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:39.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:49:39.460 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:39 smithi173 bash[58816]: cluster 2023-10-01T10:49:37.914207+0000 mgr.y (mgr.25072) 960 : cluster 0 pgmap v899: 89 pgs: 89 active+clean; 1.8 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 624 op/s 2023-10-01T10:49:39.555 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:39.582 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:39.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:39.584 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:49:39.685 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:39.720 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:39.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:39.723 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:39.824 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:39.851 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:39.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:39.855 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:49:39.956 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:39.964 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:39 smithi078 bash[78274]: cluster 2023-10-01T10:49:37.914207+0000 mgr.y (mgr.25072) 960 : cluster 0 pgmap v899: 89 pgs: 89 active+clean; 1.8 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 624 op/s 2023-10-01T10:49:39.964 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:39 smithi078 bash[75299]: cluster 2023-10-01T10:49:37.914207+0000 mgr.y (mgr.25072) 960 : cluster 0 pgmap v899: 89 pgs: 89 active+clean; 1.8 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 624 op/s 2023-10-01T10:49:39.978 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:39.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:39.984 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:49:40.084 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:40.115 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:40.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:40.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:40.221 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:40.259 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:40.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:40.260 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:40.269 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:49:40 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:49:40] "GET /metrics HTTP/1.1" 200 36315 "" "Prometheus/2.33.4" 2023-10-01T10:49:40.361 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:40.393 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:40.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:40.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:40.495 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:40.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:40.533 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:40.533 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:49:40.634 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:40.670 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:40.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:40.677 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:49:40.778 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:40.805 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:40.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:40.807 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:40.908 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:40.954 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:40.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:40.959 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:41.060 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:41.098 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:41.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:41.101 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:49:41.202 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:41.240 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:41.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:41.243 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:41.344 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:41.381 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:41.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:41.384 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:41.471 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:41 smithi173 bash[58816]: cluster 2023-10-01T10:49:39.914756+0000 mgr.y (mgr.25072) 961 : cluster 0 pgmap v900: 89 pgs: 89 active+clean; 1.6 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 830 op/s 2023-10-01T10:49:41.484 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:41.519 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:41.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:41.524 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:49:41.615 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:41 smithi078 bash[75299]: cluster 2023-10-01T10:49:39.914756+0000 mgr.y (mgr.25072) 961 : cluster 0 pgmap v900: 89 pgs: 89 active+clean; 1.6 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 830 op/s 2023-10-01T10:49:41.615 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:41 smithi078 bash[78274]: cluster 2023-10-01T10:49:39.914756+0000 mgr.y (mgr.25072) 961 : cluster 0 pgmap v900: 89 pgs: 89 active+clean; 1.6 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 830 op/s 2023-10-01T10:49:41.625 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:41.653 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:41.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:41.657 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:49:41.758 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:41.796 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:41.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:41.800 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:49:41.901 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:41.927 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:41.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:41.933 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:49:42.033 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:42.079 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:42.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:42.087 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:42.188 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:42.235 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:42.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:42.240 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:42.341 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:42.369 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:42.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:42.371 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:49:42.472 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:42.498 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:42.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:42.501 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:49:42.601 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:42.626 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:42.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:42.630 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:49:42.731 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:42.764 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:42.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:42.769 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:42.870 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:42.941 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:42.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:42.944 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:43.044 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:43.071 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:43.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:43.074 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:49:43.174 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:43.213 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:43.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:43.220 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:43.321 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:43.366 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:43.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:43.369 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:49:43.470 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:43.499 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:43.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:43.505 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:49:43.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:43 smithi078 bash[75299]: cluster 2023-10-01T10:49:41.915905+0000 mgr.y (mgr.25072) 962 : cluster 0 pgmap v901: 89 pgs: 89 active+clean; 1.6 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 652 op/s 2023-10-01T10:49:43.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:43 smithi078 bash[78274]: cluster 2023-10-01T10:49:41.915905+0000 mgr.y (mgr.25072) 962 : cluster 0 pgmap v901: 89 pgs: 89 active+clean; 1.6 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 652 op/s 2023-10-01T10:49:43.606 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:43.642 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:43.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:43.647 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:43.747 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:43.771 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:43.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:43.776 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:49:43.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:43 smithi173 bash[58816]: cluster 2023-10-01T10:49:41.915905+0000 mgr.y (mgr.25072) 962 : cluster 0 pgmap v901: 89 pgs: 89 active+clean; 1.6 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 652 op/s 2023-10-01T10:49:43.877 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:43.902 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:43.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:43.907 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:49:44.008 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:44.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:44.036 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:44.036 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:49:44.137 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:44.173 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:44.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:44.177 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:49:44.277 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:44.312 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:44.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:44.314 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:49:44.415 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:44.449 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:44.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:44.455 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:44.556 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:44.587 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:44.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:44.589 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:44.689 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:44.715 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:44.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:44.719 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:44.819 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:44.857 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:44.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:44.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:44.964 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:44.991 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:44.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:44.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:49:45.094 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:45.122 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:45.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:45.126 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:49:45.226 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:45.256 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:45.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:45.257 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:49:45.358 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:45.384 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:45.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:45.388 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:49:45.488 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:45.521 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:45.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:45.528 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:49:45.620 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:45 smithi078 bash[75299]: cluster 2023-10-01T10:49:43.916455+0000 mgr.y (mgr.25072) 963 : cluster 0 pgmap v902: 89 pgs: 89 active+clean; 1.5 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 698 op/s 2023-10-01T10:49:45.621 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:45 smithi078 bash[78274]: cluster 2023-10-01T10:49:43.916455+0000 mgr.y (mgr.25072) 963 : cluster 0 pgmap v902: 89 pgs: 89 active+clean; 1.5 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 698 op/s 2023-10-01T10:49:45.629 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:45.666 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:45.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:45.671 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:49:45.772 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:45.808 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:45.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:45.811 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:49:45.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:45 smithi173 bash[58816]: cluster 2023-10-01T10:49:43.916455+0000 mgr.y (mgr.25072) 963 : cluster 0 pgmap v902: 89 pgs: 89 active+clean; 1.5 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 698 op/s 2023-10-01T10:49:45.912 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:45.944 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:45.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:45.949 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:49:46.050 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:46.082 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:46.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:46.087 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:49:46.188 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:46.213 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:46.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:46.218 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:46.319 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:46.347 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:46.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:46.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:49:46.451 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:46.484 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:46.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:46.487 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:49:46.588 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:46.622 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:46.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:46.627 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:46.727 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:46.750 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:46.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:46.756 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:49:46.856 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:46.879 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:46.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:46.883 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:49:46.984 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:47.014 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:47.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:47.017 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:47.118 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:47.150 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:47.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:47.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:47.256 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:47.288 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:47.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:47.290 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:47.391 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:47.440 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:47.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:47.445 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:49:47.546 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:47.575 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:47.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:47.578 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:49:47.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:47 smithi173 bash[58816]: cluster 2023-10-01T10:49:45.917031+0000 mgr.y (mgr.25072) 964 : cluster 0 pgmap v903: 89 pgs: 89 active+clean; 1.4 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 719 op/s 2023-10-01T10:49:47.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:47 smithi173 bash[58816]: audit 2023-10-01T10:49:47.430165+0000 mon.a (mon.0) 1193 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:49:47.678 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:47.716 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:47.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:47.722 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:47.823 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:47.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:47 smithi078 bash[78274]: cluster 2023-10-01T10:49:45.917031+0000 mgr.y (mgr.25072) 964 : cluster 0 pgmap v903: 89 pgs: 89 active+clean; 1.4 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 719 op/s 2023-10-01T10:49:47.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:47 smithi078 bash[78274]: audit 2023-10-01T10:49:47.430165+0000 mon.a (mon.0) 1193 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:49:47.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:47 smithi078 bash[75299]: cluster 2023-10-01T10:49:45.917031+0000 mgr.y (mgr.25072) 964 : cluster 0 pgmap v903: 89 pgs: 89 active+clean; 1.4 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 719 op/s 2023-10-01T10:49:47.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:47 smithi078 bash[75299]: audit 2023-10-01T10:49:47.430165+0000 mon.a (mon.0) 1193 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:49:47.845 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:47.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:47.847 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:49:47.948 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:47.976 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:47.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:47.980 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:49:48.081 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:48.112 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:48.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:48.115 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:49:48.216 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:48.255 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:48.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:48.263 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:48.363 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:48.389 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:48.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:48.395 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:48.496 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:48.520 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:48.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:48.522 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:49:48.623 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:48.655 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:48.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:48.659 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:49:48.760 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:48.787 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:48.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:48.789 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:48.890 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:48.936 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:48.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:48.938 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:49.039 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:49.071 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:49.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:49.075 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:49.175 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:49.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:49.207 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:49.208 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:49.308 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:49.346 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:49.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:49.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:49:49.455 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:49.480 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:49.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:49.485 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:49:49.489 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:49 smithi078 bash[75299]: cluster 2023-10-01T10:49:47.917599+0000 mgr.y (mgr.25072) 965 : cluster 0 pgmap v904: 89 pgs: 89 active+clean; 1.4 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 649 op/s 2023-10-01T10:49:49.489 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:49 smithi078 bash[78274]: cluster 2023-10-01T10:49:47.917599+0000 mgr.y (mgr.25072) 965 : cluster 0 pgmap v904: 89 pgs: 89 active+clean; 1.4 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 649 op/s 2023-10-01T10:49:49.586 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:49.611 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:49.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:49.613 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:49:49.714 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:49.740 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:49.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:49.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:49:49.842 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:49.849 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:49 smithi173 bash[58816]: cluster 2023-10-01T10:49:47.917599+0000 mgr.y (mgr.25072) 965 : cluster 0 pgmap v904: 89 pgs: 89 active+clean; 1.4 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 649 op/s 2023-10-01T10:49:49.879 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:49.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:49.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:49:49.983 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:50.026 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:50.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:50.030 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:49:50.082 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T10:49:50.115 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T10:49:50.131 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:50.158 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T10:49:50.161 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:50.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:50.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:49:50.212 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T10:49:50.255 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T10:49:50.267 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:50.269 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:49:50 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:49:50] "GET /metrics HTTP/1.1" 200 36311 "" "Prometheus/2.33.4" 2023-10-01T10:49:50.297 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T10:49:50.298 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:50.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:50.300 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:50.357 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T10:49:50.401 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:50.408 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T10:49:50.417 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T10:49:50.417 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T10:49:50.417 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T10:49:50.417 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T10:49:50.417 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T10:49:50.417 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T10:49:50.417 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T10:49:50.418 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T10:49:50.430 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:50.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:50.435 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:49:50.527 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:50 smithi078 bash[75299]: audit 2023-10-01T10:49:50.046339+0000 mon.c (mon.2) 166 : audit 0 from='client.? 172.21.15.78:0/3512637224' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:49:50.528 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:50 smithi078 bash[75299]: audit 2023-10-01T10:49:50.287613+0000 mon.a (mon.0) 1194 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:49:50.528 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:50 smithi078 bash[78274]: audit 2023-10-01T10:49:50.046339+0000 mon.c (mon.2) 166 : audit 0 from='client.? 172.21.15.78:0/3512637224' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:49:50.528 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:50 smithi078 bash[78274]: audit 2023-10-01T10:49:50.287613+0000 mon.a (mon.0) 1194 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:49:50.535 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:50.598 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:50.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:50.603 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:49:50.690 INFO:teuthology.orchestra.run.smithi078.stdout:ERROR: (22) Invalid argument 2023-10-01T10:49:50.690 INFO:teuthology.orchestra.run.smithi078.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T10:49:50.703 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:50.712 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:50 smithi173 bash[58816]: audit 2023-10-01T10:49:50.046339+0000 mon.c (mon.2) 166 : audit 0 from='client.? 172.21.15.78:0/3512637224' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:49:50.713 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:50 smithi173 bash[58816]: audit 2023-10-01T10:49:50.287613+0000 mon.a (mon.0) 1194 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:49:50.734 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:50.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:50.736 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:49:50.837 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:50.861 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:50.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:50.863 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:49:50.964 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:50.986 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:50.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:50.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:49:51.089 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:51.119 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:51.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:51.122 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:51.223 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:51.247 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:51.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:51.249 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:49:51.350 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:51.375 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:51.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:51.381 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:49:51.482 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:51.513 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:51.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:51.519 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:51.529 INFO:teuthology.orchestra.run.smithi078.stderr:reweighted osd.4 to 1 (10000) 2023-10-01T10:49:51.534 INFO:teuthology.orchestra.run.smithi078.stderr:noscrub is set 2023-10-01T10:49:51.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:51 smithi173 bash[58816]: cluster 2023-10-01T10:49:49.918231+0000 mgr.y (mgr.25072) 966 : cluster 0 pgmap v905: 89 pgs: 89 active+clean; 1.2 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 831 op/s 2023-10-01T10:49:51.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:51 smithi173 bash[58816]: audit 2023-10-01T10:49:50.790258+0000 mon.c (mon.2) 167 : audit 1 from='client.? 172.21.15.78:0/1970400243' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-01T10:49:51.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:51 smithi173 bash[58816]: audit 2023-10-01T10:49:50.790907+0000 mon.a (mon.0) 1195 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-01T10:49:51.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:51 smithi173 bash[58816]: audit 2023-10-01T10:49:51.091671+0000 mon.c (mon.2) 168 : audit 1 from='client.? 172.21.15.78:0/1439276838' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:49:51.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:51 smithi173 bash[58816]: audit 2023-10-01T10:49:51.092050+0000 mon.a (mon.0) 1196 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:49:51.620 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:51.652 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:51.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:51.655 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:51.756 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:51.791 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:51.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:51.794 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:49:51.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:51 smithi078 bash[78274]: cluster 2023-10-01T10:49:49.918231+0000 mgr.y (mgr.25072) 966 : cluster 0 pgmap v905: 89 pgs: 89 active+clean; 1.2 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 831 op/s 2023-10-01T10:49:51.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:51 smithi078 bash[78274]: audit 2023-10-01T10:49:50.790258+0000 mon.c (mon.2) 167 : audit 1 from='client.? 172.21.15.78:0/1970400243' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-01T10:49:51.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:51 smithi078 bash[78274]: audit 2023-10-01T10:49:50.790907+0000 mon.a (mon.0) 1195 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-01T10:49:51.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:51 smithi078 bash[78274]: audit 2023-10-01T10:49:51.091671+0000 mon.c (mon.2) 168 : audit 1 from='client.? 172.21.15.78:0/1439276838' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:49:51.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:51 smithi078 bash[78274]: audit 2023-10-01T10:49:51.092050+0000 mon.a (mon.0) 1196 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:49:51.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:51 smithi078 bash[75299]: cluster 2023-10-01T10:49:49.918231+0000 mgr.y (mgr.25072) 966 : cluster 0 pgmap v905: 89 pgs: 89 active+clean; 1.2 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 831 op/s 2023-10-01T10:49:51.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:51 smithi078 bash[75299]: audit 2023-10-01T10:49:50.790258+0000 mon.c (mon.2) 167 : audit 1 from='client.? 172.21.15.78:0/1970400243' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-01T10:49:51.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:51 smithi078 bash[75299]: audit 2023-10-01T10:49:50.790907+0000 mon.a (mon.0) 1195 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-01T10:49:51.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:51 smithi078 bash[75299]: audit 2023-10-01T10:49:51.091671+0000 mon.c (mon.2) 168 : audit 1 from='client.? 172.21.15.78:0/1439276838' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:49:51.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:51 smithi078 bash[75299]: audit 2023-10-01T10:49:51.092050+0000 mon.a (mon.0) 1196 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:49:51.895 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:51.921 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:51.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:51.926 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:49:52.027 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:52.051 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:52.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:52.055 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:49:52.156 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:52.175 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:52.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:52.178 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:52.278 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:52.309 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:52.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:52.317 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:52.417 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:52.445 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:52.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:52.448 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:49:52.504 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:52 smithi173 bash[58816]: cluster 2023-10-01T10:49:51.499445+0000 mon.a (mon.0) 1197 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:49:52.505 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:52 smithi173 bash[58816]: cluster 2023-10-01T10:49:51.517511+0000 osd.5 (osd.5) 97 : cluster 3 failed to encode map e168 with expected crc 2023-10-01T10:49:52.505 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:52 smithi173 bash[58816]: cluster 2023-10-01T10:49:51.520965+0000 osd.7 (osd.7) 120 : cluster 3 failed to encode map e168 with expected crc 2023-10-01T10:49:52.505 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:52 smithi173 bash[58816]: audit 2023-10-01T10:49:51.521891+0000 mon.a (mon.0) 1198 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-01T10:49:52.505 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:52 smithi173 bash[58816]: audit 2023-10-01T10:49:51.521996+0000 mon.a (mon.0) 1199 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T10:49:52.505 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:52 smithi173 bash[58816]: cluster 2023-10-01T10:49:51.522951+0000 mon.a (mon.0) 1200 : cluster 0 osdmap e168: 8 total, 8 up, 8 in 2023-10-01T10:49:52.505 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:52 smithi173 bash[58816]: cluster 2023-10-01T10:49:51.526948+0000 osd.6 (osd.6) 85 : cluster 3 failed to encode map e168 with expected crc 2023-10-01T10:49:52.549 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:52.584 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:52.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:52.587 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:49:52.688 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:52.725 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:52.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:52.729 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:49:52.830 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:52.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:52 smithi078 bash[75299]: cluster 2023-10-01T10:49:51.499445+0000 mon.a (mon.0) 1197 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:49:52.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:52 smithi078 bash[75299]: cluster 2023-10-01T10:49:51.517511+0000 osd.5 (osd.5) 97 : cluster 3 failed to encode map e168 with expected crc 2023-10-01T10:49:52.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:52 smithi078 bash[75299]: cluster 2023-10-01T10:49:51.520965+0000 osd.7 (osd.7) 120 : cluster 3 failed to encode map e168 with expected crc 2023-10-01T10:49:52.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:52 smithi078 bash[75299]: audit 2023-10-01T10:49:51.521891+0000 mon.a (mon.0) 1198 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-01T10:49:52.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:52 smithi078 bash[75299]: audit 2023-10-01T10:49:51.521996+0000 mon.a (mon.0) 1199 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T10:49:52.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:52 smithi078 bash[75299]: cluster 2023-10-01T10:49:51.522951+0000 mon.a (mon.0) 1200 : cluster 0 osdmap e168: 8 total, 8 up, 8 in 2023-10-01T10:49:52.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:52 smithi078 bash[75299]: cluster 2023-10-01T10:49:51.526948+0000 osd.6 (osd.6) 85 : cluster 3 failed to encode map e168 with expected crc 2023-10-01T10:49:52.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:52 smithi078 bash[78274]: cluster 2023-10-01T10:49:51.499445+0000 mon.a (mon.0) 1197 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:49:52.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:52 smithi078 bash[78274]: cluster 2023-10-01T10:49:51.517511+0000 osd.5 (osd.5) 97 : cluster 3 failed to encode map e168 with expected crc 2023-10-01T10:49:52.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:52 smithi078 bash[78274]: cluster 2023-10-01T10:49:51.520965+0000 osd.7 (osd.7) 120 : cluster 3 failed to encode map e168 with expected crc 2023-10-01T10:49:52.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:52 smithi078 bash[78274]: audit 2023-10-01T10:49:51.521891+0000 mon.a (mon.0) 1198 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-01T10:49:52.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:52 smithi078 bash[78274]: audit 2023-10-01T10:49:51.521996+0000 mon.a (mon.0) 1199 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T10:49:52.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:52 smithi078 bash[78274]: cluster 2023-10-01T10:49:51.522951+0000 mon.a (mon.0) 1200 : cluster 0 osdmap e168: 8 total, 8 up, 8 in 2023-10-01T10:49:52.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:52 smithi078 bash[78274]: cluster 2023-10-01T10:49:51.526948+0000 osd.6 (osd.6) 85 : cluster 3 failed to encode map e168 with expected crc 2023-10-01T10:49:52.858 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:52.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:52.860 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:52.961 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:53.003 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:53.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:53.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:53.111 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:53.140 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:53.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:53.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:53.248 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:53.273 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:53.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:53.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:53.377 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:53.412 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:53.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:53.419 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:49:53.421 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-01T10:49:53.520 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:53.553 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:53.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:53.557 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:49:53.658 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:53.670 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:53 smithi173 bash[58816]: cluster 2023-10-01T10:49:51.918733+0000 mgr.y (mgr.25072) 967 : cluster 0 pgmap v907: 89 pgs: 89 active+clean; 1.2 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 696 op/s 2023-10-01T10:49:53.690 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:53.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:53.696 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:53.797 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:53.807 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:53 smithi078 bash[75299]: cluster 2023-10-01T10:49:51.918733+0000 mgr.y (mgr.25072) 967 : cluster 0 pgmap v907: 89 pgs: 89 active+clean; 1.2 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 696 op/s 2023-10-01T10:49:53.807 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:53 smithi078 bash[78274]: cluster 2023-10-01T10:49:51.918733+0000 mgr.y (mgr.25072) 967 : cluster 0 pgmap v907: 89 pgs: 89 active+clean; 1.2 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 696 op/s 2023-10-01T10:49:53.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:53.838 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:53.838 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:49:53.939 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:53.965 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:53.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:53.973 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:49:54.074 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:54.122 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:54.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:54.127 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:54.228 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:54.256 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:54.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:54.265 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:54.366 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:54.393 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:54.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:54.401 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:54.502 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:54.542 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:54.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:54.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:49:54.553 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-01T10:49:54.651 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:54.683 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:54.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:54.688 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:54.789 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:54.820 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:54.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:54.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:49:54.925 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:54.964 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:54.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:54.968 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:49:55.069 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:55.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:55.104 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:55.104 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:55.201 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T10:49:55.205 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:55.245 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:55.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:55.251 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:49:55.352 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:55.416 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:55.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:55.419 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:55.520 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:55.526 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:55 smithi078 bash[75299]: cluster 2023-10-01T10:49:53.919124+0000 mgr.y (mgr.25072) 968 : cluster 0 pgmap v908: 89 pgs: 89 active+clean; 1.1 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2023-10-01T10:49:55.527 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:55 smithi078 bash[78274]: cluster 2023-10-01T10:49:53.919124+0000 mgr.y (mgr.25072) 968 : cluster 0 pgmap v908: 89 pgs: 89 active+clean; 1.1 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2023-10-01T10:49:55.549 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:55.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:55.558 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:49:55.659 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:55.705 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:55.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:55.710 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:49:55.810 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:55.839 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:55.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:55.842 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:49:55.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:55 smithi173 bash[58816]: cluster 2023-10-01T10:49:53.919124+0000 mgr.y (mgr.25072) 968 : cluster 0 pgmap v908: 89 pgs: 89 active+clean; 1.1 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2023-10-01T10:49:55.942 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:55.976 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:55.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:55.978 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:49:56.079 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:56.120 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:56.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:56.123 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:49:56.224 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:56.252 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:56.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:56.255 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:56.356 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:56.379 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:56.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:56.381 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:49:56.482 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:56.513 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:56.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:56.518 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:49:56.530 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T10:49:56.619 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:56.661 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:56.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:56.664 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:49:56.765 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:56.793 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:56.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:56.798 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:56.899 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:56.943 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:56.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:56.947 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:49:57.048 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:57.084 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:57.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:57.088 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:49:57.189 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:57.231 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:57.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:57.239 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:57.339 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:57.365 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:57.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:57.366 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:49:57.467 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:57.503 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:57.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:57.507 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:49:57.531 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:57 smithi173 bash[58816]: cluster 2023-10-01T10:49:55.919854+0000 mgr.y (mgr.25072) 969 : cluster 0 pgmap v909: 89 pgs: 89 active+clean; 1.0 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 688 op/s 2023-10-01T10:49:57.532 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:57 smithi173 bash[58816]: audit 2023-10-01T10:49:56.965655+0000 mon.a (mon.0) 1201 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:49:57.532 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:57 smithi173 bash[58816]: audit 2023-10-01T10:49:56.977610+0000 mon.a (mon.0) 1202 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:49:57.608 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:57.616 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:57 smithi078 bash[75299]: cluster 2023-10-01T10:49:55.919854+0000 mgr.y (mgr.25072) 969 : cluster 0 pgmap v909: 89 pgs: 89 active+clean; 1.0 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 688 op/s 2023-10-01T10:49:57.617 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:57 smithi078 bash[75299]: audit 2023-10-01T10:49:56.965655+0000 mon.a (mon.0) 1201 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:49:57.617 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:57 smithi078 bash[75299]: audit 2023-10-01T10:49:56.977610+0000 mon.a (mon.0) 1202 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:49:57.617 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:57 smithi078 bash[78274]: cluster 2023-10-01T10:49:55.919854+0000 mgr.y (mgr.25072) 969 : cluster 0 pgmap v909: 89 pgs: 89 active+clean; 1.0 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 688 op/s 2023-10-01T10:49:57.617 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:57 smithi078 bash[78274]: audit 2023-10-01T10:49:56.965655+0000 mon.a (mon.0) 1201 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:49:57.617 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:57 smithi078 bash[78274]: audit 2023-10-01T10:49:56.977610+0000 mon.a (mon.0) 1202 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:49:57.632 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:57.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:57.635 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:49:57.736 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:57.783 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:57.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:57.787 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:49:57.888 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:57.923 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:57.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:57.928 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:49:58.029 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:58.057 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:49:58.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:58.066 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:49:58.166 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:58.203 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:58.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:58.208 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:49:58.309 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:58.336 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:58.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:58.339 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:49:58.440 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:58.470 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:58.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:58.479 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:49:58.580 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:58.621 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:49:58.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:58.628 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:49:58.729 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:58.775 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:58.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:58.780 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:49:58.881 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:58.918 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:49:58.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:58.922 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:49:59.023 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:59.056 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:49:59.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:59.060 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:49:59.124 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:49:59.128 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:49:59.129 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:49:59.161 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:59.188 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:59.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:59.195 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:49:59.295 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:59.337 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:59.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:59.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:49:59.447 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:59.482 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:49:59.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:59.486 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:49:59.587 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:59.616 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:49:59.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:59.618 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:49:59.719 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:59.748 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:49:59.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:59.750 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:49:59.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:49:59 smithi078 bash[75299]: cluster 2023-10-01T10:49:57.920447+0000 mgr.y (mgr.25072) 970 : cluster 0 pgmap v910: 89 pgs: 89 active+clean; 987 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 680 op/s 2023-10-01T10:49:59.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:49:59 smithi078 bash[78274]: cluster 2023-10-01T10:49:57.920447+0000 mgr.y (mgr.25072) 970 : cluster 0 pgmap v910: 89 pgs: 89 active+clean; 987 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 680 op/s 2023-10-01T10:49:59.851 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:59.860 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:49:59 smithi173 bash[58816]: cluster 2023-10-01T10:49:57.920447+0000 mgr.y (mgr.25072) 970 : cluster 0 pgmap v910: 89 pgs: 89 active+clean; 987 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 680 op/s 2023-10-01T10:49:59.884 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:49:59.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:49:59.891 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:49:59.991 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:00.018 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:00.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:00.021 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:50:00.121 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:00.147 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:00.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:00.150 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:50:00.250 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:00.292 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:00.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:00.294 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:50:00.395 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:00.424 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:00.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:00.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:50:00.527 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:00.555 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:00.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:00.559 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:50:00.592 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:50:00 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:50:00] "GET /metrics HTTP/1.1" 200 36311 "" "Prometheus/2.33.4" 2023-10-01T10:50:00.660 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:00.684 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:00.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:00.689 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:50:00.790 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:00.811 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:00.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:00.816 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:50:00.917 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:00.950 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:00.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:00.953 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:50:01.053 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:01.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:00 smithi078 bash[78274]: cluster 2023-10-01T10:49:59.921167+0000 mgr.y (mgr.25072) 971 : cluster 0 pgmap v911: 89 pgs: 89 active+clean; 817 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 667 op/s 2023-10-01T10:50:01.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:00 smithi078 bash[78274]: audit 2023-10-01T10:49:59.937788+0000 mon.a (mon.0) 1203 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:50:01.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:00 smithi078 bash[78274]: audit 2023-10-01T10:49:59.947976+0000 mon.a (mon.0) 1204 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:50:01.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:00 smithi078 bash[78274]: cluster 2023-10-01T10:50:00.000096+0000 mon.a (mon.0) 1205 : cluster 3 Health detail: HEALTH_WARN noscrub flag(s) set 2023-10-01T10:50:01.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:00 smithi078 bash[78274]: cluster 2023-10-01T10:50:00.000121+0000 mon.a (mon.0) 1206 : cluster 3 [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-10-01T10:50:01.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:00 smithi078 bash[78274]: audit 2023-10-01T10:50:00.584701+0000 mon.a (mon.0) 1207 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:50:01.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:00 smithi078 bash[78274]: audit 2023-10-01T10:50:00.590116+0000 mon.a (mon.0) 1208 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:50:01.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:00 smithi078 bash[78274]: audit 2023-10-01T10:50:00.596966+0000 mon.a (mon.0) 1209 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:50:01.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:00 smithi078 bash[75299]: cluster 2023-10-01T10:49:59.921167+0000 mgr.y (mgr.25072) 971 : cluster 0 pgmap v911: 89 pgs: 89 active+clean; 817 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 667 op/s 2023-10-01T10:50:01.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:00 smithi078 bash[75299]: audit 2023-10-01T10:49:59.937788+0000 mon.a (mon.0) 1203 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:50:01.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:00 smithi078 bash[75299]: audit 2023-10-01T10:49:59.947976+0000 mon.a (mon.0) 1204 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:50:01.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:00 smithi078 bash[75299]: cluster 2023-10-01T10:50:00.000096+0000 mon.a (mon.0) 1205 : cluster 3 Health detail: HEALTH_WARN noscrub flag(s) set 2023-10-01T10:50:01.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:00 smithi078 bash[75299]: cluster 2023-10-01T10:50:00.000121+0000 mon.a (mon.0) 1206 : cluster 3 [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-10-01T10:50:01.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:00 smithi078 bash[75299]: audit 2023-10-01T10:50:00.584701+0000 mon.a (mon.0) 1207 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:50:01.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:00 smithi078 bash[75299]: audit 2023-10-01T10:50:00.590116+0000 mon.a (mon.0) 1208 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:50:01.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:00 smithi078 bash[75299]: audit 2023-10-01T10:50:00.596966+0000 mon.a (mon.0) 1209 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:50:01.084 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:01.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:01.088 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:50:01.157 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:50:01.189 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:01.215 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:01.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:01.219 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:50:01.319 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:01.329 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:00 smithi173 bash[58816]: cluster 2023-10-01T10:49:59.921167+0000 mgr.y (mgr.25072) 971 : cluster 0 pgmap v911: 89 pgs: 89 active+clean; 817 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 667 op/s 2023-10-01T10:50:01.329 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:00 smithi173 bash[58816]: audit 2023-10-01T10:49:59.937788+0000 mon.a (mon.0) 1203 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:50:01.329 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:00 smithi173 bash[58816]: audit 2023-10-01T10:49:59.947976+0000 mon.a (mon.0) 1204 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:50:01.329 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:00 smithi173 bash[58816]: cluster 2023-10-01T10:50:00.000096+0000 mon.a (mon.0) 1205 : cluster 3 Health detail: HEALTH_WARN noscrub flag(s) set 2023-10-01T10:50:01.329 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:00 smithi173 bash[58816]: cluster 2023-10-01T10:50:00.000121+0000 mon.a (mon.0) 1206 : cluster 3 [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-10-01T10:50:01.329 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:00 smithi173 bash[58816]: audit 2023-10-01T10:50:00.584701+0000 mon.a (mon.0) 1207 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:50:01.330 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:00 smithi173 bash[58816]: audit 2023-10-01T10:50:00.590116+0000 mon.a (mon.0) 1208 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:50:01.330 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:00 smithi173 bash[58816]: audit 2023-10-01T10:50:00.596966+0000 mon.a (mon.0) 1209 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:50:01.355 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:01.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:01.364 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:50:01.465 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:01.490 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:01.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:01.495 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:50:01.596 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:01.623 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:01.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:01.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:50:01.728 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:01.769 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:01.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:01.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:50:01.872 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:01.896 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:01.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:01.898 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:50:01.999 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:02.028 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:02.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:02.035 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:50:02.136 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:02.164 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:02.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:02.171 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:50:02.271 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:02.295 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:02.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:02.299 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:50:02.399 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:02.424 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:02.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:02.430 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:50:02.531 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:02.568 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:02.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:02.573 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:50:02.674 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:02.709 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:02.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:02.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:50:02.814 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:02.836 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:02.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:02.838 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:50:02.939 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:02.970 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:02.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:02.972 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:50:03.072 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:03.097 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:03.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:03.110 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:50:03.211 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:03.238 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:03.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:03.241 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:50:03.341 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:03.369 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:03.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:03.372 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:50:03.473 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:03.505 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:03.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:03.508 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:50:03.603 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:03 smithi078 bash[75299]: cluster 2023-10-01T10:50:01.921683+0000 mgr.y (mgr.25072) 972 : cluster 0 pgmap v912: 89 pgs: 89 active+clean; 779 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 659 op/s 2023-10-01T10:50:03.603 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:03 smithi078 bash[75299]: audit 2023-10-01T10:50:02.438139+0000 mon.a (mon.0) 1210 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:50:03.603 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:03 smithi078 bash[75299]: audit 2023-10-01T10:50:02.439498+0000 mon.a (mon.0) 1211 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:50:03.604 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:03 smithi078 bash[78274]: cluster 2023-10-01T10:50:01.921683+0000 mgr.y (mgr.25072) 972 : cluster 0 pgmap v912: 89 pgs: 89 active+clean; 779 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 659 op/s 2023-10-01T10:50:03.604 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:03 smithi078 bash[78274]: audit 2023-10-01T10:50:02.438139+0000 mon.a (mon.0) 1210 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:50:03.604 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:03 smithi078 bash[78274]: audit 2023-10-01T10:50:02.439498+0000 mon.a (mon.0) 1211 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:50:03.608 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:03.631 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:03.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:03.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:50:03.735 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:03.765 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:03.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:03.770 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:50:03.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:03 smithi173 bash[58816]: cluster 2023-10-01T10:50:01.921683+0000 mgr.y (mgr.25072) 972 : cluster 0 pgmap v912: 89 pgs: 89 active+clean; 779 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 659 op/s 2023-10-01T10:50:03.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:03 smithi173 bash[58816]: audit 2023-10-01T10:50:02.438139+0000 mon.a (mon.0) 1210 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:50:03.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:03 smithi173 bash[58816]: audit 2023-10-01T10:50:02.439498+0000 mon.a (mon.0) 1211 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:50:03.871 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:03.904 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:03.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:03.906 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:50:04.007 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:04.034 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:04.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:04.043 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:50:04.144 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:04.169 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:04.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:04.172 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:50:04.273 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:04.304 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:04.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:04.310 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:50:04.411 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:04.448 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:04.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:04.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:50:04.555 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:04.584 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:04.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:04.589 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:50:04.690 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:04.724 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:04.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:04.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:50:04.827 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:04.867 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:04.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:04.874 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:50:04.974 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:05.016 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:05.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:05.018 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:50:05.119 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:05.157 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:05.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:05.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:50:05.265 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:05.301 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:05.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:05.303 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:50:05.404 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:05.429 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:05.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:05.433 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:50:05.534 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:05.542 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:05 smithi173 bash[58816]: cluster 2023-10-01T10:50:03.922232+0000 mgr.y (mgr.25072) 973 : cluster 0 pgmap v913: 89 pgs: 89 active+clean; 709 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 647 op/s 2023-10-01T10:50:05.563 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:05.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:05.568 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:50:05.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:05 smithi078 bash[78274]: cluster 2023-10-01T10:50:03.922232+0000 mgr.y (mgr.25072) 973 : cluster 0 pgmap v913: 89 pgs: 89 active+clean; 709 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 647 op/s 2023-10-01T10:50:05.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:05 smithi078 bash[75299]: cluster 2023-10-01T10:50:03.922232+0000 mgr.y (mgr.25072) 973 : cluster 0 pgmap v913: 89 pgs: 89 active+clean; 709 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 647 op/s 2023-10-01T10:50:05.669 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:05.693 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:05.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:05.700 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:50:05.801 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:05.840 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:05.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:05.845 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:50:05.946 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:05.975 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:05.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:05.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:50:06.083 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:06.114 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:06.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:06.119 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:50:06.220 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:06.245 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:06.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:06.248 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:50:06.349 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:06.380 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:06.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:06.383 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:50:06.483 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:06.515 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:06.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:06.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:50:06.624 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:06.658 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:06.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:06.660 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:50:06.761 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:06.793 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:06.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:06.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:50:06.900 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:06.941 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:06.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:06.943 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:50:07.044 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:07.079 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:07.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:07.082 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:50:07.183 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:07.214 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:07.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:07.218 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:50:07.319 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:07.344 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:07.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:07.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:50:07.442 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:07 smithi173 bash[58816]: cluster 2023-10-01T10:50:05.923190+0000 mgr.y (mgr.25072) 974 : cluster 0 pgmap v914: 89 pgs: 89 active+clean; 605 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2023-10-01T10:50:07.448 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:07.481 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:07.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:07.485 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:50:07.586 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:07.610 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:07.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:07.622 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:50:07.723 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:07.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:07.753 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:07.753 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:50:07.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:07 smithi078 bash[78274]: cluster 2023-10-01T10:50:05.923190+0000 mgr.y (mgr.25072) 974 : cluster 0 pgmap v914: 89 pgs: 89 active+clean; 605 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2023-10-01T10:50:07.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:07 smithi078 bash[75299]: cluster 2023-10-01T10:50:05.923190+0000 mgr.y (mgr.25072) 974 : cluster 0 pgmap v914: 89 pgs: 89 active+clean; 605 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2023-10-01T10:50:07.854 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:07.888 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:07.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:07.891 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:50:07.992 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:08.016 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:08.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:08.019 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:50:08.119 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:08.152 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:08.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:08.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:50:08.258 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:08.287 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:08.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:08.297 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:50:08.398 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:08.440 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:08.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:08.442 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:50:08.543 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:08.571 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:08.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:08.578 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:50:08.678 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:08.721 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:08.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:08.725 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:50:08.826 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:08.860 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:08.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:08.864 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:50:08.965 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:08.995 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:08.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:09.000 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:50:09.101 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:09.132 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:09.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:09.137 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:50:09.238 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:09.276 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:09.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:09.278 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:50:09.379 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:09.406 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:09.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:09.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:50:09.512 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:09.538 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:09.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:09.546 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:50:09.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:09 smithi173 bash[58816]: cluster 2023-10-01T10:50:07.924005+0000 mgr.y (mgr.25072) 975 : cluster 0 pgmap v915: 89 pgs: 89 active+clean; 566 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 611 op/s 2023-10-01T10:50:09.646 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:09.677 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:09.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:09.683 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:50:09.784 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:09.821 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:09.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:09.827 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:50:09.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:09 smithi078 bash[78274]: cluster 2023-10-01T10:50:07.924005+0000 mgr.y (mgr.25072) 975 : cluster 0 pgmap v915: 89 pgs: 89 active+clean; 566 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 611 op/s 2023-10-01T10:50:09.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:09 smithi078 bash[75299]: cluster 2023-10-01T10:50:07.924005+0000 mgr.y (mgr.25072) 975 : cluster 0 pgmap v915: 89 pgs: 89 active+clean; 566 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 611 op/s 2023-10-01T10:50:09.928 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:09.960 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:09.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:09.963 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:50:10.063 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:10.106 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:10.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:10.111 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:50:10.212 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:10.236 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:10.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:10.241 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:50:10.342 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:10.380 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:10.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:10.382 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:50:10.482 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:10.509 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:10.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:10.513 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:50:10.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:50:10 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:50:10] "GET /metrics HTTP/1.1" 200 36308 "" "Prometheus/2.33.4" 2023-10-01T10:50:10.614 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:10.636 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:10.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:10.642 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:50:10.742 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:10.770 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:10.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:10.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:50:10.876 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:10.910 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:10.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:10.917 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:50:11.018 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:11.046 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:11.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:11.050 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:50:11.150 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:11.178 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:11.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:11.184 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:50:11.284 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:11.309 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:11.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:11.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:50:11.421 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:11.464 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:11.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:11.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:50:11.568 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:11.586 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:11 smithi173 bash[58816]: cluster 2023-10-01T10:50:09.924773+0000 mgr.y (mgr.25072) 976 : cluster 0 pgmap v916: 89 pgs: 89 active+clean; 396 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 786 op/s 2023-10-01T10:50:11.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:11.609 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:11.609 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:50:11.710 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:11.758 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:11.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:11.763 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:50:11.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:11 smithi078 bash[75299]: cluster 2023-10-01T10:50:09.924773+0000 mgr.y (mgr.25072) 976 : cluster 0 pgmap v916: 89 pgs: 89 active+clean; 396 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 786 op/s 2023-10-01T10:50:11.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:11 smithi078 bash[78274]: cluster 2023-10-01T10:50:09.924773+0000 mgr.y (mgr.25072) 976 : cluster 0 pgmap v916: 89 pgs: 89 active+clean; 396 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 786 op/s 2023-10-01T10:50:11.864 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:11.889 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:11.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:11.895 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:50:11.996 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:12.024 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:12.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:12.027 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:50:12.127 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:12.165 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:12.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:12.171 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:50:12.271 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:12.298 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:12.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:12.304 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:50:12.405 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:12.444 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:12.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:12.453 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:50:12.553 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:12.578 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:12.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:12.583 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:50:12.684 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:12.712 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:12.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:12.717 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:50:12.818 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:12.848 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:12.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:12.850 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:50:12.951 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:12.982 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:12.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:12.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:50:13.089 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:13.122 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:13.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:13.127 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:50:13.227 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:13.261 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:13.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:13.264 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:50:13.364 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:13.396 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:13.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:13.399 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:50:13.499 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:13.525 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:13.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:13.528 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:50:13.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:13 smithi173 bash[58816]: cluster 2023-10-01T10:50:11.925321+0000 mgr.y (mgr.25072) 977 : cluster 0 pgmap v917: 89 pgs: 89 active+clean; 355 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 616 op/s 2023-10-01T10:50:13.629 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:13.662 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:13.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:13.670 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:50:13.770 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:13.778 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:13 smithi078 bash[75299]: cluster 2023-10-01T10:50:11.925321+0000 mgr.y (mgr.25072) 977 : cluster 0 pgmap v917: 89 pgs: 89 active+clean; 355 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 616 op/s 2023-10-01T10:50:13.779 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:13 smithi078 bash[78274]: cluster 2023-10-01T10:50:11.925321+0000 mgr.y (mgr.25072) 977 : cluster 0 pgmap v917: 89 pgs: 89 active+clean; 355 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 616 op/s 2023-10-01T10:50:13.796 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:13.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:13.800 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:50:13.900 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:13.930 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:13.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:13.935 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:50:14.036 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:14.067 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:14.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:14.071 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:50:14.172 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:14.204 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:14.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:14.208 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:50:14.308 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:14.335 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:14.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:14.337 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:50:14.438 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:14.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:14.470 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:14.470 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:50:14.504 INFO:teuthology.orchestra.run.smithi078.stdout:ERROR: (22) Invalid argument 2023-10-01T10:50:14.505 INFO:teuthology.orchestra.run.smithi078.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T10:50:14.571 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:14.595 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:14.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:14.598 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:50:14.699 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:14.727 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:14.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:14.731 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:50:14.832 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:14.859 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:14.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:14.862 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:50:14.963 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:14.990 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:14.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:14.994 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:50:15.094 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:15.127 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T10:50:15.134 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:15.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:15.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:50:15.185 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T10:50:15.228 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T10:50:15.240 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:15.284 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:15.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:15.289 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:50:15.301 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T10:50:15.350 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T10:50:15.390 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:15.404 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T10:50:15.422 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:15.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:15.427 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:50:15.447 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T10:50:15.483 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T10:50:15.491 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T10:50:15.491 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T10:50:15.491 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T10:50:15.491 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T10:50:15.491 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T10:50:15.491 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T10:50:15.493 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T10:50:15.494 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T10:50:15.528 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:15.562 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:15.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:15.567 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:50:15.569 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:15 smithi078 bash[75299]: cluster 2023-10-01T10:50:13.925810+0000 mgr.y (mgr.25072) 978 : cluster 0 pgmap v918: 89 pgs: 89 active+clean; 284 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 660 op/s 2023-10-01T10:50:15.569 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:15 smithi078 bash[75299]: audit 2023-10-01T10:50:15.053396+0000 mon.c (mon.2) 169 : audit 0 from='client.? 172.21.15.78:0/4033653161' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:50:15.570 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:15 smithi078 bash[78274]: cluster 2023-10-01T10:50:13.925810+0000 mgr.y (mgr.25072) 978 : cluster 0 pgmap v918: 89 pgs: 89 active+clean; 284 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 660 op/s 2023-10-01T10:50:15.570 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:15 smithi078 bash[78274]: audit 2023-10-01T10:50:15.053396+0000 mon.c (mon.2) 169 : audit 0 from='client.? 172.21.15.78:0/4033653161' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:50:15.667 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:15.676 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:15 smithi173 bash[58816]: cluster 2023-10-01T10:50:13.925810+0000 mgr.y (mgr.25072) 978 : cluster 0 pgmap v918: 89 pgs: 89 active+clean; 284 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 660 op/s 2023-10-01T10:50:15.677 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:15 smithi173 bash[58816]: audit 2023-10-01T10:50:15.053396+0000 mon.c (mon.2) 169 : audit 0 from='client.? 172.21.15.78:0/4033653161' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:50:15.720 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:15.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:15.722 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:50:15.822 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:15.847 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:15.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:15.850 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:50:15.951 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:15.995 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:15.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:15.999 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:50:16.100 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:16.129 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:16.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:16.132 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:50:16.233 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:16.282 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:16.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:16.285 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:50:16.386 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:16.410 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:16.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:16.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:50:16.496 INFO:teuthology.orchestra.run.smithi078.stderr:nodeep-scrub is set 2023-10-01T10:50:16.497 INFO:teuthology.orchestra.run.smithi078.stderr:reweighted osd.5 to 1 (10000) 2023-10-01T10:50:16.512 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:16.521 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:16 smithi173 bash[58816]: audit 2023-10-01T10:50:15.515104+0000 mon.a (mon.0) 1212 : audit 1 from='client.? 172.21.15.78:0/2677538471' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-01T10:50:16.521 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:16 smithi173 bash[58816]: audit 2023-10-01T10:50:15.935330+0000 mon.a (mon.0) 1213 : audit 1 from='client.? 172.21.15.78:0/964339289' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:50:16.535 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:16.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:16.539 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:50:16.639 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:16.673 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:16.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:16.678 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:50:16.779 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:16.805 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:16.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:16.810 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:50:16.811 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:16 smithi078 bash[75299]: audit 2023-10-01T10:50:15.515104+0000 mon.a (mon.0) 1212 : audit 1 from='client.? 172.21.15.78:0/2677538471' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-01T10:50:16.811 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:16 smithi078 bash[75299]: audit 2023-10-01T10:50:15.935330+0000 mon.a (mon.0) 1213 : audit 1 from='client.? 172.21.15.78:0/964339289' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:50:16.811 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:16 smithi078 bash[78274]: audit 2023-10-01T10:50:15.515104+0000 mon.a (mon.0) 1212 : audit 1 from='client.? 172.21.15.78:0/2677538471' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-01T10:50:16.811 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:16 smithi078 bash[78274]: audit 2023-10-01T10:50:15.935330+0000 mon.a (mon.0) 1213 : audit 1 from='client.? 172.21.15.78:0/964339289' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:50:16.912 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:16.960 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:16.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:16.962 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:50:17.063 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:17.089 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:17.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:17.092 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:50:17.193 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:17.225 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:17.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:17.229 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:50:17.330 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:17.362 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:17.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:17.367 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:50:17.468 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:17.491 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:17 smithi173 bash[58816]: cluster 2023-10-01T10:50:15.926358+0000 mgr.y (mgr.25072) 979 : cluster 0 pgmap v919: 89 pgs: 89 active+clean; 167 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 722 op/s 2023-10-01T10:50:17.491 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:17 smithi173 bash[58816]: cluster 2023-10-01T10:50:16.479028+0000 mon.a (mon.0) 1214 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:50:17.491 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:17 smithi173 bash[58816]: cluster 2023-10-01T10:50:16.486058+0000 osd.5 (osd.5) 98 : cluster 3 failed to encode map e169 with expected crc 2023-10-01T10:50:17.491 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:17 smithi173 bash[58816]: audit 2023-10-01T10:50:16.489967+0000 mon.a (mon.0) 1215 : audit 1 from='client.? 172.21.15.78:0/2677538471' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-01T10:50:17.491 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:17 smithi173 bash[58816]: audit 2023-10-01T10:50:16.490115+0000 mon.a (mon.0) 1216 : audit 1 from='client.? 172.21.15.78:0/964339289' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T10:50:17.491 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:17 smithi173 bash[58816]: cluster 2023-10-01T10:50:16.490344+0000 mon.a (mon.0) 1217 : cluster 0 osdmap e169: 8 total, 8 up, 8 in 2023-10-01T10:50:17.492 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:17 smithi173 bash[58816]: cluster 2023-10-01T10:50:16.491161+0000 osd.7 (osd.7) 121 : cluster 3 failed to encode map e169 with expected crc 2023-10-01T10:50:17.492 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:17 smithi173 bash[58816]: cluster 2023-10-01T10:50:16.497495+0000 osd.6 (osd.6) 86 : cluster 3 failed to encode map e169 with expected crc 2023-10-01T10:50:17.492 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:17 smithi173 bash[58816]: audit 2023-10-01T10:50:17.430754+0000 mon.a (mon.0) 1218 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:50:17.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:17.493 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:17.493 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:50:17.594 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:17.602 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:17 smithi078 bash[78274]: cluster 2023-10-01T10:50:15.926358+0000 mgr.y (mgr.25072) 979 : cluster 0 pgmap v919: 89 pgs: 89 active+clean; 167 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 722 op/s 2023-10-01T10:50:17.602 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:17 smithi078 bash[78274]: cluster 2023-10-01T10:50:16.479028+0000 mon.a (mon.0) 1214 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:50:17.603 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:17 smithi078 bash[78274]: cluster 2023-10-01T10:50:16.486058+0000 osd.5 (osd.5) 98 : cluster 3 failed to encode map e169 with expected crc 2023-10-01T10:50:17.603 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:17 smithi078 bash[78274]: audit 2023-10-01T10:50:16.489967+0000 mon.a (mon.0) 1215 : audit 1 from='client.? 172.21.15.78:0/2677538471' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-01T10:50:17.603 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:17 smithi078 bash[78274]: audit 2023-10-01T10:50:16.490115+0000 mon.a (mon.0) 1216 : audit 1 from='client.? 172.21.15.78:0/964339289' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T10:50:17.603 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:17 smithi078 bash[78274]: cluster 2023-10-01T10:50:16.490344+0000 mon.a (mon.0) 1217 : cluster 0 osdmap e169: 8 total, 8 up, 8 in 2023-10-01T10:50:17.603 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:17 smithi078 bash[78274]: cluster 2023-10-01T10:50:16.491161+0000 osd.7 (osd.7) 121 : cluster 3 failed to encode map e169 with expected crc 2023-10-01T10:50:17.603 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:17 smithi078 bash[78274]: cluster 2023-10-01T10:50:16.497495+0000 osd.6 (osd.6) 86 : cluster 3 failed to encode map e169 with expected crc 2023-10-01T10:50:17.603 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:17 smithi078 bash[78274]: audit 2023-10-01T10:50:17.430754+0000 mon.a (mon.0) 1218 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:50:17.604 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:17 smithi078 bash[75299]: cluster 2023-10-01T10:50:15.926358+0000 mgr.y (mgr.25072) 979 : cluster 0 pgmap v919: 89 pgs: 89 active+clean; 167 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 722 op/s 2023-10-01T10:50:17.604 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:17 smithi078 bash[75299]: cluster 2023-10-01T10:50:16.479028+0000 mon.a (mon.0) 1214 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:50:17.604 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:17 smithi078 bash[75299]: cluster 2023-10-01T10:50:16.486058+0000 osd.5 (osd.5) 98 : cluster 3 failed to encode map e169 with expected crc 2023-10-01T10:50:17.604 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:17 smithi078 bash[75299]: audit 2023-10-01T10:50:16.489967+0000 mon.a (mon.0) 1215 : audit 1 from='client.? 172.21.15.78:0/2677538471' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-01T10:50:17.604 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:17 smithi078 bash[75299]: audit 2023-10-01T10:50:16.490115+0000 mon.a (mon.0) 1216 : audit 1 from='client.? 172.21.15.78:0/964339289' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T10:50:17.604 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:17 smithi078 bash[75299]: cluster 2023-10-01T10:50:16.490344+0000 mon.a (mon.0) 1217 : cluster 0 osdmap e169: 8 total, 8 up, 8 in 2023-10-01T10:50:17.604 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:17 smithi078 bash[75299]: cluster 2023-10-01T10:50:16.491161+0000 osd.7 (osd.7) 121 : cluster 3 failed to encode map e169 with expected crc 2023-10-01T10:50:17.604 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:17 smithi078 bash[75299]: cluster 2023-10-01T10:50:16.497495+0000 osd.6 (osd.6) 86 : cluster 3 failed to encode map e169 with expected crc 2023-10-01T10:50:17.604 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:17 smithi078 bash[75299]: audit 2023-10-01T10:50:17.430754+0000 mon.a (mon.0) 1218 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:50:17.617 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:17.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:17.619 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:50:17.661 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-01T10:50:17.720 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:17.738 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Removed 109910 objects 2023-10-01T10:50:17.738 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Clean up completed and total clean up time :154.417 2023-10-01T10:50:17.745 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:17.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:17.750 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:50:17.851 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:17.882 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:17.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:17.886 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:50:17.947 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_5 2023-10-01T10:50:17.948 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_5 unique_pool_5 --yes-i-really-really-mean-it 2023-10-01T10:50:17.986 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:18.016 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:18.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:18.018 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:50:18.119 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:18.138 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:18.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:18.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:50:18.241 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:18.261 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:18.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:18.263 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:50:18.365 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:18.387 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:18.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:18.389 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:50:18.490 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:18.521 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:18.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:18.524 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:50:18.625 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:18.660 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:18.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:18.662 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:50:18.763 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:18.802 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:18.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:18.804 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:50:18.905 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:18.936 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:18.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:18.939 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:50:19.040 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:19.074 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:19.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:19.077 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:50:19.178 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:19.208 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:19.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:19.211 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:50:19.312 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:19.329 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-01T10:50:19.336 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:19.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:19.340 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:50:19.441 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:19.474 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:19.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:19.478 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:50:19.579 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:19.617 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:19.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:19.621 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:50:19.721 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:19.756 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:19.758 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:19 smithi173 bash[58816]: cluster 2023-10-01T10:50:17.926739+0000 mgr.y (mgr.25072) 980 : cluster 0 pgmap v921: 89 pgs: 89 active+clean; 127 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2023-10-01T10:50:19.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:19.759 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:50:19.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:19 smithi078 bash[75299]: cluster 2023-10-01T10:50:17.926739+0000 mgr.y (mgr.25072) 980 : cluster 0 pgmap v921: 89 pgs: 89 active+clean; 127 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2023-10-01T10:50:19.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:19 smithi078 bash[78274]: cluster 2023-10-01T10:50:17.926739+0000 mgr.y (mgr.25072) 980 : cluster 0 pgmap v921: 89 pgs: 89 active+clean; 127 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2023-10-01T10:50:19.860 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:19.894 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:19.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:19.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:50:19.998 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:20.034 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:20.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:20.037 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:50:20.138 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:20.169 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:20.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:20.171 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:50:20.272 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:20.305 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:20.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:20.308 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:50:20.389 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T10:50:20.409 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:20.458 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:20.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:20.460 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:50:20.560 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:20.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:50:20 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:50:20] "GET /metrics HTTP/1.1" 200 36307 "" "Prometheus/2.33.4" 2023-10-01T10:50:20.593 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:20.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:20.596 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:50:20.697 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:20.729 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:20.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:20.732 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:50:20.834 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:20.864 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:20.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:20.867 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:50:20.968 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:21.003 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:21.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:21.006 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:50:21.107 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:21.137 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:21.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:21.140 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:50:21.240 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:21.273 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:21.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:21.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:50:21.315 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T10:50:21.377 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:21.413 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:21.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:21.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:50:21.516 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:21.544 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:21.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:21.548 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:50:21.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:21 smithi078 bash[75299]: cluster 2023-10-01T10:50:19.927551+0000 mgr.y (mgr.25072) 981 : cluster 0 pgmap v922: 89 pgs: 89 active+clean; 12 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 615 op/s 2023-10-01T10:50:21.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:21 smithi078 bash[78274]: cluster 2023-10-01T10:50:19.927551+0000 mgr.y (mgr.25072) 981 : cluster 0 pgmap v922: 89 pgs: 89 active+clean; 12 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 615 op/s 2023-10-01T10:50:21.649 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:21.677 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:21.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:21.680 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:50:21.781 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:21.815 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:21.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:21.818 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:50:21.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:21 smithi173 bash[58816]: cluster 2023-10-01T10:50:19.927551+0000 mgr.y (mgr.25072) 981 : cluster 0 pgmap v922: 89 pgs: 89 active+clean; 12 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 615 op/s 2023-10-01T10:50:21.919 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:21.954 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:21.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:21.956 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:50:22.057 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:22.090 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:22.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:22.093 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:50:22.194 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:22.227 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:22.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:22.230 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:50:22.331 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:22.366 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:22.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:22.369 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:50:22.464 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:50:22.464 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:50:22.469 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:22.501 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:22.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:22.504 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:50:22.605 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:22.626 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:22.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:22.629 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:50:22.729 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:22.752 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:22.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:22.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:50:22.855 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:22.885 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:22.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:22.888 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:50:22.988 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:23.022 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:23.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:23.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:50:23.125 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:23.153 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:23.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:23.156 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:50:23.256 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:23.289 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:23.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:23.292 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:50:23.393 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:23.427 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:23.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:23.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:50:23.479 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:50:23.531 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:23.562 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:23.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:23.564 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:50:23.665 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:23.676 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:23 smithi173 bash[58816]: cluster 2023-10-01T10:50:21.928399+0000 mgr.y (mgr.25072) 982 : cluster 0 pgmap v923: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 566 op/s 2023-10-01T10:50:23.699 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:23.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:23.702 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:50:23.803 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:23.835 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:23.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:23.838 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:50:23.838 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:23 smithi078 bash[75299]: cluster 2023-10-01T10:50:21.928399+0000 mgr.y (mgr.25072) 982 : cluster 0 pgmap v923: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 566 op/s 2023-10-01T10:50:23.839 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:23 smithi078 bash[78274]: cluster 2023-10-01T10:50:21.928399+0000 mgr.y (mgr.25072) 982 : cluster 0 pgmap v923: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 566 op/s 2023-10-01T10:50:23.939 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:23.971 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:23.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:23.973 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:50:24.074 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:24.100 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:24.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:24.103 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:50:24.204 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:24.228 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:24.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:24.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:50:24.331 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:24.360 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:24.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:24.363 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:50:24.464 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:24.490 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:50:24.496 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:24.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:24.499 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:50:24.600 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:24.628 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:24.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:24.631 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:50:24.731 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:24.759 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:24.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:24.762 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:50:24.862 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:24.894 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:24.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:24.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:50:24.998 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:25.031 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:25.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:25.035 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:50:25.135 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:25.166 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:25.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:25.169 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:50:25.270 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:25.293 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:25.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:25.295 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:50:25.396 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:25.418 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:25.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:25.420 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:50:25.508 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:50:25.521 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:25.532 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:25 smithi078 bash[78274]: cluster 2023-10-01T10:50:23.929010+0000 mgr.y (mgr.25072) 983 : cluster 0 pgmap v924: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 453 op/s 2023-10-01T10:50:25.532 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:25 smithi078 bash[75299]: cluster 2023-10-01T10:50:23.929010+0000 mgr.y (mgr.25072) 983 : cluster 0 pgmap v924: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 453 op/s 2023-10-01T10:50:25.553 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:25.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:25.555 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:50:25.656 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:25.678 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:25.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:25.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:50:25.782 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:25.796 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:25 smithi173 bash[58816]: cluster 2023-10-01T10:50:23.929010+0000 mgr.y (mgr.25072) 983 : cluster 0 pgmap v924: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 453 op/s 2023-10-01T10:50:25.814 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:25.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:25.817 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:50:25.918 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:25.951 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:25.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:25.954 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:50:26.055 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:26.090 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:26.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:26.093 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:50:26.194 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:26.222 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:26.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:26.225 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:50:26.326 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:26.356 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:26.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:26.359 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:50:26.460 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:26.494 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:26.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:26.497 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:50:26.598 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:26.625 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:26.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:26.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:50:26.728 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:26.750 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:26.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:26.752 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:50:26.853 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:26.888 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:26.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:26.891 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:50:26.991 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:27.013 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:27.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:27.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:50:27.116 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:27.137 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:27.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:27.138 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:50:27.239 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:27.274 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:27.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:27.277 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:50:27.378 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:27.409 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:27.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:27.412 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:50:27.513 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:27.533 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:27.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:27.535 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:50:27.636 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:27.648 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:27 smithi078 bash[75299]: cluster 2023-10-01T10:50:25.929879+0000 mgr.y (mgr.25072) 984 : cluster 0 pgmap v925: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 266 op/s 2023-10-01T10:50:27.648 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:27 smithi078 bash[78274]: cluster 2023-10-01T10:50:25.929879+0000 mgr.y (mgr.25072) 984 : cluster 0 pgmap v925: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 266 op/s 2023-10-01T10:50:27.664 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:27.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:27.667 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:50:27.768 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:27.791 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:27.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:27.794 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:50:27.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:27 smithi173 bash[58816]: cluster 2023-10-01T10:50:25.929879+0000 mgr.y (mgr.25072) 984 : cluster 0 pgmap v925: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 266 op/s 2023-10-01T10:50:27.895 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:27.929 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:27.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:27.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:50:28.033 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:28.060 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:28.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:28.063 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:50:28.164 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:28.194 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:28.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:28.198 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:50:28.298 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:28.329 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:28.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:28.332 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:50:28.433 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:28.467 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:28.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:28.470 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:50:28.571 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:28.600 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:28.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:28.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:50:28.704 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:28.731 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:28.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:28.733 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:50:28.834 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:28.869 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:28.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:28.871 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:50:28.972 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:29.003 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:29.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:29.006 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:50:29.106 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:29.138 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:29.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:29.141 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:50:29.242 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:29.274 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:29.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:29.277 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:50:29.377 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:29.415 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:29.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:29.418 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:50:29.519 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:29.550 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:29.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:29.553 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:50:29.654 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:29.666 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:29 smithi173 bash[58816]: cluster 2023-10-01T10:50:27.930663+0000 mgr.y (mgr.25072) 985 : cluster 0 pgmap v926: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 195 op/s 2023-10-01T10:50:29.678 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:29 smithi078 bash[78274]: cluster 2023-10-01T10:50:27.930663+0000 mgr.y (mgr.25072) 985 : cluster 0 pgmap v926: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 195 op/s 2023-10-01T10:50:29.678 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:29 smithi078 bash[75299]: cluster 2023-10-01T10:50:27.930663+0000 mgr.y (mgr.25072) 985 : cluster 0 pgmap v926: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 195 op/s 2023-10-01T10:50:29.685 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:29.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:29.688 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:50:29.789 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:29.812 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:29.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:29.814 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:50:29.914 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:29.947 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:29.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:29.950 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:50:30.051 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:30.072 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:30.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:30.075 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:50:30.175 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:30.211 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:30.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:30.214 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:50:30.315 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:30.330 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:50:30 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:50:30] "GET /metrics HTTP/1.1" 200 36307 "" "Prometheus/2.33.4" 2023-10-01T10:50:30.345 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:30.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:30.348 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:50:30.449 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:30.483 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:30.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:30.485 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:50:30.586 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:30.617 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:30.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:30.620 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:50:30.721 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:30.747 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:30.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:30.749 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:50:30.849 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:30.880 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:30.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:30.883 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:50:30.984 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:31.009 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:31.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:31.011 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:50:31.112 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:31.140 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:31.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:31.143 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:50:31.244 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:31.268 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:31.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:31.270 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:50:31.371 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:31.405 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:31.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:31.409 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:50:31.509 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:31.540 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:31.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:31.542 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:50:31.643 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:31.656 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:31 smithi173 bash[58816]: cluster 2023-10-01T10:50:29.931284+0000 mgr.y (mgr.25072) 986 : cluster 0 pgmap v927: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 168 op/s 2023-10-01T10:50:31.679 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:31.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:31.682 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:50:31.783 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:31.821 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:31.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:31.824 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:50:31.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:31 smithi078 bash[78274]: cluster 2023-10-01T10:50:29.931284+0000 mgr.y (mgr.25072) 986 : cluster 0 pgmap v927: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 168 op/s 2023-10-01T10:50:31.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:31 smithi078 bash[75299]: cluster 2023-10-01T10:50:29.931284+0000 mgr.y (mgr.25072) 986 : cluster 0 pgmap v927: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 168 op/s 2023-10-01T10:50:31.925 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:31.946 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:31.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:31.948 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:50:32.049 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:32.081 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:32.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:32.084 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:50:32.185 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:32.207 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:32.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:32.210 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:50:32.311 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:32.339 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:32.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:32.342 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:50:32.443 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:32.481 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:32.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:32.484 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:50:32.585 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:32.618 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:32.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:32.620 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:50:32.722 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:32.733 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:32 smithi173 bash[58816]: audit 2023-10-01T10:50:32.431135+0000 mon.a (mon.0) 1219 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:50:32.755 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:32.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:32.758 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:50:32.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:32 smithi078 bash[75299]: audit 2023-10-01T10:50:32.431135+0000 mon.a (mon.0) 1219 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:50:32.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:32 smithi078 bash[78274]: audit 2023-10-01T10:50:32.431135+0000 mon.a (mon.0) 1219 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:50:32.859 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:32.880 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:32.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:32.882 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:50:32.983 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:33.004 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:33.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:33.006 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:50:33.107 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:33.132 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:33.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:33.135 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:50:33.235 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:33.266 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:33.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:33.269 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:50:33.370 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:33.404 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:33.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:33.407 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:50:33.507 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:33.538 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:33.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:33.540 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:50:33.640 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:33.666 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:33.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:33.669 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:50:33.770 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:33.777 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:33 smithi078 bash[78274]: cluster 2023-10-01T10:50:31.932072+0000 mgr.y (mgr.25072) 987 : cluster 0 pgmap v928: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 14 op/s 2023-10-01T10:50:33.778 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:33 smithi078 bash[75299]: cluster 2023-10-01T10:50:31.932072+0000 mgr.y (mgr.25072) 987 : cluster 0 pgmap v928: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 14 op/s 2023-10-01T10:50:33.796 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:33.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:33.798 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:50:33.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:33 smithi173 bash[58816]: cluster 2023-10-01T10:50:31.932072+0000 mgr.y (mgr.25072) 987 : cluster 0 pgmap v928: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 14 op/s 2023-10-01T10:50:33.900 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:33.933 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:33.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:33.936 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:50:34.037 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:34.061 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:34.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:34.063 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:50:34.163 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:34.188 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:34.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:34.191 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:50:34.292 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:34.318 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:34.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:34.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:50:34.421 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:34.452 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:34.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:34.456 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:50:34.556 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:34.577 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:34.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:34.579 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:50:34.680 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:34.713 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:34.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:34.715 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:50:34.816 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:34.848 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:34.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:34.852 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:50:34.952 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:34.987 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:34.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:34.992 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:50:35.094 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:35.126 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:35.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:35.128 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:50:35.229 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:35.268 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:35.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:35.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:50:35.371 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:35.396 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:35.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:35.398 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:50:35.498 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:35.534 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:35.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:35.536 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:50:35.637 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:35.669 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:35.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:35.672 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:50:35.772 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:35.781 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:35 smithi078 bash[75299]: cluster 2023-10-01T10:50:33.932888+0000 mgr.y (mgr.25072) 988 : cluster 0 pgmap v929: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:50:35.781 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:35 smithi078 bash[78274]: cluster 2023-10-01T10:50:33.932888+0000 mgr.y (mgr.25072) 988 : cluster 0 pgmap v929: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:50:35.797 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:35.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:35.799 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:50:35.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:35 smithi173 bash[58816]: cluster 2023-10-01T10:50:33.932888+0000 mgr.y (mgr.25072) 988 : cluster 0 pgmap v929: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:50:35.900 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:35.931 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:35.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:35.933 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:50:36.034 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:36.059 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:36.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:36.062 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:50:36.163 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:36.188 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:36.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:36.190 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:50:36.291 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:36.320 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:36.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:36.323 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:50:36.423 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:36.460 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:36.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:36.464 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:50:36.564 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:36.595 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:36.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:36.598 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:50:36.699 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:36.721 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:36.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:36.724 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:50:36.824 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:36.856 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:36.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:36.860 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:50:36.961 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:36.982 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:36.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:36.985 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:50:37.085 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:37.116 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:37.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:37.120 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:50:37.220 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:37.251 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:37.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:37.254 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:50:37.355 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:37.384 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:37.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:37.385 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:50:37.486 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:37.509 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:37.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:37.510 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:50:37.546 INFO:teuthology.orchestra.run.smithi078.stderr:pool 'unique_pool_5' removed 2023-10-01T10:50:37.612 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:37.632 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:37.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:37.643 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:50:37.744 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:37.765 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:37 smithi078 bash[78274]: cluster 2023-10-01T10:50:35.933574+0000 mgr.y (mgr.25072) 989 : cluster 0 pgmap v930: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:50:37.765 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:37 smithi078 bash[78274]: audit 2023-10-01T10:50:37.292925+0000 mon.c (mon.2) 170 : audit 1 from='client.? 172.21.15.78:0/2590937252' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T10:50:37.765 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:37 smithi078 bash[78274]: audit 2023-10-01T10:50:37.293379+0000 mon.a (mon.0) 1220 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T10:50:37.766 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:37 smithi078 bash[75299]: cluster 2023-10-01T10:50:35.933574+0000 mgr.y (mgr.25072) 989 : cluster 0 pgmap v930: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:50:37.766 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:37 smithi078 bash[75299]: audit 2023-10-01T10:50:37.292925+0000 mon.c (mon.2) 170 : audit 1 from='client.? 172.21.15.78:0/2590937252' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T10:50:37.766 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:37 smithi078 bash[75299]: audit 2023-10-01T10:50:37.293379+0000 mon.a (mon.0) 1220 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T10:50:37.777 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:37.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:37.779 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:50:37.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:37 smithi173 bash[58816]: cluster 2023-10-01T10:50:35.933574+0000 mgr.y (mgr.25072) 989 : cluster 0 pgmap v930: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:50:37.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:37 smithi173 bash[58816]: audit 2023-10-01T10:50:37.292925+0000 mon.c (mon.2) 170 : audit 1 from='client.? 172.21.15.78:0/2590937252' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T10:50:37.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:37 smithi173 bash[58816]: audit 2023-10-01T10:50:37.293379+0000 mon.a (mon.0) 1220 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T10:50:37.880 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:37.902 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:37.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:37.907 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:50:38.008 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:38.026 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:38.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:38.028 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:50:38.129 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:38.150 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:38.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:38.152 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:50:38.253 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:38.277 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:38.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:38.279 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:50:38.380 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:38.400 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:38.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:38.403 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:50:38.504 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:38.526 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:38.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:38.528 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:50:38.629 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:38.656 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:38.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:38.659 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:50:38.760 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:38.772 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:38 smithi078 bash[78274]: cluster 2023-10-01T10:50:37.536047+0000 osd.5 (osd.5) 99 : cluster 3 failed to encode map e170 with expected crc 2023-10-01T10:50:38.772 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:38 smithi078 bash[78274]: cluster 2023-10-01T10:50:37.537086+0000 osd.7 (osd.7) 122 : cluster 3 failed to encode map e170 with expected crc 2023-10-01T10:50:38.772 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:38 smithi078 bash[78274]: audit 2023-10-01T10:50:37.537118+0000 mon.a (mon.0) 1221 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-10-01T10:50:38.772 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:38 smithi078 bash[78274]: cluster 2023-10-01T10:50:37.537200+0000 mon.a (mon.0) 1222 : cluster 0 osdmap e170: 8 total, 8 up, 8 in 2023-10-01T10:50:38.772 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:38 smithi078 bash[75299]: cluster 2023-10-01T10:50:37.536047+0000 osd.5 (osd.5) 99 : cluster 3 failed to encode map e170 with expected crc 2023-10-01T10:50:38.772 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:38 smithi078 bash[75299]: cluster 2023-10-01T10:50:37.537086+0000 osd.7 (osd.7) 122 : cluster 3 failed to encode map e170 with expected crc 2023-10-01T10:50:38.772 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:38 smithi078 bash[75299]: audit 2023-10-01T10:50:37.537118+0000 mon.a (mon.0) 1221 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-10-01T10:50:38.772 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:38 smithi078 bash[75299]: cluster 2023-10-01T10:50:37.537200+0000 mon.a (mon.0) 1222 : cluster 0 osdmap e170: 8 total, 8 up, 8 in 2023-10-01T10:50:38.785 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:38.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:38.787 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:50:38.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:38 smithi173 bash[58816]: cluster 2023-10-01T10:50:37.536047+0000 osd.5 (osd.5) 99 : cluster 3 failed to encode map e170 with expected crc 2023-10-01T10:50:38.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:38 smithi173 bash[58816]: cluster 2023-10-01T10:50:37.537086+0000 osd.7 (osd.7) 122 : cluster 3 failed to encode map e170 with expected crc 2023-10-01T10:50:38.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:38 smithi173 bash[58816]: audit 2023-10-01T10:50:37.537118+0000 mon.a (mon.0) 1221 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-10-01T10:50:38.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:38 smithi173 bash[58816]: cluster 2023-10-01T10:50:37.537200+0000 mon.a (mon.0) 1222 : cluster 0 osdmap e170: 8 total, 8 up, 8 in 2023-10-01T10:50:38.888 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:38.912 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:38.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:38.915 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:50:39.015 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:39.037 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:39.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:39.040 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:50:39.141 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:39.170 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:39.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:39.172 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:50:39.273 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:39.300 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:39.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:39.303 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:50:39.404 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:39.428 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:39.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:39.431 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:50:39.532 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:39.557 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:39.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:39.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:50:39.660 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:39.670 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:39 smithi078 bash[78274]: cluster 2023-10-01T10:50:37.933968+0000 mgr.y (mgr.25072) 990 : cluster 0 pgmap v932: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:50:39.670 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:39 smithi078 bash[75299]: cluster 2023-10-01T10:50:37.933968+0000 mgr.y (mgr.25072) 990 : cluster 0 pgmap v932: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:50:39.682 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:39.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:39.686 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:50:39.787 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:39.821 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:39.823 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:39 smithi173 bash[58816]: cluster 2023-10-01T10:50:37.933968+0000 mgr.y (mgr.25072) 990 : cluster 0 pgmap v932: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:50:39.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:39.824 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:50:39.926 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:39.947 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:39.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:39.949 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:50:40.050 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:40.074 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2023-10-01T10:50:40.074 INFO:teuthology.orchestra.run.smithi078.stdout: "ops": [], 2023-10-01T10:50:40.075 INFO:teuthology.orchestra.run.smithi078.stdout: "complaint_time": 30, 2023-10-01T10:50:40.075 INFO:teuthology.orchestra.run.smithi078.stdout: "num_blocked_ops": 0 2023-10-01T10:50:40.075 INFO:teuthology.orchestra.run.smithi078.stdout:} 2023-10-01T10:50:40.082 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:40.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:40.086 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:50:40.099 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-01T10:50:40.099 INFO:tasks.radosbench:Beginning radosbench... 2023-10-01T10:50:40.099 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_6 2023-10-01T10:50:40.100 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_6 16 2023-10-01T10:50:40.187 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:40.206 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:40.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:40.210 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:50:40.310 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:40.331 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:40.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:40.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:50:40.339 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T10:50:40.360 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T10:50:40.384 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T10:50:40.410 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T10:50:40.435 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:40.435 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T10:50:40.460 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T10:50:40.465 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:40.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:40.468 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:50:40.481 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T10:50:40.512 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T10:50:40.515 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T10:50:40.515 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T10:50:40.515 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T10:50:40.515 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T10:50:40.515 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T10:50:40.515 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T10:50:40.515 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T10:50:40.515 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T10:50:40.541 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:50:40 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:50:40] "GET /metrics HTTP/1.1" 200 36261 "" "Prometheus/2.33.4" 2023-10-01T10:50:40.567 INFO:teuthology.orchestra.run.smithi078.stderr:reweighted osd.6 to 1 (10000) 2023-10-01T10:50:40.569 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:40.592 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:40.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:40.595 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:50:40.695 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:40.709 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:40 smithi173 bash[58816]: audit 2023-10-01T10:50:39.984051+0000 mon.a (mon.0) 1223 : audit 1 from='client.? 172.21.15.78:0/3504080311' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-01T10:50:40.709 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:40 smithi173 bash[58816]: audit 2023-10-01T10:50:40.316434+0000 mon.a (mon.0) 1224 : audit 0 from='client.? 172.21.15.78:0/944017041' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:50:40.730 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:40.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:40.733 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:50:40.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:40 smithi078 bash[75299]: audit 2023-10-01T10:50:39.984051+0000 mon.a (mon.0) 1223 : audit 1 from='client.? 172.21.15.78:0/3504080311' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-01T10:50:40.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:40 smithi078 bash[75299]: audit 2023-10-01T10:50:40.316434+0000 mon.a (mon.0) 1224 : audit 0 from='client.? 172.21.15.78:0/944017041' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:50:40.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:40 smithi078 bash[78274]: audit 2023-10-01T10:50:39.984051+0000 mon.a (mon.0) 1223 : audit 1 from='client.? 172.21.15.78:0/3504080311' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-01T10:50:40.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:40 smithi078 bash[78274]: audit 2023-10-01T10:50:40.316434+0000 mon.a (mon.0) 1224 : audit 0 from='client.? 172.21.15.78:0/944017041' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:50:40.834 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:40.859 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:40.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:40.862 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:50:40.963 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:40.997 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:41.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:41.001 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:50:41.102 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:41.126 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:41.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:41.128 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:50:41.229 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:41.262 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:41.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:41.266 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:50:41.367 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:41.395 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:41.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:41.399 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:50:41.500 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:41.530 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:41.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:41.534 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:50:41.576 INFO:teuthology.orchestra.run.smithi078.stderr:noscrub is unset 2023-10-01T10:50:41.635 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:41.667 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:41.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:41.670 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:50:41.771 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:41.793 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:41.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:41.795 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:50:41.825 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:41 smithi078 bash[78274]: cluster 2023-10-01T10:50:39.934621+0000 mgr.y (mgr.25072) 991 : cluster 0 pgmap v933: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:50:41.825 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:41 smithi078 bash[78274]: audit 2023-10-01T10:50:40.562917+0000 mon.a (mon.0) 1225 : audit 1 from='client.? 172.21.15.78:0/3504080311' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-01T10:50:41.825 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:41 smithi078 bash[78274]: cluster 2023-10-01T10:50:40.562964+0000 mon.a (mon.0) 1226 : cluster 0 osdmap e171: 8 total, 8 up, 8 in 2023-10-01T10:50:41.825 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:41 smithi078 bash[78274]: audit 2023-10-01T10:50:40.729126+0000 mon.a (mon.0) 1227 : audit 1 from='client.? 172.21.15.78:0/445529644' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T10:50:41.826 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:41 smithi078 bash[75299]: cluster 2023-10-01T10:50:39.934621+0000 mgr.y (mgr.25072) 991 : cluster 0 pgmap v933: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:50:41.826 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:41 smithi078 bash[75299]: audit 2023-10-01T10:50:40.562917+0000 mon.a (mon.0) 1225 : audit 1 from='client.? 172.21.15.78:0/3504080311' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-01T10:50:41.826 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:41 smithi078 bash[75299]: cluster 2023-10-01T10:50:40.562964+0000 mon.a (mon.0) 1226 : cluster 0 osdmap e171: 8 total, 8 up, 8 in 2023-10-01T10:50:41.826 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:41 smithi078 bash[75299]: audit 2023-10-01T10:50:40.729126+0000 mon.a (mon.0) 1227 : audit 1 from='client.? 172.21.15.78:0/445529644' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T10:50:41.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:41 smithi173 bash[58816]: cluster 2023-10-01T10:50:39.934621+0000 mgr.y (mgr.25072) 991 : cluster 0 pgmap v933: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:50:41.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:41 smithi173 bash[58816]: audit 2023-10-01T10:50:40.562917+0000 mon.a (mon.0) 1225 : audit 1 from='client.? 172.21.15.78:0/3504080311' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-01T10:50:41.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:41 smithi173 bash[58816]: cluster 2023-10-01T10:50:40.562964+0000 mon.a (mon.0) 1226 : cluster 0 osdmap e171: 8 total, 8 up, 8 in 2023-10-01T10:50:41.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:41 smithi173 bash[58816]: audit 2023-10-01T10:50:40.729126+0000 mon.a (mon.0) 1227 : audit 1 from='client.? 172.21.15.78:0/445529644' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T10:50:41.896 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:41.934 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:41.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:41.935 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:50:42.036 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:42.063 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:42.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:42.067 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:50:42.167 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:42.169 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-01T10:50:42.196 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:42.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:42.198 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:50:42.299 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:42.330 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:42.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:42.332 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:50:42.433 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:42.468 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:42.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:42.471 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:50:42.572 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:42.600 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:42.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:42.603 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:50:42.704 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:42.730 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:42.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:42.733 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:50:42.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:42 smithi078 bash[78274]: cluster 2023-10-01T10:50:41.474925+0000 osd.5 (osd.5) 100 : cluster 3 failed to encode map e171 with expected crc 2023-10-01T10:50:42.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:42 smithi078 bash[78274]: cluster 2023-10-01T10:50:41.493417+0000 osd.6 (osd.6) 87 : cluster 3 failed to encode map e171 with expected crc 2023-10-01T10:50:42.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:42 smithi078 bash[78274]: cluster 2023-10-01T10:50:41.558429+0000 mon.a (mon.0) 1228 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:50:42.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:42 smithi078 bash[78274]: cluster 2023-10-01T10:50:41.570258+0000 osd.7 (osd.7) 123 : cluster 3 failed to encode map e171 with expected crc 2023-10-01T10:50:42.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:42 smithi078 bash[78274]: audit 2023-10-01T10:50:41.571173+0000 mon.a (mon.0) 1229 : audit 1 from='client.? 172.21.15.78:0/445529644' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T10:50:42.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:42 smithi078 bash[78274]: cluster 2023-10-01T10:50:41.571258+0000 mon.a (mon.0) 1230 : cluster 0 osdmap e172: 8 total, 8 up, 8 in 2023-10-01T10:50:42.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:42 smithi078 bash[78274]: cluster 2023-10-01T10:50:41.962542+0000 osd.6 (osd.6) 88 : cluster 3 failed to encode map e172 with expected crc 2023-10-01T10:50:42.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:42 smithi078 bash[78274]: cluster 2023-10-01T10:50:41.965323+0000 osd.6 (osd.6) 89 : cluster 3 failed to encode map e172 with expected crc 2023-10-01T10:50:42.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:42 smithi078 bash[75299]: cluster 2023-10-01T10:50:41.474925+0000 osd.5 (osd.5) 100 : cluster 3 failed to encode map e171 with expected crc 2023-10-01T10:50:42.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:42 smithi078 bash[75299]: cluster 2023-10-01T10:50:41.493417+0000 osd.6 (osd.6) 87 : cluster 3 failed to encode map e171 with expected crc 2023-10-01T10:50:42.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:42 smithi078 bash[75299]: cluster 2023-10-01T10:50:41.558429+0000 mon.a (mon.0) 1228 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:50:42.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:42 smithi078 bash[75299]: cluster 2023-10-01T10:50:41.570258+0000 osd.7 (osd.7) 123 : cluster 3 failed to encode map e171 with expected crc 2023-10-01T10:50:42.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:42 smithi078 bash[75299]: audit 2023-10-01T10:50:41.571173+0000 mon.a (mon.0) 1229 : audit 1 from='client.? 172.21.15.78:0/445529644' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T10:50:42.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:42 smithi078 bash[75299]: cluster 2023-10-01T10:50:41.571258+0000 mon.a (mon.0) 1230 : cluster 0 osdmap e172: 8 total, 8 up, 8 in 2023-10-01T10:50:42.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:42 smithi078 bash[75299]: cluster 2023-10-01T10:50:41.962542+0000 osd.6 (osd.6) 88 : cluster 3 failed to encode map e172 with expected crc 2023-10-01T10:50:42.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:42 smithi078 bash[75299]: cluster 2023-10-01T10:50:41.965323+0000 osd.6 (osd.6) 89 : cluster 3 failed to encode map e172 with expected crc 2023-10-01T10:50:42.833 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:42.861 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:42.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:42.864 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:50:42.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:42 smithi173 bash[58816]: cluster 2023-10-01T10:50:41.474925+0000 osd.5 (osd.5) 100 : cluster 3 failed to encode map e171 with expected crc 2023-10-01T10:50:42.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:42 smithi173 bash[58816]: cluster 2023-10-01T10:50:41.493417+0000 osd.6 (osd.6) 87 : cluster 3 failed to encode map e171 with expected crc 2023-10-01T10:50:42.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:42 smithi173 bash[58816]: cluster 2023-10-01T10:50:41.558429+0000 mon.a (mon.0) 1228 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:50:42.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:42 smithi173 bash[58816]: cluster 2023-10-01T10:50:41.570258+0000 osd.7 (osd.7) 123 : cluster 3 failed to encode map e171 with expected crc 2023-10-01T10:50:42.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:42 smithi173 bash[58816]: audit 2023-10-01T10:50:41.571173+0000 mon.a (mon.0) 1229 : audit 1 from='client.? 172.21.15.78:0/445529644' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T10:50:42.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:42 smithi173 bash[58816]: cluster 2023-10-01T10:50:41.571258+0000 mon.a (mon.0) 1230 : cluster 0 osdmap e172: 8 total, 8 up, 8 in 2023-10-01T10:50:42.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:42 smithi173 bash[58816]: cluster 2023-10-01T10:50:41.962542+0000 osd.6 (osd.6) 88 : cluster 3 failed to encode map e172 with expected crc 2023-10-01T10:50:42.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:42 smithi173 bash[58816]: cluster 2023-10-01T10:50:41.965323+0000 osd.6 (osd.6) 89 : cluster 3 failed to encode map e172 with expected crc 2023-10-01T10:50:42.965 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:42.998 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:43.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:43.001 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:50:43.102 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:43.134 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:43.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:43.137 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:50:43.238 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:43.510 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-01T10:50:43.578 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:43.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:43.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:50:43.680 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:43.715 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:43.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:43.719 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:50:43.820 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:43.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:43 smithi078 bash[78274]: cluster 2023-10-01T10:50:41.935344+0000 mgr.y (mgr.25072) 992 : cluster 0 pgmap v936: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:50:43.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:43 smithi078 bash[78274]: cluster 2023-10-01T10:50:42.153247+0000 osd.5 (osd.5) 101 : cluster 3 failed to encode map e172 with expected crc 2023-10-01T10:50:43.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:43 smithi078 bash[78274]: cluster 2023-10-01T10:50:42.155979+0000 osd.5 (osd.5) 102 : cluster 3 failed to encode map e172 with expected crc 2023-10-01T10:50:43.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:43 smithi078 bash[78274]: cluster 2023-10-01T10:50:42.158563+0000 osd.5 (osd.5) 103 : cluster 3 failed to encode map e172 with expected crc 2023-10-01T10:50:43.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:43 smithi078 bash[75299]: cluster 2023-10-01T10:50:41.935344+0000 mgr.y (mgr.25072) 992 : cluster 0 pgmap v936: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:50:43.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:43 smithi078 bash[75299]: cluster 2023-10-01T10:50:42.153247+0000 osd.5 (osd.5) 101 : cluster 3 failed to encode map e172 with expected crc 2023-10-01T10:50:43.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:43 smithi078 bash[75299]: cluster 2023-10-01T10:50:42.155979+0000 osd.5 (osd.5) 102 : cluster 3 failed to encode map e172 with expected crc 2023-10-01T10:50:43.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:43 smithi078 bash[75299]: cluster 2023-10-01T10:50:42.158563+0000 osd.5 (osd.5) 103 : cluster 3 failed to encode map e172 with expected crc 2023-10-01T10:50:43.832 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:43 smithi173 bash[58816]: cluster 2023-10-01T10:50:41.935344+0000 mgr.y (mgr.25072) 992 : cluster 0 pgmap v936: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:50:43.832 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:43 smithi173 bash[58816]: cluster 2023-10-01T10:50:42.153247+0000 osd.5 (osd.5) 101 : cluster 3 failed to encode map e172 with expected crc 2023-10-01T10:50:43.832 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:43 smithi173 bash[58816]: cluster 2023-10-01T10:50:42.155979+0000 osd.5 (osd.5) 102 : cluster 3 failed to encode map e172 with expected crc 2023-10-01T10:50:43.832 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:43 smithi173 bash[58816]: cluster 2023-10-01T10:50:42.158563+0000 osd.5 (osd.5) 103 : cluster 3 failed to encode map e172 with expected crc 2023-10-01T10:50:43.850 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:43.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:43.852 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:50:43.953 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:43.985 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:43.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:43.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:50:44.089 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:44.118 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:44.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:44.122 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:50:44.222 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:44.253 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:44.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:44.255 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:50:44.356 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:44.388 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:44.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:44.391 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:50:44.492 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:44.524 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:44.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:44.527 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:50:44.628 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:44.656 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:44.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:44.658 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:50:44.759 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:44.791 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:44.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:44.794 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:50:44.895 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:44.930 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:44.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:44.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:50:45.033 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:45.062 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:45.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:45.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:50:45.166 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:45.200 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:45.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:45.203 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:50:45.303 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:45.335 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:45.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:45.338 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:50:45.438 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:45.472 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:45.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:45.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:50:45.510 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T10:50:45.576 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:45.605 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:45.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:45.607 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:50:45.708 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:45.731 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:45.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:45.733 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:50:45.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:45 smithi078 bash[75299]: cluster 2023-10-01T10:50:43.936090+0000 mgr.y (mgr.25072) 993 : cluster 0 pgmap v937: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:50:45.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:45 smithi078 bash[78274]: cluster 2023-10-01T10:50:43.936090+0000 mgr.y (mgr.25072) 993 : cluster 0 pgmap v937: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:50:45.834 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:45.866 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:45.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:45.869 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:50:45.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:45 smithi173 bash[58816]: cluster 2023-10-01T10:50:43.936090+0000 mgr.y (mgr.25072) 993 : cluster 0 pgmap v937: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:50:45.877 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T10:50:45.970 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:46.003 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:46.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:46.006 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:50:46.106 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:46.139 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:46.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:46.142 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:50:46.245 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:46.278 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:46.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:46.281 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:50:46.382 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:46.407 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:46.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:46.409 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:50:46.510 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:46.546 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:46.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:46.550 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:50:46.650 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:46.682 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:46.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:46.686 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:50:46.786 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:46.819 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:46.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:46.823 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:50:46.924 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:46.950 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:46.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:46.953 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:50:47.054 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:47.085 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:47.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:47.088 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:50:47.189 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:47.220 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:47.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:47.223 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:50:47.324 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:47.355 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:47.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:47.359 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:50:47.408 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:50:47.459 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:47.487 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:47.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:47.489 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:50:47.590 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:47.601 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:47 smithi173 bash[58816]: cluster 2023-10-01T10:50:45.937061+0000 mgr.y (mgr.25072) 994 : cluster 0 pgmap v938: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:50:47.602 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:47 smithi173 bash[58816]: audit 2023-10-01T10:50:47.432167+0000 mon.a (mon.0) 1231 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:50:47.620 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:47.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:47.623 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:50:47.724 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:47.764 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:47.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:47.767 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:50:47.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:47 smithi078 bash[75299]: cluster 2023-10-01T10:50:45.937061+0000 mgr.y (mgr.25072) 994 : cluster 0 pgmap v938: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:50:47.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:47 smithi078 bash[75299]: audit 2023-10-01T10:50:47.432167+0000 mon.a (mon.0) 1231 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:50:47.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:47 smithi078 bash[78274]: cluster 2023-10-01T10:50:45.937061+0000 mgr.y (mgr.25072) 994 : cluster 0 pgmap v938: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:50:47.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:47 smithi078 bash[78274]: audit 2023-10-01T10:50:47.432167+0000 mon.a (mon.0) 1231 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:50:47.868 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:47.904 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:47.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:47.909 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:50:48.010 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:48.051 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:48.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:48.054 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:50:48.154 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:48.188 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:48.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:48.192 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:50:48.293 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:48.326 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:48.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:48.331 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:50:48.426 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:50:48.427 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:50:48.431 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:48.465 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:48.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:48.469 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:50:48.569 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:48.603 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:48.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:48.605 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:50:48.706 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:48.733 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:48.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:48.736 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:50:48.837 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:48.867 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:48.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:48.871 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:50:48.972 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:49.009 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:49.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:49.012 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:50:49.113 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:49.149 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:49.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:49.152 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:50:49.253 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:49.287 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:49.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:49.291 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:50:49.392 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:49.417 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:49.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:49.419 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:50:49.439 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:50:49.520 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:49.545 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:49.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:49.547 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:50:49.648 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:49.670 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:49.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:49.673 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:50:49.688 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:49 smithi078 bash[75299]: cluster 2023-10-01T10:50:47.937626+0000 mgr.y (mgr.25072) 995 : cluster 0 pgmap v939: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:50:49.688 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:49 smithi078 bash[78274]: cluster 2023-10-01T10:50:47.937626+0000 mgr.y (mgr.25072) 995 : cluster 0 pgmap v939: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:50:49.774 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:49.804 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:49.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:49.805 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:50:49.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:49 smithi173 bash[58816]: cluster 2023-10-01T10:50:47.937626+0000 mgr.y (mgr.25072) 995 : cluster 0 pgmap v939: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:50:49.906 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:49.926 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:49.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:49.929 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:50:50.030 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:50.060 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:50.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:50.063 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:50:50.164 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:50.188 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:50.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:50.190 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:50:50.291 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:50.328 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:50.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:50.331 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:50:50.331 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:50:50 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:50:50] "GET /metrics HTTP/1.1" 200 33903 "" "Prometheus/2.33.4" 2023-10-01T10:50:50.432 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:50.452 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:50:50.463 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:50.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:50.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:50:50.569 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:50.594 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:50.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:50.596 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:50:50.697 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:50.727 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:50.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:50.730 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:50:50.831 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:50.866 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:50.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:50.868 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:50:50.969 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:50.998 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:51.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:51.000 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:50:51.101 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:51.135 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:51.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:51.138 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:50:51.238 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:51.269 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:51.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:51.272 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:50:51.373 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:51.394 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:51.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:51.396 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:50:51.497 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:51.527 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:51.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:51.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:50:51.631 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:51.655 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:51 smithi078 bash[75299]: cluster 2023-10-01T10:50:49.938698+0000 mgr.y (mgr.25072) 996 : cluster 0 pgmap v940: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:50:51.656 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:51 smithi078 bash[78274]: cluster 2023-10-01T10:50:49.938698+0000 mgr.y (mgr.25072) 996 : cluster 0 pgmap v940: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:50:51.662 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:51.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:51.665 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:50:51.766 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:51.778 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:51 smithi173 bash[58816]: cluster 2023-10-01T10:50:49.938698+0000 mgr.y (mgr.25072) 996 : cluster 0 pgmap v940: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:50:51.798 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:51.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:51.802 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:50:51.902 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:51.928 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:51.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:51.930 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:50:52.031 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:52.052 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:52.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:52.054 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:50:52.155 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:52.176 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:52.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:52.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:50:52.279 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:52.309 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:52.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:52.312 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:50:52.412 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:52.447 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:52.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:52.450 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:50:52.551 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:52.581 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:52.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:52.584 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:50:52.685 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:52.718 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:52.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:52.720 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:50:52.821 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:52.851 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:52.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:52.855 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:50:52.956 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:52.981 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:52.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:52.984 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:50:53.085 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:53.120 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:53.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:53.124 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:50:53.225 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:53.250 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:53.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:53.253 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:50:53.353 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:53.378 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:53.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:53.381 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:50:53.482 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:53.505 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:53.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:53.507 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:50:53.608 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:53.637 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:53.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:53.639 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:50:53.740 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:53.771 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:53.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:53.775 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:50:53.835 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:53 smithi078 bash[75299]: cluster 2023-10-01T10:50:51.939511+0000 mgr.y (mgr.25072) 997 : cluster 0 pgmap v941: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:50:53.836 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:53 smithi078 bash[78274]: cluster 2023-10-01T10:50:51.939511+0000 mgr.y (mgr.25072) 997 : cluster 0 pgmap v941: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:50:53.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:53 smithi173 bash[58816]: cluster 2023-10-01T10:50:51.939511+0000 mgr.y (mgr.25072) 997 : cluster 0 pgmap v941: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:50:53.876 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:53.909 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:53.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:53.911 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:50:54.012 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:54.043 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:54.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:54.047 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:50:54.147 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:54.181 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:54.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:54.183 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:50:54.284 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:54.311 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:54.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:54.315 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:50:54.416 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:54.440 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:54.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:54.443 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:50:54.544 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:54.571 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:54.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:54.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:50:54.675 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:54.706 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:54.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:54.709 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:50:54.809 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:54.836 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:54.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:54.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:50:54.939 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:54.964 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:54.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:54.967 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:50:55.068 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:55.096 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:55.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:55.099 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:50:55.199 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:55.221 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:55.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:55.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:50:55.324 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:55.352 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:55.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:55.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:50:55.456 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:55.486 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:55.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:55.489 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:50:55.590 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:55.614 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:55.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:55.616 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:50:55.717 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:55.751 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:55.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:55.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:50:55.854 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:55 smithi078 bash[78274]: cluster 2023-10-01T10:50:53.939963+0000 mgr.y (mgr.25072) 998 : cluster 0 pgmap v942: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:50:55.854 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:55 smithi078 bash[75299]: cluster 2023-10-01T10:50:53.939963+0000 mgr.y (mgr.25072) 998 : cluster 0 pgmap v942: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:50:55.855 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:55.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:55 smithi173 bash[58816]: cluster 2023-10-01T10:50:53.939963+0000 mgr.y (mgr.25072) 998 : cluster 0 pgmap v942: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:50:55.884 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:55.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:55.887 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:50:55.987 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:56.017 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:56.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:56.020 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:50:56.121 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:56.151 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:56.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:56.153 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:50:56.254 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:56.289 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:56.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:56.291 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:50:56.392 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:56.418 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:56.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:56.420 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:50:56.521 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:56.542 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:56.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:56.544 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:50:56.645 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:56.668 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:56.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:56.671 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:50:56.771 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:56.794 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:56.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:56.796 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:50:56.897 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:56.920 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:56.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:56.923 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:50:57.023 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:57.055 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:57.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:57.059 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:50:57.160 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:57.196 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:57.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:57.199 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:50:57.300 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:57.329 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:57.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:57.332 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:50:57.433 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:57.457 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:57.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:57.460 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:50:57.561 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:57.589 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:57.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:57.592 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:50:57.693 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:57.725 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:57.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:57.729 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:50:57.830 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:57.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:57 smithi078 bash[75299]: cluster 2023-10-01T10:50:55.940603+0000 mgr.y (mgr.25072) 999 : cluster 0 pgmap v943: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:50:57.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:57 smithi078 bash[78274]: cluster 2023-10-01T10:50:55.940603+0000 mgr.y (mgr.25072) 999 : cluster 0 pgmap v943: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:50:57.863 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:57.865 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:57 smithi173 bash[58816]: cluster 2023-10-01T10:50:55.940603+0000 mgr.y (mgr.25072) 999 : cluster 0 pgmap v943: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:50:57.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:57.866 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:50:57.967 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:57.990 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:57.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:57.992 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:50:58.093 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:58.126 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:50:58.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:58.129 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:50:58.230 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:58.256 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:58.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:58.259 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:50:58.360 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:58.393 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:58.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:58.395 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:50:58.496 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:58.517 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:50:58.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:58.519 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:50:58.620 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:58.647 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:58.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:58.650 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:50:58.750 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:58.776 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:58.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:58.779 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:50:58.880 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:58.908 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:50:58.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:58.910 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:50:59.010 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:59.036 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:59.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:59.038 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:50:59.140 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:59.168 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:59.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:59.171 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:50:59.271 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:59.304 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:59.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:59.308 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:50:59.408 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:59.434 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:50:59.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:59.436 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:50:59.537 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:59.570 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:50:59.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:59.573 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:50:59.674 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:59.682 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:50:59 smithi078 bash[78274]: cluster 2023-10-01T10:50:57.941418+0000 mgr.y (mgr.25072) 1000 : cluster 0 pgmap v944: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:50:59.682 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:50:59 smithi078 bash[75299]: cluster 2023-10-01T10:50:57.941418+0000 mgr.y (mgr.25072) 1000 : cluster 0 pgmap v944: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:50:59.698 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:50:59.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:59.700 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:50:59.801 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:59.813 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:50:59 smithi173 bash[58816]: cluster 2023-10-01T10:50:57.941418+0000 mgr.y (mgr.25072) 1000 : cluster 0 pgmap v944: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:50:59.830 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:50:59.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:59.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:50:59.934 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:59.964 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:50:59.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:50:59.967 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:51:00.068 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:00.101 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:00.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:00.104 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:51:00.205 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:00.236 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:00.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:00.238 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:51:00.331 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:51:00 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:51:00] "GET /metrics HTTP/1.1" 200 33903 "" "Prometheus/2.33.4" 2023-10-01T10:51:00.338 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:00.371 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:00.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:00.374 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:51:00.475 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:00.503 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:00.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:00.515 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:51:00.616 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:00.648 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:00.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:00.652 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:51:00.753 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:00.777 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:00.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:00.780 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:51:00.880 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:00.907 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:00.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:00.910 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:51:00.953 INFO:teuthology.orchestra.run.smithi078.stderr:pool 'unique_pool_6' created 2023-10-01T10:51:01.011 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:01.031 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:01.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:01.033 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:51:01.134 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:01.147 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:00 smithi173 bash[58816]: cluster 2023-10-01T10:50:59.942185+0000 mgr.y (mgr.25072) 1001 : cluster 0 pgmap v945: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:51:01.147 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:00 smithi173 bash[58816]: audit 2023-10-01T10:51:00.439248+0000 mon.c (mon.2) 171 : audit 1 from='client.? 172.21.15.78:0/349679068' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]: dispatch 2023-10-01T10:51:01.147 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:00 smithi173 bash[58816]: audit 2023-10-01T10:51:00.439665+0000 mon.a (mon.0) 1232 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]: dispatch 2023-10-01T10:51:01.147 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:00 smithi173 bash[58816]: audit 2023-10-01T10:51:00.879860+0000 mon.a (mon.0) 1233 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:51:01.166 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:01.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:01.168 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:51:01.269 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:01.298 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:01.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:01.301 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:51:01.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:01 smithi078 bash[78274]: cluster 2023-10-01T10:50:59.942185+0000 mgr.y (mgr.25072) 1001 : cluster 0 pgmap v945: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:51:01.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:01 smithi078 bash[78274]: audit 2023-10-01T10:51:00.439248+0000 mon.c (mon.2) 171 : audit 1 from='client.? 172.21.15.78:0/349679068' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]: dispatch 2023-10-01T10:51:01.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:01 smithi078 bash[78274]: audit 2023-10-01T10:51:00.439665+0000 mon.a (mon.0) 1232 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]: dispatch 2023-10-01T10:51:01.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:01 smithi078 bash[78274]: audit 2023-10-01T10:51:00.879860+0000 mon.a (mon.0) 1233 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:51:01.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:01 smithi078 bash[75299]: cluster 2023-10-01T10:50:59.942185+0000 mgr.y (mgr.25072) 1001 : cluster 0 pgmap v945: 73 pgs: 73 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:51:01.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:01 smithi078 bash[75299]: audit 2023-10-01T10:51:00.439248+0000 mon.c (mon.2) 171 : audit 1 from='client.? 172.21.15.78:0/349679068' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]: dispatch 2023-10-01T10:51:01.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:01 smithi078 bash[75299]: audit 2023-10-01T10:51:00.439665+0000 mon.a (mon.0) 1232 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]: dispatch 2023-10-01T10:51:01.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:01 smithi078 bash[75299]: audit 2023-10-01T10:51:00.879860+0000 mon.a (mon.0) 1233 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:51:01.402 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:01.435 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:01.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:01.438 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:51:01.539 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:01.573 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:01.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:01.576 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:51:01.677 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:01.704 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:01.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:01.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:51:01.807 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:01.839 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:01.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:01.841 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:51:01.942 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:01.953 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:01 smithi078 bash[75299]: cluster 2023-10-01T10:51:00.937302+0000 osd.7 (osd.7) 124 : cluster 3 failed to encode map e173 with expected crc 2023-10-01T10:51:01.953 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:01 smithi078 bash[75299]: cluster 2023-10-01T10:51:00.939519+0000 osd.7 (osd.7) 125 : cluster 3 failed to encode map e173 with expected crc 2023-10-01T10:51:01.953 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:01 smithi078 bash[75299]: audit 2023-10-01T10:51:00.939945+0000 mon.a (mon.0) 1234 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]': finished 2023-10-01T10:51:01.953 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:01 smithi078 bash[75299]: cluster 2023-10-01T10:51:00.940097+0000 mon.a (mon.0) 1235 : cluster 0 osdmap e173: 8 total, 8 up, 8 in 2023-10-01T10:51:01.967 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:01.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:01.969 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:51:02.070 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:02.094 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:02.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:02.096 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:51:02.197 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:02.223 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:02.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:02.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:51:02.289 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:01 smithi078 bash[75299]: cluster 2023-10-01T10:51:00.941132+0000 osd.6 (osd.6) 90 : cluster 3 failed to encode map e173 with expected crc 2023-10-01T10:51:02.289 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:01 smithi078 bash[75299]: cluster 2023-10-01T10:51:00.942574+0000 osd.5 (osd.5) 104 : cluster 3 failed to encode map e173 with expected crc 2023-10-01T10:51:02.289 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:01 smithi078 bash[75299]: cluster 2023-10-01T10:51:00.943044+0000 osd.6 (osd.6) 91 : cluster 3 failed to encode map e173 with expected crc 2023-10-01T10:51:02.289 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:01 smithi078 bash[75299]: cluster 2023-10-01T10:51:00.944414+0000 osd.7 (osd.7) 126 : cluster 3 failed to encode map e173 with expected crc 2023-10-01T10:51:02.289 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:01 smithi078 bash[75299]: cluster 2023-10-01T10:51:01.937946+0000 mon.a (mon.0) 1236 : cluster 0 osdmap e174: 8 total, 8 up, 8 in 2023-10-01T10:51:02.290 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:01 smithi078 bash[78274]: cluster 2023-10-01T10:51:00.937302+0000 osd.7 (osd.7) 124 : cluster 3 failed to encode map e173 with expected crc 2023-10-01T10:51:02.290 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:01 smithi078 bash[78274]: cluster 2023-10-01T10:51:00.939519+0000 osd.7 (osd.7) 125 : cluster 3 failed to encode map e173 with expected crc 2023-10-01T10:51:02.290 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:01 smithi078 bash[78274]: audit 2023-10-01T10:51:00.939945+0000 mon.a (mon.0) 1234 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]': finished 2023-10-01T10:51:02.290 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:01 smithi078 bash[78274]: cluster 2023-10-01T10:51:00.940097+0000 mon.a (mon.0) 1235 : cluster 0 osdmap e173: 8 total, 8 up, 8 in 2023-10-01T10:51:02.290 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:01 smithi078 bash[78274]: cluster 2023-10-01T10:51:00.941132+0000 osd.6 (osd.6) 90 : cluster 3 failed to encode map e173 with expected crc 2023-10-01T10:51:02.290 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:01 smithi078 bash[78274]: cluster 2023-10-01T10:51:00.942574+0000 osd.5 (osd.5) 104 : cluster 3 failed to encode map e173 with expected crc 2023-10-01T10:51:02.290 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:01 smithi078 bash[78274]: cluster 2023-10-01T10:51:00.943044+0000 osd.6 (osd.6) 91 : cluster 3 failed to encode map e173 with expected crc 2023-10-01T10:51:02.290 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:01 smithi078 bash[78274]: cluster 2023-10-01T10:51:00.944414+0000 osd.7 (osd.7) 126 : cluster 3 failed to encode map e173 with expected crc 2023-10-01T10:51:02.290 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:01 smithi078 bash[78274]: cluster 2023-10-01T10:51:01.937946+0000 mon.a (mon.0) 1236 : cluster 0 osdmap e174: 8 total, 8 up, 8 in 2023-10-01T10:51:02.327 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:02.354 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:02.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:02.356 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:51:02.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:01 smithi173 bash[58816]: cluster 2023-10-01T10:51:00.937302+0000 osd.7 (osd.7) 124 : cluster 3 failed to encode map e173 with expected crc 2023-10-01T10:51:02.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:01 smithi173 bash[58816]: cluster 2023-10-01T10:51:00.939519+0000 osd.7 (osd.7) 125 : cluster 3 failed to encode map e173 with expected crc 2023-10-01T10:51:02.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:01 smithi173 bash[58816]: audit 2023-10-01T10:51:00.939945+0000 mon.a (mon.0) 1234 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]': finished 2023-10-01T10:51:02.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:01 smithi173 bash[58816]: cluster 2023-10-01T10:51:00.940097+0000 mon.a (mon.0) 1235 : cluster 0 osdmap e173: 8 total, 8 up, 8 in 2023-10-01T10:51:02.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:01 smithi173 bash[58816]: cluster 2023-10-01T10:51:00.941132+0000 osd.6 (osd.6) 90 : cluster 3 failed to encode map e173 with expected crc 2023-10-01T10:51:02.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:01 smithi173 bash[58816]: cluster 2023-10-01T10:51:00.942574+0000 osd.5 (osd.5) 104 : cluster 3 failed to encode map e173 with expected crc 2023-10-01T10:51:02.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:01 smithi173 bash[58816]: cluster 2023-10-01T10:51:00.943044+0000 osd.6 (osd.6) 91 : cluster 3 failed to encode map e173 with expected crc 2023-10-01T10:51:02.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:01 smithi173 bash[58816]: cluster 2023-10-01T10:51:00.944414+0000 osd.7 (osd.7) 126 : cluster 3 failed to encode map e173 with expected crc 2023-10-01T10:51:02.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:01 smithi173 bash[58816]: cluster 2023-10-01T10:51:01.937946+0000 mon.a (mon.0) 1236 : cluster 0 osdmap e174: 8 total, 8 up, 8 in 2023-10-01T10:51:02.457 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:02.487 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:02.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:02.491 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:51:02.591 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:02.620 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:02.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:02.623 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:51:02.723 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:02.748 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:02.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:02.751 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:51:02.853 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:02.880 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:02.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:02.883 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:51:02.984 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:03.010 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:03.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:03.013 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:51:03.113 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:03.146 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:03.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:03.149 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:51:03.250 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:03.259 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:03 smithi078 bash[75299]: cluster 2023-10-01T10:51:01.936535+0000 osd.5 (osd.5) 105 : cluster 3 failed to encode map e174 with expected crc 2023-10-01T10:51:03.259 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:03 smithi078 bash[75299]: cluster 2023-10-01T10:51:01.940356+0000 osd.6 (osd.6) 92 : cluster 3 failed to encode map e174 with expected crc 2023-10-01T10:51:03.260 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:03 smithi078 bash[75299]: cluster 2023-10-01T10:51:01.940953+0000 osd.5 (osd.5) 106 : cluster 3 failed to encode map e174 with expected crc 2023-10-01T10:51:03.260 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:03 smithi078 bash[75299]: cluster 2023-10-01T10:51:01.941830+0000 osd.7 (osd.7) 127 : cluster 3 failed to encode map e174 with expected crc 2023-10-01T10:51:03.260 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:03 smithi078 bash[75299]: cluster 2023-10-01T10:51:01.942590+0000 mgr.y (mgr.25072) 1002 : cluster 0 pgmap v948: 89 pgs: 7 creating+peering, 9 unknown, 73 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:51:03.260 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:03 smithi078 bash[75299]: audit 2023-10-01T10:51:02.432333+0000 mon.a (mon.0) 1237 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:51:03.260 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:03 smithi078 bash[75299]: cluster 2023-10-01T10:51:02.945395+0000 mon.a (mon.0) 1238 : cluster 0 osdmap e175: 8 total, 8 up, 8 in 2023-10-01T10:51:03.260 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:03 smithi078 bash[78274]: cluster 2023-10-01T10:51:01.936535+0000 osd.5 (osd.5) 105 : cluster 3 failed to encode map e174 with expected crc 2023-10-01T10:51:03.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:03 smithi078 bash[78274]: cluster 2023-10-01T10:51:01.940356+0000 osd.6 (osd.6) 92 : cluster 3 failed to encode map e174 with expected crc 2023-10-01T10:51:03.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:03 smithi078 bash[78274]: cluster 2023-10-01T10:51:01.940953+0000 osd.5 (osd.5) 106 : cluster 3 failed to encode map e174 with expected crc 2023-10-01T10:51:03.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:03 smithi078 bash[78274]: cluster 2023-10-01T10:51:01.941830+0000 osd.7 (osd.7) 127 : cluster 3 failed to encode map e174 with expected crc 2023-10-01T10:51:03.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:03 smithi078 bash[78274]: cluster 2023-10-01T10:51:01.942590+0000 mgr.y (mgr.25072) 1002 : cluster 0 pgmap v948: 89 pgs: 7 creating+peering, 9 unknown, 73 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:51:03.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:03 smithi078 bash[78274]: audit 2023-10-01T10:51:02.432333+0000 mon.a (mon.0) 1237 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:51:03.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:03 smithi078 bash[78274]: cluster 2023-10-01T10:51:02.945395+0000 mon.a (mon.0) 1238 : cluster 0 osdmap e175: 8 total, 8 up, 8 in 2023-10-01T10:51:03.277 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:03.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:03.279 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:51:03.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:02 smithi173 bash[58816]: cluster 2023-10-01T10:51:01.936535+0000 osd.5 (osd.5) 105 : cluster 3 failed to encode map e174 with expected crc 2023-10-01T10:51:03.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:02 smithi173 bash[58816]: cluster 2023-10-01T10:51:01.940356+0000 osd.6 (osd.6) 92 : cluster 3 failed to encode map e174 with expected crc 2023-10-01T10:51:03.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:02 smithi173 bash[58816]: cluster 2023-10-01T10:51:01.940953+0000 osd.5 (osd.5) 106 : cluster 3 failed to encode map e174 with expected crc 2023-10-01T10:51:03.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:02 smithi173 bash[58816]: cluster 2023-10-01T10:51:01.941830+0000 osd.7 (osd.7) 127 : cluster 3 failed to encode map e174 with expected crc 2023-10-01T10:51:03.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:02 smithi173 bash[58816]: cluster 2023-10-01T10:51:01.942590+0000 mgr.y (mgr.25072) 1002 : cluster 0 pgmap v948: 89 pgs: 7 creating+peering, 9 unknown, 73 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:51:03.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:02 smithi173 bash[58816]: audit 2023-10-01T10:51:02.432333+0000 mon.a (mon.0) 1237 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:51:03.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:02 smithi173 bash[58816]: cluster 2023-10-01T10:51:02.945395+0000 mon.a (mon.0) 1238 : cluster 0 osdmap e175: 8 total, 8 up, 8 in 2023-10-01T10:51:03.379 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:03.409 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:03.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:03.412 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:51:03.513 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:03.543 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:03.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:03.546 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:51:03.647 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:03.679 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:03.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:03.682 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:51:03.783 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:03.809 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:03.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:03.812 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:51:03.913 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:03.936 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_6 rados --yes-i-really-mean-it || true 2023-10-01T10:51:03.938 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:03.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:03.940 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:51:04.041 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:04.050 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:03 smithi078 bash[75299]: cluster 2023-10-01T10:51:02.943717+0000 osd.6 (osd.6) 93 : cluster 3 failed to encode map e175 with expected crc 2023-10-01T10:51:04.050 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:03 smithi078 bash[75299]: cluster 2023-10-01T10:51:03.058023+0000 osd.5 (osd.5) 107 : cluster 3 failed to encode map e175 with expected crc 2023-10-01T10:51:04.051 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:03 smithi078 bash[75299]: cluster 2023-10-01T10:51:03.060863+0000 osd.5 (osd.5) 108 : cluster 3 failed to encode map e175 with expected crc 2023-10-01T10:51:04.051 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:03 smithi078 bash[75299]: cluster 2023-10-01T10:51:03.063712+0000 osd.5 (osd.5) 109 : cluster 3 failed to encode map e175 with expected crc 2023-10-01T10:51:04.051 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:03 smithi078 bash[75299]: cluster 2023-10-01T10:51:03.134177+0000 osd.7 (osd.7) 128 : cluster 3 failed to encode map e175 with expected crc 2023-10-01T10:51:04.051 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:03 smithi078 bash[78274]: cluster 2023-10-01T10:51:02.943717+0000 osd.6 (osd.6) 93 : cluster 3 failed to encode map e175 with expected crc 2023-10-01T10:51:04.051 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:03 smithi078 bash[78274]: cluster 2023-10-01T10:51:03.058023+0000 osd.5 (osd.5) 107 : cluster 3 failed to encode map e175 with expected crc 2023-10-01T10:51:04.051 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:03 smithi078 bash[78274]: cluster 2023-10-01T10:51:03.060863+0000 osd.5 (osd.5) 108 : cluster 3 failed to encode map e175 with expected crc 2023-10-01T10:51:04.051 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:03 smithi078 bash[78274]: cluster 2023-10-01T10:51:03.063712+0000 osd.5 (osd.5) 109 : cluster 3 failed to encode map e175 with expected crc 2023-10-01T10:51:04.051 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:03 smithi078 bash[78274]: cluster 2023-10-01T10:51:03.134177+0000 osd.7 (osd.7) 128 : cluster 3 failed to encode map e175 with expected crc 2023-10-01T10:51:04.066 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:04.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:04.068 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:51:04.157 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T10:51:04.169 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:04.177 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T10:51:04.196 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T10:51:04.202 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:04.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:04.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:51:04.219 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T10:51:04.242 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T10:51:04.266 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T10:51:04.292 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T10:51:04.305 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:04.322 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T10:51:04.324 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T10:51:04.325 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T10:51:04.325 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T10:51:04.325 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T10:51:04.325 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T10:51:04.325 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T10:51:04.325 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T10:51:04.325 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T10:51:04.331 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:04.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:04.334 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:51:04.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:03 smithi173 bash[58816]: cluster 2023-10-01T10:51:02.943717+0000 osd.6 (osd.6) 93 : cluster 3 failed to encode map e175 with expected crc 2023-10-01T10:51:04.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:03 smithi173 bash[58816]: cluster 2023-10-01T10:51:03.058023+0000 osd.5 (osd.5) 107 : cluster 3 failed to encode map e175 with expected crc 2023-10-01T10:51:04.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:03 smithi173 bash[58816]: cluster 2023-10-01T10:51:03.060863+0000 osd.5 (osd.5) 108 : cluster 3 failed to encode map e175 with expected crc 2023-10-01T10:51:04.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:03 smithi173 bash[58816]: cluster 2023-10-01T10:51:03.063712+0000 osd.5 (osd.5) 109 : cluster 3 failed to encode map e175 with expected crc 2023-10-01T10:51:04.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:03 smithi173 bash[58816]: cluster 2023-10-01T10:51:03.134177+0000 osd.7 (osd.7) 128 : cluster 3 failed to encode map e175 with expected crc 2023-10-01T10:51:04.434 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:04.468 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:04.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:04.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:51:04.572 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:04.602 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:04.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:04.604 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:51:04.671 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2023-10-01T10:51:04.671 INFO:teuthology.orchestra.run.smithi078.stdout: "size": 20, 2023-10-01T10:51:04.672 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 600, 2023-10-01T10:51:04.672 INFO:teuthology.orchestra.run.smithi078.stdout: "ops": [ 2023-10-01T10:51:04.672 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.672 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.35649.0:75435 9.c 9:35a3dedc:::benchmark_data_smithi078_113121_object75434:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e132)", 2023-10-01T10:51:04.672 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:41:57.427643+0000", 2023-10-01T10:51:04.672 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 547.24045691499998, 2023-10-01T10:51:04.672 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.20691067399999999, 2023-10-01T10:51:04.672 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:51:04.673 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:51:04.673 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:51:04.673 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.35649", 2023-10-01T10:51:04.673 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:51:04.673 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 75435 2023-10-01T10:51:04.673 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.673 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:51:04.673 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.673 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:51:04.673 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.427643+0000", 2023-10-01T10:51:04.674 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:51:04.674 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.674 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.674 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:51:04.674 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.427643+0000", 2023-10-01T10:51:04.674 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:51:04.674 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.674 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.674 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:51:04.675 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.427642+0000", 2023-10-01T10:51:04.675 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999986 2023-10-01T10:51:04.675 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.675 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.675 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:51:04.675 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.427665+0000", 2023-10-01T10:51:04.675 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.3399000000000001e-05 2023-10-01T10:51:04.675 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.675 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.675 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:51:04.675 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.427677+0000", 2023-10-01T10:51:04.676 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1813e-05 2023-10-01T10:51:04.676 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.676 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.676 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:51:04.676 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.427685+0000", 2023-10-01T10:51:04.676 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.7759999999999997e-06 2023-10-01T10:51:04.676 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.676 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.676 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:51:04.676 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.427721+0000", 2023-10-01T10:51:04.676 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.6562e-05 2023-10-01T10:51:04.676 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.677 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.677 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:51:04.677 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.427806+0000", 2023-10-01T10:51:04.677 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.5396999999999996e-05 2023-10-01T10:51:04.677 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.677 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.677 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 4,7", 2023-10-01T10:51:04.677 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.427892+0000", 2023-10-01T10:51:04.677 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.5191000000000005e-05 2023-10-01T10:51:04.677 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.677 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.678 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:51:04.678 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.430024+0000", 2023-10-01T10:51:04.678 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.002132802 2023-10-01T10:51:04.678 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.678 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.678 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:51:04.678 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.633364+0000", 2023-10-01T10:51:04.678 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.20333915699999999 2023-10-01T10:51:04.678 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.678 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.678 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:51:04.679 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.633825+0000", 2023-10-01T10:51:04.679 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00046099999999999998 2023-10-01T10:51:04.679 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.679 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.679 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:51:04.679 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.634505+0000", 2023-10-01T10:51:04.679 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00068042099999999996 2023-10-01T10:51:04.679 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.679 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.679 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:51:04.679 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.634553+0000", 2023-10-01T10:51:04.680 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.8429e-05 2023-10-01T10:51:04.680 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:51:04.680 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:51:04.680 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:51:04.680 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.680 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.680 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.35649.0:75440 9.c 9:363a79ad:::benchmark_data_smithi078_113121_object75439:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e132)", 2023-10-01T10:51:04.680 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:41:57.429993+0000", 2023-10-01T10:51:04.680 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 547.23810641800003, 2023-10-01T10:51:04.681 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.204878119, 2023-10-01T10:51:04.681 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:51:04.681 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:51:04.681 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:51:04.681 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.35649", 2023-10-01T10:51:04.681 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:51:04.681 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 75440 2023-10-01T10:51:04.681 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.681 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:51:04.682 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.682 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:51:04.682 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.429993+0000", 2023-10-01T10:51:04.682 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:51:04.682 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.682 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.682 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:51:04.682 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.429993+0000", 2023-10-01T10:51:04.682 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:51:04.682 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.682 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.683 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:51:04.683 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.429992+0000", 2023-10-01T10:51:04.683 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:51:04.683 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.683 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.683 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:51:04.683 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.430013+0000", 2023-10-01T10:51:04.683 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.0625999999999999e-05 2023-10-01T10:51:04.683 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.683 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.683 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:51:04.683 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.430023+0000", 2023-10-01T10:51:04.684 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.0319999999999999e-05 2023-10-01T10:51:04.684 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.684 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.684 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:51:04.684 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.430030+0000", 2023-10-01T10:51:04.684 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.1380000000000001e-06 2023-10-01T10:51:04.684 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.684 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.684 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:51:04.684 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.430067+0000", 2023-10-01T10:51:04.684 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.6430000000000002e-05 2023-10-01T10:51:04.685 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.685 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.685 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:51:04.685 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.430158+0000", 2023-10-01T10:51:04.685 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.0628e-05 2023-10-01T10:51:04.685 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.685 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.685 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 4,7", 2023-10-01T10:51:04.685 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.430297+0000", 2023-10-01T10:51:04.685 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000139777 2023-10-01T10:51:04.685 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.685 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.686 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:51:04.686 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.432629+0000", 2023-10-01T10:51:04.686 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0023313280000000001 2023-10-01T10:51:04.686 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.686 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.686 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:51:04.686 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.634616+0000", 2023-10-01T10:51:04.686 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.20198776700000001 2023-10-01T10:51:04.686 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.686 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.686 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:51:04.687 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.634648+0000", 2023-10-01T10:51:04.687 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.1571e-05 2023-10-01T10:51:04.687 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.687 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.687 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:51:04.687 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.634828+0000", 2023-10-01T10:51:04.687 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000180187 2023-10-01T10:51:04.687 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.687 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.687 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:51:04.687 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.634871+0000", 2023-10-01T10:51:04.687 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.3279999999999999e-05 2023-10-01T10:51:04.688 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:51:04.688 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:51:04.688 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:51:04.688 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.688 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.688 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.35649.0:75442 9.1 9:818b3f9a:::benchmark_data_smithi078_113121_object75441:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e132)", 2023-10-01T10:51:04.690 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:41:57.430836+0000", 2023-10-01T10:51:04.691 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 547.23726328500004, 2023-10-01T10:51:04.691 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.21003887700000001, 2023-10-01T10:51:04.691 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:51:04.691 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:51:04.691 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:51:04.691 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.35649", 2023-10-01T10:51:04.691 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:51:04.691 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 75442 2023-10-01T10:51:04.691 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.691 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:51:04.691 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.692 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:51:04.692 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.430836+0000", 2023-10-01T10:51:04.692 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:51:04.692 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.692 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.692 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:51:04.692 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.430836+0000", 2023-10-01T10:51:04.692 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:51:04.692 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.692 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.692 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:51:04.693 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.430835+0000", 2023-10-01T10:51:04.693 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:51:04.693 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.693 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.693 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:51:04.693 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.430859+0000", 2023-10-01T10:51:04.693 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.3326e-05 2023-10-01T10:51:04.693 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.693 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.693 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:51:04.693 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.430868+0000", 2023-10-01T10:51:04.694 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.6269999999999995e-06 2023-10-01T10:51:04.694 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.694 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.694 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:51:04.694 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.430874+0000", 2023-10-01T10:51:04.694 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.3990000000000003e-06 2023-10-01T10:51:04.694 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.694 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.694 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:51:04.694 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.430920+0000", 2023-10-01T10:51:04.694 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.6486000000000002e-05 2023-10-01T10:51:04.694 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.695 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.695 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:51:04.695 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.431034+0000", 2023-10-01T10:51:04.695 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000113694 2023-10-01T10:51:04.695 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.695 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.695 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 6,7", 2023-10-01T10:51:04.695 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.431156+0000", 2023-10-01T10:51:04.695 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000121891 2023-10-01T10:51:04.695 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.695 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.696 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:51:04.696 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.624620+0000", 2023-10-01T10:51:04.696 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.19346456400000001 2023-10-01T10:51:04.696 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.696 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.696 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:51:04.696 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.624752+0000", 2023-10-01T10:51:04.696 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00013177100000000001 2023-10-01T10:51:04.696 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.696 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.696 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:51:04.696 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.640802+0000", 2023-10-01T10:51:04.697 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.016049752 2023-10-01T10:51:04.697 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.697 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.697 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:51:04.697 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.640839+0000", 2023-10-01T10:51:04.697 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.7577999999999998e-05 2023-10-01T10:51:04.697 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.697 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.697 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:51:04.697 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.640875+0000", 2023-10-01T10:51:04.697 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.5861000000000001e-05 2023-10-01T10:51:04.698 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:51:04.698 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:51:04.698 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:51:04.698 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.698 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.698 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.35649.0:75436 9.3 e132/128 9:c2f783d2:::benchmark_data_smithi078_113121_object75435:head v 132'4665, mlcod=132'4663)", 2023-10-01T10:51:04.698 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:41:57.430995+0000", 2023-10-01T10:51:04.698 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 547.23710450399994, 2023-10-01T10:51:04.698 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.194232663, 2023-10-01T10:51:04.698 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:51:04.698 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:51:04.699 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:51:04.699 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.699 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:51:04.699 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.430995+0000", 2023-10-01T10:51:04.699 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:51:04.699 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.699 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.699 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:51:04.699 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.430995+0000", 2023-10-01T10:51:04.699 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:51:04.699 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.699 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.700 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:51:04.700 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.430995+0000", 2023-10-01T10:51:04.700 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999995 2023-10-01T10:51:04.700 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.700 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.700 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:51:04.700 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.431077+0000", 2023-10-01T10:51:04.700 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.1965999999999994e-05 2023-10-01T10:51:04.700 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.700 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.700 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:51:04.700 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.431086+0000", 2023-10-01T10:51:04.701 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.9429999999999995e-06 2023-10-01T10:51:04.701 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.701 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.701 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:51:04.701 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.431093+0000", 2023-10-01T10:51:04.701 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.4740000000000002e-06 2023-10-01T10:51:04.701 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.701 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.701 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:51:04.701 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.431140+0000", 2023-10-01T10:51:04.701 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.6739000000000001e-05 2023-10-01T10:51:04.701 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.702 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.702 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:51:04.702 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.431155+0000", 2023-10-01T10:51:04.702 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.5498e-05 2023-10-01T10:51:04.702 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.702 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.702 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:51:04.702 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.435610+0000", 2023-10-01T10:51:04.702 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0044546689999999996 2023-10-01T10:51:04.702 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.702 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.703 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:51:04.703 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.625228+0000", 2023-10-01T10:51:04.703 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.18961800300000001 2023-10-01T10:51:04.703 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:51:04.703 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:51:04.703 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:51:04.703 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.703 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.703 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.35649.0:75437 9.7 e132/128 9:eee78d3d:::benchmark_data_smithi078_113121_object75436:head v 132'4767, mlcod=132'4765)", 2023-10-01T10:51:04.703 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:41:57.432106+0000", 2023-10-01T10:51:04.703 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 547.23599386700005, 2023-10-01T10:51:04.704 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.19307002300000001, 2023-10-01T10:51:04.704 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:51:04.704 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:51:04.704 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:51:04.704 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.704 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:51:04.704 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.432106+0000", 2023-10-01T10:51:04.704 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:51:04.704 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.704 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.704 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:51:04.704 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.432106+0000", 2023-10-01T10:51:04.705 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:51:04.705 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.705 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.705 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:51:04.705 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.432105+0000", 2023-10-01T10:51:04.705 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:51:04.705 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.705 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.705 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:51:04.705 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.432169+0000", 2023-10-01T10:51:04.705 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.3794000000000006e-05 2023-10-01T10:51:04.706 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.706 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.706 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:51:04.706 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.432181+0000", 2023-10-01T10:51:04.706 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.2415000000000001e-05 2023-10-01T10:51:04.706 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.706 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.706 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:51:04.706 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.432189+0000", 2023-10-01T10:51:04.706 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.7249999999999997e-06 2023-10-01T10:51:04.706 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.707 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.707 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:51:04.707 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.432237+0000", 2023-10-01T10:51:04.707 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.8260000000000002e-05 2023-10-01T10:51:04.707 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.707 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.707 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:51:04.707 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.432261+0000", 2023-10-01T10:51:04.707 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.3819e-05 2023-10-01T10:51:04.707 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.707 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.707 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:51:04.708 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.625119+0000", 2023-10-01T10:51:04.708 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.192858376 2023-10-01T10:51:04.708 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.708 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.708 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:51:04.708 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.625176+0000", 2023-10-01T10:51:04.708 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.6455000000000001e-05 2023-10-01T10:51:04.708 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:51:04.708 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:51:04.708 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:51:04.708 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.709 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.709 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.35649.0:75445 9.c 9:3c7722e5:::benchmark_data_smithi078_113121_object75444:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e132)", 2023-10-01T10:51:04.709 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:41:57.433553+0000", 2023-10-01T10:51:04.709 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 547.234546249, 2023-10-01T10:51:04.709 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.20628380700000001, 2023-10-01T10:51:04.709 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:51:04.709 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:51:04.709 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:51:04.709 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.35649", 2023-10-01T10:51:04.710 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:51:04.710 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 75445 2023-10-01T10:51:04.710 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.710 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:51:04.710 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.710 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:51:04.710 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.433553+0000", 2023-10-01T10:51:04.710 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:51:04.710 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.710 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.710 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:51:04.711 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.433553+0000", 2023-10-01T10:51:04.711 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:51:04.711 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.711 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.711 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:51:04.711 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.433552+0000", 2023-10-01T10:51:04.711 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999986 2023-10-01T10:51:04.711 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.711 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.711 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:51:04.711 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.433606+0000", 2023-10-01T10:51:04.712 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.4082999999999997e-05 2023-10-01T10:51:04.712 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.712 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.712 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:51:04.712 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.433632+0000", 2023-10-01T10:51:04.712 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.5792e-05 2023-10-01T10:51:04.712 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.712 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.712 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:51:04.712 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.433641+0000", 2023-10-01T10:51:04.712 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.2299999999999997e-06 2023-10-01T10:51:04.713 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.713 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.713 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:51:04.713 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.433752+0000", 2023-10-01T10:51:04.713 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00011108299999999999 2023-10-01T10:51:04.713 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.713 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.713 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:51:04.713 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.433900+0000", 2023-10-01T10:51:04.713 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000147823 2023-10-01T10:51:04.713 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.713 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.714 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 4,7", 2023-10-01T10:51:04.714 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.624952+0000", 2023-10-01T10:51:04.714 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.19105241100000001 2023-10-01T10:51:04.714 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.714 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.714 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:51:04.714 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.633606+0000", 2023-10-01T10:51:04.714 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0086534890000000003 2023-10-01T10:51:04.714 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.714 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.714 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:51:04.715 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.634928+0000", 2023-10-01T10:51:04.715 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0013220669999999999 2023-10-01T10:51:04.715 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.715 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.715 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:51:04.715 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.639652+0000", 2023-10-01T10:51:04.715 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0047245330000000004 2023-10-01T10:51:04.715 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.715 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.715 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:51:04.715 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.639798+0000", 2023-10-01T10:51:04.715 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000145532 2023-10-01T10:51:04.716 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.716 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.716 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:51:04.716 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.639837+0000", 2023-10-01T10:51:04.716 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.9390000000000001e-05 2023-10-01T10:51:04.716 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:51:04.716 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:51:04.716 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:51:04.716 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.716 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.716 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.35649.0:75446 9.c 9:3afa93b1:::benchmark_data_smithi078_113121_object75445:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e132)", 2023-10-01T10:51:04.717 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:41:57.433659+0000", 2023-10-01T10:51:04.717 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 547.23444035099999, 2023-10-01T10:51:04.717 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.207073217, 2023-10-01T10:51:04.717 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:51:04.717 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:51:04.717 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:51:04.717 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.35649", 2023-10-01T10:51:04.717 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:51:04.717 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 75446 2023-10-01T10:51:04.717 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.717 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:51:04.717 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.718 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:51:04.718 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.433659+0000", 2023-10-01T10:51:04.718 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:51:04.718 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.718 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.718 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:51:04.722 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.433659+0000", 2023-10-01T10:51:04.722 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:51:04.723 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.723 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.723 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:51:04.723 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.433659+0000", 2023-10-01T10:51:04.723 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999995 2023-10-01T10:51:04.724 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.724 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.724 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:51:04.724 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.433679+0000", 2023-10-01T10:51:04.724 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.0443999999999999e-05 2023-10-01T10:51:04.724 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.725 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.725 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:51:04.725 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.433702+0000", 2023-10-01T10:51:04.725 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.3264e-05 2023-10-01T10:51:04.725 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.725 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.726 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:51:04.726 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.433710+0000", 2023-10-01T10:51:04.726 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.4139999999999997e-06 2023-10-01T10:51:04.726 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.727 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.727 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:51:04.727 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.628458+0000", 2023-10-01T10:51:04.727 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.19474844999999999 2023-10-01T10:51:04.727 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.727 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.728 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:51:04.728 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.628657+0000", 2023-10-01T10:51:04.728 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000198507 2023-10-01T10:51:04.728 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.728 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.728 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 4,7", 2023-10-01T10:51:04.728 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.628932+0000", 2023-10-01T10:51:04.728 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000275684 2023-10-01T10:51:04.728 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.728 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.728 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:51:04.729 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.639589+0000", 2023-10-01T10:51:04.729 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.010656677999999999 2023-10-01T10:51:04.729 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.729 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.729 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:51:04.729 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.639874+0000", 2023-10-01T10:51:04.729 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00028527499999999998 2023-10-01T10:51:04.729 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.729 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.729 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:51:04.729 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.639900+0000", 2023-10-01T10:51:04.729 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.5545e-05 2023-10-01T10:51:04.730 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.730 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.730 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:51:04.730 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.640691+0000", 2023-10-01T10:51:04.730 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00079071899999999995 2023-10-01T10:51:04.730 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.730 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.730 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:51:04.730 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.640733+0000", 2023-10-01T10:51:04.730 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.1896e-05 2023-10-01T10:51:04.730 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:51:04.731 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:51:04.731 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:51:04.731 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.731 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.731 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop_reply(client.35649.0:75442 9.1 e132/128 ondisk, result = 0)", 2023-10-01T10:51:04.731 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:41:57.434538+0000", 2023-10-01T10:51:04.731 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 547.23356188399998, 2023-10-01T10:51:04.731 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.19021684599999999, 2023-10-01T10:51:04.731 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:51:04.731 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "started", 2023-10-01T10:51:04.731 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:51:04.732 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.732 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:51:04.732 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.434538+0000", 2023-10-01T10:51:04.732 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:51:04.732 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.732 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.732 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:51:04.732 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.434538+0000", 2023-10-01T10:51:04.732 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:51:04.732 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.732 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.733 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:51:04.733 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.434537+0000", 2023-10-01T10:51:04.733 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:51:04.733 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.733 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.733 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:51:04.733 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.434546+0000", 2023-10-01T10:51:04.733 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.149e-06 2023-10-01T10:51:04.733 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.733 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.733 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:51:04.734 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.624667+0000", 2023-10-01T10:51:04.734 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.19012103399999999 2023-10-01T10:51:04.734 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.734 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.734 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:51:04.734 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.624690+0000", 2023-10-01T10:51:04.734 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.3223000000000001e-05 2023-10-01T10:51:04.734 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.735 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.735 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:51:04.735 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.624740+0000", 2023-10-01T10:51:04.735 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.9660999999999998e-05 2023-10-01T10:51:04.735 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.736 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.736 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:51:04.736 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.624748+0000", 2023-10-01T10:51:04.736 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.6240000000000002e-06 2023-10-01T10:51:04.737 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.737 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.737 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:51:04.737 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.624755+0000", 2023-10-01T10:51:04.737 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.0489999999999998e-06 2023-10-01T10:51:04.738 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:51:04.738 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:51:04.738 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:51:04.738 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.738 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.738 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop_reply(client.35649.0:75435 9.c e132/128 ondisk, result = 0)", 2023-10-01T10:51:04.738 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:41:57.436699+0000", 2023-10-01T10:51:04.738 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 547.23140109799999, 2023-10-01T10:51:04.738 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.19666953300000001, 2023-10-01T10:51:04.739 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:51:04.739 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "started", 2023-10-01T10:51:04.739 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:51:04.739 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.739 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:51:04.739 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.436699+0000", 2023-10-01T10:51:04.739 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:51:04.739 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.739 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.739 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:51:04.739 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.436699+0000", 2023-10-01T10:51:04.740 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:51:04.740 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.740 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.740 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:51:04.740 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.436698+0000", 2023-10-01T10:51:04.740 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999995 2023-10-01T10:51:04.740 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.740 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.740 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:51:04.740 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.436707+0000", 2023-10-01T10:51:04.740 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.2620000000000004e-06 2023-10-01T10:51:04.741 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.741 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.741 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:51:04.741 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.624621+0000", 2023-10-01T10:51:04.741 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.18791355200000001 2023-10-01T10:51:04.741 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.741 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.741 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:51:04.741 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.624793+0000", 2023-10-01T10:51:04.741 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000172607 2023-10-01T10:51:04.741 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.741 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.742 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:51:04.742 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.633345+0000", 2023-10-01T10:51:04.742 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0085519399999999992 2023-10-01T10:51:04.742 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.742 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.742 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:51:04.742 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.633355+0000", 2023-10-01T10:51:04.742 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.9320000000000008e-06 2023-10-01T10:51:04.742 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.742 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.742 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:51:04.742 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.633368+0000", 2023-10-01T10:51:04.743 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.2901999999999999e-05 2023-10-01T10:51:04.743 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:51:04.743 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:51:04.743 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:51:04.743 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.743 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.743 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.35649.0:75453 9.c 9:34e2c81b:::benchmark_data_smithi078_113121_object75452:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e132)", 2023-10-01T10:51:04.743 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:41:57.445798+0000", 2023-10-01T10:51:04.743 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 547.22230156800003, 2023-10-01T10:51:04.743 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.20244446599999999, 2023-10-01T10:51:04.743 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:51:04.744 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:51:04.744 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:51:04.744 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.35649", 2023-10-01T10:51:04.744 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:51:04.744 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 75453 2023-10-01T10:51:04.744 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.744 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:51:04.744 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.744 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:51:04.744 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.445798+0000", 2023-10-01T10:51:04.744 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:51:04.744 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.745 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.745 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:51:04.745 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.445798+0000", 2023-10-01T10:51:04.745 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:51:04.745 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.745 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.745 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:51:04.745 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.445796+0000", 2023-10-01T10:51:04.745 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999976 2023-10-01T10:51:04.745 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.745 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.746 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:51:04.746 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.445897+0000", 2023-10-01T10:51:04.746 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000100864 2023-10-01T10:51:04.746 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.746 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.746 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:51:04.746 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.624650+0000", 2023-10-01T10:51:04.746 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.178753207 2023-10-01T10:51:04.746 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.746 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.746 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:51:04.746 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.624664+0000", 2023-10-01T10:51:04.747 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.3658e-05 2023-10-01T10:51:04.747 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.747 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.747 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:51:04.747 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.635264+0000", 2023-10-01T10:51:04.747 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.010600547 2023-10-01T10:51:04.747 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.747 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.747 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:51:04.747 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.635409+0000", 2023-10-01T10:51:04.747 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00014442599999999999 2023-10-01T10:51:04.747 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.748 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.748 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 4,7", 2023-10-01T10:51:04.748 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.635555+0000", 2023-10-01T10:51:04.748 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000146592 2023-10-01T10:51:04.748 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.748 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.748 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:51:04.748 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.640775+0000", 2023-10-01T10:51:04.748 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0052196389999999999 2023-10-01T10:51:04.748 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.748 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.749 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:51:04.749 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.640783+0000", 2023-10-01T10:51:04.749 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.1380000000000007e-06 2023-10-01T10:51:04.749 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.749 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.749 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:51:04.749 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.648149+0000", 2023-10-01T10:51:04.749 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0073663519999999996 2023-10-01T10:51:04.749 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.749 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.749 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:51:04.749 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.648199+0000", 2023-10-01T10:51:04.750 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.9561999999999997e-05 2023-10-01T10:51:04.750 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.750 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.750 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:51:04.750 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:41:57.648243+0000", 2023-10-01T10:51:04.750 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.3642e-05 2023-10-01T10:51:04.750 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:51:04.750 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:51:04.750 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:51:04.750 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.750 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.750 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.35649.0:158065 9.2 9:4fdf2531:::benchmark_data_smithi078_113121_object46250:head [delete] snapc 0=[] ondisk+write+known_if_redirected e137)", 2023-10-01T10:51:04.751 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:43:39.400238+0000", 2023-10-01T10:51:04.751 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 445.267862116, 2023-10-01T10:51:04.751 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.224930456, 2023-10-01T10:51:04.751 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:51:04.751 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:51:04.751 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:51:04.751 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.35649", 2023-10-01T10:51:04.751 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:51:04.751 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 158065 2023-10-01T10:51:04.751 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.751 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:51:04.752 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.752 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:51:04.752 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.400238+0000", 2023-10-01T10:51:04.752 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:51:04.752 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.752 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.752 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:51:04.752 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.400238+0000", 2023-10-01T10:51:04.752 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:51:04.752 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.752 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.753 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:51:04.753 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.400237+0000", 2023-10-01T10:51:04.753 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:51:04.753 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.753 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.753 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:51:04.753 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.400242+0000", 2023-10-01T10:51:04.753 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.8790000000000002e-06 2023-10-01T10:51:04.753 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.754 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.754 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:51:04.754 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.400251+0000", 2023-10-01T10:51:04.754 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.8529999999999998e-06 2023-10-01T10:51:04.754 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.754 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.754 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:51:04.754 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.400258+0000", 2023-10-01T10:51:04.754 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.3699999999999997e-06 2023-10-01T10:51:04.754 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.754 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.755 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:51:04.755 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.406170+0000", 2023-10-01T10:51:04.755 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0059117900000000001 2023-10-01T10:51:04.755 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.755 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.755 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:51:04.755 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.406348+0000", 2023-10-01T10:51:04.755 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000177851 2023-10-01T10:51:04.755 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.755 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.755 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 1,2", 2023-10-01T10:51:04.755 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.406506+0000", 2023-10-01T10:51:04.756 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000158252 2023-10-01T10:51:04.756 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.756 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.756 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:51:04.756 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.409929+0000", 2023-10-01T10:51:04.756 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.003422876 2023-10-01T10:51:04.756 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.756 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.756 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:51:04.756 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.412218+0000", 2023-10-01T10:51:04.756 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0022896090000000002 2023-10-01T10:51:04.756 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.757 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.757 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:51:04.757 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.625077+0000", 2023-10-01T10:51:04.757 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.21285906399999999 2023-10-01T10:51:04.757 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.757 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.757 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:51:04.757 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.625127+0000", 2023-10-01T10:51:04.757 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.9465000000000002e-05 2023-10-01T10:51:04.757 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.757 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.758 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:51:04.758 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:39.625168+0000", 2023-10-01T10:51:04.758 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.1186999999999997e-05 2023-10-01T10:51:04.758 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:51:04.758 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:51:04.758 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:51:04.758 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.758 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.758 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.35649.0:158810 9.2 9:48008bee:::benchmark_data_smithi078_113121_object46995:head [delete] snapc 0=[] ondisk+write+known_if_redirected e139)", 2023-10-01T10:51:04.758 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:43:40.577229+0000", 2023-10-01T10:51:04.758 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 444.09087076700001, 2023-10-01T10:51:04.759 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.21258954699999999, 2023-10-01T10:51:04.759 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:51:04.759 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:51:04.759 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:51:04.759 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.35649", 2023-10-01T10:51:04.759 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:51:04.759 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 158810 2023-10-01T10:51:04.759 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.759 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:51:04.759 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.759 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:51:04.760 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.577229+0000", 2023-10-01T10:51:04.760 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:51:04.760 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.760 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.760 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:51:04.760 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.577229+0000", 2023-10-01T10:51:04.760 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:51:04.760 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.760 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.760 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:51:04.760 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.577228+0000", 2023-10-01T10:51:04.761 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999995 2023-10-01T10:51:04.761 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.761 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.761 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:51:04.761 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.577232+0000", 2023-10-01T10:51:04.761 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.3830000000000001e-06 2023-10-01T10:51:04.761 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.761 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.761 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:51:04.761 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.577238+0000", 2023-10-01T10:51:04.761 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.4919999999999998e-06 2023-10-01T10:51:04.761 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.762 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.762 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:51:04.762 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.577242+0000", 2023-10-01T10:51:04.762 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.41e-06 2023-10-01T10:51:04.762 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.762 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.762 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:51:04.762 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.672105+0000", 2023-10-01T10:51:04.763 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.094863399000000001 2023-10-01T10:51:04.763 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.763 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.763 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:51:04.763 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.672507+0000", 2023-10-01T10:51:04.763 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00040220000000000002 2023-10-01T10:51:04.763 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.763 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.763 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 1,2", 2023-10-01T10:51:04.763 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.672669+0000", 2023-10-01T10:51:04.763 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000161953 2023-10-01T10:51:04.764 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.764 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.764 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:51:04.764 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.686715+0000", 2023-10-01T10:51:04.764 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.014046018 2023-10-01T10:51:04.764 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.764 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.764 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:51:04.764 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.686723+0000", 2023-10-01T10:51:04.764 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.5689999999999999e-06 2023-10-01T10:51:04.764 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.765 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.765 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:51:04.765 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.789080+0000", 2023-10-01T10:51:04.765 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.10235736300000001 2023-10-01T10:51:04.765 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.765 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.765 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:51:04.765 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.789131+0000", 2023-10-01T10:51:04.765 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.0568000000000001e-05 2023-10-01T10:51:04.765 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.765 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.765 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:51:04.766 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.789818+0000", 2023-10-01T10:51:04.766 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00068768799999999999 2023-10-01T10:51:04.766 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:51:04.766 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:51:04.766 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:51:04.766 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.766 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.766 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.35649.0:158811 9.2 9:43e6941c:::benchmark_data_smithi078_113121_object46996:head [delete] snapc 0=[] ondisk+write+known_if_redirected e139)", 2023-10-01T10:51:04.766 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:43:40.577279+0000", 2023-10-01T10:51:04.766 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 444.09082049300002, 2023-10-01T10:51:04.766 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.212757111, 2023-10-01T10:51:04.766 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:51:04.767 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:51:04.767 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:51:04.767 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.35649", 2023-10-01T10:51:04.767 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:51:04.767 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 158811 2023-10-01T10:51:04.767 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.767 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:51:04.767 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.767 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:51:04.767 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.577279+0000", 2023-10-01T10:51:04.767 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:51:04.768 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.768 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.768 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:51:04.768 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.577279+0000", 2023-10-01T10:51:04.768 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:51:04.768 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.768 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.768 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:51:04.768 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.577279+0000", 2023-10-01T10:51:04.768 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999995 2023-10-01T10:51:04.768 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.769 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.769 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:51:04.769 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.577283+0000", 2023-10-01T10:51:04.769 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.0579999999999999e-06 2023-10-01T10:51:04.769 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.769 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.769 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:51:04.769 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.577292+0000", 2023-10-01T10:51:04.769 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.5289999999999997e-06 2023-10-01T10:51:04.769 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.769 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.770 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:51:04.770 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.577299+0000", 2023-10-01T10:51:04.770 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.7349999999999999e-06 2023-10-01T10:51:04.770 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.770 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.770 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:51:04.770 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.681513+0000", 2023-10-01T10:51:04.770 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.10421393299999999 2023-10-01T10:51:04.770 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.770 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.770 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:51:04.771 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.681619+0000", 2023-10-01T10:51:04.771 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000106477 2023-10-01T10:51:04.771 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.771 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.771 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 1,2", 2023-10-01T10:51:04.771 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.681807+0000", 2023-10-01T10:51:04.771 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00018774300000000001 2023-10-01T10:51:04.771 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.771 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.771 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:51:04.771 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.686728+0000", 2023-10-01T10:51:04.771 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0049204100000000001 2023-10-01T10:51:04.772 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.772 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.772 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:51:04.772 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.686817+0000", 2023-10-01T10:51:04.772 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.9351999999999994e-05 2023-10-01T10:51:04.772 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.772 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.772 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:51:04.772 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.789872+0000", 2023-10-01T10:51:04.772 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.10305476700000001 2023-10-01T10:51:04.772 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.773 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.773 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:51:04.773 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.789959+0000", 2023-10-01T10:51:04.773 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.7702999999999997e-05 2023-10-01T10:51:04.773 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.773 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.773 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:51:04.773 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.790036+0000", 2023-10-01T10:51:04.773 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.6994000000000002e-05 2023-10-01T10:51:04.773 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:51:04.773 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:51:04.773 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:51:04.774 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.774 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.774 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.35649.0:158815 9.1 9:81709652:::benchmark_data_smithi078_113121_object47000:head [delete] snapc 0=[] ondisk+write+known_if_redirected e139)", 2023-10-01T10:51:04.774 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:43:40.583378+0000", 2023-10-01T10:51:04.774 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 444.084722081, 2023-10-01T10:51:04.774 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.20006031099999999, 2023-10-01T10:51:04.774 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:51:04.774 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:51:04.774 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:51:04.774 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.35649", 2023-10-01T10:51:04.774 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:51:04.775 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 158815 2023-10-01T10:51:04.775 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.775 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:51:04.775 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.775 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:51:04.775 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.583378+0000", 2023-10-01T10:51:04.775 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:51:04.775 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.775 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.775 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:51:04.775 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.583378+0000", 2023-10-01T10:51:04.776 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:51:04.776 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.776 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.776 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:51:04.776 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.583376+0000", 2023-10-01T10:51:04.776 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999981 2023-10-01T10:51:04.776 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.776 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.776 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:51:04.776 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.583384+0000", 2023-10-01T10:51:04.776 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.1270000000000003e-06 2023-10-01T10:51:04.776 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.777 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.777 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:51:04.777 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.672010+0000", 2023-10-01T10:51:04.777 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.088626446999999997 2023-10-01T10:51:04.777 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.777 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.777 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:51:04.777 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.672027+0000", 2023-10-01T10:51:04.777 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.6361000000000001e-05 2023-10-01T10:51:04.777 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.777 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.777 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:51:04.778 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.676860+0000", 2023-10-01T10:51:04.778 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00483279 2023-10-01T10:51:04.778 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.778 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.778 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:51:04.778 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.676962+0000", 2023-10-01T10:51:04.778 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000102567 2023-10-01T10:51:04.778 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.778 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.778 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 6,7", 2023-10-01T10:51:04.778 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.677052+0000", 2023-10-01T10:51:04.779 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.0008999999999998e-05 2023-10-01T10:51:04.779 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.779 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.779 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:51:04.779 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.681732+0000", 2023-10-01T10:51:04.779 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0046803000000000001 2023-10-01T10:51:04.779 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.779 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.779 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:51:04.779 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.691375+0000", 2023-10-01T10:51:04.779 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0096430100000000005 2023-10-01T10:51:04.779 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.780 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.780 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:51:04.780 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.783275+0000", 2023-10-01T10:51:04.780 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.091899397999999993 2023-10-01T10:51:04.780 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.780 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.780 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:51:04.780 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.783369+0000", 2023-10-01T10:51:04.780 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.3764999999999994e-05 2023-10-01T10:51:04.780 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.780 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.781 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:51:04.781 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:40.783438+0000", 2023-10-01T10:51:04.781 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.9298e-05 2023-10-01T10:51:04.781 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:51:04.781 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:51:04.781 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:51:04.781 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.781 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.781 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.35649.0:159544 9.1 9:83c5b7cd:::benchmark_data_smithi078_113121_object47729:head [delete] snapc 0=[] ondisk+write+known_if_redirected e139)", 2023-10-01T10:51:04.781 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:43:41.686189+0000", 2023-10-01T10:51:04.781 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 442.98191034500002, 2023-10-01T10:51:04.781 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.19142947499999999, 2023-10-01T10:51:04.782 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:51:04.782 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:51:04.782 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:51:04.782 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.35649", 2023-10-01T10:51:04.782 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:51:04.782 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 159544 2023-10-01T10:51:04.782 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.782 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:51:04.782 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.782 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:51:04.782 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.686189+0000", 2023-10-01T10:51:04.783 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:51:04.783 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.783 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.783 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:51:04.783 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.686189+0000", 2023-10-01T10:51:04.783 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:51:04.783 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.783 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.783 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:51:04.783 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.686188+0000", 2023-10-01T10:51:04.783 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:51:04.783 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.784 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.784 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:51:04.784 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.686194+0000", 2023-10-01T10:51:04.784 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.8039999999999998e-06 2023-10-01T10:51:04.784 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.784 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.784 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:51:04.784 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.686204+0000", 2023-10-01T10:51:04.784 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.0305e-05 2023-10-01T10:51:04.784 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.784 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.784 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:51:04.785 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.686212+0000", 2023-10-01T10:51:04.785 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.0020000000000006e-06 2023-10-01T10:51:04.785 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.785 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.785 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:51:04.785 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.686254+0000", 2023-10-01T10:51:04.785 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.1561999999999999e-05 2023-10-01T10:51:04.785 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.785 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.785 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:51:04.785 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.686360+0000", 2023-10-01T10:51:04.786 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000105676 2023-10-01T10:51:04.786 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.786 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.786 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 6,7", 2023-10-01T10:51:04.786 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.686464+0000", 2023-10-01T10:51:04.786 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000104209 2023-10-01T10:51:04.786 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.786 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.786 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:51:04.786 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.690429+0000", 2023-10-01T10:51:04.786 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0039646619999999999 2023-10-01T10:51:04.786 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.787 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.787 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:51:04.787 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.691141+0000", 2023-10-01T10:51:04.787 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00071247699999999999 2023-10-01T10:51:04.787 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.787 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.787 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:51:04.787 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.877553+0000", 2023-10-01T10:51:04.787 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.18641245300000001 2023-10-01T10:51:04.787 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.787 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.788 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:51:04.788 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.877584+0000", 2023-10-01T10:51:04.788 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.0642000000000002e-05 2023-10-01T10:51:04.788 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.788 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.788 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:51:04.788 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.877619+0000", 2023-10-01T10:51:04.788 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.4727999999999997e-05 2023-10-01T10:51:04.788 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:51:04.788 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:51:04.788 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:51:04.788 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.789 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.789 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.35649.0:159554 9.1 9:866e6940:::benchmark_data_smithi078_113121_object47739:head [delete] snapc 0=[] ondisk+write+known_if_redirected e139)", 2023-10-01T10:51:04.789 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:43:41.702172+0000", 2023-10-01T10:51:04.789 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 442.96592804300002, 2023-10-01T10:51:04.789 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.17563100000000001, 2023-10-01T10:51:04.789 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:51:04.789 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:51:04.789 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:51:04.789 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.35649", 2023-10-01T10:51:04.789 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:51:04.789 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 159554 2023-10-01T10:51:04.789 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.790 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:51:04.790 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.790 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:51:04.790 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.702172+0000", 2023-10-01T10:51:04.790 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:51:04.790 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.790 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.790 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:51:04.790 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.702172+0000", 2023-10-01T10:51:04.790 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:51:04.790 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.791 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.791 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:51:04.791 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.702171+0000", 2023-10-01T10:51:04.791 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:51:04.791 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.791 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.791 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:51:04.791 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.702177+0000", 2023-10-01T10:51:04.791 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.8699999999999997e-06 2023-10-01T10:51:04.791 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.791 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.792 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:51:04.792 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.702188+0000", 2023-10-01T10:51:04.792 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1494e-05 2023-10-01T10:51:04.792 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.792 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.792 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:51:04.792 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.702197+0000", 2023-10-01T10:51:04.792 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.986e-06 2023-10-01T10:51:04.792 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.792 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.792 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:51:04.792 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.702238+0000", 2023-10-01T10:51:04.793 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.1057999999999999e-05 2023-10-01T10:51:04.793 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.793 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.793 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:51:04.793 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.702373+0000", 2023-10-01T10:51:04.793 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00013459400000000001 2023-10-01T10:51:04.793 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.793 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.793 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 6,7", 2023-10-01T10:51:04.793 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.702458+0000", 2023-10-01T10:51:04.793 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.5320000000000003e-05 2023-10-01T10:51:04.793 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.794 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.794 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:51:04.794 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.705539+0000", 2023-10-01T10:51:04.794 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0030812299999999999 2023-10-01T10:51:04.794 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.794 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.794 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:51:04.794 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.707125+0000", 2023-10-01T10:51:04.794 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0015853270000000001 2023-10-01T10:51:04.794 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.795 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.795 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:51:04.795 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.877694+0000", 2023-10-01T10:51:04.795 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.17056949900000001 2023-10-01T10:51:04.795 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.795 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.795 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:51:04.795 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.877737+0000", 2023-10-01T10:51:04.795 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.2760999999999999e-05 2023-10-01T10:51:04.795 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.795 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.795 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:51:04.796 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:43:41.877803+0000", 2023-10-01T10:51:04.796 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.5870999999999993e-05 2023-10-01T10:51:04.796 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:51:04.796 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:51:04.796 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:51:04.796 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.796 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.796 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.45818.0:118224 10.7 10:e7e19940:::benchmark_data_smithi078_135100_object8311:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e160)", 2023-10-01T10:51:04.796 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:47:53.736061+0000", 2023-10-01T10:51:04.796 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 190.93203855100001, 2023-10-01T10:51:04.796 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.0454423639999999, 2023-10-01T10:51:04.796 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:51:04.797 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:51:04.797 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:51:04.797 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.45818", 2023-10-01T10:51:04.797 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1151403832", 2023-10-01T10:51:04.797 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 118224 2023-10-01T10:51:04.797 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.797 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:51:04.797 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.797 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:51:04.797 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:53.736061+0000", 2023-10-01T10:51:04.797 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:51:04.797 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.798 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.798 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:51:04.798 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:53.736061+0000", 2023-10-01T10:51:04.798 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:51:04.798 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.798 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.798 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:51:04.798 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:53.736059+0000", 2023-10-01T10:51:04.799 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999981 2023-10-01T10:51:04.799 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.799 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.799 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:51:04.799 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:53.736066+0000", 2023-10-01T10:51:04.799 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.0169999999999999e-06 2023-10-01T10:51:04.800 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.800 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.800 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:51:04.800 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:53.736083+0000", 2023-10-01T10:51:04.800 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.6813999999999999e-05 2023-10-01T10:51:04.801 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.801 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.801 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:51:04.801 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:53.736092+0000", 2023-10-01T10:51:04.801 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.3189999999999995e-06 2023-10-01T10:51:04.801 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.802 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.802 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:51:04.802 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:53.744504+0000", 2023-10-01T10:51:04.802 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0084126949999999995 2023-10-01T10:51:04.802 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.802 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.803 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T10:51:04.803 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:53.744512+0000", 2023-10-01T10:51:04.803 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.1650000000000006e-06 2023-10-01T10:51:04.803 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.803 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.804 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:51:04.804 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:54.759075+0000", 2023-10-01T10:51:04.804 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.0145625060000001 2023-10-01T10:51:04.804 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.804 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.804 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:51:04.804 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:54.762631+0000", 2023-10-01T10:51:04.805 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.003556254 2023-10-01T10:51:04.805 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.805 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.805 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 5,6", 2023-10-01T10:51:04.805 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:54.762807+0000", 2023-10-01T10:51:04.805 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000175978 2023-10-01T10:51:04.806 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.806 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.806 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:51:04.806 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:54.773287+0000", 2023-10-01T10:51:04.806 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0104793 2023-10-01T10:51:04.807 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.807 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.807 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:51:04.807 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:54.777746+0000", 2023-10-01T10:51:04.807 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0044596979999999998 2023-10-01T10:51:04.808 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.808 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.808 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:51:04.808 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:54.781389+0000", 2023-10-01T10:51:04.808 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0036423649999999998 2023-10-01T10:51:04.808 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.809 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.809 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:51:04.809 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:54.781450+0000", 2023-10-01T10:51:04.809 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.1104000000000003e-05 2023-10-01T10:51:04.809 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.809 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.810 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:51:04.810 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:54.781504+0000", 2023-10-01T10:51:04.810 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.3853000000000001e-05 2023-10-01T10:51:04.810 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:51:04.810 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:51:04.810 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:51:04.811 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.811 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.811 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.45818.0:118236 10.7 10:e348914e:::benchmark_data_smithi078_135100_object8323:head [delete] snapc 0=[] ondisk+write+known_if_redirected e160)", 2023-10-01T10:51:04.811 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:47:53.745118+0000", 2023-10-01T10:51:04.811 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 190.92298217800001, 2023-10-01T10:51:04.812 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.044474221, 2023-10-01T10:51:04.812 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:51:04.812 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:51:04.812 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:51:04.812 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.45818", 2023-10-01T10:51:04.812 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1151403832", 2023-10-01T10:51:04.813 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 118236 2023-10-01T10:51:04.813 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.813 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:51:04.813 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.813 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:51:04.813 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:53.745118+0000", 2023-10-01T10:51:04.814 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:51:04.814 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.814 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.814 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:51:04.814 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:53.745118+0000", 2023-10-01T10:51:04.814 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:51:04.815 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.815 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.815 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:51:04.815 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:53.745116+0000", 2023-10-01T10:51:04.815 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:51:04.816 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.816 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.816 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:51:04.816 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:53.745121+0000", 2023-10-01T10:51:04.816 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.9540000000000003e-06 2023-10-01T10:51:04.816 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.817 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.817 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:51:04.817 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:53.745131+0000", 2023-10-01T10:51:04.817 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.3039999999999994e-06 2023-10-01T10:51:04.817 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.817 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.818 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:51:04.818 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:53.745137+0000", 2023-10-01T10:51:04.818 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.2249999999999997e-06 2023-10-01T10:51:04.818 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.818 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.818 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:51:04.819 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:53.745167+0000", 2023-10-01T10:51:04.819 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.0414999999999999e-05 2023-10-01T10:51:04.819 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.819 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.819 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T10:51:04.820 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:53.745173+0000", 2023-10-01T10:51:04.820 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.7169999999999996e-06 2023-10-01T10:51:04.820 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.820 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.820 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:51:04.820 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:54.766565+0000", 2023-10-01T10:51:04.821 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.021392375 2023-10-01T10:51:04.821 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.821 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.821 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:51:04.821 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:54.766719+0000", 2023-10-01T10:51:04.821 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00015320399999999999 2023-10-01T10:51:04.822 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.822 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.822 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 5,6", 2023-10-01T10:51:04.822 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:54.766884+0000", 2023-10-01T10:51:04.822 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00016531900000000001 2023-10-01T10:51:04.822 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.823 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.823 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:51:04.823 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:54.777797+0000", 2023-10-01T10:51:04.823 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.010913101 2023-10-01T10:51:04.823 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.823 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.824 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:51:04.824 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:54.789145+0000", 2023-10-01T10:51:04.824 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.011348001 2023-10-01T10:51:04.824 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.824 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.824 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:51:04.825 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:54.789492+0000", 2023-10-01T10:51:04.825 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000347253 2023-10-01T10:51:04.825 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.825 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.825 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:51:04.826 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:54.789549+0000", 2023-10-01T10:51:04.826 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.6854000000000001e-05 2023-10-01T10:51:04.826 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.826 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.826 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:51:04.826 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:54.789592+0000", 2023-10-01T10:51:04.827 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.2633000000000003e-05 2023-10-01T10:51:04.827 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:51:04.827 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:51:04.827 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:51:04.827 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.827 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.828 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.45818.0:118251 10.7 10:ed214ea9:::benchmark_data_smithi078_135100_object8338:head [delete] snapc 0=[] ondisk+write+known_if_redirected e160)", 2023-10-01T10:51:04.828 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:47:53.772600+0000", 2023-10-01T10:51:04.828 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 190.895499548, 2023-10-01T10:51:04.828 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.026610547, 2023-10-01T10:51:04.828 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:51:04.828 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:51:04.829 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:51:04.829 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.45818", 2023-10-01T10:51:04.829 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1151403832", 2023-10-01T10:51:04.829 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 118251 2023-10-01T10:51:04.829 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.830 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:51:04.830 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.830 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:51:04.830 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:53.772600+0000", 2023-10-01T10:51:04.830 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:51:04.830 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.831 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.831 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:51:04.831 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:53.772600+0000", 2023-10-01T10:51:04.831 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:51:04.831 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.831 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.832 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:51:04.832 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:53.772599+0000", 2023-10-01T10:51:04.832 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999986 2023-10-01T10:51:04.832 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.832 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.832 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:51:04.833 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:53.772606+0000", 2023-10-01T10:51:04.833 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.9000000000000006e-06 2023-10-01T10:51:04.833 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.833 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.833 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:51:04.833 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:53.772620+0000", 2023-10-01T10:51:04.834 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.3818000000000001e-05 2023-10-01T10:51:04.834 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.834 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.834 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:51:04.834 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:53.772629+0000", 2023-10-01T10:51:04.834 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.0869999999999996e-06 2023-10-01T10:51:04.835 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.835 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.835 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:51:04.835 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:53.772688+0000", 2023-10-01T10:51:04.835 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.893e-05 2023-10-01T10:51:04.836 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.836 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.836 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T10:51:04.836 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:53.772700+0000", 2023-10-01T10:51:04.836 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.2099999999999999e-05 2023-10-01T10:51:04.836 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.837 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.837 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:51:04.837 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:54.770305+0000", 2023-10-01T10:51:04.837 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.997604198 2023-10-01T10:51:04.837 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.837 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.838 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:51:04.838 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:54.770422+0000", 2023-10-01T10:51:04.838 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000117269 2023-10-01T10:51:04.838 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.838 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.838 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 5,6", 2023-10-01T10:51:04.839 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:54.770497+0000", 2023-10-01T10:51:04.839 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.4970999999999998e-05 2023-10-01T10:51:04.839 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.839 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.839 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:51:04.840 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:54.781333+0000", 2023-10-01T10:51:04.840 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.010835829999999999 2023-10-01T10:51:04.840 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.840 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.840 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:51:04.840 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:54.789632+0000", 2023-10-01T10:51:04.841 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0082996000000000007 2023-10-01T10:51:04.841 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.841 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.841 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:51:04.841 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:54.799109+0000", 2023-10-01T10:51:04.842 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0094770359999999994 2023-10-01T10:51:04.842 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.842 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.842 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:51:04.842 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:54.799158+0000", 2023-10-01T10:51:04.843 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.8387999999999998e-05 2023-10-01T10:51:04.843 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.843 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.843 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:51:04.843 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:54.799211+0000", 2023-10-01T10:51:04.843 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.2995999999999999e-05 2023-10-01T10:51:04.844 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:51:04.844 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:51:04.844 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:51:04.844 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.844 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.844 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.45818.0:118260 10.7 10:ec02b11a:::benchmark_data_smithi078_135100_object8347:head [delete] snapc 0=[] ondisk+write+known_if_redirected e160)", 2023-10-01T10:51:04.845 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:47:53.786487+0000", 2023-10-01T10:51:04.845 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 190.881613031, 2023-10-01T10:51:04.845 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.012876162, 2023-10-01T10:51:04.845 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:51:04.845 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:51:04.846 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:51:04.846 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.45818", 2023-10-01T10:51:04.846 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1151403832", 2023-10-01T10:51:04.846 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 118260 2023-10-01T10:51:04.846 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.846 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:51:04.847 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.847 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:51:04.847 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:53.786487+0000", 2023-10-01T10:51:04.847 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:51:04.847 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.847 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.848 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:51:04.848 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:53.786487+0000", 2023-10-01T10:51:04.848 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:51:04.848 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.848 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.848 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:51:04.849 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:53.786485+0000", 2023-10-01T10:51:04.849 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999981 2023-10-01T10:51:04.849 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.849 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.849 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:51:04.849 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:53.786493+0000", 2023-10-01T10:51:04.849 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.9680000000000006e-06 2023-10-01T10:51:04.849 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.850 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.850 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:51:04.850 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:53.786509+0000", 2023-10-01T10:51:04.850 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.6144e-05 2023-10-01T10:51:04.850 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.850 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.850 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:51:04.850 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:53.786517+0000", 2023-10-01T10:51:04.850 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.3270000000000001e-06 2023-10-01T10:51:04.851 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.851 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.851 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:51:04.851 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:53.786579+0000", 2023-10-01T10:51:04.851 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.1420000000000005e-05 2023-10-01T10:51:04.851 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.851 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.851 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T10:51:04.851 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:53.786589+0000", 2023-10-01T10:51:04.851 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.0066e-05 2023-10-01T10:51:04.851 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.851 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.852 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:51:04.852 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:54.773354+0000", 2023-10-01T10:51:04.852 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.98676514199999998 2023-10-01T10:51:04.852 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.852 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.852 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:51:04.852 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:54.773538+0000", 2023-10-01T10:51:04.852 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00018421600000000001 2023-10-01T10:51:04.852 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.852 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.852 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 5,6", 2023-10-01T10:51:04.852 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:54.773680+0000", 2023-10-01T10:51:04.853 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00014177800000000001 2023-10-01T10:51:04.853 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.853 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.853 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:51:04.853 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:54.781342+0000", 2023-10-01T10:51:04.853 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.007662543 2023-10-01T10:51:04.853 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.853 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.853 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:51:04.853 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:54.789673+0000", 2023-10-01T10:51:04.853 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0083302670000000006 2023-10-01T10:51:04.853 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.854 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.854 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:51:04.854 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:54.799266+0000", 2023-10-01T10:51:04.854 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0095933819999999993 2023-10-01T10:51:04.854 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.854 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.854 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:51:04.854 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:54.799314+0000", 2023-10-01T10:51:04.854 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.8103999999999999e-05 2023-10-01T10:51:04.854 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:51:04.854 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:51:04.854 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:51:04.855 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:54.799363+0000", 2023-10-01T10:51:04.855 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.8723000000000001e-05 2023-10-01T10:51:04.855 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:51:04.855 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:51:04.855 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:51:04.855 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:51:04.855 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:51:04.855 INFO:teuthology.orchestra.run.smithi078.stdout:} 2023-10-01T10:51:04.855 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:04.879 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:04.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:04.881 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:51:04.970 INFO:teuthology.orchestra.run.smithi078.stderr:reweighted osd.7 to 1 (10000) 2023-10-01T10:51:04.974 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:04 smithi078 bash[75299]: cluster 2023-10-01T10:51:03.943043+0000 mgr.y (mgr.25072) 1003 : cluster 0 pgmap v950: 89 pgs: 7 creating+peering, 6 unknown, 76 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:51:04.974 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:04 smithi078 bash[75299]: audit 2023-10-01T10:51:04.127086+0000 mon.c (mon.2) 172 : audit 0 from='client.? 172.21.15.78:0/2169145524' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:51:04.974 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:04 smithi078 bash[75299]: audit 2023-10-01T10:51:04.681500+0000 mon.a (mon.0) 1239 : audit 1 from='client.? 172.21.15.78:0/1763911812' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T10:51:04.974 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:04 smithi078 bash[78274]: cluster 2023-10-01T10:51:03.943043+0000 mgr.y (mgr.25072) 1003 : cluster 0 pgmap v950: 89 pgs: 7 creating+peering, 6 unknown, 76 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:51:04.974 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:04 smithi078 bash[78274]: audit 2023-10-01T10:51:04.127086+0000 mon.c (mon.2) 172 : audit 0 from='client.? 172.21.15.78:0/2169145524' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:51:04.974 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:04 smithi078 bash[78274]: audit 2023-10-01T10:51:04.681500+0000 mon.a (mon.0) 1239 : audit 1 from='client.? 172.21.15.78:0/1763911812' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T10:51:04.982 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:05.008 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:05.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:05.011 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:51:05.112 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:05.143 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:05.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:05.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:51:05.248 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:05.261 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:04 smithi173 bash[58816]: cluster 2023-10-01T10:51:03.943043+0000 mgr.y (mgr.25072) 1003 : cluster 0 pgmap v950: 89 pgs: 7 creating+peering, 6 unknown, 76 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:51:05.261 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:04 smithi173 bash[58816]: audit 2023-10-01T10:51:04.127086+0000 mon.c (mon.2) 172 : audit 0 from='client.? 172.21.15.78:0/2169145524' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:51:05.261 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:04 smithi173 bash[58816]: audit 2023-10-01T10:51:04.681500+0000 mon.a (mon.0) 1239 : audit 1 from='client.? 172.21.15.78:0/1763911812' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T10:51:05.280 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:05.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:05.283 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:51:05.384 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:05.415 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:05.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:05.417 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:51:05.518 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:05.550 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:05.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:05.554 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:51:05.654 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:05.681 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:05.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:05.684 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:51:05.785 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:05.811 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:05.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:05.813 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:51:05.914 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:05.939 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:05.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:05.941 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:51:05.983 INFO:teuthology.orchestra.run.smithi078.stderr:nodeep-scrub is unset 2023-10-01T10:51:06.042 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:06.070 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:06.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:06.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:51:06.173 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:06.196 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:06.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:06.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:51:06.218 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:05 smithi078 bash[75299]: cluster 2023-10-01T10:51:04.964268+0000 osd.5 (osd.5) 110 : cluster 3 failed to encode map e176 with expected crc 2023-10-01T10:51:06.218 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:05 smithi078 bash[75299]: audit 2023-10-01T10:51:04.965048+0000 mon.a (mon.0) 1240 : audit 1 from='client.? 172.21.15.78:0/1763911812' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-01T10:51:06.218 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:05 smithi078 bash[75299]: cluster 2023-10-01T10:51:04.965143+0000 mon.a (mon.0) 1241 : cluster 0 osdmap e176: 8 total, 8 up, 8 in 2023-10-01T10:51:06.219 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:05 smithi078 bash[75299]: cluster 2023-10-01T10:51:04.967387+0000 osd.7 (osd.7) 129 : cluster 3 failed to encode map e176 with expected crc 2023-10-01T10:51:06.219 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:05 smithi078 bash[75299]: audit 2023-10-01T10:51:05.108418+0000 mon.c (mon.2) 173 : audit 1 from='client.? 172.21.15.78:0/1693407373' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:51:06.219 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:05 smithi078 bash[75299]: audit 2023-10-01T10:51:05.108700+0000 mon.a (mon.0) 1242 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:51:06.219 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:06 smithi078 bash[78274]: cluster 2023-10-01T10:51:04.964268+0000 osd.5 (osd.5) 110 : cluster 3 failed to encode map e176 with expected crc 2023-10-01T10:51:06.219 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:06 smithi078 bash[78274]: audit 2023-10-01T10:51:04.965048+0000 mon.a (mon.0) 1240 : audit 1 from='client.? 172.21.15.78:0/1763911812' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-01T10:51:06.219 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:06 smithi078 bash[78274]: cluster 2023-10-01T10:51:04.965143+0000 mon.a (mon.0) 1241 : cluster 0 osdmap e176: 8 total, 8 up, 8 in 2023-10-01T10:51:06.219 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:06 smithi078 bash[78274]: cluster 2023-10-01T10:51:04.967387+0000 osd.7 (osd.7) 129 : cluster 3 failed to encode map e176 with expected crc 2023-10-01T10:51:06.219 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:06 smithi078 bash[78274]: audit 2023-10-01T10:51:05.108418+0000 mon.c (mon.2) 173 : audit 1 from='client.? 172.21.15.78:0/1693407373' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:51:06.219 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:06 smithi078 bash[78274]: audit 2023-10-01T10:51:05.108700+0000 mon.a (mon.0) 1242 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:51:06.300 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:06.313 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:05 smithi173 bash[58816]: cluster 2023-10-01T10:51:04.964268+0000 osd.5 (osd.5) 110 : cluster 3 failed to encode map e176 with expected crc 2023-10-01T10:51:06.313 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:05 smithi173 bash[58816]: audit 2023-10-01T10:51:04.965048+0000 mon.a (mon.0) 1240 : audit 1 from='client.? 172.21.15.78:0/1763911812' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-01T10:51:06.314 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:05 smithi173 bash[58816]: cluster 2023-10-01T10:51:04.965143+0000 mon.a (mon.0) 1241 : cluster 0 osdmap e176: 8 total, 8 up, 8 in 2023-10-01T10:51:06.314 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:05 smithi173 bash[58816]: cluster 2023-10-01T10:51:04.967387+0000 osd.7 (osd.7) 129 : cluster 3 failed to encode map e176 with expected crc 2023-10-01T10:51:06.314 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:05 smithi173 bash[58816]: audit 2023-10-01T10:51:05.108418+0000 mon.c (mon.2) 173 : audit 1 from='client.? 172.21.15.78:0/1693407373' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:51:06.314 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:05 smithi173 bash[58816]: audit 2023-10-01T10:51:05.108700+0000 mon.a (mon.0) 1242 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:51:06.326 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:06.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:06.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:51:06.430 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:06.454 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:06.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:06.457 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:51:06.558 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:06.579 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:06.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:06.581 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:51:06.682 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:06.710 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:06.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:06.714 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:51:06.815 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:06.844 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:06.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:06.847 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:51:06.949 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:06.977 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:06.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:06.980 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:51:07.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:06 smithi078 bash[78274]: cluster 2023-10-01T10:51:05.608935+0000 osd.6 (osd.6) 94 : cluster 3 failed to encode map e176 with expected crc 2023-10-01T10:51:07.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:06 smithi078 bash[78274]: cluster 2023-10-01T10:51:05.943615+0000 mgr.y (mgr.25072) 1004 : cluster 0 pgmap v952: 89 pgs: 7 creating+peering, 82 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:51:07.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:06 smithi078 bash[78274]: cluster 2023-10-01T10:51:05.971367+0000 mon.a (mon.0) 1243 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T10:51:07.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:06 smithi078 bash[78274]: cluster 2023-10-01T10:51:05.971393+0000 mon.a (mon.0) 1244 : cluster 1 Cluster is now healthy 2023-10-01T10:51:07.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:06 smithi078 bash[78274]: cluster 2023-10-01T10:51:05.975135+0000 osd.7 (osd.7) 130 : cluster 3 failed to encode map e177 with expected crc 2023-10-01T10:51:07.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:06 smithi078 bash[78274]: audit 2023-10-01T10:51:05.977211+0000 mon.a (mon.0) 1245 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T10:51:07.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:06 smithi078 bash[78274]: cluster 2023-10-01T10:51:05.977364+0000 mon.a (mon.0) 1246 : cluster 0 osdmap e177: 8 total, 8 up, 8 in 2023-10-01T10:51:07.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:06 smithi078 bash[78274]: cluster 2023-10-01T10:51:06.143658+0000 osd.5 (osd.5) 111 : cluster 3 failed to encode map e177 with expected crc 2023-10-01T10:51:07.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:06 smithi078 bash[78274]: cluster 2023-10-01T10:51:06.143727+0000 osd.6 (osd.6) 95 : cluster 3 failed to encode map e177 with expected crc 2023-10-01T10:51:07.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:06 smithi078 bash[75299]: cluster 2023-10-01T10:51:05.608935+0000 osd.6 (osd.6) 94 : cluster 3 failed to encode map e176 with expected crc 2023-10-01T10:51:07.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:06 smithi078 bash[75299]: cluster 2023-10-01T10:51:05.943615+0000 mgr.y (mgr.25072) 1004 : cluster 0 pgmap v952: 89 pgs: 7 creating+peering, 82 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:51:07.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:06 smithi078 bash[75299]: cluster 2023-10-01T10:51:05.971367+0000 mon.a (mon.0) 1243 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T10:51:07.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:06 smithi078 bash[75299]: cluster 2023-10-01T10:51:05.971393+0000 mon.a (mon.0) 1244 : cluster 1 Cluster is now healthy 2023-10-01T10:51:07.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:06 smithi078 bash[75299]: cluster 2023-10-01T10:51:05.975135+0000 osd.7 (osd.7) 130 : cluster 3 failed to encode map e177 with expected crc 2023-10-01T10:51:07.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:06 smithi078 bash[75299]: audit 2023-10-01T10:51:05.977211+0000 mon.a (mon.0) 1245 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T10:51:07.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:06 smithi078 bash[75299]: cluster 2023-10-01T10:51:05.977364+0000 mon.a (mon.0) 1246 : cluster 0 osdmap e177: 8 total, 8 up, 8 in 2023-10-01T10:51:07.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:06 smithi078 bash[75299]: cluster 2023-10-01T10:51:06.143658+0000 osd.5 (osd.5) 111 : cluster 3 failed to encode map e177 with expected crc 2023-10-01T10:51:07.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:06 smithi078 bash[75299]: cluster 2023-10-01T10:51:06.143727+0000 osd.6 (osd.6) 95 : cluster 3 failed to encode map e177 with expected crc 2023-10-01T10:51:07.083 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:07.108 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:07.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:07.111 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:51:07.211 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:07.236 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:07.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:07.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:51:07.339 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:07.366 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:07.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:07.370 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:51:07.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:06 smithi173 bash[58816]: cluster 2023-10-01T10:51:05.608935+0000 osd.6 (osd.6) 94 : cluster 3 failed to encode map e176 with expected crc 2023-10-01T10:51:07.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:06 smithi173 bash[58816]: cluster 2023-10-01T10:51:05.943615+0000 mgr.y (mgr.25072) 1004 : cluster 0 pgmap v952: 89 pgs: 7 creating+peering, 82 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:51:07.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:06 smithi173 bash[58816]: cluster 2023-10-01T10:51:05.971367+0000 mon.a (mon.0) 1243 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T10:51:07.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:06 smithi173 bash[58816]: cluster 2023-10-01T10:51:05.971393+0000 mon.a (mon.0) 1244 : cluster 1 Cluster is now healthy 2023-10-01T10:51:07.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:06 smithi173 bash[58816]: cluster 2023-10-01T10:51:05.975135+0000 osd.7 (osd.7) 130 : cluster 3 failed to encode map e177 with expected crc 2023-10-01T10:51:07.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:06 smithi173 bash[58816]: audit 2023-10-01T10:51:05.977211+0000 mon.a (mon.0) 1245 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T10:51:07.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:06 smithi173 bash[58816]: cluster 2023-10-01T10:51:05.977364+0000 mon.a (mon.0) 1246 : cluster 0 osdmap e177: 8 total, 8 up, 8 in 2023-10-01T10:51:07.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:06 smithi173 bash[58816]: cluster 2023-10-01T10:51:06.143658+0000 osd.5 (osd.5) 111 : cluster 3 failed to encode map e177 with expected crc 2023-10-01T10:51:07.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:06 smithi173 bash[58816]: cluster 2023-10-01T10:51:06.143727+0000 osd.6 (osd.6) 95 : cluster 3 failed to encode map e177 with expected crc 2023-10-01T10:51:07.470 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:07.497 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:07.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:07.499 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:51:07.600 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:07.630 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:07.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:07.632 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:51:07.734 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:07.755 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:07.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:07.757 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:51:07.858 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:07.890 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:07.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:07.893 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:51:07.993 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:08.027 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:08.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:08.031 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:51:08.132 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:08.154 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:08.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:08.157 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:51:08.258 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:08.289 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:08.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:08.292 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:51:08.392 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:08.393 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:08 smithi078 bash[75299]: audit 2023-10-01T10:51:07.121565+0000 mon.a (mon.0) 1247 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:51:08.393 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:08 smithi078 bash[75299]: audit 2023-10-01T10:51:07.128364+0000 mon.a (mon.0) 1248 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:51:08.393 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:08 smithi078 bash[78274]: audit 2023-10-01T10:51:07.121565+0000 mon.a (mon.0) 1247 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:51:08.393 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:08 smithi078 bash[78274]: audit 2023-10-01T10:51:07.128364+0000 mon.a (mon.0) 1248 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:51:08.394 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-01T10:51:08.406 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:08 smithi173 bash[58816]: audit 2023-10-01T10:51:07.121565+0000 mon.a (mon.0) 1247 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:51:08.406 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:08 smithi173 bash[58816]: audit 2023-10-01T10:51:07.128364+0000 mon.a (mon.0) 1248 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:51:08.423 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:08.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:08.426 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:51:08.516 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-01T10:51:08.517 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T10:51:08.527 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:08.554 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:08.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:08.556 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:51:08.637 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T10:51:08.657 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:08.691 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:08.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:08.694 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:51:08.794 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:08.829 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:08.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:08.832 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:51:08.933 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:08.968 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:08.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:08.971 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:51:09.071 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:09.105 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:09.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:09.109 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:51:09.210 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:09.242 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:09.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:09.246 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:51:09.346 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:09.381 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:09.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:09.383 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:51:09.484 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:09.512 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:09.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:09.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:51:09.616 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:09.642 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:09.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:09.645 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:51:09.746 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:09.757 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:09 smithi173 bash[58816]: cluster 2023-10-01T10:51:07.944181+0000 mgr.y (mgr.25072) 1005 : cluster 0 pgmap v954: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:51:09.778 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:09.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:09.781 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:51:09.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:09 smithi078 bash[75299]: cluster 2023-10-01T10:51:07.944181+0000 mgr.y (mgr.25072) 1005 : cluster 0 pgmap v954: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:51:09.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:09 smithi078 bash[78274]: cluster 2023-10-01T10:51:07.944181+0000 mgr.y (mgr.25072) 1005 : cluster 0 pgmap v954: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:51:09.882 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:09.916 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:09.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:09.920 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:51:10.020 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:10.052 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:10.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:10.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:51:10.156 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:10.186 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:10.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:10.189 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:51:10.290 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:10.316 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:10.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:10.319 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:51:10.420 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:10.453 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:10.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:10.456 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:51:10.527 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T10:51:10.540 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:51:10 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:51:10] "GET /metrics HTTP/1.1" 200 36242 "" "Prometheus/2.33.4" 2023-10-01T10:51:10.557 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:10.588 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:10.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:10.591 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:51:10.692 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:10.725 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:10.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:10.729 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:51:10.829 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:10.865 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:10.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:10.868 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:51:10.969 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:11.000 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:11.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:11.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:51:11.048 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:51:11.104 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:11.135 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:11.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:11.138 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:51:11.239 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:11.267 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:11.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:11.271 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:51:11.371 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:11.405 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:11.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:11.408 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:51:11.509 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:11.521 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:11 smithi173 bash[58816]: cluster 2023-10-01T10:51:09.945235+0000 mgr.y (mgr.25072) 1006 : cluster 0 pgmap v955: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:51:11.522 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:11 smithi173 bash[58816]: audit 2023-10-01T10:51:11.242995+0000 mon.a (mon.0) 1249 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:51:11.522 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:11 smithi173 bash[58816]: audit 2023-10-01T10:51:11.249938+0000 mon.a (mon.0) 1250 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:51:11.544 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:11.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:11.547 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:51:11.648 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:11.674 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:11.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:11.677 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:51:11.778 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:11.790 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:11 smithi078 bash[75299]: cluster 2023-10-01T10:51:09.945235+0000 mgr.y (mgr.25072) 1006 : cluster 0 pgmap v955: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:51:11.791 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:11 smithi078 bash[75299]: audit 2023-10-01T10:51:11.242995+0000 mon.a (mon.0) 1249 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:51:11.791 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:11 smithi078 bash[75299]: audit 2023-10-01T10:51:11.249938+0000 mon.a (mon.0) 1250 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:51:11.791 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:11 smithi078 bash[78274]: cluster 2023-10-01T10:51:09.945235+0000 mgr.y (mgr.25072) 1006 : cluster 0 pgmap v955: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:51:11.791 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:11 smithi078 bash[78274]: audit 2023-10-01T10:51:11.242995+0000 mon.a (mon.0) 1249 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:51:11.791 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:11 smithi078 bash[78274]: audit 2023-10-01T10:51:11.249938+0000 mon.a (mon.0) 1250 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:51:11.810 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:11.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:11.812 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:51:11.913 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:11.948 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:11.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:11.952 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:51:12.053 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:12.089 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:12.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:12.092 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:51:12.192 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:12.218 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:12.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:12.220 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:51:12.321 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:12.350 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:12.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:12.353 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:51:12.453 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:12.484 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:12.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:12.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:51:12.588 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:12.621 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:12.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:12.624 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:51:12.725 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:12.727 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:12 smithi078 bash[78274]: audit 2023-10-01T10:51:11.671288+0000 mon.a (mon.0) 1251 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:51:12.727 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:12 smithi078 bash[78274]: audit 2023-10-01T10:51:11.672416+0000 mon.a (mon.0) 1252 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:51:12.727 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:12 smithi078 bash[78274]: audit 2023-10-01T10:51:11.682138+0000 mon.a (mon.0) 1253 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:51:12.727 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:12 smithi078 bash[75299]: audit 2023-10-01T10:51:11.671288+0000 mon.a (mon.0) 1251 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:51:12.727 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:12 smithi078 bash[75299]: audit 2023-10-01T10:51:11.672416+0000 mon.a (mon.0) 1252 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:51:12.728 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:12 smithi078 bash[75299]: audit 2023-10-01T10:51:11.682138+0000 mon.a (mon.0) 1253 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:51:12.751 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:12.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:12.754 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:51:12.855 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:12.866 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:12 smithi173 bash[58816]: audit 2023-10-01T10:51:11.671288+0000 mon.a (mon.0) 1251 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:51:12.867 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:12 smithi173 bash[58816]: audit 2023-10-01T10:51:11.672416+0000 mon.a (mon.0) 1252 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:51:12.867 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:12 smithi173 bash[58816]: audit 2023-10-01T10:51:11.682138+0000 mon.a (mon.0) 1253 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:51:12.886 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:12.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:12.888 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:51:12.989 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:13.020 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:13.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:13.023 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:51:13.124 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:13.152 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:13.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:13.156 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:51:13.176 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T10:51:13.256 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:13.285 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:13.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:13.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:51:13.389 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:13.422 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:13.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:13.425 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:51:13.526 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:13.549 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:13.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:13.551 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:51:13.652 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:13.686 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:13.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:13.727 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:51:13.827 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:13.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:13 smithi078 bash[75299]: cluster 2023-10-01T10:51:11.945956+0000 mgr.y (mgr.25072) 1007 : cluster 0 pgmap v956: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:51:13.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:13 smithi078 bash[75299]: audit 2023-10-01T10:51:12.880042+0000 mon.a (mon.0) 1254 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:51:13.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:13 smithi078 bash[78274]: cluster 2023-10-01T10:51:11.945956+0000 mgr.y (mgr.25072) 1007 : cluster 0 pgmap v956: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:51:13.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:13 smithi078 bash[78274]: audit 2023-10-01T10:51:12.880042+0000 mon.a (mon.0) 1254 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:51:13.835 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:13 smithi173 bash[58816]: cluster 2023-10-01T10:51:11.945956+0000 mgr.y (mgr.25072) 1007 : cluster 0 pgmap v956: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:51:13.835 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:13 smithi173 bash[58816]: audit 2023-10-01T10:51:12.880042+0000 mon.a (mon.0) 1254 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:51:13.850 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:13.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:13.853 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:51:13.954 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:13.977 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:13.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:13.980 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:51:14.009 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:51:14.011 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:51:14.081 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:14.104 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:14.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:14.106 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:51:14.207 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:14.241 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:14.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:14.244 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:51:14.345 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:14.373 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:14.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:14.376 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:51:14.477 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:14.503 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:14.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:14.507 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:51:14.608 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:14.634 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:14.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:14.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:51:14.738 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:14.766 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:14.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:14.769 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:51:14.870 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:14.892 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:14.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:14.894 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:51:14.995 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:15.019 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:15.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:15.022 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:51:15.123 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:15.154 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:15.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:15.157 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:51:15.258 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:15.282 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:15.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:15.285 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:51:15.386 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:15.418 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:15.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:15.420 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:51:15.521 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:15.552 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:15.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:15.554 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:51:15.655 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:15.679 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:15.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:15.682 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:51:15.783 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:15.812 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:15.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:15.814 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:51:15.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:15 smithi173 bash[58816]: cluster 2023-10-01T10:51:13.946448+0000 mgr.y (mgr.25072) 1008 : cluster 0 pgmap v957: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:51:15.914 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:15.943 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:15.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:15.946 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:51:16.047 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:16.055 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:15 smithi078 bash[75299]: cluster 2023-10-01T10:51:13.946448+0000 mgr.y (mgr.25072) 1008 : cluster 0 pgmap v957: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:51:16.055 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:15 smithi078 bash[78274]: cluster 2023-10-01T10:51:13.946448+0000 mgr.y (mgr.25072) 1008 : cluster 0 pgmap v957: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:51:16.070 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:16.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:16.073 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:51:16.174 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:16.205 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:16.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:16.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:51:16.308 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:16.336 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:16.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:16.339 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:51:16.440 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:16.465 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:16.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:16.469 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:51:16.569 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:16.580 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:51:16.603 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:16.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:16.606 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:51:16.701 INFO:teuthology.orchestra.run.smithi078.stderr:enabled application 'rados' on pool 'unique_pool_6' 2023-10-01T10:51:16.707 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:16.730 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:16.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:16.732 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:51:16.782 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:16 smithi078 bash[75299]: audit 2023-10-01T10:51:16.578463+0000 mon.a (mon.0) 1255 : audit 1 from='client.? 172.21.15.78:0/163986095' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T10:51:16.832 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:16.839 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-10-01T10:51:16.840 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [], 2023-10-01T10:51:16.840 INFO:teuthology.orchestra.run.smithi173.stdout: "num_ops": 0 2023-10-01T10:51:16.840 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-10-01T10:51:16.845 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:16 smithi173 bash[58816]: audit 2023-10-01T10:51:16.578463+0000 mon.a (mon.0) 1255 : audit 1 from='client.? 172.21.15.78:0/163986095' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T10:51:16.860 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:16.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:16.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:51:16.963 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:16.996 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:16.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:16.999 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:51:17.064 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:16 smithi078 bash[78274]: audit 2023-10-01T10:51:16.578463+0000 mon.a (mon.0) 1255 : audit 1 from='client.? 172.21.15.78:0/163986095' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T10:51:17.099 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:17.127 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:17.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:17.130 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:51:17.231 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:17.264 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:17.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:17.268 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:51:17.368 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:17.394 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:17.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:17.396 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:51:17.497 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:17.523 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:17.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:17.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:51:17.627 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:17.654 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:17.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:17.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:51:17.676 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-01T10:51:17.758 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:17.770 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:17 smithi173 bash[58816]: cluster 2023-10-01T10:51:15.947160+0000 mgr.y (mgr.25072) 1009 : cluster 0 pgmap v958: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:51:17.771 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:17 smithi173 bash[58816]: cluster 2023-10-01T10:51:16.696306+0000 osd.5 (osd.5) 112 : cluster 3 failed to encode map e178 with expected crc 2023-10-01T10:51:17.771 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:17 smithi173 bash[58816]: audit 2023-10-01T10:51:16.696930+0000 mon.a (mon.0) 1256 : audit 1 from='client.? 172.21.15.78:0/163986095' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-01T10:51:17.771 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:17 smithi173 bash[58816]: cluster 2023-10-01T10:51:16.696985+0000 mon.a (mon.0) 1257 : cluster 0 osdmap e178: 8 total, 8 up, 8 in 2023-10-01T10:51:17.771 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:17 smithi173 bash[58816]: cluster 2023-10-01T10:51:16.697515+0000 osd.7 (osd.7) 131 : cluster 3 failed to encode map e178 with expected crc 2023-10-01T10:51:17.771 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:17 smithi173 bash[58816]: audit 2023-10-01T10:51:17.437096+0000 mon.a (mon.0) 1258 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:51:17.771 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:17 smithi173 bash[58816]: audit 2023-10-01T10:51:17.438316+0000 mon.a (mon.0) 1259 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:51:17.787 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:17.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:17.791 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:51:17.891 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:17.913 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:17.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:17.916 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:51:17.958 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:17 smithi078 bash[78274]: cluster 2023-10-01T10:51:15.947160+0000 mgr.y (mgr.25072) 1009 : cluster 0 pgmap v958: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:51:17.958 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:17 smithi078 bash[78274]: cluster 2023-10-01T10:51:16.696306+0000 osd.5 (osd.5) 112 : cluster 3 failed to encode map e178 with expected crc 2023-10-01T10:51:17.959 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:17 smithi078 bash[78274]: audit 2023-10-01T10:51:16.696930+0000 mon.a (mon.0) 1256 : audit 1 from='client.? 172.21.15.78:0/163986095' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-01T10:51:17.959 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:17 smithi078 bash[78274]: cluster 2023-10-01T10:51:16.696985+0000 mon.a (mon.0) 1257 : cluster 0 osdmap e178: 8 total, 8 up, 8 in 2023-10-01T10:51:17.959 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:17 smithi078 bash[78274]: cluster 2023-10-01T10:51:16.697515+0000 osd.7 (osd.7) 131 : cluster 3 failed to encode map e178 with expected crc 2023-10-01T10:51:17.959 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:17 smithi078 bash[78274]: audit 2023-10-01T10:51:17.437096+0000 mon.a (mon.0) 1258 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:51:17.959 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:17 smithi078 bash[78274]: audit 2023-10-01T10:51:17.438316+0000 mon.a (mon.0) 1259 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:51:17.959 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:17 smithi078 bash[75299]: cluster 2023-10-01T10:51:15.947160+0000 mgr.y (mgr.25072) 1009 : cluster 0 pgmap v958: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:51:17.960 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:17 smithi078 bash[75299]: cluster 2023-10-01T10:51:16.696306+0000 osd.5 (osd.5) 112 : cluster 3 failed to encode map e178 with expected crc 2023-10-01T10:51:17.960 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:17 smithi078 bash[75299]: audit 2023-10-01T10:51:16.696930+0000 mon.a (mon.0) 1256 : audit 1 from='client.? 172.21.15.78:0/163986095' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-01T10:51:17.960 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:17 smithi078 bash[75299]: cluster 2023-10-01T10:51:16.696985+0000 mon.a (mon.0) 1257 : cluster 0 osdmap e178: 8 total, 8 up, 8 in 2023-10-01T10:51:17.960 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:17 smithi078 bash[75299]: cluster 2023-10-01T10:51:16.697515+0000 osd.7 (osd.7) 131 : cluster 3 failed to encode map e178 with expected crc 2023-10-01T10:51:17.960 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:17 smithi078 bash[75299]: audit 2023-10-01T10:51:17.437096+0000 mon.a (mon.0) 1258 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:51:17.960 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:17 smithi078 bash[75299]: audit 2023-10-01T10:51:17.438316+0000 mon.a (mon.0) 1259 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:51:18.017 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:18.040 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:18.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:18.042 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:51:18.143 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:18.165 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:18.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:18.167 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:51:18.268 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:18.301 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:18.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:18.304 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:51:18.405 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:18.430 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:18.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:18.433 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:51:18.533 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:18.566 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:18.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:18.570 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:51:18.671 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:18.697 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:18.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:18.700 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:51:18.801 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:18.827 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:18.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:18.831 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:51:18.932 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:18.968 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:18.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:18.970 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:51:19.071 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:19.097 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:19.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:19.100 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:51:19.201 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:19.235 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:19.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:19.238 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:51:19.339 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:19.362 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:19.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:19.364 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:51:19.465 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:19.497 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:19.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:19.499 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:51:19.600 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:19.631 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:19.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:19.634 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:51:19.735 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:19.768 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:19.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:19.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:51:19.871 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:19.901 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:19.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:19.905 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:51:20.006 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:20.032 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:20.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:20.034 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:51:20.057 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:19 smithi078 bash[78274]: cluster 2023-10-01T10:51:17.947797+0000 mgr.y (mgr.25072) 1010 : cluster 0 pgmap v960: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:51:20.058 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:19 smithi078 bash[75299]: cluster 2023-10-01T10:51:17.947797+0000 mgr.y (mgr.25072) 1010 : cluster 0 pgmap v960: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:51:20.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:19 smithi173 bash[58816]: cluster 2023-10-01T10:51:17.947797+0000 mgr.y (mgr.25072) 1010 : cluster 0 pgmap v960: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:51:20.134 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:20.165 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:20.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:20.169 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:51:20.270 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:20.292 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:20.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:20.295 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:51:20.331 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:51:20 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:51:20] "GET /metrics HTTP/1.1" 200 36247 "" "Prometheus/2.33.4" 2023-10-01T10:51:20.396 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:20.420 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:20.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:20.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:51:20.523 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:20.548 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:20.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:20.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:51:20.651 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:20.681 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:20.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:20.685 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:51:20.785 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:20.818 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:20.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:20.821 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:51:20.922 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:20.948 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:20.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:20.951 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:51:21.051 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:21.077 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:21.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:21.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:51:21.180 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:21.209 DEBUG:teuthology.orchestra.run.smithi078:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_6 bench 90 write' 2023-10-01T10:51:21.211 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-01T10:51:21.211 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-01T10:51:21.215 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:21.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:21.218 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:51:21.253 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:hints = 1 2023-10-01T10:51:21.254 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-01T10:51:21.254 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Object prefix: benchmark_data_smithi078_157677 2023-10-01T10:51:21.254 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T10:51:21.254 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 0 0 0 0 0 0 - 0 2023-10-01T10:51:21.318 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:21.353 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:21.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:21.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:51:21.456 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:21.488 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:21.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:21.491 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:51:21.592 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:21.621 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:21.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:21.624 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:51:21.725 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:21.737 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:21 smithi173 bash[58816]: cluster 2023-10-01T10:51:19.948931+0000 mgr.y (mgr.25072) 1011 : cluster 0 pgmap v961: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:51:21.755 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:21.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:21.758 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:51:21.859 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:21.887 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:21.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:21.890 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:51:21.991 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:22.000 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:21 smithi078 bash[78274]: cluster 2023-10-01T10:51:19.948931+0000 mgr.y (mgr.25072) 1011 : cluster 0 pgmap v961: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:51:22.001 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:21 smithi078 bash[75299]: cluster 2023-10-01T10:51:19.948931+0000 mgr.y (mgr.25072) 1011 : cluster 0 pgmap v961: 89 pgs: 89 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-10-01T10:51:22.014 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:22.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:22.016 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:51:22.117 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:22.143 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:22.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:22.146 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:51:22.247 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:22.254 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 1 16 2465 2449 153.054 153.062 0.00572895 0.00651516 2023-10-01T10:51:22.272 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:22.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:22.274 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:51:22.375 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:22.398 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:22.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:22.402 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:51:22.452 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T10:51:22.502 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:22.534 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:22.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:22.537 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:51:22.638 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:22.667 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:22.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:22.670 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:51:22.771 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:22.793 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:22.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:22.795 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:51:22.896 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:22.927 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:22.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:22.930 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:51:23.030 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:23.059 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:23.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:23.062 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:51:23.163 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:23.188 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:23.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:23.193 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:51:23.254 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 2 16 4988 4972 155.364 157.688 0.00420329 0.00642068 2023-10-01T10:51:23.295 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:23.320 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:23.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:23.323 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:51:23.423 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:23.447 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:23.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:23.451 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:51:23.552 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:23.581 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:23.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:23.584 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:51:23.685 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:23.713 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:23.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:23.717 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:51:23.818 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:23.848 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:23.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:23.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:51:23.951 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:23.960 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:23 smithi078 bash[75299]: cluster 2023-10-01T10:51:21.949467+0000 mgr.y (mgr.25072) 1012 : cluster 0 pgmap v962: 89 pgs: 89 active+clean; 31 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 3.0 MiB/s wr, 48 op/s 2023-10-01T10:51:23.960 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:23 smithi078 bash[78274]: cluster 2023-10-01T10:51:21.949467+0000 mgr.y (mgr.25072) 1012 : cluster 0 pgmap v962: 89 pgs: 89 active+clean; 31 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 3.0 MiB/s wr, 48 op/s 2023-10-01T10:51:23.979 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:23.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:23.981 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:51:24.082 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:24.090 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:23 smithi173 bash[58816]: cluster 2023-10-01T10:51:21.949467+0000 mgr.y (mgr.25072) 1012 : cluster 0 pgmap v962: 89 pgs: 89 active+clean; 31 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 3.0 MiB/s wr, 48 op/s 2023-10-01T10:51:24.104 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:24.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:24.106 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:51:24.207 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:24.235 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:24.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:24.237 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:51:24.254 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 3 16 7537 7521 156.676 159.312 0.00879171 0.00637184 2023-10-01T10:51:24.339 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:24.372 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:24.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:24.375 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:51:24.476 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:24.505 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:24.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:24.507 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:51:24.608 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:24.635 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:24.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:24.638 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:51:24.738 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:24.764 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:24.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:24.766 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:51:24.867 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:24.895 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:24.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:24.898 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:51:24.999 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:25.036 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:25.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:25.038 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:51:25.139 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:25.167 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:25.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:25.172 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:51:25.254 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 4 16 10104 10088 157.612 160.438 0.0105709 0.00633472 2023-10-01T10:51:25.273 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:25.293 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:25.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:25.295 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:51:25.395 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:25.426 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:25.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:25.428 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:51:25.529 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:25.715 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:25 smithi173 bash[58816]: cluster 2023-10-01T10:51:23.950134+0000 mgr.y (mgr.25072) 1013 : cluster 0 pgmap v963: 89 pgs: 89 active+clean; 182 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 18 MiB/s wr, 289 op/s 2023-10-01T10:51:25.810 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:25.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:25.814 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:51:25.915 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:25.946 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:25.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:25.949 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:51:26.050 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:26.155 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:26 smithi078 bash[78274]: cluster 2023-10-01T10:51:23.950134+0000 mgr.y (mgr.25072) 1013 : cluster 0 pgmap v963: 89 pgs: 89 active+clean; 182 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 18 MiB/s wr, 289 op/s 2023-10-01T10:51:26.175 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:26.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:26.177 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:51:26.254 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 5 16 12035 12019 150.225 120.688 0.00723608 0.00664854 2023-10-01T10:51:26.278 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:26.301 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:26.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:26.303 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:51:26.404 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:26.434 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:26.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:26.438 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:51:26.539 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:26.567 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:26.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:26.569 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:51:26.578 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:26 smithi078 bash[75299]: cluster 2023-10-01T10:51:23.950134+0000 mgr.y (mgr.25072) 1013 : cluster 0 pgmap v963: 89 pgs: 89 active+clean; 182 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 18 MiB/s wr, 289 op/s 2023-10-01T10:51:26.670 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:26.689 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:26.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:26.692 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:51:26.793 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:26.822 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:26.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:26.826 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:51:26.926 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:26.959 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:26.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:26.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:51:27.063 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:27.089 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:27.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:27.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:51:27.193 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:27.220 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:27.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:27.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:51:27.254 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 6 16 14238 14222 148.133 137.688 0.00468334 0.00674166 2023-10-01T10:51:27.325 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:27.353 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:27.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:27.357 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:51:27.458 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:27.486 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:27.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:27.488 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:51:27.589 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:27.621 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:27.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:27.624 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:51:27.725 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:27.757 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:27.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:27.763 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:51:27.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:27 smithi078 bash[75299]: cluster 2023-10-01T10:51:25.950615+0000 mgr.y (mgr.25072) 1014 : cluster 0 pgmap v964: 89 pgs: 89 active+clean; 272 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 27 MiB/s wr, 433 op/s 2023-10-01T10:51:27.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:27 smithi078 bash[78274]: cluster 2023-10-01T10:51:25.950615+0000 mgr.y (mgr.25072) 1014 : cluster 0 pgmap v964: 89 pgs: 89 active+clean; 272 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 27 MiB/s wr, 433 op/s 2023-10-01T10:51:27.859 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:27 smithi173 bash[58816]: cluster 2023-10-01T10:51:25.950615+0000 mgr.y (mgr.25072) 1014 : cluster 0 pgmap v964: 89 pgs: 89 active+clean; 272 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 27 MiB/s wr, 433 op/s 2023-10-01T10:51:27.863 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:27.888 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:27.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:27.891 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:51:27.991 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:28.019 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:28.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:28.022 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:51:28.123 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:28.125 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-10-01T10:51:28.126 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [], 2023-10-01T10:51:28.126 INFO:teuthology.orchestra.run.smithi173.stdout: "complaint_time": 30, 2023-10-01T10:51:28.126 INFO:teuthology.orchestra.run.smithi173.stdout: "num_blocked_ops": 0 2023-10-01T10:51:28.126 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-10-01T10:51:28.147 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:28.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:28.151 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:51:28.253 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:28.255 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 7 16 16525 16509 147.389 142.938 0.0091272 0.00677692 2023-10-01T10:51:28.279 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:28.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:28.282 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:51:28.382 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:28.414 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:28.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:28.417 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:51:28.518 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:28.547 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:28.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:28.550 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:51:28.651 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:28.688 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:28.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:28.692 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:51:28.793 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:28.817 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:28.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:28.819 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:51:28.920 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:28.945 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:28.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:28.948 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:51:29.048 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:29.073 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:29.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:29.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:51:29.180 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:29.203 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:29.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:29.205 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:51:29.255 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 8 16 18231 18215 142.292 106.625 0.00459995 0.00699024 2023-10-01T10:51:29.306 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:29.334 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:29.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:29.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:51:29.439 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:29.465 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:29.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:29.471 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:51:29.572 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:29.574 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-01T10:51:29.645 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:29.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:29.650 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:51:29.750 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:29.764 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T10:51:29.775 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:29.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:29.780 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:51:29.826 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T10:51:29.862 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T10:51:29.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:29 smithi173 bash[58816]: cluster 2023-10-01T10:51:27.951153+0000 mgr.y (mgr.25072) 1015 : cluster 0 pgmap v965: 89 pgs: 89 active+clean; 596 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 53 MiB/s wr, 846 op/s 2023-10-01T10:51:29.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:29 smithi173 bash[58816]: audit 2023-10-01T10:51:29.700262+0000 mon.a (mon.0) 1260 : audit 0 from='client.? 172.21.15.78:0/113543518' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:51:29.880 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:29.908 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T10:51:29.916 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:29.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:29.921 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:51:29.946 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T10:51:29.991 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T10:51:30.021 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:30.035 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T10:51:30.051 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:30.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:30.056 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:51:30.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:29 smithi078 bash[78274]: cluster 2023-10-01T10:51:27.951153+0000 mgr.y (mgr.25072) 1015 : cluster 0 pgmap v965: 89 pgs: 89 active+clean; 596 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 53 MiB/s wr, 846 op/s 2023-10-01T10:51:30.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:29 smithi078 bash[78274]: audit 2023-10-01T10:51:29.700262+0000 mon.a (mon.0) 1260 : audit 0 from='client.? 172.21.15.78:0/113543518' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:51:30.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:29 smithi078 bash[75299]: cluster 2023-10-01T10:51:27.951153+0000 mgr.y (mgr.25072) 1015 : cluster 0 pgmap v965: 89 pgs: 89 active+clean; 596 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 53 MiB/s wr, 846 op/s 2023-10-01T10:51:30.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:29 smithi078 bash[75299]: audit 2023-10-01T10:51:29.700262+0000 mon.a (mon.0) 1260 : audit 0 from='client.? 172.21.15.78:0/113543518' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:51:30.084 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T10:51:30.090 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T10:51:30.090 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T10:51:30.090 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T10:51:30.090 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T10:51:30.090 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T10:51:30.090 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T10:51:30.091 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T10:51:30.091 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T10:51:30.156 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:30.181 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:30.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:30.185 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:51:30.255 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 9 16 19900 19884 138.071 104.312 0.0152936 0.00723437 2023-10-01T10:51:30.286 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:30.318 INFO:teuthology.orchestra.run.smithi078.stdout: 2023-10-01T10:51:30.319 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:30.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:30.320 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:51:30.323 INFO:teuthology.orchestra.run.smithi078.stderr:dumped all 2023-10-01T10:51:30.414 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:51:30 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:51:30] "GET /metrics HTTP/1.1" 200 36247 "" "Prometheus/2.33.4" 2023-10-01T10:51:30.421 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:30.454 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:30.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:30.456 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:51:30.557 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:30.582 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:30.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:30.586 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:51:30.686 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:30.729 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:30.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:30.738 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:51:30.839 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:30.874 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:30.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:30.877 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:51:30.977 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:31.005 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:31.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:31.009 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:51:31.110 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:31.144 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:31.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:31.148 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:51:31.249 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:31.255 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 10 16 21801 21785 136.144 118.812 0.0074595 0.00733894 2023-10-01T10:51:31.281 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:31.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:31.287 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:51:31.390 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:31.415 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:31.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:31.419 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:51:31.519 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:31.545 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:31.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:31.555 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:51:31.656 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:31.681 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:31.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:31.684 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:51:31.726 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:31 smithi173 bash[58816]: cluster 2023-10-01T10:51:29.952560+0000 mgr.y (mgr.25072) 1016 : cluster 0 pgmap v966: 89 pgs: 89 active+clean; 973 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-10-01T10:51:31.726 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:31 smithi173 bash[58816]: audit 2023-10-01T10:51:30.296890+0000 mgr.y (mgr.25072) 1017 : audit 0 from='client.36087 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:51:31.726 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:31 smithi173 bash[58816]: audit 2023-10-01T10:51:30.815678+0000 mon.b (mon.1) 45 : audit 1 from='client.? 172.21.15.78:0/1338046572' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:51:31.726 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:31 smithi173 bash[58816]: audit 2023-10-01T10:51:30.816598+0000 mon.a (mon.0) 1261 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:51:31.758 INFO:teuthology.orchestra.run.smithi078.stderr:noscrub is set 2023-10-01T10:51:31.785 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:31.795 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:31 smithi078 bash[75299]: cluster 2023-10-01T10:51:29.952560+0000 mgr.y (mgr.25072) 1016 : cluster 0 pgmap v966: 89 pgs: 89 active+clean; 973 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-10-01T10:51:31.796 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:31 smithi078 bash[75299]: audit 2023-10-01T10:51:30.296890+0000 mgr.y (mgr.25072) 1017 : audit 0 from='client.36087 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:51:31.796 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:31 smithi078 bash[75299]: audit 2023-10-01T10:51:30.815678+0000 mon.b (mon.1) 45 : audit 1 from='client.? 172.21.15.78:0/1338046572' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:51:31.796 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:31 smithi078 bash[75299]: audit 2023-10-01T10:51:30.816598+0000 mon.a (mon.0) 1261 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:51:31.796 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:31 smithi078 bash[78274]: cluster 2023-10-01T10:51:29.952560+0000 mgr.y (mgr.25072) 1016 : cluster 0 pgmap v966: 89 pgs: 89 active+clean; 973 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-10-01T10:51:31.796 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:31 smithi078 bash[78274]: audit 2023-10-01T10:51:30.296890+0000 mgr.y (mgr.25072) 1017 : audit 0 from='client.36087 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:51:31.796 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:31 smithi078 bash[78274]: audit 2023-10-01T10:51:30.815678+0000 mon.b (mon.1) 45 : audit 1 from='client.? 172.21.15.78:0/1338046572' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:51:31.796 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:31 smithi078 bash[78274]: audit 2023-10-01T10:51:30.816598+0000 mon.a (mon.0) 1261 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:51:31.813 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:31.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:31.817 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:51:31.917 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:31.942 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:31.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:31.947 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:51:32.048 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:32.073 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:32.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:32.075 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:51:32.176 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:32.204 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:32.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:32.207 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:51:32.255 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 11 16 23842 23826 135.363 127.562 0.00412247 0.00738102 2023-10-01T10:51:32.307 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:32.333 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:32.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:32.337 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:51:32.438 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:32.466 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:32.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:32.467 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:51:32.568 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:32.597 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:32.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:32.600 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:51:32.701 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:32.732 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:32.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:32.734 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:51:32.835 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:32.852 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:32.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:32.854 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:51:32.955 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:32.962 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:32 smithi078 bash[78274]: cluster 2023-10-01T10:51:31.722668+0000 mon.a (mon.0) 1262 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:51:32.963 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:32 smithi078 bash[78274]: cluster 2023-10-01T10:51:31.748299+0000 osd.6 (osd.6) 96 : cluster 3 failed to encode map e179 with expected crc 2023-10-01T10:51:32.963 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:32 smithi078 bash[78274]: audit 2023-10-01T10:51:31.748662+0000 mon.a (mon.0) 1263 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T10:51:32.963 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:32 smithi078 bash[78274]: cluster 2023-10-01T10:51:31.748755+0000 mon.a (mon.0) 1264 : cluster 0 osdmap e179: 8 total, 8 up, 8 in 2023-10-01T10:51:32.963 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:32 smithi078 bash[78274]: cluster 2023-10-01T10:51:31.756363+0000 osd.7 (osd.7) 132 : cluster 3 failed to encode map e179 with expected crc 2023-10-01T10:51:32.963 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:32 smithi078 bash[78274]: cluster 2023-10-01T10:51:31.762438+0000 osd.7 (osd.7) 133 : cluster 3 failed to encode map e179 with expected crc 2023-10-01T10:51:32.963 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:32 smithi078 bash[78274]: audit 2023-10-01T10:51:32.436890+0000 mon.a (mon.0) 1265 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:51:32.963 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:32 smithi078 bash[78274]: audit 2023-10-01T10:51:32.438363+0000 mon.a (mon.0) 1266 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:51:32.963 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:32 smithi078 bash[75299]: cluster 2023-10-01T10:51:31.722668+0000 mon.a (mon.0) 1262 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:51:32.964 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:32 smithi078 bash[75299]: cluster 2023-10-01T10:51:31.748299+0000 osd.6 (osd.6) 96 : cluster 3 failed to encode map e179 with expected crc 2023-10-01T10:51:32.964 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:32 smithi078 bash[75299]: audit 2023-10-01T10:51:31.748662+0000 mon.a (mon.0) 1263 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T10:51:32.964 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:32 smithi078 bash[75299]: cluster 2023-10-01T10:51:31.748755+0000 mon.a (mon.0) 1264 : cluster 0 osdmap e179: 8 total, 8 up, 8 in 2023-10-01T10:51:32.964 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:32 smithi078 bash[75299]: cluster 2023-10-01T10:51:31.756363+0000 osd.7 (osd.7) 132 : cluster 3 failed to encode map e179 with expected crc 2023-10-01T10:51:32.964 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:32 smithi078 bash[75299]: cluster 2023-10-01T10:51:31.762438+0000 osd.7 (osd.7) 133 : cluster 3 failed to encode map e179 with expected crc 2023-10-01T10:51:32.964 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:32 smithi078 bash[75299]: audit 2023-10-01T10:51:32.436890+0000 mon.a (mon.0) 1265 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:51:32.964 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:32 smithi078 bash[75299]: audit 2023-10-01T10:51:32.438363+0000 mon.a (mon.0) 1266 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:51:32.991 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:32.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:32.995 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:51:32.996 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:32 smithi173 bash[58816]: cluster 2023-10-01T10:51:31.722668+0000 mon.a (mon.0) 1262 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:51:32.996 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:32 smithi173 bash[58816]: cluster 2023-10-01T10:51:31.748299+0000 osd.6 (osd.6) 96 : cluster 3 failed to encode map e179 with expected crc 2023-10-01T10:51:32.996 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:32 smithi173 bash[58816]: audit 2023-10-01T10:51:31.748662+0000 mon.a (mon.0) 1263 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T10:51:32.996 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:32 smithi173 bash[58816]: cluster 2023-10-01T10:51:31.748755+0000 mon.a (mon.0) 1264 : cluster 0 osdmap e179: 8 total, 8 up, 8 in 2023-10-01T10:51:32.996 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:32 smithi173 bash[58816]: cluster 2023-10-01T10:51:31.756363+0000 osd.7 (osd.7) 132 : cluster 3 failed to encode map e179 with expected crc 2023-10-01T10:51:32.996 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:32 smithi173 bash[58816]: cluster 2023-10-01T10:51:31.762438+0000 osd.7 (osd.7) 133 : cluster 3 failed to encode map e179 with expected crc 2023-10-01T10:51:32.996 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:32 smithi173 bash[58816]: audit 2023-10-01T10:51:32.436890+0000 mon.a (mon.0) 1265 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:51:32.996 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:32 smithi173 bash[58816]: audit 2023-10-01T10:51:32.438363+0000 mon.a (mon.0) 1266 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:51:33.096 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:33.121 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:33.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:33.125 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:51:33.226 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:33.251 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:33.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:33.254 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:51:33.255 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 12 16 25600 25584 133.238 109.875 0.00473025 0.00749644 2023-10-01T10:51:33.354 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:33.378 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:33.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:33.380 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:51:33.481 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:33.510 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:33.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:33.513 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:51:33.614 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:33.644 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:33.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:33.649 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:51:33.750 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:33.780 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:33.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:33.784 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:51:33.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:33 smithi078 bash[78274]: cluster 2023-10-01T10:51:31.750922+0000 osd.5 (osd.5) 113 : cluster 3 failed to encode map e179 with expected crc 2023-10-01T10:51:33.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:33 smithi078 bash[78274]: cluster 2023-10-01T10:51:31.754414+0000 osd.5 (osd.5) 114 : cluster 3 failed to encode map e179 with expected crc 2023-10-01T10:51:33.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:33 smithi078 bash[78274]: cluster 2023-10-01T10:51:31.952985+0000 mgr.y (mgr.25072) 1018 : cluster 0 pgmap v968: 89 pgs: 89 active+clean; 1.2 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 120 MiB/s wr, 1.93k op/s 2023-10-01T10:51:33.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:33 smithi078 bash[75299]: cluster 2023-10-01T10:51:31.750922+0000 osd.5 (osd.5) 113 : cluster 3 failed to encode map e179 with expected crc 2023-10-01T10:51:33.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:33 smithi078 bash[75299]: cluster 2023-10-01T10:51:31.754414+0000 osd.5 (osd.5) 114 : cluster 3 failed to encode map e179 with expected crc 2023-10-01T10:51:33.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:33 smithi078 bash[75299]: cluster 2023-10-01T10:51:31.952985+0000 mgr.y (mgr.25072) 1018 : cluster 0 pgmap v968: 89 pgs: 89 active+clean; 1.2 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 120 MiB/s wr, 1.93k op/s 2023-10-01T10:51:33.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:33 smithi173 bash[58816]: cluster 2023-10-01T10:51:31.750922+0000 osd.5 (osd.5) 113 : cluster 3 failed to encode map e179 with expected crc 2023-10-01T10:51:33.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:33 smithi173 bash[58816]: cluster 2023-10-01T10:51:31.754414+0000 osd.5 (osd.5) 114 : cluster 3 failed to encode map e179 with expected crc 2023-10-01T10:51:33.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:33 smithi173 bash[58816]: cluster 2023-10-01T10:51:31.952985+0000 mgr.y (mgr.25072) 1018 : cluster 0 pgmap v968: 89 pgs: 89 active+clean; 1.2 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 120 MiB/s wr, 1.93k op/s 2023-10-01T10:51:33.885 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:33.912 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:33.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:33.916 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:51:34.017 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:34.046 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:34.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:34.048 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:51:34.149 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:34.177 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:34.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:34.183 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:51:34.255 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 13 16 27263 27247 130.983 103.938 0.00206156 0.00761732 2023-10-01T10:51:34.284 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:34.310 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:34.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:34.312 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:51:34.413 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:34.442 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T10:51:34.443 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:34.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:34.446 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:51:34.546 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:34.573 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:34.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:34.579 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:51:34.679 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:34.715 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:34.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:34.717 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:51:34.815 INFO:teuthology.orchestra.run.smithi078.stdout:{"pg_ready":true,"pg_map":{"version":966,"stamp":"2023-10-01T10:51:29.952023+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1020336032,"num_objects":15584,"num_object_clones":0,"num_object_copies":46752,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":15584,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":15821,"num_write_kb":998286,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":15755,"ondisk_log_size":15755,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":7240812,"kb_used_data":2856940,"kb_used_omap":30,"kb_used_meta":4383777,"kb_avail":742720404,"statfs":{"total":767960285184,"available":760545693696,"internally_reserved":0,"allocated":2925506560,"data_stored":2920513281,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31502,"internal_metadata":4488987890},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[7,6,5,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1019740160,"num_objects":15560,"num_object_clones":0,"num_object_copies":46680,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":15560,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15560,"num_write_kb":995840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004654"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":169,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:05.976834+0000","last_change":"2023-10-01T10:38:16.032258+0000","last_active":"2023-10-01T10:51:05.976834+0000","last_peered":"2023-10-01T10:51:05.976834+0000","last_clean":"2023-10-01T10:51:05.976834+0000","last_became_active":"2023-10-01T10:38:16.031633+0000","last_became_peered":"2023-10-01T10:38:16.031633+0000","last_unstale":"2023-10-01T10:51:05.976834+0000","last_undegraded":"2023-10-01T10:51:05.976834+0000","last_fullsized":"2023-10-01T10:51:05.976834+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:17:02.446430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":157,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:05.975996+0000","last_change":"2023-10-01T10:38:16.025881+0000","last_active":"2023-10-01T10:51:05.975996+0000","last_peered":"2023-10-01T10:51:05.975996+0000","last_clean":"2023-10-01T10:51:05.975996+0000","last_became_active":"2023-10-01T10:38:16.025518+0000","last_became_peered":"2023-10-01T10:38:16.025518+0000","last_unstale":"2023-10-01T10:51:05.975996+0000","last_undegraded":"2023-10-01T10:51:05.975996+0000","last_fullsized":"2023-10-01T10:51:05.975996+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:48:37.826604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":135,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:06.145205+0000","last_change":"2023-10-01T10:38:16.025075+0000","last_active":"2023-10-01T10:51:06.145205+0000","last_peered":"2023-10-01T10:51:06.145205+0000","last_clean":"2023-10-01T10:51:06.145205+0000","last_became_active":"2023-10-01T10:38:16.024735+0000","last_became_peered":"2023-10-01T10:38:16.024735+0000","last_unstale":"2023-10-01T10:51:06.145205+0000","last_undegraded":"2023-10-01T10:51:06.145205+0000","last_fullsized":"2023-10-01T10:51:06.145205+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:38:01.450741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":140,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:06.144449+0000","last_change":"2023-10-01T10:37:42.772008+0000","last_active":"2023-10-01T10:51:06.144449+0000","last_peered":"2023-10-01T10:51:06.144449+0000","last_clean":"2023-10-01T10:51:06.144449+0000","last_became_active":"2023-10-01T10:37:42.771244+0000","last_became_peered":"2023-10-01T10:37:42.771244+0000","last_unstale":"2023-10-01T10:51:06.144449+0000","last_undegraded":"2023-10-01T10:51:06.144449+0000","last_fullsized":"2023-10-01T10:51:06.144449+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:38:55.180790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":158,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:05.975102+0000","last_change":"2023-10-01T10:37:42.766600+0000","last_active":"2023-10-01T10:51:05.975102+0000","last_peered":"2023-10-01T10:51:05.975102+0000","last_clean":"2023-10-01T10:51:05.975102+0000","last_became_active":"2023-10-01T10:37:42.766051+0000","last_became_peered":"2023-10-01T10:37:42.766051+0000","last_unstale":"2023-10-01T10:51:05.975102+0000","last_undegraded":"2023-10-01T10:51:05.975102+0000","last_fullsized":"2023-10-01T10:51:05.975102+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:51:17.707963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":143,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:06.149365+0000","last_change":"2023-10-01T10:32:31.575109+0000","last_active":"2023-10-01T10:51:06.149365+0000","last_peered":"2023-10-01T10:51:06.149365+0000","last_clean":"2023-10-01T10:51:06.149365+0000","last_became_active":"2023-10-01T10:32:31.574729+0000","last_became_peered":"2023-10-01T10:32:31.574729+0000","last_unstale":"2023-10-01T10:51:06.149365+0000","last_undegraded":"2023-10-01T10:51:06.149365+0000","last_fullsized":"2023-10-01T10:51:06.149365+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":137,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:06.144259+0000","last_change":"2023-10-01T10:37:42.766667+0000","last_active":"2023-10-01T10:51:06.144259+0000","last_peered":"2023-10-01T10:51:06.144259+0000","last_clean":"2023-10-01T10:51:06.144259+0000","last_became_active":"2023-10-01T10:37:42.766369+0000","last_became_peered":"2023-10-01T10:37:42.766369+0000","last_unstale":"2023-10-01T10:51:06.144259+0000","last_undegraded":"2023-10-01T10:51:06.144259+0000","last_fullsized":"2023-10-01T10:51:06.144259+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:02.708788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":142,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:05.974942+0000","last_change":"2023-10-01T10:37:42.765925+0000","last_active":"2023-10-01T10:51:05.974942+0000","last_peered":"2023-10-01T10:51:05.974942+0000","last_clean":"2023-10-01T10:51:05.974942+0000","last_became_active":"2023-10-01T10:37:42.765218+0000","last_became_peered":"2023-10-01T10:37:42.765218+0000","last_unstale":"2023-10-01T10:51:05.974942+0000","last_undegraded":"2023-10-01T10:51:05.974942+0000","last_fullsized":"2023-10-01T10:51:05.974942+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T22:10:34.338031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":145,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:06.147386+0000","last_change":"2023-10-01T10:32:31.579805+0000","last_active":"2023-10-01T10:51:06.147386+0000","last_peered":"2023-10-01T10:51:06.147386+0000","last_clean":"2023-10-01T10:51:06.147386+0000","last_became_active":"2023-10-01T10:32:31.579423+0000","last_became_peered":"2023-10-01T10:32:31.579423+0000","last_unstale":"2023-10-01T10:51:06.147386+0000","last_undegraded":"2023-10-01T10:51:06.147386+0000","last_fullsized":"2023-10-01T10:51:06.147386+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":126,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:05.975279+0000","last_change":"2023-10-01T10:37:42.772531+0000","last_active":"2023-10-01T10:51:05.975279+0000","last_peered":"2023-10-01T10:51:05.975279+0000","last_clean":"2023-10-01T10:51:05.975279+0000","last_became_active":"2023-10-01T10:37:42.772216+0000","last_became_peered":"2023-10-01T10:37:42.772216+0000","last_unstale":"2023-10-01T10:51:05.975279+0000","last_undegraded":"2023-10-01T10:51:05.975279+0000","last_fullsized":"2023-10-01T10:51:05.975279+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:01:53.373628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":162,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:05.980900+0000","last_change":"2023-10-01T10:38:16.027085+0000","last_active":"2023-10-01T10:51:05.980900+0000","last_peered":"2023-10-01T10:51:05.980900+0000","last_clean":"2023-10-01T10:51:05.980900+0000","last_became_active":"2023-10-01T10:38:16.026555+0000","last_became_peered":"2023-10-01T10:38:16.026555+0000","last_unstale":"2023-10-01T10:51:05.980900+0000","last_undegraded":"2023-10-01T10:51:05.980900+0000","last_fullsized":"2023-10-01T10:51:05.980900+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":121,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:11.020653+0000","last_change":"2023-10-01T10:37:05.018755+0000","last_active":"2023-10-01T10:51:11.020653+0000","last_peered":"2023-10-01T10:51:11.020653+0000","last_clean":"2023-10-01T10:51:11.020653+0000","last_became_active":"2023-10-01T10:37:05.018446+0000","last_became_peered":"2023-10-01T10:37:05.018446+0000","last_unstale":"2023-10-01T10:51:11.020653+0000","last_undegraded":"2023-10-01T10:51:11.020653+0000","last_fullsized":"2023-10-01T10:51:11.020653+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:13:53.740503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":163,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:05.977496+0000","last_change":"2023-10-01T10:38:16.030826+0000","last_active":"2023-10-01T10:51:05.977496+0000","last_peered":"2023-10-01T10:51:05.977496+0000","last_clean":"2023-10-01T10:51:05.977496+0000","last_became_active":"2023-10-01T10:38:16.030277+0000","last_became_peered":"2023-10-01T10:38:16.030277+0000","last_unstale":"2023-10-01T10:51:05.977496+0000","last_undegraded":"2023-10-01T10:51:05.977496+0000","last_fullsized":"2023-10-01T10:51:05.977496+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:31:43.040424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":137,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:05.976913+0000","last_change":"2023-10-01T10:38:16.034741+0000","last_active":"2023-10-01T10:51:05.976913+0000","last_peered":"2023-10-01T10:51:05.976913+0000","last_clean":"2023-10-01T10:51:05.976913+0000","last_became_active":"2023-10-01T10:38:16.034407+0000","last_became_peered":"2023-10-01T10:38:16.034407+0000","last_unstale":"2023-10-01T10:51:05.976913+0000","last_undegraded":"2023-10-01T10:51:05.976913+0000","last_fullsized":"2023-10-01T10:51:05.976913+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:47:29.911477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":161,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:05.980386+0000","last_change":"2023-10-01T10:38:16.026777+0000","last_active":"2023-10-01T10:51:05.980386+0000","last_peered":"2023-10-01T10:51:05.980386+0000","last_clean":"2023-10-01T10:51:05.980386+0000","last_became_active":"2023-10-01T10:38:16.026371+0000","last_became_peered":"2023-10-01T10:38:16.026371+0000","last_unstale":"2023-10-01T10:51:05.980386+0000","last_undegraded":"2023-10-01T10:51:05.980386+0000","last_fullsized":"2023-10-01T10:51:05.980386+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":165,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:05.975277+0000","last_change":"2023-10-01T10:37:42.772319+0000","last_active":"2023-10-01T10:51:05.975277+0000","last_peered":"2023-10-01T10:51:05.975277+0000","last_clean":"2023-10-01T10:51:05.975277+0000","last_became_active":"2023-10-01T10:37:42.771977+0000","last_became_peered":"2023-10-01T10:37:42.771977+0000","last_unstale":"2023-10-01T10:51:05.975277+0000","last_undegraded":"2023-10-01T10:51:05.975277+0000","last_fullsized":"2023-10-01T10:51:05.975277+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:45:12.783214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":137,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:06.143877+0000","last_change":"2023-10-01T10:37:42.767706+0000","last_active":"2023-10-01T10:51:06.143877+0000","last_peered":"2023-10-01T10:51:06.143877+0000","last_clean":"2023-10-01T10:51:06.143877+0000","last_became_active":"2023-10-01T10:37:42.767211+0000","last_became_peered":"2023-10-01T10:37:42.767211+0000","last_unstale":"2023-10-01T10:51:06.143877+0000","last_undegraded":"2023-10-01T10:51:06.143877+0000","last_fullsized":"2023-10-01T10:51:06.143877+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:37:20.357407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":144,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:06.147676+0000","last_change":"2023-10-01T10:37:42.768642+0000","last_active":"2023-10-01T10:51:06.147676+0000","last_peered":"2023-10-01T10:51:06.147676+0000","last_clean":"2023-10-01T10:51:06.147676+0000","last_became_active":"2023-10-01T10:37:42.768239+0000","last_became_peered":"2023-10-01T10:37:42.768239+0000","last_unstale":"2023-10-01T10:51:06.147676+0000","last_undegraded":"2023-10-01T10:51:06.147676+0000","last_fullsized":"2023-10-01T10:51:06.147676+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":144,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:05.979887+0000","last_change":"2023-10-01T10:38:16.028563+0000","last_active":"2023-10-01T10:51:05.979887+0000","last_peered":"2023-10-01T10:51:05.979887+0000","last_clean":"2023-10-01T10:51:05.979887+0000","last_became_active":"2023-10-01T10:38:16.028162+0000","last_became_peered":"2023-10-01T10:38:16.028162+0000","last_unstale":"2023-10-01T10:51:05.979887+0000","last_undegraded":"2023-10-01T10:51:05.979887+0000","last_fullsized":"2023-10-01T10:51:05.979887+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":138,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:06.144543+0000","last_change":"2023-10-01T10:37:42.772247+0000","last_active":"2023-10-01T10:51:06.144543+0000","last_peered":"2023-10-01T10:51:06.144543+0000","last_clean":"2023-10-01T10:51:06.144543+0000","last_became_active":"2023-10-01T10:37:42.771697+0000","last_became_peered":"2023-10-01T10:37:42.771697+0000","last_unstale":"2023-10-01T10:51:06.144543+0000","last_undegraded":"2023-10-01T10:51:06.144543+0000","last_fullsized":"2023-10-01T10:51:06.144543+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:23:15.018248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":152,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:06.147957+0000","last_change":"2023-10-01T10:28:20.211798+0000","last_active":"2023-10-01T10:51:06.147957+0000","last_peered":"2023-10-01T10:51:06.147957+0000","last_clean":"2023-10-01T10:51:06.147957+0000","last_became_active":"2023-10-01T10:28:20.210791+0000","last_became_peered":"2023-10-01T10:28:20.210791+0000","last_unstale":"2023-10-01T10:51:06.147957+0000","last_undegraded":"2023-10-01T10:51:06.147957+0000","last_fullsized":"2023-10-01T10:51:06.147957+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":142,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:05.975925+0000","last_change":"2023-10-01T10:37:42.773127+0000","last_active":"2023-10-01T10:51:05.975925+0000","last_peered":"2023-10-01T10:51:05.975925+0000","last_clean":"2023-10-01T10:51:05.975925+0000","last_became_active":"2023-10-01T10:37:42.772102+0000","last_became_peered":"2023-10-01T10:37:42.772102+0000","last_unstale":"2023-10-01T10:51:05.975925+0000","last_undegraded":"2023-10-01T10:51:05.975925+0000","last_fullsized":"2023-10-01T10:51:05.975925+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:23:26.202295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"178'1145","reported_seq":1158,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:29.521522+0000","last_change":"2023-10-01T10:51:01.948311+0000","last_active":"2023-10-01T10:51:29.521522+0000","last_peered":"2023-10-01T10:51:29.521522+0000","last_clean":"2023-10-01T10:51:29.521522+0000","last_became_active":"2023-10-01T10:51:01.947921+0000","last_became_peered":"2023-10-01T10:51:01.947921+0000","last_unstale":"2023-10-01T10:51:29.521522+0000","last_undegraded":"2023-10-01T10:51:29.521522+0000","last_fullsized":"2023-10-01T10:51:29.521522+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":1145,"log_dups_size":0,"ondisk_log_size":1145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":75038720,"num_objects":1145,"num_object_clones":0,"num_object_copies":3435,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1145,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1145,"num_write_kb":73280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":160,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:05.981517+0000","last_change":"2023-10-01T10:38:16.026955+0000","last_active":"2023-10-01T10:51:05.981517+0000","last_peered":"2023-10-01T10:51:05.981517+0000","last_clean":"2023-10-01T10:51:05.981517+0000","last_became_active":"2023-10-01T10:38:16.026552+0000","last_became_peered":"2023-10-01T10:38:16.026552+0000","last_unstale":"2023-10-01T10:51:05.981517+0000","last_undegraded":"2023-10-01T10:51:05.981517+0000","last_fullsized":"2023-10-01T10:51:05.981517+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":129,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:06.146872+0000","last_change":"2023-10-01T10:15:40.318702+0000","last_active":"2023-10-01T10:51:06.146872+0000","last_peered":"2023-10-01T10:51:06.146872+0000","last_clean":"2023-10-01T10:51:06.146872+0000","last_became_active":"2023-10-01T10:15:40.318333+0000","last_became_peered":"2023-10-01T10:15:40.318333+0000","last_unstale":"2023-10-01T10:51:06.146872+0000","last_undegraded":"2023-10-01T10:51:06.146872+0000","last_fullsized":"2023-10-01T10:51:06.146872+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"178'1070","reported_seq":1084,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:29.081084+0000","last_change":"2023-10-01T10:51:01.947670+0000","last_active":"2023-10-01T10:51:29.081084+0000","last_peered":"2023-10-01T10:51:29.081084+0000","last_clean":"2023-10-01T10:51:29.081084+0000","last_became_active":"2023-10-01T10:51:01.947301+0000","last_became_peered":"2023-10-01T10:51:01.947301+0000","last_unstale":"2023-10-01T10:51:29.081084+0000","last_undegraded":"2023-10-01T10:51:29.081084+0000","last_fullsized":"2023-10-01T10:51:29.081084+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":1070,"log_dups_size":0,"ondisk_log_size":1070,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T22:33:54.269148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70123520,"num_objects":1070,"num_object_clones":0,"num_object_copies":3210,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1070,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1070,"num_write_kb":68480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":161,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:05.981237+0000","last_change":"2023-10-01T10:38:16.027753+0000","last_active":"2023-10-01T10:51:05.981237+0000","last_peered":"2023-10-01T10:51:05.981237+0000","last_clean":"2023-10-01T10:51:05.981237+0000","last_became_active":"2023-10-01T10:38:16.027384+0000","last_became_peered":"2023-10-01T10:38:16.027384+0000","last_unstale":"2023-10-01T10:51:05.981237+0000","last_undegraded":"2023-10-01T10:51:05.981237+0000","last_fullsized":"2023-10-01T10:51:05.981237+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":172,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:14.664273+0000","last_change":"2023-10-01T10:38:16.033203+0000","last_active":"2023-10-01T10:51:14.664273+0000","last_peered":"2023-10-01T10:51:14.664273+0000","last_clean":"2023-10-01T10:51:14.664273+0000","last_became_active":"2023-10-01T10:38:16.032681+0000","last_became_peered":"2023-10-01T10:38:16.032681+0000","last_unstale":"2023-10-01T10:51:14.664273+0000","last_undegraded":"2023-10-01T10:51:14.664273+0000","last_fullsized":"2023-10-01T10:51:14.664273+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:42:36.642284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"178'1089","reported_seq":1102,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:28.920544+0000","last_change":"2023-10-01T10:51:01.956461+0000","last_active":"2023-10-01T10:51:28.920544+0000","last_peered":"2023-10-01T10:51:28.920544+0000","last_clean":"2023-10-01T10:51:28.920544+0000","last_became_active":"2023-10-01T10:51:01.956123+0000","last_became_peered":"2023-10-01T10:51:01.956123+0000","last_unstale":"2023-10-01T10:51:28.920544+0000","last_undegraded":"2023-10-01T10:51:28.920544+0000","last_fullsized":"2023-10-01T10:51:28.920544+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":1089,"log_dups_size":0,"ondisk_log_size":1089,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71368704,"num_objects":1089,"num_object_clones":0,"num_object_copies":3267,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1089,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1089,"num_write_kb":69696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":130,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:05.980644+0000","last_change":"2023-10-01T10:15:39.293587+0000","last_active":"2023-10-01T10:51:05.980644+0000","last_peered":"2023-10-01T10:51:05.980644+0000","last_clean":"2023-10-01T10:51:05.980644+0000","last_became_active":"2023-10-01T10:15:39.293273+0000","last_became_peered":"2023-10-01T10:15:39.293273+0000","last_unstale":"2023-10-01T10:51:05.980644+0000","last_undegraded":"2023-10-01T10:51:05.980644+0000","last_fullsized":"2023-10-01T10:51:05.980644+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":143,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:06.148902+0000","last_change":"2023-10-01T10:32:31.571971+0000","last_active":"2023-10-01T10:51:06.148902+0000","last_peered":"2023-10-01T10:51:06.148902+0000","last_clean":"2023-10-01T10:51:06.148902+0000","last_became_active":"2023-10-01T10:32:31.571410+0000","last_became_peered":"2023-10-01T10:32:31.571410+0000","last_unstale":"2023-10-01T10:51:06.148902+0000","last_undegraded":"2023-10-01T10:51:06.148902+0000","last_fullsized":"2023-10-01T10:51:06.148902+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"178'1131","reported_seq":1144,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:28.939666+0000","last_change":"2023-10-01T10:51:01.955379+0000","last_active":"2023-10-01T10:51:28.939666+0000","last_peered":"2023-10-01T10:51:28.939666+0000","last_clean":"2023-10-01T10:51:28.939666+0000","last_became_active":"2023-10-01T10:51:01.954911+0000","last_became_peered":"2023-10-01T10:51:01.954911+0000","last_unstale":"2023-10-01T10:51:28.939666+0000","last_undegraded":"2023-10-01T10:51:28.939666+0000","last_fullsized":"2023-10-01T10:51:28.939666+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":1131,"log_dups_size":0,"ondisk_log_size":1131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":74121216,"num_objects":1131,"num_object_clones":0,"num_object_copies":3393,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1131,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1131,"num_write_kb":72384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":145,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:06.147365+0000","last_change":"2023-10-01T10:37:42.765733+0000","last_active":"2023-10-01T10:51:06.147365+0000","last_peered":"2023-10-01T10:51:06.147365+0000","last_clean":"2023-10-01T10:51:06.147365+0000","last_became_active":"2023-10-01T10:37:42.765311+0000","last_became_peered":"2023-10-01T10:37:42.765311+0000","last_unstale":"2023-10-01T10:51:06.147365+0000","last_undegraded":"2023-10-01T10:51:06.147365+0000","last_fullsized":"2023-10-01T10:51:06.147365+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":154,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:06.143789+0000","last_change":"2023-10-01T10:38:16.026348+0000","last_active":"2023-10-01T10:51:06.143789+0000","last_peered":"2023-10-01T10:51:06.143789+0000","last_clean":"2023-10-01T10:51:06.143789+0000","last_became_active":"2023-10-01T10:38:16.025747+0000","last_became_peered":"2023-10-01T10:38:16.025747+0000","last_unstale":"2023-10-01T10:51:06.143789+0000","last_undegraded":"2023-10-01T10:51:06.143789+0000","last_fullsized":"2023-10-01T10:51:06.143789+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:28:18.934480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"178'1127","reported_seq":1140,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:28.833871+0000","last_change":"2023-10-01T10:51:01.954225+0000","last_active":"2023-10-01T10:51:28.833871+0000","last_peered":"2023-10-01T10:51:28.833871+0000","last_clean":"2023-10-01T10:51:28.833871+0000","last_became_active":"2023-10-01T10:51:01.953830+0000","last_became_peered":"2023-10-01T10:51:01.953830+0000","last_unstale":"2023-10-01T10:51:28.833871+0000","last_undegraded":"2023-10-01T10:51:28.833871+0000","last_fullsized":"2023-10-01T10:51:28.833871+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":1127,"log_dups_size":0,"ondisk_log_size":1127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73859072,"num_objects":1127,"num_object_clones":0,"num_object_copies":3381,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1127,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1127,"num_write_kb":72128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":146,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:11.743992+0000","last_change":"2023-10-01T10:38:16.022493+0000","last_active":"2023-10-01T10:51:11.743992+0000","last_peered":"2023-10-01T10:51:11.743992+0000","last_clean":"2023-10-01T10:51:11.743992+0000","last_became_active":"2023-10-01T10:38:16.022033+0000","last_became_peered":"2023-10-01T10:38:16.022033+0000","last_unstale":"2023-10-01T10:51:11.743992+0000","last_undegraded":"2023-10-01T10:51:11.743992+0000","last_fullsized":"2023-10-01T10:51:11.743992+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:49:05.262763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":156,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:05.977648+0000","last_change":"2023-10-01T10:38:16.030613+0000","last_active":"2023-10-01T10:51:05.977648+0000","last_peered":"2023-10-01T10:51:05.977648+0000","last_clean":"2023-10-01T10:51:05.977648+0000","last_became_active":"2023-10-01T10:38:16.028975+0000","last_became_peered":"2023-10-01T10:38:16.028975+0000","last_unstale":"2023-10-01T10:51:05.977648+0000","last_undegraded":"2023-10-01T10:51:05.977648+0000","last_fullsized":"2023-10-01T10:51:05.977648+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:46:07.287911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"178'1048","reported_seq":1060,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:28.495100+0000","last_change":"2023-10-01T10:51:01.946522+0000","last_active":"2023-10-01T10:51:28.495100+0000","last_peered":"2023-10-01T10:51:28.495100+0000","last_clean":"2023-10-01T10:51:28.495100+0000","last_became_active":"2023-10-01T10:51:01.945971+0000","last_became_peered":"2023-10-01T10:51:01.945971+0000","last_unstale":"2023-10-01T10:51:28.495100+0000","last_undegraded":"2023-10-01T10:51:28.495100+0000","last_fullsized":"2023-10-01T10:51:28.495100+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":1048,"log_dups_size":0,"ondisk_log_size":1048,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:34:58.733457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68681728,"num_objects":1048,"num_object_clones":0,"num_object_copies":3144,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1048,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1048,"num_write_kb":67072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":129,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:05.978189+0000","last_change":"2023-10-01T10:47:54.750295+0000","last_active":"2023-10-01T10:51:05.978189+0000","last_peered":"2023-10-01T10:51:05.978189+0000","last_clean":"2023-10-01T10:51:05.978189+0000","last_became_active":"2023-10-01T10:47:54.749917+0000","last_became_peered":"2023-10-01T10:47:54.749917+0000","last_unstale":"2023-10-01T10:51:05.978189+0000","last_undegraded":"2023-10-01T10:51:05.978189+0000","last_fullsized":"2023-10-01T10:51:05.978189+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:30:59.236210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":161,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:05.978218+0000","last_change":"2023-10-01T10:38:16.035906+0000","last_active":"2023-10-01T10:51:05.978218+0000","last_peered":"2023-10-01T10:51:05.978218+0000","last_clean":"2023-10-01T10:51:05.978218+0000","last_became_active":"2023-10-01T10:38:16.035383+0000","last_became_peered":"2023-10-01T10:38:16.035383+0000","last_unstale":"2023-10-01T10:51:05.978218+0000","last_undegraded":"2023-10-01T10:51:05.978218+0000","last_fullsized":"2023-10-01T10:51:05.978218+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:44:41.500097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"178'816","reported_seq":830,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:26.725316+0000","last_change":"2023-10-01T10:51:01.954256+0000","last_active":"2023-10-01T10:51:26.725316+0000","last_peered":"2023-10-01T10:51:26.725316+0000","last_clean":"2023-10-01T10:51:26.725316+0000","last_became_active":"2023-10-01T10:51:01.953595+0000","last_became_peered":"2023-10-01T10:51:01.953595+0000","last_unstale":"2023-10-01T10:51:26.725316+0000","last_undegraded":"2023-10-01T10:51:26.725316+0000","last_fullsized":"2023-10-01T10:51:26.725316+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":816,"log_dups_size":0,"ondisk_log_size":816,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:55:22.868887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53477376,"num_objects":816,"num_object_clones":0,"num_object_copies":2448,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":816,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":816,"num_write_kb":52224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":154,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:14.664318+0000","last_change":"2023-10-01T10:38:16.026611+0000","last_active":"2023-10-01T10:51:14.664318+0000","last_peered":"2023-10-01T10:51:14.664318+0000","last_clean":"2023-10-01T10:51:14.664318+0000","last_became_active":"2023-10-01T10:38:16.026267+0000","last_became_peered":"2023-10-01T10:38:16.026267+0000","last_unstale":"2023-10-01T10:51:14.664318+0000","last_undegraded":"2023-10-01T10:51:14.664318+0000","last_fullsized":"2023-10-01T10:51:14.664318+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:06.135288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":137,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:05.977959+0000","last_change":"2023-10-01T10:38:16.032162+0000","last_active":"2023-10-01T10:51:05.977959+0000","last_peered":"2023-10-01T10:51:05.977959+0000","last_clean":"2023-10-01T10:51:05.977959+0000","last_became_active":"2023-10-01T10:38:16.031785+0000","last_became_peered":"2023-10-01T10:38:16.031785+0000","last_unstale":"2023-10-01T10:51:05.977959+0000","last_undegraded":"2023-10-01T10:51:05.977959+0000","last_fullsized":"2023-10-01T10:51:05.977959+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:38:43.003626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"178'771","reported_seq":784,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:26.020080+0000","last_change":"2023-10-01T10:51:01.953089+0000","last_active":"2023-10-01T10:51:26.020080+0000","last_peered":"2023-10-01T10:51:26.020080+0000","last_clean":"2023-10-01T10:51:26.020080+0000","last_became_active":"2023-10-01T10:51:01.952789+0000","last_became_peered":"2023-10-01T10:51:01.952789+0000","last_unstale":"2023-10-01T10:51:26.020080+0000","last_undegraded":"2023-10-01T10:51:26.020080+0000","last_fullsized":"2023-10-01T10:51:26.020080+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":771,"log_dups_size":0,"ondisk_log_size":771,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:28:59.885703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50528256,"num_objects":771,"num_object_clones":0,"num_object_copies":2313,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":771,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":771,"num_write_kb":49344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":144,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:05.975212+0000","last_change":"2023-10-01T10:37:42.772398+0000","last_active":"2023-10-01T10:51:05.975212+0000","last_peered":"2023-10-01T10:51:05.975212+0000","last_clean":"2023-10-01T10:51:05.975212+0000","last_became_active":"2023-10-01T10:37:42.772084+0000","last_became_peered":"2023-10-01T10:37:42.772084+0000","last_unstale":"2023-10-01T10:51:05.975212+0000","last_undegraded":"2023-10-01T10:51:05.975212+0000","last_fullsized":"2023-10-01T10:51:05.975212+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:32:43.846419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":162,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:06.144857+0000","last_change":"2023-10-01T10:38:16.019755+0000","last_active":"2023-10-01T10:51:06.144857+0000","last_peered":"2023-10-01T10:51:06.144857+0000","last_clean":"2023-10-01T10:51:06.144857+0000","last_became_active":"2023-10-01T10:38:16.019397+0000","last_became_peered":"2023-10-01T10:38:16.019397+0000","last_unstale":"2023-10-01T10:51:06.144857+0000","last_undegraded":"2023-10-01T10:51:06.144857+0000","last_fullsized":"2023-10-01T10:51:06.144857+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:42:56.623225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.d","version":"178'1003","reported_seq":1017,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:28.490969+0000","last_change":"2023-10-01T10:51:01.952187+0000","last_active":"2023-10-01T10:51:28.490969+0000","last_peered":"2023-10-01T10:51:28.490969+0000","last_clean":"2023-10-01T10:51:28.490969+0000","last_became_active":"2023-10-01T10:51:01.951373+0000","last_became_peered":"2023-10-01T10:51:01.951373+0000","last_unstale":"2023-10-01T10:51:28.490969+0000","last_undegraded":"2023-10-01T10:51:28.490969+0000","last_fullsized":"2023-10-01T10:51:28.490969+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":1003,"log_dups_size":0,"ondisk_log_size":1003,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:10:27.939483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65732608,"num_objects":1003,"num_object_clones":0,"num_object_copies":3009,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1003,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1003,"num_write_kb":64192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":156,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:16.021600+0000","last_change":"2023-10-01T10:32:31.561108+0000","last_active":"2023-10-01T10:51:16.021600+0000","last_peered":"2023-10-01T10:51:16.021600+0000","last_clean":"2023-10-01T10:51:16.021600+0000","last_became_active":"2023-10-01T10:32:31.560791+0000","last_became_peered":"2023-10-01T10:32:31.560791+0000","last_unstale":"2023-10-01T10:51:16.021600+0000","last_undegraded":"2023-10-01T10:51:16.021600+0000","last_fullsized":"2023-10-01T10:51:16.021600+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:42:47.477671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":160,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:06.147584+0000","last_change":"2023-10-01T10:37:42.766814+0000","last_active":"2023-10-01T10:51:06.147584+0000","last_peered":"2023-10-01T10:51:06.147584+0000","last_clean":"2023-10-01T10:51:06.147584+0000","last_became_active":"2023-10-01T10:37:42.766451+0000","last_became_peered":"2023-10-01T10:37:42.766451+0000","last_unstale":"2023-10-01T10:51:06.147584+0000","last_undegraded":"2023-10-01T10:51:06.147584+0000","last_fullsized":"2023-10-01T10:51:06.147584+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":122,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:16.021572+0000","last_change":"2023-10-01T10:38:16.034357+0000","last_active":"2023-10-01T10:51:16.021572+0000","last_peered":"2023-10-01T10:51:16.021572+0000","last_clean":"2023-10-01T10:51:16.021572+0000","last_became_active":"2023-10-01T10:38:16.033905+0000","last_became_peered":"2023-10-01T10:38:16.033905+0000","last_unstale":"2023-10-01T10:51:16.021572+0000","last_undegraded":"2023-10-01T10:51:16.021572+0000","last_fullsized":"2023-10-01T10:51:16.021572+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:47:58.369835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.b","version":"178'859","reported_seq":873,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:26.719925+0000","last_change":"2023-10-01T10:51:01.950278+0000","last_active":"2023-10-01T10:51:26.719925+0000","last_peered":"2023-10-01T10:51:26.719925+0000","last_clean":"2023-10-01T10:51:26.719925+0000","last_became_active":"2023-10-01T10:51:01.949902+0000","last_became_peered":"2023-10-01T10:51:01.949902+0000","last_unstale":"2023-10-01T10:51:26.719925+0000","last_undegraded":"2023-10-01T10:51:26.719925+0000","last_fullsized":"2023-10-01T10:51:26.719925+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":859,"log_dups_size":0,"ondisk_log_size":859,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:42:45.648175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56295424,"num_objects":859,"num_object_clones":0,"num_object_copies":2577,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":859,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":859,"num_write_kb":54976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":149,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:06.146997+0000","last_change":"2023-10-01T10:32:31.572482+0000","last_active":"2023-10-01T10:51:06.146997+0000","last_peered":"2023-10-01T10:51:06.146997+0000","last_clean":"2023-10-01T10:51:06.146997+0000","last_became_active":"2023-10-01T10:32:31.571993+0000","last_became_peered":"2023-10-01T10:32:31.571993+0000","last_unstale":"2023-10-01T10:51:06.146997+0000","last_undegraded":"2023-10-01T10:51:06.146997+0000","last_fullsized":"2023-10-01T10:51:06.146997+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":124,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:05.977321+0000","last_change":"2023-10-01T10:38:16.030202+0000","last_active":"2023-10-01T10:51:05.977321+0000","last_peered":"2023-10-01T10:51:05.977321+0000","last_clean":"2023-10-01T10:51:05.977321+0000","last_became_active":"2023-10-01T10:38:16.029917+0000","last_became_peered":"2023-10-01T10:38:16.029917+0000","last_unstale":"2023-10-01T10:51:05.977321+0000","last_undegraded":"2023-10-01T10:51:05.977321+0000","last_fullsized":"2023-10-01T10:51:05.977321+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:20:06.763909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":137,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:16.021632+0000","last_change":"2023-10-01T10:37:42.761153+0000","last_active":"2023-10-01T10:51:16.021632+0000","last_peered":"2023-10-01T10:51:16.021632+0000","last_clean":"2023-10-01T10:51:16.021632+0000","last_became_active":"2023-10-01T10:37:42.760715+0000","last_became_peered":"2023-10-01T10:37:42.760715+0000","last_unstale":"2023-10-01T10:51:16.021632+0000","last_undegraded":"2023-10-01T10:51:16.021632+0000","last_fullsized":"2023-10-01T10:51:16.021632+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:52:50.237174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.8","version":"178'869","reported_seq":882,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:26.742235+0000","last_change":"2023-10-01T10:51:01.955996+0000","last_active":"2023-10-01T10:51:26.742235+0000","last_peered":"2023-10-01T10:51:26.742235+0000","last_clean":"2023-10-01T10:51:26.742235+0000","last_became_active":"2023-10-01T10:51:01.955685+0000","last_became_peered":"2023-10-01T10:51:01.955685+0000","last_unstale":"2023-10-01T10:51:26.742235+0000","last_undegraded":"2023-10-01T10:51:26.742235+0000","last_fullsized":"2023-10-01T10:51:26.742235+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":869,"log_dups_size":0,"ondisk_log_size":869,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:54:42.840196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56950784,"num_objects":869,"num_object_clones":0,"num_object_copies":2607,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":869,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":869,"num_write_kb":55616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":153,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:06.143887+0000","last_change":"2023-10-01T10:37:42.762524+0000","last_active":"2023-10-01T10:51:06.143887+0000","last_peered":"2023-10-01T10:51:06.143887+0000","last_clean":"2023-10-01T10:51:06.143887+0000","last_became_active":"2023-10-01T10:37:42.762209+0000","last_became_peered":"2023-10-01T10:37:42.762209+0000","last_unstale":"2023-10-01T10:51:06.143887+0000","last_undegraded":"2023-10-01T10:51:06.143887+0000","last_fullsized":"2023-10-01T10:51:06.143887+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:45:38.171397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":122,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:06.145392+0000","last_change":"2023-10-01T10:37:05.019330+0000","last_active":"2023-10-01T10:51:06.145392+0000","last_peered":"2023-10-01T10:51:06.145392+0000","last_clean":"2023-10-01T10:51:06.145392+0000","last_became_active":"2023-10-01T10:37:05.018842+0000","last_became_peered":"2023-10-01T10:37:05.018842+0000","last_unstale":"2023-10-01T10:51:06.145392+0000","last_undegraded":"2023-10-01T10:51:06.145392+0000","last_fullsized":"2023-10-01T10:51:06.145392+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:48:09.551917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":150,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:06.217738+0000","last_change":"2023-10-01T10:32:31.569162+0000","last_active":"2023-10-01T10:51:06.217738+0000","last_peered":"2023-10-01T10:51:06.217738+0000","last_clean":"2023-10-01T10:51:06.217738+0000","last_became_active":"2023-10-01T10:32:31.568956+0000","last_became_peered":"2023-10-01T10:32:31.568956+0000","last_unstale":"2023-10-01T10:51:06.217738+0000","last_undegraded":"2023-10-01T10:51:06.217738+0000","last_fullsized":"2023-10-01T10:51:06.217738+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:02:08.285304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.a","version":"178'833","reported_seq":847,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:26.732082+0000","last_change":"2023-10-01T10:51:01.955145+0000","last_active":"2023-10-01T10:51:26.732082+0000","last_peered":"2023-10-01T10:51:26.732082+0000","last_clean":"2023-10-01T10:51:26.732082+0000","last_became_active":"2023-10-01T10:51:01.954793+0000","last_became_peered":"2023-10-01T10:51:01.954793+0000","last_unstale":"2023-10-01T10:51:26.732082+0000","last_undegraded":"2023-10-01T10:51:26.732082+0000","last_fullsized":"2023-10-01T10:51:26.732082+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":833,"log_dups_size":0,"ondisk_log_size":833,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:25:33.468595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54591488,"num_objects":833,"num_object_clones":0,"num_object_copies":2499,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":833,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":833,"num_write_kb":53312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":149,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:06.147158+0000","last_change":"2023-10-01T10:37:05.024009+0000","last_active":"2023-10-01T10:51:06.147158+0000","last_peered":"2023-10-01T10:51:06.147158+0000","last_clean":"2023-10-01T10:51:06.147158+0000","last_became_active":"2023-10-01T10:37:05.023458+0000","last_became_peered":"2023-10-01T10:37:05.023458+0000","last_unstale":"2023-10-01T10:51:06.147158+0000","last_undegraded":"2023-10-01T10:51:06.147158+0000","last_fullsized":"2023-10-01T10:51:06.147158+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'139","reported_seq":380,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:05.979760+0000","last_change":"2023-10-01T10:28:20.217386+0000","last_active":"2023-10-01T10:51:05.979760+0000","last_peered":"2023-10-01T10:51:05.979760+0000","last_clean":"2023-10-01T10:51:05.979760+0000","last_became_active":"2023-10-01T10:28:20.216803+0000","last_became_peered":"2023-10-01T10:28:20.216803+0000","last_unstale":"2023-10-01T10:51:05.979760+0000","last_undegraded":"2023-10-01T10:51:05.979760+0000","last_fullsized":"2023-10-01T10:51:05.979760+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:09:17.221082+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:09:17.221082+0000","last_clean_scrub_stamp":"2023-10-01T10:09:17.221082+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":126,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:05.975437+0000","last_change":"2023-10-01T10:37:42.765100+0000","last_active":"2023-10-01T10:51:05.975437+0000","last_peered":"2023-10-01T10:51:05.975437+0000","last_clean":"2023-10-01T10:51:05.975437+0000","last_became_active":"2023-10-01T10:37:42.764761+0000","last_became_peered":"2023-10-01T10:37:42.764761+0000","last_unstale":"2023-10-01T10:51:05.975437+0000","last_undegraded":"2023-10-01T10:51:05.975437+0000","last_fullsized":"2023-10-01T10:51:05.975437+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:24:56.907377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":113,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:05.977865+0000","last_change":"2023-10-01T10:47:54.782475+0000","last_active":"2023-10-01T10:51:05.977865+0000","last_peered":"2023-10-01T10:51:05.977865+0000","last_clean":"2023-10-01T10:51:05.977865+0000","last_became_active":"2023-10-01T10:47:54.781725+0000","last_became_peered":"2023-10-01T10:47:54.781725+0000","last_unstale":"2023-10-01T10:51:05.977865+0000","last_undegraded":"2023-10-01T10:51:05.977865+0000","last_fullsized":"2023-10-01T10:51:05.977865+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:21:30.978552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"178'815","reported_seq":829,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:26.733916+0000","last_change":"2023-10-01T10:51:01.952824+0000","last_active":"2023-10-01T10:51:26.733916+0000","last_peered":"2023-10-01T10:51:26.733916+0000","last_clean":"2023-10-01T10:51:26.733916+0000","last_became_active":"2023-10-01T10:51:01.952483+0000","last_became_peered":"2023-10-01T10:51:01.952483+0000","last_unstale":"2023-10-01T10:51:26.733916+0000","last_undegraded":"2023-10-01T10:51:26.733916+0000","last_fullsized":"2023-10-01T10:51:26.733916+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":815,"log_dups_size":0,"ondisk_log_size":815,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:42:42.460140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53411840,"num_objects":815,"num_object_clones":0,"num_object_copies":2445,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":815,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":815,"num_write_kb":52160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":163,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:05.979748+0000","last_change":"2023-10-01T10:32:31.565868+0000","last_active":"2023-10-01T10:51:05.979748+0000","last_peered":"2023-10-01T10:51:05.979748+0000","last_clean":"2023-10-01T10:51:05.979748+0000","last_became_active":"2023-10-01T10:32:31.565428+0000","last_became_peered":"2023-10-01T10:32:31.565428+0000","last_unstale":"2023-10-01T10:51:05.979748+0000","last_undegraded":"2023-10-01T10:51:05.979748+0000","last_fullsized":"2023-10-01T10:51:05.979748+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":154,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:14.078950+0000","last_change":"2023-10-01T10:38:16.021968+0000","last_active":"2023-10-01T10:51:14.078950+0000","last_peered":"2023-10-01T10:51:14.078950+0000","last_clean":"2023-10-01T10:51:14.078950+0000","last_became_active":"2023-10-01T10:38:16.021539+0000","last_became_peered":"2023-10-01T10:38:16.021539+0000","last_unstale":"2023-10-01T10:51:14.078950+0000","last_undegraded":"2023-10-01T10:51:14.078950+0000","last_fullsized":"2023-10-01T10:51:14.078950+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:12:53.105747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":154,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:06.144564+0000","last_change":"2023-10-01T10:37:05.017431+0000","last_active":"2023-10-01T10:51:06.144564+0000","last_peered":"2023-10-01T10:51:06.144564+0000","last_clean":"2023-10-01T10:51:06.144564+0000","last_became_active":"2023-10-01T10:37:05.017094+0000","last_became_peered":"2023-10-01T10:37:05.017094+0000","last_unstale":"2023-10-01T10:51:06.144564+0000","last_undegraded":"2023-10-01T10:51:06.144564+0000","last_fullsized":"2023-10-01T10:51:06.144564+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:10:40.664686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"178'714","reported_seq":727,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:26.011617+0000","last_change":"2023-10-01T10:51:01.947328+0000","last_active":"2023-10-01T10:51:26.011617+0000","last_peered":"2023-10-01T10:51:26.011617+0000","last_clean":"2023-10-01T10:51:26.011617+0000","last_became_active":"2023-10-01T10:51:01.946666+0000","last_became_peered":"2023-10-01T10:51:01.946666+0000","last_unstale":"2023-10-01T10:51:26.011617+0000","last_undegraded":"2023-10-01T10:51:26.011617+0000","last_fullsized":"2023-10-01T10:51:26.011617+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":714,"log_dups_size":0,"ondisk_log_size":714,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:19:24.372691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":46792704,"num_objects":714,"num_object_clones":0,"num_object_copies":2142,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":714,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":714,"num_write_kb":45696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":163,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:05.980745+0000","last_change":"2023-10-01T10:38:16.032774+0000","last_active":"2023-10-01T10:51:05.980745+0000","last_peered":"2023-10-01T10:51:05.980745+0000","last_clean":"2023-10-01T10:51:05.980745+0000","last_became_active":"2023-10-01T10:38:16.032283+0000","last_became_peered":"2023-10-01T10:38:16.032283+0000","last_unstale":"2023-10-01T10:51:05.980745+0000","last_undegraded":"2023-10-01T10:51:05.980745+0000","last_fullsized":"2023-10-01T10:51:05.980745+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":122,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:16.021666+0000","last_change":"2023-10-01T10:37:05.021228+0000","last_active":"2023-10-01T10:51:16.021666+0000","last_peered":"2023-10-01T10:51:16.021666+0000","last_clean":"2023-10-01T10:51:16.021666+0000","last_became_active":"2023-10-01T10:37:05.020888+0000","last_became_peered":"2023-10-01T10:37:05.020888+0000","last_unstale":"2023-10-01T10:51:16.021666+0000","last_undegraded":"2023-10-01T10:51:16.021666+0000","last_fullsized":"2023-10-01T10:51:16.021666+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:59:57.641247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":125,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:14.078976+0000","last_change":"2023-10-01T10:28:20.220812+0000","last_active":"2023-10-01T10:51:14.078976+0000","last_peered":"2023-10-01T10:51:14.078976+0000","last_clean":"2023-10-01T10:51:14.078976+0000","last_became_active":"2023-10-01T10:28:20.215962+0000","last_became_peered":"2023-10-01T10:28:20.215962+0000","last_unstale":"2023-10-01T10:51:14.078976+0000","last_undegraded":"2023-10-01T10:51:14.078976+0000","last_fullsized":"2023-10-01T10:51:14.078976+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:35:15.275097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"178'1065","reported_seq":1078,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:28.919304+0000","last_change":"2023-10-01T10:51:01.953666+0000","last_active":"2023-10-01T10:51:28.919304+0000","last_peered":"2023-10-01T10:51:28.919304+0000","last_clean":"2023-10-01T10:51:28.919304+0000","last_became_active":"2023-10-01T10:51:01.952070+0000","last_became_peered":"2023-10-01T10:51:01.952070+0000","last_unstale":"2023-10-01T10:51:28.919304+0000","last_undegraded":"2023-10-01T10:51:28.919304+0000","last_fullsized":"2023-10-01T10:51:28.919304+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":1065,"log_dups_size":0,"ondisk_log_size":1065,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69795840,"num_objects":1065,"num_object_clones":0,"num_object_copies":3195,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1065,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1065,"num_write_kb":68160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":131,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:05.975977+0000","last_change":"2023-10-01T10:37:42.773999+0000","last_active":"2023-10-01T10:51:05.975977+0000","last_peered":"2023-10-01T10:51:05.975977+0000","last_clean":"2023-10-01T10:51:05.975977+0000","last_became_active":"2023-10-01T10:37:42.773355+0000","last_became_peered":"2023-10-01T10:37:42.773355+0000","last_unstale":"2023-10-01T10:51:05.975977+0000","last_undegraded":"2023-10-01T10:51:05.975977+0000","last_fullsized":"2023-10-01T10:51:05.975977+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:46:51.317151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":163,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:06.145728+0000","last_change":"2023-10-01T10:38:16.034193+0000","last_active":"2023-10-01T10:51:06.145728+0000","last_peered":"2023-10-01T10:51:06.145728+0000","last_clean":"2023-10-01T10:51:06.145728+0000","last_became_active":"2023-10-01T10:38:16.033703+0000","last_became_peered":"2023-10-01T10:38:16.033703+0000","last_unstale":"2023-10-01T10:51:06.145728+0000","last_undegraded":"2023-10-01T10:51:06.145728+0000","last_fullsized":"2023-10-01T10:51:06.145728+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:21:26.061595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":128,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:05.975180+0000","last_change":"2023-10-01T10:37:42.773538+0000","last_active":"2023-10-01T10:51:05.975180+0000","last_peered":"2023-10-01T10:51:05.975180+0000","last_clean":"2023-10-01T10:51:05.975180+0000","last_became_active":"2023-10-01T10:37:42.771863+0000","last_became_peered":"2023-10-01T10:37:42.771863+0000","last_unstale":"2023-10-01T10:51:05.975180+0000","last_undegraded":"2023-10-01T10:51:05.975180+0000","last_fullsized":"2023-10-01T10:51:05.975180+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:46:46.698627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"178'1205","reported_seq":1219,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:29.666616+0000","last_change":"2023-10-01T10:51:01.952403+0000","last_active":"2023-10-01T10:51:29.666616+0000","last_peered":"2023-10-01T10:51:29.666616+0000","last_clean":"2023-10-01T10:51:29.666616+0000","last_became_active":"2023-10-01T10:51:01.951884+0000","last_became_peered":"2023-10-01T10:51:01.951884+0000","last_unstale":"2023-10-01T10:51:29.666616+0000","last_undegraded":"2023-10-01T10:51:29.666616+0000","last_fullsized":"2023-10-01T10:51:29.666616+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":1205,"log_dups_size":0,"ondisk_log_size":1205,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:33:03.736176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78970880,"num_objects":1205,"num_object_clones":0,"num_object_copies":3615,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1205,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1205,"num_write_kb":77120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":148,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:06.149395+0000","last_change":"2023-10-01T10:37:05.023034+0000","last_active":"2023-10-01T10:51:06.149395+0000","last_peered":"2023-10-01T10:51:06.149395+0000","last_clean":"2023-10-01T10:51:06.149395+0000","last_became_active":"2023-10-01T10:37:05.022670+0000","last_became_peered":"2023-10-01T10:37:05.022670+0000","last_unstale":"2023-10-01T10:51:06.149395+0000","last_undegraded":"2023-10-01T10:51:06.149395+0000","last_fullsized":"2023-10-01T10:51:06.149395+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":157,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:06.144348+0000","last_change":"2023-10-01T10:38:16.027472+0000","last_active":"2023-10-01T10:51:06.144348+0000","last_peered":"2023-10-01T10:51:06.144348+0000","last_clean":"2023-10-01T10:51:06.144348+0000","last_became_active":"2023-10-01T10:38:16.026810+0000","last_became_peered":"2023-10-01T10:38:16.026810+0000","last_unstale":"2023-10-01T10:51:06.144348+0000","last_undegraded":"2023-10-01T10:51:06.144348+0000","last_fullsized":"2023-10-01T10:51:06.144348+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:30:06.978187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":122,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:05.976873+0000","last_change":"2023-10-01T10:38:16.035705+0000","last_active":"2023-10-01T10:51:05.976873+0000","last_peered":"2023-10-01T10:51:05.976873+0000","last_clean":"2023-10-01T10:51:05.976873+0000","last_became_active":"2023-10-01T10:38:16.035389+0000","last_became_peered":"2023-10-01T10:38:16.035389+0000","last_unstale":"2023-10-01T10:51:05.976873+0000","last_undegraded":"2023-10-01T10:51:05.976873+0000","last_fullsized":"2023-10-01T10:51:05.976873+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:16:10.201409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":141,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:06.143891+0000","last_change":"2023-10-01T10:38:16.028491+0000","last_active":"2023-10-01T10:51:06.143891+0000","last_peered":"2023-10-01T10:51:06.143891+0000","last_clean":"2023-10-01T10:51:06.143891+0000","last_became_active":"2023-10-01T10:38:16.028175+0000","last_became_peered":"2023-10-01T10:38:16.028175+0000","last_unstale":"2023-10-01T10:51:06.143891+0000","last_undegraded":"2023-10-01T10:51:06.143891+0000","last_fullsized":"2023-10-01T10:51:06.143891+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:44:29.083129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":162,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:06.144113+0000","last_change":"2023-10-01T10:37:42.762360+0000","last_active":"2023-10-01T10:51:06.144113+0000","last_peered":"2023-10-01T10:51:06.144113+0000","last_clean":"2023-10-01T10:51:06.144113+0000","last_became_active":"2023-10-01T10:37:42.761936+0000","last_became_peered":"2023-10-01T10:37:42.761936+0000","last_unstale":"2023-10-01T10:51:06.144113+0000","last_undegraded":"2023-10-01T10:51:06.144113+0000","last_fullsized":"2023-10-01T10:51:06.144113+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:50.637582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":162,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:06.147442+0000","last_change":"2023-10-01T10:38:16.037809+0000","last_active":"2023-10-01T10:51:06.147442+0000","last_peered":"2023-10-01T10:51:06.147442+0000","last_clean":"2023-10-01T10:51:06.147442+0000","last_became_active":"2023-10-01T10:38:16.037318+0000","last_became_peered":"2023-10-01T10:38:16.037318+0000","last_unstale":"2023-10-01T10:51:06.147442+0000","last_undegraded":"2023-10-01T10:51:06.147442+0000","last_fullsized":"2023-10-01T10:51:06.147442+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":171,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:05.977188+0000","last_change":"2023-10-01T10:38:16.034282+0000","last_active":"2023-10-01T10:51:05.977188+0000","last_peered":"2023-10-01T10:51:05.977188+0000","last_clean":"2023-10-01T10:51:05.977188+0000","last_became_active":"2023-10-01T10:38:16.033820+0000","last_became_peered":"2023-10-01T10:38:16.033820+0000","last_unstale":"2023-10-01T10:51:05.977188+0000","last_undegraded":"2023-10-01T10:51:05.977188+0000","last_fullsized":"2023-10-01T10:51:05.977188+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:48:23.815832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":147,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:06.147323+0000","last_change":"2023-10-01T10:38:16.038077+0000","last_active":"2023-10-01T10:51:06.147323+0000","last_peered":"2023-10-01T10:51:06.147323+0000","last_clean":"2023-10-01T10:51:06.147323+0000","last_became_active":"2023-10-01T10:38:16.037313+0000","last_became_peered":"2023-10-01T10:38:16.037313+0000","last_unstale":"2023-10-01T10:51:06.147323+0000","last_undegraded":"2023-10-01T10:51:06.147323+0000","last_fullsized":"2023-10-01T10:51:06.147323+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":171,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:05.977279+0000","last_change":"2023-10-01T10:38:16.025600+0000","last_active":"2023-10-01T10:51:05.977279+0000","last_peered":"2023-10-01T10:51:05.977279+0000","last_clean":"2023-10-01T10:51:05.977279+0000","last_became_active":"2023-10-01T10:38:16.025252+0000","last_became_peered":"2023-10-01T10:38:16.025252+0000","last_unstale":"2023-10-01T10:51:05.977279+0000","last_undegraded":"2023-10-01T10:51:05.977279+0000","last_fullsized":"2023-10-01T10:51:05.977279+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:24:18.871963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":159,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:05.975842+0000","last_change":"2023-10-01T10:37:42.761243+0000","last_active":"2023-10-01T10:51:05.975842+0000","last_peered":"2023-10-01T10:51:05.975842+0000","last_clean":"2023-10-01T10:51:05.975842+0000","last_became_active":"2023-10-01T10:37:42.760834+0000","last_became_peered":"2023-10-01T10:37:42.760834+0000","last_unstale":"2023-10-01T10:51:05.975842+0000","last_undegraded":"2023-10-01T10:51:05.975842+0000","last_fullsized":"2023-10-01T10:51:05.975842+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:37:21.976095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":171,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:05.975168+0000","last_change":"2023-10-01T10:37:42.765142+0000","last_active":"2023-10-01T10:51:05.975168+0000","last_peered":"2023-10-01T10:51:05.975168+0000","last_clean":"2023-10-01T10:51:05.975168+0000","last_became_active":"2023-10-01T10:37:42.764770+0000","last_became_peered":"2023-10-01T10:37:42.764770+0000","last_unstale":"2023-10-01T10:51:05.975168+0000","last_undegraded":"2023-10-01T10:51:05.975168+0000","last_fullsized":"2023-10-01T10:51:05.975168+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:23:33.527876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":143,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:14.079046+0000","last_change":"2023-10-01T10:37:05.021621+0000","last_active":"2023-10-01T10:51:14.079046+0000","last_peered":"2023-10-01T10:51:14.079046+0000","last_clean":"2023-10-01T10:51:14.079046+0000","last_became_active":"2023-10-01T10:37:05.021173+0000","last_became_peered":"2023-10-01T10:37:05.021173+0000","last_unstale":"2023-10-01T10:51:14.079046+0000","last_undegraded":"2023-10-01T10:51:14.079046+0000","last_fullsized":"2023-10-01T10:51:14.079046+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:13:25.364499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":137,"reported_epoch":177,"state":"active+clean","last_fresh":"2023-10-01T10:51:05.976671+0000","last_change":"2023-10-01T10:38:16.031427+0000","last_active":"2023-10-01T10:51:05.976671+0000","last_peered":"2023-10-01T10:51:05.976671+0000","last_clean":"2023-10-01T10:51:05.976671+0000","last_became_active":"2023-10-01T10:38:16.029786+0000","last_became_peered":"2023-10-01T10:38:16.029786+0000","last_unstale":"2023-10-01T10:51:05.976671+0000","last_undegraded":"2023-10-01T10:51:05.976671+0000","last_fullsized":"2023-10-01T10:51:05.976671+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:53:22.796191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":1019740160,"num_objects":15560,"num_object_clones":0,"num_object_copies":46680,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":15560,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15560,"num_write_kb":995840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3164143616,"data_stored":3164143616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":15560,"ondisk_log_size":15560,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30319,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430674,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1284780,"kb_used_data":394532,"kb_used_omap":0,"kb_used_meta":890240,"kb_avail":92460372,"statfs":{"total":95995035648,"available":94679420928,"internally_reserved":0,"allocated":404000768,"data_stored":403493506,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":911605760},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:51:06 2023","interfaces":[{"interface":"back","average":{"1min":3.4209999999999998,"5min":3.3660000000000001,"15min":2.3399999999999999},"min":{"1min":0.33000000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":52.762,"5min":137.34200000000001,"15min":137.34200000000001},"last":0.41299999999999998},{"interface":"front","average":{"1min":3.4830000000000001,"5min":3.4780000000000002,"15min":2.2040000000000002},"min":{"1min":0.39200000000000002,"5min":0.308,"15min":0.28799999999999998},"max":{"1min":52.725999999999999,"5min":137.39500000000001,"15min":137.39500000000001},"last":0.432}]},{"osd":1,"last update":"Sun Oct 1 10:51:12 2023","interfaces":[{"interface":"back","average":{"1min":0.88600000000000001,"5min":1.1220000000000001,"15min":1.625},"min":{"1min":0.44400000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":1.4910000000000001,"5min":5.5700000000000003,"15min":56.110999999999997},"last":1.1479999999999999},{"interface":"front","average":{"1min":0.93600000000000005,"5min":1.1180000000000001,"15min":1.698},"min":{"1min":0.54500000000000004,"5min":0.316,"15min":0.30499999999999999},"max":{"1min":1.2450000000000001,"5min":5.8099999999999996,"15min":51.258000000000003},"last":0.94099999999999995}]},{"osd":2,"last update":"Sun Oct 1 10:50:27 2023","interfaces":[{"interface":"back","average":{"1min":1.452,"5min":1.4410000000000001,"15min":1.254},"min":{"1min":0.33200000000000002,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":7.1139999999999999,"5min":7.1139999999999999,"15min":7.1139999999999999},"last":1.4490000000000001},{"interface":"front","average":{"1min":1.1599999999999999,"5min":1.155,"15min":1.3049999999999999},"min":{"1min":0.377,"5min":0.35099999999999998,"15min":0.33200000000000002},"max":{"1min":3.8559999999999999,"5min":8.2110000000000003,"15min":10.766},"last":2.7709999999999999}]},{"osd":3,"last update":"Sun Oct 1 10:51:06 2023","interfaces":[{"interface":"back","average":{"1min":0.94099999999999995,"5min":1.601,"15min":1.7709999999999999},"min":{"1min":0.45600000000000002,"5min":0.374,"15min":0.34200000000000003},"max":{"1min":1.339,"5min":8.6829999999999998,"15min":51.334000000000003},"last":0.371},{"interface":"front","average":{"1min":1.038,"5min":1.2749999999999999,"15min":1.7909999999999999},"min":{"1min":0.38300000000000001,"5min":0.313,"15min":0.313},"max":{"1min":3.2570000000000001,"5min":8.7469999999999999,"15min":51.404000000000003},"last":1.1719999999999999}]},{"osd":4,"last update":"Sun Oct 1 10:50:40 2023","interfaces":[{"interface":"back","average":{"1min":1.6020000000000001,"5min":1.5509999999999999,"15min":1.298},"min":{"1min":0.40400000000000003,"5min":0.35099999999999998,"15min":0.317},"max":{"1min":6.9509999999999996,"5min":11.081,"15min":11.081},"last":1.081},{"interface":"front","average":{"1min":1.3160000000000001,"5min":1.7330000000000001,"15min":1.4239999999999999},"min":{"1min":0.50700000000000001,"5min":0.40699999999999997,"15min":0.32100000000000001},"max":{"1min":4.2309999999999999,"5min":12.292,"15min":12.292},"last":0.54400000000000004}]},{"osd":5,"last update":"Sun Oct 1 10:51:12 2023","interfaces":[{"interface":"back","average":{"1min":1.149,"5min":1.5720000000000001,"15min":1.6499999999999999},"min":{"1min":0.48199999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":1.534,"5min":9.7140000000000004,"15min":12.555},"last":0.76700000000000002},{"interface":"front","average":{"1min":1.097,"5min":1.419,"15min":1.708},"min":{"1min":0.59699999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":1.7170000000000001,"5min":10.737,"15min":17.536999999999999},"last":1.2589999999999999}]},{"osd":6,"last update":"Sun Oct 1 10:51:12 2023","interfaces":[{"interface":"back","average":{"1min":1.157,"5min":1.819,"15min":1.5389999999999999},"min":{"1min":0.46000000000000002,"5min":0.36399999999999999,"15min":0.34499999999999997},"max":{"1min":1.595,"5min":12.329000000000001,"15min":12.329000000000001},"last":2.5390000000000001},{"interface":"front","average":{"1min":1.1020000000000001,"5min":1.5680000000000001,"15min":1.3580000000000001},"min":{"1min":0.64700000000000002,"5min":0.37,"15min":0.36599999999999999},"max":{"1min":1.6279999999999999,"5min":7.9740000000000002,"15min":7.9740000000000002},"last":1.119}]}]},{"osd":6,"up_from":42,"seq":180388626908,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1141340,"kb_used_data":543444,"kb_used_omap":0,"kb_used_meta":597888,"kb_avail":92603812,"statfs":{"total":95995035648,"available":94826303488,"internally_reserved":0,"allocated":556486656,"data_stored":555995317,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":612237312},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:50:57 2023","interfaces":[{"interface":"back","average":{"1min":1.488,"5min":4.343,"15min":2.6099999999999999},"min":{"1min":0.29199999999999998,"5min":0.25,"15min":0.25},"max":{"1min":8.1639999999999997,"5min":139.994,"15min":140.524},"last":0.87},{"interface":"front","average":{"1min":1.2989999999999999,"5min":4.2759999999999998,"15min":2.5489999999999999},"min":{"1min":0.28299999999999997,"5min":0.249,"15min":0.249},"max":{"1min":7.9980000000000002,"5min":140.86500000000001,"15min":140.86500000000001},"last":1.8089999999999999}]},{"osd":1,"last update":"Sun Oct 1 10:51:03 2023","interfaces":[{"interface":"back","average":{"1min":1.016,"5min":1.7370000000000001,"15min":1.3740000000000001},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":3.9769999999999999,"5min":8.2560000000000002,"15min":9.673},"last":0.61099999999999999},{"interface":"front","average":{"1min":1.048,"5min":1.845,"15min":1.3919999999999999},"min":{"1min":0.48799999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":3.2959999999999998,"5min":12.968999999999999,"15min":12.968999999999999},"last":1.78}]},{"osd":2,"last update":"Sun Oct 1 10:51:26 2023","interfaces":[{"interface":"back","average":{"1min":0.92700000000000005,"5min":3.0299999999999998,"15min":1.7170000000000001},"min":{"1min":0.48199999999999998,"5min":0.32700000000000001,"15min":0.32300000000000001},"max":{"1min":2.5870000000000002,"5min":64.509,"15min":64.509},"last":2.5870000000000002},{"interface":"front","average":{"1min":1.02,"5min":3.0110000000000001,"15min":1.7470000000000001},"min":{"1min":0.44,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":2.6970000000000001,"5min":64.435000000000002,"15min":64.435000000000002},"last":2.6970000000000001}]},{"osd":3,"last update":"Sun Oct 1 10:51:06 2023","interfaces":[{"interface":"back","average":{"1min":1.1399999999999999,"5min":2.1880000000000002,"15min":1.9910000000000001},"min":{"1min":0.71299999999999997,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":3.9399999999999999,"5min":8.577,"15min":61.817999999999998},"last":0.54800000000000004},{"interface":"front","average":{"1min":1.149,"5min":1.9470000000000001,"15min":1.8029999999999999},"min":{"1min":0.69399999999999995,"5min":0.38100000000000001,"15min":0.315},"max":{"1min":2.8519999999999999,"5min":11.099,"15min":61.219000000000001},"last":7.4569999999999999}]},{"osd":4,"last update":"Sun Oct 1 10:50:46 2023","interfaces":[{"interface":"back","average":{"1min":1.7190000000000001,"5min":2.3149999999999999,"15min":1.415},"min":{"1min":0.35099999999999998,"5min":0.34200000000000003,"15min":0.32400000000000001},"max":{"1min":8.2089999999999996,"5min":13.1,"15min":13.1},"last":1.7230000000000001},{"interface":"front","average":{"1min":1.907,"5min":2.0779999999999998,"15min":1.4259999999999999},"min":{"1min":0.438,"5min":0.35799999999999998,"15min":0.315},"max":{"1min":8.1500000000000004,"5min":11.193,"15min":11.193},"last":2.6480000000000001}]},{"osd":5,"last update":"Sun Oct 1 10:51:06 2023","interfaces":[{"interface":"back","average":{"1min":1.21,"5min":2,"15min":1.53},"min":{"1min":0.41099999999999998,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":4.0090000000000003,"5min":8.2279999999999998,"15min":12.398999999999999},"last":0.58199999999999996},{"interface":"front","average":{"1min":1.095,"5min":2.1880000000000002,"15min":1.6739999999999999},"min":{"1min":0.41899999999999998,"5min":0.308,"15min":0.308},"max":{"1min":1.514,"5min":12.999000000000001,"15min":14.459},"last":1.7509999999999999}]},{"osd":7,"last update":"Sun Oct 1 10:50:57 2023","interfaces":[{"interface":"back","average":{"1min":1.766,"5min":2.2309999999999999,"15min":1.5149999999999999},"min":{"1min":0.32300000000000001,"5min":0.311,"15min":0.311},"max":{"1min":13.276999999999999,"5min":13.276999999999999,"15min":13.276999999999999},"last":2.0680000000000001},{"interface":"front","average":{"1min":1.8560000000000001,"5min":2.0449999999999999,"15min":1.599},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.35999999999999999},"max":{"1min":10.023,"5min":11.416,"15min":11.416},"last":0.93200000000000005}]}]},{"osd":1,"up_from":101,"seq":433791697125,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":745460,"kb_used_data":328556,"kb_used_omap":4,"kb_used_meta":416891,"kb_avail":92999692,"statfs":{"total":95995035648,"available":95231684608,"internally_reserved":0,"allocated":336441344,"data_stored":335698325,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":426897294},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:51:15 2023","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":1.546,"15min":1.1439999999999999},"min":{"1min":0.374,"5min":0.20599999999999999,"15min":0.189},"max":{"1min":0.97799999999999998,"5min":37.058,"15min":37.058},"last":2.2309999999999999},{"interface":"front","average":{"1min":0.58899999999999997,"5min":1.71,"15min":1.5529999999999999},"min":{"1min":0.35399999999999998,"5min":0.215,"15min":0.19400000000000001},"max":{"1min":1.036,"5min":37.337000000000003,"15min":61.207999999999998},"last":0.51200000000000001}]},{"osd":2,"last update":"Sun Oct 1 10:50:32 2023","interfaces":[{"interface":"back","average":{"1min":1.772,"5min":1.5169999999999999,"15min":1.3520000000000001},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":10.042999999999999,"5min":10.042999999999999,"15min":61.491},"last":1.6180000000000001},{"interface":"front","average":{"1min":1.556,"5min":1.3180000000000001,"15min":1.371},"min":{"1min":0.28699999999999998,"5min":0.24099999999999999,"15min":0.23699999999999999},"max":{"1min":5.516,"5min":8.9049999999999994,"15min":61.371000000000002},"last":2.8340000000000001}]},{"osd":3,"last update":"Sun Oct 1 10:51:15 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.95999999999999996,"15min":1.3500000000000001},"min":{"1min":0.36699999999999999,"5min":0.23999999999999999,"15min":0.20000000000000001},"max":{"1min":0.84499999999999997,"5min":5.9160000000000004,"15min":62.020000000000003},"last":2.9820000000000002},{"interface":"front","average":{"1min":0.68100000000000005,"5min":1.2310000000000001,"15min":1.5329999999999999},"min":{"1min":0.40999999999999998,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.98199999999999998,"5min":7.7560000000000002,"15min":62.338999999999999},"last":2.8559999999999999}]},{"osd":4,"last update":"Sun Oct 1 10:50:32 2023","interfaces":[{"interface":"back","average":{"1min":1.169,"5min":1.429,"15min":1.389},"min":{"1min":0.36699999999999999,"5min":0.33200000000000002,"15min":0.29199999999999998},"max":{"1min":4.3070000000000004,"5min":10.004,"15min":61.588000000000001},"last":2.2879999999999998},{"interface":"front","average":{"1min":1.7390000000000001,"5min":1.6659999999999999,"15min":1.6379999999999999},"min":{"1min":0.378,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":6.6189999999999998,"5min":10.880000000000001,"15min":61.411000000000001},"last":0.58299999999999996}]},{"osd":5,"last update":"Sun Oct 1 10:51:15 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":1.728,"15min":1.7430000000000001},"min":{"1min":0.68799999999999994,"5min":0.39400000000000002,"15min":0.38900000000000001},"max":{"1min":1.677,"5min":7.9260000000000002,"15min":61.560000000000002},"last":1.6559999999999999},{"interface":"front","average":{"1min":1.018,"5min":1.732,"15min":1.6200000000000001},"min":{"1min":0.622,"5min":0.35199999999999998,"15min":0.34699999999999998},"max":{"1min":1.647,"5min":10.865,"15min":61.667999999999999},"last":2.3210000000000002}]},{"osd":6,"last update":"Sun Oct 1 10:51:15 2023","interfaces":[{"interface":"back","average":{"1min":1.018,"5min":1.633,"15min":1.5289999999999999},"min":{"1min":0.65000000000000002,"5min":0.41099999999999998,"15min":0.39100000000000001},"max":{"1min":1.718,"5min":10.933,"15min":62.219999999999999},"last":2.3540000000000001},{"interface":"front","average":{"1min":1.026,"5min":1.6639999999999999,"15min":1.6619999999999999},"min":{"1min":0.67700000000000005,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":1.524,"5min":9.7870000000000008,"15min":61.462000000000003},"last":0.61699999999999999}]},{"osd":7,"last update":"Sun Oct 1 10:51:15 2023","interfaces":[{"interface":"back","average":{"1min":1.016,"5min":1.4419999999999999,"15min":1.782},"min":{"1min":0.63800000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":1.583,"5min":10.465,"15min":62.646999999999998},"last":2.9239999999999999},{"interface":"front","average":{"1min":1.0860000000000001,"5min":1.6439999999999999,"15min":1.7250000000000001},"min":{"1min":0.749,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":1.4950000000000001,"5min":6.415,"15min":62.048000000000002},"last":1.6879999999999999}]}]},{"osd":0,"up_from":86,"seq":369367187736,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":962336,"kb_used_data":350488,"kb_used_omap":5,"kb_used_meta":611834,"kb_avail":92782816,"statfs":{"total":95995035648,"available":95009603584,"internally_reserved":0,"allocated":358899712,"data_stored":358173009,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":626518650},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 10:51:15 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":1.1279999999999999,"15min":1.244},"min":{"1min":0.23499999999999999,"5min":0.193,"15min":0.184},"max":{"1min":1.7829999999999999,"5min":7.7359999999999998,"15min":15.074},"last":0.85199999999999998},{"interface":"front","average":{"1min":0.63300000000000001,"5min":1.075,"15min":1.3069999999999999},"min":{"1min":0.25,"5min":0.19900000000000001,"15min":0.19700000000000001},"max":{"1min":1.3959999999999999,"5min":8.4900000000000002,"15min":13.601000000000001},"last":1.7330000000000001}]},{"osd":2,"last update":"Sun Oct 1 10:50:29 2023","interfaces":[{"interface":"back","average":{"1min":1.718,"5min":1.575,"15min":1.097},"min":{"1min":0.26800000000000002,"5min":0.253,"15min":0.223},"max":{"1min":9.4309999999999992,"5min":12.872,"15min":12.872},"last":0.93799999999999994},{"interface":"front","average":{"1min":1.653,"5min":1.7450000000000001,"15min":1.135},"min":{"1min":0.36299999999999999,"5min":0.221,"15min":0.221},"max":{"1min":5.5449999999999999,"5min":13.137,"15min":13.137},"last":0.83799999999999997}]},{"osd":3,"last update":"Sun Oct 1 10:51:15 2023","interfaces":[{"interface":"back","average":{"1min":0.874,"5min":1.5309999999999999,"15min":1.3400000000000001},"min":{"1min":0.47299999999999998,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":3.097,"5min":12.944000000000001,"15min":12.944000000000001},"last":0.88100000000000001},{"interface":"front","average":{"1min":0.76600000000000001,"5min":1.5760000000000001,"15min":1.2909999999999999},"min":{"1min":0.31900000000000001,"5min":0.29099999999999998,"15min":0.245},"max":{"1min":2.2109999999999999,"5min":9.9090000000000007,"15min":10.984999999999999},"last":0.33000000000000002}]},{"osd":4,"last update":"Sun Oct 1 10:50:49 2023","interfaces":[{"interface":"back","average":{"1min":1.5449999999999999,"5min":2.3140000000000001,"15min":1.5169999999999999},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":9.3580000000000005,"5min":11.379,"15min":11.379},"last":0.47699999999999998},{"interface":"front","average":{"1min":1.254,"5min":2,"15min":1.4359999999999999},"min":{"1min":0.373,"5min":0.33400000000000002,"15min":0.31900000000000001},"max":{"1min":9.4009999999999998,"5min":11.362,"15min":11.362},"last":0.73599999999999999}]},{"osd":5,"last update":"Sun Oct 1 10:51:15 2023","interfaces":[{"interface":"back","average":{"1min":1.145,"5min":1.966,"15min":1.5669999999999999},"min":{"1min":0.64900000000000002,"5min":0.42299999999999999,"15min":0.35799999999999998},"max":{"1min":3.1859999999999999,"5min":12.91,"15min":12.91},"last":0.91300000000000003},{"interface":"front","average":{"1min":1.175,"5min":1.9159999999999999,"15min":1.5760000000000001},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.34699999999999998},"max":{"1min":3.2170000000000001,"5min":9.8620000000000001,"15min":10.666},"last":0.438}]},{"osd":6,"last update":"Sun Oct 1 10:51:15 2023","interfaces":[{"interface":"back","average":{"1min":1.2509999999999999,"5min":2.0619999999999998,"15min":1.5429999999999999},"min":{"1min":0.76200000000000001,"5min":0.376,"15min":0.36399999999999999},"max":{"1min":3.2410000000000001,"5min":9.8279999999999994,"15min":9.8279999999999994},"last":1.022},{"interface":"front","average":{"1min":1.526,"5min":2.085,"15min":1.5189999999999999},"min":{"1min":0.50800000000000001,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":10.111000000000001,"5min":10.111000000000001,"15min":10.111000000000001},"last":0.77100000000000002}]},{"osd":7,"last update":"Sun Oct 1 10:51:15 2023","interfaces":[{"interface":"back","average":{"1min":1.2529999999999999,"5min":1.786,"15min":1.603},"min":{"1min":0.40799999999999997,"5min":0.39700000000000002,"15min":0.36899999999999999},"max":{"1min":4.4969999999999999,"5min":15.839,"15min":15.839},"last":1.302},{"interface":"front","average":{"1min":1.0229999999999999,"5min":1.526,"15min":1.5109999999999999},"min":{"1min":0.53600000000000003,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":1.4690000000000001,"5min":8.9440000000000008,"15min":8.9440000000000008},"last":0.96799999999999997}]}]},{"osd":2,"up_from":116,"seq":498216206511,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":650736,"kb_used_data":361704,"kb_used_omap":7,"kb_used_meta":289016,"kb_avail":93094416,"statfs":{"total":95995035648,"available":95328681984,"internally_reserved":0,"allocated":370384896,"data_stored":369699950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":295952465},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:50:30 2023","interfaces":[{"interface":"back","average":{"1min":1.601,"5min":3.835,"15min":1.827},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":11.013999999999999,"5min":147.87,"15min":147.87},"last":1.355},{"interface":"front","average":{"1min":2.0110000000000001,"5min":4.0570000000000004,"15min":1.9510000000000001},"min":{"1min":0.36499999999999999,"5min":0.224,"15min":0.19500000000000001},"max":{"1min":8.4870000000000001,"5min":148.03100000000001,"15min":148.03100000000001},"last":0.36499999999999999}]},{"osd":1,"last update":"Sun Oct 1 10:50:30 2023","interfaces":[{"interface":"back","average":{"1min":1.6060000000000001,"5min":1.893,"15min":1.339},"min":{"1min":0.34300000000000003,"5min":0.246,"15min":0.219},"max":{"1min":7.7709999999999999,"5min":60.826000000000001,"15min":60.826000000000001},"last":0.40500000000000003},{"interface":"front","average":{"1min":1.208,"5min":2.1760000000000002,"15min":1.3700000000000001},"min":{"1min":0.25,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":7.234,"5min":62.957999999999998,"15min":62.957999999999998},"last":0.54800000000000004}]},{"osd":3,"last update":"Sun Oct 1 10:51:03 2023","interfaces":[{"interface":"back","average":{"1min":1.248,"5min":1.919,"15min":1.4650000000000001},"min":{"1min":0.28699999999999998,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":9.532,"5min":60.393000000000001,"15min":64.266000000000005},"last":1.319},{"interface":"front","average":{"1min":0.93700000000000006,"5min":2.2919999999999998,"15min":1.6439999999999999},"min":{"1min":0.378,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":6.5039999999999996,"5min":61.939999999999998,"15min":67.620999999999995},"last":0.498}]},{"osd":4,"last update":"Sun Oct 1 10:50:42 2023","interfaces":[{"interface":"back","average":{"1min":1.468,"5min":2.3690000000000002,"15min":1.476},"min":{"1min":0.47399999999999998,"5min":0.36499999999999999,"15min":0.35799999999999998},"max":{"1min":8.5190000000000001,"5min":60.994,"15min":60.994},"last":0.94299999999999995},{"interface":"front","average":{"1min":1.153,"5min":2.7269999999999999,"15min":1.534},"min":{"1min":0.43099999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":7.3049999999999997,"5min":63.012,"15min":63.012},"last":0.52200000000000002}]},{"osd":5,"last update":"Sun Oct 1 10:50:30 2023","interfaces":[{"interface":"back","average":{"1min":1.994,"5min":2.4350000000000001,"15min":1.641},"min":{"1min":0.441,"5min":0.39800000000000002,"15min":0.34699999999999998},"max":{"1min":8.4529999999999994,"5min":61.021000000000001,"15min":61.021000000000001},"last":0.89000000000000001},{"interface":"front","average":{"1min":2.4409999999999998,"5min":2.5510000000000002,"15min":1.6220000000000001},"min":{"1min":0.45400000000000001,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":7.282,"5min":63.145000000000003,"15min":63.145000000000003},"last":0.98699999999999999}]},{"osd":6,"last update":"Sun Oct 1 10:50:30 2023","interfaces":[{"interface":"back","average":{"1min":1.552,"5min":2.6139999999999999,"15min":1.5489999999999999},"min":{"1min":0.39600000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":7.3259999999999996,"5min":63.289000000000001,"15min":63.289000000000001},"last":0.68000000000000005},{"interface":"front","average":{"1min":1.4790000000000001,"5min":2.2879999999999998,"15min":1.4339999999999999},"min":{"1min":0.375,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":7.0919999999999996,"5min":60.579000000000001,"15min":60.579000000000001},"last":2.75}]},{"osd":7,"last update":"Sun Oct 1 10:50:30 2023","interfaces":[{"interface":"back","average":{"1min":1.1879999999999999,"5min":2.04,"15min":1.4259999999999999},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":3.5880000000000001,"5min":60.493000000000002,"15min":60.493000000000002},"last":1.3819999999999999},{"interface":"front","average":{"1min":1.919,"5min":2.2789999999999999,"15min":1.7070000000000001},"min":{"1min":0.59199999999999997,"5min":0.38500000000000001,"15min":0.376},"max":{"1min":8.4100000000000001,"5min":60.956000000000003,"15min":60.956000000000003},"last":0.43099999999999999}]}]},{"osd":3,"up_from":121,"seq":519691042983,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":672924,"kb_used_data":312980,"kb_used_omap":2,"kb_used_meta":359933,"kb_avail":93072228,"statfs":{"total":95995035648,"available":95305961472,"internally_reserved":0,"allocated":320491520,"data_stored":319824524,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":368571445},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3,3],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:51:06 2023","interfaces":[{"interface":"back","average":{"1min":7.21,"5min":4.242,"15min":2.4900000000000002},"min":{"1min":0.33400000000000002,"5min":0.189,"15min":0.187},"max":{"1min":69.435000000000002,"5min":129.953,"15min":129.953},"last":1.6020000000000001},{"interface":"front","average":{"1min":7.6130000000000004,"5min":4.7679999999999998,"15min":2.7320000000000002},"min":{"1min":0.38200000000000001,"5min":0.20799999999999999,"15min":0.17299999999999999},"max":{"1min":74.027000000000001,"5min":129.53999999999999,"15min":129.53999999999999},"last":0.41399999999999998}]},{"osd":1,"last update":"Sun Oct 1 10:51:06 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.97399999999999998,"15min":1.4930000000000001},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.96499999999999997,"5min":9.6560000000000006,"15min":77.361000000000004},"last":2.1230000000000002},{"interface":"front","average":{"1min":0.68200000000000005,"5min":1.071,"15min":1.633},"min":{"1min":0.247,"5min":0.247,"15min":0.24099999999999999},"max":{"1min":1.6000000000000001,"5min":10.305,"15min":77.867000000000004},"last":2.157}]},{"osd":2,"last update":"Sun Oct 1 10:51:06 2023","interfaces":[{"interface":"back","average":{"1min":0.72799999999999998,"5min":1.228,"15min":1.079},"min":{"1min":0.32400000000000001,"5min":0.27700000000000002,"15min":0.222},"max":{"1min":4.0119999999999996,"5min":9.5749999999999993,"15min":10.028},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.72499999999999998,"5min":1.252,"15min":1.329},"min":{"1min":0.34499999999999997,"5min":0.27900000000000003,"15min":0.26900000000000002},"max":{"1min":2.0619999999999998,"5min":9.8409999999999993,"15min":75.299999999999997},"last":1.71}]},{"osd":4,"last update":"Sun Oct 1 10:50:35 2023","interfaces":[{"interface":"back","average":{"1min":1.464,"5min":1.3700000000000001,"15min":1.4319999999999999},"min":{"1min":0.39400000000000002,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":5.0330000000000004,"5min":9.7370000000000001,"15min":75.388999999999996},"last":1.7470000000000001},{"interface":"front","average":{"1min":1.7410000000000001,"5min":1.609,"15min":1.577},"min":{"1min":0.53500000000000003,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":5.9249999999999998,"5min":6.375,"15min":75.165999999999997},"last":0.53400000000000003}]},{"osd":5,"last update":"Sun Oct 1 10:51:06 2023","interfaces":[{"interface":"back","average":{"1min":0.91000000000000003,"5min":1.3720000000000001,"15min":1.6839999999999999},"min":{"1min":0.39700000000000002,"5min":0.37,"15min":0.33100000000000002},"max":{"1min":1.8300000000000001,"5min":6.9260000000000002,"15min":75.221000000000004},"last":0.47699999999999998},{"interface":"front","average":{"1min":1.012,"5min":1.4119999999999999,"15min":1.7889999999999999},"min":{"1min":0.52100000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":4.1550000000000002,"5min":7.1820000000000004,"15min":75.231999999999999},"last":0.46000000000000002}]},{"osd":6,"last update":"Sun Oct 1 10:51:06 2023","interfaces":[{"interface":"back","average":{"1min":1.0489999999999999,"5min":1.855,"15min":1.762},"min":{"1min":0.42699999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":2.4750000000000001,"5min":7.8140000000000001,"15min":75.257000000000005},"last":0.56599999999999995},{"interface":"front","average":{"1min":1.4950000000000001,"5min":2.0779999999999998,"15min":1.887},"min":{"1min":0.44900000000000001,"5min":0.42299999999999999,"15min":0.33500000000000002},"max":{"1min":7.0060000000000002,"5min":8.5760000000000005,"15min":75.194000000000003},"last":0.54700000000000004}]},{"osd":7,"last update":"Sun Oct 1 10:51:06 2023","interfaces":[{"interface":"back","average":{"1min":1.054,"5min":1.3680000000000001,"15min":1.712},"min":{"1min":0.55100000000000005,"5min":0.376,"15min":0.35099999999999998},"max":{"1min":4.1210000000000004,"5min":8.1039999999999992,"15min":75.138999999999996},"last":0.497},{"interface":"front","average":{"1min":0.93200000000000005,"5min":1.4419999999999999,"15min":1.6359999999999999},"min":{"1min":0.57899999999999996,"5min":0.39500000000000002,"15min":0.37},"max":{"1min":1.1799999999999999,"5min":9.8059999999999992,"15min":75.355000000000004},"last":1.806}]}]},{"osd":4,"up_from":126,"seq":541165879457,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":607028,"kb_used_data":289196,"kb_used_omap":10,"kb_used_meta":317813,"kb_avail":93138124,"statfs":{"total":95995035648,"available":95373438976,"internally_reserved":0,"allocated":296136704,"data_stored":295447465,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":325441124},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,1,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:50:40 2023","interfaces":[{"interface":"back","average":{"1min":2.2730000000000001,"5min":2.6600000000000001,"15min":1.6080000000000001},"min":{"1min":0.30599999999999999,"5min":0.245,"15min":0.245},"max":{"1min":9.6140000000000008,"5min":62.576000000000001,"15min":62.576000000000001},"last":0.24399999999999999},{"interface":"front","average":{"1min":2.2589999999999999,"5min":2.79,"15min":1.637},"min":{"1min":0.28999999999999998,"5min":0.251,"15min":0.251},"max":{"1min":9.6530000000000005,"5min":64.941000000000003,"15min":64.941000000000003},"last":0.66100000000000003}]},{"osd":1,"last update":"Sun Oct 1 10:50:40 2023","interfaces":[{"interface":"back","average":{"1min":1.9910000000000001,"5min":1.7330000000000001,"15min":1.6919999999999999},"min":{"1min":0.34899999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":6.6100000000000003,"5min":7.0800000000000001,"15min":97.622},"last":0.96799999999999997},{"interface":"front","average":{"1min":1.6240000000000001,"5min":1.8149999999999999,"15min":1.7609999999999999},"min":{"1min":0.34499999999999997,"5min":0.28699999999999998,"15min":0.28599999999999998},"max":{"1min":5.1230000000000002,"5min":14.93,"15min":97.346999999999994},"last":0.36599999999999999}]},{"osd":2,"last update":"Sun Oct 1 10:50:40 2023","interfaces":[{"interface":"back","average":{"1min":1.6180000000000001,"5min":3.552,"15min":1.782},"min":{"1min":0.372,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":5.468,"5min":117.229,"15min":117.229},"last":1.532},{"interface":"front","average":{"1min":1.4510000000000001,"5min":3.2010000000000001,"15min":1.7210000000000001},"min":{"1min":0.33100000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":7.4080000000000004,"5min":116.922,"15min":116.922},"last":0.873}]},{"osd":3,"last update":"Sun Oct 1 10:50:40 2023","interfaces":[{"interface":"back","average":{"1min":1.8500000000000001,"5min":1.903,"15min":2.0139999999999998},"min":{"1min":0.35899999999999999,"5min":0.32900000000000001,"15min":0.29499999999999998},"max":{"1min":7.6559999999999997,"5min":12.513,"15min":61.398000000000003},"last":0.81799999999999995},{"interface":"front","average":{"1min":1.7310000000000001,"5min":1.9710000000000001,"15min":1.925},"min":{"1min":0.432,"5min":0.35399999999999998,"15min":0.34699999999999998},"max":{"1min":5.8680000000000003,"5min":7.4139999999999997,"15min":61.463000000000001},"last":1.0089999999999999}]},{"osd":5,"last update":"Sun Oct 1 10:50:40 2023","interfaces":[{"interface":"back","average":{"1min":1.9139999999999999,"5min":2.081,"15min":1.482},"min":{"1min":0.48499999999999999,"5min":0.34499999999999997,"15min":0.32700000000000001},"max":{"1min":8.7799999999999994,"5min":12.601000000000001,"15min":12.601000000000001},"last":0.59399999999999997},{"interface":"front","average":{"1min":1.9279999999999999,"5min":1.986,"15min":1.4930000000000001},"min":{"1min":0.51600000000000001,"5min":0.35499999999999998,"15min":0.28999999999999998},"max":{"1min":5.1429999999999998,"5min":11.725,"15min":16.779},"last":0.39700000000000002}]},{"osd":6,"last update":"Sun Oct 1 10:50:40 2023","interfaces":[{"interface":"back","average":{"1min":2.4580000000000002,"5min":2.5640000000000001,"15min":1.569},"min":{"1min":0.38600000000000001,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":12.114000000000001,"5min":13.646000000000001,"15min":13.646000000000001},"last":0.63200000000000001},{"interface":"front","average":{"1min":2.3959999999999999,"5min":2.411,"15min":1.5509999999999999},"min":{"1min":0.61399999999999999,"5min":0.42299999999999999,"15min":0.41099999999999998},"max":{"1min":5.7670000000000003,"5min":14.446999999999999,"15min":14.446999999999999},"last":1.2569999999999999}]},{"osd":7,"last update":"Sun Oct 1 10:50:40 2023","interfaces":[{"interface":"back","average":{"1min":1.849,"5min":2.032,"15min":1.4590000000000001},"min":{"1min":0.51200000000000001,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":5.8159999999999998,"5min":10.423999999999999,"15min":10.423999999999999},"last":1.0429999999999999},{"interface":"front","average":{"1min":1.738,"5min":1.841,"15min":1.639},"min":{"1min":0.52100000000000002,"5min":0.39200000000000002,"15min":0.30399999999999999},"max":{"1min":5.0979999999999999,"5min":11.798999999999999,"15min":11.798999999999999},"last":1.5800000000000001}]}]},{"osd":5,"up_from":36,"seq":154618823138,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1176208,"kb_used_data":276040,"kb_used_omap":0,"kb_used_meta":900160,"kb_avail":92568944,"statfs":{"total":95995035648,"available":94790598656,"internally_reserved":0,"allocated":282664960,"data_stored":282181185,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":921763840},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:51:02 2023","interfaces":[{"interface":"back","average":{"1min":1.0009999999999999,"5min":2.0150000000000001,"15min":1.522},"min":{"1min":0.34999999999999998,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":4.6440000000000001,"5min":16.309000000000001,"15min":52.094999999999999},"last":0.32600000000000001},{"interface":"front","average":{"1min":1.099,"5min":2.2050000000000001,"15min":1.7649999999999999},"min":{"1min":0.47799999999999998,"5min":0.29199999999999998,"15min":0.26900000000000002},"max":{"1min":4.9880000000000004,"5min":23.507000000000001,"15min":52.048000000000002},"last":0.65700000000000003}]},{"osd":1,"last update":"Sun Oct 1 10:51:02 2023","interfaces":[{"interface":"back","average":{"1min":1.024,"5min":1.6180000000000001,"15min":2.3050000000000002},"min":{"1min":0.436,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":4.6950000000000003,"5min":7.7000000000000002,"15min":202.96600000000001},"last":0.27700000000000002},{"interface":"front","average":{"1min":1.0580000000000001,"5min":1.6299999999999999,"15min":2.6680000000000001},"min":{"1min":0.63600000000000001,"5min":0.31900000000000001,"15min":0.28100000000000003},"max":{"1min":3.9769999999999999,"5min":9.9209999999999994,"15min":203.05099999999999},"last":1.327}]},{"osd":2,"last update":"Sun Oct 1 10:50:32 2023","interfaces":[{"interface":"back","average":{"1min":1.706,"5min":2.1230000000000002,"15min":1.385},"min":{"1min":0.317,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":9.5839999999999996,"5min":14.872,"15min":14.872},"last":0.97699999999999998},{"interface":"front","average":{"1min":1.909,"5min":2.3069999999999999,"15min":1.647},"min":{"1min":0.38200000000000001,"5min":0.32300000000000001,"15min":0.317},"max":{"1min":5.9509999999999996,"5min":14.746,"15min":14.746},"last":1.202}]},{"osd":3,"last update":"Sun Oct 1 10:51:08 2023","interfaces":[{"interface":"back","average":{"1min":1.091,"5min":1.8420000000000001,"15min":2.3500000000000001},"min":{"1min":0.64500000000000002,"5min":0.35199999999999998,"15min":0.316},"max":{"1min":3.944,"5min":9.9900000000000002,"15min":70.506},"last":0.69699999999999995},{"interface":"front","average":{"1min":0.92000000000000004,"5min":1.73,"15min":2.1619999999999999},"min":{"1min":0.55100000000000005,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.3779999999999999,"5min":10.898,"15min":70.346999999999994},"last":0.54300000000000004}]},{"osd":4,"last update":"Sun Oct 1 10:50:32 2023","interfaces":[{"interface":"back","average":{"1min":2.1949999999999998,"5min":1.833,"15min":1.4610000000000001},"min":{"1min":0.39500000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":9.8889999999999993,"5min":11.002000000000001,"15min":15.170999999999999},"last":0.58999999999999997},{"interface":"front","average":{"1min":3.0459999999999998,"5min":2.3679999999999999,"15min":1.5189999999999999},"min":{"1min":0.54100000000000004,"5min":0.316,"15min":0.316},"max":{"1min":9.7270000000000003,"5min":9.7270000000000003,"15min":9.7270000000000003},"last":0.52800000000000002}]},{"osd":6,"last update":"Sun Oct 1 10:51:08 2023","interfaces":[{"interface":"back","average":{"1min":1.0429999999999999,"5min":2.3740000000000001,"15min":1.5569999999999999},"min":{"1min":0.56200000000000006,"5min":0.45000000000000001,"15min":0.38500000000000001},"max":{"1min":1.474,"5min":11.629,"15min":11.629},"last":2.25},{"interface":"front","average":{"1min":0.98299999999999998,"5min":2.4129999999999998,"15min":1.7030000000000001},"min":{"1min":0.51300000000000001,"5min":0.33200000000000002,"15min":0.312},"max":{"1min":1.5369999999999999,"5min":14.263,"15min":14.263},"last":0.67600000000000005}]},{"osd":7,"last update":"Sun Oct 1 10:51:08 2023","interfaces":[{"interface":"back","average":{"1min":1.4750000000000001,"5min":2.198,"15min":1.845},"min":{"1min":0.57799999999999996,"5min":0.375,"15min":0.34699999999999998},"max":{"1min":9.2159999999999993,"5min":13.673999999999999,"15min":13.673999999999999},"last":0.63300000000000001},{"interface":"front","average":{"1min":1.097,"5min":1.9890000000000001,"15min":1.677},"min":{"1min":0.41099999999999998,"5min":0.38,"15min":0.372},"max":{"1min":1.5980000000000001,"5min":9.6929999999999996,"15min":11.635},"last":0.753}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":365428736,"data_stored":365428736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":377946112,"data_stored":377946112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":406323200,"data_stored":406323200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":383844352,"data_stored":383844352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":306774016,"data_stored":306774016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":309198848,"data_stored":309198848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":575275008,"data_stored":575275008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":439353344,"data_stored":439353344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T10:51:34.823 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T10:51:34.824 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:34.853 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:34.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:34.856 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:51:34.958 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:34.988 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:34.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:34.990 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:51:35.092 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:35.122 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:35.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:35.124 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:51:35.226 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:35.255 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 14 16 28577 28561 127.493 82.125 0.00878196 0.00783673 2023-10-01T10:51:35.258 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:35.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:35.261 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:51:35.362 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:35.392 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:35.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:35.396 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:51:35.497 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:35.523 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:35.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:35.529 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:51:35.630 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:35.656 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:35.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:35.660 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:51:35.760 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:35.769 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:35 smithi173 bash[58816]: cluster 2023-10-01T10:51:33.953538+0000 mgr.y (mgr.25072) 1019 : cluster 0 pgmap v969: 89 pgs: 89 active+clean; 1.4 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 127 MiB/s wr, 2.03k op/s 2023-10-01T10:51:35.786 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:35.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:35.790 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:51:35.890 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:35.899 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:35 smithi078 bash[78274]: cluster 2023-10-01T10:51:33.953538+0000 mgr.y (mgr.25072) 1019 : cluster 0 pgmap v969: 89 pgs: 89 active+clean; 1.4 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 127 MiB/s wr, 2.03k op/s 2023-10-01T10:51:35.899 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:35 smithi078 bash[75299]: cluster 2023-10-01T10:51:33.953538+0000 mgr.y (mgr.25072) 1019 : cluster 0 pgmap v969: 89 pgs: 89 active+clean; 1.4 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 127 MiB/s wr, 2.03k op/s 2023-10-01T10:51:35.914 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:35.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:35.916 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:51:36.017 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:36.042 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:36.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:36.044 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:51:36.145 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:36.171 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:36.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:36.174 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:51:36.255 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 15 15 29925 29910 124.613 84.3125 0.0087801 0.00801843 2023-10-01T10:51:36.276 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:36.305 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:36.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:36.308 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:51:36.409 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:36.440 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:36.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:36.442 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:51:36.543 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:36.575 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:36.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:36.577 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:51:36.638 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T10:51:36.678 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:36.745 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:36.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:36.748 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:51:36.848 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:36.878 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:36.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:36.880 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:51:36.896 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T10:51:36.981 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:37.012 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:37.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:37.016 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:51:37.117 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:37.140 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:37.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:37.143 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:51:37.244 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:37.258 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 16 16 31514 31498 123.017 99.25 0.0100726 0.00812249 2023-10-01T10:51:37.272 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:37.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:37.275 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:51:37.375 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:37.405 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:37.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:37.409 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:51:37.510 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:37.540 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:37.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:37.542 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:51:37.643 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:37.677 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:37.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:37.682 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:51:37.783 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:37.800 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:37 smithi173 bash[58816]: cluster 2023-10-01T10:51:35.954339+0000 mgr.y (mgr.25072) 1020 : cluster 0 pgmap v970: 89 pgs: 89 active+clean; 1.5 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s 2023-10-01T10:51:37.822 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:37.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:37.827 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:51:37.927 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:37.937 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:37 smithi078 bash[75299]: cluster 2023-10-01T10:51:35.954339+0000 mgr.y (mgr.25072) 1020 : cluster 0 pgmap v970: 89 pgs: 89 active+clean; 1.5 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s 2023-10-01T10:51:37.938 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:37 smithi078 bash[78274]: cluster 2023-10-01T10:51:35.954339+0000 mgr.y (mgr.25072) 1020 : cluster 0 pgmap v970: 89 pgs: 89 active+clean; 1.5 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s 2023-10-01T10:51:37.953 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:37.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:37.956 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:51:38.057 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:38.086 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:38.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:38.090 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:51:38.192 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:38.221 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:38.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:38.225 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:51:38.257 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 17 16 33165 33149 121.85 103.188 0.0276786 0.00819864 2023-10-01T10:51:38.326 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:38.351 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:38.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:38.360 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:51:38.460 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:38.484 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:38.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:38.489 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:51:38.590 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:38.617 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:38.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:38.623 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:51:38.724 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:38.761 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:38.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:38.763 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:51:38.864 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:38.894 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:38.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:38.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:51:39.004 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:39.038 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:39.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:39.043 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:51:39.143 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:39.172 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:39.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:39.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:51:39.257 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 18 16 34802 34786 120.764 102.312 0.00985336 0.00827466 2023-10-01T10:51:39.275 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:39.301 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:39.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:39.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:51:39.406 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:39.442 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:39.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:39.445 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:51:39.546 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:39.573 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:39.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:39.575 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:51:39.676 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:39.682 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:51:39.702 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:39.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:39.707 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:51:39.807 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:39.850 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:39.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:39.853 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:51:39.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:39 smithi173 bash[58816]: cluster 2023-10-01T10:51:37.954978+0000 mgr.y (mgr.25072) 1021 : cluster 0 pgmap v971: 89 pgs: 89 active+clean; 1.7 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2023-10-01T10:51:39.953 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:39.986 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:39.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:39.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:51:40.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:39 smithi078 bash[78274]: cluster 2023-10-01T10:51:37.954978+0000 mgr.y (mgr.25072) 1021 : cluster 0 pgmap v971: 89 pgs: 89 active+clean; 1.7 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2023-10-01T10:51:40.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:39 smithi078 bash[75299]: cluster 2023-10-01T10:51:37.954978+0000 mgr.y (mgr.25072) 1021 : cluster 0 pgmap v971: 89 pgs: 89 active+clean; 1.7 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2023-10-01T10:51:40.090 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:40.130 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:40.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:40.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:51:40.235 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:40.257 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 19 16 36359 36343 119.53 97.3125 0.0113959 0.00835998 2023-10-01T10:51:40.273 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:40.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:40.277 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:51:40.340 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-10-01T10:51:40.340 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T10:51:40.378 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:40.388 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:51:40 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:51:40] "GET /metrics HTTP/1.1" 200 36303 "" "Prometheus/2.33.4" 2023-10-01T10:51:40.404 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:40.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:40.407 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:51:40.507 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:40.534 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:40.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:40.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:51:40.638 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:40.663 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:40.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:40.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:51:40.705 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:51:40.771 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:40.807 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:40.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:40.810 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:51:40.911 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:40.939 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:40.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:40.941 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:51:41.042 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:41.070 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:41.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:41.073 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:51:41.174 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:41.197 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:41.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:41.201 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:51:41.257 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:2023-10-01T10:51:41.257164+0000 min lat: 0.00119944 max lat: 0.258785 avg lat: 0.00843198 2023-10-01T10:51:41.257 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T10:51:41.257 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 20 16 37946 37930 118.512 99.1875 0.00551626 0.00843198 2023-10-01T10:51:41.302 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:41.332 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:41.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:41.336 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:51:41.437 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:41.438 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-01T10:51:41.505 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:41.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:41.511 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:51:41.612 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:41.639 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:41.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:41.643 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:51:41.724 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:51:41.743 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:41.771 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:41.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:41.775 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:51:41.876 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:41.902 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:41.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:41.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:51:42.004 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:42.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:41 smithi078 bash[78274]: cluster 2023-10-01T10:51:39.955598+0000 mgr.y (mgr.25072) 1022 : cluster 0 pgmap v972: 89 pgs: 89 active+clean; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s 2023-10-01T10:51:42.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:41 smithi078 bash[75299]: cluster 2023-10-01T10:51:39.955598+0000 mgr.y (mgr.25072) 1022 : cluster 0 pgmap v972: 89 pgs: 89 active+clean; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s 2023-10-01T10:51:42.030 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:42.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:42.034 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:51:42.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:41 smithi173 bash[58816]: cluster 2023-10-01T10:51:39.955598+0000 mgr.y (mgr.25072) 1022 : cluster 0 pgmap v972: 89 pgs: 89 active+clean; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s 2023-10-01T10:51:42.134 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:42.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:42.162 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:42.162 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:51:42.257 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 21 16 39347 39331 117.038 87.5625 0.0120012 0.00853853 2023-10-01T10:51:42.263 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:42.294 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:42.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:42.300 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:51:42.401 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:42.433 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:42.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:42.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:51:42.536 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:42.562 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:42.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:42.564 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:51:42.664 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:42.697 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:42.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:42.700 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:51:42.801 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:42.827 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:42.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:42.830 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:51:42.930 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:42.962 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:42.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:42.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:51:43.064 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:43.090 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:43.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:43.094 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:51:43.194 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:43.225 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:43.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:43.230 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:51:43.257 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 22 16 40556 40540 115.153 75.5625 0.00557803 0.0086783 2023-10-01T10:51:43.331 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:43.368 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:43.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:43.371 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:51:43.472 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:43.508 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:43.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:43.511 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:51:43.611 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:43.638 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:43.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:43.643 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:51:43.743 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:43.768 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:43.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:43.772 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:51:43.873 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:43.901 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:43.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:43.905 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:51:44.006 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:44.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:43 smithi078 bash[78274]: cluster 2023-10-01T10:51:41.956183+0000 mgr.y (mgr.25072) 1023 : cluster 0 pgmap v973: 89 pgs: 89 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2023-10-01T10:51:44.016 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:43 smithi078 bash[75299]: cluster 2023-10-01T10:51:41.956183+0000 mgr.y (mgr.25072) 1023 : cluster 0 pgmap v973: 89 pgs: 89 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2023-10-01T10:51:44.030 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:44.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:44.033 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:51:44.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:43 smithi173 bash[58816]: cluster 2023-10-01T10:51:41.956183+0000 mgr.y (mgr.25072) 1023 : cluster 0 pgmap v973: 89 pgs: 89 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2023-10-01T10:51:44.134 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:44.169 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:44.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:44.172 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:51:44.258 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 23 15 42013 41998 114.108 91.125 0.0121349 0.00875789 2023-10-01T10:51:44.273 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:44.303 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:44.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:44.305 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:51:44.406 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:44.436 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:44.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:44.439 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:51:44.540 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:44.566 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:44.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:44.568 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:51:44.669 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:44.695 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:44.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:44.699 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:51:44.799 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:44.833 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:44.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:44.835 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:51:44.936 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:44.962 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:44.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:44.964 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:51:45.065 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:45.089 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:45.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:45.092 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:51:45.193 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:45.241 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:45.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:45.245 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:51:45.258 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 24 16 43516 43500 113.264 93.875 0.00801665 0.0088225 2023-10-01T10:51:45.346 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:45.371 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:45.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:45.375 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:51:45.476 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:45.503 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:45.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:45.505 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:51:45.606 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:45.632 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:45.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:45.636 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:51:45.737 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:45.763 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:45.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:45.765 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:51:45.866 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:45.875 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:45 smithi078 bash[78274]: cluster 2023-10-01T10:51:43.956684+0000 mgr.y (mgr.25072) 1024 : cluster 0 pgmap v974: 89 pgs: 89 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-10-01T10:51:45.875 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:45 smithi078 bash[75299]: cluster 2023-10-01T10:51:43.956684+0000 mgr.y (mgr.25072) 1024 : cluster 0 pgmap v974: 89 pgs: 89 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-10-01T10:51:45.892 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:45.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:45.895 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:51:45.996 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:46.022 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:46.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:46.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:51:46.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:45 smithi173 bash[58816]: cluster 2023-10-01T10:51:43.956684+0000 mgr.y (mgr.25072) 1024 : cluster 0 pgmap v974: 89 pgs: 89 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-10-01T10:51:46.126 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:46.155 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:46.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:46.159 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:51:46.258 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 25 16 45013 44997 112.476 93.5625 0.00729362 0.00888509 2023-10-01T10:51:46.260 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:46.292 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:46.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:46.294 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:51:46.308 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T10:51:46.395 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:46.416 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:46.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:46.418 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:51:46.519 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:46.547 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:46.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:46.554 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:51:46.655 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:46.682 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:46.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:46.684 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:51:46.785 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:46.809 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:46.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:46.813 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:51:46.913 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:46.954 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:46.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:46.956 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:51:47.057 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:47.082 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:47.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:47.085 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:51:47.185 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:47.208 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:47.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:47.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:51:47.258 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 26 16 46300 46284 111.243 80.4375 0.00456203 0.00898399 2023-10-01T10:51:47.311 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:47.341 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:47.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:47.345 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:51:47.445 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:47.477 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:47.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:47.480 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:51:47.581 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:47.611 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:47.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:47.614 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:51:47.715 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:47.755 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:47.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:47.759 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:51:47.859 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:47.869 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:47 smithi078 bash[75299]: cluster 2023-10-01T10:51:45.957156+0000 mgr.y (mgr.25072) 1025 : cluster 0 pgmap v975: 89 pgs: 89 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-10-01T10:51:47.869 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:47 smithi078 bash[75299]: audit 2023-10-01T10:51:47.433797+0000 mon.a (mon.0) 1267 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:51:47.869 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:47 smithi078 bash[78274]: cluster 2023-10-01T10:51:45.957156+0000 mgr.y (mgr.25072) 1025 : cluster 0 pgmap v975: 89 pgs: 89 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-10-01T10:51:47.869 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:47 smithi078 bash[78274]: audit 2023-10-01T10:51:47.433797+0000 mon.a (mon.0) 1267 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:51:47.894 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:47.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:47.897 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:51:47.997 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:48.023 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:48.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:48.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:51:48.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:47 smithi173 bash[58816]: cluster 2023-10-01T10:51:45.957156+0000 mgr.y (mgr.25072) 1025 : cluster 0 pgmap v975: 89 pgs: 89 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-10-01T10:51:48.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:47 smithi173 bash[58816]: audit 2023-10-01T10:51:47.433797+0000 mon.a (mon.0) 1267 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:51:48.127 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:48.172 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:48.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:48.176 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:51:48.258 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 27 16 47632 47616 110.206 83.25 0.0039291 0.00905677 2023-10-01T10:51:48.276 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:48.303 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:48.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:48.307 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:51:48.407 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:48.437 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:48.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:48.441 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:51:48.542 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:48.568 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:48.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:48.572 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:51:48.672 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:48.701 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:48.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:48.707 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:51:48.808 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:48.836 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:48.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:48.840 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:51:48.941 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:48.977 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:48.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:48.982 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:51:49.083 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:49.117 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:49.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:49.120 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:51:49.221 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:49.246 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:49.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:49.253 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:51:49.258 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 28 16 48932 48916 109.172 81.25 0.00714133 0.00915435 2023-10-01T10:51:49.354 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:49.387 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:49.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:49.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:51:49.490 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:49.525 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:49.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:49.531 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:51:49.632 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:49.668 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:49.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:49.671 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:51:49.772 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:49.790 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:49 smithi173 bash[58816]: cluster 2023-10-01T10:51:47.957716+0000 mgr.y (mgr.25072) 1026 : cluster 0 pgmap v976: 89 pgs: 89 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-10-01T10:51:49.800 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:49.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:49.804 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:51:49.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:49 smithi078 bash[75299]: cluster 2023-10-01T10:51:47.957716+0000 mgr.y (mgr.25072) 1026 : cluster 0 pgmap v976: 89 pgs: 89 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-10-01T10:51:49.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:49 smithi078 bash[78274]: cluster 2023-10-01T10:51:47.957716+0000 mgr.y (mgr.25072) 1026 : cluster 0 pgmap v976: 89 pgs: 89 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-10-01T10:51:49.904 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:49.936 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:49.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:49.942 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:51:50.043 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:50.071 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:50.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:50.073 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:51:50.174 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:50.208 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:50.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:50.213 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:51:50.258 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 29 16 50260 50244 108.269 83 0.00682715 0.00923044 2023-10-01T10:51:50.314 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:50.340 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:50.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:50.342 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:51:50.443 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:50.452 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:51:50 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:51:50] "GET /metrics HTTP/1.1" 200 36310 "" "Prometheus/2.33.4" 2023-10-01T10:51:50.474 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:50.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:50.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:51:50.578 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:50.613 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:50.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:50.618 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:51:50.719 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:50.756 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:50.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:50.758 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:51:50.859 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:50.886 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:50.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:50.890 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:51:50.991 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:51.019 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:51.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:51.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:51:51.122 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:51.152 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:51.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:51.156 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:51:51.256 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:51.258 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 30 16 51588 51572 107.427 83 0.00321985 0.00930328 2023-10-01T10:51:51.283 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:51.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:51.289 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:51:51.390 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:51.421 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:51.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:51.426 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:51:51.527 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:51.555 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:51.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:51.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:51:51.658 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:51.686 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:51.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:51.688 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:51:51.789 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:51.798 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:51 smithi078 bash[75299]: cluster 2023-10-01T10:51:49.958365+0000 mgr.y (mgr.25072) 1027 : cluster 0 pgmap v977: 89 pgs: 89 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2023-10-01T10:51:51.798 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:51 smithi173 bash[58816]: cluster 2023-10-01T10:51:49.958365+0000 mgr.y (mgr.25072) 1027 : cluster 0 pgmap v977: 89 pgs: 89 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2023-10-01T10:51:51.807 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-10-01T10:51:51.807 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T10:51:51.815 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:51.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:51.817 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:51:51.918 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:51.945 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:51.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:51.948 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:51:52.049 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:52.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:51 smithi078 bash[78274]: cluster 2023-10-01T10:51:49.958365+0000 mgr.y (mgr.25072) 1027 : cluster 0 pgmap v977: 89 pgs: 89 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2023-10-01T10:51:52.081 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:52.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:52.085 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:51:52.186 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:52.207 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:52.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:52.213 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:51:52.258 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 31 16 52798 52782 106.401 75.625 0.0165161 0.00939308 2023-10-01T10:51:52.315 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:52.349 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:52.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:52.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:51:52.452 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:52.482 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:52.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:52.485 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:51:52.586 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:52.611 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:52.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:52.615 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:51:52.716 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:52.743 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:52.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:52.747 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:51:52.847 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:52.865 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:52.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:52.867 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:51:52.946 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-01T10:51:52.967 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:52.995 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:52.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:52.999 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:51:53.100 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:53.129 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:53.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:53.133 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:51:53.234 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:53.261 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 32 16 53951 53935 105.327 72.0625 0.0066381 0.0094875 2023-10-01T10:51:53.267 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:53.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:53.270 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:51:53.371 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:53.394 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:53.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:53.397 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:51:53.498 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:53.525 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:53.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:53.531 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:51:53.632 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:53.659 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:53.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:53.732 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:51:53.804 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:53 smithi173 bash[58816]: cluster 2023-10-01T10:51:51.958965+0000 mgr.y (mgr.25072) 1028 : cluster 0 pgmap v978: 89 pgs: 89 active+clean; 3.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2023-10-01T10:51:53.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:53 smithi078 bash[78274]: cluster 2023-10-01T10:51:51.958965+0000 mgr.y (mgr.25072) 1028 : cluster 0 pgmap v978: 89 pgs: 89 active+clean; 3.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2023-10-01T10:51:53.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:53 smithi078 bash[75299]: cluster 2023-10-01T10:51:51.958965+0000 mgr.y (mgr.25072) 1028 : cluster 0 pgmap v978: 89 pgs: 89 active+clean; 3.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2023-10-01T10:51:53.832 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:53.863 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:53.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:53.866 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:51:53.967 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:53.993 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:53.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:53.998 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:51:54.099 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:54.133 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:54.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:54.142 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:51:54.242 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:54.258 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 33 16 55116 55100 104.342 72.8125 0.0134171 0.00957638 2023-10-01T10:51:54.273 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:54.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:54.277 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:51:54.338 INFO:teuthology.orchestra.run.smithi078.stdout: 2023-10-01T10:51:54.348 INFO:teuthology.orchestra.run.smithi078.stderr:dumped all 2023-10-01T10:51:54.378 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:54.387 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T10:51:54.403 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:54.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:54.405 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:51:54.439 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T10:51:54.484 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T10:51:54.507 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:54.526 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T10:51:54.533 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:54.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:54.537 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:51:54.569 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T10:51:54.606 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T10:51:54.637 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:54.652 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T10:51:54.666 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:54.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:54.672 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:51:54.697 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T10:51:54.701 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T10:51:54.701 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T10:51:54.701 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T10:51:54.701 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T10:51:54.702 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T10:51:54.702 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T10:51:54.702 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T10:51:54.702 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T10:51:54.772 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:54.800 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:54.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:54.804 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:51:54.829 INFO:teuthology.orchestra.run.smithi078.stderr:nodeep-scrub is set 2023-10-01T10:51:54.905 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:54.935 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:54.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:54.937 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:51:55.038 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:55.046 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:54 smithi078 bash[75299]: audit 2023-10-01T10:51:54.358499+0000 mon.a (mon.0) 1268 : audit 0 from='client.? 172.21.15.78:0/268504485' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:51:55.047 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:54 smithi078 bash[75299]: audit 2023-10-01T10:51:54.753173+0000 mon.a (mon.0) 1269 : audit 1 from='client.? 172.21.15.78:0/257270179' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:51:55.047 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:54 smithi078 bash[78274]: audit 2023-10-01T10:51:54.358499+0000 mon.a (mon.0) 1268 : audit 0 from='client.? 172.21.15.78:0/268504485' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:51:55.047 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:54 smithi078 bash[78274]: audit 2023-10-01T10:51:54.753173+0000 mon.a (mon.0) 1269 : audit 1 from='client.? 172.21.15.78:0/257270179' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:51:55.061 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:55.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:55.066 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:51:55.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:54 smithi173 bash[58816]: audit 2023-10-01T10:51:54.358499+0000 mon.a (mon.0) 1268 : audit 0 from='client.? 172.21.15.78:0/268504485' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:51:55.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:54 smithi173 bash[58816]: audit 2023-10-01T10:51:54.753173+0000 mon.a (mon.0) 1269 : audit 1 from='client.? 172.21.15.78:0/257270179' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:51:55.167 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:55.199 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:55.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:55.206 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:51:55.258 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 34 16 56138 56122 103.152 63.875 0.0102225 0.00968942 2023-10-01T10:51:55.306 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:55.335 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:55.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:55.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:51:55.437 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:55.468 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:55.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:55.472 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:51:55.573 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:55.600 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:55.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:55.604 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:51:55.705 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:55.734 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:55.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:55.736 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:51:55.837 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:55.867 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:55.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:55.871 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:51:55.972 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:55.979 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:55 smithi078 bash[75299]: cluster 2023-10-01T10:51:53.959540+0000 mgr.y (mgr.25072) 1029 : cluster 0 pgmap v979: 89 pgs: 89 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-01T10:51:55.979 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:55 smithi078 bash[75299]: audit 2023-10-01T10:51:54.319266+0000 mgr.y (mgr.25072) 1030 : audit 0 from='client.36096 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:51:55.980 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:55 smithi078 bash[75299]: cluster 2023-10-01T10:51:54.803184+0000 mon.a (mon.0) 1270 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:51:55.980 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:55 smithi078 bash[75299]: audit 2023-10-01T10:51:54.822720+0000 mon.a (mon.0) 1271 : audit 1 from='client.? 172.21.15.78:0/257270179' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T10:51:55.980 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:55 smithi078 bash[75299]: cluster 2023-10-01T10:51:54.822871+0000 mon.a (mon.0) 1272 : cluster 0 osdmap e180: 8 total, 8 up, 8 in 2023-10-01T10:51:55.980 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:55 smithi078 bash[75299]: cluster 2023-10-01T10:51:54.825174+0000 osd.7 (osd.7) 134 : cluster 3 failed to encode map e180 with expected crc 2023-10-01T10:51:55.980 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:55 smithi078 bash[75299]: cluster 2023-10-01T10:51:54.825287+0000 osd.5 (osd.5) 115 : cluster 3 failed to encode map e180 with expected crc 2023-10-01T10:51:55.981 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:55 smithi078 bash[75299]: cluster 2023-10-01T10:51:54.825325+0000 osd.6 (osd.6) 97 : cluster 3 failed to encode map e180 with expected crc 2023-10-01T10:51:55.981 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:55 smithi078 bash[78274]: cluster 2023-10-01T10:51:53.959540+0000 mgr.y (mgr.25072) 1029 : cluster 0 pgmap v979: 89 pgs: 89 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-01T10:51:55.981 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:55 smithi078 bash[78274]: audit 2023-10-01T10:51:54.319266+0000 mgr.y (mgr.25072) 1030 : audit 0 from='client.36096 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:51:55.981 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:55 smithi078 bash[78274]: cluster 2023-10-01T10:51:54.803184+0000 mon.a (mon.0) 1270 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:51:55.981 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:55 smithi078 bash[78274]: audit 2023-10-01T10:51:54.822720+0000 mon.a (mon.0) 1271 : audit 1 from='client.? 172.21.15.78:0/257270179' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T10:51:55.982 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:55 smithi078 bash[78274]: cluster 2023-10-01T10:51:54.822871+0000 mon.a (mon.0) 1272 : cluster 0 osdmap e180: 8 total, 8 up, 8 in 2023-10-01T10:51:55.982 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:55 smithi078 bash[78274]: cluster 2023-10-01T10:51:54.825174+0000 osd.7 (osd.7) 134 : cluster 3 failed to encode map e180 with expected crc 2023-10-01T10:51:55.982 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:55 smithi078 bash[78274]: cluster 2023-10-01T10:51:54.825287+0000 osd.5 (osd.5) 115 : cluster 3 failed to encode map e180 with expected crc 2023-10-01T10:51:55.982 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:55 smithi078 bash[78274]: cluster 2023-10-01T10:51:54.825325+0000 osd.6 (osd.6) 97 : cluster 3 failed to encode map e180 with expected crc 2023-10-01T10:51:56.003 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:56.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:56.008 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:51:56.109 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:56.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:55 smithi173 bash[58816]: cluster 2023-10-01T10:51:53.959540+0000 mgr.y (mgr.25072) 1029 : cluster 0 pgmap v979: 89 pgs: 89 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2023-10-01T10:51:56.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:55 smithi173 bash[58816]: audit 2023-10-01T10:51:54.319266+0000 mgr.y (mgr.25072) 1030 : audit 0 from='client.36096 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:51:56.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:55 smithi173 bash[58816]: cluster 2023-10-01T10:51:54.803184+0000 mon.a (mon.0) 1270 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:51:56.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:55 smithi173 bash[58816]: audit 2023-10-01T10:51:54.822720+0000 mon.a (mon.0) 1271 : audit 1 from='client.? 172.21.15.78:0/257270179' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T10:51:56.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:55 smithi173 bash[58816]: cluster 2023-10-01T10:51:54.822871+0000 mon.a (mon.0) 1272 : cluster 0 osdmap e180: 8 total, 8 up, 8 in 2023-10-01T10:51:56.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:55 smithi173 bash[58816]: cluster 2023-10-01T10:51:54.825174+0000 osd.7 (osd.7) 134 : cluster 3 failed to encode map e180 with expected crc 2023-10-01T10:51:56.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:55 smithi173 bash[58816]: cluster 2023-10-01T10:51:54.825287+0000 osd.5 (osd.5) 115 : cluster 3 failed to encode map e180 with expected crc 2023-10-01T10:51:56.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:55 smithi173 bash[58816]: cluster 2023-10-01T10:51:54.825325+0000 osd.6 (osd.6) 97 : cluster 3 failed to encode map e180 with expected crc 2023-10-01T10:51:56.137 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:56.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:56.144 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:51:56.245 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:56.258 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 35 15 57353 57338 102.376 76 0.0147817 0.00976172 2023-10-01T10:51:56.275 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:56.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:56.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:51:56.380 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:56.402 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:56.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:56.405 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:51:56.506 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:56.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:56.537 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:56.537 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:51:56.638 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:56.667 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:56.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:56.673 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:51:56.773 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:56.810 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:56.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:56.813 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:51:56.914 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:56.947 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:56.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:56.950 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:51:57.051 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:57.081 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:57.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:57.084 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:51:57.184 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:57.220 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:57.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:57.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:51:57.259 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 36 16 58583 58567 101.665 76.8125 0.00793029 0.00983016 2023-10-01T10:51:57.326 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:57.355 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:51:57.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:57.357 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:51:57.458 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:57.484 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:57.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:57.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:51:57.587 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:57.623 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:57.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:57.627 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:51:57.728 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:57.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:57.763 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:57.764 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:51:57.776 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T10:51:57.853 INFO:teuthology.orchestra.run.smithi078.stdout:{"pg_ready":true,"pg_map":{"version":979,"stamp":"2023-10-01T10:51:53.959218+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3435075488,"num_objects":52430,"num_object_clones":0,"num_object_copies":157290,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":52430,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":52667,"num_write_kb":3356430,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":52601,"ondisk_log_size":52601,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":14761916,"kb_used_data":9930940,"kb_used_omap":30,"kb_used_meta":4830881,"kb_avail":735199300,"statfs":{"total":767960285184,"available":752844083200,"internally_reserved":0,"allocated":10169282560,"data_stored":10164270941,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31502,"internal_metadata":4946822386},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1067515904,"num_objects":16289,"num_object_clones":0,"num_object_copies":48867,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":16289,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16289,"num_write_kb":1042496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003403"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":170,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:16.699129+0000","last_change":"2023-10-01T10:38:16.032258+0000","last_active":"2023-10-01T10:51:16.699129+0000","last_peered":"2023-10-01T10:51:16.699129+0000","last_clean":"2023-10-01T10:51:16.699129+0000","last_became_active":"2023-10-01T10:38:16.031633+0000","last_became_peered":"2023-10-01T10:38:16.031633+0000","last_unstale":"2023-10-01T10:51:16.699129+0000","last_undegraded":"2023-10-01T10:51:16.699129+0000","last_fullsized":"2023-10-01T10:51:16.699129+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:17:02.446430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":158,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:18.883266+0000","last_change":"2023-10-01T10:38:16.025881+0000","last_active":"2023-10-01T10:51:18.883266+0000","last_peered":"2023-10-01T10:51:18.883266+0000","last_clean":"2023-10-01T10:51:18.883266+0000","last_became_active":"2023-10-01T10:38:16.025518+0000","last_became_peered":"2023-10-01T10:38:16.025518+0000","last_unstale":"2023-10-01T10:51:18.883266+0000","last_undegraded":"2023-10-01T10:51:18.883266+0000","last_fullsized":"2023-10-01T10:51:18.883266+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:48:37.826604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":137,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-01T10:51:31.747059+0000","last_change":"2023-10-01T10:38:16.025075+0000","last_active":"2023-10-01T10:51:31.747059+0000","last_peered":"2023-10-01T10:51:31.747059+0000","last_clean":"2023-10-01T10:51:31.747059+0000","last_became_active":"2023-10-01T10:38:16.024735+0000","last_became_peered":"2023-10-01T10:38:16.024735+0000","last_unstale":"2023-10-01T10:51:31.747059+0000","last_undegraded":"2023-10-01T10:51:31.747059+0000","last_fullsized":"2023-10-01T10:51:31.747059+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:38:01.450741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":141,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:17.531944+0000","last_change":"2023-10-01T10:37:42.772008+0000","last_active":"2023-10-01T10:51:17.531944+0000","last_peered":"2023-10-01T10:51:17.531944+0000","last_clean":"2023-10-01T10:51:17.531944+0000","last_became_active":"2023-10-01T10:37:42.771244+0000","last_became_peered":"2023-10-01T10:37:42.771244+0000","last_unstale":"2023-10-01T10:51:17.531944+0000","last_undegraded":"2023-10-01T10:51:17.531944+0000","last_fullsized":"2023-10-01T10:51:17.531944+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:38:55.180790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":159,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:18.882907+0000","last_change":"2023-10-01T10:37:42.766600+0000","last_active":"2023-10-01T10:51:18.882907+0000","last_peered":"2023-10-01T10:51:18.882907+0000","last_clean":"2023-10-01T10:51:18.882907+0000","last_became_active":"2023-10-01T10:37:42.766051+0000","last_became_peered":"2023-10-01T10:37:42.766051+0000","last_unstale":"2023-10-01T10:51:18.882907+0000","last_undegraded":"2023-10-01T10:51:18.882907+0000","last_fullsized":"2023-10-01T10:51:18.882907+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:51:17.707963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":144,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:18.073083+0000","last_change":"2023-10-01T10:32:31.575109+0000","last_active":"2023-10-01T10:51:18.073083+0000","last_peered":"2023-10-01T10:51:18.073083+0000","last_clean":"2023-10-01T10:51:18.073083+0000","last_became_active":"2023-10-01T10:32:31.574729+0000","last_became_peered":"2023-10-01T10:32:31.574729+0000","last_unstale":"2023-10-01T10:51:18.073083+0000","last_undegraded":"2023-10-01T10:51:18.073083+0000","last_fullsized":"2023-10-01T10:51:18.073083+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":138,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:17.532014+0000","last_change":"2023-10-01T10:37:42.766667+0000","last_active":"2023-10-01T10:51:17.532014+0000","last_peered":"2023-10-01T10:51:17.532014+0000","last_clean":"2023-10-01T10:51:17.532014+0000","last_became_active":"2023-10-01T10:37:42.766369+0000","last_became_peered":"2023-10-01T10:37:42.766369+0000","last_unstale":"2023-10-01T10:51:17.532014+0000","last_undegraded":"2023-10-01T10:51:17.532014+0000","last_fullsized":"2023-10-01T10:51:17.532014+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:02.708788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":144,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-01T10:51:31.749348+0000","last_change":"2023-10-01T10:37:42.765925+0000","last_active":"2023-10-01T10:51:31.749348+0000","last_peered":"2023-10-01T10:51:31.749348+0000","last_clean":"2023-10-01T10:51:31.749348+0000","last_became_active":"2023-10-01T10:37:42.765218+0000","last_became_peered":"2023-10-01T10:37:42.765218+0000","last_unstale":"2023-10-01T10:51:31.749348+0000","last_undegraded":"2023-10-01T10:51:31.749348+0000","last_fullsized":"2023-10-01T10:51:31.749348+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T22:10:34.338031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":146,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:16.700480+0000","last_change":"2023-10-01T10:32:31.579805+0000","last_active":"2023-10-01T10:51:16.700480+0000","last_peered":"2023-10-01T10:51:16.700480+0000","last_clean":"2023-10-01T10:51:16.700480+0000","last_became_active":"2023-10-01T10:32:31.579423+0000","last_became_peered":"2023-10-01T10:32:31.579423+0000","last_unstale":"2023-10-01T10:51:16.700480+0000","last_undegraded":"2023-10-01T10:51:16.700480+0000","last_fullsized":"2023-10-01T10:51:16.700480+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":128,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-01T10:51:31.750208+0000","last_change":"2023-10-01T10:37:42.772531+0000","last_active":"2023-10-01T10:51:31.750208+0000","last_peered":"2023-10-01T10:51:31.750208+0000","last_clean":"2023-10-01T10:51:31.750208+0000","last_became_active":"2023-10-01T10:37:42.772216+0000","last_became_peered":"2023-10-01T10:37:42.772216+0000","last_unstale":"2023-10-01T10:51:31.750208+0000","last_undegraded":"2023-10-01T10:51:31.750208+0000","last_fullsized":"2023-10-01T10:51:31.750208+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:01:53.373628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":163,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:16.703552+0000","last_change":"2023-10-01T10:38:16.027085+0000","last_active":"2023-10-01T10:51:16.703552+0000","last_peered":"2023-10-01T10:51:16.703552+0000","last_clean":"2023-10-01T10:51:16.703552+0000","last_became_active":"2023-10-01T10:38:16.026555+0000","last_became_peered":"2023-10-01T10:38:16.026555+0000","last_unstale":"2023-10-01T10:51:16.703552+0000","last_undegraded":"2023-10-01T10:51:16.703552+0000","last_fullsized":"2023-10-01T10:51:16.703552+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":122,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:18.320947+0000","last_change":"2023-10-01T10:37:05.018755+0000","last_active":"2023-10-01T10:51:18.320947+0000","last_peered":"2023-10-01T10:51:18.320947+0000","last_clean":"2023-10-01T10:51:18.320947+0000","last_became_active":"2023-10-01T10:37:05.018446+0000","last_became_peered":"2023-10-01T10:37:05.018446+0000","last_unstale":"2023-10-01T10:51:18.320947+0000","last_undegraded":"2023-10-01T10:51:18.320947+0000","last_fullsized":"2023-10-01T10:51:18.320947+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:13:53.740503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":164,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:16.698905+0000","last_change":"2023-10-01T10:38:16.030826+0000","last_active":"2023-10-01T10:51:16.698905+0000","last_peered":"2023-10-01T10:51:16.698905+0000","last_clean":"2023-10-01T10:51:16.698905+0000","last_became_active":"2023-10-01T10:38:16.030277+0000","last_became_peered":"2023-10-01T10:38:16.030277+0000","last_unstale":"2023-10-01T10:51:16.698905+0000","last_undegraded":"2023-10-01T10:51:16.698905+0000","last_fullsized":"2023-10-01T10:51:16.698905+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:31:43.040424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":138,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:16.698217+0000","last_change":"2023-10-01T10:38:16.034741+0000","last_active":"2023-10-01T10:51:16.698217+0000","last_peered":"2023-10-01T10:51:16.698217+0000","last_clean":"2023-10-01T10:51:16.698217+0000","last_became_active":"2023-10-01T10:38:16.034407+0000","last_became_peered":"2023-10-01T10:38:16.034407+0000","last_unstale":"2023-10-01T10:51:16.698217+0000","last_undegraded":"2023-10-01T10:51:16.698217+0000","last_fullsized":"2023-10-01T10:51:16.698217+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:47:29.911477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":162,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:16.703391+0000","last_change":"2023-10-01T10:38:16.026777+0000","last_active":"2023-10-01T10:51:16.703391+0000","last_peered":"2023-10-01T10:51:16.703391+0000","last_clean":"2023-10-01T10:51:16.703391+0000","last_became_active":"2023-10-01T10:38:16.026371+0000","last_became_peered":"2023-10-01T10:38:16.026371+0000","last_unstale":"2023-10-01T10:51:16.703391+0000","last_undegraded":"2023-10-01T10:51:16.703391+0000","last_fullsized":"2023-10-01T10:51:16.703391+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":167,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-01T10:51:31.751607+0000","last_change":"2023-10-01T10:37:42.772319+0000","last_active":"2023-10-01T10:51:31.751607+0000","last_peered":"2023-10-01T10:51:31.751607+0000","last_clean":"2023-10-01T10:51:31.751607+0000","last_became_active":"2023-10-01T10:37:42.771977+0000","last_became_peered":"2023-10-01T10:37:42.771977+0000","last_unstale":"2023-10-01T10:51:31.751607+0000","last_undegraded":"2023-10-01T10:51:31.751607+0000","last_fullsized":"2023-10-01T10:51:31.751607+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:45:12.783214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":139,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-01T10:51:31.753976+0000","last_change":"2023-10-01T10:37:42.767706+0000","last_active":"2023-10-01T10:51:31.753976+0000","last_peered":"2023-10-01T10:51:31.753976+0000","last_clean":"2023-10-01T10:51:31.753976+0000","last_became_active":"2023-10-01T10:37:42.767211+0000","last_became_peered":"2023-10-01T10:37:42.767211+0000","last_unstale":"2023-10-01T10:51:31.753976+0000","last_undegraded":"2023-10-01T10:51:31.753976+0000","last_fullsized":"2023-10-01T10:51:31.753976+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:37:20.357407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":145,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:16.700766+0000","last_change":"2023-10-01T10:37:42.768642+0000","last_active":"2023-10-01T10:51:16.700766+0000","last_peered":"2023-10-01T10:51:16.700766+0000","last_clean":"2023-10-01T10:51:16.700766+0000","last_became_active":"2023-10-01T10:37:42.768239+0000","last_became_peered":"2023-10-01T10:37:42.768239+0000","last_unstale":"2023-10-01T10:51:16.700766+0000","last_undegraded":"2023-10-01T10:51:16.700766+0000","last_fullsized":"2023-10-01T10:51:16.700766+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":145,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:16.702919+0000","last_change":"2023-10-01T10:38:16.028563+0000","last_active":"2023-10-01T10:51:16.702919+0000","last_peered":"2023-10-01T10:51:16.702919+0000","last_clean":"2023-10-01T10:51:16.702919+0000","last_became_active":"2023-10-01T10:38:16.028162+0000","last_became_peered":"2023-10-01T10:38:16.028162+0000","last_unstale":"2023-10-01T10:51:16.702919+0000","last_undegraded":"2023-10-01T10:51:16.702919+0000","last_fullsized":"2023-10-01T10:51:16.702919+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":139,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:17.532056+0000","last_change":"2023-10-01T10:37:42.772247+0000","last_active":"2023-10-01T10:51:17.532056+0000","last_peered":"2023-10-01T10:51:17.532056+0000","last_clean":"2023-10-01T10:51:17.532056+0000","last_became_active":"2023-10-01T10:37:42.771697+0000","last_became_peered":"2023-10-01T10:37:42.771697+0000","last_unstale":"2023-10-01T10:51:17.532056+0000","last_undegraded":"2023-10-01T10:51:17.532056+0000","last_fullsized":"2023-10-01T10:51:17.532056+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:23:15.018248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":153,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:18.072087+0000","last_change":"2023-10-01T10:28:20.211798+0000","last_active":"2023-10-01T10:51:18.072087+0000","last_peered":"2023-10-01T10:51:18.072087+0000","last_clean":"2023-10-01T10:51:18.072087+0000","last_became_active":"2023-10-01T10:28:20.210791+0000","last_became_peered":"2023-10-01T10:28:20.210791+0000","last_unstale":"2023-10-01T10:51:18.072087+0000","last_undegraded":"2023-10-01T10:51:18.072087+0000","last_fullsized":"2023-10-01T10:51:18.072087+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":143,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:18.883371+0000","last_change":"2023-10-01T10:37:42.773127+0000","last_active":"2023-10-01T10:51:18.883371+0000","last_peered":"2023-10-01T10:51:18.883371+0000","last_clean":"2023-10-01T10:51:18.883371+0000","last_became_active":"2023-10-01T10:37:42.772102+0000","last_became_peered":"2023-10-01T10:37:42.772102+0000","last_unstale":"2023-10-01T10:51:18.883371+0000","last_undegraded":"2023-10-01T10:51:18.883371+0000","last_fullsized":"2023-10-01T10:51:18.883371+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:23:26.202295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"179'3013","reported_seq":3027,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-01T10:51:49.524291+0000","last_change":"2023-10-01T10:51:01.948311+0000","last_active":"2023-10-01T10:51:49.524291+0000","last_peered":"2023-10-01T10:51:49.524291+0000","last_clean":"2023-10-01T10:51:49.524291+0000","last_became_active":"2023-10-01T10:51:01.947921+0000","last_became_peered":"2023-10-01T10:51:01.947921+0000","last_unstale":"2023-10-01T10:51:49.524291+0000","last_undegraded":"2023-10-01T10:51:49.524291+0000","last_fullsized":"2023-10-01T10:51:49.524291+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":3013,"log_dups_size":0,"ondisk_log_size":3013,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":197459968,"num_objects":3013,"num_object_clones":0,"num_object_copies":9039,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3013,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3013,"num_write_kb":192832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":161,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:16.703634+0000","last_change":"2023-10-01T10:38:16.026955+0000","last_active":"2023-10-01T10:51:16.703634+0000","last_peered":"2023-10-01T10:51:16.703634+0000","last_clean":"2023-10-01T10:51:16.703634+0000","last_became_active":"2023-10-01T10:38:16.026552+0000","last_became_peered":"2023-10-01T10:38:16.026552+0000","last_unstale":"2023-10-01T10:51:16.703634+0000","last_undegraded":"2023-10-01T10:51:16.703634+0000","last_fullsized":"2023-10-01T10:51:16.703634+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":130,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:16.700025+0000","last_change":"2023-10-01T10:15:40.318702+0000","last_active":"2023-10-01T10:51:16.700025+0000","last_peered":"2023-10-01T10:51:16.700025+0000","last_clean":"2023-10-01T10:51:16.700025+0000","last_became_active":"2023-10-01T10:15:40.318333+0000","last_became_peered":"2023-10-01T10:15:40.318333+0000","last_unstale":"2023-10-01T10:51:16.700025+0000","last_undegraded":"2023-10-01T10:51:16.700025+0000","last_fullsized":"2023-10-01T10:51:16.700025+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"179'2985","reported_seq":3000,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-01T10:51:49.078435+0000","last_change":"2023-10-01T10:51:01.947670+0000","last_active":"2023-10-01T10:51:49.078435+0000","last_peered":"2023-10-01T10:51:49.078435+0000","last_clean":"2023-10-01T10:51:49.078435+0000","last_became_active":"2023-10-01T10:51:01.947301+0000","last_became_peered":"2023-10-01T10:51:01.947301+0000","last_unstale":"2023-10-01T10:51:49.078435+0000","last_undegraded":"2023-10-01T10:51:49.078435+0000","last_fullsized":"2023-10-01T10:51:49.078435+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":2985,"log_dups_size":0,"ondisk_log_size":2985,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T22:33:54.269148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":195624960,"num_objects":2985,"num_object_clones":0,"num_object_copies":8955,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2985,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2985,"num_write_kb":191040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":162,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:16.703455+0000","last_change":"2023-10-01T10:38:16.027753+0000","last_active":"2023-10-01T10:51:16.703455+0000","last_peered":"2023-10-01T10:51:16.703455+0000","last_clean":"2023-10-01T10:51:16.703455+0000","last_became_active":"2023-10-01T10:38:16.027384+0000","last_became_peered":"2023-10-01T10:38:16.027384+0000","last_unstale":"2023-10-01T10:51:16.703455+0000","last_undegraded":"2023-10-01T10:51:16.703455+0000","last_fullsized":"2023-10-01T10:51:16.703455+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":173,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:16.698318+0000","last_change":"2023-10-01T10:38:16.033203+0000","last_active":"2023-10-01T10:51:16.698318+0000","last_peered":"2023-10-01T10:51:16.698318+0000","last_clean":"2023-10-01T10:51:16.698318+0000","last_became_active":"2023-10-01T10:38:16.032681+0000","last_became_peered":"2023-10-01T10:38:16.032681+0000","last_unstale":"2023-10-01T10:51:16.698318+0000","last_undegraded":"2023-10-01T10:51:16.698318+0000","last_fullsized":"2023-10-01T10:51:16.698318+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:42:36.642284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"179'3478","reported_seq":3491,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-01T10:51:53.936202+0000","last_change":"2023-10-01T10:51:01.956461+0000","last_active":"2023-10-01T10:51:53.936202+0000","last_peered":"2023-10-01T10:51:53.936202+0000","last_clean":"2023-10-01T10:51:53.936202+0000","last_became_active":"2023-10-01T10:51:01.956123+0000","last_became_peered":"2023-10-01T10:51:01.956123+0000","last_unstale":"2023-10-01T10:51:53.936202+0000","last_undegraded":"2023-10-01T10:51:53.936202+0000","last_fullsized":"2023-10-01T10:51:53.936202+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":3478,"log_dups_size":0,"ondisk_log_size":3478,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":227934208,"num_objects":3478,"num_object_clones":0,"num_object_copies":10434,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3478,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3478,"num_write_kb":222592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":131,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:16.703337+0000","last_change":"2023-10-01T10:15:39.293587+0000","last_active":"2023-10-01T10:51:16.703337+0000","last_peered":"2023-10-01T10:51:16.703337+0000","last_clean":"2023-10-01T10:51:16.703337+0000","last_became_active":"2023-10-01T10:15:39.293273+0000","last_became_peered":"2023-10-01T10:15:39.293273+0000","last_unstale":"2023-10-01T10:51:16.703337+0000","last_undegraded":"2023-10-01T10:51:16.703337+0000","last_fullsized":"2023-10-01T10:51:16.703337+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":144,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:18.072317+0000","last_change":"2023-10-01T10:32:31.571971+0000","last_active":"2023-10-01T10:51:18.072317+0000","last_peered":"2023-10-01T10:51:18.072317+0000","last_clean":"2023-10-01T10:51:18.072317+0000","last_became_active":"2023-10-01T10:32:31.571410+0000","last_became_peered":"2023-10-01T10:32:31.571410+0000","last_unstale":"2023-10-01T10:51:18.072317+0000","last_undegraded":"2023-10-01T10:51:18.072317+0000","last_fullsized":"2023-10-01T10:51:18.072317+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"179'3345","reported_seq":3359,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-01T10:51:53.874134+0000","last_change":"2023-10-01T10:51:01.955379+0000","last_active":"2023-10-01T10:51:53.874134+0000","last_peered":"2023-10-01T10:51:53.874134+0000","last_clean":"2023-10-01T10:51:53.874134+0000","last_became_active":"2023-10-01T10:51:01.954911+0000","last_became_peered":"2023-10-01T10:51:01.954911+0000","last_unstale":"2023-10-01T10:51:53.874134+0000","last_undegraded":"2023-10-01T10:51:53.874134+0000","last_fullsized":"2023-10-01T10:51:53.874134+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":3345,"log_dups_size":0,"ondisk_log_size":3345,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":219217920,"num_objects":3345,"num_object_clones":0,"num_object_copies":10035,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3345,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3345,"num_write_kb":214080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":146,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:16.700446+0000","last_change":"2023-10-01T10:37:42.765733+0000","last_active":"2023-10-01T10:51:16.700446+0000","last_peered":"2023-10-01T10:51:16.700446+0000","last_clean":"2023-10-01T10:51:16.700446+0000","last_became_active":"2023-10-01T10:37:42.765311+0000","last_became_peered":"2023-10-01T10:37:42.765311+0000","last_unstale":"2023-10-01T10:51:16.700446+0000","last_undegraded":"2023-10-01T10:51:16.700446+0000","last_fullsized":"2023-10-01T10:51:16.700446+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":155,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:17.531634+0000","last_change":"2023-10-01T10:38:16.026348+0000","last_active":"2023-10-01T10:51:17.531634+0000","last_peered":"2023-10-01T10:51:17.531634+0000","last_clean":"2023-10-01T10:51:17.531634+0000","last_became_active":"2023-10-01T10:38:16.025747+0000","last_became_peered":"2023-10-01T10:38:16.025747+0000","last_unstale":"2023-10-01T10:51:17.531634+0000","last_undegraded":"2023-10-01T10:51:17.531634+0000","last_fullsized":"2023-10-01T10:51:17.531634+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:28:18.934480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"179'3481","reported_seq":3495,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-01T10:51:53.825789+0000","last_change":"2023-10-01T10:51:01.954225+0000","last_active":"2023-10-01T10:51:53.825789+0000","last_peered":"2023-10-01T10:51:53.825789+0000","last_clean":"2023-10-01T10:51:53.825789+0000","last_became_active":"2023-10-01T10:51:01.953830+0000","last_became_peered":"2023-10-01T10:51:01.953830+0000","last_unstale":"2023-10-01T10:51:53.825789+0000","last_undegraded":"2023-10-01T10:51:53.825789+0000","last_fullsized":"2023-10-01T10:51:53.825789+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":3481,"log_dups_size":0,"ondisk_log_size":3481,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":228130816,"num_objects":3481,"num_object_clones":0,"num_object_copies":10443,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3481,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3481,"num_write_kb":222784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":147,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:18.883134+0000","last_change":"2023-10-01T10:38:16.022493+0000","last_active":"2023-10-01T10:51:18.883134+0000","last_peered":"2023-10-01T10:51:18.883134+0000","last_clean":"2023-10-01T10:51:18.883134+0000","last_became_active":"2023-10-01T10:38:16.022033+0000","last_became_peered":"2023-10-01T10:38:16.022033+0000","last_unstale":"2023-10-01T10:51:18.883134+0000","last_undegraded":"2023-10-01T10:51:18.883134+0000","last_fullsized":"2023-10-01T10:51:18.883134+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:49:05.262763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":158,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-01T10:51:31.759648+0000","last_change":"2023-10-01T10:38:16.030613+0000","last_active":"2023-10-01T10:51:31.759648+0000","last_peered":"2023-10-01T10:51:31.759648+0000","last_clean":"2023-10-01T10:51:31.759648+0000","last_became_active":"2023-10-01T10:38:16.028975+0000","last_became_peered":"2023-10-01T10:38:16.028975+0000","last_unstale":"2023-10-01T10:51:31.759648+0000","last_undegraded":"2023-10-01T10:51:31.759648+0000","last_fullsized":"2023-10-01T10:51:31.759648+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:46:07.287911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"179'3392","reported_seq":3407,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-01T10:51:53.604616+0000","last_change":"2023-10-01T10:51:01.946522+0000","last_active":"2023-10-01T10:51:53.604616+0000","last_peered":"2023-10-01T10:51:53.604616+0000","last_clean":"2023-10-01T10:51:53.604616+0000","last_became_active":"2023-10-01T10:51:01.945971+0000","last_became_peered":"2023-10-01T10:51:01.945971+0000","last_unstale":"2023-10-01T10:51:53.604616+0000","last_undegraded":"2023-10-01T10:51:53.604616+0000","last_fullsized":"2023-10-01T10:51:53.604616+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":3392,"log_dups_size":0,"ondisk_log_size":3392,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:34:58.733457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":222298112,"num_objects":3392,"num_object_clones":0,"num_object_copies":10176,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3392,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3392,"num_write_kb":217088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":130,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:16.698016+0000","last_change":"2023-10-01T10:47:54.750295+0000","last_active":"2023-10-01T10:51:16.698016+0000","last_peered":"2023-10-01T10:51:16.698016+0000","last_clean":"2023-10-01T10:51:16.698016+0000","last_became_active":"2023-10-01T10:47:54.749917+0000","last_became_peered":"2023-10-01T10:47:54.749917+0000","last_unstale":"2023-10-01T10:51:16.698016+0000","last_undegraded":"2023-10-01T10:51:16.698016+0000","last_fullsized":"2023-10-01T10:51:16.698016+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:30:59.236210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":163,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-01T10:51:31.759408+0000","last_change":"2023-10-01T10:38:16.035906+0000","last_active":"2023-10-01T10:51:31.759408+0000","last_peered":"2023-10-01T10:51:31.759408+0000","last_clean":"2023-10-01T10:51:31.759408+0000","last_became_active":"2023-10-01T10:38:16.035383+0000","last_became_peered":"2023-10-01T10:38:16.035383+0000","last_unstale":"2023-10-01T10:51:31.759408+0000","last_undegraded":"2023-10-01T10:51:31.759408+0000","last_fullsized":"2023-10-01T10:51:31.759408+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:44:41.500097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"179'3304","reported_seq":3318,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-01T10:51:51.753407+0000","last_change":"2023-10-01T10:51:01.954256+0000","last_active":"2023-10-01T10:51:51.753407+0000","last_peered":"2023-10-01T10:51:51.753407+0000","last_clean":"2023-10-01T10:51:51.753407+0000","last_became_active":"2023-10-01T10:51:01.953595+0000","last_became_peered":"2023-10-01T10:51:01.953595+0000","last_unstale":"2023-10-01T10:51:51.753407+0000","last_undegraded":"2023-10-01T10:51:51.753407+0000","last_fullsized":"2023-10-01T10:51:51.753407+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":3304,"log_dups_size":0,"ondisk_log_size":3304,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:55:22.868887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":216530944,"num_objects":3304,"num_object_clones":0,"num_object_copies":9912,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3304,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3304,"num_write_kb":211456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":155,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:16.698719+0000","last_change":"2023-10-01T10:38:16.026611+0000","last_active":"2023-10-01T10:51:16.698719+0000","last_peered":"2023-10-01T10:51:16.698719+0000","last_clean":"2023-10-01T10:51:16.698719+0000","last_became_active":"2023-10-01T10:38:16.026267+0000","last_became_peered":"2023-10-01T10:38:16.026267+0000","last_unstale":"2023-10-01T10:51:16.698719+0000","last_undegraded":"2023-10-01T10:51:16.698719+0000","last_fullsized":"2023-10-01T10:51:16.698719+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:06.135288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":139,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-01T10:51:31.759391+0000","last_change":"2023-10-01T10:38:16.032162+0000","last_active":"2023-10-01T10:51:31.759391+0000","last_peered":"2023-10-01T10:51:31.759391+0000","last_clean":"2023-10-01T10:51:31.759391+0000","last_became_active":"2023-10-01T10:38:16.031785+0000","last_became_peered":"2023-10-01T10:38:16.031785+0000","last_unstale":"2023-10-01T10:51:31.759391+0000","last_undegraded":"2023-10-01T10:51:31.759391+0000","last_fullsized":"2023-10-01T10:51:31.759391+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:38:43.003626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"179'3241","reported_seq":3256,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-01T10:51:51.023680+0000","last_change":"2023-10-01T10:51:01.953089+0000","last_active":"2023-10-01T10:51:51.023680+0000","last_peered":"2023-10-01T10:51:51.023680+0000","last_clean":"2023-10-01T10:51:51.023680+0000","last_became_active":"2023-10-01T10:51:01.952789+0000","last_became_peered":"2023-10-01T10:51:01.952789+0000","last_unstale":"2023-10-01T10:51:51.023680+0000","last_undegraded":"2023-10-01T10:51:51.023680+0000","last_fullsized":"2023-10-01T10:51:51.023680+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":3241,"log_dups_size":0,"ondisk_log_size":3241,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:28:59.885703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212402176,"num_objects":3241,"num_object_clones":0,"num_object_copies":9723,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3241,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3241,"num_write_kb":207424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":145,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:18.883455+0000","last_change":"2023-10-01T10:37:42.772398+0000","last_active":"2023-10-01T10:51:18.883455+0000","last_peered":"2023-10-01T10:51:18.883455+0000","last_clean":"2023-10-01T10:51:18.883455+0000","last_became_active":"2023-10-01T10:37:42.772084+0000","last_became_peered":"2023-10-01T10:37:42.772084+0000","last_unstale":"2023-10-01T10:51:18.883455+0000","last_undegraded":"2023-10-01T10:51:18.883455+0000","last_fullsized":"2023-10-01T10:51:18.883455+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:32:43.846419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":163,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:18.741052+0000","last_change":"2023-10-01T10:38:16.019755+0000","last_active":"2023-10-01T10:51:18.741052+0000","last_peered":"2023-10-01T10:51:18.741052+0000","last_clean":"2023-10-01T10:51:18.741052+0000","last_became_active":"2023-10-01T10:38:16.019397+0000","last_became_peered":"2023-10-01T10:38:16.019397+0000","last_unstale":"2023-10-01T10:51:18.741052+0000","last_undegraded":"2023-10-01T10:51:18.741052+0000","last_fullsized":"2023-10-01T10:51:18.741052+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:42:56.623225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.d","version":"179'3352","reported_seq":3366,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-01T10:51:53.612336+0000","last_change":"2023-10-01T10:51:01.952187+0000","last_active":"2023-10-01T10:51:53.612336+0000","last_peered":"2023-10-01T10:51:53.612336+0000","last_clean":"2023-10-01T10:51:53.612336+0000","last_became_active":"2023-10-01T10:51:01.951373+0000","last_became_peered":"2023-10-01T10:51:01.951373+0000","last_unstale":"2023-10-01T10:51:53.612336+0000","last_undegraded":"2023-10-01T10:51:53.612336+0000","last_fullsized":"2023-10-01T10:51:53.612336+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":3352,"log_dups_size":0,"ondisk_log_size":3352,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:10:27.939483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":219676672,"num_objects":3352,"num_object_clones":0,"num_object_copies":10056,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3352,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3352,"num_write_kb":214528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":157,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:18.320921+0000","last_change":"2023-10-01T10:32:31.561108+0000","last_active":"2023-10-01T10:51:18.320921+0000","last_peered":"2023-10-01T10:51:18.320921+0000","last_clean":"2023-10-01T10:51:18.320921+0000","last_became_active":"2023-10-01T10:32:31.560791+0000","last_became_peered":"2023-10-01T10:32:31.560791+0000","last_unstale":"2023-10-01T10:51:18.320921+0000","last_undegraded":"2023-10-01T10:51:18.320921+0000","last_fullsized":"2023-10-01T10:51:18.320921+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:42:47.477671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":161,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:16.700695+0000","last_change":"2023-10-01T10:37:42.766814+0000","last_active":"2023-10-01T10:51:16.700695+0000","last_peered":"2023-10-01T10:51:16.700695+0000","last_clean":"2023-10-01T10:51:16.700695+0000","last_became_active":"2023-10-01T10:37:42.766451+0000","last_became_peered":"2023-10-01T10:37:42.766451+0000","last_unstale":"2023-10-01T10:51:16.700695+0000","last_undegraded":"2023-10-01T10:51:16.700695+0000","last_fullsized":"2023-10-01T10:51:16.700695+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":123,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:18.321095+0000","last_change":"2023-10-01T10:38:16.034357+0000","last_active":"2023-10-01T10:51:18.321095+0000","last_peered":"2023-10-01T10:51:18.321095+0000","last_clean":"2023-10-01T10:51:18.321095+0000","last_became_active":"2023-10-01T10:38:16.033905+0000","last_became_peered":"2023-10-01T10:38:16.033905+0000","last_unstale":"2023-10-01T10:51:18.321095+0000","last_undegraded":"2023-10-01T10:51:18.321095+0000","last_fullsized":"2023-10-01T10:51:18.321095+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:47:58.369835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.b","version":"179'3268","reported_seq":3283,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-01T10:51:51.786171+0000","last_change":"2023-10-01T10:51:01.950278+0000","last_active":"2023-10-01T10:51:51.786171+0000","last_peered":"2023-10-01T10:51:51.786171+0000","last_clean":"2023-10-01T10:51:51.786171+0000","last_became_active":"2023-10-01T10:51:01.949902+0000","last_became_peered":"2023-10-01T10:51:01.949902+0000","last_unstale":"2023-10-01T10:51:51.786171+0000","last_undegraded":"2023-10-01T10:51:51.786171+0000","last_fullsized":"2023-10-01T10:51:51.786171+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":3268,"log_dups_size":0,"ondisk_log_size":3268,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:42:45.648175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":214171648,"num_objects":3268,"num_object_clones":0,"num_object_copies":9804,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3268,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3268,"num_write_kb":209152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":150,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:16.700190+0000","last_change":"2023-10-01T10:32:31.572482+0000","last_active":"2023-10-01T10:51:16.700190+0000","last_peered":"2023-10-01T10:51:16.700190+0000","last_clean":"2023-10-01T10:51:16.700190+0000","last_became_active":"2023-10-01T10:32:31.571993+0000","last_became_peered":"2023-10-01T10:32:31.571993+0000","last_unstale":"2023-10-01T10:51:16.700190+0000","last_undegraded":"2023-10-01T10:51:16.700190+0000","last_fullsized":"2023-10-01T10:51:16.700190+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":125,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:16.698571+0000","last_change":"2023-10-01T10:38:16.030202+0000","last_active":"2023-10-01T10:51:16.698571+0000","last_peered":"2023-10-01T10:51:16.698571+0000","last_clean":"2023-10-01T10:51:16.698571+0000","last_became_active":"2023-10-01T10:38:16.029917+0000","last_became_peered":"2023-10-01T10:38:16.029917+0000","last_unstale":"2023-10-01T10:51:16.698571+0000","last_undegraded":"2023-10-01T10:51:16.698571+0000","last_fullsized":"2023-10-01T10:51:16.698571+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:20:06.763909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":138,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:18.321141+0000","last_change":"2023-10-01T10:37:42.761153+0000","last_active":"2023-10-01T10:51:18.321141+0000","last_peered":"2023-10-01T10:51:18.321141+0000","last_clean":"2023-10-01T10:51:18.321141+0000","last_became_active":"2023-10-01T10:37:42.760715+0000","last_became_peered":"2023-10-01T10:37:42.760715+0000","last_unstale":"2023-10-01T10:51:18.321141+0000","last_undegraded":"2023-10-01T10:51:18.321141+0000","last_fullsized":"2023-10-01T10:51:18.321141+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:52:50.237174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.8","version":"179'3321","reported_seq":3335,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-01T10:51:51.770339+0000","last_change":"2023-10-01T10:51:01.955996+0000","last_active":"2023-10-01T10:51:51.770339+0000","last_peered":"2023-10-01T10:51:51.770339+0000","last_clean":"2023-10-01T10:51:51.770339+0000","last_became_active":"2023-10-01T10:51:01.955685+0000","last_became_peered":"2023-10-01T10:51:01.955685+0000","last_unstale":"2023-10-01T10:51:51.770339+0000","last_undegraded":"2023-10-01T10:51:51.770339+0000","last_fullsized":"2023-10-01T10:51:51.770339+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":3321,"log_dups_size":0,"ondisk_log_size":3321,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:54:42.840196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":217645056,"num_objects":3321,"num_object_clones":0,"num_object_copies":9963,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3321,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3321,"num_write_kb":212544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":155,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-01T10:51:31.755069+0000","last_change":"2023-10-01T10:37:42.762524+0000","last_active":"2023-10-01T10:51:31.755069+0000","last_peered":"2023-10-01T10:51:31.755069+0000","last_clean":"2023-10-01T10:51:31.755069+0000","last_became_active":"2023-10-01T10:37:42.762209+0000","last_became_peered":"2023-10-01T10:37:42.762209+0000","last_unstale":"2023-10-01T10:51:31.755069+0000","last_undegraded":"2023-10-01T10:51:31.755069+0000","last_fullsized":"2023-10-01T10:51:31.755069+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:45:38.171397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":123,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:18.320873+0000","last_change":"2023-10-01T10:37:05.019330+0000","last_active":"2023-10-01T10:51:18.320873+0000","last_peered":"2023-10-01T10:51:18.320873+0000","last_clean":"2023-10-01T10:51:18.320873+0000","last_became_active":"2023-10-01T10:37:05.018842+0000","last_became_peered":"2023-10-01T10:37:05.018842+0000","last_unstale":"2023-10-01T10:51:18.320873+0000","last_undegraded":"2023-10-01T10:51:18.320873+0000","last_fullsized":"2023-10-01T10:51:18.320873+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:48:09.551917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":151,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:18.321397+0000","last_change":"2023-10-01T10:32:31.569162+0000","last_active":"2023-10-01T10:51:18.321397+0000","last_peered":"2023-10-01T10:51:18.321397+0000","last_clean":"2023-10-01T10:51:18.321397+0000","last_became_active":"2023-10-01T10:32:31.568956+0000","last_became_peered":"2023-10-01T10:32:31.568956+0000","last_unstale":"2023-10-01T10:51:18.321397+0000","last_undegraded":"2023-10-01T10:51:18.321397+0000","last_fullsized":"2023-10-01T10:51:18.321397+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:02:08.285304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.a","version":"179'3243","reported_seq":3257,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-01T10:51:51.773689+0000","last_change":"2023-10-01T10:51:01.955145+0000","last_active":"2023-10-01T10:51:51.773689+0000","last_peered":"2023-10-01T10:51:51.773689+0000","last_clean":"2023-10-01T10:51:51.773689+0000","last_became_active":"2023-10-01T10:51:01.954793+0000","last_became_peered":"2023-10-01T10:51:01.954793+0000","last_unstale":"2023-10-01T10:51:51.773689+0000","last_undegraded":"2023-10-01T10:51:51.773689+0000","last_fullsized":"2023-10-01T10:51:51.773689+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":3243,"log_dups_size":0,"ondisk_log_size":3243,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:25:33.468595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212533248,"num_objects":3243,"num_object_clones":0,"num_object_copies":9729,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3243,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3243,"num_write_kb":207552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":150,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:16.700198+0000","last_change":"2023-10-01T10:37:05.024009+0000","last_active":"2023-10-01T10:51:16.700198+0000","last_peered":"2023-10-01T10:51:16.700198+0000","last_clean":"2023-10-01T10:51:16.700198+0000","last_became_active":"2023-10-01T10:37:05.023458+0000","last_became_peered":"2023-10-01T10:37:05.023458+0000","last_unstale":"2023-10-01T10:51:16.700198+0000","last_undegraded":"2023-10-01T10:51:16.700198+0000","last_fullsized":"2023-10-01T10:51:16.700198+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'139","reported_seq":381,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:16.702762+0000","last_change":"2023-10-01T10:28:20.217386+0000","last_active":"2023-10-01T10:51:16.702762+0000","last_peered":"2023-10-01T10:51:16.702762+0000","last_clean":"2023-10-01T10:51:16.702762+0000","last_became_active":"2023-10-01T10:28:20.216803+0000","last_became_peered":"2023-10-01T10:28:20.216803+0000","last_unstale":"2023-10-01T10:51:16.702762+0000","last_undegraded":"2023-10-01T10:51:16.702762+0000","last_fullsized":"2023-10-01T10:51:16.702762+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:09:17.221082+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:09:17.221082+0000","last_clean_scrub_stamp":"2023-10-01T10:09:17.221082+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":127,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:18.882053+0000","last_change":"2023-10-01T10:37:42.765100+0000","last_active":"2023-10-01T10:51:18.882053+0000","last_peered":"2023-10-01T10:51:18.882053+0000","last_clean":"2023-10-01T10:51:18.882053+0000","last_became_active":"2023-10-01T10:37:42.764761+0000","last_became_peered":"2023-10-01T10:37:42.764761+0000","last_unstale":"2023-10-01T10:51:18.882053+0000","last_undegraded":"2023-10-01T10:51:18.882053+0000","last_fullsized":"2023-10-01T10:51:18.882053+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:24:56.907377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":114,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:16.699234+0000","last_change":"2023-10-01T10:47:54.782475+0000","last_active":"2023-10-01T10:51:16.699234+0000","last_peered":"2023-10-01T10:51:16.699234+0000","last_clean":"2023-10-01T10:51:16.699234+0000","last_became_active":"2023-10-01T10:47:54.781725+0000","last_became_peered":"2023-10-01T10:47:54.781725+0000","last_unstale":"2023-10-01T10:51:16.699234+0000","last_undegraded":"2023-10-01T10:51:16.699234+0000","last_fullsized":"2023-10-01T10:51:16.699234+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:21:30.978552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"179'3292","reported_seq":3304,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-01T10:51:51.778216+0000","last_change":"2023-10-01T10:51:01.952824+0000","last_active":"2023-10-01T10:51:51.778216+0000","last_peered":"2023-10-01T10:51:51.778216+0000","last_clean":"2023-10-01T10:51:51.778216+0000","last_became_active":"2023-10-01T10:51:01.952483+0000","last_became_peered":"2023-10-01T10:51:01.952483+0000","last_unstale":"2023-10-01T10:51:51.778216+0000","last_undegraded":"2023-10-01T10:51:51.778216+0000","last_fullsized":"2023-10-01T10:51:51.778216+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":3292,"log_dups_size":0,"ondisk_log_size":3292,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:42:42.460140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":215744512,"num_objects":3292,"num_object_clones":0,"num_object_copies":9876,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3292,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3292,"num_write_kb":210688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":164,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:16.702712+0000","last_change":"2023-10-01T10:32:31.565868+0000","last_active":"2023-10-01T10:51:16.702712+0000","last_peered":"2023-10-01T10:51:16.702712+0000","last_clean":"2023-10-01T10:51:16.702712+0000","last_became_active":"2023-10-01T10:32:31.565428+0000","last_became_peered":"2023-10-01T10:32:31.565428+0000","last_unstale":"2023-10-01T10:51:16.702712+0000","last_undegraded":"2023-10-01T10:51:16.702712+0000","last_fullsized":"2023-10-01T10:51:16.702712+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":155,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:17.531076+0000","last_change":"2023-10-01T10:38:16.021968+0000","last_active":"2023-10-01T10:51:17.531076+0000","last_peered":"2023-10-01T10:51:17.531076+0000","last_clean":"2023-10-01T10:51:17.531076+0000","last_became_active":"2023-10-01T10:38:16.021539+0000","last_became_peered":"2023-10-01T10:38:16.021539+0000","last_unstale":"2023-10-01T10:51:17.531076+0000","last_undegraded":"2023-10-01T10:51:17.531076+0000","last_fullsized":"2023-10-01T10:51:17.531076+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:12:53.105747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":155,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:17.532057+0000","last_change":"2023-10-01T10:37:05.017431+0000","last_active":"2023-10-01T10:51:17.532057+0000","last_peered":"2023-10-01T10:51:17.532057+0000","last_clean":"2023-10-01T10:51:17.532057+0000","last_became_active":"2023-10-01T10:37:05.017094+0000","last_became_peered":"2023-10-01T10:37:05.017094+0000","last_unstale":"2023-10-01T10:51:17.532057+0000","last_undegraded":"2023-10-01T10:51:17.532057+0000","last_fullsized":"2023-10-01T10:51:17.532057+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:10:40.664686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"179'3230","reported_seq":3245,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-01T10:51:51.028570+0000","last_change":"2023-10-01T10:51:01.947328+0000","last_active":"2023-10-01T10:51:51.028570+0000","last_peered":"2023-10-01T10:51:51.028570+0000","last_clean":"2023-10-01T10:51:51.028570+0000","last_became_active":"2023-10-01T10:51:01.946666+0000","last_became_peered":"2023-10-01T10:51:01.946666+0000","last_unstale":"2023-10-01T10:51:51.028570+0000","last_undegraded":"2023-10-01T10:51:51.028570+0000","last_fullsized":"2023-10-01T10:51:51.028570+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":3230,"log_dups_size":0,"ondisk_log_size":3230,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:19:24.372691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":211681280,"num_objects":3230,"num_object_clones":0,"num_object_copies":9690,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3230,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3230,"num_write_kb":206720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":164,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:16.703533+0000","last_change":"2023-10-01T10:38:16.032774+0000","last_active":"2023-10-01T10:51:16.703533+0000","last_peered":"2023-10-01T10:51:16.703533+0000","last_clean":"2023-10-01T10:51:16.703533+0000","last_became_active":"2023-10-01T10:38:16.032283+0000","last_became_peered":"2023-10-01T10:38:16.032283+0000","last_unstale":"2023-10-01T10:51:16.703533+0000","last_undegraded":"2023-10-01T10:51:16.703533+0000","last_fullsized":"2023-10-01T10:51:16.703533+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":123,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:18.321150+0000","last_change":"2023-10-01T10:37:05.021228+0000","last_active":"2023-10-01T10:51:18.321150+0000","last_peered":"2023-10-01T10:51:18.321150+0000","last_clean":"2023-10-01T10:51:18.321150+0000","last_became_active":"2023-10-01T10:37:05.020888+0000","last_became_peered":"2023-10-01T10:37:05.020888+0000","last_unstale":"2023-10-01T10:51:18.321150+0000","last_undegraded":"2023-10-01T10:51:18.321150+0000","last_fullsized":"2023-10-01T10:51:18.321150+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:59:57.641247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":126,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:17.531536+0000","last_change":"2023-10-01T10:28:20.220812+0000","last_active":"2023-10-01T10:51:17.531536+0000","last_peered":"2023-10-01T10:51:17.531536+0000","last_clean":"2023-10-01T10:51:17.531536+0000","last_became_active":"2023-10-01T10:28:20.215962+0000","last_became_peered":"2023-10-01T10:28:20.215962+0000","last_unstale":"2023-10-01T10:51:17.531536+0000","last_undegraded":"2023-10-01T10:51:17.531536+0000","last_fullsized":"2023-10-01T10:51:17.531536+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:35:15.275097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"179'3358","reported_seq":3372,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-01T10:51:53.880340+0000","last_change":"2023-10-01T10:51:01.953666+0000","last_active":"2023-10-01T10:51:53.880340+0000","last_peered":"2023-10-01T10:51:53.880340+0000","last_clean":"2023-10-01T10:51:53.880340+0000","last_became_active":"2023-10-01T10:51:01.952070+0000","last_became_peered":"2023-10-01T10:51:01.952070+0000","last_unstale":"2023-10-01T10:51:53.880340+0000","last_undegraded":"2023-10-01T10:51:53.880340+0000","last_fullsized":"2023-10-01T10:51:53.880340+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":3358,"log_dups_size":0,"ondisk_log_size":3358,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":220069888,"num_objects":3358,"num_object_clones":0,"num_object_copies":10074,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3358,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3358,"num_write_kb":214912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":132,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:18.883443+0000","last_change":"2023-10-01T10:37:42.773999+0000","last_active":"2023-10-01T10:51:18.883443+0000","last_peered":"2023-10-01T10:51:18.883443+0000","last_clean":"2023-10-01T10:51:18.883443+0000","last_became_active":"2023-10-01T10:37:42.773355+0000","last_became_peered":"2023-10-01T10:37:42.773355+0000","last_unstale":"2023-10-01T10:51:18.883443+0000","last_undegraded":"2023-10-01T10:51:18.883443+0000","last_fullsized":"2023-10-01T10:51:18.883443+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:46:51.317151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":164,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:18.321337+0000","last_change":"2023-10-01T10:38:16.034193+0000","last_active":"2023-10-01T10:51:18.321337+0000","last_peered":"2023-10-01T10:51:18.321337+0000","last_clean":"2023-10-01T10:51:18.321337+0000","last_became_active":"2023-10-01T10:38:16.033703+0000","last_became_peered":"2023-10-01T10:38:16.033703+0000","last_unstale":"2023-10-01T10:51:18.321337+0000","last_undegraded":"2023-10-01T10:51:18.321337+0000","last_fullsized":"2023-10-01T10:51:18.321337+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:21:26.061595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":130,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-01T10:51:31.749587+0000","last_change":"2023-10-01T10:37:42.773538+0000","last_active":"2023-10-01T10:51:31.749587+0000","last_peered":"2023-10-01T10:51:31.749587+0000","last_clean":"2023-10-01T10:51:31.749587+0000","last_became_active":"2023-10-01T10:37:42.771863+0000","last_became_peered":"2023-10-01T10:37:42.771863+0000","last_unstale":"2023-10-01T10:51:31.749587+0000","last_undegraded":"2023-10-01T10:51:31.749587+0000","last_fullsized":"2023-10-01T10:51:31.749587+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:46:46.698627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"179'3103","reported_seq":3118,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-01T10:51:49.641165+0000","last_change":"2023-10-01T10:51:01.952403+0000","last_active":"2023-10-01T10:51:49.641165+0000","last_peered":"2023-10-01T10:51:49.641165+0000","last_clean":"2023-10-01T10:51:49.641165+0000","last_became_active":"2023-10-01T10:51:01.951884+0000","last_became_peered":"2023-10-01T10:51:01.951884+0000","last_unstale":"2023-10-01T10:51:49.641165+0000","last_undegraded":"2023-10-01T10:51:49.641165+0000","last_fullsized":"2023-10-01T10:51:49.641165+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":3103,"log_dups_size":0,"ondisk_log_size":3103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:33:03.736176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":203358208,"num_objects":3103,"num_object_clones":0,"num_object_copies":9309,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3103,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3103,"num_write_kb":198592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":149,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:18.072922+0000","last_change":"2023-10-01T10:37:05.023034+0000","last_active":"2023-10-01T10:51:18.072922+0000","last_peered":"2023-10-01T10:51:18.072922+0000","last_clean":"2023-10-01T10:51:18.072922+0000","last_became_active":"2023-10-01T10:37:05.022670+0000","last_became_peered":"2023-10-01T10:37:05.022670+0000","last_unstale":"2023-10-01T10:51:18.072922+0000","last_undegraded":"2023-10-01T10:51:18.072922+0000","last_fullsized":"2023-10-01T10:51:18.072922+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":158,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:17.531860+0000","last_change":"2023-10-01T10:38:16.027472+0000","last_active":"2023-10-01T10:51:17.531860+0000","last_peered":"2023-10-01T10:51:17.531860+0000","last_clean":"2023-10-01T10:51:17.531860+0000","last_became_active":"2023-10-01T10:38:16.026810+0000","last_became_peered":"2023-10-01T10:38:16.026810+0000","last_unstale":"2023-10-01T10:51:17.531860+0000","last_undegraded":"2023-10-01T10:51:17.531860+0000","last_fullsized":"2023-10-01T10:51:17.531860+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:30:06.978187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":123,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:16.698419+0000","last_change":"2023-10-01T10:38:16.035705+0000","last_active":"2023-10-01T10:51:16.698419+0000","last_peered":"2023-10-01T10:51:16.698419+0000","last_clean":"2023-10-01T10:51:16.698419+0000","last_became_active":"2023-10-01T10:38:16.035389+0000","last_became_peered":"2023-10-01T10:38:16.035389+0000","last_unstale":"2023-10-01T10:51:16.698419+0000","last_undegraded":"2023-10-01T10:51:16.698419+0000","last_fullsized":"2023-10-01T10:51:16.698419+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:16:10.201409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":142,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:17.531709+0000","last_change":"2023-10-01T10:38:16.028491+0000","last_active":"2023-10-01T10:51:17.531709+0000","last_peered":"2023-10-01T10:51:17.531709+0000","last_clean":"2023-10-01T10:51:17.531709+0000","last_became_active":"2023-10-01T10:38:16.028175+0000","last_became_peered":"2023-10-01T10:38:16.028175+0000","last_unstale":"2023-10-01T10:51:17.531709+0000","last_undegraded":"2023-10-01T10:51:17.531709+0000","last_fullsized":"2023-10-01T10:51:17.531709+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:44:29.083129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":163,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:18.740639+0000","last_change":"2023-10-01T10:37:42.762360+0000","last_active":"2023-10-01T10:51:18.740639+0000","last_peered":"2023-10-01T10:51:18.740639+0000","last_clean":"2023-10-01T10:51:18.740639+0000","last_became_active":"2023-10-01T10:37:42.761936+0000","last_became_peered":"2023-10-01T10:37:42.761936+0000","last_unstale":"2023-10-01T10:51:18.740639+0000","last_undegraded":"2023-10-01T10:51:18.740639+0000","last_fullsized":"2023-10-01T10:51:18.740639+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:50.637582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":163,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:16.700395+0000","last_change":"2023-10-01T10:38:16.037809+0000","last_active":"2023-10-01T10:51:16.700395+0000","last_peered":"2023-10-01T10:51:16.700395+0000","last_clean":"2023-10-01T10:51:16.700395+0000","last_became_active":"2023-10-01T10:38:16.037318+0000","last_became_peered":"2023-10-01T10:38:16.037318+0000","last_unstale":"2023-10-01T10:51:16.700395+0000","last_undegraded":"2023-10-01T10:51:16.700395+0000","last_fullsized":"2023-10-01T10:51:16.700395+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":172,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:16.698428+0000","last_change":"2023-10-01T10:38:16.034282+0000","last_active":"2023-10-01T10:51:16.698428+0000","last_peered":"2023-10-01T10:51:16.698428+0000","last_clean":"2023-10-01T10:51:16.698428+0000","last_became_active":"2023-10-01T10:38:16.033820+0000","last_became_peered":"2023-10-01T10:38:16.033820+0000","last_unstale":"2023-10-01T10:51:16.698428+0000","last_undegraded":"2023-10-01T10:51:16.698428+0000","last_fullsized":"2023-10-01T10:51:16.698428+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:48:23.815832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":148,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:16.700561+0000","last_change":"2023-10-01T10:38:16.038077+0000","last_active":"2023-10-01T10:51:16.700561+0000","last_peered":"2023-10-01T10:51:16.700561+0000","last_clean":"2023-10-01T10:51:16.700561+0000","last_became_active":"2023-10-01T10:38:16.037313+0000","last_became_peered":"2023-10-01T10:38:16.037313+0000","last_unstale":"2023-10-01T10:51:16.700561+0000","last_undegraded":"2023-10-01T10:51:16.700561+0000","last_fullsized":"2023-10-01T10:51:16.700561+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":172,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:16.698103+0000","last_change":"2023-10-01T10:38:16.025600+0000","last_active":"2023-10-01T10:51:16.698103+0000","last_peered":"2023-10-01T10:51:16.698103+0000","last_clean":"2023-10-01T10:51:16.698103+0000","last_became_active":"2023-10-01T10:38:16.025252+0000","last_became_peered":"2023-10-01T10:38:16.025252+0000","last_unstale":"2023-10-01T10:51:16.698103+0000","last_undegraded":"2023-10-01T10:51:16.698103+0000","last_fullsized":"2023-10-01T10:51:16.698103+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:24:18.871963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":161,"reported_epoch":179,"state":"active+clean","last_fresh":"2023-10-01T10:51:31.751893+0000","last_change":"2023-10-01T10:37:42.761243+0000","last_active":"2023-10-01T10:51:31.751893+0000","last_peered":"2023-10-01T10:51:31.751893+0000","last_clean":"2023-10-01T10:51:31.751893+0000","last_became_active":"2023-10-01T10:37:42.760834+0000","last_became_peered":"2023-10-01T10:37:42.760834+0000","last_unstale":"2023-10-01T10:51:31.751893+0000","last_undegraded":"2023-10-01T10:51:31.751893+0000","last_fullsized":"2023-10-01T10:51:31.751893+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:37:21.976095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":172,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:18.882328+0000","last_change":"2023-10-01T10:37:42.765142+0000","last_active":"2023-10-01T10:51:18.882328+0000","last_peered":"2023-10-01T10:51:18.882328+0000","last_clean":"2023-10-01T10:51:18.882328+0000","last_became_active":"2023-10-01T10:37:42.764770+0000","last_became_peered":"2023-10-01T10:37:42.764770+0000","last_unstale":"2023-10-01T10:51:18.882328+0000","last_undegraded":"2023-10-01T10:51:18.882328+0000","last_fullsized":"2023-10-01T10:51:18.882328+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:23:33.527876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":144,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:17.531894+0000","last_change":"2023-10-01T10:37:05.021621+0000","last_active":"2023-10-01T10:51:17.531894+0000","last_peered":"2023-10-01T10:51:17.531894+0000","last_clean":"2023-10-01T10:51:17.531894+0000","last_became_active":"2023-10-01T10:37:05.021173+0000","last_became_peered":"2023-10-01T10:37:05.021173+0000","last_unstale":"2023-10-01T10:51:17.531894+0000","last_undegraded":"2023-10-01T10:51:17.531894+0000","last_fullsized":"2023-10-01T10:51:17.531894+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:13:25.364499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":138,"reported_epoch":178,"state":"active+clean","last_fresh":"2023-10-01T10:51:16.697925+0000","last_change":"2023-10-01T10:38:16.031427+0000","last_active":"2023-10-01T10:51:16.697925+0000","last_peered":"2023-10-01T10:51:16.697925+0000","last_clean":"2023-10-01T10:51:16.697925+0000","last_became_active":"2023-10-01T10:38:16.029786+0000","last_became_peered":"2023-10-01T10:38:16.029786+0000","last_unstale":"2023-10-01T10:51:16.697925+0000","last_undegraded":"2023-10-01T10:51:16.697925+0000","last_fullsized":"2023-10-01T10:51:16.697925+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:53:22.796191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":3434479616,"num_objects":52406,"num_object_clones":0,"num_object_copies":157218,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":52406,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":52406,"num_write_kb":3353984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":10267000832,"data_stored":10267000832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":52406,"ondisk_log_size":52406,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30319,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430679,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2192628,"kb_used_data":1298284,"kb_used_omap":0,"kb_used_meta":894336,"kb_avail":91552524,"statfs":{"total":95995035648,"available":93749784576,"internally_reserved":0,"allocated":1329442816,"data_stored":1328935144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":915800064},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:51:06 2023","interfaces":[{"interface":"back","average":{"1min":3.4209999999999998,"5min":3.3660000000000001,"15min":2.3399999999999999},"min":{"1min":0.33000000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":52.762,"5min":137.34200000000001,"15min":137.34200000000001},"last":1.3009999999999999},{"interface":"front","average":{"1min":3.4830000000000001,"5min":3.4780000000000002,"15min":2.2040000000000002},"min":{"1min":0.39200000000000002,"5min":0.308,"15min":0.28799999999999998},"max":{"1min":52.725999999999999,"5min":137.39500000000001,"15min":137.39500000000001},"last":0.34799999999999998}]},{"osd":1,"last update":"Sun Oct 1 10:51:12 2023","interfaces":[{"interface":"back","average":{"1min":0.88600000000000001,"5min":1.1220000000000001,"15min":1.625},"min":{"1min":0.44400000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":1.4910000000000001,"5min":5.5700000000000003,"15min":56.110999999999997},"last":0.308},{"interface":"front","average":{"1min":0.93600000000000005,"5min":1.1180000000000001,"15min":1.698},"min":{"1min":0.54500000000000004,"5min":0.316,"15min":0.30499999999999999},"max":{"1min":1.2450000000000001,"5min":5.8099999999999996,"15min":51.258000000000003},"last":2.0840000000000001}]},{"osd":2,"last update":"Sun Oct 1 10:51:30 2023","interfaces":[{"interface":"back","average":{"1min":1.2030000000000001,"5min":1.476,"15min":1.266},"min":{"1min":0.73399999999999999,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":3.625,"5min":7.1139999999999999,"15min":7.1139999999999999},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.95799999999999996,"5min":1.1819999999999999,"15min":1.306},"min":{"1min":0.441,"5min":0.35099999999999998,"15min":0.33200000000000002},"max":{"1min":2.7709999999999999,"5min":8.2110000000000003,"15min":10.766},"last":1.4139999999999999}]},{"osd":3,"last update":"Sun Oct 1 10:51:06 2023","interfaces":[{"interface":"back","average":{"1min":0.94099999999999995,"5min":1.601,"15min":1.7709999999999999},"min":{"1min":0.45600000000000002,"5min":0.374,"15min":0.34200000000000003},"max":{"1min":1.339,"5min":8.6829999999999998,"15min":51.334000000000003},"last":3.3679999999999999},{"interface":"front","average":{"1min":1.038,"5min":1.2749999999999999,"15min":1.7909999999999999},"min":{"1min":0.38300000000000001,"5min":0.313,"15min":0.313},"max":{"1min":3.2570000000000001,"5min":8.7469999999999999,"15min":51.404000000000003},"last":1.3560000000000001}]},{"osd":4,"last update":"Sun Oct 1 10:51:41 2023","interfaces":[{"interface":"back","average":{"1min":0.89000000000000001,"5min":1.5249999999999999,"15min":1.296},"min":{"1min":0.40200000000000002,"5min":0.35099999999999998,"15min":0.317},"max":{"1min":1.5580000000000001,"5min":11.081,"15min":11.081},"last":1.2090000000000001},{"interface":"front","average":{"1min":0.95899999999999996,"5min":1.6759999999999999,"15min":1.409},"min":{"1min":0.45100000000000001,"5min":0.40699999999999997,"15min":0.32100000000000001},"max":{"1min":1.476,"5min":12.292,"15min":12.292},"last":2.0419999999999998}]},{"osd":5,"last update":"Sun Oct 1 10:51:12 2023","interfaces":[{"interface":"back","average":{"1min":1.149,"5min":1.5720000000000001,"15min":1.6499999999999999},"min":{"1min":0.48199999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":1.534,"5min":9.7140000000000004,"15min":12.555},"last":0.54800000000000004},{"interface":"front","average":{"1min":1.097,"5min":1.419,"15min":1.708},"min":{"1min":0.59699999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":1.7170000000000001,"5min":10.737,"15min":17.536999999999999},"last":1.385}]},{"osd":6,"last update":"Sun Oct 1 10:51:12 2023","interfaces":[{"interface":"back","average":{"1min":1.157,"5min":1.819,"15min":1.5389999999999999},"min":{"1min":0.46000000000000002,"5min":0.36399999999999999,"15min":0.34499999999999997},"max":{"1min":1.595,"5min":12.329000000000001,"15min":12.329000000000001},"last":0.57699999999999996},{"interface":"front","average":{"1min":1.1020000000000001,"5min":1.5680000000000001,"15min":1.3580000000000001},"min":{"1min":0.64700000000000002,"5min":0.37,"15min":0.36599999999999999},"max":{"1min":1.6279999999999999,"5min":7.9740000000000002,"15min":7.9740000000000002},"last":3.0310000000000001}]}]},{"osd":6,"up_from":42,"seq":180388626913,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2636452,"kb_used_data":1743452,"kb_used_omap":0,"kb_used_meta":892992,"kb_avail":91108700,"statfs":{"total":95995035648,"available":93295308800,"internally_reserved":0,"allocated":1785294848,"data_stored":1784803099,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":914423808},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:50:57 2023","interfaces":[{"interface":"back","average":{"1min":1.488,"5min":4.343,"15min":2.6099999999999999},"min":{"1min":0.29199999999999998,"5min":0.25,"15min":0.25},"max":{"1min":8.1639999999999997,"5min":139.994,"15min":140.524},"last":0.434},{"interface":"front","average":{"1min":1.2989999999999999,"5min":4.2759999999999998,"15min":2.5489999999999999},"min":{"1min":0.28299999999999997,"5min":0.249,"15min":0.249},"max":{"1min":7.9980000000000002,"5min":140.86500000000001,"15min":140.86500000000001},"last":6.0999999999999996}]},{"osd":1,"last update":"Sun Oct 1 10:51:03 2023","interfaces":[{"interface":"back","average":{"1min":1.016,"5min":1.7370000000000001,"15min":1.3740000000000001},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":3.9769999999999999,"5min":8.2560000000000002,"15min":9.673},"last":0.498},{"interface":"front","average":{"1min":1.048,"5min":1.845,"15min":1.3919999999999999},"min":{"1min":0.48799999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":3.2959999999999998,"5min":12.968999999999999,"15min":12.968999999999999},"last":6.1609999999999996}]},{"osd":2,"last update":"Sun Oct 1 10:51:26 2023","interfaces":[{"interface":"back","average":{"1min":0.92700000000000005,"5min":3.0299999999999998,"15min":1.7170000000000001},"min":{"1min":0.48199999999999998,"5min":0.32700000000000001,"15min":0.32300000000000001},"max":{"1min":2.5870000000000002,"5min":64.509,"15min":64.509},"last":3.032},{"interface":"front","average":{"1min":1.02,"5min":3.0110000000000001,"15min":1.7470000000000001},"min":{"1min":0.44,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":2.6970000000000001,"5min":64.435000000000002,"15min":64.435000000000002},"last":3.0720000000000001}]},{"osd":3,"last update":"Sun Oct 1 10:51:06 2023","interfaces":[{"interface":"back","average":{"1min":1.1399999999999999,"5min":2.1880000000000002,"15min":1.9910000000000001},"min":{"1min":0.71299999999999997,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":3.9399999999999999,"5min":8.577,"15min":61.817999999999998},"last":10.452},{"interface":"front","average":{"1min":1.149,"5min":1.9470000000000001,"15min":1.8029999999999999},"min":{"1min":0.69399999999999995,"5min":0.38100000000000001,"15min":0.315},"max":{"1min":2.8519999999999999,"5min":11.099,"15min":61.219000000000001},"last":11.601000000000001}]},{"osd":4,"last update":"Sun Oct 1 10:51:47 2023","interfaces":[{"interface":"back","average":{"1min":0.95299999999999996,"5min":2.298,"15min":1.411},"min":{"1min":0.42499999999999999,"5min":0.34200000000000003,"15min":0.32400000000000001},"max":{"1min":1.7230000000000001,"5min":13.1,"15min":13.1},"last":7.7800000000000002},{"interface":"front","average":{"1min":1.1919999999999999,"5min":2.0489999999999999,"15min":1.4319999999999999},"min":{"1min":0.40999999999999998,"5min":0.35799999999999998,"15min":0.315},"max":{"1min":3.4740000000000002,"5min":11.193,"15min":11.193},"last":3.1080000000000001}]},{"osd":5,"last update":"Sun Oct 1 10:51:06 2023","interfaces":[{"interface":"back","average":{"1min":1.21,"5min":2,"15min":1.53},"min":{"1min":0.41099999999999998,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":4.0090000000000003,"5min":8.2279999999999998,"15min":12.398999999999999},"last":0.84499999999999997},{"interface":"front","average":{"1min":1.095,"5min":2.1880000000000002,"15min":1.6739999999999999},"min":{"1min":0.41899999999999998,"5min":0.308,"15min":0.308},"max":{"1min":1.514,"5min":12.999000000000001,"15min":14.459},"last":6.1790000000000003}]},{"osd":7,"last update":"Sun Oct 1 10:50:57 2023","interfaces":[{"interface":"back","average":{"1min":1.766,"5min":2.2309999999999999,"15min":1.5149999999999999},"min":{"1min":0.32300000000000001,"5min":0.311,"15min":0.311},"max":{"1min":13.276999999999999,"5min":13.276999999999999,"15min":13.276999999999999},"last":6.1420000000000003},{"interface":"front","average":{"1min":1.8560000000000001,"5min":2.0449999999999999,"15min":1.599},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.35999999999999999},"max":{"1min":10.023,"5min":11.416,"15min":11.416},"last":0.46999999999999997}]}]},{"osd":1,"up_from":101,"seq":433791697130,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2067776,"kb_used_data":1612920,"kb_used_omap":4,"kb_used_meta":454843,"kb_avail":91677376,"statfs":{"total":95995035648,"available":93877633024,"internally_reserved":0,"allocated":1651630080,"data_stored":1650882886,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":465760142},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:51:15 2023","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":1.546,"15min":1.1439999999999999},"min":{"1min":0.374,"5min":0.20599999999999999,"15min":0.189},"max":{"1min":0.97799999999999998,"5min":37.058,"15min":37.058},"last":1.2030000000000001},{"interface":"front","average":{"1min":0.58899999999999997,"5min":1.71,"15min":1.5529999999999999},"min":{"1min":0.35399999999999998,"5min":0.215,"15min":0.19400000000000001},"max":{"1min":1.036,"5min":37.337000000000003,"15min":61.207999999999998},"last":0.98399999999999999}]},{"osd":2,"last update":"Sun Oct 1 10:51:33 2023","interfaces":[{"interface":"back","average":{"1min":1.1619999999999999,"5min":1.5980000000000001,"15min":1.385},"min":{"1min":0.46600000000000003,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":5.6289999999999996,"5min":10.042999999999999,"15min":61.491},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.89100000000000001,"5min":1.3460000000000001,"15min":1.379},"min":{"1min":0.39100000000000001,"5min":0.24099999999999999,"15min":0.23699999999999999},"max":{"1min":2.8340000000000001,"5min":8.9049999999999994,"15min":61.371000000000002},"last":1.0740000000000001}]},{"osd":3,"last update":"Sun Oct 1 10:51:15 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.95999999999999996,"15min":1.3500000000000001},"min":{"1min":0.36699999999999999,"5min":0.23999999999999999,"15min":0.20000000000000001},"max":{"1min":0.84499999999999997,"5min":5.9160000000000004,"15min":62.020000000000003},"last":1.4490000000000001},{"interface":"front","average":{"1min":0.68100000000000005,"5min":1.2310000000000001,"15min":1.5329999999999999},"min":{"1min":0.40999999999999998,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.98199999999999998,"5min":7.7560000000000002,"15min":62.338999999999999},"last":0.54600000000000004}]},{"osd":4,"last update":"Sun Oct 1 10:51:33 2023","interfaces":[{"interface":"back","average":{"1min":1.2170000000000001,"5min":1.49,"15min":1.4119999999999999},"min":{"1min":0.309,"5min":0.309,"15min":0.29199999999999998},"max":{"1min":4.9909999999999997,"5min":10.004,"15min":61.588000000000001},"last":2.0099999999999998},{"interface":"front","average":{"1min":0.97099999999999997,"5min":1.671,"15min":1.6359999999999999},"min":{"1min":0.436,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":2.4740000000000002,"5min":10.880000000000001,"15min":61.411000000000001},"last":3.0379999999999998}]},{"osd":5,"last update":"Sun Oct 1 10:51:15 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":1.728,"15min":1.7430000000000001},"min":{"1min":0.68799999999999994,"5min":0.39400000000000002,"15min":0.38900000000000001},"max":{"1min":1.677,"5min":7.9260000000000002,"15min":61.560000000000002},"last":0.52200000000000002},{"interface":"front","average":{"1min":1.018,"5min":1.732,"15min":1.6200000000000001},"min":{"1min":0.622,"5min":0.35199999999999998,"15min":0.34699999999999998},"max":{"1min":1.647,"5min":10.865,"15min":61.667999999999999},"last":1.4059999999999999}]},{"osd":6,"last update":"Sun Oct 1 10:51:15 2023","interfaces":[{"interface":"back","average":{"1min":1.018,"5min":1.633,"15min":1.5289999999999999},"min":{"1min":0.65000000000000002,"5min":0.41099999999999998,"15min":0.39100000000000001},"max":{"1min":1.718,"5min":10.933,"15min":62.219999999999999},"last":2.3159999999999998},{"interface":"front","average":{"1min":1.026,"5min":1.6639999999999999,"15min":1.6619999999999999},"min":{"1min":0.67700000000000005,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":1.524,"5min":9.7870000000000008,"15min":61.462000000000003},"last":2.427}]},{"osd":7,"last update":"Sun Oct 1 10:51:15 2023","interfaces":[{"interface":"back","average":{"1min":1.016,"5min":1.4419999999999999,"15min":1.782},"min":{"1min":0.63800000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":1.583,"5min":10.465,"15min":62.646999999999998},"last":1.3859999999999999},{"interface":"front","average":{"1min":1.0860000000000001,"5min":1.6439999999999999,"15min":1.7250000000000001},"min":{"1min":0.749,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":1.4950000000000001,"5min":6.415,"15min":62.048000000000002},"last":0.80600000000000005}]}]},{"osd":0,"up_from":86,"seq":369367187740,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1589292,"kb_used_data":971684,"kb_used_omap":5,"kb_used_meta":617594,"kb_avail":92155860,"statfs":{"total":95995035648,"available":94367600640,"internally_reserved":0,"allocated":995004416,"data_stored":994273538,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":632416890},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 10:51:15 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":1.1279999999999999,"15min":1.244},"min":{"1min":0.23499999999999999,"5min":0.193,"15min":0.184},"max":{"1min":1.7829999999999999,"5min":7.7359999999999998,"15min":15.074},"last":0.218},{"interface":"front","average":{"1min":0.63300000000000001,"5min":1.075,"15min":1.3069999999999999},"min":{"1min":0.25,"5min":0.19900000000000001,"15min":0.19700000000000001},"max":{"1min":1.3959999999999999,"5min":8.4900000000000002,"15min":13.601000000000001},"last":0.90700000000000003}]},{"osd":2,"last update":"Sun Oct 1 10:51:31 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":1.5669999999999999,"15min":1.0900000000000001},"min":{"1min":0.38600000000000001,"5min":0.253,"15min":0.223},"max":{"1min":0.93799999999999994,"5min":12.872,"15min":12.872},"last":0.95499999999999996},{"interface":"front","average":{"1min":0.69799999999999995,"5min":1.716,"15min":1.1319999999999999},"min":{"1min":0.36599999999999999,"5min":0.221,"15min":0.221},"max":{"1min":1.0940000000000001,"5min":13.137,"15min":13.137},"last":0.72199999999999998}]},{"osd":3,"last update":"Sun Oct 1 10:51:15 2023","interfaces":[{"interface":"back","average":{"1min":0.874,"5min":1.5309999999999999,"15min":1.3400000000000001},"min":{"1min":0.47299999999999998,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":3.097,"5min":12.944000000000001,"15min":12.944000000000001},"last":0.68500000000000005},{"interface":"front","average":{"1min":0.76600000000000001,"5min":1.5760000000000001,"15min":1.2909999999999999},"min":{"1min":0.31900000000000001,"5min":0.29099999999999998,"15min":0.245},"max":{"1min":2.2109999999999999,"5min":9.9090000000000007,"15min":10.984999999999999},"last":0.46999999999999997}]},{"osd":4,"last update":"Sun Oct 1 10:50:49 2023","interfaces":[{"interface":"back","average":{"1min":1.5449999999999999,"5min":2.3140000000000001,"15min":1.5169999999999999},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":9.3580000000000005,"5min":11.379,"15min":11.379},"last":0.66600000000000004},{"interface":"front","average":{"1min":1.254,"5min":2,"15min":1.4359999999999999},"min":{"1min":0.373,"5min":0.33400000000000002,"15min":0.31900000000000001},"max":{"1min":9.4009999999999998,"5min":11.362,"15min":11.362},"last":0.63900000000000001}]},{"osd":5,"last update":"Sun Oct 1 10:51:15 2023","interfaces":[{"interface":"back","average":{"1min":1.145,"5min":1.966,"15min":1.5669999999999999},"min":{"1min":0.64900000000000002,"5min":0.42299999999999999,"15min":0.35799999999999998},"max":{"1min":3.1859999999999999,"5min":12.91,"15min":12.91},"last":0.746},{"interface":"front","average":{"1min":1.175,"5min":1.9159999999999999,"15min":1.5760000000000001},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.34699999999999998},"max":{"1min":3.2170000000000001,"5min":9.8620000000000001,"15min":10.666},"last":0.5}]},{"osd":6,"last update":"Sun Oct 1 10:51:15 2023","interfaces":[{"interface":"back","average":{"1min":1.2509999999999999,"5min":2.0619999999999998,"15min":1.5429999999999999},"min":{"1min":0.76200000000000001,"5min":0.376,"15min":0.36399999999999999},"max":{"1min":3.2410000000000001,"5min":9.8279999999999994,"15min":9.8279999999999994},"last":0.52100000000000002},{"interface":"front","average":{"1min":1.526,"5min":2.085,"15min":1.5189999999999999},"min":{"1min":0.50800000000000001,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":10.111000000000001,"5min":10.111000000000001,"15min":10.111000000000001},"last":2.1339999999999999}]},{"osd":7,"last update":"Sun Oct 1 10:51:15 2023","interfaces":[{"interface":"back","average":{"1min":1.2529999999999999,"5min":1.786,"15min":1.603},"min":{"1min":0.40799999999999997,"5min":0.39700000000000002,"15min":0.36899999999999999},"max":{"1min":4.4969999999999999,"5min":15.839,"15min":15.839},"last":1.0049999999999999},{"interface":"front","average":{"1min":1.0229999999999999,"5min":1.526,"15min":1.5109999999999999},"min":{"1min":0.53600000000000003,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":1.4690000000000001,"5min":8.9440000000000008,"15min":8.9440000000000008},"last":2.3380000000000001}]}]},{"osd":2,"up_from":116,"seq":498216206516,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1601084,"kb_used_data":1271860,"kb_used_omap":7,"kb_used_meta":329208,"kb_avail":92144068,"statfs":{"total":95995035648,"available":94355525632,"internally_reserved":0,"allocated":1302384640,"data_stored":1301695519,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":337109073},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:51:32 2023","interfaces":[{"interface":"back","average":{"1min":0.65900000000000003,"5min":3.8460000000000001,"15min":1.8260000000000001},"min":{"1min":0.16900000000000001,"5min":0.16900000000000001,"15min":0.16900000000000001},"max":{"1min":1.4179999999999999,"5min":147.87,"15min":147.87},"last":1.756},{"interface":"front","average":{"1min":0.67000000000000004,"5min":4.0549999999999997,"15min":1.9419999999999999},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.19500000000000001},"max":{"1min":2.9180000000000001,"5min":148.03100000000001,"15min":148.03100000000001},"last":1.7410000000000001}]},{"osd":1,"last update":"Sun Oct 1 10:51:32 2023","interfaces":[{"interface":"back","average":{"1min":6.657,"5min":3.044,"15min":1.736},"min":{"1min":0.32700000000000001,"5min":0.246,"15min":0.219},"max":{"1min":122.249,"5min":122.249,"15min":122.249},"last":0.65400000000000003},{"interface":"front","average":{"1min":6.8959999999999999,"5min":3.3679999999999999,"15min":1.776},"min":{"1min":0.34499999999999997,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":121.495,"5min":121.495,"15min":121.495},"last":4.5410000000000004}]},{"osd":3,"last update":"Sun Oct 1 10:51:03 2023","interfaces":[{"interface":"back","average":{"1min":1.248,"5min":1.919,"15min":1.4650000000000001},"min":{"1min":0.28699999999999998,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":9.532,"5min":60.393000000000001,"15min":64.266000000000005},"last":3.706},{"interface":"front","average":{"1min":0.93700000000000006,"5min":2.2919999999999998,"15min":1.6439999999999999},"min":{"1min":0.378,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":6.5039999999999996,"5min":61.939999999999998,"15min":67.620999999999995},"last":0.34599999999999997}]},{"osd":4,"last update":"Sun Oct 1 10:51:43 2023","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":2.3119999999999998,"15min":1.4770000000000001},"min":{"1min":0.371,"5min":0.36499999999999999,"15min":0.35799999999999998},"max":{"1min":3.0209999999999999,"5min":60.994,"15min":60.994},"last":0.38300000000000001},{"interface":"front","average":{"1min":1.1870000000000001,"5min":2.6880000000000002,"15min":1.55},"min":{"1min":0.42799999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":6.5839999999999996,"5min":63.012,"15min":63.012},"last":4.5739999999999998}]},{"osd":5,"last update":"Sun Oct 1 10:51:32 2023","interfaces":[{"interface":"back","average":{"1min":1.149,"5min":2.4569999999999999,"15min":1.643},"min":{"1min":0.45100000000000001,"5min":0.39800000000000002,"15min":0.34699999999999998},"max":{"1min":3.0630000000000002,"5min":61.021000000000001,"15min":61.021000000000001},"last":0.60499999999999998},{"interface":"front","average":{"1min":1.2150000000000001,"5min":2.5800000000000001,"15min":1.635},"min":{"1min":0.38400000000000001,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":6.6210000000000004,"5min":63.145000000000003,"15min":63.145000000000003},"last":4.6260000000000003}]},{"osd":6,"last update":"Sun Oct 1 10:51:32 2023","interfaces":[{"interface":"back","average":{"1min":1.224,"5min":2.641,"15min":1.5569999999999999},"min":{"1min":0.40999999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":6.6500000000000004,"5min":63.289000000000001,"15min":63.289000000000001},"last":4.5949999999999998},{"interface":"front","average":{"1min":1.0169999999999999,"5min":2.2360000000000002,"15min":1.425},"min":{"1min":0.41899999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":2.75,"5min":60.579000000000001,"15min":60.579000000000001},"last":0.624}]},{"osd":7,"last update":"Sun Oct 1 10:51:32 2023","interfaces":[{"interface":"back","average":{"1min":1.018,"5min":2.0529999999999999,"15min":1.419},"min":{"1min":0.39600000000000002,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":1.3819999999999999,"5min":60.493000000000002,"15min":60.493000000000002},"last":0.59199999999999997},{"interface":"front","average":{"1min":1.0980000000000001,"5min":2.278,"15min":1.698},"min":{"1min":0.43099999999999999,"5min":0.38500000000000001,"15min":0.376},"max":{"1min":3.0950000000000002,"5min":60.956000000000003,"15min":60.956000000000003},"last":1.806}]}]},{"osd":3,"up_from":121,"seq":519691042988,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1861288,"kb_used_data":1469344,"kb_used_omap":2,"kb_used_meta":391933,"kb_avail":91883864,"statfs":{"total":95995035648,"available":94089076736,"internally_reserved":0,"allocated":1504608256,"data_stored":1503937085,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":401339445},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:51:06 2023","interfaces":[{"interface":"back","average":{"1min":7.21,"5min":4.242,"15min":2.4900000000000002},"min":{"1min":0.33400000000000002,"5min":0.189,"15min":0.187},"max":{"1min":69.435000000000002,"5min":129.953,"15min":129.953},"last":0.61699999999999999},{"interface":"front","average":{"1min":7.6130000000000004,"5min":4.7679999999999998,"15min":2.7320000000000002},"min":{"1min":0.38200000000000001,"5min":0.20799999999999999,"15min":0.17299999999999999},"max":{"1min":74.027000000000001,"5min":129.53999999999999,"15min":129.53999999999999},"last":0.19600000000000001}]},{"osd":1,"last update":"Sun Oct 1 10:51:06 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.97399999999999998,"15min":1.4930000000000001},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.96499999999999997,"5min":9.6560000000000006,"15min":77.361000000000004},"last":0.78100000000000003},{"interface":"front","average":{"1min":0.68200000000000005,"5min":1.071,"15min":1.633},"min":{"1min":0.247,"5min":0.247,"15min":0.24099999999999999},"max":{"1min":1.6000000000000001,"5min":10.305,"15min":77.867000000000004},"last":1.373}]},{"osd":2,"last update":"Sun Oct 1 10:51:06 2023","interfaces":[{"interface":"back","average":{"1min":0.72799999999999998,"5min":1.228,"15min":1.079},"min":{"1min":0.32400000000000001,"5min":0.27700000000000002,"15min":0.222},"max":{"1min":4.0119999999999996,"5min":9.5749999999999993,"15min":10.028},"last":1.1870000000000001},{"interface":"front","average":{"1min":0.72499999999999998,"5min":1.252,"15min":1.329},"min":{"1min":0.34499999999999997,"5min":0.27900000000000003,"15min":0.26900000000000002},"max":{"1min":2.0619999999999998,"5min":9.8409999999999993,"15min":75.299999999999997},"last":0.89700000000000002}]},{"osd":4,"last update":"Sun Oct 1 10:51:41 2023","interfaces":[{"interface":"back","average":{"1min":1.171,"5min":1.4059999999999999,"15min":1.4470000000000001},"min":{"1min":0.60399999999999998,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":3.9319999999999999,"5min":9.7370000000000001,"15min":75.388999999999996},"last":0.65300000000000002},{"interface":"front","average":{"1min":0.84899999999999998,"5min":1.5600000000000001,"15min":1.571},"min":{"1min":0.42099999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":1.276,"5min":6.375,"15min":75.165999999999997},"last":0.308}]},{"osd":5,"last update":"Sun Oct 1 10:51:06 2023","interfaces":[{"interface":"back","average":{"1min":0.91000000000000003,"5min":1.3720000000000001,"15min":1.6839999999999999},"min":{"1min":0.39700000000000002,"5min":0.37,"15min":0.33100000000000002},"max":{"1min":1.8300000000000001,"5min":6.9260000000000002,"15min":75.221000000000004},"last":0.376},{"interface":"front","average":{"1min":1.012,"5min":1.4119999999999999,"15min":1.7889999999999999},"min":{"1min":0.52100000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":4.1550000000000002,"5min":7.1820000000000004,"15min":75.231999999999999},"last":1.411}]},{"osd":6,"last update":"Sun Oct 1 10:51:06 2023","interfaces":[{"interface":"back","average":{"1min":1.0489999999999999,"5min":1.855,"15min":1.762},"min":{"1min":0.42699999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":2.4750000000000001,"5min":7.8140000000000001,"15min":75.257000000000005},"last":0.35399999999999998},{"interface":"front","average":{"1min":1.4950000000000001,"5min":2.0779999999999998,"15min":1.887},"min":{"1min":0.44900000000000001,"5min":0.42299999999999999,"15min":0.33500000000000002},"max":{"1min":7.0060000000000002,"5min":8.5760000000000005,"15min":75.194000000000003},"last":1.3080000000000001}]},{"osd":7,"last update":"Sun Oct 1 10:51:06 2023","interfaces":[{"interface":"back","average":{"1min":1.054,"5min":1.3680000000000001,"15min":1.712},"min":{"1min":0.55100000000000005,"5min":0.376,"15min":0.35099999999999998},"max":{"1min":4.1210000000000004,"5min":8.1039999999999992,"15min":75.138999999999996},"last":1.2789999999999999},{"interface":"front","average":{"1min":0.93200000000000005,"5min":1.4419999999999999,"15min":1.6359999999999999},"min":{"1min":0.57899999999999996,"5min":0.39500000000000002,"15min":0.37},"max":{"1min":1.1799999999999999,"5min":9.8059999999999992,"15min":75.355000000000004},"last":0.68100000000000005}]}]},{"osd":4,"up_from":126,"seq":541165879461,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1131324,"kb_used_data":781492,"kb_used_omap":10,"kb_used_meta":349813,"kb_avail":92613828,"statfs":{"total":95995035648,"available":94836559872,"internally_reserved":0,"allocated":800247808,"data_stored":799558159,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":358209124},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:51:43 2023","interfaces":[{"interface":"back","average":{"1min":3.0659999999999998,"5min":3.1160000000000001,"15min":1.752},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":49.917000000000002,"5min":62.576000000000001,"15min":62.576000000000001},"last":1.9019999999999999},{"interface":"front","average":{"1min":3.04,"5min":3.1560000000000001,"15min":1.7729999999999999},"min":{"1min":0.41299999999999998,"5min":0.251,"15min":0.251},"max":{"1min":50.042000000000002,"5min":64.941000000000003,"15min":64.941000000000003},"last":0.39700000000000002}]},{"osd":1,"last update":"Sun Oct 1 10:51:43 2023","interfaces":[{"interface":"back","average":{"1min":2.2050000000000001,"5min":1.9410000000000001,"15min":1.7769999999999999},"min":{"1min":0.29199999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":29.344000000000001,"5min":29.344000000000001,"15min":97.622},"last":0.32000000000000001},{"interface":"front","average":{"1min":2.3719999999999999,"5min":2.1110000000000002,"15min":1.8560000000000001},"min":{"1min":0.36599999999999999,"5min":0.28699999999999998,"15min":0.28599999999999998},"max":{"1min":29.251999999999999,"5min":29.251999999999999,"15min":97.346999999999994},"last":0.93300000000000005}]},{"osd":2,"last update":"Sun Oct 1 10:51:43 2023","interfaces":[{"interface":"back","average":{"1min":1.0069999999999999,"5min":3.504,"15min":1.786},"min":{"1min":0.42899999999999999,"5min":0.30499999999999999,"15min":0.29899999999999999},"max":{"1min":1.532,"5min":117.229,"15min":117.229},"last":0.99399999999999999},{"interface":"front","average":{"1min":0.84599999999999997,"5min":3.149,"15min":1.712},"min":{"1min":0.34899999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":1.226,"5min":116.922,"15min":116.922},"last":0.59299999999999997}]},{"osd":3,"last update":"Sun Oct 1 10:51:43 2023","interfaces":[{"interface":"back","average":{"1min":2.2810000000000001,"5min":2.161,"15min":2.0939999999999999},"min":{"1min":0.42699999999999999,"5min":0.32900000000000001,"15min":0.29499999999999998},"max":{"1min":30.637,"5min":30.637,"15min":61.398000000000003},"last":0.46600000000000003},{"interface":"front","average":{"1min":2.2269999999999999,"5min":2.2040000000000002,"15min":2.008},"min":{"1min":0.36499999999999999,"5min":0.35399999999999998,"15min":0.34699999999999998},"max":{"1min":30.242999999999999,"5min":30.242999999999999,"15min":61.463000000000001},"last":0.48099999999999998}]},{"osd":5,"last update":"Sun Oct 1 10:51:43 2023","interfaces":[{"interface":"back","average":{"1min":0.88400000000000001,"5min":2.0499999999999998,"15min":1.468},"min":{"1min":0.38400000000000001,"5min":0.34499999999999997,"15min":0.32700000000000001},"max":{"1min":1.3440000000000001,"5min":12.601000000000001,"15min":12.601000000000001},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.93700000000000006,"5min":1.9410000000000001,"15min":1.482},"min":{"1min":0.39700000000000002,"5min":0.35499999999999998,"15min":0.28999999999999998},"max":{"1min":1.5580000000000001,"5min":11.725,"15min":16.779},"last":0.83499999999999996}]},{"osd":6,"last update":"Sun Oct 1 10:51:43 2023","interfaces":[{"interface":"back","average":{"1min":1.038,"5min":2.444,"15min":1.5620000000000001},"min":{"1min":0.499,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":2.3999999999999999,"5min":12.563000000000001,"15min":13.646000000000001},"last":0.63200000000000001},{"interface":"front","average":{"1min":1.087,"5min":2.2669999999999999,"15min":1.552},"min":{"1min":0.57899999999999996,"5min":0.42299999999999999,"15min":0.41099999999999998},"max":{"1min":3.3260000000000001,"5min":10.537000000000001,"15min":14.446999999999999},"last":0.44600000000000001}]},{"osd":7,"last update":"Sun Oct 1 10:51:43 2023","interfaces":[{"interface":"back","average":{"1min":1.0129999999999999,"5min":1.9990000000000001,"15min":1.448},"min":{"1min":0.47099999999999997,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":1.367,"5min":10.423999999999999,"15min":10.423999999999999},"last":1.073},{"interface":"front","average":{"1min":1.5620000000000001,"5min":1.9410000000000001,"15min":1.665},"min":{"1min":0.48499999999999999,"5min":0.39200000000000002,"15min":0.30399999999999999},"max":{"1min":6.2960000000000003,"5min":11.798999999999999,"15min":11.798999999999999},"last":0.89200000000000002}]}]},{"osd":5,"up_from":36,"seq":154618823142,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1682072,"kb_used_data":781904,"kb_used_omap":0,"kb_used_meta":900160,"kb_avail":92063080,"statfs":{"total":95995035648,"available":94272593920,"internally_reserved":0,"allocated":800669696,"data_stored":800185511,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":921763840},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:51:02 2023","interfaces":[{"interface":"back","average":{"1min":1.0009999999999999,"5min":2.0150000000000001,"15min":1.522},"min":{"1min":0.34999999999999998,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":4.6440000000000001,"5min":16.309000000000001,"15min":52.094999999999999},"last":0.55500000000000005},{"interface":"front","average":{"1min":1.099,"5min":2.2050000000000001,"15min":1.7649999999999999},"min":{"1min":0.47799999999999998,"5min":0.29199999999999998,"15min":0.26900000000000002},"max":{"1min":4.9880000000000004,"5min":23.507000000000001,"15min":52.048000000000002},"last":2.851}]},{"osd":1,"last update":"Sun Oct 1 10:51:02 2023","interfaces":[{"interface":"back","average":{"1min":1.024,"5min":1.6180000000000001,"15min":2.3050000000000002},"min":{"1min":0.436,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":4.6950000000000003,"5min":7.7000000000000002,"15min":202.96600000000001},"last":0.59299999999999997},{"interface":"front","average":{"1min":1.0580000000000001,"5min":1.6299999999999999,"15min":2.6680000000000001},"min":{"1min":0.63600000000000001,"5min":0.31900000000000001,"15min":0.28100000000000003},"max":{"1min":3.9769999999999999,"5min":9.9209999999999994,"15min":203.05099999999999},"last":1.7150000000000001}]},{"osd":2,"last update":"Sun Oct 1 10:51:36 2023","interfaces":[{"interface":"back","average":{"1min":0.83099999999999996,"5min":2.0779999999999998,"15min":1.375},"min":{"1min":0.36399999999999999,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":1.2230000000000001,"5min":14.872,"15min":14.872},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.96599999999999997,"5min":2.2789999999999999,"15min":1.647},"min":{"1min":0.40699999999999997,"5min":0.32300000000000001,"15min":0.317},"max":{"1min":1.333,"5min":14.746,"15min":14.746},"last":1.7729999999999999}]},{"osd":3,"last update":"Sun Oct 1 10:51:08 2023","interfaces":[{"interface":"back","average":{"1min":1.091,"5min":1.8420000000000001,"15min":2.3500000000000001},"min":{"1min":0.64500000000000002,"5min":0.35199999999999998,"15min":0.316},"max":{"1min":3.944,"5min":9.9900000000000002,"15min":70.506},"last":1.8160000000000001},{"interface":"front","average":{"1min":0.92000000000000004,"5min":1.73,"15min":2.1619999999999999},"min":{"1min":0.55100000000000005,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.3779999999999999,"5min":10.898,"15min":70.346999999999994},"last":2.7229999999999999}]},{"osd":4,"last update":"Sun Oct 1 10:51:36 2023","interfaces":[{"interface":"back","average":{"1min":0.89900000000000002,"5min":1.8129999999999999,"15min":1.4490000000000001},"min":{"1min":0.313,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":1.4379999999999999,"5min":11.002000000000001,"15min":15.170999999999999},"last":2.3239999999999998},{"interface":"front","average":{"1min":1.022,"5min":2.3370000000000002,"15min":1.5149999999999999},"min":{"1min":0.52800000000000002,"5min":0.316,"15min":0.316},"max":{"1min":1.4750000000000001,"5min":9.7270000000000003,"15min":9.7270000000000003},"last":0.63200000000000001}]},{"osd":6,"last update":"Sun Oct 1 10:51:08 2023","interfaces":[{"interface":"back","average":{"1min":1.0429999999999999,"5min":2.3740000000000001,"15min":1.5569999999999999},"min":{"1min":0.56200000000000006,"5min":0.45000000000000001,"15min":0.38500000000000001},"max":{"1min":1.474,"5min":11.629,"15min":11.629},"last":1.0189999999999999},{"interface":"front","average":{"1min":0.98299999999999998,"5min":2.4129999999999998,"15min":1.7030000000000001},"min":{"1min":0.51300000000000001,"5min":0.33200000000000002,"15min":0.312},"max":{"1min":1.5369999999999999,"5min":14.263,"15min":14.263},"last":2.3780000000000001}]},{"osd":7,"last update":"Sun Oct 1 10:51:08 2023","interfaces":[{"interface":"back","average":{"1min":1.4750000000000001,"5min":2.198,"15min":1.845},"min":{"1min":0.57799999999999996,"5min":0.375,"15min":0.34699999999999998},"max":{"1min":9.2159999999999993,"5min":13.673999999999999,"15min":13.673999999999999},"last":2.411},{"interface":"front","average":{"1min":1.097,"5min":1.9890000000000001,"15min":1.677},"min":{"1min":0.41099999999999998,"5min":0.38,"15min":0.372},"max":{"1min":1.5980000000000001,"5min":9.6929999999999996,"15min":11.635},"last":1.8759999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":998113280,"data_stored":998113280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1675427840,"data_stored":1675427840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1322975232,"data_stored":1322975232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1507196928,"data_stored":1507196928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":804978688,"data_stored":804978688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":808517632,"data_stored":808517632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1799880704,"data_stored":1799880704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1349910528,"data_stored":1349910528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T10:51:57.860 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-01T10:51:57.860 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T10:51:57.860 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-01T10:51:57.860 INFO:tasks.thrashosds.thrasher:Setting osd 1 primary_affinity to 0.000000 2023-10-01T10:51:57.860 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 1 0 2023-10-01T10:51:57.864 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:57.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:57 smithi173 bash[58816]: cluster 2023-10-01T10:51:55.959884+0000 mgr.y (mgr.25072) 1031 : cluster 0 pgmap v981: 89 pgs: 89 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-01T10:51:57.892 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:57.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:57.895 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:51:57.996 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:58.006 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:57 smithi078 bash[75299]: cluster 2023-10-01T10:51:55.959884+0000 mgr.y (mgr.25072) 1031 : cluster 0 pgmap v981: 89 pgs: 89 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-01T10:51:58.006 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:57 smithi078 bash[78274]: cluster 2023-10-01T10:51:55.959884+0000 mgr.y (mgr.25072) 1031 : cluster 0 pgmap v981: 89 pgs: 89 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-01T10:51:58.025 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:58.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:58.027 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:51:58.128 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:58.151 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:58.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:58.154 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:51:58.254 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:58.259 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 37 16 59771 59755 100.924 74.25 0.00425025 0.00990202 2023-10-01T10:51:58.293 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:58.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:58.297 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:51:58.398 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:58.423 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:58.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:58.425 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:51:58.525 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:58.558 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:58.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:58.560 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:51:58.661 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:58.689 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:58.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:58.692 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:51:58.793 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:58.816 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:58.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:58.822 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:51:58.923 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:58.955 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:58.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:58.958 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:51:59.059 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:59.089 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:51:59.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:59.093 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:51:59.194 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:59.222 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:51:59.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:59.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:51:59.259 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 38 16 60773 60757 99.9163 62.625 0.015128 0.0100024 2023-10-01T10:51:59.325 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:59.354 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:51:59.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:59.358 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:51:59.459 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:59.489 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:51:59.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:59.497 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:51:59.598 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:59.641 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:51:59.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:59.643 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:51:59.744 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:59.775 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:51:59.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:59.778 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:51:59.862 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T10:51:59.866 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T10:51:59.872 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:51:59 smithi078 bash[78274]: cluster 2023-10-01T10:51:57.960423+0000 mgr.y (mgr.25072) 1032 : cluster 0 pgmap v982: 89 pgs: 89 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-01T10:51:59.872 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:51:59 smithi078 bash[75299]: cluster 2023-10-01T10:51:57.960423+0000 mgr.y (mgr.25072) 1032 : cluster 0 pgmap v982: 89 pgs: 89 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-01T10:51:59.878 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:59.903 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:51:59.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:51:59.906 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:52:00.007 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:00.051 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:00.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:00.056 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:52:00.102 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:51:59 smithi173 bash[58816]: cluster 2023-10-01T10:51:57.960423+0000 mgr.y (mgr.25072) 1032 : cluster 0 pgmap v982: 89 pgs: 89 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-10-01T10:52:00.157 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:00.183 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:00.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:00.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:52:00.259 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 39 16 61818 61802 99.0289 65.3125 0.0178862 0.010091 2023-10-01T10:52:00.269 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:52:00 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:52:00] "GET /metrics HTTP/1.1" 200 36310 "" "Prometheus/2.33.4" 2023-10-01T10:52:00.287 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:00.324 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:00.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:00.328 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:52:00.429 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:00.454 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:00.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:00.462 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:52:00.562 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:00.594 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:00.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:00.597 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:52:00.698 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:00.734 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:00.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:00.737 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:52:00.838 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:00.864 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:00.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:00.871 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:00.972 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:01.000 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:01.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:01.004 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:52:01.105 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:01.130 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:01.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:01.133 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:52:01.234 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:01.259 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:2023-10-01T10:52:01.259059+0000 min lat: 0.00119944 max lat: 0.258785 avg lat: 0.0101699 2023-10-01T10:52:01.259 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T10:52:01.259 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 40 16 62909 62893 98.2578 68.1875 0.00775103 0.0101699 2023-10-01T10:52:01.271 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:01.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:01.275 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:52:01.376 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:01.402 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:01.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:01.407 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:52:01.508 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:01.535 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:01.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:01.542 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:52:01.643 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:01.672 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:01.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:01.676 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:52:01.776 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:01.799 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:01.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:01.802 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:52:01.902 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:01.915 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:01 smithi078 bash[78274]: cluster 2023-10-01T10:51:59.961128+0000 mgr.y (mgr.25072) 1033 : cluster 0 pgmap v983: 89 pgs: 89 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-10-01T10:52:01.915 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:01 smithi078 bash[75299]: cluster 2023-10-01T10:51:59.961128+0000 mgr.y (mgr.25072) 1033 : cluster 0 pgmap v983: 89 pgs: 89 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-10-01T10:52:01.936 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:01.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:01.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:52:02.040 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:02.069 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:02.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:02.075 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:02.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:01 smithi173 bash[58816]: cluster 2023-10-01T10:51:59.961128+0000 mgr.y (mgr.25072) 1033 : cluster 0 pgmap v983: 89 pgs: 89 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-10-01T10:52:02.176 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:02.219 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:02.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:02.222 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:52:02.259 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 41 16 63893 63877 97.3611 61.5 0.0141777 0.0102654 2023-10-01T10:52:02.323 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:02.348 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:02.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:02.351 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:52:02.452 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:02.484 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:02.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:02.487 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:02.588 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:02.611 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:02.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:02.618 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:52:02.708 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:52:02.718 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:02.749 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-10-01T10:52:02.749 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [], 2023-10-01T10:52:02.749 INFO:teuthology.orchestra.run.smithi173.stdout: "complaint_time": 30, 2023-10-01T10:52:02.749 INFO:teuthology.orchestra.run.smithi173.stdout: "num_blocked_ops": 0 2023-10-01T10:52:02.749 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-10-01T10:52:02.773 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:02.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:02.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:52:02.881 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:02.918 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:02.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:02.926 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:03.027 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:03.036 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:02 smithi173 bash[58816]: audit 2023-10-01T10:52:02.434091+0000 mon.a (mon.0) 1273 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:52:03.056 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:03.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:03.059 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:52:03.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:02 smithi078 bash[78274]: audit 2023-10-01T10:52:02.434091+0000 mon.a (mon.0) 1273 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:52:03.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:02 smithi078 bash[75299]: audit 2023-10-01T10:52:02.434091+0000 mon.a (mon.0) 1273 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:52:03.159 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:03.195 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:03.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:03.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:52:03.259 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 42 16 64971 64955 96.647 67.375 0.00823615 0.0103404 2023-10-01T10:52:03.300 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:03.328 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:03.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:03.331 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:52:03.432 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:03.460 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:03.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:03.462 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:52:03.563 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:03.591 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:03.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:03.596 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:52:03.697 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:03.722 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:03.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:03.726 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:03.827 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:03.847 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:03 smithi173 bash[58816]: cluster 2023-10-01T10:52:01.961812+0000 mgr.y (mgr.25072) 1034 : cluster 0 pgmap v984: 89 pgs: 89 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-10-01T10:52:03.855 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-01T10:52:03.858 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:03.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:03.863 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:52:03.964 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:04.007 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:04.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:04.012 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:52:04.113 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:04.122 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:03 smithi078 bash[78274]: cluster 2023-10-01T10:52:01.961812+0000 mgr.y (mgr.25072) 1034 : cluster 0 pgmap v984: 89 pgs: 89 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-10-01T10:52:04.122 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:03 smithi078 bash[75299]: cluster 2023-10-01T10:52:01.961812+0000 mgr.y (mgr.25072) 1034 : cluster 0 pgmap v984: 89 pgs: 89 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-10-01T10:52:04.145 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:04.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:04.146 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:52:04.247 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:04.259 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 43 16 66092 66076 96.0286 70.0625 0.0122324 0.0104074 2023-10-01T10:52:04.269 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:04.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:04.272 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:52:04.372 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:04.412 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:04.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:04.414 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:04.515 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:04.548 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:04.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:04.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:52:04.651 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:04.681 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:04.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:04.683 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:52:04.744 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:52:04.745 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:52:04.784 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:04.816 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:04.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:04.822 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:04.923 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:04.949 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:04.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:04.954 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:52:05.054 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:05.082 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:05.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:05.086 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:52:05.187 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:05.221 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:05.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:05.223 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:52:05.259 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 44 16 67262 67246 95.5079 73.125 0.0103012 0.0104645 2023-10-01T10:52:05.324 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:05.351 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:05.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:05.356 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:52:05.456 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:05.484 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:05.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:05.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:52:05.588 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:05.614 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:05.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:05.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:52:05.720 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:05.748 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:05.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:05.752 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:52:05.853 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:05.856 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:05 smithi173 bash[58816]: cluster 2023-10-01T10:52:03.962212+0000 mgr.y (mgr.25072) 1035 : cluster 0 pgmap v985: 89 pgs: 89 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 62 MiB/s wr, 988 op/s 2023-10-01T10:52:05.889 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:05.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:05.891 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:05.992 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:06.023 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:06.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:06.026 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:06.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:05 smithi078 bash[75299]: cluster 2023-10-01T10:52:03.962212+0000 mgr.y (mgr.25072) 1035 : cluster 0 pgmap v985: 89 pgs: 89 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 62 MiB/s wr, 988 op/s 2023-10-01T10:52:06.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:05 smithi078 bash[78274]: cluster 2023-10-01T10:52:03.962212+0000 mgr.y (mgr.25072) 1035 : cluster 0 pgmap v985: 89 pgs: 89 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 62 MiB/s wr, 988 op/s 2023-10-01T10:52:06.127 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:06.166 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:06.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:06.169 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:52:06.260 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 45 16 68439 68423 95.0201 73.5625 0.0066553 0.010519 2023-10-01T10:52:06.270 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:06.297 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:06.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:06.301 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:52:06.402 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:06.437 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:06.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:06.443 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:52:06.544 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:06.570 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:06.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:06.572 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:52:06.673 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:06.705 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:06.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:06.709 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:52:06.810 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:06.834 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:06.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:06.836 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:52:06.937 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:06.969 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:06.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:06.971 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:52:07.072 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:07.105 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:07.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:07.108 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:52:07.209 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:07.239 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:07.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:07.248 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:07.259 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 46 16 69560 69544 94.4774 70.0625 0.0173469 0.010579 2023-10-01T10:52:07.349 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:07.373 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:07.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:07.376 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:52:07.476 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:07.509 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:07.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:07.512 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:07.613 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:07.646 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:07.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:07.651 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:52:07.752 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:07.777 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:07.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:07.781 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:52:07.882 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:07.908 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:07.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:07.913 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:52:08.014 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:08.042 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:08.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:08.044 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:52:08.056 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:07 smithi078 bash[75299]: cluster 2023-10-01T10:52:05.962752+0000 mgr.y (mgr.25072) 1036 : cluster 0 pgmap v986: 89 pgs: 89 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 62 MiB/s wr, 992 op/s 2023-10-01T10:52:08.057 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:07 smithi078 bash[78274]: cluster 2023-10-01T10:52:05.962752+0000 mgr.y (mgr.25072) 1036 : cluster 0 pgmap v986: 89 pgs: 89 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 62 MiB/s wr, 992 op/s 2023-10-01T10:52:08.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:07 smithi173 bash[58816]: cluster 2023-10-01T10:52:05.962752+0000 mgr.y (mgr.25072) 1036 : cluster 0 pgmap v986: 89 pgs: 89 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 62 MiB/s wr, 992 op/s 2023-10-01T10:52:08.145 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:08.179 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:08.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:08.186 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:52:08.260 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 47 16 70608 70592 93.8608 65.5 0.0196256 0.0106487 2023-10-01T10:52:08.287 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:08.312 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:08.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:08.315 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:52:08.416 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:08.448 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:08.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:08.451 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:52:08.551 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:08.578 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:08.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:08.581 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:08.682 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:08.689 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T10:52:08.711 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:08.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:08.715 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:08.815 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:08.848 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:08.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:08.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:52:08.951 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:08.981 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:08.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:08.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:52:09.089 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:09.123 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:09.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:09.127 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:52:09.228 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:09.260 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 48 16 71628 71612 93.2334 63.75 0.0160345 0.0107196 2023-10-01T10:52:09.260 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:09.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:09.262 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:52:09.363 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:09.391 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:09.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:09.396 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:52:09.497 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:09.530 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:09.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:09.536 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:52:09.637 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:09.655 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:09.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:09.658 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:09.758 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:09.781 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:09.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:09.783 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:52:09.884 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:09.905 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:09.906 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:09 smithi173 bash[58816]: cluster 2023-10-01T10:52:07.963283+0000 mgr.y (mgr.25072) 1037 : cluster 0 pgmap v987: 89 pgs: 89 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-01T10:52:09.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:09.907 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:10.008 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:10.030 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:10.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:10.035 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:52:10.045 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:09 smithi078 bash[75299]: cluster 2023-10-01T10:52:07.963283+0000 mgr.y (mgr.25072) 1037 : cluster 0 pgmap v987: 89 pgs: 89 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-01T10:52:10.045 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:09 smithi078 bash[78274]: cluster 2023-10-01T10:52:07.963283+0000 mgr.y (mgr.25072) 1037 : cluster 0 pgmap v987: 89 pgs: 89 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-01T10:52:10.136 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:10.166 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:10.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:10.172 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:52:10.260 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 49 16 72345 72329 92.245 44.8125 0.00380527 0.0108212 2023-10-01T10:52:10.274 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:10.304 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:10.305 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:52:10 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:52:10] "GET /metrics HTTP/1.1" 200 36311 "" "Prometheus/2.33.4" 2023-10-01T10:52:10.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:10.306 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:52:10.406 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:10.430 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:10.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:10.435 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:52:10.536 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:10.563 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:10.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:10.569 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:52:10.670 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:10.698 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:10.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:10.702 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:52:10.803 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:10.840 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:10.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:10.843 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:52:10.943 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:10.970 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:10.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:10.972 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:52:11.073 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:11.098 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:11.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:11.101 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:52:11.201 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:11.239 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:11.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:11.245 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:52:11.260 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 50 16 73373 73357 91.685 64.25 0.0167186 0.0109017 2023-10-01T10:52:11.346 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:11.388 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:11.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:11.391 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:52:11.492 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:11.521 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:11.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:11.528 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:52:11.629 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:11.658 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:11.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:11.666 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:11.767 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:11.808 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:11.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:11.810 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:52:11.911 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:11.921 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:11 smithi173 bash[58816]: cluster 2023-10-01T10:52:09.964144+0000 mgr.y (mgr.25072) 1038 : cluster 0 pgmap v988: 89 pgs: 89 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 73 MiB/s wr, 1.18k op/s 2023-10-01T10:52:11.922 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:11 smithi173 bash[58816]: audit 2023-10-01T10:52:11.766185+0000 mon.a (mon.0) 1274 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:52:11.962 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:11.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:11.969 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:12.070 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:12.112 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:12.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:12.117 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:52:12.218 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:12.255 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:12.260 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 51 16 74421 74405 91.1715 65.5 0.00978768 0.0109631 2023-10-01T10:52:12.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:12.262 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:52:12.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:11 smithi078 bash[75299]: cluster 2023-10-01T10:52:09.964144+0000 mgr.y (mgr.25072) 1038 : cluster 0 pgmap v988: 89 pgs: 89 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 73 MiB/s wr, 1.18k op/s 2023-10-01T10:52:12.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:11 smithi078 bash[75299]: audit 2023-10-01T10:52:11.766185+0000 mon.a (mon.0) 1274 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:52:12.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:11 smithi078 bash[78274]: cluster 2023-10-01T10:52:09.964144+0000 mgr.y (mgr.25072) 1038 : cluster 0 pgmap v988: 89 pgs: 89 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 73 MiB/s wr, 1.18k op/s 2023-10-01T10:52:12.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:11 smithi078 bash[78274]: audit 2023-10-01T10:52:11.766185+0000 mon.a (mon.0) 1274 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:52:12.363 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:12.391 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:12.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:12.394 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:52:12.494 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:12.525 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:12.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:12.528 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:52:12.628 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:12.661 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:12.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:12.664 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:12.765 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:12.784 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:12.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:12.786 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:52:12.887 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:12.933 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:12.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:12.937 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:52:13.038 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:13.063 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:13.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:13.067 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:52:13.167 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:13.196 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:13.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:13.202 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:52:13.260 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 52 16 75449 75433 90.6537 64.25 0.00825614 0.0110238 2023-10-01T10:52:13.303 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:13.334 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:13.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:13.338 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:52:13.439 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:13.468 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:13.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:13.473 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:52:13.532 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-10-01T10:52:13.533 INFO:teuthology.orchestra.run.smithi173.stdout: "size": 20, 2023-10-01T10:52:13.533 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 600, 2023-10-01T10:52:13.533 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [ 2023-10-01T10:52:13.533 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.533 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.35649.0:158051 9.6 9:6c542c4a:::benchmark_data_smithi078_113121_object46236:head [delete] snapc 0=[] ondisk+write+known_if_redirected e137)", 2023-10-01T10:52:13.533 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:43:39.378615+0000", 2023-10-01T10:52:13.533 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 514.15098192400001, 2023-10-01T10:52:13.533 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.23057741400000001, 2023-10-01T10:52:13.533 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:52:13.533 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:52:13.533 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:52:13.534 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.35649", 2023-10-01T10:52:13.534 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:52:13.534 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 158051 2023-10-01T10:52:13.534 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.534 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:52:13.534 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.534 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:52:13.534 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.378615+0000", 2023-10-01T10:52:13.534 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:52:13.534 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.534 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.535 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:52:13.535 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.378615+0000", 2023-10-01T10:52:13.535 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-10-01T10:52:13.535 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.535 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.535 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:52:13.535 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.378613+0000", 2023-10-01T10:52:13.535 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.3070000000000003e-06 2023-10-01T10:52:13.535 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.535 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.535 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:52:13.535 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.378621+0000", 2023-10-01T10:52:13.535 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.9786999999999999e-05 2023-10-01T10:52:13.536 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.536 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.536 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:52:13.536 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.378641+0000", 2023-10-01T10:52:13.536 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.8250000000000005e-06 2023-10-01T10:52:13.536 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.536 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.536 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:52:13.536 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.378649+0000", 2023-10-01T10:52:13.536 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.6515999999999999e-05 2023-10-01T10:52:13.536 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.536 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.537 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:52:13.537 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.378705+0000", 2023-10-01T10:52:13.537 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000144108 2023-10-01T10:52:13.537 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.537 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.537 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:52:13.537 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.378850+0000", 2023-10-01T10:52:13.537 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00015646099999999999 2023-10-01T10:52:13.537 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.537 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.537 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1,2", 2023-10-01T10:52:13.537 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.379006+0000", 2023-10-01T10:52:13.538 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.016587009 2023-10-01T10:52:13.538 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.538 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.538 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:52:13.538 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.395593+0000", 2023-10-01T10:52:13.538 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.3212000000000003e-05 2023-10-01T10:52:13.538 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.538 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.538 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:52:13.538 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.395666+0000", 2023-10-01T10:52:13.538 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.213244132 2023-10-01T10:52:13.538 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.539 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.539 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:52:13.539 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.608910+0000", 2023-10-01T10:52:13.539 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00016685699999999999 2023-10-01T10:52:13.539 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.539 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.539 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:52:13.539 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.609077+0000", 2023-10-01T10:52:13.539 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000115271 2023-10-01T10:52:13.539 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.539 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.539 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:52:13.540 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.609192+0000", 2023-10-01T10:52:13.540 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.23057741400000001 2023-10-01T10:52:13.540 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:13.540 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:52:13.540 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:13.540 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.540 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.540 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.35649.0:158056 9.6 9:68800f44:::benchmark_data_smithi078_113121_object46241:head [delete] snapc 0=[] ondisk+write+known_if_redirected e137)", 2023-10-01T10:52:13.540 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:43:39.383099+0000", 2023-10-01T10:52:13.540 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 514.14649830400003, 2023-10-01T10:52:13.540 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.24195572700000001, 2023-10-01T10:52:13.542 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:52:13.542 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:52:13.542 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:52:13.542 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.35649", 2023-10-01T10:52:13.543 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:52:13.543 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 158056 2023-10-01T10:52:13.543 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.543 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:52:13.543 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.543 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:52:13.543 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.383099+0000", 2023-10-01T10:52:13.543 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:52:13.543 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.543 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.543 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:52:13.543 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.383099+0000", 2023-10-01T10:52:13.544 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-10-01T10:52:13.544 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.544 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.544 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:52:13.544 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.383097+0000", 2023-10-01T10:52:13.544 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.044e-05 2023-10-01T10:52:13.544 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.544 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.544 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:52:13.544 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.383108+0000", 2023-10-01T10:52:13.544 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1627999999999999e-05 2023-10-01T10:52:13.544 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.545 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.545 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:52:13.545 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.383119+0000", 2023-10-01T10:52:13.545 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.5639999999999996e-06 2023-10-01T10:52:13.545 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.545 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.545 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:52:13.545 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.383125+0000", 2023-10-01T10:52:13.545 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.001306923 2023-10-01T10:52:13.545 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.545 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.545 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:52:13.546 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.384432+0000", 2023-10-01T10:52:13.546 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00010113 2023-10-01T10:52:13.546 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.546 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.546 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:52:13.546 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.384533+0000", 2023-10-01T10:52:13.546 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.4908999999999995e-05 2023-10-01T10:52:13.546 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.546 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.546 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1,2", 2023-10-01T10:52:13.546 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.384628+0000", 2023-10-01T10:52:13.547 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.010997521999999999 2023-10-01T10:52:13.547 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.547 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.547 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:52:13.547 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.395625+0000", 2023-10-01T10:52:13.547 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.6918e-05 2023-10-01T10:52:13.547 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.547 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.547 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:52:13.547 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.395672+0000", 2023-10-01T10:52:13.547 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.22925029099999999 2023-10-01T10:52:13.547 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.548 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.548 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:52:13.548 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.624923+0000", 2023-10-01T10:52:13.548 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.3591999999999997e-05 2023-10-01T10:52:13.548 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.548 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.548 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:52:13.548 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.624986+0000", 2023-10-01T10:52:13.548 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.8189999999999996e-05 2023-10-01T10:52:13.548 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.548 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.548 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:52:13.549 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.625054+0000", 2023-10-01T10:52:13.549 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.24195572700000001 2023-10-01T10:52:13.549 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:13.549 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:52:13.549 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:13.549 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.549 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.549 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.35649.0:158057 9.a 9:5726c16d:::benchmark_data_smithi078_113121_object46242:head [delete] snapc 0=[] ondisk+write+known_if_redirected e137)", 2023-10-01T10:52:13.549 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:43:39.386088+0000", 2023-10-01T10:52:13.549 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 514.14350875399998, 2023-10-01T10:52:13.549 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.23333553600000001, 2023-10-01T10:52:13.549 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:52:13.550 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:52:13.550 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:52:13.550 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.35649", 2023-10-01T10:52:13.550 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:52:13.550 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 158057 2023-10-01T10:52:13.550 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.550 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:52:13.550 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.550 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:52:13.550 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.386088+0000", 2023-10-01T10:52:13.550 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:52:13.551 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.551 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.551 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:52:13.551 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.386088+0000", 2023-10-01T10:52:13.551 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-10-01T10:52:13.551 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.551 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.551 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:52:13.551 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.386086+0000", 2023-10-01T10:52:13.551 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.9789999999999993e-06 2023-10-01T10:52:13.551 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.551 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.551 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:52:13.552 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.386094+0000", 2023-10-01T10:52:13.552 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4647999999999999e-05 2023-10-01T10:52:13.552 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.552 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.552 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:52:13.552 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.386109+0000", 2023-10-01T10:52:13.552 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.0670000000000003e-06 2023-10-01T10:52:13.552 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.552 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.552 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:52:13.552 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.386116+0000", 2023-10-01T10:52:13.552 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.1812999999999999e-05 2023-10-01T10:52:13.553 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.553 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.553 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:52:13.553 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.386178+0000", 2023-10-01T10:52:13.553 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000114607 2023-10-01T10:52:13.553 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.553 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.553 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:52:13.553 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.386292+0000", 2023-10-01T10:52:13.553 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000136711 2023-10-01T10:52:13.553 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.553 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.554 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1,7", 2023-10-01T10:52:13.554 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.386429+0000", 2023-10-01T10:52:13.554 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.010457914 2023-10-01T10:52:13.554 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.554 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.554 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:52:13.554 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.396887+0000", 2023-10-01T10:52:13.554 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0020678089999999999 2023-10-01T10:52:13.554 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.554 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.554 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:52:13.554 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.398955+0000", 2023-10-01T10:52:13.555 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.220305955 2023-10-01T10:52:13.555 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.555 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.555 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:52:13.555 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.619261+0000", 2023-10-01T10:52:13.555 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.9376999999999995e-05 2023-10-01T10:52:13.555 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.555 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.555 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:52:13.555 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.619350+0000", 2023-10-01T10:52:13.555 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.3505999999999997e-05 2023-10-01T10:52:13.555 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.556 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.556 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:52:13.556 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.619424+0000", 2023-10-01T10:52:13.556 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.23333553600000001 2023-10-01T10:52:13.556 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:13.556 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:52:13.556 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:13.556 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.556 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.556 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.35649.0:158058 9.6 9:687242d5:::benchmark_data_smithi078_113121_object46243:head [delete] snapc 0=[] ondisk+write+known_if_redirected e137)", 2023-10-01T10:52:13.556 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:43:39.386184+0000", 2023-10-01T10:52:13.556 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 514.14341325199996, 2023-10-01T10:52:13.557 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.239007572, 2023-10-01T10:52:13.557 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:52:13.557 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:52:13.557 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:52:13.557 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.35649", 2023-10-01T10:52:13.557 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:52:13.557 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 158058 2023-10-01T10:52:13.557 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.557 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:52:13.557 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.557 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:52:13.557 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.386184+0000", 2023-10-01T10:52:13.558 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:52:13.558 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.558 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.558 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:52:13.558 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.386184+0000", 2023-10-01T10:52:13.558 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-10-01T10:52:13.558 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.558 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.558 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:52:13.558 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.386183+0000", 2023-10-01T10:52:13.558 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.6150000000000003e-06 2023-10-01T10:52:13.558 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.559 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.559 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:52:13.559 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.386192+0000", 2023-10-01T10:52:13.559 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1348999999999999e-05 2023-10-01T10:52:13.559 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.559 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.559 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:52:13.559 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.386204+0000", 2023-10-01T10:52:13.559 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.1600000000000003e-06 2023-10-01T10:52:13.559 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.559 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.559 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:52:13.560 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.386210+0000", 2023-10-01T10:52:13.560 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0031139729999999999 2023-10-01T10:52:13.560 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.560 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.560 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:52:13.560 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.389324+0000", 2023-10-01T10:52:13.560 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000119659 2023-10-01T10:52:13.560 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.560 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.560 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:52:13.561 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.389444+0000", 2023-10-01T10:52:13.561 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00013454600000000001 2023-10-01T10:52:13.561 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.561 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.561 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1,2", 2023-10-01T10:52:13.561 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.389578+0000", 2023-10-01T10:52:13.561 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0060741800000000002 2023-10-01T10:52:13.561 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.561 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.561 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:52:13.561 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.395652+0000", 2023-10-01T10:52:13.561 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0031828770000000002 2023-10-01T10:52:13.562 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.562 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.562 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:52:13.562 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.398835+0000", 2023-10-01T10:52:13.562 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.22626552999999999 2023-10-01T10:52:13.562 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.562 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.562 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:52:13.562 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.625101+0000", 2023-10-01T10:52:13.562 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.9758e-05 2023-10-01T10:52:13.562 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.562 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.563 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:52:13.563 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.625150+0000", 2023-10-01T10:52:13.563 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.0877999999999997e-05 2023-10-01T10:52:13.563 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.563 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.563 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:52:13.563 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.625191+0000", 2023-10-01T10:52:13.563 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.239007572 2023-10-01T10:52:13.563 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:13.563 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:52:13.563 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:13.563 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.564 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.564 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.35649.0:158071 9.a 9:59beab16:::benchmark_data_smithi078_113121_object46256:head [delete] snapc 0=[] ondisk+write+known_if_redirected e138)", 2023-10-01T10:52:13.564 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:43:39.419012+0000", 2023-10-01T10:52:13.564 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 514.11058528599995, 2023-10-01T10:52:13.564 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.200749969, 2023-10-01T10:52:13.564 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:52:13.564 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:52:13.564 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:52:13.564 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.35649", 2023-10-01T10:52:13.564 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:52:13.564 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 158071 2023-10-01T10:52:13.564 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.565 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:52:13.565 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.565 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:52:13.565 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.419012+0000", 2023-10-01T10:52:13.565 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:52:13.565 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.565 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.565 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:52:13.565 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.419012+0000", 2023-10-01T10:52:13.565 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-10-01T10:52:13.565 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.566 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.566 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:52:13.566 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.419010+0000", 2023-10-01T10:52:13.566 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.2259999999999994e-06 2023-10-01T10:52:13.566 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.566 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.566 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:52:13.566 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.419019+0000", 2023-10-01T10:52:13.566 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.5914999999999999e-05 2023-10-01T10:52:13.566 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.566 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.567 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:52:13.567 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.419035+0000", 2023-10-01T10:52:13.567 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.477e-06 2023-10-01T10:52:13.567 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.567 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.567 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:52:13.567 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.419041+0000", 2023-10-01T10:52:13.567 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.0578999999999997e-05 2023-10-01T10:52:13.567 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.567 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.567 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:52:13.567 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.419102+0000", 2023-10-01T10:52:13.568 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00012139899999999999 2023-10-01T10:52:13.568 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.568 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.568 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:52:13.568 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.419223+0000", 2023-10-01T10:52:13.568 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000121236 2023-10-01T10:52:13.568 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.568 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.568 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1,7", 2023-10-01T10:52:13.568 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.419345+0000", 2023-10-01T10:52:13.568 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0047990990000000002 2023-10-01T10:52:13.568 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.569 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.569 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:52:13.569 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.424144+0000", 2023-10-01T10:52:13.569 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00019378900000000001 2023-10-01T10:52:13.569 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.569 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.569 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:52:13.569 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.424337+0000", 2023-10-01T10:52:13.569 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.195223853 2023-10-01T10:52:13.569 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.569 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.569 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:52:13.570 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.619561+0000", 2023-10-01T10:52:13.570 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000106698 2023-10-01T10:52:13.570 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.570 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.570 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:52:13.570 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.619668+0000", 2023-10-01T10:52:13.570 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.3640999999999993e-05 2023-10-01T10:52:13.570 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.570 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.570 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:52:13.570 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:39.619762+0000", 2023-10-01T10:52:13.570 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.200749969 2023-10-01T10:52:13.571 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:13.571 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:52:13.571 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:13.571 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.571 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.571 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.35649.0:166281 9.f 9:fd064da1:::benchmark_data_smithi078_113121_object54466:head [delete] snapc 0=[] ondisk+write+known_if_redirected e139)", 2023-10-01T10:52:13.571 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:43:51.304959+0000", 2023-10-01T10:52:13.571 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 502.22463811400002, 2023-10-01T10:52:13.571 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.25730191099999999, 2023-10-01T10:52:13.571 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:52:13.571 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:52:13.571 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:52:13.572 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.35649", 2023-10-01T10:52:13.572 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:52:13.572 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 166281 2023-10-01T10:52:13.572 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.572 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:52:13.572 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.572 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:52:13.572 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.304959+0000", 2023-10-01T10:52:13.572 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:52:13.572 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.572 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.572 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:52:13.573 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.304959+0000", 2023-10-01T10:52:13.573 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-10-01T10:52:13.573 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.573 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.573 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:52:13.573 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.304958+0000", 2023-10-01T10:52:13.573 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.9640000000000002e-06 2023-10-01T10:52:13.573 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.573 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.573 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:52:13.573 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.304963+0000", 2023-10-01T10:52:13.574 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.8380000000000002e-06 2023-10-01T10:52:13.574 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.574 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.574 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:52:13.574 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.304971+0000", 2023-10-01T10:52:13.574 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.5754000000000001e-05 2023-10-01T10:52:13.574 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.574 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.574 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:52:13.574 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.304997+0000", 2023-10-01T10:52:13.574 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.1581999999999993e-05 2023-10-01T10:52:13.574 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.575 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.575 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:52:13.575 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.305068+0000", 2023-10-01T10:52:13.575 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000140288 2023-10-01T10:52:13.575 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.575 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.575 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:52:13.575 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.305209+0000", 2023-10-01T10:52:13.575 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00014844200000000001 2023-10-01T10:52:13.575 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.575 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.576 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1,3", 2023-10-01T10:52:13.576 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.305357+0000", 2023-10-01T10:52:13.576 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.022858590000000002 2023-10-01T10:52:13.576 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.576 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.576 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:52:13.576 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.328216+0000", 2023-10-01T10:52:13.576 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.6810000000000005e-05 2023-10-01T10:52:13.576 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.576 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.576 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:52:13.577 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.328292+0000", 2023-10-01T10:52:13.577 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.23366745899999999 2023-10-01T10:52:13.577 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.577 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.577 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:52:13.577 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.561960+0000", 2023-10-01T10:52:13.577 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00015784999999999999 2023-10-01T10:52:13.577 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.577 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.577 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:52:13.577 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.562118+0000", 2023-10-01T10:52:13.577 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00014305000000000001 2023-10-01T10:52:13.578 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.578 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.578 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:52:13.578 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.562261+0000", 2023-10-01T10:52:13.578 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.25730191099999999 2023-10-01T10:52:13.578 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:13.578 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:52:13.578 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:13.578 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.578 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.578 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.35649.0:166289 9.f 9:f0eeae49:::benchmark_data_smithi078_113121_object54474:head [delete] snapc 0=[] ondisk+write+known_if_redirected e139)", 2023-10-01T10:52:13.579 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:43:51.311529+0000", 2023-10-01T10:52:13.579 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 502.218068324, 2023-10-01T10:52:13.579 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.26435192099999999, 2023-10-01T10:52:13.579 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:52:13.579 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:52:13.579 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:52:13.579 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.35649", 2023-10-01T10:52:13.579 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:52:13.579 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 166289 2023-10-01T10:52:13.579 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.579 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:52:13.579 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.580 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:52:13.580 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.311529+0000", 2023-10-01T10:52:13.580 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:52:13.580 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.580 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.580 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:52:13.580 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.311529+0000", 2023-10-01T10:52:13.580 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-10-01T10:52:13.580 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.580 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.580 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:52:13.580 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.311527+0000", 2023-10-01T10:52:13.581 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.8909999999999998e-06 2023-10-01T10:52:13.581 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.581 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.581 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:52:13.581 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.311533+0000", 2023-10-01T10:52:13.581 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.6220999999999999e-05 2023-10-01T10:52:13.581 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.581 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.581 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:52:13.581 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.311550+0000", 2023-10-01T10:52:13.581 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.5630000000000003e-06 2023-10-01T10:52:13.581 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.582 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.582 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:52:13.582 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.311557+0000", 2023-10-01T10:52:13.582 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0022710740000000001 2023-10-01T10:52:13.582 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.582 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.582 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:52:13.582 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.313828+0000", 2023-10-01T10:52:13.582 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000110944 2023-10-01T10:52:13.582 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.582 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.583 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:52:13.583 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.313939+0000", 2023-10-01T10:52:13.583 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00013319499999999999 2023-10-01T10:52:13.583 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.583 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.583 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1,3", 2023-10-01T10:52:13.583 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.314072+0000", 2023-10-01T10:52:13.583 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.01415136 2023-10-01T10:52:13.583 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.583 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.583 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:52:13.583 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.328224+0000", 2023-10-01T10:52:13.584 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.001166952 2023-10-01T10:52:13.584 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.584 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.584 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:52:13.584 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.329391+0000", 2023-10-01T10:52:13.584 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.24640379800000001 2023-10-01T10:52:13.584 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.584 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.584 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:52:13.584 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.575795+0000", 2023-10-01T10:52:13.584 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.6008999999999999e-05 2023-10-01T10:52:13.584 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.585 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.585 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:52:13.585 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.575841+0000", 2023-10-01T10:52:13.585 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.9950000000000002e-05 2023-10-01T10:52:13.585 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.585 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.585 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:52:13.585 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.575881+0000", 2023-10-01T10:52:13.585 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.26435192099999999 2023-10-01T10:52:13.585 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:13.585 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:52:13.585 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:13.586 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.586 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.586 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.35649.0:166292 9.f 9:f0255b7f:::benchmark_data_smithi078_113121_object54477:head [delete] snapc 0=[] ondisk+write+known_if_redirected e139)", 2023-10-01T10:52:13.586 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:43:51.316157+0000", 2023-10-01T10:52:13.586 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 502.21343959799998, 2023-10-01T10:52:13.586 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.25986541400000002, 2023-10-01T10:52:13.586 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:52:13.586 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:52:13.586 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:52:13.586 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.35649", 2023-10-01T10:52:13.586 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:52:13.587 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 166292 2023-10-01T10:52:13.587 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.587 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:52:13.587 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.587 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:52:13.587 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.316157+0000", 2023-10-01T10:52:13.587 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:52:13.587 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.587 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.587 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:52:13.587 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.316157+0000", 2023-10-01T10:52:13.587 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-10-01T10:52:13.588 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.588 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.588 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:52:13.588 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.316156+0000", 2023-10-01T10:52:13.588 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.4810000000000003e-06 2023-10-01T10:52:13.588 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.588 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.588 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:52:13.588 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.316162+0000", 2023-10-01T10:52:13.588 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.6629999999999998e-05 2023-10-01T10:52:13.588 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.588 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.589 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:52:13.589 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.316179+0000", 2023-10-01T10:52:13.589 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.2309999999999999e-06 2023-10-01T10:52:13.589 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.589 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.589 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:52:13.589 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.316186+0000", 2023-10-01T10:52:13.589 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0022799230000000001 2023-10-01T10:52:13.589 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.589 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.589 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:52:13.589 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.318466+0000", 2023-10-01T10:52:13.590 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.4634000000000002e-05 2023-10-01T10:52:13.590 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.590 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.590 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:52:13.590 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.318560+0000", 2023-10-01T10:52:13.590 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.6883000000000001e-05 2023-10-01T10:52:13.590 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.590 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.590 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1,3", 2023-10-01T10:52:13.590 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.318657+0000", 2023-10-01T10:52:13.590 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0095721520000000004 2023-10-01T10:52:13.590 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.591 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.591 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:52:13.591 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.328229+0000", 2023-10-01T10:52:13.591 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.001190913 2023-10-01T10:52:13.591 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.591 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.591 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:52:13.591 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.329420+0000", 2023-10-01T10:52:13.591 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.246501003 2023-10-01T10:52:13.591 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.591 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.592 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:52:13.592 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.575921+0000", 2023-10-01T10:52:13.592 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.1399999999999997e-05 2023-10-01T10:52:13.592 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.592 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.592 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:52:13.592 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.575963+0000", 2023-10-01T10:52:13.592 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.0003999999999997e-05 2023-10-01T10:52:13.592 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.592 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.592 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:52:13.592 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.576023+0000", 2023-10-01T10:52:13.593 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.25986541400000002 2023-10-01T10:52:13.593 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:13.593 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:52:13.593 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:13.593 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.593 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.593 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.35649.0:166294 9.f 9:f325518b:::benchmark_data_smithi078_113121_object54479:head [delete] snapc 0=[] ondisk+write+known_if_redirected e139)", 2023-10-01T10:52:13.593 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:43:51.316515+0000", 2023-10-01T10:52:13.593 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 502.21308162499997, 2023-10-01T10:52:13.593 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.25958063999999997, 2023-10-01T10:52:13.593 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:52:13.593 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:52:13.593 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:52:13.594 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.35649", 2023-10-01T10:52:13.594 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:52:13.594 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 166294 2023-10-01T10:52:13.594 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.594 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:52:13.594 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.594 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:52:13.594 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.316515+0000", 2023-10-01T10:52:13.594 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:52:13.594 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.594 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.594 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:52:13.595 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.316515+0000", 2023-10-01T10:52:13.595 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-10-01T10:52:13.595 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.595 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.595 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:52:13.595 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.316514+0000", 2023-10-01T10:52:13.595 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.8499999999999999e-06 2023-10-01T10:52:13.595 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.595 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.595 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:52:13.595 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.316520+0000", 2023-10-01T10:52:13.595 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.7333000000000002e-05 2023-10-01T10:52:13.596 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.596 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.596 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:52:13.596 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.316538+0000", 2023-10-01T10:52:13.596 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.3350000000000003e-06 2023-10-01T10:52:13.596 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.596 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.596 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:52:13.596 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.316544+0000", 2023-10-01T10:52:13.596 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0063673200000000001 2023-10-01T10:52:13.596 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.596 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.597 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:52:13.597 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.322911+0000", 2023-10-01T10:52:13.597 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000111931 2023-10-01T10:52:13.597 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.597 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.597 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:52:13.597 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.323023+0000", 2023-10-01T10:52:13.597 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00013561399999999999 2023-10-01T10:52:13.597 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.597 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.597 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1,3", 2023-10-01T10:52:13.597 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.323159+0000", 2023-10-01T10:52:13.598 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0052132979999999999 2023-10-01T10:52:13.598 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.598 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.598 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:52:13.598 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.328372+0000", 2023-10-01T10:52:13.598 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0011226299999999999 2023-10-01T10:52:13.598 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.598 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.598 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:52:13.598 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.329495+0000", 2023-10-01T10:52:13.598 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.24655533399999999 2023-10-01T10:52:13.599 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.599 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.599 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:52:13.599 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.576050+0000", 2023-10-01T10:52:13.599 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.4501000000000001e-05 2023-10-01T10:52:13.599 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.599 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.599 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:52:13.599 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.576074+0000", 2023-10-01T10:52:13.599 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.1484e-05 2023-10-01T10:52:13.599 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.599 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.600 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:52:13.600 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.576096+0000", 2023-10-01T10:52:13.600 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.25958063999999997 2023-10-01T10:52:13.600 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:13.600 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:52:13.600 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:13.600 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.600 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.600 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.35649.0:166304 9.f 9:f52a2f72:::benchmark_data_smithi078_113121_object54489:head [delete] snapc 0=[] ondisk+write+known_if_redirected e139)", 2023-10-01T10:52:13.600 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:43:51.343789+0000", 2023-10-01T10:52:13.600 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 502.18580792400002, 2023-10-01T10:52:13.600 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.24844228900000001, 2023-10-01T10:52:13.601 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:52:13.601 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:52:13.601 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:52:13.601 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.35649", 2023-10-01T10:52:13.601 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/1558509921", 2023-10-01T10:52:13.601 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 166304 2023-10-01T10:52:13.601 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.601 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:52:13.601 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.601 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:52:13.601 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.343789+0000", 2023-10-01T10:52:13.601 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:52:13.602 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.602 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.602 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:52:13.602 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.343789+0000", 2023-10-01T10:52:13.602 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-10-01T10:52:13.602 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.602 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.602 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:52:13.602 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.343788+0000", 2023-10-01T10:52:13.602 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.1620000000000001e-06 2023-10-01T10:52:13.602 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.602 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.603 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:52:13.603 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.343792+0000", 2023-10-01T10:52:13.603 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.5560000000000008e-06 2023-10-01T10:52:13.603 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.603 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.603 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:52:13.603 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.343801+0000", 2023-10-01T10:52:13.603 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.4750000000000004e-06 2023-10-01T10:52:13.603 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.603 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.603 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:52:13.604 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.343805+0000", 2023-10-01T10:52:13.604 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.1152000000000001e-05 2023-10-01T10:52:13.604 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.604 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.604 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:52:13.604 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.343856+0000", 2023-10-01T10:52:13.604 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000121626 2023-10-01T10:52:13.604 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.604 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.604 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:52:13.604 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.343978+0000", 2023-10-01T10:52:13.604 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000125766 2023-10-01T10:52:13.605 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.605 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.605 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1,3", 2023-10-01T10:52:13.605 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.344104+0000", 2023-10-01T10:52:13.605 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0049757450000000002 2023-10-01T10:52:13.605 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.605 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.605 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:52:13.605 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.349079+0000", 2023-10-01T10:52:13.605 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000178254 2023-10-01T10:52:13.605 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.605 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.606 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:52:13.606 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.349258+0000", 2023-10-01T10:52:13.606 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.24281037499999999 2023-10-01T10:52:13.606 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.606 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.606 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:52:13.606 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.592068+0000", 2023-10-01T10:52:13.606 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00011524 2023-10-01T10:52:13.606 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.606 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.606 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:52:13.606 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.592183+0000", 2023-10-01T10:52:13.607 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.8003000000000001e-05 2023-10-01T10:52:13.607 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.607 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.607 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:52:13.607 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:43:51.592231+0000", 2023-10-01T10:52:13.607 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.24844228900000001 2023-10-01T10:52:13.607 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:13.607 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:52:13.607 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:13.607 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.607 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.608 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.45818.0:42735 10.9 10:98e88831:::benchmark_data_smithi078_135100_object42734:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e155)", 2023-10-01T10:52:13.608 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:46:36.223388+0000", 2023-10-01T10:52:13.608 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 337.306208551, 2023-10-01T10:52:13.608 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.205289428, 2023-10-01T10:52:13.608 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:52:13.608 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:52:13.608 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:52:13.608 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.45818", 2023-10-01T10:52:13.608 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/1151403832", 2023-10-01T10:52:13.608 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 42735 2023-10-01T10:52:13.609 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.609 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:52:13.609 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.609 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:52:13.609 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:36.223388+0000", 2023-10-01T10:52:13.609 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:52:13.609 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.609 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.609 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:52:13.609 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:36.223388+0000", 2023-10-01T10:52:13.609 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-10-01T10:52:13.609 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.610 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.610 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:52:13.610 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:36.223387+0000", 2023-10-01T10:52:13.610 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00033217000000000001 2023-10-01T10:52:13.610 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.610 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.610 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:52:13.610 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:36.223720+0000", 2023-10-01T10:52:13.610 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.1759999999999998e-05 2023-10-01T10:52:13.610 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.610 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.611 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:52:13.611 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:36.223741+0000", 2023-10-01T10:52:13.611 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.2179999999999999e-06 2023-10-01T10:52:13.611 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.611 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.611 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:52:13.611 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:36.223750+0000", 2023-10-01T10:52:13.611 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00320098 2023-10-01T10:52:13.611 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.611 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.611 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:52:13.611 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:36.226951+0000", 2023-10-01T10:52:13.612 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000120907 2023-10-01T10:52:13.612 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.612 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.612 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:52:13.612 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:36.227072+0000", 2023-10-01T10:52:13.612 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00016030999999999999 2023-10-01T10:52:13.612 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.612 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.612 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 2,7", 2023-10-01T10:52:13.612 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:36.227232+0000", 2023-10-01T10:52:13.612 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0019706440000000001 2023-10-01T10:52:13.612 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.613 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.613 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:52:13.613 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:36.229202+0000", 2023-10-01T10:52:13.613 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0048461149999999998 2023-10-01T10:52:13.613 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.613 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.613 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:52:13.613 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:36.234049+0000", 2023-10-01T10:52:13.613 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.194514941 2023-10-01T10:52:13.613 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.613 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.613 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:52:13.614 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:36.428564+0000", 2023-10-01T10:52:13.614 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.3306999999999995e-05 2023-10-01T10:52:13.614 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.614 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.614 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:52:13.614 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:36.428637+0000", 2023-10-01T10:52:13.614 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.1010999999999997e-05 2023-10-01T10:52:13.614 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.614 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.614 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:52:13.614 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:36.428678+0000", 2023-10-01T10:52:13.615 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.205289428 2023-10-01T10:52:13.615 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:13.615 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:52:13.615 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:13.615 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.615 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.615 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.45818.0:92530 10.9 10:94b49bbd:::benchmark_data_smithi078_135100_object92529:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-10-01T10:52:13.615 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:47:22.366206+0000", 2023-10-01T10:52:13.615 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 291.163390902, 2023-10-01T10:52:13.616 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.191573506, 2023-10-01T10:52:13.616 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:52:13.616 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:52:13.616 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:52:13.616 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.45818", 2023-10-01T10:52:13.616 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/1151403832", 2023-10-01T10:52:13.616 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 92530 2023-10-01T10:52:13.616 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.616 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:52:13.616 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.616 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:52:13.616 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.366206+0000", 2023-10-01T10:52:13.617 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:52:13.617 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.617 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.617 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:52:13.617 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.366206+0000", 2023-10-01T10:52:13.617 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-10-01T10:52:13.617 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.617 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.617 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:52:13.617 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.366204+0000", 2023-10-01T10:52:13.617 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.0287000000000002e-05 2023-10-01T10:52:13.617 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.618 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.618 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:52:13.618 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.366244+0000", 2023-10-01T10:52:13.618 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.3964e-05 2023-10-01T10:52:13.618 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.618 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.618 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:52:13.618 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.366268+0000", 2023-10-01T10:52:13.618 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.7439999999999996e-06 2023-10-01T10:52:13.618 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.618 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.618 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:52:13.619 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.366277+0000", 2023-10-01T10:52:13.619 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00014535399999999999 2023-10-01T10:52:13.619 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.619 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.619 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:52:13.619 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.366422+0000", 2023-10-01T10:52:13.619 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000128353 2023-10-01T10:52:13.619 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.619 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.619 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:52:13.619 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.366551+0000", 2023-10-01T10:52:13.620 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00015384700000000001 2023-10-01T10:52:13.620 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.620 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.620 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 2,7", 2023-10-01T10:52:13.620 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.366705+0000", 2023-10-01T10:52:13.620 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0031600370000000001 2023-10-01T10:52:13.620 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.620 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.620 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:52:13.620 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.369865+0000", 2023-10-01T10:52:13.620 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0068597149999999997 2023-10-01T10:52:13.620 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.621 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.621 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:52:13.621 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.376724+0000", 2023-10-01T10:52:13.621 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.180840423 2023-10-01T10:52:13.621 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.621 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.621 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:52:13.621 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.557565+0000", 2023-10-01T10:52:13.621 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000107865 2023-10-01T10:52:13.621 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.621 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.621 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:52:13.622 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.557673+0000", 2023-10-01T10:52:13.622 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00010685 2023-10-01T10:52:13.622 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.622 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.622 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:52:13.622 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.557780+0000", 2023-10-01T10:52:13.622 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.191573506 2023-10-01T10:52:13.622 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:13.622 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:52:13.622 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:13.622 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.623 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.623 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.45818.0:92544 10.9 10:97aed680:::benchmark_data_smithi078_135100_object92543:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-10-01T10:52:13.623 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:47:22.379481+0000", 2023-10-01T10:52:13.623 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 291.150116056, 2023-10-01T10:52:13.623 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.17864908099999999, 2023-10-01T10:52:13.623 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:52:13.623 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:52:13.623 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:52:13.623 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.45818", 2023-10-01T10:52:13.623 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/1151403832", 2023-10-01T10:52:13.623 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 92544 2023-10-01T10:52:13.623 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.624 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:52:13.624 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.624 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:52:13.624 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.379481+0000", 2023-10-01T10:52:13.624 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:52:13.624 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.624 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.624 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:52:13.624 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.379481+0000", 2023-10-01T10:52:13.624 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-10-01T10:52:13.624 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.624 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.625 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:52:13.625 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.379479+0000", 2023-10-01T10:52:13.625 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.0435000000000002e-05 2023-10-01T10:52:13.625 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.625 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.625 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:52:13.625 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.379519+0000", 2023-10-01T10:52:13.625 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.3774999999999999e-05 2023-10-01T10:52:13.625 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.625 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.626 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:52:13.626 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.379543+0000", 2023-10-01T10:52:13.626 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.5650000000000007e-06 2023-10-01T10:52:13.626 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.626 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.626 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:52:13.626 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.379553+0000", 2023-10-01T10:52:13.626 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.7650000000000005e-05 2023-10-01T10:52:13.626 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.626 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.626 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:52:13.626 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.379620+0000", 2023-10-01T10:52:13.627 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00015788999999999999 2023-10-01T10:52:13.627 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.627 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.627 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:52:13.627 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.379778+0000", 2023-10-01T10:52:13.627 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00018393500000000001 2023-10-01T10:52:13.627 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.627 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.627 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 2,7", 2023-10-01T10:52:13.627 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.379962+0000", 2023-10-01T10:52:13.627 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0085282380000000005 2023-10-01T10:52:13.628 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.628 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.628 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:52:13.628 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.388491+0000", 2023-10-01T10:52:13.628 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00095209099999999996 2023-10-01T10:52:13.628 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.628 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.628 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:52:13.628 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.389443+0000", 2023-10-01T10:52:13.628 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.16837856800000001 2023-10-01T10:52:13.628 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.628 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.629 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:52:13.629 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.557821+0000", 2023-10-01T10:52:13.629 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00026100599999999999 2023-10-01T10:52:13.629 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.629 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.629 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:52:13.629 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.558082+0000", 2023-10-01T10:52:13.629 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.7814e-05 2023-10-01T10:52:13.629 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.629 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.629 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:52:13.629 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.558130+0000", 2023-10-01T10:52:13.630 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.17864908099999999 2023-10-01T10:52:13.630 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:13.630 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:52:13.630 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:13.630 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.630 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.630 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.45818.0:92550 10.9 10:90d700e9:::benchmark_data_smithi078_135100_object92549:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-10-01T10:52:13.630 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:47:22.389238+0000", 2023-10-01T10:52:13.630 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 291.14035876499997, 2023-10-01T10:52:13.630 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.17912829499999999, 2023-10-01T10:52:13.630 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:52:13.630 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:52:13.631 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:52:13.631 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.45818", 2023-10-01T10:52:13.631 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/1151403832", 2023-10-01T10:52:13.631 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 92550 2023-10-01T10:52:13.631 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.631 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:52:13.631 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.631 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:52:13.631 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.389238+0000", 2023-10-01T10:52:13.631 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:52:13.632 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.632 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.632 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:52:13.632 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.389238+0000", 2023-10-01T10:52:13.632 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-10-01T10:52:13.632 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.632 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.632 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:52:13.632 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.389236+0000", 2023-10-01T10:52:13.632 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.5151999999999997e-05 2023-10-01T10:52:13.632 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.632 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.632 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:52:13.633 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.389281+0000", 2023-10-01T10:52:13.633 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4997999999999999e-05 2023-10-01T10:52:13.633 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.633 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.633 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:52:13.633 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.389296+0000", 2023-10-01T10:52:13.633 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.9680000000000006e-06 2023-10-01T10:52:13.633 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.633 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.633 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:52:13.633 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.389304+0000", 2023-10-01T10:52:13.633 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00015711699999999999 2023-10-01T10:52:13.634 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.634 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.634 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:52:13.634 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.389461+0000", 2023-10-01T10:52:13.634 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.9151000000000001e-05 2023-10-01T10:52:13.634 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.634 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.634 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:52:13.634 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.389551+0000", 2023-10-01T10:52:13.634 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000120294 2023-10-01T10:52:13.634 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.634 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.635 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 2,7", 2023-10-01T10:52:13.635 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.389671+0000", 2023-10-01T10:52:13.635 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0076907599999999996 2023-10-01T10:52:13.635 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.635 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.635 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:52:13.635 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.397362+0000", 2023-10-01T10:52:13.635 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000157715 2023-10-01T10:52:13.635 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.635 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.635 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:52:13.635 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.397519+0000", 2023-10-01T10:52:13.635 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.166123615 2023-10-01T10:52:13.636 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.636 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.636 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:52:13.636 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.563643+0000", 2023-10-01T10:52:13.636 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0046712560000000004 2023-10-01T10:52:13.636 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.636 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.636 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:52:13.636 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.568314+0000", 2023-10-01T10:52:13.636 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.2237000000000001e-05 2023-10-01T10:52:13.636 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.636 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.637 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:52:13.637 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.568366+0000", 2023-10-01T10:52:13.637 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.17912829499999999 2023-10-01T10:52:13.637 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:13.637 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:52:13.637 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:13.637 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.637 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.637 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46037.0:63469 11.7 11:eef61f7b:::benchmark_data_smithi078_157677_object63468:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e180)", 2023-10-01T10:52:13.637 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:52:01.739210+0000", 2023-10-01T10:52:13.637 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 11.790387304999999, 2023-10-01T10:52:13.637 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.20259924300000001, 2023-10-01T10:52:13.638 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:52:13.638 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:52:13.638 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:52:13.638 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46037", 2023-10-01T10:52:13.638 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:52:13.638 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 63469 2023-10-01T10:52:13.638 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.638 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:52:13.638 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.638 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:52:13.638 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.739210+0000", 2023-10-01T10:52:13.638 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:52:13.639 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.639 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.639 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:52:13.639 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.739210+0000", 2023-10-01T10:52:13.639 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999976 2023-10-01T10:52:13.639 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.639 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.639 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:52:13.639 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.739207+0000", 2023-10-01T10:52:13.639 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.5088999999999997e-05 2023-10-01T10:52:13.639 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.640 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.640 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:52:13.640 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.739303+0000", 2023-10-01T10:52:13.640 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.1024000000000001e-05 2023-10-01T10:52:13.640 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.640 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.640 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:52:13.640 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.739324+0000", 2023-10-01T10:52:13.640 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1515000000000001e-05 2023-10-01T10:52:13.640 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.640 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.640 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:52:13.641 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.739335+0000", 2023-10-01T10:52:13.641 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.3280000000000004e-05 2023-10-01T10:52:13.641 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.641 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.641 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:52:13.641 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.739398+0000", 2023-10-01T10:52:13.641 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00016312499999999999 2023-10-01T10:52:13.641 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.641 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.641 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:52:13.641 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.739561+0000", 2023-10-01T10:52:13.641 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000143199 2023-10-01T10:52:13.642 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.642 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.642 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1,4", 2023-10-01T10:52:13.642 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.739705+0000", 2023-10-01T10:52:13.642 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0043341100000000004 2023-10-01T10:52:13.642 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.642 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.642 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:52:13.642 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.744039+0000", 2023-10-01T10:52:13.642 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0024068879999999998 2023-10-01T10:52:13.642 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.643 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.643 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:52:13.643 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.746446+0000", 2023-10-01T10:52:13.643 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0051348449999999999 2023-10-01T10:52:13.643 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.643 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.643 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:52:13.643 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.751581+0000", 2023-10-01T10:52:13.643 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.190168541 2023-10-01T10:52:13.643 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.643 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.643 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:52:13.644 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.941749+0000", 2023-10-01T10:52:13.644 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.982e-05 2023-10-01T10:52:13.644 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.644 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.644 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:52:13.644 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.941809+0000", 2023-10-01T10:52:13.644 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.20259924300000001 2023-10-01T10:52:13.644 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:13.644 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:52:13.644 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:13.644 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.644 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.645 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46037.0:63480 11.7 11:e660f6d9:::benchmark_data_smithi078_157677_object63479:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e180)", 2023-10-01T10:52:13.645 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:52:01.748323+0000", 2023-10-01T10:52:13.645 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 11.781273736999999, 2023-10-01T10:52:13.645 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.193787028, 2023-10-01T10:52:13.645 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:52:13.645 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:52:13.645 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:52:13.645 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46037", 2023-10-01T10:52:13.645 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:52:13.645 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 63480 2023-10-01T10:52:13.645 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.646 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:52:13.646 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.646 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:52:13.646 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.748323+0000", 2023-10-01T10:52:13.646 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:52:13.646 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.646 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.646 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:52:13.646 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.748323+0000", 2023-10-01T10:52:13.646 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-10-01T10:52:13.646 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.647 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.647 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:52:13.647 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.748321+0000", 2023-10-01T10:52:13.647 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.9839000000000002e-05 2023-10-01T10:52:13.647 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.647 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.647 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:52:13.647 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.748371+0000", 2023-10-01T10:52:13.647 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.3478e-05 2023-10-01T10:52:13.647 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.647 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.647 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:52:13.647 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.748385+0000", 2023-10-01T10:52:13.648 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.8870000000000003e-06 2023-10-01T10:52:13.648 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.648 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.648 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:52:13.648 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.748391+0000", 2023-10-01T10:52:13.648 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.5112999999999999e-05 2023-10-01T10:52:13.648 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.648 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.648 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:52:13.648 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.748447+0000", 2023-10-01T10:52:13.648 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000167598 2023-10-01T10:52:13.648 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.649 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.649 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:52:13.649 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.748614+0000", 2023-10-01T10:52:13.649 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000193838 2023-10-01T10:52:13.649 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.649 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.649 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1,4", 2023-10-01T10:52:13.649 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.748808+0000", 2023-10-01T10:52:13.649 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.19304511499999999 2023-10-01T10:52:13.649 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.649 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.649 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:52:13.650 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.941853+0000", 2023-10-01T10:52:13.650 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.8409999999999999e-05 2023-10-01T10:52:13.650 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.650 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.650 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:52:13.650 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.941902+0000", 2023-10-01T10:52:13.650 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00015376699999999999 2023-10-01T10:52:13.650 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.650 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.650 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:52:13.650 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.942055+0000", 2023-10-01T10:52:13.650 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.2771999999999997e-05 2023-10-01T10:52:13.651 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.651 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.651 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:52:13.651 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.942088+0000", 2023-10-01T10:52:13.651 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.2192999999999999e-05 2023-10-01T10:52:13.651 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.651 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.651 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:52:13.651 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.942110+0000", 2023-10-01T10:52:13.651 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.193787028 2023-10-01T10:52:13.651 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:13.651 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:52:13.652 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:13.652 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.652 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.652 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop_reply(client.46037.0:63469 11.7 e180/173 ondisk, result = 0)", 2023-10-01T10:52:13.652 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:52:01.748853+0000", 2023-10-01T10:52:13.652 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 11.780743589, 2023-10-01T10:52:13.652 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.19296834800000001, 2023-10-01T10:52:13.652 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:52:13.652 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "started", 2023-10-01T10:52:13.652 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:52:13.652 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.653 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:52:13.653 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.748853+0000", 2023-10-01T10:52:13.653 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:52:13.653 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.653 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.653 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:52:13.653 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.748853+0000", 2023-10-01T10:52:13.653 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-10-01T10:52:13.653 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.653 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.653 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:52:13.653 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.748852+0000", 2023-10-01T10:52:13.654 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.2829999999999998e-06 2023-10-01T10:52:13.654 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.654 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.654 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:52:13.654 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.748859+0000", 2023-10-01T10:52:13.654 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.06e-05 2023-10-01T10:52:13.654 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.654 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.654 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:52:13.654 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.748869+0000", 2023-10-01T10:52:13.654 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.0929999999999999e-06 2023-10-01T10:52:13.654 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.655 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.655 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:52:13.655 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.748875+0000", 2023-10-01T10:52:13.655 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0026898249999999999 2023-10-01T10:52:13.655 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.655 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.655 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:52:13.655 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.751565+0000", 2023-10-01T10:52:13.655 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.8569999999999995e-06 2023-10-01T10:52:13.655 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.655 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.655 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:52:13.656 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.751573+0000", 2023-10-01T10:52:13.656 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.19024866400000001 2023-10-01T10:52:13.656 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.656 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.656 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:52:13.656 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.941822+0000", 2023-10-01T10:52:13.656 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.19296834800000001 2023-10-01T10:52:13.656 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:13.656 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:52:13.656 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:13.656 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.656 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.656 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.46037.0:63478 11.a e180/173 11:52b20f23:::benchmark_data_smithi078_157677_object63477:head v 180'3945, mlcod=180'3942)", 2023-10-01T10:52:13.657 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:52:01.749896+0000", 2023-10-01T10:52:13.657 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 11.779700765999999, 2023-10-01T10:52:13.657 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.19190481600000001, 2023-10-01T10:52:13.657 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:52:13.657 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:52:13.657 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:52:13.657 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.657 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:52:13.657 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.749896+0000", 2023-10-01T10:52:13.657 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:52:13.657 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.657 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.658 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:52:13.658 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.749896+0000", 2023-10-01T10:52:13.658 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-10-01T10:52:13.658 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.658 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.658 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:52:13.658 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.749895+0000", 2023-10-01T10:52:13.658 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.9036999999999999e-05 2023-10-01T10:52:13.658 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.658 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.658 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:52:13.658 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.749934+0000", 2023-10-01T10:52:13.659 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4068e-05 2023-10-01T10:52:13.659 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.659 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.659 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:52:13.659 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.749948+0000", 2023-10-01T10:52:13.659 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.3699999999999997e-06 2023-10-01T10:52:13.659 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.659 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.659 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:52:13.659 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.749956+0000", 2023-10-01T10:52:13.659 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.3328000000000002e-05 2023-10-01T10:52:13.660 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.660 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.660 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:52:13.660 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.750009+0000", 2023-10-01T10:52:13.660 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.0475e-05 2023-10-01T10:52:13.660 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.660 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.660 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:52:13.660 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.750030+0000", 2023-10-01T10:52:13.660 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.19173195200000001 2023-10-01T10:52:13.660 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.660 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.661 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:52:13.661 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.941762+0000", 2023-10-01T10:52:13.661 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.9478999999999999e-05 2023-10-01T10:52:13.661 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.661 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.661 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:52:13.661 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.941801+0000", 2023-10-01T10:52:13.661 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.19190481600000001 2023-10-01T10:52:13.661 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:13.661 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:52:13.661 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:13.661 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.662 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.662 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop_reply(client.46037.0:63480 11.7 e180/173 ondisk, result = 0)", 2023-10-01T10:52:13.662 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:52:01.751858+0000", 2023-10-01T10:52:13.662 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 11.777738486000001, 2023-10-01T10:52:13.662 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.18999654299999999, 2023-10-01T10:52:13.662 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:52:13.662 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "started", 2023-10-01T10:52:13.662 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:52:13.662 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.662 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:52:13.662 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.751858+0000", 2023-10-01T10:52:13.662 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:52:13.663 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.663 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.663 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:52:13.663 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.751858+0000", 2023-10-01T10:52:13.663 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-10-01T10:52:13.663 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.663 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.663 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:52:13.663 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.751858+0000", 2023-10-01T10:52:13.663 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.5080000000000002e-06 2023-10-01T10:52:13.663 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.663 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.664 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:52:13.664 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.751864+0000", 2023-10-01T10:52:13.664 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.18987274000000001 2023-10-01T10:52:13.664 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.664 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.664 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:52:13.664 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.941737+0000", 2023-10-01T10:52:13.664 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.8646999999999999e-05 2023-10-01T10:52:13.664 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.664 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.664 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:52:13.664 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.941765+0000", 2023-10-01T10:52:13.665 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.5757e-05 2023-10-01T10:52:13.665 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.665 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.665 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:52:13.665 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.941841+0000", 2023-10-01T10:52:13.665 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.088e-06 2023-10-01T10:52:13.665 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.665 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.665 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:52:13.665 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.941847+0000", 2023-10-01T10:52:13.665 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.7300000000000005e-06 2023-10-01T10:52:13.665 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.666 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.666 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:52:13.666 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.941855+0000", 2023-10-01T10:52:13.666 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.18999654299999999 2023-10-01T10:52:13.666 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:13.666 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:52:13.666 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:13.666 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.666 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.666 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.46037.0:63481 11.5 e180/173 11:a7f52d12:::benchmark_data_smithi078_157677_object63480:head v 180'3998, mlcod=180'3996)", 2023-10-01T10:52:13.666 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:52:01.751946+0000", 2023-10-01T10:52:13.667 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 11.777650886, 2023-10-01T10:52:13.667 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.20245234200000001, 2023-10-01T10:52:13.667 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:52:13.667 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:52:13.667 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:52:13.667 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.667 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:52:13.667 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.751946+0000", 2023-10-01T10:52:13.667 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:52:13.667 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.667 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.668 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:52:13.668 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.751946+0000", 2023-10-01T10:52:13.668 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-10-01T10:52:13.668 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.668 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.668 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:52:13.668 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.751945+0000", 2023-10-01T10:52:13.668 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.7599000000000001e-05 2023-10-01T10:52:13.668 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.668 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.668 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:52:13.668 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.751973+0000", 2023-10-01T10:52:13.669 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.189765928 2023-10-01T10:52:13.669 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.669 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.669 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:52:13.669 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.941739+0000", 2023-10-01T10:52:13.669 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.2520000000000001e-05 2023-10-01T10:52:13.669 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.669 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.669 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:52:13.669 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.941761+0000", 2023-10-01T10:52:13.669 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000115344 2023-10-01T10:52:13.669 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.669 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.670 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:52:13.670 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.941877+0000", 2023-10-01T10:52:13.670 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.0764999999999998e-05 2023-10-01T10:52:13.670 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.670 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.670 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:52:13.670 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.941937+0000", 2023-10-01T10:52:13.670 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.012407655 2023-10-01T10:52:13.670 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.670 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.670 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:52:13.671 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.954345+0000", 2023-10-01T10:52:13.671 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.3310000000000003e-05 2023-10-01T10:52:13.671 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:13.671 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:13.671 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:52:13.671 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:01.954398+0000", 2023-10-01T10:52:13.671 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.20245234200000001 2023-10-01T10:52:13.671 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:13.671 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:52:13.671 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:13.671 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:13.671 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:52:13.671 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-10-01T10:52:13.672 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:13.707 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:13.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:13.710 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:52:13.810 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:13.844 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:13.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:13.851 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:52:13.952 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:13.961 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:13 smithi078 bash[75299]: cluster 2023-10-01T10:52:11.964675+0000 mgr.y (mgr.25072) 1039 : cluster 0 pgmap v989: 89 pgs: 89 active+clean; 4.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-01T10:52:13.961 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:13 smithi078 bash[78274]: cluster 2023-10-01T10:52:11.964675+0000 mgr.y (mgr.25072) 1039 : cluster 0 pgmap v989: 89 pgs: 89 active+clean; 4.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-01T10:52:13.979 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:13.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:13.983 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:52:14.084 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:14.093 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:13 smithi173 bash[58816]: cluster 2023-10-01T10:52:11.964675+0000 mgr.y (mgr.25072) 1039 : cluster 0 pgmap v989: 89 pgs: 89 active+clean; 4.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-10-01T10:52:14.123 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:14.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:14.127 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:52:14.227 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:14.260 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:14.260 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 53 16 76441 76425 90.113 62 0.00705514 0.0110902 2023-10-01T10:52:14.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:14.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:52:14.363 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:14.390 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:14.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:14.393 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:14.494 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:14.521 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:14.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:14.522 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:52:14.623 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:14.646 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:14.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:14.652 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:52:14.655 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-01T10:52:14.752 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:14.779 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:14.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:14.785 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:52:14.886 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:14.911 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:14.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:14.916 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:52:15.016 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:15.037 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:15.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:15.039 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:52:15.139 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:15.181 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:15.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:15.186 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:52:15.260 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 54 16 77434 77418 89.5934 62.0625 0.00892688 0.0111551 2023-10-01T10:52:15.287 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:15.318 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:15.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:15.322 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:52:15.422 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:15.451 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:15.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:15.455 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:52:15.556 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:15.584 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:15.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:15.588 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:52:15.688 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:15.715 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:15.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:15.723 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:52:15.824 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:15.857 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:15.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:15.860 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:52:15.961 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:15.988 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:15.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:15.992 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:52:16.093 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:16.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:15 smithi173 bash[58816]: cluster 2023-10-01T10:52:13.965066+0000 mgr.y (mgr.25072) 1040 : cluster 0 pgmap v990: 89 pgs: 89 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 58 MiB/s wr, 932 op/s 2023-10-01T10:52:16.120 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:16.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:16.123 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:52:16.224 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:16.260 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 55 16 78459 78443 89.1291 64.0625 0.0194407 0.0112141 2023-10-01T10:52:16.264 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:16.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:16.267 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:52:16.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:15 smithi078 bash[78274]: cluster 2023-10-01T10:52:13.965066+0000 mgr.y (mgr.25072) 1040 : cluster 0 pgmap v990: 89 pgs: 89 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 58 MiB/s wr, 932 op/s 2023-10-01T10:52:16.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:15 smithi078 bash[75299]: cluster 2023-10-01T10:52:13.965066+0000 mgr.y (mgr.25072) 1040 : cluster 0 pgmap v990: 89 pgs: 89 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 58 MiB/s wr, 932 op/s 2023-10-01T10:52:16.369 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:16.395 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:16.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:16.400 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:16.501 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:16.531 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:16.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:16.534 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:52:16.635 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:16.664 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:16.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:16.702 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:16.802 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:16.831 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:16.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:16.834 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:52:16.934 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:16.960 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:16.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:16.966 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:52:17.066 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:17.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:16 smithi078 bash[75299]: cluster 2023-10-01T10:52:15.965852+0000 mgr.y (mgr.25072) 1041 : cluster 0 pgmap v991: 89 pgs: 89 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-01T10:52:17.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:16 smithi078 bash[78274]: cluster 2023-10-01T10:52:15.965852+0000 mgr.y (mgr.25072) 1041 : cluster 0 pgmap v991: 89 pgs: 89 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-01T10:52:17.094 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:17.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:17.100 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:17.201 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:17.213 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:16 smithi173 bash[58816]: cluster 2023-10-01T10:52:15.965852+0000 mgr.y (mgr.25072) 1041 : cluster 0 pgmap v991: 89 pgs: 89 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-01T10:52:17.227 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:17.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:17.229 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:52:17.261 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 56 16 79271 79255 88.4436 50.75 0.0121874 0.0112996 2023-10-01T10:52:17.277 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T10:52:17.330 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:17.339 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T10:52:17.377 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:17.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:17.378 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:52:17.397 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T10:52:17.448 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T10:52:17.479 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:17.497 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T10:52:17.508 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:17.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:17.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:52:17.535 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T10:52:17.598 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T10:52:17.612 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:17.637 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T10:52:17.645 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:17.645 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T10:52:17.645 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T10:52:17.645 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T10:52:17.646 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T10:52:17.646 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T10:52:17.646 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T10:52:17.646 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T10:52:17.646 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T10:52:17.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:17.648 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:52:17.749 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:17.777 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:17.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:17.779 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:52:17.880 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:17.910 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:17.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:17.912 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:52:17.964 INFO:teuthology.orchestra.run.smithi078.stderr:set osd.1 primary-affinity to 0 (802) 2023-10-01T10:52:17.969 INFO:teuthology.orchestra.run.smithi078.stderr:noscrub is unset 2023-10-01T10:52:18.014 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:18.052 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:18.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:18.055 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:52:18.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:17 smithi078 bash[75299]: audit 2023-10-01T10:52:17.211271+0000 mon.a (mon.0) 1275 : audit 1 from='client.? 172.21.15.78:0/1313597249' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]: dispatch 2023-10-01T10:52:18.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:17 smithi078 bash[75299]: audit 2023-10-01T10:52:17.229272+0000 mon.a (mon.0) 1276 : audit 0 from='client.? 172.21.15.78:0/3736736764' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:52:18.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:17 smithi078 bash[75299]: audit 2023-10-01T10:52:17.434528+0000 mon.a (mon.0) 1277 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:52:18.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:17 smithi078 bash[75299]: audit 2023-10-01T10:52:17.561846+0000 mon.c (mon.2) 174 : audit 1 from='client.? 172.21.15.78:0/2499809540' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T10:52:18.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:17 smithi078 bash[75299]: audit 2023-10-01T10:52:17.562901+0000 mon.a (mon.0) 1278 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T10:52:18.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:17 smithi078 bash[78274]: audit 2023-10-01T10:52:17.211271+0000 mon.a (mon.0) 1275 : audit 1 from='client.? 172.21.15.78:0/1313597249' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]: dispatch 2023-10-01T10:52:18.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:17 smithi078 bash[78274]: audit 2023-10-01T10:52:17.229272+0000 mon.a (mon.0) 1276 : audit 0 from='client.? 172.21.15.78:0/3736736764' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:52:18.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:17 smithi078 bash[78274]: audit 2023-10-01T10:52:17.434528+0000 mon.a (mon.0) 1277 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:52:18.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:17 smithi078 bash[78274]: audit 2023-10-01T10:52:17.561846+0000 mon.c (mon.2) 174 : audit 1 from='client.? 172.21.15.78:0/2499809540' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T10:52:18.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:17 smithi078 bash[78274]: audit 2023-10-01T10:52:17.562901+0000 mon.a (mon.0) 1278 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T10:52:18.156 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:18.180 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:18.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:18.182 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:52:18.261 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 57 16 79854 79838 87.5311 36.4375 0.0042432 0.0113671 2023-10-01T10:52:18.283 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:18.309 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:18.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:18.312 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:52:18.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:17 smithi173 bash[58816]: audit 2023-10-01T10:52:17.211271+0000 mon.a (mon.0) 1275 : audit 1 from='client.? 172.21.15.78:0/1313597249' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]: dispatch 2023-10-01T10:52:18.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:17 smithi173 bash[58816]: audit 2023-10-01T10:52:17.229272+0000 mon.a (mon.0) 1276 : audit 0 from='client.? 172.21.15.78:0/3736736764' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:52:18.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:17 smithi173 bash[58816]: audit 2023-10-01T10:52:17.434528+0000 mon.a (mon.0) 1277 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:52:18.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:17 smithi173 bash[58816]: audit 2023-10-01T10:52:17.561846+0000 mon.c (mon.2) 174 : audit 1 from='client.? 172.21.15.78:0/2499809540' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T10:52:18.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:17 smithi173 bash[58816]: audit 2023-10-01T10:52:17.562901+0000 mon.a (mon.0) 1278 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T10:52:18.412 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:18.436 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:18.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:18.438 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:52:18.539 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:18.567 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:18.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:18.572 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:52:18.672 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:18.699 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:18.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:18.701 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:52:18.802 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:18.834 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:18.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:18.837 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:52:18.938 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:18.969 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:18.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:18.975 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:52:19.076 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:19.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:18 smithi078 bash[75299]: cluster 2023-10-01T10:52:17.932287+0000 mon.a (mon.0) 1279 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:52:19.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:18 smithi078 bash[75299]: cluster 2023-10-01T10:52:17.944561+0000 osd.5 (osd.5) 116 : cluster 3 failed to encode map e181 with expected crc 2023-10-01T10:52:19.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:18 smithi078 bash[75299]: cluster 2023-10-01T10:52:17.947057+0000 osd.6 (osd.6) 98 : cluster 3 failed to encode map e181 with expected crc 2023-10-01T10:52:19.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:18 smithi078 bash[75299]: cluster 2023-10-01T10:52:17.951112+0000 osd.6 (osd.6) 99 : cluster 3 failed to encode map e181 with expected crc 2023-10-01T10:52:19.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:18 smithi078 bash[75299]: cluster 2023-10-01T10:52:17.951554+0000 osd.5 (osd.5) 117 : cluster 3 failed to encode map e181 with expected crc 2023-10-01T10:52:19.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:18 smithi078 bash[75299]: cluster 2023-10-01T10:52:17.952195+0000 osd.7 (osd.7) 135 : cluster 3 failed to encode map e181 with expected crc 2023-10-01T10:52:19.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:18 smithi078 bash[75299]: cluster 2023-10-01T10:52:17.952908+0000 osd.5 (osd.5) 118 : cluster 3 failed to encode map e181 with expected crc 2023-10-01T10:52:19.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:18 smithi078 bash[75299]: cluster 2023-10-01T10:52:17.953712+0000 osd.7 (osd.7) 136 : cluster 3 failed to encode map e181 with expected crc 2023-10-01T10:52:19.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:18 smithi078 bash[75299]: cluster 2023-10-01T10:52:17.954462+0000 osd.6 (osd.6) 100 : cluster 3 failed to encode map e181 with expected crc 2023-10-01T10:52:19.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:18 smithi078 bash[75299]: cluster 2023-10-01T10:52:17.954914+0000 osd.7 (osd.7) 137 : cluster 3 failed to encode map e181 with expected crc 2023-10-01T10:52:19.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:18 smithi078 bash[75299]: cluster 2023-10-01T10:52:17.955662+0000 osd.6 (osd.6) 101 : cluster 3 failed to encode map e181 with expected crc 2023-10-01T10:52:19.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:18 smithi078 bash[75299]: audit 2023-10-01T10:52:17.955754+0000 mon.a (mon.0) 1280 : audit 1 from='client.? 172.21.15.78:0/1313597249' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]': finished 2023-10-01T10:52:19.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:18 smithi078 bash[75299]: audit 2023-10-01T10:52:17.956006+0000 mon.a (mon.0) 1281 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T10:52:19.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:18 smithi078 bash[75299]: cluster 2023-10-01T10:52:17.956461+0000 mon.a (mon.0) 1282 : cluster 0 osdmap e181: 8 total, 8 up, 8 in 2023-10-01T10:52:19.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:18 smithi078 bash[75299]: cluster 2023-10-01T10:52:17.959871+0000 osd.5 (osd.5) 119 : cluster 3 failed to encode map e181 with expected crc 2023-10-01T10:52:19.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:18 smithi078 bash[75299]: cluster 2023-10-01T10:52:17.966293+0000 mgr.y (mgr.25072) 1042 : cluster 0 pgmap v993: 89 pgs: 89 active+clean; 4.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-10-01T10:52:19.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:18 smithi078 bash[75299]: audit 2023-10-01T10:52:18.177664+0000 mon.a (mon.0) 1283 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:52:19.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:18 smithi078 bash[75299]: audit 2023-10-01T10:52:18.185344+0000 mon.a (mon.0) 1284 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:52:19.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:18 smithi078 bash[75299]: audit 2023-10-01T10:52:18.690335+0000 mon.a (mon.0) 1285 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:52:19.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:18 smithi078 bash[75299]: audit 2023-10-01T10:52:18.697468+0000 mon.a (mon.0) 1286 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:52:19.085 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:18 smithi078 bash[78274]: cluster 2023-10-01T10:52:17.932287+0000 mon.a (mon.0) 1279 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:52:19.085 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:18 smithi078 bash[78274]: cluster 2023-10-01T10:52:17.944561+0000 osd.5 (osd.5) 116 : cluster 3 failed to encode map e181 with expected crc 2023-10-01T10:52:19.085 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:18 smithi078 bash[78274]: cluster 2023-10-01T10:52:17.947057+0000 osd.6 (osd.6) 98 : cluster 3 failed to encode map e181 with expected crc 2023-10-01T10:52:19.085 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:18 smithi078 bash[78274]: cluster 2023-10-01T10:52:17.951112+0000 osd.6 (osd.6) 99 : cluster 3 failed to encode map e181 with expected crc 2023-10-01T10:52:19.085 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:18 smithi078 bash[78274]: cluster 2023-10-01T10:52:17.951554+0000 osd.5 (osd.5) 117 : cluster 3 failed to encode map e181 with expected crc 2023-10-01T10:52:19.086 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:18 smithi078 bash[78274]: cluster 2023-10-01T10:52:17.952195+0000 osd.7 (osd.7) 135 : cluster 3 failed to encode map e181 with expected crc 2023-10-01T10:52:19.086 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:18 smithi078 bash[78274]: cluster 2023-10-01T10:52:17.952908+0000 osd.5 (osd.5) 118 : cluster 3 failed to encode map e181 with expected crc 2023-10-01T10:52:19.086 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:18 smithi078 bash[78274]: cluster 2023-10-01T10:52:17.953712+0000 osd.7 (osd.7) 136 : cluster 3 failed to encode map e181 with expected crc 2023-10-01T10:52:19.086 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:18 smithi078 bash[78274]: cluster 2023-10-01T10:52:17.954462+0000 osd.6 (osd.6) 100 : cluster 3 failed to encode map e181 with expected crc 2023-10-01T10:52:19.086 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:18 smithi078 bash[78274]: cluster 2023-10-01T10:52:17.954914+0000 osd.7 (osd.7) 137 : cluster 3 failed to encode map e181 with expected crc 2023-10-01T10:52:19.087 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:18 smithi078 bash[78274]: cluster 2023-10-01T10:52:17.955662+0000 osd.6 (osd.6) 101 : cluster 3 failed to encode map e181 with expected crc 2023-10-01T10:52:19.087 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:18 smithi078 bash[78274]: audit 2023-10-01T10:52:17.955754+0000 mon.a (mon.0) 1280 : audit 1 from='client.? 172.21.15.78:0/1313597249' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]': finished 2023-10-01T10:52:19.087 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:18 smithi078 bash[78274]: audit 2023-10-01T10:52:17.956006+0000 mon.a (mon.0) 1281 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T10:52:19.087 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:18 smithi078 bash[78274]: cluster 2023-10-01T10:52:17.956461+0000 mon.a (mon.0) 1282 : cluster 0 osdmap e181: 8 total, 8 up, 8 in 2023-10-01T10:52:19.087 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:18 smithi078 bash[78274]: cluster 2023-10-01T10:52:17.959871+0000 osd.5 (osd.5) 119 : cluster 3 failed to encode map e181 with expected crc 2023-10-01T10:52:19.087 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:18 smithi078 bash[78274]: cluster 2023-10-01T10:52:17.966293+0000 mgr.y (mgr.25072) 1042 : cluster 0 pgmap v993: 89 pgs: 89 active+clean; 4.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-10-01T10:52:19.088 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:18 smithi078 bash[78274]: audit 2023-10-01T10:52:18.177664+0000 mon.a (mon.0) 1283 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:52:19.088 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:18 smithi078 bash[78274]: audit 2023-10-01T10:52:18.185344+0000 mon.a (mon.0) 1284 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:52:19.088 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:18 smithi078 bash[78274]: audit 2023-10-01T10:52:18.690335+0000 mon.a (mon.0) 1285 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:52:19.088 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:18 smithi078 bash[78274]: audit 2023-10-01T10:52:18.697468+0000 mon.a (mon.0) 1286 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:52:19.106 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:19.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:19.110 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:52:19.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:18 smithi173 bash[58816]: cluster 2023-10-01T10:52:17.932287+0000 mon.a (mon.0) 1279 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:52:19.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:18 smithi173 bash[58816]: cluster 2023-10-01T10:52:17.944561+0000 osd.5 (osd.5) 116 : cluster 3 failed to encode map e181 with expected crc 2023-10-01T10:52:19.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:18 smithi173 bash[58816]: cluster 2023-10-01T10:52:17.947057+0000 osd.6 (osd.6) 98 : cluster 3 failed to encode map e181 with expected crc 2023-10-01T10:52:19.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:18 smithi173 bash[58816]: cluster 2023-10-01T10:52:17.951112+0000 osd.6 (osd.6) 99 : cluster 3 failed to encode map e181 with expected crc 2023-10-01T10:52:19.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:18 smithi173 bash[58816]: cluster 2023-10-01T10:52:17.951554+0000 osd.5 (osd.5) 117 : cluster 3 failed to encode map e181 with expected crc 2023-10-01T10:52:19.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:18 smithi173 bash[58816]: cluster 2023-10-01T10:52:17.952195+0000 osd.7 (osd.7) 135 : cluster 3 failed to encode map e181 with expected crc 2023-10-01T10:52:19.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:18 smithi173 bash[58816]: cluster 2023-10-01T10:52:17.952908+0000 osd.5 (osd.5) 118 : cluster 3 failed to encode map e181 with expected crc 2023-10-01T10:52:19.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:18 smithi173 bash[58816]: cluster 2023-10-01T10:52:17.953712+0000 osd.7 (osd.7) 136 : cluster 3 failed to encode map e181 with expected crc 2023-10-01T10:52:19.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:18 smithi173 bash[58816]: cluster 2023-10-01T10:52:17.954462+0000 osd.6 (osd.6) 100 : cluster 3 failed to encode map e181 with expected crc 2023-10-01T10:52:19.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:18 smithi173 bash[58816]: cluster 2023-10-01T10:52:17.954914+0000 osd.7 (osd.7) 137 : cluster 3 failed to encode map e181 with expected crc 2023-10-01T10:52:19.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:18 smithi173 bash[58816]: cluster 2023-10-01T10:52:17.955662+0000 osd.6 (osd.6) 101 : cluster 3 failed to encode map e181 with expected crc 2023-10-01T10:52:19.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:18 smithi173 bash[58816]: audit 2023-10-01T10:52:17.955754+0000 mon.a (mon.0) 1280 : audit 1 from='client.? 172.21.15.78:0/1313597249' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]': finished 2023-10-01T10:52:19.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:18 smithi173 bash[58816]: audit 2023-10-01T10:52:17.956006+0000 mon.a (mon.0) 1281 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T10:52:19.122 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:18 smithi173 bash[58816]: cluster 2023-10-01T10:52:17.956461+0000 mon.a (mon.0) 1282 : cluster 0 osdmap e181: 8 total, 8 up, 8 in 2023-10-01T10:52:19.122 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:18 smithi173 bash[58816]: cluster 2023-10-01T10:52:17.959871+0000 osd.5 (osd.5) 119 : cluster 3 failed to encode map e181 with expected crc 2023-10-01T10:52:19.122 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:18 smithi173 bash[58816]: cluster 2023-10-01T10:52:17.966293+0000 mgr.y (mgr.25072) 1042 : cluster 0 pgmap v993: 89 pgs: 89 active+clean; 4.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-10-01T10:52:19.122 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:18 smithi173 bash[58816]: audit 2023-10-01T10:52:18.177664+0000 mon.a (mon.0) 1283 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:52:19.122 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:18 smithi173 bash[58816]: audit 2023-10-01T10:52:18.185344+0000 mon.a (mon.0) 1284 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:52:19.122 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:18 smithi173 bash[58816]: audit 2023-10-01T10:52:18.690335+0000 mon.a (mon.0) 1285 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:52:19.123 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:18 smithi173 bash[58816]: audit 2023-10-01T10:52:18.697468+0000 mon.a (mon.0) 1286 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:52:19.206 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T10:52:19.211 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:19.253 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:19.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:19.258 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:52:19.261 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 58 16 80110 80094 86.2978 16 0.0368912 0.0115816 2023-10-01T10:52:19.359 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:19.381 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:19.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:19.383 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:52:19.484 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:19.518 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:19.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:19.524 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:52:19.625 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:19.655 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:19.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:19.660 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:52:19.761 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:19.792 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:19.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:19.795 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:52:19.895 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:19.920 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:19.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:19.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:19.950 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:19 smithi078 bash[78274]: cluster 2023-10-01T10:52:18.945981+0000 osd.7 (osd.7) 138 : cluster 3 failed to encode map e182 with expected crc 2023-10-01T10:52:19.950 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:19 smithi078 bash[78274]: cluster 2023-10-01T10:52:18.948078+0000 osd.6 (osd.6) 102 : cluster 3 failed to encode map e182 with expected crc 2023-10-01T10:52:19.950 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:19 smithi078 bash[78274]: cluster 2023-10-01T10:52:18.948238+0000 osd.5 (osd.5) 120 : cluster 3 failed to encode map e182 with expected crc 2023-10-01T10:52:19.950 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:19 smithi078 bash[78274]: cluster 2023-10-01T10:52:18.948957+0000 mon.a (mon.0) 1287 : cluster 0 osdmap e182: 8 total, 8 up, 8 in 2023-10-01T10:52:19.950 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:19 smithi078 bash[78274]: cluster 2023-10-01T10:52:18.952084+0000 osd.6 (osd.6) 103 : cluster 3 failed to encode map e182 with expected crc 2023-10-01T10:52:20.024 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:20.050 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:20.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:20.058 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:52:20.159 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:20.182 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:20.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:20.185 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:52:20.261 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 59 16 80992 80976 85.7694 55.125 0.00921892 0.0116539 2023-10-01T10:52:20.268 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:19 smithi078 bash[75299]: cluster 2023-10-01T10:52:18.945981+0000 osd.7 (osd.7) 138 : cluster 3 failed to encode map e182 with expected crc 2023-10-01T10:52:20.268 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:19 smithi078 bash[75299]: cluster 2023-10-01T10:52:18.948078+0000 osd.6 (osd.6) 102 : cluster 3 failed to encode map e182 with expected crc 2023-10-01T10:52:20.269 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:19 smithi078 bash[75299]: cluster 2023-10-01T10:52:18.948238+0000 osd.5 (osd.5) 120 : cluster 3 failed to encode map e182 with expected crc 2023-10-01T10:52:20.269 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:19 smithi078 bash[75299]: cluster 2023-10-01T10:52:18.948957+0000 mon.a (mon.0) 1287 : cluster 0 osdmap e182: 8 total, 8 up, 8 in 2023-10-01T10:52:20.269 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:19 smithi078 bash[75299]: cluster 2023-10-01T10:52:18.952084+0000 osd.6 (osd.6) 103 : cluster 3 failed to encode map e182 with expected crc 2023-10-01T10:52:20.269 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:19 smithi078 bash[75299]: cluster 2023-10-01T10:52:18.952197+0000 osd.5 (osd.5) 121 : cluster 3 failed to encode map e182 with expected crc 2023-10-01T10:52:20.269 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:19 smithi078 bash[75299]: cluster 2023-10-01T10:52:18.953875+0000 osd.5 (osd.5) 122 : cluster 3 failed to encode map e182 with expected crc 2023-10-01T10:52:20.269 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:19 smithi078 bash[75299]: cluster 2023-10-01T10:52:18.955710+0000 osd.7 (osd.7) 139 : cluster 3 failed to encode map e182 with expected crc 2023-10-01T10:52:20.269 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:19 smithi078 bash[75299]: audit 2023-10-01T10:52:19.219242+0000 mon.a (mon.0) 1288 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:52:20.269 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:19 smithi078 bash[75299]: audit 2023-10-01T10:52:19.223818+0000 mon.a (mon.0) 1289 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:52:20.269 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:19 smithi078 bash[75299]: audit 2023-10-01T10:52:19.231011+0000 mon.a (mon.0) 1290 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:52:20.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:19 smithi078 bash[78274]: cluster 2023-10-01T10:52:18.952197+0000 osd.5 (osd.5) 121 : cluster 3 failed to encode map e182 with expected crc 2023-10-01T10:52:20.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:19 smithi078 bash[78274]: cluster 2023-10-01T10:52:18.953875+0000 osd.5 (osd.5) 122 : cluster 3 failed to encode map e182 with expected crc 2023-10-01T10:52:20.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:19 smithi078 bash[78274]: cluster 2023-10-01T10:52:18.955710+0000 osd.7 (osd.7) 139 : cluster 3 failed to encode map e182 with expected crc 2023-10-01T10:52:20.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:19 smithi078 bash[78274]: audit 2023-10-01T10:52:19.219242+0000 mon.a (mon.0) 1288 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:52:20.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:19 smithi078 bash[78274]: audit 2023-10-01T10:52:19.223818+0000 mon.a (mon.0) 1289 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:52:20.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:19 smithi078 bash[78274]: audit 2023-10-01T10:52:19.231011+0000 mon.a (mon.0) 1290 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:52:20.270 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:52:20 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:52:20] "GET /metrics HTTP/1.1" 200 36313 "" "Prometheus/2.33.4" 2023-10-01T10:52:20.287 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:20.296 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:19 smithi173 bash[58816]: cluster 2023-10-01T10:52:18.945981+0000 osd.7 (osd.7) 138 : cluster 3 failed to encode map e182 with expected crc 2023-10-01T10:52:20.296 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:19 smithi173 bash[58816]: cluster 2023-10-01T10:52:18.948078+0000 osd.6 (osd.6) 102 : cluster 3 failed to encode map e182 with expected crc 2023-10-01T10:52:20.296 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:19 smithi173 bash[58816]: cluster 2023-10-01T10:52:18.948238+0000 osd.5 (osd.5) 120 : cluster 3 failed to encode map e182 with expected crc 2023-10-01T10:52:20.296 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:19 smithi173 bash[58816]: cluster 2023-10-01T10:52:18.948957+0000 mon.a (mon.0) 1287 : cluster 0 osdmap e182: 8 total, 8 up, 8 in 2023-10-01T10:52:20.296 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:19 smithi173 bash[58816]: cluster 2023-10-01T10:52:18.952084+0000 osd.6 (osd.6) 103 : cluster 3 failed to encode map e182 with expected crc 2023-10-01T10:52:20.296 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:19 smithi173 bash[58816]: cluster 2023-10-01T10:52:18.952197+0000 osd.5 (osd.5) 121 : cluster 3 failed to encode map e182 with expected crc 2023-10-01T10:52:20.296 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:19 smithi173 bash[58816]: cluster 2023-10-01T10:52:18.953875+0000 osd.5 (osd.5) 122 : cluster 3 failed to encode map e182 with expected crc 2023-10-01T10:52:20.296 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:19 smithi173 bash[58816]: cluster 2023-10-01T10:52:18.955710+0000 osd.7 (osd.7) 139 : cluster 3 failed to encode map e182 with expected crc 2023-10-01T10:52:20.297 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:19 smithi173 bash[58816]: audit 2023-10-01T10:52:19.219242+0000 mon.a (mon.0) 1288 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:52:20.297 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:19 smithi173 bash[58816]: audit 2023-10-01T10:52:19.223818+0000 mon.a (mon.0) 1289 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:52:20.297 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:19 smithi173 bash[58816]: audit 2023-10-01T10:52:19.231011+0000 mon.a (mon.0) 1290 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:52:20.319 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:20.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:20.323 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:20.423 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:20.459 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:20.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:20.462 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:20.564 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:20.587 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:20.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:20.589 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:52:20.690 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:20.720 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:20.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:20.723 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:20.824 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:20.858 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:20.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:20.865 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:52:20.963 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:20 smithi173 bash[58816]: cluster 2023-10-01T10:52:19.966840+0000 mgr.y (mgr.25072) 1043 : cluster 0 pgmap v995: 89 pgs: 4 peering, 85 active+clean; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 56 MiB/s wr, 898 op/s 2023-10-01T10:52:20.966 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:20.998 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:21.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:21.001 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:52:21.102 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:21.135 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:21.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:21.138 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:52:21.213 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:20 smithi078 bash[75299]: cluster 2023-10-01T10:52:19.966840+0000 mgr.y (mgr.25072) 1043 : cluster 0 pgmap v995: 89 pgs: 4 peering, 85 active+clean; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 56 MiB/s wr, 898 op/s 2023-10-01T10:52:21.214 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:20 smithi078 bash[78274]: cluster 2023-10-01T10:52:19.966840+0000 mgr.y (mgr.25072) 1043 : cluster 0 pgmap v995: 89 pgs: 4 peering, 85 active+clean; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 56 MiB/s wr, 898 op/s 2023-10-01T10:52:21.238 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:21.261 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:2023-10-01T10:52:21.261101+0000 min lat: 0.00119944 max lat: 1.04899 avg lat: 0.0117021 2023-10-01T10:52:21.261 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T10:52:21.261 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 60 16 82025 82009 85.4159 64.5625 0.00979876 0.0117021 2023-10-01T10:52:21.266 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:21.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:21.268 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:21.369 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:21.397 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:21.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:21.402 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:52:21.502 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:21.526 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:21.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:21.528 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:21.629 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:21.667 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:21.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:21.669 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:21.770 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:21.795 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:21.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:21.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:52:21.899 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:21.934 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:21.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:21.941 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:52:22.042 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:22.056 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:21 smithi173 bash[58816]: cluster 2023-10-01T10:52:20.945126+0000 mon.a (mon.0) 1291 : cluster 3 Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-10-01T10:52:22.076 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:22.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:22.081 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:22.182 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:22.221 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:22.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:22.227 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:22.262 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 61 16 83007 82991 85.0216 61.375 0.0187577 0.0117567 2023-10-01T10:52:22.328 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:22.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:21 smithi078 bash[75299]: cluster 2023-10-01T10:52:20.945126+0000 mon.a (mon.0) 1291 : cluster 3 Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-10-01T10:52:22.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:21 smithi078 bash[78274]: cluster 2023-10-01T10:52:20.945126+0000 mon.a (mon.0) 1291 : cluster 3 Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-10-01T10:52:22.356 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:22.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:22.365 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:52:22.467 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:22.495 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:22.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:22.498 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:52:22.599 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:22.633 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:22.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:22.638 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:52:22.739 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:22.777 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:22.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:22.778 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:52:22.879 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:22.905 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:22.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:22.910 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:52:23.011 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:23.035 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:23.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:23.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:52:23.067 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T10:52:23.144 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:23.176 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:23.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:23.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:52:23.193 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T10:52:23.261 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 62 16 83919 83903 84.5696 57 0.00835349 0.0118186 2023-10-01T10:52:23.280 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:23.319 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:23.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:23.321 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:52:23.422 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:23.459 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:23.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:23.464 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:52:23.564 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:23.589 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:23.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:23.594 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:52:23.695 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:23.724 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:23.726 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:23 smithi078 bash[78274]: cluster 2023-10-01T10:52:21.967368+0000 mgr.y (mgr.25072) 1044 : cluster 0 pgmap v996: 89 pgs: 4 peering, 85 active+clean; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 62 MiB/s wr, 998 op/s; 8.0 KiB/s, 0 objects/s recovering 2023-10-01T10:52:23.726 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:23 smithi078 bash[75299]: cluster 2023-10-01T10:52:21.967368+0000 mgr.y (mgr.25072) 1044 : cluster 0 pgmap v996: 89 pgs: 4 peering, 85 active+clean; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 62 MiB/s wr, 998 op/s; 8.0 KiB/s, 0 objects/s recovering 2023-10-01T10:52:23.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:23.727 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:52:23.828 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:23.837 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:23 smithi173 bash[58816]: cluster 2023-10-01T10:52:21.967368+0000 mgr.y (mgr.25072) 1044 : cluster 0 pgmap v996: 89 pgs: 4 peering, 85 active+clean; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 62 MiB/s wr, 998 op/s; 8.0 KiB/s, 0 objects/s recovering 2023-10-01T10:52:23.852 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:23.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:23.854 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:52:23.955 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:23.985 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:23.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:23.988 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:52:24.089 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:24.118 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:24.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:24.122 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:52:24.223 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:24.253 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:24.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:24.256 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:52:24.261 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 63 16 84828 84812 84.129 56.8125 0.0210825 0.0118806 2023-10-01T10:52:24.291 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-10-01T10:52:24.291 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T10:52:24.356 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:24.388 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:24.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:24.389 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:24.490 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:24.514 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:24.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:24.519 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:24.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:24 smithi173 bash[58816]: cluster 2023-10-01T10:52:24.512744+0000 mon.a (mon.0) 1292 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-10-01T10:52:24.620 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:24.645 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:24.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:24.651 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:52:24.751 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:24.777 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:24.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:24.782 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:24.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:24 smithi078 bash[78274]: cluster 2023-10-01T10:52:24.512744+0000 mon.a (mon.0) 1292 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-10-01T10:52:24.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:24 smithi078 bash[75299]: cluster 2023-10-01T10:52:24.512744+0000 mon.a (mon.0) 1292 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-10-01T10:52:24.883 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:24.910 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:24.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:24.912 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:52:25.012 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:25.038 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:25.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:25.040 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:25.141 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:25.168 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:25.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:25.172 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:52:25.261 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 64 16 85796 85780 83.7597 60.5 0.0146915 0.0119337 2023-10-01T10:52:25.273 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:25.302 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:25.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:25.305 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:52:25.406 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:25.432 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:25.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:25.439 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:52:25.487 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-01T10:52:25.539 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:25.568 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:25.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:25.570 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:52:25.571 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:25 smithi078 bash[75299]: cluster 2023-10-01T10:52:23.968696+0000 mgr.y (mgr.25072) 1045 : cluster 0 pgmap v997: 89 pgs: 89 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 813 op/s; 8.0 KiB/s, 0 objects/s recovering 2023-10-01T10:52:25.672 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:25.703 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:25.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:25.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:25.810 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:25.821 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:25 smithi078 bash[78274]: cluster 2023-10-01T10:52:23.968696+0000 mgr.y (mgr.25072) 1045 : cluster 0 pgmap v997: 89 pgs: 89 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 813 op/s; 8.0 KiB/s, 0 objects/s recovering 2023-10-01T10:52:25.843 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:25.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:25.845 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:52:25.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:25 smithi173 bash[58816]: cluster 2023-10-01T10:52:23.968696+0000 mgr.y (mgr.25072) 1045 : cluster 0 pgmap v997: 89 pgs: 89 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 51 MiB/s wr, 813 op/s; 8.0 KiB/s, 0 objects/s recovering 2023-10-01T10:52:25.945 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:25.982 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:25.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:25.987 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:52:26.088 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:26.115 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:26.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:26.117 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:26.214 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T10:52:26.214 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T10:52:26.214 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-01T10:52:26.214 INFO:tasks.thrashosds.thrasher:Setting osd 7 primary_affinity to 0.302250 2023-10-01T10:52:26.215 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 7 0.3022501332390072 2023-10-01T10:52:26.218 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:26.262 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 65 16 86698 86682 83.3383 56.375 0.0105338 0.0119932 2023-10-01T10:52:26.302 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:26.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:26.306 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:52:26.406 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:26.451 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:26.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:26.457 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:52:26.557 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:26.588 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:26.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:26.593 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:52:26.694 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:26.719 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:26.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:26.725 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:26.826 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:26.854 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:26.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:26.857 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:52:26.957 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:26.980 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:26.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:26.983 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:52:27.084 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:27.120 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:27.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:27.124 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:52:27.226 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:27.258 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:27.261 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 66 16 87597 87581 82.9269 56.1875 0.0116557 0.0120531 2023-10-01T10:52:27.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:27.263 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:52:27.363 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:27.395 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:27.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:27.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:52:27.500 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:27.526 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:27.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:27.529 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:52:27.576 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:27 smithi078 bash[78274]: cluster 2023-10-01T10:52:25.969430+0000 mgr.y (mgr.25072) 1046 : cluster 0 pgmap v998: 89 pgs: 89 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 50 MiB/s wr, 804 op/s; 8.0 KiB/s, 0 objects/s recovering 2023-10-01T10:52:27.576 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:27 smithi078 bash[75299]: cluster 2023-10-01T10:52:25.969430+0000 mgr.y (mgr.25072) 1046 : cluster 0 pgmap v998: 89 pgs: 89 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 50 MiB/s wr, 804 op/s; 8.0 KiB/s, 0 objects/s recovering 2023-10-01T10:52:27.632 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:27.659 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:27.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:27.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:52:27.764 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:27.774 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:27 smithi173 bash[58816]: cluster 2023-10-01T10:52:25.969430+0000 mgr.y (mgr.25072) 1046 : cluster 0 pgmap v998: 89 pgs: 89 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 50 MiB/s wr, 804 op/s; 8.0 KiB/s, 0 objects/s recovering 2023-10-01T10:52:27.794 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:27.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:27.799 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:27.899 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:27.930 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:27.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:27.933 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:27.957 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:52:27.958 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:52:28.034 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:28.076 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:28.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:28.084 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:52:28.185 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:28.211 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:28.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:28.216 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:52:28.262 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 67 16 88512 88496 82.5426 57.1875 0.020963 0.0121096 2023-10-01T10:52:28.316 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:28.345 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:28.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:28.347 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:52:28.449 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:28.486 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:28.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:28.491 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:52:28.592 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:28.632 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:28.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:28.635 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:52:28.736 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:28.877 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:28.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:28.879 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:52:28.980 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:29.009 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:29.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:29.014 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:52:29.115 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:29.142 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:29.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:29.149 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:52:29.250 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:29.262 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 68 16 89418 89402 82.1614 56.625 0.00916731 0.0121657 2023-10-01T10:52:29.282 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:29.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:29.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:29.387 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:29.417 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:29.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:29.421 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:52:29.522 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:29.553 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:29.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:29.559 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:29.659 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:29.745 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:29.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:29.746 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:52:29.847 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:29.856 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:29 smithi078 bash[75299]: cluster 2023-10-01T10:52:27.970052+0000 mgr.y (mgr.25072) 1047 : cluster 0 pgmap v999: 89 pgs: 89 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 812 op/s; 6.4 KiB/s, 0 objects/s recovering 2023-10-01T10:52:29.856 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:29 smithi078 bash[78274]: cluster 2023-10-01T10:52:27.970052+0000 mgr.y (mgr.25072) 1047 : cluster 0 pgmap v999: 89 pgs: 89 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 812 op/s; 6.4 KiB/s, 0 objects/s recovering 2023-10-01T10:52:29.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:29 smithi173 bash[58816]: cluster 2023-10-01T10:52:27.970052+0000 mgr.y (mgr.25072) 1047 : cluster 0 pgmap v999: 89 pgs: 89 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 812 op/s; 6.4 KiB/s, 0 objects/s recovering 2023-10-01T10:52:29.873 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:29.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:29.876 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:52:29.977 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:30.007 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:30.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:30.015 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:52:30.115 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:30.154 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:30.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:30.159 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:52:30.260 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:30.262 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 69 16 90328 90312 81.7948 56.875 0.0244234 0.0122201 2023-10-01T10:52:30.269 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:52:30 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:52:30] "GET /metrics HTTP/1.1" 200 36313 "" "Prometheus/2.33.4" 2023-10-01T10:52:30.283 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:30.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:30.285 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:52:30.326 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T10:52:30.386 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:30.418 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:30.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:30.424 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:52:30.524 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:30.546 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:30.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:30.548 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:52:30.649 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:30.676 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:30.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:30.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:30.779 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:30.804 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:30.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:30.811 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:30.911 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:30.939 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:30.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:30.942 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:52:31.012 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:52:31.043 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:31.070 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:31.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:31.074 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:52:31.174 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:31.215 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:31.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:31.217 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:52:31.262 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 70 16 91235 91219 81.4361 56.6875 0.00495728 0.0122738 2023-10-01T10:52:31.318 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:31.344 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:31.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:31.346 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:52:31.447 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:31.474 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:31.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:31.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:52:31.579 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:31.590 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:31 smithi173 bash[58816]: cluster 2023-10-01T10:52:29.970683+0000 mgr.y (mgr.25072) 1048 : cluster 0 pgmap v1000: 89 pgs: 89 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 59 MiB/s wr, 939 op/s; 5.8 KiB/s, 0 objects/s recovering 2023-10-01T10:52:31.611 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:31.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:31.614 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:52:31.714 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:31.723 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:31 smithi078 bash[78274]: cluster 2023-10-01T10:52:29.970683+0000 mgr.y (mgr.25072) 1048 : cluster 0 pgmap v1000: 89 pgs: 89 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 59 MiB/s wr, 939 op/s; 5.8 KiB/s, 0 objects/s recovering 2023-10-01T10:52:31.723 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:31 smithi078 bash[75299]: cluster 2023-10-01T10:52:29.970683+0000 mgr.y (mgr.25072) 1048 : cluster 0 pgmap v1000: 89 pgs: 89 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 59 MiB/s wr, 939 op/s; 5.8 KiB/s, 0 objects/s recovering 2023-10-01T10:52:31.740 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:31.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:31.743 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:52:31.844 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:31.874 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:31.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:31.878 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:52:31.979 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:32.002 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:32.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:32.006 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:32.107 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:32.136 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:32.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:32.142 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:52:32.243 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:32.262 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 71 16 92209 92193 81.1464 60.875 0.0135537 0.012318 2023-10-01T10:52:32.275 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:32.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:32.279 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:52:32.379 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:32.404 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:32.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:32.406 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:32.507 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:32.531 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:32.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:32.533 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:52:32.597 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:32 smithi078 bash[75299]: audit 2023-10-01T10:52:32.434786+0000 mon.a (mon.0) 1293 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:52:32.634 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:32.659 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:32.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:32.662 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:52:32.763 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:32.771 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:32 smithi173 bash[58816]: audit 2023-10-01T10:52:32.434786+0000 mon.a (mon.0) 1293 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:52:32.787 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:32.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:32.789 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:52:32.889 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:32.915 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:32.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:32.919 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:52:33.020 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:33.047 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:32 smithi078 bash[78274]: audit 2023-10-01T10:52:32.434786+0000 mon.a (mon.0) 1293 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:52:33.048 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:33.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:33.052 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:52:33.153 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:33.178 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:33.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:33.180 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:33.262 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 72 16 93050 93034 80.7493 52.5625 0.00581203 0.0123605 2023-10-01T10:52:33.281 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:33.311 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:33.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:33.318 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:52:33.419 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:33.566 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:33.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:33.570 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:52:33.601 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:33 smithi173 bash[58816]: cluster 2023-10-01T10:52:31.971211+0000 mgr.y (mgr.25072) 1049 : cluster 0 pgmap v1001: 89 pgs: 89 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 55 MiB/s wr, 874 op/s; 5.3 KiB/s, 0 objects/s recovering 2023-10-01T10:52:33.601 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:33 smithi173 bash[58816]: audit 2023-10-01T10:52:32.909321+0000 mon.a (mon.0) 1294 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:52:33.670 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:33.699 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:33.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:33.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:52:33.806 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:33.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:33 smithi078 bash[78274]: cluster 2023-10-01T10:52:31.971211+0000 mgr.y (mgr.25072) 1049 : cluster 0 pgmap v1001: 89 pgs: 89 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 55 MiB/s wr, 874 op/s; 5.3 KiB/s, 0 objects/s recovering 2023-10-01T10:52:33.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:33 smithi078 bash[78274]: audit 2023-10-01T10:52:32.909321+0000 mon.a (mon.0) 1294 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:52:33.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:33 smithi078 bash[75299]: cluster 2023-10-01T10:52:31.971211+0000 mgr.y (mgr.25072) 1049 : cluster 0 pgmap v1001: 89 pgs: 89 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 55 MiB/s wr, 874 op/s; 5.3 KiB/s, 0 objects/s recovering 2023-10-01T10:52:33.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:33 smithi078 bash[75299]: audit 2023-10-01T10:52:32.909321+0000 mon.a (mon.0) 1294 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:52:33.832 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:33.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:33.833 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:52:33.934 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:33.975 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:33.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:33.978 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:52:34.079 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:34.118 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:34.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:34.123 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:52:34.224 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:34.246 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:34.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:34.251 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:52:34.262 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 73 16 93955 93939 80.418 56.5625 0.0106021 0.0124291 2023-10-01T10:52:34.353 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:34.386 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:34.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:34.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:52:34.494 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:34.533 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:34.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:34.539 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:52:34.640 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:34.672 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:34.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:34.676 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:52:34.777 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:34.805 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:34.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:34.814 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:52:34.914 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:34.945 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:34.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:34.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:52:35.052 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:35.084 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:35.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:35.087 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:52:35.188 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:35.217 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:35.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:35.225 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:52:35.262 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 74 16 94768 94752 80.0178 50.8125 0.00679836 0.0124919 2023-10-01T10:52:35.284 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-10-01T10:52:35.284 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T10:52:35.326 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:35.351 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:35.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:35.358 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:52:35.459 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:35.481 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:35.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:35.482 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:52:35.583 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:35.607 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:35.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:35.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:52:35.710 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:35.717 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:35 smithi078 bash[78274]: cluster 2023-10-01T10:52:33.971602+0000 mgr.y (mgr.25072) 1050 : cluster 0 pgmap v1002: 89 pgs: 89 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 52 MiB/s wr, 835 op/s 2023-10-01T10:52:35.718 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:35 smithi078 bash[75299]: cluster 2023-10-01T10:52:33.971602+0000 mgr.y (mgr.25072) 1050 : cluster 0 pgmap v1002: 89 pgs: 89 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 52 MiB/s wr, 835 op/s 2023-10-01T10:52:35.733 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:35.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:35.735 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:52:35.837 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:35.848 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:35 smithi173 bash[58816]: cluster 2023-10-01T10:52:33.971602+0000 mgr.y (mgr.25072) 1050 : cluster 0 pgmap v1002: 89 pgs: 89 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 52 MiB/s wr, 835 op/s 2023-10-01T10:52:35.861 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:35.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:35.864 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:35.965 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:35.996 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:36.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:36.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:52:36.103 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:36.130 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:36.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:36.135 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:52:36.205 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-01T10:52:36.236 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:36.262 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 75 16 95511 95495 79.57 46.4375 0.0126473 0.0125614 2023-10-01T10:52:36.281 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:36.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:36.289 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:52:36.389 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:36.420 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:36.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:36.423 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:52:36.524 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:36.554 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:36.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:36.556 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:52:36.657 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:36.681 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:36.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:36.683 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:36.784 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:36.821 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:36.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:36.825 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:52:36.926 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:36.951 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:36.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:36.953 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:37.054 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:37.080 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:37.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:37.085 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:52:37.185 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:37.217 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:37.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:37.226 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:37.263 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 76 16 96390 96374 79.2459 54.9375 0.0125205 0.0126135 2023-10-01T10:52:37.327 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:37.358 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:37.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:37.367 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:52:37.468 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:37.504 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:37.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:37.509 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:37.610 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:37.637 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:37.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:37.646 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:52:37.747 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:37.776 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:37.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:37.779 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:52:37.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:37 smithi173 bash[58816]: cluster 2023-10-01T10:52:35.972352+0000 mgr.y (mgr.25072) 1051 : cluster 0 pgmap v1003: 89 pgs: 89 active+clean; 5.7 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 59 MiB/s wr, 940 op/s 2023-10-01T10:52:37.880 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:37.889 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:37 smithi078 bash[75299]: cluster 2023-10-01T10:52:35.972352+0000 mgr.y (mgr.25072) 1051 : cluster 0 pgmap v1003: 89 pgs: 89 active+clean; 5.7 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 59 MiB/s wr, 940 op/s 2023-10-01T10:52:37.889 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:37 smithi078 bash[78274]: cluster 2023-10-01T10:52:35.972352+0000 mgr.y (mgr.25072) 1051 : cluster 0 pgmap v1003: 89 pgs: 89 active+clean; 5.7 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 59 MiB/s wr, 940 op/s 2023-10-01T10:52:37.907 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:37.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:37.908 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:52:38.009 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:38.042 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:38.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:38.049 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:52:38.150 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:38.170 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:38.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:38.175 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:52:38.263 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 77 16 97252 97236 78.9163 53.875 0.00576563 0.0126663 2023-10-01T10:52:38.276 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:38.301 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:38.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:38.309 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:52:38.410 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:38.437 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:38.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:38.443 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:52:38.544 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:38.575 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:38.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:38.579 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:52:38.679 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:38.711 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:38.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:38.718 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:52:38.819 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:38.852 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:38.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:38.854 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:52:38.955 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:38.989 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:38.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:38.994 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:52:39.095 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:39.133 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:39.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:39.138 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:52:39.239 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:39.263 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 78 16 98117 98101 78.5976 54.0625 0.0144295 0.012717 2023-10-01T10:52:39.266 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:39.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:39.274 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:52:39.375 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:39.401 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:39.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:39.403 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:52:39.504 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:39.535 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:39.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:39.540 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:52:39.640 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:39.693 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:39.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:39.698 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:52:39.765 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:39 smithi078 bash[78274]: cluster 2023-10-01T10:52:37.972927+0000 mgr.y (mgr.25072) 1052 : cluster 0 pgmap v1004: 89 pgs: 89 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 56 MiB/s wr, 893 op/s 2023-10-01T10:52:39.765 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:39 smithi078 bash[75299]: cluster 2023-10-01T10:52:37.972927+0000 mgr.y (mgr.25072) 1052 : cluster 0 pgmap v1004: 89 pgs: 89 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 56 MiB/s wr, 893 op/s 2023-10-01T10:52:39.799 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:39.808 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:39 smithi173 bash[58816]: cluster 2023-10-01T10:52:37.972927+0000 mgr.y (mgr.25072) 1052 : cluster 0 pgmap v1004: 89 pgs: 89 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 56 MiB/s wr, 893 op/s 2023-10-01T10:52:39.826 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:39.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:39.834 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:39.935 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:39.968 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:39.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:39.971 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:52:40.072 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:40.103 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:40.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:40.110 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:52:40.211 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:40.255 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:40.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:40.259 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:52:40.263 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 79 16 99024 99008 78.3202 56.6875 0.0108066 0.0127626 2023-10-01T10:52:40.359 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:40.389 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:40.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:40.392 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:52:40.492 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:40.527 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:40.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:40.531 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:52:40.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:52:40 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:52:40] "GET /metrics HTTP/1.1" 200 36318 "" "Prometheus/2.33.4" 2023-10-01T10:52:40.632 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:40.653 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:40.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:40.656 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:52:40.757 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:40.789 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:40.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:40.793 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:52:40.894 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:40.923 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:40.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:40.928 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:52:41.029 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:41.050 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T10:52:41.054 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:41.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:41.060 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:52:41.160 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:41.192 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:41.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:41.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:52:41.263 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:2023-10-01T10:52:41.263001+0000 min lat: 0.00119944 max lat: 1.04899 avg lat: 0.0128122 2023-10-01T10:52:41.263 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T10:52:41.263 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 80 16 99889 99873 78.017 54.0625 0.0166257 0.0128122 2023-10-01T10:52:41.296 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:41.325 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:41.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:41.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:52:41.428 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:41.460 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:41.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:41.463 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:52:41.564 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:41.588 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:41.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:41.590 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:52:41.691 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:41.701 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:41 smithi078 bash[78274]: cluster 2023-10-01T10:52:39.973613+0000 mgr.y (mgr.25072) 1053 : cluster 0 pgmap v1005: 89 pgs: 89 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 61 MiB/s wr, 972 op/s 2023-10-01T10:52:41.701 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:41 smithi078 bash[75299]: cluster 2023-10-01T10:52:39.973613+0000 mgr.y (mgr.25072) 1053 : cluster 0 pgmap v1005: 89 pgs: 89 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 61 MiB/s wr, 972 op/s 2023-10-01T10:52:41.728 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:41.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:41.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:52:41.835 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:41.856 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:41.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:41.859 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:52:41.859 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:41 smithi173 bash[58816]: cluster 2023-10-01T10:52:39.973613+0000 mgr.y (mgr.25072) 1053 : cluster 0 pgmap v1005: 89 pgs: 89 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 61 MiB/s wr, 972 op/s 2023-10-01T10:52:41.959 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:41.986 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:41.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:41.991 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:52:42.091 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:42.120 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:42.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:42.122 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:52:42.223 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:42.257 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:42.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:42.258 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:52:42.263 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 81 16 100580 100564 77.5869 43.1875 0.0206955 0.0128832 2023-10-01T10:52:42.269 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T10:52:42.298 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T10:52:42.329 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T10:52:42.359 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:42.370 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T10:52:42.392 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:42.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:42.395 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:52:42.405 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T10:52:42.443 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T10:52:42.491 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T10:52:42.496 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:42.526 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T10:52:42.528 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:42.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:42.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:52:42.534 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T10:52:42.534 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T10:52:42.534 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T10:52:42.534 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T10:52:42.534 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T10:52:42.534 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T10:52:42.534 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T10:52:42.534 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T10:52:42.633 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:42.656 INFO:teuthology.orchestra.run.smithi078.stderr:nodeep-scrub is unset 2023-10-01T10:52:42.662 INFO:teuthology.orchestra.run.smithi078.stderr:set osd.7 primary-affinity to 0.30225 (8198082) 2023-10-01T10:52:42.673 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:42.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:42.678 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:52:42.779 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:42.793 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:42 smithi078 bash[78274]: audit 2023-10-01T10:52:41.885064+0000 mon.c (mon.2) 175 : audit 1 from='client.? 172.21.15.78:0/2304631211' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:52:42.793 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:42 smithi078 bash[78274]: audit 2023-10-01T10:52:41.885675+0000 mon.a (mon.0) 1295 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:52:42.793 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:42 smithi078 bash[78274]: audit 2023-10-01T10:52:42.070634+0000 mon.c (mon.2) 176 : audit 1 from='client.? 172.21.15.78:0/1708459263' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.3022501332390072}]: dispatch 2023-10-01T10:52:42.793 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:42 smithi078 bash[78274]: audit 2023-10-01T10:52:42.071502+0000 mon.a (mon.0) 1296 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.3022501332390072}]: dispatch 2023-10-01T10:52:42.794 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:42 smithi078 bash[78274]: audit 2023-10-01T10:52:42.232805+0000 mon.c (mon.2) 177 : audit 0 from='client.? 172.21.15.78:0/3764062410' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:52:42.794 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:42 smithi078 bash[75299]: audit 2023-10-01T10:52:41.885064+0000 mon.c (mon.2) 175 : audit 1 from='client.? 172.21.15.78:0/2304631211' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:52:42.794 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:42 smithi078 bash[75299]: audit 2023-10-01T10:52:41.885675+0000 mon.a (mon.0) 1295 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:52:42.794 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:42 smithi078 bash[75299]: audit 2023-10-01T10:52:42.070634+0000 mon.c (mon.2) 176 : audit 1 from='client.? 172.21.15.78:0/1708459263' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.3022501332390072}]: dispatch 2023-10-01T10:52:42.795 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:42 smithi078 bash[75299]: audit 2023-10-01T10:52:42.071502+0000 mon.a (mon.0) 1296 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.3022501332390072}]: dispatch 2023-10-01T10:52:42.795 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:42 smithi078 bash[75299]: audit 2023-10-01T10:52:42.232805+0000 mon.c (mon.2) 177 : audit 0 from='client.? 172.21.15.78:0/3764062410' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:52:42.805 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:42.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:42.808 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:52:42.909 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:42.929 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:42 smithi173 bash[58816]: audit 2023-10-01T10:52:41.885064+0000 mon.c (mon.2) 175 : audit 1 from='client.? 172.21.15.78:0/2304631211' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:52:42.929 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:42 smithi173 bash[58816]: audit 2023-10-01T10:52:41.885675+0000 mon.a (mon.0) 1295 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:52:42.929 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:42 smithi173 bash[58816]: audit 2023-10-01T10:52:42.070634+0000 mon.c (mon.2) 176 : audit 1 from='client.? 172.21.15.78:0/1708459263' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.3022501332390072}]: dispatch 2023-10-01T10:52:42.930 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:42 smithi173 bash[58816]: audit 2023-10-01T10:52:42.071502+0000 mon.a (mon.0) 1296 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.3022501332390072}]: dispatch 2023-10-01T10:52:42.930 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:42 smithi173 bash[58816]: audit 2023-10-01T10:52:42.232805+0000 mon.c (mon.2) 177 : audit 0 from='client.? 172.21.15.78:0/3764062410' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:52:42.948 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:42.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:42.949 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:52:43.050 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:43.096 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:43.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:43.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:52:43.199 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:43.226 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:43.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:43.229 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:52:43.265 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 82 16 100948 100932 76.9212 23 0.00599445 0.0129073 2023-10-01T10:52:43.330 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:43.366 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:43.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:43.368 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:52:43.469 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:43.499 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:43.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:43.501 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:52:43.602 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:43.635 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:43.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:43.638 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:52:43.702 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:43 smithi078 bash[75299]: cluster 2023-10-01T10:52:41.974062+0000 mgr.y (mgr.25072) 1054 : cluster 0 pgmap v1006: 89 pgs: 89 active+clean; 6.0 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 54 MiB/s wr, 866 op/s 2023-10-01T10:52:43.702 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:43 smithi078 bash[75299]: cluster 2023-10-01T10:52:42.622832+0000 mon.a (mon.0) 1297 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T10:52:43.702 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:43 smithi078 bash[75299]: cluster 2023-10-01T10:52:42.622916+0000 mon.a (mon.0) 1298 : cluster 1 Cluster is now healthy 2023-10-01T10:52:43.702 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:43 smithi078 bash[75299]: cluster 2023-10-01T10:52:42.636627+0000 osd.5 (osd.5) 123 : cluster 3 failed to encode map e183 with expected crc 2023-10-01T10:52:43.702 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:43 smithi078 bash[75299]: audit 2023-10-01T10:52:42.639542+0000 mon.a (mon.0) 1299 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T10:52:43.702 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:43 smithi078 bash[75299]: audit 2023-10-01T10:52:42.639669+0000 mon.a (mon.0) 1300 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.3022501332390072}]': finished 2023-10-01T10:52:43.703 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:43 smithi078 bash[75299]: cluster 2023-10-01T10:52:42.639813+0000 mon.a (mon.0) 1301 : cluster 0 osdmap e183: 8 total, 8 up, 8 in 2023-10-01T10:52:43.703 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:43 smithi078 bash[75299]: cluster 2023-10-01T10:52:42.640285+0000 osd.6 (osd.6) 104 : cluster 3 failed to encode map e183 with expected crc 2023-10-01T10:52:43.703 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:43 smithi078 bash[75299]: cluster 2023-10-01T10:52:42.641500+0000 osd.7 (osd.7) 140 : cluster 3 failed to encode map e183 with expected crc 2023-10-01T10:52:43.703 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:43 smithi078 bash[75299]: cluster 2023-10-01T10:52:42.644967+0000 osd.7 (osd.7) 141 : cluster 3 failed to encode map e183 with expected crc 2023-10-01T10:52:43.739 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:43.751 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:43 smithi173 bash[58816]: cluster 2023-10-01T10:52:41.974062+0000 mgr.y (mgr.25072) 1054 : cluster 0 pgmap v1006: 89 pgs: 89 active+clean; 6.0 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 54 MiB/s wr, 866 op/s 2023-10-01T10:52:43.751 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:43 smithi173 bash[58816]: cluster 2023-10-01T10:52:42.622832+0000 mon.a (mon.0) 1297 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T10:52:43.751 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:43 smithi173 bash[58816]: cluster 2023-10-01T10:52:42.622916+0000 mon.a (mon.0) 1298 : cluster 1 Cluster is now healthy 2023-10-01T10:52:43.751 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:43 smithi173 bash[58816]: cluster 2023-10-01T10:52:42.636627+0000 osd.5 (osd.5) 123 : cluster 3 failed to encode map e183 with expected crc 2023-10-01T10:52:43.751 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:43 smithi173 bash[58816]: audit 2023-10-01T10:52:42.639542+0000 mon.a (mon.0) 1299 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T10:52:43.751 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:43 smithi173 bash[58816]: audit 2023-10-01T10:52:42.639669+0000 mon.a (mon.0) 1300 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.3022501332390072}]': finished 2023-10-01T10:52:43.751 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:43 smithi173 bash[58816]: cluster 2023-10-01T10:52:42.639813+0000 mon.a (mon.0) 1301 : cluster 0 osdmap e183: 8 total, 8 up, 8 in 2023-10-01T10:52:43.751 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:43 smithi173 bash[58816]: cluster 2023-10-01T10:52:42.640285+0000 osd.6 (osd.6) 104 : cluster 3 failed to encode map e183 with expected crc 2023-10-01T10:52:43.751 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:43 smithi173 bash[58816]: cluster 2023-10-01T10:52:42.641500+0000 osd.7 (osd.7) 140 : cluster 3 failed to encode map e183 with expected crc 2023-10-01T10:52:43.752 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:43 smithi173 bash[58816]: cluster 2023-10-01T10:52:42.644967+0000 osd.7 (osd.7) 141 : cluster 3 failed to encode map e183 with expected crc 2023-10-01T10:52:43.771 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:43.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:43.775 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:43.876 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:43.904 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:43.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:43.908 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:52:44.008 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:44.024 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:43 smithi078 bash[78274]: cluster 2023-10-01T10:52:41.974062+0000 mgr.y (mgr.25072) 1054 : cluster 0 pgmap v1006: 89 pgs: 89 active+clean; 6.0 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 54 MiB/s wr, 866 op/s 2023-10-01T10:52:44.024 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:43 smithi078 bash[78274]: cluster 2023-10-01T10:52:42.622832+0000 mon.a (mon.0) 1297 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T10:52:44.024 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:43 smithi078 bash[78274]: cluster 2023-10-01T10:52:42.622916+0000 mon.a (mon.0) 1298 : cluster 1 Cluster is now healthy 2023-10-01T10:52:44.024 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:43 smithi078 bash[78274]: cluster 2023-10-01T10:52:42.636627+0000 osd.5 (osd.5) 123 : cluster 3 failed to encode map e183 with expected crc 2023-10-01T10:52:44.024 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:43 smithi078 bash[78274]: audit 2023-10-01T10:52:42.639542+0000 mon.a (mon.0) 1299 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T10:52:44.024 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:43 smithi078 bash[78274]: audit 2023-10-01T10:52:42.639669+0000 mon.a (mon.0) 1300 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.3022501332390072}]': finished 2023-10-01T10:52:44.024 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:43 smithi078 bash[78274]: cluster 2023-10-01T10:52:42.639813+0000 mon.a (mon.0) 1301 : cluster 0 osdmap e183: 8 total, 8 up, 8 in 2023-10-01T10:52:44.024 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:43 smithi078 bash[78274]: cluster 2023-10-01T10:52:42.640285+0000 osd.6 (osd.6) 104 : cluster 3 failed to encode map e183 with expected crc 2023-10-01T10:52:44.024 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:43 smithi078 bash[78274]: cluster 2023-10-01T10:52:42.641500+0000 osd.7 (osd.7) 140 : cluster 3 failed to encode map e183 with expected crc 2023-10-01T10:52:44.024 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:43 smithi078 bash[78274]: cluster 2023-10-01T10:52:42.644967+0000 osd.7 (osd.7) 141 : cluster 3 failed to encode map e183 with expected crc 2023-10-01T10:52:44.034 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:44.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:44.038 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:52:44.138 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:44.191 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:44.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:44.202 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:52:44.263 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 83 16 101450 101434 76.3724 31.375 0.0216192 0.0130881 2023-10-01T10:52:44.303 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:44.329 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:44.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:44.336 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:44.436 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:44.463 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:44.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:44.469 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:52:44.570 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:44.600 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:44.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:44.604 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:52:44.705 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:44.715 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:44 smithi173 bash[58816]: cluster 2023-10-01T10:52:43.647079+0000 mon.a (mon.0) 1302 : cluster 0 osdmap e184: 8 total, 8 up, 8 in 2023-10-01T10:52:44.715 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:44 smithi173 bash[58816]: cluster 2023-10-01T10:52:43.648094+0000 osd.5 (osd.5) 124 : cluster 3 failed to encode map e184 with expected crc 2023-10-01T10:52:44.715 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:44 smithi173 bash[58816]: cluster 2023-10-01T10:52:43.648257+0000 osd.6 (osd.6) 105 : cluster 3 failed to encode map e184 with expected crc 2023-10-01T10:52:44.715 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:44 smithi173 bash[58816]: cluster 2023-10-01T10:52:43.650624+0000 osd.6 (osd.6) 106 : cluster 3 failed to encode map e184 with expected crc 2023-10-01T10:52:44.715 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:44 smithi173 bash[58816]: cluster 2023-10-01T10:52:43.651376+0000 osd.7 (osd.7) 142 : cluster 3 failed to encode map e184 with expected crc 2023-10-01T10:52:44.715 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:44 smithi173 bash[58816]: cluster 2023-10-01T10:52:43.655732+0000 osd.7 (osd.7) 143 : cluster 3 failed to encode map e184 with expected crc 2023-10-01T10:52:44.741 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:44.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:44.745 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:44.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:44 smithi078 bash[78274]: cluster 2023-10-01T10:52:43.647079+0000 mon.a (mon.0) 1302 : cluster 0 osdmap e184: 8 total, 8 up, 8 in 2023-10-01T10:52:44.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:44 smithi078 bash[78274]: cluster 2023-10-01T10:52:43.648094+0000 osd.5 (osd.5) 124 : cluster 3 failed to encode map e184 with expected crc 2023-10-01T10:52:44.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:44 smithi078 bash[78274]: cluster 2023-10-01T10:52:43.648257+0000 osd.6 (osd.6) 105 : cluster 3 failed to encode map e184 with expected crc 2023-10-01T10:52:44.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:44 smithi078 bash[78274]: cluster 2023-10-01T10:52:43.650624+0000 osd.6 (osd.6) 106 : cluster 3 failed to encode map e184 with expected crc 2023-10-01T10:52:44.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:44 smithi078 bash[78274]: cluster 2023-10-01T10:52:43.651376+0000 osd.7 (osd.7) 142 : cluster 3 failed to encode map e184 with expected crc 2023-10-01T10:52:44.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:44 smithi078 bash[78274]: cluster 2023-10-01T10:52:43.655732+0000 osd.7 (osd.7) 143 : cluster 3 failed to encode map e184 with expected crc 2023-10-01T10:52:44.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:44 smithi078 bash[75299]: cluster 2023-10-01T10:52:43.647079+0000 mon.a (mon.0) 1302 : cluster 0 osdmap e184: 8 total, 8 up, 8 in 2023-10-01T10:52:44.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:44 smithi078 bash[75299]: cluster 2023-10-01T10:52:43.648094+0000 osd.5 (osd.5) 124 : cluster 3 failed to encode map e184 with expected crc 2023-10-01T10:52:44.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:44 smithi078 bash[75299]: cluster 2023-10-01T10:52:43.648257+0000 osd.6 (osd.6) 105 : cluster 3 failed to encode map e184 with expected crc 2023-10-01T10:52:44.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:44 smithi078 bash[75299]: cluster 2023-10-01T10:52:43.650624+0000 osd.6 (osd.6) 106 : cluster 3 failed to encode map e184 with expected crc 2023-10-01T10:52:44.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:44 smithi078 bash[75299]: cluster 2023-10-01T10:52:43.651376+0000 osd.7 (osd.7) 142 : cluster 3 failed to encode map e184 with expected crc 2023-10-01T10:52:44.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:44 smithi078 bash[75299]: cluster 2023-10-01T10:52:43.655732+0000 osd.7 (osd.7) 143 : cluster 3 failed to encode map e184 with expected crc 2023-10-01T10:52:44.846 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:44.883 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:44.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:44.887 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:52:44.987 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:45.027 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:45.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:45.031 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:52:45.132 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:45.217 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:45.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:45.221 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:52:45.263 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 84 16 102323 102307 76.1127 54.5625 0.0112273 0.0131324 2023-10-01T10:52:45.322 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:45.348 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:45.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:45.351 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:52:45.452 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:45.486 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:45.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:45.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:45.592 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:45.617 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:45.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:45.620 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:52:45.658 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-10-01T10:52:45.658 INFO:teuthology.orchestra.run.smithi173.stdout: "size": 20, 2023-10-01T10:52:45.658 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 600, 2023-10-01T10:52:45.658 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [ 2023-10-01T10:52:45.658 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.658 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.45818.0:40395 10.f 10:fd14ddf1:::benchmark_data_smithi078_135100_object40394:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e155)", 2023-10-01T10:52:45.658 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:46:34.564619+0000", 2023-10-01T10:52:45.658 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 371.08617075900003, 2023-10-01T10:52:45.658 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.15826623100000001, 2023-10-01T10:52:45.659 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:52:45.659 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:52:45.659 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:52:45.659 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.45818", 2023-10-01T10:52:45.659 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/1151403832", 2023-10-01T10:52:45.659 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 40395 2023-10-01T10:52:45.659 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.659 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:52:45.659 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.659 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:52:45.659 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:34.564619+0000", 2023-10-01T10:52:45.659 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:52:45.660 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.660 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.660 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:52:45.660 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:34.564619+0000", 2023-10-01T10:52:45.660 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-10-01T10:52:45.660 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.660 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.660 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:52:45.660 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:34.564617+0000", 2023-10-01T10:52:45.660 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.9567000000000003e-05 2023-10-01T10:52:45.660 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.660 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.661 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:52:45.661 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:34.564657+0000", 2023-10-01T10:52:45.661 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.3528e-05 2023-10-01T10:52:45.661 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.661 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.661 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:52:45.661 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:34.564671+0000", 2023-10-01T10:52:45.661 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.4959999999999999e-06 2023-10-01T10:52:45.661 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.661 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.661 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:52:45.661 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:34.564678+0000", 2023-10-01T10:52:45.662 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.2753999999999997e-05 2023-10-01T10:52:45.662 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.662 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.662 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:52:45.662 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:34.564731+0000", 2023-10-01T10:52:45.662 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00013365 2023-10-01T10:52:45.662 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.662 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.662 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:52:45.662 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:34.564864+0000", 2023-10-01T10:52:45.662 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00018103999999999999 2023-10-01T10:52:45.663 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.663 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.663 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 2,3", 2023-10-01T10:52:45.663 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:34.565045+0000", 2023-10-01T10:52:45.663 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0037431140000000001 2023-10-01T10:52:45.663 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.663 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.663 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:52:45.663 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:34.568789+0000", 2023-10-01T10:52:45.663 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0021277319999999998 2023-10-01T10:52:45.663 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.663 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.664 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:52:45.664 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:34.570916+0000", 2023-10-01T10:52:45.664 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.15166995799999999 2023-10-01T10:52:45.664 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.664 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.664 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:52:45.664 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:34.722586+0000", 2023-10-01T10:52:45.664 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00016752299999999999 2023-10-01T10:52:45.664 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.664 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.664 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:52:45.664 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:34.722754+0000", 2023-10-01T10:52:45.665 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00013125499999999999 2023-10-01T10:52:45.665 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.665 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.665 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:52:45.665 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:34.722885+0000", 2023-10-01T10:52:45.665 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.15826623100000001 2023-10-01T10:52:45.665 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:45.665 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:52:45.665 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:45.665 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.665 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.666 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.45818.0:40399 10.f 10:f608ea2b:::benchmark_data_smithi078_135100_object40398:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e155)", 2023-10-01T10:52:45.666 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:46:34.567012+0000", 2023-10-01T10:52:45.666 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 371.08377761600002, 2023-10-01T10:52:45.666 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.15933244899999999, 2023-10-01T10:52:45.666 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:52:45.666 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:52:45.666 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:52:45.666 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.45818", 2023-10-01T10:52:45.666 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/1151403832", 2023-10-01T10:52:45.666 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 40399 2023-10-01T10:52:45.666 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.667 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:52:45.667 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.667 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:52:45.667 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:34.567012+0000", 2023-10-01T10:52:45.667 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:52:45.667 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.667 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.667 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:52:45.667 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:34.567012+0000", 2023-10-01T10:52:45.667 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999948 2023-10-01T10:52:45.667 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.667 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.668 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:52:45.668 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:34.567007+0000", 2023-10-01T10:52:45.668 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.8414e-05 2023-10-01T10:52:45.668 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.668 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.668 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:52:45.668 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:34.567065+0000", 2023-10-01T10:52:45.668 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.3709e-05 2023-10-01T10:52:45.668 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.668 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.668 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:52:45.669 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:34.567079+0000", 2023-10-01T10:52:45.669 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.6229999999999997e-06 2023-10-01T10:52:45.669 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.669 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.669 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:52:45.670 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:34.567087+0000", 2023-10-01T10:52:45.670 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0017310870000000001 2023-10-01T10:52:45.670 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.670 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.670 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:52:45.670 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:34.568818+0000", 2023-10-01T10:52:45.670 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00016505799999999999 2023-10-01T10:52:45.670 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.670 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.670 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:52:45.670 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:34.568983+0000", 2023-10-01T10:52:45.670 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000220516 2023-10-01T10:52:45.670 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.671 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.671 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 2,3", 2023-10-01T10:52:45.671 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:34.569203+0000", 2023-10-01T10:52:45.671 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0017540730000000001 2023-10-01T10:52:45.671 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.671 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.671 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:52:45.671 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:34.570957+0000", 2023-10-01T10:52:45.671 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00017533300000000001 2023-10-01T10:52:45.671 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.671 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.671 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:52:45.672 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:34.571133+0000", 2023-10-01T10:52:45.672 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.15503409400000001 2023-10-01T10:52:45.672 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.672 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.672 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:52:45.672 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:34.726167+0000", 2023-10-01T10:52:45.672 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000101578 2023-10-01T10:52:45.672 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.672 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.672 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:52:45.672 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:34.726268+0000", 2023-10-01T10:52:45.672 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.6181999999999997e-05 2023-10-01T10:52:45.673 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.673 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.673 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:52:45.673 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:34.726344+0000", 2023-10-01T10:52:45.673 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.15933244899999999 2023-10-01T10:52:45.673 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:45.673 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:52:45.673 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:45.673 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.673 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.673 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.45818.0:54048 10.a 10:568e2248:::benchmark_data_smithi078_135100_object54047:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e156)", 2023-10-01T10:52:45.673 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:46:44.818323+0000", 2023-10-01T10:52:45.674 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 360.83246665500002, 2023-10-01T10:52:45.674 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.162339487, 2023-10-01T10:52:45.674 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:52:45.674 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:52:45.674 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:52:45.674 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.45818", 2023-10-01T10:52:45.674 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/1151403832", 2023-10-01T10:52:45.674 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 54048 2023-10-01T10:52:45.674 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.674 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:52:45.674 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.674 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:52:45.675 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:44.818323+0000", 2023-10-01T10:52:45.675 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:52:45.675 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.675 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.675 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:52:45.675 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:44.818323+0000", 2023-10-01T10:52:45.675 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-10-01T10:52:45.675 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.675 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.675 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:52:45.675 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:44.818322+0000", 2023-10-01T10:52:45.675 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.7429999999999997e-05 2023-10-01T10:52:45.676 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.676 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.676 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:52:45.676 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:44.818380+0000", 2023-10-01T10:52:45.676 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0757999999999999e-05 2023-10-01T10:52:45.676 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.676 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.676 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:52:45.676 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:44.818390+0000", 2023-10-01T10:52:45.676 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.7189999999999998e-06 2023-10-01T10:52:45.676 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.676 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.677 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:52:45.677 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:44.818396+0000", 2023-10-01T10:52:45.677 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0039145059999999999 2023-10-01T10:52:45.677 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.677 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.677 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:52:45.677 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:44.822311+0000", 2023-10-01T10:52:45.677 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00013184500000000001 2023-10-01T10:52:45.677 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.677 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.677 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:52:45.677 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:44.822442+0000", 2023-10-01T10:52:45.678 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0020845350000000002 2023-10-01T10:52:45.678 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.678 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.678 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 0,5", 2023-10-01T10:52:45.678 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:44.824527+0000", 2023-10-01T10:52:45.678 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0021300809999999998 2023-10-01T10:52:45.678 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.678 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.678 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:52:45.678 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:44.826657+0000", 2023-10-01T10:52:45.678 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00198738 2023-10-01T10:52:45.678 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.679 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.679 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:52:45.679 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:44.828644+0000", 2023-10-01T10:52:45.679 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.15183876800000001 2023-10-01T10:52:45.679 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.679 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.679 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:52:45.679 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:44.980483+0000", 2023-10-01T10:52:45.679 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000104491 2023-10-01T10:52:45.679 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.679 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.679 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:52:45.680 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:44.980588+0000", 2023-10-01T10:52:45.680 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.4754999999999998e-05 2023-10-01T10:52:45.680 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.680 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.680 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:52:45.680 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:44.980662+0000", 2023-10-01T10:52:45.680 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.162339487 2023-10-01T10:52:45.680 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:45.680 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:52:45.680 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:45.680 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.680 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.681 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.45818.0:54051 10.f 10:f692b50f:::benchmark_data_smithi078_135100_object54050:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e156)", 2023-10-01T10:52:45.681 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:46:44.823199+0000", 2023-10-01T10:52:45.681 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 360.82759055399998, 2023-10-01T10:52:45.681 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.15797988600000001, 2023-10-01T10:52:45.681 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:52:45.681 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:52:45.681 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:52:45.681 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.45818", 2023-10-01T10:52:45.681 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/1151403832", 2023-10-01T10:52:45.681 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 54051 2023-10-01T10:52:45.681 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.681 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:52:45.681 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.682 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:52:45.682 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:44.823199+0000", 2023-10-01T10:52:45.682 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:52:45.682 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.682 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.682 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:52:45.682 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:44.823199+0000", 2023-10-01T10:52:45.682 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-10-01T10:52:45.682 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.682 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.682 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:52:45.682 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:44.823197+0000", 2023-10-01T10:52:45.683 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.6746000000000001e-05 2023-10-01T10:52:45.683 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.683 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.683 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:52:45.683 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:44.823264+0000", 2023-10-01T10:52:45.683 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1559000000000001e-05 2023-10-01T10:52:45.683 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.683 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.683 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:52:45.683 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:44.823275+0000", 2023-10-01T10:52:45.683 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.1599999999999997e-06 2023-10-01T10:52:45.683 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.684 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.684 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:52:45.684 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:44.823281+0000", 2023-10-01T10:52:45.684 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0074047719999999996 2023-10-01T10:52:45.684 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.684 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.684 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:52:45.684 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:44.830685+0000", 2023-10-01T10:52:45.684 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00013793099999999999 2023-10-01T10:52:45.684 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.684 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.684 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:52:45.685 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:44.830823+0000", 2023-10-01T10:52:45.685 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00016590800000000001 2023-10-01T10:52:45.685 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.685 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.685 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 2,3", 2023-10-01T10:52:45.685 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:44.830989+0000", 2023-10-01T10:52:45.685 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0071129990000000001 2023-10-01T10:52:45.685 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.685 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.685 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:52:45.685 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:44.838102+0000", 2023-10-01T10:52:45.685 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.2657999999999997e-05 2023-10-01T10:52:45.685 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.686 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.686 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:52:45.686 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:44.838145+0000", 2023-10-01T10:52:45.686 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.14290087900000001 2023-10-01T10:52:45.686 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.686 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.686 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:52:45.686 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:44.981046+0000", 2023-10-01T10:52:45.686 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.7082000000000005e-05 2023-10-01T10:52:45.686 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.686 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.686 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:52:45.687 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:44.981123+0000", 2023-10-01T10:52:45.687 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.6125000000000002e-05 2023-10-01T10:52:45.687 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.687 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.687 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:52:45.687 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:46:44.981179+0000", 2023-10-01T10:52:45.687 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.15797988600000001 2023-10-01T10:52:45.687 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:45.687 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:52:45.687 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:45.687 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.687 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.688 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.45818.0:92529 10.f 10:fba54a62:::benchmark_data_smithi078_135100_object92528:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-10-01T10:52:45.688 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:47:22.366082+0000", 2023-10-01T10:52:45.688 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 323.28470780499998, 2023-10-01T10:52:45.688 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.19421096500000001, 2023-10-01T10:52:45.688 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:52:45.688 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:52:45.688 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:52:45.688 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.45818", 2023-10-01T10:52:45.688 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/1151403832", 2023-10-01T10:52:45.688 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 92529 2023-10-01T10:52:45.688 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.688 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:52:45.689 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.689 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:52:45.689 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.366082+0000", 2023-10-01T10:52:45.689 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:52:45.689 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.689 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.689 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:52:45.689 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.366082+0000", 2023-10-01T10:52:45.689 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-10-01T10:52:45.689 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.689 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.689 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:52:45.690 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.366080+0000", 2023-10-01T10:52:45.690 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.8968000000000002e-05 2023-10-01T10:52:45.690 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.690 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.690 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:52:45.690 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.366139+0000", 2023-10-01T10:52:45.690 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.0355000000000001e-05 2023-10-01T10:52:45.690 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.690 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.690 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:52:45.690 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.366159+0000", 2023-10-01T10:52:45.691 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.5800000000000003e-06 2023-10-01T10:52:45.691 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.691 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.691 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:52:45.691 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.366167+0000", 2023-10-01T10:52:45.691 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.1838e-05 2023-10-01T10:52:45.691 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.691 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.691 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:52:45.691 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.366229+0000", 2023-10-01T10:52:45.691 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000157336 2023-10-01T10:52:45.691 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.692 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.692 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:52:45.692 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.366386+0000", 2023-10-01T10:52:45.692 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00018444699999999999 2023-10-01T10:52:45.692 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.692 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.692 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 2,3", 2023-10-01T10:52:45.692 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.366570+0000", 2023-10-01T10:52:45.692 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0064647230000000003 2023-10-01T10:52:45.692 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.692 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.692 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:52:45.693 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.373035+0000", 2023-10-01T10:52:45.693 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00029585800000000003 2023-10-01T10:52:45.693 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.693 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.693 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:52:45.693 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.373331+0000", 2023-10-01T10:52:45.693 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.18677656400000001 2023-10-01T10:52:45.693 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.693 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.693 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:52:45.693 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.560108+0000", 2023-10-01T10:52:45.694 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000128792 2023-10-01T10:52:45.694 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.694 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.694 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:52:45.694 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.560236+0000", 2023-10-01T10:52:45.694 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.6332000000000002e-05 2023-10-01T10:52:45.694 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.694 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.694 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:52:45.694 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.560293+0000", 2023-10-01T10:52:45.694 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.19421096500000001 2023-10-01T10:52:45.694 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:45.695 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:52:45.695 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:45.695 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.695 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.695 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.45818.0:92538 10.f 10:ff40a27e:::benchmark_data_smithi078_135100_object92537:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-10-01T10:52:45.695 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:47:22.375477+0000", 2023-10-01T10:52:45.695 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 323.275313084, 2023-10-01T10:52:45.695 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.18502510899999999, 2023-10-01T10:52:45.695 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:52:45.695 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:52:45.695 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:52:45.695 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.45818", 2023-10-01T10:52:45.696 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/1151403832", 2023-10-01T10:52:45.696 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 92538 2023-10-01T10:52:45.696 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.696 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:52:45.696 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.696 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:52:45.696 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.375477+0000", 2023-10-01T10:52:45.696 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:52:45.696 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.696 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.696 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:52:45.696 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.375477+0000", 2023-10-01T10:52:45.697 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-10-01T10:52:45.697 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.697 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.697 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:52:45.697 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.375475+0000", 2023-10-01T10:52:45.697 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.9310999999999999e-05 2023-10-01T10:52:45.697 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.697 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.697 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:52:45.697 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.375565+0000", 2023-10-01T10:52:45.697 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.2473999999999999e-05 2023-10-01T10:52:45.697 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.697 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.698 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:52:45.698 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.375587+0000", 2023-10-01T10:52:45.698 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.6760000000000004e-06 2023-10-01T10:52:45.698 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.698 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.698 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:52:45.698 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.375593+0000", 2023-10-01T10:52:45.698 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.1240000000000002e-05 2023-10-01T10:52:45.698 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.698 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.698 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:52:45.699 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.375664+0000", 2023-10-01T10:52:45.699 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00013187600000000001 2023-10-01T10:52:45.699 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.699 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.699 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:52:45.699 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.375796+0000", 2023-10-01T10:52:45.699 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00016366199999999999 2023-10-01T10:52:45.699 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.699 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.699 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 2,3", 2023-10-01T10:52:45.699 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.375960+0000", 2023-10-01T10:52:45.699 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.011772863 2023-10-01T10:52:45.700 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.700 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.700 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:52:45.700 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.387732+0000", 2023-10-01T10:52:45.700 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.9469999999999997e-06 2023-10-01T10:52:45.700 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.700 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.700 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:52:45.700 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.387739+0000", 2023-10-01T10:52:45.700 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.17260742900000001 2023-10-01T10:52:45.700 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.700 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.701 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:52:45.701 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.560347+0000", 2023-10-01T10:52:45.701 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.9650999999999998e-05 2023-10-01T10:52:45.701 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.701 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.701 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:52:45.701 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.560446+0000", 2023-10-01T10:52:45.701 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.5223000000000003e-05 2023-10-01T10:52:45.701 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.701 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.701 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:52:45.701 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.560502+0000", 2023-10-01T10:52:45.702 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.18502510899999999 2023-10-01T10:52:45.702 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:45.702 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:52:45.702 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:45.702 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.702 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.702 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.45818.0:92556 10.f 10:fefa7784:::benchmark_data_smithi078_135100_object92555:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-10-01T10:52:45.702 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:47:22.400766+0000", 2023-10-01T10:52:45.702 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 323.25002339899999, 2023-10-01T10:52:45.702 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.16219724899999999, 2023-10-01T10:52:45.702 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:52:45.702 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:52:45.703 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:52:45.703 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.45818", 2023-10-01T10:52:45.703 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/1151403832", 2023-10-01T10:52:45.703 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 92556 2023-10-01T10:52:45.703 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.703 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:52:45.703 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.703 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:52:45.703 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.400766+0000", 2023-10-01T10:52:45.703 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:52:45.703 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.704 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.704 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:52:45.704 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.400766+0000", 2023-10-01T10:52:45.704 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-10-01T10:52:45.704 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.704 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.704 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:52:45.704 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.400765+0000", 2023-10-01T10:52:45.704 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.4985000000000003e-05 2023-10-01T10:52:45.704 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.704 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.704 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:52:45.705 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.400820+0000", 2023-10-01T10:52:45.705 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1789e-05 2023-10-01T10:52:45.705 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.705 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.705 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:52:45.705 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.400832+0000", 2023-10-01T10:52:45.705 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.7440000000000004e-06 2023-10-01T10:52:45.705 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.705 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.705 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:52:45.705 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.400837+0000", 2023-10-01T10:52:45.705 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.3876999999999999e-05 2023-10-01T10:52:45.706 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.706 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.706 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:52:45.706 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.400891+0000", 2023-10-01T10:52:45.706 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000205532 2023-10-01T10:52:45.706 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.706 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.706 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:52:45.706 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.401097+0000", 2023-10-01T10:52:45.706 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00024609600000000001 2023-10-01T10:52:45.706 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.707 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.707 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 2,3", 2023-10-01T10:52:45.707 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.401343+0000", 2023-10-01T10:52:45.707 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0029969800000000002 2023-10-01T10:52:45.707 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.707 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.707 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:52:45.707 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.404340+0000", 2023-10-01T10:52:45.707 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00045888100000000001 2023-10-01T10:52:45.707 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.707 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.707 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:52:45.708 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.404799+0000", 2023-10-01T10:52:45.708 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.15804105900000001 2023-10-01T10:52:45.708 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.708 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.708 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:52:45.708 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.562840+0000", 2023-10-01T10:52:45.708 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.8532e-05 2023-10-01T10:52:45.708 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.708 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.708 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:52:45.708 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.562908+0000", 2023-10-01T10:52:45.708 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.5034999999999997e-05 2023-10-01T10:52:45.709 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.709 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.709 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:52:45.709 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:47:22.562963+0000", 2023-10-01T10:52:45.709 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.16219724899999999 2023-10-01T10:52:45.709 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:45.709 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:52:45.709 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:45.709 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.709 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.709 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46037.0:10191 11.3 11:c406fcde:::benchmark_data_smithi078_157677_object10190:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e178)", 2023-10-01T10:52:45.709 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:51:25.290935+0000", 2023-10-01T10:52:45.710 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 80.359854932999994, 2023-10-01T10:52:45.710 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.199248963, 2023-10-01T10:52:45.710 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:52:45.710 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:52:45.710 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:52:45.710 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46037", 2023-10-01T10:52:45.710 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:52:45.710 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 10191 2023-10-01T10:52:45.710 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.710 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:52:45.710 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.710 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:52:45.711 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:51:25.290935+0000", 2023-10-01T10:52:45.711 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:52:45.711 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.711 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.711 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:52:45.711 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:51:25.290935+0000", 2023-10-01T10:52:45.711 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-10-01T10:52:45.711 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.711 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.711 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:52:45.711 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:51:25.290933+0000", 2023-10-01T10:52:45.711 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.6320999999999999e-05 2023-10-01T10:52:45.712 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.712 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.712 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:52:45.712 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:51:25.290980+0000", 2023-10-01T10:52:45.712 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1838000000000001e-05 2023-10-01T10:52:45.712 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.712 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.712 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:52:45.712 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:51:25.290991+0000", 2023-10-01T10:52:45.712 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.9170000000000003e-06 2023-10-01T10:52:45.712 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.712 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.712 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:52:45.713 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:51:25.290997+0000", 2023-10-01T10:52:45.713 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.6177e-05 2023-10-01T10:52:45.713 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.713 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.713 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:52:45.713 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:51:25.291064+0000", 2023-10-01T10:52:45.713 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00013519199999999999 2023-10-01T10:52:45.713 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.713 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.713 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:52:45.713 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:51:25.291199+0000", 2023-10-01T10:52:45.713 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00015611100000000001 2023-10-01T10:52:45.714 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.714 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.714 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1,7", 2023-10-01T10:52:45.714 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:51:25.291355+0000", 2023-10-01T10:52:45.714 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00078479400000000001 2023-10-01T10:52:45.714 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.714 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.714 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:52:45.714 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:51:25.292140+0000", 2023-10-01T10:52:45.714 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000207868 2023-10-01T10:52:45.714 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.714 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.715 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:52:45.715 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:51:25.292347+0000", 2023-10-01T10:52:45.715 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.197731135 2023-10-01T10:52:45.715 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.715 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.715 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:52:45.715 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:51:25.490079+0000", 2023-10-01T10:52:45.715 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.5930999999999997e-05 2023-10-01T10:52:45.715 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.715 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.715 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:52:45.715 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:51:25.490135+0000", 2023-10-01T10:52:45.716 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.9097000000000001e-05 2023-10-01T10:52:45.716 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.716 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.716 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:52:45.716 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:51:25.490184+0000", 2023-10-01T10:52:45.716 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.199248963 2023-10-01T10:52:45.716 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:45.716 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:52:45.716 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:45.716 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.716 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.716 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.46037.0:60473 11.a e180/173 11:58f76fa1:::benchmark_data_smithi078_157677_object60472:head v 180'3765, mlcod=180'3763)", 2023-10-01T10:52:45.717 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:51:58.818038+0000", 2023-10-01T10:52:45.717 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 46.832751254000001, 2023-10-01T10:52:45.717 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.21667719399999999, 2023-10-01T10:52:45.717 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:52:45.717 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:52:45.717 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:52:45.717 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.717 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:52:45.717 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:51:58.818038+0000", 2023-10-01T10:52:45.717 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:52:45.717 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.717 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.718 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:52:45.718 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:51:58.818038+0000", 2023-10-01T10:52:45.718 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999995 2023-10-01T10:52:45.718 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.718 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.718 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:52:45.718 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:51:58.818038+0000", 2023-10-01T10:52:45.718 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.6649000000000001e-05 2023-10-01T10:52:45.718 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.718 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.718 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:52:45.718 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:51:58.818075+0000", 2023-10-01T10:52:45.718 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.5320000000000001e-06 2023-10-01T10:52:45.719 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.719 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.719 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:52:45.719 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:51:58.818082+0000", 2023-10-01T10:52:45.719 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.5580000000000002e-06 2023-10-01T10:52:45.719 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.719 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.719 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:52:45.719 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:51:58.818087+0000", 2023-10-01T10:52:45.719 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.1649000000000006e-05 2023-10-01T10:52:45.719 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.719 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.720 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:52:45.720 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:51:58.818148+0000", 2023-10-01T10:52:45.720 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.2026999999999997e-05 2023-10-01T10:52:45.720 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.720 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.720 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:52:45.720 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:51:58.818180+0000", 2023-10-01T10:52:45.720 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.21644777700000001 2023-10-01T10:52:45.720 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.720 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.720 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:52:45.720 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:51:59.034628+0000", 2023-10-01T10:52:45.721 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.7406999999999998e-05 2023-10-01T10:52:45.721 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.721 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.721 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:52:45.721 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:51:59.034716+0000", 2023-10-01T10:52:45.721 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.21667719399999999 2023-10-01T10:52:45.721 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:45.721 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:52:45.721 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:45.721 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.721 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.721 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.46037.0:60477 11.5 e180/173 11:ac2078b8:::benchmark_data_smithi078_157677_object60476:head v 180'3807, mlcod=180'3805)", 2023-10-01T10:52:45.722 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:51:58.819957+0000", 2023-10-01T10:52:45.722 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 46.830832940000001, 2023-10-01T10:52:45.722 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.214752321, 2023-10-01T10:52:45.722 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:52:45.722 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:52:45.722 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:52:45.722 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.722 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:52:45.722 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:51:58.819957+0000", 2023-10-01T10:52:45.722 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:52:45.722 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.722 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.723 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:52:45.723 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:51:58.819957+0000", 2023-10-01T10:52:45.723 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-10-01T10:52:45.723 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.723 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.723 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:52:45.723 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:51:58.819955+0000", 2023-10-01T10:52:45.723 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.9813000000000001e-05 2023-10-01T10:52:45.723 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.723 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.723 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:52:45.723 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:51:58.819985+0000", 2023-10-01T10:52:45.724 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2653999999999999e-05 2023-10-01T10:52:45.724 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.724 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.724 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:52:45.724 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:51:58.819998+0000", 2023-10-01T10:52:45.724 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.6340000000000003e-06 2023-10-01T10:52:45.724 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.724 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.724 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:52:45.724 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:51:58.820005+0000", 2023-10-01T10:52:45.724 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.4379000000000001e-05 2023-10-01T10:52:45.724 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.725 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.725 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:52:45.725 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:51:58.820079+0000", 2023-10-01T10:52:45.725 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.8791e-05 2023-10-01T10:52:45.725 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.725 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.725 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:52:45.725 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:51:58.820098+0000", 2023-10-01T10:52:45.725 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.21455823900000001 2023-10-01T10:52:45.725 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.725 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.725 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:52:45.726 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:51:59.034656+0000", 2023-10-01T10:52:45.726 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.3043e-05 2023-10-01T10:52:45.726 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.726 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.726 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:52:45.726 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:51:59.034709+0000", 2023-10-01T10:52:45.726 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.214752321 2023-10-01T10:52:45.726 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:45.726 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:52:45.726 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:45.726 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.726 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.727 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.46037.0:60476 11.8 e180/173 11:10c172dc:::benchmark_data_smithi078_157677_object60475:head v 180'3820, mlcod=180'3818)", 2023-10-01T10:52:45.727 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:51:58.820741+0000", 2023-10-01T10:52:45.727 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 46.830048136000002, 2023-10-01T10:52:45.727 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.21888052699999999, 2023-10-01T10:52:45.727 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:52:45.727 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:52:45.727 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:52:45.727 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.727 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:52:45.727 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:51:58.820741+0000", 2023-10-01T10:52:45.727 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:52:45.728 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.728 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.728 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:52:45.728 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:51:58.820741+0000", 2023-10-01T10:52:45.728 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-10-01T10:52:45.728 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.728 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.729 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:52:45.729 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:51:58.820741+0000", 2023-10-01T10:52:45.729 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.7154999999999998e-05 2023-10-01T10:52:45.729 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.729 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.729 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:52:45.729 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:51:58.820788+0000", 2023-10-01T10:52:45.729 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.3293e-05 2023-10-01T10:52:45.729 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.729 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.729 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:52:45.729 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:51:58.820801+0000", 2023-10-01T10:52:45.730 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.3130000000000004e-06 2023-10-01T10:52:45.730 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.730 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.730 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:52:45.730 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:51:58.820807+0000", 2023-10-01T10:52:45.730 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00024813999999999999 2023-10-01T10:52:45.730 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.730 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.730 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:52:45.730 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:51:58.821056+0000", 2023-10-01T10:52:45.730 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.9746999999999999e-05 2023-10-01T10:52:45.730 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.731 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.731 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:52:45.731 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:51:58.821075+0000", 2023-10-01T10:52:45.731 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.218500745 2023-10-01T10:52:45.731 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.731 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.731 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:52:45.731 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:51:59.039576+0000", 2023-10-01T10:52:45.731 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.5862999999999998e-05 2023-10-01T10:52:45.731 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.731 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.731 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:52:45.732 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:51:59.039622+0000", 2023-10-01T10:52:45.732 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.21888052699999999 2023-10-01T10:52:45.732 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:45.732 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:52:45.732 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:45.732 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.732 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.732 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.46037.0:60482 11.a e180/173 11:50b10957:::benchmark_data_smithi078_157677_object60481:head v 180'3766, mlcod=180'3763)", 2023-10-01T10:52:45.732 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:51:58.826055+0000", 2023-10-01T10:52:45.732 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 46.824734282000001, 2023-10-01T10:52:45.732 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.218781897, 2023-10-01T10:52:45.733 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:52:45.733 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:52:45.733 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:52:45.733 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.733 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:52:45.733 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:51:58.826055+0000", 2023-10-01T10:52:45.733 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:52:45.733 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.733 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.733 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:52:45.733 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:51:58.826055+0000", 2023-10-01T10:52:45.733 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999952 2023-10-01T10:52:45.733 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.734 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.734 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:52:45.734 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:51:58.826050+0000", 2023-10-01T10:52:45.734 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.3078999999999998e-05 2023-10-01T10:52:45.734 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.734 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.734 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:52:45.734 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:51:58.826103+0000", 2023-10-01T10:52:45.734 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.20852035799999999 2023-10-01T10:52:45.734 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.734 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.735 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:52:45.735 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:51:59.034624+0000", 2023-10-01T10:52:45.735 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4745e-05 2023-10-01T10:52:45.735 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.735 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.735 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:52:45.735 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:51:59.034639+0000", 2023-10-01T10:52:45.735 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00013633699999999999 2023-10-01T10:52:45.735 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.735 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.735 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:52:45.735 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:51:59.034775+0000", 2023-10-01T10:52:45.736 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.1019000000000001e-05 2023-10-01T10:52:45.736 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.736 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.736 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:52:45.736 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:51:59.034826+0000", 2023-10-01T10:52:45.736 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0099642289999999998 2023-10-01T10:52:45.736 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.736 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.736 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:52:45.736 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:51:59.044790+0000", 2023-10-01T10:52:45.736 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.7042000000000001e-05 2023-10-01T10:52:45.736 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.737 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.737 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:52:45.737 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:51:59.044837+0000", 2023-10-01T10:52:45.737 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.218781897 2023-10-01T10:52:45.737 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:45.737 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:52:45.737 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:45.737 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.737 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.737 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46037.0:72067 11.3 11:cb5e1bd9:::benchmark_data_smithi078_157677_object72066:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e180)", 2023-10-01T10:52:45.737 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:52:09.676667+0000", 2023-10-01T10:52:45.737 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 35.974122483999999, 2023-10-01T10:52:45.738 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.28515953500000002, 2023-10-01T10:52:45.738 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:52:45.738 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:52:45.738 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:52:45.738 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46037", 2023-10-01T10:52:45.738 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:52:45.738 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 72067 2023-10-01T10:52:45.738 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.738 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:52:45.738 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.738 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:52:45.738 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:09.676667+0000", 2023-10-01T10:52:45.739 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:52:45.739 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.739 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.739 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:52:45.739 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:09.676667+0000", 2023-10-01T10:52:45.739 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-10-01T10:52:45.739 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.739 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.739 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:52:45.739 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:09.676665+0000", 2023-10-01T10:52:45.739 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.3195999999999997e-05 2023-10-01T10:52:45.740 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.740 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.740 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:52:45.740 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:09.676718+0000", 2023-10-01T10:52:45.740 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.3059000000000001e-05 2023-10-01T10:52:45.740 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.740 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.740 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:52:45.740 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:09.676741+0000", 2023-10-01T10:52:45.740 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0087e-05 2023-10-01T10:52:45.740 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.740 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.740 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:52:45.741 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:09.676751+0000", 2023-10-01T10:52:45.741 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.1897999999999997e-05 2023-10-01T10:52:45.741 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.741 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.741 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:52:45.741 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:09.676803+0000", 2023-10-01T10:52:45.741 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000115622 2023-10-01T10:52:45.741 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.741 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.741 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:52:45.741 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:09.676919+0000", 2023-10-01T10:52:45.742 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00015119299999999999 2023-10-01T10:52:45.742 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.742 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.742 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1,7", 2023-10-01T10:52:45.742 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:09.677070+0000", 2023-10-01T10:52:45.742 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0023375700000000002 2023-10-01T10:52:45.742 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.742 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.742 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:52:45.742 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:09.679408+0000", 2023-10-01T10:52:45.742 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.001669682 2023-10-01T10:52:45.742 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.742 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.743 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:52:45.743 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:09.681077+0000", 2023-10-01T10:52:45.743 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.280660195 2023-10-01T10:52:45.743 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.743 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.743 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:52:45.743 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:09.961738+0000", 2023-10-01T10:52:45.743 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.3776000000000002e-05 2023-10-01T10:52:45.743 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.743 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.743 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:52:45.743 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:09.961781+0000", 2023-10-01T10:52:45.744 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.5343999999999998e-05 2023-10-01T10:52:45.744 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.744 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.744 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:52:45.744 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:09.961827+0000", 2023-10-01T10:52:45.744 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.28515953500000002 2023-10-01T10:52:45.744 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:45.744 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:52:45.744 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:45.744 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.744 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.744 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46037.0:100869 11.4 11:2a1dd82b:::benchmark_data_smithi078_157677_object100868:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e183)", 2023-10-01T10:52:45.745 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:52:42.653757+0000", 2023-10-01T10:52:45.745 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 2.9970330380000001, 2023-10-01T10:52:45.745 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.016734341, 2023-10-01T10:52:45.745 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:52:45.745 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "reached pg", 2023-10-01T10:52:45.745 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:52:45.745 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46037", 2023-10-01T10:52:45.745 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:52:45.745 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 100869 2023-10-01T10:52:45.745 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.745 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:52:45.745 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.746 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:52:45.746 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.653757+0000", 2023-10-01T10:52:45.746 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:52:45.746 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.746 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.746 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:52:45.746 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.653757+0000", 2023-10-01T10:52:45.746 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-10-01T10:52:45.746 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.746 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.746 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:52:45.746 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.653755+0000", 2023-10-01T10:52:45.747 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.2917999999999998e-05 2023-10-01T10:52:45.747 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.747 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.747 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:52:45.747 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.653798+0000", 2023-10-01T10:52:45.747 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1556000000000001e-05 2023-10-01T10:52:45.747 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.747 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.747 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:52:45.747 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.653809+0000", 2023-10-01T10:52:45.747 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.486e-06 2023-10-01T10:52:45.747 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.748 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.748 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:52:45.748 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.653817+0000", 2023-10-01T10:52:45.748 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.8060000000000003e-05 2023-10-01T10:52:45.748 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.748 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.748 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:52:45.748 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.653875+0000", 2023-10-01T10:52:45.748 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0322e-05 2023-10-01T10:52:45.748 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.748 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.748 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T10:52:45.749 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.653885+0000", 2023-10-01T10:52:45.749 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.014193041 2023-10-01T10:52:45.749 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.749 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.749 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:52:45.749 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.668078+0000", 2023-10-01T10:52:45.749 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0024127089999999999 2023-10-01T10:52:45.749 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.749 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.749 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:52:45.749 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.670491+0000", 2023-10-01T10:52:45.750 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.016734341 2023-10-01T10:52:45.750 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:45.750 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:52:45.750 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:45.750 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.750 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.750 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46037.0:100871 11.4 11:2e1c9fed:::benchmark_data_smithi078_157677_object100870:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e183)", 2023-10-01T10:52:45.750 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:52:42.653837+0000", 2023-10-01T10:52:45.750 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 2.9969528840000002, 2023-10-01T10:52:45.750 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0167943660000001, 2023-10-01T10:52:45.751 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:52:45.751 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "reached pg", 2023-10-01T10:52:45.751 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:52:45.751 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46037", 2023-10-01T10:52:45.751 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:52:45.751 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 100871 2023-10-01T10:52:45.751 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.751 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:52:45.751 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.751 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:52:45.751 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.653837+0000", 2023-10-01T10:52:45.751 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:52:45.752 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.752 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.752 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:52:45.752 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.653837+0000", 2023-10-01T10:52:45.752 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-10-01T10:52:45.752 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.752 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.752 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:52:45.752 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.653836+0000", 2023-10-01T10:52:45.752 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.7914999999999999e-05 2023-10-01T10:52:45.753 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.753 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.753 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:52:45.753 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.653874+0000", 2023-10-01T10:52:45.753 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.9150000000000008e-06 2023-10-01T10:52:45.753 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.753 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.753 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:52:45.753 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.653884+0000", 2023-10-01T10:52:45.753 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.6919999999999996e-06 2023-10-01T10:52:45.753 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.753 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.754 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:52:45.754 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.653890+0000", 2023-10-01T10:52:45.754 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00041903099999999999 2023-10-01T10:52:45.754 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.754 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.754 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:52:45.754 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.654309+0000", 2023-10-01T10:52:45.754 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.6250000000000001e-06 2023-10-01T10:52:45.754 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.754 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.755 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T10:52:45.755 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.654316+0000", 2023-10-01T10:52:45.755 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0162162880000001 2023-10-01T10:52:45.755 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.755 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.755 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:52:45.755 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.670532+0000", 2023-10-01T10:52:45.755 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.8696999999999994e-05 2023-10-01T10:52:45.755 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.755 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.755 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:52:45.755 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.670631+0000", 2023-10-01T10:52:45.755 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0167943660000001 2023-10-01T10:52:45.756 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:45.756 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:52:45.756 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:45.756 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.756 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.756 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46037.0:100893 11.4 11:299e35c5:::benchmark_data_smithi078_157677_object100892:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e183)", 2023-10-01T10:52:45.756 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:52:42.679768+0000", 2023-10-01T10:52:45.756 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 2.9710218990000001, 2023-10-01T10:52:45.756 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.015671319, 2023-10-01T10:52:45.756 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:52:45.756 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:52:45.756 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:52:45.757 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46037", 2023-10-01T10:52:45.757 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:52:45.757 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 100893 2023-10-01T10:52:45.757 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.757 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:52:45.757 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.757 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:52:45.757 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.679768+0000", 2023-10-01T10:52:45.757 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:52:45.757 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.757 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.758 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:52:45.758 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.679768+0000", 2023-10-01T10:52:45.758 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999919 2023-10-01T10:52:45.758 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.758 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.758 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:52:45.758 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.679760+0000", 2023-10-01T10:52:45.758 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.3638e-05 2023-10-01T10:52:45.758 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.758 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.758 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:52:45.758 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.679853+0000", 2023-10-01T10:52:45.759 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.8518999999999999e-05 2023-10-01T10:52:45.759 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.759 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.759 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:52:45.759 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.679882+0000", 2023-10-01T10:52:45.759 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.6180000000000001e-05 2023-10-01T10:52:45.759 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.759 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.759 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:52:45.759 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.679898+0000", 2023-10-01T10:52:45.759 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.5018000000000006e-05 2023-10-01T10:52:45.759 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.759 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.760 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:52:45.760 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.679973+0000", 2023-10-01T10:52:45.760 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.3920999999999999e-05 2023-10-01T10:52:45.760 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.760 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.760 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T10:52:45.760 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.679987+0000", 2023-10-01T10:52:45.760 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.990682022 2023-10-01T10:52:45.760 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.760 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.760 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:52:45.761 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.670669+0000", 2023-10-01T10:52:45.761 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00079621800000000001 2023-10-01T10:52:45.761 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.761 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.761 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:52:45.761 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.671465+0000", 2023-10-01T10:52:45.761 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000313391 2023-10-01T10:52:45.761 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.761 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.761 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 2,7", 2023-10-01T10:52:45.761 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.671779+0000", 2023-10-01T10:52:45.761 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.022933056 2023-10-01T10:52:45.762 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.762 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.762 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:52:45.762 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.694712+0000", 2023-10-01T10:52:45.762 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000538264 2023-10-01T10:52:45.762 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.762 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.762 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:52:45.762 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.695250+0000", 2023-10-01T10:52:45.762 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.0857000000000001e-05 2023-10-01T10:52:45.762 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.762 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.763 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:52:45.763 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.695341+0000", 2023-10-01T10:52:45.763 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.5923e-05 2023-10-01T10:52:45.763 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.763 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.763 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:52:45.763 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.695397+0000", 2023-10-01T10:52:45.763 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.2191999999999999e-05 2023-10-01T10:52:45.763 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.763 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.763 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:52:45.763 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.695439+0000", 2023-10-01T10:52:45.763 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.015671319 2023-10-01T10:52:45.764 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:45.764 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:52:45.764 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:45.764 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.764 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.764 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46037.0:100924 11.4 11:2bd98104:::benchmark_data_smithi078_157677_object100923:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e183)", 2023-10-01T10:52:45.764 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:52:42.725788+0000", 2023-10-01T10:52:45.764 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 2.925002095, 2023-10-01T10:52:45.764 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.96978407200000005, 2023-10-01T10:52:45.764 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:52:45.764 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:52:45.765 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:52:45.765 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46037", 2023-10-01T10:52:45.765 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:52:45.765 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 100924 2023-10-01T10:52:45.765 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.765 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:52:45.765 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.765 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:52:45.765 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.725788+0000", 2023-10-01T10:52:45.765 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:52:45.765 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.766 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.766 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:52:45.766 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.725788+0000", 2023-10-01T10:52:45.766 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-10-01T10:52:45.766 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.766 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.766 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:52:45.766 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.725786+0000", 2023-10-01T10:52:45.767 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.6391000000000003e-05 2023-10-01T10:52:45.767 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.767 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.767 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:52:45.767 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.725842+0000", 2023-10-01T10:52:45.767 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2128e-05 2023-10-01T10:52:45.767 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.767 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.767 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:52:45.767 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.725855+0000", 2023-10-01T10:52:45.767 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.9869999999999999e-06 2023-10-01T10:52:45.767 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.768 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.768 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:52:45.768 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.725860+0000", 2023-10-01T10:52:45.768 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.8262999999999993e-05 2023-10-01T10:52:45.768 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.768 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.768 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:52:45.768 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.725928+0000", 2023-10-01T10:52:45.768 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4052e-05 2023-10-01T10:52:45.768 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.768 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.768 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T10:52:45.769 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.725942+0000", 2023-10-01T10:52:45.769 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.95033384700000001 2023-10-01T10:52:45.769 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.769 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.769 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:52:45.769 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.676276+0000", 2023-10-01T10:52:45.769 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000222174 2023-10-01T10:52:45.769 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.769 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.769 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:52:45.769 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.676498+0000", 2023-10-01T10:52:45.769 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000173439 2023-10-01T10:52:45.770 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.770 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.770 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 2,7", 2023-10-01T10:52:45.770 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.676671+0000", 2023-10-01T10:52:45.770 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.018048453999999998 2023-10-01T10:52:45.770 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.770 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.770 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:52:45.770 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.694720+0000", 2023-10-01T10:52:45.770 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00057570600000000005 2023-10-01T10:52:45.770 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.771 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.771 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:52:45.771 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.695295+0000", 2023-10-01T10:52:45.771 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000191282 2023-10-01T10:52:45.771 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.771 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.771 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:52:45.771 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.695487+0000", 2023-10-01T10:52:45.771 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.7954000000000002e-05 2023-10-01T10:52:45.771 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.771 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.771 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:52:45.772 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.695535+0000", 2023-10-01T10:52:45.772 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.6878999999999997e-05 2023-10-01T10:52:45.772 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.772 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.772 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:52:45.772 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.695572+0000", 2023-10-01T10:52:45.772 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.96978407200000005 2023-10-01T10:52:45.772 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:45.772 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:52:45.772 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:45.772 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.772 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.773 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46037.0:100939 11.4 11:2028671a:::benchmark_data_smithi078_157677_object100938:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e183)", 2023-10-01T10:52:45.773 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:52:42.745263+0000", 2023-10-01T10:52:45.773 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 2.9055267320000002, 2023-10-01T10:52:45.773 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.95557181300000005, 2023-10-01T10:52:45.773 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:52:45.773 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:52:45.773 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:52:45.773 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46037", 2023-10-01T10:52:45.773 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:52:45.773 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 100939 2023-10-01T10:52:45.773 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.774 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:52:45.774 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.774 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:52:45.774 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.745263+0000", 2023-10-01T10:52:45.774 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:52:45.774 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.774 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.774 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:52:45.774 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.745263+0000", 2023-10-01T10:52:45.774 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-10-01T10:52:45.774 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.774 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.775 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:52:45.775 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.745262+0000", 2023-10-01T10:52:45.775 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.5864000000000003e-05 2023-10-01T10:52:45.775 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.775 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.775 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:52:45.775 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.745347+0000", 2023-10-01T10:52:45.775 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.5156000000000001e-05 2023-10-01T10:52:45.775 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.775 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.775 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:52:45.775 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.745373+0000", 2023-10-01T10:52:45.776 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.8609999999999993e-06 2023-10-01T10:52:45.776 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.776 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.776 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:52:45.776 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.745382+0000", 2023-10-01T10:52:45.777 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.2941999999999995e-05 2023-10-01T10:52:45.777 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.777 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.777 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:52:45.777 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.745445+0000", 2023-10-01T10:52:45.777 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1527e-05 2023-10-01T10:52:45.777 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.777 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.777 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T10:52:45.777 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.745457+0000", 2023-10-01T10:52:45.778 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.936374763 2023-10-01T10:52:45.778 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.778 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.778 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:52:45.778 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.681832+0000", 2023-10-01T10:52:45.778 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00016292800000000001 2023-10-01T10:52:45.778 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.778 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.778 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:52:45.778 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.681995+0000", 2023-10-01T10:52:45.779 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000195513 2023-10-01T10:52:45.779 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.779 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.779 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 2,7", 2023-10-01T10:52:45.779 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.682190+0000", 2023-10-01T10:52:45.779 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.012611195 2023-10-01T10:52:45.779 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.779 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.779 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:52:45.779 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.694801+0000", 2023-10-01T10:52:45.780 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00080843600000000005 2023-10-01T10:52:45.780 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.780 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.780 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:52:45.780 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.695610+0000", 2023-10-01T10:52:45.780 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0051403439999999998 2023-10-01T10:52:45.780 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.780 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.780 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:52:45.780 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.700750+0000", 2023-10-01T10:52:45.780 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.7506000000000002e-05 2023-10-01T10:52:45.780 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.781 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.781 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:52:45.781 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.700798+0000", 2023-10-01T10:52:45.781 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.7110000000000002e-05 2023-10-01T10:52:45.781 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.781 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.781 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:52:45.781 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.700835+0000", 2023-10-01T10:52:45.781 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.95557181300000005 2023-10-01T10:52:45.781 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:45.782 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:52:45.782 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:45.782 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.782 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.782 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46037.0:100941 11.4 11:2e4290a5:::benchmark_data_smithi078_157677_object100940:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e183)", 2023-10-01T10:52:45.782 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:52:42.748755+0000", 2023-10-01T10:52:45.782 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 2.902034338, 2023-10-01T10:52:45.782 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.95237100900000005, 2023-10-01T10:52:45.782 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:52:45.782 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:52:45.782 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:52:45.782 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46037", 2023-10-01T10:52:45.783 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:52:45.783 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 100941 2023-10-01T10:52:45.783 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.783 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:52:45.783 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.783 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:52:45.783 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.748755+0000", 2023-10-01T10:52:45.783 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:52:45.783 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.783 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.783 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:52:45.783 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.748755+0000", 2023-10-01T10:52:45.784 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999976 2023-10-01T10:52:45.784 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.784 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.784 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:52:45.784 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.748753+0000", 2023-10-01T10:52:45.784 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.1923999999999997e-05 2023-10-01T10:52:45.784 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.784 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.784 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:52:45.784 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.748835+0000", 2023-10-01T10:52:45.784 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4911e-05 2023-10-01T10:52:45.784 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.785 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.785 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:52:45.785 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.748850+0000", 2023-10-01T10:52:45.785 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.1849999999999993e-06 2023-10-01T10:52:45.785 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.785 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.785 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:52:45.785 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.748859+0000", 2023-10-01T10:52:45.785 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.2833000000000001e-05 2023-10-01T10:52:45.785 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.785 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.785 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:52:45.785 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.748912+0000", 2023-10-01T10:52:45.786 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.2600000000000002e-06 2023-10-01T10:52:45.786 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.786 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.786 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T10:52:45.786 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.748918+0000", 2023-10-01T10:52:45.786 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.93701625399999999 2023-10-01T10:52:45.786 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.786 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.786 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:52:45.786 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.685934+0000", 2023-10-01T10:52:45.786 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000163122 2023-10-01T10:52:45.786 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.787 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.787 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:52:45.787 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.686097+0000", 2023-10-01T10:52:45.787 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00021060600000000001 2023-10-01T10:52:45.787 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.787 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.787 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 2,7", 2023-10-01T10:52:45.787 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.686308+0000", 2023-10-01T10:52:45.787 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.010899489 2023-10-01T10:52:45.787 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.787 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.788 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:52:45.788 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.697208+0000", 2023-10-01T10:52:45.788 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.5974000000000001e-05 2023-10-01T10:52:45.788 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.788 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.788 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:52:45.788 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.697284+0000", 2023-10-01T10:52:45.788 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.003714551 2023-10-01T10:52:45.788 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.788 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.788 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:52:45.789 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.700998+0000", 2023-10-01T10:52:45.789 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.8604999999999997e-05 2023-10-01T10:52:45.789 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.789 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.789 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:52:45.789 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.701077+0000", 2023-10-01T10:52:45.789 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.9589000000000002e-05 2023-10-01T10:52:45.789 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.789 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.789 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:52:45.789 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.701126+0000", 2023-10-01T10:52:45.790 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.95237100900000005 2023-10-01T10:52:45.790 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:45.790 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:52:45.790 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:45.790 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.790 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.790 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46037.0:100944 11.4 11:277410f8:::benchmark_data_smithi078_157677_object100943:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e183)", 2023-10-01T10:52:45.791 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:52:42.754063+0000", 2023-10-01T10:52:45.791 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 2.8967269849999999, 2023-10-01T10:52:45.791 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.95059931799999997, 2023-10-01T10:52:45.791 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:52:45.791 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:52:45.791 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:52:45.791 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46037", 2023-10-01T10:52:45.791 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:52:45.791 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 100944 2023-10-01T10:52:45.791 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.791 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:52:45.792 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.792 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:52:45.792 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.754063+0000", 2023-10-01T10:52:45.792 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:52:45.792 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.792 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.792 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:52:45.792 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.754063+0000", 2023-10-01T10:52:45.793 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999976 2023-10-01T10:52:45.793 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.793 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.793 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:52:45.793 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.754060+0000", 2023-10-01T10:52:45.793 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.4354999999999997e-05 2023-10-01T10:52:45.793 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.793 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.793 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:52:45.793 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.754114+0000", 2023-10-01T10:52:45.793 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.8628000000000001e-05 2023-10-01T10:52:45.793 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.794 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.794 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:52:45.794 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.754143+0000", 2023-10-01T10:52:45.794 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4732999999999999e-05 2023-10-01T10:52:45.795 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.795 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.795 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:52:45.795 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.754158+0000", 2023-10-01T10:52:45.795 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.6470999999999994e-05 2023-10-01T10:52:45.795 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.795 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.795 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:52:45.795 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.754224+0000", 2023-10-01T10:52:45.796 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.1769999999999998e-06 2023-10-01T10:52:45.796 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.796 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.796 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T10:52:45.796 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.754232+0000", 2023-10-01T10:52:45.796 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.93633348500000002 2023-10-01T10:52:45.796 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.797 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.797 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:52:45.797 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.690566+0000", 2023-10-01T10:52:45.797 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00014709200000000001 2023-10-01T10:52:45.797 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.797 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.797 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:52:45.797 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.690713+0000", 2023-10-01T10:52:45.797 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000184437 2023-10-01T10:52:45.797 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.798 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.798 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 2,7", 2023-10-01T10:52:45.798 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.690897+0000", 2023-10-01T10:52:45.798 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0065650999999999999 2023-10-01T10:52:45.798 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.798 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.799 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:52:45.799 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.697463+0000", 2023-10-01T10:52:45.799 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00092019599999999997 2023-10-01T10:52:45.799 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.799 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.799 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:52:45.800 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.698383+0000", 2023-10-01T10:52:45.800 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0062149279999999998 2023-10-01T10:52:45.800 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.800 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.800 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:52:45.800 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.704598+0000", 2023-10-01T10:52:45.801 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.7228000000000003e-05 2023-10-01T10:52:45.801 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.801 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.801 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:52:45.801 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.704635+0000", 2023-10-01T10:52:45.801 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.7005999999999999e-05 2023-10-01T10:52:45.802 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:45.802 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:45.802 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:52:45.802 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.704662+0000", 2023-10-01T10:52:45.802 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.95059931799999997 2023-10-01T10:52:45.802 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:45.802 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:52:45.803 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:45.803 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:45.803 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:52:45.803 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-10-01T10:52:45.803 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:45.846 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:45.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:45.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:52:45.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:45 smithi173 bash[58816]: cluster 2023-10-01T10:52:43.974576+0000 mgr.y (mgr.25072) 1055 : cluster 0 pgmap v1009: 89 pgs: 1 peering, 88 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 53 MiB/s wr, 841 op/s 2023-10-01T10:52:45.949 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:45.959 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:45 smithi078 bash[78274]: cluster 2023-10-01T10:52:43.974576+0000 mgr.y (mgr.25072) 1055 : cluster 0 pgmap v1009: 89 pgs: 1 peering, 88 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 53 MiB/s wr, 841 op/s 2023-10-01T10:52:45.959 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:45 smithi078 bash[75299]: cluster 2023-10-01T10:52:43.974576+0000 mgr.y (mgr.25072) 1055 : cluster 0 pgmap v1009: 89 pgs: 1 peering, 88 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 53 MiB/s wr, 841 op/s 2023-10-01T10:52:45.981 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:45.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:45.984 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:52:46.085 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:46.109 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:46.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:46.117 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:52:46.218 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:46.256 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:46.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:46.259 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:52:46.263 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 85 16 103157 103141 75.8305 52.125 0.0149194 0.013181 2023-10-01T10:52:46.360 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:46.392 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:46.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:46.396 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:52:46.497 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:46.532 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:46.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:46.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:52:46.636 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:46.645 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-01T10:52:46.676 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:46.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:46.681 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:52:46.782 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:46.811 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:46.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:46.817 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:52:46.918 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:46.952 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:46.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:46.956 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:52:47.057 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:47.089 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:47.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:47.093 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:52:47.128 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T10:52:47.148 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T10:52:47.194 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:47.233 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:47.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:47.236 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:52:47.264 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 86 16 103963 103947 75.534 50.375 0.0240477 0.0132328 2023-10-01T10:52:47.337 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:47.379 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:47.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:47.383 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:52:47.484 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:47.523 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:47.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:47.525 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:52:47.625 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:47.649 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:47.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:47.655 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:52:47.756 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:47.765 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:47 smithi078 bash[75299]: cluster 2023-10-01T10:52:45.975226+0000 mgr.y (mgr.25072) 1056 : cluster 0 pgmap v1010: 89 pgs: 1 peering, 88 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 51 MiB/s wr, 811 op/s 2023-10-01T10:52:47.765 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:47 smithi078 bash[75299]: audit 2023-10-01T10:52:47.452801+0000 mon.a (mon.0) 1303 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:52:47.766 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:47 smithi078 bash[75299]: audit 2023-10-01T10:52:47.454370+0000 mon.a (mon.0) 1304 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:52:47.766 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:47 smithi078 bash[78274]: cluster 2023-10-01T10:52:45.975226+0000 mgr.y (mgr.25072) 1056 : cluster 0 pgmap v1010: 89 pgs: 1 peering, 88 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 51 MiB/s wr, 811 op/s 2023-10-01T10:52:47.766 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:47 smithi078 bash[78274]: audit 2023-10-01T10:52:47.452801+0000 mon.a (mon.0) 1303 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:52:47.766 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:47 smithi078 bash[78274]: audit 2023-10-01T10:52:47.454370+0000 mon.a (mon.0) 1304 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:52:47.783 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:47.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:47.785 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:52:47.885 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:48.011 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:47 smithi173 bash[58816]: cluster 2023-10-01T10:52:45.975226+0000 mgr.y (mgr.25072) 1056 : cluster 0 pgmap v1010: 89 pgs: 1 peering, 88 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 51 MiB/s wr, 811 op/s 2023-10-01T10:52:48.012 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:47 smithi173 bash[58816]: audit 2023-10-01T10:52:47.452801+0000 mon.a (mon.0) 1303 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:52:48.012 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:47 smithi173 bash[58816]: audit 2023-10-01T10:52:47.454370+0000 mon.a (mon.0) 1304 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:52:48.033 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:48.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:48.037 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:48.139 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:48.176 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:48.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:48.184 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:52:48.264 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 87 16 104758 104742 75.2368 49.6875 0.0169611 0.0132861 2023-10-01T10:52:48.286 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:48.322 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:48.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:48.329 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:52:48.430 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:48.469 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:48.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:48.475 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:52:48.576 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:48.606 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:48.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:48.615 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:52:48.716 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:48.741 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:48.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:48.747 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:52:48.847 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:48.872 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:48.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:48.876 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:52:48.976 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:49.009 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:49.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:49.012 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:52:49.113 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:49.142 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:49.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:49.148 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:49.249 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:49.264 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 88 16 105583 105567 74.9677 51.5625 0.01888 0.0133334 2023-10-01T10:52:49.284 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:49.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:49.290 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:52:49.391 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:49.428 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:49.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:49.432 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:52:49.533 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:49.569 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:49.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:49.576 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:52:49.670 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:49 smithi078 bash[75299]: cluster 2023-10-01T10:52:47.977754+0000 mgr.y (mgr.25072) 1057 : cluster 0 pgmap v1011: 89 pgs: 1 peering, 88 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 39 MiB/s wr, 626 op/s 2023-10-01T10:52:49.670 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:49 smithi078 bash[78274]: cluster 2023-10-01T10:52:47.977754+0000 mgr.y (mgr.25072) 1057 : cluster 0 pgmap v1011: 89 pgs: 1 peering, 88 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 39 MiB/s wr, 626 op/s 2023-10-01T10:52:49.677 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:49.704 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:49.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:49.706 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:52:49.807 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:49.836 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:49.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:49.839 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:52:49.940 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:49.972 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:49.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:49.976 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:52:50.076 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:50.107 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T10:52:50.108 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T10:52:50.108 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2023-10-01T10:52:50.108 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-01T10:52:50.108 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-01T10:52:50.108 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 config set filestore_inject_stall 3 2023-10-01T10:52:50.115 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:50.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:49 smithi173 bash[58816]: cluster 2023-10-01T10:52:47.977754+0000 mgr.y (mgr.25072) 1057 : cluster 0 pgmap v1011: 89 pgs: 1 peering, 88 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 39 MiB/s wr, 626 op/s 2023-10-01T10:52:50.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:50.120 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:52:50.221 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:50.257 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:50.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:50.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:52:50.264 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 89 16 106369 106353 74.6773 49.125 0.0206427 0.0133847 2023-10-01T10:52:50.270 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:52:50 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:52:50] "GET /metrics HTTP/1.1" 200 36313 "" "Prometheus/2.33.4" 2023-10-01T10:52:50.362 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:50.394 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:50.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:50.396 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:52:50.496 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:50.520 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:50.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:50.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:52:50.626 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:50.649 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:50.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:50.652 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:52:50.753 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:50.796 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:50.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:50.800 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:52:50.901 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:50.923 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:50.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:50.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:52:51.029 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:51.067 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:51.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:51.069 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:51.170 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:51.199 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:51.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:51.202 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:52:51.264 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 90 4 107168 107164 74.4107 50.6875 0.0119513 0.0134337 2023-10-01T10:52:51.302 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:51.331 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:51.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:51.334 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:52:51.435 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:51.463 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:51.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:51.466 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:52:51.481 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T10:52:51.566 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:51.600 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:51.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:51.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:52:51.704 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:51.715 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:51 smithi173 bash[58816]: cluster 2023-10-01T10:52:49.978402+0000 mgr.y (mgr.25072) 1058 : cluster 0 pgmap v1012: 89 pgs: 89 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 45 MiB/s wr, 713 op/s; 8.0 KiB/s, 0 objects/s recovering 2023-10-01T10:52:51.727 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:51.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:51.729 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:52:51.830 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:51.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:51 smithi078 bash[75299]: cluster 2023-10-01T10:52:49.978402+0000 mgr.y (mgr.25072) 1058 : cluster 0 pgmap v1012: 89 pgs: 89 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 45 MiB/s wr, 713 op/s; 8.0 KiB/s, 0 objects/s recovering 2023-10-01T10:52:51.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:51 smithi078 bash[78274]: cluster 2023-10-01T10:52:49.978402+0000 mgr.y (mgr.25072) 1058 : cluster 0 pgmap v1012: 89 pgs: 89 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 45 MiB/s wr, 713 op/s; 8.0 KiB/s, 0 objects/s recovering 2023-10-01T10:52:51.865 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:51.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:51.867 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:52:51.967 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:51.999 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:52.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:52.002 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:52.033 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:52:52.034 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:52:52.102 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:52.128 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:52.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:52.131 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:52:52.232 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:52.261 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:52.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:52.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:52:52.265 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Total time run: 90.011 2023-10-01T10:52:52.265 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Total writes made: 107168 2023-10-01T10:52:52.265 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Write size: 65536 2023-10-01T10:52:52.265 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Object size: 65536 2023-10-01T10:52:52.265 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Bandwidth (MB/sec): 74.4131 2023-10-01T10:52:52.266 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Stddev Bandwidth: 29.4438 2023-10-01T10:52:52.266 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Max bandwidth (MB/sec): 160.438 2023-10-01T10:52:52.266 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Min bandwidth (MB/sec): 16 2023-10-01T10:52:52.266 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Average IOPS: 1190 2023-10-01T10:52:52.266 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Stddev IOPS: 471.101 2023-10-01T10:52:52.266 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Max IOPS: 2567 2023-10-01T10:52:52.266 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Min IOPS: 256 2023-10-01T10:52:52.266 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Average Latency(s): 0.0134339 2023-10-01T10:52:52.266 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Stddev Latency(s): 0.0209204 2023-10-01T10:52:52.267 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Max latency(s): 1.05375 2023-10-01T10:52:52.267 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Min latency(s): 0.00119944 2023-10-01T10:52:52.275 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Cleaning up (deleting benchmark objects) 2023-10-01T10:52:52.365 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:52.394 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:52.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:52.400 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:52:52.500 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:52.545 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:52.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:52.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:52:52.652 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:52.683 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:52.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:52.686 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:52:52.787 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:52.828 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:52.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:52.837 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:52:52.937 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:52.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:52.981 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:52.981 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:52:53.082 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:53.108 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:53.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:53.114 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:52:53.214 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:53.253 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:53.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:53.255 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:52:53.356 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:53.380 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:53.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:53.386 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:52:53.487 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:53.518 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:53.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:53.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:52:53.628 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:53.651 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:53.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:53.657 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:53.758 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:53.766 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:53 smithi173 bash[58816]: cluster 2023-10-01T10:52:51.979207+0000 mgr.y (mgr.25072) 1059 : cluster 0 pgmap v1013: 89 pgs: 89 active+clean; 6.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 47 MiB/s wr, 746 op/s; 6.8 KiB/s, 0 objects/s recovering 2023-10-01T10:52:53.797 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:53.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:53.799 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:53.900 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:53.926 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:53.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:53.929 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:52:54.030 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:54.051 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:54.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:54.053 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:54.073 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:52:54.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:53 smithi078 bash[75299]: cluster 2023-10-01T10:52:51.979207+0000 mgr.y (mgr.25072) 1059 : cluster 0 pgmap v1013: 89 pgs: 89 active+clean; 6.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 47 MiB/s wr, 746 op/s; 6.8 KiB/s, 0 objects/s recovering 2023-10-01T10:52:54.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:53 smithi078 bash[78274]: cluster 2023-10-01T10:52:51.979207+0000 mgr.y (mgr.25072) 1059 : cluster 0 pgmap v1013: 89 pgs: 89 active+clean; 6.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 47 MiB/s wr, 746 op/s; 6.8 KiB/s, 0 objects/s recovering 2023-10-01T10:52:54.153 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:54.180 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:54.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:54.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:54.288 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:54.313 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:54.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:54.318 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:52:54.419 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:54.452 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:54.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:54.459 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:54.559 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:54.588 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:54.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:54.595 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:52:54.696 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:54.733 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:54.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:54.739 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:52:54.839 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:54.871 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:54.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:54.875 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:52:54.976 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:55.003 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:55.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:55.006 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:52:55.107 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:55.138 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:55.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:55.143 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:52:55.244 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:55.279 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:55.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:55.281 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:52:55.382 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:55.413 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:55.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:55.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:52:55.520 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:55.554 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:55.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:55.561 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:52:55.662 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:55.695 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:55.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:55.697 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:55.798 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:55.835 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:55.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:55.840 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:52:55.941 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:55.949 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:55 smithi173 bash[58816]: cluster 2023-10-01T10:52:53.979885+0000 mgr.y (mgr.25072) 1060 : cluster 0 pgmap v1014: 89 pgs: 89 active+clean; 6.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 45 MiB/s wr, 751 op/s; 6.2 KiB/s, 0 objects/s recovering 2023-10-01T10:52:55.970 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:55.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:55.972 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:56.073 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:56.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:55 smithi078 bash[75299]: cluster 2023-10-01T10:52:53.979885+0000 mgr.y (mgr.25072) 1060 : cluster 0 pgmap v1014: 89 pgs: 89 active+clean; 6.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 45 MiB/s wr, 751 op/s; 6.2 KiB/s, 0 objects/s recovering 2023-10-01T10:52:56.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:55 smithi078 bash[78274]: cluster 2023-10-01T10:52:53.979885+0000 mgr.y (mgr.25072) 1060 : cluster 0 pgmap v1014: 89 pgs: 89 active+clean; 6.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 45 MiB/s wr, 751 op/s; 6.2 KiB/s, 0 objects/s recovering 2023-10-01T10:52:56.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:56.107 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:56.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:52:56.208 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:56.249 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:56.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:56.251 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:52:56.283 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-10-01T10:52:56.283 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [ 2023-10-01T10:52:56.283 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:56.283 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.46037.0:110433 11.e e184/173 11:7bf26cd4:::benchmark_data_smithi078_157677_object3262:head v 184'6910, mlcod=184'6908)", 2023-10-01T10:52:56.284 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:52:56.279720+0000", 2023-10-01T10:52:56.284 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 0.0029819619999999999, 2023-10-01T10:52:56.284 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0030055250000000002, 2023-10-01T10:52:56.284 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:52:56.284 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "started", 2023-10-01T10:52:56.284 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:52:56.284 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:56.284 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:52:56.285 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:56.279720+0000", 2023-10-01T10:52:56.285 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:52:56.285 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:56.285 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:56.285 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:52:56.285 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:56.279720+0000", 2023-10-01T10:52:56.285 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-10-01T10:52:56.285 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:56.285 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:56.285 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:52:56.285 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:56.279720+0000", 2023-10-01T10:52:56.285 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.7788999999999999e-05 2023-10-01T10:52:56.286 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:56.286 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:56.286 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:52:56.286 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:56.279767+0000", 2023-10-01T10:52:56.286 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.5136999999999999e-05 2023-10-01T10:52:56.286 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:56.286 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:56.286 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:52:56.286 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:56.279782+0000", 2023-10-01T10:52:56.286 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.6830000000000002e-06 2023-10-01T10:52:56.286 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:56.287 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:56.287 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:52:56.287 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:56.279789+0000", 2023-10-01T10:52:56.287 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.7973000000000002e-05 2023-10-01T10:52:56.287 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:56.287 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:56.287 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:52:56.287 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:56.279847+0000", 2023-10-01T10:52:56.287 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.9901000000000001e-05 2023-10-01T10:52:56.288 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:52:56.289 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:52:56.289 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:52:56.289 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:56.279867+0000", 2023-10-01T10:52:56.290 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00014652099999999999 2023-10-01T10:52:56.290 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:56.291 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:52:56.291 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:56.291 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:52:56.291 INFO:teuthology.orchestra.run.smithi173.stdout: ], 2023-10-01T10:52:56.291 INFO:teuthology.orchestra.run.smithi173.stdout: "num_ops": 1 2023-10-01T10:52:56.291 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-10-01T10:52:56.352 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:56.380 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:56.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:56.383 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:52:56.483 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:56.508 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:56.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:56.511 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:52:56.612 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:56.645 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:56.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:56.647 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:52:56.748 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:56.783 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:56.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:56.786 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:52:56.887 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:56.915 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:56.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:56.918 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:52:57.018 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:57.044 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:57.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:57.050 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:52:57.151 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:57.193 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-01T10:52:57.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:57.199 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:57.199 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:57.300 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:57.323 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:57.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:57.326 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:52:57.428 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:57.459 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:57.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:57.462 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:52:57.563 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:57.591 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:57.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:57.595 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:52:57.696 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:57.722 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:57.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:57.727 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:52:57.828 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:57.852 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:52:57.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:57.859 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:52:57.961 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:57.970 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:57 smithi173 bash[58816]: cluster 2023-10-01T10:52:55.980522+0000 mgr.y (mgr.25072) 1061 : cluster 0 pgmap v1015: 89 pgs: 89 active+clean; 6.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 39 MiB/s wr, 678 op/s; 5.3 KiB/s, 0 objects/s recovering 2023-10-01T10:52:57.985 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:57.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:57.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:52:58.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:57 smithi078 bash[75299]: cluster 2023-10-01T10:52:55.980522+0000 mgr.y (mgr.25072) 1061 : cluster 0 pgmap v1015: 89 pgs: 89 active+clean; 6.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 39 MiB/s wr, 678 op/s; 5.3 KiB/s, 0 objects/s recovering 2023-10-01T10:52:58.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:57 smithi078 bash[78274]: cluster 2023-10-01T10:52:55.980522+0000 mgr.y (mgr.25072) 1061 : cluster 0 pgmap v1015: 89 pgs: 89 active+clean; 6.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 39 MiB/s wr, 678 op/s; 5.3 KiB/s, 0 objects/s recovering 2023-10-01T10:52:58.090 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:58.114 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:58.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:58.119 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:52:58.221 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:58.248 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:58.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:58.252 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:52:58.353 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:58.383 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:58.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:58.387 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:52:58.488 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:58.526 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:58.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:58.531 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:52:58.632 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:58.663 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:52:58.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:58.667 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:52:58.768 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:58.796 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:52:58.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:58.806 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:52:58.907 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:58.935 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:58 smithi078 bash[78274]: audit 2023-10-01T10:52:57.924185+0000 mon.a (mon.0) 1305 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:52:58.936 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:52:58 smithi078 bash[78274]: cluster 2023-10-01T10:52:57.981085+0000 mgr.y (mgr.25072) 1062 : cluster 0 pgmap v1016: 89 pgs: 89 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 32 MiB/s wr, 686 op/s; 5.3 KiB/s, 0 objects/s recovering 2023-10-01T10:52:58.936 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:58 smithi078 bash[75299]: audit 2023-10-01T10:52:57.924185+0000 mon.a (mon.0) 1305 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:52:58.936 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:52:58 smithi078 bash[75299]: cluster 2023-10-01T10:52:57.981085+0000 mgr.y (mgr.25072) 1062 : cluster 0 pgmap v1016: 89 pgs: 89 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 32 MiB/s wr, 686 op/s; 5.3 KiB/s, 0 objects/s recovering 2023-10-01T10:52:58.940 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:58.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:58.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:52:59.044 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:59.068 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:59.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:59.071 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:52:59.172 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:59.207 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:52:59.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:59.210 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:52:59.311 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:59.319 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:58 smithi173 bash[58816]: audit 2023-10-01T10:52:57.924185+0000 mon.a (mon.0) 1305 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:52:59.319 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:52:58 smithi173 bash[58816]: cluster 2023-10-01T10:52:57.981085+0000 mgr.y (mgr.25072) 1062 : cluster 0 pgmap v1016: 89 pgs: 89 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 32 MiB/s wr, 686 op/s; 5.3 KiB/s, 0 objects/s recovering 2023-10-01T10:52:59.342 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:52:59.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:59.349 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:52:59.449 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:59.482 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:59.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:59.484 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:52:59.585 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:59.610 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:52:59.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:59.611 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:52:59.712 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:59.743 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:52:59.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:59.745 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:52:59.847 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:59.871 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:52:59.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:52:59.873 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:52:59.974 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:00.010 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:00.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:00.018 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:53:00.118 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:00.143 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:00.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:00.146 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:53:00.247 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:00.270 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:53:00 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:53:00] "GET /metrics HTTP/1.1" 200 36313 "" "Prometheus/2.33.4" 2023-10-01T10:53:00.273 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:00.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:00.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:53:00.376 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:00.410 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:00.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:00.414 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:53:00.515 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:00.549 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:00.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:00.554 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:53:00.655 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:00.686 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:00.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:00.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:53:00.792 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:00.814 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:00.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:00.819 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:00.920 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:00.953 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:00.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:00.958 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:01.059 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:01.092 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:01.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:01.096 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:53:01.197 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:01.206 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:01 smithi078 bash[75299]: cluster 2023-10-01T10:52:59.981874+0000 mgr.y (mgr.25072) 1063 : cluster 0 pgmap v1017: 89 pgs: 89 active+clean; 6.3 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 26 MiB/s wr, 797 op/s; 5.3 KiB/s, 0 objects/s recovering 2023-10-01T10:53:01.206 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:01 smithi078 bash[78274]: cluster 2023-10-01T10:52:59.981874+0000 mgr.y (mgr.25072) 1063 : cluster 0 pgmap v1017: 89 pgs: 89 active+clean; 6.3 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 26 MiB/s wr, 797 op/s; 5.3 KiB/s, 0 objects/s recovering 2023-10-01T10:53:01.226 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:01.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:01.232 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:01.332 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:01.342 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:01 smithi173 bash[58816]: cluster 2023-10-01T10:52:59.981874+0000 mgr.y (mgr.25072) 1063 : cluster 0 pgmap v1017: 89 pgs: 89 active+clean; 6.3 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 26 MiB/s wr, 797 op/s; 5.3 KiB/s, 0 objects/s recovering 2023-10-01T10:53:01.362 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:01.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:01.364 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:53:01.465 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:01.502 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:01.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:01.508 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:53:01.608 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:01.636 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:01.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:01.638 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:53:01.739 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:01.769 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:01.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:01.772 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:53:01.873 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:01.910 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:01.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:01.913 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:53:02.014 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:02.039 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:02.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:02.046 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:53:02.056 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T10:53:02.147 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:02.172 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:02.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:02.175 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:53:02.277 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:02.303 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:02.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:02.305 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:02.406 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:02.432 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:02.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:02.437 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:53:02.538 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:02.571 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:02.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:02.576 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:53:02.677 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:02.702 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:02.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:02.709 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:53:02.810 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:02.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:02.844 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:02.845 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:53:02.945 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:02.980 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:02.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:02.984 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:53:03.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:03 smithi078 bash[78274]: cluster 2023-10-01T10:53:01.982447+0000 mgr.y (mgr.25072) 1064 : cluster 0 pgmap v1018: 89 pgs: 89 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 14 MiB/s wr, 605 op/s 2023-10-01T10:53:03.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:03 smithi078 bash[78274]: audit 2023-10-01T10:53:02.438714+0000 mon.a (mon.0) 1306 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:53:03.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:03 smithi078 bash[75299]: cluster 2023-10-01T10:53:01.982447+0000 mgr.y (mgr.25072) 1064 : cluster 0 pgmap v1018: 89 pgs: 89 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 14 MiB/s wr, 605 op/s 2023-10-01T10:53:03.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:03 smithi078 bash[75299]: audit 2023-10-01T10:53:02.438714+0000 mon.a (mon.0) 1306 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:53:03.085 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:03.115 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:03.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:03.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:53:03.219 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:03.246 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:03.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:03.249 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:53:03.295 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2023-10-01T10:53:03.295 INFO:teuthology.orchestra.run.smithi078.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-01T10:53:03.295 INFO:teuthology.orchestra.run.smithi078.stdout:} 2023-10-01T10:53:03.317 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T10:53:03.350 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:03.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:03 smithi173 bash[58816]: cluster 2023-10-01T10:53:01.982447+0000 mgr.y (mgr.25072) 1064 : cluster 0 pgmap v1018: 89 pgs: 89 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 14 MiB/s wr, 605 op/s 2023-10-01T10:53:03.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:03 smithi173 bash[58816]: audit 2023-10-01T10:53:02.438714+0000 mon.a (mon.0) 1306 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:53:03.369 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T10:53:03.392 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:03.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:03.395 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:53:03.398 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T10:53:03.437 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T10:53:03.471 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T10:53:03.495 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:03.513 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T10:53:03.525 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:03.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:03.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:53:03.562 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T10:53:03.617 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T10:53:03.623 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T10:53:03.623 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T10:53:03.623 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T10:53:03.623 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T10:53:03.623 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T10:53:03.623 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T10:53:03.624 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T10:53:03.624 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T10:53:03.631 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:03.666 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:03.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:03.672 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:53:03.773 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:03.804 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:03.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:03.811 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:53:03.911 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:03.943 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:03.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:03.950 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:53:04.050 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:04.053 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:04 smithi173 bash[58816]: audit 2023-10-01T10:53:03.271338+0000 mon.b (mon.1) 46 : audit 0 from='client.? 172.21.15.78:0/432128668' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:53:04.053 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:04 smithi173 bash[58816]: audit 2023-10-01T10:53:03.688667+0000 mon.c (mon.2) 178 : audit 1 from='client.? 172.21.15.78:0/910572395' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:53:04.053 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:04 smithi173 bash[58816]: audit 2023-10-01T10:53:03.689412+0000 mon.a (mon.0) 1307 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:53:04.072 INFO:teuthology.orchestra.run.smithi078.stderr:noscrub is set 2023-10-01T10:53:04.076 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:04.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:04.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:04.179 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:04.204 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:04.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:04.207 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:53:04.308 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:04.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:04 smithi078 bash[78274]: audit 2023-10-01T10:53:03.271338+0000 mon.b (mon.1) 46 : audit 0 from='client.? 172.21.15.78:0/432128668' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:53:04.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:04 smithi078 bash[78274]: audit 2023-10-01T10:53:03.688667+0000 mon.c (mon.2) 178 : audit 1 from='client.? 172.21.15.78:0/910572395' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:53:04.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:04 smithi078 bash[78274]: audit 2023-10-01T10:53:03.689412+0000 mon.a (mon.0) 1307 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:53:04.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:04 smithi078 bash[75299]: audit 2023-10-01T10:53:03.271338+0000 mon.b (mon.1) 46 : audit 0 from='client.? 172.21.15.78:0/432128668' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:53:04.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:04 smithi078 bash[75299]: audit 2023-10-01T10:53:03.688667+0000 mon.c (mon.2) 178 : audit 1 from='client.? 172.21.15.78:0/910572395' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:53:04.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:04 smithi078 bash[75299]: audit 2023-10-01T10:53:03.689412+0000 mon.a (mon.0) 1307 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:53:04.344 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:04.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:04.350 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:53:04.450 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:04.484 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:04.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:04.487 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:53:04.588 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:04.622 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:04.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:04.623 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:53:04.724 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:04.745 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:04.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:04.749 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:53:04.850 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:04.874 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:04.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:04.880 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:53:04.981 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:05.008 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:05.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:05.011 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:53:05.112 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:05.133 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:05 smithi078 bash[78274]: cluster 2023-10-01T10:53:03.983162+0000 mgr.y (mgr.25072) 1065 : cluster 0 pgmap v1019: 89 pgs: 89 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 7.7 MiB/s wr, 801 op/s 2023-10-01T10:53:05.133 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:05 smithi078 bash[78274]: cluster 2023-10-01T10:53:04.041842+0000 mon.a (mon.0) 1308 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:53:05.133 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:05 smithi078 bash[78274]: cluster 2023-10-01T10:53:04.056560+0000 osd.7 (osd.7) 144 : cluster 3 failed to encode map e185 with expected crc 2023-10-01T10:53:05.133 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:05 smithi078 bash[78274]: audit 2023-10-01T10:53:04.058459+0000 mon.a (mon.0) 1309 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T10:53:05.133 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:05 smithi078 bash[78274]: cluster 2023-10-01T10:53:04.058981+0000 mon.a (mon.0) 1310 : cluster 0 osdmap e185: 8 total, 8 up, 8 in 2023-10-01T10:53:05.133 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:05 smithi078 bash[78274]: cluster 2023-10-01T10:53:04.060486+0000 osd.6 (osd.6) 107 : cluster 3 failed to encode map e185 with expected crc 2023-10-01T10:53:05.134 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:05 smithi078 bash[78274]: cluster 2023-10-01T10:53:04.080093+0000 osd.5 (osd.5) 125 : cluster 3 failed to encode map e185 with expected crc 2023-10-01T10:53:05.134 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:05 smithi078 bash[75299]: cluster 2023-10-01T10:53:03.983162+0000 mgr.y (mgr.25072) 1065 : cluster 0 pgmap v1019: 89 pgs: 89 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 7.7 MiB/s wr, 801 op/s 2023-10-01T10:53:05.134 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:05 smithi078 bash[75299]: cluster 2023-10-01T10:53:04.041842+0000 mon.a (mon.0) 1308 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:53:05.134 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:05 smithi078 bash[75299]: cluster 2023-10-01T10:53:04.056560+0000 osd.7 (osd.7) 144 : cluster 3 failed to encode map e185 with expected crc 2023-10-01T10:53:05.134 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:05 smithi078 bash[75299]: audit 2023-10-01T10:53:04.058459+0000 mon.a (mon.0) 1309 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T10:53:05.134 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:05 smithi078 bash[75299]: cluster 2023-10-01T10:53:04.058981+0000 mon.a (mon.0) 1310 : cluster 0 osdmap e185: 8 total, 8 up, 8 in 2023-10-01T10:53:05.134 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:05 smithi078 bash[75299]: cluster 2023-10-01T10:53:04.060486+0000 osd.6 (osd.6) 107 : cluster 3 failed to encode map e185 with expected crc 2023-10-01T10:53:05.134 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:05 smithi078 bash[75299]: cluster 2023-10-01T10:53:04.080093+0000 osd.5 (osd.5) 125 : cluster 3 failed to encode map e185 with expected crc 2023-10-01T10:53:05.157 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:05.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:05.166 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:53:05.266 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:05.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:05 smithi173 bash[58816]: cluster 2023-10-01T10:53:03.983162+0000 mgr.y (mgr.25072) 1065 : cluster 0 pgmap v1019: 89 pgs: 89 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 7.7 MiB/s wr, 801 op/s 2023-10-01T10:53:05.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:05 smithi173 bash[58816]: cluster 2023-10-01T10:53:04.041842+0000 mon.a (mon.0) 1308 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:53:05.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:05 smithi173 bash[58816]: cluster 2023-10-01T10:53:04.056560+0000 osd.7 (osd.7) 144 : cluster 3 failed to encode map e185 with expected crc 2023-10-01T10:53:05.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:05 smithi173 bash[58816]: audit 2023-10-01T10:53:04.058459+0000 mon.a (mon.0) 1309 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T10:53:05.278 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:05 smithi173 bash[58816]: cluster 2023-10-01T10:53:04.058981+0000 mon.a (mon.0) 1310 : cluster 0 osdmap e185: 8 total, 8 up, 8 in 2023-10-01T10:53:05.279 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:05 smithi173 bash[58816]: cluster 2023-10-01T10:53:04.060486+0000 osd.6 (osd.6) 107 : cluster 3 failed to encode map e185 with expected crc 2023-10-01T10:53:05.279 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:05 smithi173 bash[58816]: cluster 2023-10-01T10:53:04.080093+0000 osd.5 (osd.5) 125 : cluster 3 failed to encode map e185 with expected crc 2023-10-01T10:53:05.293 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:05.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:05.298 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:53:05.399 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:05.417 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:05.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:05.419 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:53:05.520 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:05.543 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:05.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:05.545 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:53:05.646 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:05.672 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:05.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:05.677 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:53:05.778 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:05.801 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:05.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:05.806 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:53:05.907 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:05.932 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:05.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:05.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:53:06.034 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:06.070 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:06.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:06.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:53:06.175 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:06.200 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:06.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:06.211 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:53:06.312 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:06.336 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:06.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:06.340 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:53:06.441 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:06.469 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:06.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:06.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:53:06.545 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-10-01T10:53:06.545 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T10:53:06.576 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:06.599 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:06.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:06.602 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:53:06.703 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:06.730 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:06.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:06.734 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:53:06.835 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:06.881 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:06.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:06.886 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:06.988 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:07.015 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:07.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:07.018 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:53:07.118 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:07.150 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:07.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:07.157 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:53:07.258 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:07.268 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:07 smithi078 bash[75299]: cluster 2023-10-01T10:53:05.983794+0000 mgr.y (mgr.25072) 1066 : cluster 0 pgmap v1021: 89 pgs: 89 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 805 op/s 2023-10-01T10:53:07.268 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:07 smithi078 bash[78274]: cluster 2023-10-01T10:53:05.983794+0000 mgr.y (mgr.25072) 1066 : cluster 0 pgmap v1021: 89 pgs: 89 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 805 op/s 2023-10-01T10:53:07.291 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:07.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:07.298 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:53:07.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:07 smithi173 bash[58816]: cluster 2023-10-01T10:53:05.983794+0000 mgr.y (mgr.25072) 1066 : cluster 0 pgmap v1021: 89 pgs: 89 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 805 op/s 2023-10-01T10:53:07.399 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:07.424 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:07.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:07.425 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:53:07.526 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:07.529 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-01T10:53:07.554 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:07.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:07.559 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:53:07.659 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:07.685 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:07.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:07.692 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:53:07.792 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:07.825 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:07.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:07.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:07.928 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:07.957 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:07.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:07.960 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:53:08.060 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:08.097 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:08.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:08.100 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:53:08.201 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:08.225 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:08.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:08.231 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:53:08.332 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:08.372 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:08.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:08.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:53:08.479 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:08.515 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:08.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:08.519 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:53:08.620 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:08.643 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:08.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:08.649 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:53:08.749 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:08.776 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:08.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:08.779 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:08.879 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:08.904 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:08.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:08.907 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:53:08.911 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T10:53:08.933 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T10:53:09.008 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:09.051 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:09.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:09.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:53:09.158 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:09.202 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:09.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:09.208 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:53:09.308 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:09.337 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:09.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:09.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:53:09.440 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:09.467 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:09.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:09.470 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:53:09.571 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:09.578 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:09 smithi173 bash[58816]: cluster 2023-10-01T10:53:07.984288+0000 mgr.y (mgr.25072) 1067 : cluster 0 pgmap v1022: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 807 op/s 2023-10-01T10:53:09.594 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:09.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:09.600 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:53:09.701 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:09.726 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:09.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:09.734 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:53:09.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:09 smithi078 bash[75299]: cluster 2023-10-01T10:53:07.984288+0000 mgr.y (mgr.25072) 1067 : cluster 0 pgmap v1022: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 807 op/s 2023-10-01T10:53:09.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:09 smithi078 bash[78274]: cluster 2023-10-01T10:53:07.984288+0000 mgr.y (mgr.25072) 1067 : cluster 0 pgmap v1022: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 807 op/s 2023-10-01T10:53:09.835 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:09.863 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:09.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:09.873 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:53:09.974 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:10.000 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:10.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:10.003 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:10.103 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:10.133 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:10.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:10.136 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:53:10.237 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:10.271 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:10.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:10.276 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:53:10.376 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:10.407 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:10.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:10.410 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:53:10.511 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:10.541 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:10.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:10.544 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:53:10.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:53:10 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:53:10] "GET /metrics HTTP/1.1" 200 36318 "" "Prometheus/2.33.4" 2023-10-01T10:53:10.645 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:10.681 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:10.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:10.684 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:53:10.785 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:10.819 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:10.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:10.822 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:53:10.855 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T10:53:10.855 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T10:53:10.856 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2023-10-01T10:53:10.856 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-01T10:53:10.856 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-01T10:53:10.856 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 config set filestore_inject_stall 3 2023-10-01T10:53:10.923 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:10.971 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:10.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:10.977 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:53:11.078 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:11.110 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:11.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:11.115 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:53:11.216 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:11.242 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:11.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:11.249 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:53:11.350 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:11.376 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:11.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:11.379 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:53:11.480 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:11.511 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:11.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:11.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:11.617 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:11.627 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:11 smithi078 bash[75299]: cluster 2023-10-01T10:53:09.987944+0000 mgr.y (mgr.25072) 1068 : cluster 0 pgmap v1023: 89 pgs: 89 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 794 op/s 2023-10-01T10:53:11.627 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:11 smithi078 bash[78274]: cluster 2023-10-01T10:53:09.987944+0000 mgr.y (mgr.25072) 1068 : cluster 0 pgmap v1023: 89 pgs: 89 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 794 op/s 2023-10-01T10:53:11.644 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:11.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:11.649 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:53:11.750 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:11.774 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:11.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:11.784 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:53:11.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:11 smithi173 bash[58816]: cluster 2023-10-01T10:53:09.987944+0000 mgr.y (mgr.25072) 1068 : cluster 0 pgmap v1023: 89 pgs: 89 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 794 op/s 2023-10-01T10:53:11.885 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:11.915 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:11.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:11.919 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:53:12.020 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:12.043 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:12.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:12.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:53:12.152 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:12.191 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:12.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:12.197 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:53:12.298 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:12.332 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:12.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:12.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:53:12.341 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T10:53:12.436 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:12.466 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:12.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:12.471 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:12.571 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:12.605 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:12.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:12.609 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:53:12.710 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:12.736 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:12.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:12.742 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:53:12.842 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:12.882 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:12.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:12.886 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:53:12.987 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:13.022 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:13.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:13.024 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:53:13.125 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:13.165 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:13.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:13.167 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:53:13.268 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:13.297 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:13.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:13.299 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:53:13.400 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:13.436 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:13.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:13.440 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:53:13.540 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:13.565 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:13.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:13.570 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:13.671 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:13.693 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:13.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:13.701 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:53:13.803 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:13.817 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:13 smithi078 bash[78274]: cluster 2023-10-01T10:53:11.990580+0000 mgr.y (mgr.25072) 1069 : cluster 0 pgmap v1024: 89 pgs: 89 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 794 op/s 2023-10-01T10:53:13.817 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:13 smithi078 bash[75299]: cluster 2023-10-01T10:53:11.990580+0000 mgr.y (mgr.25072) 1069 : cluster 0 pgmap v1024: 89 pgs: 89 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 794 op/s 2023-10-01T10:53:13.826 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:53:13.826 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:53:13.845 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:13.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:13.851 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:13.882 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:13 smithi173 bash[58816]: cluster 2023-10-01T10:53:11.990580+0000 mgr.y (mgr.25072) 1069 : cluster 0 pgmap v1024: 89 pgs: 89 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 794 op/s 2023-10-01T10:53:13.953 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:13.987 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:13.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:13.989 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:53:14.090 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:14.132 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:14.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:14.136 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:14.236 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:14.265 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:14.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:14.268 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:53:14.368 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:14.397 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:14.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:14.402 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:53:14.503 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:14.527 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:14.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:14.529 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:53:14.630 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:14.660 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:14.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:14.667 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:53:14.768 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:14.806 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:14.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:14.809 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:53:14.909 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:14.951 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:14.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:14.955 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:15.056 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:15.091 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:15.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:15.094 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:53:15.197 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:15.221 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:15.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:15.228 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:53:15.328 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:15.359 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:15.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:15.360 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:53:15.461 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:15.498 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:15.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:15.501 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:53:15.601 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:15.609 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:15 smithi173 bash[58816]: cluster 2023-10-01T10:53:13.991227+0000 mgr.y (mgr.25072) 1070 : cluster 0 pgmap v1025: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 759 op/s 2023-10-01T10:53:15.634 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:15.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:15.641 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:53:15.742 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:15.767 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:15.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:15.774 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:53:15.791 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T10:53:15.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:15 smithi078 bash[78274]: cluster 2023-10-01T10:53:13.991227+0000 mgr.y (mgr.25072) 1070 : cluster 0 pgmap v1025: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 759 op/s 2023-10-01T10:53:15.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:15 smithi078 bash[75299]: cluster 2023-10-01T10:53:13.991227+0000 mgr.y (mgr.25072) 1070 : cluster 0 pgmap v1025: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 759 op/s 2023-10-01T10:53:15.875 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:15.902 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:15.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:15.909 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:53:16.010 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:16.041 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:16.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:16.044 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:16.146 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:16.170 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:16.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:16.175 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:53:16.276 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:16.299 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:16.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:16.303 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:53:16.403 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:16.437 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:16.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:16.443 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:53:16.544 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:16.569 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:16.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:16.577 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:53:16.678 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:16.704 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:16.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:16.712 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:53:16.812 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:16.836 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:16.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:16.841 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:53:16.942 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:16.967 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:16.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:16.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:17.076 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:17.103 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:17.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:17.107 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:53:17.208 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:17.235 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:17.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:17.238 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:53:17.339 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:17.369 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:17.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:17.374 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:53:17.475 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:17.507 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:17.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:17.511 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:17.563 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:17 smithi173 bash[58816]: cluster 2023-10-01T10:53:15.991699+0000 mgr.y (mgr.25072) 1071 : cluster 0 pgmap v1026: 89 pgs: 89 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2023-10-01T10:53:17.563 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:17 smithi173 bash[58816]: audit 2023-10-01T10:53:17.449052+0000 mon.a (mon.0) 1311 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:53:17.563 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:17 smithi173 bash[58816]: audit 2023-10-01T10:53:17.454434+0000 mon.a (mon.0) 1312 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:53:17.612 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:17.648 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:17.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:17.654 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:53:17.755 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:17.777 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:17.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:17.780 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:53:17.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:17 smithi078 bash[75299]: cluster 2023-10-01T10:53:15.991699+0000 mgr.y (mgr.25072) 1071 : cluster 0 pgmap v1026: 89 pgs: 89 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2023-10-01T10:53:17.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:17 smithi078 bash[75299]: audit 2023-10-01T10:53:17.449052+0000 mon.a (mon.0) 1311 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:53:17.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:17 smithi078 bash[75299]: audit 2023-10-01T10:53:17.454434+0000 mon.a (mon.0) 1312 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:53:17.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:17 smithi078 bash[78274]: cluster 2023-10-01T10:53:15.991699+0000 mgr.y (mgr.25072) 1071 : cluster 0 pgmap v1026: 89 pgs: 89 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2023-10-01T10:53:17.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:17 smithi078 bash[78274]: audit 2023-10-01T10:53:17.449052+0000 mon.a (mon.0) 1311 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:53:17.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:17 smithi078 bash[78274]: audit 2023-10-01T10:53:17.454434+0000 mon.a (mon.0) 1312 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:53:17.880 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:17.901 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:17.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:17.903 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:53:17.969 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-10-01T10:53:17.969 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T10:53:18.004 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:18.037 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:18.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:18.040 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:53:18.140 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:18.173 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:18.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:18.178 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:53:18.278 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:18.306 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:18.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:18.312 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:18.412 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:18.445 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:18.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:18.448 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:53:18.549 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:18.591 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:18.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:18.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:53:18.696 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:18.729 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:18.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:18.732 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:53:18.833 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:18.864 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:18.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:18.906 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:53:19.007 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:19.034 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:19.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:19.038 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:53:19.138 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:19.167 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:19.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:19.173 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:53:19.274 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:19.304 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:19.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:19.308 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:53:19.409 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:19.447 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:19.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:19.449 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:19.549 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:19.590 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:19.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:19.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:53:19.693 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:19.709 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:19 smithi078 bash[75299]: cluster 2023-10-01T10:53:17.992127+0000 mgr.y (mgr.25072) 1072 : cluster 0 pgmap v1027: 89 pgs: 89 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 741 op/s 2023-10-01T10:53:19.709 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:19 smithi078 bash[75299]: audit 2023-10-01T10:53:19.317631+0000 mon.a (mon.0) 1313 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:53:19.709 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:19 smithi078 bash[78274]: cluster 2023-10-01T10:53:17.992127+0000 mgr.y (mgr.25072) 1072 : cluster 0 pgmap v1027: 89 pgs: 89 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 741 op/s 2023-10-01T10:53:19.709 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:19 smithi078 bash[78274]: audit 2023-10-01T10:53:19.317631+0000 mon.a (mon.0) 1313 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:53:19.729 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:19.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:19.731 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:53:19.786 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-01T10:53:19.828 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:19 smithi173 bash[58816]: cluster 2023-10-01T10:53:17.992127+0000 mgr.y (mgr.25072) 1072 : cluster 0 pgmap v1027: 89 pgs: 89 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 741 op/s 2023-10-01T10:53:19.828 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:19 smithi173 bash[58816]: audit 2023-10-01T10:53:19.317631+0000 mon.a (mon.0) 1313 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:53:19.831 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:19.864 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:19.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:19.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:19.972 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:20.005 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:20.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:20.010 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:53:20.111 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:20.144 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:20.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:20.147 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:53:20.247 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:20.272 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:20.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:20.275 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:53:20.375 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:20.383 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:53:20 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:53:20] "GET /metrics HTTP/1.1" 200 36314 "" "Prometheus/2.33.4" 2023-10-01T10:53:20.420 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:20.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:20.426 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:53:20.527 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:20.575 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:20.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:20.580 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:20.680 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:20.725 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:20.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:20.729 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:53:20.829 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:20.852 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:20.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:20.855 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:53:20.956 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:20.985 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:20.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:20.988 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:53:21.089 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:21.121 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:21.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:21.127 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:53:21.228 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:21.255 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:21.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:21.259 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:53:21.342 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-10-01T10:53:21.342 INFO:teuthology.orchestra.run.smithi173.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-01T10:53:21.342 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-10-01T10:53:21.361 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:21.393 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:21.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:21.399 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:21.500 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:21.522 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:21.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:21.523 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:53:21.624 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:21.658 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:21.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:21.663 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:53:21.764 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:21.803 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:21.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:21.805 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:53:21.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:21 smithi078 bash[75299]: cluster 2023-10-01T10:53:19.992808+0000 mgr.y (mgr.25072) 1073 : cluster 0 pgmap v1028: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 817 op/s 2023-10-01T10:53:21.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:21 smithi078 bash[78274]: cluster 2023-10-01T10:53:19.992808+0000 mgr.y (mgr.25072) 1073 : cluster 0 pgmap v1028: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 817 op/s 2023-10-01T10:53:21.906 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:21.948 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:21.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:21.950 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:53:21.988 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:21 smithi173 bash[58816]: cluster 2023-10-01T10:53:19.992808+0000 mgr.y (mgr.25072) 1073 : cluster 0 pgmap v1028: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 817 op/s 2023-10-01T10:53:22.051 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:22.081 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:22.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:22.085 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:53:22.186 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:22.212 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:22.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:22.216 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:53:22.317 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:22.347 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:22.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:22.352 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:53:22.453 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:22.477 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:22.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:22.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:53:22.580 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:22.602 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:22.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:22.607 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:53:22.708 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:22.742 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:22.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:22.747 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:53:22.847 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:22.879 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:22.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:22.885 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:53:22.986 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:23.021 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T10:53:23.023 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:23.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:23.027 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:23.054 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T10:53:23.093 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T10:53:23.128 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:23.140 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T10:53:23.170 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:23.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:23.178 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:53:23.186 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T10:53:23.217 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T10:53:23.267 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T10:53:23.279 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:23.307 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T10:53:23.312 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:23.313 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T10:53:23.313 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T10:53:23.313 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T10:53:23.314 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T10:53:23.314 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T10:53:23.314 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T10:53:23.314 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T10:53:23.315 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T10:53:23.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:23.316 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:23.418 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:23.450 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:23.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:23.453 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:53:23.554 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:23.593 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:23.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:23.597 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:53:23.601 INFO:teuthology.orchestra.run.smithi078.stderr:nodeep-scrub is set 2023-10-01T10:53:23.697 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:23.733 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:23.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:23.737 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:53:23.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:23 smithi078 bash[75299]: cluster 2023-10-01T10:53:21.993237+0000 mgr.y (mgr.25072) 1074 : cluster 0 pgmap v1029: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 620 op/s 2023-10-01T10:53:23.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:23 smithi078 bash[75299]: audit 2023-10-01T10:53:22.976821+0000 mon.c (mon.2) 179 : audit 0 from='client.? 172.21.15.78:0/3244570452' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:53:23.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:23 smithi078 bash[75299]: audit 2023-10-01T10:53:23.429858+0000 mon.a (mon.0) 1314 : audit 1 from='client.? 172.21.15.78:0/2838972139' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:53:23.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:23 smithi078 bash[78274]: cluster 2023-10-01T10:53:21.993237+0000 mgr.y (mgr.25072) 1074 : cluster 0 pgmap v1029: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 620 op/s 2023-10-01T10:53:23.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:23 smithi078 bash[78274]: audit 2023-10-01T10:53:22.976821+0000 mon.c (mon.2) 179 : audit 0 from='client.? 172.21.15.78:0/3244570452' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:53:23.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:23 smithi078 bash[78274]: audit 2023-10-01T10:53:23.429858+0000 mon.a (mon.0) 1314 : audit 1 from='client.? 172.21.15.78:0/2838972139' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:53:23.839 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:23.846 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:23 smithi173 bash[58816]: cluster 2023-10-01T10:53:21.993237+0000 mgr.y (mgr.25072) 1074 : cluster 0 pgmap v1029: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 620 op/s 2023-10-01T10:53:23.847 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:23 smithi173 bash[58816]: audit 2023-10-01T10:53:22.976821+0000 mon.c (mon.2) 179 : audit 0 from='client.? 172.21.15.78:0/3244570452' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:53:23.847 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:23 smithi173 bash[58816]: audit 2023-10-01T10:53:23.429858+0000 mon.a (mon.0) 1314 : audit 1 from='client.? 172.21.15.78:0/2838972139' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:53:23.872 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:23.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:23.875 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:53:23.976 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:24.004 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:24.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:24.008 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:53:24.108 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:24.130 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:24.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:24.135 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:53:24.237 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:24.270 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:24.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:24.272 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:53:24.373 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:24.405 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:24.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:24.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:53:24.509 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:24.544 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:24.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:24.547 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:53:24.648 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:24.676 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:24.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:24.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:53:24.780 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:24.789 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:24 smithi173 bash[58816]: cluster 2023-10-01T10:53:23.575180+0000 mon.a (mon.0) 1315 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:53:24.789 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:24 smithi173 bash[58816]: cluster 2023-10-01T10:53:23.592041+0000 osd.7 (osd.7) 145 : cluster 3 failed to encode map e186 with expected crc 2023-10-01T10:53:24.789 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:24 smithi173 bash[58816]: audit 2023-10-01T10:53:23.592633+0000 mon.a (mon.0) 1316 : audit 1 from='client.? 172.21.15.78:0/2838972139' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T10:53:24.789 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:24 smithi173 bash[58816]: cluster 2023-10-01T10:53:23.592682+0000 mon.a (mon.0) 1317 : cluster 0 osdmap e186: 8 total, 8 up, 8 in 2023-10-01T10:53:24.789 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:24 smithi173 bash[58816]: cluster 2023-10-01T10:53:23.598543+0000 osd.5 (osd.5) 126 : cluster 3 failed to encode map e186 with expected crc 2023-10-01T10:53:24.789 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:24 smithi173 bash[58816]: cluster 2023-10-01T10:53:23.599973+0000 osd.6 (osd.6) 108 : cluster 3 failed to encode map e186 with expected crc 2023-10-01T10:53:24.827 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:24.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:24.828 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:53:24.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:24 smithi078 bash[75299]: cluster 2023-10-01T10:53:23.575180+0000 mon.a (mon.0) 1315 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:53:24.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:24 smithi078 bash[75299]: cluster 2023-10-01T10:53:23.592041+0000 osd.7 (osd.7) 145 : cluster 3 failed to encode map e186 with expected crc 2023-10-01T10:53:24.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:24 smithi078 bash[75299]: audit 2023-10-01T10:53:23.592633+0000 mon.a (mon.0) 1316 : audit 1 from='client.? 172.21.15.78:0/2838972139' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T10:53:24.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:24 smithi078 bash[75299]: cluster 2023-10-01T10:53:23.592682+0000 mon.a (mon.0) 1317 : cluster 0 osdmap e186: 8 total, 8 up, 8 in 2023-10-01T10:53:24.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:24 smithi078 bash[75299]: cluster 2023-10-01T10:53:23.598543+0000 osd.5 (osd.5) 126 : cluster 3 failed to encode map e186 with expected crc 2023-10-01T10:53:24.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:24 smithi078 bash[75299]: cluster 2023-10-01T10:53:23.599973+0000 osd.6 (osd.6) 108 : cluster 3 failed to encode map e186 with expected crc 2023-10-01T10:53:24.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:24 smithi078 bash[78274]: cluster 2023-10-01T10:53:23.575180+0000 mon.a (mon.0) 1315 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:53:24.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:24 smithi078 bash[78274]: cluster 2023-10-01T10:53:23.592041+0000 osd.7 (osd.7) 145 : cluster 3 failed to encode map e186 with expected crc 2023-10-01T10:53:24.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:24 smithi078 bash[78274]: audit 2023-10-01T10:53:23.592633+0000 mon.a (mon.0) 1316 : audit 1 from='client.? 172.21.15.78:0/2838972139' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T10:53:24.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:24 smithi078 bash[78274]: cluster 2023-10-01T10:53:23.592682+0000 mon.a (mon.0) 1317 : cluster 0 osdmap e186: 8 total, 8 up, 8 in 2023-10-01T10:53:24.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:24 smithi078 bash[78274]: cluster 2023-10-01T10:53:23.598543+0000 osd.5 (osd.5) 126 : cluster 3 failed to encode map e186 with expected crc 2023-10-01T10:53:24.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:24 smithi078 bash[78274]: cluster 2023-10-01T10:53:23.599973+0000 osd.6 (osd.6) 108 : cluster 3 failed to encode map e186 with expected crc 2023-10-01T10:53:24.929 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:24.955 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:24.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:24.957 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:53:25.058 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:25.100 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:25.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:25.103 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:53:25.204 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:25.244 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:25.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:25.246 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:53:25.347 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:25.380 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:25.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:25.386 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:53:25.487 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:25.526 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:25.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:25.529 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:53:25.629 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:25.665 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:25.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:25.667 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:53:25.767 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:25.775 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:25 smithi078 bash[78274]: cluster 2023-10-01T10:53:23.993637+0000 mgr.y (mgr.25072) 1075 : cluster 0 pgmap v1031: 89 pgs: 89 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 737 op/s 2023-10-01T10:53:25.776 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:25 smithi078 bash[75299]: cluster 2023-10-01T10:53:23.993637+0000 mgr.y (mgr.25072) 1075 : cluster 0 pgmap v1031: 89 pgs: 89 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 737 op/s 2023-10-01T10:53:25.801 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:25.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:25.805 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:53:25.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:25 smithi173 bash[58816]: cluster 2023-10-01T10:53:23.993637+0000 mgr.y (mgr.25072) 1075 : cluster 0 pgmap v1031: 89 pgs: 89 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 737 op/s 2023-10-01T10:53:25.892 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:53:25.906 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:25.932 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:25.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:25.937 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:53:26.038 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:26.069 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:26.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:26.075 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:53:26.176 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:26.218 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:26.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:26.224 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:26.325 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:26.363 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:26.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:26.372 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:53:26.472 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:26.508 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:26.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:26.511 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:53:26.612 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:26.643 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:26.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:26.648 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:53:26.749 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:26.784 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:26.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:26.787 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:53:26.888 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:26.934 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:26.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:26.936 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:53:27.037 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:27.061 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:27.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:27.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:27.168 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:27.195 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:27.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:27.201 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:53:27.301 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:27.329 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:27.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:27.334 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:53:27.409 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T10:53:27.421 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T10:53:27.434 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:27.456 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T10:53:27.456 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T10:53:27.456 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_6 2023-10-01T10:53:27.456 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T10:53:27.505 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:27.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:27.515 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:53:27.616 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:27.648 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:27.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:27.650 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:53:27.748 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:27 smithi078 bash[78274]: cluster 2023-10-01T10:53:25.994237+0000 mgr.y (mgr.25072) 1076 : cluster 0 pgmap v1032: 89 pgs: 89 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 740 op/s 2023-10-01T10:53:27.748 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:27 smithi078 bash[78274]: audit 2023-10-01T10:53:26.597018+0000 mon.a (mon.0) 1318 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:53:27.748 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:27 smithi078 bash[78274]: audit 2023-10-01T10:53:26.626389+0000 mon.a (mon.0) 1319 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:53:27.748 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:27 smithi078 bash[75299]: cluster 2023-10-01T10:53:25.994237+0000 mgr.y (mgr.25072) 1076 : cluster 0 pgmap v1032: 89 pgs: 89 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 740 op/s 2023-10-01T10:53:27.748 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:27 smithi078 bash[75299]: audit 2023-10-01T10:53:26.597018+0000 mon.a (mon.0) 1318 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:53:27.748 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:27 smithi078 bash[75299]: audit 2023-10-01T10:53:26.626389+0000 mon.a (mon.0) 1319 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:53:27.751 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:27.776 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:27.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:27.779 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:53:27.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:27 smithi173 bash[58816]: cluster 2023-10-01T10:53:25.994237+0000 mgr.y (mgr.25072) 1076 : cluster 0 pgmap v1032: 89 pgs: 89 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 740 op/s 2023-10-01T10:53:27.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:27 smithi173 bash[58816]: audit 2023-10-01T10:53:26.597018+0000 mon.a (mon.0) 1318 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:53:27.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:27 smithi173 bash[58816]: audit 2023-10-01T10:53:26.626389+0000 mon.a (mon.0) 1319 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:53:27.880 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:27.920 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:27.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:27.928 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:53:28.028 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:28.066 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:28.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:28.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:53:28.170 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:28.197 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:28.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:28.202 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:53:28.303 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:28.331 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:28.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:28.334 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:53:28.435 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:28.468 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:28.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:28.472 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:28.572 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:28.595 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:28.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:28.598 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:53:28.699 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:28.730 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:28.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:28.735 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:53:28.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:28 smithi078 bash[78274]: audit 2023-10-01T10:53:27.789642+0000 mon.a (mon.0) 1320 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:53:28.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:28 smithi078 bash[78274]: audit 2023-10-01T10:53:27.798344+0000 mon.a (mon.0) 1321 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:53:28.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:28 smithi078 bash[78274]: audit 2023-10-01T10:53:28.388583+0000 mon.a (mon.0) 1322 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:53:28.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:28 smithi078 bash[78274]: audit 2023-10-01T10:53:28.389700+0000 mon.a (mon.0) 1323 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:53:28.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:28 smithi078 bash[78274]: audit 2023-10-01T10:53:28.397743+0000 mon.a (mon.0) 1324 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:53:28.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:28 smithi078 bash[75299]: audit 2023-10-01T10:53:27.789642+0000 mon.a (mon.0) 1320 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:53:28.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:28 smithi078 bash[75299]: audit 2023-10-01T10:53:27.798344+0000 mon.a (mon.0) 1321 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:53:28.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:28 smithi078 bash[75299]: audit 2023-10-01T10:53:28.388583+0000 mon.a (mon.0) 1322 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:53:28.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:28 smithi078 bash[75299]: audit 2023-10-01T10:53:28.389700+0000 mon.a (mon.0) 1323 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:53:28.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:28 smithi078 bash[75299]: audit 2023-10-01T10:53:28.397743+0000 mon.a (mon.0) 1324 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:53:28.836 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:28.877 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:28.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:28.880 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:28.980 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:29.013 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:29.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:29.015 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:29.116 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:29.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:28 smithi173 bash[58816]: audit 2023-10-01T10:53:27.789642+0000 mon.a (mon.0) 1320 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:53:29.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:28 smithi173 bash[58816]: audit 2023-10-01T10:53:27.798344+0000 mon.a (mon.0) 1321 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:53:29.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:28 smithi173 bash[58816]: audit 2023-10-01T10:53:28.388583+0000 mon.a (mon.0) 1322 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:53:29.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:28 smithi173 bash[58816]: audit 2023-10-01T10:53:28.389700+0000 mon.a (mon.0) 1323 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:53:29.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:28 smithi173 bash[58816]: audit 2023-10-01T10:53:28.397743+0000 mon.a (mon.0) 1324 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:53:29.143 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:29.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:29.145 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:53:29.246 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:29.269 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:29.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:29.273 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:53:29.374 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:29.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:29.413 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:29.413 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:53:29.514 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:29.541 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:29.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:29.543 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:53:29.644 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:29.671 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:29.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:29.673 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:53:29.774 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:29.795 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:29.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:29.798 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:29.899 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:29.908 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:29 smithi173 bash[58816]: cluster 2023-10-01T10:53:27.994749+0000 mgr.y (mgr.25072) 1077 : cluster 0 pgmap v1033: 89 pgs: 89 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 727 op/s 2023-10-01T10:53:29.925 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:29.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:29.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:53:30.027 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:30.036 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:29 smithi078 bash[78274]: cluster 2023-10-01T10:53:27.994749+0000 mgr.y (mgr.25072) 1077 : cluster 0 pgmap v1033: 89 pgs: 89 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 727 op/s 2023-10-01T10:53:30.037 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:29 smithi078 bash[75299]: cluster 2023-10-01T10:53:27.994749+0000 mgr.y (mgr.25072) 1077 : cluster 0 pgmap v1033: 89 pgs: 89 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 727 op/s 2023-10-01T10:53:30.055 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:30.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:30.057 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:53:30.158 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:30.204 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:30.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:30.210 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:53:30.311 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:30.327 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:53:30 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:53:30] "GET /metrics HTTP/1.1" 200 36314 "" "Prometheus/2.33.4" 2023-10-01T10:53:30.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:30.354 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:30.354 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:53:30.455 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:30.490 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:30.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:30.499 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:53:30.599 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:30.628 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:30.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:30.631 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:53:30.731 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:30.756 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2023-10-01T10:53:30.756 INFO:teuthology.orchestra.run.smithi078.stdout: "ops": [ 2023-10-01T10:53:30.756 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.756 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.46037.0:136036 11.2 e186/173 11:42593676:::benchmark_data_smithi078_157677_object28865:head v 186'8595, mlcod=186'8590)", 2023-10-01T10:53:30.756 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:53:30.731330+0000", 2023-10-01T10:53:30.756 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 0.022258126999999999, 2023-10-01T10:53:30.756 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.022281386, 2023-10-01T10:53:30.757 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:53:30.757 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "started", 2023-10-01T10:53:30.757 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:53:30.757 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.757 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:53:30.757 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.731330+0000", 2023-10-01T10:53:30.757 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:53:30.757 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.757 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.757 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:53:30.757 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.731330+0000", 2023-10-01T10:53:30.757 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:53:30.758 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.758 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.758 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:53:30.758 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.731330+0000", 2023-10-01T10:53:30.758 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:53:30.758 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.758 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.758 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:53:30.758 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.731337+0000", 2023-10-01T10:53:30.758 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.3699999999999997e-06 2023-10-01T10:53:30.759 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.759 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.759 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:53:30.759 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.731352+0000", 2023-10-01T10:53:30.759 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.4885999999999999e-05 2023-10-01T10:53:30.759 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.759 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.759 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:53:30.759 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.731359+0000", 2023-10-01T10:53:30.759 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.4660000000000002e-06 2023-10-01T10:53:30.759 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.759 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.760 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:53:30.760 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.741270+0000", 2023-10-01T10:53:30.760 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0099110099999999996 2023-10-01T10:53:30.760 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.760 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.760 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:53:30.760 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.741354+0000", 2023-10-01T10:53:30.760 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.3419999999999998e-05 2023-10-01T10:53:30.760 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:53:30.760 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:53:30.761 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:53:30.761 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.761 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.761 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46037.0:136048 11.6 11:6b5b6cd6:::benchmark_data_smithi078_157677_object28877:head [delete] snapc 0=[] ondisk+write+known_if_redirected e186)", 2023-10-01T10:53:30.761 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:53:30.733501+0000", 2023-10-01T10:53:30.761 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 0.020087904, 2023-10-01T10:53:30.761 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.020180766999999999, 2023-10-01T10:53:30.761 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:53:30.761 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "waiting for sub ops", 2023-10-01T10:53:30.761 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:53:30.761 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46037", 2023-10-01T10:53:30.762 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:53:30.762 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 136048 2023-10-01T10:53:30.762 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.762 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:53:30.762 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.762 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:53:30.762 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.733501+0000", 2023-10-01T10:53:30.762 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:53:30.762 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.762 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.762 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:53:30.763 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.733501+0000", 2023-10-01T10:53:30.763 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:53:30.763 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.763 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.763 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:53:30.763 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.733499+0000", 2023-10-01T10:53:30.763 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999986 2023-10-01T10:53:30.763 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.763 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.764 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:53:30.764 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.733507+0000", 2023-10-01T10:53:30.764 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.8310000000000001e-06 2023-10-01T10:53:30.764 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.764 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.764 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:53:30.764 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.733525+0000", 2023-10-01T10:53:30.764 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.8057999999999999e-05 2023-10-01T10:53:30.764 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.764 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.764 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:53:30.765 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.733534+0000", 2023-10-01T10:53:30.765 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.1030000000000008e-06 2023-10-01T10:53:30.765 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.765 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.765 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:53:30.765 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.733587+0000", 2023-10-01T10:53:30.765 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.3056000000000002e-05 2023-10-01T10:53:30.765 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.765 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.765 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:53:30.765 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.733831+0000", 2023-10-01T10:53:30.765 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00024389400000000001 2023-10-01T10:53:30.766 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.766 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.766 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 1,2", 2023-10-01T10:53:30.766 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.733991+0000", 2023-10-01T10:53:30.766 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00015956600000000001 2023-10-01T10:53:30.766 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.766 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.766 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:53:30.766 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.744098+0000", 2023-10-01T10:53:30.766 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.010107659999999999 2023-10-01T10:53:30.766 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:53:30.766 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:53:30.767 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:53:30.767 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.767 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.767 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.46037.0:136039 11.2 e186/173 11:411c1ad0:::benchmark_data_smithi078_157677_object28868:head v 186'8596, mlcod=186'8592)", 2023-10-01T10:53:30.767 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:53:30.741149+0000", 2023-10-01T10:53:30.767 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 0.012439145, 2023-10-01T10:53:30.767 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.012587622999999999, 2023-10-01T10:53:30.767 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:53:30.767 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "started", 2023-10-01T10:53:30.767 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:53:30.767 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.767 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:53:30.768 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.741149+0000", 2023-10-01T10:53:30.768 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:53:30.768 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.768 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.768 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:53:30.768 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.741149+0000", 2023-10-01T10:53:30.768 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:53:30.768 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.768 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.769 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:53:30.769 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.741149+0000", 2023-10-01T10:53:30.769 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999995 2023-10-01T10:53:30.769 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.769 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.769 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:53:30.769 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.741155+0000", 2023-10-01T10:53:30.769 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.8429999999999998e-06 2023-10-01T10:53:30.769 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.769 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.769 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:53:30.769 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.741167+0000", 2023-10-01T10:53:30.770 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.2046e-05 2023-10-01T10:53:30.770 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.770 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.770 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:53:30.770 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.741173+0000", 2023-10-01T10:53:30.770 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.4520000000000002e-06 2023-10-01T10:53:30.770 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.770 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.770 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:53:30.770 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.747019+0000", 2023-10-01T10:53:30.770 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0058456669999999997 2023-10-01T10:53:30.770 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.771 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.771 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:53:30.771 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.747074+0000", 2023-10-01T10:53:30.771 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.4783e-05 2023-10-01T10:53:30.771 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:53:30.771 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:53:30.771 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:53:30.771 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.771 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.771 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.46037.0:136040 11.2 e186/173)", 2023-10-01T10:53:30.771 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:53:30.741242+0000", 2023-10-01T10:53:30.771 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 0.012346954, 2023-10-01T10:53:30.772 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.01252899, 2023-10-01T10:53:30.772 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:53:30.772 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "queued for pg", 2023-10-01T10:53:30.772 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:53:30.772 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.772 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:53:30.772 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.741242+0000", 2023-10-01T10:53:30.772 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:53:30.772 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.772 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.772 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:53:30.772 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.741242+0000", 2023-10-01T10:53:30.773 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:53:30.773 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.773 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.773 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:53:30.773 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.741241+0000", 2023-10-01T10:53:30.773 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999995 2023-10-01T10:53:30.773 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.773 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.773 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:53:30.773 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.741247+0000", 2023-10-01T10:53:30.773 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.4569999999999996e-06 2023-10-01T10:53:30.773 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.774 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.774 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:53:30.774 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.741277+0000", 2023-10-01T10:53:30.774 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.0787000000000002e-05 2023-10-01T10:53:30.774 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.774 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.774 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:53:30.774 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.741284+0000", 2023-10-01T10:53:30.774 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.9399999999999996e-06 2023-10-01T10:53:30.774 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:53:30.775 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:53:30.775 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:53:30.775 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.775 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.775 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46037.0:136054 11.6 11:65195004:::benchmark_data_smithi078_157677_object28883:head [delete] snapc 0=[] ondisk+write+known_if_redirected e186)", 2023-10-01T10:53:30.775 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:53:30.747158+0000", 2023-10-01T10:53:30.775 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 0.0064309479999999997, 2023-10-01T10:53:30.775 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0066454050000000001, 2023-10-01T10:53:30.775 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:53:30.775 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "waiting for sub ops", 2023-10-01T10:53:30.775 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:53:30.776 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46037", 2023-10-01T10:53:30.776 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:53:30.776 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 136054 2023-10-01T10:53:30.776 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.776 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:53:30.776 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.776 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:53:30.776 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.747158+0000", 2023-10-01T10:53:30.776 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:53:30.776 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.776 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.776 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:53:30.777 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.747158+0000", 2023-10-01T10:53:30.777 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:53:30.777 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.777 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.777 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:53:30.777 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.747156+0000", 2023-10-01T10:53:30.777 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999981 2023-10-01T10:53:30.777 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.777 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.777 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:53:30.777 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.747165+0000", 2023-10-01T10:53:30.778 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.8039999999999996e-06 2023-10-01T10:53:30.778 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.778 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.778 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:53:30.778 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.747179+0000", 2023-10-01T10:53:30.778 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.3932e-05 2023-10-01T10:53:30.778 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.778 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.778 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:53:30.778 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.747189+0000", 2023-10-01T10:53:30.778 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.0039e-05 2023-10-01T10:53:30.778 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.779 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.779 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:53:30.779 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.747341+0000", 2023-10-01T10:53:30.779 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00015196099999999999 2023-10-01T10:53:30.779 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.779 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.779 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:53:30.779 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.747726+0000", 2023-10-01T10:53:30.779 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00038474199999999998 2023-10-01T10:53:30.779 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.780 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.780 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 1,2", 2023-10-01T10:53:30.780 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.747854+0000", 2023-10-01T10:53:30.780 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00012776399999999999 2023-10-01T10:53:30.781 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:53:30.781 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:53:30.781 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:53:30.781 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.781 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.781 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.46037.0:136041 11.2 e186/173)", 2023-10-01T10:53:30.781 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:53:30.747262+0000", 2023-10-01T10:53:30.781 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 0.0063267840000000002, 2023-10-01T10:53:30.782 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0065828570000000001, 2023-10-01T10:53:30.782 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:53:30.782 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "queued for pg", 2023-10-01T10:53:30.782 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:53:30.782 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.782 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:53:30.782 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.747262+0000", 2023-10-01T10:53:30.782 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:53:30.782 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.782 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.782 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:53:30.783 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.747262+0000", 2023-10-01T10:53:30.783 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:53:30.783 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.783 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.783 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:53:30.783 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.747251+0000", 2023-10-01T10:53:30.783 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999989 2023-10-01T10:53:30.783 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.783 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.783 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:53:30.783 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.747269+0000", 2023-10-01T10:53:30.783 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.7852000000000001e-05 2023-10-01T10:53:30.784 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.784 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.784 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:53:30.784 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.747288+0000", 2023-10-01T10:53:30.784 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.9408000000000001e-05 2023-10-01T10:53:30.784 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.784 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.784 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:53:30.784 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.747309+0000", 2023-10-01T10:53:30.784 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.1373999999999999e-05 2023-10-01T10:53:30.784 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:53:30.784 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:53:30.785 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:53:30.785 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.785 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.785 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop_reply(client.46037.0:136048 11.6 e186/173)", 2023-10-01T10:53:30.785 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:53:30.747335+0000", 2023-10-01T10:53:30.785 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 0.0062537130000000002, 2023-10-01T10:53:30.785 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0065368149999999996, 2023-10-01T10:53:30.785 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:53:30.785 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "queued for pg", 2023-10-01T10:53:30.785 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:53:30.785 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.786 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:53:30.786 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.747335+0000", 2023-10-01T10:53:30.786 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:53:30.786 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.786 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.786 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:53:30.786 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.747335+0000", 2023-10-01T10:53:30.786 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:53:30.786 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.786 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.786 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:53:30.786 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.747334+0000", 2023-10-01T10:53:30.787 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999995 2023-10-01T10:53:30.787 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.787 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.787 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:53:30.787 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.747339+0000", 2023-10-01T10:53:30.787 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.0130000000000001e-06 2023-10-01T10:53:30.787 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.787 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.787 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:53:30.787 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.747350+0000", 2023-10-01T10:53:30.787 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.059e-05 2023-10-01T10:53:30.788 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.788 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.788 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:53:30.788 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.747357+0000", 2023-10-01T10:53:30.788 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.0319999999999998e-06 2023-10-01T10:53:30.788 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:53:30.788 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:53:30.788 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:53:30.788 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.788 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.788 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46037.0:136055 11.6 11.be2bba26 (undecoded) ondisk+write+known_if_redirected e186)", 2023-10-01T10:53:30.788 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:53:30.747554+0000", 2023-10-01T10:53:30.789 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 0.0060340560000000003, 2023-10-01T10:53:30.789 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0063455120000000002, 2023-10-01T10:53:30.789 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:53:30.789 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "queued for pg", 2023-10-01T10:53:30.789 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:53:30.789 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46037", 2023-10-01T10:53:30.789 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:53:30.789 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 136055 2023-10-01T10:53:30.789 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.789 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:53:30.789 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.789 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:53:30.789 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.747554+0000", 2023-10-01T10:53:30.790 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:53:30.790 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.790 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.790 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:53:30.790 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.747554+0000", 2023-10-01T10:53:30.790 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:53:30.790 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.790 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.790 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:53:30.790 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.747553+0000", 2023-10-01T10:53:30.790 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:53:30.791 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.791 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.791 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:53:30.791 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.747559+0000", 2023-10-01T10:53:30.791 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.7819999999999999e-06 2023-10-01T10:53:30.791 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.791 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.791 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:53:30.791 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.747575+0000", 2023-10-01T10:53:30.791 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.6096e-05 2023-10-01T10:53:30.791 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.791 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.792 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:53:30.792 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.747583+0000", 2023-10-01T10:53:30.792 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.5379999999999997e-06 2023-10-01T10:53:30.792 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:53:30.792 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:53:30.792 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:53:30.792 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.792 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.792 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46037.0:136056 11.6 11.8986306 (undecoded) ondisk+write+known_if_redirected e186)", 2023-10-01T10:53:30.792 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:53:30.747621+0000", 2023-10-01T10:53:30.792 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 0.0059671450000000001, 2023-10-01T10:53:30.793 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0063135149999999996, 2023-10-01T10:53:30.793 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:53:30.793 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "queued for pg", 2023-10-01T10:53:30.793 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:53:30.793 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46037", 2023-10-01T10:53:30.793 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:53:30.793 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 136056 2023-10-01T10:53:30.793 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.793 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:53:30.793 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.793 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:53:30.793 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.747621+0000", 2023-10-01T10:53:30.794 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:53:30.794 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.794 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.794 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:53:30.794 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.747621+0000", 2023-10-01T10:53:30.794 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:53:30.794 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.794 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.794 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:53:30.794 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.747621+0000", 2023-10-01T10:53:30.794 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999995 2023-10-01T10:53:30.795 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.795 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.795 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:53:30.795 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.747624+0000", 2023-10-01T10:53:30.795 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.2210000000000002e-06 2023-10-01T10:53:30.795 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.795 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.795 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:53:30.795 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.747639+0000", 2023-10-01T10:53:30.795 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.4867e-05 2023-10-01T10:53:30.795 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.796 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.796 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:53:30.796 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.747645+0000", 2023-10-01T10:53:30.796 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.6269999999999997e-06 2023-10-01T10:53:30.796 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:53:30.796 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:53:30.796 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:53:30.796 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.796 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.796 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop_reply(client.46037.0:136048 11.6 e186/173)", 2023-10-01T10:53:30.796 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:53:30.747867+0000", 2023-10-01T10:53:30.796 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 0.0057217539999999999, 2023-10-01T10:53:30.797 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.006099284, 2023-10-01T10:53:30.797 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:53:30.797 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "queued for pg", 2023-10-01T10:53:30.797 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:53:30.797 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.797 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:53:30.797 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.747867+0000", 2023-10-01T10:53:30.797 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:53:30.797 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.797 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.797 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:53:30.797 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.747867+0000", 2023-10-01T10:53:30.798 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:53:30.798 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.798 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.798 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:53:30.798 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.747866+0000", 2023-10-01T10:53:30.798 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:53:30.798 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.798 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.799 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:53:30.799 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.747871+0000", 2023-10-01T10:53:30.799 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.0050000000000004e-06 2023-10-01T10:53:30.799 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.799 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.799 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:53:30.799 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.747881+0000", 2023-10-01T10:53:30.799 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.5259999999999999e-06 2023-10-01T10:53:30.799 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.799 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.799 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:53:30.799 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.747895+0000", 2023-10-01T10:53:30.800 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.438e-05 2023-10-01T10:53:30.800 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:53:30.800 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:53:30.800 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:53:30.800 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.800 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.800 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.46037.0:136058 11.9 e186/173 11:96cbe6a0:::benchmark_data_smithi078_157677_object28887:head v 186'8579, mlcod=186'8577)", 2023-10-01T10:53:30.800 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:53:30.749660+0000", 2023-10-01T10:53:30.800 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 0.0039280519999999996, 2023-10-01T10:53:30.800 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0043344810000000003, 2023-10-01T10:53:30.800 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:53:30.800 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "started", 2023-10-01T10:53:30.801 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:53:30.801 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.801 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:53:30.801 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.749660+0000", 2023-10-01T10:53:30.801 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:53:30.801 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.801 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.801 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:53:30.801 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.749660+0000", 2023-10-01T10:53:30.801 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:53:30.801 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.801 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.802 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:53:30.802 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.749660+0000", 2023-10-01T10:53:30.802 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:53:30.802 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.802 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.802 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:53:30.802 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.749668+0000", 2023-10-01T10:53:30.802 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.9209999999999998e-06 2023-10-01T10:53:30.802 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.802 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.802 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:53:30.803 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.749686+0000", 2023-10-01T10:53:30.803 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.7540000000000001e-05 2023-10-01T10:53:30.803 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.803 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.803 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:53:30.803 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.749694+0000", 2023-10-01T10:53:30.803 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.9249999999999995e-06 2023-10-01T10:53:30.803 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.803 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.803 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:53:30.803 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.749764+0000", 2023-10-01T10:53:30.803 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.0179999999999996e-05 2023-10-01T10:53:30.804 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.804 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.804 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:53:30.804 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.752482+0000", 2023-10-01T10:53:30.804 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.002718247 2023-10-01T10:53:30.804 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:53:30.804 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:53:30.804 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:53:30.804 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.804 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.804 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46037.0:136062 11.6 11.547c0f96 (undecoded) ondisk+write+known_if_redirected e186)", 2023-10-01T10:53:30.804 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:53:30.749791+0000", 2023-10-01T10:53:30.805 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 0.0037979569999999998, 2023-10-01T10:53:30.805 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0042386919999999996, 2023-10-01T10:53:30.805 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:53:30.805 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "queued for pg", 2023-10-01T10:53:30.805 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:53:30.805 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46037", 2023-10-01T10:53:30.805 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:53:30.805 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 136062 2023-10-01T10:53:30.805 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.805 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:53:30.805 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.805 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:53:30.806 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.749791+0000", 2023-10-01T10:53:30.806 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:53:30.806 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.806 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.806 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:53:30.806 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.749791+0000", 2023-10-01T10:53:30.806 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:53:30.806 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.806 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.806 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:53:30.806 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.749789+0000", 2023-10-01T10:53:30.807 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999986 2023-10-01T10:53:30.807 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.807 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.807 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:53:30.807 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.749795+0000", 2023-10-01T10:53:30.807 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.4380000000000003e-06 2023-10-01T10:53:30.807 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.807 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.807 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:53:30.807 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.749806+0000", 2023-10-01T10:53:30.807 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.0885e-05 2023-10-01T10:53:30.807 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.808 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.808 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:53:30.808 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.749814+0000", 2023-10-01T10:53:30.808 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.4710000000000007e-06 2023-10-01T10:53:30.808 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:53:30.808 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:53:30.808 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:53:30.808 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.808 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.808 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop_reply(client.46037.0:136054 11.6 e186/173)", 2023-10-01T10:53:30.808 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:53:30.752921+0000", 2023-10-01T10:53:30.808 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 0.000667203, 2023-10-01T10:53:30.809 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0011384500000000001, 2023-10-01T10:53:30.809 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:53:30.809 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "queued for pg", 2023-10-01T10:53:30.809 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:53:30.809 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.809 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:53:30.809 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.752921+0000", 2023-10-01T10:53:30.809 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:53:30.809 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.809 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.809 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:53:30.809 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.752921+0000", 2023-10-01T10:53:30.810 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:53:30.810 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.810 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.810 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:53:30.810 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.752920+0000", 2023-10-01T10:53:30.810 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:53:30.810 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.810 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.810 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:53:30.810 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.752927+0000", 2023-10-01T10:53:30.810 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.0849999999999999e-06 2023-10-01T10:53:30.811 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.811 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.811 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:53:30.811 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.752943+0000", 2023-10-01T10:53:30.811 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.6087e-05 2023-10-01T10:53:30.811 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.811 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.811 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:53:30.811 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.752951+0000", 2023-10-01T10:53:30.811 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.8990000000000001e-06 2023-10-01T10:53:30.811 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:53:30.811 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:53:30.811 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:53:30.812 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.812 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.812 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.46037.0:136035 11.2 e186/173 11:4d04a9f2:::benchmark_data_smithi078_157677_object28864:head v 186'8594, mlcod=186'8590)", 2023-10-01T10:53:30.812 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:53:30.717190+0000", 2023-10-01T10:53:30.812 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 0.036398944000000003, 2023-10-01T10:53:30.812 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.036900075999999997, 2023-10-01T10:53:30.812 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:53:30.812 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "started", 2023-10-01T10:53:30.812 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:53:30.812 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.812 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:53:30.812 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.717190+0000", 2023-10-01T10:53:30.813 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:53:30.813 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.813 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.813 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:53:30.813 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.717190+0000", 2023-10-01T10:53:30.813 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:53:30.813 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.813 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.813 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:53:30.813 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.717189+0000", 2023-10-01T10:53:30.813 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:53:30.813 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.814 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.814 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:53:30.814 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.717197+0000", 2023-10-01T10:53:30.814 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.6829999999999997e-06 2023-10-01T10:53:30.814 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.814 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.814 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:53:30.814 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.717209+0000", 2023-10-01T10:53:30.814 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1950000000000001e-05 2023-10-01T10:53:30.814 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.815 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.815 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:53:30.815 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.717218+0000", 2023-10-01T10:53:30.815 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.1929999999999993e-06 2023-10-01T10:53:30.815 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.815 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.815 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:53:30.815 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.717312+0000", 2023-10-01T10:53:30.815 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.4139000000000001e-05 2023-10-01T10:53:30.815 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:53:30.815 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:53:30.815 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:53:30.816 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:53:30.717336+0000", 2023-10-01T10:53:30.816 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.3955999999999999e-05 2023-10-01T10:53:30.816 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:53:30.816 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:53:30.816 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:53:30.816 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:53:30.816 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2023-10-01T10:53:30.816 INFO:teuthology.orchestra.run.smithi078.stdout: "num_ops": 14 2023-10-01T10:53:30.816 INFO:teuthology.orchestra.run.smithi078.stdout:} 2023-10-01T10:53:30.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:30.817 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:30.818 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:53:30.918 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:30.940 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:30.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:30.946 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:53:31.046 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:31.071 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:31.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:31.074 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:53:31.174 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:31.219 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:31.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:31.222 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:53:31.323 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:31.363 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:31.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:31.368 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:53:31.469 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:31.493 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:31.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:31.500 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:53:31.601 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:31.634 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:31.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:31.641 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:53:31.741 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:31.767 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:31.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:31.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:53:31.806 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:31 smithi173 bash[58816]: cluster 2023-10-01T10:53:29.995480+0000 mgr.y (mgr.25072) 1078 : cluster 0 pgmap v1034: 89 pgs: 89 active+clean; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2023-10-01T10:53:31.871 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:31.881 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:31 smithi078 bash[78274]: cluster 2023-10-01T10:53:29.995480+0000 mgr.y (mgr.25072) 1078 : cluster 0 pgmap v1034: 89 pgs: 89 active+clean; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2023-10-01T10:53:31.881 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:31 smithi078 bash[75299]: cluster 2023-10-01T10:53:29.995480+0000 mgr.y (mgr.25072) 1078 : cluster 0 pgmap v1034: 89 pgs: 89 active+clean; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2023-10-01T10:53:31.903 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:31.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:31.906 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:53:31.988 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-01T10:53:32.007 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:32.039 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:32.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:32.043 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:53:32.143 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:32.179 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:32.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:32.181 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:53:32.282 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:32.326 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:32.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:32.328 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:53:32.429 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:32.455 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:32.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:32.458 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:53:32.559 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:32.592 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:32.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:32.597 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:53:32.698 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:32.723 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:32.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:32.728 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:53:32.815 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:32 smithi078 bash[75299]: audit 2023-10-01T10:53:32.439534+0000 mon.a (mon.0) 1325 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:53:32.829 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:32.865 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:32.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:32.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:53:32.968 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:32.977 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:32 smithi173 bash[58816]: audit 2023-10-01T10:53:32.439534+0000 mon.a (mon.0) 1325 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:53:32.991 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:32.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:32.994 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:53:33.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:32 smithi078 bash[78274]: audit 2023-10-01T10:53:32.439534+0000 mon.a (mon.0) 1325 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:53:33.095 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:33.117 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:33.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:33.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:53:33.220 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:33.250 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:33.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:33.256 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:53:33.357 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:33.394 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:33.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:33.402 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:33.492 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:53:33.493 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:53:33.498 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:53:33.503 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:33.533 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:33.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:33.536 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:53:33.637 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:33.675 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:33.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:33.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:33.782 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:33.807 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:33.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:33.809 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:33.910 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:33.931 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:33 smithi078 bash[75299]: cluster 2023-10-01T10:53:31.995856+0000 mgr.y (mgr.25072) 1079 : cluster 0 pgmap v1035: 89 pgs: 89 active+clean; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2023-10-01T10:53:33.931 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:33 smithi078 bash[78274]: cluster 2023-10-01T10:53:31.995856+0000 mgr.y (mgr.25072) 1079 : cluster 0 pgmap v1035: 89 pgs: 89 active+clean; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2023-10-01T10:53:33.939 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:33.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:33.944 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:53:34.044 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:34.073 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:34.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:34.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:53:34.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:33 smithi173 bash[58816]: cluster 2023-10-01T10:53:31.995856+0000 mgr.y (mgr.25072) 1079 : cluster 0 pgmap v1035: 89 pgs: 89 active+clean; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2023-10-01T10:53:34.177 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:34.202 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:34.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:34.209 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:53:34.310 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:34.340 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:34.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:34.344 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:53:34.445 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:34.478 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:34.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:34.480 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:53:34.581 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:34.626 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:34.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:34.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:53:34.731 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:34.756 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:34.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:34.758 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:34.859 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:34.884 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:34.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:34.887 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:34.988 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:35.019 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:35.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:35.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:53:35.126 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:35.170 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:35.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:35.174 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:53:35.274 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:35.308 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:35.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:35.311 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:35.411 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:35.436 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:35.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:35.439 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:53:35.540 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:35.565 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:35.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:35.571 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:35.672 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:35.701 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:35.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:35.702 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:53:35.803 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:35.825 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:35 smithi078 bash[78274]: cluster 2023-10-01T10:53:34.001370+0000 mgr.y (mgr.25072) 1080 : cluster 0 pgmap v1036: 89 pgs: 89 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 655 op/s 2023-10-01T10:53:35.825 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:35 smithi078 bash[75299]: cluster 2023-10-01T10:53:34.001370+0000 mgr.y (mgr.25072) 1080 : cluster 0 pgmap v1036: 89 pgs: 89 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 655 op/s 2023-10-01T10:53:35.830 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:35.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:35.837 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:53:35.938 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:35.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:35.970 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:35.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:53:36.071 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:36.098 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:36.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:36.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:53:36.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:35 smithi173 bash[58816]: cluster 2023-10-01T10:53:34.001370+0000 mgr.y (mgr.25072) 1080 : cluster 0 pgmap v1036: 89 pgs: 89 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 655 op/s 2023-10-01T10:53:36.202 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:36.231 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:36.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:36.236 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:36.337 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:36.361 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:36.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:36.366 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:36.467 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:36.501 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:36.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:36.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:53:36.549 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:53:36.609 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:36.654 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:36.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:36.661 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:36.762 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:36.790 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:36.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:36.795 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:53:36.895 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:36.923 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:36.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:36.926 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:53:37.027 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:37.062 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:37.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:37.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:53:37.169 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:37.199 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:37.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:37.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:53:37.305 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:37.337 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:37.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:37.338 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:53:37.439 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:37.470 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:37.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:37.476 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:53:37.576 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:37.610 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:37.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:37.619 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:53:37.719 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:37.738 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:37.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:37.741 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:37.842 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:37.851 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:37 smithi173 bash[58816]: cluster 2023-10-01T10:53:36.001924+0000 mgr.y (mgr.25072) 1081 : cluster 0 pgmap v1037: 89 pgs: 89 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 604 op/s 2023-10-01T10:53:37.884 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:37.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:37.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:53:37.988 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:38.017 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:38.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:38.021 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:53:38.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:37 smithi078 bash[78274]: cluster 2023-10-01T10:53:36.001924+0000 mgr.y (mgr.25072) 1081 : cluster 0 pgmap v1037: 89 pgs: 89 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 604 op/s 2023-10-01T10:53:38.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:37 smithi078 bash[75299]: cluster 2023-10-01T10:53:36.001924+0000 mgr.y (mgr.25072) 1081 : cluster 0 pgmap v1037: 89 pgs: 89 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 604 op/s 2023-10-01T10:53:38.122 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:38.152 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:38.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:38.164 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:53:38.265 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:38.296 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:38.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:38.301 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:53:38.401 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:38.436 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:38.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:38.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:53:38.543 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:38.563 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:38.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:38.567 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:38.668 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:38.697 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:38.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:38.703 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:53:38.804 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:38.836 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:38.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:38.838 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:38.939 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:38.971 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:38.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:38.975 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:53:39.075 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:39.103 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:39.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:39.107 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:53:39.208 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:39.242 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:39.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:39.244 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:53:39.345 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:39.378 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:39.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:39.382 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:53:39.483 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:39.518 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:39.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:39.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:53:39.624 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:39.655 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:39.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:39.660 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:39.761 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:39.798 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:39.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:39.804 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:53:39.905 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:39.931 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:39.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:39.937 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:53:40.038 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:40.046 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:39 smithi078 bash[75299]: cluster 2023-10-01T10:53:38.002454+0000 mgr.y (mgr.25072) 1082 : cluster 0 pgmap v1038: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 682 op/s 2023-10-01T10:53:40.047 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:39 smithi078 bash[78274]: cluster 2023-10-01T10:53:38.002454+0000 mgr.y (mgr.25072) 1082 : cluster 0 pgmap v1038: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 682 op/s 2023-10-01T10:53:40.066 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:40.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:40.070 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:53:40.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:39 smithi173 bash[58816]: cluster 2023-10-01T10:53:38.002454+0000 mgr.y (mgr.25072) 1082 : cluster 0 pgmap v1038: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 682 op/s 2023-10-01T10:53:40.171 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:40.198 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:40.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:40.203 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:40.304 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:40.312 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:53:40 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:53:40] "GET /metrics HTTP/1.1" 200 36313 "" "Prometheus/2.33.4" 2023-10-01T10:53:40.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:40.330 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:40.330 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:40.431 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:40.460 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:40.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:40.464 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:40.565 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:40.590 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:40.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:40.592 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:53:40.693 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:40.719 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:40.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:40.722 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:53:40.823 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:40.848 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:40.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:40.854 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:53:40.954 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:41.003 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:41.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:41.006 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:53:41.106 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:41.139 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:41.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:41.146 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:53:41.247 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:41.274 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:41.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:41.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:53:41.380 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:41.408 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:41.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:41.414 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:53:41.515 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:41.540 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:41.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:41.544 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:53:41.645 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:41.688 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:41.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:41.692 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:53:41.793 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:41.819 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:41.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:41.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:53:41.926 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:41.945 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:41.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:41.948 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:53:42.049 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:42.058 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:41 smithi173 bash[58816]: cluster 2023-10-01T10:53:40.004765+0000 mgr.y (mgr.25072) 1083 : cluster 0 pgmap v1039: 89 pgs: 89 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 764 op/s 2023-10-01T10:53:42.075 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:42.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:42.083 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:53:42.183 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:42.192 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:41 smithi078 bash[75299]: cluster 2023-10-01T10:53:40.004765+0000 mgr.y (mgr.25072) 1083 : cluster 0 pgmap v1039: 89 pgs: 89 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 764 op/s 2023-10-01T10:53:42.192 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:41 smithi078 bash[78274]: cluster 2023-10-01T10:53:40.004765+0000 mgr.y (mgr.25072) 1083 : cluster 0 pgmap v1039: 89 pgs: 89 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 764 op/s 2023-10-01T10:53:42.211 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:42.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:42.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:53:42.318 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:42.338 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:42.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:42.341 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:42.442 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:42.467 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:42.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:42.470 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:53:42.571 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:42.600 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:42.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:42.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:53:42.703 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:42.733 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:42.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:42.738 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:53:42.839 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:42.874 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:42.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:42.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:53:42.980 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:43.010 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:43.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:43.018 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:53:43.119 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:43.155 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:43.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:43.158 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:53:43.259 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:43.287 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:43.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:43.292 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:53:43.393 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:43.415 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:43.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:43.417 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:53:43.518 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:43.552 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:43.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:43.555 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:53:43.655 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:43.680 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:43.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:43.685 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:53:43.786 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:43.815 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:43.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:43.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:53:43.919 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:43.954 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:43.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:43.956 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:53:44.057 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:44.085 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:44.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:44.088 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:53:44.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:43 smithi173 bash[58816]: cluster 2023-10-01T10:53:42.005541+0000 mgr.y (mgr.25072) 1084 : cluster 0 pgmap v1040: 89 pgs: 89 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2023-10-01T10:53:44.188 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:44.196 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:43 smithi078 bash[78274]: cluster 2023-10-01T10:53:42.005541+0000 mgr.y (mgr.25072) 1084 : cluster 0 pgmap v1040: 89 pgs: 89 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2023-10-01T10:53:44.197 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:43 smithi078 bash[75299]: cluster 2023-10-01T10:53:42.005541+0000 mgr.y (mgr.25072) 1084 : cluster 0 pgmap v1040: 89 pgs: 89 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 591 op/s 2023-10-01T10:53:44.213 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:44.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:44.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:53:44.317 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:44.344 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:44.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:44.347 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:53:44.448 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:44.472 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:44.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:44.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:53:44.578 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:44.608 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:44.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:44.610 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:44.711 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:44.735 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:44.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:44.739 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:44.839 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:44.864 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:44.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:44.868 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:44.968 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:44.996 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:45.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:45.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:53:45.103 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:45.139 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:45.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:45.144 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:53:45.245 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:45.273 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:45.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:45.279 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:53:45.379 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:45.404 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:45.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:45.410 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:53:45.510 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:45.557 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:45.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:45.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:53:45.659 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:45.688 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:45.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:45.700 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:53:45.800 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:45.830 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:45.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:45.834 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:53:45.935 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:45.972 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:45.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:45.974 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:53:46.075 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:46.080 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:45 smithi078 bash[78274]: cluster 2023-10-01T10:53:44.006202+0000 mgr.y (mgr.25072) 1085 : cluster 0 pgmap v1041: 89 pgs: 89 active+clean; 4.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 836 op/s 2023-10-01T10:53:46.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:45 smithi078 bash[75299]: cluster 2023-10-01T10:53:44.006202+0000 mgr.y (mgr.25072) 1085 : cluster 0 pgmap v1041: 89 pgs: 89 active+clean; 4.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 836 op/s 2023-10-01T10:53:46.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:46.105 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:46.105 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:53:46.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:45 smithi173 bash[58816]: cluster 2023-10-01T10:53:44.006202+0000 mgr.y (mgr.25072) 1085 : cluster 0 pgmap v1041: 89 pgs: 89 active+clean; 4.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 836 op/s 2023-10-01T10:53:46.206 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:46.232 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:46.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:46.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:46.338 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:46.366 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:46.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:46.370 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:53:46.471 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:46.503 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:46.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:46.510 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:53:46.610 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:46.639 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:46.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:46.641 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:53:46.742 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:46.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:46.766 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:46.766 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:53:46.867 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:46.909 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:46.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:46.914 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:53:47.014 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:47.048 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:47.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:47.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:53:47.154 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:47.176 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:47.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:47.179 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:53:47.280 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:47.317 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:47.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:47.324 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:53:47.424 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:47.459 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:47.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:47.462 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:53:47.563 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:47.595 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:47.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:47.600 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:53:47.700 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:47.731 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:47.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:47.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:53:47.836 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:47.882 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:47.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:47.884 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:53:47.984 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:47.992 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:47 smithi173 bash[58816]: cluster 2023-10-01T10:53:46.006768+0000 mgr.y (mgr.25072) 1086 : cluster 0 pgmap v1042: 89 pgs: 89 active+clean; 4.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 624 op/s 2023-10-01T10:53:47.993 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:47 smithi173 bash[58816]: audit 2023-10-01T10:53:47.439637+0000 mon.a (mon.0) 1326 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:53:48.014 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:48.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:48.017 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:53:48.117 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:48.126 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:47 smithi078 bash[75299]: cluster 2023-10-01T10:53:46.006768+0000 mgr.y (mgr.25072) 1086 : cluster 0 pgmap v1042: 89 pgs: 89 active+clean; 4.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 624 op/s 2023-10-01T10:53:48.126 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:47 smithi078 bash[75299]: audit 2023-10-01T10:53:47.439637+0000 mon.a (mon.0) 1326 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:53:48.127 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:47 smithi078 bash[78274]: cluster 2023-10-01T10:53:46.006768+0000 mgr.y (mgr.25072) 1086 : cluster 0 pgmap v1042: 89 pgs: 89 active+clean; 4.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 624 op/s 2023-10-01T10:53:48.127 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:47 smithi078 bash[78274]: audit 2023-10-01T10:53:47.439637+0000 mon.a (mon.0) 1326 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:53:48.148 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:48.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:48.150 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:53:48.251 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:48.284 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:48.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:48.289 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:53:48.390 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:48.424 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:48.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:48.432 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:53:48.533 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:48.555 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:48.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:48.559 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:53:48.660 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:48.692 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:48.693 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T10:53:48.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:48.699 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:53:48.757 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T10:53:48.799 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:48.817 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T10:53:48.845 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:48.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:48.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:53:48.865 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:48 smithi173 bash[58816]: audit 2023-10-01T10:53:48.596946+0000 mon.a (mon.0) 1327 : audit 0 from='client.? 172.21.15.78:0/3583330691' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:53:48.865 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:48 smithi173 bash[58816]: audit 2023-10-01T10:53:48.648886+0000 mon.a (mon.0) 1328 : audit 1 from='client.? 172.21.15.78:0/1478790856' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T10:53:48.869 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T10:53:48.892 INFO:teuthology.orchestra.run.smithi078.stderr:noscrub is unset 2023-10-01T10:53:48.921 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T10:53:48.932 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:48 smithi078 bash[78274]: audit 2023-10-01T10:53:48.596946+0000 mon.a (mon.0) 1327 : audit 0 from='client.? 172.21.15.78:0/3583330691' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:53:48.933 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:48 smithi078 bash[78274]: audit 2023-10-01T10:53:48.648886+0000 mon.a (mon.0) 1328 : audit 1 from='client.? 172.21.15.78:0/1478790856' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T10:53:48.934 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:48 smithi078 bash[75299]: audit 2023-10-01T10:53:48.596946+0000 mon.a (mon.0) 1327 : audit 0 from='client.? 172.21.15.78:0/3583330691' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:53:48.934 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:48 smithi078 bash[75299]: audit 2023-10-01T10:53:48.648886+0000 mon.a (mon.0) 1328 : audit 1 from='client.? 172.21.15.78:0/1478790856' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T10:53:48.949 INFO:teuthology.orchestra.run.smithi078.stdout:ERROR: (22) Invalid argument 2023-10-01T10:53:48.949 INFO:teuthology.orchestra.run.smithi078.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T10:53:48.952 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:48.968 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T10:53:48.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:48.994 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:48.994 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:53:49.007 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T10:53:49.053 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T10:53:49.064 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T10:53:49.064 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T10:53:49.064 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T10:53:49.064 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T10:53:49.064 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T10:53:49.064 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T10:53:49.064 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T10:53:49.065 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T10:53:49.095 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:49.119 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:49.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:49.122 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:53:49.222 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:49.244 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:49.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:49.246 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:53:49.347 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:49.369 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:49.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:49.376 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:53:49.408 INFO:teuthology.orchestra.run.smithi078.stdout: 2023-10-01T10:53:49.412 INFO:teuthology.orchestra.run.smithi078.stderr:dumped all 2023-10-01T10:53:49.477 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:49.504 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:49.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:49.513 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:49.613 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:49.645 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:49.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:49.649 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:53:49.750 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:49.781 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:49.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:49.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:49.885 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:49.899 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:49 smithi173 bash[58816]: cluster 2023-10-01T10:53:48.007299+0000 mgr.y (mgr.25072) 1087 : cluster 0 pgmap v1043: 89 pgs: 89 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 707 op/s 2023-10-01T10:53:49.900 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:49 smithi173 bash[58816]: cluster 2023-10-01T10:53:48.851624+0000 mon.a (mon.0) 1329 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:53:49.900 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:49 smithi173 bash[58816]: cluster 2023-10-01T10:53:48.872485+0000 osd.7 (osd.7) 146 : cluster 3 failed to encode map e187 with expected crc 2023-10-01T10:53:49.900 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:49 smithi173 bash[58816]: audit 2023-10-01T10:53:48.877492+0000 mon.a (mon.0) 1330 : audit 1 from='client.? 172.21.15.78:0/1478790856' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T10:53:49.900 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:49 smithi173 bash[58816]: cluster 2023-10-01T10:53:48.877813+0000 mon.a (mon.0) 1331 : cluster 0 osdmap e187: 8 total, 8 up, 8 in 2023-10-01T10:53:49.900 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:49 smithi173 bash[58816]: cluster 2023-10-01T10:53:48.887378+0000 osd.6 (osd.6) 109 : cluster 3 failed to encode map e187 with expected crc 2023-10-01T10:53:49.900 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:49 smithi173 bash[58816]: cluster 2023-10-01T10:53:48.891418+0000 osd.5 (osd.5) 127 : cluster 3 failed to encode map e187 with expected crc 2023-10-01T10:53:49.900 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:49 smithi173 bash[58816]: cluster 2023-10-01T10:53:48.893623+0000 osd.5 (osd.5) 128 : cluster 3 failed to encode map e187 with expected crc 2023-10-01T10:53:49.926 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:50.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:50.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:49 smithi078 bash[75299]: cluster 2023-10-01T10:53:48.007299+0000 mgr.y (mgr.25072) 1087 : cluster 0 pgmap v1043: 89 pgs: 89 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 707 op/s 2023-10-01T10:53:50.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:49 smithi078 bash[75299]: cluster 2023-10-01T10:53:48.851624+0000 mon.a (mon.0) 1329 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:53:50.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:49 smithi078 bash[75299]: cluster 2023-10-01T10:53:48.872485+0000 osd.7 (osd.7) 146 : cluster 3 failed to encode map e187 with expected crc 2023-10-01T10:53:50.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:49 smithi078 bash[75299]: audit 2023-10-01T10:53:48.877492+0000 mon.a (mon.0) 1330 : audit 1 from='client.? 172.21.15.78:0/1478790856' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T10:53:50.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:49 smithi078 bash[75299]: cluster 2023-10-01T10:53:48.877813+0000 mon.a (mon.0) 1331 : cluster 0 osdmap e187: 8 total, 8 up, 8 in 2023-10-01T10:53:50.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:49 smithi078 bash[75299]: cluster 2023-10-01T10:53:48.887378+0000 osd.6 (osd.6) 109 : cluster 3 failed to encode map e187 with expected crc 2023-10-01T10:53:50.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:49 smithi078 bash[75299]: cluster 2023-10-01T10:53:48.891418+0000 osd.5 (osd.5) 127 : cluster 3 failed to encode map e187 with expected crc 2023-10-01T10:53:50.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:49 smithi078 bash[75299]: cluster 2023-10-01T10:53:48.893623+0000 osd.5 (osd.5) 128 : cluster 3 failed to encode map e187 with expected crc 2023-10-01T10:53:50.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:49 smithi078 bash[78274]: cluster 2023-10-01T10:53:48.007299+0000 mgr.y (mgr.25072) 1087 : cluster 0 pgmap v1043: 89 pgs: 89 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 707 op/s 2023-10-01T10:53:50.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:49 smithi078 bash[78274]: cluster 2023-10-01T10:53:48.851624+0000 mon.a (mon.0) 1329 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:53:50.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:49 smithi078 bash[78274]: cluster 2023-10-01T10:53:48.872485+0000 osd.7 (osd.7) 146 : cluster 3 failed to encode map e187 with expected crc 2023-10-01T10:53:50.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:49 smithi078 bash[78274]: audit 2023-10-01T10:53:48.877492+0000 mon.a (mon.0) 1330 : audit 1 from='client.? 172.21.15.78:0/1478790856' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T10:53:50.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:49 smithi078 bash[78274]: cluster 2023-10-01T10:53:48.877813+0000 mon.a (mon.0) 1331 : cluster 0 osdmap e187: 8 total, 8 up, 8 in 2023-10-01T10:53:50.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:49 smithi078 bash[78274]: cluster 2023-10-01T10:53:48.887378+0000 osd.6 (osd.6) 109 : cluster 3 failed to encode map e187 with expected crc 2023-10-01T10:53:50.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:49 smithi078 bash[78274]: cluster 2023-10-01T10:53:48.891418+0000 osd.5 (osd.5) 127 : cluster 3 failed to encode map e187 with expected crc 2023-10-01T10:53:50.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:49 smithi078 bash[78274]: cluster 2023-10-01T10:53:48.893623+0000 osd.5 (osd.5) 128 : cluster 3 failed to encode map e187 with expected crc 2023-10-01T10:53:50.336 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:53:50.437 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:50.469 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:50.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:50.480 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:53:50.581 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:50.584 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:53:50 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:53:50] "GET /metrics HTTP/1.1" 200 36317 "" "Prometheus/2.33.4" 2023-10-01T10:53:50.611 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:50.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:50.616 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:53:50.716 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:50.753 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:50.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:50.756 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:53:50.856 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:50.887 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:50.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:50.890 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:53:50.990 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:51.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:51.030 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:51.030 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:53:51.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:50 smithi173 bash[58816]: audit 2023-10-01T10:53:49.388181+0000 mgr.y (mgr.25072) 1088 : audit 0 from='client.46118 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:53:51.131 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:51.143 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:50 smithi078 bash[78274]: audit 2023-10-01T10:53:49.388181+0000 mgr.y (mgr.25072) 1088 : audit 0 from='client.46118 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:53:51.144 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:50 smithi078 bash[75299]: audit 2023-10-01T10:53:49.388181+0000 mgr.y (mgr.25072) 1088 : audit 0 from='client.46118 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:53:51.173 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:51.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:51.175 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:51.275 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:51.305 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:51.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:51.312 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:53:51.412 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:51.436 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:51.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:51.440 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:53:51.540 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:51.558 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:51.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:51.560 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:53:51.660 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:51.687 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:51.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:51.689 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:53:51.790 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:51.821 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:51.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:51.826 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:53:51.927 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:51.962 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:51.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:51.966 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:53:52.067 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:52.076 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:51 smithi173 bash[58816]: cluster 2023-10-01T10:53:50.007928+0000 mgr.y (mgr.25072) 1089 : cluster 0 pgmap v1045: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 696 op/s 2023-10-01T10:53:52.080 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:51 smithi078 bash[78274]: cluster 2023-10-01T10:53:50.007928+0000 mgr.y (mgr.25072) 1089 : cluster 0 pgmap v1045: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 696 op/s 2023-10-01T10:53:52.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:51 smithi078 bash[75299]: cluster 2023-10-01T10:53:50.007928+0000 mgr.y (mgr.25072) 1089 : cluster 0 pgmap v1045: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 696 op/s 2023-10-01T10:53:52.098 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:52.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:52.103 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:53:52.204 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:52.247 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:52.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:52.249 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:53:52.350 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:52.379 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:52.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:52.385 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:53:52.486 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:52.516 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:52.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:52.520 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:53:52.620 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:52.650 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:52.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:52.653 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:53:52.754 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:52.785 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:52.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:52.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:53:52.892 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:52.918 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:52.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:52.921 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:53.022 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:53.050 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:53.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:53.052 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:53:53.153 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:53.177 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:53.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:53.181 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:53:53.225 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-01T10:53:53.238 INFO:teuthology.orchestra.run.smithi078.stdout:{"pg_ready":true,"pg_map":{"version":1044,"stamp":"2023-10-01T10:53:48.862568+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4362737592,"num_objects":66586,"num_object_clones":0,"num_object_copies":199758,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":66586,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":148037,"num_write_kb":6861199,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":137571,"ondisk_log_size":137571,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":17918216,"kb_used_data":12906120,"kb_used_omap":30,"kb_used_meta":5011809,"kb_avail":732043000,"statfs":{"total":767960285184,"available":749612032000,"internally_reserved":0,"allocated":13215866880,"data_stored":13210700165,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31502,"internal_metadata":5132092658},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,2,9,8,15,6,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-577961984,"num_objects":-8819,"num_object_clones":0,"num_object_copies":-26457,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-8819,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8819,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.860449"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":178,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:14.021174+0000","last_change":"2023-10-01T10:38:16.032258+0000","last_active":"2023-10-01T10:53:14.021174+0000","last_peered":"2023-10-01T10:53:14.021174+0000","last_clean":"2023-10-01T10:53:14.021174+0000","last_became_active":"2023-10-01T10:38:16.031633+0000","last_became_peered":"2023-10-01T10:38:16.031633+0000","last_unstale":"2023-10-01T10:53:14.021174+0000","last_undegraded":"2023-10-01T10:53:14.021174+0000","last_fullsized":"2023-10-01T10:53:14.021174+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:17:02.446430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":166,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:04.079901+0000","last_change":"2023-10-01T10:38:16.025881+0000","last_active":"2023-10-01T10:53:04.079901+0000","last_peered":"2023-10-01T10:53:04.079901+0000","last_clean":"2023-10-01T10:53:04.079901+0000","last_became_active":"2023-10-01T10:38:16.025518+0000","last_became_peered":"2023-10-01T10:38:16.025518+0000","last_unstale":"2023-10-01T10:53:04.079901+0000","last_undegraded":"2023-10-01T10:53:04.079901+0000","last_fullsized":"2023-10-01T10:53:04.079901+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:48:37.826604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":86,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:04.079286+0000","last_change":"2023-10-01T10:52:18.956544+0000","last_active":"2023-10-01T10:53:04.079286+0000","last_peered":"2023-10-01T10:53:04.079286+0000","last_clean":"2023-10-01T10:53:04.079286+0000","last_became_active":"2023-10-01T10:52:18.956217+0000","last_became_peered":"2023-10-01T10:52:18.956217+0000","last_unstale":"2023-10-01T10:53:04.079286+0000","last_undegraded":"2023-10-01T10:53:04.079286+0000","last_fullsized":"2023-10-01T10:53:04.079286+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:34:01.952235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":149,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:04.083791+0000","last_change":"2023-10-01T10:37:42.772008+0000","last_active":"2023-10-01T10:53:04.083791+0000","last_peered":"2023-10-01T10:53:04.083791+0000","last_clean":"2023-10-01T10:53:04.083791+0000","last_became_active":"2023-10-01T10:37:42.771244+0000","last_became_peered":"2023-10-01T10:37:42.771244+0000","last_unstale":"2023-10-01T10:53:04.083791+0000","last_undegraded":"2023-10-01T10:53:04.083791+0000","last_fullsized":"2023-10-01T10:53:04.083791+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:38:55.180790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":167,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:04.079089+0000","last_change":"2023-10-01T10:37:42.766600+0000","last_active":"2023-10-01T10:53:04.079089+0000","last_peered":"2023-10-01T10:53:04.079089+0000","last_clean":"2023-10-01T10:53:04.079089+0000","last_became_active":"2023-10-01T10:37:42.766051+0000","last_became_peered":"2023-10-01T10:37:42.766051+0000","last_unstale":"2023-10-01T10:53:04.079089+0000","last_undegraded":"2023-10-01T10:53:04.079089+0000","last_fullsized":"2023-10-01T10:53:04.079089+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:51:17.707963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":151,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:04.065644+0000","last_change":"2023-10-01T10:32:31.575109+0000","last_active":"2023-10-01T10:53:04.065644+0000","last_peered":"2023-10-01T10:53:04.065644+0000","last_clean":"2023-10-01T10:53:04.065644+0000","last_became_active":"2023-10-01T10:32:31.574729+0000","last_became_peered":"2023-10-01T10:32:31.574729+0000","last_unstale":"2023-10-01T10:53:04.065644+0000","last_undegraded":"2023-10-01T10:53:04.065644+0000","last_fullsized":"2023-10-01T10:53:04.065644+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":146,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:04.083503+0000","last_change":"2023-10-01T10:37:42.766667+0000","last_active":"2023-10-01T10:53:04.083503+0000","last_peered":"2023-10-01T10:53:04.083503+0000","last_clean":"2023-10-01T10:53:04.083503+0000","last_became_active":"2023-10-01T10:37:42.766369+0000","last_became_peered":"2023-10-01T10:37:42.766369+0000","last_unstale":"2023-10-01T10:53:04.083503+0000","last_undegraded":"2023-10-01T10:53:04.083503+0000","last_fullsized":"2023-10-01T10:53:04.083503+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:02.708788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":151,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:04.078998+0000","last_change":"2023-10-01T10:37:42.765925+0000","last_active":"2023-10-01T10:53:04.078998+0000","last_peered":"2023-10-01T10:53:04.078998+0000","last_clean":"2023-10-01T10:53:04.078998+0000","last_became_active":"2023-10-01T10:37:42.765218+0000","last_became_peered":"2023-10-01T10:37:42.765218+0000","last_unstale":"2023-10-01T10:53:04.078998+0000","last_undegraded":"2023-10-01T10:53:04.078998+0000","last_fullsized":"2023-10-01T10:53:04.078998+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T22:10:34.338031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":153,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:04.083851+0000","last_change":"2023-10-01T10:32:31.579805+0000","last_active":"2023-10-01T10:53:04.083851+0000","last_peered":"2023-10-01T10:53:04.083851+0000","last_clean":"2023-10-01T10:53:04.083851+0000","last_became_active":"2023-10-01T10:32:31.579423+0000","last_became_peered":"2023-10-01T10:32:31.579423+0000","last_unstale":"2023-10-01T10:53:04.083851+0000","last_undegraded":"2023-10-01T10:53:04.083851+0000","last_fullsized":"2023-10-01T10:53:04.083851+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":135,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:04.079222+0000","last_change":"2023-10-01T10:37:42.772531+0000","last_active":"2023-10-01T10:53:04.079222+0000","last_peered":"2023-10-01T10:53:04.079222+0000","last_clean":"2023-10-01T10:53:04.079222+0000","last_became_active":"2023-10-01T10:37:42.772216+0000","last_became_peered":"2023-10-01T10:37:42.772216+0000","last_unstale":"2023-10-01T10:53:04.079222+0000","last_undegraded":"2023-10-01T10:53:04.079222+0000","last_fullsized":"2023-10-01T10:53:04.079222+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:01:53.373628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":25,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:04.079827+0000","last_change":"2023-10-01T10:52:43.661650+0000","last_active":"2023-10-01T10:53:04.079827+0000","last_peered":"2023-10-01T10:53:04.079827+0000","last_clean":"2023-10-01T10:53:04.079827+0000","last_became_active":"2023-10-01T10:52:43.661202+0000","last_became_peered":"2023-10-01T10:52:43.661202+0000","last_unstale":"2023-10-01T10:53:04.079827+0000","last_undegraded":"2023-10-01T10:53:04.079827+0000","last_fullsized":"2023-10-01T10:53:04.079827+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:28:35.479124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":60,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:04.065529+0000","last_change":"2023-10-01T10:52:18.966695+0000","last_active":"2023-10-01T10:53:04.065529+0000","last_peered":"2023-10-01T10:53:04.065529+0000","last_clean":"2023-10-01T10:53:04.065529+0000","last_became_active":"2023-10-01T10:52:18.966450+0000","last_became_peered":"2023-10-01T10:52:18.966450+0000","last_unstale":"2023-10-01T10:53:04.065529+0000","last_undegraded":"2023-10-01T10:53:04.065529+0000","last_fullsized":"2023-10-01T10:53:04.065529+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":172,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:04.054308+0000","last_change":"2023-10-01T10:38:16.030826+0000","last_active":"2023-10-01T10:53:04.054308+0000","last_peered":"2023-10-01T10:53:04.054308+0000","last_clean":"2023-10-01T10:53:04.054308+0000","last_became_active":"2023-10-01T10:38:16.030277+0000","last_became_peered":"2023-10-01T10:38:16.030277+0000","last_unstale":"2023-10-01T10:53:04.054308+0000","last_undegraded":"2023-10-01T10:53:04.054308+0000","last_fullsized":"2023-10-01T10:53:04.054308+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:31:43.040424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":146,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:04.054161+0000","last_change":"2023-10-01T10:38:16.034741+0000","last_active":"2023-10-01T10:53:04.054161+0000","last_peered":"2023-10-01T10:53:04.054161+0000","last_clean":"2023-10-01T10:53:04.054161+0000","last_became_active":"2023-10-01T10:38:16.034407+0000","last_became_peered":"2023-10-01T10:38:16.034407+0000","last_unstale":"2023-10-01T10:53:04.054161+0000","last_undegraded":"2023-10-01T10:53:04.054161+0000","last_fullsized":"2023-10-01T10:53:04.054161+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:47:29.911477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":169,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:04.061050+0000","last_change":"2023-10-01T10:38:16.026777+0000","last_active":"2023-10-01T10:53:04.061050+0000","last_peered":"2023-10-01T10:53:04.061050+0000","last_clean":"2023-10-01T10:53:04.061050+0000","last_became_active":"2023-10-01T10:38:16.026371+0000","last_became_peered":"2023-10-01T10:38:16.026371+0000","last_unstale":"2023-10-01T10:53:04.061050+0000","last_undegraded":"2023-10-01T10:53:04.061050+0000","last_fullsized":"2023-10-01T10:53:04.061050+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":174,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:04.079925+0000","last_change":"2023-10-01T10:37:42.772319+0000","last_active":"2023-10-01T10:53:04.079925+0000","last_peered":"2023-10-01T10:53:04.079925+0000","last_clean":"2023-10-01T10:53:04.079925+0000","last_became_active":"2023-10-01T10:37:42.771977+0000","last_became_peered":"2023-10-01T10:37:42.771977+0000","last_unstale":"2023-10-01T10:53:04.079925+0000","last_undegraded":"2023-10-01T10:53:04.079925+0000","last_fullsized":"2023-10-01T10:53:04.079925+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:45:12.783214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":146,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:04.085419+0000","last_change":"2023-10-01T10:37:42.767706+0000","last_active":"2023-10-01T10:53:04.085419+0000","last_peered":"2023-10-01T10:53:04.085419+0000","last_clean":"2023-10-01T10:53:04.085419+0000","last_became_active":"2023-10-01T10:37:42.767211+0000","last_became_peered":"2023-10-01T10:37:42.767211+0000","last_unstale":"2023-10-01T10:53:04.085419+0000","last_undegraded":"2023-10-01T10:53:04.085419+0000","last_fullsized":"2023-10-01T10:53:04.085419+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:37:20.357407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":152,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:04.083840+0000","last_change":"2023-10-01T10:37:42.768642+0000","last_active":"2023-10-01T10:53:04.083840+0000","last_peered":"2023-10-01T10:53:04.083840+0000","last_clean":"2023-10-01T10:53:04.083840+0000","last_became_active":"2023-10-01T10:37:42.768239+0000","last_became_peered":"2023-10-01T10:37:42.768239+0000","last_unstale":"2023-10-01T10:53:04.083840+0000","last_undegraded":"2023-10-01T10:53:04.083840+0000","last_fullsized":"2023-10-01T10:53:04.083840+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":99,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:04.053760+0000","last_change":"2023-10-01T10:52:43.661451+0000","last_active":"2023-10-01T10:53:04.053760+0000","last_peered":"2023-10-01T10:53:04.053760+0000","last_clean":"2023-10-01T10:53:04.053760+0000","last_became_active":"2023-10-01T10:52:43.661095+0000","last_became_peered":"2023-10-01T10:52:43.661095+0000","last_unstale":"2023-10-01T10:53:04.053760+0000","last_undegraded":"2023-10-01T10:53:04.053760+0000","last_fullsized":"2023-10-01T10:53:04.053760+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:08:59.976920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":147,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:04.084089+0000","last_change":"2023-10-01T10:37:42.772247+0000","last_active":"2023-10-01T10:53:04.084089+0000","last_peered":"2023-10-01T10:53:04.084089+0000","last_clean":"2023-10-01T10:53:04.084089+0000","last_became_active":"2023-10-01T10:37:42.771697+0000","last_became_peered":"2023-10-01T10:37:42.771697+0000","last_unstale":"2023-10-01T10:53:04.084089+0000","last_undegraded":"2023-10-01T10:53:04.084089+0000","last_fullsized":"2023-10-01T10:53:04.084089+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:23:15.018248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":160,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:04.065235+0000","last_change":"2023-10-01T10:28:20.211798+0000","last_active":"2023-10-01T10:53:04.065235+0000","last_peered":"2023-10-01T10:53:04.065235+0000","last_clean":"2023-10-01T10:53:04.065235+0000","last_became_active":"2023-10-01T10:28:20.210791+0000","last_became_peered":"2023-10-01T10:28:20.210791+0000","last_unstale":"2023-10-01T10:53:04.065235+0000","last_undegraded":"2023-10-01T10:53:04.065235+0000","last_fullsized":"2023-10-01T10:53:04.065235+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":151,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:04.080135+0000","last_change":"2023-10-01T10:37:42.773127+0000","last_active":"2023-10-01T10:53:04.080135+0000","last_peered":"2023-10-01T10:53:04.080135+0000","last_clean":"2023-10-01T10:53:04.080135+0000","last_became_active":"2023-10-01T10:37:42.772102+0000","last_became_peered":"2023-10-01T10:37:42.772102+0000","last_unstale":"2023-10-01T10:53:04.080135+0000","last_undegraded":"2023-10-01T10:53:04.080135+0000","last_fullsized":"2023-10-01T10:53:04.080135+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:23:26.202295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"186'9179","reported_seq":9200,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-01T10:53:48.254427+0000","last_change":"2023-10-01T10:51:01.948311+0000","last_active":"2023-10-01T10:53:48.254427+0000","last_peered":"2023-10-01T10:53:48.254427+0000","last_clean":"2023-10-01T10:53:48.254427+0000","last_became_active":"2023-10-01T10:51:01.947921+0000","last_became_peered":"2023-10-01T10:51:01.947921+0000","last_unstale":"2023-10-01T10:53:48.254427+0000","last_undegraded":"2023-10-01T10:53:48.254427+0000","last_fullsized":"2023-10-01T10:53:48.254427+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":9179,"log_dups_size":0,"ondisk_log_size":9179,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":270467072,"num_objects":4127,"num_object_clones":0,"num_object_copies":12381,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4127,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9179,"num_write_kb":425792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":25,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:04.054425+0000","last_change":"2023-10-01T10:52:43.663145+0000","last_active":"2023-10-01T10:53:04.054425+0000","last_peered":"2023-10-01T10:53:04.054425+0000","last_clean":"2023-10-01T10:53:04.054425+0000","last_became_active":"2023-10-01T10:52:43.662783+0000","last_became_peered":"2023-10-01T10:52:43.662783+0000","last_unstale":"2023-10-01T10:53:04.054425+0000","last_undegraded":"2023-10-01T10:53:04.054425+0000","last_fullsized":"2023-10-01T10:53:04.054425+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T22:10:32.190922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":137,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:04.083523+0000","last_change":"2023-10-01T10:15:40.318702+0000","last_active":"2023-10-01T10:53:04.083523+0000","last_peered":"2023-10-01T10:53:04.083523+0000","last_clean":"2023-10-01T10:53:04.083523+0000","last_became_active":"2023-10-01T10:15:40.318333+0000","last_became_peered":"2023-10-01T10:15:40.318333+0000","last_unstale":"2023-10-01T10:53:04.083523+0000","last_undegraded":"2023-10-01T10:53:04.083523+0000","last_fullsized":"2023-10-01T10:53:04.083523+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"186'9087","reported_seq":9109,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-01T10:53:44.213495+0000","last_change":"2023-10-01T10:51:01.947670+0000","last_active":"2023-10-01T10:53:44.213495+0000","last_peered":"2023-10-01T10:53:44.213495+0000","last_clean":"2023-10-01T10:53:44.213495+0000","last_became_active":"2023-10-01T10:51:01.947301+0000","last_became_peered":"2023-10-01T10:51:01.947301+0000","last_unstale":"2023-10-01T10:53:44.213495+0000","last_undegraded":"2023-10-01T10:53:44.213495+0000","last_fullsized":"2023-10-01T10:53:44.213495+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":9087,"log_dups_size":0,"ondisk_log_size":9087,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T22:33:54.269148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":283574272,"num_objects":4327,"num_object_clones":0,"num_object_copies":12981,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4327,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9087,"num_write_kb":429248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":25,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:04.054928+0000","last_change":"2023-10-01T10:52:43.662540+0000","last_active":"2023-10-01T10:53:04.054928+0000","last_peered":"2023-10-01T10:53:04.054928+0000","last_clean":"2023-10-01T10:53:04.054928+0000","last_became_active":"2023-10-01T10:52:43.662128+0000","last_became_peered":"2023-10-01T10:52:43.662128+0000","last_unstale":"2023-10-01T10:53:04.054928+0000","last_undegraded":"2023-10-01T10:53:04.054928+0000","last_fullsized":"2023-10-01T10:53:04.054928+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:09:44.807003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":181,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-01T10:53:23.595727+0000","last_change":"2023-10-01T10:38:16.033203+0000","last_active":"2023-10-01T10:53:23.595727+0000","last_peered":"2023-10-01T10:53:23.595727+0000","last_clean":"2023-10-01T10:53:23.595727+0000","last_became_active":"2023-10-01T10:38:16.032681+0000","last_became_peered":"2023-10-01T10:38:16.032681+0000","last_unstale":"2023-10-01T10:53:23.595727+0000","last_undegraded":"2023-10-01T10:53:23.595727+0000","last_fullsized":"2023-10-01T10:53:23.595727+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:42:36.642284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"186'9323","reported_seq":9351,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-01T10:53:48.298899+0000","last_change":"2023-10-01T10:52:43.687462+0000","last_active":"2023-10-01T10:53:48.298899+0000","last_peered":"2023-10-01T10:53:48.298899+0000","last_clean":"2023-10-01T10:53:48.298899+0000","last_became_active":"2023-10-01T10:52:43.669323+0000","last_became_peered":"2023-10-01T10:52:43.669323+0000","last_unstale":"2023-10-01T10:53:48.298899+0000","last_undegraded":"2023-10-01T10:53:48.298899+0000","last_fullsized":"2023-10-01T10:53:48.298899+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":9323,"log_dups_size":0,"ondisk_log_size":9323,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":273350656,"num_objects":4171,"num_object_clones":0,"num_object_copies":12513,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4171,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9323,"num_write_kb":431808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":25,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:04.083895+0000","last_change":"2023-10-01T10:52:43.669543+0000","last_active":"2023-10-01T10:53:04.083895+0000","last_peered":"2023-10-01T10:53:04.083895+0000","last_clean":"2023-10-01T10:53:04.083895+0000","last_became_active":"2023-10-01T10:52:43.669141+0000","last_became_peered":"2023-10-01T10:52:43.669141+0000","last_unstale":"2023-10-01T10:53:04.083895+0000","last_undegraded":"2023-10-01T10:53:04.083895+0000","last_fullsized":"2023-10-01T10:53:04.083895+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":151,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:04.065547+0000","last_change":"2023-10-01T10:32:31.571971+0000","last_active":"2023-10-01T10:53:04.065547+0000","last_peered":"2023-10-01T10:53:04.065547+0000","last_clean":"2023-10-01T10:53:04.065547+0000","last_became_active":"2023-10-01T10:32:31.571410+0000","last_became_peered":"2023-10-01T10:32:31.571410+0000","last_unstale":"2023-10-01T10:53:04.065547+0000","last_undegraded":"2023-10-01T10:53:04.065547+0000","last_fullsized":"2023-10-01T10:53:04.065547+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"186'9311","reported_seq":9339,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-01T10:53:48.380901+0000","last_change":"2023-10-01T10:52:43.695039+0000","last_active":"2023-10-01T10:53:48.380901+0000","last_peered":"2023-10-01T10:53:48.380901+0000","last_clean":"2023-10-01T10:53:48.380901+0000","last_became_active":"2023-10-01T10:52:43.661479+0000","last_became_peered":"2023-10-01T10:52:43.661479+0000","last_unstale":"2023-10-01T10:53:48.380901+0000","last_undegraded":"2023-10-01T10:53:48.380901+0000","last_fullsized":"2023-10-01T10:53:48.380901+0000","mapping_epoch":183,"log_start":"178'700","ondisk_log_start":"178'700","created":173,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":8611,"log_dups_size":0,"ondisk_log_size":8611,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":272826368,"num_objects":4163,"num_object_clones":0,"num_object_copies":12489,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4163,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9311,"num_write_kb":431168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":153,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:04.083651+0000","last_change":"2023-10-01T10:37:42.765733+0000","last_active":"2023-10-01T10:53:04.083651+0000","last_peered":"2023-10-01T10:53:04.083651+0000","last_clean":"2023-10-01T10:53:04.083651+0000","last_became_active":"2023-10-01T10:37:42.765311+0000","last_became_peered":"2023-10-01T10:37:42.765311+0000","last_unstale":"2023-10-01T10:53:04.083651+0000","last_undegraded":"2023-10-01T10:53:04.083651+0000","last_fullsized":"2023-10-01T10:53:04.083651+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":163,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:04.182539+0000","last_change":"2023-10-01T10:38:16.026348+0000","last_active":"2023-10-01T10:53:04.182539+0000","last_peered":"2023-10-01T10:53:04.182539+0000","last_clean":"2023-10-01T10:53:04.182539+0000","last_became_active":"2023-10-01T10:38:16.025747+0000","last_became_peered":"2023-10-01T10:38:16.025747+0000","last_unstale":"2023-10-01T10:53:04.182539+0000","last_undegraded":"2023-10-01T10:53:04.182539+0000","last_fullsized":"2023-10-01T10:53:04.182539+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:28:18.934480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"186'9427","reported_seq":9448,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-01T10:53:48.321103+0000","last_change":"2023-10-01T10:51:01.954225+0000","last_active":"2023-10-01T10:53:48.321103+0000","last_peered":"2023-10-01T10:53:48.321103+0000","last_clean":"2023-10-01T10:53:48.321103+0000","last_became_active":"2023-10-01T10:51:01.953830+0000","last_became_peered":"2023-10-01T10:51:01.953830+0000","last_unstale":"2023-10-01T10:53:48.321103+0000","last_undegraded":"2023-10-01T10:53:48.321103+0000","last_fullsized":"2023-10-01T10:53:48.321103+0000","mapping_epoch":173,"log_start":"178'800","ondisk_log_start":"178'800","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":8627,"log_dups_size":0,"ondisk_log_size":8627,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":274137088,"num_objects":4183,"num_object_clones":0,"num_object_copies":12549,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4183,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9427,"num_write_kb":435520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":155,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-01T10:53:23.596766+0000","last_change":"2023-10-01T10:38:16.022493+0000","last_active":"2023-10-01T10:53:23.596766+0000","last_peered":"2023-10-01T10:53:23.596766+0000","last_clean":"2023-10-01T10:53:23.596766+0000","last_became_active":"2023-10-01T10:38:16.022033+0000","last_became_peered":"2023-10-01T10:38:16.022033+0000","last_unstale":"2023-10-01T10:53:23.596766+0000","last_undegraded":"2023-10-01T10:53:23.596766+0000","last_fullsized":"2023-10-01T10:53:23.596766+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:49:05.262763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":165,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:04.054473+0000","last_change":"2023-10-01T10:38:16.030613+0000","last_active":"2023-10-01T10:53:04.054473+0000","last_peered":"2023-10-01T10:53:04.054473+0000","last_clean":"2023-10-01T10:53:04.054473+0000","last_became_active":"2023-10-01T10:38:16.028975+0000","last_became_peered":"2023-10-01T10:38:16.028975+0000","last_unstale":"2023-10-01T10:53:04.054473+0000","last_undegraded":"2023-10-01T10:53:04.054473+0000","last_fullsized":"2023-10-01T10:53:04.054473+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:46:07.287911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"186'9363","reported_seq":9385,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-01T10:53:48.175714+0000","last_change":"2023-10-01T10:51:01.946522+0000","last_active":"2023-10-01T10:53:48.175714+0000","last_peered":"2023-10-01T10:53:48.175714+0000","last_clean":"2023-10-01T10:53:48.175714+0000","last_became_active":"2023-10-01T10:51:01.945971+0000","last_became_peered":"2023-10-01T10:51:01.945971+0000","last_unstale":"2023-10-01T10:53:48.175714+0000","last_undegraded":"2023-10-01T10:53:48.175714+0000","last_fullsized":"2023-10-01T10:53:48.175714+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":9363,"log_dups_size":0,"ondisk_log_size":9363,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:34:58.733457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":273350656,"num_objects":4171,"num_object_clones":0,"num_object_copies":12513,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4171,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9363,"num_write_kb":433088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":138,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:04.054067+0000","last_change":"2023-10-01T10:47:54.750295+0000","last_active":"2023-10-01T10:53:04.054067+0000","last_peered":"2023-10-01T10:53:04.054067+0000","last_clean":"2023-10-01T10:53:04.054067+0000","last_became_active":"2023-10-01T10:47:54.749917+0000","last_became_peered":"2023-10-01T10:47:54.749917+0000","last_unstale":"2023-10-01T10:53:04.054067+0000","last_undegraded":"2023-10-01T10:53:04.054067+0000","last_fullsized":"2023-10-01T10:53:04.054067+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:30:59.236210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":170,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:04.056100+0000","last_change":"2023-10-01T10:38:16.035906+0000","last_active":"2023-10-01T10:53:04.056100+0000","last_peered":"2023-10-01T10:53:04.056100+0000","last_clean":"2023-10-01T10:53:04.056100+0000","last_became_active":"2023-10-01T10:38:16.035383+0000","last_became_peered":"2023-10-01T10:38:16.035383+0000","last_unstale":"2023-10-01T10:53:04.056100+0000","last_undegraded":"2023-10-01T10:53:04.056100+0000","last_fullsized":"2023-10-01T10:53:04.056100+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:44:41.500097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"186'9210","reported_seq":9232,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-01T10:53:47.080826+0000","last_change":"2023-10-01T10:51:01.954256+0000","last_active":"2023-10-01T10:53:47.080826+0000","last_peered":"2023-10-01T10:53:47.080826+0000","last_clean":"2023-10-01T10:53:47.080826+0000","last_became_active":"2023-10-01T10:51:01.953595+0000","last_became_peered":"2023-10-01T10:51:01.953595+0000","last_unstale":"2023-10-01T10:53:47.080826+0000","last_undegraded":"2023-10-01T10:53:47.080826+0000","last_fullsized":"2023-10-01T10:53:47.080826+0000","mapping_epoch":173,"log_start":"178'1300","ondisk_log_start":"178'1300","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":7910,"log_dups_size":0,"ondisk_log_size":7910,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:55:22.868887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":272105472,"num_objects":4152,"num_object_clones":0,"num_object_copies":12456,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4152,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9210,"num_write_kb":427584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":163,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-01T10:53:23.595876+0000","last_change":"2023-10-01T10:38:16.026611+0000","last_active":"2023-10-01T10:53:23.595876+0000","last_peered":"2023-10-01T10:53:23.595876+0000","last_clean":"2023-10-01T10:53:23.595876+0000","last_became_active":"2023-10-01T10:38:16.026267+0000","last_became_peered":"2023-10-01T10:38:16.026267+0000","last_unstale":"2023-10-01T10:53:23.595876+0000","last_undegraded":"2023-10-01T10:53:23.595876+0000","last_fullsized":"2023-10-01T10:53:23.595876+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:06.135288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":146,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:04.054709+0000","last_change":"2023-10-01T10:38:16.032162+0000","last_active":"2023-10-01T10:53:04.054709+0000","last_peered":"2023-10-01T10:53:04.054709+0000","last_clean":"2023-10-01T10:53:04.054709+0000","last_became_active":"2023-10-01T10:38:16.031785+0000","last_became_peered":"2023-10-01T10:38:16.031785+0000","last_unstale":"2023-10-01T10:53:04.054709+0000","last_undegraded":"2023-10-01T10:53:04.054709+0000","last_fullsized":"2023-10-01T10:53:04.054709+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:38:43.003626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"186'9307","reported_seq":9341,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-01T10:53:47.045181+0000","last_change":"2023-10-01T10:52:19.021061+0000","last_active":"2023-10-01T10:53:47.045181+0000","last_peered":"2023-10-01T10:53:47.045181+0000","last_clean":"2023-10-01T10:53:47.045181+0000","last_became_active":"2023-10-01T10:52:18.959403+0000","last_became_peered":"2023-10-01T10:52:18.959403+0000","last_unstale":"2023-10-01T10:53:47.045181+0000","last_undegraded":"2023-10-01T10:53:47.045181+0000","last_fullsized":"2023-10-01T10:53:47.045181+0000","mapping_epoch":181,"log_start":"178'1400","ondisk_log_start":"178'1400","created":173,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":7907,"log_dups_size":0,"ondisk_log_size":7907,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:25:34.494121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":272564224,"num_objects":4159,"num_object_clones":0,"num_object_copies":12477,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4159,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9307,"num_write_kb":430912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":153,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:04.079368+0000","last_change":"2023-10-01T10:37:42.772398+0000","last_active":"2023-10-01T10:53:04.079368+0000","last_peered":"2023-10-01T10:53:04.079368+0000","last_clean":"2023-10-01T10:53:04.079368+0000","last_became_active":"2023-10-01T10:37:42.772084+0000","last_became_peered":"2023-10-01T10:37:42.772084+0000","last_unstale":"2023-10-01T10:53:04.079368+0000","last_undegraded":"2023-10-01T10:53:04.079368+0000","last_fullsized":"2023-10-01T10:53:04.079368+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:32:43.846419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":171,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:04.069370+0000","last_change":"2023-10-01T10:38:16.019755+0000","last_active":"2023-10-01T10:53:04.069370+0000","last_peered":"2023-10-01T10:53:04.069370+0000","last_clean":"2023-10-01T10:53:04.069370+0000","last_became_active":"2023-10-01T10:38:16.019397+0000","last_became_peered":"2023-10-01T10:38:16.019397+0000","last_unstale":"2023-10-01T10:53:04.069370+0000","last_undegraded":"2023-10-01T10:53:04.069370+0000","last_fullsized":"2023-10-01T10:53:04.069370+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:42:56.623225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.d","version":"186'9109","reported_seq":9133,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-01T10:53:48.157225+0000","last_change":"2023-10-01T10:51:01.952187+0000","last_active":"2023-10-01T10:53:48.157225+0000","last_peered":"2023-10-01T10:53:48.157225+0000","last_clean":"2023-10-01T10:53:48.157225+0000","last_became_active":"2023-10-01T10:51:01.951373+0000","last_became_peered":"2023-10-01T10:51:01.951373+0000","last_unstale":"2023-10-01T10:53:48.157225+0000","last_undegraded":"2023-10-01T10:53:48.157225+0000","last_fullsized":"2023-10-01T10:53:48.157225+0000","mapping_epoch":173,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":9109,"log_dups_size":0,"ondisk_log_size":9109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:10:27.939483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":266207256,"num_objects":4063,"num_object_clones":0,"num_object_copies":12189,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4063,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9109,"num_write_kb":421441,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":78,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:04.082114+0000","last_change":"2023-10-01T10:52:18.960164+0000","last_active":"2023-10-01T10:53:04.082114+0000","last_peered":"2023-10-01T10:53:04.082114+0000","last_clean":"2023-10-01T10:53:04.082114+0000","last_became_active":"2023-10-01T10:52:18.959782+0000","last_became_peered":"2023-10-01T10:52:18.959782+0000","last_unstale":"2023-10-01T10:53:04.082114+0000","last_undegraded":"2023-10-01T10:53:04.082114+0000","last_fullsized":"2023-10-01T10:53:04.082114+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:08:27.603412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":168,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:04.083761+0000","last_change":"2023-10-01T10:37:42.766814+0000","last_active":"2023-10-01T10:53:04.083761+0000","last_peered":"2023-10-01T10:53:04.083761+0000","last_clean":"2023-10-01T10:53:04.083761+0000","last_became_active":"2023-10-01T10:37:42.766451+0000","last_became_peered":"2023-10-01T10:37:42.766451+0000","last_unstale":"2023-10-01T10:53:04.083761+0000","last_undegraded":"2023-10-01T10:53:04.083761+0000","last_fullsized":"2023-10-01T10:53:04.083761+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":71,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:04.083310+0000","last_change":"2023-10-01T10:52:18.960606+0000","last_active":"2023-10-01T10:53:04.083310+0000","last_peered":"2023-10-01T10:53:04.083310+0000","last_clean":"2023-10-01T10:53:04.083310+0000","last_became_active":"2023-10-01T10:52:18.960270+0000","last_became_peered":"2023-10-01T10:52:18.960270+0000","last_unstale":"2023-10-01T10:53:04.083310+0000","last_undegraded":"2023-10-01T10:53:04.083310+0000","last_fullsized":"2023-10-01T10:53:04.083310+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.b","version":"186'9108","reported_seq":9131,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-01T10:53:47.110907+0000","last_change":"2023-10-01T10:51:01.950278+0000","last_active":"2023-10-01T10:53:47.110907+0000","last_peered":"2023-10-01T10:53:47.110907+0000","last_clean":"2023-10-01T10:53:47.110907+0000","last_became_active":"2023-10-01T10:51:01.949902+0000","last_became_peered":"2023-10-01T10:51:01.949902+0000","last_unstale":"2023-10-01T10:53:47.110907+0000","last_undegraded":"2023-10-01T10:53:47.110907+0000","last_fullsized":"2023-10-01T10:53:47.110907+0000","mapping_epoch":173,"log_start":"178'1200","ondisk_log_start":"178'1200","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":7908,"log_dups_size":0,"ondisk_log_size":7908,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:42:45.648175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264765440,"num_objects":4040,"num_object_clones":0,"num_object_copies":12120,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4040,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9108,"num_write_kb":420736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":157,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:04.083367+0000","last_change":"2023-10-01T10:32:31.572482+0000","last_active":"2023-10-01T10:53:04.083367+0000","last_peered":"2023-10-01T10:53:04.083367+0000","last_clean":"2023-10-01T10:53:04.083367+0000","last_became_active":"2023-10-01T10:32:31.571993+0000","last_became_peered":"2023-10-01T10:32:31.571993+0000","last_unstale":"2023-10-01T10:53:04.083367+0000","last_undegraded":"2023-10-01T10:53:04.083367+0000","last_fullsized":"2023-10-01T10:53:04.083367+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":133,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:04.054847+0000","last_change":"2023-10-01T10:38:16.030202+0000","last_active":"2023-10-01T10:53:04.054847+0000","last_peered":"2023-10-01T10:53:04.054847+0000","last_clean":"2023-10-01T10:53:04.054847+0000","last_became_active":"2023-10-01T10:38:16.029917+0000","last_became_peered":"2023-10-01T10:38:16.029917+0000","last_unstale":"2023-10-01T10:53:04.054847+0000","last_undegraded":"2023-10-01T10:53:04.054847+0000","last_fullsized":"2023-10-01T10:53:04.054847+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:20:06.763909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":81,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:04.067287+0000","last_change":"2023-10-01T10:52:18.963437+0000","last_active":"2023-10-01T10:53:04.067287+0000","last_peered":"2023-10-01T10:53:04.067287+0000","last_clean":"2023-10-01T10:53:04.067287+0000","last_became_active":"2023-10-01T10:52:18.955165+0000","last_became_peered":"2023-10-01T10:52:18.955165+0000","last_unstale":"2023-10-01T10:53:04.067287+0000","last_undegraded":"2023-10-01T10:53:04.067287+0000","last_fullsized":"2023-10-01T10:53:04.067287+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:57:49.129100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.8","version":"186'9244","reported_seq":9267,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-01T10:53:47.108679+0000","last_change":"2023-10-01T10:51:01.955996+0000","last_active":"2023-10-01T10:53:47.108679+0000","last_peered":"2023-10-01T10:53:47.108679+0000","last_clean":"2023-10-01T10:53:47.108679+0000","last_became_active":"2023-10-01T10:51:01.955685+0000","last_became_peered":"2023-10-01T10:51:01.955685+0000","last_unstale":"2023-10-01T10:53:47.108679+0000","last_undegraded":"2023-10-01T10:53:47.108679+0000","last_fullsized":"2023-10-01T10:53:47.108679+0000","mapping_epoch":173,"log_start":"178'1300","ondisk_log_start":"178'1300","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":7944,"log_dups_size":0,"ondisk_log_size":7944,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:54:42.840196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":268042240,"num_objects":4090,"num_object_clones":0,"num_object_copies":12270,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4090,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9244,"num_write_kb":426688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":163,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-01T10:53:23.599674+0000","last_change":"2023-10-01T10:37:42.762524+0000","last_active":"2023-10-01T10:53:23.599674+0000","last_peered":"2023-10-01T10:53:23.599674+0000","last_clean":"2023-10-01T10:53:23.599674+0000","last_became_active":"2023-10-01T10:37:42.762209+0000","last_became_peered":"2023-10-01T10:37:42.762209+0000","last_unstale":"2023-10-01T10:53:23.599674+0000","last_undegraded":"2023-10-01T10:53:23.599674+0000","last_fullsized":"2023-10-01T10:53:23.599674+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:45:38.171397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":61,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:04.066951+0000","last_change":"2023-10-01T10:52:18.962766+0000","last_active":"2023-10-01T10:53:04.066951+0000","last_peered":"2023-10-01T10:53:04.066951+0000","last_clean":"2023-10-01T10:53:04.066951+0000","last_became_active":"2023-10-01T10:52:18.962461+0000","last_became_peered":"2023-10-01T10:52:18.962461+0000","last_unstale":"2023-10-01T10:53:04.066951+0000","last_undegraded":"2023-10-01T10:53:04.066951+0000","last_fullsized":"2023-10-01T10:53:04.066951+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:25:54.051672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":73,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:04.084251+0000","last_change":"2023-10-01T10:52:18.961040+0000","last_active":"2023-10-01T10:53:04.084251+0000","last_peered":"2023-10-01T10:53:04.084251+0000","last_clean":"2023-10-01T10:53:04.084251+0000","last_became_active":"2023-10-01T10:52:18.960606+0000","last_became_peered":"2023-10-01T10:52:18.960606+0000","last_unstale":"2023-10-01T10:53:04.084251+0000","last_undegraded":"2023-10-01T10:53:04.084251+0000","last_fullsized":"2023-10-01T10:53:04.084251+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.a","version":"186'9232","reported_seq":9255,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-01T10:53:47.069138+0000","last_change":"2023-10-01T10:51:01.955145+0000","last_active":"2023-10-01T10:53:47.069138+0000","last_peered":"2023-10-01T10:53:47.069138+0000","last_clean":"2023-10-01T10:53:47.069138+0000","last_became_active":"2023-10-01T10:51:01.954793+0000","last_became_peered":"2023-10-01T10:51:01.954793+0000","last_unstale":"2023-10-01T10:53:47.069138+0000","last_undegraded":"2023-10-01T10:53:47.069138+0000","last_fullsized":"2023-10-01T10:53:47.069138+0000","mapping_epoch":173,"log_start":"178'1300","ondisk_log_start":"178'1300","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":7932,"log_dups_size":0,"ondisk_log_size":7932,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:25:33.468595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":270794752,"num_objects":4132,"num_object_clones":0,"num_object_copies":12396,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4132,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9232,"num_write_kb":427648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":157,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:04.083579+0000","last_change":"2023-10-01T10:37:05.024009+0000","last_active":"2023-10-01T10:53:04.083579+0000","last_peered":"2023-10-01T10:53:04.083579+0000","last_clean":"2023-10-01T10:53:04.083579+0000","last_became_active":"2023-10-01T10:37:05.023458+0000","last_became_peered":"2023-10-01T10:37:05.023458+0000","last_unstale":"2023-10-01T10:53:04.083579+0000","last_undegraded":"2023-10-01T10:53:04.083579+0000","last_fullsized":"2023-10-01T10:53:04.083579+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'139","reported_seq":276,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:04.081827+0000","last_change":"2023-10-01T10:52:43.660674+0000","last_active":"2023-10-01T10:53:04.081827+0000","last_peered":"2023-10-01T10:53:04.081827+0000","last_clean":"2023-10-01T10:53:04.081827+0000","last_became_active":"2023-10-01T10:52:43.660276+0000","last_became_peered":"2023-10-01T10:52:43.660276+0000","last_unstale":"2023-10-01T10:53:04.081827+0000","last_undegraded":"2023-10-01T10:53:04.081827+0000","last_fullsized":"2023-10-01T10:53:04.081827+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:09:17.221082+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:09:17.221082+0000","last_clean_scrub_stamp":"2023-10-01T10:09:17.221082+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:47:37.225755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":135,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:04.079108+0000","last_change":"2023-10-01T10:37:42.765100+0000","last_active":"2023-10-01T10:53:04.079108+0000","last_peered":"2023-10-01T10:53:04.079108+0000","last_clean":"2023-10-01T10:53:04.079108+0000","last_became_active":"2023-10-01T10:37:42.764761+0000","last_became_peered":"2023-10-01T10:37:42.764761+0000","last_unstale":"2023-10-01T10:53:04.079108+0000","last_undegraded":"2023-10-01T10:53:04.079108+0000","last_fullsized":"2023-10-01T10:53:04.079108+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:24:56.907377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":122,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:04.056001+0000","last_change":"2023-10-01T10:47:54.782475+0000","last_active":"2023-10-01T10:53:04.056001+0000","last_peered":"2023-10-01T10:53:04.056001+0000","last_clean":"2023-10-01T10:53:04.056001+0000","last_became_active":"2023-10-01T10:47:54.781725+0000","last_became_peered":"2023-10-01T10:47:54.781725+0000","last_unstale":"2023-10-01T10:53:04.056001+0000","last_undegraded":"2023-10-01T10:53:04.056001+0000","last_fullsized":"2023-10-01T10:53:04.056001+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:21:30.978552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"186'9350","reported_seq":9373,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-01T10:53:47.065455+0000","last_change":"2023-10-01T10:51:01.952824+0000","last_active":"2023-10-01T10:53:47.065455+0000","last_peered":"2023-10-01T10:53:47.065455+0000","last_clean":"2023-10-01T10:53:47.065455+0000","last_became_active":"2023-10-01T10:51:01.952483+0000","last_became_peered":"2023-10-01T10:51:01.952483+0000","last_unstale":"2023-10-01T10:53:47.065455+0000","last_undegraded":"2023-10-01T10:53:47.065455+0000","last_fullsized":"2023-10-01T10:53:47.065455+0000","mapping_epoch":173,"log_start":"178'1400","ondisk_log_start":"178'1400","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":7950,"log_dups_size":0,"ondisk_log_size":7950,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:42:42.460140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":275644416,"num_objects":4206,"num_object_clones":0,"num_object_copies":12618,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4206,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9350,"num_write_kb":433792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":93,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:04.081748+0000","last_change":"2023-10-01T10:52:43.661009+0000","last_active":"2023-10-01T10:53:04.081748+0000","last_peered":"2023-10-01T10:53:04.081748+0000","last_clean":"2023-10-01T10:53:04.081748+0000","last_became_active":"2023-10-01T10:52:43.660679+0000","last_became_peered":"2023-10-01T10:52:43.660679+0000","last_unstale":"2023-10-01T10:53:04.081748+0000","last_undegraded":"2023-10-01T10:53:04.081748+0000","last_fullsized":"2023-10-01T10:53:04.081748+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:32:21.336835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":163,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:19.189117+0000","last_change":"2023-10-01T10:38:16.021968+0000","last_active":"2023-10-01T10:53:19.189117+0000","last_peered":"2023-10-01T10:53:19.189117+0000","last_clean":"2023-10-01T10:53:19.189117+0000","last_became_active":"2023-10-01T10:38:16.021539+0000","last_became_peered":"2023-10-01T10:38:16.021539+0000","last_unstale":"2023-10-01T10:53:19.189117+0000","last_undegraded":"2023-10-01T10:53:19.189117+0000","last_fullsized":"2023-10-01T10:53:19.189117+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:12:53.105747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":163,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:04.083857+0000","last_change":"2023-10-01T10:37:05.017431+0000","last_active":"2023-10-01T10:53:04.083857+0000","last_peered":"2023-10-01T10:53:04.083857+0000","last_clean":"2023-10-01T10:53:04.083857+0000","last_became_active":"2023-10-01T10:37:05.017094+0000","last_became_peered":"2023-10-01T10:37:05.017094+0000","last_unstale":"2023-10-01T10:53:04.083857+0000","last_undegraded":"2023-10-01T10:53:04.083857+0000","last_fullsized":"2023-10-01T10:53:04.083857+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:10:40.664686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"186'9293","reported_seq":9320,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-01T10:53:48.112728+0000","last_change":"2023-10-01T10:52:18.964236+0000","last_active":"2023-10-01T10:53:48.112728+0000","last_peered":"2023-10-01T10:53:48.112728+0000","last_clean":"2023-10-01T10:53:48.112728+0000","last_became_active":"2023-10-01T10:52:18.955030+0000","last_became_peered":"2023-10-01T10:52:18.955030+0000","last_unstale":"2023-10-01T10:53:48.112728+0000","last_undegraded":"2023-10-01T10:53:48.112728+0000","last_fullsized":"2023-10-01T10:53:48.112728+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":9293,"log_dups_size":0,"ondisk_log_size":9293,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T22:29:23.161365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":271122432,"num_objects":4137,"num_object_clones":0,"num_object_copies":12411,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4137,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9293,"num_write_kb":429760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":171,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:04.061356+0000","last_change":"2023-10-01T10:38:16.032774+0000","last_active":"2023-10-01T10:53:04.061356+0000","last_peered":"2023-10-01T10:53:04.061356+0000","last_clean":"2023-10-01T10:53:04.061356+0000","last_became_active":"2023-10-01T10:38:16.032283+0000","last_became_peered":"2023-10-01T10:38:16.032283+0000","last_unstale":"2023-10-01T10:53:04.061356+0000","last_undegraded":"2023-10-01T10:53:04.061356+0000","last_fullsized":"2023-10-01T10:53:04.061356+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":61,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:04.067344+0000","last_change":"2023-10-01T10:52:18.964479+0000","last_active":"2023-10-01T10:53:04.067344+0000","last_peered":"2023-10-01T10:53:04.067344+0000","last_clean":"2023-10-01T10:53:04.067344+0000","last_became_active":"2023-10-01T10:52:18.963881+0000","last_became_peered":"2023-10-01T10:52:18.963881+0000","last_unstale":"2023-10-01T10:53:04.067344+0000","last_undegraded":"2023-10-01T10:53:04.067344+0000","last_fullsized":"2023-10-01T10:53:04.067344+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:36:30.405755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":134,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:19.189145+0000","last_change":"2023-10-01T10:28:20.220812+0000","last_active":"2023-10-01T10:53:19.189145+0000","last_peered":"2023-10-01T10:53:19.189145+0000","last_clean":"2023-10-01T10:53:19.189145+0000","last_became_active":"2023-10-01T10:28:20.215962+0000","last_became_peered":"2023-10-01T10:28:20.215962+0000","last_unstale":"2023-10-01T10:53:19.189145+0000","last_undegraded":"2023-10-01T10:53:19.189145+0000","last_fullsized":"2023-10-01T10:53:19.189145+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:35:15.275097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"186'9089","reported_seq":9123,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-01T10:53:48.074379+0000","last_change":"2023-10-01T10:52:43.703049+0000","last_active":"2023-10-01T10:53:48.074379+0000","last_peered":"2023-10-01T10:53:48.074379+0000","last_clean":"2023-10-01T10:53:48.074379+0000","last_became_active":"2023-10-01T10:52:43.664078+0000","last_became_peered":"2023-10-01T10:52:43.664078+0000","last_unstale":"2023-10-01T10:53:48.074379+0000","last_undegraded":"2023-10-01T10:53:48.074379+0000","last_fullsized":"2023-10-01T10:53:48.074379+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":173,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":9089,"log_dups_size":0,"ondisk_log_size":9089,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:44:08.983664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":271646720,"num_objects":4145,"num_object_clones":0,"num_object_copies":12435,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4145,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9089,"num_write_kb":423488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":140,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:04.080201+0000","last_change":"2023-10-01T10:37:42.773999+0000","last_active":"2023-10-01T10:53:04.080201+0000","last_peered":"2023-10-01T10:53:04.080201+0000","last_clean":"2023-10-01T10:53:04.080201+0000","last_became_active":"2023-10-01T10:37:42.773355+0000","last_became_peered":"2023-10-01T10:37:42.773355+0000","last_unstale":"2023-10-01T10:53:04.080201+0000","last_undegraded":"2023-10-01T10:53:04.080201+0000","last_fullsized":"2023-10-01T10:53:04.080201+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:46:51.317151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":113,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:04.054592+0000","last_change":"2023-10-01T10:52:18.962219+0000","last_active":"2023-10-01T10:53:04.054592+0000","last_peered":"2023-10-01T10:53:04.054592+0000","last_clean":"2023-10-01T10:53:04.054592+0000","last_became_active":"2023-10-01T10:52:18.961826+0000","last_became_peered":"2023-10-01T10:52:18.961826+0000","last_unstale":"2023-10-01T10:53:04.054592+0000","last_undegraded":"2023-10-01T10:53:04.054592+0000","last_fullsized":"2023-10-01T10:53:04.054592+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:01:46.130522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":138,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-01T10:53:23.596218+0000","last_change":"2023-10-01T10:37:42.773538+0000","last_active":"2023-10-01T10:53:23.596218+0000","last_peered":"2023-10-01T10:53:23.596218+0000","last_clean":"2023-10-01T10:53:23.596218+0000","last_became_active":"2023-10-01T10:37:42.771863+0000","last_became_peered":"2023-10-01T10:37:42.771863+0000","last_unstale":"2023-10-01T10:53:23.596218+0000","last_undegraded":"2023-10-01T10:53:23.596218+0000","last_fullsized":"2023-10-01T10:53:23.596218+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:46:46.698627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"186'9144","reported_seq":9167,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-01T10:53:43.996936+0000","last_change":"2023-10-01T10:51:01.952403+0000","last_active":"2023-10-01T10:53:43.996936+0000","last_peered":"2023-10-01T10:53:43.996936+0000","last_clean":"2023-10-01T10:53:43.996936+0000","last_became_active":"2023-10-01T10:51:01.951884+0000","last_became_peered":"2023-10-01T10:51:01.951884+0000","last_unstale":"2023-10-01T10:53:43.996936+0000","last_undegraded":"2023-10-01T10:53:43.996936+0000","last_fullsized":"2023-10-01T10:53:43.996936+0000","mapping_epoch":173,"log_start":"178'1000","ondisk_log_start":"178'1000","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":8144,"log_dups_size":0,"ondisk_log_size":8144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:33:03.736176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":281542656,"num_objects":4296,"num_object_clones":0,"num_object_copies":12888,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4296,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9144,"num_write_kb":430080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":156,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:04.065704+0000","last_change":"2023-10-01T10:37:05.023034+0000","last_active":"2023-10-01T10:53:04.065704+0000","last_peered":"2023-10-01T10:53:04.065704+0000","last_clean":"2023-10-01T10:53:04.065704+0000","last_became_active":"2023-10-01T10:37:05.022670+0000","last_became_peered":"2023-10-01T10:37:05.022670+0000","last_unstale":"2023-10-01T10:53:04.065704+0000","last_undegraded":"2023-10-01T10:53:04.065704+0000","last_fullsized":"2023-10-01T10:53:04.065704+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":166,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:04.083130+0000","last_change":"2023-10-01T10:38:16.027472+0000","last_active":"2023-10-01T10:53:04.083130+0000","last_peered":"2023-10-01T10:53:04.083130+0000","last_clean":"2023-10-01T10:53:04.083130+0000","last_became_active":"2023-10-01T10:38:16.026810+0000","last_became_peered":"2023-10-01T10:38:16.026810+0000","last_unstale":"2023-10-01T10:53:04.083130+0000","last_undegraded":"2023-10-01T10:53:04.083130+0000","last_fullsized":"2023-10-01T10:53:04.083130+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:30:06.978187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":131,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:04.054530+0000","last_change":"2023-10-01T10:38:16.035705+0000","last_active":"2023-10-01T10:53:04.054530+0000","last_peered":"2023-10-01T10:53:04.054530+0000","last_clean":"2023-10-01T10:53:04.054530+0000","last_became_active":"2023-10-01T10:38:16.035389+0000","last_became_peered":"2023-10-01T10:38:16.035389+0000","last_unstale":"2023-10-01T10:53:04.054530+0000","last_undegraded":"2023-10-01T10:53:04.054530+0000","last_fullsized":"2023-10-01T10:53:04.054530+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:16:10.201409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":150,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:19.189178+0000","last_change":"2023-10-01T10:38:16.028491+0000","last_active":"2023-10-01T10:53:19.189178+0000","last_peered":"2023-10-01T10:53:19.189178+0000","last_clean":"2023-10-01T10:53:19.189178+0000","last_became_active":"2023-10-01T10:38:16.028175+0000","last_became_peered":"2023-10-01T10:38:16.028175+0000","last_unstale":"2023-10-01T10:53:19.189178+0000","last_undegraded":"2023-10-01T10:53:19.189178+0000","last_fullsized":"2023-10-01T10:53:19.189178+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:44:29.083129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":171,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:04.067217+0000","last_change":"2023-10-01T10:37:42.762360+0000","last_active":"2023-10-01T10:53:04.067217+0000","last_peered":"2023-10-01T10:53:04.067217+0000","last_clean":"2023-10-01T10:53:04.067217+0000","last_became_active":"2023-10-01T10:37:42.761936+0000","last_became_peered":"2023-10-01T10:37:42.761936+0000","last_unstale":"2023-10-01T10:53:04.067217+0000","last_undegraded":"2023-10-01T10:53:04.067217+0000","last_fullsized":"2023-10-01T10:53:04.067217+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:50.637582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":170,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:04.083759+0000","last_change":"2023-10-01T10:38:16.037809+0000","last_active":"2023-10-01T10:53:04.083759+0000","last_peered":"2023-10-01T10:53:04.083759+0000","last_clean":"2023-10-01T10:53:04.083759+0000","last_became_active":"2023-10-01T10:38:16.037318+0000","last_became_peered":"2023-10-01T10:38:16.037318+0000","last_unstale":"2023-10-01T10:53:04.083759+0000","last_undegraded":"2023-10-01T10:53:04.083759+0000","last_fullsized":"2023-10-01T10:53:04.083759+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":180,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:14.021247+0000","last_change":"2023-10-01T10:38:16.034282+0000","last_active":"2023-10-01T10:53:14.021247+0000","last_peered":"2023-10-01T10:53:14.021247+0000","last_clean":"2023-10-01T10:53:14.021247+0000","last_became_active":"2023-10-01T10:38:16.033820+0000","last_became_peered":"2023-10-01T10:38:16.033820+0000","last_unstale":"2023-10-01T10:53:14.021247+0000","last_undegraded":"2023-10-01T10:53:14.021247+0000","last_fullsized":"2023-10-01T10:53:14.021247+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:48:23.815832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":155,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:04.083702+0000","last_change":"2023-10-01T10:38:16.038077+0000","last_active":"2023-10-01T10:53:04.083702+0000","last_peered":"2023-10-01T10:53:04.083702+0000","last_clean":"2023-10-01T10:53:04.083702+0000","last_became_active":"2023-10-01T10:38:16.037313+0000","last_became_peered":"2023-10-01T10:38:16.037313+0000","last_unstale":"2023-10-01T10:53:04.083702+0000","last_undegraded":"2023-10-01T10:53:04.083702+0000","last_fullsized":"2023-10-01T10:53:04.083702+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":180,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:14.021214+0000","last_change":"2023-10-01T10:38:16.025600+0000","last_active":"2023-10-01T10:53:14.021214+0000","last_peered":"2023-10-01T10:53:14.021214+0000","last_clean":"2023-10-01T10:53:14.021214+0000","last_became_active":"2023-10-01T10:38:16.025252+0000","last_became_peered":"2023-10-01T10:38:16.025252+0000","last_unstale":"2023-10-01T10:53:14.021214+0000","last_undegraded":"2023-10-01T10:53:14.021214+0000","last_fullsized":"2023-10-01T10:53:14.021214+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:24:18.871963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":169,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-10-01T10:53:23.596185+0000","last_change":"2023-10-01T10:37:42.761243+0000","last_active":"2023-10-01T10:53:23.596185+0000","last_peered":"2023-10-01T10:53:23.596185+0000","last_clean":"2023-10-01T10:53:23.596185+0000","last_became_active":"2023-10-01T10:37:42.760834+0000","last_became_peered":"2023-10-01T10:37:42.760834+0000","last_unstale":"2023-10-01T10:53:23.596185+0000","last_undegraded":"2023-10-01T10:53:23.596185+0000","last_fullsized":"2023-10-01T10:53:23.596185+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:37:21.976095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":180,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:04.079152+0000","last_change":"2023-10-01T10:37:42.765142+0000","last_active":"2023-10-01T10:53:04.079152+0000","last_peered":"2023-10-01T10:53:04.079152+0000","last_clean":"2023-10-01T10:53:04.079152+0000","last_became_active":"2023-10-01T10:37:42.764770+0000","last_became_peered":"2023-10-01T10:37:42.764770+0000","last_unstale":"2023-10-01T10:53:04.079152+0000","last_undegraded":"2023-10-01T10:53:04.079152+0000","last_fullsized":"2023-10-01T10:53:04.079152+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:23:33.527876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":152,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:19.189241+0000","last_change":"2023-10-01T10:37:05.021621+0000","last_active":"2023-10-01T10:53:19.189241+0000","last_peered":"2023-10-01T10:53:19.189241+0000","last_clean":"2023-10-01T10:53:19.189241+0000","last_became_active":"2023-10-01T10:37:05.021173+0000","last_became_peered":"2023-10-01T10:37:05.021173+0000","last_unstale":"2023-10-01T10:53:19.189241+0000","last_undegraded":"2023-10-01T10:53:19.189241+0000","last_fullsized":"2023-10-01T10:53:19.189241+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:13:25.364499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":146,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-10-01T10:53:04.053753+0000","last_change":"2023-10-01T10:38:16.031427+0000","last_active":"2023-10-01T10:53:04.053753+0000","last_peered":"2023-10-01T10:53:04.053753+0000","last_clean":"2023-10-01T10:53:04.053753+0000","last_became_active":"2023-10-01T10:38:16.029786+0000","last_became_peered":"2023-10-01T10:38:16.029786+0000","last_unstale":"2023-10-01T10:53:04.053753+0000","last_undegraded":"2023-10-01T10:53:04.053753+0000","last_fullsized":"2023-10-01T10:53:04.053753+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:53:22.796191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":4362141720,"num_objects":66562,"num_object_clones":0,"num_object_copies":199686,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":66562,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":147776,"num_write_kb":6858753,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13126610944,"data_stored":13126598728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":137376,"ondisk_log_size":137376,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30319,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430702,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2491436,"kb_used_data":1601188,"kb_used_omap":0,"kb_used_meta":890240,"kb_avail":91253716,"statfs":{"total":95995035648,"available":93443805184,"internally_reserved":0,"allocated":1639616512,"data_stored":1639105970,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":911605760},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:53:10 2023","interfaces":[{"interface":"back","average":{"1min":1.8260000000000001,"5min":3.2360000000000002,"15min":2.3479999999999999},"min":{"1min":0.30099999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":6.734,"5min":137.34200000000001,"15min":137.34200000000001},"last":0.41799999999999998},{"interface":"front","average":{"1min":1.5109999999999999,"5min":3.2400000000000002,"15min":2.181},"min":{"1min":0.36499999999999999,"5min":0.308,"15min":0.28799999999999998},"max":{"1min":4.2960000000000003,"5min":137.39500000000001,"15min":137.39500000000001},"last":4.4379999999999997}]},{"osd":1,"last update":"Sun Oct 1 10:53:15 2023","interfaces":[{"interface":"back","average":{"1min":2.1800000000000002,"5min":1.3180000000000001,"15min":1.712},"min":{"1min":0.314,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":7.3419999999999996,"5min":7.3419999999999996,"15min":56.110999999999997},"last":4.4100000000000001},{"interface":"front","average":{"1min":1.4410000000000001,"5min":1.1060000000000001,"15min":1.6890000000000001},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":7.1779999999999999,"5min":7.1779999999999999,"15min":51.258000000000003},"last":1.3660000000000001}]},{"osd":2,"last update":"Sun Oct 1 10:53:33 2023","interfaces":[{"interface":"back","average":{"1min":2.3069999999999999,"5min":1.6120000000000001,"15min":1.377},"min":{"1min":0.39400000000000002,"5min":0.315,"15min":0.315},"max":{"1min":9.7560000000000002,"5min":9.7560000000000002,"15min":9.7560000000000002},"last":1.3169999999999999},{"interface":"front","average":{"1min":1.8120000000000001,"5min":1.3700000000000001,"15min":1.4119999999999999},"min":{"1min":0.36399999999999999,"5min":0.34000000000000002,"15min":0.33200000000000002},"max":{"1min":6.1980000000000004,"5min":7.6020000000000003,"15min":10.766},"last":1.3069999999999999}]},{"osd":3,"last update":"Sun Oct 1 10:53:10 2023","interfaces":[{"interface":"back","average":{"1min":1.2410000000000001,"5min":1.6160000000000001,"15min":1.823},"min":{"1min":0.32300000000000001,"5min":0.318,"15min":0.318},"max":{"1min":7.6050000000000004,"5min":8.6829999999999998,"15min":51.334000000000003},"last":0.40100000000000002},{"interface":"front","average":{"1min":1.8759999999999999,"5min":1.306,"15min":1.859},"min":{"1min":0.33400000000000002,"5min":0.313,"15min":0.313},"max":{"1min":7.0860000000000003,"5min":8.7469999999999999,"15min":51.404000000000003},"last":1.3400000000000001}]},{"osd":4,"last update":"Sun Oct 1 10:53:43 2023","interfaces":[{"interface":"back","average":{"1min":1.6899999999999999,"5min":1.194,"15min":1.3240000000000001},"min":{"1min":0.35699999999999998,"5min":0.33500000000000002,"15min":0.317},"max":{"1min":7.7800000000000002,"5min":7.7800000000000002,"15min":11.081},"last":4.3380000000000001},{"interface":"front","average":{"1min":1.097,"5min":1.083,"15min":1.393},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.32100000000000001},"max":{"1min":3.9910000000000001,"5min":4.2309999999999999,"15min":12.292},"last":0.443}]},{"osd":5,"last update":"Sun Oct 1 10:53:15 2023","interfaces":[{"interface":"back","average":{"1min":1.7250000000000001,"5min":1.9930000000000001,"15min":1.8120000000000001},"min":{"1min":0.375,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":7.6529999999999996,"5min":40.381,"15min":40.381},"last":0.47999999999999998},{"interface":"front","average":{"1min":1.629,"5min":1.8169999999999999,"15min":1.877},"min":{"1min":0.41699999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":7.0860000000000003,"5min":39.590000000000003,"15min":39.590000000000003},"last":1.385}]},{"osd":6,"last update":"Sun Oct 1 10:53:15 2023","interfaces":[{"interface":"back","average":{"1min":1.4279999999999999,"5min":1.5580000000000001,"15min":1.5089999999999999},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.34499999999999997},"max":{"1min":5.2009999999999996,"5min":8.5419999999999998,"15min":12.329000000000001},"last":0.57899999999999996},{"interface":"front","average":{"1min":2.7480000000000002,"5min":1.7250000000000001,"15min":1.4350000000000001},"min":{"1min":0.441,"5min":0.41599999999999998,"15min":0.36599999999999999},"max":{"1min":9.4329999999999998,"5min":9.4329999999999998,"15min":9.4329999999999998},"last":4.3799999999999999}]}]},{"osd":6,"up_from":42,"seq":180388626936,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3023200,"kb_used_data":2126104,"kb_used_omap":0,"kb_used_meta":897088,"kb_avail":90721952,"statfs":{"total":95995035648,"available":92899278848,"internally_reserved":0,"allocated":2177130496,"data_stored":2176631805,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":918618112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:53:01 2023","interfaces":[{"interface":"back","average":{"1min":1.3109999999999999,"5min":2.2050000000000001,"15min":2.5840000000000001},"min":{"1min":0.30499999999999999,"5min":0.25800000000000001,"15min":0.25},"max":{"1min":6.7990000000000004,"5min":26.670999999999999,"15min":140.524},"last":1.2370000000000001},{"interface":"front","average":{"1min":2.137,"5min":2.1960000000000002,"15min":2.6059999999999999},"min":{"1min":0.35799999999999998,"5min":0.249,"15min":0.249},"max":{"1min":5.9400000000000004,"5min":27.099,"15min":140.86500000000001},"last":9.5259999999999998}]},{"osd":1,"last update":"Sun Oct 1 10:53:09 2023","interfaces":[{"interface":"back","average":{"1min":1.143,"5min":1.522,"15min":1.367},"min":{"1min":0.29099999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":3.0600000000000001,"5min":8.2560000000000002,"15min":9.673},"last":0.47899999999999998},{"interface":"front","average":{"1min":1.8160000000000001,"5min":1.7450000000000001,"15min":1.4530000000000001},"min":{"1min":0.33900000000000002,"5min":0.28000000000000003,"15min":0.26300000000000001},"max":{"1min":4.5049999999999999,"5min":12.968999999999999,"15min":12.968999999999999},"last":9.4809999999999999}]},{"osd":2,"last update":"Sun Oct 1 10:53:28 2023","interfaces":[{"interface":"back","average":{"1min":2.585,"5min":2.956,"15min":2.2570000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32300000000000001},"max":{"1min":12.113,"5min":102.776,"15min":102.776},"last":0.48999999999999999},{"interface":"front","average":{"1min":1.5980000000000001,"5min":2.8210000000000002,"15min":2.2269999999999999},"min":{"1min":0.39900000000000002,"5min":0.36699999999999999,"15min":0.32100000000000001},"max":{"1min":3.5859999999999999,"5min":101.202,"15min":101.202},"last":1.399}]},{"osd":3,"last update":"Sun Oct 1 10:53:09 2023","interfaces":[{"interface":"back","average":{"1min":1.5660000000000001,"5min":1.9139999999999999,"15min":2.069},"min":{"1min":0.36599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":4.1100000000000003,"5min":10.452,"15min":61.817999999999998},"last":4.1310000000000002},{"interface":"front","average":{"1min":3.375,"5min":2.262,"15min":2.0430000000000001},"min":{"1min":0.56100000000000005,"5min":0.38100000000000001,"15min":0.315},"max":{"1min":12.042,"5min":12.042,"15min":61.219000000000001},"last":0.54100000000000004}]},{"osd":4,"last update":"Sun Oct 1 10:52:49 2023","interfaces":[{"interface":"back","average":{"1min":1.9159999999999999,"5min":2.2909999999999999,"15min":1.47},"min":{"1min":0.40500000000000003,"5min":0.35099999999999998,"15min":0.32400000000000001},"max":{"1min":7.7800000000000002,"5min":13.1,"15min":13.1},"last":1.2210000000000001},{"interface":"front","average":{"1min":2.4849999999999999,"5min":1.9930000000000001,"15min":1.524},"min":{"1min":0.39600000000000002,"5min":0.35799999999999998,"15min":0.315},"max":{"1min":12.157999999999999,"5min":12.157999999999999,"15min":12.157999999999999},"last":0.51800000000000002}]},{"osd":5,"last update":"Sun Oct 1 10:53:09 2023","interfaces":[{"interface":"back","average":{"1min":0.88600000000000001,"5min":1.472,"15min":1.4570000000000001},"min":{"1min":0.39300000000000002,"5min":0.36099999999999999,"15min":0.33000000000000002},"max":{"1min":3.0030000000000001,"5min":8.2279999999999998,"15min":12.398999999999999},"last":0.44600000000000001},{"interface":"front","average":{"1min":1.8180000000000001,"5min":1.9650000000000001,"15min":1.706},"min":{"1min":0.40000000000000002,"5min":0.308,"15min":0.308},"max":{"1min":5.9870000000000001,"5min":12.999000000000001,"15min":14.459},"last":9.9030000000000005}]},{"osd":7,"last update":"Sun Oct 1 10:53:01 2023","interfaces":[{"interface":"back","average":{"1min":1.7609999999999999,"5min":2.278,"15min":1.5700000000000001},"min":{"1min":0.438,"5min":0.32300000000000001,"15min":0.311},"max":{"1min":4.2949999999999999,"5min":13.276999999999999,"15min":13.276999999999999},"last":1.26},{"interface":"front","average":{"1min":1.663,"5min":1.984,"15min":1.6200000000000001},"min":{"1min":0.45000000000000001,"5min":0.38500000000000001,"15min":0.35999999999999999},"max":{"1min":5.0910000000000002,"5min":11.416,"15min":11.416},"last":0.50700000000000001}]}]},{"osd":1,"up_from":101,"seq":433791697153,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2720528,"kb_used_data":2171336,"kb_used_omap":4,"kb_used_meta":549179,"kb_avail":91024624,"statfs":{"total":95995035648,"available":93209214976,"internally_reserved":0,"allocated":2223448064,"data_stored":2222675522,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":562360206},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,2,1,6,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:53:18 2023","interfaces":[{"interface":"back","average":{"1min":1.4570000000000001,"5min":1.145,"15min":1.2290000000000001},"min":{"1min":0.224,"5min":0.215,"15min":0.189},"max":{"1min":9.1739999999999995,"5min":9.1739999999999995,"15min":37.058},"last":1.5449999999999999},{"interface":"front","average":{"1min":1.24,"5min":1.133,"15min":1.55},"min":{"1min":0.19700000000000001,"5min":0.192,"15min":0.192},"max":{"1min":7.4050000000000002,"5min":11.087,"15min":61.207999999999998},"last":1.6240000000000001}]},{"osd":2,"last update":"Sun Oct 1 10:53:41 2023","interfaces":[{"interface":"back","average":{"1min":11.474,"5min":3.5579999999999998,"15min":2.1859999999999999},"min":{"1min":0.25900000000000001,"5min":0.222,"15min":0.222},"max":{"1min":161.905,"5min":161.905,"15min":161.905},"last":0.84699999999999998},{"interface":"front","average":{"1min":11.602,"5min":3.4740000000000002,"15min":2.1930000000000001},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.23699999999999999},"max":{"1min":162.059,"5min":162.059,"15min":162.059},"last":1.7450000000000001}]},{"osd":3,"last update":"Sun Oct 1 10:53:18 2023","interfaces":[{"interface":"back","average":{"1min":2.214,"5min":1.3500000000000001,"15min":1.5389999999999999},"min":{"1min":0.255,"5min":0.23999999999999999,"15min":0.20000000000000001},"max":{"1min":7.3739999999999997,"5min":7.3739999999999997,"15min":62.020000000000003},"last":1.3080000000000001},{"interface":"front","average":{"1min":1.7629999999999999,"5min":1.454,"15min":1.6990000000000001},"min":{"1min":0.28100000000000003,"5min":0.27800000000000002,"15min":0.23999999999999999},"max":{"1min":7.5359999999999996,"5min":10.747999999999999,"15min":62.338999999999999},"last":0.78500000000000003}]},{"osd":4,"last update":"Sun Oct 1 10:53:41 2023","interfaces":[{"interface":"back","average":{"1min":1.278,"5min":1.3320000000000001,"15min":1.4990000000000001},"min":{"1min":0.34100000000000003,"5min":0.309,"15min":0.29199999999999998},"max":{"1min":3.4529999999999998,"5min":7.2839999999999998,"15min":61.588000000000001},"last":1.351},{"interface":"front","average":{"1min":1.228,"5min":1.5369999999999999,"15min":1.7070000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":3.6709999999999998,"5min":10.816000000000001,"15min":61.411000000000001},"last":1.6659999999999999}]},{"osd":5,"last update":"Sun Oct 1 10:53:18 2023","interfaces":[{"interface":"back","average":{"1min":1.4059999999999999,"5min":1.6910000000000001,"15min":1.778},"min":{"1min":0.46999999999999997,"5min":0.41799999999999998,"15min":0.38900000000000001},"max":{"1min":5.0650000000000004,"5min":7.9260000000000002,"15min":61.560000000000002},"last":0.751},{"interface":"front","average":{"1min":1.7649999999999999,"5min":1.696,"15min":1.6919999999999999},"min":{"1min":0.42899999999999999,"5min":0.35199999999999998,"15min":0.34699999999999998},"max":{"1min":7.4800000000000004,"5min":7.4800000000000004,"15min":61.667999999999999},"last":1.3819999999999999}]},{"osd":6,"last update":"Sun Oct 1 10:53:18 2023","interfaces":[{"interface":"back","average":{"1min":2.851,"5min":1.7609999999999999,"15min":1.7090000000000001},"min":{"1min":0.46200000000000002,"5min":0.371,"15min":0.371},"max":{"1min":8.1140000000000008,"5min":8.1140000000000008,"15min":62.219999999999999},"last":1.409},{"interface":"front","average":{"1min":1.6919999999999999,"5min":1.645,"15min":1.706},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":4.9729999999999999,"5min":9.7870000000000008,"15min":61.462000000000003},"last":1.722}]},{"osd":7,"last update":"Sun Oct 1 10:53:18 2023","interfaces":[{"interface":"back","average":{"1min":1.4430000000000001,"5min":1.5860000000000001,"15min":1.8320000000000001},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":5.0179999999999998,"5min":10.465,"15min":62.646999999999998},"last":1.698},{"interface":"front","average":{"1min":1.8320000000000001,"5min":1.6080000000000001,"15min":1.786},"min":{"1min":0.53300000000000003,"5min":0.39100000000000001,"15min":0.38100000000000001},"max":{"1min":7.7839999999999998,"5min":7.7839999999999998,"15min":62.048000000000002},"last":8.0429999999999993}]}]},{"osd":0,"up_from":86,"seq":369367187763,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2047740,"kb_used_data":1381684,"kb_used_omap":5,"kb_used_meta":665978,"kb_avail":91697412,"statfs":{"total":95995035648,"available":93898149888,"internally_reserved":0,"allocated":1414844416,"data_stored":1414088190,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":681962106},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,2,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 10:53:19 2023","interfaces":[{"interface":"back","average":{"1min":1.625,"5min":1.3979999999999999,"15min":1.2490000000000001},"min":{"1min":0.316,"5min":0.19700000000000001,"15min":0.184},"max":{"1min":6.0410000000000004,"5min":8.6479999999999997,"15min":15.074},"last":1.1160000000000001},{"interface":"front","average":{"1min":2.9129999999999998,"5min":1.4399999999999999,"15min":1.4079999999999999},"min":{"1min":0.25800000000000001,"5min":0.19900000000000001,"15min":0.19700000000000001},"max":{"1min":11.164999999999999,"5min":11.164999999999999,"15min":13.601000000000001},"last":0.255}]},{"osd":2,"last update":"Sun Oct 1 10:53:37 2023","interfaces":[{"interface":"back","average":{"1min":2.21,"5min":2.0489999999999999,"15min":1.3939999999999999},"min":{"1min":0.25600000000000001,"5min":0.23599999999999999,"15min":0.223},"max":{"1min":7.4790000000000001,"5min":42.709000000000003,"15min":42.709000000000003},"last":0.46400000000000002},{"interface":"front","average":{"1min":2.4380000000000002,"5min":2.1789999999999998,"15min":1.4410000000000001},"min":{"1min":0.29299999999999998,"5min":0.27800000000000002,"15min":0.221},"max":{"1min":5.4630000000000001,"5min":41.091999999999999,"15min":41.091999999999999},"last":0.60699999999999998}]},{"osd":3,"last update":"Sun Oct 1 10:53:19 2023","interfaces":[{"interface":"back","average":{"1min":1.8260000000000001,"5min":1.4730000000000001,"15min":1.4450000000000001},"min":{"1min":0.32700000000000001,"5min":0.29399999999999998,"15min":0.26200000000000001},"max":{"1min":7.4260000000000002,"5min":10.512,"15min":12.944000000000001},"last":0.65000000000000002},{"interface":"front","average":{"1min":2.0739999999999998,"5min":1.571,"15min":1.3779999999999999},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.245},"max":{"1min":11.888999999999999,"5min":11.888999999999999,"15min":11.888999999999999},"last":0.39000000000000001}]},{"osd":4,"last update":"Sun Oct 1 10:52:52 2023","interfaces":[{"interface":"back","average":{"1min":0.85199999999999998,"5min":1.8819999999999999,"15min":1.5029999999999999},"min":{"1min":0.38800000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":3.1360000000000001,"5min":11.379,"15min":11.379},"last":0.48299999999999998},{"interface":"front","average":{"1min":1.163,"5min":1.764,"15min":1.4299999999999999},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.31900000000000001},"max":{"1min":5.0289999999999999,"5min":11.362,"15min":11.362},"last":0.311}]},{"osd":5,"last update":"Sun Oct 1 10:53:19 2023","interfaces":[{"interface":"back","average":{"1min":1.73,"5min":1.667,"15min":1.597},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":7.3799999999999999,"5min":7.3799999999999999,"15min":12.91},"last":0.629},{"interface":"front","average":{"1min":1.3580000000000001,"5min":1.651,"15min":1.5109999999999999},"min":{"1min":0.47999999999999998,"5min":0.35699999999999998,"15min":0.34699999999999998},"max":{"1min":3.7069999999999999,"5min":9.8620000000000001,"15min":9.8620000000000001},"last":0.51800000000000002}]},{"osd":6,"last update":"Sun Oct 1 10:53:19 2023","interfaces":[{"interface":"back","average":{"1min":2.0190000000000001,"5min":1.8320000000000001,"15min":1.504},"min":{"1min":0.44500000000000001,"5min":0.373,"15min":0.36399999999999999},"max":{"1min":6.7969999999999997,"5min":9.8279999999999994,"15min":9.8279999999999994},"last":0.54400000000000004},{"interface":"front","average":{"1min":2.3380000000000001,"5min":1.7809999999999999,"15min":1.534},"min":{"1min":0.372,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":6.0999999999999996,"5min":10.111000000000001,"15min":10.111000000000001},"last":0.40100000000000002}]},{"osd":7,"last update":"Sun Oct 1 10:53:19 2023","interfaces":[{"interface":"back","average":{"1min":2.226,"5min":1.8049999999999999,"15min":1.591},"min":{"1min":0.45300000000000001,"5min":0.39700000000000002,"15min":0.36899999999999999},"max":{"1min":7.9370000000000003,"5min":15.839,"15min":15.839},"last":0.57399999999999995},{"interface":"front","average":{"1min":2.7200000000000002,"5min":1.7430000000000001,"15min":1.5840000000000001},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.38400000000000001},"max":{"1min":6.1479999999999997,"5min":8.5169999999999995,"15min":8.9440000000000008},"last":0.437}]}]},{"osd":2,"up_from":116,"seq":498216206539,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1957072,"kb_used_data":1599048,"kb_used_omap":7,"kb_used_meta":357880,"kb_avail":91788080,"statfs":{"total":95995035648,"available":93990993920,"internally_reserved":0,"allocated":1637425152,"data_stored":1636706611,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":366469201},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,0,5,2,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:53:37 2023","interfaces":[{"interface":"back","average":{"1min":1.744,"5min":1.5640000000000001,"15min":1.909},"min":{"1min":0.219,"5min":0.16900000000000001,"15min":0.16900000000000001},"max":{"1min":7.609,"5min":14.192,"15min":147.87},"last":0.34799999999999998},{"interface":"front","average":{"1min":2.0289999999999999,"5min":1.6759999999999999,"15min":2.0339999999999998},"min":{"1min":0.26300000000000001,"5min":0.20599999999999999,"15min":0.19500000000000001},"max":{"1min":8.8480000000000008,"5min":14.592000000000001,"15min":148.03100000000001},"last":0.33600000000000002}]},{"osd":1,"last update":"Sun Oct 1 10:53:37 2023","interfaces":[{"interface":"back","average":{"1min":1.756,"5min":2.5640000000000001,"15min":1.8260000000000001},"min":{"1min":0.23799999999999999,"5min":0.217,"15min":0.217},"max":{"1min":7.9880000000000004,"5min":122.249,"15min":122.249},"last":2.351},{"interface":"front","average":{"1min":3.3399999999999999,"5min":3.0150000000000001,"15min":2.0089999999999999},"min":{"1min":0.245,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":10.071999999999999,"5min":121.495,"15min":121.495},"last":4.8979999999999997}]},{"osd":3,"last update":"Sun Oct 1 10:53:06 2023","interfaces":[{"interface":"back","average":{"1min":1.4530000000000001,"5min":2.6989999999999998,"15min":1.9750000000000001},"min":{"1min":0.25800000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":3.2879999999999998,"5min":124.393,"15min":124.393},"last":0.38900000000000001},{"interface":"front","average":{"1min":1.6160000000000001,"5min":3.0179999999999998,"15min":2.1560000000000001},"min":{"1min":0.27800000000000002,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":4.1360000000000001,"5min":122.52200000000001,"15min":122.52200000000001},"last":0.221}]},{"osd":4,"last update":"Sun Oct 1 10:52:45 2023","interfaces":[{"interface":"back","average":{"1min":1.2609999999999999,"5min":1.75,"15min":1.4950000000000001},"min":{"1min":0.38300000000000001,"5min":0.36499999999999999,"15min":0.35799999999999998},"max":{"1min":3.6400000000000001,"5min":13.676,"15min":60.994},"last":1.7969999999999999},{"interface":"front","average":{"1min":1.7769999999999999,"5min":2.0459999999999998,"15min":1.6040000000000001},"min":{"1min":0.34100000000000003,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":4.5739999999999998,"5min":17.925000000000001,"15min":63.012},"last":4.9379999999999997}]},{"osd":5,"last update":"Sun Oct 1 10:53:37 2023","interfaces":[{"interface":"back","average":{"1min":1.573,"5min":1.889,"15min":1.7},"min":{"1min":0.53300000000000003,"5min":0.39800000000000002,"15min":0.34699999999999998},"max":{"1min":4.8630000000000004,"5min":10.178000000000001,"15min":61.021000000000001},"last":0.36599999999999999},{"interface":"front","average":{"1min":3.3239999999999998,"5min":2.1379999999999999,"15min":1.8360000000000001},"min":{"1min":0.36499999999999999,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":7.8520000000000003,"5min":7.8520000000000003,"15min":63.145000000000003},"last":4.9710000000000001}]},{"osd":6,"last update":"Sun Oct 1 10:53:37 2023","interfaces":[{"interface":"back","average":{"1min":3.5710000000000002,"5min":2.2469999999999999,"15min":1.7909999999999999},"min":{"1min":0.38200000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":9.6289999999999996,"5min":9.6289999999999996,"15min":63.289000000000001},"last":6.3860000000000001},{"interface":"front","average":{"1min":1.946,"5min":1.4890000000000001,"15min":1.4870000000000001},"min":{"1min":0.36599999999999999,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":7.3860000000000001,"5min":7.3860000000000001,"15min":60.579000000000001},"last":6.0380000000000003}]},{"osd":7,"last update":"Sun Oct 1 10:53:37 2023","interfaces":[{"interface":"back","average":{"1min":1.8069999999999999,"5min":1.3919999999999999,"15min":1.46},"min":{"1min":0.39900000000000002,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":7.5700000000000003,"5min":7.8289999999999997,"15min":60.493000000000002},"last":0.46000000000000002},{"interface":"front","average":{"1min":1.8500000000000001,"5min":1.7130000000000001,"15min":1.76},"min":{"1min":0.48299999999999998,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":4.234,"5min":8.4100000000000001,"15min":60.956000000000003},"last":0.40100000000000002}]}]},{"osd":3,"up_from":121,"seq":519691043011,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2286844,"kb_used_data":1862900,"kb_used_omap":2,"kb_used_meta":423933,"kb_avail":91458308,"statfs":{"total":95995035648,"available":93653307392,"internally_reserved":0,"allocated":1907609600,"data_stored":1906909316,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":434107445},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:53:09 2023","interfaces":[{"interface":"back","average":{"1min":1.375,"5min":4.5110000000000001,"15min":2.5939999999999999},"min":{"1min":0.192,"5min":0.192,"15min":0.187},"max":{"1min":4.7539999999999996,"5min":129.953,"15min":129.953},"last":3.0510000000000002},{"interface":"front","average":{"1min":2.0489999999999999,"5min":4.9089999999999998,"15min":2.8719999999999999},"min":{"1min":0.189,"5min":0.189,"15min":0.17299999999999999},"max":{"1min":6.0469999999999997,"5min":129.53999999999999,"15min":129.53999999999999},"last":3.1499999999999999}]},{"osd":1,"last update":"Sun Oct 1 10:53:09 2023","interfaces":[{"interface":"back","average":{"1min":2.5179999999999998,"5min":1.5669999999999999,"15min":1.694},"min":{"1min":0.29699999999999999,"5min":0.193,"15min":0.193},"max":{"1min":8.7910000000000004,"5min":9.6560000000000006,"15min":77.361000000000004},"last":2.9820000000000002},{"interface":"front","average":{"1min":2.48,"5min":1.679,"15min":1.819},"min":{"1min":0.25800000000000001,"5min":0.247,"15min":0.24099999999999999},"max":{"1min":6.7309999999999999,"5min":10.305,"15min":77.867000000000004},"last":3.177}]},{"osd":2,"last update":"Sun Oct 1 10:53:09 2023","interfaces":[{"interface":"back","average":{"1min":3.0059999999999998,"5min":1.6699999999999999,"15min":1.284},"min":{"1min":0.251,"5min":0.24299999999999999,"15min":0.222},"max":{"1min":15.148,"5min":15.148,"15min":15.148},"last":1.149},{"interface":"front","average":{"1min":2.7280000000000002,"5min":1.726,"15min":1.502},"min":{"1min":0.373,"5min":0.27900000000000003,"15min":0.26900000000000002},"max":{"1min":6.9020000000000001,"5min":9.8409999999999993,"15min":75.299999999999997},"last":3.0190000000000001}]},{"osd":4,"last update":"Sun Oct 1 10:53:46 2023","interfaces":[{"interface":"back","average":{"1min":1.321,"5min":1.454,"15min":1.5249999999999999},"min":{"1min":0.42899999999999999,"5min":0.39400000000000002,"15min":0.34599999999999997},"max":{"1min":5.9809999999999999,"5min":9.7370000000000001,"15min":75.388999999999996},"last":3.073},{"interface":"front","average":{"1min":1.5289999999999999,"5min":1.53,"15min":1.681},"min":{"1min":0.377,"5min":0.308,"15min":0.29999999999999999},"max":{"1min":4.4240000000000004,"5min":6.375,"15min":75.165999999999997},"last":0.92600000000000005}]},{"osd":5,"last update":"Sun Oct 1 10:53:09 2023","interfaces":[{"interface":"back","average":{"1min":1.9410000000000001,"5min":1.494,"15min":1.7529999999999999},"min":{"1min":0.39600000000000002,"5min":0.37,"15min":0.33100000000000002},"max":{"1min":4.5300000000000002,"5min":6.9260000000000002,"15min":75.221000000000004},"last":0.89600000000000002},{"interface":"front","average":{"1min":1.6359999999999999,"5min":1.5569999999999999,"15min":1.887},"min":{"1min":0.39400000000000002,"5min":0.34999999999999998,"15min":0.33200000000000002},"max":{"1min":6.1050000000000004,"5min":7.1820000000000004,"15min":75.231999999999999},"last":1.1779999999999999}]},{"osd":6,"last update":"Sun Oct 1 10:53:09 2023","interfaces":[{"interface":"back","average":{"1min":2.907,"5min":1.893,"15min":1.899},"min":{"1min":0.46000000000000002,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":8.0549999999999997,"5min":8.0549999999999997,"15min":75.257000000000005},"last":0.94899999999999995},{"interface":"front","average":{"1min":2.1640000000000001,"5min":1.917,"15min":2.0009999999999999},"min":{"1min":0.435,"5min":0.42299999999999999,"15min":0.33500000000000002},"max":{"1min":6.8209999999999997,"5min":7.1390000000000002,"15min":75.194000000000003},"last":1.109}]},{"osd":7,"last update":"Sun Oct 1 10:53:09 2023","interfaces":[{"interface":"back","average":{"1min":1.7250000000000001,"5min":1.5980000000000001,"15min":1.802},"min":{"1min":0.40999999999999998,"5min":0.376,"15min":0.35099999999999998},"max":{"1min":6.8410000000000002,"5min":7.2080000000000002,"15min":75.138999999999996},"last":1.206},{"interface":"front","average":{"1min":2.0619999999999998,"5min":1.7410000000000001,"15min":1.734},"min":{"1min":0.56799999999999995,"5min":0.41299999999999998,"15min":0.37},"max":{"1min":6.0259999999999998,"5min":9.8059999999999992,"15min":75.355000000000004},"last":3.101}]}]},{"osd":4,"up_from":126,"seq":541165879484,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1416148,"kb_used_data":1101068,"kb_used_omap":10,"kb_used_meta":315061,"kb_avail":92329004,"statfs":{"total":95995035648,"available":94544900096,"internally_reserved":0,"allocated":1127493632,"data_stored":1126770798,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":322623076},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:52:45 2023","interfaces":[{"interface":"back","average":{"1min":1.8109999999999999,"5min":2.3250000000000002,"15min":1.8120000000000001},"min":{"1min":0.314,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":9.4700000000000006,"5min":49.917000000000002,"15min":62.576000000000001},"last":4.9850000000000003},{"interface":"front","average":{"1min":1.1619999999999999,"5min":2.3290000000000002,"15min":1.7849999999999999},"min":{"1min":0.30199999999999999,"5min":0.28999999999999998,"15min":0.251},"max":{"1min":5.8239999999999998,"5min":50.042000000000002,"15min":64.941000000000003},"last":0.59099999999999997}]},{"osd":1,"last update":"Sun Oct 1 10:52:45 2023","interfaces":[{"interface":"back","average":{"1min":1.165,"5min":1.8029999999999999,"15min":1.7929999999999999},"min":{"1min":0.28699999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":7.2039999999999997,"5min":29.344000000000001,"15min":97.622},"last":4.2140000000000004},{"interface":"front","average":{"1min":2.0680000000000001,"5min":2.109,"15min":1.9299999999999999},"min":{"1min":0.35399999999999998,"5min":0.28699999999999998,"15min":0.28599999999999998},"max":{"1min":7.5730000000000004,"5min":29.251999999999999,"15min":97.346999999999994},"last":4.8979999999999997}]},{"osd":2,"last update":"Sun Oct 1 10:52:45 2023","interfaces":[{"interface":"back","average":{"1min":1.6060000000000001,"5min":2.9119999999999999,"15min":1.829},"min":{"1min":0.32700000000000001,"5min":0.30499999999999999,"15min":0.29899999999999999},"max":{"1min":7.5949999999999998,"5min":117.229,"15min":117.229},"last":5.0839999999999996},{"interface":"front","average":{"1min":1.4790000000000001,"5min":2.77,"15min":1.746},"min":{"1min":0.378,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.1360000000000001,"5min":116.922,"15min":116.922},"last":0.50600000000000001}]},{"osd":3,"last update":"Sun Oct 1 10:52:45 2023","interfaces":[{"interface":"back","average":{"1min":0.67300000000000004,"5min":2.0739999999999998,"15min":2.0670000000000002},"min":{"1min":0.36399999999999999,"5min":0.32900000000000001,"15min":0.29499999999999998},"max":{"1min":1.9590000000000001,"5min":30.637,"15min":61.398000000000003},"last":1.5920000000000001},{"interface":"front","average":{"1min":1.155,"5min":2.0310000000000001,"15min":2.02},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":7.2809999999999997,"5min":30.242999999999999,"15min":61.463000000000001},"last":4.2930000000000001}]},{"osd":5,"last update":"Sun Oct 1 10:52:45 2023","interfaces":[{"interface":"back","average":{"1min":0.71699999999999997,"5min":1.956,"15min":1.4419999999999999},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":1.9359999999999999,"5min":12.601000000000001,"15min":12.601000000000001},"last":4.6710000000000003},{"interface":"front","average":{"1min":1.4239999999999999,"5min":1.7869999999999999,"15min":1.504},"min":{"1min":0.38500000000000001,"5min":0.35499999999999998,"15min":0.28999999999999998},"max":{"1min":8.6709999999999994,"5min":11.725,"15min":16.779},"last":4.4400000000000004}]},{"osd":6,"last update":"Sun Oct 1 10:52:45 2023","interfaces":[{"interface":"back","average":{"1min":0.86599999999999999,"5min":2.214,"15min":1.544},"min":{"1min":0.46400000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":1.9670000000000001,"5min":12.563000000000001,"15min":13.646000000000001},"last":0.55300000000000005},{"interface":"front","average":{"1min":1.571,"5min":2.1379999999999999,"15min":1.585},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":8.5700000000000003,"5min":10.537000000000001,"15min":14.446999999999999},"last":4.2519999999999998}]},{"osd":7,"last update":"Sun Oct 1 10:52:45 2023","interfaces":[{"interface":"back","average":{"1min":8.0830000000000002,"5min":3.121,"15min":1.9079999999999999},"min":{"1min":0.49299999999999999,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":119.56100000000001,"5min":119.56100000000001,"15min":119.56100000000001},"last":8.2289999999999992},{"interface":"front","average":{"1min":8.1059999999999999,"5min":3.2109999999999999,"15min":2.1280000000000001},"min":{"1min":0.40200000000000002,"5min":0.39200000000000002,"15min":0.30399999999999999},"max":{"1min":120.44499999999999,"5min":120.44499999999999,"15min":120.44499999999999},"last":4.5430000000000001}]}]},{"osd":5,"up_from":36,"seq":154618823166,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1975248,"kb_used_data":1062792,"kb_used_omap":0,"kb_used_meta":912448,"kb_avail":91769904,"statfs":{"total":95995035648,"available":93972381696,"internally_reserved":0,"allocated":1088299008,"data_stored":1087811953,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":934346752},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,2,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:53:04 2023","interfaces":[{"interface":"back","average":{"1min":7.569,"5min":2.7559999999999998,"15min":1.9390000000000001},"min":{"1min":0.28000000000000003,"5min":0.27800000000000002,"15min":0.24099999999999999},"max":{"1min":123.39700000000001,"5min":123.39700000000001,"15min":123.39700000000001},"last":0.35999999999999999},{"interface":"front","average":{"1min":8.0139999999999993,"5min":3.0790000000000002,"15min":2.1579999999999999},"min":{"1min":0.29699999999999999,"5min":0.29299999999999998,"15min":0.26900000000000002},"max":{"1min":124.051,"5min":124.051,"15min":124.051},"last":2.7719999999999998}]},{"osd":1,"last update":"Sun Oct 1 10:53:04 2023","interfaces":[{"interface":"back","average":{"1min":1.1539999999999999,"5min":1.401,"15min":2.2370000000000001},"min":{"1min":0.32700000000000001,"5min":0.27700000000000002,"15min":0.27100000000000002},"max":{"1min":4.7949999999999999,"5min":6.3719999999999999,"15min":202.96600000000001},"last":0.39300000000000002},{"interface":"front","average":{"1min":1.9690000000000001,"5min":1.6719999999999999,"15min":2.5910000000000002},"min":{"1min":0.32500000000000001,"5min":0.31900000000000001,"15min":0.28100000000000003},"max":{"1min":6.6769999999999996,"5min":9.9209999999999994,"15min":203.05099999999999},"last":2.9950000000000001}]},{"osd":2,"last update":"Sun Oct 1 10:53:43 2023","interfaces":[{"interface":"back","average":{"1min":3.1669999999999998,"5min":1.9570000000000001,"15min":1.536},"min":{"1min":0.34499999999999997,"5min":0.307,"15min":0.29199999999999998},"max":{"1min":26.242000000000001,"5min":26.242000000000001,"15min":26.242000000000001},"last":0.41799999999999998},{"interface":"front","average":{"1min":2.6899999999999999,"5min":2.157,"15min":1.8440000000000001},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.317},"max":{"1min":25.643999999999998,"5min":25.643999999999998,"15min":25.643999999999998},"last":2.7330000000000001}]},{"osd":3,"last update":"Sun Oct 1 10:53:11 2023","interfaces":[{"interface":"back","average":{"1min":1.161,"5min":1.8560000000000001,"15min":2.4329999999999998},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.316},"max":{"1min":6.0599999999999996,"5min":13.625,"15min":70.506},"last":6.1269999999999998},{"interface":"front","average":{"1min":2.4060000000000001,"5min":2.0680000000000001,"15min":2.327},"min":{"1min":0.34799999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":7.6230000000000002,"5min":13.845000000000001,"15min":70.346999999999994},"last":0.495}]},{"osd":4,"last update":"Sun Oct 1 10:53:43 2023","interfaces":[{"interface":"back","average":{"1min":1.786,"5min":1.512,"15min":1.5089999999999999},"min":{"1min":0.46100000000000002,"5min":0.313,"15min":0.29399999999999998},"max":{"1min":6.6349999999999998,"5min":9.8889999999999993,"15min":15.170999999999999},"last":0.46300000000000002},{"interface":"front","average":{"1min":1.6120000000000001,"5min":1.9530000000000001,"15min":1.5429999999999999},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.316},"max":{"1min":9.1699999999999999,"5min":9.7270000000000003,"15min":9.7270000000000003},"last":2.214}]},{"osd":6,"last update":"Sun Oct 1 10:53:11 2023","interfaces":[{"interface":"back","average":{"1min":1.798,"5min":2.4889999999999999,"15min":1.7729999999999999},"min":{"1min":0.41699999999999998,"5min":0.39800000000000002,"15min":0.39200000000000002},"max":{"1min":4.7510000000000003,"5min":61.768999999999998,"15min":61.768999999999998},"last":1.3959999999999999},{"interface":"front","average":{"1min":1.98,"5min":2.427,"15min":1.9410000000000001},"min":{"1min":0.40100000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":5.5819999999999999,"5min":62.396999999999998,"15min":62.396999999999998},"last":0.65200000000000002}]},{"osd":7,"last update":"Sun Oct 1 10:53:11 2023","interfaces":[{"interface":"back","average":{"1min":1.4870000000000001,"5min":1.8280000000000001,"15min":1.8160000000000001},"min":{"1min":0.503,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":4.2679999999999998,"5min":13.673999999999999,"15min":13.673999999999999},"last":3.8290000000000002},{"interface":"front","average":{"1min":1.2390000000000001,"5min":1.7509999999999999,"15min":1.6879999999999999},"min":{"1min":0.46500000000000002,"5min":0.41099999999999998,"15min":0.372},"max":{"1min":3.9390000000000001,"5min":9.6929999999999996,"15min":11.635},"last":0.44500000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1405288448,"data_stored":1405288448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2204368896,"data_stored":2204368896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1630081024,"data_stored":1630076952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1899298816,"data_stored":1899298816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1123880960,"data_stored":1123876888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1075970048,"data_stored":1075970048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2157383680,"data_stored":2157379608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1630339072,"data_stored":1630339072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T10:53:53.245 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-01T10:53:53.245 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_6 pg_num 26 2023-10-01T10:53:53.282 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:53.342 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:53.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:53.349 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:53:53.450 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:53.472 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:53.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:53.485 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:53:53.585 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:53.616 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:53.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:53.626 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:53:53.726 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:53.772 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:53.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:53.778 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:53:53.875 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:53 smithi078 bash[75299]: cluster 2023-10-01T10:53:52.008281+0000 mgr.y (mgr.25072) 1090 : cluster 0 pgmap v1046: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 696 op/s 2023-10-01T10:53:53.880 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:53.911 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:53.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:53.913 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:53:54.014 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:54.036 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:54.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:54.038 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:53:54.139 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:54.147 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:53 smithi173 bash[58816]: cluster 2023-10-01T10:53:52.008281+0000 mgr.y (mgr.25072) 1090 : cluster 0 pgmap v1046: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 696 op/s 2023-10-01T10:53:54.174 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:54.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:54.181 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:53:54.281 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:54.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:54.317 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:54.317 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:53:54.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:53 smithi078 bash[78274]: cluster 2023-10-01T10:53:52.008281+0000 mgr.y (mgr.25072) 1090 : cluster 0 pgmap v1046: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 696 op/s 2023-10-01T10:53:54.418 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:54.457 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:54.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:54.464 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:53:54.485 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T10:53:54.564 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:54.592 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:54.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:54.601 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:54.702 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:54.734 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:54.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:54.738 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:54.839 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:54.867 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:54.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:54.870 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:54.971 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:54.996 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:54.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:54.998 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:55.099 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:55.126 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:55.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:55.128 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:53:55.229 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:55.247 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T10:53:55.254 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:55.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:55.260 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:53:55.360 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:55.400 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:55.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:55.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:53:55.506 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:55.551 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:55.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:55.556 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:53:55.657 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:55.683 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:55.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:55.688 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:53:55.789 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:55.813 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:55.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:55.815 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:53:55.915 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:55.940 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:55.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:55.943 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:56.044 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:56.051 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:55 smithi173 bash[58816]: cluster 2023-10-01T10:53:54.008966+0000 mgr.y (mgr.25072) 1091 : cluster 0 pgmap v1047: 89 pgs: 89 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2023-10-01T10:53:56.067 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:56.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:56.070 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:53:56.171 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:56.180 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:55 smithi078 bash[78274]: cluster 2023-10-01T10:53:54.008966+0000 mgr.y (mgr.25072) 1091 : cluster 0 pgmap v1047: 89 pgs: 89 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2023-10-01T10:53:56.180 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:55 smithi078 bash[75299]: cluster 2023-10-01T10:53:54.008966+0000 mgr.y (mgr.25072) 1091 : cluster 0 pgmap v1047: 89 pgs: 89 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2023-10-01T10:53:56.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:56.208 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:56.208 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:56.309 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:56.339 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:56.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:56.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:53:56.447 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:56.474 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:56.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:56.477 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:56.577 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:56.606 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:56.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:56.611 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:53:56.711 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:56.750 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:56.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:56.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:53:56.856 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:56.884 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:56.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:56.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:53:56.987 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:57.014 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:57.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:57.024 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:53:57.124 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:57.154 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:57.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:57.161 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:53:57.261 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:57.289 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:57.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:57.294 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:53:57.395 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:57.442 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:57.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:57.449 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:53:57.550 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:57.587 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:57.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:57.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:53:57.695 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:57.726 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:57.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:57.733 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:53:57.834 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:57.858 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:53:57.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:57.863 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:53:57.964 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:57.973 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:57 smithi078 bash[78274]: cluster 2023-10-01T10:53:56.009611+0000 mgr.y (mgr.25072) 1092 : cluster 0 pgmap v1048: 89 pgs: 89 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 691 op/s 2023-10-01T10:53:57.974 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:57 smithi078 bash[75299]: cluster 2023-10-01T10:53:56.009611+0000 mgr.y (mgr.25072) 1092 : cluster 0 pgmap v1048: 89 pgs: 89 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 691 op/s 2023-10-01T10:53:57.994 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:58.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:58.001 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:53:58.080 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:53:58.081 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:53:58.102 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:58.126 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:58.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:58.131 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:53:58.232 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:58.240 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:57 smithi173 bash[58816]: cluster 2023-10-01T10:53:56.009611+0000 mgr.y (mgr.25072) 1092 : cluster 0 pgmap v1048: 89 pgs: 89 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 691 op/s 2023-10-01T10:53:58.256 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:53:58.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:58.261 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:53:58.361 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:58.399 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:58.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:58.406 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:53:58.506 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:58.535 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:58.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:58.542 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:53:58.643 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:58.672 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:58.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:58.675 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:53:58.775 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:58.805 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:58.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:58.811 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:53:58.911 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:58.936 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:53:58.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:58.939 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:53:59.040 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:59.059 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:59.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:59.061 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:53:59.098 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:53:59.103 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:53:59.162 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:59.186 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:53:59.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:59.193 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:53:59.294 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:59.325 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:53:59.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:59.328 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:53:59.429 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:59.456 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:59.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:59.460 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:53:59.560 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:59.591 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:53:59.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:59.595 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:53:59.696 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:59.724 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:53:59.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:59.726 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:53:59.827 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:59.855 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:59.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:59.857 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:53:59.958 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:59.967 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:53:59 smithi078 bash[75299]: cluster 2023-10-01T10:53:58.010132+0000 mgr.y (mgr.25072) 1093 : cluster 0 pgmap v1049: 89 pgs: 89 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 668 op/s 2023-10-01T10:53:59.968 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:53:59 smithi078 bash[78274]: cluster 2023-10-01T10:53:58.010132+0000 mgr.y (mgr.25072) 1093 : cluster 0 pgmap v1049: 89 pgs: 89 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 668 op/s 2023-10-01T10:53:59.988 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:53:59.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:53:59.991 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:54:00.092 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:00.122 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:00.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:00.127 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:54:00.228 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:00.236 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:53:59 smithi173 bash[58816]: cluster 2023-10-01T10:53:58.010132+0000 mgr.y (mgr.25072) 1093 : cluster 0 pgmap v1049: 89 pgs: 89 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 668 op/s 2023-10-01T10:54:00.257 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:00.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:00.261 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:54:00.269 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:54:00 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:54:00] "GET /metrics HTTP/1.1" 200 36317 "" "Prometheus/2.33.4" 2023-10-01T10:54:00.362 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:00.384 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:00.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:00.386 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:54:00.488 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:00.525 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:00.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:00.533 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:54:00.634 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:00.658 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:00.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:00.661 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:54:00.762 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:00.793 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:00.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:00.797 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:54:00.897 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:00.924 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:00.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:00.927 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:54:01.027 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:01.057 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:01.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:01.062 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:01.163 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:01.187 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:01.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:01.192 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:54:01.293 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:01.316 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:01.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:01.319 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:54:01.420 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:01.456 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:01.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:01.459 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:54:01.559 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:01.587 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:01.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:01.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:54:01.690 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:01.721 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:01.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:01.725 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:01.826 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:01.871 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:01.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:01.873 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:54:01.973 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:01.983 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:01 smithi078 bash[75299]: cluster 2023-10-01T10:54:00.013946+0000 mgr.y (mgr.25072) 1094 : cluster 0 pgmap v1050: 89 pgs: 89 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 636 op/s 2023-10-01T10:54:01.984 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:01 smithi078 bash[78274]: cluster 2023-10-01T10:54:00.013946+0000 mgr.y (mgr.25072) 1094 : cluster 0 pgmap v1050: 89 pgs: 89 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 636 op/s 2023-10-01T10:54:01.996 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:01.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:02.000 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:54:02.101 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:02.128 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:02.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:02.130 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:54:02.231 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:02.261 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:02.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:02.263 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:54:02.364 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:02.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:01 smithi173 bash[58816]: cluster 2023-10-01T10:54:00.013946+0000 mgr.y (mgr.25072) 1094 : cluster 0 pgmap v1050: 89 pgs: 89 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 636 op/s 2023-10-01T10:54:02.403 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:02.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:02.407 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:54:02.507 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:02.539 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:02.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:02.544 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:54:02.645 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:02.669 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:02.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:02.671 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:54:02.772 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:02.815 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:02.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:02.821 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:54:02.911 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:02 smithi078 bash[75299]: audit 2023-10-01T10:54:02.439996+0000 mon.a (mon.0) 1332 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:54:02.922 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:02.946 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:02.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:02.954 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:54:03.055 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:03.079 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:03.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:03.082 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:03.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:02 smithi173 bash[58816]: audit 2023-10-01T10:54:02.439996+0000 mon.a (mon.0) 1332 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:54:03.183 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:03.217 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:03.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:03.223 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:54:03.313 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:02 smithi078 bash[78274]: audit 2023-10-01T10:54:02.439996+0000 mon.a (mon.0) 1332 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:54:03.323 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:03.346 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:03.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:03.351 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:54:03.452 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:03.480 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:03.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:03.483 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:54:03.584 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:03.612 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:03.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:03.619 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:54:03.720 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:03.742 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:03.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:03.745 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:54:03.846 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:03.868 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:03.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:03.871 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:54:03.972 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:03.980 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:03 smithi173 bash[58816]: cluster 2023-10-01T10:54:02.014414+0000 mgr.y (mgr.25072) 1095 : cluster 0 pgmap v1051: 89 pgs: 89 active+clean; 3.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 559 op/s 2023-10-01T10:54:03.997 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:03.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:03.999 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:54:04.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:03 smithi078 bash[78274]: cluster 2023-10-01T10:54:02.014414+0000 mgr.y (mgr.25072) 1095 : cluster 0 pgmap v1051: 89 pgs: 89 active+clean; 3.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 559 op/s 2023-10-01T10:54:04.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:03 smithi078 bash[75299]: cluster 2023-10-01T10:54:02.014414+0000 mgr.y (mgr.25072) 1095 : cluster 0 pgmap v1051: 89 pgs: 89 active+clean; 3.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 559 op/s 2023-10-01T10:54:04.100 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:04.130 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:04.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:04.137 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:54:04.238 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:04.263 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:04.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:04.266 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:54:04.367 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:04.394 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:04.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:04.403 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:54:04.504 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:04.535 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:04.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:04.539 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:54:04.639 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:04.663 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:04.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:04.669 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:04.770 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:04.809 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:04.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:04.815 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:54:04.915 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:04.920 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:04 smithi078 bash[75299]: cluster 2023-10-01T10:54:04.015023+0000 mgr.y (mgr.25072) 1096 : cluster 0 pgmap v1052: 89 pgs: 89 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 787 op/s 2023-10-01T10:54:04.920 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:04 smithi078 bash[78274]: cluster 2023-10-01T10:54:04.015023+0000 mgr.y (mgr.25072) 1096 : cluster 0 pgmap v1052: 89 pgs: 89 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 787 op/s 2023-10-01T10:54:04.952 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:04.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:04.955 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:54:05.055 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:05.081 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:05.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:05.084 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:54:05.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:04 smithi173 bash[58816]: cluster 2023-10-01T10:54:04.015023+0000 mgr.y (mgr.25072) 1096 : cluster 0 pgmap v1052: 89 pgs: 89 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 787 op/s 2023-10-01T10:54:05.184 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:05.223 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:05.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:05.227 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:54:05.328 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:05.355 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:05.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:05.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:54:05.464 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:05.485 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:05.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:05.488 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:54:05.589 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:05.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:05.630 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:05.630 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:05.731 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:05.756 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:05.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:05.761 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:54:05.861 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:05.887 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:05.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:05.891 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:54:05.992 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:06.013 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:06.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:06.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:54:06.117 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:06.141 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:06.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:06.143 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:54:06.244 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:06.278 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:06.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:06.284 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:54:06.385 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:06.416 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:06.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:06.418 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:54:06.519 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:06.544 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:06.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:06.549 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:54:06.649 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:06.684 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:06.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:06.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:54:06.788 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:06.824 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:06.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:06.826 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:54:06.930 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:06.962 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:06.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:06.966 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:54:07.067 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:07.100 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:07.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:07.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:07.203 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:07.225 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:07.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:07.228 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:54:07.328 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:07.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:07 smithi078 bash[75299]: cluster 2023-10-01T10:54:06.015848+0000 mgr.y (mgr.25072) 1097 : cluster 0 pgmap v1053: 89 pgs: 89 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 573 op/s 2023-10-01T10:54:07.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:07 smithi078 bash[78274]: cluster 2023-10-01T10:54:06.015848+0000 mgr.y (mgr.25072) 1097 : cluster 0 pgmap v1053: 89 pgs: 89 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 573 op/s 2023-10-01T10:54:07.339 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:07 smithi173 bash[58816]: cluster 2023-10-01T10:54:06.015848+0000 mgr.y (mgr.25072) 1097 : cluster 0 pgmap v1053: 89 pgs: 89 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 573 op/s 2023-10-01T10:54:07.355 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:07.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:07.362 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:54:07.463 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:07.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:07.509 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:07.509 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:54:07.610 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:07.644 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:07.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:07.648 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:54:07.749 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:07.780 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:07.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:07.785 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:54:07.885 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:07.916 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:07.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:07.927 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:54:08.028 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:08.061 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:08.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:08.064 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:54:08.165 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:08.187 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:08.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:08.190 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:54:08.291 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:08.314 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:08.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:08.321 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:54:08.422 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:08.443 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:08.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:08.450 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:54:08.551 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:08.577 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:08.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:08.580 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:54:08.681 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:08.713 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:08.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:08.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:54:08.815 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:08.862 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:08.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:08.865 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:54:08.966 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:08.989 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:08.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:08.997 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:54:09.098 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:09.129 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:09.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:09.135 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:54:09.236 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:09.267 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:09.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:09.274 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:54:09.375 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:09.403 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:09.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:09.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:54:09.508 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:09.557 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:09.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:09.564 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:09.664 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:09.695 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:09.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:09.699 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:09.800 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:09.808 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:09 smithi078 bash[75299]: cluster 2023-10-01T10:54:08.016374+0000 mgr.y (mgr.25072) 1098 : cluster 0 pgmap v1054: 89 pgs: 89 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 644 op/s 2023-10-01T10:54:09.808 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:09 smithi078 bash[78274]: cluster 2023-10-01T10:54:08.016374+0000 mgr.y (mgr.25072) 1098 : cluster 0 pgmap v1054: 89 pgs: 89 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 644 op/s 2023-10-01T10:54:09.829 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:09.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:09.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:54:09.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:09 smithi173 bash[58816]: cluster 2023-10-01T10:54:08.016374+0000 mgr.y (mgr.25072) 1098 : cluster 0 pgmap v1054: 89 pgs: 89 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 644 op/s 2023-10-01T10:54:09.934 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:09.964 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:09.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:09.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:54:10.070 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:10.101 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:10.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:10.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:10.204 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:10.229 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:10.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:10.234 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:54:10.331 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:54:10 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:54:10] "GET /metrics HTTP/1.1" 200 36316 "" "Prometheus/2.33.4" 2023-10-01T10:54:10.334 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:10.359 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:10.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:10.361 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:54:10.462 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:10.484 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:10.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:10.488 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:54:10.589 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:10.613 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:10.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:10.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:54:10.716 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:10.763 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:10.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:10.767 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:54:10.867 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:10.897 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:10.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:10.901 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:11.003 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:11.045 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:11.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:11.047 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:11.148 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:11.175 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:11.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:11.182 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:54:11.283 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:11.316 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:11.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:11.318 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:54:11.419 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:11.451 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:11.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:11.459 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:54:11.559 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:11.611 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:11.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:11.616 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:54:11.717 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:11.724 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:11 smithi173 bash[58816]: cluster 2023-10-01T10:54:10.016942+0000 mgr.y (mgr.25072) 1099 : cluster 0 pgmap v1055: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 715 op/s 2023-10-01T10:54:11.749 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:11.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:11.751 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:54:11.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:11 smithi078 bash[78274]: cluster 2023-10-01T10:54:10.016942+0000 mgr.y (mgr.25072) 1099 : cluster 0 pgmap v1055: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 715 op/s 2023-10-01T10:54:11.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:11 smithi078 bash[75299]: cluster 2023-10-01T10:54:10.016942+0000 mgr.y (mgr.25072) 1099 : cluster 0 pgmap v1055: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 715 op/s 2023-10-01T10:54:11.852 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:11.874 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2023-10-01T10:54:11.874 INFO:teuthology.orchestra.run.smithi078.stdout: "size": 20, 2023-10-01T10:54:11.874 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 600, 2023-10-01T10:54:11.874 INFO:teuthology.orchestra.run.smithi078.stdout: "ops": [ 2023-10-01T10:54:11.875 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.875 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.45818.0:42745 10.8 10:19bd7cbb:::benchmark_data_smithi078_135100_object42744:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e155)", 2023-10-01T10:54:11.875 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:46:36.231461+0000", 2023-10-01T10:54:11.875 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 455.63932395799998, 2023-10-01T10:54:11.875 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.205847792, 2023-10-01T10:54:11.875 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:54:11.875 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:54:11.875 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:54:11.875 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.45818", 2023-10-01T10:54:11.875 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1151403832", 2023-10-01T10:54:11.875 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 42745 2023-10-01T10:54:11.875 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.876 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:54:11.876 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.876 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:54:11.876 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.231461+0000", 2023-10-01T10:54:11.876 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:54:11.876 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.876 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.876 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:54:11.876 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.231461+0000", 2023-10-01T10:54:11.876 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:54:11.876 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.876 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.877 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:54:11.877 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.231459+0000", 2023-10-01T10:54:11.877 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999986 2023-10-01T10:54:11.877 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.877 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.878 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:54:11.878 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.231481+0000", 2023-10-01T10:54:11.878 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.2208000000000002e-05 2023-10-01T10:54:11.878 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.878 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.878 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:54:11.879 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.231492+0000", 2023-10-01T10:54:11.879 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1042e-05 2023-10-01T10:54:11.879 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.879 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.879 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:54:11.879 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.231498+0000", 2023-10-01T10:54:11.879 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.9050000000000002e-06 2023-10-01T10:54:11.879 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.879 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.879 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:54:11.879 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.231548+0000", 2023-10-01T10:54:11.879 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.0059000000000003e-05 2023-10-01T10:54:11.880 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.880 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.880 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:54:11.880 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.231697+0000", 2023-10-01T10:54:11.880 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00014874899999999999 2023-10-01T10:54:11.880 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.880 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.880 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 2,7", 2023-10-01T10:54:11.880 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.231822+0000", 2023-10-01T10:54:11.880 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000124559 2023-10-01T10:54:11.880 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.880 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.880 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:54:11.881 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.233086+0000", 2023-10-01T10:54:11.881 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.001264041 2023-10-01T10:54:11.881 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.881 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.881 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:54:11.881 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.234282+0000", 2023-10-01T10:54:11.881 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.001195907 2023-10-01T10:54:11.881 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.881 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.881 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:54:11.881 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.437224+0000", 2023-10-01T10:54:11.881 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.202942768 2023-10-01T10:54:11.882 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.882 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.882 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:54:11.882 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.437284+0000", 2023-10-01T10:54:11.882 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.9472999999999998e-05 2023-10-01T10:54:11.882 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.882 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.882 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:54:11.882 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.437308+0000", 2023-10-01T10:54:11.882 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.4477e-05 2023-10-01T10:54:11.882 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:54:11.882 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:54:11.883 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:54:11.883 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.883 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.883 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.45818.0:42751 10.8 10:105edb8c:::benchmark_data_smithi078_135100_object42750:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e155)", 2023-10-01T10:54:11.883 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:46:36.234798+0000", 2023-10-01T10:54:11.883 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 455.635986022, 2023-10-01T10:54:11.883 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.20421113599999999, 2023-10-01T10:54:11.883 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:54:11.884 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:54:11.884 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:54:11.884 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.45818", 2023-10-01T10:54:11.884 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1151403832", 2023-10-01T10:54:11.884 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 42751 2023-10-01T10:54:11.884 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.884 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:54:11.884 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.884 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:54:11.885 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.234798+0000", 2023-10-01T10:54:11.885 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:54:11.885 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.885 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.885 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:54:11.885 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.234798+0000", 2023-10-01T10:54:11.885 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:54:11.885 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.885 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.886 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:54:11.886 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.234797+0000", 2023-10-01T10:54:11.886 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:54:11.886 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.886 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.886 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:54:11.886 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.234817+0000", 2023-10-01T10:54:11.886 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.9562e-05 2023-10-01T10:54:11.886 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.886 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.886 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:54:11.886 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.234827+0000", 2023-10-01T10:54:11.886 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.0122000000000001e-05 2023-10-01T10:54:11.887 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.887 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.887 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:54:11.887 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.234833+0000", 2023-10-01T10:54:11.887 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.806e-06 2023-10-01T10:54:11.887 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.887 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.887 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:54:11.888 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.234868+0000", 2023-10-01T10:54:11.888 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.4915e-05 2023-10-01T10:54:11.888 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.888 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.888 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:54:11.888 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.234965+0000", 2023-10-01T10:54:11.888 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.7515000000000004e-05 2023-10-01T10:54:11.888 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.888 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.888 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 2,7", 2023-10-01T10:54:11.888 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.235057+0000", 2023-10-01T10:54:11.888 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.1446000000000005e-05 2023-10-01T10:54:11.889 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.889 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.889 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:54:11.889 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.236999+0000", 2023-10-01T10:54:11.889 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0019417169999999999 2023-10-01T10:54:11.889 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.889 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.889 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:54:11.890 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.238601+0000", 2023-10-01T10:54:11.890 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0016025670000000001 2023-10-01T10:54:11.890 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.890 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.890 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:54:11.890 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.438955+0000", 2023-10-01T10:54:11.890 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.20035378500000001 2023-10-01T10:54:11.890 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.890 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.890 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:54:11.890 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.438986+0000", 2023-10-01T10:54:11.890 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.0976000000000003e-05 2023-10-01T10:54:11.891 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.891 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.891 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:54:11.891 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.439010+0000", 2023-10-01T10:54:11.891 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.3685000000000001e-05 2023-10-01T10:54:11.891 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:54:11.891 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:54:11.891 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:54:11.892 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.892 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.892 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.45818.0:42752 10.8 10:17219ec3:::benchmark_data_smithi078_135100_object42751:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e155)", 2023-10-01T10:54:11.892 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:46:36.236052+0000", 2023-10-01T10:54:11.892 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 455.63473243700003, 2023-10-01T10:54:11.892 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.20305263000000001, 2023-10-01T10:54:11.892 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:54:11.892 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:54:11.892 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:54:11.892 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.45818", 2023-10-01T10:54:11.892 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1151403832", 2023-10-01T10:54:11.892 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 42752 2023-10-01T10:54:11.893 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.893 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:54:11.893 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.893 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:54:11.893 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.236052+0000", 2023-10-01T10:54:11.893 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:54:11.893 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.893 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.893 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:54:11.894 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.236052+0000", 2023-10-01T10:54:11.894 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:54:11.894 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.894 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.894 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:54:11.894 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.236051+0000", 2023-10-01T10:54:11.894 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:54:11.894 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.894 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.895 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:54:11.895 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.236073+0000", 2023-10-01T10:54:11.895 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.1943e-05 2023-10-01T10:54:11.895 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.895 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.895 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:54:11.895 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.236086+0000", 2023-10-01T10:54:11.895 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.2495e-05 2023-10-01T10:54:11.896 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.896 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.896 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:54:11.896 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.236095+0000", 2023-10-01T10:54:11.896 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.8920000000000006e-06 2023-10-01T10:54:11.897 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.897 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.897 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:54:11.897 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.236128+0000", 2023-10-01T10:54:11.897 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.3485000000000002e-05 2023-10-01T10:54:11.898 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.898 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.898 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:54:11.898 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.236211+0000", 2023-10-01T10:54:11.898 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.2773000000000002e-05 2023-10-01T10:54:11.898 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.898 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.898 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 2,7", 2023-10-01T10:54:11.898 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.236300+0000", 2023-10-01T10:54:11.898 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.9381000000000004e-05 2023-10-01T10:54:11.898 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.898 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.899 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:54:11.899 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.237130+0000", 2023-10-01T10:54:11.899 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00082942300000000003 2023-10-01T10:54:11.899 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.899 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.899 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:54:11.899 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.239639+0000", 2023-10-01T10:54:11.899 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0025098410000000001 2023-10-01T10:54:11.899 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.899 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.899 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:54:11.899 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.439043+0000", 2023-10-01T10:54:11.900 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.19940406799999999 2023-10-01T10:54:11.900 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.900 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.900 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:54:11.900 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.439079+0000", 2023-10-01T10:54:11.900 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.5794000000000003e-05 2023-10-01T10:54:11.900 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.900 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.900 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:54:11.900 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.439105+0000", 2023-10-01T10:54:11.900 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.5378e-05 2023-10-01T10:54:11.900 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:54:11.900 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:54:11.901 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:54:11.901 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.901 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.901 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.45818.0:42753 10.8 10:1df96728:::benchmark_data_smithi078_135100_object42752:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e155)", 2023-10-01T10:54:11.901 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:46:36.237240+0000", 2023-10-01T10:54:11.901 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 455.63354485799999, 2023-10-01T10:54:11.901 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.20196054099999999, 2023-10-01T10:54:11.901 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:54:11.901 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:54:11.901 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:54:11.901 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.45818", 2023-10-01T10:54:11.901 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1151403832", 2023-10-01T10:54:11.902 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 42753 2023-10-01T10:54:11.902 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.902 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:54:11.902 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.902 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:54:11.902 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.237240+0000", 2023-10-01T10:54:11.902 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:54:11.902 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.902 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.902 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:54:11.903 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.237240+0000", 2023-10-01T10:54:11.903 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:54:11.903 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.903 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.903 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:54:11.903 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.237239+0000", 2023-10-01T10:54:11.903 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:54:11.903 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.903 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.903 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:54:11.903 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.237279+0000", 2023-10-01T10:54:11.903 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.0454999999999998e-05 2023-10-01T10:54:11.904 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.904 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.904 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:54:11.904 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.237295+0000", 2023-10-01T10:54:11.904 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.5495999999999999e-05 2023-10-01T10:54:11.904 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.904 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.904 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:54:11.904 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.237304+0000", 2023-10-01T10:54:11.904 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.6309999999999998e-06 2023-10-01T10:54:11.904 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.904 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.904 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:54:11.905 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.237353+0000", 2023-10-01T10:54:11.905 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.9175999999999998e-05 2023-10-01T10:54:11.905 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.905 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.905 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:54:11.905 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.237464+0000", 2023-10-01T10:54:11.905 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000110742 2023-10-01T10:54:11.905 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.905 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.905 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 2,7", 2023-10-01T10:54:11.905 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.237588+0000", 2023-10-01T10:54:11.905 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000124065 2023-10-01T10:54:11.906 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.906 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.906 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:54:11.906 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.238744+0000", 2023-10-01T10:54:11.906 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0011562619999999999 2023-10-01T10:54:11.906 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.906 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.906 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:54:11.906 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.239781+0000", 2023-10-01T10:54:11.906 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.001036854 2023-10-01T10:54:11.906 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.906 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.907 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:54:11.907 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.439142+0000", 2023-10-01T10:54:11.907 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.19936047800000001 2023-10-01T10:54:11.907 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.907 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.907 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:54:11.907 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.439175+0000", 2023-10-01T10:54:11.907 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.3507999999999998e-05 2023-10-01T10:54:11.907 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.907 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.907 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:54:11.907 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.439200+0000", 2023-10-01T10:54:11.907 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.4808e-05 2023-10-01T10:54:11.908 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:54:11.908 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:54:11.908 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:54:11.908 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.908 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.908 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.45818.0:42757 10.8 10:11807276:::benchmark_data_smithi078_135100_object42756:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e155)", 2023-10-01T10:54:11.908 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:46:36.240015+0000", 2023-10-01T10:54:11.908 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 455.63076984200001, 2023-10-01T10:54:11.908 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.199250345, 2023-10-01T10:54:11.908 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:54:11.908 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:54:11.908 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:54:11.909 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.45818", 2023-10-01T10:54:11.909 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1151403832", 2023-10-01T10:54:11.909 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 42757 2023-10-01T10:54:11.909 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.909 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:54:11.909 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.909 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:54:11.909 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.240015+0000", 2023-10-01T10:54:11.909 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:54:11.909 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.909 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.909 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:54:11.909 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.240015+0000", 2023-10-01T10:54:11.910 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:54:11.910 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.910 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.910 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:54:11.910 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.240013+0000", 2023-10-01T10:54:11.910 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:54:11.910 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.910 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.910 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:54:11.910 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.240037+0000", 2023-10-01T10:54:11.910 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.3091999999999998e-05 2023-10-01T10:54:11.911 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.911 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.911 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:54:11.911 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.240046+0000", 2023-10-01T10:54:11.911 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.1759999999999999e-06 2023-10-01T10:54:11.911 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.911 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.911 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:54:11.911 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.240051+0000", 2023-10-01T10:54:11.911 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.242e-06 2023-10-01T10:54:11.911 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.911 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.912 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:54:11.912 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.240084+0000", 2023-10-01T10:54:11.912 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.2706000000000002e-05 2023-10-01T10:54:11.912 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.912 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.912 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:54:11.912 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.240174+0000", 2023-10-01T10:54:11.912 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.0367999999999999e-05 2023-10-01T10:54:11.912 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.913 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.913 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 2,7", 2023-10-01T10:54:11.913 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.240260+0000", 2023-10-01T10:54:11.913 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.6346000000000003e-05 2023-10-01T10:54:11.913 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.913 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.913 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:54:11.913 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.241355+0000", 2023-10-01T10:54:11.913 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0010945480000000001 2023-10-01T10:54:11.913 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.913 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.913 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:54:11.914 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.241891+0000", 2023-10-01T10:54:11.914 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00053648400000000005 2023-10-01T10:54:11.914 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.914 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.914 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:54:11.914 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.439223+0000", 2023-10-01T10:54:11.914 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.197331125 2023-10-01T10:54:11.914 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.914 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.914 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:54:11.914 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.439247+0000", 2023-10-01T10:54:11.914 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.4085000000000001e-05 2023-10-01T10:54:11.915 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.915 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.915 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:54:11.915 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:36.439265+0000", 2023-10-01T10:54:11.915 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.8326999999999999e-05 2023-10-01T10:54:11.915 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:54:11.915 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:54:11.915 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:54:11.915 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.915 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.915 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.45818.0:54054 10.4 10:20463acc:::benchmark_data_smithi078_135100_object54053:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e156)", 2023-10-01T10:54:11.915 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:46:44.823896+0000", 2023-10-01T10:54:11.916 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 447.04688860099998, 2023-10-01T10:54:11.916 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.17607024800000001, 2023-10-01T10:54:11.916 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:54:11.916 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:54:11.916 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:54:11.916 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.45818", 2023-10-01T10:54:11.916 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1151403832", 2023-10-01T10:54:11.916 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 54054 2023-10-01T10:54:11.916 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.916 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:54:11.916 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.917 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:54:11.917 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.823896+0000", 2023-10-01T10:54:11.917 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:54:11.917 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.917 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.917 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:54:11.917 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.823896+0000", 2023-10-01T10:54:11.917 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:54:11.917 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.917 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.917 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:54:11.917 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.823895+0000", 2023-10-01T10:54:11.918 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:54:11.918 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.918 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.918 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:54:11.918 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.823918+0000", 2023-10-01T10:54:11.918 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.2527e-05 2023-10-01T10:54:11.918 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.918 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.919 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:54:11.919 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.823929+0000", 2023-10-01T10:54:11.919 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1566000000000001e-05 2023-10-01T10:54:11.919 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.919 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.919 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:54:11.919 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.823983+0000", 2023-10-01T10:54:11.919 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.3429e-05 2023-10-01T10:54:11.919 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.919 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.920 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:54:11.920 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.983693+0000", 2023-10-01T10:54:11.920 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.159710037 2023-10-01T10:54:11.920 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.920 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.920 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:54:11.920 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.983916+0000", 2023-10-01T10:54:11.920 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000223253 2023-10-01T10:54:11.921 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.921 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.921 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 3,4", 2023-10-01T10:54:11.921 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.984235+0000", 2023-10-01T10:54:11.921 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00031897099999999998 2023-10-01T10:54:11.921 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.921 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.921 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:54:11.922 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.989508+0000", 2023-10-01T10:54:11.922 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0052732079999999997 2023-10-01T10:54:11.922 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.922 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.922 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:54:11.922 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.989643+0000", 2023-10-01T10:54:11.922 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000135088 2023-10-01T10:54:11.922 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.922 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.922 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:54:11.922 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.999901+0000", 2023-10-01T10:54:11.922 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.010257547000000001 2023-10-01T10:54:11.923 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.923 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.923 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:54:11.923 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.999935+0000", 2023-10-01T10:54:11.923 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.4712999999999998e-05 2023-10-01T10:54:11.923 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.923 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.923 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:54:11.924 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:46:44.999966+0000", 2023-10-01T10:54:11.924 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.0772999999999998e-05 2023-10-01T10:54:11.924 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:54:11.924 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:54:11.924 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:54:11.924 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.924 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.924 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.45818.0:92524 10.8 10:19f5325b:::benchmark_data_smithi078_135100_object92523:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-10-01T10:54:11.924 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:47:22.357629+0000", 2023-10-01T10:54:11.924 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 409.51315588599999, 2023-10-01T10:54:11.924 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.193231981, 2023-10-01T10:54:11.924 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:54:11.924 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:54:11.925 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:54:11.925 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.45818", 2023-10-01T10:54:11.925 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1151403832", 2023-10-01T10:54:11.925 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 92524 2023-10-01T10:54:11.925 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.925 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:54:11.925 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.925 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:54:11.925 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.357629+0000", 2023-10-01T10:54:11.925 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:54:11.925 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.926 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.926 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:54:11.926 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.357629+0000", 2023-10-01T10:54:11.926 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:54:11.926 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.926 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.926 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:54:11.927 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.357626+0000", 2023-10-01T10:54:11.927 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999971 2023-10-01T10:54:11.927 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.927 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.927 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:54:11.927 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.357654+0000", 2023-10-01T10:54:11.927 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.7929e-05 2023-10-01T10:54:11.927 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.927 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.927 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:54:11.927 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.357668+0000", 2023-10-01T10:54:11.928 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.4324999999999999e-05 2023-10-01T10:54:11.928 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.928 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.928 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:54:11.928 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.357678+0000", 2023-10-01T10:54:11.928 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.0059e-05 2023-10-01T10:54:11.928 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.928 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.928 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:54:11.928 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.357756+0000", 2023-10-01T10:54:11.929 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.8045000000000002e-05 2023-10-01T10:54:11.929 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.929 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.929 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:54:11.929 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.357925+0000", 2023-10-01T10:54:11.929 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000168474 2023-10-01T10:54:11.929 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.929 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.929 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 2,7", 2023-10-01T10:54:11.929 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.360941+0000", 2023-10-01T10:54:11.930 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0030160410000000001 2023-10-01T10:54:11.930 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.930 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.930 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:54:11.930 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.366151+0000", 2023-10-01T10:54:11.930 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0052104849999999999 2023-10-01T10:54:11.930 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.930 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.930 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:54:11.931 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.371651+0000", 2023-10-01T10:54:11.931 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0055003150000000004 2023-10-01T10:54:11.931 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.931 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.931 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:54:11.931 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.550728+0000", 2023-10-01T10:54:11.931 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.17907624799999999 2023-10-01T10:54:11.931 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.931 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.931 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:54:11.932 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.550806+0000", 2023-10-01T10:54:11.932 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.8757000000000005e-05 2023-10-01T10:54:11.932 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.932 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.932 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:54:11.932 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.550861+0000", 2023-10-01T10:54:11.932 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.4184000000000002e-05 2023-10-01T10:54:11.932 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:54:11.932 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:54:11.933 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:54:11.933 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.933 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.933 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.45818.0:92546 10.8 10:14e40b42:::benchmark_data_smithi078_135100_object92545:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e158)", 2023-10-01T10:54:11.933 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:47:22.379228+0000", 2023-10-01T10:54:11.933 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 409.49155689700001, 2023-10-01T10:54:11.933 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.17459656600000001, 2023-10-01T10:54:11.933 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:54:11.933 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:54:11.934 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:54:11.934 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.45818", 2023-10-01T10:54:11.934 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1151403832", 2023-10-01T10:54:11.934 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 92546 2023-10-01T10:54:11.934 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.934 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:54:11.934 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.934 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:54:11.934 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.379228+0000", 2023-10-01T10:54:11.935 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:54:11.935 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.935 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.935 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:54:11.935 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.379228+0000", 2023-10-01T10:54:11.935 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:54:11.935 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.935 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.935 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:54:11.936 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.379226+0000", 2023-10-01T10:54:11.936 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999986 2023-10-01T10:54:11.936 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.936 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.936 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:54:11.936 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.379251+0000", 2023-10-01T10:54:11.936 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.4451999999999999e-05 2023-10-01T10:54:11.936 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.936 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.936 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:54:11.936 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.379263+0000", 2023-10-01T10:54:11.936 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.2632999999999999e-05 2023-10-01T10:54:11.937 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.937 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.937 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:54:11.937 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.379271+0000", 2023-10-01T10:54:11.937 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.3760000000000002e-06 2023-10-01T10:54:11.937 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.937 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.937 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:54:11.937 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.379318+0000", 2023-10-01T10:54:11.937 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.7802000000000001e-05 2023-10-01T10:54:11.937 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.937 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.937 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:54:11.938 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.379456+0000", 2023-10-01T10:54:11.938 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000137832 2023-10-01T10:54:11.938 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.938 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.938 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 2,7", 2023-10-01T10:54:11.938 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.379603+0000", 2023-10-01T10:54:11.938 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00014640000000000001 2023-10-01T10:54:11.938 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.938 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.938 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:54:11.938 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.383268+0000", 2023-10-01T10:54:11.938 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.003665354 2023-10-01T10:54:11.938 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.939 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.939 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:54:11.939 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.385098+0000", 2023-10-01T10:54:11.939 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0018296689999999999 2023-10-01T10:54:11.939 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.939 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.939 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:54:11.939 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.553673+0000", 2023-10-01T10:54:11.939 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.168575055 2023-10-01T10:54:11.939 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.939 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.939 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:54:11.939 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.553777+0000", 2023-10-01T10:54:11.939 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00010442000000000001 2023-10-01T10:54:11.940 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.940 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.941 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:54:11.941 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:47:22.553824+0000", 2023-10-01T10:54:11.941 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.6996999999999997e-05 2023-10-01T10:54:11.941 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:54:11.941 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:54:11.941 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:54:11.941 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.942 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.942 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.45818.0:167384 10.4 10:294e870d:::benchmark_data_smithi078_135100_object57471:head [delete] snapc 0=[] ondisk+write+known_if_redirected e165)", 2023-10-01T10:54:11.942 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:49:01.717710+0000", 2023-10-01T10:54:11.942 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 310.15307443799998, 2023-10-01T10:54:11.942 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.18218202, 2023-10-01T10:54:11.942 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:54:11.942 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:54:11.942 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:54:11.942 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.45818", 2023-10-01T10:54:11.942 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1151403832", 2023-10-01T10:54:11.942 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 167384 2023-10-01T10:54:11.943 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.943 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:54:11.943 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.943 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:54:11.943 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:49:01.717710+0000", 2023-10-01T10:54:11.943 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:54:11.943 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.943 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.943 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:54:11.943 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:49:01.717710+0000", 2023-10-01T10:54:11.943 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:54:11.943 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.944 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.944 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:54:11.944 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:49:01.717709+0000", 2023-10-01T10:54:11.944 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999986 2023-10-01T10:54:11.944 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.944 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.944 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:54:11.944 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:49:01.717713+0000", 2023-10-01T10:54:11.944 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.7979999999999996e-06 2023-10-01T10:54:11.944 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.944 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.944 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:54:11.944 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:49:01.862169+0000", 2023-10-01T10:54:11.945 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.144455798 2023-10-01T10:54:11.945 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.945 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.945 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:54:11.945 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:49:01.862180+0000", 2023-10-01T10:54:11.945 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.0363000000000001e-05 2023-10-01T10:54:11.945 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.945 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.945 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:54:11.945 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:49:01.862227+0000", 2023-10-01T10:54:11.945 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.7101000000000002e-05 2023-10-01T10:54:11.946 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.946 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.946 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:54:11.946 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:49:01.862375+0000", 2023-10-01T10:54:11.946 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000148443 2023-10-01T10:54:11.946 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.946 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.946 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 3,4", 2023-10-01T10:54:11.946 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:49:01.862512+0000", 2023-10-01T10:54:11.946 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000136565 2023-10-01T10:54:11.946 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.946 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.947 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:54:11.947 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:49:01.871946+0000", 2023-10-01T10:54:11.947 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0094341430000000007 2023-10-01T10:54:11.947 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.947 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.947 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:54:11.947 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:49:01.882176+0000", 2023-10-01T10:54:11.947 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.010230484 2023-10-01T10:54:11.947 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.947 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.947 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:54:11.947 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:49:01.899791+0000", 2023-10-01T10:54:11.948 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.017614538999999999 2023-10-01T10:54:11.948 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.948 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.948 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:54:11.948 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:49:01.899849+0000", 2023-10-01T10:54:11.948 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.8632999999999999e-05 2023-10-01T10:54:11.948 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.948 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.948 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:54:11.948 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:49:01.899892+0000", 2023-10-01T10:54:11.949 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.2617000000000002e-05 2023-10-01T10:54:11.949 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:54:11.949 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:54:11.949 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:54:11.949 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.949 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.949 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.46037.0:1705 11.9 e178/173 11:992a5915:::benchmark_data_smithi078_157677_object1704:head v 178'110, mlcod=178'106)", 2023-10-01T10:54:11.949 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:51:21.803189+0000", 2023-10-01T10:54:11.949 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 170.06759534299999, 2023-10-01T10:54:11.949 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.17539738599999999, 2023-10-01T10:54:11.949 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:54:11.949 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:54:11.950 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:54:11.950 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.950 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:54:11.950 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:21.803189+0000", 2023-10-01T10:54:11.950 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:54:11.950 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.950 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.950 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:54:11.950 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:21.803189+0000", 2023-10-01T10:54:11.950 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:54:11.950 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.950 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.951 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:54:11.951 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:21.803188+0000", 2023-10-01T10:54:11.951 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999995 2023-10-01T10:54:11.951 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.951 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.951 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:54:11.951 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:21.803218+0000", 2023-10-01T10:54:11.951 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.9583000000000001e-05 2023-10-01T10:54:11.951 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.951 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.951 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:54:11.952 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:21.803226+0000", 2023-10-01T10:54:11.952 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.4260000000000003e-06 2023-10-01T10:54:11.952 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.952 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.952 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:54:11.952 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:21.803262+0000", 2023-10-01T10:54:11.952 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.5692999999999999e-05 2023-10-01T10:54:11.952 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.952 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.952 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:54:11.952 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:21.803302+0000", 2023-10-01T10:54:11.952 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.9558999999999999e-05 2023-10-01T10:54:11.953 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.953 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.953 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:54:11.953 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:21.803315+0000", 2023-10-01T10:54:11.953 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.2829e-05 2023-10-01T10:54:11.953 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.953 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.953 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:54:11.953 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:21.978493+0000", 2023-10-01T10:54:11.953 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.175178103 2023-10-01T10:54:11.953 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.953 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.953 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:54:11.954 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:21.978587+0000", 2023-10-01T10:54:11.954 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.3840999999999998e-05 2023-10-01T10:54:11.954 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:54:11.954 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:54:11.954 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:54:11.954 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.954 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.954 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46037.0:1723 11.6 11:6daca9cf:::benchmark_data_smithi078_157677_object1722:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e178)", 2023-10-01T10:54:11.954 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:51:21.803626+0000", 2023-10-01T10:54:11.954 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 170.06715896, 2023-10-01T10:54:11.954 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.179061997, 2023-10-01T10:54:11.955 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:54:11.955 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:54:11.955 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:54:11.955 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46037", 2023-10-01T10:54:11.955 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:54:11.955 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 1723 2023-10-01T10:54:11.955 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.955 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:54:11.955 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.955 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:54:11.955 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:21.803626+0000", 2023-10-01T10:54:11.955 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:54:11.955 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.956 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.956 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:54:11.956 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:21.803626+0000", 2023-10-01T10:54:11.956 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:54:11.956 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.956 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.956 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:54:11.956 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:21.803624+0000", 2023-10-01T10:54:11.956 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999986 2023-10-01T10:54:11.956 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.956 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.957 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:54:11.957 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:21.803643+0000", 2023-10-01T10:54:11.957 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.8655999999999999e-05 2023-10-01T10:54:11.957 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.957 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.957 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:54:11.957 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:21.978452+0000", 2023-10-01T10:54:11.957 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.174809405 2023-10-01T10:54:11.957 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.957 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.957 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:54:11.957 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:21.978479+0000", 2023-10-01T10:54:11.958 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.6602999999999999e-05 2023-10-01T10:54:11.958 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.958 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.958 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:54:11.958 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:21.978548+0000", 2023-10-01T10:54:11.958 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.9326000000000007e-05 2023-10-01T10:54:11.958 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.958 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.958 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:54:11.958 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:21.978700+0000", 2023-10-01T10:54:11.958 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00015182300000000001 2023-10-01T10:54:11.958 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.959 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.959 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 1,2", 2023-10-01T10:54:11.959 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:21.978864+0000", 2023-10-01T10:54:11.959 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00016357200000000001 2023-10-01T10:54:11.959 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.959 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.959 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:54:11.959 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:21.979844+0000", 2023-10-01T10:54:11.959 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00098083900000000006 2023-10-01T10:54:11.959 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.959 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.959 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:54:11.960 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:21.980761+0000", 2023-10-01T10:54:11.960 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00091686299999999997 2023-10-01T10:54:11.960 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.960 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.960 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:54:11.960 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:21.982603+0000", 2023-10-01T10:54:11.960 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.001842116 2023-10-01T10:54:11.960 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.960 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.961 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:54:11.961 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:21.982651+0000", 2023-10-01T10:54:11.961 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.7173999999999999e-05 2023-10-01T10:54:11.961 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.961 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.961 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:54:11.961 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:21.982687+0000", 2023-10-01T10:54:11.961 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.6881999999999997e-05 2023-10-01T10:54:11.961 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:54:11.961 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:54:11.961 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:54:11.961 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.962 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.962 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46037.0:10166 11.6 11:6bf7ba40:::benchmark_data_smithi078_157677_object10165:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e178)", 2023-10-01T10:54:11.962 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:51:25.278548+0000", 2023-10-01T10:54:11.962 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 166.592236117, 2023-10-01T10:54:11.962 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.20550955800000001, 2023-10-01T10:54:11.962 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:54:11.962 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:54:11.962 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:54:11.962 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46037", 2023-10-01T10:54:11.963 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:54:11.963 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 10166 2023-10-01T10:54:11.963 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.963 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:54:11.963 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.963 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:54:11.963 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:25.278548+0000", 2023-10-01T10:54:11.963 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:54:11.963 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.963 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.964 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:54:11.964 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:25.278548+0000", 2023-10-01T10:54:11.964 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:54:11.964 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.964 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.964 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:54:11.964 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:25.278547+0000", 2023-10-01T10:54:11.964 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:54:11.964 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.964 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.964 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:54:11.965 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:25.278596+0000", 2023-10-01T10:54:11.965 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.8681999999999999e-05 2023-10-01T10:54:11.965 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.965 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.965 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:54:11.965 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:25.278705+0000", 2023-10-01T10:54:11.965 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00010932199999999999 2023-10-01T10:54:11.965 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.966 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.966 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:54:11.966 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:25.278786+0000", 2023-10-01T10:54:11.966 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.0581000000000007e-05 2023-10-01T10:54:11.966 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.966 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.966 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:54:11.966 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:25.278896+0000", 2023-10-01T10:54:11.966 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000110146 2023-10-01T10:54:11.966 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.966 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.967 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:54:11.967 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:25.279021+0000", 2023-10-01T10:54:11.967 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00012549300000000001 2023-10-01T10:54:11.967 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.967 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.967 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 1,2", 2023-10-01T10:54:11.967 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:25.279115+0000", 2023-10-01T10:54:11.967 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.3138000000000002e-05 2023-10-01T10:54:11.967 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.967 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.968 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:54:11.968 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:25.280181+0000", 2023-10-01T10:54:11.968 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0010669200000000001 2023-10-01T10:54:11.968 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.968 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.968 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:54:11.968 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:25.282563+0000", 2023-10-01T10:54:11.968 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0023815450000000001 2023-10-01T10:54:11.968 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.968 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.968 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:54:11.969 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:25.483958+0000", 2023-10-01T10:54:11.969 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.20139542799999999 2023-10-01T10:54:11.969 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.969 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.969 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:54:11.969 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:25.484017+0000", 2023-10-01T10:54:11.969 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.8471999999999999e-05 2023-10-01T10:54:11.969 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.969 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.969 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:54:11.970 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:25.484058+0000", 2023-10-01T10:54:11.970 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.0992999999999999e-05 2023-10-01T10:54:11.970 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:54:11.970 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:54:11.970 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:54:11.970 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.970 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.970 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46037.0:10167 11.6 11:6f38b541:::benchmark_data_smithi078_157677_object10166:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e178)", 2023-10-01T10:54:11.970 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:51:25.278801+0000", 2023-10-01T10:54:11.971 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 166.59198384699999, 2023-10-01T10:54:11.971 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.20546421500000001, 2023-10-01T10:54:11.971 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:54:11.971 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:54:11.971 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:54:11.971 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46037", 2023-10-01T10:54:11.971 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:54:11.971 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 10167 2023-10-01T10:54:11.972 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.972 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:54:11.972 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.972 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:54:11.972 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:25.278801+0000", 2023-10-01T10:54:11.972 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:54:11.972 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.972 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.972 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:54:11.973 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:25.278801+0000", 2023-10-01T10:54:11.973 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:54:11.973 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.973 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.973 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:54:11.973 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:25.278800+0000", 2023-10-01T10:54:11.973 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999995 2023-10-01T10:54:11.974 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.974 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.974 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:54:11.974 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:25.278839+0000", 2023-10-01T10:54:11.974 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.8998000000000002e-05 2023-10-01T10:54:11.974 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.974 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.974 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:54:11.974 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:25.278848+0000", 2023-10-01T10:54:11.974 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.6459999999999999e-06 2023-10-01T10:54:11.974 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.975 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.975 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:54:11.975 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:25.278854+0000", 2023-10-01T10:54:11.975 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.652e-06 2023-10-01T10:54:11.975 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.975 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.975 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:54:11.975 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:25.279536+0000", 2023-10-01T10:54:11.975 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00068122300000000001 2023-10-01T10:54:11.975 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.975 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.976 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:54:11.976 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:25.279607+0000", 2023-10-01T10:54:11.976 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.1653e-05 2023-10-01T10:54:11.976 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.976 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.976 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 1,2", 2023-10-01T10:54:11.976 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:25.279720+0000", 2023-10-01T10:54:11.976 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000112734 2023-10-01T10:54:11.976 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.976 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.976 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:54:11.976 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:25.281188+0000", 2023-10-01T10:54:11.977 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.001468502 2023-10-01T10:54:11.977 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.977 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.977 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:54:11.977 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:25.283641+0000", 2023-10-01T10:54:11.977 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0024526249999999999 2023-10-01T10:54:11.977 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.977 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.977 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:54:11.977 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:25.484170+0000", 2023-10-01T10:54:11.977 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.20052882499999999 2023-10-01T10:54:11.977 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.977 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.978 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:54:11.978 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:25.484221+0000", 2023-10-01T10:54:11.978 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.0674000000000003e-05 2023-10-01T10:54:11.978 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.978 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.978 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:54:11.978 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:25.484265+0000", 2023-10-01T10:54:11.978 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.4221999999999998e-05 2023-10-01T10:54:11.978 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:54:11.978 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:54:11.978 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:54:11.978 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.979 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.979 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46037.0:10172 11.6 11:6b80df2b:::benchmark_data_smithi078_157677_object10171:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e178)", 2023-10-01T10:54:11.979 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:51:25.280313+0000", 2023-10-01T10:54:11.979 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 166.59047109599999, 2023-10-01T10:54:11.979 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.207107032, 2023-10-01T10:54:11.979 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:54:11.979 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:54:11.979 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:54:11.979 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46037", 2023-10-01T10:54:11.979 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:54:11.979 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 10172 2023-10-01T10:54:11.980 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.980 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:54:11.980 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.980 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:54:11.980 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:25.280313+0000", 2023-10-01T10:54:11.980 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:54:11.980 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.980 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.980 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:54:11.980 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:25.280313+0000", 2023-10-01T10:54:11.980 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:54:11.980 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.981 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.981 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:54:11.981 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:25.280312+0000", 2023-10-01T10:54:11.981 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999986 2023-10-01T10:54:11.981 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.981 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.981 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:54:11.981 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:25.280366+0000", 2023-10-01T10:54:11.981 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.4373999999999998e-05 2023-10-01T10:54:11.981 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.981 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.981 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:54:11.981 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:25.280390+0000", 2023-10-01T10:54:11.982 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.3584999999999999e-05 2023-10-01T10:54:11.982 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.982 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.982 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:54:11.982 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:25.280398+0000", 2023-10-01T10:54:11.982 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.8299999999999996e-06 2023-10-01T10:54:11.982 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.982 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.982 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:54:11.982 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:25.280443+0000", 2023-10-01T10:54:11.982 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.5461999999999999e-05 2023-10-01T10:54:11.983 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.983 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.983 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:54:11.983 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:25.280567+0000", 2023-10-01T10:54:11.983 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00012329100000000001 2023-10-01T10:54:11.983 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.983 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.983 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 1,2", 2023-10-01T10:54:11.983 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:25.280693+0000", 2023-10-01T10:54:11.983 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00012629300000000001 2023-10-01T10:54:11.983 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.983 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.983 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:54:11.984 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:25.281558+0000", 2023-10-01T10:54:11.984 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00086481999999999998 2023-10-01T10:54:11.984 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.984 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.984 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:54:11.984 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:25.285220+0000", 2023-10-01T10:54:11.984 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0036626749999999998 2023-10-01T10:54:11.984 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.984 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.984 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:54:11.984 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:25.487074+0000", 2023-10-01T10:54:11.984 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.20185375799999999 2023-10-01T10:54:11.985 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.985 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.985 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:54:11.985 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:25.487128+0000", 2023-10-01T10:54:11.985 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.3588999999999998e-05 2023-10-01T10:54:11.985 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:11.985 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:11.985 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:54:11.985 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:51:25.487420+0000", 2023-10-01T10:54:11.985 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00029268300000000002 2023-10-01T10:54:11.986 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:54:11.986 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:54:11.986 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:54:11.986 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:54:11.986 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:54:11.986 INFO:teuthology.orchestra.run.smithi078.stdout:} 2023-10-01T10:54:11.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:11.987 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T10:54:11.987 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T10:54:11.988 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T10:54:11.988 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:11.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:54:12.024 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T10:54:12.074 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T10:54:12.089 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:12.132 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T10:54:12.136 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:12.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:12.139 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:12.177 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T10:54:12.223 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T10:54:12.230 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T10:54:12.230 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T10:54:12.230 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T10:54:12.230 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T10:54:12.231 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T10:54:12.231 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T10:54:12.231 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T10:54:12.231 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T10:54:12.241 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:12.266 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:12.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:12.271 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:54:12.371 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:12.404 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:12.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:12.408 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:54:12.509 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:12.542 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:12.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:12.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:54:12.556 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:12 smithi173 bash[58816]: audit 2023-10-01T10:54:11.605611+0000 mon.a (mon.0) 1333 : audit 1 from='client.? 172.21.15.78:0/2617045253' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:54:12.556 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:12 smithi173 bash[58816]: audit 2023-10-01T10:54:11.812012+0000 mon.b (mon.1) 47 : audit 0 from='client.? 172.21.15.78:0/3688153614' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:54:12.556 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:12 smithi173 bash[58816]: audit 2023-10-01T10:54:12.215550+0000 mon.c (mon.2) 180 : audit 1 from='client.? 172.21.15.78:0/502092324' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pg_num", "val": "26"}]: dispatch 2023-10-01T10:54:12.556 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:12 smithi173 bash[58816]: audit 2023-10-01T10:54:12.216682+0000 mon.a (mon.0) 1334 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pg_num", "val": "26"}]: dispatch 2023-10-01T10:54:12.569 INFO:teuthology.orchestra.run.smithi078.stderr:nodeep-scrub is unset 2023-10-01T10:54:12.574 INFO:teuthology.orchestra.run.smithi078.stderr:set pool 11 pg_num to 26 2023-10-01T10:54:12.647 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:12.675 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:12.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:12.679 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:12.780 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:12.806 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:12.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:12.812 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:54:12.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:12 smithi078 bash[78274]: audit 2023-10-01T10:54:11.605611+0000 mon.a (mon.0) 1333 : audit 1 from='client.? 172.21.15.78:0/2617045253' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:54:12.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:12 smithi078 bash[78274]: audit 2023-10-01T10:54:11.812012+0000 mon.b (mon.1) 47 : audit 0 from='client.? 172.21.15.78:0/3688153614' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:54:12.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:12 smithi078 bash[78274]: audit 2023-10-01T10:54:12.215550+0000 mon.c (mon.2) 180 : audit 1 from='client.? 172.21.15.78:0/502092324' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pg_num", "val": "26"}]: dispatch 2023-10-01T10:54:12.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:12 smithi078 bash[78274]: audit 2023-10-01T10:54:12.216682+0000 mon.a (mon.0) 1334 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pg_num", "val": "26"}]: dispatch 2023-10-01T10:54:12.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:12 smithi078 bash[75299]: audit 2023-10-01T10:54:11.605611+0000 mon.a (mon.0) 1333 : audit 1 from='client.? 172.21.15.78:0/2617045253' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:54:12.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:12 smithi078 bash[75299]: audit 2023-10-01T10:54:11.812012+0000 mon.b (mon.1) 47 : audit 0 from='client.? 172.21.15.78:0/3688153614' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:54:12.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:12 smithi078 bash[75299]: audit 2023-10-01T10:54:12.215550+0000 mon.c (mon.2) 180 : audit 1 from='client.? 172.21.15.78:0/502092324' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pg_num", "val": "26"}]: dispatch 2023-10-01T10:54:12.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:12 smithi078 bash[75299]: audit 2023-10-01T10:54:12.216682+0000 mon.a (mon.0) 1334 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pg_num", "val": "26"}]: dispatch 2023-10-01T10:54:12.913 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:12.955 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:12.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:12.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:13.063 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:13.098 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:13.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:13.100 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:54:13.201 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:13.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:13.251 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:13.251 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:13.352 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:13.375 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:13.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:13.377 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:54:13.478 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:13.498 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:13.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:13.500 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:13.601 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:13.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:13 smithi173 bash[58816]: cluster 2023-10-01T10:54:12.017738+0000 mgr.y (mgr.25072) 1100 : cluster 0 pgmap v1056: 89 pgs: 89 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 537 op/s 2023-10-01T10:54:13.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:13 smithi173 bash[58816]: cluster 2023-10-01T10:54:12.540802+0000 mon.a (mon.0) 1335 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T10:54:13.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:13 smithi173 bash[58816]: cluster 2023-10-01T10:54:12.541002+0000 mon.a (mon.0) 1336 : cluster 1 Cluster is now healthy 2023-10-01T10:54:13.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:13 smithi173 bash[58816]: cluster 2023-10-01T10:54:12.553899+0000 osd.6 (osd.6) 110 : cluster 3 failed to encode map e188 with expected crc 2023-10-01T10:54:13.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:13 smithi173 bash[58816]: cluster 2023-10-01T10:54:12.554347+0000 osd.7 (osd.7) 147 : cluster 3 failed to encode map e188 with expected crc 2023-10-01T10:54:13.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:13 smithi173 bash[58816]: audit 2023-10-01T10:54:12.559136+0000 mon.a (mon.0) 1337 : audit 1 from='client.? 172.21.15.78:0/2617045253' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T10:54:13.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:13 smithi173 bash[58816]: audit 2023-10-01T10:54:12.559207+0000 mon.a (mon.0) 1338 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pg_num", "val": "26"}]': finished 2023-10-01T10:54:13.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:13 smithi173 bash[58816]: cluster 2023-10-01T10:54:12.559270+0000 mon.a (mon.0) 1339 : cluster 0 osdmap e188: 8 total, 8 up, 8 in 2023-10-01T10:54:13.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:13 smithi173 bash[58816]: cluster 2023-10-01T10:54:12.565965+0000 osd.5 (osd.5) 129 : cluster 3 failed to encode map e188 with expected crc 2023-10-01T10:54:13.633 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:13.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:13.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:54:13.736 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:13.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:13 smithi078 bash[78274]: cluster 2023-10-01T10:54:12.017738+0000 mgr.y (mgr.25072) 1100 : cluster 0 pgmap v1056: 89 pgs: 89 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 537 op/s 2023-10-01T10:54:13.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:13 smithi078 bash[78274]: cluster 2023-10-01T10:54:12.540802+0000 mon.a (mon.0) 1335 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T10:54:13.763 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:13 smithi078 bash[78274]: cluster 2023-10-01T10:54:12.541002+0000 mon.a (mon.0) 1336 : cluster 1 Cluster is now healthy 2023-10-01T10:54:13.764 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:13 smithi078 bash[78274]: cluster 2023-10-01T10:54:12.553899+0000 osd.6 (osd.6) 110 : cluster 3 failed to encode map e188 with expected crc 2023-10-01T10:54:13.764 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:13 smithi078 bash[78274]: cluster 2023-10-01T10:54:12.554347+0000 osd.7 (osd.7) 147 : cluster 3 failed to encode map e188 with expected crc 2023-10-01T10:54:13.764 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:13 smithi078 bash[78274]: audit 2023-10-01T10:54:12.559136+0000 mon.a (mon.0) 1337 : audit 1 from='client.? 172.21.15.78:0/2617045253' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T10:54:13.764 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:13 smithi078 bash[78274]: audit 2023-10-01T10:54:12.559207+0000 mon.a (mon.0) 1338 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pg_num", "val": "26"}]': finished 2023-10-01T10:54:13.764 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:13 smithi078 bash[78274]: cluster 2023-10-01T10:54:12.559270+0000 mon.a (mon.0) 1339 : cluster 0 osdmap e188: 8 total, 8 up, 8 in 2023-10-01T10:54:13.764 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:13 smithi078 bash[78274]: cluster 2023-10-01T10:54:12.565965+0000 osd.5 (osd.5) 129 : cluster 3 failed to encode map e188 with expected crc 2023-10-01T10:54:13.764 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:13 smithi078 bash[75299]: cluster 2023-10-01T10:54:12.017738+0000 mgr.y (mgr.25072) 1100 : cluster 0 pgmap v1056: 89 pgs: 89 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 537 op/s 2023-10-01T10:54:13.764 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:13 smithi078 bash[75299]: cluster 2023-10-01T10:54:12.540802+0000 mon.a (mon.0) 1335 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T10:54:13.764 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:13 smithi078 bash[75299]: cluster 2023-10-01T10:54:12.541002+0000 mon.a (mon.0) 1336 : cluster 1 Cluster is now healthy 2023-10-01T10:54:13.765 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:13 smithi078 bash[75299]: cluster 2023-10-01T10:54:12.553899+0000 osd.6 (osd.6) 110 : cluster 3 failed to encode map e188 with expected crc 2023-10-01T10:54:13.765 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:13 smithi078 bash[75299]: cluster 2023-10-01T10:54:12.554347+0000 osd.7 (osd.7) 147 : cluster 3 failed to encode map e188 with expected crc 2023-10-01T10:54:13.765 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:13 smithi078 bash[75299]: audit 2023-10-01T10:54:12.559136+0000 mon.a (mon.0) 1337 : audit 1 from='client.? 172.21.15.78:0/2617045253' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T10:54:13.765 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:13 smithi078 bash[75299]: audit 2023-10-01T10:54:12.559207+0000 mon.a (mon.0) 1338 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pg_num", "val": "26"}]': finished 2023-10-01T10:54:13.765 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:13 smithi078 bash[75299]: cluster 2023-10-01T10:54:12.559270+0000 mon.a (mon.0) 1339 : cluster 0 osdmap e188: 8 total, 8 up, 8 in 2023-10-01T10:54:13.765 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:13 smithi078 bash[75299]: cluster 2023-10-01T10:54:12.565965+0000 osd.5 (osd.5) 129 : cluster 3 failed to encode map e188 with expected crc 2023-10-01T10:54:13.767 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:13.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:13.769 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:13.870 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:13.895 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:13.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:13.901 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:54:14.001 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:14.045 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:14.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:14.046 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:54:14.147 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:14.172 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:14.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:14.175 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:54:14.276 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:14.306 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:14.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:14.309 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:54:14.409 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:14.441 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:14.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:14.443 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:14.523 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-01T10:54:14.543 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:14.570 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:14.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:14.574 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:54:14.675 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:14.744 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:14.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:14.747 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:14.848 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:14.860 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:14 smithi173 bash[58816]: audit 2023-10-01T10:54:14.023405+0000 mon.a (mon.0) 1340 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-01T10:54:14.888 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:14.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:14.890 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:54:14.991 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:15.028 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:15.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:15.030 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:54:15.034 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:14 smithi078 bash[75299]: audit 2023-10-01T10:54:14.023405+0000 mon.a (mon.0) 1340 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-01T10:54:15.034 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:14 smithi078 bash[78274]: audit 2023-10-01T10:54:14.023405+0000 mon.a (mon.0) 1340 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-01T10:54:15.132 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:15.150 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:15.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:15.152 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:54:15.253 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:15.284 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:15.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:15.287 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:54:15.388 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:15.413 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:15.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:15.416 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:54:15.517 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:15.548 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:15.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:15.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:54:15.651 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:15.659 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:15 smithi078 bash[78274]: cluster 2023-10-01T10:54:14.021710+0000 mgr.y (mgr.25072) 1101 : cluster 0 pgmap v1058: 89 pgs: 89 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 667 op/s 2023-10-01T10:54:15.659 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:15 smithi078 bash[78274]: audit 2023-10-01T10:54:14.570921+0000 mon.a (mon.0) 1341 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-01T10:54:15.659 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:15 smithi078 bash[78274]: cluster 2023-10-01T10:54:14.572805+0000 osd.5 (osd.5) 130 : cluster 3 failed to encode map e189 with expected crc 2023-10-01T10:54:15.659 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:15 smithi078 bash[78274]: cluster 2023-10-01T10:54:14.572940+0000 osd.6 (osd.6) 111 : cluster 3 failed to encode map e189 with expected crc 2023-10-01T10:54:15.660 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:15 smithi078 bash[78274]: cluster 2023-10-01T10:54:14.573124+0000 osd.7 (osd.7) 148 : cluster 3 failed to encode map e189 with expected crc 2023-10-01T10:54:15.660 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:15 smithi078 bash[78274]: cluster 2023-10-01T10:54:14.575813+0000 mon.a (mon.0) 1342 : cluster 0 osdmap e189: 8 total, 8 up, 8 in 2023-10-01T10:54:15.660 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:15 smithi078 bash[75299]: cluster 2023-10-01T10:54:14.021710+0000 mgr.y (mgr.25072) 1101 : cluster 0 pgmap v1058: 89 pgs: 89 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 667 op/s 2023-10-01T10:54:15.660 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:15 smithi078 bash[75299]: audit 2023-10-01T10:54:14.570921+0000 mon.a (mon.0) 1341 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-01T10:54:15.660 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:15 smithi078 bash[75299]: cluster 2023-10-01T10:54:14.572805+0000 osd.5 (osd.5) 130 : cluster 3 failed to encode map e189 with expected crc 2023-10-01T10:54:15.660 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:15 smithi078 bash[75299]: cluster 2023-10-01T10:54:14.572940+0000 osd.6 (osd.6) 111 : cluster 3 failed to encode map e189 with expected crc 2023-10-01T10:54:15.660 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:15 smithi078 bash[75299]: cluster 2023-10-01T10:54:14.573124+0000 osd.7 (osd.7) 148 : cluster 3 failed to encode map e189 with expected crc 2023-10-01T10:54:15.660 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:15 smithi078 bash[75299]: cluster 2023-10-01T10:54:14.575813+0000 mon.a (mon.0) 1342 : cluster 0 osdmap e189: 8 total, 8 up, 8 in 2023-10-01T10:54:15.676 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:15.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:15.679 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:54:15.780 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:15.810 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:15.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:15.818 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:54:15.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:15 smithi173 bash[58816]: cluster 2023-10-01T10:54:14.021710+0000 mgr.y (mgr.25072) 1101 : cluster 0 pgmap v1058: 89 pgs: 89 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 667 op/s 2023-10-01T10:54:15.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:15 smithi173 bash[58816]: audit 2023-10-01T10:54:14.570921+0000 mon.a (mon.0) 1341 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-01T10:54:15.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:15 smithi173 bash[58816]: cluster 2023-10-01T10:54:14.572805+0000 osd.5 (osd.5) 130 : cluster 3 failed to encode map e189 with expected crc 2023-10-01T10:54:15.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:15 smithi173 bash[58816]: cluster 2023-10-01T10:54:14.572940+0000 osd.6 (osd.6) 111 : cluster 3 failed to encode map e189 with expected crc 2023-10-01T10:54:15.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:15 smithi173 bash[58816]: cluster 2023-10-01T10:54:14.573124+0000 osd.7 (osd.7) 148 : cluster 3 failed to encode map e189 with expected crc 2023-10-01T10:54:15.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:15 smithi173 bash[58816]: cluster 2023-10-01T10:54:14.575813+0000 mon.a (mon.0) 1342 : cluster 0 osdmap e189: 8 total, 8 up, 8 in 2023-10-01T10:54:15.918 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:15.965 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:15.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:15.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:16.069 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:16.099 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:16.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:16.103 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:54:16.204 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:16.251 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:16.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:16.255 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:54:16.355 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:16.384 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:16.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:16.392 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:54:16.492 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:16.526 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:16.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:16.532 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:54:16.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:16 smithi173 bash[58816]: cluster 2023-10-01T10:54:15.574807+0000 osd.5 (osd.5) 131 : cluster 3 failed to encode map e190 with expected crc 2023-10-01T10:54:16.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:16 smithi173 bash[58816]: cluster 2023-10-01T10:54:15.575654+0000 mon.a (mon.0) 1343 : cluster 0 osdmap e190: 8 total, 8 up, 8 in 2023-10-01T10:54:16.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:16 smithi173 bash[58816]: cluster 2023-10-01T10:54:15.576265+0000 osd.7 (osd.7) 149 : cluster 3 failed to encode map e190 with expected crc 2023-10-01T10:54:16.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:16 smithi173 bash[58816]: cluster 2023-10-01T10:54:15.577921+0000 osd.6 (osd.6) 112 : cluster 3 failed to encode map e190 with expected crc 2023-10-01T10:54:16.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:16 smithi173 bash[58816]: cluster 2023-10-01T10:54:15.578437+0000 osd.5 (osd.5) 132 : cluster 3 failed to encode map e190 with expected crc 2023-10-01T10:54:16.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:16 smithi173 bash[58816]: cluster 2023-10-01T10:54:15.579913+0000 osd.7 (osd.7) 150 : cluster 3 failed to encode map e190 with expected crc 2023-10-01T10:54:16.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:16 smithi173 bash[58816]: cluster 2023-10-01T10:54:15.579961+0000 osd.5 (osd.5) 133 : cluster 3 failed to encode map e190 with expected crc 2023-10-01T10:54:16.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:16 smithi173 bash[58816]: cluster 2023-10-01T10:54:15.581744+0000 osd.5 (osd.5) 134 : cluster 3 failed to encode map e190 with expected crc 2023-10-01T10:54:16.633 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:16.660 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:16.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:16.663 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:16.763 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:16.773 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:16 smithi078 bash[75299]: cluster 2023-10-01T10:54:15.574807+0000 osd.5 (osd.5) 131 : cluster 3 failed to encode map e190 with expected crc 2023-10-01T10:54:16.773 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:16 smithi078 bash[75299]: cluster 2023-10-01T10:54:15.575654+0000 mon.a (mon.0) 1343 : cluster 0 osdmap e190: 8 total, 8 up, 8 in 2023-10-01T10:54:16.773 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:16 smithi078 bash[75299]: cluster 2023-10-01T10:54:15.576265+0000 osd.7 (osd.7) 149 : cluster 3 failed to encode map e190 with expected crc 2023-10-01T10:54:16.773 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:16 smithi078 bash[75299]: cluster 2023-10-01T10:54:15.577921+0000 osd.6 (osd.6) 112 : cluster 3 failed to encode map e190 with expected crc 2023-10-01T10:54:16.773 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:16 smithi078 bash[75299]: cluster 2023-10-01T10:54:15.578437+0000 osd.5 (osd.5) 132 : cluster 3 failed to encode map e190 with expected crc 2023-10-01T10:54:16.773 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:16 smithi078 bash[75299]: cluster 2023-10-01T10:54:15.579913+0000 osd.7 (osd.7) 150 : cluster 3 failed to encode map e190 with expected crc 2023-10-01T10:54:16.773 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:16 smithi078 bash[75299]: cluster 2023-10-01T10:54:15.579961+0000 osd.5 (osd.5) 133 : cluster 3 failed to encode map e190 with expected crc 2023-10-01T10:54:16.773 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:16 smithi078 bash[75299]: cluster 2023-10-01T10:54:15.581744+0000 osd.5 (osd.5) 134 : cluster 3 failed to encode map e190 with expected crc 2023-10-01T10:54:16.774 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:16 smithi078 bash[78274]: cluster 2023-10-01T10:54:15.574807+0000 osd.5 (osd.5) 131 : cluster 3 failed to encode map e190 with expected crc 2023-10-01T10:54:16.774 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:16 smithi078 bash[78274]: cluster 2023-10-01T10:54:15.575654+0000 mon.a (mon.0) 1343 : cluster 0 osdmap e190: 8 total, 8 up, 8 in 2023-10-01T10:54:16.774 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:16 smithi078 bash[78274]: cluster 2023-10-01T10:54:15.576265+0000 osd.7 (osd.7) 149 : cluster 3 failed to encode map e190 with expected crc 2023-10-01T10:54:16.774 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:16 smithi078 bash[78274]: cluster 2023-10-01T10:54:15.577921+0000 osd.6 (osd.6) 112 : cluster 3 failed to encode map e190 with expected crc 2023-10-01T10:54:16.774 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:16 smithi078 bash[78274]: cluster 2023-10-01T10:54:15.578437+0000 osd.5 (osd.5) 132 : cluster 3 failed to encode map e190 with expected crc 2023-10-01T10:54:16.774 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:16 smithi078 bash[78274]: cluster 2023-10-01T10:54:15.579913+0000 osd.7 (osd.7) 150 : cluster 3 failed to encode map e190 with expected crc 2023-10-01T10:54:16.774 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:16 smithi078 bash[78274]: cluster 2023-10-01T10:54:15.579961+0000 osd.5 (osd.5) 133 : cluster 3 failed to encode map e190 with expected crc 2023-10-01T10:54:16.774 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:16 smithi078 bash[78274]: cluster 2023-10-01T10:54:15.581744+0000 osd.5 (osd.5) 134 : cluster 3 failed to encode map e190 with expected crc 2023-10-01T10:54:16.803 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:16.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:16.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:54:16.909 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:16.961 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:16.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:16.966 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:54:17.067 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:17.092 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:17.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:17.098 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:54:17.199 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:17.238 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:17.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:17.242 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:17.343 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:17.369 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:17.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:17.372 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:54:17.473 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:17.502 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:17.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:17.505 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:54:17.606 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:17.637 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:17.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:17.642 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:54:17.743 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:17.772 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:17.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:17.775 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:54:17.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:17 smithi078 bash[78274]: cluster 2023-10-01T10:54:16.022383+0000 mgr.y (mgr.25072) 1102 : cluster 0 pgmap v1061: 99 pgs: 10 unknown, 89 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 563 op/s 2023-10-01T10:54:17.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:17 smithi078 bash[78274]: cluster 2023-10-01T10:54:16.730029+0000 osd.0 (osd.0) 1 : cluster 0 11.6 scrub starts 2023-10-01T10:54:17.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:17 smithi078 bash[78274]: cluster 2023-10-01T10:54:16.852838+0000 osd.3 (osd.3) 1 : cluster 0 11.8 scrub starts 2023-10-01T10:54:17.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:17 smithi078 bash[78274]: cluster 2023-10-01T10:54:17.257222+0000 osd.0 (osd.0) 2 : cluster 0 11.6 scrub ok 2023-10-01T10:54:17.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:17 smithi078 bash[78274]: audit 2023-10-01T10:54:17.444099+0000 mon.a (mon.0) 1344 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:54:17.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:17 smithi078 bash[78274]: audit 2023-10-01T10:54:17.445646+0000 mon.a (mon.0) 1345 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:54:17.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:17 smithi078 bash[75299]: cluster 2023-10-01T10:54:16.022383+0000 mgr.y (mgr.25072) 1102 : cluster 0 pgmap v1061: 99 pgs: 10 unknown, 89 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 563 op/s 2023-10-01T10:54:17.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:17 smithi078 bash[75299]: cluster 2023-10-01T10:54:16.730029+0000 osd.0 (osd.0) 1 : cluster 0 11.6 scrub starts 2023-10-01T10:54:17.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:17 smithi078 bash[75299]: cluster 2023-10-01T10:54:16.852838+0000 osd.3 (osd.3) 1 : cluster 0 11.8 scrub starts 2023-10-01T10:54:17.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:17 smithi078 bash[75299]: cluster 2023-10-01T10:54:17.257222+0000 osd.0 (osd.0) 2 : cluster 0 11.6 scrub ok 2023-10-01T10:54:17.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:17 smithi078 bash[75299]: audit 2023-10-01T10:54:17.444099+0000 mon.a (mon.0) 1344 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:54:17.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:17 smithi078 bash[75299]: audit 2023-10-01T10:54:17.445646+0000 mon.a (mon.0) 1345 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:54:17.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:17 smithi173 bash[58816]: cluster 2023-10-01T10:54:16.022383+0000 mgr.y (mgr.25072) 1102 : cluster 0 pgmap v1061: 99 pgs: 10 unknown, 89 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 563 op/s 2023-10-01T10:54:17.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:17 smithi173 bash[58816]: cluster 2023-10-01T10:54:16.730029+0000 osd.0 (osd.0) 1 : cluster 0 11.6 scrub starts 2023-10-01T10:54:17.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:17 smithi173 bash[58816]: cluster 2023-10-01T10:54:16.852838+0000 osd.3 (osd.3) 1 : cluster 0 11.8 scrub starts 2023-10-01T10:54:17.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:17 smithi173 bash[58816]: cluster 2023-10-01T10:54:17.257222+0000 osd.0 (osd.0) 2 : cluster 0 11.6 scrub ok 2023-10-01T10:54:17.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:17 smithi173 bash[58816]: audit 2023-10-01T10:54:17.444099+0000 mon.a (mon.0) 1344 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:54:17.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:17 smithi173 bash[58816]: audit 2023-10-01T10:54:17.445646+0000 mon.a (mon.0) 1345 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:54:17.877 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:17.906 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:17.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:17.913 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:54:18.013 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:18.039 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:18.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:18.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:18.144 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:18.173 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:18.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:18.177 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:54:18.207 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T10:54:18.278 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:18.304 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:18.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:18.309 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:54:18.410 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:18.439 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:18.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:18.443 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:54:18.544 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:18.572 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:18.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:18.575 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:54:18.676 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:18.685 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:18 smithi173 bash[58816]: cluster 2023-10-01T10:54:17.445215+0000 osd.3 (osd.3) 2 : cluster 0 11.8 scrub ok 2023-10-01T10:54:18.685 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:18 smithi173 bash[58816]: cluster 2023-10-01T10:54:17.847374+0000 osd.6 (osd.6) 113 : cluster 0 11.4 scrub starts 2023-10-01T10:54:18.685 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:18 smithi173 bash[58816]: cluster 2023-10-01T10:54:17.874230+0000 osd.3 (osd.3) 3 : cluster 0 11.0 deep-scrub starts 2023-10-01T10:54:18.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:18.719 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:18.719 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:54:18.820 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:18.851 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:18.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:18.855 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:54:18.897 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T10:54:18.911 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:18 smithi078 bash[75299]: cluster 2023-10-01T10:54:17.445215+0000 osd.3 (osd.3) 2 : cluster 0 11.8 scrub ok 2023-10-01T10:54:18.911 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:18 smithi078 bash[75299]: cluster 2023-10-01T10:54:17.847374+0000 osd.6 (osd.6) 113 : cluster 0 11.4 scrub starts 2023-10-01T10:54:18.911 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:18 smithi078 bash[75299]: cluster 2023-10-01T10:54:17.874230+0000 osd.3 (osd.3) 3 : cluster 0 11.0 deep-scrub starts 2023-10-01T10:54:18.911 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:18 smithi078 bash[78274]: cluster 2023-10-01T10:54:17.445215+0000 osd.3 (osd.3) 2 : cluster 0 11.8 scrub ok 2023-10-01T10:54:18.911 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:18 smithi078 bash[78274]: cluster 2023-10-01T10:54:17.847374+0000 osd.6 (osd.6) 113 : cluster 0 11.4 scrub starts 2023-10-01T10:54:18.911 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:18 smithi078 bash[78274]: cluster 2023-10-01T10:54:17.874230+0000 osd.3 (osd.3) 3 : cluster 0 11.0 deep-scrub starts 2023-10-01T10:54:18.956 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:18.997 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:19.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:19.002 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:19.103 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:19.126 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:19.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:19.128 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:54:19.229 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:19.252 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:19.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:19.255 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:54:19.356 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:19.386 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:19.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:19.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:19.409 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:54:19.493 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:19.520 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:19.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:19.523 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:54:19.624 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:19.658 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:19.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:19.662 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:54:19.763 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:19.771 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:19 smithi173 bash[58816]: cluster 2023-10-01T10:54:18.022871+0000 mgr.y (mgr.25072) 1103 : cluster 0 pgmap v1062: 99 pgs: 1 active+clean+scrubbing, 6 unknown, 92 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 754 op/s 2023-10-01T10:54:19.771 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:19 smithi173 bash[58816]: cluster 2023-10-01T10:54:18.685903+0000 osd.0 (osd.0) 3 : cluster 0 11.16 scrub starts 2023-10-01T10:54:19.771 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:19 smithi173 bash[58816]: cluster 2023-10-01T10:54:18.773057+0000 osd.6 (osd.6) 114 : cluster 0 11.4 scrub ok 2023-10-01T10:54:19.772 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:19 smithi173 bash[58816]: cluster 2023-10-01T10:54:18.869111+0000 osd.3 (osd.3) 4 : cluster 0 11.1 scrub starts 2023-10-01T10:54:19.789 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:19.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:19.793 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:54:19.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:19 smithi078 bash[78274]: cluster 2023-10-01T10:54:18.022871+0000 mgr.y (mgr.25072) 1103 : cluster 0 pgmap v1062: 99 pgs: 1 active+clean+scrubbing, 6 unknown, 92 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 754 op/s 2023-10-01T10:54:19.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:19 smithi078 bash[78274]: cluster 2023-10-01T10:54:18.685903+0000 osd.0 (osd.0) 3 : cluster 0 11.16 scrub starts 2023-10-01T10:54:19.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:19 smithi078 bash[78274]: cluster 2023-10-01T10:54:18.773057+0000 osd.6 (osd.6) 114 : cluster 0 11.4 scrub ok 2023-10-01T10:54:19.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:19 smithi078 bash[78274]: cluster 2023-10-01T10:54:18.869111+0000 osd.3 (osd.3) 4 : cluster 0 11.1 scrub starts 2023-10-01T10:54:19.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:19 smithi078 bash[75299]: cluster 2023-10-01T10:54:18.022871+0000 mgr.y (mgr.25072) 1103 : cluster 0 pgmap v1062: 99 pgs: 1 active+clean+scrubbing, 6 unknown, 92 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 754 op/s 2023-10-01T10:54:19.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:19 smithi078 bash[75299]: cluster 2023-10-01T10:54:18.685903+0000 osd.0 (osd.0) 3 : cluster 0 11.16 scrub starts 2023-10-01T10:54:19.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:19 smithi078 bash[75299]: cluster 2023-10-01T10:54:18.773057+0000 osd.6 (osd.6) 114 : cluster 0 11.4 scrub ok 2023-10-01T10:54:19.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:19 smithi078 bash[75299]: cluster 2023-10-01T10:54:18.869111+0000 osd.3 (osd.3) 4 : cluster 0 11.1 scrub starts 2023-10-01T10:54:19.893 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:19.922 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:19.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:19.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:54:20.024 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:20.053 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:20.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:20.057 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:54:20.158 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:20.185 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:20.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:20.190 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:54:20.291 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:20.320 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:20.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:20.325 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:54:20.426 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:20.435 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:54:20 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:54:20] "GET /metrics HTTP/1.1" 200 36314 "" "Prometheus/2.33.4" 2023-10-01T10:54:20.462 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:20.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:20.471 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:54:20.572 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:20.606 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:20.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:20.611 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:54:20.712 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:20.745 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:20.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:20.750 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:54:20.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:20 smithi078 bash[75299]: cluster 2023-10-01T10:54:19.406031+0000 osd.3 (osd.3) 5 : cluster 0 11.1 scrub ok 2023-10-01T10:54:20.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:20 smithi078 bash[75299]: audit 2023-10-01T10:54:20.024829+0000 mon.a (mon.0) 1346 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-10-01T10:54:20.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:20 smithi078 bash[78274]: cluster 2023-10-01T10:54:19.406031+0000 osd.3 (osd.3) 5 : cluster 0 11.1 scrub ok 2023-10-01T10:54:20.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:20 smithi078 bash[78274]: audit 2023-10-01T10:54:20.024829+0000 mon.a (mon.0) 1346 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-10-01T10:54:20.851 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:20.860 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:20 smithi173 bash[58816]: cluster 2023-10-01T10:54:19.406031+0000 osd.3 (osd.3) 5 : cluster 0 11.1 scrub ok 2023-10-01T10:54:20.860 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:20 smithi173 bash[58816]: audit 2023-10-01T10:54:20.024829+0000 mon.a (mon.0) 1346 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-10-01T10:54:20.883 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:20.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:20.886 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:54:20.987 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:21.012 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:21.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:21.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:54:21.117 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:21.149 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:21.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:21.158 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:54:21.259 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:21.286 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:21.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:21.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:54:21.389 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:21.414 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:21.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:21.417 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:54:21.517 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:21.553 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:21.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:21.556 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:54:21.657 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:21.681 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:21.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:21.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:54:21.785 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:21.813 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:21.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:21.816 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:54:21.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:21 smithi173 bash[58816]: cluster 2023-10-01T10:54:20.023625+0000 mgr.y (mgr.25072) 1104 : cluster 0 pgmap v1063: 99 pgs: 2 active+clean+scrubbing, 97 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail 2023-10-01T10:54:21.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:21 smithi173 bash[58816]: audit 2023-10-01T10:54:20.615388+0000 mon.a (mon.0) 1347 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "17"}]': finished 2023-10-01T10:54:21.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:21 smithi173 bash[58816]: cluster 2023-10-01T10:54:20.615561+0000 mon.a (mon.0) 1348 : cluster 0 osdmap e191: 8 total, 8 up, 8 in 2023-10-01T10:54:21.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:21 smithi173 bash[58816]: cluster 2023-10-01T10:54:20.621361+0000 osd.6 (osd.6) 115 : cluster 3 failed to encode map e191 with expected crc 2023-10-01T10:54:21.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:21 smithi173 bash[58816]: cluster 2023-10-01T10:54:20.622257+0000 osd.5 (osd.5) 135 : cluster 3 failed to encode map e191 with expected crc 2023-10-01T10:54:21.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:21 smithi173 bash[58816]: cluster 2023-10-01T10:54:20.623345+0000 osd.7 (osd.7) 151 : cluster 3 failed to encode map e191 with expected crc 2023-10-01T10:54:21.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:21 smithi173 bash[58816]: cluster 2023-10-01T10:54:20.701357+0000 osd.0 (osd.0) 4 : cluster 0 11.16 scrub starts 2023-10-01T10:54:21.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:21 smithi173 bash[58816]: cluster 2023-10-01T10:54:20.858745+0000 osd.6 (osd.6) 116 : cluster 0 11.3 scrub starts 2023-10-01T10:54:21.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:21 smithi173 bash[58816]: cluster 2023-10-01T10:54:21.123422+0000 osd.0 (osd.0) 5 : cluster 0 11.16 scrub ok 2023-10-01T10:54:21.877 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T10:54:21.877 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T10:54:21.878 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_6 2023-10-01T10:54:21.878 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T10:54:21.886 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:21 smithi078 bash[75299]: cluster 2023-10-01T10:54:20.023625+0000 mgr.y (mgr.25072) 1104 : cluster 0 pgmap v1063: 99 pgs: 2 active+clean+scrubbing, 97 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail 2023-10-01T10:54:21.886 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:21 smithi078 bash[75299]: audit 2023-10-01T10:54:20.615388+0000 mon.a (mon.0) 1347 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "17"}]': finished 2023-10-01T10:54:21.886 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:21 smithi078 bash[75299]: cluster 2023-10-01T10:54:20.615561+0000 mon.a (mon.0) 1348 : cluster 0 osdmap e191: 8 total, 8 up, 8 in 2023-10-01T10:54:21.887 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:21 smithi078 bash[75299]: cluster 2023-10-01T10:54:20.621361+0000 osd.6 (osd.6) 115 : cluster 3 failed to encode map e191 with expected crc 2023-10-01T10:54:21.887 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:21 smithi078 bash[75299]: cluster 2023-10-01T10:54:20.622257+0000 osd.5 (osd.5) 135 : cluster 3 failed to encode map e191 with expected crc 2023-10-01T10:54:21.887 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:21 smithi078 bash[75299]: cluster 2023-10-01T10:54:20.623345+0000 osd.7 (osd.7) 151 : cluster 3 failed to encode map e191 with expected crc 2023-10-01T10:54:21.887 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:21 smithi078 bash[75299]: cluster 2023-10-01T10:54:20.701357+0000 osd.0 (osd.0) 4 : cluster 0 11.16 scrub starts 2023-10-01T10:54:21.887 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:21 smithi078 bash[75299]: cluster 2023-10-01T10:54:20.858745+0000 osd.6 (osd.6) 116 : cluster 0 11.3 scrub starts 2023-10-01T10:54:21.887 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:21 smithi078 bash[75299]: cluster 2023-10-01T10:54:21.123422+0000 osd.0 (osd.0) 5 : cluster 0 11.16 scrub ok 2023-10-01T10:54:21.887 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:21 smithi078 bash[78274]: cluster 2023-10-01T10:54:20.023625+0000 mgr.y (mgr.25072) 1104 : cluster 0 pgmap v1063: 99 pgs: 2 active+clean+scrubbing, 97 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail 2023-10-01T10:54:21.887 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:21 smithi078 bash[78274]: audit 2023-10-01T10:54:20.615388+0000 mon.a (mon.0) 1347 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "17"}]': finished 2023-10-01T10:54:21.887 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:21 smithi078 bash[78274]: cluster 2023-10-01T10:54:20.615561+0000 mon.a (mon.0) 1348 : cluster 0 osdmap e191: 8 total, 8 up, 8 in 2023-10-01T10:54:21.888 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:21 smithi078 bash[78274]: cluster 2023-10-01T10:54:20.621361+0000 osd.6 (osd.6) 115 : cluster 3 failed to encode map e191 with expected crc 2023-10-01T10:54:21.888 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:21 smithi078 bash[78274]: cluster 2023-10-01T10:54:20.622257+0000 osd.5 (osd.5) 135 : cluster 3 failed to encode map e191 with expected crc 2023-10-01T10:54:21.888 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:21 smithi078 bash[78274]: cluster 2023-10-01T10:54:20.623345+0000 osd.7 (osd.7) 151 : cluster 3 failed to encode map e191 with expected crc 2023-10-01T10:54:21.888 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:21 smithi078 bash[78274]: cluster 2023-10-01T10:54:20.701357+0000 osd.0 (osd.0) 4 : cluster 0 11.16 scrub starts 2023-10-01T10:54:21.888 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:21 smithi078 bash[78274]: cluster 2023-10-01T10:54:20.858745+0000 osd.6 (osd.6) 116 : cluster 0 11.3 scrub starts 2023-10-01T10:54:21.888 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:21 smithi078 bash[78274]: cluster 2023-10-01T10:54:21.123422+0000 osd.0 (osd.0) 5 : cluster 0 11.16 scrub ok 2023-10-01T10:54:21.917 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:21.948 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:21.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:21.951 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:54:22.052 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:22.081 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:22.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:22.085 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:54:22.185 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:22.217 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:22.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:22.220 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:54:22.321 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:22.352 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:22.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:22.355 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:54:22.455 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:22.466 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:54:22.487 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:22.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:22.489 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:22.590 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:22.622 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:22.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:22.625 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:54:22.726 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:22.758 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:22.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:22.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:54:22.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:22 smithi078 bash[78274]: cluster 2023-10-01T10:54:21.624901+0000 osd.6 (osd.6) 117 : cluster 3 failed to encode map e192 with expected crc 2023-10-01T10:54:22.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:22 smithi078 bash[78274]: cluster 2023-10-01T10:54:21.625113+0000 osd.7 (osd.7) 152 : cluster 3 failed to encode map e192 with expected crc 2023-10-01T10:54:22.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:22 smithi078 bash[78274]: cluster 2023-10-01T10:54:21.627142+0000 osd.6 (osd.6) 118 : cluster 3 failed to encode map e192 with expected crc 2023-10-01T10:54:22.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:22 smithi078 bash[78274]: cluster 2023-10-01T10:54:21.627393+0000 mon.a (mon.0) 1349 : cluster 0 osdmap e192: 8 total, 8 up, 8 in 2023-10-01T10:54:22.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:22 smithi078 bash[78274]: cluster 2023-10-01T10:54:21.627851+0000 osd.7 (osd.7) 153 : cluster 3 failed to encode map e192 with expected crc 2023-10-01T10:54:22.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:22 smithi078 bash[78274]: cluster 2023-10-01T10:54:21.842280+0000 osd.3 (osd.3) 6 : cluster 0 11.9 scrub starts 2023-10-01T10:54:22.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:22 smithi078 bash[78274]: audit 2023-10-01T10:54:22.025704+0000 mon.a (mon.0) 1350 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-10-01T10:54:22.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:22 smithi078 bash[75299]: cluster 2023-10-01T10:54:21.624901+0000 osd.6 (osd.6) 117 : cluster 3 failed to encode map e192 with expected crc 2023-10-01T10:54:22.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:22 smithi078 bash[75299]: cluster 2023-10-01T10:54:21.625113+0000 osd.7 (osd.7) 152 : cluster 3 failed to encode map e192 with expected crc 2023-10-01T10:54:22.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:22 smithi078 bash[75299]: cluster 2023-10-01T10:54:21.627142+0000 osd.6 (osd.6) 118 : cluster 3 failed to encode map e192 with expected crc 2023-10-01T10:54:22.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:22 smithi078 bash[75299]: cluster 2023-10-01T10:54:21.627393+0000 mon.a (mon.0) 1349 : cluster 0 osdmap e192: 8 total, 8 up, 8 in 2023-10-01T10:54:22.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:22 smithi078 bash[75299]: cluster 2023-10-01T10:54:21.627851+0000 osd.7 (osd.7) 153 : cluster 3 failed to encode map e192 with expected crc 2023-10-01T10:54:22.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:22 smithi078 bash[75299]: cluster 2023-10-01T10:54:21.842280+0000 osd.3 (osd.3) 6 : cluster 0 11.9 scrub starts 2023-10-01T10:54:22.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:22 smithi078 bash[75299]: audit 2023-10-01T10:54:22.025704+0000 mon.a (mon.0) 1350 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-10-01T10:54:22.861 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:22.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:22 smithi173 bash[58816]: cluster 2023-10-01T10:54:21.624901+0000 osd.6 (osd.6) 117 : cluster 3 failed to encode map e192 with expected crc 2023-10-01T10:54:22.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:22 smithi173 bash[58816]: cluster 2023-10-01T10:54:21.625113+0000 osd.7 (osd.7) 152 : cluster 3 failed to encode map e192 with expected crc 2023-10-01T10:54:22.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:22 smithi173 bash[58816]: cluster 2023-10-01T10:54:21.627142+0000 osd.6 (osd.6) 118 : cluster 3 failed to encode map e192 with expected crc 2023-10-01T10:54:22.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:22 smithi173 bash[58816]: cluster 2023-10-01T10:54:21.627393+0000 mon.a (mon.0) 1349 : cluster 0 osdmap e192: 8 total, 8 up, 8 in 2023-10-01T10:54:22.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:22 smithi173 bash[58816]: cluster 2023-10-01T10:54:21.627851+0000 osd.7 (osd.7) 153 : cluster 3 failed to encode map e192 with expected crc 2023-10-01T10:54:22.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:22 smithi173 bash[58816]: cluster 2023-10-01T10:54:21.842280+0000 osd.3 (osd.3) 6 : cluster 0 11.9 scrub starts 2023-10-01T10:54:22.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:22 smithi173 bash[58816]: audit 2023-10-01T10:54:22.025704+0000 mon.a (mon.0) 1350 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-10-01T10:54:22.890 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:22.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:22.893 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:22.993 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:23.020 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:23.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:23.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:54:23.124 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:23.148 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:23.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:23.152 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:23.252 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:23.276 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:23.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:23.281 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:54:23.382 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:23.405 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:23.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:23.407 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:23.508 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:23.527 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:23.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:23.529 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:54:23.629 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:23.657 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:23.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:23.660 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:54:23.760 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:23.789 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:23.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:23.791 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:54:23.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:23 smithi078 bash[78274]: cluster 2023-10-01T10:54:22.024256+0000 mgr.y (mgr.25072) 1105 : cluster 0 pgmap v1066: 99 pgs: 2 active+clean+scrubbing, 97 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail 2023-10-01T10:54:23.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:23 smithi078 bash[78274]: cluster 2023-10-01T10:54:22.063830+0000 osd.3 (osd.3) 7 : cluster 0 11.9 scrub ok 2023-10-01T10:54:23.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:23 smithi078 bash[78274]: cluster 2023-10-01T10:54:22.630775+0000 osd.6 (osd.6) 119 : cluster 3 failed to encode map e193 with expected crc 2023-10-01T10:54:23.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:23 smithi078 bash[78274]: audit 2023-10-01T10:54:22.633411+0000 mon.a (mon.0) 1351 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "18"}]': finished 2023-10-01T10:54:23.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:23 smithi078 bash[78274]: cluster 2023-10-01T10:54:22.633518+0000 mon.a (mon.0) 1352 : cluster 0 osdmap e193: 8 total, 8 up, 8 in 2023-10-01T10:54:23.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:23 smithi078 bash[78274]: cluster 2023-10-01T10:54:22.635063+0000 osd.1 (osd.1) 1 : cluster 0 11.10 starting backfill to osd.0 from (0'0,0'0] MAX to 190'10584 2023-10-01T10:54:23.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:23 smithi078 bash[78274]: cluster 2023-10-01T10:54:22.638892+0000 osd.7 (osd.7) 154 : cluster 3 failed to encode map e193 with expected crc 2023-10-01T10:54:23.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:23 smithi078 bash[78274]: cluster 2023-10-01T10:54:22.645594+0000 osd.1 (osd.1) 2 : cluster 0 11.10 starting backfill to osd.2 from (0'0,0'0] MAX to 190'10584 2023-10-01T10:54:23.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:23 smithi078 bash[78274]: cluster 2023-10-01T10:54:22.652602+0000 osd.1 (osd.1) 3 : cluster 0 11.10 starting backfill to osd.6 from (0'0,0'0] MAX to 190'10584 2023-10-01T10:54:23.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:23 smithi078 bash[78274]: cluster 2023-10-01T10:54:22.764164+0000 osd.5 (osd.5) 136 : cluster 3 failed to encode map e193 with expected crc 2023-10-01T10:54:23.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:23 smithi078 bash[78274]: cluster 2023-10-01T10:54:22.869680+0000 osd.3 (osd.3) 8 : cluster 0 11.19 scrub starts 2023-10-01T10:54:23.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:23 smithi078 bash[75299]: cluster 2023-10-01T10:54:22.024256+0000 mgr.y (mgr.25072) 1105 : cluster 0 pgmap v1066: 99 pgs: 2 active+clean+scrubbing, 97 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail 2023-10-01T10:54:23.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:23 smithi078 bash[75299]: cluster 2023-10-01T10:54:22.063830+0000 osd.3 (osd.3) 7 : cluster 0 11.9 scrub ok 2023-10-01T10:54:23.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:23 smithi078 bash[75299]: cluster 2023-10-01T10:54:22.630775+0000 osd.6 (osd.6) 119 : cluster 3 failed to encode map e193 with expected crc 2023-10-01T10:54:23.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:23 smithi078 bash[75299]: audit 2023-10-01T10:54:22.633411+0000 mon.a (mon.0) 1351 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "18"}]': finished 2023-10-01T10:54:23.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:23 smithi078 bash[75299]: cluster 2023-10-01T10:54:22.633518+0000 mon.a (mon.0) 1352 : cluster 0 osdmap e193: 8 total, 8 up, 8 in 2023-10-01T10:54:23.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:23 smithi078 bash[75299]: cluster 2023-10-01T10:54:22.635063+0000 osd.1 (osd.1) 1 : cluster 0 11.10 starting backfill to osd.0 from (0'0,0'0] MAX to 190'10584 2023-10-01T10:54:23.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:23 smithi078 bash[75299]: cluster 2023-10-01T10:54:22.638892+0000 osd.7 (osd.7) 154 : cluster 3 failed to encode map e193 with expected crc 2023-10-01T10:54:23.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:23 smithi078 bash[75299]: cluster 2023-10-01T10:54:22.645594+0000 osd.1 (osd.1) 2 : cluster 0 11.10 starting backfill to osd.2 from (0'0,0'0] MAX to 190'10584 2023-10-01T10:54:23.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:23 smithi078 bash[75299]: cluster 2023-10-01T10:54:22.652602+0000 osd.1 (osd.1) 3 : cluster 0 11.10 starting backfill to osd.6 from (0'0,0'0] MAX to 190'10584 2023-10-01T10:54:23.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:23 smithi078 bash[75299]: cluster 2023-10-01T10:54:22.764164+0000 osd.5 (osd.5) 136 : cluster 3 failed to encode map e193 with expected crc 2023-10-01T10:54:23.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:23 smithi078 bash[75299]: cluster 2023-10-01T10:54:22.869680+0000 osd.3 (osd.3) 8 : cluster 0 11.19 scrub starts 2023-10-01T10:54:23.891 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:23.919 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:23.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:23.921 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:54:24.022 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:24.035 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:23 smithi173 bash[58816]: cluster 2023-10-01T10:54:22.024256+0000 mgr.y (mgr.25072) 1105 : cluster 0 pgmap v1066: 99 pgs: 2 active+clean+scrubbing, 97 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail 2023-10-01T10:54:24.035 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:23 smithi173 bash[58816]: cluster 2023-10-01T10:54:22.063830+0000 osd.3 (osd.3) 7 : cluster 0 11.9 scrub ok 2023-10-01T10:54:24.035 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:23 smithi173 bash[58816]: cluster 2023-10-01T10:54:22.630775+0000 osd.6 (osd.6) 119 : cluster 3 failed to encode map e193 with expected crc 2023-10-01T10:54:24.036 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:23 smithi173 bash[58816]: audit 2023-10-01T10:54:22.633411+0000 mon.a (mon.0) 1351 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "18"}]': finished 2023-10-01T10:54:24.036 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:23 smithi173 bash[58816]: cluster 2023-10-01T10:54:22.633518+0000 mon.a (mon.0) 1352 : cluster 0 osdmap e193: 8 total, 8 up, 8 in 2023-10-01T10:54:24.036 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:23 smithi173 bash[58816]: cluster 2023-10-01T10:54:22.635063+0000 osd.1 (osd.1) 1 : cluster 0 11.10 starting backfill to osd.0 from (0'0,0'0] MAX to 190'10584 2023-10-01T10:54:24.036 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:23 smithi173 bash[58816]: cluster 2023-10-01T10:54:22.638892+0000 osd.7 (osd.7) 154 : cluster 3 failed to encode map e193 with expected crc 2023-10-01T10:54:24.036 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:23 smithi173 bash[58816]: cluster 2023-10-01T10:54:22.645594+0000 osd.1 (osd.1) 2 : cluster 0 11.10 starting backfill to osd.2 from (0'0,0'0] MAX to 190'10584 2023-10-01T10:54:24.036 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:23 smithi173 bash[58816]: cluster 2023-10-01T10:54:22.652602+0000 osd.1 (osd.1) 3 : cluster 0 11.10 starting backfill to osd.6 from (0'0,0'0] MAX to 190'10584 2023-10-01T10:54:24.036 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:23 smithi173 bash[58816]: cluster 2023-10-01T10:54:22.764164+0000 osd.5 (osd.5) 136 : cluster 3 failed to encode map e193 with expected crc 2023-10-01T10:54:24.036 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:23 smithi173 bash[58816]: cluster 2023-10-01T10:54:22.869680+0000 osd.3 (osd.3) 8 : cluster 0 11.19 scrub starts 2023-10-01T10:54:24.053 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:24.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:24.056 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:54:24.157 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:24.184 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:24.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:24.187 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:54:24.288 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:24.318 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:24.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:24.322 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:54:24.423 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:24.449 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:24.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:24.451 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:54:24.552 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:24.581 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:24.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:24.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:54:24.684 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:24.696 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:24 smithi078 bash[75299]: cluster 2023-10-01T10:54:23.197672+0000 osd.5 (osd.5) 137 : cluster 0 11.5 scrub starts 2023-10-01T10:54:24.696 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:24 smithi078 bash[75299]: cluster 2023-10-01T10:54:23.640203+0000 mon.a (mon.0) 1353 : cluster 0 osdmap e194: 8 total, 8 up, 8 in 2023-10-01T10:54:24.696 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:24 smithi078 bash[75299]: cluster 2023-10-01T10:54:23.645138+0000 osd.7 (osd.7) 155 : cluster 3 failed to encode map e194 with expected crc 2023-10-01T10:54:24.696 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:24 smithi078 bash[75299]: cluster 2023-10-01T10:54:23.645724+0000 osd.6 (osd.6) 120 : cluster 3 failed to encode map e194 with expected crc 2023-10-01T10:54:24.696 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:24 smithi078 bash[75299]: cluster 2023-10-01T10:54:23.647626+0000 osd.6 (osd.6) 121 : cluster 3 failed to encode map e194 with expected crc 2023-10-01T10:54:24.696 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:24 smithi078 bash[78274]: cluster 2023-10-01T10:54:23.197672+0000 osd.5 (osd.5) 137 : cluster 0 11.5 scrub starts 2023-10-01T10:54:24.697 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:24 smithi078 bash[78274]: cluster 2023-10-01T10:54:23.640203+0000 mon.a (mon.0) 1353 : cluster 0 osdmap e194: 8 total, 8 up, 8 in 2023-10-01T10:54:24.697 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:24 smithi078 bash[78274]: cluster 2023-10-01T10:54:23.645138+0000 osd.7 (osd.7) 155 : cluster 3 failed to encode map e194 with expected crc 2023-10-01T10:54:24.697 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:24 smithi078 bash[78274]: cluster 2023-10-01T10:54:23.645724+0000 osd.6 (osd.6) 120 : cluster 3 failed to encode map e194 with expected crc 2023-10-01T10:54:24.697 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:24 smithi078 bash[78274]: cluster 2023-10-01T10:54:23.647626+0000 osd.6 (osd.6) 121 : cluster 3 failed to encode map e194 with expected crc 2023-10-01T10:54:24.710 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:24.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:24.713 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:54:24.815 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:24.825 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:24 smithi173 bash[58816]: cluster 2023-10-01T10:54:23.197672+0000 osd.5 (osd.5) 137 : cluster 0 11.5 scrub starts 2023-10-01T10:54:24.826 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:24 smithi173 bash[58816]: cluster 2023-10-01T10:54:23.640203+0000 mon.a (mon.0) 1353 : cluster 0 osdmap e194: 8 total, 8 up, 8 in 2023-10-01T10:54:24.826 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:24 smithi173 bash[58816]: cluster 2023-10-01T10:54:23.645138+0000 osd.7 (osd.7) 155 : cluster 3 failed to encode map e194 with expected crc 2023-10-01T10:54:24.826 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:24 smithi173 bash[58816]: cluster 2023-10-01T10:54:23.645724+0000 osd.6 (osd.6) 120 : cluster 3 failed to encode map e194 with expected crc 2023-10-01T10:54:24.826 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:24 smithi173 bash[58816]: cluster 2023-10-01T10:54:23.647626+0000 osd.6 (osd.6) 121 : cluster 3 failed to encode map e194 with expected crc 2023-10-01T10:54:24.846 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:24.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:24.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:54:24.950 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:24.975 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:24.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:24.981 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:24.994 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:54:25.082 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:25.113 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:25.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:25.117 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:54:25.218 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:25.254 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:25.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:25.258 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:54:25.359 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:25.387 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:25.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:25.392 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:54:25.493 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:25.522 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:25.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:25.526 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:54:25.627 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:25.649 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:25.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:25.660 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:25.760 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:25.792 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:25.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:25.795 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:54:25.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:25 smithi173 bash[58816]: cluster 2023-10-01T10:54:24.024731+0000 mgr.y (mgr.25072) 1106 : cluster 0 pgmap v1069: 99 pgs: 1 unknown, 98 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail 2023-10-01T10:54:25.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:25 smithi173 bash[58816]: cluster 2023-10-01T10:54:24.148606+0000 osd.5 (osd.5) 138 : cluster 3 failed to encode map e194 with expected crc 2023-10-01T10:54:25.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:25 smithi173 bash[58816]: cluster 2023-10-01T10:54:24.646738+0000 osd.6 (osd.6) 122 : cluster 3 failed to encode map e195 with expected crc 2023-10-01T10:54:25.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:25 smithi173 bash[58816]: cluster 2023-10-01T10:54:24.651084+0000 mon.a (mon.0) 1354 : cluster 0 osdmap e195: 8 total, 8 up, 8 in 2023-10-01T10:54:25.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:25 smithi173 bash[58816]: cluster 2023-10-01T10:54:24.653906+0000 osd.0 (osd.0) 6 : cluster 0 11.11 starting backfill to osd.2 from (0'0,0'0] MAX to 191'10571 2023-10-01T10:54:25.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:25 smithi173 bash[58816]: cluster 2023-10-01T10:54:24.663059+0000 osd.0 (osd.0) 7 : cluster 0 11.11 starting backfill to osd.7 from (0'0,0'0] MAX to 191'10571 2023-10-01T10:54:25.896 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:25.918 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:25.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:25.921 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:54:26.022 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:26.051 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:26.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:26.055 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:54:26.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:25 smithi078 bash[78274]: cluster 2023-10-01T10:54:24.024731+0000 mgr.y (mgr.25072) 1106 : cluster 0 pgmap v1069: 99 pgs: 1 unknown, 98 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail 2023-10-01T10:54:26.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:25 smithi078 bash[78274]: cluster 2023-10-01T10:54:24.148606+0000 osd.5 (osd.5) 138 : cluster 3 failed to encode map e194 with expected crc 2023-10-01T10:54:26.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:25 smithi078 bash[78274]: cluster 2023-10-01T10:54:24.646738+0000 osd.6 (osd.6) 122 : cluster 3 failed to encode map e195 with expected crc 2023-10-01T10:54:26.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:25 smithi078 bash[78274]: cluster 2023-10-01T10:54:24.651084+0000 mon.a (mon.0) 1354 : cluster 0 osdmap e195: 8 total, 8 up, 8 in 2023-10-01T10:54:26.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:25 smithi078 bash[78274]: cluster 2023-10-01T10:54:24.653906+0000 osd.0 (osd.0) 6 : cluster 0 11.11 starting backfill to osd.2 from (0'0,0'0] MAX to 191'10571 2023-10-01T10:54:26.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:25 smithi078 bash[78274]: cluster 2023-10-01T10:54:24.663059+0000 osd.0 (osd.0) 7 : cluster 0 11.11 starting backfill to osd.7 from (0'0,0'0] MAX to 191'10571 2023-10-01T10:54:26.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:25 smithi078 bash[75299]: cluster 2023-10-01T10:54:24.024731+0000 mgr.y (mgr.25072) 1106 : cluster 0 pgmap v1069: 99 pgs: 1 unknown, 98 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail 2023-10-01T10:54:26.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:25 smithi078 bash[75299]: cluster 2023-10-01T10:54:24.148606+0000 osd.5 (osd.5) 138 : cluster 3 failed to encode map e194 with expected crc 2023-10-01T10:54:26.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:25 smithi078 bash[75299]: cluster 2023-10-01T10:54:24.646738+0000 osd.6 (osd.6) 122 : cluster 3 failed to encode map e195 with expected crc 2023-10-01T10:54:26.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:25 smithi078 bash[75299]: cluster 2023-10-01T10:54:24.651084+0000 mon.a (mon.0) 1354 : cluster 0 osdmap e195: 8 total, 8 up, 8 in 2023-10-01T10:54:26.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:25 smithi078 bash[75299]: cluster 2023-10-01T10:54:24.653906+0000 osd.0 (osd.0) 6 : cluster 0 11.11 starting backfill to osd.2 from (0'0,0'0] MAX to 191'10571 2023-10-01T10:54:26.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:25 smithi078 bash[75299]: cluster 2023-10-01T10:54:24.663059+0000 osd.0 (osd.0) 7 : cluster 0 11.11 starting backfill to osd.7 from (0'0,0'0] MAX to 191'10571 2023-10-01T10:54:26.156 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:26.181 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:26.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:26.184 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:54:26.284 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:26.323 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:26.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:26.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:54:26.428 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:26.447 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:26.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:26.449 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:54:26.550 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:26.567 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:26.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:26.569 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:54:26.669 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:26.707 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:26.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:26.710 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:54:26.811 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:26.838 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:26.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:26.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:54:26.943 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:26.974 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:26.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:26.978 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:54:27.079 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:27.109 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:27.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:27.112 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:27.213 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:27.240 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:27.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:27.243 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:27.343 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:27.378 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:27.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:27.383 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:54:27.484 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:27.509 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:27.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:27.518 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:54:27.583 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:54:27.620 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:27.651 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:27.654 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:27 smithi078 bash[75299]: cluster 2023-10-01T10:54:25.937120+0000 osd.3 (osd.3) 9 : cluster 0 11.18 scrub starts 2023-10-01T10:54:27.655 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:27 smithi078 bash[75299]: cluster 2023-10-01T10:54:26.025328+0000 mgr.y (mgr.25072) 1107 : cluster 0 pgmap v1071: 99 pgs: 1 remapped+peering, 98 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail 2023-10-01T10:54:27.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:27.656 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:54:27.756 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:27.781 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:27.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:27.784 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:27.885 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:27.909 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:27 smithi078 bash[78274]: cluster 2023-10-01T10:54:25.937120+0000 osd.3 (osd.3) 9 : cluster 0 11.18 scrub starts 2023-10-01T10:54:27.909 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:27 smithi078 bash[78274]: cluster 2023-10-01T10:54:26.025328+0000 mgr.y (mgr.25072) 1107 : cluster 0 pgmap v1071: 99 pgs: 1 remapped+peering, 98 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail 2023-10-01T10:54:27.909 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:27.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:27.911 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:54:28.012 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:28.021 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:27 smithi173 bash[58816]: cluster 2023-10-01T10:54:25.937120+0000 osd.3 (osd.3) 9 : cluster 0 11.18 scrub starts 2023-10-01T10:54:28.021 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:27 smithi173 bash[58816]: cluster 2023-10-01T10:54:26.025328+0000 mgr.y (mgr.25072) 1107 : cluster 0 pgmap v1071: 99 pgs: 1 remapped+peering, 98 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail 2023-10-01T10:54:28.045 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:28.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:28.053 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:54:28.154 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:28.201 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:28.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:28.207 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:54:28.307 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:28.339 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:28.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:28.342 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:54:28.443 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:28.464 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:28.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:28.467 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:54:28.568 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:28.598 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:28.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:28.603 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:54:28.704 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:28.751 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:28.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:28.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:54:28.861 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:28.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:28 smithi173 bash[58816]: cluster 2023-10-01T10:54:26.967192+0000 osd.3 (osd.3) 10 : cluster 0 11.0 scrub starts 2023-10-01T10:54:28.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:28 smithi173 bash[58816]: audit 2023-10-01T10:54:27.961083+0000 mon.a (mon.0) 1355 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:54:28.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:28 smithi173 bash[58816]: audit 2023-10-01T10:54:28.487654+0000 mon.a (mon.0) 1356 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:54:28.895 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:28.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:28.898 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:54:28.999 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:29.027 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:29.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:29.031 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:54:29.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:28 smithi078 bash[75299]: cluster 2023-10-01T10:54:26.967192+0000 osd.3 (osd.3) 10 : cluster 0 11.0 scrub starts 2023-10-01T10:54:29.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:28 smithi078 bash[75299]: audit 2023-10-01T10:54:27.961083+0000 mon.a (mon.0) 1355 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:54:29.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:28 smithi078 bash[75299]: audit 2023-10-01T10:54:28.487654+0000 mon.a (mon.0) 1356 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:54:29.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:28 smithi078 bash[78274]: cluster 2023-10-01T10:54:26.967192+0000 osd.3 (osd.3) 10 : cluster 0 11.0 scrub starts 2023-10-01T10:54:29.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:28 smithi078 bash[78274]: audit 2023-10-01T10:54:27.961083+0000 mon.a (mon.0) 1355 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:54:29.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:28 smithi078 bash[78274]: audit 2023-10-01T10:54:28.487654+0000 mon.a (mon.0) 1356 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:54:29.132 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:29.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:29.166 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:29.166 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:54:29.267 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:29.299 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:29.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:29.301 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:54:29.401 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:29.426 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:29.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:29.429 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:54:29.529 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:29.566 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:29.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:29.571 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:29.672 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:29.707 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:29.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:29.709 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:54:29.719 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:29 smithi078 bash[78274]: cluster 2023-10-01T10:54:28.025978+0000 mgr.y (mgr.25072) 1108 : cluster 0 pgmap v1072: 99 pgs: 1 active+remapped+backfilling, 1 remapped+peering, 97 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 2094/128613 objects misplaced (1.628%); 6.4 MiB/s, 102 objects/s recovering 2023-10-01T10:54:29.720 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:29 smithi078 bash[78274]: cluster 2023-10-01T10:54:28.170393+0000 osd.5 (osd.5) 139 : cluster 0 11.5 scrub starts 2023-10-01T10:54:29.720 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:29 smithi078 bash[75299]: cluster 2023-10-01T10:54:28.025978+0000 mgr.y (mgr.25072) 1108 : cluster 0 pgmap v1072: 99 pgs: 1 active+remapped+backfilling, 1 remapped+peering, 97 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 2094/128613 objects misplaced (1.628%); 6.4 MiB/s, 102 objects/s recovering 2023-10-01T10:54:29.720 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:29 smithi078 bash[75299]: cluster 2023-10-01T10:54:28.170393+0000 osd.5 (osd.5) 139 : cluster 0 11.5 scrub starts 2023-10-01T10:54:29.810 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:29.836 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:29.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:29.838 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:54:29.939 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:29.948 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:29 smithi173 bash[58816]: cluster 2023-10-01T10:54:28.025978+0000 mgr.y (mgr.25072) 1108 : cluster 0 pgmap v1072: 99 pgs: 1 active+remapped+backfilling, 1 remapped+peering, 97 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 2094/128613 objects misplaced (1.628%); 6.4 MiB/s, 102 objects/s recovering 2023-10-01T10:54:29.949 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:29 smithi173 bash[58816]: cluster 2023-10-01T10:54:28.170393+0000 osd.5 (osd.5) 139 : cluster 0 11.5 scrub starts 2023-10-01T10:54:29.964 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:29.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:29.967 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:54:30.067 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:30.103 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:30.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:30.108 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:30.209 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:30.242 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:30.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:30.246 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:30.331 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:54:30 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:54:30] "GET /metrics HTTP/1.1" 200 36314 "" "Prometheus/2.33.4" 2023-10-01T10:54:30.347 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:30.371 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:30.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:30.379 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:54:30.480 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:30.503 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:30.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:30.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:30.608 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:30.639 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:30.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:30.648 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:54:30.749 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:30.778 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:30.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:30.781 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:54:30.882 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:30.894 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:30 smithi078 bash[75299]: cluster 2023-10-01T10:54:29.126218+0000 osd.5 (osd.5) 140 : cluster 0 11.5 scrub starts 2023-10-01T10:54:30.895 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:30 smithi078 bash[75299]: audit 2023-10-01T10:54:30.027552+0000 mon.a (mon.0) 1357 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-10-01T10:54:30.895 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:30 smithi078 bash[78274]: cluster 2023-10-01T10:54:29.126218+0000 osd.5 (osd.5) 140 : cluster 0 11.5 scrub starts 2023-10-01T10:54:30.895 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:30 smithi078 bash[78274]: audit 2023-10-01T10:54:30.027552+0000 mon.a (mon.0) 1357 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-10-01T10:54:30.919 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:30.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:30.922 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:54:30.922 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:30 smithi173 bash[58816]: cluster 2023-10-01T10:54:29.126218+0000 osd.5 (osd.5) 140 : cluster 0 11.5 scrub starts 2023-10-01T10:54:30.922 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:30 smithi173 bash[58816]: audit 2023-10-01T10:54:30.027552+0000 mon.a (mon.0) 1357 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-10-01T10:54:31.023 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:31.046 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:31.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:31.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:54:31.151 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:31.179 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:31.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:31.181 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:54:31.282 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:31.312 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:31.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:31.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:54:31.415 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:31.450 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:31.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:31.454 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:54:31.555 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:31.586 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:31.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:31.588 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:54:31.689 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:31.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:31.717 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:31.717 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:54:31.818 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:31.842 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:31.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:31.844 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:31.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:31 smithi173 bash[58816]: cluster 2023-10-01T10:54:30.026744+0000 mgr.y (mgr.25072) 1109 : cluster 0 pgmap v1073: 99 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 97 active+clean; 2.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 4486/122979 objects misplaced (3.648%); 5.2 MiB/s, 83 objects/s recovering 2023-10-01T10:54:31.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:31 smithi173 bash[58816]: cluster 2023-10-01T10:54:30.669342+0000 osd.7 (osd.7) 156 : cluster 3 failed to encode map e196 with expected crc 2023-10-01T10:54:31.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:31 smithi173 bash[58816]: cluster 2023-10-01T10:54:30.674125+0000 osd.6 (osd.6) 123 : cluster 3 failed to encode map e196 with expected crc 2023-10-01T10:54:31.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:31 smithi173 bash[58816]: cluster 2023-10-01T10:54:30.678089+0000 osd.6 (osd.6) 124 : cluster 3 failed to encode map e196 with expected crc 2023-10-01T10:54:31.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:31 smithi173 bash[58816]: audit 2023-10-01T10:54:30.678471+0000 mon.a (mon.0) 1358 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "19"}]': finished 2023-10-01T10:54:31.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:31 smithi173 bash[58816]: cluster 2023-10-01T10:54:30.678838+0000 osd.5 (osd.5) 141 : cluster 3 failed to encode map e196 with expected crc 2023-10-01T10:54:31.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:31 smithi173 bash[58816]: cluster 2023-10-01T10:54:30.679169+0000 mon.a (mon.0) 1359 : cluster 0 osdmap e196: 8 total, 8 up, 8 in 2023-10-01T10:54:31.945 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:31.975 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:31.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:31.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:54:32.080 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:32.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:31 smithi078 bash[78274]: cluster 2023-10-01T10:54:30.026744+0000 mgr.y (mgr.25072) 1109 : cluster 0 pgmap v1073: 99 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 97 active+clean; 2.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 4486/122979 objects misplaced (3.648%); 5.2 MiB/s, 83 objects/s recovering 2023-10-01T10:54:32.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:31 smithi078 bash[78274]: cluster 2023-10-01T10:54:30.669342+0000 osd.7 (osd.7) 156 : cluster 3 failed to encode map e196 with expected crc 2023-10-01T10:54:32.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:31 smithi078 bash[78274]: cluster 2023-10-01T10:54:30.674125+0000 osd.6 (osd.6) 123 : cluster 3 failed to encode map e196 with expected crc 2023-10-01T10:54:32.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:31 smithi078 bash[78274]: cluster 2023-10-01T10:54:30.678089+0000 osd.6 (osd.6) 124 : cluster 3 failed to encode map e196 with expected crc 2023-10-01T10:54:32.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:31 smithi078 bash[78274]: audit 2023-10-01T10:54:30.678471+0000 mon.a (mon.0) 1358 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "19"}]': finished 2023-10-01T10:54:32.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:31 smithi078 bash[78274]: cluster 2023-10-01T10:54:30.678838+0000 osd.5 (osd.5) 141 : cluster 3 failed to encode map e196 with expected crc 2023-10-01T10:54:32.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:31 smithi078 bash[78274]: cluster 2023-10-01T10:54:30.679169+0000 mon.a (mon.0) 1359 : cluster 0 osdmap e196: 8 total, 8 up, 8 in 2023-10-01T10:54:32.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:31 smithi078 bash[75299]: cluster 2023-10-01T10:54:30.026744+0000 mgr.y (mgr.25072) 1109 : cluster 0 pgmap v1073: 99 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 97 active+clean; 2.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 4486/122979 objects misplaced (3.648%); 5.2 MiB/s, 83 objects/s recovering 2023-10-01T10:54:32.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:31 smithi078 bash[75299]: cluster 2023-10-01T10:54:30.669342+0000 osd.7 (osd.7) 156 : cluster 3 failed to encode map e196 with expected crc 2023-10-01T10:54:32.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:31 smithi078 bash[75299]: cluster 2023-10-01T10:54:30.674125+0000 osd.6 (osd.6) 123 : cluster 3 failed to encode map e196 with expected crc 2023-10-01T10:54:32.085 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:31 smithi078 bash[75299]: cluster 2023-10-01T10:54:30.678089+0000 osd.6 (osd.6) 124 : cluster 3 failed to encode map e196 with expected crc 2023-10-01T10:54:32.085 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:31 smithi078 bash[75299]: audit 2023-10-01T10:54:30.678471+0000 mon.a (mon.0) 1358 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "19"}]': finished 2023-10-01T10:54:32.085 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:31 smithi078 bash[75299]: cluster 2023-10-01T10:54:30.678838+0000 osd.5 (osd.5) 141 : cluster 3 failed to encode map e196 with expected crc 2023-10-01T10:54:32.085 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:31 smithi078 bash[75299]: cluster 2023-10-01T10:54:30.679169+0000 mon.a (mon.0) 1359 : cluster 0 osdmap e196: 8 total, 8 up, 8 in 2023-10-01T10:54:32.111 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:32.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:32.113 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:32.214 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:32.245 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:32.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:32.248 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:54:32.349 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:32.372 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:32.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:32.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:54:32.475 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:32.498 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:32.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:32.500 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:32.601 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:32.631 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:32.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:32.634 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:54:32.735 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:32.759 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:32.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:32.761 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:54:32.862 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:32.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:32 smithi173 bash[58816]: cluster 2023-10-01T10:54:31.679419+0000 osd.7 (osd.7) 157 : cluster 3 failed to encode map e197 with expected crc 2023-10-01T10:54:32.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:32 smithi173 bash[58816]: cluster 2023-10-01T10:54:31.680686+0000 mon.a (mon.0) 1360 : cluster 0 osdmap e197: 8 total, 8 up, 8 in 2023-10-01T10:54:32.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:32 smithi173 bash[58816]: cluster 2023-10-01T10:54:31.687700+0000 osd.5 (osd.5) 142 : cluster 3 failed to encode map e197 with expected crc 2023-10-01T10:54:32.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:32 smithi173 bash[58816]: cluster 2023-10-01T10:54:31.687957+0000 osd.6 (osd.6) 125 : cluster 3 failed to encode map e197 with expected crc 2023-10-01T10:54:32.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:32 smithi173 bash[58816]: cluster 2023-10-01T10:54:31.689790+0000 osd.6 (osd.6) 126 : cluster 3 failed to encode map e197 with expected crc 2023-10-01T10:54:32.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:32 smithi173 bash[58816]: audit 2023-10-01T10:54:32.027981+0000 mon.a (mon.0) 1361 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-10-01T10:54:32.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:32 smithi173 bash[58816]: audit 2023-10-01T10:54:32.440452+0000 mon.a (mon.0) 1362 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:54:32.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:32 smithi173 bash[58816]: audit 2023-10-01T10:54:32.543923+0000 mon.a (mon.0) 1363 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "20"}]': finished 2023-10-01T10:54:32.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:32 smithi173 bash[58816]: cluster 2023-10-01T10:54:32.544051+0000 mon.a (mon.0) 1364 : cluster 0 osdmap e198: 8 total, 8 up, 8 in 2023-10-01T10:54:32.893 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:32.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:32.895 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:54:32.996 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:33.025 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:32 smithi078 bash[78274]: cluster 2023-10-01T10:54:31.679419+0000 osd.7 (osd.7) 157 : cluster 3 failed to encode map e197 with expected crc 2023-10-01T10:54:33.025 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:32 smithi078 bash[78274]: cluster 2023-10-01T10:54:31.680686+0000 mon.a (mon.0) 1360 : cluster 0 osdmap e197: 8 total, 8 up, 8 in 2023-10-01T10:54:33.025 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:32 smithi078 bash[78274]: cluster 2023-10-01T10:54:31.687700+0000 osd.5 (osd.5) 142 : cluster 3 failed to encode map e197 with expected crc 2023-10-01T10:54:33.025 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:32 smithi078 bash[78274]: cluster 2023-10-01T10:54:31.687957+0000 osd.6 (osd.6) 125 : cluster 3 failed to encode map e197 with expected crc 2023-10-01T10:54:33.026 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:32 smithi078 bash[78274]: cluster 2023-10-01T10:54:31.689790+0000 osd.6 (osd.6) 126 : cluster 3 failed to encode map e197 with expected crc 2023-10-01T10:54:33.026 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:32 smithi078 bash[78274]: audit 2023-10-01T10:54:32.027981+0000 mon.a (mon.0) 1361 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-10-01T10:54:33.026 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:32 smithi078 bash[78274]: audit 2023-10-01T10:54:32.440452+0000 mon.a (mon.0) 1362 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:54:33.026 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:32 smithi078 bash[78274]: audit 2023-10-01T10:54:32.543923+0000 mon.a (mon.0) 1363 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "20"}]': finished 2023-10-01T10:54:33.026 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:32 smithi078 bash[78274]: cluster 2023-10-01T10:54:32.544051+0000 mon.a (mon.0) 1364 : cluster 0 osdmap e198: 8 total, 8 up, 8 in 2023-10-01T10:54:33.027 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:32 smithi078 bash[75299]: cluster 2023-10-01T10:54:31.679419+0000 osd.7 (osd.7) 157 : cluster 3 failed to encode map e197 with expected crc 2023-10-01T10:54:33.027 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:32 smithi078 bash[75299]: cluster 2023-10-01T10:54:31.680686+0000 mon.a (mon.0) 1360 : cluster 0 osdmap e197: 8 total, 8 up, 8 in 2023-10-01T10:54:33.027 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:32 smithi078 bash[75299]: cluster 2023-10-01T10:54:31.687700+0000 osd.5 (osd.5) 142 : cluster 3 failed to encode map e197 with expected crc 2023-10-01T10:54:33.027 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:32 smithi078 bash[75299]: cluster 2023-10-01T10:54:31.687957+0000 osd.6 (osd.6) 125 : cluster 3 failed to encode map e197 with expected crc 2023-10-01T10:54:33.027 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:32 smithi078 bash[75299]: cluster 2023-10-01T10:54:31.689790+0000 osd.6 (osd.6) 126 : cluster 3 failed to encode map e197 with expected crc 2023-10-01T10:54:33.027 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:32 smithi078 bash[75299]: audit 2023-10-01T10:54:32.027981+0000 mon.a (mon.0) 1361 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-10-01T10:54:33.027 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:32 smithi078 bash[75299]: audit 2023-10-01T10:54:32.440452+0000 mon.a (mon.0) 1362 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:54:33.027 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:32 smithi078 bash[75299]: audit 2023-10-01T10:54:32.543923+0000 mon.a (mon.0) 1363 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "20"}]': finished 2023-10-01T10:54:33.027 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:32 smithi078 bash[75299]: cluster 2023-10-01T10:54:32.544051+0000 mon.a (mon.0) 1364 : cluster 0 osdmap e198: 8 total, 8 up, 8 in 2023-10-01T10:54:33.028 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:33.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:33.031 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:54:33.132 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:33.158 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:33.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:33.163 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:54:33.264 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:33.293 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:33.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:33.296 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:54:33.397 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:33.427 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:33.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:33.430 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:54:33.531 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:33.567 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:33.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:33.569 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:54:33.669 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:33.701 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:33.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:33.704 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:33.792 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2023-10-01T10:54:33.792 INFO:teuthology.orchestra.run.smithi078.stdout: "ops": [ 2023-10-01T10:54:33.792 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.792 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46037.0:176554 11.13 11.6330f233 (undecoded) ondisk+write+known_if_redirected e199)", 2023-10-01T10:54:33.792 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:54:33.723328+0000", 2023-10-01T10:54:33.792 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 0.063440914000000001, 2023-10-01T10:54:33.792 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.063472129000000002, 2023-10-01T10:54:33.793 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:54:33.793 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "delayed", 2023-10-01T10:54:33.793 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:54:33.793 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46037", 2023-10-01T10:54:33.793 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:54:33.793 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 176554 2023-10-01T10:54:33.793 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.793 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:54:33.793 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.793 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:54:33.793 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.723328+0000", 2023-10-01T10:54:33.794 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:54:33.794 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.794 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.794 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:54:33.794 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.723328+0000", 2023-10-01T10:54:33.794 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:54:33.794 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.794 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.794 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:54:33.794 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.723326+0000", 2023-10-01T10:54:33.794 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999981 2023-10-01T10:54:33.794 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.795 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.795 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:54:33.795 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.723333+0000", 2023-10-01T10:54:33.795 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.4259999999999997e-06 2023-10-01T10:54:33.795 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.795 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.795 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:54:33.795 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.723346+0000", 2023-10-01T10:54:33.795 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.2442e-05 2023-10-01T10:54:33.795 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.795 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.795 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:54:33.796 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.723355+0000", 2023-10-01T10:54:33.796 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.363e-06 2023-10-01T10:54:33.796 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.796 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.796 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:54:33.796 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.723986+0000", 2023-10-01T10:54:33.796 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00063095700000000002 2023-10-01T10:54:33.796 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.796 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.796 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T10:54:33.796 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.723999+0000", 2023-10-01T10:54:33.796 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.3088000000000001e-05 2023-10-01T10:54:33.797 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:54:33.797 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:54:33.797 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:54:33.797 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.797 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.797 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46037.0:176533 11.13 11.17b4a2f3 (undecoded) ondisk+write+known_if_redirected e199)", 2023-10-01T10:54:33.797 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:54:33.686751+0000", 2023-10-01T10:54:33.797 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 0.100018076, 2023-10-01T10:54:33.797 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.100130858, 2023-10-01T10:54:33.797 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:54:33.797 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "delayed", 2023-10-01T10:54:33.798 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:54:33.798 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46037", 2023-10-01T10:54:33.798 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:54:33.798 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 176533 2023-10-01T10:54:33.798 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.798 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:54:33.798 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.798 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:54:33.798 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.686751+0000", 2023-10-01T10:54:33.798 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:54:33.798 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.799 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.799 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:54:33.799 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.686751+0000", 2023-10-01T10:54:33.799 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:54:33.799 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.799 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.799 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:54:33.799 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.686749+0000", 2023-10-01T10:54:33.799 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999976 2023-10-01T10:54:33.799 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.799 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.799 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:54:33.800 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.686759+0000", 2023-10-01T10:54:33.800 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.0138e-05 2023-10-01T10:54:33.800 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.800 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.800 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:54:33.800 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.686779+0000", 2023-10-01T10:54:33.800 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.0115e-05 2023-10-01T10:54:33.800 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.800 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.800 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:54:33.800 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.686790+0000", 2023-10-01T10:54:33.800 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1034e-05 2023-10-01T10:54:33.801 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.801 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.801 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:54:33.801 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.686879+0000", 2023-10-01T10:54:33.801 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.8918999999999993e-05 2023-10-01T10:54:33.801 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.801 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.801 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T10:54:33.801 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.686892+0000", 2023-10-01T10:54:33.801 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.3482000000000001e-05 2023-10-01T10:54:33.801 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:54:33.802 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:54:33.802 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:54:33.802 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.802 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.802 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46037.0:176584 11.13 11.503558f3 (undecoded) ondisk+write+known_if_redirected e199)", 2023-10-01T10:54:33.802 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:54:33.769745+0000", 2023-10-01T10:54:33.802 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 0.017023914000000001, 2023-10-01T10:54:33.802 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.017209107000000001, 2023-10-01T10:54:33.802 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:54:33.802 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "delayed", 2023-10-01T10:54:33.802 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:54:33.802 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46037", 2023-10-01T10:54:33.803 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:54:33.803 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 176584 2023-10-01T10:54:33.803 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.803 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:54:33.803 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.803 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:54:33.803 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.769745+0000", 2023-10-01T10:54:33.803 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:54:33.803 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.803 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.803 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:54:33.803 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.769745+0000", 2023-10-01T10:54:33.804 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:54:33.804 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.804 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.804 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:54:33.804 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.769743+0000", 2023-10-01T10:54:33.804 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999981 2023-10-01T10:54:33.804 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.804 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.804 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:54:33.804 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.769751+0000", 2023-10-01T10:54:33.804 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.5340000000000002e-06 2023-10-01T10:54:33.805 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.805 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.805 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:54:33.805 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.769766+0000", 2023-10-01T10:54:33.805 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.5347000000000001e-05 2023-10-01T10:54:33.805 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.805 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.805 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:54:33.805 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.769774+0000", 2023-10-01T10:54:33.805 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.4700000000000002e-06 2023-10-01T10:54:33.805 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.805 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.806 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:54:33.806 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.769825+0000", 2023-10-01T10:54:33.806 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.0061000000000001e-05 2023-10-01T10:54:33.806 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.806 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.806 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T10:54:33.806 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.769835+0000", 2023-10-01T10:54:33.806 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.0808000000000001e-05 2023-10-01T10:54:33.806 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:54:33.806 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:54:33.806 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:54:33.806 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.807 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.807 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46037.0:176589 11.13 11.c1b1b233 (undecoded) ondisk+write+known_if_redirected e199)", 2023-10-01T10:54:33.807 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:54:33.776428+0000", 2023-10-01T10:54:33.807 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 0.010341268000000001, 2023-10-01T10:54:33.807 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.010599021, 2023-10-01T10:54:33.807 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:54:33.807 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "delayed", 2023-10-01T10:54:33.807 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:54:33.807 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46037", 2023-10-01T10:54:33.808 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:54:33.808 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 176589 2023-10-01T10:54:33.808 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.808 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:54:33.808 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.808 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:54:33.808 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.776428+0000", 2023-10-01T10:54:33.808 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:54:33.808 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.808 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.808 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:54:33.809 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.776428+0000", 2023-10-01T10:54:33.809 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:54:33.809 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.809 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.809 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:54:33.809 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.776426+0000", 2023-10-01T10:54:33.809 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999981 2023-10-01T10:54:33.809 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.809 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.809 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:54:33.809 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.776433+0000", 2023-10-01T10:54:33.809 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.8790000000000003e-06 2023-10-01T10:54:33.810 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.810 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.810 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:54:33.810 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.776451+0000", 2023-10-01T10:54:33.810 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.7453e-05 2023-10-01T10:54:33.810 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.810 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.810 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:54:33.810 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.776460+0000", 2023-10-01T10:54:33.810 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.7299999999999994e-06 2023-10-01T10:54:33.810 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.811 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.811 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:54:33.811 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.776656+0000", 2023-10-01T10:54:33.811 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00019599900000000001 2023-10-01T10:54:33.811 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.811 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.811 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T10:54:33.811 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.776684+0000", 2023-10-01T10:54:33.811 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.8699000000000001e-05 2023-10-01T10:54:33.811 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:54:33.811 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:54:33.811 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:54:33.812 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.812 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.812 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46037.0:176163 11.13 11.6fd86013 (undecoded) ondisk+retry+write+known_if_redirected e199)", 2023-10-01T10:54:33.812 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:54:33.553395+0000", 2023-10-01T10:54:33.812 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 0.233373683, 2023-10-01T10:54:33.812 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.233708523, 2023-10-01T10:54:33.812 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:54:33.812 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "delayed", 2023-10-01T10:54:33.812 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:54:33.812 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46037", 2023-10-01T10:54:33.812 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:54:33.813 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 176163 2023-10-01T10:54:33.813 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.813 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:54:33.813 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.813 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:54:33.813 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553395+0000", 2023-10-01T10:54:33.813 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:54:33.813 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.813 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.813 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:54:33.813 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553395+0000", 2023-10-01T10:54:33.813 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:54:33.814 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.814 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.814 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:54:33.814 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553393+0000", 2023-10-01T10:54:33.814 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999976 2023-10-01T10:54:33.814 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.814 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.814 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:54:33.814 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553403+0000", 2023-10-01T10:54:33.814 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.0597999999999999e-05 2023-10-01T10:54:33.814 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.815 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.815 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:54:33.815 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553421+0000", 2023-10-01T10:54:33.815 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.7847999999999999e-05 2023-10-01T10:54:33.815 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.815 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.815 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:54:33.815 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553434+0000", 2023-10-01T10:54:33.815 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.2746e-05 2023-10-01T10:54:33.815 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.816 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.816 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:54:33.816 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.556358+0000", 2023-10-01T10:54:33.816 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0029241969999999999 2023-10-01T10:54:33.816 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.816 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.816 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T10:54:33.816 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.556366+0000", 2023-10-01T10:54:33.816 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.0260000000000007e-06 2023-10-01T10:54:33.816 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:54:33.818 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:54:33.818 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:54:33.818 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.818 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.818 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.46037.0:176591 11.f e199/183 11:ff189e42:::benchmark_data_smithi078_157677_object69420:head v 199'10879, mlcod=199'10876)", 2023-10-01T10:54:33.818 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:54:33.781123+0000", 2023-10-01T10:54:33.818 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 0.0056461669999999997, 2023-10-01T10:54:33.818 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0060521819999999997, 2023-10-01T10:54:33.818 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:54:33.818 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:54:33.819 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:54:33.819 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.819 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:54:33.819 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.781123+0000", 2023-10-01T10:54:33.819 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:54:33.819 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.819 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.819 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:54:33.819 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.781123+0000", 2023-10-01T10:54:33.819 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:54:33.819 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.819 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.819 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:54:33.820 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.781122+0000", 2023-10-01T10:54:33.820 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T10:54:33.820 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.820 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.820 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:54:33.820 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.781128+0000", 2023-10-01T10:54:33.820 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.4439999999999996e-06 2023-10-01T10:54:33.820 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.820 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.820 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:54:33.820 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.781140+0000", 2023-10-01T10:54:33.820 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1796e-05 2023-10-01T10:54:33.821 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.821 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.821 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:54:33.821 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.781148+0000", 2023-10-01T10:54:33.821 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.6669999999999996e-06 2023-10-01T10:54:33.821 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.821 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.821 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:54:33.821 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.781337+0000", 2023-10-01T10:54:33.821 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00018903299999999999 2023-10-01T10:54:33.821 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.821 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.822 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:54:33.822 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.781390+0000", 2023-10-01T10:54:33.822 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.3001999999999998e-05 2023-10-01T10:54:33.822 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.822 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.822 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:54:33.822 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.787031+0000", 2023-10-01T10:54:33.822 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0056410449999999999 2023-10-01T10:54:33.822 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.822 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.822 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:54:33.822 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.787175+0000", 2023-10-01T10:54:33.822 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00014414500000000001 2023-10-01T10:54:33.823 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:54:33.823 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:54:33.823 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:54:33.823 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.823 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.823 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46037.0:176170 11.13 11.ec438833 (undecoded) ondisk+retry+write+known_if_redirected e199)", 2023-10-01T10:54:33.823 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:54:33.553478+0000", 2023-10-01T10:54:33.824 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 0.23329081700000001, 2023-10-01T10:54:33.824 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.23377946399999999, 2023-10-01T10:54:33.824 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:54:33.824 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "delayed", 2023-10-01T10:54:33.824 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:54:33.824 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46037", 2023-10-01T10:54:33.824 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:54:33.824 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 176170 2023-10-01T10:54:33.824 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.824 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:54:33.824 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.824 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:54:33.825 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553478+0000", 2023-10-01T10:54:33.825 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:54:33.825 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.825 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.825 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:54:33.825 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553478+0000", 2023-10-01T10:54:33.825 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:54:33.825 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.825 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.826 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:54:33.826 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553477+0000", 2023-10-01T10:54:33.826 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999995 2023-10-01T10:54:33.826 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.826 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.826 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:54:33.826 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553481+0000", 2023-10-01T10:54:33.826 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.9639999999999997e-06 2023-10-01T10:54:33.826 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.826 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.826 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:54:33.827 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553490+0000", 2023-10-01T10:54:33.827 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.6149999999999997e-06 2023-10-01T10:54:33.827 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.827 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.827 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:54:33.827 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553495+0000", 2023-10-01T10:54:33.827 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.113e-06 2023-10-01T10:54:33.827 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.827 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.827 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:54:33.827 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.556534+0000", 2023-10-01T10:54:33.828 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.003038443 2023-10-01T10:54:33.828 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.828 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.828 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T10:54:33.828 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.556541+0000", 2023-10-01T10:54:33.828 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.7039999999999994e-06 2023-10-01T10:54:33.828 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:54:33.828 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:54:33.829 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:54:33.829 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.829 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.829 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46037.0:176346 11.13 11.e42d6793 (undecoded) ondisk+retry+write+known_if_redirected e199)", 2023-10-01T10:54:33.829 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:54:33.553500+0000", 2023-10-01T10:54:33.829 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 0.23326918599999999, 2023-10-01T10:54:33.829 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.23384781499999999, 2023-10-01T10:54:33.829 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:54:33.829 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "delayed", 2023-10-01T10:54:33.829 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:54:33.829 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46037", 2023-10-01T10:54:33.829 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:54:33.829 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 176346 2023-10-01T10:54:33.830 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.830 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:54:33.830 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.830 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:54:33.830 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553500+0000", 2023-10-01T10:54:33.830 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:54:33.830 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.830 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.830 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:54:33.830 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553500+0000", 2023-10-01T10:54:33.830 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:54:33.831 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.831 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.831 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:54:33.831 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553499+0000", 2023-10-01T10:54:33.831 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999995 2023-10-01T10:54:33.831 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.831 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.831 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:54:33.831 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553501+0000", 2023-10-01T10:54:33.831 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.1160000000000002e-06 2023-10-01T10:54:33.831 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.832 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.832 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:54:33.832 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553506+0000", 2023-10-01T10:54:33.832 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.3449999999999999e-06 2023-10-01T10:54:33.832 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.832 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.832 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:54:33.832 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553510+0000", 2023-10-01T10:54:33.832 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.9929999999999997e-06 2023-10-01T10:54:33.833 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.833 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.833 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:54:33.833 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.556567+0000", 2023-10-01T10:54:33.833 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0030572609999999999 2023-10-01T10:54:33.833 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.833 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.833 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T10:54:33.834 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.556575+0000", 2023-10-01T10:54:33.834 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.4390000000000003e-06 2023-10-01T10:54:33.834 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:54:33.834 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:54:33.834 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:54:33.834 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.834 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.834 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46037.0:176347 11.13 11.ae4d05f3 (undecoded) ondisk+retry+write+known_if_redirected e199)", 2023-10-01T10:54:33.834 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:54:33.553513+0000", 2023-10-01T10:54:33.834 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 0.23325594799999999, 2023-10-01T10:54:33.834 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.23392729000000001, 2023-10-01T10:54:33.835 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:54:33.835 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "delayed", 2023-10-01T10:54:33.835 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:54:33.835 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46037", 2023-10-01T10:54:33.835 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:54:33.835 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 176347 2023-10-01T10:54:33.835 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.835 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:54:33.836 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.836 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:54:33.836 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553513+0000", 2023-10-01T10:54:33.836 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:54:33.836 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.836 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.836 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:54:33.836 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553513+0000", 2023-10-01T10:54:33.836 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:54:33.837 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.837 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.837 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:54:33.837 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553513+0000", 2023-10-01T10:54:33.837 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999995 2023-10-01T10:54:33.837 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.837 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.837 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:54:33.838 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553514+0000", 2023-10-01T10:54:33.838 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.818e-06 2023-10-01T10:54:33.838 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.838 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.838 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:54:33.838 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553518+0000", 2023-10-01T10:54:33.838 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.7799999999999998e-06 2023-10-01T10:54:33.838 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.838 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.838 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:54:33.839 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553523+0000", 2023-10-01T10:54:33.839 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.5129999999999998e-06 2023-10-01T10:54:33.839 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.839 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.839 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:54:33.839 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.556600+0000", 2023-10-01T10:54:33.839 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0030775009999999998 2023-10-01T10:54:33.839 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.839 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.839 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T10:54:33.839 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.556607+0000", 2023-10-01T10:54:33.839 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.934e-06 2023-10-01T10:54:33.839 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:54:33.840 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:54:33.840 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:54:33.840 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.840 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.840 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46037.0:176390 11.13 11.c5d149f3 (undecoded) ondisk+retry+write+known_if_redirected e199)", 2023-10-01T10:54:33.840 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:54:33.553526+0000", 2023-10-01T10:54:33.840 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 0.233243062, 2023-10-01T10:54:33.840 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.233987158, 2023-10-01T10:54:33.840 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:54:33.840 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "delayed", 2023-10-01T10:54:33.840 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:54:33.840 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46037", 2023-10-01T10:54:33.841 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:54:33.841 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 176390 2023-10-01T10:54:33.841 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.841 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:54:33.841 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.841 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:54:33.841 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553526+0000", 2023-10-01T10:54:33.841 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:54:33.841 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.841 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.841 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:54:33.841 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553526+0000", 2023-10-01T10:54:33.841 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:54:33.842 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.842 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.842 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:54:33.842 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553526+0000", 2023-10-01T10:54:33.842 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999995 2023-10-01T10:54:33.842 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.842 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.842 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:54:33.842 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553528+0000", 2023-10-01T10:54:33.842 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.3649999999999998e-06 2023-10-01T10:54:33.842 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.843 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.843 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:54:33.843 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553534+0000", 2023-10-01T10:54:33.843 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.387e-06 2023-10-01T10:54:33.843 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.843 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.843 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:54:33.843 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553540+0000", 2023-10-01T10:54:33.843 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.3750000000000002e-06 2023-10-01T10:54:33.843 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.843 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.844 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:54:33.844 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.556634+0000", 2023-10-01T10:54:33.844 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0030939000000000001 2023-10-01T10:54:33.844 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.844 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.844 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T10:54:33.844 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.556641+0000", 2023-10-01T10:54:33.844 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.25e-06 2023-10-01T10:54:33.844 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:54:33.844 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:54:33.844 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:54:33.844 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.844 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.845 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46037.0:176477 11.13 11.51b15773 (undecoded) ondisk+retry+write+known_if_redirected e199)", 2023-10-01T10:54:33.845 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:54:33.553546+0000", 2023-10-01T10:54:33.845 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 0.233223182, 2023-10-01T10:54:33.845 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.23404540400000001, 2023-10-01T10:54:33.845 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:54:33.845 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "delayed", 2023-10-01T10:54:33.845 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:54:33.845 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46037", 2023-10-01T10:54:33.846 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:54:33.846 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 176477 2023-10-01T10:54:33.846 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.846 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:54:33.846 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.846 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:54:33.846 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553546+0000", 2023-10-01T10:54:33.846 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:54:33.846 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.846 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.846 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:54:33.846 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553546+0000", 2023-10-01T10:54:33.847 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:54:33.847 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.847 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.847 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:54:33.847 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553545+0000", 2023-10-01T10:54:33.847 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999995 2023-10-01T10:54:33.847 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.847 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.847 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:54:33.847 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553548+0000", 2023-10-01T10:54:33.847 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.005e-06 2023-10-01T10:54:33.847 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.848 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.848 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:54:33.848 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553555+0000", 2023-10-01T10:54:33.848 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.4099999999999996e-06 2023-10-01T10:54:33.848 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.848 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.848 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:54:33.848 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553560+0000", 2023-10-01T10:54:33.848 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.1370000000000001e-06 2023-10-01T10:54:33.848 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.848 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.848 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:54:33.849 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.556662+0000", 2023-10-01T10:54:33.849 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00310247 2023-10-01T10:54:33.849 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.849 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.849 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T10:54:33.849 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.556670+0000", 2023-10-01T10:54:33.849 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.7209999999999994e-06 2023-10-01T10:54:33.849 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:54:33.849 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:54:33.849 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:54:33.849 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.849 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.850 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46037.0:176550 11.13 11.31dca2f3 (undecoded) ondisk+write+known_if_redirected e199)", 2023-10-01T10:54:33.850 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:54:33.716087+0000", 2023-10-01T10:54:33.850 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 0.070681657999999994, 2023-10-01T10:54:33.850 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.071578303999999995, 2023-10-01T10:54:33.850 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:54:33.850 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "delayed", 2023-10-01T10:54:33.850 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:54:33.850 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46037", 2023-10-01T10:54:33.850 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:54:33.850 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 176550 2023-10-01T10:54:33.850 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.850 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:54:33.850 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.851 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:54:33.851 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.716087+0000", 2023-10-01T10:54:33.851 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:54:33.851 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.851 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.851 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:54:33.851 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.716087+0000", 2023-10-01T10:54:33.851 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:54:33.851 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.851 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.851 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:54:33.852 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.716085+0000", 2023-10-01T10:54:33.852 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999981 2023-10-01T10:54:33.852 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.852 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.852 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:54:33.852 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.716093+0000", 2023-10-01T10:54:33.852 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.4229999999999999e-06 2023-10-01T10:54:33.853 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.853 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.853 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:54:33.853 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.716110+0000", 2023-10-01T10:54:33.853 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.7414999999999998e-05 2023-10-01T10:54:33.853 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.853 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.853 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:54:33.853 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.716119+0000", 2023-10-01T10:54:33.853 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.5010000000000004e-06 2023-10-01T10:54:33.853 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.853 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.854 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:54:33.854 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.716309+0000", 2023-10-01T10:54:33.854 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00018996200000000001 2023-10-01T10:54:33.854 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:54:33.854 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:54:33.854 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T10:54:33.854 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.716320+0000", 2023-10-01T10:54:33.854 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1402e-05 2023-10-01T10:54:33.854 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:54:33.854 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:54:33.854 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:54:33.854 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:54:33.855 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2023-10-01T10:54:33.855 INFO:teuthology.orchestra.run.smithi078.stdout: "num_ops": 12 2023-10-01T10:54:33.855 INFO:teuthology.orchestra.run.smithi078.stdout:} 2023-10-01T10:54:33.855 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:33.856 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:33 smithi078 bash[78274]: cluster 2023-10-01T10:54:32.022549+0000 osd.3 (osd.3) 11 : cluster 0 11.19 scrub starts 2023-10-01T10:54:33.856 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:33 smithi078 bash[78274]: cluster 2023-10-01T10:54:32.027209+0000 mgr.y (mgr.25072) 1110 : cluster 0 pgmap v1076: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 2.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 386 op/s; 3118/122568 objects misplaced (2.544%); 8.7 MiB/s, 138 objects/s recovering 2023-10-01T10:54:33.856 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:33 smithi078 bash[78274]: cluster 2023-10-01T10:54:32.331655+0000 osd.1 (osd.1) 4 : cluster 0 11.10 scrub starts 2023-10-01T10:54:33.856 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:33 smithi078 bash[78274]: cluster 2023-10-01T10:54:32.540880+0000 osd.5 (osd.5) 143 : cluster 3 failed to encode map e198 with expected crc 2023-10-01T10:54:33.856 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:33 smithi078 bash[78274]: cluster 2023-10-01T10:54:32.545646+0000 osd.6 (osd.6) 127 : cluster 3 failed to encode map e198 with expected crc 2023-10-01T10:54:33.857 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:33 smithi078 bash[78274]: cluster 2023-10-01T10:54:32.546668+0000 osd.7 (osd.7) 158 : cluster 3 failed to encode map e198 with expected crc 2023-10-01T10:54:33.857 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:33 smithi078 bash[78274]: cluster 2023-10-01T10:54:32.548256+0000 osd.2 (osd.2) 1 : cluster 0 11.12 starting backfill to osd.7 from (0'0,0'0] MAX to 195'10739 2023-10-01T10:54:33.857 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:33 smithi078 bash[78274]: cluster 2023-10-01T10:54:33.547635+0000 mon.a (mon.0) 1365 : cluster 0 osdmap e199: 8 total, 8 up, 8 in 2023-10-01T10:54:33.857 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:33 smithi078 bash[75299]: cluster 2023-10-01T10:54:32.022549+0000 osd.3 (osd.3) 11 : cluster 0 11.19 scrub starts 2023-10-01T10:54:33.857 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:33 smithi078 bash[75299]: cluster 2023-10-01T10:54:32.027209+0000 mgr.y (mgr.25072) 1110 : cluster 0 pgmap v1076: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 2.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 386 op/s; 3118/122568 objects misplaced (2.544%); 8.7 MiB/s, 138 objects/s recovering 2023-10-01T10:54:33.857 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:33 smithi078 bash[75299]: cluster 2023-10-01T10:54:32.331655+0000 osd.1 (osd.1) 4 : cluster 0 11.10 scrub starts 2023-10-01T10:54:33.857 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:33 smithi078 bash[75299]: cluster 2023-10-01T10:54:32.540880+0000 osd.5 (osd.5) 143 : cluster 3 failed to encode map e198 with expected crc 2023-10-01T10:54:33.857 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:33 smithi078 bash[75299]: cluster 2023-10-01T10:54:32.545646+0000 osd.6 (osd.6) 127 : cluster 3 failed to encode map e198 with expected crc 2023-10-01T10:54:33.857 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:33 smithi078 bash[75299]: cluster 2023-10-01T10:54:32.546668+0000 osd.7 (osd.7) 158 : cluster 3 failed to encode map e198 with expected crc 2023-10-01T10:54:33.857 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:33 smithi078 bash[75299]: cluster 2023-10-01T10:54:32.548256+0000 osd.2 (osd.2) 1 : cluster 0 11.12 starting backfill to osd.7 from (0'0,0'0] MAX to 195'10739 2023-10-01T10:54:33.857 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:33 smithi078 bash[75299]: cluster 2023-10-01T10:54:33.547635+0000 mon.a (mon.0) 1365 : cluster 0 osdmap e199: 8 total, 8 up, 8 in 2023-10-01T10:54:33.890 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:33.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:33.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:54:33.959 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:33 smithi173 bash[58816]: cluster 2023-10-01T10:54:32.022549+0000 osd.3 (osd.3) 11 : cluster 0 11.19 scrub starts 2023-10-01T10:54:33.959 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:33 smithi173 bash[58816]: cluster 2023-10-01T10:54:32.027209+0000 mgr.y (mgr.25072) 1110 : cluster 0 pgmap v1076: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 2.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 386 op/s; 3118/122568 objects misplaced (2.544%); 8.7 MiB/s, 138 objects/s recovering 2023-10-01T10:54:33.959 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:33 smithi173 bash[58816]: cluster 2023-10-01T10:54:32.331655+0000 osd.1 (osd.1) 4 : cluster 0 11.10 scrub starts 2023-10-01T10:54:33.959 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:33 smithi173 bash[58816]: cluster 2023-10-01T10:54:32.540880+0000 osd.5 (osd.5) 143 : cluster 3 failed to encode map e198 with expected crc 2023-10-01T10:54:33.959 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:33 smithi173 bash[58816]: cluster 2023-10-01T10:54:32.545646+0000 osd.6 (osd.6) 127 : cluster 3 failed to encode map e198 with expected crc 2023-10-01T10:54:33.959 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:33 smithi173 bash[58816]: cluster 2023-10-01T10:54:32.546668+0000 osd.7 (osd.7) 158 : cluster 3 failed to encode map e198 with expected crc 2023-10-01T10:54:33.959 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:33 smithi173 bash[58816]: cluster 2023-10-01T10:54:32.548256+0000 osd.2 (osd.2) 1 : cluster 0 11.12 starting backfill to osd.7 from (0'0,0'0] MAX to 195'10739 2023-10-01T10:54:33.960 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:33 smithi173 bash[58816]: cluster 2023-10-01T10:54:33.547635+0000 mon.a (mon.0) 1365 : cluster 0 osdmap e199: 8 total, 8 up, 8 in 2023-10-01T10:54:33.995 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:34.033 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:34.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:34.035 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:54:34.136 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:34.161 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:34.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:34.164 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:54:34.264 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:34.294 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:34.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:34.330 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:54:34.431 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:34.450 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:34.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:34.452 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:34.553 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:34.578 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:34.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:34.581 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:54:34.682 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:34.713 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:34.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:34.716 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:54:34.817 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:34.828 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:34 smithi078 bash[78274]: cluster 2023-10-01T10:54:33.014539+0000 osd.3 (osd.3) 12 : cluster 0 11.18 scrub starts 2023-10-01T10:54:34.828 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:34 smithi078 bash[78274]: cluster 2023-10-01T10:54:33.232618+0000 osd.3 (osd.3) 13 : cluster 0 11.18 scrub ok 2023-10-01T10:54:34.828 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:34 smithi078 bash[78274]: cluster 2023-10-01T10:54:33.548792+0000 osd.5 (osd.5) 144 : cluster 3 failed to encode map e199 with expected crc 2023-10-01T10:54:34.828 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:34 smithi078 bash[78274]: cluster 2023-10-01T10:54:33.554106+0000 osd.7 (osd.7) 159 : cluster 3 failed to encode map e199 with expected crc 2023-10-01T10:54:34.828 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:34 smithi078 bash[78274]: cluster 2023-10-01T10:54:33.554647+0000 osd.6 (osd.6) 128 : cluster 3 failed to encode map e199 with expected crc 2023-10-01T10:54:34.828 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:34 smithi078 bash[78274]: audit 2023-10-01T10:54:34.028664+0000 mon.a (mon.0) 1366 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-10-01T10:54:34.828 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:34 smithi078 bash[78274]: audit 2023-10-01T10:54:34.550630+0000 mon.a (mon.0) 1367 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "21"}]': finished 2023-10-01T10:54:34.828 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:34 smithi078 bash[78274]: cluster 2023-10-01T10:54:34.550747+0000 mon.a (mon.0) 1368 : cluster 0 osdmap e200: 8 total, 8 up, 8 in 2023-10-01T10:54:34.829 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:34 smithi078 bash[75299]: cluster 2023-10-01T10:54:33.014539+0000 osd.3 (osd.3) 12 : cluster 0 11.18 scrub starts 2023-10-01T10:54:34.829 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:34 smithi078 bash[75299]: cluster 2023-10-01T10:54:33.232618+0000 osd.3 (osd.3) 13 : cluster 0 11.18 scrub ok 2023-10-01T10:54:34.829 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:34 smithi078 bash[75299]: cluster 2023-10-01T10:54:33.548792+0000 osd.5 (osd.5) 144 : cluster 3 failed to encode map e199 with expected crc 2023-10-01T10:54:34.829 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:34 smithi078 bash[75299]: cluster 2023-10-01T10:54:33.554106+0000 osd.7 (osd.7) 159 : cluster 3 failed to encode map e199 with expected crc 2023-10-01T10:54:34.829 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:34 smithi078 bash[75299]: cluster 2023-10-01T10:54:33.554647+0000 osd.6 (osd.6) 128 : cluster 3 failed to encode map e199 with expected crc 2023-10-01T10:54:34.829 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:34 smithi078 bash[75299]: audit 2023-10-01T10:54:34.028664+0000 mon.a (mon.0) 1366 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-10-01T10:54:34.829 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:34 smithi078 bash[75299]: audit 2023-10-01T10:54:34.550630+0000 mon.a (mon.0) 1367 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "21"}]': finished 2023-10-01T10:54:34.830 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:34 smithi078 bash[75299]: cluster 2023-10-01T10:54:34.550747+0000 mon.a (mon.0) 1368 : cluster 0 osdmap e200: 8 total, 8 up, 8 in 2023-10-01T10:54:34.843 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:34.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:34.846 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:34.946 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:34.968 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:34.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:34.971 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:54:35.072 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:35.103 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:35.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:35.109 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:54:35.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:34 smithi173 bash[58816]: cluster 2023-10-01T10:54:33.014539+0000 osd.3 (osd.3) 12 : cluster 0 11.18 scrub starts 2023-10-01T10:54:35.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:34 smithi173 bash[58816]: cluster 2023-10-01T10:54:33.232618+0000 osd.3 (osd.3) 13 : cluster 0 11.18 scrub ok 2023-10-01T10:54:35.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:34 smithi173 bash[58816]: cluster 2023-10-01T10:54:33.548792+0000 osd.5 (osd.5) 144 : cluster 3 failed to encode map e199 with expected crc 2023-10-01T10:54:35.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:34 smithi173 bash[58816]: cluster 2023-10-01T10:54:33.554106+0000 osd.7 (osd.7) 159 : cluster 3 failed to encode map e199 with expected crc 2023-10-01T10:54:35.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:34 smithi173 bash[58816]: cluster 2023-10-01T10:54:33.554647+0000 osd.6 (osd.6) 128 : cluster 3 failed to encode map e199 with expected crc 2023-10-01T10:54:35.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:34 smithi173 bash[58816]: audit 2023-10-01T10:54:34.028664+0000 mon.a (mon.0) 1366 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-10-01T10:54:35.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:34 smithi173 bash[58816]: audit 2023-10-01T10:54:34.550630+0000 mon.a (mon.0) 1367 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "21"}]': finished 2023-10-01T10:54:35.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:34 smithi173 bash[58816]: cluster 2023-10-01T10:54:34.550747+0000 mon.a (mon.0) 1368 : cluster 0 osdmap e200: 8 total, 8 up, 8 in 2023-10-01T10:54:35.210 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:35.234 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:35.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:35.242 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:35.343 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:35.372 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:35.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:35.375 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:54:35.476 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:35.501 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:35.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:35.505 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:54:35.606 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:35.631 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:35.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:35.636 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:54:35.738 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:35.773 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:35.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:35.776 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:54:35.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:35 smithi078 bash[78274]: cluster 2023-10-01T10:54:34.027706+0000 mgr.y (mgr.25072) 1111 : cluster 0 pgmap v1079: 99 pgs: 1 active+clean+scrubbing, 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 95 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 632 op/s; 4331/115392 objects misplaced (3.753%); 6.4 MiB/s, 102 objects/s recovering 2023-10-01T10:54:35.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:35 smithi078 bash[78274]: cluster 2023-10-01T10:54:34.140404+0000 osd.5 (osd.5) 145 : cluster 0 11.5 scrub starts 2023-10-01T10:54:35.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:35 smithi078 bash[78274]: cluster 2023-10-01T10:54:34.547346+0000 osd.7 (osd.7) 160 : cluster 3 failed to encode map e200 with expected crc 2023-10-01T10:54:35.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:35 smithi078 bash[78274]: cluster 2023-10-01T10:54:34.551378+0000 osd.1 (osd.1) 5 : cluster 0 11.13 starting backfill to osd.0 from (0'0,0'0] MAX to 197'10833 2023-10-01T10:54:35.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:35 smithi078 bash[78274]: cluster 2023-10-01T10:54:34.565117+0000 osd.1 (osd.1) 6 : cluster 0 11.13 starting backfill to osd.4 from (0'0,0'0] MAX to 197'10833 2023-10-01T10:54:35.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:35 smithi078 bash[78274]: cluster 2023-10-01T10:54:34.575002+0000 osd.1 (osd.1) 7 : cluster 0 11.13 starting backfill to osd.5 from (0'0,0'0] MAX to 197'10833 2023-10-01T10:54:35.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:35 smithi078 bash[78274]: audit 2023-10-01T10:54:34.883229+0000 mon.a (mon.0) 1369 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:54:35.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:35 smithi078 bash[78274]: audit 2023-10-01T10:54:34.889545+0000 mon.a (mon.0) 1370 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:54:35.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:35 smithi078 bash[78274]: cluster 2023-10-01T10:54:35.548411+0000 mon.a (mon.0) 1371 : cluster 0 osdmap e201: 8 total, 8 up, 8 in 2023-10-01T10:54:35.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:35 smithi078 bash[75299]: cluster 2023-10-01T10:54:34.027706+0000 mgr.y (mgr.25072) 1111 : cluster 0 pgmap v1079: 99 pgs: 1 active+clean+scrubbing, 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 95 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 632 op/s; 4331/115392 objects misplaced (3.753%); 6.4 MiB/s, 102 objects/s recovering 2023-10-01T10:54:35.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:35 smithi078 bash[75299]: cluster 2023-10-01T10:54:34.140404+0000 osd.5 (osd.5) 145 : cluster 0 11.5 scrub starts 2023-10-01T10:54:35.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:35 smithi078 bash[75299]: cluster 2023-10-01T10:54:34.547346+0000 osd.7 (osd.7) 160 : cluster 3 failed to encode map e200 with expected crc 2023-10-01T10:54:35.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:35 smithi078 bash[75299]: cluster 2023-10-01T10:54:34.551378+0000 osd.1 (osd.1) 5 : cluster 0 11.13 starting backfill to osd.0 from (0'0,0'0] MAX to 197'10833 2023-10-01T10:54:35.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:35 smithi078 bash[75299]: cluster 2023-10-01T10:54:34.565117+0000 osd.1 (osd.1) 6 : cluster 0 11.13 starting backfill to osd.4 from (0'0,0'0] MAX to 197'10833 2023-10-01T10:54:35.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:35 smithi078 bash[75299]: cluster 2023-10-01T10:54:34.575002+0000 osd.1 (osd.1) 7 : cluster 0 11.13 starting backfill to osd.5 from (0'0,0'0] MAX to 197'10833 2023-10-01T10:54:35.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:35 smithi078 bash[75299]: audit 2023-10-01T10:54:34.883229+0000 mon.a (mon.0) 1369 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:54:35.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:35 smithi078 bash[75299]: audit 2023-10-01T10:54:34.889545+0000 mon.a (mon.0) 1370 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:54:35.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:35 smithi078 bash[75299]: cluster 2023-10-01T10:54:35.548411+0000 mon.a (mon.0) 1371 : cluster 0 osdmap e201: 8 total, 8 up, 8 in 2023-10-01T10:54:35.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:35 smithi173 bash[58816]: cluster 2023-10-01T10:54:34.027706+0000 mgr.y (mgr.25072) 1111 : cluster 0 pgmap v1079: 99 pgs: 1 active+clean+scrubbing, 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 95 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 632 op/s; 4331/115392 objects misplaced (3.753%); 6.4 MiB/s, 102 objects/s recovering 2023-10-01T10:54:35.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:35 smithi173 bash[58816]: cluster 2023-10-01T10:54:34.140404+0000 osd.5 (osd.5) 145 : cluster 0 11.5 scrub starts 2023-10-01T10:54:35.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:35 smithi173 bash[58816]: cluster 2023-10-01T10:54:34.547346+0000 osd.7 (osd.7) 160 : cluster 3 failed to encode map e200 with expected crc 2023-10-01T10:54:35.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:35 smithi173 bash[58816]: cluster 2023-10-01T10:54:34.551378+0000 osd.1 (osd.1) 5 : cluster 0 11.13 starting backfill to osd.0 from (0'0,0'0] MAX to 197'10833 2023-10-01T10:54:35.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:35 smithi173 bash[58816]: cluster 2023-10-01T10:54:34.565117+0000 osd.1 (osd.1) 6 : cluster 0 11.13 starting backfill to osd.4 from (0'0,0'0] MAX to 197'10833 2023-10-01T10:54:35.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:35 smithi173 bash[58816]: cluster 2023-10-01T10:54:34.575002+0000 osd.1 (osd.1) 7 : cluster 0 11.13 starting backfill to osd.5 from (0'0,0'0] MAX to 197'10833 2023-10-01T10:54:35.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:35 smithi173 bash[58816]: audit 2023-10-01T10:54:34.883229+0000 mon.a (mon.0) 1369 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:54:35.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:35 smithi173 bash[58816]: audit 2023-10-01T10:54:34.889545+0000 mon.a (mon.0) 1370 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:54:35.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:35 smithi173 bash[58816]: cluster 2023-10-01T10:54:35.548411+0000 mon.a (mon.0) 1371 : cluster 0 osdmap e201: 8 total, 8 up, 8 in 2023-10-01T10:54:35.877 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:35.900 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:35.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:35.903 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:54:36.004 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:36.033 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:36.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:36.036 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:54:36.136 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:36.167 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:36.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:36.170 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:54:36.271 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:36.303 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:36.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:36.305 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:54:36.406 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:36.434 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:36.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:36.437 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:54:36.538 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:36.568 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:36.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:36.570 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:36.671 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:36.697 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:36.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:36.701 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:54:36.802 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:36.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:36 smithi078 bash[75299]: cluster 2023-10-01T10:54:34.971273+0000 osd.3 (osd.3) 14 : cluster 0 11.0 scrub starts 2023-10-01T10:54:36.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:36 smithi078 bash[75299]: cluster 2023-10-01T10:54:35.546884+0000 osd.7 (osd.7) 161 : cluster 3 failed to encode map e201 with expected crc 2023-10-01T10:54:36.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:36 smithi078 bash[75299]: cluster 2023-10-01T10:54:35.548942+0000 osd.5 (osd.5) 146 : cluster 3 failed to encode map e201 with expected crc 2023-10-01T10:54:36.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:36 smithi078 bash[75299]: audit 2023-10-01T10:54:36.029516+0000 mon.a (mon.0) 1372 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "22"}]: dispatch 2023-10-01T10:54:36.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:36 smithi078 bash[75299]: audit 2023-10-01T10:54:36.553535+0000 mon.a (mon.0) 1373 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "22"}]': finished 2023-10-01T10:54:36.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:36 smithi078 bash[75299]: cluster 2023-10-01T10:54:36.553680+0000 mon.a (mon.0) 1374 : cluster 0 osdmap e202: 8 total, 8 up, 8 in 2023-10-01T10:54:36.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:36 smithi078 bash[78274]: cluster 2023-10-01T10:54:34.971273+0000 osd.3 (osd.3) 14 : cluster 0 11.0 scrub starts 2023-10-01T10:54:36.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:36 smithi078 bash[78274]: cluster 2023-10-01T10:54:35.546884+0000 osd.7 (osd.7) 161 : cluster 3 failed to encode map e201 with expected crc 2023-10-01T10:54:36.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:36 smithi078 bash[78274]: cluster 2023-10-01T10:54:35.548942+0000 osd.5 (osd.5) 146 : cluster 3 failed to encode map e201 with expected crc 2023-10-01T10:54:36.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:36 smithi078 bash[78274]: audit 2023-10-01T10:54:36.029516+0000 mon.a (mon.0) 1372 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "22"}]: dispatch 2023-10-01T10:54:36.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:36 smithi078 bash[78274]: audit 2023-10-01T10:54:36.553535+0000 mon.a (mon.0) 1373 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "22"}]': finished 2023-10-01T10:54:36.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:36 smithi078 bash[78274]: cluster 2023-10-01T10:54:36.553680+0000 mon.a (mon.0) 1374 : cluster 0 osdmap e202: 8 total, 8 up, 8 in 2023-10-01T10:54:36.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:36.834 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:36.835 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:54:36.936 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:36.966 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:36.969 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:36 smithi173 bash[58816]: cluster 2023-10-01T10:54:34.971273+0000 osd.3 (osd.3) 14 : cluster 0 11.0 scrub starts 2023-10-01T10:54:36.969 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:36 smithi173 bash[58816]: cluster 2023-10-01T10:54:35.546884+0000 osd.7 (osd.7) 161 : cluster 3 failed to encode map e201 with expected crc 2023-10-01T10:54:36.970 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:36 smithi173 bash[58816]: cluster 2023-10-01T10:54:35.548942+0000 osd.5 (osd.5) 146 : cluster 3 failed to encode map e201 with expected crc 2023-10-01T10:54:36.970 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:36 smithi173 bash[58816]: audit 2023-10-01T10:54:36.029516+0000 mon.a (mon.0) 1372 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "22"}]: dispatch 2023-10-01T10:54:36.970 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:36 smithi173 bash[58816]: audit 2023-10-01T10:54:36.553535+0000 mon.a (mon.0) 1373 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "22"}]': finished 2023-10-01T10:54:36.970 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:36 smithi173 bash[58816]: cluster 2023-10-01T10:54:36.553680+0000 mon.a (mon.0) 1374 : cluster 0 osdmap e202: 8 total, 8 up, 8 in 2023-10-01T10:54:36.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:36.971 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:54:37.072 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:37.097 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:37.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:37.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:37.200 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:37.232 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:37.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:37.234 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:54:37.334 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:37.360 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:37.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:37.362 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:37.465 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:37.489 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:37.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:37.492 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:54:37.592 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:37.629 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:37.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:37.632 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:54:37.732 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:37.757 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:37.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:37.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:54:37.862 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:37.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:37 smithi173 bash[58816]: cluster 2023-10-01T10:54:36.028251+0000 mgr.y (mgr.25072) 1112 : cluster 0 pgmap v1082: 99 pgs: 1 active+remapped+backfilling, 1 active+clean+scrubbing, 1 active+remapped+backfill_wait, 96 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 697 op/s; 2579/114201 objects misplaced (2.258%); 11 MiB/s, 170 objects/s recovering 2023-10-01T10:54:37.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:37 smithi173 bash[58816]: cluster 2023-10-01T10:54:36.554764+0000 osd.2 (osd.2) 2 : cluster 0 11.14 starting backfill to osd.1 from (0'0,0'0] MAX to 199'10690 2023-10-01T10:54:37.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:37 smithi173 bash[58816]: cluster 2023-10-01T10:54:36.556734+0000 osd.7 (osd.7) 162 : cluster 3 failed to encode map e202 with expected crc 2023-10-01T10:54:37.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:37 smithi173 bash[58816]: cluster 2023-10-01T10:54:36.558043+0000 osd.6 (osd.6) 129 : cluster 3 failed to encode map e202 with expected crc 2023-10-01T10:54:37.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:37 smithi173 bash[58816]: cluster 2023-10-01T10:54:36.562862+0000 osd.5 (osd.5) 147 : cluster 3 failed to encode map e202 with expected crc 2023-10-01T10:54:37.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:37 smithi173 bash[58816]: cluster 2023-10-01T10:54:36.566398+0000 osd.5 (osd.5) 148 : cluster 3 failed to encode map e202 with expected crc 2023-10-01T10:54:37.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:37 smithi173 bash[58816]: cluster 2023-10-01T10:54:36.572629+0000 osd.2 (osd.2) 3 : cluster 0 11.14 starting backfill to osd.4 from (0'0,0'0] MAX to 199'10690 2023-10-01T10:54:37.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:37 smithi173 bash[58816]: cluster 2023-10-01T10:54:37.544123+0000 mon.a (mon.0) 1375 : cluster 0 osdmap e203: 8 total, 8 up, 8 in 2023-10-01T10:54:37.890 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:37.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:37.894 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:54:37.995 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:38.026 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:38.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:38.031 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:54:38.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:37 smithi078 bash[75299]: cluster 2023-10-01T10:54:36.028251+0000 mgr.y (mgr.25072) 1112 : cluster 0 pgmap v1082: 99 pgs: 1 active+remapped+backfilling, 1 active+clean+scrubbing, 1 active+remapped+backfill_wait, 96 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 697 op/s; 2579/114201 objects misplaced (2.258%); 11 MiB/s, 170 objects/s recovering 2023-10-01T10:54:38.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:37 smithi078 bash[75299]: cluster 2023-10-01T10:54:36.554764+0000 osd.2 (osd.2) 2 : cluster 0 11.14 starting backfill to osd.1 from (0'0,0'0] MAX to 199'10690 2023-10-01T10:54:38.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:37 smithi078 bash[75299]: cluster 2023-10-01T10:54:36.556734+0000 osd.7 (osd.7) 162 : cluster 3 failed to encode map e202 with expected crc 2023-10-01T10:54:38.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:37 smithi078 bash[75299]: cluster 2023-10-01T10:54:36.558043+0000 osd.6 (osd.6) 129 : cluster 3 failed to encode map e202 with expected crc 2023-10-01T10:54:38.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:37 smithi078 bash[75299]: cluster 2023-10-01T10:54:36.562862+0000 osd.5 (osd.5) 147 : cluster 3 failed to encode map e202 with expected crc 2023-10-01T10:54:38.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:37 smithi078 bash[75299]: cluster 2023-10-01T10:54:36.566398+0000 osd.5 (osd.5) 148 : cluster 3 failed to encode map e202 with expected crc 2023-10-01T10:54:38.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:37 smithi078 bash[75299]: cluster 2023-10-01T10:54:36.572629+0000 osd.2 (osd.2) 3 : cluster 0 11.14 starting backfill to osd.4 from (0'0,0'0] MAX to 199'10690 2023-10-01T10:54:38.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:37 smithi078 bash[75299]: cluster 2023-10-01T10:54:37.544123+0000 mon.a (mon.0) 1375 : cluster 0 osdmap e203: 8 total, 8 up, 8 in 2023-10-01T10:54:38.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:37 smithi078 bash[78274]: cluster 2023-10-01T10:54:36.028251+0000 mgr.y (mgr.25072) 1112 : cluster 0 pgmap v1082: 99 pgs: 1 active+remapped+backfilling, 1 active+clean+scrubbing, 1 active+remapped+backfill_wait, 96 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 697 op/s; 2579/114201 objects misplaced (2.258%); 11 MiB/s, 170 objects/s recovering 2023-10-01T10:54:38.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:37 smithi078 bash[78274]: cluster 2023-10-01T10:54:36.554764+0000 osd.2 (osd.2) 2 : cluster 0 11.14 starting backfill to osd.1 from (0'0,0'0] MAX to 199'10690 2023-10-01T10:54:38.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:37 smithi078 bash[78274]: cluster 2023-10-01T10:54:36.556734+0000 osd.7 (osd.7) 162 : cluster 3 failed to encode map e202 with expected crc 2023-10-01T10:54:38.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:37 smithi078 bash[78274]: cluster 2023-10-01T10:54:36.558043+0000 osd.6 (osd.6) 129 : cluster 3 failed to encode map e202 with expected crc 2023-10-01T10:54:38.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:37 smithi078 bash[78274]: cluster 2023-10-01T10:54:36.562862+0000 osd.5 (osd.5) 147 : cluster 3 failed to encode map e202 with expected crc 2023-10-01T10:54:38.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:37 smithi078 bash[78274]: cluster 2023-10-01T10:54:36.566398+0000 osd.5 (osd.5) 148 : cluster 3 failed to encode map e202 with expected crc 2023-10-01T10:54:38.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:37 smithi078 bash[78274]: cluster 2023-10-01T10:54:36.572629+0000 osd.2 (osd.2) 3 : cluster 0 11.14 starting backfill to osd.4 from (0'0,0'0] MAX to 199'10690 2023-10-01T10:54:38.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:37 smithi078 bash[78274]: cluster 2023-10-01T10:54:37.544123+0000 mon.a (mon.0) 1375 : cluster 0 osdmap e203: 8 total, 8 up, 8 in 2023-10-01T10:54:38.131 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:38.155 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:38.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:38.159 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:54:38.260 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:38.281 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:38.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:38.283 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:54:38.384 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:38.414 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:38.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:38.417 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:54:38.518 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:38.539 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:38.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:38.541 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:54:38.642 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:38.667 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:38.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:38.670 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:38.709 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:38 smithi173 bash[58816]: cluster 2023-10-01T10:54:37.542067+0000 osd.5 (osd.5) 149 : cluster 3 failed to encode map e203 with expected crc 2023-10-01T10:54:38.709 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:38 smithi173 bash[58816]: cluster 2023-10-01T10:54:37.544809+0000 osd.7 (osd.7) 163 : cluster 3 failed to encode map e203 with expected crc 2023-10-01T10:54:38.709 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:38 smithi173 bash[58816]: cluster 2023-10-01T10:54:37.790887+0000 osd.6 (osd.6) 130 : cluster 0 11.3 scrub starts 2023-10-01T10:54:38.709 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:38 smithi173 bash[58816]: audit 2023-10-01T10:54:37.978485+0000 mon.a (mon.0) 1376 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:54:38.709 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:38 smithi173 bash[58816]: audit 2023-10-01T10:54:38.029828+0000 mon.a (mon.0) 1377 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "23"}]: dispatch 2023-10-01T10:54:38.709 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:38 smithi173 bash[58816]: audit 2023-10-01T10:54:38.555798+0000 mon.a (mon.0) 1378 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "23"}]': finished 2023-10-01T10:54:38.709 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:38 smithi173 bash[58816]: cluster 2023-10-01T10:54:38.555912+0000 mon.a (mon.0) 1379 : cluster 0 osdmap e204: 8 total, 8 up, 8 in 2023-10-01T10:54:38.771 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:38.780 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:38 smithi078 bash[78274]: cluster 2023-10-01T10:54:37.542067+0000 osd.5 (osd.5) 149 : cluster 3 failed to encode map e203 with expected crc 2023-10-01T10:54:38.780 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:38 smithi078 bash[78274]: cluster 2023-10-01T10:54:37.544809+0000 osd.7 (osd.7) 163 : cluster 3 failed to encode map e203 with expected crc 2023-10-01T10:54:38.780 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:38 smithi078 bash[78274]: cluster 2023-10-01T10:54:37.790887+0000 osd.6 (osd.6) 130 : cluster 0 11.3 scrub starts 2023-10-01T10:54:38.780 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:38 smithi078 bash[78274]: audit 2023-10-01T10:54:37.978485+0000 mon.a (mon.0) 1376 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:54:38.780 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:38 smithi078 bash[78274]: audit 2023-10-01T10:54:38.029828+0000 mon.a (mon.0) 1377 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "23"}]: dispatch 2023-10-01T10:54:38.780 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:38 smithi078 bash[78274]: audit 2023-10-01T10:54:38.555798+0000 mon.a (mon.0) 1378 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "23"}]': finished 2023-10-01T10:54:38.780 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:38 smithi078 bash[78274]: cluster 2023-10-01T10:54:38.555912+0000 mon.a (mon.0) 1379 : cluster 0 osdmap e204: 8 total, 8 up, 8 in 2023-10-01T10:54:38.781 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:38 smithi078 bash[75299]: cluster 2023-10-01T10:54:37.542067+0000 osd.5 (osd.5) 149 : cluster 3 failed to encode map e203 with expected crc 2023-10-01T10:54:38.781 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:38 smithi078 bash[75299]: cluster 2023-10-01T10:54:37.544809+0000 osd.7 (osd.7) 163 : cluster 3 failed to encode map e203 with expected crc 2023-10-01T10:54:38.781 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:38 smithi078 bash[75299]: cluster 2023-10-01T10:54:37.790887+0000 osd.6 (osd.6) 130 : cluster 0 11.3 scrub starts 2023-10-01T10:54:38.781 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:38 smithi078 bash[75299]: audit 2023-10-01T10:54:37.978485+0000 mon.a (mon.0) 1376 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:54:38.781 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:38 smithi078 bash[75299]: audit 2023-10-01T10:54:38.029828+0000 mon.a (mon.0) 1377 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "23"}]: dispatch 2023-10-01T10:54:38.781 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:38 smithi078 bash[75299]: audit 2023-10-01T10:54:38.555798+0000 mon.a (mon.0) 1378 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "23"}]': finished 2023-10-01T10:54:38.781 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:38 smithi078 bash[75299]: cluster 2023-10-01T10:54:38.555912+0000 mon.a (mon.0) 1379 : cluster 0 osdmap e204: 8 total, 8 up, 8 in 2023-10-01T10:54:38.795 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:38.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:38.799 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:38.900 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:38.924 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:38.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:38.927 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:54:39.028 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:39.058 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:39.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:39.061 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:54:39.162 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:39.198 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:39.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:39.200 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:54:39.301 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:39.328 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:39.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:39.331 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:54:39.431 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:39.455 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:39.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:39.457 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:54:39.557 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:39.585 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:39.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:39.588 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:54:39.689 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:39.715 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:39.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:39.717 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:54:39.818 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:39.830 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:39 smithi173 bash[58816]: cluster 2023-10-01T10:54:37.895301+0000 osd.3 (osd.3) 15 : cluster 0 11.19 scrub starts 2023-10-01T10:54:39.830 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:39 smithi173 bash[58816]: cluster 2023-10-01T10:54:38.028684+0000 mgr.y (mgr.25072) 1113 : cluster 0 pgmap v1085: 99 pgs: 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 96 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 5444/112575 objects misplaced (4.836%); 14 MiB/s, 230 objects/s recovering 2023-10-01T10:54:39.830 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:39 smithi173 bash[58816]: cluster 2023-10-01T10:54:38.557544+0000 osd.5 (osd.5) 150 : cluster 3 failed to encode map e204 with expected crc 2023-10-01T10:54:39.831 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:39 smithi173 bash[58816]: cluster 2023-10-01T10:54:38.561677+0000 osd.7 (osd.7) 164 : cluster 3 failed to encode map e204 with expected crc 2023-10-01T10:54:39.831 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:39 smithi173 bash[58816]: cluster 2023-10-01T10:54:38.561707+0000 osd.5 (osd.5) 151 : cluster 0 11.15 starting backfill to osd.1 from (0'0,0'0] MAX to 201'10796 2023-10-01T10:54:39.831 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:39 smithi173 bash[58816]: cluster 2023-10-01T10:54:38.575982+0000 osd.5 (osd.5) 152 : cluster 0 11.15 starting backfill to osd.2 from (0'0,0'0] MAX to 201'10796 2023-10-01T10:54:39.831 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:39 smithi173 bash[58816]: cluster 2023-10-01T10:54:38.585479+0000 osd.5 (osd.5) 153 : cluster 0 11.15 starting backfill to osd.4 from (0'0,0'0] MAX to 201'10796 2023-10-01T10:54:39.831 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:39 smithi173 bash[58816]: cluster 2023-10-01T10:54:39.557957+0000 mon.a (mon.0) 1380 : cluster 0 osdmap e205: 8 total, 8 up, 8 in 2023-10-01T10:54:39.848 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:39.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:39.851 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:54:39.952 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:39.976 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:39.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:39.979 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:54:40.080 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:40.112 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:40.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:40.115 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:54:40.166 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:39 smithi078 bash[78274]: cluster 2023-10-01T10:54:37.895301+0000 osd.3 (osd.3) 15 : cluster 0 11.19 scrub starts 2023-10-01T10:54:40.166 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:39 smithi078 bash[78274]: cluster 2023-10-01T10:54:38.028684+0000 mgr.y (mgr.25072) 1113 : cluster 0 pgmap v1085: 99 pgs: 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 96 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 5444/112575 objects misplaced (4.836%); 14 MiB/s, 230 objects/s recovering 2023-10-01T10:54:40.166 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:39 smithi078 bash[78274]: cluster 2023-10-01T10:54:38.557544+0000 osd.5 (osd.5) 150 : cluster 3 failed to encode map e204 with expected crc 2023-10-01T10:54:40.166 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:39 smithi078 bash[78274]: cluster 2023-10-01T10:54:38.561677+0000 osd.7 (osd.7) 164 : cluster 3 failed to encode map e204 with expected crc 2023-10-01T10:54:40.166 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:39 smithi078 bash[78274]: cluster 2023-10-01T10:54:38.561707+0000 osd.5 (osd.5) 151 : cluster 0 11.15 starting backfill to osd.1 from (0'0,0'0] MAX to 201'10796 2023-10-01T10:54:40.166 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:39 smithi078 bash[78274]: cluster 2023-10-01T10:54:38.575982+0000 osd.5 (osd.5) 152 : cluster 0 11.15 starting backfill to osd.2 from (0'0,0'0] MAX to 201'10796 2023-10-01T10:54:40.166 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:39 smithi078 bash[78274]: cluster 2023-10-01T10:54:38.585479+0000 osd.5 (osd.5) 153 : cluster 0 11.15 starting backfill to osd.4 from (0'0,0'0] MAX to 201'10796 2023-10-01T10:54:40.167 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:39 smithi078 bash[78274]: cluster 2023-10-01T10:54:39.557957+0000 mon.a (mon.0) 1380 : cluster 0 osdmap e205: 8 total, 8 up, 8 in 2023-10-01T10:54:40.167 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:39 smithi078 bash[75299]: cluster 2023-10-01T10:54:37.895301+0000 osd.3 (osd.3) 15 : cluster 0 11.19 scrub starts 2023-10-01T10:54:40.167 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:39 smithi078 bash[75299]: cluster 2023-10-01T10:54:38.028684+0000 mgr.y (mgr.25072) 1113 : cluster 0 pgmap v1085: 99 pgs: 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 96 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 5444/112575 objects misplaced (4.836%); 14 MiB/s, 230 objects/s recovering 2023-10-01T10:54:40.167 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:39 smithi078 bash[75299]: cluster 2023-10-01T10:54:38.557544+0000 osd.5 (osd.5) 150 : cluster 3 failed to encode map e204 with expected crc 2023-10-01T10:54:40.167 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:39 smithi078 bash[75299]: cluster 2023-10-01T10:54:38.561677+0000 osd.7 (osd.7) 164 : cluster 3 failed to encode map e204 with expected crc 2023-10-01T10:54:40.167 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:39 smithi078 bash[75299]: cluster 2023-10-01T10:54:38.561707+0000 osd.5 (osd.5) 151 : cluster 0 11.15 starting backfill to osd.1 from (0'0,0'0] MAX to 201'10796 2023-10-01T10:54:40.167 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:39 smithi078 bash[75299]: cluster 2023-10-01T10:54:38.575982+0000 osd.5 (osd.5) 152 : cluster 0 11.15 starting backfill to osd.2 from (0'0,0'0] MAX to 201'10796 2023-10-01T10:54:40.167 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:39 smithi078 bash[75299]: cluster 2023-10-01T10:54:38.585479+0000 osd.5 (osd.5) 153 : cluster 0 11.15 starting backfill to osd.4 from (0'0,0'0] MAX to 201'10796 2023-10-01T10:54:40.168 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:39 smithi078 bash[75299]: cluster 2023-10-01T10:54:39.557957+0000 mon.a (mon.0) 1380 : cluster 0 osdmap e205: 8 total, 8 up, 8 in 2023-10-01T10:54:40.216 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:40.239 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:40.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:40.242 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:40.343 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:40.373 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:40.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:40.376 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:54:40.477 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:40.506 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:40.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:40.509 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:54:40.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:54:40 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:54:40] "GET /metrics HTTP/1.1" 200 36335 "" "Prometheus/2.33.4" 2023-10-01T10:54:40.609 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:40.636 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:40.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:40.639 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:54:40.739 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:40.766 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:40.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:40.773 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:40.874 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:40.894 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:40.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:40.896 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:54:40.903 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:40 smithi078 bash[75299]: cluster 2023-10-01T10:54:39.556893+0000 osd.7 (osd.7) 165 : cluster 3 failed to encode map e205 with expected crc 2023-10-01T10:54:40.904 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:40 smithi078 bash[75299]: cluster 2023-10-01T10:54:39.557018+0000 osd.5 (osd.5) 154 : cluster 3 failed to encode map e205 with expected crc 2023-10-01T10:54:40.904 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:40 smithi078 bash[75299]: cluster 2023-10-01T10:54:39.559371+0000 osd.5 (osd.5) 155 : cluster 3 failed to encode map e205 with expected crc 2023-10-01T10:54:40.904 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:40 smithi078 bash[75299]: cluster 2023-10-01T10:54:39.757149+0000 osd.6 (osd.6) 131 : cluster 3 failed to encode map e205 with expected crc 2023-10-01T10:54:40.904 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:40 smithi078 bash[75299]: cluster 2023-10-01T10:54:40.557833+0000 mon.a (mon.0) 1381 : cluster 0 osdmap e206: 8 total, 8 up, 8 in 2023-10-01T10:54:40.904 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:40 smithi078 bash[78274]: cluster 2023-10-01T10:54:39.556893+0000 osd.7 (osd.7) 165 : cluster 3 failed to encode map e205 with expected crc 2023-10-01T10:54:40.904 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:40 smithi078 bash[78274]: cluster 2023-10-01T10:54:39.557018+0000 osd.5 (osd.5) 154 : cluster 3 failed to encode map e205 with expected crc 2023-10-01T10:54:40.904 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:40 smithi078 bash[78274]: cluster 2023-10-01T10:54:39.559371+0000 osd.5 (osd.5) 155 : cluster 3 failed to encode map e205 with expected crc 2023-10-01T10:54:40.904 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:40 smithi078 bash[78274]: cluster 2023-10-01T10:54:39.757149+0000 osd.6 (osd.6) 131 : cluster 3 failed to encode map e205 with expected crc 2023-10-01T10:54:40.904 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:40 smithi078 bash[78274]: cluster 2023-10-01T10:54:40.557833+0000 mon.a (mon.0) 1381 : cluster 0 osdmap e206: 8 total, 8 up, 8 in 2023-10-01T10:54:40.997 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:41.026 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:41.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:41.030 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:54:41.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:40 smithi173 bash[58816]: cluster 2023-10-01T10:54:39.556893+0000 osd.7 (osd.7) 165 : cluster 3 failed to encode map e205 with expected crc 2023-10-01T10:54:41.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:40 smithi173 bash[58816]: cluster 2023-10-01T10:54:39.557018+0000 osd.5 (osd.5) 154 : cluster 3 failed to encode map e205 with expected crc 2023-10-01T10:54:41.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:40 smithi173 bash[58816]: cluster 2023-10-01T10:54:39.559371+0000 osd.5 (osd.5) 155 : cluster 3 failed to encode map e205 with expected crc 2023-10-01T10:54:41.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:40 smithi173 bash[58816]: cluster 2023-10-01T10:54:39.757149+0000 osd.6 (osd.6) 131 : cluster 3 failed to encode map e205 with expected crc 2023-10-01T10:54:41.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:40 smithi173 bash[58816]: cluster 2023-10-01T10:54:40.557833+0000 mon.a (mon.0) 1381 : cluster 0 osdmap e206: 8 total, 8 up, 8 in 2023-10-01T10:54:41.121 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-01T10:54:41.130 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:41.202 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:41.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:41.211 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:54:41.311 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:41.343 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:41.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:41.352 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:54:41.447 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T10:54:41.453 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:41.481 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:41.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:41.484 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:54:41.487 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T10:54:41.524 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T10:54:41.549 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T10:54:41.578 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T10:54:41.585 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:41.612 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T10:54:41.620 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:41.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:41.624 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:54:41.650 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T10:54:41.686 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T10:54:41.691 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T10:54:41.691 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T10:54:41.692 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T10:54:41.692 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T10:54:41.692 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T10:54:41.692 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T10:54:41.692 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T10:54:41.692 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T10:54:41.718 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:41 smithi078 bash[78274]: cluster 2023-10-01T10:54:40.029144+0000 mgr.y (mgr.25072) 1114 : cluster 0 pgmap v1088: 99 pgs: 1 remapped+peering, 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 95 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 5947/109983 objects misplaced (5.407%); 21 MiB/s, 343 objects/s recovering 2023-10-01T10:54:41.718 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:41 smithi078 bash[78274]: cluster 2023-10-01T10:54:40.556674+0000 osd.7 (osd.7) 166 : cluster 3 failed to encode map e206 with expected crc 2023-10-01T10:54:41.718 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:41 smithi078 bash[78274]: cluster 2023-10-01T10:54:40.561841+0000 osd.0 (osd.0) 8 : cluster 0 11.16 starting backfill to osd.3 from (0'0,0'0] MAX to 203'10707 2023-10-01T10:54:41.718 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:41 smithi078 bash[78274]: cluster 2023-10-01T10:54:40.571009+0000 osd.5 (osd.5) 156 : cluster 3 failed to encode map e206 with expected crc 2023-10-01T10:54:41.718 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:41 smithi078 bash[78274]: cluster 2023-10-01T10:54:40.572971+0000 osd.5 (osd.5) 157 : cluster 3 failed to encode map e206 with expected crc 2023-10-01T10:54:41.718 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:41 smithi078 bash[78274]: cluster 2023-10-01T10:54:40.573217+0000 osd.0 (osd.0) 9 : cluster 0 11.16 starting backfill to osd.5 from (0'0,0'0] MAX to 203'10707 2023-10-01T10:54:41.719 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:41 smithi078 bash[78274]: cluster 2023-10-01T10:54:40.585241+0000 osd.0 (osd.0) 10 : cluster 0 11.16 starting backfill to osd.7 from (0'0,0'0] MAX to 203'10707 2023-10-01T10:54:41.719 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:41 smithi078 bash[78274]: audit 2023-10-01T10:54:41.414700+0000 mon.c (mon.2) 181 : audit 0 from='client.? 172.21.15.78:0/2003152883' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:54:41.719 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:41 smithi078 bash[78274]: cluster 2023-10-01T10:54:41.561657+0000 mon.a (mon.0) 1382 : cluster 0 osdmap e207: 8 total, 8 up, 8 in 2023-10-01T10:54:41.725 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:41.754 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:41.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:41.758 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:54:41.858 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:41.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:41.886 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:41.886 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:54:41.987 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:41.996 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:41 smithi173 bash[58816]: cluster 2023-10-01T10:54:40.029144+0000 mgr.y (mgr.25072) 1114 : cluster 0 pgmap v1088: 99 pgs: 1 remapped+peering, 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 95 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 5947/109983 objects misplaced (5.407%); 21 MiB/s, 343 objects/s recovering 2023-10-01T10:54:41.996 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:41 smithi173 bash[58816]: cluster 2023-10-01T10:54:40.556674+0000 osd.7 (osd.7) 166 : cluster 3 failed to encode map e206 with expected crc 2023-10-01T10:54:41.996 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:41 smithi173 bash[58816]: cluster 2023-10-01T10:54:40.561841+0000 osd.0 (osd.0) 8 : cluster 0 11.16 starting backfill to osd.3 from (0'0,0'0] MAX to 203'10707 2023-10-01T10:54:41.996 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:41 smithi173 bash[58816]: cluster 2023-10-01T10:54:40.571009+0000 osd.5 (osd.5) 156 : cluster 3 failed to encode map e206 with expected crc 2023-10-01T10:54:41.996 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:41 smithi173 bash[58816]: cluster 2023-10-01T10:54:40.572971+0000 osd.5 (osd.5) 157 : cluster 3 failed to encode map e206 with expected crc 2023-10-01T10:54:41.997 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:41 smithi173 bash[58816]: cluster 2023-10-01T10:54:40.573217+0000 osd.0 (osd.0) 9 : cluster 0 11.16 starting backfill to osd.5 from (0'0,0'0] MAX to 203'10707 2023-10-01T10:54:41.997 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:41 smithi173 bash[58816]: cluster 2023-10-01T10:54:40.585241+0000 osd.0 (osd.0) 10 : cluster 0 11.16 starting backfill to osd.7 from (0'0,0'0] MAX to 203'10707 2023-10-01T10:54:41.997 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:41 smithi173 bash[58816]: audit 2023-10-01T10:54:41.414700+0000 mon.c (mon.2) 181 : audit 0 from='client.? 172.21.15.78:0/2003152883' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:54:41.997 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:41 smithi173 bash[58816]: cluster 2023-10-01T10:54:41.561657+0000 mon.a (mon.0) 1382 : cluster 0 osdmap e207: 8 total, 8 up, 8 in 2023-10-01T10:54:42.013 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:42.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:42.017 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:54:42.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:41 smithi078 bash[75299]: cluster 2023-10-01T10:54:40.029144+0000 mgr.y (mgr.25072) 1114 : cluster 0 pgmap v1088: 99 pgs: 1 remapped+peering, 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 95 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 5947/109983 objects misplaced (5.407%); 21 MiB/s, 343 objects/s recovering 2023-10-01T10:54:42.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:41 smithi078 bash[75299]: cluster 2023-10-01T10:54:40.556674+0000 osd.7 (osd.7) 166 : cluster 3 failed to encode map e206 with expected crc 2023-10-01T10:54:42.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:41 smithi078 bash[75299]: cluster 2023-10-01T10:54:40.561841+0000 osd.0 (osd.0) 8 : cluster 0 11.16 starting backfill to osd.3 from (0'0,0'0] MAX to 203'10707 2023-10-01T10:54:42.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:41 smithi078 bash[75299]: cluster 2023-10-01T10:54:40.571009+0000 osd.5 (osd.5) 156 : cluster 3 failed to encode map e206 with expected crc 2023-10-01T10:54:42.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:41 smithi078 bash[75299]: cluster 2023-10-01T10:54:40.572971+0000 osd.5 (osd.5) 157 : cluster 3 failed to encode map e206 with expected crc 2023-10-01T10:54:42.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:41 smithi078 bash[75299]: cluster 2023-10-01T10:54:40.573217+0000 osd.0 (osd.0) 9 : cluster 0 11.16 starting backfill to osd.5 from (0'0,0'0] MAX to 203'10707 2023-10-01T10:54:42.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:41 smithi078 bash[75299]: cluster 2023-10-01T10:54:40.585241+0000 osd.0 (osd.0) 10 : cluster 0 11.16 starting backfill to osd.7 from (0'0,0'0] MAX to 203'10707 2023-10-01T10:54:42.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:41 smithi078 bash[75299]: audit 2023-10-01T10:54:41.414700+0000 mon.c (mon.2) 181 : audit 0 from='client.? 172.21.15.78:0/2003152883' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:54:42.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:41 smithi078 bash[75299]: cluster 2023-10-01T10:54:41.561657+0000 mon.a (mon.0) 1382 : cluster 0 osdmap e207: 8 total, 8 up, 8 in 2023-10-01T10:54:42.118 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:42.148 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:42.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:42.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:54:42.255 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:42.281 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:42.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:42.283 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:54:42.384 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:42.411 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:42.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:42.414 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:54:42.515 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:42.539 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:42.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:42.541 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:54:42.642 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:42.678 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:42.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:42.683 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:54:42.784 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:42.810 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:42.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:42.815 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:54:42.916 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:42.944 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:42.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:42.948 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:54:43.049 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:43.058 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:42 smithi173 bash[58816]: cluster 2023-10-01T10:54:41.559800+0000 osd.7 (osd.7) 167 : cluster 3 failed to encode map e207 with expected crc 2023-10-01T10:54:43.058 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:42 smithi173 bash[58816]: cluster 2023-10-01T10:54:41.560021+0000 osd.5 (osd.5) 158 : cluster 3 failed to encode map e207 with expected crc 2023-10-01T10:54:43.058 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:42 smithi173 bash[58816]: cluster 2023-10-01T10:54:41.563271+0000 osd.5 (osd.5) 159 : cluster 3 failed to encode map e207 with expected crc 2023-10-01T10:54:43.058 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:42 smithi173 bash[58816]: cluster 2023-10-01T10:54:41.789660+0000 osd.6 (osd.6) 132 : cluster 0 11.3 scrub starts 2023-10-01T10:54:43.059 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:42 smithi173 bash[58816]: cluster 2023-10-01T10:54:42.561616+0000 mon.a (mon.0) 1383 : cluster 0 osdmap e208: 8 total, 8 up, 8 in 2023-10-01T10:54:43.073 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:43.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:43.076 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:54:43.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:42 smithi078 bash[75299]: cluster 2023-10-01T10:54:41.559800+0000 osd.7 (osd.7) 167 : cluster 3 failed to encode map e207 with expected crc 2023-10-01T10:54:43.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:42 smithi078 bash[75299]: cluster 2023-10-01T10:54:41.560021+0000 osd.5 (osd.5) 158 : cluster 3 failed to encode map e207 with expected crc 2023-10-01T10:54:43.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:42 smithi078 bash[75299]: cluster 2023-10-01T10:54:41.563271+0000 osd.5 (osd.5) 159 : cluster 3 failed to encode map e207 with expected crc 2023-10-01T10:54:43.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:42 smithi078 bash[75299]: cluster 2023-10-01T10:54:41.789660+0000 osd.6 (osd.6) 132 : cluster 0 11.3 scrub starts 2023-10-01T10:54:43.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:42 smithi078 bash[75299]: cluster 2023-10-01T10:54:42.561616+0000 mon.a (mon.0) 1383 : cluster 0 osdmap e208: 8 total, 8 up, 8 in 2023-10-01T10:54:43.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:42 smithi078 bash[78274]: cluster 2023-10-01T10:54:41.559800+0000 osd.7 (osd.7) 167 : cluster 3 failed to encode map e207 with expected crc 2023-10-01T10:54:43.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:42 smithi078 bash[78274]: cluster 2023-10-01T10:54:41.560021+0000 osd.5 (osd.5) 158 : cluster 3 failed to encode map e207 with expected crc 2023-10-01T10:54:43.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:42 smithi078 bash[78274]: cluster 2023-10-01T10:54:41.563271+0000 osd.5 (osd.5) 159 : cluster 3 failed to encode map e207 with expected crc 2023-10-01T10:54:43.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:42 smithi078 bash[78274]: cluster 2023-10-01T10:54:41.789660+0000 osd.6 (osd.6) 132 : cluster 0 11.3 scrub starts 2023-10-01T10:54:43.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:42 smithi078 bash[78274]: cluster 2023-10-01T10:54:42.561616+0000 mon.a (mon.0) 1383 : cluster 0 osdmap e208: 8 total, 8 up, 8 in 2023-10-01T10:54:43.176 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:43.211 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:43.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:43.213 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:54:43.314 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:43.349 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:43.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:43.352 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:54:43.452 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:43.480 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:43.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:43.482 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:54:43.583 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:43.642 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:43.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:43.645 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:54:43.745 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:43.778 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:43.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:43.785 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:43.885 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:43.894 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:43 smithi078 bash[75299]: cluster 2023-10-01T10:54:42.029626+0000 mgr.y (mgr.25072) 1115 : cluster 0 pgmap v1091: 99 pgs: 1 remapped+peering, 1 active+remapped+backfill_wait, 1 active+clean+remapped, 1 active+remapped+backfilling, 95 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 229 op/s; 3082/109824 objects misplaced (2.806%); 33 MiB/s, 528 objects/s recovering 2023-10-01T10:54:43.894 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:43 smithi078 bash[75299]: cluster 2023-10-01T10:54:42.559905+0000 osd.7 (osd.7) 168 : cluster 3 failed to encode map e208 with expected crc 2023-10-01T10:54:43.895 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:43 smithi078 bash[75299]: cluster 2023-10-01T10:54:42.560025+0000 osd.5 (osd.5) 160 : cluster 3 failed to encode map e208 with expected crc 2023-10-01T10:54:43.895 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:43 smithi078 bash[75299]: cluster 2023-10-01T10:54:42.562578+0000 osd.5 (osd.5) 161 : cluster 3 failed to encode map e208 with expected crc 2023-10-01T10:54:43.895 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:43 smithi078 bash[78274]: cluster 2023-10-01T10:54:42.029626+0000 mgr.y (mgr.25072) 1115 : cluster 0 pgmap v1091: 99 pgs: 1 remapped+peering, 1 active+remapped+backfill_wait, 1 active+clean+remapped, 1 active+remapped+backfilling, 95 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 229 op/s; 3082/109824 objects misplaced (2.806%); 33 MiB/s, 528 objects/s recovering 2023-10-01T10:54:43.895 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:43 smithi078 bash[78274]: cluster 2023-10-01T10:54:42.559905+0000 osd.7 (osd.7) 168 : cluster 3 failed to encode map e208 with expected crc 2023-10-01T10:54:43.895 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:43 smithi078 bash[78274]: cluster 2023-10-01T10:54:42.560025+0000 osd.5 (osd.5) 160 : cluster 3 failed to encode map e208 with expected crc 2023-10-01T10:54:43.895 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:43 smithi078 bash[78274]: cluster 2023-10-01T10:54:42.562578+0000 osd.5 (osd.5) 161 : cluster 3 failed to encode map e208 with expected crc 2023-10-01T10:54:43.923 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:43.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:43.928 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:54:44.029 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:44.065 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:44.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:44.068 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:54:44.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:43 smithi173 bash[58816]: cluster 2023-10-01T10:54:42.029626+0000 mgr.y (mgr.25072) 1115 : cluster 0 pgmap v1091: 99 pgs: 1 remapped+peering, 1 active+remapped+backfill_wait, 1 active+clean+remapped, 1 active+remapped+backfilling, 95 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 229 op/s; 3082/109824 objects misplaced (2.806%); 33 MiB/s, 528 objects/s recovering 2023-10-01T10:54:44.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:43 smithi173 bash[58816]: cluster 2023-10-01T10:54:42.559905+0000 osd.7 (osd.7) 168 : cluster 3 failed to encode map e208 with expected crc 2023-10-01T10:54:44.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:43 smithi173 bash[58816]: cluster 2023-10-01T10:54:42.560025+0000 osd.5 (osd.5) 160 : cluster 3 failed to encode map e208 with expected crc 2023-10-01T10:54:44.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:43 smithi173 bash[58816]: cluster 2023-10-01T10:54:42.562578+0000 osd.5 (osd.5) 161 : cluster 3 failed to encode map e208 with expected crc 2023-10-01T10:54:44.169 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:44.194 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:44.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:44.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:54:44.300 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:44.337 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:44.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:44.340 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:44.441 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:44.467 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:44.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:44.469 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:54:44.570 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:44.595 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:44.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:44.599 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:44.700 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:44.733 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:44.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:44.738 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:54:44.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:44 smithi078 bash[78274]: audit 2023-10-01T10:54:44.035418+0000 mon.a (mon.0) 1384 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "24"}]: dispatch 2023-10-01T10:54:44.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:44 smithi078 bash[78274]: audit 2023-10-01T10:54:44.524335+0000 mon.a (mon.0) 1385 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:54:44.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:44 smithi078 bash[78274]: audit 2023-10-01T10:54:44.533223+0000 mon.a (mon.0) 1386 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:54:44.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:44 smithi078 bash[75299]: audit 2023-10-01T10:54:44.035418+0000 mon.a (mon.0) 1384 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "24"}]: dispatch 2023-10-01T10:54:44.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:44 smithi078 bash[75299]: audit 2023-10-01T10:54:44.524335+0000 mon.a (mon.0) 1385 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:54:44.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:44 smithi078 bash[75299]: audit 2023-10-01T10:54:44.533223+0000 mon.a (mon.0) 1386 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:54:44.838 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:44.884 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:44.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:44.889 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:54:44.990 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:45.012 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:45.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:45.014 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:54:45.115 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:45.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:44 smithi173 bash[58816]: audit 2023-10-01T10:54:44.035418+0000 mon.a (mon.0) 1384 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "24"}]: dispatch 2023-10-01T10:54:45.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:44 smithi173 bash[58816]: audit 2023-10-01T10:54:44.524335+0000 mon.a (mon.0) 1385 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:54:45.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:44 smithi173 bash[58816]: audit 2023-10-01T10:54:44.533223+0000 mon.a (mon.0) 1386 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:54:45.136 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:45.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:45.142 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:45.243 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:45.277 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:45.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:45.279 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:54:45.380 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:45.402 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:45.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:45.405 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:54:45.506 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:45.531 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:45.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:45.539 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:54:45.639 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:45.667 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:45.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:45.672 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:54:45.760 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:45 smithi078 bash[75299]: cluster 2023-10-01T10:54:44.030270+0000 mgr.y (mgr.25072) 1116 : cluster 0 pgmap v1093: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 335 op/s; 4717/105477 objects misplaced (4.472%); 28 MiB/s, 441 objects/s recovering 2023-10-01T10:54:45.760 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:45 smithi078 bash[75299]: audit 2023-10-01T10:54:44.751807+0000 mon.a (mon.0) 1387 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "24"}]': finished 2023-10-01T10:54:45.760 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:45 smithi078 bash[75299]: cluster 2023-10-01T10:54:44.751874+0000 mon.a (mon.0) 1388 : cluster 0 osdmap e209: 8 total, 8 up, 8 in 2023-10-01T10:54:45.760 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:45 smithi078 bash[75299]: cluster 2023-10-01T10:54:44.752750+0000 osd.7 (osd.7) 169 : cluster 3 failed to encode map e209 with expected crc 2023-10-01T10:54:45.760 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:45 smithi078 bash[75299]: cluster 2023-10-01T10:54:44.754435+0000 osd.5 (osd.5) 162 : cluster 3 failed to encode map e209 with expected crc 2023-10-01T10:54:45.760 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:45 smithi078 bash[75299]: cluster 2023-10-01T10:54:44.754644+0000 osd.7 (osd.7) 170 : cluster 3 failed to encode map e209 with expected crc 2023-10-01T10:54:45.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:45 smithi078 bash[75299]: cluster 2023-10-01T10:54:44.756214+0000 osd.5 (osd.5) 163 : cluster 3 failed to encode map e209 with expected crc 2023-10-01T10:54:45.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:45 smithi078 bash[75299]: cluster 2023-10-01T10:54:44.757949+0000 osd.6 (osd.6) 133 : cluster 3 failed to encode map e209 with expected crc 2023-10-01T10:54:45.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:45 smithi078 bash[75299]: audit 2023-10-01T10:54:45.123955+0000 mon.a (mon.0) 1389 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:54:45.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:45 smithi078 bash[75299]: audit 2023-10-01T10:54:45.124917+0000 mon.a (mon.0) 1390 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:54:45.761 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:45 smithi078 bash[75299]: audit 2023-10-01T10:54:45.133312+0000 mon.a (mon.0) 1391 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:54:45.773 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:45.782 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:45 smithi078 bash[78274]: cluster 2023-10-01T10:54:44.030270+0000 mgr.y (mgr.25072) 1116 : cluster 0 pgmap v1093: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 335 op/s; 4717/105477 objects misplaced (4.472%); 28 MiB/s, 441 objects/s recovering 2023-10-01T10:54:45.782 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:45 smithi078 bash[78274]: audit 2023-10-01T10:54:44.751807+0000 mon.a (mon.0) 1387 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "24"}]': finished 2023-10-01T10:54:45.782 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:45 smithi078 bash[78274]: cluster 2023-10-01T10:54:44.751874+0000 mon.a (mon.0) 1388 : cluster 0 osdmap e209: 8 total, 8 up, 8 in 2023-10-01T10:54:45.782 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:45 smithi078 bash[78274]: cluster 2023-10-01T10:54:44.752750+0000 osd.7 (osd.7) 169 : cluster 3 failed to encode map e209 with expected crc 2023-10-01T10:54:45.782 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:45 smithi078 bash[78274]: cluster 2023-10-01T10:54:44.754435+0000 osd.5 (osd.5) 162 : cluster 3 failed to encode map e209 with expected crc 2023-10-01T10:54:45.782 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:45 smithi078 bash[78274]: cluster 2023-10-01T10:54:44.754644+0000 osd.7 (osd.7) 170 : cluster 3 failed to encode map e209 with expected crc 2023-10-01T10:54:45.782 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:45 smithi078 bash[78274]: cluster 2023-10-01T10:54:44.756214+0000 osd.5 (osd.5) 163 : cluster 3 failed to encode map e209 with expected crc 2023-10-01T10:54:45.783 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:45 smithi078 bash[78274]: cluster 2023-10-01T10:54:44.757949+0000 osd.6 (osd.6) 133 : cluster 3 failed to encode map e209 with expected crc 2023-10-01T10:54:45.783 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:45 smithi078 bash[78274]: audit 2023-10-01T10:54:45.123955+0000 mon.a (mon.0) 1389 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:54:45.783 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:45 smithi078 bash[78274]: audit 2023-10-01T10:54:45.124917+0000 mon.a (mon.0) 1390 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:54:45.783 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:45 smithi078 bash[78274]: audit 2023-10-01T10:54:45.133312+0000 mon.a (mon.0) 1391 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:54:45.803 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:45.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:45.806 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:45.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:45 smithi173 bash[58816]: cluster 2023-10-01T10:54:44.030270+0000 mgr.y (mgr.25072) 1116 : cluster 0 pgmap v1093: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 97 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 335 op/s; 4717/105477 objects misplaced (4.472%); 28 MiB/s, 441 objects/s recovering 2023-10-01T10:54:45.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:45 smithi173 bash[58816]: audit 2023-10-01T10:54:44.751807+0000 mon.a (mon.0) 1387 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "24"}]': finished 2023-10-01T10:54:45.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:45 smithi173 bash[58816]: cluster 2023-10-01T10:54:44.751874+0000 mon.a (mon.0) 1388 : cluster 0 osdmap e209: 8 total, 8 up, 8 in 2023-10-01T10:54:45.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:45 smithi173 bash[58816]: cluster 2023-10-01T10:54:44.752750+0000 osd.7 (osd.7) 169 : cluster 3 failed to encode map e209 with expected crc 2023-10-01T10:54:45.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:45 smithi173 bash[58816]: cluster 2023-10-01T10:54:44.754435+0000 osd.5 (osd.5) 162 : cluster 3 failed to encode map e209 with expected crc 2023-10-01T10:54:45.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:45 smithi173 bash[58816]: cluster 2023-10-01T10:54:44.754644+0000 osd.7 (osd.7) 170 : cluster 3 failed to encode map e209 with expected crc 2023-10-01T10:54:45.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:45 smithi173 bash[58816]: cluster 2023-10-01T10:54:44.756214+0000 osd.5 (osd.5) 163 : cluster 3 failed to encode map e209 with expected crc 2023-10-01T10:54:45.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:45 smithi173 bash[58816]: cluster 2023-10-01T10:54:44.757949+0000 osd.6 (osd.6) 133 : cluster 3 failed to encode map e209 with expected crc 2023-10-01T10:54:45.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:45 smithi173 bash[58816]: audit 2023-10-01T10:54:45.123955+0000 mon.a (mon.0) 1389 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:54:45.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:45 smithi173 bash[58816]: audit 2023-10-01T10:54:45.124917+0000 mon.a (mon.0) 1390 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:54:45.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:45 smithi173 bash[58816]: audit 2023-10-01T10:54:45.133312+0000 mon.a (mon.0) 1391 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:54:45.907 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:45.930 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:45.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:45.934 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:54:46.034 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:46.061 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:46.062 INFO:teuthology.orchestra.run.smithi078.stdout: 2023-10-01T10:54:46.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:46.068 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:54:46.073 INFO:teuthology.orchestra.run.smithi078.stderr:dumped all 2023-10-01T10:54:46.170 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:46.195 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:46.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:46.197 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:54:46.298 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:46.327 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:46.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:46.329 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:54:46.430 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:46.456 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:46.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:46.459 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:54:46.559 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:46.589 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:46.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:46.592 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:54:46.693 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:46.722 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:46.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:46.725 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:54:46.777 INFO:teuthology.orchestra.run.smithi078.stderr:noscrub is set 2023-10-01T10:54:46.825 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:46.848 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:46.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:46.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:46.951 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:46.979 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:46.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:46.982 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:54:47.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:46 smithi078 bash[75299]: cluster 2023-10-01T10:54:45.749784+0000 osd.5 (osd.5) 164 : cluster 3 failed to encode map e210 with expected crc 2023-10-01T10:54:47.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:46 smithi078 bash[75299]: cluster 2023-10-01T10:54:45.752761+0000 mon.a (mon.0) 1392 : cluster 0 osdmap e210: 8 total, 8 up, 8 in 2023-10-01T10:54:47.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:46 smithi078 bash[75299]: cluster 2023-10-01T10:54:45.753696+0000 osd.7 (osd.7) 171 : cluster 3 failed to encode map e210 with expected crc 2023-10-01T10:54:47.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:46 smithi078 bash[75299]: cluster 2023-10-01T10:54:45.753702+0000 osd.5 (osd.5) 165 : cluster 3 failed to encode map e210 with expected crc 2023-10-01T10:54:47.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:46 smithi078 bash[75299]: cluster 2023-10-01T10:54:45.755648+0000 osd.6 (osd.6) 134 : cluster 3 failed to encode map e210 with expected crc 2023-10-01T10:54:47.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:46 smithi078 bash[75299]: cluster 2023-10-01T10:54:45.755915+0000 osd.5 (osd.5) 166 : cluster 3 failed to encode map e210 with expected crc 2023-10-01T10:54:47.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:46 smithi078 bash[75299]: cluster 2023-10-01T10:54:45.758027+0000 osd.5 (osd.5) 167 : cluster 3 failed to encode map e210 with expected crc 2023-10-01T10:54:47.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:46 smithi078 bash[75299]: cluster 2023-10-01T10:54:45.758519+0000 osd.7 (osd.7) 172 : cluster 3 failed to encode map e210 with expected crc 2023-10-01T10:54:47.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:46 smithi078 bash[75299]: cluster 2023-10-01T10:54:45.781116+0000 osd.6 (osd.6) 135 : cluster 0 11.3 scrub starts 2023-10-01T10:54:47.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:46 smithi078 bash[75299]: audit 2023-10-01T10:54:46.068754+0000 mon.b (mon.1) 48 : audit 1 from='client.? 172.21.15.78:0/1488049397' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:54:47.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:46 smithi078 bash[75299]: audit 2023-10-01T10:54:46.069321+0000 mon.a (mon.0) 1393 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:54:47.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:46 smithi078 bash[78274]: cluster 2023-10-01T10:54:45.749784+0000 osd.5 (osd.5) 164 : cluster 3 failed to encode map e210 with expected crc 2023-10-01T10:54:47.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:46 smithi078 bash[78274]: cluster 2023-10-01T10:54:45.752761+0000 mon.a (mon.0) 1392 : cluster 0 osdmap e210: 8 total, 8 up, 8 in 2023-10-01T10:54:47.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:46 smithi078 bash[78274]: cluster 2023-10-01T10:54:45.753696+0000 osd.7 (osd.7) 171 : cluster 3 failed to encode map e210 with expected crc 2023-10-01T10:54:47.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:46 smithi078 bash[78274]: cluster 2023-10-01T10:54:45.753702+0000 osd.5 (osd.5) 165 : cluster 3 failed to encode map e210 with expected crc 2023-10-01T10:54:47.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:46 smithi078 bash[78274]: cluster 2023-10-01T10:54:45.755648+0000 osd.6 (osd.6) 134 : cluster 3 failed to encode map e210 with expected crc 2023-10-01T10:54:47.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:46 smithi078 bash[78274]: cluster 2023-10-01T10:54:45.755915+0000 osd.5 (osd.5) 166 : cluster 3 failed to encode map e210 with expected crc 2023-10-01T10:54:47.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:46 smithi078 bash[78274]: cluster 2023-10-01T10:54:45.758027+0000 osd.5 (osd.5) 167 : cluster 3 failed to encode map e210 with expected crc 2023-10-01T10:54:47.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:46 smithi078 bash[78274]: cluster 2023-10-01T10:54:45.758519+0000 osd.7 (osd.7) 172 : cluster 3 failed to encode map e210 with expected crc 2023-10-01T10:54:47.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:46 smithi078 bash[78274]: cluster 2023-10-01T10:54:45.781116+0000 osd.6 (osd.6) 135 : cluster 0 11.3 scrub starts 2023-10-01T10:54:47.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:46 smithi078 bash[78274]: audit 2023-10-01T10:54:46.068754+0000 mon.b (mon.1) 48 : audit 1 from='client.? 172.21.15.78:0/1488049397' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:54:47.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:46 smithi078 bash[78274]: audit 2023-10-01T10:54:46.069321+0000 mon.a (mon.0) 1393 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:54:47.083 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:47.088 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T10:54:47.092 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:46 smithi173 bash[58816]: cluster 2023-10-01T10:54:45.749784+0000 osd.5 (osd.5) 164 : cluster 3 failed to encode map e210 with expected crc 2023-10-01T10:54:47.092 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:46 smithi173 bash[58816]: cluster 2023-10-01T10:54:45.752761+0000 mon.a (mon.0) 1392 : cluster 0 osdmap e210: 8 total, 8 up, 8 in 2023-10-01T10:54:47.092 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:46 smithi173 bash[58816]: cluster 2023-10-01T10:54:45.753696+0000 osd.7 (osd.7) 171 : cluster 3 failed to encode map e210 with expected crc 2023-10-01T10:54:47.092 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:46 smithi173 bash[58816]: cluster 2023-10-01T10:54:45.753702+0000 osd.5 (osd.5) 165 : cluster 3 failed to encode map e210 with expected crc 2023-10-01T10:54:47.092 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:46 smithi173 bash[58816]: cluster 2023-10-01T10:54:45.755648+0000 osd.6 (osd.6) 134 : cluster 3 failed to encode map e210 with expected crc 2023-10-01T10:54:47.092 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:46 smithi173 bash[58816]: cluster 2023-10-01T10:54:45.755915+0000 osd.5 (osd.5) 166 : cluster 3 failed to encode map e210 with expected crc 2023-10-01T10:54:47.092 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:46 smithi173 bash[58816]: cluster 2023-10-01T10:54:45.758027+0000 osd.5 (osd.5) 167 : cluster 3 failed to encode map e210 with expected crc 2023-10-01T10:54:47.092 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:46 smithi173 bash[58816]: cluster 2023-10-01T10:54:45.758519+0000 osd.7 (osd.7) 172 : cluster 3 failed to encode map e210 with expected crc 2023-10-01T10:54:47.093 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:46 smithi173 bash[58816]: cluster 2023-10-01T10:54:45.781116+0000 osd.6 (osd.6) 135 : cluster 0 11.3 scrub starts 2023-10-01T10:54:47.093 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:46 smithi173 bash[58816]: audit 2023-10-01T10:54:46.068754+0000 mon.b (mon.1) 48 : audit 1 from='client.? 172.21.15.78:0/1488049397' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:54:47.093 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:46 smithi173 bash[58816]: audit 2023-10-01T10:54:46.069321+0000 mon.a (mon.0) 1393 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:54:47.123 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:47.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:47.125 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:54:47.226 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:47.256 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:47.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:47.258 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:54:47.359 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:47.393 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:47.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:47.397 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:54:47.498 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:47.528 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:47.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:47.531 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:54:47.632 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:47.670 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:47.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:47.673 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:54:47.766 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:47 smithi173 bash[58816]: cluster 2023-10-01T10:54:46.030799+0000 mgr.y (mgr.25072) 1117 : cluster 0 pgmap v1096: 99 pgs: 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 96 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 415 op/s; 6943/104250 objects misplaced (6.660%); 19 MiB/s, 300 objects/s recovering 2023-10-01T10:54:47.766 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:47 smithi173 bash[58816]: audit 2023-10-01T10:54:46.047697+0000 mgr.y (mgr.25072) 1118 : audit 0 from='client.36195 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:54:47.766 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:47 smithi173 bash[58816]: cluster 2023-10-01T10:54:46.757062+0000 mon.a (mon.0) 1394 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:54:47.766 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:47 smithi173 bash[58816]: cluster 2023-10-01T10:54:46.765976+0000 osd.5 (osd.5) 168 : cluster 3 failed to encode map e211 with expected crc 2023-10-01T10:54:47.767 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:47 smithi173 bash[58816]: cluster 2023-10-01T10:54:46.767477+0000 osd.1 (osd.1) 8 : cluster 0 11.17 starting backfill to osd.6 from (0'0,0'0] MAX to 208'10700 2023-10-01T10:54:47.767 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:47 smithi173 bash[58816]: audit 2023-10-01T10:54:46.768746+0000 mon.a (mon.0) 1395 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T10:54:47.767 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:47 smithi173 bash[58816]: cluster 2023-10-01T10:54:46.768829+0000 mon.a (mon.0) 1396 : cluster 0 osdmap e211: 8 total, 8 up, 8 in 2023-10-01T10:54:47.767 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:47 smithi173 bash[58816]: cluster 2023-10-01T10:54:46.770264+0000 osd.5 (osd.5) 169 : cluster 3 failed to encode map e211 with expected crc 2023-10-01T10:54:47.767 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:47 smithi173 bash[58816]: cluster 2023-10-01T10:54:46.785531+0000 osd.1 (osd.1) 9 : cluster 0 11.17 starting backfill to osd.7 from (0'0,0'0] MAX to 208'10700 2023-10-01T10:54:47.767 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:47 smithi173 bash[58816]: cluster 2023-10-01T10:54:46.818078+0000 osd.6 (osd.6) 136 : cluster 3 failed to encode map e211 with expected crc 2023-10-01T10:54:47.767 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:47 smithi173 bash[58816]: audit 2023-10-01T10:54:47.444957+0000 mon.a (mon.0) 1397 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:54:47.767 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:47 smithi173 bash[58816]: audit 2023-10-01T10:54:47.449003+0000 mon.a (mon.0) 1398 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:54:47.773 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:47.806 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:47.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:47.811 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:54:47.911 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:47.921 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:47 smithi078 bash[75299]: cluster 2023-10-01T10:54:46.030799+0000 mgr.y (mgr.25072) 1117 : cluster 0 pgmap v1096: 99 pgs: 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 96 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 415 op/s; 6943/104250 objects misplaced (6.660%); 19 MiB/s, 300 objects/s recovering 2023-10-01T10:54:47.921 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:47 smithi078 bash[75299]: audit 2023-10-01T10:54:46.047697+0000 mgr.y (mgr.25072) 1118 : audit 0 from='client.36195 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:54:47.921 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:47 smithi078 bash[75299]: cluster 2023-10-01T10:54:46.757062+0000 mon.a (mon.0) 1394 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:54:47.921 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:47 smithi078 bash[75299]: cluster 2023-10-01T10:54:46.765976+0000 osd.5 (osd.5) 168 : cluster 3 failed to encode map e211 with expected crc 2023-10-01T10:54:47.921 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:47 smithi078 bash[75299]: cluster 2023-10-01T10:54:46.767477+0000 osd.1 (osd.1) 8 : cluster 0 11.17 starting backfill to osd.6 from (0'0,0'0] MAX to 208'10700 2023-10-01T10:54:47.921 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:47 smithi078 bash[75299]: audit 2023-10-01T10:54:46.768746+0000 mon.a (mon.0) 1395 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T10:54:47.921 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:47 smithi078 bash[75299]: cluster 2023-10-01T10:54:46.768829+0000 mon.a (mon.0) 1396 : cluster 0 osdmap e211: 8 total, 8 up, 8 in 2023-10-01T10:54:47.921 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:47 smithi078 bash[75299]: cluster 2023-10-01T10:54:46.770264+0000 osd.5 (osd.5) 169 : cluster 3 failed to encode map e211 with expected crc 2023-10-01T10:54:47.922 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:47 smithi078 bash[75299]: cluster 2023-10-01T10:54:46.785531+0000 osd.1 (osd.1) 9 : cluster 0 11.17 starting backfill to osd.7 from (0'0,0'0] MAX to 208'10700 2023-10-01T10:54:47.922 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:47 smithi078 bash[75299]: cluster 2023-10-01T10:54:46.818078+0000 osd.6 (osd.6) 136 : cluster 3 failed to encode map e211 with expected crc 2023-10-01T10:54:47.922 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:47 smithi078 bash[75299]: audit 2023-10-01T10:54:47.444957+0000 mon.a (mon.0) 1397 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:54:47.922 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:47 smithi078 bash[75299]: audit 2023-10-01T10:54:47.449003+0000 mon.a (mon.0) 1398 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:54:47.922 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:47 smithi078 bash[78274]: cluster 2023-10-01T10:54:46.030799+0000 mgr.y (mgr.25072) 1117 : cluster 0 pgmap v1096: 99 pgs: 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 96 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 415 op/s; 6943/104250 objects misplaced (6.660%); 19 MiB/s, 300 objects/s recovering 2023-10-01T10:54:47.922 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:47 smithi078 bash[78274]: audit 2023-10-01T10:54:46.047697+0000 mgr.y (mgr.25072) 1118 : audit 0 from='client.36195 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:54:47.922 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:47 smithi078 bash[78274]: cluster 2023-10-01T10:54:46.757062+0000 mon.a (mon.0) 1394 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:54:47.922 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:47 smithi078 bash[78274]: cluster 2023-10-01T10:54:46.765976+0000 osd.5 (osd.5) 168 : cluster 3 failed to encode map e211 with expected crc 2023-10-01T10:54:47.922 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:47 smithi078 bash[78274]: cluster 2023-10-01T10:54:46.767477+0000 osd.1 (osd.1) 8 : cluster 0 11.17 starting backfill to osd.6 from (0'0,0'0] MAX to 208'10700 2023-10-01T10:54:47.923 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:47 smithi078 bash[78274]: audit 2023-10-01T10:54:46.768746+0000 mon.a (mon.0) 1395 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T10:54:47.923 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:47 smithi078 bash[78274]: cluster 2023-10-01T10:54:46.768829+0000 mon.a (mon.0) 1396 : cluster 0 osdmap e211: 8 total, 8 up, 8 in 2023-10-01T10:54:47.923 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:47 smithi078 bash[78274]: cluster 2023-10-01T10:54:46.770264+0000 osd.5 (osd.5) 169 : cluster 3 failed to encode map e211 with expected crc 2023-10-01T10:54:47.923 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:47 smithi078 bash[78274]: cluster 2023-10-01T10:54:46.785531+0000 osd.1 (osd.1) 9 : cluster 0 11.17 starting backfill to osd.7 from (0'0,0'0] MAX to 208'10700 2023-10-01T10:54:47.923 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:47 smithi078 bash[78274]: cluster 2023-10-01T10:54:46.818078+0000 osd.6 (osd.6) 136 : cluster 3 failed to encode map e211 with expected crc 2023-10-01T10:54:47.923 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:47 smithi078 bash[78274]: audit 2023-10-01T10:54:47.444957+0000 mon.a (mon.0) 1397 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:54:47.923 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:47 smithi078 bash[78274]: audit 2023-10-01T10:54:47.449003+0000 mon.a (mon.0) 1398 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:54:47.936 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:47.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:47.941 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:54:48.041 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:48.069 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:48.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:48.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:54:48.172 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:48.206 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:48.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:48.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:54:48.310 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:48.334 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:48.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:48.337 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:54:48.438 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:48.466 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:48.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:48.473 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:54:48.574 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:48.606 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:48.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:48.611 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:54:48.712 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:48.738 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:48.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:48.741 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:54:48.841 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:48.862 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:48.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:48.865 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:48.965 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:48.989 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:48.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:48.993 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:54:49.094 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:49.125 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:49.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:49.130 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:54:49.231 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:49.279 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:49.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:49.281 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:54:49.318 INFO:teuthology.orchestra.run.smithi078.stdout:{"pg_ready":true,"pg_map":{"version":1096,"stamp":"2023-10-01T10:54:46.030527+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2276235192,"num_objects":34750,"num_object_clones":0,"num_object_copies":104250,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6943,"num_objects_unfound":0,"num_objects_dirty":34750,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":142717,"num_write_kb":5358991,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5412,"num_bytes_recovered":354615296,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":236992,"ondisk_log_size":236992,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":312,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":12551928,"kb_used_data":7410744,"kb_used_omap":30,"kb_used_meta":5140961,"kb_avail":737409288,"statfs":{"total":767960285184,"available":755107110912,"internally_reserved":0,"allocated":7588601856,"data_stored":7583076816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31502,"internal_metadata":5264344306},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2],"upper_bound":2},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-121765888,"num_objects":-1858,"num_object_clones":0,"num_object_copies":-5574,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3861,"num_objects_unfound":0,"num_objects_dirty":-1858,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1858,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1345,"num_bytes_recovered":88145920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.471671"},"pg_stats":[{"pgid":"11.12","version":"208'10851","reported_seq":13280,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-01T10:54:43.127766+0000","last_change":"2023-10-01T10:54:41.595171+0000","last_active":"2023-10-01T10:54:43.127766+0000","last_peered":"2023-10-01T10:54:43.127766+0000","last_clean":"2023-10-01T10:54:43.127766+0000","last_became_active":"2023-10-01T10:54:41.572806+0000","last_became_peered":"2023-10-01T10:54:41.572806+0000","last_unstale":"2023-10-01T10:54:43.127766+0000","last_undegraded":"2023-10-01T10:54:43.127766+0000","last_fullsized":"2023-10-01T10:54:43.127766+0000","mapping_epoch":206,"log_start":"178'400","ondisk_log_start":"178'400","created":189,"last_epoch_clean":207,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":10451,"log_dups_size":0,"ondisk_log_size":10451,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73760768,"num_objects":1126,"num_object_clones":0,"num_object_copies":3378,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1126,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5642,"num_write_kb":216544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1174,"num_bytes_recovered":76873728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"60'1","reported_seq":200,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:41.559415+0000","last_change":"2023-10-01T10:38:16.032258+0000","last_active":"2023-10-01T10:54:41.559415+0000","last_peered":"2023-10-01T10:54:41.559415+0000","last_clean":"2023-10-01T10:54:41.559415+0000","last_became_active":"2023-10-01T10:38:16.031633+0000","last_became_peered":"2023-10-01T10:38:16.031633+0000","last_unstale":"2023-10-01T10:54:41.559415+0000","last_undegraded":"2023-10-01T10:54:41.559415+0000","last_fullsized":"2023-10-01T10:54:41.559415+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:17:02.446430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":187,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-01T10:54:40.557631+0000","last_change":"2023-10-01T10:38:16.025881+0000","last_active":"2023-10-01T10:54:40.557631+0000","last_peered":"2023-10-01T10:54:40.557631+0000","last_clean":"2023-10-01T10:54:40.557631+0000","last_became_active":"2023-10-01T10:38:16.025518+0000","last_became_peered":"2023-10-01T10:38:16.025518+0000","last_unstale":"2023-10-01T10:54:40.557631+0000","last_undegraded":"2023-10-01T10:54:40.557631+0000","last_fullsized":"2023-10-01T10:54:40.557631+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:48:37.826604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.11","version":"208'10842","reported_seq":14271,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-01T10:54:43.191889+0000","last_change":"2023-10-01T10:54:37.578685+0000","last_active":"2023-10-01T10:54:43.191889+0000","last_peered":"2023-10-01T10:54:43.191889+0000","last_clean":"2023-10-01T10:54:43.191889+0000","last_became_active":"2023-10-01T10:54:37.555911+0000","last_became_peered":"2023-10-01T10:54:37.555911+0000","last_unstale":"2023-10-01T10:54:43.191889+0000","last_undegraded":"2023-10-01T10:54:43.191889+0000","last_fullsized":"2023-10-01T10:54:43.191889+0000","mapping_epoch":202,"log_start":"179'2500","ondisk_log_start":"179'2500","created":189,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":8342,"log_dups_size":0,"ondisk_log_size":8342,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67928064,"num_objects":1037,"num_object_clones":0,"num_object_copies":3111,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1037,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5645,"num_write_kb":213792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1119,"num_bytes_recovered":73334784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":107,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-01T10:54:40.557337+0000","last_change":"2023-10-01T10:52:18.956544+0000","last_active":"2023-10-01T10:54:40.557337+0000","last_peered":"2023-10-01T10:54:40.557337+0000","last_clean":"2023-10-01T10:54:40.557337+0000","last_became_active":"2023-10-01T10:52:18.956217+0000","last_became_peered":"2023-10-01T10:52:18.956217+0000","last_unstale":"2023-10-01T10:54:40.557337+0000","last_undegraded":"2023-10-01T10:54:40.557337+0000","last_fullsized":"2023-10-01T10:54:40.557337+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:34:01.952235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":172,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:41.570803+0000","last_change":"2023-10-01T10:37:42.772008+0000","last_active":"2023-10-01T10:54:41.570803+0000","last_peered":"2023-10-01T10:54:41.570803+0000","last_clean":"2023-10-01T10:54:41.570803+0000","last_became_active":"2023-10-01T10:37:42.771244+0000","last_became_peered":"2023-10-01T10:37:42.771244+0000","last_unstale":"2023-10-01T10:54:41.570803+0000","last_undegraded":"2023-10-01T10:54:41.570803+0000","last_fullsized":"2023-10-01T10:54:41.570803+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:38:55.180790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.10","version":"208'10911","reported_seq":16047,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-01T10:54:44.237423+0000","last_change":"2023-10-01T10:54:33.589236+0000","last_active":"2023-10-01T10:54:44.237423+0000","last_peered":"2023-10-01T10:54:44.237423+0000","last_clean":"2023-10-01T10:54:44.237423+0000","last_became_active":"2023-10-01T10:54:33.560657+0000","last_became_peered":"2023-10-01T10:54:33.560657+0000","last_unstale":"2023-10-01T10:54:44.237423+0000","last_undegraded":"2023-10-01T10:54:44.237423+0000","last_fullsized":"2023-10-01T10:54:44.237423+0000","mapping_epoch":198,"log_start":"179'2500","ondisk_log_start":"179'2500","created":189,"last_epoch_clean":199,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":8411,"log_dups_size":0,"ondisk_log_size":8411,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68419584,"num_objects":1044,"num_object_clones":0,"num_object_copies":3132,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1044,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5689,"num_write_kb":215456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1264,"num_bytes_recovered":82804736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":189,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-01T10:54:40.556973+0000","last_change":"2023-10-01T10:37:42.766600+0000","last_active":"2023-10-01T10:54:40.556973+0000","last_peered":"2023-10-01T10:54:40.556973+0000","last_clean":"2023-10-01T10:54:40.556973+0000","last_became_active":"2023-10-01T10:37:42.766051+0000","last_became_peered":"2023-10-01T10:37:42.766051+0000","last_unstale":"2023-10-01T10:54:40.556973+0000","last_undegraded":"2023-10-01T10:54:40.556973+0000","last_fullsized":"2023-10-01T10:54:40.556973+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:51:17.707963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":173,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:41.570722+0000","last_change":"2023-10-01T10:32:31.575109+0000","last_active":"2023-10-01T10:54:41.570722+0000","last_peered":"2023-10-01T10:54:41.570722+0000","last_clean":"2023-10-01T10:54:41.570722+0000","last_became_active":"2023-10-01T10:32:31.574729+0000","last_became_peered":"2023-10-01T10:32:31.574729+0000","last_unstale":"2023-10-01T10:54:41.570722+0000","last_undegraded":"2023-10-01T10:54:41.570722+0000","last_fullsized":"2023-10-01T10:54:41.570722+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":168,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:41.569955+0000","last_change":"2023-10-01T10:37:42.766667+0000","last_active":"2023-10-01T10:54:41.569955+0000","last_peered":"2023-10-01T10:54:41.569955+0000","last_clean":"2023-10-01T10:54:41.569955+0000","last_became_active":"2023-10-01T10:37:42.766369+0000","last_became_peered":"2023-10-01T10:37:42.766369+0000","last_unstale":"2023-10-01T10:54:41.569955+0000","last_undegraded":"2023-10-01T10:54:41.569955+0000","last_fullsized":"2023-10-01T10:54:41.569955+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:02.708788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":173,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-01T10:54:40.557145+0000","last_change":"2023-10-01T10:37:42.765925+0000","last_active":"2023-10-01T10:54:40.557145+0000","last_peered":"2023-10-01T10:54:40.557145+0000","last_clean":"2023-10-01T10:54:40.557145+0000","last_became_active":"2023-10-01T10:37:42.765218+0000","last_became_peered":"2023-10-01T10:37:42.765218+0000","last_unstale":"2023-10-01T10:54:40.557145+0000","last_undegraded":"2023-10-01T10:54:40.557145+0000","last_fullsized":"2023-10-01T10:54:40.557145+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T22:10:34.338031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":175,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:41.568146+0000","last_change":"2023-10-01T10:32:31.579805+0000","last_active":"2023-10-01T10:54:41.568146+0000","last_peered":"2023-10-01T10:54:41.568146+0000","last_clean":"2023-10-01T10:54:41.568146+0000","last_became_active":"2023-10-01T10:32:31.579423+0000","last_became_peered":"2023-10-01T10:32:31.579423+0000","last_unstale":"2023-10-01T10:54:41.568146+0000","last_undegraded":"2023-10-01T10:54:41.568146+0000","last_fullsized":"2023-10-01T10:54:41.568146+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":157,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-01T10:54:40.556898+0000","last_change":"2023-10-01T10:37:42.772531+0000","last_active":"2023-10-01T10:54:40.556898+0000","last_peered":"2023-10-01T10:54:40.556898+0000","last_clean":"2023-10-01T10:54:40.556898+0000","last_became_active":"2023-10-01T10:37:42.772216+0000","last_became_peered":"2023-10-01T10:37:42.772216+0000","last_unstale":"2023-10-01T10:54:40.556898+0000","last_undegraded":"2023-10-01T10:54:40.556898+0000","last_fullsized":"2023-10-01T10:54:40.556898+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:01:53.373628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":46,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-01T10:54:40.557494+0000","last_change":"2023-10-01T10:52:43.661650+0000","last_active":"2023-10-01T10:54:40.557494+0000","last_peered":"2023-10-01T10:54:40.557494+0000","last_clean":"2023-10-01T10:54:40.557494+0000","last_became_active":"2023-10-01T10:52:43.661202+0000","last_became_peered":"2023-10-01T10:52:43.661202+0000","last_unstale":"2023-10-01T10:54:40.557494+0000","last_undegraded":"2023-10-01T10:54:40.557494+0000","last_fullsized":"2023-10-01T10:54:40.557494+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:28:35.479124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":82,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:41.570231+0000","last_change":"2023-10-01T10:52:18.966695+0000","last_active":"2023-10-01T10:54:41.570231+0000","last_peered":"2023-10-01T10:54:41.570231+0000","last_clean":"2023-10-01T10:54:41.570231+0000","last_became_active":"2023-10-01T10:52:18.966450+0000","last_became_peered":"2023-10-01T10:52:18.966450+0000","last_unstale":"2023-10-01T10:54:41.570231+0000","last_undegraded":"2023-10-01T10:54:41.570231+0000","last_fullsized":"2023-10-01T10:54:41.570231+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":195,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:41.560285+0000","last_change":"2023-10-01T10:38:16.030826+0000","last_active":"2023-10-01T10:54:41.560285+0000","last_peered":"2023-10-01T10:54:41.560285+0000","last_clean":"2023-10-01T10:54:41.560285+0000","last_became_active":"2023-10-01T10:38:16.030277+0000","last_became_peered":"2023-10-01T10:38:16.030277+0000","last_unstale":"2023-10-01T10:54:41.560285+0000","last_undegraded":"2023-10-01T10:54:41.560285+0000","last_fullsized":"2023-10-01T10:54:41.560285+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:31:43.040424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":169,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:41.559854+0000","last_change":"2023-10-01T10:38:16.034741+0000","last_active":"2023-10-01T10:54:41.559854+0000","last_peered":"2023-10-01T10:54:41.559854+0000","last_clean":"2023-10-01T10:54:41.559854+0000","last_became_active":"2023-10-01T10:38:16.034407+0000","last_became_peered":"2023-10-01T10:38:16.034407+0000","last_unstale":"2023-10-01T10:54:41.559854+0000","last_undegraded":"2023-10-01T10:54:41.559854+0000","last_fullsized":"2023-10-01T10:54:41.559854+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:47:29.911477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":191,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:41.564945+0000","last_change":"2023-10-01T10:38:16.026777+0000","last_active":"2023-10-01T10:54:41.564945+0000","last_peered":"2023-10-01T10:54:41.564945+0000","last_clean":"2023-10-01T10:54:41.564945+0000","last_became_active":"2023-10-01T10:38:16.026371+0000","last_became_peered":"2023-10-01T10:38:16.026371+0000","last_unstale":"2023-10-01T10:54:41.564945+0000","last_undegraded":"2023-10-01T10:54:41.564945+0000","last_fullsized":"2023-10-01T10:54:41.564945+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":196,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-01T10:54:40.556991+0000","last_change":"2023-10-01T10:37:42.772319+0000","last_active":"2023-10-01T10:54:40.556991+0000","last_peered":"2023-10-01T10:54:40.556991+0000","last_clean":"2023-10-01T10:54:40.556991+0000","last_became_active":"2023-10-01T10:37:42.771977+0000","last_became_peered":"2023-10-01T10:37:42.771977+0000","last_unstale":"2023-10-01T10:54:40.556991+0000","last_undegraded":"2023-10-01T10:54:40.556991+0000","last_fullsized":"2023-10-01T10:54:40.556991+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:45:12.783214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":169,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:41.570137+0000","last_change":"2023-10-01T10:37:42.767706+0000","last_active":"2023-10-01T10:54:41.570137+0000","last_peered":"2023-10-01T10:54:41.570137+0000","last_clean":"2023-10-01T10:54:41.570137+0000","last_became_active":"2023-10-01T10:37:42.767211+0000","last_became_peered":"2023-10-01T10:37:42.767211+0000","last_unstale":"2023-10-01T10:54:41.570137+0000","last_undegraded":"2023-10-01T10:54:41.570137+0000","last_fullsized":"2023-10-01T10:54:41.570137+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:37:20.357407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":174,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:41.568352+0000","last_change":"2023-10-01T10:37:42.768642+0000","last_active":"2023-10-01T10:54:41.568352+0000","last_peered":"2023-10-01T10:54:41.568352+0000","last_clean":"2023-10-01T10:54:41.568352+0000","last_became_active":"2023-10-01T10:37:42.768239+0000","last_became_peered":"2023-10-01T10:37:42.768239+0000","last_unstale":"2023-10-01T10:54:41.568352+0000","last_undegraded":"2023-10-01T10:54:41.568352+0000","last_fullsized":"2023-10-01T10:54:41.568352+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.19","version":"207'10890","reported_seq":10946,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:42.110790+0000","last_change":"2023-10-01T10:54:37.935496+0000","last_active":"2023-10-01T10:54:42.110790+0000","last_peered":"2023-10-01T10:54:42.110790+0000","last_clean":"2023-10-01T10:54:42.110790+0000","last_became_active":"2023-10-01T10:54:15.648834+0000","last_became_peered":"2023-10-01T10:54:15.648834+0000","last_unstale":"2023-10-01T10:54:42.110790+0000","last_undegraded":"2023-10-01T10:54:42.110790+0000","last_fullsized":"2023-10-01T10:54:42.110790+0000","mapping_epoch":189,"log_start":"179'2500","ondisk_log_start":"179'2500","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":8390,"log_dups_size":0,"ondisk_log_size":8390,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":74285056,"num_objects":1134,"num_object_clones":0,"num_object_copies":3402,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1134,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5645,"num_write_kb":216896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":121,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:41.559333+0000","last_change":"2023-10-01T10:52:43.661451+0000","last_active":"2023-10-01T10:54:41.559333+0000","last_peered":"2023-10-01T10:54:41.559333+0000","last_clean":"2023-10-01T10:54:41.559333+0000","last_became_active":"2023-10-01T10:52:43.661095+0000","last_became_peered":"2023-10-01T10:52:43.661095+0000","last_unstale":"2023-10-01T10:54:41.559333+0000","last_undegraded":"2023-10-01T10:54:41.559333+0000","last_fullsized":"2023-10-01T10:54:41.559333+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:08:59.976920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":169,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:41.570952+0000","last_change":"2023-10-01T10:37:42.772247+0000","last_active":"2023-10-01T10:54:41.570952+0000","last_peered":"2023-10-01T10:54:41.570952+0000","last_clean":"2023-10-01T10:54:41.570952+0000","last_became_active":"2023-10-01T10:37:42.771697+0000","last_became_peered":"2023-10-01T10:37:42.771697+0000","last_unstale":"2023-10-01T10:54:41.570952+0000","last_undegraded":"2023-10-01T10:54:41.570952+0000","last_fullsized":"2023-10-01T10:54:41.570952+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:23:15.018248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.18","version":"207'10756","reported_seq":10845,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:41.838706+0000","last_change":"2023-10-01T10:54:33.232866+0000","last_active":"2023-10-01T10:54:41.838706+0000","last_peered":"2023-10-01T10:54:41.838706+0000","last_clean":"2023-10-01T10:54:41.838706+0000","last_became_active":"2023-10-01T10:54:15.674351+0000","last_became_peered":"2023-10-01T10:54:15.674351+0000","last_unstale":"2023-10-01T10:54:41.838706+0000","last_undegraded":"2023-10-01T10:54:41.838706+0000","last_fullsized":"2023-10-01T10:54:41.838706+0000","mapping_epoch":189,"log_start":"179'2400","ondisk_log_start":"179'2400","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":5,"last_scrub":"198'10678","last_scrub_stamp":"2023-10-01T10:54:33.232623+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:54:33.232623+0000","objects_scrubbed":1147,"log_size":8356,"log_dups_size":0,"ondisk_log_size":8356,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:48:35.873116+0000","scrub_duration":0.21799253099999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70057984,"num_objects":1069,"num_object_clones":0,"num_object_copies":3207,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1069,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":78,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":182,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:41.569782+0000","last_change":"2023-10-01T10:28:20.211798+0000","last_active":"2023-10-01T10:54:41.569782+0000","last_peered":"2023-10-01T10:54:41.569782+0000","last_clean":"2023-10-01T10:54:41.569782+0000","last_became_active":"2023-10-01T10:28:20.210791+0000","last_became_peered":"2023-10-01T10:28:20.210791+0000","last_unstale":"2023-10-01T10:54:41.569782+0000","last_undegraded":"2023-10-01T10:54:41.569782+0000","last_fullsized":"2023-10-01T10:54:41.569782+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":173,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-01T10:54:40.557541+0000","last_change":"2023-10-01T10:37:42.773127+0000","last_active":"2023-10-01T10:54:40.557541+0000","last_peered":"2023-10-01T10:54:40.557541+0000","last_clean":"2023-10-01T10:54:40.557541+0000","last_became_active":"2023-10-01T10:37:42.772102+0000","last_became_peered":"2023-10-01T10:37:42.772102+0000","last_unstale":"2023-10-01T10:54:40.557541+0000","last_undegraded":"2023-10-01T10:54:40.557541+0000","last_fullsized":"2023-10-01T10:54:40.557541+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:23:26.202295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"208'10667","reported_seq":10719,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-01T10:54:43.277247+0000","last_change":"2023-10-01T10:54:15.638904+0000","last_active":"2023-10-01T10:54:43.277247+0000","last_peered":"2023-10-01T10:54:43.277247+0000","last_clean":"2023-10-01T10:54:43.277247+0000","last_became_active":"2023-10-01T10:54:15.616718+0000","last_became_peered":"2023-10-01T10:54:15.616718+0000","last_unstale":"2023-10-01T10:54:43.277247+0000","last_undegraded":"2023-10-01T10:54:43.277247+0000","last_fullsized":"2023-10-01T10:54:43.277247+0000","mapping_epoch":173,"log_start":"178'200","ondisk_log_start":"178'200","created":173,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":10467,"log_dups_size":0,"ondisk_log_size":10467,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72024064,"num_objects":1099,"num_object_clones":0,"num_object_copies":3297,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1099,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5554,"num_write_kb":212896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":47,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:41.560972+0000","last_change":"2023-10-01T10:52:43.663145+0000","last_active":"2023-10-01T10:54:41.560972+0000","last_peered":"2023-10-01T10:54:41.560972+0000","last_clean":"2023-10-01T10:54:41.560972+0000","last_became_active":"2023-10-01T10:52:43.662783+0000","last_became_peered":"2023-10-01T10:52:43.662783+0000","last_unstale":"2023-10-01T10:54:41.560972+0000","last_undegraded":"2023-10-01T10:54:41.560972+0000","last_fullsized":"2023-10-01T10:54:41.560972+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T22:10:32.190922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":159,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:41.567763+0000","last_change":"2023-10-01T10:15:40.318702+0000","last_active":"2023-10-01T10:54:41.567763+0000","last_peered":"2023-10-01T10:54:41.567763+0000","last_clean":"2023-10-01T10:54:41.567763+0000","last_became_active":"2023-10-01T10:15:40.318333+0000","last_became_peered":"2023-10-01T10:15:40.318333+0000","last_unstale":"2023-10-01T10:54:41.567763+0000","last_undegraded":"2023-10-01T10:54:41.567763+0000","last_fullsized":"2023-10-01T10:54:41.567763+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"208'10800","reported_seq":10896,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-01T10:54:44.219131+0000","last_change":"2023-10-01T10:54:17.257453+0000","last_active":"2023-10-01T10:54:44.219131+0000","last_peered":"2023-10-01T10:54:44.219131+0000","last_clean":"2023-10-01T10:54:44.219131+0000","last_became_active":"2023-10-01T10:54:15.627377+0000","last_became_peered":"2023-10-01T10:54:15.627377+0000","last_unstale":"2023-10-01T10:54:44.219131+0000","last_undegraded":"2023-10-01T10:54:44.219131+0000","last_fullsized":"2023-10-01T10:54:44.219131+0000","mapping_epoch":173,"log_start":"178'600","ondisk_log_start":"178'600","created":173,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"190'10367","last_scrub_stamp":"2023-10-01T10:54:17.257230+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:54:17.257230+0000","objects_scrubbed":1500,"log_size":10200,"log_dups_size":0,"ondisk_log_size":10200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:07:21.492956+0000","scrub_duration":0.52710383199999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69337088,"num_objects":1058,"num_object_clones":0,"num_object_copies":3174,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1058,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":442,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":47,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:41.560682+0000","last_change":"2023-10-01T10:52:43.662540+0000","last_active":"2023-10-01T10:54:41.560682+0000","last_peered":"2023-10-01T10:54:41.560682+0000","last_clean":"2023-10-01T10:54:41.560682+0000","last_became_active":"2023-10-01T10:52:43.662128+0000","last_became_peered":"2023-10-01T10:52:43.662128+0000","last_unstale":"2023-10-01T10:54:41.560682+0000","last_undegraded":"2023-10-01T10:54:41.560682+0000","last_fullsized":"2023-10-01T10:54:41.560682+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:09:44.807003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":203,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:41.559315+0000","last_change":"2023-10-01T10:38:16.033203+0000","last_active":"2023-10-01T10:54:41.559315+0000","last_peered":"2023-10-01T10:54:41.559315+0000","last_clean":"2023-10-01T10:54:41.559315+0000","last_became_active":"2023-10-01T10:38:16.032681+0000","last_became_peered":"2023-10-01T10:38:16.032681+0000","last_unstale":"2023-10-01T10:54:41.559315+0000","last_undegraded":"2023-10-01T10:54:41.559315+0000","last_fullsized":"2023-10-01T10:54:41.559315+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:42:36.642284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"208'10884","reported_seq":10952,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-01T10:54:43.105328+0000","last_change":"2023-10-01T10:54:34.141577+0000","last_active":"2023-10-01T10:54:43.105328+0000","last_peered":"2023-10-01T10:54:43.105328+0000","last_clean":"2023-10-01T10:54:43.105328+0000","last_became_active":"2023-10-01T10:54:15.612272+0000","last_became_peered":"2023-10-01T10:54:15.612272+0000","last_unstale":"2023-10-01T10:54:43.105328+0000","last_undegraded":"2023-10-01T10:54:43.105328+0000","last_fullsized":"2023-10-01T10:54:43.105328+0000","mapping_epoch":183,"log_start":"178'400","ondisk_log_start":"178'400","created":173,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":10484,"log_dups_size":0,"ondisk_log_size":10484,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70811648,"num_objects":1080,"num_object_clones":0,"num_object_copies":3240,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1080,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5666,"num_write_kb":215904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":47,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:41.568487+0000","last_change":"2023-10-01T10:52:43.669543+0000","last_active":"2023-10-01T10:54:41.568487+0000","last_peered":"2023-10-01T10:54:41.568487+0000","last_clean":"2023-10-01T10:54:41.568487+0000","last_became_active":"2023-10-01T10:52:43.669141+0000","last_became_peered":"2023-10-01T10:52:43.669141+0000","last_unstale":"2023-10-01T10:54:41.568487+0000","last_undegraded":"2023-10-01T10:54:41.568487+0000","last_fullsized":"2023-10-01T10:54:41.568487+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":173,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:41.570014+0000","last_change":"2023-10-01T10:32:31.571971+0000","last_active":"2023-10-01T10:54:41.570014+0000","last_peered":"2023-10-01T10:54:41.570014+0000","last_clean":"2023-10-01T10:54:41.570014+0000","last_became_active":"2023-10-01T10:32:31.571410+0000","last_became_peered":"2023-10-01T10:32:31.571410+0000","last_unstale":"2023-10-01T10:54:41.570014+0000","last_undegraded":"2023-10-01T10:54:41.570014+0000","last_fullsized":"2023-10-01T10:54:41.570014+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"208'10800","reported_seq":10864,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-01T10:54:43.371939+0000","last_change":"2023-10-01T10:54:18.775349+0000","last_active":"2023-10-01T10:54:43.371939+0000","last_peered":"2023-10-01T10:54:43.371939+0000","last_clean":"2023-10-01T10:54:43.371939+0000","last_became_active":"2023-10-01T10:54:15.606066+0000","last_became_peered":"2023-10-01T10:54:15.606066+0000","last_unstale":"2023-10-01T10:54:43.371939+0000","last_undegraded":"2023-10-01T10:54:43.371939+0000","last_fullsized":"2023-10-01T10:54:43.371939+0000","mapping_epoch":183,"log_start":"179'1700","ondisk_log_start":"179'1700","created":173,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"190'10418","last_scrub_stamp":"2023-10-01T10:54:18.773065+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:54:18.773065+0000","objects_scrubbed":0,"log_size":9100,"log_dups_size":0,"ondisk_log_size":9100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69206016,"num_objects":1056,"num_object_clones":0,"num_object_copies":3168,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1056,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":399,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":175,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:41.567882+0000","last_change":"2023-10-01T10:37:42.765733+0000","last_active":"2023-10-01T10:54:41.567882+0000","last_peered":"2023-10-01T10:54:41.567882+0000","last_clean":"2023-10-01T10:54:41.567882+0000","last_became_active":"2023-10-01T10:37:42.765311+0000","last_became_peered":"2023-10-01T10:37:42.765311+0000","last_unstale":"2023-10-01T10:54:41.567882+0000","last_undegraded":"2023-10-01T10:54:41.567882+0000","last_fullsized":"2023-10-01T10:54:41.567882+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":185,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:41.569996+0000","last_change":"2023-10-01T10:38:16.026348+0000","last_active":"2023-10-01T10:54:41.569996+0000","last_peered":"2023-10-01T10:54:41.569996+0000","last_clean":"2023-10-01T10:54:41.569996+0000","last_became_active":"2023-10-01T10:38:16.025747+0000","last_became_peered":"2023-10-01T10:38:16.025747+0000","last_unstale":"2023-10-01T10:54:41.569996+0000","last_undegraded":"2023-10-01T10:54:41.569996+0000","last_fullsized":"2023-10-01T10:54:41.569996+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:28:18.934480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"208'10928","reported_seq":10987,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-01T10:54:43.402194+0000","last_change":"2023-10-01T10:54:41.790187+0000","last_active":"2023-10-01T10:54:43.402194+0000","last_peered":"2023-10-01T10:54:43.402194+0000","last_clean":"2023-10-01T10:54:43.402194+0000","last_became_active":"2023-10-01T10:54:15.622989+0000","last_became_peered":"2023-10-01T10:54:15.622989+0000","last_unstale":"2023-10-01T10:54:43.402194+0000","last_undegraded":"2023-10-01T10:54:43.402194+0000","last_fullsized":"2023-10-01T10:54:43.402194+0000","mapping_epoch":173,"log_start":"179'1900","ondisk_log_start":"179'1900","created":173,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":9028,"log_dups_size":0,"ondisk_log_size":9028,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":74285056,"num_objects":1133,"num_object_clones":0,"num_object_copies":3399,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1133,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5671,"num_write_kb":217760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":176,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-01T10:54:40.556994+0000","last_change":"2023-10-01T10:38:16.022493+0000","last_active":"2023-10-01T10:54:40.556994+0000","last_peered":"2023-10-01T10:54:40.556994+0000","last_clean":"2023-10-01T10:54:40.556994+0000","last_became_active":"2023-10-01T10:38:16.022033+0000","last_became_peered":"2023-10-01T10:38:16.022033+0000","last_unstale":"2023-10-01T10:54:40.556994+0000","last_undegraded":"2023-10-01T10:54:40.556994+0000","last_fullsized":"2023-10-01T10:54:40.556994+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:49:05.262763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":188,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:41.560411+0000","last_change":"2023-10-01T10:38:16.030613+0000","last_active":"2023-10-01T10:54:41.560411+0000","last_peered":"2023-10-01T10:54:41.560411+0000","last_clean":"2023-10-01T10:54:41.560411+0000","last_became_active":"2023-10-01T10:38:16.028975+0000","last_became_peered":"2023-10-01T10:38:16.028975+0000","last_unstale":"2023-10-01T10:54:41.560411+0000","last_undegraded":"2023-10-01T10:54:41.560411+0000","last_fullsized":"2023-10-01T10:54:41.560411+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:46:07.287911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"208'10895","reported_seq":10950,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-01T10:54:43.168245+0000","last_change":"2023-10-01T10:54:15.617195+0000","last_active":"2023-10-01T10:54:43.168245+0000","last_peered":"2023-10-01T10:54:43.168245+0000","last_clean":"2023-10-01T10:54:43.168245+0000","last_became_active":"2023-10-01T10:54:15.593182+0000","last_became_peered":"2023-10-01T10:54:15.593182+0000","last_unstale":"2023-10-01T10:54:43.168245+0000","last_undegraded":"2023-10-01T10:54:43.168245+0000","last_fullsized":"2023-10-01T10:54:43.168245+0000","mapping_epoch":173,"log_start":"178'400","ondisk_log_start":"178'400","created":173,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":10495,"log_dups_size":0,"ondisk_log_size":10495,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70877184,"num_objects":1081,"num_object_clones":0,"num_object_copies":3243,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1081,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5685,"num_write_kb":216544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":161,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:41.559609+0000","last_change":"2023-10-01T10:47:54.750295+0000","last_active":"2023-10-01T10:54:41.559609+0000","last_peered":"2023-10-01T10:54:41.559609+0000","last_clean":"2023-10-01T10:54:41.559609+0000","last_became_active":"2023-10-01T10:47:54.749917+0000","last_became_peered":"2023-10-01T10:47:54.749917+0000","last_unstale":"2023-10-01T10:54:41.559609+0000","last_undegraded":"2023-10-01T10:54:41.559609+0000","last_fullsized":"2023-10-01T10:54:41.559609+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:30:59.236210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":193,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:41.560527+0000","last_change":"2023-10-01T10:38:16.035906+0000","last_active":"2023-10-01T10:54:41.560527+0000","last_peered":"2023-10-01T10:54:41.560527+0000","last_clean":"2023-10-01T10:54:41.560527+0000","last_became_active":"2023-10-01T10:38:16.035383+0000","last_became_peered":"2023-10-01T10:38:16.035383+0000","last_unstale":"2023-10-01T10:54:41.560527+0000","last_undegraded":"2023-10-01T10:54:41.560527+0000","last_fullsized":"2023-10-01T10:54:41.560527+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:44:41.500097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"207'10818","reported_seq":10908,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:42.225094+0000","last_change":"2023-10-01T10:54:19.406543+0000","last_active":"2023-10-01T10:54:42.225094+0000","last_peered":"2023-10-01T10:54:42.225094+0000","last_clean":"2023-10-01T10:54:42.225094+0000","last_became_active":"2023-10-01T10:54:15.588522+0000","last_became_peered":"2023-10-01T10:54:15.588522+0000","last_unstale":"2023-10-01T10:54:42.225094+0000","last_undegraded":"2023-10-01T10:54:42.225094+0000","last_fullsized":"2023-10-01T10:54:42.225094+0000","mapping_epoch":173,"log_start":"179'2500","ondisk_log_start":"179'2500","created":173,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"190'10512","last_scrub_stamp":"2023-10-01T10:54:19.406037+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:54:19.406037+0000","objects_scrubbed":1377,"log_size":8318,"log_dups_size":0,"ondisk_log_size":8318,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:53:49.769321+0000","scrub_duration":0.53684859900000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69599232,"num_objects":1062,"num_object_clones":0,"num_object_copies":3186,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1062,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":315,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":185,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:41.559676+0000","last_change":"2023-10-01T10:38:16.026611+0000","last_active":"2023-10-01T10:54:41.559676+0000","last_peered":"2023-10-01T10:54:41.559676+0000","last_clean":"2023-10-01T10:54:41.559676+0000","last_became_active":"2023-10-01T10:38:16.026267+0000","last_became_peered":"2023-10-01T10:38:16.026267+0000","last_unstale":"2023-10-01T10:54:41.559676+0000","last_undegraded":"2023-10-01T10:54:41.559676+0000","last_fullsized":"2023-10-01T10:54:41.559676+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:06.135288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":169,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:41.560555+0000","last_change":"2023-10-01T10:38:16.032162+0000","last_active":"2023-10-01T10:54:41.560555+0000","last_peered":"2023-10-01T10:54:41.560555+0000","last_clean":"2023-10-01T10:54:41.560555+0000","last_became_active":"2023-10-01T10:38:16.031785+0000","last_became_peered":"2023-10-01T10:38:16.031785+0000","last_unstale":"2023-10-01T10:54:41.560555+0000","last_undegraded":"2023-10-01T10:54:41.560555+0000","last_fullsized":"2023-10-01T10:54:41.560555+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:38:43.003626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"207'10874","reported_seq":10942,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:42.242282+0000","last_change":"2023-10-01T10:54:35.126391+0000","last_active":"2023-10-01T10:54:42.242282+0000","last_peered":"2023-10-01T10:54:42.242282+0000","last_clean":"2023-10-01T10:54:42.242282+0000","last_became_active":"2023-10-01T10:54:15.583965+0000","last_became_peered":"2023-10-01T10:54:15.583965+0000","last_unstale":"2023-10-01T10:54:42.242282+0000","last_undegraded":"2023-10-01T10:54:42.242282+0000","last_fullsized":"2023-10-01T10:54:42.242282+0000","mapping_epoch":181,"log_start":"179'2500","ondisk_log_start":"179'2500","created":173,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":8374,"log_dups_size":0,"ondisk_log_size":8374,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70844416,"num_objects":1081,"num_object_clones":0,"num_object_copies":3243,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1081,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5652,"num_write_kb":215456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":32768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":174,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-01T10:54:40.556656+0000","last_change":"2023-10-01T10:37:42.772398+0000","last_active":"2023-10-01T10:54:40.556656+0000","last_peered":"2023-10-01T10:54:40.556656+0000","last_clean":"2023-10-01T10:54:40.556656+0000","last_became_active":"2023-10-01T10:37:42.772084+0000","last_became_peered":"2023-10-01T10:37:42.772084+0000","last_unstale":"2023-10-01T10:54:40.556656+0000","last_undegraded":"2023-10-01T10:54:40.556656+0000","last_fullsized":"2023-10-01T10:54:40.556656+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:32:43.846419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":193,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:41.563871+0000","last_change":"2023-10-01T10:38:16.019755+0000","last_active":"2023-10-01T10:54:41.563871+0000","last_peered":"2023-10-01T10:54:41.563871+0000","last_clean":"2023-10-01T10:54:41.563871+0000","last_became_active":"2023-10-01T10:38:16.019397+0000","last_became_peered":"2023-10-01T10:38:16.019397+0000","last_unstale":"2023-10-01T10:54:41.563871+0000","last_undegraded":"2023-10-01T10:54:41.563871+0000","last_fullsized":"2023-10-01T10:54:41.563871+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:42:56.623225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.d","version":"208'11024","reported_seq":11070,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-01T10:54:43.213445+0000","last_change":"2023-10-01T10:51:01.952187+0000","last_active":"2023-10-01T10:54:43.213445+0000","last_peered":"2023-10-01T10:54:43.213445+0000","last_clean":"2023-10-01T10:54:43.213445+0000","last_became_active":"2023-10-01T10:51:01.951373+0000","last_became_peered":"2023-10-01T10:51:01.951373+0000","last_unstale":"2023-10-01T10:54:43.213445+0000","last_undegraded":"2023-10-01T10:54:43.213445+0000","last_fullsized":"2023-10-01T10:54:43.213445+0000","mapping_epoch":173,"log_start":"178'1324","ondisk_log_start":"178'1324","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":9700,"log_dups_size":0,"ondisk_log_size":9700,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:10:27.939483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140705816,"num_objects":2148,"num_object_clones":0,"num_object_copies":6444,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2148,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11024,"num_write_kb":421441,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":101,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:41.570073+0000","last_change":"2023-10-01T10:52:18.960164+0000","last_active":"2023-10-01T10:54:41.570073+0000","last_peered":"2023-10-01T10:54:41.570073+0000","last_clean":"2023-10-01T10:54:41.570073+0000","last_became_active":"2023-10-01T10:52:18.959782+0000","last_became_peered":"2023-10-01T10:52:18.959782+0000","last_unstale":"2023-10-01T10:54:41.570073+0000","last_undegraded":"2023-10-01T10:54:41.570073+0000","last_fullsized":"2023-10-01T10:54:41.570073+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:08:27.603412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":190,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:41.568237+0000","last_change":"2023-10-01T10:37:42.766814+0000","last_active":"2023-10-01T10:54:41.568237+0000","last_peered":"2023-10-01T10:54:41.568237+0000","last_clean":"2023-10-01T10:54:41.568237+0000","last_became_active":"2023-10-01T10:37:42.766451+0000","last_became_peered":"2023-10-01T10:37:42.766451+0000","last_unstale":"2023-10-01T10:54:41.568237+0000","last_undegraded":"2023-10-01T10:54:41.568237+0000","last_fullsized":"2023-10-01T10:54:41.568237+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":93,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:41.567872+0000","last_change":"2023-10-01T10:52:18.960606+0000","last_active":"2023-10-01T10:54:41.567872+0000","last_peered":"2023-10-01T10:54:41.567872+0000","last_clean":"2023-10-01T10:54:41.567872+0000","last_became_active":"2023-10-01T10:52:18.960270+0000","last_became_peered":"2023-10-01T10:52:18.960270+0000","last_unstale":"2023-10-01T10:54:41.567872+0000","last_undegraded":"2023-10-01T10:54:41.567872+0000","last_fullsized":"2023-10-01T10:54:41.567872+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.b","version":"207'11019","reported_seq":11063,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:42.238330+0000","last_change":"2023-10-01T10:51:01.950278+0000","last_active":"2023-10-01T10:54:42.238330+0000","last_peered":"2023-10-01T10:54:42.238330+0000","last_clean":"2023-10-01T10:54:42.238330+0000","last_became_active":"2023-10-01T10:51:01.949902+0000","last_became_peered":"2023-10-01T10:51:01.949902+0000","last_unstale":"2023-10-01T10:54:42.238330+0000","last_undegraded":"2023-10-01T10:54:42.238330+0000","last_fullsized":"2023-10-01T10:54:42.238330+0000","mapping_epoch":173,"log_start":"180'4142","ondisk_log_start":"180'4142","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":6877,"log_dups_size":0,"ondisk_log_size":6877,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:42:45.648175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139526144,"num_objects":2129,"num_object_clones":0,"num_object_copies":6387,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2129,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11019,"num_write_kb":420736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":179,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:41.567992+0000","last_change":"2023-10-01T10:32:31.572482+0000","last_active":"2023-10-01T10:54:41.567992+0000","last_peered":"2023-10-01T10:54:41.567992+0000","last_clean":"2023-10-01T10:54:41.567992+0000","last_became_active":"2023-10-01T10:32:31.571993+0000","last_became_peered":"2023-10-01T10:32:31.571993+0000","last_unstale":"2023-10-01T10:54:41.567992+0000","last_undegraded":"2023-10-01T10:54:41.567992+0000","last_fullsized":"2023-10-01T10:54:41.567992+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":155,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:41.561165+0000","last_change":"2023-10-01T10:38:16.030202+0000","last_active":"2023-10-01T10:54:41.561165+0000","last_peered":"2023-10-01T10:54:41.561165+0000","last_clean":"2023-10-01T10:54:41.561165+0000","last_became_active":"2023-10-01T10:38:16.029917+0000","last_became_peered":"2023-10-01T10:38:16.029917+0000","last_unstale":"2023-10-01T10:54:41.561165+0000","last_undegraded":"2023-10-01T10:54:41.561165+0000","last_fullsized":"2023-10-01T10:54:41.561165+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:20:06.763909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":104,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:41.563721+0000","last_change":"2023-10-01T10:52:18.963437+0000","last_active":"2023-10-01T10:54:41.563721+0000","last_peered":"2023-10-01T10:54:41.563721+0000","last_clean":"2023-10-01T10:54:41.563721+0000","last_became_active":"2023-10-01T10:52:18.955165+0000","last_became_peered":"2023-10-01T10:52:18.955165+0000","last_unstale":"2023-10-01T10:54:41.563721+0000","last_undegraded":"2023-10-01T10:54:41.563721+0000","last_fullsized":"2023-10-01T10:54:41.563721+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:57:49.129100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.8","version":"207'10811","reported_seq":10906,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:41.852224+0000","last_change":"2023-10-01T10:54:17.446257+0000","last_active":"2023-10-01T10:54:41.852224+0000","last_peered":"2023-10-01T10:54:41.852224+0000","last_clean":"2023-10-01T10:54:41.852224+0000","last_became_active":"2023-10-01T10:54:15.618659+0000","last_became_peered":"2023-10-01T10:54:15.618659+0000","last_unstale":"2023-10-01T10:54:41.852224+0000","last_undegraded":"2023-10-01T10:54:41.852224+0000","last_fullsized":"2023-10-01T10:54:41.852224+0000","mapping_epoch":173,"log_start":"179'2400","ondisk_log_start":"179'2400","created":173,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"190'10432","last_scrub_stamp":"2023-10-01T10:54:17.445223+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:54:17.445223+0000","objects_scrubbed":1455,"log_size":8411,"log_dups_size":0,"ondisk_log_size":8411,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T22:15:47.498205+0000","scrub_duration":0.59228419099999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69861376,"num_objects":1066,"num_object_clones":0,"num_object_copies":3198,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1066,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":389,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":185,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:41.563271+0000","last_change":"2023-10-01T10:37:42.762524+0000","last_active":"2023-10-01T10:54:41.563271+0000","last_peered":"2023-10-01T10:54:41.563271+0000","last_clean":"2023-10-01T10:54:41.563271+0000","last_became_active":"2023-10-01T10:37:42.762209+0000","last_became_peered":"2023-10-01T10:37:42.762209+0000","last_unstale":"2023-10-01T10:54:41.563271+0000","last_undegraded":"2023-10-01T10:54:41.563271+0000","last_fullsized":"2023-10-01T10:54:41.563271+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:45:38.171397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":84,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:41.563236+0000","last_change":"2023-10-01T10:52:18.962766+0000","last_active":"2023-10-01T10:54:41.563236+0000","last_peered":"2023-10-01T10:54:41.563236+0000","last_clean":"2023-10-01T10:54:41.563236+0000","last_became_active":"2023-10-01T10:52:18.962461+0000","last_became_peered":"2023-10-01T10:52:18.962461+0000","last_unstale":"2023-10-01T10:54:41.563236+0000","last_undegraded":"2023-10-01T10:54:41.563236+0000","last_fullsized":"2023-10-01T10:54:41.563236+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:25:54.051672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":95,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:41.568452+0000","last_change":"2023-10-01T10:52:18.961040+0000","last_active":"2023-10-01T10:54:41.568452+0000","last_peered":"2023-10-01T10:54:41.568452+0000","last_clean":"2023-10-01T10:54:41.568452+0000","last_became_active":"2023-10-01T10:52:18.960606+0000","last_became_peered":"2023-10-01T10:52:18.960606+0000","last_unstale":"2023-10-01T10:54:41.568452+0000","last_undegraded":"2023-10-01T10:54:41.568452+0000","last_fullsized":"2023-10-01T10:54:41.568452+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.a","version":"207'11189","reported_seq":11233,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:42.062079+0000","last_change":"2023-10-01T10:51:01.955145+0000","last_active":"2023-10-01T10:54:42.062079+0000","last_peered":"2023-10-01T10:54:42.062079+0000","last_clean":"2023-10-01T10:54:42.062079+0000","last_became_active":"2023-10-01T10:51:01.954793+0000","last_became_peered":"2023-10-01T10:51:01.954793+0000","last_unstale":"2023-10-01T10:54:42.062079+0000","last_undegraded":"2023-10-01T10:54:42.062079+0000","last_fullsized":"2023-10-01T10:54:42.062079+0000","mapping_epoch":173,"log_start":"180'4323","ondisk_log_start":"180'4323","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":6866,"log_dups_size":0,"ondisk_log_size":6866,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:25:33.468595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":142540800,"num_objects":2175,"num_object_clones":0,"num_object_copies":6525,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2175,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11189,"num_write_kb":427648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":179,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:41.569172+0000","last_change":"2023-10-01T10:37:05.024009+0000","last_active":"2023-10-01T10:54:41.569172+0000","last_peered":"2023-10-01T10:54:41.569172+0000","last_clean":"2023-10-01T10:54:41.569172+0000","last_became_active":"2023-10-01T10:37:05.023458+0000","last_became_peered":"2023-10-01T10:37:05.023458+0000","last_unstale":"2023-10-01T10:54:41.569172+0000","last_undegraded":"2023-10-01T10:54:41.569172+0000","last_fullsized":"2023-10-01T10:54:41.569172+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'139","reported_seq":299,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-01T10:54:42.566347+0000","last_change":"2023-10-01T10:52:43.660674+0000","last_active":"2023-10-01T10:54:42.566347+0000","last_peered":"2023-10-01T10:54:42.566347+0000","last_clean":"2023-10-01T10:54:42.566347+0000","last_became_active":"2023-10-01T10:52:43.660276+0000","last_became_peered":"2023-10-01T10:52:43.660276+0000","last_unstale":"2023-10-01T10:54:42.566347+0000","last_undegraded":"2023-10-01T10:54:42.566347+0000","last_fullsized":"2023-10-01T10:54:42.566347+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:09:17.221082+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:09:17.221082+0000","last_clean_scrub_stamp":"2023-10-01T10:09:17.221082+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:47:37.225755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":157,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-01T10:54:40.557993+0000","last_change":"2023-10-01T10:37:42.765100+0000","last_active":"2023-10-01T10:54:40.557993+0000","last_peered":"2023-10-01T10:54:40.557993+0000","last_clean":"2023-10-01T10:54:40.557993+0000","last_became_active":"2023-10-01T10:37:42.764761+0000","last_became_peered":"2023-10-01T10:37:42.764761+0000","last_unstale":"2023-10-01T10:54:40.557993+0000","last_undegraded":"2023-10-01T10:54:40.557993+0000","last_fullsized":"2023-10-01T10:54:40.557993+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:24:56.907377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":145,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:41.560286+0000","last_change":"2023-10-01T10:47:54.782475+0000","last_active":"2023-10-01T10:54:41.560286+0000","last_peered":"2023-10-01T10:54:41.560286+0000","last_clean":"2023-10-01T10:54:41.560286+0000","last_became_active":"2023-10-01T10:47:54.781725+0000","last_became_peered":"2023-10-01T10:47:54.781725+0000","last_unstale":"2023-10-01T10:54:41.560286+0000","last_undegraded":"2023-10-01T10:54:41.560286+0000","last_fullsized":"2023-10-01T10:54:41.560286+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:21:30.978552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"207'10910","reported_seq":11003,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:42.029952+0000","last_change":"2023-10-01T10:54:22.063987+0000","last_active":"2023-10-01T10:54:42.029952+0000","last_peered":"2023-10-01T10:54:42.029952+0000","last_clean":"2023-10-01T10:54:42.029952+0000","last_became_active":"2023-10-01T10:54:15.586144+0000","last_became_peered":"2023-10-01T10:54:15.586144+0000","last_unstale":"2023-10-01T10:54:42.029952+0000","last_undegraded":"2023-10-01T10:54:42.029952+0000","last_fullsized":"2023-10-01T10:54:42.029952+0000","mapping_epoch":173,"log_start":"179'2500","ondisk_log_start":"179'2500","created":173,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":0,"last_scrub":"191'10657","last_scrub_stamp":"2023-10-01T10:54:22.063841+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:54:22.063841+0000","objects_scrubbed":1434,"log_size":8410,"log_dups_size":0,"ondisk_log_size":8410,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T22:29:46.958786+0000","scrub_duration":0.221481707,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77398016,"num_objects":1181,"num_object_clones":0,"num_object_copies":3543,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1181,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":253,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":116,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-01T10:54:42.566115+0000","last_change":"2023-10-01T10:52:43.661009+0000","last_active":"2023-10-01T10:54:42.566115+0000","last_peered":"2023-10-01T10:54:42.566115+0000","last_clean":"2023-10-01T10:54:42.566115+0000","last_became_active":"2023-10-01T10:52:43.660679+0000","last_became_peered":"2023-10-01T10:52:43.660679+0000","last_unstale":"2023-10-01T10:54:42.566115+0000","last_undegraded":"2023-10-01T10:54:42.566115+0000","last_fullsized":"2023-10-01T10:54:42.566115+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:32:21.336835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":185,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:41.573125+0000","last_change":"2023-10-01T10:38:16.021968+0000","last_active":"2023-10-01T10:54:41.573125+0000","last_peered":"2023-10-01T10:54:41.573125+0000","last_clean":"2023-10-01T10:54:41.573125+0000","last_became_active":"2023-10-01T10:38:16.021539+0000","last_became_peered":"2023-10-01T10:38:16.021539+0000","last_unstale":"2023-10-01T10:54:41.573125+0000","last_undegraded":"2023-10-01T10:54:41.573125+0000","last_fullsized":"2023-10-01T10:54:41.573125+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:12:53.105747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":186,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:41.570876+0000","last_change":"2023-10-01T10:37:05.017431+0000","last_active":"2023-10-01T10:54:41.570876+0000","last_peered":"2023-10-01T10:54:41.570876+0000","last_clean":"2023-10-01T10:54:41.570876+0000","last_became_active":"2023-10-01T10:37:05.017094+0000","last_became_peered":"2023-10-01T10:37:05.017094+0000","last_unstale":"2023-10-01T10:54:41.570876+0000","last_undegraded":"2023-10-01T10:54:41.570876+0000","last_fullsized":"2023-10-01T10:54:41.570876+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:10:40.664686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.c","version":"208'11243","reported_seq":11293,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-01T10:54:43.189606+0000","last_change":"2023-10-01T10:52:18.964236+0000","last_active":"2023-10-01T10:54:43.189606+0000","last_peered":"2023-10-01T10:54:43.189606+0000","last_clean":"2023-10-01T10:54:43.189606+0000","last_became_active":"2023-10-01T10:52:18.955030+0000","last_became_peered":"2023-10-01T10:52:18.955030+0000","last_unstale":"2023-10-01T10:54:43.189606+0000","last_undegraded":"2023-10-01T10:54:43.189606+0000","last_fullsized":"2023-10-01T10:54:43.189606+0000","mapping_epoch":181,"log_start":"179'1510","ondisk_log_start":"179'1510","created":173,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":9733,"log_dups_size":0,"ondisk_log_size":9733,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T22:29:23.161365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143327232,"num_objects":2187,"num_object_clones":0,"num_object_copies":6561,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2187,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11243,"num_write_kb":429760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":193,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:41.564523+0000","last_change":"2023-10-01T10:38:16.032774+0000","last_active":"2023-10-01T10:54:41.564523+0000","last_peered":"2023-10-01T10:54:41.564523+0000","last_clean":"2023-10-01T10:54:41.564523+0000","last_became_active":"2023-10-01T10:38:16.032283+0000","last_became_peered":"2023-10-01T10:38:16.032283+0000","last_unstale":"2023-10-01T10:54:41.564523+0000","last_undegraded":"2023-10-01T10:54:41.564523+0000","last_fullsized":"2023-10-01T10:54:41.564523+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":84,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:41.564103+0000","last_change":"2023-10-01T10:52:18.964479+0000","last_active":"2023-10-01T10:54:41.564103+0000","last_peered":"2023-10-01T10:54:41.564103+0000","last_clean":"2023-10-01T10:54:41.564103+0000","last_became_active":"2023-10-01T10:52:18.963881+0000","last_became_peered":"2023-10-01T10:52:18.963881+0000","last_unstale":"2023-10-01T10:54:41.564103+0000","last_undegraded":"2023-10-01T10:54:41.564103+0000","last_fullsized":"2023-10-01T10:54:41.564103+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:36:30.405755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":156,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:41.570239+0000","last_change":"2023-10-01T10:28:20.220812+0000","last_active":"2023-10-01T10:54:41.570239+0000","last_peered":"2023-10-01T10:54:41.570239+0000","last_clean":"2023-10-01T10:54:41.570239+0000","last_became_active":"2023-10-01T10:28:20.215962+0000","last_became_peered":"2023-10-01T10:28:20.215962+0000","last_unstale":"2023-10-01T10:54:41.570239+0000","last_undegraded":"2023-10-01T10:54:41.570239+0000","last_fullsized":"2023-10-01T10:54:41.570239+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:35:15.275097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"208'11078","reported_seq":11134,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-01T10:54:43.227747+0000","last_change":"2023-10-01T10:52:43.703049+0000","last_active":"2023-10-01T10:54:43.227747+0000","last_peered":"2023-10-01T10:54:43.227747+0000","last_clean":"2023-10-01T10:54:43.227747+0000","last_became_active":"2023-10-01T10:52:43.664078+0000","last_became_peered":"2023-10-01T10:52:43.664078+0000","last_unstale":"2023-10-01T10:54:43.227747+0000","last_undegraded":"2023-10-01T10:54:43.227747+0000","last_fullsized":"2023-10-01T10:54:43.227747+0000","mapping_epoch":183,"log_start":"178'1338","ondisk_log_start":"178'1338","created":173,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":9740,"log_dups_size":0,"ondisk_log_size":9740,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:44:08.983664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141295616,"num_objects":2156,"num_object_clones":0,"num_object_copies":6468,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2156,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11078,"num_write_kb":423488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":161,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-01T10:54:40.557689+0000","last_change":"2023-10-01T10:37:42.773999+0000","last_active":"2023-10-01T10:54:40.557689+0000","last_peered":"2023-10-01T10:54:40.557689+0000","last_clean":"2023-10-01T10:54:40.557689+0000","last_became_active":"2023-10-01T10:37:42.773355+0000","last_became_peered":"2023-10-01T10:37:42.773355+0000","last_unstale":"2023-10-01T10:54:40.557689+0000","last_undegraded":"2023-10-01T10:54:40.557689+0000","last_fullsized":"2023-10-01T10:54:40.557689+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:46:51.317151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":136,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:41.560438+0000","last_change":"2023-10-01T10:52:18.962219+0000","last_active":"2023-10-01T10:54:41.560438+0000","last_peered":"2023-10-01T10:54:41.560438+0000","last_clean":"2023-10-01T10:54:41.560438+0000","last_became_active":"2023-10-01T10:52:18.961826+0000","last_became_peered":"2023-10-01T10:52:18.961826+0000","last_unstale":"2023-10-01T10:54:41.560438+0000","last_undegraded":"2023-10-01T10:54:41.560438+0000","last_fullsized":"2023-10-01T10:54:41.560438+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:01:46.130522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":159,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-01T10:54:40.556565+0000","last_change":"2023-10-01T10:37:42.773538+0000","last_active":"2023-10-01T10:54:40.556565+0000","last_peered":"2023-10-01T10:54:40.556565+0000","last_clean":"2023-10-01T10:54:40.556565+0000","last_became_active":"2023-10-01T10:37:42.771863+0000","last_became_peered":"2023-10-01T10:37:42.771863+0000","last_unstale":"2023-10-01T10:54:40.556565+0000","last_undegraded":"2023-10-01T10:54:40.556565+0000","last_fullsized":"2023-10-01T10:54:40.556565+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:46:46.698627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"208'11312","reported_seq":11356,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-01T10:54:44.048452+0000","last_change":"2023-10-01T10:51:01.952403+0000","last_active":"2023-10-01T10:54:44.048452+0000","last_peered":"2023-10-01T10:54:44.048452+0000","last_clean":"2023-10-01T10:54:44.048452+0000","last_became_active":"2023-10-01T10:51:01.951884+0000","last_became_peered":"2023-10-01T10:51:01.951884+0000","last_unstale":"2023-10-01T10:54:44.048452+0000","last_undegraded":"2023-10-01T10:54:44.048452+0000","last_fullsized":"2023-10-01T10:54:44.048452+0000","mapping_epoch":173,"log_start":"180'3951","ondisk_log_start":"180'3951","created":173,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":7361,"log_dups_size":0,"ondisk_log_size":7361,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:33:03.736176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139460608,"num_objects":2128,"num_object_clones":0,"num_object_copies":6384,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2128,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11312,"num_write_kb":430080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":178,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:41.570594+0000","last_change":"2023-10-01T10:37:05.023034+0000","last_active":"2023-10-01T10:54:41.570594+0000","last_peered":"2023-10-01T10:54:41.570594+0000","last_clean":"2023-10-01T10:54:41.570594+0000","last_became_active":"2023-10-01T10:37:05.022670+0000","last_became_peered":"2023-10-01T10:37:05.022670+0000","last_unstale":"2023-10-01T10:54:41.570594+0000","last_undegraded":"2023-10-01T10:54:41.570594+0000","last_fullsized":"2023-10-01T10:54:41.570594+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":189,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:41.570717+0000","last_change":"2023-10-01T10:38:16.027472+0000","last_active":"2023-10-01T10:54:41.570717+0000","last_peered":"2023-10-01T10:54:41.570717+0000","last_clean":"2023-10-01T10:54:41.570717+0000","last_became_active":"2023-10-01T10:38:16.026810+0000","last_became_peered":"2023-10-01T10:38:16.026810+0000","last_unstale":"2023-10-01T10:54:41.570717+0000","last_undegraded":"2023-10-01T10:54:41.570717+0000","last_fullsized":"2023-10-01T10:54:41.570717+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:30:06.978187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":153,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:41.559492+0000","last_change":"2023-10-01T10:38:16.035705+0000","last_active":"2023-10-01T10:54:41.559492+0000","last_peered":"2023-10-01T10:54:41.559492+0000","last_clean":"2023-10-01T10:54:41.559492+0000","last_became_active":"2023-10-01T10:38:16.035389+0000","last_became_peered":"2023-10-01T10:38:16.035389+0000","last_unstale":"2023-10-01T10:54:41.559492+0000","last_undegraded":"2023-10-01T10:54:41.559492+0000","last_fullsized":"2023-10-01T10:54:41.559492+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:16:10.201409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.13","version":"208'10954","reported_seq":15923,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-01T10:54:43.296635+0000","last_change":"2023-10-01T10:54:42.592873+0000","last_active":"2023-10-01T10:54:43.296635+0000","last_peered":"2023-10-01T10:54:43.296635+0000","last_clean":"2023-10-01T10:54:43.296635+0000","last_became_active":"2023-10-01T10:54:42.567693+0000","last_became_peered":"2023-10-01T10:54:42.567693+0000","last_unstale":"2023-10-01T10:54:43.296635+0000","last_undegraded":"2023-10-01T10:54:43.296635+0000","last_fullsized":"2023-10-01T10:54:43.296635+0000","mapping_epoch":207,"log_start":"179'1900","ondisk_log_start":"179'1900","created":189,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":9054,"log_dups_size":0,"ondisk_log_size":9054,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72581120,"num_objects":1108,"num_object_clones":0,"num_object_copies":3324,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1108,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5698,"num_write_kb":217760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1224,"num_bytes_recovered":80216064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":172,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:41.570168+0000","last_change":"2023-10-01T10:38:16.028491+0000","last_active":"2023-10-01T10:54:41.570168+0000","last_peered":"2023-10-01T10:54:41.570168+0000","last_clean":"2023-10-01T10:54:41.570168+0000","last_became_active":"2023-10-01T10:38:16.028175+0000","last_became_peered":"2023-10-01T10:38:16.028175+0000","last_unstale":"2023-10-01T10:54:41.570168+0000","last_undegraded":"2023-10-01T10:54:41.570168+0000","last_fullsized":"2023-10-01T10:54:41.570168+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:44:29.083129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":194,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:41.564323+0000","last_change":"2023-10-01T10:37:42.762360+0000","last_active":"2023-10-01T10:54:41.564323+0000","last_peered":"2023-10-01T10:54:41.564323+0000","last_clean":"2023-10-01T10:54:41.564323+0000","last_became_active":"2023-10-01T10:37:42.761936+0000","last_became_peered":"2023-10-01T10:37:42.761936+0000","last_unstale":"2023-10-01T10:54:41.564323+0000","last_undegraded":"2023-10-01T10:54:41.564323+0000","last_fullsized":"2023-10-01T10:54:41.564323+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:50.637582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.14","version":"208'10787","reported_seq":10853,"reported_epoch":208,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-01T10:54:43.129022+0000","last_change":"2023-10-01T10:54:36.691887+0000","last_active":"2023-10-01T10:54:43.129022+0000","last_peered":"2023-10-01T10:54:43.129022+0000","last_clean":"2023-10-01T10:54:34.324796+0000","last_became_active":"2023-10-01T10:54:36.667649+0000","last_became_peered":"2023-10-01T10:54:36.667649+0000","last_unstale":"2023-10-01T10:54:43.129022+0000","last_undegraded":"2023-10-01T10:54:43.129022+0000","last_fullsized":"2023-10-01T10:54:43.129022+0000","mapping_epoch":201,"log_start":"179'1700","ondisk_log_start":"179'1700","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":9087,"log_dups_size":0,"ondisk_log_size":9087,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73236480,"num_objects":1118,"num_object_clones":0,"num_object_copies":3354,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2236,"num_objects_unfound":0,"num_objects_dirty":1118,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5620,"num_write_kb":215584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[2,6,7],"avail_no_missing":["2","6","7"],"object_location_counts":[{"shards":"2,6,7","objects":1118}],"blocked_by":[],"up_primary":4,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":192,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:41.568003+0000","last_change":"2023-10-01T10:38:16.037809+0000","last_active":"2023-10-01T10:54:41.568003+0000","last_peered":"2023-10-01T10:54:41.568003+0000","last_clean":"2023-10-01T10:54:41.568003+0000","last_became_active":"2023-10-01T10:38:16.037318+0000","last_became_peered":"2023-10-01T10:38:16.037318+0000","last_unstale":"2023-10-01T10:54:41.568003+0000","last_undegraded":"2023-10-01T10:54:41.568003+0000","last_fullsized":"2023-10-01T10:54:41.568003+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":202,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:41.560143+0000","last_change":"2023-10-01T10:38:16.034282+0000","last_active":"2023-10-01T10:54:41.560143+0000","last_peered":"2023-10-01T10:54:41.560143+0000","last_clean":"2023-10-01T10:54:41.560143+0000","last_became_active":"2023-10-01T10:38:16.033820+0000","last_became_peered":"2023-10-01T10:38:16.033820+0000","last_unstale":"2023-10-01T10:54:41.560143+0000","last_undegraded":"2023-10-01T10:54:41.560143+0000","last_fullsized":"2023-10-01T10:54:41.560143+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:48:23.815832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.15","version":"208'10860","reported_seq":12050,"reported_epoch":208,"state":"active+remapped+backfilling","last_fresh":"2023-10-01T10:54:43.345084+0000","last_change":"2023-10-01T10:54:41.305667+0000","last_active":"2023-10-01T10:54:43.345084+0000","last_peered":"2023-10-01T10:54:43.345084+0000","last_clean":"2023-10-01T10:54:35.641183+0000","last_became_active":"2023-10-01T10:54:38.671972+0000","last_became_peered":"2023-10-01T10:54:38.671972+0000","last_unstale":"2023-10-01T10:54:43.345084+0000","last_undegraded":"2023-10-01T10:54:43.345084+0000","last_fullsized":"2023-10-01T10:54:43.345084+0000","mapping_epoch":203,"log_start":"178'400","ondisk_log_start":"178'400","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":10460,"log_dups_size":0,"ondisk_log_size":10460,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72384512,"num_objects":1105,"num_object_clones":0,"num_object_copies":3315,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2481,"num_objects_unfound":0,"num_objects_dirty":1105,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5643,"num_write_kb":215904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":280,"num_bytes_recovered":18350080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[5,6,7],"avail_no_missing":["5","6","7"],"object_location_counts":[{"shards":"5,6,7","objects":1105}],"blocked_by":[],"up_primary":2,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":177,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:41.568362+0000","last_change":"2023-10-01T10:38:16.038077+0000","last_active":"2023-10-01T10:54:41.568362+0000","last_peered":"2023-10-01T10:54:41.568362+0000","last_clean":"2023-10-01T10:54:41.568362+0000","last_became_active":"2023-10-01T10:38:16.037313+0000","last_became_peered":"2023-10-01T10:38:16.037313+0000","last_unstale":"2023-10-01T10:54:41.568362+0000","last_undegraded":"2023-10-01T10:54:41.568362+0000","last_fullsized":"2023-10-01T10:54:41.568362+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":202,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:41.559708+0000","last_change":"2023-10-01T10:38:16.025600+0000","last_active":"2023-10-01T10:54:41.559708+0000","last_peered":"2023-10-01T10:54:41.559708+0000","last_clean":"2023-10-01T10:54:41.559708+0000","last_became_active":"2023-10-01T10:38:16.025252+0000","last_became_peered":"2023-10-01T10:38:16.025252+0000","last_unstale":"2023-10-01T10:54:41.559708+0000","last_undegraded":"2023-10-01T10:54:41.559708+0000","last_fullsized":"2023-10-01T10:54:41.559708+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:24:18.871963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.16","version":"208'10802","reported_seq":12308,"reported_epoch":208,"state":"active+remapped+backfilling","last_fresh":"2023-10-01T10:54:44.287637+0000","last_change":"2023-10-01T10:54:41.305840+0000","last_active":"2023-10-01T10:54:44.287637+0000","last_peered":"2023-10-01T10:54:44.287637+0000","last_clean":"2023-10-01T10:54:38.200493+0000","last_became_active":"2023-10-01T10:54:40.685657+0000","last_became_peered":"2023-10-01T10:54:40.685657+0000","last_unstale":"2023-10-01T10:54:44.287637+0000","last_undegraded":"2023-10-01T10:54:44.287637+0000","last_fullsized":"2023-10-01T10:54:44.287637+0000","mapping_epoch":205,"log_start":"178'600","ondisk_log_start":"178'600","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":5,"last_scrub":"191'10469","last_scrub_stamp":"2023-10-01T10:54:21.123431+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:54:21.123431+0000","objects_scrubbed":1417,"log_size":10202,"log_dups_size":0,"ondisk_log_size":10202,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:31:59.233459+0000","scrub_duration":0.421937534,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70713344,"num_objects":1079,"num_object_clones":0,"num_object_copies":3237,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2226,"num_objects_unfound":0,"num_objects_dirty":1079,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":338,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":350,"num_bytes_recovered":22937600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[0,1,2],"avail_no_missing":["0","1","2"],"object_location_counts":[{"shards":"0,1,2","objects":1078},{"shards":"5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":190,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-01T10:54:40.557457+0000","last_change":"2023-10-01T10:37:42.761243+0000","last_active":"2023-10-01T10:54:40.557457+0000","last_peered":"2023-10-01T10:54:40.557457+0000","last_clean":"2023-10-01T10:54:40.557457+0000","last_became_active":"2023-10-01T10:37:42.760834+0000","last_became_peered":"2023-10-01T10:37:42.760834+0000","last_unstale":"2023-10-01T10:54:40.557457+0000","last_undegraded":"2023-10-01T10:54:40.557457+0000","last_fullsized":"2023-10-01T10:54:40.557457+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:37:21.976095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":202,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-10-01T10:54:40.556847+0000","last_change":"2023-10-01T10:37:42.765142+0000","last_active":"2023-10-01T10:54:40.556847+0000","last_peered":"2023-10-01T10:54:40.556847+0000","last_clean":"2023-10-01T10:54:40.556847+0000","last_became_active":"2023-10-01T10:37:42.764770+0000","last_became_peered":"2023-10-01T10:37:42.764770+0000","last_unstale":"2023-10-01T10:54:40.556847+0000","last_undegraded":"2023-10-01T10:54:40.556847+0000","last_fullsized":"2023-10-01T10:54:40.556847+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:23:33.527876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.17","version":"208'10680","reported_seq":10731,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-10-01T10:54:43.214205+0000","last_change":"2023-10-01T10:54:15.665568+0000","last_active":"2023-10-01T10:54:43.214205+0000","last_peered":"2023-10-01T10:54:43.214205+0000","last_clean":"2023-10-01T10:54:43.214205+0000","last_became_active":"2023-10-01T10:54:15.641394+0000","last_became_peered":"2023-10-01T10:54:15.641394+0000","last_unstale":"2023-10-01T10:54:43.214205+0000","last_undegraded":"2023-10-01T10:54:43.214205+0000","last_fullsized":"2023-10-01T10:54:43.214205+0000","mapping_epoch":189,"log_start":"178'200","ondisk_log_start":"178'200","created":189,"last_epoch_clean":190,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:51:00.928758+0000","last_clean_scrub_stamp":"2023-10-01T10:51:00.928758+0000","objects_scrubbed":0,"log_size":10480,"log_dups_size":0,"ondisk_log_size":10480,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71172096,"num_objects":1086,"num_object_clones":0,"num_object_copies":3258,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1086,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5567,"num_write_kb":212896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":174,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:41.569810+0000","last_change":"2023-10-01T10:37:05.021621+0000","last_active":"2023-10-01T10:54:41.569810+0000","last_peered":"2023-10-01T10:54:41.569810+0000","last_clean":"2023-10-01T10:54:41.569810+0000","last_became_active":"2023-10-01T10:37:05.021173+0000","last_became_peered":"2023-10-01T10:37:05.021173+0000","last_unstale":"2023-10-01T10:54:41.569810+0000","last_undegraded":"2023-10-01T10:54:41.569810+0000","last_fullsized":"2023-10-01T10:54:41.569810+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:13:25.364499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":168,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-10-01T10:54:41.559230+0000","last_change":"2023-10-01T10:38:16.031427+0000","last_active":"2023-10-01T10:54:41.559230+0000","last_peered":"2023-10-01T10:54:41.559230+0000","last_clean":"2023-10-01T10:54:41.559230+0000","last_became_active":"2023-10-01T10:38:16.029786+0000","last_became_peered":"2023-10-01T10:38:16.029786+0000","last_unstale":"2023-10-01T10:54:41.559230+0000","last_undegraded":"2023-10-01T10:54:41.559230+0000","last_fullsized":"2023-10-01T10:54:41.559230+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:53:22.796191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":26,"stat_sum":{"num_bytes":2275639320,"num_objects":34726,"num_object_clones":0,"num_object_copies":104178,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6943,"num_objects_unfound":0,"num_objects_dirty":34726,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":142456,"num_write_kb":5356545,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5412,"num_bytes_recovered":354615296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7513518080,"data_stored":7513505864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":236797,"ondisk_log_size":236797,"up":78,"acting":78,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30319,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430713,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1881244,"kb_used_data":986900,"kb_used_omap":0,"kb_used_meta":894336,"kb_avail":91863908,"statfs":{"total":95995035648,"available":94068641792,"internally_reserved":0,"allocated":1010585600,"data_stored":1010067663,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":915800064},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:54:10 2023","interfaces":[{"interface":"back","average":{"1min":2.5310000000000001,"5min":3.4830000000000001,"15min":2.452},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":14.315,"5min":137.34200000000001,"15min":137.34200000000001},"last":3.875},{"interface":"front","average":{"1min":2.3159999999999998,"5min":3.3730000000000002,"15min":2.2709999999999999},"min":{"1min":0.32500000000000001,"5min":0.317,"15min":0.28799999999999998},"max":{"1min":14.336,"5min":137.39500000000001,"15min":137.39500000000001},"last":3.8919999999999999}]},{"osd":1,"last update":"Sun Oct 1 10:54:17 2023","interfaces":[{"interface":"back","average":{"1min":2.706,"5min":1.629,"15min":1.8380000000000001},"min":{"1min":0.33400000000000002,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":15.662000000000001,"5min":15.662000000000001,"15min":56.110999999999997},"last":2.6320000000000001},{"interface":"front","average":{"1min":3.2890000000000001,"5min":1.512,"15min":1.8460000000000001},"min":{"1min":0.30599999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":18.768999999999998,"5min":18.768999999999998,"15min":51.258000000000003},"last":2.8010000000000002}]},{"osd":2,"last update":"Sun Oct 1 10:54:35 2023","interfaces":[{"interface":"back","average":{"1min":4.2590000000000003,"5min":2.1320000000000001,"15min":1.597},"min":{"1min":0.44400000000000001,"5min":0.315,"15min":0.315},"max":{"1min":36.308,"5min":36.308,"15min":36.308},"last":0.33300000000000002},{"interface":"front","average":{"1min":3.984,"5min":1.9179999999999999,"15min":1.6180000000000001},"min":{"1min":0.39700000000000002,"5min":0.34000000000000002,"15min":0.33200000000000002},"max":{"1min":36.198,"5min":36.198,"15min":36.198},"last":8.4860000000000007}]},{"osd":3,"last update":"Sun Oct 1 10:54:10 2023","interfaces":[{"interface":"back","average":{"1min":2.125,"5min":1.5600000000000001,"15min":1.8999999999999999},"min":{"1min":0.40100000000000002,"5min":0.318,"15min":0.318},"max":{"1min":8.6839999999999993,"5min":8.6839999999999993,"15min":51.334000000000003},"last":0.38200000000000001},{"interface":"front","average":{"1min":2.5209999999999999,"5min":1.504,"15min":1.964},"min":{"1min":0.47899999999999998,"5min":0.33400000000000002,"15min":0.313},"max":{"1min":8.9299999999999997,"5min":8.9299999999999997,"15min":51.404000000000003},"last":0.34699999999999998}]},{"osd":4,"last update":"Sun Oct 1 10:53:43 2023","interfaces":[{"interface":"back","average":{"1min":1.6899999999999999,"5min":1.194,"15min":1.3240000000000001},"min":{"1min":0.35699999999999998,"5min":0.33500000000000002,"15min":0.317},"max":{"1min":7.7800000000000002,"5min":7.7800000000000002,"15min":11.081},"last":0.44700000000000001},{"interface":"front","average":{"1min":1.097,"5min":1.083,"15min":1.393},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.32100000000000001},"max":{"1min":3.9910000000000001,"5min":4.2309999999999999,"15min":12.292},"last":0.51800000000000002}]},{"osd":5,"last update":"Sun Oct 1 10:54:17 2023","interfaces":[{"interface":"back","average":{"1min":2.6400000000000001,"5min":2.1669999999999998,"15min":1.925},"min":{"1min":0.40200000000000002,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":15.726000000000001,"5min":40.381,"15min":40.381},"last":0.48099999999999998},{"interface":"front","average":{"1min":1.887,"5min":1.883,"15min":1.9410000000000001},"min":{"1min":0.41399999999999998,"5min":0.33600000000000002,"15min":0.32200000000000001},"max":{"1min":9.1709999999999994,"5min":39.590000000000003,"15min":39.590000000000003},"last":0.46899999999999997}]},{"osd":6,"last update":"Sun Oct 1 10:54:17 2023","interfaces":[{"interface":"back","average":{"1min":1.714,"5min":1.482,"15min":1.5529999999999999},"min":{"1min":0.434,"5min":0.35799999999999998,"15min":0.34499999999999997},"max":{"1min":9.4990000000000006,"5min":9.4990000000000006,"15min":12.329000000000001},"last":0.433},{"interface":"front","average":{"1min":1.504,"5min":1.677,"15min":1.474},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.36599999999999999},"max":{"1min":4.7240000000000002,"5min":9.4329999999999998,"15min":9.4329999999999998},"last":0.39700000000000002}]}]},{"osd":6,"up_from":42,"seq":180388626947,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2083472,"kb_used_data":1186376,"kb_used_omap":0,"kb_used_meta":897088,"kb_avail":91661680,"statfs":{"total":95995035648,"available":93861560320,"internally_reserved":0,"allocated":1214849024,"data_stored":1214342938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":918618112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:54:03 2023","interfaces":[{"interface":"back","average":{"1min":1.677,"5min":1.5920000000000001,"15min":2.6419999999999999},"min":{"1min":0.28299999999999997,"5min":0.25800000000000001,"15min":0.25},"max":{"1min":6.9489999999999998,"5min":8.6129999999999995,"15min":140.524},"last":0.61099999999999999},{"interface":"front","average":{"1min":1.4419999999999999,"5min":1.748,"15min":2.641},"min":{"1min":0.30199999999999999,"5min":0.249,"15min":0.249},"max":{"1min":9.5259999999999998,"5min":12.920999999999999,"15min":140.86500000000001},"last":0.496}]},{"osd":1,"last update":"Sun Oct 1 10:54:12 2023","interfaces":[{"interface":"back","average":{"1min":1.8069999999999999,"5min":1.395,"15min":1.429},"min":{"1min":0.29199999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":7.2809999999999997,"5min":8.2560000000000002,"15min":9.673},"last":0.76500000000000001},{"interface":"front","average":{"1min":2.6619999999999999,"5min":1.7689999999999999,"15min":1.5700000000000001},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26300000000000001},"max":{"1min":9.4809999999999999,"5min":9.4809999999999999,"15min":12.968999999999999},"last":0.55400000000000005}]},{"osd":2,"last update":"Sun Oct 1 10:54:29 2023","interfaces":[{"interface":"back","average":{"1min":8.1560000000000006,"5min":4.1500000000000004,"15min":2.742},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":122.768,"5min":122.768,"15min":122.768},"last":0.73699999999999999},{"interface":"front","average":{"1min":9.1180000000000003,"5min":4.1680000000000001,"15min":2.7709999999999999},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.32100000000000001},"max":{"1min":123.02200000000001,"5min":123.02200000000001,"15min":123.02200000000001},"last":0.46400000000000002}]},{"osd":3,"last update":"Sun Oct 1 10:54:12 2023","interfaces":[{"interface":"back","average":{"1min":2.1459999999999999,"5min":1.7729999999999999,"15min":2.1469999999999998},"min":{"1min":0.35099999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":9.0350000000000001,"5min":10.452,"15min":61.817999999999998},"last":0.56899999999999995},{"interface":"front","average":{"1min":1.423,"5min":2.097,"15min":2.0699999999999998},"min":{"1min":0.41599999999999998,"5min":0.38100000000000001,"15min":0.315},"max":{"1min":3.4489999999999998,"5min":12.042,"15min":61.219000000000001},"last":0.58499999999999996}]},{"osd":4,"last update":"Sun Oct 1 10:53:51 2023","interfaces":[{"interface":"back","average":{"1min":1.3819999999999999,"5min":1.8999999999999999,"15min":1.494},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":3.746,"5min":13.1,"15min":13.1},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.89600000000000002,"5min":1.665,"15min":1.51},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.315},"max":{"1min":2.77,"5min":12.157999999999999,"15min":12.157999999999999},"last":0.54100000000000004}]},{"osd":5,"last update":"Sun Oct 1 10:54:12 2023","interfaces":[{"interface":"back","average":{"1min":2.306,"5min":1.488,"15min":1.5489999999999999},"min":{"1min":0.40899999999999997,"5min":0.39300000000000002,"15min":0.33000000000000002},"max":{"1min":7.5800000000000001,"5min":8.2279999999999998,"15min":12.398999999999999},"last":0.51900000000000002},{"interface":"front","average":{"1min":2.681,"5min":1.877,"15min":1.8200000000000001},"min":{"1min":0.42799999999999999,"5min":0.308,"15min":0.308},"max":{"1min":12.44,"5min":12.44,"15min":14.459},"last":0.77900000000000003}]},{"osd":7,"last update":"Sun Oct 1 10:54:03 2023","interfaces":[{"interface":"back","average":{"1min":1.97,"5min":2.2330000000000001,"15min":1.6379999999999999},"min":{"1min":0.33900000000000002,"5min":0.32300000000000001,"15min":0.311},"max":{"1min":8.4199999999999999,"5min":13.276999999999999,"15min":13.276999999999999},"last":0.69999999999999996},{"interface":"front","average":{"1min":1.6220000000000001,"5min":1.706,"15min":1.6599999999999999},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":7.0229999999999997,"5min":10.023,"15min":11.416},"last":0.67400000000000004}]}]},{"osd":1,"up_from":101,"seq":433791697164,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1707492,"kb_used_data":1148444,"kb_used_omap":4,"kb_used_meta":559035,"kb_avail":92037660,"statfs":{"total":95995035648,"available":94246563840,"internally_reserved":0,"allocated":1176006656,"data_stored":1175174506,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":572452750},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:54:19 2023","interfaces":[{"interface":"back","average":{"1min":1.831,"5min":1.3340000000000001,"15min":1.306},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.189},"max":{"1min":8.3610000000000007,"5min":9.1739999999999995,"15min":37.058},"last":0.71899999999999997},{"interface":"front","average":{"1min":2.1110000000000002,"5min":1.3500000000000001,"15min":1.6479999999999999},"min":{"1min":0.26600000000000001,"5min":0.192,"15min":0.192},"max":{"1min":8.6189999999999998,"5min":11.087,"15min":61.207999999999998},"last":0.85999999999999999}]},{"osd":2,"last update":"Sun Oct 1 10:53:41 2023","interfaces":[{"interface":"back","average":{"1min":11.474,"5min":3.5579999999999998,"15min":2.1859999999999999},"min":{"1min":0.25900000000000001,"5min":0.222,"15min":0.222},"max":{"1min":161.905,"5min":161.905,"15min":161.905},"last":0.59899999999999998},{"interface":"front","average":{"1min":11.602,"5min":3.4740000000000002,"15min":2.1930000000000001},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.23699999999999999},"max":{"1min":162.059,"5min":162.059,"15min":162.059},"last":0.32800000000000001}]},{"osd":3,"last update":"Sun Oct 1 10:54:19 2023","interfaces":[{"interface":"back","average":{"1min":9.9580000000000002,"5min":3.165,"15min":2.1579999999999999},"min":{"1min":0.318,"5min":0.23999999999999999,"15min":0.20000000000000001},"max":{"1min":143.73400000000001,"5min":143.73400000000001,"15min":143.73400000000001},"last":0.30399999999999999},{"interface":"front","average":{"1min":9.7690000000000001,"5min":3.181,"15min":2.2999999999999998},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.23999999999999999},"max":{"1min":143.00399999999999,"5min":143.00399999999999,"15min":143.00399999999999},"last":0.69199999999999995}]},{"osd":4,"last update":"Sun Oct 1 10:53:41 2023","interfaces":[{"interface":"back","average":{"1min":1.278,"5min":1.3320000000000001,"15min":1.4990000000000001},"min":{"1min":0.34100000000000003,"5min":0.309,"15min":0.29199999999999998},"max":{"1min":3.4529999999999998,"5min":7.2839999999999998,"15min":61.588000000000001},"last":0.40600000000000003},{"interface":"front","average":{"1min":1.228,"5min":1.5369999999999999,"15min":1.7070000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":3.6709999999999998,"5min":10.816000000000001,"15min":61.411000000000001},"last":0.84499999999999997}]},{"osd":5,"last update":"Sun Oct 1 10:54:19 2023","interfaces":[{"interface":"back","average":{"1min":1.841,"5min":1.6080000000000001,"15min":1.8340000000000001},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":9.1809999999999992,"5min":9.1809999999999992,"15min":61.560000000000002},"last":0.65300000000000002},{"interface":"front","average":{"1min":1.9730000000000001,"5min":1.722,"15min":1.7549999999999999},"min":{"1min":0.42899999999999999,"5min":0.38200000000000001,"15min":0.34699999999999998},"max":{"1min":8.7949999999999999,"5min":8.7949999999999999,"15min":61.667999999999999},"last":0.51600000000000001}]},{"osd":6,"last update":"Sun Oct 1 10:54:19 2023","interfaces":[{"interface":"back","average":{"1min":2.375,"5min":1.9530000000000001,"15min":1.798},"min":{"1min":0.45600000000000002,"5min":0.371,"15min":0.371},"max":{"1min":8.8309999999999995,"5min":8.8309999999999995,"15min":62.219999999999999},"last":0.67700000000000005},{"interface":"front","average":{"1min":2.1019999999999999,"5min":1.762,"15min":1.778},"min":{"1min":0.41499999999999998,"5min":0.38,"15min":0.38},"max":{"1min":8.8970000000000002,"5min":9.7870000000000008,"15min":61.462000000000003},"last":0.79600000000000004}]},{"osd":7,"last update":"Sun Oct 1 10:54:19 2023","interfaces":[{"interface":"back","average":{"1min":1.631,"5min":1.655,"15min":1.871},"min":{"1min":0.45400000000000001,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":9.7140000000000004,"5min":10.465,"15min":62.646999999999998},"last":0.82599999999999996},{"interface":"front","average":{"1min":2.7949999999999999,"5min":1.7869999999999999,"15min":1.8999999999999999},"min":{"1min":0.46000000000000002,"5min":0.39100000000000001,"15min":0.38100000000000001},"max":{"1min":9.1449999999999996,"5min":9.1449999999999996,"15min":62.048000000000002},"last":0.76900000000000002}]}]},{"osd":0,"up_from":86,"seq":369367187775,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1512300,"kb_used_data":823012,"kb_used_omap":5,"kb_used_meta":689274,"kb_avail":92232852,"statfs":{"total":95995035648,"available":94446440448,"internally_reserved":0,"allocated":842764288,"data_stored":841943494,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":705817210},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 10:54:19 2023","interfaces":[{"interface":"back","average":{"1min":1.964,"5min":1.3759999999999999,"15min":1.3340000000000001},"min":{"1min":0.23000000000000001,"5min":0.215,"15min":0.184},"max":{"1min":6.7850000000000001,"5min":8.6479999999999997,"15min":15.074},"last":5.6920000000000002},{"interface":"front","average":{"1min":2.1459999999999999,"5min":1.575,"15min":1.506},"min":{"1min":0.20699999999999999,"5min":0.19900000000000001,"15min":0.19700000000000001},"max":{"1min":8.2569999999999997,"5min":11.164999999999999,"15min":13.601000000000001},"last":5.6210000000000004}]},{"osd":2,"last update":"Sun Oct 1 10:54:38 2023","interfaces":[{"interface":"back","average":{"1min":1.1379999999999999,"5min":1.8819999999999999,"15min":1.4259999999999999},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.223},"max":{"1min":4.4370000000000003,"5min":42.709000000000003,"15min":42.709000000000003},"last":0.41699999999999998},{"interface":"front","average":{"1min":1.7290000000000001,"5min":2.0459999999999998,"15min":1.5089999999999999},"min":{"1min":0.29199999999999998,"5min":0.27800000000000002,"15min":0.221},"max":{"1min":4.5069999999999997,"5min":41.091999999999999,"15min":41.091999999999999},"last":0.92100000000000004}]},{"osd":3,"last update":"Sun Oct 1 10:54:19 2023","interfaces":[{"interface":"back","average":{"1min":2.5950000000000002,"5min":1.5640000000000001,"15min":1.5629999999999999},"min":{"1min":0.32000000000000001,"5min":0.31900000000000001,"15min":0.26200000000000001},"max":{"1min":6.7649999999999997,"5min":7.4260000000000002,"15min":12.944000000000001},"last":0.497},{"interface":"front","average":{"1min":2.5,"5min":1.548,"15min":1.49},"min":{"1min":0.377,"5min":0.26900000000000002,"15min":0.245},"max":{"1min":13.641,"5min":13.641,"15min":13.641},"last":1.1659999999999999}]},{"osd":4,"last update":"Sun Oct 1 10:53:52 2023","interfaces":[{"interface":"back","average":{"1min":1.651,"5min":1.5700000000000001,"15min":1.5449999999999999},"min":{"1min":0.34599999999999997,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":8.5790000000000006,"5min":9.3580000000000005,"15min":11.379},"last":1.0820000000000001},{"interface":"front","average":{"1min":2.3340000000000001,"5min":1.657,"15min":1.5169999999999999},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":6.9080000000000004,"5min":9.4009999999999998,"15min":11.362},"last":0.378}]},{"osd":5,"last update":"Sun Oct 1 10:54:19 2023","interfaces":[{"interface":"back","average":{"1min":1.7190000000000001,"5min":1.585,"15min":1.6419999999999999},"min":{"1min":0.434,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":5.0579999999999998,"5min":7.3799999999999999,"15min":12.91},"last":0.55600000000000005},{"interface":"front","average":{"1min":1.1850000000000001,"5min":1.3400000000000001,"15min":1.516},"min":{"1min":0.41199999999999998,"5min":0.35699999999999998,"15min":0.34699999999999998},"max":{"1min":4.5949999999999998,"5min":6.3739999999999997,"15min":9.8620000000000001},"last":1.0469999999999999}]},{"osd":6,"last update":"Sun Oct 1 10:54:19 2023","interfaces":[{"interface":"back","average":{"1min":1.5549999999999999,"5min":1.633,"15min":1.536},"min":{"1min":0.39800000000000002,"5min":0.373,"15min":0.36399999999999999},"max":{"1min":4.5449999999999999,"5min":6.7969999999999997,"15min":9.8279999999999994},"last":1.111},{"interface":"front","average":{"1min":1.984,"5min":1.7929999999999999,"15min":1.5960000000000001},"min":{"1min":0.37,"5min":0.37,"15min":0.36899999999999999},"max":{"1min":7.3120000000000003,"5min":10.111000000000001,"15min":10.111000000000001},"last":0.54000000000000004}]},{"osd":7,"last update":"Sun Oct 1 10:54:19 2023","interfaces":[{"interface":"back","average":{"1min":2.359,"5min":1.6970000000000001,"15min":1.673},"min":{"1min":0.48299999999999998,"5min":0.40799999999999997,"15min":0.36899999999999999},"max":{"1min":13.699,"5min":13.699,"15min":15.839},"last":1.1379999999999999},{"interface":"front","average":{"1min":2.218,"5min":1.7929999999999999,"15min":1.657},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":7.2679999999999998,"5min":8.5169999999999995,"15min":8.9440000000000008},"last":0.57899999999999996}]}]},{"osd":2,"up_from":116,"seq":498216206550,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1370372,"kb_used_data":998524,"kb_used_omap":7,"kb_used_meta":371832,"kb_avail":92374780,"statfs":{"total":95995035648,"available":94591774720,"internally_reserved":0,"allocated":1022488576,"data_stored":1021701780,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":380756049},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:54:41 2023","interfaces":[{"interface":"back","average":{"1min":1.6930000000000001,"5min":1.3120000000000001,"15min":1.98},"min":{"1min":0.23300000000000001,"5min":0.16900000000000001,"15min":0.16900000000000001},"max":{"1min":8.4870000000000001,"5min":11.013999999999999,"15min":147.87},"last":1.0600000000000001},{"interface":"front","average":{"1min":1.4299999999999999,"5min":1.413,"15min":2.0800000000000001},"min":{"1min":0.29799999999999999,"5min":0.20599999999999999,"15min":0.19500000000000001},"max":{"1min":6.1529999999999996,"5min":8.8480000000000008,"15min":148.03100000000001},"last":0.501}]},{"osd":1,"last update":"Sun Oct 1 10:54:41 2023","interfaces":[{"interface":"back","average":{"1min":1.766,"5min":2.5609999999999999,"15min":1.895},"min":{"1min":0.23899999999999999,"5min":0.217,"15min":0.217},"max":{"1min":6.8979999999999997,"5min":122.249,"15min":122.249},"last":0.60999999999999999},{"interface":"front","average":{"1min":2.3679999999999999,"5min":3.1139999999999999,"15min":2.1230000000000002},"min":{"1min":0.28799999999999998,"5min":0.245,"15min":0.20599999999999999},"max":{"1min":10.018000000000001,"5min":121.495,"15min":121.495},"last":0.40000000000000002}]},{"osd":3,"last update":"Sun Oct 1 10:54:07 2023","interfaces":[{"interface":"back","average":{"1min":2.3700000000000001,"5min":2.7829999999999999,"15min":2.0840000000000001},"min":{"1min":0.27600000000000002,"5min":0.23200000000000001,"15min":0.20999999999999999},"max":{"1min":10.811,"5min":124.393,"15min":124.393},"last":1.2589999999999999},{"interface":"front","average":{"1min":1.3280000000000001,"5min":2.8170000000000002,"15min":2.1859999999999999},"min":{"1min":0.221,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":4.9039999999999999,"5min":122.52200000000001,"15min":122.52200000000001},"last":1.1080000000000001}]},{"osd":4,"last update":"Sun Oct 1 10:53:48 2023","interfaces":[{"interface":"back","average":{"1min":1.8520000000000001,"5min":1.706,"15min":1.5529999999999999},"min":{"1min":0.45000000000000001,"5min":0.371,"15min":0.35799999999999998},"max":{"1min":7.9480000000000004,"5min":13.676,"15min":60.994},"last":0.58399999999999996},{"interface":"front","average":{"1min":3.2599999999999998,"5min":2.246,"15min":1.758},"min":{"1min":0.38,"5min":0.311,"15min":0.30299999999999999},"max":{"1min":7.9000000000000004,"5min":17.925000000000001,"15min":63.012},"last":4.3170000000000002}]},{"osd":5,"last update":"Sun Oct 1 10:54:41 2023","interfaces":[{"interface":"back","average":{"1min":1.335,"5min":1.5129999999999999,"15min":1.72},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.34699999999999998},"max":{"1min":8.5440000000000005,"5min":8.5440000000000005,"15min":61.021000000000001},"last":0.76800000000000002},{"interface":"front","average":{"1min":1.6759999999999999,"5min":2.077,"15min":1.879},"min":{"1min":0.49299999999999999,"5min":0.36499999999999999,"15min":0.34100000000000003},"max":{"1min":10.130000000000001,"5min":10.130000000000001,"15min":63.145000000000003},"last":0.55300000000000005}]},{"osd":6,"last update":"Sun Oct 1 10:54:41 2023","interfaces":[{"interface":"back","average":{"1min":1.4059999999999999,"5min":1.9790000000000001,"15min":1.8169999999999999},"min":{"1min":0.44400000000000001,"5min":0.38200000000000001,"15min":0.32100000000000001},"max":{"1min":6.3860000000000001,"5min":9.6289999999999996,"15min":63.289000000000001},"last":0.44400000000000001},{"interface":"front","average":{"1min":2.2290000000000001,"5min":1.5880000000000001,"15min":1.5600000000000001},"min":{"1min":0.38300000000000001,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":8.5310000000000006,"5min":8.5310000000000006,"15min":60.579000000000001},"last":1.214}]},{"osd":7,"last update":"Sun Oct 1 10:54:41 2023","interfaces":[{"interface":"back","average":{"1min":1.772,"5min":1.373,"15min":1.5049999999999999},"min":{"1min":0.46000000000000002,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":8.6859999999999999,"5min":8.6859999999999999,"15min":60.493000000000002},"last":1.3029999999999999},{"interface":"front","average":{"1min":1.7769999999999999,"5min":1.6339999999999999,"15min":1.8029999999999999},"min":{"1min":0.40100000000000002,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":6.4960000000000004,"5min":8.4100000000000001,"15min":60.956000000000003},"last":0.94999999999999996}]}]},{"osd":3,"up_from":121,"seq":519691043022,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1447660,"kb_used_data":969508,"kb_used_omap":2,"kb_used_meta":478077,"kb_avail":92297492,"statfs":{"total":95995035648,"available":94512631808,"internally_reserved":0,"allocated":992776192,"data_stored":992004211,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":489550901},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:54:11 2023","interfaces":[{"interface":"back","average":{"1min":1.954,"5min":4.5389999999999997,"15min":2.6800000000000002},"min":{"1min":0.26600000000000001,"5min":0.192,"15min":0.187},"max":{"1min":6.5069999999999997,"5min":129.953,"15min":129.953},"last":0.23999999999999999},{"interface":"front","average":{"1min":2.4769999999999999,"5min":4.8630000000000004,"15min":2.9980000000000002},"min":{"1min":0.27700000000000002,"5min":0.189,"15min":0.17299999999999999},"max":{"1min":6.3700000000000001,"5min":129.53999999999999,"15min":129.53999999999999},"last":0.25900000000000001}]},{"osd":1,"last update":"Sun Oct 1 10:54:11 2023","interfaces":[{"interface":"back","average":{"1min":2.3700000000000001,"5min":1.833,"15min":1.806},"min":{"1min":0.41399999999999998,"5min":0.193,"15min":0.193},"max":{"1min":9.2870000000000008,"5min":9.6560000000000006,"15min":77.361000000000004},"last":0.54000000000000004},{"interface":"front","average":{"1min":2.2280000000000002,"5min":1.855,"15min":1.9159999999999999},"min":{"1min":0.29699999999999999,"5min":0.247,"15min":0.24099999999999999},"max":{"1min":13.529999999999999,"5min":13.529999999999999,"15min":77.867000000000004},"last":0.61499999999999999}]},{"osd":2,"last update":"Sun Oct 1 10:54:11 2023","interfaces":[{"interface":"back","average":{"1min":2.149,"5min":1.8029999999999999,"15min":1.3819999999999999},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.222},"max":{"1min":6.3010000000000002,"5min":15.148,"15min":15.148},"last":0.33300000000000002},{"interface":"front","average":{"1min":2.4729999999999999,"5min":1.879,"15min":1.615},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":9.2289999999999992,"5min":9.8409999999999993,"15min":75.299999999999997},"last":0.35699999999999998}]},{"osd":4,"last update":"Sun Oct 1 10:53:46 2023","interfaces":[{"interface":"back","average":{"1min":1.321,"5min":1.454,"15min":1.5249999999999999},"min":{"1min":0.42899999999999999,"5min":0.39400000000000002,"15min":0.34599999999999997},"max":{"1min":5.9809999999999999,"5min":9.7370000000000001,"15min":75.388999999999996},"last":0.46100000000000002},{"interface":"front","average":{"1min":1.5289999999999999,"5min":1.53,"15min":1.681},"min":{"1min":0.377,"5min":0.308,"15min":0.29999999999999999},"max":{"1min":4.4240000000000004,"5min":6.375,"15min":75.165999999999997},"last":0.36799999999999999}]},{"osd":5,"last update":"Sun Oct 1 10:54:11 2023","interfaces":[{"interface":"back","average":{"1min":2.7109999999999999,"5min":1.661,"15min":1.871},"min":{"1min":0.441,"5min":0.37,"15min":0.33100000000000002},"max":{"1min":13.333,"5min":13.333,"15min":75.221000000000004},"last":0.86399999999999999},{"interface":"front","average":{"1min":1.6259999999999999,"5min":1.5640000000000001,"15min":1.9279999999999999},"min":{"1min":0.38900000000000001,"5min":0.34999999999999998,"15min":0.33200000000000002},"max":{"1min":6.8150000000000004,"5min":7.1820000000000004,"15min":75.231999999999999},"last":0.876}]},{"osd":6,"last update":"Sun Oct 1 10:54:11 2023","interfaces":[{"interface":"back","average":{"1min":2.7490000000000001,"5min":2.0339999999999998,"15min":2.0150000000000001},"min":{"1min":0.58299999999999996,"5min":0.35399999999999998,"15min":0.34999999999999998},"max":{"1min":8.8740000000000006,"5min":8.8740000000000006,"15min":75.257000000000005},"last":0.5},{"interface":"front","average":{"1min":1.873,"5min":1.9330000000000001,"15min":2.052},"min":{"1min":0.498,"5min":0.435,"15min":0.33500000000000002},"max":{"1min":7.181,"5min":7.181,"15min":75.194000000000003},"last":0.48699999999999999}]},{"osd":7,"last update":"Sun Oct 1 10:54:11 2023","interfaces":[{"interface":"back","average":{"1min":1.669,"5min":1.6200000000000001,"15min":1.843},"min":{"1min":0.43099999999999999,"5min":0.376,"15min":0.35099999999999998},"max":{"1min":6.8540000000000001,"5min":7.2080000000000002,"15min":75.138999999999996},"last":0.66900000000000004},{"interface":"front","average":{"1min":2.258,"5min":1.825,"15min":1.8160000000000001},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":5.9429999999999996,"5min":9.8059999999999992,"15min":75.355000000000004},"last":0.51800000000000002}]}]},{"osd":4,"up_from":126,"seq":541165879496,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":998476,"kb_used_data":651332,"kb_used_omap":10,"kb_used_meta":347061,"kb_avail":92746676,"statfs":{"total":95995035648,"available":94972596224,"internally_reserved":0,"allocated":666963968,"data_stored":666169122,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":355391076},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:53:49 2023","interfaces":[{"interface":"back","average":{"1min":1.55,"5min":2.1429999999999998,"15min":1.855},"min":{"1min":0.27300000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":8.032,"5min":49.917000000000002,"15min":62.576000000000001},"last":0.93700000000000006},{"interface":"front","average":{"1min":1.8300000000000001,"5min":2.286,"15min":1.841},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.251},"max":{"1min":8.7409999999999997,"5min":50.042000000000002,"15min":64.941000000000003},"last":0.58299999999999996}]},{"osd":1,"last update":"Sun Oct 1 10:53:49 2023","interfaces":[{"interface":"back","average":{"1min":1.889,"5min":1.8879999999999999,"15min":1.857},"min":{"1min":0.29199999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":7.2069999999999999,"5min":29.344000000000001,"15min":97.622},"last":0.439},{"interface":"front","average":{"1min":2.2440000000000002,"5min":2.0630000000000002,"15min":2.016},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.28599999999999998},"max":{"1min":19.719999999999999,"5min":29.251999999999999,"15min":97.346999999999994},"last":1.0740000000000001}]},{"osd":2,"last update":"Sun Oct 1 10:53:49 2023","interfaces":[{"interface":"back","average":{"1min":2.0619999999999998,"5min":1.8029999999999999,"15min":1.903},"min":{"1min":0.40300000000000002,"5min":0.30499999999999999,"15min":0.29899999999999999},"max":{"1min":8.9930000000000003,"5min":14.835000000000001,"15min":117.229},"last":1.127},{"interface":"front","average":{"1min":2.004,"5min":1.784,"15min":1.8140000000000001},"min":{"1min":0.33300000000000002,"5min":0.316,"15min":0.29699999999999999},"max":{"1min":8.6809999999999992,"5min":12.407999999999999,"15min":116.922},"last":0.503}]},{"osd":3,"last update":"Sun Oct 1 10:53:49 2023","interfaces":[{"interface":"back","average":{"1min":2.2789999999999999,"5min":2.056,"15min":2.1480000000000001},"min":{"1min":0.307,"5min":0.307,"15min":0.29499999999999998},"max":{"1min":8.8100000000000005,"5min":30.637,"15min":61.398000000000003},"last":0.69499999999999995},{"interface":"front","average":{"1min":1.7809999999999999,"5min":1.8879999999999999,"15min":2.073},"min":{"1min":0.41299999999999998,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":5.2599999999999998,"5min":30.242999999999999,"15min":61.463000000000001},"last":0.84299999999999997}]},{"osd":5,"last update":"Sun Oct 1 10:53:49 2023","interfaces":[{"interface":"back","average":{"1min":1.5089999999999999,"5min":1.7430000000000001,"15min":1.47},"min":{"1min":0.34899999999999998,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":8.7910000000000004,"5min":12.601000000000001,"15min":12.601000000000001},"last":0.751},{"interface":"front","average":{"1min":1.226,"5min":1.506,"15min":1.5129999999999999},"min":{"1min":0.309,"5min":0.309,"15min":0.28999999999999998},"max":{"1min":7.0860000000000003,"5min":11.725,"15min":16.779},"last":1.1799999999999999}]},{"osd":6,"last update":"Sun Oct 1 10:53:49 2023","interfaces":[{"interface":"back","average":{"1min":2.0449999999999999,"5min":2.0499999999999998,"15min":1.6040000000000001},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.33800000000000002},"max":{"1min":8.7710000000000008,"5min":12.563000000000001,"15min":13.646000000000001},"last":0.79500000000000004},{"interface":"front","average":{"1min":1.3919999999999999,"5min":1.992,"15min":1.6060000000000001},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":5.3899999999999997,"5min":10.537000000000001,"15min":14.446999999999999},"last":0.90600000000000003}]},{"osd":7,"last update":"Sun Oct 1 10:53:49 2023","interfaces":[{"interface":"back","average":{"1min":1.633,"5min":3.1699999999999999,"15min":1.9379999999999999},"min":{"1min":0.36699999999999999,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":8.2289999999999992,"5min":119.56100000000001,"15min":119.56100000000001},"last":1.01},{"interface":"front","average":{"1min":1.5129999999999999,"5min":3,"15min":2.1509999999999998},"min":{"1min":0.41799999999999998,"5min":0.39200000000000002,"15min":0.30399999999999999},"max":{"1min":5.5229999999999997,"5min":120.44499999999999,"15min":120.44499999999999},"last":1.2310000000000001}]}]},{"osd":5,"up_from":36,"seq":154618823177,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1550912,"kb_used_data":646648,"kb_used_omap":0,"kb_used_meta":904256,"kb_avail":92194240,"statfs":{"total":95995035648,"available":94406901760,"internally_reserved":0,"allocated":662167552,"data_stored":661673102,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":925958144},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:54:09 2023","interfaces":[{"interface":"back","average":{"1min":1.0209999999999999,"5min":2.5329999999999999,"15min":1.9490000000000001},"min":{"1min":0.29799999999999999,"5min":0.28000000000000003,"15min":0.24099999999999999},"max":{"1min":5.7910000000000004,"5min":123.39700000000001,"15min":123.39700000000001},"last":1.8580000000000001},{"interface":"front","average":{"1min":0.98099999999999998,"5min":2.7480000000000002,"15min":2.1619999999999999},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.26900000000000002},"max":{"1min":3.464,"5min":124.051,"15min":124.051},"last":4.423}]},{"osd":1,"last update":"Sun Oct 1 10:54:09 2023","interfaces":[{"interface":"back","average":{"1min":2.2839999999999998,"5min":1.498,"15min":2.331},"min":{"1min":0.28799999999999998,"5min":0.27700000000000002,"15min":0.27100000000000002},"max":{"1min":9.766,"5min":9.766,"15min":202.96600000000001},"last":1.8149999999999999},{"interface":"front","average":{"1min":1.9219999999999999,"5min":1.661,"15min":2.6549999999999998},"min":{"1min":0.35199999999999998,"5min":0.32500000000000001,"15min":0.28100000000000003},"max":{"1min":7.7290000000000001,"5min":7.7290000000000001,"15min":203.05099999999999},"last":0.81299999999999994}]},{"osd":2,"last update":"Sun Oct 1 10:54:43 2023","interfaces":[{"interface":"back","average":{"1min":1.7649999999999999,"5min":1.738,"15min":1.595},"min":{"1min":0.38600000000000001,"5min":0.307,"15min":0.29199999999999998},"max":{"1min":8.5129999999999999,"5min":26.242000000000001,"15min":26.242000000000001},"last":2.0139999999999998},{"interface":"front","average":{"1min":1.8029999999999999,"5min":1.9099999999999999,"15min":1.899},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":5.5490000000000004,"5min":25.643999999999998,"15min":25.643999999999998},"last":0.56499999999999995}]},{"osd":3,"last update":"Sun Oct 1 10:54:13 2023","interfaces":[{"interface":"back","average":{"1min":10.741,"5min":3.452,"15min":3.0870000000000002},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.316},"max":{"1min":166.29599999999999,"5min":166.29599999999999,"15min":166.29599999999999},"last":1.2170000000000001},{"interface":"front","average":{"1min":10.885,"5min":3.7269999999999999,"15min":2.9900000000000002},"min":{"1min":0.36499999999999999,"5min":0.34799999999999998,"15min":0.34699999999999998},"max":{"1min":166.565,"5min":166.565,"15min":166.565},"last":1.0029999999999999}]},{"osd":4,"last update":"Sun Oct 1 10:54:43 2023","interfaces":[{"interface":"back","average":{"1min":1.1970000000000001,"5min":1.4690000000000001,"15min":1.5169999999999999},"min":{"1min":0.40699999999999997,"5min":0.313,"15min":0.29399999999999998},"max":{"1min":3.9409999999999998,"5min":9.8889999999999993,"15min":15.170999999999999},"last":0.92000000000000004},{"interface":"front","average":{"1min":0.98699999999999999,"5min":1.5289999999999999,"15min":1.5369999999999999},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.316},"max":{"1min":3.3820000000000001,"5min":9.7270000000000003,"15min":9.7270000000000003},"last":1.944}]},{"osd":6,"last update":"Sun Oct 1 10:54:13 2023","interfaces":[{"interface":"back","average":{"1min":1.6160000000000001,"5min":2.3740000000000001,"15min":1.8160000000000001},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":9.0790000000000006,"5min":61.768999999999998,"15min":61.768999999999998},"last":1.901},{"interface":"front","average":{"1min":1.6339999999999999,"5min":2.3079999999999998,"15min":1.99},"min":{"1min":0.38700000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":5.9690000000000003,"5min":62.396999999999998,"15min":62.396999999999998},"last":0.49199999999999999}]},{"osd":7,"last update":"Sun Oct 1 10:54:13 2023","interfaces":[{"interface":"back","average":{"1min":2.4060000000000001,"5min":1.907,"15min":1.9139999999999999},"min":{"1min":0.42999999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":9.1470000000000002,"5min":13.673999999999999,"15min":13.673999999999999},"last":0.623},{"interface":"front","average":{"1min":1.5589999999999999,"5min":1.603,"15min":1.7250000000000001},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38},"max":{"1min":9.1340000000000003,"5min":9.6929999999999996,"15min":11.635},"last":1.9730000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":828047360,"data_stored":828047360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1159528448,"data_stored":1159528448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1015222272,"data_stored":1015218200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":994181120,"data_stored":994181120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":656084992,"data_stored":656080920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":658636800,"data_stored":658636800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1197477888,"data_stored":1197473816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1004339200,"data_stored":1004339200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T10:54:49.327 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_6 pgp_num 26 2023-10-01T10:54:49.382 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:49.412 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:49.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:49.415 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:49.516 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:49.540 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:49.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:49.548 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:54:49.649 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:49.676 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:49.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:49.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:54:49.778 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:49.804 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:49.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:49.808 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:49.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:49 smithi173 bash[58816]: cluster 2023-10-01T10:54:48.031509+0000 mgr.y (mgr.25072) 1119 : cluster 0 pgmap v1098: 99 pgs: 1 remapped+peering, 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 95 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 394 op/s; 6943/101832 objects misplaced (6.818%); 15 MiB/s, 245 objects/s recovering 2023-10-01T10:54:49.909 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:49.931 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:49.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:49.935 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:54:50.035 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:50.058 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:50.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:50.060 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:54:50.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:49 smithi078 bash[75299]: cluster 2023-10-01T10:54:48.031509+0000 mgr.y (mgr.25072) 1119 : cluster 0 pgmap v1098: 99 pgs: 1 remapped+peering, 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 95 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 394 op/s; 6943/101832 objects misplaced (6.818%); 15 MiB/s, 245 objects/s recovering 2023-10-01T10:54:50.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:49 smithi078 bash[78274]: cluster 2023-10-01T10:54:48.031509+0000 mgr.y (mgr.25072) 1119 : cluster 0 pgmap v1098: 99 pgs: 1 remapped+peering, 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 95 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 394 op/s; 6943/101832 objects misplaced (6.818%); 15 MiB/s, 245 objects/s recovering 2023-10-01T10:54:50.161 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:50.186 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:50.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:50.191 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:54:50.291 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:50.320 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:50.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:50.325 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:54:50.426 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:50.436 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:54:50 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:54:50] "GET /metrics HTTP/1.1" 200 36333 "" "Prometheus/2.33.4" 2023-10-01T10:54:50.453 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:50.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:50.455 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:50.556 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:50.583 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:50.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:50.586 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:50.686 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:50.713 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:50.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:50.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:50.816 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:50.820 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:54:50.846 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:50.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:50.858 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:54:50.958 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:50.967 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:50 smithi078 bash[75299]: cluster 2023-10-01T10:54:49.780904+0000 osd.5 (osd.5) 170 : cluster 3 failed to encode map e212 with expected crc 2023-10-01T10:54:50.968 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:50 smithi078 bash[75299]: cluster 2023-10-01T10:54:49.783937+0000 osd.5 (osd.5) 171 : cluster 3 failed to encode map e212 with expected crc 2023-10-01T10:54:50.968 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:50 smithi078 bash[75299]: cluster 2023-10-01T10:54:49.784101+0000 osd.6 (osd.6) 137 : cluster 3 failed to encode map e212 with expected crc 2023-10-01T10:54:50.968 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:50 smithi078 bash[75299]: cluster 2023-10-01T10:54:49.785118+0000 mon.a (mon.0) 1399 : cluster 0 osdmap e212: 8 total, 8 up, 8 in 2023-10-01T10:54:50.968 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:50 smithi078 bash[75299]: cluster 2023-10-01T10:54:49.788585+0000 osd.7 (osd.7) 173 : cluster 3 failed to encode map e212 with expected crc 2023-10-01T10:54:50.968 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:50 smithi078 bash[75299]: cluster 2023-10-01T10:54:49.791509+0000 osd.7 (osd.7) 174 : cluster 3 failed to encode map e212 with expected crc 2023-10-01T10:54:50.968 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:50 smithi078 bash[78274]: cluster 2023-10-01T10:54:49.780904+0000 osd.5 (osd.5) 170 : cluster 3 failed to encode map e212 with expected crc 2023-10-01T10:54:50.968 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:50 smithi078 bash[78274]: cluster 2023-10-01T10:54:49.783937+0000 osd.5 (osd.5) 171 : cluster 3 failed to encode map e212 with expected crc 2023-10-01T10:54:50.968 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:50 smithi078 bash[78274]: cluster 2023-10-01T10:54:49.784101+0000 osd.6 (osd.6) 137 : cluster 3 failed to encode map e212 with expected crc 2023-10-01T10:54:50.969 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:50 smithi078 bash[78274]: cluster 2023-10-01T10:54:49.785118+0000 mon.a (mon.0) 1399 : cluster 0 osdmap e212: 8 total, 8 up, 8 in 2023-10-01T10:54:50.969 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:50 smithi078 bash[78274]: cluster 2023-10-01T10:54:49.788585+0000 osd.7 (osd.7) 173 : cluster 3 failed to encode map e212 with expected crc 2023-10-01T10:54:50.969 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:50 smithi078 bash[78274]: cluster 2023-10-01T10:54:49.791509+0000 osd.7 (osd.7) 174 : cluster 3 failed to encode map e212 with expected crc 2023-10-01T10:54:50.992 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:50.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:50.996 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:51.096 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:51.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:50 smithi173 bash[58816]: cluster 2023-10-01T10:54:49.780904+0000 osd.5 (osd.5) 170 : cluster 3 failed to encode map e212 with expected crc 2023-10-01T10:54:51.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:50 smithi173 bash[58816]: cluster 2023-10-01T10:54:49.783937+0000 osd.5 (osd.5) 171 : cluster 3 failed to encode map e212 with expected crc 2023-10-01T10:54:51.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:50 smithi173 bash[58816]: cluster 2023-10-01T10:54:49.784101+0000 osd.6 (osd.6) 137 : cluster 3 failed to encode map e212 with expected crc 2023-10-01T10:54:51.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:50 smithi173 bash[58816]: cluster 2023-10-01T10:54:49.785118+0000 mon.a (mon.0) 1399 : cluster 0 osdmap e212: 8 total, 8 up, 8 in 2023-10-01T10:54:51.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:50 smithi173 bash[58816]: cluster 2023-10-01T10:54:49.788585+0000 osd.7 (osd.7) 173 : cluster 3 failed to encode map e212 with expected crc 2023-10-01T10:54:51.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:50 smithi173 bash[58816]: cluster 2023-10-01T10:54:49.791509+0000 osd.7 (osd.7) 174 : cluster 3 failed to encode map e212 with expected crc 2023-10-01T10:54:51.130 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:51.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:51.135 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:51.236 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:51.261 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:51.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:51.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:54:51.364 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:51.392 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:51.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:51.394 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:54:51.495 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:51.519 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:51.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:51.521 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:54:51.622 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:51.648 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:51.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:51.650 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:54:51.726 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T10:54:51.752 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:51.796 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:51.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:51.800 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:54:51.901 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:51.933 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:51.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:51.935 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:54:52.036 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:52.045 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:51 smithi173 bash[58816]: cluster 2023-10-01T10:54:50.031993+0000 mgr.y (mgr.25072) 1120 : cluster 0 pgmap v1100: 99 pgs: 1 active+remapped+backfill_wait, 1 remapped+peering, 2 active+remapped+backfilling, 95 active+clean; 2.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 496 op/s; 2746/96375 objects misplaced (2.849%); 15 MiB/s, 240 objects/s recovering 2023-10-01T10:54:52.045 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:51 smithi173 bash[58816]: cluster 2023-10-01T10:54:50.786437+0000 osd.6 (osd.6) 138 : cluster 3 failed to encode map e213 with expected crc 2023-10-01T10:54:52.046 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:51 smithi173 bash[58816]: cluster 2023-10-01T10:54:50.786706+0000 osd.7 (osd.7) 175 : cluster 3 failed to encode map e213 with expected crc 2023-10-01T10:54:52.046 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:51 smithi173 bash[58816]: cluster 2023-10-01T10:54:50.786714+0000 osd.5 (osd.5) 172 : cluster 3 failed to encode map e213 with expected crc 2023-10-01T10:54:52.046 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:51 smithi173 bash[58816]: cluster 2023-10-01T10:54:50.787920+0000 mon.a (mon.0) 1400 : cluster 0 osdmap e213: 8 total, 8 up, 8 in 2023-10-01T10:54:52.046 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:51 smithi173 bash[58816]: cluster 2023-10-01T10:54:50.788658+0000 osd.7 (osd.7) 176 : cluster 3 failed to encode map e213 with expected crc 2023-10-01T10:54:52.046 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:51 smithi173 bash[58816]: cluster 2023-10-01T10:54:50.788838+0000 osd.5 (osd.5) 173 : cluster 3 failed to encode map e213 with expected crc 2023-10-01T10:54:52.046 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:51 smithi173 bash[58816]: cluster 2023-10-01T10:54:50.790553+0000 osd.5 (osd.5) 174 : cluster 3 failed to encode map e213 with expected crc 2023-10-01T10:54:52.046 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:51 smithi173 bash[58816]: cluster 2023-10-01T10:54:50.791881+0000 osd.7 (osd.7) 177 : cluster 3 failed to encode map e213 with expected crc 2023-10-01T10:54:52.046 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:51 smithi173 bash[58816]: cluster 2023-10-01T10:54:50.792506+0000 osd.5 (osd.5) 175 : cluster 3 failed to encode map e213 with expected crc 2023-10-01T10:54:52.065 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:52.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:52.072 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:54:52.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:51 smithi078 bash[78274]: cluster 2023-10-01T10:54:50.031993+0000 mgr.y (mgr.25072) 1120 : cluster 0 pgmap v1100: 99 pgs: 1 active+remapped+backfill_wait, 1 remapped+peering, 2 active+remapped+backfilling, 95 active+clean; 2.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 496 op/s; 2746/96375 objects misplaced (2.849%); 15 MiB/s, 240 objects/s recovering 2023-10-01T10:54:52.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:51 smithi078 bash[78274]: cluster 2023-10-01T10:54:50.786437+0000 osd.6 (osd.6) 138 : cluster 3 failed to encode map e213 with expected crc 2023-10-01T10:54:52.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:51 smithi078 bash[78274]: cluster 2023-10-01T10:54:50.786706+0000 osd.7 (osd.7) 175 : cluster 3 failed to encode map e213 with expected crc 2023-10-01T10:54:52.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:51 smithi078 bash[78274]: cluster 2023-10-01T10:54:50.786714+0000 osd.5 (osd.5) 172 : cluster 3 failed to encode map e213 with expected crc 2023-10-01T10:54:52.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:51 smithi078 bash[78274]: cluster 2023-10-01T10:54:50.787920+0000 mon.a (mon.0) 1400 : cluster 0 osdmap e213: 8 total, 8 up, 8 in 2023-10-01T10:54:52.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:51 smithi078 bash[78274]: cluster 2023-10-01T10:54:50.788658+0000 osd.7 (osd.7) 176 : cluster 3 failed to encode map e213 with expected crc 2023-10-01T10:54:52.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:51 smithi078 bash[78274]: cluster 2023-10-01T10:54:50.788838+0000 osd.5 (osd.5) 173 : cluster 3 failed to encode map e213 with expected crc 2023-10-01T10:54:52.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:51 smithi078 bash[78274]: cluster 2023-10-01T10:54:50.790553+0000 osd.5 (osd.5) 174 : cluster 3 failed to encode map e213 with expected crc 2023-10-01T10:54:52.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:51 smithi078 bash[78274]: cluster 2023-10-01T10:54:50.791881+0000 osd.7 (osd.7) 177 : cluster 3 failed to encode map e213 with expected crc 2023-10-01T10:54:52.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:51 smithi078 bash[78274]: cluster 2023-10-01T10:54:50.792506+0000 osd.5 (osd.5) 175 : cluster 3 failed to encode map e213 with expected crc 2023-10-01T10:54:52.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:51 smithi078 bash[75299]: cluster 2023-10-01T10:54:50.031993+0000 mgr.y (mgr.25072) 1120 : cluster 0 pgmap v1100: 99 pgs: 1 active+remapped+backfill_wait, 1 remapped+peering, 2 active+remapped+backfilling, 95 active+clean; 2.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 496 op/s; 2746/96375 objects misplaced (2.849%); 15 MiB/s, 240 objects/s recovering 2023-10-01T10:54:52.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:51 smithi078 bash[75299]: cluster 2023-10-01T10:54:50.786437+0000 osd.6 (osd.6) 138 : cluster 3 failed to encode map e213 with expected crc 2023-10-01T10:54:52.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:51 smithi078 bash[75299]: cluster 2023-10-01T10:54:50.786706+0000 osd.7 (osd.7) 175 : cluster 3 failed to encode map e213 with expected crc 2023-10-01T10:54:52.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:51 smithi078 bash[75299]: cluster 2023-10-01T10:54:50.786714+0000 osd.5 (osd.5) 172 : cluster 3 failed to encode map e213 with expected crc 2023-10-01T10:54:52.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:51 smithi078 bash[75299]: cluster 2023-10-01T10:54:50.787920+0000 mon.a (mon.0) 1400 : cluster 0 osdmap e213: 8 total, 8 up, 8 in 2023-10-01T10:54:52.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:51 smithi078 bash[75299]: cluster 2023-10-01T10:54:50.788658+0000 osd.7 (osd.7) 176 : cluster 3 failed to encode map e213 with expected crc 2023-10-01T10:54:52.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:51 smithi078 bash[75299]: cluster 2023-10-01T10:54:50.788838+0000 osd.5 (osd.5) 173 : cluster 3 failed to encode map e213 with expected crc 2023-10-01T10:54:52.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:51 smithi078 bash[75299]: cluster 2023-10-01T10:54:50.790553+0000 osd.5 (osd.5) 174 : cluster 3 failed to encode map e213 with expected crc 2023-10-01T10:54:52.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:51 smithi078 bash[75299]: cluster 2023-10-01T10:54:50.791881+0000 osd.7 (osd.7) 177 : cluster 3 failed to encode map e213 with expected crc 2023-10-01T10:54:52.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:51 smithi078 bash[75299]: cluster 2023-10-01T10:54:50.792506+0000 osd.5 (osd.5) 175 : cluster 3 failed to encode map e213 with expected crc 2023-10-01T10:54:52.173 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:52.221 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:52.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:52.227 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:54:52.327 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:52.361 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:53.244 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:52 smithi173 bash[58816]: cluster 2023-10-01T10:54:51.798039+0000 mon.a (mon.0) 1401 : cluster 0 osdmap e214: 8 total, 8 up, 8 in 2023-10-01T10:54:53.245 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:52 smithi173 bash[58816]: cluster 2023-10-01T10:54:51.804887+0000 osd.6 (osd.6) 139 : cluster 3 failed to encode map e214 with expected crc 2023-10-01T10:54:53.245 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:52 smithi173 bash[58816]: cluster 2023-10-01T10:54:51.805202+0000 osd.7 (osd.7) 178 : cluster 3 failed to encode map e214 with expected crc 2023-10-01T10:54:53.245 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:52 smithi173 bash[58816]: cluster 2023-10-01T10:54:51.808449+0000 osd.7 (osd.7) 179 : cluster 3 failed to encode map e214 with expected crc 2023-10-01T10:54:53.245 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:52 smithi173 bash[58816]: cluster 2023-10-01T10:54:51.808514+0000 osd.6 (osd.6) 140 : cluster 3 failed to encode map e214 with expected crc 2023-10-01T10:54:53.245 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:52 smithi173 bash[58816]: audit 2023-10-01T10:54:52.033407+0000 mon.a (mon.0) 1402 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "25"}]: dispatch 2023-10-01T10:54:53.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:53.246 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:54:53.246 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:52 smithi078 bash[78274]: cluster 2023-10-01T10:54:51.798039+0000 mon.a (mon.0) 1401 : cluster 0 osdmap e214: 8 total, 8 up, 8 in 2023-10-01T10:54:53.246 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:52 smithi078 bash[78274]: cluster 2023-10-01T10:54:51.804887+0000 osd.6 (osd.6) 139 : cluster 3 failed to encode map e214 with expected crc 2023-10-01T10:54:53.246 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:52 smithi078 bash[78274]: cluster 2023-10-01T10:54:51.805202+0000 osd.7 (osd.7) 178 : cluster 3 failed to encode map e214 with expected crc 2023-10-01T10:54:53.246 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:52 smithi078 bash[78274]: cluster 2023-10-01T10:54:51.808449+0000 osd.7 (osd.7) 179 : cluster 3 failed to encode map e214 with expected crc 2023-10-01T10:54:53.247 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:52 smithi078 bash[78274]: cluster 2023-10-01T10:54:51.808514+0000 osd.6 (osd.6) 140 : cluster 3 failed to encode map e214 with expected crc 2023-10-01T10:54:53.247 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:52 smithi078 bash[78274]: audit 2023-10-01T10:54:52.033407+0000 mon.a (mon.0) 1402 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "25"}]: dispatch 2023-10-01T10:54:53.247 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:52 smithi078 bash[75299]: cluster 2023-10-01T10:54:51.798039+0000 mon.a (mon.0) 1401 : cluster 0 osdmap e214: 8 total, 8 up, 8 in 2023-10-01T10:54:53.247 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:52 smithi078 bash[75299]: cluster 2023-10-01T10:54:51.804887+0000 osd.6 (osd.6) 139 : cluster 3 failed to encode map e214 with expected crc 2023-10-01T10:54:53.247 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:52 smithi078 bash[75299]: cluster 2023-10-01T10:54:51.805202+0000 osd.7 (osd.7) 178 : cluster 3 failed to encode map e214 with expected crc 2023-10-01T10:54:53.247 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:52 smithi078 bash[75299]: cluster 2023-10-01T10:54:51.808449+0000 osd.7 (osd.7) 179 : cluster 3 failed to encode map e214 with expected crc 2023-10-01T10:54:53.247 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:52 smithi078 bash[75299]: cluster 2023-10-01T10:54:51.808514+0000 osd.6 (osd.6) 140 : cluster 3 failed to encode map e214 with expected crc 2023-10-01T10:54:53.247 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:52 smithi078 bash[75299]: audit 2023-10-01T10:54:52.033407+0000 mon.a (mon.0) 1402 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "25"}]: dispatch 2023-10-01T10:54:53.347 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:53.378 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:53.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:53.381 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:54:53.482 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:53.508 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:53.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:53.510 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:54:53.611 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:53.637 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:53.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:53.640 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:54:53.741 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:53.767 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:53.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:53.771 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:54:53.872 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:53.882 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:54:53.885 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:54:53.903 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:53.907 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:53 smithi078 bash[75299]: cluster 2023-10-01T10:54:51.855789+0000 osd.5 (osd.5) 176 : cluster 3 failed to encode map e214 with expected crc 2023-10-01T10:54:53.907 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:53 smithi078 bash[75299]: cluster 2023-10-01T10:54:52.032455+0000 mgr.y (mgr.25072) 1121 : cluster 0 pgmap v1103: 99 pgs: 1 active+remapped+backfill_wait, 3 active+remapped+backfilling, 95 active+clean; 2.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 516 op/s; 4086/96030 objects misplaced (4.255%); 18 MiB/s, 295 objects/s recovering 2023-10-01T10:54:53.907 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:53 smithi078 bash[75299]: cluster 2023-10-01T10:54:52.807572+0000 osd.7 (osd.7) 180 : cluster 3 failed to encode map e215 with expected crc 2023-10-01T10:54:53.907 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:53 smithi078 bash[75299]: audit 2023-10-01T10:54:52.810949+0000 mon.a (mon.0) 1403 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "25"}]': finished 2023-10-01T10:54:53.907 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:53 smithi078 bash[75299]: cluster 2023-10-01T10:54:52.811237+0000 mon.a (mon.0) 1404 : cluster 0 osdmap e215: 8 total, 8 up, 8 in 2023-10-01T10:54:53.907 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:53 smithi078 bash[75299]: cluster 2023-10-01T10:54:52.813067+0000 osd.5 (osd.5) 177 : cluster 3 failed to encode map e215 with expected crc 2023-10-01T10:54:53.907 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:53 smithi078 bash[75299]: cluster 2023-10-01T10:54:52.815530+0000 osd.6 (osd.6) 141 : cluster 3 failed to encode map e215 with expected crc 2023-10-01T10:54:53.908 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:53 smithi078 bash[75299]: cluster 2023-10-01T10:54:52.818044+0000 osd.6 (osd.6) 142 : cluster 3 failed to encode map e215 with expected crc 2023-10-01T10:54:53.908 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:53 smithi078 bash[75299]: cluster 2023-10-01T10:54:52.820141+0000 osd.6 (osd.6) 143 : cluster 3 failed to encode map e215 with expected crc 2023-10-01T10:54:53.908 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:53 smithi078 bash[78274]: cluster 2023-10-01T10:54:51.855789+0000 osd.5 (osd.5) 176 : cluster 3 failed to encode map e214 with expected crc 2023-10-01T10:54:53.908 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:53 smithi078 bash[78274]: cluster 2023-10-01T10:54:52.032455+0000 mgr.y (mgr.25072) 1121 : cluster 0 pgmap v1103: 99 pgs: 1 active+remapped+backfill_wait, 3 active+remapped+backfilling, 95 active+clean; 2.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 516 op/s; 4086/96030 objects misplaced (4.255%); 18 MiB/s, 295 objects/s recovering 2023-10-01T10:54:53.908 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:53 smithi078 bash[78274]: cluster 2023-10-01T10:54:52.807572+0000 osd.7 (osd.7) 180 : cluster 3 failed to encode map e215 with expected crc 2023-10-01T10:54:53.908 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:53 smithi078 bash[78274]: audit 2023-10-01T10:54:52.810949+0000 mon.a (mon.0) 1403 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "25"}]': finished 2023-10-01T10:54:53.908 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:53 smithi078 bash[78274]: cluster 2023-10-01T10:54:52.811237+0000 mon.a (mon.0) 1404 : cluster 0 osdmap e215: 8 total, 8 up, 8 in 2023-10-01T10:54:53.909 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:53 smithi078 bash[78274]: cluster 2023-10-01T10:54:52.813067+0000 osd.5 (osd.5) 177 : cluster 3 failed to encode map e215 with expected crc 2023-10-01T10:54:53.909 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:53 smithi078 bash[78274]: cluster 2023-10-01T10:54:52.815530+0000 osd.6 (osd.6) 141 : cluster 3 failed to encode map e215 with expected crc 2023-10-01T10:54:53.909 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:53 smithi078 bash[78274]: cluster 2023-10-01T10:54:52.818044+0000 osd.6 (osd.6) 142 : cluster 3 failed to encode map e215 with expected crc 2023-10-01T10:54:53.909 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:53 smithi078 bash[78274]: cluster 2023-10-01T10:54:52.820141+0000 osd.6 (osd.6) 143 : cluster 3 failed to encode map e215 with expected crc 2023-10-01T10:54:53.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:53.910 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:54:54.010 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:54.049 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:54.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:54.055 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:54:54.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:53 smithi173 bash[58816]: cluster 2023-10-01T10:54:51.855789+0000 osd.5 (osd.5) 176 : cluster 3 failed to encode map e214 with expected crc 2023-10-01T10:54:54.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:53 smithi173 bash[58816]: cluster 2023-10-01T10:54:52.032455+0000 mgr.y (mgr.25072) 1121 : cluster 0 pgmap v1103: 99 pgs: 1 active+remapped+backfill_wait, 3 active+remapped+backfilling, 95 active+clean; 2.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 516 op/s; 4086/96030 objects misplaced (4.255%); 18 MiB/s, 295 objects/s recovering 2023-10-01T10:54:54.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:53 smithi173 bash[58816]: cluster 2023-10-01T10:54:52.807572+0000 osd.7 (osd.7) 180 : cluster 3 failed to encode map e215 with expected crc 2023-10-01T10:54:54.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:53 smithi173 bash[58816]: audit 2023-10-01T10:54:52.810949+0000 mon.a (mon.0) 1403 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "25"}]': finished 2023-10-01T10:54:54.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:53 smithi173 bash[58816]: cluster 2023-10-01T10:54:52.811237+0000 mon.a (mon.0) 1404 : cluster 0 osdmap e215: 8 total, 8 up, 8 in 2023-10-01T10:54:54.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:53 smithi173 bash[58816]: cluster 2023-10-01T10:54:52.813067+0000 osd.5 (osd.5) 177 : cluster 3 failed to encode map e215 with expected crc 2023-10-01T10:54:54.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:53 smithi173 bash[58816]: cluster 2023-10-01T10:54:52.815530+0000 osd.6 (osd.6) 141 : cluster 3 failed to encode map e215 with expected crc 2023-10-01T10:54:54.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:53 smithi173 bash[58816]: cluster 2023-10-01T10:54:52.818044+0000 osd.6 (osd.6) 142 : cluster 3 failed to encode map e215 with expected crc 2023-10-01T10:54:54.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:53 smithi173 bash[58816]: cluster 2023-10-01T10:54:52.820141+0000 osd.6 (osd.6) 143 : cluster 3 failed to encode map e215 with expected crc 2023-10-01T10:54:54.156 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:54.184 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:54.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:54.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:54:54.287 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:54.319 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:54.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:54.322 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:54:54.423 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:54.448 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:54.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:54.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:54:54.552 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:54.575 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:54.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:54.578 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:54:54.679 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:54.707 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:54.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:54.710 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:54.811 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:54.828 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:54 smithi173 bash[58816]: cluster 2023-10-01T10:54:53.817170+0000 osd.5 (osd.5) 178 : cluster 3 failed to encode map e216 with expected crc 2023-10-01T10:54:54.828 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:54 smithi173 bash[58816]: cluster 2023-10-01T10:54:53.818706+0000 mon.a (mon.0) 1405 : cluster 0 osdmap e216: 8 total, 8 up, 8 in 2023-10-01T10:54:54.829 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:54 smithi173 bash[58816]: cluster 2023-10-01T10:54:53.820048+0000 osd.7 (osd.7) 181 : cluster 3 failed to encode map e216 with expected crc 2023-10-01T10:54:54.829 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:54 smithi173 bash[58816]: cluster 2023-10-01T10:54:53.825437+0000 osd.6 (osd.6) 144 : cluster 3 failed to encode map e216 with expected crc 2023-10-01T10:54:54.841 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:54.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:54.843 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:54:54.944 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:54.971 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:54.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:54.974 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:54:55.075 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:55.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:54 smithi078 bash[75299]: cluster 2023-10-01T10:54:53.817170+0000 osd.5 (osd.5) 178 : cluster 3 failed to encode map e216 with expected crc 2023-10-01T10:54:55.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:54 smithi078 bash[75299]: cluster 2023-10-01T10:54:53.818706+0000 mon.a (mon.0) 1405 : cluster 0 osdmap e216: 8 total, 8 up, 8 in 2023-10-01T10:54:55.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:54 smithi078 bash[75299]: cluster 2023-10-01T10:54:53.820048+0000 osd.7 (osd.7) 181 : cluster 3 failed to encode map e216 with expected crc 2023-10-01T10:54:55.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:54 smithi078 bash[75299]: cluster 2023-10-01T10:54:53.825437+0000 osd.6 (osd.6) 144 : cluster 3 failed to encode map e216 with expected crc 2023-10-01T10:54:55.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:54 smithi078 bash[78274]: cluster 2023-10-01T10:54:53.817170+0000 osd.5 (osd.5) 178 : cluster 3 failed to encode map e216 with expected crc 2023-10-01T10:54:55.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:54 smithi078 bash[78274]: cluster 2023-10-01T10:54:53.818706+0000 mon.a (mon.0) 1405 : cluster 0 osdmap e216: 8 total, 8 up, 8 in 2023-10-01T10:54:55.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:54 smithi078 bash[78274]: cluster 2023-10-01T10:54:53.820048+0000 osd.7 (osd.7) 181 : cluster 3 failed to encode map e216 with expected crc 2023-10-01T10:54:55.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:54 smithi078 bash[78274]: cluster 2023-10-01T10:54:53.825437+0000 osd.6 (osd.6) 144 : cluster 3 failed to encode map e216 with expected crc 2023-10-01T10:54:55.098 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:55.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:55.100 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:54:55.201 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:55.247 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:55.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:55.254 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:54:55.354 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:55.392 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:55.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:55.396 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:54:55.497 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:55.521 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:55.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:55.522 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:54:55.623 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:55.653 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:55.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:55.655 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:54:55.756 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:55.785 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:55.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:55.788 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:54:55.830 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:55 smithi078 bash[75299]: cluster 2023-10-01T10:54:54.033102+0000 mgr.y (mgr.25072) 1122 : cluster 0 pgmap v1106: 99 pgs: 1 unknown, 2 active+remapped+backfilling, 96 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 2302/85806 objects misplaced (2.683%); 15 MiB/s, 235 objects/s recovering 2023-10-01T10:54:55.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:55 smithi078 bash[75299]: cluster 2023-10-01T10:54:54.826909+0000 osd.5 (osd.5) 179 : cluster 3 failed to encode map e217 with expected crc 2023-10-01T10:54:55.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:55 smithi078 bash[75299]: cluster 2023-10-01T10:54:54.830569+0000 mon.a (mon.0) 1406 : cluster 0 osdmap e217: 8 total, 8 up, 8 in 2023-10-01T10:54:55.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:55 smithi078 bash[75299]: cluster 2023-10-01T10:54:54.831776+0000 osd.7 (osd.7) 182 : cluster 3 failed to encode map e217 with expected crc 2023-10-01T10:54:55.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:55 smithi078 bash[75299]: cluster 2023-10-01T10:54:54.832010+0000 osd.3 (osd.3) 16 : cluster 0 11.18 starting backfill to osd.2 from (0'0,0'0] MAX to 214'10953 2023-10-01T10:54:55.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:55 smithi078 bash[75299]: cluster 2023-10-01T10:54:54.844897+0000 osd.3 (osd.3) 17 : cluster 0 11.18 starting backfill to osd.4 from (0'0,0'0] MAX to 214'10953 2023-10-01T10:54:55.889 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:55.898 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:55 smithi173 bash[58816]: cluster 2023-10-01T10:54:54.033102+0000 mgr.y (mgr.25072) 1122 : cluster 0 pgmap v1106: 99 pgs: 1 unknown, 2 active+remapped+backfilling, 96 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 2302/85806 objects misplaced (2.683%); 15 MiB/s, 235 objects/s recovering 2023-10-01T10:54:55.899 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:55 smithi173 bash[58816]: cluster 2023-10-01T10:54:54.826909+0000 osd.5 (osd.5) 179 : cluster 3 failed to encode map e217 with expected crc 2023-10-01T10:54:55.899 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:55 smithi173 bash[58816]: cluster 2023-10-01T10:54:54.830569+0000 mon.a (mon.0) 1406 : cluster 0 osdmap e217: 8 total, 8 up, 8 in 2023-10-01T10:54:55.899 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:55 smithi173 bash[58816]: cluster 2023-10-01T10:54:54.831776+0000 osd.7 (osd.7) 182 : cluster 3 failed to encode map e217 with expected crc 2023-10-01T10:54:55.899 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:55 smithi173 bash[58816]: cluster 2023-10-01T10:54:54.832010+0000 osd.3 (osd.3) 16 : cluster 0 11.18 starting backfill to osd.2 from (0'0,0'0] MAX to 214'10953 2023-10-01T10:54:55.899 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:55 smithi173 bash[58816]: cluster 2023-10-01T10:54:54.844897+0000 osd.3 (osd.3) 17 : cluster 0 11.18 starting backfill to osd.4 from (0'0,0'0] MAX to 214'10953 2023-10-01T10:54:55.913 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:55.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:55.919 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:54:56.020 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:56.050 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:56.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:56.052 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:54:56.080 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:55 smithi078 bash[78274]: cluster 2023-10-01T10:54:54.033102+0000 mgr.y (mgr.25072) 1122 : cluster 0 pgmap v1106: 99 pgs: 1 unknown, 2 active+remapped+backfilling, 96 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 593 op/s; 2302/85806 objects misplaced (2.683%); 15 MiB/s, 235 objects/s recovering 2023-10-01T10:54:56.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:55 smithi078 bash[78274]: cluster 2023-10-01T10:54:54.826909+0000 osd.5 (osd.5) 179 : cluster 3 failed to encode map e217 with expected crc 2023-10-01T10:54:56.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:55 smithi078 bash[78274]: cluster 2023-10-01T10:54:54.830569+0000 mon.a (mon.0) 1406 : cluster 0 osdmap e217: 8 total, 8 up, 8 in 2023-10-01T10:54:56.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:55 smithi078 bash[78274]: cluster 2023-10-01T10:54:54.831776+0000 osd.7 (osd.7) 182 : cluster 3 failed to encode map e217 with expected crc 2023-10-01T10:54:56.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:55 smithi078 bash[78274]: cluster 2023-10-01T10:54:54.832010+0000 osd.3 (osd.3) 16 : cluster 0 11.18 starting backfill to osd.2 from (0'0,0'0] MAX to 214'10953 2023-10-01T10:54:56.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:55 smithi078 bash[78274]: cluster 2023-10-01T10:54:54.844897+0000 osd.3 (osd.3) 17 : cluster 0 11.18 starting backfill to osd.4 from (0'0,0'0] MAX to 214'10953 2023-10-01T10:54:56.152 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:56.178 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:56.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:56.185 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:54:56.286 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:56.311 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:54:56.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:56.313 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:54:56.414 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:56.437 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:56.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:56.440 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:54:56.541 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:56.562 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:56.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:56.565 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:56.665 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:56.690 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:56.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:56.694 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:56.795 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:56.824 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:56.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:56.826 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:54:56.927 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:56.935 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:56 smithi078 bash[78274]: cluster 2023-10-01T10:54:55.839495+0000 mon.a (mon.0) 1407 : cluster 0 osdmap e218: 8 total, 8 up, 8 in 2023-10-01T10:54:56.935 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:56 smithi078 bash[78274]: cluster 2023-10-01T10:54:55.841525+0000 osd.5 (osd.5) 180 : cluster 3 failed to encode map e218 with expected crc 2023-10-01T10:54:56.935 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:56 smithi078 bash[78274]: cluster 2023-10-01T10:54:55.842062+0000 osd.7 (osd.7) 183 : cluster 3 failed to encode map e218 with expected crc 2023-10-01T10:54:56.935 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:56 smithi078 bash[78274]: cluster 2023-10-01T10:54:55.844378+0000 osd.6 (osd.6) 145 : cluster 3 failed to encode map e218 with expected crc 2023-10-01T10:54:56.935 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:56 smithi078 bash[78274]: cluster 2023-10-01T10:54:55.845136+0000 osd.7 (osd.7) 184 : cluster 3 failed to encode map e218 with expected crc 2023-10-01T10:54:56.935 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:56 smithi078 bash[78274]: cluster 2023-10-01T10:54:55.847201+0000 osd.7 (osd.7) 185 : cluster 3 failed to encode map e218 with expected crc 2023-10-01T10:54:56.936 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:56 smithi078 bash[75299]: cluster 2023-10-01T10:54:55.839495+0000 mon.a (mon.0) 1407 : cluster 0 osdmap e218: 8 total, 8 up, 8 in 2023-10-01T10:54:56.936 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:56 smithi078 bash[75299]: cluster 2023-10-01T10:54:55.841525+0000 osd.5 (osd.5) 180 : cluster 3 failed to encode map e218 with expected crc 2023-10-01T10:54:56.936 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:56 smithi078 bash[75299]: cluster 2023-10-01T10:54:55.842062+0000 osd.7 (osd.7) 183 : cluster 3 failed to encode map e218 with expected crc 2023-10-01T10:54:56.936 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:56 smithi078 bash[75299]: cluster 2023-10-01T10:54:55.844378+0000 osd.6 (osd.6) 145 : cluster 3 failed to encode map e218 with expected crc 2023-10-01T10:54:56.936 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:56 smithi078 bash[75299]: cluster 2023-10-01T10:54:55.845136+0000 osd.7 (osd.7) 184 : cluster 3 failed to encode map e218 with expected crc 2023-10-01T10:54:56.936 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:56 smithi078 bash[75299]: cluster 2023-10-01T10:54:55.847201+0000 osd.7 (osd.7) 185 : cluster 3 failed to encode map e218 with expected crc 2023-10-01T10:54:56.971 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:56.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:56.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:57.074 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:57.098 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:57.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:57.104 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:54:57.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:56 smithi173 bash[58816]: cluster 2023-10-01T10:54:55.839495+0000 mon.a (mon.0) 1407 : cluster 0 osdmap e218: 8 total, 8 up, 8 in 2023-10-01T10:54:57.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:56 smithi173 bash[58816]: cluster 2023-10-01T10:54:55.841525+0000 osd.5 (osd.5) 180 : cluster 3 failed to encode map e218 with expected crc 2023-10-01T10:54:57.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:56 smithi173 bash[58816]: cluster 2023-10-01T10:54:55.842062+0000 osd.7 (osd.7) 183 : cluster 3 failed to encode map e218 with expected crc 2023-10-01T10:54:57.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:56 smithi173 bash[58816]: cluster 2023-10-01T10:54:55.844378+0000 osd.6 (osd.6) 145 : cluster 3 failed to encode map e218 with expected crc 2023-10-01T10:54:57.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:56 smithi173 bash[58816]: cluster 2023-10-01T10:54:55.845136+0000 osd.7 (osd.7) 184 : cluster 3 failed to encode map e218 with expected crc 2023-10-01T10:54:57.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:56 smithi173 bash[58816]: cluster 2023-10-01T10:54:55.847201+0000 osd.7 (osd.7) 185 : cluster 3 failed to encode map e218 with expected crc 2023-10-01T10:54:57.205 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:57.238 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:57.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:57.246 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:54:57.346 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:57.371 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:57.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:57.377 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:54:57.396 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:54:57.478 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:57.509 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:57.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:57.515 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:54:57.616 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:57.642 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:57.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:57.647 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:54:57.747 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:57.788 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:57.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:57.793 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:57.894 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:57.920 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:57.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:57.923 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:58.023 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:58.031 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:57 smithi078 bash[78274]: cluster 2023-10-01T10:54:56.033595+0000 mgr.y (mgr.25072) 1123 : cluster 0 pgmap v1109: 99 pgs: 1 unknown, 2 active+remapped+backfilling, 96 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 644 op/s; 2302/84858 objects misplaced (2.713%); 10 MiB/s, 163 objects/s recovering 2023-10-01T10:54:58.031 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:57 smithi078 bash[78274]: cluster 2023-10-01T10:54:56.850790+0000 osd.7 (osd.7) 186 : cluster 3 failed to encode map e219 with expected crc 2023-10-01T10:54:58.031 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:57 smithi078 bash[78274]: cluster 2023-10-01T10:54:56.853275+0000 osd.6 (osd.6) 146 : cluster 3 failed to encode map e219 with expected crc 2023-10-01T10:54:58.032 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:57 smithi078 bash[78274]: cluster 2023-10-01T10:54:56.854131+0000 mon.a (mon.0) 1408 : cluster 0 osdmap e219: 8 total, 8 up, 8 in 2023-10-01T10:54:58.032 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:57 smithi078 bash[78274]: cluster 2023-10-01T10:54:56.854721+0000 osd.6 (osd.6) 147 : cluster 3 failed to encode map e219 with expected crc 2023-10-01T10:54:58.032 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:57 smithi078 bash[78274]: cluster 2023-10-01T10:54:56.866319+0000 osd.5 (osd.5) 181 : cluster 3 failed to encode map e219 with expected crc 2023-10-01T10:54:58.032 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:57 smithi078 bash[75299]: cluster 2023-10-01T10:54:56.033595+0000 mgr.y (mgr.25072) 1123 : cluster 0 pgmap v1109: 99 pgs: 1 unknown, 2 active+remapped+backfilling, 96 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 644 op/s; 2302/84858 objects misplaced (2.713%); 10 MiB/s, 163 objects/s recovering 2023-10-01T10:54:58.032 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:57 smithi078 bash[75299]: cluster 2023-10-01T10:54:56.850790+0000 osd.7 (osd.7) 186 : cluster 3 failed to encode map e219 with expected crc 2023-10-01T10:54:58.032 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:57 smithi078 bash[75299]: cluster 2023-10-01T10:54:56.853275+0000 osd.6 (osd.6) 146 : cluster 3 failed to encode map e219 with expected crc 2023-10-01T10:54:58.032 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:57 smithi078 bash[75299]: cluster 2023-10-01T10:54:56.854131+0000 mon.a (mon.0) 1408 : cluster 0 osdmap e219: 8 total, 8 up, 8 in 2023-10-01T10:54:58.032 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:57 smithi078 bash[75299]: cluster 2023-10-01T10:54:56.854721+0000 osd.6 (osd.6) 147 : cluster 3 failed to encode map e219 with expected crc 2023-10-01T10:54:58.033 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:57 smithi078 bash[75299]: cluster 2023-10-01T10:54:56.866319+0000 osd.5 (osd.5) 181 : cluster 3 failed to encode map e219 with expected crc 2023-10-01T10:54:58.049 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:58.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:58.052 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:54:58.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:57 smithi173 bash[58816]: cluster 2023-10-01T10:54:56.033595+0000 mgr.y (mgr.25072) 1123 : cluster 0 pgmap v1109: 99 pgs: 1 unknown, 2 active+remapped+backfilling, 96 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 644 op/s; 2302/84858 objects misplaced (2.713%); 10 MiB/s, 163 objects/s recovering 2023-10-01T10:54:58.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:57 smithi173 bash[58816]: cluster 2023-10-01T10:54:56.850790+0000 osd.7 (osd.7) 186 : cluster 3 failed to encode map e219 with expected crc 2023-10-01T10:54:58.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:57 smithi173 bash[58816]: cluster 2023-10-01T10:54:56.853275+0000 osd.6 (osd.6) 146 : cluster 3 failed to encode map e219 with expected crc 2023-10-01T10:54:58.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:57 smithi173 bash[58816]: cluster 2023-10-01T10:54:56.854131+0000 mon.a (mon.0) 1408 : cluster 0 osdmap e219: 8 total, 8 up, 8 in 2023-10-01T10:54:58.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:57 smithi173 bash[58816]: cluster 2023-10-01T10:54:56.854721+0000 osd.6 (osd.6) 147 : cluster 3 failed to encode map e219 with expected crc 2023-10-01T10:54:58.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:57 smithi173 bash[58816]: cluster 2023-10-01T10:54:56.866319+0000 osd.5 (osd.5) 181 : cluster 3 failed to encode map e219 with expected crc 2023-10-01T10:54:58.153 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:58.200 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:58.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:58.203 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:54:58.304 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:58.349 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:58.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:58.350 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:54:58.451 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:58.493 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:54:58.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:58.495 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:54:58.595 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:58.618 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:58.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:58.624 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:54:58.724 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:58.757 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:54:58.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:58.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:54:58.866 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:58.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:58 smithi173 bash[58816]: audit 2023-10-01T10:54:58.035013+0000 mon.a (mon.0) 1409 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "26"}]: dispatch 2023-10-01T10:54:58.894 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:58.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:58.899 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:59.000 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:59.038 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:54:59.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:59.044 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:54:59.121 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:58 smithi078 bash[75299]: audit 2023-10-01T10:54:58.035013+0000 mon.a (mon.0) 1409 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "26"}]: dispatch 2023-10-01T10:54:59.121 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:58 smithi078 bash[78274]: audit 2023-10-01T10:54:58.035013+0000 mon.a (mon.0) 1409 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "26"}]: dispatch 2023-10-01T10:54:59.145 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:59.169 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:59.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:59.171 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:54:59.272 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:59.294 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:54:59.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:59.299 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:54:59.399 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:59.424 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:54:59.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:59.427 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:54:59.528 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:59.551 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:59.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:59.558 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:59.658 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:59.679 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:59.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:59.682 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:54:59.782 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:59.813 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:54:59.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:59.816 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:54:59.917 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:59.944 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:54:59.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:54:59.948 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:55:00.048 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:00.062 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:59 smithi173 bash[58816]: cluster 2023-10-01T10:54:58.034120+0000 mgr.y (mgr.25072) 1124 : cluster 0 pgmap v1111: 99 pgs: 3 active+remapped+backfilling, 96 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 307 op/s; 3020/85359 objects misplaced (3.538%); 6.9 MiB/s, 110 objects/s recovering 2023-10-01T10:55:00.062 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:59 smithi173 bash[58816]: cluster 2023-10-01T10:54:58.861500+0000 osd.5 (osd.5) 182 : cluster 3 failed to encode map e220 with expected crc 2023-10-01T10:55:00.062 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:59 smithi173 bash[58816]: cluster 2023-10-01T10:54:58.864049+0000 osd.7 (osd.7) 187 : cluster 3 failed to encode map e220 with expected crc 2023-10-01T10:55:00.062 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:59 smithi173 bash[58816]: audit 2023-10-01T10:54:58.864495+0000 mon.a (mon.0) 1410 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "26"}]': finished 2023-10-01T10:55:00.062 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:59 smithi173 bash[58816]: cluster 2023-10-01T10:54:58.864687+0000 mon.a (mon.0) 1411 : cluster 0 osdmap e220: 8 total, 8 up, 8 in 2023-10-01T10:55:00.062 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:59 smithi173 bash[58816]: cluster 2023-10-01T10:54:58.872509+0000 osd.6 (osd.6) 148 : cluster 3 failed to encode map e220 with expected crc 2023-10-01T10:55:00.062 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:54:59 smithi173 bash[58816]: cluster 2023-10-01T10:54:58.876230+0000 osd.6 (osd.6) 149 : cluster 3 failed to encode map e220 with expected crc 2023-10-01T10:55:00.082 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:00.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:00.084 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:55:00.165 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:59 smithi078 bash[78274]: cluster 2023-10-01T10:54:58.034120+0000 mgr.y (mgr.25072) 1124 : cluster 0 pgmap v1111: 99 pgs: 3 active+remapped+backfilling, 96 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 307 op/s; 3020/85359 objects misplaced (3.538%); 6.9 MiB/s, 110 objects/s recovering 2023-10-01T10:55:00.166 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:59 smithi078 bash[78274]: cluster 2023-10-01T10:54:58.861500+0000 osd.5 (osd.5) 182 : cluster 3 failed to encode map e220 with expected crc 2023-10-01T10:55:00.166 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:59 smithi078 bash[78274]: cluster 2023-10-01T10:54:58.864049+0000 osd.7 (osd.7) 187 : cluster 3 failed to encode map e220 with expected crc 2023-10-01T10:55:00.166 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:59 smithi078 bash[78274]: audit 2023-10-01T10:54:58.864495+0000 mon.a (mon.0) 1410 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "26"}]': finished 2023-10-01T10:55:00.166 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:59 smithi078 bash[78274]: cluster 2023-10-01T10:54:58.864687+0000 mon.a (mon.0) 1411 : cluster 0 osdmap e220: 8 total, 8 up, 8 in 2023-10-01T10:55:00.166 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:59 smithi078 bash[78274]: cluster 2023-10-01T10:54:58.872509+0000 osd.6 (osd.6) 148 : cluster 3 failed to encode map e220 with expected crc 2023-10-01T10:55:00.166 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:54:59 smithi078 bash[78274]: cluster 2023-10-01T10:54:58.876230+0000 osd.6 (osd.6) 149 : cluster 3 failed to encode map e220 with expected crc 2023-10-01T10:55:00.167 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:59 smithi078 bash[75299]: cluster 2023-10-01T10:54:58.034120+0000 mgr.y (mgr.25072) 1124 : cluster 0 pgmap v1111: 99 pgs: 3 active+remapped+backfilling, 96 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 307 op/s; 3020/85359 objects misplaced (3.538%); 6.9 MiB/s, 110 objects/s recovering 2023-10-01T10:55:00.167 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:59 smithi078 bash[75299]: cluster 2023-10-01T10:54:58.861500+0000 osd.5 (osd.5) 182 : cluster 3 failed to encode map e220 with expected crc 2023-10-01T10:55:00.167 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:59 smithi078 bash[75299]: cluster 2023-10-01T10:54:58.864049+0000 osd.7 (osd.7) 187 : cluster 3 failed to encode map e220 with expected crc 2023-10-01T10:55:00.167 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:59 smithi078 bash[75299]: audit 2023-10-01T10:54:58.864495+0000 mon.a (mon.0) 1410 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num_actual", "val": "26"}]': finished 2023-10-01T10:55:00.167 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:59 smithi078 bash[75299]: cluster 2023-10-01T10:54:58.864687+0000 mon.a (mon.0) 1411 : cluster 0 osdmap e220: 8 total, 8 up, 8 in 2023-10-01T10:55:00.167 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:59 smithi078 bash[75299]: cluster 2023-10-01T10:54:58.872509+0000 osd.6 (osd.6) 148 : cluster 3 failed to encode map e220 with expected crc 2023-10-01T10:55:00.167 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:54:59 smithi078 bash[75299]: cluster 2023-10-01T10:54:58.876230+0000 osd.6 (osd.6) 149 : cluster 3 failed to encode map e220 with expected crc 2023-10-01T10:55:00.186 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:00.222 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:00.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:00.225 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:55:00.326 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:00.347 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:00.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:00.350 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:55:00.450 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:00.471 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:00.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:00.473 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:55:00.573 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:00.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:55:00 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:55:00] "GET /metrics HTTP/1.1" 200 36333 "" "Prometheus/2.33.4" 2023-10-01T10:55:00.606 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:00.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:00.608 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:55:00.709 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:00.745 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:00.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:00.749 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:00.850 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:00.886 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:00.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:00.895 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:55:00.996 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:01.020 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:01.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:01.029 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:55:01.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:00 smithi173 bash[58816]: cluster 2023-10-01T10:54:59.865333+0000 osd.5 (osd.5) 183 : cluster 3 failed to encode map e221 with expected crc 2023-10-01T10:55:01.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:00 smithi173 bash[58816]: cluster 2023-10-01T10:54:59.865988+0000 osd.7 (osd.7) 188 : cluster 3 failed to encode map e221 with expected crc 2023-10-01T10:55:01.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:00 smithi173 bash[58816]: cluster 2023-10-01T10:54:59.868054+0000 mon.a (mon.0) 1412 : cluster 0 osdmap e221: 8 total, 8 up, 8 in 2023-10-01T10:55:01.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:00 smithi173 bash[58816]: cluster 2023-10-01T10:54:59.870291+0000 osd.6 (osd.6) 150 : cluster 3 failed to encode map e221 with expected crc 2023-10-01T10:55:01.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:00 smithi173 bash[58816]: cluster 2023-10-01T10:54:59.870960+0000 osd.5 (osd.5) 184 : cluster 3 failed to encode map e221 with expected crc 2023-10-01T10:55:01.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:00 smithi173 bash[58816]: cluster 2023-10-01T10:54:59.873892+0000 osd.6 (osd.6) 151 : cluster 3 failed to encode map e221 with expected crc 2023-10-01T10:55:01.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:00 smithi173 bash[58816]: cluster 2023-10-01T10:54:59.873982+0000 osd.7 (osd.7) 189 : cluster 3 failed to encode map e221 with expected crc 2023-10-01T10:55:01.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:00 smithi173 bash[58816]: cluster 2023-10-01T10:55:00.867586+0000 mon.a (mon.0) 1413 : cluster 0 osdmap e222: 8 total, 8 up, 8 in 2023-10-01T10:55:01.127 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:00 smithi078 bash[78274]: cluster 2023-10-01T10:54:59.865333+0000 osd.5 (osd.5) 183 : cluster 3 failed to encode map e221 with expected crc 2023-10-01T10:55:01.127 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:00 smithi078 bash[78274]: cluster 2023-10-01T10:54:59.865988+0000 osd.7 (osd.7) 188 : cluster 3 failed to encode map e221 with expected crc 2023-10-01T10:55:01.128 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:00 smithi078 bash[78274]: cluster 2023-10-01T10:54:59.868054+0000 mon.a (mon.0) 1412 : cluster 0 osdmap e221: 8 total, 8 up, 8 in 2023-10-01T10:55:01.128 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:00 smithi078 bash[78274]: cluster 2023-10-01T10:54:59.870291+0000 osd.6 (osd.6) 150 : cluster 3 failed to encode map e221 with expected crc 2023-10-01T10:55:01.128 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:00 smithi078 bash[78274]: cluster 2023-10-01T10:54:59.870960+0000 osd.5 (osd.5) 184 : cluster 3 failed to encode map e221 with expected crc 2023-10-01T10:55:01.128 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:00 smithi078 bash[78274]: cluster 2023-10-01T10:54:59.873892+0000 osd.6 (osd.6) 151 : cluster 3 failed to encode map e221 with expected crc 2023-10-01T10:55:01.128 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:00 smithi078 bash[78274]: cluster 2023-10-01T10:54:59.873982+0000 osd.7 (osd.7) 189 : cluster 3 failed to encode map e221 with expected crc 2023-10-01T10:55:01.128 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:00 smithi078 bash[78274]: cluster 2023-10-01T10:55:00.867586+0000 mon.a (mon.0) 1413 : cluster 0 osdmap e222: 8 total, 8 up, 8 in 2023-10-01T10:55:01.129 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:00 smithi078 bash[75299]: cluster 2023-10-01T10:54:59.865333+0000 osd.5 (osd.5) 183 : cluster 3 failed to encode map e221 with expected crc 2023-10-01T10:55:01.129 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:00 smithi078 bash[75299]: cluster 2023-10-01T10:54:59.865988+0000 osd.7 (osd.7) 188 : cluster 3 failed to encode map e221 with expected crc 2023-10-01T10:55:01.129 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:00 smithi078 bash[75299]: cluster 2023-10-01T10:54:59.868054+0000 mon.a (mon.0) 1412 : cluster 0 osdmap e221: 8 total, 8 up, 8 in 2023-10-01T10:55:01.129 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:00 smithi078 bash[75299]: cluster 2023-10-01T10:54:59.870291+0000 osd.6 (osd.6) 150 : cluster 3 failed to encode map e221 with expected crc 2023-10-01T10:55:01.130 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:00 smithi078 bash[75299]: cluster 2023-10-01T10:54:59.870960+0000 osd.5 (osd.5) 184 : cluster 3 failed to encode map e221 with expected crc 2023-10-01T10:55:01.130 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:00 smithi078 bash[75299]: cluster 2023-10-01T10:54:59.873892+0000 osd.6 (osd.6) 151 : cluster 3 failed to encode map e221 with expected crc 2023-10-01T10:55:01.130 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:00 smithi078 bash[75299]: cluster 2023-10-01T10:54:59.873982+0000 osd.7 (osd.7) 189 : cluster 3 failed to encode map e221 with expected crc 2023-10-01T10:55:01.130 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:00 smithi078 bash[75299]: cluster 2023-10-01T10:55:00.867586+0000 mon.a (mon.0) 1413 : cluster 0 osdmap e222: 8 total, 8 up, 8 in 2023-10-01T10:55:01.130 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:01.173 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:01.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:01.176 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:55:01.278 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:01.313 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:01.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:01.316 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:55:01.417 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:01.447 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:01.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:01.450 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:55:01.551 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:01.578 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:01.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:01.581 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:55:01.682 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:01.708 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:01.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:01.714 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:55:01.815 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:01.842 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:01.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:01.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:55:01.891 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:01 smithi078 bash[75299]: cluster 2023-10-01T10:55:00.034581+0000 mgr.y (mgr.25072) 1125 : cluster 0 pgmap v1114: 99 pgs: 1 unknown, 1 active+remapped+backfilling, 97 active+clean; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 664 op/s; 718/79935 objects misplaced (0.898%); 25 MiB/s, 399 objects/s recovering 2023-10-01T10:55:01.891 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:01 smithi078 bash[75299]: cluster 2023-10-01T10:55:00.866060+0000 osd.7 (osd.7) 190 : cluster 3 failed to encode map e222 with expected crc 2023-10-01T10:55:01.891 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:01 smithi078 bash[75299]: cluster 2023-10-01T10:55:00.871138+0000 osd.0 (osd.0) 11 : cluster 0 11.19 starting backfill to osd.2 from (0'0,0'0] MAX to 219'11170 2023-10-01T10:55:01.891 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:01 smithi078 bash[75299]: cluster 2023-10-01T10:55:00.871239+0000 osd.6 (osd.6) 152 : cluster 3 failed to encode map e222 with expected crc 2023-10-01T10:55:01.891 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:01 smithi078 bash[75299]: cluster 2023-10-01T10:55:00.883169+0000 osd.0 (osd.0) 12 : cluster 0 11.19 starting backfill to osd.7 from (0'0,0'0] MAX to 219'11170 2023-10-01T10:55:01.891 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:01 smithi078 bash[75299]: cluster 2023-10-01T10:55:00.950333+0000 osd.5 (osd.5) 185 : cluster 3 failed to encode map e222 with expected crc 2023-10-01T10:55:01.892 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:01 smithi078 bash[78274]: cluster 2023-10-01T10:55:00.034581+0000 mgr.y (mgr.25072) 1125 : cluster 0 pgmap v1114: 99 pgs: 1 unknown, 1 active+remapped+backfilling, 97 active+clean; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 664 op/s; 718/79935 objects misplaced (0.898%); 25 MiB/s, 399 objects/s recovering 2023-10-01T10:55:01.892 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:01 smithi078 bash[78274]: cluster 2023-10-01T10:55:00.866060+0000 osd.7 (osd.7) 190 : cluster 3 failed to encode map e222 with expected crc 2023-10-01T10:55:01.892 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:01 smithi078 bash[78274]: cluster 2023-10-01T10:55:00.871138+0000 osd.0 (osd.0) 11 : cluster 0 11.19 starting backfill to osd.2 from (0'0,0'0] MAX to 219'11170 2023-10-01T10:55:01.892 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:01 smithi078 bash[78274]: cluster 2023-10-01T10:55:00.871239+0000 osd.6 (osd.6) 152 : cluster 3 failed to encode map e222 with expected crc 2023-10-01T10:55:01.892 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:01 smithi078 bash[78274]: cluster 2023-10-01T10:55:00.883169+0000 osd.0 (osd.0) 12 : cluster 0 11.19 starting backfill to osd.7 from (0'0,0'0] MAX to 219'11170 2023-10-01T10:55:01.892 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:01 smithi078 bash[78274]: cluster 2023-10-01T10:55:00.950333+0000 osd.5 (osd.5) 185 : cluster 3 failed to encode map e222 with expected crc 2023-10-01T10:55:01.949 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:01.993 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:01.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:01.999 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:55:02.100 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:02.126 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:02.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:02.131 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:55:02.232 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:02.253 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:02.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:02.256 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:55:02.356 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:02.366 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:01 smithi173 bash[58816]: cluster 2023-10-01T10:55:00.034581+0000 mgr.y (mgr.25072) 1125 : cluster 0 pgmap v1114: 99 pgs: 1 unknown, 1 active+remapped+backfilling, 97 active+clean; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 664 op/s; 718/79935 objects misplaced (0.898%); 25 MiB/s, 399 objects/s recovering 2023-10-01T10:55:02.366 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:01 smithi173 bash[58816]: cluster 2023-10-01T10:55:00.866060+0000 osd.7 (osd.7) 190 : cluster 3 failed to encode map e222 with expected crc 2023-10-01T10:55:02.366 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:01 smithi173 bash[58816]: cluster 2023-10-01T10:55:00.871138+0000 osd.0 (osd.0) 11 : cluster 0 11.19 starting backfill to osd.2 from (0'0,0'0] MAX to 219'11170 2023-10-01T10:55:02.367 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:01 smithi173 bash[58816]: cluster 2023-10-01T10:55:00.871239+0000 osd.6 (osd.6) 152 : cluster 3 failed to encode map e222 with expected crc 2023-10-01T10:55:02.367 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:01 smithi173 bash[58816]: cluster 2023-10-01T10:55:00.883169+0000 osd.0 (osd.0) 12 : cluster 0 11.19 starting backfill to osd.7 from (0'0,0'0] MAX to 219'11170 2023-10-01T10:55:02.367 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:01 smithi173 bash[58816]: cluster 2023-10-01T10:55:00.950333+0000 osd.5 (osd.5) 185 : cluster 3 failed to encode map e222 with expected crc 2023-10-01T10:55:02.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:02.391 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:02.392 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:55:02.492 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:02.518 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:02.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:02.523 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:02.624 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:02.648 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:02.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:02.651 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:55:02.751 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:02.782 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:02.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:02.788 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:02.889 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:02.891 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:02 smithi078 bash[75299]: audit 2023-10-01T10:55:02.441456+0000 mon.a (mon.0) 1414 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:55:02.917 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:02.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:02.923 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:55:03.024 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:03.057 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:03.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:03.059 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:55:03.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:02 smithi173 bash[58816]: audit 2023-10-01T10:55:02.441456+0000 mon.a (mon.0) 1414 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:55:03.159 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:03.167 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:02 smithi078 bash[78274]: audit 2023-10-01T10:55:02.441456+0000 mon.a (mon.0) 1414 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:55:03.193 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:03.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:03.195 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:55:03.296 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:03.326 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:03.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:03.330 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:55:03.430 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:03.458 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:03.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:03.465 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:03.566 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:03.598 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:03.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:03.601 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:55:03.702 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:03.728 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:03.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:03.730 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:55:03.741 INFO:teuthology.orchestra.run.smithi078.stdout:ERROR: (22) Invalid argument 2023-10-01T10:55:03.741 INFO:teuthology.orchestra.run.smithi078.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T10:55:03.831 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:03.854 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:03.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:03.856 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:55:03.957 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:03.976 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:03.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:03.978 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:55:04.079 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:04.101 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:04.103 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:03 smithi078 bash[75299]: cluster 2023-10-01T10:55:02.035033+0000 mgr.y (mgr.25072) 1126 : cluster 0 pgmap v1116: 99 pgs: 1 unknown, 1 active+remapped+backfilling, 97 active+clean; 1.6 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 537 op/s; 718/79935 objects misplaced (0.898%); 20 MiB/s, 323 objects/s recovering 2023-10-01T10:55:04.103 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:03 smithi078 bash[78274]: cluster 2023-10-01T10:55:02.035033+0000 mgr.y (mgr.25072) 1126 : cluster 0 pgmap v1116: 99 pgs: 1 unknown, 1 active+remapped+backfilling, 97 active+clean; 1.6 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 537 op/s; 718/79935 objects misplaced (0.898%); 20 MiB/s, 323 objects/s recovering 2023-10-01T10:55:04.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:04.104 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:55:04.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:03 smithi173 bash[58816]: cluster 2023-10-01T10:55:02.035033+0000 mgr.y (mgr.25072) 1126 : cluster 0 pgmap v1116: 99 pgs: 1 unknown, 1 active+remapped+backfilling, 97 active+clean; 1.6 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 537 op/s; 718/79935 objects misplaced (0.898%); 20 MiB/s, 323 objects/s recovering 2023-10-01T10:55:04.204 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:04.230 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:04.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:04.231 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:55:04.332 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:04.362 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:04.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:04.368 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:55:04.469 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:04.508 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:04.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:04.510 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:55:04.611 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:04.638 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:04.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:04.641 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:55:04.741 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:04.768 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:04.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:04.773 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:55:04.873 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:04.910 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:04.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:04.915 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:55:05.016 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:05.039 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:05.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:05.043 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:55:05.144 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:05.172 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:05.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:05.176 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:55:05.277 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:05.310 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:05.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:05.314 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:55:05.415 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:05.450 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:05.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:05.455 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:55:05.556 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:05.591 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:05.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:05.593 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:55:05.694 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:05.726 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:05.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:05.729 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:55:05.830 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:05.864 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:05.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:05.867 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:05.965 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:05 smithi078 bash[75299]: cluster 2023-10-01T10:55:04.035682+0000 mgr.y (mgr.25072) 1127 : cluster 0 pgmap v1117: 99 pgs: 1 unknown, 98 active+clean; 1.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 16 MiB/s, 258 objects/s recovering 2023-10-01T10:55:05.967 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:06.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:06.010 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:06.010 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:55:06.111 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:06.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:05 smithi173 bash[58816]: cluster 2023-10-01T10:55:04.035682+0000 mgr.y (mgr.25072) 1127 : cluster 0 pgmap v1117: 99 pgs: 1 unknown, 98 active+clean; 1.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 16 MiB/s, 258 objects/s recovering 2023-10-01T10:55:06.142 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:06.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:06.148 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:55:06.248 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:06.273 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:06.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:06.275 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:55:06.318 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:05 smithi078 bash[78274]: cluster 2023-10-01T10:55:04.035682+0000 mgr.y (mgr.25072) 1127 : cluster 0 pgmap v1117: 99 pgs: 1 unknown, 98 active+clean; 1.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 16 MiB/s, 258 objects/s recovering 2023-10-01T10:55:06.376 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:06.400 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:06.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:06.404 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:06.505 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:06.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:06.549 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:06.549 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:55:06.650 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:06.680 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:06.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:06.682 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:55:06.783 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:06.811 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:06.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:06.813 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:55:06.914 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:06.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:06.945 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:06.945 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:55:07.046 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:07.082 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:07.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:07.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:55:07.188 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:07.221 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:07.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:07.229 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:55:07.329 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:07.356 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:07.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:07.359 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:55:07.460 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:07.493 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:07.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:07.496 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:55:07.596 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:07.620 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:07.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:07.625 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:55:07.726 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:07.762 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:07.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:07.767 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:55:07.868 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:07.892 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:07.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:07.897 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:55:07.998 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:08.042 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:08.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:08.047 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:08.148 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:08.173 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:08.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:08.175 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:55:08.276 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:08.286 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:07 smithi173 bash[58816]: cluster 2023-10-01T10:55:06.036308+0000 mgr.y (mgr.25072) 1128 : cluster 0 pgmap v1118: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 1.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 503 op/s; 1000/70707 objects misplaced (1.414%); 10 MiB/s, 160 objects/s recovering 2023-10-01T10:55:08.310 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:08.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:08.318 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:08.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:07 smithi078 bash[75299]: cluster 2023-10-01T10:55:06.036308+0000 mgr.y (mgr.25072) 1128 : cluster 0 pgmap v1118: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 1.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 503 op/s; 1000/70707 objects misplaced (1.414%); 10 MiB/s, 160 objects/s recovering 2023-10-01T10:55:08.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:07 smithi078 bash[78274]: cluster 2023-10-01T10:55:06.036308+0000 mgr.y (mgr.25072) 1128 : cluster 0 pgmap v1118: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 1.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 503 op/s; 1000/70707 objects misplaced (1.414%); 10 MiB/s, 160 objects/s recovering 2023-10-01T10:55:08.419 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:08.441 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:08.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:08.443 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:55:08.544 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:08.572 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:08.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:08.578 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:55:08.678 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:08.709 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:08.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:08.715 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:08.816 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:08.846 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:08.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:08.851 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:55:08.952 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:08.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:08.982 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:08.982 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:55:09.083 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:09.114 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:09.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:09.116 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:55:09.159 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-01T10:55:09.217 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:09.247 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:09.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:09.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:55:09.356 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:09.380 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:09.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:09.382 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:55:09.483 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:09.512 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:09.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:09.515 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:55:09.616 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:09.642 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:09.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:09.646 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:55:09.747 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:09.770 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:09.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:09.772 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:55:09.873 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:09.900 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:09.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:09.905 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:55:10.006 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:10.039 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:10.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:10.046 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:55:10.146 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:10.175 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:10.181 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:09 smithi173 bash[58816]: cluster 2023-10-01T10:55:08.037619+0000 mgr.y (mgr.25072) 1129 : cluster 0 pgmap v1119: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 1.4 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 511 op/s; 1000/67398 objects misplaced (1.484%); 4.8 MiB/s, 76 objects/s recovering 2023-10-01T10:55:10.181 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:09 smithi173 bash[58816]: cluster 2023-10-01T10:55:08.913246+0000 osd.7 (osd.7) 191 : cluster 3 failed to encode map e223 with expected crc 2023-10-01T10:55:10.181 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:09 smithi173 bash[58816]: cluster 2023-10-01T10:55:08.914612+0000 osd.5 (osd.5) 186 : cluster 3 failed to encode map e223 with expected crc 2023-10-01T10:55:10.181 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:09 smithi173 bash[58816]: cluster 2023-10-01T10:55:08.916439+0000 mon.a (mon.0) 1415 : cluster 0 osdmap e223: 8 total, 8 up, 8 in 2023-10-01T10:55:10.181 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:09 smithi173 bash[58816]: cluster 2023-10-01T10:55:08.918163+0000 osd.6 (osd.6) 153 : cluster 3 failed to encode map e223 with expected crc 2023-10-01T10:55:10.181 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:09 smithi173 bash[58816]: cluster 2023-10-01T10:55:08.918182+0000 osd.5 (osd.5) 187 : cluster 3 failed to encode map e223 with expected crc 2023-10-01T10:55:10.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:10.182 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:10.268 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:09 smithi078 bash[75299]: cluster 2023-10-01T10:55:08.037619+0000 mgr.y (mgr.25072) 1129 : cluster 0 pgmap v1119: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 1.4 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 511 op/s; 1000/67398 objects misplaced (1.484%); 4.8 MiB/s, 76 objects/s recovering 2023-10-01T10:55:10.269 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:09 smithi078 bash[75299]: cluster 2023-10-01T10:55:08.913246+0000 osd.7 (osd.7) 191 : cluster 3 failed to encode map e223 with expected crc 2023-10-01T10:55:10.269 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:09 smithi078 bash[75299]: cluster 2023-10-01T10:55:08.914612+0000 osd.5 (osd.5) 186 : cluster 3 failed to encode map e223 with expected crc 2023-10-01T10:55:10.269 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:09 smithi078 bash[75299]: cluster 2023-10-01T10:55:08.916439+0000 mon.a (mon.0) 1415 : cluster 0 osdmap e223: 8 total, 8 up, 8 in 2023-10-01T10:55:10.269 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:09 smithi078 bash[75299]: cluster 2023-10-01T10:55:08.918163+0000 osd.6 (osd.6) 153 : cluster 3 failed to encode map e223 with expected crc 2023-10-01T10:55:10.269 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:09 smithi078 bash[75299]: cluster 2023-10-01T10:55:08.918182+0000 osd.5 (osd.5) 187 : cluster 3 failed to encode map e223 with expected crc 2023-10-01T10:55:10.269 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:09 smithi078 bash[78274]: cluster 2023-10-01T10:55:08.037619+0000 mgr.y (mgr.25072) 1129 : cluster 0 pgmap v1119: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 1.4 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 511 op/s; 1000/67398 objects misplaced (1.484%); 4.8 MiB/s, 76 objects/s recovering 2023-10-01T10:55:10.269 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:09 smithi078 bash[78274]: cluster 2023-10-01T10:55:08.913246+0000 osd.7 (osd.7) 191 : cluster 3 failed to encode map e223 with expected crc 2023-10-01T10:55:10.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:09 smithi078 bash[78274]: cluster 2023-10-01T10:55:08.914612+0000 osd.5 (osd.5) 186 : cluster 3 failed to encode map e223 with expected crc 2023-10-01T10:55:10.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:09 smithi078 bash[78274]: cluster 2023-10-01T10:55:08.916439+0000 mon.a (mon.0) 1415 : cluster 0 osdmap e223: 8 total, 8 up, 8 in 2023-10-01T10:55:10.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:09 smithi078 bash[78274]: cluster 2023-10-01T10:55:08.918163+0000 osd.6 (osd.6) 153 : cluster 3 failed to encode map e223 with expected crc 2023-10-01T10:55:10.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:09 smithi078 bash[78274]: cluster 2023-10-01T10:55:08.918182+0000 osd.5 (osd.5) 187 : cluster 3 failed to encode map e223 with expected crc 2023-10-01T10:55:10.282 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:10.311 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:10.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:10.314 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:55:10.415 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:10.444 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:10.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:10.446 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:55:10.547 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:10.557 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:55:10 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:55:10] "GET /metrics HTTP/1.1" 200 36337 "" "Prometheus/2.33.4" 2023-10-01T10:55:10.578 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:10.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:10.582 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:55:10.683 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:10.706 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:10.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:10.712 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:55:10.813 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:10.849 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:10.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:10.853 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:10.954 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:10.987 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:10.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:10.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:55:11.089 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:11.098 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:10 smithi173 bash[58816]: cluster 2023-10-01T10:55:09.924709+0000 mon.a (mon.0) 1416 : cluster 0 osdmap e224: 8 total, 8 up, 8 in 2023-10-01T10:55:11.098 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:10 smithi173 bash[58816]: cluster 2023-10-01T10:55:09.926119+0000 osd.7 (osd.7) 192 : cluster 3 failed to encode map e224 with expected crc 2023-10-01T10:55:11.098 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:10 smithi173 bash[58816]: cluster 2023-10-01T10:55:09.927627+0000 osd.6 (osd.6) 154 : cluster 3 failed to encode map e224 with expected crc 2023-10-01T10:55:11.098 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:10 smithi173 bash[58816]: cluster 2023-10-01T10:55:09.939970+0000 osd.5 (osd.5) 188 : cluster 3 failed to encode map e224 with expected crc 2023-10-01T10:55:11.123 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:11.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:11.125 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:11.226 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:11.248 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:11.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:11.251 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:11.351 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:11.356 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:10 smithi078 bash[78274]: cluster 2023-10-01T10:55:09.924709+0000 mon.a (mon.0) 1416 : cluster 0 osdmap e224: 8 total, 8 up, 8 in 2023-10-01T10:55:11.356 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:10 smithi078 bash[78274]: cluster 2023-10-01T10:55:09.926119+0000 osd.7 (osd.7) 192 : cluster 3 failed to encode map e224 with expected crc 2023-10-01T10:55:11.356 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:10 smithi078 bash[78274]: cluster 2023-10-01T10:55:09.927627+0000 osd.6 (osd.6) 154 : cluster 3 failed to encode map e224 with expected crc 2023-10-01T10:55:11.356 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:10 smithi078 bash[78274]: cluster 2023-10-01T10:55:09.939970+0000 osd.5 (osd.5) 188 : cluster 3 failed to encode map e224 with expected crc 2023-10-01T10:55:11.356 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:10 smithi078 bash[75299]: cluster 2023-10-01T10:55:09.924709+0000 mon.a (mon.0) 1416 : cluster 0 osdmap e224: 8 total, 8 up, 8 in 2023-10-01T10:55:11.356 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:10 smithi078 bash[75299]: cluster 2023-10-01T10:55:09.926119+0000 osd.7 (osd.7) 192 : cluster 3 failed to encode map e224 with expected crc 2023-10-01T10:55:11.357 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:10 smithi078 bash[75299]: cluster 2023-10-01T10:55:09.927627+0000 osd.6 (osd.6) 154 : cluster 3 failed to encode map e224 with expected crc 2023-10-01T10:55:11.357 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:10 smithi078 bash[75299]: cluster 2023-10-01T10:55:09.939970+0000 osd.5 (osd.5) 188 : cluster 3 failed to encode map e224 with expected crc 2023-10-01T10:55:11.387 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:11.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:11.391 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:11.492 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:11.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:11.527 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:11.527 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:55:11.628 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:11.663 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:11.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:11.668 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:55:11.769 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:11.797 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:11.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:11.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:55:11.904 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:11.930 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:11.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:11.932 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:12.033 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:12.043 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:11 smithi078 bash[75299]: cluster 2023-10-01T10:55:10.038022+0000 mgr.y (mgr.25072) 1130 : cluster 0 pgmap v1122: 99 pgs: 1 peering, 98 active+clean; 1.2 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 917 op/s; 7.9 MiB/s, 127 objects/s recovering 2023-10-01T10:55:12.044 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:11 smithi078 bash[78274]: cluster 2023-10-01T10:55:10.038022+0000 mgr.y (mgr.25072) 1130 : cluster 0 pgmap v1122: 99 pgs: 1 peering, 98 active+clean; 1.2 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 917 op/s; 7.9 MiB/s, 127 objects/s recovering 2023-10-01T10:55:12.072 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:12.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:12.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:55:12.175 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:12.184 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:11 smithi173 bash[58816]: cluster 2023-10-01T10:55:10.038022+0000 mgr.y (mgr.25072) 1130 : cluster 0 pgmap v1122: 99 pgs: 1 peering, 98 active+clean; 1.2 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 917 op/s; 7.9 MiB/s, 127 objects/s recovering 2023-10-01T10:55:12.219 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:12.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:12.223 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:55:12.324 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:12.348 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:12.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:12.353 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:12.454 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:12.478 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:12.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:12.487 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:55:12.587 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:12.616 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:12.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:12.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:55:12.723 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:12.749 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:12.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:12.751 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:55:12.798 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T10:55:12.842 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T10:55:12.852 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:12.892 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:12.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:12.898 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:55:12.902 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T10:55:12.961 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T10:55:12.998 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:13.003 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T10:55:13.034 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:13.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:13.037 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:55:13.044 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T10:55:13.093 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T10:55:13.138 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:13.140 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T10:55:13.141 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T10:55:13.142 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T10:55:13.142 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T10:55:13.142 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T10:55:13.142 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T10:55:13.142 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T10:55:13.142 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T10:55:13.143 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T10:55:13.176 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:13.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:13.178 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:55:13.279 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:13.327 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:13.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:13.331 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:55:13.431 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:13.459 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:13.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:13.461 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:13.562 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:13.564 INFO:teuthology.orchestra.run.smithi078.stderr:nodeep-scrub is set 2023-10-01T10:55:13.571 INFO:teuthology.orchestra.run.smithi078.stderr:set pool 11 pgp_num to 26 2023-10-01T10:55:13.599 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:13.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:13.604 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:13.705 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:13.717 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:13 smithi173 bash[58816]: cluster 2023-10-01T10:55:12.038480+0000 mgr.y (mgr.25072) 1131 : cluster 0 pgmap v1123: 99 pgs: 1 peering, 98 active+clean; 1.2 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 5.3 MiB/s, 84 objects/s recovering 2023-10-01T10:55:13.717 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:13 smithi173 bash[58816]: audit 2023-10-01T10:55:12.595704+0000 mon.c (mon.2) 182 : audit 1 from='client.? 172.21.15.78:0/2025627819' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-01T10:55:13.717 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:13 smithi173 bash[58816]: audit 2023-10-01T10:55:12.596616+0000 mon.a (mon.0) 1417 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-01T10:55:13.717 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:13 smithi173 bash[58816]: audit 2023-10-01T10:55:12.740801+0000 mon.a (mon.0) 1418 : audit 0 from='client.? 172.21.15.78:0/9830828' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:55:13.717 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:13 smithi173 bash[58816]: audit 2023-10-01T10:55:13.100489+0000 mon.a (mon.0) 1419 : audit 1 from='client.? 172.21.15.78:0/3985229060' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:55:13.739 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:13.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:13.747 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:55:13.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:13 smithi078 bash[75299]: cluster 2023-10-01T10:55:12.038480+0000 mgr.y (mgr.25072) 1131 : cluster 0 pgmap v1123: 99 pgs: 1 peering, 98 active+clean; 1.2 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 5.3 MiB/s, 84 objects/s recovering 2023-10-01T10:55:13.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:13 smithi078 bash[75299]: audit 2023-10-01T10:55:12.595704+0000 mon.c (mon.2) 182 : audit 1 from='client.? 172.21.15.78:0/2025627819' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-01T10:55:13.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:13 smithi078 bash[75299]: audit 2023-10-01T10:55:12.596616+0000 mon.a (mon.0) 1417 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-01T10:55:13.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:13 smithi078 bash[75299]: audit 2023-10-01T10:55:12.740801+0000 mon.a (mon.0) 1418 : audit 0 from='client.? 172.21.15.78:0/9830828' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:55:13.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:13 smithi078 bash[75299]: audit 2023-10-01T10:55:13.100489+0000 mon.a (mon.0) 1419 : audit 1 from='client.? 172.21.15.78:0/3985229060' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:55:13.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:13 smithi078 bash[78274]: cluster 2023-10-01T10:55:12.038480+0000 mgr.y (mgr.25072) 1131 : cluster 0 pgmap v1123: 99 pgs: 1 peering, 98 active+clean; 1.2 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 5.3 MiB/s, 84 objects/s recovering 2023-10-01T10:55:13.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:13 smithi078 bash[78274]: audit 2023-10-01T10:55:12.595704+0000 mon.c (mon.2) 182 : audit 1 from='client.? 172.21.15.78:0/2025627819' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-01T10:55:13.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:13 smithi078 bash[78274]: audit 2023-10-01T10:55:12.596616+0000 mon.a (mon.0) 1417 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num", "val": "26"}]: dispatch 2023-10-01T10:55:13.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:13 smithi078 bash[78274]: audit 2023-10-01T10:55:12.740801+0000 mon.a (mon.0) 1418 : audit 0 from='client.? 172.21.15.78:0/9830828' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:55:13.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:13 smithi078 bash[78274]: audit 2023-10-01T10:55:13.100489+0000 mon.a (mon.0) 1419 : audit 1 from='client.? 172.21.15.78:0/3985229060' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:55:13.849 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:13.883 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:13.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:13.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:55:13.988 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:14.027 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:14.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:14.030 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:55:14.131 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:14.160 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:14.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:14.165 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:55:14.266 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:14.297 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:14.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:14.300 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:55:14.400 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:14.429 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:14.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:14.431 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:55:14.532 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:14.564 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:14.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:14.567 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:55:14.669 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:14.678 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:14 smithi078 bash[75299]: cluster 2023-10-01T10:55:13.549538+0000 mon.a (mon.0) 1420 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:55:14.679 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:14 smithi078 bash[75299]: audit 2023-10-01T10:55:13.554409+0000 mon.a (mon.0) 1421 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num", "val": "26"}]': finished 2023-10-01T10:55:14.679 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:14 smithi078 bash[75299]: audit 2023-10-01T10:55:13.554928+0000 mon.a (mon.0) 1422 : audit 1 from='client.? 172.21.15.78:0/3985229060' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T10:55:14.679 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:14 smithi078 bash[75299]: cluster 2023-10-01T10:55:13.554983+0000 mon.a (mon.0) 1423 : cluster 0 osdmap e225: 8 total, 8 up, 8 in 2023-10-01T10:55:14.679 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:14 smithi078 bash[75299]: cluster 2023-10-01T10:55:13.560306+0000 osd.7 (osd.7) 193 : cluster 3 failed to encode map e225 with expected crc 2023-10-01T10:55:14.679 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:14 smithi078 bash[75299]: cluster 2023-10-01T10:55:13.569264+0000 osd.7 (osd.7) 194 : cluster 3 failed to encode map e225 with expected crc 2023-10-01T10:55:14.679 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:14 smithi078 bash[75299]: cluster 2023-10-01T10:55:13.573529+0000 osd.6 (osd.6) 155 : cluster 3 failed to encode map e225 with expected crc 2023-10-01T10:55:14.679 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:14 smithi078 bash[75299]: cluster 2023-10-01T10:55:13.582601+0000 osd.5 (osd.5) 189 : cluster 3 failed to encode map e225 with expected crc 2023-10-01T10:55:14.680 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:14 smithi078 bash[78274]: cluster 2023-10-01T10:55:13.549538+0000 mon.a (mon.0) 1420 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:55:14.680 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:14 smithi078 bash[78274]: audit 2023-10-01T10:55:13.554409+0000 mon.a (mon.0) 1421 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num", "val": "26"}]': finished 2023-10-01T10:55:14.680 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:14 smithi078 bash[78274]: audit 2023-10-01T10:55:13.554928+0000 mon.a (mon.0) 1422 : audit 1 from='client.? 172.21.15.78:0/3985229060' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T10:55:14.680 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:14 smithi078 bash[78274]: cluster 2023-10-01T10:55:13.554983+0000 mon.a (mon.0) 1423 : cluster 0 osdmap e225: 8 total, 8 up, 8 in 2023-10-01T10:55:14.680 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:14 smithi078 bash[78274]: cluster 2023-10-01T10:55:13.560306+0000 osd.7 (osd.7) 193 : cluster 3 failed to encode map e225 with expected crc 2023-10-01T10:55:14.680 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:14 smithi078 bash[78274]: cluster 2023-10-01T10:55:13.569264+0000 osd.7 (osd.7) 194 : cluster 3 failed to encode map e225 with expected crc 2023-10-01T10:55:14.680 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:14 smithi078 bash[78274]: cluster 2023-10-01T10:55:13.573529+0000 osd.6 (osd.6) 155 : cluster 3 failed to encode map e225 with expected crc 2023-10-01T10:55:14.680 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:14 smithi078 bash[78274]: cluster 2023-10-01T10:55:13.582601+0000 osd.5 (osd.5) 189 : cluster 3 failed to encode map e225 with expected crc 2023-10-01T10:55:14.694 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:14.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:14.696 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:55:14.797 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:14.819 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:14.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:14.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:55:14.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:14 smithi173 bash[58816]: cluster 2023-10-01T10:55:13.549538+0000 mon.a (mon.0) 1420 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:55:14.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:14 smithi173 bash[58816]: audit 2023-10-01T10:55:13.554409+0000 mon.a (mon.0) 1421 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num", "val": "26"}]': finished 2023-10-01T10:55:14.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:14 smithi173 bash[58816]: audit 2023-10-01T10:55:13.554928+0000 mon.a (mon.0) 1422 : audit 1 from='client.? 172.21.15.78:0/3985229060' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T10:55:14.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:14 smithi173 bash[58816]: cluster 2023-10-01T10:55:13.554983+0000 mon.a (mon.0) 1423 : cluster 0 osdmap e225: 8 total, 8 up, 8 in 2023-10-01T10:55:14.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:14 smithi173 bash[58816]: cluster 2023-10-01T10:55:13.560306+0000 osd.7 (osd.7) 193 : cluster 3 failed to encode map e225 with expected crc 2023-10-01T10:55:14.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:14 smithi173 bash[58816]: cluster 2023-10-01T10:55:13.569264+0000 osd.7 (osd.7) 194 : cluster 3 failed to encode map e225 with expected crc 2023-10-01T10:55:14.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:14 smithi173 bash[58816]: cluster 2023-10-01T10:55:13.573529+0000 osd.6 (osd.6) 155 : cluster 3 failed to encode map e225 with expected crc 2023-10-01T10:55:14.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:14 smithi173 bash[58816]: cluster 2023-10-01T10:55:13.582601+0000 osd.5 (osd.5) 189 : cluster 3 failed to encode map e225 with expected crc 2023-10-01T10:55:14.923 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:14.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:14.960 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:14.960 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:15.061 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:15.085 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:15.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:15.088 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:55:15.189 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:15.218 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:15.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:15.222 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:55:15.323 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:15.348 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:15.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:15.351 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:55:15.451 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:15.481 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:15.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:15.486 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:55:15.587 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:15.611 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:15.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:15.615 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:55:15.716 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:15.739 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:15.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:15.745 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:55:15.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:15 smithi078 bash[75299]: cluster 2023-10-01T10:55:14.038922+0000 mgr.y (mgr.25072) 1132 : cluster 0 pgmap v1125: 99 pgs: 1 peering, 98 active+clean; 1005 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 4.1 MiB/s, 65 objects/s recovering 2023-10-01T10:55:15.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:15 smithi078 bash[78274]: cluster 2023-10-01T10:55:14.038922+0000 mgr.y (mgr.25072) 1132 : cluster 0 pgmap v1125: 99 pgs: 1 peering, 98 active+clean; 1005 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 4.1 MiB/s, 65 objects/s recovering 2023-10-01T10:55:15.846 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:15.867 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:15.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:15 smithi173 bash[58816]: cluster 2023-10-01T10:55:14.038922+0000 mgr.y (mgr.25072) 1132 : cluster 0 pgmap v1125: 99 pgs: 1 peering, 98 active+clean; 1005 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 4.1 MiB/s, 65 objects/s recovering 2023-10-01T10:55:15.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:15.870 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:55:15.971 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:16.000 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:16.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:16.001 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:55:16.102 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:16.127 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:16.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:16.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:55:16.232 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:16.262 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:16.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:16.267 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:55:16.367 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:16.390 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:16.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:16.393 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:16.463 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T10:55:16.494 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:16.536 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:16.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:16.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:55:16.589 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:55:16.640 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:16.666 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:16.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:16.669 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:16.769 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:16.800 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:16.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:16.804 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:55:16.905 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:16.941 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:16.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:16.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:55:17.044 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:17.080 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:17.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:17.083 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:55:17.184 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:17.215 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:17.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:17.219 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:55:17.321 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:17.355 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:17.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:17.359 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:55:17.460 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:17.483 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:17.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:17.490 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:55:17.590 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:17.604 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:17 smithi173 bash[58816]: cluster 2023-10-01T10:55:16.039510+0000 mgr.y (mgr.25072) 1133 : cluster 0 pgmap v1126: 99 pgs: 99 active+clean; 958 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 3.4 MiB/s, 55 objects/s recovering 2023-10-01T10:55:17.604 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:17 smithi173 bash[58816]: audit 2023-10-01T10:55:17.442588+0000 mon.a (mon.0) 1424 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:55:17.624 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:17.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:17.626 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:55:17.727 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:17.735 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:17 smithi078 bash[75299]: cluster 2023-10-01T10:55:16.039510+0000 mgr.y (mgr.25072) 1133 : cluster 0 pgmap v1126: 99 pgs: 99 active+clean; 958 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 3.4 MiB/s, 55 objects/s recovering 2023-10-01T10:55:17.735 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:17 smithi078 bash[75299]: audit 2023-10-01T10:55:17.442588+0000 mon.a (mon.0) 1424 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:55:17.736 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:17 smithi078 bash[78274]: cluster 2023-10-01T10:55:16.039510+0000 mgr.y (mgr.25072) 1133 : cluster 0 pgmap v1126: 99 pgs: 99 active+clean; 958 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 3.4 MiB/s, 55 objects/s recovering 2023-10-01T10:55:17.736 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:17 smithi078 bash[78274]: audit 2023-10-01T10:55:17.442588+0000 mon.a (mon.0) 1424 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:55:17.763 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:17.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:17.765 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:55:17.866 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:17.894 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:17.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:17.899 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:55:18.000 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:18.025 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:18.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:18.032 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:55:18.133 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:18.161 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:18.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:18.168 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:55:18.220 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T10:55:18.269 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:18.296 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:18.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:18.298 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:55:18.399 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:18.427 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:18.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:18.430 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:55:18.531 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:18.558 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:18.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:18.563 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:55:18.663 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:18.702 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:18.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:18.706 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:18.807 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:18.835 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:18.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:18.841 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:18.942 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:18.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:18.975 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:18.976 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:55:19.077 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:19.119 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:19.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:19.125 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:55:19.226 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:19.252 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:19.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:19.256 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:55:19.357 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:19.387 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:19.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:19.391 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:55:19.492 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:19.529 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:19.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:19.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:55:19.632 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:19.664 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:19.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:19.667 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:55:19.768 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:19.776 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:19 smithi078 bash[78274]: cluster 2023-10-01T10:55:18.040105+0000 mgr.y (mgr.25072) 1134 : cluster 0 pgmap v1127: 99 pgs: 99 active+clean; 890 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 0 B/s, 0 objects/s recovering 2023-10-01T10:55:19.778 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:19 smithi078 bash[75299]: cluster 2023-10-01T10:55:18.040105+0000 mgr.y (mgr.25072) 1134 : cluster 0 pgmap v1127: 99 pgs: 99 active+clean; 890 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 0 B/s, 0 objects/s recovering 2023-10-01T10:55:19.795 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:19.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:19.800 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:55:19.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:19 smithi173 bash[58816]: cluster 2023-10-01T10:55:18.040105+0000 mgr.y (mgr.25072) 1134 : cluster 0 pgmap v1127: 99 pgs: 99 active+clean; 890 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 0 B/s, 0 objects/s recovering 2023-10-01T10:55:19.901 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:19.943 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:19.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:19.947 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:20.048 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:20.076 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:20.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:20.083 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:20.184 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:20.216 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:20.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:20.218 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:55:20.319 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:20.350 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:20.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:20.356 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:55:20.456 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:20.486 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:20.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:20.488 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:55:20.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:55:20 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:55:20] "GET /metrics HTTP/1.1" 200 36329 "" "Prometheus/2.33.4" 2023-10-01T10:55:20.588 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:20.617 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:20.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:20.623 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:55:20.724 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:20.748 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:20.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:20.753 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:55:20.763 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:55:20.854 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:20.884 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:20.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:20.886 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:55:20.987 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:21.015 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:21.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:21.018 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:55:21.119 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:21.144 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:21.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:21.147 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:55:21.237 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T10:55:21.238 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T10:55:21.238 INFO:tasks.thrashosds.thrasher:Removing osd 6, in_osds are: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T10:55:21.238 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd out 6 2023-10-01T10:55:21.248 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:21.326 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:21.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:21.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:55:21.433 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:21.475 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:21.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:21.477 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:55:21.578 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:21.606 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:21.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:21.609 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:21.710 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:21.746 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:21.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:21.751 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:55:21.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:21 smithi078 bash[78274]: cluster 2023-10-01T10:55:20.042712+0000 mgr.y (mgr.25072) 1135 : cluster 0 pgmap v1128: 99 pgs: 99 active+clean; 676 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 0 B/s, 0 objects/s recovering 2023-10-01T10:55:21.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:21 smithi078 bash[75299]: cluster 2023-10-01T10:55:20.042712+0000 mgr.y (mgr.25072) 1135 : cluster 0 pgmap v1128: 99 pgs: 99 active+clean; 676 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 0 B/s, 0 objects/s recovering 2023-10-01T10:55:21.851 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:21.860 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:21 smithi173 bash[58816]: cluster 2023-10-01T10:55:20.042712+0000 mgr.y (mgr.25072) 1135 : cluster 0 pgmap v1128: 99 pgs: 99 active+clean; 676 MiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 0 B/s, 0 objects/s recovering 2023-10-01T10:55:21.883 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:21.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:21.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:55:21.987 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:22.004 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:22.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:22.006 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:55:22.107 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:22.139 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:22.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:22.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:55:22.244 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:22.266 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:22.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:22.270 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:55:22.370 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:22.398 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:22.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:22.402 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:55:22.503 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:22.529 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:22.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:22.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:55:22.632 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:22.669 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:22.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:22.670 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:55:22.736 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2023-10-01T10:55:22.736 INFO:teuthology.orchestra.run.smithi078.stdout: "size": 20, 2023-10-01T10:55:22.736 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 600, 2023-10-01T10:55:22.736 INFO:teuthology.orchestra.run.smithi078.stdout: "ops": [ 2023-10-01T10:55:22.736 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.737 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46037.0:170451 11.10 11:086e41b9:::benchmark_data_smithi078_157677_object63280:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e192)", 2023-10-01T10:55:22.737 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:54:21.631789+0000", 2023-10-01T10:55:22.737 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 61.097146713000001, 2023-10-01T10:55:22.737 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1682773260000001, 2023-10-01T10:55:22.737 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:55:22.737 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:55:22.737 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:55:22.737 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46037", 2023-10-01T10:55:22.737 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:55:22.737 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 170451 2023-10-01T10:55:22.737 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.737 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:55:22.738 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.738 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:55:22.738 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631789+0000", 2023-10-01T10:55:22.738 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:55:22.738 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.738 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.738 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:55:22.738 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631789+0000", 2023-10-01T10:55:22.738 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:55:22.738 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.738 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.738 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:55:22.739 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631789+0000", 2023-10-01T10:55:22.739 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999995 2023-10-01T10:55:22.739 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.739 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.739 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:55:22.739 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631791+0000", 2023-10-01T10:55:22.739 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.781e-06 2023-10-01T10:55:22.739 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.739 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.739 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:55:22.739 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631794+0000", 2023-10-01T10:55:22.739 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.6399999999999999e-06 2023-10-01T10:55:22.740 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.740 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.740 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:55:22.740 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631798+0000", 2023-10-01T10:55:22.740 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.4460000000000001e-06 2023-10-01T10:55:22.740 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.740 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.740 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:55:22.740 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.632029+0000", 2023-10-01T10:55:22.740 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00023185100000000001 2023-10-01T10:55:22.740 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.740 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.741 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T10:55:22.741 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.632038+0000", 2023-10-01T10:55:22.741 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.9800000000000004e-06 2023-10-01T10:55:22.741 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.741 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.741 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:55:22.741 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.766878+0000", 2023-10-01T10:55:22.741 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.134839084 2023-10-01T10:55:22.741 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.741 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.741 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:55:22.741 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.767018+0000", 2023-10-01T10:55:22.742 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00014021199999999999 2023-10-01T10:55:22.742 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.742 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.742 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 0,2,3,6,7", 2023-10-01T10:55:22.742 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.767354+0000", 2023-10-01T10:55:22.742 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00033653600000000002 2023-10-01T10:55:22.742 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.742 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.742 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.742 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.775456+0000", 2023-10-01T10:55:22.742 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0081021900000000004 2023-10-01T10:55:22.743 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.743 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.743 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.743 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.775494+0000", 2023-10-01T10:55:22.743 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.7963999999999998e-05 2023-10-01T10:55:22.743 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.743 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.743 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.743 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.775530+0000", 2023-10-01T10:55:22.743 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.5546999999999997e-05 2023-10-01T10:55:22.743 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.743 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.744 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.744 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.786108+0000", 2023-10-01T10:55:22.744 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.010578392000000001 2023-10-01T10:55:22.744 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.744 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.744 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.745 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.799819+0000", 2023-10-01T10:55:22.745 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.013711004000000001 2023-10-01T10:55:22.746 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.746 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.746 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:55:22.746 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.799867+0000", 2023-10-01T10:55:22.746 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.7336000000000002e-05 2023-10-01T10:55:22.746 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.746 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.746 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:55:22.746 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.799922+0000", 2023-10-01T10:55:22.746 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.4821000000000002e-05 2023-10-01T10:55:22.747 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.747 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.747 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:55:22.747 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.800066+0000", 2023-10-01T10:55:22.747 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00014478099999999999 2023-10-01T10:55:22.747 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:55:22.747 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:55:22.747 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:55:22.747 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.747 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.747 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46037.0:170460 11.10 11:0dddcbba:::benchmark_data_smithi078_157677_object63289:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e192)", 2023-10-01T10:55:22.747 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:54:21.631800+0000", 2023-10-01T10:55:22.748 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 61.097135737000002, 2023-10-01T10:55:22.748 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1686122050000001, 2023-10-01T10:55:22.748 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:55:22.748 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:55:22.748 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:55:22.748 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46037", 2023-10-01T10:55:22.748 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:55:22.748 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 170460 2023-10-01T10:55:22.748 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.748 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:55:22.748 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.748 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:55:22.749 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631800+0000", 2023-10-01T10:55:22.749 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:55:22.749 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.749 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.749 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:55:22.749 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631800+0000", 2023-10-01T10:55:22.749 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:55:22.749 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.749 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.749 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:55:22.749 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631800+0000", 2023-10-01T10:55:22.750 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967296 2023-10-01T10:55:22.750 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.750 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.750 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:55:22.750 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631801+0000", 2023-10-01T10:55:22.750 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.4389999999999999e-06 2023-10-01T10:55:22.750 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.750 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.750 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:55:22.750 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631804+0000", 2023-10-01T10:55:22.750 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.202e-06 2023-10-01T10:55:22.751 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.751 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.751 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:55:22.751 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631807+0000", 2023-10-01T10:55:22.751 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.0000000000000001e-06 2023-10-01T10:55:22.751 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.751 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.751 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:55:22.751 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.632065+0000", 2023-10-01T10:55:22.751 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000257667 2023-10-01T10:55:22.751 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.751 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.752 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T10:55:22.752 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.632069+0000", 2023-10-01T10:55:22.752 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.1200000000000004e-06 2023-10-01T10:55:22.752 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.752 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.752 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:55:22.752 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.769227+0000", 2023-10-01T10:55:22.752 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1371582929999999 2023-10-01T10:55:22.752 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.752 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.752 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:55:22.752 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.769401+0000", 2023-10-01T10:55:22.753 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000173303 2023-10-01T10:55:22.753 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.753 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.753 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 0,2,3,6,7", 2023-10-01T10:55:22.753 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.769674+0000", 2023-10-01T10:55:22.753 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00027319600000000002 2023-10-01T10:55:22.753 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.753 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.753 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.753 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.775569+0000", 2023-10-01T10:55:22.753 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0058948639999999997 2023-10-01T10:55:22.754 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.754 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.754 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.754 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.786075+0000", 2023-10-01T10:55:22.754 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.010505782 2023-10-01T10:55:22.754 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.754 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.754 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.754 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.786209+0000", 2023-10-01T10:55:22.754 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00013463200000000001 2023-10-01T10:55:22.754 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.754 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.755 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.755 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.788659+0000", 2023-10-01T10:55:22.755 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.002449522 2023-10-01T10:55:22.755 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.755 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.755 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:55:22.755 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.800076+0000", 2023-10-01T10:55:22.755 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.011417437000000001 2023-10-01T10:55:22.755 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.755 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.755 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.755 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.800226+0000", 2023-10-01T10:55:22.756 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00015015100000000001 2023-10-01T10:55:22.756 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.756 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.756 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:55:22.756 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.800273+0000", 2023-10-01T10:55:22.756 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.6693000000000002e-05 2023-10-01T10:55:22.756 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.756 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.756 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:55:22.756 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.800412+0000", 2023-10-01T10:55:22.756 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000139135 2023-10-01T10:55:22.756 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:55:22.757 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:55:22.757 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:55:22.757 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.757 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.757 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46037.0:170498 11.10 11:0970561e:::benchmark_data_smithi078_157677_object63327:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e192)", 2023-10-01T10:55:22.757 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:54:21.631810+0000", 2023-10-01T10:55:22.757 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 61.097126039999999, 2023-10-01T10:55:22.757 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.172262259, 2023-10-01T10:55:22.757 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:55:22.757 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:55:22.757 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:55:22.758 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46037", 2023-10-01T10:55:22.758 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:55:22.758 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 170498 2023-10-01T10:55:22.758 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.758 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:55:22.758 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.758 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:55:22.758 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631810+0000", 2023-10-01T10:55:22.758 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:55:22.758 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.758 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.758 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:55:22.759 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631810+0000", 2023-10-01T10:55:22.759 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:55:22.759 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.759 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.759 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:55:22.759 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631809+0000", 2023-10-01T10:55:22.759 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967296 2023-10-01T10:55:22.759 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.759 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.759 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:55:22.759 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631811+0000", 2023-10-01T10:55:22.759 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.5209999999999999e-06 2023-10-01T10:55:22.760 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.760 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.760 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:55:22.760 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631814+0000", 2023-10-01T10:55:22.760 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.157e-06 2023-10-01T10:55:22.760 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.760 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.760 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:55:22.760 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631818+0000", 2023-10-01T10:55:22.760 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.8670000000000004e-06 2023-10-01T10:55:22.760 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.760 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.760 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:55:22.761 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.632086+0000", 2023-10-01T10:55:22.761 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00026807000000000003 2023-10-01T10:55:22.761 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.761 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.761 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T10:55:22.761 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.632090+0000", 2023-10-01T10:55:22.761 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.1729999999999997e-06 2023-10-01T10:55:22.761 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.761 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.761 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:55:22.761 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.771382+0000", 2023-10-01T10:55:22.761 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1392917570000001 2023-10-01T10:55:22.762 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.762 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.762 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:55:22.762 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.771510+0000", 2023-10-01T10:55:22.762 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00012846299999999999 2023-10-01T10:55:22.762 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.762 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.762 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 0,2,3,6,7", 2023-10-01T10:55:22.762 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.771772+0000", 2023-10-01T10:55:22.762 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00026122299999999999 2023-10-01T10:55:22.762 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.762 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.763 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.763 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.786038+0000", 2023-10-01T10:55:22.763 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.014266778000000001 2023-10-01T10:55:22.763 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.763 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.763 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.763 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.786177+0000", 2023-10-01T10:55:22.763 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00013871900000000001 2023-10-01T10:55:22.763 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.763 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.763 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.763 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.786307+0000", 2023-10-01T10:55:22.764 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00012935900000000001 2023-10-01T10:55:22.764 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.764 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.764 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.764 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.788691+0000", 2023-10-01T10:55:22.764 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.002384803 2023-10-01T10:55:22.764 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.764 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.764 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:55:22.764 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.800083+0000", 2023-10-01T10:55:22.764 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.011391434000000001 2023-10-01T10:55:22.764 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.765 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.765 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.765 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.803877+0000", 2023-10-01T10:55:22.765 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0037939190000000002 2023-10-01T10:55:22.765 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.765 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.765 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:55:22.765 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.803945+0000", 2023-10-01T10:55:22.765 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.8113000000000003e-05 2023-10-01T10:55:22.765 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.765 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.765 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:55:22.766 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.804072+0000", 2023-10-01T10:55:22.766 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000127133 2023-10-01T10:55:22.766 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:55:22.766 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:55:22.766 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:55:22.766 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.766 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.766 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46037.0:170504 11.10 11:0d37d827:::benchmark_data_smithi078_157677_object63333:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e192)", 2023-10-01T10:55:22.766 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:54:21.631822+0000", 2023-10-01T10:55:22.766 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 61.097113655999998, 2023-10-01T10:55:22.766 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1725379440000001, 2023-10-01T10:55:22.766 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:55:22.767 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:55:22.767 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:55:22.767 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46037", 2023-10-01T10:55:22.767 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:55:22.767 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 170504 2023-10-01T10:55:22.767 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.767 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:55:22.767 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.767 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:55:22.767 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631822+0000", 2023-10-01T10:55:22.767 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:55:22.767 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.768 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.768 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:55:22.768 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631822+0000", 2023-10-01T10:55:22.768 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:55:22.768 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.768 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.768 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:55:22.768 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631822+0000", 2023-10-01T10:55:22.768 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967296 2023-10-01T10:55:22.768 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.768 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.768 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:55:22.769 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631827+0000", 2023-10-01T10:55:22.769 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.7770000000000002e-06 2023-10-01T10:55:22.769 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.769 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.769 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:55:22.769 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631831+0000", 2023-10-01T10:55:22.769 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.9530000000000001e-06 2023-10-01T10:55:22.769 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.769 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.769 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:55:22.769 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631834+0000", 2023-10-01T10:55:22.769 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.2619999999999999e-06 2023-10-01T10:55:22.770 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.770 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.770 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:55:22.770 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.632122+0000", 2023-10-01T10:55:22.770 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000288354 2023-10-01T10:55:22.770 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.770 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.770 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T10:55:22.770 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.632131+0000", 2023-10-01T10:55:22.770 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.2259999999999994e-06 2023-10-01T10:55:22.770 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.770 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.771 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:55:22.771 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.773447+0000", 2023-10-01T10:55:22.771 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.141315662 2023-10-01T10:55:22.771 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.771 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.771 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:55:22.771 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.773574+0000", 2023-10-01T10:55:22.771 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00012720599999999999 2023-10-01T10:55:22.771 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.771 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.771 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 0,2,3,6,7", 2023-10-01T10:55:22.772 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.773798+0000", 2023-10-01T10:55:22.772 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000223594 2023-10-01T10:55:22.772 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.772 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.772 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.772 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.786141+0000", 2023-10-01T10:55:22.772 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.012343394000000001 2023-10-01T10:55:22.772 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.772 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.772 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.772 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.786274+0000", 2023-10-01T10:55:22.772 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00013266600000000001 2023-10-01T10:55:22.772 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.773 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.773 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.773 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.786403+0000", 2023-10-01T10:55:22.773 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00012877099999999999 2023-10-01T10:55:22.773 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.773 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.773 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.773 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.788723+0000", 2023-10-01T10:55:22.773 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.002320707 2023-10-01T10:55:22.773 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.773 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.773 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:55:22.774 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.800089+0000", 2023-10-01T10:55:22.774 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.011366029 2023-10-01T10:55:22.774 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.774 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.774 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.774 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.804150+0000", 2023-10-01T10:55:22.774 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.004060124 2023-10-01T10:55:22.774 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.774 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.774 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:55:22.774 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.804208+0000", 2023-10-01T10:55:22.775 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.8869000000000002e-05 2023-10-01T10:55:22.775 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.775 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.775 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:55:22.775 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.804360+0000", 2023-10-01T10:55:22.775 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00015158000000000001 2023-10-01T10:55:22.775 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:55:22.775 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:55:22.775 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:55:22.775 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.775 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.775 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46037.0:170514 11.10 11:0a306d56:::benchmark_data_smithi078_157677_object63343:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e192)", 2023-10-01T10:55:22.776 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:54:21.631836+0000", 2023-10-01T10:55:22.776 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 61.097099432999997, 2023-10-01T10:55:22.776 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.172768504, 2023-10-01T10:55:22.776 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:55:22.776 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:55:22.776 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:55:22.776 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46037", 2023-10-01T10:55:22.776 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:55:22.776 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 170514 2023-10-01T10:55:22.776 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.776 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:55:22.776 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.776 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:55:22.777 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631836+0000", 2023-10-01T10:55:22.777 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:55:22.777 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.777 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.777 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:55:22.777 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631836+0000", 2023-10-01T10:55:22.777 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:55:22.777 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.777 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.777 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:55:22.777 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631836+0000", 2023-10-01T10:55:22.777 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967296 2023-10-01T10:55:22.778 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.778 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.778 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:55:22.778 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631838+0000", 2023-10-01T10:55:22.778 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.5579999999999999e-06 2023-10-01T10:55:22.778 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.778 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.778 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:55:22.778 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631842+0000", 2023-10-01T10:55:22.778 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.1099999999999996e-06 2023-10-01T10:55:22.778 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.778 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.779 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:55:22.779 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631845+0000", 2023-10-01T10:55:22.779 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.0920000000000002e-06 2023-10-01T10:55:22.779 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.779 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.779 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:55:22.779 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.632159+0000", 2023-10-01T10:55:22.779 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00031405499999999999 2023-10-01T10:55:22.779 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.779 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.779 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T10:55:22.780 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.632163+0000", 2023-10-01T10:55:22.780 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.924e-06 2023-10-01T10:55:22.780 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.780 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.780 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:55:22.780 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.775595+0000", 2023-10-01T10:55:22.780 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1434320099999999 2023-10-01T10:55:22.780 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.780 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.780 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:55:22.780 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.775723+0000", 2023-10-01T10:55:22.780 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00012839399999999999 2023-10-01T10:55:22.781 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.781 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.781 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 0,2,3,6,7", 2023-10-01T10:55:22.781 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.775945+0000", 2023-10-01T10:55:22.781 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00022158099999999999 2023-10-01T10:55:22.781 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.781 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.781 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.781 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.786243+0000", 2023-10-01T10:55:22.781 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.010297763999999999 2023-10-01T10:55:22.781 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.782 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.782 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.782 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.786371+0000", 2023-10-01T10:55:22.782 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00012810100000000001 2023-10-01T10:55:22.782 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.782 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.782 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.782 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.788759+0000", 2023-10-01T10:55:22.782 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0023886739999999999 2023-10-01T10:55:22.782 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.782 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.782 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:55:22.783 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.800095+0000", 2023-10-01T10:55:22.783 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.011336112000000001 2023-10-01T10:55:22.783 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.783 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.783 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.783 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.800193+0000", 2023-10-01T10:55:22.783 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.8064000000000002e-05 2023-10-01T10:55:22.783 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.783 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.783 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.783 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.804417+0000", 2023-10-01T10:55:22.783 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0042237170000000001 2023-10-01T10:55:22.784 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.784 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.784 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:55:22.784 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.804472+0000", 2023-10-01T10:55:22.784 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.4425999999999997e-05 2023-10-01T10:55:22.784 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.784 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.784 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:55:22.784 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.804605+0000", 2023-10-01T10:55:22.784 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000133156 2023-10-01T10:55:22.784 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:55:22.784 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:55:22.785 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:55:22.785 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.785 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.785 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46037.0:170522 11.10 11:0d343739:::benchmark_data_smithi078_157677_object63351:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e192)", 2023-10-01T10:55:22.785 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:54:21.631855+0000", 2023-10-01T10:55:22.785 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 61.097080235, 2023-10-01T10:55:22.785 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1730097100000001, 2023-10-01T10:55:22.785 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:55:22.785 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:55:22.785 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:55:22.785 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46037", 2023-10-01T10:55:22.785 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:55:22.786 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 170522 2023-10-01T10:55:22.786 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.786 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:55:22.786 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.786 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:55:22.786 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631855+0000", 2023-10-01T10:55:22.786 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:55:22.786 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.786 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.786 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:55:22.786 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631855+0000", 2023-10-01T10:55:22.786 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:55:22.787 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.787 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.787 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:55:22.787 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631855+0000", 2023-10-01T10:55:22.787 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999995 2023-10-01T10:55:22.787 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.787 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.787 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:55:22.787 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631857+0000", 2023-10-01T10:55:22.787 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.7689999999999999e-06 2023-10-01T10:55:22.787 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.787 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.787 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:55:22.788 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631861+0000", 2023-10-01T10:55:22.788 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.6969999999999999e-06 2023-10-01T10:55:22.788 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.788 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.788 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:55:22.788 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631864+0000", 2023-10-01T10:55:22.788 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.1020000000000001e-06 2023-10-01T10:55:22.788 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.788 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.788 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:55:22.788 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.632194+0000", 2023-10-01T10:55:22.788 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000330458 2023-10-01T10:55:22.789 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.789 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.789 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T10:55:22.789 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.632205+0000", 2023-10-01T10:55:22.789 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.0445000000000001e-05 2023-10-01T10:55:22.789 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.789 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.789 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:55:22.789 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.777248+0000", 2023-10-01T10:55:22.789 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1450429390000001 2023-10-01T10:55:22.789 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.789 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.790 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:55:22.790 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.777397+0000", 2023-10-01T10:55:22.790 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00014985700000000001 2023-10-01T10:55:22.790 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.790 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.790 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 0,2,3,6,7", 2023-10-01T10:55:22.790 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.777606+0000", 2023-10-01T10:55:22.790 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000208877 2023-10-01T10:55:22.790 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.790 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.790 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.790 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.786339+0000", 2023-10-01T10:55:22.791 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0087329869999999993 2023-10-01T10:55:22.791 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.791 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.791 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.791 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.786441+0000", 2023-10-01T10:55:22.791 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00010124399999999999 2023-10-01T10:55:22.791 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.791 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.791 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.791 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.799857+0000", 2023-10-01T10:55:22.791 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.013416328 2023-10-01T10:55:22.792 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.792 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.792 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:55:22.792 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.800102+0000", 2023-10-01T10:55:22.792 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00024478400000000001 2023-10-01T10:55:22.792 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.792 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.792 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.792 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.800499+0000", 2023-10-01T10:55:22.792 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000397437 2023-10-01T10:55:22.792 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.793 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.793 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.793 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.804664+0000", 2023-10-01T10:55:22.793 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0041646310000000002 2023-10-01T10:55:22.793 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.793 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.793 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:55:22.793 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.804727+0000", 2023-10-01T10:55:22.793 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.3121999999999996e-05 2023-10-01T10:55:22.793 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.793 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.794 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:55:22.794 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.804865+0000", 2023-10-01T10:55:22.794 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00013831499999999999 2023-10-01T10:55:22.794 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:55:22.794 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:55:22.794 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:55:22.794 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.794 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.794 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46037.0:170605 11.10 11:0843b868:::benchmark_data_smithi078_157677_object63434:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e192)", 2023-10-01T10:55:22.794 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:54:21.631874+0000", 2023-10-01T10:55:22.794 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 61.097062203, 2023-10-01T10:55:22.794 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1732039730000001, 2023-10-01T10:55:22.795 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:55:22.795 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:55:22.795 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:55:22.795 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46037", 2023-10-01T10:55:22.795 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:55:22.795 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 170605 2023-10-01T10:55:22.795 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.795 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:55:22.795 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.795 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:55:22.795 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631874+0000", 2023-10-01T10:55:22.795 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:55:22.796 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.796 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.796 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:55:22.796 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631874+0000", 2023-10-01T10:55:22.796 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:55:22.796 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.796 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.796 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:55:22.796 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631873+0000", 2023-10-01T10:55:22.796 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999995 2023-10-01T10:55:22.796 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.796 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.797 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:55:22.797 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631875+0000", 2023-10-01T10:55:22.797 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.7719999999999999e-06 2023-10-01T10:55:22.797 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.797 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.797 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:55:22.797 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631879+0000", 2023-10-01T10:55:22.797 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.9020000000000001e-06 2023-10-01T10:55:22.797 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.797 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.797 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:55:22.798 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631882+0000", 2023-10-01T10:55:22.798 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.3009999999999998e-06 2023-10-01T10:55:22.798 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.798 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.798 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:55:22.798 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.632232+0000", 2023-10-01T10:55:22.798 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00034966799999999997 2023-10-01T10:55:22.798 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.798 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.798 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T10:55:22.798 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.632236+0000", 2023-10-01T10:55:22.798 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.0520000000000003e-06 2023-10-01T10:55:22.799 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.799 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.799 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:55:22.799 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.783536+0000", 2023-10-01T10:55:22.799 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1512997519999999 2023-10-01T10:55:22.799 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.799 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.799 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:55:22.799 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.783777+0000", 2023-10-01T10:55:22.799 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00024158 2023-10-01T10:55:22.799 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.799 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.800 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 0,2,3,6,7", 2023-10-01T10:55:22.800 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.784109+0000", 2023-10-01T10:55:22.800 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000331887 2023-10-01T10:55:22.800 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.800 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.800 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.800 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.788592+0000", 2023-10-01T10:55:22.800 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0044826980000000002 2023-10-01T10:55:22.800 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.800 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.800 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.800 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.788626+0000", 2023-10-01T10:55:22.801 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.3733999999999999e-05 2023-10-01T10:55:22.801 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.801 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.801 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:55:22.801 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.800108+0000", 2023-10-01T10:55:22.801 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.011482137999999999 2023-10-01T10:55:22.801 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.801 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.801 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.801 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.800461+0000", 2023-10-01T10:55:22.801 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00035373000000000002 2023-10-01T10:55:22.802 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.802 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.802 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.802 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.800576+0000", 2023-10-01T10:55:22.802 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00011446399999999999 2023-10-01T10:55:22.802 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.802 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.802 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.802 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.804906+0000", 2023-10-01T10:55:22.802 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0043302549999999999 2023-10-01T10:55:22.802 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.802 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.803 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:55:22.803 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.804951+0000", 2023-10-01T10:55:22.803 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.4369000000000002e-05 2023-10-01T10:55:22.803 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.803 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.803 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:55:22.803 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.805077+0000", 2023-10-01T10:55:22.803 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00012697300000000001 2023-10-01T10:55:22.803 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:55:22.803 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:55:22.803 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:55:22.804 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.804 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.804 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46037.0:170611 11.10 11:0fb7c3db:::benchmark_data_smithi078_157677_object63440:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e192)", 2023-10-01T10:55:22.804 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:54:21.631885+0000", 2023-10-01T10:55:22.804 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 61.097050961000001, 2023-10-01T10:55:22.804 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.17341871, 2023-10-01T10:55:22.804 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:55:22.804 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:55:22.804 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:55:22.804 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46037", 2023-10-01T10:55:22.804 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:55:22.804 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 170611 2023-10-01T10:55:22.805 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.805 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:55:22.805 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.805 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:55:22.805 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631885+0000", 2023-10-01T10:55:22.805 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:55:22.805 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.805 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.805 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:55:22.805 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631885+0000", 2023-10-01T10:55:22.805 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:55:22.805 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.806 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.806 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:55:22.806 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631885+0000", 2023-10-01T10:55:22.806 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967296 2023-10-01T10:55:22.806 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.806 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.806 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:55:22.806 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631886+0000", 2023-10-01T10:55:22.806 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.6899999999999999e-06 2023-10-01T10:55:22.806 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.806 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.806 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:55:22.807 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631889+0000", 2023-10-01T10:55:22.807 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.2040000000000002e-06 2023-10-01T10:55:22.807 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.807 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.807 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:55:22.807 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631892+0000", 2023-10-01T10:55:22.807 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.9280000000000002e-06 2023-10-01T10:55:22.807 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.807 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.807 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:55:22.807 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.632264+0000", 2023-10-01T10:55:22.807 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00037162000000000002 2023-10-01T10:55:22.807 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.808 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.808 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T10:55:22.808 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.632273+0000", 2023-10-01T10:55:22.808 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.4080000000000004e-06 2023-10-01T10:55:22.808 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.808 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.808 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:55:22.808 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.786463+0000", 2023-10-01T10:55:22.808 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1541899019999999 2023-10-01T10:55:22.808 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.808 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.808 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:55:22.809 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.786581+0000", 2023-10-01T10:55:22.809 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000117694 2023-10-01T10:55:22.809 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.809 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.809 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 0,2,3,6,7", 2023-10-01T10:55:22.809 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.786866+0000", 2023-10-01T10:55:22.809 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000285376 2023-10-01T10:55:22.809 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.809 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.809 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.809 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.799766+0000", 2023-10-01T10:55:22.810 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.012899476999999999 2023-10-01T10:55:22.810 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.810 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.810 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:55:22.810 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.800114+0000", 2023-10-01T10:55:22.810 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00034815600000000002 2023-10-01T10:55:22.810 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.810 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.810 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.810 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.800539+0000", 2023-10-01T10:55:22.810 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00042501399999999998 2023-10-01T10:55:22.811 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.811 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.811 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.811 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.800612+0000", 2023-10-01T10:55:22.811 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.2849999999999995e-05 2023-10-01T10:55:22.811 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.811 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.811 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.811 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.800691+0000", 2023-10-01T10:55:22.811 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.9516000000000002e-05 2023-10-01T10:55:22.811 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.811 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.812 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.812 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.805116+0000", 2023-10-01T10:55:22.812 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0044245389999999999 2023-10-01T10:55:22.812 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.812 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.812 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:55:22.812 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.805158+0000", 2023-10-01T10:55:22.812 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.2015999999999999e-05 2023-10-01T10:55:22.812 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.812 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.812 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:55:22.813 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.805303+0000", 2023-10-01T10:55:22.813 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00014555300000000001 2023-10-01T10:55:22.813 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:55:22.813 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:55:22.813 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:55:22.813 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.813 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.813 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46037.0:176163 11.13 11:c8061bf6:::benchmark_data_smithi078_157677_object68992:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e199)", 2023-10-01T10:55:22.813 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:54:33.553395+0000", 2023-10-01T10:55:22.813 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 49.175540550999997, 2023-10-01T10:55:22.813 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.531561218, 2023-10-01T10:55:22.813 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:55:22.814 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:55:22.814 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:55:22.814 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46037", 2023-10-01T10:55:22.814 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:55:22.814 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 176163 2023-10-01T10:55:22.814 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.814 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:55:22.814 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.814 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:55:22.814 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553395+0000", 2023-10-01T10:55:22.814 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:55:22.814 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.815 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.815 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:55:22.815 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553395+0000", 2023-10-01T10:55:22.815 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:55:22.815 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.815 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.815 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:55:22.815 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553393+0000", 2023-10-01T10:55:22.815 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999976 2023-10-01T10:55:22.815 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.815 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.815 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:55:22.816 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553403+0000", 2023-10-01T10:55:22.816 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.0597999999999999e-05 2023-10-01T10:55:22.816 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.816 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.816 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:55:22.816 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553421+0000", 2023-10-01T10:55:22.816 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.7847999999999999e-05 2023-10-01T10:55:22.816 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.816 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.816 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:55:22.816 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553434+0000", 2023-10-01T10:55:22.816 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.2746e-05 2023-10-01T10:55:22.817 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.817 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.817 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:55:22.817 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.556358+0000", 2023-10-01T10:55:22.817 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0029241969999999999 2023-10-01T10:55:22.817 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.817 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.817 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T10:55:22.817 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.556366+0000", 2023-10-01T10:55:22.817 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.0260000000000007e-06 2023-10-01T10:55:22.817 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.817 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.818 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:55:22.818 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.070523+0000", 2023-10-01T10:55:22.818 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.514156469 2023-10-01T10:55:22.818 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.818 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.818 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:55:22.818 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.070672+0000", 2023-10-01T10:55:22.818 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000149157 2023-10-01T10:55:22.818 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.818 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.818 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 0,4,5,6,7", 2023-10-01T10:55:22.819 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.071071+0000", 2023-10-01T10:55:22.819 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00039965299999999998 2023-10-01T10:55:22.819 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.819 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.819 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:55:22.819 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.075827+0000", 2023-10-01T10:55:22.819 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0047560390000000001 2023-10-01T10:55:22.819 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.819 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.819 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.819 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.078651+0000", 2023-10-01T10:55:22.819 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0028232510000000001 2023-10-01T10:55:22.820 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.820 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.820 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.820 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.078688+0000", 2023-10-01T10:55:22.820 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.7246000000000002e-05 2023-10-01T10:55:22.820 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.820 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.820 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.820 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.078755+0000", 2023-10-01T10:55:22.820 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.7024000000000001e-05 2023-10-01T10:55:22.820 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.820 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.821 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.821 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.078787+0000", 2023-10-01T10:55:22.821 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.1642e-05 2023-10-01T10:55:22.821 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.821 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.821 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.821 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.084731+0000", 2023-10-01T10:55:22.821 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0059444759999999998 2023-10-01T10:55:22.821 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.821 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.821 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:55:22.822 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.084799+0000", 2023-10-01T10:55:22.822 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.7883e-05 2023-10-01T10:55:22.822 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.822 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.822 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:55:22.822 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.084956+0000", 2023-10-01T10:55:22.822 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000157472 2023-10-01T10:55:22.822 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:55:22.822 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:55:22.822 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:55:22.822 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.822 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.823 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46037.0:176170 11.13 11:cc11c237:::benchmark_data_smithi078_157677_object68999:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e199)", 2023-10-01T10:55:22.823 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:54:33.553478+0000", 2023-10-01T10:55:22.823 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 49.175457684999998, 2023-10-01T10:55:22.823 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.531709996, 2023-10-01T10:55:22.823 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:55:22.823 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:55:22.823 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:55:22.823 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46037", 2023-10-01T10:55:22.823 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:55:22.823 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 176170 2023-10-01T10:55:22.823 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.824 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:55:22.824 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.824 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:55:22.824 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553478+0000", 2023-10-01T10:55:22.824 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:55:22.824 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.824 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.824 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:55:22.824 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553478+0000", 2023-10-01T10:55:22.824 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:55:22.824 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.825 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.825 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:55:22.825 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553477+0000", 2023-10-01T10:55:22.825 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999995 2023-10-01T10:55:22.825 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.825 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.825 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:55:22.825 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553481+0000", 2023-10-01T10:55:22.825 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.9639999999999997e-06 2023-10-01T10:55:22.825 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.825 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.825 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:55:22.826 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553490+0000", 2023-10-01T10:55:22.826 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.6149999999999997e-06 2023-10-01T10:55:22.826 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.826 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.826 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:55:22.826 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553495+0000", 2023-10-01T10:55:22.826 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.113e-06 2023-10-01T10:55:22.826 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.826 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.826 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:55:22.827 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.556534+0000", 2023-10-01T10:55:22.827 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.003038443 2023-10-01T10:55:22.827 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.827 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.827 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T10:55:22.827 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.556541+0000", 2023-10-01T10:55:22.827 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.7039999999999994e-06 2023-10-01T10:55:22.827 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.827 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.827 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:55:22.827 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.073377+0000", 2023-10-01T10:55:22.827 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.5168358629999998 2023-10-01T10:55:22.827 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.828 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.828 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:55:22.828 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.073520+0000", 2023-10-01T10:55:22.828 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00014291400000000001 2023-10-01T10:55:22.828 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.828 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.828 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 0,4,5,6,7", 2023-10-01T10:55:22.828 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.073795+0000", 2023-10-01T10:55:22.828 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00027470199999999999 2023-10-01T10:55:22.828 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.828 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.829 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.829 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.078722+0000", 2023-10-01T10:55:22.829 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0049276019999999997 2023-10-01T10:55:22.829 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.829 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.829 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:55:22.829 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.084525+0000", 2023-10-01T10:55:22.829 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0058027829999999997 2023-10-01T10:55:22.829 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.829 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.829 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.829 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.084569+0000", 2023-10-01T10:55:22.830 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.3498000000000003e-05 2023-10-01T10:55:22.830 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.830 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.830 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.830 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.084644+0000", 2023-10-01T10:55:22.830 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.5512999999999993e-05 2023-10-01T10:55:22.830 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.830 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.830 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.830 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.084684+0000", 2023-10-01T10:55:22.830 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.0086000000000002e-05 2023-10-01T10:55:22.831 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.831 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.831 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.831 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.084998+0000", 2023-10-01T10:55:22.831 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000314235 2023-10-01T10:55:22.831 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.831 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.831 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:55:22.831 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.085050+0000", 2023-10-01T10:55:22.831 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.1100999999999997e-05 2023-10-01T10:55:22.831 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.831 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.832 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:55:22.832 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.085188+0000", 2023-10-01T10:55:22.832 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00013848300000000001 2023-10-01T10:55:22.832 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:55:22.832 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:55:22.832 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:55:22.832 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.832 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.832 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46037.0:176346 11.13 11:c9e6b427:::benchmark_data_smithi078_157677_object69175:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e199)", 2023-10-01T10:55:22.832 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:54:33.553500+0000", 2023-10-01T10:55:22.832 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 49.175436054000002, 2023-10-01T10:55:22.832 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.532346797, 2023-10-01T10:55:22.833 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:55:22.833 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:55:22.833 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:55:22.833 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46037", 2023-10-01T10:55:22.833 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:55:22.833 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 176346 2023-10-01T10:55:22.833 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.833 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:55:22.833 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.833 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:55:22.833 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553500+0000", 2023-10-01T10:55:22.833 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:55:22.833 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.834 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.834 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:55:22.834 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553500+0000", 2023-10-01T10:55:22.834 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:55:22.834 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.834 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.834 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:55:22.834 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553499+0000", 2023-10-01T10:55:22.834 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999995 2023-10-01T10:55:22.834 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.834 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.835 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:55:22.835 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553501+0000", 2023-10-01T10:55:22.835 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.1160000000000002e-06 2023-10-01T10:55:22.835 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.835 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.835 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:55:22.835 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553506+0000", 2023-10-01T10:55:22.835 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.3449999999999999e-06 2023-10-01T10:55:22.835 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.835 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.835 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:55:22.836 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553510+0000", 2023-10-01T10:55:22.836 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.9929999999999997e-06 2023-10-01T10:55:22.836 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.836 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.836 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:55:22.836 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.556567+0000", 2023-10-01T10:55:22.836 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0030572609999999999 2023-10-01T10:55:22.836 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.836 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.836 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T10:55:22.836 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.556575+0000", 2023-10-01T10:55:22.836 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.4390000000000003e-06 2023-10-01T10:55:22.836 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.837 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.837 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:55:22.837 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.075881+0000", 2023-10-01T10:55:22.837 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.5193060630000002 2023-10-01T10:55:22.837 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.837 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.837 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:55:22.837 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.076054+0000", 2023-10-01T10:55:22.837 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00017294500000000001 2023-10-01T10:55:22.837 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.837 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.837 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 0,4,5,6,7", 2023-10-01T10:55:22.838 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.076458+0000", 2023-10-01T10:55:22.838 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000404134 2023-10-01T10:55:22.838 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.838 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.838 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.838 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.084504+0000", 2023-10-01T10:55:22.838 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0080465829999999995 2023-10-01T10:55:22.838 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.838 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.838 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:55:22.838 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.084535+0000", 2023-10-01T10:55:22.838 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.0889999999999997e-05 2023-10-01T10:55:22.839 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.839 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.839 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.839 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.085233+0000", 2023-10-01T10:55:22.839 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00069764899999999995 2023-10-01T10:55:22.839 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.839 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.839 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.839 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.085431+0000", 2023-10-01T10:55:22.839 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00019848500000000001 2023-10-01T10:55:22.839 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.839 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.840 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.840 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.085519+0000", 2023-10-01T10:55:22.840 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.7348999999999993e-05 2023-10-01T10:55:22.840 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.840 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.840 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.840 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.085655+0000", 2023-10-01T10:55:22.840 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000135986 2023-10-01T10:55:22.840 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.840 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.840 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:55:22.840 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.085699+0000", 2023-10-01T10:55:22.841 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.4536999999999997e-05 2023-10-01T10:55:22.841 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.841 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.841 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:55:22.841 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.085846+0000", 2023-10-01T10:55:22.841 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000147331 2023-10-01T10:55:22.841 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:55:22.841 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:55:22.841 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:55:22.841 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.841 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.841 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46037.0:176347 11.13 11:cfa0b275:::benchmark_data_smithi078_157677_object69176:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e199)", 2023-10-01T10:55:22.841 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:54:33.553513+0000", 2023-10-01T10:55:22.842 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 49.175422816000001, 2023-10-01T10:55:22.842 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.53910002, 2023-10-01T10:55:22.842 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:55:22.842 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:55:22.842 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:55:22.842 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46037", 2023-10-01T10:55:22.842 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:55:22.842 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 176347 2023-10-01T10:55:22.842 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.842 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:55:22.842 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.843 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:55:22.843 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553513+0000", 2023-10-01T10:55:22.843 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:55:22.843 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.843 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.843 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:55:22.843 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553513+0000", 2023-10-01T10:55:22.843 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:55:22.843 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.843 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.843 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:55:22.843 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553513+0000", 2023-10-01T10:55:22.844 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999995 2023-10-01T10:55:22.844 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.844 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.844 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:55:22.844 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553514+0000", 2023-10-01T10:55:22.844 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.818e-06 2023-10-01T10:55:22.844 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.844 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.844 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:55:22.844 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553518+0000", 2023-10-01T10:55:22.844 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.7799999999999998e-06 2023-10-01T10:55:22.844 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.845 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.845 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:55:22.845 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553523+0000", 2023-10-01T10:55:22.845 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.5129999999999998e-06 2023-10-01T10:55:22.845 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.845 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.845 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:55:22.845 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.556600+0000", 2023-10-01T10:55:22.845 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0030775009999999998 2023-10-01T10:55:22.845 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.845 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.845 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T10:55:22.846 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.556607+0000", 2023-10-01T10:55:22.846 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.934e-06 2023-10-01T10:55:22.846 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.846 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.846 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:55:22.846 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.079062+0000", 2023-10-01T10:55:22.846 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.522454889 2023-10-01T10:55:22.846 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.846 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.846 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:55:22.846 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.079267+0000", 2023-10-01T10:55:22.847 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000205365 2023-10-01T10:55:22.847 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.847 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.847 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 0,4,5,6,7", 2023-10-01T10:55:22.847 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.079684+0000", 2023-10-01T10:55:22.847 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00041616599999999999 2023-10-01T10:55:22.847 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.847 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.847 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:55:22.847 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.084582+0000", 2023-10-01T10:55:22.847 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0048984450000000004 2023-10-01T10:55:22.847 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.847 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.848 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.848 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.085484+0000", 2023-10-01T10:55:22.848 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00090182999999999995 2023-10-01T10:55:22.848 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.848 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.848 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.848 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.085591+0000", 2023-10-01T10:55:22.848 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000107333 2023-10-01T10:55:22.848 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.848 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.848 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.848 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.085624+0000", 2023-10-01T10:55:22.849 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.2836000000000002e-05 2023-10-01T10:55:22.849 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.849 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.849 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.849 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.088522+0000", 2023-10-01T10:55:22.849 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0028975440000000002 2023-10-01T10:55:22.849 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.849 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.849 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.849 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.092422+0000", 2023-10-01T10:55:22.849 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0039003570000000001 2023-10-01T10:55:22.849 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.850 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.850 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:55:22.850 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.092476+0000", 2023-10-01T10:55:22.850 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.3724000000000003e-05 2023-10-01T10:55:22.850 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.850 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.850 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:55:22.850 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.092613+0000", 2023-10-01T10:55:22.850 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000137253 2023-10-01T10:55:22.850 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:55:22.850 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:55:22.851 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:55:22.851 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.851 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.851 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46037.0:176390 11.13 11:cf928ba3:::benchmark_data_smithi078_157677_object69219:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e199)", 2023-10-01T10:55:22.851 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:54:33.553526+0000", 2023-10-01T10:55:22.851 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 49.175409930000001, 2023-10-01T10:55:22.851 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.539296327, 2023-10-01T10:55:22.851 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:55:22.851 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:55:22.851 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:55:22.851 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46037", 2023-10-01T10:55:22.851 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:55:22.852 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 176390 2023-10-01T10:55:22.852 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.852 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:55:22.852 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.852 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:55:22.852 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553526+0000", 2023-10-01T10:55:22.852 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:55:22.852 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.852 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.852 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:55:22.852 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553526+0000", 2023-10-01T10:55:22.852 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:55:22.853 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.853 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.853 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:55:22.853 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553526+0000", 2023-10-01T10:55:22.853 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999995 2023-10-01T10:55:22.853 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.853 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.853 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:55:22.853 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553528+0000", 2023-10-01T10:55:22.853 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.3649999999999998e-06 2023-10-01T10:55:22.853 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.853 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.854 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:55:22.854 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553534+0000", 2023-10-01T10:55:22.854 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.387e-06 2023-10-01T10:55:22.854 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.854 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.854 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:55:22.854 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553540+0000", 2023-10-01T10:55:22.854 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.3750000000000002e-06 2023-10-01T10:55:22.854 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.854 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.854 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:55:22.855 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.556634+0000", 2023-10-01T10:55:22.855 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0030939000000000001 2023-10-01T10:55:22.855 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.855 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.855 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T10:55:22.855 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.556641+0000", 2023-10-01T10:55:22.855 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.25e-06 2023-10-01T10:55:22.855 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.855 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.855 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:55:22.855 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.082071+0000", 2023-10-01T10:55:22.856 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.5254299260000002 2023-10-01T10:55:22.856 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.856 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.856 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:55:22.856 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.082238+0000", 2023-10-01T10:55:22.856 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000167658 2023-10-01T10:55:22.856 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.856 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.856 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 0,4,5,6,7", 2023-10-01T10:55:22.856 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.082604+0000", 2023-10-01T10:55:22.856 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000365149 2023-10-01T10:55:22.856 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.856 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.857 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.857 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.085887+0000", 2023-10-01T10:55:22.857 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0032833889999999998 2023-10-01T10:55:22.857 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.857 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.857 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.857 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.085921+0000", 2023-10-01T10:55:22.857 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.4317999999999999e-05 2023-10-01T10:55:22.857 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.857 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.857 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.857 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.088488+0000", 2023-10-01T10:55:22.857 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0025663690000000002 2023-10-01T10:55:22.858 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.858 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.858 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.858 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.088553+0000", 2023-10-01T10:55:22.858 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.5677000000000002e-05 2023-10-01T10:55:22.858 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.858 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.858 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:55:22.858 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.088562+0000", 2023-10-01T10:55:22.858 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.303e-06 2023-10-01T10:55:22.858 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.859 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.859 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.859 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.092655+0000", 2023-10-01T10:55:22.859 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0040938800000000003 2023-10-01T10:55:22.859 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.859 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.859 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:55:22.859 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.092697+0000", 2023-10-01T10:55:22.859 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.1888000000000003e-05 2023-10-01T10:55:22.859 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.859 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.859 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:55:22.860 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.092822+0000", 2023-10-01T10:55:22.860 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000124759 2023-10-01T10:55:22.860 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:55:22.860 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:55:22.860 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:55:22.860 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.860 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.860 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46037.0:176477 11.13 11:ceea8d8a:::benchmark_data_smithi078_157677_object69306:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e199)", 2023-10-01T10:55:22.860 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:54:33.553546+0000", 2023-10-01T10:55:22.860 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 49.175390049999997, 2023-10-01T10:55:22.860 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.578554842, 2023-10-01T10:55:22.860 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:55:22.861 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:55:22.861 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:55:22.861 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46037", 2023-10-01T10:55:22.861 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:55:22.861 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 176477 2023-10-01T10:55:22.861 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.861 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:55:22.861 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.861 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:55:22.861 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553546+0000", 2023-10-01T10:55:22.861 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:55:22.861 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.862 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.862 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:55:22.862 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553546+0000", 2023-10-01T10:55:22.862 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:55:22.862 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.862 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.862 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:55:22.862 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553545+0000", 2023-10-01T10:55:22.862 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999995 2023-10-01T10:55:22.862 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.862 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.863 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:55:22.863 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553548+0000", 2023-10-01T10:55:22.863 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.005e-06 2023-10-01T10:55:22.863 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.863 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.863 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:55:22.863 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553555+0000", 2023-10-01T10:55:22.863 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.4099999999999996e-06 2023-10-01T10:55:22.863 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.864 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.864 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:55:22.864 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553560+0000", 2023-10-01T10:55:22.864 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.1370000000000001e-06 2023-10-01T10:55:22.864 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.864 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.864 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:55:22.864 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.556662+0000", 2023-10-01T10:55:22.864 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00310247 2023-10-01T10:55:22.864 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.864 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.864 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T10:55:22.865 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.556670+0000", 2023-10-01T10:55:22.865 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.7209999999999994e-06 2023-10-01T10:55:22.865 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.865 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.865 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:55:22.865 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.086002+0000", 2023-10-01T10:55:22.865 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.529332304 2023-10-01T10:55:22.865 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.865 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.865 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:55:22.865 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.086139+0000", 2023-10-01T10:55:22.865 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00013701499999999999 2023-10-01T10:55:22.866 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.866 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.866 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 0,4,5,6,7", 2023-10-01T10:55:22.866 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.086559+0000", 2023-10-01T10:55:22.866 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00042014800000000002 2023-10-01T10:55:22.866 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.866 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.866 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.866 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.092253+0000", 2023-10-01T10:55:22.866 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0056931600000000001 2023-10-01T10:55:22.866 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.867 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.867 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.867 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.092289+0000", 2023-10-01T10:55:22.867 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.6709999999999999e-05 2023-10-01T10:55:22.867 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.867 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.867 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.867 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.092388+0000", 2023-10-01T10:55:22.867 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.9223999999999997e-05 2023-10-01T10:55:22.867 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.867 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.867 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:55:22.868 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.097292+0000", 2023-10-01T10:55:22.868 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0049035099999999998 2023-10-01T10:55:22.868 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.868 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.868 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.868 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.102683+0000", 2023-10-01T10:55:22.868 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0053909609999999997 2023-10-01T10:55:22.868 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.868 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.868 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.868 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.131932+0000", 2023-10-01T10:55:22.868 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.029249139 2023-10-01T10:55:22.869 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.869 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.869 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:55:22.869 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.131987+0000", 2023-10-01T10:55:22.869 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.5053999999999998e-05 2023-10-01T10:55:22.869 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.869 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.869 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:55:22.869 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.132101+0000", 2023-10-01T10:55:22.869 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000113341 2023-10-01T10:55:22.869 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:55:22.869 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:55:22.870 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:55:22.870 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.870 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.870 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46037.0:176533 11.13 11:cf452de8:::benchmark_data_smithi078_157677_object69362:head [delete] snapc 0=[] ondisk+write+known_if_redirected e199)", 2023-10-01T10:55:22.870 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:54:33.686751+0000", 2023-10-01T10:55:22.870 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 49.042184943999999, 2023-10-01T10:55:22.870 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.44577048, 2023-10-01T10:55:22.870 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:55:22.870 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:55:22.870 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:55:22.870 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46037", 2023-10-01T10:55:22.870 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:55:22.871 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 176533 2023-10-01T10:55:22.871 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.871 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:55:22.871 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.871 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:55:22.871 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.686751+0000", 2023-10-01T10:55:22.871 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:55:22.871 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.871 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.871 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:55:22.871 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.686751+0000", 2023-10-01T10:55:22.871 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:55:22.872 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.872 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.872 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:55:22.872 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.686749+0000", 2023-10-01T10:55:22.872 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999976 2023-10-01T10:55:22.872 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.872 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.872 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:55:22.872 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.686759+0000", 2023-10-01T10:55:22.872 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.0138e-05 2023-10-01T10:55:22.872 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.872 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.873 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:55:22.873 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.686779+0000", 2023-10-01T10:55:22.873 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.0115e-05 2023-10-01T10:55:22.873 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.873 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.873 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:55:22.873 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.686790+0000", 2023-10-01T10:55:22.873 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1034e-05 2023-10-01T10:55:22.873 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.873 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.873 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:55:22.873 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.686879+0000", 2023-10-01T10:55:22.873 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.8918999999999993e-05 2023-10-01T10:55:22.874 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.874 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.874 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T10:55:22.874 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.686892+0000", 2023-10-01T10:55:22.874 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.3482000000000001e-05 2023-10-01T10:55:22.874 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.874 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.874 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:55:22.874 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.088585+0000", 2023-10-01T10:55:22.874 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.4016922489999999 2023-10-01T10:55:22.874 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.875 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.875 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:55:22.875 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.088729+0000", 2023-10-01T10:55:22.875 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00014457599999999999 2023-10-01T10:55:22.875 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.875 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.875 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 0,4,5,6,7", 2023-10-01T10:55:22.875 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.089056+0000", 2023-10-01T10:55:22.875 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00032674499999999998 2023-10-01T10:55:22.875 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.875 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.876 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.876 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.092323+0000", 2023-10-01T10:55:22.876 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0032670630000000002 2023-10-01T10:55:22.876 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.876 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.876 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.876 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.092356+0000", 2023-10-01T10:55:22.876 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.3086000000000001e-05 2023-10-01T10:55:22.876 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.876 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.876 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.876 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.097172+0000", 2023-10-01T10:55:22.876 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0048162869999999998 2023-10-01T10:55:22.877 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.877 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.877 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:55:22.877 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.097299+0000", 2023-10-01T10:55:22.877 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00012659999999999999 2023-10-01T10:55:22.877 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.877 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.877 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.877 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.102732+0000", 2023-10-01T10:55:22.877 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0054326579999999999 2023-10-01T10:55:22.877 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.877 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.878 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.878 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.132372+0000", 2023-10-01T10:55:22.878 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.029640307000000001 2023-10-01T10:55:22.878 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.878 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.878 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:55:22.878 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.132418+0000", 2023-10-01T10:55:22.878 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.5809000000000001e-05 2023-10-01T10:55:22.878 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.878 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.878 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:55:22.878 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.132521+0000", 2023-10-01T10:55:22.879 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000103566 2023-10-01T10:55:22.879 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:55:22.879 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:55:22.879 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:55:22.879 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.879 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.879 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46037.0:176550 11.13 11:cf453b8c:::benchmark_data_smithi078_157677_object69379:head [delete] snapc 0=[] ondisk+write+known_if_redirected e199)", 2023-10-01T10:55:22.879 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:54:33.716087+0000", 2023-10-01T10:55:22.879 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 49.012848525999999, 2023-10-01T10:55:22.879 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.416674075, 2023-10-01T10:55:22.879 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:55:22.879 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:55:22.880 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:55:22.880 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46037", 2023-10-01T10:55:22.880 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:55:22.880 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 176550 2023-10-01T10:55:22.880 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.880 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:55:22.880 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.880 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:55:22.880 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.716087+0000", 2023-10-01T10:55:22.880 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:55:22.880 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.880 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.881 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:55:22.881 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.716087+0000", 2023-10-01T10:55:22.881 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:55:22.881 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.881 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.881 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:55:22.881 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.716085+0000", 2023-10-01T10:55:22.881 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999981 2023-10-01T10:55:22.881 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.881 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.881 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:55:22.882 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.716093+0000", 2023-10-01T10:55:22.882 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.4229999999999999e-06 2023-10-01T10:55:22.882 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.882 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.882 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:55:22.882 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.716110+0000", 2023-10-01T10:55:22.882 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.7414999999999998e-05 2023-10-01T10:55:22.882 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.882 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.882 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:55:22.882 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.716119+0000", 2023-10-01T10:55:22.882 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.5010000000000004e-06 2023-10-01T10:55:22.883 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.883 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.883 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:55:22.883 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.716309+0000", 2023-10-01T10:55:22.883 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00018996200000000001 2023-10-01T10:55:22.883 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.883 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.883 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T10:55:22.883 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.716320+0000", 2023-10-01T10:55:22.883 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1402e-05 2023-10-01T10:55:22.883 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.883 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.884 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:55:22.884 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.092851+0000", 2023-10-01T10:55:22.884 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.376530918 2023-10-01T10:55:22.884 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.884 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.884 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:55:22.884 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.093167+0000", 2023-10-01T10:55:22.884 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00031600500000000001 2023-10-01T10:55:22.884 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.884 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.884 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 0,4,5,6,7", 2023-10-01T10:55:22.884 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.093531+0000", 2023-10-01T10:55:22.885 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000364003 2023-10-01T10:55:22.885 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.885 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.885 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.885 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.097278+0000", 2023-10-01T10:55:22.885 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0037469980000000001 2023-10-01T10:55:22.885 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.885 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.885 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:55:22.885 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.097304+0000", 2023-10-01T10:55:22.885 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.6342e-05 2023-10-01T10:55:22.885 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.886 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.886 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.886 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.097345+0000", 2023-10-01T10:55:22.886 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.0494000000000002e-05 2023-10-01T10:55:22.886 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.886 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.886 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.886 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.102316+0000", 2023-10-01T10:55:22.886 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0049712569999999998 2023-10-01T10:55:22.886 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.886 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.886 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.887 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.132145+0000", 2023-10-01T10:55:22.887 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.029828397 2023-10-01T10:55:22.887 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.887 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.887 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.887 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.132591+0000", 2023-10-01T10:55:22.887 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00044686900000000002 2023-10-01T10:55:22.887 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.887 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.887 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:55:22.887 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.132637+0000", 2023-10-01T10:55:22.887 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.5144e-05 2023-10-01T10:55:22.888 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.888 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.888 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:55:22.888 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.132761+0000", 2023-10-01T10:55:22.888 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00012464600000000001 2023-10-01T10:55:22.888 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:55:22.888 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:55:22.888 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:55:22.888 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.888 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.888 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46037.0:176554 11.13 11:cc4f0cc6:::benchmark_data_smithi078_157677_object69383:head [delete] snapc 0=[] ondisk+write+known_if_redirected e199)", 2023-10-01T10:55:22.888 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:54:33.723328+0000", 2023-10-01T10:55:22.889 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 49.005607781999998, 2023-10-01T10:55:22.889 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.4096755830000001, 2023-10-01T10:55:22.889 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:55:22.889 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:55:22.889 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:55:22.889 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46037", 2023-10-01T10:55:22.889 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:55:22.889 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 176554 2023-10-01T10:55:22.889 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.889 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:55:22.889 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.889 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:55:22.889 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.723328+0000", 2023-10-01T10:55:22.890 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:55:22.890 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.890 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.890 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:55:22.890 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.723328+0000", 2023-10-01T10:55:22.890 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:55:22.890 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.890 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.890 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:55:22.890 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.723326+0000", 2023-10-01T10:55:22.890 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999981 2023-10-01T10:55:22.891 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.891 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.891 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:55:22.891 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.723333+0000", 2023-10-01T10:55:22.891 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.4259999999999997e-06 2023-10-01T10:55:22.891 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.891 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.891 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:55:22.891 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.723346+0000", 2023-10-01T10:55:22.891 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.2442e-05 2023-10-01T10:55:22.891 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.891 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.892 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:55:22.892 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.723355+0000", 2023-10-01T10:55:22.892 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.363e-06 2023-10-01T10:55:22.892 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.892 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.892 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:55:22.892 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.723986+0000", 2023-10-01T10:55:22.892 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00063095700000000002 2023-10-01T10:55:22.892 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.892 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.892 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T10:55:22.892 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.723999+0000", 2023-10-01T10:55:22.893 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.3088000000000001e-05 2023-10-01T10:55:22.893 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.893 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.893 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:55:22.893 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.095277+0000", 2023-10-01T10:55:22.893 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.371277962 2023-10-01T10:55:22.893 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.893 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.893 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:55:22.893 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.095420+0000", 2023-10-01T10:55:22.893 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000142831 2023-10-01T10:55:22.893 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.894 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.894 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 0,4,5,6,7", 2023-10-01T10:55:22.894 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.095688+0000", 2023-10-01T10:55:22.894 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00026794600000000003 2023-10-01T10:55:22.894 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.894 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.894 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.894 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.102256+0000", 2023-10-01T10:55:22.894 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0065676709999999998 2023-10-01T10:55:22.894 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.894 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.894 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:55:22.895 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.102268+0000", 2023-10-01T10:55:22.895 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1863e-05 2023-10-01T10:55:22.895 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.895 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.895 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.895 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.102373+0000", 2023-10-01T10:55:22.895 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000105851 2023-10-01T10:55:22.895 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.895 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.895 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.895 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.102416+0000", 2023-10-01T10:55:22.896 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.2734000000000001e-05 2023-10-01T10:55:22.896 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.896 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.896 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.896 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.132285+0000", 2023-10-01T10:55:22.896 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.029868551 2023-10-01T10:55:22.896 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.896 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.896 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.896 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.132808+0000", 2023-10-01T10:55:22.896 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00052275499999999996 2023-10-01T10:55:22.896 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.897 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.897 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:55:22.897 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.132841+0000", 2023-10-01T10:55:22.897 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.3908000000000001e-05 2023-10-01T10:55:22.897 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.897 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.897 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:55:22.897 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.133004+0000", 2023-10-01T10:55:22.897 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00016209699999999999 2023-10-01T10:55:22.897 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:55:22.897 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:55:22.897 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:55:22.897 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.898 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.898 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46037.0:176584 11.13 11:cf1aac0a:::benchmark_data_smithi078_157677_object69413:head [delete] snapc 0=[] ondisk+write+known_if_redirected e199)", 2023-10-01T10:55:22.898 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:54:33.769745+0000", 2023-10-01T10:55:22.898 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 48.959190782, 2023-10-01T10:55:22.898 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.3634320180000001, 2023-10-01T10:55:22.898 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:55:22.898 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:55:22.898 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:55:22.898 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46037", 2023-10-01T10:55:22.898 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:55:22.899 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 176584 2023-10-01T10:55:22.899 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.899 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:55:22.899 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.899 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:55:22.899 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.769745+0000", 2023-10-01T10:55:22.899 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:55:22.899 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.899 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.899 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:55:22.899 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.769745+0000", 2023-10-01T10:55:22.899 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:55:22.900 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.900 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.900 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:55:22.900 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.769743+0000", 2023-10-01T10:55:22.900 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999981 2023-10-01T10:55:22.900 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.900 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.900 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:55:22.900 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.769751+0000", 2023-10-01T10:55:22.900 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.5340000000000002e-06 2023-10-01T10:55:22.900 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.900 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.900 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:55:22.901 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.769766+0000", 2023-10-01T10:55:22.901 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.5347000000000001e-05 2023-10-01T10:55:22.901 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.901 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.901 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:55:22.901 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.769774+0000", 2023-10-01T10:55:22.901 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.4700000000000002e-06 2023-10-01T10:55:22.901 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.901 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.901 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:55:22.901 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.769825+0000", 2023-10-01T10:55:22.901 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.0061000000000001e-05 2023-10-01T10:55:22.902 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.902 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.902 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T10:55:22.902 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.769835+0000", 2023-10-01T10:55:22.902 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.0808000000000001e-05 2023-10-01T10:55:22.902 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.902 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.902 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:55:22.902 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.097372+0000", 2023-10-01T10:55:22.902 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.3275364459999999 2023-10-01T10:55:22.902 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.903 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.903 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:55:22.903 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.097519+0000", 2023-10-01T10:55:22.903 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000147348 2023-10-01T10:55:22.903 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.903 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.903 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 0,4,5,6,7", 2023-10-01T10:55:22.903 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.097838+0000", 2023-10-01T10:55:22.903 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00031927000000000002 2023-10-01T10:55:22.903 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.903 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.903 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:55:22.904 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.102276+0000", 2023-10-01T10:55:22.904 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0044371760000000001 2023-10-01T10:55:22.904 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.904 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.904 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.904 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.102779+0000", 2023-10-01T10:55:22.904 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00050379400000000001 2023-10-01T10:55:22.904 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.904 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.904 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.904 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.102822+0000", 2023-10-01T10:55:22.904 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.2648999999999998e-05 2023-10-01T10:55:22.905 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.905 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.905 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.905 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.104932+0000", 2023-10-01T10:55:22.905 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0021096740000000002 2023-10-01T10:55:22.905 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.905 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.905 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.905 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.132316+0000", 2023-10-01T10:55:22.905 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.027384526999999999 2023-10-01T10:55:22.905 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.905 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.906 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.906 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.133041+0000", 2023-10-01T10:55:22.906 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00072457499999999998 2023-10-01T10:55:22.906 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.906 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.906 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:55:22.906 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.133081+0000", 2023-10-01T10:55:22.906 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.9787999999999999e-05 2023-10-01T10:55:22.906 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.906 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.906 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:55:22.907 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.133177+0000", 2023-10-01T10:55:22.907 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.6349999999999997e-05 2023-10-01T10:55:22.907 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:55:22.907 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:55:22.907 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:55:22.907 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.907 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.907 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46037.0:176589 11.13 11:cc4d8d83:::benchmark_data_smithi078_157677_object69418:head [delete] snapc 0=[] ondisk+write+known_if_redirected e199)", 2023-10-01T10:55:22.907 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:54:33.776428+0000", 2023-10-01T10:55:22.907 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 48.952508135999999, 2023-10-01T10:55:22.907 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.357038306, 2023-10-01T10:55:22.907 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:55:22.908 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:55:22.908 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:55:22.908 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46037", 2023-10-01T10:55:22.908 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:55:22.908 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 176589 2023-10-01T10:55:22.908 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.908 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:55:22.908 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.908 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:55:22.908 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.776428+0000", 2023-10-01T10:55:22.908 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:55:22.908 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.909 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.909 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:55:22.909 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.776428+0000", 2023-10-01T10:55:22.909 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:55:22.909 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.909 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.909 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:55:22.909 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.776426+0000", 2023-10-01T10:55:22.909 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999981 2023-10-01T10:55:22.909 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.909 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.909 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:55:22.909 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.776433+0000", 2023-10-01T10:55:22.910 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.8790000000000003e-06 2023-10-01T10:55:22.910 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.910 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.910 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:55:22.910 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.776451+0000", 2023-10-01T10:55:22.910 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.7453e-05 2023-10-01T10:55:22.910 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.910 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.910 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:55:22.910 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.776460+0000", 2023-10-01T10:55:22.910 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.7299999999999994e-06 2023-10-01T10:55:22.910 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.911 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.911 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:55:22.911 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.776656+0000", 2023-10-01T10:55:22.911 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00019599900000000001 2023-10-01T10:55:22.911 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.911 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.911 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T10:55:22.911 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.776684+0000", 2023-10-01T10:55:22.911 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.8699000000000001e-05 2023-10-01T10:55:22.911 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.911 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.911 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:55:22.911 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.100005+0000", 2023-10-01T10:55:22.912 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.323320432 2023-10-01T10:55:22.912 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.912 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.912 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:55:22.912 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.100137+0000", 2023-10-01T10:55:22.912 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00013244 2023-10-01T10:55:22.912 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.912 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.912 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 0,4,5,6,7", 2023-10-01T10:55:22.912 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.100431+0000", 2023-10-01T10:55:22.912 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00029331799999999999 2023-10-01T10:55:22.912 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.913 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.913 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:55:22.913 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.102738+0000", 2023-10-01T10:55:22.913 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0023078009999999999 2023-10-01T10:55:22.913 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.913 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.913 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.913 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.131755+0000", 2023-10-01T10:55:22.913 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.029016429999999999 2023-10-01T10:55:22.913 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.913 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.913 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.913 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.132174+0000", 2023-10-01T10:55:22.914 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00041950700000000001 2023-10-01T10:55:22.914 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.914 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.914 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.914 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.132228+0000", 2023-10-01T10:55:22.914 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.3334000000000001e-05 2023-10-01T10:55:22.914 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.914 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.914 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.914 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.132344+0000", 2023-10-01T10:55:22.914 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000116424 2023-10-01T10:55:22.915 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.915 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.915 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.915 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.133296+0000", 2023-10-01T10:55:22.915 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00095235899999999995 2023-10-01T10:55:22.915 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.915 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.915 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:55:22.915 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.133339+0000", 2023-10-01T10:55:22.915 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.2771000000000001e-05 2023-10-01T10:55:22.915 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.915 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.916 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:55:22.916 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.133466+0000", 2023-10-01T10:55:22.916 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00012671700000000001 2023-10-01T10:55:22.916 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:55:22.916 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:55:22.916 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:55:22.916 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.916 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.916 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46037.0:176678 11.13 11:c8f2364d:::benchmark_data_smithi078_157677_object69507:head [delete] snapc 0=[] ondisk+write+known_if_redirected e199)", 2023-10-01T10:55:22.916 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:54:33.956844+0000", 2023-10-01T10:55:22.916 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 48.772091375999999, 2023-10-01T10:55:22.916 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1774833840000001, 2023-10-01T10:55:22.917 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T10:55:22.917 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:55:22.917 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T10:55:22.917 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46037", 2023-10-01T10:55:22.917 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:55:22.917 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 176678 2023-10-01T10:55:22.917 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.917 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T10:55:22.917 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.917 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T10:55:22.917 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.956844+0000", 2023-10-01T10:55:22.918 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:55:22.918 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.918 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.918 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T10:55:22.918 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.956844+0000", 2023-10-01T10:55:22.918 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T10:55:22.918 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.918 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.918 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T10:55:22.918 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.956842+0000", 2023-10-01T10:55:22.918 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999971 2023-10-01T10:55:22.918 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.919 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.919 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T10:55:22.919 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.956851+0000", 2023-10-01T10:55:22.919 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.8339999999999994e-06 2023-10-01T10:55:22.919 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.919 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.919 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T10:55:22.919 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.956868+0000", 2023-10-01T10:55:22.919 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.6637999999999999e-05 2023-10-01T10:55:22.919 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.919 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.919 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T10:55:22.920 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.956878+0000", 2023-10-01T10:55:22.920 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.0113e-05 2023-10-01T10:55:22.920 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.920 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.920 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:55:22.920 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.956940+0000", 2023-10-01T10:55:22.920 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.2275999999999998e-05 2023-10-01T10:55:22.920 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.920 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.920 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T10:55:22.920 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.956951+0000", 2023-10-01T10:55:22.920 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.0576e-05 2023-10-01T10:55:22.921 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.921 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.921 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T10:55:22.921 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.102865+0000", 2023-10-01T10:55:22.921 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.145914238 2023-10-01T10:55:22.921 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.921 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.921 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T10:55:22.921 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.103040+0000", 2023-10-01T10:55:22.921 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00017503700000000001 2023-10-01T10:55:22.921 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.921 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.922 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 0,4,5,6,7", 2023-10-01T10:55:22.922 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.103316+0000", 2023-10-01T10:55:22.922 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00027566500000000002 2023-10-01T10:55:22.922 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.922 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.922 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T10:55:22.922 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.111360+0000", 2023-10-01T10:55:22.922 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0080440990000000007 2023-10-01T10:55:22.922 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.922 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.922 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.923 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.131844+0000", 2023-10-01T10:55:22.923 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.020483596 2023-10-01T10:55:22.923 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.923 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.923 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.923 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.132201+0000", 2023-10-01T10:55:22.923 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000357189 2023-10-01T10:55:22.923 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.923 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.923 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.923 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.132256+0000", 2023-10-01T10:55:22.923 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.4735999999999999e-05 2023-10-01T10:55:22.923 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.924 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.924 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.924 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.133510+0000", 2023-10-01T10:55:22.924 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0012545799999999999 2023-10-01T10:55:22.924 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.924 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.924 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:55:22.924 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.134179+0000", 2023-10-01T10:55:22.924 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00066873099999999999 2023-10-01T10:55:22.924 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.924 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.924 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T10:55:22.925 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.134220+0000", 2023-10-01T10:55:22.925 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.1638999999999999e-05 2023-10-01T10:55:22.925 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T10:55:22.925 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T10:55:22.925 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T10:55:22.925 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.134328+0000", 2023-10-01T10:55:22.925 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000107247 2023-10-01T10:55:22.925 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:55:22.925 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:55:22.925 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:55:22.925 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T10:55:22.926 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T10:55:22.926 INFO:teuthology.orchestra.run.smithi078.stdout:} 2023-10-01T10:55:22.926 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:22.927 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:55:22.959 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:22.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:22.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:55:23.062 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:23.096 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:23.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:23.102 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:55:23.202 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:23.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:23.244 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:23.244 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:55:23.345 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:23.368 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:23.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:23.371 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:55:23.471 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:23.495 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:23.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:23.503 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:55:23.603 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:23.622 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:23 smithi078 bash[78274]: cluster 2023-10-01T10:55:22.043215+0000 mgr.y (mgr.25072) 1136 : cluster 0 pgmap v1129: 99 pgs: 99 active+clean; 676 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 0 B/s, 0 objects/s recovering 2023-10-01T10:55:23.622 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:23 smithi078 bash[78274]: audit 2023-10-01T10:55:22.999360+0000 mon.a (mon.0) 1425 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:55:23.622 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:23 smithi078 bash[75299]: cluster 2023-10-01T10:55:22.043215+0000 mgr.y (mgr.25072) 1136 : cluster 0 pgmap v1129: 99 pgs: 99 active+clean; 676 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 0 B/s, 0 objects/s recovering 2023-10-01T10:55:23.622 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:23 smithi078 bash[75299]: audit 2023-10-01T10:55:22.999360+0000 mon.a (mon.0) 1425 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:55:23.631 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:23.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:23.638 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:55:23.739 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:23.767 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:23.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:23.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:55:23.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:23 smithi173 bash[58816]: cluster 2023-10-01T10:55:22.043215+0000 mgr.y (mgr.25072) 1136 : cluster 0 pgmap v1129: 99 pgs: 99 active+clean; 676 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 0 B/s, 0 objects/s recovering 2023-10-01T10:55:23.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:23 smithi173 bash[58816]: audit 2023-10-01T10:55:22.999360+0000 mon.a (mon.0) 1425 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:55:23.871 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:23.899 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:23.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:23.903 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:24.004 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:24.030 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:24.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:24.035 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:55:24.136 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:24.169 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:24.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:24.171 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:55:24.272 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:24.314 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:24.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:24.325 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:55:24.426 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:24.457 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:24.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:24.462 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:24.563 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:24.588 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:24.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:24.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:55:24.694 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:24.720 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:24.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:24.723 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:55:24.823 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:24.851 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:24.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:24.856 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:55:24.957 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:24.994 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:24.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:24.999 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:25.100 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:25.130 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:25.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:25.135 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:55:25.235 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:25.270 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:25.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:25.273 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:55:25.374 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:25.399 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:25.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:25.404 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:55:25.505 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:25.529 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:25.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:25.531 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:55:25.631 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:25.641 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:25 smithi078 bash[75299]: cluster 2023-10-01T10:55:24.043854+0000 mgr.y (mgr.25072) 1137 : cluster 0 pgmap v1130: 99 pgs: 99 active+clean; 455 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 839 op/s; 0 B/s, 0 objects/s recovering 2023-10-01T10:55:25.641 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:25 smithi078 bash[78274]: cluster 2023-10-01T10:55:24.043854+0000 mgr.y (mgr.25072) 1137 : cluster 0 pgmap v1130: 99 pgs: 99 active+clean; 455 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 839 op/s; 0 B/s, 0 objects/s recovering 2023-10-01T10:55:25.668 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:25.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:25.669 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:55:25.769 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:25.804 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:25.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:25.809 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:55:25.910 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:25.919 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:25 smithi173 bash[58816]: cluster 2023-10-01T10:55:24.043854+0000 mgr.y (mgr.25072) 1137 : cluster 0 pgmap v1130: 99 pgs: 99 active+clean; 455 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 839 op/s; 0 B/s, 0 objects/s recovering 2023-10-01T10:55:25.938 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:25.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:25.942 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:55:26.043 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:26.071 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:26.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:26.079 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:55:26.180 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:26.205 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:26.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:26.208 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:26.308 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:26.337 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:26.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:26.339 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:55:26.440 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:26.475 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:26.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:26.480 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:55:26.581 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:26.620 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:26.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:26.626 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:55:26.727 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:26.755 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:26.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:26.758 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:55:26.859 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:26.883 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:26.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:26.885 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:55:26.986 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:27.024 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:27.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:27.028 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:55:27.117 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-01T10:55:27.129 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:27.196 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:27.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:27.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:55:27.300 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:27.323 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:27.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:27.328 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:55:27.429 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:27.459 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:27.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:27.463 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:55:27.563 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:27.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:27.597 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:27.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:55:27.698 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:27.721 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:27.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:27.731 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:55:27.832 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:27.854 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:27.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:27.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:55:27.953 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:27 smithi078 bash[75299]: cluster 2023-10-01T10:55:26.044508+0000 mgr.y (mgr.25072) 1138 : cluster 0 pgmap v1131: 99 pgs: 99 active+clean; 402 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 803 op/s; 0 B/s, 0 objects/s recovering 2023-10-01T10:55:27.958 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:27.968 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:27 smithi173 bash[58816]: cluster 2023-10-01T10:55:26.044508+0000 mgr.y (mgr.25072) 1138 : cluster 0 pgmap v1131: 99 pgs: 99 active+clean; 402 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 803 op/s; 0 B/s, 0 objects/s recovering 2023-10-01T10:55:27.989 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:27.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:27.993 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:55:28.094 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:28.117 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:28.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:28.119 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:55:28.220 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:28.251 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:28.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:28.259 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:55:28.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:27 smithi078 bash[78274]: cluster 2023-10-01T10:55:26.044508+0000 mgr.y (mgr.25072) 1138 : cluster 0 pgmap v1131: 99 pgs: 99 active+clean; 402 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 803 op/s; 0 B/s, 0 objects/s recovering 2023-10-01T10:55:28.359 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:28.385 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:28.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:28.388 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:55:28.488 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:28.521 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:28.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:28.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:55:28.628 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:28.663 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:28.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:28.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:55:28.767 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:28.800 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:28.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:28.806 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:55:28.907 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:28.933 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:28.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:28.937 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:29.038 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:29.075 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:29.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:29.078 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:55:29.178 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:29.210 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:29.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:29.213 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:55:29.313 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:29.347 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:29.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:29.350 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:55:29.451 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:29.493 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:29.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:29.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:55:29.599 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:29.625 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:29.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:29.628 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:55:29.728 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:29.757 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:29.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:29.763 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:55:29.864 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:29.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:29 smithi173 bash[58816]: cluster 2023-10-01T10:55:28.045016+0000 mgr.y (mgr.25072) 1139 : cluster 0 pgmap v1132: 99 pgs: 99 active+clean; 341 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 821 op/s 2023-10-01T10:55:29.888 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:29.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:29.891 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:55:29.914 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:29 smithi078 bash[75299]: cluster 2023-10-01T10:55:28.045016+0000 mgr.y (mgr.25072) 1139 : cluster 0 pgmap v1132: 99 pgs: 99 active+clean; 341 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 821 op/s 2023-10-01T10:55:29.915 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:29 smithi078 bash[78274]: cluster 2023-10-01T10:55:28.045016+0000 mgr.y (mgr.25072) 1139 : cluster 0 pgmap v1132: 99 pgs: 99 active+clean; 341 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 821 op/s 2023-10-01T10:55:29.991 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:30.025 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:30.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:30.027 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:55:30.128 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:30.149 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:30.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:30.155 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:55:30.256 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:30.293 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:30.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:30.298 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:55:30.399 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:30.426 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:30.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:30.428 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:55:30.529 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:30.537 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:55:30 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:55:30] "GET /metrics HTTP/1.1" 200 36329 "" "Prometheus/2.33.4" 2023-10-01T10:55:30.565 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:30.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:30.567 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:55:30.668 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:30.699 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:30.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:30.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:55:30.801 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Removed 107168 objects 2023-10-01T10:55:30.801 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Clean up completed and total clean up time :158.526 2023-10-01T10:55:30.805 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:30.828 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:30.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:30.830 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:55:30.931 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:30.967 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:30.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:30.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:55:31.071 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:31.104 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:31.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:31.106 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:31.207 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:31.234 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:31.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:31.237 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:55:31.338 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:31.369 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_6 2023-10-01T10:55:31.369 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_6 unique_pool_6 --yes-i-really-really-mean-it 2023-10-01T10:55:31.369 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:31.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:31.372 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:55:31.473 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:31.496 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:31.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:31.499 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:55:31.600 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:31.632 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:31.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:31.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:55:31.735 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:31.764 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:31.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:31.768 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:55:31.868 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:31.900 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:31.902 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:31 smithi173 bash[58816]: cluster 2023-10-01T10:55:30.045828+0000 mgr.y (mgr.25072) 1140 : cluster 0 pgmap v1133: 99 pgs: 99 active+clean; 132 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2023-10-01T10:55:31.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:31.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:55:32.004 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:32.036 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:32.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:32.039 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:55:32.140 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:32.155 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:31 smithi078 bash[78274]: cluster 2023-10-01T10:55:30.045828+0000 mgr.y (mgr.25072) 1140 : cluster 0 pgmap v1133: 99 pgs: 99 active+clean; 132 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2023-10-01T10:55:32.155 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:31 smithi078 bash[75299]: cluster 2023-10-01T10:55:30.045828+0000 mgr.y (mgr.25072) 1140 : cluster 0 pgmap v1133: 99 pgs: 99 active+clean; 132 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2023-10-01T10:55:32.159 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:32.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:32.162 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:55:32.263 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:32.293 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:32.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:32.296 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:32.397 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:32.426 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:32.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:32.430 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:32.531 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:32.556 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:32.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:32.558 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:55:32.659 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:32.692 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:32.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:32.694 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:55:32.795 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:32.807 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T10:55:32.812 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:32 smithi078 bash[75299]: audit 2023-10-01T10:55:32.443532+0000 mon.a (mon.0) 1426 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:55:32.812 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:32 smithi078 bash[78274]: audit 2023-10-01T10:55:32.443532+0000 mon.a (mon.0) 1426 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:55:32.827 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:32.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:32.830 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:55:32.831 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T10:55:32.852 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T10:55:32.875 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T10:55:32.903 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T10:55:32.925 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T10:55:32.932 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:32.959 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:32.960 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T10:55:32.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:32.963 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:55:32.984 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T10:55:32.987 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T10:55:32.987 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T10:55:32.987 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T10:55:32.987 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T10:55:32.987 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T10:55:32.987 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T10:55:32.987 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T10:55:32.987 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T10:55:33.064 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:33.085 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:33.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:33.088 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:55:33.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:32 smithi173 bash[58816]: audit 2023-10-01T10:55:32.443532+0000 mon.a (mon.0) 1426 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:55:33.189 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:33.222 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:33.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:33.225 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:55:33.326 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:33.353 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:33.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:33.355 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:55:33.456 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:33.491 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:33.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:33.494 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:33.594 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:33.622 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:33.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:33.625 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:55:33.656 INFO:teuthology.orchestra.run.smithi078.stderr:noscrub is unset 2023-10-01T10:55:33.726 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:33.756 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:33.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:33.759 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:55:33.860 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:33.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:33 smithi173 bash[58816]: cluster 2023-10-01T10:55:32.046323+0000 mgr.y (mgr.25072) 1141 : cluster 0 pgmap v1134: 99 pgs: 99 active+clean; 132 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 725 op/s 2023-10-01T10:55:33.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:33 smithi173 bash[58816]: audit 2023-10-01T10:55:32.781839+0000 mon.c (mon.2) 183 : audit 0 from='client.? 172.21.15.78:0/735612422' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:55:33.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:33 smithi173 bash[58816]: audit 2023-10-01T10:55:33.233077+0000 mon.c (mon.2) 184 : audit 1 from='client.? 172.21.15.78:0/986862631' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T10:55:33.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:33 smithi173 bash[58816]: audit 2023-10-01T10:55:33.233481+0000 mon.a (mon.0) 1427 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T10:55:33.871 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:33 smithi078 bash[78274]: cluster 2023-10-01T10:55:32.046323+0000 mgr.y (mgr.25072) 1141 : cluster 0 pgmap v1134: 99 pgs: 99 active+clean; 132 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 725 op/s 2023-10-01T10:55:33.871 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:33 smithi078 bash[78274]: audit 2023-10-01T10:55:32.781839+0000 mon.c (mon.2) 183 : audit 0 from='client.? 172.21.15.78:0/735612422' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:55:33.871 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:33 smithi078 bash[78274]: audit 2023-10-01T10:55:33.233077+0000 mon.c (mon.2) 184 : audit 1 from='client.? 172.21.15.78:0/986862631' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T10:55:33.871 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:33 smithi078 bash[78274]: audit 2023-10-01T10:55:33.233481+0000 mon.a (mon.0) 1427 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T10:55:33.871 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:33 smithi078 bash[75299]: cluster 2023-10-01T10:55:32.046323+0000 mgr.y (mgr.25072) 1141 : cluster 0 pgmap v1134: 99 pgs: 99 active+clean; 132 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 725 op/s 2023-10-01T10:55:33.871 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:33 smithi078 bash[75299]: audit 2023-10-01T10:55:32.781839+0000 mon.c (mon.2) 183 : audit 0 from='client.? 172.21.15.78:0/735612422' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:55:33.872 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:33 smithi078 bash[75299]: audit 2023-10-01T10:55:33.233077+0000 mon.c (mon.2) 184 : audit 1 from='client.? 172.21.15.78:0/986862631' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T10:55:33.872 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:33 smithi078 bash[75299]: audit 2023-10-01T10:55:33.233481+0000 mon.a (mon.0) 1427 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T10:55:33.885 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:33.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:33.889 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:55:33.989 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:34.015 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:34.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:34.018 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:55:34.118 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:34.151 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:34.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:34.155 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:55:34.256 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:34.289 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:34.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:34.292 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:55:34.393 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:34.419 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:34.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:34.422 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:55:34.523 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:34.556 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:34.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:34.558 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:55:34.659 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:34.687 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:34.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:34.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:34.791 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:34.820 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:34.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:34.823 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:55:34.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:34 smithi173 bash[58816]: cluster 2023-10-01T10:55:33.642567+0000 mon.a (mon.0) 1428 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:55:34.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:34 smithi173 bash[58816]: cluster 2023-10-01T10:55:33.646790+0000 osd.7 (osd.7) 195 : cluster 3 failed to encode map e226 with expected crc 2023-10-01T10:55:34.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:34 smithi173 bash[58816]: audit 2023-10-01T10:55:33.648668+0000 mon.a (mon.0) 1429 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T10:55:34.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:34 smithi173 bash[58816]: cluster 2023-10-01T10:55:33.648852+0000 mon.a (mon.0) 1430 : cluster 0 osdmap e226: 8 total, 8 up, 8 in 2023-10-01T10:55:34.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:34 smithi173 bash[58816]: cluster 2023-10-01T10:55:34.035270+0000 osd.6 (osd.6) 156 : cluster 3 failed to encode map e226 with expected crc 2023-10-01T10:55:34.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:34 smithi173 bash[58816]: cluster 2023-10-01T10:55:34.038191+0000 osd.6 (osd.6) 157 : cluster 3 failed to encode map e226 with expected crc 2023-10-01T10:55:34.924 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:34.958 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:34.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:34.961 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:35.062 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:35.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:34 smithi078 bash[75299]: cluster 2023-10-01T10:55:33.642567+0000 mon.a (mon.0) 1428 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:55:35.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:34 smithi078 bash[75299]: cluster 2023-10-01T10:55:33.646790+0000 osd.7 (osd.7) 195 : cluster 3 failed to encode map e226 with expected crc 2023-10-01T10:55:35.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:34 smithi078 bash[75299]: audit 2023-10-01T10:55:33.648668+0000 mon.a (mon.0) 1429 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T10:55:35.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:34 smithi078 bash[75299]: cluster 2023-10-01T10:55:33.648852+0000 mon.a (mon.0) 1430 : cluster 0 osdmap e226: 8 total, 8 up, 8 in 2023-10-01T10:55:35.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:34 smithi078 bash[75299]: cluster 2023-10-01T10:55:34.035270+0000 osd.6 (osd.6) 156 : cluster 3 failed to encode map e226 with expected crc 2023-10-01T10:55:35.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:34 smithi078 bash[75299]: cluster 2023-10-01T10:55:34.038191+0000 osd.6 (osd.6) 157 : cluster 3 failed to encode map e226 with expected crc 2023-10-01T10:55:35.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:34 smithi078 bash[78274]: cluster 2023-10-01T10:55:33.642567+0000 mon.a (mon.0) 1428 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:55:35.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:34 smithi078 bash[78274]: cluster 2023-10-01T10:55:33.646790+0000 osd.7 (osd.7) 195 : cluster 3 failed to encode map e226 with expected crc 2023-10-01T10:55:35.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:34 smithi078 bash[78274]: audit 2023-10-01T10:55:33.648668+0000 mon.a (mon.0) 1429 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T10:55:35.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:34 smithi078 bash[78274]: cluster 2023-10-01T10:55:33.648852+0000 mon.a (mon.0) 1430 : cluster 0 osdmap e226: 8 total, 8 up, 8 in 2023-10-01T10:55:35.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:34 smithi078 bash[78274]: cluster 2023-10-01T10:55:34.035270+0000 osd.6 (osd.6) 156 : cluster 3 failed to encode map e226 with expected crc 2023-10-01T10:55:35.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:34 smithi078 bash[78274]: cluster 2023-10-01T10:55:34.038191+0000 osd.6 (osd.6) 157 : cluster 3 failed to encode map e226 with expected crc 2023-10-01T10:55:35.093 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:35.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:35.096 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:55:35.197 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:35.226 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:35.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:35.228 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:55:35.330 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:35.361 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:35.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:35.363 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:55:35.464 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:35.497 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:35.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:35.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:55:35.601 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:35.632 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:35.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:35.635 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:55:35.736 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:35.739 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:55:35.739 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:55:35.772 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:35.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:35.777 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:55:35.878 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:35.911 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:35.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:35.914 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:55:36.015 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:36.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:35 smithi173 bash[58816]: cluster 2023-10-01T10:55:34.034476+0000 osd.4 (osd.4) 1 : cluster 0 11.14 scrub starts 2023-10-01T10:55:36.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:35 smithi173 bash[58816]: cluster 2023-10-01T10:55:34.046927+0000 mgr.y (mgr.25072) 1142 : cluster 0 pgmap v1136: 99 pgs: 99 active+clean; 25 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2023-10-01T10:55:36.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:35 smithi173 bash[58816]: cluster 2023-10-01T10:55:34.065969+0000 osd.4 (osd.4) 2 : cluster 0 11.14 scrub ok 2023-10-01T10:55:36.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:35 smithi173 bash[58816]: cluster 2023-10-01T10:55:34.757440+0000 osd.5 (osd.5) 190 : cluster 3 failed to encode map e226 with expected crc 2023-10-01T10:55:36.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:35 smithi173 bash[58816]: cluster 2023-10-01T10:55:35.101752+0000 osd.2 (osd.2) 4 : cluster 0 11.2 scrub starts 2023-10-01T10:55:36.029 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:35 smithi173 bash[58816]: cluster 2023-10-01T10:55:35.125662+0000 osd.2 (osd.2) 5 : cluster 0 11.2 scrub ok 2023-10-01T10:55:36.046 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:36.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:36.050 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:55:36.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:35 smithi078 bash[78274]: cluster 2023-10-01T10:55:34.034476+0000 osd.4 (osd.4) 1 : cluster 0 11.14 scrub starts 2023-10-01T10:55:36.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:35 smithi078 bash[78274]: cluster 2023-10-01T10:55:34.046927+0000 mgr.y (mgr.25072) 1142 : cluster 0 pgmap v1136: 99 pgs: 99 active+clean; 25 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2023-10-01T10:55:36.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:35 smithi078 bash[78274]: cluster 2023-10-01T10:55:34.065969+0000 osd.4 (osd.4) 2 : cluster 0 11.14 scrub ok 2023-10-01T10:55:36.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:35 smithi078 bash[78274]: cluster 2023-10-01T10:55:34.757440+0000 osd.5 (osd.5) 190 : cluster 3 failed to encode map e226 with expected crc 2023-10-01T10:55:36.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:35 smithi078 bash[78274]: cluster 2023-10-01T10:55:35.101752+0000 osd.2 (osd.2) 4 : cluster 0 11.2 scrub starts 2023-10-01T10:55:36.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:35 smithi078 bash[78274]: cluster 2023-10-01T10:55:35.125662+0000 osd.2 (osd.2) 5 : cluster 0 11.2 scrub ok 2023-10-01T10:55:36.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:35 smithi078 bash[75299]: cluster 2023-10-01T10:55:34.034476+0000 osd.4 (osd.4) 1 : cluster 0 11.14 scrub starts 2023-10-01T10:55:36.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:35 smithi078 bash[75299]: cluster 2023-10-01T10:55:34.046927+0000 mgr.y (mgr.25072) 1142 : cluster 0 pgmap v1136: 99 pgs: 99 active+clean; 25 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2023-10-01T10:55:36.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:35 smithi078 bash[75299]: cluster 2023-10-01T10:55:34.065969+0000 osd.4 (osd.4) 2 : cluster 0 11.14 scrub ok 2023-10-01T10:55:36.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:35 smithi078 bash[75299]: cluster 2023-10-01T10:55:34.757440+0000 osd.5 (osd.5) 190 : cluster 3 failed to encode map e226 with expected crc 2023-10-01T10:55:36.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:35 smithi078 bash[75299]: cluster 2023-10-01T10:55:35.101752+0000 osd.2 (osd.2) 4 : cluster 0 11.2 scrub starts 2023-10-01T10:55:36.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:35 smithi078 bash[75299]: cluster 2023-10-01T10:55:35.125662+0000 osd.2 (osd.2) 5 : cluster 0 11.2 scrub ok 2023-10-01T10:55:36.150 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:36.182 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:36.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:36.186 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:36.286 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:36.320 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:36.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:36.323 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:55:36.424 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:36.451 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:36.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:36.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:55:36.467 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T10:55:36.555 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:36.589 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:36.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:36.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:55:36.694 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:36.718 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:36.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:36.720 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:55:36.821 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:36.845 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:36.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:36.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:55:36.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:36 smithi173 bash[58816]: cluster 2023-10-01T10:55:35.754716+0000 osd.5 (osd.5) 191 : cluster 0 11.5 scrub starts 2023-10-01T10:55:36.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:36 smithi173 bash[58816]: cluster 2023-10-01T10:55:35.771656+0000 osd.5 (osd.5) 192 : cluster 0 11.5 scrub ok 2023-10-01T10:55:36.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:36 smithi173 bash[58816]: cluster 2023-10-01T10:55:35.788684+0000 osd.7 (osd.7) 196 : cluster 0 11.17 scrub starts 2023-10-01T10:55:36.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:36 smithi173 bash[58816]: cluster 2023-10-01T10:55:35.798881+0000 osd.7 (osd.7) 197 : cluster 0 11.17 scrub ok 2023-10-01T10:55:36.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:36 smithi173 bash[58816]: cluster 2023-10-01T10:55:35.861427+0000 osd.3 (osd.3) 18 : cluster 0 11.0 scrub starts 2023-10-01T10:55:36.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:36 smithi173 bash[58816]: cluster 2023-10-01T10:55:35.881546+0000 osd.6 (osd.6) 158 : cluster 0 11.3 scrub starts 2023-10-01T10:55:36.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:36 smithi173 bash[58816]: cluster 2023-10-01T10:55:35.884231+0000 osd.3 (osd.3) 19 : cluster 0 11.0 scrub ok 2023-10-01T10:55:36.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:36 smithi173 bash[58816]: cluster 2023-10-01T10:55:35.984388+0000 osd.0 (osd.0) 13 : cluster 0 11.10 scrub starts 2023-10-01T10:55:36.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:36 smithi173 bash[58816]: cluster 2023-10-01T10:55:36.006642+0000 osd.0 (osd.0) 14 : cluster 0 11.10 scrub ok 2023-10-01T10:55:36.948 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:36.981 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:36.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:36.985 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:37.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:36 smithi078 bash[78274]: cluster 2023-10-01T10:55:35.754716+0000 osd.5 (osd.5) 191 : cluster 0 11.5 scrub starts 2023-10-01T10:55:37.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:36 smithi078 bash[78274]: cluster 2023-10-01T10:55:35.771656+0000 osd.5 (osd.5) 192 : cluster 0 11.5 scrub ok 2023-10-01T10:55:37.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:36 smithi078 bash[78274]: cluster 2023-10-01T10:55:35.788684+0000 osd.7 (osd.7) 196 : cluster 0 11.17 scrub starts 2023-10-01T10:55:37.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:36 smithi078 bash[78274]: cluster 2023-10-01T10:55:35.798881+0000 osd.7 (osd.7) 197 : cluster 0 11.17 scrub ok 2023-10-01T10:55:37.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:36 smithi078 bash[78274]: cluster 2023-10-01T10:55:35.861427+0000 osd.3 (osd.3) 18 : cluster 0 11.0 scrub starts 2023-10-01T10:55:37.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:36 smithi078 bash[78274]: cluster 2023-10-01T10:55:35.881546+0000 osd.6 (osd.6) 158 : cluster 0 11.3 scrub starts 2023-10-01T10:55:37.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:36 smithi078 bash[78274]: cluster 2023-10-01T10:55:35.884231+0000 osd.3 (osd.3) 19 : cluster 0 11.0 scrub ok 2023-10-01T10:55:37.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:36 smithi078 bash[78274]: cluster 2023-10-01T10:55:35.984388+0000 osd.0 (osd.0) 13 : cluster 0 11.10 scrub starts 2023-10-01T10:55:37.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:36 smithi078 bash[78274]: cluster 2023-10-01T10:55:36.006642+0000 osd.0 (osd.0) 14 : cluster 0 11.10 scrub ok 2023-10-01T10:55:37.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:36 smithi078 bash[75299]: cluster 2023-10-01T10:55:35.754716+0000 osd.5 (osd.5) 191 : cluster 0 11.5 scrub starts 2023-10-01T10:55:37.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:36 smithi078 bash[75299]: cluster 2023-10-01T10:55:35.771656+0000 osd.5 (osd.5) 192 : cluster 0 11.5 scrub ok 2023-10-01T10:55:37.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:36 smithi078 bash[75299]: cluster 2023-10-01T10:55:35.788684+0000 osd.7 (osd.7) 196 : cluster 0 11.17 scrub starts 2023-10-01T10:55:37.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:36 smithi078 bash[75299]: cluster 2023-10-01T10:55:35.798881+0000 osd.7 (osd.7) 197 : cluster 0 11.17 scrub ok 2023-10-01T10:55:37.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:36 smithi078 bash[75299]: cluster 2023-10-01T10:55:35.861427+0000 osd.3 (osd.3) 18 : cluster 0 11.0 scrub starts 2023-10-01T10:55:37.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:36 smithi078 bash[75299]: cluster 2023-10-01T10:55:35.881546+0000 osd.6 (osd.6) 158 : cluster 0 11.3 scrub starts 2023-10-01T10:55:37.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:36 smithi078 bash[75299]: cluster 2023-10-01T10:55:35.884231+0000 osd.3 (osd.3) 19 : cluster 0 11.0 scrub ok 2023-10-01T10:55:37.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:36 smithi078 bash[75299]: cluster 2023-10-01T10:55:35.984388+0000 osd.0 (osd.0) 13 : cluster 0 11.10 scrub starts 2023-10-01T10:55:37.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:36 smithi078 bash[75299]: cluster 2023-10-01T10:55:36.006642+0000 osd.0 (osd.0) 14 : cluster 0 11.10 scrub ok 2023-10-01T10:55:37.085 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:37.109 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:37.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:37.111 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:55:37.212 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:37.239 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:37.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:37.241 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:37.319 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T10:55:37.342 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:37.392 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:37.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:37.394 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:55:37.495 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:37.519 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:37.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:37.521 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:55:37.622 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:37.646 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:37.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:37.649 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:55:37.750 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:37.783 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:37.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:37.786 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:55:37.886 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:37.918 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:37.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:37.922 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:55:38.023 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:38.035 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:37 smithi078 bash[78274]: cluster 2023-10-01T10:55:36.047866+0000 mgr.y (mgr.25072) 1143 : cluster 0 pgmap v1137: 99 pgs: 99 active+clean; 13 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-10-01T10:55:38.035 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:37 smithi078 bash[78274]: cluster 2023-10-01T10:55:36.771621+0000 osd.5 (osd.5) 193 : cluster 0 11.7 scrub starts 2023-10-01T10:55:38.035 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:37 smithi078 bash[78274]: cluster 2023-10-01T10:55:36.786720+0000 osd.5 (osd.5) 194 : cluster 0 11.7 scrub ok 2023-10-01T10:55:38.035 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:37 smithi078 bash[78274]: cluster 2023-10-01T10:55:36.974768+0000 osd.0 (osd.0) 15 : cluster 0 11.19 scrub starts 2023-10-01T10:55:38.035 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:37 smithi078 bash[78274]: cluster 2023-10-01T10:55:36.988538+0000 osd.0 (osd.0) 16 : cluster 0 11.19 scrub ok 2023-10-01T10:55:38.036 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:37 smithi078 bash[75299]: cluster 2023-10-01T10:55:36.047866+0000 mgr.y (mgr.25072) 1143 : cluster 0 pgmap v1137: 99 pgs: 99 active+clean; 13 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-10-01T10:55:38.036 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:37 smithi078 bash[75299]: cluster 2023-10-01T10:55:36.771621+0000 osd.5 (osd.5) 193 : cluster 0 11.7 scrub starts 2023-10-01T10:55:38.036 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:37 smithi078 bash[75299]: cluster 2023-10-01T10:55:36.786720+0000 osd.5 (osd.5) 194 : cluster 0 11.7 scrub ok 2023-10-01T10:55:38.036 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:37 smithi078 bash[75299]: cluster 2023-10-01T10:55:36.974768+0000 osd.0 (osd.0) 15 : cluster 0 11.19 scrub starts 2023-10-01T10:55:38.036 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:37 smithi078 bash[75299]: cluster 2023-10-01T10:55:36.988538+0000 osd.0 (osd.0) 16 : cluster 0 11.19 scrub ok 2023-10-01T10:55:38.051 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:38.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:38.054 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:55:38.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:37 smithi173 bash[58816]: cluster 2023-10-01T10:55:36.047866+0000 mgr.y (mgr.25072) 1143 : cluster 0 pgmap v1137: 99 pgs: 99 active+clean; 13 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-10-01T10:55:38.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:37 smithi173 bash[58816]: cluster 2023-10-01T10:55:36.771621+0000 osd.5 (osd.5) 193 : cluster 0 11.7 scrub starts 2023-10-01T10:55:38.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:37 smithi173 bash[58816]: cluster 2023-10-01T10:55:36.786720+0000 osd.5 (osd.5) 194 : cluster 0 11.7 scrub ok 2023-10-01T10:55:38.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:37 smithi173 bash[58816]: cluster 2023-10-01T10:55:36.974768+0000 osd.0 (osd.0) 15 : cluster 0 11.19 scrub starts 2023-10-01T10:55:38.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:37 smithi173 bash[58816]: cluster 2023-10-01T10:55:36.988538+0000 osd.0 (osd.0) 16 : cluster 0 11.19 scrub ok 2023-10-01T10:55:38.155 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:38.176 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:38.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:38.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:55:38.279 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:38.304 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:38.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:38.307 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:55:38.408 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:38.442 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:38.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:38.446 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:55:38.546 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:38.582 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:38.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:38.586 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:55:38.687 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:38.717 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:38.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:38.720 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:38.821 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:38.845 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:38.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:38.848 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:55:38.948 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:38.974 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:38.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:38.977 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:55:39.077 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:39.105 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:39.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:39.108 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:55:39.209 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:39.241 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:39.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:39.244 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:55:39.345 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:39.376 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:39.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:39.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:39.479 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:39.510 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:39.510 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:55:39.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:39.513 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:55:39.614 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:39.644 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:39.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:39.647 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:55:39.748 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:39.774 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:39.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:39.776 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:55:39.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:39 smithi078 bash[75299]: cluster 2023-10-01T10:55:38.048543+0000 mgr.y (mgr.25072) 1144 : cluster 0 pgmap v1138: 99 pgs: 99 active+clean; 13 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-10-01T10:55:39.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:39 smithi078 bash[75299]: cluster 2023-10-01T10:55:38.796752+0000 osd.5 (osd.5) 195 : cluster 0 11.13 scrub starts 2023-10-01T10:55:39.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:39 smithi078 bash[75299]: cluster 2023-10-01T10:55:38.808937+0000 osd.5 (osd.5) 196 : cluster 0 11.13 scrub ok 2023-10-01T10:55:39.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:39 smithi078 bash[78274]: cluster 2023-10-01T10:55:38.048543+0000 mgr.y (mgr.25072) 1144 : cluster 0 pgmap v1138: 99 pgs: 99 active+clean; 13 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-10-01T10:55:39.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:39 smithi078 bash[78274]: cluster 2023-10-01T10:55:38.796752+0000 osd.5 (osd.5) 195 : cluster 0 11.13 scrub starts 2023-10-01T10:55:39.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:39 smithi078 bash[78274]: cluster 2023-10-01T10:55:38.808937+0000 osd.5 (osd.5) 196 : cluster 0 11.13 scrub ok 2023-10-01T10:55:39.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:39 smithi173 bash[58816]: cluster 2023-10-01T10:55:38.048543+0000 mgr.y (mgr.25072) 1144 : cluster 0 pgmap v1138: 99 pgs: 99 active+clean; 13 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-10-01T10:55:39.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:39 smithi173 bash[58816]: cluster 2023-10-01T10:55:38.796752+0000 osd.5 (osd.5) 195 : cluster 0 11.13 scrub starts 2023-10-01T10:55:39.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:39 smithi173 bash[58816]: cluster 2023-10-01T10:55:38.808937+0000 osd.5 (osd.5) 196 : cluster 0 11.13 scrub ok 2023-10-01T10:55:39.878 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:39.910 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:39.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:39.913 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:40.014 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:40.046 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:40.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:40.049 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:55:40.149 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:40.177 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:40.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:40.180 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:55:40.281 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:40.312 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:40.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:40.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:40.415 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:40.423 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:55:40 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:55:40] "GET /metrics HTTP/1.1" 200 36307 "" "Prometheus/2.33.4" 2023-10-01T10:55:40.441 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:40.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:40.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:55:40.544 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:40.580 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:40.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:40.583 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:40.684 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:40.714 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:40.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:40.718 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:55:40.819 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:40.842 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:40.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:40.844 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:55:40.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:40 smithi173 bash[58816]: cluster 2023-10-01T10:55:39.834250+0000 osd.6 (osd.6) 159 : cluster 0 11.3 scrub starts 2023-10-01T10:55:40.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:40 smithi173 bash[58816]: cluster 2023-10-01T10:55:39.851633+0000 osd.6 (osd.6) 160 : cluster 0 11.3 scrub ok 2023-10-01T10:55:40.945 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:40.956 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:40 smithi078 bash[78274]: cluster 2023-10-01T10:55:39.834250+0000 osd.6 (osd.6) 159 : cluster 0 11.3 scrub starts 2023-10-01T10:55:40.956 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:40 smithi078 bash[78274]: cluster 2023-10-01T10:55:39.851633+0000 osd.6 (osd.6) 160 : cluster 0 11.3 scrub ok 2023-10-01T10:55:40.956 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:40 smithi078 bash[75299]: cluster 2023-10-01T10:55:39.834250+0000 osd.6 (osd.6) 159 : cluster 0 11.3 scrub starts 2023-10-01T10:55:40.956 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:40 smithi078 bash[75299]: cluster 2023-10-01T10:55:39.851633+0000 osd.6 (osd.6) 160 : cluster 0 11.3 scrub ok 2023-10-01T10:55:40.972 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:40.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:40.974 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:55:41.075 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:41.108 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:41.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:41.111 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:41.212 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:41.239 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:41.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:41.241 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:55:41.342 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:41.359 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:55:41.360 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:55:41.374 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:41.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:41.376 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:55:41.477 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:41.498 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:41.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:41.500 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:55:41.601 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:41.629 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:41.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:41.631 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:55:41.732 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:41.766 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:41.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:41.768 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:55:41.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:41 smithi078 bash[75299]: cluster 2023-10-01T10:55:40.049749+0000 mgr.y (mgr.25072) 1145 : cluster 0 pgmap v1139: 99 pgs: 99 active+clean; 6.1 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-10-01T10:55:41.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:41 smithi078 bash[78274]: cluster 2023-10-01T10:55:40.049749+0000 mgr.y (mgr.25072) 1145 : cluster 0 pgmap v1139: 99 pgs: 99 active+clean; 6.1 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-10-01T10:55:41.869 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:41.895 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:41.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:41.898 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:55:41.999 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:42.010 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:41 smithi173 bash[58816]: cluster 2023-10-01T10:55:40.049749+0000 mgr.y (mgr.25072) 1145 : cluster 0 pgmap v1139: 99 pgs: 99 active+clean; 6.1 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-10-01T10:55:42.032 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:42.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:42.035 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:42.136 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:42.168 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:42.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:42.172 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:55:42.272 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:42.302 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:42.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:42.304 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:55:42.405 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:42.426 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:42.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:42.428 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:55:42.529 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:42.557 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:42.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:42.559 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:55:42.662 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:42.694 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:42.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:42.697 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:42.797 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:42.824 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:42.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:42.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:55:42.927 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:42.959 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:42.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:42.962 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:43.063 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:43.087 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:43.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:43.089 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:55:43.189 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:43.223 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:43.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:43.225 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:43.326 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:43.358 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:43.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:43.360 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:55:43.461 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:43.486 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:43.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:43.488 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:55:43.589 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:43.622 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:43.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:43.625 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:43.718 INFO:teuthology.orchestra.run.smithi078.stderr:marked out osd.6. 2023-10-01T10:55:43.726 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:43.746 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:43.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:43.749 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:43.817 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:43 smithi078 bash[75299]: debug 2023-10-01T10:55:43.504+0000 7f31c8dd0700 -1 mon.a@0(leader).osd e226 definitely_dead 0 2023-10-01T10:55:43.817 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:43 smithi078 bash[75299]: cluster 2023-10-01T10:55:42.050456+0000 mgr.y (mgr.25072) 1146 : cluster 0 pgmap v1140: 99 pgs: 99 active+clean; 6.1 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-10-01T10:55:43.817 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:43 smithi078 bash[75299]: audit 2023-10-01T10:55:43.508907+0000 mon.a (mon.0) 1431 : audit 1 from='client.? 172.21.15.78:0/3396088582' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-01T10:55:43.850 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:43.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:43 smithi173 bash[58816]: cluster 2023-10-01T10:55:42.050456+0000 mgr.y (mgr.25072) 1146 : cluster 0 pgmap v1140: 99 pgs: 99 active+clean; 6.1 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-10-01T10:55:43.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:43 smithi173 bash[58816]: audit 2023-10-01T10:55:43.508907+0000 mon.a (mon.0) 1431 : audit 1 from='client.? 172.21.15.78:0/3396088582' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-01T10:55:43.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:43 smithi173 bash[58816]: cluster 2023-10-01T10:55:43.508984+0000 mon.a (mon.0) 1432 : cluster 1 Client client.admin marked osd.6 out, while it was still marked up 2023-10-01T10:55:43.871 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:43.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:43.873 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:55:43.974 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:44.008 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:44.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:44.012 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:55:44.113 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:44.142 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:44.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:44.146 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:55:44.212 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:43 smithi078 bash[75299]: cluster 2023-10-01T10:55:43.508984+0000 mon.a (mon.0) 1432 : cluster 1 Client client.admin marked osd.6 out, while it was still marked up 2023-10-01T10:55:44.213 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:43 smithi078 bash[78274]: cluster 2023-10-01T10:55:42.050456+0000 mgr.y (mgr.25072) 1146 : cluster 0 pgmap v1140: 99 pgs: 99 active+clean; 6.1 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-10-01T10:55:44.213 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:43 smithi078 bash[78274]: audit 2023-10-01T10:55:43.508907+0000 mon.a (mon.0) 1431 : audit 1 from='client.? 172.21.15.78:0/3396088582' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-01T10:55:44.213 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:43 smithi078 bash[78274]: cluster 2023-10-01T10:55:43.508984+0000 mon.a (mon.0) 1432 : cluster 1 Client client.admin marked osd.6 out, while it was still marked up 2023-10-01T10:55:44.246 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:44.281 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:44.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:44.284 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:55:44.385 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:44.410 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:44.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:44.412 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:55:44.513 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:44.545 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:44.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:44.548 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:44.649 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:44.680 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:44.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:44.684 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:55:44.784 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:44.809 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:44.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:44.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:55:44.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:44 smithi173 bash[58816]: cluster 2023-10-01T10:55:43.697226+0000 osd.7 (osd.7) 198 : cluster 3 failed to encode map e227 with expected crc 2023-10-01T10:55:44.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:44 smithi173 bash[58816]: cluster 2023-10-01T10:55:43.701097+0000 osd.6 (osd.6) 161 : cluster 3 failed to encode map e227 with expected crc 2023-10-01T10:55:44.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:44 smithi173 bash[58816]: cluster 2023-10-01T10:55:43.701618+0000 osd.7 (osd.7) 199 : cluster 3 failed to encode map e227 with expected crc 2023-10-01T10:55:44.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:44 smithi173 bash[58816]: cluster 2023-10-01T10:55:43.701849+0000 osd.5 (osd.5) 197 : cluster 3 failed to encode map e227 with expected crc 2023-10-01T10:55:44.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:44 smithi173 bash[58816]: audit 2023-10-01T10:55:43.702917+0000 mon.a (mon.0) 1433 : audit 1 from='client.? 172.21.15.78:0/3396088582' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-10-01T10:55:44.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:44 smithi173 bash[58816]: cluster 2023-10-01T10:55:43.703115+0000 mon.a (mon.0) 1434 : cluster 0 osdmap e227: 8 total, 8 up, 7 in 2023-10-01T10:55:44.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:44 smithi173 bash[58816]: cluster 2023-10-01T10:55:43.706836+0000 osd.5 (osd.5) 198 : cluster 3 failed to encode map e227 with expected crc 2023-10-01T10:55:44.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:44 smithi173 bash[58816]: cluster 2023-10-01T10:55:43.707133+0000 osd.6 (osd.6) 162 : cluster 3 failed to encode map e227 with expected crc 2023-10-01T10:55:44.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:44 smithi173 bash[58816]: cluster 2023-10-01T10:55:43.709054+0000 osd.5 (osd.5) 199 : cluster 3 failed to encode map e227 with expected crc 2023-10-01T10:55:44.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:44 smithi173 bash[58816]: cluster 2023-10-01T10:55:43.711601+0000 osd.6 (osd.6) 163 : cluster 3 failed to encode map e227 with expected crc 2023-10-01T10:55:44.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:44 smithi173 bash[58816]: cluster 2023-10-01T10:55:43.713483+0000 osd.6 (osd.6) 164 : cluster 3 failed to encode map e227 with expected crc 2023-10-01T10:55:44.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:44 smithi173 bash[58816]: cluster 2023-10-01T10:55:44.703185+0000 mon.a (mon.0) 1435 : cluster 0 osdmap e228: 8 total, 8 up, 7 in 2023-10-01T10:55:44.912 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:44.936 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:44.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:44.938 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:55:45.039 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:45.054 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:44 smithi078 bash[75299]: cluster 2023-10-01T10:55:43.697226+0000 osd.7 (osd.7) 198 : cluster 3 failed to encode map e227 with expected crc 2023-10-01T10:55:45.054 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:44 smithi078 bash[75299]: cluster 2023-10-01T10:55:43.701097+0000 osd.6 (osd.6) 161 : cluster 3 failed to encode map e227 with expected crc 2023-10-01T10:55:45.055 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:44 smithi078 bash[75299]: cluster 2023-10-01T10:55:43.701618+0000 osd.7 (osd.7) 199 : cluster 3 failed to encode map e227 with expected crc 2023-10-01T10:55:45.055 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:44 smithi078 bash[75299]: cluster 2023-10-01T10:55:43.701849+0000 osd.5 (osd.5) 197 : cluster 3 failed to encode map e227 with expected crc 2023-10-01T10:55:45.055 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:44 smithi078 bash[75299]: audit 2023-10-01T10:55:43.702917+0000 mon.a (mon.0) 1433 : audit 1 from='client.? 172.21.15.78:0/3396088582' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-10-01T10:55:45.055 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:44 smithi078 bash[75299]: cluster 2023-10-01T10:55:43.703115+0000 mon.a (mon.0) 1434 : cluster 0 osdmap e227: 8 total, 8 up, 7 in 2023-10-01T10:55:45.055 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:44 smithi078 bash[75299]: cluster 2023-10-01T10:55:43.706836+0000 osd.5 (osd.5) 198 : cluster 3 failed to encode map e227 with expected crc 2023-10-01T10:55:45.055 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:44 smithi078 bash[75299]: cluster 2023-10-01T10:55:43.707133+0000 osd.6 (osd.6) 162 : cluster 3 failed to encode map e227 with expected crc 2023-10-01T10:55:45.055 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:44 smithi078 bash[75299]: cluster 2023-10-01T10:55:43.709054+0000 osd.5 (osd.5) 199 : cluster 3 failed to encode map e227 with expected crc 2023-10-01T10:55:45.055 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:44 smithi078 bash[75299]: cluster 2023-10-01T10:55:43.711601+0000 osd.6 (osd.6) 163 : cluster 3 failed to encode map e227 with expected crc 2023-10-01T10:55:45.056 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:44 smithi078 bash[75299]: cluster 2023-10-01T10:55:43.713483+0000 osd.6 (osd.6) 164 : cluster 3 failed to encode map e227 with expected crc 2023-10-01T10:55:45.056 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:44 smithi078 bash[75299]: cluster 2023-10-01T10:55:44.703185+0000 mon.a (mon.0) 1435 : cluster 0 osdmap e228: 8 total, 8 up, 7 in 2023-10-01T10:55:45.056 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:44 smithi078 bash[78274]: cluster 2023-10-01T10:55:43.697226+0000 osd.7 (osd.7) 198 : cluster 3 failed to encode map e227 with expected crc 2023-10-01T10:55:45.056 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:44 smithi078 bash[78274]: cluster 2023-10-01T10:55:43.701097+0000 osd.6 (osd.6) 161 : cluster 3 failed to encode map e227 with expected crc 2023-10-01T10:55:45.056 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:44 smithi078 bash[78274]: cluster 2023-10-01T10:55:43.701618+0000 osd.7 (osd.7) 199 : cluster 3 failed to encode map e227 with expected crc 2023-10-01T10:55:45.056 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:44 smithi078 bash[78274]: cluster 2023-10-01T10:55:43.701849+0000 osd.5 (osd.5) 197 : cluster 3 failed to encode map e227 with expected crc 2023-10-01T10:55:45.056 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:44 smithi078 bash[78274]: audit 2023-10-01T10:55:43.702917+0000 mon.a (mon.0) 1433 : audit 1 from='client.? 172.21.15.78:0/3396088582' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-10-01T10:55:45.056 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:44 smithi078 bash[78274]: cluster 2023-10-01T10:55:43.703115+0000 mon.a (mon.0) 1434 : cluster 0 osdmap e227: 8 total, 8 up, 7 in 2023-10-01T10:55:45.056 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:44 smithi078 bash[78274]: cluster 2023-10-01T10:55:43.706836+0000 osd.5 (osd.5) 198 : cluster 3 failed to encode map e227 with expected crc 2023-10-01T10:55:45.057 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:44 smithi078 bash[78274]: cluster 2023-10-01T10:55:43.707133+0000 osd.6 (osd.6) 162 : cluster 3 failed to encode map e227 with expected crc 2023-10-01T10:55:45.057 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:44 smithi078 bash[78274]: cluster 2023-10-01T10:55:43.709054+0000 osd.5 (osd.5) 199 : cluster 3 failed to encode map e227 with expected crc 2023-10-01T10:55:45.057 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:44 smithi078 bash[78274]: cluster 2023-10-01T10:55:43.711601+0000 osd.6 (osd.6) 163 : cluster 3 failed to encode map e227 with expected crc 2023-10-01T10:55:45.057 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:44 smithi078 bash[78274]: cluster 2023-10-01T10:55:43.713483+0000 osd.6 (osd.6) 164 : cluster 3 failed to encode map e227 with expected crc 2023-10-01T10:55:45.057 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:44 smithi078 bash[78274]: cluster 2023-10-01T10:55:44.703185+0000 mon.a (mon.0) 1435 : cluster 0 osdmap e228: 8 total, 8 up, 7 in 2023-10-01T10:55:45.065 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:45.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:45.071 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:45.171 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:45.196 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:45.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:45.199 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:55:45.300 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:45.322 INFO:teuthology.orchestra.run.smithi078.stdout:ERROR: (22) Invalid argument 2023-10-01T10:55:45.322 INFO:teuthology.orchestra.run.smithi078.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T10:55:45.326 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:45.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:45.329 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:55:45.429 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:45.458 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:45.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:45.460 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:55:45.561 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:45.593 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:45.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:45.596 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:45.696 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:45.748 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:45.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:45.760 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:55:45.861 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:45.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:45 smithi173 bash[58816]: cluster 2023-10-01T10:55:44.051060+0000 mgr.y (mgr.25072) 1147 : cluster 0 pgmap v1142: 99 pgs: 99 active+clean; 2.7 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail 2023-10-01T10:55:45.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:45 smithi173 bash[58816]: cluster 2023-10-01T10:55:44.701537+0000 osd.7 (osd.7) 200 : cluster 3 failed to encode map e228 with expected crc 2023-10-01T10:55:45.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:45 smithi173 bash[58816]: cluster 2023-10-01T10:55:44.704539+0000 osd.5 (osd.5) 200 : cluster 3 failed to encode map e228 with expected crc 2023-10-01T10:55:45.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:45 smithi173 bash[58816]: cluster 2023-10-01T10:55:44.706435+0000 osd.7 (osd.7) 201 : cluster 3 failed to encode map e228 with expected crc 2023-10-01T10:55:45.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:45 smithi173 bash[58816]: cluster 2023-10-01T10:55:44.707317+0000 osd.6 (osd.6) 165 : cluster 3 failed to encode map e228 with expected crc 2023-10-01T10:55:45.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:45 smithi173 bash[58816]: cluster 2023-10-01T10:55:44.708664+0000 osd.5 (osd.5) 201 : cluster 3 failed to encode map e228 with expected crc 2023-10-01T10:55:45.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:45 smithi173 bash[58816]: cluster 2023-10-01T10:55:44.708960+0000 osd.7 (osd.7) 202 : cluster 3 failed to encode map e228 with expected crc 2023-10-01T10:55:45.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:45 smithi173 bash[58816]: cluster 2023-10-01T10:55:44.710579+0000 osd.5 (osd.5) 202 : cluster 3 failed to encode map e228 with expected crc 2023-10-01T10:55:45.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:45 smithi173 bash[58816]: audit 2023-10-01T10:55:45.213724+0000 mon.a (mon.0) 1436 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:55:45.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:45 smithi173 bash[58816]: cluster 2023-10-01T10:55:45.704573+0000 mon.a (mon.0) 1437 : cluster 0 osdmap e229: 8 total, 8 up, 7 in 2023-10-01T10:55:45.870 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:45 smithi078 bash[78274]: cluster 2023-10-01T10:55:44.051060+0000 mgr.y (mgr.25072) 1147 : cluster 0 pgmap v1142: 99 pgs: 99 active+clean; 2.7 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail 2023-10-01T10:55:45.870 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:45 smithi078 bash[78274]: cluster 2023-10-01T10:55:44.701537+0000 osd.7 (osd.7) 200 : cluster 3 failed to encode map e228 with expected crc 2023-10-01T10:55:45.871 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:45 smithi078 bash[78274]: cluster 2023-10-01T10:55:44.704539+0000 osd.5 (osd.5) 200 : cluster 3 failed to encode map e228 with expected crc 2023-10-01T10:55:45.871 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:45 smithi078 bash[78274]: cluster 2023-10-01T10:55:44.706435+0000 osd.7 (osd.7) 201 : cluster 3 failed to encode map e228 with expected crc 2023-10-01T10:55:45.871 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:45 smithi078 bash[78274]: cluster 2023-10-01T10:55:44.707317+0000 osd.6 (osd.6) 165 : cluster 3 failed to encode map e228 with expected crc 2023-10-01T10:55:45.871 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:45 smithi078 bash[78274]: cluster 2023-10-01T10:55:44.708664+0000 osd.5 (osd.5) 201 : cluster 3 failed to encode map e228 with expected crc 2023-10-01T10:55:45.871 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:45 smithi078 bash[78274]: cluster 2023-10-01T10:55:44.708960+0000 osd.7 (osd.7) 202 : cluster 3 failed to encode map e228 with expected crc 2023-10-01T10:55:45.871 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:45 smithi078 bash[78274]: cluster 2023-10-01T10:55:44.710579+0000 osd.5 (osd.5) 202 : cluster 3 failed to encode map e228 with expected crc 2023-10-01T10:55:45.871 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:45 smithi078 bash[78274]: audit 2023-10-01T10:55:45.213724+0000 mon.a (mon.0) 1436 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:55:45.871 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:45 smithi078 bash[78274]: cluster 2023-10-01T10:55:45.704573+0000 mon.a (mon.0) 1437 : cluster 0 osdmap e229: 8 total, 8 up, 7 in 2023-10-01T10:55:45.872 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:45 smithi078 bash[75299]: cluster 2023-10-01T10:55:44.051060+0000 mgr.y (mgr.25072) 1147 : cluster 0 pgmap v1142: 99 pgs: 99 active+clean; 2.7 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail 2023-10-01T10:55:45.872 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:45 smithi078 bash[75299]: cluster 2023-10-01T10:55:44.701537+0000 osd.7 (osd.7) 200 : cluster 3 failed to encode map e228 with expected crc 2023-10-01T10:55:45.872 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:45 smithi078 bash[75299]: cluster 2023-10-01T10:55:44.704539+0000 osd.5 (osd.5) 200 : cluster 3 failed to encode map e228 with expected crc 2023-10-01T10:55:45.872 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:45 smithi078 bash[75299]: cluster 2023-10-01T10:55:44.706435+0000 osd.7 (osd.7) 201 : cluster 3 failed to encode map e228 with expected crc 2023-10-01T10:55:45.872 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:45 smithi078 bash[75299]: cluster 2023-10-01T10:55:44.707317+0000 osd.6 (osd.6) 165 : cluster 3 failed to encode map e228 with expected crc 2023-10-01T10:55:45.872 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:45 smithi078 bash[75299]: cluster 2023-10-01T10:55:44.708664+0000 osd.5 (osd.5) 201 : cluster 3 failed to encode map e228 with expected crc 2023-10-01T10:55:45.872 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:45 smithi078 bash[75299]: cluster 2023-10-01T10:55:44.708960+0000 osd.7 (osd.7) 202 : cluster 3 failed to encode map e228 with expected crc 2023-10-01T10:55:45.872 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:45 smithi078 bash[75299]: cluster 2023-10-01T10:55:44.710579+0000 osd.5 (osd.5) 202 : cluster 3 failed to encode map e228 with expected crc 2023-10-01T10:55:45.872 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:45 smithi078 bash[75299]: audit 2023-10-01T10:55:45.213724+0000 mon.a (mon.0) 1436 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:55:45.872 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:45 smithi078 bash[75299]: cluster 2023-10-01T10:55:45.704573+0000 mon.a (mon.0) 1437 : cluster 0 osdmap e229: 8 total, 8 up, 7 in 2023-10-01T10:55:45.914 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:45.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:45.916 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:55:46.016 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:46.040 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:46.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:46.042 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:46.143 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:46.166 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:46.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:46.167 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:55:46.268 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:46.298 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:46.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:46.301 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:55:46.402 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:46.426 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:46.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:46.429 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:55:46.530 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:46.552 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:46.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:46.554 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:55:46.655 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:46.687 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:46.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:46.690 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:55:46.730 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:46 smithi078 bash[75299]: cluster 2023-10-01T10:55:45.703981+0000 osd.4 (osd.4) 3 : cluster 0 11.14 starting backfill to osd.2 from (0'0,0'0] MAX to 225'11953 2023-10-01T10:55:46.730 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:46 smithi078 bash[75299]: cluster 2023-10-01T10:55:45.705675+0000 osd.2 (osd.2) 6 : cluster 0 11.18 starting backfill to osd.5 from (0'0,0'0] MAX to 225'11825 2023-10-01T10:55:46.731 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:46 smithi078 bash[75299]: cluster 2023-10-01T10:55:45.705704+0000 osd.5 (osd.5) 203 : cluster 3 failed to encode map e229 with expected crc 2023-10-01T10:55:46.731 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:46 smithi078 bash[75299]: cluster 2023-10-01T10:55:45.706059+0000 osd.3 (osd.3) 20 : cluster 0 11.1 starting backfill to osd.2 from (0'0,0'0] MAX to 225'11880 2023-10-01T10:55:46.731 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:46 smithi078 bash[75299]: cluster 2023-10-01T10:55:45.706490+0000 osd.2 (osd.2) 7 : cluster 0 11.d starting backfill to osd.1 from (0'0,0'0] MAX to 225'13172 2023-10-01T10:55:46.731 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:46 smithi078 bash[75299]: cluster 2023-10-01T10:55:45.706495+0000 osd.3 (osd.3) 21 : cluster 0 11.8 starting backfill to osd.0 from (0'0,0'0] MAX to 225'11877 2023-10-01T10:55:46.731 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:46 smithi078 bash[75299]: cluster 2023-10-01T10:55:45.706631+0000 osd.3 (osd.3) 22 : cluster 0 11.a starting backfill to osd.1 from (0'0,0'0] MAX to 225'13364 2023-10-01T10:55:46.731 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:46 smithi078 bash[75299]: cluster 2023-10-01T10:55:45.707351+0000 osd.7 (osd.7) 203 : cluster 3 failed to encode map e229 with expected crc 2023-10-01T10:55:46.731 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:46 smithi078 bash[75299]: cluster 2023-10-01T10:55:45.709506+0000 osd.3 (osd.3) 23 : cluster 0 11.9 starting backfill to osd.1 from (0'0,0'0] MAX to 225'12091 2023-10-01T10:55:46.731 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:46 smithi078 bash[75299]: cluster 2023-10-01T10:55:45.711076+0000 osd.7 (osd.7) 204 : cluster 3 failed to encode map e229 with expected crc 2023-10-01T10:55:46.731 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:46 smithi078 bash[75299]: cluster 2023-10-01T10:55:45.712119+0000 osd.1 (osd.1) 10 : cluster 0 11.3 starting backfill to osd.5 from (0'0,0'0] MAX to 225'12008 2023-10-01T10:55:46.731 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:46 smithi078 bash[75299]: cluster 2023-10-01T10:55:45.714268+0000 osd.7 (osd.7) 205 : cluster 0 11.17 starting backfill to osd.1 from (0'0,0'0] MAX to 225'11801 2023-10-01T10:55:46.731 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:46 smithi078 bash[75299]: cluster 2023-10-01T10:55:45.714868+0000 osd.0 (osd.0) 17 : cluster 0 11.10 starting backfill to osd.5 from (0'0,0'0] MAX to 225'11951 2023-10-01T10:55:46.732 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:46 smithi078 bash[75299]: cluster 2023-10-01T10:55:45.716014+0000 osd.5 (osd.5) 204 : cluster 0 11.5 starting backfill to osd.3 from (0'0,0'0] MAX to 225'11997 2023-10-01T10:55:46.732 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:46 smithi078 bash[75299]: cluster 2023-10-01T10:55:45.716185+0000 osd.2 (osd.2) 8 : cluster 0 11.4 starting backfill to osd.1 from (0'0,0'0] MAX to 225'11856 2023-10-01T10:55:46.732 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:46 smithi078 bash[75299]: cluster 2023-10-01T10:55:45.717251+0000 osd.6 (osd.6) 166 : cluster 3 failed to encode map e229 with expected crc 2023-10-01T10:55:46.732 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:46 smithi078 bash[75299]: cluster 2023-10-01T10:55:46.153684+0000 osd.2 (osd.2) 9 : cluster 0 11.11 scrub starts 2023-10-01T10:55:46.732 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:46 smithi078 bash[75299]: cluster 2023-10-01T10:55:46.163068+0000 osd.2 (osd.2) 10 : cluster 0 11.11 scrub ok 2023-10-01T10:55:46.791 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:46.824 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:46.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:46.827 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:46.928 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:46.939 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:46 smithi173 bash[58816]: cluster 2023-10-01T10:55:45.703981+0000 osd.4 (osd.4) 3 : cluster 0 11.14 starting backfill to osd.2 from (0'0,0'0] MAX to 225'11953 2023-10-01T10:55:46.939 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:46 smithi173 bash[58816]: cluster 2023-10-01T10:55:45.705675+0000 osd.2 (osd.2) 6 : cluster 0 11.18 starting backfill to osd.5 from (0'0,0'0] MAX to 225'11825 2023-10-01T10:55:46.940 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:46 smithi173 bash[58816]: cluster 2023-10-01T10:55:45.705704+0000 osd.5 (osd.5) 203 : cluster 3 failed to encode map e229 with expected crc 2023-10-01T10:55:46.940 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:46 smithi173 bash[58816]: cluster 2023-10-01T10:55:45.706059+0000 osd.3 (osd.3) 20 : cluster 0 11.1 starting backfill to osd.2 from (0'0,0'0] MAX to 225'11880 2023-10-01T10:55:46.940 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:46 smithi173 bash[58816]: cluster 2023-10-01T10:55:45.706490+0000 osd.2 (osd.2) 7 : cluster 0 11.d starting backfill to osd.1 from (0'0,0'0] MAX to 225'13172 2023-10-01T10:55:46.940 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:46 smithi173 bash[58816]: cluster 2023-10-01T10:55:45.706495+0000 osd.3 (osd.3) 21 : cluster 0 11.8 starting backfill to osd.0 from (0'0,0'0] MAX to 225'11877 2023-10-01T10:55:46.940 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:46 smithi173 bash[58816]: cluster 2023-10-01T10:55:45.706631+0000 osd.3 (osd.3) 22 : cluster 0 11.a starting backfill to osd.1 from (0'0,0'0] MAX to 225'13364 2023-10-01T10:55:46.940 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:46 smithi173 bash[58816]: cluster 2023-10-01T10:55:45.707351+0000 osd.7 (osd.7) 203 : cluster 3 failed to encode map e229 with expected crc 2023-10-01T10:55:46.940 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:46 smithi173 bash[58816]: cluster 2023-10-01T10:55:45.709506+0000 osd.3 (osd.3) 23 : cluster 0 11.9 starting backfill to osd.1 from (0'0,0'0] MAX to 225'12091 2023-10-01T10:55:46.940 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:46 smithi173 bash[58816]: cluster 2023-10-01T10:55:45.711076+0000 osd.7 (osd.7) 204 : cluster 3 failed to encode map e229 with expected crc 2023-10-01T10:55:46.940 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:46 smithi173 bash[58816]: cluster 2023-10-01T10:55:45.712119+0000 osd.1 (osd.1) 10 : cluster 0 11.3 starting backfill to osd.5 from (0'0,0'0] MAX to 225'12008 2023-10-01T10:55:46.940 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:46 smithi173 bash[58816]: cluster 2023-10-01T10:55:45.714268+0000 osd.7 (osd.7) 205 : cluster 0 11.17 starting backfill to osd.1 from (0'0,0'0] MAX to 225'11801 2023-10-01T10:55:46.941 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:46 smithi173 bash[58816]: cluster 2023-10-01T10:55:45.714868+0000 osd.0 (osd.0) 17 : cluster 0 11.10 starting backfill to osd.5 from (0'0,0'0] MAX to 225'11951 2023-10-01T10:55:46.941 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:46 smithi173 bash[58816]: cluster 2023-10-01T10:55:45.716014+0000 osd.5 (osd.5) 204 : cluster 0 11.5 starting backfill to osd.3 from (0'0,0'0] MAX to 225'11997 2023-10-01T10:55:46.941 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:46 smithi173 bash[58816]: cluster 2023-10-01T10:55:45.716185+0000 osd.2 (osd.2) 8 : cluster 0 11.4 starting backfill to osd.1 from (0'0,0'0] MAX to 225'11856 2023-10-01T10:55:46.941 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:46 smithi173 bash[58816]: cluster 2023-10-01T10:55:45.717251+0000 osd.6 (osd.6) 166 : cluster 3 failed to encode map e229 with expected crc 2023-10-01T10:55:46.941 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:46 smithi173 bash[58816]: cluster 2023-10-01T10:55:46.153684+0000 osd.2 (osd.2) 9 : cluster 0 11.11 scrub starts 2023-10-01T10:55:46.941 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:46 smithi173 bash[58816]: cluster 2023-10-01T10:55:46.163068+0000 osd.2 (osd.2) 10 : cluster 0 11.11 scrub ok 2023-10-01T10:55:46.956 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:46.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:46.958 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:55:47.059 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:47.080 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:47.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:47.085 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:55:47.098 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:46 smithi078 bash[78274]: cluster 2023-10-01T10:55:45.703981+0000 osd.4 (osd.4) 3 : cluster 0 11.14 starting backfill to osd.2 from (0'0,0'0] MAX to 225'11953 2023-10-01T10:55:47.098 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:46 smithi078 bash[78274]: cluster 2023-10-01T10:55:45.705675+0000 osd.2 (osd.2) 6 : cluster 0 11.18 starting backfill to osd.5 from (0'0,0'0] MAX to 225'11825 2023-10-01T10:55:47.098 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:46 smithi078 bash[78274]: cluster 2023-10-01T10:55:45.705704+0000 osd.5 (osd.5) 203 : cluster 3 failed to encode map e229 with expected crc 2023-10-01T10:55:47.099 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:46 smithi078 bash[78274]: cluster 2023-10-01T10:55:45.706059+0000 osd.3 (osd.3) 20 : cluster 0 11.1 starting backfill to osd.2 from (0'0,0'0] MAX to 225'11880 2023-10-01T10:55:47.099 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:46 smithi078 bash[78274]: cluster 2023-10-01T10:55:45.706490+0000 osd.2 (osd.2) 7 : cluster 0 11.d starting backfill to osd.1 from (0'0,0'0] MAX to 225'13172 2023-10-01T10:55:47.099 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:46 smithi078 bash[78274]: cluster 2023-10-01T10:55:45.706495+0000 osd.3 (osd.3) 21 : cluster 0 11.8 starting backfill to osd.0 from (0'0,0'0] MAX to 225'11877 2023-10-01T10:55:47.099 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:46 smithi078 bash[78274]: cluster 2023-10-01T10:55:45.706631+0000 osd.3 (osd.3) 22 : cluster 0 11.a starting backfill to osd.1 from (0'0,0'0] MAX to 225'13364 2023-10-01T10:55:47.099 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:46 smithi078 bash[78274]: cluster 2023-10-01T10:55:45.707351+0000 osd.7 (osd.7) 203 : cluster 3 failed to encode map e229 with expected crc 2023-10-01T10:55:47.099 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:46 smithi078 bash[78274]: cluster 2023-10-01T10:55:45.709506+0000 osd.3 (osd.3) 23 : cluster 0 11.9 starting backfill to osd.1 from (0'0,0'0] MAX to 225'12091 2023-10-01T10:55:47.099 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:46 smithi078 bash[78274]: cluster 2023-10-01T10:55:45.711076+0000 osd.7 (osd.7) 204 : cluster 3 failed to encode map e229 with expected crc 2023-10-01T10:55:47.099 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:46 smithi078 bash[78274]: cluster 2023-10-01T10:55:45.712119+0000 osd.1 (osd.1) 10 : cluster 0 11.3 starting backfill to osd.5 from (0'0,0'0] MAX to 225'12008 2023-10-01T10:55:47.099 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:46 smithi078 bash[78274]: cluster 2023-10-01T10:55:45.714268+0000 osd.7 (osd.7) 205 : cluster 0 11.17 starting backfill to osd.1 from (0'0,0'0] MAX to 225'11801 2023-10-01T10:55:47.099 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:46 smithi078 bash[78274]: cluster 2023-10-01T10:55:45.714868+0000 osd.0 (osd.0) 17 : cluster 0 11.10 starting backfill to osd.5 from (0'0,0'0] MAX to 225'11951 2023-10-01T10:55:47.099 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:46 smithi078 bash[78274]: cluster 2023-10-01T10:55:45.716014+0000 osd.5 (osd.5) 204 : cluster 0 11.5 starting backfill to osd.3 from (0'0,0'0] MAX to 225'11997 2023-10-01T10:55:47.099 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:46 smithi078 bash[78274]: cluster 2023-10-01T10:55:45.716185+0000 osd.2 (osd.2) 8 : cluster 0 11.4 starting backfill to osd.1 from (0'0,0'0] MAX to 225'11856 2023-10-01T10:55:47.100 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:46 smithi078 bash[78274]: cluster 2023-10-01T10:55:45.717251+0000 osd.6 (osd.6) 166 : cluster 3 failed to encode map e229 with expected crc 2023-10-01T10:55:47.100 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:46 smithi078 bash[78274]: cluster 2023-10-01T10:55:46.153684+0000 osd.2 (osd.2) 9 : cluster 0 11.11 scrub starts 2023-10-01T10:55:47.100 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:46 smithi078 bash[78274]: cluster 2023-10-01T10:55:46.163068+0000 osd.2 (osd.2) 10 : cluster 0 11.11 scrub ok 2023-10-01T10:55:47.185 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:47.215 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:47.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:47.217 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:55:47.318 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:47.347 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:47.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:47.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:55:47.450 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:47.485 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:47.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:47.489 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:55:47.589 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:47.620 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:47.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:47.623 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:55:47.724 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:47.763 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:47.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:47.767 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:55:47.868 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:47.895 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:47.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:47.898 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:55:47.999 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:48.012 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:47 smithi173 bash[58816]: cluster 2023-10-01T10:55:46.051512+0000 mgr.y (mgr.25072) 1148 : cluster 0 pgmap v1145: 99 pgs: 15 peering, 84 active+clean; 2.7 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail 2023-10-01T10:55:48.012 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:47 smithi173 bash[58816]: cluster 2023-10-01T10:55:46.720622+0000 mon.a (mon.0) 1438 : cluster 3 Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-01T10:55:48.012 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:47 smithi173 bash[58816]: cluster 2023-10-01T10:55:46.729476+0000 osd.5 (osd.5) 205 : cluster 3 failed to encode map e230 with expected crc 2023-10-01T10:55:48.012 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:47 smithi173 bash[58816]: cluster 2023-10-01T10:55:46.730629+0000 osd.7 (osd.7) 206 : cluster 3 failed to encode map e230 with expected crc 2023-10-01T10:55:48.012 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:47 smithi173 bash[58816]: cluster 2023-10-01T10:55:46.731951+0000 mon.a (mon.0) 1439 : cluster 0 osdmap e230: 8 total, 8 up, 7 in 2023-10-01T10:55:48.012 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:47 smithi173 bash[58816]: cluster 2023-10-01T10:55:46.735039+0000 osd.5 (osd.5) 206 : cluster 3 failed to encode map e230 with expected crc 2023-10-01T10:55:48.012 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:47 smithi173 bash[58816]: cluster 2023-10-01T10:55:46.736064+0000 osd.6 (osd.6) 167 : cluster 3 failed to encode map e230 with expected crc 2023-10-01T10:55:48.012 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:47 smithi173 bash[58816]: cluster 2023-10-01T10:55:46.738023+0000 osd.5 (osd.5) 207 : cluster 3 failed to encode map e230 with expected crc 2023-10-01T10:55:48.012 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:47 smithi173 bash[58816]: cluster 2023-10-01T10:55:46.738106+0000 osd.7 (osd.7) 207 : cluster 3 failed to encode map e230 with expected crc 2023-10-01T10:55:48.012 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:47 smithi173 bash[58816]: cluster 2023-10-01T10:55:46.738318+0000 osd.6 (osd.6) 168 : cluster 3 failed to encode map e230 with expected crc 2023-10-01T10:55:48.013 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:47 smithi173 bash[58816]: cluster 2023-10-01T10:55:46.739629+0000 osd.7 (osd.7) 208 : cluster 3 failed to encode map e230 with expected crc 2023-10-01T10:55:48.013 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:47 smithi173 bash[58816]: cluster 2023-10-01T10:55:46.740377+0000 osd.5 (osd.5) 208 : cluster 3 failed to encode map e230 with expected crc 2023-10-01T10:55:48.013 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:47 smithi173 bash[58816]: audit 2023-10-01T10:55:47.447227+0000 mon.a (mon.0) 1440 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:55:48.013 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:47 smithi173 bash[58816]: audit 2023-10-01T10:55:47.448996+0000 mon.a (mon.0) 1441 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:55:48.029 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:48.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:48.032 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:55:48.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:47 smithi078 bash[75299]: cluster 2023-10-01T10:55:46.051512+0000 mgr.y (mgr.25072) 1148 : cluster 0 pgmap v1145: 99 pgs: 15 peering, 84 active+clean; 2.7 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail 2023-10-01T10:55:48.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:47 smithi078 bash[75299]: cluster 2023-10-01T10:55:46.720622+0000 mon.a (mon.0) 1438 : cluster 3 Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-01T10:55:48.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:47 smithi078 bash[75299]: cluster 2023-10-01T10:55:46.729476+0000 osd.5 (osd.5) 205 : cluster 3 failed to encode map e230 with expected crc 2023-10-01T10:55:48.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:47 smithi078 bash[75299]: cluster 2023-10-01T10:55:46.730629+0000 osd.7 (osd.7) 206 : cluster 3 failed to encode map e230 with expected crc 2023-10-01T10:55:48.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:47 smithi078 bash[75299]: cluster 2023-10-01T10:55:46.731951+0000 mon.a (mon.0) 1439 : cluster 0 osdmap e230: 8 total, 8 up, 7 in 2023-10-01T10:55:48.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:47 smithi078 bash[75299]: cluster 2023-10-01T10:55:46.735039+0000 osd.5 (osd.5) 206 : cluster 3 failed to encode map e230 with expected crc 2023-10-01T10:55:48.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:47 smithi078 bash[75299]: cluster 2023-10-01T10:55:46.736064+0000 osd.6 (osd.6) 167 : cluster 3 failed to encode map e230 with expected crc 2023-10-01T10:55:48.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:47 smithi078 bash[75299]: cluster 2023-10-01T10:55:46.738023+0000 osd.5 (osd.5) 207 : cluster 3 failed to encode map e230 with expected crc 2023-10-01T10:55:48.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:47 smithi078 bash[75299]: cluster 2023-10-01T10:55:46.738106+0000 osd.7 (osd.7) 207 : cluster 3 failed to encode map e230 with expected crc 2023-10-01T10:55:48.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:47 smithi078 bash[75299]: cluster 2023-10-01T10:55:46.738318+0000 osd.6 (osd.6) 168 : cluster 3 failed to encode map e230 with expected crc 2023-10-01T10:55:48.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:47 smithi078 bash[75299]: cluster 2023-10-01T10:55:46.739629+0000 osd.7 (osd.7) 208 : cluster 3 failed to encode map e230 with expected crc 2023-10-01T10:55:48.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:47 smithi078 bash[75299]: cluster 2023-10-01T10:55:46.740377+0000 osd.5 (osd.5) 208 : cluster 3 failed to encode map e230 with expected crc 2023-10-01T10:55:48.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:47 smithi078 bash[75299]: audit 2023-10-01T10:55:47.447227+0000 mon.a (mon.0) 1440 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:55:48.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:47 smithi078 bash[75299]: audit 2023-10-01T10:55:47.448996+0000 mon.a (mon.0) 1441 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:55:48.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:47 smithi078 bash[78274]: cluster 2023-10-01T10:55:46.051512+0000 mgr.y (mgr.25072) 1148 : cluster 0 pgmap v1145: 99 pgs: 15 peering, 84 active+clean; 2.7 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail 2023-10-01T10:55:48.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:47 smithi078 bash[78274]: cluster 2023-10-01T10:55:46.720622+0000 mon.a (mon.0) 1438 : cluster 3 Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-01T10:55:48.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:47 smithi078 bash[78274]: cluster 2023-10-01T10:55:46.729476+0000 osd.5 (osd.5) 205 : cluster 3 failed to encode map e230 with expected crc 2023-10-01T10:55:48.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:47 smithi078 bash[78274]: cluster 2023-10-01T10:55:46.730629+0000 osd.7 (osd.7) 206 : cluster 3 failed to encode map e230 with expected crc 2023-10-01T10:55:48.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:47 smithi078 bash[78274]: cluster 2023-10-01T10:55:46.731951+0000 mon.a (mon.0) 1439 : cluster 0 osdmap e230: 8 total, 8 up, 7 in 2023-10-01T10:55:48.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:47 smithi078 bash[78274]: cluster 2023-10-01T10:55:46.735039+0000 osd.5 (osd.5) 206 : cluster 3 failed to encode map e230 with expected crc 2023-10-01T10:55:48.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:47 smithi078 bash[78274]: cluster 2023-10-01T10:55:46.736064+0000 osd.6 (osd.6) 167 : cluster 3 failed to encode map e230 with expected crc 2023-10-01T10:55:48.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:47 smithi078 bash[78274]: cluster 2023-10-01T10:55:46.738023+0000 osd.5 (osd.5) 207 : cluster 3 failed to encode map e230 with expected crc 2023-10-01T10:55:48.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:47 smithi078 bash[78274]: cluster 2023-10-01T10:55:46.738106+0000 osd.7 (osd.7) 207 : cluster 3 failed to encode map e230 with expected crc 2023-10-01T10:55:48.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:47 smithi078 bash[78274]: cluster 2023-10-01T10:55:46.738318+0000 osd.6 (osd.6) 168 : cluster 3 failed to encode map e230 with expected crc 2023-10-01T10:55:48.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:47 smithi078 bash[78274]: cluster 2023-10-01T10:55:46.739629+0000 osd.7 (osd.7) 208 : cluster 3 failed to encode map e230 with expected crc 2023-10-01T10:55:48.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:47 smithi078 bash[78274]: cluster 2023-10-01T10:55:46.740377+0000 osd.5 (osd.5) 208 : cluster 3 failed to encode map e230 with expected crc 2023-10-01T10:55:48.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:47 smithi078 bash[78274]: audit 2023-10-01T10:55:47.447227+0000 mon.a (mon.0) 1440 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:55:48.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:47 smithi078 bash[78274]: audit 2023-10-01T10:55:47.448996+0000 mon.a (mon.0) 1441 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:55:48.133 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:48.161 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:48.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:48.164 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:55:48.265 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:48.300 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:48.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:48.303 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:55:48.403 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:48.426 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:48.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:48.428 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:55:48.529 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:48.560 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:48.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:48.563 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:55:48.663 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:48.694 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:48.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:48.697 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:55:48.798 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:48.831 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:48.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:48.834 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:55:48.935 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:48.960 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:48.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:48.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:55:49.063 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:49.074 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:48 smithi078 bash[78274]: cluster 2023-10-01T10:55:47.740418+0000 osd.5 (osd.5) 209 : cluster 3 failed to encode map e231 with expected crc 2023-10-01T10:55:49.074 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:48 smithi078 bash[78274]: cluster 2023-10-01T10:55:47.740848+0000 osd.7 (osd.7) 209 : cluster 3 failed to encode map e231 with expected crc 2023-10-01T10:55:49.074 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:48 smithi078 bash[78274]: cluster 2023-10-01T10:55:47.741852+0000 mon.a (mon.0) 1442 : cluster 0 osdmap e231: 8 total, 8 up, 7 in 2023-10-01T10:55:49.074 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:48 smithi078 bash[78274]: cluster 2023-10-01T10:55:47.742972+0000 osd.5 (osd.5) 210 : cluster 3 failed to encode map e231 with expected crc 2023-10-01T10:55:49.074 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:48 smithi078 bash[78274]: cluster 2023-10-01T10:55:47.743803+0000 osd.7 (osd.7) 210 : cluster 3 failed to encode map e231 with expected crc 2023-10-01T10:55:49.074 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:48 smithi078 bash[78274]: cluster 2023-10-01T10:55:47.774646+0000 osd.6 (osd.6) 169 : cluster 3 failed to encode map e231 with expected crc 2023-10-01T10:55:49.075 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:48 smithi078 bash[75299]: cluster 2023-10-01T10:55:47.740418+0000 osd.5 (osd.5) 209 : cluster 3 failed to encode map e231 with expected crc 2023-10-01T10:55:49.075 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:48 smithi078 bash[75299]: cluster 2023-10-01T10:55:47.740848+0000 osd.7 (osd.7) 209 : cluster 3 failed to encode map e231 with expected crc 2023-10-01T10:55:49.075 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:48 smithi078 bash[75299]: cluster 2023-10-01T10:55:47.741852+0000 mon.a (mon.0) 1442 : cluster 0 osdmap e231: 8 total, 8 up, 7 in 2023-10-01T10:55:49.075 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:48 smithi078 bash[75299]: cluster 2023-10-01T10:55:47.742972+0000 osd.5 (osd.5) 210 : cluster 3 failed to encode map e231 with expected crc 2023-10-01T10:55:49.075 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:48 smithi078 bash[75299]: cluster 2023-10-01T10:55:47.743803+0000 osd.7 (osd.7) 210 : cluster 3 failed to encode map e231 with expected crc 2023-10-01T10:55:49.075 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:48 smithi078 bash[75299]: cluster 2023-10-01T10:55:47.774646+0000 osd.6 (osd.6) 169 : cluster 3 failed to encode map e231 with expected crc 2023-10-01T10:55:49.087 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:49.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:49.090 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:55:49.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:48 smithi173 bash[58816]: cluster 2023-10-01T10:55:47.740418+0000 osd.5 (osd.5) 209 : cluster 3 failed to encode map e231 with expected crc 2023-10-01T10:55:49.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:48 smithi173 bash[58816]: cluster 2023-10-01T10:55:47.740848+0000 osd.7 (osd.7) 209 : cluster 3 failed to encode map e231 with expected crc 2023-10-01T10:55:49.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:48 smithi173 bash[58816]: cluster 2023-10-01T10:55:47.741852+0000 mon.a (mon.0) 1442 : cluster 0 osdmap e231: 8 total, 8 up, 7 in 2023-10-01T10:55:49.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:48 smithi173 bash[58816]: cluster 2023-10-01T10:55:47.742972+0000 osd.5 (osd.5) 210 : cluster 3 failed to encode map e231 with expected crc 2023-10-01T10:55:49.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:48 smithi173 bash[58816]: cluster 2023-10-01T10:55:47.743803+0000 osd.7 (osd.7) 210 : cluster 3 failed to encode map e231 with expected crc 2023-10-01T10:55:49.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:48 smithi173 bash[58816]: cluster 2023-10-01T10:55:47.774646+0000 osd.6 (osd.6) 169 : cluster 3 failed to encode map e231 with expected crc 2023-10-01T10:55:49.180 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-01T10:55:49.191 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:49.253 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:49.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:49.256 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:55:49.357 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:49.376 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:49.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:49.379 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:55:49.479 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:49.508 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:49.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:49.510 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:55:49.611 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:49.633 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:49.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:49.635 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:55:49.736 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:49.761 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:49.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:49.763 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:55:49.863 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:49.896 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:49.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:49.900 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:55:50.000 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:50.011 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:49 smithi173 bash[58816]: cluster 2023-10-01T10:55:47.863626+0000 osd.5 (osd.5) 211 : cluster 0 11.3 scrub starts 2023-10-01T10:55:50.011 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:49 smithi173 bash[58816]: cluster 2023-10-01T10:55:47.875768+0000 osd.5 (osd.5) 212 : cluster 0 11.3 scrub ok 2023-10-01T10:55:50.012 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:49 smithi173 bash[58816]: cluster 2023-10-01T10:55:48.052196+0000 mgr.y (mgr.25072) 1149 : cluster 0 pgmap v1148: 99 pgs: 1 active+clean+remapped, 19 peering, 79 active+clean; 6.1 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 53 MiB/s wr, 1.69k op/s 2023-10-01T10:55:50.033 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:50.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:50.036 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:55:50.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:49 smithi078 bash[78274]: cluster 2023-10-01T10:55:47.863626+0000 osd.5 (osd.5) 211 : cluster 0 11.3 scrub starts 2023-10-01T10:55:50.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:49 smithi078 bash[78274]: cluster 2023-10-01T10:55:47.875768+0000 osd.5 (osd.5) 212 : cluster 0 11.3 scrub ok 2023-10-01T10:55:50.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:49 smithi078 bash[78274]: cluster 2023-10-01T10:55:48.052196+0000 mgr.y (mgr.25072) 1149 : cluster 0 pgmap v1148: 99 pgs: 1 active+clean+remapped, 19 peering, 79 active+clean; 6.1 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 53 MiB/s wr, 1.69k op/s 2023-10-01T10:55:50.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:49 smithi078 bash[75299]: cluster 2023-10-01T10:55:47.863626+0000 osd.5 (osd.5) 211 : cluster 0 11.3 scrub starts 2023-10-01T10:55:50.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:49 smithi078 bash[75299]: cluster 2023-10-01T10:55:47.875768+0000 osd.5 (osd.5) 212 : cluster 0 11.3 scrub ok 2023-10-01T10:55:50.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:49 smithi078 bash[75299]: cluster 2023-10-01T10:55:48.052196+0000 mgr.y (mgr.25072) 1149 : cluster 0 pgmap v1148: 99 pgs: 1 active+clean+remapped, 19 peering, 79 active+clean; 6.1 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 53 MiB/s wr, 1.69k op/s 2023-10-01T10:55:50.137 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:50.157 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:50.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:50.160 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:55:50.261 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:50.283 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:50.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:50.285 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:55:50.386 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:50.413 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:50.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:50.416 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:55:50.517 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:50.551 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:50.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:50.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:50.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:55:50 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:55:50] "GET /metrics HTTP/1.1" 200 36334 "" "Prometheus/2.33.4" 2023-10-01T10:55:50.655 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:50.684 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:50.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:50.686 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:55:50.787 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:50.809 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:50.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:50.811 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:55:50.912 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:50.919 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:50 smithi078 bash[75299]: cluster 2023-10-01T10:55:50.210726+0000 osd.2 (osd.2) 11 : cluster 0 11.12 scrub starts 2023-10-01T10:55:50.919 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:50 smithi078 bash[75299]: cluster 2023-10-01T10:55:50.228828+0000 osd.2 (osd.2) 12 : cluster 0 11.12 scrub ok 2023-10-01T10:55:50.920 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:50 smithi078 bash[78274]: cluster 2023-10-01T10:55:50.210726+0000 osd.2 (osd.2) 11 : cluster 0 11.12 scrub starts 2023-10-01T10:55:50.920 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:50 smithi078 bash[78274]: cluster 2023-10-01T10:55:50.228828+0000 osd.2 (osd.2) 12 : cluster 0 11.12 scrub ok 2023-10-01T10:55:50.934 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:50.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:50.936 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:55:51.037 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:51.068 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:51.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:51.070 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:55:51.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:50 smithi173 bash[58816]: cluster 2023-10-01T10:55:50.210726+0000 osd.2 (osd.2) 11 : cluster 0 11.12 scrub starts 2023-10-01T10:55:51.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:50 smithi173 bash[58816]: cluster 2023-10-01T10:55:50.228828+0000 osd.2 (osd.2) 12 : cluster 0 11.12 scrub ok 2023-10-01T10:55:51.171 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:51.197 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:51.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:51.200 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:55:51.301 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:51.327 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:51.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:51.329 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:55:51.430 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:51.461 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:51.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:51.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:51.565 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:51.591 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:51.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:51.594 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:51.695 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:51.724 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:51.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:51.727 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:55:51.827 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:51.853 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:51.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:51.856 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:55:51.957 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:51.982 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:51.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:51.985 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:55:52.086 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:52.090 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:51 smithi078 bash[78274]: cluster 2023-10-01T10:55:50.053493+0000 mgr.y (mgr.25072) 1150 : cluster 0 pgmap v1149: 99 pgs: 4 peering, 95 active+clean; 2.6 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail 2023-10-01T10:55:52.090 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:51 smithi078 bash[78274]: cluster 2023-10-01T10:55:50.749717+0000 mon.a (mon.0) 1443 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-01T10:55:52.090 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:51 smithi078 bash[78274]: audit 2023-10-01T10:55:51.400281+0000 mon.a (mon.0) 1444 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:55:52.091 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:51 smithi078 bash[78274]: audit 2023-10-01T10:55:51.408679+0000 mon.a (mon.0) 1445 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:55:52.091 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:51 smithi078 bash[75299]: cluster 2023-10-01T10:55:50.053493+0000 mgr.y (mgr.25072) 1150 : cluster 0 pgmap v1149: 99 pgs: 4 peering, 95 active+clean; 2.6 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail 2023-10-01T10:55:52.091 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:51 smithi078 bash[75299]: cluster 2023-10-01T10:55:50.749717+0000 mon.a (mon.0) 1443 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-01T10:55:52.091 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:51 smithi078 bash[75299]: audit 2023-10-01T10:55:51.400281+0000 mon.a (mon.0) 1444 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:55:52.091 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:51 smithi078 bash[75299]: audit 2023-10-01T10:55:51.408679+0000 mon.a (mon.0) 1445 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:55:52.098 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:51 smithi173 bash[58816]: cluster 2023-10-01T10:55:50.053493+0000 mgr.y (mgr.25072) 1150 : cluster 0 pgmap v1149: 99 pgs: 4 peering, 95 active+clean; 2.6 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail 2023-10-01T10:55:52.098 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:51 smithi173 bash[58816]: cluster 2023-10-01T10:55:50.749717+0000 mon.a (mon.0) 1443 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-01T10:55:52.098 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:51 smithi173 bash[58816]: audit 2023-10-01T10:55:51.400281+0000 mon.a (mon.0) 1444 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:55:52.098 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:51 smithi173 bash[58816]: audit 2023-10-01T10:55:51.408679+0000 mon.a (mon.0) 1445 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:55:52.116 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:52.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:52.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:52.220 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:52.248 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:52.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:52.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:55:52.351 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:52.378 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:52.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:52.381 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:55:52.481 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:52.513 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:52.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:52.516 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:55:52.617 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:52.644 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:52.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:52.647 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:55:52.748 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:52.781 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:52.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:52.784 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:55:52.787 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 7] out_osds: [6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T10:55:52.787 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T10:55:52.787 INFO:tasks.thrashosds.thrasher:Adding osd 6 2023-10-01T10:55:52.788 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd in 6 2023-10-01T10:55:52.885 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:52.916 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:52.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:52.920 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:55:53.020 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:53.044 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:53.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:53.046 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:55:53.147 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:53.183 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:53.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:53.186 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:55:53.287 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:53.318 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:53.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:53.322 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:53.422 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:53.454 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:53.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:53.458 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:55:53.558 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:53.592 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:53.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:53.595 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:55:53.696 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:53.726 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:53.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:53.728 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:55:53.829 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:53.861 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:53.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:53.864 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:53.965 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:53.972 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:53 smithi078 bash[78274]: cluster 2023-10-01T10:55:52.054145+0000 mgr.y (mgr.25072) 1151 : cluster 0 pgmap v1150: 99 pgs: 4 peering, 95 active+clean; 2.6 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail 2023-10-01T10:55:53.973 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:53 smithi078 bash[75299]: cluster 2023-10-01T10:55:52.054145+0000 mgr.y (mgr.25072) 1151 : cluster 0 pgmap v1150: 99 pgs: 4 peering, 95 active+clean; 2.6 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail 2023-10-01T10:55:53.988 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:53.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:53.991 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:55:54.092 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:54.114 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:54.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:54.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:55:54.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:53 smithi173 bash[58816]: cluster 2023-10-01T10:55:52.054145+0000 mgr.y (mgr.25072) 1151 : cluster 0 pgmap v1150: 99 pgs: 4 peering, 95 active+clean; 2.6 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail 2023-10-01T10:55:54.218 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:54.244 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:54.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:54.545 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:55:54.645 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:54.670 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:54.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:54.673 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:55:54.774 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:54.806 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:54.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:54.809 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:55:54.910 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:54.938 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:54.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:54.941 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:55:55.042 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:55.075 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:55.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:55.078 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:55:55.179 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:55.210 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:55.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:55.213 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:55:55.314 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:55.340 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:55.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:55.344 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:55:55.445 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:55.477 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:55.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:55.480 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:55:55.526 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:55:55.581 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:55.605 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:55.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:55.609 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:55:55.709 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:55.731 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:55.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:55.733 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:55:55.782 INFO:teuthology.orchestra.run.smithi078.stderr:pool 'unique_pool_6' removed 2023-10-01T10:55:55.830 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:55 smithi078 bash[78274]: cluster 2023-10-01T10:55:54.055225+0000 mgr.y (mgr.25072) 1152 : cluster 0 pgmap v1151: 99 pgs: 99 active+clean; 2.6 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail 2023-10-01T10:55:55.830 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:55 smithi078 bash[78274]: audit 2023-10-01T10:55:55.180537+0000 mon.c (mon.2) 185 : audit 1 from='client.? 172.21.15.78:0/3124921531' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T10:55:55.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:55 smithi078 bash[78274]: audit 2023-10-01T10:55:55.180943+0000 mon.a (mon.0) 1446 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T10:55:55.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:55 smithi078 bash[75299]: cluster 2023-10-01T10:55:54.055225+0000 mgr.y (mgr.25072) 1152 : cluster 0 pgmap v1151: 99 pgs: 99 active+clean; 2.6 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail 2023-10-01T10:55:55.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:55 smithi078 bash[75299]: audit 2023-10-01T10:55:55.180537+0000 mon.c (mon.2) 185 : audit 1 from='client.? 172.21.15.78:0/3124921531' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T10:55:55.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:55 smithi078 bash[75299]: audit 2023-10-01T10:55:55.180943+0000 mon.a (mon.0) 1446 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T10:55:55.834 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:55.858 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:55.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:55.860 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:55:55.961 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:55.979 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:55.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:55.981 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:55:56.082 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:56.095 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:55 smithi173 bash[58816]: cluster 2023-10-01T10:55:54.055225+0000 mgr.y (mgr.25072) 1152 : cluster 0 pgmap v1151: 99 pgs: 99 active+clean; 2.6 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail 2023-10-01T10:55:56.095 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:55 smithi173 bash[58816]: audit 2023-10-01T10:55:55.180537+0000 mon.c (mon.2) 185 : audit 1 from='client.? 172.21.15.78:0/3124921531' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T10:55:56.095 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:55 smithi173 bash[58816]: audit 2023-10-01T10:55:55.180943+0000 mon.a (mon.0) 1446 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T10:55:56.113 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:56.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:56.117 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:55:56.218 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:56.238 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:56.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:56.240 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:55:56.340 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:56.375 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:56.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:56.378 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:55:56.478 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:56.512 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:56.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:56.516 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:56.616 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:56.644 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:56.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:56.650 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:55:56.751 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:56.770 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:56.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:56.772 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:55:56.873 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:56.886 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:56 smithi173 bash[58816]: cluster 2023-10-01T10:55:55.770869+0000 osd.5 (osd.5) 213 : cluster 3 failed to encode map e232 with expected crc 2023-10-01T10:55:56.886 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:56 smithi173 bash[58816]: audit 2023-10-01T10:55:55.774723+0000 mon.a (mon.0) 1447 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]': finished 2023-10-01T10:55:56.886 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:56 smithi173 bash[58816]: cluster 2023-10-01T10:55:55.775038+0000 mon.a (mon.0) 1448 : cluster 0 osdmap e232: 8 total, 8 up, 7 in 2023-10-01T10:55:56.906 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:56.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:56.909 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:55:57.010 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:57.021 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:56 smithi078 bash[75299]: cluster 2023-10-01T10:55:55.770869+0000 osd.5 (osd.5) 213 : cluster 3 failed to encode map e232 with expected crc 2023-10-01T10:55:57.021 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:56 smithi078 bash[75299]: audit 2023-10-01T10:55:55.774723+0000 mon.a (mon.0) 1447 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]': finished 2023-10-01T10:55:57.021 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:56 smithi078 bash[75299]: cluster 2023-10-01T10:55:55.775038+0000 mon.a (mon.0) 1448 : cluster 0 osdmap e232: 8 total, 8 up, 7 in 2023-10-01T10:55:57.021 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:56 smithi078 bash[78274]: cluster 2023-10-01T10:55:55.770869+0000 osd.5 (osd.5) 213 : cluster 3 failed to encode map e232 with expected crc 2023-10-01T10:55:57.021 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:56 smithi078 bash[78274]: audit 2023-10-01T10:55:55.774723+0000 mon.a (mon.0) 1447 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]': finished 2023-10-01T10:55:57.021 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:56 smithi078 bash[78274]: cluster 2023-10-01T10:55:55.775038+0000 mon.a (mon.0) 1448 : cluster 0 osdmap e232: 8 total, 8 up, 7 in 2023-10-01T10:55:57.038 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:57.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:57.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:55:57.141 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:57.173 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:57.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:57.176 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:55:57.277 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:57.311 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:57.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:57.313 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:57.414 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:57.433 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:57.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:57.436 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:55:57.493 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T10:55:57.512 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T10:55:57.533 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T10:55:57.537 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:57.554 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T10:55:57.571 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:57.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:57.573 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:55:57.577 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T10:55:57.599 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T10:55:57.628 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T10:55:57.650 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T10:55:57.651 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T10:55:57.651 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T10:55:57.652 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T10:55:57.652 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T10:55:57.652 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T10:55:57.652 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T10:55:57.652 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T10:55:57.652 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T10:55:57.675 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:57.697 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:57.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:57.699 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:55:57.800 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:57.813 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:57 smithi173 bash[58816]: cluster 2023-10-01T10:55:56.055672+0000 mgr.y (mgr.25072) 1153 : cluster 0 pgmap v1153: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 622 GiB / 626 GiB avail 2023-10-01T10:55:57.813 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:57 smithi173 bash[58816]: cluster 2023-10-01T10:55:56.605961+0000 osd.7 (osd.7) 211 : cluster 3 failed to encode map e232 with expected crc 2023-10-01T10:55:57.813 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:57 smithi173 bash[58816]: cluster 2023-10-01T10:55:56.606052+0000 osd.6 (osd.6) 170 : cluster 3 failed to encode map e232 with expected crc 2023-10-01T10:55:57.813 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:57 smithi173 bash[58816]: audit 2023-10-01T10:55:57.467990+0000 mon.c (mon.2) 186 : audit 0 from='client.? 172.21.15.78:0/3469766879' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:55:57.813 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:57 smithi173 bash[58816]: audit 2023-10-01T10:55:57.593004+0000 mon.a (mon.0) 1449 : audit 1 from='client.? 172.21.15.78:0/698552711' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:55:57.833 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:55:57.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:57.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:55:57.936 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:57.949 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:57 smithi078 bash[78274]: cluster 2023-10-01T10:55:56.055672+0000 mgr.y (mgr.25072) 1153 : cluster 0 pgmap v1153: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 622 GiB / 626 GiB avail 2023-10-01T10:55:57.950 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:57 smithi078 bash[78274]: cluster 2023-10-01T10:55:56.605961+0000 osd.7 (osd.7) 211 : cluster 3 failed to encode map e232 with expected crc 2023-10-01T10:55:57.950 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:57 smithi078 bash[78274]: cluster 2023-10-01T10:55:56.606052+0000 osd.6 (osd.6) 170 : cluster 3 failed to encode map e232 with expected crc 2023-10-01T10:55:57.950 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:57 smithi078 bash[78274]: audit 2023-10-01T10:55:57.467990+0000 mon.c (mon.2) 186 : audit 0 from='client.? 172.21.15.78:0/3469766879' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:55:57.950 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:57 smithi078 bash[78274]: audit 2023-10-01T10:55:57.593004+0000 mon.a (mon.0) 1449 : audit 1 from='client.? 172.21.15.78:0/698552711' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:55:57.950 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:57 smithi078 bash[75299]: cluster 2023-10-01T10:55:56.055672+0000 mgr.y (mgr.25072) 1153 : cluster 0 pgmap v1153: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 622 GiB / 626 GiB avail 2023-10-01T10:55:57.950 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:57 smithi078 bash[75299]: cluster 2023-10-01T10:55:56.605961+0000 osd.7 (osd.7) 211 : cluster 3 failed to encode map e232 with expected crc 2023-10-01T10:55:57.950 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:57 smithi078 bash[75299]: cluster 2023-10-01T10:55:56.606052+0000 osd.6 (osd.6) 170 : cluster 3 failed to encode map e232 with expected crc 2023-10-01T10:55:57.950 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:57 smithi078 bash[75299]: audit 2023-10-01T10:55:57.467990+0000 mon.c (mon.2) 186 : audit 0 from='client.? 172.21.15.78:0/3469766879' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:55:57.950 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:57 smithi078 bash[75299]: audit 2023-10-01T10:55:57.593004+0000 mon.a (mon.0) 1449 : audit 1 from='client.? 172.21.15.78:0/698552711' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:55:57.963 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:57.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:57.965 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:55:58.066 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:58.087 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:58.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:58.090 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:55:58.190 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:58.222 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:58.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:58.226 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:55:58.327 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:58.349 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:58.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:58.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:55:58.452 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:58.486 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:55:58.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:58.489 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:55:58.575 INFO:teuthology.orchestra.run.smithi078.stderr:nodeep-scrub is unset 2023-10-01T10:55:58.589 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:58.616 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:58.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:58.619 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:55:58.720 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:58.744 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:58.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:58.746 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:55:58.847 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:58.869 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:55:58.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:58.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:55:58.871 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-01T10:55:58.872 INFO:tasks.radosbench:Beginning radosbench... 2023-10-01T10:55:58.872 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_7 2023-10-01T10:55:58.872 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_7 16 2023-10-01T10:55:58.972 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:58.994 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:58.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:58.997 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:55:59.097 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:59.128 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:55:59.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:59.132 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:55:59.233 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:59.243 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:59 smithi078 bash[75299]: audit 2023-10-01T10:55:58.018896+0000 mon.a (mon.0) 1450 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:55:59.243 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:59 smithi078 bash[75299]: cluster 2023-10-01T10:55:58.056186+0000 mgr.y (mgr.25072) 1154 : cluster 0 pgmap v1154: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 622 GiB / 626 GiB avail 2023-10-01T10:55:59.243 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:59 smithi078 bash[75299]: cluster 2023-10-01T10:55:58.564292+0000 mon.a (mon.0) 1451 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T10:55:59.244 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:59 smithi078 bash[75299]: cluster 2023-10-01T10:55:58.564337+0000 mon.a (mon.0) 1452 : cluster 1 Cluster is now healthy 2023-10-01T10:55:59.244 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:59 smithi078 bash[75299]: cluster 2023-10-01T10:55:58.568059+0000 osd.5 (osd.5) 214 : cluster 3 failed to encode map e233 with expected crc 2023-10-01T10:55:59.244 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:59 smithi078 bash[75299]: audit 2023-10-01T10:55:58.569565+0000 mon.a (mon.0) 1453 : audit 1 from='client.? 172.21.15.78:0/698552711' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T10:55:59.244 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:55:59 smithi078 bash[75299]: cluster 2023-10-01T10:55:58.569656+0000 mon.a (mon.0) 1454 : cluster 0 osdmap e233: 8 total, 8 up, 7 in 2023-10-01T10:55:59.244 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:59 smithi078 bash[78274]: audit 2023-10-01T10:55:58.018896+0000 mon.a (mon.0) 1450 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:55:59.244 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:59 smithi078 bash[78274]: cluster 2023-10-01T10:55:58.056186+0000 mgr.y (mgr.25072) 1154 : cluster 0 pgmap v1154: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 622 GiB / 626 GiB avail 2023-10-01T10:55:59.244 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:59 smithi078 bash[78274]: cluster 2023-10-01T10:55:58.564292+0000 mon.a (mon.0) 1451 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T10:55:59.244 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:59 smithi078 bash[78274]: cluster 2023-10-01T10:55:58.564337+0000 mon.a (mon.0) 1452 : cluster 1 Cluster is now healthy 2023-10-01T10:55:59.244 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:59 smithi078 bash[78274]: cluster 2023-10-01T10:55:58.568059+0000 osd.5 (osd.5) 214 : cluster 3 failed to encode map e233 with expected crc 2023-10-01T10:55:59.244 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:59 smithi078 bash[78274]: audit 2023-10-01T10:55:58.569565+0000 mon.a (mon.0) 1453 : audit 1 from='client.? 172.21.15.78:0/698552711' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T10:55:59.245 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:55:59 smithi078 bash[78274]: cluster 2023-10-01T10:55:58.569656+0000 mon.a (mon.0) 1454 : cluster 0 osdmap e233: 8 total, 8 up, 7 in 2023-10-01T10:55:59.255 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:55:59.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:59.257 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:55:59.358 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:59.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:59 smithi173 bash[58816]: audit 2023-10-01T10:55:58.018896+0000 mon.a (mon.0) 1450 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:55:59.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:59 smithi173 bash[58816]: cluster 2023-10-01T10:55:58.056186+0000 mgr.y (mgr.25072) 1154 : cluster 0 pgmap v1154: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 622 GiB / 626 GiB avail 2023-10-01T10:55:59.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:59 smithi173 bash[58816]: cluster 2023-10-01T10:55:58.564292+0000 mon.a (mon.0) 1451 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T10:55:59.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:59 smithi173 bash[58816]: cluster 2023-10-01T10:55:58.564337+0000 mon.a (mon.0) 1452 : cluster 1 Cluster is now healthy 2023-10-01T10:55:59.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:59 smithi173 bash[58816]: cluster 2023-10-01T10:55:58.568059+0000 osd.5 (osd.5) 214 : cluster 3 failed to encode map e233 with expected crc 2023-10-01T10:55:59.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:59 smithi173 bash[58816]: audit 2023-10-01T10:55:58.569565+0000 mon.a (mon.0) 1453 : audit 1 from='client.? 172.21.15.78:0/698552711' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T10:55:59.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:55:59 smithi173 bash[58816]: cluster 2023-10-01T10:55:58.569656+0000 mon.a (mon.0) 1454 : cluster 0 osdmap e233: 8 total, 8 up, 7 in 2023-10-01T10:55:59.379 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:59.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:59.381 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:55:59.482 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:59.503 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:55:59.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:59.505 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:55:59.606 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:59.644 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:59.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:59.647 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:55:59.748 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:59.773 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:55:59.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:59.775 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:55:59.876 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:59.909 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:55:59.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:55:59.913 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:56:00.014 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:00.039 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:00.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:00.041 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:56:00.142 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:00.174 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:00.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:00.177 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:56:00.278 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:00.286 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:56:00 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:56:00] "GET /metrics HTTP/1.1" 200 36334 "" "Prometheus/2.33.4" 2023-10-01T10:56:00.287 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:00 smithi078 bash[75299]: cluster 2023-10-01T10:55:58.568213+0000 osd.7 (osd.7) 212 : cluster 3 failed to encode map e233 with expected crc 2023-10-01T10:56:00.287 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:00 smithi078 bash[75299]: audit 2023-10-01T10:55:59.263844+0000 mon.a (mon.0) 1455 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:56:00.287 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:00 smithi078 bash[75299]: audit 2023-10-01T10:55:59.272057+0000 mon.a (mon.0) 1456 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:56:00.287 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:00 smithi078 bash[75299]: audit 2023-10-01T10:55:59.743487+0000 mon.a (mon.0) 1457 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:56:00.287 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:00 smithi078 bash[75299]: audit 2023-10-01T10:55:59.744591+0000 mon.a (mon.0) 1458 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:56:00.287 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:00 smithi078 bash[75299]: audit 2023-10-01T10:55:59.754142+0000 mon.a (mon.0) 1459 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:56:00.288 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:00 smithi078 bash[78274]: cluster 2023-10-01T10:55:58.568213+0000 osd.7 (osd.7) 212 : cluster 3 failed to encode map e233 with expected crc 2023-10-01T10:56:00.288 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:00 smithi078 bash[78274]: audit 2023-10-01T10:55:59.263844+0000 mon.a (mon.0) 1455 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:56:00.288 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:00 smithi078 bash[78274]: audit 2023-10-01T10:55:59.272057+0000 mon.a (mon.0) 1456 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:56:00.288 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:00 smithi078 bash[78274]: audit 2023-10-01T10:55:59.743487+0000 mon.a (mon.0) 1457 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:56:00.288 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:00 smithi078 bash[78274]: audit 2023-10-01T10:55:59.744591+0000 mon.a (mon.0) 1458 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:56:00.288 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:00 smithi078 bash[78274]: audit 2023-10-01T10:55:59.754142+0000 mon.a (mon.0) 1459 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:56:00.299 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:00.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:00.301 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:00.402 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:00.415 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:00 smithi173 bash[58816]: cluster 2023-10-01T10:55:58.568213+0000 osd.7 (osd.7) 212 : cluster 3 failed to encode map e233 with expected crc 2023-10-01T10:56:00.415 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:00 smithi173 bash[58816]: audit 2023-10-01T10:55:59.263844+0000 mon.a (mon.0) 1455 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:56:00.415 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:00 smithi173 bash[58816]: audit 2023-10-01T10:55:59.272057+0000 mon.a (mon.0) 1456 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:56:00.415 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:00 smithi173 bash[58816]: audit 2023-10-01T10:55:59.743487+0000 mon.a (mon.0) 1457 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:56:00.415 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:00 smithi173 bash[58816]: audit 2023-10-01T10:55:59.744591+0000 mon.a (mon.0) 1458 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:56:00.415 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:00 smithi173 bash[58816]: audit 2023-10-01T10:55:59.754142+0000 mon.a (mon.0) 1459 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:56:00.433 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:00.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:00.436 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:56:00.537 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:00.568 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:00.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:00.572 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:56:00.673 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:00.699 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:00.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:00.701 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:56:00.802 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:00.834 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:00.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:00.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:56:00.938 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:00.972 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:00.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:00.975 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:56:01.076 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:01.108 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:01.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:01.111 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:01.212 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:01.243 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:01.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:01.245 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:56:01.346 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:01.379 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:01.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:01.383 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:56:01.419 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:56:01.484 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:01.510 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:01.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:01.512 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:56:01.613 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:01.643 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:01.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:01.646 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:56:01.747 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:01.771 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:01.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:01.775 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:56:01.875 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:01.909 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:01.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:01.911 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:02.012 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:02.023 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:01 smithi173 bash[58816]: cluster 2023-10-01T10:56:00.056696+0000 mgr.y (mgr.25072) 1155 : cluster 0 pgmap v1156: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 622 GiB / 626 GiB avail 2023-10-01T10:56:02.041 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:02.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:02.044 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:56:02.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:01 smithi078 bash[75299]: cluster 2023-10-01T10:56:00.056696+0000 mgr.y (mgr.25072) 1155 : cluster 0 pgmap v1156: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 622 GiB / 626 GiB avail 2023-10-01T10:56:02.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:01 smithi078 bash[78274]: cluster 2023-10-01T10:56:00.056696+0000 mgr.y (mgr.25072) 1155 : cluster 0 pgmap v1156: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 622 GiB / 626 GiB avail 2023-10-01T10:56:02.146 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:02.176 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:02.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:02.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:56:02.280 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:02.306 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:02.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:02.308 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:56:02.409 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:02.434 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:02.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:02.437 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:56:02.537 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:02.567 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:02.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:02.570 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:56:02.671 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:02.700 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:02.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:02.703 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:56:02.804 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:02.833 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T10:56:02.840 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:02.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:02.843 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:56:02.934 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T10:56:02.943 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:02.974 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:02.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:02.978 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:56:03.079 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:03.110 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:03.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:03.114 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:56:03.215 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:03.249 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:03.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:03.252 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:56:03.353 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:03.373 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:03.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:03.376 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:56:03.476 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:03.489 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:03 smithi173 bash[58816]: cluster 2023-10-01T10:56:02.057278+0000 mgr.y (mgr.25072) 1156 : cluster 0 pgmap v1157: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 622 GiB / 626 GiB avail 2023-10-01T10:56:03.490 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:03 smithi173 bash[58816]: audit 2023-10-01T10:56:02.447288+0000 mon.a (mon.0) 1460 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:56:03.490 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:03 smithi173 bash[58816]: audit 2023-10-01T10:56:02.448315+0000 mon.a (mon.0) 1461 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:56:03.508 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:03.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:03.511 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:56:03.612 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:03.648 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:03.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:03.651 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:56:03.752 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:03.786 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:03.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:03.789 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:56:03.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:03 smithi078 bash[75299]: cluster 2023-10-01T10:56:02.057278+0000 mgr.y (mgr.25072) 1156 : cluster 0 pgmap v1157: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 622 GiB / 626 GiB avail 2023-10-01T10:56:03.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:03 smithi078 bash[75299]: audit 2023-10-01T10:56:02.447288+0000 mon.a (mon.0) 1460 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:56:03.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:03 smithi078 bash[75299]: audit 2023-10-01T10:56:02.448315+0000 mon.a (mon.0) 1461 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:56:03.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:03 smithi078 bash[78274]: cluster 2023-10-01T10:56:02.057278+0000 mgr.y (mgr.25072) 1156 : cluster 0 pgmap v1157: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 622 GiB / 626 GiB avail 2023-10-01T10:56:03.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:03 smithi078 bash[78274]: audit 2023-10-01T10:56:02.447288+0000 mon.a (mon.0) 1460 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:56:03.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:03 smithi078 bash[78274]: audit 2023-10-01T10:56:02.448315+0000 mon.a (mon.0) 1461 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:56:03.889 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:03.916 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:03.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:03.919 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:04.020 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:04.047 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:04.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:04.050 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:56:04.151 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:04.175 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:04.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:04.178 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:56:04.279 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:04.318 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:04.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:04.321 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:56:04.421 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:04.447 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:04.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:04.449 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:56:04.550 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:04.582 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:04.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:04.585 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:56:04.686 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:04.713 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:04.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:04.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:56:04.816 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:04.846 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:04.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:04.849 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:56:04.950 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:04.980 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:04.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:04.984 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:56:05.085 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:05.119 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:05.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:05.122 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:56:05.223 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:05.255 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:05.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:05.259 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:56:05.360 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:05.389 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:05.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:05.392 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:56:05.397 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2023-10-01T10:56:05.397 INFO:teuthology.orchestra.run.smithi078.stdout: "ops": [], 2023-10-01T10:56:05.397 INFO:teuthology.orchestra.run.smithi078.stdout: "complaint_time": 30, 2023-10-01T10:56:05.397 INFO:teuthology.orchestra.run.smithi078.stdout: "num_blocked_ops": 0 2023-10-01T10:56:05.397 INFO:teuthology.orchestra.run.smithi078.stdout:} 2023-10-01T10:56:05.494 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:05.527 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:05.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:05.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:56:05.572 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:56:05.632 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:05.639 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:05 smithi078 bash[75299]: cluster 2023-10-01T10:56:04.057913+0000 mgr.y (mgr.25072) 1157 : cluster 0 pgmap v1158: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 622 GiB / 626 GiB avail 2023-10-01T10:56:05.640 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:05 smithi078 bash[78274]: cluster 2023-10-01T10:56:04.057913+0000 mgr.y (mgr.25072) 1157 : cluster 0 pgmap v1158: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 622 GiB / 626 GiB avail 2023-10-01T10:56:05.654 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:05.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:05.657 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:56:05.758 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:05.783 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:05.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:05.785 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:56:05.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:05 smithi173 bash[58816]: cluster 2023-10-01T10:56:04.057913+0000 mgr.y (mgr.25072) 1157 : cluster 0 pgmap v1158: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 622 GiB / 626 GiB avail 2023-10-01T10:56:05.887 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:05.908 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:05.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:05.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:06.011 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:06.043 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:06.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:06.047 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:56:06.148 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:06.175 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:06.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:06.177 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:56:06.278 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:06.311 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:06.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:06.314 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:56:06.415 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:06.449 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:06.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:06.452 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:56:06.553 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:06.585 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:06.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:06.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:56:06.690 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:06.726 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:06.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:06.730 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:56:06.830 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:06.857 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:06.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:06.859 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:56:06.961 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:06.996 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:06.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:06.999 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:56:07.099 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:07.137 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:07.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:07.138 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:56:07.239 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:07.269 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:07.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:07.271 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:56:07.372 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:07.398 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:07.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:07.401 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:56:07.488 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-01T10:56:07.502 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:07.536 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:07.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:07.539 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:56:07.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:07 smithi078 bash[75299]: cluster 2023-10-01T10:56:06.058730+0000 mgr.y (mgr.25072) 1158 : cluster 0 pgmap v1159: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 622 GiB / 626 GiB avail 2023-10-01T10:56:07.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:07 smithi078 bash[78274]: cluster 2023-10-01T10:56:06.058730+0000 mgr.y (mgr.25072) 1158 : cluster 0 pgmap v1159: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 622 GiB / 626 GiB avail 2023-10-01T10:56:07.640 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:07.668 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:07.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:07.671 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:56:07.772 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:07.797 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:07.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:07.799 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:56:07.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:07 smithi173 bash[58816]: cluster 2023-10-01T10:56:06.058730+0000 mgr.y (mgr.25072) 1158 : cluster 0 pgmap v1159: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 622 GiB / 626 GiB avail 2023-10-01T10:56:07.900 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:07.928 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:07.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:07.931 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:07.958 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:56:07.958 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:56:08.032 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:08.053 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:08.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:08.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:56:08.157 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:08.192 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:08.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:08.196 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:56:08.296 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:08.318 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:08.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:08.320 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:08.421 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:08.457 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:08.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:08.460 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:56:08.561 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:08.587 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:08.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:08.589 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:56:08.690 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:08.720 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:08.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:08.723 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:56:08.824 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:08.847 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:08.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:08.851 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:56:08.952 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:08.983 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:08.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:08.987 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:56:09.087 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:09.116 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:09.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:09.118 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:09.219 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:09.251 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:09.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:09.253 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:09.354 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:09.384 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:09.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:09.387 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:56:09.488 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:09.519 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:09.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:09.522 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:56:09.622 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:09.655 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:09.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:09.658 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:56:09.674 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:09 smithi078 bash[78274]: cluster 2023-10-01T10:56:08.059469+0000 mgr.y (mgr.25072) 1159 : cluster 0 pgmap v1160: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 622 GiB / 626 GiB avail 2023-10-01T10:56:09.675 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:09 smithi078 bash[75299]: cluster 2023-10-01T10:56:08.059469+0000 mgr.y (mgr.25072) 1159 : cluster 0 pgmap v1160: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 622 GiB / 626 GiB avail 2023-10-01T10:56:09.759 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:09.792 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:09.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:09.796 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:56:09.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:09 smithi173 bash[58816]: cluster 2023-10-01T10:56:08.059469+0000 mgr.y (mgr.25072) 1159 : cluster 0 pgmap v1160: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 622 GiB / 626 GiB avail 2023-10-01T10:56:09.897 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:09.927 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:09.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:09.930 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:56:10.031 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:10.061 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:10.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:10.064 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:56:10.165 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:10.195 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:10.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:10.199 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:56:10.299 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:10.333 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:10.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:10.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:56:10.437 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:10.461 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:10.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:10.463 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:10.564 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:10.579 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:56:10 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:56:10] "GET /metrics HTTP/1.1" 200 33982 "" "Prometheus/2.33.4" 2023-10-01T10:56:10.592 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:10.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:10.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:10.695 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:10.730 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:10.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:10.733 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:56:10.834 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:10.868 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:10.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:10.871 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:56:10.972 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:10.997 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:10.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:11.000 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:56:11.101 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:11.133 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:11.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:11.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:56:11.237 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:11.263 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:11.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:11.265 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:56:11.366 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:11.401 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:11.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:11.404 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:56:11.505 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:11.528 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:11.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:11.530 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:56:11.631 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:11.652 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:11.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:11.654 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:56:11.755 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:11.767 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:11 smithi173 bash[58816]: cluster 2023-10-01T10:56:10.060182+0000 mgr.y (mgr.25072) 1160 : cluster 0 pgmap v1161: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 622 GiB / 626 GiB avail 2023-10-01T10:56:11.788 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:11.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:11.791 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:56:11.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:11 smithi078 bash[75299]: cluster 2023-10-01T10:56:10.060182+0000 mgr.y (mgr.25072) 1160 : cluster 0 pgmap v1161: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 622 GiB / 626 GiB avail 2023-10-01T10:56:11.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:11 smithi078 bash[78274]: cluster 2023-10-01T10:56:10.060182+0000 mgr.y (mgr.25072) 1160 : cluster 0 pgmap v1161: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 622 GiB / 626 GiB avail 2023-10-01T10:56:11.892 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:11.913 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:11.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:11.915 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:12.016 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:12.038 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:12.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:12.042 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:56:12.142 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:12.170 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:12.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:12.173 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:12.274 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:12.306 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:12.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:12.308 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:56:12.409 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:12.441 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:12.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:12.445 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:56:12.546 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:12.573 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:12.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:12.575 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:56:12.676 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:12.697 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:12.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:12.700 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:56:12.765 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:56:12.801 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:12.835 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:12.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:12.838 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:56:12.939 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:12.970 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:12.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:12.974 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:56:13.075 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:13.105 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:13.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:13.107 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:13.208 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:13.226 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:13.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:13.228 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:56:13.328 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:13.365 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:13.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:13.369 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:56:13.470 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:13.479 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:13 smithi078 bash[75299]: debug 2023-10-01T10:56:13.195+0000 7f31c8dd0700 -1 mon.a@0(leader).osd e233 definitely_dead 0 2023-10-01T10:56:13.493 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:13.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:13.495 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:56:13.596 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:13.597 INFO:teuthology.orchestra.run.smithi078.stderr:marked in osd.6. 2023-10-01T10:56:13.618 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:13.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:13.620 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:13.721 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:13.743 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:13 smithi078 bash[75299]: cluster 2023-10-01T10:56:12.060850+0000 mgr.y (mgr.25072) 1161 : cluster 0 pgmap v1162: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 622 GiB / 626 GiB avail 2023-10-01T10:56:13.744 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:13 smithi078 bash[75299]: audit 2023-10-01T10:56:13.200383+0000 mon.a (mon.0) 1462 : audit 0 from='client.? 172.21.15.78:0/2017168393' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-01T10:56:13.744 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:13 smithi078 bash[78274]: cluster 2023-10-01T10:56:12.060850+0000 mgr.y (mgr.25072) 1161 : cluster 0 pgmap v1162: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 622 GiB / 626 GiB avail 2023-10-01T10:56:13.744 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:13 smithi078 bash[78274]: audit 2023-10-01T10:56:13.200383+0000 mon.a (mon.0) 1462 : audit 0 from='client.? 172.21.15.78:0/2017168393' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-01T10:56:13.754 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:13.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:13.757 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:56:13.858 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:13.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:13 smithi173 bash[58816]: cluster 2023-10-01T10:56:12.060850+0000 mgr.y (mgr.25072) 1161 : cluster 0 pgmap v1162: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 622 GiB / 626 GiB avail 2023-10-01T10:56:13.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:13 smithi173 bash[58816]: audit 2023-10-01T10:56:13.200383+0000 mon.a (mon.0) 1462 : audit 0 from='client.? 172.21.15.78:0/2017168393' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-01T10:56:13.890 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:13.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:13.894 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:56:13.994 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:14.020 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:14.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:14.022 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:56:14.123 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:14.152 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:14.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:14.154 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:14.255 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:14.280 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:14.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:14.283 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:56:14.384 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:14.414 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:14.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:14.417 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:56:14.518 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:14.549 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:14.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:14.552 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:56:14.653 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:14.660 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:14 smithi078 bash[75299]: cluster 2023-10-01T10:56:13.581708+0000 osd.6 (osd.6) 171 : cluster 3 failed to encode map e234 with expected crc 2023-10-01T10:56:14.661 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:14 smithi078 bash[75299]: audit 2023-10-01T10:56:13.582822+0000 mon.a (mon.0) 1463 : audit 1 from='client.? 172.21.15.78:0/2017168393' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-10-01T10:56:14.661 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:14 smithi078 bash[75299]: cluster 2023-10-01T10:56:13.582898+0000 mon.a (mon.0) 1464 : cluster 0 osdmap e234: 8 total, 8 up, 8 in 2023-10-01T10:56:14.661 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:14 smithi078 bash[75299]: cluster 2023-10-01T10:56:13.584979+0000 osd.7 (osd.7) 213 : cluster 3 failed to encode map e234 with expected crc 2023-10-01T10:56:14.661 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:14 smithi078 bash[75299]: cluster 2023-10-01T10:56:13.585855+0000 osd.5 (osd.5) 215 : cluster 3 failed to encode map e234 with expected crc 2023-10-01T10:56:14.661 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:14 smithi078 bash[75299]: cluster 2023-10-01T10:56:13.589128+0000 osd.7 (osd.7) 214 : cluster 3 failed to encode map e234 with expected crc 2023-10-01T10:56:14.661 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:14 smithi078 bash[75299]: cluster 2023-10-01T10:56:13.590925+0000 osd.7 (osd.7) 215 : cluster 3 failed to encode map e234 with expected crc 2023-10-01T10:56:14.661 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:14 smithi078 bash[78274]: cluster 2023-10-01T10:56:13.581708+0000 osd.6 (osd.6) 171 : cluster 3 failed to encode map e234 with expected crc 2023-10-01T10:56:14.661 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:14 smithi078 bash[78274]: audit 2023-10-01T10:56:13.582822+0000 mon.a (mon.0) 1463 : audit 1 from='client.? 172.21.15.78:0/2017168393' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-10-01T10:56:14.662 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:14 smithi078 bash[78274]: cluster 2023-10-01T10:56:13.582898+0000 mon.a (mon.0) 1464 : cluster 0 osdmap e234: 8 total, 8 up, 8 in 2023-10-01T10:56:14.662 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:14 smithi078 bash[78274]: cluster 2023-10-01T10:56:13.584979+0000 osd.7 (osd.7) 213 : cluster 3 failed to encode map e234 with expected crc 2023-10-01T10:56:14.662 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:14 smithi078 bash[78274]: cluster 2023-10-01T10:56:13.585855+0000 osd.5 (osd.5) 215 : cluster 3 failed to encode map e234 with expected crc 2023-10-01T10:56:14.662 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:14 smithi078 bash[78274]: cluster 2023-10-01T10:56:13.589128+0000 osd.7 (osd.7) 214 : cluster 3 failed to encode map e234 with expected crc 2023-10-01T10:56:14.662 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:14 smithi078 bash[78274]: cluster 2023-10-01T10:56:13.590925+0000 osd.7 (osd.7) 215 : cluster 3 failed to encode map e234 with expected crc 2023-10-01T10:56:14.675 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:14.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:14.677 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:56:14.778 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:14.790 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:14 smithi173 bash[58816]: cluster 2023-10-01T10:56:13.581708+0000 osd.6 (osd.6) 171 : cluster 3 failed to encode map e234 with expected crc 2023-10-01T10:56:14.791 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:14 smithi173 bash[58816]: audit 2023-10-01T10:56:13.582822+0000 mon.a (mon.0) 1463 : audit 1 from='client.? 172.21.15.78:0/2017168393' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-10-01T10:56:14.791 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:14 smithi173 bash[58816]: cluster 2023-10-01T10:56:13.582898+0000 mon.a (mon.0) 1464 : cluster 0 osdmap e234: 8 total, 8 up, 8 in 2023-10-01T10:56:14.791 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:14 smithi173 bash[58816]: cluster 2023-10-01T10:56:13.584979+0000 osd.7 (osd.7) 213 : cluster 3 failed to encode map e234 with expected crc 2023-10-01T10:56:14.791 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:14 smithi173 bash[58816]: cluster 2023-10-01T10:56:13.585855+0000 osd.5 (osd.5) 215 : cluster 3 failed to encode map e234 with expected crc 2023-10-01T10:56:14.791 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:14 smithi173 bash[58816]: cluster 2023-10-01T10:56:13.589128+0000 osd.7 (osd.7) 214 : cluster 3 failed to encode map e234 with expected crc 2023-10-01T10:56:14.791 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:14 smithi173 bash[58816]: cluster 2023-10-01T10:56:13.590925+0000 osd.7 (osd.7) 215 : cluster 3 failed to encode map e234 with expected crc 2023-10-01T10:56:14.811 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:14.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:14.814 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:56:14.914 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:14.937 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:14.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:14.940 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:56:15.041 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:15.061 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:15.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:15.063 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:15.164 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:15.199 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:15.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:15.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:56:15.303 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:15.335 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:15.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:15.338 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:56:15.439 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:15.462 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:15.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:15.464 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:56:15.565 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:15.591 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:15.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:15.595 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:56:15.695 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:15.717 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:15.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:15.720 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:56:15.769 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:15 smithi078 bash[75299]: cluster 2023-10-01T10:56:14.061557+0000 mgr.y (mgr.25072) 1162 : cluster 0 pgmap v1164: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 622 GiB / 626 GiB avail 2023-10-01T10:56:15.769 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:15 smithi078 bash[75299]: cluster 2023-10-01T10:56:14.599421+0000 osd.5 (osd.5) 216 : cluster 3 failed to encode map e235 with expected crc 2023-10-01T10:56:15.769 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:15 smithi078 bash[75299]: cluster 2023-10-01T10:56:14.600106+0000 osd.7 (osd.7) 216 : cluster 3 failed to encode map e235 with expected crc 2023-10-01T10:56:15.769 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:15 smithi078 bash[75299]: cluster 2023-10-01T10:56:14.602319+0000 mon.a (mon.0) 1465 : cluster 0 osdmap e235: 8 total, 8 up, 8 in 2023-10-01T10:56:15.769 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:15 smithi078 bash[75299]: cluster 2023-10-01T10:56:14.603396+0000 osd.5 (osd.5) 217 : cluster 3 failed to encode map e235 with expected crc 2023-10-01T10:56:15.770 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:15 smithi078 bash[75299]: cluster 2023-10-01T10:56:14.603405+0000 osd.6 (osd.6) 172 : cluster 3 failed to encode map e235 with expected crc 2023-10-01T10:56:15.770 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:15 smithi078 bash[75299]: cluster 2023-10-01T10:56:14.608406+0000 osd.6 (osd.6) 173 : cluster 3 failed to encode map e235 with expected crc 2023-10-01T10:56:15.770 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:15 smithi078 bash[75299]: cluster 2023-10-01T10:56:14.608721+0000 osd.5 (osd.5) 218 : cluster 3 failed to encode map e235 with expected crc 2023-10-01T10:56:15.770 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:15 smithi078 bash[75299]: cluster 2023-10-01T10:56:14.611496+0000 osd.5 (osd.5) 219 : cluster 3 failed to encode map e235 with expected crc 2023-10-01T10:56:15.770 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:15 smithi078 bash[75299]: cluster 2023-10-01T10:56:14.612630+0000 osd.6 (osd.6) 174 : cluster 3 failed to encode map e235 with expected crc 2023-10-01T10:56:15.770 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:15 smithi078 bash[78274]: cluster 2023-10-01T10:56:14.061557+0000 mgr.y (mgr.25072) 1162 : cluster 0 pgmap v1164: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 622 GiB / 626 GiB avail 2023-10-01T10:56:15.770 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:15 smithi078 bash[78274]: cluster 2023-10-01T10:56:14.599421+0000 osd.5 (osd.5) 216 : cluster 3 failed to encode map e235 with expected crc 2023-10-01T10:56:15.770 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:15 smithi078 bash[78274]: cluster 2023-10-01T10:56:14.600106+0000 osd.7 (osd.7) 216 : cluster 3 failed to encode map e235 with expected crc 2023-10-01T10:56:15.770 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:15 smithi078 bash[78274]: cluster 2023-10-01T10:56:14.602319+0000 mon.a (mon.0) 1465 : cluster 0 osdmap e235: 8 total, 8 up, 8 in 2023-10-01T10:56:15.771 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:15 smithi078 bash[78274]: cluster 2023-10-01T10:56:14.603396+0000 osd.5 (osd.5) 217 : cluster 3 failed to encode map e235 with expected crc 2023-10-01T10:56:15.771 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:15 smithi078 bash[78274]: cluster 2023-10-01T10:56:14.603405+0000 osd.6 (osd.6) 172 : cluster 3 failed to encode map e235 with expected crc 2023-10-01T10:56:15.771 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:15 smithi078 bash[78274]: cluster 2023-10-01T10:56:14.608406+0000 osd.6 (osd.6) 173 : cluster 3 failed to encode map e235 with expected crc 2023-10-01T10:56:15.771 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:15 smithi078 bash[78274]: cluster 2023-10-01T10:56:14.608721+0000 osd.5 (osd.5) 218 : cluster 3 failed to encode map e235 with expected crc 2023-10-01T10:56:15.771 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:15 smithi078 bash[78274]: cluster 2023-10-01T10:56:14.611496+0000 osd.5 (osd.5) 219 : cluster 3 failed to encode map e235 with expected crc 2023-10-01T10:56:15.771 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:15 smithi078 bash[78274]: cluster 2023-10-01T10:56:14.612630+0000 osd.6 (osd.6) 174 : cluster 3 failed to encode map e235 with expected crc 2023-10-01T10:56:15.820 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:15.853 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:15.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:15.856 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:56:15.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:15 smithi173 bash[58816]: cluster 2023-10-01T10:56:14.061557+0000 mgr.y (mgr.25072) 1162 : cluster 0 pgmap v1164: 73 pgs: 73 active+clean; 582 KiB data, 4.2 GiB used, 622 GiB / 626 GiB avail 2023-10-01T10:56:15.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:15 smithi173 bash[58816]: cluster 2023-10-01T10:56:14.599421+0000 osd.5 (osd.5) 216 : cluster 3 failed to encode map e235 with expected crc 2023-10-01T10:56:15.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:15 smithi173 bash[58816]: cluster 2023-10-01T10:56:14.600106+0000 osd.7 (osd.7) 216 : cluster 3 failed to encode map e235 with expected crc 2023-10-01T10:56:15.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:15 smithi173 bash[58816]: cluster 2023-10-01T10:56:14.602319+0000 mon.a (mon.0) 1465 : cluster 0 osdmap e235: 8 total, 8 up, 8 in 2023-10-01T10:56:15.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:15 smithi173 bash[58816]: cluster 2023-10-01T10:56:14.603396+0000 osd.5 (osd.5) 217 : cluster 3 failed to encode map e235 with expected crc 2023-10-01T10:56:15.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:15 smithi173 bash[58816]: cluster 2023-10-01T10:56:14.603405+0000 osd.6 (osd.6) 172 : cluster 3 failed to encode map e235 with expected crc 2023-10-01T10:56:15.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:15 smithi173 bash[58816]: cluster 2023-10-01T10:56:14.608406+0000 osd.6 (osd.6) 173 : cluster 3 failed to encode map e235 with expected crc 2023-10-01T10:56:15.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:15 smithi173 bash[58816]: cluster 2023-10-01T10:56:14.608721+0000 osd.5 (osd.5) 218 : cluster 3 failed to encode map e235 with expected crc 2023-10-01T10:56:15.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:15 smithi173 bash[58816]: cluster 2023-10-01T10:56:14.611496+0000 osd.5 (osd.5) 219 : cluster 3 failed to encode map e235 with expected crc 2023-10-01T10:56:15.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:15 smithi173 bash[58816]: cluster 2023-10-01T10:56:14.612630+0000 osd.6 (osd.6) 174 : cluster 3 failed to encode map e235 with expected crc 2023-10-01T10:56:15.957 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:15.990 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:15.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:15.993 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:56:16.094 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:16.119 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:16.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:16.122 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:56:16.222 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:16.251 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:16.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:16.254 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:56:16.355 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:16.385 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:16.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:16.388 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:56:16.489 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:16.514 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:16.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:16.518 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:56:16.618 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:16.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:16 smithi173 bash[58816]: cluster 2023-10-01T10:56:15.602976+0000 osd.5 (osd.5) 220 : cluster 3 failed to encode map e236 with expected crc 2023-10-01T10:56:16.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:16 smithi173 bash[58816]: cluster 2023-10-01T10:56:15.606222+0000 mon.a (mon.0) 1466 : cluster 0 osdmap e236: 8 total, 8 up, 8 in 2023-10-01T10:56:16.644 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:16.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:16.646 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:56:16.747 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:16.770 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:16.771 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:16 smithi078 bash[78274]: cluster 2023-10-01T10:56:15.602976+0000 osd.5 (osd.5) 220 : cluster 3 failed to encode map e236 with expected crc 2023-10-01T10:56:16.771 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:16 smithi078 bash[78274]: cluster 2023-10-01T10:56:15.606222+0000 mon.a (mon.0) 1466 : cluster 0 osdmap e236: 8 total, 8 up, 8 in 2023-10-01T10:56:16.771 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:16 smithi078 bash[78274]: cluster 2023-10-01T10:56:15.608396+0000 osd.7 (osd.7) 217 : cluster 3 failed to encode map e236 with expected crc 2023-10-01T10:56:16.771 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:16 smithi078 bash[78274]: cluster 2023-10-01T10:56:15.849419+0000 osd.6 (osd.6) 175 : cluster 3 failed to encode map e236 with expected crc 2023-10-01T10:56:16.771 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:16 smithi078 bash[75299]: cluster 2023-10-01T10:56:15.602976+0000 osd.5 (osd.5) 220 : cluster 3 failed to encode map e236 with expected crc 2023-10-01T10:56:16.771 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:16 smithi078 bash[75299]: cluster 2023-10-01T10:56:15.606222+0000 mon.a (mon.0) 1466 : cluster 0 osdmap e236: 8 total, 8 up, 8 in 2023-10-01T10:56:16.771 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:16 smithi078 bash[75299]: cluster 2023-10-01T10:56:15.608396+0000 osd.7 (osd.7) 217 : cluster 3 failed to encode map e236 with expected crc 2023-10-01T10:56:16.771 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:16 smithi078 bash[75299]: cluster 2023-10-01T10:56:15.849419+0000 osd.6 (osd.6) 175 : cluster 3 failed to encode map e236 with expected crc 2023-10-01T10:56:16.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:16.772 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:16.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:16 smithi173 bash[58816]: cluster 2023-10-01T10:56:15.608396+0000 osd.7 (osd.7) 217 : cluster 3 failed to encode map e236 with expected crc 2023-10-01T10:56:16.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:16 smithi173 bash[58816]: cluster 2023-10-01T10:56:15.849419+0000 osd.6 (osd.6) 175 : cluster 3 failed to encode map e236 with expected crc 2023-10-01T10:56:16.873 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:16.896 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:16.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:16.899 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:56:17.000 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:17.027 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:17.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:17.030 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:56:17.131 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:17.158 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:17.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:17.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:56:17.262 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:17.294 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:17.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:17.298 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:56:17.399 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:17.426 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:17.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:17.429 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:56:17.530 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:17.557 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:17.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:17.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:56:17.660 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:17.688 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:17.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:17.692 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:56:17.793 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:17.819 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:17.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:17.821 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:56:17.922 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:17.947 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:17.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:17.949 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:56:18.030 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:17 smithi078 bash[75299]: cluster 2023-10-01T10:56:16.062215+0000 mgr.y (mgr.25072) 1163 : cluster 0 pgmap v1167: 73 pgs: 12 peering, 61 active+clean; 582 KiB data, 4.2 GiB used, 622 GiB / 626 GiB avail 2023-10-01T10:56:18.031 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:17 smithi078 bash[75299]: cluster 2023-10-01T10:56:16.618942+0000 osd.7 (osd.7) 218 : cluster 3 failed to encode map e237 with expected crc 2023-10-01T10:56:18.031 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:17 smithi078 bash[75299]: cluster 2023-10-01T10:56:16.620608+0000 mon.a (mon.0) 1467 : cluster 0 osdmap e237: 8 total, 8 up, 8 in 2023-10-01T10:56:18.031 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:17 smithi078 bash[75299]: cluster 2023-10-01T10:56:16.625471+0000 osd.6 (osd.6) 176 : cluster 3 failed to encode map e237 with expected crc 2023-10-01T10:56:18.031 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:17 smithi078 bash[75299]: audit 2023-10-01T10:56:17.443888+0000 mon.a (mon.0) 1468 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:56:18.031 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:17 smithi078 bash[78274]: cluster 2023-10-01T10:56:16.062215+0000 mgr.y (mgr.25072) 1163 : cluster 0 pgmap v1167: 73 pgs: 12 peering, 61 active+clean; 582 KiB data, 4.2 GiB used, 622 GiB / 626 GiB avail 2023-10-01T10:56:18.031 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:17 smithi078 bash[78274]: cluster 2023-10-01T10:56:16.618942+0000 osd.7 (osd.7) 218 : cluster 3 failed to encode map e237 with expected crc 2023-10-01T10:56:18.031 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:17 smithi078 bash[78274]: cluster 2023-10-01T10:56:16.620608+0000 mon.a (mon.0) 1467 : cluster 0 osdmap e237: 8 total, 8 up, 8 in 2023-10-01T10:56:18.032 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:17 smithi078 bash[78274]: cluster 2023-10-01T10:56:16.625471+0000 osd.6 (osd.6) 176 : cluster 3 failed to encode map e237 with expected crc 2023-10-01T10:56:18.032 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:17 smithi078 bash[78274]: audit 2023-10-01T10:56:17.443888+0000 mon.a (mon.0) 1468 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:56:18.050 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:18.076 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:18.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:18.079 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:18.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:17 smithi173 bash[58816]: cluster 2023-10-01T10:56:16.062215+0000 mgr.y (mgr.25072) 1163 : cluster 0 pgmap v1167: 73 pgs: 12 peering, 61 active+clean; 582 KiB data, 4.2 GiB used, 622 GiB / 626 GiB avail 2023-10-01T10:56:18.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:17 smithi173 bash[58816]: cluster 2023-10-01T10:56:16.618942+0000 osd.7 (osd.7) 218 : cluster 3 failed to encode map e237 with expected crc 2023-10-01T10:56:18.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:17 smithi173 bash[58816]: cluster 2023-10-01T10:56:16.620608+0000 mon.a (mon.0) 1467 : cluster 0 osdmap e237: 8 total, 8 up, 8 in 2023-10-01T10:56:18.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:17 smithi173 bash[58816]: cluster 2023-10-01T10:56:16.625471+0000 osd.6 (osd.6) 176 : cluster 3 failed to encode map e237 with expected crc 2023-10-01T10:56:18.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:17 smithi173 bash[58816]: audit 2023-10-01T10:56:17.443888+0000 mon.a (mon.0) 1468 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:56:18.179 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:18.207 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:18.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:18.209 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:56:18.310 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:18.343 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:18.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:18.346 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:56:18.447 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:18.477 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:18.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:18.480 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:56:18.581 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:18.602 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:18.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:18.605 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:56:18.705 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:18.728 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:18.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:18.731 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:56:18.832 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:18.867 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:18.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:18.870 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:56:18.971 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:18.993 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:18.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:18.995 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:56:19.096 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:19.104 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:18 smithi078 bash[75299]: cluster 2023-10-01T10:56:17.580546+0000 osd.5 (osd.5) 221 : cluster 3 failed to encode map e237 with expected crc 2023-10-01T10:56:19.104 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:18 smithi078 bash[75299]: cluster 2023-10-01T10:56:17.625387+0000 osd.5 (osd.5) 222 : cluster 3 failed to encode map e238 with expected crc 2023-10-01T10:56:19.104 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:18 smithi078 bash[75299]: cluster 2023-10-01T10:56:17.625881+0000 osd.7 (osd.7) 219 : cluster 3 failed to encode map e238 with expected crc 2023-10-01T10:56:19.104 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:18 smithi078 bash[75299]: cluster 2023-10-01T10:56:17.626167+0000 mon.a (mon.0) 1469 : cluster 0 osdmap e238: 8 total, 8 up, 8 in 2023-10-01T10:56:19.104 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:18 smithi078 bash[75299]: cluster 2023-10-01T10:56:17.630159+0000 osd.6 (osd.6) 177 : cluster 3 failed to encode map e238 with expected crc 2023-10-01T10:56:19.105 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:18 smithi078 bash[78274]: cluster 2023-10-01T10:56:17.580546+0000 osd.5 (osd.5) 221 : cluster 3 failed to encode map e237 with expected crc 2023-10-01T10:56:19.105 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:18 smithi078 bash[78274]: cluster 2023-10-01T10:56:17.625387+0000 osd.5 (osd.5) 222 : cluster 3 failed to encode map e238 with expected crc 2023-10-01T10:56:19.105 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:18 smithi078 bash[78274]: cluster 2023-10-01T10:56:17.625881+0000 osd.7 (osd.7) 219 : cluster 3 failed to encode map e238 with expected crc 2023-10-01T10:56:19.105 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:18 smithi078 bash[78274]: cluster 2023-10-01T10:56:17.626167+0000 mon.a (mon.0) 1469 : cluster 0 osdmap e238: 8 total, 8 up, 8 in 2023-10-01T10:56:19.105 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:18 smithi078 bash[78274]: cluster 2023-10-01T10:56:17.630159+0000 osd.6 (osd.6) 177 : cluster 3 failed to encode map e238 with expected crc 2023-10-01T10:56:19.117 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:19.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:19.119 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:19.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:18 smithi173 bash[58816]: cluster 2023-10-01T10:56:17.580546+0000 osd.5 (osd.5) 221 : cluster 3 failed to encode map e237 with expected crc 2023-10-01T10:56:19.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:18 smithi173 bash[58816]: cluster 2023-10-01T10:56:17.625387+0000 osd.5 (osd.5) 222 : cluster 3 failed to encode map e238 with expected crc 2023-10-01T10:56:19.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:18 smithi173 bash[58816]: cluster 2023-10-01T10:56:17.625881+0000 osd.7 (osd.7) 219 : cluster 3 failed to encode map e238 with expected crc 2023-10-01T10:56:19.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:18 smithi173 bash[58816]: cluster 2023-10-01T10:56:17.626167+0000 mon.a (mon.0) 1469 : cluster 0 osdmap e238: 8 total, 8 up, 8 in 2023-10-01T10:56:19.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:18 smithi173 bash[58816]: cluster 2023-10-01T10:56:17.630159+0000 osd.6 (osd.6) 177 : cluster 3 failed to encode map e238 with expected crc 2023-10-01T10:56:19.219 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:19.256 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:19.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:19.258 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:56:19.359 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:19.382 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:19.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:19.384 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:56:19.485 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:19.512 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:19.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:19.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:19.615 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:19.638 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:19.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:19.640 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:56:19.741 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:19.748 INFO:tasks.thrashosds.thrasher:Added osd 6 2023-10-01T10:56:19.767 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:19.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:19.772 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:56:19.872 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:19.884 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:19 smithi173 bash[58816]: cluster 2023-10-01T10:56:18.062696+0000 mgr.y (mgr.25072) 1164 : cluster 0 pgmap v1170: 73 pgs: 12 peering, 61 active+clean; 582 KiB data, 4.2 GiB used, 622 GiB / 626 GiB avail 2023-10-01T10:56:19.904 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:19.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:19.908 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:56:20.008 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:20.039 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:20.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:20.042 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:56:20.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:19 smithi078 bash[75299]: cluster 2023-10-01T10:56:18.062696+0000 mgr.y (mgr.25072) 1164 : cluster 0 pgmap v1170: 73 pgs: 12 peering, 61 active+clean; 582 KiB data, 4.2 GiB used, 622 GiB / 626 GiB avail 2023-10-01T10:56:20.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:19 smithi078 bash[78274]: cluster 2023-10-01T10:56:18.062696+0000 mgr.y (mgr.25072) 1164 : cluster 0 pgmap v1170: 73 pgs: 12 peering, 61 active+clean; 582 KiB data, 4.2 GiB used, 622 GiB / 626 GiB avail 2023-10-01T10:56:20.143 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:20.174 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:20.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:20.178 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:56:20.279 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:20.311 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:20.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:20.314 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:56:20.415 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:20.440 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:20.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:20.442 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:20.543 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:20.555 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:56:20 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:56:20] "GET /metrics HTTP/1.1" 200 33965 "" "Prometheus/2.33.4" 2023-10-01T10:56:20.574 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:20.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:20.576 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:20.657 INFO:teuthology.orchestra.run.smithi078.stderr:pool 'unique_pool_7' created 2023-10-01T10:56:20.678 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:20.699 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:20.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:20.702 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:56:20.803 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:20.810 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:20 smithi078 bash[78274]: audit 2023-10-01T10:56:20.003848+0000 mon.c (mon.2) 187 : audit 1 from='client.? 172.21.15.78:0/640149668' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]: dispatch 2023-10-01T10:56:20.811 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:20 smithi078 bash[78274]: audit 2023-10-01T10:56:20.004271+0000 mon.a (mon.0) 1470 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]: dispatch 2023-10-01T10:56:20.811 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:20 smithi078 bash[75299]: audit 2023-10-01T10:56:20.003848+0000 mon.c (mon.2) 187 : audit 1 from='client.? 172.21.15.78:0/640149668' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]: dispatch 2023-10-01T10:56:20.811 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:20 smithi078 bash[75299]: audit 2023-10-01T10:56:20.004271+0000 mon.a (mon.0) 1470 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]: dispatch 2023-10-01T10:56:20.825 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:20.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:20.829 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:56:20.930 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:20.943 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:20 smithi173 bash[58816]: audit 2023-10-01T10:56:20.003848+0000 mon.c (mon.2) 187 : audit 1 from='client.? 172.21.15.78:0/640149668' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]: dispatch 2023-10-01T10:56:20.943 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:20 smithi173 bash[58816]: audit 2023-10-01T10:56:20.004271+0000 mon.a (mon.0) 1470 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]: dispatch 2023-10-01T10:56:20.964 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:20.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:20.967 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:56:21.067 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:21.097 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:21.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:21.100 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:21.201 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:21.231 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:21.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:21.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:56:21.335 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:21.360 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:21.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:21.363 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:56:21.463 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:21.490 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:21.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:21.494 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:56:21.595 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:21.622 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:21.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:21.625 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:56:21.725 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:21.734 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:21 smithi078 bash[75299]: cluster 2023-10-01T10:56:20.063210+0000 mgr.y (mgr.25072) 1165 : cluster 0 pgmap v1171: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 106 KiB/s, 2 objects/s recovering 2023-10-01T10:56:21.734 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:21 smithi078 bash[75299]: audit 2023-10-01T10:56:20.648147+0000 mon.a (mon.0) 1471 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]': finished 2023-10-01T10:56:21.734 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:21 smithi078 bash[75299]: cluster 2023-10-01T10:56:20.648237+0000 mon.a (mon.0) 1472 : cluster 0 osdmap e239: 8 total, 8 up, 8 in 2023-10-01T10:56:21.734 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:21 smithi078 bash[75299]: cluster 2023-10-01T10:56:20.649097+0000 osd.7 (osd.7) 220 : cluster 3 failed to encode map e239 with expected crc 2023-10-01T10:56:21.734 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:21 smithi078 bash[75299]: cluster 2023-10-01T10:56:20.649371+0000 osd.6 (osd.6) 178 : cluster 3 failed to encode map e239 with expected crc 2023-10-01T10:56:21.734 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:21 smithi078 bash[75299]: cluster 2023-10-01T10:56:20.651384+0000 osd.7 (osd.7) 221 : cluster 3 failed to encode map e239 with expected crc 2023-10-01T10:56:21.734 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:21 smithi078 bash[75299]: cluster 2023-10-01T10:56:20.653254+0000 osd.6 (osd.6) 179 : cluster 3 failed to encode map e239 with expected crc 2023-10-01T10:56:21.734 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:21 smithi078 bash[75299]: cluster 2023-10-01T10:56:20.653409+0000 osd.5 (osd.5) 223 : cluster 3 failed to encode map e239 with expected crc 2023-10-01T10:56:21.734 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:21 smithi078 bash[75299]: cluster 2023-10-01T10:56:20.655411+0000 osd.6 (osd.6) 180 : cluster 3 failed to encode map e239 with expected crc 2023-10-01T10:56:21.735 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:21 smithi078 bash[78274]: cluster 2023-10-01T10:56:20.063210+0000 mgr.y (mgr.25072) 1165 : cluster 0 pgmap v1171: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 106 KiB/s, 2 objects/s recovering 2023-10-01T10:56:21.735 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:21 smithi078 bash[78274]: audit 2023-10-01T10:56:20.648147+0000 mon.a (mon.0) 1471 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]': finished 2023-10-01T10:56:21.735 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:21 smithi078 bash[78274]: cluster 2023-10-01T10:56:20.648237+0000 mon.a (mon.0) 1472 : cluster 0 osdmap e239: 8 total, 8 up, 8 in 2023-10-01T10:56:21.735 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:21 smithi078 bash[78274]: cluster 2023-10-01T10:56:20.649097+0000 osd.7 (osd.7) 220 : cluster 3 failed to encode map e239 with expected crc 2023-10-01T10:56:21.735 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:21 smithi078 bash[78274]: cluster 2023-10-01T10:56:20.649371+0000 osd.6 (osd.6) 178 : cluster 3 failed to encode map e239 with expected crc 2023-10-01T10:56:21.735 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:21 smithi078 bash[78274]: cluster 2023-10-01T10:56:20.651384+0000 osd.7 (osd.7) 221 : cluster 3 failed to encode map e239 with expected crc 2023-10-01T10:56:21.735 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:21 smithi078 bash[78274]: cluster 2023-10-01T10:56:20.653254+0000 osd.6 (osd.6) 179 : cluster 3 failed to encode map e239 with expected crc 2023-10-01T10:56:21.735 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:21 smithi078 bash[78274]: cluster 2023-10-01T10:56:20.653409+0000 osd.5 (osd.5) 223 : cluster 3 failed to encode map e239 with expected crc 2023-10-01T10:56:21.735 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:21 smithi078 bash[78274]: cluster 2023-10-01T10:56:20.655411+0000 osd.6 (osd.6) 180 : cluster 3 failed to encode map e239 with expected crc 2023-10-01T10:56:21.749 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:21.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:21.752 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:56:21.852 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:21.886 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:21.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:21.890 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:56:21.991 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:22.020 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:22.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:22.022 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:56:22.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:21 smithi173 bash[58816]: cluster 2023-10-01T10:56:20.063210+0000 mgr.y (mgr.25072) 1165 : cluster 0 pgmap v1171: 73 pgs: 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 106 KiB/s, 2 objects/s recovering 2023-10-01T10:56:22.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:21 smithi173 bash[58816]: audit 2023-10-01T10:56:20.648147+0000 mon.a (mon.0) 1471 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]': finished 2023-10-01T10:56:22.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:21 smithi173 bash[58816]: cluster 2023-10-01T10:56:20.648237+0000 mon.a (mon.0) 1472 : cluster 0 osdmap e239: 8 total, 8 up, 8 in 2023-10-01T10:56:22.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:21 smithi173 bash[58816]: cluster 2023-10-01T10:56:20.649097+0000 osd.7 (osd.7) 220 : cluster 3 failed to encode map e239 with expected crc 2023-10-01T10:56:22.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:21 smithi173 bash[58816]: cluster 2023-10-01T10:56:20.649371+0000 osd.6 (osd.6) 178 : cluster 3 failed to encode map e239 with expected crc 2023-10-01T10:56:22.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:21 smithi173 bash[58816]: cluster 2023-10-01T10:56:20.651384+0000 osd.7 (osd.7) 221 : cluster 3 failed to encode map e239 with expected crc 2023-10-01T10:56:22.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:21 smithi173 bash[58816]: cluster 2023-10-01T10:56:20.653254+0000 osd.6 (osd.6) 179 : cluster 3 failed to encode map e239 with expected crc 2023-10-01T10:56:22.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:21 smithi173 bash[58816]: cluster 2023-10-01T10:56:20.653409+0000 osd.5 (osd.5) 223 : cluster 3 failed to encode map e239 with expected crc 2023-10-01T10:56:22.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:21 smithi173 bash[58816]: cluster 2023-10-01T10:56:20.655411+0000 osd.6 (osd.6) 180 : cluster 3 failed to encode map e239 with expected crc 2023-10-01T10:56:22.122 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:22.147 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:22.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:22.150 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:22.251 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:22.278 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:22.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:22.281 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:56:22.382 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:22.410 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:22.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:22.412 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:56:22.513 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:22.545 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:22.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:22.547 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:56:22.648 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:22.672 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:22.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:22.675 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:56:22.775 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:22.791 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:22 smithi173 bash[58816]: cluster 2023-10-01T10:56:21.654954+0000 osd.7 (osd.7) 222 : cluster 3 failed to encode map e240 with expected crc 2023-10-01T10:56:22.791 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:22 smithi173 bash[58816]: cluster 2023-10-01T10:56:21.656564+0000 mon.a (mon.0) 1473 : cluster 0 osdmap e240: 8 total, 8 up, 8 in 2023-10-01T10:56:22.792 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:22 smithi173 bash[58816]: cluster 2023-10-01T10:56:21.658219+0000 osd.7 (osd.7) 223 : cluster 3 failed to encode map e240 with expected crc 2023-10-01T10:56:22.792 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:22 smithi173 bash[58816]: cluster 2023-10-01T10:56:21.660809+0000 osd.6 (osd.6) 181 : cluster 3 failed to encode map e240 with expected crc 2023-10-01T10:56:22.806 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:22.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:22.809 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:56:22.910 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:22.934 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:22.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:22.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:56:23.037 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:23.068 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:23.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:23.071 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:56:23.172 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:23.205 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:23.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:23.208 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:23.308 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:23.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:22 smithi078 bash[75299]: cluster 2023-10-01T10:56:21.654954+0000 osd.7 (osd.7) 222 : cluster 3 failed to encode map e240 with expected crc 2023-10-01T10:56:23.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:22 smithi078 bash[75299]: cluster 2023-10-01T10:56:21.656564+0000 mon.a (mon.0) 1473 : cluster 0 osdmap e240: 8 total, 8 up, 8 in 2023-10-01T10:56:23.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:22 smithi078 bash[75299]: cluster 2023-10-01T10:56:21.658219+0000 osd.7 (osd.7) 223 : cluster 3 failed to encode map e240 with expected crc 2023-10-01T10:56:23.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:22 smithi078 bash[75299]: cluster 2023-10-01T10:56:21.660809+0000 osd.6 (osd.6) 181 : cluster 3 failed to encode map e240 with expected crc 2023-10-01T10:56:23.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:22 smithi078 bash[78274]: cluster 2023-10-01T10:56:21.654954+0000 osd.7 (osd.7) 222 : cluster 3 failed to encode map e240 with expected crc 2023-10-01T10:56:23.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:22 smithi078 bash[78274]: cluster 2023-10-01T10:56:21.656564+0000 mon.a (mon.0) 1473 : cluster 0 osdmap e240: 8 total, 8 up, 8 in 2023-10-01T10:56:23.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:22 smithi078 bash[78274]: cluster 2023-10-01T10:56:21.658219+0000 osd.7 (osd.7) 223 : cluster 3 failed to encode map e240 with expected crc 2023-10-01T10:56:23.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:22 smithi078 bash[78274]: cluster 2023-10-01T10:56:21.660809+0000 osd.6 (osd.6) 181 : cluster 3 failed to encode map e240 with expected crc 2023-10-01T10:56:23.340 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:23.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:23.344 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:56:23.445 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:23.475 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:23.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:23.477 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:56:23.578 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:23.602 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:23.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:23.605 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:23.705 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:23.737 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:23.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:23.741 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:56:23.842 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:23.876 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:23.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:23.880 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:56:23.980 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:23.989 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:23 smithi078 bash[75299]: cluster 2023-10-01T10:56:21.890663+0000 osd.5 (osd.5) 224 : cluster 3 failed to encode map e240 with expected crc 2023-10-01T10:56:23.989 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:23 smithi078 bash[75299]: cluster 2023-10-01T10:56:22.063843+0000 mgr.y (mgr.25072) 1166 : cluster 0 pgmap v1174: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 107 KiB/s, 2 objects/s recovering 2023-10-01T10:56:23.989 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:23 smithi078 bash[75299]: cluster 2023-10-01T10:56:22.661220+0000 osd.7 (osd.7) 224 : cluster 3 failed to encode map e241 with expected crc 2023-10-01T10:56:23.990 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:23 smithi078 bash[75299]: cluster 2023-10-01T10:56:22.663240+0000 mon.a (mon.0) 1474 : cluster 0 osdmap e241: 8 total, 8 up, 8 in 2023-10-01T10:56:23.990 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:23 smithi078 bash[75299]: cluster 2023-10-01T10:56:22.782172+0000 osd.6 (osd.6) 182 : cluster 3 failed to encode map e241 with expected crc 2023-10-01T10:56:23.990 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:23 smithi078 bash[75299]: cluster 2023-10-01T10:56:22.782499+0000 osd.5 (osd.5) 225 : cluster 3 failed to encode map e241 with expected crc 2023-10-01T10:56:23.990 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:23 smithi078 bash[78274]: cluster 2023-10-01T10:56:21.890663+0000 osd.5 (osd.5) 224 : cluster 3 failed to encode map e240 with expected crc 2023-10-01T10:56:23.990 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:23 smithi078 bash[78274]: cluster 2023-10-01T10:56:22.063843+0000 mgr.y (mgr.25072) 1166 : cluster 0 pgmap v1174: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 107 KiB/s, 2 objects/s recovering 2023-10-01T10:56:23.990 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:23 smithi078 bash[78274]: cluster 2023-10-01T10:56:22.661220+0000 osd.7 (osd.7) 224 : cluster 3 failed to encode map e241 with expected crc 2023-10-01T10:56:23.990 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:23 smithi078 bash[78274]: cluster 2023-10-01T10:56:22.663240+0000 mon.a (mon.0) 1474 : cluster 0 osdmap e241: 8 total, 8 up, 8 in 2023-10-01T10:56:23.990 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:23 smithi078 bash[78274]: cluster 2023-10-01T10:56:22.782172+0000 osd.6 (osd.6) 182 : cluster 3 failed to encode map e241 with expected crc 2023-10-01T10:56:23.990 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:23 smithi078 bash[78274]: cluster 2023-10-01T10:56:22.782499+0000 osd.5 (osd.5) 225 : cluster 3 failed to encode map e241 with expected crc 2023-10-01T10:56:24.005 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:24.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:24.008 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:24.108 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:24.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:23 smithi173 bash[58816]: cluster 2023-10-01T10:56:21.890663+0000 osd.5 (osd.5) 224 : cluster 3 failed to encode map e240 with expected crc 2023-10-01T10:56:24.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:23 smithi173 bash[58816]: cluster 2023-10-01T10:56:22.063843+0000 mgr.y (mgr.25072) 1166 : cluster 0 pgmap v1174: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 107 KiB/s, 2 objects/s recovering 2023-10-01T10:56:24.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:23 smithi173 bash[58816]: cluster 2023-10-01T10:56:22.661220+0000 osd.7 (osd.7) 224 : cluster 3 failed to encode map e241 with expected crc 2023-10-01T10:56:24.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:23 smithi173 bash[58816]: cluster 2023-10-01T10:56:22.663240+0000 mon.a (mon.0) 1474 : cluster 0 osdmap e241: 8 total, 8 up, 8 in 2023-10-01T10:56:24.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:23 smithi173 bash[58816]: cluster 2023-10-01T10:56:22.782172+0000 osd.6 (osd.6) 182 : cluster 3 failed to encode map e241 with expected crc 2023-10-01T10:56:24.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:23 smithi173 bash[58816]: cluster 2023-10-01T10:56:22.782499+0000 osd.5 (osd.5) 225 : cluster 3 failed to encode map e241 with expected crc 2023-10-01T10:56:24.142 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:24.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:24.146 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:56:24.246 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:24.267 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:24.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:24.269 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:56:24.369 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:24.398 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:24.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:24.402 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:56:24.503 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:24.524 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:24.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:24.526 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:56:24.627 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:24.659 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:24.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:24.662 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:56:24.749 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 7, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T10:56:24.749 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T10:56:24.763 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:24.788 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:24.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:24.790 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:56:24.891 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:24.921 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:24.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:24.924 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:56:24.956 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_7 rados --yes-i-really-mean-it || true 2023-10-01T10:56:25.025 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:25.055 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:25.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:25.059 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:56:25.159 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:25.182 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:25.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:25.184 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:25.285 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:25.332 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:25.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:25.335 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:56:25.436 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:25.476 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:25.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:25.479 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:56:25.580 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:25.617 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:25.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:25.621 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:56:25.721 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:25.743 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:25.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:25.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:25.845 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:25.869 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:25.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:25 smithi173 bash[58816]: cluster 2023-10-01T10:56:24.064605+0000 mgr.y (mgr.25072) 1167 : cluster 0 pgmap v1176: 89 pgs: 6 unknown, 83 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 97 KiB/s, 1 objects/s recovering 2023-10-01T10:56:25.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:25.871 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:25.971 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:25.998 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:26.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:26.001 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:26.035 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:25 smithi078 bash[75299]: cluster 2023-10-01T10:56:24.064605+0000 mgr.y (mgr.25072) 1167 : cluster 0 pgmap v1176: 89 pgs: 6 unknown, 83 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 97 KiB/s, 1 objects/s recovering 2023-10-01T10:56:26.102 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:26.121 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:26.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:26.124 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:26.224 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:26.247 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:26.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:26.249 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:56:26.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:26 smithi078 bash[78274]: cluster 2023-10-01T10:56:24.064605+0000 mgr.y (mgr.25072) 1167 : cluster 0 pgmap v1176: 89 pgs: 6 unknown, 83 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 97 KiB/s, 1 objects/s recovering 2023-10-01T10:56:26.350 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:26.365 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T10:56:26.372 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:26.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:26.375 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:56:26.391 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T10:56:26.415 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T10:56:26.442 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T10:56:26.468 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T10:56:26.476 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:26.494 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T10:56:26.498 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:26.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:26.501 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:56:26.523 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T10:56:26.552 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T10:56:26.555 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T10:56:26.555 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T10:56:26.555 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T10:56:26.555 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T10:56:26.555 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T10:56:26.555 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T10:56:26.555 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T10:56:26.555 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T10:56:26.602 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:26.637 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:26.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:26.641 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:56:26.684 INFO:teuthology.orchestra.run.smithi078.stderr:noscrub is set 2023-10-01T10:56:26.742 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:26.775 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:26.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:26.778 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:56:26.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:26 smithi173 bash[58816]: audit 2023-10-01T10:56:25.907623+0000 mon.c (mon.2) 188 : audit 1 from='client.? 172.21.15.78:0/2146795744' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:56:26.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:26 smithi173 bash[58816]: audit 2023-10-01T10:56:25.907971+0000 mon.a (mon.0) 1475 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:56:26.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:26 smithi173 bash[58816]: audit 2023-10-01T10:56:26.344184+0000 mon.c (mon.2) 189 : audit 0 from='client.? 172.21.15.78:0/1384637078' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:56:26.879 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:26.914 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:26.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:26.917 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:56:27.018 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:27.052 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:27.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:27.054 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:56:27.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:26 smithi078 bash[75299]: audit 2023-10-01T10:56:25.907623+0000 mon.c (mon.2) 188 : audit 1 from='client.? 172.21.15.78:0/2146795744' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:56:27.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:26 smithi078 bash[75299]: audit 2023-10-01T10:56:25.907971+0000 mon.a (mon.0) 1475 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:56:27.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:26 smithi078 bash[75299]: audit 2023-10-01T10:56:26.344184+0000 mon.c (mon.2) 189 : audit 0 from='client.? 172.21.15.78:0/1384637078' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:56:27.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:26 smithi078 bash[78274]: audit 2023-10-01T10:56:25.907623+0000 mon.c (mon.2) 188 : audit 1 from='client.? 172.21.15.78:0/2146795744' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:56:27.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:26 smithi078 bash[78274]: audit 2023-10-01T10:56:25.907971+0000 mon.a (mon.0) 1475 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:56:27.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:26 smithi078 bash[78274]: audit 2023-10-01T10:56:26.344184+0000 mon.c (mon.2) 189 : audit 0 from='client.? 172.21.15.78:0/1384637078' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:56:27.155 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:27.179 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:27.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:27.182 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:56:27.283 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:27.311 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:27.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:27.313 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:56:27.414 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:27.443 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:27.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:27.445 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:56:27.546 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:27.575 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:27.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:27.579 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:56:27.680 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:27.706 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:27.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:27.709 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:56:27.810 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:27.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:27 smithi078 bash[75299]: cluster 2023-10-01T10:56:26.065479+0000 mgr.y (mgr.25072) 1168 : cluster 0 pgmap v1177: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-10-01T10:56:27.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:27 smithi078 bash[75299]: cluster 2023-10-01T10:56:26.665461+0000 mon.a (mon.0) 1476 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:56:27.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:27 smithi078 bash[75299]: cluster 2023-10-01T10:56:26.675564+0000 osd.6 (osd.6) 183 : cluster 3 failed to encode map e242 with expected crc 2023-10-01T10:56:27.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:27 smithi078 bash[75299]: audit 2023-10-01T10:56:26.677486+0000 mon.a (mon.0) 1477 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T10:56:27.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:27 smithi078 bash[75299]: cluster 2023-10-01T10:56:26.677602+0000 mon.a (mon.0) 1478 : cluster 0 osdmap e242: 8 total, 8 up, 8 in 2023-10-01T10:56:27.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:27 smithi078 bash[78274]: cluster 2023-10-01T10:56:26.065479+0000 mgr.y (mgr.25072) 1168 : cluster 0 pgmap v1177: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-10-01T10:56:27.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:27 smithi078 bash[78274]: cluster 2023-10-01T10:56:26.665461+0000 mon.a (mon.0) 1476 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:56:27.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:27 smithi078 bash[78274]: cluster 2023-10-01T10:56:26.675564+0000 osd.6 (osd.6) 183 : cluster 3 failed to encode map e242 with expected crc 2023-10-01T10:56:27.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:27 smithi078 bash[78274]: audit 2023-10-01T10:56:26.677486+0000 mon.a (mon.0) 1477 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T10:56:27.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:27 smithi078 bash[78274]: cluster 2023-10-01T10:56:26.677602+0000 mon.a (mon.0) 1478 : cluster 0 osdmap e242: 8 total, 8 up, 8 in 2023-10-01T10:56:27.842 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:27.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:27.844 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:56:27.945 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:27.958 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:27 smithi173 bash[58816]: cluster 2023-10-01T10:56:26.065479+0000 mgr.y (mgr.25072) 1168 : cluster 0 pgmap v1177: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-10-01T10:56:27.958 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:27 smithi173 bash[58816]: cluster 2023-10-01T10:56:26.665461+0000 mon.a (mon.0) 1476 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:56:27.958 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:27 smithi173 bash[58816]: cluster 2023-10-01T10:56:26.675564+0000 osd.6 (osd.6) 183 : cluster 3 failed to encode map e242 with expected crc 2023-10-01T10:56:27.958 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:27 smithi173 bash[58816]: audit 2023-10-01T10:56:26.677486+0000 mon.a (mon.0) 1477 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T10:56:27.959 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:27 smithi173 bash[58816]: cluster 2023-10-01T10:56:26.677602+0000 mon.a (mon.0) 1478 : cluster 0 osdmap e242: 8 total, 8 up, 8 in 2023-10-01T10:56:27.975 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:27.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:27.978 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:56:28.079 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:28.104 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:28.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:28.107 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:28.208 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:28.235 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:28.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:28.237 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:56:28.338 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:28.373 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:28.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:28.376 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:56:28.477 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:28.509 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:28.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:28.512 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:56:28.613 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:28.635 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:28.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:28.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:56:28.738 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:28.768 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:28.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:28.771 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:56:28.872 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:28.903 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:28.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:28.905 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:56:28.956 INFO:teuthology.orchestra.run.smithi078.stdout:ERROR: (22) Invalid argument 2023-10-01T10:56:28.956 INFO:teuthology.orchestra.run.smithi078.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T10:56:29.006 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:29.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:28 smithi078 bash[78274]: cluster 2023-10-01T10:56:27.400067+0000 osd.5 (osd.5) 226 : cluster 3 failed to encode map e242 with expected crc 2023-10-01T10:56:29.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:28 smithi078 bash[78274]: cluster 2023-10-01T10:56:27.400220+0000 osd.7 (osd.7) 225 : cluster 3 failed to encode map e242 with expected crc 2023-10-01T10:56:29.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:28 smithi078 bash[78274]: audit 2023-10-01T10:56:28.036602+0000 mon.a (mon.0) 1479 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:56:29.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:28 smithi078 bash[75299]: cluster 2023-10-01T10:56:27.400067+0000 osd.5 (osd.5) 226 : cluster 3 failed to encode map e242 with expected crc 2023-10-01T10:56:29.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:28 smithi078 bash[75299]: cluster 2023-10-01T10:56:27.400220+0000 osd.7 (osd.7) 225 : cluster 3 failed to encode map e242 with expected crc 2023-10-01T10:56:29.015 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:28 smithi078 bash[75299]: audit 2023-10-01T10:56:28.036602+0000 mon.a (mon.0) 1479 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:56:29.028 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:29.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:29.030 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:56:29.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:28 smithi173 bash[58816]: cluster 2023-10-01T10:56:27.400067+0000 osd.5 (osd.5) 226 : cluster 3 failed to encode map e242 with expected crc 2023-10-01T10:56:29.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:28 smithi173 bash[58816]: cluster 2023-10-01T10:56:27.400220+0000 osd.7 (osd.7) 225 : cluster 3 failed to encode map e242 with expected crc 2023-10-01T10:56:29.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:28 smithi173 bash[58816]: audit 2023-10-01T10:56:28.036602+0000 mon.a (mon.0) 1479 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:56:29.131 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:29.160 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:29.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:29.163 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:56:29.264 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:29.294 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:29.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:29.297 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:56:29.398 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:29.424 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:29.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:29.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:56:29.527 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:29.561 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:29.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:29.564 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:56:29.665 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:29.694 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:29.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:29.697 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:29.798 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:29.811 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:29 smithi173 bash[58816]: cluster 2023-10-01T10:56:28.066032+0000 mgr.y (mgr.25072) 1169 : cluster 0 pgmap v1179: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-10-01T10:56:29.831 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:29.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:29.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:56:29.935 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:29.948 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:29 smithi078 bash[78274]: cluster 2023-10-01T10:56:28.066032+0000 mgr.y (mgr.25072) 1169 : cluster 0 pgmap v1179: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-10-01T10:56:29.948 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:29 smithi078 bash[75299]: cluster 2023-10-01T10:56:28.066032+0000 mgr.y (mgr.25072) 1169 : cluster 0 pgmap v1179: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-10-01T10:56:29.965 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:29.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:29.968 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:56:30.069 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:30.100 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:30.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:30.103 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:56:30.203 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:30.229 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:30.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:30.232 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:56:30.333 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:30.365 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:30.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:30.369 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:56:30.402 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-01T10:56:30.469 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:30.481 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:56:30 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:56:30] "GET /metrics HTTP/1.1" 200 33965 "" "Prometheus/2.33.4" 2023-10-01T10:56:30.499 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:30.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:30.501 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:56:30.602 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:30.627 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:30.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:30.629 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:56:30.730 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:30.764 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:30.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:30.767 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:30.868 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:30.901 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:30.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:30.903 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:56:31.004 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:31.038 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:31.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:31.041 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:56:31.141 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:31.173 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:31.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:31.177 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:56:31.260 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T10:56:31.277 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:31.301 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:31.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:31.304 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:56:31.404 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:31.439 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:31.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:31.441 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:56:31.542 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:31.574 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:31.576 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T10:56:31.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:31.577 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:56:31.679 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:31.714 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:31.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:31.717 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:56:31.818 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:31.830 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:31 smithi173 bash[58816]: cluster 2023-10-01T10:56:30.067133+0000 mgr.y (mgr.25072) 1170 : cluster 0 pgmap v1180: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-10-01T10:56:31.851 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:31.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:31.854 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:56:31.954 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:31.987 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:31.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:31.990 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:56:32.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:31 smithi078 bash[75299]: cluster 2023-10-01T10:56:30.067133+0000 mgr.y (mgr.25072) 1170 : cluster 0 pgmap v1180: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-10-01T10:56:32.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:31 smithi078 bash[78274]: cluster 2023-10-01T10:56:30.067133+0000 mgr.y (mgr.25072) 1170 : cluster 0 pgmap v1180: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-10-01T10:56:32.092 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:32.125 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:32.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:32.129 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:56:32.229 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:32.263 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:32.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:32.266 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:56:32.367 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:32.402 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:32.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:32.405 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:56:32.505 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:32.540 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:32.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:32.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:56:32.643 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:32.669 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:32.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:32.672 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:56:32.773 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:32.805 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:32.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:32.809 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:56:32.909 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:32.941 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:32.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:32.944 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:33.045 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:33.077 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:33.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:33.082 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:56:33.151 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:56:33.183 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:33.205 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:33.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:33.207 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:56:33.307 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:33.340 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:33.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:33.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:56:33.444 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:33.473 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:33.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:33.476 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:33.577 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:33.590 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:33 smithi173 bash[58816]: cluster 2023-10-01T10:56:32.067802+0000 mgr.y (mgr.25072) 1171 : cluster 0 pgmap v1181: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-10-01T10:56:33.590 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:33 smithi173 bash[58816]: audit 2023-10-01T10:56:32.453107+0000 mon.a (mon.0) 1480 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:56:33.591 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:33 smithi173 bash[58816]: audit 2023-10-01T10:56:32.454537+0000 mon.a (mon.0) 1481 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:56:33.610 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:33.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:33.613 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:56:33.714 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:33.749 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:33.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:33.752 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:56:33.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:33 smithi078 bash[78274]: cluster 2023-10-01T10:56:32.067802+0000 mgr.y (mgr.25072) 1171 : cluster 0 pgmap v1181: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-10-01T10:56:33.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:33 smithi078 bash[78274]: audit 2023-10-01T10:56:32.453107+0000 mon.a (mon.0) 1480 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:56:33.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:33 smithi078 bash[78274]: audit 2023-10-01T10:56:32.454537+0000 mon.a (mon.0) 1481 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:56:33.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:33 smithi078 bash[75299]: cluster 2023-10-01T10:56:32.067802+0000 mgr.y (mgr.25072) 1171 : cluster 0 pgmap v1181: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-10-01T10:56:33.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:33 smithi078 bash[75299]: audit 2023-10-01T10:56:32.453107+0000 mon.a (mon.0) 1480 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:56:33.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:33 smithi078 bash[75299]: audit 2023-10-01T10:56:32.454537+0000 mon.a (mon.0) 1481 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:56:33.853 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:33.884 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:33.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:33.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:56:33.988 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:34.022 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:34.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:34.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:56:34.127 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:34.159 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:34.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:34.162 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:56:34.263 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:34.289 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:34.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:34.292 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:56:34.392 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:34.421 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:34.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:34.424 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:56:34.525 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:34.555 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:34.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:34.559 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:56:34.660 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:34.695 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:34.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:34.698 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:56:34.801 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:34.835 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:34.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:34.838 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:56:34.939 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:34.965 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:34.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:34.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:56:35.069 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:35.103 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:35.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:35.106 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:56:35.206 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:35.238 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:35.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:35.241 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:56:35.342 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:35.374 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:35.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:35.376 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:56:35.477 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:35.506 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:35.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:35.510 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:56:35.611 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:35.643 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:35.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:35.646 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:56:35.747 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:35.759 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:35 smithi173 bash[58816]: cluster 2023-10-01T10:56:34.068853+0000 mgr.y (mgr.25072) 1172 : cluster 0 pgmap v1182: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-10-01T10:56:35.780 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:35.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:35.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:56:35.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:35 smithi078 bash[75299]: cluster 2023-10-01T10:56:34.068853+0000 mgr.y (mgr.25072) 1172 : cluster 0 pgmap v1182: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-10-01T10:56:35.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:35 smithi078 bash[78274]: cluster 2023-10-01T10:56:34.068853+0000 mgr.y (mgr.25072) 1172 : cluster 0 pgmap v1182: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-10-01T10:56:35.884 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:35.912 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:35.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:35.914 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:56:36.015 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:36.049 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:36.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:36.052 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:56:36.063 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:56:36.063 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:56:36.153 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:36.181 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:36.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:36.184 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:56:36.201 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:56:36.285 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:36.320 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:36.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:36.324 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:56:36.425 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:36.456 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:36.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:36.458 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:56:36.559 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:36.587 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:36.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:36.589 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:56:36.689 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:36.722 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:36.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:36.725 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:56:36.826 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:36.857 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:36.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:36.859 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:56:36.960 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:36.985 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:36.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:36.988 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:56:37.089 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:37.123 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:37.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:37.127 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:56:37.227 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:37.251 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:37.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:37.253 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:56:37.354 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:37.384 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:37.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:37.387 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:56:37.488 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:37.516 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:37.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:37.519 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:37.545 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:37 smithi078 bash[75299]: cluster 2023-10-01T10:56:36.069524+0000 mgr.y (mgr.25072) 1173 : cluster 0 pgmap v1183: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-10-01T10:56:37.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:37 smithi173 bash[58816]: cluster 2023-10-01T10:56:36.069524+0000 mgr.y (mgr.25072) 1173 : cluster 0 pgmap v1183: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-10-01T10:56:37.619 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:37.650 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:37.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:37.653 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:56:37.754 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:37.779 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:37.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:37.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:37.803 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:37 smithi078 bash[78274]: cluster 2023-10-01T10:56:36.069524+0000 mgr.y (mgr.25072) 1173 : cluster 0 pgmap v1183: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-10-01T10:56:37.882 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:37.906 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:37.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:37.908 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:56:38.009 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:38.046 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:38.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:38.050 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:56:38.151 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:38.175 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:38.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:38.177 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:38.278 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:38.303 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:38.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:38.306 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:56:38.406 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:38.431 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:38.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:38.434 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:38.534 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:38.556 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:38.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:38.559 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:56:38.660 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:38.686 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:38.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:38.690 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:56:38.790 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:38.821 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:38.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:38.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:56:38.924 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:38.960 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:38.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:38.963 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:56:39.064 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:39.095 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:39.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:39.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:56:39.199 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:39.221 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:39.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:39.223 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:56:39.324 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:39.349 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:39.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:39.352 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:56:39.453 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:39.481 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:39.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:39.484 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:56:39.585 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:39.620 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:39.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:39.623 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:56:39.723 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:39.756 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:39.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:39.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:56:39.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:39 smithi078 bash[75299]: cluster 2023-10-01T10:56:38.070238+0000 mgr.y (mgr.25072) 1174 : cluster 0 pgmap v1184: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-10-01T10:56:39.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:39 smithi078 bash[78274]: cluster 2023-10-01T10:56:38.070238+0000 mgr.y (mgr.25072) 1174 : cluster 0 pgmap v1184: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-10-01T10:56:39.861 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:39.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:39 smithi173 bash[58816]: cluster 2023-10-01T10:56:38.070238+0000 mgr.y (mgr.25072) 1174 : cluster 0 pgmap v1184: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-10-01T10:56:39.893 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:39.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:39.895 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:56:39.996 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:40.018 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:40.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:40.020 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:56:40.121 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:40.151 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:40.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:40.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:56:40.255 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:40.284 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:40.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:40.286 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:56:40.375 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:56:40 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:56:40] "GET /metrics HTTP/1.1" 200 36317 "" "Prometheus/2.33.4" 2023-10-01T10:56:40.387 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:40.421 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:40.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:40.425 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:56:40.526 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:40.551 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:40.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:40.553 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:40.598 INFO:teuthology.orchestra.run.smithi078.stderr:enabled application 'rados' on pool 'unique_pool_7' 2023-10-01T10:56:40.627 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:40 smithi078 bash[75299]: audit 2023-10-01T10:56:39.835190+0000 mon.c (mon.2) 190 : audit 1 from='client.? 172.21.15.78:0/4034214806' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T10:56:40.627 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:40 smithi078 bash[75299]: audit 2023-10-01T10:56:39.835792+0000 mon.a (mon.0) 1482 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T10:56:40.627 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:40 smithi078 bash[78274]: audit 2023-10-01T10:56:39.835190+0000 mon.c (mon.2) 190 : audit 1 from='client.? 172.21.15.78:0/4034214806' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T10:56:40.627 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:40 smithi078 bash[78274]: audit 2023-10-01T10:56:39.835792+0000 mon.a (mon.0) 1482 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T10:56:40.655 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:40.668 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:40 smithi173 bash[58816]: audit 2023-10-01T10:56:39.835190+0000 mon.c (mon.2) 190 : audit 1 from='client.? 172.21.15.78:0/4034214806' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T10:56:40.668 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:40 smithi173 bash[58816]: audit 2023-10-01T10:56:39.835792+0000 mon.a (mon.0) 1482 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T10:56:40.689 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:40.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:40.691 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:56:40.792 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:40.830 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:40.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:40.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:56:40.934 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:40.965 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:40.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:40.969 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:56:41.070 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:41.109 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:41.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:41.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:56:41.213 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:41.240 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:41.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:41.242 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:41.343 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:41.368 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:41.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:41.371 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:41.472 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:41.505 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:41.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:41.508 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:56:41.609 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:41.644 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:41.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:41.647 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:56:41.748 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:41.778 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:41.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:41.781 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:56:41.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:41 smithi078 bash[78274]: cluster 2023-10-01T10:56:40.071283+0000 mgr.y (mgr.25072) 1175 : cluster 0 pgmap v1185: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-10-01T10:56:41.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:41 smithi078 bash[78274]: cluster 2023-10-01T10:56:40.591202+0000 osd.7 (osd.7) 226 : cluster 3 failed to encode map e243 with expected crc 2023-10-01T10:56:41.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:41 smithi078 bash[78274]: audit 2023-10-01T10:56:40.592552+0000 mon.a (mon.0) 1483 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-01T10:56:41.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:41 smithi078 bash[78274]: cluster 2023-10-01T10:56:40.592614+0000 mon.a (mon.0) 1484 : cluster 0 osdmap e243: 8 total, 8 up, 8 in 2023-10-01T10:56:41.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:41 smithi078 bash[78274]: cluster 2023-10-01T10:56:41.275457+0000 osd.6 (osd.6) 184 : cluster 3 failed to encode map e243 with expected crc 2023-10-01T10:56:41.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:41 smithi078 bash[75299]: cluster 2023-10-01T10:56:40.071283+0000 mgr.y (mgr.25072) 1175 : cluster 0 pgmap v1185: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-10-01T10:56:41.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:41 smithi078 bash[75299]: cluster 2023-10-01T10:56:40.591202+0000 osd.7 (osd.7) 226 : cluster 3 failed to encode map e243 with expected crc 2023-10-01T10:56:41.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:41 smithi078 bash[75299]: audit 2023-10-01T10:56:40.592552+0000 mon.a (mon.0) 1483 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-01T10:56:41.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:41 smithi078 bash[75299]: cluster 2023-10-01T10:56:40.592614+0000 mon.a (mon.0) 1484 : cluster 0 osdmap e243: 8 total, 8 up, 8 in 2023-10-01T10:56:41.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:41 smithi078 bash[75299]: cluster 2023-10-01T10:56:41.275457+0000 osd.6 (osd.6) 184 : cluster 3 failed to encode map e243 with expected crc 2023-10-01T10:56:41.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:41 smithi173 bash[58816]: cluster 2023-10-01T10:56:40.071283+0000 mgr.y (mgr.25072) 1175 : cluster 0 pgmap v1185: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-10-01T10:56:41.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:41 smithi173 bash[58816]: cluster 2023-10-01T10:56:40.591202+0000 osd.7 (osd.7) 226 : cluster 3 failed to encode map e243 with expected crc 2023-10-01T10:56:41.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:41 smithi173 bash[58816]: audit 2023-10-01T10:56:40.592552+0000 mon.a (mon.0) 1483 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-01T10:56:41.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:41 smithi173 bash[58816]: cluster 2023-10-01T10:56:40.592614+0000 mon.a (mon.0) 1484 : cluster 0 osdmap e243: 8 total, 8 up, 8 in 2023-10-01T10:56:41.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:41 smithi173 bash[58816]: cluster 2023-10-01T10:56:41.275457+0000 osd.6 (osd.6) 184 : cluster 3 failed to encode map e243 with expected crc 2023-10-01T10:56:41.881 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:41.911 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:41.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:41.914 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:56:42.015 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:42.039 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:42.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:42.042 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:56:42.142 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:42.167 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:42.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:42.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:42.270 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:42.301 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:42.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:42.303 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:56:42.404 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:42.435 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:42.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:42.437 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:56:42.538 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:42.572 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:42.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:42.576 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:56:42.677 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:42.702 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:42.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:42.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:56:42.806 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:42.817 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:42 smithi173 bash[58816]: cluster 2023-10-01T10:56:41.275422+0000 osd.5 (osd.5) 227 : cluster 3 failed to encode map e243 with expected crc 2023-10-01T10:56:42.837 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:42.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:42.840 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:56:42.941 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:42.954 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:42 smithi078 bash[75299]: cluster 2023-10-01T10:56:41.275422+0000 osd.5 (osd.5) 227 : cluster 3 failed to encode map e243 with expected crc 2023-10-01T10:56:42.954 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:42 smithi078 bash[78274]: cluster 2023-10-01T10:56:41.275422+0000 osd.5 (osd.5) 227 : cluster 3 failed to encode map e243 with expected crc 2023-10-01T10:56:42.969 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:42.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:42.972 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:43.072 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:43.099 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:43.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:43.101 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:56:43.202 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:43.235 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:43.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:43.239 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:56:43.340 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:43.369 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:43.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:43.371 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:56:43.472 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:43.504 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:43.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:43.507 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:56:43.608 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:43.638 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:43.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:43.641 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:56:43.742 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:43.775 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:43.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:43.779 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:56:43.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:43 smithi078 bash[75299]: cluster 2023-10-01T10:56:42.071974+0000 mgr.y (mgr.25072) 1176 : cluster 0 pgmap v1187: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-10-01T10:56:43.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:43 smithi078 bash[78274]: cluster 2023-10-01T10:56:42.071974+0000 mgr.y (mgr.25072) 1176 : cluster 0 pgmap v1187: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-10-01T10:56:43.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:43 smithi173 bash[58816]: cluster 2023-10-01T10:56:42.071974+0000 mgr.y (mgr.25072) 1176 : cluster 0 pgmap v1187: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-10-01T10:56:43.879 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:43.910 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:43.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:43.914 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:56:44.014 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:44.045 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:44.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:44.048 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:56:44.149 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:44.176 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:44.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:44.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:56:44.280 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:44.313 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:44.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:44.316 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:56:44.416 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:44.441 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:44.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:44.443 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:44.544 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:44.580 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:44.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:44.584 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:56:44.684 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:44.716 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:44.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:44.719 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:56:44.819 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:44.843 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:44.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:44.846 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:56:44.946 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:44.968 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:44.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:44.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:56:45.071 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:45.092 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:45.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:45.094 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:56:45.195 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:45.220 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:45.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:45.222 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:56:45.323 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:45.354 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:45.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:45.357 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:56:45.457 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:45.482 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:45.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:45.484 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:45.585 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:45.616 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:45.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:45.619 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:56:45.720 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:45.752 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:45.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:45.755 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:56:45.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:45 smithi078 bash[78274]: cluster 2023-10-01T10:56:44.072901+0000 mgr.y (mgr.25072) 1177 : cluster 0 pgmap v1188: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-10-01T10:56:45.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:45 smithi078 bash[75299]: cluster 2023-10-01T10:56:44.072901+0000 mgr.y (mgr.25072) 1177 : cluster 0 pgmap v1188: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-10-01T10:56:45.856 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:45.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:45 smithi173 bash[58816]: cluster 2023-10-01T10:56:44.072901+0000 mgr.y (mgr.25072) 1177 : cluster 0 pgmap v1188: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-10-01T10:56:45.887 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:45.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:45.889 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:56:45.918 DEBUG:teuthology.orchestra.run.smithi078:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_7 bench 90 write' 2023-10-01T10:56:45.919 INFO:tasks.thrashosds.thrasher:fixing pg num pool device_health_metrics 2023-10-01T10:56:45.919 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T10:56:45.939 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-01T10:56:45.939 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-01T10:56:45.990 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:46.022 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:46.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:46.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:56:46.038 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:hints = 1 2023-10-01T10:56:46.038 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-01T10:56:46.038 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Object prefix: benchmark_data_smithi078_180197 2023-10-01T10:56:46.039 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T10:56:46.039 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 0 0 0 0 0 0 - 0 2023-10-01T10:56:46.125 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:46.153 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:46.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:46.156 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:56:46.257 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:46.286 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:46.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:46.290 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:56:46.391 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:46.420 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:46.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:46.424 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:56:46.525 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:46.547 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:46.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:46.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:46.650 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:46.676 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:46.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:46.678 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:56:46.779 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:46.805 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:46.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:46.807 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:46.908 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:46.933 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:46.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:46.937 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:56:47.038 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:47.039 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 1 16 2403 2387 149.179 149.188 0.00374543 0.0066746 2023-10-01T10:56:47.064 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:47.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:47.068 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:56:47.169 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:47.194 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:47.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:47.196 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:47.297 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:47.324 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:47.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:47.325 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:47.426 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:47.460 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:47.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:47.462 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:56:47.563 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:47.592 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:47.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:47.595 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:56:47.696 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:47.720 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:47.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:47.722 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:56:47.823 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:47.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:47 smithi078 bash[75299]: cluster 2023-10-01T10:56:46.073477+0000 mgr.y (mgr.25072) 1178 : cluster 0 pgmap v1189: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-10-01T10:56:47.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:47 smithi078 bash[75299]: audit 2023-10-01T10:56:47.446357+0000 mon.a (mon.0) 1485 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:56:47.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:47 smithi078 bash[78274]: cluster 2023-10-01T10:56:46.073477+0000 mgr.y (mgr.25072) 1178 : cluster 0 pgmap v1189: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-10-01T10:56:47.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:47 smithi078 bash[78274]: audit 2023-10-01T10:56:47.446357+0000 mon.a (mon.0) 1485 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:56:47.849 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:47.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:47.851 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:47.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:47 smithi173 bash[58816]: cluster 2023-10-01T10:56:46.073477+0000 mgr.y (mgr.25072) 1178 : cluster 0 pgmap v1189: 89 pgs: 89 active+clean; 582 KiB data, 5.1 GiB used, 710 GiB / 715 GiB avail 2023-10-01T10:56:47.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:47 smithi173 bash[58816]: audit 2023-10-01T10:56:47.446357+0000 mon.a (mon.0) 1485 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:56:47.952 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:47.972 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:47.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:47.974 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:56:48.040 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 2 16 4398 4382 136.924 124.688 0.00335917 0.00701054 2023-10-01T10:56:48.074 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:48.105 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:48.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:48.108 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:56:48.209 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:48.236 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:48.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:48.238 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:56:48.339 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:48.364 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:48.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:48.367 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:56:48.467 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:48.496 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:48.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:48.498 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:56:48.599 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:48.622 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:48.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:48.624 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:56:48.641 INFO:teuthology.orchestra.run.smithi078.stderr:nodeep-scrub is set 2023-10-01T10:56:48.725 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:48.751 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:48.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:48.754 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:56:48.855 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:48.864 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:48 smithi078 bash[78274]: audit 2023-10-01T10:56:48.011538+0000 mon.a (mon.0) 1486 : audit 1 from='client.? 172.21.15.78:0/964484711' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:56:48.864 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:48 smithi078 bash[75299]: audit 2023-10-01T10:56:48.011538+0000 mon.a (mon.0) 1486 : audit 1 from='client.? 172.21.15.78:0/964484711' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:56:48.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:48 smithi173 bash[58816]: audit 2023-10-01T10:56:48.011538+0000 mon.a (mon.0) 1486 : audit 1 from='client.? 172.21.15.78:0/964484711' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:56:48.883 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:48.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:48.886 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:56:48.986 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:49.012 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:49.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:49.015 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:56:49.040 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 3 16 6935 6919 144.131 158.562 0.00799306 0.0069245 2023-10-01T10:56:49.116 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:49.146 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:49.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:49.149 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:56:49.250 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:49.275 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:49.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:49.278 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:56:49.378 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:49.408 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:49.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:49.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:56:49.512 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:49.536 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:49.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:49.539 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:56:49.640 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:49.664 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:49.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:49.666 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:56:49.767 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:49.789 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:49.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:49.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:49.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:49 smithi173 bash[58816]: cluster 2023-10-01T10:56:48.074001+0000 mgr.y (mgr.25072) 1179 : cluster 0 pgmap v1190: 89 pgs: 89 active+clean; 51 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 5.1 MiB/s wr, 80 op/s 2023-10-01T10:56:49.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:49 smithi173 bash[58816]: cluster 2023-10-01T10:56:48.611977+0000 mon.a (mon.0) 1487 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:56:49.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:49 smithi173 bash[58816]: audit 2023-10-01T10:56:48.634684+0000 mon.a (mon.0) 1488 : audit 1 from='client.? 172.21.15.78:0/964484711' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T10:56:49.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:49 smithi173 bash[58816]: cluster 2023-10-01T10:56:48.635021+0000 mon.a (mon.0) 1489 : cluster 0 osdmap e244: 8 total, 8 up, 8 in 2023-10-01T10:56:49.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:49 smithi173 bash[58816]: cluster 2023-10-01T10:56:48.635248+0000 osd.6 (osd.6) 185 : cluster 3 failed to encode map e244 with expected crc 2023-10-01T10:56:49.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:49 smithi173 bash[58816]: cluster 2023-10-01T10:56:48.637382+0000 osd.6 (osd.6) 186 : cluster 3 failed to encode map e244 with expected crc 2023-10-01T10:56:49.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:49 smithi173 bash[58816]: cluster 2023-10-01T10:56:48.639155+0000 osd.7 (osd.7) 227 : cluster 3 failed to encode map e244 with expected crc 2023-10-01T10:56:49.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:49 smithi173 bash[58816]: cluster 2023-10-01T10:56:48.639911+0000 osd.6 (osd.6) 187 : cluster 3 failed to encode map e244 with expected crc 2023-10-01T10:56:49.892 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:49.902 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:49 smithi078 bash[78274]: cluster 2023-10-01T10:56:48.074001+0000 mgr.y (mgr.25072) 1179 : cluster 0 pgmap v1190: 89 pgs: 89 active+clean; 51 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 5.1 MiB/s wr, 80 op/s 2023-10-01T10:56:49.902 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:49 smithi078 bash[78274]: cluster 2023-10-01T10:56:48.611977+0000 mon.a (mon.0) 1487 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:56:49.902 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:49 smithi078 bash[78274]: audit 2023-10-01T10:56:48.634684+0000 mon.a (mon.0) 1488 : audit 1 from='client.? 172.21.15.78:0/964484711' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T10:56:49.902 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:49 smithi078 bash[78274]: cluster 2023-10-01T10:56:48.635021+0000 mon.a (mon.0) 1489 : cluster 0 osdmap e244: 8 total, 8 up, 8 in 2023-10-01T10:56:49.902 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:49 smithi078 bash[78274]: cluster 2023-10-01T10:56:48.635248+0000 osd.6 (osd.6) 185 : cluster 3 failed to encode map e244 with expected crc 2023-10-01T10:56:49.902 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:49 smithi078 bash[78274]: cluster 2023-10-01T10:56:48.637382+0000 osd.6 (osd.6) 186 : cluster 3 failed to encode map e244 with expected crc 2023-10-01T10:56:49.903 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:49 smithi078 bash[78274]: cluster 2023-10-01T10:56:48.639155+0000 osd.7 (osd.7) 227 : cluster 3 failed to encode map e244 with expected crc 2023-10-01T10:56:49.903 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:49 smithi078 bash[78274]: cluster 2023-10-01T10:56:48.639911+0000 osd.6 (osd.6) 187 : cluster 3 failed to encode map e244 with expected crc 2023-10-01T10:56:49.903 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:49 smithi078 bash[75299]: cluster 2023-10-01T10:56:48.074001+0000 mgr.y (mgr.25072) 1179 : cluster 0 pgmap v1190: 89 pgs: 89 active+clean; 51 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 5.1 MiB/s wr, 80 op/s 2023-10-01T10:56:49.903 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:49 smithi078 bash[75299]: cluster 2023-10-01T10:56:48.611977+0000 mon.a (mon.0) 1487 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:56:49.903 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:49 smithi078 bash[75299]: audit 2023-10-01T10:56:48.634684+0000 mon.a (mon.0) 1488 : audit 1 from='client.? 172.21.15.78:0/964484711' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T10:56:49.903 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:49 smithi078 bash[75299]: cluster 2023-10-01T10:56:48.635021+0000 mon.a (mon.0) 1489 : cluster 0 osdmap e244: 8 total, 8 up, 8 in 2023-10-01T10:56:49.903 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:49 smithi078 bash[75299]: cluster 2023-10-01T10:56:48.635248+0000 osd.6 (osd.6) 185 : cluster 3 failed to encode map e244 with expected crc 2023-10-01T10:56:49.903 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:49 smithi078 bash[75299]: cluster 2023-10-01T10:56:48.637382+0000 osd.6 (osd.6) 186 : cluster 3 failed to encode map e244 with expected crc 2023-10-01T10:56:49.903 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:49 smithi078 bash[75299]: cluster 2023-10-01T10:56:48.639155+0000 osd.7 (osd.7) 227 : cluster 3 failed to encode map e244 with expected crc 2023-10-01T10:56:49.903 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:49 smithi078 bash[75299]: cluster 2023-10-01T10:56:48.639911+0000 osd.6 (osd.6) 187 : cluster 3 failed to encode map e244 with expected crc 2023-10-01T10:56:49.922 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:49.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:49.926 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:56:50.026 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:50.040 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 4 16 9206 9190 143.58 141.938 0.00484877 0.00695495 2023-10-01T10:56:50.050 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:50.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:50.052 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:56:50.153 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:50.179 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:50.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:50.186 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:50.229 INFO:teuthology.orchestra.run.smithi078.stdout:ERROR: (22) Invalid argument 2023-10-01T10:56:50.229 INFO:teuthology.orchestra.run.smithi078.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T10:56:50.287 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:50.315 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:50.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:50.318 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:56:50.419 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:50.446 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:50.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:50.449 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:56:50.550 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:50.559 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:56:50 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:56:50] "GET /metrics HTTP/1.1" 200 36314 "" "Prometheus/2.33.4" 2023-10-01T10:56:50.577 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:50.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:50.580 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:56:50.681 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:50.708 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:50.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:50.710 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:56:50.811 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:50.831 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:50.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:50.833 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:56:50.934 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:50.963 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:50.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:50.966 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:56:51.040 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 5 16 11542 11526 144.061 146 0.00374372 0.00693122 2023-10-01T10:56:51.067 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:51.084 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:51.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:51.085 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:56:51.186 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:51.213 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:51.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:51.216 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:56:51.316 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:51.336 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:51.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:51.338 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:56:51.439 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:51.467 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:51.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:51.470 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:51.571 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:51.597 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:51.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:51.599 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:56:51.700 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:51.731 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:51.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:51.733 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:56:51.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:51 smithi078 bash[75299]: cluster 2023-10-01T10:56:50.074891+0000 mgr.y (mgr.25072) 1180 : cluster 0 pgmap v1192: 89 pgs: 89 active+clean; 356 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 37 MiB/s wr, 599 op/s 2023-10-01T10:56:51.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:51 smithi078 bash[78274]: cluster 2023-10-01T10:56:50.074891+0000 mgr.y (mgr.25072) 1180 : cluster 0 pgmap v1192: 89 pgs: 89 active+clean; 356 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 37 MiB/s wr, 599 op/s 2023-10-01T10:56:51.834 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:51.845 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:51 smithi173 bash[58816]: cluster 2023-10-01T10:56:50.074891+0000 mgr.y (mgr.25072) 1180 : cluster 0 pgmap v1192: 89 pgs: 89 active+clean; 356 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 37 MiB/s wr, 599 op/s 2023-10-01T10:56:51.860 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:51.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:51.864 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:56:51.964 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:51.994 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:52.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:52.001 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:56:52.040 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 6 16 13555 13539 141.018 125.812 0.00856007 0.00708246 2023-10-01T10:56:52.102 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:52.128 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:52.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:52.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:56:52.231 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:52.266 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:52.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:52.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:56:52.338 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T10:56:52.356 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T10:56:52.371 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:52.475 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T10:56:52.493 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:52.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:52.496 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:52.510 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T10:56:52.545 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T10:56:52.586 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T10:56:52.597 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:52.621 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T10:56:52.625 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:52.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:52.628 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:56:52.666 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T10:56:52.669 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T10:56:52.669 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T10:56:52.669 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T10:56:52.669 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T10:56:52.669 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T10:56:52.669 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T10:56:52.669 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T10:56:52.670 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T10:56:52.728 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:52.759 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:52.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:52.762 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:56:52.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:52 smithi078 bash[75299]: audit 2023-10-01T10:56:52.317690+0000 mon.c (mon.2) 191 : audit 0 from='client.? 172.21.15.78:0/3599188682' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:56:52.834 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:52 smithi078 bash[78274]: audit 2023-10-01T10:56:52.317690+0000 mon.c (mon.2) 191 : audit 0 from='client.? 172.21.15.78:0/3599188682' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:56:52.863 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:52.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:52 smithi173 bash[58816]: audit 2023-10-01T10:56:52.317690+0000 mon.c (mon.2) 191 : audit 0 from='client.? 172.21.15.78:0/3599188682' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:56:52.898 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:52.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:52.903 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:56:53.004 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:53.033 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:53.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:53.038 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:56:53.040 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 7 16 15296 15280 136.416 108.812 0.00496816 0.0073239 2023-10-01T10:56:53.138 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:53.165 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:53.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:53.169 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:56:53.270 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:53.303 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:53.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:53.307 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:53.407 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:53.434 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:53.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:53.438 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:56:53.539 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:53.565 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:53.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:53.568 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:56:53.669 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:53.678 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:53 smithi173 bash[58816]: cluster 2023-10-01T10:56:52.075410+0000 mgr.y (mgr.25072) 1181 : cluster 0 pgmap v1193: 89 pgs: 89 active+clean; 356 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 36 MiB/s wr, 568 op/s 2023-10-01T10:56:53.691 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:53.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:53.694 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:56:53.795 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:53.823 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:53.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:53.826 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:56:53.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:53 smithi078 bash[75299]: cluster 2023-10-01T10:56:52.075410+0000 mgr.y (mgr.25072) 1181 : cluster 0 pgmap v1193: 89 pgs: 89 active+clean; 356 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 36 MiB/s wr, 568 op/s 2023-10-01T10:56:53.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:53 smithi078 bash[78274]: cluster 2023-10-01T10:56:52.075410+0000 mgr.y (mgr.25072) 1181 : cluster 0 pgmap v1193: 89 pgs: 89 active+clean; 356 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 36 MiB/s wr, 568 op/s 2023-10-01T10:56:53.928 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:53.957 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:53.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:53.962 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:56:54.040 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 8 16 17168 17152 133.987 117 0.0153259 0.00745236 2023-10-01T10:56:54.063 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:54.091 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:54.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:54.093 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:56:54.194 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:54.222 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:54.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:54.225 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:56:54.326 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:54.351 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:54.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:54.353 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:54.454 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:54.482 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:54.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:54.485 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:56:54.586 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:54.615 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:54.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:54.619 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:54.720 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:54.746 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:54.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:54.750 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:56:54.851 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:54.883 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:54.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:54.886 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:56:54.987 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:55.011 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:55.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:55.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:56:55.040 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 9 16 18946 18930 131.446 111.125 0.0117669 0.00760202 2023-10-01T10:56:55.114 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:55.137 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:55.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:55.141 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:55.242 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:55.268 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:55.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:55.271 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:56:55.372 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:55.403 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:55.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:55.407 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:55.508 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:55.533 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:55.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:55.536 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:56:55.636 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:55.659 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:55.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:55.663 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:56:55.728 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-01T10:56:55.764 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:55.802 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:55.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:55.806 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:56:55.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:55 smithi173 bash[58816]: cluster 2023-10-01T10:56:54.076135+0000 mgr.y (mgr.25072) 1182 : cluster 0 pgmap v1194: 89 pgs: 89 active+clean; 780 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-01T10:56:55.907 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:55.916 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:55 smithi078 bash[75299]: cluster 2023-10-01T10:56:54.076135+0000 mgr.y (mgr.25072) 1182 : cluster 0 pgmap v1194: 89 pgs: 89 active+clean; 780 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-01T10:56:55.917 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:55 smithi078 bash[78274]: cluster 2023-10-01T10:56:54.076135+0000 mgr.y (mgr.25072) 1182 : cluster 0 pgmap v1194: 89 pgs: 89 active+clean; 780 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-10-01T10:56:55.942 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:55.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:55.945 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:56:56.032 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T10:56:56.041 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 10 16 20808 20792 129.927 116.375 0.0107712 0.00768941 2023-10-01T10:56:56.045 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:56.076 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:56.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:56.078 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:56:56.179 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:56.205 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:56.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:56.240 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:56:56.341 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:56.382 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:56.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:56.387 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:56:56.487 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:56.513 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:56.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:56.516 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:56:56.617 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:56.642 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:56.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:56.645 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:56:56.745 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:56.769 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:56.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:56.772 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:56:56.873 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:56.901 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:56.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:56.904 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:56:57.005 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:57.036 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:57.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:57.040 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:56:57.041 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 11 16 22535 22519 127.924 107.938 0.00983701 0.0078093 2023-10-01T10:56:57.141 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:57.169 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:57.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:57.174 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:56:57.274 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:57.307 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:57.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:57.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:56:57.411 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:57.440 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:57.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:57.442 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:56:57.543 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:57.574 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:57.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:57.579 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:56:57.679 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:57.688 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:57 smithi078 bash[75299]: cluster 2023-10-01T10:56:56.076769+0000 mgr.y (mgr.25072) 1183 : cluster 0 pgmap v1195: 89 pgs: 89 active+clean; 1015 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s 2023-10-01T10:56:57.689 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:57 smithi078 bash[78274]: cluster 2023-10-01T10:56:56.076769+0000 mgr.y (mgr.25072) 1183 : cluster 0 pgmap v1195: 89 pgs: 89 active+clean; 1015 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s 2023-10-01T10:56:57.714 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T10:56:57.715 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:57.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:57.720 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:56:57.820 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:57.830 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:57 smithi173 bash[58816]: cluster 2023-10-01T10:56:56.076769+0000 mgr.y (mgr.25072) 1183 : cluster 0 pgmap v1195: 89 pgs: 89 active+clean; 1015 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s 2023-10-01T10:56:57.848 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:57.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:57.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:56:57.952 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:57.981 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:57.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:57.984 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:56:58.042 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 12 15 24248 24233 126.19 107.125 0.00675842 0.00791738 2023-10-01T10:56:58.085 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:58.113 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:58.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:58.116 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:56:58.216 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:58.244 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:58.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:58.247 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:56:58.348 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:58.373 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:58.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:58.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:56:58.478 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:58.505 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:58.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:58.508 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:56:58.594 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:56:58.609 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:58.651 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:58.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:58.656 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:56:58.757 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:58.787 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:56:58.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:58.791 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:56:58.891 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:58.923 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:56:58.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:58.928 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:56:59.029 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:59.042 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 13 16 25922 25906 124.526 104.562 0.0159793 0.00802312 2023-10-01T10:56:59.057 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:59.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:59.061 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:56:59.162 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:59.189 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:56:59.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:59.193 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:56:59.293 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:59.322 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:56:59.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:59.324 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:56:59.425 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:59.461 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:59.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:59.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:56:59.564 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:59.594 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:56:59.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:59.597 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:56:59.698 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:59.728 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:56:59.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:59.736 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:56:59.837 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:59.847 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:56:59 smithi173 bash[58816]: cluster 2023-10-01T10:56:58.077472+0000 mgr.y (mgr.25072) 1184 : cluster 0 pgmap v1196: 89 pgs: 89 active+clean; 1.1 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2023-10-01T10:56:59.864 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:56:59.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:56:59.867 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:56:59.968 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:56:59.976 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:56:59 smithi078 bash[78274]: cluster 2023-10-01T10:56:58.077472+0000 mgr.y (mgr.25072) 1184 : cluster 0 pgmap v1196: 89 pgs: 89 active+clean; 1.1 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2023-10-01T10:56:59.977 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:56:59 smithi078 bash[75299]: cluster 2023-10-01T10:56:58.077472+0000 mgr.y (mgr.25072) 1184 : cluster 0 pgmap v1196: 89 pgs: 89 active+clean; 1.1 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2023-10-01T10:57:00.043 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 14 16 27532 27516 122.818 100.625 0.0083475 0.00813572 2023-10-01T10:57:00.222 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:00.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:00.224 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:57:00.272 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:57:00 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:57:00] "GET /metrics HTTP/1.1" 200 36314 "" "Prometheus/2.33.4" 2023-10-01T10:57:00.325 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:00.353 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:00.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:00.358 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:57:00.458 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:00.486 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:00.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:00.490 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:00.591 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:00.618 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:00.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:00.620 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:57:00.632 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:57:00.632 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:57:00.721 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:00.752 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:00.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:00.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:57:00.856 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:00.866 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:00 smithi173 bash[58816]: audit 2023-10-01T10:56:59.812993+0000 mon.a (mon.0) 1490 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:57:00.866 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:00 smithi173 bash[58816]: audit 2023-10-01T10:57:00.343440+0000 mon.a (mon.0) 1491 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:57:00.866 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:00 smithi173 bash[58816]: audit 2023-10-01T10:57:00.353220+0000 mon.a (mon.0) 1492 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:57:00.882 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:00.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:00.886 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:00.987 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:00.995 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:00 smithi078 bash[78274]: audit 2023-10-01T10:56:59.812993+0000 mon.a (mon.0) 1490 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:57:00.995 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:00 smithi078 bash[78274]: audit 2023-10-01T10:57:00.343440+0000 mon.a (mon.0) 1491 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:57:00.995 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:00 smithi078 bash[78274]: audit 2023-10-01T10:57:00.353220+0000 mon.a (mon.0) 1492 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:57:00.996 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:00 smithi078 bash[75299]: audit 2023-10-01T10:56:59.812993+0000 mon.a (mon.0) 1490 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:57:00.996 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:00 smithi078 bash[75299]: audit 2023-10-01T10:57:00.343440+0000 mon.a (mon.0) 1491 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:57:00.996 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:00 smithi078 bash[75299]: audit 2023-10-01T10:57:00.353220+0000 mon.a (mon.0) 1492 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:57:01.018 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:01.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:01.023 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:57:01.042 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 15 16 28899 28883 120.325 85.4375 0.00690702 0.00830427 2023-10-01T10:57:01.124 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:01.157 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:01.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:01.160 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:01.261 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:01.284 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:01.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:01.287 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:57:01.387 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:01.445 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:01.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:01.449 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:57:01.549 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:01.655 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:57:01.661 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:01.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:01.668 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:57:01.769 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:01.794 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:01.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:01.796 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:57:01.897 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:01.922 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:01.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:01.926 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:57:01.926 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:01 smithi173 bash[58816]: cluster 2023-10-01T10:57:00.081691+0000 mgr.y (mgr.25072) 1185 : cluster 0 pgmap v1197: 89 pgs: 89 active+clean; 1.5 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 121 MiB/s wr, 1.94k op/s 2023-10-01T10:57:02.027 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:02.037 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:01 smithi078 bash[75299]: cluster 2023-10-01T10:57:00.081691+0000 mgr.y (mgr.25072) 1185 : cluster 0 pgmap v1197: 89 pgs: 89 active+clean; 1.5 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 121 MiB/s wr, 1.94k op/s 2023-10-01T10:57:02.037 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:01 smithi078 bash[78274]: cluster 2023-10-01T10:57:00.081691+0000 mgr.y (mgr.25072) 1185 : cluster 0 pgmap v1197: 89 pgs: 89 active+clean; 1.5 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 121 MiB/s wr, 1.94k op/s 2023-10-01T10:57:02.042 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 16 16 30401 30385 118.672 93.875 0.00767364 0.00841894 2023-10-01T10:57:02.057 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:02.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:02.060 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:57:02.161 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:02.209 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:02.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:02.212 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:57:02.312 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:02.342 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:02.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:02.346 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:02.446 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:02.467 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:02.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:02.469 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:02.569 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:02.595 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:02.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:02.598 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:57:02.699 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:02.709 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:02 smithi078 bash[75299]: audit 2023-10-01T10:57:02.446976+0000 mon.a (mon.0) 1493 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:57:02.709 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:02 smithi078 bash[78274]: audit 2023-10-01T10:57:02.446976+0000 mon.a (mon.0) 1493 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:57:02.721 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:02.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:02.723 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:57:02.824 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:02.853 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:02.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:02.856 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:57:02.957 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:02.969 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:02 smithi173 bash[58816]: audit 2023-10-01T10:57:02.446976+0000 mon.a (mon.0) 1493 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:57:02.986 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:02.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:02.989 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:57:03.042 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 17 16 31209 31193 114.662 50.5 0.00570568 0.00871527 2023-10-01T10:57:03.089 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:03.116 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:03.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:03.119 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:03.220 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:03.249 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:03.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:03.254 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:57:03.354 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:03.381 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:03.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:03.385 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:03.486 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:03.514 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:03.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:03.518 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:03.618 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:03.648 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:03.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:03.651 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:57:03.752 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:03.765 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:03 smithi173 bash[58816]: cluster 2023-10-01T10:57:02.082229+0000 mgr.y (mgr.25072) 1186 : cluster 0 pgmap v1198: 89 pgs: 89 active+clean; 1.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 101 MiB/s wr, 1.61k op/s 2023-10-01T10:57:03.788 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:03.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:03.790 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:03.891 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:03.919 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:03.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:03.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:04.024 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:04.042 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 18 16 32674 32658 113.378 91.5625 0.0125205 0.00881361 2023-10-01T10:57:04.051 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:04.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:04.055 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:04.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:03 smithi078 bash[78274]: cluster 2023-10-01T10:57:02.082229+0000 mgr.y (mgr.25072) 1186 : cluster 0 pgmap v1198: 89 pgs: 89 active+clean; 1.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 101 MiB/s wr, 1.61k op/s 2023-10-01T10:57:04.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:03 smithi078 bash[75299]: cluster 2023-10-01T10:57:02.082229+0000 mgr.y (mgr.25072) 1186 : cluster 0 pgmap v1198: 89 pgs: 89 active+clean; 1.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 101 MiB/s wr, 1.61k op/s 2023-10-01T10:57:04.155 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:04.183 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:04.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:04.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:57:04.287 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:04.314 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:04.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:04.317 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:04.418 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:04.469 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:04.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:04.474 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:57:04.574 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:04.597 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:04.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:04.599 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:57:04.700 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:04.729 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:04.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:04.733 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:57:04.834 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:04.864 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:04.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:04.868 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:57:04.969 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:04.998 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:05.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:05.001 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:57:05.042 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 19 16 34091 34075 112.072 88.5625 0.011834 0.00891679 2023-10-01T10:57:05.102 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:05.224 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:05.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:05.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:57:05.328 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:05.353 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:05.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:05.357 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:57:05.458 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:05.494 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:05.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:05.498 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:57:05.599 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:05.703 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:05.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:05.706 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:57:05.807 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:05.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:05 smithi078 bash[75299]: cluster 2023-10-01T10:57:04.083429+0000 mgr.y (mgr.25072) 1187 : cluster 0 pgmap v1199: 89 pgs: 89 active+clean; 1.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 124 MiB/s wr, 1.98k op/s 2023-10-01T10:57:05.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:05 smithi078 bash[78274]: cluster 2023-10-01T10:57:04.083429+0000 mgr.y (mgr.25072) 1187 : cluster 0 pgmap v1199: 89 pgs: 89 active+clean; 1.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 124 MiB/s wr, 1.98k op/s 2023-10-01T10:57:05.832 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:05.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:05.835 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:57:05.936 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:05.971 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:05.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:05.974 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:57:06.043 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:2023-10-01T10:57:06.042934+0000 min lat: 0.00107177 max lat: 0.29942 avg lat: 0.00905902 2023-10-01T10:57:06.043 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T10:57:06.043 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 20 16 35311 35295 110.277 76.25 0.0181336 0.00905902 2023-10-01T10:57:06.075 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:06.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:05 smithi173 bash[58816]: cluster 2023-10-01T10:57:04.083429+0000 mgr.y (mgr.25072) 1187 : cluster 0 pgmap v1199: 89 pgs: 89 active+clean; 1.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 124 MiB/s wr, 1.98k op/s 2023-10-01T10:57:06.120 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:06.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:06.124 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:57:06.225 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:06.250 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:06.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:06.253 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:06.354 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:06.384 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:06.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:06.386 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:57:06.487 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:06.554 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:06.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:06.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:57:06.658 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:06.723 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:06.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:06.725 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:06.826 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:06.855 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:06.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:06.858 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:57:06.959 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:06.989 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:06.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:06.993 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:57:07.043 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 21 16 36512 36496 108.6 75.0625 0.016574 0.00920021 2023-10-01T10:57:07.094 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:07.125 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:07.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:07.134 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:57:07.235 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:07.269 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:07.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:07.273 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:57:07.374 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:07.407 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:07.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:07.410 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:07.511 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:07.542 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:07.544 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:07 smithi173 bash[58816]: cluster 2023-10-01T10:57:06.084113+0000 mgr.y (mgr.25072) 1188 : cluster 0 pgmap v1200: 89 pgs: 89 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-10-01T10:57:07.545 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:07 smithi173 bash[58816]: audit 2023-10-01T10:57:06.281829+0000 mon.a (mon.0) 1494 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:57:07.545 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:07 smithi173 bash[58816]: audit 2023-10-01T10:57:06.299368+0000 mon.a (mon.0) 1495 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:57:07.545 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:07 smithi173 bash[58816]: audit 2023-10-01T10:57:06.621938+0000 mon.a (mon.0) 1496 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:57:07.545 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:07 smithi173 bash[58816]: audit 2023-10-01T10:57:06.639024+0000 mon.a (mon.0) 1497 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:57:07.545 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:07 smithi173 bash[58816]: audit 2023-10-01T10:57:07.268866+0000 mon.a (mon.0) 1498 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:57:07.545 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:07 smithi173 bash[58816]: audit 2023-10-01T10:57:07.272199+0000 mon.a (mon.0) 1499 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:57:07.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:07.546 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:07.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:07 smithi078 bash[75299]: cluster 2023-10-01T10:57:06.084113+0000 mgr.y (mgr.25072) 1188 : cluster 0 pgmap v1200: 89 pgs: 89 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-10-01T10:57:07.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:07 smithi078 bash[75299]: audit 2023-10-01T10:57:06.281829+0000 mon.a (mon.0) 1494 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:57:07.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:07 smithi078 bash[75299]: audit 2023-10-01T10:57:06.299368+0000 mon.a (mon.0) 1495 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:57:07.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:07 smithi078 bash[75299]: audit 2023-10-01T10:57:06.621938+0000 mon.a (mon.0) 1496 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:57:07.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:07 smithi078 bash[75299]: audit 2023-10-01T10:57:06.639024+0000 mon.a (mon.0) 1497 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:57:07.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:07 smithi078 bash[75299]: audit 2023-10-01T10:57:07.268866+0000 mon.a (mon.0) 1498 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:57:07.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:07 smithi078 bash[75299]: audit 2023-10-01T10:57:07.272199+0000 mon.a (mon.0) 1499 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:57:07.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:07 smithi078 bash[78274]: cluster 2023-10-01T10:57:06.084113+0000 mgr.y (mgr.25072) 1188 : cluster 0 pgmap v1200: 89 pgs: 89 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2023-10-01T10:57:07.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:07 smithi078 bash[78274]: audit 2023-10-01T10:57:06.281829+0000 mon.a (mon.0) 1494 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:57:07.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:07 smithi078 bash[78274]: audit 2023-10-01T10:57:06.299368+0000 mon.a (mon.0) 1495 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:57:07.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:07 smithi078 bash[78274]: audit 2023-10-01T10:57:06.621938+0000 mon.a (mon.0) 1496 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:57:07.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:07 smithi078 bash[78274]: audit 2023-10-01T10:57:06.639024+0000 mon.a (mon.0) 1497 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:57:07.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:07 smithi078 bash[78274]: audit 2023-10-01T10:57:07.268866+0000 mon.a (mon.0) 1498 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:57:07.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:07 smithi078 bash[78274]: audit 2023-10-01T10:57:07.272199+0000 mon.a (mon.0) 1499 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:57:07.647 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:07.682 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:07.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:07.685 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:57:07.786 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:07.816 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:07.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:07.822 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:57:07.923 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:07.961 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:07.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:07.964 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:57:08.043 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 22 16 37828 37812 107.402 82.25 0.00702833 0.00930393 2023-10-01T10:57:08.065 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:08.084 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:08.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:08.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:57:08.187 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:08.216 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:08.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:08.219 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:57:08.320 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:08.347 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:08.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:08.352 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:57:08.452 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:08.479 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:08.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:08.482 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:08.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:08 smithi078 bash[78274]: audit 2023-10-01T10:57:07.282501+0000 mon.a (mon.0) 1500 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:57:08.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:08 smithi078 bash[75299]: audit 2023-10-01T10:57:07.282501+0000 mon.a (mon.0) 1500 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:57:08.582 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:08.592 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:08 smithi173 bash[58816]: audit 2023-10-01T10:57:07.282501+0000 mon.a (mon.0) 1500 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:57:08.611 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:08.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:08.612 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:57:08.713 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:08.739 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:08.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:08.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:57:08.845 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:08.875 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:08.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:08.881 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:57:08.982 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:09.014 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:09.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:09.018 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:09.043 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 23 16 39244 39228 106.58 88.5 0.0041369 0.00937632 2023-10-01T10:57:09.119 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:09.279 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:09.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:09.282 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:57:09.383 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:09.408 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:09.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:09.409 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:09.510 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:09.537 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:09.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:09.541 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:57:09.642 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:09.669 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:09.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:09.672 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:57:09.773 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:09.804 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:09.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:09.807 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:57:09.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:09 smithi078 bash[78274]: cluster 2023-10-01T10:57:08.084818+0000 mgr.y (mgr.25072) 1189 : cluster 0 pgmap v1201: 89 pgs: 89 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2023-10-01T10:57:09.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:09 smithi078 bash[75299]: cluster 2023-10-01T10:57:08.084818+0000 mgr.y (mgr.25072) 1189 : cluster 0 pgmap v1201: 89 pgs: 89 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2023-10-01T10:57:09.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:09 smithi173 bash[58816]: cluster 2023-10-01T10:57:08.084818+0000 mgr.y (mgr.25072) 1189 : cluster 0 pgmap v1201: 89 pgs: 89 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2023-10-01T10:57:09.907 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:09.932 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:09.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:09.935 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:57:10.036 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:10.043 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 24 16 40742 40726 106.04 93.625 0.0057342 0.00942405 2023-10-01T10:57:10.066 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:10.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:10.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:57:10.172 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:10.198 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:10.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:10.203 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:57:10.303 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:10.330 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:10.331 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:57:10 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:57:10] "GET /metrics HTTP/1.1" 200 36380 "" "Prometheus/2.33.4" 2023-10-01T10:57:10.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:10.335 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:10.436 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:10.454 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:10.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:10.456 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:57:10.557 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:10.588 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:10.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:10.590 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:10.691 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:10.723 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:10.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:10.726 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:10.827 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:10.864 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:10.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:10.870 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:10.970 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:10.994 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:10.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:10.998 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:11.043 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 25 16 42178 42162 105.388 89.75 0.0149598 0.00948312 2023-10-01T10:57:11.098 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:11.125 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:11.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:11.127 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:57:11.228 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:11.255 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:11.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:11.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:57:11.360 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:11.389 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:11.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:11.392 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:57:11.493 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:11.518 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:11.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:11.520 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:57:11.621 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:11.649 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:11.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:11.651 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:57:11.752 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:11.831 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:11.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:11.833 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:11.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:11 smithi173 bash[58816]: cluster 2023-10-01T10:57:10.085546+0000 mgr.y (mgr.25072) 1190 : cluster 0 pgmap v1202: 89 pgs: 89 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-10-01T10:57:11.872 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:11 smithi078 bash[75299]: cluster 2023-10-01T10:57:10.085546+0000 mgr.y (mgr.25072) 1190 : cluster 0 pgmap v1202: 89 pgs: 89 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-10-01T10:57:11.872 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:11 smithi078 bash[78274]: cluster 2023-10-01T10:57:10.085546+0000 mgr.y (mgr.25072) 1190 : cluster 0 pgmap v1202: 89 pgs: 89 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-10-01T10:57:11.934 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:11.964 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:11.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:11.966 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:57:12.044 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 26 16 43548 43532 104.627 85.625 0.00612686 0.00955179 2023-10-01T10:57:12.067 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:12.093 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:12.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:12.096 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:12.196 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:12.226 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:12.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:12.231 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:57:12.332 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:12.359 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:12.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:12.362 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:57:12.463 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:12.491 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:12.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:12.493 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:12.594 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:12.626 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:12.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:12.631 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:57:12.731 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:12.758 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:12.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:12.763 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:57:12.863 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:12.888 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:12.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:12.890 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:57:12.991 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:13.019 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:13.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:13.023 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:57:13.044 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 27 16 44890 44874 103.858 83.875 0.00557505 0.00962335 2023-10-01T10:57:13.123 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:13.151 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:13.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:13.154 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:13.254 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:13.301 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:13.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:13.304 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:57:13.405 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:13.437 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:13.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:13.441 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:57:13.542 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:13.570 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:13.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:13.576 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:57:13.676 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:13.689 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:13 smithi173 bash[58816]: cluster 2023-10-01T10:57:12.085912+0000 mgr.y (mgr.25072) 1191 : cluster 0 pgmap v1203: 89 pgs: 89 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-01T10:57:13.715 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:13.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:13.719 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:13.820 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:13.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:13 smithi078 bash[78274]: cluster 2023-10-01T10:57:12.085912+0000 mgr.y (mgr.25072) 1191 : cluster 0 pgmap v1203: 89 pgs: 89 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-01T10:57:13.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:13 smithi078 bash[75299]: cluster 2023-10-01T10:57:12.085912+0000 mgr.y (mgr.25072) 1191 : cluster 0 pgmap v1203: 89 pgs: 89 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-10-01T10:57:13.854 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:13.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:13.858 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:13.958 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:13.984 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:13.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:13.987 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:57:14.044 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 28 16 46119 46103 102.892 76.8125 0.00802333 0.00971171 2023-10-01T10:57:14.087 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:14.108 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:14.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:14.110 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:57:14.210 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:14.238 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:14.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:14.241 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:14.342 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:14.369 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:14.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:14.371 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:57:14.472 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:14.503 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:14.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:14.506 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:57:14.607 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:14.636 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:14.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:14.639 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:14.740 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:14.764 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:14.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:14.766 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:57:14.867 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:14.899 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:14.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:14.903 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:57:15.003 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:15.039 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:15.044 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 29 16 47363 47347 102.025 77.75 0.00705987 0.00979484 2023-10-01T10:57:15.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:15.045 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:57:15.145 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:15.544 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:15.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:15.548 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:15.649 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:15.675 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:15.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:15.679 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:57:15.779 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:15.788 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:15 smithi078 bash[78274]: cluster 2023-10-01T10:57:14.086527+0000 mgr.y (mgr.25072) 1192 : cluster 0 pgmap v1204: 89 pgs: 89 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2023-10-01T10:57:15.788 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:15 smithi078 bash[75299]: cluster 2023-10-01T10:57:14.086527+0000 mgr.y (mgr.25072) 1192 : cluster 0 pgmap v1204: 89 pgs: 89 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2023-10-01T10:57:15.813 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:15.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:15.816 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:15.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:15 smithi173 bash[58816]: cluster 2023-10-01T10:57:14.086527+0000 mgr.y (mgr.25072) 1192 : cluster 0 pgmap v1204: 89 pgs: 89 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2023-10-01T10:57:15.917 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:15.954 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:15.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:15.959 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:57:16.044 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 30 16 48713 48697 101.437 84.375 0.00792405 0.00984893 2023-10-01T10:57:16.060 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:16.092 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:16.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:16.097 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:57:16.198 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:16.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:16.229 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:16.229 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:57:16.330 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:16.420 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:16.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:16.424 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:16.458 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2023-10-01T10:57:16.459 INFO:teuthology.orchestra.run.smithi078.stdout: "ops": [], 2023-10-01T10:57:16.459 INFO:teuthology.orchestra.run.smithi078.stdout: "complaint_time": 30, 2023-10-01T10:57:16.459 INFO:teuthology.orchestra.run.smithi078.stdout: "num_blocked_ops": 0 2023-10-01T10:57:16.459 INFO:teuthology.orchestra.run.smithi078.stdout:} 2023-10-01T10:57:16.524 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:16.556 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:16.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:16.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:16.661 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:16.686 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:16.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:16.689 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:57:16.790 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:16.814 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:16.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:16.818 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:57:16.918 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:16.944 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:16.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:16.947 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:57:17.044 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 31 16 49943 49927 100.644 76.875 0.0060019 0.00992987 2023-10-01T10:57:17.048 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:17.076 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:17.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:17.082 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:17.183 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:17.217 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:17.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:17.222 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:17.246 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T10:57:17.288 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T10:57:17.323 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:17.345 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:17.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:17.348 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:17.432 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T10:57:17.449 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:17.476 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:17.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:17.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:17.483 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T10:57:17.527 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T10:57:17.568 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T10:57:17.580 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:17.595 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T10:57:17.606 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:17.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:17.612 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:17.627 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T10:57:17.636 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T10:57:17.636 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T10:57:17.636 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T10:57:17.636 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T10:57:17.636 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T10:57:17.637 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T10:57:17.637 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T10:57:17.637 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T10:57:17.712 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:17.739 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:17.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:17.742 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:57:17.798 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:17 smithi078 bash[78274]: cluster 2023-10-01T10:57:16.087074+0000 mgr.y (mgr.25072) 1193 : cluster 0 pgmap v1205: 89 pgs: 89 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-01T10:57:17.798 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:17 smithi078 bash[78274]: audit 2023-10-01T10:57:17.149751+0000 mon.c (mon.2) 192 : audit 0 from='client.? 172.21.15.78:0/3817544124' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:57:17.798 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:17 smithi078 bash[78274]: audit 2023-10-01T10:57:17.446903+0000 mon.a (mon.0) 1501 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:57:17.798 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:17 smithi078 bash[75299]: cluster 2023-10-01T10:57:16.087074+0000 mgr.y (mgr.25072) 1193 : cluster 0 pgmap v1205: 89 pgs: 89 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-01T10:57:17.799 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:17 smithi078 bash[75299]: audit 2023-10-01T10:57:17.149751+0000 mon.c (mon.2) 192 : audit 0 from='client.? 172.21.15.78:0/3817544124' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:57:17.799 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:17 smithi078 bash[75299]: audit 2023-10-01T10:57:17.446903+0000 mon.a (mon.0) 1501 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:57:17.843 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:17.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:17 smithi173 bash[58816]: cluster 2023-10-01T10:57:16.087074+0000 mgr.y (mgr.25072) 1193 : cluster 0 pgmap v1205: 89 pgs: 89 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2023-10-01T10:57:17.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:17 smithi173 bash[58816]: audit 2023-10-01T10:57:17.149751+0000 mon.c (mon.2) 192 : audit 0 from='client.? 172.21.15.78:0/3817544124' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:57:17.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:17 smithi173 bash[58816]: audit 2023-10-01T10:57:17.446903+0000 mon.a (mon.0) 1501 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:57:17.872 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:17.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:17.877 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:57:17.977 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:18.020 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:18.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:18.026 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:18.044 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 32 16 50800 50784 99.1728 53.5625 0.0143296 0.0100777 2023-10-01T10:57:18.127 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:18.156 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:18.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:18.162 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:57:18.225 INFO:teuthology.orchestra.run.smithi078.stdout: 2023-10-01T10:57:18.229 INFO:teuthology.orchestra.run.smithi078.stderr:dumped all 2023-10-01T10:57:18.263 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:18.290 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:18.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:18.294 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:57:18.395 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:18.417 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:18.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:18.420 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:57:18.520 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:18.554 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:18.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:18.558 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:57:18.643 INFO:teuthology.orchestra.run.smithi078.stderr:noscrub is unset 2023-10-01T10:57:18.660 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:18.686 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:18.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:18.688 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:18.789 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:18.798 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:18 smithi078 bash[75299]: audit 2023-10-01T10:57:18.439521+0000 mon.a (mon.0) 1502 : audit 1 from='client.? 172.21.15.78:0/1653173169' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T10:57:18.798 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:18 smithi078 bash[78274]: audit 2023-10-01T10:57:18.439521+0000 mon.a (mon.0) 1502 : audit 1 from='client.? 172.21.15.78:0/1653173169' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T10:57:18.818 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:18.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:18.821 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:57:18.922 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:18.949 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:18.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:18.953 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:57:19.044 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 33 16 51856 51840 98.1675 66 0.00616569 0.0101811 2023-10-01T10:57:19.054 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:19.062 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:18 smithi173 bash[58816]: audit 2023-10-01T10:57:18.439521+0000 mon.a (mon.0) 1502 : audit 1 from='client.? 172.21.15.78:0/1653173169' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T10:57:19.083 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:19.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:19.085 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:57:19.186 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:19.210 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:19.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:19.213 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:57:19.314 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:19.335 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:19.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:19.337 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:57:19.438 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:19.472 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:19.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:19.476 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:19.577 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:19.606 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:19.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:19.611 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:57:19.712 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:19.739 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:19.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:19.742 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:57:19.843 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:19.854 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:19 smithi173 bash[58816]: cluster 2023-10-01T10:57:18.087606+0000 mgr.y (mgr.25072) 1194 : cluster 0 pgmap v1206: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-10-01T10:57:19.854 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:19 smithi173 bash[58816]: audit 2023-10-01T10:57:18.197154+0000 mgr.y (mgr.25072) 1195 : audit 0 from='client.46256 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:57:19.854 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:19 smithi173 bash[58816]: cluster 2023-10-01T10:57:18.620623+0000 mon.a (mon.0) 1503 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:57:19.854 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:19 smithi173 bash[58816]: cluster 2023-10-01T10:57:18.629224+0000 osd.6 (osd.6) 188 : cluster 3 failed to encode map e245 with expected crc 2023-10-01T10:57:19.854 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:19 smithi173 bash[58816]: cluster 2023-10-01T10:57:18.629270+0000 osd.7 (osd.7) 228 : cluster 3 failed to encode map e245 with expected crc 2023-10-01T10:57:19.854 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:19 smithi173 bash[58816]: audit 2023-10-01T10:57:18.631080+0000 mon.a (mon.0) 1504 : audit 1 from='client.? 172.21.15.78:0/1653173169' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T10:57:19.854 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:19 smithi173 bash[58816]: cluster 2023-10-01T10:57:18.631161+0000 mon.a (mon.0) 1505 : cluster 0 osdmap e245: 8 total, 8 up, 8 in 2023-10-01T10:57:19.872 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:19.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:19.876 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:19.977 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:20.012 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:20.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:20.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:20.044 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 34 16 53020 53004 97.4197 72.75 0.0174734 0.0102595 2023-10-01T10:57:20.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:19 smithi078 bash[78274]: cluster 2023-10-01T10:57:18.087606+0000 mgr.y (mgr.25072) 1194 : cluster 0 pgmap v1206: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-10-01T10:57:20.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:19 smithi078 bash[78274]: audit 2023-10-01T10:57:18.197154+0000 mgr.y (mgr.25072) 1195 : audit 0 from='client.46256 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:57:20.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:19 smithi078 bash[78274]: cluster 2023-10-01T10:57:18.620623+0000 mon.a (mon.0) 1503 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:57:20.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:19 smithi078 bash[78274]: cluster 2023-10-01T10:57:18.629224+0000 osd.6 (osd.6) 188 : cluster 3 failed to encode map e245 with expected crc 2023-10-01T10:57:20.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:19 smithi078 bash[78274]: cluster 2023-10-01T10:57:18.629270+0000 osd.7 (osd.7) 228 : cluster 3 failed to encode map e245 with expected crc 2023-10-01T10:57:20.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:19 smithi078 bash[78274]: audit 2023-10-01T10:57:18.631080+0000 mon.a (mon.0) 1504 : audit 1 from='client.? 172.21.15.78:0/1653173169' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T10:57:20.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:19 smithi078 bash[78274]: cluster 2023-10-01T10:57:18.631161+0000 mon.a (mon.0) 1505 : cluster 0 osdmap e245: 8 total, 8 up, 8 in 2023-10-01T10:57:20.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:19 smithi078 bash[75299]: cluster 2023-10-01T10:57:18.087606+0000 mgr.y (mgr.25072) 1194 : cluster 0 pgmap v1206: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-10-01T10:57:20.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:19 smithi078 bash[75299]: audit 2023-10-01T10:57:18.197154+0000 mgr.y (mgr.25072) 1195 : audit 0 from='client.46256 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:57:20.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:19 smithi078 bash[75299]: cluster 2023-10-01T10:57:18.620623+0000 mon.a (mon.0) 1503 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:57:20.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:19 smithi078 bash[75299]: cluster 2023-10-01T10:57:18.629224+0000 osd.6 (osd.6) 188 : cluster 3 failed to encode map e245 with expected crc 2023-10-01T10:57:20.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:19 smithi078 bash[75299]: cluster 2023-10-01T10:57:18.629270+0000 osd.7 (osd.7) 228 : cluster 3 failed to encode map e245 with expected crc 2023-10-01T10:57:20.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:19 smithi078 bash[75299]: audit 2023-10-01T10:57:18.631080+0000 mon.a (mon.0) 1504 : audit 1 from='client.? 172.21.15.78:0/1653173169' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T10:57:20.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:19 smithi078 bash[75299]: cluster 2023-10-01T10:57:18.631161+0000 mon.a (mon.0) 1505 : cluster 0 osdmap e245: 8 total, 8 up, 8 in 2023-10-01T10:57:20.117 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:20.146 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:20.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:20.151 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:20.252 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:20.280 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:20.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:20.289 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:57:20.390 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:20.398 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:57:20 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:57:20] "GET /metrics HTTP/1.1" 200 36376 "" "Prometheus/2.33.4" 2023-10-01T10:57:20.416 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:20.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:20.420 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:20.486 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-01T10:57:20.521 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:20.549 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:20.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:20.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:57:20.652 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:20.671 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:20.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:20.673 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:57:20.774 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:20.800 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:20.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:20.805 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:57:20.905 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:20.934 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:20.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:20.937 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:57:21.038 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:21.044 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 35 16 54061 54045 96.4951 65.0625 0.00770402 0.0103578 2023-10-01T10:57:21.076 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:21.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:21.079 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:57:21.180 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:21.208 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:21.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:21.213 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:21.314 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:21.348 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:21.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:21.351 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:57:21.452 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:21.485 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:21.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:21.487 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:21.588 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:21.617 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:21.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:21.619 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:57:21.721 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:21.754 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:21.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:21.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:57:21.862 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:21.872 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:21 smithi078 bash[75299]: cluster 2023-10-01T10:57:20.088088+0000 mgr.y (mgr.25072) 1196 : cluster 0 pgmap v1208: 89 pgs: 89 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-10-01T10:57:21.872 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:21 smithi078 bash[78274]: cluster 2023-10-01T10:57:20.088088+0000 mgr.y (mgr.25072) 1196 : cluster 0 pgmap v1208: 89 pgs: 89 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-10-01T10:57:21.894 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:21.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:21.899 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:57:22.000 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:22.026 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:22.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:22.028 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:22.044 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 36 16 55049 55033 95.5299 61.75 0.0112755 0.0104626 2023-10-01T10:57:22.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:21 smithi173 bash[58816]: cluster 2023-10-01T10:57:20.088088+0000 mgr.y (mgr.25072) 1196 : cluster 0 pgmap v1208: 89 pgs: 89 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-10-01T10:57:22.129 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:22.162 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:22.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:22.167 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:57:22.268 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:22.297 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:22.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:22.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:22.402 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:22.432 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:22.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:22.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:57:22.536 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:22.563 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:22.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:22.566 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:57:22.667 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:22.695 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:22.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:22.698 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:57:22.799 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:22.832 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:22.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:22.837 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:22.902 INFO:teuthology.orchestra.run.smithi078.stdout:{"pg_ready":true,"pg_map":{"version":1206,"stamp":"2023-10-01T10:57:18.087261+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3079608224,"num_objects":47006,"num_object_clones":0,"num_object_copies":141018,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":47006,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":47243,"num_write_kb":3009294,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":1190344,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":47177,"ondisk_log_size":47177,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":14321088,"kb_used_data":8857600,"kb_used_omap":30,"kb_used_meta":5463393,"kb_avail":735640128,"statfs":{"total":767960285184,"available":753295491072,"internally_reserved":0,"allocated":9070182400,"data_stored":9063854246,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31502,"internal_metadata":5594514674},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,9,5,4,4,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":978518016,"num_objects":14931,"num_object_clones":0,"num_object_copies":44793,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":14931,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14931,"num_write_kb":955584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003597"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":237,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-01T10:56:40.591653+0000","last_change":"2023-10-01T10:38:16.032258+0000","last_active":"2023-10-01T10:56:40.591653+0000","last_peered":"2023-10-01T10:56:40.591653+0000","last_clean":"2023-10-01T10:56:40.591653+0000","last_became_active":"2023-10-01T10:38:16.031633+0000","last_became_peered":"2023-10-01T10:38:16.031633+0000","last_unstale":"2023-10-01T10:56:40.591653+0000","last_undegraded":"2023-10-01T10:56:40.591653+0000","last_fullsized":"2023-10-01T10:56:40.591653+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:17:02.446430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":225,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-01T10:56:40.591331+0000","last_change":"2023-10-01T10:38:16.025881+0000","last_active":"2023-10-01T10:56:40.591331+0000","last_peered":"2023-10-01T10:56:40.591331+0000","last_clean":"2023-10-01T10:56:40.591331+0000","last_became_active":"2023-10-01T10:38:16.025518+0000","last_became_peered":"2023-10-01T10:38:16.025518+0000","last_unstale":"2023-10-01T10:56:40.591331+0000","last_undegraded":"2023-10-01T10:56:40.591331+0000","last_fullsized":"2023-10-01T10:56:40.591331+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:48:37.826604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":146,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-01T10:56:48.634638+0000","last_change":"2023-10-01T10:52:18.956544+0000","last_active":"2023-10-01T10:56:48.634638+0000","last_peered":"2023-10-01T10:56:48.634638+0000","last_clean":"2023-10-01T10:56:48.634638+0000","last_became_active":"2023-10-01T10:52:18.956217+0000","last_became_peered":"2023-10-01T10:52:18.956217+0000","last_unstale":"2023-10-01T10:56:48.634638+0000","last_undegraded":"2023-10-01T10:56:48.634638+0000","last_fullsized":"2023-10-01T10:56:48.634638+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:34:01.952235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":209,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-01T10:56:41.277662+0000","last_change":"2023-10-01T10:37:42.772008+0000","last_active":"2023-10-01T10:56:41.277662+0000","last_peered":"2023-10-01T10:56:41.277662+0000","last_clean":"2023-10-01T10:56:41.277662+0000","last_became_active":"2023-10-01T10:37:42.771244+0000","last_became_peered":"2023-10-01T10:37:42.771244+0000","last_unstale":"2023-10-01T10:56:41.277662+0000","last_undegraded":"2023-10-01T10:56:41.277662+0000","last_fullsized":"2023-10-01T10:56:41.277662+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:38:55.180790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":227,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-01T10:56:48.632863+0000","last_change":"2023-10-01T10:37:42.766600+0000","last_active":"2023-10-01T10:56:48.632863+0000","last_peered":"2023-10-01T10:56:48.632863+0000","last_clean":"2023-10-01T10:56:48.632863+0000","last_became_active":"2023-10-01T10:37:42.766051+0000","last_became_peered":"2023-10-01T10:37:42.766051+0000","last_unstale":"2023-10-01T10:56:48.632863+0000","last_undegraded":"2023-10-01T10:56:48.632863+0000","last_fullsized":"2023-10-01T10:56:48.632863+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:51:17.707963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":19,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-01T10:56:41.282582+0000","last_change":"2023-10-01T10:56:14.626119+0000","last_active":"2023-10-01T10:56:41.282582+0000","last_peered":"2023-10-01T10:56:41.282582+0000","last_clean":"2023-10-01T10:56:41.282582+0000","last_became_active":"2023-10-01T10:56:14.625777+0000","last_became_peered":"2023-10-01T10:56:14.625777+0000","last_unstale":"2023-10-01T10:56:41.282582+0000","last_undegraded":"2023-10-01T10:56:41.282582+0000","last_fullsized":"2023-10-01T10:56:41.282582+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":205,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-01T10:56:41.277627+0000","last_change":"2023-10-01T10:37:42.766667+0000","last_active":"2023-10-01T10:56:41.277627+0000","last_peered":"2023-10-01T10:56:41.277627+0000","last_clean":"2023-10-01T10:56:41.277627+0000","last_became_active":"2023-10-01T10:37:42.766369+0000","last_became_peered":"2023-10-01T10:37:42.766369+0000","last_unstale":"2023-10-01T10:56:41.277627+0000","last_undegraded":"2023-10-01T10:56:41.277627+0000","last_fullsized":"2023-10-01T10:56:41.277627+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:02.708788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":229,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-01T10:56:40.590683+0000","last_change":"2023-10-01T10:56:14.622767+0000","last_active":"2023-10-01T10:56:40.590683+0000","last_peered":"2023-10-01T10:56:40.590683+0000","last_clean":"2023-10-01T10:56:40.590683+0000","last_became_active":"2023-10-01T10:56:14.622446+0000","last_became_peered":"2023-10-01T10:56:14.622446+0000","last_unstale":"2023-10-01T10:56:40.590683+0000","last_undegraded":"2023-10-01T10:56:40.590683+0000","last_fullsized":"2023-10-01T10:56:40.590683+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:32:09.354415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":211,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-01T10:56:41.280741+0000","last_change":"2023-10-01T10:32:31.579805+0000","last_active":"2023-10-01T10:56:41.280741+0000","last_peered":"2023-10-01T10:56:41.280741+0000","last_clean":"2023-10-01T10:56:41.280741+0000","last_became_active":"2023-10-01T10:32:31.579423+0000","last_became_peered":"2023-10-01T10:32:31.579423+0000","last_unstale":"2023-10-01T10:56:41.280741+0000","last_undegraded":"2023-10-01T10:56:41.280741+0000","last_fullsized":"2023-10-01T10:56:41.280741+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":213,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-01T10:56:40.590909+0000","last_change":"2023-10-01T10:56:14.619280+0000","last_active":"2023-10-01T10:56:40.590909+0000","last_peered":"2023-10-01T10:56:40.590909+0000","last_clean":"2023-10-01T10:56:40.590909+0000","last_became_active":"2023-10-01T10:56:14.618892+0000","last_became_peered":"2023-10-01T10:56:14.618892+0000","last_unstale":"2023-10-01T10:56:40.590909+0000","last_undegraded":"2023-10-01T10:56:40.590909+0000","last_fullsized":"2023-10-01T10:56:40.590909+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:14:29.635055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":84,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-01T10:56:40.591076+0000","last_change":"2023-10-01T10:52:43.661650+0000","last_active":"2023-10-01T10:56:40.591076+0000","last_peered":"2023-10-01T10:56:40.591076+0000","last_clean":"2023-10-01T10:56:40.591076+0000","last_became_active":"2023-10-01T10:52:43.661202+0000","last_became_peered":"2023-10-01T10:52:43.661202+0000","last_unstale":"2023-10-01T10:56:40.591076+0000","last_undegraded":"2023-10-01T10:56:40.591076+0000","last_fullsized":"2023-10-01T10:56:40.591076+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:28:35.479124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":19,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-01T10:56:41.282216+0000","last_change":"2023-10-01T10:56:14.620228+0000","last_active":"2023-10-01T10:56:41.282216+0000","last_peered":"2023-10-01T10:56:41.282216+0000","last_clean":"2023-10-01T10:56:41.282216+0000","last_became_active":"2023-10-01T10:56:14.619632+0000","last_became_peered":"2023-10-01T10:56:14.619632+0000","last_unstale":"2023-10-01T10:56:41.282216+0000","last_undegraded":"2023-10-01T10:56:41.282216+0000","last_fullsized":"2023-10-01T10:56:41.282216+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":270,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-01T10:56:48.636427+0000","last_change":"2023-10-01T10:56:14.634698+0000","last_active":"2023-10-01T10:56:48.636427+0000","last_peered":"2023-10-01T10:56:48.636427+0000","last_clean":"2023-10-01T10:56:48.636427+0000","last_became_active":"2023-10-01T10:56:14.625320+0000","last_became_peered":"2023-10-01T10:56:14.625320+0000","last_unstale":"2023-10-01T10:56:48.636427+0000","last_undegraded":"2023-10-01T10:56:48.636427+0000","last_fullsized":"2023-10-01T10:56:48.636427+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:56.480434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":224,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-01T10:56:48.636227+0000","last_change":"2023-10-01T10:56:14.622580+0000","last_active":"2023-10-01T10:56:48.636227+0000","last_peered":"2023-10-01T10:56:48.636227+0000","last_clean":"2023-10-01T10:56:48.636227+0000","last_became_active":"2023-10-01T10:56:14.622075+0000","last_became_peered":"2023-10-01T10:56:14.622075+0000","last_unstale":"2023-10-01T10:56:48.636227+0000","last_undegraded":"2023-10-01T10:56:48.636227+0000","last_fullsized":"2023-10-01T10:56:48.636227+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:52:32.352383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":227,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-01T10:56:40.596729+0000","last_change":"2023-10-01T10:38:16.026777+0000","last_active":"2023-10-01T10:56:40.596729+0000","last_peered":"2023-10-01T10:56:40.596729+0000","last_clean":"2023-10-01T10:56:40.596729+0000","last_became_active":"2023-10-01T10:38:16.026371+0000","last_became_peered":"2023-10-01T10:38:16.026371+0000","last_unstale":"2023-10-01T10:56:40.596729+0000","last_undegraded":"2023-10-01T10:56:40.596729+0000","last_fullsized":"2023-10-01T10:56:40.596729+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":234,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-01T10:56:40.590872+0000","last_change":"2023-10-01T10:37:42.772319+0000","last_active":"2023-10-01T10:56:40.590872+0000","last_peered":"2023-10-01T10:56:40.590872+0000","last_clean":"2023-10-01T10:56:40.590872+0000","last_became_active":"2023-10-01T10:37:42.771977+0000","last_became_peered":"2023-10-01T10:37:42.771977+0000","last_unstale":"2023-10-01T10:56:40.590872+0000","last_undegraded":"2023-10-01T10:56:40.590872+0000","last_fullsized":"2023-10-01T10:56:40.590872+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:45:12.783214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":206,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-01T10:56:41.276954+0000","last_change":"2023-10-01T10:37:42.767706+0000","last_active":"2023-10-01T10:56:41.276954+0000","last_peered":"2023-10-01T10:56:41.276954+0000","last_clean":"2023-10-01T10:56:41.276954+0000","last_became_active":"2023-10-01T10:37:42.767211+0000","last_became_peered":"2023-10-01T10:37:42.767211+0000","last_unstale":"2023-10-01T10:56:41.276954+0000","last_undegraded":"2023-10-01T10:56:41.276954+0000","last_fullsized":"2023-10-01T10:56:41.276954+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:37:20.357407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":210,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-01T10:56:41.281263+0000","last_change":"2023-10-01T10:37:42.768642+0000","last_active":"2023-10-01T10:56:41.281263+0000","last_peered":"2023-10-01T10:56:41.281263+0000","last_clean":"2023-10-01T10:56:41.281263+0000","last_became_active":"2023-10-01T10:37:42.768239+0000","last_became_peered":"2023-10-01T10:37:42.768239+0000","last_unstale":"2023-10-01T10:56:41.281263+0000","last_undegraded":"2023-10-01T10:56:41.281263+0000","last_fullsized":"2023-10-01T10:56:41.281263+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":176,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-01T10:56:40.591336+0000","last_change":"2023-10-01T10:56:14.620937+0000","last_active":"2023-10-01T10:56:40.591336+0000","last_peered":"2023-10-01T10:56:40.591336+0000","last_clean":"2023-10-01T10:56:40.591336+0000","last_became_active":"2023-10-01T10:56:14.620579+0000","last_became_peered":"2023-10-01T10:56:14.620579+0000","last_unstale":"2023-10-01T10:56:40.591336+0000","last_undegraded":"2023-10-01T10:56:40.591336+0000","last_fullsized":"2023-10-01T10:56:40.591336+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:53:21.947263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":206,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-01T10:56:41.277933+0000","last_change":"2023-10-01T10:37:42.772247+0000","last_active":"2023-10-01T10:56:41.277933+0000","last_peered":"2023-10-01T10:56:41.277933+0000","last_clean":"2023-10-01T10:56:41.277933+0000","last_became_active":"2023-10-01T10:37:42.771697+0000","last_became_peered":"2023-10-01T10:37:42.771697+0000","last_unstale":"2023-10-01T10:56:41.277933+0000","last_undegraded":"2023-10-01T10:56:41.277933+0000","last_fullsized":"2023-10-01T10:56:41.277933+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:23:15.018248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":29,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-01T10:56:41.281224+0000","last_change":"2023-10-01T10:56:14.636139+0000","last_active":"2023-10-01T10:56:41.281224+0000","last_peered":"2023-10-01T10:56:41.281224+0000","last_clean":"2023-10-01T10:56:41.281224+0000","last_became_active":"2023-10-01T10:56:14.624426+0000","last_became_peered":"2023-10-01T10:56:14.624426+0000","last_unstale":"2023-10-01T10:56:41.281224+0000","last_undegraded":"2023-10-01T10:56:41.281224+0000","last_fullsized":"2023-10-01T10:56:41.281224+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":211,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-01T10:56:48.632603+0000","last_change":"2023-10-01T10:37:42.773127+0000","last_active":"2023-10-01T10:56:48.632603+0000","last_peered":"2023-10-01T10:56:48.632603+0000","last_clean":"2023-10-01T10:56:48.632603+0000","last_became_active":"2023-10-01T10:37:42.772102+0000","last_became_peered":"2023-10-01T10:37:42.772102+0000","last_unstale":"2023-10-01T10:56:48.632603+0000","last_undegraded":"2023-10-01T10:56:48.632603+0000","last_fullsized":"2023-10-01T10:56:48.632603+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:23:26.202295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.0","version":"244'2905","reported_seq":2919,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-01T10:57:14.065649+0000","last_change":"2023-10-01T10:56:21.668720+0000","last_active":"2023-10-01T10:57:14.065649+0000","last_peered":"2023-10-01T10:57:14.065649+0000","last_clean":"2023-10-01T10:57:14.065649+0000","last_became_active":"2023-10-01T10:56:21.668210+0000","last_became_peered":"2023-10-01T10:56:21.668210+0000","last_unstale":"2023-10-01T10:57:14.065649+0000","last_undegraded":"2023-10-01T10:57:14.065649+0000","last_fullsized":"2023-10-01T10:57:14.065649+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":2905,"log_dups_size":0,"ondisk_log_size":2905,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:32:59.602604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":190382080,"num_objects":2905,"num_object_clones":0,"num_object_copies":8715,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2905,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2905,"num_write_kb":185920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":84,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-01T10:56:40.592888+0000","last_change":"2023-10-01T10:52:43.663145+0000","last_active":"2023-10-01T10:56:40.592888+0000","last_peered":"2023-10-01T10:56:40.592888+0000","last_clean":"2023-10-01T10:56:40.592888+0000","last_became_active":"2023-10-01T10:52:43.662783+0000","last_became_peered":"2023-10-01T10:52:43.662783+0000","last_unstale":"2023-10-01T10:56:40.592888+0000","last_undegraded":"2023-10-01T10:56:40.592888+0000","last_fullsized":"2023-10-01T10:56:40.592888+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T22:10:32.190922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":213,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-01T10:56:41.281640+0000","last_change":"2023-10-01T10:56:14.627958+0000","last_active":"2023-10-01T10:56:41.281640+0000","last_peered":"2023-10-01T10:56:41.281640+0000","last_clean":"2023-10-01T10:56:41.281640+0000","last_became_active":"2023-10-01T10:56:14.627506+0000","last_became_peered":"2023-10-01T10:56:14.627506+0000","last_unstale":"2023-10-01T10:56:41.281640+0000","last_undegraded":"2023-10-01T10:56:41.281640+0000","last_fullsized":"2023-10-01T10:56:41.281640+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"244'3079","reported_seq":3093,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-01T10:57:17.451680+0000","last_change":"2023-10-01T10:56:21.667913+0000","last_active":"2023-10-01T10:57:17.451680+0000","last_peered":"2023-10-01T10:57:17.451680+0000","last_clean":"2023-10-01T10:57:17.451680+0000","last_became_active":"2023-10-01T10:56:21.667502+0000","last_became_peered":"2023-10-01T10:56:21.667502+0000","last_unstale":"2023-10-01T10:57:17.451680+0000","last_undegraded":"2023-10-01T10:57:17.451680+0000","last_fullsized":"2023-10-01T10:57:17.451680+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":3079,"log_dups_size":0,"ondisk_log_size":3079,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:53:11.947301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":201785344,"num_objects":3079,"num_object_clones":0,"num_object_copies":9237,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3079,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3079,"num_write_kb":197056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":85,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-01T10:56:48.637089+0000","last_change":"2023-10-01T10:52:43.662540+0000","last_active":"2023-10-01T10:56:48.637089+0000","last_peered":"2023-10-01T10:56:48.637089+0000","last_clean":"2023-10-01T10:56:48.637089+0000","last_became_active":"2023-10-01T10:52:43.662128+0000","last_became_peered":"2023-10-01T10:52:43.662128+0000","last_unstale":"2023-10-01T10:56:48.637089+0000","last_undegraded":"2023-10-01T10:56:48.637089+0000","last_fullsized":"2023-10-01T10:56:48.637089+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:09:44.807003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":240,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-01T10:56:40.591384+0000","last_change":"2023-10-01T10:38:16.033203+0000","last_active":"2023-10-01T10:56:40.591384+0000","last_peered":"2023-10-01T10:56:40.591384+0000","last_clean":"2023-10-01T10:56:40.591384+0000","last_became_active":"2023-10-01T10:38:16.032681+0000","last_became_peered":"2023-10-01T10:38:16.032681+0000","last_unstale":"2023-10-01T10:56:40.591384+0000","last_undegraded":"2023-10-01T10:56:40.591384+0000","last_fullsized":"2023-10-01T10:56:40.591384+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:42:36.642284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"244'2802","reported_seq":2816,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-01T10:57:14.152771+0000","last_change":"2023-10-01T10:56:21.664087+0000","last_active":"2023-10-01T10:57:14.152771+0000","last_peered":"2023-10-01T10:57:14.152771+0000","last_clean":"2023-10-01T10:57:14.152771+0000","last_became_active":"2023-10-01T10:56:21.663655+0000","last_became_peered":"2023-10-01T10:56:21.663655+0000","last_unstale":"2023-10-01T10:57:14.152771+0000","last_undegraded":"2023-10-01T10:57:14.152771+0000","last_fullsized":"2023-10-01T10:57:14.152771+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":2802,"log_dups_size":0,"ondisk_log_size":2802,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:34:34.434364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":183631872,"num_objects":2802,"num_object_clones":0,"num_object_copies":8406,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2802,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2802,"num_write_kb":179328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":109,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-01T10:56:41.281429+0000","last_change":"2023-10-01T10:56:14.638810+0000","last_active":"2023-10-01T10:56:41.281429+0000","last_peered":"2023-10-01T10:56:41.281429+0000","last_clean":"2023-10-01T10:56:41.281429+0000","last_became_active":"2023-10-01T10:56:14.624601+0000","last_became_peered":"2023-10-01T10:56:14.624601+0000","last_unstale":"2023-10-01T10:56:41.281429+0000","last_undegraded":"2023-10-01T10:56:41.281429+0000","last_fullsized":"2023-10-01T10:56:41.281429+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":19,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-01T10:56:41.281945+0000","last_change":"2023-10-01T10:56:14.623448+0000","last_active":"2023-10-01T10:56:41.281945+0000","last_peered":"2023-10-01T10:56:41.281945+0000","last_clean":"2023-10-01T10:56:41.281945+0000","last_became_active":"2023-10-01T10:56:14.623123+0000","last_became_peered":"2023-10-01T10:56:14.623123+0000","last_unstale":"2023-10-01T10:56:41.281945+0000","last_undegraded":"2023-10-01T10:56:41.281945+0000","last_fullsized":"2023-10-01T10:56:41.281945+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"244'2895","reported_seq":2909,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-01T10:57:14.327967+0000","last_change":"2023-10-01T10:56:21.664942+0000","last_active":"2023-10-01T10:57:14.327967+0000","last_peered":"2023-10-01T10:57:14.327967+0000","last_clean":"2023-10-01T10:57:14.327967+0000","last_became_active":"2023-10-01T10:56:21.664236+0000","last_became_peered":"2023-10-01T10:56:21.664236+0000","last_unstale":"2023-10-01T10:57:14.327967+0000","last_undegraded":"2023-10-01T10:57:14.327967+0000","last_fullsized":"2023-10-01T10:57:14.327967+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":2895,"log_dups_size":0,"ondisk_log_size":2895,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:51:15.411254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189726720,"num_objects":2895,"num_object_clones":0,"num_object_copies":8685,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2895,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2895,"num_write_kb":185280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":229,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-01T10:56:41.281167+0000","last_change":"2023-10-01T10:56:14.624999+0000","last_active":"2023-10-01T10:56:41.281167+0000","last_peered":"2023-10-01T10:56:41.281167+0000","last_clean":"2023-10-01T10:56:41.281167+0000","last_became_active":"2023-10-01T10:56:14.624597+0000","last_became_peered":"2023-10-01T10:56:14.624597+0000","last_unstale":"2023-10-01T10:56:41.281167+0000","last_undegraded":"2023-10-01T10:56:41.281167+0000","last_fullsized":"2023-10-01T10:56:41.281167+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":222,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-01T10:56:41.276581+0000","last_change":"2023-10-01T10:38:16.026348+0000","last_active":"2023-10-01T10:56:41.276581+0000","last_peered":"2023-10-01T10:56:41.276581+0000","last_clean":"2023-10-01T10:56:41.276581+0000","last_became_active":"2023-10-01T10:38:16.025747+0000","last_became_peered":"2023-10-01T10:38:16.025747+0000","last_unstale":"2023-10-01T10:56:41.276581+0000","last_undegraded":"2023-10-01T10:56:41.276581+0000","last_fullsized":"2023-10-01T10:56:41.276581+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:28:18.934480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"244'3146","reported_seq":3160,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-01T10:57:17.459796+0000","last_change":"2023-10-01T10:56:22.013729+0000","last_active":"2023-10-01T10:57:17.459796+0000","last_peered":"2023-10-01T10:57:17.459796+0000","last_clean":"2023-10-01T10:57:17.459796+0000","last_became_active":"2023-10-01T10:56:22.013403+0000","last_became_peered":"2023-10-01T10:56:22.013403+0000","last_unstale":"2023-10-01T10:57:17.459796+0000","last_undegraded":"2023-10-01T10:57:17.459796+0000","last_fullsized":"2023-10-01T10:57:17.459796+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":3146,"log_dups_size":0,"ondisk_log_size":3146,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:34:17.163618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":206176256,"num_objects":3146,"num_object_clones":0,"num_object_copies":9438,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3146,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3146,"num_write_kb":201344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":214,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-01T10:56:40.591002+0000","last_change":"2023-10-01T10:38:16.022493+0000","last_active":"2023-10-01T10:56:40.591002+0000","last_peered":"2023-10-01T10:56:40.591002+0000","last_clean":"2023-10-01T10:56:40.591002+0000","last_became_active":"2023-10-01T10:38:16.022033+0000","last_became_peered":"2023-10-01T10:38:16.022033+0000","last_unstale":"2023-10-01T10:56:40.591002+0000","last_undegraded":"2023-10-01T10:56:40.591002+0000","last_fullsized":"2023-10-01T10:56:40.591002+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:49:05.262763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":243,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-01T10:56:40.592158+0000","last_change":"2023-10-01T10:56:14.623628+0000","last_active":"2023-10-01T10:56:40.592158+0000","last_peered":"2023-10-01T10:56:40.592158+0000","last_clean":"2023-10-01T10:56:40.592158+0000","last_became_active":"2023-10-01T10:56:14.622190+0000","last_became_peered":"2023-10-01T10:56:14.622190+0000","last_unstale":"2023-10-01T10:56:40.592158+0000","last_undegraded":"2023-10-01T10:56:40.592158+0000","last_fullsized":"2023-10-01T10:56:40.592158+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:57:06.093638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"244'2743","reported_seq":2757,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-01T10:57:13.497684+0000","last_change":"2023-10-01T10:56:22.014079+0000","last_active":"2023-10-01T10:57:13.497684+0000","last_peered":"2023-10-01T10:57:13.497684+0000","last_clean":"2023-10-01T10:57:13.497684+0000","last_became_active":"2023-10-01T10:56:22.013703+0000","last_became_peered":"2023-10-01T10:56:22.013703+0000","last_unstale":"2023-10-01T10:57:13.497684+0000","last_undegraded":"2023-10-01T10:57:13.497684+0000","last_fullsized":"2023-10-01T10:57:13.497684+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":2743,"log_dups_size":0,"ondisk_log_size":2743,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:07:01.254677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179765248,"num_objects":2743,"num_object_clones":0,"num_object_copies":8229,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2743,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2743,"num_write_kb":175552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":216,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-01T10:56:40.592264+0000","last_change":"2023-10-01T10:56:14.622835+0000","last_active":"2023-10-01T10:56:40.592264+0000","last_peered":"2023-10-01T10:56:40.592264+0000","last_clean":"2023-10-01T10:56:40.592264+0000","last_became_active":"2023-10-01T10:56:14.622449+0000","last_became_peered":"2023-10-01T10:56:14.622449+0000","last_unstale":"2023-10-01T10:56:40.592264+0000","last_undegraded":"2023-10-01T10:56:40.592264+0000","last_fullsized":"2023-10-01T10:56:40.592264+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:29:48.934875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":230,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-01T10:56:40.592290+0000","last_change":"2023-10-01T10:38:16.035906+0000","last_active":"2023-10-01T10:56:40.592290+0000","last_peered":"2023-10-01T10:56:40.592290+0000","last_clean":"2023-10-01T10:56:40.592290+0000","last_became_active":"2023-10-01T10:38:16.035383+0000","last_became_peered":"2023-10-01T10:38:16.035383+0000","last_unstale":"2023-10-01T10:56:40.592290+0000","last_undegraded":"2023-10-01T10:56:40.592290+0000","last_fullsized":"2023-10-01T10:56:40.592290+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:44:41.500097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"244'2887","reported_seq":2901,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-01T10:57:14.325493+0000","last_change":"2023-10-01T10:56:22.013872+0000","last_active":"2023-10-01T10:57:14.325493+0000","last_peered":"2023-10-01T10:57:14.325493+0000","last_clean":"2023-10-01T10:57:14.325493+0000","last_became_active":"2023-10-01T10:56:22.013554+0000","last_became_peered":"2023-10-01T10:56:22.013554+0000","last_unstale":"2023-10-01T10:57:14.325493+0000","last_undegraded":"2023-10-01T10:57:14.325493+0000","last_fullsized":"2023-10-01T10:57:14.325493+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":2887,"log_dups_size":0,"ondisk_log_size":2887,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:10:34.060296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189202432,"num_objects":2887,"num_object_clones":0,"num_object_copies":8661,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2887,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2887,"num_write_kb":184768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":222,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-01T10:56:40.591812+0000","last_change":"2023-10-01T10:38:16.026611+0000","last_active":"2023-10-01T10:56:40.591812+0000","last_peered":"2023-10-01T10:56:40.591812+0000","last_clean":"2023-10-01T10:56:40.591812+0000","last_became_active":"2023-10-01T10:38:16.026267+0000","last_became_peered":"2023-10-01T10:38:16.026267+0000","last_unstale":"2023-10-01T10:56:40.591812+0000","last_undegraded":"2023-10-01T10:56:40.591812+0000","last_fullsized":"2023-10-01T10:56:40.591812+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:06.135288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":224,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-01T10:56:40.592723+0000","last_change":"2023-10-01T10:56:14.623011+0000","last_active":"2023-10-01T10:56:40.592723+0000","last_peered":"2023-10-01T10:56:40.592723+0000","last_clean":"2023-10-01T10:56:40.592723+0000","last_became_active":"2023-10-01T10:56:14.622649+0000","last_became_peered":"2023-10-01T10:56:14.622649+0000","last_unstale":"2023-10-01T10:56:40.592723+0000","last_undegraded":"2023-10-01T10:56:40.592723+0000","last_fullsized":"2023-10-01T10:56:40.592723+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:32:42.042469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"244'2873","reported_seq":2887,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-01T10:57:13.490086+0000","last_change":"2023-10-01T10:56:22.016312+0000","last_active":"2023-10-01T10:57:13.490086+0000","last_peered":"2023-10-01T10:57:13.490086+0000","last_clean":"2023-10-01T10:57:13.490086+0000","last_became_active":"2023-10-01T10:56:22.015964+0000","last_became_peered":"2023-10-01T10:56:22.015964+0000","last_unstale":"2023-10-01T10:57:13.490086+0000","last_undegraded":"2023-10-01T10:57:13.490086+0000","last_fullsized":"2023-10-01T10:57:13.490086+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":2873,"log_dups_size":0,"ondisk_log_size":2873,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:30:15.203782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188284928,"num_objects":2873,"num_object_clones":0,"num_object_copies":8619,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2873,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2873,"num_write_kb":183872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":213,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-01T10:56:48.632743+0000","last_change":"2023-10-01T10:37:42.772398+0000","last_active":"2023-10-01T10:56:48.632743+0000","last_peered":"2023-10-01T10:56:48.632743+0000","last_clean":"2023-10-01T10:56:48.632743+0000","last_became_active":"2023-10-01T10:37:42.772084+0000","last_became_peered":"2023-10-01T10:37:42.772084+0000","last_unstale":"2023-10-01T10:56:48.632743+0000","last_undegraded":"2023-10-01T10:56:48.632743+0000","last_fullsized":"2023-10-01T10:56:48.632743+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:32:43.846419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":230,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-01T10:56:41.277691+0000","last_change":"2023-10-01T10:38:16.019755+0000","last_active":"2023-10-01T10:56:41.277691+0000","last_peered":"2023-10-01T10:56:41.277691+0000","last_clean":"2023-10-01T10:56:41.277691+0000","last_became_active":"2023-10-01T10:38:16.019397+0000","last_became_peered":"2023-10-01T10:38:16.019397+0000","last_unstale":"2023-10-01T10:56:41.277691+0000","last_undegraded":"2023-10-01T10:56:41.277691+0000","last_fullsized":"2023-10-01T10:56:41.277691+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:42:56.623225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.a","version":"244'2831","reported_seq":2845,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-01T10:57:13.515701+0000","last_change":"2023-10-01T10:56:21.663116+0000","last_active":"2023-10-01T10:57:13.515701+0000","last_peered":"2023-10-01T10:57:13.515701+0000","last_clean":"2023-10-01T10:57:13.515701+0000","last_became_active":"2023-10-01T10:56:21.662762+0000","last_became_peered":"2023-10-01T10:56:21.662762+0000","last_unstale":"2023-10-01T10:57:13.515701+0000","last_undegraded":"2023-10-01T10:57:13.515701+0000","last_fullsized":"2023-10-01T10:57:13.515701+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":2831,"log_dups_size":0,"ondisk_log_size":2831,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:00:59.657772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185532416,"num_objects":2831,"num_object_clones":0,"num_object_copies":8493,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2831,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2831,"num_write_kb":181184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":138,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-01T10:56:41.277299+0000","last_change":"2023-10-01T10:52:18.960164+0000","last_active":"2023-10-01T10:56:41.277299+0000","last_peered":"2023-10-01T10:56:41.277299+0000","last_clean":"2023-10-01T10:56:41.277299+0000","last_became_active":"2023-10-01T10:52:18.959782+0000","last_became_peered":"2023-10-01T10:52:18.959782+0000","last_unstale":"2023-10-01T10:56:41.277299+0000","last_undegraded":"2023-10-01T10:56:41.277299+0000","last_fullsized":"2023-10-01T10:56:41.277299+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:08:27.603412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":226,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-01T10:56:41.281144+0000","last_change":"2023-10-01T10:37:42.766814+0000","last_active":"2023-10-01T10:56:41.281144+0000","last_peered":"2023-10-01T10:56:41.281144+0000","last_clean":"2023-10-01T10:56:41.281144+0000","last_became_active":"2023-10-01T10:37:42.766451+0000","last_became_peered":"2023-10-01T10:37:42.766451+0000","last_unstale":"2023-10-01T10:56:41.281144+0000","last_undegraded":"2023-10-01T10:56:41.281144+0000","last_fullsized":"2023-10-01T10:56:41.281144+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":129,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-01T10:56:41.280425+0000","last_change":"2023-10-01T10:52:18.960606+0000","last_active":"2023-10-01T10:56:41.280425+0000","last_peered":"2023-10-01T10:56:41.280425+0000","last_clean":"2023-10-01T10:56:41.280425+0000","last_became_active":"2023-10-01T10:52:18.960270+0000","last_became_peered":"2023-10-01T10:52:18.960270+0000","last_unstale":"2023-10-01T10:56:41.280425+0000","last_undegraded":"2023-10-01T10:56:41.280425+0000","last_fullsized":"2023-10-01T10:56:41.280425+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.c","version":"244'2902","reported_seq":2916,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-01T10:57:14.319742+0000","last_change":"2023-10-01T10:56:21.895915+0000","last_active":"2023-10-01T10:57:14.319742+0000","last_peered":"2023-10-01T10:57:14.319742+0000","last_clean":"2023-10-01T10:57:14.319742+0000","last_became_active":"2023-10-01T10:56:21.895501+0000","last_became_peered":"2023-10-01T10:56:21.895501+0000","last_unstale":"2023-10-01T10:57:14.319742+0000","last_undegraded":"2023-10-01T10:57:14.319742+0000","last_fullsized":"2023-10-01T10:57:14.319742+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":2902,"log_dups_size":0,"ondisk_log_size":2902,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:21:21.588297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":190185472,"num_objects":2902,"num_object_clones":0,"num_object_copies":8706,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2902,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2902,"num_write_kb":185728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":233,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-01T10:56:41.280952+0000","last_change":"2023-10-01T10:56:14.624787+0000","last_active":"2023-10-01T10:56:41.280952+0000","last_peered":"2023-10-01T10:56:41.280952+0000","last_clean":"2023-10-01T10:56:41.280952+0000","last_became_active":"2023-10-01T10:56:14.624357+0000","last_became_peered":"2023-10-01T10:56:14.624357+0000","last_unstale":"2023-10-01T10:56:41.280952+0000","last_undegraded":"2023-10-01T10:56:41.280952+0000","last_fullsized":"2023-10-01T10:56:41.280952+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":211,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-01T10:56:48.636703+0000","last_change":"2023-10-01T10:56:14.620354+0000","last_active":"2023-10-01T10:56:48.636703+0000","last_peered":"2023-10-01T10:56:48.636703+0000","last_clean":"2023-10-01T10:56:48.636703+0000","last_became_active":"2023-10-01T10:56:14.619900+0000","last_became_peered":"2023-10-01T10:56:14.619900+0000","last_unstale":"2023-10-01T10:56:48.636703+0000","last_undegraded":"2023-10-01T10:56:48.636703+0000","last_fullsized":"2023-10-01T10:56:48.636703+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:09:36.449841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":141,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-01T10:56:41.277959+0000","last_change":"2023-10-01T10:52:18.963437+0000","last_active":"2023-10-01T10:56:41.277959+0000","last_peered":"2023-10-01T10:56:41.277959+0000","last_clean":"2023-10-01T10:56:41.277959+0000","last_became_active":"2023-10-01T10:52:18.955165+0000","last_became_peered":"2023-10-01T10:52:18.955165+0000","last_unstale":"2023-10-01T10:56:41.277959+0000","last_undegraded":"2023-10-01T10:56:41.277959+0000","last_fullsized":"2023-10-01T10:56:41.277959+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:57:49.129100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.f","version":"244'2936","reported_seq":2949,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-01T10:57:13.491384+0000","last_change":"2023-10-01T10:56:21.896411+0000","last_active":"2023-10-01T10:57:13.491384+0000","last_peered":"2023-10-01T10:57:13.491384+0000","last_clean":"2023-10-01T10:57:13.491384+0000","last_became_active":"2023-10-01T10:56:21.895807+0000","last_became_peered":"2023-10-01T10:56:21.895807+0000","last_unstale":"2023-10-01T10:57:13.491384+0000","last_undegraded":"2023-10-01T10:57:13.491384+0000","last_fullsized":"2023-10-01T10:57:13.491384+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":2936,"log_dups_size":0,"ondisk_log_size":2936,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192413696,"num_objects":2936,"num_object_clones":0,"num_object_copies":8808,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2936,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2936,"num_write_kb":187904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":222,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-01T10:56:41.276704+0000","last_change":"2023-10-01T10:37:42.762524+0000","last_active":"2023-10-01T10:56:41.276704+0000","last_peered":"2023-10-01T10:56:41.276704+0000","last_clean":"2023-10-01T10:56:41.276704+0000","last_became_active":"2023-10-01T10:37:42.762209+0000","last_became_peered":"2023-10-01T10:37:42.762209+0000","last_unstale":"2023-10-01T10:56:41.276704+0000","last_undegraded":"2023-10-01T10:56:41.276704+0000","last_fullsized":"2023-10-01T10:56:41.276704+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:45:38.171397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":139,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-01T10:56:41.276328+0000","last_change":"2023-10-01T10:56:14.621343+0000","last_active":"2023-10-01T10:56:41.276328+0000","last_peered":"2023-10-01T10:56:41.276328+0000","last_clean":"2023-10-01T10:56:41.276328+0000","last_became_active":"2023-10-01T10:56:14.620940+0000","last_became_peered":"2023-10-01T10:56:14.620940+0000","last_unstale":"2023-10-01T10:56:41.276328+0000","last_undegraded":"2023-10-01T10:56:41.276328+0000","last_fullsized":"2023-10-01T10:56:41.276328+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:43:39.290894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":131,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-01T10:56:41.281882+0000","last_change":"2023-10-01T10:52:18.961040+0000","last_active":"2023-10-01T10:56:41.281882+0000","last_peered":"2023-10-01T10:56:41.281882+0000","last_clean":"2023-10-01T10:56:41.281882+0000","last_became_active":"2023-10-01T10:52:18.960606+0000","last_became_peered":"2023-10-01T10:52:18.960606+0000","last_unstale":"2023-10-01T10:56:41.281882+0000","last_undegraded":"2023-10-01T10:56:41.281882+0000","last_fullsized":"2023-10-01T10:56:41.281882+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.d","version":"244'2840","reported_seq":2853,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-01T10:57:13.414879+0000","last_change":"2023-10-01T10:56:22.015481+0000","last_active":"2023-10-01T10:57:13.414879+0000","last_peered":"2023-10-01T10:57:13.414879+0000","last_clean":"2023-10-01T10:57:13.414879+0000","last_became_active":"2023-10-01T10:56:22.014648+0000","last_became_peered":"2023-10-01T10:56:22.014648+0000","last_unstale":"2023-10-01T10:57:13.414879+0000","last_undegraded":"2023-10-01T10:57:13.414879+0000","last_fullsized":"2023-10-01T10:57:13.414879+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":2840,"log_dups_size":0,"ondisk_log_size":2840,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":186122240,"num_objects":2840,"num_object_clones":0,"num_object_copies":8520,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2840,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2840,"num_write_kb":181760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":215,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-01T10:56:41.280218+0000","last_change":"2023-10-01T10:37:05.024009+0000","last_active":"2023-10-01T10:56:41.280218+0000","last_peered":"2023-10-01T10:56:41.280218+0000","last_clean":"2023-10-01T10:56:41.280218+0000","last_became_active":"2023-10-01T10:37:05.023458+0000","last_became_peered":"2023-10-01T10:37:05.023458+0000","last_unstale":"2023-10-01T10:56:41.280218+0000","last_undegraded":"2023-10-01T10:56:41.280218+0000","last_fullsized":"2023-10-01T10:56:41.280218+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'139","reported_seq":382,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-01T10:56:44.337728+0000","last_change":"2023-10-01T10:56:17.635571+0000","last_active":"2023-10-01T10:56:44.337728+0000","last_peered":"2023-10-01T10:56:44.337728+0000","last_clean":"2023-10-01T10:56:44.337728+0000","last_became_active":"2023-10-01T10:56:17.635055+0000","last_became_peered":"2023-10-01T10:56:17.635055+0000","last_unstale":"2023-10-01T10:56:44.337728+0000","last_undegraded":"2023-10-01T10:56:44.337728+0000","last_fullsized":"2023-10-01T10:56:44.337728+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:09:17.221082+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:09:17.221082+0000","last_clean_scrub_stamp":"2023-10-01T10:09:17.221082+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:37:54.527937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":213,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-01T10:56:40.590999+0000","last_change":"2023-10-01T10:56:14.622079+0000","last_active":"2023-10-01T10:56:40.590999+0000","last_peered":"2023-10-01T10:56:40.590999+0000","last_clean":"2023-10-01T10:56:40.590999+0000","last_became_active":"2023-10-01T10:56:14.621644+0000","last_became_peered":"2023-10-01T10:56:14.621644+0000","last_unstale":"2023-10-01T10:56:40.590999+0000","last_undegraded":"2023-10-01T10:56:40.590999+0000","last_fullsized":"2023-10-01T10:56:40.590999+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:32:47.233878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":200,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-01T10:56:40.592039+0000","last_change":"2023-10-01T10:56:14.622469+0000","last_active":"2023-10-01T10:56:40.592039+0000","last_peered":"2023-10-01T10:56:40.592039+0000","last_clean":"2023-10-01T10:56:40.592039+0000","last_became_active":"2023-10-01T10:56:14.622092+0000","last_became_peered":"2023-10-01T10:56:14.622092+0000","last_unstale":"2023-10-01T10:56:40.592039+0000","last_undegraded":"2023-10-01T10:56:40.592039+0000","last_fullsized":"2023-10-01T10:56:40.592039+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:43:56.657361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.e","version":"244'2910","reported_seq":2924,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-01T10:57:14.348212+0000","last_change":"2023-10-01T10:56:21.663535+0000","last_active":"2023-10-01T10:57:14.348212+0000","last_peered":"2023-10-01T10:57:14.348212+0000","last_clean":"2023-10-01T10:57:14.348212+0000","last_became_active":"2023-10-01T10:56:21.663135+0000","last_became_peered":"2023-10-01T10:56:21.663135+0000","last_unstale":"2023-10-01T10:57:14.348212+0000","last_undegraded":"2023-10-01T10:57:14.348212+0000","last_fullsized":"2023-10-01T10:57:14.348212+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":2910,"log_dups_size":0,"ondisk_log_size":2910,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:11:25.118806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":190709760,"num_objects":2910,"num_object_clones":0,"num_object_copies":8730,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2910,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2910,"num_write_kb":186240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":153,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-01T10:56:44.337770+0000","last_change":"2023-10-01T10:52:43.661009+0000","last_active":"2023-10-01T10:56:44.337770+0000","last_peered":"2023-10-01T10:56:44.337770+0000","last_clean":"2023-10-01T10:56:44.337770+0000","last_became_active":"2023-10-01T10:52:43.660679+0000","last_became_peered":"2023-10-01T10:52:43.660679+0000","last_unstale":"2023-10-01T10:56:44.337770+0000","last_undegraded":"2023-10-01T10:56:44.337770+0000","last_fullsized":"2023-10-01T10:56:44.337770+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:32:21.336835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":222,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-01T10:56:41.276580+0000","last_change":"2023-10-01T10:38:16.021968+0000","last_active":"2023-10-01T10:56:41.276580+0000","last_peered":"2023-10-01T10:56:41.276580+0000","last_clean":"2023-10-01T10:56:41.276580+0000","last_became_active":"2023-10-01T10:38:16.021539+0000","last_became_peered":"2023-10-01T10:38:16.021539+0000","last_unstale":"2023-10-01T10:56:41.276580+0000","last_undegraded":"2023-10-01T10:56:41.276580+0000","last_fullsized":"2023-10-01T10:56:41.276580+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:12:53.105747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":223,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-01T10:56:41.277844+0000","last_change":"2023-10-01T10:37:05.017431+0000","last_active":"2023-10-01T10:56:41.277844+0000","last_peered":"2023-10-01T10:56:41.277844+0000","last_clean":"2023-10-01T10:56:41.277844+0000","last_became_active":"2023-10-01T10:37:05.017094+0000","last_became_peered":"2023-10-01T10:37:05.017094+0000","last_unstale":"2023-10-01T10:56:41.277844+0000","last_undegraded":"2023-10-01T10:56:41.277844+0000","last_fullsized":"2023-10-01T10:56:41.277844+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:10:40.664686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.b","version":"244'3185","reported_seq":3199,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-01T10:57:17.452185+0000","last_change":"2023-10-01T10:56:21.667103+0000","last_active":"2023-10-01T10:57:17.452185+0000","last_peered":"2023-10-01T10:57:17.452185+0000","last_clean":"2023-10-01T10:57:17.452185+0000","last_became_active":"2023-10-01T10:56:21.666784+0000","last_became_peered":"2023-10-01T10:56:21.666784+0000","last_unstale":"2023-10-01T10:57:17.452185+0000","last_undegraded":"2023-10-01T10:57:17.452185+0000","last_fullsized":"2023-10-01T10:57:17.452185+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":3185,"log_dups_size":0,"ondisk_log_size":3185,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:45:11.964390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":208732160,"num_objects":3185,"num_object_clones":0,"num_object_copies":9555,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3185,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3185,"num_write_kb":203840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":229,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-01T10:56:40.596545+0000","last_change":"2023-10-01T10:38:16.032774+0000","last_active":"2023-10-01T10:56:40.596545+0000","last_peered":"2023-10-01T10:56:40.596545+0000","last_clean":"2023-10-01T10:56:40.596545+0000","last_became_active":"2023-10-01T10:38:16.032283+0000","last_became_peered":"2023-10-01T10:38:16.032283+0000","last_unstale":"2023-10-01T10:56:40.596545+0000","last_undegraded":"2023-10-01T10:56:40.596545+0000","last_fullsized":"2023-10-01T10:56:40.596545+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":121,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-01T10:56:41.278040+0000","last_change":"2023-10-01T10:52:18.964479+0000","last_active":"2023-10-01T10:56:41.278040+0000","last_peered":"2023-10-01T10:56:41.278040+0000","last_clean":"2023-10-01T10:56:41.278040+0000","last_became_active":"2023-10-01T10:52:18.963881+0000","last_became_peered":"2023-10-01T10:52:18.963881+0000","last_unstale":"2023-10-01T10:56:41.278040+0000","last_undegraded":"2023-10-01T10:56:41.278040+0000","last_fullsized":"2023-10-01T10:56:41.278040+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:36:30.405755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":193,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-01T10:56:41.277186+0000","last_change":"2023-10-01T10:28:20.220812+0000","last_active":"2023-10-01T10:56:41.277186+0000","last_peered":"2023-10-01T10:56:41.277186+0000","last_clean":"2023-10-01T10:56:41.277186+0000","last_became_active":"2023-10-01T10:28:20.215962+0000","last_became_peered":"2023-10-01T10:28:20.215962+0000","last_unstale":"2023-10-01T10:56:41.277186+0000","last_undegraded":"2023-10-01T10:56:41.277186+0000","last_fullsized":"2023-10-01T10:56:41.277186+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:35:15.275097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.8","version":"244'3197","reported_seq":3211,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-01T10:57:17.319589+0000","last_change":"2023-10-01T10:56:21.664695+0000","last_active":"2023-10-01T10:57:17.319589+0000","last_peered":"2023-10-01T10:57:17.319589+0000","last_clean":"2023-10-01T10:57:17.319589+0000","last_became_active":"2023-10-01T10:56:21.664315+0000","last_became_peered":"2023-10-01T10:56:21.664315+0000","last_unstale":"2023-10-01T10:57:17.319589+0000","last_undegraded":"2023-10-01T10:57:17.319589+0000","last_fullsized":"2023-10-01T10:57:17.319589+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":3197,"log_dups_size":0,"ondisk_log_size":3197,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:27:54.090426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":209518592,"num_objects":3197,"num_object_clones":0,"num_object_copies":9591,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3197,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3197,"num_write_kb":204608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":200,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-01T10:56:48.632713+0000","last_change":"2023-10-01T10:37:42.773999+0000","last_active":"2023-10-01T10:56:48.632713+0000","last_peered":"2023-10-01T10:56:48.632713+0000","last_clean":"2023-10-01T10:56:48.632713+0000","last_became_active":"2023-10-01T10:37:42.773355+0000","last_became_peered":"2023-10-01T10:37:42.773355+0000","last_unstale":"2023-10-01T10:56:48.632713+0000","last_undegraded":"2023-10-01T10:56:48.632713+0000","last_fullsized":"2023-10-01T10:56:48.632713+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:46:51.317151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":191,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-01T10:56:40.592715+0000","last_change":"2023-10-01T10:56:14.623768+0000","last_active":"2023-10-01T10:56:40.592715+0000","last_peered":"2023-10-01T10:56:40.592715+0000","last_clean":"2023-10-01T10:56:40.592715+0000","last_became_active":"2023-10-01T10:56:14.623353+0000","last_became_peered":"2023-10-01T10:56:14.623353+0000","last_unstale":"2023-10-01T10:56:40.592715+0000","last_undegraded":"2023-10-01T10:56:40.592715+0000","last_fullsized":"2023-10-01T10:56:40.592715+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:53:19.776540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":197,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-01T10:56:40.590459+0000","last_change":"2023-10-01T10:37:42.773538+0000","last_active":"2023-10-01T10:56:40.590459+0000","last_peered":"2023-10-01T10:56:40.590459+0000","last_clean":"2023-10-01T10:56:40.590459+0000","last_became_active":"2023-10-01T10:37:42.771863+0000","last_became_peered":"2023-10-01T10:37:42.771863+0000","last_unstale":"2023-10-01T10:56:40.590459+0000","last_undegraded":"2023-10-01T10:56:40.590459+0000","last_fullsized":"2023-10-01T10:56:40.590459+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:46:46.698627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"244'2851","reported_seq":2865,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-01T10:57:13.511044+0000","last_change":"2023-10-01T10:56:21.667837+0000","last_active":"2023-10-01T10:57:13.511044+0000","last_peered":"2023-10-01T10:57:13.511044+0000","last_clean":"2023-10-01T10:57:13.511044+0000","last_became_active":"2023-10-01T10:56:21.667513+0000","last_became_peered":"2023-10-01T10:56:21.667513+0000","last_unstale":"2023-10-01T10:57:13.511044+0000","last_undegraded":"2023-10-01T10:57:13.511044+0000","last_fullsized":"2023-10-01T10:57:13.511044+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":2851,"log_dups_size":0,"ondisk_log_size":2851,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:25:17.334326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":186843136,"num_objects":2851,"num_object_clones":0,"num_object_copies":8553,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2851,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2851,"num_write_kb":182464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-01T10:56:41.282616+0000","last_change":"2023-10-01T10:56:14.621086+0000","last_active":"2023-10-01T10:56:41.282616+0000","last_peered":"2023-10-01T10:56:41.282616+0000","last_clean":"2023-10-01T10:56:41.282616+0000","last_became_active":"2023-10-01T10:56:14.620638+0000","last_became_peered":"2023-10-01T10:56:14.620638+0000","last_unstale":"2023-10-01T10:56:41.282616+0000","last_undegraded":"2023-10-01T10:56:41.282616+0000","last_fullsized":"2023-10-01T10:56:41.282616+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":226,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-01T10:56:41.277574+0000","last_change":"2023-10-01T10:38:16.027472+0000","last_active":"2023-10-01T10:56:41.277574+0000","last_peered":"2023-10-01T10:56:41.277574+0000","last_clean":"2023-10-01T10:56:41.277574+0000","last_became_active":"2023-10-01T10:38:16.026810+0000","last_became_peered":"2023-10-01T10:38:16.026810+0000","last_unstale":"2023-10-01T10:56:41.277574+0000","last_undegraded":"2023-10-01T10:56:41.277574+0000","last_fullsized":"2023-10-01T10:56:41.277574+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:30:06.978187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":209,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-01T10:56:48.635909+0000","last_change":"2023-10-01T10:56:14.626521+0000","last_active":"2023-10-01T10:56:48.635909+0000","last_peered":"2023-10-01T10:56:48.635909+0000","last_clean":"2023-10-01T10:56:48.635909+0000","last_became_active":"2023-10-01T10:56:14.626107+0000","last_became_peered":"2023-10-01T10:56:14.626107+0000","last_unstale":"2023-10-01T10:56:48.635909+0000","last_undegraded":"2023-10-01T10:56:48.635909+0000","last_fullsized":"2023-10-01T10:56:48.635909+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:44:12.588719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":209,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-01T10:56:41.277101+0000","last_change":"2023-10-01T10:38:16.028491+0000","last_active":"2023-10-01T10:56:41.277101+0000","last_peered":"2023-10-01T10:56:41.277101+0000","last_clean":"2023-10-01T10:56:41.277101+0000","last_became_active":"2023-10-01T10:38:16.028175+0000","last_became_peered":"2023-10-01T10:38:16.028175+0000","last_unstale":"2023-10-01T10:56:41.277101+0000","last_undegraded":"2023-10-01T10:56:41.277101+0000","last_fullsized":"2023-10-01T10:56:41.277101+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:44:29.083129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":231,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-01T10:56:48.638224+0000","last_change":"2023-10-01T10:37:42.762360+0000","last_active":"2023-10-01T10:56:48.638224+0000","last_peered":"2023-10-01T10:56:48.638224+0000","last_clean":"2023-10-01T10:56:48.638224+0000","last_became_active":"2023-10-01T10:37:42.761936+0000","last_became_peered":"2023-10-01T10:37:42.761936+0000","last_unstale":"2023-10-01T10:56:48.638224+0000","last_undegraded":"2023-10-01T10:56:48.638224+0000","last_fullsized":"2023-10-01T10:56:48.638224+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:50.637582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":228,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-01T10:56:41.281336+0000","last_change":"2023-10-01T10:38:16.037809+0000","last_active":"2023-10-01T10:56:41.281336+0000","last_peered":"2023-10-01T10:56:41.281336+0000","last_clean":"2023-10-01T10:56:41.281336+0000","last_became_active":"2023-10-01T10:38:16.037318+0000","last_became_peered":"2023-10-01T10:38:16.037318+0000","last_unstale":"2023-10-01T10:56:41.281336+0000","last_undegraded":"2023-10-01T10:56:41.281336+0000","last_fullsized":"2023-10-01T10:56:41.281336+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":239,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-01T10:56:40.592392+0000","last_change":"2023-10-01T10:38:16.034282+0000","last_active":"2023-10-01T10:56:40.592392+0000","last_peered":"2023-10-01T10:56:40.592392+0000","last_clean":"2023-10-01T10:56:40.592392+0000","last_became_active":"2023-10-01T10:38:16.033820+0000","last_became_peered":"2023-10-01T10:38:16.033820+0000","last_unstale":"2023-10-01T10:56:40.592392+0000","last_undegraded":"2023-10-01T10:56:40.592392+0000","last_fullsized":"2023-10-01T10:56:40.592392+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:48:23.815832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":239,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-01T10:56:41.280689+0000","last_change":"2023-10-01T10:56:14.636784+0000","last_active":"2023-10-01T10:56:41.280689+0000","last_peered":"2023-10-01T10:56:41.280689+0000","last_clean":"2023-10-01T10:56:41.280689+0000","last_became_active":"2023-10-01T10:56:14.624338+0000","last_became_peered":"2023-10-01T10:56:14.624338+0000","last_unstale":"2023-10-01T10:56:41.280689+0000","last_undegraded":"2023-10-01T10:56:41.280689+0000","last_fullsized":"2023-10-01T10:56:41.280689+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":239,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-01T10:56:40.592349+0000","last_change":"2023-10-01T10:38:16.025600+0000","last_active":"2023-10-01T10:56:40.592349+0000","last_peered":"2023-10-01T10:56:40.592349+0000","last_clean":"2023-10-01T10:56:40.592349+0000","last_became_active":"2023-10-01T10:38:16.025252+0000","last_became_peered":"2023-10-01T10:38:16.025252+0000","last_unstale":"2023-10-01T10:56:40.592349+0000","last_undegraded":"2023-10-01T10:56:40.592349+0000","last_fullsized":"2023-10-01T10:56:40.592349+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:24:18.871963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":246,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-01T10:56:40.591439+0000","last_change":"2023-10-01T10:56:14.621544+0000","last_active":"2023-10-01T10:56:40.591439+0000","last_peered":"2023-10-01T10:56:40.591439+0000","last_clean":"2023-10-01T10:56:40.591439+0000","last_became_active":"2023-10-01T10:56:14.621134+0000","last_became_peered":"2023-10-01T10:56:14.621134+0000","last_unstale":"2023-10-01T10:56:40.591439+0000","last_undegraded":"2023-10-01T10:56:40.591439+0000","last_fullsized":"2023-10-01T10:56:40.591439+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:48:44.436158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":240,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-10-01T10:56:48.632738+0000","last_change":"2023-10-01T10:37:42.765142+0000","last_active":"2023-10-01T10:56:48.632738+0000","last_peered":"2023-10-01T10:56:48.632738+0000","last_clean":"2023-10-01T10:56:48.632738+0000","last_became_active":"2023-10-01T10:37:42.764770+0000","last_became_peered":"2023-10-01T10:37:42.764770+0000","last_unstale":"2023-10-01T10:56:48.632738+0000","last_undegraded":"2023-10-01T10:56:48.632738+0000","last_fullsized":"2023-10-01T10:56:48.632738+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:23:33.527876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":211,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-01T10:56:41.277468+0000","last_change":"2023-10-01T10:37:05.021621+0000","last_active":"2023-10-01T10:56:41.277468+0000","last_peered":"2023-10-01T10:56:41.277468+0000","last_clean":"2023-10-01T10:56:41.277468+0000","last_became_active":"2023-10-01T10:37:05.021173+0000","last_became_peered":"2023-10-01T10:37:05.021173+0000","last_unstale":"2023-10-01T10:56:41.277468+0000","last_undegraded":"2023-10-01T10:56:41.277468+0000","last_fullsized":"2023-10-01T10:56:41.277468+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:13:25.364499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":223,"reported_epoch":243,"state":"active+clean","last_fresh":"2023-10-01T10:56:40.591293+0000","last_change":"2023-10-01T10:56:14.621791+0000","last_active":"2023-10-01T10:56:40.591293+0000","last_peered":"2023-10-01T10:56:40.591293+0000","last_clean":"2023-10-01T10:56:40.591293+0000","last_became_active":"2023-10-01T10:56:14.621443+0000","last_became_peered":"2023-10-01T10:56:14.621443+0000","last_unstale":"2023-10-01T10:56:40.591293+0000","last_undegraded":"2023-10-01T10:56:40.591293+0000","last_fullsized":"2023-10-01T10:56:40.591293+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:13:58.651222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":3079012352,"num_objects":46982,"num_object_clones":0,"num_object_copies":140946,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":46982,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":46982,"num_write_kb":3006848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9204793344,"data_stored":9204793344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":46982,"ondisk_log_size":46982,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":9608,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30319,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430743,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2138888,"kb_used_data":1252736,"kb_used_omap":0,"kb_used_meta":886144,"kb_avail":91606264,"statfs":{"total":95995035648,"available":93804814336,"internally_reserved":0,"allocated":1282801664,"data_stored":1282256355,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":907411456},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,4],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:56:12 2023","interfaces":[{"interface":"back","average":{"1min":1.4099999999999999,"5min":1.734,"15min":2.528},"min":{"1min":0.25700000000000001,"5min":0.252,"15min":0.252},"max":{"1min":5.8570000000000002,"5min":14.315,"15min":137.34200000000001},"last":0.56599999999999995},{"interface":"front","average":{"1min":1.4419999999999999,"5min":1.5129999999999999,"15min":2.3239999999999998},"min":{"1min":0.59999999999999998,"5min":0.32500000000000001,"15min":0.28799999999999998},"max":{"1min":7.6689999999999996,"5min":14.336,"15min":137.39500000000001},"last":2.2909999999999999}]},{"osd":1,"last update":"Sun Oct 1 10:56:20 2023","interfaces":[{"interface":"back","average":{"1min":1.444,"5min":1.891,"15min":1.9430000000000001},"min":{"1min":0.65200000000000002,"5min":0.308,"15min":0.29999999999999999},"max":{"1min":7.7220000000000004,"5min":15.662000000000001,"15min":56.110999999999997},"last":1.4690000000000001},{"interface":"front","average":{"1min":1.016,"5min":1.6639999999999999,"15min":1.8919999999999999},"min":{"1min":0.46999999999999997,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":2.7229999999999999,"5min":18.768999999999998,"15min":51.258000000000003},"last":0.68999999999999995}]},{"osd":2,"last update":"Sun Oct 1 10:56:36 2023","interfaces":[{"interface":"back","average":{"1min":0.97699999999999998,"5min":2.7210000000000001,"15min":1.845},"min":{"1min":0.60299999999999998,"5min":0.315,"15min":0.315},"max":{"1min":1.5920000000000001,"5min":42.804000000000002,"15min":42.804000000000002},"last":0.61399999999999999},{"interface":"front","average":{"1min":1.01,"5min":2.54,"15min":1.845},"min":{"1min":0.56499999999999995,"5min":0.34000000000000002,"15min":0.33200000000000002},"max":{"1min":1.6679999999999999,"5min":42.819000000000003,"15min":42.819000000000003},"last":0.63}]},{"osd":3,"last update":"Sun Oct 1 10:56:12 2023","interfaces":[{"interface":"back","average":{"1min":1.599,"5min":1.619,"15min":1.9810000000000001},"min":{"1min":0.65300000000000002,"5min":0.318,"15min":0.318},"max":{"1min":5.7789999999999999,"5min":8.6839999999999993,"15min":51.334000000000003},"last":0.95899999999999996},{"interface":"front","average":{"1min":1.5589999999999999,"5min":1.734,"15min":2.0510000000000002},"min":{"1min":0.53100000000000003,"5min":0.33400000000000002,"15min":0.313},"max":{"1min":5.883,"5min":8.9299999999999997,"15min":51.404000000000003},"last":0.77400000000000002}]},{"osd":4,"last update":"Sun Oct 1 10:56:45 2023","interfaces":[{"interface":"back","average":{"1min":1.135,"5min":1.3340000000000001,"15min":1.417},"min":{"1min":0.77000000000000002,"5min":0.33500000000000002,"15min":0.317},"max":{"1min":1.677,"5min":7.7800000000000002,"15min":11.081},"last":1.4419999999999999},{"interface":"front","average":{"1min":1.2989999999999999,"5min":1.4330000000000001,"15min":1.4990000000000001},"min":{"1min":0.86699999999999999,"5min":0.35699999999999998,"15min":0.33900000000000002},"max":{"1min":2.5630000000000002,"5min":8.7370000000000001,"15min":12.292},"last":0.66600000000000004}]},{"osd":5,"last update":"Sun Oct 1 10:56:20 2023","interfaces":[{"interface":"back","average":{"1min":1.371,"5min":2.0659999999999998,"15min":1.9590000000000001},"min":{"1min":0.39100000000000001,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":5.8140000000000001,"5min":40.381,"15min":40.381},"last":0.76300000000000001},{"interface":"front","average":{"1min":1.5249999999999999,"5min":1.9379999999999999,"15min":2.0059999999999998},"min":{"1min":0.65700000000000003,"5min":0.33600000000000002,"15min":0.32200000000000001},"max":{"1min":5.9470000000000001,"5min":39.590000000000003,"15min":39.590000000000003},"last":0.72099999999999997}]},{"osd":6,"last update":"Sun Oct 1 10:56:20 2023","interfaces":[{"interface":"back","average":{"1min":1.357,"5min":1.431,"15min":1.589},"min":{"1min":0.52300000000000002,"5min":0.35799999999999998,"15min":0.34499999999999997},"max":{"1min":2.661,"5min":9.4990000000000006,"15min":12.329000000000001},"last":0.73899999999999999},{"interface":"front","average":{"1min":1.7929999999999999,"5min":1.6919999999999999,"15min":1.5349999999999999},"min":{"1min":0.44400000000000001,"5min":0.39700000000000002,"15min":0.36599999999999999},"max":{"1min":8.0120000000000005,"5min":9.4329999999999998,"15min":9.4329999999999998},"last":1.488}]}]},{"osd":6,"up_from":42,"seq":180388626977,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2136248,"kb_used_data":1251440,"kb_used_omap":0,"kb_used_meta":884800,"kb_avail":91608904,"statfs":{"total":95995035648,"available":93807517696,"internally_reserved":0,"allocated":1281474560,"data_stored":1280945174,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":906035200},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:57:11 2023","interfaces":[{"interface":"back","average":{"1min":1.109,"5min":1.468,"15min":2.7490000000000001},"min":{"1min":0.54200000000000004,"5min":0.28299999999999997,"15min":0.25},"max":{"1min":3.4750000000000001,"5min":8.2899999999999991,"15min":140.524},"last":1.284},{"interface":"front","average":{"1min":0.99299999999999999,"5min":1.5509999999999999,"15min":2.7440000000000002},"min":{"1min":0.34000000000000002,"5min":0.29399999999999998,"15min":0.249},"max":{"1min":2.3159999999999998,"5min":9.5259999999999998,"15min":140.86500000000001},"last":0.84099999999999997}]},{"osd":1,"last update":"Sun Oct 1 10:56:18 2023","interfaces":[{"interface":"back","average":{"1min":1.175,"5min":1.5740000000000001,"15min":1.5489999999999999},"min":{"1min":0.58099999999999996,"5min":0.29099999999999998,"15min":0.26000000000000001},"max":{"1min":4.7069999999999999,"5min":9.843,"15min":9.843},"last":0.27100000000000002},{"interface":"front","average":{"1min":0.86699999999999999,"5min":1.6699999999999999,"15min":1.615},"min":{"1min":0.374,"5min":0.26400000000000001,"15min":0.26300000000000001},"max":{"1min":1.3280000000000001,"5min":9.4809999999999999,"15min":12.968999999999999},"last":0.90000000000000002}]},{"osd":2,"last update":"Sun Oct 1 10:56:36 2023","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":4.1779999999999999,"15min":2.8130000000000002},"min":{"1min":0.58099999999999996,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":1.173,"5min":122.768,"15min":122.768},"last":3.181},{"interface":"front","average":{"1min":0.97399999999999998,"5min":4.4299999999999997,"15min":2.903},"min":{"1min":0.68300000000000005,"5min":0.36199999999999999,"15min":0.32100000000000001},"max":{"1min":1.3240000000000001,"5min":123.02200000000001,"15min":123.02200000000001},"last":3.214}]},{"osd":3,"last update":"Sun Oct 1 10:56:18 2023","interfaces":[{"interface":"back","average":{"1min":1.3380000000000001,"5min":2.036,"15min":2.3399999999999999},"min":{"1min":0.53800000000000003,"5min":0.35099999999999998,"15min":0.30599999999999999},"max":{"1min":5.2839999999999998,"5min":36.698999999999998,"15min":61.817999999999998},"last":0.31},{"interface":"front","average":{"1min":1.4790000000000001,"5min":2.4620000000000002,"15min":2.282},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.315},"max":{"1min":6.3869999999999996,"5min":33.555999999999997,"15min":61.219000000000001},"last":3.2839999999999998}]},{"osd":4,"last update":"Sun Oct 1 10:57:01 2023","interfaces":[{"interface":"back","average":{"1min":0.93200000000000005,"5min":1.383,"15min":1.54},"min":{"1min":0.44400000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":1.335,"5min":7.7800000000000002,"15min":13.1},"last":0.877},{"interface":"front","average":{"1min":1.0569999999999999,"5min":1.6910000000000001,"15min":1.6419999999999999},"min":{"1min":0.49099999999999999,"5min":0.35999999999999999,"15min":0.315},"max":{"1min":1.518,"5min":12.157999999999999,"15min":12.157999999999999},"last":3.343}]},{"osd":5,"last update":"Sun Oct 1 10:56:18 2023","interfaces":[{"interface":"back","average":{"1min":1.0760000000000001,"5min":1.4370000000000001,"15min":1.6200000000000001},"min":{"1min":0.44800000000000001,"5min":0.36599999999999999,"15min":0.33000000000000002},"max":{"1min":3.0910000000000002,"5min":15.238,"15min":15.238},"last":0.35499999999999998},{"interface":"front","average":{"1min":1.9079999999999999,"5min":1.962,"15min":1.917},"min":{"1min":0.66400000000000003,"5min":0.36599999999999999,"15min":0.308},"max":{"1min":7.8550000000000004,"5min":13.295,"15min":14.459},"last":0.92100000000000004}]},{"osd":7,"last update":"Sun Oct 1 10:57:11 2023","interfaces":[{"interface":"back","average":{"1min":1.0329999999999999,"5min":1.5089999999999999,"15min":1.6639999999999999},"min":{"1min":0.433,"5min":0.33900000000000002,"15min":0.311},"max":{"1min":1.4379999999999999,"5min":8.4199999999999999,"15min":13.276999999999999},"last":0.93600000000000005},{"interface":"front","average":{"1min":1.2310000000000001,"5min":1.637,"15min":1.73},"min":{"1min":0.60499999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":1.9330000000000001,"5min":7.1379999999999999,"15min":11.416},"last":0.38500000000000001}]}]},{"osd":1,"up_from":101,"seq":433791697195,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1602872,"kb_used_data":969968,"kb_used_omap":4,"kb_used_meta":632891,"kb_avail":92142280,"statfs":{"total":95995035648,"available":94353694720,"internally_reserved":0,"allocated":993247232,"data_stored":992256662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":648081294},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:56:28 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":1.514,"15min":1.3939999999999999},"min":{"1min":0.376,"5min":0.19900000000000001,"15min":0.189},"max":{"1min":0.93100000000000005,"5min":12.888999999999999,"15min":37.058},"last":0.85599999999999998},{"interface":"front","average":{"1min":0.60599999999999998,"5min":1.506,"15min":1.764},"min":{"1min":0.375,"5min":0.192,"15min":0.192},"max":{"1min":1.1020000000000001,"5min":15.416,"15min":61.207999999999998},"last":3.2690000000000001}]},{"osd":2,"last update":"Sun Oct 1 10:56:44 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":3.7679999999999998,"15min":2.391},"min":{"1min":0.39200000000000002,"5min":0.222,"15min":0.222},"max":{"1min":1.0449999999999999,"5min":161.905,"15min":161.905},"last":2.6280000000000001},{"interface":"front","average":{"1min":0.73399999999999999,"5min":3.9649999999999999,"15min":2.431},"min":{"1min":0.48999999999999999,"5min":0.25700000000000001,"15min":0.23699999999999999},"max":{"1min":1.1140000000000001,"5min":162.059,"15min":162.059},"last":1.5669999999999999}]},{"osd":3,"last update":"Sun Oct 1 10:56:28 2023","interfaces":[{"interface":"back","average":{"1min":0.64600000000000002,"5min":3.1299999999999999,"15min":2.161},"min":{"1min":0.41699999999999998,"5min":0.255,"15min":0.20000000000000001},"max":{"1min":0.90300000000000002,"5min":143.73400000000001,"15min":143.73400000000001},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.67000000000000004,"5min":3.2000000000000002,"15min":2.3530000000000002},"min":{"1min":0.44500000000000001,"5min":0.25900000000000001,"15min":0.23999999999999999},"max":{"1min":0.92700000000000005,"5min":143.00399999999999,"15min":143.00399999999999},"last":2.573}]},{"osd":4,"last update":"Sun Oct 1 10:56:44 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":1.448,"15min":1.5860000000000001},"min":{"1min":0.67500000000000004,"5min":0.307,"15min":0.29199999999999998},"max":{"1min":1.246,"5min":8.593,"15min":61.588000000000001},"last":0.83099999999999996},{"interface":"front","average":{"1min":1.0169999999999999,"5min":1.464,"15min":1.7749999999999999},"min":{"1min":0.65700000000000003,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":1.3560000000000001,"5min":8.7530000000000001,"15min":61.411000000000001},"last":0.82199999999999995}]},{"osd":5,"last update":"Sun Oct 1 10:56:28 2023","interfaces":[{"interface":"back","average":{"1min":0.90900000000000003,"5min":1.48,"15min":1.879},"min":{"1min":0.45300000000000001,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":1.427,"5min":9.1809999999999992,"15min":61.560000000000002},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.999,"5min":1.401,"15min":1.724},"min":{"1min":0.75800000000000001,"5min":0.38200000000000001,"15min":0.34699999999999998},"max":{"1min":1.3799999999999999,"5min":8.7949999999999999,"15min":61.667999999999999},"last":0.76000000000000001}]},{"osd":6,"last update":"Sun Oct 1 10:56:28 2023","interfaces":[{"interface":"back","average":{"1min":0.97799999999999998,"5min":1.9359999999999999,"15min":1.8140000000000001},"min":{"1min":0.69199999999999995,"5min":0.371,"15min":0.371},"max":{"1min":1.216,"5min":8.8309999999999995,"15min":62.219999999999999},"last":0.871},{"interface":"front","average":{"1min":1.028,"5min":1.5760000000000001,"15min":1.8080000000000001},"min":{"1min":0.57499999999999996,"5min":0.38,"15min":0.38},"max":{"1min":1.4199999999999999,"5min":8.8970000000000002,"15min":61.462000000000003},"last":0.89700000000000002}]},{"osd":7,"last update":"Sun Oct 1 10:56:28 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":1.5189999999999999,"15min":1.8839999999999999},"min":{"1min":0.60199999999999998,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":1.4890000000000001,"5min":9.7140000000000004,"15min":62.646999999999998},"last":0.77700000000000002},{"interface":"front","average":{"1min":1.143,"5min":1.6879999999999999,"15min":1.903},"min":{"1min":0.56499999999999995,"5min":0.436,"15min":0.38100000000000001},"max":{"1min":3.8740000000000001,"5min":9.1449999999999996,"15min":62.048000000000002},"last":0.53600000000000003}]}]},{"osd":0,"up_from":86,"seq":369367187805,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1873424,"kb_used_data":1107848,"kb_used_omap":5,"kb_used_meta":765562,"kb_avail":91871728,"statfs":{"total":95995035648,"available":94076649472,"internally_reserved":0,"allocated":1134436352,"data_stored":1133480124,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":783936122},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 10:56:22 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":1.4039999999999999,"15min":1.397},"min":{"1min":0.23999999999999999,"5min":0.218,"15min":0.184},"max":{"1min":1.0820000000000001,"5min":8.6479999999999997,"15min":15.074},"last":0.23699999999999999},{"interface":"front","average":{"1min":0.56299999999999994,"5min":1.728,"15min":1.6040000000000001},"min":{"1min":0.20200000000000001,"5min":0.19900000000000001,"15min":0.19700000000000001},"max":{"1min":1.0549999999999999,"5min":11.164999999999999,"15min":13.601000000000001},"last":0.29599999999999999}]},{"osd":2,"last update":"Sun Oct 1 10:56:42 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":2.3420000000000001,"15min":1.647},"min":{"1min":0.30399999999999999,"5min":0.22900000000000001,"15min":0.223},"max":{"1min":0.73699999999999999,"5min":44.744999999999997,"15min":44.744999999999997},"last":1.0389999999999999},{"interface":"front","average":{"1min":0.63700000000000001,"5min":2.5979999999999999,"15min":1.7529999999999999},"min":{"1min":0.28799999999999998,"5min":0.27800000000000002,"15min":0.221},"max":{"1min":1.0580000000000001,"5min":44.691000000000003,"15min":44.691000000000003},"last":1.224}]},{"osd":3,"last update":"Sun Oct 1 10:56:22 2023","interfaces":[{"interface":"back","average":{"1min":0.747,"5min":1.641,"15min":1.6259999999999999},"min":{"1min":0.377,"5min":0.31900000000000001,"15min":0.26200000000000001},"max":{"1min":3.1680000000000001,"5min":7.4260000000000002,"15min":12.944000000000001},"last":1.0820000000000001},{"interface":"front","average":{"1min":0.747,"5min":1.4339999999999999,"15min":1.514},"min":{"1min":0.33300000000000002,"5min":0.246,"15min":0.245},"max":{"1min":2.7370000000000001,"5min":13.641,"15min":13.641},"last":0.52800000000000002}]},{"osd":4,"last update":"Sun Oct 1 10:57:03 2023","interfaces":[{"interface":"back","average":{"1min":0.93999999999999995,"5min":1.1419999999999999,"15min":1.5600000000000001},"min":{"1min":0.503,"5min":0.34300000000000003,"15min":0.32000000000000001},"max":{"1min":1.321,"5min":8.5790000000000006,"15min":11.379},"last":0.80200000000000005},{"interface":"front","average":{"1min":1.087,"5min":1.375,"15min":1.5429999999999999},"min":{"1min":0.40799999999999997,"5min":0.311,"15min":0.311},"max":{"1min":2.9910000000000001,"5min":6.9080000000000004,"15min":11.362},"last":0.39200000000000002}]},{"osd":5,"last update":"Sun Oct 1 10:56:22 2023","interfaces":[{"interface":"back","average":{"1min":1.1259999999999999,"5min":1.544,"15min":1.6930000000000001},"min":{"1min":0.44500000000000001,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":3.1379999999999999,"5min":7.3799999999999999,"15min":12.91},"last":1.1140000000000001},{"interface":"front","average":{"1min":0.96999999999999997,"5min":1.1619999999999999,"15min":1.5389999999999999},"min":{"1min":0.58699999999999997,"5min":0.41099999999999998,"15min":0.34699999999999998},"max":{"1min":1.2929999999999999,"5min":4.5949999999999998,"15min":9.8620000000000001},"last":0.86099999999999999}]},{"osd":6,"last update":"Sun Oct 1 10:56:22 2023","interfaces":[{"interface":"back","average":{"1min":1.0309999999999999,"5min":1.351,"15min":1.552},"min":{"1min":0.46200000000000002,"5min":0.373,"15min":0.36399999999999999},"max":{"1min":1.9350000000000001,"5min":6.7969999999999997,"15min":9.8279999999999994},"last":0.94099999999999995},{"interface":"front","average":{"1min":0.92100000000000004,"5min":1.506,"15min":1.623},"min":{"1min":0.56499999999999995,"5min":0.37,"15min":0.36899999999999999},"max":{"1min":1.2689999999999999,"5min":7.3120000000000003,"15min":10.111000000000001},"last":0.54100000000000004}]},{"osd":7,"last update":"Sun Oct 1 10:56:22 2023","interfaces":[{"interface":"back","average":{"1min":0.95799999999999996,"5min":1.5720000000000001,"15min":1.6870000000000001},"min":{"1min":0.54200000000000004,"5min":0.45200000000000001,"15min":0.36899999999999999},"max":{"1min":1.284,"5min":13.699,"15min":15.839},"last":0.83399999999999996},{"interface":"front","average":{"1min":0.95399999999999996,"5min":1.837,"15min":1.71},"min":{"1min":0.51200000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":1.389,"5min":7.2679999999999998,"15min":8.9440000000000008},"last":0.61099999999999999}]}]},{"osd":2,"up_from":116,"seq":498216206580,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2255096,"kb_used_data":1784816,"kb_used_omap":7,"kb_used_meta":470264,"kb_avail":91490056,"statfs":{"total":95995035648,"available":93685817344,"internally_reserved":0,"allocated":1827651584,"data_stored":1826715039,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":481550417},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3,0,0,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:56:45 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":1.2809999999999999,"15min":2.0390000000000001},"min":{"1min":0.245,"5min":0.19800000000000001,"15min":0.16900000000000001},"max":{"1min":0.73799999999999999,"5min":8.4870000000000001,"15min":147.87},"last":2.8650000000000002},{"interface":"front","average":{"1min":0.57799999999999996,"5min":1.4370000000000001,"15min":2.1749999999999998},"min":{"1min":0.30499999999999999,"5min":0.216,"15min":0.19500000000000001},"max":{"1min":0.97099999999999997,"5min":9.0649999999999995,"15min":148.03100000000001},"last":6.6399999999999997}]},{"osd":1,"last update":"Sun Oct 1 10:56:45 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":1.369,"15min":1.948},"min":{"1min":0.39500000000000002,"5min":0.217,"15min":0.217},"max":{"1min":0.84599999999999997,"5min":7.9880000000000004,"15min":122.249},"last":0.95799999999999996},{"interface":"front","average":{"1min":0.53700000000000003,"5min":2.1600000000000001,"15min":2.2559999999999998},"min":{"1min":0.32200000000000001,"5min":0.245,"15min":0.20599999999999999},"max":{"1min":0.83999999999999997,"5min":11.535,"15min":121.495},"last":0.46400000000000002}]},{"osd":3,"last update":"Sun Oct 1 10:57:11 2023","interfaces":[{"interface":"back","average":{"1min":1.21,"5min":1.631,"15min":2.222},"min":{"1min":0.36599999999999999,"5min":0.25800000000000001,"15min":0.20999999999999999},"max":{"1min":3.484,"5min":11.029999999999999,"15min":124.393},"last":2.8889999999999998},{"interface":"front","average":{"1min":1.2549999999999999,"5min":1.4239999999999999,"15min":2.29},"min":{"1min":0.34599999999999997,"5min":0.221,"15min":0.20899999999999999},"max":{"1min":4.7990000000000004,"5min":8.5220000000000002,"15min":122.52200000000001},"last":1.5820000000000001}]},{"osd":4,"last update":"Sun Oct 1 10:56:53 2023","interfaces":[{"interface":"back","average":{"1min":1.097,"5min":1.4370000000000001,"15min":1.631},"min":{"1min":0.69299999999999995,"5min":0.38300000000000001,"15min":0.35799999999999998},"max":{"1min":3.9590000000000001,"5min":7.9480000000000004,"15min":60.994},"last":0.995},{"interface":"front","average":{"1min":0.97699999999999998,"5min":2.073,"15min":1.9259999999999999},"min":{"1min":0.63400000000000001,"5min":0.34100000000000003,"15min":0.30299999999999999},"max":{"1min":1.546,"5min":11.574,"15min":63.012},"last":0.54100000000000004}]},{"osd":5,"last update":"Sun Oct 1 10:56:45 2023","interfaces":[{"interface":"back","average":{"1min":0.97499999999999998,"5min":1.548,"15min":1.8},"min":{"1min":0.63600000000000001,"5min":0.36599999999999999,"15min":0.34699999999999998},"max":{"1min":1.3520000000000001,"5min":8.5440000000000005,"15min":61.021000000000001},"last":1.0229999999999999},{"interface":"front","average":{"1min":0.92900000000000005,"5min":2.218,"15min":2.0339999999999998},"min":{"1min":0.71599999999999997,"5min":0.36499999999999999,"15min":0.34100000000000003},"max":{"1min":1.3140000000000001,"5min":11.609,"15min":63.145000000000003},"last":0.498}]},{"osd":6,"last update":"Sun Oct 1 10:56:45 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":2.1880000000000002,"15min":1.9370000000000001},"min":{"1min":0.64600000000000002,"5min":0.38200000000000001,"15min":0.32100000000000001},"max":{"1min":1.419,"5min":11.661,"15min":63.289000000000001},"last":2.2919999999999998},{"interface":"front","average":{"1min":0.94599999999999995,"5min":1.7509999999999999,"15min":1.6459999999999999},"min":{"1min":0.65500000000000003,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":1.3919999999999999,"5min":9.2279999999999998,"15min":60.579000000000001},"last":0.42799999999999999}]},{"osd":7,"last update":"Sun Oct 1 10:56:45 2023","interfaces":[{"interface":"back","average":{"1min":0.96099999999999997,"5min":1.46,"15min":1.5409999999999999},"min":{"1min":0.55800000000000005,"5min":0.35599999999999998,"15min":0.34300000000000003},"max":{"1min":1.4650000000000001,"5min":8.6859999999999999,"15min":60.493000000000002},"last":2.8330000000000002},{"interface":"front","average":{"1min":0.99299999999999999,"5min":1.569,"15min":1.831},"min":{"1min":0.72799999999999998,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":1.373,"5min":7.9279999999999999,"15min":60.956000000000003},"last":2.3980000000000001}]}]},{"osd":3,"up_from":121,"seq":519691043053,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2104280,"kb_used_data":1586000,"kb_used_omap":2,"kb_used_meta":518269,"kb_avail":91640872,"statfs":{"total":95995035648,"available":93840252928,"internally_reserved":0,"allocated":1624064000,"data_stored":1623156585,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":530707509},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,2,3,2,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:56:12 2023","interfaces":[{"interface":"back","average":{"1min":1.1279999999999999,"5min":1.4610000000000001,"15min":2.758},"min":{"1min":0.252,"5min":0.192,"15min":0.187},"max":{"1min":10.608000000000001,"5min":10.608000000000001,"15min":129.953},"last":0.443},{"interface":"front","average":{"1min":1.3169999999999999,"5min":1.8160000000000001,"15min":3.133},"min":{"1min":0.34799999999999998,"5min":0.189,"15min":0.17299999999999999},"max":{"1min":9.8680000000000003,"5min":11.624000000000001,"15min":129.53999999999999},"last":0.34599999999999997}]},{"osd":1,"last update":"Sun Oct 1 10:56:12 2023","interfaces":[{"interface":"back","average":{"1min":1.119,"5min":1.851,"15min":1.8779999999999999},"min":{"1min":0.32200000000000001,"5min":0.29699999999999999,"15min":0.193},"max":{"1min":10.574999999999999,"5min":10.574999999999999,"15min":77.361000000000004},"last":0.36599999999999999},{"interface":"front","average":{"1min":1.218,"5min":1.919,"15min":2.0139999999999998},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":10.685,"5min":13.529999999999999,"15min":77.867000000000004},"last":0.39600000000000002}]},{"osd":2,"last update":"Sun Oct 1 10:56:12 2023","interfaces":[{"interface":"back","average":{"1min":1.5669999999999999,"5min":1.9450000000000001,"15min":1.498},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":12.238,"5min":15.148,"15min":15.148},"last":0.83899999999999997},{"interface":"front","average":{"1min":1.6040000000000001,"5min":1.944,"15min":1.7110000000000001},"min":{"1min":0.41599999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":10.66,"5min":10.66,"15min":75.299999999999997},"last":0.59399999999999997}]},{"osd":4,"last update":"Sun Oct 1 10:56:58 2023","interfaces":[{"interface":"back","average":{"1min":1.371,"5min":1.571,"15min":1.6439999999999999},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.34599999999999997},"max":{"1min":2.9980000000000002,"5min":10.724,"15min":75.388999999999996},"last":2.294},{"interface":"front","average":{"1min":1.1719999999999999,"5min":1.7290000000000001,"15min":1.831},"min":{"1min":0.373,"5min":0.308,"15min":0.29999999999999999},"max":{"1min":3.1549999999999998,"5min":13.391999999999999,"15min":75.165999999999997},"last":0.95299999999999996}]},{"osd":5,"last update":"Sun Oct 1 10:56:12 2023","interfaces":[{"interface":"back","average":{"1min":1.0620000000000001,"5min":1.619,"15min":1.9039999999999999},"min":{"1min":0.60099999999999998,"5min":0.376,"15min":0.33100000000000002},"max":{"1min":2.4809999999999999,"5min":13.333,"15min":75.221000000000004},"last":0.42399999999999999},{"interface":"front","average":{"1min":1.601,"5min":1.7310000000000001,"15min":2.0310000000000001},"min":{"1min":0.53000000000000003,"5min":0.35999999999999999,"15min":0.33200000000000002},"max":{"1min":12.335000000000001,"5min":12.335000000000001,"15min":75.231999999999999},"last":0.88300000000000001}]},{"osd":6,"last update":"Sun Oct 1 10:56:12 2023","interfaces":[{"interface":"back","average":{"1min":1.1359999999999999,"5min":1.9510000000000001,"15min":2.0699999999999998},"min":{"1min":0.55300000000000005,"5min":0.35399999999999998,"15min":0.34999999999999998},"max":{"1min":2.4609999999999999,"5min":8.8740000000000006,"15min":75.257000000000005},"last":4.4480000000000004},{"interface":"front","average":{"1min":1.774,"5min":1.8320000000000001,"15min":2.133},"min":{"1min":0.58199999999999996,"5min":0.36799999999999999,"15min":0.33500000000000002},"max":{"1min":12.378,"5min":12.378,"15min":75.194000000000003},"last":4.399}]},{"osd":7,"last update":"Sun Oct 1 10:56:12 2023","interfaces":[{"interface":"back","average":{"1min":1.726,"5min":1.8129999999999999,"15min":1.97},"min":{"1min":0.73099999999999998,"5min":0.40999999999999998,"15min":0.35099999999999998},"max":{"1min":12.416,"5min":12.416,"15min":75.138999999999996},"last":0.91600000000000004},{"interface":"front","average":{"1min":1.597,"5min":1.7390000000000001,"15min":1.867},"min":{"1min":0.63800000000000001,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":10.755000000000001,"5min":10.755000000000001,"15min":75.355000000000004},"last":2.327}]}]},{"osd":4,"up_from":126,"seq":541165879526,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":945020,"kb_used_data":543796,"kb_used_omap":10,"kb_used_meta":401205,"kb_avail":92800132,"statfs":{"total":95995035648,"available":95027335168,"internally_reserved":0,"allocated":556847104,"data_stored":555906225,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":410834532},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:56:52 2023","interfaces":[{"interface":"back","average":{"1min":0.83699999999999997,"5min":1.2170000000000001,"15min":1.859},"min":{"1min":0.503,"5min":0.27300000000000002,"15min":0.24399999999999999},"max":{"1min":1.238,"5min":9.4700000000000006,"15min":62.576000000000001},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.91300000000000003,"5min":1.2230000000000001,"15min":1.863},"min":{"1min":0.38600000000000001,"5min":0.26200000000000001,"15min":0.251},"max":{"1min":1.377,"5min":8.7409999999999997,"15min":64.941000000000003},"last":0.48999999999999999}]},{"osd":1,"last update":"Sun Oct 1 10:56:52 2023","interfaces":[{"interface":"back","average":{"1min":0.81000000000000005,"5min":1.349,"15min":1.913},"min":{"1min":0.50600000000000001,"5min":0.28699999999999998,"15min":0.27300000000000002},"max":{"1min":1.355,"5min":11.664,"15min":97.622},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.89900000000000002,"5min":1.5229999999999999,"15min":2.0390000000000001},"min":{"1min":0.35999999999999999,"5min":0.33900000000000002,"15min":0.28599999999999998},"max":{"1min":1.284,"5min":19.719999999999999,"15min":97.346999999999994},"last":0.69599999999999995}]},{"osd":2,"last update":"Sun Oct 1 10:56:52 2023","interfaces":[{"interface":"back","average":{"1min":1.093,"5min":2.2759999999999998,"15min":2.2309999999999999},"min":{"1min":0.77600000000000002,"5min":0.32700000000000001,"15min":0.29899999999999999},"max":{"1min":2.879,"5min":68.064999999999998,"15min":117.229},"last":0.627},{"interface":"front","average":{"1min":1.1100000000000001,"5min":2.2810000000000001,"15min":2.1509999999999998},"min":{"1min":0.46700000000000003,"5min":0.33300000000000002,"15min":0.29699999999999999},"max":{"1min":3.5369999999999999,"5min":68.379000000000005,"15min":116.922},"last":0.65400000000000003}]},{"osd":3,"last update":"Sun Oct 1 10:56:52 2023","interfaces":[{"interface":"back","average":{"1min":0.97699999999999998,"5min":1.3089999999999999,"15min":2.1760000000000002},"min":{"1min":0.71199999999999997,"5min":0.307,"15min":0.29499999999999998},"max":{"1min":1.2809999999999999,"5min":8.8100000000000005,"15min":61.398000000000003},"last":3.3450000000000002},{"interface":"front","average":{"1min":0.999,"5min":1.4730000000000001,"15min":2.1840000000000002},"min":{"1min":0.60799999999999998,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":1.4410000000000001,"5min":7.6210000000000004,"15min":61.463000000000001},"last":2.6890000000000001}]},{"osd":5,"last update":"Sun Oct 1 10:56:52 2023","interfaces":[{"interface":"back","average":{"1min":0.96699999999999997,"5min":2.1269999999999998,"15min":1.804},"min":{"1min":0.44800000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":1.292,"5min":84.307000000000002,"15min":84.307000000000002},"last":0.55200000000000005},{"interface":"front","average":{"1min":1.0229999999999999,"5min":2.3849999999999998,"15min":1.9199999999999999},"min":{"1min":0.45000000000000001,"5min":0.309,"15min":0.28999999999999998},"max":{"1min":1.4770000000000001,"5min":84.234999999999999,"15min":84.234999999999999},"last":2.5049999999999999}]},{"osd":6,"last update":"Sun Oct 1 10:56:52 2023","interfaces":[{"interface":"back","average":{"1min":1.0569999999999999,"5min":1.3100000000000001,"15min":1.6339999999999999},"min":{"1min":0.57299999999999995,"5min":0.36099999999999999,"15min":0.33800000000000002},"max":{"1min":1.546,"5min":9.0109999999999992,"15min":13.646000000000001},"last":0.86499999999999999},{"interface":"front","average":{"1min":1.0049999999999999,"5min":1.3220000000000001,"15min":1.641},"min":{"1min":0.59499999999999997,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":1.3500000000000001,"5min":8.5700000000000003,"15min":14.446999999999999},"last":0.89900000000000002}]},{"osd":7,"last update":"Sun Oct 1 10:56:52 2023","interfaces":[{"interface":"back","average":{"1min":1.071,"5min":2.706,"15min":1.9670000000000001},"min":{"1min":0.64300000000000002,"5min":0.36699999999999999,"15min":0.33900000000000002},"max":{"1min":1.3939999999999999,"5min":119.56100000000001,"15min":119.56100000000001},"last":0.998},{"interface":"front","average":{"1min":1.127,"5min":2.5529999999999999,"15min":2.1259999999999999},"min":{"1min":0.62,"5min":0.40200000000000002,"15min":0.30399999999999999},"max":{"1min":1.5229999999999999,"5min":120.44499999999999,"15min":120.44499999999999},"last":1.7490000000000001}]}]},{"osd":5,"up_from":36,"seq":154618823207,"num_pgs":23,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1265260,"kb_used_data":360996,"kb_used_omap":0,"kb_used_meta":904256,"kb_avail":92479892,"statfs":{"total":95995035648,"available":94699409408,"internally_reserved":0,"allocated":369659904,"data_stored":369138082,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":925958144},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:56:15 2023","interfaces":[{"interface":"back","average":{"1min":1.171,"5min":2.3769999999999998,"15min":1.9950000000000001},"min":{"1min":0.28499999999999998,"5min":0.28000000000000003,"15min":0.24099999999999999},"max":{"1min":6.9299999999999997,"5min":123.39700000000001,"15min":123.39700000000001},"last":2.1560000000000001},{"interface":"front","average":{"1min":1.1399999999999999,"5min":2.5960000000000001,"15min":2.2389999999999999},"min":{"1min":0.39600000000000002,"5min":0.28399999999999997,"15min":0.26900000000000002},"max":{"1min":3.7610000000000001,"5min":124.051,"15min":124.051},"last":0.44}]},{"osd":1,"last update":"Sun Oct 1 10:56:15 2023","interfaces":[{"interface":"back","average":{"1min":0.93600000000000005,"5min":1.2589999999999999,"15min":2.3559999999999999},"min":{"1min":0.313,"5min":0.27700000000000002,"15min":0.27100000000000002},"max":{"1min":2.9449999999999998,"5min":9.766,"15min":202.96600000000001},"last":2.0270000000000001},{"interface":"front","average":{"1min":1.0389999999999999,"5min":1.482,"15min":2.6829999999999998},"min":{"1min":0.47199999999999998,"5min":0.32500000000000001,"15min":0.28100000000000003},"max":{"1min":3.2679999999999998,"5min":7.7290000000000001,"15min":203.05099999999999},"last":0.52500000000000002}]},{"osd":2,"last update":"Sun Oct 1 10:56:48 2023","interfaces":[{"interface":"back","average":{"1min":0.95299999999999996,"5min":1.7470000000000001,"15min":1.6359999999999999},"min":{"1min":0.66200000000000003,"5min":0.307,"15min":0.29199999999999998},"max":{"1min":1.24,"5min":26.242000000000001,"15min":26.242000000000001},"last":3.0049999999999999},{"interface":"front","average":{"1min":0.92900000000000005,"5min":1.889,"15min":1.948},"min":{"1min":0.57499999999999996,"5min":0.317,"15min":0.317},"max":{"1min":1.177,"5min":25.643999999999998,"15min":25.643999999999998},"last":2.2040000000000002}]},{"osd":3,"last update":"Sun Oct 1 10:56:15 2023","interfaces":[{"interface":"back","average":{"1min":1.2829999999999999,"5min":3.3050000000000002,"15min":3.1360000000000001},"min":{"1min":0.54900000000000004,"5min":0.32400000000000001,"15min":0.316},"max":{"1min":3.7149999999999999,"5min":166.29599999999999,"15min":166.29599999999999},"last":0.79100000000000004},{"interface":"front","average":{"1min":1.6120000000000001,"5min":3.589,"15min":3.0350000000000001},"min":{"1min":0.67500000000000004,"5min":0.34799999999999998,"15min":0.34699999999999998},"max":{"1min":6.0609999999999999,"5min":166.565,"15min":166.565},"last":3.464}]},{"osd":4,"last update":"Sun Oct 1 10:56:48 2023","interfaces":[{"interface":"back","average":{"1min":1.0369999999999999,"5min":1.27,"15min":1.5069999999999999},"min":{"1min":0.67200000000000004,"5min":0.379,"15min":0.29399999999999998},"max":{"1min":1.518,"5min":6.6349999999999998,"15min":15.170999999999999},"last":0.55000000000000004},{"interface":"front","average":{"1min":1.101,"5min":1.1879999999999999,"15min":1.536},"min":{"1min":0.66100000000000003,"5min":0.35699999999999998,"15min":0.316},"max":{"1min":1.363,"5min":9.1699999999999999,"15min":9.7270000000000003},"last":2.0840000000000001}]},{"osd":6,"last update":"Sun Oct 1 10:56:15 2023","interfaces":[{"interface":"back","average":{"1min":1.8180000000000001,"5min":2.0840000000000001,"15min":1.871},"min":{"1min":0.65600000000000003,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":8.9730000000000008,"5min":61.768999999999998,"15min":61.768999999999998},"last":2.1230000000000002},{"interface":"front","average":{"1min":1.085,"5min":1.9990000000000001,"15min":1.9730000000000001},"min":{"1min":0.497,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":2.5630000000000002,"5min":62.396999999999998,"15min":62.396999999999998},"last":3.5099999999999998}]},{"osd":7,"last update":"Sun Oct 1 10:56:15 2023","interfaces":[{"interface":"back","average":{"1min":1.1399999999999999,"5min":1.415,"15min":1.9059999999999999},"min":{"1min":0.53500000000000003,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":4.1230000000000002,"5min":9.1470000000000002,"15min":13.673999999999999},"last":0.84299999999999997},{"interface":"front","average":{"1min":1.514,"5min":1.3859999999999999,"15min":1.758},"min":{"1min":0.42099999999999999,"5min":0.38700000000000001,"15min":0.38},"max":{"1min":5.3239999999999998,"5min":9.1340000000000003,"15min":11.635},"last":2.3460000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1143341056,"data_stored":1143341056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1001390080,"data_stored":1001390080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1873674240,"data_stored":1873674240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1642332160,"data_stored":1642332160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":563347456,"data_stored":563347456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":377028608,"data_stored":377028608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1307705344,"data_stored":1307705344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1295974400,"data_stored":1295974400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T10:57:22.908 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd pool set device_health_metrics pgp_num 11 2023-10-01T10:57:22.938 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:22.999 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:23.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:23.002 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:57:23.045 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 37 16 56261 56245 94.9951 75.75 0.012347 0.01052 2023-10-01T10:57:23.103 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:23.129 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:23.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:23.131 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:23.232 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:23.267 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:23.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:23.269 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:23.370 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:23.400 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:23.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:23.406 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:57:23.507 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:23.534 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:23.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:23.536 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:57:23.634 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:23 smithi173 bash[58816]: cluster 2023-10-01T10:57:22.088719+0000 mgr.y (mgr.25072) 1197 : cluster 0 pgmap v1209: 89 pgs: 89 active+clean; 3.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-10-01T10:57:23.637 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:23.666 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:23.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:23.668 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:23.769 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:23.802 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:23.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:23.803 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:23.904 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:23.928 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:23.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:23.931 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:57:24.032 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:24.042 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:23 smithi078 bash[75299]: cluster 2023-10-01T10:57:22.088719+0000 mgr.y (mgr.25072) 1197 : cluster 0 pgmap v1209: 89 pgs: 89 active+clean; 3.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-10-01T10:57:24.042 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:23 smithi078 bash[78274]: cluster 2023-10-01T10:57:22.088719+0000 mgr.y (mgr.25072) 1197 : cluster 0 pgmap v1209: 89 pgs: 89 active+clean; 3.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-10-01T10:57:24.044 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 38 16 57430 57414 94.4178 73.0625 0.0161961 0.0105829 2023-10-01T10:57:24.062 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:24.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:24.068 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:57:24.169 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:24.192 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:24.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:24.195 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:24.295 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:24.324 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:24.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:24.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:24.428 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:24.451 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:24.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:24.454 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:24.555 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:24.586 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:24.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:24.591 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:57:24.691 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:24.719 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:24.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:24.723 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:57:24.824 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:24.850 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:24.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:24.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:57:24.877 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T10:57:24.923 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T10:57:24.954 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:24.986 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:24.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:24.988 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:25.045 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 39 16 58517 58501 93.7384 67.9375 0.0173402 0.0106605 2023-10-01T10:57:25.089 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:25.115 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:25.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:25.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:25.219 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:25.245 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:57:25.250 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:25.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:25.255 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:57:25.356 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:25.384 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:25.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:25.390 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:25.491 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:25.518 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:25.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:25.522 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:25.623 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:25.656 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:25.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:25.659 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:57:25.759 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:25.786 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:25.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:25.792 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:25.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:25 smithi173 bash[58816]: cluster 2023-10-01T10:57:24.089548+0000 mgr.y (mgr.25072) 1198 : cluster 0 pgmap v1210: 89 pgs: 89 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-10-01T10:57:25.893 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:25.903 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:25 smithi078 bash[75299]: cluster 2023-10-01T10:57:24.089548+0000 mgr.y (mgr.25072) 1198 : cluster 0 pgmap v1210: 89 pgs: 89 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-10-01T10:57:25.903 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:25 smithi078 bash[78274]: cluster 2023-10-01T10:57:24.089548+0000 mgr.y (mgr.25072) 1198 : cluster 0 pgmap v1210: 89 pgs: 89 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-10-01T10:57:25.920 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:25.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:25.923 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:57:26.023 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:26.045 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:2023-10-01T10:57:26.044868+0000 min lat: 0.00107177 max lat: 0.29942 avg lat: 0.0107448 2023-10-01T10:57:26.045 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T10:57:26.046 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 40 16 59557 59541 93.0199 65 0.0184688 0.0107448 2023-10-01T10:57:26.059 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:26.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:26.062 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:26.163 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:26.190 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:26.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:26.193 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:57:26.294 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:26.332 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:26.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:26.335 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:57:26.436 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:26.461 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:26.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:26.463 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:57:26.564 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:26.589 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:26.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:26.594 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:26.694 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:26.721 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:26.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:26.724 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:57:26.825 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:26.854 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:26.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:26.859 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:26.959 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:26.983 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:26.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:26.986 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:27.045 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 41 16 60343 60327 91.9492 49.125 0.0036852 0.0108684 2023-10-01T10:57:27.087 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:27.109 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:27.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:27.112 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:57:27.213 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:27.255 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:27.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:27.259 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:57:27.276 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:57:27.360 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:27.392 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:27.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:27.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:57:27.495 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:27.520 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:27.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:27.522 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:57:27.623 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:27.654 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:27.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:27.657 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:27.757 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:27.766 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:27 smithi078 bash[75299]: cluster 2023-10-01T10:57:26.090138+0000 mgr.y (mgr.25072) 1199 : cluster 0 pgmap v1211: 89 pgs: 89 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-10-01T10:57:27.767 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:27 smithi078 bash[78274]: cluster 2023-10-01T10:57:26.090138+0000 mgr.y (mgr.25072) 1199 : cluster 0 pgmap v1211: 89 pgs: 89 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-10-01T10:57:27.786 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:27.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:27.790 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:57:27.891 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:27.901 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:27 smithi173 bash[58816]: cluster 2023-10-01T10:57:26.090138+0000 mgr.y (mgr.25072) 1199 : cluster 0 pgmap v1211: 89 pgs: 89 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2023-10-01T10:57:27.920 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:27.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:27.924 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:57:28.024 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:28.045 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 42 16 61381 61365 91.3045 64.875 0.00453009 0.0109446 2023-10-01T10:57:28.048 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:28.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:28.050 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:57:28.151 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:28.184 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:28.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:28.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:57:28.293 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:28.325 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:28.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:28.328 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:57:28.428 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:28.458 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:28.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:28.462 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:57:28.562 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:28.589 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:28.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:28.595 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:57:28.696 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:28.725 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:28.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:28.729 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:57:28.830 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:28.862 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:28.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:28.869 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:57:28.969 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:28.992 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:28.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:28.994 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:57:29.045 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 43 16 62269 62253 90.4717 55.5 0.0125082 0.0110443 2023-10-01T10:57:29.095 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:29.116 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:29.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:29.118 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:57:29.219 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:29.252 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:29.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:29.255 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:57:29.308 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:57:29.309 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:57:29.357 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:29.380 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:29.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:29.384 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:57:29.485 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:29.515 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:29.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:29.518 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:29.619 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:29.645 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:29 smithi078 bash[75299]: cluster 2023-10-01T10:57:28.090840+0000 mgr.y (mgr.25072) 1200 : cluster 0 pgmap v1212: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2023-10-01T10:57:29.646 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:29.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:29.649 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:29.750 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:29.771 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:29.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:29.775 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:57:29.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:29 smithi173 bash[58816]: cluster 2023-10-01T10:57:28.090840+0000 mgr.y (mgr.25072) 1200 : cluster 0 pgmap v1212: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2023-10-01T10:57:29.876 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:29.903 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:29.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:29.905 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:57:30.006 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:30.032 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:30.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:30.037 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:57:30.045 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 44 16 63384 63368 89.9992 69.6875 0.00896877 0.0111052 2023-10-01T10:57:30.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:29 smithi078 bash[78274]: cluster 2023-10-01T10:57:28.090840+0000 mgr.y (mgr.25072) 1200 : cluster 0 pgmap v1212: 89 pgs: 89 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2023-10-01T10:57:30.138 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:30.169 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:30.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:30.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:57:30.274 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:30.299 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:30.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:30.302 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:30.401 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:57:30 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:57:30] "GET /metrics HTTP/1.1" 200 36376 "" "Prometheus/2.33.4" 2023-10-01T10:57:30.404 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:30.437 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:30.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:30.441 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:30.542 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:30.567 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:30.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:30.570 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:57:30.671 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:30.708 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:30.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:30.712 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:57:30.812 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:30.840 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:30.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:30.843 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:57:30.944 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:30.975 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:30.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:30.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:31.045 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 45 16 64496 64480 89.5437 69.5 0.0232805 0.0111615 2023-10-01T10:57:31.079 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:31.102 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:31.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:31.107 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:57:31.207 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:31.229 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:31.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:31.232 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:57:31.333 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:31.356 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:31.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:31.359 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:57:31.460 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:31.492 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:31.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:31.495 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:31.596 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:31.614 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:31.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:31.616 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:57:31.717 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:31.726 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:31 smithi078 bash[78274]: cluster 2023-10-01T10:57:30.091523+0000 mgr.y (mgr.25072) 1201 : cluster 0 pgmap v1213: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-01T10:57:31.726 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:31 smithi078 bash[75299]: cluster 2023-10-01T10:57:30.091523+0000 mgr.y (mgr.25072) 1201 : cluster 0 pgmap v1213: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-01T10:57:31.742 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:31.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:31.746 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:31.847 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:31.877 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:31.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:31.881 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:57:31.982 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:32.012 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:32.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:32.016 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:57:32.045 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 46 16 65450 65434 88.8932 59.625 0.0136326 0.0112439 2023-10-01T10:57:32.117 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:32.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:31 smithi173 bash[58816]: cluster 2023-10-01T10:57:30.091523+0000 mgr.y (mgr.25072) 1201 : cluster 0 pgmap v1213: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-10-01T10:57:32.141 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:32.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:32.145 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:57:32.246 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:32.272 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:32.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:32.275 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:57:32.375 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:32.399 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:32.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:32.404 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:57:32.505 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:32.532 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:32.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:32.535 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:32.636 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:32.662 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:32.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:32.666 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:57:32.767 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:32.796 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:32.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:32.799 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:32.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:32 smithi173 bash[58816]: audit 2023-10-01T10:57:32.447065+0000 mon.a (mon.0) 1506 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:57:32.900 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:32.936 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:32.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:32.938 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:33.039 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:33.045 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 47 15 66523 66508 88.4299 67.125 0.00940633 0.011303 2023-10-01T10:57:33.062 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:33.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:33.065 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:57:33.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:32 smithi078 bash[75299]: audit 2023-10-01T10:57:32.447065+0000 mon.a (mon.0) 1506 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:57:33.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:32 smithi078 bash[78274]: audit 2023-10-01T10:57:32.447065+0000 mon.a (mon.0) 1506 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:57:33.166 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:33.196 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:33.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:33.202 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:33.302 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:33.328 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:33.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:33.332 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:57:33.433 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:33.457 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:33.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:33.460 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:33.561 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:33.589 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:33.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:33.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:57:33.695 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:33.729 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:33.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:33.736 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:57:33.836 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:33.860 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:33.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:33.862 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:57:33.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:33 smithi173 bash[58816]: cluster 2023-10-01T10:57:32.091935+0000 mgr.y (mgr.25072) 1202 : cluster 0 pgmap v1214: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 54 MiB/s wr, 870 op/s 2023-10-01T10:57:33.963 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:33.988 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:33.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:33.990 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:57:34.041 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:33 smithi078 bash[75299]: cluster 2023-10-01T10:57:32.091935+0000 mgr.y (mgr.25072) 1202 : cluster 0 pgmap v1214: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 54 MiB/s wr, 870 op/s 2023-10-01T10:57:34.042 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:33 smithi078 bash[78274]: cluster 2023-10-01T10:57:32.091935+0000 mgr.y (mgr.25072) 1202 : cluster 0 pgmap v1214: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 54 MiB/s wr, 870 op/s 2023-10-01T10:57:34.045 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 48 16 67474 67458 87.8245 59.375 0.140189 0.0113622 2023-10-01T10:57:34.091 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:34.124 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:34.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:34.128 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:34.229 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:34.254 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:34.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:34.257 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:57:34.358 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:34.382 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:34.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:34.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:57:34.485 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:34.516 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:34.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:34.520 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:57:34.620 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:34.655 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:34.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:34.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:57:34.763 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:34.785 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:34.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:34.788 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:57:34.889 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:34.925 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:34.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:34.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:57:35.028 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:35.046 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 49 16 68204 68188 86.9632 45.625 0.0158204 0.0114934 2023-10-01T10:57:35.054 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:35.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:35.058 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:35.159 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:35.184 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:35.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:35.186 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:57:35.287 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:35.313 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:35.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:35.317 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:57:35.418 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:35.444 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:35.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:35.446 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:57:35.547 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:35.583 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:35.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:35.587 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:57:35.688 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:35.716 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:35.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:35.720 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:57:35.821 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:35.830 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:35 smithi173 bash[58816]: cluster 2023-10-01T10:57:34.092620+0000 mgr.y (mgr.25072) 1203 : cluster 0 pgmap v1215: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 72 MiB/s wr, 1.14k op/s 2023-10-01T10:57:35.847 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:35.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:35.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:57:35.951 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:35.977 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:35 smithi078 bash[75299]: cluster 2023-10-01T10:57:34.092620+0000 mgr.y (mgr.25072) 1203 : cluster 0 pgmap v1215: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 72 MiB/s wr, 1.14k op/s 2023-10-01T10:57:35.977 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:35 smithi078 bash[78274]: cluster 2023-10-01T10:57:34.092620+0000 mgr.y (mgr.25072) 1203 : cluster 0 pgmap v1215: 89 pgs: 89 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 72 MiB/s wr, 1.14k op/s 2023-10-01T10:57:35.982 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:35.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:35.986 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:57:36.046 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 50 16 69290 69274 86.5813 67.875 0.0112014 0.0115448 2023-10-01T10:57:36.087 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:36.110 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:36.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:36.114 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:57:36.214 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:36.241 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:36.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:36.243 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:57:36.344 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:36.370 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:36.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:36.374 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:36.475 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:36.492 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:36.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:36.493 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:57:36.594 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:36.622 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:36.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:36.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:36.724 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:36.750 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:36.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:36.755 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:57:36.856 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:36.884 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:36.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:36.887 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:36.988 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:37.015 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:37.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:37.017 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:37.046 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 51 16 70312 70296 86.136 63.875 0.0250353 0.0116043 2023-10-01T10:57:37.118 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:37.136 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:37.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:37.138 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:57:37.238 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:37.265 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:37.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:37.268 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:37.369 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:37.400 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:37.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:37.404 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:37.505 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:37.538 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:37.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:37.544 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:57:37.644 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:37.678 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:37.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:37.681 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:57:37.782 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:37.792 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:37 smithi078 bash[78274]: cluster 2023-10-01T10:57:36.093225+0000 mgr.y (mgr.25072) 1204 : cluster 0 pgmap v1216: 89 pgs: 89 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-01T10:57:37.792 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:37 smithi078 bash[75299]: cluster 2023-10-01T10:57:36.093225+0000 mgr.y (mgr.25072) 1204 : cluster 0 pgmap v1216: 89 pgs: 89 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-01T10:57:37.808 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:37.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:37.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:57:37.912 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:37.923 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:37 smithi173 bash[58816]: cluster 2023-10-01T10:57:36.093225+0000 mgr.y (mgr.25072) 1204 : cluster 0 pgmap v1216: 89 pgs: 89 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-10-01T10:57:37.944 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:37.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:37.946 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:57:38.046 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 52 16 71264 71248 85.6237 59.5 0.0224769 0.0116733 2023-10-01T10:57:38.046 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:38.079 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:38.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:38.085 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:38.185 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:38.221 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:38.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:38.230 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:38.331 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:38.356 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:38.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:38.359 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:57:38.460 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:38.489 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:38.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:38.493 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:38.594 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:38.625 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:38.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:38.629 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:57:38.729 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:38.759 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:38.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:38.766 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:57:38.866 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:38.890 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:38.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:38.895 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:57:38.996 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:39.028 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:39.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:39.032 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:57:39.046 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 53 16 72194 72178 85.1048 58.125 0.0217573 0.0117438 2023-10-01T10:57:39.133 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:39.168 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:39.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:39.174 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:57:39.274 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:39.296 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:39.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:39.298 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:39.399 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:39.435 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:39.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:39.439 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:39.540 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:39.571 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:39.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:39.576 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:39.676 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:39.703 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:39.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:39.708 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:39.734 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:39 smithi078 bash[78274]: cluster 2023-10-01T10:57:38.096509+0000 mgr.y (mgr.25072) 1205 : cluster 0 pgmap v1217: 89 pgs: 89 active+clean; 4.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 59 MiB/s wr, 950 op/s 2023-10-01T10:57:39.735 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:39 smithi078 bash[75299]: cluster 2023-10-01T10:57:38.096509+0000 mgr.y (mgr.25072) 1205 : cluster 0 pgmap v1217: 89 pgs: 89 active+clean; 4.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 59 MiB/s wr, 950 op/s 2023-10-01T10:57:39.809 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:39.837 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:39.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:39.842 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:57:39.942 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:39.951 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:39 smithi173 bash[58816]: cluster 2023-10-01T10:57:38.096509+0000 mgr.y (mgr.25072) 1205 : cluster 0 pgmap v1217: 89 pgs: 89 active+clean; 4.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 59 MiB/s wr, 950 op/s 2023-10-01T10:57:39.968 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:39.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:39.971 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:57:40.046 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 54 16 73212 73196 84.7069 63.625 0.0116786 0.0117998 2023-10-01T10:57:40.072 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:40.101 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:40.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:40.105 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:57:40.206 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:40.243 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:40.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:40.248 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:40.349 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:40.378 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:40.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:40.381 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:57:40.482 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:40.493 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:57:40 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:57:40] "GET /metrics HTTP/1.1" 200 36377 "" "Prometheus/2.33.4" 2023-10-01T10:57:40.507 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:40.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:40.509 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:57:40.610 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:40.642 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:40.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:40.646 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:57:40.747 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:40.771 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:40.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:40.776 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:40.876 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:40.906 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:40.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:40.910 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:57:41.011 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:41.032 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:41.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:41.042 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:57:41.046 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 55 16 74093 74077 84.1679 55.0625 0.0163837 0.011875 2023-10-01T10:57:41.143 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:41.171 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:41.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:41.173 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:57:41.274 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:41.304 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:41.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:41.308 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:57:41.409 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:41.434 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:41.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:41.436 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:41.537 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:41.561 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:41.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:41.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:41.667 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:41.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:41.709 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:41.710 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:41.810 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:41.821 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:41 smithi173 bash[58816]: cluster 2023-10-01T10:57:40.097209+0000 mgr.y (mgr.25072) 1206 : cluster 0 pgmap v1218: 89 pgs: 89 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-10-01T10:57:41.838 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:41.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:41.841 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:41.942 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:41.965 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:41.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:41.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:57:42.046 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 56 16 75028 75012 83.7083 58.4375 0.0062329 0.0119295 2023-10-01T10:57:42.071 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:42.089 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:42.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:42.090 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:42.119 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:41 smithi078 bash[75299]: cluster 2023-10-01T10:57:40.097209+0000 mgr.y (mgr.25072) 1206 : cluster 0 pgmap v1218: 89 pgs: 89 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-10-01T10:57:42.122 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:41 smithi078 bash[78274]: cluster 2023-10-01T10:57:40.097209+0000 mgr.y (mgr.25072) 1206 : cluster 0 pgmap v1218: 89 pgs: 89 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-10-01T10:57:42.192 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:42.231 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:42.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:42.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:57:42.337 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:42.360 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:42.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:42.366 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:57:42.467 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:42.492 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:42.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:42.498 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:57:42.599 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:42.626 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:42.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:42.628 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:42.729 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:42.758 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:42.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:42.763 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:57:42.864 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:42.887 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:42.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:42.891 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:57:42.991 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:43.022 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:43.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:43.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:57:43.046 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 57 16 75987 75971 83.2912 59.9375 0.0175517 0.0119997 2023-10-01T10:57:43.128 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:43.152 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:43.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:43.154 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:43.255 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:43.283 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:43.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:43.285 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:57:43.386 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:43.412 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:43.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:43.414 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:43.515 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:43.543 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:43.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:43.548 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:57:43.649 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:43.676 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:43.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:43.681 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:43.781 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:43.808 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:43.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:43.811 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:57:43.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:43 smithi078 bash[78274]: cluster 2023-10-01T10:57:42.097742+0000 mgr.y (mgr.25072) 1207 : cluster 0 pgmap v1219: 89 pgs: 89 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 52 MiB/s wr, 829 op/s 2023-10-01T10:57:43.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:43 smithi078 bash[75299]: cluster 2023-10-01T10:57:42.097742+0000 mgr.y (mgr.25072) 1207 : cluster 0 pgmap v1219: 89 pgs: 89 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 52 MiB/s wr, 829 op/s 2023-10-01T10:57:43.911 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:43.940 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:43.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:43.943 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:43.969 INFO:teuthology.orchestra.run.smithi078.stdout:ERROR: (22) Invalid argument 2023-10-01T10:57:43.969 INFO:teuthology.orchestra.run.smithi078.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T10:57:44.044 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:44.046 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 58 16 76944 76928 82.8863 59.8125 0.0130086 0.0120577 2023-10-01T10:57:44.067 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:44.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:44.069 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:44.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:43 smithi173 bash[58816]: cluster 2023-10-01T10:57:42.097742+0000 mgr.y (mgr.25072) 1207 : cluster 0 pgmap v1219: 89 pgs: 89 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 52 MiB/s wr, 829 op/s 2023-10-01T10:57:44.169 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:44.199 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:44.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:44.203 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:44.304 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:44.333 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:44.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:44.337 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:57:44.438 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:44.468 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:44.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:44.471 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:57:44.572 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:44.605 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:44.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:44.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:44.710 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:44.746 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:44.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:44.751 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:44.852 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:44.882 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:44.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:44.887 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:57:44.988 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:45.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:45.022 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:45.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:57:45.047 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 59 16 77876 77860 82.4687 58.25 0.0112663 0.0121205 2023-10-01T10:57:45.123 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:45.152 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:45.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:45.157 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:57:45.257 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:45.283 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:45.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:45.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:57:45.387 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:45.413 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:45.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:45.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:45.479 INFO:teuthology.orchestra.run.smithi078.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-10-01T10:57:45.519 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:45.546 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:45.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:45.552 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:45.653 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:45.680 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:45.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:45.683 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:57:45.784 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:45.793 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:45 smithi078 bash[75299]: cluster 2023-10-01T10:57:44.099875+0000 mgr.y (mgr.25072) 1208 : cluster 0 pgmap v1220: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-01T10:57:45.793 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:45 smithi078 bash[75299]: audit 2023-10-01T10:57:45.472514+0000 mon.a (mon.0) 1507 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-01T10:57:45.793 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:45 smithi078 bash[75299]: audit 2023-10-01T10:57:45.472755+0000 mon.b (mon.1) 49 : audit 1 from='client.? 172.21.15.78:0/29933818' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-01T10:57:45.793 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:45 smithi078 bash[78274]: cluster 2023-10-01T10:57:44.099875+0000 mgr.y (mgr.25072) 1208 : cluster 0 pgmap v1220: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-01T10:57:45.793 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:45 smithi078 bash[78274]: audit 2023-10-01T10:57:45.472514+0000 mon.a (mon.0) 1507 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-01T10:57:45.793 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:45 smithi078 bash[78274]: audit 2023-10-01T10:57:45.472755+0000 mon.b (mon.1) 49 : audit 1 from='client.? 172.21.15.78:0/29933818' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-01T10:57:45.812 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:45.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:45.819 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:57:45.919 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:45.950 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:45.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:45.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:57:46.047 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:2023-10-01T10:57:46.046665+0000 min lat: 0.00107177 max lat: 0.325279 avg lat: 0.0121796 2023-10-01T10:57:46.047 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T10:57:46.047 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 60 16 78792 78776 82.0483 57.25 0.00470218 0.0121796 2023-10-01T10:57:46.057 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:46.080 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:46.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:46.082 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:46.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:45 smithi173 bash[58816]: cluster 2023-10-01T10:57:44.099875+0000 mgr.y (mgr.25072) 1208 : cluster 0 pgmap v1220: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-01T10:57:46.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:45 smithi173 bash[58816]: audit 2023-10-01T10:57:45.472514+0000 mon.a (mon.0) 1507 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-01T10:57:46.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:45 smithi173 bash[58816]: audit 2023-10-01T10:57:45.472755+0000 mon.b (mon.1) 49 : audit 1 from='client.? 172.21.15.78:0/29933818' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-01T10:57:46.183 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:46.215 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:46.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:46.220 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:57:46.321 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:46.344 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:46.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:46.346 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:57:46.446 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:46.484 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:46.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:46.488 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:57:46.589 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:46.618 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:46.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:46.621 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:57:46.722 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:46.748 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:46.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:46.753 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:57:46.854 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:46.882 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:46.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:46.892 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:57:46.993 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:47.026 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:47.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:47.032 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:57:47.047 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 61 16 79661 79645 81.5935 54.3125 0.0188435 0.0122505 2023-10-01T10:57:47.134 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:47.160 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:47.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:47.164 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:57:47.264 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:47.291 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:47.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:47.296 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:57:47.397 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:47.419 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:47.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:47.421 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:57:47.522 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:47.551 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:47.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:47.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:47.656 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:47.688 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:47.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:47.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:57:47.791 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:47.820 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:47.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:47.823 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:57:47.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:47 smithi173 bash[58816]: cluster 2023-10-01T10:57:46.100584+0000 mgr.y (mgr.25072) 1209 : cluster 0 pgmap v1221: 89 pgs: 89 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 59 MiB/s wr, 949 op/s 2023-10-01T10:57:47.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:47 smithi173 bash[58816]: audit 2023-10-01T10:57:47.447278+0000 mon.a (mon.0) 1508 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:57:47.924 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:47.948 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:47.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:47.950 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:48.047 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 62 16 80635 80619 81.2592 60.875 0.00559692 0.0123009 2023-10-01T10:57:48.051 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:48.059 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:47 smithi078 bash[78274]: cluster 2023-10-01T10:57:46.100584+0000 mgr.y (mgr.25072) 1209 : cluster 0 pgmap v1221: 89 pgs: 89 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 59 MiB/s wr, 949 op/s 2023-10-01T10:57:48.060 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:47 smithi078 bash[78274]: audit 2023-10-01T10:57:47.447278+0000 mon.a (mon.0) 1508 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:57:48.060 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:47 smithi078 bash[75299]: cluster 2023-10-01T10:57:46.100584+0000 mgr.y (mgr.25072) 1209 : cluster 0 pgmap v1221: 89 pgs: 89 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 59 MiB/s wr, 949 op/s 2023-10-01T10:57:48.060 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:47 smithi078 bash[75299]: audit 2023-10-01T10:57:47.447278+0000 mon.a (mon.0) 1508 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:57:48.077 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:48.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:48.081 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:57:48.182 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:48.222 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:48.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:48.227 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:57:48.328 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:48.353 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:48.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:48.360 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:57:48.461 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:48.485 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:48.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:48.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:48.593 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:48.622 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:48.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:48.625 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:57:48.726 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:48.755 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:48.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:48.760 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:57:48.861 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:48.898 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:48.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:48.901 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:57:49.001 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:49.031 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:49.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:49.037 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:57:49.047 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 63 16 81613 81597 80.9396 61.125 0.00414907 0.0123487 2023-10-01T10:57:49.137 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:49.162 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:49.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:49.165 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:57:49.266 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:49.289 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:49.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:49.292 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:57:49.392 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:49.417 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:49.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:49.420 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:57:49.521 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:49.547 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:49.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:49.550 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:49.651 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:49.687 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:49.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:49.690 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:57:49.703 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:49 smithi078 bash[78274]: cluster 2023-10-01T10:57:48.101076+0000 mgr.y (mgr.25072) 1210 : cluster 0 pgmap v1222: 89 pgs: 89 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 55 MiB/s wr, 886 op/s 2023-10-01T10:57:49.703 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:49 smithi078 bash[75299]: cluster 2023-10-01T10:57:48.101076+0000 mgr.y (mgr.25072) 1210 : cluster 0 pgmap v1222: 89 pgs: 89 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 55 MiB/s wr, 886 op/s 2023-10-01T10:57:49.792 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:49.799 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:49 smithi173 bash[58816]: cluster 2023-10-01T10:57:48.101076+0000 mgr.y (mgr.25072) 1210 : cluster 0 pgmap v1222: 89 pgs: 89 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 55 MiB/s wr, 886 op/s 2023-10-01T10:57:49.818 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:49.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:49.821 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:57:49.922 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:49.951 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:49.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:49.957 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:57:50.047 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 64 16 82372 82356 80.416 47.4375 0.0109714 0.0124297 2023-10-01T10:57:50.058 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:50.083 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:50.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:50.085 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:50.186 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:50.225 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:50.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:50.232 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:50.315 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-01T10:57:50.332 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:50.383 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:50.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:50.386 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:57:50.451 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-10-01T10:57:50.487 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:50.495 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:57:50 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:57:50] "GET /metrics HTTP/1.1" 200 36380 "" "Prometheus/2.33.4" 2023-10-01T10:57:50.517 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:50.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:50.521 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:57:50.621 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:50.646 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:50.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:50.648 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:57:50.748 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:50.779 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:50.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:50.782 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:57:50.882 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:50.907 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:50.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:50.909 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:57:51.010 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:51.033 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:51.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:51.040 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:51.047 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 65 16 83243 83227 80.0163 54.4375 0.0468569 0.012491 2023-10-01T10:57:51.136 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T10:57:51.141 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:51.167 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:51.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:51.170 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:57:51.175 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T10:57:51.226 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T10:57:51.271 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:51.273 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T10:57:51.311 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T10:57:51.312 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:51.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:51.314 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:51.347 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T10:57:51.391 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T10:57:51.415 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:51.429 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T10:57:51.444 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T10:57:51.444 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T10:57:51.444 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T10:57:51.444 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T10:57:51.444 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T10:57:51.444 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T10:57:51.445 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T10:57:51.445 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T10:57:51.457 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:51.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:51.461 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:57:51.562 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:51.593 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:51.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:51.597 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:57:51.698 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:51.729 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:51.730 INFO:teuthology.orchestra.run.smithi078.stderr:nodeep-scrub is unset 2023-10-01T10:57:51.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:51.732 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:57:51.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:51 smithi078 bash[75299]: cluster 2023-10-01T10:57:50.101865+0000 mgr.y (mgr.25072) 1211 : cluster 0 pgmap v1223: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-01T10:57:51.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:51 smithi078 bash[75299]: audit 2023-10-01T10:57:51.091371+0000 mon.a (mon.0) 1509 : audit 0 from='client.? 172.21.15.78:0/1782629181' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:57:51.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:51 smithi078 bash[75299]: audit 2023-10-01T10:57:51.524480+0000 mon.c (mon.2) 193 : audit 1 from='client.? 172.21.15.78:0/1169249095' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:57:51.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:51 smithi078 bash[75299]: audit 2023-10-01T10:57:51.525080+0000 mon.a (mon.0) 1510 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:57:51.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:51 smithi078 bash[78274]: cluster 2023-10-01T10:57:50.101865+0000 mgr.y (mgr.25072) 1211 : cluster 0 pgmap v1223: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-01T10:57:51.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:51 smithi078 bash[78274]: audit 2023-10-01T10:57:51.091371+0000 mon.a (mon.0) 1509 : audit 0 from='client.? 172.21.15.78:0/1782629181' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:57:51.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:51 smithi078 bash[78274]: audit 2023-10-01T10:57:51.524480+0000 mon.c (mon.2) 193 : audit 1 from='client.? 172.21.15.78:0/1169249095' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:57:51.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:51 smithi078 bash[78274]: audit 2023-10-01T10:57:51.525080+0000 mon.a (mon.0) 1510 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:57:51.832 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:51.859 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:51.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:51.867 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:57:51.968 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:51.978 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:51 smithi173 bash[58816]: cluster 2023-10-01T10:57:50.101865+0000 mgr.y (mgr.25072) 1211 : cluster 0 pgmap v1223: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-10-01T10:57:51.978 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:51 smithi173 bash[58816]: audit 2023-10-01T10:57:51.091371+0000 mon.a (mon.0) 1509 : audit 0 from='client.? 172.21.15.78:0/1782629181' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:57:51.978 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:51 smithi173 bash[58816]: audit 2023-10-01T10:57:51.524480+0000 mon.c (mon.2) 193 : audit 1 from='client.? 172.21.15.78:0/1169249095' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:57:51.978 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:51 smithi173 bash[58816]: audit 2023-10-01T10:57:51.525080+0000 mon.a (mon.0) 1510 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:57:51.999 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:52.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:52.001 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:57:52.047 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 66 16 84086 84070 79.6022 52.6875 0.0161585 0.0125573 2023-10-01T10:57:52.102 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:52.134 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:52.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:52.136 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:57:52.237 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:52.264 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:52.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:52.267 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:57:52.367 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:52.395 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:52.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:52.399 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:57:52.500 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:52.527 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:52.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:52.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:57:52.630 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:52.655 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:52.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:52.659 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:57:52.760 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:52.771 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:52 smithi173 bash[58816]: cluster 2023-10-01T10:57:51.702910+0000 mon.a (mon.0) 1511 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T10:57:52.771 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:52 smithi173 bash[58816]: cluster 2023-10-01T10:57:51.702950+0000 mon.a (mon.0) 1512 : cluster 1 Cluster is now healthy 2023-10-01T10:57:52.772 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:52 smithi173 bash[58816]: cluster 2023-10-01T10:57:51.718976+0000 osd.5 (osd.5) 228 : cluster 3 failed to encode map e246 with expected crc 2023-10-01T10:57:52.772 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:52 smithi173 bash[58816]: audit 2023-10-01T10:57:51.719957+0000 mon.a (mon.0) 1513 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T10:57:52.772 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:52 smithi173 bash[58816]: cluster 2023-10-01T10:57:51.720119+0000 mon.a (mon.0) 1514 : cluster 0 osdmap e246: 8 total, 8 up, 8 in 2023-10-01T10:57:52.772 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:52 smithi173 bash[58816]: cluster 2023-10-01T10:57:51.720356+0000 osd.6 (osd.6) 189 : cluster 3 failed to encode map e246 with expected crc 2023-10-01T10:57:52.772 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:52 smithi173 bash[58816]: cluster 2023-10-01T10:57:51.725971+0000 osd.7 (osd.7) 229 : cluster 3 failed to encode map e246 with expected crc 2023-10-01T10:57:52.772 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:52 smithi173 bash[58816]: cluster 2023-10-01T10:57:51.727048+0000 osd.7 (osd.7) 230 : cluster 3 failed to encode map e246 with expected crc 2023-10-01T10:57:52.792 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:52.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:52.796 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:52.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:52 smithi078 bash[78274]: cluster 2023-10-01T10:57:51.702910+0000 mon.a (mon.0) 1511 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T10:57:52.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:52 smithi078 bash[78274]: cluster 2023-10-01T10:57:51.702950+0000 mon.a (mon.0) 1512 : cluster 1 Cluster is now healthy 2023-10-01T10:57:52.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:52 smithi078 bash[78274]: cluster 2023-10-01T10:57:51.718976+0000 osd.5 (osd.5) 228 : cluster 3 failed to encode map e246 with expected crc 2023-10-01T10:57:52.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:52 smithi078 bash[78274]: audit 2023-10-01T10:57:51.719957+0000 mon.a (mon.0) 1513 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T10:57:52.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:52 smithi078 bash[78274]: cluster 2023-10-01T10:57:51.720119+0000 mon.a (mon.0) 1514 : cluster 0 osdmap e246: 8 total, 8 up, 8 in 2023-10-01T10:57:52.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:52 smithi078 bash[78274]: cluster 2023-10-01T10:57:51.720356+0000 osd.6 (osd.6) 189 : cluster 3 failed to encode map e246 with expected crc 2023-10-01T10:57:52.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:52 smithi078 bash[78274]: cluster 2023-10-01T10:57:51.725971+0000 osd.7 (osd.7) 229 : cluster 3 failed to encode map e246 with expected crc 2023-10-01T10:57:52.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:52 smithi078 bash[78274]: cluster 2023-10-01T10:57:51.727048+0000 osd.7 (osd.7) 230 : cluster 3 failed to encode map e246 with expected crc 2023-10-01T10:57:52.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:52 smithi078 bash[75299]: cluster 2023-10-01T10:57:51.702910+0000 mon.a (mon.0) 1511 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T10:57:52.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:52 smithi078 bash[75299]: cluster 2023-10-01T10:57:51.702950+0000 mon.a (mon.0) 1512 : cluster 1 Cluster is now healthy 2023-10-01T10:57:52.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:52 smithi078 bash[75299]: cluster 2023-10-01T10:57:51.718976+0000 osd.5 (osd.5) 228 : cluster 3 failed to encode map e246 with expected crc 2023-10-01T10:57:52.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:52 smithi078 bash[75299]: audit 2023-10-01T10:57:51.719957+0000 mon.a (mon.0) 1513 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T10:57:52.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:52 smithi078 bash[75299]: cluster 2023-10-01T10:57:51.720119+0000 mon.a (mon.0) 1514 : cluster 0 osdmap e246: 8 total, 8 up, 8 in 2023-10-01T10:57:52.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:52 smithi078 bash[75299]: cluster 2023-10-01T10:57:51.720356+0000 osd.6 (osd.6) 189 : cluster 3 failed to encode map e246 with expected crc 2023-10-01T10:57:52.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:52 smithi078 bash[75299]: cluster 2023-10-01T10:57:51.725971+0000 osd.7 (osd.7) 229 : cluster 3 failed to encode map e246 with expected crc 2023-10-01T10:57:52.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:52 smithi078 bash[75299]: cluster 2023-10-01T10:57:51.727048+0000 osd.7 (osd.7) 230 : cluster 3 failed to encode map e246 with expected crc 2023-10-01T10:57:52.896 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:52.922 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:52.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:52.925 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:53.026 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:53.044 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:53.047 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 67 16 84998 84982 79.2648 57 0.0827525 0.0126026 2023-10-01T10:57:53.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:53.048 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:53.149 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:53.176 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:53.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:53.180 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:57:53.280 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:53.308 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:53.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:53.310 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:57:53.411 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:53.439 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:53.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:53.442 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:57:53.543 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:53.573 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:53.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:53.576 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:57:53.677 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:53.707 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:53.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:53.709 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:53.810 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:53.821 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:53 smithi078 bash[78274]: cluster 2023-10-01T10:57:52.102314+0000 mgr.y (mgr.25072) 1212 : cluster 0 pgmap v1225: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 936 op/s 2023-10-01T10:57:53.822 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:53 smithi078 bash[75299]: cluster 2023-10-01T10:57:52.102314+0000 mgr.y (mgr.25072) 1212 : cluster 0 pgmap v1225: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 936 op/s 2023-10-01T10:57:53.837 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:53.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:53.840 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:57:53.941 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:53.967 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:53.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:53.970 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:57:54.048 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 68 16 85840 85824 78.8722 52.625 0.0381197 0.0126723 2023-10-01T10:57:54.071 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:54.104 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:54.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:54.108 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:54.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:53 smithi173 bash[58816]: cluster 2023-10-01T10:57:52.102314+0000 mgr.y (mgr.25072) 1212 : cluster 0 pgmap v1225: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 936 op/s 2023-10-01T10:57:54.208 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:54.243 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:54.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:54.245 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:54.346 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:54.384 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:54.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:54.392 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:54.492 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:54.525 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:54.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:54.531 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:57:54.632 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:54.663 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:54.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:54.666 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:57:54.767 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:54.799 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:54.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:54.801 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:57:54.902 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:54.932 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:54.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:54.936 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:57:55.037 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:55.049 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 69 16 86694 86678 78.5013 53.375 0.019902 0.0127322 2023-10-01T10:57:55.071 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:55.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:55.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:57:55.124 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T10:57:55.179 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:55.214 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:55.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:55.220 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:55.321 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:55.344 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:55.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:55.346 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:57:55.447 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:55.451 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 7, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T10:57:55.452 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T10:57:55.452 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_7 2023-10-01T10:57:55.452 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T10:57:55.476 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:55.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:55.484 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:55.585 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:55.610 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:55.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:55.613 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:57:55.714 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:55.742 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:55.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:55.745 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:57:55.845 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:55.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:55 smithi173 bash[58816]: cluster 2023-10-01T10:57:54.102984+0000 mgr.y (mgr.25072) 1213 : cluster 0 pgmap v1226: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 57 MiB/s wr, 918 op/s 2023-10-01T10:57:55.875 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:55.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:55.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:57:55.983 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:56.010 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:56.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:56.014 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:57:56.049 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 70 16 87530 87514 78.1263 52.25 0.0252843 0.0127938 2023-10-01T10:57:56.078 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T10:57:56.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:55 smithi078 bash[78274]: cluster 2023-10-01T10:57:54.102984+0000 mgr.y (mgr.25072) 1213 : cluster 0 pgmap v1226: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 57 MiB/s wr, 918 op/s 2023-10-01T10:57:56.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:55 smithi078 bash[75299]: cluster 2023-10-01T10:57:54.102984+0000 mgr.y (mgr.25072) 1213 : cluster 0 pgmap v1226: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 57 MiB/s wr, 918 op/s 2023-10-01T10:57:56.090 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T10:57:56.115 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:56.170 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:56.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:56.175 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:57:56.276 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:56.329 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:56.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:56.332 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:57:56.433 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:56.457 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:56.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:56.463 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:56.564 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:56.624 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:56.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:56.629 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:57:56.730 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:56.758 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:56.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:56.761 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:57:56.862 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:56.901 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:56.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:56.903 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:57:57.004 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:57.031 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:57.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:57.034 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:57:57.049 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 71 16 88339 88323 77.738 50.5625 0.0259227 0.0128575 2023-10-01T10:57:57.135 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:57.165 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:57.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:57.168 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:57:57.268 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:57.292 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:57:57.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:57.297 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:57:57.398 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:57.430 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:57.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:57.435 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:57.535 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:57.566 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:57.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:57.572 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:57.672 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:57.708 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:57:57.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:57.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:57:57.815 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:57.841 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:57.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:57.846 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:57:57.947 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:57.985 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:57 smithi173 bash[58816]: cluster 2023-10-01T10:57:56.104520+0000 mgr.y (mgr.25072) 1214 : cluster 0 pgmap v1227: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 902 op/s 2023-10-01T10:57:57.989 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:57.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:57.996 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:57:58.050 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 72 15 89180 89165 77.3892 52.625 0.0172188 0.0129155 2023-10-01T10:57:58.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:57 smithi078 bash[75299]: cluster 2023-10-01T10:57:56.104520+0000 mgr.y (mgr.25072) 1214 : cluster 0 pgmap v1227: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 902 op/s 2023-10-01T10:57:58.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:57 smithi078 bash[78274]: cluster 2023-10-01T10:57:56.104520+0000 mgr.y (mgr.25072) 1214 : cluster 0 pgmap v1227: 89 pgs: 89 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 902 op/s 2023-10-01T10:57:58.097 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:58.128 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:58.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:58.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:57:58.235 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:58.259 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:58.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:58.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:57:58.364 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:58.387 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:58.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:58.394 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:57:58.495 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:58.517 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:58.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:58.520 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:57:58.621 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:58.672 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:58.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:58.678 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:58.779 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:58.804 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:57:58.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:58.807 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:57:58.908 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:58.932 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:58.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:58.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:59.036 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:59.050 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 73 16 90009 89993 77.0379 51.75 0.0182576 0.0129749 2023-10-01T10:57:59.069 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:57:59.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:59.071 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:57:59.172 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:59.203 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:59.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:59.207 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:57:59.308 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:59.333 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:57:59.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:59.335 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:57:59.436 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:59.461 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:57:59.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:59.464 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:57:59.565 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:59.590 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:59.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:59.593 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:57:59.694 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:59.722 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:57:59.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:59.728 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:57:59.735 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:57:59 smithi078 bash[75299]: cluster 2023-10-01T10:57:58.107762+0000 mgr.y (mgr.25072) 1215 : cluster 0 pgmap v1228: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 55 MiB/s wr, 883 op/s 2023-10-01T10:57:59.829 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:59.868 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:57:59.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:57:59 smithi173 bash[58816]: cluster 2023-10-01T10:57:58.107762+0000 mgr.y (mgr.25072) 1215 : cluster 0 pgmap v1228: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 55 MiB/s wr, 883 op/s 2023-10-01T10:57:59.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:57:59.872 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:57:59.972 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:00.007 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:00.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:00.013 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:58:00.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:57:59 smithi078 bash[78274]: cluster 2023-10-01T10:57:58.107762+0000 mgr.y (mgr.25072) 1215 : cluster 0 pgmap v1228: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 55 MiB/s wr, 883 op/s 2023-10-01T10:58:00.050 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 74 16 90852 90836 76.7088 52.6875 0.00561936 0.0130305 2023-10-01T10:58:00.114 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:00.144 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:00.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:00.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:58:00.247 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:00.272 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:58:00 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:58:00] "GET /metrics HTTP/1.1" 200 36380 "" "Prometheus/2.33.4" 2023-10-01T10:58:00.292 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:00.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:00.297 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:58:00.347 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:58:00.348 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:58:00.348 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:58:00.391 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-10-01T10:58:00.391 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [], 2023-10-01T10:58:00.392 INFO:teuthology.orchestra.run.smithi173.stdout: "num_ops": 0 2023-10-01T10:58:00.392 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-10-01T10:58:00.398 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:00.422 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:00.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:00.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:58:00.529 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:00.556 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:00.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:00.558 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:58:00.658 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:00.692 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:00.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:00.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:58:00.799 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:00.837 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:00.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:00.841 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:58:00.942 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:00.995 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:00.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:00.997 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:58:01.050 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 75 16 91656 91640 76.3559 50.25 0.0131989 0.0130905 2023-10-01T10:58:01.098 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:01.123 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:01.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:01.126 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:58:01.227 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:01.251 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:01.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:01.258 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:58:01.325 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-01T10:58:01.359 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:01.385 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:01.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:01.391 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:58:01.492 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:01.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:01.519 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:01.519 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:58:01.620 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:01.645 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:01.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:01.648 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:58:01.748 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:01.782 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:01.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:01.785 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:58:01.886 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:01.896 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:01 smithi173 bash[58816]: cluster 2023-10-01T10:58:00.108522+0000 mgr.y (mgr.25072) 1216 : cluster 0 pgmap v1229: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 53 MiB/s wr, 852 op/s 2023-10-01T10:58:01.918 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:01.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:01.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:58:02.024 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:02.049 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:02.050 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 76 16 92516 92500 76.0584 53.75 0.0181394 0.0131417 2023-10-01T10:58:02.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:02.052 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:58:02.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:01 smithi078 bash[78274]: cluster 2023-10-01T10:58:00.108522+0000 mgr.y (mgr.25072) 1216 : cluster 0 pgmap v1229: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 53 MiB/s wr, 852 op/s 2023-10-01T10:58:02.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:01 smithi078 bash[75299]: cluster 2023-10-01T10:58:00.108522+0000 mgr.y (mgr.25072) 1216 : cluster 0 pgmap v1229: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 53 MiB/s wr, 852 op/s 2023-10-01T10:58:02.153 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:02.176 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:02.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:02.179 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:58:02.280 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:02.305 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:02.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:02.308 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:58:02.409 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:02.437 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:02.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:02.441 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:58:02.542 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:02.575 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:02.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:02.579 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:58:02.679 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:02.704 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:02.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:02.707 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:58:02.807 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:02.834 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:02.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:02.836 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:58:02.937 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:02.968 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:02.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:02.970 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:58:03.050 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 77 15 93271 93256 75.6843 47.25 0.0067954 0.013207 2023-10-01T10:58:03.072 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:03.099 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:03.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:03.103 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:58:03.204 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:03.227 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:03.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:03.232 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:58:03.333 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:03.361 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:03.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:03.364 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:58:03.464 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:03.495 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:03.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:03.499 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:58:03.600 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:03.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:03 smithi173 bash[58816]: cluster 2023-10-01T10:58:02.108883+0000 mgr.y (mgr.25072) 1217 : cluster 0 pgmap v1230: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 51 MiB/s wr, 821 op/s 2023-10-01T10:58:03.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:03 smithi173 bash[58816]: audit 2023-10-01T10:58:02.452598+0000 mon.a (mon.0) 1515 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:58:03.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:03 smithi173 bash[58816]: audit 2023-10-01T10:58:02.454404+0000 mon.a (mon.0) 1516 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:58:03.638 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:03.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:03.642 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:58:03.743 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:03.770 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:03.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:03.772 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:58:03.873 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:03.905 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:03.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:03.908 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:58:03.922 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:03 smithi078 bash[75299]: cluster 2023-10-01T10:58:02.108883+0000 mgr.y (mgr.25072) 1217 : cluster 0 pgmap v1230: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 51 MiB/s wr, 821 op/s 2023-10-01T10:58:03.922 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:03 smithi078 bash[75299]: audit 2023-10-01T10:58:02.452598+0000 mon.a (mon.0) 1515 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:58:03.922 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:03 smithi078 bash[75299]: audit 2023-10-01T10:58:02.454404+0000 mon.a (mon.0) 1516 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:58:03.922 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:03 smithi078 bash[78274]: cluster 2023-10-01T10:58:02.108883+0000 mgr.y (mgr.25072) 1217 : cluster 0 pgmap v1230: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 51 MiB/s wr, 821 op/s 2023-10-01T10:58:03.923 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:03 smithi078 bash[78274]: audit 2023-10-01T10:58:02.452598+0000 mon.a (mon.0) 1515 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:58:03.923 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:03 smithi078 bash[78274]: audit 2023-10-01T10:58:02.454404+0000 mon.a (mon.0) 1516 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:58:04.009 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:04.038 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:04.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:04.041 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:58:04.050 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 78 16 94006 93990 75.3021 45.875 0.0214721 0.0132744 2023-10-01T10:58:04.143 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:04.169 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:04.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:04.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:58:04.275 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:04.321 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:04.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:04.326 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:58:04.427 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:04.463 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:04.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:04.466 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:58:04.567 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:04.594 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:04.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:04.602 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:58:04.703 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:04.727 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:04.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:04.732 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:58:04.833 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:04.886 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:04.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:04.889 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:58:04.990 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:05.014 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:05.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:05.016 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:58:05.050 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 79 16 94849 94833 75.0157 52.6875 0.016039 0.013325 2023-10-01T10:58:05.117 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:05.142 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:05.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:05.145 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:58:05.246 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:05.274 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:05.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:05.277 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:58:05.378 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:05.408 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:05.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:05.411 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:58:05.512 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:05.522 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:05 smithi173 bash[58816]: cluster 2023-10-01T10:58:04.109486+0000 mgr.y (mgr.25072) 1218 : cluster 0 pgmap v1231: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 58 MiB/s wr, 921 op/s 2023-10-01T10:58:05.538 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:05.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:05.542 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:58:05.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:05 smithi078 bash[78274]: cluster 2023-10-01T10:58:04.109486+0000 mgr.y (mgr.25072) 1218 : cluster 0 pgmap v1231: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 58 MiB/s wr, 921 op/s 2023-10-01T10:58:05.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:05 smithi078 bash[75299]: cluster 2023-10-01T10:58:04.109486+0000 mgr.y (mgr.25072) 1218 : cluster 0 pgmap v1231: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 58 MiB/s wr, 921 op/s 2023-10-01T10:58:05.643 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:05.674 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:05.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:05.682 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:58:05.782 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:05.812 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:05.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:05.815 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:58:05.916 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:05.943 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:05.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:05.946 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:58:06.047 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:06.050 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:2023-10-01T10:58:06.050306+0000 min lat: 0.00107177 max lat: 0.325279 avg lat: 0.0133781 2023-10-01T10:58:06.050 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T10:58:06.050 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 80 16 95668 95652 74.7178 51.1875 0.014532 0.0133781 2023-10-01T10:58:06.069 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:06.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:06.071 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:58:06.131 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T10:58:06.172 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:06.200 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:06.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:06.207 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:58:06.307 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:06.330 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:06.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:06.332 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:58:06.433 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:06.459 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:06.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:06.465 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:58:06.566 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:06.597 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:06.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:06.602 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:58:06.702 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:06.728 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:06.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:06.730 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:58:06.831 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:06.861 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:06.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:06.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:58:06.967 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:06.990 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:06.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:06.994 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:58:07.051 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 81 16 96500 96484 74.4373 52 0.0144668 0.0134283 2023-10-01T10:58:07.095 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:07.126 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:07.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:07.133 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:58:07.234 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:07.262 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:07.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:07.266 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:58:07.367 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:07.387 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:07.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:07.392 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:58:07.493 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:07.523 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:07.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:07.531 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:58:07.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:07 smithi173 bash[58816]: cluster 2023-10-01T10:58:06.110140+0000 mgr.y (mgr.25072) 1219 : cluster 0 pgmap v1232: 89 pgs: 89 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 51 MiB/s wr, 817 op/s 2023-10-01T10:58:07.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:07 smithi173 bash[58816]: audit 2023-10-01T10:58:07.366780+0000 mon.a (mon.0) 1517 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:58:07.632 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:07.660 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:07.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:07.663 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:58:07.764 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:07.790 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:07.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:07.793 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:58:07.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:07 smithi078 bash[75299]: cluster 2023-10-01T10:58:06.110140+0000 mgr.y (mgr.25072) 1219 : cluster 0 pgmap v1232: 89 pgs: 89 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 51 MiB/s wr, 817 op/s 2023-10-01T10:58:07.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:07 smithi078 bash[75299]: audit 2023-10-01T10:58:07.366780+0000 mon.a (mon.0) 1517 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:58:07.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:07 smithi078 bash[78274]: cluster 2023-10-01T10:58:06.110140+0000 mgr.y (mgr.25072) 1219 : cluster 0 pgmap v1232: 89 pgs: 89 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 51 MiB/s wr, 817 op/s 2023-10-01T10:58:07.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:07 smithi078 bash[78274]: audit 2023-10-01T10:58:07.366780+0000 mon.a (mon.0) 1517 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:58:07.893 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:07.944 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:07.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:07.947 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:58:08.047 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:08.050 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 82 16 97300 97284 74.1393 50 0.0199031 0.0134831 2023-10-01T10:58:08.075 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:08.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:08.082 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:58:08.183 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:08.216 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:08.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:08.219 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:58:08.319 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:08.349 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:08.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:08.353 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:58:08.454 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:08.485 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:08.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:08.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:58:08.592 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:08.616 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:08.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:08.618 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:58:08.719 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:08.763 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:08.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:08.765 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:58:08.866 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:08.898 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:08.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:08.906 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:58:09.007 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:09.033 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:09.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:09.036 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:58:09.051 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 83 16 98111 98095 73.8567 50.6875 0.0145119 0.0135337 2023-10-01T10:58:09.136 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:09.155 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:09.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:09.156 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:58:09.257 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:09.287 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:09.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:09.289 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:58:09.390 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:09.425 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:09.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:09.429 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:58:09.529 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:09.555 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:09.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:09.558 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:58:09.659 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:09.668 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:09 smithi078 bash[75299]: cluster 2023-10-01T10:58:08.111161+0000 mgr.y (mgr.25072) 1220 : cluster 0 pgmap v1233: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 48 MiB/s wr, 768 op/s 2023-10-01T10:58:09.668 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:09 smithi078 bash[78274]: cluster 2023-10-01T10:58:08.111161+0000 mgr.y (mgr.25072) 1220 : cluster 0 pgmap v1233: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 48 MiB/s wr, 768 op/s 2023-10-01T10:58:09.686 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:09.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:09.691 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:58:09.791 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:09.819 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:09.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:09.823 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:58:09.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:09 smithi173 bash[58816]: cluster 2023-10-01T10:58:08.111161+0000 mgr.y (mgr.25072) 1220 : cluster 0 pgmap v1233: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 48 MiB/s wr, 768 op/s 2023-10-01T10:58:09.924 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:09.961 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:09.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:09.965 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:58:10.051 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 84 16 98896 98880 73.5615 49.0625 0.0226796 0.0135889 2023-10-01T10:58:10.065 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:10.092 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:10.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:10.094 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:58:10.195 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:10.225 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:10.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:10.230 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:58:10.331 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:10.360 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:10.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:10.366 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:58:10.467 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:10.493 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:10.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:10.496 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:58:10.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:58:10 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:58:10] "GET /metrics HTTP/1.1" 200 36383 "" "Prometheus/2.33.4" 2023-10-01T10:58:10.597 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:10.621 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-10-01T10:58:10.621 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [], 2023-10-01T10:58:10.621 INFO:teuthology.orchestra.run.smithi173.stdout: "complaint_time": 30, 2023-10-01T10:58:10.622 INFO:teuthology.orchestra.run.smithi173.stdout: "num_blocked_ops": 0 2023-10-01T10:58:10.622 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-10-01T10:58:10.636 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:10.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:10.642 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:58:10.743 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:10.771 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:10.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:10.776 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:58:10.876 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:10.907 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:10.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:10.911 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:58:11.012 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:11.039 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:11.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:11.042 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:58:11.051 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 85 16 99737 99721 73.3144 52.5625 0.0228421 0.0136344 2023-10-01T10:58:11.142 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:11.168 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:11.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:11.174 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:58:11.275 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:11.304 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:11.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:11.308 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:58:11.408 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:11.433 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:11.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:11.439 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:58:11.540 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:11.569 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:11.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:11.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:58:11.582 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-01T10:58:11.674 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:11.683 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:11 smithi173 bash[58816]: cluster 2023-10-01T10:58:10.111860+0000 mgr.y (mgr.25072) 1221 : cluster 0 pgmap v1234: 89 pgs: 89 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 58 MiB/s wr, 935 op/s 2023-10-01T10:58:11.712 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:11.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:11.716 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:58:11.817 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:11.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:11 smithi078 bash[75299]: cluster 2023-10-01T10:58:10.111860+0000 mgr.y (mgr.25072) 1221 : cluster 0 pgmap v1234: 89 pgs: 89 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 58 MiB/s wr, 935 op/s 2023-10-01T10:58:11.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:11 smithi078 bash[78274]: cluster 2023-10-01T10:58:10.111860+0000 mgr.y (mgr.25072) 1221 : cluster 0 pgmap v1234: 89 pgs: 89 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 58 MiB/s wr, 935 op/s 2023-10-01T10:58:11.852 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:11.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:11.855 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:58:11.955 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:11.999 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:12.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:12.003 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:58:12.051 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 86 16 100566 100550 73.0643 51.8125 0.0197593 0.0136814 2023-10-01T10:58:12.105 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:12.125 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:12.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:12.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:58:12.228 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:12.251 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:12.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:12.254 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:58:12.354 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:12.378 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:12.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:12.380 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:58:12.481 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:12.509 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:12.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:12.512 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:58:12.613 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:12.641 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:12.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:12.645 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:58:12.746 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:12.771 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:12.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:12.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:58:12.876 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:12.907 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:12.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:12.911 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:58:13.012 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:13.036 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:13.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:13.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:58:13.051 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 87 16 101185 101169 72.6691 38.6875 0.0179329 0.0137551 2023-10-01T10:58:13.142 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:13.178 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:13.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:13.181 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:58:13.281 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:13.327 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:13.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:13.330 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:58:13.431 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:13.467 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:13.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:13.470 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:58:13.571 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:13.609 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:13.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:13.611 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:58:13.711 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:13.721 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:13 smithi173 bash[58816]: cluster 2023-10-01T10:58:12.112320+0000 mgr.y (mgr.25072) 1222 : cluster 0 pgmap v1235: 89 pgs: 89 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 42 MiB/s wr, 675 op/s 2023-10-01T10:58:13.744 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:13.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:13.752 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:58:13.853 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:13.883 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:13.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:13.886 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:58:13.906 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:13 smithi078 bash[75299]: cluster 2023-10-01T10:58:12.112320+0000 mgr.y (mgr.25072) 1222 : cluster 0 pgmap v1235: 89 pgs: 89 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 42 MiB/s wr, 675 op/s 2023-10-01T10:58:13.907 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:13 smithi078 bash[78274]: cluster 2023-10-01T10:58:12.112320+0000 mgr.y (mgr.25072) 1222 : cluster 0 pgmap v1235: 89 pgs: 89 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 42 MiB/s wr, 675 op/s 2023-10-01T10:58:13.987 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:14.014 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:14.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:14.017 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:58:14.051 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 88 16 101770 101754 72.2588 36.5625 0.0228066 0.0138293 2023-10-01T10:58:14.064 INFO:teuthology.orchestra.run.smithi078.stdout: 2023-10-01T10:58:14.080 INFO:teuthology.orchestra.run.smithi078.stderr:dumped all 2023-10-01T10:58:14.118 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:14.178 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:14.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:14.182 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:58:14.257 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T10:58:14.282 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:14.321 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T10:58:14.330 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:14.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:14.333 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:58:14.361 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T10:58:14.422 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T10:58:14.434 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:14.459 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:14.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:14.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:58:14.465 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T10:58:14.505 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T10:58:14.540 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T10:58:14.564 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:14.587 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T10:58:14.595 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:14.596 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T10:58:14.596 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T10:58:14.596 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T10:58:14.596 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T10:58:14.596 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T10:58:14.597 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T10:58:14.597 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T10:58:14.597 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T10:58:14.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:14.598 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:58:14.698 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:14.731 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:14.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:14.733 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:58:14.834 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:14.865 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:14.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:14.869 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:14 smithi078 bash[75299]: audit 2023-10-01T10:58:13.734890+0000 mon.a (mon.0) 1518 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:58:14.869 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:14 smithi078 bash[75299]: audit 2023-10-01T10:58:13.747611+0000 mon.a (mon.0) 1519 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:58:14.869 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:14 smithi078 bash[75299]: audit 2023-10-01T10:58:14.207141+0000 mon.a (mon.0) 1520 : audit 0 from='client.? 172.21.15.78:0/154768573' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:58:14.869 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:14 smithi078 bash[75299]: audit 2023-10-01T10:58:14.504634+0000 mon.a (mon.0) 1521 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:58:14.869 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:14 smithi078 bash[75299]: audit 2023-10-01T10:58:14.519369+0000 mon.a (mon.0) 1522 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:58:14.869 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:14 smithi078 bash[75299]: audit 2023-10-01T10:58:14.534572+0000 mon.b (mon.1) 50 : audit 1 from='client.? 172.21.15.78:0/1859160509' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:58:14.870 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:14 smithi078 bash[75299]: audit 2023-10-01T10:58:14.534615+0000 mon.a (mon.0) 1523 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:58:14.871 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:14 smithi078 bash[78274]: audit 2023-10-01T10:58:13.734890+0000 mon.a (mon.0) 1518 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:58:14.871 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:14 smithi078 bash[78274]: audit 2023-10-01T10:58:13.747611+0000 mon.a (mon.0) 1519 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:58:14.871 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:14 smithi078 bash[78274]: audit 2023-10-01T10:58:14.207141+0000 mon.a (mon.0) 1520 : audit 0 from='client.? 172.21.15.78:0/154768573' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:58:14.871 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:14 smithi078 bash[78274]: audit 2023-10-01T10:58:14.504634+0000 mon.a (mon.0) 1521 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:58:14.871 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:14 smithi078 bash[78274]: audit 2023-10-01T10:58:14.519369+0000 mon.a (mon.0) 1522 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:58:14.871 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:14 smithi078 bash[78274]: audit 2023-10-01T10:58:14.534572+0000 mon.b (mon.1) 50 : audit 1 from='client.? 172.21.15.78:0/1859160509' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:58:14.871 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:14 smithi078 bash[78274]: audit 2023-10-01T10:58:14.534615+0000 mon.a (mon.0) 1523 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:58:14.872 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:58:14.973 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:15.004 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:15.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:15.009 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:58:15.051 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 89 16 102456 102440 71.9285 42.875 0.0165831 0.0138974 2023-10-01T10:58:15.110 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:15.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:14 smithi173 bash[58816]: audit 2023-10-01T10:58:13.734890+0000 mon.a (mon.0) 1518 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:58:15.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:14 smithi173 bash[58816]: audit 2023-10-01T10:58:13.747611+0000 mon.a (mon.0) 1519 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:58:15.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:14 smithi173 bash[58816]: audit 2023-10-01T10:58:14.207141+0000 mon.a (mon.0) 1520 : audit 0 from='client.? 172.21.15.78:0/154768573' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:58:15.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:14 smithi173 bash[58816]: audit 2023-10-01T10:58:14.504634+0000 mon.a (mon.0) 1521 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:58:15.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:14 smithi173 bash[58816]: audit 2023-10-01T10:58:14.519369+0000 mon.a (mon.0) 1522 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:58:15.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:14 smithi173 bash[58816]: audit 2023-10-01T10:58:14.534572+0000 mon.b (mon.1) 50 : audit 1 from='client.? 172.21.15.78:0/1859160509' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:58:15.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:14 smithi173 bash[58816]: audit 2023-10-01T10:58:14.534615+0000 mon.a (mon.0) 1523 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:58:15.137 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:15.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:15.138 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:58:15.239 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:15.270 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:15.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:15.272 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:58:15.373 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:15.402 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:15.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:15.406 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:58:15.506 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:15.530 INFO:teuthology.orchestra.run.smithi078.stderr:noscrub is set 2023-10-01T10:58:15.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:15.536 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:15.537 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:58:15.637 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:15.668 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:15.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:15.672 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:58:15.773 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:15.805 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:15.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:15.811 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:58:15.911 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:15.939 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:15.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:15.942 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:58:16.042 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:16.051 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:15 smithi078 bash[78274]: audit 2023-10-01T10:58:14.050717+0000 mgr.y (mgr.25072) 1223 : audit 0 from='client.46280 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:58:16.051 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:15 smithi078 bash[78274]: cluster 2023-10-01T10:58:14.112873+0000 mgr.y (mgr.25072) 1224 : cluster 0 pgmap v1236: 89 pgs: 89 active+clean; 6.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 55 MiB/s wr, 878 op/s 2023-10-01T10:58:16.051 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:15 smithi078 bash[78274]: audit 2023-10-01T10:58:15.093598+0000 mon.a (mon.0) 1524 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:58:16.051 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:15 smithi078 bash[78274]: audit 2023-10-01T10:58:15.094252+0000 mon.a (mon.0) 1525 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:58:16.051 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:15 smithi078 bash[78274]: audit 2023-10-01T10:58:15.098501+0000 mon.a (mon.0) 1526 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:58:16.051 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:15 smithi078 bash[78274]: cluster 2023-10-01T10:58:15.510747+0000 mon.a (mon.0) 1527 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:58:16.052 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:15 smithi078 bash[78274]: audit 2023-10-01T10:58:15.516659+0000 mon.a (mon.0) 1528 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T10:58:16.052 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:15 smithi078 bash[78274]: cluster 2023-10-01T10:58:15.516726+0000 mon.a (mon.0) 1529 : cluster 0 osdmap e247: 8 total, 8 up, 8 in 2023-10-01T10:58:16.052 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:15 smithi078 bash[75299]: audit 2023-10-01T10:58:14.050717+0000 mgr.y (mgr.25072) 1223 : audit 0 from='client.46280 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:58:16.052 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:15 smithi078 bash[75299]: cluster 2023-10-01T10:58:14.112873+0000 mgr.y (mgr.25072) 1224 : cluster 0 pgmap v1236: 89 pgs: 89 active+clean; 6.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 55 MiB/s wr, 878 op/s 2023-10-01T10:58:16.052 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:15 smithi078 bash[75299]: audit 2023-10-01T10:58:15.093598+0000 mon.a (mon.0) 1524 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:58:16.052 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:15 smithi078 bash[75299]: audit 2023-10-01T10:58:15.094252+0000 mon.a (mon.0) 1525 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:58:16.052 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:15 smithi078 bash[75299]: audit 2023-10-01T10:58:15.098501+0000 mon.a (mon.0) 1526 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:58:16.052 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:15 smithi078 bash[75299]: cluster 2023-10-01T10:58:15.510747+0000 mon.a (mon.0) 1527 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:58:16.052 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:15 smithi078 bash[75299]: audit 2023-10-01T10:58:15.516659+0000 mon.a (mon.0) 1528 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T10:58:16.053 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:15 smithi078 bash[75299]: cluster 2023-10-01T10:58:15.516726+0000 mon.a (mon.0) 1529 : cluster 0 osdmap e247: 8 total, 8 up, 8 in 2023-10-01T10:58:16.053 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 90 5 103128 103123 71.6036 42.6875 0.0326697 0.0139604 2023-10-01T10:58:16.062 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:16.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:16.064 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:58:16.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:15 smithi173 bash[58816]: audit 2023-10-01T10:58:14.050717+0000 mgr.y (mgr.25072) 1223 : audit 0 from='client.46280 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:58:16.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:15 smithi173 bash[58816]: cluster 2023-10-01T10:58:14.112873+0000 mgr.y (mgr.25072) 1224 : cluster 0 pgmap v1236: 89 pgs: 89 active+clean; 6.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 55 MiB/s wr, 878 op/s 2023-10-01T10:58:16.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:15 smithi173 bash[58816]: audit 2023-10-01T10:58:15.093598+0000 mon.a (mon.0) 1524 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:58:16.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:15 smithi173 bash[58816]: audit 2023-10-01T10:58:15.094252+0000 mon.a (mon.0) 1525 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:58:16.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:15 smithi173 bash[58816]: audit 2023-10-01T10:58:15.098501+0000 mon.a (mon.0) 1526 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:58:16.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:15 smithi173 bash[58816]: cluster 2023-10-01T10:58:15.510747+0000 mon.a (mon.0) 1527 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:58:16.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:15 smithi173 bash[58816]: audit 2023-10-01T10:58:15.516659+0000 mon.a (mon.0) 1528 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T10:58:16.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:15 smithi173 bash[58816]: cluster 2023-10-01T10:58:15.516726+0000 mon.a (mon.0) 1529 : cluster 0 osdmap e247: 8 total, 8 up, 8 in 2023-10-01T10:58:16.141 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T10:58:16.165 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:16.182 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:16.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:16.184 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:58:16.285 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:16.305 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:16.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:16.307 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:58:16.410 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:16.434 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:16.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:16.436 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:58:16.537 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:16.560 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:16.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:16.562 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:58:16.663 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:16.692 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:16.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:16.694 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:58:16.795 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:16.826 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:16.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:16.829 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:58:16.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:16 smithi078 bash[75299]: cluster 2023-10-01T10:58:15.517143+0000 osd.7 (osd.7) 231 : cluster 3 failed to encode map e247 with expected crc 2023-10-01T10:58:16.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:16 smithi078 bash[75299]: cluster 2023-10-01T10:58:15.519014+0000 osd.7 (osd.7) 232 : cluster 3 failed to encode map e247 with expected crc 2023-10-01T10:58:16.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:16 smithi078 bash[75299]: cluster 2023-10-01T10:58:15.521659+0000 osd.6 (osd.6) 190 : cluster 3 failed to encode map e247 with expected crc 2023-10-01T10:58:16.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:16 smithi078 bash[75299]: cluster 2023-10-01T10:58:15.523212+0000 osd.6 (osd.6) 191 : cluster 3 failed to encode map e247 with expected crc 2023-10-01T10:58:16.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:16 smithi078 bash[75299]: cluster 2023-10-01T10:58:15.548845+0000 osd.5 (osd.5) 229 : cluster 3 failed to encode map e247 with expected crc 2023-10-01T10:58:16.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:16 smithi078 bash[78274]: cluster 2023-10-01T10:58:15.517143+0000 osd.7 (osd.7) 231 : cluster 3 failed to encode map e247 with expected crc 2023-10-01T10:58:16.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:16 smithi078 bash[78274]: cluster 2023-10-01T10:58:15.519014+0000 osd.7 (osd.7) 232 : cluster 3 failed to encode map e247 with expected crc 2023-10-01T10:58:16.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:16 smithi078 bash[78274]: cluster 2023-10-01T10:58:15.521659+0000 osd.6 (osd.6) 190 : cluster 3 failed to encode map e247 with expected crc 2023-10-01T10:58:16.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:16 smithi078 bash[78274]: cluster 2023-10-01T10:58:15.523212+0000 osd.6 (osd.6) 191 : cluster 3 failed to encode map e247 with expected crc 2023-10-01T10:58:16.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:16 smithi078 bash[78274]: cluster 2023-10-01T10:58:15.548845+0000 osd.5 (osd.5) 229 : cluster 3 failed to encode map e247 with expected crc 2023-10-01T10:58:16.930 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:16.955 INFO:teuthology.orchestra.run.smithi078.stdout:{"pg_ready":true,"pg_map":{"version":1235,"stamp":"2023-10-01T10:58:12.112059+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6405429152,"num_objects":97754,"num_object_clones":0,"num_object_copies":293262,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":97754,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":97991,"num_write_kb":6257166,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":1190344,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97925,"ondisk_log_size":97925,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":24362976,"kb_used_data":18733280,"kb_used_omap":30,"kb_used_meta":5629601,"kb_avail":725598240,"statfs":{"total":767960285184,"available":743012597760,"internally_reserved":0,"allocated":19182878720,"data_stored":19176513886,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31502,"internal_metadata":5764711666},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,3,9,6,9,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":531562496,"num_objects":8111,"num_object_clones":0,"num_object_copies":24333,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8111,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8111,"num_write_kb":519104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004100"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":239,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:18.634006+0000","last_change":"2023-10-01T10:38:16.032258+0000","last_active":"2023-10-01T10:57:18.634006+0000","last_peered":"2023-10-01T10:57:18.634006+0000","last_clean":"2023-10-01T10:57:18.634006+0000","last_became_active":"2023-10-01T10:38:16.031633+0000","last_became_peered":"2023-10-01T10:38:16.031633+0000","last_unstale":"2023-10-01T10:57:18.634006+0000","last_undegraded":"2023-10-01T10:57:18.634006+0000","last_fullsized":"2023-10-01T10:57:18.634006+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:17:02.446430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":228,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:42.491577+0000","last_change":"2023-10-01T10:38:16.025881+0000","last_active":"2023-10-01T10:57:42.491577+0000","last_peered":"2023-10-01T10:57:42.491577+0000","last_clean":"2023-10-01T10:57:42.491577+0000","last_became_active":"2023-10-01T10:38:16.025518+0000","last_became_peered":"2023-10-01T10:38:16.025518+0000","last_unstale":"2023-10-01T10:57:42.491577+0000","last_undegraded":"2023-10-01T10:57:42.491577+0000","last_fullsized":"2023-10-01T10:57:42.491577+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:48:37.826604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":148,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:18.645202+0000","last_change":"2023-10-01T10:52:18.956544+0000","last_active":"2023-10-01T10:57:18.645202+0000","last_peered":"2023-10-01T10:57:18.645202+0000","last_clean":"2023-10-01T10:57:18.645202+0000","last_became_active":"2023-10-01T10:52:18.956217+0000","last_became_peered":"2023-10-01T10:52:18.956217+0000","last_unstale":"2023-10-01T10:57:18.645202+0000","last_undegraded":"2023-10-01T10:57:18.645202+0000","last_fullsized":"2023-10-01T10:57:18.645202+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:34:01.952235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":211,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:18.655295+0000","last_change":"2023-10-01T10:37:42.772008+0000","last_active":"2023-10-01T10:57:18.655295+0000","last_peered":"2023-10-01T10:57:18.655295+0000","last_clean":"2023-10-01T10:57:18.655295+0000","last_became_active":"2023-10-01T10:37:42.771244+0000","last_became_peered":"2023-10-01T10:37:42.771244+0000","last_unstale":"2023-10-01T10:57:18.655295+0000","last_undegraded":"2023-10-01T10:57:18.655295+0000","last_fullsized":"2023-10-01T10:57:18.655295+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:38:55.180790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":229,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:18.646012+0000","last_change":"2023-10-01T10:37:42.766600+0000","last_active":"2023-10-01T10:57:18.646012+0000","last_peered":"2023-10-01T10:57:18.646012+0000","last_clean":"2023-10-01T10:57:18.646012+0000","last_became_active":"2023-10-01T10:37:42.766051+0000","last_became_peered":"2023-10-01T10:37:42.766051+0000","last_unstale":"2023-10-01T10:57:18.646012+0000","last_undegraded":"2023-10-01T10:57:18.646012+0000","last_fullsized":"2023-10-01T10:57:18.646012+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:51:17.707963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":21,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:18.634757+0000","last_change":"2023-10-01T10:56:14.626119+0000","last_active":"2023-10-01T10:57:18.634757+0000","last_peered":"2023-10-01T10:57:18.634757+0000","last_clean":"2023-10-01T10:57:18.634757+0000","last_became_active":"2023-10-01T10:56:14.625777+0000","last_became_peered":"2023-10-01T10:56:14.625777+0000","last_unstale":"2023-10-01T10:57:18.634757+0000","last_undegraded":"2023-10-01T10:57:18.634757+0000","last_fullsized":"2023-10-01T10:57:18.634757+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":208,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:19.352733+0000","last_change":"2023-10-01T10:37:42.766667+0000","last_active":"2023-10-01T10:57:19.352733+0000","last_peered":"2023-10-01T10:57:19.352733+0000","last_clean":"2023-10-01T10:57:19.352733+0000","last_became_active":"2023-10-01T10:37:42.766369+0000","last_became_peered":"2023-10-01T10:37:42.766369+0000","last_unstale":"2023-10-01T10:57:19.352733+0000","last_undegraded":"2023-10-01T10:57:19.352733+0000","last_fullsized":"2023-10-01T10:57:19.352733+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:02.708788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":231,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:18.646113+0000","last_change":"2023-10-01T10:56:14.622767+0000","last_active":"2023-10-01T10:57:18.646113+0000","last_peered":"2023-10-01T10:57:18.646113+0000","last_clean":"2023-10-01T10:57:18.646113+0000","last_became_active":"2023-10-01T10:56:14.622446+0000","last_became_peered":"2023-10-01T10:56:14.622446+0000","last_unstale":"2023-10-01T10:57:18.646113+0000","last_undegraded":"2023-10-01T10:57:18.646113+0000","last_fullsized":"2023-10-01T10:57:18.646113+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:32:09.354415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":213,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:18.657129+0000","last_change":"2023-10-01T10:32:31.579805+0000","last_active":"2023-10-01T10:57:18.657129+0000","last_peered":"2023-10-01T10:57:18.657129+0000","last_clean":"2023-10-01T10:57:18.657129+0000","last_became_active":"2023-10-01T10:32:31.579423+0000","last_became_peered":"2023-10-01T10:32:31.579423+0000","last_unstale":"2023-10-01T10:57:18.657129+0000","last_undegraded":"2023-10-01T10:57:18.657129+0000","last_fullsized":"2023-10-01T10:57:18.657129+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":215,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:18.645098+0000","last_change":"2023-10-01T10:56:14.619280+0000","last_active":"2023-10-01T10:57:18.645098+0000","last_peered":"2023-10-01T10:57:18.645098+0000","last_clean":"2023-10-01T10:57:18.645098+0000","last_became_active":"2023-10-01T10:56:14.618892+0000","last_became_peered":"2023-10-01T10:56:14.618892+0000","last_unstale":"2023-10-01T10:57:18.645098+0000","last_undegraded":"2023-10-01T10:57:18.645098+0000","last_fullsized":"2023-10-01T10:57:18.645098+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:14:29.635055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":87,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:37.488829+0000","last_change":"2023-10-01T10:52:43.661650+0000","last_active":"2023-10-01T10:57:37.488829+0000","last_peered":"2023-10-01T10:57:37.488829+0000","last_clean":"2023-10-01T10:57:37.488829+0000","last_became_active":"2023-10-01T10:52:43.661202+0000","last_became_peered":"2023-10-01T10:52:43.661202+0000","last_unstale":"2023-10-01T10:57:37.488829+0000","last_undegraded":"2023-10-01T10:57:37.488829+0000","last_fullsized":"2023-10-01T10:57:37.488829+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:28:35.479124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":21,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:18.634570+0000","last_change":"2023-10-01T10:56:14.620228+0000","last_active":"2023-10-01T10:57:18.634570+0000","last_peered":"2023-10-01T10:57:18.634570+0000","last_clean":"2023-10-01T10:57:18.634570+0000","last_became_active":"2023-10-01T10:56:14.619632+0000","last_became_peered":"2023-10-01T10:56:14.619632+0000","last_unstale":"2023-10-01T10:57:18.634570+0000","last_undegraded":"2023-10-01T10:57:18.634570+0000","last_fullsized":"2023-10-01T10:57:18.634570+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":272,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:18.634217+0000","last_change":"2023-10-01T10:56:14.634698+0000","last_active":"2023-10-01T10:57:18.634217+0000","last_peered":"2023-10-01T10:57:18.634217+0000","last_clean":"2023-10-01T10:57:18.634217+0000","last_became_active":"2023-10-01T10:56:14.625320+0000","last_became_peered":"2023-10-01T10:56:14.625320+0000","last_unstale":"2023-10-01T10:57:18.634217+0000","last_undegraded":"2023-10-01T10:57:18.634217+0000","last_fullsized":"2023-10-01T10:57:18.634217+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:56.480434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":226,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:18.634468+0000","last_change":"2023-10-01T10:56:14.622580+0000","last_active":"2023-10-01T10:57:18.634468+0000","last_peered":"2023-10-01T10:57:18.634468+0000","last_clean":"2023-10-01T10:57:18.634468+0000","last_became_active":"2023-10-01T10:56:14.622075+0000","last_became_peered":"2023-10-01T10:56:14.622075+0000","last_unstale":"2023-10-01T10:57:18.634468+0000","last_undegraded":"2023-10-01T10:57:18.634468+0000","last_fullsized":"2023-10-01T10:57:18.634468+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:52:32.352383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":229,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:18.633518+0000","last_change":"2023-10-01T10:38:16.026777+0000","last_active":"2023-10-01T10:57:18.633518+0000","last_peered":"2023-10-01T10:57:18.633518+0000","last_clean":"2023-10-01T10:57:18.633518+0000","last_became_active":"2023-10-01T10:38:16.026371+0000","last_became_peered":"2023-10-01T10:38:16.026371+0000","last_unstale":"2023-10-01T10:57:18.633518+0000","last_undegraded":"2023-10-01T10:57:18.633518+0000","last_fullsized":"2023-10-01T10:57:18.633518+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":236,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:18.645356+0000","last_change":"2023-10-01T10:37:42.772319+0000","last_active":"2023-10-01T10:57:18.645356+0000","last_peered":"2023-10-01T10:57:18.645356+0000","last_clean":"2023-10-01T10:57:18.645356+0000","last_became_active":"2023-10-01T10:37:42.771977+0000","last_became_peered":"2023-10-01T10:37:42.771977+0000","last_unstale":"2023-10-01T10:57:18.645356+0000","last_undegraded":"2023-10-01T10:57:18.645356+0000","last_fullsized":"2023-10-01T10:57:18.645356+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:45:12.783214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":208,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:18.656388+0000","last_change":"2023-10-01T10:37:42.767706+0000","last_active":"2023-10-01T10:57:18.656388+0000","last_peered":"2023-10-01T10:57:18.656388+0000","last_clean":"2023-10-01T10:57:18.656388+0000","last_became_active":"2023-10-01T10:37:42.767211+0000","last_became_peered":"2023-10-01T10:37:42.767211+0000","last_unstale":"2023-10-01T10:57:18.656388+0000","last_undegraded":"2023-10-01T10:57:18.656388+0000","last_fullsized":"2023-10-01T10:57:18.656388+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:37:20.357407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":212,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:18.657276+0000","last_change":"2023-10-01T10:37:42.768642+0000","last_active":"2023-10-01T10:57:18.657276+0000","last_peered":"2023-10-01T10:57:18.657276+0000","last_clean":"2023-10-01T10:57:18.657276+0000","last_became_active":"2023-10-01T10:37:42.768239+0000","last_became_peered":"2023-10-01T10:37:42.768239+0000","last_unstale":"2023-10-01T10:57:18.657276+0000","last_undegraded":"2023-10-01T10:57:18.657276+0000","last_fullsized":"2023-10-01T10:57:18.657276+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":179,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:24.173628+0000","last_change":"2023-10-01T10:56:14.620937+0000","last_active":"2023-10-01T10:57:24.173628+0000","last_peered":"2023-10-01T10:57:24.173628+0000","last_clean":"2023-10-01T10:57:24.173628+0000","last_became_active":"2023-10-01T10:56:14.620579+0000","last_became_peered":"2023-10-01T10:56:14.620579+0000","last_unstale":"2023-10-01T10:57:24.173628+0000","last_undegraded":"2023-10-01T10:57:24.173628+0000","last_fullsized":"2023-10-01T10:57:24.173628+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:53:21.947263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":209,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:19.352696+0000","last_change":"2023-10-01T10:37:42.772247+0000","last_active":"2023-10-01T10:57:19.352696+0000","last_peered":"2023-10-01T10:57:19.352696+0000","last_clean":"2023-10-01T10:57:19.352696+0000","last_became_active":"2023-10-01T10:37:42.771697+0000","last_became_peered":"2023-10-01T10:37:42.771697+0000","last_unstale":"2023-10-01T10:57:19.352696+0000","last_undegraded":"2023-10-01T10:57:19.352696+0000","last_fullsized":"2023-10-01T10:57:19.352696+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:23:15.018248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":31,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:18.634117+0000","last_change":"2023-10-01T10:56:14.636139+0000","last_active":"2023-10-01T10:57:18.634117+0000","last_peered":"2023-10-01T10:57:18.634117+0000","last_clean":"2023-10-01T10:57:18.634117+0000","last_became_active":"2023-10-01T10:56:14.624426+0000","last_became_peered":"2023-10-01T10:56:14.624426+0000","last_unstale":"2023-10-01T10:57:18.634117+0000","last_undegraded":"2023-10-01T10:57:18.634117+0000","last_fullsized":"2023-10-01T10:57:18.634117+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":213,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:18.645154+0000","last_change":"2023-10-01T10:37:42.773127+0000","last_active":"2023-10-01T10:57:18.645154+0000","last_peered":"2023-10-01T10:57:18.645154+0000","last_clean":"2023-10-01T10:57:18.645154+0000","last_became_active":"2023-10-01T10:37:42.772102+0000","last_became_peered":"2023-10-01T10:37:42.772102+0000","last_unstale":"2023-10-01T10:57:18.645154+0000","last_undegraded":"2023-10-01T10:57:18.645154+0000","last_fullsized":"2023-10-01T10:57:18.645154+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:23:26.202295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.0","version":"246'6096","reported_seq":6110,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-01T10:58:09.121898+0000","last_change":"2023-10-01T10:56:21.668720+0000","last_active":"2023-10-01T10:58:09.121898+0000","last_peered":"2023-10-01T10:58:09.121898+0000","last_clean":"2023-10-01T10:58:09.121898+0000","last_became_active":"2023-10-01T10:56:21.668210+0000","last_became_peered":"2023-10-01T10:56:21.668210+0000","last_unstale":"2023-10-01T10:58:09.121898+0000","last_undegraded":"2023-10-01T10:58:09.121898+0000","last_fullsized":"2023-10-01T10:58:09.121898+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":6096,"log_dups_size":0,"ondisk_log_size":6096,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:32:59.602604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":399507456,"num_objects":6096,"num_object_clones":0,"num_object_copies":18288,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6096,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6096,"num_write_kb":390144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":87,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:19.170358+0000","last_change":"2023-10-01T10:52:43.663145+0000","last_active":"2023-10-01T10:57:19.170358+0000","last_peered":"2023-10-01T10:57:19.170358+0000","last_clean":"2023-10-01T10:57:19.170358+0000","last_became_active":"2023-10-01T10:52:43.662783+0000","last_became_peered":"2023-10-01T10:52:43.662783+0000","last_unstale":"2023-10-01T10:57:19.170358+0000","last_undegraded":"2023-10-01T10:57:19.170358+0000","last_fullsized":"2023-10-01T10:57:19.170358+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T22:10:32.190922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":215,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:18.656185+0000","last_change":"2023-10-01T10:56:14.627958+0000","last_active":"2023-10-01T10:57:18.656185+0000","last_peered":"2023-10-01T10:57:18.656185+0000","last_clean":"2023-10-01T10:57:18.656185+0000","last_became_active":"2023-10-01T10:56:14.627506+0000","last_became_peered":"2023-10-01T10:56:14.627506+0000","last_unstale":"2023-10-01T10:57:18.656185+0000","last_undegraded":"2023-10-01T10:57:18.656185+0000","last_fullsized":"2023-10-01T10:57:18.656185+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"246'6080","reported_seq":6096,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-01T10:58:07.515017+0000","last_change":"2023-10-01T10:56:21.667913+0000","last_active":"2023-10-01T10:58:07.515017+0000","last_peered":"2023-10-01T10:58:07.515017+0000","last_clean":"2023-10-01T10:58:07.515017+0000","last_became_active":"2023-10-01T10:56:21.667502+0000","last_became_peered":"2023-10-01T10:56:21.667502+0000","last_unstale":"2023-10-01T10:58:07.515017+0000","last_undegraded":"2023-10-01T10:58:07.515017+0000","last_fullsized":"2023-10-01T10:58:07.515017+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":6080,"log_dups_size":0,"ondisk_log_size":6080,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:53:11.947301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":398458880,"num_objects":6080,"num_object_clones":0,"num_object_copies":18240,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6080,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6080,"num_write_kb":389120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":87,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:18.635324+0000","last_change":"2023-10-01T10:52:43.662540+0000","last_active":"2023-10-01T10:57:18.635324+0000","last_peered":"2023-10-01T10:57:18.635324+0000","last_clean":"2023-10-01T10:57:18.635324+0000","last_became_active":"2023-10-01T10:52:43.662128+0000","last_became_peered":"2023-10-01T10:52:43.662128+0000","last_unstale":"2023-10-01T10:57:18.635324+0000","last_undegraded":"2023-10-01T10:57:18.635324+0000","last_fullsized":"2023-10-01T10:57:18.635324+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:09:44.807003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":243,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:44.187503+0000","last_change":"2023-10-01T10:38:16.033203+0000","last_active":"2023-10-01T10:57:44.187503+0000","last_peered":"2023-10-01T10:57:44.187503+0000","last_clean":"2023-10-01T10:57:44.187503+0000","last_became_active":"2023-10-01T10:38:16.032681+0000","last_became_peered":"2023-10-01T10:38:16.032681+0000","last_unstale":"2023-10-01T10:57:44.187503+0000","last_undegraded":"2023-10-01T10:57:44.187503+0000","last_fullsized":"2023-10-01T10:57:44.187503+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:42:36.642284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"246'5988","reported_seq":6004,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-01T10:58:09.058275+0000","last_change":"2023-10-01T10:56:21.664087+0000","last_active":"2023-10-01T10:58:09.058275+0000","last_peered":"2023-10-01T10:58:09.058275+0000","last_clean":"2023-10-01T10:58:09.058275+0000","last_became_active":"2023-10-01T10:56:21.663655+0000","last_became_peered":"2023-10-01T10:56:21.663655+0000","last_unstale":"2023-10-01T10:58:09.058275+0000","last_undegraded":"2023-10-01T10:58:09.058275+0000","last_fullsized":"2023-10-01T10:58:09.058275+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":5988,"log_dups_size":0,"ondisk_log_size":5988,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:34:34.434364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":392429568,"num_objects":5988,"num_object_clones":0,"num_object_copies":17964,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5988,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5988,"num_write_kb":383232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":111,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:18.657469+0000","last_change":"2023-10-01T10:56:14.638810+0000","last_active":"2023-10-01T10:57:18.657469+0000","last_peered":"2023-10-01T10:57:18.657469+0000","last_clean":"2023-10-01T10:57:18.657469+0000","last_became_active":"2023-10-01T10:56:14.624601+0000","last_became_peered":"2023-10-01T10:56:14.624601+0000","last_unstale":"2023-10-01T10:57:18.657469+0000","last_undegraded":"2023-10-01T10:57:18.657469+0000","last_fullsized":"2023-10-01T10:57:18.657469+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":21,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:18.634221+0000","last_change":"2023-10-01T10:56:14.623448+0000","last_active":"2023-10-01T10:57:18.634221+0000","last_peered":"2023-10-01T10:57:18.634221+0000","last_clean":"2023-10-01T10:57:18.634221+0000","last_became_active":"2023-10-01T10:56:14.623123+0000","last_became_peered":"2023-10-01T10:56:14.623123+0000","last_unstale":"2023-10-01T10:57:18.634221+0000","last_undegraded":"2023-10-01T10:57:18.634221+0000","last_fullsized":"2023-10-01T10:57:18.634221+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"246'6062","reported_seq":6078,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-01T10:58:09.331267+0000","last_change":"2023-10-01T10:56:21.664942+0000","last_active":"2023-10-01T10:58:09.331267+0000","last_peered":"2023-10-01T10:58:09.331267+0000","last_clean":"2023-10-01T10:58:09.331267+0000","last_became_active":"2023-10-01T10:56:21.664236+0000","last_became_peered":"2023-10-01T10:56:21.664236+0000","last_unstale":"2023-10-01T10:58:09.331267+0000","last_undegraded":"2023-10-01T10:58:09.331267+0000","last_fullsized":"2023-10-01T10:58:09.331267+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":6062,"log_dups_size":0,"ondisk_log_size":6062,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:51:15.411254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":397279232,"num_objects":6062,"num_object_clones":0,"num_object_copies":18186,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6062,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6062,"num_write_kb":387968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":231,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:18.659201+0000","last_change":"2023-10-01T10:56:14.624999+0000","last_active":"2023-10-01T10:57:18.659201+0000","last_peered":"2023-10-01T10:57:18.659201+0000","last_clean":"2023-10-01T10:57:18.659201+0000","last_became_active":"2023-10-01T10:56:14.624597+0000","last_became_peered":"2023-10-01T10:56:14.624597+0000","last_unstale":"2023-10-01T10:57:18.659201+0000","last_undegraded":"2023-10-01T10:57:18.659201+0000","last_fullsized":"2023-10-01T10:57:18.659201+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":225,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:34.368513+0000","last_change":"2023-10-01T10:38:16.026348+0000","last_active":"2023-10-01T10:57:34.368513+0000","last_peered":"2023-10-01T10:57:34.368513+0000","last_clean":"2023-10-01T10:57:34.368513+0000","last_became_active":"2023-10-01T10:38:16.025747+0000","last_became_peered":"2023-10-01T10:38:16.025747+0000","last_unstale":"2023-10-01T10:57:34.368513+0000","last_undegraded":"2023-10-01T10:57:34.368513+0000","last_fullsized":"2023-10-01T10:57:34.368513+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:28:18.934480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"246'6075","reported_seq":6091,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-01T10:58:07.629590+0000","last_change":"2023-10-01T10:56:22.013729+0000","last_active":"2023-10-01T10:58:07.629590+0000","last_peered":"2023-10-01T10:58:07.629590+0000","last_clean":"2023-10-01T10:58:07.629590+0000","last_became_active":"2023-10-01T10:56:22.013403+0000","last_became_peered":"2023-10-01T10:56:22.013403+0000","last_unstale":"2023-10-01T10:58:07.629590+0000","last_undegraded":"2023-10-01T10:58:07.629590+0000","last_fullsized":"2023-10-01T10:58:07.629590+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":6075,"log_dups_size":0,"ondisk_log_size":6075,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:34:17.163618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":398131200,"num_objects":6075,"num_object_clones":0,"num_object_copies":18225,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6075,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6075,"num_write_kb":388800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":217,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-01T10:57:51.729829+0000","last_change":"2023-10-01T10:38:16.022493+0000","last_active":"2023-10-01T10:57:51.729829+0000","last_peered":"2023-10-01T10:57:51.729829+0000","last_clean":"2023-10-01T10:57:51.729829+0000","last_became_active":"2023-10-01T10:38:16.022033+0000","last_became_peered":"2023-10-01T10:38:16.022033+0000","last_unstale":"2023-10-01T10:57:51.729829+0000","last_undegraded":"2023-10-01T10:57:51.729829+0000","last_fullsized":"2023-10-01T10:57:51.729829+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:49:05.262763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":245,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:18.634313+0000","last_change":"2023-10-01T10:56:14.623628+0000","last_active":"2023-10-01T10:57:18.634313+0000","last_peered":"2023-10-01T10:57:18.634313+0000","last_clean":"2023-10-01T10:57:18.634313+0000","last_became_active":"2023-10-01T10:56:14.622190+0000","last_became_peered":"2023-10-01T10:56:14.622190+0000","last_unstale":"2023-10-01T10:57:18.634313+0000","last_undegraded":"2023-10-01T10:57:18.634313+0000","last_fullsized":"2023-10-01T10:57:18.634313+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:57:06.093638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"246'5923","reported_seq":5939,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-01T10:58:08.654461+0000","last_change":"2023-10-01T10:56:22.014079+0000","last_active":"2023-10-01T10:58:08.654461+0000","last_peered":"2023-10-01T10:58:08.654461+0000","last_clean":"2023-10-01T10:58:08.654461+0000","last_became_active":"2023-10-01T10:56:22.013703+0000","last_became_peered":"2023-10-01T10:56:22.013703+0000","last_unstale":"2023-10-01T10:58:08.654461+0000","last_undegraded":"2023-10-01T10:58:08.654461+0000","last_fullsized":"2023-10-01T10:58:08.654461+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":5923,"log_dups_size":0,"ondisk_log_size":5923,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:07:01.254677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":388169728,"num_objects":5923,"num_object_clones":0,"num_object_copies":17769,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5923,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5923,"num_write_kb":379072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":218,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:18.634183+0000","last_change":"2023-10-01T10:56:14.622835+0000","last_active":"2023-10-01T10:57:18.634183+0000","last_peered":"2023-10-01T10:57:18.634183+0000","last_clean":"2023-10-01T10:57:18.634183+0000","last_became_active":"2023-10-01T10:56:14.622449+0000","last_became_peered":"2023-10-01T10:56:14.622449+0000","last_unstale":"2023-10-01T10:57:18.634183+0000","last_undegraded":"2023-10-01T10:57:18.634183+0000","last_fullsized":"2023-10-01T10:57:18.634183+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:29:48.934875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":232,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:18.635234+0000","last_change":"2023-10-01T10:38:16.035906+0000","last_active":"2023-10-01T10:57:18.635234+0000","last_peered":"2023-10-01T10:57:18.635234+0000","last_clean":"2023-10-01T10:57:18.635234+0000","last_became_active":"2023-10-01T10:38:16.035383+0000","last_became_peered":"2023-10-01T10:38:16.035383+0000","last_unstale":"2023-10-01T10:57:18.635234+0000","last_undegraded":"2023-10-01T10:57:18.635234+0000","last_fullsized":"2023-10-01T10:57:18.635234+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:44:41.500097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"246'6170","reported_seq":6186,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-01T10:58:09.379555+0000","last_change":"2023-10-01T10:56:22.013872+0000","last_active":"2023-10-01T10:58:09.379555+0000","last_peered":"2023-10-01T10:58:09.379555+0000","last_clean":"2023-10-01T10:58:09.379555+0000","last_became_active":"2023-10-01T10:56:22.013554+0000","last_became_peered":"2023-10-01T10:56:22.013554+0000","last_unstale":"2023-10-01T10:58:09.379555+0000","last_undegraded":"2023-10-01T10:58:09.379555+0000","last_fullsized":"2023-10-01T10:58:09.379555+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":6170,"log_dups_size":0,"ondisk_log_size":6170,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:10:34.060296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":404357120,"num_objects":6170,"num_object_clones":0,"num_object_copies":18510,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6170,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6170,"num_write_kb":394880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":225,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:44.187547+0000","last_change":"2023-10-01T10:38:16.026611+0000","last_active":"2023-10-01T10:57:44.187547+0000","last_peered":"2023-10-01T10:57:44.187547+0000","last_clean":"2023-10-01T10:57:44.187547+0000","last_became_active":"2023-10-01T10:38:16.026267+0000","last_became_peered":"2023-10-01T10:38:16.026267+0000","last_unstale":"2023-10-01T10:57:44.187547+0000","last_undegraded":"2023-10-01T10:57:44.187547+0000","last_fullsized":"2023-10-01T10:57:44.187547+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:06.135288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":226,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:18.635273+0000","last_change":"2023-10-01T10:56:14.623011+0000","last_active":"2023-10-01T10:57:18.635273+0000","last_peered":"2023-10-01T10:57:18.635273+0000","last_clean":"2023-10-01T10:57:18.635273+0000","last_became_active":"2023-10-01T10:56:14.622649+0000","last_became_peered":"2023-10-01T10:56:14.622649+0000","last_unstale":"2023-10-01T10:57:18.635273+0000","last_undegraded":"2023-10-01T10:57:18.635273+0000","last_fullsized":"2023-10-01T10:57:18.635273+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:32:42.042469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"246'6196","reported_seq":6212,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-01T10:58:08.664702+0000","last_change":"2023-10-01T10:56:22.016312+0000","last_active":"2023-10-01T10:58:08.664702+0000","last_peered":"2023-10-01T10:58:08.664702+0000","last_clean":"2023-10-01T10:58:08.664702+0000","last_became_active":"2023-10-01T10:56:22.015964+0000","last_became_peered":"2023-10-01T10:56:22.015964+0000","last_unstale":"2023-10-01T10:58:08.664702+0000","last_undegraded":"2023-10-01T10:58:08.664702+0000","last_fullsized":"2023-10-01T10:58:08.664702+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":6196,"log_dups_size":0,"ondisk_log_size":6196,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:30:15.203782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":406061056,"num_objects":6196,"num_object_clones":0,"num_object_copies":18588,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6196,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6196,"num_write_kb":396544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":215,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:18.645912+0000","last_change":"2023-10-01T10:37:42.772398+0000","last_active":"2023-10-01T10:57:18.645912+0000","last_peered":"2023-10-01T10:57:18.645912+0000","last_clean":"2023-10-01T10:57:18.645912+0000","last_became_active":"2023-10-01T10:37:42.772084+0000","last_became_peered":"2023-10-01T10:37:42.772084+0000","last_unstale":"2023-10-01T10:57:18.645912+0000","last_undegraded":"2023-10-01T10:57:18.645912+0000","last_fullsized":"2023-10-01T10:57:18.645912+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:32:43.846419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":233,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:23.533893+0000","last_change":"2023-10-01T10:38:16.019755+0000","last_active":"2023-10-01T10:57:23.533893+0000","last_peered":"2023-10-01T10:57:23.533893+0000","last_clean":"2023-10-01T10:57:23.533893+0000","last_became_active":"2023-10-01T10:38:16.019397+0000","last_became_peered":"2023-10-01T10:38:16.019397+0000","last_unstale":"2023-10-01T10:57:23.533893+0000","last_undegraded":"2023-10-01T10:57:23.533893+0000","last_fullsized":"2023-10-01T10:57:23.533893+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:42:56.623225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.a","version":"246'6070","reported_seq":6086,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-01T10:58:08.635000+0000","last_change":"2023-10-01T10:56:21.663116+0000","last_active":"2023-10-01T10:58:08.635000+0000","last_peered":"2023-10-01T10:58:08.635000+0000","last_clean":"2023-10-01T10:58:08.635000+0000","last_became_active":"2023-10-01T10:56:21.662762+0000","last_became_peered":"2023-10-01T10:56:21.662762+0000","last_unstale":"2023-10-01T10:58:08.635000+0000","last_undegraded":"2023-10-01T10:58:08.635000+0000","last_fullsized":"2023-10-01T10:58:08.635000+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":6070,"log_dups_size":0,"ondisk_log_size":6070,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:00:59.657772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":397803520,"num_objects":6070,"num_object_clones":0,"num_object_copies":18210,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6070,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6070,"num_write_kb":388480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":140,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:18.654573+0000","last_change":"2023-10-01T10:52:18.960164+0000","last_active":"2023-10-01T10:57:18.654573+0000","last_peered":"2023-10-01T10:57:18.654573+0000","last_clean":"2023-10-01T10:57:18.654573+0000","last_became_active":"2023-10-01T10:52:18.959782+0000","last_became_peered":"2023-10-01T10:52:18.959782+0000","last_unstale":"2023-10-01T10:57:18.654573+0000","last_undegraded":"2023-10-01T10:57:18.654573+0000","last_fullsized":"2023-10-01T10:57:18.654573+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:08:27.603412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":228,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:18.657054+0000","last_change":"2023-10-01T10:37:42.766814+0000","last_active":"2023-10-01T10:57:18.657054+0000","last_peered":"2023-10-01T10:57:18.657054+0000","last_clean":"2023-10-01T10:57:18.657054+0000","last_became_active":"2023-10-01T10:37:42.766451+0000","last_became_peered":"2023-10-01T10:37:42.766451+0000","last_unstale":"2023-10-01T10:57:18.657054+0000","last_undegraded":"2023-10-01T10:57:18.657054+0000","last_fullsized":"2023-10-01T10:57:18.657054+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":131,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:18.656264+0000","last_change":"2023-10-01T10:52:18.960606+0000","last_active":"2023-10-01T10:57:18.656264+0000","last_peered":"2023-10-01T10:57:18.656264+0000","last_clean":"2023-10-01T10:57:18.656264+0000","last_became_active":"2023-10-01T10:52:18.960270+0000","last_became_peered":"2023-10-01T10:52:18.960270+0000","last_unstale":"2023-10-01T10:57:18.656264+0000","last_undegraded":"2023-10-01T10:57:18.656264+0000","last_fullsized":"2023-10-01T10:57:18.656264+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.c","version":"246'6204","reported_seq":6220,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-01T10:58:09.408936+0000","last_change":"2023-10-01T10:56:21.895915+0000","last_active":"2023-10-01T10:58:09.408936+0000","last_peered":"2023-10-01T10:58:09.408936+0000","last_clean":"2023-10-01T10:58:09.408936+0000","last_became_active":"2023-10-01T10:56:21.895501+0000","last_became_peered":"2023-10-01T10:56:21.895501+0000","last_unstale":"2023-10-01T10:58:09.408936+0000","last_undegraded":"2023-10-01T10:58:09.408936+0000","last_fullsized":"2023-10-01T10:58:09.408936+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":6204,"log_dups_size":0,"ondisk_log_size":6204,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:21:21.588297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":406585344,"num_objects":6204,"num_object_clones":0,"num_object_copies":18612,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6204,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6204,"num_write_kb":397056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":235,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:18.656386+0000","last_change":"2023-10-01T10:56:14.624787+0000","last_active":"2023-10-01T10:57:18.656386+0000","last_peered":"2023-10-01T10:57:18.656386+0000","last_clean":"2023-10-01T10:57:18.656386+0000","last_became_active":"2023-10-01T10:56:14.624357+0000","last_became_peered":"2023-10-01T10:56:14.624357+0000","last_unstale":"2023-10-01T10:57:18.656386+0000","last_undegraded":"2023-10-01T10:57:18.656386+0000","last_fullsized":"2023-10-01T10:57:18.656386+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":213,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:18.634863+0000","last_change":"2023-10-01T10:56:14.620354+0000","last_active":"2023-10-01T10:57:18.634863+0000","last_peered":"2023-10-01T10:57:18.634863+0000","last_clean":"2023-10-01T10:57:18.634863+0000","last_became_active":"2023-10-01T10:56:14.619900+0000","last_became_peered":"2023-10-01T10:56:14.619900+0000","last_unstale":"2023-10-01T10:57:18.634863+0000","last_undegraded":"2023-10-01T10:57:18.634863+0000","last_fullsized":"2023-10-01T10:57:18.634863+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:09:36.449841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":143,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:18.651784+0000","last_change":"2023-10-01T10:52:18.963437+0000","last_active":"2023-10-01T10:57:18.651784+0000","last_peered":"2023-10-01T10:57:18.651784+0000","last_clean":"2023-10-01T10:57:18.651784+0000","last_became_active":"2023-10-01T10:52:18.955165+0000","last_became_peered":"2023-10-01T10:52:18.955165+0000","last_unstale":"2023-10-01T10:57:18.651784+0000","last_undegraded":"2023-10-01T10:57:18.651784+0000","last_fullsized":"2023-10-01T10:57:18.651784+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:57:49.129100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.f","version":"246'6223","reported_seq":6238,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-01T10:58:08.475619+0000","last_change":"2023-10-01T10:56:21.896411+0000","last_active":"2023-10-01T10:58:08.475619+0000","last_peered":"2023-10-01T10:58:08.475619+0000","last_clean":"2023-10-01T10:58:08.475619+0000","last_became_active":"2023-10-01T10:56:21.895807+0000","last_became_peered":"2023-10-01T10:56:21.895807+0000","last_unstale":"2023-10-01T10:58:08.475619+0000","last_undegraded":"2023-10-01T10:58:08.475619+0000","last_fullsized":"2023-10-01T10:58:08.475619+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":6223,"log_dups_size":0,"ondisk_log_size":6223,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":407830528,"num_objects":6223,"num_object_clones":0,"num_object_copies":18669,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6223,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6223,"num_write_kb":398272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":225,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-01T10:57:51.715115+0000","last_change":"2023-10-01T10:37:42.762524+0000","last_active":"2023-10-01T10:57:51.715115+0000","last_peered":"2023-10-01T10:57:51.715115+0000","last_clean":"2023-10-01T10:57:51.715115+0000","last_became_active":"2023-10-01T10:37:42.762209+0000","last_became_peered":"2023-10-01T10:37:42.762209+0000","last_unstale":"2023-10-01T10:57:51.715115+0000","last_undegraded":"2023-10-01T10:57:51.715115+0000","last_fullsized":"2023-10-01T10:57:51.715115+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:45:38.171397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":141,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:18.644813+0000","last_change":"2023-10-01T10:56:14.621343+0000","last_active":"2023-10-01T10:57:18.644813+0000","last_peered":"2023-10-01T10:57:18.644813+0000","last_clean":"2023-10-01T10:57:18.644813+0000","last_became_active":"2023-10-01T10:56:14.620940+0000","last_became_peered":"2023-10-01T10:56:14.620940+0000","last_unstale":"2023-10-01T10:57:18.644813+0000","last_undegraded":"2023-10-01T10:57:18.644813+0000","last_fullsized":"2023-10-01T10:57:18.644813+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:43:39.290894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":133,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:18.658089+0000","last_change":"2023-10-01T10:52:18.961040+0000","last_active":"2023-10-01T10:57:18.658089+0000","last_peered":"2023-10-01T10:57:18.658089+0000","last_clean":"2023-10-01T10:57:18.658089+0000","last_became_active":"2023-10-01T10:52:18.960606+0000","last_became_peered":"2023-10-01T10:52:18.960606+0000","last_unstale":"2023-10-01T10:57:18.658089+0000","last_undegraded":"2023-10-01T10:57:18.658089+0000","last_fullsized":"2023-10-01T10:57:18.658089+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.d","version":"246'6161","reported_seq":6176,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-01T10:58:08.403990+0000","last_change":"2023-10-01T10:56:22.015481+0000","last_active":"2023-10-01T10:58:08.403990+0000","last_peered":"2023-10-01T10:58:08.403990+0000","last_clean":"2023-10-01T10:58:08.403990+0000","last_became_active":"2023-10-01T10:56:22.014648+0000","last_became_peered":"2023-10-01T10:56:22.014648+0000","last_unstale":"2023-10-01T10:58:08.403990+0000","last_undegraded":"2023-10-01T10:58:08.403990+0000","last_fullsized":"2023-10-01T10:58:08.403990+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":6161,"log_dups_size":0,"ondisk_log_size":6161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":403767296,"num_objects":6161,"num_object_clones":0,"num_object_copies":18483,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6161,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6161,"num_write_kb":394304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":217,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:18.656557+0000","last_change":"2023-10-01T10:37:05.024009+0000","last_active":"2023-10-01T10:57:18.656557+0000","last_peered":"2023-10-01T10:57:18.656557+0000","last_clean":"2023-10-01T10:57:18.656557+0000","last_became_active":"2023-10-01T10:37:05.023458+0000","last_became_peered":"2023-10-01T10:37:05.023458+0000","last_unstale":"2023-10-01T10:57:18.656557+0000","last_undegraded":"2023-10-01T10:57:18.656557+0000","last_fullsized":"2023-10-01T10:57:18.656557+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'139","reported_seq":384,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:18.650112+0000","last_change":"2023-10-01T10:56:17.635571+0000","last_active":"2023-10-01T10:57:18.650112+0000","last_peered":"2023-10-01T10:57:18.650112+0000","last_clean":"2023-10-01T10:57:18.650112+0000","last_became_active":"2023-10-01T10:56:17.635055+0000","last_became_peered":"2023-10-01T10:56:17.635055+0000","last_unstale":"2023-10-01T10:57:18.650112+0000","last_undegraded":"2023-10-01T10:57:18.650112+0000","last_fullsized":"2023-10-01T10:57:18.650112+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:09:17.221082+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:09:17.221082+0000","last_clean_scrub_stamp":"2023-10-01T10:09:17.221082+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:37:54.527937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":215,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:18.648555+0000","last_change":"2023-10-01T10:56:14.622079+0000","last_active":"2023-10-01T10:57:18.648555+0000","last_peered":"2023-10-01T10:57:18.648555+0000","last_clean":"2023-10-01T10:57:18.648555+0000","last_became_active":"2023-10-01T10:56:14.621644+0000","last_became_peered":"2023-10-01T10:56:14.621644+0000","last_unstale":"2023-10-01T10:57:18.648555+0000","last_undegraded":"2023-10-01T10:57:18.648555+0000","last_fullsized":"2023-10-01T10:57:18.648555+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:32:47.233878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":202,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:18.635334+0000","last_change":"2023-10-01T10:56:14.622469+0000","last_active":"2023-10-01T10:57:18.635334+0000","last_peered":"2023-10-01T10:57:18.635334+0000","last_clean":"2023-10-01T10:57:18.635334+0000","last_became_active":"2023-10-01T10:56:14.622092+0000","last_became_peered":"2023-10-01T10:56:14.622092+0000","last_unstale":"2023-10-01T10:57:18.635334+0000","last_undegraded":"2023-10-01T10:57:18.635334+0000","last_fullsized":"2023-10-01T10:57:18.635334+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:43:56.657361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.e","version":"246'6161","reported_seq":6177,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-01T10:58:09.410357+0000","last_change":"2023-10-01T10:56:21.663535+0000","last_active":"2023-10-01T10:58:09.410357+0000","last_peered":"2023-10-01T10:58:09.410357+0000","last_clean":"2023-10-01T10:58:09.410357+0000","last_became_active":"2023-10-01T10:56:21.663135+0000","last_became_peered":"2023-10-01T10:56:21.663135+0000","last_unstale":"2023-10-01T10:58:09.410357+0000","last_undegraded":"2023-10-01T10:58:09.410357+0000","last_fullsized":"2023-10-01T10:58:09.410357+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":6161,"log_dups_size":0,"ondisk_log_size":6161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:11:25.118806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":403767296,"num_objects":6161,"num_object_clones":0,"num_object_copies":18483,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6161,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6161,"num_write_kb":394304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":155,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:18.650249+0000","last_change":"2023-10-01T10:52:43.661009+0000","last_active":"2023-10-01T10:57:18.650249+0000","last_peered":"2023-10-01T10:57:18.650249+0000","last_clean":"2023-10-01T10:57:18.650249+0000","last_became_active":"2023-10-01T10:52:43.660679+0000","last_became_peered":"2023-10-01T10:52:43.660679+0000","last_unstale":"2023-10-01T10:57:18.650249+0000","last_undegraded":"2023-10-01T10:57:18.650249+0000","last_fullsized":"2023-10-01T10:57:18.650249+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:32:21.336835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":225,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:29.354740+0000","last_change":"2023-10-01T10:38:16.021968+0000","last_active":"2023-10-01T10:57:29.354740+0000","last_peered":"2023-10-01T10:57:29.354740+0000","last_clean":"2023-10-01T10:57:29.354740+0000","last_became_active":"2023-10-01T10:38:16.021539+0000","last_became_peered":"2023-10-01T10:38:16.021539+0000","last_unstale":"2023-10-01T10:57:29.354740+0000","last_undegraded":"2023-10-01T10:57:29.354740+0000","last_fullsized":"2023-10-01T10:57:29.354740+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:12:53.105747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":225,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:18.655488+0000","last_change":"2023-10-01T10:37:05.017431+0000","last_active":"2023-10-01T10:57:18.655488+0000","last_peered":"2023-10-01T10:57:18.655488+0000","last_clean":"2023-10-01T10:57:18.655488+0000","last_became_active":"2023-10-01T10:37:05.017094+0000","last_became_peered":"2023-10-01T10:37:05.017094+0000","last_unstale":"2023-10-01T10:57:18.655488+0000","last_undegraded":"2023-10-01T10:57:18.655488+0000","last_fullsized":"2023-10-01T10:57:18.655488+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:10:40.664686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.b","version":"246'6150","reported_seq":6166,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-01T10:58:07.496114+0000","last_change":"2023-10-01T10:56:21.667103+0000","last_active":"2023-10-01T10:58:07.496114+0000","last_peered":"2023-10-01T10:58:07.496114+0000","last_clean":"2023-10-01T10:58:07.496114+0000","last_became_active":"2023-10-01T10:56:21.666784+0000","last_became_peered":"2023-10-01T10:56:21.666784+0000","last_unstale":"2023-10-01T10:58:07.496114+0000","last_undegraded":"2023-10-01T10:58:07.496114+0000","last_fullsized":"2023-10-01T10:58:07.496114+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":6150,"log_dups_size":0,"ondisk_log_size":6150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:45:11.964390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":403046400,"num_objects":6150,"num_object_clones":0,"num_object_copies":18450,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6150,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6150,"num_write_kb":393600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":231,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:18.633374+0000","last_change":"2023-10-01T10:38:16.032774+0000","last_active":"2023-10-01T10:57:18.633374+0000","last_peered":"2023-10-01T10:57:18.633374+0000","last_clean":"2023-10-01T10:57:18.633374+0000","last_became_active":"2023-10-01T10:38:16.032283+0000","last_became_peered":"2023-10-01T10:38:16.032283+0000","last_unstale":"2023-10-01T10:57:18.633374+0000","last_undegraded":"2023-10-01T10:57:18.633374+0000","last_fullsized":"2023-10-01T10:57:18.633374+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":123,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:18.645973+0000","last_change":"2023-10-01T10:52:18.964479+0000","last_active":"2023-10-01T10:57:18.645973+0000","last_peered":"2023-10-01T10:57:18.645973+0000","last_clean":"2023-10-01T10:57:18.645973+0000","last_became_active":"2023-10-01T10:52:18.963881+0000","last_became_peered":"2023-10-01T10:52:18.963881+0000","last_unstale":"2023-10-01T10:57:18.645973+0000","last_undegraded":"2023-10-01T10:57:18.645973+0000","last_fullsized":"2023-10-01T10:57:18.645973+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:36:30.405755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":196,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:29.354812+0000","last_change":"2023-10-01T10:28:20.220812+0000","last_active":"2023-10-01T10:57:29.354812+0000","last_peered":"2023-10-01T10:57:29.354812+0000","last_clean":"2023-10-01T10:57:29.354812+0000","last_became_active":"2023-10-01T10:28:20.215962+0000","last_became_peered":"2023-10-01T10:28:20.215962+0000","last_unstale":"2023-10-01T10:57:29.354812+0000","last_undegraded":"2023-10-01T10:57:29.354812+0000","last_fullsized":"2023-10-01T10:57:29.354812+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:35:15.275097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.8","version":"246'6119","reported_seq":6135,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-01T10:58:07.615718+0000","last_change":"2023-10-01T10:56:21.664695+0000","last_active":"2023-10-01T10:58:07.615718+0000","last_peered":"2023-10-01T10:58:07.615718+0000","last_clean":"2023-10-01T10:58:07.615718+0000","last_became_active":"2023-10-01T10:56:21.664315+0000","last_became_peered":"2023-10-01T10:56:21.664315+0000","last_unstale":"2023-10-01T10:58:07.615718+0000","last_undegraded":"2023-10-01T10:58:07.615718+0000","last_fullsized":"2023-10-01T10:58:07.615718+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":6119,"log_dups_size":0,"ondisk_log_size":6119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:27:54.090426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":401014784,"num_objects":6119,"num_object_clones":0,"num_object_copies":18357,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6119,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6119,"num_write_kb":391616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":202,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:18.645300+0000","last_change":"2023-10-01T10:37:42.773999+0000","last_active":"2023-10-01T10:57:18.645300+0000","last_peered":"2023-10-01T10:57:18.645300+0000","last_clean":"2023-10-01T10:57:18.645300+0000","last_became_active":"2023-10-01T10:37:42.773355+0000","last_became_peered":"2023-10-01T10:37:42.773355+0000","last_unstale":"2023-10-01T10:57:18.645300+0000","last_undegraded":"2023-10-01T10:57:18.645300+0000","last_fullsized":"2023-10-01T10:57:18.645300+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:46:51.317151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":193,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:18.635205+0000","last_change":"2023-10-01T10:56:14.623768+0000","last_active":"2023-10-01T10:57:18.635205+0000","last_peered":"2023-10-01T10:57:18.635205+0000","last_clean":"2023-10-01T10:57:18.635205+0000","last_became_active":"2023-10-01T10:56:14.623353+0000","last_became_peered":"2023-10-01T10:56:14.623353+0000","last_unstale":"2023-10-01T10:57:18.635205+0000","last_undegraded":"2023-10-01T10:57:18.635205+0000","last_fullsized":"2023-10-01T10:57:18.635205+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:53:19.776540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":200,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:42.491429+0000","last_change":"2023-10-01T10:37:42.773538+0000","last_active":"2023-10-01T10:57:42.491429+0000","last_peered":"2023-10-01T10:57:42.491429+0000","last_clean":"2023-10-01T10:57:42.491429+0000","last_became_active":"2023-10-01T10:37:42.771863+0000","last_became_peered":"2023-10-01T10:37:42.771863+0000","last_unstale":"2023-10-01T10:57:42.491429+0000","last_undegraded":"2023-10-01T10:57:42.491429+0000","last_fullsized":"2023-10-01T10:57:42.491429+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:46:46.698627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"246'6052","reported_seq":6068,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-10-01T10:58:08.647451+0000","last_change":"2023-10-01T10:56:21.667837+0000","last_active":"2023-10-01T10:58:08.647451+0000","last_peered":"2023-10-01T10:58:08.647451+0000","last_clean":"2023-10-01T10:58:08.647451+0000","last_became_active":"2023-10-01T10:56:21.667513+0000","last_became_peered":"2023-10-01T10:56:21.667513+0000","last_unstale":"2023-10-01T10:58:08.647451+0000","last_undegraded":"2023-10-01T10:58:08.647451+0000","last_fullsized":"2023-10-01T10:58:08.647451+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":6052,"log_dups_size":0,"ondisk_log_size":6052,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:25:17.334326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":396623872,"num_objects":6052,"num_object_clones":0,"num_object_copies":18156,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6052,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6052,"num_write_kb":387328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":21,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:18.634744+0000","last_change":"2023-10-01T10:56:14.621086+0000","last_active":"2023-10-01T10:57:18.634744+0000","last_peered":"2023-10-01T10:57:18.634744+0000","last_clean":"2023-10-01T10:57:18.634744+0000","last_became_active":"2023-10-01T10:56:14.620638+0000","last_became_peered":"2023-10-01T10:56:14.620638+0000","last_unstale":"2023-10-01T10:57:18.634744+0000","last_undegraded":"2023-10-01T10:57:18.634744+0000","last_fullsized":"2023-10-01T10:57:18.634744+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":228,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:18.655218+0000","last_change":"2023-10-01T10:38:16.027472+0000","last_active":"2023-10-01T10:57:18.655218+0000","last_peered":"2023-10-01T10:57:18.655218+0000","last_clean":"2023-10-01T10:57:18.655218+0000","last_became_active":"2023-10-01T10:38:16.026810+0000","last_became_peered":"2023-10-01T10:38:16.026810+0000","last_unstale":"2023-10-01T10:57:18.655218+0000","last_undegraded":"2023-10-01T10:57:18.655218+0000","last_fullsized":"2023-10-01T10:57:18.655218+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:30:06.978187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":211,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:18.633798+0000","last_change":"2023-10-01T10:56:14.626521+0000","last_active":"2023-10-01T10:57:18.633798+0000","last_peered":"2023-10-01T10:57:18.633798+0000","last_clean":"2023-10-01T10:57:18.633798+0000","last_became_active":"2023-10-01T10:56:14.626107+0000","last_became_peered":"2023-10-01T10:56:14.626107+0000","last_unstale":"2023-10-01T10:57:18.633798+0000","last_undegraded":"2023-10-01T10:57:18.633798+0000","last_fullsized":"2023-10-01T10:57:18.633798+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:44:12.588719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":211,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:18.656118+0000","last_change":"2023-10-01T10:38:16.028491+0000","last_active":"2023-10-01T10:57:18.656118+0000","last_peered":"2023-10-01T10:57:18.656118+0000","last_clean":"2023-10-01T10:57:18.656118+0000","last_became_active":"2023-10-01T10:38:16.028175+0000","last_became_peered":"2023-10-01T10:38:16.028175+0000","last_unstale":"2023-10-01T10:57:18.656118+0000","last_undegraded":"2023-10-01T10:57:18.656118+0000","last_fullsized":"2023-10-01T10:57:18.656118+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:44:29.083129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":233,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:18.646125+0000","last_change":"2023-10-01T10:37:42.762360+0000","last_active":"2023-10-01T10:57:18.646125+0000","last_peered":"2023-10-01T10:57:18.646125+0000","last_clean":"2023-10-01T10:57:18.646125+0000","last_became_active":"2023-10-01T10:37:42.761936+0000","last_became_peered":"2023-10-01T10:37:42.761936+0000","last_unstale":"2023-10-01T10:57:18.646125+0000","last_undegraded":"2023-10-01T10:57:18.646125+0000","last_fullsized":"2023-10-01T10:57:18.646125+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:50.637582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":230,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:18.659311+0000","last_change":"2023-10-01T10:38:16.037809+0000","last_active":"2023-10-01T10:57:18.659311+0000","last_peered":"2023-10-01T10:57:18.659311+0000","last_clean":"2023-10-01T10:57:18.659311+0000","last_became_active":"2023-10-01T10:38:16.037318+0000","last_became_peered":"2023-10-01T10:38:16.037318+0000","last_unstale":"2023-10-01T10:57:18.659311+0000","last_undegraded":"2023-10-01T10:57:18.659311+0000","last_fullsized":"2023-10-01T10:57:18.659311+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":241,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:18.634687+0000","last_change":"2023-10-01T10:38:16.034282+0000","last_active":"2023-10-01T10:57:18.634687+0000","last_peered":"2023-10-01T10:57:18.634687+0000","last_clean":"2023-10-01T10:57:18.634687+0000","last_became_active":"2023-10-01T10:38:16.033820+0000","last_became_peered":"2023-10-01T10:38:16.033820+0000","last_unstale":"2023-10-01T10:57:18.634687+0000","last_undegraded":"2023-10-01T10:57:18.634687+0000","last_fullsized":"2023-10-01T10:57:18.634687+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:48:23.815832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":241,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:18.656863+0000","last_change":"2023-10-01T10:56:14.636784+0000","last_active":"2023-10-01T10:57:18.656863+0000","last_peered":"2023-10-01T10:57:18.656863+0000","last_clean":"2023-10-01T10:57:18.656863+0000","last_became_active":"2023-10-01T10:56:14.624338+0000","last_became_peered":"2023-10-01T10:56:14.624338+0000","last_unstale":"2023-10-01T10:57:18.656863+0000","last_undegraded":"2023-10-01T10:57:18.656863+0000","last_fullsized":"2023-10-01T10:57:18.656863+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":241,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:18.634274+0000","last_change":"2023-10-01T10:38:16.025600+0000","last_active":"2023-10-01T10:57:18.634274+0000","last_peered":"2023-10-01T10:57:18.634274+0000","last_clean":"2023-10-01T10:57:18.634274+0000","last_became_active":"2023-10-01T10:38:16.025252+0000","last_became_peered":"2023-10-01T10:38:16.025252+0000","last_unstale":"2023-10-01T10:57:18.634274+0000","last_undegraded":"2023-10-01T10:57:18.634274+0000","last_fullsized":"2023-10-01T10:57:18.634274+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:24:18.871963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":249,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:42.491595+0000","last_change":"2023-10-01T10:56:14.621544+0000","last_active":"2023-10-01T10:57:42.491595+0000","last_peered":"2023-10-01T10:57:42.491595+0000","last_clean":"2023-10-01T10:57:42.491595+0000","last_became_active":"2023-10-01T10:56:14.621134+0000","last_became_peered":"2023-10-01T10:56:14.621134+0000","last_unstale":"2023-10-01T10:57:42.491595+0000","last_undegraded":"2023-10-01T10:57:42.491595+0000","last_fullsized":"2023-10-01T10:57:42.491595+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:48:44.436158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":242,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:18.645054+0000","last_change":"2023-10-01T10:37:42.765142+0000","last_active":"2023-10-01T10:57:18.645054+0000","last_peered":"2023-10-01T10:57:18.645054+0000","last_clean":"2023-10-01T10:57:18.645054+0000","last_became_active":"2023-10-01T10:37:42.764770+0000","last_became_peered":"2023-10-01T10:37:42.764770+0000","last_unstale":"2023-10-01T10:57:18.645054+0000","last_undegraded":"2023-10-01T10:57:18.645054+0000","last_fullsized":"2023-10-01T10:57:18.645054+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:23:33.527876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":214,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:29.354965+0000","last_change":"2023-10-01T10:37:05.021621+0000","last_active":"2023-10-01T10:57:29.354965+0000","last_peered":"2023-10-01T10:57:29.354965+0000","last_clean":"2023-10-01T10:57:29.354965+0000","last_became_active":"2023-10-01T10:37:05.021173+0000","last_became_peered":"2023-10-01T10:37:05.021173+0000","last_unstale":"2023-10-01T10:57:29.354965+0000","last_undegraded":"2023-10-01T10:57:29.354965+0000","last_fullsized":"2023-10-01T10:57:29.354965+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:13:25.364499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":226,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-10-01T10:57:34.175259+0000","last_change":"2023-10-01T10:56:14.621791+0000","last_active":"2023-10-01T10:57:34.175259+0000","last_peered":"2023-10-01T10:57:34.175259+0000","last_clean":"2023-10-01T10:57:34.175259+0000","last_became_active":"2023-10-01T10:56:14.621443+0000","last_became_peered":"2023-10-01T10:56:14.621443+0000","last_unstale":"2023-10-01T10:57:34.175259+0000","last_undegraded":"2023-10-01T10:57:34.175259+0000","last_fullsized":"2023-10-01T10:57:34.175259+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:13:58.651222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":6404833280,"num_objects":97730,"num_object_clones":0,"num_object_copies":293190,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":97730,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":97730,"num_write_kb":6254720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":19251331072,"data_stored":19251331072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97730,"ondisk_log_size":97730,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":9608,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30319,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430754,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3610584,"kb_used_data":2720336,"kb_used_omap":0,"kb_used_meta":890240,"kb_avail":90134568,"statfs":{"total":95995035648,"available":92297797632,"internally_reserved":0,"allocated":2785624064,"data_stored":2785077935,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":911605760},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:57:17 2023","interfaces":[{"interface":"back","average":{"1min":0.94299999999999995,"5min":1.651,"15min":2.4470000000000001},"min":{"1min":0.27500000000000002,"5min":0.252,"15min":0.252},"max":{"1min":2.9119999999999999,"5min":14.315,"15min":137.34200000000001},"last":4.423},{"interface":"front","average":{"1min":1.1479999999999999,"5min":1.522,"15min":2.3180000000000001},"min":{"1min":0.377,"5min":0.32500000000000001,"15min":0.28799999999999998},"max":{"1min":3.137,"5min":14.336,"15min":137.39500000000001},"last":4.4489999999999998}]},{"osd":1,"last update":"Sun Oct 1 10:57:23 2023","interfaces":[{"interface":"back","average":{"1min":1.1639999999999999,"5min":1.8660000000000001,"15min":1.8240000000000001},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":4.4240000000000004,"5min":15.662000000000001,"15min":56.110999999999997},"last":0.45000000000000001},{"interface":"front","average":{"1min":1.0660000000000001,"5min":1.6539999999999999,"15min":1.7769999999999999},"min":{"1min":0.35599999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":4.1920000000000002,"5min":18.768999999999998,"15min":51.258000000000003},"last":2.048}]},{"osd":2,"last update":"Sun Oct 1 10:57:36 2023","interfaces":[{"interface":"back","average":{"1min":1.538,"5min":2.7410000000000001,"15min":1.8879999999999999},"min":{"1min":0.58199999999999996,"5min":0.33300000000000002,"15min":0.315},"max":{"1min":6.6189999999999998,"5min":42.804000000000002,"15min":42.804000000000002},"last":2.0720000000000001},{"interface":"front","average":{"1min":1.474,"5min":2.4990000000000001,"15min":1.8009999999999999},"min":{"1min":0.54300000000000004,"5min":0.36399999999999999,"15min":0.33200000000000002},"max":{"1min":4.5060000000000002,"5min":42.819000000000003,"15min":42.819000000000003},"last":0.50600000000000001}]},{"osd":3,"last update":"Sun Oct 1 10:57:17 2023","interfaces":[{"interface":"back","average":{"1min":1.071,"5min":1.536,"15min":1.857},"min":{"1min":0.40400000000000003,"5min":0.32300000000000001,"15min":0.318},"max":{"1min":3.879,"5min":8.6839999999999993,"15min":51.334000000000003},"last":2.145},{"interface":"front","average":{"1min":1.327,"5min":1.792,"15min":1.8919999999999999},"min":{"1min":0.38,"5min":0.33400000000000002,"15min":0.313},"max":{"1min":11.244999999999999,"5min":11.244999999999999,"15min":51.404000000000003},"last":0.56000000000000005}]},{"osd":4,"last update":"Sun Oct 1 10:57:46 2023","interfaces":[{"interface":"back","average":{"1min":1.4530000000000001,"5min":1.5129999999999999,"15min":1.4319999999999999},"min":{"1min":0.41199999999999998,"5min":0.35699999999999998,"15min":0.33500000000000002},"max":{"1min":4.5510000000000002,"5min":7.7800000000000002,"15min":11.081},"last":0.47999999999999998},{"interface":"front","average":{"1min":1.1120000000000001,"5min":1.446,"15min":1.4850000000000001},"min":{"1min":0.51200000000000001,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":4.3150000000000004,"5min":8.7370000000000001,"15min":12.292},"last":2.1240000000000001}]},{"osd":5,"last update":"Sun Oct 1 10:57:23 2023","interfaces":[{"interface":"back","average":{"1min":0.96099999999999997,"5min":1.603,"15min":1.9179999999999999},"min":{"1min":0.41499999999999998,"5min":0.375,"15min":0.34599999999999997},"max":{"1min":1.732,"5min":15.726000000000001,"15min":40.381},"last":2.1080000000000001},{"interface":"front","average":{"1min":1.151,"5min":1.5389999999999999,"15min":1.931},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.32200000000000001},"max":{"1min":4.5739999999999998,"5min":9.1709999999999994,"15min":39.590000000000003},"last":0.53300000000000003}]},{"osd":6,"last update":"Sun Oct 1 10:57:23 2023","interfaces":[{"interface":"back","average":{"1min":1.022,"5min":1.391,"15min":1.5800000000000001},"min":{"1min":0.45300000000000001,"5min":0.35799999999999998,"15min":0.34499999999999997},"max":{"1min":1.651,"5min":9.4990000000000006,"15min":12.329000000000001},"last":2.0910000000000002},{"interface":"front","average":{"1min":1.4159999999999999,"5min":1.7669999999999999,"15min":1.5289999999999999},"min":{"1min":0.52800000000000002,"5min":0.39700000000000002,"15min":0.36599999999999999},"max":{"1min":4.681,"5min":9.4329999999999998,"15min":9.4329999999999998},"last":0.41099999999999998}]}]},{"osd":6,"up_from":42,"seq":180388626988,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3608200,"kb_used_data":2719296,"kb_used_omap":0,"kb_used_meta":888896,"kb_avail":90136952,"statfs":{"total":95995035648,"available":92300238848,"internally_reserved":0,"allocated":2784559104,"data_stored":2784028898,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":910229504},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:57:11 2023","interfaces":[{"interface":"back","average":{"1min":1.109,"5min":1.468,"15min":2.7490000000000001},"min":{"1min":0.54200000000000004,"5min":0.28299999999999997,"15min":0.25},"max":{"1min":3.4750000000000001,"5min":8.2899999999999991,"15min":140.524},"last":2.887},{"interface":"front","average":{"1min":0.99299999999999999,"5min":1.5509999999999999,"15min":2.7440000000000002},"min":{"1min":0.34000000000000002,"5min":0.29399999999999998,"15min":0.249},"max":{"1min":2.3159999999999998,"5min":9.5259999999999998,"15min":140.86500000000001},"last":3.5049999999999999}]},{"osd":1,"last update":"Sun Oct 1 10:57:22 2023","interfaces":[{"interface":"back","average":{"1min":0.92800000000000005,"5min":1.5269999999999999,"15min":1.532},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.26000000000000001},"max":{"1min":2.0019999999999998,"5min":9.843,"15min":9.843},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.89900000000000002,"5min":1.577,"15min":1.607},"min":{"1min":0.32700000000000001,"5min":0.26400000000000001,"15min":0.26300000000000001},"max":{"1min":1.7829999999999999,"5min":9.4809999999999999,"15min":12.968999999999999},"last":0.45000000000000001}]},{"osd":2,"last update":"Sun Oct 1 10:57:37 2023","interfaces":[{"interface":"back","average":{"1min":3.629,"5min":3.4540000000000002,"15min":2.9790000000000001},"min":{"1min":0.49299999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":24.974,"5min":122.768,"15min":122.768},"last":5.7320000000000002},{"interface":"front","average":{"1min":4.056,"5min":3.746,"15min":3.1030000000000002},"min":{"1min":0.45800000000000002,"5min":0.36199999999999999,"15min":0.32100000000000001},"max":{"1min":25.763999999999999,"5min":123.02200000000001,"15min":123.02200000000001},"last":3.4870000000000001}]},{"osd":3,"last update":"Sun Oct 1 10:57:22 2023","interfaces":[{"interface":"back","average":{"1min":1.0449999999999999,"5min":1.9330000000000001,"15min":2.3029999999999999},"min":{"1min":0.31,"5min":0.31,"15min":0.30599999999999999},"max":{"1min":1.9379999999999999,"5min":36.698999999999998,"15min":61.817999999999998},"last":3.8170000000000002},{"interface":"front","average":{"1min":1.216,"5min":2.254,"15min":2.2890000000000001},"min":{"1min":0.44900000000000001,"5min":0.36499999999999999,"15min":0.315},"max":{"1min":3.2839999999999998,"5min":33.555999999999997,"15min":61.219000000000001},"last":4.4909999999999997}]},{"osd":4,"last update":"Sun Oct 1 10:58:03 2023","interfaces":[{"interface":"back","average":{"1min":0.69099999999999995,"5min":1.1379999999999999,"15min":1.532},"min":{"1min":0.35799999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":2.1509999999999998,"5min":7.5369999999999999,"15min":13.1},"last":0.59299999999999997},{"interface":"front","average":{"1min":1.8340000000000001,"5min":1.5609999999999999,"15min":1.6639999999999999},"min":{"1min":0.38700000000000001,"5min":0.35999999999999999,"15min":0.315},"max":{"1min":7.4509999999999996,"5min":8.016,"15min":12.157999999999999},"last":3.6499999999999999}]},{"osd":5,"last update":"Sun Oct 1 10:57:22 2023","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":1.4990000000000001,"15min":1.619},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.33000000000000002},"max":{"1min":1.8939999999999999,"5min":15.238,"15min":15.238},"last":0.63600000000000001},{"interface":"front","average":{"1min":1.0309999999999999,"5min":1.849,"15min":1.909},"min":{"1min":0.39800000000000002,"5min":0.36599999999999999,"15min":0.308},"max":{"1min":1.839,"5min":13.295,"15min":14.459},"last":0.53500000000000003}]},{"osd":7,"last update":"Sun Oct 1 10:57:11 2023","interfaces":[{"interface":"back","average":{"1min":1.0329999999999999,"5min":1.5089999999999999,"15min":1.6639999999999999},"min":{"1min":0.433,"5min":0.33900000000000002,"15min":0.311},"max":{"1min":1.4379999999999999,"5min":8.4199999999999999,"15min":13.276999999999999},"last":3.6309999999999998},{"interface":"front","average":{"1min":1.2310000000000001,"5min":1.637,"15min":1.73},"min":{"1min":0.60499999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":1.9330000000000001,"5min":7.1379999999999999,"15min":11.416},"last":0.79900000000000004}]}]},{"osd":1,"up_from":101,"seq":433791697206,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2654476,"kb_used_data":1995332,"kb_used_omap":4,"kb_used_meta":659131,"kb_avail":91090676,"statfs":{"total":95995035648,"available":93276852224,"internally_reserved":0,"allocated":2043219968,"data_stored":2042224813,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":674951054},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:57:29 2023","interfaces":[{"interface":"back","average":{"1min":0.82399999999999995,"5min":1.3480000000000001,"15min":1.377},"min":{"1min":0.28100000000000003,"5min":0.19900000000000001,"15min":0.19600000000000001},"max":{"1min":2.9039999999999999,"5min":12.888999999999999,"15min":37.058},"last":0.51600000000000001},{"interface":"front","average":{"1min":1.3080000000000001,"5min":1.548,"15min":1.792},"min":{"1min":0.20699999999999999,"5min":0.19700000000000001,"15min":0.192},"max":{"1min":5.1769999999999996,"5min":15.416,"15min":61.207999999999998},"last":1.5629999999999999}]},{"osd":2,"last update":"Sun Oct 1 10:57:47 2023","interfaces":[{"interface":"back","average":{"1min":2.0859999999999999,"5min":3.8090000000000002,"15min":2.448},"min":{"1min":0.376,"5min":0.25900000000000001,"15min":0.222},"max":{"1min":12.865,"5min":161.905,"15min":161.905},"last":0.60399999999999998},{"interface":"front","average":{"1min":1.736,"5min":3.8839999999999999,"15min":2.4569999999999999},"min":{"1min":0.29599999999999999,"5min":0.25700000000000001,"15min":0.23699999999999999},"max":{"1min":14.997,"5min":162.059,"15min":162.059},"last":1.522}]},{"osd":3,"last update":"Sun Oct 1 10:57:29 2023","interfaces":[{"interface":"back","average":{"1min":0.94299999999999995,"5min":2.9239999999999999,"15min":2.137},"min":{"1min":0.26300000000000001,"5min":0.255,"15min":0.20000000000000001},"max":{"1min":3.5750000000000002,"5min":143.73400000000001,"15min":143.73400000000001},"last":1.1559999999999999},{"interface":"front","average":{"1min":1.738,"5min":3.1019999999999999,"15min":2.359},"min":{"1min":0.34200000000000003,"5min":0.25900000000000001,"15min":0.23999999999999999},"max":{"1min":4.6340000000000003,"5min":143.00399999999999,"15min":143.00399999999999},"last":1.0509999999999999}]},{"osd":4,"last update":"Sun Oct 1 10:57:47 2023","interfaces":[{"interface":"back","average":{"1min":1.117,"5min":1.3140000000000001,"15min":1.5700000000000001},"min":{"1min":0.37,"5min":0.307,"15min":0.29199999999999998},"max":{"1min":6.827,"5min":8.593,"15min":61.588000000000001},"last":0.55400000000000005},{"interface":"front","average":{"1min":1.0640000000000001,"5min":1.306,"15min":1.744},"min":{"1min":0.40600000000000003,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":5.2069999999999999,"5min":8.7530000000000001,"15min":61.411000000000001},"last":1.595}]},{"osd":5,"last update":"Sun Oct 1 10:57:29 2023","interfaces":[{"interface":"back","average":{"1min":1.2929999999999999,"5min":1.4330000000000001,"15min":1.8420000000000001},"min":{"1min":0.49299999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":4.4939999999999998,"5min":9.1809999999999992,"15min":61.560000000000002},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.83899999999999997,"5min":1.2490000000000001,"15min":1.6899999999999999},"min":{"1min":0.379,"5min":0.379,"15min":0.35199999999999998},"max":{"1min":1.7050000000000001,"5min":8.7949999999999999,"15min":61.667999999999999},"last":1.6970000000000001}]},{"osd":6,"last update":"Sun Oct 1 10:57:29 2023","interfaces":[{"interface":"back","average":{"1min":0.96299999999999997,"5min":1.7150000000000001,"15min":1.8},"min":{"1min":0.46300000000000002,"5min":0.45600000000000002,"15min":0.371},"max":{"1min":2.956,"5min":8.8309999999999995,"15min":62.219999999999999},"last":1.1970000000000001},{"interface":"front","average":{"1min":1.1180000000000001,"5min":1.502,"15min":1.7929999999999999},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":5.226,"5min":8.8970000000000002,"15min":61.462000000000003},"last":1.6240000000000001}]},{"osd":7,"last update":"Sun Oct 1 10:57:29 2023","interfaces":[{"interface":"back","average":{"1min":1.097,"5min":1.325,"15min":1.8400000000000001},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":5.2409999999999997,"5min":9.7140000000000004,"15min":62.646999999999998},"last":1.72},{"interface":"front","average":{"1min":1.3380000000000001,"5min":1.6299999999999999,"15min":1.879},"min":{"1min":0.53300000000000003,"5min":0.436,"15min":0.38100000000000001},"max":{"1min":4.5519999999999996,"5min":9.1449999999999996,"15min":62.048000000000002},"last":0.42899999999999999}]}]},{"osd":0,"up_from":86,"seq":369367187816,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3147176,"kb_used_data":2363552,"kb_used_omap":5,"kb_used_meta":783610,"kb_avail":90597976,"statfs":{"total":95995035648,"available":92772327424,"internally_reserved":0,"allocated":2420277248,"data_stored":2419312670,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":802417274},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 10:57:26 2023","interfaces":[{"interface":"back","average":{"1min":1.0209999999999999,"5min":1.339,"15min":1.383},"min":{"1min":0.23499999999999999,"5min":0.23000000000000001,"15min":0.184},"max":{"1min":5.3620000000000001,"5min":6.7850000000000001,"15min":15.074},"last":3.6880000000000002},{"interface":"front","average":{"1min":1.3360000000000001,"5min":1.8129999999999999,"15min":1.6160000000000001},"min":{"1min":0.29599999999999999,"5min":0.20200000000000001,"15min":0.19700000000000001},"max":{"1min":4.1360000000000001,"5min":11.164999999999999,"15min":13.601000000000001},"last":0.497}]},{"osd":2,"last update":"Sun Oct 1 10:57:43 2023","interfaces":[{"interface":"back","average":{"1min":3.444,"5min":2.2829999999999999,"15min":1.76},"min":{"1min":0.45200000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":16.135999999999999,"5min":44.744999999999997,"15min":44.744999999999997},"last":5.46},{"interface":"front","average":{"1min":3.133,"5min":2.4820000000000002,"15min":1.8979999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.221},"max":{"1min":13.507999999999999,"5min":44.691000000000003,"15min":44.691000000000003},"last":5.4939999999999998}]},{"osd":3,"last update":"Sun Oct 1 10:57:26 2023","interfaces":[{"interface":"back","average":{"1min":1.2889999999999999,"5min":1.6240000000000001,"15min":1.581},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.26200000000000001},"max":{"1min":4.6429999999999998,"5min":7.4260000000000002,"15min":12.944000000000001},"last":13.632999999999999},{"interface":"front","average":{"1min":1.0449999999999999,"5min":1.466,"15min":1.5209999999999999},"min":{"1min":0.28799999999999998,"5min":0.246,"15min":0.245},"max":{"1min":6.1349999999999998,"5min":13.641,"15min":13.641},"last":10.173}]},{"osd":4,"last update":"Sun Oct 1 10:58:05 2023","interfaces":[{"interface":"back","average":{"1min":1.6599999999999999,"5min":1.304,"15min":1.5740000000000001},"min":{"1min":0.35799999999999998,"5min":0.34300000000000003,"15min":0.32000000000000001},"max":{"1min":9.2889999999999997,"5min":9.2889999999999997,"15min":11.379},"last":4.0970000000000004},{"interface":"front","average":{"1min":1.4890000000000001,"5min":1.4399999999999999,"15min":1.5369999999999999},"min":{"1min":0.32300000000000001,"5min":0.311,"15min":0.311},"max":{"1min":4.883,"5min":6.9080000000000004,"15min":11.362},"last":0.57199999999999995}]},{"osd":5,"last update":"Sun Oct 1 10:57:26 2023","interfaces":[{"interface":"back","average":{"1min":1.3120000000000001,"5min":1.512,"15min":1.7070000000000001},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":4.5990000000000002,"5min":7.3799999999999999,"15min":12.91},"last":5.524},{"interface":"front","average":{"1min":1.069,"5min":1.1870000000000001,"15min":1.5409999999999999},"min":{"1min":0.47499999999999998,"5min":0.41199999999999998,"15min":0.34699999999999998},"max":{"1min":3.544,"5min":4.5949999999999998,"15min":9.8620000000000001},"last":4.1900000000000004}]},{"osd":6,"last update":"Sun Oct 1 10:57:26 2023","interfaces":[{"interface":"back","average":{"1min":1.1319999999999999,"5min":1.3819999999999999,"15min":1.548},"min":{"1min":0.499,"5min":0.39800000000000002,"15min":0.36399999999999999},"max":{"1min":3.3530000000000002,"5min":6.7969999999999997,"15min":9.8279999999999994},"last":4.1289999999999996},{"interface":"front","average":{"1min":1.6699999999999999,"5min":1.661,"15min":1.669},"min":{"1min":0.378,"5min":0.37,"15min":0.36899999999999999},"max":{"1min":4.8419999999999996,"5min":7.3120000000000003,"15min":10.111000000000001},"last":0.63}]},{"osd":7,"last update":"Sun Oct 1 10:57:26 2023","interfaces":[{"interface":"back","average":{"1min":1.0880000000000001,"5min":1.583,"15min":1.6619999999999999},"min":{"1min":0.45400000000000001,"5min":0.45300000000000001,"15min":0.39700000000000002},"max":{"1min":3.5630000000000002,"5min":13.699,"15min":15.839},"last":4.1559999999999997},{"interface":"front","average":{"1min":1.6759999999999999,"5min":1.8700000000000001,"15min":1.718},"min":{"1min":0.40899999999999997,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":4.915,"5min":7.2679999999999998,"15min":8.9440000000000008},"last":0.59899999999999998}]}]},{"osd":2,"up_from":116,"seq":498216206591,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4405452,"kb_used_data":3903172,"kb_used_omap":7,"kb_used_meta":502264,"kb_avail":89339700,"statfs":{"total":95995035648,"available":91483852800,"internally_reserved":0,"allocated":3996848128,"data_stored":3995906998,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":514318417},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,3,3,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:57:46 2023","interfaces":[{"interface":"back","average":{"1min":7.3410000000000002,"5min":2.5760000000000001,"15min":2.4889999999999999},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.16900000000000001},"max":{"1min":104.42,"5min":104.42,"15min":147.87},"last":0.70199999999999996},{"interface":"front","average":{"1min":8.2739999999999991,"5min":2.9060000000000001,"15min":2.6549999999999998},"min":{"1min":0.42299999999999999,"5min":0.26300000000000001,"15min":0.19500000000000001},"max":{"1min":105.10899999999999,"5min":105.10899999999999,"15min":148.03100000000001},"last":4.3600000000000003}]},{"osd":1,"last update":"Sun Oct 1 10:57:46 2023","interfaces":[{"interface":"back","average":{"1min":1.744,"5min":1.5129999999999999,"15min":1.954},"min":{"1min":0.35399999999999998,"5min":0.23799999999999999,"15min":0.217},"max":{"1min":6.1529999999999996,"5min":7.9880000000000004,"15min":122.249},"last":4.3159999999999998},{"interface":"front","average":{"1min":1.4990000000000001,"5min":2.1080000000000001,"15min":2.2879999999999998},"min":{"1min":0.26500000000000001,"5min":0.245,"15min":0.20599999999999999},"max":{"1min":3.7989999999999999,"5min":11.535,"15min":121.495},"last":2.081}]},{"osd":3,"last update":"Sun Oct 1 10:57:11 2023","interfaces":[{"interface":"back","average":{"1min":1.21,"5min":1.631,"15min":2.222},"min":{"1min":0.36599999999999999,"5min":0.25800000000000001,"15min":0.20999999999999999},"max":{"1min":3.484,"5min":11.029999999999999,"15min":124.393},"last":0.62},{"interface":"front","average":{"1min":1.2549999999999999,"5min":1.4239999999999999,"15min":2.29},"min":{"1min":0.34599999999999997,"5min":0.221,"15min":0.20899999999999999},"max":{"1min":4.7990000000000004,"5min":8.5220000000000002,"15min":122.52200000000001},"last":4.4889999999999999}]},{"osd":4,"last update":"Sun Oct 1 10:57:57 2023","interfaces":[{"interface":"back","average":{"1min":1.9199999999999999,"5min":1.569,"15min":1.6899999999999999},"min":{"1min":0.45200000000000001,"5min":0.40400000000000003,"15min":0.36499999999999999},"max":{"1min":6.2830000000000004,"5min":7.9480000000000004,"15min":60.994},"last":4.4059999999999997},{"interface":"front","average":{"1min":1.4970000000000001,"5min":2.0169999999999999,"15min":1.9670000000000001},"min":{"1min":0.439,"5min":0.36599999999999999,"15min":0.30299999999999999},"max":{"1min":3.8610000000000002,"5min":11.574,"15min":63.012},"last":1.712}]},{"osd":5,"last update":"Sun Oct 1 10:57:46 2023","interfaces":[{"interface":"back","average":{"1min":1.9470000000000001,"5min":1.6339999999999999,"15min":1.8460000000000001},"min":{"1min":0.53200000000000003,"5min":0.36599999999999999,"15min":0.34699999999999998},"max":{"1min":6.3129999999999997,"5min":8.5440000000000005,"15min":61.021000000000001},"last":4.4640000000000004},{"interface":"front","average":{"1min":2.1539999999999999,"5min":2.3029999999999999,"15min":2.1000000000000001},"min":{"1min":0.47499999999999998,"5min":0.36499999999999999,"15min":0.34100000000000003},"max":{"1min":12.835000000000001,"5min":12.835000000000001,"15min":63.145000000000003},"last":1.75}]},{"osd":6,"last update":"Sun Oct 1 10:57:46 2023","interfaces":[{"interface":"back","average":{"1min":3.3420000000000001,"5min":2.4279999999999999,"15min":2.1200000000000001},"min":{"1min":0.52500000000000002,"5min":0.38200000000000001,"15min":0.32100000000000001},"max":{"1min":19.908000000000001,"5min":19.908000000000001,"15min":63.289000000000001},"last":1.798},{"interface":"front","average":{"1min":1.4970000000000001,"5min":1.7969999999999999,"15min":1.6879999999999999},"min":{"1min":0.42799999999999999,"5min":0.36599999999999999,"15min":0.35299999999999998},"max":{"1min":4.2229999999999999,"5min":9.2279999999999998,"15min":60.579000000000001},"last":0.67300000000000004}]},{"osd":7,"last update":"Sun Oct 1 10:57:46 2023","interfaces":[{"interface":"back","average":{"1min":1.6259999999999999,"5min":1.569,"15min":1.5860000000000001},"min":{"1min":0.51800000000000002,"5min":0.39900000000000002,"15min":0.34300000000000003},"max":{"1min":4.2560000000000002,"5min":8.6859999999999999,"15min":60.493000000000002},"last":0.64900000000000002},{"interface":"front","average":{"1min":2.1190000000000002,"5min":1.6870000000000001,"15min":1.839},"min":{"1min":0.65000000000000002,"5min":0.40100000000000002,"15min":0.36399999999999999},"max":{"1min":6.3769999999999998,"5min":7.9279999999999999,"15min":60.956000000000003},"last":4.4329999999999998}]}]},{"osd":3,"up_from":121,"seq":519691043063,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3663984,"kb_used_data":3077928,"kb_used_omap":2,"kb_used_meta":586045,"kb_avail":90081168,"statfs":{"total":95995035648,"available":92243116032,"internally_reserved":0,"allocated":3151798272,"data_stored":3150882507,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":600110133},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,1,6,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:57:17 2023","interfaces":[{"interface":"back","average":{"1min":1.075,"5min":1.3700000000000001,"15min":2.6000000000000001},"min":{"1min":0.19900000000000001,"5min":0.192,"15min":0.187},"max":{"1min":3.028,"5min":10.608000000000001,"15min":129.953},"last":3.3580000000000001},{"interface":"front","average":{"1min":1.036,"5min":1.7809999999999999,"15min":2.956},"min":{"1min":0.28199999999999997,"5min":0.189,"15min":0.17299999999999999},"max":{"1min":3.1789999999999998,"5min":11.624000000000001,"15min":129.53999999999999},"last":0.23300000000000001}]},{"osd":1,"last update":"Sun Oct 1 10:57:17 2023","interfaces":[{"interface":"back","average":{"1min":0.96799999999999997,"5min":1.6619999999999999,"15min":1.875},"min":{"1min":0.249,"5min":0.249,"15min":0.193},"max":{"1min":2.855,"5min":10.574999999999999,"15min":77.361000000000004},"last":3.4100000000000001},{"interface":"front","average":{"1min":1.325,"5min":1.8069999999999999,"15min":2.0310000000000001},"min":{"1min":0.27200000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":5.5019999999999998,"5min":13.529999999999999,"15min":77.867000000000004},"last":3.8050000000000002}]},{"osd":2,"last update":"Sun Oct 1 10:57:17 2023","interfaces":[{"interface":"back","average":{"1min":1.423,"5min":1.9430000000000001,"15min":1.546},"min":{"1min":0.28100000000000003,"5min":0.217,"15min":0.217},"max":{"1min":6.0330000000000004,"5min":15.148,"15min":15.148},"last":3.4569999999999999},{"interface":"front","average":{"1min":1.3320000000000001,"5min":1.923,"15min":1.7430000000000001},"min":{"1min":0.59399999999999997,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":4.6459999999999999,"5min":10.66,"15min":75.299999999999997},"last":5.4299999999999997}]},{"osd":4,"last update":"Sun Oct 1 10:58:01 2023","interfaces":[{"interface":"back","average":{"1min":1.2250000000000001,"5min":1.4730000000000001,"15min":1.655},"min":{"1min":0.49199999999999999,"5min":0.36799999999999999,"15min":0.34599999999999997},"max":{"1min":4.6269999999999998,"5min":10.724,"15min":75.388999999999996},"last":3.5470000000000002},{"interface":"front","average":{"1min":1.4850000000000001,"5min":1.6339999999999999,"15min":1.8},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.29999999999999999},"max":{"1min":7.6639999999999997,"5min":13.391999999999999,"15min":75.165999999999997},"last":0.68000000000000005}]},{"osd":5,"last update":"Sun Oct 1 10:57:17 2023","interfaces":[{"interface":"back","average":{"1min":1.1579999999999999,"5min":1.655,"15min":1.869},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.33100000000000002},"max":{"1min":3.0819999999999999,"5min":13.333,"15min":75.221000000000004},"last":0.61699999999999999},{"interface":"front","average":{"1min":1.532,"5min":1.669,"15min":2.0449999999999999},"min":{"1min":0.41999999999999998,"5min":0.38900000000000001,"15min":0.33200000000000002},"max":{"1min":5.2789999999999999,"5min":12.335000000000001,"15min":75.231999999999999},"last":3.5750000000000002}]},{"osd":6,"last update":"Sun Oct 1 10:57:17 2023","interfaces":[{"interface":"back","average":{"1min":1.516,"5min":2.0179999999999998,"15min":2.0710000000000002},"min":{"1min":0.435,"5min":0.435,"15min":0.34999999999999998},"max":{"1min":4.4480000000000004,"5min":8.8740000000000006,"15min":75.257000000000005},"last":0.65200000000000002},{"interface":"front","average":{"1min":1.954,"5min":1.8700000000000001,"15min":2.198},"min":{"1min":0.38700000000000001,"5min":0.36799999999999999,"15min":0.35899999999999999},"max":{"1min":5.2549999999999999,"5min":12.378,"15min":75.194000000000003},"last":3.6200000000000001}]},{"osd":7,"last update":"Sun Oct 1 10:57:17 2023","interfaces":[{"interface":"back","average":{"1min":1.913,"5min":1.845,"15min":2.0270000000000001},"min":{"1min":0.44500000000000001,"5min":0.40999999999999998,"15min":0.35099999999999998},"max":{"1min":6.0730000000000004,"5min":12.416,"15min":75.138999999999996},"last":3.488},{"interface":"front","average":{"1min":1.423,"5min":1.7350000000000001,"15min":1.881},"min":{"1min":0.45500000000000002,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":2.9780000000000002,"5min":10.755000000000001,"15min":75.355000000000004},"last":3.601}]}]},{"osd":4,"up_from":126,"seq":541165879537,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1585748,"kb_used_data":1162380,"kb_used_omap":10,"kb_used_meta":423349,"kb_avail":92159404,"statfs":{"total":95995035648,"available":94371229696,"internally_reserved":0,"allocated":1190277120,"data_stored":1189327891,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":433509988},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:57:54 2023","interfaces":[{"interface":"back","average":{"1min":2.6179999999999999,"5min":1.3779999999999999,"15min":1.962},"min":{"1min":0.313,"5min":0.27300000000000002,"15min":0.24399999999999999},"max":{"1min":35.741999999999997,"5min":35.741999999999997,"15min":62.576000000000001},"last":0.377},{"interface":"front","average":{"1min":2.823,"5min":1.5549999999999999,"15min":1.9990000000000001},"min":{"1min":0.28799999999999998,"5min":0.26200000000000001,"15min":0.251},"max":{"1min":35.701000000000001,"5min":35.701000000000001,"15min":64.941000000000003},"last":0.42199999999999999}]},{"osd":1,"last update":"Sun Oct 1 10:57:54 2023","interfaces":[{"interface":"back","average":{"1min":1.339,"5min":1.383,"15min":1.893},"min":{"1min":0.312,"5min":0.29199999999999998,"15min":0.27300000000000002},"max":{"1min":5.3760000000000003,"5min":11.664,"15min":97.622},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.94099999999999995,"5min":1.298,"15min":2.0110000000000001},"min":{"1min":0.35899999999999999,"5min":0.33900000000000002,"15min":0.28699999999999998},"max":{"1min":3.9460000000000002,"5min":19.719999999999999,"15min":97.346999999999994},"last":0.51500000000000001}]},{"osd":2,"last update":"Sun Oct 1 10:57:54 2023","interfaces":[{"interface":"back","average":{"1min":6.1040000000000001,"5min":3.1760000000000002,"15min":2.5779999999999998},"min":{"1min":0.375,"5min":0.375,"15min":0.29899999999999999},"max":{"1min":66.596999999999994,"5min":68.064999999999998,"15min":117.229},"last":2.5910000000000002},{"interface":"front","average":{"1min":6.1210000000000004,"5min":3.21,"15min":2.5089999999999999},"min":{"1min":0.35399999999999998,"5min":0.33300000000000002,"15min":0.29699999999999999},"max":{"1min":66.412000000000006,"5min":68.379000000000005,"15min":116.922},"last":0.60199999999999998}]},{"osd":3,"last update":"Sun Oct 1 10:57:54 2023","interfaces":[{"interface":"back","average":{"1min":1.8720000000000001,"5min":1.5489999999999999,"15min":1.98},"min":{"1min":0.32700000000000001,"5min":0.307,"15min":0.29499999999999998},"max":{"1min":9.1780000000000008,"5min":9.1780000000000008,"15min":61.398000000000003},"last":0.52900000000000003},{"interface":"front","average":{"1min":7.9169999999999998,"5min":2.8260000000000001,"15min":2.3660000000000001},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":130.88300000000001,"5min":130.88300000000001,"15min":130.88300000000001},"last":0.61699999999999999}]},{"osd":5,"last update":"Sun Oct 1 10:57:54 2023","interfaces":[{"interface":"back","average":{"1min":0.86799999999999999,"5min":2.157,"15min":1.8049999999999999},"min":{"1min":0.38200000000000001,"5min":0.34899999999999998,"15min":0.30199999999999999},"max":{"1min":2.7370000000000001,"5min":84.307000000000002,"15min":84.307000000000002},"last":0.63800000000000001},{"interface":"front","average":{"1min":1.1259999999999999,"5min":2.3250000000000002,"15min":1.8799999999999999},"min":{"1min":0.40500000000000003,"5min":0.309,"15min":0.28999999999999998},"max":{"1min":3.8460000000000001,"5min":84.234999999999999,"15min":84.234999999999999},"last":0.75900000000000001}]},{"osd":6,"last update":"Sun Oct 1 10:57:54 2023","interfaces":[{"interface":"back","average":{"1min":0.84099999999999997,"5min":1.3049999999999999,"15min":1.6419999999999999},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.33800000000000002},"max":{"1min":2.3399999999999999,"5min":9.0109999999999992,"15min":13.646000000000001},"last":0.80900000000000005},{"interface":"front","average":{"1min":0.98099999999999998,"5min":1.204,"15min":1.603},"min":{"1min":0.42099999999999999,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":3.778,"5min":7.5510000000000002,"15min":14.446999999999999},"last":0.65800000000000003}]},{"osd":7,"last update":"Sun Oct 1 10:57:54 2023","interfaces":[{"interface":"back","average":{"1min":1.2010000000000001,"5min":1.3300000000000001,"15min":1.9179999999999999},"min":{"1min":0.43099999999999999,"5min":0.36699999999999999,"15min":0.33900000000000002},"max":{"1min":3.823,"5min":8.2289999999999992,"15min":119.56100000000001},"last":1.542},{"interface":"front","average":{"1min":1.1339999999999999,"5min":1.1579999999999999,"15min":2.0579999999999998},"min":{"1min":0.439,"5min":0.40300000000000002,"15min":0.30399999999999999},"max":{"1min":3.4750000000000001,"5min":5.5229999999999997,"15min":120.44499999999999},"last":1.3959999999999999}]}]},{"osd":5,"up_from":36,"seq":154618823218,"num_pgs":23,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1687356,"kb_used_data":791284,"kb_used_omap":0,"kb_used_meta":896064,"kb_avail":92057796,"statfs":{"total":95995035648,"available":94267183104,"internally_reserved":0,"allocated":810274816,"data_stored":809752174,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":917569536},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:57:15 2023","interfaces":[{"interface":"back","average":{"1min":0.876,"5min":2.3860000000000001,"15min":1.974},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24099999999999999},"max":{"1min":2.5870000000000002,"5min":123.39700000000001,"15min":123.39700000000001},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.83999999999999997,"5min":2.5720000000000001,"15min":2.2189999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":1.8049999999999999,"5min":124.051,"15min":124.051},"last":3.7669999999999999}]},{"osd":1,"last update":"Sun Oct 1 10:57:15 2023","interfaces":[{"interface":"back","average":{"1min":0.88400000000000001,"5min":1.3089999999999999,"15min":2.331},"min":{"1min":0.30499999999999999,"5min":0.28799999999999998,"15min":0.27100000000000002},"max":{"1min":2.0270000000000001,"5min":9.766,"15min":202.96600000000001},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.81499999999999995,"5min":1.415,"15min":2.633},"min":{"1min":0.38200000000000001,"5min":0.32500000000000001,"15min":0.28100000000000003},"max":{"1min":1.6990000000000001,"5min":7.7290000000000001,"15min":203.05099999999999},"last":1.2270000000000001}]},{"osd":2,"last update":"Sun Oct 1 10:57:52 2023","interfaces":[{"interface":"back","average":{"1min":1.704,"5min":1.8440000000000001,"15min":1.6799999999999999},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.29199999999999998},"max":{"1min":10.510999999999999,"5min":26.242000000000001,"15min":26.242000000000001},"last":0.52300000000000002},{"interface":"front","average":{"1min":1.661,"5min":1.784,"15min":1.956},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":10.826000000000001,"5min":25.643999999999998,"15min":25.643999999999998},"last":0.499}]},{"osd":3,"last update":"Sun Oct 1 10:57:15 2023","interfaces":[{"interface":"back","average":{"1min":1.081,"5min":3.1309999999999998,"15min":3.024},"min":{"1min":0.42299999999999999,"5min":0.32400000000000001,"15min":0.316},"max":{"1min":2.8450000000000002,"5min":166.29599999999999,"15min":166.29599999999999},"last":0.53700000000000003},{"interface":"front","average":{"1min":1.4359999999999999,"5min":3.488,"15min":3.044},"min":{"1min":0.50900000000000001,"5min":0.34799999999999998,"15min":0.34699999999999998},"max":{"1min":4.9320000000000004,"5min":166.565,"15min":166.565},"last":0.374}]},{"osd":4,"last update":"Sun Oct 1 10:57:52 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":1.131,"15min":1.4730000000000001},"min":{"1min":0.42699999999999999,"5min":0.40699999999999997,"15min":0.29399999999999998},"max":{"1min":0.97299999999999998,"5min":6.6349999999999998,"15min":15.170999999999999},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.76600000000000001,"5min":1.145,"15min":1.48},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":2.0840000000000001,"5min":9.1699999999999999,"15min":9.7270000000000003},"last":0.58299999999999996}]},{"osd":6,"last update":"Sun Oct 1 10:57:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0089999999999999,"5min":1.492,"15min":1.8480000000000001},"min":{"1min":0.38600000000000001,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":2.1230000000000002,"5min":9.0790000000000006,"15min":61.768999999999998},"last":0.64200000000000002},{"interface":"front","average":{"1min":1.167,"5min":1.3440000000000001,"15min":1.9419999999999999},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.28899999999999998},"max":{"1min":3.5099999999999998,"5min":5.9690000000000003,"15min":62.396999999999998},"last":0.68600000000000005}]},{"osd":7,"last update":"Sun Oct 1 10:57:15 2023","interfaces":[{"interface":"back","average":{"1min":0.98999999999999999,"5min":1.4159999999999999,"15min":1.8400000000000001},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.32800000000000001},"max":{"1min":1.887,"5min":9.1470000000000002,"15min":13.673999999999999},"last":1.3720000000000001},{"interface":"front","average":{"1min":1.1779999999999999,"5min":1.3720000000000001,"15min":1.7230000000000001},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":2.375,"5min":9.1340000000000003,"15min":11.635},"last":0.85999999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2419851264,"data_stored":2419851264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2047737856,"data_stored":2047737856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4015980544,"data_stored":4015980544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3167420416,"data_stored":3167420416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1194459136,"data_stored":1194459136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":810680320,"data_stored":810680320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2798387200,"data_stored":2798387200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2796814336,"data_stored":2796814336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T10:58:16.962 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_7 pgp_num 16 2023-10-01T10:58:16.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:16.963 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:16.964 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:58:17.055 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Total time run: 90.0138 2023-10-01T10:58:17.055 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Total writes made: 103128 2023-10-01T10:58:17.055 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Write size: 65536 2023-10-01T10:58:17.055 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Object size: 65536 2023-10-01T10:58:17.055 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Bandwidth (MB/sec): 71.6057 2023-10-01T10:58:17.055 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Stddev Bandwidth: 26.5816 2023-10-01T10:58:17.055 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Max bandwidth (MB/sec): 158.562 2023-10-01T10:58:17.056 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Min bandwidth (MB/sec): 36.5625 2023-10-01T10:58:17.056 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Average IOPS: 1145 2023-10-01T10:58:17.056 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Stddev IOPS: 425.306 2023-10-01T10:58:17.056 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Max IOPS: 2537 2023-10-01T10:58:17.056 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Min IOPS: 585 2023-10-01T10:58:17.056 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Average Latency(s): 0.0139608 2023-10-01T10:58:17.056 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Stddev Latency(s): 0.0139469 2023-10-01T10:58:17.056 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Max latency(s): 0.325279 2023-10-01T10:58:17.056 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Min latency(s): 0.00107177 2023-10-01T10:58:17.058 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Cleaning up (deleting benchmark objects) 2023-10-01T10:58:17.065 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:17.074 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:16 smithi173 bash[58816]: cluster 2023-10-01T10:58:15.517143+0000 osd.7 (osd.7) 231 : cluster 3 failed to encode map e247 with expected crc 2023-10-01T10:58:17.074 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:16 smithi173 bash[58816]: cluster 2023-10-01T10:58:15.519014+0000 osd.7 (osd.7) 232 : cluster 3 failed to encode map e247 with expected crc 2023-10-01T10:58:17.074 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:16 smithi173 bash[58816]: cluster 2023-10-01T10:58:15.521659+0000 osd.6 (osd.6) 190 : cluster 3 failed to encode map e247 with expected crc 2023-10-01T10:58:17.074 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:16 smithi173 bash[58816]: cluster 2023-10-01T10:58:15.523212+0000 osd.6 (osd.6) 191 : cluster 3 failed to encode map e247 with expected crc 2023-10-01T10:58:17.074 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:16 smithi173 bash[58816]: cluster 2023-10-01T10:58:15.548845+0000 osd.5 (osd.5) 229 : cluster 3 failed to encode map e247 with expected crc 2023-10-01T10:58:17.095 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:17.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:17.101 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:58:17.202 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:17.243 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:17.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:17.247 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:58:17.347 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:17.373 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:17.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:17.380 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:58:17.481 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:17.514 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:17.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:17.519 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:58:17.620 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:17.653 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:17.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:17.655 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:58:17.755 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:17.779 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:17.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:17.782 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:58:17.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:17 smithi078 bash[75299]: cluster 2023-10-01T10:58:16.113199+0000 mgr.y (mgr.25072) 1225 : cluster 0 pgmap v1238: 89 pgs: 89 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 49 MiB/s wr, 785 op/s 2023-10-01T10:58:17.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:17 smithi078 bash[75299]: audit 2023-10-01T10:58:17.456849+0000 mon.a (mon.0) 1530 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:58:17.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:17 smithi078 bash[75299]: audit 2023-10-01T10:58:17.458205+0000 mon.a (mon.0) 1531 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:58:17.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:17 smithi078 bash[78274]: cluster 2023-10-01T10:58:16.113199+0000 mgr.y (mgr.25072) 1225 : cluster 0 pgmap v1238: 89 pgs: 89 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 49 MiB/s wr, 785 op/s 2023-10-01T10:58:17.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:17 smithi078 bash[78274]: audit 2023-10-01T10:58:17.456849+0000 mon.a (mon.0) 1530 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:58:17.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:17 smithi078 bash[78274]: audit 2023-10-01T10:58:17.458205+0000 mon.a (mon.0) 1531 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:58:17.882 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:17.909 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:17.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:17.914 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:58:18.015 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:18.048 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:18.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:18.055 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:58:18.074 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:17 smithi173 bash[58816]: cluster 2023-10-01T10:58:16.113199+0000 mgr.y (mgr.25072) 1225 : cluster 0 pgmap v1238: 89 pgs: 89 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 49 MiB/s wr, 785 op/s 2023-10-01T10:58:18.075 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:17 smithi173 bash[58816]: audit 2023-10-01T10:58:17.456849+0000 mon.a (mon.0) 1530 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:58:18.075 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:17 smithi173 bash[58816]: audit 2023-10-01T10:58:17.458205+0000 mon.a (mon.0) 1531 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:58:18.155 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:18.193 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:18.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:18.198 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:58:18.299 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:18.324 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:18.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:18.329 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:58:18.429 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:18.453 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:18.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:18.456 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:58:18.558 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:18.586 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:18.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:18.588 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:58:18.689 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:18.718 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:18.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:18.720 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:58:18.821 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:18.849 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:18.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:18.854 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:58:18.955 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:18.988 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:18.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:18.991 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:58:19.092 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:19.119 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:19.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:19.125 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:58:19.226 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:19.253 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:19.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:19.258 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:58:19.359 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:19.381 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:19.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:19.383 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:58:19.483 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:19.513 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:19.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:19.514 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:58:19.615 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:19.646 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:19.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:19.651 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:58:19.752 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:19.794 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:19.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:19.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:58:19.904 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:19.938 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:19.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:19.942 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:58:20.008 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T10:58:20.016 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:19 smithi078 bash[75299]: cluster 2023-10-01T10:58:18.113671+0000 mgr.y (mgr.25072) 1226 : cluster 0 pgmap v1239: 89 pgs: 89 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 46 MiB/s wr, 745 op/s 2023-10-01T10:58:20.016 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:19 smithi078 bash[78274]: cluster 2023-10-01T10:58:18.113671+0000 mgr.y (mgr.25072) 1226 : cluster 0 pgmap v1239: 89 pgs: 89 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 46 MiB/s wr, 745 op/s 2023-10-01T10:58:20.031 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T10:58:20.043 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:20.050 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:19 smithi173 bash[58816]: cluster 2023-10-01T10:58:18.113671+0000 mgr.y (mgr.25072) 1226 : cluster 0 pgmap v1239: 89 pgs: 89 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 46 MiB/s wr, 745 op/s 2023-10-01T10:58:20.062 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:20.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:20.064 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:58:20.165 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:20.190 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:20.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:20.196 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:58:20.269 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:58:20 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:58:20] "GET /metrics HTTP/1.1" 200 36382 "" "Prometheus/2.33.4" 2023-10-01T10:58:20.296 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:20.353 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:20.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:20.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:58:20.456 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:20.488 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:20.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:20.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:58:20.586 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-10-01T10:58:20.587 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T10:58:20.593 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:20.626 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:20.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:20.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:58:20.734 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:20.758 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:20.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:20.760 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:58:20.861 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:20.896 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:20.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:20.902 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:58:21.002 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:21.046 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:21.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:21.052 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:58:21.153 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:21.183 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:21.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:21.188 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:58:21.288 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:21.319 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:21.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:21.320 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:58:21.421 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:21.452 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:21.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:21.456 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:58:21.546 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-01T10:58:21.557 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:21.620 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:21.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:21.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:58:21.729 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:21.763 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:21.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:21.768 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:58:21.845 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:58:21.869 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:21.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:21 smithi173 bash[58816]: cluster 2023-10-01T10:58:20.116308+0000 mgr.y (mgr.25072) 1227 : cluster 0 pgmap v1240: 89 pgs: 89 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 34 MiB/s wr, 647 op/s 2023-10-01T10:58:21.904 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:21.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:21.907 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:58:22.007 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:22.036 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:22.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:22.038 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:58:22.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:21 smithi078 bash[78274]: cluster 2023-10-01T10:58:20.116308+0000 mgr.y (mgr.25072) 1227 : cluster 0 pgmap v1240: 89 pgs: 89 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 34 MiB/s wr, 647 op/s 2023-10-01T10:58:22.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:21 smithi078 bash[75299]: cluster 2023-10-01T10:58:20.116308+0000 mgr.y (mgr.25072) 1227 : cluster 0 pgmap v1240: 89 pgs: 89 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 34 MiB/s wr, 647 op/s 2023-10-01T10:58:22.139 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:22.170 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:22.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:22.176 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:58:22.277 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:22.305 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:22.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:22.307 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:58:22.408 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:22.433 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:22.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:22.439 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:58:22.539 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:22.566 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:22.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:22.572 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:58:22.673 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:22.702 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:22.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:22.707 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:58:22.808 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:22.829 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:22.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:22.831 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:58:22.932 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:22.963 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:22.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:22.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:58:23.065 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:23.086 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:23.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:23.088 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:58:23.189 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:23.214 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:23.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:23.219 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:58:23.320 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:23.346 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:23.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:23.353 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:58:23.454 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:23.942 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:23.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:23.947 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:58:24.048 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:24.058 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:23 smithi078 bash[75299]: cluster 2023-10-01T10:58:22.116791+0000 mgr.y (mgr.25072) 1228 : cluster 0 pgmap v1241: 89 pgs: 89 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 34 MiB/s wr, 647 op/s 2023-10-01T10:58:24.059 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:23 smithi078 bash[78274]: cluster 2023-10-01T10:58:22.116791+0000 mgr.y (mgr.25072) 1228 : cluster 0 pgmap v1241: 89 pgs: 89 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 34 MiB/s wr, 647 op/s 2023-10-01T10:58:24.085 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:24.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:24.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:58:24.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:23 smithi173 bash[58816]: cluster 2023-10-01T10:58:22.116791+0000 mgr.y (mgr.25072) 1228 : cluster 0 pgmap v1241: 89 pgs: 89 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 34 MiB/s wr, 647 op/s 2023-10-01T10:58:24.188 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:24.214 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:24.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:24.216 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:58:24.317 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:24.350 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:24.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:24.353 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:58:24.454 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:24.480 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:24.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:24.482 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:58:24.582 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:24.607 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:24.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:24.611 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:58:24.712 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:24.736 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:24.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:24.738 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:58:24.838 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:24.864 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:24.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:24.866 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:58:24.895 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:58:24.899 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:58:24.967 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:25.007 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:25.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:25.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:58:25.111 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:25.133 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:25.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:25.138 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:58:25.238 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:25.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:25.282 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:25.283 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:58:25.383 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:25.414 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:25.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:25.418 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:58:25.519 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:25.545 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:25.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:25.547 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:58:25.648 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:25.678 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:25.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:25.684 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:58:25.781 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:25 smithi173 bash[58816]: cluster 2023-10-01T10:58:24.117488+0000 mgr.y (mgr.25072) 1229 : cluster 0 pgmap v1242: 89 pgs: 89 active+clean; 6.0 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 19 MiB/s wr, 749 op/s 2023-10-01T10:58:25.784 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:25.806 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:25.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:25.809 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:58:25.910 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:25.922 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:25 smithi078 bash[75299]: cluster 2023-10-01T10:58:24.117488+0000 mgr.y (mgr.25072) 1229 : cluster 0 pgmap v1242: 89 pgs: 89 active+clean; 6.0 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 19 MiB/s wr, 749 op/s 2023-10-01T10:58:25.923 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:25 smithi078 bash[78274]: cluster 2023-10-01T10:58:24.117488+0000 mgr.y (mgr.25072) 1229 : cluster 0 pgmap v1242: 89 pgs: 89 active+clean; 6.0 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 19 MiB/s wr, 749 op/s 2023-10-01T10:58:25.943 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:25.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:25.951 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:58:26.052 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:26.075 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:26.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:26.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:58:26.180 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:26.209 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:26.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:26.219 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:58:26.320 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:26.344 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:26.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:26.348 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:58:26.358 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T10:58:26.449 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:26.481 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:26.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:26.490 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:58:26.591 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:26.616 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:26.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:26.620 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:58:26.721 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:26.744 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:26.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:26.747 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:58:26.847 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:26.885 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:26.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:26.888 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:58:26.989 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:27.012 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:27.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:27.014 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:58:27.115 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:27.142 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:27.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:27.145 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:58:27.246 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:27.270 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:27.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:27.275 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:58:27.376 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:27.404 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:27.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:27.411 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:58:27.512 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:27.534 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:27.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:27.537 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:58:27.638 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:27.681 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:27.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:27.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:58:27.785 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:27.790 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:27 smithi173 bash[58816]: cluster 2023-10-01T10:58:26.117900+0000 mgr.y (mgr.25072) 1230 : cluster 0 pgmap v1243: 89 pgs: 89 active+clean; 6.0 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 96 B/s rd, 9.4 MiB/s wr, 625 op/s 2023-10-01T10:58:27.812 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:27.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:27.814 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:58:27.915 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:27.942 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:27.943 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:27 smithi078 bash[78274]: cluster 2023-10-01T10:58:26.117900+0000 mgr.y (mgr.25072) 1230 : cluster 0 pgmap v1243: 89 pgs: 89 active+clean; 6.0 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 96 B/s rd, 9.4 MiB/s wr, 625 op/s 2023-10-01T10:58:27.943 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:27 smithi078 bash[75299]: cluster 2023-10-01T10:58:26.117900+0000 mgr.y (mgr.25072) 1230 : cluster 0 pgmap v1243: 89 pgs: 89 active+clean; 6.0 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 96 B/s rd, 9.4 MiB/s wr, 625 op/s 2023-10-01T10:58:27.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:27.947 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:58:28.047 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:28.077 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:28.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:28.082 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:58:28.183 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:28.213 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:28.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:28.215 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:58:28.316 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:28.343 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:28.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:28.346 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:58:28.446 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:28.477 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:28.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:28.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:58:28.586 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:28.606 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:28.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:28.609 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:58:28.710 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:28.737 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:28.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:28.739 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:58:28.840 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:28.872 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:28.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:28.874 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:58:28.975 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:28.996 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:28.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:28.998 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:58:29.099 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:29.122 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:29.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:29.128 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:58:29.229 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:29.252 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:29.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:29.254 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:58:29.355 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:29.384 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:29.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:29.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:58:29.490 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:29.520 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:29.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:29.521 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:58:29.622 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:29.647 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:29.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:29.649 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:58:29.750 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:29.775 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:29.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:29.779 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:58:29.879 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:29.913 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:29.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:29.916 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:58:30.017 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:30.026 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:29 smithi078 bash[75299]: cluster 2023-10-01T10:58:28.118389+0000 mgr.y (mgr.25072) 1231 : cluster 0 pgmap v1244: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 8.3 MiB/s wr, 635 op/s 2023-10-01T10:58:30.026 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:29 smithi078 bash[78274]: cluster 2023-10-01T10:58:28.118389+0000 mgr.y (mgr.25072) 1231 : cluster 0 pgmap v1244: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 8.3 MiB/s wr, 635 op/s 2023-10-01T10:58:30.060 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:30.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:30.063 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:58:30.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:29 smithi173 bash[58816]: cluster 2023-10-01T10:58:28.118389+0000 mgr.y (mgr.25072) 1231 : cluster 0 pgmap v1244: 89 pgs: 89 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 8.3 MiB/s wr, 635 op/s 2023-10-01T10:58:30.164 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:30.193 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:30.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:30.199 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:58:30.299 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:30.308 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:58:30 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:58:30] "GET /metrics HTTP/1.1" 200 36382 "" "Prometheus/2.33.4" 2023-10-01T10:58:30.328 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:30.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:30.332 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:58:30.433 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:30.458 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:30.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:30.461 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:58:30.561 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:30.585 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:30.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:30.590 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:58:30.691 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:30.718 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:30.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:30.723 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:58:30.763 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-10-01T10:58:30.763 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T10:58:30.824 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:30.859 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:30.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:30.864 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:58:30.965 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:30.989 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:30.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:30.991 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:58:31.092 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:31.114 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:31.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:31.119 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:58:31.220 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:31.255 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:31.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:31.259 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:58:31.360 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:31.393 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:31.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:31.396 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:58:31.497 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:31.541 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:31.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:31.543 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:58:31.643 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:31.678 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:31.680 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-01T10:58:31.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:31.686 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:58:31.787 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:31.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:31.829 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:31.829 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:58:31.930 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:31.954 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:31.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:31.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:58:32.059 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:32.069 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:31 smithi078 bash[75299]: cluster 2023-10-01T10:58:30.119197+0000 mgr.y (mgr.25072) 1232 : cluster 0 pgmap v1245: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 5.7 MiB/s wr, 820 op/s 2023-10-01T10:58:32.069 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:31 smithi078 bash[78274]: cluster 2023-10-01T10:58:30.119197+0000 mgr.y (mgr.25072) 1232 : cluster 0 pgmap v1245: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 5.7 MiB/s wr, 820 op/s 2023-10-01T10:58:32.092 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:32.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:32.099 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:58:32.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:31 smithi173 bash[58816]: cluster 2023-10-01T10:58:30.119197+0000 mgr.y (mgr.25072) 1232 : cluster 0 pgmap v1245: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 5.7 MiB/s wr, 820 op/s 2023-10-01T10:58:32.200 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:32.239 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:32.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:32.247 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:58:32.348 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:32.394 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:32.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:32.397 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:58:32.498 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:32.523 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:32.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:32.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:58:32.627 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:32.659 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:32.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:32.661 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:58:32.762 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:32.785 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:32.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:32.789 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:58:32.822 INFO:teuthology.orchestra.run.smithi078.stderr:set pool 12 pgp_num to 16 2023-10-01T10:58:32.890 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:32.920 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:32.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:32.925 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:58:33.026 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:33.053 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:33.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:33.056 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:58:33.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:32 smithi078 bash[75299]: audit 2023-10-01T10:58:31.825749+0000 mon.a (mon.0) 1532 : audit 1 from='client.? 172.21.15.78:0/2417684448' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-01T10:58:33.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:32 smithi078 bash[75299]: audit 2023-10-01T10:58:32.448500+0000 mon.a (mon.0) 1533 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:58:33.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:32 smithi078 bash[78274]: audit 2023-10-01T10:58:31.825749+0000 mon.a (mon.0) 1532 : audit 1 from='client.? 172.21.15.78:0/2417684448' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-01T10:58:33.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:32 smithi078 bash[78274]: audit 2023-10-01T10:58:32.448500+0000 mon.a (mon.0) 1533 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:58:33.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:32 smithi173 bash[58816]: audit 2023-10-01T10:58:31.825749+0000 mon.a (mon.0) 1532 : audit 1 from='client.? 172.21.15.78:0/2417684448' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-01T10:58:33.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:32 smithi173 bash[58816]: audit 2023-10-01T10:58:32.448500+0000 mon.a (mon.0) 1533 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:58:33.157 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:33.184 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:33.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:33.190 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:58:33.292 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:33.317 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:33.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:33.323 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:58:33.424 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:33.449 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:33.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:33.453 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:58:33.553 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:33.581 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:33.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:33.584 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:58:33.684 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:33.707 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:33.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:33.711 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:58:33.811 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:33.843 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:33.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:33.847 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:58:33.949 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:33.958 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:33 smithi078 bash[78274]: cluster 2023-10-01T10:58:32.119621+0000 mgr.y (mgr.25072) 1233 : cluster 0 pgmap v1246: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 649 op/s 2023-10-01T10:58:33.958 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:33 smithi078 bash[78274]: cluster 2023-10-01T10:58:32.808243+0000 osd.7 (osd.7) 233 : cluster 3 failed to encode map e248 with expected crc 2023-10-01T10:58:33.958 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:33 smithi078 bash[78274]: cluster 2023-10-01T10:58:32.810177+0000 osd.5 (osd.5) 230 : cluster 3 failed to encode map e248 with expected crc 2023-10-01T10:58:33.958 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:33 smithi078 bash[78274]: audit 2023-10-01T10:58:32.812362+0000 mon.a (mon.0) 1534 : audit 1 from='client.? 172.21.15.78:0/2417684448' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num", "val": "16"}]': finished 2023-10-01T10:58:33.958 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:33 smithi078 bash[78274]: cluster 2023-10-01T10:58:32.812439+0000 mon.a (mon.0) 1535 : cluster 0 osdmap e248: 8 total, 8 up, 8 in 2023-10-01T10:58:33.958 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:33 smithi078 bash[78274]: cluster 2023-10-01T10:58:32.819941+0000 osd.6 (osd.6) 192 : cluster 3 failed to encode map e248 with expected crc 2023-10-01T10:58:33.958 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:33 smithi078 bash[75299]: cluster 2023-10-01T10:58:32.119621+0000 mgr.y (mgr.25072) 1233 : cluster 0 pgmap v1246: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 649 op/s 2023-10-01T10:58:33.959 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:33 smithi078 bash[75299]: cluster 2023-10-01T10:58:32.808243+0000 osd.7 (osd.7) 233 : cluster 3 failed to encode map e248 with expected crc 2023-10-01T10:58:33.959 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:33 smithi078 bash[75299]: cluster 2023-10-01T10:58:32.810177+0000 osd.5 (osd.5) 230 : cluster 3 failed to encode map e248 with expected crc 2023-10-01T10:58:33.959 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:33 smithi078 bash[75299]: audit 2023-10-01T10:58:32.812362+0000 mon.a (mon.0) 1534 : audit 1 from='client.? 172.21.15.78:0/2417684448' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num", "val": "16"}]': finished 2023-10-01T10:58:33.959 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:33 smithi078 bash[75299]: cluster 2023-10-01T10:58:32.812439+0000 mon.a (mon.0) 1535 : cluster 0 osdmap e248: 8 total, 8 up, 8 in 2023-10-01T10:58:33.959 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:33 smithi078 bash[75299]: cluster 2023-10-01T10:58:32.819941+0000 osd.6 (osd.6) 192 : cluster 3 failed to encode map e248 with expected crc 2023-10-01T10:58:33.980 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:33.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:33.984 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:58:34.084 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:34.094 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:33 smithi173 bash[58816]: cluster 2023-10-01T10:58:32.119621+0000 mgr.y (mgr.25072) 1233 : cluster 0 pgmap v1246: 89 pgs: 89 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 649 op/s 2023-10-01T10:58:34.094 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:33 smithi173 bash[58816]: cluster 2023-10-01T10:58:32.808243+0000 osd.7 (osd.7) 233 : cluster 3 failed to encode map e248 with expected crc 2023-10-01T10:58:34.094 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:33 smithi173 bash[58816]: cluster 2023-10-01T10:58:32.810177+0000 osd.5 (osd.5) 230 : cluster 3 failed to encode map e248 with expected crc 2023-10-01T10:58:34.094 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:33 smithi173 bash[58816]: audit 2023-10-01T10:58:32.812362+0000 mon.a (mon.0) 1534 : audit 1 from='client.? 172.21.15.78:0/2417684448' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num", "val": "16"}]': finished 2023-10-01T10:58:34.094 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:33 smithi173 bash[58816]: cluster 2023-10-01T10:58:32.812439+0000 mon.a (mon.0) 1535 : cluster 0 osdmap e248: 8 total, 8 up, 8 in 2023-10-01T10:58:34.094 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:33 smithi173 bash[58816]: cluster 2023-10-01T10:58:32.819941+0000 osd.6 (osd.6) 192 : cluster 3 failed to encode map e248 with expected crc 2023-10-01T10:58:34.108 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:34.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:34.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:58:34.211 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:34.240 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:34.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:34.243 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:58:34.344 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:34.369 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:34.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:34.371 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:58:34.472 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:34.493 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:34.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:34.495 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:58:34.596 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:34.627 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:34.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:34.637 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:58:34.738 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:34.762 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:34.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:34.767 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:58:34.868 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:34.898 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:34.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:34.904 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:58:35.005 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:35.034 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:35.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:35.036 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:58:35.137 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:35.177 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:35.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:35.180 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:58:35.280 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:35.306 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:35.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:35.311 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:58:35.411 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:35.448 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:35.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:35.455 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:58:35.556 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:35.596 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:35.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:35.598 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:58:35.699 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:35.726 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:35.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:35.731 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:58:35.832 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:35.866 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:35.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:35.869 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:58:35.970 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:35.981 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:35 smithi078 bash[75299]: cluster 2023-10-01T10:58:34.120276+0000 mgr.y (mgr.25072) 1234 : cluster 0 pgmap v1248: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 765 op/s 2023-10-01T10:58:35.982 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:35 smithi078 bash[78274]: cluster 2023-10-01T10:58:34.120276+0000 mgr.y (mgr.25072) 1234 : cluster 0 pgmap v1248: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 765 op/s 2023-10-01T10:58:36.001 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:36.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:36.005 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:58:36.106 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:36.113 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:35 smithi173 bash[58816]: cluster 2023-10-01T10:58:34.120276+0000 mgr.y (mgr.25072) 1234 : cluster 0 pgmap v1248: 89 pgs: 89 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 765 op/s 2023-10-01T10:58:36.130 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:36.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:36.135 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:58:36.235 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:36.268 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:36.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:36.274 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:58:36.375 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:36.399 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:36.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:36.405 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:58:36.506 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:36.535 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:36.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:36.536 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:58:36.544 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T10:58:36.637 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:36.666 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:36.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:36.672 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:58:36.773 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:36.813 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:36.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:36.814 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:58:36.915 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:36.955 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:36.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:36.964 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:58:37.064 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:37.090 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:37.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:37.092 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:58:37.193 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:37.221 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:37.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:37.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:58:37.324 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:37.355 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:37.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:37.359 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:58:37.460 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:37.484 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:37.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:37.487 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:58:37.587 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:37.611 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:37.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:37.614 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:58:37.715 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:37.739 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:37.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:37.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:58:37.815 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:37 smithi078 bash[75299]: cluster 2023-10-01T10:58:36.120763+0000 mgr.y (mgr.25072) 1235 : cluster 0 pgmap v1249: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 758 op/s 2023-10-01T10:58:37.815 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:37 smithi078 bash[78274]: cluster 2023-10-01T10:58:36.120763+0000 mgr.y (mgr.25072) 1235 : cluster 0 pgmap v1249: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 758 op/s 2023-10-01T10:58:37.845 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:37.855 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:37 smithi173 bash[58816]: cluster 2023-10-01T10:58:36.120763+0000 mgr.y (mgr.25072) 1235 : cluster 0 pgmap v1249: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 758 op/s 2023-10-01T10:58:37.883 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:37.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:37.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:58:37.988 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:38.021 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:38.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:38.026 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:58:38.127 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:38.149 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:38.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:38.151 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:58:38.154 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T10:58:38.190 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T10:58:38.234 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T10:58:38.252 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:38.273 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T10:58:38.283 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:38.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:38.285 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:58:38.314 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T10:58:38.346 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T10:58:38.379 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T10:58:38.385 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:38.409 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T10:58:38.412 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:38.413 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T10:58:38.413 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T10:58:38.413 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T10:58:38.413 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T10:58:38.413 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T10:58:38.414 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T10:58:38.414 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T10:58:38.414 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T10:58:38.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:38.417 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:58:38.517 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:38.544 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:38.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:38.547 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:58:38.648 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:38.674 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:38.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:38.682 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:58:38.783 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:38.810 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:38.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:38.815 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:58:38.838 INFO:teuthology.orchestra.run.smithi078.stderr:nodeep-scrub is set 2023-10-01T10:58:38.884 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:38 smithi078 bash[75299]: audit 2023-10-01T10:58:38.105472+0000 mon.c (mon.2) 194 : audit 0 from='client.? 172.21.15.78:0/2223305535' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:58:38.884 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:38 smithi078 bash[75299]: audit 2023-10-01T10:58:38.571318+0000 mon.b (mon.1) 51 : audit 1 from='client.? 172.21.15.78:0/2913505703' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:58:38.884 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:38 smithi078 bash[75299]: audit 2023-10-01T10:58:38.571347+0000 mon.a (mon.0) 1536 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:58:38.884 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:38 smithi078 bash[78274]: audit 2023-10-01T10:58:38.105472+0000 mon.c (mon.2) 194 : audit 0 from='client.? 172.21.15.78:0/2223305535' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:58:38.884 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:38 smithi078 bash[78274]: audit 2023-10-01T10:58:38.571318+0000 mon.b (mon.1) 51 : audit 1 from='client.? 172.21.15.78:0/2913505703' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:58:38.884 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:38 smithi078 bash[78274]: audit 2023-10-01T10:58:38.571347+0000 mon.a (mon.0) 1536 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:58:38.916 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:38.936 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:38 smithi173 bash[58816]: audit 2023-10-01T10:58:38.105472+0000 mon.c (mon.2) 194 : audit 0 from='client.? 172.21.15.78:0/2223305535' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:58:38.936 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:38 smithi173 bash[58816]: audit 2023-10-01T10:58:38.571318+0000 mon.b (mon.1) 51 : audit 1 from='client.? 172.21.15.78:0/2913505703' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:58:38.936 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:38 smithi173 bash[58816]: audit 2023-10-01T10:58:38.571347+0000 mon.a (mon.0) 1536 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:58:38.936 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:38.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:38.938 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:58:39.039 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:39.065 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:39.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:39.067 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:58:39.168 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:39.199 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:39.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:39.203 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:58:39.303 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:39.332 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:39.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:39.334 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:58:39.435 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:39.459 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:39.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:39.463 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:58:39.563 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:39.601 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:39.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:39.608 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:58:39.708 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:39.730 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:39.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:39.732 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:58:39.833 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:39.867 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:39.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:39.868 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:58:39.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:39 smithi173 bash[58816]: cluster 2023-10-01T10:58:38.121235+0000 mgr.y (mgr.25072) 1236 : cluster 0 pgmap v1250: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 759 op/s 2023-10-01T10:58:39.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:39 smithi173 bash[58816]: cluster 2023-10-01T10:58:38.813159+0000 mon.a (mon.0) 1537 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:58:39.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:39 smithi173 bash[58816]: audit 2023-10-01T10:58:38.831837+0000 mon.a (mon.0) 1538 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T10:58:39.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:39 smithi173 bash[58816]: cluster 2023-10-01T10:58:38.832112+0000 mon.a (mon.0) 1539 : cluster 0 osdmap e249: 8 total, 8 up, 8 in 2023-10-01T10:58:39.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:39 smithi173 bash[58816]: cluster 2023-10-01T10:58:38.853442+0000 osd.6 (osd.6) 193 : cluster 3 failed to encode map e249 with expected crc 2023-10-01T10:58:39.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:39 smithi173 bash[58816]: cluster 2023-10-01T10:58:38.854517+0000 osd.6 (osd.6) 194 : cluster 3 failed to encode map e249 with expected crc 2023-10-01T10:58:39.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:39 smithi173 bash[58816]: cluster 2023-10-01T10:58:38.857432+0000 osd.7 (osd.7) 234 : cluster 3 failed to encode map e249 with expected crc 2023-10-01T10:58:39.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:39 smithi173 bash[58816]: cluster 2023-10-01T10:58:38.859600+0000 osd.7 (osd.7) 235 : cluster 3 failed to encode map e249 with expected crc 2023-10-01T10:58:39.969 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:40.000 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:40.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:40.004 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:58:40.069 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:39 smithi078 bash[75299]: cluster 2023-10-01T10:58:38.121235+0000 mgr.y (mgr.25072) 1236 : cluster 0 pgmap v1250: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 759 op/s 2023-10-01T10:58:40.069 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:39 smithi078 bash[75299]: cluster 2023-10-01T10:58:38.813159+0000 mon.a (mon.0) 1537 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:58:40.069 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:39 smithi078 bash[75299]: audit 2023-10-01T10:58:38.831837+0000 mon.a (mon.0) 1538 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T10:58:40.069 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:39 smithi078 bash[75299]: cluster 2023-10-01T10:58:38.832112+0000 mon.a (mon.0) 1539 : cluster 0 osdmap e249: 8 total, 8 up, 8 in 2023-10-01T10:58:40.069 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:39 smithi078 bash[75299]: cluster 2023-10-01T10:58:38.853442+0000 osd.6 (osd.6) 193 : cluster 3 failed to encode map e249 with expected crc 2023-10-01T10:58:40.069 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:39 smithi078 bash[75299]: cluster 2023-10-01T10:58:38.854517+0000 osd.6 (osd.6) 194 : cluster 3 failed to encode map e249 with expected crc 2023-10-01T10:58:40.069 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:39 smithi078 bash[75299]: cluster 2023-10-01T10:58:38.857432+0000 osd.7 (osd.7) 234 : cluster 3 failed to encode map e249 with expected crc 2023-10-01T10:58:40.069 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:39 smithi078 bash[75299]: cluster 2023-10-01T10:58:38.859600+0000 osd.7 (osd.7) 235 : cluster 3 failed to encode map e249 with expected crc 2023-10-01T10:58:40.070 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:39 smithi078 bash[78274]: cluster 2023-10-01T10:58:38.121235+0000 mgr.y (mgr.25072) 1236 : cluster 0 pgmap v1250: 89 pgs: 89 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 759 op/s 2023-10-01T10:58:40.070 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:39 smithi078 bash[78274]: cluster 2023-10-01T10:58:38.813159+0000 mon.a (mon.0) 1537 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:58:40.070 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:39 smithi078 bash[78274]: audit 2023-10-01T10:58:38.831837+0000 mon.a (mon.0) 1538 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T10:58:40.070 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:39 smithi078 bash[78274]: cluster 2023-10-01T10:58:38.832112+0000 mon.a (mon.0) 1539 : cluster 0 osdmap e249: 8 total, 8 up, 8 in 2023-10-01T10:58:40.070 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:39 smithi078 bash[78274]: cluster 2023-10-01T10:58:38.853442+0000 osd.6 (osd.6) 193 : cluster 3 failed to encode map e249 with expected crc 2023-10-01T10:58:40.070 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:39 smithi078 bash[78274]: cluster 2023-10-01T10:58:38.854517+0000 osd.6 (osd.6) 194 : cluster 3 failed to encode map e249 with expected crc 2023-10-01T10:58:40.070 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:39 smithi078 bash[78274]: cluster 2023-10-01T10:58:38.857432+0000 osd.7 (osd.7) 234 : cluster 3 failed to encode map e249 with expected crc 2023-10-01T10:58:40.070 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:39 smithi078 bash[78274]: cluster 2023-10-01T10:58:38.859600+0000 osd.7 (osd.7) 235 : cluster 3 failed to encode map e249 with expected crc 2023-10-01T10:58:40.105 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:40.141 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:40.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:40.147 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:58:40.248 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:40.280 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:40.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:40.285 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:58:40.331 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:58:40 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:58:40] "GET /metrics HTTP/1.1" 200 36382 "" "Prometheus/2.33.4" 2023-10-01T10:58:40.386 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:40.415 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:40.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:40.424 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:58:40.525 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:40.550 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:40.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:40.554 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:58:40.655 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:40.686 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:40.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:40.689 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:58:40.790 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:40.812 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:40.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:40.814 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:58:40.915 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:40.936 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:40.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:40.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:58:41.040 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:41.050 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-10-01T10:58:41.050 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [], 2023-10-01T10:58:41.050 INFO:teuthology.orchestra.run.smithi173.stdout: "complaint_time": 30, 2023-10-01T10:58:41.050 INFO:teuthology.orchestra.run.smithi173.stdout: "num_blocked_ops": 0 2023-10-01T10:58:41.050 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-10-01T10:58:41.066 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:41.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:41.072 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:58:41.173 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:41.203 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:41.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:41.207 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:58:41.307 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:41.320 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 7, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T10:58:41.320 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T10:58:41.320 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_7 2023-10-01T10:58:41.321 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T10:58:41.331 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:41.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:41.335 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:58:41.436 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:41.464 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:41.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:41.467 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:58:41.568 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:41.593 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:41.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:41.595 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:58:41.696 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:41.720 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:41.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:41.723 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:58:41.823 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:41.860 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:41.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:41.862 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:58:41.963 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:41.993 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:41.997 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:41 smithi173 bash[58816]: cluster 2023-10-01T10:58:40.121743+0000 mgr.y (mgr.25072) 1237 : cluster 0 pgmap v1252: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 937 op/s 2023-10-01T10:58:41.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:41.998 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:58:42.077 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-01T10:58:42.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:41 smithi078 bash[75299]: cluster 2023-10-01T10:58:40.121743+0000 mgr.y (mgr.25072) 1237 : cluster 0 pgmap v1252: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 937 op/s 2023-10-01T10:58:42.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:41 smithi078 bash[78274]: cluster 2023-10-01T10:58:40.121743+0000 mgr.y (mgr.25072) 1237 : cluster 0 pgmap v1252: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 937 op/s 2023-10-01T10:58:42.099 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:42.160 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:42.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:42.165 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:58:42.266 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:42.302 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:42.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:42.305 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:58:42.405 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:42.430 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:42.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:42.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:58:42.446 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T10:58:42.536 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:42.565 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:42.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:42.570 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:58:42.671 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:42.714 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:42.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:42.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:58:42.822 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:42.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:42.854 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:42.854 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:58:42.955 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:42.982 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:42.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:42.985 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:58:43.086 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:43.122 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:43.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:43.124 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:58:43.225 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:43.252 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:43.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:43.255 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:58:43.354 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T10:58:43.355 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:43.402 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:43.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:43.407 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:58:43.507 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:43.534 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:43.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:43.537 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:58:43.638 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:43.665 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:43.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:43.670 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:58:43.771 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:43.805 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:43.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:43.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:58:43.830 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:43 smithi173 bash[58816]: cluster 2023-10-01T10:58:42.122155+0000 mgr.y (mgr.25072) 1238 : cluster 0 pgmap v1253: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2023-10-01T10:58:43.911 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:43.960 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:43.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:43.964 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:58:44.064 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:44.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:43 smithi078 bash[75299]: cluster 2023-10-01T10:58:42.122155+0000 mgr.y (mgr.25072) 1238 : cluster 0 pgmap v1253: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2023-10-01T10:58:44.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:43 smithi078 bash[78274]: cluster 2023-10-01T10:58:42.122155+0000 mgr.y (mgr.25072) 1238 : cluster 0 pgmap v1253: 89 pgs: 89 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2023-10-01T10:58:44.089 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:44.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:44.093 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:58:44.194 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:44.227 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:44.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:44.231 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:58:44.332 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:44.358 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:44.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:44.362 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:58:44.463 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:44.495 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:44.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:44.497 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:58:44.598 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:44.623 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:44.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:44.637 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:58:44.738 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:44.763 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:44.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:44.767 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:58:44.867 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:44.920 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:44.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:44.924 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:58:45.024 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:45.058 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:45.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:45.060 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:58:45.161 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:45.207 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:45.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:45.213 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:58:45.314 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:45.343 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:45.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:45.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:58:45.447 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:45.471 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:45.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:45.474 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:58:45.575 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:45.608 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:45.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:45.611 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:58:45.712 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:45.746 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:45.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:45.752 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:58:45.853 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:45.883 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:45.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:45.885 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:58:45.987 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:45.995 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:45 smithi078 bash[78274]: cluster 2023-10-01T10:58:44.124086+0000 mgr.y (mgr.25072) 1239 : cluster 0 pgmap v1254: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 740 op/s 2023-10-01T10:58:45.998 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:45 smithi078 bash[75299]: cluster 2023-10-01T10:58:44.124086+0000 mgr.y (mgr.25072) 1239 : cluster 0 pgmap v1254: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 740 op/s 2023-10-01T10:58:46.018 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:46.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:46.023 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:58:46.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:45 smithi173 bash[58816]: cluster 2023-10-01T10:58:44.124086+0000 mgr.y (mgr.25072) 1239 : cluster 0 pgmap v1254: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 740 op/s 2023-10-01T10:58:46.124 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:46.152 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:46.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:46.157 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:58:46.158 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:58:46.258 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:46.302 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:46.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:46.306 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:58:46.406 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:46.441 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:46.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:46.446 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:58:46.547 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:46.577 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:46.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:46.580 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:58:46.681 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:46.705 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:46.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:46.707 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:58:46.808 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:46.834 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:46.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:46.836 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:58:46.901 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T10:58:46.937 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:46.969 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:46.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:46.972 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:58:47.072 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:47.100 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:47.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:47.103 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:58:47.174 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:58:47.203 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:47.226 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:47.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:47.230 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:58:47.331 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:47.362 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:47.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:47.369 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:58:47.470 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:47.497 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:47.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:47.500 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:58:47.600 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:47.624 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:47.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:47.633 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:58:47.734 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:47.762 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:47.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:47.769 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:58:47.838 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:47 smithi173 bash[58816]: cluster 2023-10-01T10:58:46.124617+0000 mgr.y (mgr.25072) 1240 : cluster 0 pgmap v1255: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 735 op/s 2023-10-01T10:58:47.838 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:47 smithi173 bash[58816]: audit 2023-10-01T10:58:47.448955+0000 mon.a (mon.0) 1540 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:58:47.869 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:47.896 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:47.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:47.898 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:58:47.999 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:48.028 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:48.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:48.032 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:58:48.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:47 smithi078 bash[78274]: cluster 2023-10-01T10:58:46.124617+0000 mgr.y (mgr.25072) 1240 : cluster 0 pgmap v1255: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 735 op/s 2023-10-01T10:58:48.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:47 smithi078 bash[78274]: audit 2023-10-01T10:58:47.448955+0000 mon.a (mon.0) 1540 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:58:48.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:47 smithi078 bash[75299]: cluster 2023-10-01T10:58:46.124617+0000 mgr.y (mgr.25072) 1240 : cluster 0 pgmap v1255: 89 pgs: 89 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 735 op/s 2023-10-01T10:58:48.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:47 smithi078 bash[75299]: audit 2023-10-01T10:58:47.448955+0000 mon.a (mon.0) 1540 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:58:48.133 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:48.174 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:48.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:48.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:58:48.201 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:58:48.278 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:48.304 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:48.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:48.307 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:58:48.408 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:48.435 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:48.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:48.437 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:58:48.538 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:48.573 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:48.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:48.579 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:58:48.679 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:48.703 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:48.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:48.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:58:48.809 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:48.850 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:48.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:48.853 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:58:48.953 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:48.976 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:48.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:48.983 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:58:49.084 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:49.111 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:49.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:49.116 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:58:49.216 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:49.246 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:49.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:49.248 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:58:49.349 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:49.376 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:49.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:49.379 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:58:49.480 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:49.520 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:49.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:49.524 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:58:49.625 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:49.652 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:49.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:49.654 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:58:49.755 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:49.779 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:49.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:49.781 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:58:49.882 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:49.902 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:49.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:49.905 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:58:49.914 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:49 smithi173 bash[58816]: cluster 2023-10-01T10:58:48.125155+0000 mgr.y (mgr.25072) 1241 : cluster 0 pgmap v1256: 89 pgs: 89 active+clean; 5.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 722 op/s 2023-10-01T10:58:50.005 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:50.022 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:50.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:50.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:58:50.124 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:50.163 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:50.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:50.166 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:58:50.166 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:49 smithi078 bash[75299]: cluster 2023-10-01T10:58:48.125155+0000 mgr.y (mgr.25072) 1241 : cluster 0 pgmap v1256: 89 pgs: 89 active+clean; 5.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 722 op/s 2023-10-01T10:58:50.167 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:49 smithi078 bash[78274]: cluster 2023-10-01T10:58:48.125155+0000 mgr.y (mgr.25072) 1241 : cluster 0 pgmap v1256: 89 pgs: 89 active+clean; 5.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 722 op/s 2023-10-01T10:58:50.267 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:50.293 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:50.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:50.299 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:58:50.400 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:50.426 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:50.432 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:58:50 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:58:50] "GET /metrics HTTP/1.1" 200 36385 "" "Prometheus/2.33.4" 2023-10-01T10:58:50.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:50.433 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:58:50.534 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:50.569 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:50.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:50.574 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:58:50.675 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:50.700 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:50.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:50.702 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:58:50.803 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:50.826 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:50.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:50.830 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:58:50.930 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:50.965 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:50.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:50.970 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:58:51.072 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:51.101 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:51.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:51.106 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:58:51.206 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:51.248 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:51.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:51.255 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:58:51.356 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:51.395 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:51.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:51.399 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:58:51.402 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-10-01T10:58:51.402 INFO:teuthology.orchestra.run.smithi173.stdout: "size": 20, 2023-10-01T10:58:51.402 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 600, 2023-10-01T10:58:51.403 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [ 2023-10-01T10:58:51.403 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.403 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46037.0:100885 11.5 11:a8bf3b53:::benchmark_data_smithi078_157677_object100884:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e183)", 2023-10-01T10:58:51.403 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:52:42.662272+0000", 2023-10-01T10:58:51.403 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 368.73674266500001, 2023-10-01T10:58:51.403 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0382194119999999, 2023-10-01T10:58:51.403 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:58:51.403 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:58:51.403 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:58:51.403 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46037", 2023-10-01T10:58:51.404 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:58:51.404 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 100885 2023-10-01T10:58:51.404 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.404 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:58:51.404 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.404 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:58:51.404 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.662272+0000", 2023-10-01T10:58:51.404 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:58:51.405 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.405 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.405 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:58:51.405 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.662272+0000", 2023-10-01T10:58:51.405 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999971 2023-10-01T10:58:51.405 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.405 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.405 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:58:51.406 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.662269+0000", 2023-10-01T10:58:51.406 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.3422000000000003e-05 2023-10-01T10:58:51.406 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.406 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.406 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:58:51.406 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.662343+0000", 2023-10-01T10:58:51.406 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.7986e-05 2023-10-01T10:58:51.407 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.407 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.407 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:58:51.407 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.662361+0000", 2023-10-01T10:58:51.407 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.5179999999999993e-06 2023-10-01T10:58:51.407 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.408 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.408 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:58:51.408 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.662370+0000", 2023-10-01T10:58:51.408 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.4979999999999999e-05 2023-10-01T10:58:51.408 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.408 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.409 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:58:51.409 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.662425+0000", 2023-10-01T10:58:51.409 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.0429999999999996e-06 2023-10-01T10:58:51.409 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.410 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.410 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T10:58:51.410 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.662434+0000", 2023-10-01T10:58:51.410 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0157098099999999 2023-10-01T10:58:51.410 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.411 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.411 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:58:51.411 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.678144+0000", 2023-10-01T10:58:51.411 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0044951189999999997 2023-10-01T10:58:51.411 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.411 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.412 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:58:51.412 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.682639+0000", 2023-10-01T10:58:51.412 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00021618900000000001 2023-10-01T10:58:51.412 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.412 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.412 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 6,7", 2023-10-01T10:58:51.413 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.682855+0000", 2023-10-01T10:58:51.413 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0047396299999999999 2023-10-01T10:58:51.413 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.413 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.413 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:58:51.413 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.687595+0000", 2023-10-01T10:58:51.414 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0085244929999999993 2023-10-01T10:58:51.414 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.414 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.414 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:58:51.415 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.696119+0000", 2023-10-01T10:58:51.415 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0042609229999999998 2023-10-01T10:58:51.415 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.415 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.415 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:58:51.416 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.700380+0000", 2023-10-01T10:58:51.416 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.4591999999999994e-05 2023-10-01T10:58:51.416 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.416 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.416 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:58:51.416 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.700445+0000", 2023-10-01T10:58:51.417 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.6448e-05 2023-10-01T10:58:51.417 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.417 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.417 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:58:51.417 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.700491+0000", 2023-10-01T10:58:51.417 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0382194119999999 2023-10-01T10:58:51.417 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:58:51.418 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:58:51.418 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:58:51.418 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.418 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.418 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46037.0:100899 11.5 11:a263adfb:::benchmark_data_smithi078_157677_object100898:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e183)", 2023-10-01T10:58:51.418 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:52:42.689207+0000", 2023-10-01T10:58:51.419 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 368.70980782999999, 2023-10-01T10:58:51.419 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.02878008, 2023-10-01T10:58:51.419 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:58:51.419 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:58:51.419 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:58:51.419 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46037", 2023-10-01T10:58:51.419 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:58:51.420 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 100899 2023-10-01T10:58:51.420 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.420 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:58:51.420 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.420 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:58:51.420 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.689207+0000", 2023-10-01T10:58:51.420 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:58:51.421 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.421 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.421 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:58:51.421 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.689207+0000", 2023-10-01T10:58:51.421 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-10-01T10:58:51.421 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.422 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.422 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:58:51.422 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.689205+0000", 2023-10-01T10:58:51.422 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00011599600000000001 2023-10-01T10:58:51.422 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.422 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.423 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:58:51.423 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.689321+0000", 2023-10-01T10:58:51.423 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2656999999999999e-05 2023-10-01T10:58:51.423 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.423 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.423 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:58:51.423 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.689334+0000", 2023-10-01T10:58:51.424 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.1739999999999997e-06 2023-10-01T10:58:51.424 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.424 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.424 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:58:51.424 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.689340+0000", 2023-10-01T10:58:51.424 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000115093 2023-10-01T10:58:51.425 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.425 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.425 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:58:51.425 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.689455+0000", 2023-10-01T10:58:51.425 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.8561999999999999e-05 2023-10-01T10:58:51.425 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.425 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.426 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T10:58:51.426 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.689483+0000", 2023-10-01T10:58:51.426 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.99813689800000005 2023-10-01T10:58:51.426 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.426 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.426 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:58:51.427 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.687620+0000", 2023-10-01T10:58:51.427 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00013331500000000001 2023-10-01T10:58:51.427 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.427 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.427 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:58:51.427 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.687754+0000", 2023-10-01T10:58:51.427 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00016973100000000001 2023-10-01T10:58:51.428 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.428 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.428 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 6,7", 2023-10-01T10:58:51.428 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.687923+0000", 2023-10-01T10:58:51.428 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0082216949999999994 2023-10-01T10:58:51.428 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.428 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.429 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:58:51.429 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.696145+0000", 2023-10-01T10:58:51.429 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0044566409999999999 2023-10-01T10:58:51.429 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.429 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.429 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:58:51.429 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.700602+0000", 2023-10-01T10:58:51.430 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.017304987000000001 2023-10-01T10:58:51.430 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.430 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.430 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:58:51.430 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.717907+0000", 2023-10-01T10:58:51.430 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.9972999999999997e-05 2023-10-01T10:58:51.430 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.431 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.431 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:58:51.431 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.717957+0000", 2023-10-01T10:58:51.431 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.0179e-05 2023-10-01T10:58:51.431 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.431 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.431 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:58:51.432 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.717987+0000", 2023-10-01T10:58:51.432 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.02878008 2023-10-01T10:58:51.432 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:58:51.432 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:58:51.432 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:58:51.432 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.432 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.433 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46037.0:100935 11.5 11:abf982ee:::benchmark_data_smithi078_157677_object100934:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e183)", 2023-10-01T10:58:51.433 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:52:42.739979+0000", 2023-10-01T10:58:51.433 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 368.659035276, 2023-10-01T10:58:51.433 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.97826487699999998, 2023-10-01T10:58:51.433 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:58:51.433 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:58:51.433 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:58:51.434 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46037", 2023-10-01T10:58:51.434 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:58:51.434 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 100935 2023-10-01T10:58:51.434 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.434 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:58:51.434 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.435 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:58:51.435 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.739979+0000", 2023-10-01T10:58:51.435 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:58:51.435 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.435 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.435 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:58:51.435 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.739979+0000", 2023-10-01T10:58:51.435 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999976 2023-10-01T10:58:51.436 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.436 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.436 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:58:51.436 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.739977+0000", 2023-10-01T10:58:51.436 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.8652e-05 2023-10-01T10:58:51.436 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.436 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.437 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:58:51.437 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.740036+0000", 2023-10-01T10:58:51.437 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.7019000000000001e-05 2023-10-01T10:58:51.437 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.437 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.437 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:58:51.437 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.740053+0000", 2023-10-01T10:58:51.438 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1243000000000001e-05 2023-10-01T10:58:51.438 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.438 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.438 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:58:51.438 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.740064+0000", 2023-10-01T10:58:51.438 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.9073999999999997e-05 2023-10-01T10:58:51.438 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.439 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.439 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:58:51.439 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.740123+0000", 2023-10-01T10:58:51.439 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0672999999999999e-05 2023-10-01T10:58:51.439 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.439 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.439 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T10:58:51.440 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:42.740134+0000", 2023-10-01T10:58:51.440 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.95174533299999997 2023-10-01T10:58:51.440 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.440 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.440 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:58:51.440 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.691879+0000", 2023-10-01T10:58:51.440 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00017638199999999999 2023-10-01T10:58:51.441 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.441 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.441 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:58:51.441 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.692055+0000", 2023-10-01T10:58:51.441 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00016038700000000001 2023-10-01T10:58:51.441 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.441 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.442 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 6,7", 2023-10-01T10:58:51.442 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.692216+0000", 2023-10-01T10:58:51.442 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0039323140000000001 2023-10-01T10:58:51.442 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.442 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.442 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:58:51.442 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.696148+0000", 2023-10-01T10:58:51.443 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0045511520000000001 2023-10-01T10:58:51.443 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.443 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.443 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:58:51.443 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.700699+0000", 2023-10-01T10:58:51.443 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.017418930999999999 2023-10-01T10:58:51.443 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.444 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.444 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:58:51.444 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.718118+0000", 2023-10-01T10:58:51.444 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.0555000000000005e-05 2023-10-01T10:58:51.444 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.444 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.444 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:58:51.445 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.718189+0000", 2023-10-01T10:58:51.445 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.5376e-05 2023-10-01T10:58:51.445 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.445 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.445 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:58:51.445 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:52:43.718244+0000", 2023-10-01T10:58:51.446 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.97826487699999998 2023-10-01T10:58:51.446 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:58:51.446 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:58:51.446 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:58:51.446 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.446 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.446 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46037.0:165744 11.7 11:e5cfcb2a:::benchmark_data_smithi078_157677_object58573:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e189)", 2023-10-01T10:58:51.447 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:54:14.595875+0000", 2023-10-01T10:58:51.447 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 276.803139821, 2023-10-01T10:58:51.447 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.042697521, 2023-10-01T10:58:51.447 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:58:51.447 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "reached pg", 2023-10-01T10:58:51.447 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:58:51.447 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46037", 2023-10-01T10:58:51.448 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:58:51.448 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 165744 2023-10-01T10:58:51.448 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.448 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:58:51.448 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.448 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:58:51.448 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:14.595875+0000", 2023-10-01T10:58:51.449 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:58:51.449 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.449 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.449 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:58:51.449 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:14.595875+0000", 2023-10-01T10:58:51.449 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999967 2023-10-01T10:58:51.450 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.450 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.450 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:58:51.450 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:14.595871+0000", 2023-10-01T10:58:51.450 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.8224999999999999e-05 2023-10-01T10:58:51.450 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.450 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.451 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:58:51.451 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:14.595890+0000", 2023-10-01T10:58:51.451 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.0704e-05 2023-10-01T10:58:51.451 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.451 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.451 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:58:51.451 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:14.595910+0000", 2023-10-01T10:58:51.452 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1457000000000001e-05 2023-10-01T10:58:51.452 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.452 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.452 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:58:51.452 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:14.595922+0000", 2023-10-01T10:58:51.452 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.317788348 2023-10-01T10:58:51.453 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.453 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.453 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:58:51.453 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:14.913710+0000", 2023-10-01T10:58:51.453 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0366e-05 2023-10-01T10:58:51.453 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.454 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.454 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T10:58:51.454 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:14.913721+0000", 2023-10-01T10:58:51.454 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.72476449300000001 2023-10-01T10:58:51.454 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.454 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.455 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:58:51.455 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:15.638485+0000", 2023-10-01T10:58:51.455 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.7224000000000004e-05 2023-10-01T10:58:51.455 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.456 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.456 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:58:51.456 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:15.638572+0000", 2023-10-01T10:58:51.456 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.042697521 2023-10-01T10:58:51.456 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:58:51.456 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:58:51.456 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:58:51.457 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.457 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.457 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46037.0:165748 11.15 11:aa0793e0:::benchmark_data_smithi078_157677_object58577:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e189)", 2023-10-01T10:58:51.457 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:54:14.595931+0000", 2023-10-01T10:58:51.457 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 276.80308397099998, 2023-10-01T10:58:51.457 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.065062323, 2023-10-01T10:58:51.458 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:58:51.458 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:58:51.458 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:58:51.458 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46037", 2023-10-01T10:58:51.458 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:58:51.459 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 165748 2023-10-01T10:58:51.459 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.459 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:58:51.459 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.459 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:58:51.459 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:14.595931+0000", 2023-10-01T10:58:51.459 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:58:51.460 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.460 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.460 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:58:51.460 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:14.595931+0000", 2023-10-01T10:58:51.460 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999995 2023-10-01T10:58:51.460 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.460 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.460 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:58:51.460 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:14.595930+0000", 2023-10-01T10:58:51.460 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.0419999999999996e-06 2023-10-01T10:58:51.460 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.461 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.461 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:58:51.461 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:14.595934+0000", 2023-10-01T10:58:51.461 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.8880000000000002e-06 2023-10-01T10:58:51.461 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.461 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.461 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:58:51.461 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:14.595940+0000", 2023-10-01T10:58:51.461 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.3220000000000003e-06 2023-10-01T10:58:51.461 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.461 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.461 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:58:51.462 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:14.595944+0000", 2023-10-01T10:58:51.462 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.26720145000000001 2023-10-01T10:58:51.462 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.462 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.462 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:58:51.462 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:14.863146+0000", 2023-10-01T10:58:51.462 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.064e-06 2023-10-01T10:58:51.462 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.462 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.462 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T10:58:51.462 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:14.863155+0000", 2023-10-01T10:58:51.463 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.79351720800000003 2023-10-01T10:58:51.463 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.463 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.463 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:58:51.463 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:15.656672+0000", 2023-10-01T10:58:51.463 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00016405900000000001 2023-10-01T10:58:51.463 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.463 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.463 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:58:51.463 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:15.656836+0000", 2023-10-01T10:58:51.463 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00021579100000000001 2023-10-01T10:58:51.463 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.464 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.464 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 6,7", 2023-10-01T10:58:51.464 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:15.657052+0000", 2023-10-01T10:58:51.464 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0036623279999999999 2023-10-01T10:58:51.464 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.464 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.464 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:58:51.464 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:15.660714+0000", 2023-10-01T10:58:51.464 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00016588400000000001 2023-10-01T10:58:51.465 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.465 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.465 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:58:51.465 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:15.660880+0000", 2023-10-01T10:58:51.465 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.7450000000000007e-06 2023-10-01T10:58:51.465 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.465 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.465 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:58:51.466 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:15.660890+0000", 2023-10-01T10:58:51.466 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.6746999999999997e-05 2023-10-01T10:58:51.466 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.466 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.466 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:58:51.466 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:15.660947+0000", 2023-10-01T10:58:51.466 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.6325000000000001e-05 2023-10-01T10:58:51.466 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.467 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.467 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:58:51.467 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:15.660993+0000", 2023-10-01T10:58:51.467 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.065062323 2023-10-01T10:58:51.467 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:58:51.467 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:58:51.467 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:58:51.468 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.468 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.468 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46037.0:176163 11.13 11.6fd86013 (undecoded) ondisk+write+known_if_redirected e198)", 2023-10-01T10:58:51.468 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:54:32.678028+0000", 2023-10-01T10:58:51.468 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 258.72098654000001, 2023-10-01T10:58:51.468 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.87486806100000003, 2023-10-01T10:58:51.468 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:58:51.468 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "reached pg", 2023-10-01T10:58:51.468 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:58:51.469 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46037", 2023-10-01T10:58:51.469 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:58:51.469 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 176163 2023-10-01T10:58:51.469 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.469 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:58:51.469 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.469 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:58:51.469 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:32.678028+0000", 2023-10-01T10:58:51.469 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:58:51.469 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.469 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.469 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:58:51.469 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:32.678028+0000", 2023-10-01T10:58:51.470 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-10-01T10:58:51.470 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.470 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.470 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:58:51.470 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:32.678026+0000", 2023-10-01T10:58:51.470 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2632999999999999e-05 2023-10-01T10:58:51.470 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.470 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.470 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:58:51.470 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:32.678039+0000", 2023-10-01T10:58:51.470 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.4204000000000001e-05 2023-10-01T10:58:51.470 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.471 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.471 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:58:51.471 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:32.678063+0000", 2023-10-01T10:58:51.471 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0682e-05 2023-10-01T10:58:51.471 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.471 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.471 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:58:51.471 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:32.678073+0000", 2023-10-01T10:58:51.471 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.5694999999999998e-05 2023-10-01T10:58:51.471 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.471 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.471 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:58:51.471 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:32.678169+0000", 2023-10-01T10:58:51.472 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.6915999999999999e-05 2023-10-01T10:58:51.472 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.472 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.472 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T10:58:51.472 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:32.678186+0000", 2023-10-01T10:58:51.472 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.87469577200000004 2023-10-01T10:58:51.472 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.472 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.472 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:58:51.472 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:33.552882+0000", 2023-10-01T10:58:51.472 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4250000000000001e-05 2023-10-01T10:58:51.472 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.473 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.473 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:58:51.473 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:33.552896+0000", 2023-10-01T10:58:51.473 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.87486806100000003 2023-10-01T10:58:51.473 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:58:51.473 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:58:51.473 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:58:51.473 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.473 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.473 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46037.0:176170 11.13 11.ec438833 (undecoded) ondisk+write+known_if_redirected e198)", 2023-10-01T10:58:51.473 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:54:32.696758+0000", 2023-10-01T10:58:51.473 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 258.702257035, 2023-10-01T10:58:51.474 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.85627318299999999, 2023-10-01T10:58:51.474 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:58:51.474 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "reached pg", 2023-10-01T10:58:51.474 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:58:51.474 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46037", 2023-10-01T10:58:51.474 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:58:51.474 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 176170 2023-10-01T10:58:51.474 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.474 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:58:51.474 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.474 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:58:51.474 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:32.696758+0000", 2023-10-01T10:58:51.474 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:58:51.475 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.475 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.475 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:58:51.475 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:32.696758+0000", 2023-10-01T10:58:51.475 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-10-01T10:58:51.475 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.475 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.475 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:58:51.475 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:32.696756+0000", 2023-10-01T10:58:51.475 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.4769999999999992e-06 2023-10-01T10:58:51.475 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.475 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.476 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:58:51.476 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:32.696765+0000", 2023-10-01T10:58:51.476 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.6453e-05 2023-10-01T10:58:51.476 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.476 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.476 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:58:51.476 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:32.696781+0000", 2023-10-01T10:58:51.476 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.3969999999999997e-06 2023-10-01T10:58:51.476 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.476 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.476 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:58:51.476 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:32.696789+0000", 2023-10-01T10:58:51.477 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.5915999999999998e-05 2023-10-01T10:58:51.477 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.477 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.477 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:58:51.477 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:32.696845+0000", 2023-10-01T10:58:51.477 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.7440000000000007e-06 2023-10-01T10:58:51.477 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.477 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.477 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T10:58:51.477 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:32.696853+0000", 2023-10-01T10:58:51.477 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.85614665899999998 2023-10-01T10:58:51.477 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.478 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.478 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:58:51.478 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:33.552999+0000", 2023-10-01T10:58:51.478 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.1501999999999998e-05 2023-10-01T10:58:51.478 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.478 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.478 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:58:51.478 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:33.553031+0000", 2023-10-01T10:58:51.478 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.85627318299999999 2023-10-01T10:58:51.478 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:58:51.478 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:58:51.478 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:58:51.479 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.479 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.479 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46037.0:177250 11.15 11:a9a2129b:::benchmark_data_smithi078_157677_object70079:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e203)", 2023-10-01T10:58:51.479 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:54:37.552328+0000", 2023-10-01T10:58:51.479 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 253.84668633199999, 2023-10-01T10:58:51.479 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.151765398, 2023-10-01T10:58:51.479 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:58:51.479 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:58:51.479 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:58:51.479 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46037", 2023-10-01T10:58:51.479 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:58:51.479 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 177250 2023-10-01T10:58:51.480 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.480 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:58:51.480 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.480 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:58:51.480 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.552328+0000", 2023-10-01T10:58:51.480 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:58:51.480 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.480 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.480 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:58:51.480 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.552328+0000", 2023-10-01T10:58:51.480 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-10-01T10:58:51.480 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.481 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.481 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:58:51.481 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.552326+0000", 2023-10-01T10:58:51.481 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.7419999999999998e-05 2023-10-01T10:58:51.481 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.481 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.481 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:58:51.481 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.552354+0000", 2023-10-01T10:58:51.481 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.2529000000000001e-05 2023-10-01T10:58:51.481 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.481 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.481 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:58:51.482 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.552376+0000", 2023-10-01T10:58:51.482 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.3520000000000007e-06 2023-10-01T10:58:51.482 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.482 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.482 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:58:51.482 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.552385+0000", 2023-10-01T10:58:51.482 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0020613820000000001 2023-10-01T10:58:51.482 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.482 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.482 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:58:51.482 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.554446+0000", 2023-10-01T10:58:51.482 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.0690000000000002e-06 2023-10-01T10:58:51.482 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.483 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.483 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T10:58:51.483 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.554454+0000", 2023-10-01T10:58:51.483 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1414079960000001 2023-10-01T10:58:51.483 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.483 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.483 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:58:51.483 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.695862+0000", 2023-10-01T10:58:51.483 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000229736 2023-10-01T10:58:51.483 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.483 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.483 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:58:51.484 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.696092+0000", 2023-10-01T10:58:51.484 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00055690199999999996 2023-10-01T10:58:51.484 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.484 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.484 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1,2,4,6,7", 2023-10-01T10:58:51.484 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.696649+0000", 2023-10-01T10:58:51.484 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.003106878 2023-10-01T10:58:51.484 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.484 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.484 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:58:51.484 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.699756+0000", 2023-10-01T10:58:51.484 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1595e-05 2023-10-01T10:58:51.485 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.485 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.485 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:58:51.485 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.699767+0000", 2023-10-01T10:58:51.485 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000103874 2023-10-01T10:58:51.485 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.485 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.485 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:58:51.485 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.699871+0000", 2023-10-01T10:58:51.485 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00337937 2023-10-01T10:58:51.485 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.485 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.485 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:58:51.486 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.703250+0000", 2023-10-01T10:58:51.486 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00036137700000000002 2023-10-01T10:58:51.486 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.486 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.486 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:58:51.486 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.703612+0000", 2023-10-01T10:58:51.486 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000260556 2023-10-01T10:58:51.486 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.486 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.486 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:58:51.486 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.703872+0000", 2023-10-01T10:58:51.486 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.9154999999999996e-05 2023-10-01T10:58:51.487 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.487 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.487 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:58:51.487 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.703961+0000", 2023-10-01T10:58:51.487 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00013218600000000001 2023-10-01T10:58:51.487 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.487 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.487 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:58:51.487 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.704094+0000", 2023-10-01T10:58:51.487 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.151765398 2023-10-01T10:58:51.487 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:58:51.487 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:58:51.487 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:58:51.488 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.488 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.488 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46037.0:177347 11.15 11:ad9e2004:::benchmark_data_smithi078_157677_object70176:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e203)", 2023-10-01T10:58:51.488 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:54:37.552401+0000", 2023-10-01T10:58:51.488 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 253.846613983, 2023-10-01T10:58:51.488 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.152022506, 2023-10-01T10:58:51.488 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:58:51.488 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:58:51.488 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:58:51.488 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46037", 2023-10-01T10:58:51.488 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:58:51.488 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 177347 2023-10-01T10:58:51.489 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.489 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:58:51.489 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.489 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:58:51.489 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.552401+0000", 2023-10-01T10:58:51.489 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:58:51.489 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.489 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.489 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:58:51.489 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.552401+0000", 2023-10-01T10:58:51.489 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-10-01T10:58:51.489 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.489 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.490 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:58:51.490 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.552400+0000", 2023-10-01T10:58:51.490 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.9810000000000003e-06 2023-10-01T10:58:51.490 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.490 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.490 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:58:51.490 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.552405+0000", 2023-10-01T10:58:51.490 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.7049999999999994e-06 2023-10-01T10:58:51.490 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.490 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.490 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:58:51.490 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.552414+0000", 2023-10-01T10:58:51.491 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.7459999999999997e-06 2023-10-01T10:58:51.491 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.491 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.491 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:58:51.491 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.552420+0000", 2023-10-01T10:58:51.491 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0020463159999999998 2023-10-01T10:58:51.491 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.491 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.491 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:58:51.491 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.554466+0000", 2023-10-01T10:58:51.491 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.8800000000000001e-06 2023-10-01T10:58:51.491 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.491 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.492 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T10:58:51.492 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.554470+0000", 2023-10-01T10:58:51.492 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1454487769999999 2023-10-01T10:58:51.492 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.492 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.492 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:58:51.492 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.699919+0000", 2023-10-01T10:58:51.492 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.6296000000000001e-05 2023-10-01T10:58:51.492 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.492 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.492 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:58:51.492 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.700015+0000", 2023-10-01T10:58:51.493 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00043841300000000002 2023-10-01T10:58:51.493 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.493 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.493 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1,2,4,6,7", 2023-10-01T10:58:51.493 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.700454+0000", 2023-10-01T10:58:51.493 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0032219660000000002 2023-10-01T10:58:51.493 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.493 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.493 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:58:51.493 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.703676+0000", 2023-10-01T10:58:51.493 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000106832 2023-10-01T10:58:51.493 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.494 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.494 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:58:51.494 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.703783+0000", 2023-10-01T10:58:51.494 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4966999999999999e-05 2023-10-01T10:58:51.494 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.494 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.494 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:58:51.494 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.703798+0000", 2023-10-01T10:58:51.494 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00036441199999999999 2023-10-01T10:58:51.494 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.494 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.495 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:58:51.495 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.704162+0000", 2023-10-01T10:58:51.495 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00010304 2023-10-01T10:58:51.495 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.495 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.495 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:58:51.495 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.704265+0000", 2023-10-01T10:58:51.495 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.9666000000000003e-05 2023-10-01T10:58:51.495 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.495 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.495 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:58:51.495 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.704305+0000", 2023-10-01T10:58:51.496 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.1791000000000001e-05 2023-10-01T10:58:51.496 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.496 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.496 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:58:51.496 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.704336+0000", 2023-10-01T10:58:51.496 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.6644999999999995e-05 2023-10-01T10:58:51.496 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.496 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.496 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:58:51.496 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.704423+0000", 2023-10-01T10:58:51.496 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.152022506 2023-10-01T10:58:51.496 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:58:51.496 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:58:51.497 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:58:51.497 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.497 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.497 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46037.0:177359 11.15 11:ad05a3ed:::benchmark_data_smithi078_157677_object70188:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e203)", 2023-10-01T10:58:51.497 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:54:37.552426+0000", 2023-10-01T10:58:51.497 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 253.846589056, 2023-10-01T10:58:51.497 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1657133159999999, 2023-10-01T10:58:51.497 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:58:51.497 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:58:51.497 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:58:51.497 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46037", 2023-10-01T10:58:51.497 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:58:51.498 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 177359 2023-10-01T10:58:51.498 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.498 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:58:51.498 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.498 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:58:51.498 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.552426+0000", 2023-10-01T10:58:51.498 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:58:51.498 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.498 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.498 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:58:51.498 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.552426+0000", 2023-10-01T10:58:51.498 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999995 2023-10-01T10:58:51.499 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.499 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.499 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:58:51.499 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.552425+0000", 2023-10-01T10:58:51.499 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.9480000000000001e-06 2023-10-01T10:58:51.499 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.499 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.499 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:58:51.499 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.552429+0000", 2023-10-01T10:58:51.499 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.8970000000000008e-06 2023-10-01T10:58:51.499 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.499 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.499 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:58:51.500 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.552437+0000", 2023-10-01T10:58:51.500 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.1329999999999998e-06 2023-10-01T10:58:51.500 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.500 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.500 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:58:51.500 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.552442+0000", 2023-10-01T10:58:51.500 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0020414230000000001 2023-10-01T10:58:51.500 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.500 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.500 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:58:51.500 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.554483+0000", 2023-10-01T10:58:51.500 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.8419999999999999e-06 2023-10-01T10:58:51.500 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.501 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.501 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T10:58:51.501 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.554490+0000", 2023-10-01T10:58:51.501 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1499767110000001 2023-10-01T10:58:51.501 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.501 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.501 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:58:51.501 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.704467+0000", 2023-10-01T10:58:51.501 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00025299099999999999 2023-10-01T10:58:51.501 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.501 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.501 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:58:51.501 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.704720+0000", 2023-10-01T10:58:51.502 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0049821179999999998 2023-10-01T10:58:51.502 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.502 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.502 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1,2,4,6,7", 2023-10-01T10:58:51.502 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.709702+0000", 2023-10-01T10:58:51.502 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0079960039999999993 2023-10-01T10:58:51.502 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.502 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.502 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:58:51.502 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.717698+0000", 2023-10-01T10:58:51.502 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.2432000000000001e-05 2023-10-01T10:58:51.502 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.503 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.503 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:58:51.503 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.717760+0000", 2023-10-01T10:58:51.503 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.3830000000000003e-05 2023-10-01T10:58:51.503 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.503 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.503 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:58:51.503 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.717834+0000", 2023-10-01T10:58:51.503 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.588e-06 2023-10-01T10:58:51.503 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.503 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.503 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:58:51.503 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.717842+0000", 2023-10-01T10:58:51.504 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.8615e-05 2023-10-01T10:58:51.504 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.504 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.504 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:58:51.504 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.717900+0000", 2023-10-01T10:58:51.504 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.7382000000000002e-05 2023-10-01T10:58:51.504 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.504 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.504 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:58:51.504 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.717938+0000", 2023-10-01T10:58:51.504 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.2284999999999999e-05 2023-10-01T10:58:51.504 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.504 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.505 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:58:51.505 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.717980+0000", 2023-10-01T10:58:51.505 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000158756 2023-10-01T10:58:51.505 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.505 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.505 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:58:51.505 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.718139+0000", 2023-10-01T10:58:51.505 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1657133159999999 2023-10-01T10:58:51.505 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:58:51.505 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:58:51.505 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:58:51.505 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.506 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.506 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46037.0:177385 11.15 11:aa5e7b30:::benchmark_data_smithi078_157677_object70214:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e203)", 2023-10-01T10:58:51.506 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:54:37.552447+0000", 2023-10-01T10:58:51.506 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 253.84656767800001, 2023-10-01T10:58:51.506 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1805803370000001, 2023-10-01T10:58:51.506 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:58:51.506 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:58:51.506 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:58:51.506 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46037", 2023-10-01T10:58:51.506 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:58:51.506 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 177385 2023-10-01T10:58:51.506 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.507 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:58:51.507 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.507 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:58:51.507 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.552447+0000", 2023-10-01T10:58:51.507 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:58:51.507 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.507 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.507 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:58:51.507 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.552447+0000", 2023-10-01T10:58:51.507 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999995 2023-10-01T10:58:51.507 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.507 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.507 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:58:51.508 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.552446+0000", 2023-10-01T10:58:51.508 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.9469999999999996e-06 2023-10-01T10:58:51.508 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.508 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.508 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:58:51.508 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.552450+0000", 2023-10-01T10:58:51.508 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.6649999999999997e-06 2023-10-01T10:58:51.508 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.508 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.508 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:58:51.508 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.552457+0000", 2023-10-01T10:58:51.508 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.8330000000000001e-06 2023-10-01T10:58:51.509 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.509 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.509 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:58:51.509 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.552462+0000", 2023-10-01T10:58:51.509 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.002050343 2023-10-01T10:58:51.509 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.509 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.509 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:58:51.509 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.554512+0000", 2023-10-01T10:58:51.509 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.4059999999999999e-06 2023-10-01T10:58:51.509 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.509 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.510 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T10:58:51.510 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.554519+0000", 2023-10-01T10:58:51.510 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1589589629999999 2023-10-01T10:58:51.510 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.510 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.510 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:58:51.510 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.713478+0000", 2023-10-01T10:58:51.510 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00022794199999999999 2023-10-01T10:58:51.510 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.510 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.510 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:58:51.510 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.713706+0000", 2023-10-01T10:58:51.511 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00052423400000000005 2023-10-01T10:58:51.511 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.511 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.511 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1,2,4,6,7", 2023-10-01T10:58:51.511 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.714231+0000", 2023-10-01T10:58:51.511 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.003920064 2023-10-01T10:58:51.511 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.511 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.511 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:58:51.511 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.718151+0000", 2023-10-01T10:58:51.511 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.2490000000000001e-05 2023-10-01T10:58:51.511 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.512 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.512 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:58:51.512 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.718203+0000", 2023-10-01T10:58:51.512 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.2979000000000002e-05 2023-10-01T10:58:51.512 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.512 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.512 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:58:51.512 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.718256+0000", 2023-10-01T10:58:51.512 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.0484000000000002e-05 2023-10-01T10:58:51.512 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.512 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.512 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:58:51.512 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.718287+0000", 2023-10-01T10:58:51.513 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0072039319999999997 2023-10-01T10:58:51.513 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.513 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.513 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:58:51.513 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.725491+0000", 2023-10-01T10:58:51.513 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0072907859999999996 2023-10-01T10:58:51.513 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.513 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.513 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:58:51.513 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.732781+0000", 2023-10-01T10:58:51.513 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.0488999999999997e-05 2023-10-01T10:58:51.513 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.513 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.514 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:58:51.514 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.732832+0000", 2023-10-01T10:58:51.514 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000195424 2023-10-01T10:58:51.514 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.514 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.514 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:58:51.514 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.733027+0000", 2023-10-01T10:58:51.514 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1805803370000001 2023-10-01T10:58:51.514 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:58:51.514 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:58:51.514 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:58:51.514 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.515 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.515 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46037.0:177507 11.15 11:ae1399bb:::benchmark_data_smithi078_157677_object70336:head [delete] snapc 0=[] ondisk+write+known_if_redirected e203)", 2023-10-01T10:58:51.515 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:54:37.570482+0000", 2023-10-01T10:58:51.515 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 253.82853246799999, 2023-10-01T10:58:51.515 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1723155279999999, 2023-10-01T10:58:51.515 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:58:51.515 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:58:51.515 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:58:51.515 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46037", 2023-10-01T10:58:51.515 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:58:51.515 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 177507 2023-10-01T10:58:51.515 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.516 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:58:51.516 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.516 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:58:51.516 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.570482+0000", 2023-10-01T10:58:51.516 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:58:51.516 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.516 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.516 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:58:51.516 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.570482+0000", 2023-10-01T10:58:51.516 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999924 2023-10-01T10:58:51.516 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.516 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.517 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:58:51.517 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.570474+0000", 2023-10-01T10:58:51.517 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.6597000000000001e-05 2023-10-01T10:58:51.517 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.517 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.517 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:58:51.517 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.570491+0000", 2023-10-01T10:58:51.517 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.3068999999999999e-05 2023-10-01T10:58:51.517 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.517 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.517 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:58:51.517 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.570514+0000", 2023-10-01T10:58:51.518 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4055e-05 2023-10-01T10:58:51.518 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.518 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.518 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:58:51.518 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.570528+0000", 2023-10-01T10:58:51.518 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.1231999999999998e-05 2023-10-01T10:58:51.518 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.518 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.518 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:58:51.518 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.570599+0000", 2023-10-01T10:58:51.518 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.6036e-05 2023-10-01T10:58:51.518 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.519 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.519 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T10:58:51.519 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.570615+0000", 2023-10-01T10:58:51.519 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1477053150000001 2023-10-01T10:58:51.519 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.519 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.519 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:58:51.519 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.718321+0000", 2023-10-01T10:58:51.519 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00018349100000000001 2023-10-01T10:58:51.519 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.519 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.519 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:58:51.520 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.718504+0000", 2023-10-01T10:58:51.520 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00056659699999999996 2023-10-01T10:58:51.520 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.520 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.520 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1,2,4,6,7", 2023-10-01T10:58:51.520 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.719071+0000", 2023-10-01T10:58:51.520 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0064289919999999997 2023-10-01T10:58:51.520 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.520 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.520 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:58:51.520 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.725500+0000", 2023-10-01T10:58:51.520 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.2632e-05 2023-10-01T10:58:51.520 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.521 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.521 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:58:51.521 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.725553+0000", 2023-10-01T10:58:51.521 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.0664999999999998e-05 2023-10-01T10:58:51.521 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.521 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.521 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:58:51.521 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.725593+0000", 2023-10-01T10:58:51.521 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0074954160000000004 2023-10-01T10:58:51.521 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.521 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.521 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:58:51.521 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.733089+0000", 2023-10-01T10:58:51.522 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.4382999999999995e-05 2023-10-01T10:58:51.522 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.522 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.522 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:58:51.522 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.733173+0000", 2023-10-01T10:58:51.522 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0093188639999999996 2023-10-01T10:58:51.522 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.522 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.522 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:58:51.522 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.742492+0000", 2023-10-01T10:58:51.522 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.2553000000000006e-05 2023-10-01T10:58:51.522 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.523 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.523 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:58:51.523 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.742584+0000", 2023-10-01T10:58:51.523 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00021324000000000001 2023-10-01T10:58:51.523 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.523 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.523 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:58:51.523 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.742798+0000", 2023-10-01T10:58:51.523 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1723155279999999 2023-10-01T10:58:51.523 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:58:51.523 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:58:51.523 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:58:51.524 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.524 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.524 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46037.0:177511 11.15 11:abaf81a2:::benchmark_data_smithi078_157677_object70340:head [delete] snapc 0=[] ondisk+write+known_if_redirected e203)", 2023-10-01T10:58:51.524 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:54:37.592417+0000", 2023-10-01T10:58:51.524 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 253.806598065, 2023-10-01T10:58:51.524 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.150762976, 2023-10-01T10:58:51.524 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:58:51.524 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:58:51.524 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:58:51.524 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46037", 2023-10-01T10:58:51.524 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:58:51.524 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 177511 2023-10-01T10:58:51.524 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.525 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:58:51.525 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.525 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:58:51.525 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.592417+0000", 2023-10-01T10:58:51.525 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:58:51.525 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.525 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.525 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:58:51.525 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.592417+0000", 2023-10-01T10:58:51.525 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-10-01T10:58:51.525 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.525 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.526 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:58:51.526 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.592415+0000", 2023-10-01T10:58:51.526 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.0490000000000004e-06 2023-10-01T10:58:51.526 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.526 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.526 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:58:51.526 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.592423+0000", 2023-10-01T10:58:51.526 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.9443999999999999e-05 2023-10-01T10:58:51.526 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.526 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.526 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:58:51.526 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.592442+0000", 2023-10-01T10:58:51.527 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.3550000000000005e-06 2023-10-01T10:58:51.527 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.527 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.527 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:58:51.527 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.592451+0000", 2023-10-01T10:58:51.527 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.0699000000000003e-05 2023-10-01T10:58:51.527 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.527 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.527 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:58:51.527 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.592511+0000", 2023-10-01T10:58:51.527 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2172e-05 2023-10-01T10:58:51.528 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.528 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.528 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T10:58:51.528 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.592523+0000", 2023-10-01T10:58:51.528 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.129729274 2023-10-01T10:58:51.528 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.528 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.528 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:58:51.528 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.722253+0000", 2023-10-01T10:58:51.528 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00011512 2023-10-01T10:58:51.528 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.529 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.529 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:58:51.529 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.722368+0000", 2023-10-01T10:58:51.529 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00042662100000000002 2023-10-01T10:58:51.529 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.529 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.529 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1,2,4,6,7", 2023-10-01T10:58:51.529 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.722794+0000", 2023-10-01T10:58:51.529 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0028398400000000002 2023-10-01T10:58:51.529 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.529 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.529 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:58:51.530 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.725634+0000", 2023-10-01T10:58:51.530 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.4036999999999999e-05 2023-10-01T10:58:51.530 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.530 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.530 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:58:51.530 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.725678+0000", 2023-10-01T10:58:51.530 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0070478199999999998 2023-10-01T10:58:51.530 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.530 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.530 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:58:51.530 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.732726+0000", 2023-10-01T10:58:51.530 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00048971599999999996 2023-10-01T10:58:51.531 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.531 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.531 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:58:51.531 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.733216+0000", 2023-10-01T10:58:51.531 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.8441000000000001e-05 2023-10-01T10:58:51.531 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.531 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.531 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:58:51.531 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.733244+0000", 2023-10-01T10:58:51.531 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0096583989999999998 2023-10-01T10:58:51.531 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.531 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.531 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:58:51.532 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.742903+0000", 2023-10-01T10:58:51.532 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.9646000000000004e-05 2023-10-01T10:58:51.532 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.532 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.532 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:58:51.532 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.742972+0000", 2023-10-01T10:58:51.532 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00020713599999999999 2023-10-01T10:58:51.532 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.532 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.532 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:58:51.532 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.743179+0000", 2023-10-01T10:58:51.532 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.150762976 2023-10-01T10:58:51.533 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:58:51.533 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:58:51.533 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:58:51.533 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.533 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.533 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46037.0:177601 11.15 11:abde5588:::benchmark_data_smithi078_157677_object70430:head [delete] snapc 0=[] ondisk+write+known_if_redirected e203)", 2023-10-01T10:58:51.533 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:54:37.722609+0000", 2023-10-01T10:58:51.533 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 253.67640514600001, 2023-10-01T10:58:51.533 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.020872121, 2023-10-01T10:58:51.533 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:58:51.533 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:58:51.533 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:58:51.534 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46037", 2023-10-01T10:58:51.534 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:58:51.534 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 177601 2023-10-01T10:58:51.534 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.534 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:58:51.534 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.534 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:58:51.534 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.722609+0000", 2023-10-01T10:58:51.534 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:58:51.534 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.534 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.534 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:58:51.534 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.722609+0000", 2023-10-01T10:58:51.535 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999976 2023-10-01T10:58:51.535 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.535 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.535 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:58:51.535 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.722607+0000", 2023-10-01T10:58:51.535 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1804000000000001e-05 2023-10-01T10:58:51.535 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.535 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.535 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:58:51.535 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.722619+0000", 2023-10-01T10:58:51.535 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.5117e-05 2023-10-01T10:58:51.535 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.535 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.536 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:58:51.536 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.722644+0000", 2023-10-01T10:58:51.536 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2021999999999999e-05 2023-10-01T10:58:51.536 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.536 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.536 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:58:51.536 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.722656+0000", 2023-10-01T10:58:51.536 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000121989 2023-10-01T10:58:51.536 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.536 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.536 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:58:51.536 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.722778+0000", 2023-10-01T10:58:51.537 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.3059999999999999e-05 2023-10-01T10:58:51.537 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.537 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.537 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T10:58:51.537 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.722801+0000", 2023-10-01T10:58:51.537 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0029135360000001 2023-10-01T10:58:51.537 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.537 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.537 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:58:51.537 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.725715+0000", 2023-10-01T10:58:51.537 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000148961 2023-10-01T10:58:51.537 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.537 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.538 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:58:51.538 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.725863+0000", 2023-10-01T10:58:51.538 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00048295100000000002 2023-10-01T10:58:51.538 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.538 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.538 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1,2,4,6,7", 2023-10-01T10:58:51.538 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.726346+0000", 2023-10-01T10:58:51.538 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.006698028 2023-10-01T10:58:51.538 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.538 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.538 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:58:51.538 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.733044+0000", 2023-10-01T10:58:51.539 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.6065999999999999e-05 2023-10-01T10:58:51.539 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.539 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.539 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:58:51.539 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.733131+0000", 2023-10-01T10:58:51.539 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00014932300000000001 2023-10-01T10:58:51.539 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.539 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.539 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:58:51.539 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.733280+0000", 2023-10-01T10:58:51.539 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.1257e-05 2023-10-01T10:58:51.539 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.540 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.540 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:58:51.540 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.733301+0000", 2023-10-01T10:58:51.540 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.9737000000000001e-05 2023-10-01T10:58:51.540 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.540 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.540 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:58:51.540 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.733321+0000", 2023-10-01T10:58:51.540 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0099182720000000005 2023-10-01T10:58:51.540 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.540 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.540 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:58:51.540 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.743239+0000", 2023-10-01T10:58:51.541 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.5034999999999997e-05 2023-10-01T10:58:51.541 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.541 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.541 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:58:51.541 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.743294+0000", 2023-10-01T10:58:51.541 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00018738899999999999 2023-10-01T10:58:51.541 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.541 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.541 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:58:51.541 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.743482+0000", 2023-10-01T10:58:51.541 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.020872121 2023-10-01T10:58:51.541 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:58:51.541 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:58:51.542 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:58:51.542 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.542 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.542 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46037.0:177611 11.15 11:aa25f454:::benchmark_data_smithi078_157677_object70440:head [delete] snapc 0=[] ondisk+write+known_if_redirected e203)", 2023-10-01T10:58:51.542 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:54:37.742316+0000", 2023-10-01T10:58:51.542 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 253.65669903200001, 2023-10-01T10:58:51.542 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.001495657, 2023-10-01T10:58:51.542 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:58:51.542 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:58:51.542 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:58:51.542 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46037", 2023-10-01T10:58:51.542 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:58:51.543 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 177611 2023-10-01T10:58:51.543 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.543 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:58:51.543 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.543 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:58:51.543 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.742316+0000", 2023-10-01T10:58:51.543 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:58:51.543 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.543 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.543 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:58:51.543 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.742316+0000", 2023-10-01T10:58:51.543 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-10-01T10:58:51.544 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.544 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.544 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:58:51.544 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.742314+0000", 2023-10-01T10:58:51.544 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.2490000000000004e-06 2023-10-01T10:58:51.544 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.544 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.544 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:58:51.544 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.742321+0000", 2023-10-01T10:58:51.544 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.6172999999999999e-05 2023-10-01T10:58:51.544 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.544 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.544 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:58:51.545 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.742337+0000", 2023-10-01T10:58:51.545 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.7919999999999993e-06 2023-10-01T10:58:51.545 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.545 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.545 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:58:51.545 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.742345+0000", 2023-10-01T10:58:51.545 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.2776999999999998e-05 2023-10-01T10:58:51.545 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.545 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.545 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:58:51.545 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.742418+0000", 2023-10-01T10:58:51.545 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4798e-05 2023-10-01T10:58:51.545 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.546 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.546 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T10:58:51.546 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.742433+0000", 2023-10-01T10:58:51.546 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.98722147500000001 2023-10-01T10:58:51.546 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.546 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.546 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:58:51.546 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.729654+0000", 2023-10-01T10:58:51.546 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.4846000000000006e-05 2023-10-01T10:58:51.546 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.546 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.546 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:58:51.547 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.729749+0000", 2023-10-01T10:58:51.547 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00041996299999999998 2023-10-01T10:58:51.547 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.547 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.547 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1,2,4,6,7", 2023-10-01T10:58:51.547 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.730169+0000", 2023-10-01T10:58:51.547 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0031853910000000001 2023-10-01T10:58:51.547 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.547 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.547 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:58:51.547 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.733354+0000", 2023-10-01T10:58:51.547 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.1692000000000001e-05 2023-10-01T10:58:51.547 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.548 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.548 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:58:51.548 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.733386+0000", 2023-10-01T10:58:51.548 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.6009e-05 2023-10-01T10:58:51.548 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.548 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.548 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:58:51.548 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.733422+0000", 2023-10-01T10:58:51.548 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.8815e-05 2023-10-01T10:58:51.548 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.548 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.548 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:58:51.549 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.733461+0000", 2023-10-01T10:58:51.549 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0058218419999999998 2023-10-01T10:58:51.549 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.549 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.549 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:58:51.549 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.739283+0000", 2023-10-01T10:58:51.549 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0043028529999999997 2023-10-01T10:58:51.549 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.549 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.549 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:58:51.549 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.743585+0000", 2023-10-01T10:58:51.549 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.8921000000000001e-05 2023-10-01T10:58:51.549 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.550 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.550 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:58:51.550 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.743634+0000", 2023-10-01T10:58:51.550 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00017684499999999999 2023-10-01T10:58:51.550 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.550 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.550 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:58:51.550 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.743811+0000", 2023-10-01T10:58:51.550 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.001495657 2023-10-01T10:58:51.550 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:58:51.550 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:58:51.550 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:58:51.551 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.551 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.551 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46037.0:177692 11.15 11:af830584:::benchmark_data_smithi078_157677_object70521:head [delete] snapc 0=[] ondisk+write+known_if_redirected e203)", 2023-10-01T10:58:51.551 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:54:37.852540+0000", 2023-10-01T10:58:51.551 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 253.546474596, 2023-10-01T10:58:51.551 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.891678843, 2023-10-01T10:58:51.551 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:58:51.551 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:58:51.551 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:58:51.551 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46037", 2023-10-01T10:58:51.551 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:58:51.551 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 177692 2023-10-01T10:58:51.552 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.552 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:58:51.552 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.552 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:58:51.552 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.852540+0000", 2023-10-01T10:58:51.552 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:58:51.552 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.552 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.552 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:58:51.552 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.852540+0000", 2023-10-01T10:58:51.552 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-10-01T10:58:51.552 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.552 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.553 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:58:51.553 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.852538+0000", 2023-10-01T10:58:51.553 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0111e-05 2023-10-01T10:58:51.553 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.553 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.553 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:58:51.553 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.852549+0000", 2023-10-01T10:58:51.553 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.3579e-05 2023-10-01T10:58:51.553 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.553 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.553 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:58:51.553 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.852562+0000", 2023-10-01T10:58:51.554 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.7119999999999997e-06 2023-10-01T10:58:51.554 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.554 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.554 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:58:51.554 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.852568+0000", 2023-10-01T10:58:51.554 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:51.555 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.6125999999999998e-05 2023-10-01T10:58:51.555 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.556 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.556 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:58:51.556 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.852624+0000", 2023-10-01T10:58:51.556 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.8140000000000007e-06 2023-10-01T10:58:51.556 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.556 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.556 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T10:58:51.556 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.852633+0000", 2023-10-01T10:58:51.556 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.88084269000000004 2023-10-01T10:58:51.556 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.556 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.556 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:58:51.557 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.733475+0000", 2023-10-01T10:58:51.557 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000119584 2023-10-01T10:58:51.557 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.557 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.557 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:58:51.557 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.733595+0000", 2023-10-01T10:58:51.557 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00047065399999999998 2023-10-01T10:58:51.557 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.557 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.557 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1,2,4,6,7", 2023-10-01T10:58:51.557 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.734066+0000", 2023-10-01T10:58:51.557 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0052662550000000001 2023-10-01T10:58:51.558 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.558 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.558 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:58:51.558 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.739332+0000", 2023-10-01T10:58:51.558 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.1670000000000005e-06 2023-10-01T10:58:51.558 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.558 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.558 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:58:51.558 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.739341+0000", 2023-10-01T10:58:51.558 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.4882999999999997e-05 2023-10-01T10:58:51.558 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.558 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.559 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:58:51.559 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.739386+0000", 2023-10-01T10:58:51.559 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.3299e-05 2023-10-01T10:58:51.559 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.559 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.559 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:58:51.559 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.739429+0000", 2023-10-01T10:58:51.559 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.2058000000000003e-05 2023-10-01T10:58:51.559 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.559 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.559 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:58:51.559 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.739471+0000", 2023-10-01T10:58:51.560 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0044855490000000001 2023-10-01T10:58:51.560 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.560 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.560 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:58:51.560 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.743957+0000", 2023-10-01T10:58:51.560 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.0143999999999998e-05 2023-10-01T10:58:51.560 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.560 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.560 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:58:51.560 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.744027+0000", 2023-10-01T10:58:51.560 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00019173399999999999 2023-10-01T10:58:51.560 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.560 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.561 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:58:51.561 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.744219+0000", 2023-10-01T10:58:51.561 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.891678843 2023-10-01T10:58:51.561 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:58:51.561 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:58:51.561 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:58:51.561 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.561 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.561 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46037.0:177786 11.15 11:ac1e9503:::benchmark_data_smithi078_157677_object70615:head [delete] snapc 0=[] ondisk+write+known_if_redirected e203)", 2023-10-01T10:58:51.561 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:54:37.995551+0000", 2023-10-01T10:58:51.561 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 253.40346376100001, 2023-10-01T10:58:51.561 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.753596936, 2023-10-01T10:58:51.562 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:58:51.562 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:58:51.562 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:58:51.562 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46037", 2023-10-01T10:58:51.562 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:58:51.562 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 177786 2023-10-01T10:58:51.562 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.562 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:58:51.562 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.562 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:58:51.562 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.995551+0000", 2023-10-01T10:58:51.562 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:58:51.563 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.563 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.563 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:58:51.563 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.995551+0000", 2023-10-01T10:58:51.563 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-10-01T10:58:51.563 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.563 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.563 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:58:51.563 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.995549+0000", 2023-10-01T10:58:51.563 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.5089999999999999e-06 2023-10-01T10:58:51.563 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.563 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.564 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:58:51.564 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.995558+0000", 2023-10-01T10:58:51.564 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.8289999999999999e-05 2023-10-01T10:58:51.564 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.564 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.564 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:58:51.564 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.995577+0000", 2023-10-01T10:58:51.564 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.9919999999999996e-06 2023-10-01T10:58:51.564 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.564 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.564 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:58:51.564 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.995586+0000", 2023-10-01T10:58:51.564 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.3052999999999994e-05 2023-10-01T10:58:51.565 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.565 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.565 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:58:51.565 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.995649+0000", 2023-10-01T10:58:51.565 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1875999999999999e-05 2023-10-01T10:58:51.565 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.565 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.565 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T10:58:51.565 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:37.995661+0000", 2023-10-01T10:58:51.565 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.74070833000000003 2023-10-01T10:58:51.565 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.566 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.566 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:58:51.566 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.736369+0000", 2023-10-01T10:58:51.566 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000118622 2023-10-01T10:58:51.566 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.566 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.566 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:58:51.566 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.736488+0000", 2023-10-01T10:58:51.566 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00041400299999999997 2023-10-01T10:58:51.566 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.566 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.567 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1,2,4,6,7", 2023-10-01T10:58:51.567 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.736902+0000", 2023-10-01T10:58:51.567 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0026319020000000002 2023-10-01T10:58:51.567 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.567 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.567 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:58:51.567 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.739533+0000", 2023-10-01T10:58:51.567 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0028985109999999999 2023-10-01T10:58:51.567 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.567 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.567 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:58:51.567 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.742432+0000", 2023-10-01T10:58:51.568 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2911000000000001e-05 2023-10-01T10:58:51.568 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.568 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.568 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:58:51.568 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.742445+0000", 2023-10-01T10:58:51.568 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00041232200000000001 2023-10-01T10:58:51.568 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.568 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.568 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:58:51.568 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.742857+0000", 2023-10-01T10:58:51.568 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0010021539999999999 2023-10-01T10:58:51.568 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.568 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.569 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:58:51.569 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.743859+0000", 2023-10-01T10:58:51.569 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0050862440000000002 2023-10-01T10:58:51.569 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.569 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.569 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:58:51.569 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.748946+0000", 2023-10-01T10:58:51.569 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.8140999999999999e-05 2023-10-01T10:58:51.569 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.569 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.569 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:58:51.569 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.748994+0000", 2023-10-01T10:58:51.570 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000153976 2023-10-01T10:58:51.570 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.570 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.570 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:58:51.570 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.749148+0000", 2023-10-01T10:58:51.570 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.753596936 2023-10-01T10:58:51.570 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:58:51.570 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:58:51.570 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:58:51.570 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.570 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.570 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46037.0:177806 11.15 11:ae49c9de:::benchmark_data_smithi078_157677_object70635:head [delete] snapc 0=[] ondisk+write+known_if_redirected e203)", 2023-10-01T10:58:51.571 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:54:38.029064+0000", 2023-10-01T10:58:51.571 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 253.369950235, 2023-10-01T10:58:51.571 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.74026546900000001, 2023-10-01T10:58:51.571 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:58:51.571 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T10:58:51.571 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:58:51.571 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46037", 2023-10-01T10:58:51.571 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:58:51.571 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 177806 2023-10-01T10:58:51.571 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.573 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:58:51.573 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.573 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:58:51.573 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.029064+0000", 2023-10-01T10:58:51.573 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:58:51.573 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.573 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.573 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:58:51.574 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.029064+0000", 2023-10-01T10:58:51.574 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999976 2023-10-01T10:58:51.574 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.574 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.574 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:58:51.574 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.029062+0000", 2023-10-01T10:58:51.574 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1443999999999999e-05 2023-10-01T10:58:51.574 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.574 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.574 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:58:51.574 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.029073+0000", 2023-10-01T10:58:51.574 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.2702999999999999e-05 2023-10-01T10:58:51.575 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.575 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.575 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:58:51.575 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.029096+0000", 2023-10-01T10:58:51.575 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.3810000000000005e-06 2023-10-01T10:58:51.575 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.575 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.575 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:58:51.575 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.029106+0000", 2023-10-01T10:58:51.575 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.1379999999999998e-05 2023-10-01T10:58:51.575 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.575 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.575 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:58:51.576 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.029177+0000", 2023-10-01T10:58:51.576 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.7822999999999999e-05 2023-10-01T10:58:51.576 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.576 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.576 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T10:58:51.576 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.029195+0000", 2023-10-01T10:58:51.576 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.720037382 2023-10-01T10:58:51.576 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.576 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.576 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:58:51.576 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.749232+0000", 2023-10-01T10:58:51.576 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00012164599999999999 2023-10-01T10:58:51.577 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.577 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.577 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T10:58:51.577 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.749354+0000", 2023-10-01T10:58:51.577 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00054277700000000004 2023-10-01T10:58:51.577 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.577 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.577 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1,2,4,6,7", 2023-10-01T10:58:51.577 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.749897+0000", 2023-10-01T10:58:51.577 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.010329171 2023-10-01T10:58:51.577 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.577 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.578 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T10:58:51.578 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.760226+0000", 2023-10-01T10:58:51.578 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0084307789999999994 2023-10-01T10:58:51.578 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.578 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.578 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:58:51.578 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.768657+0000", 2023-10-01T10:58:51.578 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.3720999999999995e-05 2023-10-01T10:58:51.578 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.578 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.578 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:58:51.578 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.768720+0000", 2023-10-01T10:58:51.579 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.6632999999999997e-05 2023-10-01T10:58:51.579 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.579 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.579 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:58:51.579 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.768787+0000", 2023-10-01T10:58:51.579 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000111875 2023-10-01T10:58:51.579 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.579 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.579 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:58:51.579 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.768899+0000", 2023-10-01T10:58:51.579 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00026668900000000002 2023-10-01T10:58:51.579 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.580 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.580 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T10:58:51.580 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.769165+0000", 2023-10-01T10:58:51.580 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.9293999999999999e-05 2023-10-01T10:58:51.580 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.580 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.580 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T10:58:51.580 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.769215+0000", 2023-10-01T10:58:51.580 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00011505 2023-10-01T10:58:51.580 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.580 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.580 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:58:51.581 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.769330+0000", 2023-10-01T10:58:51.581 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.74026546900000001 2023-10-01T10:58:51.581 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:58:51.581 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:58:51.581 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:58:51.581 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.581 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.581 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46037.0:177895 11.16 11.e255ca56 (undecoded) ondisk+write+known_if_redirected e204)", 2023-10-01T10:58:51.581 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:54:38.733290+0000", 2023-10-01T10:58:51.581 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 252.665724579, 2023-10-01T10:58:51.581 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.83136820600000005, 2023-10-01T10:58:51.581 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:58:51.581 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "reached pg", 2023-10-01T10:58:51.582 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:58:51.582 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46037", 2023-10-01T10:58:51.582 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:58:51.582 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 177895 2023-10-01T10:58:51.582 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.582 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:58:51.582 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.582 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:58:51.582 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.733290+0000", 2023-10-01T10:58:51.582 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:58:51.582 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.582 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.583 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:58:51.583 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.733290+0000", 2023-10-01T10:58:51.583 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-10-01T10:58:51.583 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.583 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.583 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:58:51.583 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.733289+0000", 2023-10-01T10:58:51.583 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.404e-06 2023-10-01T10:58:51.583 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.583 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.583 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:58:51.583 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.733295+0000", 2023-10-01T10:58:51.584 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2098e-05 2023-10-01T10:58:51.584 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.584 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.584 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:58:51.584 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.733307+0000", 2023-10-01T10:58:51.584 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.0870000000000001e-06 2023-10-01T10:58:51.584 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.584 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.584 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:58:51.584 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.733314+0000", 2023-10-01T10:58:51.584 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.5364999999999997e-05 2023-10-01T10:58:51.584 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.585 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.585 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:58:51.585 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.733360+0000", 2023-10-01T10:58:51.585 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.3960000000000006e-06 2023-10-01T10:58:51.585 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.585 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.585 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T10:58:51.585 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.733368+0000", 2023-10-01T10:58:51.585 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.831265264 2023-10-01T10:58:51.585 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.585 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.585 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:58:51.585 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:39.564633+0000", 2023-10-01T10:58:51.586 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.4975999999999999e-05 2023-10-01T10:58:51.586 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.586 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.586 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:58:51.586 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:39.564658+0000", 2023-10-01T10:58:51.586 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.83136820600000005 2023-10-01T10:58:51.586 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:58:51.586 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:58:51.586 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:58:51.586 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.586 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.586 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46037.0:177960 11.16 11.de4cf0f6 (undecoded) ondisk+write+known_if_redirected e204)", 2023-10-01T10:58:51.587 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:54:38.813973+0000", 2023-10-01T10:58:51.587 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 252.58504201100001, 2023-10-01T10:58:51.587 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.75079969300000005, 2023-10-01T10:58:51.587 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T10:58:51.587 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "reached pg", 2023-10-01T10:58:51.587 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T10:58:51.587 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46037", 2023-10-01T10:58:51.587 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T10:58:51.587 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 177960 2023-10-01T10:58:51.587 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.587 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T10:58:51.587 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.588 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T10:58:51.588 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.813973+0000", 2023-10-01T10:58:51.588 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T10:58:51.588 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.588 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.588 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T10:58:51.588 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.813973+0000", 2023-10-01T10:58:51.588 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-10-01T10:58:51.588 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.588 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.588 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T10:58:51.588 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.813971+0000", 2023-10-01T10:58:51.589 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.0589999999999999e-06 2023-10-01T10:58:51.589 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.589 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.589 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T10:58:51.589 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.813977+0000", 2023-10-01T10:58:51.589 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0557000000000001e-05 2023-10-01T10:58:51.589 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.589 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.589 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T10:58:51.589 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.813988+0000", 2023-10-01T10:58:51.589 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.6200000000000004e-06 2023-10-01T10:58:51.589 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.590 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.590 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T10:58:51.590 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.813993+0000", 2023-10-01T10:58:51.590 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.9910000000000002e-05 2023-10-01T10:58:51.590 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.590 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.590 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:58:51.590 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.814033+0000", 2023-10-01T10:58:51.590 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.2099999999999998e-06 2023-10-01T10:58:51.590 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.590 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.590 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T10:58:51.591 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:38.814040+0000", 2023-10-01T10:58:51.591 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.75070671200000005 2023-10-01T10:58:51.591 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.591 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.591 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T10:58:51.591 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:39.564746+0000", 2023-10-01T10:58:51.591 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.5922e-05 2023-10-01T10:58:51.591 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T10:58:51.591 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T10:58:51.591 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T10:58:51.591 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:54:39.564772+0000", 2023-10-01T10:58:51.591 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.75079969300000005 2023-10-01T10:58:51.592 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:58:51.592 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:58:51.592 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:58:51.592 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T10:58:51.592 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T10:58:51.592 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-10-01T10:58:51.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:51.594 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:51.594 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:58:51.694 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:51.717 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:51.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:51.722 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:58:51.822 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:51.854 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:51.858 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:51 smithi173 bash[58816]: cluster 2023-10-01T10:58:50.125815+0000 mgr.y (mgr.25072) 1242 : cluster 0 pgmap v1257: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 674 op/s 2023-10-01T10:58:51.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:51.859 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:58:51.960 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:51.983 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:51.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:51.985 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:58:52.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:51 smithi078 bash[75299]: cluster 2023-10-01T10:58:50.125815+0000 mgr.y (mgr.25072) 1242 : cluster 0 pgmap v1257: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 674 op/s 2023-10-01T10:58:52.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:51 smithi078 bash[78274]: cluster 2023-10-01T10:58:50.125815+0000 mgr.y (mgr.25072) 1242 : cluster 0 pgmap v1257: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 674 op/s 2023-10-01T10:58:52.086 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:52.110 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:52.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:52.112 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:58:52.213 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:52.246 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:52.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:52.248 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:58:52.349 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:52.371 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:52.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:52.373 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:58:52.396 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-01T10:58:52.474 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:52.515 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:52.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:52.520 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:58:52.621 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:52.659 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:52.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:52.663 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:58:52.764 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:52.790 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:52.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:52.793 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:58:52.894 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:52.919 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:52.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:52.926 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:58:53.027 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:53.058 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:53.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:53.064 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:58:53.164 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:53.198 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:53.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:53.203 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:58:53.304 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:53.328 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:53.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:53.332 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:58:53.432 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:53.456 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:53.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:53.462 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:58:53.563 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:53.607 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:53.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:53.611 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:58:53.712 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:53.735 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:53.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:53.743 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:58:53.844 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:53.848 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:53 smithi173 bash[58816]: cluster 2023-10-01T10:58:52.126338+0000 mgr.y (mgr.25072) 1243 : cluster 0 pgmap v1258: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 596 op/s 2023-10-01T10:58:53.872 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:53.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:53.875 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:58:53.976 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:54.009 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:54.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:54.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:58:54.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:53 smithi078 bash[78274]: cluster 2023-10-01T10:58:52.126338+0000 mgr.y (mgr.25072) 1243 : cluster 0 pgmap v1258: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 596 op/s 2023-10-01T10:58:54.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:53 smithi078 bash[75299]: cluster 2023-10-01T10:58:52.126338+0000 mgr.y (mgr.25072) 1243 : cluster 0 pgmap v1258: 89 pgs: 89 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 596 op/s 2023-10-01T10:58:54.113 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:54.140 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:54.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:54.143 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:58:54.243 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:54.269 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:54.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:54.277 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:58:54.378 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:54.413 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:54.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:54.419 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:58:54.520 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:54.549 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:54.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:54.552 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:58:54.652 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:54.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:54.693 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:54.693 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:58:54.794 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:54.818 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:54.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:54.822 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:58:54.923 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:54.954 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:54.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:54.957 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:58:55.057 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:55.091 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:55.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:55.096 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:58:55.197 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:55.231 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:55.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:55.234 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:58:55.334 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:55.367 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:55.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:55.369 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:58:55.470 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:55.496 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:55.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:55.502 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:58:55.603 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:55.635 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:55.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:55.642 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:58:55.743 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:55.772 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:55.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:55.775 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:58:55.876 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:55.884 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:55 smithi078 bash[75299]: cluster 2023-10-01T10:58:54.126934+0000 mgr.y (mgr.25072) 1244 : cluster 0 pgmap v1259: 89 pgs: 89 active+clean; 4.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 858 op/s 2023-10-01T10:58:55.884 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:55 smithi078 bash[78274]: cluster 2023-10-01T10:58:54.126934+0000 mgr.y (mgr.25072) 1244 : cluster 0 pgmap v1259: 89 pgs: 89 active+clean; 4.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 858 op/s 2023-10-01T10:58:55.909 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:55.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:55.912 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:58:56.013 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:56.022 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:55 smithi173 bash[58816]: cluster 2023-10-01T10:58:54.126934+0000 mgr.y (mgr.25072) 1244 : cluster 0 pgmap v1259: 89 pgs: 89 active+clean; 4.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 858 op/s 2023-10-01T10:58:56.051 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:56.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:56.053 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:58:56.154 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:56.177 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:56.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:56.179 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:58:56.280 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:56.304 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:56.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:56.306 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:58:56.407 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:56.435 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:56.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:56.438 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:58:56.539 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:56.566 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:56.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:56.568 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:58:56.669 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:56.690 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:56.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:56.694 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:58:56.795 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:56.831 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:56.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:56.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:58:56.934 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:56.964 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:56.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:56.971 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:58:57.071 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:57.101 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:57.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:57.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:58:57.208 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:57.225 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T10:58:57.251 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:57.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:57.256 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:58:57.357 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:57.392 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:57.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:57.395 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:58:57.496 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:57.527 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:57.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:57.530 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:58:57.631 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:57.659 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:57.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:57.662 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:58:57.762 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:57.792 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:57.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:57.794 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:58:57.879 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:57 smithi173 bash[58816]: cluster 2023-10-01T10:58:56.127390+0000 mgr.y (mgr.25072) 1245 : cluster 0 pgmap v1260: 89 pgs: 89 active+clean; 4.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 627 op/s 2023-10-01T10:58:57.895 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:57.918 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:57.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:57.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:58:58.026 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:58.062 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:58:58.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:58.068 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:58:58.169 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:58.200 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:58.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:58.203 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:58:58.304 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:58.313 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:57 smithi078 bash[75299]: cluster 2023-10-01T10:58:56.127390+0000 mgr.y (mgr.25072) 1245 : cluster 0 pgmap v1260: 89 pgs: 89 active+clean; 4.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 627 op/s 2023-10-01T10:58:58.313 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:57 smithi078 bash[78274]: cluster 2023-10-01T10:58:56.127390+0000 mgr.y (mgr.25072) 1245 : cluster 0 pgmap v1260: 89 pgs: 89 active+clean; 4.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 627 op/s 2023-10-01T10:58:58.337 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:58:58.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:58.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:58:58.440 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:58.470 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:58.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:58.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:58:58.576 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:58.603 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:58.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:58.606 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:58:58.707 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:58.738 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:58.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:58.741 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:58:58.842 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:58.865 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:58:58.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:58.867 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:58:58.968 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:58.991 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:58.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:58.993 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:58:59.093 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:59.122 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:58:59.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:59.125 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:58:59.226 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:59.261 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:59.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:59.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:58:59.364 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:59.391 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:58:59.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:59.395 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:58:59.495 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:59.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:59.531 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:58:59.531 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:58:59.632 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:59.661 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:58:59.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:59.665 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:58:59.765 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:59.801 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:59.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:59.808 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:58:59.909 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:59.934 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:58:59.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:58:59.937 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:59:00.037 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:00.057 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:00.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:00.058 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:59:00.060 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:58:59 smithi078 bash[78274]: cluster 2023-10-01T10:58:58.127919+0000 mgr.y (mgr.25072) 1246 : cluster 0 pgmap v1261: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 667 op/s 2023-10-01T10:59:00.061 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:58:59 smithi078 bash[75299]: cluster 2023-10-01T10:58:58.127919+0000 mgr.y (mgr.25072) 1246 : cluster 0 pgmap v1261: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 667 op/s 2023-10-01T10:59:00.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:58:59 smithi173 bash[58816]: cluster 2023-10-01T10:58:58.127919+0000 mgr.y (mgr.25072) 1246 : cluster 0 pgmap v1261: 89 pgs: 89 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 667 op/s 2023-10-01T10:59:00.159 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:00.186 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:00.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:00.192 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:59:00.293 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:00.317 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:59:00 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:59:00] "GET /metrics HTTP/1.1" 200 36385 "" "Prometheus/2.33.4" 2023-10-01T10:59:00.319 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:00.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:00.322 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:59:00.423 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:00.445 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:00.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:00.447 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:00.548 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:00.571 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:00.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:00.576 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:00.677 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:00.721 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:00.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:00.723 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:59:00.824 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:00.861 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:00.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:00.867 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:59:00.926 INFO:teuthology.orchestra.run.smithi078.stdout: 2023-10-01T10:59:00.943 INFO:teuthology.orchestra.run.smithi078.stderr:dumped all 2023-10-01T10:59:00.967 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:01.002 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:01.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:01.008 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:59:01.091 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T10:59:01.109 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:01.149 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T10:59:01.162 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:01.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:01.168 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:59:01.200 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T10:59:01.243 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T10:59:01.269 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:01.294 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T10:59:01.297 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:01.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:01.302 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:59:01.330 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T10:59:01.369 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T10:59:01.403 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:01.412 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T10:59:01.419 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T10:59:01.420 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T10:59:01.420 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T10:59:01.420 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T10:59:01.420 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T10:59:01.420 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T10:59:01.420 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T10:59:01.420 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T10:59:01.428 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:01.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:01.431 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:59:01.531 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:01.553 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:01.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:01.556 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:01.650 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-10-01T10:59:01.650 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T10:59:01.657 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:01.688 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:01.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:01.691 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:59:01.791 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:01.821 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:01.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:01.826 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:59:01.881 INFO:teuthology.orchestra.run.smithi078.stderr:noscrub is unset 2023-10-01T10:59:01.924 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:01 smithi078 bash[78274]: cluster 2023-10-01T10:59:00.128613+0000 mgr.y (mgr.25072) 1247 : cluster 0 pgmap v1262: 89 pgs: 89 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 818 op/s 2023-10-01T10:59:01.925 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:01 smithi078 bash[78274]: audit 2023-10-01T10:59:00.909522+0000 mgr.y (mgr.25072) 1248 : audit 0 from='client.46304 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:59:01.925 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:01 smithi078 bash[78274]: audit 2023-10-01T10:59:01.034882+0000 mon.c (mon.2) 195 : audit 0 from='client.? 172.21.15.78:0/998770008' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:59:01.925 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:01 smithi078 bash[78274]: audit 2023-10-01T10:59:01.401471+0000 mon.a (mon.0) 1541 : audit 1 from='client.? 172.21.15.78:0/2741778675' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T10:59:01.925 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:01 smithi078 bash[75299]: cluster 2023-10-01T10:59:00.128613+0000 mgr.y (mgr.25072) 1247 : cluster 0 pgmap v1262: 89 pgs: 89 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 818 op/s 2023-10-01T10:59:01.925 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:01 smithi078 bash[75299]: audit 2023-10-01T10:59:00.909522+0000 mgr.y (mgr.25072) 1248 : audit 0 from='client.46304 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:59:01.925 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:01 smithi078 bash[75299]: audit 2023-10-01T10:59:01.034882+0000 mon.c (mon.2) 195 : audit 0 from='client.? 172.21.15.78:0/998770008' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:59:01.925 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:01 smithi078 bash[75299]: audit 2023-10-01T10:59:01.401471+0000 mon.a (mon.0) 1541 : audit 1 from='client.? 172.21.15.78:0/2741778675' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T10:59:01.927 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:01.934 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:01 smithi173 bash[58816]: cluster 2023-10-01T10:59:00.128613+0000 mgr.y (mgr.25072) 1247 : cluster 0 pgmap v1262: 89 pgs: 89 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 818 op/s 2023-10-01T10:59:01.934 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:01 smithi173 bash[58816]: audit 2023-10-01T10:59:00.909522+0000 mgr.y (mgr.25072) 1248 : audit 0 from='client.46304 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:59:01.934 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:01 smithi173 bash[58816]: audit 2023-10-01T10:59:01.034882+0000 mon.c (mon.2) 195 : audit 0 from='client.? 172.21.15.78:0/998770008' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:59:01.934 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:01 smithi173 bash[58816]: audit 2023-10-01T10:59:01.401471+0000 mon.a (mon.0) 1541 : audit 1 from='client.? 172.21.15.78:0/2741778675' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T10:59:01.948 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:01.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:01.952 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:02.053 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:02.071 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:02.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:02.073 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:59:02.174 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:02.200 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:02.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:02.205 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:59:02.305 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:02.332 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:02.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:02.339 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:59:02.440 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:02.467 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:02.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:02.472 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:59:02.541 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-01T10:59:02.573 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:02.611 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:02.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:02.615 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:02.716 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:02.746 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:02.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:02.753 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:59:02.801 INFO:teuthology.orchestra.run.smithi078.stdout:{"pg_ready":true,"pg_map":{"version":1262,"stamp":"2023-10-01T10:59:00.128106+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4748679096,"num_objects":72475,"num_object_clones":0,"num_object_copies":217425,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":72475,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":134068,"num_write_kb":6602639,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":1190344,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":131102,"ondisk_log_size":131102,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":19767464,"kb_used_data":14018984,"kb_used_omap":30,"kb_used_meta":5748193,"kb_avail":730193752,"statfs":{"total":767960285184,"available":747718402048,"internally_reserved":0,"allocated":14355439616,"data_stored":14348997951,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31502,"internal_metadata":5886149874},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,4,6,11,6,5,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-643497984,"num_objects":-9819,"num_object_clones":0,"num_object_copies":-29457,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-9819,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9819,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003270"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":243,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:32.810751+0000","last_change":"2023-10-01T10:38:16.032258+0000","last_active":"2023-10-01T10:58:32.810751+0000","last_peered":"2023-10-01T10:58:32.810751+0000","last_clean":"2023-10-01T10:58:32.810751+0000","last_became_active":"2023-10-01T10:38:16.031633+0000","last_became_peered":"2023-10-01T10:38:16.031633+0000","last_unstale":"2023-10-01T10:58:32.810751+0000","last_undegraded":"2023-10-01T10:58:32.810751+0000","last_fullsized":"2023-10-01T10:58:32.810751+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:17:02.446430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":231,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:32.821379+0000","last_change":"2023-10-01T10:38:16.025881+0000","last_active":"2023-10-01T10:58:32.821379+0000","last_peered":"2023-10-01T10:58:32.821379+0000","last_clean":"2023-10-01T10:58:32.821379+0000","last_became_active":"2023-10-01T10:38:16.025518+0000","last_became_peered":"2023-10-01T10:38:16.025518+0000","last_unstale":"2023-10-01T10:58:32.821379+0000","last_undegraded":"2023-10-01T10:58:32.821379+0000","last_fullsized":"2023-10-01T10:58:32.821379+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:48:37.826604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":151,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:32.827946+0000","last_change":"2023-10-01T10:52:18.956544+0000","last_active":"2023-10-01T10:58:32.827946+0000","last_peered":"2023-10-01T10:58:32.827946+0000","last_clean":"2023-10-01T10:58:32.827946+0000","last_became_active":"2023-10-01T10:52:18.956217+0000","last_became_peered":"2023-10-01T10:52:18.956217+0000","last_unstale":"2023-10-01T10:58:32.827946+0000","last_undegraded":"2023-10-01T10:58:32.827946+0000","last_fullsized":"2023-10-01T10:58:32.827946+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:34:01.952235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":215,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:32.816189+0000","last_change":"2023-10-01T10:37:42.772008+0000","last_active":"2023-10-01T10:58:32.816189+0000","last_peered":"2023-10-01T10:58:32.816189+0000","last_clean":"2023-10-01T10:58:32.816189+0000","last_became_active":"2023-10-01T10:37:42.771244+0000","last_became_peered":"2023-10-01T10:37:42.771244+0000","last_unstale":"2023-10-01T10:58:32.816189+0000","last_undegraded":"2023-10-01T10:58:32.816189+0000","last_fullsized":"2023-10-01T10:58:32.816189+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:38:55.180790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":232,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:32.822244+0000","last_change":"2023-10-01T10:37:42.766600+0000","last_active":"2023-10-01T10:58:32.822244+0000","last_peered":"2023-10-01T10:58:32.822244+0000","last_clean":"2023-10-01T10:58:32.822244+0000","last_became_active":"2023-10-01T10:37:42.766051+0000","last_became_peered":"2023-10-01T10:37:42.766051+0000","last_unstale":"2023-10-01T10:58:32.822244+0000","last_undegraded":"2023-10-01T10:58:32.822244+0000","last_fullsized":"2023-10-01T10:58:32.822244+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:51:17.707963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":24,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:32.822573+0000","last_change":"2023-10-01T10:56:14.626119+0000","last_active":"2023-10-01T10:58:32.822573+0000","last_peered":"2023-10-01T10:58:32.822573+0000","last_clean":"2023-10-01T10:58:32.822573+0000","last_became_active":"2023-10-01T10:56:14.625777+0000","last_became_peered":"2023-10-01T10:56:14.625777+0000","last_unstale":"2023-10-01T10:58:32.822573+0000","last_undegraded":"2023-10-01T10:58:32.822573+0000","last_fullsized":"2023-10-01T10:58:32.822573+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":211,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:32.816294+0000","last_change":"2023-10-01T10:37:42.766667+0000","last_active":"2023-10-01T10:58:32.816294+0000","last_peered":"2023-10-01T10:58:32.816294+0000","last_clean":"2023-10-01T10:58:32.816294+0000","last_became_active":"2023-10-01T10:37:42.766369+0000","last_became_peered":"2023-10-01T10:37:42.766369+0000","last_unstale":"2023-10-01T10:58:32.816294+0000","last_undegraded":"2023-10-01T10:58:32.816294+0000","last_fullsized":"2023-10-01T10:58:32.816294+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:02.708788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":235,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-10-01T10:58:38.849911+0000","last_change":"2023-10-01T10:56:14.622767+0000","last_active":"2023-10-01T10:58:38.849911+0000","last_peered":"2023-10-01T10:58:38.849911+0000","last_clean":"2023-10-01T10:58:38.849911+0000","last_became_active":"2023-10-01T10:56:14.622446+0000","last_became_peered":"2023-10-01T10:56:14.622446+0000","last_unstale":"2023-10-01T10:58:38.849911+0000","last_undegraded":"2023-10-01T10:58:38.849911+0000","last_fullsized":"2023-10-01T10:58:38.849911+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:32:09.354415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":216,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:32.815284+0000","last_change":"2023-10-01T10:32:31.579805+0000","last_active":"2023-10-01T10:58:32.815284+0000","last_peered":"2023-10-01T10:58:32.815284+0000","last_clean":"2023-10-01T10:58:32.815284+0000","last_became_active":"2023-10-01T10:32:31.579423+0000","last_became_peered":"2023-10-01T10:32:31.579423+0000","last_unstale":"2023-10-01T10:58:32.815284+0000","last_undegraded":"2023-10-01T10:58:32.815284+0000","last_fullsized":"2023-10-01T10:58:32.815284+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":219,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-10-01T10:58:38.851291+0000","last_change":"2023-10-01T10:56:14.619280+0000","last_active":"2023-10-01T10:58:38.851291+0000","last_peered":"2023-10-01T10:58:38.851291+0000","last_clean":"2023-10-01T10:58:38.851291+0000","last_became_active":"2023-10-01T10:56:14.618892+0000","last_became_peered":"2023-10-01T10:56:14.618892+0000","last_unstale":"2023-10-01T10:58:38.851291+0000","last_undegraded":"2023-10-01T10:58:38.851291+0000","last_fullsized":"2023-10-01T10:58:38.851291+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:14:29.635055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":90,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:32.821354+0000","last_change":"2023-10-01T10:52:43.661650+0000","last_active":"2023-10-01T10:58:32.821354+0000","last_peered":"2023-10-01T10:58:32.821354+0000","last_clean":"2023-10-01T10:58:32.821354+0000","last_became_active":"2023-10-01T10:52:43.661202+0000","last_became_peered":"2023-10-01T10:52:43.661202+0000","last_unstale":"2023-10-01T10:58:32.821354+0000","last_undegraded":"2023-10-01T10:58:32.821354+0000","last_fullsized":"2023-10-01T10:58:32.821354+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:28:35.479124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":24,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:32.822182+0000","last_change":"2023-10-01T10:56:14.620228+0000","last_active":"2023-10-01T10:58:32.822182+0000","last_peered":"2023-10-01T10:58:32.822182+0000","last_clean":"2023-10-01T10:58:32.822182+0000","last_became_active":"2023-10-01T10:56:14.619632+0000","last_became_peered":"2023-10-01T10:56:14.619632+0000","last_unstale":"2023-10-01T10:58:32.822182+0000","last_undegraded":"2023-10-01T10:58:32.822182+0000","last_fullsized":"2023-10-01T10:58:32.822182+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":275,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:32.811033+0000","last_change":"2023-10-01T10:56:14.634698+0000","last_active":"2023-10-01T10:58:32.811033+0000","last_peered":"2023-10-01T10:58:32.811033+0000","last_clean":"2023-10-01T10:58:32.811033+0000","last_became_active":"2023-10-01T10:56:14.625320+0000","last_became_peered":"2023-10-01T10:56:14.625320+0000","last_unstale":"2023-10-01T10:58:32.811033+0000","last_undegraded":"2023-10-01T10:58:32.811033+0000","last_fullsized":"2023-10-01T10:58:32.811033+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:56.480434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":229,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:32.810788+0000","last_change":"2023-10-01T10:56:14.622580+0000","last_active":"2023-10-01T10:58:32.810788+0000","last_peered":"2023-10-01T10:58:32.810788+0000","last_clean":"2023-10-01T10:58:32.810788+0000","last_became_active":"2023-10-01T10:56:14.622075+0000","last_became_peered":"2023-10-01T10:56:14.622075+0000","last_unstale":"2023-10-01T10:58:32.810788+0000","last_undegraded":"2023-10-01T10:58:32.810788+0000","last_fullsized":"2023-10-01T10:58:32.810788+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:52:32.352383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":232,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:32.816037+0000","last_change":"2023-10-01T10:38:16.026777+0000","last_active":"2023-10-01T10:58:32.816037+0000","last_peered":"2023-10-01T10:58:32.816037+0000","last_clean":"2023-10-01T10:58:32.816037+0000","last_became_active":"2023-10-01T10:38:16.026371+0000","last_became_peered":"2023-10-01T10:38:16.026371+0000","last_unstale":"2023-10-01T10:58:32.816037+0000","last_undegraded":"2023-10-01T10:58:32.816037+0000","last_fullsized":"2023-10-01T10:58:32.816037+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":240,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-10-01T10:58:38.849451+0000","last_change":"2023-10-01T10:37:42.772319+0000","last_active":"2023-10-01T10:58:38.849451+0000","last_peered":"2023-10-01T10:58:38.849451+0000","last_clean":"2023-10-01T10:58:38.849451+0000","last_became_active":"2023-10-01T10:37:42.771977+0000","last_became_peered":"2023-10-01T10:37:42.771977+0000","last_unstale":"2023-10-01T10:58:38.849451+0000","last_undegraded":"2023-10-01T10:58:38.849451+0000","last_fullsized":"2023-10-01T10:58:38.849451+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:45:12.783214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":212,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:38.241847+0000","last_change":"2023-10-01T10:37:42.767706+0000","last_active":"2023-10-01T10:58:38.241847+0000","last_peered":"2023-10-01T10:58:38.241847+0000","last_clean":"2023-10-01T10:58:38.241847+0000","last_became_active":"2023-10-01T10:37:42.767211+0000","last_became_peered":"2023-10-01T10:37:42.767211+0000","last_unstale":"2023-10-01T10:58:38.241847+0000","last_undegraded":"2023-10-01T10:58:38.241847+0000","last_fullsized":"2023-10-01T10:58:38.241847+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:37:20.357407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":215,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:32.815473+0000","last_change":"2023-10-01T10:37:42.768642+0000","last_active":"2023-10-01T10:58:32.815473+0000","last_peered":"2023-10-01T10:58:32.815473+0000","last_clean":"2023-10-01T10:58:32.815473+0000","last_became_active":"2023-10-01T10:37:42.768239+0000","last_became_peered":"2023-10-01T10:37:42.768239+0000","last_unstale":"2023-10-01T10:58:32.815473+0000","last_undegraded":"2023-10-01T10:58:32.815473+0000","last_fullsized":"2023-10-01T10:58:32.815473+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":182,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:32.810584+0000","last_change":"2023-10-01T10:56:14.620937+0000","last_active":"2023-10-01T10:58:32.810584+0000","last_peered":"2023-10-01T10:58:32.810584+0000","last_clean":"2023-10-01T10:58:32.810584+0000","last_became_active":"2023-10-01T10:56:14.620579+0000","last_became_peered":"2023-10-01T10:56:14.620579+0000","last_unstale":"2023-10-01T10:58:32.810584+0000","last_undegraded":"2023-10-01T10:58:32.810584+0000","last_fullsized":"2023-10-01T10:58:32.810584+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:53:21.947263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":212,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:32.816438+0000","last_change":"2023-10-01T10:37:42.772247+0000","last_active":"2023-10-01T10:58:32.816438+0000","last_peered":"2023-10-01T10:58:32.816438+0000","last_clean":"2023-10-01T10:58:32.816438+0000","last_became_active":"2023-10-01T10:37:42.771697+0000","last_became_peered":"2023-10-01T10:37:42.771697+0000","last_unstale":"2023-10-01T10:58:32.816438+0000","last_undegraded":"2023-10-01T10:58:32.816438+0000","last_fullsized":"2023-10-01T10:58:32.816438+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:23:15.018248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":34,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:32.822114+0000","last_change":"2023-10-01T10:56:14.636139+0000","last_active":"2023-10-01T10:58:32.822114+0000","last_peered":"2023-10-01T10:58:32.822114+0000","last_clean":"2023-10-01T10:58:32.822114+0000","last_became_active":"2023-10-01T10:56:14.624426+0000","last_became_peered":"2023-10-01T10:56:14.624426+0000","last_unstale":"2023-10-01T10:58:32.822114+0000","last_undegraded":"2023-10-01T10:58:32.822114+0000","last_fullsized":"2023-10-01T10:58:32.822114+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":216,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:32.821576+0000","last_change":"2023-10-01T10:37:42.773127+0000","last_active":"2023-10-01T10:58:32.821576+0000","last_peered":"2023-10-01T10:58:32.821576+0000","last_clean":"2023-10-01T10:58:32.821576+0000","last_became_active":"2023-10-01T10:37:42.772102+0000","last_became_peered":"2023-10-01T10:37:42.772102+0000","last_unstale":"2023-10-01T10:58:32.821576+0000","last_undegraded":"2023-10-01T10:58:32.821576+0000","last_fullsized":"2023-10-01T10:58:32.821576+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:23:26.202295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.0","version":"249'8336","reported_seq":8355,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-10-01T10:58:59.187295+0000","last_change":"2023-10-01T10:56:21.668720+0000","last_active":"2023-10-01T10:58:59.187295+0000","last_peered":"2023-10-01T10:58:59.187295+0000","last_clean":"2023-10-01T10:58:59.187295+0000","last_became_active":"2023-10-01T10:56:21.668210+0000","last_became_peered":"2023-10-01T10:56:21.668210+0000","last_unstale":"2023-10-01T10:58:59.187295+0000","last_undegraded":"2023-10-01T10:58:59.187295+0000","last_fullsized":"2023-10-01T10:58:59.187295+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":8336,"log_dups_size":0,"ondisk_log_size":8336,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:32:59.602604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":290324480,"num_objects":4430,"num_object_clones":0,"num_object_copies":13290,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4430,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8336,"num_write_kb":408512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":90,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:32.811348+0000","last_change":"2023-10-01T10:52:43.663145+0000","last_active":"2023-10-01T10:58:32.811348+0000","last_peered":"2023-10-01T10:58:32.811348+0000","last_clean":"2023-10-01T10:58:32.811348+0000","last_became_active":"2023-10-01T10:52:43.662783+0000","last_became_peered":"2023-10-01T10:52:43.662783+0000","last_unstale":"2023-10-01T10:58:32.811348+0000","last_undegraded":"2023-10-01T10:58:32.811348+0000","last_fullsized":"2023-10-01T10:58:32.811348+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T22:10:32.190922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":218,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:32.814834+0000","last_change":"2023-10-01T10:56:14.627958+0000","last_active":"2023-10-01T10:58:32.814834+0000","last_peered":"2023-10-01T10:58:32.814834+0000","last_clean":"2023-10-01T10:58:32.814834+0000","last_became_active":"2023-10-01T10:56:14.627506+0000","last_became_peered":"2023-10-01T10:56:14.627506+0000","last_unstale":"2023-10-01T10:58:32.814834+0000","last_undegraded":"2023-10-01T10:58:32.814834+0000","last_fullsized":"2023-10-01T10:58:32.814834+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"249'8339","reported_seq":8359,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-10-01T10:58:57.714483+0000","last_change":"2023-10-01T10:56:21.667913+0000","last_active":"2023-10-01T10:58:57.714483+0000","last_peered":"2023-10-01T10:58:57.714483+0000","last_clean":"2023-10-01T10:58:57.714483+0000","last_became_active":"2023-10-01T10:56:21.667502+0000","last_became_peered":"2023-10-01T10:56:21.667502+0000","last_unstale":"2023-10-01T10:58:57.714483+0000","last_undegraded":"2023-10-01T10:58:57.714483+0000","last_fullsized":"2023-10-01T10:58:57.714483+0000","mapping_epoch":239,"log_start":"244'600","ondisk_log_start":"244'600","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":7739,"log_dups_size":0,"ondisk_log_size":7739,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:53:11.947301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301531136,"num_objects":4601,"num_object_clones":0,"num_object_copies":13803,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4601,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8339,"num_write_kb":414080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":90,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:32.811393+0000","last_change":"2023-10-01T10:52:43.662540+0000","last_active":"2023-10-01T10:58:32.811393+0000","last_peered":"2023-10-01T10:58:32.811393+0000","last_clean":"2023-10-01T10:58:32.811393+0000","last_became_active":"2023-10-01T10:52:43.662128+0000","last_became_peered":"2023-10-01T10:52:43.662128+0000","last_unstale":"2023-10-01T10:58:32.811393+0000","last_undegraded":"2023-10-01T10:58:32.811393+0000","last_fullsized":"2023-10-01T10:58:32.811393+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:09:44.807003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":246,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:32.810965+0000","last_change":"2023-10-01T10:38:16.033203+0000","last_active":"2023-10-01T10:58:32.810965+0000","last_peered":"2023-10-01T10:58:32.810965+0000","last_clean":"2023-10-01T10:58:32.810965+0000","last_became_active":"2023-10-01T10:38:16.032681+0000","last_became_peered":"2023-10-01T10:38:16.032681+0000","last_unstale":"2023-10-01T10:58:32.810965+0000","last_undegraded":"2023-10-01T10:58:32.810965+0000","last_fullsized":"2023-10-01T10:58:32.810965+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:42:36.642284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"249'8179","reported_seq":8199,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-10-01T10:58:59.188967+0000","last_change":"2023-10-01T10:56:21.664087+0000","last_active":"2023-10-01T10:58:59.188967+0000","last_peered":"2023-10-01T10:58:59.188967+0000","last_clean":"2023-10-01T10:58:59.188967+0000","last_became_active":"2023-10-01T10:56:21.663655+0000","last_became_peered":"2023-10-01T10:56:21.663655+0000","last_unstale":"2023-10-01T10:58:59.188967+0000","last_undegraded":"2023-10-01T10:58:59.188967+0000","last_fullsized":"2023-10-01T10:58:59.188967+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":8179,"log_dups_size":0,"ondisk_log_size":8179,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:34:34.434364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":291176448,"num_objects":4443,"num_object_clones":0,"num_object_copies":13329,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4443,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8179,"num_write_kb":403904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":114,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:32.815582+0000","last_change":"2023-10-01T10:56:14.638810+0000","last_active":"2023-10-01T10:58:32.815582+0000","last_peered":"2023-10-01T10:58:32.815582+0000","last_clean":"2023-10-01T10:58:32.815582+0000","last_became_active":"2023-10-01T10:56:14.624601+0000","last_became_peered":"2023-10-01T10:56:14.624601+0000","last_unstale":"2023-10-01T10:58:32.815582+0000","last_undegraded":"2023-10-01T10:58:32.815582+0000","last_fullsized":"2023-10-01T10:58:32.815582+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":24,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:32.822096+0000","last_change":"2023-10-01T10:56:14.623448+0000","last_active":"2023-10-01T10:58:32.822096+0000","last_peered":"2023-10-01T10:58:32.822096+0000","last_clean":"2023-10-01T10:58:32.822096+0000","last_became_active":"2023-10-01T10:56:14.623123+0000","last_became_peered":"2023-10-01T10:56:14.623123+0000","last_unstale":"2023-10-01T10:58:32.822096+0000","last_undegraded":"2023-10-01T10:58:32.822096+0000","last_fullsized":"2023-10-01T10:58:32.822096+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"249'8262","reported_seq":8282,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-10-01T10:58:58.301573+0000","last_change":"2023-10-01T10:56:21.664942+0000","last_active":"2023-10-01T10:58:58.301573+0000","last_peered":"2023-10-01T10:58:58.301573+0000","last_clean":"2023-10-01T10:58:58.301573+0000","last_became_active":"2023-10-01T10:56:21.664236+0000","last_became_peered":"2023-10-01T10:56:21.664236+0000","last_unstale":"2023-10-01T10:58:58.301573+0000","last_undegraded":"2023-10-01T10:58:58.301573+0000","last_fullsized":"2023-10-01T10:58:58.301573+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":8262,"log_dups_size":0,"ondisk_log_size":8262,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:51:15.411254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295043072,"num_objects":4502,"num_object_clones":0,"num_object_copies":13506,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4502,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8262,"num_write_kb":408448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":234,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:32.816094+0000","last_change":"2023-10-01T10:56:14.624999+0000","last_active":"2023-10-01T10:58:32.816094+0000","last_peered":"2023-10-01T10:58:32.816094+0000","last_clean":"2023-10-01T10:58:32.816094+0000","last_became_active":"2023-10-01T10:56:14.624597+0000","last_became_peered":"2023-10-01T10:56:14.624597+0000","last_unstale":"2023-10-01T10:58:32.816094+0000","last_undegraded":"2023-10-01T10:58:32.816094+0000","last_fullsized":"2023-10-01T10:58:32.816094+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":228,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:32.815699+0000","last_change":"2023-10-01T10:38:16.026348+0000","last_active":"2023-10-01T10:58:32.815699+0000","last_peered":"2023-10-01T10:58:32.815699+0000","last_clean":"2023-10-01T10:58:32.815699+0000","last_became_active":"2023-10-01T10:38:16.025747+0000","last_became_peered":"2023-10-01T10:38:16.025747+0000","last_unstale":"2023-10-01T10:58:32.815699+0000","last_undegraded":"2023-10-01T10:58:32.815699+0000","last_fullsized":"2023-10-01T10:58:32.815699+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:28:18.934480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"249'8336","reported_seq":8356,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-10-01T10:58:57.711924+0000","last_change":"2023-10-01T10:56:22.013729+0000","last_active":"2023-10-01T10:58:57.711924+0000","last_peered":"2023-10-01T10:58:57.711924+0000","last_clean":"2023-10-01T10:58:57.711924+0000","last_became_active":"2023-10-01T10:56:22.013403+0000","last_became_peered":"2023-10-01T10:56:22.013403+0000","last_unstale":"2023-10-01T10:58:57.711924+0000","last_undegraded":"2023-10-01T10:58:57.711924+0000","last_fullsized":"2023-10-01T10:58:57.711924+0000","mapping_epoch":239,"log_start":"244'600","ondisk_log_start":"244'600","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":7736,"log_dups_size":0,"ondisk_log_size":7736,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:34:17.163618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":298975232,"num_objects":4562,"num_object_clones":0,"num_object_copies":13686,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4562,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8336,"num_write_kb":412736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":220,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:32.821496+0000","last_change":"2023-10-01T10:38:16.022493+0000","last_active":"2023-10-01T10:58:32.821496+0000","last_peered":"2023-10-01T10:58:32.821496+0000","last_clean":"2023-10-01T10:58:32.821496+0000","last_became_active":"2023-10-01T10:38:16.022033+0000","last_became_peered":"2023-10-01T10:38:16.022033+0000","last_unstale":"2023-10-01T10:58:32.821496+0000","last_undegraded":"2023-10-01T10:58:32.821496+0000","last_fullsized":"2023-10-01T10:58:32.821496+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:49:05.262763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":249,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:32.811064+0000","last_change":"2023-10-01T10:56:14.623628+0000","last_active":"2023-10-01T10:58:32.811064+0000","last_peered":"2023-10-01T10:58:32.811064+0000","last_clean":"2023-10-01T10:58:32.811064+0000","last_became_active":"2023-10-01T10:56:14.622190+0000","last_became_peered":"2023-10-01T10:56:14.622190+0000","last_unstale":"2023-10-01T10:58:32.811064+0000","last_undegraded":"2023-10-01T10:58:32.811064+0000","last_fullsized":"2023-10-01T10:58:32.811064+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:57:06.093638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"249'8129","reported_seq":8149,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-10-01T10:58:58.744525+0000","last_change":"2023-10-01T10:56:22.014079+0000","last_active":"2023-10-01T10:58:58.744525+0000","last_peered":"2023-10-01T10:58:58.744525+0000","last_clean":"2023-10-01T10:58:58.744525+0000","last_became_active":"2023-10-01T10:56:22.013703+0000","last_became_peered":"2023-10-01T10:56:22.013703+0000","last_unstale":"2023-10-01T10:58:58.744525+0000","last_undegraded":"2023-10-01T10:58:58.744525+0000","last_fullsized":"2023-10-01T10:58:58.744525+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":8129,"log_dups_size":0,"ondisk_log_size":8129,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:07:01.254677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":288555008,"num_objects":4403,"num_object_clones":0,"num_object_copies":13209,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4403,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8129,"num_write_kb":401024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":222,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-10-01T10:58:38.857155+0000","last_change":"2023-10-01T10:56:14.622835+0000","last_active":"2023-10-01T10:58:38.857155+0000","last_peered":"2023-10-01T10:58:38.857155+0000","last_clean":"2023-10-01T10:58:38.857155+0000","last_became_active":"2023-10-01T10:56:14.622449+0000","last_became_peered":"2023-10-01T10:56:14.622449+0000","last_unstale":"2023-10-01T10:58:38.857155+0000","last_undegraded":"2023-10-01T10:58:38.857155+0000","last_fullsized":"2023-10-01T10:58:38.857155+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:29:48.934875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":236,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:32.811383+0000","last_change":"2023-10-01T10:38:16.035906+0000","last_active":"2023-10-01T10:58:32.811383+0000","last_peered":"2023-10-01T10:58:32.811383+0000","last_clean":"2023-10-01T10:58:32.811383+0000","last_became_active":"2023-10-01T10:38:16.035383+0000","last_became_peered":"2023-10-01T10:38:16.035383+0000","last_unstale":"2023-10-01T10:58:32.811383+0000","last_undegraded":"2023-10-01T10:58:32.811383+0000","last_fullsized":"2023-10-01T10:58:32.811383+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:44:41.500097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"249'8370","reported_seq":8390,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-10-01T10:58:58.277446+0000","last_change":"2023-10-01T10:56:22.013872+0000","last_active":"2023-10-01T10:58:58.277446+0000","last_peered":"2023-10-01T10:58:58.277446+0000","last_clean":"2023-10-01T10:58:58.277446+0000","last_became_active":"2023-10-01T10:56:22.013554+0000","last_became_peered":"2023-10-01T10:56:22.013554+0000","last_unstale":"2023-10-01T10:58:58.277446+0000","last_undegraded":"2023-10-01T10:58:58.277446+0000","last_fullsized":"2023-10-01T10:58:58.277446+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":8370,"log_dups_size":0,"ondisk_log_size":8370,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:10:34.060296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":296878080,"num_objects":4530,"num_object_clones":0,"num_object_copies":13590,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4530,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8370,"num_write_kb":412800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":228,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:32.810931+0000","last_change":"2023-10-01T10:38:16.026611+0000","last_active":"2023-10-01T10:58:32.810931+0000","last_peered":"2023-10-01T10:58:32.810931+0000","last_clean":"2023-10-01T10:58:32.810931+0000","last_became_active":"2023-10-01T10:38:16.026267+0000","last_became_peered":"2023-10-01T10:38:16.026267+0000","last_unstale":"2023-10-01T10:58:32.810931+0000","last_undegraded":"2023-10-01T10:58:32.810931+0000","last_fullsized":"2023-10-01T10:58:32.810931+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:06.135288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":230,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:32.811337+0000","last_change":"2023-10-01T10:56:14.623011+0000","last_active":"2023-10-01T10:58:32.811337+0000","last_peered":"2023-10-01T10:58:32.811337+0000","last_clean":"2023-10-01T10:58:32.811337+0000","last_became_active":"2023-10-01T10:56:14.622649+0000","last_became_peered":"2023-10-01T10:56:14.622649+0000","last_unstale":"2023-10-01T10:58:32.811337+0000","last_undegraded":"2023-10-01T10:58:32.811337+0000","last_fullsized":"2023-10-01T10:58:32.811337+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:32:42.042469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"249'8545","reported_seq":8565,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-10-01T10:58:58.732908+0000","last_change":"2023-10-01T10:56:22.016312+0000","last_active":"2023-10-01T10:58:58.732908+0000","last_peered":"2023-10-01T10:58:58.732908+0000","last_clean":"2023-10-01T10:58:58.732908+0000","last_became_active":"2023-10-01T10:56:22.015964+0000","last_became_peered":"2023-10-01T10:56:22.015964+0000","last_unstale":"2023-10-01T10:58:58.732908+0000","last_undegraded":"2023-10-01T10:58:58.732908+0000","last_fullsized":"2023-10-01T10:58:58.732908+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":8545,"log_dups_size":0,"ondisk_log_size":8545,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:30:15.203782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299565056,"num_objects":4571,"num_object_clones":0,"num_object_copies":13713,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4571,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8545,"num_write_kb":419712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":218,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:32.822212+0000","last_change":"2023-10-01T10:37:42.772398+0000","last_active":"2023-10-01T10:58:32.822212+0000","last_peered":"2023-10-01T10:58:32.822212+0000","last_clean":"2023-10-01T10:58:32.822212+0000","last_became_active":"2023-10-01T10:37:42.772084+0000","last_became_peered":"2023-10-01T10:37:42.772084+0000","last_unstale":"2023-10-01T10:58:32.822212+0000","last_undegraded":"2023-10-01T10:58:32.822212+0000","last_fullsized":"2023-10-01T10:58:32.822212+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:32:43.846419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":236,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:32.818113+0000","last_change":"2023-10-01T10:38:16.019755+0000","last_active":"2023-10-01T10:58:32.818113+0000","last_peered":"2023-10-01T10:58:32.818113+0000","last_clean":"2023-10-01T10:58:32.818113+0000","last_became_active":"2023-10-01T10:38:16.019397+0000","last_became_peered":"2023-10-01T10:38:16.019397+0000","last_unstale":"2023-10-01T10:58:32.818113+0000","last_undegraded":"2023-10-01T10:58:32.818113+0000","last_fullsized":"2023-10-01T10:58:32.818113+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:42:56.623225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.a","version":"249'8304","reported_seq":8324,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-10-01T10:58:58.700782+0000","last_change":"2023-10-01T10:56:21.663116+0000","last_active":"2023-10-01T10:58:58.700782+0000","last_peered":"2023-10-01T10:58:58.700782+0000","last_clean":"2023-10-01T10:58:58.700782+0000","last_became_active":"2023-10-01T10:56:21.662762+0000","last_became_peered":"2023-10-01T10:56:21.662762+0000","last_unstale":"2023-10-01T10:58:58.700782+0000","last_undegraded":"2023-10-01T10:58:58.700782+0000","last_fullsized":"2023-10-01T10:58:58.700782+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":8304,"log_dups_size":0,"ondisk_log_size":8304,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:00:59.657772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":294518784,"num_objects":4494,"num_object_clones":0,"num_object_copies":13482,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4494,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8304,"num_write_kb":409536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":144,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:38.241917+0000","last_change":"2023-10-01T10:52:18.960164+0000","last_active":"2023-10-01T10:58:38.241917+0000","last_peered":"2023-10-01T10:58:38.241917+0000","last_clean":"2023-10-01T10:58:38.241917+0000","last_became_active":"2023-10-01T10:52:18.959782+0000","last_became_peered":"2023-10-01T10:52:18.959782+0000","last_unstale":"2023-10-01T10:58:38.241917+0000","last_undegraded":"2023-10-01T10:58:38.241917+0000","last_fullsized":"2023-10-01T10:58:38.241917+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:08:27.603412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":231,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:32.815383+0000","last_change":"2023-10-01T10:37:42.766814+0000","last_active":"2023-10-01T10:58:32.815383+0000","last_peered":"2023-10-01T10:58:32.815383+0000","last_clean":"2023-10-01T10:58:32.815383+0000","last_became_active":"2023-10-01T10:37:42.766451+0000","last_became_peered":"2023-10-01T10:37:42.766451+0000","last_unstale":"2023-10-01T10:58:32.815383+0000","last_undegraded":"2023-10-01T10:58:32.815383+0000","last_fullsized":"2023-10-01T10:58:32.815383+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":134,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:32.814958+0000","last_change":"2023-10-01T10:52:18.960606+0000","last_active":"2023-10-01T10:58:32.814958+0000","last_peered":"2023-10-01T10:58:32.814958+0000","last_clean":"2023-10-01T10:58:32.814958+0000","last_became_active":"2023-10-01T10:52:18.960270+0000","last_became_peered":"2023-10-01T10:52:18.960270+0000","last_unstale":"2023-10-01T10:58:32.814958+0000","last_undegraded":"2023-10-01T10:58:32.814958+0000","last_fullsized":"2023-10-01T10:58:32.814958+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.c","version":"249'8435","reported_seq":8455,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-10-01T10:58:58.193077+0000","last_change":"2023-10-01T10:56:21.895915+0000","last_active":"2023-10-01T10:58:58.193077+0000","last_peered":"2023-10-01T10:58:58.193077+0000","last_clean":"2023-10-01T10:58:58.193077+0000","last_became_active":"2023-10-01T10:56:21.895501+0000","last_became_peered":"2023-10-01T10:56:21.895501+0000","last_unstale":"2023-10-01T10:58:58.193077+0000","last_undegraded":"2023-10-01T10:58:58.193077+0000","last_fullsized":"2023-10-01T10:58:58.193077+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":8435,"log_dups_size":0,"ondisk_log_size":8435,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:21:21.588297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300875776,"num_objects":4591,"num_object_clones":0,"num_object_copies":13773,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4591,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8435,"num_write_kb":416832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":238,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:32.815050+0000","last_change":"2023-10-01T10:56:14.624787+0000","last_active":"2023-10-01T10:58:32.815050+0000","last_peered":"2023-10-01T10:58:32.815050+0000","last_clean":"2023-10-01T10:58:32.815050+0000","last_became_active":"2023-10-01T10:56:14.624357+0000","last_became_peered":"2023-10-01T10:56:14.624357+0000","last_unstale":"2023-10-01T10:58:32.815050+0000","last_undegraded":"2023-10-01T10:58:32.815050+0000","last_fullsized":"2023-10-01T10:58:32.815050+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":216,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:32.811214+0000","last_change":"2023-10-01T10:56:14.620354+0000","last_active":"2023-10-01T10:58:32.811214+0000","last_peered":"2023-10-01T10:58:32.811214+0000","last_clean":"2023-10-01T10:58:32.811214+0000","last_became_active":"2023-10-01T10:56:14.619900+0000","last_became_peered":"2023-10-01T10:56:14.619900+0000","last_unstale":"2023-10-01T10:58:32.811214+0000","last_undegraded":"2023-10-01T10:58:32.811214+0000","last_fullsized":"2023-10-01T10:58:32.811214+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:09:36.449841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":147,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:33.779327+0000","last_change":"2023-10-01T10:52:18.963437+0000","last_active":"2023-10-01T10:58:33.779327+0000","last_peered":"2023-10-01T10:58:33.779327+0000","last_clean":"2023-10-01T10:58:33.779327+0000","last_became_active":"2023-10-01T10:52:18.955165+0000","last_became_peered":"2023-10-01T10:52:18.955165+0000","last_unstale":"2023-10-01T10:58:33.779327+0000","last_undegraded":"2023-10-01T10:58:33.779327+0000","last_fullsized":"2023-10-01T10:58:33.779327+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:57:49.129100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.f","version":"249'8511","reported_seq":8529,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-10-01T10:58:58.520780+0000","last_change":"2023-10-01T10:56:21.896411+0000","last_active":"2023-10-01T10:58:58.520780+0000","last_peered":"2023-10-01T10:58:58.520780+0000","last_clean":"2023-10-01T10:58:58.520780+0000","last_became_active":"2023-10-01T10:56:21.895807+0000","last_became_peered":"2023-10-01T10:56:21.895807+0000","last_unstale":"2023-10-01T10:58:58.520780+0000","last_undegraded":"2023-10-01T10:58:58.520780+0000","last_fullsized":"2023-10-01T10:58:58.520780+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":8511,"log_dups_size":0,"ondisk_log_size":8511,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299171840,"num_objects":4565,"num_object_clones":0,"num_object_copies":13695,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4565,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8511,"num_write_kb":418432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":228,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:32.817760+0000","last_change":"2023-10-01T10:37:42.762524+0000","last_active":"2023-10-01T10:58:32.817760+0000","last_peered":"2023-10-01T10:58:32.817760+0000","last_clean":"2023-10-01T10:58:32.817760+0000","last_became_active":"2023-10-01T10:37:42.762209+0000","last_became_peered":"2023-10-01T10:37:42.762209+0000","last_unstale":"2023-10-01T10:58:32.817760+0000","last_undegraded":"2023-10-01T10:58:32.817760+0000","last_fullsized":"2023-10-01T10:58:32.817760+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:45:38.171397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":145,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:32.837399+0000","last_change":"2023-10-01T10:56:14.621343+0000","last_active":"2023-10-01T10:58:32.837399+0000","last_peered":"2023-10-01T10:58:32.837399+0000","last_clean":"2023-10-01T10:58:32.837399+0000","last_became_active":"2023-10-01T10:56:14.620940+0000","last_became_peered":"2023-10-01T10:56:14.620940+0000","last_unstale":"2023-10-01T10:58:32.837399+0000","last_undegraded":"2023-10-01T10:58:32.837399+0000","last_fullsized":"2023-10-01T10:58:32.837399+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:43:39.290894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":136,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:32.815654+0000","last_change":"2023-10-01T10:52:18.961040+0000","last_active":"2023-10-01T10:58:32.815654+0000","last_peered":"2023-10-01T10:58:32.815654+0000","last_clean":"2023-10-01T10:58:32.815654+0000","last_became_active":"2023-10-01T10:52:18.960606+0000","last_became_peered":"2023-10-01T10:52:18.960606+0000","last_unstale":"2023-10-01T10:58:32.815654+0000","last_undegraded":"2023-10-01T10:58:32.815654+0000","last_fullsized":"2023-10-01T10:58:32.815654+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.d","version":"249'8416","reported_seq":8435,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-10-01T10:58:58.479861+0000","last_change":"2023-10-01T10:56:22.015481+0000","last_active":"2023-10-01T10:58:58.479861+0000","last_peered":"2023-10-01T10:58:58.479861+0000","last_clean":"2023-10-01T10:58:58.479861+0000","last_became_active":"2023-10-01T10:56:22.014648+0000","last_became_peered":"2023-10-01T10:56:22.014648+0000","last_unstale":"2023-10-01T10:58:58.479861+0000","last_undegraded":"2023-10-01T10:58:58.479861+0000","last_fullsized":"2023-10-01T10:58:58.479861+0000","mapping_epoch":239,"log_start":"243'300","ondisk_log_start":"243'300","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":8116,"log_dups_size":0,"ondisk_log_size":8116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":302841880,"num_objects":4622,"num_object_clones":0,"num_object_copies":13866,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4622,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8416,"num_write_kb":417153,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":220,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:32.814972+0000","last_change":"2023-10-01T10:37:05.024009+0000","last_active":"2023-10-01T10:58:32.814972+0000","last_peered":"2023-10-01T10:58:32.814972+0000","last_clean":"2023-10-01T10:58:32.814972+0000","last_became_active":"2023-10-01T10:37:05.023458+0000","last_became_peered":"2023-10-01T10:37:05.023458+0000","last_unstale":"2023-10-01T10:58:32.814972+0000","last_undegraded":"2023-10-01T10:58:32.814972+0000","last_fullsized":"2023-10-01T10:58:32.814972+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'139","reported_seq":388,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-10-01T10:58:38.852595+0000","last_change":"2023-10-01T10:56:17.635571+0000","last_active":"2023-10-01T10:58:38.852595+0000","last_peered":"2023-10-01T10:58:38.852595+0000","last_clean":"2023-10-01T10:58:38.852595+0000","last_became_active":"2023-10-01T10:56:17.635055+0000","last_became_peered":"2023-10-01T10:56:17.635055+0000","last_unstale":"2023-10-01T10:58:38.852595+0000","last_undegraded":"2023-10-01T10:58:38.852595+0000","last_fullsized":"2023-10-01T10:58:38.852595+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:09:17.221082+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:09:17.221082+0000","last_clean_scrub_stamp":"2023-10-01T10:09:17.221082+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:37:54.527937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":219,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:32.832089+0000","last_change":"2023-10-01T10:56:14.622079+0000","last_active":"2023-10-01T10:58:32.832089+0000","last_peered":"2023-10-01T10:58:32.832089+0000","last_clean":"2023-10-01T10:58:32.832089+0000","last_became_active":"2023-10-01T10:56:14.621644+0000","last_became_peered":"2023-10-01T10:56:14.621644+0000","last_unstale":"2023-10-01T10:58:32.832089+0000","last_undegraded":"2023-10-01T10:58:32.832089+0000","last_fullsized":"2023-10-01T10:58:32.832089+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:32:47.233878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":206,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:32.811215+0000","last_change":"2023-10-01T10:56:14.622469+0000","last_active":"2023-10-01T10:58:32.811215+0000","last_peered":"2023-10-01T10:58:32.811215+0000","last_clean":"2023-10-01T10:58:32.811215+0000","last_became_active":"2023-10-01T10:56:14.622092+0000","last_became_peered":"2023-10-01T10:56:14.622092+0000","last_unstale":"2023-10-01T10:58:32.811215+0000","last_undegraded":"2023-10-01T10:58:32.811215+0000","last_fullsized":"2023-10-01T10:58:32.811215+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:43:56.657361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.e","version":"249'8421","reported_seq":8441,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-10-01T10:58:58.305181+0000","last_change":"2023-10-01T10:56:21.663535+0000","last_active":"2023-10-01T10:58:58.305181+0000","last_peered":"2023-10-01T10:58:58.305181+0000","last_clean":"2023-10-01T10:58:58.305181+0000","last_became_active":"2023-10-01T10:56:21.663135+0000","last_became_peered":"2023-10-01T10:56:21.663135+0000","last_unstale":"2023-10-01T10:58:58.305181+0000","last_undegraded":"2023-10-01T10:58:58.305181+0000","last_fullsized":"2023-10-01T10:58:58.305181+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":8421,"log_dups_size":0,"ondisk_log_size":8421,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:11:25.118806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":298909696,"num_objects":4561,"num_object_clones":0,"num_object_copies":13683,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4561,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8421,"num_write_kb":415424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":159,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-10-01T10:58:38.852738+0000","last_change":"2023-10-01T10:52:43.661009+0000","last_active":"2023-10-01T10:58:38.852738+0000","last_peered":"2023-10-01T10:58:38.852738+0000","last_clean":"2023-10-01T10:58:38.852738+0000","last_became_active":"2023-10-01T10:52:43.660679+0000","last_became_peered":"2023-10-01T10:52:43.660679+0000","last_unstale":"2023-10-01T10:58:38.852738+0000","last_undegraded":"2023-10-01T10:58:38.852738+0000","last_fullsized":"2023-10-01T10:58:38.852738+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:32:21.336835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":228,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:32.815799+0000","last_change":"2023-10-01T10:38:16.021968+0000","last_active":"2023-10-01T10:58:32.815799+0000","last_peered":"2023-10-01T10:58:32.815799+0000","last_clean":"2023-10-01T10:58:32.815799+0000","last_became_active":"2023-10-01T10:38:16.021539+0000","last_became_peered":"2023-10-01T10:38:16.021539+0000","last_unstale":"2023-10-01T10:58:32.815799+0000","last_undegraded":"2023-10-01T10:58:32.815799+0000","last_fullsized":"2023-10-01T10:58:32.815799+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:12:53.105747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":229,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:38.241974+0000","last_change":"2023-10-01T10:37:05.017431+0000","last_active":"2023-10-01T10:58:38.241974+0000","last_peered":"2023-10-01T10:58:38.241974+0000","last_clean":"2023-10-01T10:58:38.241974+0000","last_became_active":"2023-10-01T10:37:05.017094+0000","last_became_peered":"2023-10-01T10:37:05.017094+0000","last_unstale":"2023-10-01T10:58:38.241974+0000","last_undegraded":"2023-10-01T10:58:38.241974+0000","last_fullsized":"2023-10-01T10:58:38.241974+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:10:40.664686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.b","version":"249'8430","reported_seq":8449,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-10-01T10:58:57.726052+0000","last_change":"2023-10-01T10:56:21.667103+0000","last_active":"2023-10-01T10:58:57.726052+0000","last_peered":"2023-10-01T10:58:57.726052+0000","last_clean":"2023-10-01T10:58:57.726052+0000","last_became_active":"2023-10-01T10:56:21.666784+0000","last_became_peered":"2023-10-01T10:56:21.666784+0000","last_unstale":"2023-10-01T10:58:57.726052+0000","last_undegraded":"2023-10-01T10:58:57.726052+0000","last_fullsized":"2023-10-01T10:58:57.726052+0000","mapping_epoch":239,"log_start":"244'700","ondisk_log_start":"244'700","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":7730,"log_dups_size":0,"ondisk_log_size":7730,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:45:11.964390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":302514176,"num_objects":4616,"num_object_clones":0,"num_object_copies":13848,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4616,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8430,"num_write_kb":417472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":234,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:32.815916+0000","last_change":"2023-10-01T10:38:16.032774+0000","last_active":"2023-10-01T10:58:32.815916+0000","last_peered":"2023-10-01T10:58:32.815916+0000","last_clean":"2023-10-01T10:58:32.815916+0000","last_became_active":"2023-10-01T10:38:16.032283+0000","last_became_peered":"2023-10-01T10:38:16.032283+0000","last_unstale":"2023-10-01T10:58:32.815916+0000","last_undegraded":"2023-10-01T10:58:32.815916+0000","last_fullsized":"2023-10-01T10:58:32.815916+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":127,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:33.779338+0000","last_change":"2023-10-01T10:52:18.964479+0000","last_active":"2023-10-01T10:58:33.779338+0000","last_peered":"2023-10-01T10:58:33.779338+0000","last_clean":"2023-10-01T10:58:33.779338+0000","last_became_active":"2023-10-01T10:52:18.963881+0000","last_became_peered":"2023-10-01T10:52:18.963881+0000","last_unstale":"2023-10-01T10:58:33.779338+0000","last_undegraded":"2023-10-01T10:58:33.779338+0000","last_fullsized":"2023-10-01T10:58:33.779338+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:36:30.405755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":199,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:32.815996+0000","last_change":"2023-10-01T10:28:20.220812+0000","last_active":"2023-10-01T10:58:32.815996+0000","last_peered":"2023-10-01T10:58:32.815996+0000","last_clean":"2023-10-01T10:58:32.815996+0000","last_became_active":"2023-10-01T10:28:20.215962+0000","last_became_peered":"2023-10-01T10:28:20.215962+0000","last_unstale":"2023-10-01T10:58:32.815996+0000","last_undegraded":"2023-10-01T10:58:32.815996+0000","last_fullsized":"2023-10-01T10:58:32.815996+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:35:15.275097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.8","version":"249'8472","reported_seq":8492,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-10-01T10:58:57.680609+0000","last_change":"2023-10-01T10:56:21.664695+0000","last_active":"2023-10-01T10:58:57.680609+0000","last_peered":"2023-10-01T10:58:57.680609+0000","last_clean":"2023-10-01T10:58:57.680609+0000","last_became_active":"2023-10-01T10:56:21.664315+0000","last_became_peered":"2023-10-01T10:56:21.664315+0000","last_unstale":"2023-10-01T10:58:57.680609+0000","last_undegraded":"2023-10-01T10:58:57.680609+0000","last_fullsized":"2023-10-01T10:58:57.680609+0000","mapping_epoch":239,"log_start":"244'700","ondisk_log_start":"244'700","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":7772,"log_dups_size":0,"ondisk_log_size":7772,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:27:54.090426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295698432,"num_objects":4512,"num_object_clones":0,"num_object_copies":13536,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4512,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8472,"num_write_kb":415488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":205,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:32.821624+0000","last_change":"2023-10-01T10:37:42.773999+0000","last_active":"2023-10-01T10:58:32.821624+0000","last_peered":"2023-10-01T10:58:32.821624+0000","last_clean":"2023-10-01T10:58:32.821624+0000","last_became_active":"2023-10-01T10:37:42.773355+0000","last_became_peered":"2023-10-01T10:37:42.773355+0000","last_unstale":"2023-10-01T10:58:32.821624+0000","last_undegraded":"2023-10-01T10:58:32.821624+0000","last_fullsized":"2023-10-01T10:58:32.821624+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:46:51.317151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":197,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-10-01T10:58:38.858166+0000","last_change":"2023-10-01T10:56:14.623768+0000","last_active":"2023-10-01T10:58:38.858166+0000","last_peered":"2023-10-01T10:58:38.858166+0000","last_clean":"2023-10-01T10:58:38.858166+0000","last_became_active":"2023-10-01T10:56:14.623353+0000","last_became_peered":"2023-10-01T10:56:14.623353+0000","last_unstale":"2023-10-01T10:58:38.858166+0000","last_undegraded":"2023-10-01T10:58:38.858166+0000","last_fullsized":"2023-10-01T10:58:38.858166+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:53:19.776540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":203,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:32.822131+0000","last_change":"2023-10-01T10:37:42.773538+0000","last_active":"2023-10-01T10:58:32.822131+0000","last_peered":"2023-10-01T10:58:32.822131+0000","last_clean":"2023-10-01T10:58:32.822131+0000","last_became_active":"2023-10-01T10:37:42.771863+0000","last_became_peered":"2023-10-01T10:37:42.771863+0000","last_unstale":"2023-10-01T10:58:32.822131+0000","last_undegraded":"2023-10-01T10:58:32.822131+0000","last_fullsized":"2023-10-01T10:58:32.822131+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:46:46.698627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"249'8322","reported_seq":8342,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-10-01T10:58:58.729445+0000","last_change":"2023-10-01T10:56:21.667837+0000","last_active":"2023-10-01T10:58:58.729445+0000","last_peered":"2023-10-01T10:58:58.729445+0000","last_clean":"2023-10-01T10:58:58.729445+0000","last_became_active":"2023-10-01T10:56:21.667513+0000","last_became_peered":"2023-10-01T10:56:21.667513+0000","last_unstale":"2023-10-01T10:58:58.729445+0000","last_undegraded":"2023-10-01T10:58:58.729445+0000","last_fullsized":"2023-10-01T10:58:58.729445+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":8322,"log_dups_size":0,"ondisk_log_size":8322,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:25:17.334326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":291504128,"num_objects":4448,"num_object_clones":0,"num_object_copies":13344,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4448,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8322,"num_write_kb":408640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":24,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:32.822429+0000","last_change":"2023-10-01T10:56:14.621086+0000","last_active":"2023-10-01T10:58:32.822429+0000","last_peered":"2023-10-01T10:58:32.822429+0000","last_clean":"2023-10-01T10:58:32.822429+0000","last_became_active":"2023-10-01T10:56:14.620638+0000","last_became_peered":"2023-10-01T10:56:14.620638+0000","last_unstale":"2023-10-01T10:58:32.822429+0000","last_undegraded":"2023-10-01T10:58:32.822429+0000","last_fullsized":"2023-10-01T10:58:32.822429+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":232,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:38.241942+0000","last_change":"2023-10-01T10:38:16.027472+0000","last_active":"2023-10-01T10:58:38.241942+0000","last_peered":"2023-10-01T10:58:38.241942+0000","last_clean":"2023-10-01T10:58:38.241942+0000","last_became_active":"2023-10-01T10:38:16.026810+0000","last_became_peered":"2023-10-01T10:38:16.026810+0000","last_unstale":"2023-10-01T10:58:38.241942+0000","last_undegraded":"2023-10-01T10:58:38.241942+0000","last_fullsized":"2023-10-01T10:58:38.241942+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:30:06.978187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":214,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:32.810734+0000","last_change":"2023-10-01T10:56:14.626521+0000","last_active":"2023-10-01T10:58:32.810734+0000","last_peered":"2023-10-01T10:58:32.810734+0000","last_clean":"2023-10-01T10:58:32.810734+0000","last_became_active":"2023-10-01T10:56:14.626107+0000","last_became_peered":"2023-10-01T10:56:14.626107+0000","last_unstale":"2023-10-01T10:58:32.810734+0000","last_undegraded":"2023-10-01T10:58:32.810734+0000","last_fullsized":"2023-10-01T10:58:32.810734+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:44:12.588719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":215,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:32.815987+0000","last_change":"2023-10-01T10:38:16.028491+0000","last_active":"2023-10-01T10:58:32.815987+0000","last_peered":"2023-10-01T10:58:32.815987+0000","last_clean":"2023-10-01T10:58:32.815987+0000","last_became_active":"2023-10-01T10:38:16.028175+0000","last_became_peered":"2023-10-01T10:38:16.028175+0000","last_unstale":"2023-10-01T10:58:32.815987+0000","last_undegraded":"2023-10-01T10:58:32.815987+0000","last_fullsized":"2023-10-01T10:58:32.815987+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:44:29.083129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":236,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:32.817742+0000","last_change":"2023-10-01T10:37:42.762360+0000","last_active":"2023-10-01T10:58:32.817742+0000","last_peered":"2023-10-01T10:58:32.817742+0000","last_clean":"2023-10-01T10:58:32.817742+0000","last_became_active":"2023-10-01T10:37:42.761936+0000","last_became_peered":"2023-10-01T10:37:42.761936+0000","last_unstale":"2023-10-01T10:58:32.817742+0000","last_undegraded":"2023-10-01T10:58:32.817742+0000","last_fullsized":"2023-10-01T10:58:32.817742+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:50.637582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":233,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:32.816152+0000","last_change":"2023-10-01T10:38:16.037809+0000","last_active":"2023-10-01T10:58:32.816152+0000","last_peered":"2023-10-01T10:58:32.816152+0000","last_clean":"2023-10-01T10:58:32.816152+0000","last_became_active":"2023-10-01T10:38:16.037318+0000","last_became_peered":"2023-10-01T10:38:16.037318+0000","last_unstale":"2023-10-01T10:58:32.816152+0000","last_undegraded":"2023-10-01T10:58:32.816152+0000","last_fullsized":"2023-10-01T10:58:32.816152+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":245,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:32.811001+0000","last_change":"2023-10-01T10:38:16.034282+0000","last_active":"2023-10-01T10:58:32.811001+0000","last_peered":"2023-10-01T10:58:32.811001+0000","last_clean":"2023-10-01T10:58:32.811001+0000","last_became_active":"2023-10-01T10:38:16.033820+0000","last_became_peered":"2023-10-01T10:38:16.033820+0000","last_unstale":"2023-10-01T10:58:32.811001+0000","last_undegraded":"2023-10-01T10:58:32.811001+0000","last_fullsized":"2023-10-01T10:58:32.811001+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:48:23.815832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":244,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:32.814953+0000","last_change":"2023-10-01T10:56:14.636784+0000","last_active":"2023-10-01T10:58:32.814953+0000","last_peered":"2023-10-01T10:58:32.814953+0000","last_clean":"2023-10-01T10:58:32.814953+0000","last_became_active":"2023-10-01T10:56:14.624338+0000","last_became_peered":"2023-10-01T10:56:14.624338+0000","last_unstale":"2023-10-01T10:58:32.814953+0000","last_undegraded":"2023-10-01T10:58:32.814953+0000","last_fullsized":"2023-10-01T10:58:32.814953+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":245,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:32.810960+0000","last_change":"2023-10-01T10:38:16.025600+0000","last_active":"2023-10-01T10:58:32.810960+0000","last_peered":"2023-10-01T10:58:32.810960+0000","last_clean":"2023-10-01T10:58:32.810960+0000","last_became_active":"2023-10-01T10:38:16.025252+0000","last_became_peered":"2023-10-01T10:38:16.025252+0000","last_unstale":"2023-10-01T10:58:32.810960+0000","last_undegraded":"2023-10-01T10:58:32.810960+0000","last_fullsized":"2023-10-01T10:58:32.810960+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:24:18.871963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":252,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:32.821419+0000","last_change":"2023-10-01T10:56:14.621544+0000","last_active":"2023-10-01T10:58:32.821419+0000","last_peered":"2023-10-01T10:58:32.821419+0000","last_clean":"2023-10-01T10:58:32.821419+0000","last_became_active":"2023-10-01T10:56:14.621134+0000","last_became_peered":"2023-10-01T10:56:14.621134+0000","last_unstale":"2023-10-01T10:58:32.821419+0000","last_undegraded":"2023-10-01T10:58:32.821419+0000","last_fullsized":"2023-10-01T10:58:32.821419+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:48:44.436158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":245,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:32.827813+0000","last_change":"2023-10-01T10:37:42.765142+0000","last_active":"2023-10-01T10:58:32.827813+0000","last_peered":"2023-10-01T10:58:32.827813+0000","last_clean":"2023-10-01T10:58:32.827813+0000","last_became_active":"2023-10-01T10:37:42.764770+0000","last_became_peered":"2023-10-01T10:37:42.764770+0000","last_unstale":"2023-10-01T10:58:32.827813+0000","last_undegraded":"2023-10-01T10:58:32.827813+0000","last_fullsized":"2023-10-01T10:58:32.827813+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:23:33.527876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":217,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:32.816202+0000","last_change":"2023-10-01T10:37:05.021621+0000","last_active":"2023-10-01T10:58:32.816202+0000","last_peered":"2023-10-01T10:58:32.816202+0000","last_clean":"2023-10-01T10:58:32.816202+0000","last_became_active":"2023-10-01T10:37:05.021173+0000","last_became_peered":"2023-10-01T10:37:05.021173+0000","last_unstale":"2023-10-01T10:58:32.816202+0000","last_undegraded":"2023-10-01T10:58:32.816202+0000","last_fullsized":"2023-10-01T10:58:32.816202+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:13:25.364499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":229,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-10-01T10:58:32.810565+0000","last_change":"2023-10-01T10:56:14.621791+0000","last_active":"2023-10-01T10:58:32.810565+0000","last_peered":"2023-10-01T10:58:32.810565+0000","last_clean":"2023-10-01T10:58:32.810565+0000","last_became_active":"2023-10-01T10:56:14.621443+0000","last_became_peered":"2023-10-01T10:56:14.621443+0000","last_unstale":"2023-10-01T10:58:32.810565+0000","last_undegraded":"2023-10-01T10:58:32.810565+0000","last_fullsized":"2023-10-01T10:58:32.810565+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:13:58.651222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":4748083224,"num_objects":72451,"num_object_clones":0,"num_object_copies":217353,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":72451,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":133807,"num_write_kb":6600193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14270803968,"data_stored":14270791752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":130907,"ondisk_log_size":130907,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":9608,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30319,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430764,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2929972,"kb_used_data":2043820,"kb_used_omap":0,"kb_used_meta":886144,"kb_avail":90815180,"statfs":{"total":95995035648,"available":92994744320,"internally_reserved":0,"allocated":2092871680,"data_stored":2092320249,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":907411456},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:58:21 2023","interfaces":[{"interface":"back","average":{"1min":1.339,"5min":1.554,"15min":2.431},"min":{"1min":0.33300000000000002,"5min":0.252,"15min":0.252},"max":{"1min":4.423,"5min":14.315,"15min":137.34200000000001},"last":0.67300000000000004},{"interface":"front","average":{"1min":1.4670000000000001,"5min":1.5129999999999999,"15min":2.339},"min":{"1min":0.373,"5min":0.32500000000000001,"15min":0.307},"max":{"1min":5.0750000000000002,"5min":14.336,"15min":137.39500000000001},"last":3.7759999999999998}]},{"osd":1,"last update":"Sun Oct 1 10:58:24 2023","interfaces":[{"interface":"back","average":{"1min":0.81299999999999994,"5min":1.593,"15min":1.6859999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":2.9929999999999999,"5min":15.662000000000001,"15min":56.110999999999997},"last":3.73},{"interface":"front","average":{"1min":1.323,"5min":1.6299999999999999,"15min":1.6659999999999999},"min":{"1min":0.33000000000000002,"5min":0.30599999999999999,"15min":0.27100000000000002},"max":{"1min":4.3940000000000001,"5min":18.768999999999998,"15min":51.258000000000003},"last":0.68999999999999995}]},{"osd":2,"last update":"Sun Oct 1 10:58:40 2023","interfaces":[{"interface":"back","average":{"1min":4.484,"5min":3.1760000000000002,"15min":2.0739999999999998},"min":{"1min":0.36499999999999999,"5min":0.33300000000000002,"15min":0.315},"max":{"1min":39.658000000000001,"5min":42.804000000000002,"15min":42.804000000000002},"last":0.73599999999999999},{"interface":"front","average":{"1min":3.4980000000000002,"5min":2.8370000000000002,"15min":1.883},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":42.072000000000003,"5min":42.819000000000003,"15min":42.819000000000003},"last":0.80700000000000005}]},{"osd":3,"last update":"Sun Oct 1 10:58:21 2023","interfaces":[{"interface":"back","average":{"1min":1.6859999999999999,"5min":1.625,"15min":1.8440000000000001},"min":{"1min":0.47899999999999998,"5min":0.38200000000000001,"15min":0.318},"max":{"1min":7.0590000000000002,"5min":8.6839999999999993,"15min":51.334000000000003},"last":0.91200000000000003},{"interface":"front","average":{"1min":1.458,"5min":1.708,"15min":1.8640000000000001},"min":{"1min":0.35999999999999999,"5min":0.34699999999999998,"15min":0.313},"max":{"1min":4.6260000000000003,"5min":11.244999999999999,"15min":51.404000000000003},"last":0.76000000000000001}]},{"osd":4,"last update":"Sun Oct 1 10:58:47 2023","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":1.3460000000000001,"15min":1.4099999999999999},"min":{"1min":0.433,"5min":0.40500000000000003,"15min":0.33500000000000002},"max":{"1min":3.6890000000000001,"5min":7.7009999999999996,"15min":11.081},"last":3.968},{"interface":"front","average":{"1min":1.3799999999999999,"5min":1.5029999999999999,"15min":1.5049999999999999},"min":{"1min":0.44900000000000001,"5min":0.43099999999999999,"15min":0.35699999999999998},"max":{"1min":5.5499999999999998,"5min":8.7370000000000001,"15min":12.292},"last":0.82299999999999995}]},{"osd":5,"last update":"Sun Oct 1 10:58:24 2023","interfaces":[{"interface":"back","average":{"1min":1.5009999999999999,"5min":1.5580000000000001,"15min":1.8460000000000001},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.34599999999999997},"max":{"1min":4.4329999999999998,"5min":15.726000000000001,"15min":40.381},"last":0.77200000000000002},{"interface":"front","average":{"1min":0.81000000000000005,"5min":1.375,"15min":1.774},"min":{"1min":0.46100000000000002,"5min":0.39200000000000002,"15min":0.32200000000000001},"max":{"1min":2.625,"5min":9.1709999999999994,"15min":39.590000000000003},"last":0.89900000000000002}]},{"osd":6,"last update":"Sun Oct 1 10:58:24 2023","interfaces":[{"interface":"back","average":{"1min":1.595,"5min":1.425,"15min":1.556},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.34499999999999997},"max":{"1min":5.4050000000000002,"5min":9.4990000000000006,"15min":12.329000000000001},"last":0.79800000000000004},{"interface":"front","average":{"1min":1.3420000000000001,"5min":1.486,"15min":1.5429999999999999},"min":{"1min":0.378,"5min":0.378,"15min":0.37},"max":{"1min":5.1260000000000003,"5min":8.0120000000000005,"15min":9.4329999999999998},"last":4.0019999999999998}]}]},{"osd":6,"up_from":42,"seq":180388626998,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2912480,"kb_used_data":2027544,"kb_used_omap":0,"kb_used_meta":884800,"kb_avail":90832672,"statfs":{"total":95995035648,"available":93012656128,"internally_reserved":0,"allocated":2076205056,"data_stored":2075673620,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":906035200},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:58:15 2023","interfaces":[{"interface":"back","average":{"1min":1.198,"5min":1.446,"15min":2.7669999999999999},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.25},"max":{"1min":2.887,"5min":8.2899999999999991,"15min":140.524},"last":0.56100000000000005},{"interface":"front","average":{"1min":1.554,"5min":1.4339999999999999,"15min":2.774},"min":{"1min":0.36399999999999999,"5min":0.29399999999999998,"15min":0.249},"max":{"1min":6.4870000000000001,"5min":9.5259999999999998,"15min":140.86500000000001},"last":0.42099999999999999}]},{"osd":1,"last update":"Sun Oct 1 10:58:25 2023","interfaces":[{"interface":"back","average":{"1min":1.0720000000000001,"5min":1.5129999999999999,"15min":1.5229999999999999},"min":{"1min":0.34899999999999998,"5min":0.27100000000000002,"15min":0.26000000000000001},"max":{"1min":3.7410000000000001,"5min":9.843,"15min":9.843},"last":2.9079999999999999},{"interface":"front","average":{"1min":1.522,"5min":1.5189999999999999,"15min":1.609},"min":{"1min":0.36099999999999999,"5min":0.26400000000000001,"15min":0.26300000000000001},"max":{"1min":4.5250000000000004,"5min":9.4809999999999999,"15min":12.968999999999999},"last":0.81399999999999995}]},{"osd":2,"last update":"Sun Oct 1 10:58:37 2023","interfaces":[{"interface":"back","average":{"1min":8.4619999999999997,"5min":4.6289999999999996,"15min":3.4340000000000002},"min":{"1min":0.34799999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":106.705,"5min":122.768,"15min":122.768},"last":0.67700000000000005},{"interface":"front","average":{"1min":9.0099999999999998,"5min":5.2279999999999998,"15min":3.6059999999999999},"min":{"1min":0.44500000000000001,"5min":0.36199999999999999,"15min":0.32100000000000001},"max":{"1min":106.806,"5min":123.02200000000001,"15min":123.02200000000001},"last":0.70199999999999996}]},{"osd":3,"last update":"Sun Oct 1 10:58:25 2023","interfaces":[{"interface":"back","average":{"1min":1.173,"5min":1.8540000000000001,"15min":2.0070000000000001},"min":{"1min":0.36499999999999999,"5min":0.31,"15min":0.30599999999999999},"max":{"1min":3.972,"5min":36.698999999999998,"15min":36.698999999999998},"last":7.5910000000000002},{"interface":"front","average":{"1min":1.8779999999999999,"5min":1.954,"15min":2.0840000000000001},"min":{"1min":0.54300000000000004,"5min":0.36499999999999999,"15min":0.315},"max":{"1min":4.4909999999999997,"5min":33.555999999999997,"15min":33.555999999999997},"last":0.52400000000000002}]},{"osd":4,"last update":"Sun Oct 1 10:58:03 2023","interfaces":[{"interface":"back","average":{"1min":0.69099999999999995,"5min":1.1379999999999999,"15min":1.532},"min":{"1min":0.35799999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":2.1509999999999998,"5min":7.5369999999999999,"15min":13.1},"last":0.88500000000000001},{"interface":"front","average":{"1min":1.8340000000000001,"5min":1.5609999999999999,"15min":1.6639999999999999},"min":{"1min":0.38700000000000001,"5min":0.35999999999999999,"15min":0.315},"max":{"1min":7.4509999999999996,"5min":8.016,"15min":12.157999999999999},"last":0.63400000000000001}]},{"osd":5,"last update":"Sun Oct 1 10:58:25 2023","interfaces":[{"interface":"back","average":{"1min":0.64200000000000002,"5min":1.45,"15min":1.5569999999999999},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.33000000000000002},"max":{"1min":1.1379999999999999,"5min":15.238,"15min":15.238},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.92700000000000005,"5min":1.671,"15min":1.829},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.308},"max":{"1min":2.665,"5min":13.295,"15min":14.459},"last":0.84999999999999998}]},{"osd":7,"last update":"Sun Oct 1 10:58:15 2023","interfaces":[{"interface":"back","average":{"1min":1.228,"5min":1.403,"15min":1.675},"min":{"1min":0.42099999999999999,"5min":0.33900000000000002,"15min":0.311},"max":{"1min":6.04,"5min":8.4199999999999999,"15min":13.276999999999999},"last":3.0510000000000002},{"interface":"front","average":{"1min":0.82599999999999996,"5min":1.47,"15min":1.6890000000000001},"min":{"1min":0.38500000000000001,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":1.9770000000000001,"5min":7.1379999999999999,"15min":11.416},"last":0.83999999999999997}]}]},{"osd":1,"up_from":101,"seq":433791697215,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2141620,"kb_used_data":1490668,"kb_used_omap":4,"kb_used_meta":650939,"kb_avail":91603532,"statfs":{"total":95995035648,"available":93802016768,"internally_reserved":0,"allocated":1526444032,"data_stored":1525432620,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":666562446},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:58:29 2023","interfaces":[{"interface":"back","average":{"1min":1.96,"5min":1.4490000000000001,"15min":1.395},"min":{"1min":0.23000000000000001,"5min":0.19900000000000001,"15min":0.19600000000000001},"max":{"1min":7.0359999999999996,"5min":12.888999999999999,"15min":37.058},"last":1.9870000000000001},{"interface":"front","average":{"1min":1.282,"5min":1.556,"15min":1.7689999999999999},"min":{"1min":0.23699999999999999,"5min":0.20699999999999999,"15min":0.192},"max":{"1min":4.1040000000000001,"5min":15.416,"15min":61.207999999999998},"last":0.86699999999999999}]},{"osd":2,"last update":"Sun Oct 1 10:58:47 2023","interfaces":[{"interface":"back","average":{"1min":2.5009999999999999,"5min":2.0150000000000001,"15min":2.3610000000000002},"min":{"1min":0.252,"5min":0.252,"15min":0.222},"max":{"1min":10.522,"5min":19.149999999999999,"15min":161.905},"last":9.0779999999999994},{"interface":"front","average":{"1min":2.6960000000000002,"5min":2.1030000000000002,"15min":2.3610000000000002},"min":{"1min":0.40999999999999998,"5min":0.26900000000000002,"15min":0.23699999999999999},"max":{"1min":9.3930000000000007,"5min":18.815000000000001,"15min":162.059},"last":6.4260000000000002}]},{"osd":3,"last update":"Sun Oct 1 10:58:29 2023","interfaces":[{"interface":"back","average":{"1min":2.8159999999999998,"5min":3.044,"15min":2.198},"min":{"1min":0.27100000000000002,"5min":0.26000000000000001,"15min":0.20000000000000001},"max":{"1min":6.8620000000000001,"5min":143.73400000000001,"15min":143.73400000000001},"last":0.91400000000000003},{"interface":"front","average":{"1min":1.954,"5min":3.1400000000000001,"15min":2.3300000000000001},"min":{"1min":0.29499999999999998,"5min":0.25900000000000001,"15min":0.23999999999999999},"max":{"1min":6.5460000000000003,"5min":143.00399999999999,"15min":143.00399999999999},"last":1.038}]},{"osd":4,"last update":"Sun Oct 1 10:58:47 2023","interfaces":[{"interface":"back","average":{"1min":1.2989999999999999,"5min":1.3180000000000001,"15min":1.3520000000000001},"min":{"1min":0.34599999999999997,"5min":0.307,"15min":0.29199999999999998},"max":{"1min":5.1070000000000002,"5min":8.593,"15min":10.004},"last":0.434},{"interface":"front","average":{"1min":1.2170000000000001,"5min":1.3029999999999999,"15min":1.53},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.32700000000000001},"max":{"1min":4.4470000000000001,"5min":8.7530000000000001,"15min":13.122999999999999},"last":0.90300000000000002}]},{"osd":5,"last update":"Sun Oct 1 10:58:29 2023","interfaces":[{"interface":"back","average":{"1min":1.238,"5min":1.399,"15min":1.7809999999999999},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":6.4989999999999997,"5min":9.1809999999999992,"15min":61.560000000000002},"last":1.075},{"interface":"front","average":{"1min":2.0289999999999999,"5min":1.302,"15min":1.6879999999999999},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":6.75,"5min":8.7949999999999999,"15min":61.667999999999999},"last":2.4249999999999998}]},{"osd":6,"last update":"Sun Oct 1 10:58:29 2023","interfaces":[{"interface":"back","average":{"1min":2.048,"5min":1.5549999999999999,"15min":1.8029999999999999},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.371},"max":{"1min":6.7949999999999999,"5min":8.8309999999999995,"15min":62.219999999999999},"last":0.66600000000000004},{"interface":"front","average":{"1min":1.1000000000000001,"5min":1.3839999999999999,"15min":1.766},"min":{"1min":0.51500000000000001,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":2.1080000000000001,"5min":8.8970000000000002,"15min":61.462000000000003},"last":1.1200000000000001}]},{"osd":7,"last update":"Sun Oct 1 10:58:29 2023","interfaces":[{"interface":"back","average":{"1min":1.169,"5min":1.2709999999999999,"15min":1.776},"min":{"1min":0.48099999999999998,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":2.3730000000000002,"5min":9.7140000000000004,"15min":62.646999999999998},"last":1.4330000000000001},{"interface":"front","average":{"1min":1.3009999999999999,"5min":1.524,"15min":1.8300000000000001},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.38100000000000001},"max":{"1min":3.556,"5min":9.1449999999999996,"15min":62.048000000000002},"last":0.98899999999999999}]}]},{"osd":0,"up_from":86,"seq":369367187826,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2557388,"kb_used_data":1755716,"kb_used_omap":5,"kb_used_meta":801658,"kb_avail":91187764,"statfs":{"total":95995035648,"available":93376270336,"internally_reserved":0,"allocated":1797853184,"data_stored":1796876421,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":820898426},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 10:58:26 2023","interfaces":[{"interface":"back","average":{"1min":1.75,"5min":1.3640000000000001,"15min":1.4199999999999999},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.184},"max":{"1min":4.8239999999999998,"5min":6.7850000000000001,"15min":15.074},"last":2.0600000000000001},{"interface":"front","average":{"1min":1.169,"5min":1.464,"15min":1.548},"min":{"1min":0.22,"5min":0.20200000000000001,"15min":0.19700000000000001},"max":{"1min":7.4349999999999996,"5min":10.289,"15min":13.601000000000001},"last":0.249}]},{"osd":2,"last update":"Sun Oct 1 10:58:45 2023","interfaces":[{"interface":"back","average":{"1min":2.9710000000000001,"5min":2.4350000000000001,"15min":1.869},"min":{"1min":0.39300000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":11.108000000000001,"5min":44.744999999999997,"15min":44.744999999999997},"last":5.6139999999999999},{"interface":"front","average":{"1min":3.032,"5min":2.601,"15min":2.0259999999999998},"min":{"1min":0.252,"5min":0.252,"15min":0.221},"max":{"1min":11.073,"5min":44.691000000000003,"15min":44.691000000000003},"last":5.9640000000000004}]},{"osd":3,"last update":"Sun Oct 1 10:58:26 2023","interfaces":[{"interface":"back","average":{"1min":3.02,"5min":1.863,"15min":1.673},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":13.632999999999999,"5min":13.632999999999999,"15min":13.632999999999999},"last":5.7050000000000001},{"interface":"front","average":{"1min":3.2069999999999999,"5min":1.6930000000000001,"15min":1.6120000000000001},"min":{"1min":0.27800000000000002,"5min":0.246,"15min":0.246},"max":{"1min":12.727,"5min":13.641,"15min":13.641},"last":0.41399999999999998}]},{"osd":4,"last update":"Sun Oct 1 10:58:05 2023","interfaces":[{"interface":"back","average":{"1min":1.6599999999999999,"5min":1.304,"15min":1.5740000000000001},"min":{"1min":0.35799999999999998,"5min":0.34300000000000003,"15min":0.32000000000000001},"max":{"1min":9.2889999999999997,"5min":9.2889999999999997,"15min":11.379},"last":0.53900000000000003},{"interface":"front","average":{"1min":1.4890000000000001,"5min":1.4399999999999999,"15min":1.5369999999999999},"min":{"1min":0.32300000000000001,"5min":0.311,"15min":0.311},"max":{"1min":4.883,"5min":6.9080000000000004,"15min":11.362},"last":0.40400000000000003}]},{"osd":5,"last update":"Sun Oct 1 10:58:26 2023","interfaces":[{"interface":"back","average":{"1min":1.96,"5min":1.5580000000000001,"15min":1.71},"min":{"1min":0.35099999999999998,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":7.6890000000000001,"5min":7.6890000000000001,"15min":12.91},"last":5.766},{"interface":"front","average":{"1min":1.716,"5min":1.258,"15min":1.546},"min":{"1min":0.47199999999999998,"5min":0.41199999999999998,"15min":0.34699999999999998},"max":{"1min":4.7869999999999999,"5min":4.7869999999999999,"15min":9.8620000000000001},"last":0.69899999999999995}]},{"osd":6,"last update":"Sun Oct 1 10:58:26 2023","interfaces":[{"interface":"back","average":{"1min":1.579,"5min":1.294,"15min":1.5780000000000001},"min":{"1min":0.45200000000000001,"5min":0.39800000000000002,"15min":0.373},"max":{"1min":4.8070000000000004,"5min":4.8070000000000004,"15min":9.8279999999999994},"last":0.94699999999999995},{"interface":"front","average":{"1min":1.4410000000000001,"5min":1.4810000000000001,"15min":1.681},"min":{"1min":0.39700000000000002,"5min":0.37,"15min":0.36899999999999999},"max":{"1min":5.1340000000000003,"5min":7.3120000000000003,"15min":10.111000000000001},"last":0.96699999999999997}]},{"osd":7,"last update":"Sun Oct 1 10:58:26 2023","interfaces":[{"interface":"back","average":{"1min":2.2120000000000002,"5min":1.5800000000000001,"15min":1.677},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.39700000000000002},"max":{"1min":12.680999999999999,"5min":13.699,"15min":15.839},"last":0.61899999999999999},{"interface":"front","average":{"1min":2.0539999999999998,"5min":1.7370000000000001,"15min":1.722},"min":{"1min":0.47299999999999998,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":7.1859999999999999,"5min":7.2679999999999998,"15min":8.9440000000000008},"last":0.68500000000000005}]}]},{"osd":2,"up_from":116,"seq":498216206601,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3491636,"kb_used_data":2904428,"kb_used_omap":7,"kb_used_meta":587192,"kb_avail":90253516,"statfs":{"total":95995035648,"available":92419600384,"internally_reserved":0,"allocated":2974134272,"data_stored":2973176885,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":601284689},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,4,1,3,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:58:47 2023","interfaces":[{"interface":"back","average":{"1min":2.0390000000000001,"5min":2.6349999999999998,"15min":2.5600000000000001},"min":{"1min":0.34499999999999997,"5min":0.20200000000000001,"15min":0.16900000000000001},"max":{"1min":6.3280000000000003,"5min":104.42,"15min":147.87},"last":0.28599999999999998},{"interface":"front","average":{"1min":2.4620000000000002,"5min":2.9929999999999999,"15min":2.7410000000000001},"min":{"1min":0.222,"5min":0.222,"15min":0.20599999999999999},"max":{"1min":8.6400000000000006,"5min":105.10899999999999,"15min":148.03100000000001},"last":0.998}]},{"osd":1,"last update":"Sun Oct 1 10:58:47 2023","interfaces":[{"interface":"back","average":{"1min":2.4020000000000001,"5min":1.6419999999999999,"15min":2.0110000000000001},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.217},"max":{"1min":5.9340000000000002,"5min":7.8949999999999996,"15min":122.249},"last":0.96699999999999997},{"interface":"front","average":{"1min":3.2570000000000001,"5min":2.0910000000000002,"15min":2.4129999999999998},"min":{"1min":0.373,"5min":0.26500000000000001,"15min":0.20599999999999999},"max":{"1min":11.554,"5min":11.554,"15min":121.495},"last":0.41299999999999998}]},{"osd":3,"last update":"Sun Oct 1 10:58:13 2023","interfaces":[{"interface":"back","average":{"1min":2.9609999999999999,"5min":1.9330000000000001,"15min":2.3370000000000002},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.20999999999999999},"max":{"1min":15.897,"5min":15.897,"15min":124.393},"last":1.161},{"interface":"front","average":{"1min":3.472,"5min":1.7949999999999999,"15min":2.4609999999999999},"min":{"1min":0.26400000000000001,"5min":0.221,"15min":0.20899999999999999},"max":{"1min":11.404999999999999,"5min":11.404999999999999,"15min":122.52200000000001},"last":1.1200000000000001}]},{"osd":4,"last update":"Sun Oct 1 10:58:57 2023","interfaces":[{"interface":"back","average":{"1min":2.4460000000000002,"5min":1.6879999999999999,"15min":1.7709999999999999},"min":{"1min":0.42999999999999999,"5min":0.40400000000000003,"15min":0.36499999999999999},"max":{"1min":6.2400000000000002,"5min":7.8090000000000002,"15min":60.994},"last":0.93999999999999995},{"interface":"front","average":{"1min":2.7010000000000001,"5min":1.905,"15min":2.0950000000000002},"min":{"1min":0.39700000000000002,"5min":0.36599999999999999,"15min":0.30299999999999999},"max":{"1min":7.5129999999999999,"5min":11.574,"15min":63.012},"last":0.623}]},{"osd":5,"last update":"Sun Oct 1 10:58:47 2023","interfaces":[{"interface":"back","average":{"1min":2.3900000000000001,"5min":1.7969999999999999,"15min":1.899},"min":{"1min":0.39800000000000002,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":6.2649999999999997,"5min":8.5440000000000005,"15min":61.021000000000001},"last":0.91000000000000003},{"interface":"front","average":{"1min":2.5219999999999998,"5min":2.1429999999999998,"15min":2.1890000000000001},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.34100000000000003},"max":{"1min":7.4740000000000002,"5min":12.835000000000001,"15min":63.145000000000003},"last":0.66600000000000004}]},{"osd":6,"last update":"Sun Oct 1 10:58:47 2023","interfaces":[{"interface":"back","average":{"1min":3.4380000000000002,"5min":2.4009999999999998,"15min":2.2949999999999999},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.32100000000000001},"max":{"1min":7.7690000000000001,"5min":19.908000000000001,"15min":63.289000000000001},"last":0.69099999999999995},{"interface":"front","average":{"1min":1.742,"5min":1.756,"15min":1.754},"min":{"1min":0.42299999999999999,"5min":0.38300000000000001,"15min":0.35299999999999998},"max":{"1min":6.4020000000000001,"5min":9.2279999999999998,"15min":60.579000000000001},"last":0.38300000000000001}]},{"osd":7,"last update":"Sun Oct 1 10:58:47 2023","interfaces":[{"interface":"back","average":{"1min":2.468,"5min":1.7010000000000001,"15min":1.679},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.34300000000000003},"max":{"1min":11.209,"5min":11.209,"15min":60.493000000000002},"last":0.45000000000000001},{"interface":"front","average":{"1min":2.5179999999999998,"5min":1.821,"15min":1.9039999999999999},"min":{"1min":0.45600000000000002,"5min":0.40100000000000002,"15min":0.36399999999999999},"max":{"1min":6.407,"5min":7.9279999999999999,"15min":60.956000000000003},"last":0.85799999999999998}]}]},{"osd":3,"up_from":121,"seq":519691043073,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2968852,"kb_used_data":2346636,"kb_used_omap":2,"kb_used_meta":622141,"kb_avail":90776300,"statfs":{"total":95995035648,"available":92954931200,"internally_reserved":0,"allocated":2402955264,"data_stored":2402027314,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":637072437},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,2,4,3,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:58:23 2023","interfaces":[{"interface":"back","average":{"1min":6.3360000000000003,"5min":2.3620000000000001,"15min":2.6749999999999998},"min":{"1min":0.17199999999999999,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":100.191,"5min":100.191,"15min":129.953},"last":3.5699999999999998},{"interface":"front","average":{"1min":6.5330000000000004,"5min":2.6779999999999999,"15min":3.0299999999999998},"min":{"1min":0.23000000000000001,"5min":0.19600000000000001,"15min":0.189},"max":{"1min":99.852999999999994,"5min":99.852999999999994,"15min":129.53999999999999},"last":11.077}]},{"osd":1,"last update":"Sun Oct 1 10:58:23 2023","interfaces":[{"interface":"back","average":{"1min":1.5629999999999999,"5min":1.4710000000000001,"15min":1.423},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.193},"max":{"1min":7.2850000000000001,"5min":10.574999999999999,"15min":10.574999999999999},"last":3.5329999999999999},{"interface":"front","average":{"1min":1.3460000000000001,"5min":1.5800000000000001,"15min":1.5589999999999999},"min":{"1min":0.32100000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":3.8050000000000002,"5min":13.529999999999999,"15min":13.529999999999999},"last":3.4990000000000001}]},{"osd":2,"last update":"Sun Oct 1 10:58:23 2023","interfaces":[{"interface":"back","average":{"1min":2.9430000000000001,"5min":1.931,"15min":1.673},"min":{"1min":0.38600000000000001,"5min":0.217,"15min":0.217},"max":{"1min":11.571,"5min":12.238,"15min":15.148},"last":0.38900000000000001},{"interface":"front","average":{"1min":1.6339999999999999,"5min":1.704,"15min":1.5189999999999999},"min":{"1min":0.27100000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":5.4299999999999997,"5min":10.66,"15min":10.66},"last":10.872999999999999}]},{"osd":4,"last update":"Sun Oct 1 10:58:01 2023","interfaces":[{"interface":"back","average":{"1min":1.2250000000000001,"5min":1.4730000000000001,"15min":1.655},"min":{"1min":0.49199999999999999,"5min":0.36799999999999999,"15min":0.34599999999999997},"max":{"1min":4.6269999999999998,"5min":10.724,"15min":75.388999999999996},"last":10.808999999999999},{"interface":"front","average":{"1min":1.4850000000000001,"5min":1.6339999999999999,"15min":1.8},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.29999999999999999},"max":{"1min":7.6639999999999997,"5min":13.391999999999999,"15min":75.165999999999997},"last":10.962999999999999}]},{"osd":5,"last update":"Sun Oct 1 10:58:23 2023","interfaces":[{"interface":"back","average":{"1min":1.5389999999999999,"5min":1.575,"15min":1.556},"min":{"1min":0.36699999999999999,"5min":0.34399999999999997,"15min":0.33100000000000002},"max":{"1min":7.6360000000000001,"5min":13.333,"15min":13.333},"last":11.050000000000001},{"interface":"front","average":{"1min":2.4670000000000001,"5min":1.835,"15min":1.77},"min":{"1min":0.41399999999999998,"5min":0.38900000000000001,"15min":0.33200000000000002},"max":{"1min":11.433,"5min":12.335000000000001,"15min":12.335000000000001},"last":0.495}]},{"osd":6,"last update":"Sun Oct 1 10:58:23 2023","interfaces":[{"interface":"back","average":{"1min":1.752,"5min":1.7869999999999999,"15min":1.8300000000000001},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.34999999999999998},"max":{"1min":7.7030000000000003,"5min":8.8740000000000006,"15min":8.8740000000000006},"last":11.016999999999999},{"interface":"front","average":{"1min":2.6190000000000002,"5min":1.9610000000000001,"15min":1.986},"min":{"1min":0.47699999999999998,"5min":0.36799999999999999,"15min":0.35899999999999999},"max":{"1min":11.539999999999999,"5min":12.378,"15min":12.378},"last":0.52100000000000002}]},{"osd":7,"last update":"Sun Oct 1 10:58:23 2023","interfaces":[{"interface":"back","average":{"1min":2.665,"5min":2.0329999999999999,"15min":1.74},"min":{"1min":0.44600000000000001,"5min":0.42799999999999999,"15min":0.35099999999999998},"max":{"1min":11.384,"5min":12.416,"15min":12.416},"last":0.54400000000000004},{"interface":"front","average":{"1min":1.5589999999999999,"5min":1.635,"15min":1.617},"min":{"1min":0.56899999999999995,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":4.1200000000000001,"5min":10.755000000000001,"15min":10.755000000000001},"last":5.093}]}]},{"osd":4,"up_from":126,"seq":541165879547,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1275960,"kb_used_data":860784,"kb_used_omap":10,"kb_used_meta":415157,"kb_avail":92469192,"statfs":{"total":95995035648,"available":94688452608,"internally_reserved":0,"allocated":881442816,"data_stored":880481402,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":425121380},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:58:54 2023","interfaces":[{"interface":"back","average":{"1min":2.915,"5min":1.651,"15min":2.0739999999999998},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.24399999999999999},"max":{"1min":18.484000000000002,"5min":35.741999999999997,"15min":62.576000000000001},"last":0.34300000000000003},{"interface":"front","average":{"1min":2.6099999999999999,"5min":1.7110000000000001,"15min":2.0640000000000001},"min":{"1min":0.317,"5min":0.28799999999999998,"15min":0.251},"max":{"1min":16.966000000000001,"5min":35.701000000000001,"15min":64.941000000000003},"last":0.38600000000000001}]},{"osd":1,"last update":"Sun Oct 1 10:58:54 2023","interfaces":[{"interface":"back","average":{"1min":1.1799999999999999,"5min":1.242,"15min":1.794},"min":{"1min":0.32300000000000001,"5min":0.30199999999999999,"15min":0.27300000000000002},"max":{"1min":3.7629999999999999,"5min":11.664,"15min":97.622},"last":3.9649999999999999},{"interface":"front","average":{"1min":1.4550000000000001,"5min":1.1399999999999999,"15min":1.925},"min":{"1min":0.317,"5min":0.317,"15min":0.28699999999999998},"max":{"1min":4.1079999999999997,"5min":11.914999999999999,"15min":97.346999999999994},"last":2.7919999999999998}]},{"osd":2,"last update":"Sun Oct 1 10:58:54 2023","interfaces":[{"interface":"back","average":{"1min":3.2389999999999999,"5min":3.411,"15min":2.73},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.29899999999999999},"max":{"1min":22.774999999999999,"5min":68.064999999999998,"15min":117.229},"last":3.5430000000000001},{"interface":"front","average":{"1min":2.6549999999999998,"5min":3.3399999999999999,"15min":2.5760000000000001},"min":{"1min":0.36899999999999999,"5min":0.35399999999999998,"15min":0.29699999999999999},"max":{"1min":17.065000000000001,"5min":68.379000000000005,"15min":116.922},"last":0.51100000000000001}]},{"osd":3,"last update":"Sun Oct 1 10:58:54 2023","interfaces":[{"interface":"back","average":{"1min":1.2170000000000001,"5min":1.3360000000000001,"15min":1.8220000000000001},"min":{"1min":0.36399999999999999,"5min":0.32700000000000001,"15min":0.29499999999999998},"max":{"1min":3.976,"5min":9.1780000000000008,"15min":61.398000000000003},"last":0.42599999999999999},{"interface":"front","average":{"1min":1.25,"5min":2.7200000000000002,"15min":2.2589999999999999},"min":{"1min":0.374,"5min":0.308,"15min":0.308},"max":{"1min":4.0389999999999997,"5min":130.88300000000001,"15min":130.88300000000001},"last":7.9770000000000003}]},{"osd":5,"last update":"Sun Oct 1 10:58:54 2023","interfaces":[{"interface":"back","average":{"1min":0.872,"5min":2.0299999999999998,"15min":1.7589999999999999},"min":{"1min":0.39700000000000002,"5min":0.37,"15min":0.30199999999999999},"max":{"1min":3.9580000000000002,"5min":84.307000000000002,"15min":84.307000000000002},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.84399999999999997,"5min":2.2490000000000001,"15min":1.841},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.28999999999999998},"max":{"1min":4.1360000000000001,"5min":84.234999999999999,"15min":84.234999999999999},"last":2.7040000000000002}]},{"osd":6,"last update":"Sun Oct 1 10:58:54 2023","interfaces":[{"interface":"back","average":{"1min":1.1950000000000001,"5min":1.135,"15min":1.6279999999999999},"min":{"1min":0.39200000000000002,"5min":0.34999999999999998,"15min":0.33800000000000002},"max":{"1min":4.4109999999999996,"5min":9.0109999999999992,"15min":13.646000000000001},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.98399999999999999,"5min":1.123,"15min":1.611},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.33400000000000002},"max":{"1min":3.6469999999999998,"5min":7.5510000000000002,"15min":14.446999999999999},"last":3.0640000000000001}]},{"osd":7,"last update":"Sun Oct 1 10:58:54 2023","interfaces":[{"interface":"back","average":{"1min":1.099,"5min":1.2230000000000001,"15min":1.929},"min":{"1min":0.45800000000000002,"5min":0.43099999999999999,"15min":0.33900000000000002},"max":{"1min":4.4240000000000004,"5min":7.5960000000000001,"15min":119.56100000000001},"last":3.1110000000000002},{"interface":"front","average":{"1min":0.996,"5min":1.0549999999999999,"15min":1.958},"min":{"1min":0.46600000000000003,"5min":0.40300000000000002,"15min":0.30399999999999999},"max":{"1min":4.0659999999999998,"5min":4.0659999999999998,"15min":120.44499999999999},"last":2.7559999999999998}]}]},{"osd":5,"up_from":36,"seq":154618823228,"num_pgs":23,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1489556,"kb_used_data":589388,"kb_used_omap":0,"kb_used_meta":900160,"kb_avail":92255596,"statfs":{"total":95995035648,"available":94469730304,"internally_reserved":0,"allocated":603533312,"data_stored":603009440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":921763840},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:58:16 2023","interfaces":[{"interface":"back","average":{"1min":1.9410000000000001,"5min":1.26,"15min":2.016},"min":{"1min":0.28599999999999998,"5min":0.24299999999999999,"15min":0.24099999999999999},"max":{"1min":13.955,"5min":13.955,"15min":123.39700000000001},"last":0.27200000000000002},{"interface":"front","average":{"1min":1.716,"5min":1.3129999999999999,"15min":2.214},"min":{"1min":0.30599999999999999,"5min":0.253,"15min":0.253},"max":{"1min":13.978,"5min":13.978,"15min":124.051},"last":0.42699999999999999}]},{"osd":1,"last update":"Sun Oct 1 10:58:16 2023","interfaces":[{"interface":"back","average":{"1min":0.79800000000000004,"5min":1.238,"15min":2.3010000000000002},"min":{"1min":0.32400000000000001,"5min":0.28799999999999998,"15min":0.27100000000000002},"max":{"1min":4.9530000000000003,"5min":9.766,"15min":202.96600000000001},"last":2.2280000000000002},{"interface":"front","average":{"1min":1.147,"5min":1.2509999999999999,"15min":2.5649999999999999},"min":{"1min":0.34100000000000003,"5min":0.32800000000000001,"15min":0.28100000000000003},"max":{"1min":3.9460000000000002,"5min":7.7290000000000001,"15min":203.05099999999999},"last":0.46300000000000002}]},{"osd":2,"last update":"Sun Oct 1 10:58:55 2023","interfaces":[{"interface":"back","average":{"1min":1.454,"5min":1.5009999999999999,"15min":1.6659999999999999},"min":{"1min":0.34200000000000003,"5min":0.32600000000000001,"15min":0.29199999999999998},"max":{"1min":10.206,"5min":10.510999999999999,"15min":26.242000000000001},"last":0.311},{"interface":"front","average":{"1min":1.375,"5min":1.5209999999999999,"15min":1.925},"min":{"1min":0.318,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":5.2389999999999999,"5min":10.826000000000001,"15min":25.643999999999998},"last":0.53800000000000003}]},{"osd":3,"last update":"Sun Oct 1 10:58:16 2023","interfaces":[{"interface":"back","average":{"1min":1.5329999999999999,"5min":3.2050000000000001,"15min":2.4449999999999998},"min":{"1min":0.53700000000000003,"5min":0.32400000000000001,"15min":0.316},"max":{"1min":5.0789999999999997,"5min":166.29599999999999,"15min":166.29599999999999},"last":0.49299999999999999},{"interface":"front","average":{"1min":1.786,"5min":3.3639999999999999,"15min":2.5409999999999999},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":4.8860000000000001,"5min":166.565,"15min":166.565},"last":2.3860000000000001}]},{"osd":4,"last update":"Sun Oct 1 10:58:55 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.89400000000000002,"15min":1.431},"min":{"1min":0.372,"5min":0.372,"15min":0.29399999999999998},"max":{"1min":1.1899999999999999,"5min":3.9409999999999998,"15min":15.170999999999999},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.76900000000000002,"5min":0.97599999999999998,"15min":1.4750000000000001},"min":{"1min":0.41299999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":2.3180000000000001,"5min":6.8879999999999999,"15min":9.7270000000000003},"last":0.34100000000000003}]},{"osd":6,"last update":"Sun Oct 1 10:58:16 2023","interfaces":[{"interface":"back","average":{"1min":0.749,"5min":1.282,"15min":1.806},"min":{"1min":0.48599999999999999,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":1.256,"5min":9.0790000000000006,"15min":61.768999999999998},"last":4.7000000000000002},{"interface":"front","average":{"1min":0.81899999999999995,"5min":1.1120000000000001,"15min":1.891},"min":{"1min":0.39000000000000001,"5min":0.33000000000000002,"15min":0.28899999999999998},"max":{"1min":2.0350000000000001,"5min":5.9690000000000003,"15min":62.396999999999998},"last":0.63700000000000001}]},{"osd":7,"last update":"Sun Oct 1 10:58:16 2023","interfaces":[{"interface":"back","average":{"1min":1.0669999999999999,"5min":1.3320000000000001,"15min":1.7290000000000001},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.32800000000000001},"max":{"1min":4.8520000000000003,"5min":9.1470000000000002,"15min":13.673999999999999},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.80600000000000005,"5min":1.286,"15min":1.611},"min":{"1min":0.44600000000000001,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.8160000000000001,"5min":9.1340000000000003,"15min":11.635},"last":0.5}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1781071872,"data_stored":1781071872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1516769280,"data_stored":1516765208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2958299136,"data_stored":2958295064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2383544320,"data_stored":2383544320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":877723648,"data_stored":877723648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":599916544,"data_stored":599916544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2068971520,"data_stored":2068971520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2084507648,"data_stored":2084503576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T10:59:02.808 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_7 pgp_num 16 2023-10-01T10:59:02.853 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:02.875 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:02.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:02.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:02.978 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:03.004 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:03.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:03.007 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:59:03.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:02 smithi078 bash[75299]: cluster 2023-10-01T10:59:01.868423+0000 mon.a (mon.0) 1542 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:59:03.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:02 smithi078 bash[75299]: audit 2023-10-01T10:59:01.874766+0000 mon.a (mon.0) 1543 : audit 1 from='client.? 172.21.15.78:0/2741778675' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T10:59:03.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:02 smithi078 bash[75299]: cluster 2023-10-01T10:59:01.876541+0000 osd.7 (osd.7) 236 : cluster 3 failed to encode map e250 with expected crc 2023-10-01T10:59:03.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:02 smithi078 bash[75299]: cluster 2023-10-01T10:59:01.878360+0000 mon.a (mon.0) 1544 : cluster 0 osdmap e250: 8 total, 8 up, 8 in 2023-10-01T10:59:03.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:02 smithi078 bash[75299]: cluster 2023-10-01T10:59:01.889379+0000 osd.5 (osd.5) 231 : cluster 3 failed to encode map e250 with expected crc 2023-10-01T10:59:03.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:02 smithi078 bash[75299]: cluster 2023-10-01T10:59:01.890166+0000 osd.6 (osd.6) 195 : cluster 3 failed to encode map e250 with expected crc 2023-10-01T10:59:03.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:02 smithi078 bash[75299]: cluster 2023-10-01T10:59:01.893786+0000 osd.7 (osd.7) 237 : cluster 3 failed to encode map e250 with expected crc 2023-10-01T10:59:03.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:02 smithi078 bash[75299]: cluster 2023-10-01T10:59:01.895099+0000 osd.7 (osd.7) 238 : cluster 3 failed to encode map e250 with expected crc 2023-10-01T10:59:03.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:02 smithi078 bash[75299]: cluster 2023-10-01T10:59:01.895633+0000 osd.6 (osd.6) 196 : cluster 3 failed to encode map e250 with expected crc 2023-10-01T10:59:03.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:02 smithi078 bash[75299]: cluster 2023-10-01T10:59:01.896819+0000 osd.7 (osd.7) 239 : cluster 3 failed to encode map e250 with expected crc 2023-10-01T10:59:03.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:02 smithi078 bash[75299]: cluster 2023-10-01T10:59:01.898019+0000 osd.7 (osd.7) 240 : cluster 3 failed to encode map e250 with expected crc 2023-10-01T10:59:03.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:02 smithi078 bash[75299]: audit 2023-10-01T10:59:02.449120+0000 mon.a (mon.0) 1545 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:59:03.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:02 smithi078 bash[78274]: cluster 2023-10-01T10:59:01.868423+0000 mon.a (mon.0) 1542 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:59:03.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:02 smithi078 bash[78274]: audit 2023-10-01T10:59:01.874766+0000 mon.a (mon.0) 1543 : audit 1 from='client.? 172.21.15.78:0/2741778675' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T10:59:03.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:02 smithi078 bash[78274]: cluster 2023-10-01T10:59:01.876541+0000 osd.7 (osd.7) 236 : cluster 3 failed to encode map e250 with expected crc 2023-10-01T10:59:03.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:02 smithi078 bash[78274]: cluster 2023-10-01T10:59:01.878360+0000 mon.a (mon.0) 1544 : cluster 0 osdmap e250: 8 total, 8 up, 8 in 2023-10-01T10:59:03.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:02 smithi078 bash[78274]: cluster 2023-10-01T10:59:01.889379+0000 osd.5 (osd.5) 231 : cluster 3 failed to encode map e250 with expected crc 2023-10-01T10:59:03.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:02 smithi078 bash[78274]: cluster 2023-10-01T10:59:01.890166+0000 osd.6 (osd.6) 195 : cluster 3 failed to encode map e250 with expected crc 2023-10-01T10:59:03.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:02 smithi078 bash[78274]: cluster 2023-10-01T10:59:01.893786+0000 osd.7 (osd.7) 237 : cluster 3 failed to encode map e250 with expected crc 2023-10-01T10:59:03.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:02 smithi078 bash[78274]: cluster 2023-10-01T10:59:01.895099+0000 osd.7 (osd.7) 238 : cluster 3 failed to encode map e250 with expected crc 2023-10-01T10:59:03.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:02 smithi078 bash[78274]: cluster 2023-10-01T10:59:01.895633+0000 osd.6 (osd.6) 196 : cluster 3 failed to encode map e250 with expected crc 2023-10-01T10:59:03.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:02 smithi078 bash[78274]: cluster 2023-10-01T10:59:01.896819+0000 osd.7 (osd.7) 239 : cluster 3 failed to encode map e250 with expected crc 2023-10-01T10:59:03.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:02 smithi078 bash[78274]: cluster 2023-10-01T10:59:01.898019+0000 osd.7 (osd.7) 240 : cluster 3 failed to encode map e250 with expected crc 2023-10-01T10:59:03.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:02 smithi078 bash[78274]: audit 2023-10-01T10:59:02.449120+0000 mon.a (mon.0) 1545 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:59:03.107 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:03.115 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:02 smithi173 bash[58816]: cluster 2023-10-01T10:59:01.868423+0000 mon.a (mon.0) 1542 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:59:03.116 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:02 smithi173 bash[58816]: audit 2023-10-01T10:59:01.874766+0000 mon.a (mon.0) 1543 : audit 1 from='client.? 172.21.15.78:0/2741778675' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T10:59:03.116 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:02 smithi173 bash[58816]: cluster 2023-10-01T10:59:01.876541+0000 osd.7 (osd.7) 236 : cluster 3 failed to encode map e250 with expected crc 2023-10-01T10:59:03.116 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:02 smithi173 bash[58816]: cluster 2023-10-01T10:59:01.878360+0000 mon.a (mon.0) 1544 : cluster 0 osdmap e250: 8 total, 8 up, 8 in 2023-10-01T10:59:03.116 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:02 smithi173 bash[58816]: cluster 2023-10-01T10:59:01.889379+0000 osd.5 (osd.5) 231 : cluster 3 failed to encode map e250 with expected crc 2023-10-01T10:59:03.116 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:02 smithi173 bash[58816]: cluster 2023-10-01T10:59:01.890166+0000 osd.6 (osd.6) 195 : cluster 3 failed to encode map e250 with expected crc 2023-10-01T10:59:03.116 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:02 smithi173 bash[58816]: cluster 2023-10-01T10:59:01.893786+0000 osd.7 (osd.7) 237 : cluster 3 failed to encode map e250 with expected crc 2023-10-01T10:59:03.116 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:02 smithi173 bash[58816]: cluster 2023-10-01T10:59:01.895099+0000 osd.7 (osd.7) 238 : cluster 3 failed to encode map e250 with expected crc 2023-10-01T10:59:03.116 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:02 smithi173 bash[58816]: cluster 2023-10-01T10:59:01.895633+0000 osd.6 (osd.6) 196 : cluster 3 failed to encode map e250 with expected crc 2023-10-01T10:59:03.116 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:02 smithi173 bash[58816]: cluster 2023-10-01T10:59:01.896819+0000 osd.7 (osd.7) 239 : cluster 3 failed to encode map e250 with expected crc 2023-10-01T10:59:03.116 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:02 smithi173 bash[58816]: cluster 2023-10-01T10:59:01.898019+0000 osd.7 (osd.7) 240 : cluster 3 failed to encode map e250 with expected crc 2023-10-01T10:59:03.116 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:02 smithi173 bash[58816]: audit 2023-10-01T10:59:02.449120+0000 mon.a (mon.0) 1545 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:59:03.131 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:03.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:03.138 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:59:03.238 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:03.268 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:03.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:03.271 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:59:03.371 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:03.403 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:03.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:03.407 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:03.508 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:03.527 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:03.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:03.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:59:03.633 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:03.657 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:03.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:03.659 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:59:03.760 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:03.784 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:03.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:03.786 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:59:03.887 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:03.913 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:03.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:03.916 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:59:04.017 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:04.024 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:03 smithi173 bash[58816]: cluster 2023-10-01T10:59:02.129054+0000 mgr.y (mgr.25072) 1249 : cluster 0 pgmap v1264: 89 pgs: 89 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 713 op/s 2023-10-01T10:59:04.048 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:04.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:04.050 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:04.151 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:04.188 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:04.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:04.191 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:04.292 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:04.301 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:03 smithi078 bash[78274]: cluster 2023-10-01T10:59:02.129054+0000 mgr.y (mgr.25072) 1249 : cluster 0 pgmap v1264: 89 pgs: 89 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 713 op/s 2023-10-01T10:59:04.301 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:03 smithi078 bash[75299]: cluster 2023-10-01T10:59:02.129054+0000 mgr.y (mgr.25072) 1249 : cluster 0 pgmap v1264: 89 pgs: 89 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 713 op/s 2023-10-01T10:59:04.318 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:04.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:04.319 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:59:04.420 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:04.444 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:04.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:04.449 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:59:04.550 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:04.579 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:04.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:04.583 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:59:04.684 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:04.709 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:04.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:04.711 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:59:04.812 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:04.846 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:04.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:04.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:59:04.950 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:04.976 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:04.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:04.979 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:59:05.079 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:05.118 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:05.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:05.127 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:59:05.227 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:05.259 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:05.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:05.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:59:05.364 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:05.389 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:05.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:05.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:59:05.495 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:05.515 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:05.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:05.517 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:59:05.618 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:05.641 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:05.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:05.647 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:59:05.748 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:05.772 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:05.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:05.774 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:59:05.875 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:05.902 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:05.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:05.907 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:59:05.909 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:05 smithi173 bash[58816]: cluster 2023-10-01T10:59:04.129838+0000 mgr.y (mgr.25072) 1250 : cluster 0 pgmap v1265: 89 pgs: 89 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 654 op/s 2023-10-01T10:59:06.008 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:06.033 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:06.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:06.036 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:59:06.137 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:06.146 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:05 smithi078 bash[78274]: cluster 2023-10-01T10:59:04.129838+0000 mgr.y (mgr.25072) 1250 : cluster 0 pgmap v1265: 89 pgs: 89 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 654 op/s 2023-10-01T10:59:06.147 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:05 smithi078 bash[75299]: cluster 2023-10-01T10:59:04.129838+0000 mgr.y (mgr.25072) 1250 : cluster 0 pgmap v1265: 89 pgs: 89 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 654 op/s 2023-10-01T10:59:06.170 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:06.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:06.176 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:59:06.277 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:06.303 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:06.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:06.309 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:59:06.410 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:06.431 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T10:59:06.441 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:06.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:06.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:59:06.452 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T10:59:06.544 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:06.589 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:06.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:06.593 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:59:06.694 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:06.726 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:06.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:06.728 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:59:06.829 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:06.863 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:06.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:06.865 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:59:06.966 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:06.993 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:06.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:06.999 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:59:07.100 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:07.130 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:07.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:07.135 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:59:07.236 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:07.275 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:07.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:07.280 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:59:07.358 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T10:59:07.381 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:07.408 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:07.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:07.416 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:59:07.516 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:07.548 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:07.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:07.554 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:07.645 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:59:07.654 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:07.706 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:07.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:07.712 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:59:07.813 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:07.841 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:07.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:07.844 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:59:07.945 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:07.956 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:07 smithi078 bash[78274]: cluster 2023-10-01T10:59:06.130426+0000 mgr.y (mgr.25072) 1251 : cluster 0 pgmap v1266: 89 pgs: 89 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 651 op/s 2023-10-01T10:59:07.956 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:07 smithi078 bash[75299]: cluster 2023-10-01T10:59:06.130426+0000 mgr.y (mgr.25072) 1251 : cluster 0 pgmap v1266: 89 pgs: 89 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 651 op/s 2023-10-01T10:59:07.973 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:07.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:07.982 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:59:08.083 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:08.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:07 smithi173 bash[58816]: cluster 2023-10-01T10:59:06.130426+0000 mgr.y (mgr.25072) 1251 : cluster 0 pgmap v1266: 89 pgs: 89 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 651 op/s 2023-10-01T10:59:08.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:08.124 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:08.124 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:59:08.225 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:08.249 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:08.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:08.253 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:59:08.354 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:08.378 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:08.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:08.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:59:08.481 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:08.508 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:08.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:08.510 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:59:08.611 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:08.640 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:08.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:08.645 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:59:08.745 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:08.780 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:08.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:08.784 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:59:08.885 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:08.918 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:08.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:08.921 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:59:09.022 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:09.046 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:09.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:09.050 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:59:09.151 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:09.192 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:09.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:09.196 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:59:09.297 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:09.331 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:09.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:09.334 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:59:09.435 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:09.457 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:09.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:09.459 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:59:09.560 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:09.602 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:09.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:09.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:59:09.704 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:09.750 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:09.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:09.756 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:59:09.856 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:09.891 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:09.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:09.892 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:59:09.993 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:10.009 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:09 smithi078 bash[78274]: cluster 2023-10-01T10:59:08.130950+0000 mgr.y (mgr.25072) 1252 : cluster 0 pgmap v1267: 89 pgs: 89 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 652 op/s 2023-10-01T10:59:10.009 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:09 smithi078 bash[75299]: cluster 2023-10-01T10:59:08.130950+0000 mgr.y (mgr.25072) 1252 : cluster 0 pgmap v1267: 89 pgs: 89 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 652 op/s 2023-10-01T10:59:10.018 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:10.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:10.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:59:10.122 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:10.159 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:10.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:10.163 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:59:10.264 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:10.274 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:09 smithi173 bash[58816]: cluster 2023-10-01T10:59:08.130950+0000 mgr.y (mgr.25072) 1252 : cluster 0 pgmap v1267: 89 pgs: 89 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 652 op/s 2023-10-01T10:59:10.292 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:10.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:10.294 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:59:10.395 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:10.407 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:59:10 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:59:10] "GET /metrics HTTP/1.1" 200 36387 "" "Prometheus/2.33.4" 2023-10-01T10:59:10.430 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:10.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:10.432 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:59:10.533 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:10.559 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:10.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:10.564 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:59:10.664 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:10.695 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:10.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:10.698 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:10.798 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:10.824 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:10.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:10.826 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:59:10.927 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:10.957 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:10.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:10.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:11.063 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:11.091 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:11.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:11.093 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:59:11.194 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:11.217 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:11.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:11.225 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:11.325 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:11.348 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:11.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:11.351 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:59:11.451 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:11.490 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:11.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:11.492 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:59:11.593 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:11.625 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:11.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:11.627 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:59:11.728 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:11.734 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-10-01T10:59:11.734 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T10:59:11.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:11.760 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:11.760 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:59:11.861 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:11.883 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:11.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:11.887 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:59:11.976 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:59:11.987 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:59:11.988 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:12.010 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:12.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:12.014 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:59:12.024 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:11 smithi078 bash[75299]: cluster 2023-10-01T10:59:10.131611+0000 mgr.y (mgr.25072) 1253 : cluster 0 pgmap v1268: 89 pgs: 89 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 636 op/s 2023-10-01T10:59:12.025 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:11 smithi078 bash[78274]: cluster 2023-10-01T10:59:10.131611+0000 mgr.y (mgr.25072) 1253 : cluster 0 pgmap v1268: 89 pgs: 89 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 636 op/s 2023-10-01T10:59:12.115 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:12.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:11 smithi173 bash[58816]: cluster 2023-10-01T10:59:10.131611+0000 mgr.y (mgr.25072) 1253 : cluster 0 pgmap v1268: 89 pgs: 89 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 636 op/s 2023-10-01T10:59:12.138 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:12.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:12.141 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:12.241 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:12.274 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:12.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:12.280 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:59:12.380 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:12.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:12.414 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:12.415 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:59:12.516 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:12.547 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:12.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:12.550 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:59:12.651 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:12.671 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-01T10:59:12.683 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:12.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:12.689 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:12.789 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:12.819 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:12.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:12.821 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:59:12.922 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:12.971 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:12.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:12.975 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:59:13.076 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:13.108 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:13.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:13.110 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:13.211 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:13.246 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:13.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:13.251 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:13.351 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:13.381 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:13.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:13.387 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:59:13.488 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:13.521 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:13.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:13.529 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:59:13.629 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:13.653 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:13.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:13.656 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:59:13.757 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:13.775 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:13.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:13.777 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:59:13.878 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:13.906 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:13.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:13.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:59:14.008 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:14 smithi078 bash[75299]: cluster 2023-10-01T10:59:12.132061+0000 mgr.y (mgr.25072) 1254 : cluster 0 pgmap v1269: 89 pgs: 89 active+clean; 4.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 620 op/s 2023-10-01T10:59:14.008 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:14 smithi078 bash[78274]: cluster 2023-10-01T10:59:12.132061+0000 mgr.y (mgr.25072) 1254 : cluster 0 pgmap v1269: 89 pgs: 89 active+clean; 4.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 620 op/s 2023-10-01T10:59:14.012 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:14.033 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:14.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:14.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:59:14.136 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:14.185 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:14.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:14.189 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:59:14.290 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:14.298 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:13 smithi173 bash[58816]: cluster 2023-10-01T10:59:12.132061+0000 mgr.y (mgr.25072) 1254 : cluster 0 pgmap v1269: 89 pgs: 89 active+clean; 4.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 620 op/s 2023-10-01T10:59:14.317 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:14.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:14.323 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:59:14.423 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:14.454 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:14.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:14.456 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:59:14.557 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:14.582 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:14.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:14.589 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:59:14.690 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:14.721 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:14.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:14.730 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:59:14.831 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:14.862 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:14.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:14.870 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:59:14.971 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:15.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:15.006 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:15.006 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:59:15.107 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:15.136 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:15.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:15.138 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:59:15.239 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:15.283 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:15.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:15.290 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:59:15.391 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:15.413 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:15.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:15.420 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:59:15.521 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:15.549 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:15.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:15.554 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:59:15.655 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:15.684 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:15.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:15.692 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:59:15.792 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:15.821 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:15.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:15.825 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:59:15.926 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:15.950 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:15.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:15.959 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:59:16.010 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:15 smithi078 bash[78274]: cluster 2023-10-01T10:59:14.132651+0000 mgr.y (mgr.25072) 1255 : cluster 0 pgmap v1270: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 788 op/s 2023-10-01T10:59:16.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:15 smithi078 bash[78274]: audit 2023-10-01T10:59:15.162892+0000 mon.a (mon.0) 1546 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:59:16.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:15 smithi078 bash[78274]: audit 2023-10-01T10:59:15.637892+0000 mon.a (mon.0) 1547 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:59:16.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:15 smithi078 bash[78274]: audit 2023-10-01T10:59:15.656483+0000 mon.a (mon.0) 1548 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:59:16.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:15 smithi078 bash[75299]: cluster 2023-10-01T10:59:14.132651+0000 mgr.y (mgr.25072) 1255 : cluster 0 pgmap v1270: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 788 op/s 2023-10-01T10:59:16.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:15 smithi078 bash[75299]: audit 2023-10-01T10:59:15.162892+0000 mon.a (mon.0) 1546 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:59:16.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:15 smithi078 bash[75299]: audit 2023-10-01T10:59:15.637892+0000 mon.a (mon.0) 1547 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:59:16.011 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:15 smithi078 bash[75299]: audit 2023-10-01T10:59:15.656483+0000 mon.a (mon.0) 1548 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:59:16.060 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:16.087 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:16.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:16.091 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:59:16.191 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:16.230 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:16.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:16.235 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:59:16.335 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:16.346 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:15 smithi173 bash[58816]: cluster 2023-10-01T10:59:14.132651+0000 mgr.y (mgr.25072) 1255 : cluster 0 pgmap v1270: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 788 op/s 2023-10-01T10:59:16.347 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:15 smithi173 bash[58816]: audit 2023-10-01T10:59:15.162892+0000 mon.a (mon.0) 1546 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T10:59:16.347 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:15 smithi173 bash[58816]: audit 2023-10-01T10:59:15.637892+0000 mon.a (mon.0) 1547 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:59:16.347 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:15 smithi173 bash[58816]: audit 2023-10-01T10:59:15.656483+0000 mon.a (mon.0) 1548 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:59:16.363 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:16.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:16.365 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:59:16.466 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:16.495 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:16.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:16.497 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:59:16.598 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:16.629 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:16.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:16.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:59:16.735 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:16.765 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:16.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:16.771 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:59:16.872 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:16.906 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:16.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:16.910 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:16.948 INFO:teuthology.orchestra.run.smithi078.stderr:set pool 12 pgp_num to 16 2023-10-01T10:59:17.011 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:17.021 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:16 smithi173 bash[58816]: audit 2023-10-01T10:59:16.414751+0000 mon.a (mon.0) 1549 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-01T10:59:17.021 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:16 smithi173 bash[58816]: audit 2023-10-01T10:59:16.414965+0000 mon.b (mon.1) 52 : audit 1 from='client.? 172.21.15.78:0/1526113397' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-01T10:59:17.041 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:17.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:17.047 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:59:17.147 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:17.175 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:17.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:17.180 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:59:17.249 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:16 smithi078 bash[75299]: audit 2023-10-01T10:59:16.414751+0000 mon.a (mon.0) 1549 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-01T10:59:17.250 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:16 smithi078 bash[75299]: audit 2023-10-01T10:59:16.414965+0000 mon.b (mon.1) 52 : audit 1 from='client.? 172.21.15.78:0/1526113397' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-01T10:59:17.250 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:16 smithi078 bash[78274]: audit 2023-10-01T10:59:16.414751+0000 mon.a (mon.0) 1549 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-01T10:59:17.250 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:16 smithi078 bash[78274]: audit 2023-10-01T10:59:16.414965+0000 mon.b (mon.1) 52 : audit 1 from='client.? 172.21.15.78:0/1526113397' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num", "val": "16"}]: dispatch 2023-10-01T10:59:17.280 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:17.315 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:17.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:17.319 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:59:17.419 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:17.448 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:17.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:17.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:59:17.487 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T10:59:17.552 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:17.575 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:17.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:17.586 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:59:17.687 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:17.715 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:17.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:17.717 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:59:17.818 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:17.844 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:17.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:17.845 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:59:17.946 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:17.983 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:17.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:17.987 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:59:18.088 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:18.096 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:17 smithi173 bash[58816]: cluster 2023-10-01T10:59:16.133176+0000 mgr.y (mgr.25072) 1256 : cluster 0 pgmap v1271: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 608 op/s 2023-10-01T10:59:18.096 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:17 smithi173 bash[58816]: audit 2023-10-01T10:59:16.936881+0000 mon.a (mon.0) 1550 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num", "val": "16"}]': finished 2023-10-01T10:59:18.096 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:17 smithi173 bash[58816]: cluster 2023-10-01T10:59:16.936983+0000 mon.a (mon.0) 1551 : cluster 0 osdmap e251: 8 total, 8 up, 8 in 2023-10-01T10:59:18.096 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:17 smithi173 bash[58816]: cluster 2023-10-01T10:59:16.937491+0000 osd.6 (osd.6) 197 : cluster 3 failed to encode map e251 with expected crc 2023-10-01T10:59:18.096 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:17 smithi173 bash[58816]: cluster 2023-10-01T10:59:16.941295+0000 osd.7 (osd.7) 241 : cluster 3 failed to encode map e251 with expected crc 2023-10-01T10:59:18.097 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:17 smithi173 bash[58816]: cluster 2023-10-01T10:59:16.942026+0000 osd.6 (osd.6) 198 : cluster 3 failed to encode map e251 with expected crc 2023-10-01T10:59:18.097 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:17 smithi173 bash[58816]: audit 2023-10-01T10:59:17.449349+0000 mon.a (mon.0) 1552 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:59:18.111 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:18.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:18.114 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:59:18.214 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:18.242 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:18.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:18.244 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:18.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:17 smithi078 bash[75299]: cluster 2023-10-01T10:59:16.133176+0000 mgr.y (mgr.25072) 1256 : cluster 0 pgmap v1271: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 608 op/s 2023-10-01T10:59:18.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:17 smithi078 bash[75299]: audit 2023-10-01T10:59:16.936881+0000 mon.a (mon.0) 1550 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num", "val": "16"}]': finished 2023-10-01T10:59:18.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:17 smithi078 bash[75299]: cluster 2023-10-01T10:59:16.936983+0000 mon.a (mon.0) 1551 : cluster 0 osdmap e251: 8 total, 8 up, 8 in 2023-10-01T10:59:18.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:17 smithi078 bash[75299]: cluster 2023-10-01T10:59:16.937491+0000 osd.6 (osd.6) 197 : cluster 3 failed to encode map e251 with expected crc 2023-10-01T10:59:18.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:17 smithi078 bash[75299]: cluster 2023-10-01T10:59:16.941295+0000 osd.7 (osd.7) 241 : cluster 3 failed to encode map e251 with expected crc 2023-10-01T10:59:18.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:17 smithi078 bash[75299]: cluster 2023-10-01T10:59:16.942026+0000 osd.6 (osd.6) 198 : cluster 3 failed to encode map e251 with expected crc 2023-10-01T10:59:18.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:17 smithi078 bash[75299]: audit 2023-10-01T10:59:17.449349+0000 mon.a (mon.0) 1552 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:59:18.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:17 smithi078 bash[78274]: cluster 2023-10-01T10:59:16.133176+0000 mgr.y (mgr.25072) 1256 : cluster 0 pgmap v1271: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 608 op/s 2023-10-01T10:59:18.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:17 smithi078 bash[78274]: audit 2023-10-01T10:59:16.936881+0000 mon.a (mon.0) 1550 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num", "val": "16"}]': finished 2023-10-01T10:59:18.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:17 smithi078 bash[78274]: cluster 2023-10-01T10:59:16.936983+0000 mon.a (mon.0) 1551 : cluster 0 osdmap e251: 8 total, 8 up, 8 in 2023-10-01T10:59:18.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:17 smithi078 bash[78274]: cluster 2023-10-01T10:59:16.937491+0000 osd.6 (osd.6) 197 : cluster 3 failed to encode map e251 with expected crc 2023-10-01T10:59:18.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:17 smithi078 bash[78274]: cluster 2023-10-01T10:59:16.941295+0000 osd.7 (osd.7) 241 : cluster 3 failed to encode map e251 with expected crc 2023-10-01T10:59:18.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:17 smithi078 bash[78274]: cluster 2023-10-01T10:59:16.942026+0000 osd.6 (osd.6) 198 : cluster 3 failed to encode map e251 with expected crc 2023-10-01T10:59:18.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:17 smithi078 bash[78274]: audit 2023-10-01T10:59:17.449349+0000 mon.a (mon.0) 1552 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:59:18.345 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:18.384 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:18.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:18.388 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:59:18.488 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:18.525 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:18.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:18.534 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:59:18.634 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:18.661 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:18.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:18.667 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:18.767 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:18.802 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:18.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:18.807 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:59:18.907 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:18.951 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:18.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:18.953 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:59:19.054 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:19.099 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:19.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:19.105 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:59:19.206 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:19.232 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:19.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:19.234 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:59:19.335 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:19.360 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:19.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:19.365 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:59:19.466 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:19.491 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:19.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:19.494 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:59:19.595 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:19.624 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:19.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:19.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:59:19.732 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:19.757 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:19.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:19.759 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:59:19.861 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:19.886 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:19.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:19.893 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:59:19.994 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:20.017 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:20.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:20.022 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:20.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:19 smithi078 bash[75299]: cluster 2023-10-01T10:59:18.133646+0000 mgr.y (mgr.25072) 1257 : cluster 0 pgmap v1273: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 683 op/s 2023-10-01T10:59:20.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:19 smithi078 bash[78274]: cluster 2023-10-01T10:59:18.133646+0000 mgr.y (mgr.25072) 1257 : cluster 0 pgmap v1273: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 683 op/s 2023-10-01T10:59:20.123 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:20.132 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:19 smithi173 bash[58816]: cluster 2023-10-01T10:59:18.133646+0000 mgr.y (mgr.25072) 1257 : cluster 0 pgmap v1273: 89 pgs: 89 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 683 op/s 2023-10-01T10:59:20.146 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:20.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:20.151 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:59:20.251 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:20.280 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:20.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:20.282 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:20.383 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:20.409 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:20.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:20.415 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:59:20.516 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:20.537 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:20.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:20.541 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:59:20.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:59:20 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:59:20] "GET /metrics HTTP/1.1" 200 36388 "" "Prometheus/2.33.4" 2023-10-01T10:59:20.641 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:20.679 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:20.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:20.680 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:59:20.781 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:20.810 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:20.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:20.813 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:59:20.914 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:20.949 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:20.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:20.953 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:59:21.054 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:21.078 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:21.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:21.081 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:21.182 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:21.206 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:21.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:21.214 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:59:21.315 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:21.343 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:21.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:21.346 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:59:21.447 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:21.495 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:21.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:21.499 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:59:21.600 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:21.631 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:21.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:21.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:59:21.736 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:21.796 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:21.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:21.798 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:21.899 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:21.933 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:21.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:21.934 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:59:22.035 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:22.040 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-10-01T10:59:22.040 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T10:59:22.049 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:21 smithi173 bash[58816]: cluster 2023-10-01T10:59:20.134346+0000 mgr.y (mgr.25072) 1258 : cluster 0 pgmap v1274: 89 pgs: 89 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 675 op/s 2023-10-01T10:59:22.079 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:22.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:22.091 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:59:22.192 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:22.227 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:22.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:22.234 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:22.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:21 smithi078 bash[75299]: cluster 2023-10-01T10:59:20.134346+0000 mgr.y (mgr.25072) 1258 : cluster 0 pgmap v1274: 89 pgs: 89 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 675 op/s 2023-10-01T10:59:22.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:21 smithi078 bash[78274]: cluster 2023-10-01T10:59:20.134346+0000 mgr.y (mgr.25072) 1258 : cluster 0 pgmap v1274: 89 pgs: 89 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 675 op/s 2023-10-01T10:59:22.335 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:22.365 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:22.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:22.368 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:59:22.469 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:22.496 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:22.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:22.499 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:59:22.599 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:22.631 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:22.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:22.636 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:59:22.736 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:22.787 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:22.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:22.792 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:59:22.892 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:22.917 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:22.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:22.919 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:59:22.925 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T10:59:22.972 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T10:59:23.003 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-01T10:59:23.020 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:23.021 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T10:59:23.073 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T10:59:23.075 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:23.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:23.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:59:23.132 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T10:59:23.173 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T10:59:23.180 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:23.217 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T10:59:23.219 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:23.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:23.221 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:59:23.265 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T10:59:23.268 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T10:59:23.268 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T10:59:23.268 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T10:59:23.269 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T10:59:23.269 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T10:59:23.269 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T10:59:23.269 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T10:59:23.269 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T10:59:23.323 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:23.354 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:23.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:23.359 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:59:23.461 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:23.489 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:23.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:23.492 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:59:23.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:23 smithi078 bash[75299]: cluster 2023-10-01T10:59:22.134889+0000 mgr.y (mgr.25072) 1259 : cluster 0 pgmap v1275: 89 pgs: 89 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 675 op/s 2023-10-01T10:59:23.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:23 smithi078 bash[75299]: audit 2023-10-01T10:59:22.362672+0000 mon.a (mon.0) 1553 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:59:23.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:23 smithi078 bash[75299]: audit 2023-10-01T10:59:22.372917+0000 mon.a (mon.0) 1554 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:59:23.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:23 smithi078 bash[75299]: audit 2023-10-01T10:59:22.886755+0000 mon.a (mon.0) 1555 : audit 0 from='client.? 172.21.15.78:0/595778366' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:59:23.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:23 smithi078 bash[75299]: audit 2023-10-01T10:59:23.269756+0000 mon.c (mon.2) 196 : audit 1 from='client.? 172.21.15.78:0/397102795' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:59:23.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:23 smithi078 bash[75299]: audit 2023-10-01T10:59:23.270452+0000 mon.a (mon.0) 1556 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:59:23.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:23 smithi078 bash[78274]: cluster 2023-10-01T10:59:22.134889+0000 mgr.y (mgr.25072) 1259 : cluster 0 pgmap v1275: 89 pgs: 89 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 675 op/s 2023-10-01T10:59:23.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:23 smithi078 bash[78274]: audit 2023-10-01T10:59:22.362672+0000 mon.a (mon.0) 1553 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:59:23.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:23 smithi078 bash[78274]: audit 2023-10-01T10:59:22.372917+0000 mon.a (mon.0) 1554 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:59:23.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:23 smithi078 bash[78274]: audit 2023-10-01T10:59:22.886755+0000 mon.a (mon.0) 1555 : audit 0 from='client.? 172.21.15.78:0/595778366' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:59:23.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:23 smithi078 bash[78274]: audit 2023-10-01T10:59:23.269756+0000 mon.c (mon.2) 196 : audit 1 from='client.? 172.21.15.78:0/397102795' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:59:23.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:23 smithi078 bash[78274]: audit 2023-10-01T10:59:23.270452+0000 mon.a (mon.0) 1556 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:59:23.593 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:23.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:23 smithi173 bash[58816]: cluster 2023-10-01T10:59:22.134889+0000 mgr.y (mgr.25072) 1259 : cluster 0 pgmap v1275: 89 pgs: 89 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 675 op/s 2023-10-01T10:59:23.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:23 smithi173 bash[58816]: audit 2023-10-01T10:59:22.362672+0000 mon.a (mon.0) 1553 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:59:23.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:23 smithi173 bash[58816]: audit 2023-10-01T10:59:22.372917+0000 mon.a (mon.0) 1554 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:59:23.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:23 smithi173 bash[58816]: audit 2023-10-01T10:59:22.886755+0000 mon.a (mon.0) 1555 : audit 0 from='client.? 172.21.15.78:0/595778366' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:59:23.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:23 smithi173 bash[58816]: audit 2023-10-01T10:59:23.269756+0000 mon.c (mon.2) 196 : audit 1 from='client.? 172.21.15.78:0/397102795' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:59:23.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:23 smithi173 bash[58816]: audit 2023-10-01T10:59:23.270452+0000 mon.a (mon.0) 1556 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T10:59:23.630 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:23.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:23.633 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:59:23.734 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:23.765 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:23.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:23.771 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:59:23.872 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:23.897 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:23.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:23.900 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:59:23.966 INFO:teuthology.orchestra.run.smithi078.stderr:nodeep-scrub is unset 2023-10-01T10:59:24.001 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:24.030 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:24.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:24.034 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:59:24.134 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:24.156 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:24.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:24.159 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:59:24.259 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:24.290 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:24.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:24.297 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:59:24.397 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:24.405 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:24 smithi078 bash[75299]: cluster 2023-10-01T10:59:23.946888+0000 mon.a (mon.0) 1557 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T10:59:24.405 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:24 smithi078 bash[75299]: cluster 2023-10-01T10:59:23.946922+0000 mon.a (mon.0) 1558 : cluster 1 Cluster is now healthy 2023-10-01T10:59:24.405 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:24 smithi078 bash[75299]: cluster 2023-10-01T10:59:23.951493+0000 osd.7 (osd.7) 242 : cluster 3 failed to encode map e252 with expected crc 2023-10-01T10:59:24.405 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:24 smithi078 bash[75299]: audit 2023-10-01T10:59:23.952950+0000 mon.a (mon.0) 1559 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T10:59:24.405 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:24 smithi078 bash[75299]: cluster 2023-10-01T10:59:23.953824+0000 mon.a (mon.0) 1560 : cluster 0 osdmap e252: 8 total, 8 up, 8 in 2023-10-01T10:59:24.406 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:24 smithi078 bash[78274]: cluster 2023-10-01T10:59:23.946888+0000 mon.a (mon.0) 1557 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T10:59:24.406 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:24 smithi078 bash[78274]: cluster 2023-10-01T10:59:23.946922+0000 mon.a (mon.0) 1558 : cluster 1 Cluster is now healthy 2023-10-01T10:59:24.406 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:24 smithi078 bash[78274]: cluster 2023-10-01T10:59:23.951493+0000 osd.7 (osd.7) 242 : cluster 3 failed to encode map e252 with expected crc 2023-10-01T10:59:24.406 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:24 smithi078 bash[78274]: audit 2023-10-01T10:59:23.952950+0000 mon.a (mon.0) 1559 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T10:59:24.406 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:24 smithi078 bash[78274]: cluster 2023-10-01T10:59:23.953824+0000 mon.a (mon.0) 1560 : cluster 0 osdmap e252: 8 total, 8 up, 8 in 2023-10-01T10:59:24.418 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:24.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:24.420 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:59:24.521 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:24.553 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:24.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:24.556 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:59:24.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:24 smithi173 bash[58816]: cluster 2023-10-01T10:59:23.946888+0000 mon.a (mon.0) 1557 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T10:59:24.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:24 smithi173 bash[58816]: cluster 2023-10-01T10:59:23.946922+0000 mon.a (mon.0) 1558 : cluster 1 Cluster is now healthy 2023-10-01T10:59:24.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:24 smithi173 bash[58816]: cluster 2023-10-01T10:59:23.951493+0000 osd.7 (osd.7) 242 : cluster 3 failed to encode map e252 with expected crc 2023-10-01T10:59:24.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:24 smithi173 bash[58816]: audit 2023-10-01T10:59:23.952950+0000 mon.a (mon.0) 1559 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T10:59:24.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:24 smithi173 bash[58816]: cluster 2023-10-01T10:59:23.953824+0000 mon.a (mon.0) 1560 : cluster 0 osdmap e252: 8 total, 8 up, 8 in 2023-10-01T10:59:24.657 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:24.679 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:24.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:24.684 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:59:24.785 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:24.818 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:24.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:24.821 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:59:24.922 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:24.964 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:24.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:24.971 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:25.072 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:25.094 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:25.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:25.097 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:59:25.197 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:25.228 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:25.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:25.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:59:25.336 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:25.369 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:25.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:25.373 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:59:25.474 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:25.499 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:25.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:25.502 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:59:25.605 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:25.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:25 smithi173 bash[58816]: cluster 2023-10-01T10:59:23.962139+0000 osd.6 (osd.6) 199 : cluster 3 failed to encode map e252 with expected crc 2023-10-01T10:59:25.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:25 smithi173 bash[58816]: cluster 2023-10-01T10:59:24.135509+0000 mgr.y (mgr.25072) 1260 : cluster 0 pgmap v1277: 89 pgs: 89 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 760 op/s 2023-10-01T10:59:25.659 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:25.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:25.663 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:25 smithi078 bash[78274]: cluster 2023-10-01T10:59:23.962139+0000 osd.6 (osd.6) 199 : cluster 3 failed to encode map e252 with expected crc 2023-10-01T10:59:25.663 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:25 smithi078 bash[78274]: cluster 2023-10-01T10:59:24.135509+0000 mgr.y (mgr.25072) 1260 : cluster 0 pgmap v1277: 89 pgs: 89 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 760 op/s 2023-10-01T10:59:25.664 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:59:25.664 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:25 smithi078 bash[75299]: cluster 2023-10-01T10:59:23.962139+0000 osd.6 (osd.6) 199 : cluster 3 failed to encode map e252 with expected crc 2023-10-01T10:59:25.664 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:25 smithi078 bash[75299]: cluster 2023-10-01T10:59:24.135509+0000 mgr.y (mgr.25072) 1260 : cluster 0 pgmap v1277: 89 pgs: 89 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 760 op/s 2023-10-01T10:59:25.765 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:25.831 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:25.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:25.841 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:59:25.875 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 7, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T10:59:25.875 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T10:59:25.876 INFO:tasks.thrashosds.thrasher:fixing pg num pool device_health_metrics 2023-10-01T10:59:25.876 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T10:59:25.942 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:25.970 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:25.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:25.972 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:26.072 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:26.096 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:26.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:26.106 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:26.207 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:26.235 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:26.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:26.238 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:59:26.339 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:26.360 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:26.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:26.362 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:59:26.463 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:26.504 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:26.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:26.512 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:59:26.612 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:26.639 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:26.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:26.640 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:59:26.736 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T10:59:26.741 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:26.767 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:26.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:26.769 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:59:26.870 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:26.921 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:26.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:26.924 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:59:27.025 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:27.072 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:27.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:27.078 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:59:27.179 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:27.203 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:27.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:27.206 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:27.307 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:27.336 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:27.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:27.340 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:59:27.441 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:27.466 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:27.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:27.472 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:27.573 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:27.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:27 smithi078 bash[75299]: cluster 2023-10-01T10:59:26.136019+0000 mgr.y (mgr.25072) 1261 : cluster 0 pgmap v1278: 89 pgs: 89 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 705 op/s 2023-10-01T10:59:27.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:27 smithi078 bash[75299]: audit 2023-10-01T10:59:26.560477+0000 mon.a (mon.0) 1561 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:59:27.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:27 smithi078 bash[75299]: audit 2023-10-01T10:59:26.574071+0000 mon.a (mon.0) 1562 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:59:27.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:27 smithi078 bash[75299]: audit 2023-10-01T10:59:27.253566+0000 mon.a (mon.0) 1563 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:59:27.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:27 smithi078 bash[75299]: audit 2023-10-01T10:59:27.254555+0000 mon.a (mon.0) 1564 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:59:27.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:27 smithi078 bash[75299]: audit 2023-10-01T10:59:27.263954+0000 mon.a (mon.0) 1565 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:59:27.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:27 smithi078 bash[78274]: cluster 2023-10-01T10:59:26.136019+0000 mgr.y (mgr.25072) 1261 : cluster 0 pgmap v1278: 89 pgs: 89 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 705 op/s 2023-10-01T10:59:27.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:27 smithi078 bash[78274]: audit 2023-10-01T10:59:26.560477+0000 mon.a (mon.0) 1561 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:59:27.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:27 smithi078 bash[78274]: audit 2023-10-01T10:59:26.574071+0000 mon.a (mon.0) 1562 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:59:27.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:27 smithi078 bash[78274]: audit 2023-10-01T10:59:27.253566+0000 mon.a (mon.0) 1563 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:59:27.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:27 smithi078 bash[78274]: audit 2023-10-01T10:59:27.254555+0000 mon.a (mon.0) 1564 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:59:27.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:27 smithi078 bash[78274]: audit 2023-10-01T10:59:27.263954+0000 mon.a (mon.0) 1565 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:59:27.603 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:27.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:27.607 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:27.615 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T10:59:27.708 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:27.746 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:27.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:27.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:59:27.789 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T10:59:27.819 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:27 smithi173 bash[58816]: cluster 2023-10-01T10:59:26.136019+0000 mgr.y (mgr.25072) 1261 : cluster 0 pgmap v1278: 89 pgs: 89 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 705 op/s 2023-10-01T10:59:27.820 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:27 smithi173 bash[58816]: audit 2023-10-01T10:59:26.560477+0000 mon.a (mon.0) 1561 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:59:27.820 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:27 smithi173 bash[58816]: audit 2023-10-01T10:59:26.574071+0000 mon.a (mon.0) 1562 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:59:27.820 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:27 smithi173 bash[58816]: audit 2023-10-01T10:59:27.253566+0000 mon.a (mon.0) 1563 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T10:59:27.820 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:27 smithi173 bash[58816]: audit 2023-10-01T10:59:27.254555+0000 mon.a (mon.0) 1564 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T10:59:27.820 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:27 smithi173 bash[58816]: audit 2023-10-01T10:59:27.263954+0000 mon.a (mon.0) 1565 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:59:27.850 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:27.886 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:27.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:27.890 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:59:27.991 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:28.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:28.033 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:28.033 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:59:28.133 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:28.161 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:28.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:28.166 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:59:28.269 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:28.297 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:28.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:28.303 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:28.403 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:28.432 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:28.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:28.435 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:59:28.535 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:28.564 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:28.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:28.567 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:59:28.668 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:28.695 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:28.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:28.699 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:59:28.799 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:28.844 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:28.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:28.847 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:59:28.948 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:28.988 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:28.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:28.990 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:59:29.091 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:29.129 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:29.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:29.135 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:59:29.236 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:29.263 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:29.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:29.270 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:59:29.370 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:29.397 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:29.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:29.400 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:59:29.501 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:29.529 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:29.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:29.533 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:59:29.633 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:29.670 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:29.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:29.680 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:59:29.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:29 smithi173 bash[58816]: cluster 2023-10-01T10:59:28.136535+0000 mgr.y (mgr.25072) 1262 : cluster 0 pgmap v1279: 89 pgs: 89 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 641 op/s 2023-10-01T10:59:29.780 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:29.818 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:29.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:29.826 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:59:29.927 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:29.949 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:29.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:29.953 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:59:30.054 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:30.076 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:30.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:30.078 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:30.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:29 smithi078 bash[78274]: cluster 2023-10-01T10:59:28.136535+0000 mgr.y (mgr.25072) 1262 : cluster 0 pgmap v1279: 89 pgs: 89 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 641 op/s 2023-10-01T10:59:30.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:29 smithi078 bash[75299]: cluster 2023-10-01T10:59:28.136535+0000 mgr.y (mgr.25072) 1262 : cluster 0 pgmap v1279: 89 pgs: 89 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 641 op/s 2023-10-01T10:59:30.179 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:30.218 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:30.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:30.223 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:59:30.323 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:30.354 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:30.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:30.358 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:59:30.459 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:30.502 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:30.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:30.510 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:59:30.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:59:30 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:59:30] "GET /metrics HTTP/1.1" 200 36388 "" "Prometheus/2.33.4" 2023-10-01T10:59:30.611 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:30.638 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:30.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:30.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:59:30.746 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:30.763 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:59:30.766 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:30.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:30.770 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:30.770 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:59:30.871 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:30.898 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:30.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:30.904 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:59:31.005 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:31.032 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:31.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:31.034 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:59:31.135 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:31.177 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:31.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:31.180 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:59:31.280 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:31.315 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:31.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:31.319 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:59:31.420 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:31.461 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:31.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:31.464 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:59:31.564 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:31.597 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:31.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:31.605 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:59:31.706 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:31.734 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:31.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:31.738 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:59:31.779 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:59:31.799 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:31 smithi078 bash[75299]: cluster 2023-10-01T10:59:30.137237+0000 mgr.y (mgr.25072) 1263 : cluster 0 pgmap v1280: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 636 op/s 2023-10-01T10:59:31.799 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:31 smithi078 bash[78274]: cluster 2023-10-01T10:59:30.137237+0000 mgr.y (mgr.25072) 1263 : cluster 0 pgmap v1280: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 636 op/s 2023-10-01T10:59:31.839 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:31.863 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:31.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:31.866 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:59:31.914 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:31 smithi173 bash[58816]: cluster 2023-10-01T10:59:30.137237+0000 mgr.y (mgr.25072) 1263 : cluster 0 pgmap v1280: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 636 op/s 2023-10-01T10:59:31.966 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:31.994 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:31.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:31.997 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:59:32.087 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-10-01T10:59:32.087 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [], 2023-10-01T10:59:32.088 INFO:teuthology.orchestra.run.smithi173.stdout: "num_ops": 0 2023-10-01T10:59:32.088 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-10-01T10:59:32.098 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:32.126 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:32.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:32.128 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:59:32.229 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:32.256 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:32.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:32.258 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:32.359 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:32.400 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:32.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:32.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:59:32.507 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:32.542 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:32.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:32.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:59:32.648 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:32.676 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:32.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:32.679 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:59:32.779 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:32.806 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:32.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:32.811 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:32.911 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:32.946 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:32.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:32.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:59:32.977 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-01T10:59:33.052 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:33.088 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:33.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:33.100 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:59:33.201 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:33.229 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:33.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:33.233 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:33.334 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:33.359 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:33.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:33.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:33.464 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:33.503 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:33.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:33.506 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:59:33.606 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:33.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:33 smithi173 bash[58816]: cluster 2023-10-01T10:59:32.138021+0000 mgr.y (mgr.25072) 1264 : cluster 0 pgmap v1281: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 636 op/s 2023-10-01T10:59:33.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:33 smithi173 bash[58816]: audit 2023-10-01T10:59:32.456037+0000 mon.a (mon.0) 1566 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:59:33.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:33 smithi173 bash[58816]: audit 2023-10-01T10:59:32.462312+0000 mon.a (mon.0) 1567 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:59:33.636 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:33.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:33.639 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:59:33.739 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:33.765 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:33.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:33.767 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:59:33.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:33 smithi078 bash[78274]: cluster 2023-10-01T10:59:32.138021+0000 mgr.y (mgr.25072) 1264 : cluster 0 pgmap v1281: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 636 op/s 2023-10-01T10:59:33.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:33 smithi078 bash[78274]: audit 2023-10-01T10:59:32.456037+0000 mon.a (mon.0) 1566 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:59:33.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:33 smithi078 bash[78274]: audit 2023-10-01T10:59:32.462312+0000 mon.a (mon.0) 1567 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:59:33.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:33 smithi078 bash[75299]: cluster 2023-10-01T10:59:32.138021+0000 mgr.y (mgr.25072) 1264 : cluster 0 pgmap v1281: 89 pgs: 89 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 636 op/s 2023-10-01T10:59:33.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:33 smithi078 bash[75299]: audit 2023-10-01T10:59:32.456037+0000 mon.a (mon.0) 1566 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:59:33.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:33 smithi078 bash[75299]: audit 2023-10-01T10:59:32.462312+0000 mon.a (mon.0) 1567 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:59:33.867 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:33.894 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:33.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:33.899 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:59:34.003 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:34.041 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:34.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:34.045 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:59:34.145 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:34.174 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:34.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:34.177 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:59:34.277 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:34.307 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:34.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:34.310 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:34.411 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:34.443 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:34.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:34.451 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:34.552 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:34.575 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:34.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:34.579 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:34.680 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:34.706 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:34.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:34.709 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:59:34.809 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:34.838 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:34.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:34.841 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:59:34.941 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:34.967 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:34.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:34.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:59:35.074 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:35.106 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:35.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:35.111 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:59:35.211 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:35.240 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:35.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:35.242 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:59:35.343 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:35.369 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:35.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:35.375 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:59:35.462 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:35 smithi078 bash[78274]: cluster 2023-10-01T10:59:34.138963+0000 mgr.y (mgr.25072) 1265 : cluster 0 pgmap v1282: 89 pgs: 89 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 685 op/s 2023-10-01T10:59:35.462 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:35 smithi078 bash[75299]: cluster 2023-10-01T10:59:34.138963+0000 mgr.y (mgr.25072) 1265 : cluster 0 pgmap v1282: 89 pgs: 89 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 685 op/s 2023-10-01T10:59:35.475 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:35.486 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:35 smithi173 bash[58816]: cluster 2023-10-01T10:59:34.138963+0000 mgr.y (mgr.25072) 1265 : cluster 0 pgmap v1282: 89 pgs: 89 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 685 op/s 2023-10-01T10:59:35.502 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:35.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:35.505 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:59:35.605 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:35.639 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:35.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:35.641 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:59:35.742 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:35.768 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:35.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:35.772 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:59:35.873 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:35.898 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:35.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:35.904 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:59:36.005 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:36.035 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:36.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:36.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:59:36.142 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:36.174 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:36.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:36.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:59:36.280 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:36.308 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:36.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:36.314 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:59:36.415 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:36.439 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:36.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:36.444 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:59:36.544 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:36.576 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:36.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:36.578 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:59:36.679 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:36.704 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:36.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:36.706 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:36.807 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:36.833 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:36.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:36.836 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:59:36.937 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:36.963 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:36.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:36.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:59:37.067 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:37.101 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:37.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:37.104 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:59:37.205 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:37.224 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:37.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:37.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:59:37.326 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:37.358 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:37.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:37.363 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:59:37.464 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:37.491 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:37.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:37.497 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:59:37.598 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:37.610 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:37 smithi173 bash[58816]: cluster 2023-10-01T10:59:36.139553+0000 mgr.y (mgr.25072) 1266 : cluster 0 pgmap v1283: 89 pgs: 89 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 548 op/s 2023-10-01T10:59:37.629 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:37.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:37.633 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:37.733 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:37.748 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:37 smithi078 bash[78274]: cluster 2023-10-01T10:59:36.139553+0000 mgr.y (mgr.25072) 1266 : cluster 0 pgmap v1283: 89 pgs: 89 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 548 op/s 2023-10-01T10:59:37.749 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:37 smithi078 bash[75299]: cluster 2023-10-01T10:59:36.139553+0000 mgr.y (mgr.25072) 1266 : cluster 0 pgmap v1283: 89 pgs: 89 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 548 op/s 2023-10-01T10:59:37.786 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:37.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:37.792 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:59:37.833 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T10:59:37.892 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:37.913 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:37.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:37.915 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:59:38.016 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:38.041 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:38.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:38.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:59:38.148 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:38.178 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:38.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:38.184 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:59:38.285 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:38.307 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:38.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:38.309 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:59:38.410 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:38.439 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:38.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:38.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:59:38.545 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:38.568 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:38.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:38.571 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:59:38.671 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:38.704 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:38.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:38.710 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:59:38.810 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:38.840 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:38.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:38.846 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:59:38.946 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:38.977 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:38.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:38.983 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:39.084 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:39.110 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:39.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:39.112 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:59:39.212 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:39.240 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:39.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:39.242 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:59:39.343 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:39.371 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:39.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:39.373 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:59:39.474 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:39.502 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:39.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:39.505 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:39.606 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:39.633 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:39.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:39.638 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:59:39.739 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:39.747 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:39 smithi173 bash[58816]: cluster 2023-10-01T10:59:38.140098+0000 mgr.y (mgr.25072) 1267 : cluster 0 pgmap v1284: 89 pgs: 89 active+clean; 3.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 583 op/s 2023-10-01T10:59:39.770 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:39.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:39.773 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:39.873 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:39.894 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:39.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:39.898 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:59:39.998 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:40.023 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:40.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:40.026 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:40.085 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:39 smithi078 bash[75299]: cluster 2023-10-01T10:59:38.140098+0000 mgr.y (mgr.25072) 1267 : cluster 0 pgmap v1284: 89 pgs: 89 active+clean; 3.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 583 op/s 2023-10-01T10:59:40.086 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:39 smithi078 bash[78274]: cluster 2023-10-01T10:59:38.140098+0000 mgr.y (mgr.25072) 1267 : cluster 0 pgmap v1284: 89 pgs: 89 active+clean; 3.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 583 op/s 2023-10-01T10:59:40.126 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:40.162 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:40.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:40.168 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:59:40.269 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:40.295 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:40.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:40.299 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:40.400 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:40.426 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:40.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:40.432 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:59:40.532 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:40.561 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:40.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:40.565 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:40.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:59:40 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:59:40] "GET /metrics HTTP/1.1" 200 36388 "" "Prometheus/2.33.4" 2023-10-01T10:59:40.666 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:40.705 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:40.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:40.708 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:59:40.809 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:40.835 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:40.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:40.837 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:59:40.938 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:40.983 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:40.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:40.988 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:59:41.089 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:41.120 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:41.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:41.122 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:59:41.223 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:41.251 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:41.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:41.255 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:59:41.356 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:41.391 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:41.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:41.393 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:59:41.494 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:41.533 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:41.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:41.538 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:59:41.639 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:41.693 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:41.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:41.695 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:59:41.796 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:41.803 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:41 smithi078 bash[78274]: cluster 2023-10-01T10:59:40.140818+0000 mgr.y (mgr.25072) 1268 : cluster 0 pgmap v1285: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 715 op/s 2023-10-01T10:59:41.803 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:41 smithi078 bash[75299]: cluster 2023-10-01T10:59:40.140818+0000 mgr.y (mgr.25072) 1268 : cluster 0 pgmap v1285: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 715 op/s 2023-10-01T10:59:41.823 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:41.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:41.825 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:59:41.913 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-10-01T10:59:41.914 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [], 2023-10-01T10:59:41.914 INFO:teuthology.orchestra.run.smithi173.stdout: "complaint_time": 30, 2023-10-01T10:59:41.914 INFO:teuthology.orchestra.run.smithi173.stdout: "num_blocked_ops": 0 2023-10-01T10:59:41.914 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-10-01T10:59:41.926 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:41.936 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:41 smithi173 bash[58816]: cluster 2023-10-01T10:59:40.140818+0000 mgr.y (mgr.25072) 1268 : cluster 0 pgmap v1285: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 715 op/s 2023-10-01T10:59:41.955 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:41.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:41.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:59:42.059 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:42.094 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:42.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:42.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:59:42.117 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T10:59:42.159 INFO:teuthology.orchestra.run.smithi078.stdout: 2023-10-01T10:59:42.173 INFO:teuthology.orchestra.run.smithi078.stderr:dumped all 2023-10-01T10:59:42.177 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T10:59:42.198 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:42.229 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:42.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:42.231 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:59:42.235 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T10:59:42.273 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T10:59:42.321 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T10:59:42.332 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:42.355 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:42.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:42.359 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:59:42.363 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T10:59:42.410 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T10:59:42.447 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T10:59:42.450 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T10:59:42.450 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T10:59:42.451 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T10:59:42.451 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T10:59:42.451 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T10:59:42.451 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T10:59:42.451 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T10:59:42.451 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T10:59:42.459 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:42.485 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:42.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:42.490 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:59:42.591 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:42.612 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:42.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:42.614 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:42.656 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:42 smithi078 bash[78274]: audit 2023-10-01T10:59:42.067370+0000 mon.c (mon.2) 197 : audit 0 from='client.? 172.21.15.78:0/419644538' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:59:42.656 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:42 smithi078 bash[78274]: audit 2023-10-01T10:59:42.376205+0000 mon.a (mon.0) 1568 : audit 1 from='client.? 172.21.15.78:0/1742743966' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:59:42.656 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:42 smithi078 bash[75299]: audit 2023-10-01T10:59:42.067370+0000 mon.c (mon.2) 197 : audit 0 from='client.? 172.21.15.78:0/419644538' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:59:42.657 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:42 smithi078 bash[75299]: audit 2023-10-01T10:59:42.376205+0000 mon.a (mon.0) 1568 : audit 1 from='client.? 172.21.15.78:0/1742743966' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:59:42.670 INFO:teuthology.orchestra.run.smithi078.stderr:noscrub is set 2023-10-01T10:59:42.715 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:42.738 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:42.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:42.741 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:59:42.841 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:42.866 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:42.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:42.872 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:59:42.918 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-01T10:59:42.919 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:42 smithi173 bash[58816]: audit 2023-10-01T10:59:42.067370+0000 mon.c (mon.2) 197 : audit 0 from='client.? 172.21.15.78:0/419644538' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T10:59:42.919 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:42 smithi173 bash[58816]: audit 2023-10-01T10:59:42.376205+0000 mon.a (mon.0) 1568 : audit 1 from='client.? 172.21.15.78:0/1742743966' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T10:59:42.973 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:43.000 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:43.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:43.002 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:43.103 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:43.136 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:43.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:43.142 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:59:43.243 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:43.275 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:43.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:43.280 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:59:43.381 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:43.413 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:43.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:43.419 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:59:43.520 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:43.551 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:43.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:43.553 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:59:43.654 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:43.690 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:43.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:43.694 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:59:43.795 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:43.818 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:43.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:43.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:43.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:43 smithi078 bash[78274]: audit 2023-10-01T10:59:42.139094+0000 mgr.y (mgr.25072) 1269 : audit 0 from='client.46334 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:59:43.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:43 smithi078 bash[78274]: cluster 2023-10-01T10:59:42.156801+0000 mgr.y (mgr.25072) 1270 : cluster 0 pgmap v1286: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 514 op/s 2023-10-01T10:59:43.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:43 smithi078 bash[78274]: cluster 2023-10-01T10:59:42.644750+0000 mon.a (mon.0) 1569 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:59:43.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:43 smithi078 bash[78274]: audit 2023-10-01T10:59:42.660638+0000 mon.a (mon.0) 1570 : audit 1 from='client.? 172.21.15.78:0/1742743966' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T10:59:43.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:43 smithi078 bash[78274]: cluster 2023-10-01T10:59:42.660963+0000 mon.a (mon.0) 1571 : cluster 0 osdmap e253: 8 total, 8 up, 8 in 2023-10-01T10:59:43.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:43 smithi078 bash[78274]: cluster 2023-10-01T10:59:42.663001+0000 osd.7 (osd.7) 243 : cluster 3 failed to encode map e253 with expected crc 2023-10-01T10:59:43.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:43 smithi078 bash[78274]: cluster 2023-10-01T10:59:42.666149+0000 osd.7 (osd.7) 244 : cluster 3 failed to encode map e253 with expected crc 2023-10-01T10:59:43.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:43 smithi078 bash[78274]: cluster 2023-10-01T10:59:42.667058+0000 osd.6 (osd.6) 200 : cluster 3 failed to encode map e253 with expected crc 2023-10-01T10:59:43.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:43 smithi078 bash[78274]: cluster 2023-10-01T10:59:42.672058+0000 osd.6 (osd.6) 201 : cluster 3 failed to encode map e253 with expected crc 2023-10-01T10:59:43.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:43 smithi078 bash[75299]: audit 2023-10-01T10:59:42.139094+0000 mgr.y (mgr.25072) 1269 : audit 0 from='client.46334 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:59:43.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:43 smithi078 bash[75299]: cluster 2023-10-01T10:59:42.156801+0000 mgr.y (mgr.25072) 1270 : cluster 0 pgmap v1286: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 514 op/s 2023-10-01T10:59:43.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:43 smithi078 bash[75299]: cluster 2023-10-01T10:59:42.644750+0000 mon.a (mon.0) 1569 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:59:43.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:43 smithi078 bash[75299]: audit 2023-10-01T10:59:42.660638+0000 mon.a (mon.0) 1570 : audit 1 from='client.? 172.21.15.78:0/1742743966' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T10:59:43.833 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:43 smithi078 bash[75299]: cluster 2023-10-01T10:59:42.660963+0000 mon.a (mon.0) 1571 : cluster 0 osdmap e253: 8 total, 8 up, 8 in 2023-10-01T10:59:43.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:43 smithi078 bash[75299]: cluster 2023-10-01T10:59:42.663001+0000 osd.7 (osd.7) 243 : cluster 3 failed to encode map e253 with expected crc 2023-10-01T10:59:43.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:43 smithi078 bash[75299]: cluster 2023-10-01T10:59:42.666149+0000 osd.7 (osd.7) 244 : cluster 3 failed to encode map e253 with expected crc 2023-10-01T10:59:43.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:43 smithi078 bash[75299]: cluster 2023-10-01T10:59:42.667058+0000 osd.6 (osd.6) 200 : cluster 3 failed to encode map e253 with expected crc 2023-10-01T10:59:43.834 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:43 smithi078 bash[75299]: cluster 2023-10-01T10:59:42.672058+0000 osd.6 (osd.6) 201 : cluster 3 failed to encode map e253 with expected crc 2023-10-01T10:59:43.922 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:43.931 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:43 smithi173 bash[58816]: audit 2023-10-01T10:59:42.139094+0000 mgr.y (mgr.25072) 1269 : audit 0 from='client.46334 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T10:59:43.931 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:43 smithi173 bash[58816]: cluster 2023-10-01T10:59:42.156801+0000 mgr.y (mgr.25072) 1270 : cluster 0 pgmap v1286: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 514 op/s 2023-10-01T10:59:43.931 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:43 smithi173 bash[58816]: cluster 2023-10-01T10:59:42.644750+0000 mon.a (mon.0) 1569 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T10:59:43.931 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:43 smithi173 bash[58816]: audit 2023-10-01T10:59:42.660638+0000 mon.a (mon.0) 1570 : audit 1 from='client.? 172.21.15.78:0/1742743966' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T10:59:43.931 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:43 smithi173 bash[58816]: cluster 2023-10-01T10:59:42.660963+0000 mon.a (mon.0) 1571 : cluster 0 osdmap e253: 8 total, 8 up, 8 in 2023-10-01T10:59:43.932 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:43 smithi173 bash[58816]: cluster 2023-10-01T10:59:42.663001+0000 osd.7 (osd.7) 243 : cluster 3 failed to encode map e253 with expected crc 2023-10-01T10:59:43.932 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:43 smithi173 bash[58816]: cluster 2023-10-01T10:59:42.666149+0000 osd.7 (osd.7) 244 : cluster 3 failed to encode map e253 with expected crc 2023-10-01T10:59:43.932 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:43 smithi173 bash[58816]: cluster 2023-10-01T10:59:42.667058+0000 osd.6 (osd.6) 200 : cluster 3 failed to encode map e253 with expected crc 2023-10-01T10:59:43.932 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:43 smithi173 bash[58816]: cluster 2023-10-01T10:59:42.672058+0000 osd.6 (osd.6) 201 : cluster 3 failed to encode map e253 with expected crc 2023-10-01T10:59:43.949 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:43.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:43.952 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:44.053 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:44.087 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:44.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:44.092 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:59:44.193 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:44.221 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:44.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:44.227 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:59:44.328 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:44.358 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:44.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:44.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:59:44.463 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:44.492 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:44.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:44.500 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:59:44.601 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:44.630 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:44.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:44.632 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:59:44.733 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:44.759 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:44.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:44.766 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:59:44.867 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:44.900 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:44.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:44.906 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:59:45.007 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:45.034 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:45.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:45.040 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:59:45.140 INFO:teuthology.orchestra.run.smithi078.stdout:{"pg_ready":true,"pg_map":{"version":1285,"stamp":"2023-10-01T10:59:40.140336+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3065780152,"num_objects":46796,"num_object_clones":0,"num_object_copies":140388,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":46796,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":159747,"num_write_kb":6602639,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":1190344,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":142281,"ondisk_log_size":142281,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":14845212,"kb_used_data":9058332,"kb_used_omap":30,"kb_used_meta":5786721,"kb_avail":735116004,"statfs":{"total":767960285184,"available":752758788096,"internally_reserved":0,"allocated":9275731968,"data_stored":9269233218,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31502,"internal_metadata":5925602546},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[5,1,2,10,10,5,3],"upper_bound":128},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-562692096,"num_objects":-8586,"num_object_clones":0,"num_object_copies":-25758,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-8586,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8586,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004137"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":246,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-01T10:59:16.936705+0000","last_change":"2023-10-01T10:38:16.032258+0000","last_active":"2023-10-01T10:59:16.936705+0000","last_peered":"2023-10-01T10:59:16.936705+0000","last_clean":"2023-10-01T10:59:16.936705+0000","last_became_active":"2023-10-01T10:38:16.031633+0000","last_became_peered":"2023-10-01T10:38:16.031633+0000","last_unstale":"2023-10-01T10:59:16.936705+0000","last_undegraded":"2023-10-01T10:59:16.936705+0000","last_fullsized":"2023-10-01T10:59:16.936705+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:17:02.446430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":234,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-01T10:59:16.937167+0000","last_change":"2023-10-01T10:38:16.025881+0000","last_active":"2023-10-01T10:59:16.937167+0000","last_peered":"2023-10-01T10:59:16.937167+0000","last_clean":"2023-10-01T10:59:16.937167+0000","last_became_active":"2023-10-01T10:38:16.025518+0000","last_became_peered":"2023-10-01T10:38:16.025518+0000","last_unstale":"2023-10-01T10:59:16.937167+0000","last_undegraded":"2023-10-01T10:59:16.937167+0000","last_fullsized":"2023-10-01T10:59:16.937167+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:48:37.826604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":155,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-01T10:59:23.961122+0000","last_change":"2023-10-01T10:52:18.956544+0000","last_active":"2023-10-01T10:59:23.961122+0000","last_peered":"2023-10-01T10:59:23.961122+0000","last_clean":"2023-10-01T10:59:23.961122+0000","last_became_active":"2023-10-01T10:52:18.956217+0000","last_became_peered":"2023-10-01T10:52:18.956217+0000","last_unstale":"2023-10-01T10:59:23.961122+0000","last_undegraded":"2023-10-01T10:59:23.961122+0000","last_fullsized":"2023-10-01T10:59:23.961122+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:34:01.952235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":218,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-01T10:59:16.943366+0000","last_change":"2023-10-01T10:37:42.772008+0000","last_active":"2023-10-01T10:59:16.943366+0000","last_peered":"2023-10-01T10:59:16.943366+0000","last_clean":"2023-10-01T10:59:16.943366+0000","last_became_active":"2023-10-01T10:37:42.771244+0000","last_became_peered":"2023-10-01T10:37:42.771244+0000","last_unstale":"2023-10-01T10:59:16.943366+0000","last_undegraded":"2023-10-01T10:59:16.943366+0000","last_fullsized":"2023-10-01T10:59:16.943366+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:38:55.180790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":236,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-01T10:59:23.957473+0000","last_change":"2023-10-01T10:37:42.766600+0000","last_active":"2023-10-01T10:59:23.957473+0000","last_peered":"2023-10-01T10:59:23.957473+0000","last_clean":"2023-10-01T10:59:23.957473+0000","last_became_active":"2023-10-01T10:37:42.766051+0000","last_became_peered":"2023-10-01T10:37:42.766051+0000","last_unstale":"2023-10-01T10:59:23.957473+0000","last_undegraded":"2023-10-01T10:59:23.957473+0000","last_fullsized":"2023-10-01T10:59:23.957473+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:51:17.707963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":27,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-01T10:59:16.949222+0000","last_change":"2023-10-01T10:56:14.626119+0000","last_active":"2023-10-01T10:59:16.949222+0000","last_peered":"2023-10-01T10:59:16.949222+0000","last_clean":"2023-10-01T10:59:16.949222+0000","last_became_active":"2023-10-01T10:56:14.625777+0000","last_became_peered":"2023-10-01T10:56:14.625777+0000","last_unstale":"2023-10-01T10:59:16.949222+0000","last_undegraded":"2023-10-01T10:59:16.949222+0000","last_fullsized":"2023-10-01T10:59:16.949222+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":214,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-01T10:59:16.937150+0000","last_change":"2023-10-01T10:37:42.766667+0000","last_active":"2023-10-01T10:59:16.937150+0000","last_peered":"2023-10-01T10:59:16.937150+0000","last_clean":"2023-10-01T10:59:16.937150+0000","last_became_active":"2023-10-01T10:37:42.766369+0000","last_became_peered":"2023-10-01T10:37:42.766369+0000","last_unstale":"2023-10-01T10:59:16.937150+0000","last_undegraded":"2023-10-01T10:59:16.937150+0000","last_fullsized":"2023-10-01T10:59:16.937150+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:02.708788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":238,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-01T10:59:16.939810+0000","last_change":"2023-10-01T10:56:14.622767+0000","last_active":"2023-10-01T10:59:16.939810+0000","last_peered":"2023-10-01T10:59:16.939810+0000","last_clean":"2023-10-01T10:59:16.939810+0000","last_became_active":"2023-10-01T10:56:14.622446+0000","last_became_peered":"2023-10-01T10:56:14.622446+0000","last_unstale":"2023-10-01T10:59:16.939810+0000","last_undegraded":"2023-10-01T10:59:16.939810+0000","last_fullsized":"2023-10-01T10:59:16.939810+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:32:09.354415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":219,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-01T10:59:16.951261+0000","last_change":"2023-10-01T10:32:31.579805+0000","last_active":"2023-10-01T10:59:16.951261+0000","last_peered":"2023-10-01T10:59:16.951261+0000","last_clean":"2023-10-01T10:59:16.951261+0000","last_became_active":"2023-10-01T10:32:31.579423+0000","last_became_peered":"2023-10-01T10:32:31.579423+0000","last_unstale":"2023-10-01T10:59:16.951261+0000","last_undegraded":"2023-10-01T10:59:16.951261+0000","last_fullsized":"2023-10-01T10:59:16.951261+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":222,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-01T10:59:16.942870+0000","last_change":"2023-10-01T10:56:14.619280+0000","last_active":"2023-10-01T10:59:16.942870+0000","last_peered":"2023-10-01T10:59:16.942870+0000","last_clean":"2023-10-01T10:59:16.942870+0000","last_became_active":"2023-10-01T10:56:14.618892+0000","last_became_peered":"2023-10-01T10:56:14.618892+0000","last_unstale":"2023-10-01T10:59:16.942870+0000","last_undegraded":"2023-10-01T10:59:16.942870+0000","last_fullsized":"2023-10-01T10:59:16.942870+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:14:29.635055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":93,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-01T10:59:16.937087+0000","last_change":"2023-10-01T10:52:43.661650+0000","last_active":"2023-10-01T10:59:16.937087+0000","last_peered":"2023-10-01T10:59:16.937087+0000","last_clean":"2023-10-01T10:59:16.937087+0000","last_became_active":"2023-10-01T10:52:43.661202+0000","last_became_peered":"2023-10-01T10:52:43.661202+0000","last_unstale":"2023-10-01T10:59:16.937087+0000","last_undegraded":"2023-10-01T10:59:16.937087+0000","last_fullsized":"2023-10-01T10:59:16.937087+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:28:35.479124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":27,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-01T10:59:16.946090+0000","last_change":"2023-10-01T10:56:14.620228+0000","last_active":"2023-10-01T10:59:16.946090+0000","last_peered":"2023-10-01T10:59:16.946090+0000","last_clean":"2023-10-01T10:59:16.946090+0000","last_became_active":"2023-10-01T10:56:14.619632+0000","last_became_peered":"2023-10-01T10:56:14.619632+0000","last_unstale":"2023-10-01T10:59:16.946090+0000","last_undegraded":"2023-10-01T10:59:16.946090+0000","last_fullsized":"2023-10-01T10:59:16.946090+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":279,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-01T10:59:19.244918+0000","last_change":"2023-10-01T10:56:14.634698+0000","last_active":"2023-10-01T10:59:19.244918+0000","last_peered":"2023-10-01T10:59:19.244918+0000","last_clean":"2023-10-01T10:59:19.244918+0000","last_became_active":"2023-10-01T10:56:14.625320+0000","last_became_peered":"2023-10-01T10:56:14.625320+0000","last_unstale":"2023-10-01T10:59:19.244918+0000","last_undegraded":"2023-10-01T10:59:19.244918+0000","last_fullsized":"2023-10-01T10:59:19.244918+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:02:56.480434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":233,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-01T10:59:16.935937+0000","last_change":"2023-10-01T10:56:14.622580+0000","last_active":"2023-10-01T10:59:16.935937+0000","last_peered":"2023-10-01T10:59:16.935937+0000","last_clean":"2023-10-01T10:59:16.935937+0000","last_became_active":"2023-10-01T10:56:14.622075+0000","last_became_peered":"2023-10-01T10:56:14.622075+0000","last_unstale":"2023-10-01T10:59:16.935937+0000","last_undegraded":"2023-10-01T10:59:16.935937+0000","last_fullsized":"2023-10-01T10:59:16.935937+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:52:32.352383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":235,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-01T10:59:16.947283+0000","last_change":"2023-10-01T10:38:16.026777+0000","last_active":"2023-10-01T10:59:16.947283+0000","last_peered":"2023-10-01T10:59:16.947283+0000","last_clean":"2023-10-01T10:59:16.947283+0000","last_became_active":"2023-10-01T10:38:16.026371+0000","last_became_peered":"2023-10-01T10:38:16.026371+0000","last_unstale":"2023-10-01T10:59:16.947283+0000","last_undegraded":"2023-10-01T10:59:16.947283+0000","last_fullsized":"2023-10-01T10:59:16.947283+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":243,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-01T10:59:16.936912+0000","last_change":"2023-10-01T10:37:42.772319+0000","last_active":"2023-10-01T10:59:16.936912+0000","last_peered":"2023-10-01T10:59:16.936912+0000","last_clean":"2023-10-01T10:59:16.936912+0000","last_became_active":"2023-10-01T10:37:42.771977+0000","last_became_peered":"2023-10-01T10:37:42.771977+0000","last_unstale":"2023-10-01T10:59:16.936912+0000","last_undegraded":"2023-10-01T10:59:16.936912+0000","last_fullsized":"2023-10-01T10:59:16.936912+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:45:12.783214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":215,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-01T10:59:16.938429+0000","last_change":"2023-10-01T10:37:42.767706+0000","last_active":"2023-10-01T10:59:16.938429+0000","last_peered":"2023-10-01T10:59:16.938429+0000","last_clean":"2023-10-01T10:59:16.938429+0000","last_became_active":"2023-10-01T10:37:42.767211+0000","last_became_peered":"2023-10-01T10:37:42.767211+0000","last_unstale":"2023-10-01T10:59:16.938429+0000","last_undegraded":"2023-10-01T10:59:16.938429+0000","last_fullsized":"2023-10-01T10:59:16.938429+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:37:20.357407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":218,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-01T10:59:16.951088+0000","last_change":"2023-10-01T10:37:42.768642+0000","last_active":"2023-10-01T10:59:16.951088+0000","last_peered":"2023-10-01T10:59:16.951088+0000","last_clean":"2023-10-01T10:59:16.951088+0000","last_became_active":"2023-10-01T10:37:42.768239+0000","last_became_peered":"2023-10-01T10:37:42.768239+0000","last_unstale":"2023-10-01T10:59:16.951088+0000","last_undegraded":"2023-10-01T10:59:16.951088+0000","last_fullsized":"2023-10-01T10:59:16.951088+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":185,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-01T10:59:16.935603+0000","last_change":"2023-10-01T10:56:14.620937+0000","last_active":"2023-10-01T10:59:16.935603+0000","last_peered":"2023-10-01T10:59:16.935603+0000","last_clean":"2023-10-01T10:59:16.935603+0000","last_became_active":"2023-10-01T10:56:14.620579+0000","last_became_peered":"2023-10-01T10:56:14.620579+0000","last_unstale":"2023-10-01T10:59:16.935603+0000","last_undegraded":"2023-10-01T10:59:16.935603+0000","last_fullsized":"2023-10-01T10:59:16.935603+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:53:21.947263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":215,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-01T10:59:16.943769+0000","last_change":"2023-10-01T10:37:42.772247+0000","last_active":"2023-10-01T10:59:16.943769+0000","last_peered":"2023-10-01T10:59:16.943769+0000","last_clean":"2023-10-01T10:59:16.943769+0000","last_became_active":"2023-10-01T10:37:42.771697+0000","last_became_peered":"2023-10-01T10:37:42.771697+0000","last_unstale":"2023-10-01T10:59:16.943769+0000","last_undegraded":"2023-10-01T10:59:16.943769+0000","last_fullsized":"2023-10-01T10:59:16.943769+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:23:15.018248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":37,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-01T10:59:16.948314+0000","last_change":"2023-10-01T10:56:14.636139+0000","last_active":"2023-10-01T10:59:16.948314+0000","last_peered":"2023-10-01T10:59:16.948314+0000","last_clean":"2023-10-01T10:59:16.948314+0000","last_became_active":"2023-10-01T10:56:14.624426+0000","last_became_peered":"2023-10-01T10:56:14.624426+0000","last_unstale":"2023-10-01T10:59:16.948314+0000","last_undegraded":"2023-10-01T10:59:16.948314+0000","last_fullsized":"2023-10-01T10:59:16.948314+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":220,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-01T10:59:23.958388+0000","last_change":"2023-10-01T10:37:42.773127+0000","last_active":"2023-10-01T10:59:23.958388+0000","last_peered":"2023-10-01T10:59:23.958388+0000","last_clean":"2023-10-01T10:59:23.958388+0000","last_became_active":"2023-10-01T10:37:42.772102+0000","last_became_peered":"2023-10-01T10:37:42.772102+0000","last_unstale":"2023-10-01T10:59:23.958388+0000","last_undegraded":"2023-10-01T10:59:23.958388+0000","last_fullsized":"2023-10-01T10:59:23.958388+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:23:26.202295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.0","version":"252'9985","reported_seq":10008,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-01T10:59:39.204165+0000","last_change":"2023-10-01T10:56:21.668720+0000","last_active":"2023-10-01T10:59:39.204165+0000","last_peered":"2023-10-01T10:59:39.204165+0000","last_clean":"2023-10-01T10:59:39.204165+0000","last_became_active":"2023-10-01T10:56:21.668210+0000","last_became_peered":"2023-10-01T10:56:21.668210+0000","last_unstale":"2023-10-01T10:59:39.204165+0000","last_undegraded":"2023-10-01T10:59:39.204165+0000","last_fullsized":"2023-10-01T10:59:39.204165+0000","mapping_epoch":239,"log_start":"244'1600","ondisk_log_start":"244'1600","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":8385,"log_dups_size":0,"ondisk_log_size":8385,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:32:59.602604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182255616,"num_objects":2781,"num_object_clones":0,"num_object_copies":8343,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2781,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9985,"num_write_kb":408512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":94,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-01T10:59:23.972416+0000","last_change":"2023-10-01T10:52:43.663145+0000","last_active":"2023-10-01T10:59:23.972416+0000","last_peered":"2023-10-01T10:59:23.972416+0000","last_clean":"2023-10-01T10:59:23.972416+0000","last_became_active":"2023-10-01T10:52:43.662783+0000","last_became_peered":"2023-10-01T10:52:43.662783+0000","last_unstale":"2023-10-01T10:59:23.972416+0000","last_undegraded":"2023-10-01T10:59:23.972416+0000","last_fullsized":"2023-10-01T10:59:23.972416+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T22:10:32.190922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":221,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-01T10:59:16.950505+0000","last_change":"2023-10-01T10:56:14.627958+0000","last_active":"2023-10-01T10:59:16.950505+0000","last_peered":"2023-10-01T10:59:16.950505+0000","last_clean":"2023-10-01T10:59:16.950505+0000","last_became_active":"2023-10-01T10:56:14.627506+0000","last_became_peered":"2023-10-01T10:56:14.627506+0000","last_unstale":"2023-10-01T10:59:16.950505+0000","last_undegraded":"2023-10-01T10:59:16.950505+0000","last_fullsized":"2023-10-01T10:59:16.950505+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"252'9921","reported_seq":9944,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-01T10:59:37.863295+0000","last_change":"2023-10-01T10:56:21.667913+0000","last_active":"2023-10-01T10:59:37.863295+0000","last_peered":"2023-10-01T10:59:37.863295+0000","last_clean":"2023-10-01T10:59:37.863295+0000","last_became_active":"2023-10-01T10:56:21.667502+0000","last_became_peered":"2023-10-01T10:56:21.667502+0000","last_unstale":"2023-10-01T10:59:37.863295+0000","last_undegraded":"2023-10-01T10:59:37.863295+0000","last_fullsized":"2023-10-01T10:59:37.863295+0000","mapping_epoch":239,"log_start":"244'2200","ondisk_log_start":"244'2200","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":7721,"log_dups_size":0,"ondisk_log_size":7721,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:53:11.947301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":197853184,"num_objects":3019,"num_object_clones":0,"num_object_copies":9057,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3019,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9921,"num_write_kb":414080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":94,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-01T10:59:23.973212+0000","last_change":"2023-10-01T10:52:43.662540+0000","last_active":"2023-10-01T10:59:23.973212+0000","last_peered":"2023-10-01T10:59:23.973212+0000","last_clean":"2023-10-01T10:59:23.973212+0000","last_became_active":"2023-10-01T10:52:43.662128+0000","last_became_peered":"2023-10-01T10:52:43.662128+0000","last_unstale":"2023-10-01T10:59:23.973212+0000","last_undegraded":"2023-10-01T10:59:23.973212+0000","last_fullsized":"2023-10-01T10:59:23.973212+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:09:44.807003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":249,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-01T10:59:16.935721+0000","last_change":"2023-10-01T10:38:16.033203+0000","last_active":"2023-10-01T10:59:16.935721+0000","last_peered":"2023-10-01T10:59:16.935721+0000","last_clean":"2023-10-01T10:59:16.935721+0000","last_became_active":"2023-10-01T10:38:16.032681+0000","last_became_peered":"2023-10-01T10:38:16.032681+0000","last_unstale":"2023-10-01T10:59:16.935721+0000","last_undegraded":"2023-10-01T10:59:16.935721+0000","last_fullsized":"2023-10-01T10:59:16.935721+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:42:36.642284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"252'9800","reported_seq":9823,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-01T10:59:39.210966+0000","last_change":"2023-10-01T10:56:21.664087+0000","last_active":"2023-10-01T10:59:39.210966+0000","last_peered":"2023-10-01T10:59:39.210966+0000","last_clean":"2023-10-01T10:59:39.210966+0000","last_became_active":"2023-10-01T10:56:21.663655+0000","last_became_peered":"2023-10-01T10:56:21.663655+0000","last_unstale":"2023-10-01T10:59:39.210966+0000","last_undegraded":"2023-10-01T10:59:39.210966+0000","last_fullsized":"2023-10-01T10:59:39.210966+0000","mapping_epoch":239,"log_start":"244'1400","ondisk_log_start":"244'1400","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":8400,"log_dups_size":0,"ondisk_log_size":8400,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:34:34.434364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184942592,"num_objects":2822,"num_object_clones":0,"num_object_copies":8466,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2822,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9800,"num_write_kb":403904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":117,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-01T10:59:16.951249+0000","last_change":"2023-10-01T10:56:14.638810+0000","last_active":"2023-10-01T10:59:16.951249+0000","last_peered":"2023-10-01T10:59:16.951249+0000","last_clean":"2023-10-01T10:59:16.951249+0000","last_became_active":"2023-10-01T10:56:14.624601+0000","last_became_peered":"2023-10-01T10:56:14.624601+0000","last_unstale":"2023-10-01T10:59:16.951249+0000","last_undegraded":"2023-10-01T10:59:16.951249+0000","last_fullsized":"2023-10-01T10:59:16.951249+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":27,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-01T10:59:16.945934+0000","last_change":"2023-10-01T10:56:14.623448+0000","last_active":"2023-10-01T10:59:16.945934+0000","last_peered":"2023-10-01T10:59:16.945934+0000","last_clean":"2023-10-01T10:59:16.945934+0000","last_became_active":"2023-10-01T10:56:14.623123+0000","last_became_peered":"2023-10-01T10:56:14.623123+0000","last_unstale":"2023-10-01T10:59:16.945934+0000","last_undegraded":"2023-10-01T10:59:16.945934+0000","last_fullsized":"2023-10-01T10:59:16.945934+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"252'9863","reported_seq":9886,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-01T10:59:38.422145+0000","last_change":"2023-10-01T10:56:21.664942+0000","last_active":"2023-10-01T10:59:38.422145+0000","last_peered":"2023-10-01T10:59:38.422145+0000","last_clean":"2023-10-01T10:59:38.422145+0000","last_became_active":"2023-10-01T10:56:21.664236+0000","last_became_peered":"2023-10-01T10:56:21.664236+0000","last_unstale":"2023-10-01T10:59:38.422145+0000","last_undegraded":"2023-10-01T10:59:38.422145+0000","last_fullsized":"2023-10-01T10:59:38.422145+0000","mapping_epoch":239,"log_start":"244'400","ondisk_log_start":"244'400","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":9463,"log_dups_size":0,"ondisk_log_size":9463,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:51:15.411254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":190119936,"num_objects":2901,"num_object_clones":0,"num_object_copies":8703,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2901,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9863,"num_write_kb":408448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":237,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-01T10:59:16.951070+0000","last_change":"2023-10-01T10:56:14.624999+0000","last_active":"2023-10-01T10:59:16.951070+0000","last_peered":"2023-10-01T10:59:16.951070+0000","last_clean":"2023-10-01T10:59:16.951070+0000","last_became_active":"2023-10-01T10:56:14.624597+0000","last_became_peered":"2023-10-01T10:56:14.624597+0000","last_unstale":"2023-10-01T10:59:16.951070+0000","last_undegraded":"2023-10-01T10:59:16.951070+0000","last_fullsized":"2023-10-01T10:59:16.951070+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":231,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-01T10:59:16.953518+0000","last_change":"2023-10-01T10:38:16.026348+0000","last_active":"2023-10-01T10:59:16.953518+0000","last_peered":"2023-10-01T10:59:16.953518+0000","last_clean":"2023-10-01T10:59:16.953518+0000","last_became_active":"2023-10-01T10:38:16.025747+0000","last_became_peered":"2023-10-01T10:38:16.025747+0000","last_unstale":"2023-10-01T10:59:16.953518+0000","last_undegraded":"2023-10-01T10:59:16.953518+0000","last_fullsized":"2023-10-01T10:59:16.953518+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:28:18.934480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"252'9937","reported_seq":9960,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-01T10:59:37.835174+0000","last_change":"2023-10-01T10:56:22.013729+0000","last_active":"2023-10-01T10:59:37.835174+0000","last_peered":"2023-10-01T10:59:37.835174+0000","last_clean":"2023-10-01T10:59:37.835174+0000","last_became_active":"2023-10-01T10:56:22.013403+0000","last_became_peered":"2023-10-01T10:56:22.013403+0000","last_unstale":"2023-10-01T10:59:37.835174+0000","last_undegraded":"2023-10-01T10:59:37.835174+0000","last_fullsized":"2023-10-01T10:59:37.835174+0000","mapping_epoch":239,"log_start":"244'2200","ondisk_log_start":"244'2200","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":7737,"log_dups_size":0,"ondisk_log_size":7737,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:34:17.163618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":194052096,"num_objects":2961,"num_object_clones":0,"num_object_copies":8883,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2961,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9937,"num_write_kb":412736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":223,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-01T10:59:16.940941+0000","last_change":"2023-10-01T10:38:16.022493+0000","last_active":"2023-10-01T10:59:16.940941+0000","last_peered":"2023-10-01T10:59:16.940941+0000","last_clean":"2023-10-01T10:59:16.940941+0000","last_became_active":"2023-10-01T10:38:16.022033+0000","last_became_peered":"2023-10-01T10:38:16.022033+0000","last_unstale":"2023-10-01T10:59:16.940941+0000","last_undegraded":"2023-10-01T10:59:16.940941+0000","last_fullsized":"2023-10-01T10:59:16.940941+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:49:05.262763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":252,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-01T10:59:16.935851+0000","last_change":"2023-10-01T10:56:14.623628+0000","last_active":"2023-10-01T10:59:16.935851+0000","last_peered":"2023-10-01T10:59:16.935851+0000","last_clean":"2023-10-01T10:59:16.935851+0000","last_became_active":"2023-10-01T10:56:14.622190+0000","last_became_peered":"2023-10-01T10:56:14.622190+0000","last_unstale":"2023-10-01T10:59:16.935851+0000","last_undegraded":"2023-10-01T10:59:16.935851+0000","last_fullsized":"2023-10-01T10:59:16.935851+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:57:06.093638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"252'9706","reported_seq":9729,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-01T10:59:38.936088+0000","last_change":"2023-10-01T10:56:22.014079+0000","last_active":"2023-10-01T10:59:38.936088+0000","last_peered":"2023-10-01T10:59:38.936088+0000","last_clean":"2023-10-01T10:59:38.936088+0000","last_became_active":"2023-10-01T10:56:22.013703+0000","last_became_peered":"2023-10-01T10:56:22.013703+0000","last_unstale":"2023-10-01T10:59:38.936088+0000","last_undegraded":"2023-10-01T10:59:38.936088+0000","last_fullsized":"2023-10-01T10:59:38.936088+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":9706,"log_dups_size":0,"ondisk_log_size":9706,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:07:01.254677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185204736,"num_objects":2826,"num_object_clones":0,"num_object_copies":8478,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2826,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9706,"num_write_kb":401024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":225,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-01T10:59:16.936871+0000","last_change":"2023-10-01T10:56:14.622835+0000","last_active":"2023-10-01T10:59:16.936871+0000","last_peered":"2023-10-01T10:59:16.936871+0000","last_clean":"2023-10-01T10:59:16.936871+0000","last_became_active":"2023-10-01T10:56:14.622449+0000","last_became_peered":"2023-10-01T10:56:14.622449+0000","last_unstale":"2023-10-01T10:59:16.936871+0000","last_undegraded":"2023-10-01T10:59:16.936871+0000","last_fullsized":"2023-10-01T10:59:16.936871+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:29:48.934875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":239,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-01T10:59:16.936254+0000","last_change":"2023-10-01T10:38:16.035906+0000","last_active":"2023-10-01T10:59:16.936254+0000","last_peered":"2023-10-01T10:59:16.936254+0000","last_clean":"2023-10-01T10:59:16.936254+0000","last_became_active":"2023-10-01T10:38:16.035383+0000","last_became_peered":"2023-10-01T10:38:16.035383+0000","last_unstale":"2023-10-01T10:59:16.936254+0000","last_undegraded":"2023-10-01T10:59:16.936254+0000","last_fullsized":"2023-10-01T10:59:16.936254+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:44:41.500097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"252'9903","reported_seq":9925,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-01T10:59:38.387965+0000","last_change":"2023-10-01T10:56:22.013872+0000","last_active":"2023-10-01T10:59:38.387965+0000","last_peered":"2023-10-01T10:59:38.387965+0000","last_clean":"2023-10-01T10:59:38.387965+0000","last_became_active":"2023-10-01T10:56:22.013554+0000","last_became_peered":"2023-10-01T10:56:22.013554+0000","last_unstale":"2023-10-01T10:59:38.387965+0000","last_undegraded":"2023-10-01T10:59:38.387965+0000","last_fullsized":"2023-10-01T10:59:38.387965+0000","mapping_epoch":239,"log_start":"244'500","ondisk_log_start":"244'500","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":9403,"log_dups_size":0,"ondisk_log_size":9403,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:10:34.060296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":196411392,"num_objects":2997,"num_object_clones":0,"num_object_copies":8991,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2997,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9903,"num_write_kb":412800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":231,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-01T10:59:16.936036+0000","last_change":"2023-10-01T10:38:16.026611+0000","last_active":"2023-10-01T10:59:16.936036+0000","last_peered":"2023-10-01T10:59:16.936036+0000","last_clean":"2023-10-01T10:59:16.936036+0000","last_became_active":"2023-10-01T10:38:16.026267+0000","last_became_peered":"2023-10-01T10:38:16.026267+0000","last_unstale":"2023-10-01T10:59:16.936036+0000","last_undegraded":"2023-10-01T10:59:16.936036+0000","last_fullsized":"2023-10-01T10:59:16.936036+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:56:06.135288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":233,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-01T10:59:16.936396+0000","last_change":"2023-10-01T10:56:14.623011+0000","last_active":"2023-10-01T10:59:16.936396+0000","last_peered":"2023-10-01T10:59:16.936396+0000","last_clean":"2023-10-01T10:59:16.936396+0000","last_became_active":"2023-10-01T10:56:14.622649+0000","last_became_peered":"2023-10-01T10:56:14.622649+0000","last_unstale":"2023-10-01T10:59:16.936396+0000","last_undegraded":"2023-10-01T10:59:16.936396+0000","last_fullsized":"2023-10-01T10:59:16.936396+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:32:42.042469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"252'10119","reported_seq":10142,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-01T10:59:38.980447+0000","last_change":"2023-10-01T10:56:22.016312+0000","last_active":"2023-10-01T10:59:38.980447+0000","last_peered":"2023-10-01T10:59:38.980447+0000","last_clean":"2023-10-01T10:59:38.980447+0000","last_became_active":"2023-10-01T10:56:22.015964+0000","last_became_peered":"2023-10-01T10:56:22.015964+0000","last_unstale":"2023-10-01T10:59:38.980447+0000","last_undegraded":"2023-10-01T10:59:38.980447+0000","last_fullsized":"2023-10-01T10:59:38.980447+0000","mapping_epoch":239,"log_start":"243'100","ondisk_log_start":"243'100","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":10019,"log_dups_size":0,"ondisk_log_size":10019,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:30:15.203782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":196411392,"num_objects":2997,"num_object_clones":0,"num_object_copies":8991,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2997,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10119,"num_write_kb":419712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":222,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-01T10:59:22.828686+0000","last_change":"2023-10-01T10:37:42.772398+0000","last_active":"2023-10-01T10:59:22.828686+0000","last_peered":"2023-10-01T10:59:22.828686+0000","last_clean":"2023-10-01T10:59:22.828686+0000","last_became_active":"2023-10-01T10:37:42.772084+0000","last_became_peered":"2023-10-01T10:37:42.772084+0000","last_unstale":"2023-10-01T10:59:22.828686+0000","last_undegraded":"2023-10-01T10:59:22.828686+0000","last_fullsized":"2023-10-01T10:59:22.828686+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:32:43.846419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":240,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-01T10:59:23.984389+0000","last_change":"2023-10-01T10:38:16.019755+0000","last_active":"2023-10-01T10:59:23.984389+0000","last_peered":"2023-10-01T10:59:23.984389+0000","last_clean":"2023-10-01T10:59:23.984389+0000","last_became_active":"2023-10-01T10:38:16.019397+0000","last_became_peered":"2023-10-01T10:38:16.019397+0000","last_unstale":"2023-10-01T10:59:23.984389+0000","last_undegraded":"2023-10-01T10:59:23.984389+0000","last_fullsized":"2023-10-01T10:59:23.984389+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:42:56.623225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.a","version":"252'9927","reported_seq":9950,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-01T10:59:38.988727+0000","last_change":"2023-10-01T10:56:21.663116+0000","last_active":"2023-10-01T10:59:38.988727+0000","last_peered":"2023-10-01T10:59:38.988727+0000","last_clean":"2023-10-01T10:59:38.988727+0000","last_became_active":"2023-10-01T10:56:21.662762+0000","last_became_peered":"2023-10-01T10:56:21.662762+0000","last_unstale":"2023-10-01T10:59:38.988727+0000","last_undegraded":"2023-10-01T10:59:38.988727+0000","last_fullsized":"2023-10-01T10:59:38.988727+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":9927,"log_dups_size":0,"ondisk_log_size":9927,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:00:59.657772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188153856,"num_objects":2871,"num_object_clones":0,"num_object_copies":8613,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2871,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9927,"num_write_kb":409536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":147,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-01T10:59:16.936340+0000","last_change":"2023-10-01T10:52:18.960164+0000","last_active":"2023-10-01T10:59:16.936340+0000","last_peered":"2023-10-01T10:59:16.936340+0000","last_clean":"2023-10-01T10:59:16.936340+0000","last_became_active":"2023-10-01T10:52:18.959782+0000","last_became_peered":"2023-10-01T10:52:18.959782+0000","last_unstale":"2023-10-01T10:59:16.936340+0000","last_undegraded":"2023-10-01T10:59:16.936340+0000","last_fullsized":"2023-10-01T10:59:16.936340+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:08:27.603412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":234,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-01T10:59:16.950980+0000","last_change":"2023-10-01T10:37:42.766814+0000","last_active":"2023-10-01T10:59:16.950980+0000","last_peered":"2023-10-01T10:59:16.950980+0000","last_clean":"2023-10-01T10:59:16.950980+0000","last_became_active":"2023-10-01T10:37:42.766451+0000","last_became_peered":"2023-10-01T10:37:42.766451+0000","last_unstale":"2023-10-01T10:59:16.950980+0000","last_undegraded":"2023-10-01T10:59:16.950980+0000","last_fullsized":"2023-10-01T10:59:16.950980+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":137,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-01T10:59:16.950630+0000","last_change":"2023-10-01T10:52:18.960606+0000","last_active":"2023-10-01T10:59:16.950630+0000","last_peered":"2023-10-01T10:59:16.950630+0000","last_clean":"2023-10-01T10:59:16.950630+0000","last_became_active":"2023-10-01T10:52:18.960270+0000","last_became_peered":"2023-10-01T10:52:18.960270+0000","last_unstale":"2023-10-01T10:59:16.950630+0000","last_undegraded":"2023-10-01T10:59:16.950630+0000","last_fullsized":"2023-10-01T10:59:16.950630+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.c","version":"252'10052","reported_seq":10075,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-01T10:59:38.456548+0000","last_change":"2023-10-01T10:56:21.895915+0000","last_active":"2023-10-01T10:59:38.456548+0000","last_peered":"2023-10-01T10:59:38.456548+0000","last_clean":"2023-10-01T10:59:38.456548+0000","last_became_active":"2023-10-01T10:56:21.895501+0000","last_became_peered":"2023-10-01T10:56:21.895501+0000","last_unstale":"2023-10-01T10:59:38.456548+0000","last_undegraded":"2023-10-01T10:59:38.456548+0000","last_fullsized":"2023-10-01T10:59:38.456548+0000","mapping_epoch":239,"log_start":"244'600","ondisk_log_start":"244'600","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":9452,"log_dups_size":0,"ondisk_log_size":9452,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:21:21.588297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":194904064,"num_objects":2974,"num_object_clones":0,"num_object_copies":8922,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2974,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10052,"num_write_kb":416832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":241,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-01T10:59:16.950713+0000","last_change":"2023-10-01T10:56:14.624787+0000","last_active":"2023-10-01T10:59:16.950713+0000","last_peered":"2023-10-01T10:59:16.950713+0000","last_clean":"2023-10-01T10:59:16.950713+0000","last_became_active":"2023-10-01T10:56:14.624357+0000","last_became_peered":"2023-10-01T10:56:14.624357+0000","last_unstale":"2023-10-01T10:59:16.950713+0000","last_undegraded":"2023-10-01T10:59:16.950713+0000","last_fullsized":"2023-10-01T10:59:16.950713+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":220,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-01T10:59:16.936619+0000","last_change":"2023-10-01T10:56:14.620354+0000","last_active":"2023-10-01T10:59:16.936619+0000","last_peered":"2023-10-01T10:59:16.936619+0000","last_clean":"2023-10-01T10:59:16.936619+0000","last_became_active":"2023-10-01T10:56:14.619900+0000","last_became_peered":"2023-10-01T10:56:14.619900+0000","last_unstale":"2023-10-01T10:59:16.936619+0000","last_undegraded":"2023-10-01T10:59:16.936619+0000","last_fullsized":"2023-10-01T10:59:16.936619+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:09:36.449841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":150,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-01T10:59:16.943845+0000","last_change":"2023-10-01T10:52:18.963437+0000","last_active":"2023-10-01T10:59:16.943845+0000","last_peered":"2023-10-01T10:59:16.943845+0000","last_clean":"2023-10-01T10:59:16.943845+0000","last_became_active":"2023-10-01T10:52:18.955165+0000","last_became_peered":"2023-10-01T10:52:18.955165+0000","last_unstale":"2023-10-01T10:59:16.943845+0000","last_undegraded":"2023-10-01T10:59:16.943845+0000","last_fullsized":"2023-10-01T10:59:16.943845+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:57:49.129100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.f","version":"252'10150","reported_seq":10171,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-01T10:59:38.537201+0000","last_change":"2023-10-01T10:56:21.896411+0000","last_active":"2023-10-01T10:59:38.537201+0000","last_peered":"2023-10-01T10:59:38.537201+0000","last_clean":"2023-10-01T10:59:38.537201+0000","last_became_active":"2023-10-01T10:56:21.895807+0000","last_became_peered":"2023-10-01T10:56:21.895807+0000","last_unstale":"2023-10-01T10:59:38.537201+0000","last_undegraded":"2023-10-01T10:59:38.537201+0000","last_fullsized":"2023-10-01T10:59:38.537201+0000","mapping_epoch":239,"log_start":"244'1300","ondisk_log_start":"244'1300","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":8850,"log_dups_size":0,"ondisk_log_size":8850,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":191758336,"num_objects":2926,"num_object_clones":0,"num_object_copies":8778,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2926,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10150,"num_write_kb":418432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":231,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-01T10:59:16.947310+0000","last_change":"2023-10-01T10:37:42.762524+0000","last_active":"2023-10-01T10:59:16.947310+0000","last_peered":"2023-10-01T10:59:16.947310+0000","last_clean":"2023-10-01T10:59:16.947310+0000","last_became_active":"2023-10-01T10:37:42.762209+0000","last_became_peered":"2023-10-01T10:37:42.762209+0000","last_unstale":"2023-10-01T10:59:16.947310+0000","last_undegraded":"2023-10-01T10:59:16.947310+0000","last_fullsized":"2023-10-01T10:59:16.947310+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:45:38.171397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":148,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-01T10:59:16.954184+0000","last_change":"2023-10-01T10:56:14.621343+0000","last_active":"2023-10-01T10:59:16.954184+0000","last_peered":"2023-10-01T10:59:16.954184+0000","last_clean":"2023-10-01T10:59:16.954184+0000","last_became_active":"2023-10-01T10:56:14.620940+0000","last_became_peered":"2023-10-01T10:56:14.620940+0000","last_unstale":"2023-10-01T10:59:16.954184+0000","last_undegraded":"2023-10-01T10:59:16.954184+0000","last_fullsized":"2023-10-01T10:59:16.954184+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:43:39.290894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":139,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-01T10:59:16.954555+0000","last_change":"2023-10-01T10:52:18.961040+0000","last_active":"2023-10-01T10:59:16.954555+0000","last_peered":"2023-10-01T10:59:16.954555+0000","last_clean":"2023-10-01T10:59:16.954555+0000","last_became_active":"2023-10-01T10:52:18.960606+0000","last_became_peered":"2023-10-01T10:52:18.960606+0000","last_unstale":"2023-10-01T10:59:16.954555+0000","last_undegraded":"2023-10-01T10:59:16.954555+0000","last_fullsized":"2023-10-01T10:59:16.954555+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.d","version":"252'10073","reported_seq":10095,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-01T10:59:38.500822+0000","last_change":"2023-10-01T10:56:22.015481+0000","last_active":"2023-10-01T10:59:38.500822+0000","last_peered":"2023-10-01T10:59:38.500822+0000","last_clean":"2023-10-01T10:59:38.500822+0000","last_became_active":"2023-10-01T10:56:22.014648+0000","last_became_peered":"2023-10-01T10:56:22.014648+0000","last_unstale":"2023-10-01T10:59:38.500822+0000","last_undegraded":"2023-10-01T10:59:38.500822+0000","last_fullsized":"2023-10-01T10:59:38.500822+0000","mapping_epoch":239,"log_start":"244'1900","ondisk_log_start":"244'1900","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":8173,"log_dups_size":0,"ondisk_log_size":8173,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":194248728,"num_objects":2965,"num_object_clones":0,"num_object_copies":8895,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2965,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10073,"num_write_kb":417153,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":223,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-01T10:59:16.950617+0000","last_change":"2023-10-01T10:37:05.024009+0000","last_active":"2023-10-01T10:59:16.950617+0000","last_peered":"2023-10-01T10:59:16.950617+0000","last_clean":"2023-10-01T10:59:16.950617+0000","last_became_active":"2023-10-01T10:37:05.023458+0000","last_became_peered":"2023-10-01T10:37:05.023458+0000","last_unstale":"2023-10-01T10:59:16.950617+0000","last_undegraded":"2023-10-01T10:59:16.950617+0000","last_fullsized":"2023-10-01T10:59:16.950617+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'139","reported_seq":391,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-01T10:59:16.949186+0000","last_change":"2023-10-01T10:56:17.635571+0000","last_active":"2023-10-01T10:59:16.949186+0000","last_peered":"2023-10-01T10:59:16.949186+0000","last_clean":"2023-10-01T10:59:16.949186+0000","last_became_active":"2023-10-01T10:56:17.635055+0000","last_became_peered":"2023-10-01T10:56:17.635055+0000","last_unstale":"2023-10-01T10:59:16.949186+0000","last_undegraded":"2023-10-01T10:59:16.949186+0000","last_fullsized":"2023-10-01T10:59:16.949186+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:09:17.221082+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:09:17.221082+0000","last_clean_scrub_stamp":"2023-10-01T10:09:17.221082+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:37:54.527937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":222,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-01T10:59:16.945193+0000","last_change":"2023-10-01T10:56:14.622079+0000","last_active":"2023-10-01T10:59:16.945193+0000","last_peered":"2023-10-01T10:59:16.945193+0000","last_clean":"2023-10-01T10:59:16.945193+0000","last_became_active":"2023-10-01T10:56:14.621644+0000","last_became_peered":"2023-10-01T10:56:14.621644+0000","last_unstale":"2023-10-01T10:59:16.945193+0000","last_undegraded":"2023-10-01T10:59:16.945193+0000","last_fullsized":"2023-10-01T10:59:16.945193+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:32:47.233878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":209,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-01T10:59:16.936065+0000","last_change":"2023-10-01T10:56:14.622469+0000","last_active":"2023-10-01T10:59:16.936065+0000","last_peered":"2023-10-01T10:59:16.936065+0000","last_clean":"2023-10-01T10:59:16.936065+0000","last_became_active":"2023-10-01T10:56:14.622092+0000","last_became_peered":"2023-10-01T10:56:14.622092+0000","last_unstale":"2023-10-01T10:59:16.936065+0000","last_undegraded":"2023-10-01T10:59:16.936065+0000","last_fullsized":"2023-10-01T10:59:16.936065+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:43:56.657361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.e","version":"252'10015","reported_seq":10038,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-01T10:59:38.471666+0000","last_change":"2023-10-01T10:56:21.663535+0000","last_active":"2023-10-01T10:59:38.471666+0000","last_peered":"2023-10-01T10:59:38.471666+0000","last_clean":"2023-10-01T10:59:38.471666+0000","last_became_active":"2023-10-01T10:56:21.663135+0000","last_became_peered":"2023-10-01T10:56:21.663135+0000","last_unstale":"2023-10-01T10:59:38.471666+0000","last_undegraded":"2023-10-01T10:59:38.471666+0000","last_fullsized":"2023-10-01T10:59:38.471666+0000","mapping_epoch":239,"log_start":"244'600","ondisk_log_start":"244'600","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":9415,"log_dups_size":0,"ondisk_log_size":9415,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:11:25.118806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":194445312,"num_objects":2967,"num_object_clones":0,"num_object_copies":8901,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2967,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10015,"num_write_kb":415424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":162,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-01T10:59:16.950147+0000","last_change":"2023-10-01T10:52:43.661009+0000","last_active":"2023-10-01T10:59:16.950147+0000","last_peered":"2023-10-01T10:59:16.950147+0000","last_clean":"2023-10-01T10:59:16.950147+0000","last_became_active":"2023-10-01T10:52:43.660679+0000","last_became_peered":"2023-10-01T10:52:43.660679+0000","last_unstale":"2023-10-01T10:59:16.950147+0000","last_undegraded":"2023-10-01T10:59:16.950147+0000","last_fullsized":"2023-10-01T10:59:16.950147+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:32:21.336835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":232,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-01T10:59:23.965323+0000","last_change":"2023-10-01T10:38:16.021968+0000","last_active":"2023-10-01T10:59:23.965323+0000","last_peered":"2023-10-01T10:59:23.965323+0000","last_clean":"2023-10-01T10:59:23.965323+0000","last_became_active":"2023-10-01T10:38:16.021539+0000","last_became_peered":"2023-10-01T10:38:16.021539+0000","last_unstale":"2023-10-01T10:59:23.965323+0000","last_undegraded":"2023-10-01T10:59:23.965323+0000","last_fullsized":"2023-10-01T10:59:23.965323+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:12:53.105747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":232,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-01T10:59:16.943727+0000","last_change":"2023-10-01T10:37:05.017431+0000","last_active":"2023-10-01T10:59:16.943727+0000","last_peered":"2023-10-01T10:59:16.943727+0000","last_clean":"2023-10-01T10:59:16.943727+0000","last_became_active":"2023-10-01T10:37:05.017094+0000","last_became_peered":"2023-10-01T10:37:05.017094+0000","last_unstale":"2023-10-01T10:59:16.943727+0000","last_undegraded":"2023-10-01T10:59:16.943727+0000","last_fullsized":"2023-10-01T10:59:16.943727+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:10:40.664686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.b","version":"252'10075","reported_seq":10098,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-01T10:59:37.864243+0000","last_change":"2023-10-01T10:56:21.667103+0000","last_active":"2023-10-01T10:59:37.864243+0000","last_peered":"2023-10-01T10:59:37.864243+0000","last_clean":"2023-10-01T10:59:37.864243+0000","last_became_active":"2023-10-01T10:56:21.666784+0000","last_became_peered":"2023-10-01T10:56:21.666784+0000","last_unstale":"2023-10-01T10:59:37.864243+0000","last_undegraded":"2023-10-01T10:59:37.864243+0000","last_fullsized":"2023-10-01T10:59:37.864243+0000","mapping_epoch":239,"log_start":"244'2300","ondisk_log_start":"244'2300","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":7775,"log_dups_size":0,"ondisk_log_size":7775,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:45:11.964390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":194707456,"num_objects":2971,"num_object_clones":0,"num_object_copies":8913,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2971,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10075,"num_write_kb":417472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":237,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-01T10:59:16.946625+0000","last_change":"2023-10-01T10:38:16.032774+0000","last_active":"2023-10-01T10:59:16.946625+0000","last_peered":"2023-10-01T10:59:16.946625+0000","last_clean":"2023-10-01T10:59:16.946625+0000","last_became_active":"2023-10-01T10:38:16.032283+0000","last_became_peered":"2023-10-01T10:38:16.032283+0000","last_unstale":"2023-10-01T10:59:16.946625+0000","last_undegraded":"2023-10-01T10:59:16.946625+0000","last_fullsized":"2023-10-01T10:59:16.946625+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":130,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-01T10:59:16.943965+0000","last_change":"2023-10-01T10:52:18.964479+0000","last_active":"2023-10-01T10:59:16.943965+0000","last_peered":"2023-10-01T10:59:16.943965+0000","last_clean":"2023-10-01T10:59:16.943965+0000","last_became_active":"2023-10-01T10:52:18.963881+0000","last_became_peered":"2023-10-01T10:52:18.963881+0000","last_unstale":"2023-10-01T10:59:16.943965+0000","last_undegraded":"2023-10-01T10:59:16.943965+0000","last_fullsized":"2023-10-01T10:59:16.943965+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:36:30.405755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":203,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-01T10:59:23.966795+0000","last_change":"2023-10-01T10:28:20.220812+0000","last_active":"2023-10-01T10:59:23.966795+0000","last_peered":"2023-10-01T10:59:23.966795+0000","last_clean":"2023-10-01T10:59:23.966795+0000","last_became_active":"2023-10-01T10:28:20.215962+0000","last_became_peered":"2023-10-01T10:28:20.215962+0000","last_unstale":"2023-10-01T10:59:23.966795+0000","last_undegraded":"2023-10-01T10:59:23.966795+0000","last_fullsized":"2023-10-01T10:59:23.966795+0000","mapping_epoch":86,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":87,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:35:15.275097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.8","version":"252'10061","reported_seq":10084,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-01T10:59:37.863647+0000","last_change":"2023-10-01T10:56:21.664695+0000","last_active":"2023-10-01T10:59:37.863647+0000","last_peered":"2023-10-01T10:59:37.863647+0000","last_clean":"2023-10-01T10:59:37.863647+0000","last_became_active":"2023-10-01T10:56:21.664315+0000","last_became_peered":"2023-10-01T10:56:21.664315+0000","last_unstale":"2023-10-01T10:59:37.863647+0000","last_undegraded":"2023-10-01T10:59:37.863647+0000","last_fullsized":"2023-10-01T10:59:37.863647+0000","mapping_epoch":239,"log_start":"244'2300","ondisk_log_start":"244'2300","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":7761,"log_dups_size":0,"ondisk_log_size":7761,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:27:54.090426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":191561728,"num_objects":2923,"num_object_clones":0,"num_object_copies":8769,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2923,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10061,"num_write_kb":415488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":209,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-01T10:59:22.828775+0000","last_change":"2023-10-01T10:37:42.773999+0000","last_active":"2023-10-01T10:59:22.828775+0000","last_peered":"2023-10-01T10:59:22.828775+0000","last_clean":"2023-10-01T10:59:22.828775+0000","last_became_active":"2023-10-01T10:37:42.773355+0000","last_became_peered":"2023-10-01T10:37:42.773355+0000","last_unstale":"2023-10-01T10:59:22.828775+0000","last_undegraded":"2023-10-01T10:59:22.828775+0000","last_fullsized":"2023-10-01T10:59:22.828775+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:46:51.317151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":200,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-01T10:59:16.936272+0000","last_change":"2023-10-01T10:56:14.623768+0000","last_active":"2023-10-01T10:59:16.936272+0000","last_peered":"2023-10-01T10:59:16.936272+0000","last_clean":"2023-10-01T10:59:16.936272+0000","last_became_active":"2023-10-01T10:56:14.623353+0000","last_became_peered":"2023-10-01T10:56:14.623353+0000","last_unstale":"2023-10-01T10:59:16.936272+0000","last_undegraded":"2023-10-01T10:59:16.936272+0000","last_fullsized":"2023-10-01T10:59:16.936272+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:53:19.776540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":206,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-01T10:59:16.939587+0000","last_change":"2023-10-01T10:37:42.773538+0000","last_active":"2023-10-01T10:59:16.939587+0000","last_peered":"2023-10-01T10:59:16.939587+0000","last_clean":"2023-10-01T10:59:16.939587+0000","last_became_active":"2023-10-01T10:37:42.771863+0000","last_became_peered":"2023-10-01T10:37:42.771863+0000","last_unstale":"2023-10-01T10:59:16.939587+0000","last_undegraded":"2023-10-01T10:59:16.939587+0000","last_fullsized":"2023-10-01T10:59:16.939587+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:46:46.698627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"252'9899","reported_seq":9922,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-01T10:59:39.182613+0000","last_change":"2023-10-01T10:56:21.667837+0000","last_active":"2023-10-01T10:59:39.182613+0000","last_peered":"2023-10-01T10:59:39.182613+0000","last_clean":"2023-10-01T10:59:39.182613+0000","last_became_active":"2023-10-01T10:56:21.667513+0000","last_became_peered":"2023-10-01T10:56:21.667513+0000","last_unstale":"2023-10-01T10:59:39.182613+0000","last_undegraded":"2023-10-01T10:59:39.182613+0000","last_fullsized":"2023-10-01T10:59:39.182613+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":239,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:56:20.633482+0000","last_clean_scrub_stamp":"2023-10-01T10:56:20.633482+0000","objects_scrubbed":0,"log_size":9899,"log_dups_size":0,"ondisk_log_size":9899,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:25:17.334326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188153856,"num_objects":2871,"num_object_clones":0,"num_object_copies":8613,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2871,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9899,"num_write_kb":408640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":27,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-01T10:59:16.949135+0000","last_change":"2023-10-01T10:56:14.621086+0000","last_active":"2023-10-01T10:59:16.949135+0000","last_peered":"2023-10-01T10:59:16.949135+0000","last_clean":"2023-10-01T10:59:16.949135+0000","last_became_active":"2023-10-01T10:56:14.620638+0000","last_became_peered":"2023-10-01T10:56:14.620638+0000","last_unstale":"2023-10-01T10:59:16.949135+0000","last_undegraded":"2023-10-01T10:59:16.949135+0000","last_fullsized":"2023-10-01T10:59:16.949135+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":235,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-01T10:59:16.943287+0000","last_change":"2023-10-01T10:38:16.027472+0000","last_active":"2023-10-01T10:59:16.943287+0000","last_peered":"2023-10-01T10:59:16.943287+0000","last_clean":"2023-10-01T10:59:16.943287+0000","last_became_active":"2023-10-01T10:38:16.026810+0000","last_became_peered":"2023-10-01T10:38:16.026810+0000","last_unstale":"2023-10-01T10:59:16.943287+0000","last_undegraded":"2023-10-01T10:59:16.943287+0000","last_fullsized":"2023-10-01T10:59:16.943287+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:30:06.978187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":218,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-01T10:59:16.935834+0000","last_change":"2023-10-01T10:56:14.626521+0000","last_active":"2023-10-01T10:59:16.935834+0000","last_peered":"2023-10-01T10:59:16.935834+0000","last_clean":"2023-10-01T10:59:16.935834+0000","last_became_active":"2023-10-01T10:56:14.626107+0000","last_became_peered":"2023-10-01T10:56:14.626107+0000","last_unstale":"2023-10-01T10:59:16.935834+0000","last_undegraded":"2023-10-01T10:59:16.935834+0000","last_fullsized":"2023-10-01T10:59:16.935834+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:44:12.588719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":218,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-01T10:59:16.953684+0000","last_change":"2023-10-01T10:38:16.028491+0000","last_active":"2023-10-01T10:59:16.953684+0000","last_peered":"2023-10-01T10:59:16.953684+0000","last_clean":"2023-10-01T10:59:16.953684+0000","last_became_active":"2023-10-01T10:38:16.028175+0000","last_became_peered":"2023-10-01T10:38:16.028175+0000","last_unstale":"2023-10-01T10:59:16.953684+0000","last_undegraded":"2023-10-01T10:59:16.953684+0000","last_fullsized":"2023-10-01T10:59:16.953684+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:44:29.083129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":240,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-01T10:59:23.979041+0000","last_change":"2023-10-01T10:37:42.762360+0000","last_active":"2023-10-01T10:59:23.979041+0000","last_peered":"2023-10-01T10:59:23.979041+0000","last_clean":"2023-10-01T10:59:23.979041+0000","last_became_active":"2023-10-01T10:37:42.761936+0000","last_became_peered":"2023-10-01T10:37:42.761936+0000","last_unstale":"2023-10-01T10:59:23.979041+0000","last_undegraded":"2023-10-01T10:59:23.979041+0000","last_fullsized":"2023-10-01T10:59:23.979041+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:50.637582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":236,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-01T10:59:16.951193+0000","last_change":"2023-10-01T10:38:16.037809+0000","last_active":"2023-10-01T10:59:16.951193+0000","last_peered":"2023-10-01T10:59:16.951193+0000","last_clean":"2023-10-01T10:59:16.951193+0000","last_became_active":"2023-10-01T10:38:16.037318+0000","last_became_peered":"2023-10-01T10:38:16.037318+0000","last_unstale":"2023-10-01T10:59:16.951193+0000","last_undegraded":"2023-10-01T10:59:16.951193+0000","last_fullsized":"2023-10-01T10:59:16.951193+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":248,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-01T10:59:16.936348+0000","last_change":"2023-10-01T10:38:16.034282+0000","last_active":"2023-10-01T10:59:16.936348+0000","last_peered":"2023-10-01T10:59:16.936348+0000","last_clean":"2023-10-01T10:59:16.936348+0000","last_became_active":"2023-10-01T10:38:16.033820+0000","last_became_peered":"2023-10-01T10:38:16.033820+0000","last_unstale":"2023-10-01T10:59:16.936348+0000","last_undegraded":"2023-10-01T10:59:16.936348+0000","last_fullsized":"2023-10-01T10:59:16.936348+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:48:23.815832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":247,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-01T10:59:16.950641+0000","last_change":"2023-10-01T10:56:14.636784+0000","last_active":"2023-10-01T10:59:16.950641+0000","last_peered":"2023-10-01T10:59:16.950641+0000","last_clean":"2023-10-01T10:59:16.950641+0000","last_became_active":"2023-10-01T10:56:14.624338+0000","last_became_peered":"2023-10-01T10:56:14.624338+0000","last_unstale":"2023-10-01T10:59:16.950641+0000","last_undegraded":"2023-10-01T10:59:16.950641+0000","last_fullsized":"2023-10-01T10:59:16.950641+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":248,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-01T10:59:16.936957+0000","last_change":"2023-10-01T10:38:16.025600+0000","last_active":"2023-10-01T10:59:16.936957+0000","last_peered":"2023-10-01T10:59:16.936957+0000","last_clean":"2023-10-01T10:59:16.936957+0000","last_became_active":"2023-10-01T10:38:16.025252+0000","last_became_peered":"2023-10-01T10:38:16.025252+0000","last_unstale":"2023-10-01T10:59:16.936957+0000","last_undegraded":"2023-10-01T10:59:16.936957+0000","last_fullsized":"2023-10-01T10:59:16.936957+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:24:18.871963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":255,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-01T10:59:16.936541+0000","last_change":"2023-10-01T10:56:14.621544+0000","last_active":"2023-10-01T10:59:16.936541+0000","last_peered":"2023-10-01T10:59:16.936541+0000","last_clean":"2023-10-01T10:59:16.936541+0000","last_became_active":"2023-10-01T10:56:14.621134+0000","last_became_peered":"2023-10-01T10:56:14.621134+0000","last_unstale":"2023-10-01T10:59:16.936541+0000","last_undegraded":"2023-10-01T10:59:16.936541+0000","last_fullsized":"2023-10-01T10:59:16.936541+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:48:44.436158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":249,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-01T10:59:23.960977+0000","last_change":"2023-10-01T10:37:42.765142+0000","last_active":"2023-10-01T10:59:23.960977+0000","last_peered":"2023-10-01T10:59:23.960977+0000","last_clean":"2023-10-01T10:59:23.960977+0000","last_became_active":"2023-10-01T10:37:42.764770+0000","last_became_peered":"2023-10-01T10:37:42.764770+0000","last_unstale":"2023-10-01T10:59:23.960977+0000","last_undegraded":"2023-10-01T10:59:23.960977+0000","last_fullsized":"2023-10-01T10:59:23.960977+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:23:33.527876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":221,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-10-01T10:59:23.966509+0000","last_change":"2023-10-01T10:37:05.021621+0000","last_active":"2023-10-01T10:59:23.966509+0000","last_peered":"2023-10-01T10:59:23.966509+0000","last_clean":"2023-10-01T10:59:23.966509+0000","last_became_active":"2023-10-01T10:37:05.021173+0000","last_became_peered":"2023-10-01T10:37:05.021173+0000","last_unstale":"2023-10-01T10:59:23.966509+0000","last_undegraded":"2023-10-01T10:59:23.966509+0000","last_fullsized":"2023-10-01T10:59:23.966509+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:13:25.364499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":232,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-10-01T10:59:16.935594+0000","last_change":"2023-10-01T10:56:14.621791+0000","last_active":"2023-10-01T10:59:16.935594+0000","last_peered":"2023-10-01T10:59:16.935594+0000","last_clean":"2023-10-01T10:59:16.935594+0000","last_became_active":"2023-10-01T10:56:14.621443+0000","last_became_peered":"2023-10-01T10:56:14.621443+0000","last_unstale":"2023-10-01T10:59:16.935594+0000","last_undegraded":"2023-10-01T10:59:16.935594+0000","last_fullsized":"2023-10-01T10:59:16.935594+0000","mapping_epoch":234,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":235,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:13:58.651222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":3065184280,"num_objects":46772,"num_object_clones":0,"num_object_copies":140316,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":46772,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":159486,"num_write_kb":6600193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9217585152,"data_stored":9217572936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":142086,"ondisk_log_size":142086,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":9608,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30319,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430772,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2222860,"kb_used_data":1328516,"kb_used_omap":0,"kb_used_meta":894336,"kb_avail":91522292,"statfs":{"total":95995035648,"available":93718827008,"internally_reserved":0,"allocated":1360400384,"data_stored":1359847723,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":915800064},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:59:25 2023","interfaces":[{"interface":"back","average":{"1min":1.9950000000000001,"5min":1.4470000000000001,"15min":2.4449999999999998},"min":{"1min":0.28000000000000003,"5min":0.25700000000000001,"15min":0.252},"max":{"1min":11.166,"5min":11.166,"15min":137.34200000000001},"last":0.57799999999999996},{"interface":"front","average":{"1min":2.1309999999999998,"5min":1.476,"15min":2.411},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.307},"max":{"1min":11.18,"5min":11.18,"15min":137.39500000000001},"last":0.56799999999999995}]},{"osd":1,"last update":"Sun Oct 1 10:59:25 2023","interfaces":[{"interface":"back","average":{"1min":1.2749999999999999,"5min":1.3069999999999999,"15min":1.6859999999999999},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":3.73,"5min":9.4160000000000004,"15min":56.110999999999997},"last":0.504},{"interface":"front","average":{"1min":1.623,"5min":1.2969999999999999,"15min":1.673},"min":{"1min":0.36799999999999999,"5min":0.33000000000000002,"15min":0.27100000000000002},"max":{"1min":5.4009999999999998,"5min":5.4009999999999998,"15min":51.258000000000003},"last":0.374}]},{"osd":2,"last update":"Sun Oct 1 10:58:40 2023","interfaces":[{"interface":"back","average":{"1min":4.484,"5min":3.1760000000000002,"15min":2.0739999999999998},"min":{"1min":0.36499999999999999,"5min":0.33300000000000002,"15min":0.315},"max":{"1min":39.658000000000001,"5min":42.804000000000002,"15min":42.804000000000002},"last":0.40999999999999998},{"interface":"front","average":{"1min":3.4980000000000002,"5min":2.8370000000000002,"15min":1.883},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":42.072000000000003,"5min":42.819000000000003,"15min":42.819000000000003},"last":4.242}]},{"osd":3,"last update":"Sun Oct 1 10:59:25 2023","interfaces":[{"interface":"back","average":{"1min":2.6120000000000001,"5min":1.7230000000000001,"15min":1.9039999999999999},"min":{"1min":0.433,"5min":0.38200000000000001,"15min":0.318},"max":{"1min":9.157,"5min":9.157,"15min":51.334000000000003},"last":0.51500000000000001},{"interface":"front","average":{"1min":1.6319999999999999,"5min":1.53,"15min":1.788},"min":{"1min":0.36399999999999999,"5min":0.34699999999999998,"15min":0.313},"max":{"1min":8.516,"5min":11.244999999999999,"15min":51.404000000000003},"last":0.28299999999999997}]},{"osd":4,"last update":"Sun Oct 1 10:58:47 2023","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":1.3460000000000001,"15min":1.4099999999999999},"min":{"1min":0.433,"5min":0.40500000000000003,"15min":0.33500000000000002},"max":{"1min":3.6890000000000001,"5min":7.7009999999999996,"15min":11.081},"last":0.436},{"interface":"front","average":{"1min":1.3799999999999999,"5min":1.5029999999999999,"15min":1.5049999999999999},"min":{"1min":0.44900000000000001,"5min":0.43099999999999999,"15min":0.35699999999999998},"max":{"1min":5.5499999999999998,"5min":8.7370000000000001,"15min":12.292},"last":0.55200000000000005}]},{"osd":5,"last update":"Sun Oct 1 10:59:25 2023","interfaces":[{"interface":"back","average":{"1min":2.0880000000000001,"5min":1.4470000000000001,"15min":1.7969999999999999},"min":{"1min":0.435,"5min":0.38900000000000001,"15min":0.34599999999999997},"max":{"1min":12.291,"5min":12.291,"15min":40.381},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.98499999999999999,"5min":1.1950000000000001,"15min":1.6180000000000001},"min":{"1min":0.5,"5min":0.39200000000000002,"15min":0.32200000000000001},"max":{"1min":3.8109999999999999,"5min":7.4409999999999998,"15min":39.590000000000003},"last":0.39600000000000002}]},{"osd":6,"last update":"Sun Oct 1 10:59:25 2023","interfaces":[{"interface":"back","average":{"1min":1.492,"5min":1.3799999999999999,"15min":1.5649999999999999},"min":{"1min":0.47399999999999998,"5min":0.35299999999999998,"15min":0.34499999999999997},"max":{"1min":6.0830000000000002,"5min":6.0830000000000002,"15min":12.329000000000001},"last":0.48399999999999999},{"interface":"front","average":{"1min":1.3340000000000001,"5min":1.452,"15min":1.5580000000000001},"min":{"1min":0.46999999999999997,"5min":0.378,"15min":0.37},"max":{"1min":4.5119999999999996,"5min":8.0120000000000005,"15min":9.4329999999999998},"last":0.47599999999999998}]}]},{"osd":6,"up_from":42,"seq":180388627006,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2201336,"kb_used_data":1304240,"kb_used_omap":0,"kb_used_meta":897088,"kb_avail":91543816,"statfs":{"total":95995035648,"available":93740867584,"internally_reserved":0,"allocated":1335541760,"data_stored":1335009094,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":918618112},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,3,0,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:59:16 2023","interfaces":[{"interface":"back","average":{"1min":1.3049999999999999,"5min":1.371,"15min":2.746},"min":{"1min":0.28399999999999997,"5min":0.27900000000000003,"15min":0.25},"max":{"1min":7.0179999999999998,"5min":8.2899999999999991,"15min":140.524},"last":3.5529999999999999},{"interface":"front","average":{"1min":1.4470000000000001,"5min":1.4350000000000001,"15min":2.7719999999999998},"min":{"1min":0.314,"5min":0.29399999999999998,"15min":0.249},"max":{"1min":7.4359999999999999,"5min":7.4630000000000001,"15min":140.86500000000001},"last":0.69499999999999995}]},{"osd":1,"last update":"Sun Oct 1 10:59:28 2023","interfaces":[{"interface":"back","average":{"1min":1.9039999999999999,"5min":1.532,"15min":1.536},"min":{"1min":0.39300000000000002,"5min":0.27100000000000002,"15min":0.26000000000000001},"max":{"1min":7.1070000000000002,"5min":9.843,"15min":9.843},"last":2.9830000000000001},{"interface":"front","average":{"1min":1.3280000000000001,"5min":1.252,"15min":1.591},"min":{"1min":0.376,"5min":0.32700000000000001,"15min":0.26300000000000001},"max":{"1min":4.0940000000000003,"5min":7.0380000000000003,"15min":12.968999999999999},"last":2.8759999999999999}]},{"osd":2,"last update":"Sun Oct 1 10:58:37 2023","interfaces":[{"interface":"back","average":{"1min":8.4619999999999997,"5min":4.6289999999999996,"15min":3.4340000000000002},"min":{"1min":0.34799999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":106.705,"5min":122.768,"15min":122.768},"last":1.2609999999999999},{"interface":"front","average":{"1min":9.0099999999999998,"5min":5.2279999999999998,"15min":3.6059999999999999},"min":{"1min":0.44500000000000001,"5min":0.36199999999999999,"15min":0.32100000000000001},"max":{"1min":106.806,"5min":123.02200000000001,"15min":123.02200000000001},"last":2.9670000000000001}]},{"osd":3,"last update":"Sun Oct 1 10:59:28 2023","interfaces":[{"interface":"back","average":{"1min":3.101,"5min":2.0449999999999999,"15min":2.056},"min":{"1min":0.51400000000000001,"5min":0.31,"15min":0.30599999999999999},"max":{"1min":7.8109999999999999,"5min":36.698999999999998,"15min":36.698999999999998},"last":3.891},{"interface":"front","average":{"1min":3.1819999999999999,"5min":2.306,"15min":2.1720000000000002},"min":{"1min":0.52400000000000002,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":8.8350000000000009,"5min":33.555999999999997,"15min":33.555999999999997},"last":1.383}]},{"osd":4,"last update":"Sun Oct 1 10:59:03 2023","interfaces":[{"interface":"back","average":{"1min":1.306,"5min":1.123,"15min":1.5640000000000001},"min":{"1min":0.40999999999999998,"5min":0.35799999999999998,"15min":0.32200000000000001},"max":{"1min":5.8979999999999997,"5min":7.5369999999999999,"15min":13.1},"last":1.218},{"interface":"front","average":{"1min":1.6839999999999999,"5min":1.7190000000000001,"15min":1.708},"min":{"1min":0.47599999999999998,"5min":0.38700000000000001,"15min":0.315},"max":{"1min":6.3490000000000002,"5min":8.016,"15min":12.157999999999999},"last":1.7649999999999999}]},{"osd":5,"last update":"Sun Oct 1 10:59:28 2023","interfaces":[{"interface":"back","average":{"1min":1.2849999999999999,"5min":1.246,"15min":1.5449999999999999},"min":{"1min":0.46200000000000002,"5min":0.34200000000000003,"15min":0.33000000000000002},"max":{"1min":7.0739999999999998,"5min":15.238,"15min":15.238},"last":1.04},{"interface":"front","average":{"1min":1.5249999999999999,"5min":1.4399999999999999,"15min":1.853},"min":{"1min":0.438,"5min":0.36499999999999999,"15min":0.308},"max":{"1min":4.4050000000000002,"5min":13.295,"15min":14.459},"last":1.327}]},{"osd":7,"last update":"Sun Oct 1 10:59:16 2023","interfaces":[{"interface":"back","average":{"1min":1.395,"5min":1.288,"15min":1.6890000000000001},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.311},"max":{"1min":4.5069999999999997,"5min":6.04,"15min":13.276999999999999},"last":1.698},{"interface":"front","average":{"1min":1.3700000000000001,"5min":1.419,"15min":1.6890000000000001},"min":{"1min":0.43099999999999999,"5min":0.38500000000000001,"15min":0.35799999999999998},"max":{"1min":7.1349999999999998,"5min":7.1379999999999999,"15min":11.416},"last":1.1120000000000001}]}]},{"osd":1,"up_from":101,"seq":433791697223,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1635992,"kb_used_data":971088,"kb_used_omap":4,"kb_used_meta":664891,"kb_avail":92109160,"statfs":{"total":95995035648,"available":94319779840,"internally_reserved":0,"allocated":994394112,"data_stored":993370515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":680849294},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:59:33 2023","interfaces":[{"interface":"back","average":{"1min":1.627,"5min":1.4079999999999999,"15min":1.4159999999999999},"min":{"1min":0.28299999999999997,"5min":0.222,"15min":0.19900000000000001},"max":{"1min":11.055999999999999,"5min":12.888999999999999,"15min":37.058},"last":0.45600000000000002},{"interface":"front","average":{"1min":2.0579999999999998,"5min":1.5449999999999999,"15min":1.575},"min":{"1min":0.25,"5min":0.20699999999999999,"15min":0.192},"max":{"1min":4.819,"5min":15.416,"15min":37.337000000000003},"last":0.79200000000000004}]},{"osd":2,"last update":"Sun Oct 1 10:58:47 2023","interfaces":[{"interface":"back","average":{"1min":2.5009999999999999,"5min":2.0150000000000001,"15min":2.3610000000000002},"min":{"1min":0.252,"5min":0.252,"15min":0.222},"max":{"1min":10.522,"5min":19.149999999999999,"15min":161.905},"last":1.048},{"interface":"front","average":{"1min":2.6960000000000002,"5min":2.1030000000000002,"15min":2.3610000000000002},"min":{"1min":0.40999999999999998,"5min":0.26900000000000002,"15min":0.23699999999999999},"max":{"1min":9.3930000000000007,"5min":18.815000000000001,"15min":162.059},"last":0.95599999999999996}]},{"osd":3,"last update":"Sun Oct 1 10:59:33 2023","interfaces":[{"interface":"back","average":{"1min":2.5779999999999998,"5min":1.5680000000000001,"15min":1.974},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.20000000000000001},"max":{"1min":17.681999999999999,"5min":17.681999999999999,"15min":143.73400000000001},"last":2.02},{"interface":"front","average":{"1min":2.2480000000000002,"5min":1.6359999999999999,"15min":2.113},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":10.983000000000001,"5min":10.983000000000001,"15min":143.00399999999999},"last":1.0740000000000001}]},{"osd":4,"last update":"Sun Oct 1 10:58:47 2023","interfaces":[{"interface":"back","average":{"1min":1.2989999999999999,"5min":1.3180000000000001,"15min":1.3520000000000001},"min":{"1min":0.34599999999999997,"5min":0.307,"15min":0.29199999999999998},"max":{"1min":5.1070000000000002,"5min":8.593,"15min":10.004},"last":0.55600000000000005},{"interface":"front","average":{"1min":1.2170000000000001,"5min":1.3029999999999999,"15min":1.53},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.32700000000000001},"max":{"1min":4.4470000000000001,"5min":8.7530000000000001,"15min":13.122999999999999},"last":0.621}]},{"osd":5,"last update":"Sun Oct 1 10:59:33 2023","interfaces":[{"interface":"back","average":{"1min":1.258,"5min":1.282,"15min":1.5700000000000001},"min":{"1min":0.40600000000000003,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":5.7359999999999998,"5min":7.7190000000000003,"15min":9.1809999999999992},"last":0.41099999999999998},{"interface":"front","average":{"1min":1.266,"5min":1.161,"15min":1.4870000000000001},"min":{"1min":0.38800000000000001,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":3.7170000000000001,"5min":6.75,"15min":10.865},"last":0.59199999999999997}]},{"osd":6,"last update":"Sun Oct 1 10:59:33 2023","interfaces":[{"interface":"back","average":{"1min":1.861,"5min":1.452,"15min":1.6599999999999999},"min":{"1min":0.40400000000000003,"5min":0.38400000000000001,"15min":0.371},"max":{"1min":7.4509999999999996,"5min":7.4509999999999996,"15min":10.933},"last":0.49299999999999999},{"interface":"front","average":{"1min":1.778,"5min":1.319,"15min":1.5740000000000001},"min":{"1min":0.38,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":6.1609999999999996,"5min":6.1609999999999996,"15min":13.195},"last":2.6819999999999999}]},{"osd":7,"last update":"Sun Oct 1 10:59:33 2023","interfaces":[{"interface":"back","average":{"1min":1.9970000000000001,"5min":1.3440000000000001,"15min":1.575},"min":{"1min":0.41499999999999998,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":4.4900000000000002,"5min":5.2409999999999997,"15min":13.164999999999999},"last":0.65000000000000002},{"interface":"front","average":{"1min":2.6259999999999999,"5min":1.49,"15min":1.704},"min":{"1min":0.53200000000000003,"5min":0.42899999999999999,"15min":0.38100000000000001},"max":{"1min":11.689,"5min":11.689,"15min":11.689},"last":1.0069999999999999}]}]},{"osd":0,"up_from":86,"seq":369367187834,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1949292,"kb_used_data":1143524,"kb_used_omap":5,"kb_used_meta":805754,"kb_avail":91795860,"statfs":{"total":95995035648,"available":93998960640,"internally_reserved":0,"allocated":1170968576,"data_stored":1169983393,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":825092730},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,0,0,1,1,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 10:59:29 2023","interfaces":[{"interface":"back","average":{"1min":1.9690000000000001,"5min":1.365,"15min":1.3580000000000001},"min":{"1min":0.28399999999999997,"5min":0.19700000000000001,"15min":0.193},"max":{"1min":9.2889999999999997,"5min":9.2889999999999997,"15min":15.074},"last":0.378},{"interface":"front","average":{"1min":2.5070000000000001,"5min":1.536,"15min":1.524},"min":{"1min":0.22900000000000001,"5min":0.20200000000000001,"15min":0.19900000000000001},"max":{"1min":7.8849999999999998,"5min":10.289,"15min":13.601000000000001},"last":3.577}]},{"osd":2,"last update":"Sun Oct 1 10:58:45 2023","interfaces":[{"interface":"back","average":{"1min":2.9710000000000001,"5min":2.4350000000000001,"15min":1.869},"min":{"1min":0.39300000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":11.108000000000001,"5min":44.744999999999997,"15min":44.744999999999997},"last":0.96699999999999997},{"interface":"front","average":{"1min":3.032,"5min":2.601,"15min":2.0259999999999998},"min":{"1min":0.252,"5min":0.252,"15min":0.221},"max":{"1min":11.073,"5min":44.691000000000003,"15min":44.691000000000003},"last":0.84099999999999997}]},{"osd":3,"last update":"Sun Oct 1 10:59:29 2023","interfaces":[{"interface":"back","average":{"1min":2.7639999999999998,"5min":1.897,"15min":1.738},"min":{"1min":0.317,"5min":0.214,"15min":0.214},"max":{"1min":8.7569999999999997,"5min":13.632999999999999,"15min":13.632999999999999},"last":5.1779999999999999},{"interface":"front","average":{"1min":2.1619999999999999,"5min":1.625,"15min":1.6279999999999999},"min":{"1min":0.25800000000000001,"5min":0.246,"15min":0.246},"max":{"1min":9.423,"5min":12.727,"15min":13.641},"last":0.59999999999999998}]},{"osd":4,"last update":"Sun Oct 1 10:59:07 2023","interfaces":[{"interface":"back","average":{"1min":1.5609999999999999,"5min":1.286,"15min":1.591},"min":{"1min":0.42599999999999999,"5min":0.34300000000000003,"15min":0.32000000000000001},"max":{"1min":6.3890000000000002,"5min":9.2889999999999997,"15min":11.379},"last":0.56499999999999995},{"interface":"front","average":{"1min":1.4790000000000001,"5min":1.2689999999999999,"15min":1.5589999999999999},"min":{"1min":0.35399999999999998,"5min":0.32300000000000001,"15min":0.311},"max":{"1min":5.0019999999999998,"5min":5.3600000000000003,"15min":11.362},"last":3.9079999999999999}]},{"osd":5,"last update":"Sun Oct 1 10:59:29 2023","interfaces":[{"interface":"back","average":{"1min":2.3199999999999998,"5min":1.6779999999999999,"15min":1.6990000000000001},"min":{"1min":0.432,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":6.8369999999999997,"5min":7.6890000000000001,"15min":12.91},"last":0.63100000000000001},{"interface":"front","average":{"1min":1.837,"5min":1.389,"15min":1.534},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.35099999999999998},"max":{"1min":14.913,"5min":14.913,"15min":14.913},"last":0.49099999999999999}]},{"osd":6,"last update":"Sun Oct 1 10:59:29 2023","interfaces":[{"interface":"back","average":{"1min":2.1709999999999998,"5min":1.417,"15min":1.617},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.373},"max":{"1min":9.4570000000000007,"5min":9.4570000000000007,"15min":9.8279999999999994},"last":0.51900000000000002},{"interface":"front","average":{"1min":1.954,"5min":1.4750000000000001,"15min":1.728},"min":{"1min":0.39600000000000002,"5min":0.378,"15min":0.36899999999999999},"max":{"1min":8.0909999999999993,"5min":8.0909999999999993,"15min":10.111000000000001},"last":3.621}]},{"osd":7,"last update":"Sun Oct 1 10:59:29 2023","interfaces":[{"interface":"back","average":{"1min":1.9139999999999999,"5min":1.4910000000000001,"15min":1.6599999999999999},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":9.4819999999999993,"5min":12.680999999999999,"15min":15.839},"last":0.66200000000000003},{"interface":"front","average":{"1min":1.954,"5min":1.6839999999999999,"15min":1.7529999999999999},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":8.1199999999999992,"5min":8.1199999999999992,"15min":8.9440000000000008},"last":3.8700000000000001}]}]},{"osd":2,"up_from":116,"seq":498216206609,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2458328,"kb_used_data":1866960,"kb_used_omap":7,"kb_used_meta":591288,"kb_avail":91286824,"statfs":{"total":95995035648,"available":93477707776,"internally_reserved":0,"allocated":1911767040,"data_stored":1910797468,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":605478993},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,3,5,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:58:47 2023","interfaces":[{"interface":"back","average":{"1min":2.0390000000000001,"5min":2.6349999999999998,"15min":2.5600000000000001},"min":{"1min":0.34499999999999997,"5min":0.20200000000000001,"15min":0.16900000000000001},"max":{"1min":6.3280000000000003,"5min":104.42,"15min":147.87},"last":0.45100000000000001},{"interface":"front","average":{"1min":2.4620000000000002,"5min":2.9929999999999999,"15min":2.7410000000000001},"min":{"1min":0.222,"5min":0.222,"15min":0.20599999999999999},"max":{"1min":8.6400000000000006,"5min":105.10899999999999,"15min":148.03100000000001},"last":0.33200000000000002}]},{"osd":1,"last update":"Sun Oct 1 10:58:47 2023","interfaces":[{"interface":"back","average":{"1min":2.4020000000000001,"5min":1.6419999999999999,"15min":2.0110000000000001},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.217},"max":{"1min":5.9340000000000002,"5min":7.8949999999999996,"15min":122.249},"last":1.292},{"interface":"front","average":{"1min":3.2570000000000001,"5min":2.0910000000000002,"15min":2.4129999999999998},"min":{"1min":0.373,"5min":0.26500000000000001,"15min":0.20599999999999999},"max":{"1min":11.554,"5min":11.554,"15min":121.495},"last":5.7800000000000002}]},{"osd":3,"last update":"Sun Oct 1 10:59:17 2023","interfaces":[{"interface":"back","average":{"1min":1.819,"5min":1.823,"15min":2.1629999999999998},"min":{"1min":0.33700000000000002,"5min":0.26100000000000001,"15min":0.20999999999999999},"max":{"1min":5.8090000000000002,"5min":15.897,"15min":124.393},"last":0.53600000000000003},{"interface":"front","average":{"1min":2.6899999999999999,"5min":2.0680000000000001,"15min":2.3490000000000002},"min":{"1min":0.35299999999999998,"5min":0.26400000000000001,"15min":0.20899999999999999},"max":{"1min":9.2129999999999992,"5min":11.404999999999999,"15min":122.52200000000001},"last":2.25}]},{"osd":4,"last update":"Sun Oct 1 10:58:57 2023","interfaces":[{"interface":"back","average":{"1min":2.4460000000000002,"5min":1.6879999999999999,"15min":1.7709999999999999},"min":{"1min":0.42999999999999999,"5min":0.40400000000000003,"15min":0.36499999999999999},"max":{"1min":6.2400000000000002,"5min":7.8090000000000002,"15min":60.994},"last":1.1910000000000001},{"interface":"front","average":{"1min":2.7010000000000001,"5min":1.905,"15min":2.0950000000000002},"min":{"1min":0.39700000000000002,"5min":0.36599999999999999,"15min":0.30299999999999999},"max":{"1min":7.5129999999999999,"5min":11.574,"15min":63.012},"last":5.7400000000000002}]},{"osd":5,"last update":"Sun Oct 1 10:58:47 2023","interfaces":[{"interface":"back","average":{"1min":2.3900000000000001,"5min":1.7969999999999999,"15min":1.899},"min":{"1min":0.39800000000000002,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":6.2649999999999997,"5min":8.5440000000000005,"15min":61.021000000000001},"last":0.627},{"interface":"front","average":{"1min":2.5219999999999998,"5min":2.1429999999999998,"15min":2.1890000000000001},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.34100000000000003},"max":{"1min":7.4740000000000002,"5min":12.835000000000001,"15min":63.145000000000003},"last":2.2400000000000002}]},{"osd":6,"last update":"Sun Oct 1 10:58:47 2023","interfaces":[{"interface":"back","average":{"1min":3.4380000000000002,"5min":2.4009999999999998,"15min":2.2949999999999999},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.32100000000000001},"max":{"1min":7.7690000000000001,"5min":19.908000000000001,"15min":63.289000000000001},"last":2.2080000000000002},{"interface":"front","average":{"1min":1.742,"5min":1.756,"15min":1.754},"min":{"1min":0.42299999999999999,"5min":0.38300000000000001,"15min":0.35299999999999998},"max":{"1min":6.4020000000000001,"5min":9.2279999999999998,"15min":60.579000000000001},"last":0.51100000000000001}]},{"osd":7,"last update":"Sun Oct 1 10:58:47 2023","interfaces":[{"interface":"back","average":{"1min":2.468,"5min":1.7010000000000001,"15min":1.679},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.34300000000000003},"max":{"1min":11.209,"5min":11.209,"15min":60.493000000000002},"last":0.92400000000000004},{"interface":"front","average":{"1min":2.5179999999999998,"5min":1.821,"15min":1.9039999999999999},"min":{"1min":0.45600000000000002,"5min":0.40100000000000002,"15min":0.36399999999999999},"max":{"1min":6.407,"5min":7.9279999999999999,"15min":60.956000000000003},"last":0.70999999999999996}]}]},{"osd":3,"up_from":121,"seq":519691043081,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2129464,"kb_used_data":1511408,"kb_used_omap":2,"kb_used_meta":618045,"kb_avail":91615688,"statfs":{"total":95995035648,"available":93814464512,"internally_reserved":0,"allocated":1547681792,"data_stored":1546741657,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":632878133},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,3,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:59:25 2023","interfaces":[{"interface":"back","average":{"1min":2.3780000000000001,"5min":2.4470000000000001,"15min":2.7290000000000001},"min":{"1min":0.245,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":5.4009999999999998,"5min":100.191,"15min":129.953},"last":7.7450000000000001},{"interface":"front","average":{"1min":3.3500000000000001,"5min":2.8519999999999999,"15min":3.1299999999999999},"min":{"1min":0.248,"5min":0.19600000000000001,"15min":0.189},"max":{"1min":11.077,"5min":99.852999999999994,"15min":129.53999999999999},"last":4.1710000000000003}]},{"osd":1,"last update":"Sun Oct 1 10:59:25 2023","interfaces":[{"interface":"back","average":{"1min":2.6669999999999998,"5min":1.5309999999999999,"15min":1.4930000000000001},"min":{"1min":0.255,"5min":0.24299999999999999,"15min":0.193},"max":{"1min":8.3650000000000002,"5min":10.574999999999999,"15min":10.574999999999999},"last":4.4560000000000004},{"interface":"front","average":{"1min":2.2730000000000001,"5min":1.589,"15min":1.5429999999999999},"min":{"1min":0.33600000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":5.5220000000000002,"5min":10.685,"15min":13.529999999999999},"last":0.373}]},{"osd":2,"last update":"Sun Oct 1 10:59:25 2023","interfaces":[{"interface":"back","average":{"1min":3.593,"5min":2.2200000000000002,"15min":1.8},"min":{"1min":0.25600000000000001,"5min":0.217,"15min":0.217},"max":{"1min":11.324,"5min":12.238,"15min":15.148},"last":5.141},{"interface":"front","average":{"1min":4.1360000000000001,"5min":2.0369999999999999,"15min":1.698},"min":{"1min":0.40600000000000003,"5min":0.27100000000000002,"15min":0.26900000000000002},"max":{"1min":11.606999999999999,"5min":11.606999999999999,"15min":11.606999999999999},"last":7.7869999999999999}]},{"osd":4,"last update":"Sun Oct 1 10:59:01 2023","interfaces":[{"interface":"back","average":{"1min":2.831,"5min":1.7749999999999999,"15min":1.524},"min":{"1min":0.42999999999999999,"5min":0.36799999999999999,"15min":0.34599999999999997},"max":{"1min":10.808999999999999,"5min":10.808999999999999,"15min":10.808999999999999},"last":0.50700000000000001},{"interface":"front","average":{"1min":2.2869999999999999,"5min":1.7849999999999999,"15min":1.619},"min":{"1min":0.441,"5min":0.34899999999999998,"15min":0.29999999999999999},"max":{"1min":10.962999999999999,"5min":13.391999999999999,"15min":13.391999999999999},"last":3.9489999999999998}]},{"osd":5,"last update":"Sun Oct 1 10:59:25 2023","interfaces":[{"interface":"back","average":{"1min":2.4900000000000002,"5min":1.5309999999999999,"15min":1.5800000000000001},"min":{"1min":0.44400000000000001,"5min":0.34399999999999997,"15min":0.33100000000000002},"max":{"1min":11.050000000000001,"5min":11.050000000000001,"15min":13.333},"last":3.9900000000000002},{"interface":"front","average":{"1min":2.6400000000000001,"5min":2.0379999999999998,"15min":1.7350000000000001},"min":{"1min":0.40899999999999997,"5min":0.40300000000000002,"15min":0.33200000000000002},"max":{"1min":9.0380000000000003,"5min":12.335000000000001,"15min":12.335000000000001},"last":0.58699999999999997}]},{"osd":6,"last update":"Sun Oct 1 10:59:25 2023","interfaces":[{"interface":"back","average":{"1min":2.181,"5min":1.673,"15min":1.8400000000000001},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.34999999999999998},"max":{"1min":11.016999999999999,"5min":11.016999999999999,"15min":11.016999999999999},"last":4.0149999999999997},{"interface":"front","average":{"1min":2.71,"5min":2.1280000000000001,"15min":2.0470000000000002},"min":{"1min":0.5,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":9.0779999999999994,"5min":12.378,"15min":12.378},"last":0.78100000000000003}]},{"osd":7,"last update":"Sun Oct 1 10:59:25 2023","interfaces":[{"interface":"back","average":{"1min":2.7690000000000001,"5min":2.2530000000000001,"15min":1.8069999999999999},"min":{"1min":0.53700000000000003,"5min":0.42799999999999999,"15min":0.376},"max":{"1min":8.9719999999999995,"5min":12.416,"15min":12.416},"last":0.54400000000000004},{"interface":"front","average":{"1min":2.7850000000000001,"5min":1.74,"15min":1.653},"min":{"1min":0.51600000000000001,"5min":0.45500000000000002,"15min":0.35899999999999999},"max":{"1min":6.085,"5min":10.755000000000001,"15min":10.755000000000001},"last":0.442}]}]},{"osd":4,"up_from":126,"seq":541165879555,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":969752,"kb_used_data":550480,"kb_used_omap":10,"kb_used_meta":419253,"kb_avail":92775400,"statfs":{"total":95995035648,"available":95002009600,"internally_reserved":0,"allocated":563691520,"data_stored":562721686,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":429315684},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:58:54 2023","interfaces":[{"interface":"back","average":{"1min":2.915,"5min":1.651,"15min":2.0739999999999998},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.24399999999999999},"max":{"1min":18.484000000000002,"5min":35.741999999999997,"15min":62.576000000000001},"last":1.2230000000000001},{"interface":"front","average":{"1min":2.6099999999999999,"5min":1.7110000000000001,"15min":2.0640000000000001},"min":{"1min":0.317,"5min":0.28799999999999998,"15min":0.251},"max":{"1min":16.966000000000001,"5min":35.701000000000001,"15min":64.941000000000003},"last":5.0209999999999999}]},{"osd":1,"last update":"Sun Oct 1 10:58:54 2023","interfaces":[{"interface":"back","average":{"1min":1.1799999999999999,"5min":1.242,"15min":1.794},"min":{"1min":0.32300000000000001,"5min":0.30199999999999999,"15min":0.27300000000000002},"max":{"1min":3.7629999999999999,"5min":11.664,"15min":97.622},"last":0.47199999999999998},{"interface":"front","average":{"1min":1.4550000000000001,"5min":1.1399999999999999,"15min":1.925},"min":{"1min":0.317,"5min":0.317,"15min":0.28699999999999998},"max":{"1min":4.1079999999999997,"5min":11.914999999999999,"15min":97.346999999999994},"last":0.39400000000000002}]},{"osd":2,"last update":"Sun Oct 1 10:58:54 2023","interfaces":[{"interface":"back","average":{"1min":3.2389999999999999,"5min":3.411,"15min":2.73},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.29899999999999999},"max":{"1min":22.774999999999999,"5min":68.064999999999998,"15min":117.229},"last":0.53700000000000003},{"interface":"front","average":{"1min":2.6549999999999998,"5min":3.3399999999999999,"15min":2.5760000000000001},"min":{"1min":0.36899999999999999,"5min":0.35399999999999998,"15min":0.29699999999999999},"max":{"1min":17.065000000000001,"5min":68.379000000000005,"15min":116.922},"last":0.67400000000000004}]},{"osd":3,"last update":"Sun Oct 1 10:58:54 2023","interfaces":[{"interface":"back","average":{"1min":1.2170000000000001,"5min":1.3360000000000001,"15min":1.8220000000000001},"min":{"1min":0.36399999999999999,"5min":0.32700000000000001,"15min":0.29499999999999998},"max":{"1min":3.976,"5min":9.1780000000000008,"15min":61.398000000000003},"last":0.93799999999999994},{"interface":"front","average":{"1min":1.25,"5min":2.7200000000000002,"15min":2.2589999999999999},"min":{"1min":0.374,"5min":0.308,"15min":0.308},"max":{"1min":4.0389999999999997,"5min":130.88300000000001,"15min":130.88300000000001},"last":3.0409999999999999}]},{"osd":5,"last update":"Sun Oct 1 10:58:54 2023","interfaces":[{"interface":"back","average":{"1min":0.872,"5min":2.0299999999999998,"15min":1.7589999999999999},"min":{"1min":0.39700000000000002,"5min":0.37,"15min":0.30199999999999999},"max":{"1min":3.9580000000000002,"5min":84.307000000000002,"15min":84.307000000000002},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.84399999999999997,"5min":2.2490000000000001,"15min":1.841},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.28999999999999998},"max":{"1min":4.1360000000000001,"5min":84.234999999999999,"15min":84.234999999999999},"last":0.78600000000000003}]},{"osd":6,"last update":"Sun Oct 1 10:58:54 2023","interfaces":[{"interface":"back","average":{"1min":1.1950000000000001,"5min":1.135,"15min":1.6279999999999999},"min":{"1min":0.39200000000000002,"5min":0.34999999999999998,"15min":0.33800000000000002},"max":{"1min":4.4109999999999996,"5min":9.0109999999999992,"15min":13.646000000000001},"last":1.0049999999999999},{"interface":"front","average":{"1min":0.98399999999999999,"5min":1.123,"15min":1.611},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.33400000000000002},"max":{"1min":3.6469999999999998,"5min":7.5510000000000002,"15min":14.446999999999999},"last":0.91900000000000004}]},{"osd":7,"last update":"Sun Oct 1 10:58:54 2023","interfaces":[{"interface":"back","average":{"1min":1.099,"5min":1.2230000000000001,"15min":1.929},"min":{"1min":0.45800000000000002,"5min":0.43099999999999999,"15min":0.33900000000000002},"max":{"1min":4.4240000000000004,"5min":7.5960000000000001,"15min":119.56100000000001},"last":1.147},{"interface":"front","average":{"1min":0.996,"5min":1.0549999999999999,"15min":1.958},"min":{"1min":0.46600000000000003,"5min":0.40300000000000002,"15min":0.30399999999999999},"max":{"1min":4.0659999999999998,"5min":4.0659999999999998,"15min":120.44499999999999},"last":0.57399999999999995}]}]},{"osd":5,"up_from":36,"seq":154618823236,"num_pgs":23,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1278188,"kb_used_data":382116,"kb_used_omap":0,"kb_used_meta":896064,"kb_avail":92466964,"statfs":{"total":95995035648,"available":94686171136,"internally_reserved":0,"allocated":391286784,"data_stored":390761682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":917569536},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 10:59:20 2023","interfaces":[{"interface":"back","average":{"1min":1.452,"5min":1.347,"15min":2.0619999999999998},"min":{"1min":0.27200000000000002,"5min":0.24299999999999999,"15min":0.24099999999999999},"max":{"1min":5.5789999999999997,"5min":13.955,"15min":123.39700000000001},"last":7.8979999999999997},{"interface":"front","average":{"1min":0.98899999999999999,"5min":1.3140000000000001,"15min":2.1829999999999998},"min":{"1min":0.28499999999999998,"5min":0.253,"15min":0.253},"max":{"1min":7.6369999999999996,"5min":13.978,"15min":124.051},"last":0.42499999999999999}]},{"osd":1,"last update":"Sun Oct 1 10:59:20 2023","interfaces":[{"interface":"back","average":{"1min":1.1779999999999999,"5min":1.016,"15min":1.494},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.27100000000000002},"max":{"1min":4.992,"5min":4.992,"15min":52.18},"last":2.3759999999999999},{"interface":"front","average":{"1min":1.113,"5min":1.089,"15min":1.702},"min":{"1min":0.35299999999999998,"5min":0.32800000000000001,"15min":0.28100000000000003},"max":{"1min":2.9980000000000002,"5min":5.609,"15min":52.386000000000003},"last":0.378}]},{"osd":2,"last update":"Sun Oct 1 10:58:55 2023","interfaces":[{"interface":"back","average":{"1min":1.454,"5min":1.5009999999999999,"15min":1.6659999999999999},"min":{"1min":0.34200000000000003,"5min":0.32600000000000001,"15min":0.29199999999999998},"max":{"1min":10.206,"5min":10.510999999999999,"15min":26.242000000000001},"last":3.919},{"interface":"front","average":{"1min":1.375,"5min":1.5209999999999999,"15min":1.925},"min":{"1min":0.318,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":5.2389999999999999,"5min":10.826000000000001,"15min":25.643999999999998},"last":6.2699999999999996}]},{"osd":3,"last update":"Sun Oct 1 10:59:20 2023","interfaces":[{"interface":"back","average":{"1min":1.5920000000000001,"5min":1.3759999999999999,"15min":2.4279999999999999},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.32400000000000001},"max":{"1min":11.819000000000001,"5min":11.819000000000001,"15min":166.29599999999999},"last":4.0410000000000004},{"interface":"front","average":{"1min":1.5349999999999999,"5min":1.494,"15min":2.4990000000000001},"min":{"1min":0.40899999999999997,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":5.6699999999999999,"5min":6.0609999999999999,"15min":166.565},"last":0.47199999999999998}]},{"osd":4,"last update":"Sun Oct 1 10:58:55 2023","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.89400000000000002,"15min":1.431},"min":{"1min":0.372,"5min":0.372,"15min":0.29399999999999998},"max":{"1min":1.1899999999999999,"5min":3.9409999999999998,"15min":15.170999999999999},"last":0.503},{"interface":"front","average":{"1min":0.76900000000000002,"5min":0.97599999999999998,"15min":1.4750000000000001},"min":{"1min":0.41299999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":2.3180000000000001,"5min":6.8879999999999999,"15min":9.7270000000000003},"last":0.434}]},{"osd":6,"last update":"Sun Oct 1 10:59:20 2023","interfaces":[{"interface":"back","average":{"1min":1.4419999999999999,"5min":1.2470000000000001,"15min":1.849},"min":{"1min":0.41899999999999998,"5min":0.38600000000000001,"15min":0.27800000000000002},"max":{"1min":7.7510000000000003,"5min":8.9730000000000008,"15min":61.768999999999998},"last":0.60399999999999998},{"interface":"front","average":{"1min":1.4179999999999999,"5min":1.069,"15min":1.867},"min":{"1min":0.41699999999999998,"5min":0.33000000000000002,"15min":0.28899999999999998},"max":{"1min":4.7229999999999999,"5min":4.7229999999999999,"15min":62.396999999999998},"last":0.57699999999999996}]},{"osd":7,"last update":"Sun Oct 1 10:59:20 2023","interfaces":[{"interface":"back","average":{"1min":0.95599999999999996,"5min":1.042,"15min":1.645},"min":{"1min":0.41099999999999998,"5min":0.36099999999999999,"15min":0.32800000000000001},"max":{"1min":5.2519999999999998,"5min":5.4619999999999997,"15min":13.673999999999999},"last":0.54400000000000004},{"interface":"front","average":{"1min":1.2749999999999999,"5min":1.2290000000000001,"15min":1.5960000000000001},"min":{"1min":0.48499999999999999,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":6.6299999999999999,"5min":6.6299999999999999,"15min":11.635},"last":0.53400000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1155137536,"data_stored":1155137536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":990187520,"data_stored":990183448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1896419328,"data_stored":1896415256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1539637248,"data_stored":1539637248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":560332800,"data_stored":560332800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":387448832,"data_stored":387448832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1332150272,"data_stored":1332150272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1356271616,"data_stored":1356267544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T10:59:45.147 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd pool set device_health_metrics pgp_num 11 2023-10-01T10:59:45.148 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:45.177 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:45.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:45.180 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:59:45.281 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:45.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:45.313 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:45.313 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:59:45.414 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:45.443 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:45.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:45.445 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:59:45.546 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:45.572 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:45.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:45.575 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:59:45.675 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:45.721 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:45.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:45.727 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:59:45.828 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:45.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:45 smithi078 bash[75299]: cluster 2023-10-01T10:59:44.157348+0000 mgr.y (mgr.25072) 1271 : cluster 0 pgmap v1288: 89 pgs: 89 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 545 op/s 2023-10-01T10:59:45.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:45 smithi078 bash[78274]: cluster 2023-10-01T10:59:44.157348+0000 mgr.y (mgr.25072) 1271 : cluster 0 pgmap v1288: 89 pgs: 89 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 545 op/s 2023-10-01T10:59:45.838 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:45 smithi173 bash[58816]: cluster 2023-10-01T10:59:44.157348+0000 mgr.y (mgr.25072) 1271 : cluster 0 pgmap v1288: 89 pgs: 89 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 545 op/s 2023-10-01T10:59:45.859 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:45.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:45.861 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:45.962 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:45.996 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:46.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:46.003 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:59:46.104 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:46.132 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:46.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:46.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:59:46.237 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:46.277 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:46.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:46.282 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:59:46.383 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:46.412 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:46.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:46.415 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:59:46.517 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:46.550 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:46.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:46.556 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:59:46.656 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:46.683 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:46.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:46.686 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:59:46.786 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:46.809 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:46.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:46.814 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:46.915 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:46.948 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:46.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:46.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:59:47.056 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:47.087 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:47.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:47.091 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:59:47.192 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:47.222 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:47.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:47.226 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:59:47.327 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:47.350 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:47.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:47.353 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:47.454 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:47.480 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:47.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:47.484 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:59:47.584 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:47.611 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:47.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:47.613 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:59:47.714 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:47.732 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T10:59:47.753 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:47.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:47.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:59:47.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:47 smithi078 bash[78274]: cluster 2023-10-01T10:59:46.157932+0000 mgr.y (mgr.25072) 1272 : cluster 0 pgmap v1289: 89 pgs: 89 active+clean; 2.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 627 op/s 2023-10-01T10:59:47.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:47 smithi078 bash[78274]: audit 2023-10-01T10:59:47.456334+0000 mon.a (mon.0) 1572 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:59:47.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:47 smithi078 bash[78274]: audit 2023-10-01T10:59:47.457851+0000 mon.a (mon.0) 1573 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:59:47.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:47 smithi078 bash[75299]: cluster 2023-10-01T10:59:46.157932+0000 mgr.y (mgr.25072) 1272 : cluster 0 pgmap v1289: 89 pgs: 89 active+clean; 2.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 627 op/s 2023-10-01T10:59:47.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:47 smithi078 bash[75299]: audit 2023-10-01T10:59:47.456334+0000 mon.a (mon.0) 1572 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:59:47.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:47 smithi078 bash[75299]: audit 2023-10-01T10:59:47.457851+0000 mon.a (mon.0) 1573 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:59:47.856 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:47.865 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:47 smithi173 bash[58816]: cluster 2023-10-01T10:59:46.157932+0000 mgr.y (mgr.25072) 1272 : cluster 0 pgmap v1289: 89 pgs: 89 active+clean; 2.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 627 op/s 2023-10-01T10:59:47.865 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:47 smithi173 bash[58816]: audit 2023-10-01T10:59:47.456334+0000 mon.a (mon.0) 1572 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T10:59:47.866 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:47 smithi173 bash[58816]: audit 2023-10-01T10:59:47.457851+0000 mon.a (mon.0) 1573 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T10:59:47.884 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:47.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:47.890 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:47.991 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:48.023 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:48.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:48.030 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:59:48.080 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T10:59:48.103 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T10:59:48.131 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:48.152 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:48.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:48.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:48.256 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:48.285 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:48.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:48.287 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:59:48.387 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:48.425 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:48.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:48.430 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:59:48.531 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:48.573 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:48.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:48.575 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:59:48.676 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:48.710 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:48.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:48.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:59:48.816 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:48.842 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:48.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:48.844 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:59:48.945 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:48.975 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:48.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:48.983 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:59:49.084 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:49.115 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:49.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:49.120 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:59:49.221 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:49.265 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:49.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:49.270 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:49.371 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:49.410 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:49.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:49.413 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:59:49.514 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:49.553 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:49.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:49.559 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:59:49.660 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:49.685 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:49.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:49.691 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:59:49.792 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:49.801 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:49 smithi078 bash[75299]: cluster 2023-10-01T10:59:48.158399+0000 mgr.y (mgr.25072) 1273 : cluster 0 pgmap v1290: 89 pgs: 89 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 627 op/s 2023-10-01T10:59:49.802 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:49 smithi078 bash[78274]: cluster 2023-10-01T10:59:48.158399+0000 mgr.y (mgr.25072) 1273 : cluster 0 pgmap v1290: 89 pgs: 89 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 627 op/s 2023-10-01T10:59:49.818 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:49.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:49.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:59:49.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:49 smithi173 bash[58816]: cluster 2023-10-01T10:59:48.158399+0000 mgr.y (mgr.25072) 1273 : cluster 0 pgmap v1290: 89 pgs: 89 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 627 op/s 2023-10-01T10:59:49.926 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:49.950 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:49.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:49.953 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:59:50.004 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:59:50.054 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:50.083 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:50.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:50.087 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:59:50.188 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:50.216 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:50.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:50.218 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:59:50.319 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:50.349 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:50.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:50.356 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:59:50.457 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:50.491 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:50.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:50.495 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:59:50.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 10:59:50 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:10:59:50] "GET /metrics HTTP/1.1" 200 36385 "" "Prometheus/2.33.4" 2023-10-01T10:59:50.596 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:50.636 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:50.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:50.644 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:59:50.744 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:50.789 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:50.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:50.791 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:59:50.892 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:50.925 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:50.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:50.931 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:59:51.031 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:51.058 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:51.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:51.064 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:51.165 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:51.191 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:51.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:51.198 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:59:51.299 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:51.345 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:51.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:51.349 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:59:51.450 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:51.486 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:51.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:51.487 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:59:51.588 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:51.624 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:51.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:51.628 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:59:51.729 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:51.753 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-10-01T10:59:51.754 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T10:59:51.758 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:51.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:51.759 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:59:51.782 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:51 smithi173 bash[58816]: cluster 2023-10-01T10:59:50.160291+0000 mgr.y (mgr.25072) 1274 : cluster 0 pgmap v1291: 89 pgs: 89 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 637 op/s 2023-10-01T10:59:51.860 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:51.872 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:51 smithi078 bash[75299]: cluster 2023-10-01T10:59:50.160291+0000 mgr.y (mgr.25072) 1274 : cluster 0 pgmap v1291: 89 pgs: 89 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 637 op/s 2023-10-01T10:59:51.872 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:51 smithi078 bash[78274]: cluster 2023-10-01T10:59:50.160291+0000 mgr.y (mgr.25072) 1274 : cluster 0 pgmap v1291: 89 pgs: 89 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 637 op/s 2023-10-01T10:59:51.892 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:51.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:51.895 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:59:51.996 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:52.024 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:52.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:52.031 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:59:52.045 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:59:52.046 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T10:59:52.132 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:52.155 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:52.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:52.157 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:59:52.258 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:52.296 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:52.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:52.298 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:59:52.399 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:52.437 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:52.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:52.445 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:59:52.546 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:52.577 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:52.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:52.579 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:59:52.668 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-01T10:59:52.680 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:52.704 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:52.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:52.706 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:59:52.807 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:52.835 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:52.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:52.837 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:52.938 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:52.976 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:52.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:52.978 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:59:53.079 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:53.113 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:53.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:53.117 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:59:53.218 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:53.242 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:53.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:53.246 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:59:53.346 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:53.380 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:53.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:53.383 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:59:53.484 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:53.515 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:53.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:53.518 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:59:53.619 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:53.645 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:53.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:53.652 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:53.678 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:53 smithi078 bash[75299]: cluster 2023-10-01T10:59:52.160690+0000 mgr.y (mgr.25072) 1275 : cluster 0 pgmap v1292: 89 pgs: 89 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 638 op/s 2023-10-01T10:59:53.753 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:53.780 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:53.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:53.782 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:59:53.883 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:53.890 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:53 smithi173 bash[58816]: cluster 2023-10-01T10:59:52.160690+0000 mgr.y (mgr.25072) 1275 : cluster 0 pgmap v1292: 89 pgs: 89 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 638 op/s 2023-10-01T10:59:53.904 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:53.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:53.909 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:54.010 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:54.035 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:54.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:54.037 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:54.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:53 smithi078 bash[78274]: cluster 2023-10-01T10:59:52.160690+0000 mgr.y (mgr.25072) 1275 : cluster 0 pgmap v1292: 89 pgs: 89 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 638 op/s 2023-10-01T10:59:54.138 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:54.169 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:54.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:54.174 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:59:54.275 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:54.297 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:54.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:54.299 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:59:54.400 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:54.426 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:54.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:54.429 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:59:54.530 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:54.552 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:54.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:54.558 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:59:54.659 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:54.696 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:54.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:54.699 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:59:54.800 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:54.841 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:54.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:54.847 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:59:54.948 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:54.982 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:54.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:54.984 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:59:55.085 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:55.110 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:55.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:55.116 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:59:55.217 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:55.239 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:55.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:55.242 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:59:55.342 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:55.372 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:55.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:55.374 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:59:55.475 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:55.504 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:55.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:55.507 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:59:55.608 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:55.640 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:55.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:55.643 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:59:55.744 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:55.752 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:55 smithi078 bash[78274]: cluster 2023-10-01T10:59:54.161314+0000 mgr.y (mgr.25072) 1276 : cluster 0 pgmap v1293: 89 pgs: 89 active+clean; 2.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 735 op/s 2023-10-01T10:59:55.752 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:55 smithi078 bash[75299]: cluster 2023-10-01T10:59:54.161314+0000 mgr.y (mgr.25072) 1276 : cluster 0 pgmap v1293: 89 pgs: 89 active+clean; 2.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 735 op/s 2023-10-01T10:59:55.780 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:55.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:55.785 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:59:55.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:55 smithi173 bash[58816]: cluster 2023-10-01T10:59:54.161314+0000 mgr.y (mgr.25072) 1276 : cluster 0 pgmap v1293: 89 pgs: 89 active+clean; 2.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 735 op/s 2023-10-01T10:59:55.885 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:55.925 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:55.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:55.926 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:59:56.027 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:56.064 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:56.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:56.068 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:59:56.169 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:56.200 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:56.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:56.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:59:56.304 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:56.329 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:56.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:56.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:59:56.435 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:56.473 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T10:59:56.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:56.475 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T10:59:56.576 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:56.605 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:56.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:56.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:59:56.707 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:56.737 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:56.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:56.742 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:59:56.843 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:56.869 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:56.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:56.873 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:59:56.973 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:57.001 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:57.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:57.007 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:59:57.108 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:57.139 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:57.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:57.142 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:59:57.243 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:57.278 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:57.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:57.284 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:59:57.385 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:57.419 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:57.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:57.424 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:59:57.524 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:57.557 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:57.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:57.564 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:59:57.665 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:57.696 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:57.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:57.700 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:59:57.785 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:57 smithi078 bash[75299]: cluster 2023-10-01T10:59:56.162082+0000 mgr.y (mgr.25072) 1277 : cluster 0 pgmap v1294: 89 pgs: 89 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 636 op/s 2023-10-01T10:59:57.786 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:57 smithi078 bash[78274]: cluster 2023-10-01T10:59:56.162082+0000 mgr.y (mgr.25072) 1277 : cluster 0 pgmap v1294: 89 pgs: 89 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 636 op/s 2023-10-01T10:59:57.802 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:57.835 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:57.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:57.839 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:59:57.940 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:57.947 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:57 smithi173 bash[58816]: cluster 2023-10-01T10:59:56.162082+0000 mgr.y (mgr.25072) 1277 : cluster 0 pgmap v1294: 89 pgs: 89 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 636 op/s 2023-10-01T10:59:57.968 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:57.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:57.979 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:59:58.080 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:58.099 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:58.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:58.101 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:59:58.202 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:58.231 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:58.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:58.236 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:59:58.337 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:58.370 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T10:59:58.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:58.371 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T10:59:58.472 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:58.504 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:58.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:58.510 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:59:58.611 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:58.643 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:58.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:58.650 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:58.750 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:58.773 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:58.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:58.776 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:59:58.877 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:58.911 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:58.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:58.914 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:59:59.015 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:59.045 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T10:59:59.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:59.051 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T10:59:59.152 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:59.177 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:59.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:59.183 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:59:59.284 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:59.307 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T10:59:59.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:59.311 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T10:59:59.412 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:59.445 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T10:59:59.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:59.448 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T10:59:59.549 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:59.579 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T10:59:59.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:59.582 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T10:59:59.683 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:59.719 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:59.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:59.722 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T10:59:59.822 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:59.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 10:59:59 smithi078 bash[78274]: cluster 2023-10-01T10:59:58.162614+0000 mgr.y (mgr.25072) 1278 : cluster 0 pgmap v1295: 89 pgs: 89 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2023-10-01T10:59:59.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 10:59:59 smithi078 bash[75299]: cluster 2023-10-01T10:59:58.162614+0000 mgr.y (mgr.25072) 1278 : cluster 0 pgmap v1295: 89 pgs: 89 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2023-10-01T10:59:59.855 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T10:59:59.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:59.863 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T10:59:59.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 10:59:59 smithi173 bash[58816]: cluster 2023-10-01T10:59:58.162614+0000 mgr.y (mgr.25072) 1278 : cluster 0 pgmap v1295: 89 pgs: 89 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 600 op/s 2023-10-01T10:59:59.964 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:59.985 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T10:59:59.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T10:59:59.988 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:00:00.088 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:00.115 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:00.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:00.121 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:00:00.222 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:00.249 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:00.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:00.257 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:00:00.358 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:00.367 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:00:00 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:00:00] "GET /metrics HTTP/1.1" 200 36385 "" "Prometheus/2.33.4" 2023-10-01T11:00:00.394 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:00.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:00.398 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:00.499 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:00.525 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:00.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:00.527 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:00:00.628 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:00.662 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:00.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:00.667 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:00.767 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:00.776 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:00 smithi173 bash[58816]: cluster 2023-10-01T11:00:00.000124+0000 mon.a (mon.0) 1574 : cluster 3 overall HEALTH_WARN noscrub flag(s) set 2023-10-01T11:00:00.799 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:00.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:00.806 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:00:00.908 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:00.937 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:00.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:00.944 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:00:01.044 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:01.053 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:00 smithi078 bash[75299]: cluster 2023-10-01T11:00:00.000124+0000 mon.a (mon.0) 1574 : cluster 3 overall HEALTH_WARN noscrub flag(s) set 2023-10-01T11:00:01.053 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:00 smithi078 bash[78274]: cluster 2023-10-01T11:00:00.000124+0000 mon.a (mon.0) 1574 : cluster 3 overall HEALTH_WARN noscrub flag(s) set 2023-10-01T11:00:01.071 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:01.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:01.077 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:00:01.178 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:01.205 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:01.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:01.208 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:00:01.309 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:01.340 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:01.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:01.344 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:00:01.445 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:01.483 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:01.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:01.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:00:01.586 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:01.611 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:01.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:01.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:00:01.720 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:01.729 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:01 smithi173 bash[58816]: cluster 2023-10-01T11:00:00.163269+0000 mgr.y (mgr.25072) 1279 : cluster 0 pgmap v1296: 89 pgs: 89 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 721 op/s 2023-10-01T11:00:01.747 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:01.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:01.750 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:00:01.851 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:01.905 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:01.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:01.913 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:02.014 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:02.037 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:02.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:02.040 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:00:02.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:01 smithi078 bash[75299]: cluster 2023-10-01T11:00:00.163269+0000 mgr.y (mgr.25072) 1279 : cluster 0 pgmap v1296: 89 pgs: 89 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 721 op/s 2023-10-01T11:00:02.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:01 smithi078 bash[78274]: cluster 2023-10-01T11:00:00.163269+0000 mgr.y (mgr.25072) 1279 : cluster 0 pgmap v1296: 89 pgs: 89 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 721 op/s 2023-10-01T11:00:02.141 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:02.176 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:02.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:02.178 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:00:02.279 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:02.306 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:02.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:02.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:00:02.343 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:00:02.411 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:02.457 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:02.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:02.461 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:00:02.562 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:02.587 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:02.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:02.593 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:00:02.694 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:02.730 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:02.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:02.733 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:00:02.833 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:02.863 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:02.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:02.868 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:00:02.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:02 smithi173 bash[58816]: audit 2023-10-01T11:00:02.450863+0000 mon.a (mon.0) 1575 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:00:02.928 INFO:teuthology.orchestra.run.smithi078.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-10-01T11:00:02.969 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:03.007 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:03.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:03.012 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:00:03.071 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:02 smithi078 bash[75299]: audit 2023-10-01T11:00:02.450863+0000 mon.a (mon.0) 1575 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:00:03.072 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:02 smithi078 bash[78274]: audit 2023-10-01T11:00:02.450863+0000 mon.a (mon.0) 1575 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:00:03.113 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:03.143 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:03.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:03.148 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:00:03.249 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:03.274 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:03.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:03.279 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:00:03.380 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:03.404 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:03.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:03.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:00:03.507 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:03.540 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:03.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:03.542 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:00:03.643 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:03.686 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:03.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:03.690 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:00:03.790 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:03.800 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:03 smithi173 bash[58816]: cluster 2023-10-01T11:00:02.163643+0000 mgr.y (mgr.25072) 1280 : cluster 0 pgmap v1297: 89 pgs: 89 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 521 op/s 2023-10-01T11:00:03.800 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:03 smithi173 bash[58816]: audit 2023-10-01T11:00:02.923137+0000 mon.c (mon.2) 198 : audit 1 from='client.? 172.21.15.78:0/4028686879' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-01T11:00:03.801 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:03 smithi173 bash[58816]: audit 2023-10-01T11:00:02.924156+0000 mon.a (mon.0) 1576 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-01T11:00:03.822 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:03.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:03.824 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:00:03.925 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:03.964 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:03.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:03.969 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:00:04.070 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:04.077 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:03 smithi078 bash[78274]: cluster 2023-10-01T11:00:02.163643+0000 mgr.y (mgr.25072) 1280 : cluster 0 pgmap v1297: 89 pgs: 89 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 521 op/s 2023-10-01T11:00:04.077 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:03 smithi078 bash[78274]: audit 2023-10-01T11:00:02.923137+0000 mon.c (mon.2) 198 : audit 1 from='client.? 172.21.15.78:0/4028686879' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-01T11:00:04.077 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:03 smithi078 bash[78274]: audit 2023-10-01T11:00:02.924156+0000 mon.a (mon.0) 1576 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-01T11:00:04.078 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:03 smithi078 bash[75299]: cluster 2023-10-01T11:00:02.163643+0000 mgr.y (mgr.25072) 1280 : cluster 0 pgmap v1297: 89 pgs: 89 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 521 op/s 2023-10-01T11:00:04.078 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:03 smithi078 bash[75299]: audit 2023-10-01T11:00:02.923137+0000 mon.c (mon.2) 198 : audit 1 from='client.? 172.21.15.78:0/4028686879' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-01T11:00:04.078 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:03 smithi078 bash[75299]: audit 2023-10-01T11:00:02.924156+0000 mon.a (mon.0) 1576 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "11"}]: dispatch 2023-10-01T11:00:04.090 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:04.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:04.093 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:00:04.193 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:04.217 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:04.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:04.227 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:00:04.328 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:04.363 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:04.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:04.368 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:04.469 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:04.496 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:04.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:04.498 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:04.599 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:04.623 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:04.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:04.626 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:00:04.727 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:04.762 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:04.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:04.767 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:00:04.868 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:04.910 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:04.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:04.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:00:05.018 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:05.049 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:05.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:05.051 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:00:05.152 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:05.194 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:05.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:05.198 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:00:05.299 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:05.328 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:05.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:05.330 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:00:05.431 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:05.467 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:05.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:05.471 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:00:05.572 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:05.599 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:05.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:05.606 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:00:05.706 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:05.741 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:05.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:05.742 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:00:05.843 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:05.855 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:05 smithi173 bash[58816]: cluster 2023-10-01T11:00:04.164337+0000 mgr.y (mgr.25072) 1281 : cluster 0 pgmap v1298: 89 pgs: 89 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 688 op/s 2023-10-01T11:00:05.875 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:05.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:05.880 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:00:05.981 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:06.016 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:06.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:06.018 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:00:06.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:05 smithi078 bash[78274]: cluster 2023-10-01T11:00:04.164337+0000 mgr.y (mgr.25072) 1281 : cluster 0 pgmap v1298: 89 pgs: 89 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 688 op/s 2023-10-01T11:00:06.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:05 smithi078 bash[75299]: cluster 2023-10-01T11:00:04.164337+0000 mgr.y (mgr.25072) 1281 : cluster 0 pgmap v1298: 89 pgs: 89 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 688 op/s 2023-10-01T11:00:06.119 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:06.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:06.220 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:06.221 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:00:06.321 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:06.348 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:06.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:06.353 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:00:06.454 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:06.476 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:06.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:06.479 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:00:06.580 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:06.604 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:06.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:06.615 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:00:06.716 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:06.749 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:06.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:06.751 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:00:06.852 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:06.883 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:06.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:06.887 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:00:06.988 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:07.014 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:07.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:07.022 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:07.123 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:07.157 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:07.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:07.159 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:07.254 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-10-01T11:00:07.260 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:07.301 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:07.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:07.302 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:07.403 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:07.433 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:07.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:07.439 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:07.540 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:07.568 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:07.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:07.571 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:00:07.672 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:07.702 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:07.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:07.707 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:07.808 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:07.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:07 smithi078 bash[78274]: cluster 2023-10-01T11:00:06.164873+0000 mgr.y (mgr.25072) 1282 : cluster 0 pgmap v1299: 89 pgs: 89 active+clean; 1.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 611 op/s 2023-10-01T11:00:07.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:07 smithi078 bash[75299]: cluster 2023-10-01T11:00:06.164873+0000 mgr.y (mgr.25072) 1282 : cluster 0 pgmap v1299: 89 pgs: 89 active+clean; 1.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 611 op/s 2023-10-01T11:00:07.834 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:07.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:07.837 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:00:07.938 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:07.967 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:07.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:07.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:00:08.072 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:08.079 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:07 smithi173 bash[58816]: cluster 2023-10-01T11:00:06.164873+0000 mgr.y (mgr.25072) 1282 : cluster 0 pgmap v1299: 89 pgs: 89 active+clean; 1.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 611 op/s 2023-10-01T11:00:08.097 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:08.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:08.099 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:00:08.151 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T11:00:08.200 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:08.205 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T11:00:08.233 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:08.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:08.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:00:08.259 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T11:00:08.309 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T11:00:08.338 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:08.354 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T11:00:08.364 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:08.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:08.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:00:08.385 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T11:00:08.424 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T11:00:08.466 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T11:00:08.467 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:08.471 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T11:00:08.471 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T11:00:08.471 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T11:00:08.472 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T11:00:08.472 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T11:00:08.472 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T11:00:08.472 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T11:00:08.472 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T11:00:08.500 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:08.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:08.505 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:00:08.605 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:08.636 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:08.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:08.642 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:00:08.742 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:08 smithi078 bash[75299]: audit 2023-10-01T11:00:08.099799+0000 mon.a (mon.0) 1577 : audit 0 from='client.? 172.21.15.78:0/1088893389' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:00:08.742 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:08 smithi078 bash[75299]: audit 2023-10-01T11:00:08.384282+0000 mon.b (mon.1) 53 : audit 1 from='client.? 172.21.15.78:0/1357202229' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:00:08.742 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:08 smithi078 bash[75299]: audit 2023-10-01T11:00:08.384514+0000 mon.a (mon.0) 1578 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:00:08.742 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:08 smithi078 bash[78274]: audit 2023-10-01T11:00:08.099799+0000 mon.a (mon.0) 1577 : audit 0 from='client.? 172.21.15.78:0/1088893389' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:00:08.742 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:08 smithi078 bash[78274]: audit 2023-10-01T11:00:08.384282+0000 mon.b (mon.1) 53 : audit 1 from='client.? 172.21.15.78:0/1357202229' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:00:08.742 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:08 smithi078 bash[78274]: audit 2023-10-01T11:00:08.384514+0000 mon.a (mon.0) 1578 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:00:08.743 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:08.759 INFO:teuthology.orchestra.run.smithi078.stderr:nodeep-scrub is set 2023-10-01T11:00:08.778 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:08.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:08.783 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:00:08.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:08 smithi173 bash[58816]: audit 2023-10-01T11:00:08.099799+0000 mon.a (mon.0) 1577 : audit 0 from='client.? 172.21.15.78:0/1088893389' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:00:08.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:08 smithi173 bash[58816]: audit 2023-10-01T11:00:08.384282+0000 mon.b (mon.1) 53 : audit 1 from='client.? 172.21.15.78:0/1357202229' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:00:08.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:08 smithi173 bash[58816]: audit 2023-10-01T11:00:08.384514+0000 mon.a (mon.0) 1578 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:00:08.884 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:08.923 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:08.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:08.925 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:09.025 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:09.053 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:09.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:09.055 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:00:09.156 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:09.181 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:09.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:09.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:00:09.283 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:09.305 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:09.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:09.308 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:00:09.408 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:09.437 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:09.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:09.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:00:09.544 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:09.569 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:09.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:09.572 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:00:09.672 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:09.690 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:09.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:09.692 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:00:09.793 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:09.802 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:09 smithi078 bash[75299]: cluster 2023-10-01T11:00:08.165375+0000 mgr.y (mgr.25072) 1283 : cluster 0 pgmap v1300: 89 pgs: 89 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 586 op/s 2023-10-01T11:00:09.803 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:09 smithi078 bash[75299]: cluster 2023-10-01T11:00:08.727634+0000 mon.a (mon.0) 1579 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:00:09.803 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:09 smithi078 bash[75299]: cluster 2023-10-01T11:00:08.740658+0000 osd.7 (osd.7) 245 : cluster 3 failed to encode map e254 with expected crc 2023-10-01T11:00:09.803 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:09 smithi078 bash[75299]: audit 2023-10-01T11:00:08.744380+0000 mon.a (mon.0) 1580 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T11:00:09.803 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:09 smithi078 bash[75299]: cluster 2023-10-01T11:00:08.744645+0000 mon.a (mon.0) 1581 : cluster 0 osdmap e254: 8 total, 8 up, 8 in 2023-10-01T11:00:09.803 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:09 smithi078 bash[75299]: cluster 2023-10-01T11:00:08.745859+0000 osd.6 (osd.6) 202 : cluster 3 failed to encode map e254 with expected crc 2023-10-01T11:00:09.803 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:09 smithi078 bash[75299]: cluster 2023-10-01T11:00:08.748329+0000 osd.6 (osd.6) 203 : cluster 3 failed to encode map e254 with expected crc 2023-10-01T11:00:09.803 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:09 smithi078 bash[75299]: cluster 2023-10-01T11:00:08.751482+0000 osd.6 (osd.6) 204 : cluster 3 failed to encode map e254 with expected crc 2023-10-01T11:00:09.804 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:09 smithi078 bash[78274]: cluster 2023-10-01T11:00:08.165375+0000 mgr.y (mgr.25072) 1283 : cluster 0 pgmap v1300: 89 pgs: 89 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 586 op/s 2023-10-01T11:00:09.804 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:09 smithi078 bash[78274]: cluster 2023-10-01T11:00:08.727634+0000 mon.a (mon.0) 1579 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:00:09.804 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:09 smithi078 bash[78274]: cluster 2023-10-01T11:00:08.740658+0000 osd.7 (osd.7) 245 : cluster 3 failed to encode map e254 with expected crc 2023-10-01T11:00:09.804 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:09 smithi078 bash[78274]: audit 2023-10-01T11:00:08.744380+0000 mon.a (mon.0) 1580 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T11:00:09.804 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:09 smithi078 bash[78274]: cluster 2023-10-01T11:00:08.744645+0000 mon.a (mon.0) 1581 : cluster 0 osdmap e254: 8 total, 8 up, 8 in 2023-10-01T11:00:09.804 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:09 smithi078 bash[78274]: cluster 2023-10-01T11:00:08.745859+0000 osd.6 (osd.6) 202 : cluster 3 failed to encode map e254 with expected crc 2023-10-01T11:00:09.804 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:09 smithi078 bash[78274]: cluster 2023-10-01T11:00:08.748329+0000 osd.6 (osd.6) 203 : cluster 3 failed to encode map e254 with expected crc 2023-10-01T11:00:09.804 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:09 smithi078 bash[78274]: cluster 2023-10-01T11:00:08.751482+0000 osd.6 (osd.6) 204 : cluster 3 failed to encode map e254 with expected crc 2023-10-01T11:00:09.817 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:09.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:09.819 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:00:09.920 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:09.942 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:09.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:09.946 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:00:10.046 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:10.057 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:09 smithi173 bash[58816]: cluster 2023-10-01T11:00:08.165375+0000 mgr.y (mgr.25072) 1283 : cluster 0 pgmap v1300: 89 pgs: 89 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 586 op/s 2023-10-01T11:00:10.057 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:09 smithi173 bash[58816]: cluster 2023-10-01T11:00:08.727634+0000 mon.a (mon.0) 1579 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:00:10.057 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:09 smithi173 bash[58816]: cluster 2023-10-01T11:00:08.740658+0000 osd.7 (osd.7) 245 : cluster 3 failed to encode map e254 with expected crc 2023-10-01T11:00:10.058 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:09 smithi173 bash[58816]: audit 2023-10-01T11:00:08.744380+0000 mon.a (mon.0) 1580 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T11:00:10.058 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:09 smithi173 bash[58816]: cluster 2023-10-01T11:00:08.744645+0000 mon.a (mon.0) 1581 : cluster 0 osdmap e254: 8 total, 8 up, 8 in 2023-10-01T11:00:10.058 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:09 smithi173 bash[58816]: cluster 2023-10-01T11:00:08.745859+0000 osd.6 (osd.6) 202 : cluster 3 failed to encode map e254 with expected crc 2023-10-01T11:00:10.058 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:09 smithi173 bash[58816]: cluster 2023-10-01T11:00:08.748329+0000 osd.6 (osd.6) 203 : cluster 3 failed to encode map e254 with expected crc 2023-10-01T11:00:10.058 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:09 smithi173 bash[58816]: cluster 2023-10-01T11:00:08.751482+0000 osd.6 (osd.6) 204 : cluster 3 failed to encode map e254 with expected crc 2023-10-01T11:00:10.073 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:10.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:10.079 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:00:10.180 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:10.199 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:10.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:10.201 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:00:10.302 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:10.326 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:10.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:10.329 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:00:10.430 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:10.455 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:10.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:10.466 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:00:10.518 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:00:10 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:00:10] "GET /metrics HTTP/1.1" 200 36383 "" "Prometheus/2.33.4" 2023-10-01T11:00:10.567 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:10.595 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:10.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:10.598 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:00:10.699 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:10.733 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:10.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:10.737 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:00:10.837 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:10.875 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:10.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:10.881 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:00:10.982 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:11.006 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:11.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:11.009 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:00:11.110 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:11.157 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:11.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:11.159 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:11.259 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:11.291 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:11.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:11.293 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:11.394 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:11.427 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:11.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:11.432 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:00:11.533 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:11.567 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:11.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:11.571 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:00:11.672 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:11.697 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:11.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:11.702 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:00:11.803 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:11.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:11 smithi078 bash[75299]: cluster 2023-10-01T11:00:10.166228+0000 mgr.y (mgr.25072) 1284 : cluster 0 pgmap v1302: 89 pgs: 89 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 639 op/s 2023-10-01T11:00:11.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:11 smithi078 bash[78274]: cluster 2023-10-01T11:00:10.166228+0000 mgr.y (mgr.25072) 1284 : cluster 0 pgmap v1302: 89 pgs: 89 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 639 op/s 2023-10-01T11:00:11.832 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:11.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:11.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:00:11.940 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:11.978 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:11.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:11.984 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:00:12.084 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:12.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:11 smithi173 bash[58816]: cluster 2023-10-01T11:00:10.166228+0000 mgr.y (mgr.25072) 1284 : cluster 0 pgmap v1302: 89 pgs: 89 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 639 op/s 2023-10-01T11:00:12.120 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:12.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:12.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:00:12.228 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:12.252 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:12.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:12.255 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:00:12.255 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 7, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T11:00:12.255 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T11:00:12.255 INFO:tasks.thrashosds.thrasher:Removing osd 6, in_osds are: [0, 1, 2, 3, 4, 5, 7, 6] 2023-10-01T11:00:12.255 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd out 6 2023-10-01T11:00:12.355 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:12.387 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:12.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:12.391 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:00:12.492 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:12.516 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:12.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:12.519 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:00:12.620 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:12.645 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:12.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:12.650 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:12.750 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:12.775 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:12.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:12.777 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:00:12.878 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:12.907 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:12.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:12.911 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:13.012 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:13.042 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:13.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:13.046 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:00:13.147 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:13.170 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:13.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:13.174 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:00:13.275 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:13.301 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:13.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:13.304 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:00:13.404 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:13.428 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:13.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:13.433 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:00:13.534 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:13.559 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:13.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:13.563 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:00:13.664 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:13.692 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:13.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:13.693 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:13.794 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:13.802 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:13 smithi078 bash[78274]: cluster 2023-10-01T11:00:12.166639+0000 mgr.y (mgr.25072) 1285 : cluster 0 pgmap v1303: 89 pgs: 89 active+clean; 1.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 639 op/s 2023-10-01T11:00:13.803 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:13 smithi078 bash[75299]: cluster 2023-10-01T11:00:12.166639+0000 mgr.y (mgr.25072) 1285 : cluster 0 pgmap v1303: 89 pgs: 89 active+clean; 1.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 639 op/s 2023-10-01T11:00:13.822 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:13.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:13.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:00:13.929 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:13.956 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:13.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:13.961 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:14.062 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:14.098 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:14.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:14.104 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:00:14.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:13 smithi173 bash[58816]: cluster 2023-10-01T11:00:12.166639+0000 mgr.y (mgr.25072) 1285 : cluster 0 pgmap v1303: 89 pgs: 89 active+clean; 1.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 639 op/s 2023-10-01T11:00:14.205 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:14.230 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:14.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:14.232 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:00:14.333 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:14.369 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:14.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:14.372 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:00:14.373 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2023-10-01T11:00:14.373 INFO:teuthology.orchestra.run.smithi078.stdout: "ops": [ 2023-10-01T11:00:14.373 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:00:14.373 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46232.0:182030 12.c 12:35547150:::benchmark_data_smithi078_180197_object78899:head [delete] snapc 0=[] ondisk+write+known_if_redirected e254)", 2023-10-01T11:00:14.374 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:00:14.366591+0000", 2023-10-01T11:00:14.374 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 0.0055345489999999997, 2023-10-01T11:00:14.374 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0055582619999999996, 2023-10-01T11:00:14.374 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:00:14.374 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "waiting for sub ops", 2023-10-01T11:00:14.374 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T11:00:14.374 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46232", 2023-10-01T11:00:14.375 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/711982100", 2023-10-01T11:00:14.375 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 182030 2023-10-01T11:00:14.375 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:00:14.375 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:00:14.375 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:00:14.375 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:00:14.375 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:00:14.366591+0000", 2023-10-01T11:00:14.376 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:00:14.376 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:00:14.376 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:00:14.376 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:00:14.376 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:00:14.366591+0000", 2023-10-01T11:00:14.376 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:00:14.376 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:00:14.376 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:00:14.377 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:00:14.377 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:00:14.366588+0000", 2023-10-01T11:00:14.377 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999981 2023-10-01T11:00:14.377 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:00:14.377 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:00:14.377 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:00:14.377 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:00:14.366614+0000", 2023-10-01T11:00:14.377 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.5060000000000001e-05 2023-10-01T11:00:14.378 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:00:14.378 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:00:14.378 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:00:14.378 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:00:14.366631+0000", 2023-10-01T11:00:14.378 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.7210999999999998e-05 2023-10-01T11:00:14.378 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:00:14.378 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:00:14.379 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:00:14.379 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:00:14.366641+0000", 2023-10-01T11:00:14.379 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.7850000000000003e-06 2023-10-01T11:00:14.379 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:00:14.379 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:00:14.379 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:00:14.379 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:00:14.366691+0000", 2023-10-01T11:00:14.379 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.0349000000000002e-05 2023-10-01T11:00:14.380 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:00:14.380 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:00:14.380 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:00:14.380 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:00:14.366810+0000", 2023-10-01T11:00:14.380 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000119207 2023-10-01T11:00:14.380 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:00:14.380 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:00:14.380 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 2,5", 2023-10-01T11:00:14.381 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:00:14.366923+0000", 2023-10-01T11:00:14.381 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00011299299999999999 2023-10-01T11:00:14.381 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:00:14.381 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:00:14.381 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:00:14.381 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:00:14.381 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:00:14.381 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.46232.0:182031 12.a e254/239 12:5d806807:::benchmark_data_smithi078_180197_object78900:head v 254'11283, mlcod=254'11281)", 2023-10-01T11:00:14.382 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:00:14.367237+0000", 2023-10-01T11:00:14.382 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 0.004887771, 2023-10-01T11:00:14.382 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0049629119999999999, 2023-10-01T11:00:14.382 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:00:14.382 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "started", 2023-10-01T11:00:14.382 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:00:14.382 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:00:14.383 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:00:14.383 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:00:14.367237+0000", 2023-10-01T11:00:14.383 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:00:14.383 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:00:14.383 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:00:14.383 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:00:14.383 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:00:14.367237+0000", 2023-10-01T11:00:14.384 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:00:14.384 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:00:14.384 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:00:14.384 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:00:14.384 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:00:14.367236+0000", 2023-10-01T11:00:14.384 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T11:00:14.384 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:00:14.384 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:00:14.385 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:00:14.385 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:00:14.367244+0000", 2023-10-01T11:00:14.385 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.3429999999999998e-06 2023-10-01T11:00:14.385 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:00:14.385 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:00:14.385 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:00:14.385 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:00:14.367255+0000", 2023-10-01T11:00:14.385 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1086e-05 2023-10-01T11:00:14.386 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:00:14.386 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:00:14.386 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:00:14.386 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:00:14.367262+0000", 2023-10-01T11:00:14.386 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.0489999999999998e-06 2023-10-01T11:00:14.386 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:00:14.387 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:00:14.387 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:00:14.387 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:00:14.367308+0000", 2023-10-01T11:00:14.387 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.5760000000000002e-05 2023-10-01T11:00:14.387 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:00:14.387 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:00:14.387 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:00:14.388 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:00:14.367327+0000", 2023-10-01T11:00:14.388 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.9748000000000001e-05 2023-10-01T11:00:14.388 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:00:14.388 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:00:14.388 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:00:14.388 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:00:14.388 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:00:14.388 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop_reply(client.46232.0:182030 12.c e254/239)", 2023-10-01T11:00:14.389 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:00:14.371727+0000", 2023-10-01T11:00:14.389 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 0.00039817599999999998, 2023-10-01T11:00:14.389 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00050686700000000004, 2023-10-01T11:00:14.389 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:00:14.389 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "queued for pg", 2023-10-01T11:00:14.389 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:00:14.389 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:00:14.389 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:00:14.389 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:00:14.371727+0000", 2023-10-01T11:00:14.389 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:00:14.389 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:00:14.390 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:00:14.390 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:00:14.390 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:00:14.371727+0000", 2023-10-01T11:00:14.390 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:00:14.390 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:00:14.390 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:00:14.390 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:00:14.391 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:00:14.371726+0000", 2023-10-01T11:00:14.391 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T11:00:14.391 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:00:14.391 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:00:14.391 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:00:14.391 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:00:14.371731+0000", 2023-10-01T11:00:14.391 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.6620000000000002e-06 2023-10-01T11:00:14.392 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:00:14.392 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:00:14.392 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:00:14.392 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:00:14.371748+0000", 2023-10-01T11:00:14.392 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.647e-05 2023-10-01T11:00:14.392 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:00:14.392 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:00:14.392 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:00:14.393 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:00:14.371755+0000", 2023-10-01T11:00:14.393 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.6969999999999996e-06 2023-10-01T11:00:14.393 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:00:14.393 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:00:14.393 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:00:14.393 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:00:14.393 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2023-10-01T11:00:14.393 INFO:teuthology.orchestra.run.smithi078.stdout: "num_ops": 3 2023-10-01T11:00:14.393 INFO:teuthology.orchestra.run.smithi078.stdout:} 2023-10-01T11:00:14.473 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:14.499 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:14.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:14.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:00:14.603 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:14.637 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:14.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:14.643 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:00:14.744 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:14.779 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:14.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:14.782 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:00:14.882 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:14.918 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:14.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:14.921 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:14.943 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T11:00:15.022 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:15.059 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:15.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:15.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:00:15.168 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:15.202 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:15.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:15.206 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:00:15.306 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:15.337 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:15.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:15.342 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:00:15.443 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:15.471 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:15.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:15.479 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:00:15.580 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:15.599 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T11:00:15.611 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:15.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:15.615 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:00:15.716 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:15.742 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:15.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:15.745 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:15.846 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:15.869 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:15.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:15.876 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:00:15.928 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:15 smithi078 bash[75299]: cluster 2023-10-01T11:00:14.167224+0000 mgr.y (mgr.25072) 1286 : cluster 0 pgmap v1304: 89 pgs: 89 active+clean; 1.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 634 op/s 2023-10-01T11:00:15.929 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:15 smithi078 bash[78274]: cluster 2023-10-01T11:00:14.167224+0000 mgr.y (mgr.25072) 1286 : cluster 0 pgmap v1304: 89 pgs: 89 active+clean; 1.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 634 op/s 2023-10-01T11:00:15.929 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-01T11:00:15.977 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:16.023 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:16.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:16.024 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:00:16.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:15 smithi173 bash[58816]: cluster 2023-10-01T11:00:14.167224+0000 mgr.y (mgr.25072) 1286 : cluster 0 pgmap v1304: 89 pgs: 89 active+clean; 1.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 634 op/s 2023-10-01T11:00:16.125 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:16.153 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:16.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:16.161 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:16.261 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:16.294 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:16.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:16.301 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:00:16.402 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:16.428 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:16.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:16.430 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:00:16.531 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:16.561 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:16.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:16.568 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:00:16.669 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:16.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:16.726 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:16.726 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:00:16.827 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:16.852 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:16.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:16.858 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:00:16.958 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:16.989 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:16.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:16.994 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:00:17.095 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:17.123 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:17.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:17.132 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:17.232 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:17.256 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:17.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:17.263 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:00:17.363 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:17.404 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:17.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:17.412 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:00:17.513 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:17.540 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:17.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:17.544 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:00:17.645 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:17.681 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:17.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:17.689 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:00:17.790 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:17.820 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:17.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:17.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:17.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:17 smithi173 bash[58816]: cluster 2023-10-01T11:00:16.167666+0000 mgr.y (mgr.25072) 1287 : cluster 0 pgmap v1305: 89 pgs: 89 active+clean; 1.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 630 op/s 2023-10-01T11:00:17.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:17 smithi173 bash[58816]: audit 2023-10-01T11:00:17.451140+0000 mon.a (mon.0) 1582 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:00:17.925 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:17.954 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:17.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:17.956 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:00:18.057 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:18.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:17 smithi078 bash[78274]: cluster 2023-10-01T11:00:16.167666+0000 mgr.y (mgr.25072) 1287 : cluster 0 pgmap v1305: 89 pgs: 89 active+clean; 1.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 630 op/s 2023-10-01T11:00:18.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:17 smithi078 bash[78274]: audit 2023-10-01T11:00:17.451140+0000 mon.a (mon.0) 1582 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:00:18.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:17 smithi078 bash[75299]: cluster 2023-10-01T11:00:16.167666+0000 mgr.y (mgr.25072) 1287 : cluster 0 pgmap v1305: 89 pgs: 89 active+clean; 1.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 630 op/s 2023-10-01T11:00:18.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:17 smithi078 bash[75299]: audit 2023-10-01T11:00:17.451140+0000 mon.a (mon.0) 1582 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:00:18.093 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:18.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:18.096 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:00:18.197 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:18.223 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:18.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:18.230 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:00:18.330 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:18.353 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:18.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:18.358 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:00:18.450 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:00:18.459 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:18.481 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:18.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:18.483 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:00:18.584 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:18.612 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:18.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:18.620 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:00:18.720 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:18.748 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:18.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:18.751 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:00:18.852 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:18.880 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:18.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:18.882 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:00:18.983 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:19.006 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:19.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:19.009 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:00:19.110 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:19.136 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:19.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:19.143 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:00:19.243 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:19.280 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:19.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:19.284 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:00:19.385 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:19.424 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:19.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:19.427 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:00:19.528 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:19.558 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:19.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:19.563 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:19.664 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:19.689 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:19.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:19.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:19.767 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:19 smithi078 bash[78274]: cluster 2023-10-01T11:00:18.168157+0000 mgr.y (mgr.25072) 1288 : cluster 0 pgmap v1306: 89 pgs: 89 active+clean; 1.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 629 op/s 2023-10-01T11:00:19.768 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:19 smithi078 bash[75299]: cluster 2023-10-01T11:00:18.168157+0000 mgr.y (mgr.25072) 1288 : cluster 0 pgmap v1306: 89 pgs: 89 active+clean; 1.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 629 op/s 2023-10-01T11:00:19.792 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:19.820 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:19.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:19.824 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:00:19.924 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:19.954 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:19.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:19.958 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:00:20.058 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:20.069 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:19 smithi173 bash[58816]: cluster 2023-10-01T11:00:18.168157+0000 mgr.y (mgr.25072) 1288 : cluster 0 pgmap v1306: 89 pgs: 89 active+clean; 1.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 629 op/s 2023-10-01T11:00:20.087 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:20.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:20.091 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:00:20.192 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:20.225 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:20.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:20.228 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:00:20.329 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:20.374 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:20.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:20.378 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:00:20.479 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:20.482 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:00:20.483 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:00:20.488 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:00:20.528 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:00:20 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:00:20] "GET /metrics HTTP/1.1" 200 36384 "" "Prometheus/2.33.4" 2023-10-01T11:00:20.562 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:20.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:20.566 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:20.667 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:20.698 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:20.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:20.705 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:00:20.806 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:20.831 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:20.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:20.838 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:00:20.938 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:20.970 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:20.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:20.975 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:00:21.075 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:21.121 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:21.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:21.123 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:00:21.224 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:21.252 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:21.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:21.256 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:00:21.356 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:21.386 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:21.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:21.395 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:00:21.495 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:21.534 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:21.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:21.539 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:00:21.639 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:21.669 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:21.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:21.676 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:00:21.777 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:21.800 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:21.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:21.806 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:00:21.907 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:21.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:21.937 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:21.938 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:00:22.038 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:22.049 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:21 smithi173 bash[58816]: cluster 2023-10-01T11:00:20.168686+0000 mgr.y (mgr.25072) 1289 : cluster 0 pgmap v1307: 89 pgs: 89 active+clean; 1.3 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 661 op/s 2023-10-01T11:00:22.061 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:22.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:22.066 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:00:22.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:21 smithi078 bash[78274]: cluster 2023-10-01T11:00:20.168686+0000 mgr.y (mgr.25072) 1289 : cluster 0 pgmap v1307: 89 pgs: 89 active+clean; 1.3 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 661 op/s 2023-10-01T11:00:22.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:21 smithi078 bash[75299]: cluster 2023-10-01T11:00:20.168686+0000 mgr.y (mgr.25072) 1289 : cluster 0 pgmap v1307: 89 pgs: 89 active+clean; 1.3 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 661 op/s 2023-10-01T11:00:22.167 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:22.196 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:22.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:22.202 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:00:22.303 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:22.331 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:22.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:22.338 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:00:22.439 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:22.472 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:22.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:22.475 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:00:22.576 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:22.595 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:22.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:22.602 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:00:22.702 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:22.732 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:22.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:22.739 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:00:22.840 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:22.887 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:22.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:22.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:22.993 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:23.025 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:23.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:23.028 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:23.129 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:23.154 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:23.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:23.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:23.266 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:23.302 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:23.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:23.311 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:00:23.412 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:23.444 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:23.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:23.448 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:00:23.549 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:23.580 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:23.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:23.584 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:00:23.685 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:23.717 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:23.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:23.723 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:00:23.823 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:23.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:23 smithi078 bash[78274]: cluster 2023-10-01T11:00:22.169081+0000 mgr.y (mgr.25072) 1290 : cluster 0 pgmap v1308: 89 pgs: 89 active+clean; 1.3 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 527 op/s 2023-10-01T11:00:23.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:23 smithi078 bash[75299]: cluster 2023-10-01T11:00:22.169081+0000 mgr.y (mgr.25072) 1290 : cluster 0 pgmap v1308: 89 pgs: 89 active+clean; 1.3 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 527 op/s 2023-10-01T11:00:23.848 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:23.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:23.852 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:00:23.952 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:23.978 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:23.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:23.986 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:00:24.086 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:24.114 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:24.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:23 smithi173 bash[58816]: cluster 2023-10-01T11:00:22.169081+0000 mgr.y (mgr.25072) 1290 : cluster 0 pgmap v1308: 89 pgs: 89 active+clean; 1.3 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 527 op/s 2023-10-01T11:00:24.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:24.121 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:00:24.222 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:24.251 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:24.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:24.258 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:00:24.359 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:24.394 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:24.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:24.399 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:00:24.500 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:24.525 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:24.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:24.530 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:00:24.631 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:24.664 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:24.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:24.672 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:24.772 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:24.794 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:24.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:24.796 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:00:24.897 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:24.922 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:24.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:24.927 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:25.028 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:25.051 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:25.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:25.053 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:00:25.154 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:25.176 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:25.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:25.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:00:25.279 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:25.306 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:25.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:25.312 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:00:25.413 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:25.443 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:25.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:25.447 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:00:25.547 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:25.574 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:25.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:25.576 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:25.677 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:25.713 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:25.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:25.719 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:00:25.820 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:25.847 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:25.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:25.849 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:00:25.950 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:25.959 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:25 smithi173 bash[58816]: cluster 2023-10-01T11:00:24.169714+0000 mgr.y (mgr.25072) 1291 : cluster 0 pgmap v1309: 89 pgs: 89 active+clean; 1.2 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2023-10-01T11:00:25.976 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:25.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:25.981 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:00:26.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:25 smithi078 bash[75299]: cluster 2023-10-01T11:00:24.169714+0000 mgr.y (mgr.25072) 1291 : cluster 0 pgmap v1309: 89 pgs: 89 active+clean; 1.2 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2023-10-01T11:00:26.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:25 smithi078 bash[78274]: cluster 2023-10-01T11:00:24.169714+0000 mgr.y (mgr.25072) 1291 : cluster 0 pgmap v1309: 89 pgs: 89 active+clean; 1.2 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2023-10-01T11:00:26.082 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:26.111 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:26.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:26.113 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:26.214 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:26.236 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:26.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:26.238 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:00:26.339 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:26.372 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:26.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:26.375 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:00:26.476 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:26.507 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:26.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:26.509 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:00:26.610 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:26.645 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:26.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:26.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:00:26.751 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:26.783 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:26.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:26.788 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:00:26.889 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:26.915 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:26.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:26.922 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:00:27.022 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:27.070 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:27.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:27.074 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:00:27.175 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:27.207 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:27.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:27.211 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:00:27.312 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:27.351 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:27.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:27.353 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:00:27.454 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:27.503 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:27.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:27.518 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:27.619 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:27.648 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:27.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:27.651 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:00:27.751 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:27.799 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:27.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:27.802 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:00:27.904 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:27.915 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:27 smithi173 bash[58816]: cluster 2023-10-01T11:00:26.170242+0000 mgr.y (mgr.25072) 1292 : cluster 0 pgmap v1310: 89 pgs: 89 active+clean; 1.1 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 640 op/s 2023-10-01T11:00:27.915 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:27 smithi173 bash[58816]: audit 2023-10-01T11:00:27.343413+0000 mon.a (mon.0) 1583 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T11:00:27.932 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:27.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:27.936 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:00:28.037 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:28.045 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:27 smithi078 bash[78274]: cluster 2023-10-01T11:00:26.170242+0000 mgr.y (mgr.25072) 1292 : cluster 0 pgmap v1310: 89 pgs: 89 active+clean; 1.1 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 640 op/s 2023-10-01T11:00:28.046 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:27 smithi078 bash[78274]: audit 2023-10-01T11:00:27.343413+0000 mon.a (mon.0) 1583 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T11:00:28.046 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:27 smithi078 bash[75299]: cluster 2023-10-01T11:00:26.170242+0000 mgr.y (mgr.25072) 1292 : cluster 0 pgmap v1310: 89 pgs: 89 active+clean; 1.1 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 640 op/s 2023-10-01T11:00:28.046 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:27 smithi078 bash[75299]: audit 2023-10-01T11:00:27.343413+0000 mon.a (mon.0) 1583 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T11:00:28.046 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:27 smithi078 bash[75299]: debug 2023-10-01T11:00:27.967+0000 7f31c8dd0700 -1 mon.a@0(leader).osd e254 definitely_dead 0 2023-10-01T11:00:28.060 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:28.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:28.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:00:28.168 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:28.194 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:28.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:28.196 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:00:28.297 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:28.332 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:28.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:28.333 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:00:28.434 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:28.460 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:28.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:28.462 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:28.563 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:28.597 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:28.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:28.604 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:28.704 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:28.728 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:28.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:28.730 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:00:28.802 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:28 smithi078 bash[75299]: audit 2023-10-01T11:00:27.971910+0000 mon.a (mon.0) 1584 : audit 1 from='client.? 172.21.15.78:0/4202119613' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-01T11:00:28.802 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:28 smithi078 bash[75299]: cluster 2023-10-01T11:00:27.972309+0000 mon.a (mon.0) 1585 : cluster 1 Client client.admin marked osd.6 out, while it was still marked up 2023-10-01T11:00:28.813 INFO:teuthology.orchestra.run.smithi078.stderr:marked out osd.6. 2023-10-01T11:00:28.831 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:28.859 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:28.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:28.861 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:00:28.962 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:28.976 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:28 smithi173 bash[58816]: audit 2023-10-01T11:00:27.971910+0000 mon.a (mon.0) 1584 : audit 1 from='client.? 172.21.15.78:0/4202119613' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-01T11:00:28.976 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:28 smithi173 bash[58816]: cluster 2023-10-01T11:00:27.972309+0000 mon.a (mon.0) 1585 : cluster 1 Client client.admin marked osd.6 out, while it was still marked up 2023-10-01T11:00:28.997 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:28.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:28.999 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:00:29.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:28 smithi078 bash[78274]: audit 2023-10-01T11:00:27.971910+0000 mon.a (mon.0) 1584 : audit 1 from='client.? 172.21.15.78:0/4202119613' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-01T11:00:29.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:28 smithi078 bash[78274]: cluster 2023-10-01T11:00:27.972309+0000 mon.a (mon.0) 1585 : cluster 1 Client client.admin marked osd.6 out, while it was still marked up 2023-10-01T11:00:29.100 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:29.134 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:29.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:29.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:00:29.238 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:29.268 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:29.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:29.270 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:00:29.371 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:29.407 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:29.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:29.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:00:29.511 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:29.541 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:29.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:29.545 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:00:29.646 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:29.678 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:29.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:29.680 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:00:29.781 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:29.820 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:29.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:29.828 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:00:29.929 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:29.940 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:29 smithi078 bash[78274]: cluster 2023-10-01T11:00:28.170731+0000 mgr.y (mgr.25072) 1293 : cluster 0 pgmap v1311: 89 pgs: 89 active+clean; 1.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 611 op/s 2023-10-01T11:00:29.941 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:29 smithi078 bash[78274]: cluster 2023-10-01T11:00:28.803109+0000 osd.7 (osd.7) 246 : cluster 3 failed to encode map e255 with expected crc 2023-10-01T11:00:29.941 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:29 smithi078 bash[78274]: audit 2023-10-01T11:00:28.803971+0000 mon.a (mon.0) 1586 : audit 1 from='client.? 172.21.15.78:0/4202119613' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-10-01T11:00:29.941 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:29 smithi078 bash[78274]: cluster 2023-10-01T11:00:28.804043+0000 mon.a (mon.0) 1587 : cluster 0 osdmap e255: 8 total, 8 up, 7 in 2023-10-01T11:00:29.941 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:29 smithi078 bash[78274]: cluster 2023-10-01T11:00:28.805970+0000 osd.6 (osd.6) 205 : cluster 3 failed to encode map e255 with expected crc 2023-10-01T11:00:29.941 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:29 smithi078 bash[75299]: cluster 2023-10-01T11:00:28.170731+0000 mgr.y (mgr.25072) 1293 : cluster 0 pgmap v1311: 89 pgs: 89 active+clean; 1.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 611 op/s 2023-10-01T11:00:29.941 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:29 smithi078 bash[75299]: cluster 2023-10-01T11:00:28.803109+0000 osd.7 (osd.7) 246 : cluster 3 failed to encode map e255 with expected crc 2023-10-01T11:00:29.941 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:29 smithi078 bash[75299]: audit 2023-10-01T11:00:28.803971+0000 mon.a (mon.0) 1586 : audit 1 from='client.? 172.21.15.78:0/4202119613' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-10-01T11:00:29.941 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:29 smithi078 bash[75299]: cluster 2023-10-01T11:00:28.804043+0000 mon.a (mon.0) 1587 : cluster 0 osdmap e255: 8 total, 8 up, 7 in 2023-10-01T11:00:29.942 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:29 smithi078 bash[75299]: cluster 2023-10-01T11:00:28.805970+0000 osd.6 (osd.6) 205 : cluster 3 failed to encode map e255 with expected crc 2023-10-01T11:00:29.953 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:29.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:29.955 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:30.056 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:30.069 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:29 smithi173 bash[58816]: cluster 2023-10-01T11:00:28.170731+0000 mgr.y (mgr.25072) 1293 : cluster 0 pgmap v1311: 89 pgs: 89 active+clean; 1.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 611 op/s 2023-10-01T11:00:30.069 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:29 smithi173 bash[58816]: cluster 2023-10-01T11:00:28.803109+0000 osd.7 (osd.7) 246 : cluster 3 failed to encode map e255 with expected crc 2023-10-01T11:00:30.069 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:29 smithi173 bash[58816]: audit 2023-10-01T11:00:28.803971+0000 mon.a (mon.0) 1586 : audit 1 from='client.? 172.21.15.78:0/4202119613' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-10-01T11:00:30.069 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:29 smithi173 bash[58816]: cluster 2023-10-01T11:00:28.804043+0000 mon.a (mon.0) 1587 : cluster 0 osdmap e255: 8 total, 8 up, 7 in 2023-10-01T11:00:30.069 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:29 smithi173 bash[58816]: cluster 2023-10-01T11:00:28.805970+0000 osd.6 (osd.6) 205 : cluster 3 failed to encode map e255 with expected crc 2023-10-01T11:00:30.087 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:30.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:30.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:00:30.191 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:30.220 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:30.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:30.223 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:00:30.324 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:30.356 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:30.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:30.359 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:00:30.460 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:30.491 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:30.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:30.495 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:00:30.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:00:30 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:00:30] "GET /metrics HTTP/1.1" 200 36384 "" "Prometheus/2.33.4" 2023-10-01T11:00:30.595 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:30.623 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:30.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:30.625 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:00:30.726 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:30.751 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:30.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:30.753 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:00:30.854 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:30.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:30 smithi173 bash[58816]: cluster 2023-10-01T11:00:29.803808+0000 osd.7 (osd.7) 247 : cluster 3 failed to encode map e256 with expected crc 2023-10-01T11:00:30.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:30 smithi173 bash[58816]: cluster 2023-10-01T11:00:29.804908+0000 mon.a (mon.0) 1588 : cluster 0 osdmap e256: 8 total, 8 up, 7 in 2023-10-01T11:00:30.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:30 smithi173 bash[58816]: cluster 2023-10-01T11:00:29.806264+0000 osd.6 (osd.6) 206 : cluster 3 failed to encode map e256 with expected crc 2023-10-01T11:00:30.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:30 smithi173 bash[58816]: cluster 2023-10-01T11:00:29.806615+0000 osd.5 (osd.5) 232 : cluster 3 failed to encode map e256 with expected crc 2023-10-01T11:00:30.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:30 smithi173 bash[58816]: cluster 2023-10-01T11:00:29.809826+0000 osd.5 (osd.5) 233 : cluster 3 failed to encode map e256 with expected crc 2023-10-01T11:00:30.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:30 smithi173 bash[58816]: cluster 2023-10-01T11:00:30.807139+0000 mon.a (mon.0) 1589 : cluster 0 osdmap e257: 8 total, 8 up, 7 in 2023-10-01T11:00:30.888 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:30.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:30.890 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:00:30.991 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:30.998 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:30 smithi078 bash[75299]: cluster 2023-10-01T11:00:29.803808+0000 osd.7 (osd.7) 247 : cluster 3 failed to encode map e256 with expected crc 2023-10-01T11:00:30.998 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:30 smithi078 bash[75299]: cluster 2023-10-01T11:00:29.804908+0000 mon.a (mon.0) 1588 : cluster 0 osdmap e256: 8 total, 8 up, 7 in 2023-10-01T11:00:30.998 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:30 smithi078 bash[75299]: cluster 2023-10-01T11:00:29.806264+0000 osd.6 (osd.6) 206 : cluster 3 failed to encode map e256 with expected crc 2023-10-01T11:00:30.999 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:30 smithi078 bash[75299]: cluster 2023-10-01T11:00:29.806615+0000 osd.5 (osd.5) 232 : cluster 3 failed to encode map e256 with expected crc 2023-10-01T11:00:30.999 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:30 smithi078 bash[75299]: cluster 2023-10-01T11:00:29.809826+0000 osd.5 (osd.5) 233 : cluster 3 failed to encode map e256 with expected crc 2023-10-01T11:00:30.999 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:30 smithi078 bash[75299]: cluster 2023-10-01T11:00:30.807139+0000 mon.a (mon.0) 1589 : cluster 0 osdmap e257: 8 total, 8 up, 7 in 2023-10-01T11:00:30.999 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:30 smithi078 bash[78274]: cluster 2023-10-01T11:00:29.803808+0000 osd.7 (osd.7) 247 : cluster 3 failed to encode map e256 with expected crc 2023-10-01T11:00:30.999 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:30 smithi078 bash[78274]: cluster 2023-10-01T11:00:29.804908+0000 mon.a (mon.0) 1588 : cluster 0 osdmap e256: 8 total, 8 up, 7 in 2023-10-01T11:00:30.999 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:30 smithi078 bash[78274]: cluster 2023-10-01T11:00:29.806264+0000 osd.6 (osd.6) 206 : cluster 3 failed to encode map e256 with expected crc 2023-10-01T11:00:30.999 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:30 smithi078 bash[78274]: cluster 2023-10-01T11:00:29.806615+0000 osd.5 (osd.5) 232 : cluster 3 failed to encode map e256 with expected crc 2023-10-01T11:00:30.999 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:30 smithi078 bash[78274]: cluster 2023-10-01T11:00:29.809826+0000 osd.5 (osd.5) 233 : cluster 3 failed to encode map e256 with expected crc 2023-10-01T11:00:31.000 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:30 smithi078 bash[78274]: cluster 2023-10-01T11:00:30.807139+0000 mon.a (mon.0) 1589 : cluster 0 osdmap e257: 8 total, 8 up, 7 in 2023-10-01T11:00:31.025 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:31.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:31.031 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:31.132 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:31.155 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:31.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:31.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:31.257 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:31.280 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:31.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:31.283 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:00:31.384 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:31.414 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:31.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:31.418 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:00:31.519 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:31.546 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:31.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:31.552 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:00:31.652 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:31.682 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:31.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:31.686 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:00:31.787 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:31.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:31.822 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:31.822 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:00:31.923 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:31.963 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:31.968 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:31 smithi078 bash[78274]: cluster 2023-10-01T11:00:30.171142+0000 mgr.y (mgr.25072) 1294 : cluster 0 pgmap v1314: 89 pgs: 17 peering, 72 active+clean; 949 MiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 779 op/s 2023-10-01T11:00:31.968 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:31 smithi078 bash[78274]: cluster 2023-10-01T11:00:30.808559+0000 osd.5 (osd.5) 234 : cluster 3 failed to encode map e257 with expected crc 2023-10-01T11:00:31.968 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:31 smithi078 bash[78274]: cluster 2023-10-01T11:00:30.808865+0000 osd.3 (osd.3) 24 : cluster 0 12.4 starting backfill to osd.7 from (0'0,0'0] MAX to 254'11966 2023-10-01T11:00:31.969 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:31 smithi078 bash[78274]: cluster 2023-10-01T11:00:30.810317+0000 mon.a (mon.0) 1590 : cluster 3 Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-01T11:00:31.969 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:31 smithi078 bash[78274]: cluster 2023-10-01T11:00:30.810781+0000 osd.6 (osd.6) 207 : cluster 3 failed to encode map e257 with expected crc 2023-10-01T11:00:31.969 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:31 smithi078 bash[78274]: cluster 2023-10-01T11:00:30.811037+0000 osd.7 (osd.7) 248 : cluster 3 failed to encode map e257 with expected crc 2023-10-01T11:00:31.969 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:31 smithi078 bash[78274]: cluster 2023-10-01T11:00:30.812534+0000 osd.3 (osd.3) 25 : cluster 0 12.b starting backfill to osd.4 from (0'0,0'0] MAX to 254'12069 2023-10-01T11:00:31.969 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:31 smithi078 bash[78274]: cluster 2023-10-01T11:00:30.812591+0000 osd.3 (osd.3) 26 : cluster 0 12.1 starting backfill to osd.1 from (0'0,0'0] MAX to 254'11994 2023-10-01T11:00:31.969 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:31 smithi078 bash[78274]: cluster 2023-10-01T11:00:30.813138+0000 osd.4 (osd.4) 4 : cluster 0 12.0 starting backfill to osd.7 from (0'0,0'0] MAX to 254'11868 2023-10-01T11:00:31.969 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:31 smithi078 bash[78274]: cluster 2023-10-01T11:00:30.813725+0000 osd.2 (osd.2) 13 : cluster 0 12.5 starting backfill to osd.3 from (0'0,0'0] MAX to 254'11599 2023-10-01T11:00:31.969 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:31 smithi078 bash[78274]: cluster 2023-10-01T11:00:30.813879+0000 osd.2 (osd.2) 14 : cluster 0 12.9 starting backfill to osd.1 from (0'0,0'0] MAX to 254'11861 2023-10-01T11:00:31.969 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:31 smithi078 bash[78274]: cluster 2023-10-01T11:00:30.816119+0000 osd.5 (osd.5) 235 : cluster 3 failed to encode map e257 with expected crc 2023-10-01T11:00:31.969 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:31 smithi078 bash[78274]: cluster 2023-10-01T11:00:30.819340+0000 osd.5 (osd.5) 236 : cluster 0 12.f starting backfill to osd.4 from (0'0,0'0] MAX to 254'12176 2023-10-01T11:00:31.970 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:31 smithi078 bash[75299]: cluster 2023-10-01T11:00:30.171142+0000 mgr.y (mgr.25072) 1294 : cluster 0 pgmap v1314: 89 pgs: 17 peering, 72 active+clean; 949 MiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 779 op/s 2023-10-01T11:00:31.970 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:31 smithi078 bash[75299]: cluster 2023-10-01T11:00:30.808559+0000 osd.5 (osd.5) 234 : cluster 3 failed to encode map e257 with expected crc 2023-10-01T11:00:31.970 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:31 smithi078 bash[75299]: cluster 2023-10-01T11:00:30.808865+0000 osd.3 (osd.3) 24 : cluster 0 12.4 starting backfill to osd.7 from (0'0,0'0] MAX to 254'11966 2023-10-01T11:00:31.970 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:31 smithi078 bash[75299]: cluster 2023-10-01T11:00:30.810317+0000 mon.a (mon.0) 1590 : cluster 3 Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-01T11:00:31.970 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:31 smithi078 bash[75299]: cluster 2023-10-01T11:00:30.810781+0000 osd.6 (osd.6) 207 : cluster 3 failed to encode map e257 with expected crc 2023-10-01T11:00:31.970 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:31 smithi078 bash[75299]: cluster 2023-10-01T11:00:30.811037+0000 osd.7 (osd.7) 248 : cluster 3 failed to encode map e257 with expected crc 2023-10-01T11:00:31.970 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:31 smithi078 bash[75299]: cluster 2023-10-01T11:00:30.812534+0000 osd.3 (osd.3) 25 : cluster 0 12.b starting backfill to osd.4 from (0'0,0'0] MAX to 254'12069 2023-10-01T11:00:31.971 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:31 smithi078 bash[75299]: cluster 2023-10-01T11:00:30.812591+0000 osd.3 (osd.3) 26 : cluster 0 12.1 starting backfill to osd.1 from (0'0,0'0] MAX to 254'11994 2023-10-01T11:00:31.971 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:31 smithi078 bash[75299]: cluster 2023-10-01T11:00:30.813138+0000 osd.4 (osd.4) 4 : cluster 0 12.0 starting backfill to osd.7 from (0'0,0'0] MAX to 254'11868 2023-10-01T11:00:31.971 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:31 smithi078 bash[75299]: cluster 2023-10-01T11:00:30.813725+0000 osd.2 (osd.2) 13 : cluster 0 12.5 starting backfill to osd.3 from (0'0,0'0] MAX to 254'11599 2023-10-01T11:00:31.971 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:31 smithi078 bash[75299]: cluster 2023-10-01T11:00:30.813879+0000 osd.2 (osd.2) 14 : cluster 0 12.9 starting backfill to osd.1 from (0'0,0'0] MAX to 254'11861 2023-10-01T11:00:31.971 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:31 smithi078 bash[75299]: cluster 2023-10-01T11:00:30.816119+0000 osd.5 (osd.5) 235 : cluster 3 failed to encode map e257 with expected crc 2023-10-01T11:00:31.971 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:31 smithi078 bash[75299]: cluster 2023-10-01T11:00:30.819340+0000 osd.5 (osd.5) 236 : cluster 0 12.f starting backfill to osd.4 from (0'0,0'0] MAX to 254'12176 2023-10-01T11:00:31.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:31.972 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:00:32.073 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:32.105 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:32.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:32.113 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:00:32.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:31 smithi173 bash[58816]: cluster 2023-10-01T11:00:30.171142+0000 mgr.y (mgr.25072) 1294 : cluster 0 pgmap v1314: 89 pgs: 17 peering, 72 active+clean; 949 MiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 779 op/s 2023-10-01T11:00:32.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:31 smithi173 bash[58816]: cluster 2023-10-01T11:00:30.808559+0000 osd.5 (osd.5) 234 : cluster 3 failed to encode map e257 with expected crc 2023-10-01T11:00:32.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:31 smithi173 bash[58816]: cluster 2023-10-01T11:00:30.808865+0000 osd.3 (osd.3) 24 : cluster 0 12.4 starting backfill to osd.7 from (0'0,0'0] MAX to 254'11966 2023-10-01T11:00:32.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:31 smithi173 bash[58816]: cluster 2023-10-01T11:00:30.810317+0000 mon.a (mon.0) 1590 : cluster 3 Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-10-01T11:00:32.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:31 smithi173 bash[58816]: cluster 2023-10-01T11:00:30.810781+0000 osd.6 (osd.6) 207 : cluster 3 failed to encode map e257 with expected crc 2023-10-01T11:00:32.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:31 smithi173 bash[58816]: cluster 2023-10-01T11:00:30.811037+0000 osd.7 (osd.7) 248 : cluster 3 failed to encode map e257 with expected crc 2023-10-01T11:00:32.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:31 smithi173 bash[58816]: cluster 2023-10-01T11:00:30.812534+0000 osd.3 (osd.3) 25 : cluster 0 12.b starting backfill to osd.4 from (0'0,0'0] MAX to 254'12069 2023-10-01T11:00:32.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:31 smithi173 bash[58816]: cluster 2023-10-01T11:00:30.812591+0000 osd.3 (osd.3) 26 : cluster 0 12.1 starting backfill to osd.1 from (0'0,0'0] MAX to 254'11994 2023-10-01T11:00:32.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:31 smithi173 bash[58816]: cluster 2023-10-01T11:00:30.813138+0000 osd.4 (osd.4) 4 : cluster 0 12.0 starting backfill to osd.7 from (0'0,0'0] MAX to 254'11868 2023-10-01T11:00:32.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:31 smithi173 bash[58816]: cluster 2023-10-01T11:00:30.813725+0000 osd.2 (osd.2) 13 : cluster 0 12.5 starting backfill to osd.3 from (0'0,0'0] MAX to 254'11599 2023-10-01T11:00:32.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:31 smithi173 bash[58816]: cluster 2023-10-01T11:00:30.813879+0000 osd.2 (osd.2) 14 : cluster 0 12.9 starting backfill to osd.1 from (0'0,0'0] MAX to 254'11861 2023-10-01T11:00:32.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:31 smithi173 bash[58816]: cluster 2023-10-01T11:00:30.816119+0000 osd.5 (osd.5) 235 : cluster 3 failed to encode map e257 with expected crc 2023-10-01T11:00:32.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:31 smithi173 bash[58816]: cluster 2023-10-01T11:00:30.819340+0000 osd.5 (osd.5) 236 : cluster 0 12.f starting backfill to osd.4 from (0'0,0'0] MAX to 254'12176 2023-10-01T11:00:32.213 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:32.246 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:32.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:32.248 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:00:32.349 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:32.404 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:32.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:32.407 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:00:32.508 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:32.542 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:32.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:32.545 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:00:32.646 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:32.671 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:32.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:32.678 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:00:32.779 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:32.815 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:32.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:32.819 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:00:32.921 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:32.955 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:32.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:32.961 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:00:33.062 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:33.071 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:32 smithi173 bash[58816]: cluster 2023-10-01T11:00:31.841206+0000 mon.a (mon.0) 1591 : cluster 0 osdmap e258: 8 total, 8 up, 7 in 2023-10-01T11:00:33.072 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:32 smithi173 bash[58816]: cluster 2023-10-01T11:00:31.849051+0000 osd.6 (osd.6) 208 : cluster 3 failed to encode map e258 with expected crc 2023-10-01T11:00:33.072 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:32 smithi173 bash[58816]: cluster 2023-10-01T11:00:31.853808+0000 osd.7 (osd.7) 249 : cluster 3 failed to encode map e258 with expected crc 2023-10-01T11:00:33.072 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:32 smithi173 bash[58816]: cluster 2023-10-01T11:00:31.855750+0000 osd.7 (osd.7) 250 : cluster 3 failed to encode map e258 with expected crc 2023-10-01T11:00:33.072 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:32 smithi173 bash[58816]: cluster 2023-10-01T11:00:31.860868+0000 osd.5 (osd.5) 237 : cluster 3 failed to encode map e258 with expected crc 2023-10-01T11:00:33.072 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:32 smithi173 bash[58816]: audit 2023-10-01T11:00:32.457078+0000 mon.a (mon.0) 1592 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:00:33.072 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:32 smithi173 bash[58816]: audit 2023-10-01T11:00:32.462059+0000 mon.a (mon.0) 1593 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:00:33.103 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:33.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:33.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:00:33.212 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:33.243 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:33.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:33.268 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:00:33.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:32 smithi078 bash[75299]: cluster 2023-10-01T11:00:31.841206+0000 mon.a (mon.0) 1591 : cluster 0 osdmap e258: 8 total, 8 up, 7 in 2023-10-01T11:00:33.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:32 smithi078 bash[75299]: cluster 2023-10-01T11:00:31.849051+0000 osd.6 (osd.6) 208 : cluster 3 failed to encode map e258 with expected crc 2023-10-01T11:00:33.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:32 smithi078 bash[75299]: cluster 2023-10-01T11:00:31.853808+0000 osd.7 (osd.7) 249 : cluster 3 failed to encode map e258 with expected crc 2023-10-01T11:00:33.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:32 smithi078 bash[75299]: cluster 2023-10-01T11:00:31.855750+0000 osd.7 (osd.7) 250 : cluster 3 failed to encode map e258 with expected crc 2023-10-01T11:00:33.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:32 smithi078 bash[75299]: cluster 2023-10-01T11:00:31.860868+0000 osd.5 (osd.5) 237 : cluster 3 failed to encode map e258 with expected crc 2023-10-01T11:00:33.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:32 smithi078 bash[75299]: audit 2023-10-01T11:00:32.457078+0000 mon.a (mon.0) 1592 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:00:33.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:32 smithi078 bash[75299]: audit 2023-10-01T11:00:32.462059+0000 mon.a (mon.0) 1593 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:00:33.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:32 smithi078 bash[78274]: cluster 2023-10-01T11:00:31.841206+0000 mon.a (mon.0) 1591 : cluster 0 osdmap e258: 8 total, 8 up, 7 in 2023-10-01T11:00:33.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:32 smithi078 bash[78274]: cluster 2023-10-01T11:00:31.849051+0000 osd.6 (osd.6) 208 : cluster 3 failed to encode map e258 with expected crc 2023-10-01T11:00:33.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:32 smithi078 bash[78274]: cluster 2023-10-01T11:00:31.853808+0000 osd.7 (osd.7) 249 : cluster 3 failed to encode map e258 with expected crc 2023-10-01T11:00:33.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:32 smithi078 bash[78274]: cluster 2023-10-01T11:00:31.855750+0000 osd.7 (osd.7) 250 : cluster 3 failed to encode map e258 with expected crc 2023-10-01T11:00:33.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:32 smithi078 bash[78274]: cluster 2023-10-01T11:00:31.860868+0000 osd.5 (osd.5) 237 : cluster 3 failed to encode map e258 with expected crc 2023-10-01T11:00:33.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:32 smithi078 bash[78274]: audit 2023-10-01T11:00:32.457078+0000 mon.a (mon.0) 1592 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:00:33.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:32 smithi078 bash[78274]: audit 2023-10-01T11:00:32.462059+0000 mon.a (mon.0) 1593 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:00:33.369 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:33.398 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:33.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:33.403 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:00:33.504 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:33.547 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:33.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:33.553 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:00:33.654 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:33.682 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:33.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:33.687 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:00:33.787 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:33.813 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:33.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:33.817 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:00:33.855 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:33 smithi173 bash[58816]: cluster 2023-10-01T11:00:32.173868+0000 mgr.y (mgr.25072) 1295 : cluster 0 pgmap v1317: 89 pgs: 17 peering, 72 active+clean; 949 MiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 535 op/s 2023-10-01T11:00:33.855 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:33 smithi173 bash[58816]: cluster 2023-10-01T11:00:32.849662+0000 osd.5 (osd.5) 238 : cluster 3 failed to encode map e259 with expected crc 2023-10-01T11:00:33.855 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:33 smithi173 bash[58816]: cluster 2023-10-01T11:00:32.850874+0000 osd.7 (osd.7) 251 : cluster 3 failed to encode map e259 with expected crc 2023-10-01T11:00:33.855 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:33 smithi173 bash[58816]: cluster 2023-10-01T11:00:32.854861+0000 mon.a (mon.0) 1594 : cluster 0 osdmap e259: 8 total, 8 up, 7 in 2023-10-01T11:00:33.855 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:33 smithi173 bash[58816]: cluster 2023-10-01T11:00:32.871476+0000 osd.6 (osd.6) 209 : cluster 3 failed to encode map e259 with expected crc 2023-10-01T11:00:33.856 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:33 smithi173 bash[58816]: cluster 2023-10-01T11:00:32.873531+0000 osd.6 (osd.6) 210 : cluster 3 failed to encode map e259 with expected crc 2023-10-01T11:00:33.918 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:33.927 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:33 smithi078 bash[75299]: cluster 2023-10-01T11:00:32.173868+0000 mgr.y (mgr.25072) 1295 : cluster 0 pgmap v1317: 89 pgs: 17 peering, 72 active+clean; 949 MiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 535 op/s 2023-10-01T11:00:33.928 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:33 smithi078 bash[75299]: cluster 2023-10-01T11:00:32.849662+0000 osd.5 (osd.5) 238 : cluster 3 failed to encode map e259 with expected crc 2023-10-01T11:00:33.928 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:33 smithi078 bash[75299]: cluster 2023-10-01T11:00:32.850874+0000 osd.7 (osd.7) 251 : cluster 3 failed to encode map e259 with expected crc 2023-10-01T11:00:33.928 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:33 smithi078 bash[75299]: cluster 2023-10-01T11:00:32.854861+0000 mon.a (mon.0) 1594 : cluster 0 osdmap e259: 8 total, 8 up, 7 in 2023-10-01T11:00:33.928 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:33 smithi078 bash[75299]: cluster 2023-10-01T11:00:32.871476+0000 osd.6 (osd.6) 209 : cluster 3 failed to encode map e259 with expected crc 2023-10-01T11:00:33.928 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:33 smithi078 bash[75299]: cluster 2023-10-01T11:00:32.873531+0000 osd.6 (osd.6) 210 : cluster 3 failed to encode map e259 with expected crc 2023-10-01T11:00:33.928 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:33 smithi078 bash[78274]: cluster 2023-10-01T11:00:32.173868+0000 mgr.y (mgr.25072) 1295 : cluster 0 pgmap v1317: 89 pgs: 17 peering, 72 active+clean; 949 MiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 535 op/s 2023-10-01T11:00:33.928 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:33 smithi078 bash[78274]: cluster 2023-10-01T11:00:32.849662+0000 osd.5 (osd.5) 238 : cluster 3 failed to encode map e259 with expected crc 2023-10-01T11:00:33.928 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:33 smithi078 bash[78274]: cluster 2023-10-01T11:00:32.850874+0000 osd.7 (osd.7) 251 : cluster 3 failed to encode map e259 with expected crc 2023-10-01T11:00:33.929 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:33 smithi078 bash[78274]: cluster 2023-10-01T11:00:32.854861+0000 mon.a (mon.0) 1594 : cluster 0 osdmap e259: 8 total, 8 up, 7 in 2023-10-01T11:00:33.929 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:33 smithi078 bash[78274]: cluster 2023-10-01T11:00:32.871476+0000 osd.6 (osd.6) 209 : cluster 3 failed to encode map e259 with expected crc 2023-10-01T11:00:33.929 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:33 smithi078 bash[78274]: cluster 2023-10-01T11:00:32.873531+0000 osd.6 (osd.6) 210 : cluster 3 failed to encode map e259 with expected crc 2023-10-01T11:00:33.947 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:33.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:33.950 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:00:34.051 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:34.076 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:34.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:34.080 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:00:34.181 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:34.211 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:34.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:34.215 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:00:34.316 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:34.354 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:34.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:34.356 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:00:34.457 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:34.495 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:34.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:34.497 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:00:34.598 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:34.629 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:34.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:34.634 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:00:34.735 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:34.774 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:34.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:34.780 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:00:34.881 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:34.912 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:34.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:34.923 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:00:35.024 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:35.050 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:35.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:35.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:00:35.154 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:35.167 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:34 smithi078 bash[75299]: audit 2023-10-01T11:00:33.955263+0000 mon.a (mon.0) 1595 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:00:35.167 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:34 smithi078 bash[75299]: audit 2023-10-01T11:00:33.971130+0000 mon.a (mon.0) 1596 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:00:35.168 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:35 smithi078 bash[78274]: audit 2023-10-01T11:00:33.955263+0000 mon.a (mon.0) 1595 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:00:35.168 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:35 smithi078 bash[78274]: audit 2023-10-01T11:00:33.971130+0000 mon.a (mon.0) 1596 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:00:35.195 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:35.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:35.199 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:00:35.300 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:35.310 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:34 smithi173 bash[58816]: audit 2023-10-01T11:00:33.955263+0000 mon.a (mon.0) 1595 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:00:35.311 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:34 smithi173 bash[58816]: audit 2023-10-01T11:00:33.971130+0000 mon.a (mon.0) 1596 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:00:35.325 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:35.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:35.331 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:00:35.432 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:35.467 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:35.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:35.470 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:00:35.570 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:35.617 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:35.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:35.623 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:35.723 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:35.760 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:35.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:35.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:00:35.865 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:35.894 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:35.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:35.896 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:00:35.980 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:35 smithi078 bash[75299]: cluster 2023-10-01T11:00:34.174463+0000 mgr.y (mgr.25072) 1296 : cluster 0 pgmap v1319: 89 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 17 peering, 68 active+clean; 888 MiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 224 op/s; 3075/42648 objects misplaced (7.210%); 5.6 MiB/s, 88 objects/s recovering 2023-10-01T11:00:35.980 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:35 smithi078 bash[75299]: audit 2023-10-01T11:00:35.865395+0000 mon.a (mon.0) 1597 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:00:35.980 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:35 smithi078 bash[75299]: audit 2023-10-01T11:00:35.870193+0000 mon.a (mon.0) 1598 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:00:35.997 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:36.029 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:36.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:36.038 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:00:36.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:35 smithi173 bash[58816]: cluster 2023-10-01T11:00:34.174463+0000 mgr.y (mgr.25072) 1296 : cluster 0 pgmap v1319: 89 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 17 peering, 68 active+clean; 888 MiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 224 op/s; 3075/42648 objects misplaced (7.210%); 5.6 MiB/s, 88 objects/s recovering 2023-10-01T11:00:36.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:35 smithi173 bash[58816]: audit 2023-10-01T11:00:35.865395+0000 mon.a (mon.0) 1597 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:00:36.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:35 smithi173 bash[58816]: audit 2023-10-01T11:00:35.870193+0000 mon.a (mon.0) 1598 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:00:36.139 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:36.166 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:36.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:36.168 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:00:36.238 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:35 smithi078 bash[78274]: cluster 2023-10-01T11:00:34.174463+0000 mgr.y (mgr.25072) 1296 : cluster 0 pgmap v1319: 89 pgs: 1 active+remapped+backfilling, 3 active+remapped+backfill_wait, 17 peering, 68 active+clean; 888 MiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 224 op/s; 3075/42648 objects misplaced (7.210%); 5.6 MiB/s, 88 objects/s recovering 2023-10-01T11:00:36.238 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:35 smithi078 bash[78274]: audit 2023-10-01T11:00:35.865395+0000 mon.a (mon.0) 1597 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:00:36.238 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:35 smithi078 bash[78274]: audit 2023-10-01T11:00:35.870193+0000 mon.a (mon.0) 1598 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:00:36.269 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:36.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:36.303 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:36.303 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:36.404 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:36.426 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:36.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:36.428 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:00:36.466 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2023-10-01T11:00:36.467 INFO:teuthology.orchestra.run.smithi078.stdout: "ops": [], 2023-10-01T11:00:36.467 INFO:teuthology.orchestra.run.smithi078.stdout: "complaint_time": 30, 2023-10-01T11:00:36.467 INFO:teuthology.orchestra.run.smithi078.stdout: "num_blocked_ops": 0 2023-10-01T11:00:36.467 INFO:teuthology.orchestra.run.smithi078.stdout:} 2023-10-01T11:00:36.529 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:36.558 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:36.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:36.561 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:00:36.661 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:36.689 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:36.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:36.692 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:00:36.793 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:36.815 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:36.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:36.816 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:00:36.861 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T11:00:36.891 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T11:00:36.894 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 7] out_osds: [6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T11:00:36.894 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T11:00:36.894 INFO:tasks.thrashosds.thrasher:Adding osd 6 2023-10-01T11:00:36.894 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd in 6 2023-10-01T11:00:36.917 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:36.921 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T11:00:36.947 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T11:00:36.947 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:36.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:36.950 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:00:36.973 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T11:00:37.011 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T11:00:37.050 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T11:00:37.051 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:37.077 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:37.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:37.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:00:37.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:36 smithi078 bash[75299]: cluster 2023-10-01T11:00:35.971117+0000 osd.7 (osd.7) 252 : cluster 3 failed to encode map e260 with expected crc 2023-10-01T11:00:37.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:36 smithi078 bash[75299]: cluster 2023-10-01T11:00:35.973512+0000 osd.5 (osd.5) 239 : cluster 3 failed to encode map e260 with expected crc 2023-10-01T11:00:37.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:36 smithi078 bash[75299]: cluster 2023-10-01T11:00:35.974049+0000 osd.7 (osd.7) 253 : cluster 3 failed to encode map e260 with expected crc 2023-10-01T11:00:37.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:36 smithi078 bash[75299]: cluster 2023-10-01T11:00:35.974138+0000 mon.a (mon.0) 1599 : cluster 0 osdmap e260: 8 total, 8 up, 7 in 2023-10-01T11:00:37.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:36 smithi078 bash[75299]: cluster 2023-10-01T11:00:35.976280+0000 osd.5 (osd.5) 240 : cluster 3 failed to encode map e260 with expected crc 2023-10-01T11:00:37.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:36 smithi078 bash[75299]: cluster 2023-10-01T11:00:35.978418+0000 osd.6 (osd.6) 211 : cluster 3 failed to encode map e260 with expected crc 2023-10-01T11:00:37.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:36 smithi078 bash[75299]: audit 2023-10-01T11:00:36.444738+0000 mon.a (mon.0) 1600 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T11:00:37.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:36 smithi078 bash[75299]: audit 2023-10-01T11:00:36.445723+0000 mon.a (mon.0) 1601 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T11:00:37.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:36 smithi078 bash[75299]: audit 2023-10-01T11:00:36.452062+0000 mon.a (mon.0) 1602 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:00:37.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:36 smithi078 bash[75299]: audit 2023-10-01T11:00:36.828909+0000 mon.c (mon.2) 199 : audit 0 from='client.? 172.21.15.78:0/1987907691' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:00:37.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:36 smithi078 bash[78274]: cluster 2023-10-01T11:00:35.971117+0000 osd.7 (osd.7) 252 : cluster 3 failed to encode map e260 with expected crc 2023-10-01T11:00:37.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:36 smithi078 bash[78274]: cluster 2023-10-01T11:00:35.973512+0000 osd.5 (osd.5) 239 : cluster 3 failed to encode map e260 with expected crc 2023-10-01T11:00:37.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:36 smithi078 bash[78274]: cluster 2023-10-01T11:00:35.974049+0000 osd.7 (osd.7) 253 : cluster 3 failed to encode map e260 with expected crc 2023-10-01T11:00:37.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:36 smithi078 bash[78274]: cluster 2023-10-01T11:00:35.974138+0000 mon.a (mon.0) 1599 : cluster 0 osdmap e260: 8 total, 8 up, 7 in 2023-10-01T11:00:37.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:36 smithi078 bash[78274]: cluster 2023-10-01T11:00:35.976280+0000 osd.5 (osd.5) 240 : cluster 3 failed to encode map e260 with expected crc 2023-10-01T11:00:37.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:36 smithi078 bash[78274]: cluster 2023-10-01T11:00:35.978418+0000 osd.6 (osd.6) 211 : cluster 3 failed to encode map e260 with expected crc 2023-10-01T11:00:37.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:36 smithi078 bash[78274]: audit 2023-10-01T11:00:36.444738+0000 mon.a (mon.0) 1600 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T11:00:37.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:36 smithi078 bash[78274]: audit 2023-10-01T11:00:36.445723+0000 mon.a (mon.0) 1601 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T11:00:37.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:36 smithi078 bash[78274]: audit 2023-10-01T11:00:36.452062+0000 mon.a (mon.0) 1602 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:00:37.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:36 smithi078 bash[78274]: audit 2023-10-01T11:00:36.828909+0000 mon.c (mon.2) 199 : audit 0 from='client.? 172.21.15.78:0/1987907691' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:00:37.088 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T11:00:37.099 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T11:00:37.099 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T11:00:37.099 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T11:00:37.099 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T11:00:37.099 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T11:00:37.099 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T11:00:37.099 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T11:00:37.100 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T11:00:37.179 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:37.188 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:36 smithi173 bash[58816]: cluster 2023-10-01T11:00:35.971117+0000 osd.7 (osd.7) 252 : cluster 3 failed to encode map e260 with expected crc 2023-10-01T11:00:37.188 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:36 smithi173 bash[58816]: cluster 2023-10-01T11:00:35.973512+0000 osd.5 (osd.5) 239 : cluster 3 failed to encode map e260 with expected crc 2023-10-01T11:00:37.188 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:36 smithi173 bash[58816]: cluster 2023-10-01T11:00:35.974049+0000 osd.7 (osd.7) 253 : cluster 3 failed to encode map e260 with expected crc 2023-10-01T11:00:37.188 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:36 smithi173 bash[58816]: cluster 2023-10-01T11:00:35.974138+0000 mon.a (mon.0) 1599 : cluster 0 osdmap e260: 8 total, 8 up, 7 in 2023-10-01T11:00:37.188 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:36 smithi173 bash[58816]: cluster 2023-10-01T11:00:35.976280+0000 osd.5 (osd.5) 240 : cluster 3 failed to encode map e260 with expected crc 2023-10-01T11:00:37.188 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:36 smithi173 bash[58816]: cluster 2023-10-01T11:00:35.978418+0000 osd.6 (osd.6) 211 : cluster 3 failed to encode map e260 with expected crc 2023-10-01T11:00:37.188 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:36 smithi173 bash[58816]: audit 2023-10-01T11:00:36.444738+0000 mon.a (mon.0) 1600 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T11:00:37.189 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:36 smithi173 bash[58816]: audit 2023-10-01T11:00:36.445723+0000 mon.a (mon.0) 1601 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T11:00:37.189 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:36 smithi173 bash[58816]: audit 2023-10-01T11:00:36.452062+0000 mon.a (mon.0) 1602 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:00:37.189 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:36 smithi173 bash[58816]: audit 2023-10-01T11:00:36.828909+0000 mon.c (mon.2) 199 : audit 0 from='client.? 172.21.15.78:0/1987907691' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:00:37.204 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:37.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:37.206 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:00:37.307 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:37.332 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:37.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:37.335 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:00:37.436 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:37.491 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:37.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:37.497 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:00:37.597 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:37.621 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:37.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:37.626 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:00:37.727 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:37.757 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:37.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:37.767 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:00:37.868 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:37.904 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:37.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:37.908 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:38.009 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:38.010 INFO:teuthology.orchestra.run.smithi078.stderr:noscrub is unset 2023-10-01T11:00:38.040 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:38.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:38.047 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:00:38.148 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:38.179 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:38.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:38.182 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:00:38.283 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:38.291 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:38 smithi078 bash[78274]: cluster 2023-10-01T11:00:36.174896+0000 mgr.y (mgr.25072) 1297 : cluster 0 pgmap v1321: 89 pgs: 3 active+remapped+backfilling, 4 active+remapped+backfill_wait, 82 active+clean; 845 MiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 308 op/s; 4721/40620 objects misplaced (11.622%); 14 MiB/s, 224 objects/s recovering 2023-10-01T11:00:38.292 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:38 smithi078 bash[78274]: cluster 2023-10-01T11:00:36.978305+0000 mon.a (mon.0) 1603 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-10-01T11:00:38.292 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:38 smithi078 bash[78274]: cluster 2023-10-01T11:00:36.986411+0000 osd.6 (osd.6) 212 : cluster 3 failed to encode map e261 with expected crc 2023-10-01T11:00:38.292 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:38 smithi078 bash[78274]: cluster 2023-10-01T11:00:36.987612+0000 osd.7 (osd.7) 254 : cluster 3 failed to encode map e261 with expected crc 2023-10-01T11:00:38.292 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:38 smithi078 bash[78274]: cluster 2023-10-01T11:00:36.988680+0000 mon.a (mon.0) 1604 : cluster 0 osdmap e261: 8 total, 8 up, 7 in 2023-10-01T11:00:38.292 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:38 smithi078 bash[78274]: cluster 2023-10-01T11:00:36.991396+0000 osd.7 (osd.7) 255 : cluster 3 failed to encode map e261 with expected crc 2023-10-01T11:00:38.292 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:38 smithi078 bash[78274]: cluster 2023-10-01T11:00:36.991701+0000 osd.5 (osd.5) 241 : cluster 3 failed to encode map e261 with expected crc 2023-10-01T11:00:38.292 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:38 smithi078 bash[78274]: cluster 2023-10-01T11:00:36.994413+0000 osd.5 (osd.5) 242 : cluster 3 failed to encode map e261 with expected crc 2023-10-01T11:00:38.292 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:38 smithi078 bash[78274]: cluster 2023-10-01T11:00:36.996190+0000 osd.5 (osd.5) 243 : cluster 3 failed to encode map e261 with expected crc 2023-10-01T11:00:38.292 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:38 smithi078 bash[78274]: audit 2023-10-01T11:00:37.385623+0000 mon.c (mon.2) 200 : audit 1 from='client.? 172.21.15.78:0/2991360495' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T11:00:38.292 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:38 smithi078 bash[78274]: audit 2023-10-01T11:00:37.386292+0000 mon.a (mon.0) 1605 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T11:00:38.293 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:37 smithi078 bash[75299]: cluster 2023-10-01T11:00:36.174896+0000 mgr.y (mgr.25072) 1297 : cluster 0 pgmap v1321: 89 pgs: 3 active+remapped+backfilling, 4 active+remapped+backfill_wait, 82 active+clean; 845 MiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 308 op/s; 4721/40620 objects misplaced (11.622%); 14 MiB/s, 224 objects/s recovering 2023-10-01T11:00:38.293 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:38 smithi078 bash[75299]: cluster 2023-10-01T11:00:36.978305+0000 mon.a (mon.0) 1603 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-10-01T11:00:38.293 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:38 smithi078 bash[75299]: cluster 2023-10-01T11:00:36.986411+0000 osd.6 (osd.6) 212 : cluster 3 failed to encode map e261 with expected crc 2023-10-01T11:00:38.293 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:38 smithi078 bash[75299]: cluster 2023-10-01T11:00:36.987612+0000 osd.7 (osd.7) 254 : cluster 3 failed to encode map e261 with expected crc 2023-10-01T11:00:38.293 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:38 smithi078 bash[75299]: cluster 2023-10-01T11:00:36.988680+0000 mon.a (mon.0) 1604 : cluster 0 osdmap e261: 8 total, 8 up, 7 in 2023-10-01T11:00:38.293 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:38 smithi078 bash[75299]: cluster 2023-10-01T11:00:36.991396+0000 osd.7 (osd.7) 255 : cluster 3 failed to encode map e261 with expected crc 2023-10-01T11:00:38.293 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:38 smithi078 bash[75299]: cluster 2023-10-01T11:00:36.991701+0000 osd.5 (osd.5) 241 : cluster 3 failed to encode map e261 with expected crc 2023-10-01T11:00:38.294 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:38 smithi078 bash[75299]: cluster 2023-10-01T11:00:36.994413+0000 osd.5 (osd.5) 242 : cluster 3 failed to encode map e261 with expected crc 2023-10-01T11:00:38.294 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:38 smithi078 bash[75299]: cluster 2023-10-01T11:00:36.996190+0000 osd.5 (osd.5) 243 : cluster 3 failed to encode map e261 with expected crc 2023-10-01T11:00:38.294 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:38 smithi078 bash[75299]: audit 2023-10-01T11:00:37.385623+0000 mon.c (mon.2) 200 : audit 1 from='client.? 172.21.15.78:0/2991360495' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T11:00:38.294 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:38 smithi078 bash[75299]: audit 2023-10-01T11:00:37.386292+0000 mon.a (mon.0) 1605 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T11:00:38.306 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:38.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:38.312 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:00:38.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:37 smithi173 bash[58816]: cluster 2023-10-01T11:00:36.174896+0000 mgr.y (mgr.25072) 1297 : cluster 0 pgmap v1321: 89 pgs: 3 active+remapped+backfilling, 4 active+remapped+backfill_wait, 82 active+clean; 845 MiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 308 op/s; 4721/40620 objects misplaced (11.622%); 14 MiB/s, 224 objects/s recovering 2023-10-01T11:00:38.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:37 smithi173 bash[58816]: cluster 2023-10-01T11:00:36.978305+0000 mon.a (mon.0) 1603 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-10-01T11:00:38.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:37 smithi173 bash[58816]: cluster 2023-10-01T11:00:36.986411+0000 osd.6 (osd.6) 212 : cluster 3 failed to encode map e261 with expected crc 2023-10-01T11:00:38.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:37 smithi173 bash[58816]: cluster 2023-10-01T11:00:36.987612+0000 osd.7 (osd.7) 254 : cluster 3 failed to encode map e261 with expected crc 2023-10-01T11:00:38.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:37 smithi173 bash[58816]: cluster 2023-10-01T11:00:36.988680+0000 mon.a (mon.0) 1604 : cluster 0 osdmap e261: 8 total, 8 up, 7 in 2023-10-01T11:00:38.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:37 smithi173 bash[58816]: cluster 2023-10-01T11:00:36.991396+0000 osd.7 (osd.7) 255 : cluster 3 failed to encode map e261 with expected crc 2023-10-01T11:00:38.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:37 smithi173 bash[58816]: cluster 2023-10-01T11:00:36.991701+0000 osd.5 (osd.5) 241 : cluster 3 failed to encode map e261 with expected crc 2023-10-01T11:00:38.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:37 smithi173 bash[58816]: cluster 2023-10-01T11:00:36.994413+0000 osd.5 (osd.5) 242 : cluster 3 failed to encode map e261 with expected crc 2023-10-01T11:00:38.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:37 smithi173 bash[58816]: cluster 2023-10-01T11:00:36.996190+0000 osd.5 (osd.5) 243 : cluster 3 failed to encode map e261 with expected crc 2023-10-01T11:00:38.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:37 smithi173 bash[58816]: audit 2023-10-01T11:00:37.385623+0000 mon.c (mon.2) 200 : audit 1 from='client.? 172.21.15.78:0/2991360495' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T11:00:38.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:37 smithi173 bash[58816]: audit 2023-10-01T11:00:37.386292+0000 mon.a (mon.0) 1605 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T11:00:38.413 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:38.443 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:38.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:38.445 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:00:38.546 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:38.568 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:38.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:38.570 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:00:38.671 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:38.702 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:38.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:38.705 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:00:38.806 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:38.832 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:38.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:38.835 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:00:38.936 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:38.966 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:38.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:38.969 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:00:39.070 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:39.102 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:39.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:39.105 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:00:39.205 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:39.213 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:38 smithi078 bash[75299]: cluster 2023-10-01T11:00:37.985528+0000 mon.a (mon.0) 1606 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:00:39.213 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:38 smithi078 bash[75299]: cluster 2023-10-01T11:00:38.005637+0000 osd.7 (osd.7) 256 : cluster 3 failed to encode map e262 with expected crc 2023-10-01T11:00:39.214 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:38 smithi078 bash[75299]: audit 2023-10-01T11:00:38.006437+0000 mon.a (mon.0) 1607 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T11:00:39.214 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:38 smithi078 bash[75299]: cluster 2023-10-01T11:00:38.006592+0000 mon.a (mon.0) 1608 : cluster 0 osdmap e262: 8 total, 8 up, 7 in 2023-10-01T11:00:39.214 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:38 smithi078 bash[75299]: cluster 2023-10-01T11:00:38.030620+0000 osd.5 (osd.5) 244 : cluster 3 failed to encode map e262 with expected crc 2023-10-01T11:00:39.214 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:38 smithi078 bash[75299]: cluster 2023-10-01T11:00:38.030726+0000 osd.6 (osd.6) 213 : cluster 3 failed to encode map e262 with expected crc 2023-10-01T11:00:39.214 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:38 smithi078 bash[78274]: cluster 2023-10-01T11:00:37.985528+0000 mon.a (mon.0) 1606 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:00:39.214 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:38 smithi078 bash[78274]: cluster 2023-10-01T11:00:38.005637+0000 osd.7 (osd.7) 256 : cluster 3 failed to encode map e262 with expected crc 2023-10-01T11:00:39.214 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:38 smithi078 bash[78274]: audit 2023-10-01T11:00:38.006437+0000 mon.a (mon.0) 1607 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T11:00:39.214 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:38 smithi078 bash[78274]: cluster 2023-10-01T11:00:38.006592+0000 mon.a (mon.0) 1608 : cluster 0 osdmap e262: 8 total, 8 up, 7 in 2023-10-01T11:00:39.215 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:38 smithi078 bash[78274]: cluster 2023-10-01T11:00:38.030620+0000 osd.5 (osd.5) 244 : cluster 3 failed to encode map e262 with expected crc 2023-10-01T11:00:39.215 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:38 smithi078 bash[78274]: cluster 2023-10-01T11:00:38.030726+0000 osd.6 (osd.6) 213 : cluster 3 failed to encode map e262 with expected crc 2023-10-01T11:00:39.232 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:39.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:39.239 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:39.340 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:39.364 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:39.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:39.366 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:00:39.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:38 smithi173 bash[58816]: cluster 2023-10-01T11:00:37.985528+0000 mon.a (mon.0) 1606 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:00:39.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:38 smithi173 bash[58816]: cluster 2023-10-01T11:00:38.005637+0000 osd.7 (osd.7) 256 : cluster 3 failed to encode map e262 with expected crc 2023-10-01T11:00:39.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:38 smithi173 bash[58816]: audit 2023-10-01T11:00:38.006437+0000 mon.a (mon.0) 1607 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T11:00:39.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:38 smithi173 bash[58816]: cluster 2023-10-01T11:00:38.006592+0000 mon.a (mon.0) 1608 : cluster 0 osdmap e262: 8 total, 8 up, 7 in 2023-10-01T11:00:39.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:38 smithi173 bash[58816]: cluster 2023-10-01T11:00:38.030620+0000 osd.5 (osd.5) 244 : cluster 3 failed to encode map e262 with expected crc 2023-10-01T11:00:39.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:38 smithi173 bash[58816]: cluster 2023-10-01T11:00:38.030726+0000 osd.6 (osd.6) 213 : cluster 3 failed to encode map e262 with expected crc 2023-10-01T11:00:39.467 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:39.486 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:39.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:39.489 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:39.589 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:39.620 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:39.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:39.623 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:00:39.723 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:39.749 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:39.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:39.751 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:00:39.852 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:39.880 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:39.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:39.883 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:00:39.983 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:40.003 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:40.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:40.006 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:40.006 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:40 smithi078 bash[75299]: cluster 2023-10-01T11:00:38.175425+0000 mgr.y (mgr.25072) 1298 : cluster 0 pgmap v1324: 89 pgs: 4 active+remapped+backfilling, 3 active+remapped+backfill_wait, 82 active+clean; 808 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 422 op/s; 3836/38847 objects misplaced (9.875%); 20 MiB/s, 314 objects/s recovering 2023-10-01T11:00:40.006 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:40 smithi078 bash[75299]: cluster 2023-10-01T11:00:39.011945+0000 osd.7 (osd.7) 257 : cluster 3 failed to encode map e263 with expected crc 2023-10-01T11:00:40.006 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:40 smithi078 bash[75299]: cluster 2023-10-01T11:00:39.013423+0000 mon.a (mon.0) 1609 : cluster 0 osdmap e263: 8 total, 8 up, 7 in 2023-10-01T11:00:40.006 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:40 smithi078 bash[75299]: cluster 2023-10-01T11:00:39.016126+0000 osd.6 (osd.6) 214 : cluster 3 failed to encode map e263 with expected crc 2023-10-01T11:00:40.007 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:40 smithi078 bash[75299]: cluster 2023-10-01T11:00:39.074481+0000 osd.5 (osd.5) 245 : cluster 3 failed to encode map e263 with expected crc 2023-10-01T11:00:40.106 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:40.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:40 smithi173 bash[58816]: cluster 2023-10-01T11:00:38.175425+0000 mgr.y (mgr.25072) 1298 : cluster 0 pgmap v1324: 89 pgs: 4 active+remapped+backfilling, 3 active+remapped+backfill_wait, 82 active+clean; 808 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 422 op/s; 3836/38847 objects misplaced (9.875%); 20 MiB/s, 314 objects/s recovering 2023-10-01T11:00:40.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:40 smithi173 bash[58816]: cluster 2023-10-01T11:00:39.011945+0000 osd.7 (osd.7) 257 : cluster 3 failed to encode map e263 with expected crc 2023-10-01T11:00:40.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:40 smithi173 bash[58816]: cluster 2023-10-01T11:00:39.013423+0000 mon.a (mon.0) 1609 : cluster 0 osdmap e263: 8 total, 8 up, 7 in 2023-10-01T11:00:40.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:40 smithi173 bash[58816]: cluster 2023-10-01T11:00:39.016126+0000 osd.6 (osd.6) 214 : cluster 3 failed to encode map e263 with expected crc 2023-10-01T11:00:40.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:40 smithi173 bash[58816]: cluster 2023-10-01T11:00:39.074481+0000 osd.5 (osd.5) 245 : cluster 3 failed to encode map e263 with expected crc 2023-10-01T11:00:40.136 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:40.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:40.138 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:00:40.239 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:40.262 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:40.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:40.264 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:40.284 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:00:40 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:00:40] "GET /metrics HTTP/1.1" 200 36376 "" "Prometheus/2.33.4" 2023-10-01T11:00:40.284 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:40 smithi078 bash[78274]: cluster 2023-10-01T11:00:38.175425+0000 mgr.y (mgr.25072) 1298 : cluster 0 pgmap v1324: 89 pgs: 4 active+remapped+backfilling, 3 active+remapped+backfill_wait, 82 active+clean; 808 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 422 op/s; 3836/38847 objects misplaced (9.875%); 20 MiB/s, 314 objects/s recovering 2023-10-01T11:00:40.284 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:40 smithi078 bash[78274]: cluster 2023-10-01T11:00:39.011945+0000 osd.7 (osd.7) 257 : cluster 3 failed to encode map e263 with expected crc 2023-10-01T11:00:40.284 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:40 smithi078 bash[78274]: cluster 2023-10-01T11:00:39.013423+0000 mon.a (mon.0) 1609 : cluster 0 osdmap e263: 8 total, 8 up, 7 in 2023-10-01T11:00:40.285 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:40 smithi078 bash[78274]: cluster 2023-10-01T11:00:39.016126+0000 osd.6 (osd.6) 214 : cluster 3 failed to encode map e263 with expected crc 2023-10-01T11:00:40.285 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:40 smithi078 bash[78274]: cluster 2023-10-01T11:00:39.074481+0000 osd.5 (osd.5) 245 : cluster 3 failed to encode map e263 with expected crc 2023-10-01T11:00:40.365 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:40.388 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:40.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:40.392 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:00:40.493 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:40.518 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:40.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:40.522 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:00:40.623 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:40.644 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:40.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:40.653 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:00:40.754 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:40.786 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:40.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:40.788 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:00:40.888 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:40.914 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:40.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:40.916 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:00:41.017 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:41.020 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:41 smithi173 bash[58816]: cluster 2023-10-01T11:00:40.012931+0000 osd.5 (osd.5) 246 : cluster 3 failed to encode map e264 with expected crc 2023-10-01T11:00:41.020 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:41 smithi173 bash[58816]: cluster 2023-10-01T11:00:40.014439+0000 mon.a (mon.0) 1610 : cluster 0 osdmap e264: 8 total, 8 up, 7 in 2023-10-01T11:00:41.021 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:41 smithi173 bash[58816]: cluster 2023-10-01T11:00:40.016849+0000 osd.6 (osd.6) 215 : cluster 3 failed to encode map e264 with expected crc 2023-10-01T11:00:41.021 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:41 smithi173 bash[58816]: cluster 2023-10-01T11:00:40.018950+0000 osd.7 (osd.7) 258 : cluster 3 failed to encode map e264 with expected crc 2023-10-01T11:00:41.021 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:41 smithi173 bash[58816]: cluster 2023-10-01T11:00:40.175869+0000 mgr.y (mgr.25072) 1299 : cluster 0 pgmap v1327: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 85 active+clean; 717 MiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 513 op/s; 1941/34458 objects misplaced (5.633%); 34 MiB/s, 545 objects/s recovering 2023-10-01T11:00:41.041 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:41.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:41.043 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:00:41.144 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:41.172 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:41.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:41.175 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:00:41.259 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:41 smithi078 bash[75299]: cluster 2023-10-01T11:00:40.012931+0000 osd.5 (osd.5) 246 : cluster 3 failed to encode map e264 with expected crc 2023-10-01T11:00:41.260 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:41 smithi078 bash[75299]: cluster 2023-10-01T11:00:40.014439+0000 mon.a (mon.0) 1610 : cluster 0 osdmap e264: 8 total, 8 up, 7 in 2023-10-01T11:00:41.260 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:41 smithi078 bash[75299]: cluster 2023-10-01T11:00:40.016849+0000 osd.6 (osd.6) 215 : cluster 3 failed to encode map e264 with expected crc 2023-10-01T11:00:41.260 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:41 smithi078 bash[75299]: cluster 2023-10-01T11:00:40.018950+0000 osd.7 (osd.7) 258 : cluster 3 failed to encode map e264 with expected crc 2023-10-01T11:00:41.260 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:41 smithi078 bash[75299]: cluster 2023-10-01T11:00:40.175869+0000 mgr.y (mgr.25072) 1299 : cluster 0 pgmap v1327: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 85 active+clean; 717 MiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 513 op/s; 1941/34458 objects misplaced (5.633%); 34 MiB/s, 545 objects/s recovering 2023-10-01T11:00:41.260 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:41 smithi078 bash[78274]: cluster 2023-10-01T11:00:40.012931+0000 osd.5 (osd.5) 246 : cluster 3 failed to encode map e264 with expected crc 2023-10-01T11:00:41.260 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:41 smithi078 bash[78274]: cluster 2023-10-01T11:00:40.014439+0000 mon.a (mon.0) 1610 : cluster 0 osdmap e264: 8 total, 8 up, 7 in 2023-10-01T11:00:41.260 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:41 smithi078 bash[78274]: cluster 2023-10-01T11:00:40.016849+0000 osd.6 (osd.6) 215 : cluster 3 failed to encode map e264 with expected crc 2023-10-01T11:00:41.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:41 smithi078 bash[78274]: cluster 2023-10-01T11:00:40.018950+0000 osd.7 (osd.7) 258 : cluster 3 failed to encode map e264 with expected crc 2023-10-01T11:00:41.261 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:41 smithi078 bash[78274]: cluster 2023-10-01T11:00:40.175869+0000 mgr.y (mgr.25072) 1299 : cluster 0 pgmap v1327: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 85 active+clean; 717 MiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 513 op/s; 1941/34458 objects misplaced (5.633%); 34 MiB/s, 545 objects/s recovering 2023-10-01T11:00:41.261 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-01T11:00:41.276 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:41.302 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:41.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:41.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:00:41.407 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:41.431 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:41.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:41.433 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:00:41.534 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:41.561 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:41.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:41.563 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:00:41.664 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:41.687 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:41.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:41.690 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:00:41.791 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:41.817 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:41.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:41.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:00:41.922 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:41.957 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:41.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:41.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:00:42.065 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:42.108 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:42.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:42.112 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:42.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:42 smithi173 bash[58816]: cluster 2023-10-01T11:00:41.023103+0000 mon.a (mon.0) 1611 : cluster 0 osdmap e265: 8 total, 8 up, 7 in 2023-10-01T11:00:42.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:42 smithi173 bash[58816]: cluster 2023-10-01T11:00:41.027999+0000 osd.7 (osd.7) 259 : cluster 3 failed to encode map e265 with expected crc 2023-10-01T11:00:42.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:42 smithi173 bash[58816]: cluster 2023-10-01T11:00:41.037419+0000 osd.5 (osd.5) 247 : cluster 3 failed to encode map e265 with expected crc 2023-10-01T11:00:42.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:42 smithi173 bash[58816]: cluster 2023-10-01T11:00:41.086451+0000 osd.6 (osd.6) 216 : cluster 3 failed to encode map e265 with expected crc 2023-10-01T11:00:42.213 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:42.221 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:42 smithi078 bash[78274]: cluster 2023-10-01T11:00:41.023103+0000 mon.a (mon.0) 1611 : cluster 0 osdmap e265: 8 total, 8 up, 7 in 2023-10-01T11:00:42.221 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:42 smithi078 bash[78274]: cluster 2023-10-01T11:00:41.027999+0000 osd.7 (osd.7) 259 : cluster 3 failed to encode map e265 with expected crc 2023-10-01T11:00:42.222 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:42 smithi078 bash[78274]: cluster 2023-10-01T11:00:41.037419+0000 osd.5 (osd.5) 247 : cluster 3 failed to encode map e265 with expected crc 2023-10-01T11:00:42.222 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:42 smithi078 bash[78274]: cluster 2023-10-01T11:00:41.086451+0000 osd.6 (osd.6) 216 : cluster 3 failed to encode map e265 with expected crc 2023-10-01T11:00:42.222 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:42 smithi078 bash[75299]: cluster 2023-10-01T11:00:41.023103+0000 mon.a (mon.0) 1611 : cluster 0 osdmap e265: 8 total, 8 up, 7 in 2023-10-01T11:00:42.222 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:42 smithi078 bash[75299]: cluster 2023-10-01T11:00:41.027999+0000 osd.7 (osd.7) 259 : cluster 3 failed to encode map e265 with expected crc 2023-10-01T11:00:42.222 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:42 smithi078 bash[75299]: cluster 2023-10-01T11:00:41.037419+0000 osd.5 (osd.5) 247 : cluster 3 failed to encode map e265 with expected crc 2023-10-01T11:00:42.222 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:42 smithi078 bash[75299]: cluster 2023-10-01T11:00:41.086451+0000 osd.6 (osd.6) 216 : cluster 3 failed to encode map e265 with expected crc 2023-10-01T11:00:42.238 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:42.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:42.244 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:42.344 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:42.372 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:42.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:42.379 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:00:42.480 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:42.505 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:42.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:42.507 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:00:42.608 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:42.637 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:42.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:42.639 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:00:42.740 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:42.769 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:42.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:42.773 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:00:42.874 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:42.903 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:42.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:42.906 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:00:43.007 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:43.036 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:43.037 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:43 smithi173 bash[58816]: cluster 2023-10-01T11:00:42.030695+0000 mon.a (mon.0) 1612 : cluster 0 osdmap e266: 8 total, 8 up, 7 in 2023-10-01T11:00:43.038 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:43 smithi173 bash[58816]: cluster 2023-10-01T11:00:42.031716+0000 osd.5 (osd.5) 248 : cluster 3 failed to encode map e266 with expected crc 2023-10-01T11:00:43.038 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:43 smithi173 bash[58816]: cluster 2023-10-01T11:00:42.038786+0000 osd.7 (osd.7) 260 : cluster 3 failed to encode map e266 with expected crc 2023-10-01T11:00:43.038 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:43 smithi173 bash[58816]: cluster 2023-10-01T11:00:42.039842+0000 osd.6 (osd.6) 217 : cluster 3 failed to encode map e266 with expected crc 2023-10-01T11:00:43.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:43.039 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:00:43.140 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:43.149 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:43 smithi173 bash[58816]: cluster 2023-10-01T11:00:42.041194+0000 osd.7 (osd.7) 261 : cluster 3 failed to encode map e266 with expected crc 2023-10-01T11:00:43.149 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:43 smithi173 bash[58816]: cluster 2023-10-01T11:00:42.176356+0000 mgr.y (mgr.25072) 1300 : cluster 0 pgmap v1330: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 85 active+clean; 717 MiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 365 op/s; 1941/34458 objects misplaced (5.633%); 27 MiB/s, 427 objects/s recovering 2023-10-01T11:00:43.165 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:43.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:43.168 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:00:43.268 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:43.297 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:43.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:43.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:00:43.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:43 smithi078 bash[75299]: cluster 2023-10-01T11:00:42.030695+0000 mon.a (mon.0) 1612 : cluster 0 osdmap e266: 8 total, 8 up, 7 in 2023-10-01T11:00:43.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:43 smithi078 bash[75299]: cluster 2023-10-01T11:00:42.031716+0000 osd.5 (osd.5) 248 : cluster 3 failed to encode map e266 with expected crc 2023-10-01T11:00:43.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:43 smithi078 bash[75299]: cluster 2023-10-01T11:00:42.038786+0000 osd.7 (osd.7) 260 : cluster 3 failed to encode map e266 with expected crc 2023-10-01T11:00:43.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:43 smithi078 bash[75299]: cluster 2023-10-01T11:00:42.039842+0000 osd.6 (osd.6) 217 : cluster 3 failed to encode map e266 with expected crc 2023-10-01T11:00:43.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:43 smithi078 bash[75299]: cluster 2023-10-01T11:00:42.041194+0000 osd.7 (osd.7) 261 : cluster 3 failed to encode map e266 with expected crc 2023-10-01T11:00:43.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:43 smithi078 bash[75299]: cluster 2023-10-01T11:00:42.176356+0000 mgr.y (mgr.25072) 1300 : cluster 0 pgmap v1330: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 85 active+clean; 717 MiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 365 op/s; 1941/34458 objects misplaced (5.633%); 27 MiB/s, 427 objects/s recovering 2023-10-01T11:00:43.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:43 smithi078 bash[78274]: cluster 2023-10-01T11:00:42.030695+0000 mon.a (mon.0) 1612 : cluster 0 osdmap e266: 8 total, 8 up, 7 in 2023-10-01T11:00:43.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:43 smithi078 bash[78274]: cluster 2023-10-01T11:00:42.031716+0000 osd.5 (osd.5) 248 : cluster 3 failed to encode map e266 with expected crc 2023-10-01T11:00:43.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:43 smithi078 bash[78274]: cluster 2023-10-01T11:00:42.038786+0000 osd.7 (osd.7) 260 : cluster 3 failed to encode map e266 with expected crc 2023-10-01T11:00:43.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:43 smithi078 bash[78274]: cluster 2023-10-01T11:00:42.039842+0000 osd.6 (osd.6) 217 : cluster 3 failed to encode map e266 with expected crc 2023-10-01T11:00:43.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:43 smithi078 bash[78274]: cluster 2023-10-01T11:00:42.041194+0000 osd.7 (osd.7) 261 : cluster 3 failed to encode map e266 with expected crc 2023-10-01T11:00:43.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:43 smithi078 bash[78274]: cluster 2023-10-01T11:00:42.176356+0000 mgr.y (mgr.25072) 1300 : cluster 0 pgmap v1330: 89 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 85 active+clean; 717 MiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 365 op/s; 1941/34458 objects misplaced (5.633%); 27 MiB/s, 427 objects/s recovering 2023-10-01T11:00:43.389 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T11:00:43.402 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T11:00:43.406 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:43.437 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:43.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:43.439 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:00:43.540 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:43.564 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:43.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:43.566 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:00:43.667 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:43.714 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:43.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:43.722 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:43.822 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:43.850 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:43.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:43.853 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:00:43.953 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:43.978 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:43.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:43.980 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:00:44.038 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:44 smithi078 bash[75299]: cluster 2023-10-01T11:00:43.038952+0000 osd.7 (osd.7) 262 : cluster 3 failed to encode map e267 with expected crc 2023-10-01T11:00:44.039 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:44 smithi078 bash[75299]: cluster 2023-10-01T11:00:43.045138+0000 mon.a (mon.0) 1613 : cluster 0 osdmap e267: 8 total, 8 up, 7 in 2023-10-01T11:00:44.081 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:44.091 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:44 smithi173 bash[58816]: cluster 2023-10-01T11:00:43.038952+0000 osd.7 (osd.7) 262 : cluster 3 failed to encode map e267 with expected crc 2023-10-01T11:00:44.091 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:44 smithi173 bash[58816]: cluster 2023-10-01T11:00:43.045138+0000 mon.a (mon.0) 1613 : cluster 0 osdmap e267: 8 total, 8 up, 7 in 2023-10-01T11:00:44.091 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:44 smithi173 bash[58816]: audit 2023-10-01T11:00:43.084582+0000 mon.a (mon.0) 1614 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:00:44.091 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:44 smithi173 bash[58816]: cluster 2023-10-01T11:00:43.092929+0000 osd.6 (osd.6) 218 : cluster 3 failed to encode map e267 with expected crc 2023-10-01T11:00:44.091 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:44 smithi173 bash[58816]: cluster 2023-10-01T11:00:43.093248+0000 osd.5 (osd.5) 249 : cluster 3 failed to encode map e267 with expected crc 2023-10-01T11:00:44.107 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:44.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:44.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:00:44.210 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:44.235 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:44.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:44.238 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:00:44.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:44 smithi078 bash[78274]: cluster 2023-10-01T11:00:43.038952+0000 osd.7 (osd.7) 262 : cluster 3 failed to encode map e267 with expected crc 2023-10-01T11:00:44.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:44 smithi078 bash[78274]: cluster 2023-10-01T11:00:43.045138+0000 mon.a (mon.0) 1613 : cluster 0 osdmap e267: 8 total, 8 up, 7 in 2023-10-01T11:00:44.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:44 smithi078 bash[78274]: audit 2023-10-01T11:00:43.084582+0000 mon.a (mon.0) 1614 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:00:44.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:44 smithi078 bash[78274]: cluster 2023-10-01T11:00:43.092929+0000 osd.6 (osd.6) 218 : cluster 3 failed to encode map e267 with expected crc 2023-10-01T11:00:44.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:44 smithi078 bash[78274]: cluster 2023-10-01T11:00:43.093248+0000 osd.5 (osd.5) 249 : cluster 3 failed to encode map e267 with expected crc 2023-10-01T11:00:44.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:44 smithi078 bash[75299]: audit 2023-10-01T11:00:43.084582+0000 mon.a (mon.0) 1614 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:00:44.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:44 smithi078 bash[75299]: cluster 2023-10-01T11:00:43.092929+0000 osd.6 (osd.6) 218 : cluster 3 failed to encode map e267 with expected crc 2023-10-01T11:00:44.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:44 smithi078 bash[75299]: cluster 2023-10-01T11:00:43.093248+0000 osd.5 (osd.5) 249 : cluster 3 failed to encode map e267 with expected crc 2023-10-01T11:00:44.338 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:44.364 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:44.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:44.367 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:00:44.468 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:44.494 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:44.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:44.498 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:00:44.599 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:44.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:44.627 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:44.627 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:00:44.728 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:44.768 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:44.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:44.772 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:44.873 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:44.901 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:44.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:44.903 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:00:45.004 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:45.028 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:45.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:45.032 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:00:45.053 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:45 smithi078 bash[75299]: cluster 2023-10-01T11:00:44.177914+0000 mgr.y (mgr.25072) 1301 : cluster 0 pgmap v1332: 89 pgs: 2 active+remapped+backfilling, 87 active+clean; 652 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 250 op/s; 488/31329 objects misplaced (1.558%); 20 MiB/s, 317 objects/s recovering 2023-10-01T11:00:45.054 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:45 smithi078 bash[78274]: cluster 2023-10-01T11:00:44.177914+0000 mgr.y (mgr.25072) 1301 : cluster 0 pgmap v1332: 89 pgs: 2 active+remapped+backfilling, 87 active+clean; 652 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 250 op/s; 488/31329 objects misplaced (1.558%); 20 MiB/s, 317 objects/s recovering 2023-10-01T11:00:45.133 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:45.160 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:45.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:45.162 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:45.263 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:45.272 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:45 smithi173 bash[58816]: cluster 2023-10-01T11:00:44.177914+0000 mgr.y (mgr.25072) 1301 : cluster 0 pgmap v1332: 89 pgs: 2 active+remapped+backfilling, 87 active+clean; 652 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 250 op/s; 488/31329 objects misplaced (1.558%); 20 MiB/s, 317 objects/s recovering 2023-10-01T11:00:45.292 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:45.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:45.295 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:00:45.395 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:45.438 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:45.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:45.449 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:00:45.550 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:45.573 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:45.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:45.575 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:00:45.676 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:45.700 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:45.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:45.702 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:00:45.803 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:45.827 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:45.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:45.831 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:00:45.929 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:00:45.932 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:00:45.932 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:45.958 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:45.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:45.960 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:00:46.061 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:46.086 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:46.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:46.088 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:00:46.189 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:46.215 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:46.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:46.218 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:00:46.318 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:46.345 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:46.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:46.351 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:00:46.451 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:46.481 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:46.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:46.483 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:00:46.584 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:46.609 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:46.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:46.612 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:00:46.714 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:46.747 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:46.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:46.750 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:00:46.851 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:46.886 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:46.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:46.890 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:00:46.991 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:47.037 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:47.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:47.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:47.144 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:47.167 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:47.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:47.171 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:00:47.253 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:47 smithi078 bash[75299]: cluster 2023-10-01T11:00:46.179812+0000 mgr.y (mgr.25072) 1302 : cluster 0 pgmap v1333: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 606 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 294 op/s; 295/29157 objects misplaced (1.012%); 16 MiB/s, 252 objects/s recovering 2023-10-01T11:00:47.272 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:47.299 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:47.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:47.302 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:00:47.403 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:47.429 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:47.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:47.434 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:47.534 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:47.561 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:47.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:47.564 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:47.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:47 smithi173 bash[58816]: cluster 2023-10-01T11:00:46.179812+0000 mgr.y (mgr.25072) 1302 : cluster 0 pgmap v1333: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 606 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 294 op/s; 295/29157 objects misplaced (1.012%); 16 MiB/s, 252 objects/s recovering 2023-10-01T11:00:47.650 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:47 smithi078 bash[78274]: cluster 2023-10-01T11:00:46.179812+0000 mgr.y (mgr.25072) 1302 : cluster 0 pgmap v1333: 89 pgs: 1 active+remapped+backfilling, 88 active+clean; 606 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 294 op/s; 295/29157 objects misplaced (1.012%); 16 MiB/s, 252 objects/s recovering 2023-10-01T11:00:47.666 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:47.695 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:47.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:47.698 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:00:47.799 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:47.828 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:47.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:47.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:47.930 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:47.961 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:00:47.963 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:00:47.968 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:47.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:47.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:00:48.072 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:48.101 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:48.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:48.103 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:00:48.203 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:48.237 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:48.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:48.240 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:00:48.265 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:48 smithi078 bash[75299]: cluster 2023-10-01T11:00:47.241177+0000 osd.7 (osd.7) 263 : cluster 3 failed to encode map e268 with expected crc 2023-10-01T11:00:48.265 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:48 smithi078 bash[75299]: cluster 2023-10-01T11:00:47.244522+0000 osd.7 (osd.7) 264 : cluster 3 failed to encode map e268 with expected crc 2023-10-01T11:00:48.266 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:48 smithi078 bash[75299]: cluster 2023-10-01T11:00:47.244569+0000 mon.a (mon.0) 1615 : cluster 0 osdmap e268: 8 total, 8 up, 7 in 2023-10-01T11:00:48.266 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:48 smithi078 bash[75299]: cluster 2023-10-01T11:00:47.259018+0000 osd.5 (osd.5) 250 : cluster 3 failed to encode map e268 with expected crc 2023-10-01T11:00:48.266 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:48 smithi078 bash[75299]: cluster 2023-10-01T11:00:47.261583+0000 osd.6 (osd.6) 219 : cluster 3 failed to encode map e268 with expected crc 2023-10-01T11:00:48.267 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:48 smithi078 bash[75299]: audit 2023-10-01T11:00:47.432167+0000 mon.a (mon.0) 1616 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.15", "id": [3, 2]}]: dispatch 2023-10-01T11:00:48.267 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:48 smithi078 bash[75299]: audit 2023-10-01T11:00:47.456560+0000 mon.a (mon.0) 1617 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:00:48.268 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:48 smithi078 bash[75299]: audit 2023-10-01T11:00:47.457811+0000 mon.a (mon.0) 1618 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:00:48.340 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:48.369 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:48.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:48.371 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:00:48.472 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:48.481 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:48 smithi173 bash[58816]: cluster 2023-10-01T11:00:47.241177+0000 osd.7 (osd.7) 263 : cluster 3 failed to encode map e268 with expected crc 2023-10-01T11:00:48.481 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:48 smithi173 bash[58816]: cluster 2023-10-01T11:00:47.244522+0000 osd.7 (osd.7) 264 : cluster 3 failed to encode map e268 with expected crc 2023-10-01T11:00:48.481 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:48 smithi173 bash[58816]: cluster 2023-10-01T11:00:47.244569+0000 mon.a (mon.0) 1615 : cluster 0 osdmap e268: 8 total, 8 up, 7 in 2023-10-01T11:00:48.481 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:48 smithi173 bash[58816]: cluster 2023-10-01T11:00:47.259018+0000 osd.5 (osd.5) 250 : cluster 3 failed to encode map e268 with expected crc 2023-10-01T11:00:48.482 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:48 smithi173 bash[58816]: cluster 2023-10-01T11:00:47.261583+0000 osd.6 (osd.6) 219 : cluster 3 failed to encode map e268 with expected crc 2023-10-01T11:00:48.482 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:48 smithi173 bash[58816]: audit 2023-10-01T11:00:47.432167+0000 mon.a (mon.0) 1616 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.15", "id": [3, 2]}]: dispatch 2023-10-01T11:00:48.482 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:48 smithi173 bash[58816]: audit 2023-10-01T11:00:47.456560+0000 mon.a (mon.0) 1617 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:00:48.482 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:48 smithi173 bash[58816]: audit 2023-10-01T11:00:47.457811+0000 mon.a (mon.0) 1618 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:00:48.497 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:48.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:48.502 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:00:48.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:48 smithi078 bash[78274]: cluster 2023-10-01T11:00:47.241177+0000 osd.7 (osd.7) 263 : cluster 3 failed to encode map e268 with expected crc 2023-10-01T11:00:48.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:48 smithi078 bash[78274]: cluster 2023-10-01T11:00:47.244522+0000 osd.7 (osd.7) 264 : cluster 3 failed to encode map e268 with expected crc 2023-10-01T11:00:48.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:48 smithi078 bash[78274]: cluster 2023-10-01T11:00:47.244569+0000 mon.a (mon.0) 1615 : cluster 0 osdmap e268: 8 total, 8 up, 7 in 2023-10-01T11:00:48.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:48 smithi078 bash[78274]: cluster 2023-10-01T11:00:47.259018+0000 osd.5 (osd.5) 250 : cluster 3 failed to encode map e268 with expected crc 2023-10-01T11:00:48.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:48 smithi078 bash[78274]: cluster 2023-10-01T11:00:47.261583+0000 osd.6 (osd.6) 219 : cluster 3 failed to encode map e268 with expected crc 2023-10-01T11:00:48.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:48 smithi078 bash[78274]: audit 2023-10-01T11:00:47.432167+0000 mon.a (mon.0) 1616 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.15", "id": [3, 2]}]: dispatch 2023-10-01T11:00:48.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:48 smithi078 bash[78274]: audit 2023-10-01T11:00:47.456560+0000 mon.a (mon.0) 1617 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:00:48.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:48 smithi078 bash[78274]: audit 2023-10-01T11:00:47.457811+0000 mon.a (mon.0) 1618 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:00:48.603 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:48.631 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:48.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:48.634 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:00:48.734 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:48.780 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:48.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:48.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:48.884 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:48.911 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:48.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:48.915 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:00:49.016 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:49.048 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:49.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:49.055 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:00:49.157 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:49.180 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:49.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:49.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:00:49.283 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:49.307 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:49.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:49.310 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:00:49.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:49 smithi078 bash[75299]: cluster 2023-10-01T11:00:48.180627+0000 mgr.y (mgr.25072) 1303 : cluster 0 pgmap v1335: 89 pgs: 1 peering, 88 active+clean; 562 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 403 op/s; 18 MiB/s, 288 objects/s recovering 2023-10-01T11:00:49.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:49 smithi078 bash[75299]: cluster 2023-10-01T11:00:48.254818+0000 osd.5 (osd.5) 251 : cluster 3 failed to encode map e269 with expected crc 2023-10-01T11:00:49.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:49 smithi078 bash[75299]: cluster 2023-10-01T11:00:48.255430+0000 osd.7 (osd.7) 265 : cluster 3 failed to encode map e269 with expected crc 2023-10-01T11:00:49.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:49 smithi078 bash[75299]: audit 2023-10-01T11:00:48.255980+0000 mon.a (mon.0) 1619 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.15", "id": [3, 2]}]': finished 2023-10-01T11:00:49.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:49 smithi078 bash[75299]: cluster 2023-10-01T11:00:48.256035+0000 mon.a (mon.0) 1620 : cluster 0 osdmap e269: 8 total, 8 up, 7 in 2023-10-01T11:00:49.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:49 smithi078 bash[75299]: cluster 2023-10-01T11:00:48.269592+0000 osd.6 (osd.6) 220 : cluster 3 failed to encode map e269 with expected crc 2023-10-01T11:00:49.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:49 smithi078 bash[78274]: cluster 2023-10-01T11:00:48.180627+0000 mgr.y (mgr.25072) 1303 : cluster 0 pgmap v1335: 89 pgs: 1 peering, 88 active+clean; 562 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 403 op/s; 18 MiB/s, 288 objects/s recovering 2023-10-01T11:00:49.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:49 smithi078 bash[78274]: cluster 2023-10-01T11:00:48.254818+0000 osd.5 (osd.5) 251 : cluster 3 failed to encode map e269 with expected crc 2023-10-01T11:00:49.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:49 smithi078 bash[78274]: cluster 2023-10-01T11:00:48.255430+0000 osd.7 (osd.7) 265 : cluster 3 failed to encode map e269 with expected crc 2023-10-01T11:00:49.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:49 smithi078 bash[78274]: audit 2023-10-01T11:00:48.255980+0000 mon.a (mon.0) 1619 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.15", "id": [3, 2]}]': finished 2023-10-01T11:00:49.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:49 smithi078 bash[78274]: cluster 2023-10-01T11:00:48.256035+0000 mon.a (mon.0) 1620 : cluster 0 osdmap e269: 8 total, 8 up, 7 in 2023-10-01T11:00:49.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:49 smithi078 bash[78274]: cluster 2023-10-01T11:00:48.269592+0000 osd.6 (osd.6) 220 : cluster 3 failed to encode map e269 with expected crc 2023-10-01T11:00:49.411 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:49.419 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:49 smithi173 bash[58816]: cluster 2023-10-01T11:00:48.180627+0000 mgr.y (mgr.25072) 1303 : cluster 0 pgmap v1335: 89 pgs: 1 peering, 88 active+clean; 562 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 403 op/s; 18 MiB/s, 288 objects/s recovering 2023-10-01T11:00:49.419 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:49 smithi173 bash[58816]: cluster 2023-10-01T11:00:48.254818+0000 osd.5 (osd.5) 251 : cluster 3 failed to encode map e269 with expected crc 2023-10-01T11:00:49.419 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:49 smithi173 bash[58816]: cluster 2023-10-01T11:00:48.255430+0000 osd.7 (osd.7) 265 : cluster 3 failed to encode map e269 with expected crc 2023-10-01T11:00:49.419 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:49 smithi173 bash[58816]: audit 2023-10-01T11:00:48.255980+0000 mon.a (mon.0) 1619 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.15", "id": [3, 2]}]': finished 2023-10-01T11:00:49.419 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:49 smithi173 bash[58816]: cluster 2023-10-01T11:00:48.256035+0000 mon.a (mon.0) 1620 : cluster 0 osdmap e269: 8 total, 8 up, 7 in 2023-10-01T11:00:49.419 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:49 smithi173 bash[58816]: cluster 2023-10-01T11:00:48.269592+0000 osd.6 (osd.6) 220 : cluster 3 failed to encode map e269 with expected crc 2023-10-01T11:00:49.438 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:49.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:49.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:00:49.551 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:49.577 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:49.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:49.583 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:00:49.683 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:49.709 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:49.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:49.715 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:00:49.815 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:49.844 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:49.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:49.846 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:00:49.947 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:49.970 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:49.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:49.974 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:00:50.075 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:50.103 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:50.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:50.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:00:50.208 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:50.244 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:50.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:50.248 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:00:50.292 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:50 smithi173 bash[58816]: cluster 2023-10-01T11:00:49.283924+0000 osd.7 (osd.7) 266 : cluster 3 failed to encode map e270 with expected crc 2023-10-01T11:00:50.293 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:50 smithi173 bash[58816]: cluster 2023-10-01T11:00:49.284675+0000 osd.5 (osd.5) 252 : cluster 3 failed to encode map e270 with expected crc 2023-10-01T11:00:50.293 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:50 smithi173 bash[58816]: cluster 2023-10-01T11:00:49.286457+0000 mon.a (mon.0) 1621 : cluster 0 osdmap e270: 8 total, 8 up, 7 in 2023-10-01T11:00:50.349 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:50.357 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:50 smithi078 bash[78274]: cluster 2023-10-01T11:00:49.283924+0000 osd.7 (osd.7) 266 : cluster 3 failed to encode map e270 with expected crc 2023-10-01T11:00:50.357 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:50 smithi078 bash[78274]: cluster 2023-10-01T11:00:49.284675+0000 osd.5 (osd.5) 252 : cluster 3 failed to encode map e270 with expected crc 2023-10-01T11:00:50.357 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:50 smithi078 bash[78274]: cluster 2023-10-01T11:00:49.286457+0000 mon.a (mon.0) 1621 : cluster 0 osdmap e270: 8 total, 8 up, 7 in 2023-10-01T11:00:50.358 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:50 smithi078 bash[75299]: cluster 2023-10-01T11:00:49.283924+0000 osd.7 (osd.7) 266 : cluster 3 failed to encode map e270 with expected crc 2023-10-01T11:00:50.358 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:50 smithi078 bash[75299]: cluster 2023-10-01T11:00:49.284675+0000 osd.5 (osd.5) 252 : cluster 3 failed to encode map e270 with expected crc 2023-10-01T11:00:50.358 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:50 smithi078 bash[75299]: cluster 2023-10-01T11:00:49.286457+0000 mon.a (mon.0) 1621 : cluster 0 osdmap e270: 8 total, 8 up, 7 in 2023-10-01T11:00:50.358 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:00:50 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:00:50] "GET /metrics HTTP/1.1" 200 36402 "" "Prometheus/2.33.4" 2023-10-01T11:00:50.382 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:50.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:50.386 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:00:50.487 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:50.512 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:50.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:50.514 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:00:50.614 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:50.644 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:50.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:50.646 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:50.747 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:50.789 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:50.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:50.793 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:00:50.893 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:50.925 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:50.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:50.928 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:00:51.029 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:51.062 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:51.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:51.072 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:51.172 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:51.196 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:51.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:51.199 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:00:51.300 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:51.327 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:51.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:51.331 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:00:51.433 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:51.455 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:51.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:51.461 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:00:51.562 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:51.570 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:51 smithi173 bash[58816]: cluster 2023-10-01T11:00:49.688094+0000 osd.6 (osd.6) 221 : cluster 3 failed to encode map e270 with expected crc 2023-10-01T11:00:51.570 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:51 smithi173 bash[58816]: cluster 2023-10-01T11:00:49.689685+0000 osd.6 (osd.6) 222 : cluster 3 failed to encode map e270 with expected crc 2023-10-01T11:00:51.570 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:51 smithi173 bash[58816]: cluster 2023-10-01T11:00:50.181321+0000 mgr.y (mgr.25072) 1304 : cluster 0 pgmap v1338: 89 pgs: 2 peering, 87 active+clean; 434 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 579 op/s; 4.7 MiB/s, 75 objects/s recovering 2023-10-01T11:00:51.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:51 smithi078 bash[75299]: cluster 2023-10-01T11:00:49.688094+0000 osd.6 (osd.6) 221 : cluster 3 failed to encode map e270 with expected crc 2023-10-01T11:00:51.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:51 smithi078 bash[75299]: cluster 2023-10-01T11:00:49.689685+0000 osd.6 (osd.6) 222 : cluster 3 failed to encode map e270 with expected crc 2023-10-01T11:00:51.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:51 smithi078 bash[75299]: cluster 2023-10-01T11:00:50.181321+0000 mgr.y (mgr.25072) 1304 : cluster 0 pgmap v1338: 89 pgs: 2 peering, 87 active+clean; 434 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 579 op/s; 4.7 MiB/s, 75 objects/s recovering 2023-10-01T11:00:51.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:51 smithi078 bash[78274]: cluster 2023-10-01T11:00:49.688094+0000 osd.6 (osd.6) 221 : cluster 3 failed to encode map e270 with expected crc 2023-10-01T11:00:51.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:51 smithi078 bash[78274]: cluster 2023-10-01T11:00:49.689685+0000 osd.6 (osd.6) 222 : cluster 3 failed to encode map e270 with expected crc 2023-10-01T11:00:51.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:51 smithi078 bash[78274]: cluster 2023-10-01T11:00:50.181321+0000 mgr.y (mgr.25072) 1304 : cluster 0 pgmap v1338: 89 pgs: 2 peering, 87 active+clean; 434 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 579 op/s; 4.7 MiB/s, 75 objects/s recovering 2023-10-01T11:00:51.586 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:51.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:51.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:00:51.690 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:51.726 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:51.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:51.731 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:00:51.832 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:51.855 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:51.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:51.858 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:00:51.958 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:51.983 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:51.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:51.985 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:00:52.086 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:52.114 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:52.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:52.119 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:00:52.219 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:52.245 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:52.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:52.251 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:00:52.352 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:52.378 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:52.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:52.379 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:52.480 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:52.502 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:52.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:52.505 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:00:52.606 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:52.651 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:52.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:52.657 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:00:52.758 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:52.787 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:52.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:52.792 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:00:52.892 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:52.915 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:52.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:52.918 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:00:53.019 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:53.047 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:53.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:53.052 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:53.153 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:53.180 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:53.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:53.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:00:53.283 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:53.299 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:53.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:53.301 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:00:53.402 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:53.427 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:53.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:53.432 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:00:53.533 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:53.560 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:53.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:53.563 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:00:53.664 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:53.695 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:53.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:53.700 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:00:53.801 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:53.811 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:53 smithi173 bash[58816]: cluster 2023-10-01T11:00:52.181762+0000 mgr.y (mgr.25072) 1305 : cluster 0 pgmap v1339: 89 pgs: 2 peering, 87 active+clean; 434 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 459 op/s; 2.7 MiB/s, 43 objects/s recovering 2023-10-01T11:00:53.811 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:53 smithi173 bash[58816]: audit 2023-10-01T11:00:53.105951+0000 mon.a (mon.0) 1622 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:00:53.831 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:53.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:53.839 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:00:53.940 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:53.964 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:53.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:53.966 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:00:54.030 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:53 smithi078 bash[75299]: cluster 2023-10-01T11:00:52.181762+0000 mgr.y (mgr.25072) 1305 : cluster 0 pgmap v1339: 89 pgs: 2 peering, 87 active+clean; 434 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 459 op/s; 2.7 MiB/s, 43 objects/s recovering 2023-10-01T11:00:54.031 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:53 smithi078 bash[75299]: audit 2023-10-01T11:00:53.105951+0000 mon.a (mon.0) 1622 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:00:54.031 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:53 smithi078 bash[78274]: cluster 2023-10-01T11:00:52.181762+0000 mgr.y (mgr.25072) 1305 : cluster 0 pgmap v1339: 89 pgs: 2 peering, 87 active+clean; 434 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 459 op/s; 2.7 MiB/s, 43 objects/s recovering 2023-10-01T11:00:54.031 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:53 smithi078 bash[78274]: audit 2023-10-01T11:00:53.105951+0000 mon.a (mon.0) 1622 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:00:54.067 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:54.085 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:54.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:54.087 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:54.188 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:54.222 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:54.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:54.228 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:00:54.329 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:54.353 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:54.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:54.356 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:00:54.457 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:54.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:54.488 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:54.488 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:00:54.589 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:54.624 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:54.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:54.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:00:54.728 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:54.752 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:54.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:54.758 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:54.858 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:54.876 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:54.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:54.878 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:00:54.979 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:55.010 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:55.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:55.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:00:55.113 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:55.141 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:55.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:55.144 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:00:55.245 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:55.265 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:55.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:55.267 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:00:55.367 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:55.397 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:55.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:55.401 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:00:55.502 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:55.526 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:55.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:55.528 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:00:55.628 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:55.659 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:55.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:55.662 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:55.672 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:55 smithi173 bash[58816]: cluster 2023-10-01T11:00:54.182396+0000 mgr.y (mgr.25072) 1306 : cluster 0 pgmap v1340: 89 pgs: 89 active+clean; 301 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 704 op/s; 2.4 MiB/s, 37 objects/s recovering 2023-10-01T11:00:55.762 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:55.784 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:55.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:55.787 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:00:55.888 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:55.896 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:55 smithi078 bash[75299]: cluster 2023-10-01T11:00:54.182396+0000 mgr.y (mgr.25072) 1306 : cluster 0 pgmap v1340: 89 pgs: 89 active+clean; 301 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 704 op/s; 2.4 MiB/s, 37 objects/s recovering 2023-10-01T11:00:55.896 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:55 smithi078 bash[78274]: cluster 2023-10-01T11:00:54.182396+0000 mgr.y (mgr.25072) 1306 : cluster 0 pgmap v1340: 89 pgs: 89 active+clean; 301 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 704 op/s; 2.4 MiB/s, 37 objects/s recovering 2023-10-01T11:00:55.922 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:55.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:55.924 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:56.024 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:56.052 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:56.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:56.054 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:00:56.154 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:56.171 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:56.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:56.173 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:00:56.274 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:56.308 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:56.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:56.310 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:00:56.411 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:56.449 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:56.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:56.450 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:56.551 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:56.579 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:56.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:56.582 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:00:56.683 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:56.709 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:56.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:56.715 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:56.816 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:56.846 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:56.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:56.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:56.951 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:56.973 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:56.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:56.975 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:00:57.076 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:57.101 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:57.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:57.106 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:00:57.207 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:57.236 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:57.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:57.239 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:00:57.340 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:57.366 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:57.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:57.370 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:00:57.471 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:57.498 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:57.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:57.501 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:00:57.602 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:57.628 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:57.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:57.630 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:00:57.731 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:57.739 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:57 smithi173 bash[58816]: cluster 2023-10-01T11:00:56.182897+0000 mgr.y (mgr.25072) 1307 : cluster 0 pgmap v1341: 89 pgs: 89 active+clean; 226 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 670 op/s; 0 B/s, 0 objects/s recovering 2023-10-01T11:00:57.754 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:57.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:57.762 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:00:57.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:57 smithi078 bash[78274]: cluster 2023-10-01T11:00:56.182897+0000 mgr.y (mgr.25072) 1307 : cluster 0 pgmap v1341: 89 pgs: 89 active+clean; 226 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 670 op/s; 0 B/s, 0 objects/s recovering 2023-10-01T11:00:57.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:57 smithi078 bash[75299]: cluster 2023-10-01T11:00:56.182897+0000 mgr.y (mgr.25072) 1307 : cluster 0 pgmap v1341: 89 pgs: 89 active+clean; 226 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 670 op/s; 0 B/s, 0 objects/s recovering 2023-10-01T11:00:57.862 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:57.886 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:57.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:57.891 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:00:57.992 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:58.026 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:58.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:58.031 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:00:58.131 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:58.156 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:58.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:58.162 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:58.263 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:58.292 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:00:58.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:58.296 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:00:58.396 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:58.435 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:58.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:58.438 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:00:58.539 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:58.563 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:58.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:58.565 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:58.667 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:58.690 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:00:58.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:58.695 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:00:58.795 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:58.828 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:00:58.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:58.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:00:58.932 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:58.970 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:58.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:58.972 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:59.073 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:59.110 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:59.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:59.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:00:59.214 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:59.238 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:00:59.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:59.241 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:00:59.342 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:59.371 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:00:59.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:59.374 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:00:59.475 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:59.528 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:59.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:59.530 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:00:59.561 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Removed 103128 objects 2023-10-01T11:00:59.562 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Clean up completed and total clean up time :162.503 2023-10-01T11:00:59.631 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:59.660 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:00:59.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:59.663 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:00:59.763 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:59.798 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:00:59.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:59.800 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:00:59.901 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:59.936 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:00:59.938 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:00:59 smithi173 bash[58816]: cluster 2023-10-01T11:00:58.183338+0000 mgr.y (mgr.25072) 1308 : cluster 0 pgmap v1342: 89 pgs: 89 active+clean; 180 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 615 op/s; 0 B/s, 0 objects/s recovering 2023-10-01T11:00:59.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:00:59.939 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:01:00.040 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:00.047 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:00:59 smithi078 bash[75299]: cluster 2023-10-01T11:00:58.183338+0000 mgr.y (mgr.25072) 1308 : cluster 0 pgmap v1342: 89 pgs: 89 active+clean; 180 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 615 op/s; 0 B/s, 0 objects/s recovering 2023-10-01T11:01:00.047 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:00:59 smithi078 bash[78274]: cluster 2023-10-01T11:00:58.183338+0000 mgr.y (mgr.25072) 1308 : cluster 0 pgmap v1342: 89 pgs: 89 active+clean; 180 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 615 op/s; 0 B/s, 0 objects/s recovering 2023-10-01T11:01:00.061 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:00.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:00.063 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:00.164 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:00.188 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:00.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:00.190 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:00.291 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:00.302 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:01:00 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:01:00] "GET /metrics HTTP/1.1" 200 36402 "" "Prometheus/2.33.4" 2023-10-01T11:01:00.315 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:00.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:00.317 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:01:00.417 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:00.442 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:00.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:00.445 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:00.545 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:00.571 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:00.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:00.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:00.674 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:00.706 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:00.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:00.709 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:01:00.809 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:00.837 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:00.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:00.840 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:00.941 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:00.968 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:00.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:00.970 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:01:01.071 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:01.108 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:01.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:01.112 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:01:01.213 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:01.240 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:01.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:01.243 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:01:01.346 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:01.378 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:01.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:01.382 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:01:01.483 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:01.507 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:01.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:01.510 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:01:01.611 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:01.636 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:01.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:01.639 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:01:01.740 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:01.752 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:01 smithi078 bash[78274]: cluster 2023-10-01T11:01:00.184257+0000 mgr.y (mgr.25072) 1309 : cluster 0 pgmap v1343: 89 pgs: 89 active+clean; 31 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 687 op/s; 0 B/s, 0 objects/s recovering 2023-10-01T11:01:01.752 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:01 smithi078 bash[75299]: cluster 2023-10-01T11:01:00.184257+0000 mgr.y (mgr.25072) 1309 : cluster 0 pgmap v1343: 89 pgs: 89 active+clean; 31 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 687 op/s; 0 B/s, 0 objects/s recovering 2023-10-01T11:01:01.765 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:01.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:01.767 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:01:01.868 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:01.890 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:01.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:01.893 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:01.994 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:02.007 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:01 smithi173 bash[58816]: cluster 2023-10-01T11:01:00.184257+0000 mgr.y (mgr.25072) 1309 : cluster 0 pgmap v1343: 89 pgs: 89 active+clean; 31 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 687 op/s; 0 B/s, 0 objects/s recovering 2023-10-01T11:01:02.030 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:02.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:02.033 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:01:02.134 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:02.160 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:02.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:02.162 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:01:02.263 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:02.282 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:02.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:02.284 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:01:02.385 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:02.408 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:02.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:02.410 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:02.511 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:02.532 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:02.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:02.534 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:02.541 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_7 2023-10-01T11:01:02.542 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_7 unique_pool_7 --yes-i-really-really-mean-it 2023-10-01T11:01:02.635 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:02.654 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:02.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:02.657 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:01:02.706 INFO:teuthology.orchestra.run.smithi078.stderr:nodeep-scrub is unset 2023-10-01T11:01:02.758 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:02.784 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:02.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:02.787 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:01:02.888 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:02.900 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:02 smithi078 bash[75299]: audit 2023-10-01T11:01:02.452517+0000 mon.a (mon.0) 1623 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:01:02.900 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:02 smithi078 bash[75299]: audit 2023-10-01T11:01:02.476188+0000 mon.c (mon.2) 201 : audit 1 from='client.? 172.21.15.78:0/3467406167' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:01:02.900 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:02 smithi078 bash[75299]: audit 2023-10-01T11:01:02.476741+0000 mon.a (mon.0) 1624 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:01:02.900 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:02 smithi078 bash[78274]: audit 2023-10-01T11:01:02.452517+0000 mon.a (mon.0) 1623 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:01:02.901 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:02 smithi078 bash[78274]: audit 2023-10-01T11:01:02.476188+0000 mon.c (mon.2) 201 : audit 1 from='client.? 172.21.15.78:0/3467406167' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:01:02.901 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:02 smithi078 bash[78274]: audit 2023-10-01T11:01:02.476741+0000 mon.a (mon.0) 1624 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:01:02.912 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:02.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:02.914 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:01:03.015 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:03.042 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:03.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:03.045 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:03.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:02 smithi173 bash[58816]: audit 2023-10-01T11:01:02.452517+0000 mon.a (mon.0) 1623 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:01:03.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:02 smithi173 bash[58816]: audit 2023-10-01T11:01:02.476188+0000 mon.c (mon.2) 201 : audit 1 from='client.? 172.21.15.78:0/3467406167' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:01:03.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:02 smithi173 bash[58816]: audit 2023-10-01T11:01:02.476741+0000 mon.a (mon.0) 1624 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:01:03.146 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:03.175 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:03.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:03.177 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:01:03.278 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:03.308 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:03.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:03.311 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:01:03.412 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:03.439 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:03.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:03.442 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:01:03.543 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:03.575 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:03.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:03.578 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:01:03.679 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:03.715 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:03.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:03.719 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:01:03.819 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:03.849 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:03.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:03.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:01:03.953 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:03.962 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:03 smithi078 bash[78274]: cluster 2023-10-01T11:01:02.184692+0000 mgr.y (mgr.25072) 1310 : cluster 0 pgmap v1344: 89 pgs: 89 active+clean; 31 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 537 op/s; 0 B/s, 0 objects/s recovering 2023-10-01T11:01:03.962 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:03 smithi078 bash[78274]: cluster 2023-10-01T11:01:02.683739+0000 mon.a (mon.0) 1625 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T11:01:03.962 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:03 smithi078 bash[78274]: cluster 2023-10-01T11:01:02.683766+0000 mon.a (mon.0) 1626 : cluster 1 Cluster is now healthy 2023-10-01T11:01:03.962 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:03 smithi078 bash[78274]: audit 2023-10-01T11:01:02.702286+0000 mon.a (mon.0) 1627 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T11:01:03.962 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:03 smithi078 bash[78274]: cluster 2023-10-01T11:01:02.702349+0000 mon.a (mon.0) 1628 : cluster 0 osdmap e271: 8 total, 8 up, 7 in 2023-10-01T11:01:03.962 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:03 smithi078 bash[78274]: audit 2023-10-01T11:01:03.120773+0000 mon.a (mon.0) 1629 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:01:03.962 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:03 smithi078 bash[78274]: cluster 2023-10-01T11:01:03.218682+0000 osd.7 (osd.7) 267 : cluster 3 failed to encode map e271 with expected crc 2023-10-01T11:01:03.963 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:03 smithi078 bash[75299]: cluster 2023-10-01T11:01:02.184692+0000 mgr.y (mgr.25072) 1310 : cluster 0 pgmap v1344: 89 pgs: 89 active+clean; 31 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 537 op/s; 0 B/s, 0 objects/s recovering 2023-10-01T11:01:03.963 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:03 smithi078 bash[75299]: cluster 2023-10-01T11:01:02.683739+0000 mon.a (mon.0) 1625 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T11:01:03.963 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:03 smithi078 bash[75299]: cluster 2023-10-01T11:01:02.683766+0000 mon.a (mon.0) 1626 : cluster 1 Cluster is now healthy 2023-10-01T11:01:03.963 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:03 smithi078 bash[75299]: audit 2023-10-01T11:01:02.702286+0000 mon.a (mon.0) 1627 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T11:01:03.963 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:03 smithi078 bash[75299]: cluster 2023-10-01T11:01:02.702349+0000 mon.a (mon.0) 1628 : cluster 0 osdmap e271: 8 total, 8 up, 7 in 2023-10-01T11:01:03.963 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:03 smithi078 bash[75299]: audit 2023-10-01T11:01:03.120773+0000 mon.a (mon.0) 1629 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:01:03.963 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:03 smithi078 bash[75299]: cluster 2023-10-01T11:01:03.218682+0000 osd.7 (osd.7) 267 : cluster 3 failed to encode map e271 with expected crc 2023-10-01T11:01:03.978 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:03.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:03.981 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:01:04.082 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:04.096 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:03 smithi173 bash[58816]: cluster 2023-10-01T11:01:02.184692+0000 mgr.y (mgr.25072) 1310 : cluster 0 pgmap v1344: 89 pgs: 89 active+clean; 31 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 537 op/s; 0 B/s, 0 objects/s recovering 2023-10-01T11:01:04.096 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:03 smithi173 bash[58816]: cluster 2023-10-01T11:01:02.683739+0000 mon.a (mon.0) 1625 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T11:01:04.096 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:03 smithi173 bash[58816]: cluster 2023-10-01T11:01:02.683766+0000 mon.a (mon.0) 1626 : cluster 1 Cluster is now healthy 2023-10-01T11:01:04.096 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:03 smithi173 bash[58816]: audit 2023-10-01T11:01:02.702286+0000 mon.a (mon.0) 1627 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T11:01:04.096 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:03 smithi173 bash[58816]: cluster 2023-10-01T11:01:02.702349+0000 mon.a (mon.0) 1628 : cluster 0 osdmap e271: 8 total, 8 up, 7 in 2023-10-01T11:01:04.097 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:03 smithi173 bash[58816]: audit 2023-10-01T11:01:03.120773+0000 mon.a (mon.0) 1629 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:01:04.097 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:03 smithi173 bash[58816]: cluster 2023-10-01T11:01:03.218682+0000 osd.7 (osd.7) 267 : cluster 3 failed to encode map e271 with expected crc 2023-10-01T11:01:04.116 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:04.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:04.118 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:01:04.219 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:04.253 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:04.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:04.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:04.356 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:04.384 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:04.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:04.387 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:01:04.488 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:04.514 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:04.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:04.517 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:01:04.618 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:04.647 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:04.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:04.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:01:04.750 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:04.781 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:04.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:04.785 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:04.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:04 smithi173 bash[58816]: cluster 2023-10-01T11:01:03.308875+0000 osd.5 (osd.5) 253 : cluster 3 failed to encode map e271 with expected crc 2023-10-01T11:01:04.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:04 smithi173 bash[58816]: cluster 2023-10-01T11:01:03.628529+0000 osd.6 (osd.6) 223 : cluster 3 failed to encode map e271 with expected crc 2023-10-01T11:01:04.886 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:04.910 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:04 smithi078 bash[78274]: cluster 2023-10-01T11:01:03.308875+0000 osd.5 (osd.5) 253 : cluster 3 failed to encode map e271 with expected crc 2023-10-01T11:01:04.911 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:04 smithi078 bash[78274]: cluster 2023-10-01T11:01:03.628529+0000 osd.6 (osd.6) 223 : cluster 3 failed to encode map e271 with expected crc 2023-10-01T11:01:04.911 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:04 smithi078 bash[75299]: cluster 2023-10-01T11:01:03.308875+0000 osd.5 (osd.5) 253 : cluster 3 failed to encode map e271 with expected crc 2023-10-01T11:01:04.911 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:04 smithi078 bash[75299]: cluster 2023-10-01T11:01:03.628529+0000 osd.6 (osd.6) 223 : cluster 3 failed to encode map e271 with expected crc 2023-10-01T11:01:04.911 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:04.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:04.913 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:01:05.014 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:05.046 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:05.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:05.050 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:01:05.150 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:05.174 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:05.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:05.177 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:01:05.278 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:05.302 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:05.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:05.304 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:05.405 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:05.438 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:05.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:05.441 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:01:05.541 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:05.572 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:05.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:05.575 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:01:05.592 INFO:teuthology.orchestra.run.smithi078.stdout:ERROR: (22) Invalid argument 2023-10-01T11:01:05.592 INFO:teuthology.orchestra.run.smithi078.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T11:01:05.676 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:05.705 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:05.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:05.708 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:01:05.808 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:05.829 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:05.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:05.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:05.864 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T11:01:05.887 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T11:01:05.908 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T11:01:05.932 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:05.939 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:05 smithi078 bash[78274]: cluster 2023-10-01T11:01:04.185720+0000 mgr.y (mgr.25072) 1311 : cluster 0 pgmap v1346: 89 pgs: 89 active+clean; 2.1 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 478 op/s 2023-10-01T11:01:05.939 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:05 smithi078 bash[75299]: cluster 2023-10-01T11:01:04.185720+0000 mgr.y (mgr.25072) 1311 : cluster 0 pgmap v1346: 89 pgs: 89 active+clean; 2.1 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 478 op/s 2023-10-01T11:01:05.940 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T11:01:05.954 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:05.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:05.957 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:01:05.967 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T11:01:05.991 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T11:01:06.014 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T11:01:06.036 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T11:01:06.038 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T11:01:06.038 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T11:01:06.038 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T11:01:06.038 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T11:01:06.039 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T11:01:06.039 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T11:01:06.039 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T11:01:06.039 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T11:01:06.057 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:06.071 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:05 smithi173 bash[58816]: cluster 2023-10-01T11:01:04.185720+0000 mgr.y (mgr.25072) 1311 : cluster 0 pgmap v1346: 89 pgs: 89 active+clean; 2.1 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 478 op/s 2023-10-01T11:01:06.093 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:06.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:06.096 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:01:06.196 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:06.225 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:06.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:06.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:01:06.328 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:06.364 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:06.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:06.367 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:01:06.468 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:06.481 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:06 smithi078 bash[75299]: debug 2023-10-01T11:01:06.199+0000 7f31c8dd0700 -1 mon.a@0(leader).osd e271 definitely_dead 0 2023-10-01T11:01:06.498 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:06.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:06.500 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:06.601 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:06.631 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:06.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:06.634 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:01:06.725 INFO:teuthology.orchestra.run.smithi078.stderr:marked in osd.6. 2023-10-01T11:01:06.734 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:06.760 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:06.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:06.763 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:01:06.811 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:06 smithi078 bash[75299]: audit 2023-10-01T11:01:05.838013+0000 mon.a (mon.0) 1630 : audit 0 from='client.? 172.21.15.78:0/1878764134' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:01:06.812 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:06 smithi078 bash[75299]: audit 2023-10-01T11:01:06.203699+0000 mon.a (mon.0) 1631 : audit 0 from='client.? 172.21.15.78:0/3120034396' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-01T11:01:06.863 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:06.895 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:06.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:06.898 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:01:06.999 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:07.024 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:07.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:07.027 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:01:07.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:06 smithi078 bash[78274]: audit 2023-10-01T11:01:05.838013+0000 mon.a (mon.0) 1630 : audit 0 from='client.? 172.21.15.78:0/1878764134' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:01:07.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:06 smithi078 bash[78274]: audit 2023-10-01T11:01:06.203699+0000 mon.a (mon.0) 1631 : audit 0 from='client.? 172.21.15.78:0/3120034396' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-01T11:01:07.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:06 smithi173 bash[58816]: audit 2023-10-01T11:01:05.838013+0000 mon.a (mon.0) 1630 : audit 0 from='client.? 172.21.15.78:0/1878764134' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:01:07.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:06 smithi173 bash[58816]: audit 2023-10-01T11:01:06.203699+0000 mon.a (mon.0) 1631 : audit 0 from='client.? 172.21.15.78:0/3120034396' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-01T11:01:07.128 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:07.155 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:07.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:07.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:07.258 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:07.290 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:07.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:07.293 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:01:07.394 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:07.418 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:07.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:07.421 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:01:07.434 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T11:01:07.507 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-01T11:01:07.522 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:07.554 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:07.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:07.556 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:01:07.657 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:07.694 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:07.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:07.699 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:01:07.724 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:07 smithi078 bash[75299]: cluster 2023-10-01T11:01:06.186617+0000 mgr.y (mgr.25072) 1312 : cluster 0 pgmap v1347: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 361 op/s 2023-10-01T11:01:07.724 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:07 smithi078 bash[75299]: audit 2023-10-01T11:01:06.710934+0000 mon.a (mon.0) 1632 : audit 1 from='client.? 172.21.15.78:0/3120034396' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-10-01T11:01:07.724 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:07 smithi078 bash[75299]: cluster 2023-10-01T11:01:06.711101+0000 mon.a (mon.0) 1633 : cluster 0 osdmap e272: 8 total, 8 up, 8 in 2023-10-01T11:01:07.725 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:07 smithi078 bash[75299]: cluster 2023-10-01T11:01:06.714770+0000 osd.5 (osd.5) 254 : cluster 3 failed to encode map e272 with expected crc 2023-10-01T11:01:07.725 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:07 smithi078 bash[75299]: cluster 2023-10-01T11:01:06.715081+0000 osd.6 (osd.6) 224 : cluster 3 failed to encode map e272 with expected crc 2023-10-01T11:01:07.725 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:07 smithi078 bash[75299]: cluster 2023-10-01T11:01:06.717244+0000 osd.7 (osd.7) 268 : cluster 3 failed to encode map e272 with expected crc 2023-10-01T11:01:07.725 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:07 smithi078 bash[75299]: cluster 2023-10-01T11:01:06.717763+0000 osd.6 (osd.6) 225 : cluster 3 failed to encode map e272 with expected crc 2023-10-01T11:01:07.725 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:07 smithi078 bash[75299]: cluster 2023-10-01T11:01:06.718520+0000 osd.5 (osd.5) 255 : cluster 3 failed to encode map e272 with expected crc 2023-10-01T11:01:07.725 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:07 smithi078 bash[75299]: cluster 2023-10-01T11:01:06.719337+0000 osd.7 (osd.7) 269 : cluster 3 failed to encode map e272 with expected crc 2023-10-01T11:01:07.725 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:07 smithi078 bash[75299]: cluster 2023-10-01T11:01:06.719784+0000 osd.6 (osd.6) 226 : cluster 3 failed to encode map e272 with expected crc 2023-10-01T11:01:07.725 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:07 smithi078 bash[75299]: cluster 2023-10-01T11:01:06.720696+0000 osd.5 (osd.5) 256 : cluster 3 failed to encode map e272 with expected crc 2023-10-01T11:01:07.725 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:07 smithi078 bash[75299]: cluster 2023-10-01T11:01:06.721331+0000 osd.7 (osd.7) 270 : cluster 3 failed to encode map e272 with expected crc 2023-10-01T11:01:07.725 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:07 smithi078 bash[75299]: cluster 2023-10-01T11:01:06.721936+0000 osd.6 (osd.6) 227 : cluster 3 failed to encode map e272 with expected crc 2023-10-01T11:01:07.725 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:07 smithi078 bash[75299]: cluster 2023-10-01T11:01:06.724209+0000 osd.6 (osd.6) 228 : cluster 3 failed to encode map e272 with expected crc 2023-10-01T11:01:07.726 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:07 smithi078 bash[75299]: cluster 2023-10-01T11:01:06.725987+0000 osd.6 (osd.6) 229 : cluster 3 failed to encode map e272 with expected crc 2023-10-01T11:01:07.726 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:07 smithi078 bash[75299]: cluster 2023-10-01T11:01:07.660515+0000 mon.a (mon.0) 1634 : cluster 0 osdmap e273: 8 total, 8 up, 8 in 2023-10-01T11:01:07.799 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:07.821 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:07.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:07.823 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:07.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:07 smithi173 bash[58816]: cluster 2023-10-01T11:01:06.186617+0000 mgr.y (mgr.25072) 1312 : cluster 0 pgmap v1347: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 361 op/s 2023-10-01T11:01:07.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:07 smithi173 bash[58816]: audit 2023-10-01T11:01:06.710934+0000 mon.a (mon.0) 1632 : audit 1 from='client.? 172.21.15.78:0/3120034396' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-10-01T11:01:07.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:07 smithi173 bash[58816]: cluster 2023-10-01T11:01:06.711101+0000 mon.a (mon.0) 1633 : cluster 0 osdmap e272: 8 total, 8 up, 8 in 2023-10-01T11:01:07.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:07 smithi173 bash[58816]: cluster 2023-10-01T11:01:06.714770+0000 osd.5 (osd.5) 254 : cluster 3 failed to encode map e272 with expected crc 2023-10-01T11:01:07.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:07 smithi173 bash[58816]: cluster 2023-10-01T11:01:06.715081+0000 osd.6 (osd.6) 224 : cluster 3 failed to encode map e272 with expected crc 2023-10-01T11:01:07.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:07 smithi173 bash[58816]: cluster 2023-10-01T11:01:06.717244+0000 osd.7 (osd.7) 268 : cluster 3 failed to encode map e272 with expected crc 2023-10-01T11:01:07.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:07 smithi173 bash[58816]: cluster 2023-10-01T11:01:06.717763+0000 osd.6 (osd.6) 225 : cluster 3 failed to encode map e272 with expected crc 2023-10-01T11:01:07.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:07 smithi173 bash[58816]: cluster 2023-10-01T11:01:06.718520+0000 osd.5 (osd.5) 255 : cluster 3 failed to encode map e272 with expected crc 2023-10-01T11:01:07.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:07 smithi173 bash[58816]: cluster 2023-10-01T11:01:06.719337+0000 osd.7 (osd.7) 269 : cluster 3 failed to encode map e272 with expected crc 2023-10-01T11:01:07.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:07 smithi173 bash[58816]: cluster 2023-10-01T11:01:06.719784+0000 osd.6 (osd.6) 226 : cluster 3 failed to encode map e272 with expected crc 2023-10-01T11:01:07.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:07 smithi173 bash[58816]: cluster 2023-10-01T11:01:06.720696+0000 osd.5 (osd.5) 256 : cluster 3 failed to encode map e272 with expected crc 2023-10-01T11:01:07.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:07 smithi173 bash[58816]: cluster 2023-10-01T11:01:06.721331+0000 osd.7 (osd.7) 270 : cluster 3 failed to encode map e272 with expected crc 2023-10-01T11:01:07.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:07 smithi173 bash[58816]: cluster 2023-10-01T11:01:06.721936+0000 osd.6 (osd.6) 227 : cluster 3 failed to encode map e272 with expected crc 2023-10-01T11:01:07.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:07 smithi173 bash[58816]: cluster 2023-10-01T11:01:06.724209+0000 osd.6 (osd.6) 228 : cluster 3 failed to encode map e272 with expected crc 2023-10-01T11:01:07.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:07 smithi173 bash[58816]: cluster 2023-10-01T11:01:06.725987+0000 osd.6 (osd.6) 229 : cluster 3 failed to encode map e272 with expected crc 2023-10-01T11:01:07.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:07 smithi173 bash[58816]: cluster 2023-10-01T11:01:07.660515+0000 mon.a (mon.0) 1634 : cluster 0 osdmap e273: 8 total, 8 up, 8 in 2023-10-01T11:01:07.924 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:07.942 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:07.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:07.945 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:01:08.045 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:08.079 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:08.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:07 smithi078 bash[78274]: cluster 2023-10-01T11:01:06.186617+0000 mgr.y (mgr.25072) 1312 : cluster 0 pgmap v1347: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 361 op/s 2023-10-01T11:01:08.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:07 smithi078 bash[78274]: audit 2023-10-01T11:01:06.710934+0000 mon.a (mon.0) 1632 : audit 1 from='client.? 172.21.15.78:0/3120034396' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-10-01T11:01:08.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:07 smithi078 bash[78274]: cluster 2023-10-01T11:01:06.711101+0000 mon.a (mon.0) 1633 : cluster 0 osdmap e272: 8 total, 8 up, 8 in 2023-10-01T11:01:08.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:07 smithi078 bash[78274]: cluster 2023-10-01T11:01:06.714770+0000 osd.5 (osd.5) 254 : cluster 3 failed to encode map e272 with expected crc 2023-10-01T11:01:08.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:07 smithi078 bash[78274]: cluster 2023-10-01T11:01:06.715081+0000 osd.6 (osd.6) 224 : cluster 3 failed to encode map e272 with expected crc 2023-10-01T11:01:08.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:07 smithi078 bash[78274]: cluster 2023-10-01T11:01:06.717244+0000 osd.7 (osd.7) 268 : cluster 3 failed to encode map e272 with expected crc 2023-10-01T11:01:08.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:07 smithi078 bash[78274]: cluster 2023-10-01T11:01:06.717763+0000 osd.6 (osd.6) 225 : cluster 3 failed to encode map e272 with expected crc 2023-10-01T11:01:08.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:07 smithi078 bash[78274]: cluster 2023-10-01T11:01:06.718520+0000 osd.5 (osd.5) 255 : cluster 3 failed to encode map e272 with expected crc 2023-10-01T11:01:08.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:07 smithi078 bash[78274]: cluster 2023-10-01T11:01:06.719337+0000 osd.7 (osd.7) 269 : cluster 3 failed to encode map e272 with expected crc 2023-10-01T11:01:08.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:07 smithi078 bash[78274]: cluster 2023-10-01T11:01:06.719784+0000 osd.6 (osd.6) 226 : cluster 3 failed to encode map e272 with expected crc 2023-10-01T11:01:08.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:07 smithi078 bash[78274]: cluster 2023-10-01T11:01:06.720696+0000 osd.5 (osd.5) 256 : cluster 3 failed to encode map e272 with expected crc 2023-10-01T11:01:08.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:07 smithi078 bash[78274]: cluster 2023-10-01T11:01:06.721331+0000 osd.7 (osd.7) 270 : cluster 3 failed to encode map e272 with expected crc 2023-10-01T11:01:08.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:07 smithi078 bash[78274]: cluster 2023-10-01T11:01:06.721936+0000 osd.6 (osd.6) 227 : cluster 3 failed to encode map e272 with expected crc 2023-10-01T11:01:08.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:07 smithi078 bash[78274]: cluster 2023-10-01T11:01:06.724209+0000 osd.6 (osd.6) 228 : cluster 3 failed to encode map e272 with expected crc 2023-10-01T11:01:08.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:07 smithi078 bash[78274]: cluster 2023-10-01T11:01:06.725987+0000 osd.6 (osd.6) 229 : cluster 3 failed to encode map e272 with expected crc 2023-10-01T11:01:08.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:07 smithi078 bash[78274]: cluster 2023-10-01T11:01:07.660515+0000 mon.a (mon.0) 1634 : cluster 0 osdmap e273: 8 total, 8 up, 8 in 2023-10-01T11:01:08.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:08.085 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:01:08.186 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:08.210 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:08.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:08.212 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:01:08.313 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:08.347 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:08.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:08.351 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:01:08.451 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:08.480 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:08.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:08.484 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:01:08.585 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:08.622 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:08.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:08.624 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:01:08.725 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:08.750 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:08.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:08.753 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:01:08.854 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:08.883 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:08.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:08.886 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:08.987 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:08.990 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:08 smithi078 bash[75299]: cluster 2023-10-01T11:01:07.659501+0000 osd.5 (osd.5) 257 : cluster 3 failed to encode map e273 with expected crc 2023-10-01T11:01:08.991 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:08 smithi078 bash[75299]: cluster 2023-10-01T11:01:07.659921+0000 osd.7 (osd.7) 271 : cluster 3 failed to encode map e273 with expected crc 2023-10-01T11:01:08.991 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:08 smithi078 bash[75299]: cluster 2023-10-01T11:01:07.661908+0000 osd.6 (osd.6) 230 : cluster 3 failed to encode map e273 with expected crc 2023-10-01T11:01:08.991 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:08 smithi078 bash[75299]: cluster 2023-10-01T11:01:07.662801+0000 osd.5 (osd.5) 258 : cluster 3 failed to encode map e273 with expected crc 2023-10-01T11:01:08.991 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:08 smithi078 bash[75299]: cluster 2023-10-01T11:01:07.666713+0000 osd.5 (osd.5) 259 : cluster 3 failed to encode map e273 with expected crc 2023-10-01T11:01:08.991 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:08 smithi078 bash[75299]: cluster 2023-10-01T11:01:07.670322+0000 osd.6 (osd.6) 231 : cluster 3 failed to encode map e273 with expected crc 2023-10-01T11:01:08.991 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:08 smithi078 bash[75299]: cluster 2023-10-01T11:01:07.670678+0000 osd.7 (osd.7) 272 : cluster 3 failed to encode map e273 with expected crc 2023-10-01T11:01:08.991 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:08 smithi078 bash[75299]: cluster 2023-10-01T11:01:07.672846+0000 osd.7 (osd.7) 273 : cluster 3 failed to encode map e273 with expected crc 2023-10-01T11:01:08.991 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:08 smithi078 bash[75299]: cluster 2023-10-01T11:01:07.674513+0000 osd.6 (osd.6) 232 : cluster 3 failed to encode map e273 with expected crc 2023-10-01T11:01:08.992 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:08 smithi078 bash[75299]: cluster 2023-10-01T11:01:08.663265+0000 mon.a (mon.0) 1635 : cluster 0 osdmap e274: 8 total, 8 up, 8 in 2023-10-01T11:01:08.992 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:08 smithi078 bash[78274]: cluster 2023-10-01T11:01:07.659501+0000 osd.5 (osd.5) 257 : cluster 3 failed to encode map e273 with expected crc 2023-10-01T11:01:08.992 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:08 smithi078 bash[78274]: cluster 2023-10-01T11:01:07.659921+0000 osd.7 (osd.7) 271 : cluster 3 failed to encode map e273 with expected crc 2023-10-01T11:01:08.992 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:08 smithi078 bash[78274]: cluster 2023-10-01T11:01:07.661908+0000 osd.6 (osd.6) 230 : cluster 3 failed to encode map e273 with expected crc 2023-10-01T11:01:08.992 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:08 smithi078 bash[78274]: cluster 2023-10-01T11:01:07.662801+0000 osd.5 (osd.5) 258 : cluster 3 failed to encode map e273 with expected crc 2023-10-01T11:01:08.992 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:08 smithi078 bash[78274]: cluster 2023-10-01T11:01:07.666713+0000 osd.5 (osd.5) 259 : cluster 3 failed to encode map e273 with expected crc 2023-10-01T11:01:08.993 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:08 smithi078 bash[78274]: cluster 2023-10-01T11:01:07.670322+0000 osd.6 (osd.6) 231 : cluster 3 failed to encode map e273 with expected crc 2023-10-01T11:01:08.993 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:08 smithi078 bash[78274]: cluster 2023-10-01T11:01:07.670678+0000 osd.7 (osd.7) 272 : cluster 3 failed to encode map e273 with expected crc 2023-10-01T11:01:08.993 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:08 smithi078 bash[78274]: cluster 2023-10-01T11:01:07.672846+0000 osd.7 (osd.7) 273 : cluster 3 failed to encode map e273 with expected crc 2023-10-01T11:01:08.993 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:08 smithi078 bash[78274]: cluster 2023-10-01T11:01:07.674513+0000 osd.6 (osd.6) 232 : cluster 3 failed to encode map e273 with expected crc 2023-10-01T11:01:08.993 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:08 smithi078 bash[78274]: cluster 2023-10-01T11:01:08.663265+0000 mon.a (mon.0) 1635 : cluster 0 osdmap e274: 8 total, 8 up, 8 in 2023-10-01T11:01:08.994 INFO:tasks.thrashosds.thrasher:Added osd 6 2023-10-01T11:01:08.999 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:08 smithi173 bash[58816]: cluster 2023-10-01T11:01:07.659501+0000 osd.5 (osd.5) 257 : cluster 3 failed to encode map e273 with expected crc 2023-10-01T11:01:08.999 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:08 smithi173 bash[58816]: cluster 2023-10-01T11:01:07.659921+0000 osd.7 (osd.7) 271 : cluster 3 failed to encode map e273 with expected crc 2023-10-01T11:01:08.999 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:08 smithi173 bash[58816]: cluster 2023-10-01T11:01:07.661908+0000 osd.6 (osd.6) 230 : cluster 3 failed to encode map e273 with expected crc 2023-10-01T11:01:08.999 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:08 smithi173 bash[58816]: cluster 2023-10-01T11:01:07.662801+0000 osd.5 (osd.5) 258 : cluster 3 failed to encode map e273 with expected crc 2023-10-01T11:01:08.999 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:08 smithi173 bash[58816]: cluster 2023-10-01T11:01:07.666713+0000 osd.5 (osd.5) 259 : cluster 3 failed to encode map e273 with expected crc 2023-10-01T11:01:08.999 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:08 smithi173 bash[58816]: cluster 2023-10-01T11:01:07.670322+0000 osd.6 (osd.6) 231 : cluster 3 failed to encode map e273 with expected crc 2023-10-01T11:01:08.999 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:08 smithi173 bash[58816]: cluster 2023-10-01T11:01:07.670678+0000 osd.7 (osd.7) 272 : cluster 3 failed to encode map e273 with expected crc 2023-10-01T11:01:09.000 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:08 smithi173 bash[58816]: cluster 2023-10-01T11:01:07.672846+0000 osd.7 (osd.7) 273 : cluster 3 failed to encode map e273 with expected crc 2023-10-01T11:01:09.000 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:08 smithi173 bash[58816]: cluster 2023-10-01T11:01:07.674513+0000 osd.6 (osd.6) 232 : cluster 3 failed to encode map e273 with expected crc 2023-10-01T11:01:09.000 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:08 smithi173 bash[58816]: cluster 2023-10-01T11:01:08.663265+0000 mon.a (mon.0) 1635 : cluster 0 osdmap e274: 8 total, 8 up, 8 in 2023-10-01T11:01:09.016 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:09.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:09.019 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:01:09.119 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:09.153 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:09.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:09.156 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:09.257 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:09.289 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:09.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:09.292 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:01:09.393 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:09.424 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:09.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:09.427 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:01:09.527 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:09.562 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:09.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:09.565 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:01:09.665 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:09.696 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:09.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:09.699 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:01:09.800 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:09.834 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:09.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:09.837 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:01:09.938 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:09.969 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:09.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:09.972 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:01:10.072 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:10.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:09 smithi078 bash[75299]: cluster 2023-10-01T11:01:08.187397+0000 mgr.y (mgr.25072) 1313 : cluster 0 pgmap v1350: 89 pgs: 3 remapped+peering, 86 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 80 op/s 2023-10-01T11:01:10.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:09 smithi078 bash[75299]: cluster 2023-10-01T11:01:08.661646+0000 osd.5 (osd.5) 260 : cluster 3 failed to encode map e274 with expected crc 2023-10-01T11:01:10.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:09 smithi078 bash[75299]: cluster 2023-10-01T11:01:08.662994+0000 osd.7 (osd.7) 274 : cluster 3 failed to encode map e274 with expected crc 2023-10-01T11:01:10.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:09 smithi078 bash[75299]: cluster 2023-10-01T11:01:08.664973+0000 osd.4 (osd.4) 5 : cluster 0 12.0 starting backfill to osd.6 from (0'0,0'0] MAX to 270'12766 2023-10-01T11:01:10.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:09 smithi078 bash[75299]: cluster 2023-10-01T11:01:08.665458+0000 osd.3 (osd.3) 27 : cluster 0 12.1 starting backfill to osd.6 from (0'0,0'0] MAX to 270'12940 2023-10-01T11:01:10.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:09 smithi078 bash[75299]: cluster 2023-10-01T11:01:08.665585+0000 osd.3 (osd.3) 28 : cluster 0 12.4 starting backfill to osd.6 from (0'0,0'0] MAX to 270'12898 2023-10-01T11:01:10.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:09 smithi078 bash[75299]: cluster 2023-10-01T11:01:08.665630+0000 osd.3 (osd.3) 29 : cluster 0 12.b starting backfill to osd.6 from (0'0,0'0] MAX to 270'13046 2023-10-01T11:01:10.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:09 smithi078 bash[75299]: cluster 2023-10-01T11:01:08.667680+0000 osd.2 (osd.2) 15 : cluster 0 12.9 starting backfill to osd.6 from (0'0,0'0] MAX to 270'12770 2023-10-01T11:01:10.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:09 smithi078 bash[75299]: cluster 2023-10-01T11:01:08.667776+0000 osd.2 (osd.2) 16 : cluster 0 12.f starting backfill to osd.6 from (0'0,0'0] MAX to 270'13076 2023-10-01T11:01:10.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:09 smithi078 bash[75299]: cluster 2023-10-01T11:01:08.667946+0000 osd.2 (osd.2) 17 : cluster 0 12.5 starting backfill to osd.6 from (0'0,0'0] MAX to 270'12532 2023-10-01T11:01:10.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:09 smithi078 bash[75299]: cluster 2023-10-01T11:01:08.679940+0000 osd.6 (osd.6) 233 : cluster 3 failed to encode map e274 with expected crc 2023-10-01T11:01:10.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:09 smithi078 bash[75299]: cluster 2023-10-01T11:01:08.679942+0000 osd.5 (osd.5) 261 : cluster 3 failed to encode map e274 with expected crc 2023-10-01T11:01:10.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:09 smithi078 bash[78274]: cluster 2023-10-01T11:01:08.187397+0000 mgr.y (mgr.25072) 1313 : cluster 0 pgmap v1350: 89 pgs: 3 remapped+peering, 86 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 80 op/s 2023-10-01T11:01:10.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:09 smithi078 bash[78274]: cluster 2023-10-01T11:01:08.661646+0000 osd.5 (osd.5) 260 : cluster 3 failed to encode map e274 with expected crc 2023-10-01T11:01:10.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:09 smithi078 bash[78274]: cluster 2023-10-01T11:01:08.662994+0000 osd.7 (osd.7) 274 : cluster 3 failed to encode map e274 with expected crc 2023-10-01T11:01:10.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:09 smithi078 bash[78274]: cluster 2023-10-01T11:01:08.664973+0000 osd.4 (osd.4) 5 : cluster 0 12.0 starting backfill to osd.6 from (0'0,0'0] MAX to 270'12766 2023-10-01T11:01:10.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:09 smithi078 bash[78274]: cluster 2023-10-01T11:01:08.665458+0000 osd.3 (osd.3) 27 : cluster 0 12.1 starting backfill to osd.6 from (0'0,0'0] MAX to 270'12940 2023-10-01T11:01:10.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:09 smithi078 bash[78274]: cluster 2023-10-01T11:01:08.665585+0000 osd.3 (osd.3) 28 : cluster 0 12.4 starting backfill to osd.6 from (0'0,0'0] MAX to 270'12898 2023-10-01T11:01:10.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:09 smithi078 bash[78274]: cluster 2023-10-01T11:01:08.665630+0000 osd.3 (osd.3) 29 : cluster 0 12.b starting backfill to osd.6 from (0'0,0'0] MAX to 270'13046 2023-10-01T11:01:10.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:09 smithi078 bash[78274]: cluster 2023-10-01T11:01:08.667680+0000 osd.2 (osd.2) 15 : cluster 0 12.9 starting backfill to osd.6 from (0'0,0'0] MAX to 270'12770 2023-10-01T11:01:10.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:09 smithi078 bash[78274]: cluster 2023-10-01T11:01:08.667776+0000 osd.2 (osd.2) 16 : cluster 0 12.f starting backfill to osd.6 from (0'0,0'0] MAX to 270'13076 2023-10-01T11:01:10.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:09 smithi078 bash[78274]: cluster 2023-10-01T11:01:08.667946+0000 osd.2 (osd.2) 17 : cluster 0 12.5 starting backfill to osd.6 from (0'0,0'0] MAX to 270'12532 2023-10-01T11:01:10.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:09 smithi078 bash[78274]: cluster 2023-10-01T11:01:08.679940+0000 osd.6 (osd.6) 233 : cluster 3 failed to encode map e274 with expected crc 2023-10-01T11:01:10.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:09 smithi078 bash[78274]: cluster 2023-10-01T11:01:08.679942+0000 osd.5 (osd.5) 261 : cluster 3 failed to encode map e274 with expected crc 2023-10-01T11:01:10.085 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:09 smithi173 bash[58816]: cluster 2023-10-01T11:01:08.187397+0000 mgr.y (mgr.25072) 1313 : cluster 0 pgmap v1350: 89 pgs: 3 remapped+peering, 86 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 80 op/s 2023-10-01T11:01:10.085 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:09 smithi173 bash[58816]: cluster 2023-10-01T11:01:08.661646+0000 osd.5 (osd.5) 260 : cluster 3 failed to encode map e274 with expected crc 2023-10-01T11:01:10.085 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:09 smithi173 bash[58816]: cluster 2023-10-01T11:01:08.662994+0000 osd.7 (osd.7) 274 : cluster 3 failed to encode map e274 with expected crc 2023-10-01T11:01:10.085 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:09 smithi173 bash[58816]: cluster 2023-10-01T11:01:08.664973+0000 osd.4 (osd.4) 5 : cluster 0 12.0 starting backfill to osd.6 from (0'0,0'0] MAX to 270'12766 2023-10-01T11:01:10.085 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:09 smithi173 bash[58816]: cluster 2023-10-01T11:01:08.665458+0000 osd.3 (osd.3) 27 : cluster 0 12.1 starting backfill to osd.6 from (0'0,0'0] MAX to 270'12940 2023-10-01T11:01:10.086 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:09 smithi173 bash[58816]: cluster 2023-10-01T11:01:08.665585+0000 osd.3 (osd.3) 28 : cluster 0 12.4 starting backfill to osd.6 from (0'0,0'0] MAX to 270'12898 2023-10-01T11:01:10.086 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:09 smithi173 bash[58816]: cluster 2023-10-01T11:01:08.665630+0000 osd.3 (osd.3) 29 : cluster 0 12.b starting backfill to osd.6 from (0'0,0'0] MAX to 270'13046 2023-10-01T11:01:10.086 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:09 smithi173 bash[58816]: cluster 2023-10-01T11:01:08.667680+0000 osd.2 (osd.2) 15 : cluster 0 12.9 starting backfill to osd.6 from (0'0,0'0] MAX to 270'12770 2023-10-01T11:01:10.086 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:09 smithi173 bash[58816]: cluster 2023-10-01T11:01:08.667776+0000 osd.2 (osd.2) 16 : cluster 0 12.f starting backfill to osd.6 from (0'0,0'0] MAX to 270'13076 2023-10-01T11:01:10.086 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:09 smithi173 bash[58816]: cluster 2023-10-01T11:01:08.667946+0000 osd.2 (osd.2) 17 : cluster 0 12.5 starting backfill to osd.6 from (0'0,0'0] MAX to 270'12532 2023-10-01T11:01:10.086 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:09 smithi173 bash[58816]: cluster 2023-10-01T11:01:08.679940+0000 osd.6 (osd.6) 233 : cluster 3 failed to encode map e274 with expected crc 2023-10-01T11:01:10.086 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:09 smithi173 bash[58816]: cluster 2023-10-01T11:01:08.679942+0000 osd.5 (osd.5) 261 : cluster 3 failed to encode map e274 with expected crc 2023-10-01T11:01:10.106 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:10.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:10.108 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:01:10.209 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:10.243 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:10.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:10.247 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:01:10.348 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:10.384 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:10.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:10.387 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:01:10.488 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:10.494 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:01:10.517 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:10.521 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:01:10 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:01:10] "GET /metrics HTTP/1.1" 200 36373 "" "Prometheus/2.33.4" 2023-10-01T11:01:10.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:10.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:10.628 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:10.661 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:10.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:10.664 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:01:10.765 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:10.787 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:10.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:10.790 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:01:10.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:10 smithi078 bash[75299]: cluster 2023-10-01T11:01:09.741054+0000 osd.7 (osd.7) 275 : cluster 3 failed to encode map e275 with expected crc 2023-10-01T11:01:10.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:10 smithi078 bash[75299]: cluster 2023-10-01T11:01:09.743405+0000 mon.a (mon.0) 1636 : cluster 0 osdmap e275: 8 total, 8 up, 8 in 2023-10-01T11:01:10.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:10 smithi078 bash[75299]: cluster 2023-10-01T11:01:09.746160+0000 osd.6 (osd.6) 234 : cluster 3 failed to encode map e275 with expected crc 2023-10-01T11:01:10.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:10 smithi078 bash[75299]: cluster 2023-10-01T11:01:09.749851+0000 osd.6 (osd.6) 235 : cluster 3 failed to encode map e275 with expected crc 2023-10-01T11:01:10.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:10 smithi078 bash[75299]: cluster 2023-10-01T11:01:09.751276+0000 osd.6 (osd.6) 236 : cluster 3 failed to encode map e275 with expected crc 2023-10-01T11:01:10.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:10 smithi078 bash[75299]: cluster 2023-10-01T11:01:09.753161+0000 osd.6 (osd.6) 237 : cluster 3 failed to encode map e275 with expected crc 2023-10-01T11:01:10.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:10 smithi078 bash[78274]: cluster 2023-10-01T11:01:09.741054+0000 osd.7 (osd.7) 275 : cluster 3 failed to encode map e275 with expected crc 2023-10-01T11:01:10.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:10 smithi078 bash[78274]: cluster 2023-10-01T11:01:09.743405+0000 mon.a (mon.0) 1636 : cluster 0 osdmap e275: 8 total, 8 up, 8 in 2023-10-01T11:01:10.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:10 smithi078 bash[78274]: cluster 2023-10-01T11:01:09.746160+0000 osd.6 (osd.6) 234 : cluster 3 failed to encode map e275 with expected crc 2023-10-01T11:01:10.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:10 smithi078 bash[78274]: cluster 2023-10-01T11:01:09.749851+0000 osd.6 (osd.6) 235 : cluster 3 failed to encode map e275 with expected crc 2023-10-01T11:01:10.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:10 smithi078 bash[78274]: cluster 2023-10-01T11:01:09.751276+0000 osd.6 (osd.6) 236 : cluster 3 failed to encode map e275 with expected crc 2023-10-01T11:01:10.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:10 smithi078 bash[78274]: cluster 2023-10-01T11:01:09.753161+0000 osd.6 (osd.6) 237 : cluster 3 failed to encode map e275 with expected crc 2023-10-01T11:01:10.892 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:10.904 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T11:01:10.905 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:10 smithi173 bash[58816]: cluster 2023-10-01T11:01:09.741054+0000 osd.7 (osd.7) 275 : cluster 3 failed to encode map e275 with expected crc 2023-10-01T11:01:10.905 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:10 smithi173 bash[58816]: cluster 2023-10-01T11:01:09.743405+0000 mon.a (mon.0) 1636 : cluster 0 osdmap e275: 8 total, 8 up, 8 in 2023-10-01T11:01:10.905 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:10 smithi173 bash[58816]: cluster 2023-10-01T11:01:09.746160+0000 osd.6 (osd.6) 234 : cluster 3 failed to encode map e275 with expected crc 2023-10-01T11:01:10.905 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:10 smithi173 bash[58816]: cluster 2023-10-01T11:01:09.749851+0000 osd.6 (osd.6) 235 : cluster 3 failed to encode map e275 with expected crc 2023-10-01T11:01:10.905 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:10 smithi173 bash[58816]: cluster 2023-10-01T11:01:09.751276+0000 osd.6 (osd.6) 236 : cluster 3 failed to encode map e275 with expected crc 2023-10-01T11:01:10.905 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:10 smithi173 bash[58816]: cluster 2023-10-01T11:01:09.753161+0000 osd.6 (osd.6) 237 : cluster 3 failed to encode map e275 with expected crc 2023-10-01T11:01:10.922 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:10.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:10.924 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:01:11.025 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:11.045 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:11.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:11.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:01:11.148 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:11.172 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:11.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:11.174 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:01:11.275 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:11.302 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:11.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:11.306 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:01:11.406 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:11.432 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:11.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:11.434 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:11.535 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:11.568 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:11.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:11.571 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:01:11.671 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:11.707 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:11.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:11.710 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:01:11.811 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:11.844 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:11.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:11.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:01:11.949 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:11.978 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:11.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:11.980 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:01:12.081 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:12.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:11 smithi078 bash[75299]: cluster 2023-10-01T11:01:10.188146+0000 mgr.y (mgr.25072) 1314 : cluster 0 pgmap v1353: 89 pgs: 4 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 81 active+clean; 582 KiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 2/72 objects degraded (2.778%); 1.2 KiB/s, 2 objects/s recovering 2023-10-01T11:01:12.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:11 smithi078 bash[75299]: cluster 2023-10-01T11:01:10.739676+0000 mon.a (mon.0) 1637 : cluster 3 Health check failed: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2023-10-01T11:01:12.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:11 smithi078 bash[75299]: cluster 2023-10-01T11:01:10.749683+0000 osd.5 (osd.5) 262 : cluster 3 failed to encode map e276 with expected crc 2023-10-01T11:01:12.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:11 smithi078 bash[75299]: cluster 2023-10-01T11:01:10.750655+0000 mon.a (mon.0) 1638 : cluster 0 osdmap e276: 8 total, 8 up, 8 in 2023-10-01T11:01:12.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:11 smithi078 bash[75299]: cluster 2023-10-01T11:01:10.751763+0000 osd.6 (osd.6) 238 : cluster 3 failed to encode map e276 with expected crc 2023-10-01T11:01:12.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:11 smithi078 bash[75299]: cluster 2023-10-01T11:01:10.754738+0000 osd.7 (osd.7) 276 : cluster 3 failed to encode map e276 with expected crc 2023-10-01T11:01:12.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:11 smithi078 bash[75299]: cluster 2023-10-01T11:01:10.756354+0000 osd.7 (osd.7) 277 : cluster 3 failed to encode map e276 with expected crc 2023-10-01T11:01:12.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:11 smithi078 bash[75299]: cluster 2023-10-01T11:01:10.757449+0000 osd.6 (osd.6) 239 : cluster 3 failed to encode map e276 with expected crc 2023-10-01T11:01:12.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:11 smithi078 bash[78274]: cluster 2023-10-01T11:01:10.188146+0000 mgr.y (mgr.25072) 1314 : cluster 0 pgmap v1353: 89 pgs: 4 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 81 active+clean; 582 KiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 2/72 objects degraded (2.778%); 1.2 KiB/s, 2 objects/s recovering 2023-10-01T11:01:12.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:11 smithi078 bash[78274]: cluster 2023-10-01T11:01:10.739676+0000 mon.a (mon.0) 1637 : cluster 3 Health check failed: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2023-10-01T11:01:12.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:11 smithi078 bash[78274]: cluster 2023-10-01T11:01:10.749683+0000 osd.5 (osd.5) 262 : cluster 3 failed to encode map e276 with expected crc 2023-10-01T11:01:12.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:11 smithi078 bash[78274]: cluster 2023-10-01T11:01:10.750655+0000 mon.a (mon.0) 1638 : cluster 0 osdmap e276: 8 total, 8 up, 8 in 2023-10-01T11:01:12.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:11 smithi078 bash[78274]: cluster 2023-10-01T11:01:10.751763+0000 osd.6 (osd.6) 238 : cluster 3 failed to encode map e276 with expected crc 2023-10-01T11:01:12.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:11 smithi078 bash[78274]: cluster 2023-10-01T11:01:10.754738+0000 osd.7 (osd.7) 276 : cluster 3 failed to encode map e276 with expected crc 2023-10-01T11:01:12.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:11 smithi078 bash[78274]: cluster 2023-10-01T11:01:10.756354+0000 osd.7 (osd.7) 277 : cluster 3 failed to encode map e276 with expected crc 2023-10-01T11:01:12.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:11 smithi078 bash[78274]: cluster 2023-10-01T11:01:10.757449+0000 osd.6 (osd.6) 239 : cluster 3 failed to encode map e276 with expected crc 2023-10-01T11:01:12.114 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:12.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:12.116 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:01:12.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:11 smithi173 bash[58816]: cluster 2023-10-01T11:01:10.188146+0000 mgr.y (mgr.25072) 1314 : cluster 0 pgmap v1353: 89 pgs: 4 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 81 active+clean; 582 KiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 2/72 objects degraded (2.778%); 1.2 KiB/s, 2 objects/s recovering 2023-10-01T11:01:12.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:11 smithi173 bash[58816]: cluster 2023-10-01T11:01:10.739676+0000 mon.a (mon.0) 1637 : cluster 3 Health check failed: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2023-10-01T11:01:12.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:11 smithi173 bash[58816]: cluster 2023-10-01T11:01:10.749683+0000 osd.5 (osd.5) 262 : cluster 3 failed to encode map e276 with expected crc 2023-10-01T11:01:12.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:11 smithi173 bash[58816]: cluster 2023-10-01T11:01:10.750655+0000 mon.a (mon.0) 1638 : cluster 0 osdmap e276: 8 total, 8 up, 8 in 2023-10-01T11:01:12.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:11 smithi173 bash[58816]: cluster 2023-10-01T11:01:10.751763+0000 osd.6 (osd.6) 238 : cluster 3 failed to encode map e276 with expected crc 2023-10-01T11:01:12.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:11 smithi173 bash[58816]: cluster 2023-10-01T11:01:10.754738+0000 osd.7 (osd.7) 276 : cluster 3 failed to encode map e276 with expected crc 2023-10-01T11:01:12.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:11 smithi173 bash[58816]: cluster 2023-10-01T11:01:10.756354+0000 osd.7 (osd.7) 277 : cluster 3 failed to encode map e276 with expected crc 2023-10-01T11:01:12.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:11 smithi173 bash[58816]: cluster 2023-10-01T11:01:10.757449+0000 osd.6 (osd.6) 239 : cluster 3 failed to encode map e276 with expected crc 2023-10-01T11:01:12.217 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:12.250 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:12.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:12.253 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:01:12.354 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:12.381 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:12.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:12.384 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:01:12.485 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:12.513 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:12.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:12.516 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:01:12.520 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:01:12.520 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:01:12.617 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:12.649 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:12.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:12.652 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:01:12.753 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:12.787 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:12.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:12.790 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:01:12.890 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:12.923 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:12.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:12.926 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:01:13.027 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:13.050 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:13.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:13.053 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:01:13.154 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:13.177 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:13.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:13.179 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:01:13.280 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:13.305 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:13.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:13.308 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:01:13.408 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:13.440 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:13.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:13.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:01:13.544 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:13.577 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:13.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:13.581 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:01:13.682 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:13.713 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:13.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:13.717 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:01:13.817 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:13.850 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:13.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:13.853 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:01:13.954 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:13.968 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:13 smithi173 bash[58816]: cluster 2023-10-01T11:01:12.188791+0000 mgr.y (mgr.25072) 1315 : cluster 0 pgmap v1355: 89 pgs: 4 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 81 active+clean; 582 KiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 2/72 objects degraded (2.778%); 1.0 KiB/s, 2 objects/s recovering 2023-10-01T11:01:13.986 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:13.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:13.990 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:01:13.995 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 7, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T11:01:13.996 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T11:01:13.996 INFO:tasks.thrashosds.thrasher:Removing osd 7, in_osds are: [0, 1, 2, 3, 4, 5, 7, 6] 2023-10-01T11:01:13.996 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd out 7 2023-10-01T11:01:14.005 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:13 smithi078 bash[78274]: cluster 2023-10-01T11:01:12.188791+0000 mgr.y (mgr.25072) 1315 : cluster 0 pgmap v1355: 89 pgs: 4 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 81 active+clean; 582 KiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 2/72 objects degraded (2.778%); 1.0 KiB/s, 2 objects/s recovering 2023-10-01T11:01:14.005 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:13 smithi078 bash[75299]: cluster 2023-10-01T11:01:12.188791+0000 mgr.y (mgr.25072) 1315 : cluster 0 pgmap v1355: 89 pgs: 4 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 81 active+clean; 582 KiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 2/72 objects degraded (2.778%); 1.0 KiB/s, 2 objects/s recovering 2023-10-01T11:01:14.090 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:14.122 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:14.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:14.125 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:01:14.226 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:14.246 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:14.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:14.249 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:01:14.350 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:14.380 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:14.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:14.383 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:01:14.485 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:14.511 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:14.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:14.514 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:01:14.614 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:14.649 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:14.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:14.653 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:01:14.753 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:14.783 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:14.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:14.786 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:01:14.887 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:14.912 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:14.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:14.914 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:01:15.015 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:15.039 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:15.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:15.043 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:01:15.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:14 smithi078 bash[75299]: cluster 2023-10-01T11:01:14.749197+0000 mon.a (mon.0) 1639 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded) 2023-10-01T11:01:15.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:14 smithi078 bash[75299]: cluster 2023-10-01T11:01:14.749282+0000 mon.a (mon.0) 1640 : cluster 1 Cluster is now healthy 2023-10-01T11:01:15.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:14 smithi078 bash[78274]: cluster 2023-10-01T11:01:14.749197+0000 mon.a (mon.0) 1639 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded) 2023-10-01T11:01:15.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:14 smithi078 bash[78274]: cluster 2023-10-01T11:01:14.749282+0000 mon.a (mon.0) 1640 : cluster 1 Cluster is now healthy 2023-10-01T11:01:15.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:14 smithi173 bash[58816]: cluster 2023-10-01T11:01:14.749197+0000 mon.a (mon.0) 1639 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded) 2023-10-01T11:01:15.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:14 smithi173 bash[58816]: cluster 2023-10-01T11:01:14.749282+0000 mon.a (mon.0) 1640 : cluster 1 Cluster is now healthy 2023-10-01T11:01:15.144 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:15.177 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:15.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:15.180 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:01:15.281 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:15.313 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:15.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:15.316 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:01:15.417 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:15.445 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:15.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:15.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:01:15.548 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:15.585 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:15.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:15.589 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:01:15.690 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:15.716 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:15.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:15.719 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:01:15.820 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:15.833 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:15 smithi173 bash[58816]: cluster 2023-10-01T11:01:14.189457+0000 mgr.y (mgr.25072) 1316 : cluster 0 pgmap v1356: 89 pgs: 3 active+clean+remapped, 86 active+clean; 582 KiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 97 KiB/s, 1 objects/s recovering 2023-10-01T11:01:15.853 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:15.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:15.857 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:01:15.958 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:15.992 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:15.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:15.995 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:01:16.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:15 smithi078 bash[75299]: cluster 2023-10-01T11:01:14.189457+0000 mgr.y (mgr.25072) 1316 : cluster 0 pgmap v1356: 89 pgs: 3 active+clean+remapped, 86 active+clean; 582 KiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 97 KiB/s, 1 objects/s recovering 2023-10-01T11:01:16.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:15 smithi078 bash[78274]: cluster 2023-10-01T11:01:14.189457+0000 mgr.y (mgr.25072) 1316 : cluster 0 pgmap v1356: 89 pgs: 3 active+clean+remapped, 86 active+clean; 582 KiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 97 KiB/s, 1 objects/s recovering 2023-10-01T11:01:16.096 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:16.126 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:16.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:16.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:01:16.231 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:16.257 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:16.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:16.260 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:16.360 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:16.390 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:16.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:16.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:01:16.495 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:16.526 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:16.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:16.528 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:01:16.628 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:16.661 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:16.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:16.663 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:01:16.738 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:01:16.764 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:16.796 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:16.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:16.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:01:16.855 INFO:teuthology.orchestra.run.smithi078.stderr:pool 'unique_pool_7' removed 2023-10-01T11:01:16.900 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:16.903 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:16 smithi078 bash[75299]: audit 2023-10-01T11:01:16.645538+0000 mon.c (mon.2) 202 : audit 1 from='client.? 172.21.15.78:0/1166959636' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T11:01:16.903 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:16 smithi078 bash[75299]: audit 2023-10-01T11:01:16.646386+0000 mon.a (mon.0) 1641 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T11:01:16.903 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:16 smithi078 bash[78274]: audit 2023-10-01T11:01:16.645538+0000 mon.c (mon.2) 202 : audit 1 from='client.? 172.21.15.78:0/1166959636' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T11:01:16.903 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:16 smithi078 bash[78274]: audit 2023-10-01T11:01:16.646386+0000 mon.a (mon.0) 1641 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T11:01:16.913 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:16 smithi173 bash[58816]: audit 2023-10-01T11:01:16.645538+0000 mon.c (mon.2) 202 : audit 1 from='client.? 172.21.15.78:0/1166959636' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T11:01:16.913 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:16 smithi173 bash[58816]: audit 2023-10-01T11:01:16.646386+0000 mon.a (mon.0) 1641 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T11:01:16.931 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:16.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:16.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:01:17.035 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:17.058 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:17.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:17.061 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:01:17.161 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:17.187 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:17.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:17.189 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:01:17.290 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:17.322 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:17.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:17.326 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:01:17.426 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:17.449 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:17.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:17.452 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:01:17.553 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:17.577 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:17.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:17.579 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:17.680 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:17.711 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:17.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:17.714 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:01:17.815 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:17.844 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:17.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:17.847 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:01:17.948 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:17.973 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:17.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:17.975 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:01:18.076 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:18.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:17 smithi078 bash[75299]: cluster 2023-10-01T11:01:16.189985+0000 mgr.y (mgr.25072) 1317 : cluster 0 pgmap v1357: 89 pgs: 89 active+clean; 582 KiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 77 KiB/s, 1 objects/s recovering 2023-10-01T11:01:18.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:17 smithi078 bash[75299]: audit 2023-10-01T11:01:16.820244+0000 mon.a (mon.0) 1642 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]': finished 2023-10-01T11:01:18.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:17 smithi078 bash[75299]: cluster 2023-10-01T11:01:16.820340+0000 mon.a (mon.0) 1643 : cluster 0 osdmap e277: 8 total, 8 up, 8 in 2023-10-01T11:01:18.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:17 smithi078 bash[75299]: cluster 2023-10-01T11:01:17.281160+0000 osd.7 (osd.7) 278 : cluster 3 failed to encode map e277 with expected crc 2023-10-01T11:01:18.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:17 smithi078 bash[75299]: cluster 2023-10-01T11:01:17.283805+0000 osd.7 (osd.7) 279 : cluster 3 failed to encode map e277 with expected crc 2023-10-01T11:01:18.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:17 smithi078 bash[75299]: cluster 2023-10-01T11:01:17.296500+0000 osd.6 (osd.6) 240 : cluster 3 failed to encode map e277 with expected crc 2023-10-01T11:01:18.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:17 smithi078 bash[75299]: cluster 2023-10-01T11:01:17.298485+0000 osd.6 (osd.6) 241 : cluster 3 failed to encode map e277 with expected crc 2023-10-01T11:01:18.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:17 smithi078 bash[75299]: audit 2023-10-01T11:01:17.455638+0000 mon.a (mon.0) 1644 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:01:18.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:17 smithi078 bash[75299]: audit 2023-10-01T11:01:17.456554+0000 mon.a (mon.0) 1645 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:01:18.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:17 smithi078 bash[78274]: cluster 2023-10-01T11:01:16.189985+0000 mgr.y (mgr.25072) 1317 : cluster 0 pgmap v1357: 89 pgs: 89 active+clean; 582 KiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 77 KiB/s, 1 objects/s recovering 2023-10-01T11:01:18.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:17 smithi078 bash[78274]: audit 2023-10-01T11:01:16.820244+0000 mon.a (mon.0) 1642 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]': finished 2023-10-01T11:01:18.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:17 smithi078 bash[78274]: cluster 2023-10-01T11:01:16.820340+0000 mon.a (mon.0) 1643 : cluster 0 osdmap e277: 8 total, 8 up, 8 in 2023-10-01T11:01:18.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:17 smithi078 bash[78274]: cluster 2023-10-01T11:01:17.281160+0000 osd.7 (osd.7) 278 : cluster 3 failed to encode map e277 with expected crc 2023-10-01T11:01:18.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:17 smithi078 bash[78274]: cluster 2023-10-01T11:01:17.283805+0000 osd.7 (osd.7) 279 : cluster 3 failed to encode map e277 with expected crc 2023-10-01T11:01:18.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:17 smithi078 bash[78274]: cluster 2023-10-01T11:01:17.296500+0000 osd.6 (osd.6) 240 : cluster 3 failed to encode map e277 with expected crc 2023-10-01T11:01:18.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:17 smithi078 bash[78274]: cluster 2023-10-01T11:01:17.298485+0000 osd.6 (osd.6) 241 : cluster 3 failed to encode map e277 with expected crc 2023-10-01T11:01:18.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:17 smithi078 bash[78274]: audit 2023-10-01T11:01:17.455638+0000 mon.a (mon.0) 1644 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:01:18.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:17 smithi078 bash[78274]: audit 2023-10-01T11:01:17.456554+0000 mon.a (mon.0) 1645 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:01:18.106 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:18.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:18.109 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:18.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:17 smithi173 bash[58816]: cluster 2023-10-01T11:01:16.189985+0000 mgr.y (mgr.25072) 1317 : cluster 0 pgmap v1357: 89 pgs: 89 active+clean; 582 KiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 77 KiB/s, 1 objects/s recovering 2023-10-01T11:01:18.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:17 smithi173 bash[58816]: audit 2023-10-01T11:01:16.820244+0000 mon.a (mon.0) 1642 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]': finished 2023-10-01T11:01:18.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:17 smithi173 bash[58816]: cluster 2023-10-01T11:01:16.820340+0000 mon.a (mon.0) 1643 : cluster 0 osdmap e277: 8 total, 8 up, 8 in 2023-10-01T11:01:18.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:17 smithi173 bash[58816]: cluster 2023-10-01T11:01:17.281160+0000 osd.7 (osd.7) 278 : cluster 3 failed to encode map e277 with expected crc 2023-10-01T11:01:18.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:17 smithi173 bash[58816]: cluster 2023-10-01T11:01:17.283805+0000 osd.7 (osd.7) 279 : cluster 3 failed to encode map e277 with expected crc 2023-10-01T11:01:18.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:17 smithi173 bash[58816]: cluster 2023-10-01T11:01:17.296500+0000 osd.6 (osd.6) 240 : cluster 3 failed to encode map e277 with expected crc 2023-10-01T11:01:18.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:17 smithi173 bash[58816]: cluster 2023-10-01T11:01:17.298485+0000 osd.6 (osd.6) 241 : cluster 3 failed to encode map e277 with expected crc 2023-10-01T11:01:18.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:17 smithi173 bash[58816]: audit 2023-10-01T11:01:17.455638+0000 mon.a (mon.0) 1644 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:01:18.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:17 smithi173 bash[58816]: audit 2023-10-01T11:01:17.456554+0000 mon.a (mon.0) 1645 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:01:18.210 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:18.239 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:18.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:18.242 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:01:18.342 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:18.368 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:18.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:18.371 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:01:18.472 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:18.502 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:18.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:18.505 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:01:18.605 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:18.634 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:18.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:18.637 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:01:18.738 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:18.760 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:18.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:18.763 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:01:18.864 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:18.899 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:18.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:18.907 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:01:19.007 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:19.038 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:19.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:19.041 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:01:19.142 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:19.176 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:19.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:19.179 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:01:19.279 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:19.302 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:19.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:19.304 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:01:19.405 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:19.444 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:19.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:19.446 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:01:19.547 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:19.570 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:19.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:19.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:19.673 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:19.703 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:19.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:19.705 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:01:19.807 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:19.838 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:19.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:19.841 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:01:19.942 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:19.972 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:19.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:19.974 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:20.075 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:20.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:19 smithi078 bash[75299]: cluster 2023-10-01T11:01:18.190727+0000 mgr.y (mgr.25072) 1318 : cluster 0 pgmap v1359: 73 pgs: 73 active+clean; 582 KiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-01T11:01:20.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:19 smithi078 bash[78274]: cluster 2023-10-01T11:01:18.190727+0000 mgr.y (mgr.25072) 1318 : cluster 0 pgmap v1359: 73 pgs: 73 active+clean; 582 KiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-01T11:01:20.089 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:19 smithi173 bash[58816]: cluster 2023-10-01T11:01:18.190727+0000 mgr.y (mgr.25072) 1318 : cluster 0 pgmap v1359: 73 pgs: 73 active+clean; 582 KiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-01T11:01:20.111 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:20.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:20.113 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:01:20.214 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:20.235 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:20.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:20.237 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:01:20.338 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:20.364 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:20.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:20.366 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:01:20.467 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:20.497 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:20.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:20.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:01:20.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:01:20 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:01:20] "GET /metrics HTTP/1.1" 200 33974 "" "Prometheus/2.33.4" 2023-10-01T11:01:20.601 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:20.634 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:20.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:20.638 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:20.738 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:20.770 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:20.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:20.773 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:01:20.873 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:20.906 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:20.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:20.909 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:01:21.010 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:21.041 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:21.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:21.043 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:01:21.139 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-01T11:01:21.139 INFO:tasks.radosbench:Beginning radosbench... 2023-10-01T11:01:21.140 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_8 2023-10-01T11:01:21.140 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_8 16 2023-10-01T11:01:21.144 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:21.174 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:21.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:21.177 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:01:21.278 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:21.314 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:21.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:21.316 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:01:21.417 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:21.442 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:21.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:21.444 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:21.545 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:21.575 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:21.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:21.578 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:01:21.678 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:21.713 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:21.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:21.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:01:21.818 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:21.852 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:21.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:21.854 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:01:21.955 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:21.977 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:21.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:21.980 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:01:22.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:21 smithi078 bash[75299]: cluster 2023-10-01T11:01:20.191668+0000 mgr.y (mgr.25072) 1319 : cluster 0 pgmap v1360: 73 pgs: 73 active+clean; 582 KiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-01T11:01:22.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:21 smithi078 bash[78274]: cluster 2023-10-01T11:01:20.191668+0000 mgr.y (mgr.25072) 1319 : cluster 0 pgmap v1360: 73 pgs: 73 active+clean; 582 KiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-01T11:01:22.082 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:22.102 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:22.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:22.104 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:01:22.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:21 smithi173 bash[58816]: cluster 2023-10-01T11:01:20.191668+0000 mgr.y (mgr.25072) 1319 : cluster 0 pgmap v1360: 73 pgs: 73 active+clean; 582 KiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-01T11:01:22.205 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:22.233 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:22.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:22.236 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:01:22.337 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:22.368 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:22.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:22.371 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:01:22.472 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:22.502 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:22.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:22.506 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:01:22.606 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:22.635 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:01:22.641 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:22.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:22.645 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:01:22.745 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:22.776 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:22.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:22.780 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:01:22.881 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:22.912 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:22.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:22.915 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:01:23.016 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:23.042 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:23.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:23.044 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:01:23.145 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:23.168 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:23.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:23.171 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:01:23.272 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:23.299 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:23.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:23.302 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:23.405 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:23.436 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:23.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:23.440 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:01:23.540 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:23.569 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:23.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:23.572 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:01:23.673 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:23.699 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:23.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:23.703 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:01:23.803 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:23.830 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:23.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:23.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:01:23.934 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:23.967 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:23.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:23.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:01:24.071 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:24.080 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:23 smithi078 bash[78274]: cluster 2023-10-01T11:01:22.192213+0000 mgr.y (mgr.25072) 1320 : cluster 0 pgmap v1361: 73 pgs: 73 active+clean; 582 KiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-01T11:01:24.080 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:23 smithi078 bash[78274]: audit 2023-10-01T11:01:23.142672+0000 mon.a (mon.0) 1646 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:01:24.080 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:23 smithi078 bash[75299]: cluster 2023-10-01T11:01:22.192213+0000 mgr.y (mgr.25072) 1320 : cluster 0 pgmap v1361: 73 pgs: 73 active+clean; 582 KiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-01T11:01:24.080 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:23 smithi078 bash[75299]: audit 2023-10-01T11:01:23.142672+0000 mon.a (mon.0) 1646 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:01:24.100 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:24.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:24.103 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:01:24.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:23 smithi173 bash[58816]: cluster 2023-10-01T11:01:22.192213+0000 mgr.y (mgr.25072) 1320 : cluster 0 pgmap v1361: 73 pgs: 73 active+clean; 582 KiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-01T11:01:24.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:23 smithi173 bash[58816]: audit 2023-10-01T11:01:23.142672+0000 mon.a (mon.0) 1646 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:01:24.204 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:24.239 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:24.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:24.242 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:01:24.329 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2023-10-01T11:01:24.330 INFO:teuthology.orchestra.run.smithi078.stdout: "ops": [], 2023-10-01T11:01:24.330 INFO:teuthology.orchestra.run.smithi078.stdout: "num_ops": 0 2023-10-01T11:01:24.330 INFO:teuthology.orchestra.run.smithi078.stdout:} 2023-10-01T11:01:24.344 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:24.364 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:24.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:24.368 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:01:24.469 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:24.501 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:24.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:24.504 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:01:24.605 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:24.639 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:24.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:24.642 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:01:24.743 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:24.774 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:24.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:24.777 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:01:24.846 INFO:teuthology.orchestra.run.smithi078.stderr:noscrub is set 2023-10-01T11:01:24.872 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:24 smithi078 bash[75299]: audit 2023-10-01T11:01:24.598914+0000 mon.c (mon.2) 203 : audit 1 from='client.? 172.21.15.78:0/449477066' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T11:01:24.872 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:24 smithi078 bash[75299]: audit 2023-10-01T11:01:24.599549+0000 mon.a (mon.0) 1647 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T11:01:24.872 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:24 smithi078 bash[78274]: audit 2023-10-01T11:01:24.598914+0000 mon.c (mon.2) 203 : audit 1 from='client.? 172.21.15.78:0/449477066' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T11:01:24.872 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:24 smithi078 bash[78274]: audit 2023-10-01T11:01:24.599549+0000 mon.a (mon.0) 1647 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T11:01:24.878 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:24.910 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:24.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:24.913 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:01:25.014 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:25.046 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:25.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:25.050 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:01:25.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:24 smithi173 bash[58816]: audit 2023-10-01T11:01:24.598914+0000 mon.c (mon.2) 203 : audit 1 from='client.? 172.21.15.78:0/449477066' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T11:01:25.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:24 smithi173 bash[58816]: audit 2023-10-01T11:01:24.599549+0000 mon.a (mon.0) 1647 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T11:01:25.151 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:25.185 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:25.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:25.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:01:25.289 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:25.320 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:25.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:25.322 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:01:25.423 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:25.454 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:25.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:25.456 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:01:25.557 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:25.586 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:25.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:25.589 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:01:25.690 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:25.722 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:25.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:25.726 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:01:25.827 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:25.860 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:25.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:25.864 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:01:25.964 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:25.978 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:25 smithi173 bash[58816]: cluster 2023-10-01T11:01:24.192804+0000 mgr.y (mgr.25072) 1321 : cluster 0 pgmap v1362: 73 pgs: 73 active+clean; 582 KiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-01T11:01:25.978 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:25 smithi173 bash[58816]: cluster 2023-10-01T11:01:24.827548+0000 mon.a (mon.0) 1648 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:01:25.978 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:25 smithi173 bash[58816]: cluster 2023-10-01T11:01:24.837997+0000 osd.6 (osd.6) 242 : cluster 3 failed to encode map e278 with expected crc 2023-10-01T11:01:25.978 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:25 smithi173 bash[58816]: audit 2023-10-01T11:01:24.839772+0000 mon.a (mon.0) 1649 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T11:01:25.979 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:25 smithi173 bash[58816]: cluster 2023-10-01T11:01:24.839873+0000 mon.a (mon.0) 1650 : cluster 0 osdmap e278: 8 total, 8 up, 8 in 2023-10-01T11:01:25.979 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:25 smithi173 bash[58816]: cluster 2023-10-01T11:01:25.253094+0000 osd.7 (osd.7) 280 : cluster 3 failed to encode map e278 with expected crc 2023-10-01T11:01:25.999 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:26.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:26.001 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:01:26.102 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:26.138 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:26.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:26.141 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:01:26.186 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:25 smithi078 bash[75299]: cluster 2023-10-01T11:01:24.192804+0000 mgr.y (mgr.25072) 1321 : cluster 0 pgmap v1362: 73 pgs: 73 active+clean; 582 KiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-01T11:01:26.186 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:25 smithi078 bash[75299]: cluster 2023-10-01T11:01:24.827548+0000 mon.a (mon.0) 1648 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:01:26.186 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:25 smithi078 bash[75299]: cluster 2023-10-01T11:01:24.837997+0000 osd.6 (osd.6) 242 : cluster 3 failed to encode map e278 with expected crc 2023-10-01T11:01:26.186 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:25 smithi078 bash[75299]: audit 2023-10-01T11:01:24.839772+0000 mon.a (mon.0) 1649 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T11:01:26.186 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:25 smithi078 bash[75299]: cluster 2023-10-01T11:01:24.839873+0000 mon.a (mon.0) 1650 : cluster 0 osdmap e278: 8 total, 8 up, 8 in 2023-10-01T11:01:26.186 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:25 smithi078 bash[75299]: cluster 2023-10-01T11:01:25.253094+0000 osd.7 (osd.7) 280 : cluster 3 failed to encode map e278 with expected crc 2023-10-01T11:01:26.242 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:26.263 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:26.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:26.265 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:26.366 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:26.397 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:26.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:26.400 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:01:26.501 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:26.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:26 smithi078 bash[78274]: cluster 2023-10-01T11:01:24.192804+0000 mgr.y (mgr.25072) 1321 : cluster 0 pgmap v1362: 73 pgs: 73 active+clean; 582 KiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-01T11:01:26.511 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:26 smithi078 bash[78274]: cluster 2023-10-01T11:01:24.827548+0000 mon.a (mon.0) 1648 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:01:26.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:26 smithi078 bash[78274]: cluster 2023-10-01T11:01:24.837997+0000 osd.6 (osd.6) 242 : cluster 3 failed to encode map e278 with expected crc 2023-10-01T11:01:26.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:26 smithi078 bash[78274]: audit 2023-10-01T11:01:24.839772+0000 mon.a (mon.0) 1649 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T11:01:26.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:26 smithi078 bash[78274]: cluster 2023-10-01T11:01:24.839873+0000 mon.a (mon.0) 1650 : cluster 0 osdmap e278: 8 total, 8 up, 8 in 2023-10-01T11:01:26.512 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:26 smithi078 bash[78274]: cluster 2023-10-01T11:01:25.253094+0000 osd.7 (osd.7) 280 : cluster 3 failed to encode map e278 with expected crc 2023-10-01T11:01:26.527 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:26.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:26.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:01:26.584 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-01T11:01:26.632 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:26.655 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:26.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:26.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:26.758 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:26.781 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:26.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:26.783 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:01:26.884 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:26.916 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:26.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:26.918 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:01:27.019 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:27.051 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:27.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:27.055 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:01:27.069 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:01:27.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:26 smithi078 bash[75299]: cluster 2023-10-01T11:01:25.252974+0000 osd.5 (osd.5) 263 : cluster 3 failed to encode map e278 with expected crc 2023-10-01T11:01:27.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:26 smithi078 bash[78274]: cluster 2023-10-01T11:01:25.252974+0000 osd.5 (osd.5) 263 : cluster 3 failed to encode map e278 with expected crc 2023-10-01T11:01:27.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:26 smithi173 bash[58816]: cluster 2023-10-01T11:01:25.252974+0000 osd.5 (osd.5) 263 : cluster 3 failed to encode map e278 with expected crc 2023-10-01T11:01:27.156 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:27.186 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:27.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:27.188 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:01:27.289 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:27.322 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:27.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:27.324 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:01:27.425 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:27.454 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:27.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:27.456 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:01:27.557 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:27.582 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:27.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:27.585 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:01:27.686 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:27.716 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:27.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:27.719 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:01:27.819 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:27.849 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:27.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:27.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:01:27.953 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:27.979 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:27.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:27.983 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:01:28.083 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:28.115 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:28.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:28.118 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:01:28.218 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:28.231 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:27 smithi078 bash[75299]: cluster 2023-10-01T11:01:26.193435+0000 mgr.y (mgr.25072) 1322 : cluster 0 pgmap v1364: 73 pgs: 73 active+clean; 582 KiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-01T11:01:28.231 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:27 smithi078 bash[78274]: cluster 2023-10-01T11:01:26.193435+0000 mgr.y (mgr.25072) 1322 : cluster 0 pgmap v1364: 73 pgs: 73 active+clean; 582 KiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-01T11:01:28.247 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:28.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:28.249 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:28.350 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:28.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:27 smithi173 bash[58816]: cluster 2023-10-01T11:01:26.193435+0000 mgr.y (mgr.25072) 1322 : cluster 0 pgmap v1364: 73 pgs: 73 active+clean; 582 KiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-01T11:01:28.375 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:28.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:28.379 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:28.480 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:28.513 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:28.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:28.516 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:01:28.589 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T11:01:28.617 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:28.650 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:28.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:28.654 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:01:28.754 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:28.786 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:28.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:28.788 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:01:28.890 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:28.923 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:28.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:28.926 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:01:29.027 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:29.061 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:29.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:29.065 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:01:29.165 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:29.197 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:29.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:29.200 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:01:29.301 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:29.326 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:29.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:29.330 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:01:29.431 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:29.464 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:29.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:29.467 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:01:29.568 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:29.597 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:29.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:29.600 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:01:29.700 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:29.733 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:29.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:29.737 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:01:29.838 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:29.864 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:29.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:29.867 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:01:29.891 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T11:01:29.919 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T11:01:29.948 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T11:01:29.968 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:29.971 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T11:01:29.994 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:29.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:29.996 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:01:29.998 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T11:01:30.025 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T11:01:30.054 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T11:01:30.083 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T11:01:30.085 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T11:01:30.086 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T11:01:30.086 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T11:01:30.086 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T11:01:30.086 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T11:01:30.086 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T11:01:30.086 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T11:01:30.086 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T11:01:30.096 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:30.103 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:29 smithi078 bash[78274]: cluster 2023-10-01T11:01:28.194067+0000 mgr.y (mgr.25072) 1323 : cluster 0 pgmap v1365: 73 pgs: 73 active+clean; 582 KiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-01T11:01:30.104 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:29 smithi078 bash[75299]: cluster 2023-10-01T11:01:28.194067+0000 mgr.y (mgr.25072) 1323 : cluster 0 pgmap v1365: 73 pgs: 73 active+clean; 582 KiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-01T11:01:30.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:29 smithi173 bash[58816]: cluster 2023-10-01T11:01:28.194067+0000 mgr.y (mgr.25072) 1323 : cluster 0 pgmap v1365: 73 pgs: 73 active+clean; 582 KiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-01T11:01:30.127 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:30.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:30.129 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:30.230 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:30.257 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:30.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:30.259 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:01:30.360 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:30.384 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:30.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:30.387 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:01:30.463 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:30 smithi078 bash[75299]: debug 2023-10-01T11:01:30.186+0000 7f31c8dd0700 -1 mon.a@0(leader).osd e278 definitely_dead 0 2023-10-01T11:01:30.463 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:01:30 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:01:30] "GET /metrics HTTP/1.1" 200 33974 "" "Prometheus/2.33.4" 2023-10-01T11:01:30.488 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:30.521 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:30.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:30.524 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:01:30.624 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:30.655 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:30.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:30.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:01:30.760 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:30.791 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:30.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:30.793 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:01:30.875 INFO:teuthology.orchestra.run.smithi078.stderr:marked out osd.7. 2023-10-01T11:01:30.894 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:30.918 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:30.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:30.921 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:31.022 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:31.054 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:31.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:31.058 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:01:31.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:30 smithi078 bash[75299]: audit 2023-10-01T11:01:29.866814+0000 mon.c (mon.2) 204 : audit 0 from='client.? 172.21.15.78:0/867841935' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:01:31.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:30 smithi078 bash[75299]: audit 2023-10-01T11:01:30.190627+0000 mon.a (mon.0) 1651 : audit 1 from='client.? 172.21.15.78:0/3946118720' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-01T11:01:31.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:30 smithi078 bash[75299]: cluster 2023-10-01T11:01:30.190788+0000 mon.a (mon.0) 1652 : cluster 1 Client client.admin marked osd.7 out, while it was still marked up 2023-10-01T11:01:31.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:30 smithi078 bash[78274]: audit 2023-10-01T11:01:29.866814+0000 mon.c (mon.2) 204 : audit 0 from='client.? 172.21.15.78:0/867841935' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:01:31.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:30 smithi078 bash[78274]: audit 2023-10-01T11:01:30.190627+0000 mon.a (mon.0) 1651 : audit 1 from='client.? 172.21.15.78:0/3946118720' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-01T11:01:31.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:30 smithi078 bash[78274]: cluster 2023-10-01T11:01:30.190788+0000 mon.a (mon.0) 1652 : cluster 1 Client client.admin marked osd.7 out, while it was still marked up 2023-10-01T11:01:31.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:30 smithi173 bash[58816]: audit 2023-10-01T11:01:29.866814+0000 mon.c (mon.2) 204 : audit 0 from='client.? 172.21.15.78:0/867841935' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:01:31.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:30 smithi173 bash[58816]: audit 2023-10-01T11:01:30.190627+0000 mon.a (mon.0) 1651 : audit 1 from='client.? 172.21.15.78:0/3946118720' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-01T11:01:31.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:30 smithi173 bash[58816]: cluster 2023-10-01T11:01:30.190788+0000 mon.a (mon.0) 1652 : cluster 1 Client client.admin marked osd.7 out, while it was still marked up 2023-10-01T11:01:31.159 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:31.186 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:31.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:31.190 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:31.291 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:31.321 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:31.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:31.325 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:01:31.426 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:31.461 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:31.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:31.464 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:01:31.565 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:31.597 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:31.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:31.601 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:01:31.702 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:31.731 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:31.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:31.734 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:01:31.834 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:31.859 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:31.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:31.863 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:01:31.963 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:31.990 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:31.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:31.994 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:01:32.095 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:32.103 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:31 smithi078 bash[78274]: cluster 2023-10-01T11:01:30.194570+0000 mgr.y (mgr.25072) 1324 : cluster 0 pgmap v1366: 73 pgs: 73 active+clean; 582 KiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-01T11:01:32.103 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:31 smithi078 bash[78274]: cluster 2023-10-01T11:01:30.857550+0000 osd.6 (osd.6) 243 : cluster 3 failed to encode map e279 with expected crc 2023-10-01T11:01:32.103 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:31 smithi078 bash[78274]: cluster 2023-10-01T11:01:30.857865+0000 osd.7 (osd.7) 281 : cluster 3 failed to encode map e279 with expected crc 2023-10-01T11:01:32.103 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:31 smithi078 bash[78274]: audit 2023-10-01T11:01:30.860720+0000 mon.a (mon.0) 1653 : audit 1 from='client.? 172.21.15.78:0/3946118720' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-10-01T11:01:32.103 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:31 smithi078 bash[78274]: cluster 2023-10-01T11:01:30.860806+0000 mon.a (mon.0) 1654 : cluster 0 osdmap e279: 8 total, 8 up, 7 in 2023-10-01T11:01:32.103 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:31 smithi078 bash[78274]: cluster 2023-10-01T11:01:30.861588+0000 osd.6 (osd.6) 244 : cluster 3 failed to encode map e279 with expected crc 2023-10-01T11:01:32.103 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:31 smithi078 bash[78274]: cluster 2023-10-01T11:01:30.867516+0000 osd.5 (osd.5) 264 : cluster 3 failed to encode map e279 with expected crc 2023-10-01T11:01:32.103 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:31 smithi078 bash[78274]: cluster 2023-10-01T11:01:30.870895+0000 osd.5 (osd.5) 265 : cluster 3 failed to encode map e279 with expected crc 2023-10-01T11:01:32.104 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:31 smithi078 bash[75299]: cluster 2023-10-01T11:01:30.194570+0000 mgr.y (mgr.25072) 1324 : cluster 0 pgmap v1366: 73 pgs: 73 active+clean; 582 KiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-01T11:01:32.104 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:31 smithi078 bash[75299]: cluster 2023-10-01T11:01:30.857550+0000 osd.6 (osd.6) 243 : cluster 3 failed to encode map e279 with expected crc 2023-10-01T11:01:32.104 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:31 smithi078 bash[75299]: cluster 2023-10-01T11:01:30.857865+0000 osd.7 (osd.7) 281 : cluster 3 failed to encode map e279 with expected crc 2023-10-01T11:01:32.104 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:31 smithi078 bash[75299]: audit 2023-10-01T11:01:30.860720+0000 mon.a (mon.0) 1653 : audit 1 from='client.? 172.21.15.78:0/3946118720' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-10-01T11:01:32.104 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:31 smithi078 bash[75299]: cluster 2023-10-01T11:01:30.860806+0000 mon.a (mon.0) 1654 : cluster 0 osdmap e279: 8 total, 8 up, 7 in 2023-10-01T11:01:32.104 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:31 smithi078 bash[75299]: cluster 2023-10-01T11:01:30.861588+0000 osd.6 (osd.6) 244 : cluster 3 failed to encode map e279 with expected crc 2023-10-01T11:01:32.104 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:31 smithi078 bash[75299]: cluster 2023-10-01T11:01:30.867516+0000 osd.5 (osd.5) 264 : cluster 3 failed to encode map e279 with expected crc 2023-10-01T11:01:32.104 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:31 smithi078 bash[75299]: cluster 2023-10-01T11:01:30.870895+0000 osd.5 (osd.5) 265 : cluster 3 failed to encode map e279 with expected crc 2023-10-01T11:01:32.119 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:32.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:31 smithi173 bash[58816]: cluster 2023-10-01T11:01:30.194570+0000 mgr.y (mgr.25072) 1324 : cluster 0 pgmap v1366: 73 pgs: 73 active+clean; 582 KiB data, 5.7 GiB used, 710 GiB / 715 GiB avail 2023-10-01T11:01:32.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:31 smithi173 bash[58816]: cluster 2023-10-01T11:01:30.857550+0000 osd.6 (osd.6) 243 : cluster 3 failed to encode map e279 with expected crc 2023-10-01T11:01:32.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:31 smithi173 bash[58816]: cluster 2023-10-01T11:01:30.857865+0000 osd.7 (osd.7) 281 : cluster 3 failed to encode map e279 with expected crc 2023-10-01T11:01:32.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:31 smithi173 bash[58816]: audit 2023-10-01T11:01:30.860720+0000 mon.a (mon.0) 1653 : audit 1 from='client.? 172.21.15.78:0/3946118720' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-10-01T11:01:32.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:31 smithi173 bash[58816]: cluster 2023-10-01T11:01:30.860806+0000 mon.a (mon.0) 1654 : cluster 0 osdmap e279: 8 total, 8 up, 7 in 2023-10-01T11:01:32.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:31 smithi173 bash[58816]: cluster 2023-10-01T11:01:30.861588+0000 osd.6 (osd.6) 244 : cluster 3 failed to encode map e279 with expected crc 2023-10-01T11:01:32.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:31 smithi173 bash[58816]: cluster 2023-10-01T11:01:30.867516+0000 osd.5 (osd.5) 264 : cluster 3 failed to encode map e279 with expected crc 2023-10-01T11:01:32.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:31 smithi173 bash[58816]: cluster 2023-10-01T11:01:30.870895+0000 osd.5 (osd.5) 265 : cluster 3 failed to encode map e279 with expected crc 2023-10-01T11:01:32.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:32.121 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:32.222 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:32.248 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:32.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:32.251 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:01:32.351 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:32.372 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:32.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:32.375 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:01:32.476 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:32.505 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:32.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:32.508 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:01:32.609 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:32.629 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:32.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:32.631 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:32.732 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:32.760 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:32.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:32.763 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:01:32.864 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:32.887 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:32.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:32.889 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:32.990 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:33.005 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:32 smithi173 bash[58816]: cluster 2023-10-01T11:01:31.865568+0000 osd.5 (osd.5) 266 : cluster 3 failed to encode map e280 with expected crc 2023-10-01T11:01:33.005 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:32 smithi173 bash[58816]: cluster 2023-10-01T11:01:31.866945+0000 mon.a (mon.0) 1655 : cluster 0 osdmap e280: 8 total, 8 up, 7 in 2023-10-01T11:01:33.005 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:32 smithi173 bash[58816]: cluster 2023-10-01T11:01:31.868665+0000 osd.6 (osd.6) 245 : cluster 3 failed to encode map e280 with expected crc 2023-10-01T11:01:33.005 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:32 smithi173 bash[58816]: cluster 2023-10-01T11:01:31.868781+0000 osd.5 (osd.5) 267 : cluster 3 failed to encode map e280 with expected crc 2023-10-01T11:01:33.005 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:32 smithi173 bash[58816]: cluster 2023-10-01T11:01:31.869777+0000 osd.7 (osd.7) 282 : cluster 3 failed to encode map e280 with expected crc 2023-10-01T11:01:33.005 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:32 smithi173 bash[58816]: cluster 2023-10-01T11:01:31.871520+0000 osd.6 (osd.6) 246 : cluster 3 failed to encode map e280 with expected crc 2023-10-01T11:01:33.005 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:32 smithi173 bash[58816]: cluster 2023-10-01T11:01:31.873297+0000 osd.7 (osd.7) 283 : cluster 3 failed to encode map e280 with expected crc 2023-10-01T11:01:33.005 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:32 smithi173 bash[58816]: audit 2023-10-01T11:01:32.458990+0000 mon.a (mon.0) 1656 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:01:33.006 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:32 smithi173 bash[58816]: audit 2023-10-01T11:01:32.460201+0000 mon.a (mon.0) 1657 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:01:33.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:33.026 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:33.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:01:33.123 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:32 smithi078 bash[75299]: cluster 2023-10-01T11:01:31.865568+0000 osd.5 (osd.5) 266 : cluster 3 failed to encode map e280 with expected crc 2023-10-01T11:01:33.123 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:32 smithi078 bash[75299]: cluster 2023-10-01T11:01:31.866945+0000 mon.a (mon.0) 1655 : cluster 0 osdmap e280: 8 total, 8 up, 7 in 2023-10-01T11:01:33.123 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:32 smithi078 bash[75299]: cluster 2023-10-01T11:01:31.868665+0000 osd.6 (osd.6) 245 : cluster 3 failed to encode map e280 with expected crc 2023-10-01T11:01:33.123 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:32 smithi078 bash[75299]: cluster 2023-10-01T11:01:31.868781+0000 osd.5 (osd.5) 267 : cluster 3 failed to encode map e280 with expected crc 2023-10-01T11:01:33.123 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:32 smithi078 bash[75299]: cluster 2023-10-01T11:01:31.869777+0000 osd.7 (osd.7) 282 : cluster 3 failed to encode map e280 with expected crc 2023-10-01T11:01:33.123 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:32 smithi078 bash[75299]: cluster 2023-10-01T11:01:31.871520+0000 osd.6 (osd.6) 246 : cluster 3 failed to encode map e280 with expected crc 2023-10-01T11:01:33.123 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:32 smithi078 bash[75299]: cluster 2023-10-01T11:01:31.873297+0000 osd.7 (osd.7) 283 : cluster 3 failed to encode map e280 with expected crc 2023-10-01T11:01:33.124 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:32 smithi078 bash[75299]: audit 2023-10-01T11:01:32.458990+0000 mon.a (mon.0) 1656 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:01:33.124 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:32 smithi078 bash[75299]: audit 2023-10-01T11:01:32.460201+0000 mon.a (mon.0) 1657 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:01:33.124 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:32 smithi078 bash[78274]: cluster 2023-10-01T11:01:31.865568+0000 osd.5 (osd.5) 266 : cluster 3 failed to encode map e280 with expected crc 2023-10-01T11:01:33.124 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:32 smithi078 bash[78274]: cluster 2023-10-01T11:01:31.866945+0000 mon.a (mon.0) 1655 : cluster 0 osdmap e280: 8 total, 8 up, 7 in 2023-10-01T11:01:33.124 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:32 smithi078 bash[78274]: cluster 2023-10-01T11:01:31.868665+0000 osd.6 (osd.6) 245 : cluster 3 failed to encode map e280 with expected crc 2023-10-01T11:01:33.124 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:32 smithi078 bash[78274]: cluster 2023-10-01T11:01:31.868781+0000 osd.5 (osd.5) 267 : cluster 3 failed to encode map e280 with expected crc 2023-10-01T11:01:33.124 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:32 smithi078 bash[78274]: cluster 2023-10-01T11:01:31.869777+0000 osd.7 (osd.7) 282 : cluster 3 failed to encode map e280 with expected crc 2023-10-01T11:01:33.124 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:32 smithi078 bash[78274]: cluster 2023-10-01T11:01:31.871520+0000 osd.6 (osd.6) 246 : cluster 3 failed to encode map e280 with expected crc 2023-10-01T11:01:33.125 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:32 smithi078 bash[78274]: cluster 2023-10-01T11:01:31.873297+0000 osd.7 (osd.7) 283 : cluster 3 failed to encode map e280 with expected crc 2023-10-01T11:01:33.125 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:32 smithi078 bash[78274]: audit 2023-10-01T11:01:32.458990+0000 mon.a (mon.0) 1656 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:01:33.125 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:32 smithi078 bash[78274]: audit 2023-10-01T11:01:32.460201+0000 mon.a (mon.0) 1657 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:01:33.127 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:33.152 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:33.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:33.154 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:01:33.255 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:33.289 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:33.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:33.291 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:01:33.392 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:33.417 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:33.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:33.420 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:33.521 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:33.548 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:33.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:33.550 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:01:33.651 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:33.672 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:33.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:33.675 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:33.775 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:33.803 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:33.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:33.806 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:01:33.907 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:33.939 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:33.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:33.942 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:01:33.999 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:33 smithi078 bash[78274]: cluster 2023-10-01T11:01:32.195242+0000 mgr.y (mgr.25072) 1325 : cluster 0 pgmap v1369: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-10-01T11:01:33.999 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:33 smithi078 bash[78274]: cluster 2023-10-01T11:01:32.874453+0000 osd.7 (osd.7) 284 : cluster 3 failed to encode map e281 with expected crc 2023-10-01T11:01:33.999 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:33 smithi078 bash[78274]: cluster 2023-10-01T11:01:32.876724+0000 mon.a (mon.0) 1658 : cluster 0 osdmap e281: 8 total, 8 up, 7 in 2023-10-01T11:01:33.999 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:33 smithi078 bash[78274]: cluster 2023-10-01T11:01:32.907002+0000 osd.6 (osd.6) 247 : cluster 3 failed to encode map e281 with expected crc 2023-10-01T11:01:34.000 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:33 smithi078 bash[75299]: cluster 2023-10-01T11:01:32.195242+0000 mgr.y (mgr.25072) 1325 : cluster 0 pgmap v1369: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-10-01T11:01:34.000 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:33 smithi078 bash[75299]: cluster 2023-10-01T11:01:32.874453+0000 osd.7 (osd.7) 284 : cluster 3 failed to encode map e281 with expected crc 2023-10-01T11:01:34.000 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:33 smithi078 bash[75299]: cluster 2023-10-01T11:01:32.876724+0000 mon.a (mon.0) 1658 : cluster 0 osdmap e281: 8 total, 8 up, 7 in 2023-10-01T11:01:34.000 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:33 smithi078 bash[75299]: cluster 2023-10-01T11:01:32.907002+0000 osd.6 (osd.6) 247 : cluster 3 failed to encode map e281 with expected crc 2023-10-01T11:01:34.042 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:34.068 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:34.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:34.071 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:34.173 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:34.186 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:33 smithi173 bash[58816]: cluster 2023-10-01T11:01:32.195242+0000 mgr.y (mgr.25072) 1325 : cluster 0 pgmap v1369: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-10-01T11:01:34.186 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:33 smithi173 bash[58816]: cluster 2023-10-01T11:01:32.874453+0000 osd.7 (osd.7) 284 : cluster 3 failed to encode map e281 with expected crc 2023-10-01T11:01:34.186 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:33 smithi173 bash[58816]: cluster 2023-10-01T11:01:32.876724+0000 mon.a (mon.0) 1658 : cluster 0 osdmap e281: 8 total, 8 up, 7 in 2023-10-01T11:01:34.186 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:33 smithi173 bash[58816]: cluster 2023-10-01T11:01:32.907002+0000 osd.6 (osd.6) 247 : cluster 3 failed to encode map e281 with expected crc 2023-10-01T11:01:34.207 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:34.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:34.211 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:01:34.312 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:34.343 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:34.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:34.346 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:01:34.447 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:34.475 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:34.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:34.477 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:01:34.478 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:01:34.579 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:34.612 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:34.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:34.616 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:01:34.716 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:34.742 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:34.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:34.745 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:34.846 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:34.875 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:34.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:34.878 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:34.979 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:35.013 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:35.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:35.016 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:01:35.116 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:35.126 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T11:01:35.144 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:35.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:35.147 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:01:35.191 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:34 smithi078 bash[78274]: cluster 2023-10-01T11:01:33.882443+0000 osd.6 (osd.6) 248 : cluster 3 failed to encode map e282 with expected crc 2023-10-01T11:01:35.191 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:34 smithi078 bash[78274]: cluster 2023-10-01T11:01:33.883654+0000 mon.a (mon.0) 1659 : cluster 0 osdmap e282: 8 total, 8 up, 7 in 2023-10-01T11:01:35.191 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:34 smithi078 bash[78274]: cluster 2023-10-01T11:01:33.885181+0000 osd.7 (osd.7) 285 : cluster 3 failed to encode map e282 with expected crc 2023-10-01T11:01:35.192 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:34 smithi078 bash[75299]: cluster 2023-10-01T11:01:33.882443+0000 osd.6 (osd.6) 248 : cluster 3 failed to encode map e282 with expected crc 2023-10-01T11:01:35.192 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:34 smithi078 bash[75299]: cluster 2023-10-01T11:01:33.883654+0000 mon.a (mon.0) 1659 : cluster 0 osdmap e282: 8 total, 8 up, 7 in 2023-10-01T11:01:35.192 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:34 smithi078 bash[75299]: cluster 2023-10-01T11:01:33.885181+0000 osd.7 (osd.7) 285 : cluster 3 failed to encode map e282 with expected crc 2023-10-01T11:01:35.249 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:35.262 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:34 smithi173 bash[58816]: cluster 2023-10-01T11:01:33.882443+0000 osd.6 (osd.6) 248 : cluster 3 failed to encode map e282 with expected crc 2023-10-01T11:01:35.262 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:34 smithi173 bash[58816]: cluster 2023-10-01T11:01:33.883654+0000 mon.a (mon.0) 1659 : cluster 0 osdmap e282: 8 total, 8 up, 7 in 2023-10-01T11:01:35.262 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:34 smithi173 bash[58816]: cluster 2023-10-01T11:01:33.885181+0000 osd.7 (osd.7) 285 : cluster 3 failed to encode map e282 with expected crc 2023-10-01T11:01:35.280 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:35.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:35.285 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:01:35.386 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:35.411 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:35.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:35.414 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:01:35.515 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:35.550 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:35.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:35.553 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:01:35.654 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:35.680 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:35.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:35.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:35.783 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:35.814 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:35.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:35.817 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:01:35.918 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:35.952 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:35.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:35.956 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:01:36.021 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:35 smithi078 bash[78274]: cluster 2023-10-01T11:01:34.195874+0000 mgr.y (mgr.25072) 1326 : cluster 0 pgmap v1372: 73 pgs: 1 active+undersized+remapped, 72 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 144 KiB/s, 1 objects/s recovering 2023-10-01T11:01:36.021 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:35 smithi078 bash[78274]: cluster 2023-10-01T11:01:34.533548+0000 osd.5 (osd.5) 268 : cluster 3 failed to encode map e281 with expected crc 2023-10-01T11:01:36.022 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:35 smithi078 bash[78274]: cluster 2023-10-01T11:01:34.538919+0000 osd.5 (osd.5) 269 : cluster 3 failed to encode map e282 with expected crc 2023-10-01T11:01:36.022 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:35 smithi078 bash[78274]: cluster 2023-10-01T11:01:34.894544+0000 mon.a (mon.0) 1660 : cluster 0 osdmap e283: 8 total, 8 up, 7 in 2023-10-01T11:01:36.022 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:35 smithi078 bash[78274]: cluster 2023-10-01T11:01:34.898377+0000 osd.6 (osd.6) 249 : cluster 3 failed to encode map e283 with expected crc 2023-10-01T11:01:36.022 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:35 smithi078 bash[78274]: cluster 2023-10-01T11:01:34.984299+0000 osd.7 (osd.7) 286 : cluster 3 failed to encode map e283 with expected crc 2023-10-01T11:01:36.022 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:35 smithi078 bash[75299]: cluster 2023-10-01T11:01:34.195874+0000 mgr.y (mgr.25072) 1326 : cluster 0 pgmap v1372: 73 pgs: 1 active+undersized+remapped, 72 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 144 KiB/s, 1 objects/s recovering 2023-10-01T11:01:36.022 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:35 smithi078 bash[75299]: cluster 2023-10-01T11:01:34.533548+0000 osd.5 (osd.5) 268 : cluster 3 failed to encode map e281 with expected crc 2023-10-01T11:01:36.022 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:35 smithi078 bash[75299]: cluster 2023-10-01T11:01:34.538919+0000 osd.5 (osd.5) 269 : cluster 3 failed to encode map e282 with expected crc 2023-10-01T11:01:36.022 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:35 smithi078 bash[75299]: cluster 2023-10-01T11:01:34.894544+0000 mon.a (mon.0) 1660 : cluster 0 osdmap e283: 8 total, 8 up, 7 in 2023-10-01T11:01:36.023 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:35 smithi078 bash[75299]: cluster 2023-10-01T11:01:34.898377+0000 osd.6 (osd.6) 249 : cluster 3 failed to encode map e283 with expected crc 2023-10-01T11:01:36.023 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:35 smithi078 bash[75299]: cluster 2023-10-01T11:01:34.984299+0000 osd.7 (osd.7) 286 : cluster 3 failed to encode map e283 with expected crc 2023-10-01T11:01:36.056 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:36.069 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:35 smithi173 bash[58816]: cluster 2023-10-01T11:01:34.195874+0000 mgr.y (mgr.25072) 1326 : cluster 0 pgmap v1372: 73 pgs: 1 active+undersized+remapped, 72 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 144 KiB/s, 1 objects/s recovering 2023-10-01T11:01:36.070 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:35 smithi173 bash[58816]: cluster 2023-10-01T11:01:34.533548+0000 osd.5 (osd.5) 268 : cluster 3 failed to encode map e281 with expected crc 2023-10-01T11:01:36.070 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:35 smithi173 bash[58816]: cluster 2023-10-01T11:01:34.538919+0000 osd.5 (osd.5) 269 : cluster 3 failed to encode map e282 with expected crc 2023-10-01T11:01:36.070 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:35 smithi173 bash[58816]: cluster 2023-10-01T11:01:34.894544+0000 mon.a (mon.0) 1660 : cluster 0 osdmap e283: 8 total, 8 up, 7 in 2023-10-01T11:01:36.070 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:35 smithi173 bash[58816]: cluster 2023-10-01T11:01:34.898377+0000 osd.6 (osd.6) 249 : cluster 3 failed to encode map e283 with expected crc 2023-10-01T11:01:36.070 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:35 smithi173 bash[58816]: cluster 2023-10-01T11:01:34.984299+0000 osd.7 (osd.7) 286 : cluster 3 failed to encode map e283 with expected crc 2023-10-01T11:01:36.089 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:36.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:36.091 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:01:36.192 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:36.221 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:36.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:36.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:01:36.324 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:36.354 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:36.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:36.357 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:01:36.457 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:36.479 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:36.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:36.481 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:01:36.506 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:01:36.583 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:36.603 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:36.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:36.606 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:01:36.707 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:36.741 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:36.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:36.744 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:01:36.845 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:36.877 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:36.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:36.879 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:01:36.980 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:37.000 INFO:teuthology.orchestra.run.smithi078.stderr:pool 'unique_pool_8' created 2023-10-01T11:01:37.003 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:37.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:37.007 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:01:37.107 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:37.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:36 smithi173 bash[58816]: cluster 2023-10-01T11:01:34.984324+0000 osd.5 (osd.5) 270 : cluster 3 failed to encode map e283 with expected crc 2023-10-01T11:01:37.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:36 smithi173 bash[58816]: audit 2023-10-01T11:01:36.480292+0000 mon.a (mon.0) 1661 : audit 1 from='client.? 172.21.15.78:0/3178037826' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]: dispatch 2023-10-01T11:01:37.132 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:37.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:37.134 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:01:37.234 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:37.266 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:37.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:37.268 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:01:37.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:36 smithi078 bash[75299]: cluster 2023-10-01T11:01:34.984324+0000 osd.5 (osd.5) 270 : cluster 3 failed to encode map e283 with expected crc 2023-10-01T11:01:37.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:36 smithi078 bash[75299]: audit 2023-10-01T11:01:36.480292+0000 mon.a (mon.0) 1661 : audit 1 from='client.? 172.21.15.78:0/3178037826' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]: dispatch 2023-10-01T11:01:37.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:36 smithi078 bash[78274]: cluster 2023-10-01T11:01:34.984324+0000 osd.5 (osd.5) 270 : cluster 3 failed to encode map e283 with expected crc 2023-10-01T11:01:37.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:36 smithi078 bash[78274]: audit 2023-10-01T11:01:36.480292+0000 mon.a (mon.0) 1661 : audit 1 from='client.? 172.21.15.78:0/3178037826' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]: dispatch 2023-10-01T11:01:37.369 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:37.393 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:37.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:37.395 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:37.495 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:37.525 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:37.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:37.529 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:01:37.630 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:37.657 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:37.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:37.661 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:01:37.762 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:37.793 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:37.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:37.796 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:01:37.897 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:37.928 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:37.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:37.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:01:38.032 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:38.062 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:38.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:38.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:01:38.166 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:38.178 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:37 smithi078 bash[78274]: cluster 2023-10-01T11:01:36.196375+0000 mgr.y (mgr.25072) 1327 : cluster 0 pgmap v1374: 73 pgs: 1 active+undersized+remapped, 72 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 134 KiB/s, 2 keys/s, 3 objects/s recovering 2023-10-01T11:01:38.179 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:37 smithi078 bash[78274]: audit 2023-10-01T11:01:36.932378+0000 mon.a (mon.0) 1662 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T11:01:38.179 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:37 smithi078 bash[78274]: cluster 2023-10-01T11:01:36.990251+0000 osd.7 (osd.7) 287 : cluster 3 failed to encode map e284 with expected crc 2023-10-01T11:01:38.179 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:37 smithi078 bash[78274]: audit 2023-10-01T11:01:36.995753+0000 mon.a (mon.0) 1663 : audit 1 from='client.? 172.21.15.78:0/3178037826' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]': finished 2023-10-01T11:01:38.179 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:37 smithi078 bash[78274]: cluster 2023-10-01T11:01:36.995851+0000 mon.a (mon.0) 1664 : cluster 0 osdmap e284: 8 total, 8 up, 7 in 2023-10-01T11:01:38.179 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:37 smithi078 bash[78274]: cluster 2023-10-01T11:01:37.285141+0000 osd.5 (osd.5) 271 : cluster 3 failed to encode map e284 with expected crc 2023-10-01T11:01:38.179 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:37 smithi078 bash[78274]: cluster 2023-10-01T11:01:37.285190+0000 osd.6 (osd.6) 250 : cluster 3 failed to encode map e284 with expected crc 2023-10-01T11:01:38.180 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:37 smithi078 bash[78274]: cluster 2023-10-01T11:01:37.288508+0000 osd.5 (osd.5) 272 : cluster 3 failed to encode map e284 with expected crc 2023-10-01T11:01:38.180 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:37 smithi078 bash[78274]: cluster 2023-10-01T11:01:37.288591+0000 osd.6 (osd.6) 251 : cluster 3 failed to encode map e284 with expected crc 2023-10-01T11:01:38.180 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:37 smithi078 bash[75299]: cluster 2023-10-01T11:01:36.196375+0000 mgr.y (mgr.25072) 1327 : cluster 0 pgmap v1374: 73 pgs: 1 active+undersized+remapped, 72 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 134 KiB/s, 2 keys/s, 3 objects/s recovering 2023-10-01T11:01:38.181 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:37 smithi078 bash[75299]: audit 2023-10-01T11:01:36.932378+0000 mon.a (mon.0) 1662 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T11:01:38.181 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:37 smithi078 bash[75299]: cluster 2023-10-01T11:01:36.990251+0000 osd.7 (osd.7) 287 : cluster 3 failed to encode map e284 with expected crc 2023-10-01T11:01:38.181 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:37 smithi078 bash[75299]: audit 2023-10-01T11:01:36.995753+0000 mon.a (mon.0) 1663 : audit 1 from='client.? 172.21.15.78:0/3178037826' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]': finished 2023-10-01T11:01:38.181 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:37 smithi078 bash[75299]: cluster 2023-10-01T11:01:36.995851+0000 mon.a (mon.0) 1664 : cluster 0 osdmap e284: 8 total, 8 up, 7 in 2023-10-01T11:01:38.182 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:37 smithi078 bash[75299]: cluster 2023-10-01T11:01:37.285141+0000 osd.5 (osd.5) 271 : cluster 3 failed to encode map e284 with expected crc 2023-10-01T11:01:38.182 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:37 smithi078 bash[75299]: cluster 2023-10-01T11:01:37.285190+0000 osd.6 (osd.6) 250 : cluster 3 failed to encode map e284 with expected crc 2023-10-01T11:01:38.182 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:37 smithi078 bash[75299]: cluster 2023-10-01T11:01:37.288508+0000 osd.5 (osd.5) 272 : cluster 3 failed to encode map e284 with expected crc 2023-10-01T11:01:38.182 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:37 smithi078 bash[75299]: cluster 2023-10-01T11:01:37.288591+0000 osd.6 (osd.6) 251 : cluster 3 failed to encode map e284 with expected crc 2023-10-01T11:01:38.193 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:38.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:38.197 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:38.298 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:38.312 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:37 smithi173 bash[58816]: cluster 2023-10-01T11:01:36.196375+0000 mgr.y (mgr.25072) 1327 : cluster 0 pgmap v1374: 73 pgs: 1 active+undersized+remapped, 72 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 134 KiB/s, 2 keys/s, 3 objects/s recovering 2023-10-01T11:01:38.312 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:37 smithi173 bash[58816]: audit 2023-10-01T11:01:36.932378+0000 mon.a (mon.0) 1662 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T11:01:38.312 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:37 smithi173 bash[58816]: cluster 2023-10-01T11:01:36.990251+0000 osd.7 (osd.7) 287 : cluster 3 failed to encode map e284 with expected crc 2023-10-01T11:01:38.312 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:37 smithi173 bash[58816]: audit 2023-10-01T11:01:36.995753+0000 mon.a (mon.0) 1663 : audit 1 from='client.? 172.21.15.78:0/3178037826' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]': finished 2023-10-01T11:01:38.312 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:37 smithi173 bash[58816]: cluster 2023-10-01T11:01:36.995851+0000 mon.a (mon.0) 1664 : cluster 0 osdmap e284: 8 total, 8 up, 7 in 2023-10-01T11:01:38.312 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:37 smithi173 bash[58816]: cluster 2023-10-01T11:01:37.285141+0000 osd.5 (osd.5) 271 : cluster 3 failed to encode map e284 with expected crc 2023-10-01T11:01:38.313 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:37 smithi173 bash[58816]: cluster 2023-10-01T11:01:37.285190+0000 osd.6 (osd.6) 250 : cluster 3 failed to encode map e284 with expected crc 2023-10-01T11:01:38.313 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:37 smithi173 bash[58816]: cluster 2023-10-01T11:01:37.288508+0000 osd.5 (osd.5) 272 : cluster 3 failed to encode map e284 with expected crc 2023-10-01T11:01:38.313 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:37 smithi173 bash[58816]: cluster 2023-10-01T11:01:37.288591+0000 osd.6 (osd.6) 251 : cluster 3 failed to encode map e284 with expected crc 2023-10-01T11:01:38.332 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:38.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:38.335 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:01:38.436 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:38.463 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:38.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:38.466 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:01:38.567 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:38.602 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:38.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:38.605 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:01:38.706 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:38.740 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:38.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:38.742 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:01:38.843 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:38.865 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:38.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:38.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:38.967 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:38.995 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:38.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:38.997 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:01:39.002 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6] out_osds: [7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T11:01:39.002 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T11:01:39.002 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-01T11:01:39.002 INFO:tasks.thrashosds.thrasher:Setting osd 4 primary_affinity to 0.234609 2023-10-01T11:01:39.003 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 4 0.23460949209094606 2023-10-01T11:01:39.004 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:39 smithi078 bash[75299]: cluster 2023-10-01T11:01:37.990873+0000 mon.a (mon.0) 1665 : cluster 0 osdmap e285: 8 total, 8 up, 7 in 2023-10-01T11:01:39.004 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:39 smithi078 bash[75299]: cluster 2023-10-01T11:01:37.992373+0000 osd.6 (osd.6) 252 : cluster 3 failed to encode map e285 with expected crc 2023-10-01T11:01:39.004 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:39 smithi078 bash[75299]: cluster 2023-10-01T11:01:37.993561+0000 osd.5 (osd.5) 273 : cluster 3 failed to encode map e285 with expected crc 2023-10-01T11:01:39.098 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:39.120 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:39.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:39.124 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:01:39.224 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:39.248 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:39.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:39.250 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:01:39.286 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:39 smithi078 bash[78274]: cluster 2023-10-01T11:01:37.990873+0000 mon.a (mon.0) 1665 : cluster 0 osdmap e285: 8 total, 8 up, 7 in 2023-10-01T11:01:39.286 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:39 smithi078 bash[78274]: cluster 2023-10-01T11:01:37.992373+0000 osd.6 (osd.6) 252 : cluster 3 failed to encode map e285 with expected crc 2023-10-01T11:01:39.286 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:39 smithi078 bash[78274]: cluster 2023-10-01T11:01:37.993561+0000 osd.5 (osd.5) 273 : cluster 3 failed to encode map e285 with expected crc 2023-10-01T11:01:39.352 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:39.363 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:38 smithi173 bash[58816]: cluster 2023-10-01T11:01:37.990873+0000 mon.a (mon.0) 1665 : cluster 0 osdmap e285: 8 total, 8 up, 7 in 2023-10-01T11:01:39.363 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:38 smithi173 bash[58816]: cluster 2023-10-01T11:01:37.992373+0000 osd.6 (osd.6) 252 : cluster 3 failed to encode map e285 with expected crc 2023-10-01T11:01:39.363 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:39 smithi173 bash[58816]: cluster 2023-10-01T11:01:37.993561+0000 osd.5 (osd.5) 273 : cluster 3 failed to encode map e285 with expected crc 2023-10-01T11:01:39.382 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:39.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:39.385 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:01:39.486 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:39.510 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:39.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:39.512 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:01:39.613 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:39.647 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:39.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:39.650 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:01:39.750 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:39.784 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:39.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:39.786 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:01:39.887 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:39.910 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:39.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:39.912 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:40.013 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:40.041 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:40.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:40.044 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:01:40.144 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:40.170 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:39 smithi078 bash[75299]: cluster 2023-10-01T11:01:38.196932+0000 mgr.y (mgr.25072) 1328 : cluster 0 pgmap v1377: 89 pgs: 16 unknown, 1 active+undersized+remapped, 72 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 1.1 KiB/s, 2 keys/s, 1 objects/s recovering 2023-10-01T11:01:40.170 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:39 smithi078 bash[75299]: cluster 2023-10-01T11:01:39.002901+0000 osd.5 (osd.5) 274 : cluster 3 failed to encode map e286 with expected crc 2023-10-01T11:01:40.170 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:39 smithi078 bash[75299]: cluster 2023-10-01T11:01:39.006409+0000 mon.a (mon.0) 1666 : cluster 0 osdmap e286: 8 total, 8 up, 7 in 2023-10-01T11:01:40.171 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:39 smithi078 bash[78274]: cluster 2023-10-01T11:01:38.196932+0000 mgr.y (mgr.25072) 1328 : cluster 0 pgmap v1377: 89 pgs: 16 unknown, 1 active+undersized+remapped, 72 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 1.1 KiB/s, 2 keys/s, 1 objects/s recovering 2023-10-01T11:01:40.171 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:39 smithi078 bash[78274]: cluster 2023-10-01T11:01:39.002901+0000 osd.5 (osd.5) 274 : cluster 3 failed to encode map e286 with expected crc 2023-10-01T11:01:40.171 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:39 smithi078 bash[78274]: cluster 2023-10-01T11:01:39.006409+0000 mon.a (mon.0) 1666 : cluster 0 osdmap e286: 8 total, 8 up, 7 in 2023-10-01T11:01:40.172 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_8 rados --yes-i-really-mean-it || true 2023-10-01T11:01:40.178 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:40.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:40.181 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:01:40.282 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:40.304 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:40.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:40.307 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:40.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:39 smithi173 bash[58816]: cluster 2023-10-01T11:01:38.196932+0000 mgr.y (mgr.25072) 1328 : cluster 0 pgmap v1377: 89 pgs: 16 unknown, 1 active+undersized+remapped, 72 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 1.1 KiB/s, 2 keys/s, 1 objects/s recovering 2023-10-01T11:01:40.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:39 smithi173 bash[58816]: cluster 2023-10-01T11:01:39.002901+0000 osd.5 (osd.5) 274 : cluster 3 failed to encode map e286 with expected crc 2023-10-01T11:01:40.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:39 smithi173 bash[58816]: cluster 2023-10-01T11:01:39.006409+0000 mon.a (mon.0) 1666 : cluster 0 osdmap e286: 8 total, 8 up, 7 in 2023-10-01T11:01:40.408 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:40.438 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:40.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:40.442 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:01:40.521 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:01:40 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:01:40] "GET /metrics HTTP/1.1" 200 33967 "" "Prometheus/2.33.4" 2023-10-01T11:01:40.543 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:40.569 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:40.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:40.571 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:01:40.672 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:40.696 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:40.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:40.699 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:01:40.801 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:40.831 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:40.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:40.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:01:40.935 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:40.960 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:40.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:40.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:01:41.063 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:41.092 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:41.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:41.095 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:01:41.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:41 smithi173 bash[58816]: cluster 2023-10-01T11:01:39.703754+0000 osd.6 (osd.6) 253 : cluster 3 failed to encode map e286 with expected crc 2023-10-01T11:01:41.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:41 smithi173 bash[58816]: cluster 2023-10-01T11:01:39.703775+0000 osd.7 (osd.7) 288 : cluster 3 failed to encode map e286 with expected crc 2023-10-01T11:01:41.196 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:41.208 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:41 smithi078 bash[75299]: cluster 2023-10-01T11:01:39.703754+0000 osd.6 (osd.6) 253 : cluster 3 failed to encode map e286 with expected crc 2023-10-01T11:01:41.208 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:41 smithi078 bash[75299]: cluster 2023-10-01T11:01:39.703775+0000 osd.7 (osd.7) 288 : cluster 3 failed to encode map e286 with expected crc 2023-10-01T11:01:41.209 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:41 smithi078 bash[78274]: cluster 2023-10-01T11:01:39.703754+0000 osd.6 (osd.6) 253 : cluster 3 failed to encode map e286 with expected crc 2023-10-01T11:01:41.209 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:41 smithi078 bash[78274]: cluster 2023-10-01T11:01:39.703775+0000 osd.7 (osd.7) 288 : cluster 3 failed to encode map e286 with expected crc 2023-10-01T11:01:41.223 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:41.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:41.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:01:41.327 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:41.353 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:41.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:41.356 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:41.457 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:41.481 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:41.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:41.483 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:01:41.584 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:41.613 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:41.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:41.614 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:01:41.715 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:41.744 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:41.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:41.747 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:01:41.847 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:41.872 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:41.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:41.875 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:01:41.976 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:42.005 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:42.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:42.008 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:01:42.108 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:42.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:42 smithi173 bash[58816]: cluster 2023-10-01T11:01:40.197424+0000 mgr.y (mgr.25072) 1329 : cluster 0 pgmap v1379: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-10-01T11:01:42.151 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:42.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:42.154 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:01:42.255 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:42.285 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:42.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:42.287 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:01:42.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:42 smithi078 bash[78274]: cluster 2023-10-01T11:01:40.197424+0000 mgr.y (mgr.25072) 1329 : cluster 0 pgmap v1379: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-10-01T11:01:42.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:42 smithi078 bash[75299]: cluster 2023-10-01T11:01:40.197424+0000 mgr.y (mgr.25072) 1329 : cluster 0 pgmap v1379: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-10-01T11:01:42.388 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:42.418 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:42.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:42.421 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:01:42.521 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:42.546 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:42.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:42.550 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:01:42.651 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:42.679 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:42.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:42.683 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:01:42.784 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:42.814 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:42.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:42.816 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:01:42.917 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:42.949 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:42.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:42.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:01:43.052 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:43.103 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:43.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:43.107 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:01:43.208 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:43.238 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:43.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:43.240 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:01:43.341 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:43.370 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:43.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:43.373 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:01:43.474 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:43.504 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:43.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:43.507 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:01:43.511 INFO:teuthology.orchestra.run.smithi078.stdout:ERROR: (22) Invalid argument 2023-10-01T11:01:43.511 INFO:teuthology.orchestra.run.smithi078.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T11:01:43.608 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:43.636 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:43.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:43.639 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:01:43.740 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:43.767 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:43.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:43.769 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:43.800 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:01:43.870 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:43.921 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:43.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:43.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:44.025 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:44.048 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:44.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:44.052 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:44.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:44 smithi078 bash[78274]: cluster 2023-10-01T11:01:42.198001+0000 mgr.y (mgr.25072) 1330 : cluster 0 pgmap v1380: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-10-01T11:01:44.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:44 smithi078 bash[78274]: audit 2023-10-01T11:01:43.277495+0000 mon.a (mon.0) 1667 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:01:44.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:44 smithi078 bash[78274]: audit 2023-10-01T11:01:43.287257+0000 mon.a (mon.0) 1668 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:01:44.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:44 smithi078 bash[75299]: cluster 2023-10-01T11:01:42.198001+0000 mgr.y (mgr.25072) 1330 : cluster 0 pgmap v1380: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-10-01T11:01:44.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:44 smithi078 bash[75299]: audit 2023-10-01T11:01:43.277495+0000 mon.a (mon.0) 1667 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:01:44.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:44 smithi078 bash[75299]: audit 2023-10-01T11:01:43.287257+0000 mon.a (mon.0) 1668 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:01:44.154 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:44.176 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:44.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:44.178 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:01:44.279 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:44.339 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:44.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:44.340 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:01:44.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:44 smithi173 bash[58816]: cluster 2023-10-01T11:01:42.198001+0000 mgr.y (mgr.25072) 1330 : cluster 0 pgmap v1380: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-10-01T11:01:44.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:44 smithi173 bash[58816]: audit 2023-10-01T11:01:43.277495+0000 mon.a (mon.0) 1667 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:01:44.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:44 smithi173 bash[58816]: audit 2023-10-01T11:01:43.287257+0000 mon.a (mon.0) 1668 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:01:44.441 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:44.474 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:44.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:44.478 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:01:44.580 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:44.605 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:44.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:44.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:01:44.936 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:44.960 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:44.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:44.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:01:45.062 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:45.082 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:45.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:45.086 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:45.186 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:45.209 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:45.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:45.212 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:01:45.312 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:45.335 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:45.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:45.338 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:01:45.439 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:45.468 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:45.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:45.472 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:01:45.573 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:45.605 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:45.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:45.607 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:01:45.708 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:45.739 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:45.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:45.741 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:01:45.842 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:45.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:45 smithi173 bash[58816]: cluster 2023-10-01T11:01:44.198622+0000 mgr.y (mgr.25072) 1331 : cluster 0 pgmap v1381: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-10-01T11:01:45.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:45 smithi173 bash[58816]: audit 2023-10-01T11:01:44.831395+0000 mon.a (mon.0) 1669 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:01:45.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:45 smithi173 bash[58816]: audit 2023-10-01T11:01:44.839009+0000 mon.a (mon.0) 1670 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:01:45.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:45 smithi173 bash[58816]: audit 2023-10-01T11:01:45.258884+0000 mon.a (mon.0) 1671 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T11:01:45.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:45 smithi173 bash[58816]: audit 2023-10-01T11:01:45.259977+0000 mon.a (mon.0) 1672 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T11:01:45.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:45 smithi173 bash[58816]: audit 2023-10-01T11:01:45.269406+0000 mon.a (mon.0) 1673 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:01:45.872 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:45.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:45.874 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:01:45.975 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:45.994 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:45 smithi078 bash[78274]: cluster 2023-10-01T11:01:44.198622+0000 mgr.y (mgr.25072) 1331 : cluster 0 pgmap v1381: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-10-01T11:01:45.994 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:45 smithi078 bash[78274]: audit 2023-10-01T11:01:44.831395+0000 mon.a (mon.0) 1669 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:01:45.995 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:45 smithi078 bash[78274]: audit 2023-10-01T11:01:44.839009+0000 mon.a (mon.0) 1670 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:01:45.995 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:45 smithi078 bash[78274]: audit 2023-10-01T11:01:45.258884+0000 mon.a (mon.0) 1671 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T11:01:45.995 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:45 smithi078 bash[78274]: audit 2023-10-01T11:01:45.259977+0000 mon.a (mon.0) 1672 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T11:01:45.995 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:45 smithi078 bash[78274]: audit 2023-10-01T11:01:45.269406+0000 mon.a (mon.0) 1673 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:01:45.995 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:45 smithi078 bash[75299]: cluster 2023-10-01T11:01:44.198622+0000 mgr.y (mgr.25072) 1331 : cluster 0 pgmap v1381: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-10-01T11:01:45.995 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:45 smithi078 bash[75299]: audit 2023-10-01T11:01:44.831395+0000 mon.a (mon.0) 1669 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:01:45.995 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:45 smithi078 bash[75299]: audit 2023-10-01T11:01:44.839009+0000 mon.a (mon.0) 1670 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:01:45.996 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:45 smithi078 bash[75299]: audit 2023-10-01T11:01:45.258884+0000 mon.a (mon.0) 1671 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T11:01:45.996 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:45 smithi078 bash[75299]: audit 2023-10-01T11:01:45.259977+0000 mon.a (mon.0) 1672 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T11:01:45.996 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:45 smithi078 bash[75299]: audit 2023-10-01T11:01:45.269406+0000 mon.a (mon.0) 1673 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:01:46.008 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:46.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:46.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:46.112 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:46.148 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:46.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:46.149 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:01:46.251 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:46.283 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:46.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:46.285 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:01:46.385 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:46.422 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:46.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:46.425 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:01:46.529 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:46.561 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:46.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:46.564 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:01:46.666 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:46.694 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:46.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:46.696 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:01:46.796 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:46.832 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:46.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:46.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:01:46.937 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:46.963 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:46.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:46.965 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:01:47.066 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:47.099 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:47.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:47.103 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:01:47.203 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:47.214 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-01T11:01:47.234 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:47.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:47.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:01:47.342 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:47.373 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:47.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:47.376 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:01:47.477 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:47.508 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:47.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:47.512 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:01:47.612 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:47.648 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:47.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:47.652 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:01:47.685 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:01:47.685 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:01:47.754 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:47.780 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:47.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:47.782 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:01:47.883 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:47.919 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:47.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:47.922 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:01:48.023 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:48.061 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:48.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:48.064 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:01:48.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:47 smithi078 bash[75299]: cluster 2023-10-01T11:01:46.199271+0000 mgr.y (mgr.25072) 1332 : cluster 0 pgmap v1382: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-10-01T11:01:48.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:47 smithi078 bash[75299]: audit 2023-10-01T11:01:47.452855+0000 mon.a (mon.0) 1674 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:01:48.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:47 smithi078 bash[75299]: audit 2023-10-01T11:01:47.783040+0000 mon.c (mon.2) 205 : audit 1 from='client.? 172.21.15.78:0/2963651018' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:01:48.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:47 smithi078 bash[75299]: audit 2023-10-01T11:01:47.783422+0000 mon.a (mon.0) 1675 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:01:48.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:47 smithi078 bash[78274]: cluster 2023-10-01T11:01:46.199271+0000 mgr.y (mgr.25072) 1332 : cluster 0 pgmap v1382: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-10-01T11:01:48.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:47 smithi078 bash[78274]: audit 2023-10-01T11:01:47.452855+0000 mon.a (mon.0) 1674 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:01:48.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:47 smithi078 bash[78274]: audit 2023-10-01T11:01:47.783040+0000 mon.c (mon.2) 205 : audit 1 from='client.? 172.21.15.78:0/2963651018' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:01:48.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:47 smithi078 bash[78274]: audit 2023-10-01T11:01:47.783422+0000 mon.a (mon.0) 1675 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:01:48.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:47 smithi173 bash[58816]: cluster 2023-10-01T11:01:46.199271+0000 mgr.y (mgr.25072) 1332 : cluster 0 pgmap v1382: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-10-01T11:01:48.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:47 smithi173 bash[58816]: audit 2023-10-01T11:01:47.452855+0000 mon.a (mon.0) 1674 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:01:48.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:47 smithi173 bash[58816]: audit 2023-10-01T11:01:47.783040+0000 mon.c (mon.2) 205 : audit 1 from='client.? 172.21.15.78:0/2963651018' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:01:48.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:47 smithi173 bash[58816]: audit 2023-10-01T11:01:47.783422+0000 mon.a (mon.0) 1675 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:01:48.165 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:48.205 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:48.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:48.208 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:01:48.310 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:48.342 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:48.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:48.346 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:01:48.446 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:48.472 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:48.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:48.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:01:48.578 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:48.609 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:48.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:48.612 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:01:48.692 INFO:teuthology.orchestra.run.smithi078.stderr:nodeep-scrub is set 2023-10-01T11:01:48.713 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:48.746 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:48.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:48.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:01:48.851 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:48.888 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:48.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:48.891 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:01:48.992 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:49.021 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:49.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:49.023 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:01:49.124 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:49.150 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:49.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:49.152 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:01:49.253 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:49.283 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:49.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:49.285 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:01:49.386 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:49.417 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:49.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:49.420 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:01:49.521 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:49.546 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:49.547 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:49 smithi078 bash[78274]: audit 2023-10-01T11:01:48.163242+0000 mon.a (mon.0) 1676 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:01:49.548 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:49 smithi078 bash[78274]: cluster 2023-10-01T11:01:48.199707+0000 mgr.y (mgr.25072) 1333 : cluster 0 pgmap v1383: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-10-01T11:01:49.548 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:49 smithi078 bash[78274]: audit 2023-10-01T11:01:48.264457+0000 mon.a (mon.0) 1677 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 0]}]: dispatch 2023-10-01T11:01:49.548 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:49 smithi078 bash[78274]: audit 2023-10-01T11:01:48.264680+0000 mon.a (mon.0) 1678 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 0]}]: dispatch 2023-10-01T11:01:49.548 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:49 smithi078 bash[78274]: audit 2023-10-01T11:01:48.264883+0000 mon.a (mon.0) 1679 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [1, 5]}]: dispatch 2023-10-01T11:01:49.548 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:49 smithi078 bash[78274]: cluster 2023-10-01T11:01:48.674173+0000 mon.a (mon.0) 1680 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:01:49.548 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:49 smithi078 bash[78274]: audit 2023-10-01T11:01:48.684054+0000 mon.a (mon.0) 1681 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T11:01:49.548 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:49 smithi078 bash[78274]: audit 2023-10-01T11:01:48.684155+0000 mon.a (mon.0) 1682 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 0]}]': finished 2023-10-01T11:01:49.548 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:49 smithi078 bash[78274]: audit 2023-10-01T11:01:48.684417+0000 mon.a (mon.0) 1683 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 0]}]': finished 2023-10-01T11:01:49.548 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:49 smithi078 bash[78274]: audit 2023-10-01T11:01:48.684594+0000 mon.a (mon.0) 1684 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [1, 5]}]': finished 2023-10-01T11:01:49.549 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:49 smithi078 bash[78274]: cluster 2023-10-01T11:01:48.684685+0000 mon.a (mon.0) 1685 : cluster 0 osdmap e287: 8 total, 8 up, 7 in 2023-10-01T11:01:49.549 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:49 smithi078 bash[78274]: cluster 2023-10-01T11:01:48.687952+0000 osd.5 (osd.5) 275 : cluster 3 failed to encode map e287 with expected crc 2023-10-01T11:01:49.549 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:49 smithi078 bash[75299]: audit 2023-10-01T11:01:48.163242+0000 mon.a (mon.0) 1676 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:01:49.549 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:49 smithi078 bash[75299]: cluster 2023-10-01T11:01:48.199707+0000 mgr.y (mgr.25072) 1333 : cluster 0 pgmap v1383: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-10-01T11:01:49.549 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:49 smithi078 bash[75299]: audit 2023-10-01T11:01:48.264457+0000 mon.a (mon.0) 1677 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 0]}]: dispatch 2023-10-01T11:01:49.549 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:49 smithi078 bash[75299]: audit 2023-10-01T11:01:48.264680+0000 mon.a (mon.0) 1678 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 0]}]: dispatch 2023-10-01T11:01:49.550 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:49 smithi078 bash[75299]: audit 2023-10-01T11:01:48.264883+0000 mon.a (mon.0) 1679 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [1, 5]}]: dispatch 2023-10-01T11:01:49.550 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:49 smithi078 bash[75299]: cluster 2023-10-01T11:01:48.674173+0000 mon.a (mon.0) 1680 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:01:49.550 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:49 smithi078 bash[75299]: audit 2023-10-01T11:01:48.684054+0000 mon.a (mon.0) 1681 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T11:01:49.550 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:49 smithi078 bash[75299]: audit 2023-10-01T11:01:48.684155+0000 mon.a (mon.0) 1682 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 0]}]': finished 2023-10-01T11:01:49.550 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:49 smithi078 bash[75299]: audit 2023-10-01T11:01:48.684417+0000 mon.a (mon.0) 1683 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 0]}]': finished 2023-10-01T11:01:49.550 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:49 smithi078 bash[75299]: audit 2023-10-01T11:01:48.684594+0000 mon.a (mon.0) 1684 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [1, 5]}]': finished 2023-10-01T11:01:49.550 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:49 smithi078 bash[75299]: cluster 2023-10-01T11:01:48.684685+0000 mon.a (mon.0) 1685 : cluster 0 osdmap e287: 8 total, 8 up, 7 in 2023-10-01T11:01:49.550 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:49 smithi078 bash[75299]: cluster 2023-10-01T11:01:48.687952+0000 osd.5 (osd.5) 275 : cluster 3 failed to encode map e287 with expected crc 2023-10-01T11:01:49.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:49.551 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:49.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:49 smithi173 bash[58816]: audit 2023-10-01T11:01:48.163242+0000 mon.a (mon.0) 1676 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:01:49.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:49 smithi173 bash[58816]: cluster 2023-10-01T11:01:48.199707+0000 mgr.y (mgr.25072) 1333 : cluster 0 pgmap v1383: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-10-01T11:01:49.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:49 smithi173 bash[58816]: audit 2023-10-01T11:01:48.264457+0000 mon.a (mon.0) 1677 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 0]}]: dispatch 2023-10-01T11:01:49.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:49 smithi173 bash[58816]: audit 2023-10-01T11:01:48.264680+0000 mon.a (mon.0) 1678 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 0]}]: dispatch 2023-10-01T11:01:49.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:49 smithi173 bash[58816]: audit 2023-10-01T11:01:48.264883+0000 mon.a (mon.0) 1679 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [1, 5]}]: dispatch 2023-10-01T11:01:49.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:49 smithi173 bash[58816]: cluster 2023-10-01T11:01:48.674173+0000 mon.a (mon.0) 1680 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:01:49.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:49 smithi173 bash[58816]: audit 2023-10-01T11:01:48.684054+0000 mon.a (mon.0) 1681 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T11:01:49.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:49 smithi173 bash[58816]: audit 2023-10-01T11:01:48.684155+0000 mon.a (mon.0) 1682 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 0]}]': finished 2023-10-01T11:01:49.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:49 smithi173 bash[58816]: audit 2023-10-01T11:01:48.684417+0000 mon.a (mon.0) 1683 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 0]}]': finished 2023-10-01T11:01:49.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:49 smithi173 bash[58816]: audit 2023-10-01T11:01:48.684594+0000 mon.a (mon.0) 1684 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [1, 5]}]': finished 2023-10-01T11:01:49.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:49 smithi173 bash[58816]: cluster 2023-10-01T11:01:48.684685+0000 mon.a (mon.0) 1685 : cluster 0 osdmap e287: 8 total, 8 up, 7 in 2023-10-01T11:01:49.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:49 smithi173 bash[58816]: cluster 2023-10-01T11:01:48.687952+0000 osd.5 (osd.5) 275 : cluster 3 failed to encode map e287 with expected crc 2023-10-01T11:01:49.652 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:49.678 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:49.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:49.682 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:01:49.783 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:49.808 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:49.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:49.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:01:49.912 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:49.934 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:49.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:49.937 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:50.038 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:50.065 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:50.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:50.067 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:01:50.168 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:50.196 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:50.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:50.199 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:01:50.269 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:50 smithi078 bash[78274]: cluster 2023-10-01T11:01:48.688767+0000 osd.7 (osd.7) 289 : cluster 3 failed to encode map e287 with expected crc 2023-10-01T11:01:50.269 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:50 smithi078 bash[78274]: cluster 2023-10-01T11:01:48.690611+0000 osd.7 (osd.7) 290 : cluster 3 failed to encode map e287 with expected crc 2023-10-01T11:01:50.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:50 smithi078 bash[78274]: cluster 2023-10-01T11:01:48.690922+0000 osd.6 (osd.6) 254 : cluster 3 failed to encode map e287 with expected crc 2023-10-01T11:01:50.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:50 smithi078 bash[78274]: cluster 2023-10-01T11:01:49.684563+0000 mon.a (mon.0) 1686 : cluster 0 osdmap e288: 8 total, 8 up, 7 in 2023-10-01T11:01:50.270 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:50 smithi078 bash[75299]: cluster 2023-10-01T11:01:48.688767+0000 osd.7 (osd.7) 289 : cluster 3 failed to encode map e287 with expected crc 2023-10-01T11:01:50.270 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:50 smithi078 bash[75299]: cluster 2023-10-01T11:01:48.690611+0000 osd.7 (osd.7) 290 : cluster 3 failed to encode map e287 with expected crc 2023-10-01T11:01:50.270 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:50 smithi078 bash[75299]: cluster 2023-10-01T11:01:48.690922+0000 osd.6 (osd.6) 254 : cluster 3 failed to encode map e287 with expected crc 2023-10-01T11:01:50.271 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:50 smithi078 bash[75299]: cluster 2023-10-01T11:01:49.684563+0000 mon.a (mon.0) 1686 : cluster 0 osdmap e288: 8 total, 8 up, 7 in 2023-10-01T11:01:50.300 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:50.330 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:50.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:50.333 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:01:50.434 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:50.446 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:50 smithi173 bash[58816]: cluster 2023-10-01T11:01:48.688767+0000 osd.7 (osd.7) 289 : cluster 3 failed to encode map e287 with expected crc 2023-10-01T11:01:50.446 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:50 smithi173 bash[58816]: cluster 2023-10-01T11:01:48.690611+0000 osd.7 (osd.7) 290 : cluster 3 failed to encode map e287 with expected crc 2023-10-01T11:01:50.446 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:50 smithi173 bash[58816]: cluster 2023-10-01T11:01:48.690922+0000 osd.6 (osd.6) 254 : cluster 3 failed to encode map e287 with expected crc 2023-10-01T11:01:50.446 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:50 smithi173 bash[58816]: cluster 2023-10-01T11:01:49.684563+0000 mon.a (mon.0) 1686 : cluster 0 osdmap e288: 8 total, 8 up, 7 in 2023-10-01T11:01:50.464 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:50.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:50.467 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:01:50.567 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:50.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:01:50 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:01:50] "GET /metrics HTTP/1.1" 200 36314 "" "Prometheus/2.33.4" 2023-10-01T11:01:50.604 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:50.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:50.606 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:01:50.707 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:50.742 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:50.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:50.744 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:01:50.845 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:50.874 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:50.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:50.877 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:01:50.978 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:51.003 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:51.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:51.006 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:01:51.107 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:51.142 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:51.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:51.146 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:01:51.247 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:51.278 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:51.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:51.281 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:01:51.382 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:51.393 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:51 smithi078 bash[75299]: cluster 2023-10-01T11:01:49.789371+0000 osd.7 (osd.7) 291 : cluster 3 failed to encode map e288 with expected crc 2023-10-01T11:01:51.393 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:51 smithi078 bash[75299]: cluster 2023-10-01T11:01:49.792437+0000 osd.7 (osd.7) 292 : cluster 3 failed to encode map e288 with expected crc 2023-10-01T11:01:51.393 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:51 smithi078 bash[75299]: cluster 2023-10-01T11:01:49.795364+0000 osd.7 (osd.7) 293 : cluster 3 failed to encode map e288 with expected crc 2023-10-01T11:01:51.394 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:51 smithi078 bash[75299]: cluster 2023-10-01T11:01:50.139573+0000 osd.6 (osd.6) 255 : cluster 3 failed to encode map e288 with expected crc 2023-10-01T11:01:51.394 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:51 smithi078 bash[75299]: cluster 2023-10-01T11:01:50.200316+0000 mgr.y (mgr.25072) 1334 : cluster 0 pgmap v1386: 89 pgs: 3 peering, 86 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-10-01T11:01:51.394 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:51 smithi078 bash[75299]: cluster 2023-10-01T11:01:50.413667+0000 osd.5 (osd.5) 276 : cluster 3 failed to encode map e288 with expected crc 2023-10-01T11:01:51.394 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:51 smithi078 bash[78274]: cluster 2023-10-01T11:01:49.789371+0000 osd.7 (osd.7) 291 : cluster 3 failed to encode map e288 with expected crc 2023-10-01T11:01:51.394 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:51 smithi078 bash[78274]: cluster 2023-10-01T11:01:49.792437+0000 osd.7 (osd.7) 292 : cluster 3 failed to encode map e288 with expected crc 2023-10-01T11:01:51.394 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:51 smithi078 bash[78274]: cluster 2023-10-01T11:01:49.795364+0000 osd.7 (osd.7) 293 : cluster 3 failed to encode map e288 with expected crc 2023-10-01T11:01:51.394 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:51 smithi078 bash[78274]: cluster 2023-10-01T11:01:50.139573+0000 osd.6 (osd.6) 255 : cluster 3 failed to encode map e288 with expected crc 2023-10-01T11:01:51.394 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:51 smithi078 bash[78274]: cluster 2023-10-01T11:01:50.200316+0000 mgr.y (mgr.25072) 1334 : cluster 0 pgmap v1386: 89 pgs: 3 peering, 86 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-10-01T11:01:51.394 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:51 smithi078 bash[78274]: cluster 2023-10-01T11:01:50.413667+0000 osd.5 (osd.5) 276 : cluster 3 failed to encode map e288 with expected crc 2023-10-01T11:01:51.406 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:51.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:51.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:01:51.510 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:51.540 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:51.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:51.543 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:51.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:51 smithi173 bash[58816]: cluster 2023-10-01T11:01:49.789371+0000 osd.7 (osd.7) 291 : cluster 3 failed to encode map e288 with expected crc 2023-10-01T11:01:51.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:51 smithi173 bash[58816]: cluster 2023-10-01T11:01:49.792437+0000 osd.7 (osd.7) 292 : cluster 3 failed to encode map e288 with expected crc 2023-10-01T11:01:51.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:51 smithi173 bash[58816]: cluster 2023-10-01T11:01:49.795364+0000 osd.7 (osd.7) 293 : cluster 3 failed to encode map e288 with expected crc 2023-10-01T11:01:51.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:51 smithi173 bash[58816]: cluster 2023-10-01T11:01:50.139573+0000 osd.6 (osd.6) 255 : cluster 3 failed to encode map e288 with expected crc 2023-10-01T11:01:51.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:51 smithi173 bash[58816]: cluster 2023-10-01T11:01:50.200316+0000 mgr.y (mgr.25072) 1334 : cluster 0 pgmap v1386: 89 pgs: 3 peering, 86 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-10-01T11:01:51.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:51 smithi173 bash[58816]: cluster 2023-10-01T11:01:50.413667+0000 osd.5 (osd.5) 276 : cluster 3 failed to encode map e288 with expected crc 2023-10-01T11:01:51.643 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:51.677 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:51.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:51.681 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:01:51.781 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:51.812 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:51.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:51.815 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:01:51.916 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:51.937 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:51.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:51.940 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:01:52.041 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:52.063 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:52.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:52.065 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:52.166 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:52.189 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:52.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:52.192 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:01:52.293 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:52.318 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:52.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:52.322 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:01:52.423 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:52.445 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:52.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:52.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:01:52.548 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:52.574 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:52.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:52.577 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:52.678 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:52.708 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:52.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:52.712 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:01:52.812 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:52.845 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:52.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:52.848 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:01:52.949 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:52.978 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:52.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:52.981 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:01:53.082 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:53.113 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:53.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:53.116 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:01:53.217 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:53.245 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:53.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:53.249 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:53.350 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:53.375 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:53.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:53.378 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:01:53.479 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:53.514 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:53.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:53.520 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:01:53.621 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:53.653 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:53.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:53.656 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:01:53.757 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:53.769 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:53 smithi173 bash[58816]: cluster 2023-10-01T11:01:52.200769+0000 mgr.y (mgr.25072) 1335 : cluster 0 pgmap v1387: 89 pgs: 3 peering, 86 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-10-01T11:01:53.789 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:53.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:53.792 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:01:53.893 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:53.899 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:53 smithi078 bash[78274]: cluster 2023-10-01T11:01:52.200769+0000 mgr.y (mgr.25072) 1335 : cluster 0 pgmap v1387: 89 pgs: 3 peering, 86 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-10-01T11:01:53.900 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:53 smithi078 bash[75299]: cluster 2023-10-01T11:01:52.200769+0000 mgr.y (mgr.25072) 1335 : cluster 0 pgmap v1387: 89 pgs: 3 peering, 86 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-10-01T11:01:53.926 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:53.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:53.929 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:01:54.030 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:54.062 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:54.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:54.065 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:01:54.070 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T11:01:54.090 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T11:01:54.114 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T11:01:54.134 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T11:01:54.164 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T11:01:54.167 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:54.186 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:54.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:54.189 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:01:54.192 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T11:01:54.219 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T11:01:54.244 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T11:01:54.246 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T11:01:54.246 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T11:01:54.247 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T11:01:54.247 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T11:01:54.247 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T11:01:54.247 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T11:01:54.247 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T11:01:54.247 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T11:01:54.290 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:54.315 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:54.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:54.319 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:01:54.420 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:54.441 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:54.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:54.443 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:01:54.526 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T11:01:54.545 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:54.577 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:54.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:54.580 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:01:54.681 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:54.710 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:54.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:54.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:01:54.813 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:54.842 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:54.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:54.845 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:01:54.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:54 smithi173 bash[58816]: audit 2023-10-01T11:01:54.044687+0000 mon.c (mon.2) 206 : audit 0 from='client.? 172.21.15.78:0/429370081' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:01:54.946 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:54.976 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:54.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:54.980 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:01:55.081 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:55.108 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:55.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:55.111 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:01:55.187 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:54 smithi078 bash[75299]: audit 2023-10-01T11:01:54.044687+0000 mon.c (mon.2) 206 : audit 0 from='client.? 172.21.15.78:0/429370081' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:01:55.188 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:54 smithi078 bash[78274]: audit 2023-10-01T11:01:54.044687+0000 mon.c (mon.2) 206 : audit 0 from='client.? 172.21.15.78:0/429370081' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:01:55.211 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:55.237 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:55.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:55.240 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:55.341 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:55.362 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:55.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:55.365 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:01:55.466 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:55.501 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:55.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:55.503 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:01:55.604 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:55.629 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:55.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:55.632 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:01:55.733 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:55.762 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:55.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:55.765 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:55.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:55 smithi078 bash[78274]: cluster 2023-10-01T11:01:54.201364+0000 mgr.y (mgr.25072) 1336 : cluster 0 pgmap v1388: 89 pgs: 2 peering, 87 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-10-01T11:01:55.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:55 smithi078 bash[75299]: cluster 2023-10-01T11:01:54.201364+0000 mgr.y (mgr.25072) 1336 : cluster 0 pgmap v1388: 89 pgs: 2 peering, 87 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-10-01T11:01:55.866 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:55.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:55 smithi173 bash[58816]: cluster 2023-10-01T11:01:54.201364+0000 mgr.y (mgr.25072) 1336 : cluster 0 pgmap v1388: 89 pgs: 2 peering, 87 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-10-01T11:01:55.894 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:55.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:55.896 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:01:55.997 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:56.024 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:56.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:56.027 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:01:56.128 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:56.159 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:56.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:56.162 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:01:56.263 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:56.292 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:56.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:56.294 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:01:56.395 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:56.429 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:56.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:56.432 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:01:56.533 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:56.566 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:56.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:56.569 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:01:56.670 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:56.707 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:56.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:56.710 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:01:56.810 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:56.835 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:56.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:56.837 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:01:56.938 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:56.959 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:56.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:56.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:01:57.063 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:57.091 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:57.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:57.093 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:01:57.194 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:57.227 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:57.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:57.230 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:01:57.331 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:57.362 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:57.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:57.365 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:57.465 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:57.501 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:57.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:57.503 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:01:57.605 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:57.641 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:57.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:57.645 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:01:57.746 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:57.760 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:57 smithi173 bash[58816]: cluster 2023-10-01T11:01:56.202165+0000 mgr.y (mgr.25072) 1337 : cluster 0 pgmap v1389: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-10-01T11:01:57.779 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:57.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:57.781 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:01:57.882 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:57.918 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:57.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:57.920 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:01:57.970 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:57 smithi078 bash[75299]: cluster 2023-10-01T11:01:56.202165+0000 mgr.y (mgr.25072) 1337 : cluster 0 pgmap v1389: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-10-01T11:01:57.970 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:57 smithi078 bash[78274]: cluster 2023-10-01T11:01:56.202165+0000 mgr.y (mgr.25072) 1337 : cluster 0 pgmap v1389: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-10-01T11:01:58.022 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:58.055 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:58.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:58.059 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:01:58.160 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:58.187 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:58.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:58.190 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:01:58.291 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:58.329 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:58.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:58.331 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:01:58.432 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:58.465 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:01:58.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:58.468 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:01:58.568 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:58.572 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:01:58.601 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:01:58.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:58.603 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:01:58.704 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:58.727 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:01:58.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:58.730 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:01:58.831 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:58.858 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:58.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:58.860 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:01:58.961 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:58.992 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:58.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:58.995 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:01:59.096 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:59.129 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:01:59.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:59.132 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:01:59.234 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:59.260 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:59.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:59.264 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:01:59.365 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:59.391 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:01:59.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:59.394 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:01:59.494 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:59.516 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:59.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:59.518 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:01:59.618 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:59.641 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:01:59.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:59.643 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:01:59.744 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:59.770 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:01:59.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:59.772 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:01:59.872 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:59.906 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:01:59.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:01:59.910 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:01:59.924 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:01:59 smithi078 bash[78274]: cluster 2023-10-01T11:01:58.202777+0000 mgr.y (mgr.25072) 1338 : cluster 0 pgmap v1390: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-10-01T11:01:59.924 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:01:59 smithi078 bash[75299]: cluster 2023-10-01T11:01:58.202777+0000 mgr.y (mgr.25072) 1338 : cluster 0 pgmap v1390: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-10-01T11:02:00.011 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:00.040 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:00.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:00.042 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:02:00.088 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T11:02:00.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:01:59 smithi173 bash[58816]: cluster 2023-10-01T11:01:58.202777+0000 mgr.y (mgr.25072) 1338 : cluster 0 pgmap v1390: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-10-01T11:02:00.145 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:00.167 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:00.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:00.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:02:00.270 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:00.297 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:00.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:00.299 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:02:00.400 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:00.426 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:00.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:00.428 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:02:00.501 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:02:00 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:02:00] "GET /metrics HTTP/1.1" 200 36314 "" "Prometheus/2.33.4" 2023-10-01T11:02:00.530 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:00.557 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:00.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:00.559 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:02:00.660 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:00.680 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:00.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:00.682 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:02:00.783 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:00.815 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:00.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:00.818 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:02:00.919 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:00.949 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:00.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:00.951 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:02:01.053 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:01.083 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:01.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:01.086 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:02:01.186 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:01.222 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:01.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:01.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:02:01.327 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:01.360 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:01.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:01.362 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:02:01.463 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:01.487 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:01.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:01.490 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:02:01.591 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:01.613 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:02:01.616 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:01.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:01.620 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:02:01.720 INFO:teuthology.orchestra.run.smithi078.stderr:enabled application 'rados' on pool 'unique_pool_8' 2023-10-01T11:02:01.721 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:01.749 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:01.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:01.751 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:02:01.852 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:01.907 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:01 smithi078 bash[78274]: cluster 2023-10-01T11:02:00.203383+0000 mgr.y (mgr.25072) 1339 : cluster 0 pgmap v1391: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-10-01T11:02:01.907 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:01 smithi078 bash[78274]: audit 2023-10-01T11:02:00.852771+0000 mon.c (mon.2) 207 : audit 1 from='client.? 172.21.15.78:0/3429748850' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T11:02:01.908 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:01 smithi078 bash[78274]: audit 2023-10-01T11:02:00.853247+0000 mon.a (mon.0) 1687 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T11:02:01.909 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:01 smithi078 bash[75299]: cluster 2023-10-01T11:02:00.203383+0000 mgr.y (mgr.25072) 1339 : cluster 0 pgmap v1391: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-10-01T11:02:01.909 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:01 smithi078 bash[75299]: audit 2023-10-01T11:02:00.852771+0000 mon.c (mon.2) 207 : audit 1 from='client.? 172.21.15.78:0/3429748850' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T11:02:01.909 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:01 smithi078 bash[75299]: audit 2023-10-01T11:02:00.853247+0000 mon.a (mon.0) 1687 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T11:02:01.918 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:01.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:01.921 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:02:02.022 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:02.041 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:01 smithi173 bash[58816]: cluster 2023-10-01T11:02:00.203383+0000 mgr.y (mgr.25072) 1339 : cluster 0 pgmap v1391: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-10-01T11:02:02.042 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:01 smithi173 bash[58816]: audit 2023-10-01T11:02:00.852771+0000 mon.c (mon.2) 207 : audit 1 from='client.? 172.21.15.78:0/3429748850' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T11:02:02.042 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:01 smithi173 bash[58816]: audit 2023-10-01T11:02:00.853247+0000 mon.a (mon.0) 1687 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T11:02:02.063 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:02.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:02.067 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:02:02.168 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:02.198 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:02.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:02.201 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:02:02.302 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:02.330 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:02.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:02.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:02:02.433 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:02.456 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:02.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:02.459 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:02:02.560 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:02.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:02.589 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:02.589 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:02:02.690 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:02.716 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:02.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:02.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:02:02.737 INFO:teuthology.orchestra.run.smithi078.stderr:set osd.4 primary-affinity to 0.234609 (8153752) 2023-10-01T11:02:02.819 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:02.849 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:02.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:02.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:02:02.953 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:02.965 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:02 smithi173 bash[58816]: cluster 2023-10-01T11:02:01.709741+0000 osd.5 (osd.5) 277 : cluster 3 failed to encode map e289 with expected crc 2023-10-01T11:02:02.965 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:02 smithi173 bash[58816]: audit 2023-10-01T11:02:01.711920+0000 mon.a (mon.0) 1688 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-01T11:02:02.966 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:02 smithi173 bash[58816]: cluster 2023-10-01T11:02:01.712023+0000 mon.a (mon.0) 1689 : cluster 0 osdmap e289: 8 total, 8 up, 7 in 2023-10-01T11:02:02.966 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:02 smithi173 bash[58816]: audit 2023-10-01T11:02:02.278012+0000 mon.a (mon.0) 1690 : audit 1 from='client.? 172.21.15.78:0/1971415489' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.23460949209094606}]: dispatch 2023-10-01T11:02:02.966 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:02 smithi173 bash[58816]: audit 2023-10-01T11:02:02.453421+0000 mon.a (mon.0) 1691 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:02:02.983 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:02.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:02.987 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:02:03.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:02 smithi078 bash[75299]: cluster 2023-10-01T11:02:01.709741+0000 osd.5 (osd.5) 277 : cluster 3 failed to encode map e289 with expected crc 2023-10-01T11:02:03.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:02 smithi078 bash[75299]: audit 2023-10-01T11:02:01.711920+0000 mon.a (mon.0) 1688 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-01T11:02:03.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:02 smithi078 bash[75299]: cluster 2023-10-01T11:02:01.712023+0000 mon.a (mon.0) 1689 : cluster 0 osdmap e289: 8 total, 8 up, 7 in 2023-10-01T11:02:03.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:02 smithi078 bash[75299]: audit 2023-10-01T11:02:02.278012+0000 mon.a (mon.0) 1690 : audit 1 from='client.? 172.21.15.78:0/1971415489' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.23460949209094606}]: dispatch 2023-10-01T11:02:03.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:02 smithi078 bash[75299]: audit 2023-10-01T11:02:02.453421+0000 mon.a (mon.0) 1691 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:02:03.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:02 smithi078 bash[78274]: cluster 2023-10-01T11:02:01.709741+0000 osd.5 (osd.5) 277 : cluster 3 failed to encode map e289 with expected crc 2023-10-01T11:02:03.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:02 smithi078 bash[78274]: audit 2023-10-01T11:02:01.711920+0000 mon.a (mon.0) 1688 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-01T11:02:03.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:02 smithi078 bash[78274]: cluster 2023-10-01T11:02:01.712023+0000 mon.a (mon.0) 1689 : cluster 0 osdmap e289: 8 total, 8 up, 7 in 2023-10-01T11:02:03.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:02 smithi078 bash[78274]: audit 2023-10-01T11:02:02.278012+0000 mon.a (mon.0) 1690 : audit 1 from='client.? 172.21.15.78:0/1971415489' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.23460949209094606}]: dispatch 2023-10-01T11:02:03.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:02 smithi078 bash[78274]: audit 2023-10-01T11:02:02.453421+0000 mon.a (mon.0) 1691 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:02:03.088 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:03.118 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:03.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:03.122 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:02:03.223 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:03.244 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:03.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:03.248 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:02:03.349 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:03.382 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:03.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:03.385 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:02:03.485 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:03.517 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:03.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:03.520 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:02:03.620 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:03.650 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:03.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:03.653 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:02:03.754 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:03.782 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:03.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:03.786 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:02:03.886 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:03.896 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:03 smithi173 bash[58816]: cluster 2023-10-01T11:02:02.203819+0000 mgr.y (mgr.25072) 1340 : cluster 0 pgmap v1393: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-10-01T11:02:03.897 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:03 smithi173 bash[58816]: cluster 2023-10-01T11:02:02.723448+0000 osd.6 (osd.6) 256 : cluster 3 failed to encode map e290 with expected crc 2023-10-01T11:02:03.897 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:03 smithi173 bash[58816]: audit 2023-10-01T11:02:02.725626+0000 mon.a (mon.0) 1692 : audit 1 from='client.? 172.21.15.78:0/1971415489' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.23460949209094606}]': finished 2023-10-01T11:02:03.897 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:03 smithi173 bash[58816]: cluster 2023-10-01T11:02:02.725748+0000 mon.a (mon.0) 1693 : cluster 0 osdmap e290: 8 total, 8 up, 7 in 2023-10-01T11:02:03.897 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:03 smithi173 bash[58816]: cluster 2023-10-01T11:02:02.726706+0000 osd.5 (osd.5) 278 : cluster 3 failed to encode map e290 with expected crc 2023-10-01T11:02:03.897 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:03 smithi173 bash[58816]: cluster 2023-10-01T11:02:02.730038+0000 osd.6 (osd.6) 257 : cluster 3 failed to encode map e290 with expected crc 2023-10-01T11:02:03.897 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:03 smithi173 bash[58816]: cluster 2023-10-01T11:02:02.730733+0000 osd.5 (osd.5) 279 : cluster 3 failed to encode map e290 with expected crc 2023-10-01T11:02:03.897 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:03 smithi173 bash[58816]: cluster 2023-10-01T11:02:02.732881+0000 osd.6 (osd.6) 258 : cluster 3 failed to encode map e290 with expected crc 2023-10-01T11:02:03.897 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:03 smithi173 bash[58816]: cluster 2023-10-01T11:02:02.890213+0000 osd.7 (osd.7) 294 : cluster 3 failed to encode map e290 with expected crc 2023-10-01T11:02:03.897 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:03 smithi173 bash[58816]: audit 2023-10-01T11:02:03.179177+0000 mon.a (mon.0) 1694 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:02:03.913 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:03.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:03.916 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:02:04.017 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:04.051 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:04.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:04.054 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:02:04.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:03 smithi078 bash[75299]: cluster 2023-10-01T11:02:02.203819+0000 mgr.y (mgr.25072) 1340 : cluster 0 pgmap v1393: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-10-01T11:02:04.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:03 smithi078 bash[75299]: cluster 2023-10-01T11:02:02.723448+0000 osd.6 (osd.6) 256 : cluster 3 failed to encode map e290 with expected crc 2023-10-01T11:02:04.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:03 smithi078 bash[75299]: audit 2023-10-01T11:02:02.725626+0000 mon.a (mon.0) 1692 : audit 1 from='client.? 172.21.15.78:0/1971415489' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.23460949209094606}]': finished 2023-10-01T11:02:04.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:03 smithi078 bash[75299]: cluster 2023-10-01T11:02:02.725748+0000 mon.a (mon.0) 1693 : cluster 0 osdmap e290: 8 total, 8 up, 7 in 2023-10-01T11:02:04.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:03 smithi078 bash[75299]: cluster 2023-10-01T11:02:02.726706+0000 osd.5 (osd.5) 278 : cluster 3 failed to encode map e290 with expected crc 2023-10-01T11:02:04.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:03 smithi078 bash[75299]: cluster 2023-10-01T11:02:02.730038+0000 osd.6 (osd.6) 257 : cluster 3 failed to encode map e290 with expected crc 2023-10-01T11:02:04.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:03 smithi078 bash[75299]: cluster 2023-10-01T11:02:02.730733+0000 osd.5 (osd.5) 279 : cluster 3 failed to encode map e290 with expected crc 2023-10-01T11:02:04.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:03 smithi078 bash[75299]: cluster 2023-10-01T11:02:02.732881+0000 osd.6 (osd.6) 258 : cluster 3 failed to encode map e290 with expected crc 2023-10-01T11:02:04.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:03 smithi078 bash[75299]: cluster 2023-10-01T11:02:02.890213+0000 osd.7 (osd.7) 294 : cluster 3 failed to encode map e290 with expected crc 2023-10-01T11:02:04.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:03 smithi078 bash[75299]: audit 2023-10-01T11:02:03.179177+0000 mon.a (mon.0) 1694 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:02:04.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:03 smithi078 bash[78274]: cluster 2023-10-01T11:02:02.203819+0000 mgr.y (mgr.25072) 1340 : cluster 0 pgmap v1393: 89 pgs: 89 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-10-01T11:02:04.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:03 smithi078 bash[78274]: cluster 2023-10-01T11:02:02.723448+0000 osd.6 (osd.6) 256 : cluster 3 failed to encode map e290 with expected crc 2023-10-01T11:02:04.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:03 smithi078 bash[78274]: audit 2023-10-01T11:02:02.725626+0000 mon.a (mon.0) 1692 : audit 1 from='client.? 172.21.15.78:0/1971415489' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.23460949209094606}]': finished 2023-10-01T11:02:04.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:03 smithi078 bash[78274]: cluster 2023-10-01T11:02:02.725748+0000 mon.a (mon.0) 1693 : cluster 0 osdmap e290: 8 total, 8 up, 7 in 2023-10-01T11:02:04.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:03 smithi078 bash[78274]: cluster 2023-10-01T11:02:02.726706+0000 osd.5 (osd.5) 278 : cluster 3 failed to encode map e290 with expected crc 2023-10-01T11:02:04.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:03 smithi078 bash[78274]: cluster 2023-10-01T11:02:02.730038+0000 osd.6 (osd.6) 257 : cluster 3 failed to encode map e290 with expected crc 2023-10-01T11:02:04.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:03 smithi078 bash[78274]: cluster 2023-10-01T11:02:02.730733+0000 osd.5 (osd.5) 279 : cluster 3 failed to encode map e290 with expected crc 2023-10-01T11:02:04.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:03 smithi078 bash[78274]: cluster 2023-10-01T11:02:02.732881+0000 osd.6 (osd.6) 258 : cluster 3 failed to encode map e290 with expected crc 2023-10-01T11:02:04.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:03 smithi078 bash[78274]: cluster 2023-10-01T11:02:02.890213+0000 osd.7 (osd.7) 294 : cluster 3 failed to encode map e290 with expected crc 2023-10-01T11:02:04.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:03 smithi078 bash[78274]: audit 2023-10-01T11:02:03.179177+0000 mon.a (mon.0) 1694 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:02:04.155 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:04.181 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:04.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:04.184 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:02:04.285 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:04.308 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:04.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:04.310 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:02:04.411 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:04.442 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:04.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:04.445 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:02:04.546 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:04.579 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:04.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:04.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:02:04.682 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:04.705 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:04.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:04.707 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:02:04.808 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:04.836 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:04.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:04.838 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:02:04.939 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:04.971 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:04.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:04.974 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:02:05.074 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:05.086 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:04 smithi078 bash[75299]: cluster 2023-10-01T11:02:03.728965+0000 osd.5 (osd.5) 280 : cluster 3 failed to encode map e291 with expected crc 2023-10-01T11:02:05.086 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:04 smithi078 bash[75299]: cluster 2023-10-01T11:02:03.730353+0000 mon.a (mon.0) 1695 : cluster 0 osdmap e291: 8 total, 8 up, 7 in 2023-10-01T11:02:05.086 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:04 smithi078 bash[75299]: cluster 2023-10-01T11:02:03.731827+0000 osd.7 (osd.7) 295 : cluster 3 failed to encode map e291 with expected crc 2023-10-01T11:02:05.086 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:04 smithi078 bash[75299]: cluster 2023-10-01T11:02:03.731956+0000 osd.6 (osd.6) 259 : cluster 3 failed to encode map e291 with expected crc 2023-10-01T11:02:05.086 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:04 smithi078 bash[75299]: cluster 2023-10-01T11:02:03.733214+0000 osd.5 (osd.5) 281 : cluster 3 failed to encode map e291 with expected crc 2023-10-01T11:02:05.086 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:04 smithi078 bash[75299]: cluster 2023-10-01T11:02:03.737552+0000 osd.6 (osd.6) 260 : cluster 3 failed to encode map e291 with expected crc 2023-10-01T11:02:05.086 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:04 smithi078 bash[75299]: cluster 2023-10-01T11:02:03.739487+0000 osd.6 (osd.6) 261 : cluster 3 failed to encode map e291 with expected crc 2023-10-01T11:02:05.087 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:04 smithi078 bash[75299]: cluster 2023-10-01T11:02:03.740800+0000 osd.6 (osd.6) 262 : cluster 3 failed to encode map e291 with expected crc 2023-10-01T11:02:05.087 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:04 smithi078 bash[78274]: cluster 2023-10-01T11:02:03.728965+0000 osd.5 (osd.5) 280 : cluster 3 failed to encode map e291 with expected crc 2023-10-01T11:02:05.087 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:04 smithi078 bash[78274]: cluster 2023-10-01T11:02:03.730353+0000 mon.a (mon.0) 1695 : cluster 0 osdmap e291: 8 total, 8 up, 7 in 2023-10-01T11:02:05.087 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:04 smithi078 bash[78274]: cluster 2023-10-01T11:02:03.731827+0000 osd.7 (osd.7) 295 : cluster 3 failed to encode map e291 with expected crc 2023-10-01T11:02:05.087 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:04 smithi078 bash[78274]: cluster 2023-10-01T11:02:03.731956+0000 osd.6 (osd.6) 259 : cluster 3 failed to encode map e291 with expected crc 2023-10-01T11:02:05.087 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:04 smithi078 bash[78274]: cluster 2023-10-01T11:02:03.733214+0000 osd.5 (osd.5) 281 : cluster 3 failed to encode map e291 with expected crc 2023-10-01T11:02:05.087 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:04 smithi078 bash[78274]: cluster 2023-10-01T11:02:03.737552+0000 osd.6 (osd.6) 260 : cluster 3 failed to encode map e291 with expected crc 2023-10-01T11:02:05.087 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:04 smithi078 bash[78274]: cluster 2023-10-01T11:02:03.739487+0000 osd.6 (osd.6) 261 : cluster 3 failed to encode map e291 with expected crc 2023-10-01T11:02:05.087 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:04 smithi078 bash[78274]: cluster 2023-10-01T11:02:03.740800+0000 osd.6 (osd.6) 262 : cluster 3 failed to encode map e291 with expected crc 2023-10-01T11:02:05.096 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:05.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:05.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:02:05.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:04 smithi173 bash[58816]: cluster 2023-10-01T11:02:03.728965+0000 osd.5 (osd.5) 280 : cluster 3 failed to encode map e291 with expected crc 2023-10-01T11:02:05.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:04 smithi173 bash[58816]: cluster 2023-10-01T11:02:03.730353+0000 mon.a (mon.0) 1695 : cluster 0 osdmap e291: 8 total, 8 up, 7 in 2023-10-01T11:02:05.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:04 smithi173 bash[58816]: cluster 2023-10-01T11:02:03.731827+0000 osd.7 (osd.7) 295 : cluster 3 failed to encode map e291 with expected crc 2023-10-01T11:02:05.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:04 smithi173 bash[58816]: cluster 2023-10-01T11:02:03.731956+0000 osd.6 (osd.6) 259 : cluster 3 failed to encode map e291 with expected crc 2023-10-01T11:02:05.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:04 smithi173 bash[58816]: cluster 2023-10-01T11:02:03.733214+0000 osd.5 (osd.5) 281 : cluster 3 failed to encode map e291 with expected crc 2023-10-01T11:02:05.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:04 smithi173 bash[58816]: cluster 2023-10-01T11:02:03.737552+0000 osd.6 (osd.6) 260 : cluster 3 failed to encode map e291 with expected crc 2023-10-01T11:02:05.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:04 smithi173 bash[58816]: cluster 2023-10-01T11:02:03.739487+0000 osd.6 (osd.6) 261 : cluster 3 failed to encode map e291 with expected crc 2023-10-01T11:02:05.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:04 smithi173 bash[58816]: cluster 2023-10-01T11:02:03.740800+0000 osd.6 (osd.6) 262 : cluster 3 failed to encode map e291 with expected crc 2023-10-01T11:02:05.199 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:05.232 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:05.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:05.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:02:05.336 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:05.373 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:05.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:05.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:02:05.476 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:05.502 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:05.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:05.504 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:02:05.605 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:05.641 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:05.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:05.643 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:02:05.744 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:05.765 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:05.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:05.768 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:02:05.868 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:05.900 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:05.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:05.903 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:02:06.004 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:06.018 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:05 smithi173 bash[58816]: cluster 2023-10-01T11:02:04.204420+0000 mgr.y (mgr.25072) 1341 : cluster 0 pgmap v1396: 89 pgs: 9 peering, 80 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-10-01T11:02:06.018 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:05 smithi173 bash[58816]: cluster 2023-10-01T11:02:04.727510+0000 mon.a (mon.0) 1696 : cluster 3 Health check failed: Reduced data availability: 3 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2023-10-01T11:02:06.036 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:06.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:06.039 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:02:06.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:05 smithi078 bash[75299]: cluster 2023-10-01T11:02:04.204420+0000 mgr.y (mgr.25072) 1341 : cluster 0 pgmap v1396: 89 pgs: 9 peering, 80 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-10-01T11:02:06.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:05 smithi078 bash[75299]: cluster 2023-10-01T11:02:04.727510+0000 mon.a (mon.0) 1696 : cluster 3 Health check failed: Reduced data availability: 3 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2023-10-01T11:02:06.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:05 smithi078 bash[78274]: cluster 2023-10-01T11:02:04.204420+0000 mgr.y (mgr.25072) 1341 : cluster 0 pgmap v1396: 89 pgs: 9 peering, 80 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-10-01T11:02:06.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:05 smithi078 bash[78274]: cluster 2023-10-01T11:02:04.727510+0000 mon.a (mon.0) 1696 : cluster 3 Health check failed: Reduced data availability: 3 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2023-10-01T11:02:06.140 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:06.174 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:06.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:06.176 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:02:06.277 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:06.305 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:06.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:06.356 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:02:06.457 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:06.490 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:06.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:06.495 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:02:06.596 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:06.624 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:06.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:06.626 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:02:06.727 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:06.760 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:06.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:06.763 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:02:06.864 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:06.891 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:06.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:06.894 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:02:06.995 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:07.026 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:07.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:07.030 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:02:07.131 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:07.162 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:07.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:07.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:02:07.266 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:07.300 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:07.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:07.303 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:02:07.387 DEBUG:teuthology.orchestra.run.smithi078:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_8 bench 90 write' 2023-10-01T11:02:07.389 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-01T11:02:07.389 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-01T11:02:07.404 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:07.430 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:07.431 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:hints = 1 2023-10-01T11:02:07.431 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-01T11:02:07.431 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Object prefix: benchmark_data_smithi078_202524 2023-10-01T11:02:07.431 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T11:02:07.431 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 0 0 0 0 0 0 - 0 2023-10-01T11:02:07.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:07.433 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:02:07.534 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:07.564 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:07.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:07.567 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:02:07.668 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:07.700 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:07.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:07.703 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:02:07.803 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:07.812 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:07 smithi173 bash[58816]: cluster 2023-10-01T11:02:06.205350+0000 mgr.y (mgr.25072) 1342 : cluster 0 pgmap v1397: 89 pgs: 9 peering, 80 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-10-01T11:02:07.829 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:07.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:07.832 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:02:07.933 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:07.960 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:07.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:07.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:02:08.064 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:08.073 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:07 smithi078 bash[75299]: cluster 2023-10-01T11:02:06.205350+0000 mgr.y (mgr.25072) 1342 : cluster 0 pgmap v1397: 89 pgs: 9 peering, 80 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-10-01T11:02:08.073 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:07 smithi078 bash[78274]: cluster 2023-10-01T11:02:06.205350+0000 mgr.y (mgr.25072) 1342 : cluster 0 pgmap v1397: 89 pgs: 9 peering, 80 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-10-01T11:02:08.089 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:08.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:08.092 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:02:08.192 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:08.217 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:08.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:08.220 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:02:08.321 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:08.362 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:08.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:08.370 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:02:08.431 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 1 16 3063 3047 190.428 190.438 0.0186285 0.00522921 2023-10-01T11:02:08.471 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:08.500 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:08.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:08.507 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:02:08.607 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:08.639 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:08.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:08.642 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:02:08.743 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:08.773 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:08.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:08.777 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:02:08.878 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:08.906 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:08.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:08.911 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:02:09.012 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:09.040 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:09.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:09.043 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:02:09.143 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:09.175 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:09.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:09.177 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:02:09.278 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:09.309 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:09.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:09.312 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:02:09.413 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:09.430 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 2 16 6009 5993 187.264 184.125 0.00229125 0.00532466 2023-10-01T11:02:09.445 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:09.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:09.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:02:09.548 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:09.578 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:09.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:09.580 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:02:09.681 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:09.709 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:09.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:09.711 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:02:09.753 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:09 smithi078 bash[75299]: cluster 2023-10-01T11:02:08.205925+0000 mgr.y (mgr.25072) 1343 : cluster 0 pgmap v1398: 89 pgs: 5 peering, 84 active+clean; 582 KiB data, 4.9 GiB used, 621 GiB / 626 GiB avail 2023-10-01T11:02:09.753 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:09 smithi078 bash[78274]: cluster 2023-10-01T11:02:08.205925+0000 mgr.y (mgr.25072) 1343 : cluster 0 pgmap v1398: 89 pgs: 5 peering, 84 active+clean; 582 KiB data, 4.9 GiB used, 621 GiB / 626 GiB avail 2023-10-01T11:02:09.812 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:09.841 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:09.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:09.844 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:02:09.945 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:09.967 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:09.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:09.969 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:02:10.070 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:10.083 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:09 smithi173 bash[58816]: cluster 2023-10-01T11:02:08.205925+0000 mgr.y (mgr.25072) 1343 : cluster 0 pgmap v1398: 89 pgs: 5 peering, 84 active+clean; 582 KiB data, 4.9 GiB used, 621 GiB / 626 GiB avail 2023-10-01T11:02:10.111 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:10.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:10.116 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:02:10.217 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:10.249 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:10.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:10.252 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:02:10.352 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:10.370 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:02:10 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:02:10] "GET /metrics HTTP/1.1" 200 36313 "" "Prometheus/2.33.4" 2023-10-01T11:02:10.384 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:10.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:10.387 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:02:10.430 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 3 16 8318 8302 172.943 144.312 0.00578025 0.00577019 2023-10-01T11:02:10.488 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:10.518 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:10.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:10.522 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:02:10.623 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:10.643 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2023-10-01T11:02:10.643 INFO:teuthology.orchestra.run.smithi078.stdout: "size": 20, 2023-10-01T11:02:10.643 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 600, 2023-10-01T11:02:10.643 INFO:teuthology.orchestra.run.smithi078.stdout: "ops": [ 2023-10-01T11:02:10.644 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.644 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46037.0:170451 11.10 11:086e41b9:::benchmark_data_smithi078_157677_object63280:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e192)", 2023-10-01T11:02:10.644 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:54:21.631789+0000", 2023-10-01T11:02:10.644 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 469.002982503, 2023-10-01T11:02:10.644 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1682773260000001, 2023-10-01T11:02:10.644 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:02:10.645 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:02:10.645 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T11:02:10.645 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46037", 2023-10-01T11:02:10.645 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T11:02:10.645 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 170451 2023-10-01T11:02:10.646 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.646 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:02:10.646 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.646 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:02:10.646 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631789+0000", 2023-10-01T11:02:10.646 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:02:10.646 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.646 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.647 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:02:10.647 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631789+0000", 2023-10-01T11:02:10.647 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:02:10.647 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.647 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.647 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:02:10.647 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631789+0000", 2023-10-01T11:02:10.648 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999995 2023-10-01T11:02:10.648 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.648 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.648 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:02:10.648 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631791+0000", 2023-10-01T11:02:10.648 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.781e-06 2023-10-01T11:02:10.648 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.649 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.649 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:02:10.649 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631794+0000", 2023-10-01T11:02:10.649 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.6399999999999999e-06 2023-10-01T11:02:10.649 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.649 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.649 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:02:10.649 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631798+0000", 2023-10-01T11:02:10.650 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.4460000000000001e-06 2023-10-01T11:02:10.650 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.650 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.650 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:02:10.650 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.632029+0000", 2023-10-01T11:02:10.650 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00023185100000000001 2023-10-01T11:02:10.650 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.650 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.650 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T11:02:10.651 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.632038+0000", 2023-10-01T11:02:10.651 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.9800000000000004e-06 2023-10-01T11:02:10.651 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.652 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.652 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:02:10.652 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.766878+0000", 2023-10-01T11:02:10.652 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.134839084 2023-10-01T11:02:10.652 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.652 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.652 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:02:10.652 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.767018+0000", 2023-10-01T11:02:10.653 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00014021199999999999 2023-10-01T11:02:10.653 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.653 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.653 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 0,2,3,6,7", 2023-10-01T11:02:10.653 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.767354+0000", 2023-10-01T11:02:10.653 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00033653600000000002 2023-10-01T11:02:10.653 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.653 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.654 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.654 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.775456+0000", 2023-10-01T11:02:10.654 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0081021900000000004 2023-10-01T11:02:10.654 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.654 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.654 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.654 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.775494+0000", 2023-10-01T11:02:10.655 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.7963999999999998e-05 2023-10-01T11:02:10.655 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.655 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.655 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.655 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.775530+0000", 2023-10-01T11:02:10.655 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.5546999999999997e-05 2023-10-01T11:02:10.655 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.656 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.656 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.656 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.786108+0000", 2023-10-01T11:02:10.656 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.010578392000000001 2023-10-01T11:02:10.656 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.656 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.656 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.656 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.799819+0000", 2023-10-01T11:02:10.657 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.013711004000000001 2023-10-01T11:02:10.657 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.657 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.657 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T11:02:10.657 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.799867+0000", 2023-10-01T11:02:10.657 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.7336000000000002e-05 2023-10-01T11:02:10.658 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.658 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.658 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:02:10.658 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.799922+0000", 2023-10-01T11:02:10.658 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.4821000000000002e-05 2023-10-01T11:02:10.658 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.658 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.659 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:02:10.659 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.800066+0000", 2023-10-01T11:02:10.659 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00014478099999999999 2023-10-01T11:02:10.659 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:02:10.659 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:02:10.659 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:02:10.660 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.660 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.660 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46037.0:170460 11.10 11:0dddcbba:::benchmark_data_smithi078_157677_object63289:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e192)", 2023-10-01T11:02:10.660 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:54:21.631800+0000", 2023-10-01T11:02:10.660 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 469.002971527, 2023-10-01T11:02:10.660 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1686122050000001, 2023-10-01T11:02:10.660 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:02:10.660 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:02:10.660 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T11:02:10.660 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46037", 2023-10-01T11:02:10.661 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T11:02:10.661 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 170460 2023-10-01T11:02:10.661 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.661 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:02:10.661 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.661 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:02:10.661 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631800+0000", 2023-10-01T11:02:10.661 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:02:10.661 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.662 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.662 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:02:10.662 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631800+0000", 2023-10-01T11:02:10.662 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:02:10.662 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.662 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.662 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:02:10.663 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631800+0000", 2023-10-01T11:02:10.663 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967296 2023-10-01T11:02:10.663 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.663 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.663 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:02:10.663 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631801+0000", 2023-10-01T11:02:10.663 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.4389999999999999e-06 2023-10-01T11:02:10.663 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.664 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.664 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:02:10.664 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631804+0000", 2023-10-01T11:02:10.664 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.202e-06 2023-10-01T11:02:10.664 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.664 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.664 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:02:10.664 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631807+0000", 2023-10-01T11:02:10.664 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.0000000000000001e-06 2023-10-01T11:02:10.664 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.665 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.665 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:02:10.665 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.632065+0000", 2023-10-01T11:02:10.665 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000257667 2023-10-01T11:02:10.665 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.665 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.665 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T11:02:10.665 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.632069+0000", 2023-10-01T11:02:10.665 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.1200000000000004e-06 2023-10-01T11:02:10.666 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.666 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.666 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:02:10.666 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.769227+0000", 2023-10-01T11:02:10.666 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1371582929999999 2023-10-01T11:02:10.666 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.666 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.667 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:02:10.667 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.769401+0000", 2023-10-01T11:02:10.667 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000173303 2023-10-01T11:02:10.667 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.667 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.667 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 0,2,3,6,7", 2023-10-01T11:02:10.667 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.769674+0000", 2023-10-01T11:02:10.667 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00027319600000000002 2023-10-01T11:02:10.667 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.668 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.668 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.668 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.775569+0000", 2023-10-01T11:02:10.668 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0058948639999999997 2023-10-01T11:02:10.668 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.668 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.670 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.670 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.786075+0000", 2023-10-01T11:02:10.670 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.010505782 2023-10-01T11:02:10.670 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.670 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.671 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.671 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.786209+0000", 2023-10-01T11:02:10.671 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00013463200000000001 2023-10-01T11:02:10.672 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.672 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.672 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.672 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.788659+0000", 2023-10-01T11:02:10.672 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.002449522 2023-10-01T11:02:10.672 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.673 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.673 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T11:02:10.673 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.800076+0000", 2023-10-01T11:02:10.673 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.011417437000000001 2023-10-01T11:02:10.673 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.673 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.674 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.674 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.800226+0000", 2023-10-01T11:02:10.674 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00015015100000000001 2023-10-01T11:02:10.674 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.674 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.674 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:02:10.674 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.800273+0000", 2023-10-01T11:02:10.675 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.6693000000000002e-05 2023-10-01T11:02:10.675 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.675 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.675 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:02:10.675 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.800412+0000", 2023-10-01T11:02:10.675 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000139135 2023-10-01T11:02:10.675 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:02:10.676 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:02:10.676 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:02:10.676 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.676 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.676 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46037.0:170498 11.10 11:0970561e:::benchmark_data_smithi078_157677_object63327:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e192)", 2023-10-01T11:02:10.676 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:54:21.631810+0000", 2023-10-01T11:02:10.677 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 469.00296183, 2023-10-01T11:02:10.677 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.172262259, 2023-10-01T11:02:10.677 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:02:10.677 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:02:10.677 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T11:02:10.677 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46037", 2023-10-01T11:02:10.677 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T11:02:10.677 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 170498 2023-10-01T11:02:10.678 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.678 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:02:10.678 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.678 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:02:10.678 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631810+0000", 2023-10-01T11:02:10.679 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:02:10.679 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.679 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.679 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:02:10.680 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631810+0000", 2023-10-01T11:02:10.680 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:02:10.680 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.680 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.680 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:02:10.680 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631809+0000", 2023-10-01T11:02:10.680 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967296 2023-10-01T11:02:10.681 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.681 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.681 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:02:10.681 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631811+0000", 2023-10-01T11:02:10.681 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.5209999999999999e-06 2023-10-01T11:02:10.681 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.682 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.682 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:02:10.682 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631814+0000", 2023-10-01T11:02:10.682 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.157e-06 2023-10-01T11:02:10.682 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.682 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.683 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:02:10.683 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631818+0000", 2023-10-01T11:02:10.683 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.8670000000000004e-06 2023-10-01T11:02:10.683 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.683 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.683 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:02:10.683 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.632086+0000", 2023-10-01T11:02:10.684 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00026807000000000003 2023-10-01T11:02:10.684 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.684 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.684 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T11:02:10.684 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.632090+0000", 2023-10-01T11:02:10.684 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.1729999999999997e-06 2023-10-01T11:02:10.685 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.685 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.685 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:02:10.685 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.771382+0000", 2023-10-01T11:02:10.685 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1392917570000001 2023-10-01T11:02:10.685 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.686 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.686 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:02:10.686 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.771510+0000", 2023-10-01T11:02:10.686 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00012846299999999999 2023-10-01T11:02:10.686 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.686 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.686 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 0,2,3,6,7", 2023-10-01T11:02:10.686 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.771772+0000", 2023-10-01T11:02:10.687 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00026122299999999999 2023-10-01T11:02:10.687 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.687 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.687 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.687 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.786038+0000", 2023-10-01T11:02:10.687 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.014266778000000001 2023-10-01T11:02:10.688 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.688 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.688 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.688 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.786177+0000", 2023-10-01T11:02:10.688 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00013871900000000001 2023-10-01T11:02:10.688 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.688 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.689 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.689 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.786307+0000", 2023-10-01T11:02:10.689 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00012935900000000001 2023-10-01T11:02:10.689 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.689 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.689 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.690 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.788691+0000", 2023-10-01T11:02:10.690 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.002384803 2023-10-01T11:02:10.690 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.691 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.691 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T11:02:10.691 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.800083+0000", 2023-10-01T11:02:10.691 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.011391434000000001 2023-10-01T11:02:10.691 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.691 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.692 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.692 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.803877+0000", 2023-10-01T11:02:10.692 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0037939190000000002 2023-10-01T11:02:10.692 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.692 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.692 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:02:10.692 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.803945+0000", 2023-10-01T11:02:10.693 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.8113000000000003e-05 2023-10-01T11:02:10.693 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.693 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.693 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:02:10.693 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.804072+0000", 2023-10-01T11:02:10.693 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000127133 2023-10-01T11:02:10.693 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:02:10.694 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:02:10.694 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:02:10.694 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.694 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.694 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46037.0:170504 11.10 11:0d37d827:::benchmark_data_smithi078_157677_object63333:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e192)", 2023-10-01T11:02:10.694 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:54:21.631822+0000", 2023-10-01T11:02:10.694 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 469.002949446, 2023-10-01T11:02:10.695 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1725379440000001, 2023-10-01T11:02:10.695 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:02:10.695 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:02:10.695 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T11:02:10.695 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46037", 2023-10-01T11:02:10.695 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T11:02:10.695 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 170504 2023-10-01T11:02:10.696 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.696 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:02:10.696 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.696 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:02:10.696 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631822+0000", 2023-10-01T11:02:10.696 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:02:10.696 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.697 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.697 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:02:10.697 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631822+0000", 2023-10-01T11:02:10.697 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:02:10.697 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.697 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.697 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:02:10.698 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631822+0000", 2023-10-01T11:02:10.698 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967296 2023-10-01T11:02:10.698 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.698 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.698 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:02:10.698 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631827+0000", 2023-10-01T11:02:10.698 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.7770000000000002e-06 2023-10-01T11:02:10.699 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.699 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.699 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:02:10.699 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631831+0000", 2023-10-01T11:02:10.699 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.9530000000000001e-06 2023-10-01T11:02:10.699 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.699 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.699 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:02:10.700 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631834+0000", 2023-10-01T11:02:10.700 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.2619999999999999e-06 2023-10-01T11:02:10.700 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.701 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.701 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:02:10.701 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.632122+0000", 2023-10-01T11:02:10.701 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000288354 2023-10-01T11:02:10.701 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.701 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.701 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T11:02:10.701 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.632131+0000", 2023-10-01T11:02:10.701 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.2259999999999994e-06 2023-10-01T11:02:10.701 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.702 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.702 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:02:10.702 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.773447+0000", 2023-10-01T11:02:10.702 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.141315662 2023-10-01T11:02:10.702 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.702 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.702 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:02:10.702 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.773574+0000", 2023-10-01T11:02:10.702 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00012720599999999999 2023-10-01T11:02:10.702 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.702 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.702 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 0,2,3,6,7", 2023-10-01T11:02:10.703 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.773798+0000", 2023-10-01T11:02:10.703 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000223594 2023-10-01T11:02:10.703 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.703 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.703 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.703 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.786141+0000", 2023-10-01T11:02:10.703 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.012343394000000001 2023-10-01T11:02:10.703 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.703 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.703 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.703 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.786274+0000", 2023-10-01T11:02:10.703 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00013266600000000001 2023-10-01T11:02:10.704 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.704 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.704 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.704 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.786403+0000", 2023-10-01T11:02:10.704 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00012877099999999999 2023-10-01T11:02:10.704 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.704 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.704 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.704 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.788723+0000", 2023-10-01T11:02:10.704 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.002320707 2023-10-01T11:02:10.704 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.704 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.705 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T11:02:10.705 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.800089+0000", 2023-10-01T11:02:10.705 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.011366029 2023-10-01T11:02:10.705 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.705 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.705 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.705 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.804150+0000", 2023-10-01T11:02:10.705 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.004060124 2023-10-01T11:02:10.706 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.706 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.706 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:02:10.706 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.804208+0000", 2023-10-01T11:02:10.706 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.8869000000000002e-05 2023-10-01T11:02:10.706 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.706 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.706 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:02:10.706 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.804360+0000", 2023-10-01T11:02:10.706 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00015158000000000001 2023-10-01T11:02:10.707 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:02:10.707 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:02:10.707 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:02:10.707 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.707 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.707 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46037.0:170514 11.10 11:0a306d56:::benchmark_data_smithi078_157677_object63343:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e192)", 2023-10-01T11:02:10.707 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:54:21.631836+0000", 2023-10-01T11:02:10.707 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 469.00293522300001, 2023-10-01T11:02:10.707 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.172768504, 2023-10-01T11:02:10.707 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:02:10.708 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:02:10.708 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T11:02:10.708 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46037", 2023-10-01T11:02:10.708 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T11:02:10.708 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 170514 2023-10-01T11:02:10.708 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.708 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:02:10.708 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.708 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:02:10.708 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631836+0000", 2023-10-01T11:02:10.709 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:02:10.709 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.709 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.709 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:02:10.709 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631836+0000", 2023-10-01T11:02:10.709 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:02:10.709 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.709 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.709 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:02:10.709 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631836+0000", 2023-10-01T11:02:10.710 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967296 2023-10-01T11:02:10.710 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.710 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.710 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:02:10.710 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631838+0000", 2023-10-01T11:02:10.710 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.5579999999999999e-06 2023-10-01T11:02:10.710 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.710 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.711 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:02:10.711 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631842+0000", 2023-10-01T11:02:10.711 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.1099999999999996e-06 2023-10-01T11:02:10.711 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.711 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.711 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:02:10.711 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631845+0000", 2023-10-01T11:02:10.712 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.0920000000000002e-06 2023-10-01T11:02:10.712 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.712 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.712 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:02:10.712 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.632159+0000", 2023-10-01T11:02:10.712 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00031405499999999999 2023-10-01T11:02:10.712 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.712 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.712 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T11:02:10.712 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.632163+0000", 2023-10-01T11:02:10.713 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.924e-06 2023-10-01T11:02:10.713 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.713 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.713 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:02:10.713 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.775595+0000", 2023-10-01T11:02:10.713 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1434320099999999 2023-10-01T11:02:10.713 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.713 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.713 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:02:10.713 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.775723+0000", 2023-10-01T11:02:10.714 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00012839399999999999 2023-10-01T11:02:10.714 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.714 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.714 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 0,2,3,6,7", 2023-10-01T11:02:10.714 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.775945+0000", 2023-10-01T11:02:10.714 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00022158099999999999 2023-10-01T11:02:10.714 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.714 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.714 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.714 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.786243+0000", 2023-10-01T11:02:10.714 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.010297763999999999 2023-10-01T11:02:10.715 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.715 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.715 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.715 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.786371+0000", 2023-10-01T11:02:10.717 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00012810100000000001 2023-10-01T11:02:10.717 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.717 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.717 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.717 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.788759+0000", 2023-10-01T11:02:10.717 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0023886739999999999 2023-10-01T11:02:10.717 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.717 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.717 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T11:02:10.717 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.800095+0000", 2023-10-01T11:02:10.718 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.011336112000000001 2023-10-01T11:02:10.718 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.718 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.718 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.718 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.800193+0000", 2023-10-01T11:02:10.718 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.8064000000000002e-05 2023-10-01T11:02:10.718 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.718 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.718 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.718 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.804417+0000", 2023-10-01T11:02:10.718 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0042237170000000001 2023-10-01T11:02:10.718 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.718 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.719 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:02:10.719 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.804472+0000", 2023-10-01T11:02:10.719 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.4425999999999997e-05 2023-10-01T11:02:10.719 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.719 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.719 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:02:10.719 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.804605+0000", 2023-10-01T11:02:10.719 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000133156 2023-10-01T11:02:10.719 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:02:10.719 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:02:10.719 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:02:10.719 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.719 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.720 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46037.0:170522 11.10 11:0d343739:::benchmark_data_smithi078_157677_object63351:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e192)", 2023-10-01T11:02:10.720 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:54:21.631855+0000", 2023-10-01T11:02:10.720 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 469.00291602499999, 2023-10-01T11:02:10.720 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1730097100000001, 2023-10-01T11:02:10.721 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:02:10.722 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:02:10.722 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T11:02:10.722 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46037", 2023-10-01T11:02:10.722 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T11:02:10.722 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 170522 2023-10-01T11:02:10.722 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.722 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:02:10.723 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.723 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:02:10.723 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631855+0000", 2023-10-01T11:02:10.723 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:02:10.723 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.723 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.723 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:02:10.723 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631855+0000", 2023-10-01T11:02:10.723 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:02:10.723 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.724 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.724 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:02:10.724 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631855+0000", 2023-10-01T11:02:10.724 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999995 2023-10-01T11:02:10.724 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.724 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.724 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:02:10.724 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631857+0000", 2023-10-01T11:02:10.724 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.7689999999999999e-06 2023-10-01T11:02:10.724 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.725 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.725 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:02:10.725 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631861+0000", 2023-10-01T11:02:10.725 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.6969999999999999e-06 2023-10-01T11:02:10.725 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.725 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.726 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:02:10.726 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631864+0000", 2023-10-01T11:02:10.726 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.1020000000000001e-06 2023-10-01T11:02:10.726 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.726 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.726 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:02:10.726 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.632194+0000", 2023-10-01T11:02:10.726 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000330458 2023-10-01T11:02:10.726 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.726 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.727 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T11:02:10.727 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.632205+0000", 2023-10-01T11:02:10.727 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.0445000000000001e-05 2023-10-01T11:02:10.727 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.727 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.727 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:02:10.727 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.777248+0000", 2023-10-01T11:02:10.727 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1450429390000001 2023-10-01T11:02:10.727 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.727 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.727 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:02:10.727 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.777397+0000", 2023-10-01T11:02:10.728 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00014985700000000001 2023-10-01T11:02:10.728 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.728 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.728 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 0,2,3,6,7", 2023-10-01T11:02:10.728 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.777606+0000", 2023-10-01T11:02:10.728 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000208877 2023-10-01T11:02:10.728 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.728 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.728 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.728 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.786339+0000", 2023-10-01T11:02:10.728 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0087329869999999993 2023-10-01T11:02:10.729 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.729 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.729 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.729 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.786441+0000", 2023-10-01T11:02:10.729 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00010124399999999999 2023-10-01T11:02:10.729 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.729 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.729 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.729 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.799857+0000", 2023-10-01T11:02:10.730 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.013416328 2023-10-01T11:02:10.730 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.730 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.730 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T11:02:10.730 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.800102+0000", 2023-10-01T11:02:10.730 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00024478400000000001 2023-10-01T11:02:10.730 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.730 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.730 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.730 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.800499+0000", 2023-10-01T11:02:10.731 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000397437 2023-10-01T11:02:10.731 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.731 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.731 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.731 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.804664+0000", 2023-10-01T11:02:10.731 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0041646310000000002 2023-10-01T11:02:10.731 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.731 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.731 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:02:10.731 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.804727+0000", 2023-10-01T11:02:10.731 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.3121999999999996e-05 2023-10-01T11:02:10.732 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.732 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.732 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:02:10.732 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.804865+0000", 2023-10-01T11:02:10.732 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00013831499999999999 2023-10-01T11:02:10.732 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:02:10.732 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:02:10.732 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:02:10.732 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.732 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.732 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46037.0:170605 11.10 11:0843b868:::benchmark_data_smithi078_157677_object63434:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e192)", 2023-10-01T11:02:10.733 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:54:21.631874+0000", 2023-10-01T11:02:10.733 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 469.00289799299998, 2023-10-01T11:02:10.733 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1732039730000001, 2023-10-01T11:02:10.733 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:02:10.733 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:02:10.733 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T11:02:10.733 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46037", 2023-10-01T11:02:10.733 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T11:02:10.733 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 170605 2023-10-01T11:02:10.733 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.733 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:02:10.734 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.734 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:02:10.734 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631874+0000", 2023-10-01T11:02:10.734 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:02:10.734 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.734 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.734 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:02:10.734 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631874+0000", 2023-10-01T11:02:10.734 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:02:10.734 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.735 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.735 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:02:10.735 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631873+0000", 2023-10-01T11:02:10.735 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999995 2023-10-01T11:02:10.735 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.735 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.735 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:02:10.736 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631875+0000", 2023-10-01T11:02:10.736 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.7719999999999999e-06 2023-10-01T11:02:10.736 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.736 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.736 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:02:10.736 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631879+0000", 2023-10-01T11:02:10.736 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.9020000000000001e-06 2023-10-01T11:02:10.736 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.736 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.736 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:02:10.736 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631882+0000", 2023-10-01T11:02:10.737 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.3009999999999998e-06 2023-10-01T11:02:10.737 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.737 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.737 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:02:10.737 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.632232+0000", 2023-10-01T11:02:10.737 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00034966799999999997 2023-10-01T11:02:10.737 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.737 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.737 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T11:02:10.737 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.632236+0000", 2023-10-01T11:02:10.737 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.0520000000000003e-06 2023-10-01T11:02:10.738 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.738 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.738 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:02:10.738 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.783536+0000", 2023-10-01T11:02:10.738 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1512997519999999 2023-10-01T11:02:10.738 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.738 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.738 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:02:10.738 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.783777+0000", 2023-10-01T11:02:10.738 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00024158 2023-10-01T11:02:10.739 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.739 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.739 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 0,2,3,6,7", 2023-10-01T11:02:10.739 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.784109+0000", 2023-10-01T11:02:10.739 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000331887 2023-10-01T11:02:10.739 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.739 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.739 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.739 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.788592+0000", 2023-10-01T11:02:10.739 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0044826980000000002 2023-10-01T11:02:10.740 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.740 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.740 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.740 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.788626+0000", 2023-10-01T11:02:10.740 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.3733999999999999e-05 2023-10-01T11:02:10.740 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.740 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.740 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T11:02:10.740 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.800108+0000", 2023-10-01T11:02:10.740 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.011482137999999999 2023-10-01T11:02:10.741 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.741 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.741 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.741 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.800461+0000", 2023-10-01T11:02:10.741 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00035373000000000002 2023-10-01T11:02:10.741 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.741 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.741 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.741 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.800576+0000", 2023-10-01T11:02:10.741 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00011446399999999999 2023-10-01T11:02:10.742 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.742 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.742 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.742 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.804906+0000", 2023-10-01T11:02:10.742 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0043302549999999999 2023-10-01T11:02:10.742 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.742 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.742 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:02:10.742 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.804951+0000", 2023-10-01T11:02:10.742 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.4369000000000002e-05 2023-10-01T11:02:10.742 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.743 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.743 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:02:10.743 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.805077+0000", 2023-10-01T11:02:10.743 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00012697300000000001 2023-10-01T11:02:10.743 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:02:10.743 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:02:10.743 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:02:10.743 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.743 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.743 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46037.0:170611 11.10 11:0fb7c3db:::benchmark_data_smithi078_157677_object63440:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e192)", 2023-10-01T11:02:10.744 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:54:21.631885+0000", 2023-10-01T11:02:10.744 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 469.00288675100001, 2023-10-01T11:02:10.744 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.17341871, 2023-10-01T11:02:10.744 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:02:10.744 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:02:10.744 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T11:02:10.744 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46037", 2023-10-01T11:02:10.744 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T11:02:10.744 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 170611 2023-10-01T11:02:10.744 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.745 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:02:10.745 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.745 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:02:10.745 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631885+0000", 2023-10-01T11:02:10.745 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:02:10.745 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.745 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.745 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:02:10.745 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631885+0000", 2023-10-01T11:02:10.745 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:02:10.746 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.746 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.746 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:02:10.746 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631885+0000", 2023-10-01T11:02:10.746 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967296 2023-10-01T11:02:10.746 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.746 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.746 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:02:10.747 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631886+0000", 2023-10-01T11:02:10.747 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.6899999999999999e-06 2023-10-01T11:02:10.747 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.747 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.747 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:02:10.747 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631889+0000", 2023-10-01T11:02:10.747 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.2040000000000002e-06 2023-10-01T11:02:10.747 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.747 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.747 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:02:10.747 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.631892+0000", 2023-10-01T11:02:10.748 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.9280000000000002e-06 2023-10-01T11:02:10.748 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.748 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.748 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:02:10.748 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.632264+0000", 2023-10-01T11:02:10.748 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00037162000000000002 2023-10-01T11:02:10.748 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.748 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.748 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T11:02:10.748 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:21.632273+0000", 2023-10-01T11:02:10.749 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.4080000000000004e-06 2023-10-01T11:02:10.749 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.749 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.749 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:02:10.749 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.786463+0000", 2023-10-01T11:02:10.749 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1541899019999999 2023-10-01T11:02:10.749 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.749 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.749 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:02:10.749 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.786581+0000", 2023-10-01T11:02:10.749 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000117694 2023-10-01T11:02:10.750 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.750 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.750 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 0,2,3,6,7", 2023-10-01T11:02:10.750 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.786866+0000", 2023-10-01T11:02:10.750 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000285376 2023-10-01T11:02:10.750 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.750 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.750 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.750 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.799766+0000", 2023-10-01T11:02:10.750 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.012899476999999999 2023-10-01T11:02:10.751 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.751 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.751 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T11:02:10.751 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.800114+0000", 2023-10-01T11:02:10.751 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00034815600000000002 2023-10-01T11:02:10.751 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.751 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.751 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.751 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.800539+0000", 2023-10-01T11:02:10.752 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00042501399999999998 2023-10-01T11:02:10.752 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.752 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.752 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.752 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.800612+0000", 2023-10-01T11:02:10.752 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.2849999999999995e-05 2023-10-01T11:02:10.752 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.752 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.752 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.752 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.800691+0000", 2023-10-01T11:02:10.753 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.9516000000000002e-05 2023-10-01T11:02:10.753 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.753 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.753 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.753 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.805116+0000", 2023-10-01T11:02:10.753 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0044245389999999999 2023-10-01T11:02:10.753 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.753 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.753 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:02:10.753 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.805158+0000", 2023-10-01T11:02:10.753 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.2015999999999999e-05 2023-10-01T11:02:10.754 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.754 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.754 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:02:10.754 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:22.805303+0000", 2023-10-01T11:02:10.754 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00014555300000000001 2023-10-01T11:02:10.754 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:02:10.754 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:02:10.754 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:02:10.754 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.755 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.755 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46037.0:176163 11.13 11:c8061bf6:::benchmark_data_smithi078_157677_object68992:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e199)", 2023-10-01T11:02:10.755 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:54:33.553395+0000", 2023-10-01T11:02:10.755 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 457.08137634100001, 2023-10-01T11:02:10.755 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.531561218, 2023-10-01T11:02:10.755 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:02:10.755 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:02:10.755 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T11:02:10.755 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46037", 2023-10-01T11:02:10.755 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T11:02:10.755 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 176163 2023-10-01T11:02:10.756 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.756 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:02:10.756 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.756 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:02:10.756 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553395+0000", 2023-10-01T11:02:10.756 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:02:10.756 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.757 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.757 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:02:10.757 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553395+0000", 2023-10-01T11:02:10.757 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:02:10.757 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.757 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.757 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:02:10.757 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553393+0000", 2023-10-01T11:02:10.757 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999976 2023-10-01T11:02:10.757 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.757 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.757 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:02:10.758 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553403+0000", 2023-10-01T11:02:10.758 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.0597999999999999e-05 2023-10-01T11:02:10.758 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.758 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.758 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:02:10.758 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553421+0000", 2023-10-01T11:02:10.758 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.7847999999999999e-05 2023-10-01T11:02:10.758 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.758 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.758 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:02:10.758 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553434+0000", 2023-10-01T11:02:10.758 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.2746e-05 2023-10-01T11:02:10.759 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.759 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.759 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:02:10.759 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.556358+0000", 2023-10-01T11:02:10.759 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0029241969999999999 2023-10-01T11:02:10.759 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.759 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.759 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T11:02:10.759 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.556366+0000", 2023-10-01T11:02:10.759 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.0260000000000007e-06 2023-10-01T11:02:10.759 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.759 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.760 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:02:10.760 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.070523+0000", 2023-10-01T11:02:10.760 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.514156469 2023-10-01T11:02:10.760 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.760 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.760 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:02:10.760 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.070672+0000", 2023-10-01T11:02:10.760 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000149157 2023-10-01T11:02:10.760 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.760 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.760 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 0,4,5,6,7", 2023-10-01T11:02:10.760 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.071071+0000", 2023-10-01T11:02:10.761 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00039965299999999998 2023-10-01T11:02:10.761 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.761 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.761 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T11:02:10.761 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.075827+0000", 2023-10-01T11:02:10.761 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0047560390000000001 2023-10-01T11:02:10.761 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.761 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.761 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.761 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.078651+0000", 2023-10-01T11:02:10.761 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0028232510000000001 2023-10-01T11:02:10.761 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.761 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.762 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.762 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.078688+0000", 2023-10-01T11:02:10.762 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.7246000000000002e-05 2023-10-01T11:02:10.762 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.762 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.762 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.762 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.078755+0000", 2023-10-01T11:02:10.762 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.7024000000000001e-05 2023-10-01T11:02:10.762 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.762 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.762 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.763 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.078787+0000", 2023-10-01T11:02:10.763 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.1642e-05 2023-10-01T11:02:10.763 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.763 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.763 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.763 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.084731+0000", 2023-10-01T11:02:10.763 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0059444759999999998 2023-10-01T11:02:10.763 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.763 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.763 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:02:10.763 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.084799+0000", 2023-10-01T11:02:10.764 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.7883e-05 2023-10-01T11:02:10.764 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.764 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.764 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:02:10.764 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.084956+0000", 2023-10-01T11:02:10.764 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000157472 2023-10-01T11:02:10.764 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:02:10.764 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:02:10.764 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:02:10.764 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.764 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.765 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46037.0:176170 11.13 11:cc11c237:::benchmark_data_smithi078_157677_object68999:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e199)", 2023-10-01T11:02:10.765 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:54:33.553478+0000", 2023-10-01T11:02:10.765 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 457.081293475, 2023-10-01T11:02:10.765 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.531709996, 2023-10-01T11:02:10.765 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:02:10.765 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:02:10.765 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T11:02:10.765 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46037", 2023-10-01T11:02:10.765 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T11:02:10.765 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 176170 2023-10-01T11:02:10.765 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.765 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:02:10.766 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.766 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:02:10.766 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553478+0000", 2023-10-01T11:02:10.766 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:02:10.766 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.766 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.767 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:02:10.767 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553478+0000", 2023-10-01T11:02:10.767 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:02:10.767 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.767 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.767 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:02:10.767 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553477+0000", 2023-10-01T11:02:10.767 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999995 2023-10-01T11:02:10.767 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.767 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.767 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:02:10.767 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553481+0000", 2023-10-01T11:02:10.768 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.9639999999999997e-06 2023-10-01T11:02:10.768 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.768 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.768 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:02:10.768 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553490+0000", 2023-10-01T11:02:10.768 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.6149999999999997e-06 2023-10-01T11:02:10.768 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.768 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.768 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:02:10.768 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553495+0000", 2023-10-01T11:02:10.768 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.113e-06 2023-10-01T11:02:10.768 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.769 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.769 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:02:10.769 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.556534+0000", 2023-10-01T11:02:10.769 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.003038443 2023-10-01T11:02:10.769 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.769 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.769 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T11:02:10.769 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.556541+0000", 2023-10-01T11:02:10.769 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.7039999999999994e-06 2023-10-01T11:02:10.769 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.769 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.769 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:02:10.770 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.073377+0000", 2023-10-01T11:02:10.770 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.5168358629999998 2023-10-01T11:02:10.770 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.770 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.770 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:02:10.770 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.073520+0000", 2023-10-01T11:02:10.770 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00014291400000000001 2023-10-01T11:02:10.770 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.770 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.770 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 0,4,5,6,7", 2023-10-01T11:02:10.770 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.073795+0000", 2023-10-01T11:02:10.770 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00027470199999999999 2023-10-01T11:02:10.771 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.771 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.771 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.771 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.078722+0000", 2023-10-01T11:02:10.771 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0049276019999999997 2023-10-01T11:02:10.771 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.771 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.771 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T11:02:10.771 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.084525+0000", 2023-10-01T11:02:10.771 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0058027829999999997 2023-10-01T11:02:10.771 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.771 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.772 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.772 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.084569+0000", 2023-10-01T11:02:10.772 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.3498000000000003e-05 2023-10-01T11:02:10.772 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.772 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.772 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.772 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.084644+0000", 2023-10-01T11:02:10.772 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.5512999999999993e-05 2023-10-01T11:02:10.772 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.772 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.772 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.772 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.084684+0000", 2023-10-01T11:02:10.773 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.0086000000000002e-05 2023-10-01T11:02:10.773 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.773 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.773 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.773 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.084998+0000", 2023-10-01T11:02:10.773 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000314235 2023-10-01T11:02:10.773 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.773 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.773 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:02:10.773 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.085050+0000", 2023-10-01T11:02:10.773 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.1100999999999997e-05 2023-10-01T11:02:10.773 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.773 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.774 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:02:10.774 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.085188+0000", 2023-10-01T11:02:10.774 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00013848300000000001 2023-10-01T11:02:10.774 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:02:10.774 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:02:10.774 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:02:10.774 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.774 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.774 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46037.0:176346 11.13 11:c9e6b427:::benchmark_data_smithi078_157677_object69175:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e199)", 2023-10-01T11:02:10.774 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:54:33.553500+0000", 2023-10-01T11:02:10.774 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 457.08127184400001, 2023-10-01T11:02:10.774 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.532346797, 2023-10-01T11:02:10.775 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:02:10.775 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:02:10.775 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T11:02:10.775 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46037", 2023-10-01T11:02:10.775 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T11:02:10.775 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 176346 2023-10-01T11:02:10.775 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.775 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:02:10.775 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.775 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:02:10.775 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553500+0000", 2023-10-01T11:02:10.776 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:02:10.776 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.776 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.776 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:02:10.776 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553500+0000", 2023-10-01T11:02:10.776 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:02:10.776 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.776 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.776 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:02:10.777 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553499+0000", 2023-10-01T11:02:10.777 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999995 2023-10-01T11:02:10.777 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.778 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.778 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:02:10.778 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553501+0000", 2023-10-01T11:02:10.778 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.1160000000000002e-06 2023-10-01T11:02:10.778 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.778 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.778 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:02:10.778 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553506+0000", 2023-10-01T11:02:10.778 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.3449999999999999e-06 2023-10-01T11:02:10.778 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.778 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.779 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:02:10.779 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553510+0000", 2023-10-01T11:02:10.779 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.9929999999999997e-06 2023-10-01T11:02:10.779 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.779 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.779 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:02:10.779 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.556567+0000", 2023-10-01T11:02:10.779 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0030572609999999999 2023-10-01T11:02:10.779 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.779 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.779 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T11:02:10.779 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.556575+0000", 2023-10-01T11:02:10.780 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.4390000000000003e-06 2023-10-01T11:02:10.780 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.780 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.780 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:02:10.780 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.075881+0000", 2023-10-01T11:02:10.780 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.5193060630000002 2023-10-01T11:02:10.780 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.780 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.780 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:02:10.780 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.076054+0000", 2023-10-01T11:02:10.780 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00017294500000000001 2023-10-01T11:02:10.781 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.781 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.781 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 0,4,5,6,7", 2023-10-01T11:02:10.781 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.076458+0000", 2023-10-01T11:02:10.781 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000404134 2023-10-01T11:02:10.781 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.781 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.781 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.781 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.084504+0000", 2023-10-01T11:02:10.781 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0080465829999999995 2023-10-01T11:02:10.781 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.781 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.782 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T11:02:10.782 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.084535+0000", 2023-10-01T11:02:10.782 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.0889999999999997e-05 2023-10-01T11:02:10.782 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.782 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.782 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.782 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.085233+0000", 2023-10-01T11:02:10.782 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00069764899999999995 2023-10-01T11:02:10.782 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.782 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.782 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.782 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.085431+0000", 2023-10-01T11:02:10.782 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00019848500000000001 2023-10-01T11:02:10.783 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.783 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.783 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.783 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.085519+0000", 2023-10-01T11:02:10.783 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.7348999999999993e-05 2023-10-01T11:02:10.783 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.783 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.783 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.783 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.085655+0000", 2023-10-01T11:02:10.783 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000135986 2023-10-01T11:02:10.783 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.783 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.784 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:02:10.784 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.085699+0000", 2023-10-01T11:02:10.784 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.4536999999999997e-05 2023-10-01T11:02:10.784 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.784 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.784 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:02:10.784 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.085846+0000", 2023-10-01T11:02:10.784 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000147331 2023-10-01T11:02:10.784 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:02:10.784 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:02:10.784 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:02:10.784 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.785 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.785 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46037.0:176347 11.13 11:cfa0b275:::benchmark_data_smithi078_157677_object69176:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e199)", 2023-10-01T11:02:10.785 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:54:33.553513+0000", 2023-10-01T11:02:10.785 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 457.08125860600001, 2023-10-01T11:02:10.785 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.53910002, 2023-10-01T11:02:10.785 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:02:10.785 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:02:10.785 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T11:02:10.785 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46037", 2023-10-01T11:02:10.785 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T11:02:10.785 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 176347 2023-10-01T11:02:10.785 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.786 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:02:10.786 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.786 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:02:10.786 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553513+0000", 2023-10-01T11:02:10.786 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:02:10.786 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.786 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.786 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:02:10.786 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553513+0000", 2023-10-01T11:02:10.786 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:02:10.786 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.786 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.787 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:02:10.787 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553513+0000", 2023-10-01T11:02:10.787 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999995 2023-10-01T11:02:10.787 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.787 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.787 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:02:10.787 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553514+0000", 2023-10-01T11:02:10.787 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.818e-06 2023-10-01T11:02:10.788 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.788 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.788 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:02:10.788 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553518+0000", 2023-10-01T11:02:10.788 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.7799999999999998e-06 2023-10-01T11:02:10.788 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.788 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.788 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:02:10.788 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553523+0000", 2023-10-01T11:02:10.788 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.5129999999999998e-06 2023-10-01T11:02:10.789 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.789 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.789 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:02:10.789 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.556600+0000", 2023-10-01T11:02:10.789 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0030775009999999998 2023-10-01T11:02:10.789 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.789 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.789 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T11:02:10.789 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.556607+0000", 2023-10-01T11:02:10.789 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.934e-06 2023-10-01T11:02:10.789 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.789 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.789 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:02:10.790 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.079062+0000", 2023-10-01T11:02:10.790 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.522454889 2023-10-01T11:02:10.790 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.790 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.790 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:02:10.790 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.079267+0000", 2023-10-01T11:02:10.790 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000205365 2023-10-01T11:02:10.790 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.790 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.790 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 0,4,5,6,7", 2023-10-01T11:02:10.790 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.079684+0000", 2023-10-01T11:02:10.790 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00041616599999999999 2023-10-01T11:02:10.791 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.791 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.791 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T11:02:10.791 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.084582+0000", 2023-10-01T11:02:10.791 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0048984450000000004 2023-10-01T11:02:10.791 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.791 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.791 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.791 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.085484+0000", 2023-10-01T11:02:10.791 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00090182999999999995 2023-10-01T11:02:10.791 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.791 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.791 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.792 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.085591+0000", 2023-10-01T11:02:10.792 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000107333 2023-10-01T11:02:10.792 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.792 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.792 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.792 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.085624+0000", 2023-10-01T11:02:10.792 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.2836000000000002e-05 2023-10-01T11:02:10.792 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.792 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.792 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.792 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.088522+0000", 2023-10-01T11:02:10.792 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0028975440000000002 2023-10-01T11:02:10.792 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.793 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.793 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.793 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.092422+0000", 2023-10-01T11:02:10.793 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0039003570000000001 2023-10-01T11:02:10.793 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.793 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.793 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:02:10.793 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.092476+0000", 2023-10-01T11:02:10.793 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.3724000000000003e-05 2023-10-01T11:02:10.793 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.793 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.794 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:02:10.794 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.092613+0000", 2023-10-01T11:02:10.794 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000137253 2023-10-01T11:02:10.794 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:02:10.794 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:02:10.794 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:02:10.794 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.794 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.794 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46037.0:176390 11.13 11:cf928ba3:::benchmark_data_smithi078_157677_object69219:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e199)", 2023-10-01T11:02:10.794 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:54:33.553526+0000", 2023-10-01T11:02:10.794 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 457.08124572000003, 2023-10-01T11:02:10.794 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.539296327, 2023-10-01T11:02:10.795 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:02:10.795 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:02:10.795 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T11:02:10.795 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46037", 2023-10-01T11:02:10.795 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T11:02:10.795 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 176390 2023-10-01T11:02:10.795 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.795 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:02:10.795 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.795 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:02:10.796 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553526+0000", 2023-10-01T11:02:10.796 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:02:10.796 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.796 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.796 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:02:10.796 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553526+0000", 2023-10-01T11:02:10.797 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:02:10.797 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.797 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.797 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:02:10.797 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553526+0000", 2023-10-01T11:02:10.797 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999995 2023-10-01T11:02:10.797 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.797 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.797 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:02:10.797 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553528+0000", 2023-10-01T11:02:10.797 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.3649999999999998e-06 2023-10-01T11:02:10.797 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.798 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.798 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:02:10.798 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553534+0000", 2023-10-01T11:02:10.798 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.387e-06 2023-10-01T11:02:10.798 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.798 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.798 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:02:10.798 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553540+0000", 2023-10-01T11:02:10.798 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.3750000000000002e-06 2023-10-01T11:02:10.798 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.798 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.798 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:02:10.799 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.556634+0000", 2023-10-01T11:02:10.799 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0030939000000000001 2023-10-01T11:02:10.799 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.799 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.799 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T11:02:10.799 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.556641+0000", 2023-10-01T11:02:10.799 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.25e-06 2023-10-01T11:02:10.799 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.799 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.799 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:02:10.799 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.082071+0000", 2023-10-01T11:02:10.799 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.5254299260000002 2023-10-01T11:02:10.800 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.800 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.800 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:02:10.800 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.082238+0000", 2023-10-01T11:02:10.800 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000167658 2023-10-01T11:02:10.800 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.800 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.800 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 0,4,5,6,7", 2023-10-01T11:02:10.800 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.082604+0000", 2023-10-01T11:02:10.800 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000365149 2023-10-01T11:02:10.800 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.800 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.800 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.801 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.085887+0000", 2023-10-01T11:02:10.801 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0032833889999999998 2023-10-01T11:02:10.801 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.801 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.801 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.801 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.085921+0000", 2023-10-01T11:02:10.802 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.4317999999999999e-05 2023-10-01T11:02:10.802 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.802 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.802 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.802 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.088488+0000", 2023-10-01T11:02:10.802 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0025663690000000002 2023-10-01T11:02:10.803 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.803 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.803 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.803 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.088553+0000", 2023-10-01T11:02:10.803 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.5677000000000002e-05 2023-10-01T11:02:10.803 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.803 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.803 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T11:02:10.803 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.088562+0000", 2023-10-01T11:02:10.804 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.303e-06 2023-10-01T11:02:10.804 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.804 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.804 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.804 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.092655+0000", 2023-10-01T11:02:10.804 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0040938800000000003 2023-10-01T11:02:10.804 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.804 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.804 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:02:10.804 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.092697+0000", 2023-10-01T11:02:10.804 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.1888000000000003e-05 2023-10-01T11:02:10.805 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.805 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.805 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:02:10.805 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.092822+0000", 2023-10-01T11:02:10.805 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000124759 2023-10-01T11:02:10.805 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:02:10.805 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:02:10.805 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:02:10.805 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.806 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.806 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46037.0:176477 11.13 11:ceea8d8a:::benchmark_data_smithi078_157677_object69306:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e199)", 2023-10-01T11:02:10.806 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:54:33.553546+0000", 2023-10-01T11:02:10.806 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 457.08122584, 2023-10-01T11:02:10.806 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.578554842, 2023-10-01T11:02:10.806 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:02:10.806 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:02:10.806 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T11:02:10.806 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46037", 2023-10-01T11:02:10.806 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T11:02:10.807 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 176477 2023-10-01T11:02:10.807 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.807 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:02:10.807 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.807 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:02:10.807 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553546+0000", 2023-10-01T11:02:10.807 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:02:10.807 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.807 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.807 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:02:10.807 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553546+0000", 2023-10-01T11:02:10.807 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:02:10.807 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.808 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.808 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:02:10.808 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553545+0000", 2023-10-01T11:02:10.808 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999995 2023-10-01T11:02:10.808 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.808 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.808 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:02:10.808 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553548+0000", 2023-10-01T11:02:10.808 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.005e-06 2023-10-01T11:02:10.808 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.808 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.808 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:02:10.808 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553555+0000", 2023-10-01T11:02:10.809 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.4099999999999996e-06 2023-10-01T11:02:10.809 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.809 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.809 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:02:10.809 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.553560+0000", 2023-10-01T11:02:10.809 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.1370000000000001e-06 2023-10-01T11:02:10.809 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.809 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.809 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:02:10.809 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.556662+0000", 2023-10-01T11:02:10.809 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00310247 2023-10-01T11:02:10.810 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.810 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.810 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T11:02:10.810 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.556670+0000", 2023-10-01T11:02:10.810 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.7209999999999994e-06 2023-10-01T11:02:10.810 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.810 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.810 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:02:10.810 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.086002+0000", 2023-10-01T11:02:10.811 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.529332304 2023-10-01T11:02:10.811 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.811 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.811 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:02:10.811 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.086139+0000", 2023-10-01T11:02:10.811 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00013701499999999999 2023-10-01T11:02:10.811 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.812 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.812 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 0,4,5,6,7", 2023-10-01T11:02:10.812 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.086559+0000", 2023-10-01T11:02:10.812 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00042014800000000002 2023-10-01T11:02:10.813 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.813 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.813 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.813 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.092253+0000", 2023-10-01T11:02:10.813 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0056931600000000001 2023-10-01T11:02:10.813 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.813 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.813 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.813 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.092289+0000", 2023-10-01T11:02:10.813 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.6709999999999999e-05 2023-10-01T11:02:10.813 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.814 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.814 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.814 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.092388+0000", 2023-10-01T11:02:10.814 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.9223999999999997e-05 2023-10-01T11:02:10.814 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.814 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.814 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T11:02:10.815 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.097292+0000", 2023-10-01T11:02:10.815 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0049035099999999998 2023-10-01T11:02:10.815 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.815 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.815 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.815 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.102683+0000", 2023-10-01T11:02:10.815 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0053909609999999997 2023-10-01T11:02:10.815 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.815 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.815 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.815 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.131932+0000", 2023-10-01T11:02:10.815 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.029249139 2023-10-01T11:02:10.815 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.816 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.816 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:02:10.816 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.131987+0000", 2023-10-01T11:02:10.816 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.5053999999999998e-05 2023-10-01T11:02:10.816 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.816 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.816 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:02:10.816 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.132101+0000", 2023-10-01T11:02:10.817 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000113341 2023-10-01T11:02:10.817 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:02:10.817 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:02:10.817 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:02:10.817 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.817 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.817 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46037.0:176533 11.13 11:cf452de8:::benchmark_data_smithi078_157677_object69362:head [delete] snapc 0=[] ondisk+write+known_if_redirected e199)", 2023-10-01T11:02:10.817 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:54:33.686751+0000", 2023-10-01T11:02:10.817 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 456.94802073400001, 2023-10-01T11:02:10.817 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.44577048, 2023-10-01T11:02:10.817 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:02:10.817 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:02:10.818 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T11:02:10.818 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46037", 2023-10-01T11:02:10.818 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T11:02:10.818 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 176533 2023-10-01T11:02:10.818 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.818 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:02:10.818 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.818 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:02:10.818 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.686751+0000", 2023-10-01T11:02:10.818 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:02:10.818 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.819 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.819 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:02:10.819 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.686751+0000", 2023-10-01T11:02:10.819 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:02:10.819 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.819 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.819 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:02:10.819 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.686749+0000", 2023-10-01T11:02:10.819 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999976 2023-10-01T11:02:10.819 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.819 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.819 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:02:10.819 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.686759+0000", 2023-10-01T11:02:10.819 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.0138e-05 2023-10-01T11:02:10.820 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.820 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.820 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:02:10.820 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.686779+0000", 2023-10-01T11:02:10.820 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.0115e-05 2023-10-01T11:02:10.820 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.820 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.820 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:02:10.820 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.686790+0000", 2023-10-01T11:02:10.820 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1034e-05 2023-10-01T11:02:10.820 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.820 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.820 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:02:10.820 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.686879+0000", 2023-10-01T11:02:10.821 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.8918999999999993e-05 2023-10-01T11:02:10.821 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.821 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.821 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T11:02:10.821 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.686892+0000", 2023-10-01T11:02:10.821 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.3482000000000001e-05 2023-10-01T11:02:10.821 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.821 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.821 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:02:10.821 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.088585+0000", 2023-10-01T11:02:10.822 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.4016922489999999 2023-10-01T11:02:10.822 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.822 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.822 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:02:10.822 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.088729+0000", 2023-10-01T11:02:10.822 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00014457599999999999 2023-10-01T11:02:10.823 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.823 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.823 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 0,4,5,6,7", 2023-10-01T11:02:10.823 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.089056+0000", 2023-10-01T11:02:10.823 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00032674499999999998 2023-10-01T11:02:10.823 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.824 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.824 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.824 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.092323+0000", 2023-10-01T11:02:10.824 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0032670630000000002 2023-10-01T11:02:10.824 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.824 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.824 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.825 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.092356+0000", 2023-10-01T11:02:10.825 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.3086000000000001e-05 2023-10-01T11:02:10.825 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.825 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.825 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.825 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.097172+0000", 2023-10-01T11:02:10.825 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0048162869999999998 2023-10-01T11:02:10.825 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.825 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.825 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T11:02:10.825 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.097299+0000", 2023-10-01T11:02:10.825 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00012659999999999999 2023-10-01T11:02:10.826 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.826 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.826 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.826 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.102732+0000", 2023-10-01T11:02:10.826 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0054326579999999999 2023-10-01T11:02:10.826 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.826 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.826 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.827 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.132372+0000", 2023-10-01T11:02:10.827 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.029640307000000001 2023-10-01T11:02:10.827 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.827 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.827 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:02:10.827 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.132418+0000", 2023-10-01T11:02:10.827 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.5809000000000001e-05 2023-10-01T11:02:10.827 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.827 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.827 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:02:10.827 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.132521+0000", 2023-10-01T11:02:10.827 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000103566 2023-10-01T11:02:10.828 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:02:10.828 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:02:10.828 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:02:10.828 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.828 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.828 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46037.0:176550 11.13 11:cf453b8c:::benchmark_data_smithi078_157677_object69379:head [delete] snapc 0=[] ondisk+write+known_if_redirected e199)", 2023-10-01T11:02:10.828 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:54:33.716087+0000", 2023-10-01T11:02:10.828 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 456.918684316, 2023-10-01T11:02:10.828 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.416674075, 2023-10-01T11:02:10.828 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:02:10.828 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:02:10.828 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T11:02:10.829 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46037", 2023-10-01T11:02:10.829 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T11:02:10.829 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 176550 2023-10-01T11:02:10.829 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.829 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:02:10.829 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.829 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:02:10.829 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.716087+0000", 2023-10-01T11:02:10.829 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:02:10.829 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.830 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.830 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:02:10.830 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.716087+0000", 2023-10-01T11:02:10.830 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:02:10.830 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.830 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.830 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:02:10.830 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.716085+0000", 2023-10-01T11:02:10.830 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999981 2023-10-01T11:02:10.830 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.830 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.831 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:02:10.831 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.716093+0000", 2023-10-01T11:02:10.831 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.4229999999999999e-06 2023-10-01T11:02:10.831 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.831 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.831 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:02:10.831 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.716110+0000", 2023-10-01T11:02:10.831 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.7414999999999998e-05 2023-10-01T11:02:10.831 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.832 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.832 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:02:10.832 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.716119+0000", 2023-10-01T11:02:10.832 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.5010000000000004e-06 2023-10-01T11:02:10.832 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.832 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.834 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:02:10.834 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.716309+0000", 2023-10-01T11:02:10.834 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00018996200000000001 2023-10-01T11:02:10.834 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.834 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.835 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T11:02:10.835 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.716320+0000", 2023-10-01T11:02:10.835 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1402e-05 2023-10-01T11:02:10.835 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.835 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.835 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:02:10.835 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.092851+0000", 2023-10-01T11:02:10.835 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.376530918 2023-10-01T11:02:10.836 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.836 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.836 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:02:10.836 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.093167+0000", 2023-10-01T11:02:10.836 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00031600500000000001 2023-10-01T11:02:10.836 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.836 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.836 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 0,4,5,6,7", 2023-10-01T11:02:10.837 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.093531+0000", 2023-10-01T11:02:10.837 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000364003 2023-10-01T11:02:10.837 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.837 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.837 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.837 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.097278+0000", 2023-10-01T11:02:10.837 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0037469980000000001 2023-10-01T11:02:10.838 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.838 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.838 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T11:02:10.838 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.097304+0000", 2023-10-01T11:02:10.838 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.6342e-05 2023-10-01T11:02:10.838 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.838 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.838 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.838 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.097345+0000", 2023-10-01T11:02:10.839 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.0494000000000002e-05 2023-10-01T11:02:10.839 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.839 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.839 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.839 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.102316+0000", 2023-10-01T11:02:10.839 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0049712569999999998 2023-10-01T11:02:10.839 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.839 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.839 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.839 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.132145+0000", 2023-10-01T11:02:10.840 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.029828397 2023-10-01T11:02:10.840 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.841 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.841 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.841 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.132591+0000", 2023-10-01T11:02:10.841 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00044686900000000002 2023-10-01T11:02:10.841 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.841 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.841 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:02:10.841 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.132637+0000", 2023-10-01T11:02:10.841 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.5144e-05 2023-10-01T11:02:10.841 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.841 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.842 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:02:10.842 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.132761+0000", 2023-10-01T11:02:10.842 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00012464600000000001 2023-10-01T11:02:10.842 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:02:10.842 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:02:10.842 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:02:10.842 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.842 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.842 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46037.0:176554 11.13 11:cc4f0cc6:::benchmark_data_smithi078_157677_object69383:head [delete] snapc 0=[] ondisk+write+known_if_redirected e199)", 2023-10-01T11:02:10.842 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:54:33.723328+0000", 2023-10-01T11:02:10.842 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 456.911443572, 2023-10-01T11:02:10.842 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.4096755830000001, 2023-10-01T11:02:10.843 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:02:10.843 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:02:10.843 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T11:02:10.843 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46037", 2023-10-01T11:02:10.843 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T11:02:10.843 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 176554 2023-10-01T11:02:10.843 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.843 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:02:10.843 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.843 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:02:10.843 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.723328+0000", 2023-10-01T11:02:10.843 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:02:10.844 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.844 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.844 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:02:10.844 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.723328+0000", 2023-10-01T11:02:10.844 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:02:10.844 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.844 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.844 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:02:10.844 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.723326+0000", 2023-10-01T11:02:10.844 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999981 2023-10-01T11:02:10.844 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.844 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.844 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:02:10.845 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.723333+0000", 2023-10-01T11:02:10.845 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.4259999999999997e-06 2023-10-01T11:02:10.845 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.845 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.845 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:02:10.845 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.723346+0000", 2023-10-01T11:02:10.845 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.2442e-05 2023-10-01T11:02:10.845 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.845 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.845 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:02:10.845 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.723355+0000", 2023-10-01T11:02:10.845 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.363e-06 2023-10-01T11:02:10.846 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.846 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.846 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:02:10.846 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.723986+0000", 2023-10-01T11:02:10.846 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00063095700000000002 2023-10-01T11:02:10.846 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.846 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.846 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T11:02:10.846 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.723999+0000", 2023-10-01T11:02:10.846 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.3088000000000001e-05 2023-10-01T11:02:10.846 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.847 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.847 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:02:10.847 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.095277+0000", 2023-10-01T11:02:10.847 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.371277962 2023-10-01T11:02:10.847 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.847 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.847 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:02:10.847 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.095420+0000", 2023-10-01T11:02:10.847 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000142831 2023-10-01T11:02:10.847 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.847 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.847 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 0,4,5,6,7", 2023-10-01T11:02:10.847 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.095688+0000", 2023-10-01T11:02:10.848 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00026794600000000003 2023-10-01T11:02:10.848 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.848 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.848 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.848 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.102256+0000", 2023-10-01T11:02:10.848 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0065676709999999998 2023-10-01T11:02:10.848 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.848 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.848 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T11:02:10.849 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.102268+0000", 2023-10-01T11:02:10.849 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1863e-05 2023-10-01T11:02:10.850 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.850 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.850 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.850 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.102373+0000", 2023-10-01T11:02:10.850 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000105851 2023-10-01T11:02:10.850 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.850 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.851 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.851 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.102416+0000", 2023-10-01T11:02:10.851 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.2734000000000001e-05 2023-10-01T11:02:10.851 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.851 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.851 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.851 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.132285+0000", 2023-10-01T11:02:10.851 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.029868551 2023-10-01T11:02:10.851 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.851 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.851 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.852 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.132808+0000", 2023-10-01T11:02:10.852 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00052275499999999996 2023-10-01T11:02:10.852 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.852 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.852 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:02:10.852 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.132841+0000", 2023-10-01T11:02:10.852 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.3908000000000001e-05 2023-10-01T11:02:10.852 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.852 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.852 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:02:10.852 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.133004+0000", 2023-10-01T11:02:10.853 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00016209699999999999 2023-10-01T11:02:10.853 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:02:10.853 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:02:10.853 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:02:10.853 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.853 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.853 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46037.0:176584 11.13 11:cf1aac0a:::benchmark_data_smithi078_157677_object69413:head [delete] snapc 0=[] ondisk+write+known_if_redirected e199)", 2023-10-01T11:02:10.853 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:54:33.769745+0000", 2023-10-01T11:02:10.853 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 456.86502657199998, 2023-10-01T11:02:10.853 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.3634320180000001, 2023-10-01T11:02:10.854 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:02:10.854 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:02:10.854 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T11:02:10.854 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46037", 2023-10-01T11:02:10.854 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T11:02:10.854 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 176584 2023-10-01T11:02:10.854 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.854 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:02:10.854 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.854 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:02:10.854 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.769745+0000", 2023-10-01T11:02:10.854 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:02:10.854 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.855 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.855 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:02:10.855 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.769745+0000", 2023-10-01T11:02:10.855 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:02:10.855 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.855 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.855 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:02:10.855 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.769743+0000", 2023-10-01T11:02:10.855 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999981 2023-10-01T11:02:10.856 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.856 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.857 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:02:10.857 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.769751+0000", 2023-10-01T11:02:10.857 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.5340000000000002e-06 2023-10-01T11:02:10.857 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.857 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.857 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:02:10.858 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.769766+0000", 2023-10-01T11:02:10.858 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.5347000000000001e-05 2023-10-01T11:02:10.858 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.858 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.858 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:02:10.858 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.769774+0000", 2023-10-01T11:02:10.858 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.4700000000000002e-06 2023-10-01T11:02:10.858 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.858 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.859 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:02:10.859 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.769825+0000", 2023-10-01T11:02:10.859 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.0061000000000001e-05 2023-10-01T11:02:10.859 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.859 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.859 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T11:02:10.859 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.769835+0000", 2023-10-01T11:02:10.859 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.0808000000000001e-05 2023-10-01T11:02:10.859 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.860 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.860 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:02:10.860 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.097372+0000", 2023-10-01T11:02:10.860 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.3275364459999999 2023-10-01T11:02:10.860 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.860 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.860 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:02:10.860 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.097519+0000", 2023-10-01T11:02:10.860 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000147348 2023-10-01T11:02:10.861 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.861 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.861 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 0,4,5,6,7", 2023-10-01T11:02:10.861 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.097838+0000", 2023-10-01T11:02:10.865 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00031927000000000002 2023-10-01T11:02:10.865 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.865 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.865 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T11:02:10.865 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.102276+0000", 2023-10-01T11:02:10.865 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0044371760000000001 2023-10-01T11:02:10.865 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.865 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.865 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.865 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.102779+0000", 2023-10-01T11:02:10.865 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00050379400000000001 2023-10-01T11:02:10.866 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.866 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.866 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.866 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.102822+0000", 2023-10-01T11:02:10.866 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.2648999999999998e-05 2023-10-01T11:02:10.866 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.866 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.866 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.866 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.104932+0000", 2023-10-01T11:02:10.866 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0021096740000000002 2023-10-01T11:02:10.866 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.867 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.867 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.867 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.132316+0000", 2023-10-01T11:02:10.867 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.027384526999999999 2023-10-01T11:02:10.867 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.867 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.867 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.867 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.133041+0000", 2023-10-01T11:02:10.867 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00072457499999999998 2023-10-01T11:02:10.867 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.867 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.867 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:02:10.868 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.133081+0000", 2023-10-01T11:02:10.868 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.9787999999999999e-05 2023-10-01T11:02:10.868 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.868 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.868 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:02:10.868 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.133177+0000", 2023-10-01T11:02:10.868 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.6349999999999997e-05 2023-10-01T11:02:10.868 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:02:10.868 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:02:10.868 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:02:10.868 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.868 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.869 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46037.0:176589 11.13 11:cc4d8d83:::benchmark_data_smithi078_157677_object69418:head [delete] snapc 0=[] ondisk+write+known_if_redirected e199)", 2023-10-01T11:02:10.869 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:54:33.776428+0000", 2023-10-01T11:02:10.869 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 456.85834392599997, 2023-10-01T11:02:10.869 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.357038306, 2023-10-01T11:02:10.869 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:02:10.869 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:02:10.869 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T11:02:10.869 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46037", 2023-10-01T11:02:10.869 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T11:02:10.869 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 176589 2023-10-01T11:02:10.869 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.869 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:02:10.870 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.870 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:02:10.870 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.776428+0000", 2023-10-01T11:02:10.870 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:02:10.870 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.870 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.870 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:02:10.870 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.776428+0000", 2023-10-01T11:02:10.870 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:02:10.870 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.870 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.870 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:02:10.871 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.776426+0000", 2023-10-01T11:02:10.871 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999981 2023-10-01T11:02:10.871 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.871 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.871 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:02:10.871 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.776433+0000", 2023-10-01T11:02:10.871 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.8790000000000003e-06 2023-10-01T11:02:10.871 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.871 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.871 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:02:10.872 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.776451+0000", 2023-10-01T11:02:10.872 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.7453e-05 2023-10-01T11:02:10.872 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.872 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.872 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:02:10.872 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.776460+0000", 2023-10-01T11:02:10.872 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.7299999999999994e-06 2023-10-01T11:02:10.872 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.873 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.873 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:02:10.873 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.776656+0000", 2023-10-01T11:02:10.873 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00019599900000000001 2023-10-01T11:02:10.873 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.873 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.873 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T11:02:10.873 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.776684+0000", 2023-10-01T11:02:10.874 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.8699000000000001e-05 2023-10-01T11:02:10.874 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.874 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.874 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:02:10.874 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.100005+0000", 2023-10-01T11:02:10.874 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.323320432 2023-10-01T11:02:10.874 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.874 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.874 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:02:10.874 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.100137+0000", 2023-10-01T11:02:10.874 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00013244 2023-10-01T11:02:10.875 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.875 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.875 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 0,4,5,6,7", 2023-10-01T11:02:10.875 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.100431+0000", 2023-10-01T11:02:10.875 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00029331799999999999 2023-10-01T11:02:10.875 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.875 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.875 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T11:02:10.875 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.102738+0000", 2023-10-01T11:02:10.876 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0023078009999999999 2023-10-01T11:02:10.876 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.876 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.876 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.876 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.131755+0000", 2023-10-01T11:02:10.876 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.029016429999999999 2023-10-01T11:02:10.876 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.876 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.876 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.876 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.132174+0000", 2023-10-01T11:02:10.877 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00041950700000000001 2023-10-01T11:02:10.877 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.877 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.877 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.877 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.132228+0000", 2023-10-01T11:02:10.877 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.3334000000000001e-05 2023-10-01T11:02:10.877 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.877 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.877 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.877 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.132344+0000", 2023-10-01T11:02:10.878 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000116424 2023-10-01T11:02:10.878 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.878 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.878 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.878 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.133296+0000", 2023-10-01T11:02:10.878 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00095235899999999995 2023-10-01T11:02:10.878 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.878 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.878 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:02:10.879 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.133339+0000", 2023-10-01T11:02:10.879 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.2771000000000001e-05 2023-10-01T11:02:10.879 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.879 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.879 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:02:10.879 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.133466+0000", 2023-10-01T11:02:10.879 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00012671700000000001 2023-10-01T11:02:10.879 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:02:10.879 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:02:10.879 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:02:10.880 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.880 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.880 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46037.0:176678 11.13 11:c8f2364d:::benchmark_data_smithi078_157677_object69507:head [delete] snapc 0=[] ondisk+write+known_if_redirected e199)", 2023-10-01T11:02:10.880 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T10:54:33.956844+0000", 2023-10-01T11:02:10.880 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 456.67792716600002, 2023-10-01T11:02:10.880 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1774833840000001, 2023-10-01T11:02:10.880 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:02:10.880 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:02:10.880 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T11:02:10.880 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46037", 2023-10-01T11:02:10.881 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/3634377475", 2023-10-01T11:02:10.881 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 176678 2023-10-01T11:02:10.881 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.881 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:02:10.881 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.881 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:02:10.881 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.956844+0000", 2023-10-01T11:02:10.881 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:02:10.882 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.882 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.882 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:02:10.882 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.956844+0000", 2023-10-01T11:02:10.882 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:02:10.883 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.883 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.883 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:02:10.883 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.956842+0000", 2023-10-01T11:02:10.883 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999971 2023-10-01T11:02:10.883 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.883 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.883 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:02:10.884 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.956851+0000", 2023-10-01T11:02:10.884 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.8339999999999994e-06 2023-10-01T11:02:10.884 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.884 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.884 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:02:10.884 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.956868+0000", 2023-10-01T11:02:10.884 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.6637999999999999e-05 2023-10-01T11:02:10.884 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.884 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.885 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:02:10.885 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.956878+0000", 2023-10-01T11:02:10.885 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.0113e-05 2023-10-01T11:02:10.885 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.885 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.885 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:02:10.885 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.956940+0000", 2023-10-01T11:02:10.886 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.2275999999999998e-05 2023-10-01T11:02:10.886 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.886 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.886 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T11:02:10.886 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:33.956951+0000", 2023-10-01T11:02:10.886 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.0576e-05 2023-10-01T11:02:10.886 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.886 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.886 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:02:10.886 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.102865+0000", 2023-10-01T11:02:10.886 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.145914238 2023-10-01T11:02:10.887 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.887 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.887 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:02:10.887 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.103040+0000", 2023-10-01T11:02:10.887 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00017503700000000001 2023-10-01T11:02:10.887 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.887 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.887 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 0,4,5,6,7", 2023-10-01T11:02:10.887 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.103316+0000", 2023-10-01T11:02:10.887 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00027566500000000002 2023-10-01T11:02:10.887 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.887 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.887 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T11:02:10.888 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.111360+0000", 2023-10-01T11:02:10.888 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0080440990000000007 2023-10-01T11:02:10.888 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.888 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.888 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.888 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.131844+0000", 2023-10-01T11:02:10.888 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.020483596 2023-10-01T11:02:10.888 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.888 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.888 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.889 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.132201+0000", 2023-10-01T11:02:10.889 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000357189 2023-10-01T11:02:10.889 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.889 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.889 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.889 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.132256+0000", 2023-10-01T11:02:10.889 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.4735999999999999e-05 2023-10-01T11:02:10.889 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.889 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.889 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.889 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.133510+0000", 2023-10-01T11:02:10.889 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0012545799999999999 2023-10-01T11:02:10.890 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.890 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.890 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:02:10.890 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.134179+0000", 2023-10-01T11:02:10.890 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00066873099999999999 2023-10-01T11:02:10.890 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.890 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.890 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:02:10.890 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.134220+0000", 2023-10-01T11:02:10.890 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.1638999999999999e-05 2023-10-01T11:02:10.890 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:02:10.891 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:02:10.891 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:02:10.891 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T10:54:35.134328+0000", 2023-10-01T11:02:10.891 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000107247 2023-10-01T11:02:10.891 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:02:10.891 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:02:10.891 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:02:10.891 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:02:10.892 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:02:10.892 INFO:teuthology.orchestra.run.smithi078.stdout:} 2023-10-01T11:02:10.893 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:02:10.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:10.894 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:10.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:02:10.995 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:11.008 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:10 smithi078 bash[75299]: cluster 2023-10-01T11:02:10.750310+0000 mon.a (mon.0) 1697 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-10-01T11:02:11.009 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:10 smithi078 bash[78274]: cluster 2023-10-01T11:02:10.750310+0000 mon.a (mon.0) 1697 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-10-01T11:02:11.022 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:11.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:11.030 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:02:11.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:10 smithi173 bash[58816]: cluster 2023-10-01T11:02:10.750310+0000 mon.a (mon.0) 1697 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-10-01T11:02:11.131 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:11.162 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:11.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:11.167 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:02:11.268 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:11.293 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:11.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:11.298 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:02:11.399 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:11.430 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 4 16 10708 10692 167.048 149.375 0.00368663 0.00597615 2023-10-01T11:02:11.431 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:11.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:11.432 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:02:11.533 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:11.559 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:11.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:11.561 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:02:11.662 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:11.689 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:11.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:11.693 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:02:11.794 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:11.825 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6] out_osds: [7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T11:02:11.825 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T11:02:11.825 INFO:tasks.thrashosds.thrasher:Adding osd 7 2023-10-01T11:02:11.826 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd in 7 2023-10-01T11:02:11.827 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:11.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:11.830 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:02:11.931 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:11.943 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:11 smithi078 bash[75299]: cluster 2023-10-01T11:02:10.206645+0000 mgr.y (mgr.25072) 1344 : cluster 0 pgmap v1399: 89 pgs: 89 active+clean; 258 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 32 MiB/s wr, 515 op/s 2023-10-01T11:02:11.944 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:11 smithi078 bash[78274]: cluster 2023-10-01T11:02:10.206645+0000 mgr.y (mgr.25072) 1344 : cluster 0 pgmap v1399: 89 pgs: 89 active+clean; 258 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 32 MiB/s wr, 515 op/s 2023-10-01T11:02:11.970 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:11.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:11.973 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:02:12.074 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:12.099 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:12.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:12.103 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:02:12.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:11 smithi173 bash[58816]: cluster 2023-10-01T11:02:10.206645+0000 mgr.y (mgr.25072) 1344 : cluster 0 pgmap v1399: 89 pgs: 89 active+clean; 258 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 32 MiB/s wr, 515 op/s 2023-10-01T11:02:12.204 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:12.232 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:12.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:12.235 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:02:12.336 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:12.362 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:12.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:12.368 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:02:12.430 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 5 16 13027 13011 162.623 144.938 0.00561603 0.00613979 2023-10-01T11:02:12.470 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:12.513 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:12.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:12.517 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:02:12.617 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:12.642 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:12.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:12.645 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:02:12.746 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:12.780 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:12.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:12.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:02:12.885 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:12.912 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:12.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:12.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:02:13.017 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:13.045 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:13.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:13.048 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:02:13.149 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:13.178 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:13.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:13.181 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:02:13.281 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:13.315 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:13.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:13.320 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:02:13.421 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:13.431 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 6 16 15234 15218 158.506 137.938 0.00546405 0.00629962 2023-10-01T11:02:13.452 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:13.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:13.455 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:02:13.557 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:13.585 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:13.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:13.589 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:02:13.690 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:13.717 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:13.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:13.721 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:02:13.822 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:13.852 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:13.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:13.854 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:02:13.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:13 smithi173 bash[58816]: cluster 2023-10-01T11:02:12.207100+0000 mgr.y (mgr.25072) 1345 : cluster 0 pgmap v1400: 89 pgs: 89 active+clean; 258 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 27 MiB/s wr, 434 op/s 2023-10-01T11:02:13.956 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:13.984 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:13.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:13.988 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:02:14.072 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:13 smithi078 bash[75299]: cluster 2023-10-01T11:02:12.207100+0000 mgr.y (mgr.25072) 1345 : cluster 0 pgmap v1400: 89 pgs: 89 active+clean; 258 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 27 MiB/s wr, 434 op/s 2023-10-01T11:02:14.073 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:13 smithi078 bash[78274]: cluster 2023-10-01T11:02:12.207100+0000 mgr.y (mgr.25072) 1345 : cluster 0 pgmap v1400: 89 pgs: 89 active+clean; 258 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 27 MiB/s wr, 434 op/s 2023-10-01T11:02:14.088 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:14.120 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:14.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:14.124 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:02:14.225 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:14.250 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:14.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:14.256 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:02:14.357 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:14.385 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:14.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:14.391 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:02:14.430 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 7 16 17476 17460 155.878 140.125 0.00757615 0.00640851 2023-10-01T11:02:14.492 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:14.522 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:14.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:14.525 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:02:14.626 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:14.653 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:14.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:14.656 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:02:14.757 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:14.781 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:14.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:14.784 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:02:14.885 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:14.917 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:14.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:14.920 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:02:15.022 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:15.048 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:15.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:15.054 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:02:15.154 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:15.183 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:15.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:15.185 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:02:15.286 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:15.314 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:15.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:15.318 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:02:15.419 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:15.435 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 8 16 19260 19244 150.33 111.5 0.00652476 0.00664305 2023-10-01T11:02:15.445 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:15.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:15.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:02:15.551 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:15.582 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:15.584 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-01T11:02:15.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:15.585 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:02:15.686 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:15.717 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:15.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:15.723 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:02:15.824 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:15.853 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:15.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:15.858 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:02:15.959 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:15.970 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:15 smithi173 bash[58816]: cluster 2023-10-01T11:02:14.207848+0000 mgr.y (mgr.25072) 1346 : cluster 0 pgmap v1401: 89 pgs: 89 active+clean; 827 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-10-01T11:02:15.990 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:15.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:15.993 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:02:16.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:15 smithi078 bash[75299]: cluster 2023-10-01T11:02:14.207848+0000 mgr.y (mgr.25072) 1346 : cluster 0 pgmap v1401: 89 pgs: 89 active+clean; 827 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-10-01T11:02:16.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:15 smithi078 bash[78274]: cluster 2023-10-01T11:02:14.207848+0000 mgr.y (mgr.25072) 1346 : cluster 0 pgmap v1401: 89 pgs: 89 active+clean; 827 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-10-01T11:02:16.094 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:16.123 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:16.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:16.127 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:02:16.228 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:16.256 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:16.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:16.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:02:16.360 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:16.388 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:16.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:16.390 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:02:16.431 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 9 16 21128 21112 146.598 116.75 0.00789828 0.00681254 2023-10-01T11:02:16.491 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:16.526 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:16.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:16.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:02:16.631 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:16.658 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:16.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:16.663 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:02:16.764 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:16.787 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:16.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:16.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:02:16.827 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:16 smithi173 bash[58816]: audit 2023-10-01T11:02:16.732768+0000 mon.a (mon.0) 1698 : audit 1 from='client.? 172.21.15.78:0/2112349669' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T11:02:16.844 INFO:teuthology.orchestra.run.smithi078.stderr:noscrub is unset 2023-10-01T11:02:16.891 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:16.922 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:16.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:16.927 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:02:17.027 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:17.056 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:17.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:17.058 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:02:17.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:16 smithi078 bash[78274]: audit 2023-10-01T11:02:16.732768+0000 mon.a (mon.0) 1698 : audit 1 from='client.? 172.21.15.78:0/2112349669' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T11:02:17.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:16 smithi078 bash[75299]: audit 2023-10-01T11:02:16.732768+0000 mon.a (mon.0) 1698 : audit 1 from='client.? 172.21.15.78:0/2112349669' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T11:02:17.159 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:17.185 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:17.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:17.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:02:17.289 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:17.317 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:17.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:17.320 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:02:17.421 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:17.431 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 10 16 22850 22834 142.699 107.625 0.00334679 0.00695339 2023-10-01T11:02:17.450 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:17.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:17.453 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:02:17.554 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:17.589 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:17.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:17.591 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:02:17.692 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:17.719 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:17.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:17.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:02:17.822 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:17.831 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:17 smithi173 bash[58816]: cluster 2023-10-01T11:02:16.208762+0000 mgr.y (mgr.25072) 1347 : cluster 0 pgmap v1402: 89 pgs: 89 active+clean; 1005 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-01T11:02:17.831 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:17 smithi173 bash[58816]: cluster 2023-10-01T11:02:16.813280+0000 mon.a (mon.0) 1699 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:02:17.831 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:17 smithi173 bash[58816]: cluster 2023-10-01T11:02:16.829090+0000 osd.5 (osd.5) 282 : cluster 3 failed to encode map e292 with expected crc 2023-10-01T11:02:17.831 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:17 smithi173 bash[58816]: cluster 2023-10-01T11:02:16.831350+0000 osd.7 (osd.7) 296 : cluster 3 failed to encode map e292 with expected crc 2023-10-01T11:02:17.831 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:17 smithi173 bash[58816]: audit 2023-10-01T11:02:16.834143+0000 mon.a (mon.0) 1700 : audit 1 from='client.? 172.21.15.78:0/2112349669' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T11:02:17.831 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:17 smithi173 bash[58816]: cluster 2023-10-01T11:02:16.834244+0000 mon.a (mon.0) 1701 : cluster 0 osdmap e292: 8 total, 8 up, 7 in 2023-10-01T11:02:17.831 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:17 smithi173 bash[58816]: cluster 2023-10-01T11:02:16.836129+0000 osd.5 (osd.5) 283 : cluster 3 failed to encode map e292 with expected crc 2023-10-01T11:02:17.831 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:17 smithi173 bash[58816]: cluster 2023-10-01T11:02:16.839393+0000 osd.6 (osd.6) 263 : cluster 3 failed to encode map e292 with expected crc 2023-10-01T11:02:17.831 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:17 smithi173 bash[58816]: audit 2023-10-01T11:02:17.454032+0000 mon.a (mon.0) 1702 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:02:17.848 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:17.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:17.851 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:02:17.945 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:17 smithi078 bash[78274]: cluster 2023-10-01T11:02:16.208762+0000 mgr.y (mgr.25072) 1347 : cluster 0 pgmap v1402: 89 pgs: 89 active+clean; 1005 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-01T11:02:17.946 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:17 smithi078 bash[78274]: cluster 2023-10-01T11:02:16.813280+0000 mon.a (mon.0) 1699 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:02:17.946 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:17 smithi078 bash[78274]: cluster 2023-10-01T11:02:16.829090+0000 osd.5 (osd.5) 282 : cluster 3 failed to encode map e292 with expected crc 2023-10-01T11:02:17.946 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:17 smithi078 bash[78274]: cluster 2023-10-01T11:02:16.831350+0000 osd.7 (osd.7) 296 : cluster 3 failed to encode map e292 with expected crc 2023-10-01T11:02:17.946 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:17 smithi078 bash[78274]: audit 2023-10-01T11:02:16.834143+0000 mon.a (mon.0) 1700 : audit 1 from='client.? 172.21.15.78:0/2112349669' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T11:02:17.946 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:17 smithi078 bash[78274]: cluster 2023-10-01T11:02:16.834244+0000 mon.a (mon.0) 1701 : cluster 0 osdmap e292: 8 total, 8 up, 7 in 2023-10-01T11:02:17.946 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:17 smithi078 bash[78274]: cluster 2023-10-01T11:02:16.836129+0000 osd.5 (osd.5) 283 : cluster 3 failed to encode map e292 with expected crc 2023-10-01T11:02:17.946 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:17 smithi078 bash[78274]: cluster 2023-10-01T11:02:16.839393+0000 osd.6 (osd.6) 263 : cluster 3 failed to encode map e292 with expected crc 2023-10-01T11:02:17.946 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:17 smithi078 bash[78274]: audit 2023-10-01T11:02:17.454032+0000 mon.a (mon.0) 1702 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:02:17.947 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:17 smithi078 bash[75299]: cluster 2023-10-01T11:02:16.208762+0000 mgr.y (mgr.25072) 1347 : cluster 0 pgmap v1402: 89 pgs: 89 active+clean; 1005 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-10-01T11:02:17.947 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:17 smithi078 bash[75299]: cluster 2023-10-01T11:02:16.813280+0000 mon.a (mon.0) 1699 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:02:17.947 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:17 smithi078 bash[75299]: cluster 2023-10-01T11:02:16.829090+0000 osd.5 (osd.5) 282 : cluster 3 failed to encode map e292 with expected crc 2023-10-01T11:02:17.947 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:17 smithi078 bash[75299]: cluster 2023-10-01T11:02:16.831350+0000 osd.7 (osd.7) 296 : cluster 3 failed to encode map e292 with expected crc 2023-10-01T11:02:17.947 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:17 smithi078 bash[75299]: audit 2023-10-01T11:02:16.834143+0000 mon.a (mon.0) 1700 : audit 1 from='client.? 172.21.15.78:0/2112349669' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T11:02:17.947 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:17 smithi078 bash[75299]: cluster 2023-10-01T11:02:16.834244+0000 mon.a (mon.0) 1701 : cluster 0 osdmap e292: 8 total, 8 up, 7 in 2023-10-01T11:02:17.947 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:17 smithi078 bash[75299]: cluster 2023-10-01T11:02:16.836129+0000 osd.5 (osd.5) 283 : cluster 3 failed to encode map e292 with expected crc 2023-10-01T11:02:17.947 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:17 smithi078 bash[75299]: cluster 2023-10-01T11:02:16.839393+0000 osd.6 (osd.6) 263 : cluster 3 failed to encode map e292 with expected crc 2023-10-01T11:02:17.947 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:17 smithi078 bash[75299]: audit 2023-10-01T11:02:17.454032+0000 mon.a (mon.0) 1702 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:02:17.952 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:17.979 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:17.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:17.983 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:02:18.083 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:18.104 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:18.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:18.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:02:18.210 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:18.234 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:18.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:18.237 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:02:18.338 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:18.368 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:18.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:18.371 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:02:18.431 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 11 16 24700 24684 140.236 115.625 0.00846988 0.00712478 2023-10-01T11:02:18.472 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:18.499 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:18.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:18.501 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:02:18.603 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:18.633 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:18.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:18.638 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:02:18.739 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:18.765 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:18.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:18.767 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:02:18.868 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:18.900 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:18.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:18.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:02:19.004 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:19.029 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:19.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:19.031 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:02:19.135 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:19.163 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:19.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:19.166 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:02:19.201 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:19 smithi173 bash[58816]: audit 2023-10-01T11:02:18.195964+0000 mon.a (mon.0) 1703 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:02:19.201 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:19 smithi173 bash[58816]: cluster 2023-10-01T11:02:18.209342+0000 mgr.y (mgr.25072) 1348 : cluster 0 pgmap v1404: 89 pgs: 89 active+clean; 1005 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 100 MiB/s wr, 1.61k op/s 2023-10-01T11:02:19.267 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:19.276 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:19 smithi078 bash[78274]: audit 2023-10-01T11:02:18.195964+0000 mon.a (mon.0) 1703 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:02:19.276 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:19 smithi078 bash[78274]: cluster 2023-10-01T11:02:18.209342+0000 mgr.y (mgr.25072) 1348 : cluster 0 pgmap v1404: 89 pgs: 89 active+clean; 1005 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 100 MiB/s wr, 1.61k op/s 2023-10-01T11:02:19.276 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:19 smithi078 bash[75299]: audit 2023-10-01T11:02:18.195964+0000 mon.a (mon.0) 1703 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:02:19.277 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:19 smithi078 bash[75299]: cluster 2023-10-01T11:02:18.209342+0000 mgr.y (mgr.25072) 1348 : cluster 0 pgmap v1404: 89 pgs: 89 active+clean; 1005 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 100 MiB/s wr, 1.61k op/s 2023-10-01T11:02:19.292 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:19.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:19.295 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:02:19.396 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:19.428 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:19.431 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 12 16 26549 26533 138.179 115.562 0.0125975 0.00722997 2023-10-01T11:02:19.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:19.432 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:02:19.533 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:19.563 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:19.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:19.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:02:19.667 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:19.700 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:19.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:19.704 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:02:19.805 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:19.827 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:19.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:19.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:02:19.931 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:19.958 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:19.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:19.963 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:02:20.063 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:20.097 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:20.098 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:02:20.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:20.099 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:02:20.200 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:20.231 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:20.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:20.234 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:02:20.335 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:20.364 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:20.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:20.368 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:02:20.431 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 13 16 28247 28231 135.713 106.125 0.0053196 0.0073629 2023-10-01T11:02:20.469 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:20.496 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:20.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:20.499 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:02:20.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:02:20 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:02:20] "GET /metrics HTTP/1.1" 200 36373 "" "Prometheus/2.33.4" 2023-10-01T11:02:20.600 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:20.630 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:20.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:20.632 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:02:20.733 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:20.763 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:20.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:20.770 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:02:20.871 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:20.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:20.910 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:20.910 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:02:21.011 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:21.035 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:21.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:21.037 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:02:21.137 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:21.160 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:21.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:21.163 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:02:21.263 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:21.302 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:21.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:21.307 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:02:21.408 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:21.418 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:21 smithi078 bash[75299]: cluster 2023-10-01T11:02:20.210000+0000 mgr.y (mgr.25072) 1349 : cluster 0 pgmap v1405: 89 pgs: 89 active+clean; 1.5 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 133 MiB/s wr, 2.12k op/s 2023-10-01T11:02:21.418 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:21 smithi078 bash[78274]: cluster 2023-10-01T11:02:20.210000+0000 mgr.y (mgr.25072) 1349 : cluster 0 pgmap v1405: 89 pgs: 89 active+clean; 1.5 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 133 MiB/s wr, 2.12k op/s 2023-10-01T11:02:21.432 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 14 16 29983 29967 133.768 108.5 0.00561845 0.00747072 2023-10-01T11:02:21.440 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:21.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:21.444 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:02:21.546 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:21.556 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:21 smithi173 bash[58816]: cluster 2023-10-01T11:02:20.210000+0000 mgr.y (mgr.25072) 1349 : cluster 0 pgmap v1405: 89 pgs: 89 active+clean; 1.5 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 133 MiB/s wr, 2.12k op/s 2023-10-01T11:02:21.574 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:21.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:21.577 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:02:21.630 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T11:02:21.678 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:21.708 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:21.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:21.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:02:21.815 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:21.842 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:21.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:21.847 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:02:21.947 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:21.972 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:22.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:22.030 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:02:22.131 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:22.158 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:22.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:22.166 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:02:22.216 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:02:22.266 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:22.298 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:22.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:22.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:02:22.403 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:22.432 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 15 16 31487 31471 131.117 94 0.00880801 0.00762089 2023-10-01T11:02:22.436 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:22.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:22.439 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:02:22.540 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:22.569 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:22.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:22.576 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:02:22.677 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:22.708 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:22.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:22.712 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:02:22.812 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:22.844 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:22.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:22.847 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:02:22.947 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:22.973 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:22.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:22.975 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:02:23.076 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:23.104 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:23.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:23.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:02:23.208 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:23.238 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:23.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:23.241 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:02:23.342 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:23.372 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:23.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:23.374 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:02:23.431 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 16 16 33190 33174 129.574 106.438 0.0100978 0.00771138 2023-10-01T11:02:23.475 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:23.499 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:23.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:23.502 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:02:23.603 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:23.626 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:23.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:23.627 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:02:23.728 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:23.762 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:23.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:23.766 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:02:23.830 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:23 smithi078 bash[75299]: cluster 2023-10-01T11:02:22.210482+0000 mgr.y (mgr.25072) 1350 : cluster 0 pgmap v1406: 89 pgs: 89 active+clean; 1.5 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 133 MiB/s wr, 2.12k op/s 2023-10-01T11:02:23.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:23 smithi078 bash[78274]: cluster 2023-10-01T11:02:22.210482+0000 mgr.y (mgr.25072) 1350 : cluster 0 pgmap v1406: 89 pgs: 89 active+clean; 1.5 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 133 MiB/s wr, 2.12k op/s 2023-10-01T11:02:23.866 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:23.904 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:23.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:23.908 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:02:24.009 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:24.022 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:23 smithi173 bash[58816]: cluster 2023-10-01T11:02:22.210482+0000 mgr.y (mgr.25072) 1350 : cluster 0 pgmap v1406: 89 pgs: 89 active+clean; 1.5 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 133 MiB/s wr, 2.12k op/s 2023-10-01T11:02:24.038 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:24.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:24.041 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:02:24.142 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:24.172 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:24.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:24.175 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:02:24.276 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:24.309 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:24.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:24.312 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:02:24.413 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:24.432 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 17 16 34522 34506 126.848 83.25 0.00706382 0.00787653 2023-10-01T11:02:24.443 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:24.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:24.445 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:02:24.546 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:24.576 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:24.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:24.580 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:02:24.682 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:24.718 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:24.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:24.722 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:02:24.822 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:24.865 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:24.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:24.866 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:02:24.967 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:25.001 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:25.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:25.005 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:02:25.105 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:25.127 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T11:02:25.131 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:25.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:25.133 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:02:25.163 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T11:02:25.183 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T11:02:25.209 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T11:02:25.234 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:25.246 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T11:02:25.267 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:25.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:25.273 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:02:25.275 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T11:02:25.303 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T11:02:25.329 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T11:02:25.342 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T11:02:25.342 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T11:02:25.342 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T11:02:25.343 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T11:02:25.343 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T11:02:25.343 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T11:02:25.343 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T11:02:25.343 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T11:02:25.373 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:25.411 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:25.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:25.418 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:02:25.432 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 18 16 35982 35966 124.87 91.25 0.00233999 0.00799831 2023-10-01T11:02:25.519 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:25.544 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:25.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:25.548 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:02:25.649 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:25.677 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:25.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:25.680 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:02:25.781 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:25.830 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:25.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:25.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:02:25.934 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:25.951 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:25 smithi173 bash[58816]: cluster 2023-10-01T11:02:24.211150+0000 mgr.y (mgr.25072) 1351 : cluster 0 pgmap v1407: 89 pgs: 89 active+clean; 1.9 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 115 MiB/s wr, 1.84k op/s 2023-10-01T11:02:25.952 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:25 smithi173 bash[58816]: audit 2023-10-01T11:02:25.090889+0000 mon.b (mon.1) 54 : audit 0 from='client.? 172.21.15.78:0/3303000805' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:02:25.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:25.975 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:25.975 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:02:25.976 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:25 smithi078 bash[78274]: cluster 2023-10-01T11:02:24.211150+0000 mgr.y (mgr.25072) 1351 : cluster 0 pgmap v1407: 89 pgs: 89 active+clean; 1.9 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 115 MiB/s wr, 1.84k op/s 2023-10-01T11:02:25.977 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:25 smithi078 bash[78274]: audit 2023-10-01T11:02:25.090889+0000 mon.b (mon.1) 54 : audit 0 from='client.? 172.21.15.78:0/3303000805' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:02:25.977 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:25 smithi078 bash[75299]: cluster 2023-10-01T11:02:24.211150+0000 mgr.y (mgr.25072) 1351 : cluster 0 pgmap v1407: 89 pgs: 89 active+clean; 1.9 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 115 MiB/s wr, 1.84k op/s 2023-10-01T11:02:25.977 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:25 smithi078 bash[75299]: audit 2023-10-01T11:02:25.090889+0000 mon.b (mon.1) 54 : audit 0 from='client.? 172.21.15.78:0/3303000805' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:02:26.076 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:26.112 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:26.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:26.116 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:02:26.216 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:26.253 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:26.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:26.260 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:02:26.360 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:26.389 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:26.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:26.393 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:02:26.432 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 19 16 37520 37504 123.357 96.125 0.00699599 0.00810035 2023-10-01T11:02:26.494 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:26.523 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:26.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:26.533 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:02:26.634 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:26.675 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:26.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:26.680 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:02:26.781 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:26.812 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:26.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:26.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:02:26.921 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:26.956 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:26.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:26.960 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:02:27.061 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:27.090 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:27.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:27.094 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:02:27.195 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:27.238 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:27.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:27.241 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:02:27.343 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:27.368 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:27.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:27.371 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:02:27.432 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:2023-10-01T11:02:27.432031+0000 min lat: 0.00136129 max lat: 0.202467 avg lat: 0.00821279 2023-10-01T11:02:27.432 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T11:02:27.432 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 20 16 38953 38937 121.667 89.5625 0.0165411 0.00821279 2023-10-01T11:02:27.472 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:27.502 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:27.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:27.505 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:02:27.605 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:27.630 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:27.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:27.636 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:02:27.737 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:27.769 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:27.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:27.772 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:02:27.873 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:27.884 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:27 smithi078 bash[78274]: cluster 2023-10-01T11:02:26.212585+0000 mgr.y (mgr.25072) 1352 : cluster 0 pgmap v1408: 89 pgs: 89 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 110 MiB/s wr, 1.76k op/s 2023-10-01T11:02:27.884 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:27 smithi078 bash[75299]: cluster 2023-10-01T11:02:26.212585+0000 mgr.y (mgr.25072) 1352 : cluster 0 pgmap v1408: 89 pgs: 89 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 110 MiB/s wr, 1.76k op/s 2023-10-01T11:02:27.901 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:27.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:27.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:02:28.005 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:28.014 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:27 smithi173 bash[58816]: cluster 2023-10-01T11:02:26.212585+0000 mgr.y (mgr.25072) 1352 : cluster 0 pgmap v1408: 89 pgs: 89 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 110 MiB/s wr, 1.76k op/s 2023-10-01T11:02:28.043 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:28.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:28.047 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:02:28.148 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:28.179 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:28.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:28.183 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:02:28.285 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:28.313 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:28.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:28.319 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:02:28.420 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:28.432 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 21 16 40487 40471 120.438 95.875 0.00675581 0.00829555 2023-10-01T11:02:28.451 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:28.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:28.453 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:02:28.554 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:28.581 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:28.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:28.583 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:02:28.684 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:28.709 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:28.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:28.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:02:28.812 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:28.843 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:28.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:28.848 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:02:28.949 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:28.975 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:28.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:28.980 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:02:29.081 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:29.113 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:29.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:29.117 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:02:29.218 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:29.243 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:29.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:29.245 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:02:29.346 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:29.374 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:29.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:29.380 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:02:29.432 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 22 16 41987 41971 119.224 93.75 0.00889112 0.00838235 2023-10-01T11:02:29.481 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:29.508 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:29.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:29.510 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:02:29.611 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:29.638 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:29.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:29.641 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:02:29.704 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:29 smithi078 bash[75299]: cluster 2023-10-01T11:02:28.213014+0000 mgr.y (mgr.25072) 1353 : cluster 0 pgmap v1409: 89 pgs: 89 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 97 MiB/s wr, 1.55k op/s 2023-10-01T11:02:29.743 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:29.752 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:29 smithi173 bash[58816]: cluster 2023-10-01T11:02:28.213014+0000 mgr.y (mgr.25072) 1353 : cluster 0 pgmap v1409: 89 pgs: 89 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 97 MiB/s wr, 1.55k op/s 2023-10-01T11:02:29.774 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:29.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:29.777 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:02:29.878 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:29.887 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:29 smithi078 bash[78274]: cluster 2023-10-01T11:02:28.213014+0000 mgr.y (mgr.25072) 1353 : cluster 0 pgmap v1409: 89 pgs: 89 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 97 MiB/s wr, 1.55k op/s 2023-10-01T11:02:29.905 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:29.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:29.908 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:02:30.009 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:30.033 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:30.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:30.038 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:02:30.139 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:30.170 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:30.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:30.175 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:02:30.269 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:02:30 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:02:30] "GET /metrics HTTP/1.1" 200 36373 "" "Prometheus/2.33.4" 2023-10-01T11:02:30.276 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:30.304 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:30.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:30.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:02:30.410 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:30.432 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 23 16 43475 43459 118.084 93 0.0145472 0.00846269 2023-10-01T11:02:30.441 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:30.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:30.446 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:02:30.547 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:30.573 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:30.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:30.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:02:30.678 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:30.707 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:30.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:30.710 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:02:30.810 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:30.844 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:30.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:30.847 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:02:30.948 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:30.980 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:30.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:30.986 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:02:31.086 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:31.115 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:31.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:31.118 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:02:31.219 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:31.245 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:31.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:31.248 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:02:31.280 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T11:02:31.349 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:31.390 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:31.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:31.391 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:02:31.432 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 24 16 44882 44866 116.827 87.9375 0.00724695 0.0085526 2023-10-01T11:02:31.492 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:31.528 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:31.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:31.532 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:02:31.633 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:31.656 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:31.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:31.658 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:02:31.759 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:31.795 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:31.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:31.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:02:31.900 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:31.931 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:31.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:31.934 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:02:32.035 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:32.044 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:31 smithi173 bash[58816]: cluster 2023-10-01T11:02:30.213726+0000 mgr.y (mgr.25072) 1354 : cluster 0 pgmap v1410: 89 pgs: 89 active+clean; 2.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 130 MiB/s wr, 2.08k op/s 2023-10-01T11:02:32.071 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:32.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:32.074 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:02:32.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:31 smithi078 bash[78274]: cluster 2023-10-01T11:02:30.213726+0000 mgr.y (mgr.25072) 1354 : cluster 0 pgmap v1410: 89 pgs: 89 active+clean; 2.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 130 MiB/s wr, 2.08k op/s 2023-10-01T11:02:32.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:31 smithi078 bash[75299]: cluster 2023-10-01T11:02:30.213726+0000 mgr.y (mgr.25072) 1354 : cluster 0 pgmap v1410: 89 pgs: 89 active+clean; 2.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 130 MiB/s wr, 2.08k op/s 2023-10-01T11:02:32.086 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:02:32.175 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:32.206 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:32.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:32.211 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:02:32.312 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:32.339 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:32.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:32.342 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:02:32.433 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 25 16 46106 46090 115.214 76.5 0.0034585 0.00864324 2023-10-01T11:02:32.442 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:32.481 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:32.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:32.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:02:32.587 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:32.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:32.616 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:32.617 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:02:32.717 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:32.718 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:32 smithi078 bash[75299]: audit 2023-10-01T11:02:32.453977+0000 mon.a (mon.0) 1704 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:02:32.746 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:32.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:32.752 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:02:32.853 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:32.889 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:32.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:32.892 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:02:32.993 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:33.002 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:32 smithi078 bash[78274]: audit 2023-10-01T11:02:32.453977+0000 mon.a (mon.0) 1704 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:02:33.029 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:33.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:33.032 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:02:33.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:32 smithi173 bash[58816]: audit 2023-10-01T11:02:32.453977+0000 mon.a (mon.0) 1704 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:02:33.133 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:33.161 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:33.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:33.163 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:02:33.264 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:33.290 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:33.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:33.293 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:02:33.393 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:33.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:33.432 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:33.432 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:02:33.433 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 26 15 47466 47451 114.054 85.0625 0.00776301 0.00876203 2023-10-01T11:02:33.533 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:33.574 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:33.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:33.579 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:02:33.679 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:33.705 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:33.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:33.708 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:02:33.808 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:33.835 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:33.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:33.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:02:33.941 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:33.971 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:33.975 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:33 smithi078 bash[75299]: cluster 2023-10-01T11:02:32.214123+0000 mgr.y (mgr.25072) 1355 : cluster 0 pgmap v1411: 89 pgs: 89 active+clean; 2.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 82 MiB/s wr, 1.30k op/s 2023-10-01T11:02:33.975 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:33 smithi078 bash[78274]: cluster 2023-10-01T11:02:32.214123+0000 mgr.y (mgr.25072) 1355 : cluster 0 pgmap v1411: 89 pgs: 89 active+clean; 2.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 82 MiB/s wr, 1.30k op/s 2023-10-01T11:02:33.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:33.976 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:02:34.077 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:34.107 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:34.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:34.114 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:02:34.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:33 smithi173 bash[58816]: cluster 2023-10-01T11:02:32.214123+0000 mgr.y (mgr.25072) 1355 : cluster 0 pgmap v1411: 89 pgs: 89 active+clean; 2.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 82 MiB/s wr, 1.30k op/s 2023-10-01T11:02:34.216 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:34.261 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:34.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:34.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:02:34.364 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:34.392 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:34.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:34.395 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:02:34.433 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 27 16 48686 48670 112.651 76.1875 0.117361 0.00884305 2023-10-01T11:02:34.496 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:34.525 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:34.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:34.530 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:02:34.631 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:34.659 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:34.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:34.662 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:02:34.763 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:34.786 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:34.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:34.791 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:02:34.891 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:34.920 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:34.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:34.926 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:02:35.029 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:35.058 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:35.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:35.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:02:35.164 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:35.188 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:35.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:35.194 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:02:35.295 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:35.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:35 smithi078 bash[75299]: debug 2023-10-01T11:02:35.046+0000 7f31c8dd0700 -1 mon.a@0(leader).osd e292 definitely_dead 0 2023-10-01T11:02:35.334 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:35.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:35.336 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:02:35.433 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 28 16 50003 49987 111.567 82.3125 0.0158019 0.00895742 2023-10-01T11:02:35.437 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:35.475 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:35.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:35.482 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:02:35.583 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:35.611 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:35.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:35.613 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:02:35.714 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:35.739 INFO:teuthology.orchestra.run.smithi078.stderr:marked in osd.7. 2023-10-01T11:02:35.745 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:35.746 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:35 smithi078 bash[75299]: cluster 2023-10-01T11:02:34.216605+0000 mgr.y (mgr.25072) 1356 : cluster 0 pgmap v1412: 89 pgs: 89 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-10-01T11:02:35.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:35.749 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:02:35.850 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:35.860 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:35 smithi173 bash[58816]: cluster 2023-10-01T11:02:34.216605+0000 mgr.y (mgr.25072) 1356 : cluster 0 pgmap v1412: 89 pgs: 89 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-10-01T11:02:35.860 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:35 smithi173 bash[58816]: audit 2023-10-01T11:02:35.052043+0000 mon.b (mon.1) 55 : audit 0 from='client.? 172.21.15.78:0/1654027232' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-01T11:02:35.860 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:35 smithi173 bash[58816]: audit 2023-10-01T11:02:35.052419+0000 mon.a (mon.0) 1705 : audit 0 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-01T11:02:35.877 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:35.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:35.880 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:02:35.981 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:36.007 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:36.009 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:35 smithi078 bash[75299]: audit 2023-10-01T11:02:35.052043+0000 mon.b (mon.1) 55 : audit 0 from='client.? 172.21.15.78:0/1654027232' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-01T11:02:36.009 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:35 smithi078 bash[75299]: audit 2023-10-01T11:02:35.052419+0000 mon.a (mon.0) 1705 : audit 0 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-01T11:02:36.009 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:35 smithi078 bash[78274]: cluster 2023-10-01T11:02:34.216605+0000 mgr.y (mgr.25072) 1356 : cluster 0 pgmap v1412: 89 pgs: 89 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-10-01T11:02:36.009 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:35 smithi078 bash[78274]: audit 2023-10-01T11:02:35.052043+0000 mon.b (mon.1) 55 : audit 0 from='client.? 172.21.15.78:0/1654027232' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-01T11:02:36.009 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:35 smithi078 bash[78274]: audit 2023-10-01T11:02:35.052419+0000 mon.a (mon.0) 1705 : audit 0 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-01T11:02:36.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:36.010 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:02:36.111 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:36.137 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:36.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:36.140 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:02:36.241 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:36.265 INFO:teuthology.orchestra.run.smithi078.stdout:ERROR: (22) Invalid argument 2023-10-01T11:02:36.265 INFO:teuthology.orchestra.run.smithi078.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T11:02:36.273 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:36.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:36.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:02:36.377 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:36.413 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:36.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:36.417 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:02:36.433 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 29 16 50477 50461 108.741 29.625 0.00351028 0.00898683 2023-10-01T11:02:36.517 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:36.544 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:36.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:36.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:02:36.647 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:36.681 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:36.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:36.683 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:02:36.756 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:36 smithi078 bash[75299]: cluster 2023-10-01T11:02:35.729264+0000 osd.6 (osd.6) 264 : cluster 3 failed to encode map e293 with expected crc 2023-10-01T11:02:36.756 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:36 smithi078 bash[75299]: cluster 2023-10-01T11:02:35.730537+0000 osd.7 (osd.7) 297 : cluster 3 failed to encode map e293 with expected crc 2023-10-01T11:02:36.756 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:36 smithi078 bash[75299]: audit 2023-10-01T11:02:35.730869+0000 mon.a (mon.0) 1706 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-10-01T11:02:36.756 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:36 smithi078 bash[75299]: cluster 2023-10-01T11:02:35.730965+0000 mon.a (mon.0) 1707 : cluster 0 osdmap e293: 8 total, 8 up, 8 in 2023-10-01T11:02:36.757 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:36 smithi078 bash[75299]: cluster 2023-10-01T11:02:35.732602+0000 osd.7 (osd.7) 298 : cluster 3 failed to encode map e293 with expected crc 2023-10-01T11:02:36.757 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:36 smithi078 bash[75299]: cluster 2023-10-01T11:02:35.734476+0000 osd.5 (osd.5) 284 : cluster 3 failed to encode map e293 with expected crc 2023-10-01T11:02:36.757 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:36 smithi078 bash[75299]: cluster 2023-10-01T11:02:35.738856+0000 osd.5 (osd.5) 285 : cluster 3 failed to encode map e293 with expected crc 2023-10-01T11:02:36.757 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:36 smithi078 bash[75299]: cluster 2023-10-01T11:02:35.739250+0000 osd.7 (osd.7) 299 : cluster 3 failed to encode map e293 with expected crc 2023-10-01T11:02:36.757 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:36 smithi078 bash[75299]: cluster 2023-10-01T11:02:35.740617+0000 osd.6 (osd.6) 265 : cluster 3 failed to encode map e293 with expected crc 2023-10-01T11:02:36.757 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:36 smithi078 bash[75299]: cluster 2023-10-01T11:02:35.740753+0000 osd.7 (osd.7) 300 : cluster 3 failed to encode map e293 with expected crc 2023-10-01T11:02:36.757 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:36 smithi078 bash[75299]: cluster 2023-10-01T11:02:35.742089+0000 osd.6 (osd.6) 266 : cluster 3 failed to encode map e293 with expected crc 2023-10-01T11:02:36.757 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:36 smithi078 bash[75299]: cluster 2023-10-01T11:02:36.737437+0000 mon.a (mon.0) 1708 : cluster 0 osdmap e294: 8 total, 8 up, 8 in 2023-10-01T11:02:36.757 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:36 smithi078 bash[78274]: cluster 2023-10-01T11:02:35.729264+0000 osd.6 (osd.6) 264 : cluster 3 failed to encode map e293 with expected crc 2023-10-01T11:02:36.757 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:36 smithi078 bash[78274]: cluster 2023-10-01T11:02:35.730537+0000 osd.7 (osd.7) 297 : cluster 3 failed to encode map e293 with expected crc 2023-10-01T11:02:36.758 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:36 smithi078 bash[78274]: audit 2023-10-01T11:02:35.730869+0000 mon.a (mon.0) 1706 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-10-01T11:02:36.758 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:36 smithi078 bash[78274]: cluster 2023-10-01T11:02:35.730965+0000 mon.a (mon.0) 1707 : cluster 0 osdmap e293: 8 total, 8 up, 8 in 2023-10-01T11:02:36.758 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:36 smithi078 bash[78274]: cluster 2023-10-01T11:02:35.732602+0000 osd.7 (osd.7) 298 : cluster 3 failed to encode map e293 with expected crc 2023-10-01T11:02:36.758 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:36 smithi078 bash[78274]: cluster 2023-10-01T11:02:35.734476+0000 osd.5 (osd.5) 284 : cluster 3 failed to encode map e293 with expected crc 2023-10-01T11:02:36.758 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:36 smithi078 bash[78274]: cluster 2023-10-01T11:02:35.738856+0000 osd.5 (osd.5) 285 : cluster 3 failed to encode map e293 with expected crc 2023-10-01T11:02:36.758 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:36 smithi078 bash[78274]: cluster 2023-10-01T11:02:35.739250+0000 osd.7 (osd.7) 299 : cluster 3 failed to encode map e293 with expected crc 2023-10-01T11:02:36.758 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:36 smithi078 bash[78274]: cluster 2023-10-01T11:02:35.740617+0000 osd.6 (osd.6) 265 : cluster 3 failed to encode map e293 with expected crc 2023-10-01T11:02:36.758 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:36 smithi078 bash[78274]: cluster 2023-10-01T11:02:35.740753+0000 osd.7 (osd.7) 300 : cluster 3 failed to encode map e293 with expected crc 2023-10-01T11:02:36.758 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:36 smithi078 bash[78274]: cluster 2023-10-01T11:02:35.742089+0000 osd.6 (osd.6) 266 : cluster 3 failed to encode map e293 with expected crc 2023-10-01T11:02:36.758 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:36 smithi078 bash[78274]: cluster 2023-10-01T11:02:36.737437+0000 mon.a (mon.0) 1708 : cluster 0 osdmap e294: 8 total, 8 up, 8 in 2023-10-01T11:02:36.784 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:36.806 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:36.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:36.809 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:02:36.910 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:36.934 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:36.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:36.937 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:02:37.037 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:37.061 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:37.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:37.064 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:02:37.122 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:36 smithi173 bash[58816]: cluster 2023-10-01T11:02:35.729264+0000 osd.6 (osd.6) 264 : cluster 3 failed to encode map e293 with expected crc 2023-10-01T11:02:37.123 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:36 smithi173 bash[58816]: cluster 2023-10-01T11:02:35.730537+0000 osd.7 (osd.7) 297 : cluster 3 failed to encode map e293 with expected crc 2023-10-01T11:02:37.123 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:36 smithi173 bash[58816]: audit 2023-10-01T11:02:35.730869+0000 mon.a (mon.0) 1706 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-10-01T11:02:37.123 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:36 smithi173 bash[58816]: cluster 2023-10-01T11:02:35.730965+0000 mon.a (mon.0) 1707 : cluster 0 osdmap e293: 8 total, 8 up, 8 in 2023-10-01T11:02:37.123 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:36 smithi173 bash[58816]: cluster 2023-10-01T11:02:35.732602+0000 osd.7 (osd.7) 298 : cluster 3 failed to encode map e293 with expected crc 2023-10-01T11:02:37.123 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:36 smithi173 bash[58816]: cluster 2023-10-01T11:02:35.734476+0000 osd.5 (osd.5) 284 : cluster 3 failed to encode map e293 with expected crc 2023-10-01T11:02:37.123 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:36 smithi173 bash[58816]: cluster 2023-10-01T11:02:35.738856+0000 osd.5 (osd.5) 285 : cluster 3 failed to encode map e293 with expected crc 2023-10-01T11:02:37.123 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:36 smithi173 bash[58816]: cluster 2023-10-01T11:02:35.739250+0000 osd.7 (osd.7) 299 : cluster 3 failed to encode map e293 with expected crc 2023-10-01T11:02:37.123 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:36 smithi173 bash[58816]: cluster 2023-10-01T11:02:35.740617+0000 osd.6 (osd.6) 265 : cluster 3 failed to encode map e293 with expected crc 2023-10-01T11:02:37.123 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:36 smithi173 bash[58816]: cluster 2023-10-01T11:02:35.740753+0000 osd.7 (osd.7) 300 : cluster 3 failed to encode map e293 with expected crc 2023-10-01T11:02:37.123 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:36 smithi173 bash[58816]: cluster 2023-10-01T11:02:35.742089+0000 osd.6 (osd.6) 266 : cluster 3 failed to encode map e293 with expected crc 2023-10-01T11:02:37.124 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:36 smithi173 bash[58816]: cluster 2023-10-01T11:02:36.737437+0000 mon.a (mon.0) 1708 : cluster 0 osdmap e294: 8 total, 8 up, 8 in 2023-10-01T11:02:37.165 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:37.202 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:37.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:37.203 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:02:37.304 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:37.328 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:37.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:37.331 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:02:37.432 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:37.450 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 30 16 50477 50461 105.117 0 - 0.00898683 2023-10-01T11:02:37.486 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:37.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:37.491 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:02:37.592 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:37.620 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:37.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:37.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:02:37.723 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:37.782 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:37.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:37.786 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:02:37.887 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:37.923 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:37.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:37.926 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:02:38.027 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:38.036 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:37 smithi173 bash[58816]: cluster 2023-10-01T11:02:36.216966+0000 mgr.y (mgr.25072) 1357 : cluster 0 pgmap v1414: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-10-01T11:02:38.036 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:37 smithi173 bash[58816]: cluster 2023-10-01T11:02:36.733126+0000 osd.7 (osd.7) 301 : cluster 3 failed to encode map e294 with expected crc 2023-10-01T11:02:38.036 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:37 smithi173 bash[58816]: cluster 2023-10-01T11:02:36.734743+0000 osd.5 (osd.5) 286 : cluster 3 failed to encode map e294 with expected crc 2023-10-01T11:02:38.036 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:37 smithi173 bash[58816]: cluster 2023-10-01T11:02:36.737953+0000 osd.6 (osd.6) 267 : cluster 3 failed to encode map e294 with expected crc 2023-10-01T11:02:38.036 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:37 smithi173 bash[58816]: cluster 2023-10-01T11:02:36.738271+0000 osd.7 (osd.7) 302 : cluster 3 failed to encode map e294 with expected crc 2023-10-01T11:02:38.036 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:37 smithi173 bash[58816]: cluster 2023-10-01T11:02:36.739198+0000 osd.5 (osd.5) 287 : cluster 3 failed to encode map e294 with expected crc 2023-10-01T11:02:38.036 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:37 smithi173 bash[58816]: cluster 2023-10-01T11:02:36.739900+0000 osd.7 (osd.7) 303 : cluster 3 failed to encode map e294 with expected crc 2023-10-01T11:02:38.037 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:37 smithi173 bash[58816]: cluster 2023-10-01T11:02:36.741408+0000 osd.5 (osd.5) 288 : cluster 3 failed to encode map e294 with expected crc 2023-10-01T11:02:38.037 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:37 smithi173 bash[58816]: cluster 2023-10-01T11:02:36.741871+0000 osd.7 (osd.7) 304 : cluster 3 failed to encode map e294 with expected crc 2023-10-01T11:02:38.037 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:37 smithi173 bash[58816]: cluster 2023-10-01T11:02:36.741920+0000 osd.6 (osd.6) 268 : cluster 3 failed to encode map e294 with expected crc 2023-10-01T11:02:38.037 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:37 smithi173 bash[58816]: cluster 2023-10-01T11:02:36.743512+0000 osd.6 (osd.6) 269 : cluster 3 failed to encode map e294 with expected crc 2023-10-01T11:02:38.037 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:37 smithi173 bash[58816]: cluster 2023-10-01T11:02:36.743515+0000 osd.5 (osd.5) 289 : cluster 3 failed to encode map e294 with expected crc 2023-10-01T11:02:38.037 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:37 smithi173 bash[58816]: cluster 2023-10-01T11:02:36.743616+0000 osd.7 (osd.7) 305 : cluster 3 failed to encode map e294 with expected crc 2023-10-01T11:02:38.037 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:37 smithi173 bash[58816]: cluster 2023-10-01T11:02:36.744964+0000 osd.7 (osd.7) 306 : cluster 3 failed to encode map e294 with expected crc 2023-10-01T11:02:38.037 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:37 smithi173 bash[58816]: cluster 2023-10-01T11:02:37.740562+0000 mon.a (mon.0) 1709 : cluster 0 osdmap e295: 8 total, 8 up, 8 in 2023-10-01T11:02:38.064 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:38.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:38.070 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:02:38.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:37 smithi078 bash[78274]: cluster 2023-10-01T11:02:36.216966+0000 mgr.y (mgr.25072) 1357 : cluster 0 pgmap v1414: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-10-01T11:02:38.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:37 smithi078 bash[78274]: cluster 2023-10-01T11:02:36.733126+0000 osd.7 (osd.7) 301 : cluster 3 failed to encode map e294 with expected crc 2023-10-01T11:02:38.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:37 smithi078 bash[78274]: cluster 2023-10-01T11:02:36.734743+0000 osd.5 (osd.5) 286 : cluster 3 failed to encode map e294 with expected crc 2023-10-01T11:02:38.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:37 smithi078 bash[78274]: cluster 2023-10-01T11:02:36.737953+0000 osd.6 (osd.6) 267 : cluster 3 failed to encode map e294 with expected crc 2023-10-01T11:02:38.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:37 smithi078 bash[78274]: cluster 2023-10-01T11:02:36.738271+0000 osd.7 (osd.7) 302 : cluster 3 failed to encode map e294 with expected crc 2023-10-01T11:02:38.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:37 smithi078 bash[78274]: cluster 2023-10-01T11:02:36.739198+0000 osd.5 (osd.5) 287 : cluster 3 failed to encode map e294 with expected crc 2023-10-01T11:02:38.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:37 smithi078 bash[78274]: cluster 2023-10-01T11:02:36.739900+0000 osd.7 (osd.7) 303 : cluster 3 failed to encode map e294 with expected crc 2023-10-01T11:02:38.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:37 smithi078 bash[78274]: cluster 2023-10-01T11:02:36.741408+0000 osd.5 (osd.5) 288 : cluster 3 failed to encode map e294 with expected crc 2023-10-01T11:02:38.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:37 smithi078 bash[78274]: cluster 2023-10-01T11:02:36.741871+0000 osd.7 (osd.7) 304 : cluster 3 failed to encode map e294 with expected crc 2023-10-01T11:02:38.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:37 smithi078 bash[78274]: cluster 2023-10-01T11:02:36.741920+0000 osd.6 (osd.6) 268 : cluster 3 failed to encode map e294 with expected crc 2023-10-01T11:02:38.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:37 smithi078 bash[78274]: cluster 2023-10-01T11:02:36.743512+0000 osd.6 (osd.6) 269 : cluster 3 failed to encode map e294 with expected crc 2023-10-01T11:02:38.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:37 smithi078 bash[78274]: cluster 2023-10-01T11:02:36.743515+0000 osd.5 (osd.5) 289 : cluster 3 failed to encode map e294 with expected crc 2023-10-01T11:02:38.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:37 smithi078 bash[78274]: cluster 2023-10-01T11:02:36.743616+0000 osd.7 (osd.7) 305 : cluster 3 failed to encode map e294 with expected crc 2023-10-01T11:02:38.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:37 smithi078 bash[78274]: cluster 2023-10-01T11:02:36.744964+0000 osd.7 (osd.7) 306 : cluster 3 failed to encode map e294 with expected crc 2023-10-01T11:02:38.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:37 smithi078 bash[78274]: cluster 2023-10-01T11:02:37.740562+0000 mon.a (mon.0) 1709 : cluster 0 osdmap e295: 8 total, 8 up, 8 in 2023-10-01T11:02:38.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:37 smithi078 bash[75299]: cluster 2023-10-01T11:02:36.216966+0000 mgr.y (mgr.25072) 1357 : cluster 0 pgmap v1414: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-10-01T11:02:38.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:37 smithi078 bash[75299]: cluster 2023-10-01T11:02:36.733126+0000 osd.7 (osd.7) 301 : cluster 3 failed to encode map e294 with expected crc 2023-10-01T11:02:38.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:37 smithi078 bash[75299]: cluster 2023-10-01T11:02:36.734743+0000 osd.5 (osd.5) 286 : cluster 3 failed to encode map e294 with expected crc 2023-10-01T11:02:38.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:37 smithi078 bash[75299]: cluster 2023-10-01T11:02:36.737953+0000 osd.6 (osd.6) 267 : cluster 3 failed to encode map e294 with expected crc 2023-10-01T11:02:38.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:37 smithi078 bash[75299]: cluster 2023-10-01T11:02:36.738271+0000 osd.7 (osd.7) 302 : cluster 3 failed to encode map e294 with expected crc 2023-10-01T11:02:38.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:37 smithi078 bash[75299]: cluster 2023-10-01T11:02:36.739198+0000 osd.5 (osd.5) 287 : cluster 3 failed to encode map e294 with expected crc 2023-10-01T11:02:38.085 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:37 smithi078 bash[75299]: cluster 2023-10-01T11:02:36.739900+0000 osd.7 (osd.7) 303 : cluster 3 failed to encode map e294 with expected crc 2023-10-01T11:02:38.086 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:37 smithi078 bash[75299]: cluster 2023-10-01T11:02:36.741408+0000 osd.5 (osd.5) 288 : cluster 3 failed to encode map e294 with expected crc 2023-10-01T11:02:38.086 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:37 smithi078 bash[75299]: cluster 2023-10-01T11:02:36.741871+0000 osd.7 (osd.7) 304 : cluster 3 failed to encode map e294 with expected crc 2023-10-01T11:02:38.086 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:37 smithi078 bash[75299]: cluster 2023-10-01T11:02:36.741920+0000 osd.6 (osd.6) 268 : cluster 3 failed to encode map e294 with expected crc 2023-10-01T11:02:38.086 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:37 smithi078 bash[75299]: cluster 2023-10-01T11:02:36.743512+0000 osd.6 (osd.6) 269 : cluster 3 failed to encode map e294 with expected crc 2023-10-01T11:02:38.086 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:37 smithi078 bash[75299]: cluster 2023-10-01T11:02:36.743515+0000 osd.5 (osd.5) 289 : cluster 3 failed to encode map e294 with expected crc 2023-10-01T11:02:38.087 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:37 smithi078 bash[75299]: cluster 2023-10-01T11:02:36.743616+0000 osd.7 (osd.7) 305 : cluster 3 failed to encode map e294 with expected crc 2023-10-01T11:02:38.087 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:37 smithi078 bash[75299]: cluster 2023-10-01T11:02:36.744964+0000 osd.7 (osd.7) 306 : cluster 3 failed to encode map e294 with expected crc 2023-10-01T11:02:38.087 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:37 smithi078 bash[75299]: cluster 2023-10-01T11:02:37.740562+0000 mon.a (mon.0) 1709 : cluster 0 osdmap e295: 8 total, 8 up, 8 in 2023-10-01T11:02:38.171 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:38.195 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:38.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:38.197 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:02:38.297 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:38.341 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:38.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:38.347 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:02:38.433 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 31 16 50882 50866 102.542 12.6562 0.0168892 0.00974601 2023-10-01T11:02:38.450 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:38.477 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:38.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:38.483 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:02:38.584 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:38.613 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:38.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:38.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:02:38.720 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:38.747 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:38.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:38.751 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:02:38.838 INFO:tasks.thrashosds.thrasher:Added osd 7 2023-10-01T11:02:38.852 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:38.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:38 smithi173 bash[58816]: cluster 2023-10-01T11:02:37.737633+0000 osd.7 (osd.7) 307 : cluster 3 failed to encode map e295 with expected crc 2023-10-01T11:02:38.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:38 smithi173 bash[58816]: cluster 2023-10-01T11:02:37.740469+0000 osd.5 (osd.5) 290 : cluster 3 failed to encode map e295 with expected crc 2023-10-01T11:02:38.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:38 smithi173 bash[58816]: cluster 2023-10-01T11:02:37.741846+0000 osd.6 (osd.6) 270 : cluster 3 failed to encode map e295 with expected crc 2023-10-01T11:02:38.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:38 smithi173 bash[58816]: cluster 2023-10-01T11:02:37.741994+0000 osd.5 (osd.5) 291 : cluster 3 failed to encode map e295 with expected crc 2023-10-01T11:02:38.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:38 smithi173 bash[58816]: cluster 2023-10-01T11:02:37.744781+0000 osd.6 (osd.6) 271 : cluster 3 failed to encode map e295 with expected crc 2023-10-01T11:02:38.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:38 smithi173 bash[58816]: cluster 2023-10-01T11:02:37.747021+0000 osd.6 (osd.6) 272 : cluster 3 failed to encode map e295 with expected crc 2023-10-01T11:02:38.882 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:38.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:38.886 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:02:38.987 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:39.012 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:39.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:38 smithi078 bash[78274]: cluster 2023-10-01T11:02:37.737633+0000 osd.7 (osd.7) 307 : cluster 3 failed to encode map e295 with expected crc 2023-10-01T11:02:39.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:38 smithi078 bash[78274]: cluster 2023-10-01T11:02:37.740469+0000 osd.5 (osd.5) 290 : cluster 3 failed to encode map e295 with expected crc 2023-10-01T11:02:39.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:38 smithi078 bash[78274]: cluster 2023-10-01T11:02:37.741846+0000 osd.6 (osd.6) 270 : cluster 3 failed to encode map e295 with expected crc 2023-10-01T11:02:39.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:38 smithi078 bash[78274]: cluster 2023-10-01T11:02:37.741994+0000 osd.5 (osd.5) 291 : cluster 3 failed to encode map e295 with expected crc 2023-10-01T11:02:39.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:38 smithi078 bash[78274]: cluster 2023-10-01T11:02:37.744781+0000 osd.6 (osd.6) 271 : cluster 3 failed to encode map e295 with expected crc 2023-10-01T11:02:39.015 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:38 smithi078 bash[78274]: cluster 2023-10-01T11:02:37.747021+0000 osd.6 (osd.6) 272 : cluster 3 failed to encode map e295 with expected crc 2023-10-01T11:02:39.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:39.016 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:02:39.016 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:38 smithi078 bash[75299]: cluster 2023-10-01T11:02:37.737633+0000 osd.7 (osd.7) 307 : cluster 3 failed to encode map e295 with expected crc 2023-10-01T11:02:39.017 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:38 smithi078 bash[75299]: cluster 2023-10-01T11:02:37.740469+0000 osd.5 (osd.5) 290 : cluster 3 failed to encode map e295 with expected crc 2023-10-01T11:02:39.017 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:38 smithi078 bash[75299]: cluster 2023-10-01T11:02:37.741846+0000 osd.6 (osd.6) 270 : cluster 3 failed to encode map e295 with expected crc 2023-10-01T11:02:39.017 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:38 smithi078 bash[75299]: cluster 2023-10-01T11:02:37.741994+0000 osd.5 (osd.5) 291 : cluster 3 failed to encode map e295 with expected crc 2023-10-01T11:02:39.017 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:38 smithi078 bash[75299]: cluster 2023-10-01T11:02:37.744781+0000 osd.6 (osd.6) 271 : cluster 3 failed to encode map e295 with expected crc 2023-10-01T11:02:39.017 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:38 smithi078 bash[75299]: cluster 2023-10-01T11:02:37.747021+0000 osd.6 (osd.6) 272 : cluster 3 failed to encode map e295 with expected crc 2023-10-01T11:02:39.117 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:39.142 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:39.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:39.147 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:02:39.249 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:39.285 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:39.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:39.290 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:02:39.391 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:39.420 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:39.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:39.422 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:02:39.433 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 32 16 51787 51771 101.105 56.5625 0.0115303 0.00988377 2023-10-01T11:02:39.523 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:39.548 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:39.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:39.554 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:02:39.656 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:39.682 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:39.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:39.686 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:02:39.788 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:39.814 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:39.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:39.822 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:02:39.923 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:39.950 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:39.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:39.953 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:02:40.054 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:40.063 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:39 smithi078 bash[78274]: cluster 2023-10-01T11:02:38.217442+0000 mgr.y (mgr.25072) 1358 : cluster 0 pgmap v1417: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s; 14 B/s, 0 objects/s recovering 2023-10-01T11:02:40.063 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:39 smithi078 bash[78274]: cluster 2023-10-01T11:02:38.763378+0000 osd.7 (osd.7) 308 : cluster 3 failed to encode map e296 with expected crc 2023-10-01T11:02:40.063 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:39 smithi078 bash[78274]: cluster 2023-10-01T11:02:38.764799+0000 mon.a (mon.0) 1710 : cluster 0 osdmap e296: 8 total, 8 up, 8 in 2023-10-01T11:02:40.063 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:39 smithi078 bash[78274]: cluster 2023-10-01T11:02:38.764918+0000 osd.6 (osd.6) 273 : cluster 3 failed to encode map e296 with expected crc 2023-10-01T11:02:40.063 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:39 smithi078 bash[78274]: cluster 2023-10-01T11:02:38.773826+0000 osd.5 (osd.5) 292 : cluster 3 failed to encode map e296 with expected crc 2023-10-01T11:02:40.063 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:39 smithi078 bash[78274]: cluster 2023-10-01T11:02:38.782291+0000 osd.5 (osd.5) 293 : cluster 3 failed to encode map e296 with expected crc 2023-10-01T11:02:40.064 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:39 smithi078 bash[75299]: cluster 2023-10-01T11:02:38.217442+0000 mgr.y (mgr.25072) 1358 : cluster 0 pgmap v1417: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s; 14 B/s, 0 objects/s recovering 2023-10-01T11:02:40.064 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:39 smithi078 bash[75299]: cluster 2023-10-01T11:02:38.763378+0000 osd.7 (osd.7) 308 : cluster 3 failed to encode map e296 with expected crc 2023-10-01T11:02:40.064 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:39 smithi078 bash[75299]: cluster 2023-10-01T11:02:38.764799+0000 mon.a (mon.0) 1710 : cluster 0 osdmap e296: 8 total, 8 up, 8 in 2023-10-01T11:02:40.064 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:39 smithi078 bash[75299]: cluster 2023-10-01T11:02:38.764918+0000 osd.6 (osd.6) 273 : cluster 3 failed to encode map e296 with expected crc 2023-10-01T11:02:40.064 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:39 smithi078 bash[75299]: cluster 2023-10-01T11:02:38.773826+0000 osd.5 (osd.5) 292 : cluster 3 failed to encode map e296 with expected crc 2023-10-01T11:02:40.064 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:39 smithi078 bash[75299]: cluster 2023-10-01T11:02:38.782291+0000 osd.5 (osd.5) 293 : cluster 3 failed to encode map e296 with expected crc 2023-10-01T11:02:40.078 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:40.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:40.082 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:02:40.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:39 smithi173 bash[58816]: cluster 2023-10-01T11:02:38.217442+0000 mgr.y (mgr.25072) 1358 : cluster 0 pgmap v1417: 89 pgs: 89 active+clean; 2.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s; 14 B/s, 0 objects/s recovering 2023-10-01T11:02:40.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:39 smithi173 bash[58816]: cluster 2023-10-01T11:02:38.763378+0000 osd.7 (osd.7) 308 : cluster 3 failed to encode map e296 with expected crc 2023-10-01T11:02:40.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:39 smithi173 bash[58816]: cluster 2023-10-01T11:02:38.764799+0000 mon.a (mon.0) 1710 : cluster 0 osdmap e296: 8 total, 8 up, 8 in 2023-10-01T11:02:40.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:39 smithi173 bash[58816]: cluster 2023-10-01T11:02:38.764918+0000 osd.6 (osd.6) 273 : cluster 3 failed to encode map e296 with expected crc 2023-10-01T11:02:40.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:39 smithi173 bash[58816]: cluster 2023-10-01T11:02:38.773826+0000 osd.5 (osd.5) 292 : cluster 3 failed to encode map e296 with expected crc 2023-10-01T11:02:40.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:39 smithi173 bash[58816]: cluster 2023-10-01T11:02:38.782291+0000 osd.5 (osd.5) 293 : cluster 3 failed to encode map e296 with expected crc 2023-10-01T11:02:40.185 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:40.216 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:40.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:40.221 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:02:40.322 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:40.331 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:02:40 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:02:40] "GET /metrics HTTP/1.1" 200 36375 "" "Prometheus/2.33.4" 2023-10-01T11:02:40.347 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-01T11:02:40.352 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:40.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:40.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:02:40.435 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 33 16 52688 52672 99.7423 56.3125 0.0158251 0.0100192 2023-10-01T11:02:40.456 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:40.482 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:40.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:40.487 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:02:40.588 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:40.616 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:40.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:40.618 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:02:40.719 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:40.744 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:40.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:40.747 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:02:40.837 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:02:40.848 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:40.863 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:40 smithi078 bash[75299]: cluster 2023-10-01T11:02:39.768512+0000 osd.7 (osd.7) 309 : cluster 3 failed to encode map e297 with expected crc 2023-10-01T11:02:40.863 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:40 smithi078 bash[75299]: cluster 2023-10-01T11:02:39.768609+0000 mon.a (mon.0) 1711 : cluster 0 osdmap e297: 8 total, 8 up, 8 in 2023-10-01T11:02:40.864 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:40 smithi078 bash[75299]: cluster 2023-10-01T11:02:39.772826+0000 osd.5 (osd.5) 294 : cluster 3 failed to encode map e297 with expected crc 2023-10-01T11:02:40.864 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:40 smithi078 bash[75299]: cluster 2023-10-01T11:02:39.774420+0000 osd.6 (osd.6) 274 : cluster 3 failed to encode map e297 with expected crc 2023-10-01T11:02:40.864 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:40 smithi078 bash[75299]: cluster 2023-10-01T11:02:39.774466+0000 osd.5 (osd.5) 295 : cluster 3 failed to encode map e297 with expected crc 2023-10-01T11:02:40.864 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:40 smithi078 bash[75299]: cluster 2023-10-01T11:02:39.775453+0000 osd.7 (osd.7) 310 : cluster 3 failed to encode map e297 with expected crc 2023-10-01T11:02:40.864 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:40 smithi078 bash[75299]: cluster 2023-10-01T11:02:39.776287+0000 osd.5 (osd.5) 296 : cluster 3 failed to encode map e297 with expected crc 2023-10-01T11:02:40.865 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:40 smithi078 bash[78274]: cluster 2023-10-01T11:02:39.768512+0000 osd.7 (osd.7) 309 : cluster 3 failed to encode map e297 with expected crc 2023-10-01T11:02:40.865 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:40 smithi078 bash[78274]: cluster 2023-10-01T11:02:39.768609+0000 mon.a (mon.0) 1711 : cluster 0 osdmap e297: 8 total, 8 up, 8 in 2023-10-01T11:02:40.865 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:40 smithi078 bash[78274]: cluster 2023-10-01T11:02:39.772826+0000 osd.5 (osd.5) 294 : cluster 3 failed to encode map e297 with expected crc 2023-10-01T11:02:40.865 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:40 smithi078 bash[78274]: cluster 2023-10-01T11:02:39.774420+0000 osd.6 (osd.6) 274 : cluster 3 failed to encode map e297 with expected crc 2023-10-01T11:02:40.865 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:40 smithi078 bash[78274]: cluster 2023-10-01T11:02:39.774466+0000 osd.5 (osd.5) 295 : cluster 3 failed to encode map e297 with expected crc 2023-10-01T11:02:40.865 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:40 smithi078 bash[78274]: cluster 2023-10-01T11:02:39.775453+0000 osd.7 (osd.7) 310 : cluster 3 failed to encode map e297 with expected crc 2023-10-01T11:02:40.865 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:40 smithi078 bash[78274]: cluster 2023-10-01T11:02:39.776287+0000 osd.5 (osd.5) 296 : cluster 3 failed to encode map e297 with expected crc 2023-10-01T11:02:40.881 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:40.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:40.887 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:02:40.988 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:41.012 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:41.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:41.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:02:41.115 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:41.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:40 smithi173 bash[58816]: cluster 2023-10-01T11:02:39.768512+0000 osd.7 (osd.7) 309 : cluster 3 failed to encode map e297 with expected crc 2023-10-01T11:02:41.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:40 smithi173 bash[58816]: cluster 2023-10-01T11:02:39.768609+0000 mon.a (mon.0) 1711 : cluster 0 osdmap e297: 8 total, 8 up, 8 in 2023-10-01T11:02:41.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:40 smithi173 bash[58816]: cluster 2023-10-01T11:02:39.772826+0000 osd.5 (osd.5) 294 : cluster 3 failed to encode map e297 with expected crc 2023-10-01T11:02:41.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:40 smithi173 bash[58816]: cluster 2023-10-01T11:02:39.774420+0000 osd.6 (osd.6) 274 : cluster 3 failed to encode map e297 with expected crc 2023-10-01T11:02:41.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:40 smithi173 bash[58816]: cluster 2023-10-01T11:02:39.774466+0000 osd.5 (osd.5) 295 : cluster 3 failed to encode map e297 with expected crc 2023-10-01T11:02:41.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:40 smithi173 bash[58816]: cluster 2023-10-01T11:02:39.775453+0000 osd.7 (osd.7) 310 : cluster 3 failed to encode map e297 with expected crc 2023-10-01T11:02:41.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:40 smithi173 bash[58816]: cluster 2023-10-01T11:02:39.776287+0000 osd.5 (osd.5) 296 : cluster 3 failed to encode map e297 with expected crc 2023-10-01T11:02:41.157 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:41.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:41.163 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:02:41.264 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:41.294 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:41.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:41.297 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:02:41.398 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:41.429 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:41.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:41.435 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:02:41.435 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 34 16 53566 53550 98.4226 54.875 0.00829272 0.0101524 2023-10-01T11:02:41.537 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:41.573 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:41.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:41.576 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:02:41.677 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:41.713 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:41.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:41.715 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:02:41.788 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:41 smithi173 bash[58816]: cluster 2023-10-01T11:02:40.217907+0000 mgr.y (mgr.25072) 1359 : cluster 0 pgmap v1420: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 83 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 51 MiB/s wr, 814 op/s; 9471/153777 objects degraded (6.159%); 6381/153777 objects misplaced (4.150%); 1.5 MiB/s, 2 keys/s, 24 objects/s recovering 2023-10-01T11:02:41.789 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:41 smithi173 bash[58816]: cluster 2023-10-01T11:02:40.775861+0000 mon.a (mon.0) 1712 : cluster 3 Health check failed: Degraded data redundancy: 9471/153777 objects degraded (6.159%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T11:02:41.816 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:41.838 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:41 smithi078 bash[75299]: cluster 2023-10-01T11:02:40.217907+0000 mgr.y (mgr.25072) 1359 : cluster 0 pgmap v1420: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 83 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 51 MiB/s wr, 814 op/s; 9471/153777 objects degraded (6.159%); 6381/153777 objects misplaced (4.150%); 1.5 MiB/s, 2 keys/s, 24 objects/s recovering 2023-10-01T11:02:41.839 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:41 smithi078 bash[75299]: cluster 2023-10-01T11:02:40.775861+0000 mon.a (mon.0) 1712 : cluster 3 Health check failed: Degraded data redundancy: 9471/153777 objects degraded (6.159%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T11:02:41.839 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:41 smithi078 bash[78274]: cluster 2023-10-01T11:02:40.217907+0000 mgr.y (mgr.25072) 1359 : cluster 0 pgmap v1420: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 83 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 51 MiB/s wr, 814 op/s; 9471/153777 objects degraded (6.159%); 6381/153777 objects misplaced (4.150%); 1.5 MiB/s, 2 keys/s, 24 objects/s recovering 2023-10-01T11:02:41.839 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:41 smithi078 bash[78274]: cluster 2023-10-01T11:02:40.775861+0000 mon.a (mon.0) 1712 : cluster 3 Health check failed: Degraded data redundancy: 9471/153777 objects degraded (6.159%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T11:02:41.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:41.857 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:41.857 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:02:41.958 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:42.005 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:42.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:42.009 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:02:42.109 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:42.134 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:42.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:42.141 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:02:42.243 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:42.273 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:42.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:42.277 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:02:42.377 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:42.412 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:42.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:42.419 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:02:42.438 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 35 16 54479 54463 97.2409 57.0625 0.0274799 0.0102763 2023-10-01T11:02:42.521 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:42.554 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:42.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:42.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:02:42.658 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:42.686 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:42.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:42.689 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:02:42.790 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:42.824 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:42.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:42.827 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:02:42.928 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:42.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:42.955 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:42.955 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:02:43.056 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:43.094 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:43.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:43.100 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:02:43.201 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:43.228 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:43.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:43.232 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:02:43.334 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:43.361 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:43.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:43.364 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:02:43.435 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 36 16 55411 55395 96.1577 58.25 0.018564 0.0103918 2023-10-01T11:02:43.465 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:43.494 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:43.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:43.498 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:02:43.600 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:43.645 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:43.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:43.649 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:02:43.750 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:43.780 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:43.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:43.785 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:02:43.839 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T11:02:43.840 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T11:02:43.840 INFO:tasks.thrashosds.thrasher:Removing osd 6, in_osds are: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T11:02:43.840 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd out 6 2023-10-01T11:02:43.886 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:43.916 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:43.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:43.921 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:02:44.022 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:44.062 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:44.064 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:43 smithi078 bash[75299]: cluster 2023-10-01T11:02:42.218429+0000 mgr.y (mgr.25072) 1360 : cluster 0 pgmap v1421: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 83 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 37 MiB/s wr, 593 op/s; 9471/153777 objects degraded (6.159%); 6381/153777 objects misplaced (4.150%); 1.1 MiB/s, 1 keys/s, 17 objects/s recovering 2023-10-01T11:02:44.065 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:43 smithi078 bash[78274]: cluster 2023-10-01T11:02:42.218429+0000 mgr.y (mgr.25072) 1360 : cluster 0 pgmap v1421: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 83 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 37 MiB/s wr, 593 op/s; 9471/153777 objects degraded (6.159%); 6381/153777 objects misplaced (4.150%); 1.1 MiB/s, 1 keys/s, 17 objects/s recovering 2023-10-01T11:02:44.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:44.066 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:02:44.122 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:43 smithi173 bash[58816]: cluster 2023-10-01T11:02:42.218429+0000 mgr.y (mgr.25072) 1360 : cluster 0 pgmap v1421: 89 pgs: 1 peering, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 83 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 37 MiB/s wr, 593 op/s; 9471/153777 objects degraded (6.159%); 6381/153777 objects misplaced (4.150%); 1.1 MiB/s, 1 keys/s, 17 objects/s recovering 2023-10-01T11:02:44.167 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:44.202 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:44.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:44.209 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:02:44.310 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:44.342 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:44.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:44.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:02:44.436 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 37 16 56265 56249 95.0013 53.375 0.0124539 0.0105199 2023-10-01T11:02:44.449 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:44.477 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:44.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:44.481 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:02:44.583 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:44.613 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:44.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:44.618 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:02:44.719 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:44.756 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:44.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:44.759 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:02:44.859 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:44.900 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:44.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:44.904 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:02:45.004 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:45.033 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:45.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:45.038 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:02:45.138 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:45.157 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:45.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:45.160 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:02:45.261 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:45.290 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:45.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:45.294 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:02:45.395 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:45.433 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:45.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:45.434 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:02:45.435 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 38 16 57091 57075 93.8597 51.625 0.0397019 0.0106463 2023-10-01T11:02:45.535 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:45.568 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:45.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:45.571 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:02:45.672 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:45.705 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:45.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:45.709 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:02:45.810 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:45.824 INFO:teuthology.orchestra.run.smithi078.stderr:nodeep-scrub is unset 2023-10-01T11:02:45.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:45.852 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:45.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:02:45.953 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:45.984 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:45.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:45.987 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:02:46.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:45 smithi078 bash[75299]: cluster 2023-10-01T11:02:44.219250+0000 mgr.y (mgr.25072) 1361 : cluster 0 pgmap v1422: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s; 9471/163968 objects degraded (5.776%); 6381/163968 objects misplaced (3.892%); 919 KiB/s, 1 keys/s, 15 objects/s recovering 2023-10-01T11:02:46.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:45 smithi078 bash[75299]: audit 2023-10-01T11:02:44.934432+0000 mon.a (mon.0) 1713 : audit 1 from='client.? 172.21.15.78:0/156504321' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:02:46.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:45 smithi078 bash[75299]: audit 2023-10-01T11:02:45.347304+0000 mon.a (mon.0) 1714 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T11:02:46.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:45 smithi078 bash[78274]: cluster 2023-10-01T11:02:44.219250+0000 mgr.y (mgr.25072) 1361 : cluster 0 pgmap v1422: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s; 9471/163968 objects degraded (5.776%); 6381/163968 objects misplaced (3.892%); 919 KiB/s, 1 keys/s, 15 objects/s recovering 2023-10-01T11:02:46.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:45 smithi078 bash[78274]: audit 2023-10-01T11:02:44.934432+0000 mon.a (mon.0) 1713 : audit 1 from='client.? 172.21.15.78:0/156504321' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:02:46.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:45 smithi078 bash[78274]: audit 2023-10-01T11:02:45.347304+0000 mon.a (mon.0) 1714 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T11:02:46.088 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:46.114 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:46.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:45 smithi173 bash[58816]: cluster 2023-10-01T11:02:44.219250+0000 mgr.y (mgr.25072) 1361 : cluster 0 pgmap v1422: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s; 9471/163968 objects degraded (5.776%); 6381/163968 objects misplaced (3.892%); 919 KiB/s, 1 keys/s, 15 objects/s recovering 2023-10-01T11:02:46.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:45 smithi173 bash[58816]: audit 2023-10-01T11:02:44.934432+0000 mon.a (mon.0) 1713 : audit 1 from='client.? 172.21.15.78:0/156504321' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:02:46.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:45 smithi173 bash[58816]: audit 2023-10-01T11:02:45.347304+0000 mon.a (mon.0) 1714 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T11:02:46.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:46.120 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:02:46.222 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:46.255 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:46.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:46.258 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:02:46.359 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:46.386 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:46.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:46.392 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:02:46.435 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 39 16 57927 57911 92.7928 52.25 0.00398056 0.0107601 2023-10-01T11:02:46.494 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:46.522 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:46.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:46.528 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:02:46.603 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:02:46.629 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:46.665 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:46.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:46.667 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:02:46.768 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:46.799 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:46.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:46.803 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:02:46.903 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:46.914 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:46 smithi078 bash[78274]: cluster 2023-10-01T11:02:45.791412+0000 mon.a (mon.0) 1715 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T11:02:46.914 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:46 smithi078 bash[78274]: audit 2023-10-01T11:02:45.811598+0000 mon.a (mon.0) 1716 : audit 1 from='client.? 172.21.15.78:0/156504321' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T11:02:46.914 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:46 smithi078 bash[78274]: cluster 2023-10-01T11:02:45.811637+0000 osd.5 (osd.5) 297 : cluster 3 failed to encode map e298 with expected crc 2023-10-01T11:02:46.914 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:46 smithi078 bash[78274]: cluster 2023-10-01T11:02:45.811666+0000 mon.a (mon.0) 1717 : cluster 0 osdmap e298: 8 total, 8 up, 8 in 2023-10-01T11:02:46.914 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:46 smithi078 bash[78274]: cluster 2023-10-01T11:02:45.813498+0000 osd.7 (osd.7) 311 : cluster 3 failed to encode map e298 with expected crc 2023-10-01T11:02:46.914 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:46 smithi078 bash[78274]: cluster 2023-10-01T11:02:45.819143+0000 osd.6 (osd.6) 275 : cluster 3 failed to encode map e298 with expected crc 2023-10-01T11:02:46.914 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:46 smithi078 bash[75299]: cluster 2023-10-01T11:02:45.791412+0000 mon.a (mon.0) 1715 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T11:02:46.915 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:46 smithi078 bash[75299]: audit 2023-10-01T11:02:45.811598+0000 mon.a (mon.0) 1716 : audit 1 from='client.? 172.21.15.78:0/156504321' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T11:02:46.915 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:46 smithi078 bash[75299]: cluster 2023-10-01T11:02:45.811637+0000 osd.5 (osd.5) 297 : cluster 3 failed to encode map e298 with expected crc 2023-10-01T11:02:46.915 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:46 smithi078 bash[75299]: cluster 2023-10-01T11:02:45.811666+0000 mon.a (mon.0) 1717 : cluster 0 osdmap e298: 8 total, 8 up, 8 in 2023-10-01T11:02:46.915 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:46 smithi078 bash[75299]: cluster 2023-10-01T11:02:45.813498+0000 osd.7 (osd.7) 311 : cluster 3 failed to encode map e298 with expected crc 2023-10-01T11:02:46.915 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:46 smithi078 bash[75299]: cluster 2023-10-01T11:02:45.819143+0000 osd.6 (osd.6) 275 : cluster 3 failed to encode map e298 with expected crc 2023-10-01T11:02:46.929 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:46.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:46.932 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:02:47.033 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:47.055 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:47.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:47.057 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:02:47.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:46 smithi173 bash[58816]: cluster 2023-10-01T11:02:45.791412+0000 mon.a (mon.0) 1715 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T11:02:47.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:46 smithi173 bash[58816]: audit 2023-10-01T11:02:45.811598+0000 mon.a (mon.0) 1716 : audit 1 from='client.? 172.21.15.78:0/156504321' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T11:02:47.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:46 smithi173 bash[58816]: cluster 2023-10-01T11:02:45.811637+0000 osd.5 (osd.5) 297 : cluster 3 failed to encode map e298 with expected crc 2023-10-01T11:02:47.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:46 smithi173 bash[58816]: cluster 2023-10-01T11:02:45.811666+0000 mon.a (mon.0) 1717 : cluster 0 osdmap e298: 8 total, 8 up, 8 in 2023-10-01T11:02:47.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:46 smithi173 bash[58816]: cluster 2023-10-01T11:02:45.813498+0000 osd.7 (osd.7) 311 : cluster 3 failed to encode map e298 with expected crc 2023-10-01T11:02:47.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:46 smithi173 bash[58816]: cluster 2023-10-01T11:02:45.819143+0000 osd.6 (osd.6) 275 : cluster 3 failed to encode map e298 with expected crc 2023-10-01T11:02:47.158 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:47.201 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:47.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:47.203 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:02:47.304 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:47.338 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:47.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:47.342 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:02:47.436 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:2023-10-01T11:02:47.435834+0000 min lat: 0.00136129 max lat: 2.2796 avg lat: 0.0108875 2023-10-01T11:02:47.436 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T11:02:47.436 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 40 16 58772 58756 91.7932 52.8125 0.00735985 0.0108875 2023-10-01T11:02:47.442 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:47.482 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:47.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:47.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:02:47.588 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:47.619 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:47.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:47.623 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:02:47.724 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:47.758 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:47.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:47.760 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:02:47.861 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:47.894 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:47.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:47.897 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:02:47.997 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:48.005 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:47 smithi078 bash[78274]: cluster 2023-10-01T11:02:46.219692+0000 mgr.y (mgr.25072) 1362 : cluster 0 pgmap v1424: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 47 MiB/s wr, 759 op/s; 9471/167262 objects degraded (5.662%); 6168/167262 objects misplaced (3.688%); 2.6 MiB/s, 40 objects/s recovering 2023-10-01T11:02:48.006 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:47 smithi078 bash[78274]: cluster 2023-10-01T11:02:46.805721+0000 mon.a (mon.0) 1718 : cluster 3 Health check update: Degraded data redundancy: 9471/167262 objects degraded (5.662%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T11:02:48.006 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:47 smithi078 bash[78274]: audit 2023-10-01T11:02:47.462876+0000 mon.a (mon.0) 1719 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:02:48.006 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:47 smithi078 bash[78274]: audit 2023-10-01T11:02:47.464060+0000 mon.a (mon.0) 1720 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:02:48.020 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:48.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:48.026 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:02:48.071 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:47 smithi078 bash[75299]: cluster 2023-10-01T11:02:46.219692+0000 mgr.y (mgr.25072) 1362 : cluster 0 pgmap v1424: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 47 MiB/s wr, 759 op/s; 9471/167262 objects degraded (5.662%); 6168/167262 objects misplaced (3.688%); 2.6 MiB/s, 40 objects/s recovering 2023-10-01T11:02:48.071 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:47 smithi078 bash[75299]: cluster 2023-10-01T11:02:46.805721+0000 mon.a (mon.0) 1718 : cluster 3 Health check update: Degraded data redundancy: 9471/167262 objects degraded (5.662%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T11:02:48.071 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:47 smithi078 bash[75299]: audit 2023-10-01T11:02:47.462876+0000 mon.a (mon.0) 1719 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:02:48.071 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:47 smithi078 bash[75299]: audit 2023-10-01T11:02:47.464060+0000 mon.a (mon.0) 1720 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:02:48.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:47 smithi173 bash[58816]: cluster 2023-10-01T11:02:46.219692+0000 mgr.y (mgr.25072) 1362 : cluster 0 pgmap v1424: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 47 MiB/s wr, 759 op/s; 9471/167262 objects degraded (5.662%); 6168/167262 objects misplaced (3.688%); 2.6 MiB/s, 40 objects/s recovering 2023-10-01T11:02:48.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:47 smithi173 bash[58816]: cluster 2023-10-01T11:02:46.805721+0000 mon.a (mon.0) 1718 : cluster 3 Health check update: Degraded data redundancy: 9471/167262 objects degraded (5.662%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T11:02:48.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:47 smithi173 bash[58816]: audit 2023-10-01T11:02:47.462876+0000 mon.a (mon.0) 1719 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:02:48.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:47 smithi173 bash[58816]: audit 2023-10-01T11:02:47.464060+0000 mon.a (mon.0) 1720 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:02:48.128 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:48.174 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:48.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:48.178 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:02:48.278 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:48.313 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:48.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:48.317 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:02:48.418 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:48.436 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 41 16 59608 59592 90.8286 52.25 0.0135858 0.011002 2023-10-01T11:02:48.443 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:48.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:48.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:02:48.548 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:48.576 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:48.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:48.585 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:02:48.685 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:48.711 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:48.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:48.716 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:02:48.817 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:48.844 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:48.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:48.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:02:48.951 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:48.994 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:48.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:48.998 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:02:49.099 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:49.127 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:49.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:49.133 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:02:49.234 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:49.258 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:49.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:49.261 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:02:49.362 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:49.383 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:49.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:49.387 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:02:49.436 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 42 16 60447 60431 89.9145 52.4375 0.0522399 0.0111142 2023-10-01T11:02:49.490 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:49.526 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:49.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:49.537 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:02:49.638 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:49.661 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:49.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:49.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:02:49.764 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:49.796 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:49.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:49.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:02:49.904 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:49.914 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:49 smithi078 bash[75299]: cluster 2023-10-01T11:02:48.220302+0000 mgr.y (mgr.25072) 1363 : cluster 0 pgmap v1425: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 33 MiB/s wr, 531 op/s; 9471/167262 objects degraded (5.662%); 6168/167262 objects misplaced (3.688%); 1.6 MiB/s, 25 objects/s recovering 2023-10-01T11:02:49.914 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:49 smithi078 bash[78274]: cluster 2023-10-01T11:02:48.220302+0000 mgr.y (mgr.25072) 1363 : cluster 0 pgmap v1425: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 33 MiB/s wr, 531 op/s; 9471/167262 objects degraded (5.662%); 6168/167262 objects misplaced (3.688%); 1.6 MiB/s, 25 objects/s recovering 2023-10-01T11:02:49.929 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:49.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:49.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:02:50.035 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:50.044 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:49 smithi173 bash[58816]: cluster 2023-10-01T11:02:48.220302+0000 mgr.y (mgr.25072) 1363 : cluster 0 pgmap v1425: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 33 MiB/s wr, 531 op/s; 9471/167262 objects degraded (5.662%); 6168/167262 objects misplaced (3.688%); 1.6 MiB/s, 25 objects/s recovering 2023-10-01T11:02:50.068 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:50.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:50.075 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:02:50.177 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:50.200 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:50.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:50.204 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:02:50.305 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:50.335 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:50.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:50.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:02:50.436 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 43 16 61287 61271 89.0443 52.5 0.011713 0.0112225 2023-10-01T11:02:50.439 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:50.476 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:50.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:50.482 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:02:50.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:02:50 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:02:50] "GET /metrics HTTP/1.1" 200 36411 "" "Prometheus/2.33.4" 2023-10-01T11:02:50.583 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:50.619 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:50.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:50.626 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:02:50.727 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:50.762 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:50.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:50.765 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:02:50.866 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:50.901 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:50.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:50.908 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:02:51.009 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:51.033 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:51.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:51.037 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:02:51.081 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T11:02:51.127 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T11:02:51.138 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:51.155 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T11:02:51.170 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:51.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:51.176 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:02:51.184 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T11:02:51.217 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T11:02:51.266 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T11:02:51.278 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:51.299 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T11:02:51.303 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:51.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:51.306 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:02:51.340 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T11:02:51.344 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T11:02:51.344 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T11:02:51.344 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T11:02:51.344 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T11:02:51.345 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T11:02:51.345 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T11:02:51.345 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T11:02:51.345 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T11:02:51.407 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:51.432 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:51.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:51.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:02:51.436 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 44 16 62066 62050 88.127 48.6875 0.0312596 0.0113402 2023-10-01T11:02:51.537 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:51.564 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:51.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:51.571 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:02:51.671 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:51.742 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:51.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:51.747 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:02:51.807 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:51 smithi173 bash[58816]: cluster 2023-10-01T11:02:50.221092+0000 mgr.y (mgr.25072) 1364 : cluster 0 pgmap v1426: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 55 MiB/s wr, 872 op/s; 9471/179952 objects degraded (5.263%); 5926/179952 objects misplaced (3.293%); 2.8 MiB/s, 45 objects/s recovering 2023-10-01T11:02:51.807 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:51 smithi173 bash[58816]: audit 2023-10-01T11:02:51.025782+0000 mon.a (mon.0) 1721 : audit 0 from='client.? 172.21.15.78:0/1874430007' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:02:51.849 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:51.877 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:51.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:51.881 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:02:51.982 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:52.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:52.010 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:52.010 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:02:52.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:51 smithi078 bash[78274]: cluster 2023-10-01T11:02:50.221092+0000 mgr.y (mgr.25072) 1364 : cluster 0 pgmap v1426: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 55 MiB/s wr, 872 op/s; 9471/179952 objects degraded (5.263%); 5926/179952 objects misplaced (3.293%); 2.8 MiB/s, 45 objects/s recovering 2023-10-01T11:02:52.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:51 smithi078 bash[78274]: audit 2023-10-01T11:02:51.025782+0000 mon.a (mon.0) 1721 : audit 0 from='client.? 172.21.15.78:0/1874430007' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:02:52.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:51 smithi078 bash[75299]: cluster 2023-10-01T11:02:50.221092+0000 mgr.y (mgr.25072) 1364 : cluster 0 pgmap v1426: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 55 MiB/s wr, 872 op/s; 9471/179952 objects degraded (5.263%); 5926/179952 objects misplaced (3.293%); 2.8 MiB/s, 45 objects/s recovering 2023-10-01T11:02:52.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:51 smithi078 bash[75299]: audit 2023-10-01T11:02:51.025782+0000 mon.a (mon.0) 1721 : audit 0 from='client.? 172.21.15.78:0/1874430007' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:02:52.111 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:52.134 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:52.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:52.137 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:02:52.238 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:52.253 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T11:02:52.288 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:52.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:52.289 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:02:52.390 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:52.436 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 45 16 62798 62782 87.1853 45.75 0.00819063 0.0114616 2023-10-01T11:02:52.457 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:52.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:52.463 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:02:52.564 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:52.589 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:52.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:52.593 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:02:52.693 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:52.726 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:52.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:52.732 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:02:52.833 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:52.846 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:52 smithi173 bash[58816]: cluster 2023-10-01T11:02:52.689296+0000 mon.a (mon.0) 1722 : cluster 3 Health check update: Degraded data redundancy: 9471/179952 objects degraded (5.263%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T11:02:52.846 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:52 smithi173 bash[58816]: audit 2023-10-01T11:02:52.797284+0000 mon.a (mon.0) 1723 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:02:52.846 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:52 smithi173 bash[58816]: audit 2023-10-01T11:02:52.802776+0000 mon.a (mon.0) 1724 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:02:52.859 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:52 smithi078 bash[75299]: cluster 2023-10-01T11:02:52.689296+0000 mon.a (mon.0) 1722 : cluster 3 Health check update: Degraded data redundancy: 9471/179952 objects degraded (5.263%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T11:02:52.859 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:52 smithi078 bash[75299]: audit 2023-10-01T11:02:52.797284+0000 mon.a (mon.0) 1723 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:02:52.860 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:52 smithi078 bash[75299]: audit 2023-10-01T11:02:52.802776+0000 mon.a (mon.0) 1724 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:02:52.860 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:52 smithi078 bash[78274]: cluster 2023-10-01T11:02:52.689296+0000 mon.a (mon.0) 1722 : cluster 3 Health check update: Degraded data redundancy: 9471/179952 objects degraded (5.263%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T11:02:52.860 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:52 smithi078 bash[78274]: audit 2023-10-01T11:02:52.797284+0000 mon.a (mon.0) 1723 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:02:52.860 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:52 smithi078 bash[78274]: audit 2023-10-01T11:02:52.802776+0000 mon.a (mon.0) 1724 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:02:52.876 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:52.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:52.879 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:02:52.980 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:53.019 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:53.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:53.025 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:02:53.126 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:53.157 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:53.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:53.167 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:02:53.268 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:53.293 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:53.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:53.295 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:02:53.396 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:53.422 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:53.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:53.427 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:02:53.436 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 46 16 63460 63444 86.1894 41.375 0.0103338 0.0115957 2023-10-01T11:02:53.528 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:53.556 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:53.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:53.559 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:02:53.660 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:53.685 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:53.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:53.687 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:02:53.788 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:53.826 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:53.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:53.831 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:02:53.932 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:53.940 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:53 smithi078 bash[78274]: cluster 2023-10-01T11:02:52.221573+0000 mgr.y (mgr.25072) 1365 : cluster 0 pgmap v1427: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 55 MiB/s wr, 872 op/s; 9471/179952 objects degraded (5.263%); 5926/179952 objects misplaced (3.293%); 2.8 MiB/s, 45 objects/s recovering 2023-10-01T11:02:53.941 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:53 smithi078 bash[78274]: audit 2023-10-01T11:02:53.215680+0000 mon.a (mon.0) 1725 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:02:53.941 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:53 smithi078 bash[75299]: cluster 2023-10-01T11:02:52.221573+0000 mgr.y (mgr.25072) 1365 : cluster 0 pgmap v1427: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 55 MiB/s wr, 872 op/s; 9471/179952 objects degraded (5.263%); 5926/179952 objects misplaced (3.293%); 2.8 MiB/s, 45 objects/s recovering 2023-10-01T11:02:53.941 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:53 smithi078 bash[75299]: audit 2023-10-01T11:02:53.215680+0000 mon.a (mon.0) 1725 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:02:53.954 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:53.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:53.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:02:54.059 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:54.083 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:54.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:54.085 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:02:54.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:53 smithi173 bash[58816]: cluster 2023-10-01T11:02:52.221573+0000 mgr.y (mgr.25072) 1365 : cluster 0 pgmap v1427: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 55 MiB/s wr, 872 op/s; 9471/179952 objects degraded (5.263%); 5926/179952 objects misplaced (3.293%); 2.8 MiB/s, 45 objects/s recovering 2023-10-01T11:02:54.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:53 smithi173 bash[58816]: audit 2023-10-01T11:02:53.215680+0000 mon.a (mon.0) 1725 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:02:54.186 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:54.230 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:54.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:54.236 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:02:54.337 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:54.365 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:54.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:54.368 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:02:54.436 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 47 16 64278 64262 85.4432 51.125 0.0061413 0.0116955 2023-10-01T11:02:54.469 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:54.502 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:54.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:54.509 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:02:54.610 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:54.649 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:54.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:54.655 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:02:54.756 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:54.783 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:54.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:54.786 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:02:54.887 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:54.932 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:54.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:54.937 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:02:55.038 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:55.069 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:55.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:55.074 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:02:55.175 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:55.206 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:55.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:55.212 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:02:55.314 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:55.340 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:55.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:55.342 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:02:55.436 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 48 16 65088 65072 84.7178 50.625 0.0192707 0.0117967 2023-10-01T11:02:55.444 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:55.478 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:55.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:55.484 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:02:55.585 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:55.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:55.615 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:55.616 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:02:55.616 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:02:55.717 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:55.748 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:55.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:55.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:02:55.861 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:55.869 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:55 smithi078 bash[75299]: cluster 2023-10-01T11:02:54.222283+0000 mgr.y (mgr.25072) 1366 : cluster 0 pgmap v1428: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 51 MiB/s wr, 819 op/s; 9471/188550 objects degraded (5.023%); 5926/188550 objects misplaced (3.143%); 2.8 MiB/s, 45 objects/s recovering 2023-10-01T11:02:55.869 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:55 smithi078 bash[78274]: cluster 2023-10-01T11:02:54.222283+0000 mgr.y (mgr.25072) 1366 : cluster 0 pgmap v1428: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 51 MiB/s wr, 819 op/s; 9471/188550 objects degraded (5.023%); 5926/188550 objects misplaced (3.143%); 2.8 MiB/s, 45 objects/s recovering 2023-10-01T11:02:55.892 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:55.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:55.895 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:02:55.995 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:56.004 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:55 smithi173 bash[58816]: cluster 2023-10-01T11:02:54.222283+0000 mgr.y (mgr.25072) 1366 : cluster 0 pgmap v1428: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 51 MiB/s wr, 819 op/s; 9471/188550 objects degraded (5.023%); 5926/188550 objects misplaced (3.143%); 2.8 MiB/s, 45 objects/s recovering 2023-10-01T11:02:56.031 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:56.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:56.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:02:56.137 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:56.162 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:56.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:56.165 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:02:56.266 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:56.290 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:56.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:56.295 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:02:56.396 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:56.428 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:56.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:56.435 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:02:56.437 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 49 16 65914 65898 84.0424 51.625 0.0413726 0.0118905 2023-10-01T11:02:56.536 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:56.576 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:56.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:56.580 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:02:56.681 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:56.719 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:02:56.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:56.725 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:02:56.827 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:56.857 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:56.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:56.860 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:02:56.961 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:56.988 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:56.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:56.990 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:02:57.091 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:57.119 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:57.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:57.121 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:02:57.222 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:57.255 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:57.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:57.258 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:02:57.359 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:57.384 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:57.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:57.387 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:02:57.437 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 50 16 66651 66635 83.2827 46.0625 0.0308139 0.0120005 2023-10-01T11:02:57.458 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T11:02:57.488 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:57.520 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:57.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:57.524 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:02:57.625 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:57.666 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:57.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:57.669 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:02:57.770 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:57.799 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:57.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:57.806 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:02:57.907 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:57.933 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:57.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:57.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:02:58.039 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:58.080 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:57 smithi078 bash[75299]: cluster 2023-10-01T11:02:56.224051+0000 mgr.y (mgr.25072) 1367 : cluster 0 pgmap v1429: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 48 MiB/s wr, 772 op/s; 9471/191406 objects degraded (4.948%); 5700/191406 objects misplaced (2.978%); 2.8 MiB/s, 44 objects/s recovering 2023-10-01T11:02:58.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:57 smithi078 bash[75299]: cluster 2023-10-01T11:02:57.690248+0000 mon.a (mon.0) 1726 : cluster 3 Health check update: Degraded data redundancy: 9471/191406 objects degraded (4.948%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T11:02:58.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:57 smithi078 bash[78274]: cluster 2023-10-01T11:02:56.224051+0000 mgr.y (mgr.25072) 1367 : cluster 0 pgmap v1429: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 48 MiB/s wr, 772 op/s; 9471/191406 objects degraded (4.948%); 5700/191406 objects misplaced (2.978%); 2.8 MiB/s, 44 objects/s recovering 2023-10-01T11:02:58.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:57 smithi078 bash[78274]: cluster 2023-10-01T11:02:57.690248+0000 mon.a (mon.0) 1726 : cluster 3 Health check update: Degraded data redundancy: 9471/191406 objects degraded (4.948%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T11:02:58.085 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:58.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:58.111 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:02:58.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:57 smithi173 bash[58816]: cluster 2023-10-01T11:02:56.224051+0000 mgr.y (mgr.25072) 1367 : cluster 0 pgmap v1429: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 48 MiB/s wr, 772 op/s; 9471/191406 objects degraded (4.948%); 5700/191406 objects misplaced (2.978%); 2.8 MiB/s, 44 objects/s recovering 2023-10-01T11:02:58.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:57 smithi173 bash[58816]: cluster 2023-10-01T11:02:57.690248+0000 mon.a (mon.0) 1726 : cluster 3 Health check update: Degraded data redundancy: 9471/191406 objects degraded (4.948%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T11:02:58.212 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:58.253 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:58.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:58.255 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:02:58.356 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:58.410 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:02:58.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:58.414 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:02:58.437 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 51 16 67374 67358 82.5357 45.1875 0.019551 0.0121074 2023-10-01T11:02:58.516 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:58.555 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:58.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:58.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:02:58.660 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:58.693 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:58.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:58.703 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:02:58.710 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2023-10-01T11:02:58.710 INFO:teuthology.orchestra.run.smithi078.stdout: "ops": [], 2023-10-01T11:02:58.711 INFO:teuthology.orchestra.run.smithi078.stdout: "complaint_time": 30, 2023-10-01T11:02:58.711 INFO:teuthology.orchestra.run.smithi078.stdout: "num_blocked_ops": 0 2023-10-01T11:02:58.711 INFO:teuthology.orchestra.run.smithi078.stdout:} 2023-10-01T11:02:58.806 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:58.850 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:58.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:58.866 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:02:58.967 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:59.009 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:02:59.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:59.012 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:02:59.113 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:59.165 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:02:59.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:59.171 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:02:59.272 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:59.312 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:59.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:59.313 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:02:59.414 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:59.437 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 52 16 68142 68126 81.8715 48 0.0099873 0.012208 2023-10-01T11:02:59.457 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:59.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:59.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:02:59.568 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:59.607 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:02:59.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:59.609 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:02:59.710 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:59.741 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:02:59.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:59.747 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:02:59.834 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:59 smithi173 bash[58816]: cluster 2023-10-01T11:02:58.224568+0000 mgr.y (mgr.25072) 1368 : cluster 0 pgmap v1430: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 42 MiB/s wr, 670 op/s; 9471/191406 objects degraded (4.948%); 5700/191406 objects misplaced (2.978%); 2.4 MiB/s, 38 objects/s recovering 2023-10-01T11:02:59.834 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:59 smithi173 bash[58816]: audit 2023-10-01T11:02:59.261032+0000 mon.a (mon.0) 1727 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:02:59.834 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:02:59 smithi173 bash[58816]: audit 2023-10-01T11:02:59.269614+0000 mon.a (mon.0) 1728 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:02:59.848 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:59.877 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:02:59.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:02:59.882 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:02:59.982 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:02:59.994 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:59 smithi078 bash[78274]: cluster 2023-10-01T11:02:58.224568+0000 mgr.y (mgr.25072) 1368 : cluster 0 pgmap v1430: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 42 MiB/s wr, 670 op/s; 9471/191406 objects degraded (4.948%); 5700/191406 objects misplaced (2.978%); 2.4 MiB/s, 38 objects/s recovering 2023-10-01T11:02:59.994 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:59 smithi078 bash[78274]: audit 2023-10-01T11:02:59.261032+0000 mon.a (mon.0) 1727 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:02:59.995 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:02:59 smithi078 bash[78274]: audit 2023-10-01T11:02:59.269614+0000 mon.a (mon.0) 1728 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:02:59.995 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:59 smithi078 bash[75299]: cluster 2023-10-01T11:02:58.224568+0000 mgr.y (mgr.25072) 1368 : cluster 0 pgmap v1430: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 42 MiB/s wr, 670 op/s; 9471/191406 objects degraded (4.948%); 5700/191406 objects misplaced (2.978%); 2.4 MiB/s, 38 objects/s recovering 2023-10-01T11:02:59.995 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:59 smithi078 bash[75299]: audit 2023-10-01T11:02:59.261032+0000 mon.a (mon.0) 1727 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:02:59.995 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:02:59 smithi078 bash[75299]: audit 2023-10-01T11:02:59.269614+0000 mon.a (mon.0) 1728 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:03:00.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:00.027 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:00.027 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:03:00.128 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:00.155 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:00.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:00.160 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:03:00.260 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:00.294 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:00.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:00.299 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:03:00.401 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:00.422 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:00.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:00.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:03:00.437 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 53 16 68921 68905 81.2453 48.6875 0.0215036 0.0122999 2023-10-01T11:03:00.526 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:00.564 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:00.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:00.567 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:03:00.621 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:03:00 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:03:00] "GET /metrics HTTP/1.1" 200 36411 "" "Prometheus/2.33.4" 2023-10-01T11:03:00.668 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:00.705 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:00.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:00.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:00.812 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:00.837 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:00 smithi173 bash[58816]: audit 2023-10-01T11:02:59.896504+0000 mon.a (mon.0) 1729 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T11:03:00.845 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:00.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:00.847 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:03:00.947 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:00.972 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:00.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:00.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:01.080 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:01.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:00 smithi078 bash[75299]: audit 2023-10-01T11:02:59.896504+0000 mon.a (mon.0) 1729 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T11:03:01.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:00 smithi078 bash[75299]: audit 2023-10-01T11:02:59.897633+0000 mon.a (mon.0) 1730 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T11:03:01.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:00 smithi078 bash[75299]: audit 2023-10-01T11:02:59.905541+0000 mon.a (mon.0) 1731 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:03:01.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:00 smithi078 bash[78274]: audit 2023-10-01T11:02:59.896504+0000 mon.a (mon.0) 1729 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T11:03:01.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:00 smithi078 bash[78274]: audit 2023-10-01T11:02:59.897633+0000 mon.a (mon.0) 1730 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T11:03:01.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:00 smithi078 bash[78274]: audit 2023-10-01T11:02:59.905541+0000 mon.a (mon.0) 1731 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:03:01.105 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:01.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:01.112 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:03:01.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:00 smithi173 bash[58816]: audit 2023-10-01T11:02:59.897633+0000 mon.a (mon.0) 1730 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T11:03:01.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:00 smithi173 bash[58816]: audit 2023-10-01T11:02:59.905541+0000 mon.a (mon.0) 1731 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:03:01.203 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-01T11:03:01.213 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:01.249 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:01.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:01.254 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:01.354 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:01.379 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:01.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:01.384 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:03:01.437 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 54 16 69566 69550 80.4872 40.3125 0.0397049 0.0124181 2023-10-01T11:03:01.486 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:01.511 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:01.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:01.517 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:03:01.618 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:01.650 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:01.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:01.653 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:03:01.682 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:03:01.754 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:01.785 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:01.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:01.788 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:03:01.889 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:01.919 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:01.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:01.923 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:03:02.024 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:02.059 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:02.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:02.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:02.163 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:02.172 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:01 smithi078 bash[78274]: cluster 2023-10-01T11:03:00.225395+0000 mgr.y (mgr.25072) 1369 : cluster 0 pgmap v1431: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 62 MiB/s wr, 995 op/s; 9471/203121 objects degraded (4.663%); 5465/203121 objects misplaced (2.691%); 3.7 MiB/s, 58 objects/s recovering 2023-10-01T11:03:02.173 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:01 smithi078 bash[75299]: cluster 2023-10-01T11:03:00.225395+0000 mgr.y (mgr.25072) 1369 : cluster 0 pgmap v1431: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 62 MiB/s wr, 995 op/s; 9471/203121 objects degraded (4.663%); 5465/203121 objects misplaced (2.691%); 3.7 MiB/s, 58 objects/s recovering 2023-10-01T11:03:02.188 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:02.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:02.190 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:03:02.291 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:02.301 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:01 smithi173 bash[58816]: cluster 2023-10-01T11:03:00.225395+0000 mgr.y (mgr.25072) 1369 : cluster 0 pgmap v1431: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 62 MiB/s wr, 995 op/s; 9471/203121 objects degraded (4.663%); 5465/203121 objects misplaced (2.691%); 3.7 MiB/s, 58 objects/s recovering 2023-10-01T11:03:02.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:02.338 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:02.339 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:03:02.437 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 55 16 70369 70353 79.9363 50.1875 0.0257058 0.012501 2023-10-01T11:03:02.439 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:02.468 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:02.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:02.479 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:03:02.580 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:02.610 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:02.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:02.612 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:03:02.713 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:02.746 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:02.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:02.750 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:02.851 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:02.895 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:02.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:02.898 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:03:02.999 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:03.009 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:02 smithi078 bash[78274]: audit 2023-10-01T11:03:02.456083+0000 mon.a (mon.0) 1732 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:03:03.009 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:02 smithi078 bash[78274]: cluster 2023-10-01T11:03:02.691303+0000 mon.a (mon.0) 1733 : cluster 3 Health check update: Degraded data redundancy: 9471/203121 objects degraded (4.663%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T11:03:03.010 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:02 smithi078 bash[75299]: audit 2023-10-01T11:03:02.456083+0000 mon.a (mon.0) 1732 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:03:03.010 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:02 smithi078 bash[75299]: cluster 2023-10-01T11:03:02.691303+0000 mon.a (mon.0) 1733 : cluster 3 Health check update: Degraded data redundancy: 9471/203121 objects degraded (4.663%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T11:03:03.028 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:03.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:03.031 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:03:03.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:02 smithi173 bash[58816]: audit 2023-10-01T11:03:02.456083+0000 mon.a (mon.0) 1732 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:03:03.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:02 smithi173 bash[58816]: cluster 2023-10-01T11:03:02.691303+0000 mon.a (mon.0) 1733 : cluster 3 Health check update: Degraded data redundancy: 9471/203121 objects degraded (4.663%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T11:03:03.131 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:03.159 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:03.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:03.161 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:03:03.262 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:03.290 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:03.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:03.297 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:03:03.399 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:03.424 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:03.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:03.428 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:03.437 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 56 16 71125 71109 79.3525 47.25 0.0171657 0.0125958 2023-10-01T11:03:03.529 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:03.560 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:03.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:03.566 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:03:03.613 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:03:03.667 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:03.701 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:03.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:03.703 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:03.804 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:03.828 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:03.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:03.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:03:03.931 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:03.940 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:03 smithi173 bash[58816]: cluster 2023-10-01T11:03:02.225835+0000 mgr.y (mgr.25072) 1370 : cluster 0 pgmap v1432: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 40 MiB/s wr, 643 op/s; 9471/203121 objects degraded (4.663%); 5465/203121 objects misplaced (2.691%); 2.4 MiB/s, 38 objects/s recovering 2023-10-01T11:03:03.968 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:03.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:03.970 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:03:04.071 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:04.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:03 smithi078 bash[75299]: cluster 2023-10-01T11:03:02.225835+0000 mgr.y (mgr.25072) 1370 : cluster 0 pgmap v1432: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 40 MiB/s wr, 643 op/s; 9471/203121 objects degraded (4.663%); 5465/203121 objects misplaced (2.691%); 2.4 MiB/s, 38 objects/s recovering 2023-10-01T11:03:04.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:03 smithi078 bash[78274]: cluster 2023-10-01T11:03:02.225835+0000 mgr.y (mgr.25072) 1370 : cluster 0 pgmap v1432: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 40 MiB/s wr, 643 op/s; 9471/203121 objects degraded (4.663%); 5465/203121 objects misplaced (2.691%); 2.4 MiB/s, 38 objects/s recovering 2023-10-01T11:03:04.127 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:04.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:04.131 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:03:04.233 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:04.269 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:04.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:04.275 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:04.376 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:04.400 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:04.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:04.404 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:03:04.437 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 57 16 71869 71853 78.7761 46.5 0.0190565 0.0126877 2023-10-01T11:03:04.504 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:04.531 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:04.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:04.536 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:03:04.637 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:04.675 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:04.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:04.679 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:04.780 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:04.815 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:04.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:04.820 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:03:04.921 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:04.962 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:04.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:04.966 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:05.067 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:05.094 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:05.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:05.098 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:05.199 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:05.228 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:05.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:05.234 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:03:05.335 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:05.381 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:05.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:05.387 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:05.438 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 58 16 72624 72608 78.2314 47.1875 0.0224549 0.0127748 2023-10-01T11:03:05.488 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:05.515 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:05.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:05.523 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:05.624 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:05.641 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:03:05.651 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:05.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:05.656 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:05.757 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:05.788 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:05.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:05.794 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:03:05.895 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:05.919 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:05 smithi173 bash[58816]: cluster 2023-10-01T11:03:04.226452+0000 mgr.y (mgr.25072) 1371 : cluster 0 pgmap v1433: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 55 MiB/s wr, 878 op/s; 9471/211602 objects degraded (4.476%); 5465/211602 objects misplaced (2.583%); 2.4 MiB/s, 38 objects/s recovering 2023-10-01T11:03:05.919 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:05.922 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:05 smithi078 bash[75299]: cluster 2023-10-01T11:03:04.226452+0000 mgr.y (mgr.25072) 1371 : cluster 0 pgmap v1433: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 55 MiB/s wr, 878 op/s; 9471/211602 objects degraded (4.476%); 5465/211602 objects misplaced (2.583%); 2.4 MiB/s, 38 objects/s recovering 2023-10-01T11:03:05.923 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:05 smithi078 bash[78274]: cluster 2023-10-01T11:03:04.226452+0000 mgr.y (mgr.25072) 1371 : cluster 0 pgmap v1433: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 55 MiB/s wr, 878 op/s; 9471/211602 objects degraded (4.476%); 5465/211602 objects misplaced (2.583%); 2.4 MiB/s, 38 objects/s recovering 2023-10-01T11:03:05.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:05.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:03:06.024 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:06.068 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:06.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:06.071 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:03:06.172 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:06.216 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:06.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:06.220 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:06.321 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:06.351 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:06.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:06.357 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:06.440 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 59 16 73356 73340 77.6809 45.75 0.0321143 0.0128655 2023-10-01T11:03:06.457 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:06.479 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:06.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:06.483 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:03:06.585 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:06.610 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:06.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:06.614 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:03:06.715 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:06.750 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:06.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:06.755 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:03:06.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:06 smithi078 bash[75299]: debug 2023-10-01T11:03:06.609+0000 7f31c8dd0700 -1 mon.a@0(leader).osd e298 definitely_dead 0 2023-10-01T11:03:06.856 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:06.886 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:06.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:06.890 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:03:06.947 INFO:teuthology.orchestra.run.smithi078.stderr:marked out osd.6. 2023-10-01T11:03:06.992 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:07.012 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:07.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:07.013 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:03:07.114 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:07.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:06 smithi173 bash[58816]: audit 2023-10-01T11:03:06.615664+0000 mon.c (mon.2) 208 : audit 1 from='client.? 172.21.15.78:0/297500572' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-01T11:03:07.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:06 smithi173 bash[58816]: audit 2023-10-01T11:03:06.616170+0000 mon.a (mon.0) 1734 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-01T11:03:07.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:06 smithi173 bash[58816]: cluster 2023-10-01T11:03:06.616308+0000 mon.a (mon.0) 1735 : cluster 1 Client client.admin marked osd.6 out, while it was still marked up 2023-10-01T11:03:07.126 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:06 smithi078 bash[75299]: audit 2023-10-01T11:03:06.615664+0000 mon.c (mon.2) 208 : audit 1 from='client.? 172.21.15.78:0/297500572' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-01T11:03:07.127 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:06 smithi078 bash[75299]: audit 2023-10-01T11:03:06.616170+0000 mon.a (mon.0) 1734 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-01T11:03:07.127 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:06 smithi078 bash[75299]: cluster 2023-10-01T11:03:06.616308+0000 mon.a (mon.0) 1735 : cluster 1 Client client.admin marked osd.6 out, while it was still marked up 2023-10-01T11:03:07.127 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:06 smithi078 bash[78274]: audit 2023-10-01T11:03:06.615664+0000 mon.c (mon.2) 208 : audit 1 from='client.? 172.21.15.78:0/297500572' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-01T11:03:07.127 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:06 smithi078 bash[78274]: audit 2023-10-01T11:03:06.616170+0000 mon.a (mon.0) 1734 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-10-01T11:03:07.127 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:06 smithi078 bash[78274]: cluster 2023-10-01T11:03:06.616308+0000 mon.a (mon.0) 1735 : cluster 1 Client client.admin marked osd.6 out, while it was still marked up 2023-10-01T11:03:07.139 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:07.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:07.141 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:03:07.242 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:07.275 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:07.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:07.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:07.379 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:07.401 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:07.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:07.406 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:03:07.437 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:2023-10-01T11:03:07.437683+0000 min lat: 0.00136129 max lat: 2.2796 avg lat: 0.0129157 2023-10-01T11:03:07.438 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T11:03:07.438 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 60 16 73761 73745 76.8081 25.3125 0.00452288 0.0129157 2023-10-01T11:03:07.507 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:07.538 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:07.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:07.541 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:03:07.642 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:07.672 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:07.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:07.675 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:03:07.776 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:07.797 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:07.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:07.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:03:07.900 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:07.926 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:07.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:07.929 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:08.029 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:08.060 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:08.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:08.063 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:03:08.164 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:08.176 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:07 smithi173 bash[58816]: cluster 2023-10-01T11:03:06.226930+0000 mgr.y (mgr.25072) 1372 : cluster 0 pgmap v1434: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 45 MiB/s wr, 716 op/s; 9471/214359 objects degraded (4.418%); 5223/214359 objects misplaced (2.437%); 3.7 MiB/s, 58 objects/s recovering 2023-10-01T11:03:08.176 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:07 smithi173 bash[58816]: cluster 2023-10-01T11:03:06.927859+0000 osd.6 (osd.6) 276 : cluster 3 failed to encode map e299 with expected crc 2023-10-01T11:03:08.176 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:07 smithi173 bash[58816]: cluster 2023-10-01T11:03:06.928816+0000 osd.7 (osd.7) 312 : cluster 3 failed to encode map e299 with expected crc 2023-10-01T11:03:08.177 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:07 smithi173 bash[58816]: audit 2023-10-01T11:03:06.930585+0000 mon.a (mon.0) 1736 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-10-01T11:03:08.177 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:07 smithi173 bash[58816]: cluster 2023-10-01T11:03:06.930653+0000 mon.a (mon.0) 1737 : cluster 0 osdmap e299: 8 total, 8 up, 7 in 2023-10-01T11:03:08.177 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:07 smithi173 bash[58816]: cluster 2023-10-01T11:03:06.934038+0000 osd.5 (osd.5) 298 : cluster 3 failed to encode map e299 with expected crc 2023-10-01T11:03:08.177 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:07 smithi173 bash[58816]: cluster 2023-10-01T11:03:07.692445+0000 mon.a (mon.0) 1738 : cluster 3 Health check update: Degraded data redundancy: 9471/214359 objects degraded (4.418%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T11:03:08.177 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:07 smithi173 bash[58816]: cluster 2023-10-01T11:03:07.700158+0000 mon.a (mon.0) 1739 : cluster 0 osdmap e300: 8 total, 8 up, 7 in 2023-10-01T11:03:08.193 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:08.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:08.195 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:03:08.296 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:08.320 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:08.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:08.322 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:08.336 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:07 smithi078 bash[78274]: cluster 2023-10-01T11:03:06.226930+0000 mgr.y (mgr.25072) 1372 : cluster 0 pgmap v1434: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 45 MiB/s wr, 716 op/s; 9471/214359 objects degraded (4.418%); 5223/214359 objects misplaced (2.437%); 3.7 MiB/s, 58 objects/s recovering 2023-10-01T11:03:08.337 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:07 smithi078 bash[78274]: cluster 2023-10-01T11:03:06.927859+0000 osd.6 (osd.6) 276 : cluster 3 failed to encode map e299 with expected crc 2023-10-01T11:03:08.337 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:07 smithi078 bash[78274]: cluster 2023-10-01T11:03:06.928816+0000 osd.7 (osd.7) 312 : cluster 3 failed to encode map e299 with expected crc 2023-10-01T11:03:08.337 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:07 smithi078 bash[78274]: audit 2023-10-01T11:03:06.930585+0000 mon.a (mon.0) 1736 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-10-01T11:03:08.337 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:07 smithi078 bash[78274]: cluster 2023-10-01T11:03:06.930653+0000 mon.a (mon.0) 1737 : cluster 0 osdmap e299: 8 total, 8 up, 7 in 2023-10-01T11:03:08.337 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:07 smithi078 bash[78274]: cluster 2023-10-01T11:03:06.934038+0000 osd.5 (osd.5) 298 : cluster 3 failed to encode map e299 with expected crc 2023-10-01T11:03:08.337 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:07 smithi078 bash[78274]: cluster 2023-10-01T11:03:07.692445+0000 mon.a (mon.0) 1738 : cluster 3 Health check update: Degraded data redundancy: 9471/214359 objects degraded (4.418%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T11:03:08.337 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:07 smithi078 bash[78274]: cluster 2023-10-01T11:03:07.700158+0000 mon.a (mon.0) 1739 : cluster 0 osdmap e300: 8 total, 8 up, 7 in 2023-10-01T11:03:08.337 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:07 smithi078 bash[75299]: cluster 2023-10-01T11:03:06.226930+0000 mgr.y (mgr.25072) 1372 : cluster 0 pgmap v1434: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 45 MiB/s wr, 716 op/s; 9471/214359 objects degraded (4.418%); 5223/214359 objects misplaced (2.437%); 3.7 MiB/s, 58 objects/s recovering 2023-10-01T11:03:08.338 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:07 smithi078 bash[75299]: cluster 2023-10-01T11:03:06.927859+0000 osd.6 (osd.6) 276 : cluster 3 failed to encode map e299 with expected crc 2023-10-01T11:03:08.338 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:07 smithi078 bash[75299]: cluster 2023-10-01T11:03:06.928816+0000 osd.7 (osd.7) 312 : cluster 3 failed to encode map e299 with expected crc 2023-10-01T11:03:08.338 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:07 smithi078 bash[75299]: audit 2023-10-01T11:03:06.930585+0000 mon.a (mon.0) 1736 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-10-01T11:03:08.338 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:07 smithi078 bash[75299]: cluster 2023-10-01T11:03:06.930653+0000 mon.a (mon.0) 1737 : cluster 0 osdmap e299: 8 total, 8 up, 7 in 2023-10-01T11:03:08.338 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:07 smithi078 bash[75299]: cluster 2023-10-01T11:03:06.934038+0000 osd.5 (osd.5) 298 : cluster 3 failed to encode map e299 with expected crc 2023-10-01T11:03:08.338 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:07 smithi078 bash[75299]: cluster 2023-10-01T11:03:07.692445+0000 mon.a (mon.0) 1738 : cluster 3 Health check update: Degraded data redundancy: 9471/214359 objects degraded (4.418%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T11:03:08.338 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:07 smithi078 bash[75299]: cluster 2023-10-01T11:03:07.700158+0000 mon.a (mon.0) 1739 : cluster 0 osdmap e300: 8 total, 8 up, 7 in 2023-10-01T11:03:08.423 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:08.437 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 61 16 73761 73745 75.5489 0 - 0.0129157 2023-10-01T11:03:08.448 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:08.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:08.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:08.552 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:08.583 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:08.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:08.589 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:08.689 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:08.714 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:08.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:08.716 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:03:08.817 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:08.841 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:08.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:08.843 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:08.944 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:08.971 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:08.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:08.974 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:03:09.075 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:09.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:08 smithi078 bash[75299]: cluster 2023-10-01T11:03:06.943482+0000 osd.5 (osd.5) 299 : cluster 3 failed to encode map e299 with expected crc 2023-10-01T11:03:09.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:08 smithi078 bash[75299]: cluster 2023-10-01T11:03:06.945528+0000 osd.5 (osd.5) 300 : cluster 3 failed to encode map e299 with expected crc 2023-10-01T11:03:09.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:08 smithi078 bash[75299]: cluster 2023-10-01T11:03:07.697767+0000 osd.7 (osd.7) 313 : cluster 3 failed to encode map e300 with expected crc 2023-10-01T11:03:09.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:08 smithi078 bash[75299]: cluster 2023-10-01T11:03:07.701446+0000 osd.6 (osd.6) 277 : cluster 3 failed to encode map e300 with expected crc 2023-10-01T11:03:09.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:08 smithi078 bash[75299]: cluster 2023-10-01T11:03:07.701471+0000 osd.5 (osd.5) 301 : cluster 3 failed to encode map e300 with expected crc 2023-10-01T11:03:09.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:08 smithi078 bash[75299]: cluster 2023-10-01T11:03:07.706147+0000 osd.5 (osd.5) 302 : cluster 3 failed to encode map e300 with expected crc 2023-10-01T11:03:09.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:08 smithi078 bash[75299]: cluster 2023-10-01T11:03:07.707959+0000 osd.7 (osd.7) 314 : cluster 3 failed to encode map e300 with expected crc 2023-10-01T11:03:09.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:08 smithi078 bash[75299]: cluster 2023-10-01T11:03:07.708390+0000 osd.6 (osd.6) 278 : cluster 3 failed to encode map e300 with expected crc 2023-10-01T11:03:09.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:08 smithi078 bash[75299]: cluster 2023-10-01T11:03:07.709328+0000 osd.5 (osd.5) 303 : cluster 3 failed to encode map e300 with expected crc 2023-10-01T11:03:09.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:08 smithi078 bash[75299]: cluster 2023-10-01T11:03:07.709896+0000 osd.7 (osd.7) 315 : cluster 3 failed to encode map e300 with expected crc 2023-10-01T11:03:09.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:08 smithi078 bash[75299]: cluster 2023-10-01T11:03:07.711677+0000 osd.5 (osd.5) 304 : cluster 3 failed to encode map e300 with expected crc 2023-10-01T11:03:09.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:08 smithi078 bash[75299]: cluster 2023-10-01T11:03:08.722060+0000 mon.a (mon.0) 1740 : cluster 0 osdmap e301: 8 total, 8 up, 7 in 2023-10-01T11:03:09.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:08 smithi078 bash[78274]: cluster 2023-10-01T11:03:06.943482+0000 osd.5 (osd.5) 299 : cluster 3 failed to encode map e299 with expected crc 2023-10-01T11:03:09.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:08 smithi078 bash[78274]: cluster 2023-10-01T11:03:06.945528+0000 osd.5 (osd.5) 300 : cluster 3 failed to encode map e299 with expected crc 2023-10-01T11:03:09.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:08 smithi078 bash[78274]: cluster 2023-10-01T11:03:07.697767+0000 osd.7 (osd.7) 313 : cluster 3 failed to encode map e300 with expected crc 2023-10-01T11:03:09.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:08 smithi078 bash[78274]: cluster 2023-10-01T11:03:07.701446+0000 osd.6 (osd.6) 277 : cluster 3 failed to encode map e300 with expected crc 2023-10-01T11:03:09.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:08 smithi078 bash[78274]: cluster 2023-10-01T11:03:07.701471+0000 osd.5 (osd.5) 301 : cluster 3 failed to encode map e300 with expected crc 2023-10-01T11:03:09.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:08 smithi078 bash[78274]: cluster 2023-10-01T11:03:07.706147+0000 osd.5 (osd.5) 302 : cluster 3 failed to encode map e300 with expected crc 2023-10-01T11:03:09.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:08 smithi078 bash[78274]: cluster 2023-10-01T11:03:07.707959+0000 osd.7 (osd.7) 314 : cluster 3 failed to encode map e300 with expected crc 2023-10-01T11:03:09.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:08 smithi078 bash[78274]: cluster 2023-10-01T11:03:07.708390+0000 osd.6 (osd.6) 278 : cluster 3 failed to encode map e300 with expected crc 2023-10-01T11:03:09.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:08 smithi078 bash[78274]: cluster 2023-10-01T11:03:07.709328+0000 osd.5 (osd.5) 303 : cluster 3 failed to encode map e300 with expected crc 2023-10-01T11:03:09.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:08 smithi078 bash[78274]: cluster 2023-10-01T11:03:07.709896+0000 osd.7 (osd.7) 315 : cluster 3 failed to encode map e300 with expected crc 2023-10-01T11:03:09.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:08 smithi078 bash[78274]: cluster 2023-10-01T11:03:07.711677+0000 osd.5 (osd.5) 304 : cluster 3 failed to encode map e300 with expected crc 2023-10-01T11:03:09.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:08 smithi078 bash[78274]: cluster 2023-10-01T11:03:08.722060+0000 mon.a (mon.0) 1740 : cluster 0 osdmap e301: 8 total, 8 up, 7 in 2023-10-01T11:03:09.084 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:08 smithi173 bash[58816]: cluster 2023-10-01T11:03:06.943482+0000 osd.5 (osd.5) 299 : cluster 3 failed to encode map e299 with expected crc 2023-10-01T11:03:09.084 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:08 smithi173 bash[58816]: cluster 2023-10-01T11:03:06.945528+0000 osd.5 (osd.5) 300 : cluster 3 failed to encode map e299 with expected crc 2023-10-01T11:03:09.084 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:08 smithi173 bash[58816]: cluster 2023-10-01T11:03:07.697767+0000 osd.7 (osd.7) 313 : cluster 3 failed to encode map e300 with expected crc 2023-10-01T11:03:09.085 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:08 smithi173 bash[58816]: cluster 2023-10-01T11:03:07.701446+0000 osd.6 (osd.6) 277 : cluster 3 failed to encode map e300 with expected crc 2023-10-01T11:03:09.085 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:08 smithi173 bash[58816]: cluster 2023-10-01T11:03:07.701471+0000 osd.5 (osd.5) 301 : cluster 3 failed to encode map e300 with expected crc 2023-10-01T11:03:09.085 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:08 smithi173 bash[58816]: cluster 2023-10-01T11:03:07.706147+0000 osd.5 (osd.5) 302 : cluster 3 failed to encode map e300 with expected crc 2023-10-01T11:03:09.085 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:08 smithi173 bash[58816]: cluster 2023-10-01T11:03:07.707959+0000 osd.7 (osd.7) 314 : cluster 3 failed to encode map e300 with expected crc 2023-10-01T11:03:09.085 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:08 smithi173 bash[58816]: cluster 2023-10-01T11:03:07.708390+0000 osd.6 (osd.6) 278 : cluster 3 failed to encode map e300 with expected crc 2023-10-01T11:03:09.085 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:08 smithi173 bash[58816]: cluster 2023-10-01T11:03:07.709328+0000 osd.5 (osd.5) 303 : cluster 3 failed to encode map e300 with expected crc 2023-10-01T11:03:09.085 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:08 smithi173 bash[58816]: cluster 2023-10-01T11:03:07.709896+0000 osd.7 (osd.7) 315 : cluster 3 failed to encode map e300 with expected crc 2023-10-01T11:03:09.085 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:08 smithi173 bash[58816]: cluster 2023-10-01T11:03:07.711677+0000 osd.5 (osd.5) 304 : cluster 3 failed to encode map e300 with expected crc 2023-10-01T11:03:09.085 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:08 smithi173 bash[58816]: cluster 2023-10-01T11:03:08.722060+0000 mon.a (mon.0) 1740 : cluster 0 osdmap e301: 8 total, 8 up, 7 in 2023-10-01T11:03:09.100 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:09.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:09.103 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:09.203 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:09.221 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:09.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:09.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:03:09.324 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:09.349 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:09.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:09.352 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:03:09.438 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 62 16 73797 73781 74.3667 1.125 0.0077104 0.0130819 2023-10-01T11:03:09.452 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:09.477 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:09.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:09.480 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:09.581 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:09.600 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:09.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:09.602 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:03:09.702 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:09.729 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:09.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:09.732 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:09.832 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:09.854 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:09.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:09.857 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:03:09.958 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:09.982 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:09.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:09.984 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:03:10.085 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:10.096 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:09 smithi173 bash[58816]: cluster 2023-10-01T11:03:08.227423+0000 mgr.y (mgr.25072) 1373 : cluster 0 pgmap v1437: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 29 MiB/s wr, 468 op/s; 9471/214359 objects degraded (4.418%); 5223/214359 objects misplaced (2.437%); 1.9 MiB/s, 30 objects/s recovering 2023-10-01T11:03:10.096 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:09 smithi173 bash[58816]: cluster 2023-10-01T11:03:08.719793+0000 osd.7 (osd.7) 316 : cluster 3 failed to encode map e301 with expected crc 2023-10-01T11:03:10.096 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:09 smithi173 bash[58816]: cluster 2023-10-01T11:03:08.723008+0000 osd.7 (osd.7) 317 : cluster 3 failed to encode map e301 with expected crc 2023-10-01T11:03:10.096 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:09 smithi173 bash[58816]: cluster 2023-10-01T11:03:08.724536+0000 osd.7 (osd.7) 318 : cluster 3 failed to encode map e301 with expected crc 2023-10-01T11:03:10.096 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:09 smithi173 bash[58816]: cluster 2023-10-01T11:03:08.782499+0000 osd.5 (osd.5) 305 : cluster 3 failed to encode map e301 with expected crc 2023-10-01T11:03:10.110 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:10.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:10.113 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:10.213 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:10.220 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:09 smithi078 bash[78274]: cluster 2023-10-01T11:03:08.227423+0000 mgr.y (mgr.25072) 1373 : cluster 0 pgmap v1437: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 29 MiB/s wr, 468 op/s; 9471/214359 objects degraded (4.418%); 5223/214359 objects misplaced (2.437%); 1.9 MiB/s, 30 objects/s recovering 2023-10-01T11:03:10.220 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:09 smithi078 bash[78274]: cluster 2023-10-01T11:03:08.719793+0000 osd.7 (osd.7) 316 : cluster 3 failed to encode map e301 with expected crc 2023-10-01T11:03:10.221 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:09 smithi078 bash[78274]: cluster 2023-10-01T11:03:08.723008+0000 osd.7 (osd.7) 317 : cluster 3 failed to encode map e301 with expected crc 2023-10-01T11:03:10.221 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:09 smithi078 bash[78274]: cluster 2023-10-01T11:03:08.724536+0000 osd.7 (osd.7) 318 : cluster 3 failed to encode map e301 with expected crc 2023-10-01T11:03:10.221 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:09 smithi078 bash[78274]: cluster 2023-10-01T11:03:08.782499+0000 osd.5 (osd.5) 305 : cluster 3 failed to encode map e301 with expected crc 2023-10-01T11:03:10.221 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:09 smithi078 bash[75299]: cluster 2023-10-01T11:03:08.227423+0000 mgr.y (mgr.25072) 1373 : cluster 0 pgmap v1437: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 29 MiB/s wr, 468 op/s; 9471/214359 objects degraded (4.418%); 5223/214359 objects misplaced (2.437%); 1.9 MiB/s, 30 objects/s recovering 2023-10-01T11:03:10.221 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:09 smithi078 bash[75299]: cluster 2023-10-01T11:03:08.719793+0000 osd.7 (osd.7) 316 : cluster 3 failed to encode map e301 with expected crc 2023-10-01T11:03:10.221 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:09 smithi078 bash[75299]: cluster 2023-10-01T11:03:08.723008+0000 osd.7 (osd.7) 317 : cluster 3 failed to encode map e301 with expected crc 2023-10-01T11:03:10.222 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:09 smithi078 bash[75299]: cluster 2023-10-01T11:03:08.724536+0000 osd.7 (osd.7) 318 : cluster 3 failed to encode map e301 with expected crc 2023-10-01T11:03:10.222 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:09 smithi078 bash[75299]: cluster 2023-10-01T11:03:08.782499+0000 osd.5 (osd.5) 305 : cluster 3 failed to encode map e301 with expected crc 2023-10-01T11:03:10.231 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:10.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:10.233 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:03:10.334 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:10.359 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:10.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:10.361 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:10.438 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 63 16 73797 73781 73.1863 0 - 0.0130819 2023-10-01T11:03:10.462 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:10.480 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:03:10 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:03:10] "GET /metrics HTTP/1.1" 200 36399 "" "Prometheus/2.33.4" 2023-10-01T11:03:10.480 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:10.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:10.482 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:03:10.583 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:10.630 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:10.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:10.633 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:03:10.734 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:10.762 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:10.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:10.765 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:03:10.866 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:10.883 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:10.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:10.885 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:03:10.986 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:11.014 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:11.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:11.018 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:11.119 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:11.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:11.150 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:11.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:03:11.250 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:11.272 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:11.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:11.275 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:03:11.375 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:11.403 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:11.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:11.407 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:03:11.438 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 64 16 73797 73781 72.0428 0 - 0.0130819 2023-10-01T11:03:11.507 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:11.537 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:11.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:11.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:03:11.639 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:11.662 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:11.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:11.663 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:03:11.764 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:11.791 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:11.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:11.793 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:11.894 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:11.923 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:11.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:11.925 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:12.026 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:12.036 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:11 smithi173 bash[58816]: cluster 2023-10-01T11:03:10.228352+0000 mgr.y (mgr.25072) 1374 : cluster 0 pgmap v1439: 89 pgs: 1 peering, 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 25 MiB/s wr, 401 op/s; 6335/218829 objects degraded (2.895%); 4981/218829 objects misplaced (2.276%); 3.8 MiB/s, 61 objects/s recovering 2023-10-01T11:03:12.049 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:12.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:12.052 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:12.152 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:12.175 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:12.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:12.177 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:03:12.278 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:12.305 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:12.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:12.312 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:12.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:11 smithi078 bash[78274]: cluster 2023-10-01T11:03:10.228352+0000 mgr.y (mgr.25072) 1374 : cluster 0 pgmap v1439: 89 pgs: 1 peering, 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 25 MiB/s wr, 401 op/s; 6335/218829 objects degraded (2.895%); 4981/218829 objects misplaced (2.276%); 3.8 MiB/s, 61 objects/s recovering 2023-10-01T11:03:12.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:11 smithi078 bash[75299]: cluster 2023-10-01T11:03:10.228352+0000 mgr.y (mgr.25072) 1374 : cluster 0 pgmap v1439: 89 pgs: 1 peering, 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 25 MiB/s wr, 401 op/s; 6335/218829 objects degraded (2.895%); 4981/218829 objects misplaced (2.276%); 3.8 MiB/s, 61 objects/s recovering 2023-10-01T11:03:12.412 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:12.438 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 65 16 73797 73781 70.9344 0 - 0.0130819 2023-10-01T11:03:12.439 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:12.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:12.442 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:12.542 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:12.568 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:12.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:12.570 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:03:12.671 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:12.695 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:12.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:12.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:03:12.798 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:12.823 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:12.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:12.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:03:12.926 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:12.958 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:12.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:12.960 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:03:13.061 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:13.062 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:12 smithi078 bash[78274]: cluster 2023-10-01T11:03:12.694518+0000 mon.a (mon.0) 1741 : cluster 3 Health check update: Degraded data redundancy: 6335/218829 objects degraded (2.895%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T11:03:13.062 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:12 smithi078 bash[75299]: cluster 2023-10-01T11:03:12.694518+0000 mon.a (mon.0) 1741 : cluster 3 Health check update: Degraded data redundancy: 6335/218829 objects degraded (2.895%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T11:03:13.082 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:13.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:13.084 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:03:13.185 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:13.211 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:13.213 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:12 smithi173 bash[58816]: cluster 2023-10-01T11:03:12.694518+0000 mon.a (mon.0) 1741 : cluster 3 Health check update: Degraded data redundancy: 6335/218829 objects degraded (2.895%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T11:03:13.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:13.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:13.314 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:13.345 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:13.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:13.347 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:03:13.438 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 66 16 73801 73785 69.8634 0.0625 0.00361467 0.0131688 2023-10-01T11:03:13.448 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:13.465 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:13.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:13.467 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:03:13.568 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:13.587 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:13.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:13.589 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:03:13.690 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:13.713 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:13.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:13.716 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:03:13.817 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:13.839 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:13.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:13.842 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:03:13.943 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:13.966 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:13.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:13.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:03:14.068 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:14.089 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:14.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:14.091 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:03:14.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:13 smithi173 bash[58816]: cluster 2023-10-01T11:03:12.229104+0000 mgr.y (mgr.25072) 1375 : cluster 0 pgmap v1440: 89 pgs: 1 peering, 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 16 MiB/s wr, 248 op/s; 6335/218829 objects degraded (2.895%); 4981/218829 objects misplaced (2.276%); 1.3 MiB/s, 21 objects/s recovering 2023-10-01T11:03:14.192 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:14.213 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:14.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:14.215 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:03:14.315 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:14.322 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:13 smithi078 bash[75299]: cluster 2023-10-01T11:03:12.229104+0000 mgr.y (mgr.25072) 1375 : cluster 0 pgmap v1440: 89 pgs: 1 peering, 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 16 MiB/s wr, 248 op/s; 6335/218829 objects degraded (2.895%); 4981/218829 objects misplaced (2.276%); 1.3 MiB/s, 21 objects/s recovering 2023-10-01T11:03:14.322 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:13 smithi078 bash[78274]: cluster 2023-10-01T11:03:12.229104+0000 mgr.y (mgr.25072) 1375 : cluster 0 pgmap v1440: 89 pgs: 1 peering, 1 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 16 MiB/s wr, 248 op/s; 6335/218829 objects degraded (2.895%); 4981/218829 objects misplaced (2.276%); 1.3 MiB/s, 21 objects/s recovering 2023-10-01T11:03:14.333 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:14.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:14.335 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:03:14.435 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:14.438 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 67 16 73900 73884 68.9131 6.1875 0.00448433 0.0138738 2023-10-01T11:03:14.466 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:14.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:14.468 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:03:14.569 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:14.599 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:14.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:14.602 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:14.703 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:14.721 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:14.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:14.723 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:03:14.823 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:14.849 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:14.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:14.852 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:03:14.953 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:14.975 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:14.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:14.978 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:15.078 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:15.100 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:15.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:15.102 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:03:15.203 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:15.230 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:15.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:15.232 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:15.332 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:15.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:15.367 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:15.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:03:15.439 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 68 16 73900 73884 67.8997 0 - 0.0138738 2023-10-01T11:03:15.468 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:15.501 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:15.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:15.506 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:03:15.607 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:15.642 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:15.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:15.645 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:15.746 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:15.778 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:15.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:15.780 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:03:15.880 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:15.916 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:15.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:15.918 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:03:16.019 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:16.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:16.043 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:16.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:03:16.072 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 7] out_osds: [6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T11:03:16.073 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T11:03:16.073 INFO:tasks.thrashosds.thrasher:Removing osd 7, in_osds are: [0, 1, 2, 3, 4, 5, 7] 2023-10-01T11:03:16.073 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd out 7 2023-10-01T11:03:16.146 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:16.160 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:15 smithi173 bash[58816]: cluster 2023-10-01T11:03:14.229717+0000 mgr.y (mgr.25072) 1376 : cluster 0 pgmap v1441: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 83 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 13 MiB/s wr, 206 op/s; 9594/218892 objects degraded (4.383%); 4981/218892 objects misplaced (2.276%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T11:03:16.176 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:16.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:16.179 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:03:16.279 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:16.313 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:16.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:16.316 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:03:16.330 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:15 smithi078 bash[75299]: cluster 2023-10-01T11:03:14.229717+0000 mgr.y (mgr.25072) 1376 : cluster 0 pgmap v1441: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 83 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 13 MiB/s wr, 206 op/s; 9594/218892 objects degraded (4.383%); 4981/218892 objects misplaced (2.276%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T11:03:16.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:15 smithi078 bash[78274]: cluster 2023-10-01T11:03:14.229717+0000 mgr.y (mgr.25072) 1376 : cluster 0 pgmap v1441: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 83 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 13 MiB/s wr, 206 op/s; 9594/218892 objects degraded (4.383%); 4981/218892 objects misplaced (2.276%); 1.1 MiB/s, 17 objects/s recovering 2023-10-01T11:03:16.419 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:16.440 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 69 16 73900 73884 66.9156 0 - 0.0138738 2023-10-01T11:03:16.443 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:16.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:16.452 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:03:16.553 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:16.582 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:16.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:16.585 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:03:16.686 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:16.707 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:16.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:16.708 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:03:16.810 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:16.839 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:16.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:16.842 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:03:16.943 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:16.969 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:16.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:16.973 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:03:17.074 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:17.100 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:17.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:17.103 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:17.203 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:17.230 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:17.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:17.235 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:17.335 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:17.362 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:17.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:17.365 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:17.439 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 70 16 73900 73884 65.9597 0 - 0.0138738 2023-10-01T11:03:17.466 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:17.486 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:17.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:17.488 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:03:17.589 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:17.611 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:17.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:17.615 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:03:17.715 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:17.746 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:17.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:17.750 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:17.851 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:17.881 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:17.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:17.887 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:03:17.983 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T11:03:17.988 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:17.998 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:17 smithi173 bash[58816]: cluster 2023-10-01T11:03:16.230123+0000 mgr.y (mgr.25072) 1377 : cluster 0 pgmap v1442: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 83 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 11 MiB/s wr, 177 op/s; 9594/218913 objects degraded (4.383%); 4553/218913 objects misplaced (2.080%); 4.0 MiB/s, 65 objects/s recovering 2023-10-01T11:03:17.998 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:17 smithi173 bash[58816]: audit 2023-10-01T11:03:17.456099+0000 mon.a (mon.0) 1742 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:03:17.998 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:17 smithi173 bash[58816]: cluster 2023-10-01T11:03:17.695380+0000 mon.a (mon.0) 1743 : cluster 3 Health check update: Degraded data redundancy: 9594/218913 objects degraded (4.383%), 4 pgs degraded (PG_DEGRADED) 2023-10-01T11:03:17.998 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:17 smithi173 bash[58816]: audit 2023-10-01T11:03:17.944683+0000 mon.c (mon.2) 209 : audit 0 from='client.? 172.21.15.78:0/2550680626' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:03:18.016 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:18.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:18.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:18.019 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T11:03:18.051 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T11:03:18.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:17 smithi078 bash[75299]: cluster 2023-10-01T11:03:16.230123+0000 mgr.y (mgr.25072) 1377 : cluster 0 pgmap v1442: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 83 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 11 MiB/s wr, 177 op/s; 9594/218913 objects degraded (4.383%); 4553/218913 objects misplaced (2.080%); 4.0 MiB/s, 65 objects/s recovering 2023-10-01T11:03:18.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:17 smithi078 bash[75299]: audit 2023-10-01T11:03:17.456099+0000 mon.a (mon.0) 1742 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:03:18.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:17 smithi078 bash[75299]: cluster 2023-10-01T11:03:17.695380+0000 mon.a (mon.0) 1743 : cluster 3 Health check update: Degraded data redundancy: 9594/218913 objects degraded (4.383%), 4 pgs degraded (PG_DEGRADED) 2023-10-01T11:03:18.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:17 smithi078 bash[75299]: audit 2023-10-01T11:03:17.944683+0000 mon.c (mon.2) 209 : audit 0 from='client.? 172.21.15.78:0/2550680626' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:03:18.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:17 smithi078 bash[78274]: cluster 2023-10-01T11:03:16.230123+0000 mgr.y (mgr.25072) 1377 : cluster 0 pgmap v1442: 89 pgs: 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 83 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 11 MiB/s wr, 177 op/s; 9594/218913 objects degraded (4.383%); 4553/218913 objects misplaced (2.080%); 4.0 MiB/s, 65 objects/s recovering 2023-10-01T11:03:18.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:17 smithi078 bash[78274]: audit 2023-10-01T11:03:17.456099+0000 mon.a (mon.0) 1742 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:03:18.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:17 smithi078 bash[78274]: cluster 2023-10-01T11:03:17.695380+0000 mon.a (mon.0) 1743 : cluster 3 Health check update: Degraded data redundancy: 9594/218913 objects degraded (4.383%), 4 pgs degraded (PG_DEGRADED) 2023-10-01T11:03:18.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:17 smithi078 bash[78274]: audit 2023-10-01T11:03:17.944683+0000 mon.c (mon.2) 209 : audit 0 from='client.? 172.21.15.78:0/2550680626' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:03:18.083 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T11:03:18.119 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:18.122 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T11:03:18.149 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:18.150 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T11:03:18.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:18.155 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:03:18.173 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T11:03:18.204 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T11:03:18.206 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T11:03:18.206 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T11:03:18.206 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T11:03:18.206 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T11:03:18.206 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T11:03:18.207 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T11:03:18.207 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T11:03:18.207 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T11:03:18.255 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:18.288 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:18.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:18.292 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:18.308 INFO:teuthology.orchestra.run.smithi078.stdout:ERROR: (22) Invalid argument 2023-10-01T11:03:18.308 INFO:teuthology.orchestra.run.smithi078.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T11:03:18.392 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:18.416 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:18.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:18.419 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:03:18.439 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 71 16 74333 74317 65.4119 6.76562 0.45266 0.0152443 2023-10-01T11:03:18.520 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:18.544 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:18.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:18.552 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:03:18.652 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:18.675 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:18.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:18.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:03:18.781 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:18.808 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:18.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:18.815 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:18.916 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:18.946 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:18.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:18.948 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:03:19.048 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:19.070 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:19.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:19.072 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:03:19.173 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:19.198 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:19.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:19.201 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:03:19.248 INFO:teuthology.orchestra.run.smithi078.stderr:noscrub is set 2023-10-01T11:03:19.248 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:19 smithi078 bash[75299]: cluster 2023-10-01T11:03:18.230858+0000 mgr.y (mgr.25072) 1378 : cluster 0 pgmap v1443: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 15 MiB/s wr, 240 op/s; 26681/221574 objects degraded (12.042%); 2588/221574 objects misplaced (1.168%); 4.7 MiB/s, 76 objects/s recovering 2023-10-01T11:03:19.248 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:19 smithi078 bash[75299]: audit 2023-10-01T11:03:18.241476+0000 mon.a (mon.0) 1744 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:03:19.248 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:19 smithi078 bash[75299]: audit 2023-10-01T11:03:18.279096+0000 mon.a (mon.0) 1745 : audit 1 from='client.? 172.21.15.78:0/2179208199' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T11:03:19.301 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:19.324 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:19.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:19.327 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:03:19.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:19 smithi173 bash[58816]: cluster 2023-10-01T11:03:18.230858+0000 mgr.y (mgr.25072) 1378 : cluster 0 pgmap v1443: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 15 MiB/s wr, 240 op/s; 26681/221574 objects degraded (12.042%); 2588/221574 objects misplaced (1.168%); 4.7 MiB/s, 76 objects/s recovering 2023-10-01T11:03:19.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:19 smithi173 bash[58816]: audit 2023-10-01T11:03:18.241476+0000 mon.a (mon.0) 1744 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:03:19.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:19 smithi173 bash[58816]: audit 2023-10-01T11:03:18.279096+0000 mon.a (mon.0) 1745 : audit 1 from='client.? 172.21.15.78:0/2179208199' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T11:03:19.427 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:19.439 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 72 16 74725 74709 64.8436 24.5 0.00378393 0.0153669 2023-10-01T11:03:19.452 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:19.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:19.455 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:19.556 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:19.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:19 smithi078 bash[78274]: cluster 2023-10-01T11:03:18.230858+0000 mgr.y (mgr.25072) 1378 : cluster 0 pgmap v1443: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 15 MiB/s wr, 240 op/s; 26681/221574 objects degraded (12.042%); 2588/221574 objects misplaced (1.168%); 4.7 MiB/s, 76 objects/s recovering 2023-10-01T11:03:19.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:19 smithi078 bash[78274]: audit 2023-10-01T11:03:18.241476+0000 mon.a (mon.0) 1744 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:03:19.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:19 smithi078 bash[78274]: audit 2023-10-01T11:03:18.279096+0000 mon.a (mon.0) 1745 : audit 1 from='client.? 172.21.15.78:0/2179208199' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T11:03:19.582 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:19.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:19.587 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:03:19.688 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:19.712 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:19.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:19.714 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:03:19.815 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:19.842 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:19.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:19.847 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:19.947 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:19.966 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:19.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:19.969 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:03:20.070 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:20.100 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:20.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:20.103 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:20.204 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:20.233 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:20.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:20.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:03:20.336 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:20.362 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:20.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:20.367 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:03:20.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:20 smithi173 bash[58816]: cluster 2023-10-01T11:03:19.236383+0000 mon.a (mon.0) 1746 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:03:20.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:20 smithi173 bash[58816]: cluster 2023-10-01T11:03:19.242061+0000 osd.6 (osd.6) 279 : cluster 3 failed to encode map e302 with expected crc 2023-10-01T11:03:20.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:20 smithi173 bash[58816]: audit 2023-10-01T11:03:19.243228+0000 mon.a (mon.0) 1747 : audit 1 from='client.? 172.21.15.78:0/2179208199' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T11:03:20.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:20 smithi173 bash[58816]: cluster 2023-10-01T11:03:19.243269+0000 mon.a (mon.0) 1748 : cluster 0 osdmap e302: 8 total, 8 up, 7 in 2023-10-01T11:03:20.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:20 smithi173 bash[58816]: cluster 2023-10-01T11:03:19.243471+0000 osd.7 (osd.7) 319 : cluster 3 failed to encode map e302 with expected crc 2023-10-01T11:03:20.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:20 smithi173 bash[58816]: cluster 2023-10-01T11:03:19.247391+0000 osd.7 (osd.7) 320 : cluster 3 failed to encode map e302 with expected crc 2023-10-01T11:03:20.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:20 smithi173 bash[58816]: cluster 2023-10-01T11:03:19.257601+0000 osd.5 (osd.5) 306 : cluster 3 failed to encode map e302 with expected crc 2023-10-01T11:03:20.439 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 73 16 74942 74926 64.141 13.5625 0.00511223 0.0155179 2023-10-01T11:03:20.468 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:20.492 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:20.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:20.494 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:20.532 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:20 smithi078 bash[78274]: cluster 2023-10-01T11:03:19.236383+0000 mon.a (mon.0) 1746 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:03:20.533 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:20 smithi078 bash[78274]: cluster 2023-10-01T11:03:19.242061+0000 osd.6 (osd.6) 279 : cluster 3 failed to encode map e302 with expected crc 2023-10-01T11:03:20.533 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:20 smithi078 bash[78274]: audit 2023-10-01T11:03:19.243228+0000 mon.a (mon.0) 1747 : audit 1 from='client.? 172.21.15.78:0/2179208199' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T11:03:20.533 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:20 smithi078 bash[78274]: cluster 2023-10-01T11:03:19.243269+0000 mon.a (mon.0) 1748 : cluster 0 osdmap e302: 8 total, 8 up, 7 in 2023-10-01T11:03:20.533 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:20 smithi078 bash[78274]: cluster 2023-10-01T11:03:19.243471+0000 osd.7 (osd.7) 319 : cluster 3 failed to encode map e302 with expected crc 2023-10-01T11:03:20.533 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:20 smithi078 bash[78274]: cluster 2023-10-01T11:03:19.247391+0000 osd.7 (osd.7) 320 : cluster 3 failed to encode map e302 with expected crc 2023-10-01T11:03:20.533 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:20 smithi078 bash[78274]: cluster 2023-10-01T11:03:19.257601+0000 osd.5 (osd.5) 306 : cluster 3 failed to encode map e302 with expected crc 2023-10-01T11:03:20.533 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:20 smithi078 bash[75299]: cluster 2023-10-01T11:03:19.236383+0000 mon.a (mon.0) 1746 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:03:20.534 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:20 smithi078 bash[75299]: cluster 2023-10-01T11:03:19.242061+0000 osd.6 (osd.6) 279 : cluster 3 failed to encode map e302 with expected crc 2023-10-01T11:03:20.534 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:20 smithi078 bash[75299]: audit 2023-10-01T11:03:19.243228+0000 mon.a (mon.0) 1747 : audit 1 from='client.? 172.21.15.78:0/2179208199' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T11:03:20.534 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:20 smithi078 bash[75299]: cluster 2023-10-01T11:03:19.243269+0000 mon.a (mon.0) 1748 : cluster 0 osdmap e302: 8 total, 8 up, 7 in 2023-10-01T11:03:20.534 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:20 smithi078 bash[75299]: cluster 2023-10-01T11:03:19.243471+0000 osd.7 (osd.7) 319 : cluster 3 failed to encode map e302 with expected crc 2023-10-01T11:03:20.534 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:20 smithi078 bash[75299]: cluster 2023-10-01T11:03:19.247391+0000 osd.7 (osd.7) 320 : cluster 3 failed to encode map e302 with expected crc 2023-10-01T11:03:20.534 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:20 smithi078 bash[75299]: cluster 2023-10-01T11:03:19.257601+0000 osd.5 (osd.5) 306 : cluster 3 failed to encode map e302 with expected crc 2023-10-01T11:03:20.534 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:03:20 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:03:20] "GET /metrics HTTP/1.1" 200 36406 "" "Prometheus/2.33.4" 2023-10-01T11:03:20.595 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:20.621 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:20.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:20.624 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:03:20.725 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:20.750 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:20.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:20.754 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:03:20.855 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:20.882 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:20.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:20.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:03:20.962 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-01T11:03:20.984 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:21.026 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:21.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:21.028 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:03:21.129 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:21.155 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:21.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:21.158 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:21.259 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:21.280 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:21.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:21.283 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:03:21.384 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:21.392 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:21 smithi173 bash[58816]: cluster 2023-10-01T11:03:20.231552+0000 mgr.y (mgr.25072) 1379 : cluster 0 pgmap v1445: 89 pgs: 1 activating+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 7.9 MiB/s wr, 126 op/s; 28962/222633 objects degraded (13.009%); 5.9 MiB/s, 93 objects/s recovering 2023-10-01T11:03:21.410 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:21.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:21.415 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:03:21.439 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 74 16 75201 75185 63.493 16.1875 0.00406086 0.015703 2023-10-01T11:03:21.517 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:21.541 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:21.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:21.543 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:21.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:21 smithi078 bash[78274]: cluster 2023-10-01T11:03:20.231552+0000 mgr.y (mgr.25072) 1379 : cluster 0 pgmap v1445: 89 pgs: 1 activating+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 7.9 MiB/s wr, 126 op/s; 28962/222633 objects degraded (13.009%); 5.9 MiB/s, 93 objects/s recovering 2023-10-01T11:03:21.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:21 smithi078 bash[75299]: cluster 2023-10-01T11:03:20.231552+0000 mgr.y (mgr.25072) 1379 : cluster 0 pgmap v1445: 89 pgs: 1 activating+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 7.9 MiB/s wr, 126 op/s; 28962/222633 objects degraded (13.009%); 5.9 MiB/s, 93 objects/s recovering 2023-10-01T11:03:21.644 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:21.671 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:21.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:21.673 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:03:21.774 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:21.801 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:21.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:21.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:03:21.905 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:21.936 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:21.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:21.939 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:03:22.039 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:22.064 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:22.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:22.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:22.167 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:22.193 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:22.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:22.199 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:22.300 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:22.327 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:22.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:22.329 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:03:22.430 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:22.439 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 75 16 75504 75488 62.8989 18.9375 0.00494353 0.0158361 2023-10-01T11:03:22.460 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:22.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:22.463 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:03:22.564 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:22.592 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:22.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:22.595 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:03:22.696 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:22.722 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:22.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:22.724 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:03:22.825 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:22.858 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:22.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:22.861 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:03:22.872 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T11:03:22.963 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:22.991 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:22.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:22.993 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:03:23.094 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:23.119 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:23.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:23.123 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:23.224 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:23.250 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:23.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:23.252 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:03:23.353 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:23.361 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:23 smithi173 bash[58816]: cluster 2023-10-01T11:03:22.232479+0000 mgr.y (mgr.25072) 1380 : cluster 0 pgmap v1446: 89 pgs: 1 activating+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 7.9 MiB/s wr, 126 op/s; 28962/222633 objects degraded (13.009%); 5.9 MiB/s, 93 objects/s recovering 2023-10-01T11:03:23.362 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:23 smithi173 bash[58816]: cluster 2023-10-01T11:03:22.696725+0000 mon.a (mon.0) 1749 : cluster 3 Health check update: Degraded data redundancy: 28962/222633 objects degraded (13.009%), 9 pgs degraded (PG_DEGRADED) 2023-10-01T11:03:23.378 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:23.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:23.380 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:23.439 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 76 16 75818 75802 62.3294 19.625 0.00464693 0.0159777 2023-10-01T11:03:23.481 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:23.491 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:23 smithi078 bash[78274]: cluster 2023-10-01T11:03:22.232479+0000 mgr.y (mgr.25072) 1380 : cluster 0 pgmap v1446: 89 pgs: 1 activating+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 7.9 MiB/s wr, 126 op/s; 28962/222633 objects degraded (13.009%); 5.9 MiB/s, 93 objects/s recovering 2023-10-01T11:03:23.492 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:23 smithi078 bash[78274]: cluster 2023-10-01T11:03:22.696725+0000 mon.a (mon.0) 1749 : cluster 3 Health check update: Degraded data redundancy: 28962/222633 objects degraded (13.009%), 9 pgs degraded (PG_DEGRADED) 2023-10-01T11:03:23.492 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:23 smithi078 bash[75299]: cluster 2023-10-01T11:03:22.232479+0000 mgr.y (mgr.25072) 1380 : cluster 0 pgmap v1446: 89 pgs: 1 activating+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 7.9 MiB/s wr, 126 op/s; 28962/222633 objects degraded (13.009%); 5.9 MiB/s, 93 objects/s recovering 2023-10-01T11:03:23.492 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:23 smithi078 bash[75299]: cluster 2023-10-01T11:03:22.696725+0000 mon.a (mon.0) 1749 : cluster 3 Health check update: Degraded data redundancy: 28962/222633 objects degraded (13.009%), 9 pgs degraded (PG_DEGRADED) 2023-10-01T11:03:23.508 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:23.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:23.511 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:03:23.611 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:23.640 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:23.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:23.646 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:03:23.696 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T11:03:23.747 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:23.776 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:23.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:23.779 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:03:23.882 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:23.910 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:23.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:23.915 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:03:24.016 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:24.046 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:24.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:24.048 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:03:24.149 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:24.173 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:24.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:24.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:03:24.198 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:03:24.276 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:24.301 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:24.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:24.303 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:03:24.404 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:24.428 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:24.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:24.434 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:03:24.439 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 77 16 76080 76064 61.7326 16.375 0.00386864 0.0161456 2023-10-01T11:03:24.535 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:24.558 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:24.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:24.561 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:24.661 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:24.691 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:24.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:24.694 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:24.795 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:24.818 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:24.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:24.820 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:24.921 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:24.950 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:24.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:24.952 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:25.053 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:25.080 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:25.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:25.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:03:25.183 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:25.218 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:25.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:25.221 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:03:25.321 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:25.330 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:25 smithi078 bash[78274]: cluster 2023-10-01T11:03:24.233459+0000 mgr.y (mgr.25072) 1381 : cluster 0 pgmap v1447: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 4.6 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 17 MiB/s wr, 270 op/s; 24194/227001 objects degraded (10.658%); 4749/227001 objects misplaced (2.092%); 6.0 MiB/s, 95 objects/s recovering 2023-10-01T11:03:25.330 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:25 smithi078 bash[75299]: cluster 2023-10-01T11:03:24.233459+0000 mgr.y (mgr.25072) 1381 : cluster 0 pgmap v1447: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 4.6 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 17 MiB/s wr, 270 op/s; 24194/227001 objects degraded (10.658%); 4749/227001 objects misplaced (2.092%); 6.0 MiB/s, 95 objects/s recovering 2023-10-01T11:03:25.346 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:25.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:25.351 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:03:25.440 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 78 16 76429 76413 61.2208 21.8125 0.00734148 0.0162772 2023-10-01T11:03:25.452 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:25.480 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:25.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:25.482 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:03:25.582 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:25.604 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:25.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:25.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:03:25.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:25 smithi173 bash[58816]: cluster 2023-10-01T11:03:24.233459+0000 mgr.y (mgr.25072) 1381 : cluster 0 pgmap v1447: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 4.6 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 17 MiB/s wr, 270 op/s; 24194/227001 objects degraded (10.658%); 4749/227001 objects misplaced (2.092%); 6.0 MiB/s, 95 objects/s recovering 2023-10-01T11:03:25.707 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:25.734 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:25.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:25.740 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:25.840 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:25.864 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:25.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:25.872 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:03:25.972 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:26.001 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:26.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:26.007 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:26.108 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:26.136 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:26.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:26.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:03:26.231 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:03:26.239 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:26.269 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:26.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:26.274 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:03:26.375 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:26.405 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:26.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:26.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:26.440 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 79 16 76811 76795 60.7481 23.875 0.00697331 0.0164099 2023-10-01T11:03:26.511 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:26.539 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:26.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:26.543 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:26.646 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:26.671 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:26.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:26.673 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:26.774 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:26.797 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:26.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:26.803 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:03:26.904 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:26.932 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:26.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:26.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:27.036 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:27.064 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:27.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:27.066 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:03:27.167 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:27.192 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:27.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:27.195 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:03:27.296 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:27.327 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:27.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:27.331 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:03:27.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:27 smithi078 bash[75299]: cluster 2023-10-01T11:03:26.234205+0000 mgr.y (mgr.25072) 1382 : cluster 0 pgmap v1448: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 4.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 17 MiB/s wr, 278 op/s; 24194/227259 objects degraded (10.646%); 4749/227259 objects misplaced (2.090%); 3.3 MiB/s, 52 objects/s recovering 2023-10-01T11:03:27.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:27 smithi078 bash[78274]: cluster 2023-10-01T11:03:26.234205+0000 mgr.y (mgr.25072) 1382 : cluster 0 pgmap v1448: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 4.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 17 MiB/s wr, 278 op/s; 24194/227259 objects degraded (10.646%); 4749/227259 objects misplaced (2.090%); 3.3 MiB/s, 52 objects/s recovering 2023-10-01T11:03:27.432 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:27.440 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:2023-10-01T11:03:27.439957+0000 min lat: 0.00136129 max lat: 10.8051 avg lat: 0.0165433 2023-10-01T11:03:27.440 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T11:03:27.441 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 80 16 77129 77113 60.2371 19.875 0.00455069 0.0165433 2023-10-01T11:03:27.454 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:27.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:27.457 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:03:27.558 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:27.583 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:27.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:27.589 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:03:27.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:27 smithi173 bash[58816]: cluster 2023-10-01T11:03:26.234205+0000 mgr.y (mgr.25072) 1382 : cluster 0 pgmap v1448: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 4.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 17 MiB/s wr, 278 op/s; 24194/227259 objects degraded (10.646%); 4749/227259 objects misplaced (2.090%); 3.3 MiB/s, 52 objects/s recovering 2023-10-01T11:03:27.689 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:27.713 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:27.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:27.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:03:27.817 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:27.843 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:27.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:27.846 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:03:27.947 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:27.971 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:27.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:27.976 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:03:28.077 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:28.104 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:28.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:28.107 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:03:28.208 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:28.231 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:28.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:28.234 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:03:28.335 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:28.344 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:28 smithi173 bash[58816]: cluster 2023-10-01T11:03:27.698319+0000 mon.a (mon.0) 1750 : cluster 3 Health check update: Degraded data redundancy: 24194/227259 objects degraded (10.646%), 8 pgs degraded (PG_DEGRADED) 2023-10-01T11:03:28.359 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:28.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:28.363 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:03:28.440 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 81 16 77470 77454 59.7566 21.3125 0.00663601 0.0166851 2023-10-01T11:03:28.463 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:28.490 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:28.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:28.492 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:03:28.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:28 smithi078 bash[75299]: cluster 2023-10-01T11:03:27.698319+0000 mon.a (mon.0) 1750 : cluster 3 Health check update: Degraded data redundancy: 24194/227259 objects degraded (10.646%), 8 pgs degraded (PG_DEGRADED) 2023-10-01T11:03:28.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:28 smithi078 bash[78274]: cluster 2023-10-01T11:03:27.698319+0000 mon.a (mon.0) 1750 : cluster 3 Health check update: Degraded data redundancy: 24194/227259 objects degraded (10.646%), 8 pgs degraded (PG_DEGRADED) 2023-10-01T11:03:28.592 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:28.623 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:28.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:28.625 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:03:28.726 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:28.751 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:28.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:28.753 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:03:28.808 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:03:28.808 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:03:28.854 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:28.877 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:28.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:28.879 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:28.980 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:29.012 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:29.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:29.016 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:29.117 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:29.141 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:29.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:29.143 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:03:29.244 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:29.275 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:29.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:29.278 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:03:29.379 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:29.410 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:29.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:29.415 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:03:29.440 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 82 16 77811 77795 59.2877 21.3125 0.00645428 0.0168177 2023-10-01T11:03:29.517 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:29.538 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:29.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:29.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:03:29.641 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:29.667 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:29.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:29.671 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:03:29.772 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:29.799 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:29.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:29.802 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:03:29.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:29 smithi078 bash[75299]: cluster 2023-10-01T11:03:28.234976+0000 mgr.y (mgr.25072) 1383 : cluster 0 pgmap v1449: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 16 MiB/s wr, 248 op/s; 24194/229041 objects degraded (10.563%); 4738/229041 objects misplaced (2.069%); 2.1 MiB/s, 33 objects/s recovering 2023-10-01T11:03:29.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:29 smithi078 bash[78274]: cluster 2023-10-01T11:03:28.234976+0000 mgr.y (mgr.25072) 1383 : cluster 0 pgmap v1449: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 16 MiB/s wr, 248 op/s; 24194/229041 objects degraded (10.563%); 4738/229041 objects misplaced (2.069%); 2.1 MiB/s, 33 objects/s recovering 2023-10-01T11:03:29.902 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:29.930 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:29.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:29.932 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:03:30.034 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:30.042 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:29 smithi173 bash[58816]: cluster 2023-10-01T11:03:28.234976+0000 mgr.y (mgr.25072) 1383 : cluster 0 pgmap v1449: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 16 MiB/s wr, 248 op/s; 24194/229041 objects degraded (10.563%); 4738/229041 objects misplaced (2.069%); 2.1 MiB/s, 33 objects/s recovering 2023-10-01T11:03:30.061 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:30.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:30.064 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:30.165 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:30.193 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:30.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:30.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:03:30.300 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:30.323 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:30.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:30.325 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:03:30.427 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:30.440 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 83 16 78175 78159 58.8475 22.75 0.0117759 0.016942 2023-10-01T11:03:30.459 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:30.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:30.462 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:30.562 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:30.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:03:30 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:03:30] "GET /metrics HTTP/1.1" 200 36406 "" "Prometheus/2.33.4" 2023-10-01T11:03:30.584 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:30.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:30.587 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:03:30.687 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:30.713 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:30.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:30.715 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:03:30.816 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:30.846 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:30.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:30.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:30.951 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:30.973 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:30.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:30.975 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:03:31.076 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:31.096 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:31.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:31.101 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:03:31.202 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:31.227 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:31.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:31.229 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:03:31.330 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:31.357 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:31.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:31.358 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:03:31.440 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 84 16 78490 78474 58.3813 19.6875 0.00747956 0.0170742 2023-10-01T11:03:31.460 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:31.487 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:31.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:31.490 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:03:31.591 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:31.617 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:31.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:31.622 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:03:31.723 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:31.746 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:31.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:31.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:03:31.850 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:31.862 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:31 smithi173 bash[58816]: cluster 2023-10-01T11:03:30.235822+0000 mgr.y (mgr.25072) 1384 : cluster 0 pgmap v1450: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 18 MiB/s wr, 294 op/s; 24194/232203 objects degraded (10.419%); 4738/232203 objects misplaced (2.040%); 1.9 MiB/s, 30 objects/s recovering 2023-10-01T11:03:31.879 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:31.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:31.882 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:31.983 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:32.016 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:32.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:32.021 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:32.038 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:31 smithi078 bash[78274]: cluster 2023-10-01T11:03:30.235822+0000 mgr.y (mgr.25072) 1384 : cluster 0 pgmap v1450: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 18 MiB/s wr, 294 op/s; 24194/232203 objects degraded (10.419%); 4738/232203 objects misplaced (2.040%); 1.9 MiB/s, 30 objects/s recovering 2023-10-01T11:03:32.039 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:31 smithi078 bash[75299]: cluster 2023-10-01T11:03:30.235822+0000 mgr.y (mgr.25072) 1384 : cluster 0 pgmap v1450: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 18 MiB/s wr, 294 op/s; 24194/232203 objects degraded (10.419%); 4738/232203 objects misplaced (2.040%); 1.9 MiB/s, 30 objects/s recovering 2023-10-01T11:03:32.121 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:32.145 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:32.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:32.151 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:03:32.252 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:32.276 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:32.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:32.278 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:32.379 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:32.400 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:32.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:32.402 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:03:32.440 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 85 16 78872 78856 57.9753 23.875 0.00610147 0.0171953 2023-10-01T11:03:32.503 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:32.525 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:32.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:32.528 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:03:32.628 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:32.649 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:32.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:32.651 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:03:32.752 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:32.780 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:32.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:32.782 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:32.883 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:32.908 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:32.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:32.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:03:33.010 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:33.038 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:33.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:33.041 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:03:33.142 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:33.164 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:33.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:33.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:03:33.268 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:33.292 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:33.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:33.296 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:33.397 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:33.419 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:33.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:33.422 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:03:33.440 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 86 16 79171 79155 57.5185 18.6875 0.00510258 0.0173331 2023-10-01T11:03:33.461 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:33 smithi078 bash[75299]: cluster 2023-10-01T11:03:32.236553+0000 mgr.y (mgr.25072) 1385 : cluster 0 pgmap v1451: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 17 MiB/s wr, 265 op/s; 24194/232203 objects degraded (10.419%); 4738/232203 objects misplaced (2.040%); 139 KiB/s, 2 objects/s recovering 2023-10-01T11:03:33.461 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:33 smithi078 bash[75299]: audit 2023-10-01T11:03:32.460100+0000 mon.a (mon.0) 1751 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:03:33.461 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:33 smithi078 bash[75299]: audit 2023-10-01T11:03:32.461423+0000 mon.a (mon.0) 1752 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:03:33.461 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:33 smithi078 bash[75299]: cluster 2023-10-01T11:03:32.699560+0000 mon.a (mon.0) 1753 : cluster 3 Health check update: Degraded data redundancy: 24194/232203 objects degraded (10.419%), 8 pgs degraded (PG_DEGRADED) 2023-10-01T11:03:33.522 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:33.530 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:33 smithi173 bash[58816]: cluster 2023-10-01T11:03:32.236553+0000 mgr.y (mgr.25072) 1385 : cluster 0 pgmap v1451: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 17 MiB/s wr, 265 op/s; 24194/232203 objects degraded (10.419%); 4738/232203 objects misplaced (2.040%); 139 KiB/s, 2 objects/s recovering 2023-10-01T11:03:33.531 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:33 smithi173 bash[58816]: audit 2023-10-01T11:03:32.460100+0000 mon.a (mon.0) 1751 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:03:33.531 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:33 smithi173 bash[58816]: audit 2023-10-01T11:03:32.461423+0000 mon.a (mon.0) 1752 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:03:33.531 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:33 smithi173 bash[58816]: cluster 2023-10-01T11:03:32.699560+0000 mon.a (mon.0) 1753 : cluster 3 Health check update: Degraded data redundancy: 24194/232203 objects degraded (10.419%), 8 pgs degraded (PG_DEGRADED) 2023-10-01T11:03:33.552 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:33.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:33.555 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:03:33.656 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:33.687 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:33.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:33.689 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:03:33.729 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:33 smithi078 bash[78274]: cluster 2023-10-01T11:03:32.236553+0000 mgr.y (mgr.25072) 1385 : cluster 0 pgmap v1451: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 17 MiB/s wr, 265 op/s; 24194/232203 objects degraded (10.419%); 4738/232203 objects misplaced (2.040%); 139 KiB/s, 2 objects/s recovering 2023-10-01T11:03:33.730 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:33 smithi078 bash[78274]: audit 2023-10-01T11:03:32.460100+0000 mon.a (mon.0) 1751 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:03:33.730 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:33 smithi078 bash[78274]: audit 2023-10-01T11:03:32.461423+0000 mon.a (mon.0) 1752 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:03:33.730 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:33 smithi078 bash[78274]: cluster 2023-10-01T11:03:32.699560+0000 mon.a (mon.0) 1753 : cluster 3 Health check update: Degraded data redundancy: 24194/232203 objects degraded (10.419%), 8 pgs degraded (PG_DEGRADED) 2023-10-01T11:03:33.790 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:33.817 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:33.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:33.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:33.920 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:33.948 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:33.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:33.955 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:34.055 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:34.080 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:34.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:34.086 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:03:34.187 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:34.213 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:34.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:34.216 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:03:34.316 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:34.330 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:34 smithi078 bash[75299]: debug 2023-10-01T11:03:34.237+0000 7f31c8dd0700 -1 mon.a@0(leader).osd e302 definitely_dead 0 2023-10-01T11:03:34.344 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:34.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:34.346 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:34.442 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 87 16 79464 79448 57.0679 18.3125 0.632246 0.0174738 2023-10-01T11:03:34.446 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:34.471 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:34.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:34.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:03:34.481 INFO:teuthology.orchestra.run.smithi078.stderr:marked out osd.7. 2023-10-01T11:03:34.579 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:34.591 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:34 smithi173 bash[58816]: audit 2023-10-01T11:03:34.238082+0000 mon.c (mon.2) 210 : audit 1 from='client.? 172.21.15.78:0/3064388736' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-01T11:03:34.591 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:34 smithi173 bash[58816]: audit 2023-10-01T11:03:34.238739+0000 mon.a (mon.0) 1754 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-01T11:03:34.591 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:34 smithi173 bash[58816]: cluster 2023-10-01T11:03:34.238800+0000 mon.a (mon.0) 1755 : cluster 1 Client client.admin marked osd.7 out, while it was still marked up 2023-10-01T11:03:34.608 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:34.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:34.611 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:34.712 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:34.722 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:34 smithi078 bash[75299]: audit 2023-10-01T11:03:34.238082+0000 mon.c (mon.2) 210 : audit 1 from='client.? 172.21.15.78:0/3064388736' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-01T11:03:34.722 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:34 smithi078 bash[75299]: audit 2023-10-01T11:03:34.238739+0000 mon.a (mon.0) 1754 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-01T11:03:34.722 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:34 smithi078 bash[75299]: cluster 2023-10-01T11:03:34.238800+0000 mon.a (mon.0) 1755 : cluster 1 Client client.admin marked osd.7 out, while it was still marked up 2023-10-01T11:03:34.723 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:34 smithi078 bash[78274]: audit 2023-10-01T11:03:34.238082+0000 mon.c (mon.2) 210 : audit 1 from='client.? 172.21.15.78:0/3064388736' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-01T11:03:34.723 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:34 smithi078 bash[78274]: audit 2023-10-01T11:03:34.238739+0000 mon.a (mon.0) 1754 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-10-01T11:03:34.723 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:34 smithi078 bash[78274]: cluster 2023-10-01T11:03:34.238800+0000 mon.a (mon.0) 1755 : cluster 1 Client client.admin marked osd.7 out, while it was still marked up 2023-10-01T11:03:34.735 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:34.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:34.737 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:03:34.838 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:34.868 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:34.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:34.872 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:34.972 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:35.002 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:35.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:35.005 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:03:35.105 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:35.127 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:35.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:35.129 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:03:35.230 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:35.262 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:35.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:35.264 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:03:35.365 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:35.397 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:35.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:35.400 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:03:35.441 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 88 16 79475 79459 56.4272 0.6875 0.651822 0.0174813 2023-10-01T11:03:35.501 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:35.512 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:35 smithi173 bash[58816]: cluster 2023-10-01T11:03:34.237598+0000 mgr.y (mgr.25072) 1386 : cluster 0 pgmap v1452: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 25 MiB/s wr, 400 op/s; 24194/237060 objects degraded (10.206%); 4730/237060 objects misplaced (1.995%); 181 KiB/s, 2 objects/s recovering 2023-10-01T11:03:35.512 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:35 smithi173 bash[58816]: cluster 2023-10-01T11:03:34.464627+0000 osd.7 (osd.7) 321 : cluster 3 failed to encode map e303 with expected crc 2023-10-01T11:03:35.512 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:35 smithi173 bash[58816]: cluster 2023-10-01T11:03:34.465269+0000 osd.6 (osd.6) 280 : cluster 3 failed to encode map e303 with expected crc 2023-10-01T11:03:35.512 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:35 smithi173 bash[58816]: audit 2023-10-01T11:03:34.466162+0000 mon.a (mon.0) 1756 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-10-01T11:03:35.512 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:35 smithi173 bash[58816]: cluster 2023-10-01T11:03:34.466322+0000 mon.a (mon.0) 1757 : cluster 0 osdmap e303: 8 total, 8 up, 6 in 2023-10-01T11:03:35.512 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:35 smithi173 bash[58816]: cluster 2023-10-01T11:03:34.466765+0000 osd.5 (osd.5) 307 : cluster 3 failed to encode map e303 with expected crc 2023-10-01T11:03:35.512 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:35 smithi173 bash[58816]: cluster 2023-10-01T11:03:34.468419+0000 osd.7 (osd.7) 322 : cluster 3 failed to encode map e303 with expected crc 2023-10-01T11:03:35.512 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:35 smithi173 bash[58816]: cluster 2023-10-01T11:03:34.470677+0000 osd.7 (osd.7) 323 : cluster 3 failed to encode map e303 with expected crc 2023-10-01T11:03:35.512 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:35 smithi173 bash[58816]: cluster 2023-10-01T11:03:34.471309+0000 osd.6 (osd.6) 281 : cluster 3 failed to encode map e303 with expected crc 2023-10-01T11:03:35.513 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:35 smithi173 bash[58816]: cluster 2023-10-01T11:03:34.474797+0000 osd.7 (osd.7) 324 : cluster 3 failed to encode map e303 with expected crc 2023-10-01T11:03:35.513 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:35 smithi173 bash[58816]: cluster 2023-10-01T11:03:34.480299+0000 osd.7 (osd.7) 325 : cluster 3 failed to encode map e303 with expected crc 2023-10-01T11:03:35.531 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:35.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:35.539 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:35.640 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:35.681 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:35.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:35.683 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:03:35.784 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:35.806 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:35.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:35.809 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:35.891 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:35 smithi078 bash[75299]: cluster 2023-10-01T11:03:34.237598+0000 mgr.y (mgr.25072) 1386 : cluster 0 pgmap v1452: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 25 MiB/s wr, 400 op/s; 24194/237060 objects degraded (10.206%); 4730/237060 objects misplaced (1.995%); 181 KiB/s, 2 objects/s recovering 2023-10-01T11:03:35.891 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:35 smithi078 bash[75299]: cluster 2023-10-01T11:03:34.464627+0000 osd.7 (osd.7) 321 : cluster 3 failed to encode map e303 with expected crc 2023-10-01T11:03:35.891 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:35 smithi078 bash[75299]: cluster 2023-10-01T11:03:34.465269+0000 osd.6 (osd.6) 280 : cluster 3 failed to encode map e303 with expected crc 2023-10-01T11:03:35.892 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:35 smithi078 bash[75299]: audit 2023-10-01T11:03:34.466162+0000 mon.a (mon.0) 1756 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-10-01T11:03:35.892 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:35 smithi078 bash[75299]: cluster 2023-10-01T11:03:34.466322+0000 mon.a (mon.0) 1757 : cluster 0 osdmap e303: 8 total, 8 up, 6 in 2023-10-01T11:03:35.892 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:35 smithi078 bash[75299]: cluster 2023-10-01T11:03:34.466765+0000 osd.5 (osd.5) 307 : cluster 3 failed to encode map e303 with expected crc 2023-10-01T11:03:35.892 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:35 smithi078 bash[75299]: cluster 2023-10-01T11:03:34.468419+0000 osd.7 (osd.7) 322 : cluster 3 failed to encode map e303 with expected crc 2023-10-01T11:03:35.892 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:35 smithi078 bash[75299]: cluster 2023-10-01T11:03:34.470677+0000 osd.7 (osd.7) 323 : cluster 3 failed to encode map e303 with expected crc 2023-10-01T11:03:35.892 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:35 smithi078 bash[75299]: cluster 2023-10-01T11:03:34.471309+0000 osd.6 (osd.6) 281 : cluster 3 failed to encode map e303 with expected crc 2023-10-01T11:03:35.892 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:35 smithi078 bash[75299]: cluster 2023-10-01T11:03:34.474797+0000 osd.7 (osd.7) 324 : cluster 3 failed to encode map e303 with expected crc 2023-10-01T11:03:35.892 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:35 smithi078 bash[75299]: cluster 2023-10-01T11:03:34.480299+0000 osd.7 (osd.7) 325 : cluster 3 failed to encode map e303 with expected crc 2023-10-01T11:03:35.892 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:35 smithi078 bash[78274]: cluster 2023-10-01T11:03:34.237598+0000 mgr.y (mgr.25072) 1386 : cluster 0 pgmap v1452: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 25 MiB/s wr, 400 op/s; 24194/237060 objects degraded (10.206%); 4730/237060 objects misplaced (1.995%); 181 KiB/s, 2 objects/s recovering 2023-10-01T11:03:35.893 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:35 smithi078 bash[78274]: cluster 2023-10-01T11:03:34.464627+0000 osd.7 (osd.7) 321 : cluster 3 failed to encode map e303 with expected crc 2023-10-01T11:03:35.893 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:35 smithi078 bash[78274]: cluster 2023-10-01T11:03:34.465269+0000 osd.6 (osd.6) 280 : cluster 3 failed to encode map e303 with expected crc 2023-10-01T11:03:35.893 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:35 smithi078 bash[78274]: audit 2023-10-01T11:03:34.466162+0000 mon.a (mon.0) 1756 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-10-01T11:03:35.893 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:35 smithi078 bash[78274]: cluster 2023-10-01T11:03:34.466322+0000 mon.a (mon.0) 1757 : cluster 0 osdmap e303: 8 total, 8 up, 6 in 2023-10-01T11:03:35.893 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:35 smithi078 bash[78274]: cluster 2023-10-01T11:03:34.466765+0000 osd.5 (osd.5) 307 : cluster 3 failed to encode map e303 with expected crc 2023-10-01T11:03:35.893 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:35 smithi078 bash[78274]: cluster 2023-10-01T11:03:34.468419+0000 osd.7 (osd.7) 322 : cluster 3 failed to encode map e303 with expected crc 2023-10-01T11:03:35.893 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:35 smithi078 bash[78274]: cluster 2023-10-01T11:03:34.470677+0000 osd.7 (osd.7) 323 : cluster 3 failed to encode map e303 with expected crc 2023-10-01T11:03:35.893 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:35 smithi078 bash[78274]: cluster 2023-10-01T11:03:34.471309+0000 osd.6 (osd.6) 281 : cluster 3 failed to encode map e303 with expected crc 2023-10-01T11:03:35.893 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:35 smithi078 bash[78274]: cluster 2023-10-01T11:03:34.474797+0000 osd.7 (osd.7) 324 : cluster 3 failed to encode map e303 with expected crc 2023-10-01T11:03:35.893 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:35 smithi078 bash[78274]: cluster 2023-10-01T11:03:34.480299+0000 osd.7 (osd.7) 325 : cluster 3 failed to encode map e303 with expected crc 2023-10-01T11:03:35.910 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:35.934 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:35.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:35.938 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:03:36.039 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:36.063 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:36.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:36.066 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:03:36.167 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:36.190 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:36.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:36.192 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:03:36.293 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:36.315 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:36.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:36.318 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:03:36.419 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:36.441 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 89 16 79509 79493 55.817 2.125 0.00848396 0.0175249 2023-10-01T11:03:36.444 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:36.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:36.446 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:03:36.547 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:36.573 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:36.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:36.576 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:03:36.676 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:36.684 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:36 smithi078 bash[75299]: cluster 2023-10-01T11:03:35.478855+0000 osd.5 (osd.5) 308 : cluster 3 failed to encode map e304 with expected crc 2023-10-01T11:03:36.684 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:36 smithi078 bash[75299]: cluster 2023-10-01T11:03:35.480259+0000 mon.a (mon.0) 1758 : cluster 0 osdmap e304: 8 total, 8 up, 6 in 2023-10-01T11:03:36.684 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:36 smithi078 bash[75299]: cluster 2023-10-01T11:03:35.481784+0000 osd.5 (osd.5) 309 : cluster 3 failed to encode map e304 with expected crc 2023-10-01T11:03:36.684 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:36 smithi078 bash[75299]: cluster 2023-10-01T11:03:35.485689+0000 osd.5 (osd.5) 310 : cluster 3 failed to encode map e304 with expected crc 2023-10-01T11:03:36.685 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:36 smithi078 bash[75299]: cluster 2023-10-01T11:03:35.486029+0000 osd.7 (osd.7) 326 : cluster 3 failed to encode map e304 with expected crc 2023-10-01T11:03:36.685 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:36 smithi078 bash[75299]: cluster 2023-10-01T11:03:35.489170+0000 osd.7 (osd.7) 327 : cluster 3 failed to encode map e304 with expected crc 2023-10-01T11:03:36.685 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:36 smithi078 bash[75299]: cluster 2023-10-01T11:03:35.492047+0000 osd.7 (osd.7) 328 : cluster 3 failed to encode map e304 with expected crc 2023-10-01T11:03:36.685 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:36 smithi078 bash[75299]: cluster 2023-10-01T11:03:35.493666+0000 osd.6 (osd.6) 282 : cluster 3 failed to encode map e304 with expected crc 2023-10-01T11:03:36.686 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:36 smithi078 bash[75299]: cluster 2023-10-01T11:03:35.495991+0000 osd.6 (osd.6) 283 : cluster 3 failed to encode map e304 with expected crc 2023-10-01T11:03:36.686 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:36 smithi078 bash[78274]: cluster 2023-10-01T11:03:35.478855+0000 osd.5 (osd.5) 308 : cluster 3 failed to encode map e304 with expected crc 2023-10-01T11:03:36.686 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:36 smithi078 bash[78274]: cluster 2023-10-01T11:03:35.480259+0000 mon.a (mon.0) 1758 : cluster 0 osdmap e304: 8 total, 8 up, 6 in 2023-10-01T11:03:36.686 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:36 smithi078 bash[78274]: cluster 2023-10-01T11:03:35.481784+0000 osd.5 (osd.5) 309 : cluster 3 failed to encode map e304 with expected crc 2023-10-01T11:03:36.686 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:36 smithi078 bash[78274]: cluster 2023-10-01T11:03:35.485689+0000 osd.5 (osd.5) 310 : cluster 3 failed to encode map e304 with expected crc 2023-10-01T11:03:36.686 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:36 smithi078 bash[78274]: cluster 2023-10-01T11:03:35.486029+0000 osd.7 (osd.7) 326 : cluster 3 failed to encode map e304 with expected crc 2023-10-01T11:03:36.686 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:36 smithi078 bash[78274]: cluster 2023-10-01T11:03:35.489170+0000 osd.7 (osd.7) 327 : cluster 3 failed to encode map e304 with expected crc 2023-10-01T11:03:36.686 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:36 smithi078 bash[78274]: cluster 2023-10-01T11:03:35.492047+0000 osd.7 (osd.7) 328 : cluster 3 failed to encode map e304 with expected crc 2023-10-01T11:03:36.686 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:36 smithi078 bash[78274]: cluster 2023-10-01T11:03:35.493666+0000 osd.6 (osd.6) 282 : cluster 3 failed to encode map e304 with expected crc 2023-10-01T11:03:36.686 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:36 smithi078 bash[78274]: cluster 2023-10-01T11:03:35.495991+0000 osd.6 (osd.6) 283 : cluster 3 failed to encode map e304 with expected crc 2023-10-01T11:03:36.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:36.700 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:36.700 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:03:36.801 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:36.828 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:36.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:36.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:03:36.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:36 smithi173 bash[58816]: cluster 2023-10-01T11:03:35.478855+0000 osd.5 (osd.5) 308 : cluster 3 failed to encode map e304 with expected crc 2023-10-01T11:03:36.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:36 smithi173 bash[58816]: cluster 2023-10-01T11:03:35.480259+0000 mon.a (mon.0) 1758 : cluster 0 osdmap e304: 8 total, 8 up, 6 in 2023-10-01T11:03:36.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:36 smithi173 bash[58816]: cluster 2023-10-01T11:03:35.481784+0000 osd.5 (osd.5) 309 : cluster 3 failed to encode map e304 with expected crc 2023-10-01T11:03:36.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:36 smithi173 bash[58816]: cluster 2023-10-01T11:03:35.485689+0000 osd.5 (osd.5) 310 : cluster 3 failed to encode map e304 with expected crc 2023-10-01T11:03:36.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:36 smithi173 bash[58816]: cluster 2023-10-01T11:03:35.486029+0000 osd.7 (osd.7) 326 : cluster 3 failed to encode map e304 with expected crc 2023-10-01T11:03:36.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:36 smithi173 bash[58816]: cluster 2023-10-01T11:03:35.489170+0000 osd.7 (osd.7) 327 : cluster 3 failed to encode map e304 with expected crc 2023-10-01T11:03:36.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:36 smithi173 bash[58816]: cluster 2023-10-01T11:03:35.492047+0000 osd.7 (osd.7) 328 : cluster 3 failed to encode map e304 with expected crc 2023-10-01T11:03:36.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:36 smithi173 bash[58816]: cluster 2023-10-01T11:03:35.493666+0000 osd.6 (osd.6) 282 : cluster 3 failed to encode map e304 with expected crc 2023-10-01T11:03:36.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:36 smithi173 bash[58816]: cluster 2023-10-01T11:03:35.495991+0000 osd.6 (osd.6) 283 : cluster 3 failed to encode map e304 with expected crc 2023-10-01T11:03:36.931 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:36.959 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:36.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:36.962 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:03:37.063 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:37.093 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:37.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:37.098 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:03:37.198 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:37.223 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:37.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:37.228 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:03:37.329 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:37.359 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:37.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:37.364 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:03:37.443 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 90 16 79555 79539 55.2288 2.875 0.00530391 0.0175697 2023-10-01T11:03:37.465 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:37.494 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:37.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:37.495 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:37.596 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:37.631 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:37.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:37.633 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:37.734 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:37.743 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:37 smithi173 bash[58816]: cluster 2023-10-01T11:03:36.238299+0000 mgr.y (mgr.25072) 1387 : cluster 0 pgmap v1455: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 4.8 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 22 MiB/s wr, 346 op/s; 24194/237366 objects degraded (10.193%); 4730/237366 objects misplaced (1.993%); 64 KiB/s, 0 objects/s recovering 2023-10-01T11:03:37.744 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:37 smithi173 bash[58816]: cluster 2023-10-01T11:03:36.501079+0000 osd.7 (osd.7) 329 : cluster 3 failed to encode map e305 with expected crc 2023-10-01T11:03:37.744 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:37 smithi173 bash[58816]: cluster 2023-10-01T11:03:36.503777+0000 osd.7 (osd.7) 330 : cluster 3 failed to encode map e305 with expected crc 2023-10-01T11:03:37.744 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:37 smithi173 bash[58816]: cluster 2023-10-01T11:03:36.508290+0000 mon.a (mon.0) 1759 : cluster 0 osdmap e305: 8 total, 8 up, 6 in 2023-10-01T11:03:37.744 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:37 smithi173 bash[58816]: cluster 2023-10-01T11:03:36.515149+0000 osd.6 (osd.6) 284 : cluster 3 failed to encode map e305 with expected crc 2023-10-01T11:03:37.744 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:37 smithi173 bash[58816]: cluster 2023-10-01T11:03:36.520825+0000 osd.5 (osd.5) 311 : cluster 3 failed to encode map e305 with expected crc 2023-10-01T11:03:37.758 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:37.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:37.765 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:03:37.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:37 smithi078 bash[75299]: cluster 2023-10-01T11:03:36.238299+0000 mgr.y (mgr.25072) 1387 : cluster 0 pgmap v1455: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 4.8 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 22 MiB/s wr, 346 op/s; 24194/237366 objects degraded (10.193%); 4730/237366 objects misplaced (1.993%); 64 KiB/s, 0 objects/s recovering 2023-10-01T11:03:37.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:37 smithi078 bash[75299]: cluster 2023-10-01T11:03:36.501079+0000 osd.7 (osd.7) 329 : cluster 3 failed to encode map e305 with expected crc 2023-10-01T11:03:37.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:37 smithi078 bash[75299]: cluster 2023-10-01T11:03:36.503777+0000 osd.7 (osd.7) 330 : cluster 3 failed to encode map e305 with expected crc 2023-10-01T11:03:37.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:37 smithi078 bash[75299]: cluster 2023-10-01T11:03:36.508290+0000 mon.a (mon.0) 1759 : cluster 0 osdmap e305: 8 total, 8 up, 6 in 2023-10-01T11:03:37.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:37 smithi078 bash[75299]: cluster 2023-10-01T11:03:36.515149+0000 osd.6 (osd.6) 284 : cluster 3 failed to encode map e305 with expected crc 2023-10-01T11:03:37.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:37 smithi078 bash[75299]: cluster 2023-10-01T11:03:36.520825+0000 osd.5 (osd.5) 311 : cluster 3 failed to encode map e305 with expected crc 2023-10-01T11:03:37.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:37 smithi078 bash[78274]: cluster 2023-10-01T11:03:36.238299+0000 mgr.y (mgr.25072) 1387 : cluster 0 pgmap v1455: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 4.8 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 22 MiB/s wr, 346 op/s; 24194/237366 objects degraded (10.193%); 4730/237366 objects misplaced (1.993%); 64 KiB/s, 0 objects/s recovering 2023-10-01T11:03:37.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:37 smithi078 bash[78274]: cluster 2023-10-01T11:03:36.501079+0000 osd.7 (osd.7) 329 : cluster 3 failed to encode map e305 with expected crc 2023-10-01T11:03:37.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:37 smithi078 bash[78274]: cluster 2023-10-01T11:03:36.503777+0000 osd.7 (osd.7) 330 : cluster 3 failed to encode map e305 with expected crc 2023-10-01T11:03:37.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:37 smithi078 bash[78274]: cluster 2023-10-01T11:03:36.508290+0000 mon.a (mon.0) 1759 : cluster 0 osdmap e305: 8 total, 8 up, 6 in 2023-10-01T11:03:37.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:37 smithi078 bash[78274]: cluster 2023-10-01T11:03:36.515149+0000 osd.6 (osd.6) 284 : cluster 3 failed to encode map e305 with expected crc 2023-10-01T11:03:37.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:37 smithi078 bash[78274]: cluster 2023-10-01T11:03:36.520825+0000 osd.5 (osd.5) 311 : cluster 3 failed to encode map e305 with expected crc 2023-10-01T11:03:37.866 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:37.888 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:37.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:37.891 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:03:37.992 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:38.023 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:38.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:38.026 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:03:38.127 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:38.154 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:38.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:38.157 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:03:38.261 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:38.293 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:38.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:38.298 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:03:38.399 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:38.424 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:38.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:38.428 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:03:38.441 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 91 16 79555 79539 54.6219 0 - 0.0175697 2023-10-01T11:03:38.529 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:38.555 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:38.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:38.557 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:03:38.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:38 smithi173 bash[58816]: cluster 2023-10-01T11:03:37.700560+0000 mon.a (mon.0) 1760 : cluster 3 Health check update: Degraded data redundancy: 24194/237366 objects degraded (10.193%), 8 pgs degraded (PG_DEGRADED) 2023-10-01T11:03:38.658 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:38.672 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:38 smithi078 bash[78274]: cluster 2023-10-01T11:03:37.700560+0000 mon.a (mon.0) 1760 : cluster 3 Health check update: Degraded data redundancy: 24194/237366 objects degraded (10.193%), 8 pgs degraded (PG_DEGRADED) 2023-10-01T11:03:38.672 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:38 smithi078 bash[75299]: cluster 2023-10-01T11:03:37.700560+0000 mon.a (mon.0) 1760 : cluster 3 Health check update: Degraded data redundancy: 24194/237366 objects degraded (10.193%), 8 pgs degraded (PG_DEGRADED) 2023-10-01T11:03:38.690 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:38.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:38.693 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:03:38.794 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:38.822 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:38.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:38.827 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:03:38.928 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:38.961 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:38.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:38.964 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:03:39.065 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:39.091 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:39.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:39.093 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:03:39.194 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:39.221 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:39.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:39.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:39.328 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:39.347 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:39.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:39.349 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:03:39.441 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 92 16 79555 79539 54.0282 0 - 0.0175697 2023-10-01T11:03:39.450 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:39.467 INFO:teuthology.orchestra.run.smithi078.stdout:ERROR: (22) Invalid argument 2023-10-01T11:03:39.467 INFO:teuthology.orchestra.run.smithi078.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T11:03:39.478 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:39.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:39.481 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:03:39.581 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:39.613 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:39.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:39.615 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:39.716 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:39.743 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:39.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:39.745 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:03:39.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:39 smithi078 bash[78274]: cluster 2023-10-01T11:03:38.238907+0000 mgr.y (mgr.25072) 1388 : cluster 0 pgmap v1457: 89 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 4.8 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 20 MiB/s wr, 327 op/s; 34271/238107 objects degraded (14.393%); 864 KiB/s, 13 objects/s recovering 2023-10-01T11:03:39.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:39 smithi078 bash[75299]: cluster 2023-10-01T11:03:38.238907+0000 mgr.y (mgr.25072) 1388 : cluster 0 pgmap v1457: 89 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 4.8 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 20 MiB/s wr, 327 op/s; 34271/238107 objects degraded (14.393%); 864 KiB/s, 13 objects/s recovering 2023-10-01T11:03:39.846 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:39.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:39 smithi173 bash[58816]: cluster 2023-10-01T11:03:38.238907+0000 mgr.y (mgr.25072) 1388 : cluster 0 pgmap v1457: 89 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 4.8 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 20 MiB/s wr, 327 op/s; 34271/238107 objects degraded (14.393%); 864 KiB/s, 13 objects/s recovering 2023-10-01T11:03:39.878 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:39.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:39.881 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:03:39.982 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:40.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:40.011 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:40.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:03:40.112 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:40.140 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:40.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:40.142 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:40.244 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:40.264 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:40.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:40.266 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:03:40.367 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:40.392 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:40.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:40.394 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:03:40.442 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 93 16 79555 79539 53.4473 0 - 0.0175697 2023-10-01T11:03:40.496 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:40.519 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:40.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:40.523 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:40.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:03:40 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:03:40] "GET /metrics HTTP/1.1" 200 36397 "" "Prometheus/2.33.4" 2023-10-01T11:03:40.624 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:40.648 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:40.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:40.651 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:03:40.752 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:40.773 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:40.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:40.776 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:03:40.877 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:40.901 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:40.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:40.904 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:03:41.005 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:41.032 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:41.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:41.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:41.135 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:41.164 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:41.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:41.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:03:41.267 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:41.293 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:41.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:41.296 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:03:41.397 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:41.422 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:41.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:41.425 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:03:41.441 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 94 16 79555 79539 52.8787 0 - 0.0175697 2023-10-01T11:03:41.526 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:41.548 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:41.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:41.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:03:41.651 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:41.676 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:41.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:41.679 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:41.779 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:41.808 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:41.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:41.812 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:41.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:41 smithi078 bash[78274]: cluster 2023-10-01T11:03:40.239600+0000 mgr.y (mgr.25072) 1389 : cluster 0 pgmap v1458: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 4.9 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 5.5 MiB/s wr, 88 op/s; 28903/238659 objects degraded (12.111%); 2988/238659 objects misplaced (1.252%); 8.1 MiB/s, 130 objects/s recovering 2023-10-01T11:03:41.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:41 smithi078 bash[75299]: cluster 2023-10-01T11:03:40.239600+0000 mgr.y (mgr.25072) 1389 : cluster 0 pgmap v1458: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 4.9 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 5.5 MiB/s wr, 88 op/s; 28903/238659 objects degraded (12.111%); 2988/238659 objects misplaced (1.252%); 8.1 MiB/s, 130 objects/s recovering 2023-10-01T11:03:41.912 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:41.924 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:41 smithi173 bash[58816]: cluster 2023-10-01T11:03:40.239600+0000 mgr.y (mgr.25072) 1389 : cluster 0 pgmap v1458: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 4.9 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 5.5 MiB/s wr, 88 op/s; 28903/238659 objects degraded (12.111%); 2988/238659 objects misplaced (1.252%); 8.1 MiB/s, 130 objects/s recovering 2023-10-01T11:03:41.941 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:41.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:41.944 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:42.045 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:42.071 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:42.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:42.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:03:42.080 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-01T11:03:42.174 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:42.200 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:42.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:42.202 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:03:42.303 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:42.328 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:42.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:42.330 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:03:42.433 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:42.441 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 95 16 79555 79539 52.3221 0 - 0.0175697 2023-10-01T11:03:42.460 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:42.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:42.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:03:42.564 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:42.589 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:42.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:42.592 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:03:42.692 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:42.717 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:42.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:42.720 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:03:42.733 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T11:03:42.755 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T11:03:42.782 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T11:03:42.809 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T11:03:42.821 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:42.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:42 smithi078 bash[78274]: audit 2023-10-01T11:03:42.698989+0000 mon.c (mon.2) 211 : audit 0 from='client.? 172.21.15.78:0/506760731' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:03:42.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:42 smithi078 bash[78274]: cluster 2023-10-01T11:03:42.701833+0000 mon.a (mon.0) 1761 : cluster 3 Health check update: Degraded data redundancy: 28903/238659 objects degraded (12.111%), 7 pgs degraded (PG_DEGRADED) 2023-10-01T11:03:42.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:42 smithi078 bash[75299]: audit 2023-10-01T11:03:42.698989+0000 mon.c (mon.2) 211 : audit 0 from='client.? 172.21.15.78:0/506760731' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:03:42.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:42 smithi078 bash[75299]: cluster 2023-10-01T11:03:42.701833+0000 mon.a (mon.0) 1761 : cluster 3 Health check update: Degraded data redundancy: 28903/238659 objects degraded (12.111%), 7 pgs degraded (PG_DEGRADED) 2023-10-01T11:03:42.841 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T11:03:42.851 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:42.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:42.854 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:03:42.870 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T11:03:42.896 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T11:03:42.955 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:42.981 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:42.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:42.983 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:03:43.084 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:43.095 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:42 smithi173 bash[58816]: audit 2023-10-01T11:03:42.698989+0000 mon.c (mon.2) 211 : audit 0 from='client.? 172.21.15.78:0/506760731' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:03:43.096 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:42 smithi173 bash[58816]: cluster 2023-10-01T11:03:42.701833+0000 mon.a (mon.0) 1761 : cluster 3 Health check update: Degraded data redundancy: 28903/238659 objects degraded (12.111%), 7 pgs degraded (PG_DEGRADED) 2023-10-01T11:03:43.109 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:43.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:43.111 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:03:43.212 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:43.243 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:43.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:43.246 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:03:43.346 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:43.374 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:43.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:43.379 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:03:43.441 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 96 16 79555 79539 51.7771 0 - 0.0175697 2023-10-01T11:03:43.480 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:43.507 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:43.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:43.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:03:43.612 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:43.639 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:43.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:43.642 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:03:43.643 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5] out_osds: [6, 7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T11:03:43.643 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T11:03:43.643 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-01T11:03:43.643 INFO:tasks.thrashosds.thrasher:Setting osd 4 primary_affinity to 1.000000 2023-10-01T11:03:43.644 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 4 1 2023-10-01T11:03:43.739 INFO:teuthology.orchestra.run.smithi078.stderr:nodeep-scrub is set 2023-10-01T11:03:43.742 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:43.751 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:43 smithi173 bash[58816]: cluster 2023-10-01T11:03:42.240085+0000 mgr.y (mgr.25072) 1390 : cluster 0 pgmap v1459: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 4.9 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 3.5 MiB/s wr, 55 op/s; 28903/238659 objects degraded (12.111%); 2988/238659 objects misplaced (1.252%); 6.3 MiB/s, 100 objects/s recovering 2023-10-01T11:03:43.751 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:43 smithi173 bash[58816]: audit 2023-10-01T11:03:42.982623+0000 mon.a (mon.0) 1762 : audit 1 from='client.? 172.21.15.78:0/748243198' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:03:43.769 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:43.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:43.771 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:43.872 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:43.898 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:43.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:43.900 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:43.962 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:43 smithi078 bash[75299]: cluster 2023-10-01T11:03:42.240085+0000 mgr.y (mgr.25072) 1390 : cluster 0 pgmap v1459: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 4.9 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 3.5 MiB/s wr, 55 op/s; 28903/238659 objects degraded (12.111%); 2988/238659 objects misplaced (1.252%); 6.3 MiB/s, 100 objects/s recovering 2023-10-01T11:03:43.962 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:43 smithi078 bash[75299]: audit 2023-10-01T11:03:42.982623+0000 mon.a (mon.0) 1762 : audit 1 from='client.? 172.21.15.78:0/748243198' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:03:43.962 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:43 smithi078 bash[78274]: cluster 2023-10-01T11:03:42.240085+0000 mgr.y (mgr.25072) 1390 : cluster 0 pgmap v1459: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 4.9 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 3.5 MiB/s wr, 55 op/s; 28903/238659 objects degraded (12.111%); 2988/238659 objects misplaced (1.252%); 6.3 MiB/s, 100 objects/s recovering 2023-10-01T11:03:43.962 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:43 smithi078 bash[78274]: audit 2023-10-01T11:03:42.982623+0000 mon.a (mon.0) 1762 : audit 1 from='client.? 172.21.15.78:0/748243198' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:03:44.001 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:44.023 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:44.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:44.026 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:03:44.127 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:44.154 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:44.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:44.156 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:44.257 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:44.281 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:44.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:44.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:03:44.383 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:44.409 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:44.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:44.411 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:03:44.441 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 97 16 79555 79539 51.2433 0 - 0.0175697 2023-10-01T11:03:44.512 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:44.539 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:44.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:44.542 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:03:44.642 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:44.667 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:44.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:44.669 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:03:44.770 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:44.779 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:44 smithi078 bash[78274]: cluster 2023-10-01T11:03:43.719529+0000 mon.a (mon.0) 1763 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:03:44.779 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:44 smithi078 bash[78274]: audit 2023-10-01T11:03:43.730704+0000 mon.a (mon.0) 1764 : audit 1 from='client.? 172.21.15.78:0/748243198' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T11:03:44.779 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:44 smithi078 bash[78274]: cluster 2023-10-01T11:03:43.730792+0000 mon.a (mon.0) 1765 : cluster 0 osdmap e306: 8 total, 8 up, 6 in 2023-10-01T11:03:44.779 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:44 smithi078 bash[78274]: cluster 2023-10-01T11:03:43.930340+0000 osd.5 (osd.5) 312 : cluster 3 failed to encode map e306 with expected crc 2023-10-01T11:03:44.779 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:44 smithi078 bash[78274]: cluster 2023-10-01T11:03:43.930393+0000 osd.7 (osd.7) 331 : cluster 3 failed to encode map e306 with expected crc 2023-10-01T11:03:44.779 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:44 smithi078 bash[78274]: cluster 2023-10-01T11:03:43.932207+0000 osd.6 (osd.6) 285 : cluster 3 failed to encode map e306 with expected crc 2023-10-01T11:03:44.779 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:44 smithi078 bash[75299]: cluster 2023-10-01T11:03:43.719529+0000 mon.a (mon.0) 1763 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:03:44.780 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:44 smithi078 bash[75299]: audit 2023-10-01T11:03:43.730704+0000 mon.a (mon.0) 1764 : audit 1 from='client.? 172.21.15.78:0/748243198' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T11:03:44.780 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:44 smithi078 bash[75299]: cluster 2023-10-01T11:03:43.730792+0000 mon.a (mon.0) 1765 : cluster 0 osdmap e306: 8 total, 8 up, 6 in 2023-10-01T11:03:44.780 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:44 smithi078 bash[75299]: cluster 2023-10-01T11:03:43.930340+0000 osd.5 (osd.5) 312 : cluster 3 failed to encode map e306 with expected crc 2023-10-01T11:03:44.780 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:44 smithi078 bash[75299]: cluster 2023-10-01T11:03:43.930393+0000 osd.7 (osd.7) 331 : cluster 3 failed to encode map e306 with expected crc 2023-10-01T11:03:44.780 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:44 smithi078 bash[75299]: cluster 2023-10-01T11:03:43.932207+0000 osd.6 (osd.6) 285 : cluster 3 failed to encode map e306 with expected crc 2023-10-01T11:03:44.793 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:44.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:44.796 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:03:44.897 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:44.919 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:44.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:44.922 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:03:45.023 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:45.034 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:44 smithi173 bash[58816]: cluster 2023-10-01T11:03:43.719529+0000 mon.a (mon.0) 1763 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:03:45.034 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:44 smithi173 bash[58816]: audit 2023-10-01T11:03:43.730704+0000 mon.a (mon.0) 1764 : audit 1 from='client.? 172.21.15.78:0/748243198' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T11:03:45.034 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:44 smithi173 bash[58816]: cluster 2023-10-01T11:03:43.730792+0000 mon.a (mon.0) 1765 : cluster 0 osdmap e306: 8 total, 8 up, 6 in 2023-10-01T11:03:45.035 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:44 smithi173 bash[58816]: cluster 2023-10-01T11:03:43.930340+0000 osd.5 (osd.5) 312 : cluster 3 failed to encode map e306 with expected crc 2023-10-01T11:03:45.035 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:44 smithi173 bash[58816]: cluster 2023-10-01T11:03:43.930393+0000 osd.7 (osd.7) 331 : cluster 3 failed to encode map e306 with expected crc 2023-10-01T11:03:45.035 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:44 smithi173 bash[58816]: cluster 2023-10-01T11:03:43.932207+0000 osd.6 (osd.6) 285 : cluster 3 failed to encode map e306 with expected crc 2023-10-01T11:03:45.050 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:45.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:45.054 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:03:45.155 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:45.186 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:45.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:45.189 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:03:45.290 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:45.321 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:45.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:45.323 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:03:45.425 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:45.442 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 98 16 79555 79539 50.7204 0 - 0.0175697 2023-10-01T11:03:45.452 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:45.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:45.456 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:03:45.556 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:45.583 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:45.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:45.585 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:03:45.687 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:45.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:45.718 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:45.718 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:45.819 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:45.830 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:45 smithi173 bash[58816]: cluster 2023-10-01T11:03:44.240648+0000 mgr.y (mgr.25072) 1391 : cluster 0 pgmap v1461: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 4.9 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 3.4 MiB/s wr, 53 op/s; 28903/238659 objects degraded (12.111%); 2398/238659 objects misplaced (1.005%); 11 MiB/s, 171 objects/s recovering 2023-10-01T11:03:45.844 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:45.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:45.847 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:03:45.948 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:45.972 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:45.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:45.975 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:46.076 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:46.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:45 smithi078 bash[78274]: cluster 2023-10-01T11:03:44.240648+0000 mgr.y (mgr.25072) 1391 : cluster 0 pgmap v1461: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 4.9 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 3.4 MiB/s wr, 53 op/s; 28903/238659 objects degraded (12.111%); 2398/238659 objects misplaced (1.005%); 11 MiB/s, 171 objects/s recovering 2023-10-01T11:03:46.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:45 smithi078 bash[75299]: cluster 2023-10-01T11:03:44.240648+0000 mgr.y (mgr.25072) 1391 : cluster 0 pgmap v1461: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 4.9 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 3.4 MiB/s wr, 53 op/s; 28903/238659 objects degraded (12.111%); 2398/238659 objects misplaced (1.005%); 11 MiB/s, 171 objects/s recovering 2023-10-01T11:03:46.096 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:46.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:46.099 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:03:46.200 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:46.227 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:46.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:46.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:03:46.333 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:46.363 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:46.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:46.366 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:03:46.442 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 99 16 79555 79539 50.2081 0 - 0.0175697 2023-10-01T11:03:46.467 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:46.488 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:46.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:46.491 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:03:46.592 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:46.622 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:46.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:46.624 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:03:46.725 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:46.755 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:46.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:46.758 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:03:46.858 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:46.881 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:46.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:46.883 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:03:46.984 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:47.010 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:47.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:47.015 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:03:47.042 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T11:03:47.116 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:47.139 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:47.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:47.141 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:03:47.242 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:47.264 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:47.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:47.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:47.366 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:47.395 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:47.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:47.398 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:03:47.442 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:2023-10-01T11:03:47.441976+0000 min lat: 0.00136129 max lat: 10.8051 avg lat: 0.0175697 2023-10-01T11:03:47.442 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T11:03:47.442 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 100 16 79555 79539 49.706 0 - 0.0175697 2023-10-01T11:03:47.499 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:47.523 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:47.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:47.525 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:47.553 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:03:47.625 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:47.648 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:47.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:47.650 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:47.751 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:47.773 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:47.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:47.775 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:03:47.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:47 smithi078 bash[75299]: cluster 2023-10-01T11:03:46.241318+0000 mgr.y (mgr.25072) 1392 : cluster 0 pgmap v1462: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 4.9 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 2.8 MiB/s wr, 44 op/s; 28903/238659 objects degraded (12.111%); 1688/238659 objects misplaced (0.707%); 13 MiB/s, 213 objects/s recovering 2023-10-01T11:03:47.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:47 smithi078 bash[75299]: audit 2023-10-01T11:03:47.457810+0000 mon.a (mon.0) 1766 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:03:47.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:47 smithi078 bash[75299]: cluster 2023-10-01T11:03:47.709813+0000 mon.a (mon.0) 1767 : cluster 0 osdmap e307: 8 total, 8 up, 6 in 2023-10-01T11:03:47.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:47 smithi078 bash[78274]: cluster 2023-10-01T11:03:46.241318+0000 mgr.y (mgr.25072) 1392 : cluster 0 pgmap v1462: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 4.9 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 2.8 MiB/s wr, 44 op/s; 28903/238659 objects degraded (12.111%); 1688/238659 objects misplaced (0.707%); 13 MiB/s, 213 objects/s recovering 2023-10-01T11:03:47.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:47 smithi078 bash[78274]: audit 2023-10-01T11:03:47.457810+0000 mon.a (mon.0) 1766 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:03:47.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:47 smithi078 bash[78274]: cluster 2023-10-01T11:03:47.709813+0000 mon.a (mon.0) 1767 : cluster 0 osdmap e307: 8 total, 8 up, 6 in 2023-10-01T11:03:47.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:47 smithi173 bash[58816]: cluster 2023-10-01T11:03:46.241318+0000 mgr.y (mgr.25072) 1392 : cluster 0 pgmap v1462: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 4.9 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 2.8 MiB/s wr, 44 op/s; 28903/238659 objects degraded (12.111%); 1688/238659 objects misplaced (0.707%); 13 MiB/s, 213 objects/s recovering 2023-10-01T11:03:47.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:47 smithi173 bash[58816]: audit 2023-10-01T11:03:47.457810+0000 mon.a (mon.0) 1766 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:03:47.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:47 smithi173 bash[58816]: cluster 2023-10-01T11:03:47.709813+0000 mon.a (mon.0) 1767 : cluster 0 osdmap e307: 8 total, 8 up, 6 in 2023-10-01T11:03:47.876 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:47.906 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:47.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:47.908 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:03:48.009 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:48.034 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:48.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:48.037 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:03:48.137 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:48.159 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:48.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:48.162 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:03:48.263 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:48.282 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:48.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:48.284 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:48.385 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:48.415 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:48.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:48.417 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:03:48.442 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 101 16 79555 79539 49.2139 0 - 0.0175697 2023-10-01T11:03:48.518 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:48.543 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:48.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:48.546 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:48.569 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:03:48.647 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:48.673 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:48.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:48.676 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:48.777 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:48.796 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:48.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:48.798 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:03:48.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:48 smithi173 bash[58816]: cluster 2023-10-01T11:03:47.722977+0000 osd.7 (osd.7) 332 : cluster 3 failed to encode map e307 with expected crc 2023-10-01T11:03:48.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:48 smithi173 bash[58816]: cluster 2023-10-01T11:03:47.732856+0000 osd.5 (osd.5) 313 : cluster 3 failed to encode map e307 with expected crc 2023-10-01T11:03:48.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:48 smithi173 bash[58816]: cluster 2023-10-01T11:03:48.003031+0000 osd.6 (osd.6) 286 : cluster 3 failed to encode map e307 with expected crc 2023-10-01T11:03:48.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:48 smithi173 bash[58816]: cluster 2023-10-01T11:03:48.003053+0000 osd.7 (osd.7) 333 : cluster 3 failed to encode map e307 with expected crc 2023-10-01T11:03:48.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:48 smithi173 bash[58816]: cluster 2023-10-01T11:03:48.724296+0000 mon.a (mon.0) 1768 : cluster 0 osdmap e308: 8 total, 8 up, 6 in 2023-10-01T11:03:48.899 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:48.907 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:48 smithi078 bash[78274]: cluster 2023-10-01T11:03:47.722977+0000 osd.7 (osd.7) 332 : cluster 3 failed to encode map e307 with expected crc 2023-10-01T11:03:48.907 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:48 smithi078 bash[78274]: cluster 2023-10-01T11:03:47.732856+0000 osd.5 (osd.5) 313 : cluster 3 failed to encode map e307 with expected crc 2023-10-01T11:03:48.907 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:48 smithi078 bash[78274]: cluster 2023-10-01T11:03:48.003031+0000 osd.6 (osd.6) 286 : cluster 3 failed to encode map e307 with expected crc 2023-10-01T11:03:48.907 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:48 smithi078 bash[78274]: cluster 2023-10-01T11:03:48.003053+0000 osd.7 (osd.7) 333 : cluster 3 failed to encode map e307 with expected crc 2023-10-01T11:03:48.907 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:48 smithi078 bash[78274]: cluster 2023-10-01T11:03:48.724296+0000 mon.a (mon.0) 1768 : cluster 0 osdmap e308: 8 total, 8 up, 6 in 2023-10-01T11:03:48.908 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:48 smithi078 bash[75299]: cluster 2023-10-01T11:03:47.722977+0000 osd.7 (osd.7) 332 : cluster 3 failed to encode map e307 with expected crc 2023-10-01T11:03:48.908 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:48 smithi078 bash[75299]: cluster 2023-10-01T11:03:47.732856+0000 osd.5 (osd.5) 313 : cluster 3 failed to encode map e307 with expected crc 2023-10-01T11:03:48.908 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:48 smithi078 bash[75299]: cluster 2023-10-01T11:03:48.003031+0000 osd.6 (osd.6) 286 : cluster 3 failed to encode map e307 with expected crc 2023-10-01T11:03:48.908 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:48 smithi078 bash[75299]: cluster 2023-10-01T11:03:48.003053+0000 osd.7 (osd.7) 333 : cluster 3 failed to encode map e307 with expected crc 2023-10-01T11:03:48.908 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:48 smithi078 bash[75299]: cluster 2023-10-01T11:03:48.724296+0000 mon.a (mon.0) 1768 : cluster 0 osdmap e308: 8 total, 8 up, 6 in 2023-10-01T11:03:48.923 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:48.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:48.925 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:03:49.026 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:49.052 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:49.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:49.055 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:49.155 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:49.182 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:49.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:49.185 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:03:49.286 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:49.307 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:49.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:49.308 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:03:49.409 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:49.437 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:49.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:49.441 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:49.442 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 102 16 79555 79539 48.7314 0 - 0.0175697 2023-10-01T11:03:49.542 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:49.571 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:49.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:49.574 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:49.674 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:49.704 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:49.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:49.707 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:49.808 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:49.818 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:49 smithi173 bash[58816]: cluster 2023-10-01T11:03:48.241905+0000 mgr.y (mgr.25072) 1393 : cluster 0 pgmap v1464: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 28903/238659 objects degraded (12.111%); 1688/238659 objects misplaced (0.707%); 10 MiB/s, 162 objects/s recovering 2023-10-01T11:03:49.818 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:49 smithi173 bash[58816]: cluster 2023-10-01T11:03:48.722899+0000 osd.5 (osd.5) 314 : cluster 3 failed to encode map e308 with expected crc 2023-10-01T11:03:49.819 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:49 smithi173 bash[58816]: cluster 2023-10-01T11:03:49.435851+0000 osd.6 (osd.6) 287 : cluster 3 failed to encode map e308 with expected crc 2023-10-01T11:03:49.819 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:49 smithi173 bash[58816]: cluster 2023-10-01T11:03:49.437210+0000 osd.6 (osd.6) 288 : cluster 3 failed to encode map e308 with expected crc 2023-10-01T11:03:49.833 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:49.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:49.837 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:03:49.894 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T11:03:49.896 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T11:03:49.896 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T11:03:49.896 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T11:03:49.896 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T11:03:49.897 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T11:03:49.897 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T11:03:49.897 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T11:03:49.897 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T11:03:49.937 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:49.959 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:49.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:49.961 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:03:50.062 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:50.069 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:49 smithi078 bash[75299]: cluster 2023-10-01T11:03:48.241905+0000 mgr.y (mgr.25072) 1393 : cluster 0 pgmap v1464: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 28903/238659 objects degraded (12.111%); 1688/238659 objects misplaced (0.707%); 10 MiB/s, 162 objects/s recovering 2023-10-01T11:03:50.069 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:49 smithi078 bash[75299]: cluster 2023-10-01T11:03:48.722899+0000 osd.5 (osd.5) 314 : cluster 3 failed to encode map e308 with expected crc 2023-10-01T11:03:50.069 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:49 smithi078 bash[75299]: cluster 2023-10-01T11:03:49.435851+0000 osd.6 (osd.6) 287 : cluster 3 failed to encode map e308 with expected crc 2023-10-01T11:03:50.069 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:49 smithi078 bash[75299]: cluster 2023-10-01T11:03:49.437210+0000 osd.6 (osd.6) 288 : cluster 3 failed to encode map e308 with expected crc 2023-10-01T11:03:50.069 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:49 smithi078 bash[78274]: cluster 2023-10-01T11:03:48.241905+0000 mgr.y (mgr.25072) 1393 : cluster 0 pgmap v1464: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 80 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 28903/238659 objects degraded (12.111%); 1688/238659 objects misplaced (0.707%); 10 MiB/s, 162 objects/s recovering 2023-10-01T11:03:50.069 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:49 smithi078 bash[78274]: cluster 2023-10-01T11:03:48.722899+0000 osd.5 (osd.5) 314 : cluster 3 failed to encode map e308 with expected crc 2023-10-01T11:03:50.070 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:49 smithi078 bash[78274]: cluster 2023-10-01T11:03:49.435851+0000 osd.6 (osd.6) 287 : cluster 3 failed to encode map e308 with expected crc 2023-10-01T11:03:50.070 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:49 smithi078 bash[78274]: cluster 2023-10-01T11:03:49.437210+0000 osd.6 (osd.6) 288 : cluster 3 failed to encode map e308 with expected crc 2023-10-01T11:03:50.081 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:50.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:50.084 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:03:50.185 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:50.214 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:50.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:50.217 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:03:50.318 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:50.326 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:03:50 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:03:50] "GET /metrics HTTP/1.1" 200 36409 "" "Prometheus/2.33.4" 2023-10-01T11:03:50.343 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:50.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:50.345 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:03:50.442 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 103 16 79555 79539 48.2583 0 - 0.0175697 2023-10-01T11:03:50.447 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:50.473 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:50.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:50.475 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:03:50.576 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:50.596 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:50.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:50.599 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:03:50.699 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:50.723 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:50.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:50.726 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:03:50.827 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:50.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:50.854 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:50.855 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:50.955 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:50.968 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:50 smithi078 bash[75299]: cluster 2023-10-01T11:03:49.755946+0000 osd.5 (osd.5) 315 : cluster 3 failed to encode map e309 with expected crc 2023-10-01T11:03:50.968 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:50 smithi078 bash[75299]: cluster 2023-10-01T11:03:49.758656+0000 osd.5 (osd.5) 316 : cluster 3 failed to encode map e309 with expected crc 2023-10-01T11:03:50.969 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:50 smithi078 bash[75299]: cluster 2023-10-01T11:03:49.761509+0000 mon.a (mon.0) 1769 : cluster 0 osdmap e309: 8 total, 8 up, 6 in 2023-10-01T11:03:50.969 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:50 smithi078 bash[75299]: cluster 2023-10-01T11:03:49.762308+0000 osd.5 (osd.5) 317 : cluster 3 failed to encode map e309 with expected crc 2023-10-01T11:03:50.969 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:50 smithi078 bash[78274]: cluster 2023-10-01T11:03:49.755946+0000 osd.5 (osd.5) 315 : cluster 3 failed to encode map e309 with expected crc 2023-10-01T11:03:50.969 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:50 smithi078 bash[78274]: cluster 2023-10-01T11:03:49.758656+0000 osd.5 (osd.5) 316 : cluster 3 failed to encode map e309 with expected crc 2023-10-01T11:03:50.969 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:50 smithi078 bash[78274]: cluster 2023-10-01T11:03:49.761509+0000 mon.a (mon.0) 1769 : cluster 0 osdmap e309: 8 total, 8 up, 6 in 2023-10-01T11:03:50.969 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:50 smithi078 bash[78274]: cluster 2023-10-01T11:03:49.762308+0000 osd.5 (osd.5) 317 : cluster 3 failed to encode map e309 with expected crc 2023-10-01T11:03:50.981 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:50.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:50.983 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:03:51.084 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:51.108 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:51.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:51.110 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:03:51.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:50 smithi173 bash[58816]: cluster 2023-10-01T11:03:49.755946+0000 osd.5 (osd.5) 315 : cluster 3 failed to encode map e309 with expected crc 2023-10-01T11:03:51.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:50 smithi173 bash[58816]: cluster 2023-10-01T11:03:49.758656+0000 osd.5 (osd.5) 316 : cluster 3 failed to encode map e309 with expected crc 2023-10-01T11:03:51.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:50 smithi173 bash[58816]: cluster 2023-10-01T11:03:49.761509+0000 mon.a (mon.0) 1769 : cluster 0 osdmap e309: 8 total, 8 up, 6 in 2023-10-01T11:03:51.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:50 smithi173 bash[58816]: cluster 2023-10-01T11:03:49.762308+0000 osd.5 (osd.5) 317 : cluster 3 failed to encode map e309 with expected crc 2023-10-01T11:03:51.210 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:51.241 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:51.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:51.243 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:51.344 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:51.371 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:51.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:51.374 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:51.442 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 104 16 79555 79539 47.7942 0 - 0.0175697 2023-10-01T11:03:51.475 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:51.503 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:51.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:51.505 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:03:51.606 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:51.629 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:51.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:51.631 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:03:51.732 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:51.750 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:51.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:51.752 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:03:51.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:51 smithi078 bash[75299]: cluster 2023-10-01T11:03:50.242350+0000 mgr.y (mgr.25072) 1394 : cluster 0 pgmap v1467: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 117 KiB/s wr, 1 op/s; 28905/238692 objects degraded (12.110%); 1945/238692 objects misplaced (0.815%); 23 MiB/s, 371 objects/s recovering 2023-10-01T11:03:51.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:51 smithi078 bash[75299]: cluster 2023-10-01T11:03:50.598748+0000 osd.6 (osd.6) 289 : cluster 3 failed to encode map e309 with expected crc 2023-10-01T11:03:51.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:51 smithi078 bash[75299]: cluster 2023-10-01T11:03:50.750245+0000 mon.a (mon.0) 1770 : cluster 3 Health check update: Degraded data redundancy: 28905/238692 objects degraded (12.110%), 7 pgs degraded (PG_DEGRADED) 2023-10-01T11:03:51.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:51 smithi078 bash[75299]: cluster 2023-10-01T11:03:50.765359+0000 osd.5 (osd.5) 318 : cluster 3 failed to encode map e310 with expected crc 2023-10-01T11:03:51.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:51 smithi078 bash[75299]: cluster 2023-10-01T11:03:50.767814+0000 osd.5 (osd.5) 319 : cluster 3 failed to encode map e310 with expected crc 2023-10-01T11:03:51.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:51 smithi078 bash[75299]: cluster 2023-10-01T11:03:50.767833+0000 osd.7 (osd.7) 334 : cluster 3 failed to encode map e310 with expected crc 2023-10-01T11:03:51.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:51 smithi078 bash[75299]: cluster 2023-10-01T11:03:50.769221+0000 mon.a (mon.0) 1771 : cluster 0 osdmap e310: 8 total, 8 up, 6 in 2023-10-01T11:03:51.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:51 smithi078 bash[75299]: cluster 2023-10-01T11:03:50.769926+0000 osd.5 (osd.5) 320 : cluster 3 failed to encode map e310 with expected crc 2023-10-01T11:03:51.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:51 smithi078 bash[78274]: cluster 2023-10-01T11:03:50.242350+0000 mgr.y (mgr.25072) 1394 : cluster 0 pgmap v1467: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 117 KiB/s wr, 1 op/s; 28905/238692 objects degraded (12.110%); 1945/238692 objects misplaced (0.815%); 23 MiB/s, 371 objects/s recovering 2023-10-01T11:03:51.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:51 smithi078 bash[78274]: cluster 2023-10-01T11:03:50.598748+0000 osd.6 (osd.6) 289 : cluster 3 failed to encode map e309 with expected crc 2023-10-01T11:03:51.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:51 smithi078 bash[78274]: cluster 2023-10-01T11:03:50.750245+0000 mon.a (mon.0) 1770 : cluster 3 Health check update: Degraded data redundancy: 28905/238692 objects degraded (12.110%), 7 pgs degraded (PG_DEGRADED) 2023-10-01T11:03:51.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:51 smithi078 bash[78274]: cluster 2023-10-01T11:03:50.765359+0000 osd.5 (osd.5) 318 : cluster 3 failed to encode map e310 with expected crc 2023-10-01T11:03:51.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:51 smithi078 bash[78274]: cluster 2023-10-01T11:03:50.767814+0000 osd.5 (osd.5) 319 : cluster 3 failed to encode map e310 with expected crc 2023-10-01T11:03:51.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:51 smithi078 bash[78274]: cluster 2023-10-01T11:03:50.767833+0000 osd.7 (osd.7) 334 : cluster 3 failed to encode map e310 with expected crc 2023-10-01T11:03:51.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:51 smithi078 bash[78274]: cluster 2023-10-01T11:03:50.769221+0000 mon.a (mon.0) 1771 : cluster 0 osdmap e310: 8 total, 8 up, 6 in 2023-10-01T11:03:51.833 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:51 smithi078 bash[78274]: cluster 2023-10-01T11:03:50.769926+0000 osd.5 (osd.5) 320 : cluster 3 failed to encode map e310 with expected crc 2023-10-01T11:03:51.853 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:51.871 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:51.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:51.873 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:03:51.974 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:51.993 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:51.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:51.995 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:03:52.096 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:52.107 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:51 smithi173 bash[58816]: cluster 2023-10-01T11:03:50.242350+0000 mgr.y (mgr.25072) 1394 : cluster 0 pgmap v1467: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 117 KiB/s wr, 1 op/s; 28905/238692 objects degraded (12.110%); 1945/238692 objects misplaced (0.815%); 23 MiB/s, 371 objects/s recovering 2023-10-01T11:03:52.107 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:51 smithi173 bash[58816]: cluster 2023-10-01T11:03:50.598748+0000 osd.6 (osd.6) 289 : cluster 3 failed to encode map e309 with expected crc 2023-10-01T11:03:52.107 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:51 smithi173 bash[58816]: cluster 2023-10-01T11:03:50.750245+0000 mon.a (mon.0) 1770 : cluster 3 Health check update: Degraded data redundancy: 28905/238692 objects degraded (12.110%), 7 pgs degraded (PG_DEGRADED) 2023-10-01T11:03:52.107 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:51 smithi173 bash[58816]: cluster 2023-10-01T11:03:50.765359+0000 osd.5 (osd.5) 318 : cluster 3 failed to encode map e310 with expected crc 2023-10-01T11:03:52.107 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:51 smithi173 bash[58816]: cluster 2023-10-01T11:03:50.767814+0000 osd.5 (osd.5) 319 : cluster 3 failed to encode map e310 with expected crc 2023-10-01T11:03:52.107 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:51 smithi173 bash[58816]: cluster 2023-10-01T11:03:50.767833+0000 osd.7 (osd.7) 334 : cluster 3 failed to encode map e310 with expected crc 2023-10-01T11:03:52.107 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:51 smithi173 bash[58816]: cluster 2023-10-01T11:03:50.769221+0000 mon.a (mon.0) 1771 : cluster 0 osdmap e310: 8 total, 8 up, 6 in 2023-10-01T11:03:52.107 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:51 smithi173 bash[58816]: cluster 2023-10-01T11:03:50.769926+0000 osd.5 (osd.5) 320 : cluster 3 failed to encode map e310 with expected crc 2023-10-01T11:03:52.124 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:52.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:52.126 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:03:52.227 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:52.254 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:52.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:52.256 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:03:52.357 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:52.388 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:52.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:52.391 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:52.442 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 105 16 79555 79539 47.3391 0 - 0.0175697 2023-10-01T11:03:52.491 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:52.509 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:52.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:52.511 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:03:52.612 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:52.630 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:52.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:52.633 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:03:52.733 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:52.763 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:52.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:52.766 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:52.867 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:52.892 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:52.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:52.894 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:03:52.995 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:53.016 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:53.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:53.018 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:03:53.119 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:53.147 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:53.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:53.152 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:03:53.252 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:53.279 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:53.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:53.282 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:53.383 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:53.414 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:53.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:53.416 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:03:53.442 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 106 16 79555 79539 46.8925 0 - 0.0175697 2023-10-01T11:03:53.517 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:53.542 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:53.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:53.545 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:03:53.646 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:53.677 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:53.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:53.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:03:53.780 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:53.790 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:53 smithi173 bash[58816]: cluster 2023-10-01T11:03:52.242732+0000 mgr.y (mgr.25072) 1395 : cluster 0 pgmap v1469: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 155 KiB/s wr, 2 op/s; 28905/238692 objects degraded (12.110%); 1945/238692 objects misplaced (0.815%); 21 MiB/s, 334 objects/s recovering 2023-10-01T11:03:53.805 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:53.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:53.808 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:53.909 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:53.935 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:53.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:53.938 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:03:54.039 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:54.063 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:54.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:54.065 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:54.106 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:53 smithi078 bash[75299]: cluster 2023-10-01T11:03:52.242732+0000 mgr.y (mgr.25072) 1395 : cluster 0 pgmap v1469: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 155 KiB/s wr, 2 op/s; 28905/238692 objects degraded (12.110%); 1945/238692 objects misplaced (0.815%); 21 MiB/s, 334 objects/s recovering 2023-10-01T11:03:54.107 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:53 smithi078 bash[78274]: cluster 2023-10-01T11:03:52.242732+0000 mgr.y (mgr.25072) 1395 : cluster 0 pgmap v1469: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 155 KiB/s wr, 2 op/s; 28905/238692 objects degraded (12.110%); 1945/238692 objects misplaced (0.815%); 21 MiB/s, 334 objects/s recovering 2023-10-01T11:03:54.166 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:54.194 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:54.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:54.196 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:03:54.297 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:54.319 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:54.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:54.321 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:03:54.422 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:54.443 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 107 16 79555 79539 46.4542 0 - 0.0175697 2023-10-01T11:03:54.450 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:54.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:54.453 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:03:54.554 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:54.589 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:54.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:54.591 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:03:54.692 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:54.722 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:54.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:54.725 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:03:54.826 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:54.860 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:54.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:54.862 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:03:54.963 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:54.984 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:54.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:54.987 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:03:55.088 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:55.111 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:55.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:55.113 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:03:55.214 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:55.240 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:55.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:55.243 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:03:55.343 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:55.373 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:55.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:55.376 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:03:55.443 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 108 16 79555 79539 46.0241 0 - 0.0175697 2023-10-01T11:03:55.477 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:55.504 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:55.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:55.506 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:55.607 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:55.635 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:55.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:55.638 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:03:55.739 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:55.770 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:55.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:55.773 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:55.874 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:55.902 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:55.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:55.905 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:56.006 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:56.018 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:55 smithi173 bash[58816]: cluster 2023-10-01T11:03:54.244061+0000 mgr.y (mgr.25072) 1396 : cluster 0 pgmap v1470: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 117 KiB/s wr, 1 op/s; 24328/238692 objects degraded (10.192%); 6016/238692 objects misplaced (2.520%); 21 MiB/s, 337 objects/s recovering 2023-10-01T11:03:56.018 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:55 smithi173 bash[58816]: cluster 2023-10-01T11:03:54.780670+0000 osd.6 (osd.6) 290 : cluster 3 failed to encode map e311 with expected crc 2023-10-01T11:03:56.018 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:55 smithi173 bash[58816]: cluster 2023-10-01T11:03:54.782244+0000 mon.a (mon.0) 1772 : cluster 0 osdmap e311: 8 total, 8 up, 6 in 2023-10-01T11:03:56.018 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:55 smithi173 bash[58816]: cluster 2023-10-01T11:03:54.786912+0000 osd.7 (osd.7) 335 : cluster 3 failed to encode map e311 with expected crc 2023-10-01T11:03:56.033 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:56.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:56.036 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:03:56.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:55 smithi078 bash[75299]: cluster 2023-10-01T11:03:54.244061+0000 mgr.y (mgr.25072) 1396 : cluster 0 pgmap v1470: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 117 KiB/s wr, 1 op/s; 24328/238692 objects degraded (10.192%); 6016/238692 objects misplaced (2.520%); 21 MiB/s, 337 objects/s recovering 2023-10-01T11:03:56.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:55 smithi078 bash[75299]: cluster 2023-10-01T11:03:54.780670+0000 osd.6 (osd.6) 290 : cluster 3 failed to encode map e311 with expected crc 2023-10-01T11:03:56.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:55 smithi078 bash[75299]: cluster 2023-10-01T11:03:54.782244+0000 mon.a (mon.0) 1772 : cluster 0 osdmap e311: 8 total, 8 up, 6 in 2023-10-01T11:03:56.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:55 smithi078 bash[75299]: cluster 2023-10-01T11:03:54.786912+0000 osd.7 (osd.7) 335 : cluster 3 failed to encode map e311 with expected crc 2023-10-01T11:03:56.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:55 smithi078 bash[78274]: cluster 2023-10-01T11:03:54.244061+0000 mgr.y (mgr.25072) 1396 : cluster 0 pgmap v1470: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 117 KiB/s wr, 1 op/s; 24328/238692 objects degraded (10.192%); 6016/238692 objects misplaced (2.520%); 21 MiB/s, 337 objects/s recovering 2023-10-01T11:03:56.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:55 smithi078 bash[78274]: cluster 2023-10-01T11:03:54.780670+0000 osd.6 (osd.6) 290 : cluster 3 failed to encode map e311 with expected crc 2023-10-01T11:03:56.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:55 smithi078 bash[78274]: cluster 2023-10-01T11:03:54.782244+0000 mon.a (mon.0) 1772 : cluster 0 osdmap e311: 8 total, 8 up, 6 in 2023-10-01T11:03:56.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:55 smithi078 bash[78274]: cluster 2023-10-01T11:03:54.786912+0000 osd.7 (osd.7) 335 : cluster 3 failed to encode map e311 with expected crc 2023-10-01T11:03:56.136 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:56.167 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:56.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:56.170 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:56.225 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T11:03:56.271 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:56.296 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:56.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:56.298 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:03:56.399 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:56.421 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:56.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:56.423 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:03:56.443 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 109 16 79555 79539 45.6018 0 - 0.0175697 2023-10-01T11:03:56.524 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:56.551 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:56.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:56.553 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:56.654 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:56.685 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:56.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:56.688 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:03:56.789 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:56.812 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:56.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:56.815 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:56.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:56 smithi078 bash[75299]: cluster 2023-10-01T11:03:55.031901+0000 osd.5 (osd.5) 321 : cluster 3 failed to encode map e311 with expected crc 2023-10-01T11:03:56.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:56 smithi078 bash[75299]: cluster 2023-10-01T11:03:55.789681+0000 osd.5 (osd.5) 322 : cluster 3 failed to encode map e312 with expected crc 2023-10-01T11:03:56.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:56 smithi078 bash[75299]: cluster 2023-10-01T11:03:55.789762+0000 osd.7 (osd.7) 336 : cluster 3 failed to encode map e312 with expected crc 2023-10-01T11:03:56.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:56 smithi078 bash[75299]: cluster 2023-10-01T11:03:55.791686+0000 mon.a (mon.0) 1773 : cluster 0 osdmap e312: 8 total, 8 up, 6 in 2023-10-01T11:03:56.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:56 smithi078 bash[75299]: cluster 2023-10-01T11:03:55.837530+0000 osd.6 (osd.6) 291 : cluster 3 failed to encode map e312 with expected crc 2023-10-01T11:03:56.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:56 smithi078 bash[78274]: cluster 2023-10-01T11:03:55.031901+0000 osd.5 (osd.5) 321 : cluster 3 failed to encode map e311 with expected crc 2023-10-01T11:03:56.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:56 smithi078 bash[78274]: cluster 2023-10-01T11:03:55.789681+0000 osd.5 (osd.5) 322 : cluster 3 failed to encode map e312 with expected crc 2023-10-01T11:03:56.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:56 smithi078 bash[78274]: cluster 2023-10-01T11:03:55.789762+0000 osd.7 (osd.7) 336 : cluster 3 failed to encode map e312 with expected crc 2023-10-01T11:03:56.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:56 smithi078 bash[78274]: cluster 2023-10-01T11:03:55.791686+0000 mon.a (mon.0) 1773 : cluster 0 osdmap e312: 8 total, 8 up, 6 in 2023-10-01T11:03:56.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:56 smithi078 bash[78274]: cluster 2023-10-01T11:03:55.837530+0000 osd.6 (osd.6) 291 : cluster 3 failed to encode map e312 with expected crc 2023-10-01T11:03:56.915 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:56.946 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:56.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:56.949 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:03:57.050 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:57.060 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:56 smithi173 bash[58816]: cluster 2023-10-01T11:03:55.031901+0000 osd.5 (osd.5) 321 : cluster 3 failed to encode map e311 with expected crc 2023-10-01T11:03:57.060 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:56 smithi173 bash[58816]: cluster 2023-10-01T11:03:55.789681+0000 osd.5 (osd.5) 322 : cluster 3 failed to encode map e312 with expected crc 2023-10-01T11:03:57.060 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:56 smithi173 bash[58816]: cluster 2023-10-01T11:03:55.789762+0000 osd.7 (osd.7) 336 : cluster 3 failed to encode map e312 with expected crc 2023-10-01T11:03:57.061 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:56 smithi173 bash[58816]: cluster 2023-10-01T11:03:55.791686+0000 mon.a (mon.0) 1773 : cluster 0 osdmap e312: 8 total, 8 up, 6 in 2023-10-01T11:03:57.061 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:56 smithi173 bash[58816]: cluster 2023-10-01T11:03:55.837530+0000 osd.6 (osd.6) 291 : cluster 3 failed to encode map e312 with expected crc 2023-10-01T11:03:57.076 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:57.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:57.078 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:03:57.179 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:57.211 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:57.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:57.214 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:03:57.315 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:57.341 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:57.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:57.343 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:03:57.445 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 110 16 79555 79539 45.1873 0 - 0.0175697 2023-10-01T11:03:57.446 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:57.473 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:57.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:57.475 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:03:57.576 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:57.608 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:57.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:57.611 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:03:57.712 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:57.737 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:57.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:57.739 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:03:57.840 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:57.869 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:57.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:57.871 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:57.911 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:03:57.935 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:57 smithi078 bash[78274]: cluster 2023-10-01T11:03:56.244664+0000 mgr.y (mgr.25072) 1397 : cluster 0 pgmap v1473: 89 pgs: 1 active+recovering+undersized+degraded+remapped+laggy, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 21070/238692 objects degraded (8.827%); 9267/238692 objects misplaced (3.882%); 5.3 MiB/s, 85 objects/s recovering 2023-10-01T11:03:57.936 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:57 smithi078 bash[78274]: cluster 2023-10-01T11:03:56.788129+0000 mon.a (mon.0) 1774 : cluster 3 Health check update: Degraded data redundancy: 21070/238692 objects degraded (8.827%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T11:03:57.936 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:57 smithi078 bash[75299]: cluster 2023-10-01T11:03:56.244664+0000 mgr.y (mgr.25072) 1397 : cluster 0 pgmap v1473: 89 pgs: 1 active+recovering+undersized+degraded+remapped+laggy, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 21070/238692 objects degraded (8.827%); 9267/238692 objects misplaced (3.882%); 5.3 MiB/s, 85 objects/s recovering 2023-10-01T11:03:57.936 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:57 smithi078 bash[75299]: cluster 2023-10-01T11:03:56.788129+0000 mon.a (mon.0) 1774 : cluster 3 Health check update: Degraded data redundancy: 21070/238692 objects degraded (8.827%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T11:03:57.973 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:57.997 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:57.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:58.000 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:58.101 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:58.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:57 smithi173 bash[58816]: cluster 2023-10-01T11:03:56.244664+0000 mgr.y (mgr.25072) 1397 : cluster 0 pgmap v1473: 89 pgs: 1 active+recovering+undersized+degraded+remapped+laggy, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 21070/238692 objects degraded (8.827%); 9267/238692 objects misplaced (3.882%); 5.3 MiB/s, 85 objects/s recovering 2023-10-01T11:03:58.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:57 smithi173 bash[58816]: cluster 2023-10-01T11:03:56.788129+0000 mon.a (mon.0) 1774 : cluster 3 Health check update: Degraded data redundancy: 21070/238692 objects degraded (8.827%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T11:03:58.122 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:03:58.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:58.124 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:03:58.225 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:58.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:58.262 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:03:58.262 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:03:58.363 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:58.393 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:58.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:58.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:58.443 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 111 16 79555 79539 44.7802 0 - 0.0175697 2023-10-01T11:03:58.496 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:58.527 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:58.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:58.529 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:03:58.630 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:58.657 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:58.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:58.660 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:58.760 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:58.791 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:58.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:58.795 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:58.896 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:58.924 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:58.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:58.926 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:03:59.027 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:59.056 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:03:59.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:59.059 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:03:59.159 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:59.186 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:03:59.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:59.189 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:03:59.290 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:59.322 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:59.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:59.325 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:03:59.426 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:59.443 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 112 16 79555 79539 44.3804 0 - 0.0175697 2023-10-01T11:03:59.451 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:03:59.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:59.454 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:03:59.555 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:59.589 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:03:59.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:59.593 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:03:59.694 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:59.724 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:59.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:59.728 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:03:59.828 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:59.841 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:03:59 smithi173 bash[58816]: cluster 2023-10-01T11:03:58.245371+0000 mgr.y (mgr.25072) 1398 : cluster 0 pgmap v1474: 89 pgs: 1 active+recovering+undersized+degraded+remapped+laggy, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 25190/238692 objects degraded (10.553%); 5025/238692 objects misplaced (2.105%); 5.3 MiB/s, 84 objects/s recovering 2023-10-01T11:03:59.858 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:03:59.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:59.860 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:03:59.961 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:59.990 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:03:59.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:03:59.993 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:00.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:03:59 smithi078 bash[78274]: cluster 2023-10-01T11:03:58.245371+0000 mgr.y (mgr.25072) 1398 : cluster 0 pgmap v1474: 89 pgs: 1 active+recovering+undersized+degraded+remapped+laggy, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 25190/238692 objects degraded (10.553%); 5025/238692 objects misplaced (2.105%); 5.3 MiB/s, 84 objects/s recovering 2023-10-01T11:04:00.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:03:59 smithi078 bash[75299]: cluster 2023-10-01T11:03:58.245371+0000 mgr.y (mgr.25072) 1398 : cluster 0 pgmap v1474: 89 pgs: 1 active+recovering+undersized+degraded+remapped+laggy, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 25190/238692 objects degraded (10.553%); 5025/238692 objects misplaced (2.105%); 5.3 MiB/s, 84 objects/s recovering 2023-10-01T11:04:00.093 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:00.114 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:00.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:00.116 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:04:00.217 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:00.243 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:00.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:00.246 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:04:00.346 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:00.353 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:04:00 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:04:00] "GET /metrics HTTP/1.1" 200 36409 "" "Prometheus/2.33.4" 2023-10-01T11:04:00.367 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:00.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:00.369 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:04:00.371 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2023-10-01T11:04:00.371 INFO:teuthology.orchestra.run.smithi078.stdout: "ops": [], 2023-10-01T11:04:00.372 INFO:teuthology.orchestra.run.smithi078.stdout: "complaint_time": 30, 2023-10-01T11:04:00.372 INFO:teuthology.orchestra.run.smithi078.stdout: "num_blocked_ops": 0 2023-10-01T11:04:00.372 INFO:teuthology.orchestra.run.smithi078.stdout:} 2023-10-01T11:04:00.443 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 113 16 79555 79539 43.9876 0 - 0.0175697 2023-10-01T11:04:00.470 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:00.494 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:00.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:00.497 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:04:00.597 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:00.621 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:00.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:00.624 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:04:00.724 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:00.748 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:00.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:00.750 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:04:00.851 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:00.884 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:00.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:00.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:04:00.929 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:00 smithi078 bash[78274]: audit 2023-10-01T11:03:59.987707+0000 mon.a (mon.0) 1775 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T11:04:00.944 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:00 smithi078 bash[75299]: audit 2023-10-01T11:03:59.987707+0000 mon.a (mon.0) 1775 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T11:04:00.977 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:04:00.987 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:01.017 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:01.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:01.020 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:04:01.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:00 smithi173 bash[58816]: audit 2023-10-01T11:03:59.987707+0000 mon.a (mon.0) 1775 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T11:04:01.121 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:01.148 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:01.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:01.150 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:04:01.251 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:01.276 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:01.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:01.278 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:04:01.379 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:01.410 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:01.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:01.412 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:04:01.443 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 114 16 79555 79539 43.6018 0 - 0.0175697 2023-10-01T11:04:01.513 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:01.535 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:01.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:01.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:04:01.638 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:01.724 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:01.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:01.730 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:01.833 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:01.838 INFO:teuthology.orchestra.run.smithi078.stderr:set osd.4 primary-affinity to 1 (8655362) 2023-10-01T11:04:01.855 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:01.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:01.859 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:04:01.960 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:01.980 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:01.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:01.982 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:04:02.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:01 smithi078 bash[75299]: cluster 2023-10-01T11:04:00.245914+0000 mgr.y (mgr.25072) 1399 : cluster 0 pgmap v1475: 89 pgs: 1 active+recovering+undersized+degraded+remapped+laggy, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 25186/238692 objects degraded (10.552%); 4977/238692 objects misplaced (2.085%); 5.4 MiB/s, 85 objects/s recovering 2023-10-01T11:04:02.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:01 smithi078 bash[75299]: audit 2023-10-01T11:04:01.330696+0000 mon.a (mon.0) 1776 : audit 1 from='client.? 172.21.15.78:0/3895820170' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-10-01T11:04:02.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:01 smithi078 bash[78274]: cluster 2023-10-01T11:04:00.245914+0000 mgr.y (mgr.25072) 1399 : cluster 0 pgmap v1475: 89 pgs: 1 active+recovering+undersized+degraded+remapped+laggy, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 25186/238692 objects degraded (10.552%); 4977/238692 objects misplaced (2.085%); 5.4 MiB/s, 85 objects/s recovering 2023-10-01T11:04:02.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:01 smithi078 bash[78274]: audit 2023-10-01T11:04:01.330696+0000 mon.a (mon.0) 1776 : audit 1 from='client.? 172.21.15.78:0/3895820170' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-10-01T11:04:02.084 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:02.102 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:02.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:02.104 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:04:02.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:01 smithi173 bash[58816]: cluster 2023-10-01T11:04:00.245914+0000 mgr.y (mgr.25072) 1399 : cluster 0 pgmap v1475: 89 pgs: 1 active+recovering+undersized+degraded+remapped+laggy, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 25186/238692 objects degraded (10.552%); 4977/238692 objects misplaced (2.085%); 5.4 MiB/s, 85 objects/s recovering 2023-10-01T11:04:02.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:01 smithi173 bash[58816]: audit 2023-10-01T11:04:01.330696+0000 mon.a (mon.0) 1776 : audit 1 from='client.? 172.21.15.78:0/3895820170' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-10-01T11:04:02.204 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:02.228 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:02.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:02.235 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:04:02.336 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:02.366 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:02.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:02.371 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:02.443 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 115 16 79555 79539 43.2226 0 - 0.0175697 2023-10-01T11:04:02.471 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:02.501 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:02.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:02.504 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:02.604 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:02.637 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:02.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:02.640 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:02.740 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:02.762 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:02.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:02.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:04:02.864 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:02.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:02 smithi173 bash[58816]: cluster 2023-10-01T11:04:01.813816+0000 osd.7 (osd.7) 337 : cluster 3 failed to encode map e313 with expected crc 2023-10-01T11:04:02.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:02 smithi173 bash[58816]: audit 2023-10-01T11:04:01.815659+0000 mon.a (mon.0) 1777 : audit 1 from='client.? 172.21.15.78:0/3895820170' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-10-01T11:04:02.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:02 smithi173 bash[58816]: cluster 2023-10-01T11:04:01.815792+0000 mon.a (mon.0) 1778 : cluster 0 osdmap e313: 8 total, 8 up, 6 in 2023-10-01T11:04:02.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:02 smithi173 bash[58816]: cluster 2023-10-01T11:04:01.817355+0000 osd.5 (osd.5) 323 : cluster 3 failed to encode map e313 with expected crc 2023-10-01T11:04:02.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:02 smithi173 bash[58816]: audit 2023-10-01T11:04:02.457489+0000 mon.a (mon.0) 1779 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:04:02.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:02 smithi173 bash[58816]: cluster 2023-10-01T11:04:02.707169+0000 mon.a (mon.0) 1780 : cluster 3 Health check update: Degraded data redundancy: 25186/238692 objects degraded (10.552%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T11:04:02.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:02 smithi173 bash[58816]: cluster 2023-10-01T11:04:02.715216+0000 mon.a (mon.0) 1781 : cluster 0 osdmap e314: 8 total, 8 up, 6 in 2023-10-01T11:04:02.894 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:02.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:02.898 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:04:02.998 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:03.038 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:03.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:03.041 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:03.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:02 smithi078 bash[75299]: cluster 2023-10-01T11:04:01.813816+0000 osd.7 (osd.7) 337 : cluster 3 failed to encode map e313 with expected crc 2023-10-01T11:04:03.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:02 smithi078 bash[75299]: audit 2023-10-01T11:04:01.815659+0000 mon.a (mon.0) 1777 : audit 1 from='client.? 172.21.15.78:0/3895820170' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-10-01T11:04:03.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:02 smithi078 bash[75299]: cluster 2023-10-01T11:04:01.815792+0000 mon.a (mon.0) 1778 : cluster 0 osdmap e313: 8 total, 8 up, 6 in 2023-10-01T11:04:03.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:02 smithi078 bash[75299]: cluster 2023-10-01T11:04:01.817355+0000 osd.5 (osd.5) 323 : cluster 3 failed to encode map e313 with expected crc 2023-10-01T11:04:03.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:02 smithi078 bash[75299]: audit 2023-10-01T11:04:02.457489+0000 mon.a (mon.0) 1779 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:04:03.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:02 smithi078 bash[75299]: cluster 2023-10-01T11:04:02.707169+0000 mon.a (mon.0) 1780 : cluster 3 Health check update: Degraded data redundancy: 25186/238692 objects degraded (10.552%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T11:04:03.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:02 smithi078 bash[75299]: cluster 2023-10-01T11:04:02.715216+0000 mon.a (mon.0) 1781 : cluster 0 osdmap e314: 8 total, 8 up, 6 in 2023-10-01T11:04:03.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:02 smithi078 bash[78274]: cluster 2023-10-01T11:04:01.813816+0000 osd.7 (osd.7) 337 : cluster 3 failed to encode map e313 with expected crc 2023-10-01T11:04:03.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:02 smithi078 bash[78274]: audit 2023-10-01T11:04:01.815659+0000 mon.a (mon.0) 1777 : audit 1 from='client.? 172.21.15.78:0/3895820170' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-10-01T11:04:03.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:02 smithi078 bash[78274]: cluster 2023-10-01T11:04:01.815792+0000 mon.a (mon.0) 1778 : cluster 0 osdmap e313: 8 total, 8 up, 6 in 2023-10-01T11:04:03.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:02 smithi078 bash[78274]: cluster 2023-10-01T11:04:01.817355+0000 osd.5 (osd.5) 323 : cluster 3 failed to encode map e313 with expected crc 2023-10-01T11:04:03.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:02 smithi078 bash[78274]: audit 2023-10-01T11:04:02.457489+0000 mon.a (mon.0) 1779 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:04:03.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:02 smithi078 bash[78274]: cluster 2023-10-01T11:04:02.707169+0000 mon.a (mon.0) 1780 : cluster 3 Health check update: Degraded data redundancy: 25186/238692 objects degraded (10.552%), 6 pgs degraded (PG_DEGRADED) 2023-10-01T11:04:03.085 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:02 smithi078 bash[78274]: cluster 2023-10-01T11:04:02.715216+0000 mon.a (mon.0) 1781 : cluster 0 osdmap e314: 8 total, 8 up, 6 in 2023-10-01T11:04:03.141 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:03.173 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:03.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:03.176 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:03.277 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:03.297 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:03.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:03.300 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:04:03.401 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:03.437 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:03.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:03.442 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:04:03.443 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 116 16 79555 79539 42.85 0 - 0.0175697 2023-10-01T11:04:03.542 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:03.565 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:03.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:03.567 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:04:03.668 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:03.697 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:03.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:03.700 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:04:03.801 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:03.829 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:03.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:03.831 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:04:03.932 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:03.955 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:03.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:03.957 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:04:04.058 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:04.066 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:03 smithi078 bash[78274]: cluster 2023-10-01T11:04:02.246550+0000 mgr.y (mgr.25072) 1400 : cluster 0 pgmap v1477: 89 pgs: 1 active+recovering+undersized+degraded+remapped+laggy, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 25186/238692 objects degraded (10.552%); 4977/238692 objects misplaced (2.085%); 1.6 MiB/s, 23 objects/s recovering 2023-10-01T11:04:04.066 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:03 smithi078 bash[78274]: cluster 2023-10-01T11:04:02.711406+0000 osd.7 (osd.7) 338 : cluster 3 failed to encode map e314 with expected crc 2023-10-01T11:04:04.066 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:03 smithi078 bash[78274]: cluster 2023-10-01T11:04:02.718066+0000 osd.5 (osd.5) 324 : cluster 3 failed to encode map e314 with expected crc 2023-10-01T11:04:04.066 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:03 smithi078 bash[78274]: cluster 2023-10-01T11:04:03.721327+0000 mon.a (mon.0) 1782 : cluster 0 osdmap e315: 8 total, 8 up, 6 in 2023-10-01T11:04:04.066 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:03 smithi078 bash[75299]: cluster 2023-10-01T11:04:02.246550+0000 mgr.y (mgr.25072) 1400 : cluster 0 pgmap v1477: 89 pgs: 1 active+recovering+undersized+degraded+remapped+laggy, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 25186/238692 objects degraded (10.552%); 4977/238692 objects misplaced (2.085%); 1.6 MiB/s, 23 objects/s recovering 2023-10-01T11:04:04.067 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:03 smithi078 bash[75299]: cluster 2023-10-01T11:04:02.711406+0000 osd.7 (osd.7) 338 : cluster 3 failed to encode map e314 with expected crc 2023-10-01T11:04:04.067 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:03 smithi078 bash[75299]: cluster 2023-10-01T11:04:02.718066+0000 osd.5 (osd.5) 324 : cluster 3 failed to encode map e314 with expected crc 2023-10-01T11:04:04.067 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:03 smithi078 bash[75299]: cluster 2023-10-01T11:04:03.721327+0000 mon.a (mon.0) 1782 : cluster 0 osdmap e315: 8 total, 8 up, 6 in 2023-10-01T11:04:04.078 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:04.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:04.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:04:04.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:03 smithi173 bash[58816]: cluster 2023-10-01T11:04:02.246550+0000 mgr.y (mgr.25072) 1400 : cluster 0 pgmap v1477: 89 pgs: 1 active+recovering+undersized+degraded+remapped+laggy, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 25186/238692 objects degraded (10.552%); 4977/238692 objects misplaced (2.085%); 1.6 MiB/s, 23 objects/s recovering 2023-10-01T11:04:04.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:03 smithi173 bash[58816]: cluster 2023-10-01T11:04:02.711406+0000 osd.7 (osd.7) 338 : cluster 3 failed to encode map e314 with expected crc 2023-10-01T11:04:04.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:03 smithi173 bash[58816]: cluster 2023-10-01T11:04:02.718066+0000 osd.5 (osd.5) 324 : cluster 3 failed to encode map e314 with expected crc 2023-10-01T11:04:04.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:03 smithi173 bash[58816]: cluster 2023-10-01T11:04:03.721327+0000 mon.a (mon.0) 1782 : cluster 0 osdmap e315: 8 total, 8 up, 6 in 2023-10-01T11:04:04.182 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:04.210 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:04.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:04.213 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:04:04.314 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:04.347 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:04.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:04.352 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:04:04.396 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-01T11:04:04.444 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 117 16 79555 79539 42.4838 0 - 0.0175697 2023-10-01T11:04:04.452 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:04.473 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:04.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:04.474 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:04:04.575 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:04.602 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:04.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:04.607 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:04:04.708 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:04.733 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:04.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:04.736 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:04:04.837 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:04.858 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:04.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:04.860 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:04:04.961 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:05.006 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:05.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:05.009 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:04:05.106 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:04 smithi078 bash[75299]: cluster 2023-10-01T11:04:03.720018+0000 osd.6 (osd.6) 292 : cluster 3 failed to encode map e315 with expected crc 2023-10-01T11:04:05.107 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:04 smithi078 bash[75299]: cluster 2023-10-01T11:04:03.723301+0000 osd.7 (osd.7) 339 : cluster 3 failed to encode map e315 with expected crc 2023-10-01T11:04:05.107 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:04 smithi078 bash[75299]: cluster 2023-10-01T11:04:03.808665+0000 osd.5 (osd.5) 325 : cluster 3 failed to encode map e315 with expected crc 2023-10-01T11:04:05.110 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:05.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:04 smithi173 bash[58816]: cluster 2023-10-01T11:04:03.720018+0000 osd.6 (osd.6) 292 : cluster 3 failed to encode map e315 with expected crc 2023-10-01T11:04:05.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:04 smithi173 bash[58816]: cluster 2023-10-01T11:04:03.723301+0000 osd.7 (osd.7) 339 : cluster 3 failed to encode map e315 with expected crc 2023-10-01T11:04:05.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:04 smithi173 bash[58816]: cluster 2023-10-01T11:04:03.808665+0000 osd.5 (osd.5) 325 : cluster 3 failed to encode map e315 with expected crc 2023-10-01T11:04:05.133 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:05.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:05.135 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:04:05.236 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:05.259 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:05.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:05.262 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:04:05.363 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:05.389 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:05.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:05.391 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:05.444 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 118 16 79555 79539 42.1238 0 - 0.0175697 2023-10-01T11:04:05.492 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:05.505 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:05 smithi078 bash[78274]: cluster 2023-10-01T11:04:03.720018+0000 osd.6 (osd.6) 292 : cluster 3 failed to encode map e315 with expected crc 2023-10-01T11:04:05.506 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:05 smithi078 bash[78274]: cluster 2023-10-01T11:04:03.723301+0000 osd.7 (osd.7) 339 : cluster 3 failed to encode map e315 with expected crc 2023-10-01T11:04:05.506 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:05 smithi078 bash[78274]: cluster 2023-10-01T11:04:03.808665+0000 osd.5 (osd.5) 325 : cluster 3 failed to encode map e315 with expected crc 2023-10-01T11:04:05.520 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:05.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:05.523 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:04:05.623 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:05.652 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:05.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:05.656 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:04:05.759 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:05.787 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:05.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:05.789 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:04:05.890 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:05.912 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:05.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:05.914 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:04:06.015 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:06.040 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:06.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:06.042 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:04:06.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:05 smithi173 bash[58816]: cluster 2023-10-01T11:04:04.247391+0000 mgr.y (mgr.25072) 1401 : cluster 0 pgmap v1480: 89 pgs: 1 active+recovering+undersized+degraded+remapped+laggy, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 20580/238692 objects degraded (8.622%); 9539/238692 objects misplaced (3.996%); 1.1 MiB/s, 15 objects/s recovering 2023-10-01T11:04:06.143 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:06.155 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:05 smithi078 bash[78274]: cluster 2023-10-01T11:04:04.247391+0000 mgr.y (mgr.25072) 1401 : cluster 0 pgmap v1480: 89 pgs: 1 active+recovering+undersized+degraded+remapped+laggy, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 20580/238692 objects degraded (8.622%); 9539/238692 objects misplaced (3.996%); 1.1 MiB/s, 15 objects/s recovering 2023-10-01T11:04:06.155 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:05 smithi078 bash[75299]: cluster 2023-10-01T11:04:04.247391+0000 mgr.y (mgr.25072) 1401 : cluster 0 pgmap v1480: 89 pgs: 1 active+recovering+undersized+degraded+remapped+laggy, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 20580/238692 objects degraded (8.622%); 9539/238692 objects misplaced (3.996%); 1.1 MiB/s, 15 objects/s recovering 2023-10-01T11:04:06.166 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:06.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:06.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:04:06.270 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:06.300 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:06.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:06.303 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:04:06.404 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:06.424 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:06.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:06.426 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:04:06.444 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 119 16 79555 79539 41.7698 0 - 0.0175697 2023-10-01T11:04:06.528 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:06.556 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:06.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:06.559 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:04:06.659 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:06.692 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:06.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:06.694 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:04:06.795 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:06.821 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:06.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:06.823 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:04:06.924 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:06.944 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:06.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:06.946 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:04:07.047 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:07.071 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:07.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:07.073 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:04:07.174 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:07.195 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:07.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:07.197 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:04:07.298 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:07.323 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:07.325 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:07 smithi078 bash[75299]: audit 2023-10-01T11:04:06.173253+0000 mon.a (mon.0) 1783 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:04:07.325 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:07 smithi078 bash[75299]: audit 2023-10-01T11:04:06.178528+0000 mon.a (mon.0) 1784 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:04:07.325 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:07 smithi078 bash[75299]: cluster 2023-10-01T11:04:06.247904+0000 mgr.y (mgr.25072) 1402 : cluster 0 pgmap v1481: 89 pgs: 1 active+recovering+undersized+degraded+remapped+laggy, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 20580/238692 objects degraded (8.622%); 9539/238692 objects misplaced (3.996%); 468 KiB/s, 6 objects/s recovering 2023-10-01T11:04:07.325 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:07 smithi078 bash[78274]: audit 2023-10-01T11:04:06.173253+0000 mon.a (mon.0) 1783 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:04:07.325 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:07 smithi078 bash[78274]: audit 2023-10-01T11:04:06.178528+0000 mon.a (mon.0) 1784 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:04:07.325 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:07 smithi078 bash[78274]: cluster 2023-10-01T11:04:06.247904+0000 mgr.y (mgr.25072) 1402 : cluster 0 pgmap v1481: 89 pgs: 1 active+recovering+undersized+degraded+remapped+laggy, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 20580/238692 objects degraded (8.622%); 9539/238692 objects misplaced (3.996%); 468 KiB/s, 6 objects/s recovering 2023-10-01T11:04:07.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:07.326 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:04:07.427 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:07.444 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:2023-10-01T11:04:07.444118+0000 min lat: 0.00136129 max lat: 10.8051 avg lat: 0.0175697 2023-10-01T11:04:07.444 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T11:04:07.444 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 120 16 79555 79539 41.4217 0 - 0.0175697 2023-10-01T11:04:07.455 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:07.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:07.458 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:04:07.559 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:07.571 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:07 smithi173 bash[58816]: audit 2023-10-01T11:04:06.173253+0000 mon.a (mon.0) 1783 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:04:07.571 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:07 smithi173 bash[58816]: audit 2023-10-01T11:04:06.178528+0000 mon.a (mon.0) 1784 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:04:07.572 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:07 smithi173 bash[58816]: cluster 2023-10-01T11:04:06.247904+0000 mgr.y (mgr.25072) 1402 : cluster 0 pgmap v1481: 89 pgs: 1 active+recovering+undersized+degraded+remapped+laggy, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 20580/238692 objects degraded (8.622%); 9539/238692 objects misplaced (3.996%); 468 KiB/s, 6 objects/s recovering 2023-10-01T11:04:07.591 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:07.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:07.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:04:07.694 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:07.723 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:07.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:07.725 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:07.826 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:07.850 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:07.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:07.854 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:04:07.955 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:07.986 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:07.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:07.989 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:04:08.089 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:08.114 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:08.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:08.117 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:04:08.217 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:08.238 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:08.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:08.240 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:04:08.341 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:08.354 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:08 smithi173 bash[58816]: cluster 2023-10-01T11:04:07.708425+0000 mon.a (mon.0) 1785 : cluster 3 Health check update: Degraded data redundancy: 20580/238692 objects degraded (8.622%), 5 pgs degraded (PG_DEGRADED) 2023-10-01T11:04:08.370 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:08.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:08.372 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:04:08.444 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 121 16 79555 79539 41.0794 0 - 0.0175697 2023-10-01T11:04:08.474 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:08.502 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:08.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:08.505 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:04:08.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:08 smithi078 bash[75299]: cluster 2023-10-01T11:04:07.708425+0000 mon.a (mon.0) 1785 : cluster 3 Health check update: Degraded data redundancy: 20580/238692 objects degraded (8.622%), 5 pgs degraded (PG_DEGRADED) 2023-10-01T11:04:08.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:08 smithi078 bash[78274]: cluster 2023-10-01T11:04:07.708425+0000 mon.a (mon.0) 1785 : cluster 3 Health check update: Degraded data redundancy: 20580/238692 objects degraded (8.622%), 5 pgs degraded (PG_DEGRADED) 2023-10-01T11:04:08.606 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:08.635 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:08.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:08.638 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:04:08.739 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:08.765 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:08.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:08.767 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:08.868 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:08.896 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:08.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:08.899 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:09.000 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:09.024 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:09.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:09.026 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:04:09.127 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:09.151 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:09.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:09.153 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:04:09.253 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:09.280 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:09.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:09.283 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:04:09.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:09 smithi078 bash[75299]: cluster 2023-10-01T11:04:08.248639+0000 mgr.y (mgr.25072) 1403 : cluster 0 pgmap v1482: 89 pgs: 1 active+recovering+undersized+degraded+remapped+laggy, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 20580/238692 objects degraded (8.622%); 9176/238692 objects misplaced (3.844%); 4.0 MiB/s, 62 objects/s recovering 2023-10-01T11:04:09.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:09 smithi078 bash[78274]: cluster 2023-10-01T11:04:08.248639+0000 mgr.y (mgr.25072) 1403 : cluster 0 pgmap v1482: 89 pgs: 1 active+recovering+undersized+degraded+remapped+laggy, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 20580/238692 objects degraded (8.622%); 9176/238692 objects misplaced (3.844%); 4.0 MiB/s, 62 objects/s recovering 2023-10-01T11:04:09.383 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:09.409 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:09.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:09.413 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:04:09.444 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 122 16 79555 79539 40.7427 0 - 0.0175697 2023-10-01T11:04:09.514 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:09.526 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:09 smithi173 bash[58816]: cluster 2023-10-01T11:04:08.248639+0000 mgr.y (mgr.25072) 1403 : cluster 0 pgmap v1482: 89 pgs: 1 active+recovering+undersized+degraded+remapped+laggy, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 20580/238692 objects degraded (8.622%); 9176/238692 objects misplaced (3.844%); 4.0 MiB/s, 62 objects/s recovering 2023-10-01T11:04:09.538 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:09.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:09.541 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:04:09.642 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:09.664 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:09.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:09.666 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:04:09.767 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:09.795 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:09.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:09.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:09.899 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:09.927 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:09.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:09.929 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:04:10.030 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:10.054 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:10.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:10.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:04:10.158 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:10.186 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:10.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:10.189 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:04:10.290 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:10.319 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:10.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:10.322 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:04:10.423 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:10.434 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:04:10 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:04:10] "GET /metrics HTTP/1.1" 200 36413 "" "Prometheus/2.33.4" 2023-10-01T11:04:10.444 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 123 16 79555 79539 40.4115 0 - 0.0175697 2023-10-01T11:04:10.455 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:10.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:10.459 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:04:10.559 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:10.590 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:10.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:10.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:04:10.693 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:10.719 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:10.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:10.721 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:04:10.822 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:10.853 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:10.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:10.856 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:10.957 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:10.989 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:10.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:10.991 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:11.092 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:11.119 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:11.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:11.122 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:04:11.223 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:11.248 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:11.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:11.251 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:04:11.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:11 smithi078 bash[78274]: cluster 2023-10-01T11:04:10.249883+0000 mgr.y (mgr.25072) 1404 : cluster 0 pgmap v1483: 89 pgs: 1 active+recovering+undersized+degraded+remapped+laggy, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 20580/238692 objects degraded (8.622%); 9176/238692 objects misplaced (3.844%); 3.2 MiB/s, 50 objects/s recovering 2023-10-01T11:04:11.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:11 smithi078 bash[75299]: cluster 2023-10-01T11:04:10.249883+0000 mgr.y (mgr.25072) 1404 : cluster 0 pgmap v1483: 89 pgs: 1 active+recovering+undersized+degraded+remapped+laggy, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 20580/238692 objects degraded (8.622%); 9176/238692 objects misplaced (3.844%); 3.2 MiB/s, 50 objects/s recovering 2023-10-01T11:04:11.352 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:11.378 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:11.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:11.382 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:04:11.444 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 124 16 79555 79539 40.0856 0 - 0.0175697 2023-10-01T11:04:11.483 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:11.508 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:11.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:11.511 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:04:11.611 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:11.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:11 smithi173 bash[58816]: cluster 2023-10-01T11:04:10.249883+0000 mgr.y (mgr.25072) 1404 : cluster 0 pgmap v1483: 89 pgs: 1 active+recovering+undersized+degraded+remapped+laggy, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 20580/238692 objects degraded (8.622%); 9176/238692 objects misplaced (3.844%); 3.2 MiB/s, 50 objects/s recovering 2023-10-01T11:04:11.638 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:11.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:11.664 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:04:11.765 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:11.791 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:11.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:11.794 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:04:11.895 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:11.920 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:11.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:11.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:04:12.024 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:12.050 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:12.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:12.054 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:04:12.154 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:12.181 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:12.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:12.184 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:04:12.241 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5] out_osds: [6, 7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T11:04:12.241 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T11:04:12.241 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-01T11:04:12.241 INFO:tasks.thrashosds.thrasher:Setting osd 0 primary_affinity to 0.000000 2023-10-01T11:04:12.241 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 0 0 2023-10-01T11:04:12.285 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:12.316 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:12.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:12.318 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:04:12.419 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:12.444 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 125 16 79555 79539 39.7649 0 - 0.0175697 2023-10-01T11:04:12.448 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:12.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:12.451 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:04:12.552 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:12.576 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:12.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:12.579 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:12.680 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:12.704 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:12.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:12.706 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:04:12.807 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:12.835 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:12.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:12.838 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:12.938 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:12.966 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:12.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:12.969 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:04:13.069 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:13.097 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:13.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:13.100 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:13.200 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:13.224 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:13.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:13.226 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:04:13.326 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:13.354 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:13.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:13.356 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:13.445 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 126 16 79555 79539 39.4493 0 - 0.0175697 2023-10-01T11:04:13.457 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:13.479 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:13.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:13.481 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:04:13.582 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:13.593 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:13 smithi173 bash[58816]: cluster 2023-10-01T11:04:12.250441+0000 mgr.y (mgr.25072) 1405 : cluster 0 pgmap v1484: 89 pgs: 1 active+recovering+undersized+degraded+remapped+laggy, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 20580/238692 objects degraded (8.622%); 9176/238692 objects misplaced (3.844%); 2.4 MiB/s, 37 objects/s recovering 2023-10-01T11:04:13.593 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:13 smithi173 bash[58816]: audit 2023-10-01T11:04:12.592406+0000 mon.a (mon.0) 1786 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:04:13.593 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:13 smithi173 bash[58816]: audit 2023-10-01T11:04:12.600575+0000 mon.a (mon.0) 1787 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:04:13.609 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:13.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:13.611 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:04:13.712 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:13.721 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:13 smithi078 bash[78274]: cluster 2023-10-01T11:04:12.250441+0000 mgr.y (mgr.25072) 1405 : cluster 0 pgmap v1484: 89 pgs: 1 active+recovering+undersized+degraded+remapped+laggy, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 20580/238692 objects degraded (8.622%); 9176/238692 objects misplaced (3.844%); 2.4 MiB/s, 37 objects/s recovering 2023-10-01T11:04:13.721 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:13 smithi078 bash[78274]: audit 2023-10-01T11:04:12.592406+0000 mon.a (mon.0) 1786 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:04:13.721 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:13 smithi078 bash[78274]: audit 2023-10-01T11:04:12.600575+0000 mon.a (mon.0) 1787 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:04:13.721 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:13 smithi078 bash[78274]: cluster 2023-10-01T11:04:12.710045+0000 mon.a (mon.0) 1788 : cluster 3 Health check update: Degraded data redundancy: 20580/238692 objects degraded (8.622%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T11:04:13.721 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:13 smithi078 bash[78274]: audit 2023-10-01T11:04:13.072941+0000 mon.a (mon.0) 1789 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T11:04:13.721 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:13 smithi078 bash[78274]: audit 2023-10-01T11:04:13.074173+0000 mon.a (mon.0) 1790 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T11:04:13.722 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:13 smithi078 bash[78274]: audit 2023-10-01T11:04:13.083378+0000 mon.a (mon.0) 1791 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:04:13.722 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:13 smithi078 bash[75299]: cluster 2023-10-01T11:04:12.250441+0000 mgr.y (mgr.25072) 1405 : cluster 0 pgmap v1484: 89 pgs: 1 active+recovering+undersized+degraded+remapped+laggy, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 20580/238692 objects degraded (8.622%); 9176/238692 objects misplaced (3.844%); 2.4 MiB/s, 37 objects/s recovering 2023-10-01T11:04:13.722 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:13 smithi078 bash[75299]: audit 2023-10-01T11:04:12.592406+0000 mon.a (mon.0) 1786 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:04:13.722 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:13 smithi078 bash[75299]: audit 2023-10-01T11:04:12.600575+0000 mon.a (mon.0) 1787 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:04:13.722 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:13 smithi078 bash[75299]: cluster 2023-10-01T11:04:12.710045+0000 mon.a (mon.0) 1788 : cluster 3 Health check update: Degraded data redundancy: 20580/238692 objects degraded (8.622%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T11:04:13.722 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:13 smithi078 bash[75299]: audit 2023-10-01T11:04:13.072941+0000 mon.a (mon.0) 1789 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T11:04:13.722 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:13 smithi078 bash[75299]: audit 2023-10-01T11:04:13.074173+0000 mon.a (mon.0) 1790 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T11:04:13.722 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:13 smithi078 bash[75299]: audit 2023-10-01T11:04:13.083378+0000 mon.a (mon.0) 1791 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:04:13.734 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:13.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:13.736 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:04:13.836 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:13.860 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:13.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:13.863 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:04:13.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:13 smithi173 bash[58816]: cluster 2023-10-01T11:04:12.710045+0000 mon.a (mon.0) 1788 : cluster 3 Health check update: Degraded data redundancy: 20580/238692 objects degraded (8.622%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T11:04:13.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:13 smithi173 bash[58816]: audit 2023-10-01T11:04:13.072941+0000 mon.a (mon.0) 1789 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T11:04:13.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:13 smithi173 bash[58816]: audit 2023-10-01T11:04:13.074173+0000 mon.a (mon.0) 1790 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T11:04:13.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:13 smithi173 bash[58816]: audit 2023-10-01T11:04:13.083378+0000 mon.a (mon.0) 1791 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:04:13.964 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:13.985 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T11:04:13.992 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:13.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:13.994 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:04:14.010 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T11:04:14.041 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T11:04:14.070 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T11:04:14.080 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:04:14.094 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:14.109 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T11:04:14.124 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:14.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:14.127 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:14.131 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T11:04:14.160 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T11:04:14.228 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:14.255 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:14.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:14.261 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:04:14.362 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:14.391 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:14.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:14.393 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:04:14.445 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 127 16 79555 79539 39.1387 0 - 0.0175697 2023-10-01T11:04:14.494 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:14.524 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:14.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:14.526 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:04:14.608 INFO:teuthology.orchestra.run.smithi078.stderr:noscrub is unset 2023-10-01T11:04:14.627 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:14.650 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:14.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:14.652 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:04:14.753 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:14.783 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:14.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:14.785 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:04:14.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:14 smithi173 bash[58816]: audit 2023-10-01T11:04:13.927662+0000 mon.a (mon.0) 1792 : audit 1 from='client.? 172.21.15.78:0/2834248278' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T11:04:14.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:14 smithi173 bash[58816]: audit 2023-10-01T11:04:13.956765+0000 mon.c (mon.2) 212 : audit 0 from='client.? 172.21.15.78:0/3856160795' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:04:14.886 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:14.898 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:14 smithi078 bash[78274]: audit 2023-10-01T11:04:13.927662+0000 mon.a (mon.0) 1792 : audit 1 from='client.? 172.21.15.78:0/2834248278' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T11:04:14.899 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:14 smithi078 bash[78274]: audit 2023-10-01T11:04:13.956765+0000 mon.c (mon.2) 212 : audit 0 from='client.? 172.21.15.78:0/3856160795' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:04:14.899 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:14 smithi078 bash[75299]: audit 2023-10-01T11:04:13.927662+0000 mon.a (mon.0) 1792 : audit 1 from='client.? 172.21.15.78:0/2834248278' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T11:04:14.899 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:14 smithi078 bash[75299]: audit 2023-10-01T11:04:13.956765+0000 mon.c (mon.2) 212 : audit 0 from='client.? 172.21.15.78:0/3856160795' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:04:14.913 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:14.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:14.916 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:04:15.017 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:15.041 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:15.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:15.043 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:04:15.144 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:15.174 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:15.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:15.177 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:04:15.277 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:15.300 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:15.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:15.303 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:04:15.403 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:15.433 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:15.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:15.435 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:04:15.445 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 128 16 79555 79539 38.8329 0 - 0.0175697 2023-10-01T11:04:15.536 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:15.563 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:15.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:15.566 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:04:15.667 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:15.681 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:15 smithi173 bash[58816]: cluster 2023-10-01T11:04:14.251339+0000 mgr.y (mgr.25072) 1406 : cluster 0 pgmap v1485: 89 pgs: 1 active+recovering+undersized+degraded+remapped+laggy, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 20580/238692 objects degraded (8.622%); 8808/238692 objects misplaced (3.690%); 4.3 MiB/s, 69 objects/s recovering 2023-10-01T11:04:15.681 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:15 smithi173 bash[58816]: cluster 2023-10-01T11:04:14.591222+0000 mon.a (mon.0) 1793 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:04:15.681 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:15 smithi173 bash[58816]: cluster 2023-10-01T11:04:14.598431+0000 osd.5 (osd.5) 326 : cluster 3 failed to encode map e316 with expected crc 2023-10-01T11:04:15.681 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:15 smithi173 bash[58816]: audit 2023-10-01T11:04:14.603477+0000 mon.a (mon.0) 1794 : audit 1 from='client.? 172.21.15.78:0/2834248278' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T11:04:15.681 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:15 smithi173 bash[58816]: cluster 2023-10-01T11:04:14.603537+0000 mon.a (mon.0) 1795 : cluster 0 osdmap e316: 8 total, 8 up, 6 in 2023-10-01T11:04:15.698 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:15.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:15.700 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:04:15.801 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:15.828 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:15.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:15.831 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:04:15.932 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:15.942 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:15 smithi078 bash[75299]: cluster 2023-10-01T11:04:14.251339+0000 mgr.y (mgr.25072) 1406 : cluster 0 pgmap v1485: 89 pgs: 1 active+recovering+undersized+degraded+remapped+laggy, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 20580/238692 objects degraded (8.622%); 8808/238692 objects misplaced (3.690%); 4.3 MiB/s, 69 objects/s recovering 2023-10-01T11:04:15.942 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:15 smithi078 bash[75299]: cluster 2023-10-01T11:04:14.591222+0000 mon.a (mon.0) 1793 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:04:15.942 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:15 smithi078 bash[75299]: cluster 2023-10-01T11:04:14.598431+0000 osd.5 (osd.5) 326 : cluster 3 failed to encode map e316 with expected crc 2023-10-01T11:04:15.942 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:15 smithi078 bash[75299]: audit 2023-10-01T11:04:14.603477+0000 mon.a (mon.0) 1794 : audit 1 from='client.? 172.21.15.78:0/2834248278' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T11:04:15.942 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:15 smithi078 bash[75299]: cluster 2023-10-01T11:04:14.603537+0000 mon.a (mon.0) 1795 : cluster 0 osdmap e316: 8 total, 8 up, 6 in 2023-10-01T11:04:15.942 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:15 smithi078 bash[78274]: cluster 2023-10-01T11:04:14.251339+0000 mgr.y (mgr.25072) 1406 : cluster 0 pgmap v1485: 89 pgs: 1 active+recovering+undersized+degraded+remapped+laggy, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 20580/238692 objects degraded (8.622%); 8808/238692 objects misplaced (3.690%); 4.3 MiB/s, 69 objects/s recovering 2023-10-01T11:04:15.942 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:15 smithi078 bash[78274]: cluster 2023-10-01T11:04:14.591222+0000 mon.a (mon.0) 1793 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:04:15.943 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:15 smithi078 bash[78274]: cluster 2023-10-01T11:04:14.598431+0000 osd.5 (osd.5) 326 : cluster 3 failed to encode map e316 with expected crc 2023-10-01T11:04:15.943 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:15 smithi078 bash[78274]: audit 2023-10-01T11:04:14.603477+0000 mon.a (mon.0) 1794 : audit 1 from='client.? 172.21.15.78:0/2834248278' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T11:04:15.943 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:15 smithi078 bash[78274]: cluster 2023-10-01T11:04:14.603537+0000 mon.a (mon.0) 1795 : cluster 0 osdmap e316: 8 total, 8 up, 6 in 2023-10-01T11:04:15.955 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:15.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:15.957 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:04:16.058 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:16.087 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:16.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:16.089 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:16.190 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:16.220 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:16.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:16.226 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:16.327 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:16.354 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:16.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:16.357 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:04:16.445 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 129 16 79555 79539 38.5319 0 - 0.0175697 2023-10-01T11:04:16.458 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:16.489 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:16.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:16.491 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:04:16.592 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:16.623 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:16.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:16.625 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:04:16.726 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:16.746 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:16.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:16.748 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:04:16.849 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:16.869 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:16.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:16 smithi173 bash[58816]: cluster 2023-10-01T11:04:15.907819+0000 osd.7 (osd.7) 340 : cluster 3 failed to encode map e316 with expected crc 2023-10-01T11:04:16.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:16 smithi173 bash[58816]: cluster 2023-10-01T11:04:15.907849+0000 osd.6 (osd.6) 293 : cluster 3 failed to encode map e316 with expected crc 2023-10-01T11:04:16.870 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:16 smithi078 bash[78274]: cluster 2023-10-01T11:04:15.907819+0000 osd.7 (osd.7) 340 : cluster 3 failed to encode map e316 with expected crc 2023-10-01T11:04:16.870 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:16 smithi078 bash[78274]: cluster 2023-10-01T11:04:15.907849+0000 osd.6 (osd.6) 293 : cluster 3 failed to encode map e316 with expected crc 2023-10-01T11:04:16.871 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:16 smithi078 bash[75299]: cluster 2023-10-01T11:04:15.907819+0000 osd.7 (osd.7) 340 : cluster 3 failed to encode map e316 with expected crc 2023-10-01T11:04:16.871 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:16 smithi078 bash[75299]: cluster 2023-10-01T11:04:15.907849+0000 osd.6 (osd.6) 293 : cluster 3 failed to encode map e316 with expected crc 2023-10-01T11:04:16.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:16.872 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:04:16.972 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:17.004 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:17.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:17.006 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:04:17.107 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:17.129 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:17.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:17.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:04:17.232 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:17.251 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:17.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:17.253 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:04:17.354 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:17.384 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:17.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:17.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:04:17.445 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 130 16 79555 79539 38.2355 0 - 0.0175697 2023-10-01T11:04:17.487 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:17.517 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:17.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:17.520 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:04:17.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:17 smithi173 bash[58816]: cluster 2023-10-01T11:04:16.251853+0000 mgr.y (mgr.25072) 1407 : cluster 0 pgmap v1487: 89 pgs: 1 active+recovering+undersized+degraded+remapped+laggy, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 20580/238692 objects degraded (8.622%); 8808/238692 objects misplaced (3.690%); 4.6 MiB/s, 73 objects/s recovering 2023-10-01T11:04:17.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:17 smithi173 bash[58816]: audit 2023-10-01T11:04:17.458014+0000 mon.a (mon.0) 1796 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:04:17.621 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:17.647 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:17.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:17.651 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:04:17.751 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:17.777 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:17.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:17.779 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:04:17.880 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:17.888 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:17 smithi078 bash[78274]: cluster 2023-10-01T11:04:16.251853+0000 mgr.y (mgr.25072) 1407 : cluster 0 pgmap v1487: 89 pgs: 1 active+recovering+undersized+degraded+remapped+laggy, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 20580/238692 objects degraded (8.622%); 8808/238692 objects misplaced (3.690%); 4.6 MiB/s, 73 objects/s recovering 2023-10-01T11:04:17.888 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:17 smithi078 bash[78274]: audit 2023-10-01T11:04:17.458014+0000 mon.a (mon.0) 1796 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:04:17.888 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:17 smithi078 bash[75299]: cluster 2023-10-01T11:04:16.251853+0000 mgr.y (mgr.25072) 1407 : cluster 0 pgmap v1487: 89 pgs: 1 active+recovering+undersized+degraded+remapped+laggy, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 20580/238692 objects degraded (8.622%); 8808/238692 objects misplaced (3.690%); 4.6 MiB/s, 73 objects/s recovering 2023-10-01T11:04:17.888 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:17 smithi078 bash[75299]: audit 2023-10-01T11:04:17.458014+0000 mon.a (mon.0) 1796 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:04:17.900 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:17.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:17.903 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:04:18.004 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:18.030 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:18.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:18.033 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:04:18.134 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:18.161 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:18.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:18.164 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:04:18.265 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:18.293 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:18.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:18.295 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:04:18.396 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:18.420 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:18.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:18.423 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:04:18.445 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 131 16 79555 79539 37.9436 0 - 0.0175697 2023-10-01T11:04:18.490 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:04:18.523 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:18.547 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:18.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:18.549 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:04:18.650 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:18.675 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:18.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:18.678 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:04:18.779 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:18.799 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:18.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:18.802 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:04:18.862 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T11:04:18.864 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T11:04:18.864 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T11:04:18.864 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T11:04:18.864 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T11:04:18.865 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T11:04:18.865 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T11:04:18.865 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T11:04:18.865 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T11:04:18.903 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:18.927 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:18.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:18.930 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:04:19.030 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:19.054 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:19.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:19.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:04:19.158 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:19.182 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:19.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:19.184 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:04:19.285 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:19.310 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:19.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:19.313 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:04:19.414 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:19.436 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:19.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:19.438 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:04:19.445 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 132 13 79555 79542 37.6576 0.00446429 45.4157 0.0192695 2023-10-01T11:04:19.538 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:19.559 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:19.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:19.561 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:04:19.662 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:19.690 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:19.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:19.693 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:04:19.793 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:19.825 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:19.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:19.827 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:04:19.928 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:19.954 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:19.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:19.957 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:04:20.007 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T11:04:20.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:19 smithi078 bash[78274]: cluster 2023-10-01T11:04:18.252894+0000 mgr.y (mgr.25072) 1408 : cluster 0 pgmap v1488: 89 pgs: 1 active+recovering+undersized+degraded+remapped+laggy, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 20580/238692 objects degraded (8.622%); 8406/238692 objects misplaced (3.522%); 4.8 MiB/s, 76 objects/s recovering 2023-10-01T11:04:20.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:19 smithi078 bash[75299]: cluster 2023-10-01T11:04:18.252894+0000 mgr.y (mgr.25072) 1408 : cluster 0 pgmap v1488: 89 pgs: 1 active+recovering+undersized+degraded+remapped+laggy, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 20580/238692 objects degraded (8.622%); 8406/238692 objects misplaced (3.522%); 4.8 MiB/s, 76 objects/s recovering 2023-10-01T11:04:20.058 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:20.085 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:20.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:20.091 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:04:20.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:19 smithi173 bash[58816]: cluster 2023-10-01T11:04:18.252894+0000 mgr.y (mgr.25072) 1408 : cluster 0 pgmap v1488: 89 pgs: 1 active+recovering+undersized+degraded+remapped+laggy, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 20580/238692 objects degraded (8.622%); 8406/238692 objects misplaced (3.522%); 4.8 MiB/s, 76 objects/s recovering 2023-10-01T11:04:20.191 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:20.216 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:20.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:20.220 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:04:20.321 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:20.342 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:20.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:20.345 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:04:20.445 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Total time run: 132.808 2023-10-01T11:04:20.446 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Total writes made: 79555 2023-10-01T11:04:20.446 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Write size: 65536 2023-10-01T11:04:20.446 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Object size: 65536 2023-10-01T11:04:20.446 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Bandwidth (MB/sec): 37.439 2023-10-01T11:04:20.446 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Stddev Bandwidth: 45.2143 2023-10-01T11:04:20.446 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Max bandwidth (MB/sec): 190.438 2023-10-01T11:04:20.446 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Min bandwidth (MB/sec): 0 2023-10-01T11:04:20.446 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Average IOPS: 599 2023-10-01T11:04:20.446 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Stddev IOPS: 723.432 2023-10-01T11:04:20.446 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Max IOPS: 3047 2023-10-01T11:04:20.447 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Min IOPS: 0 2023-10-01T11:04:20.447 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Average Latency(s): 0.0265974 2023-10-01T11:04:20.447 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Stddev Latency(s): 0.649707 2023-10-01T11:04:20.447 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Max latency(s): 45.5142 2023-10-01T11:04:20.447 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Min latency(s): 0.00136129 2023-10-01T11:04:20.447 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:20.452 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Cleaning up (deleting benchmark objects) 2023-10-01T11:04:20.479 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:20.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:20.484 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:04:20.580 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:04:20 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:04:20] "GET /metrics HTTP/1.1" 200 36401 "" "Prometheus/2.33.4" 2023-10-01T11:04:20.585 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:20.611 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:20.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:20.618 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:04:20.719 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:20.743 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:20.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:20.746 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:04:20.846 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:20.856 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:20 smithi173 bash[58816]: cluster 2023-10-01T11:04:20.715749+0000 mon.a (mon.0) 1797 : cluster 3 Health check update: Degraded data redundancy: 23789/238737 objects degraded (9.965%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T11:04:20.870 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:20.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:20.872 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:04:20.973 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:21.004 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:21.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:21.007 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:04:21.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:20 smithi078 bash[75299]: cluster 2023-10-01T11:04:20.715749+0000 mon.a (mon.0) 1797 : cluster 3 Health check update: Degraded data redundancy: 23789/238737 objects degraded (9.965%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T11:04:21.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:20 smithi078 bash[78274]: cluster 2023-10-01T11:04:20.715749+0000 mon.a (mon.0) 1797 : cluster 3 Health check update: Degraded data redundancy: 23789/238737 objects degraded (9.965%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T11:04:21.107 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:21.130 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:21.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:21.133 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:04:21.234 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:21.258 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:21.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:21.260 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:04:21.361 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:21.387 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:21.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:21.390 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:04:21.491 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:21.522 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:21.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:21.524 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:04:21.625 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:21.649 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:21.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:21.651 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:04:21.752 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:21.772 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:21.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:21.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:04:21.875 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:21.901 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:21.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:21.903 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:04:22.004 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:22.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:21 smithi078 bash[78274]: cluster 2023-10-01T11:04:20.253766+0000 mgr.y (mgr.25072) 1409 : cluster 0 pgmap v1489: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 96 KiB/s wr, 1 op/s; 23789/238737 objects degraded (9.965%); 5216/238737 objects misplaced (2.185%); 4.9 MiB/s, 78 objects/s recovering 2023-10-01T11:04:22.015 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:21 smithi078 bash[75299]: cluster 2023-10-01T11:04:20.253766+0000 mgr.y (mgr.25072) 1409 : cluster 0 pgmap v1489: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 96 KiB/s wr, 1 op/s; 23789/238737 objects degraded (9.965%); 5216/238737 objects misplaced (2.185%); 4.9 MiB/s, 78 objects/s recovering 2023-10-01T11:04:22.027 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:22.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:22.030 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:04:22.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:21 smithi173 bash[58816]: cluster 2023-10-01T11:04:20.253766+0000 mgr.y (mgr.25072) 1409 : cluster 0 pgmap v1489: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 96 KiB/s wr, 1 op/s; 23789/238737 objects degraded (9.965%); 5216/238737 objects misplaced (2.185%); 4.9 MiB/s, 78 objects/s recovering 2023-10-01T11:04:22.131 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:22.160 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:22.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:22.163 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:04:22.264 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:22.292 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:22.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:22.294 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:04:22.396 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:22.423 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:22.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:22.427 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:22.528 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:22.548 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:22.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:22.550 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:04:22.651 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:22.678 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:22.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:22.681 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:04:22.781 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:22.812 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:22.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:22.814 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:04:22.915 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:22.939 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:22.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:22.942 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:04:23.043 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:23.067 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:23.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:23.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:04:23.170 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:23.191 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:23.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:23.193 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:04:23.295 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:23.321 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:23.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:23.323 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:04:23.424 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:23.450 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:23.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:23.452 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:04:23.553 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:23.578 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:23.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:23.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:04:23.681 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:23.704 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:23.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:23.706 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:04:23.807 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:23.817 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:23 smithi173 bash[58816]: cluster 2023-10-01T11:04:22.254193+0000 mgr.y (mgr.25072) 1410 : cluster 0 pgmap v1490: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 96 KiB/s wr, 1 op/s; 23789/238737 objects degraded (9.965%); 5216/238737 objects misplaced (2.185%); 4.9 MiB/s, 78 objects/s recovering 2023-10-01T11:04:23.833 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:23.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:23.835 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:04:23.936 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:23.964 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:23.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:23.966 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:04:24.003 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:23 smithi078 bash[75299]: cluster 2023-10-01T11:04:22.254193+0000 mgr.y (mgr.25072) 1410 : cluster 0 pgmap v1490: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 96 KiB/s wr, 1 op/s; 23789/238737 objects degraded (9.965%); 5216/238737 objects misplaced (2.185%); 4.9 MiB/s, 78 objects/s recovering 2023-10-01T11:04:24.003 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:23 smithi078 bash[78274]: cluster 2023-10-01T11:04:22.254193+0000 mgr.y (mgr.25072) 1410 : cluster 0 pgmap v1490: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.9 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 96 KiB/s wr, 1 op/s; 23789/238737 objects degraded (9.965%); 5216/238737 objects misplaced (2.185%); 4.9 MiB/s, 78 objects/s recovering 2023-10-01T11:04:24.067 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:24.095 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:24.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:24.099 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:04:24.199 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:24.222 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:24.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:24.225 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:04:24.326 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:24.350 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:24.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:24.354 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:04:24.455 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:24.479 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:24.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:24.482 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:04:24.583 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:24.604 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:24.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:24.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:04:24.707 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:24.740 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:24.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:24.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:04:24.843 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:24.867 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:24.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:24.870 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:04:24.971 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:24.994 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:24.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:24.997 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:04:25.097 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:25.122 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:25.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:25.126 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:25.226 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:25.251 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:25.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:25.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:04:25.355 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:25.378 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:25.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:25.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:04:25.483 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:25.507 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:25.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:25.509 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:04:25.610 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:25.633 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:25.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:25.636 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:04:25.737 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:25.763 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:25.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:25.767 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:04:25.868 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:25.889 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:25.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:25.892 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:04:25.993 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:26.024 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:26.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:26.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:04:26.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:25 smithi078 bash[78274]: cluster 2023-10-01T11:04:24.254988+0000 mgr.y (mgr.25072) 1411 : cluster 0 pgmap v1491: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 96 KiB/s wr, 52 op/s; 23763/237210 objects degraded (10.018%); 4790/237210 objects misplaced (2.019%); 5.3 MiB/s, 84 objects/s recovering 2023-10-01T11:04:26.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:25 smithi078 bash[75299]: cluster 2023-10-01T11:04:24.254988+0000 mgr.y (mgr.25072) 1411 : cluster 0 pgmap v1491: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 96 KiB/s wr, 52 op/s; 23763/237210 objects degraded (10.018%); 4790/237210 objects misplaced (2.019%); 5.3 MiB/s, 84 objects/s recovering 2023-10-01T11:04:26.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:25 smithi173 bash[58816]: cluster 2023-10-01T11:04:24.254988+0000 mgr.y (mgr.25072) 1411 : cluster 0 pgmap v1491: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 96 KiB/s wr, 52 op/s; 23763/237210 objects degraded (10.018%); 4790/237210 objects misplaced (2.019%); 5.3 MiB/s, 84 objects/s recovering 2023-10-01T11:04:26.128 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:26.156 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:26.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:26.158 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:04:26.259 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:26.281 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:26.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:26.285 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:04:26.385 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:26.410 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:26.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:26.412 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:26.513 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:26.537 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:26.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:26.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:04:26.640 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:26.669 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:26.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:26.671 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:26.772 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:26.773 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T11:04:26.797 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:26.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:26.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:04:26.900 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:26.911 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:26 smithi173 bash[58816]: cluster 2023-10-01T11:04:26.778582+0000 mon.a (mon.0) 1798 : cluster 3 Health check update: Degraded data redundancy: 23763/236697 objects degraded (10.039%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T11:04:26.926 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:26.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:26.929 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:27.030 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:27.058 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:27.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:27.060 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:04:27.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:26 smithi078 bash[75299]: cluster 2023-10-01T11:04:26.778582+0000 mon.a (mon.0) 1798 : cluster 3 Health check update: Degraded data redundancy: 23763/236697 objects degraded (10.039%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T11:04:27.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:26 smithi078 bash[78274]: cluster 2023-10-01T11:04:26.778582+0000 mon.a (mon.0) 1798 : cluster 3 Health check update: Degraded data redundancy: 23763/236697 objects degraded (10.039%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T11:04:27.161 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:27.190 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:27.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:27.193 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:04:27.294 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:27.317 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:27.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:27.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:27.420 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:27.449 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:27.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:27.452 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:04:27.553 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:27.576 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:27.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:27.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:04:27.680 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:27.703 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:27.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:27.705 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:04:27.806 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:27.807 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:04:27.831 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:27.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:27.834 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:04:27.935 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:27.945 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:27 smithi078 bash[78274]: cluster 2023-10-01T11:04:26.255906+0000 mgr.y (mgr.25072) 1412 : cluster 0 pgmap v1492: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 87 B/s rd, 82 KiB/s wr, 59 op/s; 23763/236697 objects degraded (10.039%); 4790/236697 objects misplaced (2.024%); 4.5 MiB/s, 72 objects/s recovering 2023-10-01T11:04:27.946 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:27 smithi078 bash[75299]: cluster 2023-10-01T11:04:26.255906+0000 mgr.y (mgr.25072) 1412 : cluster 0 pgmap v1492: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 87 B/s rd, 82 KiB/s wr, 59 op/s; 23763/236697 objects degraded (10.039%); 4790/236697 objects misplaced (2.024%); 4.5 MiB/s, 72 objects/s recovering 2023-10-01T11:04:27.959 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:27.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:27.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:04:28.063 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:28.092 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:28.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:28.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:04:28.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:27 smithi173 bash[58816]: cluster 2023-10-01T11:04:26.255906+0000 mgr.y (mgr.25072) 1412 : cluster 0 pgmap v1492: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 87 B/s rd, 82 KiB/s wr, 59 op/s; 23763/236697 objects degraded (10.039%); 4790/236697 objects misplaced (2.024%); 4.5 MiB/s, 72 objects/s recovering 2023-10-01T11:04:28.194 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:28.224 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:28.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:28.227 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:28.328 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:28.350 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:28.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:28.353 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:04:28.454 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:28.481 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:28.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:28.487 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:04:28.587 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:28.613 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:28.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:28.615 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:04:28.716 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:28.745 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:28.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:28.748 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:28.849 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:28.871 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:28.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:28.873 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:04:28.974 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:29.004 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:29.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:29.006 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:04:29.107 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:29.138 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:29.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:29.141 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:29.242 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:29.268 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:29.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:29.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:04:29.371 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:29.400 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:29.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:29.403 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:04:29.504 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:29.524 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:29.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:29.526 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:04:29.627 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:29.648 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:29.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:29.650 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:04:29.750 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:29.774 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:29.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:29.776 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:04:29.877 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:29.903 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:29.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:29.906 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:04:30.006 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:30.029 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:30.030 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:29 smithi078 bash[75299]: cluster 2023-10-01T11:04:28.256820+0000 mgr.y (mgr.25072) 1413 : cluster 0 pgmap v1493: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 85 B/s rd, 80 KiB/s wr, 80 op/s; 23763/235887 objects degraded (10.074%); 4364/235887 objects misplaced (1.850%); 6.6 MiB/s, 105 objects/s recovering 2023-10-01T11:04:30.031 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:29 smithi078 bash[78274]: cluster 2023-10-01T11:04:28.256820+0000 mgr.y (mgr.25072) 1413 : cluster 0 pgmap v1493: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 85 B/s rd, 80 KiB/s wr, 80 op/s; 23763/235887 objects degraded (10.074%); 4364/235887 objects misplaced (1.850%); 6.6 MiB/s, 105 objects/s recovering 2023-10-01T11:04:30.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:30.031 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:04:30.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:29 smithi173 bash[58816]: cluster 2023-10-01T11:04:28.256820+0000 mgr.y (mgr.25072) 1413 : cluster 0 pgmap v1493: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 85 B/s rd, 80 KiB/s wr, 80 op/s; 23763/235887 objects degraded (10.074%); 4364/235887 objects misplaced (1.850%); 6.6 MiB/s, 105 objects/s recovering 2023-10-01T11:04:30.133 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:30.161 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:30.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:30.164 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:04:30.265 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:30.285 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:30.286 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:04:30 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:04:30] "GET /metrics HTTP/1.1" 200 36401 "" "Prometheus/2.33.4" 2023-10-01T11:04:30.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:30.287 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:04:30.388 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:30.418 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:30.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:30.420 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:04:30.521 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:30.548 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:30.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:30.551 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:04:30.651 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:30.675 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:30.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:30.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:04:30.729 INFO:teuthology.orchestra.run.smithi078.stdout:ERROR: (22) Invalid argument 2023-10-01T11:04:30.729 INFO:teuthology.orchestra.run.smithi078.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T11:04:30.779 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:30.799 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:30.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:30.801 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:04:30.902 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:30.928 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:30.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:30.931 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:04:31.032 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:31.051 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:31.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:31.053 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:04:31.154 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:31.181 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:31.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:31.184 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:04:31.285 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:31.312 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:31.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:31.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:04:31.416 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:31.441 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:31.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:31.444 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:31.545 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:31.566 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:31.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:31.569 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:04:31.669 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:31.694 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:31.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:31.699 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:04:31.800 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:31.826 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:31.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:31.829 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:31.930 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:31.938 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:31 smithi078 bash[78274]: cluster 2023-10-01T11:04:30.257606+0000 mgr.y (mgr.25072) 1414 : cluster 0 pgmap v1494: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 85 B/s rd, 80 KiB/s wr, 109 op/s; 23738/234840 objects degraded (10.108%); 4372/234840 objects misplaced (1.862%); 4.6 MiB/s, 73 objects/s recovering 2023-10-01T11:04:31.939 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:31 smithi078 bash[75299]: cluster 2023-10-01T11:04:30.257606+0000 mgr.y (mgr.25072) 1414 : cluster 0 pgmap v1494: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 85 B/s rd, 80 KiB/s wr, 109 op/s; 23738/234840 objects degraded (10.108%); 4372/234840 objects misplaced (1.862%); 4.6 MiB/s, 73 objects/s recovering 2023-10-01T11:04:31.951 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:31.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:31.954 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:04:32.055 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:32.066 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:31 smithi173 bash[58816]: cluster 2023-10-01T11:04:30.257606+0000 mgr.y (mgr.25072) 1414 : cluster 0 pgmap v1494: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 85 B/s rd, 80 KiB/s wr, 109 op/s; 23738/234840 objects degraded (10.108%); 4372/234840 objects misplaced (1.862%); 4.6 MiB/s, 73 objects/s recovering 2023-10-01T11:04:32.083 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:32.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:32.086 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:04:32.187 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:32.219 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:32.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:32.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:04:32.324 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:32.347 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:32.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:32.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:04:32.452 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:32.475 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:32.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:32.479 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:04:32.579 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:32.605 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:32.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:32.608 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:04:32.710 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:32.742 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:32.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:32.746 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:04:32.847 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:32.867 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:32.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:32.869 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:04:32.971 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:32.987 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:32 smithi173 bash[58816]: audit 2023-10-01T11:04:32.457816+0000 mon.a (mon.0) 1799 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:04:32.988 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:32 smithi173 bash[58816]: cluster 2023-10-01T11:04:32.714340+0000 mon.a (mon.0) 1800 : cluster 3 Health check update: Degraded data redundancy: 23738/234840 objects degraded (10.108%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T11:04:32.996 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:32.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:32.999 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:04:33.100 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:33.122 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:32 smithi078 bash[78274]: audit 2023-10-01T11:04:32.457816+0000 mon.a (mon.0) 1799 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:04:33.122 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:32 smithi078 bash[78274]: cluster 2023-10-01T11:04:32.714340+0000 mon.a (mon.0) 1800 : cluster 3 Health check update: Degraded data redundancy: 23738/234840 objects degraded (10.108%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T11:04:33.123 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:32 smithi078 bash[75299]: audit 2023-10-01T11:04:32.457816+0000 mon.a (mon.0) 1799 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:04:33.123 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:32 smithi078 bash[75299]: cluster 2023-10-01T11:04:32.714340+0000 mon.a (mon.0) 1800 : cluster 3 Health check update: Degraded data redundancy: 23738/234840 objects degraded (10.108%), 5 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T11:04:33.134 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:33.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:33.136 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:04:33.236 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:33.261 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:33.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:33.263 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:04:33.364 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:33.390 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:33.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:33.393 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:04:33.494 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:33.528 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:33.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:33.532 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:04:33.632 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:33.663 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:33.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:33.666 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:04:33.767 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:33.798 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:33.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:33.801 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:33.902 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:33.925 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:33.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:33.930 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:04:34.031 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:34.043 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:33 smithi173 bash[58816]: cluster 2023-10-01T11:04:32.258187+0000 mgr.y (mgr.25072) 1415 : cluster 0 pgmap v1495: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 85 B/s rd, 85 B/s wr, 108 op/s; 23738/234840 objects degraded (10.108%); 4372/234840 objects misplaced (1.862%); 4.5 MiB/s, 72 objects/s recovering 2023-10-01T11:04:34.062 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:34.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:34.064 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:34.165 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:34.191 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:34.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:34.194 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:04:34.295 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:34.328 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:34.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:34.331 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:04:34.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:33 smithi078 bash[78274]: cluster 2023-10-01T11:04:32.258187+0000 mgr.y (mgr.25072) 1415 : cluster 0 pgmap v1495: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 85 B/s rd, 85 B/s wr, 108 op/s; 23738/234840 objects degraded (10.108%); 4372/234840 objects misplaced (1.862%); 4.5 MiB/s, 72 objects/s recovering 2023-10-01T11:04:34.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:33 smithi078 bash[75299]: cluster 2023-10-01T11:04:32.258187+0000 mgr.y (mgr.25072) 1415 : cluster 0 pgmap v1495: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 85 B/s rd, 85 B/s wr, 108 op/s; 23738/234840 objects degraded (10.108%); 4372/234840 objects misplaced (1.862%); 4.5 MiB/s, 72 objects/s recovering 2023-10-01T11:04:34.432 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:34.461 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:34.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:34.463 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:04:34.564 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:34.592 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:34.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:34.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:04:34.696 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:34.726 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:34.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:34.729 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:04:34.830 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:34.855 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:34.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:34.857 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:04:34.958 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:34.983 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:34.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:34.986 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:04:35.087 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:35.108 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:35.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:35.111 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:04:35.211 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:35.242 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:35.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:35.244 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:04:35.345 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:35.376 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:35.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:35.379 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:04:35.480 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:35.506 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:35.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:35.509 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:04:35.609 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:35.638 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:35.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:35.641 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:35.742 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:35.772 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:35.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:35.774 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:04:35.875 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:35.897 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:35.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:35.902 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:04:36.003 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:36.024 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:36.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:36.026 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:04:36.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:35 smithi078 bash[75299]: cluster 2023-10-01T11:04:34.259350+0000 mgr.y (mgr.25072) 1416 : cluster 0 pgmap v1496: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 85 B/s rd, 85 B/s wr, 140 op/s; 23708/233697 objects degraded (10.145%); 3858/233697 objects misplaced (1.651%); 7.4 MiB/s, 117 objects/s recovering 2023-10-01T11:04:36.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:35 smithi078 bash[78274]: cluster 2023-10-01T11:04:34.259350+0000 mgr.y (mgr.25072) 1416 : cluster 0 pgmap v1496: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 85 B/s rd, 85 B/s wr, 140 op/s; 23708/233697 objects degraded (10.145%); 3858/233697 objects misplaced (1.651%); 7.4 MiB/s, 117 objects/s recovering 2023-10-01T11:04:36.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:35 smithi173 bash[58816]: cluster 2023-10-01T11:04:34.259350+0000 mgr.y (mgr.25072) 1416 : cluster 0 pgmap v1496: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 85 B/s rd, 85 B/s wr, 140 op/s; 23708/233697 objects degraded (10.145%); 3858/233697 objects misplaced (1.651%); 7.4 MiB/s, 117 objects/s recovering 2023-10-01T11:04:36.127 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:36.158 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:36.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:36.160 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:04:36.182 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-01T11:04:36.261 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:36.289 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:36.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:36.292 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:04:36.393 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:36.417 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:36.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:36.419 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:04:36.520 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:36.548 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:36.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:36.550 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:36.650 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:36.680 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:36.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:36.683 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:04:36.785 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:36.815 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:36.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:36.817 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:04:36.918 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:36.948 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:36.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:36.951 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:04:37.052 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:37.082 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:37.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:37.084 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:04:37.185 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:37.206 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:37.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:37.208 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:04:37.309 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:37.339 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:37.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:37.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:04:37.444 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:37.473 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:37.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:37.476 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:04:37.577 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:37.607 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:37.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:37.609 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:04:37.710 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:37.732 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:37.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:37.735 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:04:37.836 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:37.862 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:37.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:37.865 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:04:37.966 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:37.996 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:37.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:37.998 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:04:38.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:37 smithi078 bash[78274]: cluster 2023-10-01T11:04:36.260179+0000 mgr.y (mgr.25072) 1417 : cluster 0 pgmap v1497: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 85 B/s rd, 85 B/s wr, 105 op/s; 23708/233403 objects degraded (10.158%); 3858/233403 objects misplaced (1.653%); 5.1 MiB/s, 82 objects/s recovering 2023-10-01T11:04:38.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:37 smithi078 bash[78274]: cluster 2023-10-01T11:04:37.715570+0000 mon.a (mon.0) 1801 : cluster 3 Health check update: Degraded data redundancy: 23708/233403 objects degraded (10.158%), 5 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T11:04:38.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:37 smithi078 bash[75299]: cluster 2023-10-01T11:04:36.260179+0000 mgr.y (mgr.25072) 1417 : cluster 0 pgmap v1497: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 85 B/s rd, 85 B/s wr, 105 op/s; 23708/233403 objects degraded (10.158%); 3858/233403 objects misplaced (1.653%); 5.1 MiB/s, 82 objects/s recovering 2023-10-01T11:04:38.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:37 smithi078 bash[75299]: cluster 2023-10-01T11:04:37.715570+0000 mon.a (mon.0) 1801 : cluster 3 Health check update: Degraded data redundancy: 23708/233403 objects degraded (10.158%), 5 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T11:04:38.099 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:38.111 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:37 smithi173 bash[58816]: cluster 2023-10-01T11:04:36.260179+0000 mgr.y (mgr.25072) 1417 : cluster 0 pgmap v1497: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 85 B/s rd, 85 B/s wr, 105 op/s; 23708/233403 objects degraded (10.158%); 3858/233403 objects misplaced (1.653%); 5.1 MiB/s, 82 objects/s recovering 2023-10-01T11:04:38.111 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:37 smithi173 bash[58816]: cluster 2023-10-01T11:04:37.715570+0000 mon.a (mon.0) 1801 : cluster 3 Health check update: Degraded data redundancy: 23708/233403 objects degraded (10.158%), 5 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T11:04:38.125 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:38.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:38.128 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:04:38.229 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:38.252 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:38.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:38.254 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:04:38.355 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:38.384 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:38.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:38.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:04:38.488 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:38.514 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:38.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:38.516 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:04:38.617 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:38.641 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:38.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:38.644 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:04:38.744 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:38.769 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:38.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:38.772 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:04:38.873 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:38.903 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:38.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:38.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:39.007 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:39.038 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:39.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:39.040 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:04:39.141 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:39.164 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:39.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:39.167 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:04:39.268 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:39.292 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:39.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:39.295 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:04:39.396 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:39.421 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:39.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:39.424 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:04:39.524 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:39.555 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:39.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:39.559 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:04:39.659 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:39.688 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:39.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:39.691 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:04:39.792 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:39.815 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:39.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:39.818 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:04:39.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:39 smithi173 bash[58816]: cluster 2023-10-01T11:04:38.260700+0000 mgr.y (mgr.25072) 1418 : cluster 0 pgmap v1498: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 101 op/s; 23708/233025 objects degraded (10.174%); 3234/233025 objects misplaced (1.388%); 8.4 MiB/s, 134 objects/s recovering 2023-10-01T11:04:39.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:39 smithi173 bash[58816]: audit 2023-10-01T11:04:39.011613+0000 mon.a (mon.0) 1802 : audit 1 from='client.? 172.21.15.78:0/1785418447' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.0}]: dispatch 2023-10-01T11:04:39.873 INFO:teuthology.orchestra.run.smithi078.stderr:set osd.0 primary-affinity to 0 (802) 2023-10-01T11:04:39.918 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:39.921 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:39 smithi078 bash[75299]: cluster 2023-10-01T11:04:38.260700+0000 mgr.y (mgr.25072) 1418 : cluster 0 pgmap v1498: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 101 op/s; 23708/233025 objects degraded (10.174%); 3234/233025 objects misplaced (1.388%); 8.4 MiB/s, 134 objects/s recovering 2023-10-01T11:04:39.921 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:39 smithi078 bash[75299]: audit 2023-10-01T11:04:39.011613+0000 mon.a (mon.0) 1802 : audit 1 from='client.? 172.21.15.78:0/1785418447' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.0}]: dispatch 2023-10-01T11:04:39.922 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:39 smithi078 bash[78274]: cluster 2023-10-01T11:04:38.260700+0000 mgr.y (mgr.25072) 1418 : cluster 0 pgmap v1498: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 101 op/s; 23708/233025 objects degraded (10.174%); 3234/233025 objects misplaced (1.388%); 8.4 MiB/s, 134 objects/s recovering 2023-10-01T11:04:39.922 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:39 smithi078 bash[78274]: audit 2023-10-01T11:04:39.011613+0000 mon.a (mon.0) 1802 : audit 1 from='client.? 172.21.15.78:0/1785418447' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.0}]: dispatch 2023-10-01T11:04:39.943 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:39.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:39.945 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:04:40.049 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:40.073 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:40.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:40.076 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:04:40.177 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:40.208 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:40.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:40.211 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:04:40.270 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:04:40 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:04:40] "GET /metrics HTTP/1.1" 200 36402 "" "Prometheus/2.33.4" 2023-10-01T11:04:40.312 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:40.335 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:40.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:40.337 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:04:40.438 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:40.470 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:40.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:40.475 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:04:40.575 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:40.607 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:40.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:40.610 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:04:40.711 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:40.738 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:40.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:40.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:04:40.842 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:40.876 INFO:teuthology.orchestra.run.smithi078.stderr:nodeep-scrub is unset 2023-10-01T11:04:40.881 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:40.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:40.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:04:40.980 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T11:04:40.984 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:41.012 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:41.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:41.018 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:04:41.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:40 smithi078 bash[75299]: cluster 2023-10-01T11:04:39.857980+0000 osd.7 (osd.7) 341 : cluster 3 failed to encode map e317 with expected crc 2023-10-01T11:04:41.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:40 smithi078 bash[75299]: audit 2023-10-01T11:04:39.860709+0000 mon.a (mon.0) 1803 : audit 1 from='client.? 172.21.15.78:0/1785418447' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.0}]': finished 2023-10-01T11:04:41.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:40 smithi078 bash[75299]: cluster 2023-10-01T11:04:39.861022+0000 osd.5 (osd.5) 327 : cluster 3 failed to encode map e317 with expected crc 2023-10-01T11:04:41.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:40 smithi078 bash[75299]: cluster 2023-10-01T11:04:39.861326+0000 mon.a (mon.0) 1804 : cluster 0 osdmap e317: 8 total, 8 up, 6 in 2023-10-01T11:04:41.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:40 smithi078 bash[75299]: cluster 2023-10-01T11:04:39.863858+0000 osd.5 (osd.5) 328 : cluster 3 failed to encode map e317 with expected crc 2023-10-01T11:04:41.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:40 smithi078 bash[75299]: cluster 2023-10-01T11:04:39.937967+0000 osd.6 (osd.6) 294 : cluster 3 failed to encode map e317 with expected crc 2023-10-01T11:04:41.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:40 smithi078 bash[75299]: audit 2023-10-01T11:04:40.500790+0000 mon.c (mon.2) 213 : audit 1 from='client.? 172.21.15.78:0/164908762' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:04:41.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:40 smithi078 bash[75299]: audit 2023-10-01T11:04:40.501350+0000 mon.a (mon.0) 1805 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:04:41.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:40 smithi078 bash[75299]: cluster 2023-10-01T11:04:40.856006+0000 mon.a (mon.0) 1806 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T11:04:41.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:40 smithi078 bash[75299]: audit 2023-10-01T11:04:40.862582+0000 mon.a (mon.0) 1807 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T11:04:41.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:40 smithi078 bash[75299]: cluster 2023-10-01T11:04:40.862677+0000 mon.a (mon.0) 1808 : cluster 0 osdmap e318: 8 total, 8 up, 6 in 2023-10-01T11:04:41.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:40 smithi078 bash[78274]: cluster 2023-10-01T11:04:39.857980+0000 osd.7 (osd.7) 341 : cluster 3 failed to encode map e317 with expected crc 2023-10-01T11:04:41.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:40 smithi078 bash[78274]: audit 2023-10-01T11:04:39.860709+0000 mon.a (mon.0) 1803 : audit 1 from='client.? 172.21.15.78:0/1785418447' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.0}]': finished 2023-10-01T11:04:41.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:40 smithi078 bash[78274]: cluster 2023-10-01T11:04:39.861022+0000 osd.5 (osd.5) 327 : cluster 3 failed to encode map e317 with expected crc 2023-10-01T11:04:41.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:40 smithi078 bash[78274]: cluster 2023-10-01T11:04:39.861326+0000 mon.a (mon.0) 1804 : cluster 0 osdmap e317: 8 total, 8 up, 6 in 2023-10-01T11:04:41.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:40 smithi078 bash[78274]: cluster 2023-10-01T11:04:39.863858+0000 osd.5 (osd.5) 328 : cluster 3 failed to encode map e317 with expected crc 2023-10-01T11:04:41.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:40 smithi078 bash[78274]: cluster 2023-10-01T11:04:39.937967+0000 osd.6 (osd.6) 294 : cluster 3 failed to encode map e317 with expected crc 2023-10-01T11:04:41.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:40 smithi078 bash[78274]: audit 2023-10-01T11:04:40.500790+0000 mon.c (mon.2) 213 : audit 1 from='client.? 172.21.15.78:0/164908762' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:04:41.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:40 smithi078 bash[78274]: audit 2023-10-01T11:04:40.501350+0000 mon.a (mon.0) 1805 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:04:41.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:40 smithi078 bash[78274]: cluster 2023-10-01T11:04:40.856006+0000 mon.a (mon.0) 1806 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T11:04:41.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:40 smithi078 bash[78274]: audit 2023-10-01T11:04:40.862582+0000 mon.a (mon.0) 1807 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T11:04:41.084 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:40 smithi078 bash[78274]: cluster 2023-10-01T11:04:40.862677+0000 mon.a (mon.0) 1808 : cluster 0 osdmap e318: 8 total, 8 up, 6 in 2023-10-01T11:04:41.118 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:41.143 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:41.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:41.146 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:04:41.247 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:41.267 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:41.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:41.269 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:04:41.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:40 smithi173 bash[58816]: cluster 2023-10-01T11:04:39.857980+0000 osd.7 (osd.7) 341 : cluster 3 failed to encode map e317 with expected crc 2023-10-01T11:04:41.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:40 smithi173 bash[58816]: audit 2023-10-01T11:04:39.860709+0000 mon.a (mon.0) 1803 : audit 1 from='client.? 172.21.15.78:0/1785418447' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.0}]': finished 2023-10-01T11:04:41.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:40 smithi173 bash[58816]: cluster 2023-10-01T11:04:39.861022+0000 osd.5 (osd.5) 327 : cluster 3 failed to encode map e317 with expected crc 2023-10-01T11:04:41.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:40 smithi173 bash[58816]: cluster 2023-10-01T11:04:39.861326+0000 mon.a (mon.0) 1804 : cluster 0 osdmap e317: 8 total, 8 up, 6 in 2023-10-01T11:04:41.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:40 smithi173 bash[58816]: cluster 2023-10-01T11:04:39.863858+0000 osd.5 (osd.5) 328 : cluster 3 failed to encode map e317 with expected crc 2023-10-01T11:04:41.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:40 smithi173 bash[58816]: cluster 2023-10-01T11:04:39.937967+0000 osd.6 (osd.6) 294 : cluster 3 failed to encode map e317 with expected crc 2023-10-01T11:04:41.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:40 smithi173 bash[58816]: audit 2023-10-01T11:04:40.500790+0000 mon.c (mon.2) 213 : audit 1 from='client.? 172.21.15.78:0/164908762' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:04:41.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:40 smithi173 bash[58816]: audit 2023-10-01T11:04:40.501350+0000 mon.a (mon.0) 1805 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:04:41.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:40 smithi173 bash[58816]: cluster 2023-10-01T11:04:40.856006+0000 mon.a (mon.0) 1806 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T11:04:41.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:40 smithi173 bash[58816]: audit 2023-10-01T11:04:40.862582+0000 mon.a (mon.0) 1807 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T11:04:41.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:40 smithi173 bash[58816]: cluster 2023-10-01T11:04:40.862677+0000 mon.a (mon.0) 1808 : cluster 0 osdmap e318: 8 total, 8 up, 6 in 2023-10-01T11:04:41.370 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:41.391 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:41.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:41.394 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:41.494 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:41.514 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:41.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:41.516 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:04:41.617 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:41.643 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:41.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:41.645 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:04:41.746 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:41.819 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:41.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:41.823 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:04:41.924 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:41.947 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:41.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:41.949 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:04:42.050 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:42.057 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:41 smithi078 bash[78274]: cluster 2023-10-01T11:04:40.261160+0000 mgr.y (mgr.25072) 1419 : cluster 0 pgmap v1500: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 81 op/s; 23674/232392 objects degraded (10.187%); 3237/232392 objects misplaced (1.393%); 7.5 MiB/s, 119 objects/s recovering 2023-10-01T11:04:42.057 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:41 smithi078 bash[78274]: cluster 2023-10-01T11:04:40.859660+0000 osd.7 (osd.7) 342 : cluster 3 failed to encode map e318 with expected crc 2023-10-01T11:04:42.058 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:41 smithi078 bash[78274]: cluster 2023-10-01T11:04:40.861766+0000 osd.5 (osd.5) 329 : cluster 3 failed to encode map e318 with expected crc 2023-10-01T11:04:42.058 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:41 smithi078 bash[78274]: cluster 2023-10-01T11:04:40.865087+0000 osd.5 (osd.5) 330 : cluster 3 failed to encode map e318 with expected crc 2023-10-01T11:04:42.058 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:41 smithi078 bash[78274]: cluster 2023-10-01T11:04:40.867490+0000 osd.5 (osd.5) 331 : cluster 3 failed to encode map e318 with expected crc 2023-10-01T11:04:42.058 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:41 smithi078 bash[75299]: cluster 2023-10-01T11:04:40.261160+0000 mgr.y (mgr.25072) 1419 : cluster 0 pgmap v1500: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 81 op/s; 23674/232392 objects degraded (10.187%); 3237/232392 objects misplaced (1.393%); 7.5 MiB/s, 119 objects/s recovering 2023-10-01T11:04:42.058 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:41 smithi078 bash[75299]: cluster 2023-10-01T11:04:40.859660+0000 osd.7 (osd.7) 342 : cluster 3 failed to encode map e318 with expected crc 2023-10-01T11:04:42.058 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:41 smithi078 bash[75299]: cluster 2023-10-01T11:04:40.861766+0000 osd.5 (osd.5) 329 : cluster 3 failed to encode map e318 with expected crc 2023-10-01T11:04:42.058 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:41 smithi078 bash[75299]: cluster 2023-10-01T11:04:40.865087+0000 osd.5 (osd.5) 330 : cluster 3 failed to encode map e318 with expected crc 2023-10-01T11:04:42.058 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:41 smithi078 bash[75299]: cluster 2023-10-01T11:04:40.867490+0000 osd.5 (osd.5) 331 : cluster 3 failed to encode map e318 with expected crc 2023-10-01T11:04:42.072 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:42.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:42.075 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:04:42.176 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:42.200 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:42.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:42.203 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:04:42.303 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:42.312 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:41 smithi173 bash[58816]: cluster 2023-10-01T11:04:40.261160+0000 mgr.y (mgr.25072) 1419 : cluster 0 pgmap v1500: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 81 op/s; 23674/232392 objects degraded (10.187%); 3237/232392 objects misplaced (1.393%); 7.5 MiB/s, 119 objects/s recovering 2023-10-01T11:04:42.312 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:41 smithi173 bash[58816]: cluster 2023-10-01T11:04:40.859660+0000 osd.7 (osd.7) 342 : cluster 3 failed to encode map e318 with expected crc 2023-10-01T11:04:42.312 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:41 smithi173 bash[58816]: cluster 2023-10-01T11:04:40.861766+0000 osd.5 (osd.5) 329 : cluster 3 failed to encode map e318 with expected crc 2023-10-01T11:04:42.312 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:41 smithi173 bash[58816]: cluster 2023-10-01T11:04:40.865087+0000 osd.5 (osd.5) 330 : cluster 3 failed to encode map e318 with expected crc 2023-10-01T11:04:42.312 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:41 smithi173 bash[58816]: cluster 2023-10-01T11:04:40.867490+0000 osd.5 (osd.5) 331 : cluster 3 failed to encode map e318 with expected crc 2023-10-01T11:04:42.327 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:42.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:42.330 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:42.430 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:42.459 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:42.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:42.461 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:04:42.562 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:42.589 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:42.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:42.591 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:04:42.692 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:42.718 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:42.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:42.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:04:42.822 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:42.849 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:42.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:42.851 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:42.952 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:42.981 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:42.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:42.984 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:04:43.085 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:43.105 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:42 smithi173 bash[58816]: cluster 2023-10-01T11:04:42.716858+0000 mon.a (mon.0) 1809 : cluster 3 Health check update: Degraded data redundancy: 23674/232392 objects degraded (10.187%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T11:04:43.112 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:43.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:43.118 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:04:43.218 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:43.227 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:42 smithi078 bash[78274]: cluster 2023-10-01T11:04:42.716858+0000 mon.a (mon.0) 1809 : cluster 3 Health check update: Degraded data redundancy: 23674/232392 objects degraded (10.187%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T11:04:43.228 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:42 smithi078 bash[75299]: cluster 2023-10-01T11:04:42.716858+0000 mon.a (mon.0) 1809 : cluster 3 Health check update: Degraded data redundancy: 23674/232392 objects degraded (10.187%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T11:04:43.244 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:43.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:43.246 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:04:43.300 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:04:43.347 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:43.370 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:43.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:43.373 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:04:43.475 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:43.502 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:43.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:43.505 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:04:43.605 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:43.630 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:43.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:43.633 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:04:43.734 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:43.760 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:43.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:43.763 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:04:43.863 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:43.886 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:43.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:43.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:43.988 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:44.012 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:44.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:44.017 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:04:44.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:43 smithi078 bash[75299]: cluster 2023-10-01T11:04:42.261606+0000 mgr.y (mgr.25072) 1420 : cluster 0 pgmap v1502: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 54 op/s; 23674/232392 objects degraded (10.187%); 3237/232392 objects misplaced (1.393%); 5.1 MiB/s, 81 objects/s recovering 2023-10-01T11:04:44.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:43 smithi078 bash[78274]: cluster 2023-10-01T11:04:42.261606+0000 mgr.y (mgr.25072) 1420 : cluster 0 pgmap v1502: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 54 op/s; 23674/232392 objects degraded (10.187%); 3237/232392 objects misplaced (1.393%); 5.1 MiB/s, 81 objects/s recovering 2023-10-01T11:04:44.118 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:44.144 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:44.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:44.149 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:04:44.249 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:44.258 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:43 smithi173 bash[58816]: cluster 2023-10-01T11:04:42.261606+0000 mgr.y (mgr.25072) 1420 : cluster 0 pgmap v1502: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 54 op/s; 23674/232392 objects degraded (10.187%); 3237/232392 objects misplaced (1.393%); 5.1 MiB/s, 81 objects/s recovering 2023-10-01T11:04:44.275 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:44.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:44.278 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:44.379 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:44.411 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:44.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:44.414 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:04:44.515 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:44.537 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:44.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:44.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:04:44.640 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:44.669 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:44.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:44.672 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:04:44.773 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:44.798 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:44.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:44.800 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:04:44.884 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T11:04:44.901 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:44.963 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:44.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:44.966 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:04:45.067 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:45.093 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:45.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:45.095 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:04:45.196 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:45.224 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:45.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:45.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:04:45.327 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:45.358 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:45.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:45.362 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:45.439 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-10-01T11:04:45.439 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T11:04:45.463 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:45.489 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:45.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:45.491 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:45.592 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:45.624 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:45.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:45.626 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:45.726 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:45.756 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:45.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:45.759 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:04:45.860 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:45.891 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:45.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:45.893 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:04:45.995 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:46.030 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:46.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:46.031 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:04:46.132 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:46.159 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:46.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:46.162 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:04:46.211 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:45 smithi078 bash[75299]: cluster 2023-10-01T11:04:44.262540+0000 mgr.y (mgr.25072) 1421 : cluster 0 pgmap v1503: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 81 op/s; 23644/231456 objects degraded (10.215%); 2476/231456 objects misplaced (1.070%); 11 MiB/s, 178 objects/s recovering 2023-10-01T11:04:46.262 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:46.266 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:45 smithi173 bash[58816]: cluster 2023-10-01T11:04:44.262540+0000 mgr.y (mgr.25072) 1421 : cluster 0 pgmap v1503: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 81 op/s; 23644/231456 objects degraded (10.215%); 2476/231456 objects misplaced (1.070%); 11 MiB/s, 178 objects/s recovering 2023-10-01T11:04:46.267 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-01T11:04:46.287 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:46.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:46.290 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:46.391 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:46.420 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:46.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:46.422 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:04:46.523 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:46.537 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:46 smithi078 bash[78274]: cluster 2023-10-01T11:04:44.262540+0000 mgr.y (mgr.25072) 1421 : cluster 0 pgmap v1503: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 81 op/s; 23644/231456 objects degraded (10.215%); 2476/231456 objects misplaced (1.070%); 11 MiB/s, 178 objects/s recovering 2023-10-01T11:04:46.555 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:46.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:46.557 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:04:46.658 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:46.686 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:46.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:46.689 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:46.748 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5] out_osds: [6, 7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T11:04:46.749 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T11:04:46.749 INFO:tasks.thrashosds.thrasher:Growing pool device_health_metrics 2023-10-01T11:04:46.749 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T11:04:46.790 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:46.816 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:46.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:46.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:46.919 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:46.928 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:46 smithi173 bash[58816]: cluster 2023-10-01T11:04:45.906754+0000 osd.7 (osd.7) 343 : cluster 3 failed to encode map e319 with expected crc 2023-10-01T11:04:46.928 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:46 smithi173 bash[58816]: cluster 2023-10-01T11:04:45.907891+0000 mon.a (mon.0) 1810 : cluster 0 osdmap e319: 8 total, 8 up, 6 in 2023-10-01T11:04:46.928 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:46 smithi173 bash[58816]: cluster 2023-10-01T11:04:45.913030+0000 osd.5 (osd.5) 332 : cluster 3 failed to encode map e319 with expected crc 2023-10-01T11:04:46.928 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:46 smithi173 bash[58816]: cluster 2023-10-01T11:04:45.913261+0000 osd.6 (osd.6) 295 : cluster 3 failed to encode map e319 with expected crc 2023-10-01T11:04:46.942 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:46.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:46.945 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:04:47.046 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:47.055 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:46 smithi078 bash[75299]: cluster 2023-10-01T11:04:45.906754+0000 osd.7 (osd.7) 343 : cluster 3 failed to encode map e319 with expected crc 2023-10-01T11:04:47.055 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:46 smithi078 bash[75299]: cluster 2023-10-01T11:04:45.907891+0000 mon.a (mon.0) 1810 : cluster 0 osdmap e319: 8 total, 8 up, 6 in 2023-10-01T11:04:47.055 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:46 smithi078 bash[75299]: cluster 2023-10-01T11:04:45.913030+0000 osd.5 (osd.5) 332 : cluster 3 failed to encode map e319 with expected crc 2023-10-01T11:04:47.055 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:46 smithi078 bash[75299]: cluster 2023-10-01T11:04:45.913261+0000 osd.6 (osd.6) 295 : cluster 3 failed to encode map e319 with expected crc 2023-10-01T11:04:47.055 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:46 smithi078 bash[78274]: cluster 2023-10-01T11:04:45.906754+0000 osd.7 (osd.7) 343 : cluster 3 failed to encode map e319 with expected crc 2023-10-01T11:04:47.056 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:46 smithi078 bash[78274]: cluster 2023-10-01T11:04:45.907891+0000 mon.a (mon.0) 1810 : cluster 0 osdmap e319: 8 total, 8 up, 6 in 2023-10-01T11:04:47.056 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:46 smithi078 bash[78274]: cluster 2023-10-01T11:04:45.913030+0000 osd.5 (osd.5) 332 : cluster 3 failed to encode map e319 with expected crc 2023-10-01T11:04:47.056 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:46 smithi078 bash[78274]: cluster 2023-10-01T11:04:45.913261+0000 osd.6 (osd.6) 295 : cluster 3 failed to encode map e319 with expected crc 2023-10-01T11:04:47.067 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:47.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:47.071 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:04:47.171 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:47.193 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:47.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:47.196 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:04:47.297 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:47.323 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:47.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:47.326 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:47.426 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:47.451 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:47.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:47.453 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:04:47.554 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:47.580 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:47.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:47.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:04:47.684 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:47.708 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:47.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:47.710 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:04:47.811 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:47.841 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:47.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:47.843 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:47.944 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:47.955 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:47 smithi173 bash[58816]: cluster 2023-10-01T11:04:46.263378+0000 mgr.y (mgr.25072) 1422 : cluster 0 pgmap v1505: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 64 op/s; 23644/231150 objects degraded (10.229%); 2476/231150 objects misplaced (1.071%); 7.6 MiB/s, 121 objects/s recovering 2023-10-01T11:04:47.955 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:47 smithi173 bash[58816]: cluster 2023-10-01T11:04:46.914659+0000 osd.6 (osd.6) 296 : cluster 3 failed to encode map e320 with expected crc 2023-10-01T11:04:47.955 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:47 smithi173 bash[58816]: cluster 2023-10-01T11:04:46.915874+0000 mon.a (mon.0) 1811 : cluster 0 osdmap e320: 8 total, 8 up, 6 in 2023-10-01T11:04:47.955 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:47 smithi173 bash[58816]: cluster 2023-10-01T11:04:46.918096+0000 osd.7 (osd.7) 344 : cluster 3 failed to encode map e320 with expected crc 2023-10-01T11:04:47.956 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:47 smithi173 bash[58816]: audit 2023-10-01T11:04:47.461925+0000 mon.a (mon.0) 1812 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:04:47.956 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:47 smithi173 bash[58816]: audit 2023-10-01T11:04:47.462968+0000 mon.a (mon.0) 1813 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:04:47.956 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:47 smithi173 bash[58816]: cluster 2023-10-01T11:04:47.718120+0000 mon.a (mon.0) 1814 : cluster 3 Health check update: Degraded data redundancy: 23644/231150 objects degraded (10.229%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T11:04:47.970 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:47.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:47.973 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:04:48.073 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:48.106 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:48.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:48.109 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:04:48.210 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:48.222 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:47 smithi078 bash[78274]: cluster 2023-10-01T11:04:46.263378+0000 mgr.y (mgr.25072) 1422 : cluster 0 pgmap v1505: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 64 op/s; 23644/231150 objects degraded (10.229%); 2476/231150 objects misplaced (1.071%); 7.6 MiB/s, 121 objects/s recovering 2023-10-01T11:04:48.222 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:47 smithi078 bash[78274]: cluster 2023-10-01T11:04:46.914659+0000 osd.6 (osd.6) 296 : cluster 3 failed to encode map e320 with expected crc 2023-10-01T11:04:48.222 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:47 smithi078 bash[78274]: cluster 2023-10-01T11:04:46.915874+0000 mon.a (mon.0) 1811 : cluster 0 osdmap e320: 8 total, 8 up, 6 in 2023-10-01T11:04:48.222 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:47 smithi078 bash[78274]: cluster 2023-10-01T11:04:46.918096+0000 osd.7 (osd.7) 344 : cluster 3 failed to encode map e320 with expected crc 2023-10-01T11:04:48.222 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:47 smithi078 bash[78274]: audit 2023-10-01T11:04:47.461925+0000 mon.a (mon.0) 1812 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:04:48.223 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:47 smithi078 bash[78274]: audit 2023-10-01T11:04:47.462968+0000 mon.a (mon.0) 1813 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:04:48.223 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:47 smithi078 bash[78274]: cluster 2023-10-01T11:04:47.718120+0000 mon.a (mon.0) 1814 : cluster 3 Health check update: Degraded data redundancy: 23644/231150 objects degraded (10.229%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T11:04:48.223 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:47 smithi078 bash[75299]: cluster 2023-10-01T11:04:46.263378+0000 mgr.y (mgr.25072) 1422 : cluster 0 pgmap v1505: 89 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.7 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 64 op/s; 23644/231150 objects degraded (10.229%); 2476/231150 objects misplaced (1.071%); 7.6 MiB/s, 121 objects/s recovering 2023-10-01T11:04:48.223 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:47 smithi078 bash[75299]: cluster 2023-10-01T11:04:46.914659+0000 osd.6 (osd.6) 296 : cluster 3 failed to encode map e320 with expected crc 2023-10-01T11:04:48.223 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:47 smithi078 bash[75299]: cluster 2023-10-01T11:04:46.915874+0000 mon.a (mon.0) 1811 : cluster 0 osdmap e320: 8 total, 8 up, 6 in 2023-10-01T11:04:48.223 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:47 smithi078 bash[75299]: cluster 2023-10-01T11:04:46.918096+0000 osd.7 (osd.7) 344 : cluster 3 failed to encode map e320 with expected crc 2023-10-01T11:04:48.223 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:47 smithi078 bash[75299]: audit 2023-10-01T11:04:47.461925+0000 mon.a (mon.0) 1812 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:04:48.223 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:47 smithi078 bash[75299]: audit 2023-10-01T11:04:47.462968+0000 mon.a (mon.0) 1813 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:04:48.224 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:47 smithi078 bash[75299]: cluster 2023-10-01T11:04:47.718120+0000 mon.a (mon.0) 1814 : cluster 3 Health check update: Degraded data redundancy: 23644/231150 objects degraded (10.229%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T11:04:48.238 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:48.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:48.240 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:04:48.341 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:48.372 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:48.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:48.376 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:48.477 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:48.490 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T11:04:48.501 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:48.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:48.504 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:04:48.516 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T11:04:48.540 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T11:04:48.565 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T11:04:48.593 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T11:04:48.605 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:48.623 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T11:04:48.631 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:48.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:48.634 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:04:48.653 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T11:04:48.681 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T11:04:48.683 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T11:04:48.684 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T11:04:48.684 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T11:04:48.684 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T11:04:48.684 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T11:04:48.684 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T11:04:48.684 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T11:04:48.684 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T11:04:48.735 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:48.763 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:48.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:48.766 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:04:48.867 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:48.893 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:48.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:48.896 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:48.997 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:49.018 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:49.020 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:48 smithi078 bash[75299]: audit 2023-10-01T11:04:48.463175+0000 mon.a (mon.0) 1815 : audit 0 from='client.? 172.21.15.78:0/3160358210' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:04:49.020 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:48 smithi078 bash[78274]: audit 2023-10-01T11:04:48.463175+0000 mon.a (mon.0) 1815 : audit 0 from='client.? 172.21.15.78:0/3160358210' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:04:49.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:49.021 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:04:49.122 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:49.144 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:49.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:49.147 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:04:49.248 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:49.259 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:48 smithi173 bash[58816]: audit 2023-10-01T11:04:48.463175+0000 mon.a (mon.0) 1815 : audit 0 from='client.? 172.21.15.78:0/3160358210' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:04:49.276 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:49.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:49.279 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:04:49.380 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:49.403 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:49.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:49.405 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:04:49.506 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:49.535 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:49.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:49.538 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:04:49.639 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:49.665 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:49.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:49.667 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:49.768 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:49.799 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:49.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:49.802 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:04:49.903 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:49.934 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:49.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:49.939 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:04:50.039 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:50.054 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:49 smithi078 bash[78274]: cluster 2023-10-01T11:04:48.264247+0000 mgr.y (mgr.25072) 1423 : cluster 0 pgmap v1507: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 82 op/s; 23644/230553 objects degraded (10.255%); 1782/230553 objects misplaced (0.773%); 12 MiB/s, 198 objects/s recovering 2023-10-01T11:04:50.054 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:49 smithi078 bash[75299]: cluster 2023-10-01T11:04:48.264247+0000 mgr.y (mgr.25072) 1423 : cluster 0 pgmap v1507: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 82 op/s; 23644/230553 objects degraded (10.255%); 1782/230553 objects misplaced (0.773%); 12 MiB/s, 198 objects/s recovering 2023-10-01T11:04:50.072 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:50.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:50.078 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:04:50.179 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:50.191 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:49 smithi173 bash[58816]: cluster 2023-10-01T11:04:48.264247+0000 mgr.y (mgr.25072) 1423 : cluster 0 pgmap v1507: 89 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 82 op/s; 23644/230553 objects degraded (10.255%); 1782/230553 objects misplaced (0.773%); 12 MiB/s, 198 objects/s recovering 2023-10-01T11:04:50.209 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:50.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:50.212 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:50.293 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:04:50.313 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:50.322 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:04:50 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:04:50] "GET /metrics HTTP/1.1" 200 36407 "" "Prometheus/2.33.4" 2023-10-01T11:04:50.337 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:50.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:50.340 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:04:50.440 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:50.471 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:50.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:50.474 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:04:50.575 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:50.606 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:50.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:50.609 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:04:50.710 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:50.737 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:50.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:50.739 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:04:50.839 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:50.872 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:50.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:50.875 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:50.976 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:51.003 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:51.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:51.006 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:04:51.074 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T11:04:51.106 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:51.133 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:51.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:51.135 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:04:51.236 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:51.260 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:51.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:51.263 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:04:51.308 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:04:51.365 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:51.389 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:51.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:51.391 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:04:51.492 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:51.523 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:51.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:51.525 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:04:51.625 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:51.657 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:51.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:51.660 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:04:51.761 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:51.795 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:51.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:51.797 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:04:51.813 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T11:04:51.898 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:51.928 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:51.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:51.930 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:04:52.031 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:52.058 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:52.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:52.061 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:04:52.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:51 smithi078 bash[78274]: cluster 2023-10-01T11:04:50.265468+0000 mgr.y (mgr.25072) 1424 : cluster 0 pgmap v1508: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 115 op/s; 20578/229620 objects degraded (8.962%); 4819/229620 objects misplaced (2.099%); 12 MiB/s, 187 objects/s recovering 2023-10-01T11:04:52.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:51 smithi078 bash[75299]: cluster 2023-10-01T11:04:50.265468+0000 mgr.y (mgr.25072) 1424 : cluster 0 pgmap v1508: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 115 op/s; 20578/229620 objects degraded (8.962%); 4819/229620 objects misplaced (2.099%); 12 MiB/s, 187 objects/s recovering 2023-10-01T11:04:52.162 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:52.188 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:52.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:52.190 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:04:52.291 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:52.302 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:51 smithi173 bash[58816]: cluster 2023-10-01T11:04:50.265468+0000 mgr.y (mgr.25072) 1424 : cluster 0 pgmap v1508: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 115 op/s; 20578/229620 objects degraded (8.962%); 4819/229620 objects misplaced (2.099%); 12 MiB/s, 187 objects/s recovering 2023-10-01T11:04:52.319 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:52.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:52.323 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:04:52.423 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:52.457 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:52.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:52.460 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:04:52.561 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:52.586 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:52.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:52.589 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:04:52.690 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:52.714 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:52.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:52.716 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:04:52.817 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:52.842 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:52.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:52.846 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:04:52.946 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:52.958 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:52 smithi078 bash[78274]: cluster 2023-10-01T11:04:52.719189+0000 mon.a (mon.0) 1816 : cluster 3 Health check update: Degraded data redundancy: 20578/229620 objects degraded (8.962%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T11:04:52.959 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:52 smithi078 bash[75299]: cluster 2023-10-01T11:04:52.719189+0000 mon.a (mon.0) 1816 : cluster 3 Health check update: Degraded data redundancy: 20578/229620 objects degraded (8.962%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T11:04:52.973 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:52.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:52.981 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:04:53.083 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:53.110 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:53.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:53.113 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:53.214 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:53.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:53.239 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:53.239 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:04:53.308 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:52 smithi173 bash[58816]: cluster 2023-10-01T11:04:52.719189+0000 mon.a (mon.0) 1816 : cluster 3 Health check update: Degraded data redundancy: 20578/229620 objects degraded (8.962%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T11:04:53.340 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:53.369 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:53.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:53.372 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:04:53.472 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:53.495 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:53.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:53.498 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:04:53.599 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:53.632 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:53.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:53.635 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:04:53.736 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:53.760 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:53.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:53.763 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:04:53.863 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:53.893 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:53.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:53.896 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:04:53.997 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:54.023 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:54.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:54.026 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:54.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:53 smithi078 bash[75299]: cluster 2023-10-01T11:04:52.266269+0000 mgr.y (mgr.25072) 1425 : cluster 0 pgmap v1509: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 76 op/s; 20578/229620 objects degraded (8.962%); 4819/229620 objects misplaced (2.099%); 5.6 MiB/s, 90 objects/s recovering 2023-10-01T11:04:54.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:53 smithi078 bash[78274]: cluster 2023-10-01T11:04:52.266269+0000 mgr.y (mgr.25072) 1425 : cluster 0 pgmap v1509: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 76 op/s; 20578/229620 objects degraded (8.962%); 4819/229620 objects misplaced (2.099%); 5.6 MiB/s, 90 objects/s recovering 2023-10-01T11:04:54.127 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:54.134 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:53 smithi173 bash[58816]: cluster 2023-10-01T11:04:52.266269+0000 mgr.y (mgr.25072) 1425 : cluster 0 pgmap v1509: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 76 op/s; 20578/229620 objects degraded (8.962%); 4819/229620 objects misplaced (2.099%); 5.6 MiB/s, 90 objects/s recovering 2023-10-01T11:04:54.152 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:54.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:54.154 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:04:54.255 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:54.280 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:54.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:54.285 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:04:54.386 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:54.411 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:54.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:54.413 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:04:54.514 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:54.539 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:54.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:54.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:04:54.596 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-10-01T11:04:54.596 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [], 2023-10-01T11:04:54.596 INFO:teuthology.orchestra.run.smithi173.stdout: "complaint_time": 30, 2023-10-01T11:04:54.597 INFO:teuthology.orchestra.run.smithi173.stdout: "num_blocked_ops": 0 2023-10-01T11:04:54.597 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-10-01T11:04:54.644 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:54.669 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:54.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:54.674 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:04:54.775 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:54.794 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:54.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:54.796 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:04:54.897 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:54.918 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:54.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:54.919 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:55.020 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:55.043 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:55.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:55.046 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:55.147 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:55.176 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:55.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:55.179 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:04:55.280 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:55.299 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:55.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:55.301 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:04:55.402 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:55.431 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:55.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:55.435 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:04:55.536 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:55.540 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-01T11:04:55.558 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:55.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:55.561 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:04:55.662 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:55.687 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:55.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:55.689 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:04:55.790 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:55.819 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:55.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:55.821 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:04:55.922 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:55.952 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:55.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:55.955 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:04:56.056 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:56.068 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:55 smithi173 bash[58816]: cluster 2023-10-01T11:04:54.267377+0000 mgr.y (mgr.25072) 1426 : cluster 0 pgmap v1510: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 94 op/s; 20578/228789 objects degraded (8.994%); 4786/228789 objects misplaced (2.092%); 5.7 MiB/s, 90 objects/s recovering 2023-10-01T11:04:56.084 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:56.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:56.087 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:56.188 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:56.196 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:55 smithi078 bash[75299]: cluster 2023-10-01T11:04:54.267377+0000 mgr.y (mgr.25072) 1426 : cluster 0 pgmap v1510: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 94 op/s; 20578/228789 objects degraded (8.994%); 4786/228789 objects misplaced (2.092%); 5.7 MiB/s, 90 objects/s recovering 2023-10-01T11:04:56.196 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:55 smithi078 bash[78274]: cluster 2023-10-01T11:04:54.267377+0000 mgr.y (mgr.25072) 1426 : cluster 0 pgmap v1510: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 94 op/s; 20578/228789 objects degraded (8.994%); 4786/228789 objects misplaced (2.092%); 5.7 MiB/s, 90 objects/s recovering 2023-10-01T11:04:56.210 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:56.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:56.213 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:04:56.314 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:56.340 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:56.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:56.342 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:04:56.443 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:56.465 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:56.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:56.468 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:04:56.569 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:56.595 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:56.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:56.599 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:56.699 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:56.726 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:56.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:56.729 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:04:56.829 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:56.857 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:56.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:56.860 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:56.960 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:56.988 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:56.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:56.991 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:04:57.091 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:57.114 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:57.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:57.117 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:04:57.217 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:57.245 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:57.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:57.248 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:57.349 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:57.373 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:57.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:57.376 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:04:57.476 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:57.505 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:57.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:57.507 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:04:57.608 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:57.638 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:57.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:57.641 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:04:57.742 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:57.773 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:57.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:57.775 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:04:57.876 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:57.899 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:57.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:57.902 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:04:58.003 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:58.032 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:58.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:58.035 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:04:58.054 INFO:teuthology.orchestra.run.smithi078.stdout: 2023-10-01T11:04:58.058 INFO:teuthology.orchestra.run.smithi078.stderr:dumped all 2023-10-01T11:04:58.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:57 smithi173 bash[58816]: cluster 2023-10-01T11:04:56.268129+0000 mgr.y (mgr.25072) 1427 : cluster 0 pgmap v1511: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 88 op/s; 20578/228495 objects degraded (9.006%); 4786/228495 objects misplaced (2.095%); 4.7 MiB/s, 75 objects/s recovering 2023-10-01T11:04:58.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:57 smithi173 bash[58816]: cluster 2023-10-01T11:04:57.720541+0000 mon.a (mon.0) 1817 : cluster 3 Health check update: Degraded data redundancy: 20578/228495 objects degraded (9.006%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T11:04:58.136 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:58.147 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:57 smithi078 bash[78274]: cluster 2023-10-01T11:04:56.268129+0000 mgr.y (mgr.25072) 1427 : cluster 0 pgmap v1511: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 88 op/s; 20578/228495 objects degraded (9.006%); 4786/228495 objects misplaced (2.095%); 4.7 MiB/s, 75 objects/s recovering 2023-10-01T11:04:58.148 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:57 smithi078 bash[78274]: cluster 2023-10-01T11:04:57.720541+0000 mon.a (mon.0) 1817 : cluster 3 Health check update: Degraded data redundancy: 20578/228495 objects degraded (9.006%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T11:04:58.148 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:57 smithi078 bash[75299]: cluster 2023-10-01T11:04:56.268129+0000 mgr.y (mgr.25072) 1427 : cluster 0 pgmap v1511: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 88 op/s; 20578/228495 objects degraded (9.006%); 4786/228495 objects misplaced (2.095%); 4.7 MiB/s, 75 objects/s recovering 2023-10-01T11:04:58.148 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:57 smithi078 bash[75299]: cluster 2023-10-01T11:04:57.720541+0000 mon.a (mon.0) 1817 : cluster 3 Health check update: Degraded data redundancy: 20578/228495 objects degraded (9.006%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T11:04:58.162 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:58.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:58.165 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:04:58.266 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:58.291 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:58.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:58.296 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:04:58.397 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:58.423 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:04:58.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:58.426 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:04:58.526 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:58.552 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:58.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:58.555 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:04:58.655 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:58.682 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:04:58.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:58.684 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:04:58.785 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:58.811 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:04:58.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:58.814 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:04:58.915 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:58.940 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:58.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:58.944 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:04:58.959 INFO:teuthology.orchestra.run.smithi078.stderr:noscrub is set 2023-10-01T11:04:59.044 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:59.076 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:59.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:59.079 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:04:59.080 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:58 smithi078 bash[75299]: audit 2023-10-01T11:04:58.035777+0000 mgr.y (mgr.25072) 1428 : audit 0 from='client.36597 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T11:04:59.080 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:58 smithi078 bash[75299]: audit 2023-10-01T11:04:58.062585+0000 mon.c (mon.2) 214 : audit 1 from='client.? 172.21.15.78:0/3033915762' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T11:04:59.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:58 smithi078 bash[75299]: audit 2023-10-01T11:04:58.062936+0000 mon.a (mon.0) 1818 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T11:04:59.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:58 smithi078 bash[78274]: audit 2023-10-01T11:04:58.035777+0000 mgr.y (mgr.25072) 1428 : audit 0 from='client.36597 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T11:04:59.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:58 smithi078 bash[78274]: audit 2023-10-01T11:04:58.062585+0000 mon.c (mon.2) 214 : audit 1 from='client.? 172.21.15.78:0/3033915762' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T11:04:59.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:58 smithi078 bash[78274]: audit 2023-10-01T11:04:58.062936+0000 mon.a (mon.0) 1818 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T11:04:59.180 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:59.210 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:04:59.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:59.213 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:04:59.220 INFO:teuthology.orchestra.run.smithi078.stdout:{"pg_ready":true,"pg_map":{"version":1511,"stamp":"2023-10-01T11:04:56.267579+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4990506936,"num_objects":76165,"num_object_clones":0,"num_object_copies":228495,"num_objects_missing_on_primary":3005,"num_objects_missing":3005,"num_objects_degraded":20578,"num_objects_misplaced":4786,"num_objects_unfound":0,"num_objects_dirty":76165,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":83232,"num_write_kb":5093967,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11486,"num_bytes_recovered":752381144,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":83166,"ondisk_log_size":83166,"up":267,"acting":262,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":17560824,"kb_used_data":13062920,"kb_used_omap":30,"kb_used_meta":4497825,"kb_avail":544910088,"statfs":{"total":575970213888,"available":557987930112,"internally_reserved":0,"allocated":13376430080,"data_stored":13369726580,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31502,"internal_metadata":4605773042},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-57999360,"num_objects":-885,"num_object_clones":0,"num_object_copies":-2655,"num_objects_missing_on_primary":-61,"num_objects_missing":-61,"num_objects_degraded":-3066,"num_objects_misplaced":2310,"num_objects_unfound":0,"num_objects_dirty":-885,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":885,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":756,"num_bytes_recovered":49545216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004760"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":300,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.906961+0000","last_change":"2023-10-01T11:04:02.725956+0000","last_active":"2023-10-01T11:04:45.906961+0000","last_peered":"2023-10-01T11:04:45.906961+0000","last_clean":"2023-10-01T11:04:45.906961+0000","last_became_active":"2023-10-01T11:04:02.725317+0000","last_became_peered":"2023-10-01T11:04:02.725317+0000","last_unstale":"2023-10-01T11:04:45.906961+0000","last_undegraded":"2023-10-01T11:04:45.906961+0000","last_fullsized":"2023-10-01T11:04:45.906961+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:34:07.675650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":304,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.908138+0000","last_change":"2023-10-01T10:38:16.025881+0000","last_active":"2023-10-01T11:04:45.908138+0000","last_peered":"2023-10-01T11:04:45.908138+0000","last_clean":"2023-10-01T11:04:45.908138+0000","last_became_active":"2023-10-01T10:38:16.025518+0000","last_became_peered":"2023-10-01T10:38:16.025518+0000","last_unstale":"2023-10-01T11:04:45.908138+0000","last_undegraded":"2023-10-01T11:04:45.908138+0000","last_fullsized":"2023-10-01T11:04:45.908138+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:48:37.826604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":225,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.907525+0000","last_change":"2023-10-01T10:52:18.956544+0000","last_active":"2023-10-01T11:04:45.907525+0000","last_peered":"2023-10-01T11:04:45.907525+0000","last_clean":"2023-10-01T11:04:45.907525+0000","last_became_active":"2023-10-01T10:52:18.956217+0000","last_became_peered":"2023-10-01T10:52:18.956217+0000","last_unstale":"2023-10-01T11:04:45.907525+0000","last_undegraded":"2023-10-01T11:04:45.907525+0000","last_fullsized":"2023-10-01T11:04:45.907525+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:34:01.952235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":313,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.908307+0000","last_change":"2023-10-01T11:04:40.878679+0000","last_active":"2023-10-01T11:04:45.908307+0000","last_peered":"2023-10-01T11:04:45.908307+0000","last_clean":"2023-10-01T11:04:45.908307+0000","last_became_active":"2023-10-01T11:04:40.878386+0000","last_became_peered":"2023-10-01T11:04:40.878386+0000","last_unstale":"2023-10-01T11:04:45.908307+0000","last_undegraded":"2023-10-01T11:04:45.908307+0000","last_fullsized":"2023-10-01T11:04:45.908307+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:17:07.380788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":306,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.907358+0000","last_change":"2023-10-01T10:37:42.766600+0000","last_active":"2023-10-01T11:04:45.907358+0000","last_peered":"2023-10-01T11:04:45.907358+0000","last_clean":"2023-10-01T11:04:45.907358+0000","last_became_active":"2023-10-01T10:37:42.766051+0000","last_became_peered":"2023-10-01T10:37:42.766051+0000","last_unstale":"2023-10-01T11:04:45.907358+0000","last_undegraded":"2023-10-01T11:04:45.907358+0000","last_fullsized":"2023-10-01T11:04:45.907358+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:51:17.707963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":132,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.917348+0000","last_change":"2023-10-01T11:03:07.743974+0000","last_active":"2023-10-01T11:04:45.917348+0000","last_peered":"2023-10-01T11:04:45.917348+0000","last_clean":"2023-10-01T11:04:45.917348+0000","last_became_active":"2023-10-01T11:03:07.743480+0000","last_became_peered":"2023-10-01T11:03:07.743480+0000","last_unstale":"2023-10-01T11:04:45.917348+0000","last_undegraded":"2023-10-01T11:04:45.917348+0000","last_fullsized":"2023-10-01T11:04:45.917348+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":80,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.917212+0000","last_change":"2023-10-01T11:04:40.881723+0000","last_active":"2023-10-01T11:04:45.917212+0000","last_peered":"2023-10-01T11:04:45.917212+0000","last_clean":"2023-10-01T11:04:45.917212+0000","last_became_active":"2023-10-01T11:04:40.881381+0000","last_became_peered":"2023-10-01T11:04:40.881381+0000","last_unstale":"2023-10-01T11:04:45.917212+0000","last_undegraded":"2023-10-01T11:04:45.917212+0000","last_fullsized":"2023-10-01T11:04:45.917212+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":335,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.907436+0000","last_change":"2023-10-01T11:03:07.706340+0000","last_active":"2023-10-01T11:04:45.907436+0000","last_peered":"2023-10-01T11:04:45.907436+0000","last_clean":"2023-10-01T11:04:45.907436+0000","last_became_active":"2023-10-01T11:03:07.705969+0000","last_became_peered":"2023-10-01T11:03:07.705969+0000","last_unstale":"2023-10-01T11:04:45.907436+0000","last_undegraded":"2023-10-01T11:04:45.907436+0000","last_fullsized":"2023-10-01T11:04:45.907436+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:47:22.620457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":314,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.917031+0000","last_change":"2023-10-01T11:03:35.605699+0000","last_active":"2023-10-01T11:04:45.917031+0000","last_peered":"2023-10-01T11:04:45.917031+0000","last_clean":"2023-10-01T11:04:45.917031+0000","last_became_active":"2023-10-01T11:03:35.605198+0000","last_became_peered":"2023-10-01T11:03:35.605198+0000","last_unstale":"2023-10-01T11:04:45.917031+0000","last_undegraded":"2023-10-01T11:04:45.917031+0000","last_fullsized":"2023-10-01T11:04:45.917031+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":346,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.908320+0000","last_change":"2023-10-01T11:03:35.506550+0000","last_active":"2023-10-01T11:04:45.908320+0000","last_peered":"2023-10-01T11:04:45.908320+0000","last_clean":"2023-10-01T11:04:45.908320+0000","last_became_active":"2023-10-01T11:03:35.506187+0000","last_became_peered":"2023-10-01T11:03:35.506187+0000","last_unstale":"2023-10-01T11:04:45.908320+0000","last_undegraded":"2023-10-01T11:04:45.908320+0000","last_fullsized":"2023-10-01T11:04:45.908320+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:41:31.370791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,0],"acting":[3,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":209,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-01T11:04:46.951947+0000","last_change":"2023-10-01T11:03:35.675790+0000","last_active":"2023-10-01T11:04:46.951947+0000","last_peered":"2023-10-01T11:04:46.951947+0000","last_clean":"2023-10-01T11:04:46.951947+0000","last_became_active":"2023-10-01T11:03:35.512279+0000","last_became_peered":"2023-10-01T11:03:35.512279+0000","last_unstale":"2023-10-01T11:04:46.951947+0000","last_undegraded":"2023-10-01T11:04:46.951947+0000","last_fullsized":"2023-10-01T11:04:46.951947+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:33:40.487487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":121,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.910551+0000","last_change":"2023-10-01T11:03:07.719806+0000","last_active":"2023-10-01T11:04:45.910551+0000","last_peered":"2023-10-01T11:04:45.910551+0000","last_clean":"2023-10-01T11:04:45.910551+0000","last_became_active":"2023-10-01T11:03:07.719100+0000","last_became_peered":"2023-10-01T11:03:07.719100+0000","last_unstale":"2023-10-01T11:04:45.910551+0000","last_undegraded":"2023-10-01T11:04:45.910551+0000","last_fullsized":"2023-10-01T11:04:45.910551+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:48:42.877973+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":390,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.907230+0000","last_change":"2023-10-01T11:04:02.737450+0000","last_active":"2023-10-01T11:04:45.907230+0000","last_peered":"2023-10-01T11:04:45.907230+0000","last_clean":"2023-10-01T11:04:45.907230+0000","last_became_active":"2023-10-01T11:04:02.737077+0000","last_became_peered":"2023-10-01T11:04:02.737077+0000","last_unstale":"2023-10-01T11:04:45.907230+0000","last_undegraded":"2023-10-01T11:04:45.907230+0000","last_fullsized":"2023-10-01T11:04:45.907230+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:51:42.820846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":314,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.907222+0000","last_change":"2023-10-01T11:04:02.735355+0000","last_active":"2023-10-01T11:04:45.907222+0000","last_peered":"2023-10-01T11:04:45.907222+0000","last_clean":"2023-10-01T11:04:45.907222+0000","last_became_active":"2023-10-01T11:04:02.734296+0000","last_became_peered":"2023-10-01T11:04:02.734296+0000","last_unstale":"2023-10-01T11:04:45.907222+0000","last_undegraded":"2023-10-01T11:04:45.907222+0000","last_fullsized":"2023-10-01T11:04:45.907222+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:45:58.384236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":54,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.907319+0000","last_change":"2023-10-01T11:04:02.735798+0000","last_active":"2023-10-01T11:04:45.907319+0000","last_peered":"2023-10-01T11:04:45.907319+0000","last_clean":"2023-10-01T11:04:45.907319+0000","last_became_active":"2023-10-01T11:04:02.735015+0000","last_became_peered":"2023-10-01T11:04:02.735015+0000","last_unstale":"2023-10-01T11:04:45.907319+0000","last_undegraded":"2023-10-01T11:04:45.907319+0000","last_fullsized":"2023-10-01T11:04:45.907319+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:10:48.623151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":340,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.907592+0000","last_change":"2023-10-01T11:03:35.511653+0000","last_active":"2023-10-01T11:04:45.907592+0000","last_peered":"2023-10-01T11:04:45.907592+0000","last_clean":"2023-10-01T11:04:45.907592+0000","last_became_active":"2023-10-01T11:03:35.511313+0000","last_became_peered":"2023-10-01T11:03:35.511313+0000","last_unstale":"2023-10-01T11:04:45.907592+0000","last_undegraded":"2023-10-01T11:04:45.907592+0000","last_fullsized":"2023-10-01T11:04:45.907592+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:04:33.959443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":301,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.908149+0000","last_change":"2023-10-01T11:04:40.877657+0000","last_active":"2023-10-01T11:04:45.908149+0000","last_peered":"2023-10-01T11:04:45.908149+0000","last_clean":"2023-10-01T11:04:45.908149+0000","last_became_active":"2023-10-01T11:04:40.877353+0000","last_became_peered":"2023-10-01T11:04:40.877353+0000","last_unstale":"2023-10-01T11:04:45.908149+0000","last_undegraded":"2023-10-01T11:04:45.908149+0000","last_fullsized":"2023-10-01T11:04:45.908149+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:33:29.944933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":322,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.916807+0000","last_change":"2023-10-01T11:03:35.511627+0000","last_active":"2023-10-01T11:04:45.916807+0000","last_peered":"2023-10-01T11:04:45.916807+0000","last_clean":"2023-10-01T11:04:45.916807+0000","last_became_active":"2023-10-01T11:03:35.511267+0000","last_became_peered":"2023-10-01T11:03:35.511267+0000","last_unstale":"2023-10-01T11:04:45.916807+0000","last_undegraded":"2023-10-01T11:04:45.916807+0000","last_fullsized":"2023-10-01T11:04:45.916807+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,0],"acting":[5,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":309,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.906722+0000","last_change":"2023-10-01T11:03:35.510184+0000","last_active":"2023-10-01T11:04:45.906722+0000","last_peered":"2023-10-01T11:04:45.906722+0000","last_clean":"2023-10-01T11:04:45.906722+0000","last_became_active":"2023-10-01T11:03:35.506856+0000","last_became_peered":"2023-10-01T11:03:35.506856+0000","last_unstale":"2023-10-01T11:04:45.906722+0000","last_undegraded":"2023-10-01T11:04:45.906722+0000","last_fullsized":"2023-10-01T11:04:45.906722+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:12:08.652555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":301,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.908221+0000","last_change":"2023-10-01T11:04:40.878331+0000","last_active":"2023-10-01T11:04:45.908221+0000","last_peered":"2023-10-01T11:04:45.908221+0000","last_clean":"2023-10-01T11:04:45.908221+0000","last_became_active":"2023-10-01T11:04:40.878006+0000","last_became_peered":"2023-10-01T11:04:40.878006+0000","last_unstale":"2023-10-01T11:04:45.908221+0000","last_undegraded":"2023-10-01T11:04:45.908221+0000","last_fullsized":"2023-10-01T11:04:45.908221+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:46:22.909785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":25,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.916690+0000","last_change":"2023-10-01T11:04:40.880122+0000","last_active":"2023-10-01T11:04:45.916690+0000","last_peered":"2023-10-01T11:04:45.916690+0000","last_clean":"2023-10-01T11:04:45.916690+0000","last_became_active":"2023-10-01T11:04:40.879833+0000","last_became_peered":"2023-10-01T11:04:40.879833+0000","last_unstale":"2023-10-01T11:04:45.916690+0000","last_undegraded":"2023-10-01T11:04:45.916690+0000","last_fullsized":"2023-10-01T11:04:45.916690+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":317,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.907878+0000","last_change":"2023-10-01T11:03:35.499805+0000","last_active":"2023-10-01T11:04:45.907878+0000","last_peered":"2023-10-01T11:04:45.907878+0000","last_clean":"2023-10-01T11:04:45.907878+0000","last_became_active":"2023-10-01T11:03:35.499525+0000","last_became_peered":"2023-10-01T11:03:35.499525+0000","last_unstale":"2023-10-01T11:04:45.907878+0000","last_undegraded":"2023-10-01T11:04:45.907878+0000","last_fullsized":"2023-10-01T11:04:45.907878+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:35:13.678732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.1","version":"320'5112","reported_seq":5170,"reported_epoch":320,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T11:04:51.195601+0000","last_change":"2023-10-01T11:03:35.752144+0000","last_active":"2023-10-01T11:04:51.195601+0000","last_peered":"2023-10-01T11:04:51.195601+0000","last_clean":"2023-10-01T11:03:06.875724+0000","last_became_active":"2023-10-01T11:03:35.735850+0000","last_became_peered":"2023-10-01T11:03:35.735850+0000","last_unstale":"2023-10-01T11:04:51.195601+0000","last_undegraded":"2023-10-01T11:03:35.538680+0000","last_fullsized":"2023-10-01T11:03:35.485357+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_clean_scrub_stamp":"2023-10-01T11:01:36.981250+0000","objects_scrubbed":0,"log_size":5112,"log_dups_size":0,"ondisk_log_size":5112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:14:57.753295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":308150272,"num_objects":4702,"num_object_clones":0,"num_object_copies":14106,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4902,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4702,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5112,"num_write_kb":314048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4","objects":-200},{"shards":"2,4,6","objects":4345},{"shards":"2,4,7","objects":352}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":202,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.908211+0000","last_change":"2023-10-01T11:03:35.600339+0000","last_active":"2023-10-01T11:04:45.908211+0000","last_peered":"2023-10-01T11:04:45.908211+0000","last_clean":"2023-10-01T11:04:45.908211+0000","last_became_active":"2023-10-01T11:03:35.505956+0000","last_became_peered":"2023-10-01T11:03:35.505956+0000","last_unstale":"2023-10-01T11:04:45.908211+0000","last_undegraded":"2023-10-01T11:04:45.908211+0000","last_fullsized":"2023-10-01T11:04:45.908211+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:30:13.286647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":343,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.916587+0000","last_change":"2023-10-01T11:03:35.497223+0000","last_active":"2023-10-01T11:04:45.916587+0000","last_peered":"2023-10-01T11:04:45.916587+0000","last_clean":"2023-10-01T11:04:45.916587+0000","last_became_active":"2023-10-01T11:03:35.496602+0000","last_became_peered":"2023-10-01T11:03:35.496602+0000","last_unstale":"2023-10-01T11:04:45.916587+0000","last_undegraded":"2023-10-01T11:04:45.916587+0000","last_fullsized":"2023-10-01T11:04:45.916587+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"320'5164","reported_seq":8117,"reported_epoch":320,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T11:04:51.925053+0000","last_change":"2023-10-01T11:03:35.728836+0000","last_active":"2023-10-01T11:04:51.925053+0000","last_peered":"2023-10-01T11:04:51.925053+0000","last_clean":"2023-10-01T11:02:35.746836+0000","last_became_active":"2023-10-01T11:03:35.718333+0000","last_became_peered":"2023-10-01T11:03:35.718333+0000","last_unstale":"2023-10-01T11:04:51.925053+0000","last_undegraded":"2023-10-01T11:03:35.545429+0000","last_fullsized":"2023-10-01T11:03:35.487437+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_clean_scrub_stamp":"2023-10-01T11:01:36.981250+0000","objects_scrubbed":0,"log_size":5164,"log_dups_size":0,"ondisk_log_size":5164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T22:28:05.518673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":310640640,"num_objects":4740,"num_object_clones":0,"num_object_copies":14220,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4947,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4740,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5164,"num_write_kb":316928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1445,"num_bytes_recovered":94699520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1],"avail_no_missing":["2","1","6"],"object_location_counts":[{"shards":"1,2","objects":-207},{"shards":"1,2,6","objects":1761},{"shards":"1,2,6,7","objects":1233},{"shards":"1,2,7","objects":1741}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":161,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.907948+0000","last_change":"2023-10-01T11:04:02.736196+0000","last_active":"2023-10-01T11:04:45.907948+0000","last_peered":"2023-10-01T11:04:45.907948+0000","last_clean":"2023-10-01T11:04:45.907948+0000","last_became_active":"2023-10-01T11:04:02.735884+0000","last_became_peered":"2023-10-01T11:04:02.735884+0000","last_unstale":"2023-10-01T11:04:45.907948+0000","last_undegraded":"2023-10-01T11:04:45.907948+0000","last_fullsized":"2023-10-01T11:04:45.907948+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:50:07.625601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":303,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.907986+0000","last_change":"2023-10-01T11:04:02.724840+0000","last_active":"2023-10-01T11:04:45.907986+0000","last_peered":"2023-10-01T11:04:45.907986+0000","last_clean":"2023-10-01T11:04:45.907986+0000","last_became_active":"2023-10-01T11:04:02.724476+0000","last_became_peered":"2023-10-01T11:04:02.724476+0000","last_unstale":"2023-10-01T11:04:45.907986+0000","last_undegraded":"2023-10-01T11:04:45.907986+0000","last_fullsized":"2023-10-01T11:04:45.907986+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:05:08.882495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.3","version":"320'5189","reported_seq":5243,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-01T11:04:54.279356+0000","last_change":"2023-10-01T11:04:40.889413+0000","last_active":"2023-10-01T11:04:54.279356+0000","last_peered":"2023-10-01T11:04:54.279356+0000","last_clean":"2023-10-01T11:04:54.279356+0000","last_became_active":"2023-10-01T11:04:40.877803+0000","last_became_peered":"2023-10-01T11:04:40.877803+0000","last_unstale":"2023-10-01T11:04:54.279356+0000","last_undegraded":"2023-10-01T11:04:54.279356+0000","last_fullsized":"2023-10-01T11:04:54.279356+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_clean_scrub_stamp":"2023-10-01T11:01:36.981250+0000","objects_scrubbed":0,"log_size":5189,"log_dups_size":0,"ondisk_log_size":5189,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T22:24:24.600680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":311623680,"num_objects":4755,"num_object_clones":0,"num_object_copies":14265,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4755,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5189,"num_write_kb":318208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":263,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.916871+0000","last_change":"2023-10-01T11:03:35.607770+0000","last_active":"2023-10-01T11:04:45.916871+0000","last_peered":"2023-10-01T11:04:45.916871+0000","last_clean":"2023-10-01T11:04:45.916871+0000","last_became_active":"2023-10-01T11:03:35.518159+0000","last_became_peered":"2023-10-01T11:03:35.518159+0000","last_unstale":"2023-10-01T11:04:45.916871+0000","last_undegraded":"2023-10-01T11:04:45.916871+0000","last_fullsized":"2023-10-01T11:04:45.916871+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":39,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.916211+0000","last_change":"2023-10-01T11:03:35.505922+0000","last_active":"2023-10-01T11:04:45.916211+0000","last_peered":"2023-10-01T11:04:45.916211+0000","last_clean":"2023-10-01T11:04:45.916211+0000","last_became_active":"2023-10-01T11:03:35.505109+0000","last_became_peered":"2023-10-01T11:03:35.505109+0000","last_unstale":"2023-10-01T11:04:45.916211+0000","last_undegraded":"2023-10-01T11:04:45.916211+0000","last_fullsized":"2023-10-01T11:04:45.916211+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.2","version":"320'5119","reported_seq":5163,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-01T11:04:53.491643+0000","last_change":"2023-10-01T11:01:38.006227+0000","last_active":"2023-10-01T11:04:53.491643+0000","last_peered":"2023-10-01T11:04:53.491643+0000","last_clean":"2023-10-01T11:04:53.491643+0000","last_became_active":"2023-10-01T11:01:38.005895+0000","last_became_peered":"2023-10-01T11:01:38.005895+0000","last_unstale":"2023-10-01T11:04:53.491643+0000","last_undegraded":"2023-10-01T11:04:53.491643+0000","last_fullsized":"2023-10-01T11:04:53.491643+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_clean_scrub_stamp":"2023-10-01T11:01:36.981250+0000","objects_scrubbed":0,"log_size":5119,"log_dups_size":0,"ondisk_log_size":5119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":308609024,"num_objects":4709,"num_object_clones":0,"num_object_copies":14127,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4709,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5119,"num_write_kb":314496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":332,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.917247+0000","last_change":"2023-10-01T11:03:07.724734+0000","last_active":"2023-10-01T11:04:45.917247+0000","last_peered":"2023-10-01T11:04:45.917247+0000","last_clean":"2023-10-01T11:04:45.917247+0000","last_became_active":"2023-10-01T11:03:07.724363+0000","last_became_peered":"2023-10-01T11:03:07.724363+0000","last_unstale":"2023-10-01T11:04:45.917247+0000","last_undegraded":"2023-10-01T11:04:45.917247+0000","last_fullsized":"2023-10-01T11:04:45.917247+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":102,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.907446+0000","last_change":"2023-10-01T11:04:40.878153+0000","last_active":"2023-10-01T11:04:45.907446+0000","last_peered":"2023-10-01T11:04:45.907446+0000","last_clean":"2023-10-01T11:04:45.907446+0000","last_became_active":"2023-10-01T11:04:40.877734+0000","last_became_peered":"2023-10-01T11:04:40.877734+0000","last_unstale":"2023-10-01T11:04:45.907446+0000","last_undegraded":"2023-10-01T11:04:45.907446+0000","last_fullsized":"2023-10-01T11:04:45.907446+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:36:14.253484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.5","version":"320'5213","reported_seq":5255,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-01T11:04:54.344682+0000","last_change":"2023-10-01T11:04:02.733362+0000","last_active":"2023-10-01T11:04:54.344682+0000","last_peered":"2023-10-01T11:04:54.344682+0000","last_clean":"2023-10-01T11:04:54.344682+0000","last_became_active":"2023-10-01T11:04:02.728059+0000","last_became_peered":"2023-10-01T11:04:02.728059+0000","last_unstale":"2023-10-01T11:04:54.344682+0000","last_undegraded":"2023-10-01T11:04:54.344682+0000","last_fullsized":"2023-10-01T11:04:54.344682+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_clean_scrub_stamp":"2023-10-01T11:01:36.981250+0000","objects_scrubbed":0,"log_size":5213,"log_dups_size":0,"ondisk_log_size":5213,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:46:23.560869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":310312960,"num_objects":4735,"num_object_clones":0,"num_object_copies":14205,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4735,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5213,"num_write_kb":318336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":294,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.908041+0000","last_change":"2023-10-01T10:38:16.022493+0000","last_active":"2023-10-01T11:04:45.908041+0000","last_peered":"2023-10-01T11:04:45.908041+0000","last_clean":"2023-10-01T11:04:45.908041+0000","last_became_active":"2023-10-01T10:38:16.022033+0000","last_became_peered":"2023-10-01T10:38:16.022033+0000","last_unstale":"2023-10-01T11:04:45.908041+0000","last_undegraded":"2023-10-01T11:04:45.908041+0000","last_fullsized":"2023-10-01T11:04:45.908041+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:49:05.262763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":324,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.907409+0000","last_change":"2023-10-01T11:04:02.724702+0000","last_active":"2023-10-01T11:04:45.907409+0000","last_peered":"2023-10-01T11:04:45.907409+0000","last_clean":"2023-10-01T11:04:45.907409+0000","last_became_active":"2023-10-01T11:04:02.724295+0000","last_became_peered":"2023-10-01T11:04:02.724295+0000","last_unstale":"2023-10-01T11:04:45.907409+0000","last_undegraded":"2023-10-01T11:04:45.907409+0000","last_fullsized":"2023-10-01T11:04:45.907409+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:10:07.356173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.4","version":"320'5148","reported_seq":6116,"reported_epoch":320,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T11:04:52.330129+0000","last_change":"2023-10-01T11:03:54.008568+0000","last_active":"2023-10-01T11:04:52.330129+0000","last_peered":"2023-10-01T11:04:52.330129+0000","last_clean":"2023-10-01T11:03:06.930658+0000","last_became_active":"2023-10-01T11:03:09.039081+0000","last_became_peered":"2023-10-01T11:03:09.039081+0000","last_unstale":"2023-10-01T11:04:52.330129+0000","last_undegraded":"2023-10-01T11:03:54.005136+0000","last_fullsized":"2023-10-01T11:03:08.832285+0000","mapping_epoch":300,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_clean_scrub_stamp":"2023-10-01T11:01:36.981250+0000","objects_scrubbed":0,"log_size":5148,"log_dups_size":0,"ondisk_log_size":5148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:53:01.537595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":309985280,"num_objects":4730,"num_object_clones":0,"num_object_copies":14190,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4120,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4730,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5148,"num_write_kb":316096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":457,"num_bytes_recovered":29949952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1],"avail_no_missing":["2","1","6"],"object_location_counts":[{"shards":"1,2","objects":610},{"shards":"1,2,6","objects":4120}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":297,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.906860+0000","last_change":"2023-10-01T11:04:02.724208+0000","last_active":"2023-10-01T11:04:45.906860+0000","last_peered":"2023-10-01T11:04:45.906860+0000","last_clean":"2023-10-01T11:04:45.906860+0000","last_became_active":"2023-10-01T11:04:02.723636+0000","last_became_peered":"2023-10-01T11:04:02.723636+0000","last_unstale":"2023-10-01T11:04:45.906860+0000","last_undegraded":"2023-10-01T11:04:45.906860+0000","last_fullsized":"2023-10-01T11:04:45.906860+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:47:22.548060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":293,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.908505+0000","last_change":"2023-10-01T11:04:02.727282+0000","last_active":"2023-10-01T11:04:45.908505+0000","last_peered":"2023-10-01T11:04:45.908505+0000","last_clean":"2023-10-01T11:04:45.908505+0000","last_became_active":"2023-10-01T11:04:02.726818+0000","last_became_peered":"2023-10-01T11:04:02.726818+0000","last_unstale":"2023-10-01T11:04:45.908505+0000","last_undegraded":"2023-10-01T11:04:45.908505+0000","last_fullsized":"2023-10-01T11:04:45.908505+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:11:11.868291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.7","version":"320'5291","reported_seq":6151,"reported_epoch":320,"state":"active+recovering+undersized+remapped","last_fresh":"2023-10-01T11:04:54.091256+0000","last_change":"2023-10-01T11:04:45.928778+0000","last_active":"2023-10-01T11:04:54.091256+0000","last_peered":"2023-10-01T11:04:54.091256+0000","last_clean":"2023-10-01T11:02:35.706450+0000","last_became_active":"2023-10-01T11:04:18.884932+0000","last_became_peered":"2023-10-01T11:04:18.884932+0000","last_unstale":"2023-10-01T11:04:54.091256+0000","last_undegraded":"2023-10-01T11:04:54.091256+0000","last_fullsized":"2023-10-01T11:04:03.727320+0000","mapping_epoch":314,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_clean_scrub_stamp":"2023-10-01T11:01:36.981250+0000","objects_scrubbed":0,"log_size":5291,"log_dups_size":0,"ondisk_log_size":5291,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":314769408,"num_objects":4803,"num_object_clones":0,"num_object_copies":14409,"num_objects_missing_on_primary":3005,"num_objects_missing":3005,"num_objects_degraded":0,"num_objects_misplaced":4786,"num_objects_unfound":0,"num_objects_dirty":4803,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5291,"num_write_kb":323008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":261,"num_bytes_recovered":17104896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[7,0],"avail_no_missing":["4","5","6"],"object_location_counts":[{"shards":"0,7","objects":17},{"shards":"4,5,6","objects":3005},{"shards":"4,5,6,7","objects":1424},{"shards":"4,5,7","objects":357}],"blocked_by":[],"up_primary":4,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":328,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.907483+0000","last_change":"2023-10-01T11:03:35.550325+0000","last_active":"2023-10-01T11:04:45.907483+0000","last_peered":"2023-10-01T11:04:45.907483+0000","last_clean":"2023-10-01T11:04:45.907483+0000","last_became_active":"2023-10-01T11:03:35.550022+0000","last_became_peered":"2023-10-01T11:03:35.550022+0000","last_unstale":"2023-10-01T11:04:45.907483+0000","last_undegraded":"2023-10-01T11:04:45.907483+0000","last_fullsized":"2023-10-01T11:04:45.907483+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:27:04.015121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":326,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.907829+0000","last_change":"2023-10-01T11:04:02.733306+0000","last_active":"2023-10-01T11:04:45.907829+0000","last_peered":"2023-10-01T11:04:45.907829+0000","last_clean":"2023-10-01T11:04:45.907829+0000","last_became_active":"2023-10-01T11:04:02.727962+0000","last_became_peered":"2023-10-01T11:04:02.727962+0000","last_unstale":"2023-10-01T11:04:45.907829+0000","last_undegraded":"2023-10-01T11:04:45.907829+0000","last_fullsized":"2023-10-01T11:04:45.907829+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:34:14.937942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.6","version":"320'5150","reported_seq":5192,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-01T11:04:54.177205+0000","last_change":"2023-10-01T11:04:02.733583+0000","last_active":"2023-10-01T11:04:54.177205+0000","last_peered":"2023-10-01T11:04:54.177205+0000","last_clean":"2023-10-01T11:04:54.177205+0000","last_became_active":"2023-10-01T11:04:02.727060+0000","last_became_peered":"2023-10-01T11:04:02.727060+0000","last_unstale":"2023-10-01T11:04:54.177205+0000","last_undegraded":"2023-10-01T11:04:54.177205+0000","last_fullsized":"2023-10-01T11:04:54.177205+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_clean_scrub_stamp":"2023-10-01T11:01:36.981250+0000","objects_scrubbed":0,"log_size":5150,"log_dups_size":0,"ondisk_log_size":5150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:01:54.121478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":307625984,"num_objects":4694,"num_object_clones":0,"num_object_copies":14082,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4694,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5150,"num_write_kb":315008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":319,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.907278+0000","last_change":"2023-10-01T11:03:35.498563+0000","last_active":"2023-10-01T11:04:45.907278+0000","last_peered":"2023-10-01T11:04:45.907278+0000","last_clean":"2023-10-01T11:04:45.907278+0000","last_became_active":"2023-10-01T11:03:35.498206+0000","last_became_peered":"2023-10-01T11:03:35.498206+0000","last_unstale":"2023-10-01T11:04:45.907278+0000","last_undegraded":"2023-10-01T11:04:45.907278+0000","last_fullsized":"2023-10-01T11:04:45.907278+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:32:53.496174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":310,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.911508+0000","last_change":"2023-10-01T10:38:16.019755+0000","last_active":"2023-10-01T11:04:45.911508+0000","last_peered":"2023-10-01T11:04:45.911508+0000","last_clean":"2023-10-01T11:04:45.911508+0000","last_became_active":"2023-10-01T10:38:16.019397+0000","last_became_peered":"2023-10-01T10:38:16.019397+0000","last_unstale":"2023-10-01T11:04:45.911508+0000","last_undegraded":"2023-10-01T11:04:45.911508+0000","last_fullsized":"2023-10-01T11:04:45.911508+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:42:56.623225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.b","version":"320'5057","reported_seq":5101,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-01T11:04:53.482688+0000","last_change":"2023-10-01T11:01:38.006450+0000","last_active":"2023-10-01T11:04:53.482688+0000","last_peered":"2023-10-01T11:04:53.482688+0000","last_clean":"2023-10-01T11:04:53.482688+0000","last_became_active":"2023-10-01T11:01:38.005600+0000","last_became_peered":"2023-10-01T11:01:38.005600+0000","last_unstale":"2023-10-01T11:04:53.482688+0000","last_undegraded":"2023-10-01T11:04:53.482688+0000","last_fullsized":"2023-10-01T11:04:53.482688+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_clean_scrub_stamp":"2023-10-01T11:01:36.981250+0000","objects_scrubbed":0,"log_size":5057,"log_dups_size":0,"ondisk_log_size":5057,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305201152,"num_objects":4657,"num_object_clones":0,"num_object_copies":13971,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4657,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5057,"num_write_kb":310848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":233,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.910809+0000","last_change":"2023-10-01T11:04:40.895708+0000","last_active":"2023-10-01T11:04:45.910809+0000","last_peered":"2023-10-01T11:04:45.910809+0000","last_clean":"2023-10-01T11:04:45.910809+0000","last_became_active":"2023-10-01T11:04:40.895211+0000","last_became_peered":"2023-10-01T11:04:40.895211+0000","last_unstale":"2023-10-01T11:04:45.910809+0000","last_undegraded":"2023-10-01T11:04:45.910809+0000","last_fullsized":"2023-10-01T11:04:45.910809+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:57:20.468016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":302,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.916747+0000","last_change":"2023-10-01T10:37:42.766814+0000","last_active":"2023-10-01T11:04:45.916747+0000","last_peered":"2023-10-01T11:04:45.916747+0000","last_clean":"2023-10-01T11:04:45.916747+0000","last_became_active":"2023-10-01T10:37:42.766451+0000","last_became_peered":"2023-10-01T10:37:42.766451+0000","last_unstale":"2023-10-01T11:04:45.916747+0000","last_undegraded":"2023-10-01T11:04:45.916747+0000","last_fullsized":"2023-10-01T11:04:45.916747+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":205,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.916207+0000","last_change":"2023-10-01T10:52:18.960606+0000","last_active":"2023-10-01T11:04:45.916207+0000","last_peered":"2023-10-01T11:04:45.916207+0000","last_clean":"2023-10-01T11:04:45.916207+0000","last_became_active":"2023-10-01T10:52:18.960270+0000","last_became_peered":"2023-10-01T10:52:18.960270+0000","last_unstale":"2023-10-01T11:04:45.916207+0000","last_undegraded":"2023-10-01T11:04:45.916207+0000","last_fullsized":"2023-10-01T11:04:45.916207+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.d","version":"320'5194","reported_seq":10623,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-01T11:04:54.395651+0000","last_change":"2023-10-01T11:03:48.736187+0000","last_active":"2023-10-01T11:04:54.395651+0000","last_peered":"2023-10-01T11:04:54.395651+0000","last_clean":"2023-10-01T11:04:54.395651+0000","last_became_active":"2023-10-01T11:03:48.731342+0000","last_became_peered":"2023-10-01T11:03:48.731342+0000","last_unstale":"2023-10-01T11:04:54.395651+0000","last_undegraded":"2023-10-01T11:04:54.395651+0000","last_fullsized":"2023-10-01T11:04:54.395651+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_clean_scrub_stamp":"2023-10-01T11:01:36.981250+0000","objects_scrubbed":0,"log_size":5194,"log_dups_size":0,"ondisk_log_size":5194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:33:29.188691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":310181912,"num_objects":4734,"num_object_clones":0,"num_object_copies":14202,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4734,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5194,"num_write_kb":317633,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2677,"num_bytes_recovered":175439872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":336,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.916331+0000","last_change":"2023-10-01T11:03:07.726174+0000","last_active":"2023-10-01T11:04:45.916331+0000","last_peered":"2023-10-01T11:04:45.916331+0000","last_clean":"2023-10-01T11:04:45.916331+0000","last_became_active":"2023-10-01T11:03:07.725779+0000","last_became_peered":"2023-10-01T11:03:07.725779+0000","last_unstale":"2023-10-01T11:04:45.916331+0000","last_undegraded":"2023-10-01T11:04:45.916331+0000","last_fullsized":"2023-10-01T11:04:45.916331+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":292,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.908022+0000","last_change":"2023-10-01T11:04:02.735697+0000","last_active":"2023-10-01T11:04:45.908022+0000","last_peered":"2023-10-01T11:04:45.908022+0000","last_clean":"2023-10-01T11:04:45.908022+0000","last_became_active":"2023-10-01T11:04:02.735307+0000","last_became_peered":"2023-10-01T11:04:02.735307+0000","last_unstale":"2023-10-01T11:04:45.908022+0000","last_undegraded":"2023-10-01T11:04:45.908022+0000","last_fullsized":"2023-10-01T11:04:45.908022+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:30:18.731530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":221,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-01T11:04:46.923822+0000","last_change":"2023-10-01T10:52:18.963437+0000","last_active":"2023-10-01T11:04:46.923822+0000","last_peered":"2023-10-01T11:04:46.923822+0000","last_clean":"2023-10-01T11:04:46.923822+0000","last_became_active":"2023-10-01T10:52:18.955165+0000","last_became_peered":"2023-10-01T10:52:18.955165+0000","last_unstale":"2023-10-01T11:04:46.923822+0000","last_undegraded":"2023-10-01T11:04:46.923822+0000","last_fullsized":"2023-10-01T11:04:46.923822+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:57:49.129100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.e","version":"320'5177","reported_seq":8997,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-01T11:04:53.246116+0000","last_change":"2023-10-01T11:03:50.779694+0000","last_active":"2023-10-01T11:04:53.246116+0000","last_peered":"2023-10-01T11:04:53.246116+0000","last_clean":"2023-10-01T11:04:53.246116+0000","last_became_active":"2023-10-01T11:03:50.775886+0000","last_became_peered":"2023-10-01T11:03:50.775886+0000","last_unstale":"2023-10-01T11:04:53.246116+0000","last_undegraded":"2023-10-01T11:04:53.246116+0000","last_fullsized":"2023-10-01T11:04:53.246116+0000","mapping_epoch":309,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":310,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_clean_scrub_stamp":"2023-10-01T11:01:36.981250+0000","objects_scrubbed":0,"log_size":5177,"log_dups_size":0,"ondisk_log_size":5177,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":311361536,"num_objects":4751,"num_object_clones":0,"num_object_copies":14253,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4751,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5177,"num_write_kb":317696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1873,"num_bytes_recovered":122748928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":302,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.910691+0000","last_change":"2023-10-01T10:37:42.762524+0000","last_active":"2023-10-01T11:04:45.910691+0000","last_peered":"2023-10-01T11:04:45.910691+0000","last_clean":"2023-10-01T11:04:45.910691+0000","last_became_active":"2023-10-01T10:37:42.762209+0000","last_became_peered":"2023-10-01T10:37:42.762209+0000","last_unstale":"2023-10-01T11:04:45.910691+0000","last_undegraded":"2023-10-01T11:04:45.910691+0000","last_fullsized":"2023-10-01T11:04:45.910691+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:45:38.171397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":246,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.912384+0000","last_change":"2023-10-01T11:03:07.722828+0000","last_active":"2023-10-01T11:04:45.912384+0000","last_peered":"2023-10-01T11:04:45.912384+0000","last_clean":"2023-10-01T11:04:45.912384+0000","last_became_active":"2023-10-01T11:03:07.722056+0000","last_became_peered":"2023-10-01T11:03:07.722056+0000","last_unstale":"2023-10-01T11:04:45.912384+0000","last_undegraded":"2023-10-01T11:04:45.912384+0000","last_fullsized":"2023-10-01T11:04:45.912384+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:08:14.758405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":207,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.917257+0000","last_change":"2023-10-01T10:52:18.961040+0000","last_active":"2023-10-01T11:04:45.917257+0000","last_peered":"2023-10-01T11:04:45.917257+0000","last_clean":"2023-10-01T11:04:45.917257+0000","last_became_active":"2023-10-01T10:52:18.960606+0000","last_became_peered":"2023-10-01T10:52:18.960606+0000","last_unstale":"2023-10-01T11:04:45.917257+0000","last_undegraded":"2023-10-01T11:04:45.917257+0000","last_fullsized":"2023-10-01T11:04:45.917257+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.c","version":"320'5080","reported_seq":5124,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-01T11:04:52.948053+0000","last_change":"2023-10-01T11:01:38.196508+0000","last_active":"2023-10-01T11:04:52.948053+0000","last_peered":"2023-10-01T11:04:52.948053+0000","last_clean":"2023-10-01T11:04:52.948053+0000","last_became_active":"2023-10-01T11:01:38.195817+0000","last_became_peered":"2023-10-01T11:01:38.195817+0000","last_unstale":"2023-10-01T11:04:52.948053+0000","last_undegraded":"2023-10-01T11:04:52.948053+0000","last_fullsized":"2023-10-01T11:04:52.948053+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_clean_scrub_stamp":"2023-10-01T11:01:36.981250+0000","objects_scrubbed":0,"log_size":5080,"log_dups_size":0,"ondisk_log_size":5080,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":306053120,"num_objects":4670,"num_object_clones":0,"num_object_copies":14010,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4670,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5080,"num_write_kb":312000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":318,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.916549+0000","last_change":"2023-10-01T11:03:35.502310+0000","last_active":"2023-10-01T11:04:45.916549+0000","last_peered":"2023-10-01T11:04:45.916549+0000","last_clean":"2023-10-01T11:04:45.916549+0000","last_became_active":"2023-10-01T11:03:35.501857+0000","last_became_peered":"2023-10-01T11:03:35.501857+0000","last_unstale":"2023-10-01T11:04:45.916549+0000","last_undegraded":"2023-10-01T11:04:45.916549+0000","last_fullsized":"2023-10-01T11:04:45.916549+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'139","reported_seq":562,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.910353+0000","last_change":"2023-10-01T11:04:40.885637+0000","last_active":"2023-10-01T11:04:45.910353+0000","last_peered":"2023-10-01T11:04:45.910353+0000","last_clean":"2023-10-01T11:04:45.910353+0000","last_became_active":"2023-10-01T11:04:40.885228+0000","last_became_peered":"2023-10-01T11:04:40.885228+0000","last_unstale":"2023-10-01T11:04:45.910353+0000","last_undegraded":"2023-10-01T11:04:45.910353+0000","last_fullsized":"2023-10-01T11:04:45.910353+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:09:17.221082+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:09:17.221082+0000","last_clean_scrub_stamp":"2023-10-01T10:09:17.221082+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:49:55.351411+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":320,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-01T11:04:46.953114+0000","last_change":"2023-10-01T11:03:07.725333+0000","last_active":"2023-10-01T11:04:46.953114+0000","last_peered":"2023-10-01T11:04:46.953114+0000","last_clean":"2023-10-01T11:04:46.953114+0000","last_became_active":"2023-10-01T11:03:07.724926+0000","last_became_peered":"2023-10-01T11:03:07.724926+0000","last_unstale":"2023-10-01T11:04:46.953114+0000","last_undegraded":"2023-10-01T11:04:46.953114+0000","last_fullsized":"2023-10-01T11:04:46.953114+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:10:48.812969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":333,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.908320+0000","last_change":"2023-10-01T11:03:35.508853+0000","last_active":"2023-10-01T11:04:45.908320+0000","last_peered":"2023-10-01T11:04:45.908320+0000","last_clean":"2023-10-01T11:04:45.908320+0000","last_became_active":"2023-10-01T11:03:35.508479+0000","last_became_peered":"2023-10-01T11:03:35.508479+0000","last_unstale":"2023-10-01T11:04:45.908320+0000","last_undegraded":"2023-10-01T11:04:45.908320+0000","last_fullsized":"2023-10-01T11:04:45.908320+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:16:14.758291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.f","version":"320'5276","reported_seq":5420,"reported_epoch":320,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T11:04:51.351625+0000","last_change":"2023-10-01T11:03:36.714158+0000","last_active":"2023-10-01T11:04:51.351625+0000","last_peered":"2023-10-01T11:04:51.351625+0000","last_clean":"2023-10-01T11:02:35.691576+0000","last_became_active":"2023-10-01T11:03:36.707736+0000","last_became_peered":"2023-10-01T11:03:36.707736+0000","last_unstale":"2023-10-01T11:04:51.351625+0000","last_undegraded":"2023-10-01T11:03:36.627177+0000","last_fullsized":"2023-10-01T11:03:36.510552+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_clean_scrub_stamp":"2023-10-01T11:01:36.981250+0000","objects_scrubbed":0,"log_size":5276,"log_dups_size":0,"ondisk_log_size":5276,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T22:22:33.799109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":322306048,"num_objects":4918,"num_object_clones":0,"num_object_copies":14754,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6609,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4918,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5276,"num_write_kb":326208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":2424832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"2,5","objects":-174},{"shards":"2,5,6","objects":3066},{"shards":"2,5,6,7","objects":2},{"shards":"2,5,7","objects":328},{"shards":"2,6,7","objects":1517}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":248,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.907080+0000","last_change":"2023-10-01T11:04:40.877348+0000","last_active":"2023-10-01T11:04:45.907080+0000","last_peered":"2023-10-01T11:04:45.907080+0000","last_clean":"2023-10-01T11:04:45.907080+0000","last_became_active":"2023-10-01T11:04:40.877017+0000","last_became_peered":"2023-10-01T11:04:40.877017+0000","last_unstale":"2023-10-01T11:04:45.907080+0000","last_undegraded":"2023-10-01T11:04:45.907080+0000","last_fullsized":"2023-10-01T11:04:45.907080+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:52:37.965623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":101,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.907302+0000","last_change":"2023-10-01T11:04:40.868156+0000","last_active":"2023-10-01T11:04:45.907302+0000","last_peered":"2023-10-01T11:04:45.907302+0000","last_clean":"2023-10-01T11:04:45.907302+0000","last_became_active":"2023-10-01T11:04:40.867864+0000","last_became_peered":"2023-10-01T11:04:40.867864+0000","last_unstale":"2023-10-01T11:04:45.907302+0000","last_undegraded":"2023-10-01T11:04:45.907302+0000","last_fullsized":"2023-10-01T11:04:45.907302+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:37:15.571777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":92,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.911269+0000","last_change":"2023-10-01T11:04:40.880090+0000","last_active":"2023-10-01T11:04:45.911269+0000","last_peered":"2023-10-01T11:04:45.911269+0000","last_clean":"2023-10-01T11:04:45.911269+0000","last_became_active":"2023-10-01T11:04:40.879768+0000","last_became_peered":"2023-10-01T11:04:40.879768+0000","last_unstale":"2023-10-01T11:04:45.911269+0000","last_undegraded":"2023-10-01T11:04:45.911269+0000","last_fullsized":"2023-10-01T11:04:45.911269+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:25:31.189338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.a","version":"320'5351","reported_seq":5397,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-01T11:04:52.322113+0000","last_change":"2023-10-01T11:01:38.193535+0000","last_active":"2023-10-01T11:04:52.322113+0000","last_peered":"2023-10-01T11:04:52.322113+0000","last_clean":"2023-10-01T11:04:52.322113+0000","last_became_active":"2023-10-01T11:01:38.193069+0000","last_became_peered":"2023-10-01T11:01:38.193069+0000","last_unstale":"2023-10-01T11:04:52.322113+0000","last_undegraded":"2023-10-01T11:04:52.322113+0000","last_fullsized":"2023-10-01T11:04:52.322113+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_clean_scrub_stamp":"2023-10-01T11:01:36.981250+0000","objects_scrubbed":0,"log_size":5351,"log_dups_size":0,"ondisk_log_size":5351,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:27:23.966398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":324206592,"num_objects":4947,"num_object_clones":0,"num_object_copies":14841,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4947,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5351,"num_write_kb":329536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":143,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.907780+0000","last_change":"2023-10-01T11:04:40.878192+0000","last_active":"2023-10-01T11:04:45.907780+0000","last_peered":"2023-10-01T11:04:45.907780+0000","last_clean":"2023-10-01T11:04:45.907780+0000","last_became_active":"2023-10-01T11:04:40.877838+0000","last_became_peered":"2023-10-01T11:04:40.877838+0000","last_unstale":"2023-10-01T11:04:45.907780+0000","last_undegraded":"2023-10-01T11:04:45.907780+0000","last_fullsized":"2023-10-01T11:04:45.907780+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:11:32.873620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":201,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-01T11:04:46.924002+0000","last_change":"2023-10-01T10:52:18.964479+0000","last_active":"2023-10-01T11:04:46.924002+0000","last_peered":"2023-10-01T11:04:46.924002+0000","last_clean":"2023-10-01T11:04:46.924002+0000","last_became_active":"2023-10-01T10:52:18.963881+0000","last_became_peered":"2023-10-01T10:52:18.963881+0000","last_unstale":"2023-10-01T11:04:46.924002+0000","last_undegraded":"2023-10-01T11:04:46.924002+0000","last_fullsized":"2023-10-01T11:04:46.924002+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:36:30.405755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":288,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.916639+0000","last_change":"2023-10-01T11:04:40.878262+0000","last_active":"2023-10-01T11:04:45.916639+0000","last_peered":"2023-10-01T11:04:45.916639+0000","last_clean":"2023-10-01T11:04:45.916639+0000","last_became_active":"2023-10-01T11:04:40.877922+0000","last_became_peered":"2023-10-01T11:04:40.877922+0000","last_unstale":"2023-10-01T11:04:45.916639+0000","last_undegraded":"2023-10-01T11:04:45.916639+0000","last_fullsized":"2023-10-01T11:04:45.916639+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.9","version":"320'5206","reported_seq":5250,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-01T11:04:53.456406+0000","last_change":"2023-10-01T11:01:38.007037+0000","last_active":"2023-10-01T11:04:53.456406+0000","last_peered":"2023-10-01T11:04:53.456406+0000","last_clean":"2023-10-01T11:04:53.456406+0000","last_became_active":"2023-10-01T11:01:38.006590+0000","last_became_peered":"2023-10-01T11:01:38.006590+0000","last_unstale":"2023-10-01T11:04:53.456406+0000","last_undegraded":"2023-10-01T11:04:53.456406+0000","last_fullsized":"2023-10-01T11:04:53.456406+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_clean_scrub_stamp":"2023-10-01T11:01:36.981250+0000","objects_scrubbed":0,"log_size":5206,"log_dups_size":0,"ondisk_log_size":5206,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":312344576,"num_objects":4766,"num_object_clones":0,"num_object_copies":14298,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4766,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5206,"num_write_kb":319104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":318,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.908006+0000","last_change":"2023-10-01T11:03:35.637892+0000","last_active":"2023-10-01T11:04:45.908006+0000","last_peered":"2023-10-01T11:04:45.908006+0000","last_clean":"2023-10-01T11:04:45.908006+0000","last_became_active":"2023-10-01T11:03:35.506080+0000","last_became_peered":"2023-10-01T11:03:35.506080+0000","last_unstale":"2023-10-01T11:04:45.908006+0000","last_undegraded":"2023-10-01T11:04:45.908006+0000","last_fullsized":"2023-10-01T11:04:45.908006+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T22:15:33.720146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":272,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.907614+0000","last_change":"2023-10-01T11:04:02.725949+0000","last_active":"2023-10-01T11:04:45.907614+0000","last_peered":"2023-10-01T11:04:45.907614+0000","last_clean":"2023-10-01T11:04:45.907614+0000","last_became_active":"2023-10-01T11:04:02.725266+0000","last_became_peered":"2023-10-01T11:04:02.725266+0000","last_unstale":"2023-10-01T11:04:45.907614+0000","last_undegraded":"2023-10-01T11:04:45.907614+0000","last_fullsized":"2023-10-01T11:04:45.907614+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:20:41.550613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":304,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.907177+0000","last_change":"2023-10-01T11:03:35.506332+0000","last_active":"2023-10-01T11:04:45.907177+0000","last_peered":"2023-10-01T11:04:45.907177+0000","last_clean":"2023-10-01T11:04:45.907177+0000","last_became_active":"2023-10-01T11:03:35.505525+0000","last_became_peered":"2023-10-01T11:03:35.505525+0000","last_unstale":"2023-10-01T11:04:45.907177+0000","last_undegraded":"2023-10-01T11:04:45.907177+0000","last_fullsized":"2023-10-01T11:04:45.907177+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:01:54.155187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.8","version":"320'5244","reported_seq":14664,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-01T11:04:52.291974+0000","last_change":"2023-10-01T11:04:46.935903+0000","last_active":"2023-10-01T11:04:52.291974+0000","last_peered":"2023-10-01T11:04:52.291974+0000","last_clean":"2023-10-01T11:04:52.291974+0000","last_became_active":"2023-10-01T11:04:46.927982+0000","last_became_peered":"2023-10-01T11:04:46.927982+0000","last_unstale":"2023-10-01T11:04:52.291974+0000","last_undegraded":"2023-10-01T11:04:52.291974+0000","last_fullsized":"2023-10-01T11:04:52.291974+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_clean_scrub_stamp":"2023-10-01T11:01:36.981250+0000","objects_scrubbed":0,"log_size":5244,"log_dups_size":0,"ondisk_log_size":5244,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:56:11.646367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":316538880,"num_objects":4830,"num_object_clones":0,"num_object_copies":14490,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4830,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5244,"num_write_kb":322368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4676,"num_bytes_recovered":306446336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":161,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.911650+0000","last_change":"2023-10-01T11:03:35.512100+0000","last_active":"2023-10-01T11:04:45.911650+0000","last_peered":"2023-10-01T11:04:45.911650+0000","last_clean":"2023-10-01T11:04:45.911650+0000","last_became_active":"2023-10-01T11:03:35.510692+0000","last_became_peered":"2023-10-01T11:03:35.510692+0000","last_unstale":"2023-10-01T11:04:45.911650+0000","last_undegraded":"2023-10-01T11:04:45.911650+0000","last_fullsized":"2023-10-01T11:04:45.911650+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:16:09.577268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":25,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.907519+0000","last_change":"2023-10-01T11:04:40.878201+0000","last_active":"2023-10-01T11:04:45.907519+0000","last_peered":"2023-10-01T11:04:45.907519+0000","last_clean":"2023-10-01T11:04:45.907519+0000","last_became_active":"2023-10-01T11:04:40.877847+0000","last_became_peered":"2023-10-01T11:04:40.877847+0000","last_unstale":"2023-10-01T11:04:45.907519+0000","last_undegraded":"2023-10-01T11:04:45.907519+0000","last_fullsized":"2023-10-01T11:04:45.907519+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:14:30.388066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":290,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.906943+0000","last_change":"2023-10-01T11:04:02.733599+0000","last_active":"2023-10-01T11:04:45.906943+0000","last_peered":"2023-10-01T11:04:45.906943+0000","last_clean":"2023-10-01T11:04:45.906943+0000","last_became_active":"2023-10-01T11:04:02.733210+0000","last_became_peered":"2023-10-01T11:04:02.733210+0000","last_unstale":"2023-10-01T11:04:45.906943+0000","last_undegraded":"2023-10-01T11:04:45.906943+0000","last_fullsized":"2023-10-01T11:04:45.906943+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:35:15.233575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":23,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.907907+0000","last_change":"2023-10-01T11:04:40.875312+0000","last_active":"2023-10-01T11:04:45.907907+0000","last_peered":"2023-10-01T11:04:45.907907+0000","last_clean":"2023-10-01T11:04:45.907907+0000","last_became_active":"2023-10-01T11:04:40.874972+0000","last_became_peered":"2023-10-01T11:04:40.874972+0000","last_unstale":"2023-10-01T11:04:45.907907+0000","last_undegraded":"2023-10-01T11:04:45.907907+0000","last_fullsized":"2023-10-01T11:04:45.907907+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:18:55.309174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":310,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.911296+0000","last_change":"2023-10-01T10:37:42.762360+0000","last_active":"2023-10-01T11:04:45.911296+0000","last_peered":"2023-10-01T11:04:45.911296+0000","last_clean":"2023-10-01T11:04:45.911296+0000","last_became_active":"2023-10-01T10:37:42.761936+0000","last_became_peered":"2023-10-01T10:37:42.761936+0000","last_unstale":"2023-10-01T11:04:45.911296+0000","last_undegraded":"2023-10-01T11:04:45.911296+0000","last_fullsized":"2023-10-01T11:04:45.911296+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:50.637582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":304,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.917308+0000","last_change":"2023-10-01T10:38:16.037809+0000","last_active":"2023-10-01T11:04:45.917308+0000","last_peered":"2023-10-01T11:04:45.917308+0000","last_clean":"2023-10-01T11:04:45.917308+0000","last_became_active":"2023-10-01T10:38:16.037318+0000","last_became_peered":"2023-10-01T10:38:16.037318+0000","last_unstale":"2023-10-01T11:04:45.917308+0000","last_undegraded":"2023-10-01T11:04:45.917308+0000","last_fullsized":"2023-10-01T11:04:45.917308+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":302,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.907829+0000","last_change":"2023-10-01T11:04:02.726007+0000","last_active":"2023-10-01T11:04:45.907829+0000","last_peered":"2023-10-01T11:04:45.907829+0000","last_clean":"2023-10-01T11:04:45.907829+0000","last_became_active":"2023-10-01T11:04:02.725507+0000","last_became_peered":"2023-10-01T11:04:02.725507+0000","last_unstale":"2023-10-01T11:04:45.907829+0000","last_undegraded":"2023-10-01T11:04:45.907829+0000","last_fullsized":"2023-10-01T11:04:45.907829+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:32:04.611533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":354,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.916607+0000","last_change":"2023-10-01T11:03:07.908857+0000","last_active":"2023-10-01T11:04:45.916607+0000","last_peered":"2023-10-01T11:04:45.916607+0000","last_clean":"2023-10-01T11:04:45.916607+0000","last_became_active":"2023-10-01T11:03:07.729730+0000","last_became_peered":"2023-10-01T11:03:07.729730+0000","last_unstale":"2023-10-01T11:04:45.916607+0000","last_undegraded":"2023-10-01T11:04:45.916607+0000","last_fullsized":"2023-10-01T11:04:45.916607+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":302,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.907037+0000","last_change":"2023-10-01T11:04:02.733653+0000","last_active":"2023-10-01T11:04:45.907037+0000","last_peered":"2023-10-01T11:04:45.907037+0000","last_clean":"2023-10-01T11:04:45.907037+0000","last_became_active":"2023-10-01T11:04:02.733178+0000","last_became_peered":"2023-10-01T11:04:02.733178+0000","last_unstale":"2023-10-01T11:04:45.907037+0000","last_undegraded":"2023-10-01T11:04:45.907037+0000","last_fullsized":"2023-10-01T11:04:45.907037+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:18:25.249624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":353,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.908344+0000","last_change":"2023-10-01T11:03:07.718308+0000","last_active":"2023-10-01T11:04:45.908344+0000","last_peered":"2023-10-01T11:04:45.908344+0000","last_clean":"2023-10-01T11:04:45.908344+0000","last_became_active":"2023-10-01T11:03:07.717793+0000","last_became_peered":"2023-10-01T11:03:07.717793+0000","last_unstale":"2023-10-01T11:04:45.908344+0000","last_undegraded":"2023-10-01T11:04:45.908344+0000","last_fullsized":"2023-10-01T11:04:45.908344+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:14:27.587917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":319,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.907420+0000","last_change":"2023-10-01T10:37:42.765142+0000","last_active":"2023-10-01T11:04:45.907420+0000","last_peered":"2023-10-01T11:04:45.907420+0000","last_clean":"2023-10-01T11:04:45.907420+0000","last_became_active":"2023-10-01T10:37:42.764770+0000","last_became_peered":"2023-10-01T10:37:42.764770+0000","last_unstale":"2023-10-01T11:04:45.907420+0000","last_undegraded":"2023-10-01T11:04:45.907420+0000","last_fullsized":"2023-10-01T11:04:45.907420+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:23:33.527876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":34,"reported_epoch":319,"state":"active+clean","last_fresh":"2023-10-01T11:04:45.917159+0000","last_change":"2023-10-01T11:04:40.889388+0000","last_active":"2023-10-01T11:04:45.917159+0000","last_peered":"2023-10-01T11:04:45.917159+0000","last_clean":"2023-10-01T11:04:45.917159+0000","last_became_active":"2023-10-01T11:04:40.888960+0000","last_became_peered":"2023-10-01T11:04:40.888960+0000","last_unstale":"2023-10-01T11:04:45.917159+0000","last_undegraded":"2023-10-01T11:04:45.917159+0000","last_fullsized":"2023-10-01T11:04:45.917159+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":329,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-10-01T11:04:46.945990+0000","last_change":"2023-10-01T11:03:07.710653+0000","last_active":"2023-10-01T11:04:46.945990+0000","last_peered":"2023-10-01T11:04:46.945990+0000","last_clean":"2023-10-01T11:04:46.945990+0000","last_became_active":"2023-10-01T11:03:07.710311+0000","last_became_peered":"2023-10-01T11:03:07.710311+0000","last_unstale":"2023-10-01T11:04:46.945990+0000","last_undegraded":"2023-10-01T11:04:46.945990+0000","last_fullsized":"2023-10-01T11:04:46.945990+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:26:33.319883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":16,"stat_sum":{"num_bytes":4989911064,"num_objects":76141,"num_object_clones":0,"num_object_copies":228423,"num_objects_missing_on_primary":3005,"num_objects_missing":3005,"num_objects_degraded":20578,"num_objects_misplaced":4786,"num_objects_unfound":0,"num_objects_dirty":76141,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":82971,"num_write_kb":5091521,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11426,"num_bytes_recovered":748814336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15581982720,"data_stored":15581970504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":82971,"ondisk_log_size":82971,"up":48,"acting":43,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":24600,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30319,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3542208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":206158430830,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":0,"seq":180388627069,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":101,"seq":433791697287,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2587968,"kb_used_data":1832824,"kb_used_omap":4,"kb_used_meta":755131,"kb_avail":91157184,"statfs":{"total":95995035648,"available":93344956416,"internally_reserved":0,"allocated":1876811776,"data_stored":1875531532,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":773255054},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 11:04:44 2023","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":1.028,"15min":1.256},"min":{"1min":0.28599999999999998,"5min":0.25600000000000001,"15min":0.19900000000000001},"max":{"1min":3.1880000000000002,"5min":7.742,"15min":12.888999999999999},"last":0.73099999999999998},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.97399999999999998,"15min":1.29},"min":{"1min":0.26200000000000001,"5min":0.23400000000000001,"15min":0.192},"max":{"1min":2.9980000000000002,"5min":7.1150000000000002,"15min":15.416},"last":0.69699999999999995}]},{"osd":2,"last update":"Sun Oct 1 11:04:03 2023","interfaces":[{"interface":"back","average":{"1min":2.871,"5min":2.1739999999999999,"15min":2.5819999999999999},"min":{"1min":0.33500000000000002,"5min":0.23200000000000001,"15min":0.222},"max":{"1min":41.026000000000003,"5min":41.026000000000003,"15min":161.905},"last":0.95099999999999996},{"interface":"front","average":{"1min":2.9319999999999999,"5min":2.4550000000000001,"15min":2.677},"min":{"1min":0.40200000000000002,"5min":0.247,"15min":0.247},"max":{"1min":41.148000000000003,"5min":41.148000000000003,"15min":162.059},"last":0.88400000000000001}]},{"osd":3,"last update":"Sun Oct 1 11:04:44 2023","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":0.90100000000000002,"15min":1.8779999999999999},"min":{"1min":0.32400000000000001,"5min":0.248,"15min":0.23999999999999999},"max":{"1min":3.2309999999999999,"5min":5.3739999999999997,"15min":143.73400000000001},"last":1.0149999999999999},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.996,"15min":1.9379999999999999},"min":{"1min":0.34499999999999997,"5min":0.223,"15min":0.223},"max":{"1min":0.97899999999999998,"5min":9.1219999999999999,"15min":143.00399999999999},"last":0.85399999999999998}]},{"osd":4,"last update":"Sun Oct 1 11:04:03 2023","interfaces":[{"interface":"back","average":{"1min":0.70499999999999996,"5min":1.038,"15min":1.2290000000000001},"min":{"1min":0.36299999999999999,"5min":0.32600000000000001,"15min":0.307},"max":{"1min":1.2689999999999999,"5min":5.3609999999999998,"15min":8.593},"last":0.90800000000000003},{"interface":"front","average":{"1min":0.92800000000000005,"5min":1.2010000000000001,"15min":1.347},"min":{"1min":0.39800000000000002,"5min":0.34699999999999998,"15min":0.32700000000000001},"max":{"1min":2.3929999999999998,"5min":6.5010000000000003,"15min":10.816000000000001},"last":0.98199999999999998}]},{"osd":5,"last update":"Sun Oct 1 11:04:44 2023","interfaces":[{"interface":"back","average":{"1min":0.78800000000000003,"5min":1.101,"15min":1.3300000000000001},"min":{"1min":0.48499999999999999,"5min":0.39000000000000001,"15min":0.35299999999999998},"max":{"1min":1.3129999999999999,"5min":9.048,"15min":9.1809999999999992},"last":1.3560000000000001},{"interface":"front","average":{"1min":1.204,"5min":1.1830000000000001,"15min":1.355},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.34200000000000003},"max":{"1min":9.8100000000000005,"5min":9.8100000000000005,"15min":9.8100000000000005},"last":1.0980000000000001}]},{"osd":6,"last update":"Sun Oct 1 11:04:44 2023","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":1.141,"15min":1.5149999999999999},"min":{"1min":0.51400000000000001,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":3.3159999999999998,"5min":5.2519999999999998,"15min":8.8309999999999995},"last":1.1619999999999999},{"interface":"front","average":{"1min":0.98499999999999999,"5min":1.212,"15min":1.431},"min":{"1min":0.46800000000000003,"5min":0.34699999999999998,"15min":0.33200000000000002},"max":{"1min":4.1029999999999998,"5min":6.5289999999999999,"15min":9.7870000000000008},"last":1.0409999999999999}]},{"osd":7,"last update":"Sun Oct 1 11:04:44 2023","interfaces":[{"interface":"back","average":{"1min":0.875,"5min":1.1499999999999999,"15min":1.383},"min":{"1min":0.59699999999999998,"5min":0.38,"15min":0.36399999999999999},"max":{"1min":1.304,"5min":6.5519999999999996,"15min":10.465},"last":1.131},{"interface":"front","average":{"1min":0.81899999999999995,"5min":1.1830000000000001,"15min":1.486},"min":{"1min":0.58599999999999997,"5min":0.438,"15min":0.39100000000000001},"max":{"1min":1.4119999999999999,"5min":9.0890000000000004,"15min":11.689},"last":1.2490000000000001}]}]},{"osd":0,"up_from":86,"seq":369367187897,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2103572,"kb_used_data":1223948,"kb_used_omap":5,"kb_used_meta":879610,"kb_avail":91641580,"statfs":{"total":95995035648,"available":93840977920,"internally_reserved":0,"allocated":1253322752,"data_stored":1252083744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":900721274},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 11:04:43 2023","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":2.0459999999999998,"15min":1.5960000000000001},"min":{"1min":0.29399999999999998,"5min":0.222,"15min":0.19700000000000001},"max":{"1min":0.79500000000000004,"5min":119.911,"15min":119.911},"last":1.851},{"interface":"front","average":{"1min":0.49199999999999999,"5min":2.238,"15min":1.7829999999999999},"min":{"1min":0.23799999999999999,"5min":0.20300000000000001,"15min":0.19900000000000001},"max":{"1min":0.93200000000000005,"5min":119.76300000000001,"15min":119.76300000000001},"last":1.0800000000000001}]},{"osd":2,"last update":"Sun Oct 1 11:03:55 2023","interfaces":[{"interface":"back","average":{"1min":1.7090000000000001,"5min":2.4180000000000001,"15min":2.2999999999999998},"min":{"1min":0.35199999999999998,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":29.199999999999999,"5min":72.064999999999998,"15min":72.064999999999998},"last":0.97299999999999998},{"interface":"front","average":{"1min":1.881,"5min":2.5910000000000002,"15min":2.4569999999999999},"min":{"1min":0.36399999999999999,"5min":0.25700000000000001,"15min":0.252},"max":{"1min":31.081,"5min":76.328999999999994,"15min":76.328999999999994},"last":1.647}]},{"osd":3,"last update":"Sun Oct 1 11:04:43 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":2.4209999999999998,"15min":1.96},"min":{"1min":0.32100000000000001,"5min":0.24199999999999999,"15min":0.214},"max":{"1min":1.3680000000000001,"5min":81.674999999999997,"15min":81.674999999999997},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.59999999999999998,"5min":2.2010000000000001,"15min":1.7909999999999999},"min":{"1min":0.32200000000000001,"5min":0.308,"15min":0.246},"max":{"1min":0.97299999999999998,"5min":70.950999999999993,"15min":70.950999999999993},"last":1.177}]},{"osd":4,"last update":"Sun Oct 1 11:04:18 2023","interfaces":[{"interface":"back","average":{"1min":0.81799999999999995,"5min":1.448,"15min":1.4339999999999999},"min":{"1min":0.42299999999999999,"5min":0.379,"15min":0.32000000000000001},"max":{"1min":1.6299999999999999,"5min":9.3320000000000007,"15min":9.3580000000000005},"last":1.0109999999999999},{"interface":"front","average":{"1min":0.73899999999999999,"5min":1.5900000000000001,"15min":1.5049999999999999},"min":{"1min":0.40300000000000002,"5min":0.33500000000000002,"15min":0.311},"max":{"1min":1.2789999999999999,"5min":8.0530000000000008,"15min":9.4009999999999998},"last":1.28}]},{"osd":5,"last update":"Sun Oct 1 11:04:43 2023","interfaces":[{"interface":"back","average":{"1min":0.82899999999999996,"5min":1.4890000000000001,"15min":1.5840000000000001},"min":{"1min":0.57899999999999996,"5min":0.309,"15min":0.309},"max":{"1min":1.4370000000000001,"5min":39.860999999999997,"15min":39.860999999999997},"last":0.92700000000000005},{"interface":"front","average":{"1min":0.85799999999999998,"5min":1.673,"15min":1.4670000000000001},"min":{"1min":0.51900000000000002,"5min":0.39300000000000002,"15min":0.35699999999999998},"max":{"1min":1.5529999999999999,"5min":39.646000000000001,"15min":39.646000000000001},"last":1.0549999999999999}]},{"osd":6,"last update":"Sun Oct 1 11:04:43 2023","interfaces":[{"interface":"back","average":{"1min":0.81599999999999995,"5min":1.2949999999999999,"15min":1.448},"min":{"1min":0.40899999999999997,"5min":0.39500000000000002,"15min":0.373},"max":{"1min":1.585,"5min":8.6379999999999999,"15min":9.4570000000000007},"last":1.143},{"interface":"front","average":{"1min":0.81599999999999995,"5min":1.383,"15min":1.5509999999999999},"min":{"1min":0.47599999999999998,"5min":0.45400000000000001,"15min":0.37},"max":{"1min":1.0760000000000001,"5min":11.032999999999999,"15min":11.032999999999999},"last":1.4179999999999999}]},{"osd":7,"last update":"Sun Oct 1 11:04:43 2023","interfaces":[{"interface":"back","average":{"1min":0.83599999999999997,"5min":1.3140000000000001,"15min":1.5009999999999999},"min":{"1min":0.497,"5min":0.39600000000000002,"15min":0.35499999999999998},"max":{"1min":1.607,"5min":9.1989999999999998,"15min":13.699},"last":1.1040000000000001},{"interface":"front","average":{"1min":0.85599999999999998,"5min":1.528,"15min":1.6679999999999999},"min":{"1min":0.54700000000000004,"5min":0.42599999999999999,"15min":0.32200000000000001},"max":{"1min":1.3340000000000001,"5min":9.4659999999999993,"15min":9.4659999999999993},"last":1.371}]}]},{"osd":2,"up_from":116,"seq":498216206670,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3438208,"kb_used_data":2756664,"kb_used_omap":7,"kb_used_meta":681528,"kb_avail":90306944,"statfs":{"total":95995035648,"available":92474310656,"internally_reserved":0,"allocated":2822823936,"data_stored":2821601908,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":697884753},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 11:03:57 2023","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":1.3340000000000001,"15min":1.8440000000000001},"min":{"1min":0.26400000000000001,"5min":0.21299999999999999,"15min":0.16900000000000001},"max":{"1min":0.98899999999999999,"5min":9.4510000000000005,"15min":104.42},"last":0.69199999999999995},{"interface":"front","average":{"1min":0.66100000000000003,"5min":1.546,"15min":2.0720000000000001},"min":{"1min":0.28399999999999997,"5min":0.187,"15min":0.187},"max":{"1min":2.0880000000000001,"5min":11.771000000000001,"15min":105.10899999999999},"last":0.66700000000000004}]},{"osd":1,"last update":"Sun Oct 1 11:03:57 2023","interfaces":[{"interface":"back","average":{"1min":0.77100000000000002,"5min":1.468,"15min":1.891},"min":{"1min":0.33500000000000002,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":2.141,"5min":8.6440000000000001,"15min":122.249},"last":0.72099999999999997},{"interface":"front","average":{"1min":0.58199999999999996,"5min":1.6850000000000001,"15min":2.2639999999999998},"min":{"1min":0.28799999999999998,"5min":0.23999999999999999,"15min":0.20599999999999999},"max":{"1min":0.97199999999999998,"5min":11.443,"15min":121.495},"last":0.64400000000000002}]},{"osd":3,"last update":"Sun Oct 1 11:04:26 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":1.6120000000000001,"15min":2.073},"min":{"1min":0.36899999999999999,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.86499999999999999,"5min":23.879000000000001,"15min":124.393},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.59799999999999998,"5min":1.5489999999999999,"15min":2.145},"min":{"1min":0.37,"5min":0.30399999999999999,"15min":0.20899999999999999},"max":{"1min":0.96399999999999997,"5min":23.704000000000001,"15min":122.52200000000001},"last":0.77300000000000002}]},{"osd":4,"last update":"Sun Oct 1 11:04:07 2023","interfaces":[{"interface":"back","average":{"1min":0.79200000000000004,"5min":1.5860000000000001,"15min":1.6599999999999999},"min":{"1min":0.432,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":1.254,"5min":9.5109999999999992,"15min":13.676},"last":0.90400000000000003},{"interface":"front","average":{"1min":0.79100000000000004,"5min":1.9430000000000001,"15min":2.0310000000000001},"min":{"1min":0.45800000000000002,"5min":0.317,"15min":0.311},"max":{"1min":1.1399999999999999,"5min":9.3670000000000009,"15min":17.925000000000001},"last":0.58799999999999997}]},{"osd":5,"last update":"Sun Oct 1 11:03:57 2023","interfaces":[{"interface":"back","average":{"1min":1.022,"5min":1.6819999999999999,"15min":1.7889999999999999},"min":{"1min":0.45000000000000001,"5min":0.378,"15min":0.36599999999999999},"max":{"1min":3.5960000000000001,"5min":10.180999999999999,"15min":10.180999999999999},"last":1.0469999999999999},{"interface":"front","average":{"1min":0.95199999999999996,"5min":1.9550000000000001,"15min":2.0779999999999998},"min":{"1min":0.52600000000000002,"5min":0.372,"15min":0.34100000000000003},"max":{"1min":2.9350000000000001,"5min":9.4169999999999998,"15min":12.835000000000001},"last":0.56499999999999995}]},{"osd":6,"last update":"Sun Oct 1 11:03:57 2023","interfaces":[{"interface":"back","average":{"1min":1.087,"5min":2.0249999999999999,"15min":2.2240000000000002},"min":{"1min":0.48399999999999999,"5min":0.40799999999999997,"15min":0.32100000000000001},"max":{"1min":3.544,"5min":9.5289999999999999,"15min":19.908000000000001},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.84699999999999998,"5min":1.4430000000000001,"15min":1.5629999999999999},"min":{"1min":0.59199999999999997,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":1.5529999999999999,"5min":9.6709999999999994,"15min":9.6709999999999994},"last":0.752}]},{"osd":7,"last update":"Sun Oct 1 11:03:57 2023","interfaces":[{"interface":"back","average":{"1min":1.0349999999999999,"5min":1.4339999999999999,"15min":1.5089999999999999},"min":{"1min":0.65700000000000003,"5min":0.35099999999999998,"15min":0.34300000000000003},"max":{"1min":4.056,"5min":8.0359999999999996,"15min":11.209},"last":0.80200000000000005},{"interface":"front","average":{"1min":0.995,"5min":1.6699999999999999,"15min":1.7350000000000001},"min":{"1min":0.57799999999999996,"5min":0.51200000000000001,"15min":0.36399999999999999},"max":{"1min":2.2250000000000001,"5min":9.5489999999999995,"15min":9.5489999999999995},"last":1.121}]}]},{"osd":3,"up_from":121,"seq":519691043144,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2212824,"kb_used_data":1514384,"kb_used_omap":2,"kb_used_meta":698429,"kb_avail":91532328,"statfs":{"total":95995035648,"available":93729103872,"internally_reserved":0,"allocated":1550729216,"data_stored":1549538799,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":715191349},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 11:04:39 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":1.276,"15min":2.754},"min":{"1min":0.31900000000000001,"5min":0.22600000000000001,"15min":0.17199999999999999},"max":{"1min":0.98599999999999999,"5min":13.130000000000001,"15min":129.953},"last":0.82199999999999995},{"interface":"front","average":{"1min":0.51000000000000001,"5min":1.345,"15min":3.02},"min":{"1min":0.27600000000000002,"5min":0.20000000000000001,"15min":0.189},"max":{"1min":0.83699999999999997,"5min":9.3539999999999992,"15min":129.53999999999999},"last":0.79300000000000004}]},{"osd":1,"last update":"Sun Oct 1 11:04:39 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":1.028,"15min":1.464},"min":{"1min":0.30599999999999999,"5min":0.22500000000000001,"15min":0.193},"max":{"1min":0.69599999999999995,"5min":10.128,"15min":10.574999999999999},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.55600000000000005,"5min":1.2829999999999999,"15min":1.5760000000000001},"min":{"1min":0.34100000000000003,"5min":0.26800000000000002,"15min":0.23400000000000001},"max":{"1min":0.79300000000000004,"5min":13.086,"15min":13.529999999999999},"last":0.91700000000000004}]},{"osd":2,"last update":"Sun Oct 1 11:04:39 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":2.641,"15min":2.2210000000000001},"min":{"1min":0.31,"5min":0.23499999999999999,"15min":0.217},"max":{"1min":0.97699999999999998,"5min":81.212999999999994,"15min":81.212999999999994},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.67400000000000004,"5min":2.8889999999999998,"15min":2.2679999999999998},"min":{"1min":0.40799999999999997,"5min":0.29899999999999999,"15min":0.26900000000000002},"max":{"1min":1.018,"5min":80.819000000000003,"15min":80.819000000000003},"last":0.99099999999999999}]},{"osd":4,"last update":"Sun Oct 1 11:04:16 2023","interfaces":[{"interface":"back","average":{"1min":0.76600000000000001,"5min":1.3879999999999999,"15min":1.5389999999999999},"min":{"1min":0.49299999999999999,"5min":0.437,"15min":0.36799999999999999},"max":{"1min":1.1319999999999999,"5min":8.6780000000000008,"15min":10.808999999999999},"last":1.0509999999999999},{"interface":"front","average":{"1min":0.77800000000000002,"5min":1.47,"15min":1.595},"min":{"1min":0.42199999999999999,"5min":0.35499999999999998,"15min":0.308},"max":{"1min":1.1859999999999999,"5min":8.8190000000000008,"15min":13.391999999999999},"last":0.63900000000000001}]},{"osd":5,"last update":"Sun Oct 1 11:04:39 2023","interfaces":[{"interface":"back","average":{"1min":0.80000000000000004,"5min":1.4179999999999999,"15min":1.5369999999999999},"min":{"1min":0.41399999999999998,"5min":0.378,"15min":0.34399999999999997},"max":{"1min":1.103,"5min":12.771000000000001,"15min":13.333},"last":0.67900000000000005},{"interface":"front","average":{"1min":0.73299999999999998,"5min":1.196,"15min":1.599},"min":{"1min":0.49399999999999999,"5min":0.39500000000000002,"15min":0.34999999999999998},"max":{"1min":1.127,"5min":7.7640000000000002,"15min":12.335000000000001},"last":0.70499999999999996}]},{"osd":6,"last update":"Sun Oct 1 11:04:39 2023","interfaces":[{"interface":"back","average":{"1min":0.84999999999999998,"5min":1.583,"15min":1.7629999999999999},"min":{"1min":0.51900000000000002,"5min":0.48199999999999998,"15min":0.35399999999999998},"max":{"1min":1.103,"5min":13.616,"15min":13.616},"last":0.85799999999999998},{"interface":"front","average":{"1min":0.79500000000000004,"5min":2.077,"15min":2.0459999999999998},"min":{"1min":0.627,"5min":0.433,"15min":0.36799999999999999},"max":{"1min":1.2410000000000001,"5min":73.542000000000002,"15min":73.542000000000002},"last":0.877}]},{"osd":7,"last update":"Sun Oct 1 11:04:39 2023","interfaces":[{"interface":"back","average":{"1min":0.73299999999999998,"5min":1.143,"15min":1.6719999999999999},"min":{"1min":0.496,"5min":0.42399999999999999,"15min":0.376},"max":{"1min":1.1699999999999999,"5min":6.9989999999999997,"15min":12.416},"last":0.96199999999999997},{"interface":"front","average":{"1min":0.747,"5min":1.385,"15min":1.6499999999999999},"min":{"1min":0.51700000000000002,"5min":0.41899999999999998,"15min":0.35899999999999999},"max":{"1min":1.0529999999999999,"5min":8.7200000000000006,"15min":10.755000000000001},"last":1.095}]}]},{"osd":4,"up_from":126,"seq":541165879618,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3327036,"kb_used_data":2744052,"kb_used_omap":10,"kb_used_meta":582965,"kb_avail":90418116,"statfs":{"total":95995035648,"available":92588150784,"internally_reserved":0,"allocated":2809909248,"data_stored":2808685570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":596956772},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 11:04:03 2023","interfaces":[{"interface":"back","average":{"1min":1.1819999999999999,"5min":2.0670000000000002,"15min":1.954},"min":{"1min":0.32800000000000001,"5min":0.27700000000000002,"15min":0.24399999999999999},"max":{"1min":10.321999999999999,"5min":68.611000000000004,"15min":68.611000000000004},"last":0.85099999999999998},{"interface":"front","average":{"1min":0.66600000000000004,"5min":1.915,"15min":1.97},"min":{"1min":0.37,"5min":0.30399999999999999,"15min":0.26200000000000001},"max":{"1min":1.198,"5min":68.480999999999995,"15min":68.480999999999995},"last":0.89200000000000002}]},{"osd":1,"last update":"Sun Oct 1 11:04:03 2023","interfaces":[{"interface":"back","average":{"1min":0.86899999999999999,"5min":1.0189999999999999,"15min":1.383},"min":{"1min":0.44,"5min":0.309,"15min":0.27300000000000002},"max":{"1min":2.3300000000000001,"5min":5.2240000000000002,"15min":29.344000000000001},"last":0.75700000000000001},{"interface":"front","average":{"1min":1.369,"5min":1.2829999999999999,"15min":1.4950000000000001},"min":{"1min":0.46899999999999997,"5min":0.34499999999999997,"15min":0.317},"max":{"1min":10.263,"5min":10.263,"15min":29.251999999999999},"last":0.78700000000000003}]},{"osd":2,"last update":"Sun Oct 1 11:04:03 2023","interfaces":[{"interface":"back","average":{"1min":1.4910000000000001,"5min":3.6629999999999998,"15min":2.9590000000000001},"min":{"1min":0.48299999999999998,"5min":0.34699999999999998,"15min":0.30499999999999999},"max":{"1min":10.449,"5min":133.321,"15min":133.321},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.71899999999999997,"5min":3.0720000000000001,"15min":2.7320000000000002},"min":{"1min":0.375,"5min":0.33700000000000002,"15min":0.316},"max":{"1min":1.2749999999999999,"5min":132.875,"15min":132.875},"last":0.67000000000000004}]},{"osd":3,"last update":"Sun Oct 1 11:04:03 2023","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":2.0390000000000001,"15min":1.8100000000000001},"min":{"1min":0.42399999999999999,"5min":0.36199999999999999,"15min":0.307},"max":{"1min":0.90400000000000003,"5min":61.664999999999999,"15min":61.664999999999999},"last":1.079},{"interface":"front","average":{"1min":0.91800000000000004,"5min":1.974,"15min":2.194},"min":{"1min":0.61099999999999999,"5min":0.36599999999999999,"15min":0.308},"max":{"1min":2.3580000000000001,"5min":61.707999999999998,"15min":130.88300000000001},"last":0.91600000000000004}]},{"osd":5,"last update":"Sun Oct 1 11:04:03 2023","interfaces":[{"interface":"back","average":{"1min":0.81899999999999995,"5min":1.105,"15min":1.6259999999999999},"min":{"1min":0.40799999999999997,"5min":0.36299999999999999,"15min":0.30199999999999999},"max":{"1min":1.5,"5min":10.096,"15min":84.307000000000002},"last":0.82099999999999995},{"interface":"front","average":{"1min":1.643,"5min":1.4650000000000001,"15min":1.74},"min":{"1min":0.39400000000000002,"5min":0.35799999999999998,"15min":0.309},"max":{"1min":13.629,"5min":13.629,"15min":84.234999999999999},"last":1.0049999999999999}]},{"osd":6,"last update":"Sun Oct 1 11:04:03 2023","interfaces":[{"interface":"back","average":{"1min":0.82399999999999995,"5min":1.077,"15min":1.4199999999999999},"min":{"1min":0.47499999999999998,"5min":0.36099999999999999,"15min":0.34999999999999998},"max":{"1min":1.4950000000000001,"5min":8.1859999999999999,"15min":12.563000000000001},"last":0.95699999999999996},{"interface":"front","average":{"1min":1.075,"5min":1.208,"15min":1.4410000000000001},"min":{"1min":0.51700000000000002,"5min":0.42599999999999999,"15min":0.33400000000000002},"max":{"1min":2.5379999999999998,"5min":6.9969999999999999,"15min":10.537000000000001},"last":0.97699999999999998}]},{"osd":7,"last update":"Sun Oct 1 11:04:03 2023","interfaces":[{"interface":"back","average":{"1min":1.022,"5min":1.236,"15min":1.8759999999999999},"min":{"1min":0.56100000000000005,"5min":0.45200000000000001,"15min":0.33900000000000002},"max":{"1min":2.3999999999999999,"5min":5.1529999999999996,"15min":119.56100000000001},"last":1.0589999999999999},{"interface":"front","average":{"1min":1.524,"5min":1.409,"15min":1.821},"min":{"1min":0.44,"5min":0.40999999999999998,"15min":0.39200000000000002},"max":{"1min":10.471,"5min":10.471,"15min":120.44499999999999},"last":1.202}]}]},{"osd":5,"up_from":36,"seq":154618823299,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3891216,"kb_used_data":2991048,"kb_used_omap":0,"kb_used_meta":900160,"kb_avail":89853936,"statfs":{"total":95995035648,"available":92010430464,"internally_reserved":0,"allocated":3062833152,"data_stored":3062285027,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":921763840},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 11:04:26 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":2.2389999999999999,"15min":2.04},"min":{"1min":0.32600000000000001,"5min":0.253,"15min":0.24299999999999999},"max":{"1min":0.95299999999999996,"5min":115.057,"15min":123.39700000000001},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.624,"5min":2.4199999999999999,"15min":2.161},"min":{"1min":0.36499999999999999,"5min":0.30399999999999999,"15min":0.253},"max":{"1min":0.96699999999999997,"5min":115.53,"15min":124.051},"last":0.67900000000000005}]},{"osd":1,"last update":"Sun Oct 1 11:04:26 2023","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":0.96699999999999997,"15min":1.1599999999999999},"min":{"1min":0.371,"5min":0.29899999999999999,"15min":0.27700000000000002},"max":{"1min":1.04,"5min":6.0599999999999996,"15min":9.766},"last":1.1040000000000001},{"interface":"front","average":{"1min":0.68899999999999995,"5min":1.232,"15min":1.327},"min":{"1min":0.40699999999999997,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.218,"5min":7.0069999999999997,"15min":7.7290000000000001},"last":0.73899999999999999}]},{"osd":2,"last update":"Sun Oct 1 11:04:01 2023","interfaces":[{"interface":"back","average":{"1min":0.69099999999999995,"5min":4.202,"15min":2.5529999999999999},"min":{"1min":0.38800000000000001,"5min":0.311,"15min":0.307},"max":{"1min":1.002,"5min":207.059,"15min":207.059},"last":1.1619999999999999},{"interface":"front","average":{"1min":0.89900000000000002,"5min":4.54,"15min":2.7389999999999999},"min":{"1min":0.40699999999999997,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":3.23,"5min":206.673,"15min":206.673},"last":0.84499999999999997}]},{"osd":3,"last update":"Sun Oct 1 11:04:26 2023","interfaces":[{"interface":"back","average":{"1min":0.754,"5min":1.6870000000000001,"15min":2.1709999999999998},"min":{"1min":0.46300000000000002,"5min":0.33800000000000002,"15min":0.32400000000000001},"max":{"1min":1.292,"5min":13.186999999999999,"15min":166.29599999999999},"last":0.873},{"interface":"front","average":{"1min":0.69499999999999995,"5min":1.339,"15min":2.1869999999999998},"min":{"1min":0.45400000000000001,"5min":0.36899999999999999,"15min":0.32000000000000001},"max":{"1min":0.97499999999999998,"5min":11.026999999999999,"15min":166.565},"last":0.75700000000000001}]},{"osd":4,"last update":"Sun Oct 1 11:04:01 2023","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":1.054,"15min":1.153},"min":{"1min":0.45800000000000002,"5min":0.311,"15min":0.311},"max":{"1min":3.8290000000000002,"5min":6.5410000000000004,"15min":9.8889999999999993},"last":0.91100000000000003},{"interface":"front","average":{"1min":0.80300000000000005,"5min":1.0600000000000001,"15min":1.3300000000000001},"min":{"1min":0.40999999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":1.29,"5min":5.0419999999999998,"15min":9.7270000000000003},"last":1.204}]},{"osd":6,"last update":"Sun Oct 1 11:04:26 2023","interfaces":[{"interface":"back","average":{"1min":0.80000000000000004,"5min":1.0980000000000001,"15min":1.573},"min":{"1min":0.40400000000000003,"5min":0.35199999999999998,"15min":0.27800000000000002},"max":{"1min":1.1839999999999999,"5min":5.0899999999999999,"15min":61.768999999999998},"last":1.2410000000000001},{"interface":"front","average":{"1min":0.76100000000000001,"5min":1.103,"15min":1.4930000000000001},"min":{"1min":0.378,"5min":0.34499999999999997,"15min":0.28899999999999998},"max":{"1min":1.2010000000000001,"5min":6.3849999999999998,"15min":62.396999999999998},"last":1.054}]},{"osd":7,"last update":"Sun Oct 1 11:04:26 2023","interfaces":[{"interface":"back","average":{"1min":0.80900000000000005,"5min":1.2390000000000001,"15min":1.3959999999999999},"min":{"1min":0.436,"5min":0.34899999999999998,"15min":0.32800000000000001},"max":{"1min":1.1279999999999999,"5min":6.577,"15min":13.673999999999999},"last":0.98199999999999998},{"interface":"front","average":{"1min":0.92900000000000005,"5min":1.2070000000000001,"15min":1.3460000000000001},"min":{"1min":0.61699999999999999,"5min":0.38600000000000001,"15min":0.34699999999999998},"max":{"1min":1.238,"5min":4.8520000000000003,"15min":9.6929999999999996},"last":1.3009999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1247084544,"data_stored":1247084544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1872236544,"data_stored":1872232472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2816671744,"data_stored":2816671744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1546784768,"data_stored":1546780696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2803175424,"data_stored":2803171352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3055812608,"data_stored":3055812608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1719533568,"data_stored":1719533568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":520683520,"data_stored":520683520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T11:04:59.227 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-01T11:04:59.227 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd pool set device_health_metrics pg_num 21 2023-10-01T11:04:59.313 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:59.324 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:58 smithi173 bash[58816]: audit 2023-10-01T11:04:58.035777+0000 mgr.y (mgr.25072) 1428 : audit 0 from='client.36597 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T11:04:59.325 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:58 smithi173 bash[58816]: audit 2023-10-01T11:04:58.062585+0000 mon.c (mon.2) 214 : audit 1 from='client.? 172.21.15.78:0/3033915762' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T11:04:59.325 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:58 smithi173 bash[58816]: audit 2023-10-01T11:04:58.062936+0000 mon.a (mon.0) 1818 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T11:04:59.339 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:04:59.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:59.341 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:04:59.442 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:59.469 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:59.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:59.471 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:04:59.569 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:04:59.571 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:59.598 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:04:59.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:59.601 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:04:59.701 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:59.736 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:59.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:59.738 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:04:59.840 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:59.863 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:04:59.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:59.868 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:04:59.969 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:59.978 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:59 smithi173 bash[58816]: cluster 2023-10-01T11:04:58.268690+0000 mgr.y (mgr.25072) 1429 : cluster 0 pgmap v1512: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 97 op/s; 20578/227832 objects degraded (9.032%); 4786/227832 objects misplaced (2.101%); 4.2 MiB/s, 66 objects/s recovering 2023-10-01T11:04:59.978 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:59 smithi173 bash[58816]: cluster 2023-10-01T11:04:58.942958+0000 mon.a (mon.0) 1819 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:04:59.978 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:59 smithi173 bash[58816]: cluster 2023-10-01T11:04:58.950618+0000 osd.5 (osd.5) 333 : cluster 3 failed to encode map e321 with expected crc 2023-10-01T11:04:59.978 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:59 smithi173 bash[58816]: cluster 2023-10-01T11:04:58.950996+0000 osd.7 (osd.7) 345 : cluster 3 failed to encode map e321 with expected crc 2023-10-01T11:04:59.979 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:59 smithi173 bash[58816]: audit 2023-10-01T11:04:58.954241+0000 mon.a (mon.0) 1820 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T11:04:59.979 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:04:59 smithi173 bash[58816]: cluster 2023-10-01T11:04:58.954355+0000 mon.a (mon.0) 1821 : cluster 0 osdmap e321: 8 total, 8 up, 6 in 2023-10-01T11:04:59.993 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:04:59.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:04:59.998 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:00.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:59 smithi078 bash[75299]: cluster 2023-10-01T11:04:58.268690+0000 mgr.y (mgr.25072) 1429 : cluster 0 pgmap v1512: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 97 op/s; 20578/227832 objects degraded (9.032%); 4786/227832 objects misplaced (2.101%); 4.2 MiB/s, 66 objects/s recovering 2023-10-01T11:05:00.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:59 smithi078 bash[75299]: cluster 2023-10-01T11:04:58.942958+0000 mon.a (mon.0) 1819 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:05:00.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:59 smithi078 bash[75299]: cluster 2023-10-01T11:04:58.950618+0000 osd.5 (osd.5) 333 : cluster 3 failed to encode map e321 with expected crc 2023-10-01T11:05:00.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:59 smithi078 bash[75299]: cluster 2023-10-01T11:04:58.950996+0000 osd.7 (osd.7) 345 : cluster 3 failed to encode map e321 with expected crc 2023-10-01T11:05:00.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:59 smithi078 bash[75299]: audit 2023-10-01T11:04:58.954241+0000 mon.a (mon.0) 1820 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T11:05:00.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:04:59 smithi078 bash[75299]: cluster 2023-10-01T11:04:58.954355+0000 mon.a (mon.0) 1821 : cluster 0 osdmap e321: 8 total, 8 up, 6 in 2023-10-01T11:05:00.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:59 smithi078 bash[78274]: cluster 2023-10-01T11:04:58.268690+0000 mgr.y (mgr.25072) 1429 : cluster 0 pgmap v1512: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 97 op/s; 20578/227832 objects degraded (9.032%); 4786/227832 objects misplaced (2.101%); 4.2 MiB/s, 66 objects/s recovering 2023-10-01T11:05:00.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:59 smithi078 bash[78274]: cluster 2023-10-01T11:04:58.942958+0000 mon.a (mon.0) 1819 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:05:00.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:59 smithi078 bash[78274]: cluster 2023-10-01T11:04:58.950618+0000 osd.5 (osd.5) 333 : cluster 3 failed to encode map e321 with expected crc 2023-10-01T11:05:00.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:59 smithi078 bash[78274]: cluster 2023-10-01T11:04:58.950996+0000 osd.7 (osd.7) 345 : cluster 3 failed to encode map e321 with expected crc 2023-10-01T11:05:00.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:59 smithi078 bash[78274]: audit 2023-10-01T11:04:58.954241+0000 mon.a (mon.0) 1820 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T11:05:00.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:04:59 smithi078 bash[78274]: cluster 2023-10-01T11:04:58.954355+0000 mon.a (mon.0) 1821 : cluster 0 osdmap e321: 8 total, 8 up, 6 in 2023-10-01T11:05:00.098 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:00.122 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:00.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:00.125 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:05:00.226 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:00.249 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:00.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:00.251 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:05:00.325 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T11:05:00.352 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:00.360 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:05:00 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:05:00] "GET /metrics HTTP/1.1" 200 36407 "" "Prometheus/2.33.4" 2023-10-01T11:05:00.375 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:00.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:00.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:05:00.478 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:00.505 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:00.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:00.512 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:05:00.613 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:00.643 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:00.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:00.645 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:05:00.746 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:00.769 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:00.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:00.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:05:00.872 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:00.898 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:00.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:00.900 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:05:01.001 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:01.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:01.031 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:01.031 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:05:01.133 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:01.161 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:01.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:01.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:05:01.265 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:01.294 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:01.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:01.297 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:05:01.398 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:01.420 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:01.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:01.422 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:01.523 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:01.547 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:01.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:01.550 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:05:01.650 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:01.676 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:01.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:01.679 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:01.780 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:01.807 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:01.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:01.809 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:05:01.910 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:01.933 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:01.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:01.935 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:05:02.036 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:02.049 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:01 smithi173 bash[58816]: cluster 2023-10-01T11:05:00.269483+0000 mgr.y (mgr.25072) 1430 : cluster 0 pgmap v1514: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 93 op/s; 20578/226824 objects degraded (9.072%); 4756/226824 objects misplaced (2.097%); 403 KiB/s, 6 objects/s recovering 2023-10-01T11:05:02.064 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:02.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:02.070 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:02.171 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:02.192 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:02.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:02.194 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:02.242 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:01 smithi078 bash[78274]: cluster 2023-10-01T11:05:00.269483+0000 mgr.y (mgr.25072) 1430 : cluster 0 pgmap v1514: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 93 op/s; 20578/226824 objects degraded (9.072%); 4756/226824 objects misplaced (2.097%); 403 KiB/s, 6 objects/s recovering 2023-10-01T11:05:02.242 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:01 smithi078 bash[75299]: cluster 2023-10-01T11:05:00.269483+0000 mgr.y (mgr.25072) 1430 : cluster 0 pgmap v1514: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 93 op/s; 20578/226824 objects degraded (9.072%); 4756/226824 objects misplaced (2.097%); 403 KiB/s, 6 objects/s recovering 2023-10-01T11:05:02.295 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:02.320 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:02.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:02.323 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:05:02.424 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:02.447 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:02.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:02.450 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:05:02.551 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:02.572 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:02.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:02.575 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:05:02.675 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:02.698 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:02.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:02.700 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:05:02.800 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:02.832 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:02.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:02.835 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:05:02.893 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T11:05:02.936 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:02.967 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:02.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:02.969 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:05:03.070 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:03.094 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:03.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:03.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:05:03.197 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:03.224 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:03.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:03.226 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:05:03.327 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:03.351 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:03.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:03.355 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:05:03.455 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:03.478 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:03.480 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:03 smithi173 bash[58816]: cluster 2023-10-01T11:05:02.269983+0000 mgr.y (mgr.25072) 1431 : cluster 0 pgmap v1515: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 93 op/s; 20578/226824 objects degraded (9.072%); 4756/226824 objects misplaced (2.097%); 403 KiB/s, 6 objects/s recovering 2023-10-01T11:05:03.480 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:03 smithi173 bash[58816]: audit 2023-10-01T11:05:02.463807+0000 mon.a (mon.0) 1822 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:05:03.480 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:03 smithi173 bash[58816]: audit 2023-10-01T11:05:02.465498+0000 mon.a (mon.0) 1823 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:05:03.480 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:03 smithi173 bash[58816]: cluster 2023-10-01T11:05:02.721870+0000 mon.a (mon.0) 1824 : cluster 3 Health check update: Degraded data redundancy: 20578/226824 objects degraded (9.072%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T11:05:03.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:03.481 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:05:03.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:03 smithi078 bash[78274]: cluster 2023-10-01T11:05:02.269983+0000 mgr.y (mgr.25072) 1431 : cluster 0 pgmap v1515: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 93 op/s; 20578/226824 objects degraded (9.072%); 4756/226824 objects misplaced (2.097%); 403 KiB/s, 6 objects/s recovering 2023-10-01T11:05:03.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:03 smithi078 bash[78274]: audit 2023-10-01T11:05:02.463807+0000 mon.a (mon.0) 1822 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:05:03.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:03 smithi078 bash[78274]: audit 2023-10-01T11:05:02.465498+0000 mon.a (mon.0) 1823 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:05:03.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:03 smithi078 bash[78274]: cluster 2023-10-01T11:05:02.721870+0000 mon.a (mon.0) 1824 : cluster 3 Health check update: Degraded data redundancy: 20578/226824 objects degraded (9.072%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T11:05:03.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:03 smithi078 bash[75299]: cluster 2023-10-01T11:05:02.269983+0000 mgr.y (mgr.25072) 1431 : cluster 0 pgmap v1515: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 93 op/s; 20578/226824 objects degraded (9.072%); 4756/226824 objects misplaced (2.097%); 403 KiB/s, 6 objects/s recovering 2023-10-01T11:05:03.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:03 smithi078 bash[75299]: audit 2023-10-01T11:05:02.463807+0000 mon.a (mon.0) 1822 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:05:03.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:03 smithi078 bash[75299]: audit 2023-10-01T11:05:02.465498+0000 mon.a (mon.0) 1823 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:05:03.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:03 smithi078 bash[75299]: cluster 2023-10-01T11:05:02.721870+0000 mon.a (mon.0) 1824 : cluster 3 Health check update: Degraded data redundancy: 20578/226824 objects degraded (9.072%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T11:05:03.582 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:03.611 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:03.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:03.614 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:05:03.715 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:03.740 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:03.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:03.742 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:05:03.750 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:05:03.843 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:03.873 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:03.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:03.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:05:03.976 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:04.006 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:04.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:04.008 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:05:04.109 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:04.139 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:04.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:04.142 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:05:04.243 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:04.264 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:04.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:04.267 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:04.368 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:04.396 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:04.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:04.417 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:05:04.518 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:04.550 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:04.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:04.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:05:04.655 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:04.676 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:04.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:04.678 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:05:04.715 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-10-01T11:05:04.716 INFO:teuthology.orchestra.run.smithi173.stdout: "size": 20, 2023-10-01T11:05:04.716 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 600, 2023-10-01T11:05:04.716 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [ 2023-10-01T11:05:04.716 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.716 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46232.0:30713 12.0 12:0dfa5ac4:::benchmark_data_smithi078_180197_object30712:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e244)", 2023-10-01T11:05:04.716 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:57:02.246384+0000", 2023-10-01T11:05:04.716 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 482.466428716, 2023-10-01T11:05:04.716 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.27095785500000003, 2023-10-01T11:05:04.716 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:05:04.717 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:05:04.717 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:05:04.717 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46232", 2023-10-01T11:05:04.717 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/711982100", 2023-10-01T11:05:04.717 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 30713 2023-10-01T11:05:04.717 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.717 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:05:04.717 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.717 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:05:04.717 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:57:02.246384+0000", 2023-10-01T11:05:04.717 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:05:04.717 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.718 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.718 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:05:04.718 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:57:02.246384+0000", 2023-10-01T11:05:04.718 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:05:04.718 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.718 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.718 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:05:04.718 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:57:02.246382+0000", 2023-10-01T11:05:04.718 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-10-01T11:05:04.718 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.718 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.718 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:05:04.719 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:57:02.246436+0000", 2023-10-01T11:05:04.719 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.3553e-05 2023-10-01T11:05:04.719 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.719 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.719 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:05:04.719 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:57:02.246457+0000", 2023-10-01T11:05:04.719 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.1163e-05 2023-10-01T11:05:04.719 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.719 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.719 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:05:04.719 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:57:02.246469+0000", 2023-10-01T11:05:04.720 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.151e-05 2023-10-01T11:05:04.720 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.720 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.720 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:05:04.720 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:57:02.247916+0000", 2023-10-01T11:05:04.720 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0014472529999999999 2023-10-01T11:05:04.720 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.720 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.720 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:05:04.720 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:57:02.248075+0000", 2023-10-01T11:05:04.720 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00015892900000000001 2023-10-01T11:05:04.720 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.721 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.721 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 2,6", 2023-10-01T11:05:04.721 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:57:02.248274+0000", 2023-10-01T11:05:04.721 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00019960299999999999 2023-10-01T11:05:04.721 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.721 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.721 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:05:04.721 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:57:02.250146+0000", 2023-10-01T11:05:04.721 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0018710650000000001 2023-10-01T11:05:04.721 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.721 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.721 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:04.722 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:57:02.253078+0000", 2023-10-01T11:05:04.722 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0029325419999999998 2023-10-01T11:05:04.722 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.722 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.722 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:04.722 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:57:02.517206+0000", 2023-10-01T11:05:04.722 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.26412774999999999 2023-10-01T11:05:04.722 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.722 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.722 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:05:04.722 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:57:02.517281+0000", 2023-10-01T11:05:04.723 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.4939999999999997e-05 2023-10-01T11:05:04.723 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.723 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.723 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:05:04.723 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:57:02.517342+0000", 2023-10-01T11:05:04.723 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.1119999999999998e-05 2023-10-01T11:05:04.723 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:04.723 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:05:04.723 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:04.723 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.723 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.724 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46232.0:30714 12.0 12:0a516742:::benchmark_data_smithi078_180197_object30713:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e244)", 2023-10-01T11:05:04.724 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:57:02.246625+0000", 2023-10-01T11:05:04.724 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 482.46618800800002, 2023-10-01T11:05:04.724 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.27093959400000001, 2023-10-01T11:05:04.724 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:05:04.724 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:05:04.724 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:05:04.724 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46232", 2023-10-01T11:05:04.724 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/711982100", 2023-10-01T11:05:04.724 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 30714 2023-10-01T11:05:04.724 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.724 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:05:04.725 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.725 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:05:04.725 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:57:02.246625+0000", 2023-10-01T11:05:04.725 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:05:04.725 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.725 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.725 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:05:04.725 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:57:02.246625+0000", 2023-10-01T11:05:04.725 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:05:04.725 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.725 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.725 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:05:04.726 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:57:02.246623+0000", 2023-10-01T11:05:04.726 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-10-01T11:05:04.726 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.726 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.726 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:05:04.726 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:57:02.246743+0000", 2023-10-01T11:05:04.726 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000119578 2023-10-01T11:05:04.726 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.726 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.726 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:05:04.726 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:57:02.246756+0000", 2023-10-01T11:05:04.726 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.3258000000000001e-05 2023-10-01T11:05:04.727 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.727 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.727 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:05:04.727 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:57:02.246763+0000", 2023-10-01T11:05:04.727 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.9110000000000004e-06 2023-10-01T11:05:04.727 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.727 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.727 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:05:04.727 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:57:02.250218+0000", 2023-10-01T11:05:04.727 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0034549210000000001 2023-10-01T11:05:04.727 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.727 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.728 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:05:04.728 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:57:02.250359+0000", 2023-10-01T11:05:04.728 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000141199 2023-10-01T11:05:04.728 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.728 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.728 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 2,6", 2023-10-01T11:05:04.728 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:57:02.250470+0000", 2023-10-01T11:05:04.728 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00011113899999999999 2023-10-01T11:05:04.728 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.728 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.728 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:05:04.728 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:57:02.252860+0000", 2023-10-01T11:05:04.728 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0023898410000000002 2023-10-01T11:05:04.729 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.729 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.729 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:04.729 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:57:02.253273+0000", 2023-10-01T11:05:04.729 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00041263699999999999 2023-10-01T11:05:04.729 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.729 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.729 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:04.729 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:57:02.517416+0000", 2023-10-01T11:05:04.729 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.26414309899999999 2023-10-01T11:05:04.729 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.729 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.730 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:05:04.730 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:57:02.517493+0000", 2023-10-01T11:05:04.730 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.7224000000000005e-05 2023-10-01T11:05:04.730 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.730 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.730 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:05:04.730 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:57:02.517564+0000", 2023-10-01T11:05:04.730 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.1364000000000003e-05 2023-10-01T11:05:04.730 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:04.730 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:05:04.730 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:04.730 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.731 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.731 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46232.0:134068 12.2 12:4ab3dc7e:::benchmark_data_smithi078_180197_object30937:head [delete] snapc 0=[] ondisk+write+known_if_redirected e249)", 2023-10-01T11:05:04.731 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:58:58.761179+0000", 2023-10-01T11:05:04.731 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 365.95163376099998, 2023-10-01T11:05:04.731 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.17506829400000001, 2023-10-01T11:05:04.731 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:05:04.731 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:05:04.731 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:05:04.731 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46232", 2023-10-01T11:05:04.731 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/711982100", 2023-10-01T11:05:04.731 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 134068 2023-10-01T11:05:04.731 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.732 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:05:04.732 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.732 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:05:04.732 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:58:58.761179+0000", 2023-10-01T11:05:04.732 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:05:04.732 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.732 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.732 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:05:04.732 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:58:58.761179+0000", 2023-10-01T11:05:04.732 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:05:04.732 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.732 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.733 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:05:04.733 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:58:58.761177+0000", 2023-10-01T11:05:04.733 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-10-01T11:05:04.733 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.733 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.733 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:05:04.733 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:58:58.761187+0000", 2023-10-01T11:05:04.733 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0115999999999999e-05 2023-10-01T11:05:04.733 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.733 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.733 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:05:04.734 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:58:58.761204+0000", 2023-10-01T11:05:04.734 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.6668999999999999e-05 2023-10-01T11:05:04.734 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.734 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.734 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:05:04.734 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:58:58.761214+0000", 2023-10-01T11:05:04.734 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0331e-05 2023-10-01T11:05:04.734 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.734 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.734 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:05:04.734 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:58:58.761270+0000", 2023-10-01T11:05:04.734 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.5992000000000002e-05 2023-10-01T11:05:04.735 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.735 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.735 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:05:04.735 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:58:58.761378+0000", 2023-10-01T11:05:04.735 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00010792400000000001 2023-10-01T11:05:04.735 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.735 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.735 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 3,7", 2023-10-01T11:05:04.735 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:58:58.761458+0000", 2023-10-01T11:05:04.735 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.0098999999999993e-05 2023-10-01T11:05:04.735 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.735 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.736 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:05:04.736 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:58:58.765408+0000", 2023-10-01T11:05:04.736 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0039501750000000002 2023-10-01T11:05:04.736 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.736 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.736 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:04.736 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:58:58.766352+0000", 2023-10-01T11:05:04.736 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00094351999999999999 2023-10-01T11:05:04.736 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.736 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.736 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:04.736 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:58:58.936119+0000", 2023-10-01T11:05:04.736 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.16976739299999999 2023-10-01T11:05:04.737 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.737 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.737 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:05:04.737 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:58:58.936190+0000", 2023-10-01T11:05:04.737 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.0859999999999996e-05 2023-10-01T11:05:04.737 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.737 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.737 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:05:04.737 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:58:58.936247+0000", 2023-10-01T11:05:04.737 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.7195999999999999e-05 2023-10-01T11:05:04.738 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:04.738 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:05:04.738 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:04.738 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.738 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.738 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46232.0:134081 12.2 12:4f018a62:::benchmark_data_smithi078_180197_object30950:head [delete] snapc 0=[] ondisk+write+known_if_redirected e249)", 2023-10-01T11:05:04.738 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:58:58.784033+0000", 2023-10-01T11:05:04.738 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 365.92877934699999, 2023-10-01T11:05:04.738 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.15233935000000001, 2023-10-01T11:05:04.738 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:05:04.738 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:05:04.739 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:05:04.739 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46232", 2023-10-01T11:05:04.739 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/711982100", 2023-10-01T11:05:04.739 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 134081 2023-10-01T11:05:04.739 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.739 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:05:04.739 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.739 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:05:04.739 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:58:58.784033+0000", 2023-10-01T11:05:04.739 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:05:04.739 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.739 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.740 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:05:04.740 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:58:58.784033+0000", 2023-10-01T11:05:04.740 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:05:04.740 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.740 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.740 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:05:04.740 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:58:58.784032+0000", 2023-10-01T11:05:04.740 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-10-01T11:05:04.740 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.740 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.741 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:05:04.741 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:58:58.784040+0000", 2023-10-01T11:05:04.741 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.1049999999999995e-06 2023-10-01T11:05:04.741 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.741 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.741 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:05:04.741 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:58:58.784055+0000", 2023-10-01T11:05:04.741 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.5271999999999999e-05 2023-10-01T11:05:04.741 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.741 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.741 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:05:04.742 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:58:58.784065+0000", 2023-10-01T11:05:04.742 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.6350000000000001e-06 2023-10-01T11:05:04.742 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.742 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.742 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:05:04.742 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:58:58.784106+0000", 2023-10-01T11:05:04.742 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.1554999999999998e-05 2023-10-01T11:05:04.742 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.742 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.742 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:05:04.743 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:58:58.784209+0000", 2023-10-01T11:05:04.743 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000102438 2023-10-01T11:05:04.743 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.743 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.743 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 3,7", 2023-10-01T11:05:04.743 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:58:58.784290+0000", 2023-10-01T11:05:04.743 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.1527999999999996e-05 2023-10-01T11:05:04.743 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.743 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.743 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:05:04.743 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:58:58.787784+0000", 2023-10-01T11:05:04.744 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.003493875 2023-10-01T11:05:04.744 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.744 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.744 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:04.744 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:58:58.788319+0000", 2023-10-01T11:05:04.744 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00053443900000000005 2023-10-01T11:05:04.744 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.744 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.744 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:04.744 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:58:58.936296+0000", 2023-10-01T11:05:04.744 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.147977518 2023-10-01T11:05:04.745 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.745 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.745 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:05:04.745 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:58:58.936342+0000", 2023-10-01T11:05:04.745 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.6195000000000001e-05 2023-10-01T11:05:04.745 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.745 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.745 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:05:04.745 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:58:58.936373+0000", 2023-10-01T11:05:04.745 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.0292999999999999e-05 2023-10-01T11:05:04.745 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:04.745 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:05:04.746 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:04.746 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.746 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.746 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46232.0:155371 12.2 12:4f77ffe9:::benchmark_data_smithi078_180197_object52240:head [delete] snapc 0=[] ondisk+write+known_if_redirected e252)", 2023-10-01T11:05:04.746 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:59:31.724165+0000", 2023-10-01T11:05:04.746 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 332.98864745999998, 2023-10-01T11:05:04.746 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.40746153099999999, 2023-10-01T11:05:04.746 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:05:04.746 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:05:04.746 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:05:04.746 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46232", 2023-10-01T11:05:04.746 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/711982100", 2023-10-01T11:05:04.747 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 155371 2023-10-01T11:05:04.747 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.747 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:05:04.747 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.747 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:05:04.747 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:59:31.724165+0000", 2023-10-01T11:05:04.747 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:05:04.747 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.747 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.747 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:05:04.747 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:59:31.724165+0000", 2023-10-01T11:05:04.748 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:05:04.748 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.748 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.748 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:05:04.748 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:59:31.724164+0000", 2023-10-01T11:05:04.748 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-10-01T11:05:04.748 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.748 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.748 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:05:04.748 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:59:31.724176+0000", 2023-10-01T11:05:04.749 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2092e-05 2023-10-01T11:05:04.749 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.749 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.749 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:05:04.749 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:59:31.724188+0000", 2023-10-01T11:05:04.749 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1251e-05 2023-10-01T11:05:04.749 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.749 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.750 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:05:04.750 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:59:31.724196+0000", 2023-10-01T11:05:04.750 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.2849999999999995e-06 2023-10-01T11:05:04.750 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.750 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.750 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:05:04.750 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:59:31.728972+0000", 2023-10-01T11:05:04.750 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.004775619 2023-10-01T11:05:04.750 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.750 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.750 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:05:04.751 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:59:31.729185+0000", 2023-10-01T11:05:04.751 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00021369499999999999 2023-10-01T11:05:04.751 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.751 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.751 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 3,7", 2023-10-01T11:05:04.751 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:59:31.729335+0000", 2023-10-01T11:05:04.751 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00015003099999999999 2023-10-01T11:05:04.751 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.751 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.751 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:04.751 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:59:31.735007+0000", 2023-10-01T11:05:04.751 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0056715430000000002 2023-10-01T11:05:04.751 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.752 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.752 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:05:04.752 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:59:31.736791+0000", 2023-10-01T11:05:04.752 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0017841669999999999 2023-10-01T11:05:04.752 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.752 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.752 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:04.752 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:59:32.131370+0000", 2023-10-01T11:05:04.752 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.39457919499999999 2023-10-01T11:05:04.752 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.752 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.752 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:05:04.753 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:59:32.131522+0000", 2023-10-01T11:05:04.753 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000151452 2023-10-01T11:05:04.753 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.753 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.753 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:05:04.753 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:59:32.131627+0000", 2023-10-01T11:05:04.753 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000105136 2023-10-01T11:05:04.754 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:04.754 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:05:04.754 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:04.754 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.755 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.755 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46232.0:155392 12.2 12:4f7d3271:::benchmark_data_smithi078_180197_object52261:head [delete] snapc 0=[] ondisk+write+known_if_redirected e252)", 2023-10-01T11:05:04.755 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:59:31.757168+0000", 2023-10-01T11:05:04.755 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 332.95564503200001, 2023-10-01T11:05:04.755 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.38928179200000002, 2023-10-01T11:05:04.755 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:05:04.755 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:05:04.755 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:05:04.755 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46232", 2023-10-01T11:05:04.755 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/711982100", 2023-10-01T11:05:04.755 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 155392 2023-10-01T11:05:04.756 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.756 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:05:04.756 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.756 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:05:04.756 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:59:31.757168+0000", 2023-10-01T11:05:04.756 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:05:04.756 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.756 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.756 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:05:04.756 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:59:31.757168+0000", 2023-10-01T11:05:04.756 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:05:04.756 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.757 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.757 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:05:04.757 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:59:31.757165+0000", 2023-10-01T11:05:04.757 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999971 2023-10-01T11:05:04.757 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.757 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.757 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:05:04.757 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:59:31.757184+0000", 2023-10-01T11:05:04.757 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.9026e-05 2023-10-01T11:05:04.757 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.757 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.757 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:05:04.757 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:59:31.757224+0000", 2023-10-01T11:05:04.758 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.9691999999999999e-05 2023-10-01T11:05:04.758 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.758 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.758 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:05:04.758 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:59:31.757242+0000", 2023-10-01T11:05:04.758 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.8564e-05 2023-10-01T11:05:04.758 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.758 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.758 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:05:04.758 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:59:31.757339+0000", 2023-10-01T11:05:04.758 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.6551000000000004e-05 2023-10-01T11:05:04.759 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.759 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.759 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:05:04.759 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:59:31.757477+0000", 2023-10-01T11:05:04.759 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00013773 2023-10-01T11:05:04.759 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.759 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.759 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 3,7", 2023-10-01T11:05:04.759 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:59:31.757581+0000", 2023-10-01T11:05:04.759 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00010474700000000001 2023-10-01T11:05:04.759 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.759 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.760 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:04.760 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:59:31.766892+0000", 2023-10-01T11:05:04.760 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0093110899999999993 2023-10-01T11:05:04.760 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.760 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.760 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:05:04.760 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:59:31.766909+0000", 2023-10-01T11:05:04.760 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.6673000000000001e-05 2023-10-01T11:05:04.760 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.760 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.760 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:04.760 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:59:32.146242+0000", 2023-10-01T11:05:04.761 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.37933308599999999 2023-10-01T11:05:04.761 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.761 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.761 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:05:04.761 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:59:32.146352+0000", 2023-10-01T11:05:04.761 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000109932 2023-10-01T11:05:04.761 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.761 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.761 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:05:04.761 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:59:32.146450+0000", 2023-10-01T11:05:04.761 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.7399999999999996e-05 2023-10-01T11:05:04.761 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:04.762 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:05:04.762 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:04.762 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.762 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.762 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46232.0:155393 12.2 12:44a16121:::benchmark_data_smithi078_180197_object52262:head [delete] snapc 0=[] ondisk+write+known_if_redirected e252)", 2023-10-01T11:05:04.762 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:59:31.757344+0000", 2023-10-01T11:05:04.762 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 332.95546830299998, 2023-10-01T11:05:04.762 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.38934637799999999, 2023-10-01T11:05:04.762 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:05:04.762 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:05:04.762 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:05:04.763 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46232", 2023-10-01T11:05:04.763 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/711982100", 2023-10-01T11:05:04.763 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 155393 2023-10-01T11:05:04.763 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.763 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:05:04.763 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.763 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:05:04.763 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:59:31.757344+0000", 2023-10-01T11:05:04.763 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:05:04.763 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.763 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.763 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:05:04.763 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:59:31.757344+0000", 2023-10-01T11:05:04.764 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:05:04.764 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.764 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.764 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:05:04.764 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:59:31.757343+0000", 2023-10-01T11:05:04.764 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-10-01T11:05:04.764 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.764 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.764 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:05:04.764 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:59:31.757350+0000", 2023-10-01T11:05:04.764 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.2940000000000002e-06 2023-10-01T11:05:04.764 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.765 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.765 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:05:04.765 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:59:31.757361+0000", 2023-10-01T11:05:04.765 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1014e-05 2023-10-01T11:05:04.765 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.765 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.765 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:05:04.765 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:59:31.757367+0000", 2023-10-01T11:05:04.765 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.4540000000000004e-06 2023-10-01T11:05:04.765 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.765 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.765 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:05:04.766 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:59:31.761868+0000", 2023-10-01T11:05:04.766 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0045012289999999998 2023-10-01T11:05:04.766 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.766 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.766 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:05:04.766 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:59:31.762013+0000", 2023-10-01T11:05:04.766 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000144303 2023-10-01T11:05:04.766 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.766 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.766 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 3,7", 2023-10-01T11:05:04.767 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:59:31.762152+0000", 2023-10-01T11:05:04.767 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00013930799999999999 2023-10-01T11:05:04.767 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.767 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.767 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:05:04.767 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:59:31.767019+0000", 2023-10-01T11:05:04.767 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0048672560000000004 2023-10-01T11:05:04.767 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.767 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.767 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:04.767 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:59:31.767719+0000", 2023-10-01T11:05:04.767 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000699773 2023-10-01T11:05:04.768 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.768 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.768 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:04.768 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:59:32.146532+0000", 2023-10-01T11:05:04.768 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.37881302300000003 2023-10-01T11:05:04.768 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.768 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.768 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:05:04.768 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:59:32.146620+0000", 2023-10-01T11:05:04.768 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.8350000000000006e-05 2023-10-01T11:05:04.768 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.768 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.769 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:05:04.769 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:59:32.146691+0000", 2023-10-01T11:05:04.769 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.0400000000000004e-05 2023-10-01T11:05:04.769 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:04.769 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:05:04.769 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:04.769 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.769 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.769 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46232.0:159788 12.0 12:0a5c648d:::benchmark_data_smithi078_180197_object56657:head [delete] snapc 0=[] ondisk+write+known_if_redirected e252)", 2023-10-01T11:05:04.769 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:59:38.992108+0000", 2023-10-01T11:05:04.769 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 325.72070448400001, 2023-10-01T11:05:04.769 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.19184726099999999, 2023-10-01T11:05:04.770 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:05:04.770 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:05:04.770 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:05:04.770 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46232", 2023-10-01T11:05:04.770 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/711982100", 2023-10-01T11:05:04.770 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 159788 2023-10-01T11:05:04.770 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.770 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:05:04.770 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.770 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:05:04.770 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:59:38.992108+0000", 2023-10-01T11:05:04.771 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:05:04.771 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.771 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.771 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:05:04.771 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:59:38.992108+0000", 2023-10-01T11:05:04.771 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:05:04.771 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.771 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.771 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:05:04.771 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:59:38.992107+0000", 2023-10-01T11:05:04.771 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-10-01T11:05:04.771 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.772 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.772 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:05:04.772 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:59:38.992116+0000", 2023-10-01T11:05:04.772 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.3959999999999995e-06 2023-10-01T11:05:04.772 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.772 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.772 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:05:04.772 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:59:38.992137+0000", 2023-10-01T11:05:04.772 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.1070999999999999e-05 2023-10-01T11:05:04.772 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.772 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.772 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:05:04.772 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:59:38.992148+0000", 2023-10-01T11:05:04.773 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0689999999999999e-05 2023-10-01T11:05:04.773 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.773 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.773 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:05:04.773 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:59:38.997483+0000", 2023-10-01T11:05:04.773 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.005335322 2023-10-01T11:05:04.773 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.773 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.773 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:05:04.773 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:59:38.997635+0000", 2023-10-01T11:05:04.773 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000151829 2023-10-01T11:05:04.773 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.774 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.774 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 2,6", 2023-10-01T11:05:04.774 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:59:38.997774+0000", 2023-10-01T11:05:04.774 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00013945400000000001 2023-10-01T11:05:04.774 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.774 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.774 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:05:04.774 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:59:39.002725+0000", 2023-10-01T11:05:04.774 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0049510700000000001 2023-10-01T11:05:04.774 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.774 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.775 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:04.775 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:59:39.002775+0000", 2023-10-01T11:05:04.775 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.9385999999999998e-05 2023-10-01T11:05:04.775 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.775 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.775 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:04.775 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:59:39.183776+0000", 2023-10-01T11:05:04.775 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.181000932 2023-10-01T11:05:04.775 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.775 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.775 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:05:04.775 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:59:39.183858+0000", 2023-10-01T11:05:04.776 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.1833000000000001e-05 2023-10-01T11:05:04.776 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.776 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.776 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:05:04.776 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:59:39.183956+0000", 2023-10-01T11:05:04.776 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.7931999999999997e-05 2023-10-01T11:05:04.776 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:04.776 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:05:04.776 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:04.776 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.776 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.776 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46232.0:193436 12.0 12:0b21e224:::benchmark_data_smithi078_180197_object90305:head [delete] snapc 0=[] ondisk+write+known_if_redirected e259)", 2023-10-01T11:05:04.777 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:00:34.984641+0000", 2023-10-01T11:05:04.777 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 269.728171691, 2023-10-01T11:05:04.777 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.14904957199999999, 2023-10-01T11:05:04.777 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:05:04.777 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:05:04.777 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:05:04.777 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46232", 2023-10-01T11:05:04.777 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/711982100", 2023-10-01T11:05:04.777 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 193436 2023-10-01T11:05:04.777 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.777 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:05:04.777 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.778 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:05:04.778 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:34.984641+0000", 2023-10-01T11:05:04.778 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:05:04.778 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.778 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.778 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:05:04.778 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:34.984641+0000", 2023-10-01T11:05:04.778 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:05:04.778 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.778 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.778 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:05:04.778 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:34.984640+0000", 2023-10-01T11:05:04.779 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-10-01T11:05:04.779 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.779 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.779 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:05:04.779 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:34.984644+0000", 2023-10-01T11:05:04.779 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.0029999999999996e-06 2023-10-01T11:05:04.779 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.779 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.779 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:05:04.779 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:34.984649+0000", 2023-10-01T11:05:04.779 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.5439999999999998e-06 2023-10-01T11:05:04.780 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.780 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.780 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:05:04.780 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:34.984653+0000", 2023-10-01T11:05:04.780 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.9430000000000002e-06 2023-10-01T11:05:04.780 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.780 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.780 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:05:04.780 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:34.984697+0000", 2023-10-01T11:05:04.780 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.4104999999999999e-05 2023-10-01T11:05:04.781 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.781 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.781 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:05:04.781 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:34.984832+0000", 2023-10-01T11:05:04.781 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000134182 2023-10-01T11:05:04.781 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.781 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.781 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 2,6,7", 2023-10-01T11:05:04.781 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:34.985019+0000", 2023-10-01T11:05:04.781 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00018733199999999999 2023-10-01T11:05:04.782 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.782 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.782 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:05:04.782 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:34.991117+0000", 2023-10-01T11:05:04.782 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0060982739999999999 2023-10-01T11:05:04.782 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.782 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.782 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:04.782 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:34.991181+0000", 2023-10-01T11:05:04.782 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.3368999999999996e-05 2023-10-01T11:05:04.782 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.782 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.783 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:04.783 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:34.995757+0000", 2023-10-01T11:05:04.783 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0045759620000000003 2023-10-01T11:05:04.783 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.783 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.783 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:04.783 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:35.133483+0000", 2023-10-01T11:05:04.783 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.137726072 2023-10-01T11:05:04.783 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.783 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.783 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:05:04.784 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:35.133544+0000", 2023-10-01T11:05:04.784 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.1462000000000002e-05 2023-10-01T11:05:04.784 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.784 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.784 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:05:04.784 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:35.133691+0000", 2023-10-01T11:05:04.784 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000146508 2023-10-01T11:05:04.784 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:04.784 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:05:04.784 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:04.784 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.785 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.785 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:18163 13.d 13:b28ba6e8:::benchmark_data_smithi078_202524_object18162:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e291)", 2023-10-01T11:05:04.785 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:02:14.746390+0000", 2023-10-01T11:05:04.785 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 169.96642280399999, 2023-10-01T11:05:04.785 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.19699196499999999, 2023-10-01T11:05:04.785 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:05:04.785 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:05:04.785 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:05:04.785 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:05:04.785 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:05:04.785 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 18163 2023-10-01T11:05:04.785 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.786 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:05:04.786 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.786 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:05:04.786 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.746390+0000", 2023-10-01T11:05:04.786 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:05:04.786 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.786 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.786 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:05:04.786 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.746390+0000", 2023-10-01T11:05:04.786 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:05:04.786 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.787 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.787 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:05:04.787 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.746388+0000", 2023-10-01T11:05:04.787 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-10-01T11:05:04.787 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.787 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.787 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:05:04.787 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.746423+0000", 2023-10-01T11:05:04.787 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.4480000000000002e-05 2023-10-01T11:05:04.787 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.787 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.787 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:05:04.788 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.746435+0000", 2023-10-01T11:05:04.788 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2557e-05 2023-10-01T11:05:04.788 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.788 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.788 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:05:04.788 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.746441+0000", 2023-10-01T11:05:04.788 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.365e-06 2023-10-01T11:05:04.788 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.788 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.788 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:05:04.788 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.746606+0000", 2023-10-01T11:05:04.788 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00016403199999999999 2023-10-01T11:05:04.789 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.789 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.789 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:05:04.789 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.746799+0000", 2023-10-01T11:05:04.789 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00019320200000000001 2023-10-01T11:05:04.789 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.789 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.789 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1,3", 2023-10-01T11:05:04.789 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.746989+0000", 2023-10-01T11:05:04.789 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00019067800000000001 2023-10-01T11:05:04.789 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.789 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.790 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:04.790 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.750523+0000", 2023-10-01T11:05:04.790 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0035337820000000001 2023-10-01T11:05:04.790 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.790 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.790 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:05:04.790 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.751432+0000", 2023-10-01T11:05:04.790 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00090912199999999995 2023-10-01T11:05:04.790 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.790 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.790 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:04.790 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.943180+0000", 2023-10-01T11:05:04.791 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.19174725000000001 2023-10-01T11:05:04.791 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.791 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.791 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:05:04.791 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.943295+0000", 2023-10-01T11:05:04.791 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000115109 2023-10-01T11:05:04.791 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.791 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.791 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:05:04.791 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.943382+0000", 2023-10-01T11:05:04.791 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.7254000000000002e-05 2023-10-01T11:05:04.791 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:04.792 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:05:04.792 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:04.792 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.792 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.792 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:18164 13.d 13:b656909e:::benchmark_data_smithi078_202524_object18163:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e291)", 2023-10-01T11:05:04.792 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:02:14.746557+0000", 2023-10-01T11:05:04.792 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 169.96625595099999, 2023-10-01T11:05:04.792 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.19703968499999999, 2023-10-01T11:05:04.792 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:05:04.792 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:05:04.792 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:05:04.793 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:05:04.793 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:05:04.793 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 18164 2023-10-01T11:05:04.793 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.793 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:05:04.793 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.793 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:05:04.793 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.746557+0000", 2023-10-01T11:05:04.793 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:05:04.793 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.793 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.793 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:05:04.794 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.746557+0000", 2023-10-01T11:05:04.794 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:05:04.794 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.794 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.794 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:05:04.794 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.746556+0000", 2023-10-01T11:05:04.794 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-10-01T11:05:04.794 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.794 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.794 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:05:04.794 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.746598+0000", 2023-10-01T11:05:04.794 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.2478999999999998e-05 2023-10-01T11:05:04.795 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.795 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.795 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:05:04.795 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.746611+0000", 2023-10-01T11:05:04.795 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2439999999999999e-05 2023-10-01T11:05:04.795 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.795 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.795 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:05:04.795 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.746620+0000", 2023-10-01T11:05:04.795 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.2969999999999992e-06 2023-10-01T11:05:04.795 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.796 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.796 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:05:04.796 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.749048+0000", 2023-10-01T11:05:04.796 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0024282140000000002 2023-10-01T11:05:04.796 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.796 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.796 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:05:04.796 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.749224+0000", 2023-10-01T11:05:04.796 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000176093 2023-10-01T11:05:04.796 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.796 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.796 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1,3", 2023-10-01T11:05:04.797 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.749403+0000", 2023-10-01T11:05:04.797 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000179047 2023-10-01T11:05:04.797 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.797 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.797 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:04.797 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.752435+0000", 2023-10-01T11:05:04.797 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0030312109999999998 2023-10-01T11:05:04.797 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.797 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.797 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:05:04.797 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.752448+0000", 2023-10-01T11:05:04.797 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.3908e-05 2023-10-01T11:05:04.798 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.798 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.798 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:04.798 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.943461+0000", 2023-10-01T11:05:04.798 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.191012186 2023-10-01T11:05:04.798 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.798 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.798 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:05:04.798 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.943546+0000", 2023-10-01T11:05:04.798 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.4963999999999995e-05 2023-10-01T11:05:04.798 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.799 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.799 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:05:04.799 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.943596+0000", 2023-10-01T11:05:04.799 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.0878999999999999e-05 2023-10-01T11:05:04.799 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:04.799 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:05:04.799 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:04.799 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.799 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.799 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:18168 13.d 13:b3034507:::benchmark_data_smithi078_202524_object18167:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e291)", 2023-10-01T11:05:04.799 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:02:14.747106+0000", 2023-10-01T11:05:04.799 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 169.96570720599999, 2023-10-01T11:05:04.800 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.19663782599999999, 2023-10-01T11:05:04.800 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:05:04.800 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:05:04.800 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:05:04.800 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:05:04.800 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:05:04.800 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 18168 2023-10-01T11:05:04.800 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.800 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:05:04.800 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.800 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:05:04.801 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.747106+0000", 2023-10-01T11:05:04.801 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:05:04.801 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.801 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.801 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:05:04.801 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.747106+0000", 2023-10-01T11:05:04.801 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:05:04.801 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.801 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.801 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:05:04.801 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.747105+0000", 2023-10-01T11:05:04.801 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-10-01T11:05:04.802 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.802 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.802 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:05:04.802 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.747150+0000", 2023-10-01T11:05:04.802 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.5589e-05 2023-10-01T11:05:04.802 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.802 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.802 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:05:04.802 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.747162+0000", 2023-10-01T11:05:04.802 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1562e-05 2023-10-01T11:05:04.802 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.802 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.803 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:05:04.803 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.747170+0000", 2023-10-01T11:05:04.803 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.4770000000000003e-06 2023-10-01T11:05:04.803 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.803 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.803 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:05:04.803 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.750558+0000", 2023-10-01T11:05:04.803 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0033878369999999999 2023-10-01T11:05:04.803 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.803 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.803 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:05:04.804 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.750660+0000", 2023-10-01T11:05:04.804 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000102237 2023-10-01T11:05:04.804 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.804 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.804 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1,3", 2023-10-01T11:05:04.804 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.750755+0000", 2023-10-01T11:05:04.804 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.5030000000000003e-05 2023-10-01T11:05:04.804 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.804 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.804 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:04.804 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.752491+0000", 2023-10-01T11:05:04.804 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.001735688 2023-10-01T11:05:04.805 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.805 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.805 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:05:04.805 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.752499+0000", 2023-10-01T11:05:04.805 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.9379999999999992e-06 2023-10-01T11:05:04.805 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.805 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.805 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:04.805 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.943654+0000", 2023-10-01T11:05:04.805 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.19115552899999999 2023-10-01T11:05:04.805 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.806 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.806 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:05:04.806 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.943706+0000", 2023-10-01T11:05:04.806 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.1351999999999999e-05 2023-10-01T11:05:04.806 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.806 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.806 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:05:04.806 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.943743+0000", 2023-10-01T11:05:04.806 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.7546000000000003e-05 2023-10-01T11:05:04.806 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:04.806 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:05:04.806 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:04.807 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.807 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.807 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:18169 13.d 13:ba3e613a:::benchmark_data_smithi078_202524_object18168:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e291)", 2023-10-01T11:05:04.807 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:02:14.747322+0000", 2023-10-01T11:05:04.807 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 169.96549068900001, 2023-10-01T11:05:04.807 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.19658656599999999, 2023-10-01T11:05:04.807 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:05:04.807 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:05:04.807 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:05:04.807 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:05:04.807 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:05:04.808 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 18169 2023-10-01T11:05:04.808 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.808 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:05:04.808 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.808 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:05:04.808 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.747322+0000", 2023-10-01T11:05:04.808 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:05:04.808 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.808 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.808 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:05:04.808 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.747322+0000", 2023-10-01T11:05:04.808 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:05:04.809 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.809 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.809 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:05:04.809 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.747321+0000", 2023-10-01T11:05:04.809 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-10-01T11:05:04.809 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.809 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.809 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:05:04.809 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.747395+0000", 2023-10-01T11:05:04.809 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.4233000000000006e-05 2023-10-01T11:05:04.809 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.809 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.810 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:05:04.810 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.747406+0000", 2023-10-01T11:05:04.810 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0318e-05 2023-10-01T11:05:04.810 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.810 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.810 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:05:04.810 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.747412+0000", 2023-10-01T11:05:04.810 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.9780000000000002e-06 2023-10-01T11:05:04.810 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.810 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.810 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:05:04.811 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.751472+0000", 2023-10-01T11:05:04.811 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0040598359999999998 2023-10-01T11:05:04.811 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.811 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.811 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:05:04.811 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.751609+0000", 2023-10-01T11:05:04.811 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000137341 2023-10-01T11:05:04.811 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.811 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.811 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1,3", 2023-10-01T11:05:04.811 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.751774+0000", 2023-10-01T11:05:04.811 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00016492300000000001 2023-10-01T11:05:04.812 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.812 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.812 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:04.812 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.753748+0000", 2023-10-01T11:05:04.812 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0019743069999999998 2023-10-01T11:05:04.812 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.812 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.812 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:05:04.812 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.753754+0000", 2023-10-01T11:05:04.812 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.1709999999999999e-06 2023-10-01T11:05:04.812 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.812 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.813 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:04.813 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.943782+0000", 2023-10-01T11:05:04.813 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.19002730800000001 2023-10-01T11:05:04.813 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.813 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.813 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:05:04.813 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.943846+0000", 2023-10-01T11:05:04.813 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.4237000000000002e-05 2023-10-01T11:05:04.813 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.813 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.813 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:05:04.814 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.943909+0000", 2023-10-01T11:05:04.814 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.2707999999999997e-05 2023-10-01T11:05:04.814 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:04.814 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:05:04.814 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:04.814 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.814 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.814 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:18173 13.d 13:b71e730e:::benchmark_data_smithi078_202524_object18172:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e291)", 2023-10-01T11:05:04.814 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:02:14.749647+0000", 2023-10-01T11:05:04.814 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 169.96316564899999, 2023-10-01T11:05:04.814 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.194414118, 2023-10-01T11:05:04.815 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:05:04.815 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:05:04.815 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:05:04.815 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:05:04.815 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:05:04.815 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 18173 2023-10-01T11:05:04.815 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.815 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:05:04.815 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.816 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:05:04.816 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.749647+0000", 2023-10-01T11:05:04.816 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:05:04.816 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.816 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.816 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:05:04.816 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.749647+0000", 2023-10-01T11:05:04.816 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:05:04.816 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.816 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.816 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:05:04.817 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.749643+0000", 2023-10-01T11:05:04.817 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999952 2023-10-01T11:05:04.817 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.817 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.817 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:05:04.817 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.749817+0000", 2023-10-01T11:05:04.817 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00017420900000000001 2023-10-01T11:05:04.817 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.817 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.817 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:05:04.817 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.749826+0000", 2023-10-01T11:05:04.817 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.4739999999999994e-06 2023-10-01T11:05:04.818 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.818 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.818 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:05:04.818 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.749835+0000", 2023-10-01T11:05:04.818 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.6819999999999992e-06 2023-10-01T11:05:04.818 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.818 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.818 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:05:04.818 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.752526+0000", 2023-10-01T11:05:04.818 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0026907020000000001 2023-10-01T11:05:04.818 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.818 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.819 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:05:04.819 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.753048+0000", 2023-10-01T11:05:04.819 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000522096 2023-10-01T11:05:04.819 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.819 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.819 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1,3", 2023-10-01T11:05:04.819 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.753191+0000", 2023-10-01T11:05:04.819 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00014302100000000001 2023-10-01T11:05:04.819 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.819 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.819 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:05:04.819 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.753881+0000", 2023-10-01T11:05:04.821 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00069042300000000001 2023-10-01T11:05:04.821 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.821 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.821 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:04.821 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.754279+0000", 2023-10-01T11:05:04.821 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00039833000000000002 2023-10-01T11:05:04.822 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.822 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.822 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:04.822 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.943953+0000", 2023-10-01T11:05:04.822 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.189673907 2023-10-01T11:05:04.823 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.823 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.823 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:05:04.823 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.944001+0000", 2023-10-01T11:05:04.823 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.7568000000000003e-05 2023-10-01T11:05:04.823 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.823 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.823 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:05:04.823 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:02:14.944061+0000", 2023-10-01T11:05:04.823 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.0282999999999999e-05 2023-10-01T11:05:04.823 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:04.824 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:05:04.824 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:04.824 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.824 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.824 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.46454.0:82681 13.f e320/304 13:f5b7f793:::benchmark_data_smithi078_202524_object3123:head v 320'5270, mlcod=0'0)", 2023-10-01T11:05:04.824 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:04:49.237697+0000", 2023-10-01T11:05:04.824 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 15.475115902000001, 2023-10-01T11:05:04.824 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0078325410000000002, 2023-10-01T11:05:04.824 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:05:04.824 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:05:04.824 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:05:04.824 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.825 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:05:04.825 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:49.237697+0000", 2023-10-01T11:05:04.825 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:05:04.825 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.825 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.825 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:05:04.825 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:49.237697+0000", 2023-10-01T11:05:04.826 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:05:04.826 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.826 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.826 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:05:04.826 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:49.237696+0000", 2023-10-01T11:05:04.826 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-10-01T11:05:04.826 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.826 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.826 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:05:04.826 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:49.237705+0000", 2023-10-01T11:05:04.826 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.6889999999999993e-06 2023-10-01T11:05:04.826 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.827 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.827 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:05:04.827 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:49.237719+0000", 2023-10-01T11:05:04.827 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4894e-05 2023-10-01T11:05:04.827 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.827 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.827 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:05:04.827 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:49.237729+0000", 2023-10-01T11:05:04.827 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.2569999999999996e-06 2023-10-01T11:05:04.827 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.827 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.827 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:05:04.827 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:49.237797+0000", 2023-10-01T11:05:04.828 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.8374000000000006e-05 2023-10-01T11:05:04.828 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.828 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.828 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:05:04.828 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:49.237843+0000", 2023-10-01T11:05:04.828 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.6393000000000001e-05 2023-10-01T11:05:04.828 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.828 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.828 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:05:04.828 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:49.245481+0000", 2023-10-01T11:05:04.828 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0076373099999999996 2023-10-01T11:05:04.828 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.829 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.829 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:05:04.829 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:49.245529+0000", 2023-10-01T11:05:04.829 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.8578999999999997e-05 2023-10-01T11:05:04.829 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:04.829 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:05:04.829 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:04.829 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.829 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.829 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:82788 13.3 13:cad2ef39:::benchmark_data_smithi078_202524_object3230:head [delete] snapc 0=[] ondisk+write+known_if_redirected e320)", 2023-10-01T11:05:04.829 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:04:51.160189+0000", 2023-10-01T11:05:04.829 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 13.552623546, 2023-10-01T11:05:04.830 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0081152800000000008, 2023-10-01T11:05:04.830 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:05:04.830 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:05:04.830 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:05:04.830 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:05:04.830 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:05:04.830 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 82788 2023-10-01T11:05:04.830 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.830 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:05:04.830 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.830 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:05:04.830 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:51.160189+0000", 2023-10-01T11:05:04.831 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:05:04.831 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.831 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.831 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:05:04.831 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:51.160189+0000", 2023-10-01T11:05:04.831 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:05:04.831 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.831 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.831 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:05:04.831 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:51.160188+0000", 2023-10-01T11:05:04.831 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-10-01T11:05:04.831 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.832 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.832 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:05:04.832 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:51.160195+0000", 2023-10-01T11:05:04.832 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.6850000000000001e-06 2023-10-01T11:05:04.832 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.832 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.832 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:05:04.832 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:51.160209+0000", 2023-10-01T11:05:04.832 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.3287e-05 2023-10-01T11:05:04.832 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.832 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.832 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:05:04.833 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:51.160216+0000", 2023-10-01T11:05:04.833 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.5009999999999998e-06 2023-10-01T11:05:04.833 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.833 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.833 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:05:04.833 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:51.160270+0000", 2023-10-01T11:05:04.833 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.4341e-05 2023-10-01T11:05:04.833 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.833 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.833 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:05:04.833 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:51.160408+0000", 2023-10-01T11:05:04.833 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00013808099999999999 2023-10-01T11:05:04.834 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.834 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.834 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 0,3", 2023-10-01T11:05:04.834 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:51.160535+0000", 2023-10-01T11:05:04.835 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000126854 2023-10-01T11:05:04.835 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.835 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.835 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:05:04.835 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:51.163396+0000", 2023-10-01T11:05:04.835 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0028602860000000001 2023-10-01T11:05:04.835 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.835 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.835 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:04.835 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:51.165164+0000", 2023-10-01T11:05:04.836 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.001768248 2023-10-01T11:05:04.836 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.836 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.836 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:04.836 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:51.168223+0000", 2023-10-01T11:05:04.836 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0030586760000000002 2023-10-01T11:05:04.836 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.836 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.836 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:05:04.836 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:51.168256+0000", 2023-10-01T11:05:04.836 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.3634999999999999e-05 2023-10-01T11:05:04.836 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.837 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.837 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:05:04.837 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:51.168304+0000", 2023-10-01T11:05:04.837 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.829e-05 2023-10-01T11:05:04.837 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:04.837 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:05:04.837 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:04.837 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.837 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.837 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:82987 13.5 13:ae58208b:::benchmark_data_smithi078_202524_object3429:head [delete] snapc 0=[] ondisk+write+known_if_redirected e320)", 2023-10-01T11:05:04.837 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:04:53.544561+0000", 2023-10-01T11:05:04.837 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 11.168251517, 2023-10-01T11:05:04.837 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0080313199999999998, 2023-10-01T11:05:04.838 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:05:04.838 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:05:04.838 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:05:04.838 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:05:04.838 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:05:04.838 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 82987 2023-10-01T11:05:04.838 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.838 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:05:04.838 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.838 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:05:04.838 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:53.544561+0000", 2023-10-01T11:05:04.838 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:05:04.839 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.839 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.839 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:05:04.839 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:53.544561+0000", 2023-10-01T11:05:04.839 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:05:04.839 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.839 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.839 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:05:04.839 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:53.544560+0000", 2023-10-01T11:05:04.839 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-10-01T11:05:04.839 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.839 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.840 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:05:04.840 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:53.544567+0000", 2023-10-01T11:05:04.840 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.3180000000000001e-06 2023-10-01T11:05:04.840 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.840 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.840 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:05:04.840 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:53.544582+0000", 2023-10-01T11:05:04.840 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4778e-05 2023-10-01T11:05:04.840 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.840 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.840 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:05:04.840 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:53.544590+0000", 2023-10-01T11:05:04.841 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.4349999999999997e-06 2023-10-01T11:05:04.841 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.841 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.841 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:05:04.841 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:53.544663+0000", 2023-10-01T11:05:04.841 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.2311000000000006e-05 2023-10-01T11:05:04.841 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.841 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.841 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:05:04.841 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:53.544852+0000", 2023-10-01T11:05:04.841 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00018921199999999999 2023-10-01T11:05:04.841 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.842 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.842 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 0,5", 2023-10-01T11:05:04.842 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:53.544954+0000", 2023-10-01T11:05:04.842 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000102002 2023-10-01T11:05:04.842 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.842 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.842 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:05:04.842 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:53.547909+0000", 2023-10-01T11:05:04.842 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.002955039 2023-10-01T11:05:04.842 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.842 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.842 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:04.842 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:53.548268+0000", 2023-10-01T11:05:04.843 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000359293 2023-10-01T11:05:04.843 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.843 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.843 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:04.843 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:53.552484+0000", 2023-10-01T11:05:04.843 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0042155200000000004 2023-10-01T11:05:04.843 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.843 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.843 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:05:04.843 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:53.552543+0000", 2023-10-01T11:05:04.843 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.9373999999999998e-05 2023-10-01T11:05:04.843 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.844 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.844 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:05:04.844 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:53.552593+0000", 2023-10-01T11:05:04.844 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.9484999999999998e-05 2023-10-01T11:05:04.844 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:04.844 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:05:04.844 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:04.844 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.844 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.844 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.46454.0:82992 13.f e320/304 13:f7efffbb:::benchmark_data_smithi078_202524_object3434:head v 320'5288, mlcod=0'0)", 2023-10-01T11:05:04.844 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:04:53.562841+0000", 2023-10-01T11:05:04.844 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 11.149971381, 2023-10-01T11:05:04.845 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.010295017, 2023-10-01T11:05:04.845 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:05:04.845 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:05:04.845 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:05:04.845 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.845 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:05:04.845 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:53.562841+0000", 2023-10-01T11:05:04.845 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:05:04.845 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.845 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.845 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:05:04.845 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:53.562841+0000", 2023-10-01T11:05:04.846 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:05:04.846 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.846 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.846 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:05:04.846 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:53.562840+0000", 2023-10-01T11:05:04.846 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-10-01T11:05:04.846 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.846 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.846 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:05:04.846 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:53.562854+0000", 2023-10-01T11:05:04.846 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4649e-05 2023-10-01T11:05:04.846 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.847 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.847 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:05:04.847 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:53.562873+0000", 2023-10-01T11:05:04.847 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.8893e-05 2023-10-01T11:05:04.847 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.847 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.847 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:05:04.847 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:53.562885+0000", 2023-10-01T11:05:04.847 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1642e-05 2023-10-01T11:05:04.847 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.847 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.847 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:05:04.848 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:53.566727+0000", 2023-10-01T11:05:04.848 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0038423900000000002 2023-10-01T11:05:04.848 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.848 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.848 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:05:04.848 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:53.566783+0000", 2023-10-01T11:05:04.848 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.6304000000000002e-05 2023-10-01T11:05:04.848 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.848 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.848 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:05:04.848 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:53.573094+0000", 2023-10-01T11:05:04.848 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0063109630000000002 2023-10-01T11:05:04.848 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.849 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.849 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:05:04.849 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:53.573136+0000", 2023-10-01T11:05:04.849 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.1919000000000003e-05 2023-10-01T11:05:04.849 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:04.849 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:05:04.849 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:04.849 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.849 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.849 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.46454.0:83490 13.a e320/284 13:5751ca3d:::benchmark_data_smithi078_202524_object3932:head v 320'5387, mlcod=320'5385)", 2023-10-01T11:05:04.849 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:04:58.265289+0000", 2023-10-01T11:05:04.849 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 6.4475239330000003, 2023-10-01T11:05:04.850 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0082481610000000004, 2023-10-01T11:05:04.850 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:05:04.850 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:05:04.850 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:05:04.850 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.850 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:05:04.850 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:58.265289+0000", 2023-10-01T11:05:04.850 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:05:04.850 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.851 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.851 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:05:04.851 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:58.265289+0000", 2023-10-01T11:05:04.851 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:05:04.851 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.851 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.851 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:05:04.851 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:58.265287+0000", 2023-10-01T11:05:04.851 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-10-01T11:05:04.851 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.851 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.851 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:05:04.852 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:58.265302+0000", 2023-10-01T11:05:04.852 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.5597999999999999e-05 2023-10-01T11:05:04.852 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.852 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.852 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:05:04.852 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:58.265328+0000", 2023-10-01T11:05:04.852 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.5622e-05 2023-10-01T11:05:04.852 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.852 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.852 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:05:04.852 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:58.265340+0000", 2023-10-01T11:05:04.852 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2354000000000001e-05 2023-10-01T11:05:04.853 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.853 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.853 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:05:04.853 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:58.265419+0000", 2023-10-01T11:05:04.853 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.9115000000000004e-05 2023-10-01T11:05:04.853 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.853 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.853 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:05:04.853 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:58.265466+0000", 2023-10-01T11:05:04.853 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.6684999999999998e-05 2023-10-01T11:05:04.853 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.853 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.853 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:05:04.854 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:58.273487+0000", 2023-10-01T11:05:04.854 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0080205090000000003 2023-10-01T11:05:04.854 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.854 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.854 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:05:04.854 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:58.273537+0000", 2023-10-01T11:05:04.854 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.0367000000000001e-05 2023-10-01T11:05:04.854 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:04.854 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:05:04.854 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:04.854 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.854 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.855 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.46454.0:84051 13.f e321/304 13:f8fea293:::benchmark_data_smithi078_202524_object4493:head v 321'5364, mlcod=0'0)", 2023-10-01T11:05:04.855 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:05:03.375349+0000", 2023-10-01T11:05:04.855 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 1.337463895, 2023-10-01T11:05:04.855 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.010616769999999999, 2023-10-01T11:05:04.855 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:05:04.855 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:05:04.855 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:05:04.855 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.855 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:05:04.855 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:03.375349+0000", 2023-10-01T11:05:04.855 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:05:04.855 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.856 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.856 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:05:04.856 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:03.375349+0000", 2023-10-01T11:05:04.856 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:05:04.856 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.856 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.856 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:05:04.856 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:03.375348+0000", 2023-10-01T11:05:04.856 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-10-01T11:05:04.856 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.856 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.856 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:05:04.857 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:03.375362+0000", 2023-10-01T11:05:04.857 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4062e-05 2023-10-01T11:05:04.857 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.857 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.857 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:05:04.857 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:03.375378+0000", 2023-10-01T11:05:04.857 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.6404000000000001e-05 2023-10-01T11:05:04.857 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.857 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.857 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:05:04.857 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:03.375387+0000", 2023-10-01T11:05:04.857 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.4640000000000006e-06 2023-10-01T11:05:04.858 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.858 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.858 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:05:04.858 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:03.375460+0000", 2023-10-01T11:05:04.858 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.3652999999999994e-05 2023-10-01T11:05:04.858 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.858 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.858 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:05:04.858 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:03.375503+0000", 2023-10-01T11:05:04.858 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.2852000000000002e-05 2023-10-01T11:05:04.858 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.858 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.859 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:05:04.859 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:03.385923+0000", 2023-10-01T11:05:04.859 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.010419856 2023-10-01T11:05:04.859 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:04.859 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:04.859 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:05:04.859 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:03.385966+0000", 2023-10-01T11:05:04.859 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.2669000000000001e-05 2023-10-01T11:05:04.859 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:04.859 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:05:04.859 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:04.859 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:04.860 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:05:04.860 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-10-01T11:05:04.860 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:04.885 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:04.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:04.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:05:04.987 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:05.016 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:05.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:05.019 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:05:05.120 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:05.144 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:05.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:05.146 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:05:05.247 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:05.269 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:05.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:05.273 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:05:05.373 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:05.396 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:05.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:05.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:05:05.499 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:05.526 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:05.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:05.529 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:05.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:05 smithi078 bash[78274]: cluster 2023-10-01T11:05:04.270656+0000 mgr.y (mgr.25072) 1432 : cluster 0 pgmap v1516: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 102 op/s; 20578/225702 objects degraded (9.117%); 4756/225702 objects misplaced (2.107%); 192 KiB/s, 2 objects/s recovering 2023-10-01T11:05:05.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:05 smithi078 bash[75299]: cluster 2023-10-01T11:05:04.270656+0000 mgr.y (mgr.25072) 1432 : cluster 0 pgmap v1516: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 102 op/s; 20578/225702 objects degraded (9.117%); 4756/225702 objects misplaced (2.107%); 192 KiB/s, 2 objects/s recovering 2023-10-01T11:05:05.629 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:05.656 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:05.657 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:05 smithi173 bash[58816]: cluster 2023-10-01T11:05:04.270656+0000 mgr.y (mgr.25072) 1432 : cluster 0 pgmap v1516: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 102 op/s; 20578/225702 objects degraded (9.117%); 4756/225702 objects misplaced (2.107%); 192 KiB/s, 2 objects/s recovering 2023-10-01T11:05:05.658 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-01T11:05:05.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:05.659 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:05:05.760 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:05.786 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:05.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:05.788 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:05:05.889 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:05.916 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:05.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:05.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:05:06.019 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:06.045 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:06.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:06.049 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:05:06.149 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:06.173 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:06.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:06.176 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:05:06.276 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:06.300 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:06.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:06.304 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:05:06.382 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T11:05:06.405 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:06.412 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T11:05:06.432 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:06.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:06.437 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:06.441 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T11:05:06.467 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T11:05:06.494 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T11:05:06.522 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T11:05:06.538 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:06.551 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T11:05:06.561 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:06.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:06.563 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:05:06.575 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T11:05:06.576 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T11:05:06.577 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T11:05:06.577 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T11:05:06.577 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T11:05:06.577 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T11:05:06.577 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T11:05:06.577 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T11:05:06.577 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T11:05:06.664 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:06.677 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:06 smithi173 bash[58816]: audit 2023-10-01T11:05:06.355142+0000 mon.c (mon.2) 215 : audit 0 from='client.? 172.21.15.78:0/1421969759' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:05:06.696 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:06.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:06.698 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:05:06.799 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:06.810 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:06 smithi078 bash[75299]: audit 2023-10-01T11:05:06.355142+0000 mon.c (mon.2) 215 : audit 0 from='client.? 172.21.15.78:0/1421969759' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:05:06.810 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:06 smithi078 bash[78274]: audit 2023-10-01T11:05:06.355142+0000 mon.c (mon.2) 215 : audit 0 from='client.? 172.21.15.78:0/1421969759' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:05:06.824 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:06.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:06.827 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:05:06.927 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:06.956 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:06.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:06.960 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:05:07.060 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:07.090 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:07.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:07.092 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:05:07.193 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:07.217 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:07.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:07.219 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:05:07.319 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:07.348 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:07.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:07.351 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:07.452 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:07.482 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:07.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:07.483 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:07.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:07 smithi078 bash[75299]: cluster 2023-10-01T11:05:06.271587+0000 mgr.y (mgr.25072) 1433 : cluster 0 pgmap v1517: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 106 op/s; 20578/225288 objects degraded (9.134%); 4723/225288 objects misplaced (2.096%); 403 KiB/s, 6 objects/s recovering 2023-10-01T11:05:07.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:07 smithi078 bash[78274]: cluster 2023-10-01T11:05:06.271587+0000 mgr.y (mgr.25072) 1433 : cluster 0 pgmap v1517: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 106 op/s; 20578/225288 objects degraded (9.134%); 4723/225288 objects misplaced (2.096%); 403 KiB/s, 6 objects/s recovering 2023-10-01T11:05:07.585 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:07.597 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:07 smithi173 bash[58816]: cluster 2023-10-01T11:05:06.271587+0000 mgr.y (mgr.25072) 1433 : cluster 0 pgmap v1517: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 106 op/s; 20578/225288 objects degraded (9.134%); 4723/225288 objects misplaced (2.096%); 403 KiB/s, 6 objects/s recovering 2023-10-01T11:05:07.612 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:07.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:07.615 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:07.715 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:07.743 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:07.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:07.746 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:05:07.847 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:07.876 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:07.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:07.879 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:05:07.980 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:08.010 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:08.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:08.014 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:05:08.114 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:08.141 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:08.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:08.144 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:05:08.244 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:08.270 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:08.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:08.272 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:05:08.373 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:08.403 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:08.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:08.407 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:05:08.508 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:08.533 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:08.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:08.536 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:05:08.580 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:05:08.636 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:08.657 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:08.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:08.659 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:05:08.760 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:08.767 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:08 smithi078 bash[78274]: cluster 2023-10-01T11:05:07.723619+0000 mon.a (mon.0) 1825 : cluster 3 Health check update: Degraded data redundancy: 20578/225288 objects degraded (9.134%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T11:05:08.767 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:08 smithi078 bash[75299]: cluster 2023-10-01T11:05:07.723619+0000 mon.a (mon.0) 1825 : cluster 3 Health check update: Degraded data redundancy: 20578/225288 objects degraded (9.134%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T11:05:08.780 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:08.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:08.782 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:05:08.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:08 smithi173 bash[58816]: cluster 2023-10-01T11:05:07.723619+0000 mon.a (mon.0) 1825 : cluster 3 Health check update: Degraded data redundancy: 20578/225288 objects degraded (9.134%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T11:05:08.883 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:08.910 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:08.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:08.912 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:05:09.013 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:09.042 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:09.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:09.045 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:05:09.146 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:09.169 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:09.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:09.175 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:09.276 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:09.305 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:09.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:09.307 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:05:09.408 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:09.430 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:09.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:09.435 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:05:09.539 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:09.572 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:09.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:09.576 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:05:09.677 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:09.711 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:09.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:09.713 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:05:09.814 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:09.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:09 smithi078 bash[78274]: cluster 2023-10-01T11:05:08.272229+0000 mgr.y (mgr.25072) 1434 : cluster 0 pgmap v1518: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 101 op/s; 20578/224778 objects degraded (9.155%); 4723/224778 objects misplaced (2.101%); 403 KiB/s, 6 objects/s recovering 2023-10-01T11:05:09.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:09 smithi078 bash[75299]: cluster 2023-10-01T11:05:08.272229+0000 mgr.y (mgr.25072) 1434 : cluster 0 pgmap v1518: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 101 op/s; 20578/224778 objects degraded (9.155%); 4723/224778 objects misplaced (2.101%); 403 KiB/s, 6 objects/s recovering 2023-10-01T11:05:09.845 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:09.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:09.849 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:05:09.950 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:09.957 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:09 smithi173 bash[58816]: cluster 2023-10-01T11:05:08.272229+0000 mgr.y (mgr.25072) 1434 : cluster 0 pgmap v1518: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 101 op/s; 20578/224778 objects degraded (9.155%); 4723/224778 objects misplaced (2.101%); 403 KiB/s, 6 objects/s recovering 2023-10-01T11:05:09.975 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:09.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:09.980 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:10.080 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:10.096 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T11:05:10.107 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:10.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:10.112 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:05:10.214 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:10.245 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:10.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:10.249 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:10.349 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:10.359 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:05:10 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:05:10] "GET /metrics HTTP/1.1" 200 36404 "" "Prometheus/2.33.4" 2023-10-01T11:05:10.372 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:10.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:10.375 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:05:10.426 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T11:05:10.475 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:10.501 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:10.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:10.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:05:10.604 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:10.627 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:10.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:10.630 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:05:10.731 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:10.753 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:10.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:10.756 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:10.856 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:10.888 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:10.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:10.891 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:05:10.992 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:11.017 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:11.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:11.020 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:05:11.121 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:11.144 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:11.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:11.146 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:11.247 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:11.266 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:11.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:11.269 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:05:11.349 INFO:teuthology.orchestra.run.smithi078.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-10-01T11:05:11.370 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:11.393 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:11.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:11.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:05:11.496 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:11.524 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:11.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:11.528 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:05:11.629 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:11.651 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:11.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:11.654 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:05:11.754 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:11.776 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:11.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:11.778 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:05:11.783 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:11 smithi173 bash[58816]: cluster 2023-10-01T11:05:10.272811+0000 mgr.y (mgr.25072) 1435 : cluster 0 pgmap v1519: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 100 op/s; 20578/223902 objects degraded (9.191%); 4692/223902 objects misplaced (2.096%); 531 KiB/s, 8 objects/s recovering 2023-10-01T11:05:11.783 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:11 smithi173 bash[58816]: audit 2023-10-01T11:05:11.346867+0000 mon.a (mon.0) 1826 : audit 1 from='client.? 172.21.15.78:0/214223755' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "21"}]: dispatch 2023-10-01T11:05:11.879 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:11.913 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:11.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:11.915 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:05:12.016 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:12.024 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:11 smithi078 bash[75299]: cluster 2023-10-01T11:05:10.272811+0000 mgr.y (mgr.25072) 1435 : cluster 0 pgmap v1519: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 100 op/s; 20578/223902 objects degraded (9.191%); 4692/223902 objects misplaced (2.096%); 531 KiB/s, 8 objects/s recovering 2023-10-01T11:05:12.024 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:11 smithi078 bash[75299]: audit 2023-10-01T11:05:11.346867+0000 mon.a (mon.0) 1826 : audit 1 from='client.? 172.21.15.78:0/214223755' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "21"}]: dispatch 2023-10-01T11:05:12.024 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:11 smithi078 bash[78274]: cluster 2023-10-01T11:05:10.272811+0000 mgr.y (mgr.25072) 1435 : cluster 0 pgmap v1519: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 100 op/s; 20578/223902 objects degraded (9.191%); 4692/223902 objects misplaced (2.096%); 531 KiB/s, 8 objects/s recovering 2023-10-01T11:05:12.024 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:11 smithi078 bash[78274]: audit 2023-10-01T11:05:11.346867+0000 mon.a (mon.0) 1826 : audit 1 from='client.? 172.21.15.78:0/214223755' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "21"}]: dispatch 2023-10-01T11:05:12.037 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:12.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:12.040 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:05:12.141 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:12.169 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:12.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:12.172 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:12.273 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:12.295 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:12.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:12.299 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:12.400 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:12.424 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:12.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:12.427 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:05:12.528 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:12.556 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:12.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:12.558 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:05:12.659 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:12.688 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:12.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:12.691 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:05:12.792 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:12.819 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:12.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:12.821 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:05:12.922 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:12.947 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:12.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:12.949 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:13.050 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:13.061 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:12 smithi173 bash[58816]: cluster 2023-10-01T11:05:12.725432+0000 mon.a (mon.0) 1827 : cluster 3 Health check update: Degraded data redundancy: 20578/223902 objects degraded (9.191%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T11:05:13.076 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:13.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:13.078 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:13.080 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:12 smithi078 bash[78274]: cluster 2023-10-01T11:05:12.725432+0000 mon.a (mon.0) 1827 : cluster 3 Health check update: Degraded data redundancy: 20578/223902 objects degraded (9.191%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T11:05:13.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:12 smithi078 bash[75299]: cluster 2023-10-01T11:05:12.725432+0000 mon.a (mon.0) 1827 : cluster 3 Health check update: Degraded data redundancy: 20578/223902 objects degraded (9.191%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T11:05:13.179 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:13.201 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:13.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:13.205 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:05:13.306 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:13.330 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:13.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:13.333 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:05:13.433 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:13.455 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:13.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:13.458 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:05:13.559 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:13.579 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:13.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:13.581 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:13.682 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:13.709 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:13.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:13.715 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:05:13.815 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:13.816 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:13 smithi078 bash[75299]: cluster 2023-10-01T11:05:12.273676+0000 mgr.y (mgr.25072) 1436 : cluster 0 pgmap v1520: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 81 op/s; 20578/223902 objects degraded (9.191%); 4692/223902 objects misplaced (2.096%); 341 KiB/s, 5 objects/s recovering 2023-10-01T11:05:13.816 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:13 smithi078 bash[75299]: audit 2023-10-01T11:05:13.164395+0000 mon.a (mon.0) 1828 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T11:05:13.838 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:13.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:13.842 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:05:13.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:13 smithi173 bash[58816]: cluster 2023-10-01T11:05:12.273676+0000 mgr.y (mgr.25072) 1436 : cluster 0 pgmap v1520: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 81 op/s; 20578/223902 objects degraded (9.191%); 4692/223902 objects misplaced (2.096%); 341 KiB/s, 5 objects/s recovering 2023-10-01T11:05:13.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:13 smithi173 bash[58816]: audit 2023-10-01T11:05:13.164395+0000 mon.a (mon.0) 1828 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T11:05:13.943 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:13.962 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:13.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:13.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:05:14.065 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:14.073 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:13 smithi078 bash[78274]: cluster 2023-10-01T11:05:12.273676+0000 mgr.y (mgr.25072) 1436 : cluster 0 pgmap v1520: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 81 op/s; 20578/223902 objects degraded (9.191%); 4692/223902 objects misplaced (2.096%); 341 KiB/s, 5 objects/s recovering 2023-10-01T11:05:14.073 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:13 smithi078 bash[78274]: audit 2023-10-01T11:05:13.164395+0000 mon.a (mon.0) 1828 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T11:05:14.089 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:14.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:14.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:05:14.191 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:14.206 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-10-01T11:05:14.220 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:14.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:14.223 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:05:14.293 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-10-01T11:05:14.293 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T11:05:14.323 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:14.348 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:14.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:14.351 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:05:14.451 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:14.480 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:14.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:14.484 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:05:14.584 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:14.609 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:14.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:14.611 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:05:14.712 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:14.740 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:14.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:14.742 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:05:14.843 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:14.871 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:14.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:14.873 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:05:14.974 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:15.001 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:15.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:15.004 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:05:15.104 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:15.129 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:15.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:15.132 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:15.233 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:15.256 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:15.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:15.261 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:05:15.267 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-01T11:05:15.362 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:15.388 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:15.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:15.390 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:05:15.491 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:15.519 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:15.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:15.522 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:05:15.623 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:15.653 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:15.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:15.656 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:05:15.757 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:15.758 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:15 smithi173 bash[58816]: cluster 2023-10-01T11:05:14.274946+0000 mgr.y (mgr.25072) 1437 : cluster 0 pgmap v1521: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 113 op/s; 20578/222735 objects degraded (9.239%); 4692/222735 objects misplaced (2.107%); 341 KiB/s, 5 objects/s recovering 2023-10-01T11:05:15.758 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:15 smithi173 bash[58816]: audit 2023-10-01T11:05:15.461680+0000 mon.c (mon.2) 216 : audit 1 from='client.? 172.21.15.78:0/3505663335' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:05:15.758 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:15 smithi173 bash[58816]: audit 2023-10-01T11:05:15.462137+0000 mon.a (mon.0) 1829 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:05:15.764 INFO:teuthology.orchestra.run.smithi078.stderr:nodeep-scrub is set 2023-10-01T11:05:15.783 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:15.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:15.785 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:05:15.886 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:15.898 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:15 smithi078 bash[75299]: cluster 2023-10-01T11:05:14.274946+0000 mgr.y (mgr.25072) 1437 : cluster 0 pgmap v1521: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 113 op/s; 20578/222735 objects degraded (9.239%); 4692/222735 objects misplaced (2.107%); 341 KiB/s, 5 objects/s recovering 2023-10-01T11:05:15.898 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:15 smithi078 bash[75299]: audit 2023-10-01T11:05:15.461680+0000 mon.c (mon.2) 216 : audit 1 from='client.? 172.21.15.78:0/3505663335' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:05:15.898 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:15 smithi078 bash[75299]: audit 2023-10-01T11:05:15.462137+0000 mon.a (mon.0) 1829 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:05:15.898 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:15 smithi078 bash[78274]: cluster 2023-10-01T11:05:14.274946+0000 mgr.y (mgr.25072) 1437 : cluster 0 pgmap v1521: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 113 op/s; 20578/222735 objects degraded (9.239%); 4692/222735 objects misplaced (2.107%); 341 KiB/s, 5 objects/s recovering 2023-10-01T11:05:15.898 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:15 smithi078 bash[78274]: audit 2023-10-01T11:05:15.461680+0000 mon.c (mon.2) 216 : audit 1 from='client.? 172.21.15.78:0/3505663335' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:05:15.899 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:15 smithi078 bash[78274]: audit 2023-10-01T11:05:15.462137+0000 mon.a (mon.0) 1829 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:05:15.912 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:15.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:15.915 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:05:16.015 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:16.045 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:16.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:16.048 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:05:16.149 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:16.174 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:16.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:16.177 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:05:16.277 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:16.302 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:16.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:16.305 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:05:16.406 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:16.428 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:16.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:16.431 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:05:16.532 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:16.563 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:16.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:16.567 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:05:16.668 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:16.693 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:16.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:16.696 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:05:16.797 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:16.821 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:16.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:16.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:05:16.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:16 smithi078 bash[75299]: cluster 2023-10-01T11:05:15.745669+0000 mon.a (mon.0) 1830 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:05:16.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:16 smithi078 bash[75299]: audit 2023-10-01T11:05:15.753351+0000 mon.a (mon.0) 1831 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T11:05:16.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:16 smithi078 bash[75299]: cluster 2023-10-01T11:05:15.753432+0000 mon.a (mon.0) 1832 : cluster 0 osdmap e322: 8 total, 8 up, 6 in 2023-10-01T11:05:16.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:16 smithi078 bash[75299]: cluster 2023-10-01T11:05:15.755300+0000 osd.5 (osd.5) 334 : cluster 3 failed to encode map e322 with expected crc 2023-10-01T11:05:16.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:16 smithi078 bash[75299]: cluster 2023-10-01T11:05:15.764755+0000 osd.7 (osd.7) 346 : cluster 3 failed to encode map e322 with expected crc 2023-10-01T11:05:16.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:16 smithi078 bash[75299]: cluster 2023-10-01T11:05:15.912956+0000 osd.6 (osd.6) 297 : cluster 3 failed to encode map e322 with expected crc 2023-10-01T11:05:16.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:16 smithi078 bash[78274]: cluster 2023-10-01T11:05:15.745669+0000 mon.a (mon.0) 1830 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:05:16.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:16 smithi078 bash[78274]: audit 2023-10-01T11:05:15.753351+0000 mon.a (mon.0) 1831 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T11:05:16.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:16 smithi078 bash[78274]: cluster 2023-10-01T11:05:15.753432+0000 mon.a (mon.0) 1832 : cluster 0 osdmap e322: 8 total, 8 up, 6 in 2023-10-01T11:05:16.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:16 smithi078 bash[78274]: cluster 2023-10-01T11:05:15.755300+0000 osd.5 (osd.5) 334 : cluster 3 failed to encode map e322 with expected crc 2023-10-01T11:05:16.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:16 smithi078 bash[78274]: cluster 2023-10-01T11:05:15.764755+0000 osd.7 (osd.7) 346 : cluster 3 failed to encode map e322 with expected crc 2023-10-01T11:05:16.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:16 smithi078 bash[78274]: cluster 2023-10-01T11:05:15.912956+0000 osd.6 (osd.6) 297 : cluster 3 failed to encode map e322 with expected crc 2023-10-01T11:05:16.924 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:16.948 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:16.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:16.950 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:05:17.050 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:17.063 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:16 smithi173 bash[58816]: cluster 2023-10-01T11:05:15.745669+0000 mon.a (mon.0) 1830 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:05:17.063 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:16 smithi173 bash[58816]: audit 2023-10-01T11:05:15.753351+0000 mon.a (mon.0) 1831 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T11:05:17.063 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:16 smithi173 bash[58816]: cluster 2023-10-01T11:05:15.753432+0000 mon.a (mon.0) 1832 : cluster 0 osdmap e322: 8 total, 8 up, 6 in 2023-10-01T11:05:17.063 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:16 smithi173 bash[58816]: cluster 2023-10-01T11:05:15.755300+0000 osd.5 (osd.5) 334 : cluster 3 failed to encode map e322 with expected crc 2023-10-01T11:05:17.063 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:16 smithi173 bash[58816]: cluster 2023-10-01T11:05:15.764755+0000 osd.7 (osd.7) 346 : cluster 3 failed to encode map e322 with expected crc 2023-10-01T11:05:17.063 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:16 smithi173 bash[58816]: cluster 2023-10-01T11:05:15.912956+0000 osd.6 (osd.6) 297 : cluster 3 failed to encode map e322 with expected crc 2023-10-01T11:05:17.078 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:17.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:17.082 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:05:17.182 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:17.210 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:17.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:17.213 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:05:17.313 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:17.346 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:17.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:17.349 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:05:17.450 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:17.481 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:17.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:17.485 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:05:17.585 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:17.610 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:17.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:17.613 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:05:17.714 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:17.744 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:17.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:17.746 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:05:17.847 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:17.856 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:17 smithi173 bash[58816]: cluster 2023-10-01T11:05:16.275563+0000 mgr.y (mgr.25072) 1438 : cluster 0 pgmap v1523: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 101 op/s; 20578/222243 objects degraded (9.259%); 4660/222243 objects misplaced (2.097%); 403 KiB/s, 6 objects/s recovering 2023-10-01T11:05:17.856 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:17 smithi173 bash[58816]: audit 2023-10-01T11:05:17.459857+0000 mon.a (mon.0) 1833 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:05:17.856 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:17 smithi173 bash[58816]: cluster 2023-10-01T11:05:17.727078+0000 mon.a (mon.0) 1834 : cluster 3 Health check update: Degraded data redundancy: 20578/222243 objects degraded (9.259%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T11:05:17.869 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:17.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:17.875 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:17.976 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:18.008 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:18.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:18.010 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:05:18.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:17 smithi078 bash[75299]: cluster 2023-10-01T11:05:16.275563+0000 mgr.y (mgr.25072) 1438 : cluster 0 pgmap v1523: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 101 op/s; 20578/222243 objects degraded (9.259%); 4660/222243 objects misplaced (2.097%); 403 KiB/s, 6 objects/s recovering 2023-10-01T11:05:18.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:17 smithi078 bash[75299]: audit 2023-10-01T11:05:17.459857+0000 mon.a (mon.0) 1833 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:05:18.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:17 smithi078 bash[75299]: cluster 2023-10-01T11:05:17.727078+0000 mon.a (mon.0) 1834 : cluster 3 Health check update: Degraded data redundancy: 20578/222243 objects degraded (9.259%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T11:05:18.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:17 smithi078 bash[78274]: cluster 2023-10-01T11:05:16.275563+0000 mgr.y (mgr.25072) 1438 : cluster 0 pgmap v1523: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 101 op/s; 20578/222243 objects degraded (9.259%); 4660/222243 objects misplaced (2.097%); 403 KiB/s, 6 objects/s recovering 2023-10-01T11:05:18.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:17 smithi078 bash[78274]: audit 2023-10-01T11:05:17.459857+0000 mon.a (mon.0) 1833 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:05:18.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:17 smithi078 bash[78274]: cluster 2023-10-01T11:05:17.727078+0000 mon.a (mon.0) 1834 : cluster 3 Health check update: Degraded data redundancy: 20578/222243 objects degraded (9.259%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T11:05:18.111 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:18.141 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:18.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:18.145 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:05:18.245 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:18.274 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:18.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:18.277 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:05:18.378 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:18.410 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:18.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:18.413 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:05:18.514 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:18.546 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:18.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:18.548 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:18.649 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:18.675 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:18.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:18.679 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:18.780 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:18.803 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:18.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:18.806 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:05:18.906 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:18.922 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T11:05:18.937 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:18.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:18.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:05:19.040 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:19.071 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:19.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:19.073 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:05:19.174 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:19.202 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:19.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:19.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:05:19.207 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5] out_osds: [6, 7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T11:05:19.207 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T11:05:19.207 INFO:tasks.thrashosds.thrasher:Adding osd 6 2023-10-01T11:05:19.208 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd in 6 2023-10-01T11:05:19.308 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:19.331 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:19.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:19.334 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:05:19.419 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:05:19.434 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:19.459 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:19.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:19.462 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:05:19.563 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:19.592 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:19.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:19.596 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:05:19.696 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:19.724 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:19.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:19.728 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:05:19.795 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T11:05:19.829 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:19.861 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:19.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:19.864 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:05:19.965 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:19.973 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:19 smithi173 bash[58816]: cluster 2023-10-01T11:05:18.276284+0000 mgr.y (mgr.25072) 1439 : cluster 0 pgmap v1524: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 100 op/s; 20578/221769 objects degraded (9.279%); 4660/221769 objects misplaced (2.101%); 403 KiB/s, 6 objects/s recovering 2023-10-01T11:05:19.993 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:19.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:19.995 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:20.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:19 smithi078 bash[75299]: cluster 2023-10-01T11:05:18.276284+0000 mgr.y (mgr.25072) 1439 : cluster 0 pgmap v1524: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 100 op/s; 20578/221769 objects degraded (9.279%); 4660/221769 objects misplaced (2.101%); 403 KiB/s, 6 objects/s recovering 2023-10-01T11:05:20.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:19 smithi078 bash[78274]: cluster 2023-10-01T11:05:18.276284+0000 mgr.y (mgr.25072) 1439 : cluster 0 pgmap v1524: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 100 op/s; 20578/221769 objects degraded (9.279%); 4660/221769 objects misplaced (2.101%); 403 KiB/s, 6 objects/s recovering 2023-10-01T11:05:20.095 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:20.125 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:20.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:20.130 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:05:20.230 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:20.258 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:20.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:20.260 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:05:20.361 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:20.386 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:20.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:20.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:05:20.490 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:20.515 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:20.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:20.518 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:05:20.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:05:20 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:05:20] "GET /metrics HTTP/1.1" 200 36404 "" "Prometheus/2.33.4" 2023-10-01T11:05:20.619 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:20.651 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:20.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:20.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:05:20.754 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:20.783 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:20.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:20.786 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:05:20.887 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:20.911 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:20.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:20.914 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:05:21.015 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:21.039 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:21.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:21.041 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:05:21.142 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:21.166 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:21.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:21.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:05:21.270 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:21.276 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:21 smithi078 bash[78274]: audit 2023-10-01T11:05:20.154192+0000 mon.a (mon.0) 1835 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:05:21.276 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:21 smithi078 bash[78274]: audit 2023-10-01T11:05:20.161006+0000 mon.a (mon.0) 1836 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:05:21.276 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:21 smithi078 bash[78274]: cluster 2023-10-01T11:05:20.277612+0000 mgr.y (mgr.25072) 1440 : cluster 0 pgmap v1525: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 98 op/s; 20578/220953 objects degraded (9.313%); 4629/220953 objects misplaced (2.095%); 403 KiB/s, 6 objects/s recovering 2023-10-01T11:05:21.277 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:21 smithi078 bash[78274]: audit 2023-10-01T11:05:20.860013+0000 mon.a (mon.0) 1837 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:05:21.277 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:21 smithi078 bash[78274]: audit 2023-10-01T11:05:20.883851+0000 mon.a (mon.0) 1838 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:05:21.277 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:21 smithi078 bash[75299]: audit 2023-10-01T11:05:20.154192+0000 mon.a (mon.0) 1835 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:05:21.277 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:21 smithi078 bash[75299]: audit 2023-10-01T11:05:20.161006+0000 mon.a (mon.0) 1836 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:05:21.277 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:21 smithi078 bash[75299]: cluster 2023-10-01T11:05:20.277612+0000 mgr.y (mgr.25072) 1440 : cluster 0 pgmap v1525: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 98 op/s; 20578/220953 objects degraded (9.313%); 4629/220953 objects misplaced (2.095%); 403 KiB/s, 6 objects/s recovering 2023-10-01T11:05:21.277 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:21 smithi078 bash[75299]: audit 2023-10-01T11:05:20.860013+0000 mon.a (mon.0) 1837 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:05:21.277 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:21 smithi078 bash[75299]: audit 2023-10-01T11:05:20.883851+0000 mon.a (mon.0) 1838 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:05:21.292 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:21.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:21.294 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:05:21.395 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:21.406 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:21 smithi173 bash[58816]: audit 2023-10-01T11:05:20.154192+0000 mon.a (mon.0) 1835 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:05:21.406 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:21 smithi173 bash[58816]: audit 2023-10-01T11:05:20.161006+0000 mon.a (mon.0) 1836 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:05:21.407 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:21 smithi173 bash[58816]: cluster 2023-10-01T11:05:20.277612+0000 mgr.y (mgr.25072) 1440 : cluster 0 pgmap v1525: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 98 op/s; 20578/220953 objects degraded (9.313%); 4629/220953 objects misplaced (2.095%); 403 KiB/s, 6 objects/s recovering 2023-10-01T11:05:21.407 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:21 smithi173 bash[58816]: audit 2023-10-01T11:05:20.860013+0000 mon.a (mon.0) 1837 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:05:21.407 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:21 smithi173 bash[58816]: audit 2023-10-01T11:05:20.883851+0000 mon.a (mon.0) 1838 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:05:21.425 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:21.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:21.428 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:21.528 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:21.554 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:21.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:21.556 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:05:21.657 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:21.684 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:21.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:21.686 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:05:21.787 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:21.807 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:21.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:21.811 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:05:21.912 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:21.936 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:21.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:21.938 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:22.039 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:22.067 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:22.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:22.070 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:05:22.170 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:22.202 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:22.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:22.205 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:05:22.305 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:22.330 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:22.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:22.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:05:22.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:22 smithi173 bash[58816]: audit 2023-10-01T11:05:21.356736+0000 mon.a (mon.0) 1839 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T11:05:22.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:22 smithi173 bash[58816]: audit 2023-10-01T11:05:21.358064+0000 mon.a (mon.0) 1840 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T11:05:22.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:22 smithi173 bash[58816]: audit 2023-10-01T11:05:21.368737+0000 mon.a (mon.0) 1841 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:05:22.433 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:22.446 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:22 smithi078 bash[78274]: audit 2023-10-01T11:05:21.356736+0000 mon.a (mon.0) 1839 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T11:05:22.446 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:22 smithi078 bash[78274]: audit 2023-10-01T11:05:21.358064+0000 mon.a (mon.0) 1840 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T11:05:22.447 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:22 smithi078 bash[78274]: audit 2023-10-01T11:05:21.368737+0000 mon.a (mon.0) 1841 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:05:22.447 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:22 smithi078 bash[75299]: audit 2023-10-01T11:05:21.356736+0000 mon.a (mon.0) 1839 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T11:05:22.447 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:22 smithi078 bash[75299]: audit 2023-10-01T11:05:21.358064+0000 mon.a (mon.0) 1840 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T11:05:22.447 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:22 smithi078 bash[75299]: audit 2023-10-01T11:05:21.368737+0000 mon.a (mon.0) 1841 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:05:22.461 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:22.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:22.464 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:05:22.565 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:22.588 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:22.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:22.591 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:22.691 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:22.720 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:22.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:22.722 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:05:22.823 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:22.852 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:22.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:22.856 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:05:22.957 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:22.985 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:22.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:22.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:05:23.089 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:23.113 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:23.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:23.116 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:05:23.217 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:23.239 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:23.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:23.242 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:05:23.342 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:23.355 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:23 smithi078 bash[75299]: cluster 2023-10-01T11:05:22.278127+0000 mgr.y (mgr.25072) 1441 : cluster 0 pgmap v1526: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 98 op/s; 20578/220953 objects degraded (9.313%); 4629/220953 objects misplaced (2.095%); 403 KiB/s, 6 objects/s recovering 2023-10-01T11:05:23.355 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:23 smithi078 bash[75299]: cluster 2023-10-01T11:05:22.728379+0000 mon.a (mon.0) 1842 : cluster 3 Health check update: Degraded data redundancy: 20578/220953 objects degraded (9.313%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T11:05:23.355 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:23 smithi078 bash[78274]: cluster 2023-10-01T11:05:22.278127+0000 mgr.y (mgr.25072) 1441 : cluster 0 pgmap v1526: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 98 op/s; 20578/220953 objects degraded (9.313%); 4629/220953 objects misplaced (2.095%); 403 KiB/s, 6 objects/s recovering 2023-10-01T11:05:23.356 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:23 smithi078 bash[78274]: cluster 2023-10-01T11:05:22.728379+0000 mon.a (mon.0) 1842 : cluster 3 Health check update: Degraded data redundancy: 20578/220953 objects degraded (9.313%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T11:05:23.369 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:23.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:23.371 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:05:23.472 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:23.485 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:05:23.486 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:05:23.501 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:23.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:23.504 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:05:23.604 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:23.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:23 smithi173 bash[58816]: cluster 2023-10-01T11:05:22.278127+0000 mgr.y (mgr.25072) 1441 : cluster 0 pgmap v1526: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 98 op/s; 20578/220953 objects degraded (9.313%); 4629/220953 objects misplaced (2.095%); 403 KiB/s, 6 objects/s recovering 2023-10-01T11:05:23.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:23 smithi173 bash[58816]: cluster 2023-10-01T11:05:22.728379+0000 mon.a (mon.0) 1842 : cluster 3 Health check update: Degraded data redundancy: 20578/220953 objects degraded (9.313%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T11:05:23.630 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:23.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:23.633 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:05:23.734 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:23.758 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:23.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:23.760 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:05:23.861 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:23.891 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:23.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:23.894 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:05:23.995 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:24.021 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:24.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:24.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:05:24.080 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-10-01T11:05:24.080 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T11:05:24.126 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:24.151 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:24.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:24.154 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:05:24.255 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:24.282 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:24.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:24.283 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:05:24.383 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:24.407 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:24.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:24.410 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:05:24.510 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:24.541 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:24.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:24.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:05:24.644 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:24.670 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:24.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:24.672 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:05:24.773 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:24.802 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:24.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:24.805 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:24.906 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:24.926 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:24.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:24.928 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:05:24.957 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-01T11:05:25.029 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:25.067 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:25.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:25.070 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:05:25.171 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:25.193 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:25.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:25.196 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:05:25.297 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:25.322 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:25.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:25.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:05:25.424 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:25.452 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:25 smithi078 bash[75299]: cluster 2023-10-01T11:05:24.278822+0000 mgr.y (mgr.25072) 1442 : cluster 0 pgmap v1527: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 84 op/s; 20578/220191 objects degraded (9.346%); 4629/220191 objects misplaced (2.102%); 403 KiB/s, 6 objects/s recovering 2023-10-01T11:05:25.452 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:25 smithi078 bash[78274]: cluster 2023-10-01T11:05:24.278822+0000 mgr.y (mgr.25072) 1442 : cluster 0 pgmap v1527: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 84 op/s; 20578/220191 objects degraded (9.346%); 4629/220191 objects misplaced (2.102%); 403 KiB/s, 6 objects/s recovering 2023-10-01T11:05:25.453 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:25.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:25.456 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:05:25.557 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:25.584 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:25.585 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:25 smithi173 bash[58816]: cluster 2023-10-01T11:05:24.278822+0000 mgr.y (mgr.25072) 1442 : cluster 0 pgmap v1527: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 84 op/s; 20578/220191 objects degraded (9.346%); 4629/220191 objects misplaced (2.102%); 403 KiB/s, 6 objects/s recovering 2023-10-01T11:05:25.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:25.586 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:25.687 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:25.691 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T11:05:25.718 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:25.718 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T11:05:25.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:25.719 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:05:25.741 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T11:05:25.768 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T11:05:25.792 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T11:05:25.818 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T11:05:25.820 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:25.844 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T11:05:25.850 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:25.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:25.853 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:05:25.870 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T11:05:25.872 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T11:05:25.872 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T11:05:25.872 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T11:05:25.872 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T11:05:25.872 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T11:05:25.872 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T11:05:25.872 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T11:05:25.872 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T11:05:25.954 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:25.979 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:25.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:25.980 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:05:26.081 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:26.108 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:26.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:26.111 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:05:26.213 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:26.240 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:26.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:26.242 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:05:26.343 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:26.368 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:26.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:26.371 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:05:26.472 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:26.483 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:26 smithi173 bash[58816]: audit 2023-10-01T11:05:25.666982+0000 mon.c (mon.2) 217 : audit 0 from='client.? 172.21.15.78:0/715158432' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:05:26.502 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:26.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:26.504 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:05:26.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:26 smithi078 bash[78274]: audit 2023-10-01T11:05:25.666982+0000 mon.c (mon.2) 217 : audit 0 from='client.? 172.21.15.78:0/715158432' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:05:26.584 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:26 smithi078 bash[75299]: audit 2023-10-01T11:05:25.666982+0000 mon.c (mon.2) 217 : audit 0 from='client.? 172.21.15.78:0/715158432' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:05:26.605 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:26.633 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:26.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:26.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:05:26.736 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:26.762 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:26.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:26.765 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:05:26.865 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:26.890 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:26.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:26.893 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:05:26.994 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:27.015 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:27.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:27.017 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:05:27.118 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:27.141 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:27.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:27.144 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:05:27.245 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:27.271 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:27.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:27.274 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:27.374 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:27.386 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:27 smithi173 bash[58816]: cluster 2023-10-01T11:05:26.279595+0000 mgr.y (mgr.25072) 1443 : cluster 0 pgmap v1528: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 78 op/s; 20578/219774 objects degraded (9.363%); 4598/219774 objects misplaced (2.092%); 377 KiB/s, 5 objects/s recovering 2023-10-01T11:05:27.399 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:27.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:27.402 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:05:27.502 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:27.525 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:27.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:27.528 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:05:27.629 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:27.636 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:27 smithi078 bash[75299]: cluster 2023-10-01T11:05:26.279595+0000 mgr.y (mgr.25072) 1443 : cluster 0 pgmap v1528: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 78 op/s; 20578/219774 objects degraded (9.363%); 4598/219774 objects misplaced (2.092%); 377 KiB/s, 5 objects/s recovering 2023-10-01T11:05:27.637 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:27 smithi078 bash[78274]: cluster 2023-10-01T11:05:26.279595+0000 mgr.y (mgr.25072) 1443 : cluster 0 pgmap v1528: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 78 op/s; 20578/219774 objects degraded (9.363%); 4598/219774 objects misplaced (2.092%); 377 KiB/s, 5 objects/s recovering 2023-10-01T11:05:27.653 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:27.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:27.655 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:05:27.756 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:27.782 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:27.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:27.784 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:05:27.885 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:27.908 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:27.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:27.911 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:05:28.012 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:28.042 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:28.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:28.045 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:05:28.145 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:28.168 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:28.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:28.170 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:05:28.276 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:28.299 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:28.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:28.303 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:05:28.404 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:28.431 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:28.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:28.433 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:28.534 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:28.560 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:28.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:28.562 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:05:28.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:28 smithi078 bash[75299]: cluster 2023-10-01T11:05:27.729794+0000 mon.a (mon.0) 1843 : cluster 3 Health check update: Degraded data redundancy: 20578/219774 objects degraded (9.363%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T11:05:28.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:28 smithi078 bash[78274]: cluster 2023-10-01T11:05:27.729794+0000 mon.a (mon.0) 1843 : cluster 3 Health check update: Degraded data redundancy: 20578/219774 objects degraded (9.363%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T11:05:28.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:28 smithi173 bash[58816]: cluster 2023-10-01T11:05:27.729794+0000 mon.a (mon.0) 1843 : cluster 3 Health check update: Degraded data redundancy: 20578/219774 objects degraded (9.363%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T11:05:28.664 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:28.693 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:28.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:28.695 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:05:28.796 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:28.824 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:28.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:28.827 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:28.928 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:28.953 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:28.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:28.956 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:05:29.057 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:29.086 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:29.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:29.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:05:29.189 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:29.223 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:29.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:29.226 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:05:29.326 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:29.357 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:29.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:29.360 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:29.461 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:29.486 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:29.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:29.488 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:05:29.589 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:29.612 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:29.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:29.614 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:05:29.715 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:29.740 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:29.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:29.743 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:05:29.770 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T11:05:29.843 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:29.866 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:29.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:29.868 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:05:29.969 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:29.983 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:29 smithi078 bash[75299]: cluster 2023-10-01T11:05:28.280468+0000 mgr.y (mgr.25072) 1444 : cluster 0 pgmap v1529: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 84 op/s; 20578/219198 objects degraded (9.388%); 4598/219198 objects misplaced (2.098%); 331 KiB/s, 5 objects/s recovering 2023-10-01T11:05:29.984 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:29 smithi078 bash[78274]: cluster 2023-10-01T11:05:28.280468+0000 mgr.y (mgr.25072) 1444 : cluster 0 pgmap v1529: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 84 op/s; 20578/219198 objects degraded (9.388%); 4598/219198 objects misplaced (2.098%); 331 KiB/s, 5 objects/s recovering 2023-10-01T11:05:29.992 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:29.994 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:29 smithi173 bash[58816]: cluster 2023-10-01T11:05:28.280468+0000 mgr.y (mgr.25072) 1444 : cluster 0 pgmap v1529: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 84 op/s; 20578/219198 objects degraded (9.388%); 4598/219198 objects misplaced (2.098%); 331 KiB/s, 5 objects/s recovering 2023-10-01T11:05:29.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:29.995 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:05:30.068 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T11:05:30.095 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:30.139 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:30.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:30.142 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:05:30.243 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:30.271 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:30.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:30.274 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:05:30.375 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:30.405 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:30.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:30.408 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:05:30.508 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:30.539 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:30.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:30.542 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:30.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:05:30 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:05:30] "GET /metrics HTTP/1.1" 200 36404 "" "Prometheus/2.33.4" 2023-10-01T11:05:30.643 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:30.668 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:30.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:30.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:05:30.770 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:30.795 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:30.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:30.798 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:05:30.898 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:30.924 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:30.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:30.927 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:31.028 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:31.051 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:31.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:31.053 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:05:31.153 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:31.173 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:31.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:31.176 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:05:31.276 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:31.299 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:31.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:31.302 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:05:31.403 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:31.433 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:31.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:31.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:05:31.536 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:31.564 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:31.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:31.566 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:05:31.667 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:31.696 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:31.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:31.699 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:05:31.800 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:31.826 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:31.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:31.828 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:05:31.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:31 smithi078 bash[78274]: cluster 2023-10-01T11:05:30.281628+0000 mgr.y (mgr.25072) 1445 : cluster 0 pgmap v1530: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 100 op/s; 20578/218166 objects degraded (9.432%); 4565/218166 objects misplaced (2.092%); 506 KiB/s, 7 objects/s recovering 2023-10-01T11:05:31.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:31 smithi078 bash[75299]: cluster 2023-10-01T11:05:30.281628+0000 mgr.y (mgr.25072) 1445 : cluster 0 pgmap v1530: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 100 op/s; 20578/218166 objects degraded (9.432%); 4565/218166 objects misplaced (2.092%); 506 KiB/s, 7 objects/s recovering 2023-10-01T11:05:31.928 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:31.959 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:31.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:31.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:05:32.062 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:32.087 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:32.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:32.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:05:32.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:31 smithi173 bash[58816]: cluster 2023-10-01T11:05:30.281628+0000 mgr.y (mgr.25072) 1445 : cluster 0 pgmap v1530: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 100 op/s; 20578/218166 objects degraded (9.432%); 4565/218166 objects misplaced (2.092%); 506 KiB/s, 7 objects/s recovering 2023-10-01T11:05:32.190 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:32.221 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:32.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:32.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:05:32.325 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:32.349 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:32.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:32.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:05:32.452 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:32.483 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:32.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:32.485 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:05:32.586 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:32.615 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:32.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:32.618 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:05:32.718 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:32.730 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:32 smithi078 bash[75299]: debug 2023-10-01T11:05:32.543+0000 7f31c8dd0700 -1 mon.a@0(leader).osd e322 definitely_dead 0 2023-10-01T11:05:32.749 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:32 smithi173 bash[58816]: audit 2023-10-01T11:05:32.398136+0000 mon.c (mon.2) 218 : audit 1 from='client.? 172.21.15.78:0/865129230' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T11:05:32.750 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:32 smithi173 bash[58816]: audit 2023-10-01T11:05:32.398785+0000 mon.a (mon.0) 1844 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T11:05:32.750 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:32 smithi173 bash[58816]: audit 2023-10-01T11:05:32.460734+0000 mon.a (mon.0) 1845 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:05:32.750 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:32 smithi173 bash[58816]: audit 2023-10-01T11:05:32.545469+0000 mon.c (mon.2) 219 : audit 0 from='client.? 172.21.15.78:0/2056886966' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-01T11:05:32.750 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:32 smithi173 bash[58816]: audit 2023-10-01T11:05:32.546079+0000 mon.a (mon.0) 1846 : audit 0 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-01T11:05:32.750 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:32 smithi173 bash[58816]: cluster 2023-10-01T11:05:32.732442+0000 mon.a (mon.0) 1847 : cluster 3 Health check update: Degraded data redundancy: 20578/218166 objects degraded (9.432%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T11:05:32.751 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:32.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:32.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:05:32.777 INFO:teuthology.orchestra.run.smithi078.stderr:noscrub is unset 2023-10-01T11:05:32.782 INFO:teuthology.orchestra.run.smithi078.stderr:marked in osd.6. 2023-10-01T11:05:32.856 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:32.882 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:32.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:32.885 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:05:32.986 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:32.998 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:32 smithi078 bash[78274]: audit 2023-10-01T11:05:32.398136+0000 mon.c (mon.2) 218 : audit 1 from='client.? 172.21.15.78:0/865129230' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T11:05:32.998 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:32 smithi078 bash[78274]: audit 2023-10-01T11:05:32.398785+0000 mon.a (mon.0) 1844 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T11:05:32.998 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:32 smithi078 bash[78274]: audit 2023-10-01T11:05:32.460734+0000 mon.a (mon.0) 1845 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:05:32.999 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:32 smithi078 bash[78274]: audit 2023-10-01T11:05:32.545469+0000 mon.c (mon.2) 219 : audit 0 from='client.? 172.21.15.78:0/2056886966' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-01T11:05:32.999 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:32 smithi078 bash[78274]: audit 2023-10-01T11:05:32.546079+0000 mon.a (mon.0) 1846 : audit 0 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-01T11:05:32.999 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:32 smithi078 bash[78274]: cluster 2023-10-01T11:05:32.732442+0000 mon.a (mon.0) 1847 : cluster 3 Health check update: Degraded data redundancy: 20578/218166 objects degraded (9.432%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T11:05:32.999 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:32 smithi078 bash[75299]: audit 2023-10-01T11:05:32.398136+0000 mon.c (mon.2) 218 : audit 1 from='client.? 172.21.15.78:0/865129230' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T11:05:32.999 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:32 smithi078 bash[75299]: audit 2023-10-01T11:05:32.398785+0000 mon.a (mon.0) 1844 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T11:05:33.000 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:32 smithi078 bash[75299]: audit 2023-10-01T11:05:32.460734+0000 mon.a (mon.0) 1845 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:05:33.000 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:32 smithi078 bash[75299]: audit 2023-10-01T11:05:32.545469+0000 mon.c (mon.2) 219 : audit 0 from='client.? 172.21.15.78:0/2056886966' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-01T11:05:33.000 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:32 smithi078 bash[75299]: audit 2023-10-01T11:05:32.546079+0000 mon.a (mon.0) 1846 : audit 0 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-10-01T11:05:33.000 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:32 smithi078 bash[75299]: cluster 2023-10-01T11:05:32.732442+0000 mon.a (mon.0) 1847 : cluster 3 Health check update: Degraded data redundancy: 20578/218166 objects degraded (9.432%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T11:05:33.016 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:33.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:33.019 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:05:33.120 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:33.150 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:33.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:33.152 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:05:33.253 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:33.284 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:33.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:33.286 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:05:33.387 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:33.418 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:33.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:33.419 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:05:33.520 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:33.551 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:33.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:33.553 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:05:33.654 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:33.682 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:33.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:33.685 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:05:33.786 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:33.817 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-10-01T11:05:33.818 INFO:teuthology.orchestra.run.smithi173.stdout: "size": 20, 2023-10-01T11:05:33.818 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 600, 2023-10-01T11:05:33.818 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [ 2023-10-01T11:05:33.818 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.818 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46232.0:191209 12.f 12:fc4afb62:::benchmark_data_smithi078_180197_object88078:head [delete] snapc 0=[] ondisk+write+known_if_redirected e255)", 2023-10-01T11:05:33.818 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:00:28.822223+0000", 2023-10-01T11:05:33.818 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 304.98439976999998, 2023-10-01T11:05:33.818 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.4949136900000002, 2023-10-01T11:05:33.818 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:05:33.818 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:05:33.818 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:05:33.819 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46232", 2023-10-01T11:05:33.819 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/711982100", 2023-10-01T11:05:33.819 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 191209 2023-10-01T11:05:33.819 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.819 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:05:33.819 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.819 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:05:33.819 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:28.822223+0000", 2023-10-01T11:05:33.819 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:05:33.819 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.819 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.819 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:05:33.820 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:28.822223+0000", 2023-10-01T11:05:33.820 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-10-01T11:05:33.820 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.820 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.820 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:05:33.820 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:28.822221+0000", 2023-10-01T11:05:33.820 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.7020000000000001e-06 2023-10-01T11:05:33.820 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.820 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.820 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:05:33.820 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:28.822229+0000", 2023-10-01T11:05:33.820 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0546e-05 2023-10-01T11:05:33.821 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.821 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.821 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:05:33.821 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:28.822239+0000", 2023-10-01T11:05:33.821 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.8180000000000004e-06 2023-10-01T11:05:33.821 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.821 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.821 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:05:33.821 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:28.822247+0000", 2023-10-01T11:05:33.821 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00049260299999999999 2023-10-01T11:05:33.821 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.821 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.821 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:05:33.822 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:28.822740+0000", 2023-10-01T11:05:33.822 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1816e-05 2023-10-01T11:05:33.822 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.822 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.822 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:05:33.822 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:28.822752+0000", 2023-10-01T11:05:33.822 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.002766526 2023-10-01T11:05:33.822 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.822 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.822 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:05:33.822 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:29.825518+0000", 2023-10-01T11:05:33.823 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.6858e-05 2023-10-01T11:05:33.823 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.823 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.823 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:05:33.823 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:29.825535+0000", 2023-10-01T11:05:33.823 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.210420759 2023-10-01T11:05:33.823 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.823 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.823 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:05:33.823 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.035956+0000", 2023-10-01T11:05:33.823 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0039583359999999998 2023-10-01T11:05:33.823 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.823 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.824 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:05:33.824 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.039914+0000", 2023-10-01T11:05:33.824 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000199135 2023-10-01T11:05:33.824 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.824 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.824 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 2,4,6", 2023-10-01T11:05:33.824 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.040113+0000", 2023-10-01T11:05:33.824 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0052214319999999998 2023-10-01T11:05:33.824 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.824 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.824 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:33.824 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.045335+0000", 2023-10-01T11:05:33.824 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0053388359999999996 2023-10-01T11:05:33.825 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.825 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.825 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:33.825 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.050674+0000", 2023-10-01T11:05:33.825 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.7119999999999995e-06 2023-10-01T11:05:33.825 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.825 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.825 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:05:33.825 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.050683+0000", 2023-10-01T11:05:33.825 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.26624254200000003 2023-10-01T11:05:33.825 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.825 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.826 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:33.826 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.316926+0000", 2023-10-01T11:05:33.826 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.1984999999999997e-05 2023-10-01T11:05:33.826 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.826 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.826 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:05:33.826 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.316988+0000", 2023-10-01T11:05:33.826 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00014912200000000001 2023-10-01T11:05:33.826 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.826 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.826 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:05:33.826 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.317137+0000", 2023-10-01T11:05:33.827 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.4949136900000002 2023-10-01T11:05:33.827 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:33.827 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:05:33.827 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:33.827 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.827 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.827 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46232.0:191218 12.f 12:fce854b3:::benchmark_data_smithi078_180197_object88087:head [delete] snapc 0=[] ondisk+write+known_if_redirected e255)", 2023-10-01T11:05:33.827 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:00:28.837248+0000", 2023-10-01T11:05:33.827 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 304.96937458500003, 2023-10-01T11:05:33.827 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.489574401, 2023-10-01T11:05:33.827 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:05:33.827 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:05:33.827 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:05:33.828 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46232", 2023-10-01T11:05:33.828 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/711982100", 2023-10-01T11:05:33.828 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 191218 2023-10-01T11:05:33.828 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.828 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:05:33.828 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.828 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:05:33.828 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:28.837248+0000", 2023-10-01T11:05:33.828 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:05:33.828 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.828 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.828 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:05:33.829 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:28.837248+0000", 2023-10-01T11:05:33.829 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999976 2023-10-01T11:05:33.829 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.829 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.829 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:05:33.829 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:28.837246+0000", 2023-10-01T11:05:33.829 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.3779000000000001e-05 2023-10-01T11:05:33.829 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.829 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.829 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:05:33.829 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:28.837270+0000", 2023-10-01T11:05:33.829 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.6537000000000001e-05 2023-10-01T11:05:33.829 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.830 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.830 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:05:33.830 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:28.837286+0000", 2023-10-01T11:05:33.830 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.4369999999999996e-06 2023-10-01T11:05:33.830 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.830 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.830 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:05:33.830 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:28.837296+0000", 2023-10-01T11:05:33.830 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00010752900000000001 2023-10-01T11:05:33.830 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.830 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.830 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:05:33.831 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:28.837403+0000", 2023-10-01T11:05:33.831 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.8538000000000001e-05 2023-10-01T11:05:33.831 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.831 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.831 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:05:33.831 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:28.837422+0000", 2023-10-01T11:05:33.831 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.98814465399999996 2023-10-01T11:05:33.831 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.831 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.831 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:05:33.831 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:29.825566+0000", 2023-10-01T11:05:33.831 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1473e-05 2023-10-01T11:05:33.832 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.832 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.832 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:05:33.832 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:29.825578+0000", 2023-10-01T11:05:33.832 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.219819107 2023-10-01T11:05:33.832 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.832 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.832 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:05:33.832 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.045397+0000", 2023-10-01T11:05:33.832 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00012661000000000001 2023-10-01T11:05:33.832 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.832 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.832 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:05:33.833 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.045524+0000", 2023-10-01T11:05:33.833 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00014995500000000001 2023-10-01T11:05:33.833 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.833 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.833 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 2,4,6", 2023-10-01T11:05:33.833 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.045674+0000", 2023-10-01T11:05:33.833 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.010827186000000001 2023-10-01T11:05:33.833 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.833 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.833 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:05:33.833 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.056501+0000", 2023-10-01T11:05:33.833 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0051694829999999999 2023-10-01T11:05:33.834 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.834 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.834 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:33.834 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.061670+0000", 2023-10-01T11:05:33.834 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00026799799999999998 2023-10-01T11:05:33.834 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.834 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.834 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:33.834 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.061938+0000", 2023-10-01T11:05:33.834 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.26471266100000002 2023-10-01T11:05:33.834 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.834 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.835 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:33.835 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.326651+0000", 2023-10-01T11:05:33.835 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.7376000000000003e-05 2023-10-01T11:05:33.835 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.835 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.835 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:05:33.835 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.326688+0000", 2023-10-01T11:05:33.835 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00013452399999999999 2023-10-01T11:05:33.835 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.835 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.835 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:05:33.835 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.326823+0000", 2023-10-01T11:05:33.836 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.489574401 2023-10-01T11:05:33.836 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:33.836 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:05:33.836 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:33.836 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.836 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.836 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46232.0:191220 12.f 12:f0dec486:::benchmark_data_smithi078_180197_object88089:head [delete] snapc 0=[] ondisk+write+known_if_redirected e255)", 2023-10-01T11:05:33.836 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:00:28.839042+0000", 2023-10-01T11:05:33.836 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 304.96758051299997, 2023-10-01T11:05:33.836 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.4880418620000002, 2023-10-01T11:05:33.836 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:05:33.836 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:05:33.836 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:05:33.837 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46232", 2023-10-01T11:05:33.837 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/711982100", 2023-10-01T11:05:33.837 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 191220 2023-10-01T11:05:33.837 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.837 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:05:33.837 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.837 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:05:33.837 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:28.839042+0000", 2023-10-01T11:05:33.837 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:05:33.837 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.837 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.837 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:05:33.838 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:28.839042+0000", 2023-10-01T11:05:33.838 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-10-01T11:05:33.838 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.838 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.838 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:05:33.838 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:28.839041+0000", 2023-10-01T11:05:33.838 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.6490000000000003e-06 2023-10-01T11:05:33.838 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.838 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.838 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:05:33.838 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:28.839051+0000", 2023-10-01T11:05:33.838 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.9145999999999999e-05 2023-10-01T11:05:33.839 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.839 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.839 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:05:33.839 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:28.839070+0000", 2023-10-01T11:05:33.839 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2196e-05 2023-10-01T11:05:33.839 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.839 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.839 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:05:33.839 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:28.839082+0000", 2023-10-01T11:05:33.839 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.6396000000000004e-05 2023-10-01T11:05:33.839 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.839 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.839 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:05:33.840 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:28.839168+0000", 2023-10-01T11:05:33.840 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.3798000000000001e-05 2023-10-01T11:05:33.840 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.840 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.840 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:05:33.840 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:28.839182+0000", 2023-10-01T11:05:33.840 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.98642509599999995 2023-10-01T11:05:33.840 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.840 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.840 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:05:33.840 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:29.825607+0000", 2023-10-01T11:05:33.840 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1511e-05 2023-10-01T11:05:33.841 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.841 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.841 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:05:33.841 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:29.825619+0000", 2023-10-01T11:05:33.841 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.225089791 2023-10-01T11:05:33.841 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.841 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.841 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:05:33.841 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.050709+0000", 2023-10-01T11:05:33.841 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00014783900000000001 2023-10-01T11:05:33.841 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.841 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.842 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:05:33.842 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.050856+0000", 2023-10-01T11:05:33.842 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00017628199999999999 2023-10-01T11:05:33.842 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.842 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.842 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 2,4,6", 2023-10-01T11:05:33.842 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.051033+0000", 2023-10-01T11:05:33.842 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.026714826000000001 2023-10-01T11:05:33.842 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.842 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.842 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:05:33.842 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.077748+0000", 2023-10-01T11:05:33.843 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.010377361 2023-10-01T11:05:33.843 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.843 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.843 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:33.843 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.088125+0000", 2023-10-01T11:05:33.843 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000162585 2023-10-01T11:05:33.843 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.843 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.843 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:33.843 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.088287+0000", 2023-10-01T11:05:33.843 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.238585303 2023-10-01T11:05:33.843 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.843 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.844 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:33.844 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.326873+0000", 2023-10-01T11:05:33.844 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.2073e-05 2023-10-01T11:05:33.844 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.844 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.844 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:05:33.844 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.326935+0000", 2023-10-01T11:05:33.844 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00014947199999999999 2023-10-01T11:05:33.844 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.844 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.844 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:05:33.844 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.327084+0000", 2023-10-01T11:05:33.845 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.4880418620000002 2023-10-01T11:05:33.845 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:33.845 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:05:33.845 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:33.845 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.845 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.845 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46232.0:191222 12.f 12:fb2ef8e6:::benchmark_data_smithi078_180197_object88091:head [delete] snapc 0=[] ondisk+write+known_if_redirected e255)", 2023-10-01T11:05:33.845 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:00:28.839172+0000", 2023-10-01T11:05:33.845 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 304.96745136800001, 2023-10-01T11:05:33.845 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.4881694250000002, 2023-10-01T11:05:33.845 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:05:33.845 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:05:33.845 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:05:33.846 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46232", 2023-10-01T11:05:33.846 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/711982100", 2023-10-01T11:05:33.846 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 191222 2023-10-01T11:05:33.846 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.846 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:05:33.846 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.846 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:05:33.846 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:28.839172+0000", 2023-10-01T11:05:33.846 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:05:33.846 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.846 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.846 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:05:33.847 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:28.839172+0000", 2023-10-01T11:05:33.847 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-10-01T11:05:33.847 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.847 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.847 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:05:33.847 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:28.839170+0000", 2023-10-01T11:05:33.847 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.8939999999999993e-06 2023-10-01T11:05:33.847 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.847 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.848 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:05:33.848 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:28.839178+0000", 2023-10-01T11:05:33.848 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.5603999999999998e-05 2023-10-01T11:05:33.848 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.848 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.848 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:05:33.848 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:28.839194+0000", 2023-10-01T11:05:33.848 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.4930000000000004e-06 2023-10-01T11:05:33.848 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.848 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.848 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:05:33.848 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:28.839203+0000", 2023-10-01T11:05:33.848 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.4775000000000002e-05 2023-10-01T11:05:33.849 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.849 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.849 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:05:33.849 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:28.839268+0000", 2023-10-01T11:05:33.849 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2310000000000001e-05 2023-10-01T11:05:33.849 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.849 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.849 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:05:33.849 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:28.839280+0000", 2023-10-01T11:05:33.849 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.98636649600000004 2023-10-01T11:05:33.849 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.849 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.850 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:05:33.850 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:29.825647+0000", 2023-10-01T11:05:33.850 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0922e-05 2023-10-01T11:05:33.850 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.850 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.850 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:05:33.850 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:29.825658+0000", 2023-10-01T11:05:33.850 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2308853749999999 2023-10-01T11:05:33.850 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.850 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.850 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:05:33.850 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.056543+0000", 2023-10-01T11:05:33.851 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0001615 2023-10-01T11:05:33.851 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.851 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.851 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:05:33.851 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.056705+0000", 2023-10-01T11:05:33.851 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00018902799999999999 2023-10-01T11:05:33.851 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.851 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.851 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 2,4,6", 2023-10-01T11:05:33.851 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.056894+0000", 2023-10-01T11:05:33.851 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.020861452999999999 2023-10-01T11:05:33.851 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.852 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.852 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:05:33.852 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.077755+0000", 2023-10-01T11:05:33.852 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.010464763 2023-10-01T11:05:33.852 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.852 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.852 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:33.852 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.088220+0000", 2023-10-01T11:05:33.852 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000179388 2023-10-01T11:05:33.852 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.852 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.852 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:33.852 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.088399+0000", 2023-10-01T11:05:33.853 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.238740024 2023-10-01T11:05:33.853 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.853 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.853 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:33.853 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.327139+0000", 2023-10-01T11:05:33.853 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.0050000000000003e-05 2023-10-01T11:05:33.853 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.853 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.853 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:05:33.853 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.327199+0000", 2023-10-01T11:05:33.853 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000141724 2023-10-01T11:05:33.853 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.854 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.854 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:05:33.854 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.327341+0000", 2023-10-01T11:05:33.854 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.4881694250000002 2023-10-01T11:05:33.854 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:33.854 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:05:33.854 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:33.854 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.854 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.854 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46232.0:191223 12.f 12:f071dbd8:::benchmark_data_smithi078_180197_object88092:head [delete] snapc 0=[] ondisk+write+known_if_redirected e255)", 2023-10-01T11:05:33.854 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:00:28.839691+0000", 2023-10-01T11:05:33.854 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 304.96693184600002, 2023-10-01T11:05:33.855 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.4879260570000001, 2023-10-01T11:05:33.855 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:05:33.855 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:05:33.855 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:05:33.855 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46232", 2023-10-01T11:05:33.855 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/711982100", 2023-10-01T11:05:33.855 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 191223 2023-10-01T11:05:33.855 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.855 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:05:33.855 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.855 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:05:33.855 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:28.839691+0000", 2023-10-01T11:05:33.855 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:05:33.856 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.856 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.856 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:05:33.856 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:28.839691+0000", 2023-10-01T11:05:33.856 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-10-01T11:05:33.856 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.856 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.856 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:05:33.856 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:28.839690+0000", 2023-10-01T11:05:33.856 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.1769999999999997e-06 2023-10-01T11:05:33.856 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.856 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.857 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:05:33.857 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:28.839695+0000", 2023-10-01T11:05:33.857 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1042e-05 2023-10-01T11:05:33.857 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.857 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.857 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:05:33.857 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:28.839706+0000", 2023-10-01T11:05:33.857 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.5560000000000008e-06 2023-10-01T11:05:33.857 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.857 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.859 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:05:33.859 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:28.839715+0000", 2023-10-01T11:05:33.859 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.3565e-05 2023-10-01T11:05:33.859 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.860 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.860 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:05:33.860 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:28.839768+0000", 2023-10-01T11:05:33.860 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1963999999999999e-05 2023-10-01T11:05:33.860 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.860 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.860 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:05:33.860 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:28.839780+0000", 2023-10-01T11:05:33.860 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.98590450500000004 2023-10-01T11:05:33.860 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.860 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.860 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:05:33.861 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:29.825685+0000", 2023-10-01T11:05:33.861 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0549e-05 2023-10-01T11:05:33.861 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.861 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.861 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:05:33.861 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:29.825695+0000", 2023-10-01T11:05:33.861 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2362738689999999 2023-10-01T11:05:33.861 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.861 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.861 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:05:33.862 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.061969+0000", 2023-10-01T11:05:33.862 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000176272 2023-10-01T11:05:33.862 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.862 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.862 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:05:33.862 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.062146+0000", 2023-10-01T11:05:33.862 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000235873 2023-10-01T11:05:33.862 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.862 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.863 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 2,4,6", 2023-10-01T11:05:33.863 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.062381+0000", 2023-10-01T11:05:33.863 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.01537999 2023-10-01T11:05:33.863 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.863 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.863 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:05:33.863 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.077761+0000", 2023-10-01T11:05:33.863 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.010591343 2023-10-01T11:05:33.863 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.863 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.863 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:33.864 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.088353+0000", 2023-10-01T11:05:33.864 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000161628 2023-10-01T11:05:33.864 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.864 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.864 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:33.864 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.088514+0000", 2023-10-01T11:05:33.864 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.23888900499999999 2023-10-01T11:05:33.864 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.864 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.864 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:33.864 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.327403+0000", 2023-10-01T11:05:33.865 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.4041000000000006e-05 2023-10-01T11:05:33.865 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.865 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.865 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:05:33.865 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.327468+0000", 2023-10-01T11:05:33.865 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00014968800000000001 2023-10-01T11:05:33.865 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.865 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.865 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:05:33.865 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.327617+0000", 2023-10-01T11:05:33.865 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.4879260570000001 2023-10-01T11:05:33.866 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:33.866 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:05:33.866 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:33.866 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.866 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.866 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46232.0:191230 12.f 12:f12f6696:::benchmark_data_smithi078_180197_object88099:head [delete] snapc 0=[] ondisk+write+known_if_redirected e255)", 2023-10-01T11:05:33.866 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:00:28.858113+0000", 2023-10-01T11:05:33.866 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 304.948509505, 2023-10-01T11:05:33.866 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.4697371700000001, 2023-10-01T11:05:33.866 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:05:33.866 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:05:33.867 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:05:33.867 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46232", 2023-10-01T11:05:33.867 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/711982100", 2023-10-01T11:05:33.867 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 191230 2023-10-01T11:05:33.867 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.867 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:05:33.867 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.867 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:05:33.867 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:28.858113+0000", 2023-10-01T11:05:33.867 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:05:33.867 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.868 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.868 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:05:33.868 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:28.858113+0000", 2023-10-01T11:05:33.868 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999976 2023-10-01T11:05:33.868 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.868 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.868 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:05:33.868 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:28.858111+0000", 2023-10-01T11:05:33.868 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1514e-05 2023-10-01T11:05:33.868 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.868 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.869 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:05:33.869 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:28.858123+0000", 2023-10-01T11:05:33.869 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.2863000000000001e-05 2023-10-01T11:05:33.869 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.869 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.869 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:05:33.869 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:28.858145+0000", 2023-10-01T11:05:33.869 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.6178e-05 2023-10-01T11:05:33.869 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.869 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.869 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:05:33.869 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:28.858162+0000", 2023-10-01T11:05:33.870 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.8907999999999996e-05 2023-10-01T11:05:33.870 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.870 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.870 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:05:33.870 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:28.858251+0000", 2023-10-01T11:05:33.870 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4673e-05 2023-10-01T11:05:33.870 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.870 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.870 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:05:33.870 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:28.858265+0000", 2023-10-01T11:05:33.870 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.967458127 2023-10-01T11:05:33.871 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.871 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.871 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:05:33.871 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:29.825723+0000", 2023-10-01T11:05:33.871 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.136e-05 2023-10-01T11:05:33.871 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.871 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.871 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:05:33.871 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:29.825735+0000", 2023-10-01T11:05:33.871 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2415901570000001 2023-10-01T11:05:33.871 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.871 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.872 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:05:33.872 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.067325+0000", 2023-10-01T11:05:33.872 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00016307299999999999 2023-10-01T11:05:33.872 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.872 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.872 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:05:33.872 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.067488+0000", 2023-10-01T11:05:33.872 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000215942 2023-10-01T11:05:33.872 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.872 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.872 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 2,4,6", 2023-10-01T11:05:33.873 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.067704+0000", 2023-10-01T11:05:33.873 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.010067316 2023-10-01T11:05:33.873 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.873 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.873 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:05:33.873 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.077771+0000", 2023-10-01T11:05:33.873 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.010688736000000001 2023-10-01T11:05:33.873 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.873 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.874 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:33.874 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.088460+0000", 2023-10-01T11:05:33.874 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00031148699999999999 2023-10-01T11:05:33.874 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.874 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.874 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:33.874 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.088771+0000", 2023-10-01T11:05:33.874 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.23889453299999999 2023-10-01T11:05:33.874 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.874 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.874 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:33.875 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.327666+0000", 2023-10-01T11:05:33.875 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.8690000000000003e-05 2023-10-01T11:05:33.875 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.875 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.875 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:05:33.875 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.327715+0000", 2023-10-01T11:05:33.875 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00013599200000000001 2023-10-01T11:05:33.875 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.875 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.876 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:05:33.876 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.327851+0000", 2023-10-01T11:05:33.876 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.4697371700000001 2023-10-01T11:05:33.876 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:33.876 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:05:33.876 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:33.876 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.876 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.876 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46232.0:191235 12.f 12:f74a4006:::benchmark_data_smithi078_180197_object88104:head [delete] snapc 0=[] ondisk+write+known_if_redirected e255)", 2023-10-01T11:05:33.876 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:00:28.869071+0000", 2023-10-01T11:05:33.876 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 304.93755239400002, 2023-10-01T11:05:33.877 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.4589382720000001, 2023-10-01T11:05:33.877 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:05:33.877 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:05:33.877 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:05:33.877 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46232", 2023-10-01T11:05:33.877 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/711982100", 2023-10-01T11:05:33.877 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 191235 2023-10-01T11:05:33.877 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.877 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:05:33.877 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.877 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:05:33.877 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:28.869071+0000", 2023-10-01T11:05:33.878 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:05:33.878 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.878 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.878 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:05:33.878 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:28.869071+0000", 2023-10-01T11:05:33.878 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-10-01T11:05:33.878 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.878 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.878 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:05:33.878 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:28.869069+0000", 2023-10-01T11:05:33.878 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.8300000000000008e-06 2023-10-01T11:05:33.879 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.879 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.879 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:05:33.879 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:28.869079+0000", 2023-10-01T11:05:33.879 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.0366000000000001e-05 2023-10-01T11:05:33.879 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.879 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.879 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:05:33.879 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:28.869099+0000", 2023-10-01T11:05:33.879 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0052e-05 2023-10-01T11:05:33.879 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.879 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.880 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:05:33.880 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:28.869109+0000", 2023-10-01T11:05:33.880 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00014569100000000001 2023-10-01T11:05:33.880 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.880 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.880 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:05:33.880 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:28.869255+0000", 2023-10-01T11:05:33.880 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.8994000000000001e-05 2023-10-01T11:05:33.880 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.881 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.881 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:05:33.881 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:28.869274+0000", 2023-10-01T11:05:33.881 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.95648903799999996 2023-10-01T11:05:33.881 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.881 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.881 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:05:33.881 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:29.825763+0000", 2023-10-01T11:05:33.881 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.101e-05 2023-10-01T11:05:33.881 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.881 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.882 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:05:33.882 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:29.825774+0000", 2023-10-01T11:05:33.882 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2467272789999999 2023-10-01T11:05:33.882 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.882 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.882 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:05:33.882 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.072501+0000", 2023-10-01T11:05:33.882 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00019195700000000001 2023-10-01T11:05:33.883 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.883 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.883 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:05:33.883 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.072693+0000", 2023-10-01T11:05:33.883 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00022046699999999999 2023-10-01T11:05:33.883 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.883 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.883 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 2,4,6", 2023-10-01T11:05:33.883 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.072914+0000", 2023-10-01T11:05:33.883 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.015656446000000001 2023-10-01T11:05:33.884 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.884 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.884 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:33.884 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.088570+0000", 2023-10-01T11:05:33.884 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.4190999999999997e-05 2023-10-01T11:05:33.884 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.884 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.884 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:05:33.884 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.088624+0000", 2023-10-01T11:05:33.884 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00030568000000000001 2023-10-01T11:05:33.885 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.885 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.885 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:33.885 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.088930+0000", 2023-10-01T11:05:33.885 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.23894932899999999 2023-10-01T11:05:33.885 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.885 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.885 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:33.885 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.327879+0000", 2023-10-01T11:05:33.886 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.6859000000000001e-05 2023-10-01T11:05:33.886 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.886 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.886 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:05:33.886 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.327916+0000", 2023-10-01T11:05:33.886 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.2633000000000006e-05 2023-10-01T11:05:33.886 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.886 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.887 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:05:33.887 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.328009+0000", 2023-10-01T11:05:33.887 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.4589382720000001 2023-10-01T11:05:33.887 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:33.887 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:05:33.887 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:33.887 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.888 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.888 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:73718 13.7 13:eae49b7e:::benchmark_data_smithi078_202524_object73717:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e298)", 2023-10-01T11:05:33.888 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:03:06.928786+0000", 2023-10-01T11:05:33.888 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 146.87783733200001, 2023-10-01T11:05:33.888 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.81450642699999998, 2023-10-01T11:05:33.888 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:05:33.888 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "reached pg", 2023-10-01T11:05:33.888 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:05:33.888 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:05:33.889 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:05:33.889 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 73718 2023-10-01T11:05:33.889 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.889 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:05:33.889 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.889 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:05:33.889 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.928786+0000", 2023-10-01T11:05:33.889 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:05:33.889 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.889 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.889 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:05:33.890 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.928786+0000", 2023-10-01T11:05:33.890 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-10-01T11:05:33.890 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.890 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.890 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:05:33.890 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.928784+0000", 2023-10-01T11:05:33.890 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.8640999999999998e-05 2023-10-01T11:05:33.890 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.890 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.890 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:05:33.891 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.928823+0000", 2023-10-01T11:05:33.891 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4948e-05 2023-10-01T11:05:33.891 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.891 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.891 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:05:33.891 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.928838+0000", 2023-10-01T11:05:33.891 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.9590000000000001e-06 2023-10-01T11:05:33.891 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.891 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.891 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:05:33.891 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.928847+0000", 2023-10-01T11:05:33.891 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.1929999999999999e-05 2023-10-01T11:05:33.892 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.892 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.892 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:05:33.892 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.928899+0000", 2023-10-01T11:05:33.892 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00020405799999999999 2023-10-01T11:05:33.892 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.892 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.892 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:05:33.892 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.929103+0000", 2023-10-01T11:05:33.892 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000219817 2023-10-01T11:05:33.892 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.892 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.893 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 4,6,7", 2023-10-01T11:05:33.893 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.929322+0000", 2023-10-01T11:05:33.893 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0081014009999999994 2023-10-01T11:05:33.893 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.893 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.893 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:33.893 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.937424+0000", 2023-10-01T11:05:33.893 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.001143725 2023-10-01T11:05:33.893 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.893 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.893 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:05:33.893 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.938568+0000", 2023-10-01T11:05:33.893 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.047217527000000002 2023-10-01T11:05:33.894 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.894 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.894 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:05:33.894 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.985785+0000", 2023-10-01T11:05:33.894 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1289e-05 2023-10-01T11:05:33.894 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.894 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.894 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:05:33.894 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.985796+0000", 2023-10-01T11:05:33.894 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.7574824 2023-10-01T11:05:33.894 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.894 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.895 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:05:33.895 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.743279+0000", 2023-10-01T11:05:33.895 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.3275000000000001e-05 2023-10-01T11:05:33.895 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.895 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.895 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:05:33.895 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.743292+0000", 2023-10-01T11:05:33.895 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.81450642699999998 2023-10-01T11:05:33.895 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:33.895 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:05:33.895 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:33.895 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.895 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.896 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:73722 13.7 13:efda010a:::benchmark_data_smithi078_202524_object73721:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e298)", 2023-10-01T11:05:33.896 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:03:06.932655+0000", 2023-10-01T11:05:33.896 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 146.87396767800001, 2023-10-01T11:05:33.896 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.81066804299999995, 2023-10-01T11:05:33.896 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:05:33.896 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "reached pg", 2023-10-01T11:05:33.896 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:05:33.896 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:05:33.896 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:05:33.896 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 73722 2023-10-01T11:05:33.896 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.897 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:05:33.897 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.897 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:05:33.897 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.932655+0000", 2023-10-01T11:05:33.897 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:05:33.897 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.897 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.897 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:05:33.897 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.932655+0000", 2023-10-01T11:05:33.897 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-10-01T11:05:33.897 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.897 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.898 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:05:33.898 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.932654+0000", 2023-10-01T11:05:33.898 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.7014000000000001e-05 2023-10-01T11:05:33.898 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.898 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.898 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:05:33.898 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.932701+0000", 2023-10-01T11:05:33.898 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0563e-05 2023-10-01T11:05:33.898 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.898 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.898 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:05:33.898 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.932711+0000", 2023-10-01T11:05:33.899 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.2039999999999996e-06 2023-10-01T11:05:33.899 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.899 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.899 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:05:33.899 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.932716+0000", 2023-10-01T11:05:33.899 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00013517499999999999 2023-10-01T11:05:33.899 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.899 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.899 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:05:33.899 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.932852+0000", 2023-10-01T11:05:33.899 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00018874300000000001 2023-10-01T11:05:33.899 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.899 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.900 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:05:33.900 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.933040+0000", 2023-10-01T11:05:33.900 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00045594800000000002 2023-10-01T11:05:33.900 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.900 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.900 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 4,6,7", 2023-10-01T11:05:33.900 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.933496+0000", 2023-10-01T11:05:33.900 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0050797289999999998 2023-10-01T11:05:33.900 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.900 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.901 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:05:33.901 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.938576+0000", 2023-10-01T11:05:33.901 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.047250863999999997 2023-10-01T11:05:33.901 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.901 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.901 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:05:33.901 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.985827+0000", 2023-10-01T11:05:33.901 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.8830000000000001e-06 2023-10-01T11:05:33.901 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.901 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.901 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:05:33.901 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.985837+0000", 2023-10-01T11:05:33.902 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.75747867899999999 2023-10-01T11:05:33.902 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.902 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.902 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:05:33.902 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.743315+0000", 2023-10-01T11:05:33.902 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.9780000000000005e-06 2023-10-01T11:05:33.902 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.902 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.902 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:05:33.902 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.743323+0000", 2023-10-01T11:05:33.903 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.81066804299999995 2023-10-01T11:05:33.903 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:33.903 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:05:33.903 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:33.903 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.903 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.903 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:73728 13.7 13:e775b1ab:::benchmark_data_smithi078_202524_object73727:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e298)", 2023-10-01T11:05:33.903 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:03:06.948920+0000", 2023-10-01T11:05:33.903 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 146.85770329600001, 2023-10-01T11:05:33.903 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.79442613799999995, 2023-10-01T11:05:33.903 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:05:33.903 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "reached pg", 2023-10-01T11:05:33.903 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:05:33.904 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:05:33.904 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:05:33.904 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 73728 2023-10-01T11:05:33.904 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.904 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:05:33.904 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.904 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:05:33.904 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.948920+0000", 2023-10-01T11:05:33.904 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:05:33.904 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.905 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.905 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:05:33.905 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.948920+0000", 2023-10-01T11:05:33.905 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-10-01T11:05:33.905 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.905 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.905 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:05:33.905 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.948918+0000", 2023-10-01T11:05:33.905 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.2062000000000003e-05 2023-10-01T11:05:33.905 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.905 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.905 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:05:33.905 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.948980+0000", 2023-10-01T11:05:33.906 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1464000000000001e-05 2023-10-01T11:05:33.906 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.906 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.906 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:05:33.906 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.948991+0000", 2023-10-01T11:05:33.906 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.2814999999999997e-05 2023-10-01T11:05:33.906 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.906 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.906 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:05:33.906 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.949024+0000", 2023-10-01T11:05:33.906 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0038814299999999999 2023-10-01T11:05:33.906 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.907 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.907 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:05:33.907 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.952906+0000", 2023-10-01T11:05:33.907 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00017169099999999999 2023-10-01T11:05:33.907 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.907 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.907 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:05:33.907 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.953077+0000", 2023-10-01T11:05:33.907 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00018491699999999999 2023-10-01T11:05:33.907 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.907 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.908 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 4,6,7", 2023-10-01T11:05:33.908 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.953262+0000", 2023-10-01T11:05:33.908 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.032607349000000001 2023-10-01T11:05:33.908 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.908 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.908 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:05:33.908 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.985870+0000", 2023-10-01T11:05:33.908 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.9709999999999993e-06 2023-10-01T11:05:33.908 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.908 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.908 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:05:33.908 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.985879+0000", 2023-10-01T11:05:33.909 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.75746156799999997 2023-10-01T11:05:33.909 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.909 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.909 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:05:33.909 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.743340+0000", 2023-10-01T11:05:33.909 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.7440000000000004e-06 2023-10-01T11:05:33.909 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.909 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.909 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:05:33.910 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.743346+0000", 2023-10-01T11:05:33.910 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.79442613799999995 2023-10-01T11:05:33.910 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:33.910 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:05:33.910 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:33.910 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.910 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.910 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.46454.0:82948 13.4 e320/300 13:23f16cc3:::benchmark_data_smithi078_202524_object3390:head v 320'5154, mlcod=291'457)", 2023-10-01T11:05:33.910 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:04:53.436235+0000", 2023-10-01T11:05:33.910 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 40.370387675000003, 2023-10-01T11:05:33.911 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.010507302, 2023-10-01T11:05:33.911 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:05:33.911 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:05:33.911 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:05:33.911 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.911 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:05:33.911 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:53.436235+0000", 2023-10-01T11:05:33.911 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:05:33.911 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.911 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.911 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:05:33.912 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:53.436235+0000", 2023-10-01T11:05:33.912 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-10-01T11:05:33.912 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.912 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.912 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:05:33.912 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:53.436234+0000", 2023-10-01T11:05:33.912 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0536e-05 2023-10-01T11:05:33.912 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.912 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.913 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:05:33.913 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:53.436244+0000", 2023-10-01T11:05:33.913 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4844999999999999e-05 2023-10-01T11:05:33.913 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.913 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.913 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:05:33.913 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:53.436259+0000", 2023-10-01T11:05:33.913 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.0090000000000001e-06 2023-10-01T11:05:33.913 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.914 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.914 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:05:33.914 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:53.436266+0000", 2023-10-01T11:05:33.914 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.1409000000000002e-05 2023-10-01T11:05:33.914 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.914 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.914 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:05:33.914 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:53.436318+0000", 2023-10-01T11:05:33.914 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.9663999999999999e-05 2023-10-01T11:05:33.915 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.915 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.915 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:05:33.915 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:53.436347+0000", 2023-10-01T11:05:33.915 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.010348196 2023-10-01T11:05:33.915 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.915 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.915 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:05:33.915 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:53.446696+0000", 2023-10-01T11:05:33.915 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.7105999999999999e-05 2023-10-01T11:05:33.916 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.916 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.916 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:05:33.916 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:53.446743+0000", 2023-10-01T11:05:33.916 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.010507302 2023-10-01T11:05:33.916 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:33.916 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:05:33.916 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:33.916 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.916 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.917 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.46454.0:83280 13.4 e320/300 13:2dc5a430:::benchmark_data_smithi078_202524_object3722:head v 320'5180, mlcod=291'457)", 2023-10-01T11:05:33.917 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:04:56.013812+0000", 2023-10-01T11:05:33.917 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 37.792811065999999, 2023-10-01T11:05:33.917 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.010437947, 2023-10-01T11:05:33.917 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:05:33.917 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:05:33.917 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:05:33.917 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.918 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:05:33.918 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:56.013812+0000", 2023-10-01T11:05:33.918 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:05:33.918 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.918 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.918 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:05:33.918 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:56.013812+0000", 2023-10-01T11:05:33.918 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999928 2023-10-01T11:05:33.918 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.919 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.919 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:05:33.919 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:56.013805+0000", 2023-10-01T11:05:33.919 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.0219000000000001e-05 2023-10-01T11:05:33.919 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.919 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.919 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:05:33.919 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:56.013825+0000", 2023-10-01T11:05:33.920 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.4719000000000002e-05 2023-10-01T11:05:33.920 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.920 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.920 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:05:33.920 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:56.013850+0000", 2023-10-01T11:05:33.920 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.1293000000000001e-05 2023-10-01T11:05:33.920 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.920 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.921 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:05:33.921 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:56.013871+0000", 2023-10-01T11:05:33.921 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00014392100000000001 2023-10-01T11:05:33.921 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.921 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.921 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:05:33.921 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:56.014015+0000", 2023-10-01T11:05:33.921 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.4856000000000004e-05 2023-10-01T11:05:33.922 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.922 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.922 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:05:33.922 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:56.014080+0000", 2023-10-01T11:05:33.922 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.010073587 2023-10-01T11:05:33.922 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.922 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.922 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:05:33.922 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:56.024154+0000", 2023-10-01T11:05:33.922 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.6278000000000002e-05 2023-10-01T11:05:33.922 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.922 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.925 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:05:33.925 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:56.024250+0000", 2023-10-01T11:05:33.925 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.010437947 2023-10-01T11:05:33.925 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:33.925 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:05:33.925 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:33.926 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.926 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.926 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.46454.0:83415 13.4 e320/300 13:26822c14:::benchmark_data_smithi078_202524_object3857:head v 320'5191, mlcod=291'457)", 2023-10-01T11:05:33.926 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:04:57.063534+0000", 2023-10-01T11:05:33.926 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 36.743088696999997, 2023-10-01T11:05:33.926 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.010549167999999999, 2023-10-01T11:05:33.926 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:05:33.926 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:05:33.926 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:05:33.926 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.927 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:05:33.927 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:57.063534+0000", 2023-10-01T11:05:33.927 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:05:33.927 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.927 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.927 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:05:33.927 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:57.063534+0000", 2023-10-01T11:05:33.927 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-10-01T11:05:33.927 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.927 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.928 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:05:33.928 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:57.063533+0000", 2023-10-01T11:05:33.928 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1389000000000001e-05 2023-10-01T11:05:33.928 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.928 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.928 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:05:33.928 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:57.063544+0000", 2023-10-01T11:05:33.928 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.5466999999999999e-05 2023-10-01T11:05:33.928 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.928 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.928 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:05:33.929 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:57.063560+0000", 2023-10-01T11:05:33.929 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.1340000000000004e-06 2023-10-01T11:05:33.929 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.929 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.929 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:05:33.929 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:57.063568+0000", 2023-10-01T11:05:33.929 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.7484000000000001e-05 2023-10-01T11:05:33.929 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.929 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.929 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:05:33.929 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:57.063625+0000", 2023-10-01T11:05:33.930 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.3970000000000002e-05 2023-10-01T11:05:33.930 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.930 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.930 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:05:33.930 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:57.063659+0000", 2023-10-01T11:05:33.930 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.010383081000000001 2023-10-01T11:05:33.930 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.930 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.930 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:05:33.930 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:57.074042+0000", 2023-10-01T11:05:33.930 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.1087999999999997e-05 2023-10-01T11:05:33.931 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.931 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.931 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:05:33.931 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:57.074083+0000", 2023-10-01T11:05:33.931 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.010549167999999999 2023-10-01T11:05:33.931 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:33.931 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:05:33.931 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:33.931 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.931 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.931 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:83712 13.9 13:9289f09a:::benchmark_data_smithi078_202524_object4154:head [delete] snapc 0=[] ondisk+write+known_if_redirected e321)", 2023-10-01T11:05:33.932 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:04:59.887976+0000", 2023-10-01T11:05:33.932 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 33.918646932000001, 2023-10-01T11:05:33.932 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.01037243, 2023-10-01T11:05:33.932 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:05:33.932 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:05:33.932 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:05:33.932 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:05:33.932 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:05:33.932 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 83712 2023-10-01T11:05:33.932 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.932 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:05:33.932 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.933 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:05:33.933 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:59.887976+0000", 2023-10-01T11:05:33.933 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:05:33.933 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.933 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.933 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:05:33.933 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:59.887976+0000", 2023-10-01T11:05:33.933 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-10-01T11:05:33.933 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.933 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.933 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:05:33.933 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:59.887975+0000", 2023-10-01T11:05:33.934 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.22e-06 2023-10-01T11:05:33.934 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.934 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.934 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:05:33.934 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:59.887980+0000", 2023-10-01T11:05:33.934 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.9220000000000003e-06 2023-10-01T11:05:33.934 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.934 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.934 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:05:33.934 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:59.887989+0000", 2023-10-01T11:05:33.934 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.4880000000000001e-06 2023-10-01T11:05:33.935 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.935 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.935 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:05:33.935 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:59.887994+0000", 2023-10-01T11:05:33.935 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.6338999999999998e-05 2023-10-01T11:05:33.935 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.935 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.935 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:05:33.935 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:59.888040+0000", 2023-10-01T11:05:33.935 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000113347 2023-10-01T11:05:33.935 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.936 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.936 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:05:33.936 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:59.888153+0000", 2023-10-01T11:05:33.936 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000108495 2023-10-01T11:05:33.936 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.936 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.936 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 0,2", 2023-10-01T11:05:33.936 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:59.888262+0000", 2023-10-01T11:05:33.936 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0028133839999999999 2023-10-01T11:05:33.936 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.936 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.936 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:05:33.937 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:59.891075+0000", 2023-10-01T11:05:33.937 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00087796 2023-10-01T11:05:33.937 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.937 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.937 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:33.937 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:59.891953+0000", 2023-10-01T11:05:33.937 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0062469029999999998 2023-10-01T11:05:33.937 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.937 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.937 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:33.937 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:59.898200+0000", 2023-10-01T11:05:33.937 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.6211999999999994e-05 2023-10-01T11:05:33.938 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.938 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.938 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:05:33.938 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:59.898286+0000", 2023-10-01T11:05:33.938 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.2255999999999995e-05 2023-10-01T11:05:33.938 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.938 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.938 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:05:33.938 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:59.898348+0000", 2023-10-01T11:05:33.938 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.01037243 2023-10-01T11:05:33.938 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:33.939 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:05:33.939 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:33.939 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.939 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.939 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.46454.0:83717 13.4 e321/300 13:2f28b3c1:::benchmark_data_smithi078_202524_object4159:head v 321'5208, mlcod=291'457)", 2023-10-01T11:05:33.939 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:04:59.903084+0000", 2023-10-01T11:05:33.939 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 33.903538595000001, 2023-10-01T11:05:33.939 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.011860206, 2023-10-01T11:05:33.939 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:05:33.939 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:05:33.939 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:05:33.940 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.940 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:05:33.940 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:59.903084+0000", 2023-10-01T11:05:33.940 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:05:33.940 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.940 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.940 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:05:33.940 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:59.903084+0000", 2023-10-01T11:05:33.940 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999971 2023-10-01T11:05:33.940 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.940 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.941 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:05:33.941 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:59.903082+0000", 2023-10-01T11:05:33.941 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.9762000000000002e-05 2023-10-01T11:05:33.941 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.941 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.941 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:05:33.941 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:59.903101+0000", 2023-10-01T11:05:33.941 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.9850999999999999e-05 2023-10-01T11:05:33.941 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.941 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.941 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:05:33.941 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:59.903131+0000", 2023-10-01T11:05:33.942 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.1437000000000002e-05 2023-10-01T11:05:33.942 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.942 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.942 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:05:33.942 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:59.903153+0000", 2023-10-01T11:05:33.942 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.1263e-05 2023-10-01T11:05:33.942 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.942 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.942 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:05:33.942 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:59.903204+0000", 2023-10-01T11:05:33.942 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.5154999999999999e-05 2023-10-01T11:05:33.943 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.943 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.943 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:05:33.943 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:59.903229+0000", 2023-10-01T11:05:33.943 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.011681965000000001 2023-10-01T11:05:33.943 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.943 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.943 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:05:33.943 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:59.914911+0000", 2023-10-01T11:05:33.943 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.3426999999999997e-05 2023-10-01T11:05:33.943 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.944 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.944 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:05:33.944 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:59.914945+0000", 2023-10-01T11:05:33.944 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.011860206 2023-10-01T11:05:33.944 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:33.944 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:05:33.944 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:33.944 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.944 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.944 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.46454.0:83797 13.4 e321/300 13:265df3ae:::benchmark_data_smithi078_202524_object4239:head v 321'5215, mlcod=291'457)", 2023-10-01T11:05:33.944 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:05:00.255266+0000", 2023-10-01T11:05:33.945 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 33.551356959000003, 2023-10-01T11:05:33.945 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.015680646999999999, 2023-10-01T11:05:33.945 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:05:33.945 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:05:33.945 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:05:33.945 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.945 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:05:33.945 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:00.255266+0000", 2023-10-01T11:05:33.945 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:05:33.945 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.945 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.946 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:05:33.946 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:00.255266+0000", 2023-10-01T11:05:33.946 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-10-01T11:05:33.946 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.946 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.946 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:05:33.946 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:00.255264+0000", 2023-10-01T11:05:33.946 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4949999999999999e-05 2023-10-01T11:05:33.946 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.946 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.946 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:05:33.946 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:00.255279+0000", 2023-10-01T11:05:33.947 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.9596e-05 2023-10-01T11:05:33.947 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.947 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.947 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:05:33.947 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:00.255299+0000", 2023-10-01T11:05:33.947 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0658e-05 2023-10-01T11:05:33.947 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.947 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.947 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:05:33.947 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:00.255310+0000", 2023-10-01T11:05:33.948 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0038525640000000002 2023-10-01T11:05:33.948 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.948 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.948 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:05:33.948 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:00.259162+0000", 2023-10-01T11:05:33.948 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.8619000000000003e-05 2023-10-01T11:05:33.948 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.948 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.948 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:05:33.948 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:00.259211+0000", 2023-10-01T11:05:33.948 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.011691295000000001 2023-10-01T11:05:33.949 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.949 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.949 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:05:33.949 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:00.270902+0000", 2023-10-01T11:05:33.949 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.4620000000000003e-05 2023-10-01T11:05:33.949 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.949 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.949 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:05:33.949 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:00.270947+0000", 2023-10-01T11:05:33.949 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.015680646999999999 2023-10-01T11:05:33.949 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:33.950 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:05:33.950 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:33.950 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.950 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.950 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:86361 13.9 13:9af3f917:::benchmark_data_smithi078_202524_object6803:head [delete] snapc 0=[] ondisk+write+known_if_redirected e322)", 2023-10-01T11:05:33.950 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:05:28.198995+0000", 2023-10-01T11:05:33.950 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 5.607627624, 2023-10-01T11:05:33.950 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.011165379, 2023-10-01T11:05:33.950 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:05:33.950 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:05:33.950 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:05:33.950 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:05:33.951 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:05:33.951 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 86361 2023-10-01T11:05:33.951 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.951 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:05:33.951 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.951 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:05:33.951 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:28.198995+0000", 2023-10-01T11:05:33.951 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:05:33.951 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.951 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.951 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:05:33.952 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:28.198995+0000", 2023-10-01T11:05:33.952 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-10-01T11:05:33.952 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.952 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.952 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:05:33.952 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:28.198994+0000", 2023-10-01T11:05:33.952 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.19e-06 2023-10-01T11:05:33.952 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.952 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.952 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:05:33.952 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:28.199000+0000", 2023-10-01T11:05:33.952 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.52e-05 2023-10-01T11:05:33.953 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.953 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.953 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:05:33.953 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:28.199016+0000", 2023-10-01T11:05:33.953 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.2389999999999996e-06 2023-10-01T11:05:33.953 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.953 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.953 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:05:33.953 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:28.199023+0000", 2023-10-01T11:05:33.953 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.3584999999999995e-05 2023-10-01T11:05:33.953 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.953 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.954 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:05:33.954 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:28.199086+0000", 2023-10-01T11:05:33.954 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00014976400000000001 2023-10-01T11:05:33.954 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.954 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.954 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:05:33.954 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:28.199236+0000", 2023-10-01T11:05:33.954 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000169264 2023-10-01T11:05:33.954 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.954 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.954 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 0,2", 2023-10-01T11:05:33.954 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:28.199405+0000", 2023-10-01T11:05:33.955 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0028737400000000001 2023-10-01T11:05:33.955 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.955 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.955 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:05:33.955 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:28.202279+0000", 2023-10-01T11:05:33.955 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0055475840000000004 2023-10-01T11:05:33.955 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.955 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.955 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:33.955 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:28.207827+0000", 2023-10-01T11:05:33.955 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0022096360000000001 2023-10-01T11:05:33.956 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.956 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.956 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:33.956 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:28.210036+0000", 2023-10-01T11:05:33.956 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.0048000000000003e-05 2023-10-01T11:05:33.956 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.956 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.956 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:05:33.956 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:28.210116+0000", 2023-10-01T11:05:33.957 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.4283000000000003e-05 2023-10-01T11:05:33.957 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.957 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.957 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:05:33.957 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:28.210161+0000", 2023-10-01T11:05:33.957 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.011165379 2023-10-01T11:05:33.957 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:33.957 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:05:33.957 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:33.958 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.958 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.958 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:86372 13.c 13:3a68c314:::benchmark_data_smithi078_202524_object6814:head [delete] snapc 0=[] ondisk+write+known_if_redirected e322)", 2023-10-01T11:05:33.958 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:05:28.218970+0000", 2023-10-01T11:05:33.958 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 5.5876532540000001, 2023-10-01T11:05:33.958 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.01027807, 2023-10-01T11:05:33.958 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:05:33.958 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:05:33.959 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:05:33.959 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:05:33.959 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:05:33.959 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 86372 2023-10-01T11:05:33.959 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.959 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:05:33.959 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.959 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:05:33.959 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:28.218970+0000", 2023-10-01T11:05:33.959 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:05:33.960 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.960 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.960 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:05:33.960 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:28.218970+0000", 2023-10-01T11:05:33.960 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999971 2023-10-01T11:05:33.960 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.960 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.960 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:05:33.960 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:28.218967+0000", 2023-10-01T11:05:33.960 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2206e-05 2023-10-01T11:05:33.960 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.961 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.961 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:05:33.961 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:28.218979+0000", 2023-10-01T11:05:33.961 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.1517000000000001e-05 2023-10-01T11:05:33.961 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.961 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.961 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:05:33.961 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:28.219001+0000", 2023-10-01T11:05:33.961 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1559999999999999e-05 2023-10-01T11:05:33.961 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.961 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.961 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:05:33.962 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:28.219012+0000", 2023-10-01T11:05:33.962 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.9286999999999997e-05 2023-10-01T11:05:33.962 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.962 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.962 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:05:33.962 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:28.219072+0000", 2023-10-01T11:05:33.962 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000333863 2023-10-01T11:05:33.962 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.962 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.962 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:05:33.962 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:28.219405+0000", 2023-10-01T11:05:33.963 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00022279799999999999 2023-10-01T11:05:33.963 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.963 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.963 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1,2", 2023-10-01T11:05:33.963 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:28.219628+0000", 2023-10-01T11:05:33.963 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0063016799999999996 2023-10-01T11:05:33.963 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.963 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.963 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:05:33.963 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:28.225930+0000", 2023-10-01T11:05:33.963 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.001350228 2023-10-01T11:05:33.963 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.963 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.964 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:33.964 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:28.227280+0000", 2023-10-01T11:05:33.964 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0018718389999999999 2023-10-01T11:05:33.964 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.964 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.964 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:33.964 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:28.229152+0000", 2023-10-01T11:05:33.964 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.2623000000000001e-05 2023-10-01T11:05:33.964 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.964 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.964 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:05:33.964 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:28.229205+0000", 2023-10-01T11:05:33.965 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.3242999999999999e-05 2023-10-01T11:05:33.965 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.965 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.965 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:05:33.965 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:28.229248+0000", 2023-10-01T11:05:33.965 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.01027807 2023-10-01T11:05:33.965 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:33.965 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:05:33.965 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:33.965 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.965 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.965 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:86374 13.c 13:3a32927e:::benchmark_data_smithi078_202524_object6816:head [delete] snapc 0=[] ondisk+write+known_if_redirected e322)", 2023-10-01T11:05:33.966 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:05:28.219069+0000", 2023-10-01T11:05:33.966 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 5.5875534980000001, 2023-10-01T11:05:33.966 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.010331636999999999, 2023-10-01T11:05:33.966 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:05:33.966 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:05:33.966 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:05:33.966 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:05:33.966 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:05:33.966 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 86374 2023-10-01T11:05:33.966 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.966 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:05:33.967 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.967 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:05:33.967 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:28.219069+0000", 2023-10-01T11:05:33.967 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:05:33.967 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.967 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.967 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:05:33.967 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:28.219069+0000", 2023-10-01T11:05:33.967 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-10-01T11:05:33.967 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.967 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.967 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:05:33.968 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:28.219068+0000", 2023-10-01T11:05:33.968 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.4819999999999997e-06 2023-10-01T11:05:33.968 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.968 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.968 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:05:33.968 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:28.219076+0000", 2023-10-01T11:05:33.968 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.3954000000000001e-05 2023-10-01T11:05:33.968 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.968 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.968 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:05:33.968 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:28.219090+0000", 2023-10-01T11:05:33.969 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.0850000000000003e-06 2023-10-01T11:05:33.969 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.969 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.969 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:05:33.969 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:28.219099+0000", 2023-10-01T11:05:33.969 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0036402370000000002 2023-10-01T11:05:33.969 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.969 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.969 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:05:33.969 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:28.222739+0000", 2023-10-01T11:05:33.969 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00012515700000000001 2023-10-01T11:05:33.969 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.969 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.970 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:05:33.970 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:28.222864+0000", 2023-10-01T11:05:33.970 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000149159 2023-10-01T11:05:33.970 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.970 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.970 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1,2", 2023-10-01T11:05:33.970 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:28.223013+0000", 2023-10-01T11:05:33.970 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0031463229999999999 2023-10-01T11:05:33.970 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.970 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.970 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:05:33.970 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:28.226160+0000", 2023-10-01T11:05:33.971 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.001194374 2023-10-01T11:05:33.971 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.971 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.971 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:33.971 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:28.227354+0000", 2023-10-01T11:05:33.971 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.001942736 2023-10-01T11:05:33.971 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.971 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.971 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:33.971 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:28.229297+0000", 2023-10-01T11:05:33.971 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.7194000000000002e-05 2023-10-01T11:05:33.971 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.972 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.972 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:05:33.972 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:28.229354+0000", 2023-10-01T11:05:33.972 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.7241999999999999e-05 2023-10-01T11:05:33.972 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.972 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.972 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:05:33.972 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:28.229401+0000", 2023-10-01T11:05:33.972 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.010331636999999999 2023-10-01T11:05:33.972 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:33.972 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:05:33.973 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:33.973 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.973 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.973 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:86405 13.b 13:d82dc497:::benchmark_data_smithi078_202524_object6847:head [delete] snapc 0=[] ondisk+write+known_if_redirected e322)", 2023-10-01T11:05:33.973 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:05:28.278029+0000", 2023-10-01T11:05:33.973 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 5.5285938009999995, 2023-10-01T11:05:33.973 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.010071172999999999, 2023-10-01T11:05:33.973 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:05:33.973 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:05:33.973 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:05:33.973 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:05:33.973 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:05:33.973 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 86405 2023-10-01T11:05:33.974 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.974 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:05:33.974 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.974 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:05:33.974 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:28.278029+0000", 2023-10-01T11:05:33.974 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:05:33.974 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.974 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.974 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:05:33.974 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:28.278029+0000", 2023-10-01T11:05:33.974 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-10-01T11:05:33.975 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.975 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.975 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:05:33.975 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:28.278028+0000", 2023-10-01T11:05:33.975 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.6959999999999999e-06 2023-10-01T11:05:33.975 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.975 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.975 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:05:33.975 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:28.278035+0000", 2023-10-01T11:05:33.975 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2549e-05 2023-10-01T11:05:33.975 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.976 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.976 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:05:33.976 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:28.278047+0000", 2023-10-01T11:05:33.976 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.6359999999999997e-06 2023-10-01T11:05:33.976 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.976 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.976 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:05:33.976 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:28.278054+0000", 2023-10-01T11:05:33.976 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.002401973 2023-10-01T11:05:33.976 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.976 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.977 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:05:33.977 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:28.280456+0000", 2023-10-01T11:05:33.977 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00015408800000000001 2023-10-01T11:05:33.977 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.977 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.977 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:05:33.977 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:28.280610+0000", 2023-10-01T11:05:33.977 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000113533 2023-10-01T11:05:33.977 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.977 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.977 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 2,3", 2023-10-01T11:05:33.977 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:28.280723+0000", 2023-10-01T11:05:33.978 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0025874029999999998 2023-10-01T11:05:33.978 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.978 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.978 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:05:33.978 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:28.283311+0000", 2023-10-01T11:05:33.978 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.001501491 2023-10-01T11:05:33.978 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.978 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.978 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:33.978 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:28.284812+0000", 2023-10-01T11:05:33.978 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0031551249999999999 2023-10-01T11:05:33.979 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.979 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.979 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:05:33.979 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:28.287967+0000", 2023-10-01T11:05:33.979 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.9660000000000006e-05 2023-10-01T11:05:33.979 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.979 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.979 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:05:33.980 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:28.288047+0000", 2023-10-01T11:05:33.980 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.3195999999999997e-05 2023-10-01T11:05:33.980 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:05:33.980 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:05:33.980 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:05:33.980 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:28.288100+0000", 2023-10-01T11:05:33.980 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.010071172999999999 2023-10-01T11:05:33.981 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:33.981 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:05:33.981 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:33.981 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:05:33.981 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:05:33.981 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-10-01T11:05:33.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:33.982 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:33.983 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:34.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:33 smithi078 bash[78274]: cluster 2023-10-01T11:05:32.282581+0000 mgr.y (mgr.25072) 1446 : cluster 0 pgmap v1531: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 77 op/s; 20578/218166 objects degraded (9.432%); 4565/218166 objects misplaced (2.092%); 341 KiB/s, 5 objects/s recovering 2023-10-01T11:05:34.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:33 smithi078 bash[78274]: cluster 2023-10-01T11:05:32.746966+0000 mon.a (mon.0) 1848 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:05:34.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:33 smithi078 bash[78274]: audit 2023-10-01T11:05:32.760464+0000 mon.a (mon.0) 1849 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T11:05:34.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:33 smithi078 bash[78274]: audit 2023-10-01T11:05:32.760612+0000 mon.a (mon.0) 1850 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-10-01T11:05:34.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:33 smithi078 bash[78274]: cluster 2023-10-01T11:05:32.760717+0000 mon.a (mon.0) 1851 : cluster 0 osdmap e323: 8 total, 8 up, 7 in 2023-10-01T11:05:34.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:33 smithi078 bash[78274]: cluster 2023-10-01T11:05:32.761169+0000 osd.5 (osd.5) 335 : cluster 3 failed to encode map e323 with expected crc 2023-10-01T11:05:34.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:33 smithi078 bash[78274]: cluster 2023-10-01T11:05:32.761172+0000 osd.6 (osd.6) 298 : cluster 3 failed to encode map e323 with expected crc 2023-10-01T11:05:34.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:33 smithi078 bash[78274]: cluster 2023-10-01T11:05:32.763006+0000 osd.6 (osd.6) 299 : cluster 3 failed to encode map e323 with expected crc 2023-10-01T11:05:34.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:33 smithi078 bash[78274]: cluster 2023-10-01T11:05:32.764747+0000 osd.5 (osd.5) 336 : cluster 3 failed to encode map e323 with expected crc 2023-10-01T11:05:34.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:33 smithi078 bash[78274]: cluster 2023-10-01T11:05:32.771209+0000 osd.7 (osd.7) 347 : cluster 3 failed to encode map e323 with expected crc 2023-10-01T11:05:34.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:33 smithi078 bash[75299]: cluster 2023-10-01T11:05:32.282581+0000 mgr.y (mgr.25072) 1446 : cluster 0 pgmap v1531: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 77 op/s; 20578/218166 objects degraded (9.432%); 4565/218166 objects misplaced (2.092%); 341 KiB/s, 5 objects/s recovering 2023-10-01T11:05:34.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:33 smithi078 bash[75299]: cluster 2023-10-01T11:05:32.746966+0000 mon.a (mon.0) 1848 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:05:34.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:33 smithi078 bash[75299]: audit 2023-10-01T11:05:32.760464+0000 mon.a (mon.0) 1849 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T11:05:34.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:33 smithi078 bash[75299]: audit 2023-10-01T11:05:32.760612+0000 mon.a (mon.0) 1850 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-10-01T11:05:34.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:33 smithi078 bash[75299]: cluster 2023-10-01T11:05:32.760717+0000 mon.a (mon.0) 1851 : cluster 0 osdmap e323: 8 total, 8 up, 7 in 2023-10-01T11:05:34.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:33 smithi078 bash[75299]: cluster 2023-10-01T11:05:32.761169+0000 osd.5 (osd.5) 335 : cluster 3 failed to encode map e323 with expected crc 2023-10-01T11:05:34.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:33 smithi078 bash[75299]: cluster 2023-10-01T11:05:32.761172+0000 osd.6 (osd.6) 298 : cluster 3 failed to encode map e323 with expected crc 2023-10-01T11:05:34.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:33 smithi078 bash[75299]: cluster 2023-10-01T11:05:32.763006+0000 osd.6 (osd.6) 299 : cluster 3 failed to encode map e323 with expected crc 2023-10-01T11:05:34.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:33 smithi078 bash[75299]: cluster 2023-10-01T11:05:32.764747+0000 osd.5 (osd.5) 336 : cluster 3 failed to encode map e323 with expected crc 2023-10-01T11:05:34.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:33 smithi078 bash[75299]: cluster 2023-10-01T11:05:32.771209+0000 osd.7 (osd.7) 347 : cluster 3 failed to encode map e323 with expected crc 2023-10-01T11:05:34.084 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:34.109 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:34.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:34.111 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:05:34.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:33 smithi173 bash[58816]: cluster 2023-10-01T11:05:32.282581+0000 mgr.y (mgr.25072) 1446 : cluster 0 pgmap v1531: 89 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.4 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 77 op/s; 20578/218166 objects degraded (9.432%); 4565/218166 objects misplaced (2.092%); 341 KiB/s, 5 objects/s recovering 2023-10-01T11:05:34.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:33 smithi173 bash[58816]: cluster 2023-10-01T11:05:32.746966+0000 mon.a (mon.0) 1848 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:05:34.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:33 smithi173 bash[58816]: audit 2023-10-01T11:05:32.760464+0000 mon.a (mon.0) 1849 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T11:05:34.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:33 smithi173 bash[58816]: audit 2023-10-01T11:05:32.760612+0000 mon.a (mon.0) 1850 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-10-01T11:05:34.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:33 smithi173 bash[58816]: cluster 2023-10-01T11:05:32.760717+0000 mon.a (mon.0) 1851 : cluster 0 osdmap e323: 8 total, 8 up, 7 in 2023-10-01T11:05:34.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:33 smithi173 bash[58816]: cluster 2023-10-01T11:05:32.761169+0000 osd.5 (osd.5) 335 : cluster 3 failed to encode map e323 with expected crc 2023-10-01T11:05:34.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:33 smithi173 bash[58816]: cluster 2023-10-01T11:05:32.761172+0000 osd.6 (osd.6) 298 : cluster 3 failed to encode map e323 with expected crc 2023-10-01T11:05:34.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:33 smithi173 bash[58816]: cluster 2023-10-01T11:05:32.763006+0000 osd.6 (osd.6) 299 : cluster 3 failed to encode map e323 with expected crc 2023-10-01T11:05:34.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:33 smithi173 bash[58816]: cluster 2023-10-01T11:05:32.764747+0000 osd.5 (osd.5) 336 : cluster 3 failed to encode map e323 with expected crc 2023-10-01T11:05:34.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:33 smithi173 bash[58816]: cluster 2023-10-01T11:05:32.771209+0000 osd.7 (osd.7) 347 : cluster 3 failed to encode map e323 with expected crc 2023-10-01T11:05:34.212 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:34.238 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:34.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:34.244 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:05:34.279 INFO:tasks.thrashosds.thrasher:Added osd 6 2023-10-01T11:05:34.344 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:34.365 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:34.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:34.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:05:34.468 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:34.502 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:34.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:34.505 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:05:34.605 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:34.631 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:34.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:34.633 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:05:34.734 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:34.764 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:34.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:34.768 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:05:34.869 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:34.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:34 smithi173 bash[58816]: cluster 2023-10-01T11:05:33.762788+0000 osd.5 (osd.5) 337 : cluster 3 failed to encode map e324 with expected crc 2023-10-01T11:05:34.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:34 smithi173 bash[58816]: cluster 2023-10-01T11:05:33.765414+0000 osd.5 (osd.5) 338 : cluster 3 failed to encode map e324 with expected crc 2023-10-01T11:05:34.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:34 smithi173 bash[58816]: cluster 2023-10-01T11:05:33.765597+0000 mon.a (mon.0) 1852 : cluster 0 osdmap e324: 8 total, 8 up, 7 in 2023-10-01T11:05:34.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:34 smithi173 bash[58816]: cluster 2023-10-01T11:05:33.765636+0000 osd.7 (osd.7) 348 : cluster 3 failed to encode map e324 with expected crc 2023-10-01T11:05:34.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:34 smithi173 bash[58816]: cluster 2023-10-01T11:05:33.765970+0000 osd.6 (osd.6) 300 : cluster 3 failed to encode map e324 with expected crc 2023-10-01T11:05:34.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:34 smithi173 bash[58816]: cluster 2023-10-01T11:05:33.767651+0000 osd.5 (osd.5) 339 : cluster 3 failed to encode map e324 with expected crc 2023-10-01T11:05:34.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:34 smithi173 bash[58816]: cluster 2023-10-01T11:05:33.767772+0000 osd.7 (osd.7) 349 : cluster 3 failed to encode map e324 with expected crc 2023-10-01T11:05:34.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:34 smithi173 bash[58816]: cluster 2023-10-01T11:05:33.768264+0000 osd.6 (osd.6) 301 : cluster 3 failed to encode map e324 with expected crc 2023-10-01T11:05:34.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:34 smithi173 bash[58816]: cluster 2023-10-01T11:05:33.769610+0000 osd.6 (osd.6) 302 : cluster 3 failed to encode map e324 with expected crc 2023-10-01T11:05:34.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:34 smithi173 bash[58816]: cluster 2023-10-01T11:05:33.769834+0000 osd.5 (osd.5) 340 : cluster 3 failed to encode map e324 with expected crc 2023-10-01T11:05:34.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:34 smithi173 bash[58816]: cluster 2023-10-01T11:05:33.769986+0000 osd.7 (osd.7) 350 : cluster 3 failed to encode map e324 with expected crc 2023-10-01T11:05:34.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:34 smithi173 bash[58816]: cluster 2023-10-01T11:05:33.771058+0000 osd.5 (osd.5) 341 : cluster 3 failed to encode map e324 with expected crc 2023-10-01T11:05:34.871 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:34 smithi173 bash[58816]: cluster 2023-10-01T11:05:33.772010+0000 osd.7 (osd.7) 351 : cluster 3 failed to encode map e324 with expected crc 2023-10-01T11:05:34.891 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:34.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:34.893 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:05:34.994 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:35.003 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:34 smithi078 bash[75299]: cluster 2023-10-01T11:05:33.762788+0000 osd.5 (osd.5) 337 : cluster 3 failed to encode map e324 with expected crc 2023-10-01T11:05:35.004 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:34 smithi078 bash[75299]: cluster 2023-10-01T11:05:33.765414+0000 osd.5 (osd.5) 338 : cluster 3 failed to encode map e324 with expected crc 2023-10-01T11:05:35.004 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:34 smithi078 bash[75299]: cluster 2023-10-01T11:05:33.765597+0000 mon.a (mon.0) 1852 : cluster 0 osdmap e324: 8 total, 8 up, 7 in 2023-10-01T11:05:35.004 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:34 smithi078 bash[75299]: cluster 2023-10-01T11:05:33.765636+0000 osd.7 (osd.7) 348 : cluster 3 failed to encode map e324 with expected crc 2023-10-01T11:05:35.004 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:34 smithi078 bash[75299]: cluster 2023-10-01T11:05:33.765970+0000 osd.6 (osd.6) 300 : cluster 3 failed to encode map e324 with expected crc 2023-10-01T11:05:35.004 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:34 smithi078 bash[75299]: cluster 2023-10-01T11:05:33.767651+0000 osd.5 (osd.5) 339 : cluster 3 failed to encode map e324 with expected crc 2023-10-01T11:05:35.004 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:34 smithi078 bash[75299]: cluster 2023-10-01T11:05:33.767772+0000 osd.7 (osd.7) 349 : cluster 3 failed to encode map e324 with expected crc 2023-10-01T11:05:35.004 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:34 smithi078 bash[75299]: cluster 2023-10-01T11:05:33.768264+0000 osd.6 (osd.6) 301 : cluster 3 failed to encode map e324 with expected crc 2023-10-01T11:05:35.004 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:34 smithi078 bash[75299]: cluster 2023-10-01T11:05:33.769610+0000 osd.6 (osd.6) 302 : cluster 3 failed to encode map e324 with expected crc 2023-10-01T11:05:35.004 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:34 smithi078 bash[75299]: cluster 2023-10-01T11:05:33.769834+0000 osd.5 (osd.5) 340 : cluster 3 failed to encode map e324 with expected crc 2023-10-01T11:05:35.004 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:34 smithi078 bash[75299]: cluster 2023-10-01T11:05:33.769986+0000 osd.7 (osd.7) 350 : cluster 3 failed to encode map e324 with expected crc 2023-10-01T11:05:35.005 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:34 smithi078 bash[75299]: cluster 2023-10-01T11:05:33.771058+0000 osd.5 (osd.5) 341 : cluster 3 failed to encode map e324 with expected crc 2023-10-01T11:05:35.005 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:34 smithi078 bash[75299]: cluster 2023-10-01T11:05:33.772010+0000 osd.7 (osd.7) 351 : cluster 3 failed to encode map e324 with expected crc 2023-10-01T11:05:35.005 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:34 smithi078 bash[78274]: cluster 2023-10-01T11:05:33.762788+0000 osd.5 (osd.5) 337 : cluster 3 failed to encode map e324 with expected crc 2023-10-01T11:05:35.005 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:34 smithi078 bash[78274]: cluster 2023-10-01T11:05:33.765414+0000 osd.5 (osd.5) 338 : cluster 3 failed to encode map e324 with expected crc 2023-10-01T11:05:35.005 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:34 smithi078 bash[78274]: cluster 2023-10-01T11:05:33.765597+0000 mon.a (mon.0) 1852 : cluster 0 osdmap e324: 8 total, 8 up, 7 in 2023-10-01T11:05:35.005 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:34 smithi078 bash[78274]: cluster 2023-10-01T11:05:33.765636+0000 osd.7 (osd.7) 348 : cluster 3 failed to encode map e324 with expected crc 2023-10-01T11:05:35.005 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:34 smithi078 bash[78274]: cluster 2023-10-01T11:05:33.765970+0000 osd.6 (osd.6) 300 : cluster 3 failed to encode map e324 with expected crc 2023-10-01T11:05:35.005 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:34 smithi078 bash[78274]: cluster 2023-10-01T11:05:33.767651+0000 osd.5 (osd.5) 339 : cluster 3 failed to encode map e324 with expected crc 2023-10-01T11:05:35.005 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:34 smithi078 bash[78274]: cluster 2023-10-01T11:05:33.767772+0000 osd.7 (osd.7) 349 : cluster 3 failed to encode map e324 with expected crc 2023-10-01T11:05:35.005 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:34 smithi078 bash[78274]: cluster 2023-10-01T11:05:33.768264+0000 osd.6 (osd.6) 301 : cluster 3 failed to encode map e324 with expected crc 2023-10-01T11:05:35.006 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:34 smithi078 bash[78274]: cluster 2023-10-01T11:05:33.769610+0000 osd.6 (osd.6) 302 : cluster 3 failed to encode map e324 with expected crc 2023-10-01T11:05:35.006 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:34 smithi078 bash[78274]: cluster 2023-10-01T11:05:33.769834+0000 osd.5 (osd.5) 340 : cluster 3 failed to encode map e324 with expected crc 2023-10-01T11:05:35.006 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:34 smithi078 bash[78274]: cluster 2023-10-01T11:05:33.769986+0000 osd.7 (osd.7) 350 : cluster 3 failed to encode map e324 with expected crc 2023-10-01T11:05:35.006 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:34 smithi078 bash[78274]: cluster 2023-10-01T11:05:33.771058+0000 osd.5 (osd.5) 341 : cluster 3 failed to encode map e324 with expected crc 2023-10-01T11:05:35.006 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:34 smithi078 bash[78274]: cluster 2023-10-01T11:05:33.772010+0000 osd.7 (osd.7) 351 : cluster 3 failed to encode map e324 with expected crc 2023-10-01T11:05:35.020 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:35.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:35.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:05:35.124 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:35.135 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-01T11:05:35.145 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:35.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:35.147 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:05:35.248 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:35.278 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:35.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:35.280 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:05:35.381 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:35.524 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:35.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:35.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:35.628 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:35.658 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:35.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:35.661 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:05:35.762 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:35.788 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:35.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:35.790 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:35.891 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:35.921 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:35.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:35.924 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:05:36.025 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:36.037 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:35 smithi078 bash[78274]: cluster 2023-10-01T11:05:34.283195+0000 mgr.y (mgr.25072) 1447 : cluster 0 pgmap v1534: 89 pgs: 1 remapped+peering, 15 peering, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 68 active+clean; 4.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 98 op/s; 20578/217368 objects degraded (9.467%); 4565/217368 objects misplaced (2.100%); 336 KiB/s, 4 objects/s recovering 2023-10-01T11:05:36.038 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:35 smithi078 bash[78274]: cluster 2023-10-01T11:05:34.759305+0000 mon.a (mon.0) 1853 : cluster 3 Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-01T11:05:36.038 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:35 smithi078 bash[78274]: cluster 2023-10-01T11:05:34.771958+0000 osd.6 (osd.6) 303 : cluster 3 failed to encode map e325 with expected crc 2023-10-01T11:05:36.038 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:35 smithi078 bash[78274]: cluster 2023-10-01T11:05:34.772138+0000 osd.7 (osd.7) 352 : cluster 3 failed to encode map e325 with expected crc 2023-10-01T11:05:36.038 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:35 smithi078 bash[78274]: cluster 2023-10-01T11:05:34.775807+0000 mon.a (mon.0) 1854 : cluster 0 osdmap e325: 8 total, 8 up, 7 in 2023-10-01T11:05:36.038 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:35 smithi078 bash[78274]: cluster 2023-10-01T11:05:34.778032+0000 osd.2 (osd.2) 18 : cluster 0 13.8 starting backfill to osd.6 from (0'0,298'4676] MIN to 322'5460 2023-10-01T11:05:36.038 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:35 smithi078 bash[78274]: cluster 2023-10-01T11:05:34.778297+0000 osd.7 (osd.7) 353 : cluster 3 failed to encode map e325 with expected crc 2023-10-01T11:05:36.038 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:35 smithi078 bash[78274]: cluster 2023-10-01T11:05:34.791331+0000 osd.5 (osd.5) 342 : cluster 3 failed to encode map e325 with expected crc 2023-10-01T11:05:36.039 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:35 smithi078 bash[75299]: cluster 2023-10-01T11:05:34.283195+0000 mgr.y (mgr.25072) 1447 : cluster 0 pgmap v1534: 89 pgs: 1 remapped+peering, 15 peering, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 68 active+clean; 4.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 98 op/s; 20578/217368 objects degraded (9.467%); 4565/217368 objects misplaced (2.100%); 336 KiB/s, 4 objects/s recovering 2023-10-01T11:05:36.039 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:35 smithi078 bash[75299]: cluster 2023-10-01T11:05:34.759305+0000 mon.a (mon.0) 1853 : cluster 3 Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-01T11:05:36.039 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:35 smithi078 bash[75299]: cluster 2023-10-01T11:05:34.771958+0000 osd.6 (osd.6) 303 : cluster 3 failed to encode map e325 with expected crc 2023-10-01T11:05:36.039 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:35 smithi078 bash[75299]: cluster 2023-10-01T11:05:34.772138+0000 osd.7 (osd.7) 352 : cluster 3 failed to encode map e325 with expected crc 2023-10-01T11:05:36.039 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:35 smithi078 bash[75299]: cluster 2023-10-01T11:05:34.775807+0000 mon.a (mon.0) 1854 : cluster 0 osdmap e325: 8 total, 8 up, 7 in 2023-10-01T11:05:36.039 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:35 smithi078 bash[75299]: cluster 2023-10-01T11:05:34.778032+0000 osd.2 (osd.2) 18 : cluster 0 13.8 starting backfill to osd.6 from (0'0,298'4676] MIN to 322'5460 2023-10-01T11:05:36.039 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:35 smithi078 bash[75299]: cluster 2023-10-01T11:05:34.778297+0000 osd.7 (osd.7) 353 : cluster 3 failed to encode map e325 with expected crc 2023-10-01T11:05:36.039 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:35 smithi078 bash[75299]: cluster 2023-10-01T11:05:34.791331+0000 osd.5 (osd.5) 342 : cluster 3 failed to encode map e325 with expected crc 2023-10-01T11:05:36.059 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:36.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:36.061 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:05:36.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:35 smithi173 bash[58816]: cluster 2023-10-01T11:05:34.283195+0000 mgr.y (mgr.25072) 1447 : cluster 0 pgmap v1534: 89 pgs: 1 remapped+peering, 15 peering, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 68 active+clean; 4.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 98 op/s; 20578/217368 objects degraded (9.467%); 4565/217368 objects misplaced (2.100%); 336 KiB/s, 4 objects/s recovering 2023-10-01T11:05:36.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:35 smithi173 bash[58816]: cluster 2023-10-01T11:05:34.759305+0000 mon.a (mon.0) 1853 : cluster 3 Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-10-01T11:05:36.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:35 smithi173 bash[58816]: cluster 2023-10-01T11:05:34.771958+0000 osd.6 (osd.6) 303 : cluster 3 failed to encode map e325 with expected crc 2023-10-01T11:05:36.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:35 smithi173 bash[58816]: cluster 2023-10-01T11:05:34.772138+0000 osd.7 (osd.7) 352 : cluster 3 failed to encode map e325 with expected crc 2023-10-01T11:05:36.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:35 smithi173 bash[58816]: cluster 2023-10-01T11:05:34.775807+0000 mon.a (mon.0) 1854 : cluster 0 osdmap e325: 8 total, 8 up, 7 in 2023-10-01T11:05:36.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:35 smithi173 bash[58816]: cluster 2023-10-01T11:05:34.778032+0000 osd.2 (osd.2) 18 : cluster 0 13.8 starting backfill to osd.6 from (0'0,298'4676] MIN to 322'5460 2023-10-01T11:05:36.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:35 smithi173 bash[58816]: cluster 2023-10-01T11:05:34.778297+0000 osd.7 (osd.7) 353 : cluster 3 failed to encode map e325 with expected crc 2023-10-01T11:05:36.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:35 smithi173 bash[58816]: cluster 2023-10-01T11:05:34.791331+0000 osd.5 (osd.5) 342 : cluster 3 failed to encode map e325 with expected crc 2023-10-01T11:05:36.162 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:36.192 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:36.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:36.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:05:36.258 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T11:05:36.296 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:36.326 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:36.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:36.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:05:36.429 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:36.455 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:36.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:36.457 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:05:36.557 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:36.582 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:36.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:36.585 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:05:36.686 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:36.718 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:36.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:36.720 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:05:36.789 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:05:36.821 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:36.846 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:36.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:36.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:05:36.949 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:36.979 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:36.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:36.982 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:05:37.083 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:37.114 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:37.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:37.117 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:05:37.218 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:37.242 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:37.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:37.245 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:05:37.345 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:37.371 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:37.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:37.374 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:37.474 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:37.499 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:37.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:37.501 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:37.602 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:37.629 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:37.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:37.631 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:37.732 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:37.758 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:37.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:37.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:05:37.861 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:37.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:37 smithi173 bash[58816]: cluster 2023-10-01T11:05:36.283766+0000 mgr.y (mgr.25072) 1448 : cluster 0 pgmap v1536: 89 pgs: 1 remapped+peering, 15 peering, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 68 active+clean; 4.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 52 op/s; 20578/217179 objects degraded (9.475%); 4565/217179 objects misplaced (2.102%); 97 KiB/s, 1 keys/s, 1 objects/s recovering 2023-10-01T11:05:37.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:37 smithi173 bash[58816]: cluster 2023-10-01T11:05:37.733632+0000 mon.a (mon.0) 1855 : cluster 3 Health check update: Degraded data redundancy: 20578/217179 objects degraded (9.475%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T11:05:37.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:37 smithi173 bash[58816]: cluster 2023-10-01T11:05:37.740617+0000 mon.a (mon.0) 1856 : cluster 0 osdmap e326: 8 total, 8 up, 7 in 2023-10-01T11:05:37.891 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:37.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:37.894 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:05:37.995 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:38.023 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:38.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:38.025 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:05:38.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:37 smithi078 bash[75299]: cluster 2023-10-01T11:05:36.283766+0000 mgr.y (mgr.25072) 1448 : cluster 0 pgmap v1536: 89 pgs: 1 remapped+peering, 15 peering, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 68 active+clean; 4.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 52 op/s; 20578/217179 objects degraded (9.475%); 4565/217179 objects misplaced (2.102%); 97 KiB/s, 1 keys/s, 1 objects/s recovering 2023-10-01T11:05:38.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:37 smithi078 bash[75299]: cluster 2023-10-01T11:05:37.733632+0000 mon.a (mon.0) 1855 : cluster 3 Health check update: Degraded data redundancy: 20578/217179 objects degraded (9.475%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T11:05:38.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:37 smithi078 bash[75299]: cluster 2023-10-01T11:05:37.740617+0000 mon.a (mon.0) 1856 : cluster 0 osdmap e326: 8 total, 8 up, 7 in 2023-10-01T11:05:38.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:37 smithi078 bash[78274]: cluster 2023-10-01T11:05:36.283766+0000 mgr.y (mgr.25072) 1448 : cluster 0 pgmap v1536: 89 pgs: 1 remapped+peering, 15 peering, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 68 active+clean; 4.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 52 op/s; 20578/217179 objects degraded (9.475%); 4565/217179 objects misplaced (2.102%); 97 KiB/s, 1 keys/s, 1 objects/s recovering 2023-10-01T11:05:38.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:37 smithi078 bash[78274]: cluster 2023-10-01T11:05:37.733632+0000 mon.a (mon.0) 1855 : cluster 3 Health check update: Degraded data redundancy: 20578/217179 objects degraded (9.475%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T11:05:38.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:37 smithi078 bash[78274]: cluster 2023-10-01T11:05:37.740617+0000 mon.a (mon.0) 1856 : cluster 0 osdmap e326: 8 total, 8 up, 7 in 2023-10-01T11:05:38.126 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:38.156 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:38.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:38.159 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:38.260 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:38.288 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:38.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:38.292 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:05:38.392 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:38.416 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:38.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:38.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:38.519 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:38.549 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:38.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:38.552 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:05:38.653 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:38.676 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:38.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:38.679 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:38.780 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:38.803 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:38.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:38.806 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:38.907 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:38.918 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:38 smithi173 bash[58816]: cluster 2023-10-01T11:05:37.741628+0000 osd.6 (osd.6) 304 : cluster 3 failed to encode map e326 with expected crc 2023-10-01T11:05:38.918 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:38 smithi173 bash[58816]: cluster 2023-10-01T11:05:38.754884+0000 mon.a (mon.0) 1857 : cluster 0 osdmap e327: 8 total, 8 up, 7 in 2023-10-01T11:05:38.932 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:38.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:38.935 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:05:39.035 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:39.062 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:39.062 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:38 smithi078 bash[75299]: cluster 2023-10-01T11:05:37.741628+0000 osd.6 (osd.6) 304 : cluster 3 failed to encode map e326 with expected crc 2023-10-01T11:05:39.062 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:38 smithi078 bash[75299]: cluster 2023-10-01T11:05:38.754884+0000 mon.a (mon.0) 1857 : cluster 0 osdmap e327: 8 total, 8 up, 7 in 2023-10-01T11:05:39.063 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:38 smithi078 bash[78274]: cluster 2023-10-01T11:05:37.741628+0000 osd.6 (osd.6) 304 : cluster 3 failed to encode map e326 with expected crc 2023-10-01T11:05:39.063 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:38 smithi078 bash[78274]: cluster 2023-10-01T11:05:38.754884+0000 mon.a (mon.0) 1857 : cluster 0 osdmap e327: 8 total, 8 up, 7 in 2023-10-01T11:05:39.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:39.065 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:05:39.166 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:39.194 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:39.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:39.197 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:05:39.279 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6] out_osds: [7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T11:05:39.280 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T11:05:39.280 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2023-10-01T11:05:39.280 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-01T11:05:39.280 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-01T11:05:39.280 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 config set filestore_inject_stall 3 2023-10-01T11:05:39.298 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:39.331 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:39.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:39.334 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:05:39.435 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:39.458 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:39.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:39.460 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:05:39.561 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:39.590 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:39.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:39.593 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:05:39.693 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:39.717 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:39.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:39.722 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:05:39.823 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:39.849 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:39.850 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:39 smithi078 bash[75299]: cluster 2023-10-01T11:05:38.284333+0000 mgr.y (mgr.25072) 1449 : cluster 0 pgmap v1538: 89 pgs: 1 remapped+peering, 11 peering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 29 op/s; 5303/217149 objects degraded (2.442%); 9177/217149 objects misplaced (4.226%); 4.2 MiB/s, 1 keys/s, 140 objects/s recovering 2023-10-01T11:05:39.850 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:39 smithi078 bash[75299]: cluster 2023-10-01T11:05:38.751235+0000 osd.5 (osd.5) 343 : cluster 3 failed to encode map e327 with expected crc 2023-10-01T11:05:39.850 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:39 smithi078 bash[75299]: cluster 2023-10-01T11:05:38.753335+0000 osd.7 (osd.7) 354 : cluster 3 failed to encode map e327 with expected crc 2023-10-01T11:05:39.851 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:39 smithi078 bash[75299]: cluster 2023-10-01T11:05:38.753373+0000 osd.6 (osd.6) 305 : cluster 3 failed to encode map e327 with expected crc 2023-10-01T11:05:39.851 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:39 smithi078 bash[75299]: cluster 2023-10-01T11:05:38.755596+0000 osd.6 (osd.6) 306 : cluster 3 failed to encode map e327 with expected crc 2023-10-01T11:05:39.851 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:39 smithi078 bash[78274]: cluster 2023-10-01T11:05:38.284333+0000 mgr.y (mgr.25072) 1449 : cluster 0 pgmap v1538: 89 pgs: 1 remapped+peering, 11 peering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 29 op/s; 5303/217149 objects degraded (2.442%); 9177/217149 objects misplaced (4.226%); 4.2 MiB/s, 1 keys/s, 140 objects/s recovering 2023-10-01T11:05:39.851 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:39 smithi078 bash[78274]: cluster 2023-10-01T11:05:38.751235+0000 osd.5 (osd.5) 343 : cluster 3 failed to encode map e327 with expected crc 2023-10-01T11:05:39.851 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:39 smithi078 bash[78274]: cluster 2023-10-01T11:05:38.753335+0000 osd.7 (osd.7) 354 : cluster 3 failed to encode map e327 with expected crc 2023-10-01T11:05:39.851 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:39 smithi078 bash[78274]: cluster 2023-10-01T11:05:38.753373+0000 osd.6 (osd.6) 305 : cluster 3 failed to encode map e327 with expected crc 2023-10-01T11:05:39.851 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:39 smithi078 bash[78274]: cluster 2023-10-01T11:05:38.755596+0000 osd.6 (osd.6) 306 : cluster 3 failed to encode map e327 with expected crc 2023-10-01T11:05:39.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:39.852 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:05:39.904 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T11:05:39.953 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:39.977 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:39.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:39.979 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:05:40.080 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:40.108 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:40.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:40.111 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:05:40.127 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:39 smithi173 bash[58816]: cluster 2023-10-01T11:05:38.284333+0000 mgr.y (mgr.25072) 1449 : cluster 0 pgmap v1538: 89 pgs: 1 remapped+peering, 11 peering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 29 op/s; 5303/217149 objects degraded (2.442%); 9177/217149 objects misplaced (4.226%); 4.2 MiB/s, 1 keys/s, 140 objects/s recovering 2023-10-01T11:05:40.127 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:39 smithi173 bash[58816]: cluster 2023-10-01T11:05:38.751235+0000 osd.5 (osd.5) 343 : cluster 3 failed to encode map e327 with expected crc 2023-10-01T11:05:40.127 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:39 smithi173 bash[58816]: cluster 2023-10-01T11:05:38.753335+0000 osd.7 (osd.7) 354 : cluster 3 failed to encode map e327 with expected crc 2023-10-01T11:05:40.127 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:39 smithi173 bash[58816]: cluster 2023-10-01T11:05:38.753373+0000 osd.6 (osd.6) 305 : cluster 3 failed to encode map e327 with expected crc 2023-10-01T11:05:40.127 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:39 smithi173 bash[58816]: cluster 2023-10-01T11:05:38.755596+0000 osd.6 (osd.6) 306 : cluster 3 failed to encode map e327 with expected crc 2023-10-01T11:05:40.212 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:40.237 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:40.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:40.240 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:05:40.341 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:40.367 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:40.368 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:05:40 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:05:40] "GET /metrics HTTP/1.1" 200 36401 "" "Prometheus/2.33.4" 2023-10-01T11:05:40.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:40.369 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:05:40.470 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:40.493 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:40.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:40.499 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:05:40.599 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:40.632 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:40.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:40.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:05:40.735 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:40.764 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:40.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:40.767 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:05:40.867 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:40.889 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:40.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:40.892 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:05:40.903 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:40 smithi078 bash[78274]: cluster 2023-10-01T11:05:40.785405+0000 mon.a (mon.0) 1858 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-01T11:05:40.903 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:40 smithi078 bash[75299]: cluster 2023-10-01T11:05:40.785405+0000 mon.a (mon.0) 1858 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-01T11:05:40.993 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:41.029 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:41.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:41.031 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:05:41.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:40 smithi173 bash[58816]: cluster 2023-10-01T11:05:40.785405+0000 mon.a (mon.0) 1858 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-10-01T11:05:41.131 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:41.152 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:41.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:41.154 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:05:41.255 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:41.279 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:41.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:41.281 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:05:41.343 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:05:41.383 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:41.407 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:41.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:41.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:05:41.510 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:41.535 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:41.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:41.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:05:41.638 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:41.667 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:41.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:41.671 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:41.771 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:41.800 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:41.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:41.803 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:05:41.904 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:41.931 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:41.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:41.933 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:05:42.034 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:42.042 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:41 smithi078 bash[75299]: cluster 2023-10-01T11:05:40.285177+0000 mgr.y (mgr.25072) 1450 : cluster 0 pgmap v1540: 89 pgs: 1 activating+undersized+degraded+remapped, 1 peering, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 20 op/s; 6164/217050 objects degraded (2.840%); 4612/217050 objects misplaced (2.125%); 2.1 MiB/s, 1 keys/s, 36 objects/s recovering 2023-10-01T11:05:42.042 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:41 smithi078 bash[78274]: cluster 2023-10-01T11:05:40.285177+0000 mgr.y (mgr.25072) 1450 : cluster 0 pgmap v1540: 89 pgs: 1 activating+undersized+degraded+remapped, 1 peering, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 20 op/s; 6164/217050 objects degraded (2.840%); 4612/217050 objects misplaced (2.125%); 2.1 MiB/s, 1 keys/s, 36 objects/s recovering 2023-10-01T11:05:42.060 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:42.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:42.062 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:05:42.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:41 smithi173 bash[58816]: cluster 2023-10-01T11:05:40.285177+0000 mgr.y (mgr.25072) 1450 : cluster 0 pgmap v1540: 89 pgs: 1 activating+undersized+degraded+remapped, 1 peering, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 20 op/s; 6164/217050 objects degraded (2.840%); 4612/217050 objects misplaced (2.125%); 2.1 MiB/s, 1 keys/s, 36 objects/s recovering 2023-10-01T11:05:42.164 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:42.188 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:42.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:42.191 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:05:42.291 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:42.317 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:42.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:42.319 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:05:42.420 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:42.447 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:42.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:42.450 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:05:42.551 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:42.580 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:42.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:42.582 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:05:42.683 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:42.712 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:42.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:42.714 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:42.815 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:42.841 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:42.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:42.843 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:05:42.944 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:42.950 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:42 smithi173 bash[58816]: cluster 2023-10-01T11:05:42.735084+0000 mon.a (mon.0) 1859 : cluster 3 Health check update: Degraded data redundancy: 6164/217050 objects degraded (2.840%), 5 pgs degraded (PG_DEGRADED) 2023-10-01T11:05:42.966 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:42.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:42.968 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:05:43.069 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:43.077 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:43 smithi078 bash[78274]: cluster 2023-10-01T11:05:42.735084+0000 mon.a (mon.0) 1859 : cluster 3 Health check update: Degraded data redundancy: 6164/217050 objects degraded (2.840%), 5 pgs degraded (PG_DEGRADED) 2023-10-01T11:05:43.077 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:43 smithi078 bash[75299]: cluster 2023-10-01T11:05:42.735084+0000 mon.a (mon.0) 1859 : cluster 3 Health check update: Degraded data redundancy: 6164/217050 objects degraded (2.840%), 5 pgs degraded (PG_DEGRADED) 2023-10-01T11:05:43.089 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:43.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:43.091 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:05:43.192 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:43.217 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:43.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:43.219 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:05:43.320 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:43.343 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:43.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:43.346 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:05:43.446 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:43.467 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:43.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:43.470 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:05:43.571 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:43.598 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:43.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:43.602 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:05:43.703 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:43.729 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:43.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:43.732 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:05:43.833 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:43.863 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:43.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:43.866 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:05:43.953 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T11:05:43.967 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:43.990 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:43.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:43.992 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:05:44.093 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:44.118 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:44.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:44.120 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:05:44.177 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:43 smithi173 bash[58816]: cluster 2023-10-01T11:05:42.285859+0000 mgr.y (mgr.25072) 1451 : cluster 0 pgmap v1541: 89 pgs: 1 activating+undersized+degraded+remapped, 1 peering, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 7 op/s; 6164/217050 objects degraded (2.840%); 4612/217050 objects misplaced (2.125%); 1.6 MiB/s, 28 objects/s recovering 2023-10-01T11:05:44.221 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:44.231 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:43 smithi078 bash[75299]: cluster 2023-10-01T11:05:42.285859+0000 mgr.y (mgr.25072) 1451 : cluster 0 pgmap v1541: 89 pgs: 1 activating+undersized+degraded+remapped, 1 peering, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 7 op/s; 6164/217050 objects degraded (2.840%); 4612/217050 objects misplaced (2.125%); 1.6 MiB/s, 28 objects/s recovering 2023-10-01T11:05:44.231 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:43 smithi078 bash[78274]: cluster 2023-10-01T11:05:42.285859+0000 mgr.y (mgr.25072) 1451 : cluster 0 pgmap v1541: 89 pgs: 1 activating+undersized+degraded+remapped, 1 peering, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 7 op/s; 6164/217050 objects degraded (2.840%); 4612/217050 objects misplaced (2.125%); 1.6 MiB/s, 28 objects/s recovering 2023-10-01T11:05:44.247 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:44.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:44.249 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:05:44.350 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:44.372 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:44.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:44.375 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:44.476 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:44.500 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:44.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:44.505 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:05:44.606 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:44.631 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:44.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:44.633 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:05:44.734 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:44.756 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:44.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:44.758 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:05:44.856 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T11:05:44.859 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:44.884 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:44.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:44.887 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:05:44.887 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T11:05:44.908 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T11:05:44.933 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T11:05:44.963 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T11:05:44.988 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:44.995 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T11:05:45.014 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:45.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:45.016 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:05:45.020 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T11:05:45.044 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T11:05:45.045 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T11:05:45.045 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T11:05:45.046 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T11:05:45.046 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T11:05:45.046 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T11:05:45.046 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T11:05:45.046 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T11:05:45.046 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T11:05:45.117 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:45.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:44 smithi173 bash[58816]: audit 2023-10-01T11:05:44.830879+0000 mon.b (mon.1) 56 : audit 0 from='client.? 172.21.15.78:0/2183971728' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:05:45.124 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:44 smithi078 bash[75299]: audit 2023-10-01T11:05:44.830879+0000 mon.b (mon.1) 56 : audit 0 from='client.? 172.21.15.78:0/2183971728' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:05:45.125 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:44 smithi078 bash[78274]: audit 2023-10-01T11:05:44.830879+0000 mon.b (mon.1) 56 : audit 0 from='client.? 172.21.15.78:0/2183971728' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:05:45.140 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:45.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:45.144 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:05:45.244 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:45.266 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:45.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:45.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:05:45.302 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-10-01T11:05:45.303 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T11:05:45.372 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:45.396 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:45.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:45.398 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:05:45.499 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:45.525 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:45.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:45.527 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:05:45.628 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:45.650 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:45.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:45.652 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:05:45.753 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:45.778 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:45.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:45.780 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:45.881 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:45.902 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:45.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:45.906 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:05:46.007 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:46.017 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:45 smithi173 bash[58816]: cluster 2023-10-01T11:05:44.287053+0000 mgr.y (mgr.25072) 1452 : cluster 0 pgmap v1542: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 7 op/s; 3869/217050 objects degraded (1.783%); 5925/217050 objects misplaced (2.730%); 5.7 MiB/s, 93 objects/s recovering 2023-10-01T11:05:46.031 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:46.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:46.034 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:05:46.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:45 smithi078 bash[75299]: cluster 2023-10-01T11:05:44.287053+0000 mgr.y (mgr.25072) 1452 : cluster 0 pgmap v1542: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 7 op/s; 3869/217050 objects degraded (1.783%); 5925/217050 objects misplaced (2.730%); 5.7 MiB/s, 93 objects/s recovering 2023-10-01T11:05:46.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:45 smithi078 bash[78274]: cluster 2023-10-01T11:05:44.287053+0000 mgr.y (mgr.25072) 1452 : cluster 0 pgmap v1542: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 7 op/s; 3869/217050 objects degraded (1.783%); 5925/217050 objects misplaced (2.730%); 5.7 MiB/s, 93 objects/s recovering 2023-10-01T11:05:46.135 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:46.159 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:46.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:46.162 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:05:46.262 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:46.291 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:46.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:46.295 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:05:46.395 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:46.414 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:46.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:46.416 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:05:46.516 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:46.541 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:46.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:46.543 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:05:46.645 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:46.667 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:46.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:46.670 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:05:46.771 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:46.793 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:46.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:46.799 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:05:46.899 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:46.923 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:46.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:46.926 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:05:46.959 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-01T11:05:47.026 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:47.072 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:47.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:47.075 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:47.176 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:47.198 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:47.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:47.201 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:05:47.301 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:47.327 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:47.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:47.330 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:05:47.430 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:47.452 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:47.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:47.454 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:47.555 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:47.582 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:47.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:47.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:05:47.685 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:47.710 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:47.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:47.712 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:05:47.813 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:47.843 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:47.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:47.847 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:05:47.948 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:47.958 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:47 smithi078 bash[75299]: cluster 2023-10-01T11:05:46.287835+0000 mgr.y (mgr.25072) 1453 : cluster 0 pgmap v1543: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 7 op/s; 3869/217017 objects degraded (1.783%); 5925/217017 objects misplaced (2.730%); 2.7 MiB/s, 0 objects/s recovering 2023-10-01T11:05:47.959 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:47 smithi078 bash[75299]: audit 2023-10-01T11:05:47.460193+0000 mon.a (mon.0) 1860 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:05:47.959 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:47 smithi078 bash[75299]: cluster 2023-10-01T11:05:47.736220+0000 mon.a (mon.0) 1861 : cluster 3 Health check update: Degraded data redundancy: 3869/217017 objects degraded (1.783%), 4 pgs degraded (PG_DEGRADED) 2023-10-01T11:05:47.960 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:47 smithi078 bash[78274]: cluster 2023-10-01T11:05:46.287835+0000 mgr.y (mgr.25072) 1453 : cluster 0 pgmap v1543: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 7 op/s; 3869/217017 objects degraded (1.783%); 5925/217017 objects misplaced (2.730%); 2.7 MiB/s, 0 objects/s recovering 2023-10-01T11:05:47.960 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:47 smithi078 bash[78274]: audit 2023-10-01T11:05:47.460193+0000 mon.a (mon.0) 1860 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:05:47.960 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:47 smithi078 bash[78274]: cluster 2023-10-01T11:05:47.736220+0000 mon.a (mon.0) 1861 : cluster 3 Health check update: Degraded data redundancy: 3869/217017 objects degraded (1.783%), 4 pgs degraded (PG_DEGRADED) 2023-10-01T11:05:47.974 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:47.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:47.977 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:05:48.077 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:48.085 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:47 smithi173 bash[58816]: cluster 2023-10-01T11:05:46.287835+0000 mgr.y (mgr.25072) 1453 : cluster 0 pgmap v1543: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 7 op/s; 3869/217017 objects degraded (1.783%); 5925/217017 objects misplaced (2.730%); 2.7 MiB/s, 0 objects/s recovering 2023-10-01T11:05:48.085 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:47 smithi173 bash[58816]: audit 2023-10-01T11:05:47.460193+0000 mon.a (mon.0) 1860 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:05:48.085 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:47 smithi173 bash[58816]: cluster 2023-10-01T11:05:47.736220+0000 mon.a (mon.0) 1861 : cluster 3 Health check update: Degraded data redundancy: 3869/217017 objects degraded (1.783%), 4 pgs degraded (PG_DEGRADED) 2023-10-01T11:05:48.100 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:48.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:48.104 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:05:48.205 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:48.232 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:48.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:48.235 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:05:48.336 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:48.359 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:48.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:48.361 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:05:48.462 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:48.488 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:48.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:48.489 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:05:48.590 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:48.617 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:48.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:48.620 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:48.721 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:48.748 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:48.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:48.751 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:05:48.852 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:48.878 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:48.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:48.881 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:05:48.982 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:49.005 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:49.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:49.008 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:05:49.109 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:49.135 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:49.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:49.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:05:49.240 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:49.264 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:49.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:49.271 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:05:49.372 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:49.404 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:49.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:49.407 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:05:49.410 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T11:05:49.507 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:49.528 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-10-01T11:05:49.528 INFO:teuthology.orchestra.run.smithi173.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-01T11:05:49.528 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-10-01T11:05:49.535 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:49.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:49.541 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:05:49.642 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:49.665 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:49.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:49.671 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:05:49.771 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:49.795 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:49.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:49.797 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:05:49.898 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:49.923 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:49.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:49.925 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:05:50.025 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:50.036 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:49 smithi078 bash[78274]: cluster 2023-10-01T11:05:48.288630+0000 mgr.y (mgr.25072) 1454 : cluster 0 pgmap v1544: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 28 op/s; 3869/216354 objects degraded (1.788%); 5276/216354 objects misplaced (2.439%); 6.2 MiB/s, 60 objects/s recovering 2023-10-01T11:05:50.036 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:49 smithi078 bash[75299]: cluster 2023-10-01T11:05:48.288630+0000 mgr.y (mgr.25072) 1454 : cluster 0 pgmap v1544: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 28 op/s; 3869/216354 objects degraded (1.788%); 5276/216354 objects misplaced (2.439%); 6.2 MiB/s, 60 objects/s recovering 2023-10-01T11:05:50.055 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:50.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:50.058 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:05:50.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:49 smithi173 bash[58816]: cluster 2023-10-01T11:05:48.288630+0000 mgr.y (mgr.25072) 1454 : cluster 0 pgmap v1544: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 28 op/s; 3869/216354 objects degraded (1.788%); 5276/216354 objects misplaced (2.439%); 6.2 MiB/s, 60 objects/s recovering 2023-10-01T11:05:50.159 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:50.183 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:50.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:50.186 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:05:50.286 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:50.298 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:05:50 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:05:50] "GET /metrics HTTP/1.1" 200 36405 "" "Prometheus/2.33.4" 2023-10-01T11:05:50.314 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:50.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:50.318 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:05:50.419 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:50.450 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:50.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:50.452 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:50.553 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:50.583 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:50.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:50.585 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:05:50.686 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:50.713 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:50.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:50.715 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:05:50.816 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:50.845 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:50.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:50.848 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:05:50.892 INFO:teuthology.orchestra.run.smithi078.stderr:nodeep-scrub is unset 2023-10-01T11:05:50.949 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:50.978 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:50.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:50.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:05:51.080 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:50 smithi078 bash[75299]: audit 2023-10-01T11:05:49.950019+0000 mon.a (mon.0) 1862 : audit 1 from='client.? 172.21.15.78:0/853806492' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:05:51.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:50 smithi078 bash[78274]: audit 2023-10-01T11:05:49.950019+0000 mon.a (mon.0) 1862 : audit 1 from='client.? 172.21.15.78:0/853806492' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:05:51.083 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:51.110 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:51.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:51.115 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:51.216 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:51.247 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:51.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:51.250 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:05:51.351 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:51.360 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:50 smithi173 bash[58816]: audit 2023-10-01T11:05:49.950019+0000 mon.a (mon.0) 1862 : audit 1 from='client.? 172.21.15.78:0/853806492' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:05:51.378 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:51.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:51.382 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:05:51.482 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:51.514 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:51.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:51.516 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:05:51.617 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:51.644 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:51.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:51.646 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:05:51.747 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:51.774 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:51.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:51.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:05:51.779 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T11:05:51.877 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:51.901 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:51.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:51.903 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:05:52.004 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:52.030 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:52.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:52.033 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:05:52.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:51 smithi173 bash[58816]: cluster 2023-10-01T11:05:50.289879+0000 mgr.y (mgr.25072) 1455 : cluster 0 pgmap v1545: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 67 op/s; 3869/214788 objects degraded (1.801%); 5276/214788 objects misplaced (2.456%); 6.4 MiB/s, 101 objects/s recovering 2023-10-01T11:05:52.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:51 smithi173 bash[58816]: cluster 2023-10-01T11:05:50.871914+0000 mon.a (mon.0) 1863 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T11:05:52.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:51 smithi173 bash[58816]: cluster 2023-10-01T11:05:50.883664+0000 osd.7 (osd.7) 355 : cluster 3 failed to encode map e328 with expected crc 2023-10-01T11:05:52.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:51 smithi173 bash[58816]: audit 2023-10-01T11:05:50.884959+0000 mon.a (mon.0) 1864 : audit 1 from='client.? 172.21.15.78:0/853806492' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T11:05:52.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:51 smithi173 bash[58816]: cluster 2023-10-01T11:05:50.885031+0000 mon.a (mon.0) 1865 : cluster 0 osdmap e328: 8 total, 8 up, 7 in 2023-10-01T11:05:52.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:51 smithi173 bash[58816]: cluster 2023-10-01T11:05:50.891752+0000 osd.6 (osd.6) 307 : cluster 3 failed to encode map e328 with expected crc 2023-10-01T11:05:52.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:51 smithi173 bash[58816]: cluster 2023-10-01T11:05:50.908630+0000 osd.5 (osd.5) 344 : cluster 3 failed to encode map e328 with expected crc 2023-10-01T11:05:52.133 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:52.145 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:51 smithi078 bash[78274]: cluster 2023-10-01T11:05:50.289879+0000 mgr.y (mgr.25072) 1455 : cluster 0 pgmap v1545: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 67 op/s; 3869/214788 objects degraded (1.801%); 5276/214788 objects misplaced (2.456%); 6.4 MiB/s, 101 objects/s recovering 2023-10-01T11:05:52.145 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:51 smithi078 bash[78274]: cluster 2023-10-01T11:05:50.871914+0000 mon.a (mon.0) 1863 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T11:05:52.145 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:51 smithi078 bash[78274]: cluster 2023-10-01T11:05:50.883664+0000 osd.7 (osd.7) 355 : cluster 3 failed to encode map e328 with expected crc 2023-10-01T11:05:52.145 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:51 smithi078 bash[78274]: audit 2023-10-01T11:05:50.884959+0000 mon.a (mon.0) 1864 : audit 1 from='client.? 172.21.15.78:0/853806492' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T11:05:52.145 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:51 smithi078 bash[78274]: cluster 2023-10-01T11:05:50.885031+0000 mon.a (mon.0) 1865 : cluster 0 osdmap e328: 8 total, 8 up, 7 in 2023-10-01T11:05:52.145 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:51 smithi078 bash[78274]: cluster 2023-10-01T11:05:50.891752+0000 osd.6 (osd.6) 307 : cluster 3 failed to encode map e328 with expected crc 2023-10-01T11:05:52.145 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:51 smithi078 bash[78274]: cluster 2023-10-01T11:05:50.908630+0000 osd.5 (osd.5) 344 : cluster 3 failed to encode map e328 with expected crc 2023-10-01T11:05:52.146 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:51 smithi078 bash[75299]: cluster 2023-10-01T11:05:50.289879+0000 mgr.y (mgr.25072) 1455 : cluster 0 pgmap v1545: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 67 op/s; 3869/214788 objects degraded (1.801%); 5276/214788 objects misplaced (2.456%); 6.4 MiB/s, 101 objects/s recovering 2023-10-01T11:05:52.146 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:51 smithi078 bash[75299]: cluster 2023-10-01T11:05:50.871914+0000 mon.a (mon.0) 1863 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T11:05:52.146 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:51 smithi078 bash[75299]: cluster 2023-10-01T11:05:50.883664+0000 osd.7 (osd.7) 355 : cluster 3 failed to encode map e328 with expected crc 2023-10-01T11:05:52.146 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:51 smithi078 bash[75299]: audit 2023-10-01T11:05:50.884959+0000 mon.a (mon.0) 1864 : audit 1 from='client.? 172.21.15.78:0/853806492' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T11:05:52.146 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:51 smithi078 bash[75299]: cluster 2023-10-01T11:05:50.885031+0000 mon.a (mon.0) 1865 : cluster 0 osdmap e328: 8 total, 8 up, 7 in 2023-10-01T11:05:52.146 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:51 smithi078 bash[75299]: cluster 2023-10-01T11:05:50.891752+0000 osd.6 (osd.6) 307 : cluster 3 failed to encode map e328 with expected crc 2023-10-01T11:05:52.146 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:51 smithi078 bash[75299]: cluster 2023-10-01T11:05:50.908630+0000 osd.5 (osd.5) 344 : cluster 3 failed to encode map e328 with expected crc 2023-10-01T11:05:52.160 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:52.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:52.166 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:05:52.267 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:52.290 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:52.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:52.292 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:05:52.393 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:52.420 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:52.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:52.422 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:52.523 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:52.549 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:52.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:52.552 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:52.653 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:52.678 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:52.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:52.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:05:52.782 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:52.809 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:52.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:52.815 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:05:52.915 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:52.942 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:52.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:52.944 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:05:53.045 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:53.068 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:53.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:53.070 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:05:53.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:52 smithi078 bash[75299]: cluster 2023-10-01T11:05:52.737315+0000 mon.a (mon.0) 1866 : cluster 3 Health check update: Degraded data redundancy: 3869/214788 objects degraded (1.801%), 4 pgs degraded (PG_DEGRADED) 2023-10-01T11:05:53.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:52 smithi078 bash[78274]: cluster 2023-10-01T11:05:52.737315+0000 mon.a (mon.0) 1866 : cluster 3 Health check update: Degraded data redundancy: 3869/214788 objects degraded (1.801%), 4 pgs degraded (PG_DEGRADED) 2023-10-01T11:05:53.134 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:52 smithi173 bash[58816]: cluster 2023-10-01T11:05:52.737315+0000 mon.a (mon.0) 1866 : cluster 3 Health check update: Degraded data redundancy: 3869/214788 objects degraded (1.801%), 4 pgs degraded (PG_DEGRADED) 2023-10-01T11:05:53.171 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:53.194 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:53.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:53.195 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:05:53.296 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:53.322 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:05:53.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:53.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:05:53.425 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:53.453 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:53.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:53.456 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:05:53.557 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:53.587 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:53.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:53.590 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:05:53.690 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:53.712 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:53.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:53.714 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:05:53.815 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:53.841 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:53.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:53.844 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:05:53.945 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:53.972 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:53.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:53.974 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:05:54.075 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:54.080 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:53 smithi078 bash[78274]: cluster 2023-10-01T11:05:52.290689+0000 mgr.y (mgr.25072) 1456 : cluster 0 pgmap v1547: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 80 op/s; 3869/214635 objects degraded (1.803%); 5276/214635 objects misplaced (2.458%); 7.5 MiB/s, 120 objects/s recovering 2023-10-01T11:05:54.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:53 smithi078 bash[75299]: cluster 2023-10-01T11:05:52.290689+0000 mgr.y (mgr.25072) 1456 : cluster 0 pgmap v1547: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 80 op/s; 3869/214635 objects degraded (1.803%); 5276/214635 objects misplaced (2.458%); 7.5 MiB/s, 120 objects/s recovering 2023-10-01T11:05:54.103 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:54.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:54.105 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:05:54.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:53 smithi173 bash[58816]: cluster 2023-10-01T11:05:52.290689+0000 mgr.y (mgr.25072) 1456 : cluster 0 pgmap v1547: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 80 op/s; 3869/214635 objects degraded (1.803%); 5276/214635 objects misplaced (2.458%); 7.5 MiB/s, 120 objects/s recovering 2023-10-01T11:05:54.206 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:54.235 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:54.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:54.238 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:05:54.338 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:54.369 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:54.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:54.372 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:05:54.473 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:54.500 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:54.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:54.502 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:05:54.603 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:54.626 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:54.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:54.629 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:05:54.671 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T11:05:54.729 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:54.754 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:54.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:54.757 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:05:54.858 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:54.883 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:54.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:54.887 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:05:54.988 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:55.180 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:55.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:55.184 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:05:55.186 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:05:55.285 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:55.311 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:55.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:55.313 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:05:55.414 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:55.415 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-10-01T11:05:55.415 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T11:05:55.442 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:55.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:55.444 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:05:55.459 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6] out_osds: [7] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T11:05:55.459 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T11:05:55.459 INFO:tasks.thrashosds.thrasher:Adding osd 7 2023-10-01T11:05:55.459 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd in 7 2023-10-01T11:05:55.546 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:55.570 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:55.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:55.572 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:05:55.673 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:55.695 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:55.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:55.699 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:05:55.800 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:55.826 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:55.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:55.828 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:05:55.929 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:55.955 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:55.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:55.963 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:05:56.064 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:56.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:55 smithi078 bash[75299]: cluster 2023-10-01T11:05:54.291824+0000 mgr.y (mgr.25072) 1457 : cluster 0 pgmap v1548: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 128 op/s; 3869/213201 objects degraded (1.815%); 4531/213201 objects misplaced (2.125%); 8.5 MiB/s, 139 objects/s recovering 2023-10-01T11:05:56.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:55 smithi078 bash[75299]: cluster 2023-10-01T11:05:54.894871+0000 mon.a (mon.0) 1867 : cluster 0 osdmap e329: 8 total, 8 up, 7 in 2023-10-01T11:05:56.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:55 smithi078 bash[75299]: cluster 2023-10-01T11:05:54.896777+0000 osd.6 (osd.6) 308 : cluster 3 failed to encode map e329 with expected crc 2023-10-01T11:05:56.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:55 smithi078 bash[75299]: cluster 2023-10-01T11:05:54.899094+0000 osd.6 (osd.6) 309 : cluster 3 failed to encode map e329 with expected crc 2023-10-01T11:05:56.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:55 smithi078 bash[78274]: cluster 2023-10-01T11:05:54.291824+0000 mgr.y (mgr.25072) 1457 : cluster 0 pgmap v1548: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 128 op/s; 3869/213201 objects degraded (1.815%); 4531/213201 objects misplaced (2.125%); 8.5 MiB/s, 139 objects/s recovering 2023-10-01T11:05:56.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:55 smithi078 bash[78274]: cluster 2023-10-01T11:05:54.894871+0000 mon.a (mon.0) 1867 : cluster 0 osdmap e329: 8 total, 8 up, 7 in 2023-10-01T11:05:56.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:55 smithi078 bash[78274]: cluster 2023-10-01T11:05:54.896777+0000 osd.6 (osd.6) 308 : cluster 3 failed to encode map e329 with expected crc 2023-10-01T11:05:56.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:55 smithi078 bash[78274]: cluster 2023-10-01T11:05:54.899094+0000 osd.6 (osd.6) 309 : cluster 3 failed to encode map e329 with expected crc 2023-10-01T11:05:56.089 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:56.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:56.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:05:56.193 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:56.210 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:55 smithi173 bash[58816]: cluster 2023-10-01T11:05:54.291824+0000 mgr.y (mgr.25072) 1457 : cluster 0 pgmap v1548: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 128 op/s; 3869/213201 objects degraded (1.815%); 4531/213201 objects misplaced (2.125%); 8.5 MiB/s, 139 objects/s recovering 2023-10-01T11:05:56.210 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:55 smithi173 bash[58816]: cluster 2023-10-01T11:05:54.894871+0000 mon.a (mon.0) 1867 : cluster 0 osdmap e329: 8 total, 8 up, 7 in 2023-10-01T11:05:56.210 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:55 smithi173 bash[58816]: cluster 2023-10-01T11:05:54.896777+0000 osd.6 (osd.6) 308 : cluster 3 failed to encode map e329 with expected crc 2023-10-01T11:05:56.210 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:55 smithi173 bash[58816]: cluster 2023-10-01T11:05:54.899094+0000 osd.6 (osd.6) 309 : cluster 3 failed to encode map e329 with expected crc 2023-10-01T11:05:56.230 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:56.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:56.233 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:05:56.296 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-01T11:05:56.335 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:56.369 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:56.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:56.372 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:05:56.472 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:56.502 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:56.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:56.506 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:05:56.610 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:56.637 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:56.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:56.640 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:56.741 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:56.772 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:56.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:56.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:05:56.875 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:56.902 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:56.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:56.905 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:05:57.006 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:57.029 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:57.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:57.031 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:57.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:56 smithi078 bash[75299]: cluster 2023-10-01T11:05:55.901037+0000 osd.7 (osd.7) 356 : cluster 3 failed to encode map e330 with expected crc 2023-10-01T11:05:57.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:56 smithi078 bash[75299]: cluster 2023-10-01T11:05:55.902726+0000 mon.a (mon.0) 1868 : cluster 0 osdmap e330: 8 total, 8 up, 7 in 2023-10-01T11:05:57.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:56 smithi078 bash[75299]: cluster 2023-10-01T11:05:55.903634+0000 osd.6 (osd.6) 310 : cluster 3 failed to encode map e330 with expected crc 2023-10-01T11:05:57.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:56 smithi078 bash[75299]: cluster 2023-10-01T11:05:55.907393+0000 osd.6 (osd.6) 311 : cluster 3 failed to encode map e330 with expected crc 2023-10-01T11:05:57.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:56 smithi078 bash[78274]: cluster 2023-10-01T11:05:55.901037+0000 osd.7 (osd.7) 356 : cluster 3 failed to encode map e330 with expected crc 2023-10-01T11:05:57.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:56 smithi078 bash[78274]: cluster 2023-10-01T11:05:55.902726+0000 mon.a (mon.0) 1868 : cluster 0 osdmap e330: 8 total, 8 up, 7 in 2023-10-01T11:05:57.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:56 smithi078 bash[78274]: cluster 2023-10-01T11:05:55.903634+0000 osd.6 (osd.6) 310 : cluster 3 failed to encode map e330 with expected crc 2023-10-01T11:05:57.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:56 smithi078 bash[78274]: cluster 2023-10-01T11:05:55.907393+0000 osd.6 (osd.6) 311 : cluster 3 failed to encode map e330 with expected crc 2023-10-01T11:05:57.132 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:57.160 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:57.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:57.163 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:05:57.263 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:57.274 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:56 smithi173 bash[58816]: cluster 2023-10-01T11:05:55.901037+0000 osd.7 (osd.7) 356 : cluster 3 failed to encode map e330 with expected crc 2023-10-01T11:05:57.274 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:56 smithi173 bash[58816]: cluster 2023-10-01T11:05:55.902726+0000 mon.a (mon.0) 1868 : cluster 0 osdmap e330: 8 total, 8 up, 7 in 2023-10-01T11:05:57.274 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:56 smithi173 bash[58816]: cluster 2023-10-01T11:05:55.903634+0000 osd.6 (osd.6) 310 : cluster 3 failed to encode map e330 with expected crc 2023-10-01T11:05:57.274 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:56 smithi173 bash[58816]: cluster 2023-10-01T11:05:55.907393+0000 osd.6 (osd.6) 311 : cluster 3 failed to encode map e330 with expected crc 2023-10-01T11:05:57.291 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:57.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:57.293 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:57.394 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:57.424 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:57.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:57.427 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:05:57.528 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:57.557 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:05:57.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:57.559 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:05:57.660 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:57.689 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:57.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:57.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:05:57.792 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:57.820 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:57.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:57.823 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:05:57.924 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:57.954 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:57.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:57.958 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:58.058 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:58.069 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:57 smithi173 bash[58816]: cluster 2023-10-01T11:05:56.292596+0000 mgr.y (mgr.25072) 1458 : cluster 0 pgmap v1551: 89 pgs: 2 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 124 op/s; 3010/212538 objects degraded (1.416%); 4904/212538 objects misplaced (2.307%); 8.4 MiB/s, 138 objects/s recovering 2023-10-01T11:05:58.069 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:57 smithi173 bash[58816]: cluster 2023-10-01T11:05:57.738903+0000 mon.a (mon.0) 1869 : cluster 3 Health check update: Degraded data redundancy: 3010/212538 objects degraded (1.416%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T11:05:58.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:57 smithi078 bash[78274]: cluster 2023-10-01T11:05:56.292596+0000 mgr.y (mgr.25072) 1458 : cluster 0 pgmap v1551: 89 pgs: 2 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 124 op/s; 3010/212538 objects degraded (1.416%); 4904/212538 objects misplaced (2.307%); 8.4 MiB/s, 138 objects/s recovering 2023-10-01T11:05:58.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:57 smithi078 bash[78274]: cluster 2023-10-01T11:05:57.738903+0000 mon.a (mon.0) 1869 : cluster 3 Health check update: Degraded data redundancy: 3010/212538 objects degraded (1.416%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T11:05:58.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:57 smithi078 bash[75299]: cluster 2023-10-01T11:05:56.292596+0000 mgr.y (mgr.25072) 1458 : cluster 0 pgmap v1551: 89 pgs: 2 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 124 op/s; 3010/212538 objects degraded (1.416%); 4904/212538 objects misplaced (2.307%); 8.4 MiB/s, 138 objects/s recovering 2023-10-01T11:05:58.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:57 smithi078 bash[75299]: cluster 2023-10-01T11:05:57.738903+0000 mon.a (mon.0) 1869 : cluster 3 Health check update: Degraded data redundancy: 3010/212538 objects degraded (1.416%), 3 pgs degraded (PG_DEGRADED) 2023-10-01T11:05:58.087 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:58.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:58.091 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:58.191 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:58.220 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:58.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:58.222 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:05:58.323 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:58.353 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:58.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:58.355 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:05:58.456 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:58.482 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:58.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:58.484 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:05:58.585 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:58.611 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:58.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:58.613 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:05:58.713 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:58.736 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:58.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:58.739 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:05:58.840 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:58.867 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:05:58.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:58.868 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:05:58.969 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:58.997 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:05:59.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:59.000 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:05:59.101 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:59.133 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:05:59.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:59.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:05:59.237 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:59.248 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:05:59.260 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:59.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:59.263 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:05:59.364 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:59.396 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:59.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:59.398 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:05:59.499 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:59.529 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:05:59.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:59.531 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:05:59.632 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:59.662 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:59.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:59.666 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:05:59.767 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:59.787 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:05:59.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:59.789 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:05:59.890 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:59.918 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:05:59.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:05:59.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:00.021 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:00.047 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:00.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:00.049 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:00.150 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:00.162 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:05:59 smithi173 bash[58816]: cluster 2023-10-01T11:05:58.293458+0000 mgr.y (mgr.25072) 1459 : cluster 0 pgmap v1552: 89 pgs: 1 active+remapped+backfill_wait, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 121 op/s; 3010/211926 objects degraded (1.420%); 4846/211926 objects misplaced (2.287%); 6.4 MiB/s, 104 objects/s recovering 2023-10-01T11:06:00.181 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:00.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:00.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:06:00.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:05:59 smithi078 bash[78274]: cluster 2023-10-01T11:05:58.293458+0000 mgr.y (mgr.25072) 1459 : cluster 0 pgmap v1552: 89 pgs: 1 active+remapped+backfill_wait, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 121 op/s; 3010/211926 objects degraded (1.420%); 4846/211926 objects misplaced (2.287%); 6.4 MiB/s, 104 objects/s recovering 2023-10-01T11:06:00.270 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:05:59 smithi078 bash[75299]: cluster 2023-10-01T11:05:58.293458+0000 mgr.y (mgr.25072) 1459 : cluster 0 pgmap v1552: 89 pgs: 1 active+remapped+backfill_wait, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 121 op/s; 3010/211926 objects degraded (1.420%); 4846/211926 objects misplaced (2.287%); 6.4 MiB/s, 104 objects/s recovering 2023-10-01T11:06:00.284 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:00.308 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:00.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:00.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:06:00.411 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:00.435 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:00.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:00.437 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:06:00.538 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:00.564 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:00.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:00.568 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:06:00.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:06:00 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:06:00] "GET /metrics HTTP/1.1" 200 36405 "" "Prometheus/2.33.4" 2023-10-01T11:06:00.668 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:00.693 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:00.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:00.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:06:00.796 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:00.823 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:00.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:00.826 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:06:00.927 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:00.952 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:00.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:00.955 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:06:01.056 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:01.077 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T11:06:01.086 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:01.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:01.089 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:06:01.189 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:01.216 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:01.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:01.219 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:06:01.274 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:06:01.320 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:01.349 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:01.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:01.351 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:06:01.452 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:01.475 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:01.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:01.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:01.579 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:01.599 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:01.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:01.602 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:01.703 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:01.729 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:01.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:01.732 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:06:01.807 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T11:06:01.827 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T11:06:01.832 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:01.851 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T11:06:01.856 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:01.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:01.858 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:06:01.872 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T11:06:01.894 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T11:06:01.915 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T11:06:01.935 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T11:06:01.954 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T11:06:01.956 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T11:06:01.956 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T11:06:01.956 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T11:06:01.956 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T11:06:01.956 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T11:06:01.956 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T11:06:01.956 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T11:06:01.956 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T11:06:01.959 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:01.967 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:01 smithi173 bash[58816]: cluster 2023-10-01T11:06:00.294314+0000 mgr.y (mgr.25072) 1460 : cluster 0 pgmap v1553: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 182 op/s; 3008/210249 objects degraded (1.431%); 4846/210249 objects misplaced (2.305%); 6.0 MiB/s, 149 objects/s recovering 2023-10-01T11:06:01.967 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:01 smithi173 bash[58816]: audit 2023-10-01T11:06:01.781247+0000 mon.c (mon.2) 220 : audit 0 from='client.? 172.21.15.78:0/807962323' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:06:01.981 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:01.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:01.983 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:02.080 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:01 smithi078 bash[75299]: cluster 2023-10-01T11:06:00.294314+0000 mgr.y (mgr.25072) 1460 : cluster 0 pgmap v1553: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 182 op/s; 3008/210249 objects degraded (1.431%); 4846/210249 objects misplaced (2.305%); 6.0 MiB/s, 149 objects/s recovering 2023-10-01T11:06:02.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:01 smithi078 bash[75299]: audit 2023-10-01T11:06:01.781247+0000 mon.c (mon.2) 220 : audit 0 from='client.? 172.21.15.78:0/807962323' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:06:02.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:01 smithi078 bash[78274]: cluster 2023-10-01T11:06:00.294314+0000 mgr.y (mgr.25072) 1460 : cluster 0 pgmap v1553: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 182 op/s; 3008/210249 objects degraded (1.431%); 4846/210249 objects misplaced (2.305%); 6.0 MiB/s, 149 objects/s recovering 2023-10-01T11:06:02.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:01 smithi078 bash[78274]: audit 2023-10-01T11:06:01.781247+0000 mon.c (mon.2) 220 : audit 0 from='client.? 172.21.15.78:0/807962323' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:06:02.084 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:02.106 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:02.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:02.108 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:06:02.208 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:02.231 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:02.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:02.234 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:06:02.335 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:02.361 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:02.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:02.364 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:06:02.464 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:02.485 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:02.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:02.487 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:06:02.588 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:02.616 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:02.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:02.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:06:02.720 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:02.741 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:02.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:02.744 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:06:02.844 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:02.866 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:02.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:02.868 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:06:02.969 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:02.994 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:02.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:02.996 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:06:03.097 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:03.121 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:03.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:03.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:06:03.224 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:03.247 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:03.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:03.250 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:06:03.351 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:03.381 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:03.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:03.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:06:03.485 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:03.507 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:03.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:03.509 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:03.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:03 smithi078 bash[78274]: cluster 2023-10-01T11:06:02.294800+0000 mgr.y (mgr.25072) 1461 : cluster 0 pgmap v1554: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 129 op/s; 3008/210093 objects degraded (1.432%); 4846/210093 objects misplaced (2.307%); 920 KiB/s, 65 objects/s recovering 2023-10-01T11:06:03.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:03 smithi078 bash[78274]: audit 2023-10-01T11:06:02.463777+0000 mon.a (mon.0) 1870 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:06:03.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:03 smithi078 bash[78274]: audit 2023-10-01T11:06:02.464819+0000 mon.a (mon.0) 1871 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:06:03.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:03 smithi078 bash[78274]: cluster 2023-10-01T11:06:02.740041+0000 mon.a (mon.0) 1872 : cluster 3 Health check update: Degraded data redundancy: 3008/210249 objects degraded (1.431%), 2 pgs degraded (PG_DEGRADED) 2023-10-01T11:06:03.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:03 smithi078 bash[75299]: cluster 2023-10-01T11:06:02.294800+0000 mgr.y (mgr.25072) 1461 : cluster 0 pgmap v1554: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 129 op/s; 3008/210093 objects degraded (1.432%); 4846/210093 objects misplaced (2.307%); 920 KiB/s, 65 objects/s recovering 2023-10-01T11:06:03.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:03 smithi078 bash[75299]: audit 2023-10-01T11:06:02.463777+0000 mon.a (mon.0) 1870 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:06:03.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:03 smithi078 bash[75299]: audit 2023-10-01T11:06:02.464819+0000 mon.a (mon.0) 1871 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:06:03.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:03 smithi078 bash[75299]: cluster 2023-10-01T11:06:02.740041+0000 mon.a (mon.0) 1872 : cluster 3 Health check update: Degraded data redundancy: 3008/210249 objects degraded (1.431%), 2 pgs degraded (PG_DEGRADED) 2023-10-01T11:06:03.610 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:03.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:03 smithi173 bash[58816]: cluster 2023-10-01T11:06:02.294800+0000 mgr.y (mgr.25072) 1461 : cluster 0 pgmap v1554: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 129 op/s; 3008/210093 objects degraded (1.432%); 4846/210093 objects misplaced (2.307%); 920 KiB/s, 65 objects/s recovering 2023-10-01T11:06:03.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:03 smithi173 bash[58816]: audit 2023-10-01T11:06:02.463777+0000 mon.a (mon.0) 1870 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:06:03.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:03 smithi173 bash[58816]: audit 2023-10-01T11:06:02.464819+0000 mon.a (mon.0) 1871 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:06:03.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:03 smithi173 bash[58816]: cluster 2023-10-01T11:06:02.740041+0000 mon.a (mon.0) 1872 : cluster 3 Health check update: Degraded data redundancy: 3008/210249 objects degraded (1.431%), 2 pgs degraded (PG_DEGRADED) 2023-10-01T11:06:03.639 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:03.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:03.641 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:06:03.742 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:03.773 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:03.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:03.776 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:06:03.876 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:03.896 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:03.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:03.898 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:06:03.998 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:04.022 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:04.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:04.024 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:06:04.125 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:04.153 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:04.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:04.155 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:06:04.255 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:04.276 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:04.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:04.278 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:06:04.379 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:04.398 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:04.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:04.400 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:06:04.500 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:04.518 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:04.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:04.520 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:06:04.621 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:04.645 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:04.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:04.648 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:06:04.748 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:04.772 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:04.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:04.775 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:06:04.875 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:04.904 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:04.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:04.909 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:05.009 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:05.029 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:05.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:05.033 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:06:05.110 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-10-01T11:06:05.110 INFO:teuthology.orchestra.run.smithi173.stdout: "size": 20, 2023-10-01T11:06:05.110 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 600, 2023-10-01T11:06:05.110 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [ 2023-10-01T11:06:05.110 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.111 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46232.0:30715 12.f 12:f0019da4:::benchmark_data_smithi078_180197_object30714:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e244)", 2023-10-01T11:06:05.111 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T10:57:02.246723+0000", 2023-10-01T11:06:05.111 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 542.85853391299997, 2023-10-01T11:06:05.111 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.26381739799999998, 2023-10-01T11:06:05.111 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:06:05.111 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:06:05.111 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:06:05.111 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46232", 2023-10-01T11:06:05.111 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/711982100", 2023-10-01T11:06:05.111 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 30715 2023-10-01T11:06:05.111 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.112 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:06:05.112 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.112 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:06:05.112 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:57:02.246723+0000", 2023-10-01T11:06:05.112 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:06:05.112 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.112 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.112 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:06:05.112 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:57:02.246723+0000", 2023-10-01T11:06:05.112 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999976 2023-10-01T11:06:05.112 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.112 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.112 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:06:05.113 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:57:02.246721+0000", 2023-10-01T11:06:05.113 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.4435999999999999e-05 2023-10-01T11:06:05.113 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.113 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.113 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:06:05.113 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:57:02.246776+0000", 2023-10-01T11:06:05.113 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4181e-05 2023-10-01T11:06:05.113 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.113 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.113 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:06:05.113 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:57:02.246790+0000", 2023-10-01T11:06:05.113 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.1810000000000001e-06 2023-10-01T11:06:05.114 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.114 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.114 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:06:05.114 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:57:02.246798+0000", 2023-10-01T11:06:05.114 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.5488000000000005e-05 2023-10-01T11:06:05.114 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.114 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.114 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:05.114 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:57:02.246884+0000", 2023-10-01T11:06:05.114 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00022132599999999999 2023-10-01T11:06:05.114 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.114 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.115 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:06:05.115 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:57:02.247105+0000", 2023-10-01T11:06:05.115 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00026952200000000001 2023-10-01T11:06:05.115 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.115 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.115 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 2,5", 2023-10-01T11:06:05.115 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:57:02.247374+0000", 2023-10-01T11:06:05.115 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0015796320000000001 2023-10-01T11:06:05.115 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.115 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.116 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:06:05.116 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:57:02.248954+0000", 2023-10-01T11:06:05.116 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00059958099999999996 2023-10-01T11:06:05.116 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.116 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.116 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:06:05.117 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:57:02.249554+0000", 2023-10-01T11:06:05.117 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.26078844800000001 2023-10-01T11:06:05.117 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.117 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.117 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:06:05.117 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:57:02.510342+0000", 2023-10-01T11:06:05.117 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.3645000000000002e-05 2023-10-01T11:06:05.117 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.118 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.118 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:06:05.118 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:57:02.510436+0000", 2023-10-01T11:06:05.118 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000105115 2023-10-01T11:06:05.118 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.118 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.118 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:06:05.118 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T10:57:02.510541+0000", 2023-10-01T11:06:05.118 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.26381739799999998 2023-10-01T11:06:05.119 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:05.119 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:06:05.119 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:05.119 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.119 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.119 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.46232.0:191209 12.f e257/256 12:fc4afb62:::benchmark_data_smithi078_180197_object88078:head v 257'12177, mlcod=0'0)", 2023-10-01T11:06:05.119 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:00:31.040307+0000", 2023-10-01T11:06:05.119 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 334.06495013799997, 2023-10-01T11:06:05.120 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.27612240100000002, 2023-10-01T11:06:05.120 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:06:05.120 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:06:05.120 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:06:05.120 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.120 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:06:05.120 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.040307+0000", 2023-10-01T11:06:05.120 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:06:05.120 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.120 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.120 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:06:05.120 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.040307+0000", 2023-10-01T11:06:05.121 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999971 2023-10-01T11:06:05.121 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.121 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.121 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:06:05.121 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.040305+0000", 2023-10-01T11:06:05.121 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.7931000000000002e-05 2023-10-01T11:06:05.121 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.121 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.121 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:06:05.121 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.040323+0000", 2023-10-01T11:06:05.121 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.8070000000000001e-05 2023-10-01T11:06:05.121 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.122 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.122 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:06:05.122 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.040351+0000", 2023-10-01T11:06:05.122 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0861e-05 2023-10-01T11:06:05.122 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.122 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.122 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:06:05.122 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.040361+0000", 2023-10-01T11:06:05.122 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000100654 2023-10-01T11:06:05.122 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.122 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.123 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:05.123 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.040462+0000", 2023-10-01T11:06:05.123 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.7046999999999997e-05 2023-10-01T11:06:05.123 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.123 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.123 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:06:05.123 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.040529+0000", 2023-10-01T11:06:05.123 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.275856559 2023-10-01T11:06:05.123 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.123 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.123 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:06:05.123 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.316386+0000", 2023-10-01T11:06:05.123 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.3939000000000001e-05 2023-10-01T11:06:05.124 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.124 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.124 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:06:05.124 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.316430+0000", 2023-10-01T11:06:05.124 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.27612240100000002 2023-10-01T11:06:05.124 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:05.124 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:06:05.124 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:05.124 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.124 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.124 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.46232.0:191219 12.4 e257/256 12:290006ad:::benchmark_data_smithi078_180197_object88088:head v 257'11967, mlcod=0'0)", 2023-10-01T11:06:05.125 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:00:31.041504+0000", 2023-10-01T11:06:05.125 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 334.06375341500001, 2023-10-01T11:06:05.125 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.24961704500000001, 2023-10-01T11:06:05.125 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:06:05.125 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:06:05.125 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:06:05.125 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.125 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:06:05.125 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.041504+0000", 2023-10-01T11:06:05.125 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:06:05.125 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.125 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.126 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:06:05.126 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.041504+0000", 2023-10-01T11:06:05.126 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-10-01T11:06:05.126 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.126 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.126 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:06:05.126 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.041503+0000", 2023-10-01T11:06:05.126 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.6356999999999999e-05 2023-10-01T11:06:05.126 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.126 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.126 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:06:05.126 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.041519+0000", 2023-10-01T11:06:05.127 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.7200999999999999e-05 2023-10-01T11:06:05.127 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.127 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.127 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:06:05.127 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.041546+0000", 2023-10-01T11:06:05.127 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.2290000000000003e-06 2023-10-01T11:06:05.127 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.127 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.127 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:06:05.127 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.041554+0000", 2023-10-01T11:06:05.127 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00010103300000000001 2023-10-01T11:06:05.128 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.128 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.128 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:05.128 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.041655+0000", 2023-10-01T11:06:05.128 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.7381e-05 2023-10-01T11:06:05.128 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.128 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.128 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:06:05.128 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.041693+0000", 2023-10-01T11:06:05.128 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.24930122099999999 2023-10-01T11:06:05.128 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.129 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.129 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:06:05.129 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.290994+0000", 2023-10-01T11:06:05.129 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00012693900000000001 2023-10-01T11:06:05.129 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.129 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.129 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:06:05.129 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.291121+0000", 2023-10-01T11:06:05.129 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.24961704500000001 2023-10-01T11:06:05.129 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:05.129 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:06:05.129 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:05.130 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.130 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.130 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.46232.0:191218 12.f e257/256 12:fce854b3:::benchmark_data_smithi078_180197_object88087:head v 257'12178, mlcod=0'0)", 2023-10-01T11:06:05.130 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:00:31.049630+0000", 2023-10-01T11:06:05.130 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 334.05562698300002, 2023-10-01T11:06:05.130 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.27357339800000002, 2023-10-01T11:06:05.130 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:06:05.130 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:06:05.130 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:06:05.130 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.131 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:06:05.131 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.049630+0000", 2023-10-01T11:06:05.131 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:06:05.131 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.131 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.131 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:06:05.131 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.049630+0000", 2023-10-01T11:06:05.131 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-10-01T11:06:05.131 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.131 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.131 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:06:05.132 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.049629+0000", 2023-10-01T11:06:05.132 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4204e-05 2023-10-01T11:06:05.132 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.132 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.132 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:06:05.132 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.049643+0000", 2023-10-01T11:06:05.132 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.6719999999999999e-05 2023-10-01T11:06:05.132 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.132 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.132 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:06:05.132 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.049659+0000", 2023-10-01T11:06:05.132 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.5240000000000006e-06 2023-10-01T11:06:05.133 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.133 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.133 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:06:05.133 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.049669+0000", 2023-10-01T11:06:05.133 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.146045965 2023-10-01T11:06:05.133 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.133 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.133 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:05.134 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.195715+0000", 2023-10-01T11:06:05.134 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.3194000000000001e-05 2023-10-01T11:06:05.134 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.134 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.134 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:06:05.134 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.195748+0000", 2023-10-01T11:06:05.134 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.12562382899999999 2023-10-01T11:06:05.135 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.135 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.135 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:06:05.135 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.321372+0000", 2023-10-01T11:06:05.135 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.001831802 2023-10-01T11:06:05.135 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.135 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.135 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:06:05.135 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.323204+0000", 2023-10-01T11:06:05.135 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.27357339800000002 2023-10-01T11:06:05.135 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:05.135 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:06:05.136 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:05.136 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.136 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.136 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.46232.0:191220 12.f e257/256 12:f0dec486:::benchmark_data_smithi078_180197_object88089:head v 257'12179, mlcod=0'0)", 2023-10-01T11:06:05.136 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:00:31.051265+0000", 2023-10-01T11:06:05.136 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 334.05399208199998, 2023-10-01T11:06:05.136 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.271979046, 2023-10-01T11:06:05.136 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:06:05.136 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:06:05.136 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:06:05.136 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.136 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:06:05.137 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.051265+0000", 2023-10-01T11:06:05.137 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:06:05.137 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.137 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.137 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:06:05.137 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.051265+0000", 2023-10-01T11:06:05.137 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-10-01T11:06:05.137 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.137 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.137 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:06:05.137 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.051264+0000", 2023-10-01T11:06:05.137 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.901e-06 2023-10-01T11:06:05.137 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.138 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.138 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:06:05.138 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.051273+0000", 2023-10-01T11:06:05.138 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1914e-05 2023-10-01T11:06:05.138 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.138 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.138 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:06:05.138 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.051285+0000", 2023-10-01T11:06:05.138 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.5069999999999997e-06 2023-10-01T11:06:05.138 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.138 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.139 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:06:05.139 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.051292+0000", 2023-10-01T11:06:05.139 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.23522458700000001 2023-10-01T11:06:05.139 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.139 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.139 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:05.139 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.286516+0000", 2023-10-01T11:06:05.139 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.8777999999999999e-05 2023-10-01T11:06:05.139 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.139 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.139 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:06:05.139 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.286575+0000", 2023-10-01T11:06:05.140 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.036633970000000002 2023-10-01T11:06:05.140 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.140 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.140 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:06:05.140 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.323209+0000", 2023-10-01T11:06:05.140 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.5466000000000002e-05 2023-10-01T11:06:05.140 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.140 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.140 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:06:05.140 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.323244+0000", 2023-10-01T11:06:05.140 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.271979046 2023-10-01T11:06:05.140 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:05.141 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:06:05.141 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:05.141 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.141 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.141 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.46232.0:191222 12.f e257/256 12:fb2ef8e6:::benchmark_data_smithi078_180197_object88091:head v 257'12180, mlcod=0'0)", 2023-10-01T11:06:05.141 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:00:31.057064+0000", 2023-10-01T11:06:05.141 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 334.04819322100002, 2023-10-01T11:06:05.141 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.26619957500000002, 2023-10-01T11:06:05.141 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:06:05.141 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:06:05.141 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:06:05.142 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.142 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:06:05.142 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.057064+0000", 2023-10-01T11:06:05.142 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:06:05.142 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.143 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.143 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:06:05.143 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.057064+0000", 2023-10-01T11:06:05.143 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999995 2023-10-01T11:06:05.143 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.143 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.143 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:06:05.144 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.057063+0000", 2023-10-01T11:06:05.144 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.1710000000000008e-06 2023-10-01T11:06:05.144 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.144 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.144 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:06:05.144 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.057073+0000", 2023-10-01T11:06:05.144 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.376e-05 2023-10-01T11:06:05.144 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.144 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.144 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:06:05.144 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.057086+0000", 2023-10-01T11:06:05.145 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.95e-06 2023-10-01T11:06:05.145 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.145 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.145 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:06:05.145 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.057091+0000", 2023-10-01T11:06:05.145 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.235104743 2023-10-01T11:06:05.145 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.145 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.145 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:05.145 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.292196+0000", 2023-10-01T11:06:05.145 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.6214000000000003e-05 2023-10-01T11:06:05.145 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.146 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.146 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:06:05.146 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.292232+0000", 2023-10-01T11:06:05.146 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.031014943999999999 2023-10-01T11:06:05.146 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.146 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.146 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:06:05.146 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.323247+0000", 2023-10-01T11:06:05.146 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.6456e-05 2023-10-01T11:06:05.146 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.146 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.147 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:06:05.147 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.323264+0000", 2023-10-01T11:06:05.147 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.26619957500000002 2023-10-01T11:06:05.147 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:05.147 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:06:05.147 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:05.147 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.147 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.147 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.46232.0:191223 12.f e257/256 12:f071dbd8:::benchmark_data_smithi078_180197_object88092:head v 257'12181, mlcod=0'0)", 2023-10-01T11:06:05.147 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:00:31.062547+0000", 2023-10-01T11:06:05.147 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 334.042710038, 2023-10-01T11:06:05.148 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.26080558999999998, 2023-10-01T11:06:05.148 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:06:05.148 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:06:05.148 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:06:05.148 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.148 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:06:05.148 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.062547+0000", 2023-10-01T11:06:05.148 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:06:05.148 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.149 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.149 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:06:05.149 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.062547+0000", 2023-10-01T11:06:05.149 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-10-01T11:06:05.149 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.149 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.149 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:06:05.150 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.062546+0000", 2023-10-01T11:06:05.150 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0915e-05 2023-10-01T11:06:05.150 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.150 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.150 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:06:05.150 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.062557+0000", 2023-10-01T11:06:05.150 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.7189000000000001e-05 2023-10-01T11:06:05.150 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.150 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.150 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:06:05.150 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.062574+0000", 2023-10-01T11:06:05.151 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.6249999999999996e-06 2023-10-01T11:06:05.151 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.151 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.151 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:06:05.151 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.062583+0000", 2023-10-01T11:06:05.151 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.23526127999999999 2023-10-01T11:06:05.151 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.151 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.151 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:05.151 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.297844+0000", 2023-10-01T11:06:05.152 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.7100000000000001e-05 2023-10-01T11:06:05.152 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.152 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.152 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:06:05.152 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.297872+0000", 2023-10-01T11:06:05.152 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.025394713999999999 2023-10-01T11:06:05.152 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.152 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.152 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:06:05.152 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.323266+0000", 2023-10-01T11:06:05.152 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.6710000000000002e-05 2023-10-01T11:06:05.153 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.153 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.153 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:06:05.153 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.323353+0000", 2023-10-01T11:06:05.153 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.26080558999999998 2023-10-01T11:06:05.153 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:05.153 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:06:05.153 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:05.153 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.154 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.154 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.46232.0:191230 12.f e257/256 12:f12f6696:::benchmark_data_smithi078_180197_object88099:head v 257'12182, mlcod=0'0)", 2023-10-01T11:06:05.154 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:00:31.069045+0000", 2023-10-01T11:06:05.154 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 334.03621210300003, 2023-10-01T11:06:05.154 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.25436570200000003, 2023-10-01T11:06:05.154 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:06:05.154 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:06:05.154 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:06:05.154 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.154 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:06:05.155 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.069045+0000", 2023-10-01T11:06:05.155 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:06:05.155 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.155 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.155 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:06:05.155 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.069045+0000", 2023-10-01T11:06:05.155 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-10-01T11:06:05.155 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.155 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.155 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:06:05.155 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.069044+0000", 2023-10-01T11:06:05.156 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2696e-05 2023-10-01T11:06:05.156 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.156 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.156 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:06:05.156 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.069057+0000", 2023-10-01T11:06:05.156 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.6200000000000001e-05 2023-10-01T11:06:05.156 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.156 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.156 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:06:05.156 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.069073+0000", 2023-10-01T11:06:05.157 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.3229e-05 2023-10-01T11:06:05.157 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.157 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.157 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:06:05.157 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.069086+0000", 2023-10-01T11:06:05.157 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.23287075700000001 2023-10-01T11:06:05.157 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.157 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.157 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:05.157 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.301957+0000", 2023-10-01T11:06:05.158 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.1662000000000002e-05 2023-10-01T11:06:05.158 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.158 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.158 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:06:05.158 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.301998+0000", 2023-10-01T11:06:05.158 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.021358115 2023-10-01T11:06:05.158 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.158 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.158 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:06:05.158 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.323357+0000", 2023-10-01T11:06:05.159 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.4462999999999997e-05 2023-10-01T11:06:05.159 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.159 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.159 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:06:05.159 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.323411+0000", 2023-10-01T11:06:05.159 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.25436570200000003 2023-10-01T11:06:05.159 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:05.159 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:06:05.159 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:05.159 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.160 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.160 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.46232.0:191235 12.f e257/256 12:f74a4006:::benchmark_data_smithi078_180197_object88104:head v 257'12183, mlcod=0'0)", 2023-10-01T11:06:05.160 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:00:31.073747+0000", 2023-10-01T11:06:05.160 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 334.03151056299998, 2023-10-01T11:06:05.160 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.24988876700000001, 2023-10-01T11:06:05.160 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:06:05.160 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:06:05.160 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:06:05.160 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.160 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:06:05.160 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.073747+0000", 2023-10-01T11:06:05.161 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:06:05.161 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.161 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.161 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:06:05.161 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.073747+0000", 2023-10-01T11:06:05.161 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-10-01T11:06:05.161 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.161 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.161 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:06:05.161 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.073746+0000", 2023-10-01T11:06:05.161 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2278e-05 2023-10-01T11:06:05.161 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.162 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.162 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:06:05.162 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.073758+0000", 2023-10-01T11:06:05.162 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.9879999999999999e-05 2023-10-01T11:06:05.162 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.162 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.162 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:06:05.162 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.073778+0000", 2023-10-01T11:06:05.162 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.5609999999999999e-06 2023-10-01T11:06:05.162 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.162 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.163 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:06:05.163 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.073786+0000", 2023-10-01T11:06:05.163 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.23305060599999999 2023-10-01T11:06:05.163 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.163 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.163 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:05.163 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.306837+0000", 2023-10-01T11:06:05.163 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.0885999999999999e-05 2023-10-01T11:06:05.163 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.163 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.163 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:06:05.163 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.306908+0000", 2023-10-01T11:06:05.164 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.01666538 2023-10-01T11:06:05.164 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.164 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.164 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:06:05.164 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.323573+0000", 2023-10-01T11:06:05.164 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.2293999999999997e-05 2023-10-01T11:06:05.164 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.164 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.164 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:06:05.164 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:00:31.323636+0000", 2023-10-01T11:06:05.164 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.24988876700000001 2023-10-01T11:06:05.164 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:05.165 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:06:05.165 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:05.165 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.165 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.165 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:68905 13.4 13:28a59762:::benchmark_data_smithi078_202524_object68904:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e298)", 2023-10-01T11:06:05.165 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:03:00.413094+0000", 2023-10-01T11:06:05.165 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 184.69216314100001, 2023-10-01T11:06:05.165 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.21380880599999999, 2023-10-01T11:06:05.165 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:06:05.165 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:06:05.165 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:06:05.166 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:06:05.166 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:06:05.166 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 68905 2023-10-01T11:06:05.166 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.166 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:06:05.166 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.166 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:06:05.166 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:00.413094+0000", 2023-10-01T11:06:05.166 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:06:05.166 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.167 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.167 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:06:05.167 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:00.413094+0000", 2023-10-01T11:06:05.167 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-10-01T11:06:05.167 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.167 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.167 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:06:05.167 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:00.413092+0000", 2023-10-01T11:06:05.167 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.7972999999999997e-05 2023-10-01T11:06:05.167 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.167 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.168 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:06:05.168 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:00.413130+0000", 2023-10-01T11:06:05.168 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0784e-05 2023-10-01T11:06:05.168 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.168 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.168 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:06:05.168 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:00.413141+0000", 2023-10-01T11:06:05.168 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.6849999999999998e-06 2023-10-01T11:06:05.168 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.169 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.169 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:06:05.169 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:00.413147+0000", 2023-10-01T11:06:05.169 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.4619e-05 2023-10-01T11:06:05.169 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.169 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.169 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:05.169 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:00.413202+0000", 2023-10-01T11:06:05.169 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000158214 2023-10-01T11:06:05.169 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.170 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.170 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:06:05.170 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:00.413360+0000", 2023-10-01T11:06:05.170 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00017850099999999999 2023-10-01T11:06:05.170 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.170 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.170 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1,2", 2023-10-01T11:06:05.170 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:00.413538+0000", 2023-10-01T11:06:05.170 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.002794174 2023-10-01T11:06:05.170 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.170 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.171 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:06:05.171 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:00.416332+0000", 2023-10-01T11:06:05.171 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0048371040000000001 2023-10-01T11:06:05.171 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.171 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.171 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:06:05.171 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:00.421169+0000", 2023-10-01T11:06:05.171 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.20557399000000001 2023-10-01T11:06:05.171 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.171 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.171 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:06:05.171 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:00.626743+0000", 2023-10-01T11:06:05.172 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.7689999999999995e-05 2023-10-01T11:06:05.172 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.172 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.172 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:06:05.172 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:00.626841+0000", 2023-10-01T11:06:05.172 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.1860999999999996e-05 2023-10-01T11:06:05.172 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.172 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.172 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:06:05.172 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:00.626903+0000", 2023-10-01T11:06:05.172 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.21380880599999999 2023-10-01T11:06:05.173 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:05.173 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:06:05.173 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:05.173 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.173 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.173 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:68908 13.1 13:8117ecc6:::benchmark_data_smithi078_202524_object68907:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e298)", 2023-10-01T11:06:05.173 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:03:00.414459+0000", 2023-10-01T11:06:05.173 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 184.69079873499999, 2023-10-01T11:06:05.173 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.21906904799999999, 2023-10-01T11:06:05.173 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:06:05.173 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:06:05.174 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:06:05.174 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:06:05.174 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:06:05.174 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 68908 2023-10-01T11:06:05.174 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.174 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:06:05.174 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.174 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:06:05.174 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:00.414459+0000", 2023-10-01T11:06:05.174 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:06:05.174 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.175 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.175 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:06:05.175 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:00.414459+0000", 2023-10-01T11:06:05.175 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-10-01T11:06:05.175 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.175 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.175 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:06:05.175 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:00.414457+0000", 2023-10-01T11:06:05.175 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.4363999999999997e-05 2023-10-01T11:06:05.175 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.175 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.175 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:06:05.176 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:00.414501+0000", 2023-10-01T11:06:05.176 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2779000000000001e-05 2023-10-01T11:06:05.176 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.176 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.176 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:06:05.176 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:00.414514+0000", 2023-10-01T11:06:05.176 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.3760000000000002e-06 2023-10-01T11:06:05.176 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.176 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.176 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:06:05.176 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:00.414522+0000", 2023-10-01T11:06:05.176 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00042431800000000003 2023-10-01T11:06:05.177 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.177 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.177 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:05.177 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:00.414946+0000", 2023-10-01T11:06:05.177 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00012916199999999999 2023-10-01T11:06:05.177 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.177 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.177 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:06:05.177 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:00.415075+0000", 2023-10-01T11:06:05.177 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00017811899999999999 2023-10-01T11:06:05.177 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.178 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.178 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 2,4", 2023-10-01T11:06:05.178 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:00.415253+0000", 2023-10-01T11:06:05.178 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0037784730000000001 2023-10-01T11:06:05.178 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.178 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.178 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:06:05.178 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:00.419032+0000", 2023-10-01T11:06:05.178 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0063494850000000002 2023-10-01T11:06:05.178 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.178 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.178 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:06:05.179 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:00.425381+0000", 2023-10-01T11:06:05.179 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.207988601 2023-10-01T11:06:05.179 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.179 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.179 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:06:05.179 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:00.633370+0000", 2023-10-01T11:06:05.179 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000111286 2023-10-01T11:06:05.179 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.179 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.179 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:06:05.179 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:00.633481+0000", 2023-10-01T11:06:05.180 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.6613999999999998e-05 2023-10-01T11:06:05.180 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.180 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.180 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:06:05.180 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:00.633528+0000", 2023-10-01T11:06:05.180 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.21906904799999999 2023-10-01T11:06:05.180 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:05.180 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:06:05.180 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:05.180 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.180 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.180 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:68909 13.f 13:fd9b0258:::benchmark_data_smithi078_202524_object68908:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e298)", 2023-10-01T11:06:05.181 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:03:00.414536+0000", 2023-10-01T11:06:05.181 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 184.69072172899999, 2023-10-01T11:06:05.181 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.23123769999999999, 2023-10-01T11:06:05.181 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:06:05.181 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:06:05.181 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:06:05.181 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:06:05.181 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:06:05.181 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 68909 2023-10-01T11:06:05.181 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.181 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:06:05.182 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.182 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:06:05.182 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:00.414536+0000", 2023-10-01T11:06:05.182 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:06:05.182 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.182 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.182 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:06:05.182 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:00.414536+0000", 2023-10-01T11:06:05.182 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999995 2023-10-01T11:06:05.182 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.182 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.183 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:06:05.183 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:00.414535+0000", 2023-10-01T11:06:05.183 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.8105999999999997e-05 2023-10-01T11:06:05.183 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.183 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.183 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:06:05.183 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:00.414573+0000", 2023-10-01T11:06:05.183 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.901e-06 2023-10-01T11:06:05.183 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.183 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.183 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:06:05.183 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:00.414582+0000", 2023-10-01T11:06:05.184 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.1229999999999999e-06 2023-10-01T11:06:05.184 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.184 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.184 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:06:05.184 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:00.414587+0000", 2023-10-01T11:06:05.184 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0032587850000000002 2023-10-01T11:06:05.184 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.184 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.184 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:05.184 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:00.417846+0000", 2023-10-01T11:06:05.184 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00019491800000000001 2023-10-01T11:06:05.185 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.185 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.185 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:06:05.185 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:00.418041+0000", 2023-10-01T11:06:05.185 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00023517500000000001 2023-10-01T11:06:05.185 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.185 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.185 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 2,7", 2023-10-01T11:06:05.185 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:00.418276+0000", 2023-10-01T11:06:05.185 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0034629800000000001 2023-10-01T11:06:05.185 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.186 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.186 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:06:05.186 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:00.421739+0000", 2023-10-01T11:06:05.186 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.009839067 2023-10-01T11:06:05.186 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.186 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.186 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:06:05.186 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:00.431578+0000", 2023-10-01T11:06:05.186 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.207074493 2023-10-01T11:06:05.186 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.186 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.187 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:06:05.187 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:00.638652+0000", 2023-10-01T11:06:05.187 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.2053999999999997e-05 2023-10-01T11:06:05.187 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.187 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.187 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:06:05.187 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:00.638735+0000", 2023-10-01T11:06:05.187 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0070387940000000001 2023-10-01T11:06:05.187 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.187 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.187 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:06:05.187 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:00.645773+0000", 2023-10-01T11:06:05.188 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.23123769999999999 2023-10-01T11:06:05.188 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:05.188 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:06:05.188 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:05.188 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.188 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.188 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:68916 13.f 13:ff979225:::benchmark_data_smithi078_202524_object68915:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e298)", 2023-10-01T11:06:05.188 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:03:00.421790+0000", 2023-10-01T11:06:05.188 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 184.68346765300001, 2023-10-01T11:06:05.188 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.22824936600000001, 2023-10-01T11:06:05.189 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:06:05.189 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:06:05.189 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:06:05.189 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:06:05.189 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:06:05.189 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 68916 2023-10-01T11:06:05.189 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.189 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:06:05.189 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.189 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:06:05.189 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:00.421790+0000", 2023-10-01T11:06:05.190 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:06:05.190 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.190 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.190 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:06:05.190 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:00.421790+0000", 2023-10-01T11:06:05.190 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-10-01T11:06:05.190 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.190 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.190 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:06:05.190 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:00.421788+0000", 2023-10-01T11:06:05.190 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.7129000000000003e-05 2023-10-01T11:06:05.190 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.191 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.191 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:06:05.191 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:00.421825+0000", 2023-10-01T11:06:05.191 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.3069e-05 2023-10-01T11:06:05.191 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.191 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.191 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:06:05.191 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:00.421838+0000", 2023-10-01T11:06:05.191 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.5169999999999996e-06 2023-10-01T11:06:05.191 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.191 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.191 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:06:05.192 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:00.421845+0000", 2023-10-01T11:06:05.192 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.3181e-05 2023-10-01T11:06:05.192 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.192 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.192 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:05.192 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:00.421878+0000", 2023-10-01T11:06:05.192 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000119963 2023-10-01T11:06:05.192 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.192 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.192 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:06:05.192 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:00.421998+0000", 2023-10-01T11:06:05.193 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00013839200000000001 2023-10-01T11:06:05.193 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.193 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.193 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 2,7", 2023-10-01T11:06:05.193 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:00.422136+0000", 2023-10-01T11:06:05.193 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0094527659999999996 2023-10-01T11:06:05.193 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.193 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.193 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:06:05.193 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:00.431589+0000", 2023-10-01T11:06:05.193 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000407704 2023-10-01T11:06:05.194 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.194 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.194 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:06:05.194 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:00.431997+0000", 2023-10-01T11:06:05.194 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.21384493299999999 2023-10-01T11:06:05.194 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.194 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.194 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:06:05.194 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:00.645841+0000", 2023-10-01T11:06:05.194 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00012500400000000001 2023-10-01T11:06:05.194 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.194 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.195 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:06:05.195 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:00.645966+0000", 2023-10-01T11:06:05.195 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0040726319999999996 2023-10-01T11:06:05.195 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.195 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.195 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:06:05.195 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:00.650039+0000", 2023-10-01T11:06:05.195 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.22824936600000001 2023-10-01T11:06:05.195 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:05.195 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:06:05.195 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:05.196 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.196 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.196 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:73715 13.f 13:ff65771b:::benchmark_data_smithi078_202524_object73714:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e298)", 2023-10-01T11:06:05.196 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:03:06.928963+0000", 2023-10-01T11:06:05.196 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 178.17629475300001, 2023-10-01T11:06:05.196 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.78847444700000002, 2023-10-01T11:06:05.196 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:06:05.196 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "reached pg", 2023-10-01T11:06:05.196 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:06:05.196 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:06:05.196 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:06:05.197 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 73715 2023-10-01T11:06:05.197 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.197 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:06:05.197 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.197 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:06:05.197 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.928963+0000", 2023-10-01T11:06:05.197 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:06:05.197 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.197 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.197 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:06:05.198 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.928963+0000", 2023-10-01T11:06:05.198 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-10-01T11:06:05.198 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.198 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.198 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:06:05.198 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.928961+0000", 2023-10-01T11:06:05.198 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.9901000000000001e-05 2023-10-01T11:06:05.198 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.198 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.198 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:06:05.198 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.929021+0000", 2023-10-01T11:06:05.199 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.5566e-05 2023-10-01T11:06:05.199 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.199 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.199 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:06:05.199 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.929036+0000", 2023-10-01T11:06:05.199 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.9510000000000006e-06 2023-10-01T11:06:05.199 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.199 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.199 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:06:05.199 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.929044+0000", 2023-10-01T11:06:05.199 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.5580000000000005e-05 2023-10-01T11:06:05.199 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.200 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.200 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:05.200 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.929120+0000", 2023-10-01T11:06:05.200 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000173618 2023-10-01T11:06:05.200 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.200 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.200 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:06:05.200 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.929293+0000", 2023-10-01T11:06:05.200 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000214028 2023-10-01T11:06:05.200 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.201 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.201 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 2,7", 2023-10-01T11:06:05.201 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.929507+0000", 2023-10-01T11:06:05.201 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.010736101 2023-10-01T11:06:05.201 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.201 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.201 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:06:05.201 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.940244+0000", 2023-10-01T11:06:05.201 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0030327850000000001 2023-10-01T11:06:05.201 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.201 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.202 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:06:05.202 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.943276+0000", 2023-10-01T11:06:05.202 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.029978635999999999 2023-10-01T11:06:05.202 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.202 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.202 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:05.202 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.973255+0000", 2023-10-01T11:06:05.202 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.0572999999999999e-05 2023-10-01T11:06:05.202 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.202 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.202 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:06:05.202 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.973295+0000", 2023-10-01T11:06:05.203 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.74410996699999998 2023-10-01T11:06:05.203 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.203 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.203 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:05.203 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.717405+0000", 2023-10-01T11:06:05.203 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.1605e-05 2023-10-01T11:06:05.203 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.203 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.203 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:06:05.203 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.717437+0000", 2023-10-01T11:06:05.203 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.78847444700000002 2023-10-01T11:06:05.203 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:05.204 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:06:05.204 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:05.204 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.204 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.204 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:73717 13.f 13:f618ea28:::benchmark_data_smithi078_202524_object73716:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e298)", 2023-10-01T11:06:05.204 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:03:06.929065+0000", 2023-10-01T11:06:05.204 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 178.17619227899999, 2023-10-01T11:06:05.204 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.78843320100000003, 2023-10-01T11:06:05.204 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:06:05.204 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "reached pg", 2023-10-01T11:06:05.204 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:06:05.205 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:06:05.205 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:06:05.205 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 73717 2023-10-01T11:06:05.205 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.205 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:06:05.205 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.205 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:06:05.205 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.929065+0000", 2023-10-01T11:06:05.205 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:06:05.205 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.205 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.206 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:06:05.206 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.929065+0000", 2023-10-01T11:06:05.206 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-10-01T11:06:05.206 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.206 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.206 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:06:05.206 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.929064+0000", 2023-10-01T11:06:05.206 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.1127000000000001e-05 2023-10-01T11:06:05.206 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.206 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.206 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:06:05.207 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.929105+0000", 2023-10-01T11:06:05.207 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.5490000000000006e-06 2023-10-01T11:06:05.207 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.207 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.207 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:06:05.207 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.929114+0000", 2023-10-01T11:06:05.207 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.5330000000000002e-06 2023-10-01T11:06:05.207 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.207 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.207 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:06:05.208 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.929120+0000", 2023-10-01T11:06:05.208 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.014195055999999999 2023-10-01T11:06:05.208 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.208 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.208 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:05.208 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.943315+0000", 2023-10-01T11:06:05.208 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00041657400000000001 2023-10-01T11:06:05.208 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.208 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.208 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:06:05.208 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.943731+0000", 2023-10-01T11:06:05.209 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.001347413 2023-10-01T11:06:05.209 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.209 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.209 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 2,7", 2023-10-01T11:06:05.209 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.945079+0000", 2023-10-01T11:06:05.209 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.015658522000000001 2023-10-01T11:06:05.209 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.209 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.209 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:06:05.210 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.960737+0000", 2023-10-01T11:06:05.210 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.012669303 2023-10-01T11:06:05.210 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.210 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.210 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:05.210 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.973406+0000", 2023-10-01T11:06:05.210 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0518e-05 2023-10-01T11:06:05.210 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.210 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.210 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:06:05.211 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.973417+0000", 2023-10-01T11:06:05.211 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.74406769100000003 2023-10-01T11:06:05.211 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.211 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.211 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:05.211 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.717485+0000", 2023-10-01T11:06:05.211 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.367e-05 2023-10-01T11:06:05.211 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.211 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.211 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:06:05.212 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.717498+0000", 2023-10-01T11:06:05.212 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.78843320100000003 2023-10-01T11:06:05.212 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:05.212 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:06:05.212 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:05.212 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.212 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.212 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:73721 13.f 13:f95d23cf:::benchmark_data_smithi078_202524_object73720:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e298)", 2023-10-01T11:06:05.212 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:03:06.935068+0000", 2023-10-01T11:06:05.212 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 178.170189405, 2023-10-01T11:06:05.213 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.782478173, 2023-10-01T11:06:05.213 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:06:05.213 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "reached pg", 2023-10-01T11:06:05.213 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:06:05.213 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:06:05.213 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:06:05.213 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 73721 2023-10-01T11:06:05.213 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.213 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:06:05.213 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.214 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:06:05.214 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.935068+0000", 2023-10-01T11:06:05.214 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:06:05.214 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.214 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.214 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:06:05.214 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.935068+0000", 2023-10-01T11:06:05.214 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-10-01T11:06:05.214 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.214 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.214 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:06:05.215 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.935066+0000", 2023-10-01T11:06:05.215 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.9147000000000002e-05 2023-10-01T11:06:05.215 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.215 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.215 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:06:05.215 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.935115+0000", 2023-10-01T11:06:05.215 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.8175000000000001e-05 2023-10-01T11:06:05.215 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.215 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.215 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:06:05.215 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.935134+0000", 2023-10-01T11:06:05.216 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0977e-05 2023-10-01T11:06:05.216 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.216 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.216 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:06:05.216 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.935145+0000", 2023-10-01T11:06:05.216 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.020502031 2023-10-01T11:06:05.216 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.216 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.216 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:05.216 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.955647+0000", 2023-10-01T11:06:05.216 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00015518899999999999 2023-10-01T11:06:05.217 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.217 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.217 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:06:05.217 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.955802+0000", 2023-10-01T11:06:05.217 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00018738199999999999 2023-10-01T11:06:05.217 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.217 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.217 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 2,7", 2023-10-01T11:06:05.217 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.955989+0000", 2023-10-01T11:06:05.217 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.017480155000000001 2023-10-01T11:06:05.218 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.218 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.218 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:05.218 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.973469+0000", 2023-10-01T11:06:05.218 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.357e-06 2023-10-01T11:06:05.218 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.218 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.218 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:06:05.218 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:06.973477+0000", 2023-10-01T11:06:05.218 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.74405902400000001 2023-10-01T11:06:05.218 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.219 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.219 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:05.219 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.717536+0000", 2023-10-01T11:06:05.219 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0383e-05 2023-10-01T11:06:05.219 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.219 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.219 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:06:05.219 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.717546+0000", 2023-10-01T11:06:05.219 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.782478173 2023-10-01T11:06:05.219 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:05.219 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:06:05.220 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:05.220 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.220 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.220 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:73756 13.f 13.8597a85f (undecoded) ondisk+write+known_if_redirected e299)", 2023-10-01T11:06:05.220 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:03:07.045583+0000", 2023-10-01T11:06:05.220 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 178.059674336, 2023-10-01T11:06:05.220 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.67247588300000005, 2023-10-01T11:06:05.220 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:06:05.220 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "reached pg", 2023-10-01T11:06:05.220 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:06:05.220 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:06:05.220 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:06:05.221 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 73756 2023-10-01T11:06:05.221 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.221 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:06:05.221 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.221 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:06:05.221 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.045583+0000", 2023-10-01T11:06:05.221 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:06:05.221 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.221 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.221 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:06:05.221 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.045583+0000", 2023-10-01T11:06:05.221 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999971 2023-10-01T11:06:05.222 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.222 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.222 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:06:05.222 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.045580+0000", 2023-10-01T11:06:05.222 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.4255000000000001e-05 2023-10-01T11:06:05.222 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.222 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.222 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:06:05.222 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.045634+0000", 2023-10-01T11:06:05.222 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.5843e-05 2023-10-01T11:06:05.222 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.223 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.223 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:06:05.223 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.045660+0000", 2023-10-01T11:06:05.223 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0448999999999999e-05 2023-10-01T11:06:05.223 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.223 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.223 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:06:05.223 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.045670+0000", 2023-10-01T11:06:05.223 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.8107000000000004e-05 2023-10-01T11:06:05.223 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.223 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.223 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:05.224 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.045739+0000", 2023-10-01T11:06:05.224 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.277e-05 2023-10-01T11:06:05.224 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.224 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.224 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:06:05.224 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.045751+0000", 2023-10-01T11:06:05.224 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.67228892900000004 2023-10-01T11:06:05.224 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.224 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.224 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:05.225 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.718040+0000", 2023-10-01T11:06:05.225 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.8621999999999999e-05 2023-10-01T11:06:05.225 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.225 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.225 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:06:05.225 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.718059+0000", 2023-10-01T11:06:05.225 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.67247588300000005 2023-10-01T11:06:05.225 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:05.225 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:06:05.225 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:05.225 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.225 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.226 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.46454.0:89971 13.0 e330/323 13:0d1649c4:::benchmark_data_smithi078_202524_object10413:head v 330'5591, mlcod=292'3206)", 2023-10-01T11:06:05.226 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:06:04.355752+0000", 2023-10-01T11:06:05.226 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 0.74950554899999999, 2023-10-01T11:06:05.226 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0097371639999999995, 2023-10-01T11:06:05.226 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:06:05.226 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:06:05.226 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:06:05.226 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.226 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:06:05.226 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:04.355752+0000", 2023-10-01T11:06:05.226 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:06:05.227 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.227 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.227 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:06:05.227 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:04.355752+0000", 2023-10-01T11:06:05.227 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-10-01T11:06:05.227 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.227 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.227 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:06:05.227 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:04.355750+0000", 2023-10-01T11:06:05.227 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.0418999999999999e-05 2023-10-01T11:06:05.227 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.227 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.228 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:06:05.228 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:04.355771+0000", 2023-10-01T11:06:05.228 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.2379999999999999e-05 2023-10-01T11:06:05.228 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.228 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.228 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:06:05.228 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:04.355793+0000", 2023-10-01T11:06:05.228 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4455e-05 2023-10-01T11:06:05.228 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.228 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.228 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:06:05.228 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:04.355808+0000", 2023-10-01T11:06:05.229 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.7198999999999999e-05 2023-10-01T11:06:05.229 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.229 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.229 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:05.229 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:04.355865+0000", 2023-10-01T11:06:05.229 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.9431e-05 2023-10-01T11:06:05.229 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.229 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.229 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:06:05.229 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:04.355894+0000", 2023-10-01T11:06:05.229 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0095583690000000006 2023-10-01T11:06:05.229 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.230 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.230 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:06:05.230 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:04.365453+0000", 2023-10-01T11:06:05.230 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.6245999999999998e-05 2023-10-01T11:06:05.230 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.230 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.230 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:06:05.230 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:04.365489+0000", 2023-10-01T11:06:05.230 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0097371639999999995 2023-10-01T11:06:05.230 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:05.230 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:06:05.231 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:05.231 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.231 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.231 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.46454.0:89893 13.7 e330/324 13:ebfb0b5e:::benchmark_data_smithi078_202524_object10335:head v 330'5751, mlcod=291'714)", 2023-10-01T11:06:05.231 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:06:04.810010+0000", 2023-10-01T11:06:05.231 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 0.29524777099999999, 2023-10-01T11:06:05.231 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0075933989999999998, 2023-10-01T11:06:05.231 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:06:05.231 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:06:05.231 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:06:05.231 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.232 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:06:05.232 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:04.810010+0000", 2023-10-01T11:06:05.232 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:06:05.232 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.232 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.232 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:06:05.232 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:04.810010+0000", 2023-10-01T11:06:05.232 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-10-01T11:06:05.232 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.232 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.232 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:06:05.232 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:04.810009+0000", 2023-10-01T11:06:05.233 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.1289999999999996e-06 2023-10-01T11:06:05.233 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.233 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.233 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:06:05.233 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:04.810017+0000", 2023-10-01T11:06:05.233 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1336e-05 2023-10-01T11:06:05.233 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.233 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.233 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:06:05.233 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:04.810028+0000", 2023-10-01T11:06:05.233 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.4390000000000003e-06 2023-10-01T11:06:05.234 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.234 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.234 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:06:05.234 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:04.810036+0000", 2023-10-01T11:06:05.234 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0032309180000000002 2023-10-01T11:06:05.234 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.234 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.234 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:05.234 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:04.813266+0000", 2023-10-01T11:06:05.234 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.2632000000000001e-05 2023-10-01T11:06:05.234 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.235 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.235 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:06:05.235 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:04.813309+0000", 2023-10-01T11:06:05.235 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0042659519999999999 2023-10-01T11:06:05.235 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.235 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.235 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:06:05.235 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:04.817575+0000", 2023-10-01T11:06:05.235 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.7945000000000001e-05 2023-10-01T11:06:05.235 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.235 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.236 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:06:05.236 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:04.817603+0000", 2023-10-01T11:06:05.236 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0075933989999999998 2023-10-01T11:06:05.236 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:05.236 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:06:05.236 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:05.236 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.236 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.236 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:90071 13.f 13:f59b73af:::benchmark_data_smithi078_202524_object10513:head [delete] snapc 0=[] ondisk+write+known_if_redirected e330)", 2023-10-01T11:06:05.236 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:06:04.881131+0000", 2023-10-01T11:06:05.236 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 0.224126511, 2023-10-01T11:06:05.236 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.015283435999999999, 2023-10-01T11:06:05.237 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:06:05.237 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:06:05.237 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:06:05.237 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:06:05.237 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:06:05.237 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 90071 2023-10-01T11:06:05.237 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.237 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:06:05.237 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.237 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:06:05.237 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:04.881131+0000", 2023-10-01T11:06:05.237 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:06:05.238 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.238 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.238 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:06:05.238 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:04.881131+0000", 2023-10-01T11:06:05.238 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-10-01T11:06:05.238 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.238 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.238 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:06:05.238 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:04.881129+0000", 2023-10-01T11:06:05.238 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.1889999999999996e-06 2023-10-01T11:06:05.238 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.239 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.239 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:06:05.239 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:04.881138+0000", 2023-10-01T11:06:05.239 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.8068e-05 2023-10-01T11:06:05.239 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.239 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.239 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:06:05.239 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:04.881156+0000", 2023-10-01T11:06:05.239 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.251e-06 2023-10-01T11:06:05.239 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.239 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.239 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:06:05.240 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:04.881165+0000", 2023-10-01T11:06:05.240 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.0875999999999998e-05 2023-10-01T11:06:05.240 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.240 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.240 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:05.240 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:04.881226+0000", 2023-10-01T11:06:05.240 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000113338 2023-10-01T11:06:05.240 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.240 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.240 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:06:05.240 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:04.881339+0000", 2023-10-01T11:06:05.240 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00010999500000000001 2023-10-01T11:06:05.241 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.241 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.241 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 2,5", 2023-10-01T11:06:05.241 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:04.881449+0000", 2023-10-01T11:06:05.241 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.004191012 2023-10-01T11:06:05.241 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.241 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.241 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:06:05.241 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:04.885640+0000", 2023-10-01T11:06:05.241 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00030078899999999998 2023-10-01T11:06:05.241 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.241 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.241 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:06:05.242 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:04.885941+0000", 2023-10-01T11:06:05.242 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.010374593 2023-10-01T11:06:05.242 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.242 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.242 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:06:05.242 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:04.896316+0000", 2023-10-01T11:06:05.242 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.5043999999999997e-05 2023-10-01T11:06:05.242 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.242 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.242 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:06:05.242 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:04.896371+0000", 2023-10-01T11:06:05.243 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.3420000000000001e-05 2023-10-01T11:06:05.243 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:05.243 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:05.243 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:06:05.243 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:04.896414+0000", 2023-10-01T11:06:05.243 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.015283435999999999 2023-10-01T11:06:05.243 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:05.243 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:06:05.243 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:05.243 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:05.243 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:06:05.243 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-10-01T11:06:05.244 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:05.264 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:05.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:05.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:06:05.368 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:05.390 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:05.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:05.395 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:05.496 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:05.516 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:05.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:05.518 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:06:05.619 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:05.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:05 smithi173 bash[58816]: cluster 2023-10-01T11:06:04.295902+0000 mgr.y (mgr.25072) 1462 : cluster 0 pgmap v1555: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 148 op/s; 3008/208515 objects degraded (1.443%); 4793/208515 objects misplaced (2.299%); 783 KiB/s, 56 objects/s recovering 2023-10-01T11:06:05.645 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:05.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:05.647 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:06:05.748 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:05.770 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:05.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:05.776 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:06:05.808 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:05 smithi078 bash[78274]: cluster 2023-10-01T11:06:04.295902+0000 mgr.y (mgr.25072) 1462 : cluster 0 pgmap v1555: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 148 op/s; 3008/208515 objects degraded (1.443%); 4793/208515 objects misplaced (2.299%); 783 KiB/s, 56 objects/s recovering 2023-10-01T11:06:05.808 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:05 smithi078 bash[75299]: cluster 2023-10-01T11:06:04.295902+0000 mgr.y (mgr.25072) 1462 : cluster 0 pgmap v1555: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 148 op/s; 3008/208515 objects degraded (1.443%); 4793/208515 objects misplaced (2.299%); 783 KiB/s, 56 objects/s recovering 2023-10-01T11:06:05.876 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:05.905 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:05.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:05.907 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:06:05.973 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-01T11:06:06.009 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:06.033 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:06.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:06.035 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:06:06.136 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:06.161 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:06.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:06.164 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:06:06.264 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:06.294 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:06.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:06.297 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:06:06.398 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:06.423 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:06.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:06.426 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:06:06.527 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:06.552 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:06.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:06.554 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:06.655 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:06.728 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:06.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:06.730 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:06.831 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:06.855 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:06.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:06.860 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:06:06.961 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:06.984 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:06.986 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T11:06:06.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:06.987 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:07.089 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:07.115 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:07.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:07.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:07.218 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:07.242 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:07.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:07.246 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:06:07.348 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:07.371 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:07.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:07.374 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:06:07.474 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:07.484 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:07 smithi173 bash[58816]: cluster 2023-10-01T11:06:06.296528+0000 mgr.y (mgr.25072) 1463 : cluster 0 pgmap v1556: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 152 op/s; 3008/207774 objects degraded (1.448%); 4793/207774 objects misplaced (2.307%); 720 KiB/s, 50 objects/s recovering 2023-10-01T11:06:07.500 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:07.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:07.502 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:07.603 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:07.614 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:07 smithi078 bash[75299]: cluster 2023-10-01T11:06:06.296528+0000 mgr.y (mgr.25072) 1463 : cluster 0 pgmap v1556: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 152 op/s; 3008/207774 objects degraded (1.448%); 4793/207774 objects misplaced (2.307%); 720 KiB/s, 50 objects/s recovering 2023-10-01T11:06:07.614 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:07 smithi078 bash[78274]: cluster 2023-10-01T11:06:06.296528+0000 mgr.y (mgr.25072) 1463 : cluster 0 pgmap v1556: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 152 op/s; 3008/207774 objects degraded (1.448%); 4793/207774 objects misplaced (2.307%); 720 KiB/s, 50 objects/s recovering 2023-10-01T11:06:07.630 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:07.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:07.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:06:07.735 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:07.763 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:07.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:07.768 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:06:07.868 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:07.896 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:07.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:07.899 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:06:08.000 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:08.026 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:08.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:08.031 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:08.131 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:08.158 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:08.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:08.161 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:08.262 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:08.288 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:08.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:08.292 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:06:08.393 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:08.421 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:08.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:08.423 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:08.524 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:08.551 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:08.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:08.554 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:06:08.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:08 smithi173 bash[58816]: cluster 2023-10-01T11:06:07.741562+0000 mon.a (mon.0) 1873 : cluster 3 Health check update: Degraded data redundancy: 3008/207774 objects degraded (1.448%), 2 pgs degraded (PG_DEGRADED) 2023-10-01T11:06:08.655 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:08.681 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:08.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:08.684 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:06:08.785 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:08.793 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:08 smithi078 bash[78274]: cluster 2023-10-01T11:06:07.741562+0000 mon.a (mon.0) 1873 : cluster 3 Health check update: Degraded data redundancy: 3008/207774 objects degraded (1.448%), 2 pgs degraded (PG_DEGRADED) 2023-10-01T11:06:08.793 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:08 smithi078 bash[75299]: cluster 2023-10-01T11:06:07.741562+0000 mon.a (mon.0) 1873 : cluster 3 Health check update: Degraded data redundancy: 3008/207774 objects degraded (1.448%), 2 pgs degraded (PG_DEGRADED) 2023-10-01T11:06:08.806 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:08.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:08.810 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:06:08.910 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:08.936 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:08.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:08.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:06:09.039 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:09.062 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:09.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:09.065 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:06:09.166 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:09.189 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:09.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:09.192 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:06:09.292 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:09.317 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:09.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:09.319 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:09.420 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:09.448 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:09.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:09.450 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:06:09.551 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:09.585 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:09.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:09.588 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:06:09.688 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:09.714 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:09.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:09.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:09.817 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:09.846 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:09.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:09.849 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:09.950 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:09.961 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:09 smithi173 bash[58816]: cluster 2023-10-01T11:06:08.297124+0000 mgr.y (mgr.25072) 1464 : cluster 0 pgmap v1557: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 147 op/s; 3008/207234 objects degraded (1.451%); 4751/207234 objects misplaced (2.293%); 624 KiB/s, 44 objects/s recovering 2023-10-01T11:06:09.974 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:09.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:09.978 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:06:10.011 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:09 smithi078 bash[78274]: cluster 2023-10-01T11:06:08.297124+0000 mgr.y (mgr.25072) 1464 : cluster 0 pgmap v1557: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 147 op/s; 3008/207234 objects degraded (1.451%); 4751/207234 objects misplaced (2.293%); 624 KiB/s, 44 objects/s recovering 2023-10-01T11:06:10.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:09 smithi078 bash[75299]: cluster 2023-10-01T11:06:08.297124+0000 mgr.y (mgr.25072) 1464 : cluster 0 pgmap v1557: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 147 op/s; 3008/207234 objects degraded (1.451%); 4751/207234 objects misplaced (2.293%); 624 KiB/s, 44 objects/s recovering 2023-10-01T11:06:10.079 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:10.102 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:10.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:10.105 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:06:10.205 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:10.231 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:10.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:10.235 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:06:10.336 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:10.366 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:10.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:10.368 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:06:10.469 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:10.500 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:10.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:10.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:06:10.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:10 smithi078 bash[75299]: debug 2023-10-01T11:06:10.474+0000 7f31c8dd0700 -1 mon.a@0(leader).osd e330 definitely_dead 0 2023-10-01T11:06:10.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:06:10 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:06:10] "GET /metrics HTTP/1.1" 200 36412 "" "Prometheus/2.33.4" 2023-10-01T11:06:10.605 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:10.630 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:10.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:10.633 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:06:10.734 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:10.756 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T11:06:10.761 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:10.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:10.764 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:06:10.777 INFO:teuthology.orchestra.run.smithi078.stderr:noscrub is set 2023-10-01T11:06:10.779 INFO:teuthology.orchestra.run.smithi078.stderr:marked in osd.7. 2023-10-01T11:06:10.865 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:10.887 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:10.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:10.889 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:10.989 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:10.999 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:10 smithi078 bash[78274]: audit 2023-10-01T11:06:10.112721+0000 mon.c (mon.2) 221 : audit 1 from='client.? 172.21.15.78:0/1302934045' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T11:06:10.999 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:10 smithi078 bash[78274]: audit 2023-10-01T11:06:10.113091+0000 mon.a (mon.0) 1874 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T11:06:10.999 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:10 smithi078 bash[78274]: audit 2023-10-01T11:06:10.478230+0000 mon.c (mon.2) 222 : audit 0 from='client.? 172.21.15.78:0/938562609' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-01T11:06:10.999 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:10 smithi078 bash[78274]: audit 2023-10-01T11:06:10.478761+0000 mon.a (mon.0) 1875 : audit 0 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-01T11:06:10.999 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:10 smithi078 bash[75299]: audit 2023-10-01T11:06:10.112721+0000 mon.c (mon.2) 221 : audit 1 from='client.? 172.21.15.78:0/1302934045' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T11:06:10.999 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:10 smithi078 bash[75299]: audit 2023-10-01T11:06:10.113091+0000 mon.a (mon.0) 1874 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T11:06:10.999 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:10 smithi078 bash[75299]: audit 2023-10-01T11:06:10.478230+0000 mon.c (mon.2) 222 : audit 0 from='client.? 172.21.15.78:0/938562609' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-01T11:06:10.999 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:10 smithi078 bash[75299]: audit 2023-10-01T11:06:10.478761+0000 mon.a (mon.0) 1875 : audit 0 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-01T11:06:11.015 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:11.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:11.018 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:06:11.119 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:11.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:10 smithi173 bash[58816]: audit 2023-10-01T11:06:10.112721+0000 mon.c (mon.2) 221 : audit 1 from='client.? 172.21.15.78:0/1302934045' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T11:06:11.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:10 smithi173 bash[58816]: audit 2023-10-01T11:06:10.113091+0000 mon.a (mon.0) 1874 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T11:06:11.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:10 smithi173 bash[58816]: audit 2023-10-01T11:06:10.478230+0000 mon.c (mon.2) 222 : audit 0 from='client.? 172.21.15.78:0/938562609' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-01T11:06:11.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:10 smithi173 bash[58816]: audit 2023-10-01T11:06:10.478761+0000 mon.a (mon.0) 1875 : audit 0 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-10-01T11:06:11.146 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:11.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:11.148 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:06:11.249 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:11.277 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:11.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:11.280 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:06:11.381 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:11.406 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:11.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:11.410 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:11.511 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:11.532 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:11.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:11.534 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:06:11.635 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:11.658 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:11.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:11.661 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:11.762 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:11.777 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:11 smithi078 bash[75299]: cluster 2023-10-01T11:06:10.298138+0000 mgr.y (mgr.25072) 1465 : cluster 0 pgmap v1558: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 178 op/s; 3008/205491 objects degraded (1.464%); 4751/205491 objects misplaced (2.312%); 624 KiB/s, 44 objects/s recovering 2023-10-01T11:06:11.777 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:11 smithi078 bash[75299]: cluster 2023-10-01T11:06:10.747369+0000 mon.a (mon.0) 1876 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:06:11.777 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:11 smithi078 bash[75299]: audit 2023-10-01T11:06:10.763734+0000 mon.a (mon.0) 1877 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T11:06:11.778 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:11 smithi078 bash[75299]: audit 2023-10-01T11:06:10.763861+0000 mon.a (mon.0) 1878 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-10-01T11:06:11.778 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:11 smithi078 bash[75299]: cluster 2023-10-01T11:06:10.763956+0000 mon.a (mon.0) 1879 : cluster 0 osdmap e331: 8 total, 8 up, 8 in 2023-10-01T11:06:11.778 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:11 smithi078 bash[75299]: cluster 2023-10-01T11:06:10.765257+0000 osd.7 (osd.7) 357 : cluster 3 failed to encode map e331 with expected crc 2023-10-01T11:06:11.778 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:11 smithi078 bash[75299]: cluster 2023-10-01T11:06:10.766058+0000 osd.5 (osd.5) 345 : cluster 3 failed to encode map e331 with expected crc 2023-10-01T11:06:11.778 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:11 smithi078 bash[75299]: cluster 2023-10-01T11:06:10.766192+0000 osd.6 (osd.6) 312 : cluster 3 failed to encode map e331 with expected crc 2023-10-01T11:06:11.778 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:11 smithi078 bash[75299]: cluster 2023-10-01T11:06:10.766703+0000 osd.7 (osd.7) 358 : cluster 3 failed to encode map e331 with expected crc 2023-10-01T11:06:11.778 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:11 smithi078 bash[75299]: cluster 2023-10-01T11:06:10.768810+0000 osd.6 (osd.6) 313 : cluster 3 failed to encode map e331 with expected crc 2023-10-01T11:06:11.778 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:11 smithi078 bash[75299]: cluster 2023-10-01T11:06:10.768854+0000 osd.7 (osd.7) 359 : cluster 3 failed to encode map e331 with expected crc 2023-10-01T11:06:11.778 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:11 smithi078 bash[75299]: cluster 2023-10-01T11:06:10.770856+0000 osd.7 (osd.7) 360 : cluster 3 failed to encode map e331 with expected crc 2023-10-01T11:06:11.778 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:11 smithi078 bash[75299]: cluster 2023-10-01T11:06:10.772360+0000 osd.5 (osd.5) 346 : cluster 3 failed to encode map e331 with expected crc 2023-10-01T11:06:11.779 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:11 smithi078 bash[75299]: cluster 2023-10-01T11:06:10.775925+0000 osd.5 (osd.5) 347 : cluster 3 failed to encode map e331 with expected crc 2023-10-01T11:06:11.779 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:11 smithi078 bash[78274]: cluster 2023-10-01T11:06:10.298138+0000 mgr.y (mgr.25072) 1465 : cluster 0 pgmap v1558: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 178 op/s; 3008/205491 objects degraded (1.464%); 4751/205491 objects misplaced (2.312%); 624 KiB/s, 44 objects/s recovering 2023-10-01T11:06:11.779 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:11 smithi078 bash[78274]: cluster 2023-10-01T11:06:10.747369+0000 mon.a (mon.0) 1876 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:06:11.779 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:11 smithi078 bash[78274]: audit 2023-10-01T11:06:10.763734+0000 mon.a (mon.0) 1877 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T11:06:11.779 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:11 smithi078 bash[78274]: audit 2023-10-01T11:06:10.763861+0000 mon.a (mon.0) 1878 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-10-01T11:06:11.779 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:11 smithi078 bash[78274]: cluster 2023-10-01T11:06:10.763956+0000 mon.a (mon.0) 1879 : cluster 0 osdmap e331: 8 total, 8 up, 8 in 2023-10-01T11:06:11.779 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:11 smithi078 bash[78274]: cluster 2023-10-01T11:06:10.765257+0000 osd.7 (osd.7) 357 : cluster 3 failed to encode map e331 with expected crc 2023-10-01T11:06:11.779 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:11 smithi078 bash[78274]: cluster 2023-10-01T11:06:10.766058+0000 osd.5 (osd.5) 345 : cluster 3 failed to encode map e331 with expected crc 2023-10-01T11:06:11.779 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:11 smithi078 bash[78274]: cluster 2023-10-01T11:06:10.766192+0000 osd.6 (osd.6) 312 : cluster 3 failed to encode map e331 with expected crc 2023-10-01T11:06:11.780 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:11 smithi078 bash[78274]: cluster 2023-10-01T11:06:10.766703+0000 osd.7 (osd.7) 358 : cluster 3 failed to encode map e331 with expected crc 2023-10-01T11:06:11.780 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:11 smithi078 bash[78274]: cluster 2023-10-01T11:06:10.768810+0000 osd.6 (osd.6) 313 : cluster 3 failed to encode map e331 with expected crc 2023-10-01T11:06:11.780 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:11 smithi078 bash[78274]: cluster 2023-10-01T11:06:10.768854+0000 osd.7 (osd.7) 359 : cluster 3 failed to encode map e331 with expected crc 2023-10-01T11:06:11.780 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:11 smithi078 bash[78274]: cluster 2023-10-01T11:06:10.770856+0000 osd.7 (osd.7) 360 : cluster 3 failed to encode map e331 with expected crc 2023-10-01T11:06:11.780 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:11 smithi078 bash[78274]: cluster 2023-10-01T11:06:10.772360+0000 osd.5 (osd.5) 346 : cluster 3 failed to encode map e331 with expected crc 2023-10-01T11:06:11.780 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:11 smithi078 bash[78274]: cluster 2023-10-01T11:06:10.775925+0000 osd.5 (osd.5) 347 : cluster 3 failed to encode map e331 with expected crc 2023-10-01T11:06:11.804 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:11.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:11.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:06:11.912 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:11.947 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:11.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:11.949 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:06:12.050 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:12.060 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:11 smithi173 bash[58816]: cluster 2023-10-01T11:06:10.298138+0000 mgr.y (mgr.25072) 1465 : cluster 0 pgmap v1558: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 178 op/s; 3008/205491 objects degraded (1.464%); 4751/205491 objects misplaced (2.312%); 624 KiB/s, 44 objects/s recovering 2023-10-01T11:06:12.060 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:11 smithi173 bash[58816]: cluster 2023-10-01T11:06:10.747369+0000 mon.a (mon.0) 1876 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:06:12.060 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:11 smithi173 bash[58816]: audit 2023-10-01T11:06:10.763734+0000 mon.a (mon.0) 1877 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T11:06:12.060 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:11 smithi173 bash[58816]: audit 2023-10-01T11:06:10.763861+0000 mon.a (mon.0) 1878 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-10-01T11:06:12.061 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:11 smithi173 bash[58816]: cluster 2023-10-01T11:06:10.763956+0000 mon.a (mon.0) 1879 : cluster 0 osdmap e331: 8 total, 8 up, 8 in 2023-10-01T11:06:12.061 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:11 smithi173 bash[58816]: cluster 2023-10-01T11:06:10.765257+0000 osd.7 (osd.7) 357 : cluster 3 failed to encode map e331 with expected crc 2023-10-01T11:06:12.061 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:11 smithi173 bash[58816]: cluster 2023-10-01T11:06:10.766058+0000 osd.5 (osd.5) 345 : cluster 3 failed to encode map e331 with expected crc 2023-10-01T11:06:12.061 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:11 smithi173 bash[58816]: cluster 2023-10-01T11:06:10.766192+0000 osd.6 (osd.6) 312 : cluster 3 failed to encode map e331 with expected crc 2023-10-01T11:06:12.061 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:11 smithi173 bash[58816]: cluster 2023-10-01T11:06:10.766703+0000 osd.7 (osd.7) 358 : cluster 3 failed to encode map e331 with expected crc 2023-10-01T11:06:12.061 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:11 smithi173 bash[58816]: cluster 2023-10-01T11:06:10.768810+0000 osd.6 (osd.6) 313 : cluster 3 failed to encode map e331 with expected crc 2023-10-01T11:06:12.061 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:11 smithi173 bash[58816]: cluster 2023-10-01T11:06:10.768854+0000 osd.7 (osd.7) 359 : cluster 3 failed to encode map e331 with expected crc 2023-10-01T11:06:12.061 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:11 smithi173 bash[58816]: cluster 2023-10-01T11:06:10.770856+0000 osd.7 (osd.7) 360 : cluster 3 failed to encode map e331 with expected crc 2023-10-01T11:06:12.061 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:11 smithi173 bash[58816]: cluster 2023-10-01T11:06:10.772360+0000 osd.5 (osd.5) 346 : cluster 3 failed to encode map e331 with expected crc 2023-10-01T11:06:12.062 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:11 smithi173 bash[58816]: cluster 2023-10-01T11:06:10.775925+0000 osd.5 (osd.5) 347 : cluster 3 failed to encode map e331 with expected crc 2023-10-01T11:06:12.075 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:12.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:12.078 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:12.179 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:12.205 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:12.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:12.209 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:12.310 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:12.337 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:12.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:12.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:12.440 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:12.466 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:12.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:12.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:06:12.570 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:12.597 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:12.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:12.599 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:06:12.700 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:12.727 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:12.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:12.729 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:06:12.830 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:12.855 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:12.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:12.857 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:06:12.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:12 smithi173 bash[58816]: cluster 2023-10-01T11:06:11.763429+0000 osd.7 (osd.7) 361 : cluster 3 failed to encode map e332 with expected crc 2023-10-01T11:06:12.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:12 smithi173 bash[58816]: cluster 2023-10-01T11:06:11.765835+0000 osd.5 (osd.5) 348 : cluster 3 failed to encode map e332 with expected crc 2023-10-01T11:06:12.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:12 smithi173 bash[58816]: cluster 2023-10-01T11:06:11.766010+0000 mon.a (mon.0) 1880 : cluster 0 osdmap e332: 8 total, 8 up, 8 in 2023-10-01T11:06:12.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:12 smithi173 bash[58816]: cluster 2023-10-01T11:06:11.768632+0000 osd.7 (osd.7) 362 : cluster 3 failed to encode map e332 with expected crc 2023-10-01T11:06:12.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:12 smithi173 bash[58816]: cluster 2023-10-01T11:06:11.768973+0000 osd.6 (osd.6) 314 : cluster 3 failed to encode map e332 with expected crc 2023-10-01T11:06:12.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:12 smithi173 bash[58816]: cluster 2023-10-01T11:06:11.770074+0000 osd.5 (osd.5) 349 : cluster 3 failed to encode map e332 with expected crc 2023-10-01T11:06:12.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:12 smithi173 bash[58816]: cluster 2023-10-01T11:06:11.771129+0000 osd.7 (osd.7) 363 : cluster 3 failed to encode map e332 with expected crc 2023-10-01T11:06:12.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:12 smithi173 bash[58816]: cluster 2023-10-01T11:06:11.771726+0000 osd.6 (osd.6) 315 : cluster 3 failed to encode map e332 with expected crc 2023-10-01T11:06:12.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:12 smithi173 bash[58816]: cluster 2023-10-01T11:06:11.771827+0000 osd.5 (osd.5) 350 : cluster 3 failed to encode map e332 with expected crc 2023-10-01T11:06:12.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:12 smithi173 bash[58816]: cluster 2023-10-01T11:06:11.773043+0000 osd.5 (osd.5) 351 : cluster 3 failed to encode map e332 with expected crc 2023-10-01T11:06:12.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:12 smithi173 bash[58816]: cluster 2023-10-01T11:06:11.773207+0000 osd.7 (osd.7) 364 : cluster 3 failed to encode map e332 with expected crc 2023-10-01T11:06:12.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:12 smithi173 bash[58816]: cluster 2023-10-01T11:06:11.773601+0000 osd.6 (osd.6) 316 : cluster 3 failed to encode map e332 with expected crc 2023-10-01T11:06:12.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:12 smithi173 bash[58816]: cluster 2023-10-01T11:06:12.742500+0000 mon.a (mon.0) 1881 : cluster 3 Health check update: Degraded data redundancy: 3008/205491 objects degraded (1.464%), 2 pgs degraded (PG_DEGRADED) 2023-10-01T11:06:12.958 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:12.974 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:12 smithi078 bash[78274]: cluster 2023-10-01T11:06:11.763429+0000 osd.7 (osd.7) 361 : cluster 3 failed to encode map e332 with expected crc 2023-10-01T11:06:12.975 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:12 smithi078 bash[78274]: cluster 2023-10-01T11:06:11.765835+0000 osd.5 (osd.5) 348 : cluster 3 failed to encode map e332 with expected crc 2023-10-01T11:06:12.975 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:12 smithi078 bash[78274]: cluster 2023-10-01T11:06:11.766010+0000 mon.a (mon.0) 1880 : cluster 0 osdmap e332: 8 total, 8 up, 8 in 2023-10-01T11:06:12.975 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:12 smithi078 bash[78274]: cluster 2023-10-01T11:06:11.768632+0000 osd.7 (osd.7) 362 : cluster 3 failed to encode map e332 with expected crc 2023-10-01T11:06:12.975 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:12 smithi078 bash[78274]: cluster 2023-10-01T11:06:11.768973+0000 osd.6 (osd.6) 314 : cluster 3 failed to encode map e332 with expected crc 2023-10-01T11:06:12.975 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:12 smithi078 bash[78274]: cluster 2023-10-01T11:06:11.770074+0000 osd.5 (osd.5) 349 : cluster 3 failed to encode map e332 with expected crc 2023-10-01T11:06:12.975 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:12 smithi078 bash[78274]: cluster 2023-10-01T11:06:11.771129+0000 osd.7 (osd.7) 363 : cluster 3 failed to encode map e332 with expected crc 2023-10-01T11:06:12.975 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:12 smithi078 bash[78274]: cluster 2023-10-01T11:06:11.771726+0000 osd.6 (osd.6) 315 : cluster 3 failed to encode map e332 with expected crc 2023-10-01T11:06:12.975 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:12 smithi078 bash[78274]: cluster 2023-10-01T11:06:11.771827+0000 osd.5 (osd.5) 350 : cluster 3 failed to encode map e332 with expected crc 2023-10-01T11:06:12.975 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:12 smithi078 bash[78274]: cluster 2023-10-01T11:06:11.773043+0000 osd.5 (osd.5) 351 : cluster 3 failed to encode map e332 with expected crc 2023-10-01T11:06:12.975 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:12 smithi078 bash[78274]: cluster 2023-10-01T11:06:11.773207+0000 osd.7 (osd.7) 364 : cluster 3 failed to encode map e332 with expected crc 2023-10-01T11:06:12.975 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:12 smithi078 bash[78274]: cluster 2023-10-01T11:06:11.773601+0000 osd.6 (osd.6) 316 : cluster 3 failed to encode map e332 with expected crc 2023-10-01T11:06:12.976 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:12 smithi078 bash[78274]: cluster 2023-10-01T11:06:12.742500+0000 mon.a (mon.0) 1881 : cluster 3 Health check update: Degraded data redundancy: 3008/205491 objects degraded (1.464%), 2 pgs degraded (PG_DEGRADED) 2023-10-01T11:06:12.976 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:12 smithi078 bash[75299]: cluster 2023-10-01T11:06:11.763429+0000 osd.7 (osd.7) 361 : cluster 3 failed to encode map e332 with expected crc 2023-10-01T11:06:12.976 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:12 smithi078 bash[75299]: cluster 2023-10-01T11:06:11.765835+0000 osd.5 (osd.5) 348 : cluster 3 failed to encode map e332 with expected crc 2023-10-01T11:06:12.976 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:12 smithi078 bash[75299]: cluster 2023-10-01T11:06:11.766010+0000 mon.a (mon.0) 1880 : cluster 0 osdmap e332: 8 total, 8 up, 8 in 2023-10-01T11:06:12.976 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:12 smithi078 bash[75299]: cluster 2023-10-01T11:06:11.768632+0000 osd.7 (osd.7) 362 : cluster 3 failed to encode map e332 with expected crc 2023-10-01T11:06:12.976 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:12 smithi078 bash[75299]: cluster 2023-10-01T11:06:11.768973+0000 osd.6 (osd.6) 314 : cluster 3 failed to encode map e332 with expected crc 2023-10-01T11:06:12.976 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:12 smithi078 bash[75299]: cluster 2023-10-01T11:06:11.770074+0000 osd.5 (osd.5) 349 : cluster 3 failed to encode map e332 with expected crc 2023-10-01T11:06:12.976 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:12 smithi078 bash[75299]: cluster 2023-10-01T11:06:11.771129+0000 osd.7 (osd.7) 363 : cluster 3 failed to encode map e332 with expected crc 2023-10-01T11:06:12.977 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:12 smithi078 bash[75299]: cluster 2023-10-01T11:06:11.771726+0000 osd.6 (osd.6) 315 : cluster 3 failed to encode map e332 with expected crc 2023-10-01T11:06:12.977 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:12 smithi078 bash[75299]: cluster 2023-10-01T11:06:11.771827+0000 osd.5 (osd.5) 350 : cluster 3 failed to encode map e332 with expected crc 2023-10-01T11:06:12.977 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:12 smithi078 bash[75299]: cluster 2023-10-01T11:06:11.773043+0000 osd.5 (osd.5) 351 : cluster 3 failed to encode map e332 with expected crc 2023-10-01T11:06:12.977 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:12 smithi078 bash[75299]: cluster 2023-10-01T11:06:11.773207+0000 osd.7 (osd.7) 364 : cluster 3 failed to encode map e332 with expected crc 2023-10-01T11:06:12.977 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:12 smithi078 bash[75299]: cluster 2023-10-01T11:06:11.773601+0000 osd.6 (osd.6) 316 : cluster 3 failed to encode map e332 with expected crc 2023-10-01T11:06:12.977 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:12 smithi078 bash[75299]: cluster 2023-10-01T11:06:12.742500+0000 mon.a (mon.0) 1881 : cluster 3 Health check update: Degraded data redundancy: 3008/205491 objects degraded (1.464%), 2 pgs degraded (PG_DEGRADED) 2023-10-01T11:06:12.978 INFO:tasks.thrashosds.thrasher:Added osd 7 2023-10-01T11:06:12.984 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:12.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:12.986 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:06:13.086 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:13.116 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:13.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:13.119 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:06:13.220 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:13.247 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:13.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:13.250 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:06:13.350 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:13.372 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:13.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:13.374 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:13.376 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:06:13.475 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:13.506 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:13.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:13.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:06:13.610 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:13.632 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:13.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:13.635 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:06:13.735 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:13.762 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:13.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:13.764 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:06:13.780 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:13 smithi173 bash[58816]: cluster 2023-10-01T11:06:12.298830+0000 mgr.y (mgr.25072) 1466 : cluster 0 pgmap v1561: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 132 op/s; 3008/205332 objects degraded (1.465%); 4751/205332 objects misplaced (2.314%); 864 KiB/s, 13 objects/s recovering 2023-10-01T11:06:13.780 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:13 smithi173 bash[58816]: cluster 2023-10-01T11:06:12.770174+0000 mon.a (mon.0) 1882 : cluster 0 osdmap e333: 8 total, 8 up, 8 in 2023-10-01T11:06:13.780 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:13 smithi173 bash[58816]: cluster 2023-10-01T11:06:12.771118+0000 osd.5 (osd.5) 352 : cluster 3 failed to encode map e333 with expected crc 2023-10-01T11:06:13.781 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:13 smithi173 bash[58816]: cluster 2023-10-01T11:06:12.771277+0000 osd.6 (osd.6) 317 : cluster 3 failed to encode map e333 with expected crc 2023-10-01T11:06:13.781 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:13 smithi173 bash[58816]: cluster 2023-10-01T11:06:12.773208+0000 osd.7 (osd.7) 365 : cluster 3 failed to encode map e333 with expected crc 2023-10-01T11:06:13.781 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:13 smithi173 bash[58816]: cluster 2023-10-01T11:06:12.773806+0000 osd.6 (osd.6) 318 : cluster 3 failed to encode map e333 with expected crc 2023-10-01T11:06:13.781 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:13 smithi173 bash[58816]: cluster 2023-10-01T11:06:12.775045+0000 osd.7 (osd.7) 366 : cluster 3 failed to encode map e333 with expected crc 2023-10-01T11:06:13.865 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:13.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:13 smithi173 bash[58816]: cluster 2023-10-01T11:06:12.775407+0000 osd.5 (osd.5) 353 : cluster 3 failed to encode map e333 with expected crc 2023-10-01T11:06:13.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:13 smithi173 bash[58816]: cluster 2023-10-01T11:06:12.777141+0000 osd.7 (osd.7) 367 : cluster 3 failed to encode map e333 with expected crc 2023-10-01T11:06:13.890 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:13.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:13.892 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:06:13.993 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:14.020 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:14.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:14.023 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:14.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:13 smithi078 bash[75299]: cluster 2023-10-01T11:06:12.298830+0000 mgr.y (mgr.25072) 1466 : cluster 0 pgmap v1561: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 132 op/s; 3008/205332 objects degraded (1.465%); 4751/205332 objects misplaced (2.314%); 864 KiB/s, 13 objects/s recovering 2023-10-01T11:06:14.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:13 smithi078 bash[75299]: cluster 2023-10-01T11:06:12.770174+0000 mon.a (mon.0) 1882 : cluster 0 osdmap e333: 8 total, 8 up, 8 in 2023-10-01T11:06:14.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:13 smithi078 bash[75299]: cluster 2023-10-01T11:06:12.771118+0000 osd.5 (osd.5) 352 : cluster 3 failed to encode map e333 with expected crc 2023-10-01T11:06:14.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:13 smithi078 bash[75299]: cluster 2023-10-01T11:06:12.771277+0000 osd.6 (osd.6) 317 : cluster 3 failed to encode map e333 with expected crc 2023-10-01T11:06:14.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:13 smithi078 bash[75299]: cluster 2023-10-01T11:06:12.773208+0000 osd.7 (osd.7) 365 : cluster 3 failed to encode map e333 with expected crc 2023-10-01T11:06:14.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:13 smithi078 bash[75299]: cluster 2023-10-01T11:06:12.773806+0000 osd.6 (osd.6) 318 : cluster 3 failed to encode map e333 with expected crc 2023-10-01T11:06:14.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:13 smithi078 bash[75299]: cluster 2023-10-01T11:06:12.775045+0000 osd.7 (osd.7) 366 : cluster 3 failed to encode map e333 with expected crc 2023-10-01T11:06:14.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:13 smithi078 bash[75299]: cluster 2023-10-01T11:06:12.775407+0000 osd.5 (osd.5) 353 : cluster 3 failed to encode map e333 with expected crc 2023-10-01T11:06:14.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:13 smithi078 bash[75299]: cluster 2023-10-01T11:06:12.777141+0000 osd.7 (osd.7) 367 : cluster 3 failed to encode map e333 with expected crc 2023-10-01T11:06:14.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:13 smithi078 bash[78274]: cluster 2023-10-01T11:06:12.298830+0000 mgr.y (mgr.25072) 1466 : cluster 0 pgmap v1561: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 132 op/s; 3008/205332 objects degraded (1.465%); 4751/205332 objects misplaced (2.314%); 864 KiB/s, 13 objects/s recovering 2023-10-01T11:06:14.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:13 smithi078 bash[78274]: cluster 2023-10-01T11:06:12.770174+0000 mon.a (mon.0) 1882 : cluster 0 osdmap e333: 8 total, 8 up, 8 in 2023-10-01T11:06:14.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:13 smithi078 bash[78274]: cluster 2023-10-01T11:06:12.771118+0000 osd.5 (osd.5) 352 : cluster 3 failed to encode map e333 with expected crc 2023-10-01T11:06:14.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:13 smithi078 bash[78274]: cluster 2023-10-01T11:06:12.771277+0000 osd.6 (osd.6) 317 : cluster 3 failed to encode map e333 with expected crc 2023-10-01T11:06:14.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:13 smithi078 bash[78274]: cluster 2023-10-01T11:06:12.773208+0000 osd.7 (osd.7) 365 : cluster 3 failed to encode map e333 with expected crc 2023-10-01T11:06:14.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:13 smithi078 bash[78274]: cluster 2023-10-01T11:06:12.773806+0000 osd.6 (osd.6) 318 : cluster 3 failed to encode map e333 with expected crc 2023-10-01T11:06:14.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:13 smithi078 bash[78274]: cluster 2023-10-01T11:06:12.775045+0000 osd.7 (osd.7) 366 : cluster 3 failed to encode map e333 with expected crc 2023-10-01T11:06:14.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:13 smithi078 bash[78274]: cluster 2023-10-01T11:06:12.775407+0000 osd.5 (osd.5) 353 : cluster 3 failed to encode map e333 with expected crc 2023-10-01T11:06:14.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:13 smithi078 bash[78274]: cluster 2023-10-01T11:06:12.777141+0000 osd.7 (osd.7) 367 : cluster 3 failed to encode map e333 with expected crc 2023-10-01T11:06:14.124 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:14.149 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:14.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:14.151 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:06:14.252 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:14.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:14.279 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:14.279 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:06:14.380 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:14.406 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:14.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:14.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:06:14.509 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:14.536 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:14.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:14.538 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:14.639 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:14.669 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:14.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:14.672 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:06:14.773 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:14.806 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:14.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:14.808 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:06:14.909 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:14.934 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:14.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:14.937 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:14.995 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T11:06:15.038 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:15.062 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:15.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:15.065 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:06:15.166 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:15.189 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:15.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:15.191 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:06:15.292 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:15.314 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:15.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:15.316 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:06:15.417 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:15.427 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-10-01T11:06:15.427 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [ 2023-10-01T11:06:15.427 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.427 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:90900 13.7 13:e9d6bf75:::benchmark_data_smithi078_202524_object11342:head [delete] snapc 0=[] ondisk+write+known_if_redirected e330)", 2023-10-01T11:06:15.428 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:06:09.662987+0000", 2023-10-01T11:06:15.428 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 5.7585483389999998, 2023-10-01T11:06:15.428 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.7585927899999998, 2023-10-01T11:06:15.428 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:06:15.428 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "delayed", 2023-10-01T11:06:15.429 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:06:15.429 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:06:15.429 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:06:15.429 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 90900 2023-10-01T11:06:15.429 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.429 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:06:15.429 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.429 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:06:15.430 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:09.662987+0000", 2023-10-01T11:06:15.430 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:06:15.430 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.430 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.430 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:06:15.430 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:09.662987+0000", 2023-10-01T11:06:15.430 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999976 2023-10-01T11:06:15.430 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.430 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.430 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:06:15.430 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:09.662985+0000", 2023-10-01T11:06:15.431 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0905e-05 2023-10-01T11:06:15.431 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.431 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.431 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:06:15.431 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:09.662996+0000", 2023-10-01T11:06:15.431 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.9802999999999999e-05 2023-10-01T11:06:15.431 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.431 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.431 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:06:15.431 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:09.663025+0000", 2023-10-01T11:06:15.431 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.6759999999999996e-06 2023-10-01T11:06:15.431 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.431 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.432 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:06:15.432 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:09.663034+0000", 2023-10-01T11:06:15.432 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.045649153999999997 2023-10-01T11:06:15.432 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.432 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.432 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:15.432 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:09.708683+0000", 2023-10-01T11:06:15.432 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0075206229999999997 2023-10-01T11:06:15.432 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.432 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.432 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for missing object", 2023-10-01T11:06:15.432 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:09.716204+0000", 2023-10-01T11:06:15.433 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.036829852000000003 2023-10-01T11:06:15.433 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.433 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.433 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:15.433 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:09.753034+0000", 2023-10-01T11:06:15.433 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0077225810000000001 2023-10-01T11:06:15.433 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.433 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.433 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for rw locks", 2023-10-01T11:06:15.433 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:09.760756+0000", 2023-10-01T11:06:15.433 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.098309489 2023-10-01T11:06:15.433 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.433 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.434 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:15.434 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.859066+0000", 2023-10-01T11:06:15.434 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.3839999999999999e-06 2023-10-01T11:06:15.434 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.434 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.434 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:06:15.434 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.859073+0000", 2023-10-01T11:06:15.434 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.92250128600000003 2023-10-01T11:06:15.434 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.434 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.434 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:15.434 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:11.781574+0000", 2023-10-01T11:06:15.435 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0927000000000001e-05 2023-10-01T11:06:15.435 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.435 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.435 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:06:15.435 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:11.781585+0000", 2023-10-01T11:06:15.435 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.1185982069999998 2023-10-01T11:06:15.435 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:15.435 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:06:15.435 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:15.435 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.435 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.435 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:90987 13.7 13:eaa714e7:::benchmark_data_smithi078_202524_object11429:head [delete] snapc 0=[] ondisk+write+known_if_redirected e330)", 2023-10-01T11:06:15.435 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:06:10.277364+0000", 2023-10-01T11:06:15.436 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 5.1441716709999996, 2023-10-01T11:06:15.436 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.1443911980000001, 2023-10-01T11:06:15.436 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:06:15.436 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "delayed", 2023-10-01T11:06:15.436 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:06:15.436 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:06:15.436 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:06:15.436 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 90987 2023-10-01T11:06:15.436 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.436 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:06:15.436 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.436 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:06:15.437 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.277364+0000", 2023-10-01T11:06:15.437 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:06:15.437 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.437 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.437 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:06:15.437 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.277364+0000", 2023-10-01T11:06:15.437 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-10-01T11:06:15.437 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.437 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.437 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:06:15.437 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.277362+0000", 2023-10-01T11:06:15.437 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.0080000000000002e-06 2023-10-01T11:06:15.437 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.438 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.438 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:06:15.438 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.277370+0000", 2023-10-01T11:06:15.438 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.3078000000000001e-05 2023-10-01T11:06:15.438 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.438 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.438 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:06:15.438 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.277383+0000", 2023-10-01T11:06:15.438 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.4890000000000003e-06 2023-10-01T11:06:15.438 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.438 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.439 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:06:15.439 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.277389+0000", 2023-10-01T11:06:15.439 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.064535125999999998 2023-10-01T11:06:15.439 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.439 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.439 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:15.439 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.341924+0000", 2023-10-01T11:06:15.439 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0050644560000000002 2023-10-01T11:06:15.439 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.439 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.439 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for missing object", 2023-10-01T11:06:15.439 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.346988+0000", 2023-10-01T11:06:15.439 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.059518306999999999 2023-10-01T11:06:15.440 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.440 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.440 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:15.440 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.406507+0000", 2023-10-01T11:06:15.440 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0043659399999999996 2023-10-01T11:06:15.440 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.440 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.440 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for rw locks", 2023-10-01T11:06:15.440 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.410873+0000", 2023-10-01T11:06:15.440 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.44813687800000002 2023-10-01T11:06:15.440 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.440 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.441 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:15.441 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.859010+0000", 2023-10-01T11:06:15.441 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.7330000000000003e-06 2023-10-01T11:06:15.441 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.441 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.441 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:06:15.441 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.859017+0000", 2023-10-01T11:06:15.441 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.92250221399999999 2023-10-01T11:06:15.441 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.441 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.441 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:15.441 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:11.781520+0000", 2023-10-01T11:06:15.441 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0305999999999999e-05 2023-10-01T11:06:15.442 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.442 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.442 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:06:15.442 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:11.781530+0000", 2023-10-01T11:06:15.442 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.5041660430000001 2023-10-01T11:06:15.442 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:15.442 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:06:15.442 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:15.443 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.443 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.444 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:90912 13.7 13:e9834e2e:::benchmark_data_smithi078_202524_object11354:head [delete] snapc 0=[] ondisk+write+known_if_redirected e330)", 2023-10-01T11:06:15.444 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:06:09.695303+0000", 2023-10-01T11:06:15.444 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 5.7262320569999998, 2023-10-01T11:06:15.444 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.7266074009999999, 2023-10-01T11:06:15.444 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:06:15.444 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "delayed", 2023-10-01T11:06:15.444 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:06:15.444 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:06:15.444 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:06:15.444 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 90912 2023-10-01T11:06:15.444 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.445 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:06:15.445 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.445 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:06:15.445 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:09.695303+0000", 2023-10-01T11:06:15.445 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:06:15.445 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.445 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.445 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:06:15.445 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:09.695303+0000", 2023-10-01T11:06:15.445 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999976 2023-10-01T11:06:15.446 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.446 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.446 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:06:15.446 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:09.695301+0000", 2023-10-01T11:06:15.446 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0711e-05 2023-10-01T11:06:15.447 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.447 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.447 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:06:15.447 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:09.695312+0000", 2023-10-01T11:06:15.447 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.8879e-05 2023-10-01T11:06:15.448 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.448 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.448 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:06:15.448 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:09.695330+0000", 2023-10-01T11:06:15.448 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0049e-05 2023-10-01T11:06:15.448 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.449 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.449 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:06:15.449 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:09.695340+0000", 2023-10-01T11:06:15.449 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.021727949999999999 2023-10-01T11:06:15.449 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.450 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.450 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:15.450 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:09.717068+0000", 2023-10-01T11:06:15.450 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0058438049999999997 2023-10-01T11:06:15.450 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.451 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.451 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for missing object", 2023-10-01T11:06:15.451 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:09.722912+0000", 2023-10-01T11:06:15.451 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.064880487000000001 2023-10-01T11:06:15.451 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.451 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.451 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:15.452 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:09.787793+0000", 2023-10-01T11:06:15.452 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0044240130000000001 2023-10-01T11:06:15.452 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.452 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.452 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for rw locks", 2023-10-01T11:06:15.452 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:09.792217+0000", 2023-10-01T11:06:15.452 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0668942269999999 2023-10-01T11:06:15.453 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.453 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.453 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:15.453 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.859111+0000", 2023-10-01T11:06:15.453 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.3570000000000003e-06 2023-10-01T11:06:15.453 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.453 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.454 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:06:15.454 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.859117+0000", 2023-10-01T11:06:15.454 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.92250491499999998 2023-10-01T11:06:15.454 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.454 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.455 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:15.455 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:11.781622+0000", 2023-10-01T11:06:15.455 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.9560000000000003e-06 2023-10-01T11:06:15.455 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.455 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.455 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:06:15.455 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:11.781631+0000", 2023-10-01T11:06:15.456 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.0863277820000001 2023-10-01T11:06:15.456 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:15.456 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:06:15.456 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:15.456 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.456 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.456 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:90913 13.7 13:e4ae50de:::benchmark_data_smithi078_202524_object11355:head [delete] snapc 0=[] ondisk+write+known_if_redirected e330)", 2023-10-01T11:06:15.457 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:06:09.701934+0000", 2023-10-01T11:06:15.457 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 5.7196014059999998, 2023-10-01T11:06:15.457 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.7201324529999997, 2023-10-01T11:06:15.457 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:06:15.457 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "delayed", 2023-10-01T11:06:15.457 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:06:15.457 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:06:15.457 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:06:15.458 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 90913 2023-10-01T11:06:15.458 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.458 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:06:15.458 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.458 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:06:15.458 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:09.701934+0000", 2023-10-01T11:06:15.458 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:06:15.459 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.459 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.459 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:06:15.459 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:09.701934+0000", 2023-10-01T11:06:15.459 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999976 2023-10-01T11:06:15.459 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.460 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.460 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:06:15.460 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:09.701932+0000", 2023-10-01T11:06:15.460 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1892000000000001e-05 2023-10-01T11:06:15.460 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.460 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.460 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:06:15.460 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:09.701944+0000", 2023-10-01T11:06:15.460 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.8797999999999999e-05 2023-10-01T11:06:15.460 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.460 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.461 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:06:15.461 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:09.701962+0000", 2023-10-01T11:06:15.461 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.2059999999999996e-06 2023-10-01T11:06:15.461 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.461 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.461 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:06:15.461 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:09.701972+0000", 2023-10-01T11:06:15.461 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.021584782 2023-10-01T11:06:15.461 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.461 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.461 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:15.461 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:09.723556+0000", 2023-10-01T11:06:15.461 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0054940589999999999 2023-10-01T11:06:15.462 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.462 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.462 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for missing object", 2023-10-01T11:06:15.462 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:09.729050+0000", 2023-10-01T11:06:15.462 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.082833860999999995 2023-10-01T11:06:15.462 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.462 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.462 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:15.462 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:09.811884+0000", 2023-10-01T11:06:15.462 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0043112769999999996 2023-10-01T11:06:15.462 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.463 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.463 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for rw locks", 2023-10-01T11:06:15.463 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:09.816196+0000", 2023-10-01T11:06:15.463 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0429475859999999 2023-10-01T11:06:15.463 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.463 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.463 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:15.463 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.859143+0000", 2023-10-01T11:06:15.463 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.7489999999999998e-06 2023-10-01T11:06:15.463 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.463 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.463 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:06:15.464 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.859148+0000", 2023-10-01T11:06:15.464 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.92251773800000003 2023-10-01T11:06:15.464 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.464 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.464 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:15.464 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:11.781666+0000", 2023-10-01T11:06:15.464 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.7269999999999996e-06 2023-10-01T11:06:15.464 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.464 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.464 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:06:15.465 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:11.781674+0000", 2023-10-01T11:06:15.465 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.079740358 2023-10-01T11:06:15.465 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:15.465 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:06:15.465 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:15.465 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.465 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.465 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:90992 13.7 13:eeb7223a:::benchmark_data_smithi078_202524_object11434:head [delete] snapc 0=[] ondisk+write+known_if_redirected e330)", 2023-10-01T11:06:15.465 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:06:10.313237+0000", 2023-10-01T11:06:15.465 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 5.1082988790000003, 2023-10-01T11:06:15.465 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.1090740270000001, 2023-10-01T11:06:15.466 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:06:15.466 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "delayed", 2023-10-01T11:06:15.466 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:06:15.466 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:06:15.466 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:06:15.466 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 90992 2023-10-01T11:06:15.466 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.466 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:06:15.466 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.466 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:06:15.467 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.313237+0000", 2023-10-01T11:06:15.467 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:06:15.467 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.467 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.467 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:06:15.467 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.313237+0000", 2023-10-01T11:06:15.467 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999971 2023-10-01T11:06:15.467 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.467 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.467 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:06:15.468 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.313234+0000", 2023-10-01T11:06:15.468 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2835e-05 2023-10-01T11:06:15.468 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.468 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.468 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:06:15.468 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.313247+0000", 2023-10-01T11:06:15.468 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.9996e-05 2023-10-01T11:06:15.468 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.468 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.468 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:06:15.469 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.313267+0000", 2023-10-01T11:06:15.469 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.9340000000000001e-06 2023-10-01T11:06:15.469 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.469 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.469 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:06:15.469 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.313277+0000", 2023-10-01T11:06:15.469 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.034229023999999997 2023-10-01T11:06:15.469 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.469 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.469 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:15.469 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.347506+0000", 2023-10-01T11:06:15.470 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0045258340000000003 2023-10-01T11:06:15.470 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.470 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.470 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for missing object", 2023-10-01T11:06:15.470 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.352031+0000", 2023-10-01T11:06:15.470 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.083323512000000002 2023-10-01T11:06:15.470 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.470 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.470 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:15.470 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.435355+0000", 2023-10-01T11:06:15.471 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0044127469999999999 2023-10-01T11:06:15.471 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.471 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.471 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for rw locks", 2023-10-01T11:06:15.471 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.439768+0000", 2023-10-01T11:06:15.471 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.41911722600000001 2023-10-01T11:06:15.471 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.471 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.471 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:15.471 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.858885+0000", 2023-10-01T11:06:15.471 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.4449999999999996e-06 2023-10-01T11:06:15.472 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.472 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.472 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:06:15.472 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.858893+0000", 2023-10-01T11:06:15.472 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.92192855699999998 2023-10-01T11:06:15.472 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.472 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.472 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:15.472 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:11.780822+0000", 2023-10-01T11:06:15.472 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.2779999999999993e-06 2023-10-01T11:06:15.473 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.473 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.473 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:06:15.473 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:11.780830+0000", 2023-10-01T11:06:15.473 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4675936540000001 2023-10-01T11:06:15.473 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:15.473 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:06:15.473 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:15.473 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.473 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.473 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:90920 13.7 13:e1016db1:::benchmark_data_smithi078_202524_object11362:head [delete] snapc 0=[] ondisk+write+known_if_redirected e330)", 2023-10-01T11:06:15.474 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:06:09.733925+0000", 2023-10-01T11:06:15.474 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 5.68761034, 2023-10-01T11:06:15.474 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.6886093999999998, 2023-10-01T11:06:15.474 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:06:15.474 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "delayed", 2023-10-01T11:06:15.474 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:06:15.474 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:06:15.474 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:06:15.474 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 90920 2023-10-01T11:06:15.474 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.474 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:06:15.475 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.475 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:06:15.475 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:09.733925+0000", 2023-10-01T11:06:15.475 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:06:15.475 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.475 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.475 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:06:15.475 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:09.733925+0000", 2023-10-01T11:06:15.475 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-10-01T11:06:15.475 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.476 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.476 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:06:15.476 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:09.733923+0000", 2023-10-01T11:06:15.476 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.076e-05 2023-10-01T11:06:15.476 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.476 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.476 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:06:15.476 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:09.733934+0000", 2023-10-01T11:06:15.476 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.8664999999999998e-05 2023-10-01T11:06:15.476 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.476 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.477 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:06:15.477 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:09.733953+0000", 2023-10-01T11:06:15.477 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.9729999999999997e-06 2023-10-01T11:06:15.477 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.477 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.477 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:06:15.477 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:09.733961+0000", 2023-10-01T11:06:15.477 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.083536535999999995 2023-10-01T11:06:15.477 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.477 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.477 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:15.478 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:09.817497+0000", 2023-10-01T11:06:15.478 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0045493060000000004 2023-10-01T11:06:15.478 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.478 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.478 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for missing object", 2023-10-01T11:06:15.478 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:09.822047+0000", 2023-10-01T11:06:15.478 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.095099276999999996 2023-10-01T11:06:15.478 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.478 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.479 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:15.479 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:09.917146+0000", 2023-10-01T11:06:15.479 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0067624599999999997 2023-10-01T11:06:15.479 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.479 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.479 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for rw locks", 2023-10-01T11:06:15.479 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:09.923908+0000", 2023-10-01T11:06:15.479 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.935308731 2023-10-01T11:06:15.479 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.479 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.479 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:15.480 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.859217+0000", 2023-10-01T11:06:15.480 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.5739999999999997e-06 2023-10-01T11:06:15.480 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.480 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.480 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:06:15.480 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.859222+0000", 2023-10-01T11:06:15.480 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.92258064200000001 2023-10-01T11:06:15.480 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.480 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.480 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:15.480 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:11.781802+0000", 2023-10-01T11:06:15.480 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.3119999999999994e-06 2023-10-01T11:06:15.481 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.481 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.481 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:06:15.481 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:11.781811+0000", 2023-10-01T11:06:15.481 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.0478854129999999 2023-10-01T11:06:15.481 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:15.481 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:06:15.481 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:15.481 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.481 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.481 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:90930 13.7 13:e2277410:::benchmark_data_smithi078_202524_object11372:head [delete] snapc 0=[] ondisk+write+known_if_redirected e330)", 2023-10-01T11:06:15.481 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:06:09.963053+0000", 2023-10-01T11:06:15.482 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 5.4584824340000004, 2023-10-01T11:06:15.482 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.4597027760000003, 2023-10-01T11:06:15.482 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:06:15.482 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "delayed", 2023-10-01T11:06:15.482 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:06:15.482 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:06:15.482 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:06:15.482 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 90930 2023-10-01T11:06:15.482 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.482 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:06:15.482 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.483 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:06:15.483 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:09.963053+0000", 2023-10-01T11:06:15.483 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:06:15.483 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.483 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.483 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:06:15.483 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:09.963053+0000", 2023-10-01T11:06:15.483 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999976 2023-10-01T11:06:15.483 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.483 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.484 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:06:15.484 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:09.963051+0000", 2023-10-01T11:06:15.484 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1494999999999999e-05 2023-10-01T11:06:15.484 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.484 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.484 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:06:15.484 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:09.963062+0000", 2023-10-01T11:06:15.484 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.7615999999999999e-05 2023-10-01T11:06:15.484 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.484 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.484 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:06:15.485 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:09.963080+0000", 2023-10-01T11:06:15.485 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.3770000000000002e-06 2023-10-01T11:06:15.485 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.485 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.485 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:06:15.485 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:09.963089+0000", 2023-10-01T11:06:15.485 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.038187750999999999 2023-10-01T11:06:15.485 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.485 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.485 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:15.486 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.001277+0000", 2023-10-01T11:06:15.486 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0084801770000000002 2023-10-01T11:06:15.486 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.486 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.486 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for missing object", 2023-10-01T11:06:15.486 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.009757+0000", 2023-10-01T11:06:15.486 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.051020342000000003 2023-10-01T11:06:15.486 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.486 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.486 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:15.487 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.060777+0000", 2023-10-01T11:06:15.487 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0091223459999999999 2023-10-01T11:06:15.487 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.487 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.487 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for rw locks", 2023-10-01T11:06:15.487 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.069900+0000", 2023-10-01T11:06:15.487 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.78929818299999999 2023-10-01T11:06:15.487 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.487 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.487 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:15.488 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.859198+0000", 2023-10-01T11:06:15.488 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.3250000000000001e-06 2023-10-01T11:06:15.488 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.488 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.488 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:06:15.488 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.859202+0000", 2023-10-01T11:06:15.488 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.92254991799999997 2023-10-01T11:06:15.488 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.488 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.488 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:15.488 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:11.781752+0000", 2023-10-01T11:06:15.489 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.123e-06 2023-10-01T11:06:15.489 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.489 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.489 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:06:15.489 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:11.781760+0000", 2023-10-01T11:06:15.489 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.8187073580000002 2023-10-01T11:06:15.489 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:15.489 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:06:15.489 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:15.489 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.489 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.490 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:90931 13.7 13:e3a5e152:::benchmark_data_smithi078_202524_object11373:head [delete] snapc 0=[] ondisk+write+known_if_redirected e330)", 2023-10-01T11:06:15.490 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:06:09.966789+0000", 2023-10-01T11:06:15.490 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 5.4547467169999999, 2023-10-01T11:06:15.490 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.4562018769999998, 2023-10-01T11:06:15.490 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:06:15.490 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "delayed", 2023-10-01T11:06:15.490 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:06:15.490 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:06:15.490 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:06:15.490 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 90931 2023-10-01T11:06:15.490 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.491 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:06:15.491 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.491 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:06:15.491 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:09.966789+0000", 2023-10-01T11:06:15.491 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:06:15.491 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.491 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.491 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:06:15.491 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:09.966789+0000", 2023-10-01T11:06:15.491 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-10-01T11:06:15.492 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.492 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.492 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:06:15.492 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:09.966787+0000", 2023-10-01T11:06:15.492 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0352e-05 2023-10-01T11:06:15.492 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.492 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.492 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:06:15.493 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:09.966797+0000", 2023-10-01T11:06:15.493 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.5113000000000001e-05 2023-10-01T11:06:15.493 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.493 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.493 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:06:15.493 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:09.966812+0000", 2023-10-01T11:06:15.494 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.7150000000000001e-06 2023-10-01T11:06:15.494 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.494 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.494 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:06:15.494 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:09.966819+0000", 2023-10-01T11:06:15.494 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.10356421 2023-10-01T11:06:15.494 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.494 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.494 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:15.494 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.070383+0000", 2023-10-01T11:06:15.495 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0048671749999999996 2023-10-01T11:06:15.495 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.495 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.495 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for missing object", 2023-10-01T11:06:15.495 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.075251+0000", 2023-10-01T11:06:15.495 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.103088658 2023-10-01T11:06:15.495 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.495 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.495 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:15.495 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.178339+0000", 2023-10-01T11:06:15.495 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0077670500000000002 2023-10-01T11:06:15.496 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.496 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.497 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for rw locks", 2023-10-01T11:06:15.497 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.186106+0000", 2023-10-01T11:06:15.497 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.67306498000000003 2023-10-01T11:06:15.497 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.497 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.497 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:15.497 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.859171+0000", 2023-10-01T11:06:15.497 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.4830000000000001e-06 2023-10-01T11:06:15.497 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.497 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.497 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:06:15.498 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.859176+0000", 2023-10-01T11:06:15.498 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.92253244999999995 2023-10-01T11:06:15.498 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.498 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.498 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:15.498 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:11.781708+0000", 2023-10-01T11:06:15.498 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.9709999999999993e-06 2023-10-01T11:06:15.498 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.498 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.498 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:06:15.499 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:11.781717+0000", 2023-10-01T11:06:15.499 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.814928404 2023-10-01T11:06:15.499 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:15.499 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:06:15.499 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:15.499 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.499 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.499 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:91044 13.7 13:ee7f7252:::benchmark_data_smithi078_202524_object11486:head [delete] snapc 0=[] ondisk+write+known_if_redirected e330)", 2023-10-01T11:06:15.500 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:06:10.658557+0000", 2023-10-01T11:06:15.500 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 4.7629786349999996, 2023-10-01T11:06:15.500 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.7646056750000003, 2023-10-01T11:06:15.500 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:06:15.500 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "delayed", 2023-10-01T11:06:15.500 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:06:15.500 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:06:15.501 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:06:15.501 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 91044 2023-10-01T11:06:15.501 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.501 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:06:15.501 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.501 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:06:15.501 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.658557+0000", 2023-10-01T11:06:15.501 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:06:15.501 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.501 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.501 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:06:15.502 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.658557+0000", 2023-10-01T11:06:15.502 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-10-01T11:06:15.502 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.502 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.502 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:06:15.502 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.658555+0000", 2023-10-01T11:06:15.502 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1123e-05 2023-10-01T11:06:15.502 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.502 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.502 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:06:15.502 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.658566+0000", 2023-10-01T11:06:15.503 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.8476000000000001e-05 2023-10-01T11:06:15.503 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.503 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.503 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:06:15.503 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.658584+0000", 2023-10-01T11:06:15.503 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.1339999999999993e-06 2023-10-01T11:06:15.503 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.503 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.503 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:06:15.503 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.658594+0000", 2023-10-01T11:06:15.503 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.044092635999999998 2023-10-01T11:06:15.503 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.503 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.504 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:15.504 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.702686+0000", 2023-10-01T11:06:15.504 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0052976400000000002 2023-10-01T11:06:15.504 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.504 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.504 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for missing object", 2023-10-01T11:06:15.504 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.707984+0000", 2023-10-01T11:06:15.504 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.048686976 2023-10-01T11:06:15.504 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.504 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.504 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:15.504 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.756671+0000", 2023-10-01T11:06:15.505 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.006272003 2023-10-01T11:06:15.505 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.505 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.505 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for rw locks", 2023-10-01T11:06:15.505 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.762943+0000", 2023-10-01T11:06:15.505 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.095982534999999994 2023-10-01T11:06:15.505 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.505 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.505 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:15.505 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.858925+0000", 2023-10-01T11:06:15.505 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.6060000000000003e-06 2023-10-01T11:06:15.506 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.506 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.506 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:06:15.506 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.858934+0000", 2023-10-01T11:06:15.506 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.92192976000000004 2023-10-01T11:06:15.506 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.506 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.506 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:15.506 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:11.780864+0000", 2023-10-01T11:06:15.506 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.2939999999999994e-06 2023-10-01T11:06:15.506 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.507 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.507 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:06:15.507 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:11.780873+0000", 2023-10-01T11:06:15.507 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.122316163 2023-10-01T11:06:15.507 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:15.507 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:06:15.507 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:15.507 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.507 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.507 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:90866 13.7 13:eb030853:::benchmark_data_smithi078_202524_object11308:head [delete] snapc 0=[] ondisk+write+known_if_redirected e330)", 2023-10-01T11:06:15.507 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:06:09.467492+0000", 2023-10-01T11:06:15.507 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 5.954043124, 2023-10-01T11:06:15.508 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.9558457479999998, 2023-10-01T11:06:15.508 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:06:15.508 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "delayed", 2023-10-01T11:06:15.508 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:06:15.508 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:06:15.508 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:06:15.508 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 90866 2023-10-01T11:06:15.508 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.508 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:06:15.508 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.508 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:06:15.509 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:09.467492+0000", 2023-10-01T11:06:15.509 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:06:15.509 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.509 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.509 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:06:15.509 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:09.467492+0000", 2023-10-01T11:06:15.509 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999971 2023-10-01T11:06:15.509 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.509 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.509 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:06:15.509 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:09.467489+0000", 2023-10-01T11:06:15.509 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.3587000000000001e-05 2023-10-01T11:06:15.510 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.510 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.510 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:06:15.510 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:09.467503+0000", 2023-10-01T11:06:15.510 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.262e-05 2023-10-01T11:06:15.510 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.510 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.510 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:06:15.511 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:09.467526+0000", 2023-10-01T11:06:15.511 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.6067e-05 2023-10-01T11:06:15.511 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.511 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.511 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:06:15.511 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:09.467542+0000", 2023-10-01T11:06:15.511 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.058417771 2023-10-01T11:06:15.511 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.511 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.511 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:15.512 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:09.525959+0000", 2023-10-01T11:06:15.512 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0044004509999999997 2023-10-01T11:06:15.512 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.512 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.512 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for missing object", 2023-10-01T11:06:15.512 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:09.530360+0000", 2023-10-01T11:06:15.512 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.12552844299999999 2023-10-01T11:06:15.512 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.512 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.512 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:15.512 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:09.655888+0000", 2023-10-01T11:06:15.513 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0075971390000000001 2023-10-01T11:06:15.513 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.513 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.513 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for rw locks", 2023-10-01T11:06:15.513 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:09.663486+0000", 2023-10-01T11:06:15.513 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1290324709999999 2023-10-01T11:06:15.513 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.513 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.513 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:15.513 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.792518+0000", 2023-10-01T11:06:15.513 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0070787990000000002 2023-10-01T11:06:15.513 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.514 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.514 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:06:15.514 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.799597+0000", 2023-10-01T11:06:15.514 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0078850670000000008 2023-10-01T11:06:15.514 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.514 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.514 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 4,5,6", 2023-10-01T11:06:15.514 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.807482+0000", 2023-10-01T11:06:15.514 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.051303645000000002 2023-10-01T11:06:15.514 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.514 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.514 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:15.515 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.858785+0000", 2023-10-01T11:06:15.515 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.5227e-05 2023-10-01T11:06:15.515 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.515 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.515 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:06:15.515 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.858801+0000", 2023-10-01T11:06:15.515 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.92189180699999995 2023-10-01T11:06:15.515 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.515 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.515 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:15.515 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:11.780693+0000", 2023-10-01T11:06:15.516 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.5338000000000001e-05 2023-10-01T11:06:15.516 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.516 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.516 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:06:15.516 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:11.780708+0000", 2023-10-01T11:06:15.516 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.313215552 2023-10-01T11:06:15.516 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:15.516 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:06:15.516 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:15.516 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.516 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.516 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:91096 13.7 13.1d5b7277 (undecoded) ondisk+write+known_if_redirected e330)", 2023-10-01T11:06:15.517 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:06:10.745780+0000", 2023-10-01T11:06:15.517 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 4.6757555880000004, 2023-10-01T11:06:15.517 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.6777263170000003, 2023-10-01T11:06:15.517 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:06:15.517 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "delayed", 2023-10-01T11:06:15.517 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:06:15.517 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:06:15.517 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:06:15.517 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 91096 2023-10-01T11:06:15.517 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.517 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:06:15.518 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.518 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:06:15.518 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.745780+0000", 2023-10-01T11:06:15.518 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:06:15.518 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.518 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.518 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:06:15.518 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.745780+0000", 2023-10-01T11:06:15.518 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-10-01T11:06:15.519 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.519 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.519 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:06:15.519 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.745778+0000", 2023-10-01T11:06:15.519 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.7270000000000002e-06 2023-10-01T11:06:15.519 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.519 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.519 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:06:15.519 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.745788+0000", 2023-10-01T11:06:15.519 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.859e-05 2023-10-01T11:06:15.519 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.519 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.520 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:06:15.520 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.745806+0000", 2023-10-01T11:06:15.520 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.7879999999999995e-06 2023-10-01T11:06:15.520 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.520 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.520 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:06:15.520 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.745815+0000", 2023-10-01T11:06:15.520 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.11424672600000001 2023-10-01T11:06:15.520 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.520 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.520 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:15.521 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.860062+0000", 2023-10-01T11:06:15.521 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.7209999999999999e-06 2023-10-01T11:06:15.521 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.521 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.521 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:06:15.521 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.860067+0000", 2023-10-01T11:06:15.521 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.92177177600000004 2023-10-01T11:06:15.522 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.522 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.522 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:15.522 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:11.781839+0000", 2023-10-01T11:06:15.522 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.5329999999999994e-06 2023-10-01T11:06:15.522 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.522 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.522 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:06:15.522 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:11.781848+0000", 2023-10-01T11:06:15.522 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0360679269999999 2023-10-01T11:06:15.522 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:15.522 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:06:15.523 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:15.523 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.523 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.523 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:91103 13.7 13.77173777 (undecoded) ondisk+write+known_if_redirected e331)", 2023-10-01T11:06:15.523 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:06:10.787764+0000", 2023-10-01T11:06:15.523 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 4.6337715609999997, 2023-10-01T11:06:15.523 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.6358621490000003, 2023-10-01T11:06:15.523 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:06:15.523 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "delayed", 2023-10-01T11:06:15.523 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:06:15.524 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:06:15.524 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:06:15.524 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 91103 2023-10-01T11:06:15.524 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.524 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:06:15.524 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.524 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:06:15.524 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.787764+0000", 2023-10-01T11:06:15.524 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:06:15.524 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.524 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.524 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:06:15.525 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.787764+0000", 2023-10-01T11:06:15.525 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999976 2023-10-01T11:06:15.525 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.525 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.525 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:06:15.525 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.787762+0000", 2023-10-01T11:06:15.525 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.7940000000000008e-06 2023-10-01T11:06:15.525 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.525 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.525 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:06:15.525 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.787770+0000", 2023-10-01T11:06:15.526 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.5276000000000001e-05 2023-10-01T11:06:15.526 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.526 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.526 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:06:15.526 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.787786+0000", 2023-10-01T11:06:15.526 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.3279999999999995e-06 2023-10-01T11:06:15.526 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.526 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.526 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:06:15.526 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.787795+0000", 2023-10-01T11:06:15.526 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.072294944 2023-10-01T11:06:15.527 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.527 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.527 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:15.527 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.860090+0000", 2023-10-01T11:06:15.527 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.9190000000000004e-06 2023-10-01T11:06:15.527 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.527 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.527 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:06:15.527 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.860096+0000", 2023-10-01T11:06:15.527 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.92177759299999995 2023-10-01T11:06:15.527 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.528 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.528 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:15.528 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:11.781873+0000", 2023-10-01T11:06:15.528 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.524e-06 2023-10-01T11:06:15.528 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.528 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.528 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:06:15.528 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:11.781882+0000", 2023-10-01T11:06:15.528 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.99411812399999999 2023-10-01T11:06:15.528 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:15.528 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:06:15.529 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:15.529 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.529 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.529 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:91108 13.7 13.b77cc687 (undecoded) ondisk+write+known_if_redirected e331)", 2023-10-01T11:06:15.529 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:06:10.825479+0000", 2023-10-01T11:06:15.529 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 4.5960561139999996, 2023-10-01T11:06:15.529 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.5982568549999998, 2023-10-01T11:06:15.529 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:06:15.529 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "delayed", 2023-10-01T11:06:15.529 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:06:15.529 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:06:15.529 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:06:15.530 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 91108 2023-10-01T11:06:15.530 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.530 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:06:15.530 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.530 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:06:15.530 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.825479+0000", 2023-10-01T11:06:15.530 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:06:15.530 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.530 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.530 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:06:15.530 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.825479+0000", 2023-10-01T11:06:15.531 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-10-01T11:06:15.531 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.531 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.531 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:06:15.532 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.825477+0000", 2023-10-01T11:06:15.532 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.8980000000000008e-06 2023-10-01T11:06:15.533 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.533 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.533 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:06:15.533 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.825487+0000", 2023-10-01T11:06:15.533 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4131000000000001e-05 2023-10-01T11:06:15.533 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.533 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.533 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:06:15.533 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.825501+0000", 2023-10-01T11:06:15.533 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.0619999999999997e-06 2023-10-01T11:06:15.533 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.534 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.534 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:06:15.534 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.825507+0000", 2023-10-01T11:06:15.534 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.034612048999999999 2023-10-01T11:06:15.534 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.534 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.534 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:15.534 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.860120+0000", 2023-10-01T11:06:15.534 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.5140000000000003e-06 2023-10-01T11:06:15.534 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.534 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.534 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:06:15.535 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.860124+0000", 2023-10-01T11:06:15.535 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.92217301600000001 2023-10-01T11:06:15.535 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.535 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.535 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:15.535 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:11.782297+0000", 2023-10-01T11:06:15.535 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1712999999999999e-05 2023-10-01T11:06:15.535 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.535 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.535 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:06:15.535 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:11.782309+0000", 2023-10-01T11:06:15.535 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.95682943300000001 2023-10-01T11:06:15.536 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:15.536 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:06:15.536 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:15.536 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.536 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.536 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:90973 13.7 13:ec26e768:::benchmark_data_smithi078_202524_object11415:head [delete] snapc 0=[] ondisk+write+known_if_redirected e330)", 2023-10-01T11:06:15.536 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:06:10.222614+0000", 2023-10-01T11:06:15.536 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 5.1989209890000003, 2023-10-01T11:06:15.536 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.2012375430000004, 2023-10-01T11:06:15.537 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:06:15.537 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "delayed", 2023-10-01T11:06:15.537 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:06:15.537 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:06:15.537 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:06:15.537 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 90973 2023-10-01T11:06:15.537 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.537 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:06:15.537 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.537 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:06:15.537 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.222614+0000", 2023-10-01T11:06:15.537 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:06:15.538 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.538 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.538 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:06:15.538 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.222614+0000", 2023-10-01T11:06:15.538 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-10-01T11:06:15.538 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.538 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.538 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:06:15.538 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.222613+0000", 2023-10-01T11:06:15.538 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.995e-06 2023-10-01T11:06:15.538 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.539 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.539 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:06:15.539 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.222623+0000", 2023-10-01T11:06:15.539 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.588e-05 2023-10-01T11:06:15.539 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.539 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.539 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:06:15.539 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.222639+0000", 2023-10-01T11:06:15.539 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.2089999999999999e-06 2023-10-01T11:06:15.539 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.539 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.539 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:06:15.540 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.222646+0000", 2023-10-01T11:06:15.540 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.055076323000000003 2023-10-01T11:06:15.540 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.540 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.540 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:15.540 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.277722+0000", 2023-10-01T11:06:15.540 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0051028330000000002 2023-10-01T11:06:15.540 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.540 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.540 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for missing object", 2023-10-01T11:06:15.540 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.282825+0000", 2023-10-01T11:06:15.540 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.044590273999999999 2023-10-01T11:06:15.541 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.541 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.541 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:15.541 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.327415+0000", 2023-10-01T11:06:15.541 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0080262830000000004 2023-10-01T11:06:15.541 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.541 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.541 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for rw locks", 2023-10-01T11:06:15.541 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.335442+0000", 2023-10-01T11:06:15.541 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.52352330300000005 2023-10-01T11:06:15.541 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.542 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.542 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:15.542 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.858965+0000", 2023-10-01T11:06:15.542 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.7600000000000008e-06 2023-10-01T11:06:15.542 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.542 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.542 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:06:15.542 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.858974+0000", 2023-10-01T11:06:15.542 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.92248672700000001 2023-10-01T11:06:15.542 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.542 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.543 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:15.543 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:11.781460+0000", 2023-10-01T11:06:15.543 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2374e-05 2023-10-01T11:06:15.543 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.543 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.543 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:06:15.543 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:11.781473+0000", 2023-10-01T11:06:15.543 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.5588583520000001 2023-10-01T11:06:15.543 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:15.543 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:06:15.543 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:15.544 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.544 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.544 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:90981 13.7 13:eefc6230:::benchmark_data_smithi078_202524_object11423:head [delete] snapc 0=[] ondisk+write+known_if_redirected e330)", 2023-10-01T11:06:15.544 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:06:10.244018+0000", 2023-10-01T11:06:15.544 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 5.1775177430000001, 2023-10-01T11:06:15.544 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.1800036760000001, 2023-10-01T11:06:15.544 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:06:15.544 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "delayed", 2023-10-01T11:06:15.544 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:06:15.544 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:06:15.544 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:06:15.545 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 90981 2023-10-01T11:06:15.545 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.545 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:06:15.545 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.545 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:06:15.545 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.244018+0000", 2023-10-01T11:06:15.545 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:06:15.545 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.545 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.545 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:06:15.545 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.244018+0000", 2023-10-01T11:06:15.545 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999971 2023-10-01T11:06:15.545 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.546 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.546 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:06:15.546 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.244015+0000", 2023-10-01T11:06:15.546 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4987000000000001e-05 2023-10-01T11:06:15.546 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.546 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.546 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:06:15.546 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.244030+0000", 2023-10-01T11:06:15.546 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.3081000000000001e-05 2023-10-01T11:06:15.546 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.546 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.547 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:06:15.547 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.244053+0000", 2023-10-01T11:06:15.547 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2699e-05 2023-10-01T11:06:15.547 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.547 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.547 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:06:15.547 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.244066+0000", 2023-10-01T11:06:15.547 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.091937464999999996 2023-10-01T11:06:15.547 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.547 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.547 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:15.548 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.336003+0000", 2023-10-01T11:06:15.548 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0053390499999999997 2023-10-01T11:06:15.548 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.548 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.548 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for missing object", 2023-10-01T11:06:15.548 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.341342+0000", 2023-10-01T11:06:15.548 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.030782943 2023-10-01T11:06:15.548 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.548 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.548 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:15.548 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.372125+0000", 2023-10-01T11:06:15.549 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0076532789999999998 2023-10-01T11:06:15.549 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.549 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.549 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for rw locks", 2023-10-01T11:06:15.549 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.379778+0000", 2023-10-01T11:06:15.549 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.479064394 2023-10-01T11:06:15.549 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.549 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.549 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:15.549 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.858843+0000", 2023-10-01T11:06:15.549 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.4190000000000002e-06 2023-10-01T11:06:15.549 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.550 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.550 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:06:15.550 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:10.858851+0000", 2023-10-01T11:06:15.550 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.92191364799999997 2023-10-01T11:06:15.550 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.550 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.550 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:06:15.550 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:11.780765+0000", 2023-10-01T11:06:15.550 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.9389999999999992e-06 2023-10-01T11:06:15.550 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:06:15.550 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:06:15.551 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:06:15.551 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:06:11.780775+0000", 2023-10-01T11:06:15.551 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.5367571899999999 2023-10-01T11:06:15.551 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:15.551 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:06:15.551 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:15.551 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:06:15.551 INFO:teuthology.orchestra.run.smithi173.stdout: ], 2023-10-01T11:06:15.551 INFO:teuthology.orchestra.run.smithi173.stdout: "num_ops": 15 2023-10-01T11:06:15.551 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-10-01T11:06:15.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:15.553 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:15.553 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:06:15.654 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:15.682 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:15.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:15.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:06:15.786 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:15.809 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:15.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:15.811 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:06:15.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:15 smithi173 bash[58816]: cluster 2023-10-01T11:06:14.300039+0000 mgr.y (mgr.25072) 1467 : cluster 0 pgmap v1563: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 161 op/s; 10698/204324 objects degraded (5.236%); 4770/204324 objects misplaced (2.335%); 3.7 MiB/s, 59 objects/s recovering 2023-10-01T11:06:15.912 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:15.933 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:15.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:15.936 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:16.036 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:16.049 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:15 smithi078 bash[78274]: cluster 2023-10-01T11:06:14.300039+0000 mgr.y (mgr.25072) 1467 : cluster 0 pgmap v1563: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 161 op/s; 10698/204324 objects degraded (5.236%); 4770/204324 objects misplaced (2.335%); 3.7 MiB/s, 59 objects/s recovering 2023-10-01T11:06:16.049 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:15 smithi078 bash[75299]: cluster 2023-10-01T11:06:14.300039+0000 mgr.y (mgr.25072) 1467 : cluster 0 pgmap v1563: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 161 op/s; 10698/204324 objects degraded (5.236%); 4770/204324 objects misplaced (2.335%); 3.7 MiB/s, 59 objects/s recovering 2023-10-01T11:06:16.068 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:16.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:16.071 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:06:16.172 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:16.198 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:16.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:16.200 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:06:16.301 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:16.325 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:16.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:16.328 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:06:16.429 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:16.455 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:16.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:16.459 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:06:16.559 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:16.584 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:16.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:16.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:06:16.688 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:16.719 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:16.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:16.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:06:16.775 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-01T11:06:16.822 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:16.849 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:16.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:16.854 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:06:16.955 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:16.977 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:16.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:16.979 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:06:17.080 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:17.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:16 smithi078 bash[75299]: cluster 2023-10-01T11:06:15.785109+0000 mon.a (mon.0) 1883 : cluster 0 osdmap e334: 8 total, 8 up, 8 in 2023-10-01T11:06:17.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:16 smithi078 bash[75299]: cluster 2023-10-01T11:06:15.785992+0000 osd.6 (osd.6) 319 : cluster 3 failed to encode map e334 with expected crc 2023-10-01T11:06:17.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:16 smithi078 bash[75299]: cluster 2023-10-01T11:06:15.788179+0000 osd.7 (osd.7) 368 : cluster 3 failed to encode map e334 with expected crc 2023-10-01T11:06:17.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:16 smithi078 bash[75299]: cluster 2023-10-01T11:06:15.790842+0000 osd.6 (osd.6) 320 : cluster 3 failed to encode map e334 with expected crc 2023-10-01T11:06:17.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:16 smithi078 bash[75299]: cluster 2023-10-01T11:06:15.860892+0000 osd.5 (osd.5) 354 : cluster 3 failed to encode map e334 with expected crc 2023-10-01T11:06:17.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:16 smithi078 bash[78274]: cluster 2023-10-01T11:06:15.785109+0000 mon.a (mon.0) 1883 : cluster 0 osdmap e334: 8 total, 8 up, 8 in 2023-10-01T11:06:17.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:16 smithi078 bash[78274]: cluster 2023-10-01T11:06:15.785992+0000 osd.6 (osd.6) 319 : cluster 3 failed to encode map e334 with expected crc 2023-10-01T11:06:17.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:16 smithi078 bash[78274]: cluster 2023-10-01T11:06:15.788179+0000 osd.7 (osd.7) 368 : cluster 3 failed to encode map e334 with expected crc 2023-10-01T11:06:17.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:16 smithi078 bash[78274]: cluster 2023-10-01T11:06:15.790842+0000 osd.6 (osd.6) 320 : cluster 3 failed to encode map e334 with expected crc 2023-10-01T11:06:17.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:16 smithi078 bash[78274]: cluster 2023-10-01T11:06:15.860892+0000 osd.5 (osd.5) 354 : cluster 3 failed to encode map e334 with expected crc 2023-10-01T11:06:17.091 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:16 smithi173 bash[58816]: cluster 2023-10-01T11:06:15.785109+0000 mon.a (mon.0) 1883 : cluster 0 osdmap e334: 8 total, 8 up, 8 in 2023-10-01T11:06:17.091 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:16 smithi173 bash[58816]: cluster 2023-10-01T11:06:15.785992+0000 osd.6 (osd.6) 319 : cluster 3 failed to encode map e334 with expected crc 2023-10-01T11:06:17.091 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:16 smithi173 bash[58816]: cluster 2023-10-01T11:06:15.788179+0000 osd.7 (osd.7) 368 : cluster 3 failed to encode map e334 with expected crc 2023-10-01T11:06:17.092 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:16 smithi173 bash[58816]: cluster 2023-10-01T11:06:15.790842+0000 osd.6 (osd.6) 320 : cluster 3 failed to encode map e334 with expected crc 2023-10-01T11:06:17.092 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:16 smithi173 bash[58816]: cluster 2023-10-01T11:06:15.860892+0000 osd.5 (osd.5) 354 : cluster 3 failed to encode map e334 with expected crc 2023-10-01T11:06:17.105 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:17.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:17.108 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:06:17.209 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:17.242 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:17.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:17.245 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:06:17.346 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:17.369 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:17.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:17.371 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:17.472 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:17.496 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:17.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:17.499 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:06:17.600 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:17.626 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:17.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:17.629 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:17.730 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:17.758 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:17.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:17.760 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:06:17.861 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:17.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:17 smithi173 bash[58816]: cluster 2023-10-01T11:06:16.300658+0000 mgr.y (mgr.25072) 1468 : cluster 0 pgmap v1565: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 71 op/s; 10698/204138 objects degraded (5.241%); 8973/204138 objects misplaced (4.396%); 4.3 MiB/s, 71 objects/s recovering 2023-10-01T11:06:17.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:17 smithi173 bash[58816]: cluster 2023-10-01T11:06:16.801951+0000 osd.6 (osd.6) 321 : cluster 3 failed to encode map e335 with expected crc 2023-10-01T11:06:17.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:17 smithi173 bash[58816]: cluster 2023-10-01T11:06:16.804237+0000 osd.6 (osd.6) 322 : cluster 3 failed to encode map e335 with expected crc 2023-10-01T11:06:17.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:17 smithi173 bash[58816]: cluster 2023-10-01T11:06:16.806116+0000 mon.a (mon.0) 1884 : cluster 0 osdmap e335: 8 total, 8 up, 8 in 2023-10-01T11:06:17.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:17 smithi173 bash[58816]: cluster 2023-10-01T11:06:16.807180+0000 osd.6 (osd.6) 323 : cluster 3 failed to encode map e335 with expected crc 2023-10-01T11:06:17.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:17 smithi173 bash[58816]: cluster 2023-10-01T11:06:16.807427+0000 osd.7 (osd.7) 369 : cluster 3 failed to encode map e335 with expected crc 2023-10-01T11:06:17.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:17 smithi173 bash[58816]: audit 2023-10-01T11:06:17.466499+0000 mon.a (mon.0) 1885 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:06:17.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:17 smithi173 bash[58816]: audit 2023-10-01T11:06:17.468132+0000 mon.a (mon.0) 1886 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:06:17.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:17 smithi173 bash[58816]: cluster 2023-10-01T11:06:17.743536+0000 mon.a (mon.0) 1887 : cluster 3 Health check update: Degraded data redundancy: 10698/204138 objects degraded (5.241%), 4 pgs degraded (PG_DEGRADED) 2023-10-01T11:06:17.888 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:17.888 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T11:06:17.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:17.891 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:17.912 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T11:06:17.933 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T11:06:17.958 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T11:06:17.978 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T11:06:17.978 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T11:06:17.979 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2023-10-01T11:06:17.979 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-01T11:06:17.979 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-01T11:06:17.979 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 config set filestore_inject_stall 3 2023-10-01T11:06:17.983 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T11:06:17.992 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:18.003 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:17 smithi078 bash[78274]: cluster 2023-10-01T11:06:16.300658+0000 mgr.y (mgr.25072) 1468 : cluster 0 pgmap v1565: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 71 op/s; 10698/204138 objects degraded (5.241%); 8973/204138 objects misplaced (4.396%); 4.3 MiB/s, 71 objects/s recovering 2023-10-01T11:06:18.003 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:17 smithi078 bash[78274]: cluster 2023-10-01T11:06:16.801951+0000 osd.6 (osd.6) 321 : cluster 3 failed to encode map e335 with expected crc 2023-10-01T11:06:18.003 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:17 smithi078 bash[78274]: cluster 2023-10-01T11:06:16.804237+0000 osd.6 (osd.6) 322 : cluster 3 failed to encode map e335 with expected crc 2023-10-01T11:06:18.004 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:17 smithi078 bash[78274]: cluster 2023-10-01T11:06:16.806116+0000 mon.a (mon.0) 1884 : cluster 0 osdmap e335: 8 total, 8 up, 8 in 2023-10-01T11:06:18.004 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:17 smithi078 bash[78274]: cluster 2023-10-01T11:06:16.807180+0000 osd.6 (osd.6) 323 : cluster 3 failed to encode map e335 with expected crc 2023-10-01T11:06:18.004 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:17 smithi078 bash[78274]: cluster 2023-10-01T11:06:16.807427+0000 osd.7 (osd.7) 369 : cluster 3 failed to encode map e335 with expected crc 2023-10-01T11:06:18.004 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:17 smithi078 bash[78274]: audit 2023-10-01T11:06:17.466499+0000 mon.a (mon.0) 1885 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:06:18.004 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:17 smithi078 bash[78274]: audit 2023-10-01T11:06:17.468132+0000 mon.a (mon.0) 1886 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:06:18.004 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:17 smithi078 bash[78274]: cluster 2023-10-01T11:06:17.743536+0000 mon.a (mon.0) 1887 : cluster 3 Health check update: Degraded data redundancy: 10698/204138 objects degraded (5.241%), 4 pgs degraded (PG_DEGRADED) 2023-10-01T11:06:18.004 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:17 smithi078 bash[75299]: cluster 2023-10-01T11:06:16.300658+0000 mgr.y (mgr.25072) 1468 : cluster 0 pgmap v1565: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 71 op/s; 10698/204138 objects degraded (5.241%); 8973/204138 objects misplaced (4.396%); 4.3 MiB/s, 71 objects/s recovering 2023-10-01T11:06:18.004 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:17 smithi078 bash[75299]: cluster 2023-10-01T11:06:16.801951+0000 osd.6 (osd.6) 321 : cluster 3 failed to encode map e335 with expected crc 2023-10-01T11:06:18.004 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:17 smithi078 bash[75299]: cluster 2023-10-01T11:06:16.804237+0000 osd.6 (osd.6) 322 : cluster 3 failed to encode map e335 with expected crc 2023-10-01T11:06:18.004 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:17 smithi078 bash[75299]: cluster 2023-10-01T11:06:16.806116+0000 mon.a (mon.0) 1884 : cluster 0 osdmap e335: 8 total, 8 up, 8 in 2023-10-01T11:06:18.005 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:17 smithi078 bash[75299]: cluster 2023-10-01T11:06:16.807180+0000 osd.6 (osd.6) 323 : cluster 3 failed to encode map e335 with expected crc 2023-10-01T11:06:18.005 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:17 smithi078 bash[75299]: cluster 2023-10-01T11:06:16.807427+0000 osd.7 (osd.7) 369 : cluster 3 failed to encode map e335 with expected crc 2023-10-01T11:06:18.005 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:17 smithi078 bash[75299]: audit 2023-10-01T11:06:17.466499+0000 mon.a (mon.0) 1885 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:06:18.005 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:17 smithi078 bash[75299]: audit 2023-10-01T11:06:17.468132+0000 mon.a (mon.0) 1886 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:06:18.005 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:17 smithi078 bash[75299]: cluster 2023-10-01T11:06:17.743536+0000 mon.a (mon.0) 1887 : cluster 3 Health check update: Degraded data redundancy: 10698/204138 objects degraded (5.241%), 4 pgs degraded (PG_DEGRADED) 2023-10-01T11:06:18.012 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T11:06:18.022 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:18.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:18.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:06:18.037 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T11:06:18.063 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T11:06:18.066 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T11:06:18.066 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T11:06:18.066 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T11:06:18.066 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T11:06:18.066 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T11:06:18.066 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T11:06:18.066 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T11:06:18.066 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T11:06:18.126 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:18.147 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:18.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:18.150 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:18.251 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:18.273 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:18.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:18.275 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:06:18.376 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:18.400 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:18.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:18.406 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:06:18.507 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:18.532 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:18.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:18.535 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:18.636 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:18.659 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:18.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:18.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:06:18.762 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:18.782 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:18.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:18.784 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:18.885 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:18.898 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:18 smithi078 bash[75299]: cluster 2023-10-01T11:06:17.221410+0000 osd.5 (osd.5) 355 : cluster 3 failed to encode map e335 with expected crc 2023-10-01T11:06:18.898 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:18 smithi078 bash[75299]: audit 2023-10-01T11:06:17.861308+0000 mon.c (mon.2) 223 : audit 0 from='client.? 172.21.15.78:0/746349354' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:06:18.898 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:18 smithi078 bash[78274]: cluster 2023-10-01T11:06:17.221410+0000 osd.5 (osd.5) 355 : cluster 3 failed to encode map e335 with expected crc 2023-10-01T11:06:18.898 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:18 smithi078 bash[78274]: audit 2023-10-01T11:06:17.861308+0000 mon.c (mon.2) 223 : audit 0 from='client.? 172.21.15.78:0/746349354' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:06:18.913 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:18.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:18.915 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:06:19.016 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:19.051 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:19.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:19.055 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:06:19.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:18 smithi173 bash[58816]: cluster 2023-10-01T11:06:17.221410+0000 osd.5 (osd.5) 355 : cluster 3 failed to encode map e335 with expected crc 2023-10-01T11:06:19.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:18 smithi173 bash[58816]: audit 2023-10-01T11:06:17.861308+0000 mon.c (mon.2) 223 : audit 0 from='client.? 172.21.15.78:0/746349354' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:06:19.155 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:19.186 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:19.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:19.189 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:19.290 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:19.312 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:19.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:19.314 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:19.415 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:19.442 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:19.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:19.444 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:06:19.545 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:19.565 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:19.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:19.567 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:19.668 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:19.692 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:19.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:19.695 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:19.796 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:19.827 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:19.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:19.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:06:19.931 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:19.962 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:19.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:19.966 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:06:20.067 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:20.078 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:19 smithi173 bash[58816]: cluster 2023-10-01T11:06:18.301468+0000 mgr.y (mgr.25072) 1469 : cluster 0 pgmap v1567: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+remapped, 81 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 67 op/s; 14038/204114 objects degraded (6.878%); 8599/204114 objects misplaced (4.213%); 4.1 MiB/s, 66 objects/s recovering 2023-10-01T11:06:20.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:19 smithi078 bash[75299]: cluster 2023-10-01T11:06:18.301468+0000 mgr.y (mgr.25072) 1469 : cluster 0 pgmap v1567: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+remapped, 81 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 67 op/s; 14038/204114 objects degraded (6.878%); 8599/204114 objects misplaced (4.213%); 4.1 MiB/s, 66 objects/s recovering 2023-10-01T11:06:20.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:19 smithi078 bash[78274]: cluster 2023-10-01T11:06:18.301468+0000 mgr.y (mgr.25072) 1469 : cluster 0 pgmap v1567: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+remapped, 81 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 67 op/s; 14038/204114 objects degraded (6.878%); 8599/204114 objects misplaced (4.213%); 4.1 MiB/s, 66 objects/s recovering 2023-10-01T11:06:20.094 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:20.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:20.096 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:20.197 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:20.227 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:20.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:20.230 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:06:20.331 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:20.356 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:20.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:20.359 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:20.460 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:20.488 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:20.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:20.490 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:20.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:06:20 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:06:20] "GET /metrics HTTP/1.1" 200 36414 "" "Prometheus/2.33.4" 2023-10-01T11:06:20.591 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:20.616 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:20.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:20.618 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:20.693 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:06:20.719 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:20.740 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:20.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:20.742 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:20.843 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:20.871 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:20.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:20.873 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:06:20.974 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:20.999 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:21.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:21.003 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:21.104 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:21.138 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:21.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:21.140 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:06:21.205 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T11:06:21.241 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:21.271 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:21.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:21.273 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:21.374 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:21.402 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:21.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:21.405 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:06:21.505 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:21.535 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:21.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:21.537 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:06:21.538 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T11:06:21.638 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:21.662 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:21.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:21.664 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:06:21.765 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:21.789 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:21.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:21.791 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:21.818 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:21 smithi173 bash[58816]: cluster 2023-10-01T11:06:20.302551+0000 mgr.y (mgr.25072) 1470 : cluster 0 pgmap v1568: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+remapped, 82 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 33 op/s; 14038/204114 objects degraded (6.878%); 8236/204114 objects misplaced (4.035%); 3.8 MiB/s, 122 objects/s recovering 2023-10-01T11:06:21.818 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:21 smithi173 bash[58816]: audit 2023-10-01T11:06:21.452607+0000 mon.a (mon.0) 1888 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T11:06:21.892 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:21.917 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:21.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:21.919 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:06:22.020 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:22.045 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:22.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:22.048 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:22.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:21 smithi078 bash[75299]: cluster 2023-10-01T11:06:20.302551+0000 mgr.y (mgr.25072) 1470 : cluster 0 pgmap v1568: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+remapped, 82 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 33 op/s; 14038/204114 objects degraded (6.878%); 8236/204114 objects misplaced (4.035%); 3.8 MiB/s, 122 objects/s recovering 2023-10-01T11:06:22.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:21 smithi078 bash[75299]: audit 2023-10-01T11:06:21.452607+0000 mon.a (mon.0) 1888 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T11:06:22.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:21 smithi078 bash[78274]: cluster 2023-10-01T11:06:20.302551+0000 mgr.y (mgr.25072) 1470 : cluster 0 pgmap v1568: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+remapped, 82 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 33 op/s; 14038/204114 objects degraded (6.878%); 8236/204114 objects misplaced (4.035%); 3.8 MiB/s, 122 objects/s recovering 2023-10-01T11:06:22.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:21 smithi078 bash[78274]: audit 2023-10-01T11:06:21.452607+0000 mon.a (mon.0) 1888 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T11:06:22.149 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:22.174 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:22.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:22.177 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:06:22.278 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:22.311 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:22.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:22.315 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:06:22.417 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:22.448 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:22.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:22.451 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:06:22.553 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:22.555 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T11:06:22.576 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:22.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:22.579 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:06:22.679 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:22.702 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:22.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:22.704 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:06:22.805 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:22.823 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:22 smithi173 bash[58816]: cluster 2023-10-01T11:06:22.745395+0000 mon.a (mon.0) 1889 : cluster 3 Health check update: Degraded data redundancy: 14038/204114 objects degraded (6.878%), 4 pgs degraded (PG_DEGRADED) 2023-10-01T11:06:22.838 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:22.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:22.840 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:06:22.941 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:22.949 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:22 smithi078 bash[78274]: cluster 2023-10-01T11:06:22.745395+0000 mon.a (mon.0) 1889 : cluster 3 Health check update: Degraded data redundancy: 14038/204114 objects degraded (6.878%), 4 pgs degraded (PG_DEGRADED) 2023-10-01T11:06:22.950 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:22 smithi078 bash[75299]: cluster 2023-10-01T11:06:22.745395+0000 mon.a (mon.0) 1889 : cluster 3 Health check update: Degraded data redundancy: 14038/204114 objects degraded (6.878%), 4 pgs degraded (PG_DEGRADED) 2023-10-01T11:06:22.966 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:22.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:22.969 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:06:23.069 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:23.094 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:23.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:23.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:06:23.200 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:23.221 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:23.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:23.223 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:23.324 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:23.356 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:23.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:23.360 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:06:23.461 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:23.488 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:23.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:23.491 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:06:23.592 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:23.618 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:23.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:23.623 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:06:23.724 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:23.747 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:23.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:23.749 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:06:23.850 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:23.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:23 smithi173 bash[58816]: cluster 2023-10-01T11:06:22.303193+0000 mgr.y (mgr.25072) 1471 : cluster 0 pgmap v1569: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+remapped, 82 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 8 op/s; 14038/204114 objects degraded (6.878%); 8236/204114 objects misplaced (4.035%); 3.6 MiB/s, 114 objects/s recovering 2023-10-01T11:06:23.875 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:23.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:23.878 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:06:23.978 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:23.989 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:23 smithi078 bash[78274]: cluster 2023-10-01T11:06:22.303193+0000 mgr.y (mgr.25072) 1471 : cluster 0 pgmap v1569: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+remapped, 82 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 8 op/s; 14038/204114 objects degraded (6.878%); 8236/204114 objects misplaced (4.035%); 3.6 MiB/s, 114 objects/s recovering 2023-10-01T11:06:23.989 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:23 smithi078 bash[75299]: cluster 2023-10-01T11:06:22.303193+0000 mgr.y (mgr.25072) 1471 : cluster 0 pgmap v1569: 89 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+remapped, 82 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 8 op/s; 14038/204114 objects degraded (6.878%); 8236/204114 objects misplaced (4.035%); 3.6 MiB/s, 114 objects/s recovering 2023-10-01T11:06:24.003 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:24.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:24.006 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:06:24.107 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:24.129 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:24.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:24.131 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:24.232 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:24.257 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:24.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:24.260 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:06:24.360 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:24.384 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:24.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:24.389 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:06:24.490 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:24.520 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:24.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:24.522 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:24.623 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:24.650 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:24.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:24.652 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:06:24.752 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:24.777 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:24.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:24.782 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:24.882 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:24.906 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:24.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:24.908 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:25.009 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:25.034 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:25.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:25.037 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:06:25.139 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:25.172 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:25.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:25.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:06:25.275 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:25.299 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:25.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:25.302 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:06:25.403 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:25.426 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:25.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:25.430 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:06:25.531 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:25.556 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:25.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:25.559 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:06:25.660 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:25.687 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:25.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:25.690 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:06:25.791 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:25.820 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:25.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:25.824 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:06:25.837 INFO:teuthology.orchestra.run.smithi078.stderr:nodeep-scrub is set 2023-10-01T11:06:25.925 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:25.949 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:25.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:25.951 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:06:26.051 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:26.088 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:26.090 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:25 smithi173 bash[58816]: cluster 2023-10-01T11:06:24.303851+0000 mgr.y (mgr.25072) 1472 : cluster 0 pgmap v1570: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 17 op/s; 14038/203694 objects degraded (6.892%); 8234/203694 objects misplaced (4.042%); 2.7 MiB/s, 96 objects/s recovering 2023-10-01T11:06:26.090 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:25 smithi173 bash[58816]: audit 2023-10-01T11:06:25.370228+0000 mon.b (mon.1) 57 : audit 1 from='client.? 172.21.15.78:0/2975663563' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:06:26.090 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:25 smithi173 bash[58816]: audit 2023-10-01T11:06:25.370647+0000 mon.a (mon.0) 1890 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:06:26.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:26.091 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:06:26.093 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:25 smithi078 bash[75299]: cluster 2023-10-01T11:06:24.303851+0000 mgr.y (mgr.25072) 1472 : cluster 0 pgmap v1570: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 17 op/s; 14038/203694 objects degraded (6.892%); 8234/203694 objects misplaced (4.042%); 2.7 MiB/s, 96 objects/s recovering 2023-10-01T11:06:26.094 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:25 smithi078 bash[75299]: audit 2023-10-01T11:06:25.370228+0000 mon.b (mon.1) 57 : audit 1 from='client.? 172.21.15.78:0/2975663563' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:06:26.094 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:25 smithi078 bash[75299]: audit 2023-10-01T11:06:25.370647+0000 mon.a (mon.0) 1890 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:06:26.192 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:26.220 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:26.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:26.224 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:26.325 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:26.349 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:26.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:26.351 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:06:26.352 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:26 smithi078 bash[78274]: cluster 2023-10-01T11:06:24.303851+0000 mgr.y (mgr.25072) 1472 : cluster 0 pgmap v1570: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 17 op/s; 14038/203694 objects degraded (6.892%); 8234/203694 objects misplaced (4.042%); 2.7 MiB/s, 96 objects/s recovering 2023-10-01T11:06:26.352 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:26 smithi078 bash[78274]: audit 2023-10-01T11:06:25.370228+0000 mon.b (mon.1) 57 : audit 1 from='client.? 172.21.15.78:0/2975663563' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:06:26.352 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:26 smithi078 bash[78274]: audit 2023-10-01T11:06:25.370647+0000 mon.a (mon.0) 1890 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:06:26.452 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:26.476 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:26.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:26.479 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:26.580 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:26.611 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:26.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:26.614 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:06:26.714 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:26.744 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:26.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:26.747 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:26.847 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:26.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:26 smithi173 bash[58816]: cluster 2023-10-01T11:06:25.824933+0000 mon.a (mon.0) 1891 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:06:26.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:26 smithi173 bash[58816]: cluster 2023-10-01T11:06:25.828499+0000 osd.7 (osd.7) 370 : cluster 3 failed to encode map e336 with expected crc 2023-10-01T11:06:26.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:26 smithi173 bash[58816]: audit 2023-10-01T11:06:25.831987+0000 mon.a (mon.0) 1892 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T11:06:26.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:26 smithi173 bash[58816]: cluster 2023-10-01T11:06:25.832070+0000 mon.a (mon.0) 1893 : cluster 0 osdmap e336: 8 total, 8 up, 8 in 2023-10-01T11:06:26.877 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:26.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:26.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:26.983 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:27.007 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:27.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:27.010 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:27.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:26 smithi078 bash[78274]: cluster 2023-10-01T11:06:25.824933+0000 mon.a (mon.0) 1891 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:06:27.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:26 smithi078 bash[78274]: cluster 2023-10-01T11:06:25.828499+0000 osd.7 (osd.7) 370 : cluster 3 failed to encode map e336 with expected crc 2023-10-01T11:06:27.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:26 smithi078 bash[78274]: audit 2023-10-01T11:06:25.831987+0000 mon.a (mon.0) 1892 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T11:06:27.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:26 smithi078 bash[78274]: cluster 2023-10-01T11:06:25.832070+0000 mon.a (mon.0) 1893 : cluster 0 osdmap e336: 8 total, 8 up, 8 in 2023-10-01T11:06:27.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:26 smithi078 bash[75299]: cluster 2023-10-01T11:06:25.824933+0000 mon.a (mon.0) 1891 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:06:27.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:26 smithi078 bash[75299]: cluster 2023-10-01T11:06:25.828499+0000 osd.7 (osd.7) 370 : cluster 3 failed to encode map e336 with expected crc 2023-10-01T11:06:27.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:26 smithi078 bash[75299]: audit 2023-10-01T11:06:25.831987+0000 mon.a (mon.0) 1892 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T11:06:27.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:26 smithi078 bash[75299]: cluster 2023-10-01T11:06:25.832070+0000 mon.a (mon.0) 1893 : cluster 0 osdmap e336: 8 total, 8 up, 8 in 2023-10-01T11:06:27.111 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:27.131 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:27.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:27.134 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:27.235 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:27.266 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:27.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:27.269 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:06:27.370 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:27.394 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:27.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:27.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:27.500 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:27.525 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:27.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:27.527 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:27.532 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:06:27.628 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:27.676 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:27.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:27.678 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:06:27.779 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:27.809 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:27.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:27.812 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:27.913 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:27.934 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:27.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:27.937 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:28.038 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:28.072 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:28.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:28.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:06:28.175 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:28.199 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:28.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:28.202 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:06:28.303 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:28.311 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:27 smithi078 bash[75299]: cluster 2023-10-01T11:06:26.304708+0000 mgr.y (mgr.25072) 1473 : cluster 0 pgmap v1572: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 24 op/s; 14038/203427 objects degraded (6.901%); 7918/203427 objects misplaced (3.892%); 4.5 MiB/s, 119 objects/s recovering 2023-10-01T11:06:28.311 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:27 smithi078 bash[75299]: audit 2023-10-01T11:06:26.991579+0000 mon.a (mon.0) 1894 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:06:28.311 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:27 smithi078 bash[75299]: audit 2023-10-01T11:06:26.998524+0000 mon.a (mon.0) 1895 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:06:28.311 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:27 smithi078 bash[75299]: cluster 2023-10-01T11:06:27.746602+0000 mon.a (mon.0) 1896 : cluster 3 Health check update: Degraded data redundancy: 14038/203427 objects degraded (6.901%), 5 pgs degraded (PG_DEGRADED) 2023-10-01T11:06:28.312 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:27 smithi078 bash[78274]: cluster 2023-10-01T11:06:26.304708+0000 mgr.y (mgr.25072) 1473 : cluster 0 pgmap v1572: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 24 op/s; 14038/203427 objects degraded (6.901%); 7918/203427 objects misplaced (3.892%); 4.5 MiB/s, 119 objects/s recovering 2023-10-01T11:06:28.312 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:27 smithi078 bash[78274]: audit 2023-10-01T11:06:26.991579+0000 mon.a (mon.0) 1894 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:06:28.312 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:27 smithi078 bash[78274]: audit 2023-10-01T11:06:26.998524+0000 mon.a (mon.0) 1895 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:06:28.312 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:27 smithi078 bash[78274]: cluster 2023-10-01T11:06:27.746602+0000 mon.a (mon.0) 1896 : cluster 3 Health check update: Degraded data redundancy: 14038/203427 objects degraded (6.901%), 5 pgs degraded (PG_DEGRADED) 2023-10-01T11:06:28.324 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:28.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:28.326 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:06:28.429 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:28.474 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:28.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:28.477 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:06:28.578 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:28.604 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:28.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:28.607 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:06:28.707 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:28.734 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:28.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:28.736 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:06:28.837 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:28.864 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:28.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:28.866 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:06:28.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:28 smithi173 bash[58816]: cluster 2023-10-01T11:06:26.304708+0000 mgr.y (mgr.25072) 1473 : cluster 0 pgmap v1572: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 24 op/s; 14038/203427 objects degraded (6.901%); 7918/203427 objects misplaced (3.892%); 4.5 MiB/s, 119 objects/s recovering 2023-10-01T11:06:28.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:28 smithi173 bash[58816]: audit 2023-10-01T11:06:26.991579+0000 mon.a (mon.0) 1894 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:06:28.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:28 smithi173 bash[58816]: audit 2023-10-01T11:06:26.998524+0000 mon.a (mon.0) 1895 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:06:28.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:28 smithi173 bash[58816]: cluster 2023-10-01T11:06:27.746602+0000 mon.a (mon.0) 1896 : cluster 3 Health check update: Degraded data redundancy: 14038/203427 objects degraded (6.901%), 5 pgs degraded (PG_DEGRADED) 2023-10-01T11:06:28.967 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:28.990 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:28.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:28.993 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:06:29.094 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:29.115 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T11:06:29.120 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:29.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:29.127 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:29.228 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:29.250 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:29.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:29.253 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:29.354 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:29.379 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:29.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:29.381 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:06:29.482 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:29.509 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:29.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:29.511 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:06:29.612 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:29.638 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:29.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:29.640 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:06:29.741 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:29.767 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:29.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:29.769 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:06:29.871 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:29.891 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:29.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:29.893 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:06:29.994 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:30.026 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:30.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:30.029 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:06:30.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:29 smithi078 bash[75299]: cluster 2023-10-01T11:06:28.305429+0000 mgr.y (mgr.25072) 1474 : cluster 0 pgmap v1573: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 46 op/s; 13981/202725 objects degraded (6.897%); 7863/202725 objects misplaced (3.879%); 4.5 MiB/s, 117 objects/s recovering 2023-10-01T11:06:30.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:29 smithi078 bash[75299]: audit 2023-10-01T11:06:28.651173+0000 mon.a (mon.0) 1897 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:06:30.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:29 smithi078 bash[75299]: audit 2023-10-01T11:06:28.657674+0000 mon.a (mon.0) 1898 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:06:30.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:29 smithi078 bash[75299]: audit 2023-10-01T11:06:29.181644+0000 mon.a (mon.0) 1899 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T11:06:30.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:29 smithi078 bash[75299]: audit 2023-10-01T11:06:29.182814+0000 mon.a (mon.0) 1900 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T11:06:30.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:29 smithi078 bash[75299]: audit 2023-10-01T11:06:29.189758+0000 mon.a (mon.0) 1901 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:06:30.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:29 smithi078 bash[78274]: cluster 2023-10-01T11:06:28.305429+0000 mgr.y (mgr.25072) 1474 : cluster 0 pgmap v1573: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 46 op/s; 13981/202725 objects degraded (6.897%); 7863/202725 objects misplaced (3.879%); 4.5 MiB/s, 117 objects/s recovering 2023-10-01T11:06:30.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:29 smithi078 bash[78274]: audit 2023-10-01T11:06:28.651173+0000 mon.a (mon.0) 1897 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:06:30.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:29 smithi078 bash[78274]: audit 2023-10-01T11:06:28.657674+0000 mon.a (mon.0) 1898 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:06:30.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:29 smithi078 bash[78274]: audit 2023-10-01T11:06:29.181644+0000 mon.a (mon.0) 1899 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T11:06:30.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:29 smithi078 bash[78274]: audit 2023-10-01T11:06:29.182814+0000 mon.a (mon.0) 1900 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T11:06:30.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:29 smithi078 bash[78274]: audit 2023-10-01T11:06:29.189758+0000 mon.a (mon.0) 1901 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:06:30.131 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:30.141 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:29 smithi173 bash[58816]: cluster 2023-10-01T11:06:28.305429+0000 mgr.y (mgr.25072) 1474 : cluster 0 pgmap v1573: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 46 op/s; 13981/202725 objects degraded (6.897%); 7863/202725 objects misplaced (3.879%); 4.5 MiB/s, 117 objects/s recovering 2023-10-01T11:06:30.141 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:29 smithi173 bash[58816]: audit 2023-10-01T11:06:28.651173+0000 mon.a (mon.0) 1897 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:06:30.141 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:29 smithi173 bash[58816]: audit 2023-10-01T11:06:28.657674+0000 mon.a (mon.0) 1898 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:06:30.141 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:29 smithi173 bash[58816]: audit 2023-10-01T11:06:29.181644+0000 mon.a (mon.0) 1899 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T11:06:30.141 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:29 smithi173 bash[58816]: audit 2023-10-01T11:06:29.182814+0000 mon.a (mon.0) 1900 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T11:06:30.142 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:29 smithi173 bash[58816]: audit 2023-10-01T11:06:29.189758+0000 mon.a (mon.0) 1901 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:06:30.153 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:30.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:30.156 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:30.257 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:30.285 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:30.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:30.287 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:06:30.388 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:30.413 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:30.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:30.419 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:06:30.520 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:30.554 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:30.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:30.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:06:30.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:06:30 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:06:30] "GET /metrics HTTP/1.1" 200 36414 "" "Prometheus/2.33.4" 2023-10-01T11:06:30.659 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:30.684 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:30.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:30.687 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:30.788 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:30.814 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:30.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:30.816 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:06:30.916 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:30.939 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:30.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:30.941 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:06:31.042 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:31.066 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:31.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:31.069 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:06:31.169 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:31.191 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:31.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:31.194 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:06:31.295 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:31.322 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:31.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:31.326 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:06:31.426 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:31.450 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:31.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:31.452 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:06:31.553 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:31.573 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:31.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:31.577 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:31.677 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:31.709 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:31.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:31.717 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:31.740 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T11:06:31.765 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T11:06:31.791 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T11:06:31.818 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:31.819 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T11:06:31.844 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:31.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:31.846 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:06:31.846 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T11:06:31.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:31 smithi173 bash[58816]: cluster 2023-10-01T11:06:30.306140+0000 mgr.y (mgr.25072) 1475 : cluster 0 pgmap v1574: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 123 op/s; 13981/200403 objects degraded (6.976%); 7602/200403 objects misplaced (3.793%); 3.9 MiB/s, 61 objects/s recovering 2023-10-01T11:06:31.871 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T11:06:31.896 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T11:06:31.922 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T11:06:31.923 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T11:06:31.923 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T11:06:31.923 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T11:06:31.923 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T11:06:31.923 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T11:06:31.923 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T11:06:31.923 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T11:06:31.923 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T11:06:31.945 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:31 smithi078 bash[75299]: cluster 2023-10-01T11:06:30.306140+0000 mgr.y (mgr.25072) 1475 : cluster 0 pgmap v1574: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 123 op/s; 13981/200403 objects degraded (6.976%); 7602/200403 objects misplaced (3.793%); 3.9 MiB/s, 61 objects/s recovering 2023-10-01T11:06:31.946 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:31 smithi078 bash[78274]: cluster 2023-10-01T11:06:30.306140+0000 mgr.y (mgr.25072) 1475 : cluster 0 pgmap v1574: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 123 op/s; 13981/200403 objects degraded (6.976%); 7602/200403 objects misplaced (3.793%); 3.9 MiB/s, 61 objects/s recovering 2023-10-01T11:06:31.947 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:31.971 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:31.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:31.973 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:06:32.074 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:32.111 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:32.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:32.272 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:32.296 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-10-01T11:06:32.296 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [], 2023-10-01T11:06:32.296 INFO:teuthology.orchestra.run.smithi173.stdout: "complaint_time": 30, 2023-10-01T11:06:32.296 INFO:teuthology.orchestra.run.smithi173.stdout: "num_blocked_ops": 0 2023-10-01T11:06:32.296 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-10-01T11:06:32.345 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-10-01T11:06:32.345 INFO:teuthology.orchestra.run.smithi173.stdout: "success": "" 2023-10-01T11:06:32.345 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-10-01T11:06:32.374 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:32.401 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:32.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:32.404 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:06:32.505 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:32.534 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:32.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:32.538 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:06:32.639 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:32.670 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:32.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:32.672 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:06:32.773 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:32.782 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:32 smithi173 bash[58816]: audit 2023-10-01T11:06:31.707691+0000 mon.a (mon.0) 1902 : audit 0 from='client.? 172.21.15.78:0/3490311559' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:06:32.782 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:32 smithi173 bash[58816]: audit 2023-10-01T11:06:32.460945+0000 mon.a (mon.0) 1903 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:06:32.792 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:32.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:32.796 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:06:32.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:32 smithi078 bash[75299]: audit 2023-10-01T11:06:31.707691+0000 mon.a (mon.0) 1902 : audit 0 from='client.? 172.21.15.78:0/3490311559' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:06:32.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:32 smithi078 bash[75299]: audit 2023-10-01T11:06:32.460945+0000 mon.a (mon.0) 1903 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:06:32.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:32 smithi078 bash[78274]: audit 2023-10-01T11:06:31.707691+0000 mon.a (mon.0) 1902 : audit 0 from='client.? 172.21.15.78:0/3490311559' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:06:32.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:32 smithi078 bash[78274]: audit 2023-10-01T11:06:32.460945+0000 mon.a (mon.0) 1903 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:06:32.897 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:32.927 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:32.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:32.930 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:06:33.031 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:33.060 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:33.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:33.062 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:06:33.163 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:33.193 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:33.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:33.288 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:06:33.389 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:33.416 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:33.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:33.419 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:33.520 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:33.552 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:33.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:33.557 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:06:33.658 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:33.663 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:33 smithi173 bash[58816]: cluster 2023-10-01T11:06:32.306643+0000 mgr.y (mgr.25072) 1476 : cluster 0 pgmap v1575: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 123 op/s; 13981/200403 objects degraded (6.976%); 7602/200403 objects misplaced (3.793%); 3.9 MiB/s, 61 objects/s recovering 2023-10-01T11:06:33.664 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:33 smithi173 bash[58816]: cluster 2023-10-01T11:06:32.747784+0000 mon.a (mon.0) 1904 : cluster 3 Health check update: Degraded data redundancy: 13981/200403 objects degraded (6.976%), 5 pgs degraded (PG_DEGRADED) 2023-10-01T11:06:33.682 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:33.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:33.684 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:06:33.785 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:33.798 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:33 smithi078 bash[78274]: cluster 2023-10-01T11:06:32.306643+0000 mgr.y (mgr.25072) 1476 : cluster 0 pgmap v1575: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 123 op/s; 13981/200403 objects degraded (6.976%); 7602/200403 objects misplaced (3.793%); 3.9 MiB/s, 61 objects/s recovering 2023-10-01T11:06:33.798 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:33 smithi078 bash[78274]: cluster 2023-10-01T11:06:32.747784+0000 mon.a (mon.0) 1904 : cluster 3 Health check update: Degraded data redundancy: 13981/200403 objects degraded (6.976%), 5 pgs degraded (PG_DEGRADED) 2023-10-01T11:06:33.799 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:33 smithi078 bash[75299]: cluster 2023-10-01T11:06:32.306643+0000 mgr.y (mgr.25072) 1476 : cluster 0 pgmap v1575: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 123 op/s; 13981/200403 objects degraded (6.976%); 7602/200403 objects misplaced (3.793%); 3.9 MiB/s, 61 objects/s recovering 2023-10-01T11:06:33.799 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:33 smithi078 bash[75299]: cluster 2023-10-01T11:06:32.747784+0000 mon.a (mon.0) 1904 : cluster 3 Health check update: Degraded data redundancy: 13981/200403 objects degraded (6.976%), 5 pgs degraded (PG_DEGRADED) 2023-10-01T11:06:33.821 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:33.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:33.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:06:33.925 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:33.951 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:33.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:33.953 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:06:34.054 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:34.086 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:34.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:34.092 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:06:34.193 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:34.220 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:34.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:34.223 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:34.324 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:34.343 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:34.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:34.345 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:34.443 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:06:34.446 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:34.470 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:34.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:34.472 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:06:34.573 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:34.600 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:34.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:34.603 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:34.704 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:34.727 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:34.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:34.730 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:34.830 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:34.851 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:34.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:34.853 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:34.948 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T11:06:34.954 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:35.023 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:35.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:35.025 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:06:35.126 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:35.146 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:35.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:35.150 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:06:35.251 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:35.280 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:35.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:35.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:06:35.383 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:35.404 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:35.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:35.406 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:06:35.507 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:35.534 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:35.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:35.537 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:06:35.540 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-01T11:06:35.638 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:35.668 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:35.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:35.671 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:35.772 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:35.796 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:35 smithi078 bash[78274]: cluster 2023-10-01T11:06:34.307658+0000 mgr.y (mgr.25072) 1477 : cluster 0 pgmap v1576: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 182 op/s; 13929/198210 objects degraded (7.027%); 7534/198210 objects misplaced (3.801%); 4.2 MiB/s, 67 objects/s recovering 2023-10-01T11:06:35.797 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:35 smithi078 bash[75299]: cluster 2023-10-01T11:06:34.307658+0000 mgr.y (mgr.25072) 1477 : cluster 0 pgmap v1576: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 182 op/s; 13929/198210 objects degraded (7.027%); 7534/198210 objects misplaced (3.801%); 4.2 MiB/s, 67 objects/s recovering 2023-10-01T11:06:35.801 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:35.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:35.806 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:35.907 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:35.918 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:35 smithi173 bash[58816]: cluster 2023-10-01T11:06:34.307658+0000 mgr.y (mgr.25072) 1477 : cluster 0 pgmap v1576: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 182 op/s; 13929/198210 objects degraded (7.027%); 7534/198210 objects misplaced (3.801%); 4.2 MiB/s, 67 objects/s recovering 2023-10-01T11:06:35.938 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:35.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:35.941 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:06:36.042 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:36.067 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:36.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:36.069 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:06:36.170 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:36.197 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:36.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:36.199 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:06:36.300 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:36.327 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:36.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:36.330 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:36.431 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:36.455 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:36.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:36.458 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:36.559 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:36.586 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:36.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:36.589 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:06:36.690 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:36.716 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:36.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:36.720 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:06:36.821 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:36.844 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:36.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:36.846 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:06:36.947 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:36.977 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:36.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:36.979 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:06:37.080 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:37.108 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:37.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:37.111 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:06:37.212 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:37.237 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:37.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:37.239 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:37.340 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:37.366 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:37.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:37.370 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:06:37.471 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:37.506 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:37.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:37.508 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:06:37.609 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:37.636 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:37.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:37.639 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:06:37.740 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:37.750 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:37 smithi173 bash[58816]: cluster 2023-10-01T11:06:36.308527+0000 mgr.y (mgr.25072) 1478 : cluster 0 pgmap v1577: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 190 op/s; 13929/197451 objects degraded (7.054%); 7224/197451 objects misplaced (3.659%); 4.0 MiB/s, 63 objects/s recovering 2023-10-01T11:06:37.772 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:37.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:37.774 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:37.875 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:37.900 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:37.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:37.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:06:38.003 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:38.027 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:38.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:38.032 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:06:38.064 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:37 smithi078 bash[75299]: cluster 2023-10-01T11:06:36.308527+0000 mgr.y (mgr.25072) 1478 : cluster 0 pgmap v1577: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 190 op/s; 13929/197451 objects degraded (7.054%); 7224/197451 objects misplaced (3.659%); 4.0 MiB/s, 63 objects/s recovering 2023-10-01T11:06:38.064 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:37 smithi078 bash[78274]: cluster 2023-10-01T11:06:36.308527+0000 mgr.y (mgr.25072) 1478 : cluster 0 pgmap v1577: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 190 op/s; 13929/197451 objects degraded (7.054%); 7224/197451 objects misplaced (3.659%); 4.0 MiB/s, 63 objects/s recovering 2023-10-01T11:06:38.133 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:38.158 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:38.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:38.162 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:38.263 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:38.291 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:38.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:38.294 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:06:38.395 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:38.423 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:38.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:38.426 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:06:38.527 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:38.555 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:38.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:38.558 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:38.659 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:38.673 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:38 smithi078 bash[75299]: cluster 2023-10-01T11:06:37.749091+0000 mon.a (mon.0) 1905 : cluster 3 Health check update: Degraded data redundancy: 13929/197451 objects degraded (7.054%), 5 pgs degraded (PG_DEGRADED) 2023-10-01T11:06:38.673 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:38 smithi078 bash[75299]: audit 2023-10-01T11:06:38.552022+0000 mon.c (mon.2) 224 : audit 1 from='client.? 172.21.15.78:0/638096268' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T11:06:38.673 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:38 smithi078 bash[75299]: audit 2023-10-01T11:06:38.552404+0000 mon.a (mon.0) 1906 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T11:06:38.691 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:38.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:38.693 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:38.770 INFO:teuthology.orchestra.run.smithi078.stderr:noscrub is unset 2023-10-01T11:06:38.794 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:38.803 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:38 smithi173 bash[58816]: cluster 2023-10-01T11:06:37.749091+0000 mon.a (mon.0) 1905 : cluster 3 Health check update: Degraded data redundancy: 13929/197451 objects degraded (7.054%), 5 pgs degraded (PG_DEGRADED) 2023-10-01T11:06:38.803 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:38 smithi173 bash[58816]: audit 2023-10-01T11:06:38.552022+0000 mon.c (mon.2) 224 : audit 1 from='client.? 172.21.15.78:0/638096268' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T11:06:38.803 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:38 smithi173 bash[58816]: audit 2023-10-01T11:06:38.552404+0000 mon.a (mon.0) 1906 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T11:06:38.820 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:38.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:38.822 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:06:38.923 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:38.951 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:38.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:38.953 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:06:38.963 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:38 smithi078 bash[78274]: cluster 2023-10-01T11:06:37.749091+0000 mon.a (mon.0) 1905 : cluster 3 Health check update: Degraded data redundancy: 13929/197451 objects degraded (7.054%), 5 pgs degraded (PG_DEGRADED) 2023-10-01T11:06:38.964 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:38 smithi078 bash[78274]: audit 2023-10-01T11:06:38.552022+0000 mon.c (mon.2) 224 : audit 1 from='client.? 172.21.15.78:0/638096268' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T11:06:38.964 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:38 smithi078 bash[78274]: audit 2023-10-01T11:06:38.552404+0000 mon.a (mon.0) 1906 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T11:06:39.054 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:39.089 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:39.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:39.092 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:39.193 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:39.222 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:39.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:39.224 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:06:39.325 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:39.348 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:39.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:39.351 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:06:39.451 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:39.483 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:39.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:39.487 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:39.588 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:39.617 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:39.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:39.620 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:06:39.721 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:39.754 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:39.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:39.756 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:06:39.857 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:39.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:39 smithi173 bash[58816]: cluster 2023-10-01T11:06:38.309178+0000 mgr.y (mgr.25072) 1479 : cluster 0 pgmap v1578: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 182 op/s; 13874/196860 objects degraded (7.048%); 7179/196860 objects misplaced (3.647%); 3.7 MiB/s, 60 objects/s recovering 2023-10-01T11:06:39.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:39 smithi173 bash[58816]: cluster 2023-10-01T11:06:38.752118+0000 mon.a (mon.0) 1907 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:06:39.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:39 smithi173 bash[58816]: audit 2023-10-01T11:06:38.763332+0000 mon.a (mon.0) 1908 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T11:06:39.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:39 smithi173 bash[58816]: cluster 2023-10-01T11:06:38.763453+0000 mon.a (mon.0) 1909 : cluster 0 osdmap e337: 8 total, 8 up, 8 in 2023-10-01T11:06:39.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:39 smithi173 bash[58816]: cluster 2023-10-01T11:06:38.779381+0000 osd.7 (osd.7) 371 : cluster 3 failed to encode map e337 with expected crc 2023-10-01T11:06:39.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:39 smithi173 bash[58816]: cluster 2023-10-01T11:06:38.929362+0000 osd.5 (osd.5) 356 : cluster 3 failed to encode map e337 with expected crc 2023-10-01T11:06:39.888 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:39.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:39.891 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:39.992 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:40.005 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:39 smithi078 bash[78274]: cluster 2023-10-01T11:06:38.309178+0000 mgr.y (mgr.25072) 1479 : cluster 0 pgmap v1578: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 182 op/s; 13874/196860 objects degraded (7.048%); 7179/196860 objects misplaced (3.647%); 3.7 MiB/s, 60 objects/s recovering 2023-10-01T11:06:40.005 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:39 smithi078 bash[78274]: cluster 2023-10-01T11:06:38.752118+0000 mon.a (mon.0) 1907 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:06:40.006 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:39 smithi078 bash[78274]: audit 2023-10-01T11:06:38.763332+0000 mon.a (mon.0) 1908 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T11:06:40.006 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:39 smithi078 bash[78274]: cluster 2023-10-01T11:06:38.763453+0000 mon.a (mon.0) 1909 : cluster 0 osdmap e337: 8 total, 8 up, 8 in 2023-10-01T11:06:40.006 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:39 smithi078 bash[78274]: cluster 2023-10-01T11:06:38.779381+0000 osd.7 (osd.7) 371 : cluster 3 failed to encode map e337 with expected crc 2023-10-01T11:06:40.006 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:39 smithi078 bash[78274]: cluster 2023-10-01T11:06:38.929362+0000 osd.5 (osd.5) 356 : cluster 3 failed to encode map e337 with expected crc 2023-10-01T11:06:40.007 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:39 smithi078 bash[75299]: cluster 2023-10-01T11:06:38.309178+0000 mgr.y (mgr.25072) 1479 : cluster 0 pgmap v1578: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 182 op/s; 13874/196860 objects degraded (7.048%); 7179/196860 objects misplaced (3.647%); 3.7 MiB/s, 60 objects/s recovering 2023-10-01T11:06:40.007 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:39 smithi078 bash[75299]: cluster 2023-10-01T11:06:38.752118+0000 mon.a (mon.0) 1907 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:06:40.007 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:39 smithi078 bash[75299]: audit 2023-10-01T11:06:38.763332+0000 mon.a (mon.0) 1908 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T11:06:40.007 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:39 smithi078 bash[75299]: cluster 2023-10-01T11:06:38.763453+0000 mon.a (mon.0) 1909 : cluster 0 osdmap e337: 8 total, 8 up, 8 in 2023-10-01T11:06:40.008 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:39 smithi078 bash[75299]: cluster 2023-10-01T11:06:38.779381+0000 osd.7 (osd.7) 371 : cluster 3 failed to encode map e337 with expected crc 2023-10-01T11:06:40.008 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:39 smithi078 bash[75299]: cluster 2023-10-01T11:06:38.929362+0000 osd.5 (osd.5) 356 : cluster 3 failed to encode map e337 with expected crc 2023-10-01T11:06:40.021 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:40.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:40.024 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:06:40.125 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:40.158 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:40.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:40.163 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:06:40.257 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:06:40.264 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:40.270 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:06:40 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:06:40] "GET /metrics HTTP/1.1" 200 36402 "" "Prometheus/2.33.4" 2023-10-01T11:06:40.292 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:40.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:40.295 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:06:40.338 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T11:06:40.395 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:40.426 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:40.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:40.428 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:40.529 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:40.556 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:40.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:40.558 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:40.659 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:40.684 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:40.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:40.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:06:40.788 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:40.811 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:40.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:40.813 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:06:40.818 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T11:06:40.818 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T11:06:40.818 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-01T11:06:40.818 INFO:tasks.thrashosds.thrasher:Setting osd 7 primary_affinity to 0.000000 2023-10-01T11:06:40.819 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 7 0 2023-10-01T11:06:40.915 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:40.937 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:40.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:40.940 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:06:41.041 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:41.064 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:41.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:41.066 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:06:41.167 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:41.194 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:41.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:41.196 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:06:41.297 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:41.322 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:41.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:41.325 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:41.426 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:41.456 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:41.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:41.458 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:06:41.559 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:41.586 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:41.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:41.590 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:06:41.691 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:41.698 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:41 smithi078 bash[75299]: cluster 2023-10-01T11:06:40.309853+0000 mgr.y (mgr.25072) 1480 : cluster 0 pgmap v1580: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 177 op/s; 13874/195075 objects degraded (7.112%); 6843/195075 objects misplaced (3.508%); 4.7 MiB/s, 75 objects/s recovering 2023-10-01T11:06:41.715 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:41.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:41.719 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:41.778 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T11:06:41.820 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:41.831 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:41 smithi173 bash[58816]: cluster 2023-10-01T11:06:40.309853+0000 mgr.y (mgr.25072) 1480 : cluster 0 pgmap v1580: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 177 op/s; 13874/195075 objects degraded (7.112%); 6843/195075 objects misplaced (3.508%); 4.7 MiB/s, 75 objects/s recovering 2023-10-01T11:06:41.846 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:41.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:41.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:06:41.951 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:41.978 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:41.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:41.984 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:42.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:41 smithi078 bash[78274]: cluster 2023-10-01T11:06:40.309853+0000 mgr.y (mgr.25072) 1480 : cluster 0 pgmap v1580: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 177 op/s; 13874/195075 objects degraded (7.112%); 6843/195075 objects misplaced (3.508%); 4.7 MiB/s, 75 objects/s recovering 2023-10-01T11:06:42.084 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:42.110 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:42.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:42.113 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:06:42.214 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:42.238 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:42.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:42.241 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:42.341 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:42.365 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:42.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:42.368 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:06:42.468 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:42.499 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:42.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:42.502 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:06:42.603 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:42.628 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:42.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:42.635 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:42.736 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:42.765 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:42.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:42.767 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:06:42.868 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:42.895 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:42.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:42.898 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:06:42.999 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:43.028 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:43.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:43.033 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:06:43.134 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:43.161 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:43.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:43.163 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:06:43.263 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:43.288 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:43.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:43.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:06:43.392 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:43.417 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:43.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:43.420 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:06:43.521 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:43.549 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:43.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:43.551 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:06:43.652 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:43.682 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:43.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:43.688 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:43.788 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:43.812 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:43.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:43.814 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:06:43.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:43 smithi078 bash[78274]: cluster 2023-10-01T11:06:42.310340+0000 mgr.y (mgr.25072) 1481 : cluster 0 pgmap v1581: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 177 op/s; 13874/195075 objects degraded (7.112%); 6843/195075 objects misplaced (3.508%); 4.7 MiB/s, 75 objects/s recovering 2023-10-01T11:06:43.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:43 smithi078 bash[78274]: cluster 2023-10-01T11:06:42.750487+0000 mon.a (mon.0) 1910 : cluster 3 Health check update: Degraded data redundancy: 13874/195075 objects degraded (7.112%), 5 pgs degraded (PG_DEGRADED) 2023-10-01T11:06:43.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:43 smithi078 bash[75299]: cluster 2023-10-01T11:06:42.310340+0000 mgr.y (mgr.25072) 1481 : cluster 0 pgmap v1581: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 177 op/s; 13874/195075 objects degraded (7.112%); 6843/195075 objects misplaced (3.508%); 4.7 MiB/s, 75 objects/s recovering 2023-10-01T11:06:43.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:43 smithi078 bash[75299]: cluster 2023-10-01T11:06:42.750487+0000 mon.a (mon.0) 1910 : cluster 3 Health check update: Degraded data redundancy: 13874/195075 objects degraded (7.112%), 5 pgs degraded (PG_DEGRADED) 2023-10-01T11:06:43.915 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:43.941 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:43.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:43.944 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:06:44.045 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:44.048 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:43 smithi173 bash[58816]: cluster 2023-10-01T11:06:42.310340+0000 mgr.y (mgr.25072) 1481 : cluster 0 pgmap v1581: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 177 op/s; 13874/195075 objects degraded (7.112%); 6843/195075 objects misplaced (3.508%); 4.7 MiB/s, 75 objects/s recovering 2023-10-01T11:06:44.048 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:43 smithi173 bash[58816]: cluster 2023-10-01T11:06:42.750487+0000 mon.a (mon.0) 1910 : cluster 3 Health check update: Degraded data redundancy: 13874/195075 objects degraded (7.112%), 5 pgs degraded (PG_DEGRADED) 2023-10-01T11:06:44.069 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:44.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:44.071 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:44.172 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:44.201 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:44.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:44.203 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:06:44.231 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T11:06:44.260 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T11:06:44.284 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T11:06:44.304 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:44.314 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T11:06:44.334 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:44.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:44.338 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:06:44.344 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T11:06:44.371 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T11:06:44.397 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T11:06:44.423 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T11:06:44.425 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T11:06:44.425 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T11:06:44.425 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T11:06:44.425 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T11:06:44.425 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T11:06:44.425 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T11:06:44.425 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T11:06:44.425 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T11:06:44.439 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:44.469 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:44.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:44.471 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:06:44.572 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:44.594 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:44.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:44.596 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:06:44.697 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:44.721 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:44.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:44.726 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:06:44.826 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:44.830 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:44 smithi078 bash[75299]: audit 2023-10-01T11:06:44.199679+0000 mon.a (mon.0) 1911 : audit 0 from='client.? 172.21.15.78:0/3548319795' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:06:44.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:44 smithi078 bash[78274]: audit 2023-10-01T11:06:44.199679+0000 mon.a (mon.0) 1911 : audit 0 from='client.? 172.21.15.78:0/3548319795' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:06:44.849 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:44.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:44.851 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:44.952 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:44.959 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:44 smithi173 bash[58816]: audit 2023-10-01T11:06:44.199679+0000 mon.a (mon.0) 1911 : audit 0 from='client.? 172.21.15.78:0/3548319795' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:06:44.974 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:44.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:44.976 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:06:45.077 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:45.102 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:45.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:45.105 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:06:45.205 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:45.230 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:45.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:45.233 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:06:45.333 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:45.361 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:45.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:45.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:45.468 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:45.494 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:45.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:45.496 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:06:45.597 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:45.625 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:45.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:45.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:06:45.729 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:45.741 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:45 smithi078 bash[78274]: cluster 2023-10-01T11:06:44.310964+0000 mgr.y (mgr.25072) 1482 : cluster 0 pgmap v1582: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 174 op/s; 13820/192966 objects degraded (7.162%); 6784/192966 objects misplaced (3.516%); 4.7 MiB/s, 75 objects/s recovering 2023-10-01T11:06:45.741 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:45 smithi078 bash[75299]: cluster 2023-10-01T11:06:44.310964+0000 mgr.y (mgr.25072) 1482 : cluster 0 pgmap v1582: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 174 op/s; 13820/192966 objects degraded (7.162%); 6784/192966 objects misplaced (3.516%); 4.7 MiB/s, 75 objects/s recovering 2023-10-01T11:06:45.758 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:45.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:45.761 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:06:45.861 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:45.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:45 smithi173 bash[58816]: cluster 2023-10-01T11:06:44.310964+0000 mgr.y (mgr.25072) 1482 : cluster 0 pgmap v1582: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 174 op/s; 13820/192966 objects degraded (7.162%); 6784/192966 objects misplaced (3.516%); 4.7 MiB/s, 75 objects/s recovering 2023-10-01T11:06:45.886 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:45.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:45.888 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:45.989 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:46.015 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:46.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:46.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:46.102 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:06:46.104 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:06:46.119 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:46.146 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:46.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:46.149 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:46.249 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:46.277 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:46.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:46.280 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:06:46.380 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:46.406 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:46.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:46.409 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:06:46.509 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:46.536 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:46.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:46.539 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:46.640 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:46.662 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:46.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:46.667 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:06:46.768 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:46.792 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:46.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:46.794 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:06:46.894 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:46.918 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:46.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:46.920 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:47.021 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:47.051 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:47.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:47.053 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:06:47.154 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:47.178 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:47.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:47.181 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:06:47.282 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:47.307 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:47.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:47.309 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:06:47.410 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:47.440 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:47.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:47.443 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:06:47.544 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:47.569 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:47.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:47.571 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:06:47.574 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-10-01T11:06:47.574 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T11:06:47.631 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T11:06:47.672 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:47.691 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:47.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:47.694 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:47.794 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:47.822 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:47.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:47.825 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:06:47.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:47 smithi173 bash[58816]: cluster 2023-10-01T11:06:46.311510+0000 mgr.y (mgr.25072) 1483 : cluster 0 pgmap v1583: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 174 op/s; 13820/192222 objects degraded (7.190%); 6438/192222 objects misplaced (3.349%); 4.9 MiB/s, 79 objects/s recovering 2023-10-01T11:06:47.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:47 smithi173 bash[58816]: audit 2023-10-01T11:06:47.462087+0000 mon.a (mon.0) 1912 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:06:47.926 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:47.956 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:47.957 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:47 smithi078 bash[78274]: cluster 2023-10-01T11:06:46.311510+0000 mgr.y (mgr.25072) 1483 : cluster 0 pgmap v1583: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 174 op/s; 13820/192222 objects degraded (7.190%); 6438/192222 objects misplaced (3.349%); 4.9 MiB/s, 79 objects/s recovering 2023-10-01T11:06:47.957 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:47 smithi078 bash[78274]: audit 2023-10-01T11:06:47.462087+0000 mon.a (mon.0) 1912 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:06:47.958 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:47 smithi078 bash[75299]: cluster 2023-10-01T11:06:46.311510+0000 mgr.y (mgr.25072) 1483 : cluster 0 pgmap v1583: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 174 op/s; 13820/192222 objects degraded (7.190%); 6438/192222 objects misplaced (3.349%); 4.9 MiB/s, 79 objects/s recovering 2023-10-01T11:06:47.958 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:47 smithi078 bash[75299]: audit 2023-10-01T11:06:47.462087+0000 mon.a (mon.0) 1912 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:06:47.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:47.959 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:06:48.060 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:48.088 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:48.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:48.091 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:06:48.191 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:48.224 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:48.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:48.229 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:06:48.329 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:48.355 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:48.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:48.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:06:48.458 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:48.480 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:48.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:48.486 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:48.587 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:48.612 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:48.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:48.614 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:06:48.715 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:48.746 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:48.748 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:48 smithi078 bash[75299]: cluster 2023-10-01T11:06:47.754521+0000 mon.a (mon.0) 1913 : cluster 3 Health check update: Degraded data redundancy: 13820/192222 objects degraded (7.190%), 5 pgs degraded (PG_DEGRADED) 2023-10-01T11:06:48.748 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:48 smithi078 bash[78274]: cluster 2023-10-01T11:06:47.754521+0000 mon.a (mon.0) 1913 : cluster 3 Health check update: Degraded data redundancy: 13820/192222 objects degraded (7.190%), 5 pgs degraded (PG_DEGRADED) 2023-10-01T11:06:48.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:48.749 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:06:48.850 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:48.861 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:48 smithi173 bash[58816]: cluster 2023-10-01T11:06:47.754521+0000 mon.a (mon.0) 1913 : cluster 3 Health check update: Degraded data redundancy: 13820/192222 objects degraded (7.190%), 5 pgs degraded (PG_DEGRADED) 2023-10-01T11:06:48.875 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:48.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:48.879 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:06:48.979 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:49.010 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:49.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:49.013 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:06:49.114 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:49.145 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:49.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:49.149 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:06:49.200 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-01T11:06:49.249 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:49.274 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:49.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:49.276 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:06:49.377 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:49.396 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:49.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:49.399 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:06:49.499 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:49.525 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:49.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:49.529 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:49.630 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:49.658 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:49.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:49.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:06:49.762 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:49 smithi173 bash[58816]: cluster 2023-10-01T11:06:48.312223+0000 mgr.y (mgr.25072) 1484 : cluster 0 pgmap v1584: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 177 op/s; 13764/191520 objects degraded (7.187%); 6360/191520 objects misplaced (3.321%); 4.9 MiB/s, 79 objects/s recovering 2023-10-01T11:06:49.764 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:49.787 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:49.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:49.790 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:06:49.893 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:49.917 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:49.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:49.919 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:06:50.019 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:50.044 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:50.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:50.048 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:50.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:49 smithi078 bash[75299]: cluster 2023-10-01T11:06:48.312223+0000 mgr.y (mgr.25072) 1484 : cluster 0 pgmap v1584: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 177 op/s; 13764/191520 objects degraded (7.187%); 6360/191520 objects misplaced (3.321%); 4.9 MiB/s, 79 objects/s recovering 2023-10-01T11:06:50.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:49 smithi078 bash[78274]: cluster 2023-10-01T11:06:48.312223+0000 mgr.y (mgr.25072) 1484 : cluster 0 pgmap v1584: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 177 op/s; 13764/191520 objects degraded (7.187%); 6360/191520 objects misplaced (3.321%); 4.9 MiB/s, 79 objects/s recovering 2023-10-01T11:06:50.149 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:50.177 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:50.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:50.180 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:06:50.281 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:50.304 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:50.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:50.307 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:06:50.408 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:50.432 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:50.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:50.435 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:06:50.535 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:50.560 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:50.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:50.563 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:06:50.649 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:06:50 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:06:50] "GET /metrics HTTP/1.1" 200 36408 "" "Prometheus/2.33.4" 2023-10-01T11:06:50.664 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:50.689 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:50.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:50.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:06:50.792 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:50.816 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:50.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:50.818 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:06:50.919 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:50.947 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:50.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:50.952 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:06:51.053 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:51.082 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:51.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:51.086 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:06:51.187 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:51.211 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:51.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:51.214 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:06:51.315 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:51.341 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:51.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:51.343 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:06:51.443 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:51.464 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:51.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:51.467 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:51.568 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:51.594 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:51.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:51.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:06:51.697 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:51.723 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:51.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:51.726 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:06:51.768 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:51 smithi173 bash[58816]: cluster 2023-10-01T11:06:50.313242+0000 mgr.y (mgr.25072) 1485 : cluster 0 pgmap v1585: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 170 op/s; 13764/189711 objects degraded (7.255%); 5962/189711 objects misplaced (3.143%); 6.4 MiB/s, 102 objects/s recovering 2023-10-01T11:06:51.826 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:51.855 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:51.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:51.859 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:51.960 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:51.987 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:51.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:51.989 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:06:52.089 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:52.098 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:51 smithi078 bash[75299]: cluster 2023-10-01T11:06:50.313242+0000 mgr.y (mgr.25072) 1485 : cluster 0 pgmap v1585: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 170 op/s; 13764/189711 objects degraded (7.255%); 5962/189711 objects misplaced (3.143%); 6.4 MiB/s, 102 objects/s recovering 2023-10-01T11:06:52.099 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:51 smithi078 bash[78274]: cluster 2023-10-01T11:06:50.313242+0000 mgr.y (mgr.25072) 1485 : cluster 0 pgmap v1585: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 170 op/s; 13764/189711 objects degraded (7.255%); 5962/189711 objects misplaced (3.143%); 6.4 MiB/s, 102 objects/s recovering 2023-10-01T11:06:52.112 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:52.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:52.115 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:06:52.215 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:52.247 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:52.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:52.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:06:52.350 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:52.373 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:52.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:52.378 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:06:52.403 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:06:52.479 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:52.502 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:52.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:52.504 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:06:52.605 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:52.640 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:52.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:52.642 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:52.743 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:52.766 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:52.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:52.769 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:06:52.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:52 smithi173 bash[58816]: cluster 2023-10-01T11:06:52.756114+0000 mon.a (mon.0) 1914 : cluster 3 Health check update: Degraded data redundancy: 13764/189711 objects degraded (7.255%), 5 pgs degraded (PG_DEGRADED) 2023-10-01T11:06:52.869 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:52.898 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:52.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:52.900 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:06:53.002 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:53.027 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:53.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:53.029 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:53.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:52 smithi078 bash[78274]: cluster 2023-10-01T11:06:52.756114+0000 mon.a (mon.0) 1914 : cluster 3 Health check update: Degraded data redundancy: 13764/189711 objects degraded (7.255%), 5 pgs degraded (PG_DEGRADED) 2023-10-01T11:06:53.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:52 smithi078 bash[75299]: cluster 2023-10-01T11:06:52.756114+0000 mon.a (mon.0) 1914 : cluster 3 Health check update: Degraded data redundancy: 13764/189711 objects degraded (7.255%), 5 pgs degraded (PG_DEGRADED) 2023-10-01T11:06:53.130 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:53.155 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:53.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:53.158 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:53.259 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:53.285 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:53.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:53.289 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:06:53.390 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:53.415 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:06:53.418 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:53.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:53.425 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:53.526 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:53.555 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:53.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:53.557 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:53.658 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:53.683 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:53.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:53.686 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:06:53.777 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:53 smithi078 bash[75299]: cluster 2023-10-01T11:06:52.313749+0000 mgr.y (mgr.25072) 1486 : cluster 0 pgmap v1586: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 148 op/s; 13764/189711 objects degraded (7.255%); 5962/189711 objects misplaced (3.143%); 4.4 MiB/s, 71 objects/s recovering 2023-10-01T11:06:53.777 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:53 smithi078 bash[78274]: cluster 2023-10-01T11:06:52.313749+0000 mgr.y (mgr.25072) 1486 : cluster 0 pgmap v1586: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 148 op/s; 13764/189711 objects degraded (7.255%); 5962/189711 objects misplaced (3.143%); 4.4 MiB/s, 71 objects/s recovering 2023-10-01T11:06:53.789 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:53.797 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:53 smithi173 bash[58816]: cluster 2023-10-01T11:06:52.313749+0000 mgr.y (mgr.25072) 1486 : cluster 0 pgmap v1586: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 148 op/s; 13764/189711 objects degraded (7.255%); 5962/189711 objects misplaced (3.143%); 4.4 MiB/s, 71 objects/s recovering 2023-10-01T11:06:53.814 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:53.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:53.817 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:53.918 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:53.942 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:53.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:53.945 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:06:54.045 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:54.065 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:54.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:54.067 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:06:54.168 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:54.190 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:54.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:54.192 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:06:54.293 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:54.320 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:54.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:54.323 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:54.424 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:54.454 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:54.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:54.458 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:06:54.559 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:54.587 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:54.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:54.595 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:54.695 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:54.735 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:54.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:54.738 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:54.797 INFO:teuthology.orchestra.run.smithi078.stderr:nodeep-scrub is unset 2023-10-01T11:06:54.798 INFO:teuthology.orchestra.run.smithi078.stderr:set osd.7 primary-affinity to 0 (802) 2023-10-01T11:06:54.838 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:54.860 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:54.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:54.862 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:06:54.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:54 smithi173 bash[58816]: audit 2023-10-01T11:06:54.225115+0000 mon.c (mon.2) 225 : audit 1 from='client.? 172.21.15.78:0/849328084' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:06:54.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:54 smithi173 bash[58816]: audit 2023-10-01T11:06:54.225655+0000 mon.a (mon.0) 1915 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:06:54.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:54 smithi173 bash[58816]: audit 2023-10-01T11:06:54.523050+0000 mon.c (mon.2) 226 : audit 1 from='client.? 172.21.15.78:0/641308099' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.0}]: dispatch 2023-10-01T11:06:54.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:54 smithi173 bash[58816]: audit 2023-10-01T11:06:54.523548+0000 mon.a (mon.0) 1916 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.0}]: dispatch 2023-10-01T11:06:54.963 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:54.989 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:54.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:54.991 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:06:55.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:54 smithi078 bash[78274]: audit 2023-10-01T11:06:54.225115+0000 mon.c (mon.2) 225 : audit 1 from='client.? 172.21.15.78:0/849328084' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:06:55.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:54 smithi078 bash[78274]: audit 2023-10-01T11:06:54.225655+0000 mon.a (mon.0) 1915 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:06:55.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:54 smithi078 bash[78274]: audit 2023-10-01T11:06:54.523050+0000 mon.c (mon.2) 226 : audit 1 from='client.? 172.21.15.78:0/641308099' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.0}]: dispatch 2023-10-01T11:06:55.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:54 smithi078 bash[78274]: audit 2023-10-01T11:06:54.523548+0000 mon.a (mon.0) 1916 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.0}]: dispatch 2023-10-01T11:06:55.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:54 smithi078 bash[75299]: audit 2023-10-01T11:06:54.225115+0000 mon.c (mon.2) 225 : audit 1 from='client.? 172.21.15.78:0/849328084' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:06:55.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:54 smithi078 bash[75299]: audit 2023-10-01T11:06:54.225655+0000 mon.a (mon.0) 1915 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:06:55.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:54 smithi078 bash[75299]: audit 2023-10-01T11:06:54.523050+0000 mon.c (mon.2) 226 : audit 1 from='client.? 172.21.15.78:0/641308099' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.0}]: dispatch 2023-10-01T11:06:55.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:54 smithi078 bash[75299]: audit 2023-10-01T11:06:54.523548+0000 mon.a (mon.0) 1916 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.0}]: dispatch 2023-10-01T11:06:55.092 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:55.119 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:55.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:55.122 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:55.223 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:55.249 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:55.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:55.252 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:55.352 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:55.382 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:55.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:55.385 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:06:55.486 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:55.514 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:55.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:55.516 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:06:55.617 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:55.643 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:55.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:55.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:06:55.746 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:55.782 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:55.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:55.784 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:55.885 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:55.893 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:55 smithi173 bash[58816]: cluster 2023-10-01T11:06:54.314350+0000 mgr.y (mgr.25072) 1487 : cluster 0 pgmap v1587: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 195 op/s; 13708/188019 objects degraded (7.291%); 5910/188019 objects misplaced (3.143%); 4.7 MiB/s, 75 objects/s recovering 2023-10-01T11:06:55.893 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:55 smithi173 bash[58816]: cluster 2023-10-01T11:06:54.774635+0000 mon.a (mon.0) 1917 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T11:06:55.893 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:55 smithi173 bash[58816]: audit 2023-10-01T11:06:54.787306+0000 mon.a (mon.0) 1918 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T11:06:55.893 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:55 smithi173 bash[58816]: audit 2023-10-01T11:06:54.787414+0000 mon.a (mon.0) 1919 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.0}]': finished 2023-10-01T11:06:55.893 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:55 smithi173 bash[58816]: cluster 2023-10-01T11:06:54.787498+0000 mon.a (mon.0) 1920 : cluster 0 osdmap e338: 8 total, 8 up, 8 in 2023-10-01T11:06:55.893 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:55 smithi173 bash[58816]: cluster 2023-10-01T11:06:54.790199+0000 osd.7 (osd.7) 372 : cluster 3 failed to encode map e338 with expected crc 2023-10-01T11:06:55.893 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:55 smithi173 bash[58816]: cluster 2023-10-01T11:06:54.793617+0000 osd.6 (osd.6) 324 : cluster 3 failed to encode map e338 with expected crc 2023-10-01T11:06:55.894 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:55 smithi173 bash[58816]: cluster 2023-10-01T11:06:54.814331+0000 osd.5 (osd.5) 357 : cluster 3 failed to encode map e338 with expected crc 2023-10-01T11:06:55.909 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:55.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:55.912 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:06:56.013 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:56.024 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:55 smithi078 bash[78274]: cluster 2023-10-01T11:06:54.314350+0000 mgr.y (mgr.25072) 1487 : cluster 0 pgmap v1587: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 195 op/s; 13708/188019 objects degraded (7.291%); 5910/188019 objects misplaced (3.143%); 4.7 MiB/s, 75 objects/s recovering 2023-10-01T11:06:56.024 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:55 smithi078 bash[78274]: cluster 2023-10-01T11:06:54.774635+0000 mon.a (mon.0) 1917 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T11:06:56.025 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:55 smithi078 bash[78274]: audit 2023-10-01T11:06:54.787306+0000 mon.a (mon.0) 1918 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T11:06:56.025 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:55 smithi078 bash[78274]: audit 2023-10-01T11:06:54.787414+0000 mon.a (mon.0) 1919 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.0}]': finished 2023-10-01T11:06:56.025 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:55 smithi078 bash[78274]: cluster 2023-10-01T11:06:54.787498+0000 mon.a (mon.0) 1920 : cluster 0 osdmap e338: 8 total, 8 up, 8 in 2023-10-01T11:06:56.025 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:55 smithi078 bash[78274]: cluster 2023-10-01T11:06:54.790199+0000 osd.7 (osd.7) 372 : cluster 3 failed to encode map e338 with expected crc 2023-10-01T11:06:56.025 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:55 smithi078 bash[78274]: cluster 2023-10-01T11:06:54.793617+0000 osd.6 (osd.6) 324 : cluster 3 failed to encode map e338 with expected crc 2023-10-01T11:06:56.025 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:55 smithi078 bash[78274]: cluster 2023-10-01T11:06:54.814331+0000 osd.5 (osd.5) 357 : cluster 3 failed to encode map e338 with expected crc 2023-10-01T11:06:56.025 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:55 smithi078 bash[75299]: cluster 2023-10-01T11:06:54.314350+0000 mgr.y (mgr.25072) 1487 : cluster 0 pgmap v1587: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 195 op/s; 13708/188019 objects degraded (7.291%); 5910/188019 objects misplaced (3.143%); 4.7 MiB/s, 75 objects/s recovering 2023-10-01T11:06:56.025 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:55 smithi078 bash[75299]: cluster 2023-10-01T11:06:54.774635+0000 mon.a (mon.0) 1917 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T11:06:56.025 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:55 smithi078 bash[75299]: audit 2023-10-01T11:06:54.787306+0000 mon.a (mon.0) 1918 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T11:06:56.025 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:55 smithi078 bash[75299]: audit 2023-10-01T11:06:54.787414+0000 mon.a (mon.0) 1919 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.0}]': finished 2023-10-01T11:06:56.026 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:55 smithi078 bash[75299]: cluster 2023-10-01T11:06:54.787498+0000 mon.a (mon.0) 1920 : cluster 0 osdmap e338: 8 total, 8 up, 8 in 2023-10-01T11:06:56.026 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:55 smithi078 bash[75299]: cluster 2023-10-01T11:06:54.790199+0000 osd.7 (osd.7) 372 : cluster 3 failed to encode map e338 with expected crc 2023-10-01T11:06:56.026 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:55 smithi078 bash[75299]: cluster 2023-10-01T11:06:54.793617+0000 osd.6 (osd.6) 324 : cluster 3 failed to encode map e338 with expected crc 2023-10-01T11:06:56.026 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:55 smithi078 bash[75299]: cluster 2023-10-01T11:06:54.814331+0000 osd.5 (osd.5) 357 : cluster 3 failed to encode map e338 with expected crc 2023-10-01T11:06:56.043 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:56.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:56.046 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:06:56.147 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:56.176 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:56.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:56.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:06:56.279 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:56.308 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:56.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:56.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:56.412 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:56.438 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:56.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:56.440 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:06:56.540 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:56.571 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:56.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:56.574 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:56.675 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:56.703 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:56.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:56.706 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:06:56.806 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:56.830 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:56.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:56.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:06:56.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:56 smithi173 bash[58816]: cluster 2023-10-01T11:06:55.787171+0000 osd.5 (osd.5) 358 : cluster 3 failed to encode map e339 with expected crc 2023-10-01T11:06:56.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:56 smithi173 bash[58816]: cluster 2023-10-01T11:06:55.789031+0000 mon.a (mon.0) 1921 : cluster 0 osdmap e339: 8 total, 8 up, 8 in 2023-10-01T11:06:56.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:56 smithi173 bash[58816]: cluster 2023-10-01T11:06:55.791905+0000 osd.7 (osd.7) 373 : cluster 3 failed to encode map e339 with expected crc 2023-10-01T11:06:56.870 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:56 smithi173 bash[58816]: cluster 2023-10-01T11:06:55.802055+0000 osd.6 (osd.6) 325 : cluster 3 failed to encode map e339 with expected crc 2023-10-01T11:06:56.933 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:56.957 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:56.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:56.959 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:06:57.060 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:57.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:56 smithi078 bash[78274]: cluster 2023-10-01T11:06:55.787171+0000 osd.5 (osd.5) 358 : cluster 3 failed to encode map e339 with expected crc 2023-10-01T11:06:57.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:56 smithi078 bash[78274]: cluster 2023-10-01T11:06:55.789031+0000 mon.a (mon.0) 1921 : cluster 0 osdmap e339: 8 total, 8 up, 8 in 2023-10-01T11:06:57.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:56 smithi078 bash[78274]: cluster 2023-10-01T11:06:55.791905+0000 osd.7 (osd.7) 373 : cluster 3 failed to encode map e339 with expected crc 2023-10-01T11:06:57.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:56 smithi078 bash[78274]: cluster 2023-10-01T11:06:55.802055+0000 osd.6 (osd.6) 325 : cluster 3 failed to encode map e339 with expected crc 2023-10-01T11:06:57.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:56 smithi078 bash[75299]: cluster 2023-10-01T11:06:55.787171+0000 osd.5 (osd.5) 358 : cluster 3 failed to encode map e339 with expected crc 2023-10-01T11:06:57.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:56 smithi078 bash[75299]: cluster 2023-10-01T11:06:55.789031+0000 mon.a (mon.0) 1921 : cluster 0 osdmap e339: 8 total, 8 up, 8 in 2023-10-01T11:06:57.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:56 smithi078 bash[75299]: cluster 2023-10-01T11:06:55.791905+0000 osd.7 (osd.7) 373 : cluster 3 failed to encode map e339 with expected crc 2023-10-01T11:06:57.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:56 smithi078 bash[75299]: cluster 2023-10-01T11:06:55.802055+0000 osd.6 (osd.6) 325 : cluster 3 failed to encode map e339 with expected crc 2023-10-01T11:06:57.095 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:57.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:57.099 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:57.200 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:57.239 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:57.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:57.243 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:06:57.343 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:57.380 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:57.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:57.383 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:06:57.484 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:57.518 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:06:57.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:57.521 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:06:57.622 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:57.648 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:57.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:57.650 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:06:57.751 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:57.783 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:57.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:57.788 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:06:57.799 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:57 smithi173 bash[58816]: cluster 2023-10-01T11:06:56.315057+0000 mgr.y (mgr.25072) 1488 : cluster 0 pgmap v1590: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 167 op/s; 13708/187488 objects degraded (7.311%); 5491/187488 objects misplaced (2.929%); 6.8 MiB/s, 109 objects/s recovering 2023-10-01T11:06:57.799 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:57 smithi173 bash[58816]: cluster 2023-10-01T11:06:56.795240+0000 osd.5 (osd.5) 359 : cluster 3 failed to encode map e340 with expected crc 2023-10-01T11:06:57.800 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:57 smithi173 bash[58816]: cluster 2023-10-01T11:06:56.795950+0000 osd.7 (osd.7) 374 : cluster 3 failed to encode map e340 with expected crc 2023-10-01T11:06:57.800 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:57 smithi173 bash[58816]: cluster 2023-10-01T11:06:56.797380+0000 mon.a (mon.0) 1922 : cluster 0 osdmap e340: 8 total, 8 up, 8 in 2023-10-01T11:06:57.800 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:57 smithi173 bash[58816]: cluster 2023-10-01T11:06:56.864673+0000 osd.6 (osd.6) 326 : cluster 3 failed to encode map e340 with expected crc 2023-10-01T11:06:57.800 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:57 smithi173 bash[58816]: cluster 2023-10-01T11:06:57.757345+0000 mon.a (mon.0) 1923 : cluster 3 Health check update: Degraded data redundancy: 13708/187488 objects degraded (7.311%), 5 pgs degraded (PG_DEGRADED) 2023-10-01T11:06:57.889 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:57.898 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:57 smithi078 bash[75299]: cluster 2023-10-01T11:06:56.315057+0000 mgr.y (mgr.25072) 1488 : cluster 0 pgmap v1590: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 167 op/s; 13708/187488 objects degraded (7.311%); 5491/187488 objects misplaced (2.929%); 6.8 MiB/s, 109 objects/s recovering 2023-10-01T11:06:57.898 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:57 smithi078 bash[75299]: cluster 2023-10-01T11:06:56.795240+0000 osd.5 (osd.5) 359 : cluster 3 failed to encode map e340 with expected crc 2023-10-01T11:06:57.898 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:57 smithi078 bash[75299]: cluster 2023-10-01T11:06:56.795950+0000 osd.7 (osd.7) 374 : cluster 3 failed to encode map e340 with expected crc 2023-10-01T11:06:57.898 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:57 smithi078 bash[75299]: cluster 2023-10-01T11:06:56.797380+0000 mon.a (mon.0) 1922 : cluster 0 osdmap e340: 8 total, 8 up, 8 in 2023-10-01T11:06:57.898 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:57 smithi078 bash[75299]: cluster 2023-10-01T11:06:56.864673+0000 osd.6 (osd.6) 326 : cluster 3 failed to encode map e340 with expected crc 2023-10-01T11:06:57.898 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:57 smithi078 bash[75299]: cluster 2023-10-01T11:06:57.757345+0000 mon.a (mon.0) 1923 : cluster 3 Health check update: Degraded data redundancy: 13708/187488 objects degraded (7.311%), 5 pgs degraded (PG_DEGRADED) 2023-10-01T11:06:57.899 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:57 smithi078 bash[78274]: cluster 2023-10-01T11:06:56.315057+0000 mgr.y (mgr.25072) 1488 : cluster 0 pgmap v1590: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 167 op/s; 13708/187488 objects degraded (7.311%); 5491/187488 objects misplaced (2.929%); 6.8 MiB/s, 109 objects/s recovering 2023-10-01T11:06:57.899 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:57 smithi078 bash[78274]: cluster 2023-10-01T11:06:56.795240+0000 osd.5 (osd.5) 359 : cluster 3 failed to encode map e340 with expected crc 2023-10-01T11:06:57.899 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:57 smithi078 bash[78274]: cluster 2023-10-01T11:06:56.795950+0000 osd.7 (osd.7) 374 : cluster 3 failed to encode map e340 with expected crc 2023-10-01T11:06:57.899 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:57 smithi078 bash[78274]: cluster 2023-10-01T11:06:56.797380+0000 mon.a (mon.0) 1922 : cluster 0 osdmap e340: 8 total, 8 up, 8 in 2023-10-01T11:06:57.899 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:57 smithi078 bash[78274]: cluster 2023-10-01T11:06:56.864673+0000 osd.6 (osd.6) 326 : cluster 3 failed to encode map e340 with expected crc 2023-10-01T11:06:57.899 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:57 smithi078 bash[78274]: cluster 2023-10-01T11:06:57.757345+0000 mon.a (mon.0) 1923 : cluster 3 Health check update: Degraded data redundancy: 13708/187488 objects degraded (7.311%), 5 pgs degraded (PG_DEGRADED) 2023-10-01T11:06:57.919 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:57.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:57.922 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:06:58.023 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:58.050 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:58.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:58.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:06:58.153 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:58.179 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:06:58.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:58.182 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:06:58.283 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:58.309 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:58.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:58.312 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:06:58.412 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:58.441 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:58.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:58.444 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:06:58.544 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:58.574 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:58.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:58.579 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:06:58.680 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:58.705 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:58.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:58.712 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:06:58.813 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:58.841 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:06:58.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:58.846 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:06:58.946 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:58.980 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:58.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:58.987 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:06:59.087 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:59.116 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:06:59.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:59.118 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:06:59.219 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:59.244 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:06:59.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:59.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:06:59.351 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:59.382 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:59.382 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T11:06:59.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:59.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:59.488 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:59.521 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:59.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:59.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:06:59.627 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:59.658 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:06:59.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:59.666 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:06:59.766 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:59.793 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:06:59.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:59.795 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:06:59.896 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:59.907 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:06:59 smithi173 bash[58816]: cluster 2023-10-01T11:06:58.315720+0000 mgr.y (mgr.25072) 1489 : cluster 0 pgmap v1592: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 146 op/s; 13708/187080 objects degraded (7.327%); 5449/187080 objects misplaced (2.913%); 5.0 MiB/s, 79 objects/s recovering 2023-10-01T11:06:59.934 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:06:59.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:06:59.942 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:07:00.043 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:00.053 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:06:59 smithi078 bash[75299]: cluster 2023-10-01T11:06:58.315720+0000 mgr.y (mgr.25072) 1489 : cluster 0 pgmap v1592: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 146 op/s; 13708/187080 objects degraded (7.327%); 5449/187080 objects misplaced (2.913%); 5.0 MiB/s, 79 objects/s recovering 2023-10-01T11:07:00.053 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:06:59 smithi078 bash[78274]: cluster 2023-10-01T11:06:58.315720+0000 mgr.y (mgr.25072) 1489 : cluster 0 pgmap v1592: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 146 op/s; 13708/187080 objects degraded (7.327%); 5449/187080 objects misplaced (2.913%); 5.0 MiB/s, 79 objects/s recovering 2023-10-01T11:07:00.068 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:00.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:00.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:07:00.171 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:00.194 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:00.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:00.196 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:07:00.297 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:00.331 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:07:00 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:07:00] "GET /metrics HTTP/1.1" 200 36408 "" "Prometheus/2.33.4" 2023-10-01T11:07:00.351 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:00.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:00.355 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:07:00.456 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:00.478 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:00.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:00.480 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:07:00.581 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:00.605 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:00.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:00.613 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:07:00.714 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:00.742 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:00.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:00.744 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:07:00.845 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:00.869 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:00.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:00.874 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:07:00.975 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:01.006 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:01.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:01.008 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:07:01.109 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:01.135 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:01.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:01.139 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:07:01.240 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:01.267 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:01.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:01.270 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:07:01.370 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:01.397 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:01.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:01.399 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:07:01.500 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:01.544 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:01.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:01.547 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:07:01.648 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:01.674 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:01.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:01.681 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:07:01.782 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:01.804 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:01 smithi173 bash[58816]: cluster 2023-10-01T11:07:00.316498+0000 mgr.y (mgr.25072) 1490 : cluster 0 pgmap v1593: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 254 op/s; 13584/183441 objects degraded (7.405%); 4994/183441 objects misplaced (2.722%); 10 MiB/s, 166 objects/s recovering 2023-10-01T11:07:01.808 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:01.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:01.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:07:01.912 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:01.937 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:01.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:01.942 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:07:02.043 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:02.052 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:01 smithi078 bash[78274]: cluster 2023-10-01T11:07:00.316498+0000 mgr.y (mgr.25072) 1490 : cluster 0 pgmap v1593: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 254 op/s; 13584/183441 objects degraded (7.405%); 4994/183441 objects misplaced (2.722%); 10 MiB/s, 166 objects/s recovering 2023-10-01T11:07:02.052 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:01 smithi078 bash[75299]: cluster 2023-10-01T11:07:00.316498+0000 mgr.y (mgr.25072) 1490 : cluster 0 pgmap v1593: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 254 op/s; 13584/183441 objects degraded (7.405%); 4994/183441 objects misplaced (2.722%); 10 MiB/s, 166 objects/s recovering 2023-10-01T11:07:02.068 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:02.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:02.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:07:02.175 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:02.201 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:02.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:02.203 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:07:02.304 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:02.347 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:02.348 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T11:07:02.349 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T11:07:02.349 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2023-10-01T11:07:02.349 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-10-01T11:07:02.349 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-10-01T11:07:02.349 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 config set filestore_inject_stall 3 2023-10-01T11:07:02.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:02.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:07:02.452 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:02.482 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:02.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:02.486 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:07:02.586 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:02.611 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:02.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:02.615 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:07:02.716 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:02.754 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:02.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:02.768 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:07:02.869 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:02.897 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:02.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:02.898 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:07:02.999 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:03.033 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T11:07:03.043 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:03.046 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2023-10-01T11:07:03.046 INFO:teuthology.orchestra.run.smithi078.stdout: "ops": [ 2023-10-01T11:07:03.046 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:07:03.046 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.46454.0:100784 13.3 e340/317 13:cacbb615:::benchmark_data_smithi078_202524_object21226:head v 340'6306, mlcod=340'6304)", 2023-10-01T11:07:03.047 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:07:03.041415+0000", 2023-10-01T11:07:03.047 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 0.003964436, 2023-10-01T11:07:03.047 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0039915039999999999, 2023-10-01T11:07:03.047 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:07:03.047 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:07:03.047 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:07:03.047 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:07:03.047 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:07:03.047 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:07:03.041415+0000", 2023-10-01T11:07:03.047 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:07:03.047 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:07:03.047 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:07:03.048 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:07:03.048 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:07:03.041415+0000", 2023-10-01T11:07:03.048 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:07:03.048 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:07:03.048 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:07:03.048 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:07:03.048 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:07:03.041414+0000", 2023-10-01T11:07:03.048 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999986 2023-10-01T11:07:03.048 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:07:03.048 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:07:03.048 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:07:03.048 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:07:03.041426+0000", 2023-10-01T11:07:03.049 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1883e-05 2023-10-01T11:07:03.049 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:07:03.049 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:07:03.049 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:07:03.049 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:07:03.041449+0000", 2023-10-01T11:07:03.049 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.2948000000000001e-05 2023-10-01T11:07:03.049 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:07:03.049 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:07:03.049 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:07:03.049 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:07:03.041459+0000", 2023-10-01T11:07:03.049 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.0631999999999999e-05 2023-10-01T11:07:03.049 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:07:03.050 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:07:03.050 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:07:03.050 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:07:03.041553+0000", 2023-10-01T11:07:03.050 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.4061999999999995e-05 2023-10-01T11:07:03.050 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:07:03.050 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:07:03.050 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:07:03.050 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:07:03.041592+0000", 2023-10-01T11:07:03.050 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.8988e-05 2023-10-01T11:07:03.050 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:07:03.050 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:07:03.050 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:07:03.051 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:07:03.045340+0000", 2023-10-01T11:07:03.051 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0037474700000000001 2023-10-01T11:07:03.051 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:07:03.051 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:07:03.051 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:07:03.051 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:07:03.051 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2023-10-01T11:07:03.051 INFO:teuthology.orchestra.run.smithi078.stdout: "num_ops": 1 2023-10-01T11:07:03.051 INFO:teuthology.orchestra.run.smithi078.stdout:} 2023-10-01T11:07:03.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:03.052 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:07:03.062 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T11:07:03.101 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T11:07:03.130 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T11:07:03.154 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:03.174 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T11:07:03.186 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:03.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:03.190 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:07:03.221 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T11:07:03.255 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T11:07:03.284 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T11:07:03.286 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T11:07:03.286 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T11:07:03.286 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T11:07:03.286 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T11:07:03.286 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T11:07:03.287 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T11:07:03.287 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T11:07:03.287 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T11:07:03.290 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:03.316 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:03.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:03.321 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:07:03.421 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:03.444 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:03.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:03.448 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:07:03.466 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:03 smithi173 bash[58816]: cluster 2023-10-01T11:07:02.317109+0000 mgr.y (mgr.25072) 1491 : cluster 0 pgmap v1594: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 179 op/s; 13584/183441 objects degraded (7.405%); 4994/183441 objects misplaced (2.722%); 4.8 MiB/s, 76 objects/s recovering 2023-10-01T11:07:03.466 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:03 smithi173 bash[58816]: audit 2023-10-01T11:07:02.465670+0000 mon.a (mon.0) 1924 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:07:03.466 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:03 smithi173 bash[58816]: audit 2023-10-01T11:07:02.466793+0000 mon.a (mon.0) 1925 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:07:03.466 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:03 smithi173 bash[58816]: cluster 2023-10-01T11:07:02.761334+0000 mon.a (mon.0) 1926 : cluster 3 Health check update: Degraded data redundancy: 13584/183441 objects degraded (7.405%), 5 pgs degraded (PG_DEGRADED) 2023-10-01T11:07:03.466 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:03 smithi173 bash[58816]: audit 2023-10-01T11:07:02.989211+0000 mon.a (mon.0) 1927 : audit 0 from='client.? 172.21.15.78:0/1373786285' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:07:03.549 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:03.560 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:03 smithi078 bash[78274]: cluster 2023-10-01T11:07:02.317109+0000 mgr.y (mgr.25072) 1491 : cluster 0 pgmap v1594: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 179 op/s; 13584/183441 objects degraded (7.405%); 4994/183441 objects misplaced (2.722%); 4.8 MiB/s, 76 objects/s recovering 2023-10-01T11:07:03.560 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:03 smithi078 bash[78274]: audit 2023-10-01T11:07:02.465670+0000 mon.a (mon.0) 1924 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:07:03.560 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:03 smithi078 bash[78274]: audit 2023-10-01T11:07:02.466793+0000 mon.a (mon.0) 1925 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:07:03.560 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:03 smithi078 bash[78274]: cluster 2023-10-01T11:07:02.761334+0000 mon.a (mon.0) 1926 : cluster 3 Health check update: Degraded data redundancy: 13584/183441 objects degraded (7.405%), 5 pgs degraded (PG_DEGRADED) 2023-10-01T11:07:03.560 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:03 smithi078 bash[78274]: audit 2023-10-01T11:07:02.989211+0000 mon.a (mon.0) 1927 : audit 0 from='client.? 172.21.15.78:0/1373786285' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:07:03.561 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:03 smithi078 bash[75299]: cluster 2023-10-01T11:07:02.317109+0000 mgr.y (mgr.25072) 1491 : cluster 0 pgmap v1594: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 179 op/s; 13584/183441 objects degraded (7.405%); 4994/183441 objects misplaced (2.722%); 4.8 MiB/s, 76 objects/s recovering 2023-10-01T11:07:03.561 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:03 smithi078 bash[75299]: audit 2023-10-01T11:07:02.465670+0000 mon.a (mon.0) 1924 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:07:03.561 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:03 smithi078 bash[75299]: audit 2023-10-01T11:07:02.466793+0000 mon.a (mon.0) 1925 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:07:03.561 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:03 smithi078 bash[75299]: cluster 2023-10-01T11:07:02.761334+0000 mon.a (mon.0) 1926 : cluster 3 Health check update: Degraded data redundancy: 13584/183441 objects degraded (7.405%), 5 pgs degraded (PG_DEGRADED) 2023-10-01T11:07:03.561 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:03 smithi078 bash[75299]: audit 2023-10-01T11:07:02.989211+0000 mon.a (mon.0) 1927 : audit 0 from='client.? 172.21.15.78:0/1373786285' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:07:03.578 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:03.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:03.581 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:07:03.682 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:03.707 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:03.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:03.710 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:07:03.811 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:03.840 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:03.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:03.844 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:07:03.945 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:03.985 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:03.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:03.994 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:07:04.095 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:04.145 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:04.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:04.149 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:07:04.250 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:04.277 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:04.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:04.280 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:07:04.381 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:04.403 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:04.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:04.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:07:04.507 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:04.547 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:04.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:04.556 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:07:04.657 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:04.683 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:04.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:04.687 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:07:04.788 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:04.817 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:04.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:04.820 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:07:04.921 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:04.955 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:04.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:04.958 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:07:05.059 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:05.090 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:05.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:05.093 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:07:05.193 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:05.218 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:05.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:05.223 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:07:05.247 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-01T11:07:05.323 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:05.357 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:05.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:05.364 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:07:05.466 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:05.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:05.517 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:05.517 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:07:05.618 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:05.645 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:05.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:05.647 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:07:05.748 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:05.756 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:05 smithi078 bash[78274]: cluster 2023-10-01T11:07:04.317808+0000 mgr.y (mgr.25072) 1492 : cluster 0 pgmap v1595: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 424 op/s; 13584/176625 objects degraded (7.691%); 4756/176625 objects misplaced (2.693%); 4.2 MiB/s, 67 objects/s recovering 2023-10-01T11:07:05.757 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:05 smithi078 bash[75299]: cluster 2023-10-01T11:07:04.317808+0000 mgr.y (mgr.25072) 1492 : cluster 0 pgmap v1595: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 424 op/s; 13584/176625 objects degraded (7.691%); 4756/176625 objects misplaced (2.693%); 4.2 MiB/s, 67 objects/s recovering 2023-10-01T11:07:05.772 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:05.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:05.777 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:07:05.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:05 smithi173 bash[58816]: cluster 2023-10-01T11:07:04.317808+0000 mgr.y (mgr.25072) 1492 : cluster 0 pgmap v1595: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 424 op/s; 13584/176625 objects degraded (7.691%); 4756/176625 objects misplaced (2.693%); 4.2 MiB/s, 67 objects/s recovering 2023-10-01T11:07:05.875 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:07:05.877 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:06.090 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:06.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:06.094 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:07:06.194 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:06.220 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:06.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:06.222 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:07:06.323 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:06.360 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:06.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:06.363 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:07:06.464 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:06.487 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:06.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:06.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:07:06.590 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:06.615 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:06.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:06.618 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:07:06.719 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:06.744 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:06.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:06.747 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:07:06.848 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:06.876 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:06.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:06.879 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:07:06.979 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:07.005 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:07.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:07.008 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:07:07.109 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:07.137 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:07.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:07.143 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:07:07.244 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:07.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:07.274 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:07.274 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:07:07.375 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:07.399 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:07.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:07.404 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:07:07.459 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T11:07:07.505 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:07.527 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:07.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:07.529 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:07:07.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:07 smithi078 bash[75299]: cluster 2023-10-01T11:07:06.318487+0000 mgr.y (mgr.25072) 1493 : cluster 0 pgmap v1596: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 459 op/s; 13376/173712 objects degraded (7.700%); 4558/173712 objects misplaced (2.624%); 6.2 MiB/s, 98 objects/s recovering 2023-10-01T11:07:07.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:07 smithi078 bash[78274]: cluster 2023-10-01T11:07:06.318487+0000 mgr.y (mgr.25072) 1493 : cluster 0 pgmap v1596: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 459 op/s; 13376/173712 objects degraded (7.700%); 4558/173712 objects misplaced (2.624%); 6.2 MiB/s, 98 objects/s recovering 2023-10-01T11:07:07.630 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:07.659 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:07.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:07.663 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:07:07.764 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:07.793 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:07.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:07.796 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:07:07.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:07 smithi173 bash[58816]: cluster 2023-10-01T11:07:06.318487+0000 mgr.y (mgr.25072) 1493 : cluster 0 pgmap v1596: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 459 op/s; 13376/173712 objects degraded (7.700%); 4558/173712 objects misplaced (2.624%); 6.2 MiB/s, 98 objects/s recovering 2023-10-01T11:07:07.897 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:07.927 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:07.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:07.930 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:07:08.031 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:08.060 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:08.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:08.062 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:07:08.162 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:08.189 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:08.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:08.192 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:07:08.293 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:08.328 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:08.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:08.335 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:07:08.436 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:08.462 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:08.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:08.464 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:07:08.565 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:08.587 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:08.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:08.594 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:07:08.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:08 smithi173 bash[58816]: cluster 2023-10-01T11:07:07.763526+0000 mon.a (mon.0) 1928 : cluster 3 Health check update: Degraded data redundancy: 13376/173712 objects degraded (7.700%), 5 pgs degraded (PG_DEGRADED) 2023-10-01T11:07:08.695 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:08.706 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:08 smithi078 bash[78274]: cluster 2023-10-01T11:07:07.763526+0000 mon.a (mon.0) 1928 : cluster 3 Health check update: Degraded data redundancy: 13376/173712 objects degraded (7.700%), 5 pgs degraded (PG_DEGRADED) 2023-10-01T11:07:08.706 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:08 smithi078 bash[75299]: cluster 2023-10-01T11:07:07.763526+0000 mon.a (mon.0) 1928 : cluster 3 Health check update: Degraded data redundancy: 13376/173712 objects degraded (7.700%), 5 pgs degraded (PG_DEGRADED) 2023-10-01T11:07:08.734 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:08.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:08.738 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:07:08.838 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:08.862 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:08.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:08.865 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:07:08.966 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:08.994 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:08.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:08.999 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:07:09.100 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:09.128 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:09.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:09.134 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:07:09.177 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:07:09.235 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:09.277 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:09.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:09.279 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:07:09.380 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:09.428 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:09.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:09.431 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:07:09.531 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:09.573 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:09.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:09.578 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:07:09.678 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:09.703 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:09.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:09.706 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:07:09.771 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:09 smithi173 bash[58816]: cluster 2023-10-01T11:07:08.319014+0000 mgr.y (mgr.25072) 1494 : cluster 0 pgmap v1597: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 455 op/s; 13376/171744 objects degraded (7.788%); 4346/171744 objects misplaced (2.531%); 5.3 MiB/s, 85 objects/s recovering 2023-10-01T11:07:09.806 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:09.840 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:09.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:09.846 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:07:09.947 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:09.955 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:09 smithi078 bash[78274]: cluster 2023-10-01T11:07:08.319014+0000 mgr.y (mgr.25072) 1494 : cluster 0 pgmap v1597: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 455 op/s; 13376/171744 objects degraded (7.788%); 4346/171744 objects misplaced (2.531%); 5.3 MiB/s, 85 objects/s recovering 2023-10-01T11:07:09.955 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:09 smithi078 bash[75299]: cluster 2023-10-01T11:07:08.319014+0000 mgr.y (mgr.25072) 1494 : cluster 0 pgmap v1597: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 455 op/s; 13376/171744 objects degraded (7.788%); 4346/171744 objects misplaced (2.531%); 5.3 MiB/s, 85 objects/s recovering 2023-10-01T11:07:09.969 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:09.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:09.972 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:07:10.073 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:10.099 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:10.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:10.104 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:07:10.205 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:10.225 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:10.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:10.232 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:07:10.332 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:10.372 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:10.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:10.374 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:07:10.475 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:10.528 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:10.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:10.531 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:07:10.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:07:10 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:07:10] "GET /metrics HTTP/1.1" 200 36407 "" "Prometheus/2.33.4" 2023-10-01T11:07:10.632 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:10.666 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:10.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:10.672 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:07:10.773 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:10.798 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:10.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:10.802 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:07:10.903 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:10.934 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:10.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:10.937 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:07:10.992 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:07:11.038 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:11.059 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:11.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:11.064 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:07:11.165 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:11.196 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:11.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:11.201 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:07:11.302 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:11.327 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:11.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:11.333 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:07:11.434 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:11.459 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:11.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:11.462 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:07:11.562 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:11.585 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:11.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:11.590 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:07:11.691 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:11.717 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:11.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:11.719 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:07:11.819 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:11.855 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:11.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:11.859 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:07:11.960 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:11.987 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:11.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:11.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:07:11.990 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:11 smithi078 bash[78274]: cluster 2023-10-01T11:07:10.319644+0000 mgr.y (mgr.25072) 1495 : cluster 0 pgmap v1598: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 646 op/s; 13171/163794 objects degraded (8.041%); 4128/163794 objects misplaced (2.520%); 7.3 MiB/s, 117 objects/s recovering 2023-10-01T11:07:11.990 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:11 smithi078 bash[75299]: cluster 2023-10-01T11:07:10.319644+0000 mgr.y (mgr.25072) 1495 : cluster 0 pgmap v1598: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 646 op/s; 13171/163794 objects degraded (8.041%); 4128/163794 objects misplaced (2.520%); 7.3 MiB/s, 117 objects/s recovering 2023-10-01T11:07:12.013 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:07:12.090 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:12.098 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:11 smithi173 bash[58816]: cluster 2023-10-01T11:07:10.319644+0000 mgr.y (mgr.25072) 1495 : cluster 0 pgmap v1598: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 646 op/s; 13171/163794 objects degraded (8.041%); 4128/163794 objects misplaced (2.520%); 7.3 MiB/s, 117 objects/s recovering 2023-10-01T11:07:12.124 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:12.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:12.129 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:07:12.230 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:12.257 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:12.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:12.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:07:12.364 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:12.388 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:12.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:12.391 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:07:12.492 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:12.526 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:12.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:12.532 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:07:12.633 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:12.667 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:12.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:12.671 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:07:12.772 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:12.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:12 smithi173 bash[58816]: audit 2023-10-01T11:07:11.890684+0000 mon.c (mon.2) 227 : audit 1 from='client.? 172.21.15.78:0/330847985' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T11:07:12.777 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:12 smithi173 bash[58816]: audit 2023-10-01T11:07:11.891046+0000 mon.a (mon.0) 1929 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T11:07:12.793 INFO:teuthology.orchestra.run.smithi078.stderr:noscrub is set 2023-10-01T11:07:12.805 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:12.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:12.812 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:07:12.913 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:12.941 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:12.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:12.944 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:07:13.045 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:13.069 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:12 smithi173 bash[58816]: cluster 2023-10-01T11:07:12.765170+0000 mon.a (mon.0) 1930 : cluster 3 Health check update: Degraded data redundancy: 13171/163794 objects degraded (8.041%), 5 pgs degraded (PG_DEGRADED) 2023-10-01T11:07:13.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:12 smithi078 bash[78274]: audit 2023-10-01T11:07:11.890684+0000 mon.c (mon.2) 227 : audit 1 from='client.? 172.21.15.78:0/330847985' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T11:07:13.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:12 smithi078 bash[78274]: audit 2023-10-01T11:07:11.891046+0000 mon.a (mon.0) 1929 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T11:07:13.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:12 smithi078 bash[78274]: cluster 2023-10-01T11:07:12.765170+0000 mon.a (mon.0) 1930 : cluster 3 Health check update: Degraded data redundancy: 13171/163794 objects degraded (8.041%), 5 pgs degraded (PG_DEGRADED) 2023-10-01T11:07:13.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:12 smithi078 bash[75299]: audit 2023-10-01T11:07:11.890684+0000 mon.c (mon.2) 227 : audit 1 from='client.? 172.21.15.78:0/330847985' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T11:07:13.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:12 smithi078 bash[75299]: audit 2023-10-01T11:07:11.891046+0000 mon.a (mon.0) 1929 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T11:07:13.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:12 smithi078 bash[75299]: cluster 2023-10-01T11:07:12.765170+0000 mon.a (mon.0) 1930 : cluster 3 Health check update: Degraded data redundancy: 13171/163794 objects degraded (8.041%), 5 pgs degraded (PG_DEGRADED) 2023-10-01T11:07:13.090 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:13.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:13.098 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:07:13.199 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:13.236 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:13.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:13.238 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:07:13.339 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:13.396 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:13.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:13.402 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:07:13.503 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:13.529 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:13.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:13.536 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:07:13.636 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:13.658 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:13.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:13.664 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:07:13.764 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:13.810 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:13.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:13.814 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:07:13.915 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:13.937 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:13.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:13.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:07:14.040 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:14.048 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:13 smithi078 bash[78274]: cluster 2023-10-01T11:07:12.320131+0000 mgr.y (mgr.25072) 1496 : cluster 0 pgmap v1599: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 545 op/s; 13171/163794 objects degraded (8.041%); 4128/163794 objects misplaced (2.520%); 4.3 MiB/s, 69 objects/s recovering 2023-10-01T11:07:14.048 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:13 smithi078 bash[78274]: cluster 2023-10-01T11:07:12.774161+0000 mon.a (mon.0) 1931 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:07:14.048 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:13 smithi078 bash[78274]: cluster 2023-10-01T11:07:12.784219+0000 osd.5 (osd.5) 360 : cluster 3 failed to encode map e341 with expected crc 2023-10-01T11:07:14.048 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:13 smithi078 bash[78274]: audit 2023-10-01T11:07:12.786475+0000 mon.a (mon.0) 1932 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T11:07:14.048 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:13 smithi078 bash[78274]: cluster 2023-10-01T11:07:12.786567+0000 mon.a (mon.0) 1933 : cluster 0 osdmap e341: 8 total, 8 up, 8 in 2023-10-01T11:07:14.048 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:13 smithi078 bash[78274]: cluster 2023-10-01T11:07:12.802651+0000 osd.6 (osd.6) 327 : cluster 3 failed to encode map e341 with expected crc 2023-10-01T11:07:14.049 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:13 smithi078 bash[78274]: cluster 2023-10-01T11:07:12.809141+0000 osd.7 (osd.7) 375 : cluster 3 failed to encode map e341 with expected crc 2023-10-01T11:07:14.049 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:13 smithi078 bash[75299]: cluster 2023-10-01T11:07:12.320131+0000 mgr.y (mgr.25072) 1496 : cluster 0 pgmap v1599: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 545 op/s; 13171/163794 objects degraded (8.041%); 4128/163794 objects misplaced (2.520%); 4.3 MiB/s, 69 objects/s recovering 2023-10-01T11:07:14.049 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:13 smithi078 bash[75299]: cluster 2023-10-01T11:07:12.774161+0000 mon.a (mon.0) 1931 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:07:14.049 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:13 smithi078 bash[75299]: cluster 2023-10-01T11:07:12.784219+0000 osd.5 (osd.5) 360 : cluster 3 failed to encode map e341 with expected crc 2023-10-01T11:07:14.049 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:13 smithi078 bash[75299]: audit 2023-10-01T11:07:12.786475+0000 mon.a (mon.0) 1932 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T11:07:14.049 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:13 smithi078 bash[75299]: cluster 2023-10-01T11:07:12.786567+0000 mon.a (mon.0) 1933 : cluster 0 osdmap e341: 8 total, 8 up, 8 in 2023-10-01T11:07:14.049 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:13 smithi078 bash[75299]: cluster 2023-10-01T11:07:12.802651+0000 osd.6 (osd.6) 327 : cluster 3 failed to encode map e341 with expected crc 2023-10-01T11:07:14.049 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:13 smithi078 bash[75299]: cluster 2023-10-01T11:07:12.809141+0000 osd.7 (osd.7) 375 : cluster 3 failed to encode map e341 with expected crc 2023-10-01T11:07:14.060 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:14.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:14.063 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:07:14.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:13 smithi173 bash[58816]: cluster 2023-10-01T11:07:12.320131+0000 mgr.y (mgr.25072) 1496 : cluster 0 pgmap v1599: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 545 op/s; 13171/163794 objects degraded (8.041%); 4128/163794 objects misplaced (2.520%); 4.3 MiB/s, 69 objects/s recovering 2023-10-01T11:07:14.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:13 smithi173 bash[58816]: cluster 2023-10-01T11:07:12.774161+0000 mon.a (mon.0) 1931 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:07:14.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:13 smithi173 bash[58816]: cluster 2023-10-01T11:07:12.784219+0000 osd.5 (osd.5) 360 : cluster 3 failed to encode map e341 with expected crc 2023-10-01T11:07:14.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:13 smithi173 bash[58816]: audit 2023-10-01T11:07:12.786475+0000 mon.a (mon.0) 1932 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T11:07:14.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:13 smithi173 bash[58816]: cluster 2023-10-01T11:07:12.786567+0000 mon.a (mon.0) 1933 : cluster 0 osdmap e341: 8 total, 8 up, 8 in 2023-10-01T11:07:14.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:13 smithi173 bash[58816]: cluster 2023-10-01T11:07:12.802651+0000 osd.6 (osd.6) 327 : cluster 3 failed to encode map e341 with expected crc 2023-10-01T11:07:14.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:13 smithi173 bash[58816]: cluster 2023-10-01T11:07:12.809141+0000 osd.7 (osd.7) 375 : cluster 3 failed to encode map e341 with expected crc 2023-10-01T11:07:14.163 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:14.213 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:14.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:14.220 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:07:14.320 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:14.349 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:14.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:14.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:07:14.452 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:14.471 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:14.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:14.475 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:07:14.576 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:14.602 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:14.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:14.607 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:07:14.708 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:14.752 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:14.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:14.758 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:07:14.859 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:14.888 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:14.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:14.890 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:07:14.991 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:15.025 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:15.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:15.029 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:07:15.130 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:15.164 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:15.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:15.166 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:07:15.267 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:15.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:15.311 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:15.311 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:07:15.412 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:15.446 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:15.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:15.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:07:15.552 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:15.590 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:15.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:15.596 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:07:15.697 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:15.727 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:15.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:15.732 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:07:15.832 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:15.854 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:15.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:15.856 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:07:15.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:15 smithi173 bash[58816]: cluster 2023-10-01T11:07:14.320838+0000 mgr.y (mgr.25072) 1497 : cluster 0 pgmap v1601: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 13171/157332 objects degraded (8.371%); 3923/157332 objects misplaced (2.493%); 5.2 MiB/s, 82 objects/s recovering 2023-10-01T11:07:15.957 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:16.014 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:16.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:16.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:07:16.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:15 smithi078 bash[75299]: cluster 2023-10-01T11:07:14.320838+0000 mgr.y (mgr.25072) 1497 : cluster 0 pgmap v1601: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 13171/157332 objects degraded (8.371%); 3923/157332 objects misplaced (2.493%); 5.2 MiB/s, 82 objects/s recovering 2023-10-01T11:07:16.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:15 smithi078 bash[78274]: cluster 2023-10-01T11:07:14.320838+0000 mgr.y (mgr.25072) 1497 : cluster 0 pgmap v1601: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 13171/157332 objects degraded (8.371%); 3923/157332 objects misplaced (2.493%); 5.2 MiB/s, 82 objects/s recovering 2023-10-01T11:07:16.117 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:16.147 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:16.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:16.150 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:07:16.251 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:16.289 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:16.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:16.295 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:07:16.395 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:16.428 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:16.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:16.431 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:07:16.531 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:16.556 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:16.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:16.559 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:07:16.660 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:16.723 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:16.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:16.726 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:07:16.826 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:16.848 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:16.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:16.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:07:16.951 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:16.980 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:16.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:16.989 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:07:17.089 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:17.123 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:17.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:17.127 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:07:17.227 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:17.251 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:17.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:17.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:07:17.434 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:17.461 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:17.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:17.466 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:07:17.567 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:17.598 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:17.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:17.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:07:17.704 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:17.737 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:17.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:17.742 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:07:17.843 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:17.873 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:17.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:17.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:07:17.978 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:17.987 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:17 smithi173 bash[58816]: cluster 2023-10-01T11:07:16.321592+0000 mgr.y (mgr.25072) 1498 : cluster 0 pgmap v1602: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 12971/154356 objects degraded (8.403%); 3683/154356 objects misplaced (2.386%); 5.4 MiB/s, 86 objects/s recovering 2023-10-01T11:07:17.987 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:17 smithi173 bash[58816]: audit 2023-10-01T11:07:17.468751+0000 mon.a (mon.0) 1934 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:07:17.987 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:17 smithi173 bash[58816]: audit 2023-10-01T11:07:17.470155+0000 mon.a (mon.0) 1935 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:07:17.987 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:17 smithi173 bash[58816]: cluster 2023-10-01T11:07:17.766320+0000 mon.a (mon.0) 1936 : cluster 3 Health check update: Degraded data redundancy: 12971/154356 objects degraded (8.403%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T11:07:18.003 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:18.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:18.006 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:07:18.077 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:17 smithi078 bash[75299]: cluster 2023-10-01T11:07:16.321592+0000 mgr.y (mgr.25072) 1498 : cluster 0 pgmap v1602: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 12971/154356 objects degraded (8.403%); 3683/154356 objects misplaced (2.386%); 5.4 MiB/s, 86 objects/s recovering 2023-10-01T11:07:18.077 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:17 smithi078 bash[75299]: audit 2023-10-01T11:07:17.468751+0000 mon.a (mon.0) 1934 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:07:18.077 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:17 smithi078 bash[75299]: audit 2023-10-01T11:07:17.470155+0000 mon.a (mon.0) 1935 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:07:18.077 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:17 smithi078 bash[75299]: cluster 2023-10-01T11:07:17.766320+0000 mon.a (mon.0) 1936 : cluster 3 Health check update: Degraded data redundancy: 12971/154356 objects degraded (8.403%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T11:07:18.077 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:17 smithi078 bash[78274]: cluster 2023-10-01T11:07:16.321592+0000 mgr.y (mgr.25072) 1498 : cluster 0 pgmap v1602: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 12971/154356 objects degraded (8.403%); 3683/154356 objects misplaced (2.386%); 5.4 MiB/s, 86 objects/s recovering 2023-10-01T11:07:18.077 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:17 smithi078 bash[78274]: audit 2023-10-01T11:07:17.468751+0000 mon.a (mon.0) 1934 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:07:18.077 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:17 smithi078 bash[78274]: audit 2023-10-01T11:07:17.470155+0000 mon.a (mon.0) 1935 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:07:18.077 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:17 smithi078 bash[78274]: cluster 2023-10-01T11:07:17.766320+0000 mon.a (mon.0) 1936 : cluster 3 Health check update: Degraded data redundancy: 12971/154356 objects degraded (8.403%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T11:07:18.107 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:18.133 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:18.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:18.135 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:07:18.236 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:18.267 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:18.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:18.273 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:07:18.341 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T11:07:18.374 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:18.415 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:18.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:18.418 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:07:18.518 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:18.555 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:18.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:18.561 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:07:18.662 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:18.713 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:18.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:18.721 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:07:18.822 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:18.859 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:18.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:18.862 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:07:18.964 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:18.999 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:19.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:19.007 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:07:19.108 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:19.139 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:19.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:19.142 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:07:19.243 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:19.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:19.303 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:19.303 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:07:19.404 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:19.431 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:19.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:19.437 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:07:19.538 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:19.568 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:19.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:19.572 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:07:19.673 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:19.701 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:19.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:19.704 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:07:19.805 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:19.813 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:19 smithi173 bash[58816]: cluster 2023-10-01T11:07:18.322359+0000 mgr.y (mgr.25072) 1499 : cluster 0 pgmap v1603: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 639 op/s; 12971/152544 objects degraded (8.503%); 3484/152544 objects misplaced (2.284%); 5.4 MiB/s, 86 objects/s recovering 2023-10-01T11:07:19.831 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:19.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:19.834 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:07:19.935 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:19.972 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:19.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:19.975 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:07:20.075 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:20.080 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:19 smithi078 bash[75299]: cluster 2023-10-01T11:07:18.322359+0000 mgr.y (mgr.25072) 1499 : cluster 0 pgmap v1603: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 639 op/s; 12971/152544 objects degraded (8.503%); 3484/152544 objects misplaced (2.284%); 5.4 MiB/s, 86 objects/s recovering 2023-10-01T11:07:20.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:19 smithi078 bash[78274]: cluster 2023-10-01T11:07:18.322359+0000 mgr.y (mgr.25072) 1499 : cluster 0 pgmap v1603: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.1 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 639 op/s; 12971/152544 objects degraded (8.503%); 3484/152544 objects misplaced (2.284%); 5.4 MiB/s, 86 objects/s recovering 2023-10-01T11:07:20.117 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:20.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:20.119 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:07:20.220 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:20.260 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:20.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:20.264 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:07:20.364 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:20.392 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:20.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:20.396 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:07:20.497 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:20.526 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:20.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:20.531 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:07:20.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:07:20 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:07:20] "GET /metrics HTTP/1.1" 200 36405 "" "Prometheus/2.33.4" 2023-10-01T11:07:20.632 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:20.671 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:20.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:20.675 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:07:20.776 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:20.802 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:20.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:20.805 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:07:20.905 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:20.931 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:20.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:20.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:07:21.035 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:21.077 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:21.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:21.082 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:07:21.184 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:21.234 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:21.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:21.240 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:07:21.278 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2023-10-01T11:07:21.279 INFO:teuthology.orchestra.run.smithi078.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-10-01T11:07:21.279 INFO:teuthology.orchestra.run.smithi078.stdout:} 2023-10-01T11:07:21.341 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:21.384 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:21.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:21.388 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:07:21.489 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:21.518 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:21.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:21.519 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:07:21.620 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:21.660 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:21.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:21.663 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:07:21.764 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:21.787 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:21.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:21.792 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:07:21.893 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:21.921 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:21.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:21.926 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:07:22.027 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:22.050 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:22.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:22.054 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:07:22.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:21 smithi173 bash[58816]: cluster 2023-10-01T11:07:20.323147+0000 mgr.y (mgr.25072) 1500 : cluster 0 pgmap v1604: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 12771/145074 objects degraded (8.803%); 3245/145074 objects misplaced (2.237%); 5.5 MiB/s, 87 objects/s recovering 2023-10-01T11:07:22.155 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:22.191 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:22.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:22.198 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:07:22.229 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:21 smithi078 bash[78274]: cluster 2023-10-01T11:07:20.323147+0000 mgr.y (mgr.25072) 1500 : cluster 0 pgmap v1604: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 12771/145074 objects degraded (8.803%); 3245/145074 objects misplaced (2.237%); 5.5 MiB/s, 87 objects/s recovering 2023-10-01T11:07:22.230 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:21 smithi078 bash[75299]: cluster 2023-10-01T11:07:20.323147+0000 mgr.y (mgr.25072) 1500 : cluster 0 pgmap v1604: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 12771/145074 objects degraded (8.803%); 3245/145074 objects misplaced (2.237%); 5.5 MiB/s, 87 objects/s recovering 2023-10-01T11:07:22.299 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:22.322 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:22.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:22.324 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:07:22.424 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:22.449 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:22.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:22.452 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:07:22.553 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:22.575 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:22.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:22.580 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:07:22.681 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:22.703 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:22.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:22.706 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:07:22.802 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:22 smithi078 bash[75299]: cluster 2023-10-01T11:07:22.767306+0000 mon.a (mon.0) 1937 : cluster 3 Health check update: Degraded data redundancy: 12771/145074 objects degraded (8.803%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T11:07:22.802 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:22 smithi078 bash[78274]: cluster 2023-10-01T11:07:22.767306+0000 mon.a (mon.0) 1937 : cluster 3 Health check update: Degraded data redundancy: 12771/145074 objects degraded (8.803%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T11:07:22.807 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:22.827 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:22.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:22.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:07:22.932 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:22.955 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:22.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:22.959 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:07:23.059 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:23.078 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:23.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:23.080 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:07:23.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:22 smithi173 bash[58816]: cluster 2023-10-01T11:07:22.767306+0000 mon.a (mon.0) 1937 : cluster 3 Health check update: Degraded data redundancy: 12771/145074 objects degraded (8.803%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T11:07:23.181 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:23.212 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:23.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:23.217 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:07:23.318 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:23.347 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:23.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:23.350 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:07:23.450 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:23.470 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:23.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:23.472 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:07:23.572 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:23.616 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:23.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:23.622 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:07:23.723 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:23.759 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:23.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:23.762 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:07:23.803 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:23 smithi078 bash[75299]: cluster 2023-10-01T11:07:22.323659+0000 mgr.y (mgr.25072) 1501 : cluster 0 pgmap v1605: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 12771/145074 objects degraded (8.803%); 3245/145074 objects misplaced (2.237%); 5.5 MiB/s, 87 objects/s recovering 2023-10-01T11:07:23.864 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:23.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:23 smithi173 bash[58816]: cluster 2023-10-01T11:07:22.323659+0000 mgr.y (mgr.25072) 1501 : cluster 0 pgmap v1605: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 12771/145074 objects degraded (8.803%); 3245/145074 objects misplaced (2.237%); 5.5 MiB/s, 87 objects/s recovering 2023-10-01T11:07:23.892 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:23.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:23.896 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:07:23.997 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:24.017 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:23 smithi078 bash[78274]: cluster 2023-10-01T11:07:22.323659+0000 mgr.y (mgr.25072) 1501 : cluster 0 pgmap v1605: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 12771/145074 objects degraded (8.803%); 3245/145074 objects misplaced (2.237%); 5.5 MiB/s, 87 objects/s recovering 2023-10-01T11:07:24.032 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:24.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:24.037 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:07:24.138 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:24.170 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:24.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:24.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:07:24.280 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:24.310 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:24.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:24.315 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:07:24.416 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:24.460 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:24.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:24.469 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:07:24.570 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:24.594 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:24.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:24.599 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:07:24.700 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:24.743 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:24.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:24.749 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:07:24.850 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:24.887 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:24.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:24.891 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:07:24.991 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:25.029 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:25.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:25.033 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:07:25.134 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:25.163 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:25.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:25.172 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:07:25.273 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:25.306 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:25.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:25.310 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:07:25.411 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:25.450 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:25.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:25.453 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:07:25.534 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:07:25.540 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2023-10-01T11:07:25.541 INFO:teuthology.orchestra.run.smithi078.stdout: "ops": [], 2023-10-01T11:07:25.541 INFO:teuthology.orchestra.run.smithi078.stdout: "complaint_time": 30, 2023-10-01T11:07:25.541 INFO:teuthology.orchestra.run.smithi078.stdout: "num_blocked_ops": 0 2023-10-01T11:07:25.541 INFO:teuthology.orchestra.run.smithi078.stdout:} 2023-10-01T11:07:25.554 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:25.555 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T11:07:25.586 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T11:07:25.597 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:25.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:25.599 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:07:25.623 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T11:07:25.650 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T11:07:25.681 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T11:07:25.700 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:25.733 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:25.733 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T11:07:25.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:25.739 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:07:25.771 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T11:07:25.806 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:25 smithi078 bash[75299]: cluster 2023-10-01T11:07:24.324535+0000 mgr.y (mgr.25072) 1502 : cluster 0 pgmap v1606: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 720 op/s; 12771/138849 objects degraded (9.198%); 3074/138849 objects misplaced (2.214%); 4.8 MiB/s, 76 objects/s recovering 2023-10-01T11:07:25.806 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:25 smithi078 bash[75299]: audit 2023-10-01T11:07:25.515441+0000 mon.a (mon.0) 1938 : audit 0 from='client.? 172.21.15.78:0/548420309' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:07:25.806 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T11:07:25.809 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T11:07:25.809 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T11:07:25.809 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T11:07:25.809 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T11:07:25.809 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T11:07:25.810 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T11:07:25.810 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T11:07:25.810 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T11:07:25.839 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:25.849 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:25 smithi173 bash[58816]: cluster 2023-10-01T11:07:24.324535+0000 mgr.y (mgr.25072) 1502 : cluster 0 pgmap v1606: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 720 op/s; 12771/138849 objects degraded (9.198%); 3074/138849 objects misplaced (2.214%); 4.8 MiB/s, 76 objects/s recovering 2023-10-01T11:07:25.849 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:25 smithi173 bash[58816]: audit 2023-10-01T11:07:25.515441+0000 mon.a (mon.0) 1938 : audit 0 from='client.? 172.21.15.78:0/548420309' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:07:25.868 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:25.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:25.871 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:07:25.972 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:25.998 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:26.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:26.003 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:07:26.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:25 smithi078 bash[78274]: cluster 2023-10-01T11:07:24.324535+0000 mgr.y (mgr.25072) 1502 : cluster 0 pgmap v1606: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 720 op/s; 12771/138849 objects degraded (9.198%); 3074/138849 objects misplaced (2.214%); 4.8 MiB/s, 76 objects/s recovering 2023-10-01T11:07:26.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:25 smithi078 bash[78274]: audit 2023-10-01T11:07:25.515441+0000 mon.a (mon.0) 1938 : audit 0 from='client.? 172.21.15.78:0/548420309' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:07:26.104 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:26.140 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:26.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:26.145 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:07:26.246 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:26.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:26.296 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:26.296 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:07:26.397 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:26.425 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:26.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:26.428 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:07:26.528 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:26.551 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:26.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:26.553 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:07:26.654 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:26.691 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:26.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:26.695 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:07:26.795 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:26.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:26.828 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:26.828 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:07:26.929 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:26.952 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:26.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:26.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:07:27.059 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:27.085 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:27.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:27.089 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:07:27.190 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:27.217 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:27.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:27.222 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:07:27.322 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:27.353 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:27.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:27.356 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:07:27.457 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:27.481 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:27.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:27.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:07:27.587 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:27.613 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:27.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:27.617 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:07:27.718 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:27.746 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:27.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:27.749 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:07:27.849 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:27.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:27 smithi173 bash[58816]: cluster 2023-10-01T11:07:26.325227+0000 mgr.y (mgr.25072) 1503 : cluster 0 pgmap v1607: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 595 op/s; 12572/135900 objects degraded (9.251%); 2911/135900 objects misplaced (2.142%); 6.5 MiB/s, 112 objects/s recovering 2023-10-01T11:07:27.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:27 smithi173 bash[58816]: cluster 2023-10-01T11:07:27.768387+0000 mon.a (mon.0) 1939 : cluster 3 Health check update: Degraded data redundancy: 12572/135900 objects degraded (9.251%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T11:07:27.874 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:27.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:27.876 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:07:27.977 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:28.007 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:28.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:28.012 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:07:28.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:27 smithi078 bash[75299]: cluster 2023-10-01T11:07:26.325227+0000 mgr.y (mgr.25072) 1503 : cluster 0 pgmap v1607: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 595 op/s; 12572/135900 objects degraded (9.251%); 2911/135900 objects misplaced (2.142%); 6.5 MiB/s, 112 objects/s recovering 2023-10-01T11:07:28.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:27 smithi078 bash[75299]: cluster 2023-10-01T11:07:27.768387+0000 mon.a (mon.0) 1939 : cluster 3 Health check update: Degraded data redundancy: 12572/135900 objects degraded (9.251%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T11:07:28.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:27 smithi078 bash[78274]: cluster 2023-10-01T11:07:26.325227+0000 mgr.y (mgr.25072) 1503 : cluster 0 pgmap v1607: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 595 op/s; 12572/135900 objects degraded (9.251%); 2911/135900 objects misplaced (2.142%); 6.5 MiB/s, 112 objects/s recovering 2023-10-01T11:07:28.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:27 smithi078 bash[78274]: cluster 2023-10-01T11:07:27.768387+0000 mon.a (mon.0) 1939 : cluster 3 Health check update: Degraded data redundancy: 12572/135900 objects degraded (9.251%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T11:07:28.113 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:28.141 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:28.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:28.145 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:07:28.238 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-01T11:07:28.246 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:28.324 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:28.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:28.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:07:28.428 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:28.462 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:28.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:28.468 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:07:28.568 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:28.600 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:28.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:28.604 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:07:28.704 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:28.739 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:28.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:28.744 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:07:28.844 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:28.879 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:28.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:28.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:07:28.984 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:29.012 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:29.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:29.015 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:07:29.116 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:29.139 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:29.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:29.143 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:07:29.244 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:29.271 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:29.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:29.275 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:07:29.376 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:29.405 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:29.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:29.409 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:07:29.510 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:29.551 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:29.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:29.556 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:07:29.657 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:29.691 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:29.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:29.695 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:07:29.796 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:29.827 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:29 smithi173 bash[58816]: cluster 2023-10-01T11:07:28.325954+0000 mgr.y (mgr.25072) 1504 : cluster 0 pgmap v1608: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 563 op/s; 12572/134073 objects degraded (9.377%); 2689/134073 objects misplaced (2.006%); 4.2 MiB/s, 75 objects/s recovering 2023-10-01T11:07:29.827 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:29 smithi173 bash[58816]: audit 2023-10-01T11:07:29.269459+0000 mon.a (mon.0) 1940 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T11:07:29.827 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:29 smithi173 bash[58816]: audit 2023-10-01T11:07:29.695746+0000 mon.a (mon.0) 1941 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:07:29.827 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:29 smithi173 bash[58816]: audit 2023-10-01T11:07:29.708371+0000 mon.a (mon.0) 1942 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:07:29.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:29.828 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:29.829 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:07:29.929 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:29.965 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:29.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:29.968 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:07:30.065 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T11:07:30.065 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T11:07:30.066 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_8 2023-10-01T11:07:30.066 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T11:07:30.068 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:30.074 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:29 smithi078 bash[78274]: cluster 2023-10-01T11:07:28.325954+0000 mgr.y (mgr.25072) 1504 : cluster 0 pgmap v1608: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 563 op/s; 12572/134073 objects degraded (9.377%); 2689/134073 objects misplaced (2.006%); 4.2 MiB/s, 75 objects/s recovering 2023-10-01T11:07:30.075 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:29 smithi078 bash[78274]: audit 2023-10-01T11:07:29.269459+0000 mon.a (mon.0) 1940 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T11:07:30.075 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:29 smithi078 bash[78274]: audit 2023-10-01T11:07:29.695746+0000 mon.a (mon.0) 1941 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:07:30.075 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:29 smithi078 bash[78274]: audit 2023-10-01T11:07:29.708371+0000 mon.a (mon.0) 1942 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:07:30.076 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:29 smithi078 bash[75299]: cluster 2023-10-01T11:07:28.325954+0000 mgr.y (mgr.25072) 1504 : cluster 0 pgmap v1608: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 563 op/s; 12572/134073 objects degraded (9.377%); 2689/134073 objects misplaced (2.006%); 4.2 MiB/s, 75 objects/s recovering 2023-10-01T11:07:30.076 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:29 smithi078 bash[75299]: audit 2023-10-01T11:07:29.269459+0000 mon.a (mon.0) 1940 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T11:07:30.076 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:29 smithi078 bash[75299]: audit 2023-10-01T11:07:29.695746+0000 mon.a (mon.0) 1941 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:07:30.076 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:29 smithi078 bash[75299]: audit 2023-10-01T11:07:29.708371+0000 mon.a (mon.0) 1942 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:07:30.101 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:30.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:30.106 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:07:30.207 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:30.228 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:30.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:30.232 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:07:30.257 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T11:07:30.331 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:07:30 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:07:30] "GET /metrics HTTP/1.1" 200 36405 "" "Prometheus/2.33.4" 2023-10-01T11:07:30.334 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:30.359 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:30.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:30.372 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:07:30.473 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:30.507 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:30.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:30.513 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:07:30.614 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:30.660 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:30.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:30.666 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:07:30.767 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:30.804 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:30.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:30.807 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:07:30.907 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:30.943 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:30.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:30.949 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:07:31.050 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:31.077 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:31.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:31.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:07:31.180 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:31.202 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:31.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:31.206 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:07:31.307 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:31.357 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:31.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:31.359 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:07:31.460 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:31.508 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:31.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:31.511 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:07:31.612 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:31.640 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:31.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:31.644 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:07:31.745 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:31.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:31.793 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:31.793 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:07:31.894 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:31.907 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:31 smithi173 bash[58816]: cluster 2023-10-01T11:07:30.326717+0000 mgr.y (mgr.25072) 1505 : cluster 0 pgmap v1609: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 720 op/s; 12370/126603 objects degraded (9.771%); 2689/126603 objects misplaced (2.124%); 5.2 MiB/s, 116 objects/s recovering 2023-10-01T11:07:31.947 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:31.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:31.951 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:07:32.051 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:32.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:31 smithi078 bash[75299]: cluster 2023-10-01T11:07:30.326717+0000 mgr.y (mgr.25072) 1505 : cluster 0 pgmap v1609: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 720 op/s; 12370/126603 objects degraded (9.771%); 2689/126603 objects misplaced (2.124%); 5.2 MiB/s, 116 objects/s recovering 2023-10-01T11:07:32.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:31 smithi078 bash[78274]: cluster 2023-10-01T11:07:30.326717+0000 mgr.y (mgr.25072) 1505 : cluster 0 pgmap v1609: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 720 op/s; 12370/126603 objects degraded (9.771%); 2689/126603 objects misplaced (2.124%); 5.2 MiB/s, 116 objects/s recovering 2023-10-01T11:07:32.084 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:32.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:32.087 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:07:32.188 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:32.223 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:32.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:32.231 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:07:32.331 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:32.365 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:32.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:32.369 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:07:32.470 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:32.497 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:32.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:32.499 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:07:32.600 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:32.636 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:32.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:32.640 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:07:32.741 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:32.768 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:32.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:32.773 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:07:32.861 INFO:teuthology.orchestra.run.smithi078.stderr:nodeep-scrub is set 2023-10-01T11:07:32.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:32 smithi173 bash[58816]: audit 2023-10-01T11:07:31.894277+0000 mon.a (mon.0) 1943 : audit 1 from='client.? 172.21.15.78:0/3018773452' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:07:32.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:32 smithi173 bash[58816]: audit 2023-10-01T11:07:32.462011+0000 mon.a (mon.0) 1944 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:07:32.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:32 smithi173 bash[58816]: cluster 2023-10-01T11:07:32.771140+0000 mon.a (mon.0) 1945 : cluster 3 Health check update: Degraded data redundancy: 12370/126603 objects degraded (9.771%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T11:07:32.873 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:32.916 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:32.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:32.919 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:07:33.019 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:33.058 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:33.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:33.063 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:07:33.073 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:07:33.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:32 smithi078 bash[78274]: audit 2023-10-01T11:07:31.894277+0000 mon.a (mon.0) 1943 : audit 1 from='client.? 172.21.15.78:0/3018773452' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:07:33.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:32 smithi078 bash[78274]: audit 2023-10-01T11:07:32.462011+0000 mon.a (mon.0) 1944 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:07:33.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:32 smithi078 bash[78274]: cluster 2023-10-01T11:07:32.771140+0000 mon.a (mon.0) 1945 : cluster 3 Health check update: Degraded data redundancy: 12370/126603 objects degraded (9.771%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T11:07:33.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:32 smithi078 bash[75299]: audit 2023-10-01T11:07:31.894277+0000 mon.a (mon.0) 1943 : audit 1 from='client.? 172.21.15.78:0/3018773452' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:07:33.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:32 smithi078 bash[75299]: audit 2023-10-01T11:07:32.462011+0000 mon.a (mon.0) 1944 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:07:33.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:32 smithi078 bash[75299]: cluster 2023-10-01T11:07:32.771140+0000 mon.a (mon.0) 1945 : cluster 3 Health check update: Degraded data redundancy: 12370/126603 objects degraded (9.771%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T11:07:33.164 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:33.194 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:33.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:33.197 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:07:33.297 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:33.328 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:33.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:33.330 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:07:33.431 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:33.455 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:33.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:33.460 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:07:33.561 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:33.597 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:33.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:33.600 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:07:33.701 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:33.724 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:33.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:33.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:07:33.828 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:33.857 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:33.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:33.859 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:07:33.960 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:33.968 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:33 smithi078 bash[78274]: cluster 2023-10-01T11:07:32.327323+0000 mgr.y (mgr.25072) 1506 : cluster 0 pgmap v1610: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 512 op/s; 12370/126603 objects degraded (9.771%); 2689/126603 objects misplaced (2.124%); 2.9 MiB/s, 79 objects/s recovering 2023-10-01T11:07:33.968 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:33 smithi078 bash[78274]: cluster 2023-10-01T11:07:32.830521+0000 mon.a (mon.0) 1946 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:07:33.968 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:33 smithi078 bash[78274]: audit 2023-10-01T11:07:32.846569+0000 mon.a (mon.0) 1947 : audit 1 from='client.? 172.21.15.78:0/3018773452' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T11:07:33.969 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:33 smithi078 bash[78274]: cluster 2023-10-01T11:07:32.846716+0000 mon.a (mon.0) 1948 : cluster 0 osdmap e342: 8 total, 8 up, 8 in 2023-10-01T11:07:33.969 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:33 smithi078 bash[78274]: cluster 2023-10-01T11:07:32.850336+0000 osd.5 (osd.5) 361 : cluster 3 failed to encode map e342 with expected crc 2023-10-01T11:07:33.969 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:33 smithi078 bash[78274]: cluster 2023-10-01T11:07:32.855325+0000 osd.6 (osd.6) 328 : cluster 3 failed to encode map e342 with expected crc 2023-10-01T11:07:33.969 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:33 smithi078 bash[78274]: cluster 2023-10-01T11:07:32.859910+0000 osd.7 (osd.7) 376 : cluster 3 failed to encode map e342 with expected crc 2023-10-01T11:07:33.969 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:33 smithi078 bash[78274]: cluster 2023-10-01T11:07:32.862016+0000 osd.7 (osd.7) 377 : cluster 3 failed to encode map e342 with expected crc 2023-10-01T11:07:33.969 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:33 smithi078 bash[75299]: cluster 2023-10-01T11:07:32.327323+0000 mgr.y (mgr.25072) 1506 : cluster 0 pgmap v1610: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 512 op/s; 12370/126603 objects degraded (9.771%); 2689/126603 objects misplaced (2.124%); 2.9 MiB/s, 79 objects/s recovering 2023-10-01T11:07:33.969 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:33 smithi078 bash[75299]: cluster 2023-10-01T11:07:32.830521+0000 mon.a (mon.0) 1946 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:07:33.970 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:33 smithi078 bash[75299]: audit 2023-10-01T11:07:32.846569+0000 mon.a (mon.0) 1947 : audit 1 from='client.? 172.21.15.78:0/3018773452' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T11:07:33.970 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:33 smithi078 bash[75299]: cluster 2023-10-01T11:07:32.846716+0000 mon.a (mon.0) 1948 : cluster 0 osdmap e342: 8 total, 8 up, 8 in 2023-10-01T11:07:33.970 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:33 smithi078 bash[75299]: cluster 2023-10-01T11:07:32.850336+0000 osd.5 (osd.5) 361 : cluster 3 failed to encode map e342 with expected crc 2023-10-01T11:07:33.970 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:33 smithi078 bash[75299]: cluster 2023-10-01T11:07:32.855325+0000 osd.6 (osd.6) 328 : cluster 3 failed to encode map e342 with expected crc 2023-10-01T11:07:33.970 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:33 smithi078 bash[75299]: cluster 2023-10-01T11:07:32.859910+0000 osd.7 (osd.7) 376 : cluster 3 failed to encode map e342 with expected crc 2023-10-01T11:07:33.970 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:33 smithi078 bash[75299]: cluster 2023-10-01T11:07:32.862016+0000 osd.7 (osd.7) 377 : cluster 3 failed to encode map e342 with expected crc 2023-10-01T11:07:33.984 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:33.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:33.988 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:07:34.089 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:34.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:33 smithi173 bash[58816]: cluster 2023-10-01T11:07:32.327323+0000 mgr.y (mgr.25072) 1506 : cluster 0 pgmap v1610: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 512 op/s; 12370/126603 objects degraded (9.771%); 2689/126603 objects misplaced (2.124%); 2.9 MiB/s, 79 objects/s recovering 2023-10-01T11:07:34.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:33 smithi173 bash[58816]: cluster 2023-10-01T11:07:32.830521+0000 mon.a (mon.0) 1946 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:07:34.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:33 smithi173 bash[58816]: audit 2023-10-01T11:07:32.846569+0000 mon.a (mon.0) 1947 : audit 1 from='client.? 172.21.15.78:0/3018773452' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T11:07:34.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:33 smithi173 bash[58816]: cluster 2023-10-01T11:07:32.846716+0000 mon.a (mon.0) 1948 : cluster 0 osdmap e342: 8 total, 8 up, 8 in 2023-10-01T11:07:34.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:33 smithi173 bash[58816]: cluster 2023-10-01T11:07:32.850336+0000 osd.5 (osd.5) 361 : cluster 3 failed to encode map e342 with expected crc 2023-10-01T11:07:34.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:33 smithi173 bash[58816]: cluster 2023-10-01T11:07:32.855325+0000 osd.6 (osd.6) 328 : cluster 3 failed to encode map e342 with expected crc 2023-10-01T11:07:34.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:33 smithi173 bash[58816]: cluster 2023-10-01T11:07:32.859910+0000 osd.7 (osd.7) 376 : cluster 3 failed to encode map e342 with expected crc 2023-10-01T11:07:34.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:33 smithi173 bash[58816]: cluster 2023-10-01T11:07:32.862016+0000 osd.7 (osd.7) 377 : cluster 3 failed to encode map e342 with expected crc 2023-10-01T11:07:34.122 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:34.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:34.125 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:07:34.225 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:34.248 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:34.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:34.252 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:07:34.353 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:34.378 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:34.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:34.381 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:07:34.481 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:34.543 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:34.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:34.548 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:07:34.649 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:34.704 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:34.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:34.707 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:07:34.808 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:34.844 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:34.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:34.847 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:07:34.948 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:34.977 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:34.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:34.979 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:07:35.080 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:35.118 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:35.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:35.121 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:07:35.222 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:35.243 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:35.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:35.246 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:07:35.347 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:35.378 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:35.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:35.383 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:07:35.483 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:35.511 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:35.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:35.514 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:07:35.614 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:35.639 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:35.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:35.641 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:07:35.742 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:35.761 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:35.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:35.766 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:07:35.867 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:35.898 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:35.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:35.900 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:07:36.001 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:36.014 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:35 smithi173 bash[58816]: cluster 2023-10-01T11:07:34.328150+0000 mgr.y (mgr.25072) 1507 : cluster 0 pgmap v1612: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 12370/120348 objects degraded (10.279%); 2507/120348 objects misplaced (2.083%); 3.5 MiB/s, 95 objects/s recovering 2023-10-01T11:07:36.015 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:35 smithi173 bash[58816]: cluster 2023-10-01T11:07:34.844400+0000 mon.a (mon.0) 1949 : cluster 0 osdmap e343: 8 total, 8 up, 8 in 2023-10-01T11:07:36.015 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:35 smithi173 bash[58816]: cluster 2023-10-01T11:07:34.853491+0000 osd.5 (osd.5) 362 : cluster 3 failed to encode map e343 with expected crc 2023-10-01T11:07:36.015 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:35 smithi173 bash[58816]: cluster 2023-10-01T11:07:34.855829+0000 osd.5 (osd.5) 363 : cluster 3 failed to encode map e343 with expected crc 2023-10-01T11:07:36.015 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:35 smithi173 bash[58816]: cluster 2023-10-01T11:07:34.858394+0000 osd.6 (osd.6) 329 : cluster 3 failed to encode map e343 with expected crc 2023-10-01T11:07:36.015 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:35 smithi173 bash[58816]: cluster 2023-10-01T11:07:34.863827+0000 osd.7 (osd.7) 378 : cluster 3 failed to encode map e343 with expected crc 2023-10-01T11:07:36.028 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:07:36.030 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:07:36.030 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:36.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:36.033 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:07:36.134 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:36.141 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:35 smithi078 bash[78274]: cluster 2023-10-01T11:07:34.328150+0000 mgr.y (mgr.25072) 1507 : cluster 0 pgmap v1612: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 12370/120348 objects degraded (10.279%); 2507/120348 objects misplaced (2.083%); 3.5 MiB/s, 95 objects/s recovering 2023-10-01T11:07:36.141 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:35 smithi078 bash[78274]: cluster 2023-10-01T11:07:34.844400+0000 mon.a (mon.0) 1949 : cluster 0 osdmap e343: 8 total, 8 up, 8 in 2023-10-01T11:07:36.141 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:35 smithi078 bash[78274]: cluster 2023-10-01T11:07:34.853491+0000 osd.5 (osd.5) 362 : cluster 3 failed to encode map e343 with expected crc 2023-10-01T11:07:36.141 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:35 smithi078 bash[78274]: cluster 2023-10-01T11:07:34.855829+0000 osd.5 (osd.5) 363 : cluster 3 failed to encode map e343 with expected crc 2023-10-01T11:07:36.141 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:35 smithi078 bash[78274]: cluster 2023-10-01T11:07:34.858394+0000 osd.6 (osd.6) 329 : cluster 3 failed to encode map e343 with expected crc 2023-10-01T11:07:36.141 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:35 smithi078 bash[78274]: cluster 2023-10-01T11:07:34.863827+0000 osd.7 (osd.7) 378 : cluster 3 failed to encode map e343 with expected crc 2023-10-01T11:07:36.142 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:35 smithi078 bash[75299]: cluster 2023-10-01T11:07:34.328150+0000 mgr.y (mgr.25072) 1507 : cluster 0 pgmap v1612: 89 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 12370/120348 objects degraded (10.279%); 2507/120348 objects misplaced (2.083%); 3.5 MiB/s, 95 objects/s recovering 2023-10-01T11:07:36.142 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:35 smithi078 bash[75299]: cluster 2023-10-01T11:07:34.844400+0000 mon.a (mon.0) 1949 : cluster 0 osdmap e343: 8 total, 8 up, 8 in 2023-10-01T11:07:36.142 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:35 smithi078 bash[75299]: cluster 2023-10-01T11:07:34.853491+0000 osd.5 (osd.5) 362 : cluster 3 failed to encode map e343 with expected crc 2023-10-01T11:07:36.142 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:35 smithi078 bash[75299]: cluster 2023-10-01T11:07:34.855829+0000 osd.5 (osd.5) 363 : cluster 3 failed to encode map e343 with expected crc 2023-10-01T11:07:36.142 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:35 smithi078 bash[75299]: cluster 2023-10-01T11:07:34.858394+0000 osd.6 (osd.6) 329 : cluster 3 failed to encode map e343 with expected crc 2023-10-01T11:07:36.142 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:35 smithi078 bash[75299]: cluster 2023-10-01T11:07:34.863827+0000 osd.7 (osd.7) 378 : cluster 3 failed to encode map e343 with expected crc 2023-10-01T11:07:36.185 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:36.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:36.191 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:07:36.292 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:36.321 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:36.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:36.326 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:07:36.427 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:36.462 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:36.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:36.465 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:07:36.566 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:36.599 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:36.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:36.606 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:07:36.706 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:36.738 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:36.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:36.741 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:07:36.842 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:36.871 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:36.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:36.878 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:07:36.979 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:36.988 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:36 smithi173 bash[58816]: cluster 2023-10-01T11:07:35.844599+0000 osd.7 (osd.7) 379 : cluster 3 failed to encode map e344 with expected crc 2023-10-01T11:07:36.988 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:36 smithi173 bash[58816]: cluster 2023-10-01T11:07:35.849763+0000 mon.a (mon.0) 1950 : cluster 0 osdmap e344: 8 total, 8 up, 8 in 2023-10-01T11:07:36.988 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:36 smithi173 bash[58816]: cluster 2023-10-01T11:07:35.850252+0000 osd.7 (osd.7) 380 : cluster 3 failed to encode map e344 with expected crc 2023-10-01T11:07:36.989 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:36 smithi173 bash[58816]: cluster 2023-10-01T11:07:35.906483+0000 osd.6 (osd.6) 330 : cluster 3 failed to encode map e344 with expected crc 2023-10-01T11:07:36.989 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:36 smithi173 bash[58816]: audit 2023-10-01T11:07:36.321139+0000 mon.a (mon.0) 1951 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:07:36.989 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:36 smithi173 bash[58816]: audit 2023-10-01T11:07:36.328868+0000 mon.a (mon.0) 1952 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:07:37.018 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:37.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:37.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:07:37.065 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:37 smithi078 bash[78274]: cluster 2023-10-01T11:07:35.844599+0000 osd.7 (osd.7) 379 : cluster 3 failed to encode map e344 with expected crc 2023-10-01T11:07:37.066 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:37 smithi078 bash[78274]: cluster 2023-10-01T11:07:35.849763+0000 mon.a (mon.0) 1950 : cluster 0 osdmap e344: 8 total, 8 up, 8 in 2023-10-01T11:07:37.066 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:37 smithi078 bash[78274]: cluster 2023-10-01T11:07:35.850252+0000 osd.7 (osd.7) 380 : cluster 3 failed to encode map e344 with expected crc 2023-10-01T11:07:37.066 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:37 smithi078 bash[78274]: cluster 2023-10-01T11:07:35.906483+0000 osd.6 (osd.6) 330 : cluster 3 failed to encode map e344 with expected crc 2023-10-01T11:07:37.066 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:37 smithi078 bash[78274]: audit 2023-10-01T11:07:36.321139+0000 mon.a (mon.0) 1951 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:07:37.066 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:37 smithi078 bash[78274]: audit 2023-10-01T11:07:36.328868+0000 mon.a (mon.0) 1952 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:07:37.122 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:37.153 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:37.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:37.157 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:07:37.258 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:37.281 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:37.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:37.287 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:07:37.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:37 smithi078 bash[75299]: cluster 2023-10-01T11:07:35.844599+0000 osd.7 (osd.7) 379 : cluster 3 failed to encode map e344 with expected crc 2023-10-01T11:07:37.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:37 smithi078 bash[75299]: cluster 2023-10-01T11:07:35.849763+0000 mon.a (mon.0) 1950 : cluster 0 osdmap e344: 8 total, 8 up, 8 in 2023-10-01T11:07:37.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:37 smithi078 bash[75299]: cluster 2023-10-01T11:07:35.850252+0000 osd.7 (osd.7) 380 : cluster 3 failed to encode map e344 with expected crc 2023-10-01T11:07:37.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:37 smithi078 bash[75299]: cluster 2023-10-01T11:07:35.906483+0000 osd.6 (osd.6) 330 : cluster 3 failed to encode map e344 with expected crc 2023-10-01T11:07:37.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:37 smithi078 bash[75299]: audit 2023-10-01T11:07:36.321139+0000 mon.a (mon.0) 1951 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:07:37.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:37 smithi078 bash[75299]: audit 2023-10-01T11:07:36.328868+0000 mon.a (mon.0) 1952 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:07:37.388 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:37.417 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:37.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:37.424 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:07:37.524 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:37.561 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T11:07:37.566 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:37.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:37.571 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:07:37.672 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:37.699 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:37.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:37.702 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:07:37.803 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:37.843 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:37.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:37.850 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:07:37.951 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:37.975 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:37.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:37.981 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:07:38.082 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:38.107 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:38.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:38.109 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:07:38.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:37 smithi173 bash[58816]: cluster 2023-10-01T11:07:36.328710+0000 mgr.y (mgr.25072) 1508 : cluster 0 pgmap v1615: 89 pgs: 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 509 op/s; 12170/117423 objects degraded (10.364%); 2507/117423 objects misplaced (2.135%); 2.1 MiB/s, 80 objects/s recovering 2023-10-01T11:07:38.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:37 smithi173 bash[58816]: audit 2023-10-01T11:07:37.447053+0000 mon.a (mon.0) 1953 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:07:38.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:37 smithi173 bash[58816]: audit 2023-10-01T11:07:37.455711+0000 mon.a (mon.0) 1954 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:07:38.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:37 smithi173 bash[58816]: cluster 2023-10-01T11:07:37.772331+0000 mon.a (mon.0) 1955 : cluster 3 Health check update: Degraded data redundancy: 12170/117423 objects degraded (10.364%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T11:07:38.126 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:37 smithi078 bash[75299]: cluster 2023-10-01T11:07:36.328710+0000 mgr.y (mgr.25072) 1508 : cluster 0 pgmap v1615: 89 pgs: 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 509 op/s; 12170/117423 objects degraded (10.364%); 2507/117423 objects misplaced (2.135%); 2.1 MiB/s, 80 objects/s recovering 2023-10-01T11:07:38.126 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:37 smithi078 bash[75299]: audit 2023-10-01T11:07:37.447053+0000 mon.a (mon.0) 1953 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:07:38.126 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:37 smithi078 bash[75299]: audit 2023-10-01T11:07:37.455711+0000 mon.a (mon.0) 1954 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:07:38.126 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:37 smithi078 bash[75299]: cluster 2023-10-01T11:07:37.772331+0000 mon.a (mon.0) 1955 : cluster 3 Health check update: Degraded data redundancy: 12170/117423 objects degraded (10.364%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T11:07:38.126 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:37 smithi078 bash[78274]: cluster 2023-10-01T11:07:36.328710+0000 mgr.y (mgr.25072) 1508 : cluster 0 pgmap v1615: 89 pgs: 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 509 op/s; 12170/117423 objects degraded (10.364%); 2507/117423 objects misplaced (2.135%); 2.1 MiB/s, 80 objects/s recovering 2023-10-01T11:07:38.126 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:37 smithi078 bash[78274]: audit 2023-10-01T11:07:37.447053+0000 mon.a (mon.0) 1953 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:07:38.126 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:37 smithi078 bash[78274]: audit 2023-10-01T11:07:37.455711+0000 mon.a (mon.0) 1954 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:07:38.126 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:37 smithi078 bash[78274]: cluster 2023-10-01T11:07:37.772331+0000 mon.a (mon.0) 1955 : cluster 3 Health check update: Degraded data redundancy: 12170/117423 objects degraded (10.364%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T11:07:38.211 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:38.234 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:38.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:38.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:07:38.339 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:38.378 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:38.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:38.384 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:07:38.484 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:38.515 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:38.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:38.519 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:07:38.620 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:38.666 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:38.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:38.667 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:07:38.768 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:38.800 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:38.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:38.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:07:38.906 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:38.931 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:38.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:38.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:07:39.035 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:39.063 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:39.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:39.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:07:39.167 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:39.197 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:39.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:39.200 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:07:39.301 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:39.337 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:39.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:39.344 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:07:39.444 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:39.476 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:39.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:39.489 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:07:39.491 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:39 smithi078 bash[75299]: cluster 2023-10-01T11:07:38.329357+0000 mgr.y (mgr.25072) 1509 : cluster 0 pgmap v1616: 89 pgs: 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 12170/115863 objects degraded (10.504%); 2326/115863 objects misplaced (2.008%); 2.1 MiB/s, 80 objects/s recovering 2023-10-01T11:07:39.491 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:39 smithi078 bash[75299]: audit 2023-10-01T11:07:38.346676+0000 mon.a (mon.0) 1956 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:07:39.491 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:39 smithi078 bash[75299]: audit 2023-10-01T11:07:38.357916+0000 mon.a (mon.0) 1957 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:07:39.491 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:39 smithi078 bash[75299]: audit 2023-10-01T11:07:38.358990+0000 mon.a (mon.0) 1958 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T11:07:39.491 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:39 smithi078 bash[75299]: audit 2023-10-01T11:07:38.359930+0000 mon.a (mon.0) 1959 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T11:07:39.491 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:39 smithi078 bash[75299]: audit 2023-10-01T11:07:38.365233+0000 mon.a (mon.0) 1960 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:07:39.491 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:39 smithi078 bash[78274]: cluster 2023-10-01T11:07:38.329357+0000 mgr.y (mgr.25072) 1509 : cluster 0 pgmap v1616: 89 pgs: 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 12170/115863 objects degraded (10.504%); 2326/115863 objects misplaced (2.008%); 2.1 MiB/s, 80 objects/s recovering 2023-10-01T11:07:39.491 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:39 smithi078 bash[78274]: audit 2023-10-01T11:07:38.346676+0000 mon.a (mon.0) 1956 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:07:39.492 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:39 smithi078 bash[78274]: audit 2023-10-01T11:07:38.357916+0000 mon.a (mon.0) 1957 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:07:39.492 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:39 smithi078 bash[78274]: audit 2023-10-01T11:07:38.358990+0000 mon.a (mon.0) 1958 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T11:07:39.492 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:39 smithi078 bash[78274]: audit 2023-10-01T11:07:38.359930+0000 mon.a (mon.0) 1959 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T11:07:39.492 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:39 smithi078 bash[78274]: audit 2023-10-01T11:07:38.365233+0000 mon.a (mon.0) 1960 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:07:39.590 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:39.614 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:39.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:39.617 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:07:39.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:39 smithi173 bash[58816]: cluster 2023-10-01T11:07:38.329357+0000 mgr.y (mgr.25072) 1509 : cluster 0 pgmap v1616: 89 pgs: 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 12170/115863 objects degraded (10.504%); 2326/115863 objects misplaced (2.008%); 2.1 MiB/s, 80 objects/s recovering 2023-10-01T11:07:39.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:39 smithi173 bash[58816]: audit 2023-10-01T11:07:38.346676+0000 mon.a (mon.0) 1956 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:07:39.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:39 smithi173 bash[58816]: audit 2023-10-01T11:07:38.357916+0000 mon.a (mon.0) 1957 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:07:39.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:39 smithi173 bash[58816]: audit 2023-10-01T11:07:38.358990+0000 mon.a (mon.0) 1958 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T11:07:39.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:39 smithi173 bash[58816]: audit 2023-10-01T11:07:38.359930+0000 mon.a (mon.0) 1959 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T11:07:39.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:39 smithi173 bash[58816]: audit 2023-10-01T11:07:38.365233+0000 mon.a (mon.0) 1960 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:07:39.718 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:39.752 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:39.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:39.755 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:07:39.856 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:39.882 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:39.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:39.890 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:07:39.991 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:40.023 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:40.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:40.028 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:07:40.129 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:40.154 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:40.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:40.160 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:07:40.260 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:40.279 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:40.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:40.281 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:07:40.382 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:40.391 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:07:40 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:07:40] "GET /metrics HTTP/1.1" 200 36406 "" "Prometheus/2.33.4" 2023-10-01T11:07:40.417 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:40.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:40.420 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:07:40.521 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:40.558 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:40.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:40.563 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:07:40.664 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:40.711 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:40.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:40.715 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:07:40.816 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:40.844 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:40.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:40.847 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:07:40.947 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:40.971 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:40.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:40.977 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:07:41.078 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:41.112 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:41.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:41.115 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:07:41.216 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:41.248 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:41.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:41.251 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:07:41.351 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:41.371 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:41.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:41.373 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:07:41.473 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:41.502 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:41.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:41.506 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:07:41.607 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:41.614 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:41 smithi078 bash[78274]: cluster 2023-10-01T11:07:40.330160+0000 mgr.y (mgr.25072) 1510 : cluster 0 pgmap v1617: 89 pgs: 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 8835/109305 objects degraded (8.083%); 4641/109305 objects misplaced (4.246%); 3.9 MiB/s, 100 objects/s recovering 2023-10-01T11:07:41.614 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:41 smithi078 bash[75299]: cluster 2023-10-01T11:07:40.330160+0000 mgr.y (mgr.25072) 1510 : cluster 0 pgmap v1617: 89 pgs: 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 8835/109305 objects degraded (8.083%); 4641/109305 objects misplaced (4.246%); 3.9 MiB/s, 100 objects/s recovering 2023-10-01T11:07:41.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:41 smithi173 bash[58816]: cluster 2023-10-01T11:07:40.330160+0000 mgr.y (mgr.25072) 1510 : cluster 0 pgmap v1617: 89 pgs: 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 8835/109305 objects degraded (8.083%); 4641/109305 objects misplaced (4.246%); 3.9 MiB/s, 100 objects/s recovering 2023-10-01T11:07:41.632 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:41.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:41.635 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:07:41.735 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:41.772 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:41.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:41.779 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:07:41.879 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:41.913 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:41.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:41.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:07:42.017 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:42.044 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:42.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:42.047 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:07:42.148 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:42.177 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:42.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:42.180 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:07:42.280 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:42.311 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:42.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:42.317 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:07:42.418 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:42.434 INFO:teuthology.orchestra.run.smithi078.stdout:ERROR: (22) Invalid argument 2023-10-01T11:07:42.434 INFO:teuthology.orchestra.run.smithi078.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T11:07:42.442 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:42.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:42.448 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:07:42.549 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:42.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:42.589 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:42.589 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:07:42.690 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:42.713 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:42.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:42.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:07:42.817 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:42.850 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:42.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:42.853 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:07:42.954 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:42.977 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:42.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:42.980 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:07:43.081 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:43.115 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:43.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:43.118 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:07:43.218 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:43.247 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:43.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:43.251 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:07:43.352 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:43.356 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:43 smithi173 bash[58816]: cluster 2023-10-01T11:07:42.330754+0000 mgr.y (mgr.25072) 1511 : cluster 0 pgmap v1618: 89 pgs: 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 459 op/s; 8835/109305 objects degraded (8.083%); 4641/109305 objects misplaced (4.246%); 3.7 MiB/s, 94 objects/s recovering 2023-10-01T11:07:43.356 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:43 smithi173 bash[58816]: cluster 2023-10-01T11:07:42.773552+0000 mon.a (mon.0) 1961 : cluster 3 Health check update: Degraded data redundancy: 8835/109305 objects degraded (8.083%), 5 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T11:07:43.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:43.389 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:43.390 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:07:43.482 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:43 smithi078 bash[75299]: cluster 2023-10-01T11:07:42.330754+0000 mgr.y (mgr.25072) 1511 : cluster 0 pgmap v1618: 89 pgs: 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 459 op/s; 8835/109305 objects degraded (8.083%); 4641/109305 objects misplaced (4.246%); 3.7 MiB/s, 94 objects/s recovering 2023-10-01T11:07:43.483 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:43 smithi078 bash[75299]: cluster 2023-10-01T11:07:42.773552+0000 mon.a (mon.0) 1961 : cluster 3 Health check update: Degraded data redundancy: 8835/109305 objects degraded (8.083%), 5 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T11:07:43.483 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:43 smithi078 bash[78274]: cluster 2023-10-01T11:07:42.330754+0000 mgr.y (mgr.25072) 1511 : cluster 0 pgmap v1618: 89 pgs: 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 459 op/s; 8835/109305 objects degraded (8.083%); 4641/109305 objects misplaced (4.246%); 3.7 MiB/s, 94 objects/s recovering 2023-10-01T11:07:43.483 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:43 smithi078 bash[78274]: cluster 2023-10-01T11:07:42.773552+0000 mon.a (mon.0) 1961 : cluster 3 Health check update: Degraded data redundancy: 8835/109305 objects degraded (8.083%), 5 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T11:07:43.490 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:43.523 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:43.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:43.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:07:43.627 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:43.660 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:43.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:43.663 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:07:43.764 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:43.802 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:43.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:43.806 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:07:43.907 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:43.944 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:43.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:43.946 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:07:44.047 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:44.074 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:44.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:44.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:07:44.180 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:44.210 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:44.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:44.212 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:07:44.313 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:44.346 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:44.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:44.352 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:07:44.453 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:44.497 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:44.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:44.502 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:07:44.603 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:44.631 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:44.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:44.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:07:44.736 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:44.773 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:44.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:44.775 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:07:44.851 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-01T11:07:44.875 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:44.910 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:44.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:44.915 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:07:45.018 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:45.048 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:45.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:45.055 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:07:45.155 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:45.185 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:45.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:45.191 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:07:45.292 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:45.318 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:07:45.324 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:45.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:45.331 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:07:45.432 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:45.440 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:45 smithi078 bash[78274]: cluster 2023-10-01T11:07:44.335131+0000 mgr.y (mgr.25072) 1512 : cluster 0 pgmap v1619: 89 pgs: 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 493 op/s; 8884/103374 objects degraded (8.594%); 4292/103374 objects misplaced (4.152%); 2.5 MiB/s, 40 objects/s recovering 2023-10-01T11:07:45.440 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:45 smithi078 bash[75299]: cluster 2023-10-01T11:07:44.335131+0000 mgr.y (mgr.25072) 1512 : cluster 0 pgmap v1619: 89 pgs: 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 493 op/s; 8884/103374 objects degraded (8.594%); 4292/103374 objects misplaced (4.152%); 2.5 MiB/s, 40 objects/s recovering 2023-10-01T11:07:45.458 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:45.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:45.462 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:07:45.563 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:45.589 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:45.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:45.592 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:07:45.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:45 smithi173 bash[58816]: cluster 2023-10-01T11:07:44.335131+0000 mgr.y (mgr.25072) 1512 : cluster 0 pgmap v1619: 89 pgs: 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 493 op/s; 8884/103374 objects degraded (8.594%); 4292/103374 objects misplaced (4.152%); 2.5 MiB/s, 40 objects/s recovering 2023-10-01T11:07:45.693 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:45.722 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:45.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:45.726 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:07:45.827 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:45.873 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:45.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:45.877 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:07:45.978 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:46.006 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:46.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:46.013 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:07:46.114 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:46.136 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:46.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:46.142 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:07:46.243 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:46.275 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:46.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:46.282 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:07:46.383 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:46.417 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:46.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:46.421 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:07:46.522 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:46.555 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:46.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:46.558 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:07:46.659 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:46.687 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:46.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:46.690 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:07:46.791 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:46.827 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:46.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:46.835 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:07:46.936 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:46.986 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:46.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:46.991 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:07:47.091 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:47.116 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:47.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:47.120 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:07:47.221 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:47.246 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:47.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:47.248 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:07:47.349 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:47.384 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:47.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:47.386 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:07:47.486 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:47.519 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:47.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:47.525 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:07:47.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:47 smithi173 bash[58816]: cluster 2023-10-01T11:07:46.335833+0000 mgr.y (mgr.25072) 1513 : cluster 0 pgmap v1620: 89 pgs: 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 529 op/s; 8716/100749 objects degraded (8.651%); 4292/100749 objects misplaced (4.260%); 3.3 MiB/s, 52 objects/s recovering 2023-10-01T11:07:47.626 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:47.668 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:47.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:47.672 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:07:47.772 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:47.782 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:47 smithi078 bash[75299]: cluster 2023-10-01T11:07:46.335833+0000 mgr.y (mgr.25072) 1513 : cluster 0 pgmap v1620: 89 pgs: 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 529 op/s; 8716/100749 objects degraded (8.651%); 4292/100749 objects misplaced (4.260%); 3.3 MiB/s, 52 objects/s recovering 2023-10-01T11:07:47.782 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:47 smithi078 bash[78274]: cluster 2023-10-01T11:07:46.335833+0000 mgr.y (mgr.25072) 1513 : cluster 0 pgmap v1620: 89 pgs: 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 529 op/s; 8716/100749 objects degraded (8.651%); 4292/100749 objects misplaced (4.260%); 3.3 MiB/s, 52 objects/s recovering 2023-10-01T11:07:47.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:47.801 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:47.802 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:07:47.902 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:47.927 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:47.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:47.929 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:07:48.029 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:48.062 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:48.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:48.065 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:07:48.166 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:48.208 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:48.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:48.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:07:48.314 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:48.324 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T11:07:48.350 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:48.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:48.352 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:07:48.368 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T11:07:48.409 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T11:07:48.452 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:48.454 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T11:07:48.461 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:48 smithi173 bash[58816]: audit 2023-10-01T11:07:47.465517+0000 mon.a (mon.0) 1962 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:07:48.461 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:48 smithi173 bash[58816]: cluster 2023-10-01T11:07:47.774511+0000 mon.a (mon.0) 1963 : cluster 3 Health check update: Degraded data redundancy: 8716/100749 objects degraded (8.651%), 5 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T11:07:48.461 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:48 smithi173 bash[58816]: audit 2023-10-01T11:07:48.277943+0000 mon.a (mon.0) 1964 : audit 0 from='client.? 172.21.15.78:0/2252461709' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:07:48.484 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:48.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:48.490 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:07:48.497 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T11:07:48.527 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T11:07:48.573 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T11:07:48.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:48 smithi078 bash[78274]: audit 2023-10-01T11:07:47.465517+0000 mon.a (mon.0) 1962 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:07:48.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:48 smithi078 bash[78274]: cluster 2023-10-01T11:07:47.774511+0000 mon.a (mon.0) 1963 : cluster 3 Health check update: Degraded data redundancy: 8716/100749 objects degraded (8.651%), 5 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T11:07:48.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:48 smithi078 bash[78274]: audit 2023-10-01T11:07:48.277943+0000 mon.a (mon.0) 1964 : audit 0 from='client.? 172.21.15.78:0/2252461709' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:07:48.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:48 smithi078 bash[75299]: audit 2023-10-01T11:07:47.465517+0000 mon.a (mon.0) 1962 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:07:48.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:48 smithi078 bash[75299]: cluster 2023-10-01T11:07:47.774511+0000 mon.a (mon.0) 1963 : cluster 3 Health check update: Degraded data redundancy: 8716/100749 objects degraded (8.651%), 5 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T11:07:48.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:48 smithi078 bash[75299]: audit 2023-10-01T11:07:48.277943+0000 mon.a (mon.0) 1964 : audit 0 from='client.? 172.21.15.78:0/2252461709' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:07:48.591 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:48.607 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T11:07:48.614 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T11:07:48.614 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T11:07:48.614 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T11:07:48.614 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T11:07:48.615 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T11:07:48.615 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T11:07:48.615 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T11:07:48.615 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T11:07:48.618 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:48.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:48.622 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:07:48.723 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:48.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:48.770 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:48.770 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:07:48.776 INFO:teuthology.orchestra.run.smithi078.stdout: 2023-10-01T11:07:48.790 INFO:teuthology.orchestra.run.smithi078.stderr:dumped all 2023-10-01T11:07:48.872 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:48.903 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:48.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:48.907 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:07:49.008 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:49.030 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:49.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:49.036 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:07:49.137 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:49.159 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:49.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:49.162 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:07:49.263 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:49.308 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:49.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:49.314 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:07:49.415 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:49.441 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:49.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:49.443 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:07:49.544 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:49.569 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:49.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:49.572 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:07:49.673 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:49.703 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:49.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:49.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:07:49.807 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:49.849 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:49.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:49.854 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:07:49.954 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:49.978 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:49.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:49.980 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:07:49.982 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:49 smithi078 bash[78274]: cluster 2023-10-01T11:07:48.338921+0000 mgr.y (mgr.25072) 1514 : cluster 0 pgmap v1621: 89 pgs: 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 555 op/s; 8716/97413 objects degraded (8.947%); 4113/97413 objects misplaced (4.222%); 2.9 MiB/s, 46 objects/s recovering 2023-10-01T11:07:49.982 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:49 smithi078 bash[78274]: audit 2023-10-01T11:07:48.758638+0000 mgr.y (mgr.25072) 1515 : audit 0 from='client.36732 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T11:07:49.983 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:49 smithi078 bash[75299]: cluster 2023-10-01T11:07:48.338921+0000 mgr.y (mgr.25072) 1514 : cluster 0 pgmap v1621: 89 pgs: 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 555 op/s; 8716/97413 objects degraded (8.947%); 4113/97413 objects misplaced (4.222%); 2.9 MiB/s, 46 objects/s recovering 2023-10-01T11:07:49.983 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:49 smithi078 bash[75299]: audit 2023-10-01T11:07:48.758638+0000 mgr.y (mgr.25072) 1515 : audit 0 from='client.36732 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T11:07:50.080 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:50.089 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:49 smithi173 bash[58816]: cluster 2023-10-01T11:07:48.338921+0000 mgr.y (mgr.25072) 1514 : cluster 0 pgmap v1621: 89 pgs: 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 2.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 555 op/s; 8716/97413 objects degraded (8.947%); 4113/97413 objects misplaced (4.222%); 2.9 MiB/s, 46 objects/s recovering 2023-10-01T11:07:50.090 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:49 smithi173 bash[58816]: audit 2023-10-01T11:07:48.758638+0000 mgr.y (mgr.25072) 1515 : audit 0 from='client.36732 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T11:07:50.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:50.113 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:50.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:07:50.214 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:50.244 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:50.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:50.247 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:07:50.347 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:50.394 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:50.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:50.399 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:07:50.500 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:50.508 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:07:50 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:07:50] "GET /metrics HTTP/1.1" 200 36405 "" "Prometheus/2.33.4" 2023-10-01T11:07:50.524 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:50.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:50.527 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:07:50.627 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:50.650 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:50.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:50.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:07:50.754 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:50.778 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:50.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:50.784 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:07:50.885 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:50.911 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:50.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:50.915 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:07:51.016 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:51.048 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:51.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:51.053 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:07:51.154 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:51.181 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:51.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:51.184 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:07:51.284 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:51.308 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:51.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:51.312 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:07:51.413 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:51.436 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:51.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:51.440 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:07:51.540 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:51.566 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:51.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:51.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:07:51.669 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:51.697 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:51.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:51.700 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:07:51.800 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:51.823 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:51.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:51.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:07:51.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:51 smithi173 bash[58816]: cluster 2023-10-01T11:07:50.339707+0000 mgr.y (mgr.25072) 1516 : cluster 0 pgmap v1622: 89 pgs: 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 8602/91776 objects degraded (9.373%); 3920/91776 objects misplaced (4.271%); 4.5 MiB/s, 71 objects/s recovering 2023-10-01T11:07:51.928 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:51.963 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:51.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:51.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:07:52.070 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:52.080 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:51 smithi078 bash[75299]: cluster 2023-10-01T11:07:50.339707+0000 mgr.y (mgr.25072) 1516 : cluster 0 pgmap v1622: 89 pgs: 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 8602/91776 objects degraded (9.373%); 3920/91776 objects misplaced (4.271%); 4.5 MiB/s, 71 objects/s recovering 2023-10-01T11:07:52.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:51 smithi078 bash[78274]: cluster 2023-10-01T11:07:50.339707+0000 mgr.y (mgr.25072) 1516 : cluster 0 pgmap v1622: 89 pgs: 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 8602/91776 objects degraded (9.373%); 3920/91776 objects misplaced (4.271%); 4.5 MiB/s, 71 objects/s recovering 2023-10-01T11:07:52.100 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:52.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:52.102 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:07:52.203 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:52.255 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:52.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:52.259 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:07:52.359 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:52.390 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:52.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:52.392 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:07:52.493 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:52.521 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:52.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:52.524 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:07:52.625 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:52.653 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:52.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:52.659 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:07:52.760 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:52.787 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:52.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:52.789 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:07:52.890 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:52.898 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:52 smithi078 bash[78274]: cluster 2023-10-01T11:07:52.775578+0000 mon.a (mon.0) 1965 : cluster 3 Health check update: Degraded data redundancy: 8602/91776 objects degraded (9.373%), 5 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T11:07:52.898 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:52 smithi078 bash[75299]: cluster 2023-10-01T11:07:52.775578+0000 mon.a (mon.0) 1965 : cluster 3 Health check update: Degraded data redundancy: 8602/91776 objects degraded (9.373%), 5 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T11:07:52.912 INFO:teuthology.orchestra.run.smithi078.stdout:{"pg_ready":true,"pg_map":{"version":1621,"stamp":"2023-10-01T11:07:48.338413+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2126976952,"num_objects":32471,"num_object_clones":0,"num_object_copies":97413,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8716,"num_objects_misplaced":4113,"num_objects_unfound":0,"num_objects_dirty":32471,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":126926,"num_write_kb":5093967,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22696,"num_bytes_recovered":1382520650,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":123460,"ondisk_log_size":123460,"up":267,"acting":262,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":281,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":14650964,"kb_used_data":7678228,"kb_used_omap":30,"kb_used_meta":6972641,"kb_avail":735310252,"statfs":{"total":767960285184,"available":752957698048,"internally_reserved":0,"allocated":7862505472,"data_stored":7854284816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31502,"internal_metadata":7139984626},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-437125120,"num_objects":-6670,"num_object_clones":0,"num_object_copies":-20010,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-3454,"num_objects_misplaced":1606,"num_objects_unfound":0,"num_objects_dirty":-6670,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6670,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":555,"num_bytes_recovered":36372480,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.010072"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":325,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.862699+0000","last_change":"2023-10-01T11:04:02.725956+0000","last_active":"2023-10-01T11:07:34.862699+0000","last_peered":"2023-10-01T11:07:34.862699+0000","last_clean":"2023-10-01T11:07:34.862699+0000","last_became_active":"2023-10-01T11:04:02.725317+0000","last_became_peered":"2023-10-01T11:04:02.725317+0000","last_unstale":"2023-10-01T11:07:34.862699+0000","last_undegraded":"2023-10-01T11:07:34.862699+0000","last_fullsized":"2023-10-01T11:07:34.862699+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:34:07.675650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":330,"reported_epoch":344,"state":"active+clean","last_fresh":"2023-10-01T11:07:35.845833+0000","last_change":"2023-10-01T10:38:16.025881+0000","last_active":"2023-10-01T11:07:35.845833+0000","last_peered":"2023-10-01T11:07:35.845833+0000","last_clean":"2023-10-01T11:07:35.845833+0000","last_became_active":"2023-10-01T10:38:16.025518+0000","last_became_peered":"2023-10-01T10:38:16.025518+0000","last_unstale":"2023-10-01T11:07:35.845833+0000","last_undegraded":"2023-10-01T11:07:35.845833+0000","last_fullsized":"2023-10-01T11:07:35.845833+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:48:37.826604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":250,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.846986+0000","last_change":"2023-10-01T10:52:18.956544+0000","last_active":"2023-10-01T11:07:34.846986+0000","last_peered":"2023-10-01T11:07:34.846986+0000","last_clean":"2023-10-01T11:07:34.846986+0000","last_became_active":"2023-10-01T10:52:18.956217+0000","last_became_peered":"2023-10-01T10:52:18.956217+0000","last_unstale":"2023-10-01T11:07:34.846986+0000","last_undegraded":"2023-10-01T11:07:34.846986+0000","last_fullsized":"2023-10-01T11:07:34.846986+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:34:01.952235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":347,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.847333+0000","last_change":"2023-10-01T11:06:11.786067+0000","last_active":"2023-10-01T11:07:34.847333+0000","last_peered":"2023-10-01T11:07:34.847333+0000","last_clean":"2023-10-01T11:07:34.847333+0000","last_became_active":"2023-10-01T11:06:11.785586+0000","last_became_peered":"2023-10-01T11:06:11.785586+0000","last_unstale":"2023-10-01T11:07:34.847333+0000","last_undegraded":"2023-10-01T11:07:34.847333+0000","last_fullsized":"2023-10-01T11:07:34.847333+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:09:53.087537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":331,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.846654+0000","last_change":"2023-10-01T10:37:42.766600+0000","last_active":"2023-10-01T11:07:34.846654+0000","last_peered":"2023-10-01T11:07:34.846654+0000","last_clean":"2023-10-01T11:07:34.846654+0000","last_became_active":"2023-10-01T10:37:42.766051+0000","last_became_peered":"2023-10-01T10:37:42.766051+0000","last_unstale":"2023-10-01T11:07:34.846654+0000","last_undegraded":"2023-10-01T11:07:34.846654+0000","last_fullsized":"2023-10-01T11:07:34.846654+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:51:17.707963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":30,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.861609+0000","last_change":"2023-10-01T11:05:33.800368+0000","last_active":"2023-10-01T11:07:34.861609+0000","last_peered":"2023-10-01T11:07:34.861609+0000","last_clean":"2023-10-01T11:07:34.861609+0000","last_became_active":"2023-10-01T11:05:33.799877+0000","last_became_peered":"2023-10-01T11:05:33.799877+0000","last_unstale":"2023-10-01T11:07:34.861609+0000","last_undegraded":"2023-10-01T11:07:34.861609+0000","last_fullsized":"2023-10-01T11:07:34.861609+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":104,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.859821+0000","last_change":"2023-10-01T11:04:40.881723+0000","last_active":"2023-10-01T11:07:34.859821+0000","last_peered":"2023-10-01T11:07:34.859821+0000","last_clean":"2023-10-01T11:07:34.859821+0000","last_became_active":"2023-10-01T11:04:40.881381+0000","last_became_peered":"2023-10-01T11:04:40.881381+0000","last_unstale":"2023-10-01T11:07:34.859821+0000","last_undegraded":"2023-10-01T11:07:34.859821+0000","last_fullsized":"2023-10-01T11:07:34.859821+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":369,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.846690+0000","last_change":"2023-10-01T11:05:33.782561+0000","last_active":"2023-10-01T11:07:34.846690+0000","last_peered":"2023-10-01T11:07:34.846690+0000","last_clean":"2023-10-01T11:07:34.846690+0000","last_became_active":"2023-10-01T11:05:33.782066+0000","last_became_peered":"2023-10-01T11:05:33.782066+0000","last_unstale":"2023-10-01T11:07:34.846690+0000","last_undegraded":"2023-10-01T11:07:34.846690+0000","last_fullsized":"2023-10-01T11:07:34.846690+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:57:06.451343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":347,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.870817+0000","last_change":"2023-10-01T11:06:11.792061+0000","last_active":"2023-10-01T11:07:34.870817+0000","last_peered":"2023-10-01T11:07:34.870817+0000","last_clean":"2023-10-01T11:07:34.870817+0000","last_became_active":"2023-10-01T11:06:11.789129+0000","last_became_peered":"2023-10-01T11:06:11.789129+0000","last_unstale":"2023-10-01T11:07:34.870817+0000","last_undegraded":"2023-10-01T11:07:34.870817+0000","last_fullsized":"2023-10-01T11:07:34.870817+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":389,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.846900+0000","last_change":"2023-10-01T11:06:11.807749+0000","last_active":"2023-10-01T11:07:34.846900+0000","last_peered":"2023-10-01T11:07:34.846900+0000","last_clean":"2023-10-01T11:07:34.846900+0000","last_became_active":"2023-10-01T11:06:11.807375+0000","last_became_peered":"2023-10-01T11:06:11.807375+0000","last_unstale":"2023-10-01T11:07:34.846900+0000","last_undegraded":"2023-10-01T11:07:34.846900+0000","last_fullsized":"2023-10-01T11:07:34.846900+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:44:05.945387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":249,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.847630+0000","last_change":"2023-10-01T11:06:11.816274+0000","last_active":"2023-10-01T11:07:34.847630+0000","last_peered":"2023-10-01T11:07:34.847630+0000","last_clean":"2023-10-01T11:07:34.847630+0000","last_became_active":"2023-10-01T11:06:11.790196+0000","last_became_peered":"2023-10-01T11:06:11.790196+0000","last_unstale":"2023-10-01T11:07:34.847630+0000","last_undegraded":"2023-10-01T11:07:34.847630+0000","last_fullsized":"2023-10-01T11:07:34.847630+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:09:46.957797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":30,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.860952+0000","last_change":"2023-10-01T11:05:33.785585+0000","last_active":"2023-10-01T11:07:34.860952+0000","last_peered":"2023-10-01T11:07:34.860952+0000","last_clean":"2023-10-01T11:07:34.860952+0000","last_became_active":"2023-10-01T11:05:33.783298+0000","last_became_peered":"2023-10-01T11:05:33.783298+0000","last_unstale":"2023-10-01T11:07:34.860952+0000","last_undegraded":"2023-10-01T11:07:34.860952+0000","last_fullsized":"2023-10-01T11:07:34.860952+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":453,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.862623+0000","last_change":"2023-10-01T11:06:11.836035+0000","last_active":"2023-10-01T11:07:34.862623+0000","last_peered":"2023-10-01T11:07:34.862623+0000","last_clean":"2023-10-01T11:07:34.862623+0000","last_became_active":"2023-10-01T11:06:11.794834+0000","last_became_peered":"2023-10-01T11:06:11.794834+0000","last_unstale":"2023-10-01T11:07:34.862623+0000","last_undegraded":"2023-10-01T11:07:34.862623+0000","last_fullsized":"2023-10-01T11:07:34.862623+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:25:13.103100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":33138,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":348,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.862175+0000","last_change":"2023-10-01T11:05:33.782898+0000","last_active":"2023-10-01T11:07:34.862175+0000","last_peered":"2023-10-01T11:07:34.862175+0000","last_clean":"2023-10-01T11:07:34.862175+0000","last_became_active":"2023-10-01T11:05:33.782135+0000","last_became_peered":"2023-10-01T11:05:33.782135+0000","last_unstale":"2023-10-01T11:07:34.862175+0000","last_undegraded":"2023-10-01T11:07:34.862175+0000","last_fullsized":"2023-10-01T11:07:34.862175+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:56:49.807416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":95,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.862276+0000","last_change":"2023-10-01T11:06:55.803414+0000","last_active":"2023-10-01T11:07:34.862276+0000","last_peered":"2023-10-01T11:07:34.862276+0000","last_clean":"2023-10-01T11:07:34.862276+0000","last_became_active":"2023-10-01T11:06:55.802738+0000","last_became_peered":"2023-10-01T11:06:55.802738+0000","last_unstale":"2023-10-01T11:07:34.862276+0000","last_undegraded":"2023-10-01T11:07:34.862276+0000","last_fullsized":"2023-10-01T11:07:34.862276+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:55:57.911799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":374,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.846818+0000","last_change":"2023-10-01T11:06:11.789794+0000","last_active":"2023-10-01T11:07:34.846818+0000","last_peered":"2023-10-01T11:07:34.846818+0000","last_clean":"2023-10-01T11:07:34.846818+0000","last_became_active":"2023-10-01T11:06:11.789452+0000","last_became_peered":"2023-10-01T11:06:11.789452+0000","last_unstale":"2023-10-01T11:07:34.846818+0000","last_undegraded":"2023-10-01T11:07:34.846818+0000","last_fullsized":"2023-10-01T11:07:34.846818+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:18:02.050694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":335,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.847278+0000","last_change":"2023-10-01T11:06:11.784794+0000","last_active":"2023-10-01T11:07:34.847278+0000","last_peered":"2023-10-01T11:07:34.847278+0000","last_clean":"2023-10-01T11:07:34.847278+0000","last_became_active":"2023-10-01T11:06:11.784368+0000","last_became_peered":"2023-10-01T11:06:11.784368+0000","last_unstale":"2023-10-01T11:07:34.847278+0000","last_undegraded":"2023-10-01T11:07:34.847278+0000","last_fullsized":"2023-10-01T11:07:34.847278+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:06:52.307449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":364,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.865705+0000","last_change":"2023-10-01T11:06:11.794671+0000","last_active":"2023-10-01T11:07:34.865705+0000","last_peered":"2023-10-01T11:07:34.865705+0000","last_clean":"2023-10-01T11:07:34.865705+0000","last_became_active":"2023-10-01T11:06:11.794121+0000","last_became_peered":"2023-10-01T11:06:11.794121+0000","last_unstale":"2023-10-01T11:07:34.865705+0000","last_undegraded":"2023-10-01T11:07:34.865705+0000","last_fullsized":"2023-10-01T11:07:34.865705+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":352,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.861475+0000","last_change":"2023-10-01T11:06:11.796259+0000","last_active":"2023-10-01T11:07:34.861475+0000","last_peered":"2023-10-01T11:07:34.861475+0000","last_clean":"2023-10-01T11:07:34.861475+0000","last_became_active":"2023-10-01T11:06:11.795870+0000","last_became_peered":"2023-10-01T11:06:11.795870+0000","last_unstale":"2023-10-01T11:07:34.861475+0000","last_undegraded":"2023-10-01T11:07:34.861475+0000","last_fullsized":"2023-10-01T11:07:34.861475+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:49:21.072163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":335,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.847194+0000","last_change":"2023-10-01T11:06:11.785871+0000","last_active":"2023-10-01T11:07:34.847194+0000","last_peered":"2023-10-01T11:07:34.847194+0000","last_clean":"2023-10-01T11:07:34.847194+0000","last_became_active":"2023-10-01T11:06:11.785183+0000","last_became_peered":"2023-10-01T11:06:11.785183+0000","last_unstale":"2023-10-01T11:07:34.847194+0000","last_undegraded":"2023-10-01T11:07:34.847194+0000","last_fullsized":"2023-10-01T11:07:34.847194+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:07:55.556404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":40,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.860916+0000","last_change":"2023-10-01T11:05:33.900352+0000","last_active":"2023-10-01T11:07:34.860916+0000","last_peered":"2023-10-01T11:07:34.860916+0000","last_clean":"2023-10-01T11:07:34.860916+0000","last_became_active":"2023-10-01T11:05:33.788906+0000","last_became_peered":"2023-10-01T11:05:33.788906+0000","last_unstale":"2023-10-01T11:07:34.860916+0000","last_undegraded":"2023-10-01T11:07:34.860916+0000","last_fullsized":"2023-10-01T11:07:34.860916+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":351,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.847789+0000","last_change":"2023-10-01T11:06:11.798727+0000","last_active":"2023-10-01T11:07:34.847789+0000","last_peered":"2023-10-01T11:07:34.847789+0000","last_clean":"2023-10-01T11:07:34.847789+0000","last_became_active":"2023-10-01T11:06:11.798388+0000","last_became_peered":"2023-10-01T11:06:11.798388+0000","last_unstale":"2023-10-01T11:07:34.847789+0000","last_undegraded":"2023-10-01T11:07:34.847789+0000","last_fullsized":"2023-10-01T11:07:34.847789+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:10:21.576627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.1","version":"344'7882","reported_seq":7981,"reported_epoch":344,"state":"active+clean","last_fresh":"2023-10-01T11:07:48.191823+0000","last_change":"2023-10-01T11:06:16.823243+0000","last_active":"2023-10-01T11:07:48.191823+0000","last_peered":"2023-10-01T11:07:48.191823+0000","last_clean":"2023-10-01T11:07:48.191823+0000","last_became_active":"2023-10-01T11:06:16.815377+0000","last_became_peered":"2023-10-01T11:06:16.815377+0000","last_unstale":"2023-10-01T11:07:48.191823+0000","last_undegraded":"2023-10-01T11:07:48.191823+0000","last_fullsized":"2023-10-01T11:07:48.191823+0000","mapping_epoch":334,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":335,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_clean_scrub_stamp":"2023-10-01T11:01:36.981250+0000","objects_scrubbed":0,"log_size":7882,"log_dups_size":0,"ondisk_log_size":7882,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":126615552,"num_objects":1932,"num_object_clones":0,"num_object_copies":5796,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1932,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7882,"num_write_kb":314048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":810,"num_bytes_recovered":23396352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":240,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.870914+0000","last_change":"2023-10-01T11:06:11.840628+0000","last_active":"2023-10-01T11:07:34.870914+0000","last_peered":"2023-10-01T11:07:34.870914+0000","last_clean":"2023-10-01T11:07:34.870914+0000","last_became_active":"2023-10-01T11:06:11.800187+0000","last_became_peered":"2023-10-01T11:06:11.800187+0000","last_unstale":"2023-10-01T11:07:34.870914+0000","last_undegraded":"2023-10-01T11:07:34.870914+0000","last_fullsized":"2023-10-01T11:07:34.870914+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:24:48.900425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":385,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.864887+0000","last_change":"2023-10-01T11:06:11.830255+0000","last_active":"2023-10-01T11:07:34.864887+0000","last_peered":"2023-10-01T11:07:34.864887+0000","last_clean":"2023-10-01T11:07:34.864887+0000","last_became_active":"2023-10-01T11:06:11.829847+0000","last_became_peered":"2023-10-01T11:06:11.829847+0000","last_unstale":"2023-10-01T11:07:34.864887+0000","last_undegraded":"2023-10-01T11:07:34.864887+0000","last_fullsized":"2023-10-01T11:07:34.864887+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.0","version":"344'8035","reported_seq":11030,"reported_epoch":344,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T11:07:47.957591+0000","last_change":"2023-10-01T11:06:11.952805+0000","last_active":"2023-10-01T11:07:47.957591+0000","last_peered":"2023-10-01T11:07:47.957591+0000","last_clean":"2023-10-01T11:02:35.746836+0000","last_became_active":"2023-10-01T11:06:11.943209+0000","last_became_peered":"2023-10-01T11:06:11.943209+0000","last_unstale":"2023-10-01T11:07:47.957591+0000","last_undegraded":"2023-10-01T11:06:11.787697+0000","last_fullsized":"2023-10-01T11:06:11.769959+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_clean_scrub_stamp":"2023-10-01T11:01:36.981250+0000","objects_scrubbed":0,"log_size":8035,"log_dups_size":0,"ondisk_log_size":8035,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:31:25.633240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122486784,"num_objects":1869,"num_object_clones":0,"num_object_copies":5607,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2483,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1869,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8035,"num_write_kb":316928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1445,"num_bytes_recovered":94699520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,1],"avail_no_missing":["2","1"],"object_location_counts":[{"shards":"0,1,2","objects":5},{"shards":"1,2","objects":103},{"shards":"1,2,6","objects":123}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":199,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.865449+0000","last_change":"2023-10-01T11:06:11.813139+0000","last_active":"2023-10-01T11:07:34.865449+0000","last_peered":"2023-10-01T11:07:34.865449+0000","last_clean":"2023-10-01T11:07:34.865449+0000","last_became_active":"2023-10-01T11:06:11.790124+0000","last_became_peered":"2023-10-01T11:06:11.790124+0000","last_unstale":"2023-10-01T11:07:34.865449+0000","last_undegraded":"2023-10-01T11:07:34.865449+0000","last_fullsized":"2023-10-01T11:07:34.865449+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:30:46.077244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":328,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.861555+0000","last_change":"2023-10-01T11:04:02.724840+0000","last_active":"2023-10-01T11:07:34.861555+0000","last_peered":"2023-10-01T11:07:34.861555+0000","last_clean":"2023-10-01T11:07:34.861555+0000","last_became_active":"2023-10-01T11:04:02.724476+0000","last_became_peered":"2023-10-01T11:04:02.724476+0000","last_unstale":"2023-10-01T11:07:34.861555+0000","last_undegraded":"2023-10-01T11:07:34.861555+0000","last_fullsized":"2023-10-01T11:07:34.861555+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:05:08.882495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.3","version":"344'7792","reported_seq":7871,"reported_epoch":344,"state":"active+clean","last_fresh":"2023-10-01T11:07:44.625447+0000","last_change":"2023-10-01T11:04:40.889413+0000","last_active":"2023-10-01T11:07:44.625447+0000","last_peered":"2023-10-01T11:07:44.625447+0000","last_clean":"2023-10-01T11:07:44.625447+0000","last_became_active":"2023-10-01T11:04:40.877803+0000","last_became_peered":"2023-10-01T11:04:40.877803+0000","last_unstale":"2023-10-01T11:07:44.625447+0000","last_undegraded":"2023-10-01T11:07:44.625447+0000","last_fullsized":"2023-10-01T11:07:44.625447+0000","mapping_epoch":317,"log_start":"291'600","ondisk_log_start":"291'600","created":284,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_clean_scrub_stamp":"2023-10-01T11:01:36.981250+0000","objects_scrubbed":0,"log_size":7192,"log_dups_size":0,"ondisk_log_size":7192,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T22:24:24.600680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141033472,"num_objects":2152,"num_object_clones":0,"num_object_copies":6456,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2152,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7792,"num_write_kb":318208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":313,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.865796+0000","last_change":"2023-10-01T11:06:11.820664+0000","last_active":"2023-10-01T11:07:34.865796+0000","last_peered":"2023-10-01T11:07:34.865796+0000","last_clean":"2023-10-01T11:07:34.865796+0000","last_became_active":"2023-10-01T11:06:11.792931+0000","last_became_peered":"2023-10-01T11:06:11.792931+0000","last_unstale":"2023-10-01T11:07:34.865796+0000","last_undegraded":"2023-10-01T11:07:34.865796+0000","last_fullsized":"2023-10-01T11:07:34.865796+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":700,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":39,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.861121+0000","last_change":"2023-10-01T11:06:11.795096+0000","last_active":"2023-10-01T11:07:34.861121+0000","last_peered":"2023-10-01T11:07:34.861121+0000","last_clean":"2023-10-01T11:07:34.861121+0000","last_became_active":"2023-10-01T11:06:11.794716+0000","last_became_peered":"2023-10-01T11:06:11.794716+0000","last_unstale":"2023-10-01T11:07:34.861121+0000","last_undegraded":"2023-10-01T11:07:34.861121+0000","last_fullsized":"2023-10-01T11:07:34.861121+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.2","version":"344'7720","reported_seq":7788,"reported_epoch":344,"state":"active+clean","last_fresh":"2023-10-01T11:07:43.678762+0000","last_change":"2023-10-01T11:01:38.006227+0000","last_active":"2023-10-01T11:07:43.678762+0000","last_peered":"2023-10-01T11:07:43.678762+0000","last_clean":"2023-10-01T11:07:43.678762+0000","last_became_active":"2023-10-01T11:01:38.005895+0000","last_became_peered":"2023-10-01T11:01:38.005895+0000","last_unstale":"2023-10-01T11:07:43.678762+0000","last_undegraded":"2023-10-01T11:07:43.678762+0000","last_fullsized":"2023-10-01T11:07:43.678762+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_clean_scrub_stamp":"2023-10-01T11:01:36.981250+0000","objects_scrubbed":0,"log_size":7720,"log_dups_size":0,"ondisk_log_size":7720,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138149888,"num_objects":2108,"num_object_clones":0,"num_object_copies":6324,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2108,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7720,"num_write_kb":314496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":365,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.870039+0000","last_change":"2023-10-01T11:05:33.788917+0000","last_active":"2023-10-01T11:07:34.870039+0000","last_peered":"2023-10-01T11:07:34.870039+0000","last_clean":"2023-10-01T11:07:34.870039+0000","last_became_active":"2023-10-01T11:05:33.788078+0000","last_became_peered":"2023-10-01T11:05:33.788078+0000","last_unstale":"2023-10-01T11:07:34.870039+0000","last_undegraded":"2023-10-01T11:07:34.870039+0000","last_fullsized":"2023-10-01T11:07:34.870039+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":127,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.862358+0000","last_change":"2023-10-01T11:04:40.878153+0000","last_active":"2023-10-01T11:07:34.862358+0000","last_peered":"2023-10-01T11:07:34.862358+0000","last_clean":"2023-10-01T11:07:34.862358+0000","last_became_active":"2023-10-01T11:04:40.877734+0000","last_became_peered":"2023-10-01T11:04:40.877734+0000","last_unstale":"2023-10-01T11:07:34.862358+0000","last_undegraded":"2023-10-01T11:07:34.862358+0000","last_fullsized":"2023-10-01T11:07:34.862358+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:36:14.253484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.5","version":"344'7961","reported_seq":8028,"reported_epoch":344,"state":"active+clean","last_fresh":"2023-10-01T11:07:44.617621+0000","last_change":"2023-10-01T11:04:02.733362+0000","last_active":"2023-10-01T11:07:44.617621+0000","last_peered":"2023-10-01T11:07:44.617621+0000","last_clean":"2023-10-01T11:07:44.617621+0000","last_became_active":"2023-10-01T11:04:02.728059+0000","last_became_peered":"2023-10-01T11:04:02.728059+0000","last_unstale":"2023-10-01T11:07:44.617621+0000","last_undegraded":"2023-10-01T11:07:44.617621+0000","last_fullsized":"2023-10-01T11:07:44.617621+0000","mapping_epoch":313,"log_start":"291'800","ondisk_log_start":"291'800","created":284,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_clean_scrub_stamp":"2023-10-01T11:01:36.981250+0000","objects_scrubbed":0,"log_size":7161,"log_dups_size":0,"ondisk_log_size":7161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:46:23.560869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":130220032,"num_objects":1987,"num_object_clones":0,"num_object_copies":5961,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1987,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7961,"num_write_kb":318336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":319,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.853199+0000","last_change":"2023-10-01T10:38:16.022493+0000","last_active":"2023-10-01T11:07:34.853199+0000","last_peered":"2023-10-01T11:07:34.853199+0000","last_clean":"2023-10-01T11:07:34.853199+0000","last_became_active":"2023-10-01T10:38:16.022033+0000","last_became_peered":"2023-10-01T10:38:16.022033+0000","last_unstale":"2023-10-01T11:07:34.853199+0000","last_undegraded":"2023-10-01T11:07:34.853199+0000","last_fullsized":"2023-10-01T11:07:34.853199+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:49:05.262763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":358,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.862718+0000","last_change":"2023-10-01T11:05:33.785133+0000","last_active":"2023-10-01T11:07:34.862718+0000","last_peered":"2023-10-01T11:07:34.862718+0000","last_clean":"2023-10-01T11:07:34.862718+0000","last_became_active":"2023-10-01T11:05:33.784749+0000","last_became_peered":"2023-10-01T11:05:33.784749+0000","last_unstale":"2023-10-01T11:07:34.862718+0000","last_undegraded":"2023-10-01T11:07:34.862718+0000","last_fullsized":"2023-10-01T11:07:34.862718+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:08:01.664432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.4","version":"344'7919","reported_seq":9284,"reported_epoch":344,"state":"active+clean","last_fresh":"2023-10-01T11:07:48.272338+0000","last_change":"2023-10-01T11:05:38.770432+0000","last_active":"2023-10-01T11:07:48.272338+0000","last_peered":"2023-10-01T11:07:48.272338+0000","last_clean":"2023-10-01T11:07:48.272338+0000","last_became_active":"2023-10-01T11:05:38.761825+0000","last_became_peered":"2023-10-01T11:05:38.761825+0000","last_unstale":"2023-10-01T11:07:48.272338+0000","last_undegraded":"2023-10-01T11:07:48.272338+0000","last_fullsized":"2023-10-01T11:07:48.272338+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_clean_scrub_stamp":"2023-10-01T11:01:36.981250+0000","objects_scrubbed":0,"log_size":7919,"log_dups_size":0,"ondisk_log_size":7919,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":128385024,"num_objects":1959,"num_object_clones":0,"num_object_copies":5877,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1959,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7919,"num_write_kb":316096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":634,"num_bytes_recovered":41549824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":331,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.862558+0000","last_change":"2023-10-01T11:05:33.782142+0000","last_active":"2023-10-01T11:07:34.862558+0000","last_peered":"2023-10-01T11:07:34.862558+0000","last_clean":"2023-10-01T11:07:34.862558+0000","last_became_active":"2023-10-01T11:05:33.781725+0000","last_became_peered":"2023-10-01T11:05:33.781725+0000","last_unstale":"2023-10-01T11:07:34.862558+0000","last_undegraded":"2023-10-01T11:07:34.862558+0000","last_fullsized":"2023-10-01T11:07:34.862558+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:58:37.958611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":318,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.868233+0000","last_change":"2023-10-01T11:04:02.727282+0000","last_active":"2023-10-01T11:07:34.868233+0000","last_peered":"2023-10-01T11:07:34.868233+0000","last_clean":"2023-10-01T11:07:34.868233+0000","last_became_active":"2023-10-01T11:04:02.726818+0000","last_became_peered":"2023-10-01T11:04:02.726818+0000","last_unstale":"2023-10-01T11:07:34.868233+0000","last_undegraded":"2023-10-01T11:07:34.868233+0000","last_fullsized":"2023-10-01T11:07:34.868233+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:11:11.868291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.7","version":"344'8039","reported_seq":14645,"reported_epoch":344,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T11:07:44.694161+0000","last_change":"2023-10-01T11:06:56.931001+0000","last_active":"2023-10-01T11:07:44.694161+0000","last_peered":"2023-10-01T11:07:44.694161+0000","last_clean":"2023-10-01T11:02:35.706450+0000","last_became_active":"2023-10-01T11:06:56.875224+0000","last_became_peered":"2023-10-01T11:06:56.875224+0000","last_unstale":"2023-10-01T11:07:44.694161+0000","last_undegraded":"2023-10-01T11:06:56.862861+0000","last_fullsized":"2023-10-01T11:06:56.799531+0000","mapping_epoch":339,"log_start":"291'800","ondisk_log_start":"291'800","created":284,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_clean_scrub_stamp":"2023-10-01T11:01:36.981250+0000","objects_scrubbed":0,"log_size":7239,"log_dups_size":0,"ondisk_log_size":7239,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:11:43.167681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134676480,"num_objects":2055,"num_object_clones":0,"num_object_copies":6165,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1124,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2055,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8039,"num_write_kb":323008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2979,"num_bytes_recovered":195166208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"0,4,5,7","objects":16},{"shards":"4,5,6","objects":265},{"shards":"4,5,7","objects":357},{"shards":"4,7","objects":1417}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":362,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.861891+0000","last_change":"2023-10-01T11:06:11.790944+0000","last_active":"2023-10-01T11:07:34.861891+0000","last_peered":"2023-10-01T11:07:34.861891+0000","last_clean":"2023-10-01T11:07:34.861891+0000","last_became_active":"2023-10-01T11:06:11.790151+0000","last_became_peered":"2023-10-01T11:06:11.790151+0000","last_unstale":"2023-10-01T11:07:34.861891+0000","last_undegraded":"2023-10-01T11:07:34.861891+0000","last_fullsized":"2023-10-01T11:07:34.861891+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:05:49.161868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":360,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.865377+0000","last_change":"2023-10-01T11:05:33.781934+0000","last_active":"2023-10-01T11:07:34.865377+0000","last_peered":"2023-10-01T11:07:34.865377+0000","last_clean":"2023-10-01T11:07:34.865377+0000","last_became_active":"2023-10-01T11:05:33.781555+0000","last_became_peered":"2023-10-01T11:05:33.781555+0000","last_unstale":"2023-10-01T11:07:34.865377+0000","last_undegraded":"2023-10-01T11:07:34.865377+0000","last_fullsized":"2023-10-01T11:07:34.865377+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:59:22.019053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.6","version":"344'7819","reported_seq":7886,"reported_epoch":344,"state":"active+clean","last_fresh":"2023-10-01T11:07:44.621986+0000","last_change":"2023-10-01T11:04:02.733583+0000","last_active":"2023-10-01T11:07:44.621986+0000","last_peered":"2023-10-01T11:07:44.621986+0000","last_clean":"2023-10-01T11:07:44.621986+0000","last_became_active":"2023-10-01T11:04:02.727060+0000","last_became_peered":"2023-10-01T11:04:02.727060+0000","last_unstale":"2023-10-01T11:07:44.621986+0000","last_undegraded":"2023-10-01T11:07:44.621986+0000","last_fullsized":"2023-10-01T11:07:44.621986+0000","mapping_epoch":313,"log_start":"291'600","ondisk_log_start":"291'600","created":284,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_clean_scrub_stamp":"2023-10-01T11:01:36.981250+0000","objects_scrubbed":0,"log_size":7219,"log_dups_size":0,"ondisk_log_size":7219,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:01:54.121478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132710400,"num_objects":2025,"num_object_clones":0,"num_object_copies":6075,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2025,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7819,"num_write_kb":315008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":353,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.846603+0000","last_change":"2023-10-01T11:06:11.784722+0000","last_active":"2023-10-01T11:07:34.846603+0000","last_peered":"2023-10-01T11:07:34.846603+0000","last_clean":"2023-10-01T11:07:34.846603+0000","last_became_active":"2023-10-01T11:06:11.784348+0000","last_became_peered":"2023-10-01T11:06:11.784348+0000","last_unstale":"2023-10-01T11:07:34.846603+0000","last_undegraded":"2023-10-01T11:07:34.846603+0000","last_fullsized":"2023-10-01T11:07:34.846603+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:40:29.283179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":335,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.848051+0000","last_change":"2023-10-01T10:38:16.019755+0000","last_active":"2023-10-01T11:07:34.848051+0000","last_peered":"2023-10-01T11:07:34.848051+0000","last_clean":"2023-10-01T11:07:34.848051+0000","last_became_active":"2023-10-01T10:38:16.019397+0000","last_became_peered":"2023-10-01T10:38:16.019397+0000","last_unstale":"2023-10-01T11:07:34.848051+0000","last_undegraded":"2023-10-01T11:07:34.848051+0000","last_fullsized":"2023-10-01T11:07:34.848051+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:42:56.623225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.b","version":"344'7664","reported_seq":7732,"reported_epoch":344,"state":"active+clean","last_fresh":"2023-10-01T11:07:43.581477+0000","last_change":"2023-10-01T11:01:38.006450+0000","last_active":"2023-10-01T11:07:43.581477+0000","last_peered":"2023-10-01T11:07:43.581477+0000","last_clean":"2023-10-01T11:07:43.581477+0000","last_became_active":"2023-10-01T11:01:38.005600+0000","last_became_peered":"2023-10-01T11:01:38.005600+0000","last_unstale":"2023-10-01T11:07:43.581477+0000","last_undegraded":"2023-10-01T11:07:43.581477+0000","last_fullsized":"2023-10-01T11:07:43.581477+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_clean_scrub_stamp":"2023-10-01T11:01:36.981250+0000","objects_scrubbed":0,"log_size":7664,"log_dups_size":0,"ondisk_log_size":7664,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134348800,"num_objects":2050,"num_object_clones":0,"num_object_copies":6150,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2050,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7664,"num_write_kb":310848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":245,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.853341+0000","last_change":"2023-10-01T11:06:11.788951+0000","last_active":"2023-10-01T11:07:34.853341+0000","last_peered":"2023-10-01T11:07:34.853341+0000","last_clean":"2023-10-01T11:07:34.853341+0000","last_became_active":"2023-10-01T11:06:11.788617+0000","last_became_peered":"2023-10-01T11:06:11.788617+0000","last_unstale":"2023-10-01T11:07:34.853341+0000","last_undegraded":"2023-10-01T11:07:34.853341+0000","last_fullsized":"2023-10-01T11:07:34.853341+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:07:46.629250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":326,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.865623+0000","last_change":"2023-10-01T10:37:42.766814+0000","last_active":"2023-10-01T11:07:34.865623+0000","last_peered":"2023-10-01T11:07:34.865623+0000","last_clean":"2023-10-01T11:07:34.865623+0000","last_became_active":"2023-10-01T10:37:42.766451+0000","last_became_peered":"2023-10-01T10:37:42.766451+0000","last_unstale":"2023-10-01T11:07:34.865623+0000","last_undegraded":"2023-10-01T11:07:34.865623+0000","last_fullsized":"2023-10-01T11:07:34.865623+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":229,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.859814+0000","last_change":"2023-10-01T10:52:18.960606+0000","last_active":"2023-10-01T11:07:34.859814+0000","last_peered":"2023-10-01T11:07:34.859814+0000","last_clean":"2023-10-01T11:07:34.859814+0000","last_became_active":"2023-10-01T10:52:18.960270+0000","last_became_peered":"2023-10-01T10:52:18.960270+0000","last_unstale":"2023-10-01T11:07:34.859814+0000","last_undegraded":"2023-10-01T11:07:34.859814+0000","last_fullsized":"2023-10-01T11:07:34.859814+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.d","version":"344'7830","reported_seq":23231,"reported_epoch":344,"state":"active+clean","last_fresh":"2023-10-01T11:07:44.632012+0000","last_change":"2023-10-01T11:07:35.880902+0000","last_active":"2023-10-01T11:07:44.632012+0000","last_peered":"2023-10-01T11:07:44.632012+0000","last_clean":"2023-10-01T11:07:44.632012+0000","last_became_active":"2023-10-01T11:07:35.854260+0000","last_became_peered":"2023-10-01T11:07:35.854260+0000","last_unstale":"2023-10-01T11:07:44.632012+0000","last_undegraded":"2023-10-01T11:07:44.632012+0000","last_fullsized":"2023-10-01T11:07:44.632012+0000","mapping_epoch":343,"log_start":"291'600","ondisk_log_start":"291'600","created":284,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_clean_scrub_stamp":"2023-10-01T11:01:36.981250+0000","objects_scrubbed":0,"log_size":7230,"log_dups_size":0,"ondisk_log_size":7230,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:40:25.476868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137429016,"num_objects":2098,"num_object_clones":0,"num_object_copies":6294,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2098,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7830,"num_write_kb":317633,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7641,"num_bytes_recovered":456523800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":369,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.859899+0000","last_change":"2023-10-01T11:05:33.792877+0000","last_active":"2023-10-01T11:07:34.859899+0000","last_peered":"2023-10-01T11:07:34.859899+0000","last_clean":"2023-10-01T11:07:34.859899+0000","last_became_active":"2023-10-01T11:05:33.792054+0000","last_became_peered":"2023-10-01T11:05:33.792054+0000","last_unstale":"2023-10-01T11:07:34.859899+0000","last_undegraded":"2023-10-01T11:07:34.859899+0000","last_fullsized":"2023-10-01T11:07:34.859899+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":326,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.862726+0000","last_change":"2023-10-01T11:05:33.783100+0000","last_active":"2023-10-01T11:07:34.862726+0000","last_peered":"2023-10-01T11:07:34.862726+0000","last_clean":"2023-10-01T11:07:34.862726+0000","last_became_active":"2023-10-01T11:05:33.782557+0000","last_became_peered":"2023-10-01T11:05:33.782557+0000","last_unstale":"2023-10-01T11:07:34.862726+0000","last_undegraded":"2023-10-01T11:07:34.862726+0000","last_fullsized":"2023-10-01T11:07:34.862726+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:52:05.581697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":246,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.847748+0000","last_change":"2023-10-01T10:52:18.963437+0000","last_active":"2023-10-01T11:07:34.847748+0000","last_peered":"2023-10-01T11:07:34.847748+0000","last_clean":"2023-10-01T11:07:34.847748+0000","last_became_active":"2023-10-01T10:52:18.955165+0000","last_became_peered":"2023-10-01T10:52:18.955165+0000","last_unstale":"2023-10-01T11:07:34.847748+0000","last_undegraded":"2023-10-01T11:07:34.847748+0000","last_fullsized":"2023-10-01T11:07:34.847748+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:57:49.129100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.e","version":"344'7771","reported_seq":12039,"reported_epoch":344,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-10-01T11:07:43.654303+0000","last_change":"2023-10-01T11:07:34.367871+0000","last_active":"2023-10-01T11:07:43.654303+0000","last_peered":"2023-10-01T11:07:43.654303+0000","last_clean":"2023-10-01T11:06:10.738686+0000","last_became_active":"2023-10-01T11:06:13.042167+0000","last_became_peered":"2023-10-01T11:06:13.042167+0000","last_unstale":"2023-10-01T11:07:43.654303+0000","last_undegraded":"2023-10-01T11:06:12.822146+0000","last_fullsized":"2023-10-01T11:06:12.780968+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":310,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_clean_scrub_stamp":"2023-10-01T11:01:36.981250+0000","objects_scrubbed":0,"log_size":7771,"log_dups_size":0,"ondisk_log_size":7771,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141361152,"num_objects":2157,"num_object_clones":0,"num_object_copies":6471,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1857,"num_objects_misplaced":2152,"num_objects_unfound":0,"num_objects_dirty":2157,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7771,"num_write_kb":317696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2081,"num_bytes_recovered":136314880,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":2152},{"shards":"4,5","objects":-1852}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":327,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.847591+0000","last_change":"2023-10-01T10:37:42.762524+0000","last_active":"2023-10-01T11:07:34.847591+0000","last_peered":"2023-10-01T11:07:34.847591+0000","last_clean":"2023-10-01T11:07:34.847591+0000","last_became_active":"2023-10-01T10:37:42.762209+0000","last_became_peered":"2023-10-01T10:37:42.762209+0000","last_unstale":"2023-10-01T11:07:34.847591+0000","last_undegraded":"2023-10-01T11:07:34.847591+0000","last_fullsized":"2023-10-01T11:07:34.847591+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:45:38.171397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":280,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.853070+0000","last_change":"2023-10-01T11:05:33.799413+0000","last_active":"2023-10-01T11:07:34.853070+0000","last_peered":"2023-10-01T11:07:34.853070+0000","last_clean":"2023-10-01T11:07:34.853070+0000","last_became_active":"2023-10-01T11:05:33.799158+0000","last_became_peered":"2023-10-01T11:05:33.799158+0000","last_unstale":"2023-10-01T11:07:34.853070+0000","last_undegraded":"2023-10-01T11:07:34.853070+0000","last_fullsized":"2023-10-01T11:07:34.853070+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:39:05.963181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":231,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.859867+0000","last_change":"2023-10-01T10:52:18.961040+0000","last_active":"2023-10-01T11:07:34.859867+0000","last_peered":"2023-10-01T11:07:34.859867+0000","last_clean":"2023-10-01T11:07:34.859867+0000","last_became_active":"2023-10-01T10:52:18.960606+0000","last_became_peered":"2023-10-01T10:52:18.960606+0000","last_unstale":"2023-10-01T11:07:34.859867+0000","last_undegraded":"2023-10-01T11:07:34.859867+0000","last_fullsized":"2023-10-01T11:07:34.859867+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.c","version":"344'7666","reported_seq":7734,"reported_epoch":344,"state":"active+clean","last_fresh":"2023-10-01T11:07:43.520633+0000","last_change":"2023-10-01T11:01:38.196508+0000","last_active":"2023-10-01T11:07:43.520633+0000","last_peered":"2023-10-01T11:07:43.520633+0000","last_clean":"2023-10-01T11:07:43.520633+0000","last_became_active":"2023-10-01T11:01:38.195817+0000","last_became_peered":"2023-10-01T11:01:38.195817+0000","last_unstale":"2023-10-01T11:07:43.520633+0000","last_undegraded":"2023-10-01T11:07:43.520633+0000","last_fullsized":"2023-10-01T11:07:43.520633+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_clean_scrub_stamp":"2023-10-01T11:01:36.981250+0000","objects_scrubbed":0,"log_size":7666,"log_dups_size":0,"ondisk_log_size":7666,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136577024,"num_objects":2084,"num_object_clones":0,"num_object_copies":6252,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2084,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7666,"num_write_kb":312000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":351,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.865836+0000","last_change":"2023-10-01T11:06:11.791607+0000","last_active":"2023-10-01T11:07:34.865836+0000","last_peered":"2023-10-01T11:07:34.865836+0000","last_clean":"2023-10-01T11:07:34.865836+0000","last_became_active":"2023-10-01T11:06:11.789667+0000","last_became_peered":"2023-10-01T11:06:11.789667+0000","last_unstale":"2023-10-01T11:07:34.865836+0000","last_undegraded":"2023-10-01T11:07:34.865836+0000","last_fullsized":"2023-10-01T11:07:34.865836+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'139","reported_seq":637,"reported_epoch":343,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T11:07:34.864449+0000","last_change":"2023-10-01T11:06:12.785972+0000","last_active":"2023-10-01T11:07:34.864449+0000","last_peered":"2023-10-01T11:07:34.864449+0000","last_clean":"2023-10-01T11:05:55.911294+0000","last_became_active":"2023-10-01T11:06:12.785562+0000","last_became_peered":"2023-10-01T11:06:12.785562+0000","last_unstale":"2023-10-01T11:07:34.864449+0000","last_undegraded":"2023-10-01T11:06:12.774372+0000","last_fullsized":"2023-10-01T11:06:12.773739+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:09:17.221082+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:09:17.221082+0000","last_clean_scrub_stamp":"2023-10-01T10:09:17.221082+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:58:09.966597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":4722944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[0,6],"avail_no_missing":["0","1","2","6"],"object_location_counts":[{"shards":"0,1,2,6","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":354,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.847196+0000","last_change":"2023-10-01T11:05:33.788093+0000","last_active":"2023-10-01T11:07:34.847196+0000","last_peered":"2023-10-01T11:07:34.847196+0000","last_clean":"2023-10-01T11:07:34.847196+0000","last_became_active":"2023-10-01T11:05:33.787723+0000","last_became_peered":"2023-10-01T11:05:33.787723+0000","last_unstale":"2023-10-01T11:07:34.847196+0000","last_undegraded":"2023-10-01T11:07:34.847196+0000","last_fullsized":"2023-10-01T11:07:34.847196+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:04:55.138186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":376,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.868092+0000","last_change":"2023-10-01T11:06:11.788535+0000","last_active":"2023-10-01T11:07:34.868092+0000","last_peered":"2023-10-01T11:07:34.868092+0000","last_clean":"2023-10-01T11:07:34.868092+0000","last_became_active":"2023-10-01T11:06:11.788103+0000","last_became_peered":"2023-10-01T11:06:11.788103+0000","last_unstale":"2023-10-01T11:07:34.868092+0000","last_undegraded":"2023-10-01T11:07:34.868092+0000","last_fullsized":"2023-10-01T11:07:34.868092+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:02:49.482834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.f","version":"344'8222","reported_seq":13011,"reported_epoch":344,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-10-01T11:07:48.166673+0000","last_change":"2023-10-01T11:06:22.449717+0000","last_active":"2023-10-01T11:07:48.166673+0000","last_peered":"2023-10-01T11:07:48.166673+0000","last_clean":"2023-10-01T11:06:10.735600+0000","last_became_active":"2023-10-01T11:06:22.434934+0000","last_became_peered":"2023-10-01T11:06:22.434934+0000","last_unstale":"2023-10-01T11:07:48.166673+0000","last_undegraded":"2023-10-01T11:06:12.800781+0000","last_fullsized":"2023-10-01T11:06:12.777737+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":330,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_clean_scrub_stamp":"2023-10-01T11:01:36.981250+0000","objects_scrubbed":0,"log_size":8222,"log_dups_size":0,"ondisk_log_size":8222,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129236992,"num_objects":1972,"num_object_clones":0,"num_object_copies":5916,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3250,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1972,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8222,"num_write_kb":326208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2332,"num_bytes_recovered":123666432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,2],"avail_no_missing":["6","2","5"],"object_location_counts":[{"shards":"2,5,6","objects":125},{"shards":"2,6","objects":-535}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":22,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.870728+0000","last_change":"2023-10-01T11:06:11.788462+0000","last_active":"2023-10-01T11:07:34.870728+0000","last_peered":"2023-10-01T11:07:34.870728+0000","last_clean":"2023-10-01T11:07:34.870728+0000","last_became_active":"2023-10-01T11:06:11.787195+0000","last_became_peered":"2023-10-01T11:06:11.787195+0000","last_unstale":"2023-10-01T11:07:34.870728+0000","last_undegraded":"2023-10-01T11:07:34.870728+0000","last_fullsized":"2023-10-01T11:07:34.870728+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":126,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.861791+0000","last_change":"2023-10-01T11:04:40.868156+0000","last_active":"2023-10-01T11:07:34.861791+0000","last_peered":"2023-10-01T11:07:34.861791+0000","last_clean":"2023-10-01T11:07:34.861791+0000","last_became_active":"2023-10-01T11:04:40.867864+0000","last_became_peered":"2023-10-01T11:04:40.867864+0000","last_unstale":"2023-10-01T11:07:34.861791+0000","last_undegraded":"2023-10-01T11:07:34.861791+0000","last_fullsized":"2023-10-01T11:07:34.861791+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:37:15.571777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":117,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.847747+0000","last_change":"2023-10-01T11:04:40.880090+0000","last_active":"2023-10-01T11:07:34.847747+0000","last_peered":"2023-10-01T11:07:34.847747+0000","last_clean":"2023-10-01T11:07:34.847747+0000","last_became_active":"2023-10-01T11:04:40.879768+0000","last_became_peered":"2023-10-01T11:04:40.879768+0000","last_unstale":"2023-10-01T11:07:34.847747+0000","last_undegraded":"2023-10-01T11:07:34.847747+0000","last_fullsized":"2023-10-01T11:07:34.847747+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:25:31.189338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"13.a","version":"344'8318","reported_seq":8388,"reported_epoch":344,"state":"active+clean","last_fresh":"2023-10-01T11:07:48.135389+0000","last_change":"2023-10-01T11:01:38.193535+0000","last_active":"2023-10-01T11:07:48.135389+0000","last_peered":"2023-10-01T11:07:48.135389+0000","last_clean":"2023-10-01T11:07:48.135389+0000","last_became_active":"2023-10-01T11:01:38.193069+0000","last_became_peered":"2023-10-01T11:01:38.193069+0000","last_unstale":"2023-10-01T11:07:48.135389+0000","last_undegraded":"2023-10-01T11:07:48.135389+0000","last_fullsized":"2023-10-01T11:07:48.135389+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_clean_scrub_stamp":"2023-10-01T11:01:36.981250+0000","objects_scrubbed":0,"log_size":8318,"log_dups_size":0,"ondisk_log_size":8318,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:27:23.966398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129761280,"num_objects":1980,"num_object_clones":0,"num_object_copies":5940,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1980,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8318,"num_write_kb":329536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":169,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.867879+0000","last_change":"2023-10-01T11:06:55.803529+0000","last_active":"2023-10-01T11:07:34.867879+0000","last_peered":"2023-10-01T11:07:34.867879+0000","last_clean":"2023-10-01T11:07:34.867879+0000","last_became_active":"2023-10-01T11:06:55.803266+0000","last_became_peered":"2023-10-01T11:06:55.803266+0000","last_unstale":"2023-10-01T11:07:34.867879+0000","last_undegraded":"2023-10-01T11:07:34.867879+0000","last_fullsized":"2023-10-01T11:07:34.867879+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:57:06.218433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":226,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.847873+0000","last_change":"2023-10-01T10:52:18.964479+0000","last_active":"2023-10-01T11:07:34.847873+0000","last_peered":"2023-10-01T11:07:34.847873+0000","last_clean":"2023-10-01T11:07:34.847873+0000","last_became_active":"2023-10-01T10:52:18.963881+0000","last_became_peered":"2023-10-01T10:52:18.963881+0000","last_unstale":"2023-10-01T11:07:34.847873+0000","last_undegraded":"2023-10-01T11:07:34.847873+0000","last_fullsized":"2023-10-01T11:07:34.847873+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:36:30.405755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":321,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.865909+0000","last_change":"2023-10-01T11:06:11.791253+0000","last_active":"2023-10-01T11:07:34.865909+0000","last_peered":"2023-10-01T11:07:34.865909+0000","last_clean":"2023-10-01T11:07:34.865909+0000","last_became_active":"2023-10-01T11:06:11.789517+0000","last_became_peered":"2023-10-01T11:06:11.789517+0000","last_unstale":"2023-10-01T11:07:34.865909+0000","last_undegraded":"2023-10-01T11:07:34.865909+0000","last_fullsized":"2023-10-01T11:07:34.865909+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"13.9","version":"344'7914","reported_seq":7981,"reported_epoch":344,"state":"active+clean","last_fresh":"2023-10-01T11:07:43.655570+0000","last_change":"2023-10-01T11:01:38.007037+0000","last_active":"2023-10-01T11:07:43.655570+0000","last_peered":"2023-10-01T11:07:43.655570+0000","last_clean":"2023-10-01T11:07:43.655570+0000","last_became_active":"2023-10-01T11:01:38.006590+0000","last_became_peered":"2023-10-01T11:01:38.006590+0000","last_unstale":"2023-10-01T11:07:43.655570+0000","last_undegraded":"2023-10-01T11:07:43.655570+0000","last_fullsized":"2023-10-01T11:07:43.655570+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_clean_scrub_stamp":"2023-10-01T11:01:36.981250+0000","objects_scrubbed":0,"log_size":7914,"log_dups_size":0,"ondisk_log_size":7914,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134873088,"num_objects":2058,"num_object_clones":0,"num_object_copies":6174,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2058,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7914,"num_write_kb":319104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":356,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.847876+0000","last_change":"2023-10-01T11:06:11.837719+0000","last_active":"2023-10-01T11:07:34.847876+0000","last_peered":"2023-10-01T11:07:34.847876+0000","last_clean":"2023-10-01T11:07:34.847876+0000","last_became_active":"2023-10-01T11:06:11.798523+0000","last_became_peered":"2023-10-01T11:06:11.798523+0000","last_unstale":"2023-10-01T11:07:34.847876+0000","last_undegraded":"2023-10-01T11:07:34.847876+0000","last_fullsized":"2023-10-01T11:07:34.847876+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:53:34.413543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":306,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.865257+0000","last_change":"2023-10-01T11:05:33.776132+0000","last_active":"2023-10-01T11:07:34.865257+0000","last_peered":"2023-10-01T11:07:34.865257+0000","last_clean":"2023-10-01T11:07:34.865257+0000","last_became_active":"2023-10-01T11:05:33.775706+0000","last_became_peered":"2023-10-01T11:05:33.775706+0000","last_unstale":"2023-10-01T11:07:34.865257+0000","last_undegraded":"2023-10-01T11:07:34.865257+0000","last_fullsized":"2023-10-01T11:07:34.865257+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:42:33.376356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":339,"reported_epoch":344,"state":"active+clean","last_fresh":"2023-10-01T11:07:35.844980+0000","last_change":"2023-10-01T11:06:11.784630+0000","last_active":"2023-10-01T11:07:35.844980+0000","last_peered":"2023-10-01T11:07:35.844980+0000","last_clean":"2023-10-01T11:07:35.844980+0000","last_became_active":"2023-10-01T11:06:11.784264+0000","last_became_peered":"2023-10-01T11:06:11.784264+0000","last_unstale":"2023-10-01T11:07:35.844980+0000","last_undegraded":"2023-10-01T11:07:35.844980+0000","last_fullsized":"2023-10-01T11:07:35.844980+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:03:03.888001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.8","version":"344'8113","reported_seq":17567,"reported_epoch":344,"state":"active+remapped+backfill_wait","last_fresh":"2023-10-01T11:07:48.051989+0000","last_change":"2023-10-01T11:05:34.897848+0000","last_active":"2023-10-01T11:07:48.051989+0000","last_peered":"2023-10-01T11:07:48.051989+0000","last_clean":"2023-10-01T11:05:32.094623+0000","last_became_active":"2023-10-01T11:05:34.883496+0000","last_became_peered":"2023-10-01T11:05:34.883496+0000","last_unstale":"2023-10-01T11:07:48.051989+0000","last_undegraded":"2023-10-01T11:07:48.051989+0000","last_fullsized":"2023-10-01T11:07:48.051989+0000","mapping_epoch":324,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:01:36.981250+0000","last_clean_scrub_stamp":"2023-10-01T11:01:36.981250+0000","objects_scrubbed":0,"log_size":8113,"log_dups_size":0,"ondisk_log_size":8113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:37:50.914861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":128516096,"num_objects":1961,"num_object_clones":0,"num_object_copies":5883,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1961,"num_objects_unfound":0,"num_objects_dirty":1961,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8113,"num_write_kb":322368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4676,"num_bytes_recovered":306446336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,1],"avail_no_missing":["2","1","5"],"object_location_counts":[{"shards":"1,2,5","objects":1961}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":39,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.861679+0000","last_change":"2023-10-01T11:06:11.818913+0000","last_active":"2023-10-01T11:07:34.861679+0000","last_peered":"2023-10-01T11:07:34.861679+0000","last_clean":"2023-10-01T11:07:34.861679+0000","last_became_active":"2023-10-01T11:06:11.818533+0000","last_became_peered":"2023-10-01T11:06:11.818533+0000","last_unstale":"2023-10-01T11:07:34.861679+0000","last_undegraded":"2023-10-01T11:07:34.861679+0000","last_fullsized":"2023-10-01T11:07:34.861679+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":50,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.865110+0000","last_change":"2023-10-01T11:04:40.878201+0000","last_active":"2023-10-01T11:07:34.865110+0000","last_peered":"2023-10-01T11:07:34.865110+0000","last_clean":"2023-10-01T11:07:34.865110+0000","last_became_active":"2023-10-01T11:04:40.877847+0000","last_became_peered":"2023-10-01T11:04:40.877847+0000","last_unstale":"2023-10-01T11:07:34.865110+0000","last_undegraded":"2023-10-01T11:07:34.865110+0000","last_fullsized":"2023-10-01T11:07:34.865110+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:14:30.388066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":324,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.861631+0000","last_change":"2023-10-01T11:05:33.784531+0000","last_active":"2023-10-01T11:07:34.861631+0000","last_peered":"2023-10-01T11:07:34.861631+0000","last_clean":"2023-10-01T11:07:34.861631+0000","last_became_active":"2023-10-01T11:05:33.784167+0000","last_became_peered":"2023-10-01T11:05:33.784167+0000","last_unstale":"2023-10-01T11:07:34.861631+0000","last_undegraded":"2023-10-01T11:07:34.861631+0000","last_fullsized":"2023-10-01T11:07:34.861631+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:34:16.434841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":57,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.862644+0000","last_change":"2023-10-01T11:06:11.791476+0000","last_active":"2023-10-01T11:07:34.862644+0000","last_peered":"2023-10-01T11:07:34.862644+0000","last_clean":"2023-10-01T11:07:34.862644+0000","last_became_active":"2023-10-01T11:06:11.791021+0000","last_became_peered":"2023-10-01T11:06:11.791021+0000","last_unstale":"2023-10-01T11:07:34.862644+0000","last_undegraded":"2023-10-01T11:07:34.862644+0000","last_fullsized":"2023-10-01T11:07:34.862644+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:14:03.385152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":335,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.847847+0000","last_change":"2023-10-01T10:37:42.762360+0000","last_active":"2023-10-01T11:07:34.847847+0000","last_peered":"2023-10-01T11:07:34.847847+0000","last_clean":"2023-10-01T11:07:34.847847+0000","last_became_active":"2023-10-01T10:37:42.761936+0000","last_became_peered":"2023-10-01T10:37:42.761936+0000","last_unstale":"2023-10-01T11:07:34.847847+0000","last_undegraded":"2023-10-01T11:07:34.847847+0000","last_fullsized":"2023-10-01T11:07:34.847847+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:50.637582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":328,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.870114+0000","last_change":"2023-10-01T10:38:16.037809+0000","last_active":"2023-10-01T11:07:34.870114+0000","last_peered":"2023-10-01T11:07:34.870114+0000","last_clean":"2023-10-01T11:07:34.870114+0000","last_became_active":"2023-10-01T10:38:16.037318+0000","last_became_peered":"2023-10-01T10:38:16.037318+0000","last_unstale":"2023-10-01T11:07:34.870114+0000","last_undegraded":"2023-10-01T11:07:34.870114+0000","last_fullsized":"2023-10-01T11:07:34.870114+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":327,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.862554+0000","last_change":"2023-10-01T11:04:02.726007+0000","last_active":"2023-10-01T11:07:34.862554+0000","last_peered":"2023-10-01T11:07:34.862554+0000","last_clean":"2023-10-01T11:07:34.862554+0000","last_became_active":"2023-10-01T11:04:02.725507+0000","last_became_peered":"2023-10-01T11:04:02.725507+0000","last_unstale":"2023-10-01T11:07:34.862554+0000","last_undegraded":"2023-10-01T11:07:34.862554+0000","last_fullsized":"2023-10-01T11:07:34.862554+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:32:04.611533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":391,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.865466+0000","last_change":"2023-10-01T11:05:33.830108+0000","last_active":"2023-10-01T11:07:34.865466+0000","last_peered":"2023-10-01T11:07:34.865466+0000","last_clean":"2023-10-01T11:07:34.865466+0000","last_became_active":"2023-10-01T11:05:33.791882+0000","last_became_peered":"2023-10-01T11:05:33.791882+0000","last_unstale":"2023-10-01T11:07:34.865466+0000","last_undegraded":"2023-10-01T11:07:34.865466+0000","last_fullsized":"2023-10-01T11:07:34.865466+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":327,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.862466+0000","last_change":"2023-10-01T11:04:02.733653+0000","last_active":"2023-10-01T11:07:34.862466+0000","last_peered":"2023-10-01T11:07:34.862466+0000","last_clean":"2023-10-01T11:07:34.862466+0000","last_became_active":"2023-10-01T11:04:02.733178+0000","last_became_peered":"2023-10-01T11:04:02.733178+0000","last_unstale":"2023-10-01T11:07:34.862466+0000","last_undegraded":"2023-10-01T11:07:34.862466+0000","last_fullsized":"2023-10-01T11:07:34.862466+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:18:25.249624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":388,"reported_epoch":344,"state":"active+clean","last_fresh":"2023-10-01T11:07:35.846394+0000","last_change":"2023-10-01T11:05:33.777019+0000","last_active":"2023-10-01T11:07:35.846394+0000","last_peered":"2023-10-01T11:07:35.846394+0000","last_clean":"2023-10-01T11:07:35.846394+0000","last_became_active":"2023-10-01T11:05:33.776565+0000","last_became_peered":"2023-10-01T11:05:33.776565+0000","last_unstale":"2023-10-01T11:07:35.846394+0000","last_undegraded":"2023-10-01T11:07:35.846394+0000","last_fullsized":"2023-10-01T11:07:35.846394+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T22:00:18.890326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":344,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.846818+0000","last_change":"2023-10-01T10:37:42.765142+0000","last_active":"2023-10-01T11:07:34.846818+0000","last_peered":"2023-10-01T11:07:34.846818+0000","last_clean":"2023-10-01T11:07:34.846818+0000","last_became_active":"2023-10-01T10:37:42.764770+0000","last_became_peered":"2023-10-01T10:37:42.764770+0000","last_unstale":"2023-10-01T11:07:34.846818+0000","last_undegraded":"2023-10-01T11:07:34.846818+0000","last_fullsized":"2023-10-01T11:07:34.846818+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:23:33.527876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":58,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.859770+0000","last_change":"2023-10-01T11:04:40.889388+0000","last_active":"2023-10-01T11:07:34.859770+0000","last_peered":"2023-10-01T11:07:34.859770+0000","last_clean":"2023-10-01T11:07:34.859770+0000","last_became_active":"2023-10-01T11:04:40.888960+0000","last_became_peered":"2023-10-01T11:04:40.888960+0000","last_unstale":"2023-10-01T11:07:34.859770+0000","last_undegraded":"2023-10-01T11:07:34.859770+0000","last_fullsized":"2023-10-01T11:07:34.859770+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":363,"reported_epoch":343,"state":"active+clean","last_fresh":"2023-10-01T11:07:34.861287+0000","last_change":"2023-10-01T11:05:33.785774+0000","last_active":"2023-10-01T11:07:34.861287+0000","last_peered":"2023-10-01T11:07:34.861287+0000","last_clean":"2023-10-01T11:07:34.861287+0000","last_became_active":"2023-10-01T11:05:33.785426+0000","last_became_peered":"2023-10-01T11:05:33.785426+0000","last_unstale":"2023-10-01T11:07:34.861287+0000","last_undegraded":"2023-10-01T11:07:34.861287+0000","last_fullsized":"2023-10-01T11:07:34.861287+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:41:32.634339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":16,"stat_sum":{"num_bytes":2126381080,"num_objects":32447,"num_object_clones":0,"num_object_copies":97341,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8714,"num_objects_misplaced":4113,"num_objects_unfound":0,"num_objects_dirty":32447,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":126665,"num_write_kb":5091521,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22598,"num_bytes_recovered":1377763352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7769112576,"data_stored":7769096288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":123265,"ondisk_log_size":123265,"up":48,"acting":44,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":82,"num_bytes_recovered":34354,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":110592,"data_stored":21246,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30319,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":4722944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":2,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430864,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1698284,"kb_used_data":519652,"kb_used_omap":0,"kb_used_meta":1178624,"kb_avail":92046868,"statfs":{"total":95995035648,"available":94255992832,"internally_reserved":0,"allocated":532123648,"data_stored":531548198,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1206910976},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 11:07:33 2023","interfaces":[{"interface":"back","average":{"1min":1.2270000000000001,"5min":0.81499999999999995,"15min":1.2589999999999999},"min":{"1min":0.27900000000000003,"5min":0.27400000000000002,"15min":0.252},"max":{"1min":7.2199999999999998,"5min":7.2199999999999998,"15min":14.315},"last":10.013},{"interface":"front","average":{"1min":1.8640000000000001,"5min":1.0940000000000001,"15min":1.349},"min":{"1min":0.35499999999999998,"5min":0.307,"15min":0.307},"max":{"1min":7.3440000000000003,"5min":8.548,"15min":14.336},"last":8.8949999999999996}]},{"osd":1,"last update":"Sun Oct 1 11:07:33 2023","interfaces":[{"interface":"back","average":{"1min":1.635,"5min":1.0369999999999999,"15min":1.419},"min":{"1min":0.32900000000000001,"5min":0.30199999999999999,"15min":0.27400000000000002},"max":{"1min":7.4199999999999999,"5min":8.5830000000000002,"15min":15.662000000000001},"last":8.9309999999999992},{"interface":"front","average":{"1min":1.2290000000000001,"5min":0.91500000000000004,"15min":1.3540000000000001},"min":{"1min":0.34599999999999997,"5min":0.33000000000000002,"15min":0.27100000000000002},"max":{"1min":4.0750000000000002,"5min":5.6900000000000004,"15min":18.768999999999998},"last":0.68999999999999995}]},{"osd":2,"last update":"Sun Oct 1 11:06:50 2023","interfaces":[{"interface":"back","average":{"1min":0.91400000000000003,"5min":0.94899999999999995,"15min":2.8500000000000001},"min":{"1min":0.44800000000000001,"5min":0.41899999999999998,"15min":0.315},"max":{"1min":2.9420000000000002,"5min":5.6539999999999999,"15min":215.89699999999999},"last":0.66000000000000003},{"interface":"front","average":{"1min":0.82599999999999996,"5min":0.89000000000000001,"15min":2.5990000000000002},"min":{"1min":0.374,"5min":0.35899999999999999,"15min":0.307},"max":{"1min":4.4139999999999997,"5min":5.4329999999999998,"15min":215.77600000000001},"last":0.41799999999999998}]},{"osd":3,"last update":"Sun Oct 1 11:07:33 2023","interfaces":[{"interface":"back","average":{"1min":1.1519999999999999,"5min":0.96299999999999997,"15min":1.4370000000000001},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.32300000000000001},"max":{"1min":4.1360000000000001,"5min":5.5750000000000002,"15min":9.157},"last":0.621},{"interface":"front","average":{"1min":1.427,"5min":0.94899999999999995,"15min":1.387},"min":{"1min":0.376,"5min":0.376,"15min":0.28299999999999997},"max":{"1min":7.2549999999999999,"5min":7.2549999999999999,"15min":11.244999999999999},"last":0.46200000000000002}]},{"osd":4,"last update":"Sun Oct 1 11:07:09 2023","interfaces":[{"interface":"back","average":{"1min":1.764,"5min":1.3280000000000001,"15min":1.2729999999999999},"min":{"1min":0.36199999999999999,"5min":0.34999999999999998,"15min":0.33100000000000002},"max":{"1min":14.51,"5min":17.178999999999998,"15min":17.178999999999998},"last":8.8200000000000003},{"interface":"front","average":{"1min":1.1930000000000001,"5min":1.276,"15min":1.3169999999999999},"min":{"1min":0.441,"5min":0.375,"15min":0.35699999999999998},"max":{"1min":4.2009999999999996,"5min":17.25,"15min":17.25},"last":0.57999999999999996}]},{"osd":5,"last update":"Sun Oct 1 11:07:33 2023","interfaces":[{"interface":"back","average":{"1min":3.2200000000000002,"5min":1.5980000000000001,"15min":1.5649999999999999},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":16.491,"5min":16.491,"15min":16.491},"last":0.755},{"interface":"front","average":{"1min":2.206,"5min":1.2589999999999999,"15min":1.2609999999999999},"min":{"1min":0.38300000000000001,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":7.6050000000000004,"5min":7.6050000000000004,"15min":9.1709999999999994},"last":10.1}]},{"osd":6,"last update":"Sun Oct 1 11:07:33 2023","interfaces":[{"interface":"back","average":{"1min":1.4139999999999999,"5min":1.1639999999999999,"15min":1.3540000000000001},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.35299999999999998},"max":{"1min":4.2619999999999996,"5min":15.433,"15min":15.433},"last":0.72199999999999998},{"interface":"front","average":{"1min":2.4820000000000002,"5min":1.3660000000000001,"15min":1.474},"min":{"1min":0.436,"5min":0.376,"15min":0.34899999999999998},"max":{"1min":11.536,"5min":11.536,"15min":11.536},"last":11.318}]}]},{"osd":6,"up_from":42,"seq":180388627104,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1804124,"kb_used_data":915220,"kb_used_omap":0,"kb_used_meta":888896,"kb_avail":91941028,"statfs":{"total":95995035648,"available":94147612672,"internally_reserved":0,"allocated":937185280,"data_stored":936626273,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":910229504},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 11:07:26 2023","interfaces":[{"interface":"back","average":{"1min":1.26,"5min":1.712,"15min":1.514},"min":{"1min":0.311,"5min":0.29899999999999999,"15min":0.26800000000000002},"max":{"1min":6.2039999999999997,"5min":102.23,"15min":102.23},"last":0.432},{"interface":"front","average":{"1min":1.0629999999999999,"5min":1.698,"15min":1.5669999999999999},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29399999999999998},"max":{"1min":5.5129999999999999,"5min":101.217,"15min":101.217},"last":0.58599999999999997}]},{"osd":1,"last update":"Sun Oct 1 11:07:45 2023","interfaces":[{"interface":"back","average":{"1min":1.458,"5min":0.82399999999999995,"15min":1.2090000000000001},"min":{"1min":0.31,"5min":0.29799999999999999,"15min":0.27100000000000002},"max":{"1min":6.3220000000000001,"5min":6.3220000000000001,"15min":10.679},"last":0.52100000000000002},{"interface":"front","average":{"1min":1.351,"5min":0.93700000000000006,"15min":1.278},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.26400000000000001},"max":{"1min":5.4749999999999996,"5min":5.7539999999999996,"15min":9.4809999999999999},"last":0.622}]},{"osd":2,"last update":"Sun Oct 1 11:06:54 2023","interfaces":[{"interface":"back","average":{"1min":0.71099999999999997,"5min":2.0190000000000001,"15min":3.2269999999999999},"min":{"1min":0.35399999999999998,"5min":0.32500000000000001,"15min":0.29699999999999999},"max":{"1min":1.9830000000000001,"5min":104.871,"15min":122.768},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.73499999999999999,"5min":2.161,"15min":3.5169999999999999},"min":{"1min":0.41799999999999998,"5min":0.40300000000000002,"15min":0.36199999999999999},"max":{"1min":2.0089999999999999,"5min":101.059,"15min":123.02200000000001},"last":0.70499999999999996}]},{"osd":3,"last update":"Sun Oct 1 11:07:45 2023","interfaces":[{"interface":"back","average":{"1min":1.5569999999999999,"5min":1.9119999999999999,"15min":1.867},"min":{"1min":0.377,"5min":0.31900000000000001,"15min":0.31},"max":{"1min":6.2949999999999999,"5min":102.273,"15min":102.273},"last":0.54400000000000004},{"interface":"front","average":{"1min":1.647,"5min":1.9139999999999999,"15min":2.0329999999999999},"min":{"1min":0.51000000000000001,"5min":0.41099999999999998,"15min":0.36499999999999999},"max":{"1min":4.4349999999999996,"5min":101.491,"15min":101.491},"last":0.64900000000000002}]},{"osd":4,"last update":"Sun Oct 1 11:07:18 2023","interfaces":[{"interface":"back","average":{"1min":1.304,"5min":1.0009999999999999,"15min":1.1399999999999999},"min":{"1min":0.34399999999999997,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":8.6289999999999996,"5min":8.6289999999999996,"15min":8.6289999999999996},"last":0.75800000000000001},{"interface":"front","average":{"1min":1.855,"5min":1.337,"15min":1.504},"min":{"1min":0.44400000000000001,"5min":0.36799999999999999,"15min":0.35999999999999999},"max":{"1min":9.2460000000000004,"5min":13.800000000000001,"15min":13.800000000000001},"last":0.88}]},{"osd":5,"last update":"Sun Oct 1 11:07:45 2023","interfaces":[{"interface":"back","average":{"1min":2.5430000000000001,"5min":1.145,"15min":1.2330000000000001},"min":{"1min":0.45400000000000001,"5min":0.36399999999999999,"15min":0.33000000000000002},"max":{"1min":6.2649999999999997,"5min":6.2649999999999997,"15min":15.238},"last":0.48199999999999998},{"interface":"front","average":{"1min":2.0720000000000001,"5min":1.327,"15min":1.486},"min":{"1min":0.46899999999999997,"5min":0.432,"15min":0.315},"max":{"1min":6.8739999999999997,"5min":8.3450000000000006,"15min":13.295},"last":2.9260000000000002}]},{"osd":7,"last update":"Sun Oct 1 11:07:26 2023","interfaces":[{"interface":"back","average":{"1min":1.2929999999999999,"5min":1.1359999999999999,"15min":1.2949999999999999},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.33900000000000002},"max":{"1min":6.1799999999999997,"5min":9.5419999999999998,"15min":9.5419999999999998},"last":0.55000000000000004},{"interface":"front","average":{"1min":2.3969999999999998,"5min":1.236,"15min":1.331},"min":{"1min":0.51900000000000002,"5min":0.45000000000000001,"15min":0.35799999999999998},"max":{"1min":8.6959999999999997,"5min":8.6959999999999997,"15min":8.6959999999999997},"last":0.56999999999999995}]}]},{"osd":1,"up_from":101,"seq":433791697321,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1611476,"kb_used_data":803852,"kb_used_omap":4,"kb_used_meta":807611,"kb_avail":92133676,"statfs":{"total":95995035648,"available":94344884224,"internally_reserved":0,"allocated":823144448,"data_stored":821786125,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":826994574},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 11:06:48 2023","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":0.58699999999999997,"15min":1.222},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.19900000000000001},"max":{"1min":2.9390000000000001,"5min":3.1880000000000002,"15min":12.888999999999999},"last":0.248},{"interface":"front","average":{"1min":0.629,"5min":0.60999999999999999,"15min":1.218},"min":{"1min":0.34799999999999998,"5min":0.23400000000000001,"15min":0.192},"max":{"1min":0.94099999999999995,"5min":2.9980000000000002,"15min":15.416},"last":0.27600000000000002}]},{"osd":2,"last update":"Sun Oct 1 11:07:15 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":1.0740000000000001,"15min":2.4100000000000001},"min":{"1min":0.317,"5min":0.23200000000000001,"15min":0.222},"max":{"1min":0.85699999999999998,"5min":41.026000000000003,"15min":161.905},"last":3.214},{"interface":"front","average":{"1min":0.69999999999999996,"5min":1.165,"15min":2.569},"min":{"1min":0.38700000000000001,"5min":0.32400000000000001,"15min":0.247},"max":{"1min":1.1499999999999999,"5min":41.148000000000003,"15min":162.059},"last":0.501}]},{"osd":3,"last update":"Sun Oct 1 11:06:48 2023","interfaces":[{"interface":"back","average":{"1min":0.67100000000000004,"5min":0.64100000000000001,"15min":1.853},"min":{"1min":0.308,"5min":0.28699999999999998,"15min":0.248},"max":{"1min":1.119,"5min":3.2309999999999999,"15min":143.73400000000001},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.64600000000000002,"5min":0.624,"15min":1.8839999999999999},"min":{"1min":0.36299999999999999,"5min":0.28699999999999998,"15min":0.223},"max":{"1min":0.96799999999999997,"5min":1.8100000000000001,"15min":143.00399999999999},"last":3.2629999999999999}]},{"osd":4,"last update":"Sun Oct 1 11:07:15 2023","interfaces":[{"interface":"back","average":{"1min":12.374000000000001,"5min":3.141,"15min":1.9350000000000001},"min":{"1min":0.40799999999999997,"5min":0.35899999999999999,"15min":0.307},"max":{"1min":203.89500000000001,"5min":203.89500000000001,"15min":203.89500000000001},"last":0.66900000000000004},{"interface":"front","average":{"1min":12.233000000000001,"5min":3.2719999999999998,"15min":1.954},"min":{"1min":0.51800000000000002,"5min":0.378,"15min":0.32700000000000001},"max":{"1min":203.852,"5min":203.852,"15min":203.852},"last":0.65700000000000003}]},{"osd":5,"last update":"Sun Oct 1 11:06:48 2023","interfaces":[{"interface":"back","average":{"1min":0.76900000000000002,"5min":0.89100000000000001,"15min":1.2230000000000001},"min":{"1min":0.49099999999999999,"5min":0.39000000000000001,"15min":0.35299999999999998},"max":{"1min":1.204,"5min":7.3949999999999996,"15min":9.1809999999999992},"last":8.2249999999999996},{"interface":"front","average":{"1min":0.92500000000000004,"5min":0.97899999999999998,"15min":1.2509999999999999},"min":{"1min":0.46899999999999997,"5min":0.40600000000000003,"15min":0.34200000000000003},"max":{"1min":2.0030000000000001,"5min":9.8100000000000005,"15min":9.8100000000000005},"last":8.1110000000000007}]},{"osd":6,"last update":"Sun Oct 1 11:06:48 2023","interfaces":[{"interface":"back","average":{"1min":1.032,"5min":0.96499999999999997,"15min":1.476},"min":{"1min":0.495,"5min":0.38800000000000001,"15min":0.33800000000000002},"max":{"1min":3.6099999999999999,"5min":3.6099999999999999,"15min":8.8309999999999995},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.92400000000000004,"5min":1.0289999999999999,"15min":1.3129999999999999},"min":{"1min":0.48199999999999998,"5min":0.42999999999999999,"15min":0.33200000000000002},"max":{"1min":1.5069999999999999,"5min":5.9329999999999998,"15min":8.8970000000000002},"last":0.68999999999999995}]},{"osd":7,"last update":"Sun Oct 1 11:06:48 2023","interfaces":[{"interface":"back","average":{"1min":0.97099999999999997,"5min":0.995,"15min":1.2969999999999999},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.36399999999999999},"max":{"1min":1.4319999999999999,"5min":2.9910000000000001,"15min":9.7140000000000004},"last":0.59299999999999997},{"interface":"front","average":{"1min":1.109,"5min":0.95199999999999996,"15min":1.4430000000000001},"min":{"1min":0.49099999999999999,"5min":0.48399999999999999,"15min":0.42899999999999999},"max":{"1min":4.1639999999999997,"5min":4.1639999999999997,"15min":11.689},"last":3.3660000000000001}]}]},{"osd":0,"up_from":86,"seq":369367187931,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1559192,"kb_used_data":675472,"kb_used_omap":5,"kb_used_meta":883706,"kb_avail":92185960,"statfs":{"total":95995035648,"available":94398423040,"internally_reserved":0,"allocated":691683328,"data_stored":690377776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":904915578},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 11:06:51 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":1.6619999999999999,"15min":1.544},"min":{"1min":0.27500000000000002,"5min":0.222,"15min":0.19700000000000001},"max":{"1min":1.0920000000000001,"5min":119.911,"15min":119.911},"last":0.28999999999999998},{"interface":"front","average":{"1min":0.61199999999999999,"5min":1.6910000000000001,"15min":1.738},"min":{"1min":0.308,"5min":0.23799999999999999,"15min":0.19900000000000001},"max":{"1min":1.1319999999999999,"5min":119.76300000000001,"15min":119.76300000000001},"last":0.33500000000000002}]},{"osd":2,"last update":"Sun Oct 1 11:06:58 2023","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.88100000000000001,"15min":2.1579999999999999},"min":{"1min":0.29999999999999999,"5min":0.248,"15min":0.20699999999999999},"max":{"1min":1.2190000000000001,"5min":29.199999999999999,"15min":72.064999999999998},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.72399999999999998,"5min":1.054,"15min":2.3260000000000001},"min":{"1min":0.39700000000000002,"5min":0.26100000000000001,"15min":0.252},"max":{"1min":1.883,"5min":31.081,"15min":76.328999999999994},"last":0.35599999999999998}]},{"osd":3,"last update":"Sun Oct 1 11:06:51 2023","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":0.63400000000000001,"15min":1.919},"min":{"1min":0.36499999999999999,"5min":0.32100000000000001,"15min":0.214},"max":{"1min":1.095,"5min":1.542,"15min":81.674999999999997},"last":0.38700000000000001},{"interface":"front","average":{"1min":0.77500000000000002,"5min":0.67500000000000004,"15min":1.7450000000000001},"min":{"1min":0.34300000000000003,"5min":0.308,"15min":0.246},"max":{"1min":1.1830000000000001,"5min":2.8620000000000001,"15min":70.950999999999993},"last":0.32300000000000001}]},{"osd":4,"last update":"Sun Oct 1 11:07:24 2023","interfaces":[{"interface":"back","average":{"1min":1.359,"5min":1.1359999999999999,"15min":1.2749999999999999},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.34300000000000003},"max":{"1min":8.0180000000000007,"5min":8.0180000000000007,"15min":9.3320000000000007},"last":2.8660000000000001},{"interface":"front","average":{"1min":1.6439999999999999,"5min":1.1899999999999999,"15min":1.4039999999999999},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.311},"max":{"1min":8.0370000000000008,"5min":8.0370000000000008,"15min":8.0530000000000008},"last":2.9390000000000001}]},{"osd":5,"last update":"Sun Oct 1 11:06:51 2023","interfaces":[{"interface":"back","average":{"1min":0.76600000000000001,"5min":1.226,"15min":1.496},"min":{"1min":0.48399999999999999,"5min":0.45900000000000002,"15min":0.309},"max":{"1min":1.052,"5min":39.860999999999997,"15min":39.860999999999997},"last":1.111},{"interface":"front","average":{"1min":0.82599999999999996,"5min":1.3200000000000001,"15min":1.363},"min":{"1min":0.46999999999999997,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":1.2350000000000001,"5min":39.646000000000001,"15min":39.646000000000001},"last":1.133}]},{"osd":6,"last update":"Sun Oct 1 11:06:51 2023","interfaces":[{"interface":"back","average":{"1min":0.88700000000000001,"5min":0.94299999999999995,"15min":1.3280000000000001},"min":{"1min":0.438,"5min":0.40899999999999997,"15min":0.373},"max":{"1min":1.325,"5min":5.319,"15min":9.4570000000000007},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.88300000000000001,"5min":0.90800000000000003,"15min":1.4159999999999999},"min":{"1min":0.52000000000000002,"5min":0.47599999999999998,"15min":0.37},"max":{"1min":1.413,"5min":4.3789999999999996,"15min":11.032999999999999},"last":0.88500000000000001}]},{"osd":7,"last update":"Sun Oct 1 11:06:51 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":0.99099999999999999,"15min":1.4339999999999999},"min":{"1min":0.68000000000000005,"5min":0.39600000000000002,"15min":0.35499999999999998},"max":{"1min":1.3640000000000001,"5min":9.1989999999999998,"15min":13.699},"last":1.0960000000000001},{"interface":"front","average":{"1min":1.006,"5min":0.997,"15min":1.629},"min":{"1min":0.63,"5min":0.53000000000000003,"15min":0.32200000000000001},"max":{"1min":2.2370000000000001,"5min":9.4659999999999993,"15min":9.4659999999999993},"last":0.47799999999999998}]}]},{"osd":2,"up_from":116,"seq":498216206706,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1854288,"kb_used_data":1136648,"kb_used_omap":7,"kb_used_meta":717624,"kb_avail":91890864,"statfs":{"total":95995035648,"available":94096244736,"internally_reserved":0,"allocated":1163927552,"data_stored":1162630711,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":734847057},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 11:07:01 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.73299999999999998,"15min":1.6160000000000001},"min":{"1min":0.314,"5min":0.21199999999999999,"15min":0.19800000000000001},"max":{"1min":1.0009999999999999,"5min":7.0060000000000002,"15min":104.42},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.70399999999999996,"5min":0.88700000000000001,"15min":1.8380000000000001},"min":{"1min":0.248,"5min":0.23999999999999999,"15min":0.187},"max":{"1min":1.052,"5min":9.6379999999999999,"15min":105.10899999999999},"last":0.439}]},{"osd":1,"last update":"Sun Oct 1 11:07:01 2023","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.90400000000000003,"15min":1.363},"min":{"1min":0.35699999999999998,"5min":0.29099999999999998,"15min":0.20699999999999999},"max":{"1min":1.048,"5min":5.9950000000000001,"15min":8.6440000000000001},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.79200000000000004,"15min":1.732},"min":{"1min":0.33800000000000002,"5min":0.26900000000000002,"15min":0.23999999999999999},"max":{"1min":1.091,"5min":7.1340000000000003,"15min":11.554},"last":0.46200000000000002}]},{"osd":3,"last update":"Sun Oct 1 11:07:36 2023","interfaces":[{"interface":"back","average":{"1min":0.70299999999999996,"5min":0.80700000000000005,"15min":1.544},"min":{"1min":0.313,"5min":0.27300000000000002,"15min":0.23200000000000001},"max":{"1min":3.4060000000000001,"5min":7.0410000000000004,"15min":23.879000000000001},"last":0.71699999999999997},{"interface":"front","average":{"1min":0.75800000000000001,"5min":0.95099999999999996,"15min":1.5669999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.221},"max":{"1min":2.7719999999999998,"5min":6.0289999999999999,"15min":23.704000000000001},"last":0.56599999999999995}]},{"osd":4,"last update":"Sun Oct 1 11:07:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0089999999999999,"5min":1.292,"15min":1.4870000000000001},"min":{"1min":0.38200000000000001,"5min":0.34399999999999997,"15min":0.33000000000000002},"max":{"1min":4.6820000000000004,"5min":9.5109999999999992,"15min":9.5109999999999992},"last":5.9500000000000002},{"interface":"front","average":{"1min":1.123,"5min":1.224,"15min":1.806},"min":{"1min":0.42599999999999999,"5min":0.33200000000000002,"15min":0.317},"max":{"1min":4.5060000000000002,"5min":8.9860000000000007,"15min":11.574},"last":2.7599999999999998}]},{"osd":5,"last update":"Sun Oct 1 11:07:01 2023","interfaces":[{"interface":"back","average":{"1min":0.93300000000000005,"5min":1.3859999999999999,"15min":1.5580000000000001},"min":{"1min":0.441,"5min":0.40100000000000002,"15min":0.36599999999999999},"max":{"1min":1.4359999999999999,"5min":10.180999999999999,"15min":10.180999999999999},"last":2.504},{"interface":"front","average":{"1min":1.149,"5min":1.147,"15min":1.899},"min":{"1min":0.58299999999999996,"5min":0.40600000000000003,"15min":0.36199999999999999},"max":{"1min":4.2640000000000002,"5min":8.0749999999999993,"15min":12.835000000000001},"last":1.837}]},{"osd":6,"last update":"Sun Oct 1 11:07:01 2023","interfaces":[{"interface":"back","average":{"1min":1.268,"5min":1.2969999999999999,"15min":2.0710000000000002},"min":{"1min":0.53500000000000003,"5min":0.48399999999999999,"15min":0.38200000000000001},"max":{"1min":4.4210000000000003,"5min":8.0999999999999996,"15min":19.908000000000001},"last":0.74199999999999999},{"interface":"front","average":{"1min":0.89400000000000002,"5min":1.0900000000000001,"15min":1.464},"min":{"1min":0.61599999999999999,"5min":0.42599999999999999,"15min":0.33700000000000002},"max":{"1min":1.3819999999999999,"5min":7.1719999999999997,"15min":9.6709999999999994},"last":0.77700000000000002}]},{"osd":7,"last update":"Sun Oct 1 11:07:01 2023","interfaces":[{"interface":"back","average":{"1min":0.92200000000000004,"5min":1.159,"15min":1.427},"min":{"1min":0.435,"5min":0.435,"15min":0.35099999999999998},"max":{"1min":1.341,"5min":8.0359999999999996,"15min":11.209},"last":1.887},{"interface":"front","average":{"1min":0.95899999999999996,"5min":1.2569999999999999,"15min":1.587},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.36399999999999999},"max":{"1min":1.49,"5min":9.5489999999999995,"15min":9.5489999999999995},"last":0.69299999999999995}]}]},{"osd":3,"up_from":121,"seq":519691043179,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1478572,"kb_used_data":784228,"kb_used_omap":2,"kb_used_meta":694333,"kb_avail":92266580,"statfs":{"total":95995035648,"available":94480977920,"internally_reserved":0,"allocated":803049472,"data_stored":801780899,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":710997045},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 11:07:43 2023","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.55000000000000004,"15min":1.577},"min":{"1min":0.26900000000000002,"5min":0.23899999999999999,"15min":0.17199999999999999},"max":{"1min":0.754,"5min":2.4049999999999998,"15min":100.191},"last":1.2989999999999999},{"interface":"front","average":{"1min":0.70599999999999996,"5min":0.61299999999999999,"15min":1.825},"min":{"1min":0.221,"5min":0.22,"15min":0.189},"max":{"1min":4.6280000000000001,"5min":4.6280000000000001,"15min":99.852999999999994},"last":0.68100000000000005}]},{"osd":1,"last update":"Sun Oct 1 11:07:43 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.54900000000000004,"15min":1.2969999999999999},"min":{"1min":0.32800000000000001,"5min":0.25800000000000001,"15min":0.22500000000000001},"max":{"1min":1.706,"5min":1.706,"15min":10.574999999999999},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.63,"15min":1.4059999999999999},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.23400000000000001},"max":{"1min":0.94299999999999995,"5min":2.2759999999999998,"15min":13.529999999999999},"last":0.313}]},{"osd":2,"last update":"Sun Oct 1 11:07:43 2023","interfaces":[{"interface":"back","average":{"1min":1.0229999999999999,"5min":0.753,"15min":2.129},"min":{"1min":0.33200000000000002,"5min":0.252,"15min":0.217},"max":{"1min":4.367,"5min":4.367,"15min":81.212999999999994},"last":0.93100000000000005},{"interface":"front","average":{"1min":1.056,"5min":0.83099999999999996,"15min":2.1720000000000002},"min":{"1min":0.41099999999999998,"5min":0.33600000000000002,"15min":0.26900000000000002},"max":{"1min":3.754,"5min":3.754,"15min":80.819000000000003},"last":0.91000000000000003}]},{"osd":4,"last update":"Sun Oct 1 11:07:20 2023","interfaces":[{"interface":"back","average":{"1min":1.3100000000000001,"5min":0.998,"15min":1.456},"min":{"1min":0.52000000000000002,"5min":0.437,"15min":0.36799999999999999},"max":{"1min":5.1479999999999997,"5min":6.0410000000000004,"15min":10.808999999999999},"last":2.3580000000000001},{"interface":"front","average":{"1min":1.343,"5min":1.0680000000000001,"15min":1.5149999999999999},"min":{"1min":0.495,"5min":0.39200000000000002,"15min":0.308},"max":{"1min":5.016,"5min":6.218,"15min":13.391999999999999},"last":2.4169999999999998}]},{"osd":5,"last update":"Sun Oct 1 11:07:43 2023","interfaces":[{"interface":"back","average":{"1min":1.5920000000000001,"5min":1.0109999999999999,"15min":1.5149999999999999},"min":{"1min":0.44600000000000001,"5min":0.41399999999999998,"15min":0.34399999999999997},"max":{"1min":5.1589999999999998,"5min":5.1589999999999998,"15min":13.333},"last":0.57799999999999996},{"interface":"front","average":{"1min":1.6719999999999999,"5min":1.1279999999999999,"15min":1.536},"min":{"1min":0.505,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":5.5279999999999996,"5min":6.2110000000000003,"15min":12.335000000000001},"last":0.52500000000000002}]},{"osd":6,"last update":"Sun Oct 1 11:07:43 2023","interfaces":[{"interface":"back","average":{"1min":1.7430000000000001,"5min":1.099,"15min":1.704},"min":{"1min":0.46600000000000003,"5min":0.38,"15min":0.38},"max":{"1min":8.9009999999999998,"5min":8.9009999999999998,"15min":13.616},"last":0.46800000000000003},{"interface":"front","average":{"1min":1.6830000000000001,"5min":1.0229999999999999,"15min":1.895},"min":{"1min":0.50800000000000001,"5min":0.46500000000000002,"15min":0.36799999999999999},"max":{"1min":8.8849999999999998,"5min":8.8849999999999998,"15min":73.542000000000002},"last":0.48599999999999999}]},{"osd":7,"last update":"Sun Oct 1 11:07:43 2023","interfaces":[{"interface":"back","average":{"1min":0.995,"5min":0.89200000000000002,"15min":1.5449999999999999},"min":{"1min":0.44500000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":3.5630000000000002,"5min":3.5630000000000002,"15min":12.416},"last":0.73499999999999999},{"interface":"front","average":{"1min":0.97399999999999998,"5min":0.96599999999999997,"15min":1.52},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.35899999999999999},"max":{"1min":5.6269999999999998,"5min":5.6269999999999998,"15min":10.755000000000001},"last":0.59899999999999998}]}]},{"osd":4,"up_from":126,"seq":541165879652,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1819092,"kb_used_data":1209868,"kb_used_omap":10,"kb_used_meta":609205,"kb_avail":91926060,"statfs":{"total":95995035648,"available":94132285440,"internally_reserved":0,"allocated":1238904832,"data_stored":1237599365,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":623826532},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 11:07:06 2023","interfaces":[{"interface":"back","average":{"1min":1.2569999999999999,"5min":0.98999999999999999,"15min":1.639},"min":{"1min":0.374,"5min":0.27700000000000002,"15min":0.26400000000000001},"max":{"1min":4.0970000000000004,"5min":10.321999999999999,"15min":68.611000000000004},"last":0.51600000000000001},{"interface":"front","average":{"1min":2.016,"5min":1.038,"15min":1.639},"min":{"1min":0.372,"5min":0.30499999999999999,"15min":0.26200000000000001},"max":{"1min":12.369,"5min":12.369,"15min":68.480999999999995},"last":0.46899999999999997}]},{"osd":1,"last update":"Sun Oct 1 11:07:06 2023","interfaces":[{"interface":"back","average":{"1min":1.492,"5min":0.92600000000000005,"15min":1.1399999999999999},"min":{"1min":0.433,"5min":0.34599999999999997,"15min":0.28699999999999998},"max":{"1min":9.1920000000000002,"5min":9.1920000000000002,"15min":11.664},"last":2.5070000000000001},{"interface":"front","average":{"1min":1.2649999999999999,"5min":0.999,"15min":1.272},"min":{"1min":0.442,"5min":0.34499999999999997,"15min":0.317},"max":{"1min":4.1390000000000002,"5min":10.263,"15min":19.719999999999999},"last":2.2559999999999998}]},{"osd":2,"last update":"Sun Oct 1 11:07:06 2023","interfaces":[{"interface":"back","average":{"1min":1.4179999999999999,"5min":2.3479999999999999,"15min":2.7999999999999998},"min":{"1min":0.46500000000000002,"5min":0.34100000000000003,"15min":0.32700000000000001},"max":{"1min":4.0439999999999996,"5min":133.321,"15min":133.321},"last":0.40600000000000003},{"interface":"front","average":{"1min":2.0590000000000002,"5min":2.3260000000000001,"15min":2.621},"min":{"1min":0.45200000000000001,"5min":0.33700000000000002,"15min":0.33300000000000002},"max":{"1min":12.448,"5min":132.875,"15min":132.875},"last":1.0009999999999999}]},{"osd":3,"last update":"Sun Oct 1 11:07:06 2023","interfaces":[{"interface":"back","average":{"1min":2.0110000000000001,"5min":1.119,"15min":1.5629999999999999},"min":{"1min":0.44800000000000001,"5min":0.42299999999999999,"15min":0.307},"max":{"1min":12.473000000000001,"5min":12.473000000000001,"15min":61.664999999999999},"last":0.435},{"interface":"front","average":{"1min":1.5209999999999999,"5min":0.98199999999999998,"15min":1.9610000000000001},"min":{"1min":0.378,"5min":0.378,"15min":0.308},"max":{"1min":9.1419999999999995,"5min":9.1419999999999995,"15min":130.88300000000001},"last":2.5449999999999999}]},{"osd":5,"last update":"Sun Oct 1 11:07:06 2023","interfaces":[{"interface":"back","average":{"1min":1.946,"5min":1.2729999999999999,"15min":1.429},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.30199999999999999},"max":{"1min":12.416,"5min":12.416,"15min":84.307000000000002},"last":1.7849999999999999},{"interface":"front","average":{"1min":1.7809999999999999,"5min":1.4890000000000001,"15min":1.659},"min":{"1min":0.41499999999999998,"5min":0.35799999999999998,"15min":0.309},"max":{"1min":7.1109999999999998,"5min":13.629,"15min":84.234999999999999},"last":2.2890000000000001}]},{"osd":6,"last update":"Sun Oct 1 11:07:06 2023","interfaces":[{"interface":"back","average":{"1min":2.02,"5min":1.212,"15min":1.181},"min":{"1min":0.39000000000000001,"5min":0.372,"15min":0.34999999999999998},"max":{"1min":12.635999999999999,"5min":12.635999999999999,"15min":12.635999999999999},"last":5.5739999999999998},{"interface":"front","average":{"1min":1.5409999999999999,"5min":1.1910000000000001,"15min":1.1819999999999999},"min":{"1min":0.43099999999999999,"5min":0.39300000000000002,"15min":0.33400000000000002},"max":{"1min":9.3059999999999992,"5min":9.3059999999999992,"15min":9.3059999999999992},"last":5.4710000000000001}]},{"osd":7,"last update":"Sun Oct 1 11:07:06 2023","interfaces":[{"interface":"back","average":{"1min":1.9610000000000001,"5min":1.284,"15min":1.718},"min":{"1min":0.53600000000000003,"5min":0.495,"15min":0.36699999999999999},"max":{"1min":10.851000000000001,"5min":10.851000000000001,"15min":119.56100000000001},"last":5.5119999999999996},{"interface":"front","average":{"1min":1.6439999999999999,"5min":1.242,"15min":1.6950000000000001},"min":{"1min":0.441,"5min":0.40999999999999998,"15min":0.40200000000000002},"max":{"1min":7.3879999999999999,"5min":10.471,"15min":120.44499999999999},"last":0.48199999999999998}]}]},{"osd":5,"up_from":36,"seq":154618823333,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2825936,"kb_used_data":1633288,"kb_used_omap":0,"kb_used_meta":1192640,"kb_avail":90919216,"statfs":{"total":95995035648,"available":93101277184,"internally_reserved":0,"allocated":1672486912,"data_stored":1671935469,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1221263360},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 11:07:33 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":0.86199999999999999,"15min":1.923},"min":{"1min":0.29799999999999999,"5min":0.25800000000000001,"15min":0.24299999999999999},"max":{"1min":5.8079999999999998,"5min":10.839,"15min":123.39700000000001},"last":0.59299999999999997},{"interface":"front","average":{"1min":1.9890000000000001,"5min":1.181,"15min":2.0699999999999998},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.253},"max":{"1min":7.984,"5min":7.984,"15min":124.051},"last":0.879}]},{"osd":1,"last update":"Sun Oct 1 11:07:33 2023","interfaces":[{"interface":"back","average":{"1min":0.97199999999999998,"5min":0.79400000000000004,"15min":1.0529999999999999},"min":{"1min":0.38,"5min":0.29899999999999999,"15min":0.28799999999999998},"max":{"1min":5.8559999999999999,"5min":5.8559999999999999,"15min":9.766},"last":0.67500000000000004},{"interface":"front","average":{"1min":2.1720000000000002,"5min":1.2509999999999999,"15min":1.274},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":8.2260000000000009,"5min":8.2260000000000009,"15min":8.2260000000000009},"last":0.96199999999999997}]},{"osd":2,"last update":"Sun Oct 1 11:07:06 2023","interfaces":[{"interface":"back","average":{"1min":0.74199999999999999,"5min":2.746,"15min":2.359},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.307},"max":{"1min":2.7360000000000002,"5min":207.059,"15min":207.059},"last":0.64500000000000002},{"interface":"front","average":{"1min":1.1799999999999999,"5min":3.1259999999999999,"15min":2.5169999999999999},"min":{"1min":0.502,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":4.2990000000000004,"5min":206.673,"15min":206.673},"last":0.98999999999999999}]},{"osd":3,"last update":"Sun Oct 1 11:07:33 2023","interfaces":[{"interface":"back","average":{"1min":2.0510000000000002,"5min":1.288,"15min":2.0609999999999999},"min":{"1min":0.437,"5min":0.437,"15min":0.32400000000000001},"max":{"1min":8.0999999999999996,"5min":8.0999999999999996,"15min":166.29599999999999},"last":1.012},{"interface":"front","average":{"1min":1.863,"5min":1.1040000000000001,"15min":2.0550000000000002},"min":{"1min":0.47999999999999998,"5min":0.36899999999999999,"15min":0.32000000000000001},"max":{"1min":9.2439999999999998,"5min":9.2439999999999998,"15min":166.565},"last":0.52400000000000002}]},{"osd":4,"last update":"Sun Oct 1 11:07:06 2023","interfaces":[{"interface":"back","average":{"1min":0.82799999999999996,"5min":0.84699999999999998,"15min":1.004},"min":{"1min":0.39000000000000001,"5min":0.311,"15min":0.311},"max":{"1min":3.1179999999999999,"5min":6.5410000000000004,"15min":6.6349999999999998},"last":0.48599999999999999},{"interface":"front","average":{"1min":1.077,"5min":1.0029999999999999,"15min":1.038},"min":{"1min":0.34100000000000003,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":4.2709999999999999,"5min":4.2709999999999999,"15min":9.1699999999999999},"last":0.70199999999999996}]},{"osd":6,"last update":"Sun Oct 1 11:07:33 2023","interfaces":[{"interface":"back","average":{"1min":1.218,"5min":1.083,"15min":1.2010000000000001},"min":{"1min":0.36599999999999999,"5min":0.35399999999999998,"15min":0.27800000000000002},"max":{"1min":7.7679999999999998,"5min":7.7679999999999998,"15min":9.0790000000000006},"last":0.61899999999999999},{"interface":"front","average":{"1min":2.008,"5min":1.234,"15min":1.204},"min":{"1min":0.45500000000000002,"5min":0.34499999999999997,"15min":0.33000000000000002},"max":{"1min":12.189,"5min":12.189,"15min":12.189},"last":3.5409999999999999}]},{"osd":7,"last update":"Sun Oct 1 11:07:33 2023","interfaces":[{"interface":"back","average":{"1min":2.2349999999999999,"5min":1.4179999999999999,"15min":1.2809999999999999},"min":{"1min":0.46600000000000003,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":9.1959999999999997,"5min":9.1959999999999997,"15min":9.1959999999999997},"last":0.55300000000000005},{"interface":"front","average":{"1min":1.4830000000000001,"5min":1.147,"15min":1.22},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.34699999999999998},"max":{"1min":8.8710000000000004,"5min":8.8710000000000004,"15min":9.1340000000000003},"last":0.73399999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":685441024,"data_stored":685441024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":815271936,"data_stored":815267864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1139474432,"data_stored":1139474432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":792662016,"data_stored":792657944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1225068544,"data_stored":1225064472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1657995264,"data_stored":1657995264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":924778496,"data_stored":924778496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":528420864,"data_stored":528416792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T11:07:52.918 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-01T11:07:52.918 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd pool set unique_pool_8 pg_num 26 2023-10-01T11:07:52.921 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:52.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:52.923 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:07:53.024 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:53.054 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:53.058 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:52 smithi173 bash[58816]: cluster 2023-10-01T11:07:52.775578+0000 mon.a (mon.0) 1965 : cluster 3 Health check update: Degraded data redundancy: 8602/91776 objects degraded (9.373%), 5 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T11:07:53.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:53.059 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:07:53.160 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:53.192 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:53.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:53.194 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:07:53.295 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:53.333 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:53.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:53.339 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:07:53.439 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:53.468 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:53.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:53.474 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:07:53.575 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:53.605 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:53.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:53.611 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:07:53.712 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:53.738 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:53.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:53.741 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:07:53.841 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:53.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:53 smithi173 bash[58816]: cluster 2023-10-01T11:07:52.340219+0000 mgr.y (mgr.25072) 1517 : cluster 0 pgmap v1623: 89 pgs: 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 486 op/s; 8602/91776 objects degraded (9.373%); 3920/91776 objects misplaced (4.271%); 3.1 MiB/s, 49 objects/s recovering 2023-10-01T11:07:53.872 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:53.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:53.874 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:07:53.975 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:53.985 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:07:53.999 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:54.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:54.003 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:07:54.012 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:53 smithi078 bash[75299]: cluster 2023-10-01T11:07:52.340219+0000 mgr.y (mgr.25072) 1517 : cluster 0 pgmap v1623: 89 pgs: 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 486 op/s; 8602/91776 objects degraded (9.373%); 3920/91776 objects misplaced (4.271%); 3.1 MiB/s, 49 objects/s recovering 2023-10-01T11:07:54.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:53 smithi078 bash[78274]: cluster 2023-10-01T11:07:52.340219+0000 mgr.y (mgr.25072) 1517 : cluster 0 pgmap v1623: 89 pgs: 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 486 op/s; 8602/91776 objects degraded (9.373%); 3920/91776 objects misplaced (4.271%); 3.1 MiB/s, 49 objects/s recovering 2023-10-01T11:07:54.103 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:54.140 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:54.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:54.147 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:07:54.248 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:54.272 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:54.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:54.275 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:07:54.376 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:54.408 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:54.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:54.411 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:07:54.512 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:54.534 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:54.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:54.536 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:07:54.636 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:54.661 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:54.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:54.665 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:07:54.765 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:54.801 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:54.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:54.806 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:07:54.820 INFO:teuthology.orchestra.run.smithi078.stderr:noscrub is unset 2023-10-01T11:07:54.907 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:54.943 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:54.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:54.947 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:07:55.048 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:55.058 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:54 smithi173 bash[58816]: audit 2023-10-01T11:07:54.402564+0000 mon.c (mon.2) 228 : audit 1 from='client.? 172.21.15.78:0/2806026599' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T11:07:55.058 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:54 smithi173 bash[58816]: audit 2023-10-01T11:07:54.405963+0000 mon.a (mon.0) 1966 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T11:07:55.081 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:55.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:54 smithi078 bash[78274]: audit 2023-10-01T11:07:54.402564+0000 mon.c (mon.2) 228 : audit 1 from='client.? 172.21.15.78:0/2806026599' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T11:07:55.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:54 smithi078 bash[78274]: audit 2023-10-01T11:07:54.405963+0000 mon.a (mon.0) 1966 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T11:07:55.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:54 smithi078 bash[75299]: audit 2023-10-01T11:07:54.402564+0000 mon.c (mon.2) 228 : audit 1 from='client.? 172.21.15.78:0/2806026599' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T11:07:55.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:54 smithi078 bash[75299]: audit 2023-10-01T11:07:54.405963+0000 mon.a (mon.0) 1966 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T11:07:55.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:55.087 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:07:55.188 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:55.238 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:55.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:55.243 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:07:55.344 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:55.392 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:55.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:55.394 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:07:55.494 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:55.503 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T11:07:55.531 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:55.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:55.542 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:07:55.643 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:55.676 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:55.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:55.679 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:07:55.780 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:55.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:55.821 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:55.821 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:07:55.922 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:55.923 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:55 smithi078 bash[75299]: cluster 2023-10-01T11:07:54.340917+0000 mgr.y (mgr.25072) 1518 : cluster 0 pgmap v1624: 89 pgs: 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 656 op/s; 8675/85662 objects degraded (10.127%); 3572/85662 objects misplaced (4.170%); 3.6 MiB/s, 57 objects/s recovering 2023-10-01T11:07:55.923 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:55 smithi078 bash[75299]: cluster 2023-10-01T11:07:54.803873+0000 mon.a (mon.0) 1967 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:07:55.923 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:55 smithi078 bash[75299]: audit 2023-10-01T11:07:54.815216+0000 mon.a (mon.0) 1968 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T11:07:55.924 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:55 smithi078 bash[75299]: cluster 2023-10-01T11:07:54.815293+0000 mon.a (mon.0) 1969 : cluster 0 osdmap e345: 8 total, 8 up, 8 in 2023-10-01T11:07:55.924 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:55 smithi078 bash[75299]: cluster 2023-10-01T11:07:54.815679+0000 osd.5 (osd.5) 364 : cluster 3 failed to encode map e345 with expected crc 2023-10-01T11:07:55.924 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:55 smithi078 bash[75299]: cluster 2023-10-01T11:07:54.817352+0000 osd.7 (osd.7) 381 : cluster 3 failed to encode map e345 with expected crc 2023-10-01T11:07:55.949 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:55.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:55.952 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:07:56.053 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:56.062 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:55 smithi173 bash[58816]: cluster 2023-10-01T11:07:54.340917+0000 mgr.y (mgr.25072) 1518 : cluster 0 pgmap v1624: 89 pgs: 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 656 op/s; 8675/85662 objects degraded (10.127%); 3572/85662 objects misplaced (4.170%); 3.6 MiB/s, 57 objects/s recovering 2023-10-01T11:07:56.062 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:55 smithi173 bash[58816]: cluster 2023-10-01T11:07:54.803873+0000 mon.a (mon.0) 1967 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:07:56.062 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:55 smithi173 bash[58816]: audit 2023-10-01T11:07:54.815216+0000 mon.a (mon.0) 1968 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T11:07:56.062 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:55 smithi173 bash[58816]: cluster 2023-10-01T11:07:54.815293+0000 mon.a (mon.0) 1969 : cluster 0 osdmap e345: 8 total, 8 up, 8 in 2023-10-01T11:07:56.062 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:55 smithi173 bash[58816]: cluster 2023-10-01T11:07:54.815679+0000 osd.5 (osd.5) 364 : cluster 3 failed to encode map e345 with expected crc 2023-10-01T11:07:56.062 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:55 smithi173 bash[58816]: cluster 2023-10-01T11:07:54.817352+0000 osd.7 (osd.7) 381 : cluster 3 failed to encode map e345 with expected crc 2023-10-01T11:07:56.086 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:56.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:56.088 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:07:56.189 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:56.216 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:56.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:56.220 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:07:56.289 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:55 smithi078 bash[78274]: cluster 2023-10-01T11:07:54.340917+0000 mgr.y (mgr.25072) 1518 : cluster 0 pgmap v1624: 89 pgs: 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 656 op/s; 8675/85662 objects degraded (10.127%); 3572/85662 objects misplaced (4.170%); 3.6 MiB/s, 57 objects/s recovering 2023-10-01T11:07:56.289 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:55 smithi078 bash[78274]: cluster 2023-10-01T11:07:54.803873+0000 mon.a (mon.0) 1967 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:07:56.290 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:55 smithi078 bash[78274]: audit 2023-10-01T11:07:54.815216+0000 mon.a (mon.0) 1968 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T11:07:56.290 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:55 smithi078 bash[78274]: cluster 2023-10-01T11:07:54.815293+0000 mon.a (mon.0) 1969 : cluster 0 osdmap e345: 8 total, 8 up, 8 in 2023-10-01T11:07:56.290 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:55 smithi078 bash[78274]: cluster 2023-10-01T11:07:54.815679+0000 osd.5 (osd.5) 364 : cluster 3 failed to encode map e345 with expected crc 2023-10-01T11:07:56.290 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:55 smithi078 bash[78274]: cluster 2023-10-01T11:07:54.817352+0000 osd.7 (osd.7) 381 : cluster 3 failed to encode map e345 with expected crc 2023-10-01T11:07:56.320 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:56.353 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:56.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:56.355 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:07:56.456 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:56.485 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:56.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:56.493 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:07:56.594 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:56.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:56.630 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:56.630 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:07:56.731 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:56.772 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:56.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:56.775 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:07:56.876 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:56.906 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:56.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:56.910 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:07:57.011 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:57.042 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:57.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:57.046 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:07:57.147 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:57.170 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:57.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:57.172 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:07:57.273 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:57.388 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:57.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:57.390 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:07:57.491 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:57.570 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:57.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:57.572 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:07:57.672 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:57.701 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:57.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:57.703 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:07:57.804 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:58.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:57 smithi078 bash[75299]: cluster 2023-10-01T11:07:56.341949+0000 mgr.y (mgr.25072) 1519 : cluster 0 pgmap v1626: 89 pgs: 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 8619/82824 objects degraded (10.406%); 3572/82824 objects misplaced (4.313%); 2.9 MiB/s, 46 objects/s recovering 2023-10-01T11:07:58.015 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:57 smithi078 bash[75299]: cluster 2023-10-01T11:07:57.776615+0000 mon.a (mon.0) 1970 : cluster 3 Health check update: Degraded data redundancy: 8619/82824 objects degraded (10.406%), 5 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T11:07:58.039 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:07:58.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:58.087 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:07:58.087 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:57 smithi078 bash[78274]: cluster 2023-10-01T11:07:56.341949+0000 mgr.y (mgr.25072) 1519 : cluster 0 pgmap v1626: 89 pgs: 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 8619/82824 objects degraded (10.406%); 3572/82824 objects misplaced (4.313%); 2.9 MiB/s, 46 objects/s recovering 2023-10-01T11:07:58.088 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:57 smithi078 bash[78274]: cluster 2023-10-01T11:07:57.776615+0000 mon.a (mon.0) 1970 : cluster 3 Health check update: Degraded data redundancy: 8619/82824 objects degraded (10.406%), 5 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T11:07:58.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:57 smithi173 bash[58816]: cluster 2023-10-01T11:07:56.341949+0000 mgr.y (mgr.25072) 1519 : cluster 0 pgmap v1626: 89 pgs: 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 8619/82824 objects degraded (10.406%); 3572/82824 objects misplaced (4.313%); 2.9 MiB/s, 46 objects/s recovering 2023-10-01T11:07:58.130 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:57 smithi173 bash[58816]: cluster 2023-10-01T11:07:57.776615+0000 mon.a (mon.0) 1970 : cluster 3 Health check update: Degraded data redundancy: 8619/82824 objects degraded (10.406%), 5 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-10-01T11:07:58.188 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:58.230 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:58.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:58.286 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:07:58.386 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:58.440 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:58.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:58.441 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:07:58.542 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:58.576 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:07:58.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:58.583 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:07:58.684 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:58.694 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T11:07:58.742 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:58.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:58.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:07:58.843 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:58.873 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:58.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:58.877 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:07:58.977 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:59.010 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:59.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:59.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:07:59.114 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:59.145 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:07:59.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:59.147 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:07:59.193 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:07:59.248 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:59.304 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:07:59.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:59.307 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:07:59.408 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:59.431 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:59.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:59.435 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:07:59.536 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:59.577 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:07:59.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:59.581 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:07:59.682 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:59.713 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:07:59.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:59.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:07:59.816 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:59.829 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2023-10-01T11:07:59.829 INFO:teuthology.orchestra.run.smithi078.stdout: "ops": [ 2023-10-01T11:07:59.829 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:07:59.829 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.46454.0:135187 13.4 e345/326 13:27f43e5f:::benchmark_data_smithi078_202524_object55629:head v 345'8342, mlcod=345'8340)", 2023-10-01T11:07:59.829 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:07:59.820635+0000", 2023-10-01T11:07:59.830 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 0.0072841520000000003, 2023-10-01T11:07:59.830 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0073188150000000002, 2023-10-01T11:07:59.830 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:07:59.830 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "started", 2023-10-01T11:07:59.830 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:07:59.830 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:07:59.830 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:07:59.830 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:07:59.820635+0000", 2023-10-01T11:07:59.831 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:07:59.831 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:07:59.831 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:07:59.831 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:07:59.831 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:07:59.820635+0000", 2023-10-01T11:07:59.831 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:07:59.831 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:07:59.831 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:07:59.831 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:07:59.831 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:07:59.820634+0000", 2023-10-01T11:07:59.831 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T11:07:59.832 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:07:59.832 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:07:59.832 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:07:59.832 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:07:59.820647+0000", 2023-10-01T11:07:59.832 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.3203e-05 2023-10-01T11:07:59.832 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:07:59.832 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:07:59.832 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:07:59.832 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:07:59.820667+0000", 2023-10-01T11:07:59.833 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.0007999999999999e-05 2023-10-01T11:07:59.833 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:07:59.833 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:07:59.833 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:07:59.833 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:07:59.820675+0000", 2023-10-01T11:07:59.833 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.4770000000000003e-06 2023-10-01T11:07:59.833 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:07:59.833 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:07:59.834 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:07:59.834 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:07:59.820909+0000", 2023-10-01T11:07:59.834 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00023408700000000001 2023-10-01T11:07:59.834 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:07:59.834 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:07:59.834 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:07:59.834 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:07:59.820980+0000", 2023-10-01T11:07:59.835 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.0129000000000006e-05 2023-10-01T11:07:59.835 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:07:59.835 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:07:59.835 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:07:59.835 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:07:59.835 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2023-10-01T11:07:59.835 INFO:teuthology.orchestra.run.smithi078.stdout: "num_ops": 1 2023-10-01T11:07:59.835 INFO:teuthology.orchestra.run.smithi078.stdout:} 2023-10-01T11:07:59.844 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:07:59.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:59.848 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:07:59.949 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:59.956 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:07:59 smithi078 bash[78274]: cluster 2023-10-01T11:07:58.342621+0000 mgr.y (mgr.25072) 1520 : cluster 0 pgmap v1627: 89 pgs: 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 8619/79317 objects degraded (10.867%); 3363/79317 objects misplaced (4.240%); 2.9 MiB/s, 46 objects/s recovering 2023-10-01T11:07:59.957 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:07:59 smithi078 bash[75299]: cluster 2023-10-01T11:07:58.342621+0000 mgr.y (mgr.25072) 1520 : cluster 0 pgmap v1627: 89 pgs: 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 8619/79317 objects degraded (10.867%); 3363/79317 objects misplaced (4.240%); 2.9 MiB/s, 46 objects/s recovering 2023-10-01T11:07:59.973 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:07:59.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:07:59.978 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:08:00.079 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:00.101 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:00.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:00.104 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:08:00.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:07:59 smithi173 bash[58816]: cluster 2023-10-01T11:07:58.342621+0000 mgr.y (mgr.25072) 1520 : cluster 0 pgmap v1627: 89 pgs: 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 8619/79317 objects degraded (10.867%); 3363/79317 objects misplaced (4.240%); 2.9 MiB/s, 46 objects/s recovering 2023-10-01T11:08:00.205 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:00.239 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:00.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:00.241 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:08:00.342 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:00.378 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:00.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:00.383 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:00.484 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:00.544 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:00.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:00.546 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:00.580 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:08:00 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:08:00] "GET /metrics HTTP/1.1" 200 36405 "" "Prometheus/2.33.4" 2023-10-01T11:08:00.647 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:00.687 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:00.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:00.691 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:08:00.792 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:00.824 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:00.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:00.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:00.930 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:00.958 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:00.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:00.963 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:08:01.064 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:01.091 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:01.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:01.093 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:08:01.194 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:01.224 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:01.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:01.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:08:01.328 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:01.380 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:01.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:01.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:08:01.487 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:01.511 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:01.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:01.513 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:01.614 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:01.641 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:01.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:01.643 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:08:01.744 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:01.774 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:01.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:01.781 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:08:01.882 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:01.915 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:01.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:01.918 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:02.019 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:02.027 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:01 smithi173 bash[58816]: cluster 2023-10-01T11:08:00.343302+0000 mgr.y (mgr.25072) 1521 : cluster 0 pgmap v1628: 89 pgs: 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 8745/73683 objects degraded (11.868%); 3155/73683 objects misplaced (4.282%); 1.5 MiB/s, 24 objects/s recovering 2023-10-01T11:08:02.057 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:02.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:02.063 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:08:02.164 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:02.236 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:02.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:02.239 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:02.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:01 smithi078 bash[78274]: cluster 2023-10-01T11:08:00.343302+0000 mgr.y (mgr.25072) 1521 : cluster 0 pgmap v1628: 89 pgs: 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 8745/73683 objects degraded (11.868%); 3155/73683 objects misplaced (4.282%); 1.5 MiB/s, 24 objects/s recovering 2023-10-01T11:08:02.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:01 smithi078 bash[75299]: cluster 2023-10-01T11:08:00.343302+0000 mgr.y (mgr.25072) 1521 : cluster 0 pgmap v1628: 89 pgs: 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 8745/73683 objects degraded (11.868%); 3155/73683 objects misplaced (4.282%); 1.5 MiB/s, 24 objects/s recovering 2023-10-01T11:08:02.340 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:02.363 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:02.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:02.368 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:08:02.441 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-01T11:08:02.469 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:02.499 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:02.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:02.506 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:08:02.606 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:02.655 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:02.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:02.659 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:02.760 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:02.784 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:02.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:02.786 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:08:02.849 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:02 smithi173 bash[58816]: audit 2023-10-01T11:08:02.462918+0000 mon.a (mon.0) 1971 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:08:02.849 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:02 smithi173 bash[58816]: cluster 2023-10-01T11:08:02.778068+0000 mon.a (mon.0) 1972 : cluster 3 Health check update: Degraded data redundancy: 8745/73683 objects degraded (11.868%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T11:08:02.886 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:02.911 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:02.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:02.913 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:08:02.952 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:02 smithi078 bash[75299]: audit 2023-10-01T11:08:02.462918+0000 mon.a (mon.0) 1971 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:08:02.952 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:02 smithi078 bash[75299]: cluster 2023-10-01T11:08:02.778068+0000 mon.a (mon.0) 1972 : cluster 3 Health check update: Degraded data redundancy: 8745/73683 objects degraded (11.868%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T11:08:02.952 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:02 smithi078 bash[78274]: audit 2023-10-01T11:08:02.462918+0000 mon.a (mon.0) 1971 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:08:02.952 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:02 smithi078 bash[78274]: cluster 2023-10-01T11:08:02.778068+0000 mon.a (mon.0) 1972 : cluster 3 Health check update: Degraded data redundancy: 8745/73683 objects degraded (11.868%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T11:08:02.957 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:08:03.015 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:03.046 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:03.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:03.048 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:08:03.149 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:03.195 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:03.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:03.200 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:03.301 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:03.343 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:03.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:03.346 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:08:03.447 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:03.470 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:03.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:03.472 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:03.573 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:03.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:03.605 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:03.605 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:08:03.706 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:03.730 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:03.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:03.736 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:08:03.836 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:03.844 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:03 smithi173 bash[58816]: cluster 2023-10-01T11:08:02.344017+0000 mgr.y (mgr.25072) 1522 : cluster 0 pgmap v1629: 89 pgs: 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 8745/73683 objects degraded (11.868%); 3155/73683 objects misplaced (4.282%); 1.5 MiB/s, 24 objects/s recovering 2023-10-01T11:08:03.865 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:03.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:03.867 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:08:03.968 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:04.001 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:04.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:04.009 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:04.110 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:04.148 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:04.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:04.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:08:04.255 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:04.265 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:03 smithi078 bash[75299]: cluster 2023-10-01T11:08:02.344017+0000 mgr.y (mgr.25072) 1522 : cluster 0 pgmap v1629: 89 pgs: 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 8745/73683 objects degraded (11.868%); 3155/73683 objects misplaced (4.282%); 1.5 MiB/s, 24 objects/s recovering 2023-10-01T11:08:04.265 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:03 smithi078 bash[78274]: cluster 2023-10-01T11:08:02.344017+0000 mgr.y (mgr.25072) 1522 : cluster 0 pgmap v1629: 89 pgs: 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 8745/73683 objects degraded (11.868%); 3155/73683 objects misplaced (4.282%); 1.5 MiB/s, 24 objects/s recovering 2023-10-01T11:08:04.278 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:04.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:04.283 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:08:04.384 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:04.417 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:04.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:04.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:08:04.524 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:04.549 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:04.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:04.554 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:08:04.654 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:04.677 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:04.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:04.679 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:08:04.780 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:04.807 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:04.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:04.815 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:08:04.916 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:04.944 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:04.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:04.948 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:08:05.049 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:05.088 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:05.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:05.091 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:08:05.191 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:05.221 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:05.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:05.224 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:05.255 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:08:05.325 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:05.363 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:05.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:05.369 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:05.470 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:05.500 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:05.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:05.507 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:08:05.607 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:05.635 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:05.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:05.637 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:05.738 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:05.762 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:05.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:05.764 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:08:05.865 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:05.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:05 smithi173 bash[58816]: cluster 2023-10-01T11:08:04.344876+0000 mgr.y (mgr.25072) 1523 : cluster 0 pgmap v1630: 89 pgs: 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 8826/67194 objects degraded (13.135%); 2782/67194 objects misplaced (4.140%); 1.5 MiB/s, 24 objects/s recovering 2023-10-01T11:08:05.890 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:05.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:05.896 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:05.997 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:06.049 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:06.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:06.052 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:06.153 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:06.171 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:05 smithi078 bash[75299]: cluster 2023-10-01T11:08:04.344876+0000 mgr.y (mgr.25072) 1523 : cluster 0 pgmap v1630: 89 pgs: 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 8826/67194 objects degraded (13.135%); 2782/67194 objects misplaced (4.140%); 1.5 MiB/s, 24 objects/s recovering 2023-10-01T11:08:06.171 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:05 smithi078 bash[78274]: cluster 2023-10-01T11:08:04.344876+0000 mgr.y (mgr.25072) 1523 : cluster 0 pgmap v1630: 89 pgs: 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 8826/67194 objects degraded (13.135%); 2782/67194 objects misplaced (4.140%); 1.5 MiB/s, 24 objects/s recovering 2023-10-01T11:08:06.175 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:06.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:06.177 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:06.278 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:06.318 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:06.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:06.323 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:08:06.423 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:06.451 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:06.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:06.457 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:08:06.557 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:06.576 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:06.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:06.578 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:06.679 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:06.717 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:06.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:06.723 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:06.824 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:06.859 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:06.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:06.863 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:06.963 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:06.986 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:06.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:06.994 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:08:07.095 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:07.119 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:07.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:07.124 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:08:07.224 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:07.250 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:08:07.259 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:07.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:07.260 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:07.361 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:07.391 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:07.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:07.397 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:07.498 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:07.527 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:07.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:07.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:08:07.632 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:07.676 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:07.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:07.678 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:07.779 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:07.803 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:07.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:07.807 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:08:07.908 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:07.917 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:07 smithi173 bash[58816]: cluster 2023-10-01T11:08:06.345536+0000 mgr.y (mgr.25072) 1524 : cluster 0 pgmap v1631: 89 pgs: 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 534 op/s; 8826/64329 objects degraded (13.720%); 2782/64329 objects misplaced (4.325%); 1.0 MiB/s, 16 objects/s recovering 2023-10-01T11:08:07.917 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:07 smithi173 bash[58816]: cluster 2023-10-01T11:08:07.779039+0000 mon.a (mon.0) 1973 : cluster 3 Health check update: Degraded data redundancy: 8826/64329 objects degraded (13.720%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T11:08:07.949 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:07.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:07.952 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:08:08.052 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:08.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:07 smithi078 bash[75299]: cluster 2023-10-01T11:08:06.345536+0000 mgr.y (mgr.25072) 1524 : cluster 0 pgmap v1631: 89 pgs: 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 534 op/s; 8826/64329 objects degraded (13.720%); 2782/64329 objects misplaced (4.325%); 1.0 MiB/s, 16 objects/s recovering 2023-10-01T11:08:08.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:07 smithi078 bash[75299]: cluster 2023-10-01T11:08:07.779039+0000 mon.a (mon.0) 1973 : cluster 3 Health check update: Degraded data redundancy: 8826/64329 objects degraded (13.720%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T11:08:08.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:07 smithi078 bash[78274]: cluster 2023-10-01T11:08:06.345536+0000 mgr.y (mgr.25072) 1524 : cluster 0 pgmap v1631: 89 pgs: 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 534 op/s; 8826/64329 objects degraded (13.720%); 2782/64329 objects misplaced (4.325%); 1.0 MiB/s, 16 objects/s recovering 2023-10-01T11:08:08.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:07 smithi078 bash[78274]: cluster 2023-10-01T11:08:07.779039+0000 mon.a (mon.0) 1973 : cluster 3 Health check update: Degraded data redundancy: 8826/64329 objects degraded (13.720%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T11:08:08.091 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:08.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:08.096 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:08:08.197 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:08.227 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:08.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:08.229 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:08:08.330 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:08.361 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:08.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:08.363 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:08:08.464 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:08.518 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:08.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:08.523 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:08:08.624 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:08.672 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:08.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:08.675 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:08:08.776 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:08.800 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:08.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:08.811 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:08:08.911 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:08.937 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:08.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:08.940 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:08:09.040 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:09.065 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:09.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:09.068 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:09.168 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:09.189 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:09.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:09.196 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:08:09.297 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:09.322 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:09.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:09.328 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:09.429 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:09.469 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:09.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:09.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:08:09.575 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:09.602 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:09.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:09.606 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:08:09.707 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:09.735 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:09.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:09.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:08:09.844 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:09.866 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:09.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:09.874 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:08:09.974 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:09.983 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:09 smithi173 bash[58816]: cluster 2023-10-01T11:08:08.346186+0000 mgr.y (mgr.25072) 1525 : cluster 0 pgmap v1632: 89 pgs: 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 609 op/s; 8826/60888 objects degraded (14.495%); 2575/60888 objects misplaced (4.229%); 986 KiB/s, 15 objects/s recovering 2023-10-01T11:08:10.025 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:10.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:10.030 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:08:10.131 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:10.138 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:09 smithi078 bash[78274]: cluster 2023-10-01T11:08:08.346186+0000 mgr.y (mgr.25072) 1525 : cluster 0 pgmap v1632: 89 pgs: 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 609 op/s; 8826/60888 objects degraded (14.495%); 2575/60888 objects misplaced (4.229%); 986 KiB/s, 15 objects/s recovering 2023-10-01T11:08:10.139 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:09 smithi078 bash[75299]: cluster 2023-10-01T11:08:08.346186+0000 mgr.y (mgr.25072) 1525 : cluster 0 pgmap v1632: 89 pgs: 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 609 op/s; 8826/60888 objects degraded (14.495%); 2575/60888 objects misplaced (4.229%); 986 KiB/s, 15 objects/s recovering 2023-10-01T11:08:10.152 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:10.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:10.155 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:08:10.256 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:10.286 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:10.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:10.289 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:08:10.390 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:10.399 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:08:10 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:08:10] "GET /metrics HTTP/1.1" 200 36405 "" "Prometheus/2.33.4" 2023-10-01T11:08:10.430 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:10.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:10.435 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:08:10.536 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:10.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:10.582 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:10.583 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:08:10.684 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:10.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:10.714 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:10.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:08:10.815 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:10.849 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:10.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:10.858 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:08:10.959 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:10.984 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:10.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:10.988 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:08:11.089 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:11.117 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:11.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:11.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:08:11.220 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:11.245 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:11.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:11.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:08:11.348 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:11.387 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:11.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:11.392 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:08:11.493 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:11.521 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:11.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:11.524 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:08:11.625 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:11.657 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:11.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:11.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:08:11.763 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:11.791 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:11.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:11.797 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:11.894 INFO:teuthology.orchestra.run.smithi078.stderr:set pool 13 pg_num to 26 2023-10-01T11:08:11.898 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:11.922 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:11.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:11.927 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:08:12.028 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:12.050 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:12.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:12.055 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:08:12.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:11 smithi173 bash[58816]: cluster 2023-10-01T11:08:10.346824+0000 mgr.y (mgr.25072) 1526 : cluster 0 pgmap v1633: 89 pgs: 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 8905/55260 objects degraded (16.115%); 2387/55260 objects misplaced (4.320%); 1.5 MiB/s, 24 objects/s recovering 2023-10-01T11:08:12.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:11 smithi173 bash[58816]: audit 2023-10-01T11:08:11.426730+0000 mon.c (mon.2) 229 : audit 1 from='client.? 172.21.15.78:0/1270169454' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num", "val": "26"}]: dispatch 2023-10-01T11:08:12.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:11 smithi173 bash[58816]: audit 2023-10-01T11:08:11.427221+0000 mon.a (mon.0) 1974 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num", "val": "26"}]: dispatch 2023-10-01T11:08:12.155 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:12.196 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:12.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:12.203 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:08:12.303 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:12.309 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:11 smithi078 bash[75299]: cluster 2023-10-01T11:08:10.346824+0000 mgr.y (mgr.25072) 1526 : cluster 0 pgmap v1633: 89 pgs: 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 8905/55260 objects degraded (16.115%); 2387/55260 objects misplaced (4.320%); 1.5 MiB/s, 24 objects/s recovering 2023-10-01T11:08:12.309 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:11 smithi078 bash[75299]: audit 2023-10-01T11:08:11.426730+0000 mon.c (mon.2) 229 : audit 1 from='client.? 172.21.15.78:0/1270169454' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num", "val": "26"}]: dispatch 2023-10-01T11:08:12.309 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:11 smithi078 bash[75299]: audit 2023-10-01T11:08:11.427221+0000 mon.a (mon.0) 1974 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num", "val": "26"}]: dispatch 2023-10-01T11:08:12.310 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:11 smithi078 bash[78274]: cluster 2023-10-01T11:08:10.346824+0000 mgr.y (mgr.25072) 1526 : cluster 0 pgmap v1633: 89 pgs: 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 8905/55260 objects degraded (16.115%); 2387/55260 objects misplaced (4.320%); 1.5 MiB/s, 24 objects/s recovering 2023-10-01T11:08:12.310 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:11 smithi078 bash[78274]: audit 2023-10-01T11:08:11.426730+0000 mon.c (mon.2) 229 : audit 1 from='client.? 172.21.15.78:0/1270169454' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num", "val": "26"}]: dispatch 2023-10-01T11:08:12.310 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:11 smithi078 bash[78274]: audit 2023-10-01T11:08:11.427221+0000 mon.a (mon.0) 1974 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num", "val": "26"}]: dispatch 2023-10-01T11:08:12.335 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:12.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:12.337 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:08:12.438 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:12.461 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:12.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:12.474 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:08:12.575 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:12.613 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:12.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:12.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:12.719 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:12.750 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:12.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:12.758 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:08:12.859 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:12.920 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:12.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:12.927 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:13.028 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:13.039 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:12 smithi078 bash[75299]: cluster 2023-10-01T11:08:11.879370+0000 osd.7 (osd.7) 382 : cluster 3 failed to encode map e346 with expected crc 2023-10-01T11:08:13.039 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:12 smithi078 bash[75299]: audit 2023-10-01T11:08:11.880453+0000 mon.a (mon.0) 1975 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num", "val": "26"}]': finished 2023-10-01T11:08:13.039 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:12 smithi078 bash[75299]: cluster 2023-10-01T11:08:11.880515+0000 mon.a (mon.0) 1976 : cluster 0 osdmap e346: 8 total, 8 up, 8 in 2023-10-01T11:08:13.039 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:12 smithi078 bash[75299]: cluster 2023-10-01T11:08:11.892005+0000 osd.5 (osd.5) 365 : cluster 3 failed to encode map e346 with expected crc 2023-10-01T11:08:13.039 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:12 smithi078 bash[75299]: audit 2023-10-01T11:08:12.348251+0000 mon.a (mon.0) 1977 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-01T11:08:13.040 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:12 smithi078 bash[75299]: cluster 2023-10-01T11:08:12.779968+0000 mon.a (mon.0) 1978 : cluster 3 Health check update: Degraded data redundancy: 8905/55260 objects degraded (16.115%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T11:08:13.040 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:12 smithi078 bash[78274]: cluster 2023-10-01T11:08:11.879370+0000 osd.7 (osd.7) 382 : cluster 3 failed to encode map e346 with expected crc 2023-10-01T11:08:13.040 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:12 smithi078 bash[78274]: audit 2023-10-01T11:08:11.880453+0000 mon.a (mon.0) 1975 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num", "val": "26"}]': finished 2023-10-01T11:08:13.040 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:12 smithi078 bash[78274]: cluster 2023-10-01T11:08:11.880515+0000 mon.a (mon.0) 1976 : cluster 0 osdmap e346: 8 total, 8 up, 8 in 2023-10-01T11:08:13.041 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:12 smithi078 bash[78274]: cluster 2023-10-01T11:08:11.892005+0000 osd.5 (osd.5) 365 : cluster 3 failed to encode map e346 with expected crc 2023-10-01T11:08:13.041 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:12 smithi078 bash[78274]: audit 2023-10-01T11:08:12.348251+0000 mon.a (mon.0) 1977 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-01T11:08:13.041 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:12 smithi078 bash[78274]: cluster 2023-10-01T11:08:12.779968+0000 mon.a (mon.0) 1978 : cluster 3 Health check update: Degraded data redundancy: 8905/55260 objects degraded (16.115%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T11:08:13.085 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:13.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:13.092 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:13.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:12 smithi173 bash[58816]: cluster 2023-10-01T11:08:11.879370+0000 osd.7 (osd.7) 382 : cluster 3 failed to encode map e346 with expected crc 2023-10-01T11:08:13.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:12 smithi173 bash[58816]: audit 2023-10-01T11:08:11.880453+0000 mon.a (mon.0) 1975 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num", "val": "26"}]': finished 2023-10-01T11:08:13.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:12 smithi173 bash[58816]: cluster 2023-10-01T11:08:11.880515+0000 mon.a (mon.0) 1976 : cluster 0 osdmap e346: 8 total, 8 up, 8 in 2023-10-01T11:08:13.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:12 smithi173 bash[58816]: cluster 2023-10-01T11:08:11.892005+0000 osd.5 (osd.5) 365 : cluster 3 failed to encode map e346 with expected crc 2023-10-01T11:08:13.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:12 smithi173 bash[58816]: audit 2023-10-01T11:08:12.348251+0000 mon.a (mon.0) 1977 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-10-01T11:08:13.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:12 smithi173 bash[58816]: cluster 2023-10-01T11:08:12.779968+0000 mon.a (mon.0) 1978 : cluster 3 Health check update: Degraded data redundancy: 8905/55260 objects degraded (16.115%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-10-01T11:08:13.193 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:13.239 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:13.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:13.248 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:13.348 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:13.371 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:13.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:13.373 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:08:13.473 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:13.516 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:13.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:13.519 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:13.620 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:13.654 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:13.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:13.656 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:08:13.757 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:13.779 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:13.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:13.782 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:13.883 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:13.916 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:13.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:13.919 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:08:14.019 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:14.044 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:14.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:14.050 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:08:14.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:13 smithi078 bash[75299]: cluster 2023-10-01T11:08:12.347388+0000 mgr.y (mgr.25072) 1527 : cluster 0 pgmap v1635: 89 pgs: 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 8905/55260 objects degraded (16.115%); 2387/55260 objects misplaced (4.320%); 1.3 MiB/s, 21 objects/s recovering 2023-10-01T11:08:14.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:13 smithi078 bash[75299]: audit 2023-10-01T11:08:12.889784+0000 mon.a (mon.0) 1979 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-01T11:08:14.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:13 smithi078 bash[75299]: cluster 2023-10-01T11:08:12.889846+0000 mon.a (mon.0) 1980 : cluster 0 osdmap e347: 8 total, 8 up, 8 in 2023-10-01T11:08:14.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:13 smithi078 bash[75299]: cluster 2023-10-01T11:08:12.897699+0000 osd.5 (osd.5) 366 : cluster 3 failed to encode map e347 with expected crc 2023-10-01T11:08:14.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:13 smithi078 bash[75299]: cluster 2023-10-01T11:08:12.988813+0000 osd.6 (osd.6) 331 : cluster 3 failed to encode map e347 with expected crc 2023-10-01T11:08:14.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:13 smithi078 bash[75299]: cluster 2023-10-01T11:08:13.133626+0000 osd.7 (osd.7) 383 : cluster 3 failed to encode map e347 with expected crc 2023-10-01T11:08:14.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:13 smithi078 bash[78274]: cluster 2023-10-01T11:08:12.347388+0000 mgr.y (mgr.25072) 1527 : cluster 0 pgmap v1635: 89 pgs: 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 8905/55260 objects degraded (16.115%); 2387/55260 objects misplaced (4.320%); 1.3 MiB/s, 21 objects/s recovering 2023-10-01T11:08:14.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:13 smithi078 bash[78274]: audit 2023-10-01T11:08:12.889784+0000 mon.a (mon.0) 1979 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-01T11:08:14.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:13 smithi078 bash[78274]: cluster 2023-10-01T11:08:12.889846+0000 mon.a (mon.0) 1980 : cluster 0 osdmap e347: 8 total, 8 up, 8 in 2023-10-01T11:08:14.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:13 smithi078 bash[78274]: cluster 2023-10-01T11:08:12.897699+0000 osd.5 (osd.5) 366 : cluster 3 failed to encode map e347 with expected crc 2023-10-01T11:08:14.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:13 smithi078 bash[78274]: cluster 2023-10-01T11:08:12.988813+0000 osd.6 (osd.6) 331 : cluster 3 failed to encode map e347 with expected crc 2023-10-01T11:08:14.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:13 smithi078 bash[78274]: cluster 2023-10-01T11:08:13.133626+0000 osd.7 (osd.7) 383 : cluster 3 failed to encode map e347 with expected crc 2023-10-01T11:08:14.150 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:14.179 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:14.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:14.182 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:08:14.282 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:14.292 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:13 smithi173 bash[58816]: cluster 2023-10-01T11:08:12.347388+0000 mgr.y (mgr.25072) 1527 : cluster 0 pgmap v1635: 89 pgs: 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 1.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 8905/55260 objects degraded (16.115%); 2387/55260 objects misplaced (4.320%); 1.3 MiB/s, 21 objects/s recovering 2023-10-01T11:08:14.292 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:13 smithi173 bash[58816]: audit 2023-10-01T11:08:12.889784+0000 mon.a (mon.0) 1979 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_8", "var": "pg_num_actual", "val": "26"}]': finished 2023-10-01T11:08:14.292 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:13 smithi173 bash[58816]: cluster 2023-10-01T11:08:12.889846+0000 mon.a (mon.0) 1980 : cluster 0 osdmap e347: 8 total, 8 up, 8 in 2023-10-01T11:08:14.292 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:13 smithi173 bash[58816]: cluster 2023-10-01T11:08:12.897699+0000 osd.5 (osd.5) 366 : cluster 3 failed to encode map e347 with expected crc 2023-10-01T11:08:14.293 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:13 smithi173 bash[58816]: cluster 2023-10-01T11:08:12.988813+0000 osd.6 (osd.6) 331 : cluster 3 failed to encode map e347 with expected crc 2023-10-01T11:08:14.293 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:13 smithi173 bash[58816]: cluster 2023-10-01T11:08:13.133626+0000 osd.7 (osd.7) 383 : cluster 3 failed to encode map e347 with expected crc 2023-10-01T11:08:14.306 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:14.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:14.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:08:14.412 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:14.433 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:14.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:14.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:14.537 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:14.559 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:14.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:14.561 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:14.662 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:14.686 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:14.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:14.690 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:08:14.790 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:14.820 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:14.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:14.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:14.924 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:14.934 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:14 smithi078 bash[78274]: cluster 2023-10-01T11:08:13.902217+0000 osd.6 (osd.6) 332 : cluster 3 failed to encode map e348 with expected crc 2023-10-01T11:08:14.934 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:14 smithi078 bash[78274]: cluster 2023-10-01T11:08:13.904143+0000 mon.a (mon.0) 1981 : cluster 0 osdmap e348: 8 total, 8 up, 8 in 2023-10-01T11:08:14.934 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:14 smithi078 bash[78274]: cluster 2023-10-01T11:08:13.904909+0000 osd.6 (osd.6) 333 : cluster 3 failed to encode map e348 with expected crc 2023-10-01T11:08:14.934 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:14 smithi078 bash[78274]: cluster 2023-10-01T11:08:13.906811+0000 osd.6 (osd.6) 334 : cluster 3 failed to encode map e348 with expected crc 2023-10-01T11:08:14.935 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:14 smithi078 bash[78274]: cluster 2023-10-01T11:08:13.908649+0000 osd.2 (osd.2) 19 : cluster 1 13.8 continuing backfill to osd.6 from (291'200,346'9058] MIN to 346'9058 2023-10-01T11:08:14.935 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:14 smithi078 bash[78274]: cluster 2023-10-01T11:08:13.909393+0000 osd.7 (osd.7) 384 : cluster 3 failed to encode map e348 with expected crc 2023-10-01T11:08:14.935 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:14 smithi078 bash[78274]: cluster 2023-10-01T11:08:13.911239+0000 osd.7 (osd.7) 385 : cluster 3 failed to encode map e348 with expected crc 2023-10-01T11:08:14.935 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:14 smithi078 bash[78274]: cluster 2023-10-01T11:08:13.931299+0000 osd.5 (osd.5) 367 : cluster 3 failed to encode map e348 with expected crc 2023-10-01T11:08:14.935 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:14 smithi078 bash[78274]: cluster 2023-10-01T11:08:13.933548+0000 osd.5 (osd.5) 368 : cluster 3 failed to encode map e348 with expected crc 2023-10-01T11:08:14.935 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:14 smithi078 bash[78274]: cluster 2023-10-01T11:08:13.935751+0000 osd.5 (osd.5) 369 : cluster 3 failed to encode map e348 with expected crc 2023-10-01T11:08:14.935 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:14 smithi078 bash[75299]: cluster 2023-10-01T11:08:13.902217+0000 osd.6 (osd.6) 332 : cluster 3 failed to encode map e348 with expected crc 2023-10-01T11:08:14.935 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:14 smithi078 bash[75299]: cluster 2023-10-01T11:08:13.904143+0000 mon.a (mon.0) 1981 : cluster 0 osdmap e348: 8 total, 8 up, 8 in 2023-10-01T11:08:14.936 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:14 smithi078 bash[75299]: cluster 2023-10-01T11:08:13.904909+0000 osd.6 (osd.6) 333 : cluster 3 failed to encode map e348 with expected crc 2023-10-01T11:08:14.936 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:14 smithi078 bash[75299]: cluster 2023-10-01T11:08:13.906811+0000 osd.6 (osd.6) 334 : cluster 3 failed to encode map e348 with expected crc 2023-10-01T11:08:14.936 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:14 smithi078 bash[75299]: cluster 2023-10-01T11:08:13.908649+0000 osd.2 (osd.2) 19 : cluster 1 13.8 continuing backfill to osd.6 from (291'200,346'9058] MIN to 346'9058 2023-10-01T11:08:14.936 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:14 smithi078 bash[75299]: cluster 2023-10-01T11:08:13.909393+0000 osd.7 (osd.7) 384 : cluster 3 failed to encode map e348 with expected crc 2023-10-01T11:08:14.936 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:14 smithi078 bash[75299]: cluster 2023-10-01T11:08:13.911239+0000 osd.7 (osd.7) 385 : cluster 3 failed to encode map e348 with expected crc 2023-10-01T11:08:14.936 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:14 smithi078 bash[75299]: cluster 2023-10-01T11:08:13.931299+0000 osd.5 (osd.5) 367 : cluster 3 failed to encode map e348 with expected crc 2023-10-01T11:08:14.937 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:14 smithi078 bash[75299]: cluster 2023-10-01T11:08:13.933548+0000 osd.5 (osd.5) 368 : cluster 3 failed to encode map e348 with expected crc 2023-10-01T11:08:14.937 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:14 smithi078 bash[75299]: cluster 2023-10-01T11:08:13.935751+0000 osd.5 (osd.5) 369 : cluster 3 failed to encode map e348 with expected crc 2023-10-01T11:08:14.948 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:14.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:14.951 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:08:15.052 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:15.073 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:15.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:15.075 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:15.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:14 smithi173 bash[58816]: cluster 2023-10-01T11:08:13.902217+0000 osd.6 (osd.6) 332 : cluster 3 failed to encode map e348 with expected crc 2023-10-01T11:08:15.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:14 smithi173 bash[58816]: cluster 2023-10-01T11:08:13.904143+0000 mon.a (mon.0) 1981 : cluster 0 osdmap e348: 8 total, 8 up, 8 in 2023-10-01T11:08:15.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:14 smithi173 bash[58816]: cluster 2023-10-01T11:08:13.904909+0000 osd.6 (osd.6) 333 : cluster 3 failed to encode map e348 with expected crc 2023-10-01T11:08:15.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:14 smithi173 bash[58816]: cluster 2023-10-01T11:08:13.906811+0000 osd.6 (osd.6) 334 : cluster 3 failed to encode map e348 with expected crc 2023-10-01T11:08:15.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:14 smithi173 bash[58816]: cluster 2023-10-01T11:08:13.908649+0000 osd.2 (osd.2) 19 : cluster 1 13.8 continuing backfill to osd.6 from (291'200,346'9058] MIN to 346'9058 2023-10-01T11:08:15.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:14 smithi173 bash[58816]: cluster 2023-10-01T11:08:13.909393+0000 osd.7 (osd.7) 384 : cluster 3 failed to encode map e348 with expected crc 2023-10-01T11:08:15.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:14 smithi173 bash[58816]: cluster 2023-10-01T11:08:13.911239+0000 osd.7 (osd.7) 385 : cluster 3 failed to encode map e348 with expected crc 2023-10-01T11:08:15.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:14 smithi173 bash[58816]: cluster 2023-10-01T11:08:13.931299+0000 osd.5 (osd.5) 367 : cluster 3 failed to encode map e348 with expected crc 2023-10-01T11:08:15.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:14 smithi173 bash[58816]: cluster 2023-10-01T11:08:13.933548+0000 osd.5 (osd.5) 368 : cluster 3 failed to encode map e348 with expected crc 2023-10-01T11:08:15.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:14 smithi173 bash[58816]: cluster 2023-10-01T11:08:13.935751+0000 osd.5 (osd.5) 369 : cluster 3 failed to encode map e348 with expected crc 2023-10-01T11:08:15.176 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:15.200 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:15.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:15.203 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:15.303 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:15.330 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:15.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:15.335 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:15.435 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:15.459 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:15.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:15.461 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:08:15.562 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:15.588 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:15.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:15.590 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:15.691 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:15.724 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:15.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:15.726 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:15.827 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:15.856 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:15.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:15.858 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:08:15.959 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:15.965 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:15 smithi078 bash[75299]: cluster 2023-10-01T11:08:14.348273+0000 mgr.y (mgr.25072) 1528 : cluster 0 pgmap v1638: 99 pgs: 1 activating+degraded, 2 activating, 1 active+recovery_wait+degraded, 5 peering, 79 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 6 unknown, 2 remapped+peering; 875 MiB data, 11 GiB used, 704 GiB / 715 GiB avail; 6020/42045 objects degraded (14.318%); 1021/42045 objects misplaced (2.428%) 2023-10-01T11:08:15.965 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:15 smithi078 bash[75299]: cluster 2023-10-01T11:08:14.910511+0000 osd.7 (osd.7) 386 : cluster 3 failed to encode map e349 with expected crc 2023-10-01T11:08:15.990 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:15.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:15.995 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:08:16.096 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:16.121 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:16.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:16.127 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:08:16.228 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:16.241 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:15 smithi173 bash[58816]: cluster 2023-10-01T11:08:14.348273+0000 mgr.y (mgr.25072) 1528 : cluster 0 pgmap v1638: 99 pgs: 1 activating+degraded, 2 activating, 1 active+recovery_wait+degraded, 5 peering, 79 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 6 unknown, 2 remapped+peering; 875 MiB data, 11 GiB used, 704 GiB / 715 GiB avail; 6020/42045 objects degraded (14.318%); 1021/42045 objects misplaced (2.428%) 2023-10-01T11:08:16.241 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:15 smithi173 bash[58816]: cluster 2023-10-01T11:08:14.910511+0000 osd.7 (osd.7) 386 : cluster 3 failed to encode map e349 with expected crc 2023-10-01T11:08:16.241 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:15 smithi173 bash[58816]: cluster 2023-10-01T11:08:14.913435+0000 osd.7 (osd.7) 387 : cluster 3 failed to encode map e349 with expected crc 2023-10-01T11:08:16.241 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:15 smithi173 bash[58816]: cluster 2023-10-01T11:08:14.917507+0000 osd.5 (osd.5) 370 : cluster 3 failed to encode map e349 with expected crc 2023-10-01T11:08:16.241 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:15 smithi173 bash[58816]: cluster 2023-10-01T11:08:14.917547+0000 osd.6 (osd.6) 335 : cluster 3 failed to encode map e349 with expected crc 2023-10-01T11:08:16.241 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:15 smithi173 bash[58816]: cluster 2023-10-01T11:08:14.917920+0000 osd.7 (osd.7) 388 : cluster 3 failed to encode map e349 with expected crc 2023-10-01T11:08:16.241 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:15 smithi173 bash[58816]: cluster 2023-10-01T11:08:14.919397+0000 mon.a (mon.0) 1982 : cluster 0 osdmap e349: 8 total, 8 up, 8 in 2023-10-01T11:08:16.274 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:16.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:16.278 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:08:16.379 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:16.386 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:15 smithi078 bash[75299]: cluster 2023-10-01T11:08:14.913435+0000 osd.7 (osd.7) 387 : cluster 3 failed to encode map e349 with expected crc 2023-10-01T11:08:16.386 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:15 smithi078 bash[75299]: cluster 2023-10-01T11:08:14.917507+0000 osd.5 (osd.5) 370 : cluster 3 failed to encode map e349 with expected crc 2023-10-01T11:08:16.386 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:15 smithi078 bash[75299]: cluster 2023-10-01T11:08:14.917547+0000 osd.6 (osd.6) 335 : cluster 3 failed to encode map e349 with expected crc 2023-10-01T11:08:16.386 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:15 smithi078 bash[75299]: cluster 2023-10-01T11:08:14.917920+0000 osd.7 (osd.7) 388 : cluster 3 failed to encode map e349 with expected crc 2023-10-01T11:08:16.386 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:15 smithi078 bash[75299]: cluster 2023-10-01T11:08:14.919397+0000 mon.a (mon.0) 1982 : cluster 0 osdmap e349: 8 total, 8 up, 8 in 2023-10-01T11:08:16.406 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:16.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:16.411 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:08:16.512 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:16.536 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:16.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:16.540 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:08:16.640 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:16.649 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:16 smithi078 bash[78274]: cluster 2023-10-01T11:08:14.348273+0000 mgr.y (mgr.25072) 1528 : cluster 0 pgmap v1638: 99 pgs: 1 activating+degraded, 2 activating, 1 active+recovery_wait+degraded, 5 peering, 79 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 6 unknown, 2 remapped+peering; 875 MiB data, 11 GiB used, 704 GiB / 715 GiB avail; 6020/42045 objects degraded (14.318%); 1021/42045 objects misplaced (2.428%) 2023-10-01T11:08:16.649 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:16 smithi078 bash[78274]: cluster 2023-10-01T11:08:14.910511+0000 osd.7 (osd.7) 386 : cluster 3 failed to encode map e349 with expected crc 2023-10-01T11:08:16.650 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:16 smithi078 bash[78274]: cluster 2023-10-01T11:08:14.913435+0000 osd.7 (osd.7) 387 : cluster 3 failed to encode map e349 with expected crc 2023-10-01T11:08:16.650 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:16 smithi078 bash[78274]: cluster 2023-10-01T11:08:14.917507+0000 osd.5 (osd.5) 370 : cluster 3 failed to encode map e349 with expected crc 2023-10-01T11:08:16.650 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:16 smithi078 bash[78274]: cluster 2023-10-01T11:08:14.917547+0000 osd.6 (osd.6) 335 : cluster 3 failed to encode map e349 with expected crc 2023-10-01T11:08:16.650 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:16 smithi078 bash[78274]: cluster 2023-10-01T11:08:14.917920+0000 osd.7 (osd.7) 388 : cluster 3 failed to encode map e349 with expected crc 2023-10-01T11:08:16.650 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:16 smithi078 bash[78274]: cluster 2023-10-01T11:08:14.919397+0000 mon.a (mon.0) 1982 : cluster 0 osdmap e349: 8 total, 8 up, 8 in 2023-10-01T11:08:16.679 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:16.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:16.681 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:16.783 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:16.822 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:16.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:16.830 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:08:16.930 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:16.936 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:16 smithi078 bash[78274]: cluster 2023-10-01T11:08:15.910605+0000 osd.5 (osd.5) 371 : cluster 3 failed to encode map e350 with expected crc 2023-10-01T11:08:16.936 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:16 smithi078 bash[78274]: cluster 2023-10-01T11:08:15.912007+0000 mon.a (mon.0) 1983 : cluster 0 osdmap e350: 8 total, 8 up, 8 in 2023-10-01T11:08:16.937 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:16 smithi078 bash[78274]: cluster 2023-10-01T11:08:15.913672+0000 osd.7 (osd.7) 389 : cluster 3 failed to encode map e350 with expected crc 2023-10-01T11:08:16.937 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:16 smithi078 bash[78274]: cluster 2023-10-01T11:08:15.915536+0000 osd.2 (osd.2) 20 : cluster 1 13.18 continuing backfill to osd.6 from (291'200,346'9058] MIN to 346'9058 2023-10-01T11:08:16.937 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:16 smithi078 bash[78274]: cluster 2023-10-01T11:08:15.916654+0000 osd.7 (osd.7) 390 : cluster 3 failed to encode map e350 with expected crc 2023-10-01T11:08:16.937 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:16 smithi078 bash[78274]: cluster 2023-10-01T11:08:15.917375+0000 osd.6 (osd.6) 336 : cluster 3 failed to encode map e350 with expected crc 2023-10-01T11:08:16.937 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:16 smithi078 bash[78274]: cluster 2023-10-01T11:08:15.919000+0000 osd.7 (osd.7) 391 : cluster 3 failed to encode map e350 with expected crc 2023-10-01T11:08:16.937 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:16 smithi078 bash[78274]: cluster 2023-10-01T11:08:15.920471+0000 osd.7 (osd.7) 392 : cluster 3 failed to encode map e350 with expected crc 2023-10-01T11:08:16.937 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:16 smithi078 bash[75299]: cluster 2023-10-01T11:08:15.910605+0000 osd.5 (osd.5) 371 : cluster 3 failed to encode map e350 with expected crc 2023-10-01T11:08:16.937 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:16 smithi078 bash[75299]: cluster 2023-10-01T11:08:15.912007+0000 mon.a (mon.0) 1983 : cluster 0 osdmap e350: 8 total, 8 up, 8 in 2023-10-01T11:08:16.937 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:16 smithi078 bash[75299]: cluster 2023-10-01T11:08:15.913672+0000 osd.7 (osd.7) 389 : cluster 3 failed to encode map e350 with expected crc 2023-10-01T11:08:16.937 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:16 smithi078 bash[75299]: cluster 2023-10-01T11:08:15.915536+0000 osd.2 (osd.2) 20 : cluster 1 13.18 continuing backfill to osd.6 from (291'200,346'9058] MIN to 346'9058 2023-10-01T11:08:16.937 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:16 smithi078 bash[75299]: cluster 2023-10-01T11:08:15.916654+0000 osd.7 (osd.7) 390 : cluster 3 failed to encode map e350 with expected crc 2023-10-01T11:08:16.938 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:16 smithi078 bash[75299]: cluster 2023-10-01T11:08:15.917375+0000 osd.6 (osd.6) 336 : cluster 3 failed to encode map e350 with expected crc 2023-10-01T11:08:16.938 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:16 smithi078 bash[75299]: cluster 2023-10-01T11:08:15.919000+0000 osd.7 (osd.7) 391 : cluster 3 failed to encode map e350 with expected crc 2023-10-01T11:08:16.938 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:16 smithi078 bash[75299]: cluster 2023-10-01T11:08:15.920471+0000 osd.7 (osd.7) 392 : cluster 3 failed to encode map e350 with expected crc 2023-10-01T11:08:16.963 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:16.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:16.967 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:08:17.068 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:17.090 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:17.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:17.094 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:08:17.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:16 smithi173 bash[58816]: cluster 2023-10-01T11:08:15.910605+0000 osd.5 (osd.5) 371 : cluster 3 failed to encode map e350 with expected crc 2023-10-01T11:08:17.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:16 smithi173 bash[58816]: cluster 2023-10-01T11:08:15.912007+0000 mon.a (mon.0) 1983 : cluster 0 osdmap e350: 8 total, 8 up, 8 in 2023-10-01T11:08:17.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:16 smithi173 bash[58816]: cluster 2023-10-01T11:08:15.913672+0000 osd.7 (osd.7) 389 : cluster 3 failed to encode map e350 with expected crc 2023-10-01T11:08:17.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:16 smithi173 bash[58816]: cluster 2023-10-01T11:08:15.915536+0000 osd.2 (osd.2) 20 : cluster 1 13.18 continuing backfill to osd.6 from (291'200,346'9058] MIN to 346'9058 2023-10-01T11:08:17.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:16 smithi173 bash[58816]: cluster 2023-10-01T11:08:15.916654+0000 osd.7 (osd.7) 390 : cluster 3 failed to encode map e350 with expected crc 2023-10-01T11:08:17.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:16 smithi173 bash[58816]: cluster 2023-10-01T11:08:15.917375+0000 osd.6 (osd.6) 336 : cluster 3 failed to encode map e350 with expected crc 2023-10-01T11:08:17.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:16 smithi173 bash[58816]: cluster 2023-10-01T11:08:15.919000+0000 osd.7 (osd.7) 391 : cluster 3 failed to encode map e350 with expected crc 2023-10-01T11:08:17.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:16 smithi173 bash[58816]: cluster 2023-10-01T11:08:15.920471+0000 osd.7 (osd.7) 392 : cluster 3 failed to encode map e350 with expected crc 2023-10-01T11:08:17.195 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:17.221 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:17.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:17.232 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:17.332 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:17.369 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:17.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:17.374 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:08:17.475 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:17.508 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:17.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:17.511 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:08:17.612 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:17.635 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:17.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:17.642 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:08:17.743 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:17.780 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:17.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:17.786 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:08:17.886 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:17.922 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:17.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:17.925 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:08:18.026 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:18.053 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:18.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:18.056 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:18.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:17 smithi173 bash[58816]: cluster 2023-10-01T11:08:16.348723+0000 mgr.y (mgr.25072) 1529 : cluster 0 pgmap v1641: 99 pgs: 1 activating+degraded, 2 activating, 1 active+recovery_wait+degraded, 5 peering, 79 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 6 unknown, 2 remapped+peering; 875 MiB data, 11 GiB used, 704 GiB / 715 GiB avail; 6020/42045 objects degraded (14.318%); 1021/42045 objects misplaced (2.428%) 2023-10-01T11:08:18.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:17 smithi173 bash[58816]: audit 2023-10-01T11:08:17.464650+0000 mon.a (mon.0) 1984 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:08:18.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:17 smithi173 bash[58816]: cluster 2023-10-01T11:08:17.781103+0000 mon.a (mon.0) 1985 : cluster 3 Health check update: Degraded data redundancy: 6020/42045 objects degraded (14.318%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T11:08:18.157 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:18.184 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:18.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:18.188 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:08:18.289 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:18.325 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:18.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:18.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:18.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:17 smithi078 bash[78274]: cluster 2023-10-01T11:08:16.348723+0000 mgr.y (mgr.25072) 1529 : cluster 0 pgmap v1641: 99 pgs: 1 activating+degraded, 2 activating, 1 active+recovery_wait+degraded, 5 peering, 79 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 6 unknown, 2 remapped+peering; 875 MiB data, 11 GiB used, 704 GiB / 715 GiB avail; 6020/42045 objects degraded (14.318%); 1021/42045 objects misplaced (2.428%) 2023-10-01T11:08:18.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:17 smithi078 bash[78274]: audit 2023-10-01T11:08:17.464650+0000 mon.a (mon.0) 1984 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:08:18.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:17 smithi078 bash[78274]: cluster 2023-10-01T11:08:17.781103+0000 mon.a (mon.0) 1985 : cluster 3 Health check update: Degraded data redundancy: 6020/42045 objects degraded (14.318%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T11:08:18.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:17 smithi078 bash[75299]: cluster 2023-10-01T11:08:16.348723+0000 mgr.y (mgr.25072) 1529 : cluster 0 pgmap v1641: 99 pgs: 1 activating+degraded, 2 activating, 1 active+recovery_wait+degraded, 5 peering, 79 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 6 unknown, 2 remapped+peering; 875 MiB data, 11 GiB used, 704 GiB / 715 GiB avail; 6020/42045 objects degraded (14.318%); 1021/42045 objects misplaced (2.428%) 2023-10-01T11:08:18.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:17 smithi078 bash[75299]: audit 2023-10-01T11:08:17.464650+0000 mon.a (mon.0) 1984 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:08:18.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:17 smithi078 bash[75299]: cluster 2023-10-01T11:08:17.781103+0000 mon.a (mon.0) 1985 : cluster 3 Health check update: Degraded data redundancy: 6020/42045 objects degraded (14.318%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-10-01T11:08:18.428 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:18.459 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:18.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:18.461 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:18.561 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:18.588 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:18.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:18.591 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:08:18.682 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T11:08:18.692 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:18.706 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T11:08:18.721 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:18.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:18.722 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:08:18.732 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T11:08:18.768 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T11:08:18.814 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T11:08:18.822 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:18.846 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T11:08:18.853 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:18.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:18.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:18.874 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T11:08:18.906 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T11:08:18.918 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T11:08:18.918 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T11:08:18.918 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T11:08:18.918 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T11:08:18.918 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T11:08:18.918 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T11:08:18.918 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T11:08:18.918 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T11:08:18.958 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:18.996 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:18.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:18.999 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:08:19.099 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:19.145 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:19.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:19.151 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:19.252 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:19.262 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:18 smithi173 bash[58816]: audit 2023-10-01T11:08:18.654366+0000 mon.c (mon.2) 230 : audit 0 from='client.? 172.21.15.78:0/3348917391' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:08:19.290 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:19.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:19.295 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:08:19.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:18 smithi078 bash[78274]: audit 2023-10-01T11:08:18.654366+0000 mon.c (mon.2) 230 : audit 0 from='client.? 172.21.15.78:0/3348917391' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:08:19.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:18 smithi078 bash[75299]: audit 2023-10-01T11:08:18.654366+0000 mon.c (mon.2) 230 : audit 0 from='client.? 172.21.15.78:0/3348917391' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:08:19.396 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:19.415 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:19.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:19.420 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:08:19.520 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:19.559 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:19.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:19.563 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:08:19.664 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:19.692 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:19.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:19.696 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:19.797 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:19.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:19.825 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:19.825 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:19.926 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:19.948 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:19.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:19.952 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:08:20.053 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:20.076 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:20.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:20.080 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:08:20.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:19 smithi173 bash[58816]: cluster 2023-10-01T11:08:18.350634+0000 mgr.y (mgr.25072) 1530 : cluster 0 pgmap v1642: 99 pgs: 1 activating+degraded, 2 activating, 1 active+recovery_wait+degraded, 5 peering, 79 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 6 unknown, 2 remapped+peering; 875 MiB data, 11 GiB used, 704 GiB / 715 GiB avail; 6020/42045 objects degraded (14.318%); 1021/42045 objects misplaced (2.428%) 2023-10-01T11:08:20.181 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:20.189 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:19 smithi078 bash[75299]: cluster 2023-10-01T11:08:18.350634+0000 mgr.y (mgr.25072) 1530 : cluster 0 pgmap v1642: 99 pgs: 1 activating+degraded, 2 activating, 1 active+recovery_wait+degraded, 5 peering, 79 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 6 unknown, 2 remapped+peering; 875 MiB data, 11 GiB used, 704 GiB / 715 GiB avail; 6020/42045 objects degraded (14.318%); 1021/42045 objects misplaced (2.428%) 2023-10-01T11:08:20.189 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:19 smithi078 bash[78274]: cluster 2023-10-01T11:08:18.350634+0000 mgr.y (mgr.25072) 1530 : cluster 0 pgmap v1642: 99 pgs: 1 activating+degraded, 2 activating, 1 active+recovery_wait+degraded, 5 peering, 79 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 6 unknown, 2 remapped+peering; 875 MiB data, 11 GiB used, 704 GiB / 715 GiB avail; 6020/42045 objects degraded (14.318%); 1021/42045 objects misplaced (2.428%) 2023-10-01T11:08:20.204 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:20.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:20.208 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:20.309 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:20.334 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:20.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:20.338 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:20.438 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:20.445 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:08:20 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:08:20] "GET /metrics HTTP/1.1" 200 36390 "" "Prometheus/2.33.4" 2023-10-01T11:08:20.458 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:20.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:20.461 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:08:20.562 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:20.588 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:20.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:20.591 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:20.692 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:20.727 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:20.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:20.732 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:08:20.832 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:20.864 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:20.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:20.867 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:08:20.968 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:20.999 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:21.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:21.009 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:21.110 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:21.141 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:21.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:21.144 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:08:21.245 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:21.285 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:21.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:21.288 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:08:21.388 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:21.418 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:21.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:21.421 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:21.521 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:21.550 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:21.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:21.555 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:08:21.656 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:21.681 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:21.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:21.683 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:08:21.784 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:21.814 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:21.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:21.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:08:21.920 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:21.947 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:21.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:21.951 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:08:22.051 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:22.053 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T11:08:22.053 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T11:08:22.053 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-01T11:08:22.054 INFO:tasks.thrashosds.thrasher:Setting osd 1 primary_affinity to 0.659718 2023-10-01T11:08:22.054 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 1 0.6597183621581236 2023-10-01T11:08:22.059 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:21 smithi173 bash[58816]: cluster 2023-10-01T11:08:20.351567+0000 mgr.y (mgr.25072) 1531 : cluster 0 pgmap v1643: 99 pgs: 4 active+recovery_wait+degraded, 90 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped; 823 MiB data, 11 GiB used, 705 GiB / 715 GiB avail; 96 MiB/s wr, 3.28k op/s; 9977/39558 objects degraded (25.221%); 871/39558 objects misplaced (2.202%); 5.1 MiB/s, 129 objects/s recovering 2023-10-01T11:08:22.075 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:22.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:22.077 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:08:22.178 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:22.187 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:21 smithi078 bash[75299]: cluster 2023-10-01T11:08:20.351567+0000 mgr.y (mgr.25072) 1531 : cluster 0 pgmap v1643: 99 pgs: 4 active+recovery_wait+degraded, 90 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped; 823 MiB data, 11 GiB used, 705 GiB / 715 GiB avail; 96 MiB/s wr, 3.28k op/s; 9977/39558 objects degraded (25.221%); 871/39558 objects misplaced (2.202%); 5.1 MiB/s, 129 objects/s recovering 2023-10-01T11:08:22.188 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:21 smithi078 bash[78274]: cluster 2023-10-01T11:08:20.351567+0000 mgr.y (mgr.25072) 1531 : cluster 0 pgmap v1643: 99 pgs: 4 active+recovery_wait+degraded, 90 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped; 823 MiB data, 11 GiB used, 705 GiB / 715 GiB avail; 96 MiB/s wr, 3.28k op/s; 9977/39558 objects degraded (25.221%); 871/39558 objects misplaced (2.202%); 5.1 MiB/s, 129 objects/s recovering 2023-10-01T11:08:22.208 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:22.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:22.211 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:22.312 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:22.359 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:22.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:22.364 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:08:22.464 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:22.503 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:22.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:22.507 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:08:22.598 INFO:teuthology.orchestra.run.smithi078.stdout:ERROR: (22) Invalid argument 2023-10-01T11:08:22.599 INFO:teuthology.orchestra.run.smithi078.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T11:08:22.608 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:22.641 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:22.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:22.644 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:08:22.744 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:22.770 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:22.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:22.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:08:22.876 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:22.900 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:22.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:22.904 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:22.980 INFO:teuthology.orchestra.run.smithi078.stderr:nodeep-scrub is unset 2023-10-01T11:08:23.005 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:23.014 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:22 smithi173 bash[58816]: audit 2023-10-01T11:08:22.765135+0000 mon.c (mon.2) 231 : audit 1 from='client.? 172.21.15.78:0/80913014' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:08:23.014 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:22 smithi173 bash[58816]: audit 2023-10-01T11:08:22.767703+0000 mon.a (mon.0) 1986 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:08:23.014 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:22 smithi173 bash[58816]: cluster 2023-10-01T11:08:22.783352+0000 mon.a (mon.0) 1987 : cluster 3 Health check update: Degraded data redundancy: 9977/39558 objects degraded (25.221%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T11:08:23.036 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:23.041 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:22 smithi078 bash[75299]: audit 2023-10-01T11:08:22.765135+0000 mon.c (mon.2) 231 : audit 1 from='client.? 172.21.15.78:0/80913014' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:08:23.041 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:22 smithi078 bash[75299]: audit 2023-10-01T11:08:22.767703+0000 mon.a (mon.0) 1986 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:08:23.041 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:22 smithi078 bash[75299]: cluster 2023-10-01T11:08:22.783352+0000 mon.a (mon.0) 1987 : cluster 3 Health check update: Degraded data redundancy: 9977/39558 objects degraded (25.221%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T11:08:23.041 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:22 smithi078 bash[78274]: audit 2023-10-01T11:08:22.765135+0000 mon.c (mon.2) 231 : audit 1 from='client.? 172.21.15.78:0/80913014' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:08:23.041 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:22 smithi078 bash[78274]: audit 2023-10-01T11:08:22.767703+0000 mon.a (mon.0) 1986 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:08:23.041 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:22 smithi078 bash[78274]: cluster 2023-10-01T11:08:22.783352+0000 mon.a (mon.0) 1987 : cluster 3 Health check update: Degraded data redundancy: 9977/39558 objects degraded (25.221%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T11:08:23.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:23.045 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:23.145 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:23.168 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:23.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:23.171 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:08:23.271 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:23.314 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:23.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:23.318 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:08:23.419 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:23.442 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:23.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:23.445 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:23.546 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:23.572 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:23.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:23.579 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:23.680 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:23.717 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:23.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:23.723 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:08:23.824 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:23.842 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:23.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:23.844 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:23.945 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:23.975 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:23.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:23.980 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:08:23.983 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T11:08:24.080 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:24.111 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:24.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:24.114 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:08:24.168 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:23 smithi078 bash[75299]: cluster 2023-10-01T11:08:22.352283+0000 mgr.y (mgr.25072) 1532 : cluster 0 pgmap v1644: 99 pgs: 4 active+recovery_wait+degraded, 90 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped; 823 MiB data, 11 GiB used, 705 GiB / 715 GiB avail; 77 MiB/s wr, 2.57k op/s; 9977/39558 objects degraded (25.221%); 871/39558 objects misplaced (2.202%); 4.1 MiB/s, 103 objects/s recovering 2023-10-01T11:08:24.168 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:23 smithi078 bash[75299]: cluster 2023-10-01T11:08:22.957499+0000 mon.a (mon.0) 1988 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T11:08:24.168 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:23 smithi078 bash[75299]: audit 2023-10-01T11:08:22.971216+0000 mon.a (mon.0) 1989 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T11:08:24.168 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:23 smithi078 bash[75299]: cluster 2023-10-01T11:08:22.971291+0000 mon.a (mon.0) 1990 : cluster 0 osdmap e351: 8 total, 8 up, 8 in 2023-10-01T11:08:24.168 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:23 smithi078 bash[75299]: cluster 2023-10-01T11:08:22.973413+0000 osd.7 (osd.7) 393 : cluster 3 failed to encode map e351 with expected crc 2023-10-01T11:08:24.168 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:23 smithi078 bash[75299]: cluster 2023-10-01T11:08:22.981523+0000 osd.5 (osd.5) 372 : cluster 3 failed to encode map e351 with expected crc 2023-10-01T11:08:24.168 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:23 smithi078 bash[75299]: cluster 2023-10-01T11:08:22.988854+0000 osd.5 (osd.5) 373 : cluster 3 failed to encode map e351 with expected crc 2023-10-01T11:08:24.169 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:23 smithi078 bash[78274]: cluster 2023-10-01T11:08:22.352283+0000 mgr.y (mgr.25072) 1532 : cluster 0 pgmap v1644: 99 pgs: 4 active+recovery_wait+degraded, 90 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped; 823 MiB data, 11 GiB used, 705 GiB / 715 GiB avail; 77 MiB/s wr, 2.57k op/s; 9977/39558 objects degraded (25.221%); 871/39558 objects misplaced (2.202%); 4.1 MiB/s, 103 objects/s recovering 2023-10-01T11:08:24.169 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:23 smithi078 bash[78274]: cluster 2023-10-01T11:08:22.957499+0000 mon.a (mon.0) 1988 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T11:08:24.169 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:23 smithi078 bash[78274]: audit 2023-10-01T11:08:22.971216+0000 mon.a (mon.0) 1989 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T11:08:24.169 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:23 smithi078 bash[78274]: cluster 2023-10-01T11:08:22.971291+0000 mon.a (mon.0) 1990 : cluster 0 osdmap e351: 8 total, 8 up, 8 in 2023-10-01T11:08:24.169 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:23 smithi078 bash[78274]: cluster 2023-10-01T11:08:22.973413+0000 osd.7 (osd.7) 393 : cluster 3 failed to encode map e351 with expected crc 2023-10-01T11:08:24.169 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:23 smithi078 bash[78274]: cluster 2023-10-01T11:08:22.981523+0000 osd.5 (osd.5) 372 : cluster 3 failed to encode map e351 with expected crc 2023-10-01T11:08:24.169 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:23 smithi078 bash[78274]: cluster 2023-10-01T11:08:22.988854+0000 osd.5 (osd.5) 373 : cluster 3 failed to encode map e351 with expected crc 2023-10-01T11:08:24.215 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:24.237 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:24.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:24.244 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:08:24.345 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:24.356 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:23 smithi173 bash[58816]: cluster 2023-10-01T11:08:22.352283+0000 mgr.y (mgr.25072) 1532 : cluster 0 pgmap v1644: 99 pgs: 4 active+recovery_wait+degraded, 90 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped; 823 MiB data, 11 GiB used, 705 GiB / 715 GiB avail; 77 MiB/s wr, 2.57k op/s; 9977/39558 objects degraded (25.221%); 871/39558 objects misplaced (2.202%); 4.1 MiB/s, 103 objects/s recovering 2023-10-01T11:08:24.356 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:23 smithi173 bash[58816]: cluster 2023-10-01T11:08:22.957499+0000 mon.a (mon.0) 1988 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T11:08:24.356 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:23 smithi173 bash[58816]: audit 2023-10-01T11:08:22.971216+0000 mon.a (mon.0) 1989 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T11:08:24.356 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:23 smithi173 bash[58816]: cluster 2023-10-01T11:08:22.971291+0000 mon.a (mon.0) 1990 : cluster 0 osdmap e351: 8 total, 8 up, 8 in 2023-10-01T11:08:24.356 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:23 smithi173 bash[58816]: cluster 2023-10-01T11:08:22.973413+0000 osd.7 (osd.7) 393 : cluster 3 failed to encode map e351 with expected crc 2023-10-01T11:08:24.356 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:23 smithi173 bash[58816]: cluster 2023-10-01T11:08:22.981523+0000 osd.5 (osd.5) 372 : cluster 3 failed to encode map e351 with expected crc 2023-10-01T11:08:24.356 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:23 smithi173 bash[58816]: cluster 2023-10-01T11:08:22.988854+0000 osd.5 (osd.5) 373 : cluster 3 failed to encode map e351 with expected crc 2023-10-01T11:08:24.383 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:24.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:24.387 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:08:24.448 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-01T11:08:24.487 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:24.519 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:24.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:24.522 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:08:24.622 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:24.647 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:24.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:24.651 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:08:24.751 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:24.782 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:24.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:24.786 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:08:24.887 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:24.914 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:24.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:24.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:08:25.018 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:25.052 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:25.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:25.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:08:25.156 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:25.180 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:25.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:25.183 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:25.283 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:25.317 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:25.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:25.321 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:25.422 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:25.449 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:25.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:25.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:25.552 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:25.576 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:25.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:25.579 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:08:25.680 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:25.701 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:25.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:25.705 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:08:25.806 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:25.843 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:25.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:25.847 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:08:25.948 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:25.980 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:25.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:25.987 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:08:26.087 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:26.116 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:26.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:26.120 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:26.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:25 smithi173 bash[58816]: cluster 2023-10-01T11:08:24.353011+0000 mgr.y (mgr.25072) 1533 : cluster 0 pgmap v1646: 99 pgs: 4 active+recovery_wait+degraded, 90 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 575 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 73 MiB/s wr, 2.91k op/s; 9977/27633 objects degraded (36.105%); 602/27633 objects misplaced (2.179%); 3.9 MiB/s, 111 objects/s recovering 2023-10-01T11:08:26.221 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:26.264 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:26.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:26.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:08:26.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:25 smithi078 bash[78274]: cluster 2023-10-01T11:08:24.353011+0000 mgr.y (mgr.25072) 1533 : cluster 0 pgmap v1646: 99 pgs: 4 active+recovery_wait+degraded, 90 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 575 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 73 MiB/s wr, 2.91k op/s; 9977/27633 objects degraded (36.105%); 602/27633 objects misplaced (2.179%); 3.9 MiB/s, 111 objects/s recovering 2023-10-01T11:08:26.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:25 smithi078 bash[75299]: cluster 2023-10-01T11:08:24.353011+0000 mgr.y (mgr.25072) 1533 : cluster 0 pgmap v1646: 99 pgs: 4 active+recovery_wait+degraded, 90 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 575 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 73 MiB/s wr, 2.91k op/s; 9977/27633 objects degraded (36.105%); 602/27633 objects misplaced (2.179%); 3.9 MiB/s, 111 objects/s recovering 2023-10-01T11:08:26.372 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:26.417 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:26.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:26.422 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:08:26.523 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:26.557 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:26.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:26.559 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:08:26.660 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:26.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:26.703 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:26.703 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:08:26.804 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:26.836 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:26.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:26.839 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:26.940 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:26.968 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:26.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:26.973 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:27.074 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:27.104 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:27.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:27.107 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:08:27.209 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:27.239 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:27.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:27.242 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:08:27.343 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:27.353 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T11:08:27.381 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:27.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:27.386 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:08:27.486 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:27.519 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:27.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:27.522 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:08:27.623 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:27.651 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:27.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:27.656 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:08:27.757 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:27.801 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:27.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:27.806 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:27.906 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:27.932 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:27.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:27.935 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:08:28.036 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:28.061 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:28.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:28.067 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:08:28.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:27 smithi173 bash[58816]: cluster 2023-10-01T11:08:26.353678+0000 mgr.y (mgr.25072) 1534 : cluster 0 pgmap v1647: 99 pgs: 4 active+recovery_wait+degraded, 90 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 575 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 62 MiB/s wr, 2.45k op/s; 9977/27633 objects degraded (36.105%); 602/27633 objects misplaced (2.179%); 3.3 MiB/s, 93 objects/s recovering 2023-10-01T11:08:28.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:27 smithi173 bash[58816]: cluster 2023-10-01T11:08:27.784594+0000 mon.a (mon.0) 1991 : cluster 3 Health check update: Degraded data redundancy: 9977/27633 objects degraded (36.105%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T11:08:28.168 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:28.177 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:27 smithi078 bash[75299]: cluster 2023-10-01T11:08:26.353678+0000 mgr.y (mgr.25072) 1534 : cluster 0 pgmap v1647: 99 pgs: 4 active+recovery_wait+degraded, 90 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 575 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 62 MiB/s wr, 2.45k op/s; 9977/27633 objects degraded (36.105%); 602/27633 objects misplaced (2.179%); 3.3 MiB/s, 93 objects/s recovering 2023-10-01T11:08:28.177 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:27 smithi078 bash[75299]: cluster 2023-10-01T11:08:27.784594+0000 mon.a (mon.0) 1991 : cluster 3 Health check update: Degraded data redundancy: 9977/27633 objects degraded (36.105%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T11:08:28.178 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:27 smithi078 bash[78274]: cluster 2023-10-01T11:08:26.353678+0000 mgr.y (mgr.25072) 1534 : cluster 0 pgmap v1647: 99 pgs: 4 active+recovery_wait+degraded, 90 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 575 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 62 MiB/s wr, 2.45k op/s; 9977/27633 objects degraded (36.105%); 602/27633 objects misplaced (2.179%); 3.3 MiB/s, 93 objects/s recovering 2023-10-01T11:08:28.178 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:27 smithi078 bash[78274]: cluster 2023-10-01T11:08:27.784594+0000 mon.a (mon.0) 1991 : cluster 3 Health check update: Degraded data redundancy: 9977/27633 objects degraded (36.105%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T11:08:28.197 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:28.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:28.204 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:08:28.304 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:28.333 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:28.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:28.340 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:28.440 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:28.466 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:28.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:28.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:08:28.569 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:28.593 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:28.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:28.597 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:28.698 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:28.725 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:28.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:28.727 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:08:28.828 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:28.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:28.861 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:28.861 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:08:28.877 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:08:28.962 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:28.996 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:29.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:29.002 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:29.103 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:29.138 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:29.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:29.142 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:08:29.243 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:29.272 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:29.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:29.275 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:29.375 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:29.402 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:29.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:29.404 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:08:29.504 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:29.530 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:29.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:29.538 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:08:29.639 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:29.683 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:29.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:29.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:08:29.788 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:29.817 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:29.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:29.822 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:08:29.900 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:08:29.901 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:08:29.923 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:29.958 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:29.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:29.963 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:08:29.984 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:29 smithi173 bash[58816]: cluster 2023-10-01T11:08:28.354362+0000 mgr.y (mgr.25072) 1535 : cluster 0 pgmap v1648: 99 pgs: 4 active+recovery_wait+degraded, 90 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 575 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 62 MiB/s wr, 2.45k op/s; 9977/27633 objects degraded (36.105%); 602/27633 objects misplaced (2.179%); 3.3 MiB/s, 93 objects/s recovering 2023-10-01T11:08:30.064 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:30.089 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:30.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:30.095 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:08:30.195 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:30.203 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:29 smithi078 bash[78274]: cluster 2023-10-01T11:08:28.354362+0000 mgr.y (mgr.25072) 1535 : cluster 0 pgmap v1648: 99 pgs: 4 active+recovery_wait+degraded, 90 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 575 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 62 MiB/s wr, 2.45k op/s; 9977/27633 objects degraded (36.105%); 602/27633 objects misplaced (2.179%); 3.3 MiB/s, 93 objects/s recovering 2023-10-01T11:08:30.204 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:29 smithi078 bash[75299]: cluster 2023-10-01T11:08:28.354362+0000 mgr.y (mgr.25072) 1535 : cluster 0 pgmap v1648: 99 pgs: 4 active+recovery_wait+degraded, 90 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 575 MiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 62 MiB/s wr, 2.45k op/s; 9977/27633 objects degraded (36.105%); 602/27633 objects misplaced (2.179%); 3.3 MiB/s, 93 objects/s recovering 2023-10-01T11:08:30.218 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:30.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:30.225 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:08:30.326 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:30.353 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:30.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:30.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:08:30.456 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:30.464 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:08:30 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:08:30] "GET /metrics HTTP/1.1" 200 36390 "" "Prometheus/2.33.4" 2023-10-01T11:08:30.481 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:30.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:30.483 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:08:30.584 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:30.606 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:30.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:30.609 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:08:30.709 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:30.733 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:30.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:30.736 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:08:30.836 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:30.866 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:30.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:30.871 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:08:30.972 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:31.028 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:31.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:31.030 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:08:31.131 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:31.156 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:31.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:31.158 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:08:31.258 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:31.283 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:31.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:31.288 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:31.388 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:31.422 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:31.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:31.427 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:08:31.527 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:31.571 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:31.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:31.575 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:08:31.676 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:31.700 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:31.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:31.704 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:31.805 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:31.837 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:31.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:31.844 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:31.929 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:08:31.945 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:31.969 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:31.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:31.971 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:32.071 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:32.098 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:32.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:32.102 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:32.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:31 smithi173 bash[58816]: cluster 2023-10-01T11:08:30.355117+0000 mgr.y (mgr.25072) 1536 : cluster 0 pgmap v1649: 99 pgs: 4 active+recovery_wait+degraded, 90 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 324 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 9921/15606 objects degraded (63.572%); 335/15606 objects misplaced (2.147%); 358 KiB/s, 26 objects/s recovering 2023-10-01T11:08:32.202 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:32.211 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:31 smithi078 bash[75299]: cluster 2023-10-01T11:08:30.355117+0000 mgr.y (mgr.25072) 1536 : cluster 0 pgmap v1649: 99 pgs: 4 active+recovery_wait+degraded, 90 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 324 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 9921/15606 objects degraded (63.572%); 335/15606 objects misplaced (2.147%); 358 KiB/s, 26 objects/s recovering 2023-10-01T11:08:32.211 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:31 smithi078 bash[78274]: cluster 2023-10-01T11:08:30.355117+0000 mgr.y (mgr.25072) 1536 : cluster 0 pgmap v1649: 99 pgs: 4 active+recovery_wait+degraded, 90 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 324 MiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 9921/15606 objects degraded (63.572%); 335/15606 objects misplaced (2.147%); 358 KiB/s, 26 objects/s recovering 2023-10-01T11:08:32.224 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:32.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:32.228 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:32.329 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:32.354 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:32.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:32.357 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:08:32.458 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:32.496 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:32.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:32.499 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:32.600 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:32.645 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:32.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:32.652 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:08:32.753 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:32.781 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:32.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:32.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:32.884 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:32.914 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:32.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:32.917 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:33.018 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:33.057 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:33.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:33.061 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:08:33.162 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:33.187 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:33.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:33.194 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:08:33.295 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:33.327 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:33.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:33.331 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:33.431 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:33.466 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:33.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:33.468 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:33.568 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:33.599 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:33.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:33.602 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:08:33.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:33 smithi173 bash[58816]: cluster 2023-10-01T11:08:32.355771+0000 mgr.y (mgr.25072) 1537 : cluster 0 pgmap v1650: 99 pgs: 4 active+recovery_wait+degraded, 90 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 324 MiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 9921/15606 objects degraded (63.572%); 335/15606 objects misplaced (2.147%); 358 KiB/s, 26 objects/s recovering 2023-10-01T11:08:33.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:33 smithi173 bash[58816]: audit 2023-10-01T11:08:32.473737+0000 mon.a (mon.0) 1992 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:08:33.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:33 smithi173 bash[58816]: audit 2023-10-01T11:08:32.475255+0000 mon.a (mon.0) 1993 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:08:33.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:33 smithi173 bash[58816]: cluster 2023-10-01T11:08:32.785924+0000 mon.a (mon.0) 1994 : cluster 3 Health check update: Degraded data redundancy: 9921/15606 objects degraded (63.572%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T11:08:33.703 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:33.746 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:33.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:33.749 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:08:33.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:33 smithi078 bash[75299]: cluster 2023-10-01T11:08:32.355771+0000 mgr.y (mgr.25072) 1537 : cluster 0 pgmap v1650: 99 pgs: 4 active+recovery_wait+degraded, 90 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 324 MiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 9921/15606 objects degraded (63.572%); 335/15606 objects misplaced (2.147%); 358 KiB/s, 26 objects/s recovering 2023-10-01T11:08:33.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:33 smithi078 bash[75299]: audit 2023-10-01T11:08:32.473737+0000 mon.a (mon.0) 1992 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:08:33.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:33 smithi078 bash[75299]: audit 2023-10-01T11:08:32.475255+0000 mon.a (mon.0) 1993 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:08:33.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:33 smithi078 bash[75299]: cluster 2023-10-01T11:08:32.785924+0000 mon.a (mon.0) 1994 : cluster 3 Health check update: Degraded data redundancy: 9921/15606 objects degraded (63.572%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T11:08:33.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:33 smithi078 bash[78274]: cluster 2023-10-01T11:08:32.355771+0000 mgr.y (mgr.25072) 1537 : cluster 0 pgmap v1650: 99 pgs: 4 active+recovery_wait+degraded, 90 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 324 MiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 9921/15606 objects degraded (63.572%); 335/15606 objects misplaced (2.147%); 358 KiB/s, 26 objects/s recovering 2023-10-01T11:08:33.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:33 smithi078 bash[78274]: audit 2023-10-01T11:08:32.473737+0000 mon.a (mon.0) 1992 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:08:33.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:33 smithi078 bash[78274]: audit 2023-10-01T11:08:32.475255+0000 mon.a (mon.0) 1993 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:08:33.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:33 smithi078 bash[78274]: cluster 2023-10-01T11:08:32.785924+0000 mon.a (mon.0) 1994 : cluster 3 Health check update: Degraded data redundancy: 9921/15606 objects degraded (63.572%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T11:08:33.850 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:33.878 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:33.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:33.881 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:08:33.982 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:34.009 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:34.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:34.015 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:08:34.116 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:34.142 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:34.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:34.147 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:34.248 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:34.274 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:34.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:34.280 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:34.381 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:34.407 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:34.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:34.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:34.512 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:34.549 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:34.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:34.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:34.656 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:34.681 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:34.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:34.688 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:08:34.789 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:34.810 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:34.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:34.812 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:08:34.913 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:34.935 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:34.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:34.938 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:08:35.039 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:35.068 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:35.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:35.070 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:35.171 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:35.197 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:35.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:35.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:35.205 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Removed 79555 objects 2023-10-01T11:08:35.205 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Clean up completed and total clean up time :254.753 2023-10-01T11:08:35.300 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:35.325 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:35.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:35.328 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:08:35.429 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:35.456 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:35.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:35.459 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:08:35.559 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:35.575 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:35 smithi078 bash[78274]: cluster 2023-10-01T11:08:34.356982+0000 mgr.y (mgr.25072) 1538 : cluster 0 pgmap v1651: 99 pgs: 4 active+recovery_wait+degraded, 90 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 71 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 9659/3471 objects degraded (278.277%); 79/3471 objects misplaced (2.276%); 1.7 MiB/s, 54 objects/s recovering 2023-10-01T11:08:35.575 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:35 smithi078 bash[75299]: cluster 2023-10-01T11:08:34.356982+0000 mgr.y (mgr.25072) 1538 : cluster 0 pgmap v1651: 99 pgs: 4 active+recovery_wait+degraded, 90 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 71 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 9659/3471 objects degraded (278.277%); 79/3471 objects misplaced (2.276%); 1.7 MiB/s, 54 objects/s recovering 2023-10-01T11:08:35.584 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:35.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:35.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:35.688 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:35.698 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:35 smithi173 bash[58816]: cluster 2023-10-01T11:08:34.356982+0000 mgr.y (mgr.25072) 1538 : cluster 0 pgmap v1651: 99 pgs: 4 active+recovery_wait+degraded, 90 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 71 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 9659/3471 objects degraded (278.277%); 79/3471 objects misplaced (2.276%); 1.7 MiB/s, 54 objects/s recovering 2023-10-01T11:08:35.712 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:35.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:35.714 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:08:35.815 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:35.838 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:35.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:35.841 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:08:35.942 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:35.971 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:35.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:35.973 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:08:36.074 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:36.099 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:36.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:36.102 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:08:36.202 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:36.229 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:36.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:36.231 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:36.332 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:36.356 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:36.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:36.359 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:08:36.460 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:36.486 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:36.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:36.488 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:36.589 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:36.611 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:36.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:36.613 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:08:36.714 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:37.143 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:37.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:37.145 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:37.245 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:37.271 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:37.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:37.273 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:08:37.373 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:37.398 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:37.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:37.400 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:37.501 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:37.526 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:37.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:37.528 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:08:37.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:37 smithi078 bash[78274]: cluster 2023-10-01T11:08:36.357963+0000 mgr.y (mgr.25072) 1539 : cluster 0 pgmap v1652: 99 pgs: 4 active+recovery_wait+degraded, 90 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 71 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 9659/3471 objects degraded (278.277%); 79/3471 objects misplaced (2.276%); 1.7 MiB/s, 43 objects/s recovering 2023-10-01T11:08:37.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:37 smithi078 bash[75299]: cluster 2023-10-01T11:08:36.357963+0000 mgr.y (mgr.25072) 1539 : cluster 0 pgmap v1652: 99 pgs: 4 active+recovery_wait+degraded, 90 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 71 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 9659/3471 objects degraded (278.277%); 79/3471 objects misplaced (2.276%); 1.7 MiB/s, 43 objects/s recovering 2023-10-01T11:08:37.630 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:37.656 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:37.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:37.660 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:37.760 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:37.773 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:37 smithi173 bash[58816]: cluster 2023-10-01T11:08:36.357963+0000 mgr.y (mgr.25072) 1539 : cluster 0 pgmap v1652: 99 pgs: 4 active+recovery_wait+degraded, 90 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 71 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 9659/3471 objects degraded (278.277%); 79/3471 objects misplaced (2.276%); 1.7 MiB/s, 43 objects/s recovering 2023-10-01T11:08:37.785 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:37.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:37.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:08:37.867 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_8 2023-10-01T11:08:37.868 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_8 unique_pool_8 --yes-i-really-really-mean-it 2023-10-01T11:08:37.888 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:37.915 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:37.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:37.918 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:08:38.019 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:38.045 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:38.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:38.048 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:38.148 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:38.172 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:38.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:38.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:08:38.275 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:38.302 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:38.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:38.304 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:38.405 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:38.437 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:38.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:38.440 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:08:38.541 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:38.564 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:38.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:38.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:08:38.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:38 smithi078 bash[75299]: cluster 2023-10-01T11:08:37.786944+0000 mon.a (mon.0) 1995 : cluster 3 Health check update: Degraded data redundancy: 9659/3471 objects degraded (278.277%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T11:08:38.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:38 smithi078 bash[75299]: audit 2023-10-01T11:08:38.444454+0000 mon.a (mon.0) 1996 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T11:08:38.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:38 smithi078 bash[78274]: cluster 2023-10-01T11:08:37.786944+0000 mon.a (mon.0) 1995 : cluster 3 Health check update: Degraded data redundancy: 9659/3471 objects degraded (278.277%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T11:08:38.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:38 smithi078 bash[78274]: audit 2023-10-01T11:08:38.444454+0000 mon.a (mon.0) 1996 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T11:08:38.667 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:38.687 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:38.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:38.690 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:08:38.790 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:38.803 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:38 smithi173 bash[58816]: cluster 2023-10-01T11:08:37.786944+0000 mon.a (mon.0) 1995 : cluster 3 Health check update: Degraded data redundancy: 9659/3471 objects degraded (278.277%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T11:08:38.803 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:38 smithi173 bash[58816]: audit 2023-10-01T11:08:38.444454+0000 mon.a (mon.0) 1996 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T11:08:38.818 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:38.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:38.821 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:08:38.921 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:38.948 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:38.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:38.950 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:08:39.051 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:39.079 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:39.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:39.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:39.182 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:39.210 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:39.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:39.212 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:08:39.313 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:39.342 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:39.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:39.345 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:39.446 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:39.472 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:39.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:39.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:08:39.575 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:39.603 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:39.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:39.606 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:39.707 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:39.738 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:39.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:39.740 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:08:39.841 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:39.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:39 smithi173 bash[58816]: cluster 2023-10-01T11:08:38.358729+0000 mgr.y (mgr.25072) 1540 : cluster 0 pgmap v1653: 99 pgs: 4 active+recovery_wait+degraded, 90 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 71 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 9659/3471 objects degraded (278.277%); 79/3471 objects misplaced (2.276%); 1.7 MiB/s, 43 objects/s recovering 2023-10-01T11:08:39.872 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:39.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:39.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:08:39.975 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:40.010 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:40.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:40.013 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:08:40.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:39 smithi078 bash[75299]: cluster 2023-10-01T11:08:38.358729+0000 mgr.y (mgr.25072) 1540 : cluster 0 pgmap v1653: 99 pgs: 4 active+recovery_wait+degraded, 90 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 71 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 9659/3471 objects degraded (278.277%); 79/3471 objects misplaced (2.276%); 1.7 MiB/s, 43 objects/s recovering 2023-10-01T11:08:40.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:39 smithi078 bash[78274]: cluster 2023-10-01T11:08:38.358729+0000 mgr.y (mgr.25072) 1540 : cluster 0 pgmap v1653: 99 pgs: 4 active+recovery_wait+degraded, 90 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 71 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 9659/3471 objects degraded (278.277%); 79/3471 objects misplaced (2.276%); 1.7 MiB/s, 43 objects/s recovering 2023-10-01T11:08:40.113 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:40.141 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:40.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:40.143 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:08:40.244 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:40.274 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:40.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:40.277 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:08:40.377 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:40.407 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:40.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:40.411 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:40.511 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:40.523 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:08:40 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:08:40] "GET /metrics HTTP/1.1" 200 36396 "" "Prometheus/2.33.4" 2023-10-01T11:08:40.537 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:40.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:40.540 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:08:40.641 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:40.672 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:40.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:40.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:08:40.776 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:40.808 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:40.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:40.810 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:08:40.912 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:40.941 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:40.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:40.943 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:41.044 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:41.077 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:41.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:41.080 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:08:41.181 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:41.206 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:41.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:41.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:08:41.310 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:41.334 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:41.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:41.338 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:08:41.439 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:41.473 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:41.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:41.476 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:08:41.577 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:41.601 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:41.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:41.603 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:08:41.704 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:41.734 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:41.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:41.736 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:08:41.837 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:41.867 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:41.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:41.869 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:41.970 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:41.996 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:41.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:41.998 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:08:42.087 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:41 smithi078 bash[78274]: cluster 2023-10-01T11:08:40.360136+0000 mgr.y (mgr.25072) 1541 : cluster 0 pgmap v1654: 99 pgs: 4 active+recovery_wait+degraded, 90 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 7.9 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 765 op/s; 9599/426 objects degraded (2253.286%); 15/426 objects misplaced (3.521%); 2.0 MiB/s, 57 objects/s recovering 2023-10-01T11:08:42.087 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:41 smithi078 bash[75299]: cluster 2023-10-01T11:08:40.360136+0000 mgr.y (mgr.25072) 1541 : cluster 0 pgmap v1654: 99 pgs: 4 active+recovery_wait+degraded, 90 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 7.9 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 765 op/s; 9599/426 objects degraded (2253.286%); 15/426 objects misplaced (3.521%); 2.0 MiB/s, 57 objects/s recovering 2023-10-01T11:08:42.099 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:42.112 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:41 smithi173 bash[58816]: cluster 2023-10-01T11:08:40.360136+0000 mgr.y (mgr.25072) 1541 : cluster 0 pgmap v1654: 99 pgs: 4 active+recovery_wait+degraded, 90 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 7.9 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 765 op/s; 9599/426 objects degraded (2253.286%); 15/426 objects misplaced (3.521%); 2.0 MiB/s, 57 objects/s recovering 2023-10-01T11:08:42.128 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:42.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:42.131 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:08:42.232 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:42.255 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:42.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:42.258 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:42.358 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:42.390 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:42.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:42.393 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:42.494 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:42.526 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:42.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:42.528 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:42.629 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:42.658 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:42.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:42.660 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:08:42.761 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:42.788 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:42.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:42.791 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:08:42.892 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:42.906 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:42 smithi078 bash[78274]: cluster 2023-10-01T11:08:42.788223+0000 mon.a (mon.0) 1997 : cluster 3 Health check update: Degraded data redundancy: 9599/426 objects degraded (2253.286%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T11:08:42.906 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:42 smithi078 bash[75299]: cluster 2023-10-01T11:08:42.788223+0000 mon.a (mon.0) 1997 : cluster 3 Health check update: Degraded data redundancy: 9599/426 objects degraded (2253.286%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T11:08:42.920 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:42.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:42.924 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:43.024 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:43.035 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:42 smithi173 bash[58816]: cluster 2023-10-01T11:08:42.788223+0000 mon.a (mon.0) 1997 : cluster 3 Health check update: Degraded data redundancy: 9599/426 objects degraded (2253.286%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-10-01T11:08:43.053 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:43.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:43.056 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:08:43.157 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:43.185 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:43.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:43.188 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:08:43.289 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:43.321 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:43.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:43.324 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:43.425 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:43.450 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:43.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:43.452 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:08:43.553 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:43.581 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:43.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:43.583 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:08:43.684 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:43.709 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:43.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:43.711 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:43.812 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:43.833 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:43.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:43.835 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:08:43.936 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:43.945 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:43 smithi078 bash[75299]: cluster 2023-10-01T11:08:42.360727+0000 mgr.y (mgr.25072) 1542 : cluster 0 pgmap v1655: 99 pgs: 4 active+recovery_wait+degraded, 90 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 7.9 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 431 op/s; 9599/426 objects degraded (2253.286%); 15/426 objects misplaced (3.521%); 1.7 MiB/s, 44 objects/s recovering 2023-10-01T11:08:43.945 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:43 smithi078 bash[78274]: cluster 2023-10-01T11:08:42.360727+0000 mgr.y (mgr.25072) 1542 : cluster 0 pgmap v1655: 99 pgs: 4 active+recovery_wait+degraded, 90 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 7.9 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 431 op/s; 9599/426 objects degraded (2253.286%); 15/426 objects misplaced (3.521%); 1.7 MiB/s, 44 objects/s recovering 2023-10-01T11:08:43.959 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:43.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:43.962 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:08:44.062 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:44.084 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:44.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:44.086 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:08:44.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:43 smithi173 bash[58816]: cluster 2023-10-01T11:08:42.360727+0000 mgr.y (mgr.25072) 1542 : cluster 0 pgmap v1655: 99 pgs: 4 active+recovery_wait+degraded, 90 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 7.9 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 431 op/s; 9599/426 objects degraded (2253.286%); 15/426 objects misplaced (3.521%); 1.7 MiB/s, 44 objects/s recovering 2023-10-01T11:08:44.187 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:44.207 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:44.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:44.209 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:44.310 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:44.339 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:44.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:44.341 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:08:44.442 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:44.465 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:44.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:44.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:08:44.568 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:44.595 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:44.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:44.597 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:44.697 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:44.722 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:44.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:44.724 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:08:44.825 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:44.853 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:44.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:44.855 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:08:44.956 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:44.981 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:44.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:44.984 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:08:45.084 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:45.112 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:45.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:45.115 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:08:45.215 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:45.244 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:45.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:45.247 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:08:45.347 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:45.375 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:45.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:45.379 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:08:45.480 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:45.499 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T11:08:45.502 INFO:teuthology.orchestra.run.smithi078.stdout:ERROR: (22) Invalid argument 2023-10-01T11:08:45.502 INFO:teuthology.orchestra.run.smithi078.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T11:08:45.508 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:45.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:45.511 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:08:45.523 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T11:08:45.547 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T11:08:45.570 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T11:08:45.611 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:45.612 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T11:08:45.637 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:45.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:45.639 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:08:45.650 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T11:08:45.681 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T11:08:45.719 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T11:08:45.721 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T11:08:45.721 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T11:08:45.721 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T11:08:45.722 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T11:08:45.722 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T11:08:45.722 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T11:08:45.722 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T11:08:45.722 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T11:08:45.740 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:45.760 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:45.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:45.762 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:08:45.856 INFO:teuthology.orchestra.run.smithi078.stderr:set osd.1 primary-affinity to 0.659718 (8432352) 2023-10-01T11:08:45.863 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:45.886 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:45.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:45.888 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:08:45.989 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:46.014 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:46.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:46.016 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:46.086 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:45 smithi078 bash[75299]: cluster 2023-10-01T11:08:44.361476+0000 mgr.y (mgr.25072) 1543 : cluster 0 pgmap v1656: 99 pgs: 4 active+recovery_wait+degraded, 90 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 7.9 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 431 op/s; 9599/426 objects degraded (2253.286%); 15/426 objects misplaced (3.521%); 1.7 MiB/s, 54 objects/s recovering 2023-10-01T11:08:46.086 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:45 smithi078 bash[75299]: audit 2023-10-01T11:08:44.834856+0000 mon.a (mon.0) 1998 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:08:46.086 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:45 smithi078 bash[75299]: audit 2023-10-01T11:08:44.843573+0000 mon.a (mon.0) 1999 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:08:46.086 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:45 smithi078 bash[75299]: audit 2023-10-01T11:08:44.906611+0000 mon.c (mon.2) 232 : audit 1 from='client.? 172.21.15.78:0/1109692154' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.6597183621581236}]: dispatch 2023-10-01T11:08:46.086 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:45 smithi078 bash[75299]: audit 2023-10-01T11:08:44.907098+0000 mon.a (mon.0) 2000 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.6597183621581236}]: dispatch 2023-10-01T11:08:46.086 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:45 smithi078 bash[75299]: audit 2023-10-01T11:08:45.458873+0000 mon.c (mon.2) 233 : audit 0 from='client.? 172.21.15.78:0/1424667761' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:08:46.086 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:45 smithi078 bash[75299]: audit 2023-10-01T11:08:45.734030+0000 mon.a (mon.0) 2001 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:08:46.086 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:45 smithi078 bash[75299]: audit 2023-10-01T11:08:45.741654+0000 mon.a (mon.0) 2002 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:08:46.117 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:46.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:45 smithi173 bash[58816]: cluster 2023-10-01T11:08:44.361476+0000 mgr.y (mgr.25072) 1543 : cluster 0 pgmap v1656: 99 pgs: 4 active+recovery_wait+degraded, 90 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 7.9 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 431 op/s; 9599/426 objects degraded (2253.286%); 15/426 objects misplaced (3.521%); 1.7 MiB/s, 54 objects/s recovering 2023-10-01T11:08:46.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:45 smithi173 bash[58816]: audit 2023-10-01T11:08:44.834856+0000 mon.a (mon.0) 1998 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:08:46.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:45 smithi173 bash[58816]: audit 2023-10-01T11:08:44.843573+0000 mon.a (mon.0) 1999 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:08:46.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:45 smithi173 bash[58816]: audit 2023-10-01T11:08:44.906611+0000 mon.c (mon.2) 232 : audit 1 from='client.? 172.21.15.78:0/1109692154' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.6597183621581236}]: dispatch 2023-10-01T11:08:46.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:45 smithi173 bash[58816]: audit 2023-10-01T11:08:44.907098+0000 mon.a (mon.0) 2000 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.6597183621581236}]: dispatch 2023-10-01T11:08:46.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:45 smithi173 bash[58816]: audit 2023-10-01T11:08:45.458873+0000 mon.c (mon.2) 233 : audit 0 from='client.? 172.21.15.78:0/1424667761' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:08:46.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:45 smithi173 bash[58816]: audit 2023-10-01T11:08:45.734030+0000 mon.a (mon.0) 2001 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:08:46.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:45 smithi173 bash[58816]: audit 2023-10-01T11:08:45.741654+0000 mon.a (mon.0) 2002 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:08:46.146 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:46.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:46.148 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:46.249 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:46.278 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:46.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:46.281 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:08:46.382 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:46.414 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:46.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:46.417 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:08:46.518 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:46.551 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:46.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:46.554 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:08:46.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:46 smithi078 bash[78274]: cluster 2023-10-01T11:08:44.361476+0000 mgr.y (mgr.25072) 1543 : cluster 0 pgmap v1656: 99 pgs: 4 active+recovery_wait+degraded, 90 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 7.9 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 431 op/s; 9599/426 objects degraded (2253.286%); 15/426 objects misplaced (3.521%); 1.7 MiB/s, 54 objects/s recovering 2023-10-01T11:08:46.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:46 smithi078 bash[78274]: audit 2023-10-01T11:08:44.834856+0000 mon.a (mon.0) 1998 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:08:46.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:46 smithi078 bash[78274]: audit 2023-10-01T11:08:44.843573+0000 mon.a (mon.0) 1999 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:08:46.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:46 smithi078 bash[78274]: audit 2023-10-01T11:08:44.906611+0000 mon.c (mon.2) 232 : audit 1 from='client.? 172.21.15.78:0/1109692154' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.6597183621581236}]: dispatch 2023-10-01T11:08:46.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:46 smithi078 bash[78274]: audit 2023-10-01T11:08:44.907098+0000 mon.a (mon.0) 2000 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.6597183621581236}]: dispatch 2023-10-01T11:08:46.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:46 smithi078 bash[78274]: audit 2023-10-01T11:08:45.458873+0000 mon.c (mon.2) 233 : audit 0 from='client.? 172.21.15.78:0/1424667761' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:08:46.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:46 smithi078 bash[78274]: audit 2023-10-01T11:08:45.734030+0000 mon.a (mon.0) 2001 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:08:46.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:46 smithi078 bash[78274]: audit 2023-10-01T11:08:45.741654+0000 mon.a (mon.0) 2002 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:08:46.655 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:46.681 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:46.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:46.684 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:46.785 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:46.811 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:46.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:46.815 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:08:46.874 INFO:teuthology.orchestra.run.smithi078.stderr:noscrub is set 2023-10-01T11:08:46.911 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-01T11:08:46.916 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:46.980 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:46.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:46.982 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:08:47.083 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:47.097 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:46 smithi173 bash[58816]: cluster 2023-10-01T11:08:45.843054+0000 osd.5 (osd.5) 374 : cluster 3 failed to encode map e352 with expected crc 2023-10-01T11:08:47.097 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:46 smithi173 bash[58816]: audit 2023-10-01T11:08:45.843852+0000 mon.a (mon.0) 2003 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.6597183621581236}]': finished 2023-10-01T11:08:47.097 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:46 smithi173 bash[58816]: cluster 2023-10-01T11:08:45.843932+0000 mon.a (mon.0) 2004 : cluster 0 osdmap e352: 8 total, 8 up, 8 in 2023-10-01T11:08:47.097 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:46 smithi173 bash[58816]: cluster 2023-10-01T11:08:45.845631+0000 osd.7 (osd.7) 394 : cluster 3 failed to encode map e352 with expected crc 2023-10-01T11:08:47.097 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:46 smithi173 bash[58816]: cluster 2023-10-01T11:08:45.848083+0000 osd.7 (osd.7) 395 : cluster 3 failed to encode map e352 with expected crc 2023-10-01T11:08:47.097 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:46 smithi173 bash[58816]: cluster 2023-10-01T11:08:45.848797+0000 osd.6 (osd.6) 337 : cluster 3 failed to encode map e352 with expected crc 2023-10-01T11:08:47.098 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:46 smithi173 bash[58816]: audit 2023-10-01T11:08:45.855363+0000 mon.c (mon.2) 234 : audit 1 from='client.? 172.21.15.78:0/31240127' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T11:08:47.098 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:46 smithi173 bash[58816]: audit 2023-10-01T11:08:45.855901+0000 mon.a (mon.0) 2005 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T11:08:47.113 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:47.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:47.115 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:47.209 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:46 smithi078 bash[75299]: cluster 2023-10-01T11:08:45.843054+0000 osd.5 (osd.5) 374 : cluster 3 failed to encode map e352 with expected crc 2023-10-01T11:08:47.209 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:46 smithi078 bash[75299]: audit 2023-10-01T11:08:45.843852+0000 mon.a (mon.0) 2003 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.6597183621581236}]': finished 2023-10-01T11:08:47.209 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:46 smithi078 bash[75299]: cluster 2023-10-01T11:08:45.843932+0000 mon.a (mon.0) 2004 : cluster 0 osdmap e352: 8 total, 8 up, 8 in 2023-10-01T11:08:47.209 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:46 smithi078 bash[75299]: cluster 2023-10-01T11:08:45.845631+0000 osd.7 (osd.7) 394 : cluster 3 failed to encode map e352 with expected crc 2023-10-01T11:08:47.209 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:46 smithi078 bash[75299]: cluster 2023-10-01T11:08:45.848083+0000 osd.7 (osd.7) 395 : cluster 3 failed to encode map e352 with expected crc 2023-10-01T11:08:47.209 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:46 smithi078 bash[75299]: cluster 2023-10-01T11:08:45.848797+0000 osd.6 (osd.6) 337 : cluster 3 failed to encode map e352 with expected crc 2023-10-01T11:08:47.209 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:46 smithi078 bash[75299]: audit 2023-10-01T11:08:45.855363+0000 mon.c (mon.2) 234 : audit 1 from='client.? 172.21.15.78:0/31240127' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T11:08:47.209 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:46 smithi078 bash[75299]: audit 2023-10-01T11:08:45.855901+0000 mon.a (mon.0) 2005 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T11:08:47.210 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:46 smithi078 bash[78274]: cluster 2023-10-01T11:08:45.843054+0000 osd.5 (osd.5) 374 : cluster 3 failed to encode map e352 with expected crc 2023-10-01T11:08:47.210 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:46 smithi078 bash[78274]: audit 2023-10-01T11:08:45.843852+0000 mon.a (mon.0) 2003 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.6597183621581236}]': finished 2023-10-01T11:08:47.210 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:46 smithi078 bash[78274]: cluster 2023-10-01T11:08:45.843932+0000 mon.a (mon.0) 2004 : cluster 0 osdmap e352: 8 total, 8 up, 8 in 2023-10-01T11:08:47.210 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:46 smithi078 bash[78274]: cluster 2023-10-01T11:08:45.845631+0000 osd.7 (osd.7) 394 : cluster 3 failed to encode map e352 with expected crc 2023-10-01T11:08:47.210 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:46 smithi078 bash[78274]: cluster 2023-10-01T11:08:45.848083+0000 osd.7 (osd.7) 395 : cluster 3 failed to encode map e352 with expected crc 2023-10-01T11:08:47.210 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:46 smithi078 bash[78274]: cluster 2023-10-01T11:08:45.848797+0000 osd.6 (osd.6) 337 : cluster 3 failed to encode map e352 with expected crc 2023-10-01T11:08:47.210 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:46 smithi078 bash[78274]: audit 2023-10-01T11:08:45.855363+0000 mon.c (mon.2) 234 : audit 1 from='client.? 172.21.15.78:0/31240127' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T11:08:47.210 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:46 smithi078 bash[78274]: audit 2023-10-01T11:08:45.855901+0000 mon.a (mon.0) 2005 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T11:08:47.216 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:47.244 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:47.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:47.247 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:08:47.347 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:47.373 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:47.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:47.376 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:08:47.477 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:47.505 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:47.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:47.508 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:08:47.608 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:47.633 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:47.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:47.636 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:47.736 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:47.762 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:47.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:47.764 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:08:47.865 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:47.894 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:47.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:47.898 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:08:47.998 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:48.011 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:47 smithi173 bash[58816]: cluster 2023-10-01T11:08:46.362053+0000 mgr.y (mgr.25072) 1544 : cluster 0 pgmap v1658: 99 pgs: 4 active+recovery_wait+degraded, 90 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 7.9 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 113 op/s; 9599/426 objects degraded (2253.286%); 15/426 objects misplaced (3.521%); 384 KiB/s, 28 objects/s recovering 2023-10-01T11:08:48.011 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:47 smithi173 bash[58816]: cluster 2023-10-01T11:08:46.842954+0000 mon.a (mon.0) 2006 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:08:48.011 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:47 smithi173 bash[58816]: cluster 2023-10-01T11:08:46.853957+0000 osd.7 (osd.7) 396 : cluster 3 failed to encode map e353 with expected crc 2023-10-01T11:08:48.011 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:47 smithi173 bash[58816]: audit 2023-10-01T11:08:46.856363+0000 mon.a (mon.0) 2007 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T11:08:48.012 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:47 smithi173 bash[58816]: cluster 2023-10-01T11:08:46.856471+0000 mon.a (mon.0) 2008 : cluster 0 osdmap e353: 8 total, 8 up, 8 in 2023-10-01T11:08:48.012 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:47 smithi173 bash[58816]: cluster 2023-10-01T11:08:46.860252+0000 osd.5 (osd.5) 375 : cluster 3 failed to encode map e353 with expected crc 2023-10-01T11:08:48.012 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:47 smithi173 bash[58816]: cluster 2023-10-01T11:08:46.860720+0000 osd.6 (osd.6) 338 : cluster 3 failed to encode map e353 with expected crc 2023-10-01T11:08:48.012 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:47 smithi173 bash[58816]: audit 2023-10-01T11:08:47.472480+0000 mon.a (mon.0) 2009 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:08:48.012 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:47 smithi173 bash[58816]: audit 2023-10-01T11:08:47.473873+0000 mon.a (mon.0) 2010 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:08:48.028 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:48.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:48.031 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:08:48.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:47 smithi078 bash[78274]: cluster 2023-10-01T11:08:46.362053+0000 mgr.y (mgr.25072) 1544 : cluster 0 pgmap v1658: 99 pgs: 4 active+recovery_wait+degraded, 90 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 7.9 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 113 op/s; 9599/426 objects degraded (2253.286%); 15/426 objects misplaced (3.521%); 384 KiB/s, 28 objects/s recovering 2023-10-01T11:08:48.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:47 smithi078 bash[78274]: cluster 2023-10-01T11:08:46.842954+0000 mon.a (mon.0) 2006 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:08:48.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:47 smithi078 bash[78274]: cluster 2023-10-01T11:08:46.853957+0000 osd.7 (osd.7) 396 : cluster 3 failed to encode map e353 with expected crc 2023-10-01T11:08:48.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:47 smithi078 bash[78274]: audit 2023-10-01T11:08:46.856363+0000 mon.a (mon.0) 2007 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T11:08:48.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:47 smithi078 bash[78274]: cluster 2023-10-01T11:08:46.856471+0000 mon.a (mon.0) 2008 : cluster 0 osdmap e353: 8 total, 8 up, 8 in 2023-10-01T11:08:48.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:47 smithi078 bash[78274]: cluster 2023-10-01T11:08:46.860252+0000 osd.5 (osd.5) 375 : cluster 3 failed to encode map e353 with expected crc 2023-10-01T11:08:48.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:47 smithi078 bash[78274]: cluster 2023-10-01T11:08:46.860720+0000 osd.6 (osd.6) 338 : cluster 3 failed to encode map e353 with expected crc 2023-10-01T11:08:48.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:47 smithi078 bash[78274]: audit 2023-10-01T11:08:47.472480+0000 mon.a (mon.0) 2009 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:08:48.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:47 smithi078 bash[78274]: audit 2023-10-01T11:08:47.473873+0000 mon.a (mon.0) 2010 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:08:48.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:47 smithi078 bash[75299]: cluster 2023-10-01T11:08:46.362053+0000 mgr.y (mgr.25072) 1544 : cluster 0 pgmap v1658: 99 pgs: 4 active+recovery_wait+degraded, 90 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 7.9 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 113 op/s; 9599/426 objects degraded (2253.286%); 15/426 objects misplaced (3.521%); 384 KiB/s, 28 objects/s recovering 2023-10-01T11:08:48.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:47 smithi078 bash[75299]: cluster 2023-10-01T11:08:46.842954+0000 mon.a (mon.0) 2006 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:08:48.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:47 smithi078 bash[75299]: cluster 2023-10-01T11:08:46.853957+0000 osd.7 (osd.7) 396 : cluster 3 failed to encode map e353 with expected crc 2023-10-01T11:08:48.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:47 smithi078 bash[75299]: audit 2023-10-01T11:08:46.856363+0000 mon.a (mon.0) 2007 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T11:08:48.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:47 smithi078 bash[75299]: cluster 2023-10-01T11:08:46.856471+0000 mon.a (mon.0) 2008 : cluster 0 osdmap e353: 8 total, 8 up, 8 in 2023-10-01T11:08:48.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:47 smithi078 bash[75299]: cluster 2023-10-01T11:08:46.860252+0000 osd.5 (osd.5) 375 : cluster 3 failed to encode map e353 with expected crc 2023-10-01T11:08:48.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:47 smithi078 bash[75299]: cluster 2023-10-01T11:08:46.860720+0000 osd.6 (osd.6) 338 : cluster 3 failed to encode map e353 with expected crc 2023-10-01T11:08:48.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:47 smithi078 bash[75299]: audit 2023-10-01T11:08:47.472480+0000 mon.a (mon.0) 2009 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:08:48.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:47 smithi078 bash[75299]: audit 2023-10-01T11:08:47.473873+0000 mon.a (mon.0) 2010 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:08:48.131 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:48.156 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:48.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:48.158 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:08:48.259 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:48.284 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:48.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:48.286 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:08:48.387 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:48.413 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:48.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:48.415 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:08:48.516 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:48.541 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:48.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:48.544 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:48.645 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:48.673 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:48.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:48.675 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:08:48.776 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:48.805 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:48.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:48.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:08:48.908 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:48.940 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:48.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:48.944 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:08:49.045 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:49.076 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:49.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:49.079 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:08:49.180 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:49.212 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:49.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:49.216 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:08:49.318 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:49.331 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:08:49.350 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:49.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:49.352 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:08:49.453 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:49.510 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:49.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:49.512 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:08:49.613 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:49.642 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:49.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:49.645 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:08:49.746 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:49.777 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:49.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:49.780 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:49.880 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:49.912 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:49.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:49.915 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:50.016 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:50.039 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:50.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:50.042 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:08:50.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:49 smithi173 bash[58816]: cluster 2023-10-01T11:08:48.362902+0000 mgr.y (mgr.25072) 1545 : cluster 0 pgmap v1660: 99 pgs: 4 active+recovery_wait+degraded, 90 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 7.9 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 9599/426 objects degraded (2253.286%); 15/426 objects misplaced (3.521%); 0 B/s, 14 objects/s recovering 2023-10-01T11:08:50.143 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:50.154 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:49 smithi078 bash[78274]: cluster 2023-10-01T11:08:48.362902+0000 mgr.y (mgr.25072) 1545 : cluster 0 pgmap v1660: 99 pgs: 4 active+recovery_wait+degraded, 90 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 7.9 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 9599/426 objects degraded (2253.286%); 15/426 objects misplaced (3.521%); 0 B/s, 14 objects/s recovering 2023-10-01T11:08:50.155 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:49 smithi078 bash[75299]: cluster 2023-10-01T11:08:48.362902+0000 mgr.y (mgr.25072) 1545 : cluster 0 pgmap v1660: 99 pgs: 4 active+recovery_wait+degraded, 90 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 7.9 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 9599/426 objects degraded (2253.286%); 15/426 objects misplaced (3.521%); 0 B/s, 14 objects/s recovering 2023-10-01T11:08:50.169 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:50.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:50.173 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:08:50.274 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:50.322 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:50.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:50.324 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:08:50.425 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:50.434 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:08:50 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:08:50] "GET /metrics HTTP/1.1" 200 36409 "" "Prometheus/2.33.4" 2023-10-01T11:08:50.467 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:50.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:50.470 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:08:50.570 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:50.592 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:50.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:50.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:50.696 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:50.720 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:50.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:50.723 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:08:50.823 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:50.849 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:50.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:50.851 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:08:50.951 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T11:08:50.952 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:50.965 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T11:08:50.977 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:50.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:50.979 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:08:51.079 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:51.103 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:51.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:51.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:08:51.209 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:51.240 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:51.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:51.243 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:08:51.343 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:51.367 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:51.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:51.368 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:08:51.469 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:51.492 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:51.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:51.493 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:08:51.594 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:51.615 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:51.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:51.617 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:08:51.718 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:51.750 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:51.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:51.752 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:08:51.853 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:51.875 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:51.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:51.878 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:51.979 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:51.990 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:51 smithi078 bash[75299]: cluster 2023-10-01T11:08:50.363680+0000 mgr.y (mgr.25072) 1546 : cluster 0 pgmap v1661: 99 pgs: 4 active+recovery_wait+degraded, 90 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 7.9 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 9599/426 objects degraded (2253.286%); 15/426 objects misplaced (3.521%); 0 B/s, 29 objects/s recovering 2023-10-01T11:08:51.991 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:51 smithi078 bash[75299]: audit 2023-10-01T11:08:50.806516+0000 mon.a (mon.0) 2011 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:08:51.991 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:51 smithi078 bash[75299]: audit 2023-10-01T11:08:50.814743+0000 mon.a (mon.0) 2012 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:08:51.991 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:51 smithi078 bash[78274]: cluster 2023-10-01T11:08:50.363680+0000 mgr.y (mgr.25072) 1546 : cluster 0 pgmap v1661: 99 pgs: 4 active+recovery_wait+degraded, 90 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 7.9 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 9599/426 objects degraded (2253.286%); 15/426 objects misplaced (3.521%); 0 B/s, 29 objects/s recovering 2023-10-01T11:08:51.991 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:51 smithi078 bash[78274]: audit 2023-10-01T11:08:50.806516+0000 mon.a (mon.0) 2011 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:08:51.991 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:51 smithi078 bash[78274]: audit 2023-10-01T11:08:50.814743+0000 mon.a (mon.0) 2012 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:08:52.005 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:52.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:52.008 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:08:52.109 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:52.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:51 smithi173 bash[58816]: cluster 2023-10-01T11:08:50.363680+0000 mgr.y (mgr.25072) 1546 : cluster 0 pgmap v1661: 99 pgs: 4 active+recovery_wait+degraded, 90 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 7.9 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 9599/426 objects degraded (2253.286%); 15/426 objects misplaced (3.521%); 0 B/s, 29 objects/s recovering 2023-10-01T11:08:52.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:51 smithi173 bash[58816]: audit 2023-10-01T11:08:50.806516+0000 mon.a (mon.0) 2011 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:08:52.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:51 smithi173 bash[58816]: audit 2023-10-01T11:08:50.814743+0000 mon.a (mon.0) 2012 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:08:52.137 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:52.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:52.139 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:08:52.240 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:52.265 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:52.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:52.269 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:08:52.369 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:52.400 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:52.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:52.401 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:52.502 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:52.541 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:52.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:52.543 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:08:52.645 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:52.676 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:52.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:52.678 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:08:52.779 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:52.813 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:52.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:52.815 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:52.915 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:52.947 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:52.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:52.950 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:08:53.051 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:53.077 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:53.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:53.079 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:08:53.180 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:53.206 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:53.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:53.210 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:08:53.311 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:53.341 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:53.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:53.344 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:08:53.445 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:53.476 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:53.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:53.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:53.494 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:08:53.580 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:53.608 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:53.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:53.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:08:53.712 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:53.731 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:53.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:53.734 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:08:53.826 INFO:teuthology.orchestra.run.smithi078.stderr:pool 'unique_pool_8' removed 2023-10-01T11:08:53.835 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:53.866 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:53.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:53.875 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:08:53.950 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T11:08:53.950 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T11:08:53.976 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:53.985 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:53 smithi173 bash[58816]: cluster 2023-10-01T11:08:52.364611+0000 mgr.y (mgr.25072) 1547 : cluster 0 pgmap v1662: 99 pgs: 4 active+recovery_wait+degraded, 90 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 7.9 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 9599/426 objects degraded (2253.286%); 15/426 objects misplaced (3.521%); 0 B/s, 14 objects/s recovering 2023-10-01T11:08:53.985 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:53 smithi173 bash[58816]: audit 2023-10-01T11:08:53.716555+0000 mon.a (mon.0) 2013 : audit 1 from='client.? 172.21.15.78:0/4131334372' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T11:08:53.998 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:54.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:54.000 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:54.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:53 smithi078 bash[75299]: cluster 2023-10-01T11:08:52.364611+0000 mgr.y (mgr.25072) 1547 : cluster 0 pgmap v1662: 99 pgs: 4 active+recovery_wait+degraded, 90 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 7.9 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 9599/426 objects degraded (2253.286%); 15/426 objects misplaced (3.521%); 0 B/s, 14 objects/s recovering 2023-10-01T11:08:54.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:53 smithi078 bash[75299]: audit 2023-10-01T11:08:53.716555+0000 mon.a (mon.0) 2013 : audit 1 from='client.? 172.21.15.78:0/4131334372' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T11:08:54.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:53 smithi078 bash[78274]: cluster 2023-10-01T11:08:52.364611+0000 mgr.y (mgr.25072) 1547 : cluster 0 pgmap v1662: 99 pgs: 4 active+recovery_wait+degraded, 90 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait; 7.9 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 9599/426 objects degraded (2253.286%); 15/426 objects misplaced (3.521%); 0 B/s, 14 objects/s recovering 2023-10-01T11:08:54.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:53 smithi078 bash[78274]: audit 2023-10-01T11:08:53.716555+0000 mon.a (mon.0) 2013 : audit 1 from='client.? 172.21.15.78:0/4131334372' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T11:08:54.101 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:54.128 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:54.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:54.131 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:54.232 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:54.261 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:54.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:54.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:08:54.363 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:54.393 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:54.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:54.397 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:08:54.498 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:54.526 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:54.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:54.528 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:08:54.629 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:54.654 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:54.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:54.656 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:08:54.756 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:54.834 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:54.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:54.837 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:08:54.937 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:54.970 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:54.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:54.973 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:08:55.073 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:55.235 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:54 smithi173 bash[58816]: cluster 2023-10-01T11:08:53.814553+0000 osd.7 (osd.7) 397 : cluster 3 failed to encode map e354 with expected crc 2023-10-01T11:08:55.235 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:54 smithi173 bash[58816]: audit 2023-10-01T11:08:53.814652+0000 mon.a (mon.0) 2014 : audit 1 from='client.? 172.21.15.78:0/4131334372' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]': finished 2023-10-01T11:08:55.235 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:54 smithi173 bash[58816]: cluster 2023-10-01T11:08:53.815036+0000 mon.a (mon.0) 2015 : cluster 0 osdmap e354: 8 total, 8 up, 8 in 2023-10-01T11:08:55.236 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:54 smithi078 bash[75299]: cluster 2023-10-01T11:08:53.814553+0000 osd.7 (osd.7) 397 : cluster 3 failed to encode map e354 with expected crc 2023-10-01T11:08:55.236 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:54 smithi078 bash[75299]: audit 2023-10-01T11:08:53.814652+0000 mon.a (mon.0) 2014 : audit 1 from='client.? 172.21.15.78:0/4131334372' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]': finished 2023-10-01T11:08:55.236 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:54 smithi078 bash[75299]: cluster 2023-10-01T11:08:53.815036+0000 mon.a (mon.0) 2015 : cluster 0 osdmap e354: 8 total, 8 up, 8 in 2023-10-01T11:08:55.236 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:54 smithi078 bash[78274]: cluster 2023-10-01T11:08:53.814553+0000 osd.7 (osd.7) 397 : cluster 3 failed to encode map e354 with expected crc 2023-10-01T11:08:55.236 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:54 smithi078 bash[78274]: audit 2023-10-01T11:08:53.814652+0000 mon.a (mon.0) 2014 : audit 1 from='client.? 172.21.15.78:0/4131334372' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]': finished 2023-10-01T11:08:55.236 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:54 smithi078 bash[78274]: cluster 2023-10-01T11:08:53.815036+0000 mon.a (mon.0) 2015 : cluster 0 osdmap e354: 8 total, 8 up, 8 in 2023-10-01T11:08:55.268 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:55.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:55.273 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:08:55.374 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:55.404 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:55.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:55.407 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:55.507 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:55.532 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:55.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:55.534 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:55.635 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:55.664 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:55.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:55.667 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:08:55.768 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:55.794 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:55.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:55.797 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:55.897 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:55.908 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:55 smithi173 bash[58816]: cluster 2023-10-01T11:08:54.365494+0000 mgr.y (mgr.25072) 1548 : cluster 0 pgmap v1664: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:08:55.909 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:55 smithi173 bash[58816]: cluster 2023-10-01T11:08:54.820141+0000 mon.a (mon.0) 2016 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 9599/426 objects degraded (2253.286%), 6 pgs degraded, 2 pgs undersized) 2023-10-01T11:08:55.930 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:55.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:55.932 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:56.032 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:56.064 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:56.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:56.067 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:08:56.101 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:55 smithi078 bash[75299]: cluster 2023-10-01T11:08:54.365494+0000 mgr.y (mgr.25072) 1548 : cluster 0 pgmap v1664: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:08:56.101 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:55 smithi078 bash[75299]: cluster 2023-10-01T11:08:54.820141+0000 mon.a (mon.0) 2016 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 9599/426 objects degraded (2253.286%), 6 pgs degraded, 2 pgs undersized) 2023-10-01T11:08:56.101 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:55 smithi078 bash[78274]: cluster 2023-10-01T11:08:54.365494+0000 mgr.y (mgr.25072) 1548 : cluster 0 pgmap v1664: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:08:56.102 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:55 smithi078 bash[78274]: cluster 2023-10-01T11:08:54.820141+0000 mon.a (mon.0) 2016 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 9599/426 objects degraded (2253.286%), 6 pgs degraded, 2 pgs undersized) 2023-10-01T11:08:56.103 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-10-01T11:08:56.103 INFO:tasks.radosbench:Beginning radosbench... 2023-10-01T11:08:56.103 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_9 2023-10-01T11:08:56.104 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd pool create unique_pool_9 16 2023-10-01T11:08:56.168 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:56.194 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:56.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:56.196 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:08:56.297 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:56.324 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:56.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:56.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:56.428 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:56.459 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:56.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:56.462 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:56.562 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:56.593 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:56.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:56.596 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:08:56.696 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:56.723 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:56.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:56.725 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:08:56.826 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:56.853 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:56.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:56.856 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:08:56.956 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:56.990 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:56.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:56.993 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:08:57.094 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:57.121 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:57.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:57.123 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:57.223 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:57.255 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:08:57.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:57.257 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:08:57.358 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:57.384 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:57.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:57.387 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:57.488 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:57.516 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:57.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:57.519 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:57.620 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:57.653 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:57.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:57.656 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:08:57.747 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:08:57.747 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:08:57.757 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:57.787 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:57.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:57.790 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:08:57.890 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:57.904 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:57 smithi173 bash[58816]: cluster 2023-10-01T11:08:56.366193+0000 mgr.y (mgr.25072) 1549 : cluster 0 pgmap v1665: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:08:57.904 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:57 smithi173 bash[58816]: audit 2023-10-01T11:08:57.501159+0000 mon.a (mon.0) 2017 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:08:57.904 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:57 smithi173 bash[58816]: audit 2023-10-01T11:08:57.510289+0000 mon.a (mon.0) 2018 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:08:57.922 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:57.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:57.926 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:08:58.027 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:58.037 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:57 smithi078 bash[75299]: cluster 2023-10-01T11:08:56.366193+0000 mgr.y (mgr.25072) 1549 : cluster 0 pgmap v1665: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:08:58.037 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:57 smithi078 bash[75299]: audit 2023-10-01T11:08:57.501159+0000 mon.a (mon.0) 2017 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:08:58.037 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:57 smithi078 bash[75299]: audit 2023-10-01T11:08:57.510289+0000 mon.a (mon.0) 2018 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:08:58.037 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:57 smithi078 bash[78274]: cluster 2023-10-01T11:08:56.366193+0000 mgr.y (mgr.25072) 1549 : cluster 0 pgmap v1665: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:08:58.038 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:57 smithi078 bash[78274]: audit 2023-10-01T11:08:57.501159+0000 mon.a (mon.0) 2017 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:08:58.038 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:57 smithi078 bash[78274]: audit 2023-10-01T11:08:57.510289+0000 mon.a (mon.0) 2018 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:08:58.056 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:08:58.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:58.059 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:08:58.160 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:58.191 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:58.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:58.195 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:08:58.296 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:58.323 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:08:58.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:58.325 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:08:58.426 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:58.458 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:58.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:58.461 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:58.562 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:58.592 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:58.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:58.595 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:58.696 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:58.718 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:08:58.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:58.720 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:08:58.821 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:58.855 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:08:58.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:58.858 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:08:58.958 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:58.994 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:58.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:58.997 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:08:59.098 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:59.130 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:59.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:59.134 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:08:59.235 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:59.263 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:59.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:59.266 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:08:59.366 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:59.396 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:08:59.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:59.400 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:08:59.500 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:59.535 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:08:59.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:59.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:08:59.639 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:59.667 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:59.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:59.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:08:59.771 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:59.805 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:59.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:59.809 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:08:59.910 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:59.924 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:08:59 smithi173 bash[58816]: cluster 2023-10-01T11:08:58.366769+0000 mgr.y (mgr.25072) 1550 : cluster 0 pgmap v1666: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:08:59.943 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:08:59.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:08:59.946 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:09:00.047 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:00.075 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:00.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:00.078 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:09:00.157 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2023-10-01T11:09:00.157 INFO:teuthology.orchestra.run.smithi078.stdout: "ops": [], 2023-10-01T11:09:00.157 INFO:teuthology.orchestra.run.smithi078.stdout: "num_ops": 0 2023-10-01T11:09:00.157 INFO:teuthology.orchestra.run.smithi078.stdout:} 2023-10-01T11:09:00.178 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:00.181 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:08:59 smithi078 bash[78274]: cluster 2023-10-01T11:08:58.366769+0000 mgr.y (mgr.25072) 1550 : cluster 0 pgmap v1666: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:00.182 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:08:59 smithi078 bash[75299]: cluster 2023-10-01T11:08:58.366769+0000 mgr.y (mgr.25072) 1550 : cluster 0 pgmap v1666: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:00.214 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:00.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:00.218 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:09:00.318 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:00.344 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:00.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:00.346 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:09:00.447 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:00.481 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:00.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:00.484 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:09:00.514 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:09:00 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:09:00] "GET /metrics HTTP/1.1" 200 36409 "" "Prometheus/2.33.4" 2023-10-01T11:09:00.585 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:00.610 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:00.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:00.613 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:09:00.714 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:00.751 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:00.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:00.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:09:00.855 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:00.882 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:00.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:00.885 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:00.986 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:01.019 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:01.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:01.022 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:09:01.123 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:01.147 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:01.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:01.149 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:09:01.250 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:01.272 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:01.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:01.274 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:09:01.375 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:01.406 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:01.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:01.409 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:01.510 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:01.534 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:01.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:01.536 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:09:01.637 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:01.670 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:01.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:01.673 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:09:01.773 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:01.795 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:01.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:01.797 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:09:01.898 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:01.909 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:01 smithi173 bash[58816]: cluster 2023-10-01T11:09:00.367551+0000 mgr.y (mgr.25072) 1551 : cluster 0 pgmap v1667: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:01.929 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:01.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:01.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:02.033 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:02.041 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:01 smithi078 bash[75299]: cluster 2023-10-01T11:09:00.367551+0000 mgr.y (mgr.25072) 1551 : cluster 0 pgmap v1667: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:02.041 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:01 smithi078 bash[78274]: cluster 2023-10-01T11:09:00.367551+0000 mgr.y (mgr.25072) 1551 : cluster 0 pgmap v1667: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:02.061 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:02.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:02.064 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:09:02.164 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:02.197 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:02.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:02.198 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:02.299 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:02.324 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:02.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:02.327 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:09:02.427 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:02.458 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:02.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:02.462 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:02.562 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:02.592 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:02.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:02.595 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:02.696 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:02.727 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:02.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:02.730 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:09:02.830 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:02.862 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:02.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:02.865 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:09:02.966 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:03.001 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:03.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:03.004 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:09:03.051 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-01T11:09:03.104 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:03.136 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:03.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:03.139 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:09:03.240 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:03.272 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:03.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:03.278 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:09:03.378 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:03.400 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:03.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:03.403 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:09:03.504 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:03.516 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:03 smithi173 bash[58816]: cluster 2023-10-01T11:09:02.368281+0000 mgr.y (mgr.25072) 1552 : cluster 0 pgmap v1668: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:03.516 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:03 smithi173 bash[58816]: audit 2023-10-01T11:09:02.468579+0000 mon.a (mon.0) 2019 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:09:03.516 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:03 smithi173 bash[58816]: audit 2023-10-01T11:09:02.469795+0000 mon.a (mon.0) 2020 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:09:03.516 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:03 smithi173 bash[58816]: audit 2023-10-01T11:09:03.322907+0000 mon.a (mon.0) 2021 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:09:03.538 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:03.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:03.542 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:09:03.642 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:03.651 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:03 smithi078 bash[78274]: cluster 2023-10-01T11:09:02.368281+0000 mgr.y (mgr.25072) 1552 : cluster 0 pgmap v1668: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:03.651 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:03 smithi078 bash[78274]: audit 2023-10-01T11:09:02.468579+0000 mon.a (mon.0) 2019 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:09:03.651 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:03 smithi078 bash[78274]: audit 2023-10-01T11:09:02.469795+0000 mon.a (mon.0) 2020 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:09:03.651 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:03 smithi078 bash[78274]: audit 2023-10-01T11:09:03.322907+0000 mon.a (mon.0) 2021 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:09:03.651 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:03 smithi078 bash[75299]: cluster 2023-10-01T11:09:02.368281+0000 mgr.y (mgr.25072) 1552 : cluster 0 pgmap v1668: 73 pgs: 73 active+clean; 582 KiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:03.651 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:03 smithi078 bash[75299]: audit 2023-10-01T11:09:02.468579+0000 mon.a (mon.0) 2019 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:09:03.651 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:03 smithi078 bash[75299]: audit 2023-10-01T11:09:02.469795+0000 mon.a (mon.0) 2020 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:09:03.651 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:03 smithi078 bash[75299]: audit 2023-10-01T11:09:03.322907+0000 mon.a (mon.0) 2021 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:09:03.667 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:03.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:03.669 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:09:03.770 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:03.792 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:03.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:03.794 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:09:03.895 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:03.921 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:03.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:03.923 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:09:04.024 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:04.057 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:04.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:04.060 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:09:04.161 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:04.187 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:04.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:04.189 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:09:04.290 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:04.322 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:04.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:04.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:09:04.427 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:04.459 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:04.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:04.462 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:09:04.563 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:04.596 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:04.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:04.598 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:04.699 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:04.733 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:04.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:04.736 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:09:04.837 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:04.859 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:04.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:04.862 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:09:04.963 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:04.993 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:04.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:04.995 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:09:05.096 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:05.125 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:05.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:05.128 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:09:05.229 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:05.258 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:05.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:05.260 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:09:05.361 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:05.391 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:05.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:05.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:05.496 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:05.525 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:05.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:05.528 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:09:05.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:05 smithi173 bash[58816]: cluster 2023-10-01T11:09:04.368868+0000 mgr.y (mgr.25072) 1553 : cluster 0 pgmap v1669: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:05.629 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:05.654 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:05.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:05.658 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:09:05.759 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:05.795 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:05.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:05.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:09:05.825 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:05 smithi078 bash[75299]: cluster 2023-10-01T11:09:04.368868+0000 mgr.y (mgr.25072) 1553 : cluster 0 pgmap v1669: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:05.825 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:05 smithi078 bash[78274]: cluster 2023-10-01T11:09:04.368868+0000 mgr.y (mgr.25072) 1553 : cluster 0 pgmap v1669: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:05.900 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:05.926 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:05.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:05.928 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:09:06.029 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:06.060 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:06.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:06.062 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:06.163 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:06.189 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:06.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:06.192 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:09:06.278 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:09:06.293 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:06.316 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:06.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:06.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:09:06.419 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:06.441 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:06.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:06.445 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:09:06.545 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:06.578 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:06.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:06.581 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:09:06.682 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:06.707 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:06.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:06.710 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:09:06.810 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:06.842 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:06.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:06.844 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:06.945 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:06.973 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:06.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:06.976 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:09:07.077 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:07.109 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:07.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:07.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:09:07.212 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:07.241 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:07.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:07.244 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:09:07.345 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:07.373 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:07.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:07.375 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:09:07.476 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:07.503 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:07.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:07.506 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:09:07.607 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:07.619 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:07 smithi078 bash[75299]: cluster 2023-10-01T11:09:06.369579+0000 mgr.y (mgr.25072) 1554 : cluster 0 pgmap v1670: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:07.620 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:07 smithi078 bash[78274]: cluster 2023-10-01T11:09:06.369579+0000 mgr.y (mgr.25072) 1554 : cluster 0 pgmap v1670: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:07.632 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:07.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:07.635 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:09:07.735 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:07.747 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:07 smithi173 bash[58816]: cluster 2023-10-01T11:09:06.369579+0000 mgr.y (mgr.25072) 1554 : cluster 0 pgmap v1670: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:07.765 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:07.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:07.768 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:09:07.869 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:08.174 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:08.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:08.176 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:09:08.276 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:08.307 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:08.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:08.310 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:09:08.411 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:08.436 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:08.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:08.438 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:09:08.539 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:08.570 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:08.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:08.573 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:09:08.674 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:08.700 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:08.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:08.702 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:09:08.803 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:08.830 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:08.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:08.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:09:08.845 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:09:08.933 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:08.965 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:08.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:08.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:09:09.069 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:09.102 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:09.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:09.105 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:09.205 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:09.227 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:09.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:09.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:09:09.330 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:09.358 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:09.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:09.360 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:09:09.461 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:09.488 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:09.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:09.490 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:09:09.591 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:09.626 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:09.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:09.630 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:09:09.731 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:09.765 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:09.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:09.769 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:09.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:09 smithi173 bash[58816]: cluster 2023-10-01T11:09:08.369966+0000 mgr.y (mgr.25072) 1555 : cluster 0 pgmap v1671: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:09.869 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:09.891 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:09.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:09.894 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:09:09.994 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:10.015 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:10.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:10.018 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:09:10.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:09 smithi078 bash[75299]: cluster 2023-10-01T11:09:08.369966+0000 mgr.y (mgr.25072) 1555 : cluster 0 pgmap v1671: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:10.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:09 smithi078 bash[78274]: cluster 2023-10-01T11:09:08.369966+0000 mgr.y (mgr.25072) 1555 : cluster 0 pgmap v1671: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:10.119 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:10.145 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:10.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:10.147 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:09:10.248 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:10.282 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:10.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:10.285 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:09:10.385 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:10.417 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:10.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:10.420 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:09:10.520 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:10.555 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:10.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:10.559 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:09:10.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:09:10 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:09:10] "GET /metrics HTTP/1.1" 200 33967 "" "Prometheus/2.33.4" 2023-10-01T11:09:10.660 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:10.690 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:10.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:10.693 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:09:10.794 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:10.817 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:10.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:10.819 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:09:10.920 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:10.946 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:10.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:10.948 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:09:11.049 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:11.081 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:11.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:11.084 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:09:11.185 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:11.218 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:11.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:11.222 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:09:11.322 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:11.346 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:11.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:11.349 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:09:11.449 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:11.476 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:11.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:11.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:09:11.580 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:11.613 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:11.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:11.616 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:09:11.717 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:11.746 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:11.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:11.750 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:09:11.821 INFO:teuthology.orchestra.run.smithi078.stderr:nodeep-scrub is set 2023-10-01T11:09:11.847 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:11 smithi078 bash[75299]: cluster 2023-10-01T11:09:10.370575+0000 mgr.y (mgr.25072) 1556 : cluster 0 pgmap v1672: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:11.848 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:11 smithi078 bash[75299]: audit 2023-10-01T11:09:11.202103+0000 mon.c (mon.2) 235 : audit 1 from='client.? 172.21.15.78:0/2874918186' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:09:11.848 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:11 smithi078 bash[75299]: audit 2023-10-01T11:09:11.202721+0000 mon.a (mon.0) 2022 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:09:11.851 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:11.864 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:11 smithi173 bash[58816]: cluster 2023-10-01T11:09:10.370575+0000 mgr.y (mgr.25072) 1556 : cluster 0 pgmap v1672: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:11.864 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:11 smithi173 bash[58816]: audit 2023-10-01T11:09:11.202103+0000 mon.c (mon.2) 235 : audit 1 from='client.? 172.21.15.78:0/2874918186' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:09:11.865 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:11 smithi173 bash[58816]: audit 2023-10-01T11:09:11.202721+0000 mon.a (mon.0) 2022 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:09:11.887 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:11.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:11.891 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:09:11.924 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T11:09:11.946 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T11:09:11.973 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T11:09:11.991 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:11.993 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T11:09:12.023 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T11:09:12.024 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:12.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:12.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:09:12.051 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T11:09:12.073 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T11:09:12.094 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T11:09:12.096 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T11:09:12.096 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T11:09:12.096 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T11:09:12.096 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T11:09:12.096 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T11:09:12.096 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T11:09:12.096 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T11:09:12.097 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T11:09:12.120 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:11 smithi078 bash[78274]: cluster 2023-10-01T11:09:10.370575+0000 mgr.y (mgr.25072) 1556 : cluster 0 pgmap v1672: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:12.120 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:11 smithi078 bash[78274]: audit 2023-10-01T11:09:11.202103+0000 mon.c (mon.2) 235 : audit 1 from='client.? 172.21.15.78:0/2874918186' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:09:12.120 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:11 smithi078 bash[78274]: audit 2023-10-01T11:09:11.202721+0000 mon.a (mon.0) 2022 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:09:12.128 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:12.147 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:12.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:12.149 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:09:12.250 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:12.285 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:12.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:12.290 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:09:12.390 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:12.417 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:12.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:12.419 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:09:12.519 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:12.552 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:12.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:12.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:09:12.656 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:12.686 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:12.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:12.689 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:12.789 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:12.815 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:12.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:12.818 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:09:12.918 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:12.944 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:12.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:12.946 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:09:13.047 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:13.059 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:12 smithi078 bash[78274]: cluster 2023-10-01T11:09:11.805189+0000 mon.a (mon.0) 2023 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:09:13.059 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:12 smithi078 bash[78274]: cluster 2023-10-01T11:09:11.809529+0000 osd.7 (osd.7) 398 : cluster 3 failed to encode map e355 with expected crc 2023-10-01T11:09:13.059 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:12 smithi078 bash[78274]: audit 2023-10-01T11:09:11.811313+0000 mon.a (mon.0) 2024 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T11:09:13.059 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:12 smithi078 bash[78274]: cluster 2023-10-01T11:09:11.811389+0000 mon.a (mon.0) 2025 : cluster 0 osdmap e355: 8 total, 8 up, 8 in 2023-10-01T11:09:13.059 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:12 smithi078 bash[78274]: cluster 2023-10-01T11:09:11.838745+0000 osd.5 (osd.5) 376 : cluster 3 failed to encode map e355 with expected crc 2023-10-01T11:09:13.060 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:12 smithi078 bash[78274]: cluster 2023-10-01T11:09:11.841380+0000 osd.5 (osd.5) 377 : cluster 3 failed to encode map e355 with expected crc 2023-10-01T11:09:13.060 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:12 smithi078 bash[78274]: audit 2023-10-01T11:09:11.898121+0000 mon.a (mon.0) 2026 : audit 0 from='client.? 172.21.15.78:0/887665182' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:09:13.060 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:12 smithi078 bash[78274]: cluster 2023-10-01T11:09:12.035992+0000 osd.6 (osd.6) 339 : cluster 3 failed to encode map e355 with expected crc 2023-10-01T11:09:13.060 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:12 smithi078 bash[75299]: cluster 2023-10-01T11:09:11.805189+0000 mon.a (mon.0) 2023 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:09:13.060 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:12 smithi078 bash[75299]: cluster 2023-10-01T11:09:11.809529+0000 osd.7 (osd.7) 398 : cluster 3 failed to encode map e355 with expected crc 2023-10-01T11:09:13.060 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:12 smithi078 bash[75299]: audit 2023-10-01T11:09:11.811313+0000 mon.a (mon.0) 2024 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T11:09:13.060 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:12 smithi078 bash[75299]: cluster 2023-10-01T11:09:11.811389+0000 mon.a (mon.0) 2025 : cluster 0 osdmap e355: 8 total, 8 up, 8 in 2023-10-01T11:09:13.060 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:12 smithi078 bash[75299]: cluster 2023-10-01T11:09:11.838745+0000 osd.5 (osd.5) 376 : cluster 3 failed to encode map e355 with expected crc 2023-10-01T11:09:13.060 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:12 smithi078 bash[75299]: cluster 2023-10-01T11:09:11.841380+0000 osd.5 (osd.5) 377 : cluster 3 failed to encode map e355 with expected crc 2023-10-01T11:09:13.061 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:12 smithi078 bash[75299]: audit 2023-10-01T11:09:11.898121+0000 mon.a (mon.0) 2026 : audit 0 from='client.? 172.21.15.78:0/887665182' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:09:13.061 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:12 smithi078 bash[75299]: cluster 2023-10-01T11:09:12.035992+0000 osd.6 (osd.6) 339 : cluster 3 failed to encode map e355 with expected crc 2023-10-01T11:09:13.079 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:13.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:13.081 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:09:13.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:12 smithi173 bash[58816]: cluster 2023-10-01T11:09:11.805189+0000 mon.a (mon.0) 2023 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:09:13.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:12 smithi173 bash[58816]: cluster 2023-10-01T11:09:11.809529+0000 osd.7 (osd.7) 398 : cluster 3 failed to encode map e355 with expected crc 2023-10-01T11:09:13.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:12 smithi173 bash[58816]: audit 2023-10-01T11:09:11.811313+0000 mon.a (mon.0) 2024 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T11:09:13.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:12 smithi173 bash[58816]: cluster 2023-10-01T11:09:11.811389+0000 mon.a (mon.0) 2025 : cluster 0 osdmap e355: 8 total, 8 up, 8 in 2023-10-01T11:09:13.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:12 smithi173 bash[58816]: cluster 2023-10-01T11:09:11.838745+0000 osd.5 (osd.5) 376 : cluster 3 failed to encode map e355 with expected crc 2023-10-01T11:09:13.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:12 smithi173 bash[58816]: cluster 2023-10-01T11:09:11.841380+0000 osd.5 (osd.5) 377 : cluster 3 failed to encode map e355 with expected crc 2023-10-01T11:09:13.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:12 smithi173 bash[58816]: audit 2023-10-01T11:09:11.898121+0000 mon.a (mon.0) 2026 : audit 0 from='client.? 172.21.15.78:0/887665182' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:09:13.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:12 smithi173 bash[58816]: cluster 2023-10-01T11:09:12.035992+0000 osd.6 (osd.6) 339 : cluster 3 failed to encode map e355 with expected crc 2023-10-01T11:09:13.182 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:13.209 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:13.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:13.212 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:09:13.313 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:13.340 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:13.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:13.342 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:09:13.443 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:13.481 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:13.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:13.484 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:09:13.585 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:13.614 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:13.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:13.617 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:09:13.718 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:13.750 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:13.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:13.752 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:09:13.853 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:13.878 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:13.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:13.881 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:09:13.982 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:14.009 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:14.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:14.012 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:09:14.113 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:14.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:13 smithi173 bash[58816]: cluster 2023-10-01T11:09:12.371035+0000 mgr.y (mgr.25072) 1557 : cluster 0 pgmap v1674: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:14.134 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:13 smithi078 bash[75299]: cluster 2023-10-01T11:09:12.371035+0000 mgr.y (mgr.25072) 1557 : cluster 0 pgmap v1674: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:14.135 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:13 smithi078 bash[78274]: cluster 2023-10-01T11:09:12.371035+0000 mgr.y (mgr.25072) 1557 : cluster 0 pgmap v1674: 73 pgs: 73 active+clean; 582 KiB data, 8.2 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:14.142 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:14.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:14.145 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:09:14.246 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:14.274 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:14.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:14.277 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:14.378 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:14.415 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:14.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:14.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:09:14.519 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:14.546 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:14.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:14.550 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:09:14.651 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:14.680 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:14.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:14.683 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:09:14.783 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:14.814 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:14.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:14.817 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:09:14.917 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:14.947 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:14.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:14.949 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:09:15.050 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:15.076 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:15.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:15.079 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:09:15.180 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:15.211 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:15.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:15.214 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:15.315 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:15.351 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:15.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:15.354 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:09:15.455 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:15.490 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:15.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:15.493 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:09:15.593 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:15.626 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:15.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:15.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:09:15.730 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:15.752 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:15.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:15.754 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:09:15.855 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:15.888 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:15.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:15.891 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:09:15.991 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:16.025 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:16.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:16.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:09:16.078 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:15 smithi078 bash[75299]: cluster 2023-10-01T11:09:14.371985+0000 mgr.y (mgr.25072) 1558 : cluster 0 pgmap v1675: 73 pgs: 73 active+clean; 582 KiB data, 8.1 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:16.078 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:15 smithi078 bash[78274]: cluster 2023-10-01T11:09:14.371985+0000 mgr.y (mgr.25072) 1558 : cluster 0 pgmap v1675: 73 pgs: 73 active+clean; 582 KiB data, 8.1 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:16.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:15 smithi173 bash[58816]: cluster 2023-10-01T11:09:14.371985+0000 mgr.y (mgr.25072) 1558 : cluster 0 pgmap v1675: 73 pgs: 73 active+clean; 582 KiB data, 8.1 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:16.129 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:16.150 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:16.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:16.152 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:09:16.253 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:16.281 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:16.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:16.283 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:09:16.384 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:16.416 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:16.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:16.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:09:16.519 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:16.549 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:16.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:16.552 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:09:16.653 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:16.679 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:16.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:16.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:09:16.782 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:16.812 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:16.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:16.815 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:09:16.916 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:16.946 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:16.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:16.948 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:09:17.049 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:17.078 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:17.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:17.081 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:09:17.182 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:17.190 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:17 smithi078 bash[75299]: audit 2023-10-01T11:09:16.112834+0000 mon.a (mon.0) 2027 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:09:17.190 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:17 smithi078 bash[75299]: audit 2023-10-01T11:09:16.120429+0000 mon.a (mon.0) 2028 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:09:17.190 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:17 smithi078 bash[75299]: audit 2023-10-01T11:09:16.121934+0000 mon.a (mon.0) 2029 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T11:09:17.191 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:17 smithi078 bash[75299]: audit 2023-10-01T11:09:16.123225+0000 mon.a (mon.0) 2030 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T11:09:17.191 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:17 smithi078 bash[75299]: audit 2023-10-01T11:09:16.129554+0000 mon.a (mon.0) 2031 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:09:17.191 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:17 smithi078 bash[75299]: cluster 2023-10-01T11:09:16.372863+0000 mgr.y (mgr.25072) 1559 : cluster 0 pgmap v1676: 73 pgs: 73 active+clean; 582 KiB data, 8.1 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:17.191 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:17 smithi078 bash[78274]: audit 2023-10-01T11:09:16.112834+0000 mon.a (mon.0) 2027 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:09:17.191 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:17 smithi078 bash[78274]: audit 2023-10-01T11:09:16.120429+0000 mon.a (mon.0) 2028 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:09:17.191 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:17 smithi078 bash[78274]: audit 2023-10-01T11:09:16.121934+0000 mon.a (mon.0) 2029 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T11:09:17.191 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:17 smithi078 bash[78274]: audit 2023-10-01T11:09:16.123225+0000 mon.a (mon.0) 2030 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T11:09:17.191 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:17 smithi078 bash[78274]: audit 2023-10-01T11:09:16.129554+0000 mon.a (mon.0) 2031 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:09:17.191 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:17 smithi078 bash[78274]: cluster 2023-10-01T11:09:16.372863+0000 mgr.y (mgr.25072) 1559 : cluster 0 pgmap v1676: 73 pgs: 73 active+clean; 582 KiB data, 8.1 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:17.204 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:17.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:17.206 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:09:17.307 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:17.339 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:17.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:17.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:09:17.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:17 smithi173 bash[58816]: audit 2023-10-01T11:09:16.112834+0000 mon.a (mon.0) 2027 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:09:17.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:17 smithi173 bash[58816]: audit 2023-10-01T11:09:16.120429+0000 mon.a (mon.0) 2028 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:09:17.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:17 smithi173 bash[58816]: audit 2023-10-01T11:09:16.121934+0000 mon.a (mon.0) 2029 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T11:09:17.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:17 smithi173 bash[58816]: audit 2023-10-01T11:09:16.123225+0000 mon.a (mon.0) 2030 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T11:09:17.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:17 smithi173 bash[58816]: audit 2023-10-01T11:09:16.129554+0000 mon.a (mon.0) 2031 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:09:17.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:17 smithi173 bash[58816]: cluster 2023-10-01T11:09:16.372863+0000 mgr.y (mgr.25072) 1559 : cluster 0 pgmap v1676: 73 pgs: 73 active+clean; 582 KiB data, 8.1 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:17.443 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:17.469 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:17.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:17.473 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:09:17.574 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:17.606 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:17.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:17.608 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:17.709 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:17.743 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:17.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:17.745 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:17.846 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:17.877 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:17.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:17.879 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:17.900 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T11:09:17.980 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:18.010 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:18.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:18.013 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:18.080 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T11:09:18.113 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:18.153 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:18.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:18.201 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:09:18.302 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:18.310 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:18 smithi078 bash[75299]: audit 2023-10-01T11:09:17.464484+0000 mon.a (mon.0) 2032 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:09:18.311 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:18 smithi078 bash[78274]: audit 2023-10-01T11:09:17.464484+0000 mon.a (mon.0) 2032 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:09:18.329 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:18.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:18.332 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:09:18.432 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:18.464 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:18.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:18.467 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:09:18.567 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:18.581 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:18 smithi173 bash[58816]: audit 2023-10-01T11:09:17.464484+0000 mon.a (mon.0) 2032 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:09:18.601 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:18.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:18.603 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:18.704 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:18.733 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:18.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:18.736 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:09:18.837 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:18.866 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:18.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:18.870 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:09:18.892 INFO:teuthology.orchestra.run.smithi078.stdout:ERROR: (22) Invalid argument 2023-10-01T11:09:18.892 INFO:teuthology.orchestra.run.smithi078.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T11:09:18.971 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:19.004 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:19.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:19.007 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:19.108 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:19.137 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:19.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:19.140 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:09:19.241 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:19.274 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:19.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:19.278 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:09:19.379 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:19.413 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:19.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:19.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:09:19.517 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:19.544 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:19.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:19.548 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:09:19.648 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:19.662 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:19 smithi173 bash[58816]: cluster 2023-10-01T11:09:18.373506+0000 mgr.y (mgr.25072) 1560 : cluster 0 pgmap v1677: 73 pgs: 73 active+clean; 582 KiB data, 8.1 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:19.680 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:19.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:19.683 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:09:19.784 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:19.813 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:19.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:19.816 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:09:19.917 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:19.949 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:19.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:19.952 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:19 smithi078 bash[75299]: cluster 2023-10-01T11:09:18.373506+0000 mgr.y (mgr.25072) 1560 : cluster 0 pgmap v1677: 73 pgs: 73 active+clean; 582 KiB data, 8.1 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:19.952 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:19 smithi078 bash[78274]: cluster 2023-10-01T11:09:18.373506+0000 mgr.y (mgr.25072) 1560 : cluster 0 pgmap v1677: 73 pgs: 73 active+clean; 582 KiB data, 8.1 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:19.953 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:09:19.954 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-01T11:09:20.054 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:20.086 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:20.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:20.090 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:09:20.153 INFO:teuthology.orchestra.run.smithi078.stderr:pool 'unique_pool_9' created 2023-10-01T11:09:20.191 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:20.222 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:20.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:20.225 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:09:20.326 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:20.361 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:20.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:20.364 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:09:20.465 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:20.497 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:20.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:20.500 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:09:20.579 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:09:20 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:09:20] "GET /metrics HTTP/1.1" 200 33966 "" "Prometheus/2.33.4" 2023-10-01T11:09:20.601 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:20.633 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:20.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:20.635 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:09:20.736 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:20.748 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:20 smithi173 bash[58816]: audit 2023-10-01T11:09:19.134663+0000 mon.a (mon.0) 2033 : audit 1 from='client.? 172.21.15.78:0/878863455' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]: dispatch 2023-10-01T11:09:20.749 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:20 smithi173 bash[58816]: audit 2023-10-01T11:09:20.142680+0000 mon.a (mon.0) 2034 : audit 1 from='client.? 172.21.15.78:0/878863455' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]': finished 2023-10-01T11:09:20.749 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:20 smithi173 bash[58816]: cluster 2023-10-01T11:09:20.143265+0000 mon.a (mon.0) 2035 : cluster 0 osdmap e356: 8 total, 8 up, 8 in 2023-10-01T11:09:20.768 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:20.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:20.772 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:09:20.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:20 smithi078 bash[78274]: audit 2023-10-01T11:09:19.134663+0000 mon.a (mon.0) 2033 : audit 1 from='client.? 172.21.15.78:0/878863455' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]: dispatch 2023-10-01T11:09:20.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:20 smithi078 bash[78274]: audit 2023-10-01T11:09:20.142680+0000 mon.a (mon.0) 2034 : audit 1 from='client.? 172.21.15.78:0/878863455' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]': finished 2023-10-01T11:09:20.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:20 smithi078 bash[78274]: cluster 2023-10-01T11:09:20.143265+0000 mon.a (mon.0) 2035 : cluster 0 osdmap e356: 8 total, 8 up, 8 in 2023-10-01T11:09:20.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:20 smithi078 bash[75299]: audit 2023-10-01T11:09:19.134663+0000 mon.a (mon.0) 2033 : audit 1 from='client.? 172.21.15.78:0/878863455' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]: dispatch 2023-10-01T11:09:20.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:20 smithi078 bash[75299]: audit 2023-10-01T11:09:20.142680+0000 mon.a (mon.0) 2034 : audit 1 from='client.? 172.21.15.78:0/878863455' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]': finished 2023-10-01T11:09:20.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:20 smithi078 bash[75299]: cluster 2023-10-01T11:09:20.143265+0000 mon.a (mon.0) 2035 : cluster 0 osdmap e356: 8 total, 8 up, 8 in 2023-10-01T11:09:20.872 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:20.905 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:20.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:20.906 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:09:21.007 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:21.040 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:21.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:21.043 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:09:21.144 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:21.172 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:21.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:21.174 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:09:21.187 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd pool application enable unique_pool_9 rados --yes-i-really-mean-it || true 2023-10-01T11:09:21.275 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:21.306 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:21.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:21.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:09:21.409 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:21.439 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:21.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:21.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:21.544 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:21.576 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:21.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:21.577 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:09:21.678 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:21.713 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:21.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:21.717 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:09:21.817 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:21.832 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:21 smithi173 bash[58816]: cluster 2023-10-01T11:09:20.138133+0000 osd.7 (osd.7) 399 : cluster 3 failed to encode map e356 with expected crc 2023-10-01T11:09:21.832 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:21 smithi173 bash[58816]: cluster 2023-10-01T11:09:20.144953+0000 osd.6 (osd.6) 340 : cluster 3 failed to encode map e356 with expected crc 2023-10-01T11:09:21.832 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:21 smithi173 bash[58816]: cluster 2023-10-01T11:09:20.147183+0000 osd.5 (osd.5) 378 : cluster 3 failed to encode map e356 with expected crc 2023-10-01T11:09:21.832 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:21 smithi173 bash[58816]: cluster 2023-10-01T11:09:20.374186+0000 mgr.y (mgr.25072) 1561 : cluster 0 pgmap v1679: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 8.0 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:21.832 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:21 smithi173 bash[58816]: cluster 2023-10-01T11:09:21.148088+0000 mon.a (mon.0) 2036 : cluster 0 osdmap e357: 8 total, 8 up, 8 in 2023-10-01T11:09:21.852 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:21.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:21.855 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:09:21.956 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:21.984 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:21.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:21.987 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:22.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:21 smithi078 bash[75299]: cluster 2023-10-01T11:09:20.138133+0000 osd.7 (osd.7) 399 : cluster 3 failed to encode map e356 with expected crc 2023-10-01T11:09:22.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:21 smithi078 bash[75299]: cluster 2023-10-01T11:09:20.144953+0000 osd.6 (osd.6) 340 : cluster 3 failed to encode map e356 with expected crc 2023-10-01T11:09:22.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:21 smithi078 bash[75299]: cluster 2023-10-01T11:09:20.147183+0000 osd.5 (osd.5) 378 : cluster 3 failed to encode map e356 with expected crc 2023-10-01T11:09:22.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:21 smithi078 bash[75299]: cluster 2023-10-01T11:09:20.374186+0000 mgr.y (mgr.25072) 1561 : cluster 0 pgmap v1679: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 8.0 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:22.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:21 smithi078 bash[75299]: cluster 2023-10-01T11:09:21.148088+0000 mon.a (mon.0) 2036 : cluster 0 osdmap e357: 8 total, 8 up, 8 in 2023-10-01T11:09:22.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:21 smithi078 bash[78274]: cluster 2023-10-01T11:09:20.138133+0000 osd.7 (osd.7) 399 : cluster 3 failed to encode map e356 with expected crc 2023-10-01T11:09:22.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:21 smithi078 bash[78274]: cluster 2023-10-01T11:09:20.144953+0000 osd.6 (osd.6) 340 : cluster 3 failed to encode map e356 with expected crc 2023-10-01T11:09:22.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:21 smithi078 bash[78274]: cluster 2023-10-01T11:09:20.147183+0000 osd.5 (osd.5) 378 : cluster 3 failed to encode map e356 with expected crc 2023-10-01T11:09:22.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:21 smithi078 bash[78274]: cluster 2023-10-01T11:09:20.374186+0000 mgr.y (mgr.25072) 1561 : cluster 0 pgmap v1679: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 8.0 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:22.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:21 smithi078 bash[78274]: cluster 2023-10-01T11:09:21.148088+0000 mon.a (mon.0) 2036 : cluster 0 osdmap e357: 8 total, 8 up, 8 in 2023-10-01T11:09:22.088 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:22.118 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:22.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:22.121 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:22.222 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:22.253 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:22.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:22.256 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:09:22.357 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:22.389 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:22.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:22.393 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:09:22.494 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:22.528 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:22.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:22.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:09:22.632 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:22.663 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:22.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:22.667 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:09:22.768 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:22.779 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:22 smithi173 bash[58816]: cluster 2023-10-01T11:09:21.146110+0000 osd.7 (osd.7) 400 : cluster 3 failed to encode map e357 with expected crc 2023-10-01T11:09:22.780 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:22 smithi173 bash[58816]: cluster 2023-10-01T11:09:21.149057+0000 osd.6 (osd.6) 341 : cluster 3 failed to encode map e357 with expected crc 2023-10-01T11:09:22.780 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:22 smithi173 bash[58816]: cluster 2023-10-01T11:09:21.149347+0000 osd.5 (osd.5) 379 : cluster 3 failed to encode map e357 with expected crc 2023-10-01T11:09:22.780 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:22 smithi173 bash[58816]: cluster 2023-10-01T11:09:22.153709+0000 mon.a (mon.0) 2037 : cluster 0 osdmap e358: 8 total, 8 up, 8 in 2023-10-01T11:09:22.798 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:22.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:22.802 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:22.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:22 smithi078 bash[75299]: cluster 2023-10-01T11:09:21.146110+0000 osd.7 (osd.7) 400 : cluster 3 failed to encode map e357 with expected crc 2023-10-01T11:09:22.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:22 smithi078 bash[75299]: cluster 2023-10-01T11:09:21.149057+0000 osd.6 (osd.6) 341 : cluster 3 failed to encode map e357 with expected crc 2023-10-01T11:09:22.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:22 smithi078 bash[75299]: cluster 2023-10-01T11:09:21.149347+0000 osd.5 (osd.5) 379 : cluster 3 failed to encode map e357 with expected crc 2023-10-01T11:09:22.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:22 smithi078 bash[75299]: cluster 2023-10-01T11:09:22.153709+0000 mon.a (mon.0) 2037 : cluster 0 osdmap e358: 8 total, 8 up, 8 in 2023-10-01T11:09:22.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:22 smithi078 bash[78274]: cluster 2023-10-01T11:09:21.146110+0000 osd.7 (osd.7) 400 : cluster 3 failed to encode map e357 with expected crc 2023-10-01T11:09:22.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:22 smithi078 bash[78274]: cluster 2023-10-01T11:09:21.149057+0000 osd.6 (osd.6) 341 : cluster 3 failed to encode map e357 with expected crc 2023-10-01T11:09:22.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:22 smithi078 bash[78274]: cluster 2023-10-01T11:09:21.149347+0000 osd.5 (osd.5) 379 : cluster 3 failed to encode map e357 with expected crc 2023-10-01T11:09:22.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:22 smithi078 bash[78274]: cluster 2023-10-01T11:09:22.153709+0000 mon.a (mon.0) 2037 : cluster 0 osdmap e358: 8 total, 8 up, 8 in 2023-10-01T11:09:22.902 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:22.935 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:22.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:22.938 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:09:23.039 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:23.070 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:23.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:23.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:09:23.173 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:23.199 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:23.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:23.203 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:09:23.304 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:23.332 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:23.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:23.334 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:09:23.435 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:23.465 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:23.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:23.468 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:09:23.569 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:23.601 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:23.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:23.603 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:23.695 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:09:23.696 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:09:23.704 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:23.731 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:23.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:23.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:09:23.835 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:23.847 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:23 smithi173 bash[58816]: cluster 2023-10-01T11:09:22.150064+0000 osd.5 (osd.5) 380 : cluster 3 failed to encode map e358 with expected crc 2023-10-01T11:09:23.847 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:23 smithi173 bash[58816]: cluster 2023-10-01T11:09:22.374912+0000 mgr.y (mgr.25072) 1562 : cluster 0 pgmap v1682: 89 pgs: 3 creating+peering, 13 unknown, 73 active+clean; 582 KiB data, 8.0 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:23.847 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:23 smithi173 bash[58816]: cluster 2023-10-01T11:09:23.202461+0000 osd.6 (osd.6) 342 : cluster 3 failed to encode map e358 with expected crc 2023-10-01T11:09:23.847 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:23 smithi173 bash[58816]: cluster 2023-10-01T11:09:23.202639+0000 osd.7 (osd.7) 401 : cluster 3 failed to encode map e358 with expected crc 2023-10-01T11:09:23.867 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:23.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:23.870 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:09:23.970 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:24.002 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:24.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:24.004 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:09:24.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:23 smithi078 bash[75299]: cluster 2023-10-01T11:09:22.150064+0000 osd.5 (osd.5) 380 : cluster 3 failed to encode map e358 with expected crc 2023-10-01T11:09:24.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:23 smithi078 bash[75299]: cluster 2023-10-01T11:09:22.374912+0000 mgr.y (mgr.25072) 1562 : cluster 0 pgmap v1682: 89 pgs: 3 creating+peering, 13 unknown, 73 active+clean; 582 KiB data, 8.0 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:24.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:23 smithi078 bash[75299]: cluster 2023-10-01T11:09:23.202461+0000 osd.6 (osd.6) 342 : cluster 3 failed to encode map e358 with expected crc 2023-10-01T11:09:24.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:23 smithi078 bash[75299]: cluster 2023-10-01T11:09:23.202639+0000 osd.7 (osd.7) 401 : cluster 3 failed to encode map e358 with expected crc 2023-10-01T11:09:24.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:23 smithi078 bash[78274]: cluster 2023-10-01T11:09:22.150064+0000 osd.5 (osd.5) 380 : cluster 3 failed to encode map e358 with expected crc 2023-10-01T11:09:24.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:23 smithi078 bash[78274]: cluster 2023-10-01T11:09:22.374912+0000 mgr.y (mgr.25072) 1562 : cluster 0 pgmap v1682: 89 pgs: 3 creating+peering, 13 unknown, 73 active+clean; 582 KiB data, 8.0 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:24.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:23 smithi078 bash[78274]: cluster 2023-10-01T11:09:23.202461+0000 osd.6 (osd.6) 342 : cluster 3 failed to encode map e358 with expected crc 2023-10-01T11:09:24.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:23 smithi078 bash[78274]: cluster 2023-10-01T11:09:23.202639+0000 osd.7 (osd.7) 401 : cluster 3 failed to encode map e358 with expected crc 2023-10-01T11:09:24.105 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:24.139 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:24.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:24.143 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:24.244 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:24.280 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:24.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:24.283 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:24.384 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:24.417 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:24.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:24.421 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:09:24.522 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:24.548 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:24.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:24.551 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:24.651 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:24.679 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:24.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:24.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:09:24.709 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:09:24.783 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:24.816 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:24.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:24.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:24.920 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:24.946 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:24.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:24.948 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:09:25.049 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:25.077 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:25.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:25.080 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:09:25.181 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:25.209 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:25.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:25.211 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:09:25.312 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:25.340 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:25.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:25.342 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:09:25.443 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:25.468 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:25.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:25.472 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:09:25.572 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:25.605 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:25.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:25.608 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:09:25.708 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:25.719 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:09:25.731 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:25.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:25.733 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:09:25.834 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:25.860 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:25.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:25.863 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:09:25.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:25 smithi173 bash[58816]: cluster 2023-10-01T11:09:24.375939+0000 mgr.y (mgr.25072) 1563 : cluster 0 pgmap v1683: 89 pgs: 3 creating+peering, 2 unknown, 84 active+clean; 582 KiB data, 8.0 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:25.964 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:25.998 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:26.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:26.002 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:09:26.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:25 smithi078 bash[78274]: cluster 2023-10-01T11:09:24.375939+0000 mgr.y (mgr.25072) 1563 : cluster 0 pgmap v1683: 89 pgs: 3 creating+peering, 2 unknown, 84 active+clean; 582 KiB data, 8.0 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:26.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:25 smithi078 bash[75299]: cluster 2023-10-01T11:09:24.375939+0000 mgr.y (mgr.25072) 1563 : cluster 0 pgmap v1683: 89 pgs: 3 creating+peering, 2 unknown, 84 active+clean; 582 KiB data, 8.0 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:26.103 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:26.138 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:26.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:26.142 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:09:26.243 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:26.272 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:26.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:26.275 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:26.376 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:26.397 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:26.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:26.399 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:09:26.500 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:26.525 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:26.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:26.527 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:09:26.628 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:26.663 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:26.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:26.667 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:26.768 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:26.796 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:26.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:26.798 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:09:26.898 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:26.927 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:26.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:26.931 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:09:27.032 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:27.060 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:27.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:27.063 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:09:27.163 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:27.193 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:27.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:27.196 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:09:27.297 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:27.322 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:27.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:27.325 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:09:27.426 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:27.450 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:27.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:27.453 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:09:27.553 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:27.589 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:27.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:27.592 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:09:27.692 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:27.719 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:27.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:27.721 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:09:27.822 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:27.846 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:27.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:27.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:09:27.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:27 smithi173 bash[58816]: cluster 2023-10-01T11:09:26.376608+0000 mgr.y (mgr.25072) 1564 : cluster 0 pgmap v1684: 89 pgs: 89 active+clean; 582 KiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:27.949 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:27.957 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:27 smithi078 bash[78274]: cluster 2023-10-01T11:09:26.376608+0000 mgr.y (mgr.25072) 1564 : cluster 0 pgmap v1684: 89 pgs: 89 active+clean; 582 KiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:27.958 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:27 smithi078 bash[75299]: cluster 2023-10-01T11:09:26.376608+0000 mgr.y (mgr.25072) 1564 : cluster 0 pgmap v1684: 89 pgs: 89 active+clean; 582 KiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:27.974 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:27.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:27.977 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:09:28.078 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:28.104 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:28.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:28.107 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:09:28.208 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:28.241 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:28.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:28.244 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:28.345 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:28.376 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:28.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:28.380 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:09:28.481 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:28.507 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:28.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:28.510 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:09:28.610 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:28.639 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:28.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:28.641 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:09:28.742 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:28.763 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:28.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:28.767 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:09:28.867 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:28.901 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:28.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:28.904 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:29.005 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:29.038 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:29.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:29.041 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:09:29.142 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:29.167 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:29.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:29.169 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:09:29.270 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:29.305 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:29.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:29.308 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:09:29.409 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:29.430 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:29.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:29.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:09:29.533 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:29.565 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:29.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:29.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:09:29.668 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:29.701 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:29.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:29.704 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:09:29.805 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:29.837 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:29.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:29.840 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:09:29.941 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:29.974 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:29.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:29.978 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:09:30.078 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:30.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:29 smithi078 bash[75299]: cluster 2023-10-01T11:09:28.377059+0000 mgr.y (mgr.25072) 1565 : cluster 0 pgmap v1685: 89 pgs: 89 active+clean; 582 KiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:30.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:29 smithi078 bash[78274]: cluster 2023-10-01T11:09:28.377059+0000 mgr.y (mgr.25072) 1565 : cluster 0 pgmap v1685: 89 pgs: 89 active+clean; 582 KiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:30.108 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:30.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:30.111 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:09:30.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:29 smithi173 bash[58816]: cluster 2023-10-01T11:09:28.377059+0000 mgr.y (mgr.25072) 1565 : cluster 0 pgmap v1685: 89 pgs: 89 active+clean; 582 KiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:30.212 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:30.238 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:30.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:30.241 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:09:30.341 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:30.355 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:09:30 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:09:30] "GET /metrics HTTP/1.1" 200 33966 "" "Prometheus/2.33.4" 2023-10-01T11:09:30.376 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:30.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:30.379 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:09:30.480 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:30.510 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:30.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:30.513 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:09:30.613 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:30.642 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:30.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:30.645 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:09:30.746 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:30.778 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:30.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:30.781 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:09:30.882 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:30.915 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:30.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:30.918 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:09:31.019 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:31.050 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:31.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:31.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:31.153 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:31.185 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:31.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:31.187 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:09:31.288 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:31.323 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:31.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:31.326 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:09:31.427 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:31.452 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:31.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:31.454 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:09:31.555 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:31.589 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:31.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:31.592 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:09:31.693 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:31.724 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:31.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:31.727 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:31.828 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:31.861 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:31.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:31.864 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:09:31.965 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:31.997 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:32.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:32.000 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:09:32.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:31 smithi078 bash[75299]: cluster 2023-10-01T11:09:30.378409+0000 mgr.y (mgr.25072) 1566 : cluster 0 pgmap v1686: 89 pgs: 89 active+clean; 582 KiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:32.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:31 smithi078 bash[78274]: cluster 2023-10-01T11:09:30.378409+0000 mgr.y (mgr.25072) 1566 : cluster 0 pgmap v1686: 89 pgs: 89 active+clean; 582 KiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:32.102 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:32.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:31 smithi173 bash[58816]: cluster 2023-10-01T11:09:30.378409+0000 mgr.y (mgr.25072) 1566 : cluster 0 pgmap v1686: 89 pgs: 89 active+clean; 582 KiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:32.130 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:32.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:32.133 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:09:32.234 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:32.266 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:32.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:32.270 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:09:32.370 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:32.400 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:32.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:32.405 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:32.505 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:32.529 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:32.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:32.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:09:32.631 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:32.661 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:32.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:32.663 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:09:32.764 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:32.794 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:32.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:32.797 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:32.898 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:32.910 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:32 smithi173 bash[58816]: audit 2023-10-01T11:09:32.465621+0000 mon.a (mon.0) 2038 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:09:32.930 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:32.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:32.933 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:09:33.033 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:33.046 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:32 smithi078 bash[75299]: audit 2023-10-01T11:09:32.465621+0000 mon.a (mon.0) 2038 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:09:33.046 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:32 smithi078 bash[78274]: audit 2023-10-01T11:09:32.465621+0000 mon.a (mon.0) 2038 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:09:33.063 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:33.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:33.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:09:33.166 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:33.194 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:33.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:33.197 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:09:33.298 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:33.329 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:33.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:33.332 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:09:33.434 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:33.465 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:33.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:33.468 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:33.569 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:33.600 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:33.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:33.603 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:09:33.704 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:33.731 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:33.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:33.733 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:33.834 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:33.858 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:33.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:33.861 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:09:33.961 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:33.995 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:33.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:33.997 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:09:34.087 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:33 smithi078 bash[75299]: cluster 2023-10-01T11:09:32.379305+0000 mgr.y (mgr.25072) 1567 : cluster 0 pgmap v1687: 89 pgs: 89 active+clean; 582 KiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:34.088 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:33 smithi078 bash[75299]: audit 2023-10-01T11:09:33.347103+0000 mon.a (mon.0) 2039 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:09:34.088 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:33 smithi078 bash[78274]: cluster 2023-10-01T11:09:32.379305+0000 mgr.y (mgr.25072) 1567 : cluster 0 pgmap v1687: 89 pgs: 89 active+clean; 582 KiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:34.088 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:33 smithi078 bash[78274]: audit 2023-10-01T11:09:33.347103+0000 mon.a (mon.0) 2039 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:09:34.098 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:34.111 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:33 smithi173 bash[58816]: cluster 2023-10-01T11:09:32.379305+0000 mgr.y (mgr.25072) 1567 : cluster 0 pgmap v1687: 89 pgs: 89 active+clean; 582 KiB data, 7.9 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:34.111 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:33 smithi173 bash[58816]: audit 2023-10-01T11:09:33.347103+0000 mon.a (mon.0) 2039 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:09:34.131 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:34.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:34.134 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:34.235 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:34.267 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:34.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:34.270 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:09:34.371 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:34.401 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:34.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:34.405 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:09:34.505 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:34.537 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:34.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:34.540 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:09:34.641 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:34.674 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:34.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:34.678 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:09:34.779 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:34.808 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:34.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:34.810 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:09:34.911 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:34.938 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:34.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:34.941 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:09:35.042 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:35.072 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:35.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:35.076 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:09:35.177 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:35.212 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:35.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:35.215 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:35.316 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:35.342 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:35.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:35.344 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:09:35.445 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:35.478 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:35.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:35.480 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:09:35.581 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:35.611 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:35.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:35.614 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:09:35.715 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:35.736 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:35.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:35.738 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:09:35.839 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:35.869 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:35.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:35.872 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:09:35.973 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:36.008 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:36.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:36.011 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:09:36.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:35 smithi078 bash[75299]: cluster 2023-10-01T11:09:34.380179+0000 mgr.y (mgr.25072) 1568 : cluster 0 pgmap v1688: 89 pgs: 89 active+clean; 582 KiB data, 7.8 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:36.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:35 smithi078 bash[78274]: cluster 2023-10-01T11:09:34.380179+0000 mgr.y (mgr.25072) 1568 : cluster 0 pgmap v1688: 89 pgs: 89 active+clean; 582 KiB data, 7.8 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:36.112 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:36.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:35 smithi173 bash[58816]: cluster 2023-10-01T11:09:34.380179+0000 mgr.y (mgr.25072) 1568 : cluster 0 pgmap v1688: 89 pgs: 89 active+clean; 582 KiB data, 7.8 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:36.142 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:36.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:36.145 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:09:36.245 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:36.275 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:36.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:36.277 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:09:36.378 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:36.402 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:36.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:36.404 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:09:36.505 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:36.538 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:36.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:36.541 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:09:36.642 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:36.674 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:36.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:36.677 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:09:36.777 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:36.809 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:36.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:36.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:09:36.912 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:36.938 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:36.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:36.940 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:09:36.961 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T11:09:36.983 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T11:09:37.010 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T11:09:37.034 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T11:09:37.041 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:37.059 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T11:09:37.066 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:37.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:37.069 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:09:37.087 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T11:09:37.114 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T11:09:37.139 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T11:09:37.141 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T11:09:37.141 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T11:09:37.141 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T11:09:37.141 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T11:09:37.141 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T11:09:37.141 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T11:09:37.141 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T11:09:37.141 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T11:09:37.171 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:37.202 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:37.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:37.205 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:09:37.306 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:37.327 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:37.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:37.329 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:09:37.430 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:37.463 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:37.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:37.466 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:09:37.567 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:37.599 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:37.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:37.603 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:37.704 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:37.723 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:37.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:37.725 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:09:37.826 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:37.835 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2023-10-01T11:09:37.835 INFO:teuthology.orchestra.run.smithi078.stdout: "size": 20, 2023-10-01T11:09:37.835 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 600, 2023-10-01T11:09:37.835 INFO:teuthology.orchestra.run.smithi078.stdout: "ops": [ 2023-10-01T11:09:37.835 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.835 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46232.0:191212 12.9 12:9b1b517c:::benchmark_data_smithi078_180197_object88081:head [delete] snapc 0=[] ondisk+write+known_if_redirected e255)", 2023-10-01T11:09:37.835 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:00:28.827360+0000", 2023-10-01T11:09:37.835 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 549.004271217, 2023-10-01T11:09:37.836 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.459188594, 2023-10-01T11:09:37.836 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:09:37.836 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:09:37.836 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T11:09:37.836 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46232", 2023-10-01T11:09:37.836 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/711982100", 2023-10-01T11:09:37.836 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 191212 2023-10-01T11:09:37.836 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.836 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:09:37.836 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.836 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:09:37.837 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:00:28.827360+0000", 2023-10-01T11:09:37.837 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:09:37.837 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.837 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.837 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:09:37.837 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:00:28.827360+0000", 2023-10-01T11:09:37.837 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:09:37.837 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.837 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.837 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:09:37.837 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:00:28.827358+0000", 2023-10-01T11:09:37.837 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T11:09:37.838 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.838 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.838 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:09:37.838 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:00:28.827363+0000", 2023-10-01T11:09:37.838 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.9169999999999997e-06 2023-10-01T11:09:37.838 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.838 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.838 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:09:37.838 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:00:28.827373+0000", 2023-10-01T11:09:37.838 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.2259999999999994e-06 2023-10-01T11:09:37.838 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.838 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.839 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:09:37.839 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:00:28.827378+0000", 2023-10-01T11:09:37.839 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.5990000000000001e-06 2023-10-01T11:09:37.839 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.839 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.839 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:09:37.839 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:00:28.827430+0000", 2023-10-01T11:09:37.839 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.1894000000000002e-05 2023-10-01T11:09:37.839 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.839 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.839 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T11:09:37.839 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:00:28.827441+0000", 2023-10-01T11:09:37.840 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.0597e-05 2023-10-01T11:09:37.840 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.840 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.840 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:09:37.840 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:00:29.824220+0000", 2023-10-01T11:09:37.840 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.99677930699999995 2023-10-01T11:09:37.840 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.840 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.840 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T11:09:37.840 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:00:29.824260+0000", 2023-10-01T11:09:37.840 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.0246000000000001e-05 2023-10-01T11:09:37.841 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.841 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.841 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:09:37.841 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:00:31.087634+0000", 2023-10-01T11:09:37.841 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.2633734620000001 2023-10-01T11:09:37.841 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.841 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.841 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:09:37.841 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:00:31.087732+0000", 2023-10-01T11:09:37.841 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.8657000000000001e-05 2023-10-01T11:09:37.841 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.841 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.841 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 1,3,6", 2023-10-01T11:09:37.842 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:00:31.087826+0000", 2023-10-01T11:09:37.842 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.3548000000000006e-05 2023-10-01T11:09:37.842 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.842 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.842 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T11:09:37.842 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:00:31.092494+0000", 2023-10-01T11:09:37.842 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0046685559999999999 2023-10-01T11:09:37.842 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.842 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.842 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:09:37.842 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:00:31.092533+0000", 2023-10-01T11:09:37.842 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.8578000000000002e-05 2023-10-01T11:09:37.843 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.843 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.843 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:09:37.843 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:00:31.094593+0000", 2023-10-01T11:09:37.843 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0020599279999999999 2023-10-01T11:09:37.843 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.843 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.843 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:09:37.843 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:00:31.286428+0000", 2023-10-01T11:09:37.843 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.19183481499999999 2023-10-01T11:09:37.843 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.844 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.844 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:09:37.844 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:00:31.286465+0000", 2023-10-01T11:09:37.844 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.7008000000000002e-05 2023-10-01T11:09:37.844 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.844 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.844 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:09:37.844 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:00:31.286548+0000", 2023-10-01T11:09:37.844 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.3321999999999999e-05 2023-10-01T11:09:37.844 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:09:37.844 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:09:37.844 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:09:37.845 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.845 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.845 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46454.0:73732 13.1 13:8b7358d4:::benchmark_data_smithi078_202524_object73731:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e299)", 2023-10-01T11:09:37.845 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:03:06.964378+0000", 2023-10-01T11:09:37.845 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 390.86725325499998, 2023-10-01T11:09:37.845 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.1158809710000002, 2023-10-01T11:09:37.845 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:09:37.845 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:09:37.845 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T11:09:37.846 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46454", 2023-10-01T11:09:37.846 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:09:37.846 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 73732 2023-10-01T11:09:37.846 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.846 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:09:37.846 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.846 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:09:37.846 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:06.964378+0000", 2023-10-01T11:09:37.846 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:09:37.846 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.846 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.847 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:09:37.847 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:06.964378+0000", 2023-10-01T11:09:37.847 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:09:37.847 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.847 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.847 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:09:37.847 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:06.964376+0000", 2023-10-01T11:09:37.847 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999981 2023-10-01T11:09:37.847 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.847 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.847 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:09:37.847 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:06.964402+0000", 2023-10-01T11:09:37.848 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.6227000000000001e-05 2023-10-01T11:09:37.848 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.848 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.848 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:09:37.848 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:06.964416+0000", 2023-10-01T11:09:37.848 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.3872e-05 2023-10-01T11:09:37.848 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.848 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.848 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:09:37.848 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:06.964425+0000", 2023-10-01T11:09:37.848 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.3100000000000006e-06 2023-10-01T11:09:37.848 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.849 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.849 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:09:37.849 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:06.964470+0000", 2023-10-01T11:09:37.849 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.4929000000000003e-05 2023-10-01T11:09:37.849 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.849 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.849 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T11:09:37.849 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:06.964479+0000", 2023-10-01T11:09:37.849 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.5690000000000005e-06 2023-10-01T11:09:37.849 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.849 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.849 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:09:37.850 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:07.714946+0000", 2023-10-01T11:09:37.850 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.75046762300000003 2023-10-01T11:09:37.850 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.850 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.850 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T11:09:37.850 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:07.714963+0000", 2023-10-01T11:09:37.850 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.6668e-05 2023-10-01T11:09:37.850 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.850 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.850 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:09:37.850 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:09.060576+0000", 2023-10-01T11:09:37.850 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.345612627 2023-10-01T11:09:37.850 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.851 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.851 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:09:37.851 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:09.062595+0000", 2023-10-01T11:09:37.851 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0020195389999999999 2023-10-01T11:09:37.851 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.851 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.851 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 4,7", 2023-10-01T11:09:37.851 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:09.062786+0000", 2023-10-01T11:09:37.851 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00019063700000000001 2023-10-01T11:09:37.851 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.851 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.851 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:09:37.852 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:09.069879+0000", 2023-10-01T11:09:37.852 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0070927530000000003 2023-10-01T11:09:37.852 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.852 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.852 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T11:09:37.852 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:09.071556+0000", 2023-10-01T11:09:37.852 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.001677254 2023-10-01T11:09:37.852 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.852 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.852 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:09:37.852 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:09.077672+0000", 2023-10-01T11:09:37.852 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0061164449999999999 2023-10-01T11:09:37.853 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.853 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.853 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:09:37.853 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:09.077718+0000", 2023-10-01T11:09:37.853 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.5494000000000002e-05 2023-10-01T11:09:37.853 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.853 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.853 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:09:37.853 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:09.080258+0000", 2023-10-01T11:09:37.853 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0025407210000000001 2023-10-01T11:09:37.853 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:09:37.853 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:09:37.854 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:09:37.854 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.854 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.854 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46454.0:73733 13.1 13:84a02ffb:::benchmark_data_smithi078_202524_object73732:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e299)", 2023-10-01T11:09:37.854 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:03:06.966016+0000", 2023-10-01T11:09:37.854 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 390.86561476899999, 2023-10-01T11:09:37.854 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.1164666329999999, 2023-10-01T11:09:37.854 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:09:37.854 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:09:37.854 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T11:09:37.854 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46454", 2023-10-01T11:09:37.855 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:09:37.855 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 73733 2023-10-01T11:09:37.855 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.855 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:09:37.855 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.855 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:09:37.855 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:06.966016+0000", 2023-10-01T11:09:37.855 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:09:37.855 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.855 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.855 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:09:37.855 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:06.966016+0000", 2023-10-01T11:09:37.856 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:09:37.856 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.856 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.856 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:09:37.856 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:06.966015+0000", 2023-10-01T11:09:37.856 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999986 2023-10-01T11:09:37.856 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.856 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.856 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:09:37.856 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:06.966037+0000", 2023-10-01T11:09:37.856 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.1982999999999999e-05 2023-10-01T11:09:37.856 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.856 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.857 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:09:37.857 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:06.966049+0000", 2023-10-01T11:09:37.857 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.2696e-05 2023-10-01T11:09:37.857 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.857 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.857 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:09:37.857 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:06.966056+0000", 2023-10-01T11:09:37.857 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.8569999999999997e-06 2023-10-01T11:09:37.857 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.857 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.857 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:09:37.857 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:06.966197+0000", 2023-10-01T11:09:37.858 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00014104499999999999 2023-10-01T11:09:37.858 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.858 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.858 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T11:09:37.858 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:06.966209+0000", 2023-10-01T11:09:37.858 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1476e-05 2023-10-01T11:09:37.858 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.858 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.858 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:09:37.858 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:07.714994+0000", 2023-10-01T11:09:37.858 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.74878555700000005 2023-10-01T11:09:37.859 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.859 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.859 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T11:09:37.859 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:07.715003+0000", 2023-10-01T11:09:37.859 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.0929999999999992e-06 2023-10-01T11:09:37.859 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.859 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.859 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:09:37.859 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:09.067482+0000", 2023-10-01T11:09:37.859 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.352478947 2023-10-01T11:09:37.859 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.860 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.860 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:09:37.860 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:09.067612+0000", 2023-10-01T11:09:37.860 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000129689 2023-10-01T11:09:37.860 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.860 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.860 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 4,7", 2023-10-01T11:09:37.860 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:09.067751+0000", 2023-10-01T11:09:37.860 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00013877200000000001 2023-10-01T11:09:37.860 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.860 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.861 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T11:09:37.861 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:09.071563+0000", 2023-10-01T11:09:37.861 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.003811924 2023-10-01T11:09:37.861 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.861 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.861 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:09:37.861 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:09.075668+0000", 2023-10-01T11:09:37.861 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0041049379999999998 2023-10-01T11:09:37.861 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.861 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.861 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:09:37.861 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:09.080305+0000", 2023-10-01T11:09:37.862 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0046376220000000001 2023-10-01T11:09:37.862 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.862 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.862 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:09:37.862 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:09.080342+0000", 2023-10-01T11:09:37.862 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.7224999999999997e-05 2023-10-01T11:09:37.862 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.862 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.862 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:09:37.862 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:09.082483+0000", 2023-10-01T11:09:37.862 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0021401599999999999 2023-10-01T11:09:37.862 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:09:37.863 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:09:37.863 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:09:37.863 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.863 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.863 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46454.0:73715 13.f 13:ff65771b:::benchmark_data_smithi078_202524_object73714:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e300)", 2023-10-01T11:09:37.863 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:03:07.716959+0000", 2023-10-01T11:09:37.863 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 390.114672068, 2023-10-01T11:09:37.863 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.7499220649999998, 2023-10-01T11:09:37.863 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:09:37.863 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "reached pg", 2023-10-01T11:09:37.863 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T11:09:37.864 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46454", 2023-10-01T11:09:37.864 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:09:37.864 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 73715 2023-10-01T11:09:37.864 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.864 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:09:37.864 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.864 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:09:37.864 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:07.716959+0000", 2023-10-01T11:09:37.864 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:09:37.865 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.865 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.865 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:09:37.865 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:07.716959+0000", 2023-10-01T11:09:37.865 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:09:37.865 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.865 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.865 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:09:37.865 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:07.716956+0000", 2023-10-01T11:09:37.865 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999976 2023-10-01T11:09:37.866 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.866 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.866 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:09:37.866 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:07.717039+0000", 2023-10-01T11:09:37.866 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.2498999999999997e-05 2023-10-01T11:09:37.866 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.866 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.866 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:09:37.866 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:07.717055+0000", 2023-10-01T11:09:37.867 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.5832999999999999e-05 2023-10-01T11:09:37.867 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.867 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.867 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:09:37.867 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:07.717068+0000", 2023-10-01T11:09:37.867 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.3325e-05 2023-10-01T11:09:37.867 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.867 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.867 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:09:37.868 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:07.717335+0000", 2023-10-01T11:09:37.868 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00026727199999999998 2023-10-01T11:09:37.868 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.868 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.868 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T11:09:37.868 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:07.717349+0000", 2023-10-01T11:09:37.868 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.3634e-05 2023-10-01T11:09:37.868 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.868 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.869 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:09:37.869 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.458852+0000", 2023-10-01T11:09:37.869 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.7415029579999999 2023-10-01T11:09:37.869 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.869 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.869 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:09:37.869 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.466881+0000", 2023-10-01T11:09:37.869 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0080287529999999996 2023-10-01T11:09:37.869 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:09:37.869 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:09:37.870 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:09:37.870 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.870 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.870 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46454.0:73717 13.f 13:f618ea28:::benchmark_data_smithi078_202524_object73716:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e300)", 2023-10-01T11:09:37.870 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:03:07.717086+0000", 2023-10-01T11:09:37.870 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 390.11454503300001, 2023-10-01T11:09:37.870 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 10.012722919, 2023-10-01T11:09:37.870 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:09:37.870 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "reached pg", 2023-10-01T11:09:37.870 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T11:09:37.870 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46454", 2023-10-01T11:09:37.871 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:09:37.871 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 73717 2023-10-01T11:09:37.871 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.871 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:09:37.871 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.871 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:09:37.871 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:07.717086+0000", 2023-10-01T11:09:37.871 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:09:37.871 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.871 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.871 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:09:37.871 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:07.717086+0000", 2023-10-01T11:09:37.872 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:09:37.872 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.872 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.872 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:09:37.872 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:07.717085+0000", 2023-10-01T11:09:37.872 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T11:09:37.872 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.872 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.872 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:09:37.872 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:07.717142+0000", 2023-10-01T11:09:37.872 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.6974e-05 2023-10-01T11:09:37.873 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.873 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.873 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:09:37.873 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:07.717153+0000", 2023-10-01T11:09:37.873 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.129e-05 2023-10-01T11:09:37.873 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.873 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.873 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:09:37.873 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:07.717161+0000", 2023-10-01T11:09:37.873 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.6159999999999996e-06 2023-10-01T11:09:37.873 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.874 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.874 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:09:37.874 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:07.717374+0000", 2023-10-01T11:09:37.874 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00021338999999999999 2023-10-01T11:09:37.874 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.874 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.874 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T11:09:37.874 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:07.717380+0000", 2023-10-01T11:09:37.874 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.5330000000000002e-06 2023-10-01T11:09:37.874 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.875 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.875 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:09:37.875 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.483633+0000", 2023-10-01T11:09:37.875 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.7662528329999994 2023-10-01T11:09:37.875 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.875 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.875 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for missing object", 2023-10-01T11:09:37.875 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.489528+0000", 2023-10-01T11:09:37.875 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0058951129999999996 2023-10-01T11:09:37.875 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.875 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.876 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:09:37.876 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.536484+0000", 2023-10-01T11:09:37.876 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.046956789999999998 2023-10-01T11:09:37.876 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.876 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.876 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for degraded object", 2023-10-01T11:09:37.876 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.539537+0000", 2023-10-01T11:09:37.876 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0030522069999999999 2023-10-01T11:09:37.876 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.876 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.876 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:09:37.877 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.729709+0000", 2023-10-01T11:09:37.877 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.19017236500000001 2023-10-01T11:09:37.877 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.877 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.877 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:09:37.877 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.729809+0000", 2023-10-01T11:09:37.877 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.9592999999999993e-05 2023-10-01T11:09:37.877 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:09:37.877 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:09:37.878 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:09:37.878 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.878 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.878 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46454.0:73721 13.f 13:f95d23cf:::benchmark_data_smithi078_202524_object73720:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e300)", 2023-10-01T11:09:37.878 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:03:07.717167+0000", 2023-10-01T11:09:37.878 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 390.114463306, 2023-10-01T11:09:37.878 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 10.015407279, 2023-10-01T11:09:37.878 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:09:37.878 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "reached pg", 2023-10-01T11:09:37.878 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T11:09:37.879 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46454", 2023-10-01T11:09:37.879 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:09:37.879 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 73721 2023-10-01T11:09:37.879 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.879 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:09:37.879 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.879 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:09:37.879 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:07.717167+0000", 2023-10-01T11:09:37.879 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:09:37.879 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.879 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.880 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:09:37.880 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:07.717167+0000", 2023-10-01T11:09:37.880 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:09:37.880 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.880 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.880 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:09:37.880 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:07.717167+0000", 2023-10-01T11:09:37.880 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999995 2023-10-01T11:09:37.880 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.880 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.880 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:09:37.881 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:07.717196+0000", 2023-10-01T11:09:37.881 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.9689999999999999e-05 2023-10-01T11:09:37.881 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.881 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.881 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:09:37.881 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:07.717205+0000", 2023-10-01T11:09:37.881 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.704e-06 2023-10-01T11:09:37.881 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.881 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.881 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:09:37.881 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:07.717213+0000", 2023-10-01T11:09:37.882 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.6380000000000004e-06 2023-10-01T11:09:37.882 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.882 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.882 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:09:37.882 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:07.717394+0000", 2023-10-01T11:09:37.882 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00018082800000000001 2023-10-01T11:09:37.882 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.882 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.882 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T11:09:37.882 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:07.717398+0000", 2023-10-01T11:09:37.882 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.5689999999999998e-06 2023-10-01T11:09:37.883 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.883 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.883 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:09:37.883 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.490074+0000", 2023-10-01T11:09:37.883 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.7726762069999999 2023-10-01T11:09:37.883 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.883 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.883 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for missing object", 2023-10-01T11:09:37.883 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.495202+0000", 2023-10-01T11:09:37.883 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0051278770000000003 2023-10-01T11:09:37.883 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.884 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.884 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:09:37.884 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.580077+0000", 2023-10-01T11:09:37.884 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.084874359999999996 2023-10-01T11:09:37.884 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.884 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.884 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for degraded object", 2023-10-01T11:09:37.884 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.580129+0000", 2023-10-01T11:09:37.884 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.2219000000000003e-05 2023-10-01T11:09:37.884 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.884 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.884 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:09:37.885 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.732425+0000", 2023-10-01T11:09:37.885 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.152295875 2023-10-01T11:09:37.885 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.885 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.885 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:09:37.885 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.732575+0000", 2023-10-01T11:09:37.885 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000150018 2023-10-01T11:09:37.885 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:09:37.885 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:09:37.885 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:09:37.885 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.885 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.886 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46454.0:73756 13.f 13:fa15e9a1:::benchmark_data_smithi078_202524_object73755:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e300)", 2023-10-01T11:09:37.886 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:03:07.717331+0000", 2023-10-01T11:09:37.886 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 390.11429978400002, 2023-10-01T11:09:37.886 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.8515626869999995, 2023-10-01T11:09:37.886 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:09:37.886 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:09:37.886 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T11:09:37.886 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46454", 2023-10-01T11:09:37.886 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:09:37.886 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 73756 2023-10-01T11:09:37.886 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.887 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:09:37.887 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.887 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:09:37.887 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:07.717331+0000", 2023-10-01T11:09:37.887 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:09:37.887 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.887 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.887 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:09:37.887 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:07.717331+0000", 2023-10-01T11:09:37.887 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:09:37.887 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.887 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.888 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:09:37.888 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:07.717330+0000", 2023-10-01T11:09:37.888 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T11:09:37.888 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.888 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.888 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:09:37.888 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:07.717367+0000", 2023-10-01T11:09:37.888 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.7428000000000001e-05 2023-10-01T11:09:37.888 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.888 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.888 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:09:37.888 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:07.717383+0000", 2023-10-01T11:09:37.888 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.5420000000000001e-05 2023-10-01T11:09:37.889 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.889 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.889 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:09:37.889 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:07.717391+0000", 2023-10-01T11:09:37.889 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.4719999999999995e-06 2023-10-01T11:09:37.889 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.889 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.889 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:09:37.889 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:07.717411+0000", 2023-10-01T11:09:37.889 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.9979e-05 2023-10-01T11:09:37.890 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.890 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.890 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T11:09:37.890 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:07.717416+0000", 2023-10-01T11:09:37.890 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.6569999999999996e-06 2023-10-01T11:09:37.890 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.890 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.890 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:09:37.890 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.495732+0000", 2023-10-01T11:09:37.890 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.7783158750000005 2023-10-01T11:09:37.891 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.891 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.891 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:09:37.891 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.501597+0000", 2023-10-01T11:09:37.891 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0058649059999999996 2023-10-01T11:09:37.891 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.891 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.891 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 5,7", 2023-10-01T11:09:37.891 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.510987+0000", 2023-10-01T11:09:37.891 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0093899280000000005 2023-10-01T11:09:37.892 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.892 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.892 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:09:37.892 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.542050+0000", 2023-10-01T11:09:37.892 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.031063464999999998 2023-10-01T11:09:37.892 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.892 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.892 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:09:37.892 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.543643+0000", 2023-10-01T11:09:37.892 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00159329 2023-10-01T11:09:37.892 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.892 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.893 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T11:09:37.893 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.555960+0000", 2023-10-01T11:09:37.893 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.012316817000000001 2023-10-01T11:09:37.893 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.893 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.893 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:09:37.893 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.558405+0000", 2023-10-01T11:09:37.893 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0024449839999999999 2023-10-01T11:09:37.893 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.893 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.894 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:09:37.894 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.568894+0000", 2023-10-01T11:09:37.894 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.010488409000000001 2023-10-01T11:09:37.894 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:09:37.894 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:09:37.894 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:09:37.894 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.894 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.894 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46454.0:73765 13.f 13:f6ea2151:::benchmark_data_smithi078_202524_object73764:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e301)", 2023-10-01T11:09:37.894 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:03:09.049299+0000", 2023-10-01T11:09:37.894 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 388.78233174799999, 2023-10-01T11:09:37.895 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.5514202319999999, 2023-10-01T11:09:37.895 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:09:37.895 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:09:37.895 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T11:09:37.895 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46454", 2023-10-01T11:09:37.895 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:09:37.895 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 73765 2023-10-01T11:09:37.895 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.895 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:09:37.895 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.895 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:09:37.896 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:09.049299+0000", 2023-10-01T11:09:37.896 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:09:37.896 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.896 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.896 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:09:37.896 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:09.049299+0000", 2023-10-01T11:09:37.896 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:09:37.896 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.896 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.896 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:09:37.896 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:09.049297+0000", 2023-10-01T11:09:37.896 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999981 2023-10-01T11:09:37.897 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.897 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.897 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:09:37.897 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:09.049325+0000", 2023-10-01T11:09:37.897 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.8201999999999999e-05 2023-10-01T11:09:37.897 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.897 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.897 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:09:37.897 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:09.049340+0000", 2023-10-01T11:09:37.897 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.4243999999999999e-05 2023-10-01T11:09:37.897 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.897 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.898 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:09:37.898 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:09.049348+0000", 2023-10-01T11:09:37.898 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.2919999999999996e-06 2023-10-01T11:09:37.898 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.898 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.898 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:09:37.898 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.580165+0000", 2023-10-01T11:09:37.898 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.5308173949999997 2023-10-01T11:09:37.898 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.898 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.898 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:09:37.899 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.580335+0000", 2023-10-01T11:09:37.899 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00016954200000000001 2023-10-01T11:09:37.899 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.899 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.899 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 5,7", 2023-10-01T11:09:37.899 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.580543+0000", 2023-10-01T11:09:37.899 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00020849200000000001 2023-10-01T11:09:37.899 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.899 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.899 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T11:09:37.899 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.591338+0000", 2023-10-01T11:09:37.900 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.010794125 2023-10-01T11:09:37.900 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.900 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.900 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:09:37.900 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.598888+0000", 2023-10-01T11:09:37.900 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0075503590000000004 2023-10-01T11:09:37.900 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.900 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.900 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:09:37.900 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.598931+0000", 2023-10-01T11:09:37.900 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.2806000000000003e-05 2023-10-01T11:09:37.901 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.901 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.901 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:09:37.901 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.598968+0000", 2023-10-01T11:09:37.901 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.7135000000000003e-05 2023-10-01T11:09:37.901 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.901 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.901 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:09:37.901 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.600719+0000", 2023-10-01T11:09:37.901 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0017514480000000001 2023-10-01T11:09:37.901 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:09:37.902 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:09:37.902 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:09:37.902 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.902 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.902 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46454.0:73782 13.f 13:faf103d8:::benchmark_data_smithi078_202524_object73781:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e301)", 2023-10-01T11:09:37.902 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:03:09.113994+0000", 2023-10-01T11:09:37.902 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 388.71763671100001, 2023-10-01T11:09:37.902 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.4888464199999998, 2023-10-01T11:09:37.902 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:09:37.902 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:09:37.902 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T11:09:37.903 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46454", 2023-10-01T11:09:37.903 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:09:37.903 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 73782 2023-10-01T11:09:37.903 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.903 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:09:37.903 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.903 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:09:37.903 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:09.113994+0000", 2023-10-01T11:09:37.903 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:09:37.903 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.904 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.904 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:09:37.904 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:09.113994+0000", 2023-10-01T11:09:37.904 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:09:37.904 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.904 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.904 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:09:37.904 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:09.113992+0000", 2023-10-01T11:09:37.904 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999981 2023-10-01T11:09:37.904 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.904 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.904 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:09:37.905 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:09.114029+0000", 2023-10-01T11:09:37.905 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.7376999999999998e-05 2023-10-01T11:09:37.905 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.905 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.905 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:09:37.905 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:09.114043+0000", 2023-10-01T11:09:37.905 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.382e-05 2023-10-01T11:09:37.905 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.905 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.905 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:09:37.906 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:09.114051+0000", 2023-10-01T11:09:37.906 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.605e-06 2023-10-01T11:09:37.906 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.906 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.906 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:09:37.906 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.582827+0000", 2023-10-01T11:09:37.906 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.4687765549999998 2023-10-01T11:09:37.906 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.906 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.906 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:09:37.906 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.582962+0000", 2023-10-01T11:09:37.907 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000134513 2023-10-01T11:09:37.907 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.907 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.907 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 5,7", 2023-10-01T11:09:37.907 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.583159+0000", 2023-10-01T11:09:37.907 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000197357 2023-10-01T11:09:37.907 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.907 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.907 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T11:09:37.907 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.591345+0000", 2023-10-01T11:09:37.908 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0081853840000000004 2023-10-01T11:09:37.908 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.908 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.908 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:09:37.908 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.600784+0000", 2023-10-01T11:09:37.908 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0094389269999999997 2023-10-01T11:09:37.908 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.908 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.908 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:09:37.908 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.600848+0000", 2023-10-01T11:09:37.908 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.3969999999999999e-05 2023-10-01T11:09:37.908 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.909 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.909 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:09:37.909 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.600891+0000", 2023-10-01T11:09:37.909 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.3228e-05 2023-10-01T11:09:37.909 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.909 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.909 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:09:37.909 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.602840+0000", 2023-10-01T11:09:37.909 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.001949693 2023-10-01T11:09:37.909 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:09:37.909 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:09:37.909 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:09:37.910 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.910 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.910 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46454.0:73803 13.f 13:ffc81c8f:::benchmark_data_smithi078_202524_object73802:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e301)", 2023-10-01T11:09:37.910 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:03:13.523137+0000", 2023-10-01T11:09:37.910 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 384.308494145, 2023-10-01T11:09:37.910 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.0947663670000001, 2023-10-01T11:09:37.910 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:09:37.910 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:09:37.910 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T11:09:37.910 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46454", 2023-10-01T11:09:37.910 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:09:37.911 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 73803 2023-10-01T11:09:37.911 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.911 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:09:37.911 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.911 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:09:37.911 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:13.523137+0000", 2023-10-01T11:09:37.911 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:09:37.911 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.911 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.911 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:09:37.911 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:13.523137+0000", 2023-10-01T11:09:37.912 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:09:37.912 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.912 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.912 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:09:37.912 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:13.523135+0000", 2023-10-01T11:09:37.912 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999986 2023-10-01T11:09:37.912 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.912 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.912 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:09:37.912 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:13.523165+0000", 2023-10-01T11:09:37.913 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.9454e-05 2023-10-01T11:09:37.913 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.913 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.913 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:09:37.913 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:13.523175+0000", 2023-10-01T11:09:37.913 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.0283e-05 2023-10-01T11:09:37.913 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.913 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.913 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:09:37.913 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:13.523181+0000", 2023-10-01T11:09:37.913 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.7429999999999999e-06 2023-10-01T11:09:37.913 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.914 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.914 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:09:37.914 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.585086+0000", 2023-10-01T11:09:37.914 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.0619051669999999 2023-10-01T11:09:37.914 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.914 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.914 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:09:37.914 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.585221+0000", 2023-10-01T11:09:37.914 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000134769 2023-10-01T11:09:37.915 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.915 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.915 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 5,7", 2023-10-01T11:09:37.915 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.585378+0000", 2023-10-01T11:09:37.915 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00015732800000000001 2023-10-01T11:09:37.915 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.915 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.915 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T11:09:37.915 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.593739+0000", 2023-10-01T11:09:37.915 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0083609740000000002 2023-10-01T11:09:37.915 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.916 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.916 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:09:37.916 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.611008+0000", 2023-10-01T11:09:37.916 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.017269263999999999 2023-10-01T11:09:37.916 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.916 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.916 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:09:37.916 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.616268+0000", 2023-10-01T11:09:37.916 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0052594190000000004 2023-10-01T11:09:37.916 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.916 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.917 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:09:37.917 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.616338+0000", 2023-10-01T11:09:37.917 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.0239000000000003e-05 2023-10-01T11:09:37.917 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.917 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.917 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:09:37.917 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.617903+0000", 2023-10-01T11:09:37.917 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0015650449999999999 2023-10-01T11:09:37.917 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:09:37.917 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:09:37.917 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:09:37.918 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.918 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.918 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46454.0:73811 13.f 13:f38b34a9:::benchmark_data_smithi078_202524_object73810:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e301)", 2023-10-01T11:09:37.918 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:03:13.597487+0000", 2023-10-01T11:09:37.918 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 384.23414396800001, 2023-10-01T11:09:37.918 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.0325334179999999, 2023-10-01T11:09:37.918 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:09:37.918 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:09:37.918 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T11:09:37.918 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46454", 2023-10-01T11:09:37.918 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:09:37.919 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 73811 2023-10-01T11:09:37.919 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.919 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:09:37.919 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.919 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:09:37.919 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:13.597487+0000", 2023-10-01T11:09:37.919 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:09:37.919 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.919 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.919 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:09:37.920 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:13.597487+0000", 2023-10-01T11:09:37.920 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:09:37.920 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.920 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.920 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:09:37.920 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:13.597485+0000", 2023-10-01T11:09:37.920 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999981 2023-10-01T11:09:37.920 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.920 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.921 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:09:37.921 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:13.597514+0000", 2023-10-01T11:09:37.921 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.9300000000000001e-05 2023-10-01T11:09:37.921 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.921 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.921 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:09:37.921 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:13.597531+0000", 2023-10-01T11:09:37.921 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.6588000000000001e-05 2023-10-01T11:09:37.921 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.921 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.922 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:09:37.922 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:13.597542+0000", 2023-10-01T11:09:37.922 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1491999999999999e-05 2023-10-01T11:09:37.922 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.922 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.922 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:09:37.922 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.587131+0000", 2023-10-01T11:09:37.922 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.9895887009999997 2023-10-01T11:09:37.922 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.922 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.922 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:09:37.923 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.587318+0000", 2023-10-01T11:09:37.923 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00018682200000000001 2023-10-01T11:09:37.923 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.923 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.923 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 5,7", 2023-10-01T11:09:37.923 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.587502+0000", 2023-10-01T11:09:37.923 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00018421400000000001 2023-10-01T11:09:37.923 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.923 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.923 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T11:09:37.923 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.593749+0000", 2023-10-01T11:09:37.923 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0062469639999999998 2023-10-01T11:09:37.924 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.924 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.924 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:09:37.924 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.611055+0000", 2023-10-01T11:09:37.924 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.017306038999999999 2023-10-01T11:09:37.924 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.924 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.924 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:09:37.924 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.627568+0000", 2023-10-01T11:09:37.924 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.016513153999999999 2023-10-01T11:09:37.925 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.925 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.925 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:09:37.925 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.627626+0000", 2023-10-01T11:09:37.925 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.7951999999999997e-05 2023-10-01T11:09:37.925 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.925 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.925 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:09:37.925 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.630020+0000", 2023-10-01T11:09:37.925 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0023939109999999999 2023-10-01T11:09:37.925 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:09:37.925 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:09:37.926 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:09:37.926 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.926 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.926 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46454.0:73823 13.f 13:f26cf720:::benchmark_data_smithi078_202524_object73822:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e301)", 2023-10-01T11:09:37.926 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:03:13.649419+0000", 2023-10-01T11:09:37.926 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 384.18221188199999, 2023-10-01T11:09:37.926 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.9897676349999998, 2023-10-01T11:09:37.926 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:09:37.926 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:09:37.926 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T11:09:37.926 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46454", 2023-10-01T11:09:37.927 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:09:37.927 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 73823 2023-10-01T11:09:37.927 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.927 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:09:37.927 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.927 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:09:37.927 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:13.649419+0000", 2023-10-01T11:09:37.927 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:09:37.927 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.928 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.928 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:09:37.928 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:13.649419+0000", 2023-10-01T11:09:37.928 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:09:37.928 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.928 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.928 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:09:37.928 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:13.649418+0000", 2023-10-01T11:09:37.928 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999986 2023-10-01T11:09:37.928 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.928 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.928 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:09:37.929 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:13.649439+0000", 2023-10-01T11:09:37.929 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.1447e-05 2023-10-01T11:09:37.929 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.929 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.929 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:09:37.929 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:13.649450+0000", 2023-10-01T11:09:37.929 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.0699e-05 2023-10-01T11:09:37.929 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.929 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.929 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:09:37.929 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:13.649458+0000", 2023-10-01T11:09:37.929 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.5809999999999997e-06 2023-10-01T11:09:37.930 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.930 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.930 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:09:37.930 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.589452+0000", 2023-10-01T11:09:37.930 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.9399940610000002 2023-10-01T11:09:37.930 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.930 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.930 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:09:37.930 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.589587+0000", 2023-10-01T11:09:37.930 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00013418600000000001 2023-10-01T11:09:37.930 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.930 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.931 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 5,7", 2023-10-01T11:09:37.931 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.589719+0000", 2023-10-01T11:09:37.931 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00013280200000000001 2023-10-01T11:09:37.931 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.931 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.931 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T11:09:37.931 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.598896+0000", 2023-10-01T11:09:37.931 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0091764450000000001 2023-10-01T11:09:37.931 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.931 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.932 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:09:37.932 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.617960+0000", 2023-10-01T11:09:37.932 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.019064352 2023-10-01T11:09:37.932 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.932 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.932 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:09:37.932 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.637168+0000", 2023-10-01T11:09:37.932 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.019208125 2023-10-01T11:09:37.932 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.932 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.933 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:09:37.933 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.637206+0000", 2023-10-01T11:09:37.933 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.8197e-05 2023-10-01T11:09:37.933 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.933 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.933 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:09:37.933 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.639187+0000", 2023-10-01T11:09:37.933 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0019800460000000001 2023-10-01T11:09:37.933 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:09:37.933 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:09:37.933 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:09:37.933 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.933 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.934 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46454.0:73826 13.f 13:f5427c1c:::benchmark_data_smithi078_202524_object73825:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e301)", 2023-10-01T11:09:37.934 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:03:13.688007+0000", 2023-10-01T11:09:37.934 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 384.14362328800001, 2023-10-01T11:09:37.934 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.95318353, 2023-10-01T11:09:37.934 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:09:37.934 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:09:37.934 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T11:09:37.934 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46454", 2023-10-01T11:09:37.934 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:09:37.935 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 73826 2023-10-01T11:09:37.935 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.935 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:09:37.935 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.935 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:09:37.935 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:13.688007+0000", 2023-10-01T11:09:37.935 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:09:37.935 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.935 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.935 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:09:37.935 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:13.688007+0000", 2023-10-01T11:09:37.936 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:09:37.936 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.936 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.936 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:09:37.936 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:13.688005+0000", 2023-10-01T11:09:37.936 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999976 2023-10-01T11:09:37.936 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.936 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.936 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:09:37.936 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:13.688089+0000", 2023-10-01T11:09:37.936 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.4050999999999999e-05 2023-10-01T11:09:37.936 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.937 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.937 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:09:37.937 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:13.688116+0000", 2023-10-01T11:09:37.937 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.6835999999999999e-05 2023-10-01T11:09:37.937 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.937 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.937 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:09:37.937 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:13.688137+0000", 2023-10-01T11:09:37.937 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.1149e-05 2023-10-01T11:09:37.937 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.938 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.938 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:09:37.938 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.591379+0000", 2023-10-01T11:09:37.938 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.9032423019999998 2023-10-01T11:09:37.938 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.938 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.938 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:09:37.938 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.591535+0000", 2023-10-01T11:09:37.938 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00015563999999999999 2023-10-01T11:09:37.938 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.938 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.939 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 5,7", 2023-10-01T11:09:37.939 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.591668+0000", 2023-10-01T11:09:37.939 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00013315199999999999 2023-10-01T11:09:37.939 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.939 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.939 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T11:09:37.939 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.600793+0000", 2023-10-01T11:09:37.939 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00912462 2023-10-01T11:09:37.939 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.939 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.939 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:09:37.939 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.618028+0000", 2023-10-01T11:09:37.939 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.017235622999999999 2023-10-01T11:09:37.939 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.940 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.940 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:09:37.940 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.639328+0000", 2023-10-01T11:09:37.940 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.021299753000000001 2023-10-01T11:09:37.940 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.940 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.940 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:09:37.940 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.639389+0000", 2023-10-01T11:09:37.940 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.0909000000000002e-05 2023-10-01T11:09:37.940 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.941 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.941 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:09:37.941 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.641191+0000", 2023-10-01T11:09:37.941 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0018020670000000001 2023-10-01T11:09:37.941 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:09:37.941 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:09:37.941 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:09:37.942 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.942 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.942 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46454.0:73830 13.f 13:f0c402b8:::benchmark_data_smithi078_202524_object73829:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e301)", 2023-10-01T11:09:37.942 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:03:13.706184+0000", 2023-10-01T11:09:37.942 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 384.12544679400003, 2023-10-01T11:09:37.942 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.936670345, 2023-10-01T11:09:37.942 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:09:37.942 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:09:37.942 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T11:09:37.942 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46454", 2023-10-01T11:09:37.942 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:09:37.943 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 73830 2023-10-01T11:09:37.943 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.943 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:09:37.943 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.943 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:09:37.943 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:13.706184+0000", 2023-10-01T11:09:37.943 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:09:37.943 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.943 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.943 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:09:37.943 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:13.706184+0000", 2023-10-01T11:09:37.943 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:09:37.944 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.944 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.944 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:09:37.944 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:13.706183+0000", 2023-10-01T11:09:37.944 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T11:09:37.944 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.944 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.944 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:09:37.944 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:13.706209+0000", 2023-10-01T11:09:37.944 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.5825000000000001e-05 2023-10-01T11:09:37.944 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.944 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.944 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:09:37.945 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:13.706217+0000", 2023-10-01T11:09:37.945 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.4719999999999995e-06 2023-10-01T11:09:37.945 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.945 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.945 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:09:37.945 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:13.706222+0000", 2023-10-01T11:09:37.945 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.8969999999999999e-06 2023-10-01T11:09:37.945 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.945 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.945 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:09:37.946 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.593806+0000", 2023-10-01T11:09:37.946 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.887583738 2023-10-01T11:09:37.946 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.946 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.946 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:09:37.946 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.593986+0000", 2023-10-01T11:09:37.946 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000180061 2023-10-01T11:09:37.946 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.946 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.946 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 5,7", 2023-10-01T11:09:37.946 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.594182+0000", 2023-10-01T11:09:37.946 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00019590100000000001 2023-10-01T11:09:37.947 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.947 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.947 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T11:09:37.947 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.600799+0000", 2023-10-01T11:09:37.947 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0066168199999999998 2023-10-01T11:09:37.947 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.947 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.947 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:09:37.947 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.637103+0000", 2023-10-01T11:09:37.947 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.036304548999999998 2023-10-01T11:09:37.947 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.948 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.948 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:09:37.948 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.641326+0000", 2023-10-01T11:09:37.948 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0042230949999999996 2023-10-01T11:09:37.948 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.948 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.948 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:09:37.948 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.641364+0000", 2023-10-01T11:09:37.948 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.7703000000000001e-05 2023-10-01T11:09:37.948 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.948 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.948 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:09:37.949 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.642854+0000", 2023-10-01T11:09:37.949 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.001490424 2023-10-01T11:09:37.949 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:09:37.949 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:09:37.949 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:09:37.949 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.949 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.949 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46454.0:73846 13.f 13:f66a88c6:::benchmark_data_smithi078_202524_object73845:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e301)", 2023-10-01T11:09:37.949 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:03:13.853331+0000", 2023-10-01T11:09:37.949 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 383.97830027100002, 2023-10-01T11:09:37.949 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.8004265610000001, 2023-10-01T11:09:37.949 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:09:37.950 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:09:37.950 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T11:09:37.950 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46454", 2023-10-01T11:09:37.950 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:09:37.950 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 73846 2023-10-01T11:09:37.950 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.950 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:09:37.950 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.950 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:09:37.950 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:13.853331+0000", 2023-10-01T11:09:37.950 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:09:37.950 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.951 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.951 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:09:37.951 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:13.853331+0000", 2023-10-01T11:09:37.951 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:09:37.951 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.951 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.951 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:09:37.951 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:13.853329+0000", 2023-10-01T11:09:37.951 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T11:09:37.951 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.951 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.951 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:09:37.952 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:13.853357+0000", 2023-10-01T11:09:37.952 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.8085e-05 2023-10-01T11:09:37.952 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.952 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.952 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:09:37.952 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:13.853366+0000", 2023-10-01T11:09:37.952 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.8040000000000007e-06 2023-10-01T11:09:37.952 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.952 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.952 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:09:37.952 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:13.853372+0000", 2023-10-01T11:09:37.953 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.6590000000000004e-06 2023-10-01T11:09:37.953 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.953 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.953 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:09:37.953 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.596435+0000", 2023-10-01T11:09:37.953 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.7430634220000001 2023-10-01T11:09:37.953 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.953 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.953 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:09:37.953 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.596582+0000", 2023-10-01T11:09:37.953 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00014644699999999999 2023-10-01T11:09:37.954 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.954 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.954 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 5,7", 2023-10-01T11:09:37.954 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.596765+0000", 2023-10-01T11:09:37.954 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000183528 2023-10-01T11:09:37.954 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.954 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.954 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T11:09:37.954 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.605467+0000", 2023-10-01T11:09:37.954 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0087012410000000002 2023-10-01T11:09:37.954 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.955 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.955 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:09:37.955 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.637139+0000", 2023-10-01T11:09:37.955 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.031672121999999997 2023-10-01T11:09:37.955 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.955 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.955 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:09:37.955 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.652054+0000", 2023-10-01T11:09:37.955 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.014914927999999999 2023-10-01T11:09:37.955 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.955 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.955 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:09:37.956 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.652089+0000", 2023-10-01T11:09:37.956 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.5222000000000003e-05 2023-10-01T11:09:37.956 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.956 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.956 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:09:37.956 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.653757+0000", 2023-10-01T11:09:37.956 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0016682540000000001 2023-10-01T11:09:37.956 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:09:37.956 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:09:37.956 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:09:37.956 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.957 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.957 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46454.0:73856 13.f 13:f1bab3a1:::benchmark_data_smithi078_202524_object73855:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e301)", 2023-10-01T11:09:37.957 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:03:13.962751+0000", 2023-10-01T11:09:37.957 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 383.86887955399999, 2023-10-01T11:09:37.957 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.6930807130000001, 2023-10-01T11:09:37.957 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:09:37.957 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:09:37.957 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T11:09:37.957 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46454", 2023-10-01T11:09:37.957 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:09:37.957 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 73856 2023-10-01T11:09:37.958 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.958 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:09:37.958 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.958 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:09:37.958 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:13.962751+0000", 2023-10-01T11:09:37.958 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:09:37.958 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.958 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.958 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:09:37.958 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:13.962751+0000", 2023-10-01T11:09:37.958 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:09:37.959 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.959 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.959 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:09:37.959 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:13.962750+0000", 2023-10-01T11:09:37.959 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T11:09:37.959 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.959 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.959 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:09:37.959 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:13.962789+0000", 2023-10-01T11:09:37.959 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.8794000000000002e-05 2023-10-01T11:09:37.959 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.959 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.960 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:09:37.960 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:13.962801+0000", 2023-10-01T11:09:37.960 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1698e-05 2023-10-01T11:09:37.960 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.960 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.960 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:09:37.960 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:13.962809+0000", 2023-10-01T11:09:37.960 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.3580000000000003e-06 2023-10-01T11:09:37.960 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.960 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.960 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:09:37.961 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.602897+0000", 2023-10-01T11:09:37.961 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.6400883249999998 2023-10-01T11:09:37.961 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.961 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.961 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:09:37.961 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.603011+0000", 2023-10-01T11:09:37.961 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000113287 2023-10-01T11:09:37.961 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.961 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.961 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 5,7", 2023-10-01T11:09:37.961 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.603130+0000", 2023-10-01T11:09:37.961 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000119782 2023-10-01T11:09:37.962 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.962 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.962 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T11:09:37.962 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.611066+0000", 2023-10-01T11:09:37.962 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0079356159999999995 2023-10-01T11:09:37.962 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.962 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.962 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:09:37.962 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.639270+0000", 2023-10-01T11:09:37.962 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.028203880000000001 2023-10-01T11:09:37.963 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.963 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.963 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:09:37.963 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.653828+0000", 2023-10-01T11:09:37.963 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.014558041000000001 2023-10-01T11:09:37.963 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.963 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.963 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:09:37.963 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.653883+0000", 2023-10-01T11:09:37.963 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.4975000000000001e-05 2023-10-01T11:09:37.964 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.964 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.964 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:09:37.964 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.655832+0000", 2023-10-01T11:09:37.964 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.001949079 2023-10-01T11:09:37.964 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:09:37.964 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:09:37.964 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:09:37.964 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.964 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.964 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46454.0:73857 13.f 13:f324d089:::benchmark_data_smithi078_202524_object73856:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e301)", 2023-10-01T11:09:37.965 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:03:13.965648+0000", 2023-10-01T11:09:37.965 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 383.865982772, 2023-10-01T11:09:37.965 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.6993895390000002, 2023-10-01T11:09:37.965 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:09:37.965 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:09:37.965 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T11:09:37.965 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46454", 2023-10-01T11:09:37.965 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:09:37.965 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 73857 2023-10-01T11:09:37.965 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.965 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:09:37.966 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.966 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:09:37.966 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:13.965648+0000", 2023-10-01T11:09:37.966 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:09:37.966 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.966 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.966 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:09:37.966 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:13.965648+0000", 2023-10-01T11:09:37.966 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:09:37.966 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.966 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.966 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:09:37.967 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:13.965647+0000", 2023-10-01T11:09:37.967 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999986 2023-10-01T11:09:37.967 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.967 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.967 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:09:37.967 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:13.965708+0000", 2023-10-01T11:09:37.967 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.19e-05 2023-10-01T11:09:37.967 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.967 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.967 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:09:37.967 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:13.965724+0000", 2023-10-01T11:09:37.968 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.5693e-05 2023-10-01T11:09:37.968 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.968 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.968 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:09:37.968 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:13.965735+0000", 2023-10-01T11:09:37.968 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.0963e-05 2023-10-01T11:09:37.968 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.968 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.968 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:09:37.968 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.605520+0000", 2023-10-01T11:09:37.968 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.6397844470000003 2023-10-01T11:09:37.969 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.969 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.969 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:09:37.969 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.605711+0000", 2023-10-01T11:09:37.969 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000191876 2023-10-01T11:09:37.969 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.969 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.969 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 5,7", 2023-10-01T11:09:37.969 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.605951+0000", 2023-10-01T11:09:37.969 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00023907900000000001 2023-10-01T11:09:37.969 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.970 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.970 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T11:09:37.970 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.611075+0000", 2023-10-01T11:09:37.970 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0051244740000000004 2023-10-01T11:09:37.970 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.970 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.970 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:09:37.970 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.641252+0000", 2023-10-01T11:09:37.970 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.030177121000000001 2023-10-01T11:09:37.970 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.970 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.970 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:09:37.971 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.662952+0000", 2023-10-01T11:09:37.971 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.021699501 2023-10-01T11:09:37.971 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.971 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.971 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:09:37.971 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.663010+0000", 2023-10-01T11:09:37.971 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.8146000000000002e-05 2023-10-01T11:09:37.971 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.971 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.972 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:09:37.972 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.665038+0000", 2023-10-01T11:09:37.972 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.002027793 2023-10-01T11:09:37.972 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:09:37.972 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:09:37.972 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:09:37.972 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.972 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.972 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46454.0:73892 13.f 13:f5b9b91f:::benchmark_data_smithi078_202524_object73891:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e301)", 2023-10-01T11:09:37.972 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:03:14.094153+0000", 2023-10-01T11:09:37.972 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 383.73747755400001, 2023-10-01T11:09:37.973 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.5734376230000002, 2023-10-01T11:09:37.973 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:09:37.973 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:09:37.973 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T11:09:37.973 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46454", 2023-10-01T11:09:37.973 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:09:37.973 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 73892 2023-10-01T11:09:37.973 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.973 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:09:37.973 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.973 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:09:37.973 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:14.094153+0000", 2023-10-01T11:09:37.973 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:09:37.974 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.974 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.974 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:09:37.974 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:14.094153+0000", 2023-10-01T11:09:37.974 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:09:37.974 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.974 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.974 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:09:37.974 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:14.094153+0000", 2023-10-01T11:09:37.974 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999995 2023-10-01T11:09:37.974 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.975 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.975 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:09:37.975 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:14.094169+0000", 2023-10-01T11:09:37.975 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.5996999999999999e-05 2023-10-01T11:09:37.975 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.975 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.975 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:09:37.975 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:14.094173+0000", 2023-10-01T11:09:37.975 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.4229999999999998e-06 2023-10-01T11:09:37.975 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.976 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.976 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:09:37.976 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:14.094177+0000", 2023-10-01T11:09:37.976 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.3749999999999999e-06 2023-10-01T11:09:37.976 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.976 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.976 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:09:37.976 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.608359+0000", 2023-10-01T11:09:37.976 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.5141822629999999 2023-10-01T11:09:37.976 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.976 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.977 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:09:37.977 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.608542+0000", 2023-10-01T11:09:37.977 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000183468 2023-10-01T11:09:37.977 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.977 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.977 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 5,7", 2023-10-01T11:09:37.977 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.608716+0000", 2023-10-01T11:09:37.977 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00017371300000000001 2023-10-01T11:09:37.977 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.977 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.977 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T11:09:37.978 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.617968+0000", 2023-10-01T11:09:37.978 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0092516439999999998 2023-10-01T11:09:37.978 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.978 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.978 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:09:37.978 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.642914+0000", 2023-10-01T11:09:37.978 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.024945993 2023-10-01T11:09:37.978 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.978 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.978 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:09:37.978 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.665103+0000", 2023-10-01T11:09:37.979 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.022189568 2023-10-01T11:09:37.979 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.979 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.979 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:09:37.979 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.665170+0000", 2023-10-01T11:09:37.979 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.6877000000000004e-05 2023-10-01T11:09:37.979 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.979 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.979 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:09:37.979 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.667591+0000", 2023-10-01T11:09:37.979 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0024206789999999998 2023-10-01T11:09:37.979 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:09:37.980 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:09:37.980 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:09:37.980 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.980 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.980 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46454.0:73900 13.f 13:faecdcac:::benchmark_data_smithi078_202524_object73899:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e301)", 2023-10-01T11:09:37.980 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:03:14.202442+0000", 2023-10-01T11:09:37.980 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 383.629189204, 2023-10-01T11:09:37.980 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.4707954540000001, 2023-10-01T11:09:37.980 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:09:37.980 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:09:37.980 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T11:09:37.980 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46454", 2023-10-01T11:09:37.980 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:09:37.980 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 73900 2023-10-01T11:09:37.981 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.981 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:09:37.981 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.981 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:09:37.981 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:14.202442+0000", 2023-10-01T11:09:37.981 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:09:37.981 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.981 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.981 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:09:37.981 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:14.202442+0000", 2023-10-01T11:09:37.981 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:09:37.982 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.982 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.982 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:09:37.982 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:14.202440+0000", 2023-10-01T11:09:37.982 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999981 2023-10-01T11:09:37.982 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.982 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.982 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:09:37.982 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:14.202491+0000", 2023-10-01T11:09:37.982 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.0704000000000001e-05 2023-10-01T11:09:37.982 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.983 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.983 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:09:37.983 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:14.202505+0000", 2023-10-01T11:09:37.983 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.4418e-05 2023-10-01T11:09:37.983 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.983 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.983 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:09:37.983 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:14.202515+0000", 2023-10-01T11:09:37.983 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.5519999999999993e-06 2023-10-01T11:09:37.983 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.983 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.983 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:09:37.984 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.611103+0000", 2023-10-01T11:09:37.984 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.408588736 2023-10-01T11:09:37.984 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.984 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.984 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:09:37.984 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.611271+0000", 2023-10-01T11:09:37.984 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00016753300000000001 2023-10-01T11:09:37.984 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.984 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.984 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 5,7", 2023-10-01T11:09:37.984 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.611495+0000", 2023-10-01T11:09:37.984 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00022434700000000001 2023-10-01T11:09:37.985 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.985 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.985 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T11:09:37.985 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.617974+0000", 2023-10-01T11:09:37.985 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0064789080000000002 2023-10-01T11:09:37.985 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.985 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.985 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:09:37.985 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.643036+0000", 2023-10-01T11:09:37.985 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.025061872999999998 2023-10-01T11:09:37.985 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.985 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.986 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:09:37.986 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.668759+0000", 2023-10-01T11:09:37.986 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.025722681000000001 2023-10-01T11:09:37.986 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.986 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.986 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:09:37.986 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.668816+0000", 2023-10-01T11:09:37.986 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.7433999999999999e-05 2023-10-01T11:09:37.986 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.986 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.986 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:09:37.987 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:03:17.673237+0000", 2023-10-01T11:09:37.987 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0044210029999999997 2023-10-01T11:09:37.987 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:09:37.987 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:09:37.987 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:09:37.987 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.987 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.987 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.46454.0:86779 13.8 13:130c87d1:::benchmark_data_smithi078_202524_object7221:head [delete] snapc 0=[] ondisk+write+known_if_redirected e322)", 2023-10-01T11:09:37.987 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:05:32.777202+0000", 2023-10-01T11:09:37.987 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 245.05442856799999, 2023-10-01T11:09:37.987 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.1214169809999999, 2023-10-01T11:09:37.988 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:09:37.988 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:09:37.988 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T11:09:37.988 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.46454", 2023-10-01T11:09:37.988 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:09:37.988 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 86779 2023-10-01T11:09:37.988 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.988 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:09:37.988 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.988 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:09:37.988 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:05:32.777202+0000", 2023-10-01T11:09:37.988 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:09:37.989 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.989 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.989 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:09:37.989 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:05:32.777202+0000", 2023-10-01T11:09:37.989 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:09:37.989 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.989 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.989 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:09:37.989 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:05:32.777200+0000", 2023-10-01T11:09:37.989 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999976 2023-10-01T11:09:37.989 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.989 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.990 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:09:37.990 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:05:32.777209+0000", 2023-10-01T11:09:37.990 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.7459999999999995e-06 2023-10-01T11:09:37.990 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.990 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.990 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:09:37.990 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:05:32.777227+0000", 2023-10-01T11:09:37.990 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.8121000000000001e-05 2023-10-01T11:09:37.990 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.990 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.990 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:09:37.991 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:05:32.777241+0000", 2023-10-01T11:09:37.991 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.3540000000000001e-05 2023-10-01T11:09:37.991 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.991 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.991 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:09:37.991 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:05:32.783220+0000", 2023-10-01T11:09:37.991 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0059794039999999998 2023-10-01T11:09:37.991 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.991 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.991 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T11:09:37.991 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:05:32.783235+0000", 2023-10-01T11:09:37.992 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.4875000000000001e-05 2023-10-01T11:09:37.992 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.992 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.992 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:09:37.992 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:05:33.799587+0000", 2023-10-01T11:09:37.992 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.0163521849999999 2023-10-01T11:09:37.992 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.992 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.992 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for peered", 2023-10-01T11:09:37.992 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:05:33.799669+0000", 2023-10-01T11:09:37.992 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.1265000000000002e-05 2023-10-01T11:09:37.992 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.993 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.993 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:09:37.993 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:05:34.893649+0000", 2023-10-01T11:09:37.993 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.0939799189999999 2023-10-01T11:09:37.993 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.993 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.993 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:09:37.993 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:05:34.893939+0000", 2023-10-01T11:09:37.993 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00029066399999999999 2023-10-01T11:09:37.993 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.993 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.993 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 1,5,6", 2023-10-01T11:09:37.994 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:05:34.894172+0000", 2023-10-01T11:09:37.994 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000232852 2023-10-01T11:09:37.994 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.994 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.994 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:09:37.994 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:05:34.897999+0000", 2023-10-01T11:09:37.994 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0038270629999999999 2023-10-01T11:09:37.994 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.994 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.994 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T11:09:37.994 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:05:34.898037+0000", 2023-10-01T11:09:37.995 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.7613000000000001e-05 2023-10-01T11:09:37.995 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.995 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.995 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:09:37.995 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:05:34.898281+0000", 2023-10-01T11:09:37.995 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00024459500000000001 2023-10-01T11:09:37.995 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.995 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.995 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:09:37.995 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:05:34.898496+0000", 2023-10-01T11:09:37.995 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00021452799999999999 2023-10-01T11:09:37.995 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.995 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.996 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:09:37.996 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:05:34.898542+0000", 2023-10-01T11:09:37.996 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.5623e-05 2023-10-01T11:09:37.996 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:09:37.996 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:09:37.996 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:09:37.996 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:05:34.898619+0000", 2023-10-01T11:09:37.996 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.7583999999999994e-05 2023-10-01T11:09:37.996 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:09:37.996 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:09:37.996 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:09:37.996 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:09:37.997 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:09:37.997 INFO:teuthology.orchestra.run.smithi078.stdout:} 2023-10-01T11:09:37.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:37.998 INFO:teuthology.orchestra.run.smithi078.stderr:enabled application 'rados' on pool 'unique_pool_9' 2023-10-01T11:09:37.998 INFO:teuthology.orchestra.run.smithi078.stderr:noscrub is unset 2023-10-01T11:09:37.998 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:37 smithi078 bash[75299]: cluster 2023-10-01T11:09:36.380839+0000 mgr.y (mgr.25072) 1569 : cluster 0 pgmap v1689: 89 pgs: 89 active+clean; 582 KiB data, 7.8 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:37.998 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:37 smithi078 bash[75299]: audit 2023-10-01T11:09:36.935153+0000 mon.a (mon.0) 2040 : audit 0 from='client.? 172.21.15.78:0/3737408123' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:09:37.999 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:37 smithi078 bash[75299]: audit 2023-10-01T11:09:37.694762+0000 mon.c (mon.2) 236 : audit 1 from='client.? 172.21.15.78:0/572963835' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T11:09:37.999 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:37 smithi078 bash[75299]: audit 2023-10-01T11:09:37.695635+0000 mon.a (mon.0) 2041 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T11:09:37.999 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:37 smithi078 bash[75299]: audit 2023-10-01T11:09:37.806892+0000 mon.c (mon.2) 237 : audit 1 from='client.? 172.21.15.78:0/1893778856' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T11:09:37.999 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:37 smithi078 bash[75299]: audit 2023-10-01T11:09:37.809582+0000 mon.a (mon.0) 2042 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T11:09:37.999 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:37 smithi078 bash[78274]: cluster 2023-10-01T11:09:36.380839+0000 mgr.y (mgr.25072) 1569 : cluster 0 pgmap v1689: 89 pgs: 89 active+clean; 582 KiB data, 7.8 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:37.999 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:37 smithi078 bash[78274]: audit 2023-10-01T11:09:36.935153+0000 mon.a (mon.0) 2040 : audit 0 from='client.? 172.21.15.78:0/3737408123' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:09:37.999 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:37 smithi078 bash[78274]: audit 2023-10-01T11:09:37.694762+0000 mon.c (mon.2) 236 : audit 1 from='client.? 172.21.15.78:0/572963835' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T11:09:37.999 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:37 smithi078 bash[78274]: audit 2023-10-01T11:09:37.695635+0000 mon.a (mon.0) 2041 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T11:09:37.999 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:37 smithi078 bash[78274]: audit 2023-10-01T11:09:37.806892+0000 mon.c (mon.2) 237 : audit 1 from='client.? 172.21.15.78:0/1893778856' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T11:09:37.999 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:37 smithi078 bash[78274]: audit 2023-10-01T11:09:37.809582+0000 mon.a (mon.0) 2042 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T11:09:38.000 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:38.000 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:09:38.101 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:38.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:37 smithi173 bash[58816]: cluster 2023-10-01T11:09:36.380839+0000 mgr.y (mgr.25072) 1569 : cluster 0 pgmap v1689: 89 pgs: 89 active+clean; 582 KiB data, 7.8 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:38.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:37 smithi173 bash[58816]: audit 2023-10-01T11:09:36.935153+0000 mon.a (mon.0) 2040 : audit 0 from='client.? 172.21.15.78:0/3737408123' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:09:38.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:37 smithi173 bash[58816]: audit 2023-10-01T11:09:37.694762+0000 mon.c (mon.2) 236 : audit 1 from='client.? 172.21.15.78:0/572963835' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T11:09:38.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:37 smithi173 bash[58816]: audit 2023-10-01T11:09:37.695635+0000 mon.a (mon.0) 2041 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-10-01T11:09:38.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:37 smithi173 bash[58816]: audit 2023-10-01T11:09:37.806892+0000 mon.c (mon.2) 237 : audit 1 from='client.? 172.21.15.78:0/1893778856' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T11:09:38.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:37 smithi173 bash[58816]: audit 2023-10-01T11:09:37.809582+0000 mon.a (mon.0) 2042 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T11:09:38.131 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:38.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:38.133 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:09:38.234 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:38.264 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:38.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:38.267 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:09:38.368 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:38.401 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:38.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:38.404 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:09:38.505 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:38.529 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:38.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:38.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:09:38.632 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:38.665 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:38.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:38.668 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:38.770 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:38.801 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:38.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:38.804 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:09:38.905 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:38.918 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:38 smithi173 bash[58816]: cluster 2023-10-01T11:09:37.838877+0000 mon.a (mon.0) 2043 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:09:38.918 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:38 smithi173 bash[58816]: audit 2023-10-01T11:09:37.855412+0000 mon.a (mon.0) 2044 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-01T11:09:38.918 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:38 smithi173 bash[58816]: audit 2023-10-01T11:09:37.855477+0000 mon.a (mon.0) 2045 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T11:09:38.918 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:38 smithi173 bash[58816]: cluster 2023-10-01T11:09:37.855519+0000 mon.a (mon.0) 2046 : cluster 0 osdmap e359: 8 total, 8 up, 8 in 2023-10-01T11:09:38.918 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:38 smithi173 bash[58816]: cluster 2023-10-01T11:09:38.137144+0000 osd.5 (osd.5) 381 : cluster 3 failed to encode map e359 with expected crc 2023-10-01T11:09:38.936 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:38.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:38.939 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:09:39.039 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:39.073 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:39.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:39.077 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:09:39.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:38 smithi078 bash[75299]: cluster 2023-10-01T11:09:37.838877+0000 mon.a (mon.0) 2043 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:09:39.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:38 smithi078 bash[75299]: audit 2023-10-01T11:09:37.855412+0000 mon.a (mon.0) 2044 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-01T11:09:39.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:38 smithi078 bash[75299]: audit 2023-10-01T11:09:37.855477+0000 mon.a (mon.0) 2045 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T11:09:39.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:38 smithi078 bash[75299]: cluster 2023-10-01T11:09:37.855519+0000 mon.a (mon.0) 2046 : cluster 0 osdmap e359: 8 total, 8 up, 8 in 2023-10-01T11:09:39.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:38 smithi078 bash[75299]: cluster 2023-10-01T11:09:38.137144+0000 osd.5 (osd.5) 381 : cluster 3 failed to encode map e359 with expected crc 2023-10-01T11:09:39.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:38 smithi078 bash[78274]: cluster 2023-10-01T11:09:37.838877+0000 mon.a (mon.0) 2043 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:09:39.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:38 smithi078 bash[78274]: audit 2023-10-01T11:09:37.855412+0000 mon.a (mon.0) 2044 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-10-01T11:09:39.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:38 smithi078 bash[78274]: audit 2023-10-01T11:09:37.855477+0000 mon.a (mon.0) 2045 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T11:09:39.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:38 smithi078 bash[78274]: cluster 2023-10-01T11:09:37.855519+0000 mon.a (mon.0) 2046 : cluster 0 osdmap e359: 8 total, 8 up, 8 in 2023-10-01T11:09:39.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:38 smithi078 bash[78274]: cluster 2023-10-01T11:09:38.137144+0000 osd.5 (osd.5) 381 : cluster 3 failed to encode map e359 with expected crc 2023-10-01T11:09:39.178 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:39.203 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:39.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:39.206 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:09:39.306 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:39.332 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:39.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:39.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:09:39.435 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:39.472 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:39.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:39.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:09:39.576 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:39.609 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:39.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:39.612 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:09:39.713 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:39.740 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:39.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:39.743 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:09:39.844 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:39.873 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:39.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:39.876 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:09:39.976 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:40.007 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:40.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:40.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:09:40.111 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:40.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:39 smithi173 bash[58816]: cluster 2023-10-01T11:09:38.381347+0000 mgr.y (mgr.25072) 1570 : cluster 0 pgmap v1691: 89 pgs: 89 active+clean; 582 KiB data, 7.8 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:40.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:39 smithi173 bash[58816]: cluster 2023-10-01T11:09:38.730666+0000 osd.6 (osd.6) 343 : cluster 3 failed to encode map e359 with expected crc 2023-10-01T11:09:40.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:39 smithi173 bash[58816]: cluster 2023-10-01T11:09:38.732916+0000 osd.6 (osd.6) 344 : cluster 3 failed to encode map e359 with expected crc 2023-10-01T11:09:40.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:39 smithi173 bash[58816]: cluster 2023-10-01T11:09:39.005584+0000 osd.7 (osd.7) 402 : cluster 3 failed to encode map e359 with expected crc 2023-10-01T11:09:40.122 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:39 smithi078 bash[75299]: cluster 2023-10-01T11:09:38.381347+0000 mgr.y (mgr.25072) 1570 : cluster 0 pgmap v1691: 89 pgs: 89 active+clean; 582 KiB data, 7.8 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:40.122 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:39 smithi078 bash[75299]: cluster 2023-10-01T11:09:38.730666+0000 osd.6 (osd.6) 343 : cluster 3 failed to encode map e359 with expected crc 2023-10-01T11:09:40.122 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:39 smithi078 bash[75299]: cluster 2023-10-01T11:09:38.732916+0000 osd.6 (osd.6) 344 : cluster 3 failed to encode map e359 with expected crc 2023-10-01T11:09:40.122 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:39 smithi078 bash[75299]: cluster 2023-10-01T11:09:39.005584+0000 osd.7 (osd.7) 402 : cluster 3 failed to encode map e359 with expected crc 2023-10-01T11:09:40.122 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:39 smithi078 bash[78274]: cluster 2023-10-01T11:09:38.381347+0000 mgr.y (mgr.25072) 1570 : cluster 0 pgmap v1691: 89 pgs: 89 active+clean; 582 KiB data, 7.8 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:40.123 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:39 smithi078 bash[78274]: cluster 2023-10-01T11:09:38.730666+0000 osd.6 (osd.6) 343 : cluster 3 failed to encode map e359 with expected crc 2023-10-01T11:09:40.123 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:39 smithi078 bash[78274]: cluster 2023-10-01T11:09:38.732916+0000 osd.6 (osd.6) 344 : cluster 3 failed to encode map e359 with expected crc 2023-10-01T11:09:40.123 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:39 smithi078 bash[78274]: cluster 2023-10-01T11:09:39.005584+0000 osd.7 (osd.7) 402 : cluster 3 failed to encode map e359 with expected crc 2023-10-01T11:09:40.139 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:40.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:40.142 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:09:40.243 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:40.269 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:40.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:40.271 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:09:40.372 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:40.405 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:40.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:40.409 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:09:40.509 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:40.540 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:40.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:40.543 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:09:40.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:09:40 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:09:40] "GET /metrics HTTP/1.1" 200 36317 "" "Prometheus/2.33.4" 2023-10-01T11:09:40.644 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:40.674 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:40.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:40.677 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:40.778 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:40.809 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:40.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:40.812 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:09:40.913 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:40.940 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:40.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:40.943 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:09:41.043 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:41.057 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T11:09:41.075 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:41.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:41.078 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:09:41.179 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:41.213 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:41.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:41.215 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:41.316 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:41.317 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-01T11:09:41.389 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:41.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:41.392 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:09:41.492 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:41.519 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:41.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:41.522 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:09:41.622 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:41.653 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:41.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:41.656 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:09:41.757 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:41.792 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:41.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:41.795 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:09:41.896 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:41.926 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:41.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:41.929 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:09:42.030 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:42.043 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:41 smithi078 bash[75299]: cluster 2023-10-01T11:09:40.382479+0000 mgr.y (mgr.25072) 1571 : cluster 0 pgmap v1692: 89 pgs: 89 active+clean; 582 KiB data, 7.7 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:42.043 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:41 smithi078 bash[78274]: cluster 2023-10-01T11:09:40.382479+0000 mgr.y (mgr.25072) 1571 : cluster 0 pgmap v1692: 89 pgs: 89 active+clean; 582 KiB data, 7.7 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:42.061 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:42.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:42.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:09:42.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:41 smithi173 bash[58816]: cluster 2023-10-01T11:09:40.382479+0000 mgr.y (mgr.25072) 1571 : cluster 0 pgmap v1692: 89 pgs: 89 active+clean; 582 KiB data, 7.7 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:42.166 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:42.199 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:42.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:42.201 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:09:42.302 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:42.304 DEBUG:teuthology.orchestra.run.smithi078:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_9 bench 90 write' 2023-10-01T11:09:42.305 INFO:tasks.thrashosds.thrasher:Growing pool device_health_metrics 2023-10-01T11:09:42.305 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T11:09:42.307 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-10-01T11:09:42.307 INFO:teuthology.orchestra.run:waiting for 3000 2023-10-01T11:09:42.333 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:42.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:42.336 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:09:42.350 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:hints = 1 2023-10-01T11:09:42.350 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-10-01T11:09:42.350 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Object prefix: benchmark_data_smithi078_236918 2023-10-01T11:09:42.351 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T11:09:42.351 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 0 0 0 0 0 0 - 0 2023-10-01T11:09:42.438 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:42.466 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:42.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:42.468 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:09:42.569 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:42.605 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:42.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:42.608 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:09:42.709 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:42.739 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:42.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:42.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:09:42.805 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T11:09:42.844 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:42.874 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:42.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:42.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:09:42.977 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:43.008 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:43.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:43.012 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:43.113 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:43.145 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:43.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:43.149 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:09:43.249 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:43.283 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:43.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:43.290 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:09:43.351 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 1 16 2703 2687 167.928 167.938 0.00651863 0.00593283 2023-10-01T11:09:43.390 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:43.425 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:43.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:43.429 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:09:43.530 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:43.558 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:43.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:43.560 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:09:43.661 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:43.689 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:43.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:43.692 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:09:43.793 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:43.850 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:43.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:43.852 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:09:43.953 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:43.985 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:43.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:43.992 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:09:44.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:43 smithi078 bash[75299]: cluster 2023-10-01T11:09:42.383098+0000 mgr.y (mgr.25072) 1572 : cluster 0 pgmap v1693: 89 pgs: 89 active+clean; 582 KiB data, 7.7 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:44.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:43 smithi078 bash[78274]: cluster 2023-10-01T11:09:42.383098+0000 mgr.y (mgr.25072) 1572 : cluster 0 pgmap v1693: 89 pgs: 89 active+clean; 582 KiB data, 7.7 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:44.093 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:44.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:43 smithi173 bash[58816]: cluster 2023-10-01T11:09:42.383098+0000 mgr.y (mgr.25072) 1572 : cluster 0 pgmap v1693: 89 pgs: 89 active+clean; 582 KiB data, 7.7 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:09:44.123 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:44.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:44.126 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:09:44.227 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:44.260 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:44.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:44.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:44.351 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 2 16 5347 5331 166.581 165.25 0.0082074 0.00598396 2023-10-01T11:09:44.364 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:44.390 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:44.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:44.392 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:09:44.493 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:44.524 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:44.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:44.527 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:09:44.628 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:44.659 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:44.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:44.662 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:44.762 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:44.796 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:44.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:44.798 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:09:44.899 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:44.928 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:44.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:44.932 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:09:45.033 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:45.057 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:45.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:45.059 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:09:45.161 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:45.188 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:45.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:45.190 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:09:45.291 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:45.321 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:45.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:45.329 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:09:45.351 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 3 16 7883 7867 163.882 158.5 0.00509049 0.00608855 2023-10-01T11:09:45.430 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:45.461 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:45.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:45.464 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:09:45.565 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:45.594 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:45.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:45.598 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:09:45.699 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:45.730 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:45.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:45.732 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:09:45.833 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:45.839 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:09:45.840 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:09:45.873 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:45.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:45.878 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:09:45.978 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:46.007 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:46.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:46.010 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:09:46.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:45 smithi078 bash[75299]: cluster 2023-10-01T11:09:44.384055+0000 mgr.y (mgr.25072) 1573 : cluster 0 pgmap v1694: 89 pgs: 89 active+clean; 164 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 16 MiB/s wr, 260 op/s 2023-10-01T11:09:46.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:45 smithi078 bash[78274]: cluster 2023-10-01T11:09:44.384055+0000 mgr.y (mgr.25072) 1573 : cluster 0 pgmap v1694: 89 pgs: 89 active+clean; 164 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 16 MiB/s wr, 260 op/s 2023-10-01T11:09:46.110 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:46.141 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:46.144 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:45 smithi173 bash[58816]: cluster 2023-10-01T11:09:44.384055+0000 mgr.y (mgr.25072) 1573 : cluster 0 pgmap v1694: 89 pgs: 89 active+clean; 164 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 16 MiB/s wr, 260 op/s 2023-10-01T11:09:46.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:46.145 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:09:46.246 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:46.278 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:46.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:46.281 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:09:46.352 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 4 16 10238 10222 159.705 147.188 0.00547689 0.00624658 2023-10-01T11:09:46.382 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:46.423 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:46.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:46.429 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:46.530 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:46.559 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:46.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:46.561 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:09:46.662 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:46.687 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:46.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:46.691 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:09:46.791 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:46.825 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:46.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:46.827 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:09:46.868 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:09:46.868 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:09:46.928 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:46.957 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:46.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:46.959 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:09:47.060 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:47.087 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:47.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:47.090 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:09:47.191 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:47.223 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:47.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:47.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:09:47.327 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:47.351 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 5 16 12545 12529 156.599 144.188 0.00417506 0.00637663 2023-10-01T11:09:47.354 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:47.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:47.358 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:09:47.459 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:47.482 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:47.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:47.486 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:09:47.586 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:47.611 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:47.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:47.613 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:09:47.714 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:47.741 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:47.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:47.744 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:09:47.845 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:47.876 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:47.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:47.879 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:09:47.887 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:47 smithi078 bash[75299]: cluster 2023-10-01T11:09:46.384951+0000 mgr.y (mgr.25072) 1574 : cluster 0 pgmap v1695: 89 pgs: 89 active+clean; 315 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 31 MiB/s wr, 502 op/s 2023-10-01T11:09:47.887 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:47 smithi078 bash[75299]: audit 2023-10-01T11:09:47.465482+0000 mon.a (mon.0) 2047 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:09:47.979 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:48.007 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:48.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:48.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:09:48.111 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:48.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:47 smithi173 bash[58816]: cluster 2023-10-01T11:09:46.384951+0000 mgr.y (mgr.25072) 1574 : cluster 0 pgmap v1695: 89 pgs: 89 active+clean; 315 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 31 MiB/s wr, 502 op/s 2023-10-01T11:09:48.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:47 smithi173 bash[58816]: audit 2023-10-01T11:09:47.465482+0000 mon.a (mon.0) 2047 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:09:48.140 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:48.142 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:47 smithi078 bash[78274]: cluster 2023-10-01T11:09:46.384951+0000 mgr.y (mgr.25072) 1574 : cluster 0 pgmap v1695: 89 pgs: 89 active+clean; 315 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 31 MiB/s wr, 502 op/s 2023-10-01T11:09:48.142 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:47 smithi078 bash[78274]: audit 2023-10-01T11:09:47.465482+0000 mon.a (mon.0) 2047 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:09:48.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:48.143 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:09:48.244 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:48.281 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:48.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:48.283 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:09:48.351 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 6 16 14833 14817 154.33 143 0.010362 0.00647025 2023-10-01T11:09:48.385 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:48.414 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:48.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:48.417 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:48.518 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:48.550 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:48.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:48.553 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:09:48.654 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:48.681 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:48.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:48.686 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:09:48.787 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:48.811 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:48.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:48.813 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:09:48.914 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:48.940 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:48.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:48.943 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:09:49.044 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:49.072 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:49.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:49.074 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:09:49.175 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:49.202 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:49.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:49.207 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:09:49.308 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:49.338 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:49.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:49.342 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:09:49.351 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 7 16 17105 17089 152.567 142 0.00478583 0.00654548 2023-10-01T11:09:49.443 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:49.470 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:49.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:49.473 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:49.574 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:49.600 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:49.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:49.603 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:09:49.703 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:49.731 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:49.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:49.734 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:49.834 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:49.862 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:49.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:49.864 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:09:49.965 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:49.974 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:49 smithi173 bash[58816]: cluster 2023-10-01T11:09:48.385473+0000 mgr.y (mgr.25072) 1575 : cluster 0 pgmap v1696: 89 pgs: 89 active+clean; 315 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 30 MiB/s wr, 476 op/s 2023-10-01T11:09:49.989 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:49.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:49.992 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:09:50.093 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:50.101 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:49 smithi078 bash[78274]: cluster 2023-10-01T11:09:48.385473+0000 mgr.y (mgr.25072) 1575 : cluster 0 pgmap v1696: 89 pgs: 89 active+clean; 315 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 30 MiB/s wr, 476 op/s 2023-10-01T11:09:50.102 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:49 smithi078 bash[75299]: cluster 2023-10-01T11:09:48.385473+0000 mgr.y (mgr.25072) 1575 : cluster 0 pgmap v1696: 89 pgs: 89 active+clean; 315 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 30 MiB/s wr, 476 op/s 2023-10-01T11:09:50.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:50.120 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:50.120 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:09:50.221 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:50.249 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:50.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:50.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:09:50.352 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 8 16 19188 19172 149.768 130.188 0.00293832 0.00667021 2023-10-01T11:09:50.352 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:50.360 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:09:50 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:09:50] "GET /metrics HTTP/1.1" 200 36366 "" "Prometheus/2.33.4" 2023-10-01T11:09:50.381 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:50.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:50.383 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:09:50.484 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:50.510 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:50.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:50.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:09:50.616 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:50.638 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:50.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:50.641 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:09:50.742 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:50.767 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:50.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:50.769 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:09:50.870 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:50.899 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:50.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:50.902 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:09:51.003 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:51.028 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:51.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:51.030 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:51.131 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:51.155 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:51.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:51.158 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:09:51.259 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:51.285 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:51.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:51.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:09:51.352 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 9 16 21274 21258 147.612 130.375 0.00392109 0.00676715 2023-10-01T11:09:51.390 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:51.414 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:51.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:51.416 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:09:51.517 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:51.545 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:51.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:51.549 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:09:51.650 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:51.679 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:51.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:51.682 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:09:51.782 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:51.809 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:51.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:51.814 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:09:51.915 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:51.941 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:51.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:51.944 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:09:52.045 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:52.055 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:51 smithi173 bash[58816]: cluster 2023-10-01T11:09:50.386176+0000 mgr.y (mgr.25072) 1576 : cluster 0 pgmap v1697: 89 pgs: 89 active+clean; 918 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-10-01T11:09:52.073 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:52.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:52.076 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:09:52.177 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:52.186 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:51 smithi078 bash[75299]: cluster 2023-10-01T11:09:50.386176+0000 mgr.y (mgr.25072) 1576 : cluster 0 pgmap v1697: 89 pgs: 89 active+clean; 918 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-10-01T11:09:52.186 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:51 smithi078 bash[78274]: cluster 2023-10-01T11:09:50.386176+0000 mgr.y (mgr.25072) 1576 : cluster 0 pgmap v1697: 89 pgs: 89 active+clean; 918 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-10-01T11:09:52.201 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:52.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:52.207 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:09:52.308 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:52.340 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:52.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:52.343 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:09:52.352 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 10 16 23277 23261 145.368 125.188 0.00911353 0.00687329 2023-10-01T11:09:52.444 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:52.474 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:52.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:52.477 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:09:52.578 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:52.609 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:52.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:52.613 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:09:52.714 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:52.739 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:52.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:52.742 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:09:52.843 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:52.874 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:52.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:52.876 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:09:52.977 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:53.009 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:53.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:53.013 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:53.114 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:53.141 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:53.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:53.144 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:09:53.245 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:53.277 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:53.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:53.281 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:09:53.352 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 11 16 25102 25086 142.522 114.062 0.00968254 0.00701049 2023-10-01T11:09:53.381 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:53.416 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:53.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:53.418 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:53.519 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:53.547 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:53.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:53.549 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:09:53.650 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:53.674 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:53.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:53.677 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:09:53.778 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:53.810 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:53.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:53.814 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:53.914 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:53.939 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:53.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:53.941 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:09:54.042 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:54.050 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:53 smithi173 bash[58816]: cluster 2023-10-01T11:09:52.386647+0000 mgr.y (mgr.25072) 1577 : cluster 0 pgmap v1698: 89 pgs: 89 active+clean; 1.0 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s 2023-10-01T11:09:54.069 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:54.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:54.072 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:54.173 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:54.181 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:53 smithi078 bash[78274]: cluster 2023-10-01T11:09:52.386647+0000 mgr.y (mgr.25072) 1577 : cluster 0 pgmap v1698: 89 pgs: 89 active+clean; 1.0 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s 2023-10-01T11:09:54.181 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:53 smithi078 bash[75299]: cluster 2023-10-01T11:09:52.386647+0000 mgr.y (mgr.25072) 1577 : cluster 0 pgmap v1698: 89 pgs: 89 active+clean; 1.0 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s 2023-10-01T11:09:54.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:54.201 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:54.202 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:09:54.303 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:54.328 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:09:54.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:54.331 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:09:54.352 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 12 16 26715 26699 139.045 100.812 0.00751829 0.00718495 2023-10-01T11:09:54.433 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:54.460 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:54.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:54.463 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:09:54.564 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:54.594 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:54.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:54.597 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:09:54.698 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:54.731 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:54.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:54.735 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:09:54.836 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:54.868 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:54.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:54.871 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:09:54.972 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:55.000 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:55.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:55.002 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:09:55.103 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:55.132 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:55.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:55.136 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:09:55.236 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:55.264 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:55.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:55.267 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:09:55.353 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 13 16 28438 28422 136.632 107.688 0.00722022 0.00731189 2023-10-01T11:09:55.368 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:55.393 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:55.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:55.396 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:09:55.497 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:55.525 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:55.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:55.532 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:09:55.633 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:55.664 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:55.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:55.666 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:09:55.767 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:55.796 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:55.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:55.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:09:55.900 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:55.927 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:55.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:55.929 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:09:55.992 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:55 smithi078 bash[75299]: cluster 2023-10-01T11:09:54.387406+0000 mgr.y (mgr.25072) 1578 : cluster 0 pgmap v1699: 89 pgs: 89 active+clean; 1.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-10-01T11:09:55.993 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:55 smithi078 bash[78274]: cluster 2023-10-01T11:09:54.387406+0000 mgr.y (mgr.25072) 1578 : cluster 0 pgmap v1699: 89 pgs: 89 active+clean; 1.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-10-01T11:09:56.030 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:56.059 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:56.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:56.062 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:09:56.163 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:56.173 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:55 smithi173 bash[58816]: cluster 2023-10-01T11:09:54.387406+0000 mgr.y (mgr.25072) 1578 : cluster 0 pgmap v1699: 89 pgs: 89 active+clean; 1.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2023-10-01T11:09:56.192 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:56.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:56.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:09:56.296 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:56.324 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:56.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:56.328 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:56.352 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 14 16 29928 29912 133.524 93.125 0.0142374 0.00748128 2023-10-01T11:09:56.428 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:56.455 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:09:56.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:56.459 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:09:56.560 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:56.585 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:56.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:56.588 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:56.689 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:56.723 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:09:56.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:56.728 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:09:56.829 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:56.860 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:56.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:56.863 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:56.964 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:57.002 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:57.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:57.005 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:57.106 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:57.130 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:09:57.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:57.132 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:09:57.233 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:57.262 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:57.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:57.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:57.352 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 15 16 31506 31490 131.197 98.625 0.0113448 0.00761458 2023-10-01T11:09:57.400 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:57.426 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:57.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:57.432 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:09:57.533 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:57.562 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:57.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:57.565 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:09:57.665 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:57.690 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:57.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:57.692 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:09:57.793 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:58.930 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:57 smithi078 bash[78274]: cluster 2023-10-01T11:09:56.387976+0000 mgr.y (mgr.25072) 1579 : cluster 0 pgmap v1700: 89 pgs: 89 active+clean; 1.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 124 MiB/s wr, 1.99k op/s 2023-10-01T11:09:58.931 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:57 smithi078 bash[75299]: cluster 2023-10-01T11:09:56.387976+0000 mgr.y (mgr.25072) 1579 : cluster 0 pgmap v1700: 89 pgs: 89 active+clean; 1.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 124 MiB/s wr, 1.99k op/s 2023-10-01T11:09:58.931 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 16 16 33191 33175 129.578 105.312 0.0122828 0.00770982 2023-10-01T11:09:58.932 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:57 smithi173 bash[58816]: cluster 2023-10-01T11:09:56.387976+0000 mgr.y (mgr.25072) 1579 : cluster 0 pgmap v1700: 89 pgs: 89 active+clean; 1.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 124 MiB/s wr, 1.99k op/s 2023-10-01T11:09:58.959 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:58.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:58.965 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:09:59.066 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:59.447 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 17 16 34575 34559 127.044 86.5 0.00876662 0.00786522 2023-10-01T11:09:59.477 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:09:59.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:59.586 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:09:59.686 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:59.714 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:09:59.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:59.717 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:09:59.818 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:59.844 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:09:59.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:59.847 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:09:59.948 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:59.973 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:09:59.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:09:59.976 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:10:00.077 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:00.106 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:00.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:00.109 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:10:00.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:09:59 smithi173 bash[58816]: cluster 2023-10-01T11:09:58.388579+0000 mgr.y (mgr.25072) 1580 : cluster 0 pgmap v1701: 89 pgs: 89 active+clean; 1.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-10-01T11:10:00.210 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:00.218 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:09:59 smithi078 bash[75299]: cluster 2023-10-01T11:09:58.388579+0000 mgr.y (mgr.25072) 1580 : cluster 0 pgmap v1701: 89 pgs: 89 active+clean; 1.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-10-01T11:10:00.218 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:09:59 smithi078 bash[78274]: cluster 2023-10-01T11:09:58.388579+0000 mgr.y (mgr.25072) 1580 : cluster 0 pgmap v1701: 89 pgs: 89 active+clean; 1.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-10-01T11:10:00.233 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:00.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:00.236 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:10:00.336 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:00.396 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 18 16 36139 36123 125.416 97.75 0.013204 0.00796695 2023-10-01T11:10:00.422 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:00.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:00.734 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:10:00 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:10:00] "GET /metrics HTTP/1.1" 200 36366 "" "Prometheus/2.33.4" 2023-10-01T11:10:00.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:10:00.835 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:00.856 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:00.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:00.857 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:10:00.958 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:00.999 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:01.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:01.002 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:10:01.102 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:01.112 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:00 smithi078 bash[78274]: cluster 2023-10-01T11:10:00.000113+0000 mon.a (mon.0) 2048 : cluster 3 Health detail: HEALTH_WARN nodeep-scrub flag(s) set 2023-10-01T11:10:01.241 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:00 smithi078 bash[78274]: cluster 2023-10-01T11:10:00.000157+0000 mon.a (mon.0) 2049 : cluster 3 [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-10-01T11:10:01.247 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:00 smithi078 bash[75299]: cluster 2023-10-01T11:10:00.000113+0000 mon.a (mon.0) 2048 : cluster 3 Health detail: HEALTH_WARN nodeep-scrub flag(s) set 2023-10-01T11:10:01.247 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:00 smithi078 bash[75299]: cluster 2023-10-01T11:10:00.000157+0000 mon.a (mon.0) 2049 : cluster 3 [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-10-01T11:10:01.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:01.248 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:01.248 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:10:01.349 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:01.352 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 19 16 37492 37476 123.265 84.5625 0.00641026 0.00810653 2023-10-01T11:10:01.360 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:00 smithi173 bash[58816]: cluster 2023-10-01T11:10:00.000113+0000 mon.a (mon.0) 2048 : cluster 3 Health detail: HEALTH_WARN nodeep-scrub flag(s) set 2023-10-01T11:10:01.360 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:00 smithi173 bash[58816]: cluster 2023-10-01T11:10:00.000157+0000 mon.a (mon.0) 2049 : cluster 3 [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-10-01T11:10:01.375 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:01.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:01.378 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:10:01.479 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:01.510 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:01.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:01.514 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:10:01.615 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:01.642 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:01.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:01.645 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:10:01.745 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:01.771 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:01.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:01.774 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:10:01.874 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:01.896 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:01.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:01.899 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:10:02.000 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:02.036 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:02.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:02.039 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:02.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:01 smithi078 bash[75299]: cluster 2023-10-01T11:10:00.389220+0000 mgr.y (mgr.25072) 1581 : cluster 0 pgmap v1702: 89 pgs: 89 active+clean; 2.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 146 MiB/s wr, 2.33k op/s 2023-10-01T11:10:02.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:01 smithi078 bash[78274]: cluster 2023-10-01T11:10:00.389220+0000 mgr.y (mgr.25072) 1581 : cluster 0 pgmap v1702: 89 pgs: 89 active+clean; 2.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 146 MiB/s wr, 2.33k op/s 2023-10-01T11:10:02.140 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:02.167 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:02.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:02.172 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:10:02.273 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:02.282 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:01 smithi173 bash[58816]: cluster 2023-10-01T11:10:00.389220+0000 mgr.y (mgr.25072) 1581 : cluster 0 pgmap v1702: 89 pgs: 89 active+clean; 2.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 146 MiB/s wr, 2.33k op/s 2023-10-01T11:10:02.296 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:02.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:02.299 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:02.353 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:2023-10-01T11:10:02.352597+0000 min lat: 0.00136692 max lat: 0.206832 avg lat: 0.00824457 2023-10-01T11:10:02.353 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T11:10:02.353 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 20 16 38809 38793 121.217 82.3125 0.00488493 0.00824457 2023-10-01T11:10:02.399 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:02.428 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:02.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:02.437 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:02.538 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:02.568 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:02.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:02.572 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:02.672 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:02.706 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:02.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:02.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:02.809 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:02.842 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:02.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:02.846 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:10:02.947 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:02.970 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:02.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:02.973 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:10:03.073 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:03.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:02 smithi078 bash[78274]: audit 2023-10-01T11:10:02.465623+0000 mon.a (mon.0) 2050 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:10:03.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:02 smithi078 bash[75299]: audit 2023-10-01T11:10:02.465623+0000 mon.a (mon.0) 2050 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:10:03.098 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:03.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:03.101 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:10:03.202 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:03.241 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:03.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:03.244 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:10:03.345 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:03.352 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 21 16 40065 40049 119.183 78.5 0.0133625 0.00838488 2023-10-01T11:10:03.354 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:02 smithi173 bash[58816]: audit 2023-10-01T11:10:02.465623+0000 mon.a (mon.0) 2050 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:10:03.371 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:03.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:03.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:10:03.476 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:03.500 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:03.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:03.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:10:03.605 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:03.627 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:03.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:03.630 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:10:03.731 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:03.761 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:03.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:03.766 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:10:03.867 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:03.893 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:03.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:03.895 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:03.996 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:04.006 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:03 smithi173 bash[58816]: cluster 2023-10-01T11:10:02.389645+0000 mgr.y (mgr.25072) 1582 : cluster 0 pgmap v1703: 89 pgs: 89 active+clean; 2.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2023-10-01T11:10:04.032 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:04.092 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:03 smithi078 bash[75299]: cluster 2023-10-01T11:10:02.389645+0000 mgr.y (mgr.25072) 1582 : cluster 0 pgmap v1703: 89 pgs: 89 active+clean; 2.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2023-10-01T11:10:04.092 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:03 smithi078 bash[78274]: cluster 2023-10-01T11:10:02.389645+0000 mgr.y (mgr.25072) 1582 : cluster 0 pgmap v1703: 89 pgs: 89 active+clean; 2.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s 2023-10-01T11:10:04.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:04.093 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:10:04.194 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:04.248 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:04.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:04.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:10:04.351 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:04.353 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 22 16 41448 41432 117.694 86.4375 0.00863929 0.00849051 2023-10-01T11:10:04.384 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:04.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:04.387 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:04.487 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:04.514 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:04.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:04.517 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:04.618 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:04.650 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:04.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:04.654 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:04.755 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:04.784 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:04.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:04.787 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:04.887 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:04.922 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:04.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:04.926 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:10:05.026 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:05.053 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:05.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:05.057 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:10:05.158 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:05.182 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:05.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:05.185 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:10:05.286 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:05.318 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:05.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:05.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:05.353 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 23 16 42755 42739 116.128 81.6875 0.00247925 0.00856753 2023-10-01T11:10:05.422 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:05.450 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:05.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:05.455 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:10:05.556 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:05.584 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:05.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:05.590 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:10:05.691 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:05.712 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:05.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:05.715 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:10:05.816 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:05.834 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:05.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:05.836 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:10:05.936 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:05.965 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:05.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:05.968 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:06.069 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:06.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:05 smithi078 bash[78274]: cluster 2023-10-01T11:10:04.390302+0000 mgr.y (mgr.25072) 1583 : cluster 0 pgmap v1704: 89 pgs: 89 active+clean; 2.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 117 MiB/s wr, 1.87k op/s 2023-10-01T11:10:06.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:05 smithi078 bash[75299]: cluster 2023-10-01T11:10:04.390302+0000 mgr.y (mgr.25072) 1583 : cluster 0 pgmap v1704: 89 pgs: 89 active+clean; 2.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 117 MiB/s wr, 1.87k op/s 2023-10-01T11:10:06.099 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:06.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:06.103 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:06.204 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:06.215 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:05 smithi173 bash[58816]: cluster 2023-10-01T11:10:04.390302+0000 mgr.y (mgr.25072) 1583 : cluster 0 pgmap v1704: 89 pgs: 89 active+clean; 2.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 117 MiB/s wr, 1.87k op/s 2023-10-01T11:10:06.230 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:06.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:06.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:06.335 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:06.353 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 24 16 43762 43746 113.911 62.9375 0.00709832 0.00877305 2023-10-01T11:10:06.363 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:06.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:06.372 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:06.473 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:06.493 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:06.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:06.495 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:10:06.595 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:06.630 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:06.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:06.638 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:10:06.738 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:06.765 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:06.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:06.770 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:06.871 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:06.897 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:06.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:06.900 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:10:07.001 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:07.031 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:07.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:07.034 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:10:07.135 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:07.164 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:07.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:07.167 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:10:07.269 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:07.300 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:07.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:07.303 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:10:07.353 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 25 16 44988 44972 112.419 76.625 0.0115061 0.0088895 2023-10-01T11:10:07.404 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:07.427 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:07.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:07.431 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:10:07.532 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:07.558 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:07.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:07.561 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:10:07.661 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:07.686 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:07.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:07.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:10:07.792 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:07.817 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:07.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:07.822 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:07.922 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:07.952 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:07.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:07.955 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:10:08.056 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:08.081 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:08.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:08.084 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:08.184 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:08.195 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:07 smithi078 bash[75299]: cluster 2023-10-01T11:10:06.390873+0000 mgr.y (mgr.25072) 1584 : cluster 0 pgmap v1705: 89 pgs: 89 active+clean; 2.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2023-10-01T11:10:08.196 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:07 smithi078 bash[78274]: cluster 2023-10-01T11:10:06.390873+0000 mgr.y (mgr.25072) 1584 : cluster 0 pgmap v1705: 89 pgs: 89 active+clean; 2.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2023-10-01T11:10:08.211 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T11:10:08.223 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:08.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:08.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:10:08.274 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T11:10:08.327 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:08.331 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T11:10:08.346 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2023-10-01T11:10:08.346 INFO:teuthology.orchestra.run.smithi078.stdout: "ops": [ 2023-10-01T11:10:08.346 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.346 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.36819.0:46089 14.f 14:f1ddf75b:::benchmark_data_smithi078_236918_object46088:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e359)", 2023-10-01T11:10:08.346 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:10:08.333766+0000", 2023-10-01T11:10:08.346 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 0.010528063000000001, 2023-10-01T11:10:08.346 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.010559010000000001, 2023-10-01T11:10:08.346 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:10:08.346 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "waiting for sub ops", 2023-10-01T11:10:08.346 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T11:10:08.347 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.36819", 2023-10-01T11:10:08.347 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1409915880", 2023-10-01T11:10:08.347 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 46089 2023-10-01T11:10:08.347 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.347 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:10:08.347 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.347 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:10:08.347 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:08.333766+0000", 2023-10-01T11:10:08.347 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:10:08.347 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.347 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.347 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:10:08.347 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:08.333766+0000", 2023-10-01T11:10:08.348 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:10:08.348 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.348 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.348 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:10:08.348 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:08.333764+0000", 2023-10-01T11:10:08.348 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999981 2023-10-01T11:10:08.348 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.348 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.348 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:10:08.348 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:08.333818+0000", 2023-10-01T11:10:08.348 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.3715000000000003e-05 2023-10-01T11:10:08.348 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.349 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.349 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:10:08.349 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:08.333837+0000", 2023-10-01T11:10:08.349 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.9317000000000002e-05 2023-10-01T11:10:08.349 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.349 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.349 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:10:08.349 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:08.333846+0000", 2023-10-01T11:10:08.349 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.8319999999999995e-06 2023-10-01T11:10:08.349 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.349 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.349 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:10:08.350 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:08.334395+0000", 2023-10-01T11:10:08.350 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00054969700000000003 2023-10-01T11:10:08.350 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.350 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.350 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:10:08.350 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:08.334539+0000", 2023-10-01T11:10:08.350 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00014381100000000001 2023-10-01T11:10:08.350 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.350 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.350 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 1,7", 2023-10-01T11:10:08.350 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:08.334687+0000", 2023-10-01T11:10:08.351 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000147334 2023-10-01T11:10:08.351 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.351 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.351 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T11:10:08.351 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:08.339896+0000", 2023-10-01T11:10:08.351 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0052095910000000004 2023-10-01T11:10:08.351 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:10:08.351 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:10:08.351 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:10:08.351 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.351 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.351 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.36819.0:46075 14.3 14:ce5825b0:::benchmark_data_smithi078_236918_object46074:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e359)", 2023-10-01T11:10:08.352 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:10:08.317239+0000", 2023-10-01T11:10:08.352 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 0.027055078, 2023-10-01T11:10:08.352 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.027169649000000001, 2023-10-01T11:10:08.352 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:10:08.352 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "waiting for sub ops", 2023-10-01T11:10:08.352 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T11:10:08.352 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.36819", 2023-10-01T11:10:08.352 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1409915880", 2023-10-01T11:10:08.352 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 46075 2023-10-01T11:10:08.352 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.352 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:10:08.352 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.352 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:10:08.353 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:08.317239+0000", 2023-10-01T11:10:08.353 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:10:08.353 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.353 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.353 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:10:08.353 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:08.317239+0000", 2023-10-01T11:10:08.353 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:10:08.353 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.353 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.353 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:10:08.353 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:08.317237+0000", 2023-10-01T11:10:08.353 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999986 2023-10-01T11:10:08.354 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.354 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.354 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:10:08.354 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:08.318209+0000", 2023-10-01T11:10:08.354 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00097221600000000003 2023-10-01T11:10:08.354 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.354 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.354 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:10:08.354 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:08.318223+0000", 2023-10-01T11:10:08.354 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.4071e-05 2023-10-01T11:10:08.354 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.354 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.355 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:10:08.355 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:08.318231+0000", 2023-10-01T11:10:08.355 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.9799999999999998e-06 2023-10-01T11:10:08.355 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.355 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.355 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:10:08.355 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:08.318281+0000", 2023-10-01T11:10:08.355 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.9169000000000003e-05 2023-10-01T11:10:08.355 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.355 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.355 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:10:08.355 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:08.318424+0000", 2023-10-01T11:10:08.356 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000142972 2023-10-01T11:10:08.356 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.356 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.356 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 0,1", 2023-10-01T11:10:08.356 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:08.318568+0000", 2023-10-01T11:10:08.356 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00014453900000000001 2023-10-01T11:10:08.356 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.356 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.356 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:10:08.356 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:08.331506+0000", 2023-10-01T11:10:08.356 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.012937666 2023-10-01T11:10:08.356 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.356 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.357 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T11:10:08.357 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:08.338780+0000", 2023-10-01T11:10:08.357 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0072742600000000003 2023-10-01T11:10:08.357 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:10:08.357 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:10:08.357 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:10:08.357 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.357 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.357 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.36819.0:46078 14.3 14:c26dee58:::benchmark_data_smithi078_236918_object46077:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e359)", 2023-10-01T11:10:08.357 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:10:08.320749+0000", 2023-10-01T11:10:08.357 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 0.023545087999999999, 2023-10-01T11:10:08.357 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.023723810000000001, 2023-10-01T11:10:08.358 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:10:08.358 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "waiting for sub ops", 2023-10-01T11:10:08.358 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T11:10:08.358 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.36819", 2023-10-01T11:10:08.358 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1409915880", 2023-10-01T11:10:08.358 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 46078 2023-10-01T11:10:08.358 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.358 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:10:08.358 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.358 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:10:08.358 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:08.320749+0000", 2023-10-01T11:10:08.358 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:10:08.358 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.359 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.359 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:10:08.359 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:08.320749+0000", 2023-10-01T11:10:08.359 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:10:08.359 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.359 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.359 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:10:08.359 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:08.320748+0000", 2023-10-01T11:10:08.359 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T11:10:08.359 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.359 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.359 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:10:08.360 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:08.320768+0000", 2023-10-01T11:10:08.360 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.0633999999999999e-05 2023-10-01T11:10:08.360 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.360 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.360 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:10:08.360 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:08.320781+0000", 2023-10-01T11:10:08.360 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.2422999999999999e-05 2023-10-01T11:10:08.360 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.360 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.360 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:10:08.360 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:08.320789+0000", 2023-10-01T11:10:08.360 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.1890000000000007e-06 2023-10-01T11:10:08.360 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.361 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.361 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:10:08.361 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:08.323926+0000", 2023-10-01T11:10:08.361 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0031371760000000002 2023-10-01T11:10:08.361 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.361 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.361 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:10:08.361 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:08.324395+0000", 2023-10-01T11:10:08.361 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00046924899999999998 2023-10-01T11:10:08.361 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.361 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.361 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 0,1", 2023-10-01T11:10:08.362 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:08.324541+0000", 2023-10-01T11:10:08.362 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00014595100000000001 2023-10-01T11:10:08.362 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.362 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.362 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:10:08.362 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:08.331536+0000", 2023-10-01T11:10:08.362 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.006994569 2023-10-01T11:10:08.362 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.362 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.363 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T11:10:08.363 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:08.338787+0000", 2023-10-01T11:10:08.363 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0072508909999999998 2023-10-01T11:10:08.363 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:10:08.363 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:10:08.363 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:10:08.363 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.363 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.363 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.36819.0:46082 14.3 14:cef01ad9:::benchmark_data_smithi078_236918_object46081:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e359)", 2023-10-01T11:10:08.363 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:10:08.323813+0000", 2023-10-01T11:10:08.363 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 0.020480339, 2023-10-01T11:10:08.363 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.020720849, 2023-10-01T11:10:08.364 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:10:08.364 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "waiting for sub ops", 2023-10-01T11:10:08.364 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T11:10:08.364 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.36819", 2023-10-01T11:10:08.364 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1409915880", 2023-10-01T11:10:08.364 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 46082 2023-10-01T11:10:08.364 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.364 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:10:08.364 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.364 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:10:08.364 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:08.323813+0000", 2023-10-01T11:10:08.364 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:10:08.364 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.365 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.365 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:10:08.365 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:08.323813+0000", 2023-10-01T11:10:08.365 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:10:08.365 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.365 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.365 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:10:08.365 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:08.323812+0000", 2023-10-01T11:10:08.365 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999986 2023-10-01T11:10:08.365 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.365 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.365 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:10:08.366 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:08.323835+0000", 2023-10-01T11:10:08.366 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.3382000000000001e-05 2023-10-01T11:10:08.366 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.366 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.366 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:10:08.366 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:08.323852+0000", 2023-10-01T11:10:08.366 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.6895e-05 2023-10-01T11:10:08.366 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.366 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.366 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:10:08.366 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:08.323860+0000", 2023-10-01T11:10:08.366 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.7649999999999993e-06 2023-10-01T11:10:08.367 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.367 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.367 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:10:08.367 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:08.327047+0000", 2023-10-01T11:10:08.367 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.003186641 2023-10-01T11:10:08.367 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.367 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.367 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:10:08.367 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:08.327185+0000", 2023-10-01T11:10:08.367 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00013794400000000001 2023-10-01T11:10:08.367 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.367 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.368 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 0,1", 2023-10-01T11:10:08.368 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:08.327333+0000", 2023-10-01T11:10:08.368 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00014870000000000001 2023-10-01T11:10:08.368 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.368 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.368 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T11:10:08.368 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:08.338792+0000", 2023-10-01T11:10:08.368 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.011458462000000001 2023-10-01T11:10:08.368 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:10:08.368 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:10:08.368 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:10:08.368 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.369 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.369 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.36819.0:46087 14.f 14:f31279a7:::benchmark_data_smithi078_236918_object46086:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e359)", 2023-10-01T11:10:08.369 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:10:08.329987+0000", 2023-10-01T11:10:08.369 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 0.014306753, 2023-10-01T11:10:08.369 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.014601538000000001, 2023-10-01T11:10:08.369 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:10:08.369 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "waiting for sub ops", 2023-10-01T11:10:08.369 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T11:10:08.369 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.36819", 2023-10-01T11:10:08.369 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1409915880", 2023-10-01T11:10:08.369 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 46087 2023-10-01T11:10:08.369 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.369 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:10:08.370 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.370 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:10:08.370 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:08.329987+0000", 2023-10-01T11:10:08.370 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:10:08.370 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.370 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.370 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:10:08.370 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:08.329987+0000", 2023-10-01T11:10:08.370 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:10:08.370 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.370 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.370 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:10:08.371 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:08.329985+0000", 2023-10-01T11:10:08.371 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999981 2023-10-01T11:10:08.371 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.371 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.371 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:10:08.371 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:08.330015+0000", 2023-10-01T11:10:08.371 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.0391000000000001e-05 2023-10-01T11:10:08.371 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.371 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.371 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:10:08.371 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:08.330036+0000", 2023-10-01T11:10:08.371 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.0407e-05 2023-10-01T11:10:08.371 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.372 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.372 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:10:08.372 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:08.330045+0000", 2023-10-01T11:10:08.372 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.4539999999999996e-06 2023-10-01T11:10:08.372 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.372 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.372 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:10:08.372 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:08.332042+0000", 2023-10-01T11:10:08.372 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0019969129999999999 2023-10-01T11:10:08.372 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.372 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.372 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:10:08.373 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:08.332175+0000", 2023-10-01T11:10:08.373 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000133218 2023-10-01T11:10:08.373 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.373 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.373 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 1,7", 2023-10-01T11:10:08.373 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:08.332316+0000", 2023-10-01T11:10:08.373 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000140801 2023-10-01T11:10:08.373 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.373 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.373 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T11:10:08.373 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:08.339891+0000", 2023-10-01T11:10:08.373 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0075748949999999999 2023-10-01T11:10:08.374 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:10:08.374 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:10:08.374 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:10:08.374 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.374 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.374 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.36819.0:46088 14.1 14:88bb2c99:::benchmark_data_smithi078_236918_object46087:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected e359)", 2023-10-01T11:10:08.374 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:10:08.330059+0000", 2023-10-01T11:10:08.374 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 0.014235166, 2023-10-01T11:10:08.374 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.014577817999999999, 2023-10-01T11:10:08.374 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:10:08.374 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "waiting for sub ops", 2023-10-01T11:10:08.374 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T11:10:08.375 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.36819", 2023-10-01T11:10:08.375 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1409915880", 2023-10-01T11:10:08.375 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 46088 2023-10-01T11:10:08.375 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.375 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:10:08.375 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.375 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:10:08.375 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:08.330059+0000", 2023-10-01T11:10:08.375 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:10:08.375 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.375 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.375 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:10:08.376 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:08.330059+0000", 2023-10-01T11:10:08.376 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:10:08.376 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.376 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.376 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:10:08.376 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:08.330058+0000", 2023-10-01T11:10:08.376 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999995 2023-10-01T11:10:08.376 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.376 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.376 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:10:08.376 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:08.330110+0000", 2023-10-01T11:10:08.376 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.2116e-05 2023-10-01T11:10:08.376 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.377 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.377 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:10:08.377 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:08.330118+0000", 2023-10-01T11:10:08.377 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.7009999999999996e-06 2023-10-01T11:10:08.377 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.377 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.377 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:10:08.377 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:08.330124+0000", 2023-10-01T11:10:08.377 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.9449999999999998e-06 2023-10-01T11:10:08.377 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.377 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.377 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:10:08.378 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:08.330178+0000", 2023-10-01T11:10:08.378 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.3968000000000002e-05 2023-10-01T11:10:08.378 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.378 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.378 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:10:08.378 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:08.330322+0000", 2023-10-01T11:10:08.378 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000143886 2023-10-01T11:10:08.378 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.378 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.378 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 1,2", 2023-10-01T11:10:08.378 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:08.330479+0000", 2023-10-01T11:10:08.378 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00015684299999999999 2023-10-01T11:10:08.379 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.379 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.379 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:10:08.379 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:08.339137+0000", 2023-10-01T11:10:08.379 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0086583670000000001 2023-10-01T11:10:08.379 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:08.379 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:08.379 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T11:10:08.379 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:08.339184+0000", 2023-10-01T11:10:08.379 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.7407000000000002e-05 2023-10-01T11:10:08.379 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:10:08.379 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:10:08.380 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:10:08.380 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:10:08.380 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2023-10-01T11:10:08.380 INFO:teuthology.orchestra.run.smithi078.stdout: "num_ops": 6 2023-10-01T11:10:08.380 INFO:teuthology.orchestra.run.smithi078.stdout:} 2023-10-01T11:10:08.380 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:07 smithi173 bash[58816]: cluster 2023-10-01T11:10:06.390873+0000 mgr.y (mgr.25072) 1584 : cluster 0 pgmap v1705: 89 pgs: 89 active+clean; 2.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2023-10-01T11:10:08.381 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 26 16 46092 46076 110.749 69 0.03743 0.00901543 2023-10-01T11:10:08.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:08.382 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:08.382 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:10:08.403 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T11:10:08.447 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T11:10:08.483 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:08.488 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T11:10:08.512 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:08.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:08.515 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:08.532 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T11:10:08.572 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T11:10:08.581 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T11:10:08.581 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T11:10:08.581 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T11:10:08.581 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T11:10:08.581 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T11:10:08.581 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T11:10:08.582 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T11:10:08.582 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T11:10:08.602 INFO:teuthology.orchestra.run.smithi078.stdout: 2023-10-01T11:10:08.610 INFO:teuthology.orchestra.run.smithi078.stderr:dumped all 2023-10-01T11:10:08.615 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:08.648 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:08.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:08.654 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:10:08.755 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:08.793 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:08.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:08.797 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:10:08.897 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:08.928 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:08.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:08.932 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:10:08.979 INFO:teuthology.orchestra.run.smithi078.stderr:nodeep-scrub is unset 2023-10-01T11:10:09.032 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:09.061 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:09.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:09.064 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:09.165 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:09.197 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:09.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:09.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:09.300 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:09.308 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:08 smithi078 bash[75299]: audit 2023-10-01T11:10:08.149369+0000 mon.a (mon.0) 2051 : audit 0 from='client.? 172.21.15.78:0/914612183' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:10:09.308 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:08 smithi078 bash[75299]: audit 2023-10-01T11:10:08.536924+0000 mon.c (mon.2) 238 : audit 1 from='client.? 172.21.15.78:0/1664081677' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:10:09.308 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:08 smithi078 bash[75299]: audit 2023-10-01T11:10:08.537683+0000 mon.a (mon.0) 2052 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:10:09.309 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:08 smithi078 bash[78274]: audit 2023-10-01T11:10:08.149369+0000 mon.a (mon.0) 2051 : audit 0 from='client.? 172.21.15.78:0/914612183' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:10:09.309 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:08 smithi078 bash[78274]: audit 2023-10-01T11:10:08.536924+0000 mon.c (mon.2) 238 : audit 1 from='client.? 172.21.15.78:0/1664081677' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:10:09.309 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:08 smithi078 bash[78274]: audit 2023-10-01T11:10:08.537683+0000 mon.a (mon.0) 2052 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:10:09.327 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:09.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:09.329 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:10:09.353 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 27 16 47277 47261 109.39 74.0625 0.0189158 0.00913628 2023-10-01T11:10:09.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:08 smithi173 bash[58816]: audit 2023-10-01T11:10:08.149369+0000 mon.a (mon.0) 2051 : audit 0 from='client.? 172.21.15.78:0/914612183' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:10:09.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:08 smithi173 bash[58816]: audit 2023-10-01T11:10:08.536924+0000 mon.c (mon.2) 238 : audit 1 from='client.? 172.21.15.78:0/1664081677' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:10:09.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:08 smithi173 bash[58816]: audit 2023-10-01T11:10:08.537683+0000 mon.a (mon.0) 2052 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:10:09.430 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:09.467 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:09.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:09.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:10:09.572 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:09.599 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:09.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:09.606 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:10:09.707 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:09.733 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:09.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:09.736 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:09.837 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:09.864 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:09.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:09.868 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:09.961 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:09 smithi173 bash[58816]: cluster 2023-10-01T11:10:08.391327+0000 mgr.y (mgr.25072) 1585 : cluster 0 pgmap v1706: 89 pgs: 89 active+clean; 2.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-10-01T11:10:09.961 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:09 smithi173 bash[58816]: audit 2023-10-01T11:10:08.584738+0000 mgr.y (mgr.25072) 1586 : audit 0 from='client.46784 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T11:10:09.962 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:09 smithi173 bash[58816]: cluster 2023-10-01T11:10:08.945326+0000 mon.a (mon.0) 2053 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T11:10:09.962 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:09 smithi173 bash[58816]: cluster 2023-10-01T11:10:08.945367+0000 mon.a (mon.0) 2054 : cluster 1 Cluster is now healthy 2023-10-01T11:10:09.962 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:09 smithi173 bash[58816]: cluster 2023-10-01T11:10:08.956540+0000 osd.5 (osd.5) 382 : cluster 3 failed to encode map e360 with expected crc 2023-10-01T11:10:09.962 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:09 smithi173 bash[58816]: cluster 2023-10-01T11:10:08.960925+0000 osd.7 (osd.7) 403 : cluster 3 failed to encode map e360 with expected crc 2023-10-01T11:10:09.962 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:09 smithi173 bash[58816]: audit 2023-10-01T11:10:08.962287+0000 mon.a (mon.0) 2055 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T11:10:09.962 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:09 smithi173 bash[58816]: cluster 2023-10-01T11:10:08.962372+0000 mon.a (mon.0) 2056 : cluster 0 osdmap e360: 8 total, 8 up, 8 in 2023-10-01T11:10:09.962 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:09 smithi173 bash[58816]: cluster 2023-10-01T11:10:08.964815+0000 osd.7 (osd.7) 404 : cluster 3 failed to encode map e360 with expected crc 2023-10-01T11:10:09.962 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:09 smithi173 bash[58816]: cluster 2023-10-01T11:10:08.967260+0000 osd.6 (osd.6) 345 : cluster 3 failed to encode map e360 with expected crc 2023-10-01T11:10:09.969 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:09.992 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:09.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:09.997 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:10:10.097 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:10.120 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:10.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:10.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:10.224 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:10.233 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:09 smithi078 bash[75299]: cluster 2023-10-01T11:10:08.391327+0000 mgr.y (mgr.25072) 1585 : cluster 0 pgmap v1706: 89 pgs: 89 active+clean; 2.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-10-01T11:10:10.233 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:09 smithi078 bash[75299]: audit 2023-10-01T11:10:08.584738+0000 mgr.y (mgr.25072) 1586 : audit 0 from='client.46784 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T11:10:10.233 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:09 smithi078 bash[75299]: cluster 2023-10-01T11:10:08.945326+0000 mon.a (mon.0) 2053 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T11:10:10.233 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:09 smithi078 bash[75299]: cluster 2023-10-01T11:10:08.945367+0000 mon.a (mon.0) 2054 : cluster 1 Cluster is now healthy 2023-10-01T11:10:10.234 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:09 smithi078 bash[75299]: cluster 2023-10-01T11:10:08.956540+0000 osd.5 (osd.5) 382 : cluster 3 failed to encode map e360 with expected crc 2023-10-01T11:10:10.234 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:09 smithi078 bash[75299]: cluster 2023-10-01T11:10:08.960925+0000 osd.7 (osd.7) 403 : cluster 3 failed to encode map e360 with expected crc 2023-10-01T11:10:10.234 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:09 smithi078 bash[75299]: audit 2023-10-01T11:10:08.962287+0000 mon.a (mon.0) 2055 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T11:10:10.234 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:09 smithi078 bash[75299]: cluster 2023-10-01T11:10:08.962372+0000 mon.a (mon.0) 2056 : cluster 0 osdmap e360: 8 total, 8 up, 8 in 2023-10-01T11:10:10.234 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:09 smithi078 bash[75299]: cluster 2023-10-01T11:10:08.964815+0000 osd.7 (osd.7) 404 : cluster 3 failed to encode map e360 with expected crc 2023-10-01T11:10:10.234 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:09 smithi078 bash[75299]: cluster 2023-10-01T11:10:08.967260+0000 osd.6 (osd.6) 345 : cluster 3 failed to encode map e360 with expected crc 2023-10-01T11:10:10.234 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:09 smithi078 bash[75299]: cluster 2023-10-01T11:10:08.969248+0000 osd.6 (osd.6) 346 : cluster 3 failed to encode map e360 with expected crc 2023-10-01T11:10:10.234 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:09 smithi078 bash[78274]: cluster 2023-10-01T11:10:08.391327+0000 mgr.y (mgr.25072) 1585 : cluster 0 pgmap v1706: 89 pgs: 89 active+clean; 2.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-10-01T11:10:10.234 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:09 smithi078 bash[78274]: audit 2023-10-01T11:10:08.584738+0000 mgr.y (mgr.25072) 1586 : audit 0 from='client.46784 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T11:10:10.235 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:09 smithi078 bash[78274]: cluster 2023-10-01T11:10:08.945326+0000 mon.a (mon.0) 2053 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T11:10:10.235 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:09 smithi078 bash[78274]: cluster 2023-10-01T11:10:08.945367+0000 mon.a (mon.0) 2054 : cluster 1 Cluster is now healthy 2023-10-01T11:10:10.235 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:09 smithi078 bash[78274]: cluster 2023-10-01T11:10:08.956540+0000 osd.5 (osd.5) 382 : cluster 3 failed to encode map e360 with expected crc 2023-10-01T11:10:10.235 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:09 smithi078 bash[78274]: cluster 2023-10-01T11:10:08.960925+0000 osd.7 (osd.7) 403 : cluster 3 failed to encode map e360 with expected crc 2023-10-01T11:10:10.235 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:09 smithi078 bash[78274]: audit 2023-10-01T11:10:08.962287+0000 mon.a (mon.0) 2055 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T11:10:10.235 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:09 smithi078 bash[78274]: cluster 2023-10-01T11:10:08.962372+0000 mon.a (mon.0) 2056 : cluster 0 osdmap e360: 8 total, 8 up, 8 in 2023-10-01T11:10:10.235 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:09 smithi078 bash[78274]: cluster 2023-10-01T11:10:08.964815+0000 osd.7 (osd.7) 404 : cluster 3 failed to encode map e360 with expected crc 2023-10-01T11:10:10.235 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:09 smithi078 bash[78274]: cluster 2023-10-01T11:10:08.967260+0000 osd.6 (osd.6) 345 : cluster 3 failed to encode map e360 with expected crc 2023-10-01T11:10:10.235 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:09 smithi078 bash[78274]: cluster 2023-10-01T11:10:08.969248+0000 osd.6 (osd.6) 346 : cluster 3 failed to encode map e360 with expected crc 2023-10-01T11:10:10.251 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:10.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:10.256 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:10:10.353 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 28 15 48489 48474 108.191 75.8125 0.00839951 0.00923684 2023-10-01T11:10:10.357 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:10.366 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:09 smithi173 bash[58816]: cluster 2023-10-01T11:10:08.969248+0000 osd.6 (osd.6) 346 : cluster 3 failed to encode map e360 with expected crc 2023-10-01T11:10:10.381 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:10.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:10.384 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:10.484 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:10.510 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:10.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:10.514 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:10:10.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:10:10 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:10:10] "GET /metrics HTTP/1.1" 200 36380 "" "Prometheus/2.33.4" 2023-10-01T11:10:10.615 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:10.642 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:10.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:10.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:10.745 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:10.767 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:10.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:10.770 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:10:10.871 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:10.895 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:10.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:10.900 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:10:11.001 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:11.027 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:11.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:11.030 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:10:11.132 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:11.157 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:11.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:11.162 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:10:11.263 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:11.291 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:11.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:11.293 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:10:11.354 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 29 16 49800 49784 107.283 81.875 0.0149876 0.00931627 2023-10-01T11:10:11.394 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:11.421 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:11.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:11.424 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:10:11.525 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:11.552 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:11.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:11.556 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:10:11.656 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:11.683 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:11.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:11.686 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:11.786 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:11.809 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:11.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:11.811 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:10:11.911 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:11.940 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:11.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:11.944 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:10:11.963 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:11 smithi173 bash[58816]: cluster 2023-10-01T11:10:10.391897+0000 mgr.y (mgr.25072) 1587 : cluster 0 pgmap v1708: 89 pgs: 89 active+clean; 2.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-10-01T11:10:12.045 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:12.054 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:11 smithi078 bash[78274]: cluster 2023-10-01T11:10:10.391897+0000 mgr.y (mgr.25072) 1587 : cluster 0 pgmap v1708: 89 pgs: 89 active+clean; 2.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-10-01T11:10:12.054 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:11 smithi078 bash[75299]: cluster 2023-10-01T11:10:10.391897+0000 mgr.y (mgr.25072) 1587 : cluster 0 pgmap v1708: 89 pgs: 89 active+clean; 2.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-10-01T11:10:12.076 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:12.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:12.083 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:10:12.185 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:12.212 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:12.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:12.216 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:10:12.317 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:12.351 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:12.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:12.353 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:10:12.354 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 30 16 51057 51041 106.325 78.5625 0.0146601 0.00939965 2023-10-01T11:10:12.454 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:12.485 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:12.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:12.487 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:10:12.588 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:12.614 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:12.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:12.616 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:10:12.717 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:12.753 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:12.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:12.756 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:10:12.856 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:12.883 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:12.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:12.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:12.989 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:13.021 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:13.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:13.024 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:10:13.125 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:13.152 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:13.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:13.154 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:10:13.256 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-01T11:10:13.256 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:13.282 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:13.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:13.285 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:10:13.354 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 31 16 52316 52300 105.434 78.6875 0.00815462 0.00947973 2023-10-01T11:10:13.386 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:13.416 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:13.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:13.419 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:13.519 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:13.548 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:13.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:13.551 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:13.639 INFO:teuthology.orchestra.run.smithi078.stdout:{"pg_ready":true,"pg_map":{"version":1706,"stamp":"2023-10-01T11:10:08.391068+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2705135520,"num_objects":41292,"num_object_clones":0,"num_object_copies":123876,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":41292,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":41529,"num_write_kb":2643598,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":100,"num_bytes_recovered":5347666,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":41463,"ondisk_log_size":41463,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":272,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":16073548,"kb_used_data":8142284,"kb_used_omap":30,"kb_used_meta":7931169,"kb_avail":733887668,"statfs":{"total":767960285184,"available":751500972032,"internally_reserved":0,"allocated":8337698816,"data_stored":8329221933,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31502,"internal_metadata":8121517298},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,3,10,4,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":967507968,"num_objects":14763,"num_object_clones":0,"num_object_copies":44289,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":14763,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14763,"num_write_kb":944832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003464"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":341,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:22.150787+0000","last_change":"2023-10-01T11:04:02.725956+0000","last_active":"2023-10-01T11:09:22.150787+0000","last_peered":"2023-10-01T11:09:22.150787+0000","last_clean":"2023-10-01T11:09:22.150787+0000","last_became_active":"2023-10-01T11:04:02.725317+0000","last_became_peered":"2023-10-01T11:04:02.725317+0000","last_unstale":"2023-10-01T11:09:22.150787+0000","last_undegraded":"2023-10-01T11:09:22.150787+0000","last_fullsized":"2023-10-01T11:09:22.150787+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:34:07.675650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":345,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:23.204273+0000","last_change":"2023-10-01T10:38:16.025881+0000","last_active":"2023-10-01T11:09:23.204273+0000","last_peered":"2023-10-01T11:09:23.204273+0000","last_clean":"2023-10-01T11:09:23.204273+0000","last_became_active":"2023-10-01T10:38:16.025518+0000","last_became_peered":"2023-10-01T10:38:16.025518+0000","last_unstale":"2023-10-01T11:09:23.204273+0000","last_undegraded":"2023-10-01T11:09:23.204273+0000","last_fullsized":"2023-10-01T11:09:23.204273+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:48:37.826604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":155,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:22.261043+0000","last_change":"2023-10-01T11:08:46.870944+0000","last_active":"2023-10-01T11:09:22.261043+0000","last_peered":"2023-10-01T11:09:22.261043+0000","last_clean":"2023-10-01T11:09:22.261043+0000","last_became_active":"2023-10-01T11:08:46.870425+0000","last_became_peered":"2023-10-01T11:08:46.870425+0000","last_unstale":"2023-10-01T11:09:22.261043+0000","last_undegraded":"2023-10-01T11:09:22.261043+0000","last_fullsized":"2023-10-01T11:09:22.261043+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:28:03.064900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":363,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:23.204092+0000","last_change":"2023-10-01T11:06:11.786067+0000","last_active":"2023-10-01T11:09:23.204092+0000","last_peered":"2023-10-01T11:09:23.204092+0000","last_clean":"2023-10-01T11:09:23.204092+0000","last_became_active":"2023-10-01T11:06:11.785586+0000","last_became_peered":"2023-10-01T11:06:11.785586+0000","last_unstale":"2023-10-01T11:09:23.204092+0000","last_undegraded":"2023-10-01T11:09:23.204092+0000","last_fullsized":"2023-10-01T11:09:23.204092+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:09:53.087537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":347,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:23.203448+0000","last_change":"2023-10-01T10:37:42.766600+0000","last_active":"2023-10-01T11:09:23.203448+0000","last_peered":"2023-10-01T11:09:23.203448+0000","last_clean":"2023-10-01T11:09:23.203448+0000","last_became_active":"2023-10-01T10:37:42.766051+0000","last_became_peered":"2023-10-01T10:37:42.766051+0000","last_unstale":"2023-10-01T11:09:23.203448+0000","last_undegraded":"2023-10-01T11:09:23.203448+0000","last_fullsized":"2023-10-01T11:09:23.203448+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:51:17.707963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":45,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:23.210472+0000","last_change":"2023-10-01T11:05:33.800368+0000","last_active":"2023-10-01T11:09:23.210472+0000","last_peered":"2023-10-01T11:09:23.210472+0000","last_clean":"2023-10-01T11:09:23.210472+0000","last_became_active":"2023-10-01T11:05:33.799877+0000","last_became_peered":"2023-10-01T11:05:33.799877+0000","last_unstale":"2023-10-01T11:09:23.210472+0000","last_undegraded":"2023-10-01T11:09:23.210472+0000","last_fullsized":"2023-10-01T11:09:23.210472+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":119,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:22.157363+0000","last_change":"2023-10-01T11:04:40.881723+0000","last_active":"2023-10-01T11:09:22.157363+0000","last_peered":"2023-10-01T11:09:22.157363+0000","last_clean":"2023-10-01T11:09:22.157363+0000","last_became_active":"2023-10-01T11:04:40.881381+0000","last_became_peered":"2023-10-01T11:04:40.881381+0000","last_unstale":"2023-10-01T11:09:22.157363+0000","last_undegraded":"2023-10-01T11:09:22.157363+0000","last_fullsized":"2023-10-01T11:09:22.157363+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":385,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:23.203516+0000","last_change":"2023-10-01T11:05:33.782561+0000","last_active":"2023-10-01T11:09:23.203516+0000","last_peered":"2023-10-01T11:09:23.203516+0000","last_clean":"2023-10-01T11:09:23.203516+0000","last_became_active":"2023-10-01T11:05:33.782066+0000","last_became_peered":"2023-10-01T11:05:33.782066+0000","last_unstale":"2023-10-01T11:09:23.203516+0000","last_undegraded":"2023-10-01T11:09:23.203516+0000","last_fullsized":"2023-10-01T11:09:23.203516+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:57:06.451343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":362,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:22.156996+0000","last_change":"2023-10-01T11:06:11.792061+0000","last_active":"2023-10-01T11:09:22.156996+0000","last_peered":"2023-10-01T11:09:22.156996+0000","last_clean":"2023-10-01T11:09:22.156996+0000","last_became_active":"2023-10-01T11:06:11.789129+0000","last_became_peered":"2023-10-01T11:06:11.789129+0000","last_unstale":"2023-10-01T11:09:22.156996+0000","last_undegraded":"2023-10-01T11:09:22.156996+0000","last_fullsized":"2023-10-01T11:09:22.156996+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":405,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:23.203791+0000","last_change":"2023-10-01T11:06:11.807749+0000","last_active":"2023-10-01T11:09:23.203791+0000","last_peered":"2023-10-01T11:09:23.203791+0000","last_clean":"2023-10-01T11:09:23.203791+0000","last_became_active":"2023-10-01T11:06:11.807375+0000","last_became_peered":"2023-10-01T11:06:11.807375+0000","last_unstale":"2023-10-01T11:09:23.203791+0000","last_undegraded":"2023-10-01T11:09:23.203791+0000","last_fullsized":"2023-10-01T11:09:23.203791+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:44:05.945387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":265,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-01T11:09:37.853585+0000","last_change":"2023-10-01T11:06:11.816274+0000","last_active":"2023-10-01T11:09:37.853585+0000","last_peered":"2023-10-01T11:09:37.853585+0000","last_clean":"2023-10-01T11:09:37.853585+0000","last_became_active":"2023-10-01T11:06:11.790196+0000","last_became_peered":"2023-10-01T11:06:11.790196+0000","last_unstale":"2023-10-01T11:09:37.853585+0000","last_undegraded":"2023-10-01T11:09:37.853585+0000","last_fullsized":"2023-10-01T11:09:37.853585+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:09:46.957797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":140,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:22.261372+0000","last_change":"2023-10-01T11:08:46.887634+0000","last_active":"2023-10-01T11:09:22.261372+0000","last_peered":"2023-10-01T11:09:22.261372+0000","last_clean":"2023-10-01T11:09:22.261372+0000","last_became_active":"2023-10-01T11:08:46.887226+0000","last_became_peered":"2023-10-01T11:08:46.887226+0000","last_unstale":"2023-10-01T11:09:22.261372+0000","last_undegraded":"2023-10-01T11:09:22.261372+0000","last_fullsized":"2023-10-01T11:09:22.261372+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:41:08.771365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":469,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:22.151245+0000","last_change":"2023-10-01T11:06:11.836035+0000","last_active":"2023-10-01T11:09:22.151245+0000","last_peered":"2023-10-01T11:09:22.151245+0000","last_clean":"2023-10-01T11:09:22.151245+0000","last_became_active":"2023-10-01T11:06:11.794834+0000","last_became_peered":"2023-10-01T11:06:11.794834+0000","last_unstale":"2023-10-01T11:09:22.151245+0000","last_undegraded":"2023-10-01T11:09:22.151245+0000","last_fullsized":"2023-10-01T11:09:22.151245+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:25:13.103100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":33138,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":364,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:22.150868+0000","last_change":"2023-10-01T11:05:33.782898+0000","last_active":"2023-10-01T11:09:22.150868+0000","last_peered":"2023-10-01T11:09:22.150868+0000","last_clean":"2023-10-01T11:09:22.150868+0000","last_became_active":"2023-10-01T11:05:33.782135+0000","last_became_peered":"2023-10-01T11:05:33.782135+0000","last_unstale":"2023-10-01T11:09:22.150868+0000","last_undegraded":"2023-10-01T11:09:22.150868+0000","last_fullsized":"2023-10-01T11:09:22.150868+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:56:49.807416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":111,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:22.151100+0000","last_change":"2023-10-01T11:06:55.803414+0000","last_active":"2023-10-01T11:09:22.151100+0000","last_peered":"2023-10-01T11:09:22.151100+0000","last_clean":"2023-10-01T11:09:22.151100+0000","last_became_active":"2023-10-01T11:06:55.802738+0000","last_became_peered":"2023-10-01T11:06:55.802738+0000","last_unstale":"2023-10-01T11:09:22.151100+0000","last_undegraded":"2023-10-01T11:09:22.151100+0000","last_fullsized":"2023-10-01T11:09:22.151100+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:55:57.911799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":390,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:23.203947+0000","last_change":"2023-10-01T11:06:11.789794+0000","last_active":"2023-10-01T11:09:23.203947+0000","last_peered":"2023-10-01T11:09:23.203947+0000","last_clean":"2023-10-01T11:09:23.203947+0000","last_became_active":"2023-10-01T11:06:11.789452+0000","last_became_peered":"2023-10-01T11:06:11.789452+0000","last_unstale":"2023-10-01T11:09:23.203947+0000","last_undegraded":"2023-10-01T11:09:23.203947+0000","last_fullsized":"2023-10-01T11:09:23.203947+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:18:02.050694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":351,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:25.760779+0000","last_change":"2023-10-01T11:06:11.784794+0000","last_active":"2023-10-01T11:09:25.760779+0000","last_peered":"2023-10-01T11:09:25.760779+0000","last_clean":"2023-10-01T11:09:25.760779+0000","last_became_active":"2023-10-01T11:06:11.784368+0000","last_became_peered":"2023-10-01T11:06:11.784368+0000","last_unstale":"2023-10-01T11:09:25.760779+0000","last_undegraded":"2023-10-01T11:09:25.760779+0000","last_fullsized":"2023-10-01T11:09:25.760779+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:06:52.307449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":379,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:22.157310+0000","last_change":"2023-10-01T11:06:11.794671+0000","last_active":"2023-10-01T11:09:22.157310+0000","last_peered":"2023-10-01T11:09:22.157310+0000","last_clean":"2023-10-01T11:09:22.157310+0000","last_became_active":"2023-10-01T11:06:11.794121+0000","last_became_peered":"2023-10-01T11:06:11.794121+0000","last_unstale":"2023-10-01T11:09:22.157310+0000","last_undegraded":"2023-10-01T11:09:22.157310+0000","last_fullsized":"2023-10-01T11:09:22.157310+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":368,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:22.150307+0000","last_change":"2023-10-01T11:06:11.796259+0000","last_active":"2023-10-01T11:09:22.150307+0000","last_peered":"2023-10-01T11:09:22.150307+0000","last_clean":"2023-10-01T11:09:22.150307+0000","last_became_active":"2023-10-01T11:06:11.795870+0000","last_became_peered":"2023-10-01T11:06:11.795870+0000","last_unstale":"2023-10-01T11:09:22.150307+0000","last_undegraded":"2023-10-01T11:09:22.150307+0000","last_fullsized":"2023-10-01T11:09:22.150307+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:49:21.072163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":351,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:30.761559+0000","last_change":"2023-10-01T11:06:11.785871+0000","last_active":"2023-10-01T11:09:30.761559+0000","last_peered":"2023-10-01T11:09:30.761559+0000","last_clean":"2023-10-01T11:09:30.761559+0000","last_became_active":"2023-10-01T11:06:11.785183+0000","last_became_peered":"2023-10-01T11:06:11.785183+0000","last_unstale":"2023-10-01T11:09:30.761559+0000","last_undegraded":"2023-10-01T11:09:30.761559+0000","last_fullsized":"2023-10-01T11:09:30.761559+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:07:55.556404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":55,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:23.207523+0000","last_change":"2023-10-01T11:05:33.900352+0000","last_active":"2023-10-01T11:09:23.207523+0000","last_peered":"2023-10-01T11:09:23.207523+0000","last_clean":"2023-10-01T11:09:23.207523+0000","last_became_active":"2023-10-01T11:05:33.788906+0000","last_became_peered":"2023-10-01T11:05:33.788906+0000","last_unstale":"2023-10-01T11:09:23.207523+0000","last_undegraded":"2023-10-01T11:09:23.207523+0000","last_fullsized":"2023-10-01T11:09:23.207523+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":367,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:23.204217+0000","last_change":"2023-10-01T11:06:11.798727+0000","last_active":"2023-10-01T11:09:23.204217+0000","last_peered":"2023-10-01T11:09:23.204217+0000","last_clean":"2023-10-01T11:09:23.204217+0000","last_became_active":"2023-10-01T11:06:11.798388+0000","last_became_peered":"2023-10-01T11:06:11.798388+0000","last_unstale":"2023-10-01T11:09:23.204217+0000","last_undegraded":"2023-10-01T11:09:23.204217+0000","last_fullsized":"2023-10-01T11:09:23.204217+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:10:21.576627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.2","version":"359'2588","reported_seq":2600,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-01T11:10:04.680946+0000","last_change":"2023-10-01T11:09:21.161135+0000","last_active":"2023-10-01T11:10:04.680946+0000","last_peered":"2023-10-01T11:10:04.680946+0000","last_clean":"2023-10-01T11:10:04.680946+0000","last_became_active":"2023-10-01T11:09:21.160719+0000","last_became_peered":"2023-10-01T11:09:21.160719+0000","last_unstale":"2023-10-01T11:10:04.680946+0000","last_undegraded":"2023-10-01T11:10:04.680946+0000","last_fullsized":"2023-10-01T11:10:04.680946+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":356,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_clean_scrub_stamp":"2023-10-01T11:09:20.132185+0000","objects_scrubbed":0,"log_size":2588,"log_dups_size":0,"ondisk_log_size":2588,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:53:24.785514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169607168,"num_objects":2588,"num_object_clones":0,"num_object_copies":7764,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2588,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2588,"num_write_kb":165632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":256,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:22.151832+0000","last_change":"2023-10-01T11:06:11.840628+0000","last_active":"2023-10-01T11:09:22.151832+0000","last_peered":"2023-10-01T11:09:22.151832+0000","last_clean":"2023-10-01T11:09:22.151832+0000","last_became_active":"2023-10-01T11:06:11.800187+0000","last_became_peered":"2023-10-01T11:06:11.800187+0000","last_unstale":"2023-10-01T11:09:22.151832+0000","last_undegraded":"2023-10-01T11:09:22.151832+0000","last_fullsized":"2023-10-01T11:09:22.151832+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:24:48.900425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":400,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:22.156776+0000","last_change":"2023-10-01T11:06:11.830255+0000","last_active":"2023-10-01T11:09:22.156776+0000","last_peered":"2023-10-01T11:09:22.156776+0000","last_clean":"2023-10-01T11:09:22.156776+0000","last_became_active":"2023-10-01T11:06:11.829847+0000","last_became_peered":"2023-10-01T11:06:11.829847+0000","last_unstale":"2023-10-01T11:09:22.156776+0000","last_undegraded":"2023-10-01T11:09:22.156776+0000","last_fullsized":"2023-10-01T11:09:22.156776+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.3","version":"359'2755","reported_seq":2767,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-01T11:10:05.767641+0000","last_change":"2023-10-01T11:09:22.023134+0000","last_active":"2023-10-01T11:10:05.767641+0000","last_peered":"2023-10-01T11:10:05.767641+0000","last_clean":"2023-10-01T11:10:05.767641+0000","last_became_active":"2023-10-01T11:09:22.022727+0000","last_became_peered":"2023-10-01T11:09:22.022727+0000","last_unstale":"2023-10-01T11:10:05.767641+0000","last_undegraded":"2023-10-01T11:10:05.767641+0000","last_fullsized":"2023-10-01T11:10:05.767641+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":356,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_clean_scrub_stamp":"2023-10-01T11:09:20.132185+0000","objects_scrubbed":0,"log_size":2755,"log_dups_size":0,"ondisk_log_size":2755,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:24:57.467745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180551680,"num_objects":2755,"num_object_clones":0,"num_object_copies":8265,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2755,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2755,"num_write_kb":176320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":215,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:22.151675+0000","last_change":"2023-10-01T11:06:11.813139+0000","last_active":"2023-10-01T11:09:22.151675+0000","last_peered":"2023-10-01T11:09:22.151675+0000","last_clean":"2023-10-01T11:09:22.151675+0000","last_became_active":"2023-10-01T11:06:11.790124+0000","last_became_peered":"2023-10-01T11:06:11.790124+0000","last_unstale":"2023-10-01T11:09:22.151675+0000","last_undegraded":"2023-10-01T11:09:22.151675+0000","last_fullsized":"2023-10-01T11:09:22.151675+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:30:46.077244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":344,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:22.150607+0000","last_change":"2023-10-01T11:04:02.724840+0000","last_active":"2023-10-01T11:09:22.150607+0000","last_peered":"2023-10-01T11:09:22.150607+0000","last_clean":"2023-10-01T11:09:22.150607+0000","last_became_active":"2023-10-01T11:04:02.724476+0000","last_became_peered":"2023-10-01T11:04:02.724476+0000","last_unstale":"2023-10-01T11:09:22.150607+0000","last_undegraded":"2023-10-01T11:09:22.150607+0000","last_fullsized":"2023-10-01T11:09:22.150607+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:05:08.882495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.0","version":"359'2508","reported_seq":2519,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-01T11:10:03.473066+0000","last_change":"2023-10-01T11:09:22.023014+0000","last_active":"2023-10-01T11:10:03.473066+0000","last_peered":"2023-10-01T11:10:03.473066+0000","last_clean":"2023-10-01T11:10:03.473066+0000","last_became_active":"2023-10-01T11:09:22.022611+0000","last_became_peered":"2023-10-01T11:09:22.022611+0000","last_unstale":"2023-10-01T11:10:03.473066+0000","last_undegraded":"2023-10-01T11:10:03.473066+0000","last_fullsized":"2023-10-01T11:10:03.473066+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":356,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_clean_scrub_stamp":"2023-10-01T11:09:20.132185+0000","objects_scrubbed":0,"log_size":2508,"log_dups_size":0,"ondisk_log_size":2508,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:23:59.916230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164364288,"num_objects":2508,"num_object_clones":0,"num_object_copies":7524,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2508,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2508,"num_write_kb":160512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":328,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:22.157380+0000","last_change":"2023-10-01T11:06:11.820664+0000","last_active":"2023-10-01T11:09:22.157380+0000","last_peered":"2023-10-01T11:09:22.157380+0000","last_clean":"2023-10-01T11:09:22.157380+0000","last_became_active":"2023-10-01T11:06:11.792931+0000","last_became_peered":"2023-10-01T11:06:11.792931+0000","last_unstale":"2023-10-01T11:09:22.157380+0000","last_undegraded":"2023-10-01T11:09:22.157380+0000","last_fullsized":"2023-10-01T11:09:22.157380+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":700,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":54,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:23.208371+0000","last_change":"2023-10-01T11:06:11.795096+0000","last_active":"2023-10-01T11:09:23.208371+0000","last_peered":"2023-10-01T11:09:23.208371+0000","last_clean":"2023-10-01T11:09:23.208371+0000","last_became_active":"2023-10-01T11:06:11.794716+0000","last_became_peered":"2023-10-01T11:06:11.794716+0000","last_unstale":"2023-10-01T11:09:23.208371+0000","last_undegraded":"2023-10-01T11:09:23.208371+0000","last_fullsized":"2023-10-01T11:09:23.208371+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1","version":"359'2696","reported_seq":2708,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-01T11:10:05.761179+0000","last_change":"2023-10-01T11:09:21.156985+0000","last_active":"2023-10-01T11:10:05.761179+0000","last_peered":"2023-10-01T11:10:05.761179+0000","last_clean":"2023-10-01T11:10:05.761179+0000","last_became_active":"2023-10-01T11:09:21.156452+0000","last_became_peered":"2023-10-01T11:09:21.156452+0000","last_unstale":"2023-10-01T11:10:05.761179+0000","last_undegraded":"2023-10-01T11:10:05.761179+0000","last_fullsized":"2023-10-01T11:10:05.761179+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":356,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_clean_scrub_stamp":"2023-10-01T11:09:20.132185+0000","objects_scrubbed":0,"log_size":2696,"log_dups_size":0,"ondisk_log_size":2696,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:15:04.145939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":176685056,"num_objects":2696,"num_object_clones":0,"num_object_copies":8088,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2696,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2696,"num_write_kb":172544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":380,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:22.157192+0000","last_change":"2023-10-01T11:05:33.788917+0000","last_active":"2023-10-01T11:09:22.157192+0000","last_peered":"2023-10-01T11:09:22.157192+0000","last_clean":"2023-10-01T11:09:22.157192+0000","last_became_active":"2023-10-01T11:05:33.788078+0000","last_became_peered":"2023-10-01T11:05:33.788078+0000","last_unstale":"2023-10-01T11:09:22.157192+0000","last_undegraded":"2023-10-01T11:09:22.157192+0000","last_fullsized":"2023-10-01T11:09:22.157192+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":108,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:22.261413+0000","last_change":"2023-10-01T11:08:46.873065+0000","last_active":"2023-10-01T11:09:22.261413+0000","last_peered":"2023-10-01T11:09:22.261413+0000","last_clean":"2023-10-01T11:09:22.261413+0000","last_became_active":"2023-10-01T11:08:46.872692+0000","last_became_peered":"2023-10-01T11:08:46.872692+0000","last_unstale":"2023-10-01T11:09:22.261413+0000","last_undegraded":"2023-10-01T11:09:22.261413+0000","last_fullsized":"2023-10-01T11:09:22.261413+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:27:32.806247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.6","version":"359'2564","reported_seq":2576,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-01T11:10:03.471600+0000","last_change":"2023-10-01T11:09:22.025630+0000","last_active":"2023-10-01T11:10:03.471600+0000","last_peered":"2023-10-01T11:10:03.471600+0000","last_clean":"2023-10-01T11:10:03.471600+0000","last_became_active":"2023-10-01T11:09:22.025164+0000","last_became_peered":"2023-10-01T11:09:22.025164+0000","last_unstale":"2023-10-01T11:10:03.471600+0000","last_undegraded":"2023-10-01T11:10:03.471600+0000","last_fullsized":"2023-10-01T11:10:03.471600+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":356,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_clean_scrub_stamp":"2023-10-01T11:09:20.132185+0000","objects_scrubbed":0,"log_size":2564,"log_dups_size":0,"ondisk_log_size":2564,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:05:50.808903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":168034304,"num_objects":2564,"num_object_clones":0,"num_object_copies":7692,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2564,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2564,"num_write_kb":164096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":335,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:23.204459+0000","last_change":"2023-10-01T10:38:16.022493+0000","last_active":"2023-10-01T11:09:23.204459+0000","last_peered":"2023-10-01T11:09:23.204459+0000","last_clean":"2023-10-01T11:09:23.204459+0000","last_became_active":"2023-10-01T10:38:16.022033+0000","last_became_peered":"2023-10-01T10:38:16.022033+0000","last_unstale":"2023-10-01T11:09:23.204459+0000","last_undegraded":"2023-10-01T11:09:23.204459+0000","last_fullsized":"2023-10-01T11:09:23.204459+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:49:05.262763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":374,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:22.151470+0000","last_change":"2023-10-01T11:05:33.785133+0000","last_active":"2023-10-01T11:09:22.151470+0000","last_peered":"2023-10-01T11:09:22.151470+0000","last_clean":"2023-10-01T11:09:22.151470+0000","last_became_active":"2023-10-01T11:05:33.784749+0000","last_became_peered":"2023-10-01T11:05:33.784749+0000","last_unstale":"2023-10-01T11:09:22.151470+0000","last_undegraded":"2023-10-01T11:09:22.151470+0000","last_fullsized":"2023-10-01T11:09:22.151470+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:08:01.664432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.7","version":"359'2606","reported_seq":2617,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-01T11:10:03.815898+0000","last_change":"2023-10-01T11:09:22.022235+0000","last_active":"2023-10-01T11:10:03.815898+0000","last_peered":"2023-10-01T11:10:03.815898+0000","last_clean":"2023-10-01T11:10:03.815898+0000","last_became_active":"2023-10-01T11:09:22.021331+0000","last_became_peered":"2023-10-01T11:09:22.021331+0000","last_unstale":"2023-10-01T11:10:03.815898+0000","last_undegraded":"2023-10-01T11:10:03.815898+0000","last_fullsized":"2023-10-01T11:10:03.815898+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":356,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_clean_scrub_stamp":"2023-10-01T11:09:20.132185+0000","objects_scrubbed":0,"log_size":2606,"log_dups_size":0,"ondisk_log_size":2606,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":170786816,"num_objects":2606,"num_object_clones":0,"num_object_copies":7818,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2606,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2606,"num_write_kb":166784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":347,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:22.150528+0000","last_change":"2023-10-01T11:05:33.782142+0000","last_active":"2023-10-01T11:09:22.150528+0000","last_peered":"2023-10-01T11:09:22.150528+0000","last_clean":"2023-10-01T11:09:22.150528+0000","last_became_active":"2023-10-01T11:05:33.781725+0000","last_became_peered":"2023-10-01T11:05:33.781725+0000","last_unstale":"2023-10-01T11:09:22.150528+0000","last_undegraded":"2023-10-01T11:09:22.150528+0000","last_fullsized":"2023-10-01T11:09:22.150528+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:58:37.958611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":334,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:22.152050+0000","last_change":"2023-10-01T11:04:02.727282+0000","last_active":"2023-10-01T11:09:22.152050+0000","last_peered":"2023-10-01T11:09:22.152050+0000","last_clean":"2023-10-01T11:09:22.152050+0000","last_became_active":"2023-10-01T11:04:02.726818+0000","last_became_peered":"2023-10-01T11:04:02.726818+0000","last_unstale":"2023-10-01T11:09:22.152050+0000","last_undegraded":"2023-10-01T11:09:22.152050+0000","last_fullsized":"2023-10-01T11:09:22.152050+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:11:11.868291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.4","version":"359'2473","reported_seq":2484,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-01T11:10:03.425822+0000","last_change":"2023-10-01T11:09:21.163719+0000","last_active":"2023-10-01T11:10:03.425822+0000","last_peered":"2023-10-01T11:10:03.425822+0000","last_clean":"2023-10-01T11:10:03.425822+0000","last_became_active":"2023-10-01T11:09:21.163294+0000","last_became_peered":"2023-10-01T11:09:21.163294+0000","last_unstale":"2023-10-01T11:10:03.425822+0000","last_undegraded":"2023-10-01T11:10:03.425822+0000","last_fullsized":"2023-10-01T11:10:03.425822+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":356,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_clean_scrub_stamp":"2023-10-01T11:09:20.132185+0000","objects_scrubbed":0,"log_size":2473,"log_dups_size":0,"ondisk_log_size":2473,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162070528,"num_objects":2473,"num_object_clones":0,"num_object_copies":7419,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2473,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2473,"num_write_kb":158272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":378,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:22.150985+0000","last_change":"2023-10-01T11:06:11.790944+0000","last_active":"2023-10-01T11:09:22.150985+0000","last_peered":"2023-10-01T11:09:22.150985+0000","last_clean":"2023-10-01T11:09:22.150985+0000","last_became_active":"2023-10-01T11:06:11.790151+0000","last_became_peered":"2023-10-01T11:06:11.790151+0000","last_unstale":"2023-10-01T11:09:22.150985+0000","last_undegraded":"2023-10-01T11:09:22.150985+0000","last_fullsized":"2023-10-01T11:09:22.150985+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:05:49.161868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":376,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:22.151861+0000","last_change":"2023-10-01T11:05:33.781934+0000","last_active":"2023-10-01T11:09:22.151861+0000","last_peered":"2023-10-01T11:09:22.151861+0000","last_clean":"2023-10-01T11:09:22.151861+0000","last_became_active":"2023-10-01T11:05:33.781555+0000","last_became_peered":"2023-10-01T11:05:33.781555+0000","last_unstale":"2023-10-01T11:09:22.151861+0000","last_undegraded":"2023-10-01T11:09:22.151861+0000","last_fullsized":"2023-10-01T11:09:22.151861+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:59:22.019053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.5","version":"359'2624","reported_seq":2636,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-01T11:10:04.137130+0000","last_change":"2023-10-01T11:09:22.023719+0000","last_active":"2023-10-01T11:10:04.137130+0000","last_peered":"2023-10-01T11:10:04.137130+0000","last_clean":"2023-10-01T11:10:04.137130+0000","last_became_active":"2023-10-01T11:09:22.023016+0000","last_became_peered":"2023-10-01T11:09:22.023016+0000","last_unstale":"2023-10-01T11:10:04.137130+0000","last_undegraded":"2023-10-01T11:10:04.137130+0000","last_fullsized":"2023-10-01T11:10:04.137130+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":356,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_clean_scrub_stamp":"2023-10-01T11:09:20.132185+0000","objects_scrubbed":0,"log_size":2624,"log_dups_size":0,"ondisk_log_size":2624,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:55:17.819499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":171966464,"num_objects":2624,"num_object_clones":0,"num_object_copies":7872,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2624,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2624,"num_write_kb":167936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":369,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:23.203474+0000","last_change":"2023-10-01T11:06:11.784722+0000","last_active":"2023-10-01T11:09:23.203474+0000","last_peered":"2023-10-01T11:09:23.203474+0000","last_clean":"2023-10-01T11:09:23.203474+0000","last_became_active":"2023-10-01T11:06:11.784348+0000","last_became_peered":"2023-10-01T11:06:11.784348+0000","last_unstale":"2023-10-01T11:09:23.203474+0000","last_undegraded":"2023-10-01T11:09:23.203474+0000","last_fullsized":"2023-10-01T11:09:23.203474+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:40:29.283179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":351,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:22.152644+0000","last_change":"2023-10-01T10:38:16.019755+0000","last_active":"2023-10-01T11:09:22.152644+0000","last_peered":"2023-10-01T11:09:22.152644+0000","last_clean":"2023-10-01T11:09:22.152644+0000","last_became_active":"2023-10-01T10:38:16.019397+0000","last_became_peered":"2023-10-01T10:38:16.019397+0000","last_unstale":"2023-10-01T11:09:22.152644+0000","last_undegraded":"2023-10-01T11:09:22.152644+0000","last_fullsized":"2023-10-01T11:09:22.152644+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:42:56.623225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.8","version":"359'2598","reported_seq":2610,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-01T11:10:04.672921+0000","last_change":"2023-10-01T11:09:22.022986+0000","last_active":"2023-10-01T11:10:04.672921+0000","last_peered":"2023-10-01T11:10:04.672921+0000","last_clean":"2023-10-01T11:10:04.672921+0000","last_became_active":"2023-10-01T11:09:22.022553+0000","last_became_peered":"2023-10-01T11:09:22.022553+0000","last_unstale":"2023-10-01T11:10:04.672921+0000","last_undegraded":"2023-10-01T11:10:04.672921+0000","last_fullsized":"2023-10-01T11:10:04.672921+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":356,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_clean_scrub_stamp":"2023-10-01T11:09:20.132185+0000","objects_scrubbed":0,"log_size":2598,"log_dups_size":0,"ondisk_log_size":2598,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T22:14:03.287229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":170262528,"num_objects":2598,"num_object_clones":0,"num_object_copies":7794,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2598,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2598,"num_write_kb":166272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":261,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:22.261660+0000","last_change":"2023-10-01T11:06:11.788951+0000","last_active":"2023-10-01T11:09:22.261660+0000","last_peered":"2023-10-01T11:09:22.261660+0000","last_clean":"2023-10-01T11:09:22.261660+0000","last_became_active":"2023-10-01T11:06:11.788617+0000","last_became_peered":"2023-10-01T11:06:11.788617+0000","last_unstale":"2023-10-01T11:09:22.261660+0000","last_undegraded":"2023-10-01T11:09:22.261660+0000","last_fullsized":"2023-10-01T11:09:22.261660+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:07:46.629250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":341,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:22.157280+0000","last_change":"2023-10-01T10:37:42.766814+0000","last_active":"2023-10-01T11:09:22.157280+0000","last_peered":"2023-10-01T11:09:22.157280+0000","last_clean":"2023-10-01T11:09:22.157280+0000","last_became_active":"2023-10-01T10:37:42.766451+0000","last_became_peered":"2023-10-01T10:37:42.766451+0000","last_unstale":"2023-10-01T11:09:22.157280+0000","last_undegraded":"2023-10-01T11:09:22.157280+0000","last_fullsized":"2023-10-01T11:09:22.157280+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":141,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:22.261502+0000","last_change":"2023-10-01T11:08:46.870961+0000","last_active":"2023-10-01T11:09:22.261502+0000","last_peered":"2023-10-01T11:09:22.261502+0000","last_clean":"2023-10-01T11:09:22.261502+0000","last_became_active":"2023-10-01T11:08:46.870680+0000","last_became_peered":"2023-10-01T11:08:46.870680+0000","last_unstale":"2023-10-01T11:09:22.261502+0000","last_undegraded":"2023-10-01T11:09:22.261502+0000","last_fullsized":"2023-10-01T11:09:22.261502+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:36:45.187763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.e","version":"359'2489","reported_seq":2501,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-01T11:10:04.122407+0000","last_change":"2023-10-01T11:09:21.163877+0000","last_active":"2023-10-01T11:10:04.122407+0000","last_peered":"2023-10-01T11:10:04.122407+0000","last_clean":"2023-10-01T11:10:04.122407+0000","last_became_active":"2023-10-01T11:09:21.163510+0000","last_became_peered":"2023-10-01T11:09:21.163510+0000","last_unstale":"2023-10-01T11:10:04.122407+0000","last_undegraded":"2023-10-01T11:10:04.122407+0000","last_fullsized":"2023-10-01T11:10:04.122407+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":356,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_clean_scrub_stamp":"2023-10-01T11:09:20.132185+0000","objects_scrubbed":0,"log_size":2489,"log_dups_size":0,"ondisk_log_size":2489,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:49:26.529322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163119104,"num_objects":2489,"num_object_clones":0,"num_object_copies":7467,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2489,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2489,"num_write_kb":159296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":384,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:22.156944+0000","last_change":"2023-10-01T11:05:33.792877+0000","last_active":"2023-10-01T11:09:22.156944+0000","last_peered":"2023-10-01T11:09:22.156944+0000","last_clean":"2023-10-01T11:09:22.156944+0000","last_became_active":"2023-10-01T11:05:33.792054+0000","last_became_peered":"2023-10-01T11:05:33.792054+0000","last_unstale":"2023-10-01T11:09:22.156944+0000","last_undegraded":"2023-10-01T11:09:22.156944+0000","last_fullsized":"2023-10-01T11:09:22.156944+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":342,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:22.151785+0000","last_change":"2023-10-01T11:05:33.783100+0000","last_active":"2023-10-01T11:09:22.151785+0000","last_peered":"2023-10-01T11:09:22.151785+0000","last_clean":"2023-10-01T11:09:22.151785+0000","last_became_active":"2023-10-01T11:05:33.782557+0000","last_became_peered":"2023-10-01T11:05:33.782557+0000","last_unstale":"2023-10-01T11:09:22.151785+0000","last_undegraded":"2023-10-01T11:09:22.151785+0000","last_fullsized":"2023-10-01T11:09:22.151785+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:52:05.581697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":262,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:22.152190+0000","last_change":"2023-10-01T10:52:18.963437+0000","last_active":"2023-10-01T11:09:22.152190+0000","last_peered":"2023-10-01T11:09:22.152190+0000","last_clean":"2023-10-01T11:09:22.152190+0000","last_became_active":"2023-10-01T10:52:18.955165+0000","last_became_peered":"2023-10-01T10:52:18.955165+0000","last_unstale":"2023-10-01T11:09:22.152190+0000","last_undegraded":"2023-10-01T11:09:22.152190+0000","last_fullsized":"2023-10-01T11:09:22.152190+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:57:49.129100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.d","version":"359'2527","reported_seq":2538,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-01T11:10:03.439073+0000","last_change":"2023-10-01T11:09:21.165115+0000","last_active":"2023-10-01T11:10:03.439073+0000","last_peered":"2023-10-01T11:10:03.439073+0000","last_clean":"2023-10-01T11:10:03.439073+0000","last_became_active":"2023-10-01T11:09:21.164631+0000","last_became_peered":"2023-10-01T11:09:21.164631+0000","last_unstale":"2023-10-01T11:10:03.439073+0000","last_undegraded":"2023-10-01T11:10:03.439073+0000","last_fullsized":"2023-10-01T11:10:03.439073+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":356,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_clean_scrub_stamp":"2023-10-01T11:09:20.132185+0000","objects_scrubbed":0,"log_size":2527,"log_dups_size":0,"ondisk_log_size":2527,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165609472,"num_objects":2527,"num_object_clones":0,"num_object_copies":7581,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2527,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2527,"num_write_kb":161728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":343,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:22.151860+0000","last_change":"2023-10-01T10:37:42.762524+0000","last_active":"2023-10-01T11:09:22.151860+0000","last_peered":"2023-10-01T11:09:22.151860+0000","last_clean":"2023-10-01T11:09:22.151860+0000","last_became_active":"2023-10-01T10:37:42.762209+0000","last_became_peered":"2023-10-01T10:37:42.762209+0000","last_unstale":"2023-10-01T11:09:22.151860+0000","last_undegraded":"2023-10-01T11:09:22.151860+0000","last_fullsized":"2023-10-01T11:09:22.151860+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:45:38.171397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":296,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:22.151757+0000","last_change":"2023-10-01T11:05:33.799413+0000","last_active":"2023-10-01T11:09:22.151757+0000","last_peered":"2023-10-01T11:09:22.151757+0000","last_clean":"2023-10-01T11:09:22.151757+0000","last_became_active":"2023-10-01T11:05:33.799158+0000","last_became_peered":"2023-10-01T11:05:33.799158+0000","last_unstale":"2023-10-01T11:09:22.151757+0000","last_undegraded":"2023-10-01T11:09:22.151757+0000","last_fullsized":"2023-10-01T11:09:22.151757+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:39:05.963181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":170,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:22.261825+0000","last_change":"2023-10-01T11:08:46.870350+0000","last_active":"2023-10-01T11:09:22.261825+0000","last_peered":"2023-10-01T11:09:22.261825+0000","last_clean":"2023-10-01T11:09:22.261825+0000","last_became_active":"2023-10-01T11:08:46.869990+0000","last_became_peered":"2023-10-01T11:08:46.869990+0000","last_unstale":"2023-10-01T11:09:22.261825+0000","last_undegraded":"2023-10-01T11:09:22.261825+0000","last_fullsized":"2023-10-01T11:09:22.261825+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:32:25.096683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.f","version":"359'2651","reported_seq":2663,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-01T11:10:05.684627+0000","last_change":"2023-10-01T11:09:21.156775+0000","last_active":"2023-10-01T11:10:05.684627+0000","last_peered":"2023-10-01T11:10:05.684627+0000","last_clean":"2023-10-01T11:10:05.684627+0000","last_became_active":"2023-10-01T11:09:21.156403+0000","last_became_peered":"2023-10-01T11:09:21.156403+0000","last_unstale":"2023-10-01T11:10:05.684627+0000","last_undegraded":"2023-10-01T11:10:05.684627+0000","last_fullsized":"2023-10-01T11:10:05.684627+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":356,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_clean_scrub_stamp":"2023-10-01T11:09:20.132185+0000","objects_scrubbed":0,"log_size":2651,"log_dups_size":0,"ondisk_log_size":2651,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:18:58.903554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":173735936,"num_objects":2651,"num_object_clones":0,"num_object_copies":7953,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2651,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2651,"num_write_kb":169664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":366,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:22.156808+0000","last_change":"2023-10-01T11:06:11.791607+0000","last_active":"2023-10-01T11:09:22.156808+0000","last_peered":"2023-10-01T11:09:22.156808+0000","last_clean":"2023-10-01T11:09:22.156808+0000","last_became_active":"2023-10-01T11:06:11.789667+0000","last_became_peered":"2023-10-01T11:06:11.789667+0000","last_unstale":"2023-10-01T11:09:22.156808+0000","last_undegraded":"2023-10-01T11:09:22.156808+0000","last_fullsized":"2023-10-01T11:09:22.156808+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'139","reported_seq":640,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:23.207999+0000","last_change":"2023-10-01T11:08:15.927727+0000","last_active":"2023-10-01T11:09:23.207999+0000","last_peered":"2023-10-01T11:09:23.207999+0000","last_clean":"2023-10-01T11:09:23.207999+0000","last_became_active":"2023-10-01T11:08:15.927235+0000","last_became_peered":"2023-10-01T11:08:15.927235+0000","last_unstale":"2023-10-01T11:09:23.207999+0000","last_undegraded":"2023-10-01T11:09:23.207999+0000","last_fullsized":"2023-10-01T11:09:23.207999+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:09:17.221082+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:09:17.221082+0000","last_clean_scrub_stamp":"2023-10-01T10:09:17.221082+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":5313312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":370,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:23.203757+0000","last_change":"2023-10-01T11:05:33.788093+0000","last_active":"2023-10-01T11:09:23.203757+0000","last_peered":"2023-10-01T11:09:23.203757+0000","last_clean":"2023-10-01T11:09:23.203757+0000","last_became_active":"2023-10-01T11:05:33.787723+0000","last_became_peered":"2023-10-01T11:05:33.787723+0000","last_unstale":"2023-10-01T11:09:23.203757+0000","last_undegraded":"2023-10-01T11:09:23.203757+0000","last_fullsized":"2023-10-01T11:09:23.203757+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:04:55.138186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":392,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:22.151741+0000","last_change":"2023-10-01T11:06:11.788535+0000","last_active":"2023-10-01T11:09:22.151741+0000","last_peered":"2023-10-01T11:09:22.151741+0000","last_clean":"2023-10-01T11:09:22.151741+0000","last_became_active":"2023-10-01T11:06:11.788103+0000","last_became_peered":"2023-10-01T11:06:11.788103+0000","last_unstale":"2023-10-01T11:09:22.151741+0000","last_undegraded":"2023-10-01T11:09:22.151741+0000","last_fullsized":"2023-10-01T11:09:22.151741+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:02:49.482834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.c","version":"359'2580","reported_seq":2592,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-01T11:10:04.108154+0000","last_change":"2023-10-01T11:09:22.022461+0000","last_active":"2023-10-01T11:10:04.108154+0000","last_peered":"2023-10-01T11:10:04.108154+0000","last_clean":"2023-10-01T11:10:04.108154+0000","last_became_active":"2023-10-01T11:09:22.021657+0000","last_became_peered":"2023-10-01T11:09:22.021657+0000","last_unstale":"2023-10-01T11:10:04.108154+0000","last_undegraded":"2023-10-01T11:10:04.108154+0000","last_fullsized":"2023-10-01T11:10:04.108154+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":356,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_clean_scrub_stamp":"2023-10-01T11:09:20.132185+0000","objects_scrubbed":0,"log_size":2580,"log_dups_size":0,"ondisk_log_size":2580,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:13:30.660488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169082880,"num_objects":2580,"num_object_clones":0,"num_object_copies":7740,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2580,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2580,"num_write_kb":165120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":254,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:22.260765+0000","last_change":"2023-10-01T11:08:46.989647+0000","last_active":"2023-10-01T11:09:22.260765+0000","last_peered":"2023-10-01T11:09:22.260765+0000","last_clean":"2023-10-01T11:09:22.260765+0000","last_became_active":"2023-10-01T11:08:46.988979+0000","last_became_peered":"2023-10-01T11:08:46.988979+0000","last_unstale":"2023-10-01T11:09:22.260765+0000","last_undegraded":"2023-10-01T11:09:22.260765+0000","last_fullsized":"2023-10-01T11:09:22.260765+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:10:06.594141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":142,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:22.150859+0000","last_change":"2023-10-01T11:04:40.868156+0000","last_active":"2023-10-01T11:09:22.150859+0000","last_peered":"2023-10-01T11:09:22.150859+0000","last_clean":"2023-10-01T11:09:22.150859+0000","last_became_active":"2023-10-01T11:04:40.867864+0000","last_became_peered":"2023-10-01T11:04:40.867864+0000","last_unstale":"2023-10-01T11:09:22.150859+0000","last_undegraded":"2023-10-01T11:09:22.150859+0000","last_fullsized":"2023-10-01T11:09:22.150859+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:37:15.571777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":133,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:22.152437+0000","last_change":"2023-10-01T11:04:40.880090+0000","last_active":"2023-10-01T11:09:22.152437+0000","last_peered":"2023-10-01T11:09:22.152437+0000","last_clean":"2023-10-01T11:09:22.152437+0000","last_became_active":"2023-10-01T11:04:40.879768+0000","last_became_peered":"2023-10-01T11:04:40.879768+0000","last_unstale":"2023-10-01T11:09:22.152437+0000","last_undegraded":"2023-10-01T11:09:22.152437+0000","last_fullsized":"2023-10-01T11:09:22.152437+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:25:31.189338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.9","version":"359'2548","reported_seq":2559,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-01T11:10:03.808796+0000","last_change":"2023-10-01T11:09:21.165167+0000","last_active":"2023-10-01T11:10:03.808796+0000","last_peered":"2023-10-01T11:10:03.808796+0000","last_clean":"2023-10-01T11:10:03.808796+0000","last_became_active":"2023-10-01T11:09:21.164717+0000","last_became_peered":"2023-10-01T11:09:21.164717+0000","last_unstale":"2023-10-01T11:10:03.808796+0000","last_undegraded":"2023-10-01T11:10:03.808796+0000","last_fullsized":"2023-10-01T11:10:03.808796+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":356,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_clean_scrub_stamp":"2023-10-01T11:09:20.132185+0000","objects_scrubbed":0,"log_size":2548,"log_dups_size":0,"ondisk_log_size":2548,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166985728,"num_objects":2548,"num_object_clones":0,"num_object_copies":7644,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2548,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2548,"num_write_kb":163072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":185,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:22.151407+0000","last_change":"2023-10-01T11:06:55.803529+0000","last_active":"2023-10-01T11:09:22.151407+0000","last_peered":"2023-10-01T11:09:22.151407+0000","last_clean":"2023-10-01T11:09:22.151407+0000","last_became_active":"2023-10-01T11:06:55.803266+0000","last_became_peered":"2023-10-01T11:06:55.803266+0000","last_unstale":"2023-10-01T11:09:22.151407+0000","last_undegraded":"2023-10-01T11:09:22.151407+0000","last_fullsized":"2023-10-01T11:09:22.151407+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:57:06.218433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":242,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:22.152514+0000","last_change":"2023-10-01T10:52:18.964479+0000","last_active":"2023-10-01T11:09:22.152514+0000","last_peered":"2023-10-01T11:09:22.152514+0000","last_clean":"2023-10-01T11:09:22.152514+0000","last_became_active":"2023-10-01T10:52:18.963881+0000","last_became_peered":"2023-10-01T10:52:18.963881+0000","last_unstale":"2023-10-01T11:09:22.152514+0000","last_undegraded":"2023-10-01T11:09:22.152514+0000","last_fullsized":"2023-10-01T11:09:22.152514+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:36:30.405755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":336,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:22.156904+0000","last_change":"2023-10-01T11:06:11.791253+0000","last_active":"2023-10-01T11:09:22.156904+0000","last_peered":"2023-10-01T11:09:22.156904+0000","last_clean":"2023-10-01T11:09:22.156904+0000","last_became_active":"2023-10-01T11:06:11.789517+0000","last_became_peered":"2023-10-01T11:06:11.789517+0000","last_unstale":"2023-10-01T11:09:22.156904+0000","last_undegraded":"2023-10-01T11:09:22.156904+0000","last_fullsized":"2023-10-01T11:09:22.156904+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.a","version":"359'2605","reported_seq":2615,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-01T11:10:03.770722+0000","last_change":"2023-10-01T11:09:21.162350+0000","last_active":"2023-10-01T11:10:03.770722+0000","last_peered":"2023-10-01T11:10:03.770722+0000","last_clean":"2023-10-01T11:10:03.770722+0000","last_became_active":"2023-10-01T11:09:21.162009+0000","last_became_peered":"2023-10-01T11:09:21.162009+0000","last_unstale":"2023-10-01T11:10:03.770722+0000","last_undegraded":"2023-10-01T11:10:03.770722+0000","last_fullsized":"2023-10-01T11:10:03.770722+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":356,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_clean_scrub_stamp":"2023-10-01T11:09:20.132185+0000","objects_scrubbed":0,"log_size":2605,"log_dups_size":0,"ondisk_log_size":2605,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":170721280,"num_objects":2605,"num_object_clones":0,"num_object_copies":7815,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2605,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2605,"num_write_kb":166720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":372,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:23.204428+0000","last_change":"2023-10-01T11:06:11.837719+0000","last_active":"2023-10-01T11:09:23.204428+0000","last_peered":"2023-10-01T11:09:23.204428+0000","last_clean":"2023-10-01T11:09:23.204428+0000","last_became_active":"2023-10-01T11:06:11.798523+0000","last_became_peered":"2023-10-01T11:06:11.798523+0000","last_unstale":"2023-10-01T11:09:23.204428+0000","last_undegraded":"2023-10-01T11:09:23.204428+0000","last_fullsized":"2023-10-01T11:09:23.204428+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:53:34.413543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":294,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:22.261829+0000","last_change":"2023-10-01T11:08:46.872392+0000","last_active":"2023-10-01T11:09:22.261829+0000","last_peered":"2023-10-01T11:09:22.261829+0000","last_clean":"2023-10-01T11:09:22.261829+0000","last_became_active":"2023-10-01T11:08:46.872108+0000","last_became_peered":"2023-10-01T11:08:46.872108+0000","last_unstale":"2023-10-01T11:09:22.261829+0000","last_undegraded":"2023-10-01T11:09:22.261829+0000","last_fullsized":"2023-10-01T11:09:22.261829+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:01:40.481690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":355,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-01T11:09:37.852901+0000","last_change":"2023-10-01T11:06:11.784630+0000","last_active":"2023-10-01T11:09:37.852901+0000","last_peered":"2023-10-01T11:09:37.852901+0000","last_clean":"2023-10-01T11:09:37.852901+0000","last_became_active":"2023-10-01T11:06:11.784264+0000","last_became_peered":"2023-10-01T11:06:11.784264+0000","last_unstale":"2023-10-01T11:09:37.852901+0000","last_undegraded":"2023-10-01T11:09:37.852901+0000","last_fullsized":"2023-10-01T11:09:37.852901+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:03:03.888001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.b","version":"359'2456","reported_seq":2468,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-01T11:10:03.468756+0000","last_change":"2023-10-01T11:09:22.024506+0000","last_active":"2023-10-01T11:10:03.468756+0000","last_peered":"2023-10-01T11:10:03.468756+0000","last_clean":"2023-10-01T11:10:03.468756+0000","last_became_active":"2023-10-01T11:09:22.023185+0000","last_became_peered":"2023-10-01T11:09:22.023185+0000","last_unstale":"2023-10-01T11:10:03.468756+0000","last_undegraded":"2023-10-01T11:10:03.468756+0000","last_fullsized":"2023-10-01T11:10:03.468756+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":356,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_clean_scrub_stamp":"2023-10-01T11:09:20.132185+0000","objects_scrubbed":0,"log_size":2456,"log_dups_size":0,"ondisk_log_size":2456,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:06:56.128744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160956416,"num_objects":2456,"num_object_clones":0,"num_object_copies":7368,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2456,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2456,"num_write_kb":157184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":54,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:23.210463+0000","last_change":"2023-10-01T11:06:11.818913+0000","last_active":"2023-10-01T11:09:23.210463+0000","last_peered":"2023-10-01T11:09:23.210463+0000","last_clean":"2023-10-01T11:09:23.210463+0000","last_became_active":"2023-10-01T11:06:11.818533+0000","last_became_peered":"2023-10-01T11:06:11.818533+0000","last_unstale":"2023-10-01T11:09:23.210463+0000","last_undegraded":"2023-10-01T11:09:23.210463+0000","last_fullsized":"2023-10-01T11:09:23.210463+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":38,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:22.261885+0000","last_change":"2023-10-01T11:08:46.872670+0000","last_active":"2023-10-01T11:09:22.261885+0000","last_peered":"2023-10-01T11:09:22.261885+0000","last_clean":"2023-10-01T11:09:22.261885+0000","last_became_active":"2023-10-01T11:08:46.872300+0000","last_became_peered":"2023-10-01T11:08:46.872300+0000","last_unstale":"2023-10-01T11:09:22.261885+0000","last_undegraded":"2023-10-01T11:09:22.261885+0000","last_fullsized":"2023-10-01T11:09:22.261885+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:57:28.746652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":340,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:22.150581+0000","last_change":"2023-10-01T11:05:33.784531+0000","last_active":"2023-10-01T11:09:22.150581+0000","last_peered":"2023-10-01T11:09:22.150581+0000","last_clean":"2023-10-01T11:09:22.150581+0000","last_became_active":"2023-10-01T11:05:33.784167+0000","last_became_peered":"2023-10-01T11:05:33.784167+0000","last_unstale":"2023-10-01T11:09:22.150581+0000","last_undegraded":"2023-10-01T11:09:22.150581+0000","last_fullsized":"2023-10-01T11:09:22.150581+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:34:16.434841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":73,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-01T11:09:38.236995+0000","last_change":"2023-10-01T11:06:11.791476+0000","last_active":"2023-10-01T11:09:38.236995+0000","last_peered":"2023-10-01T11:09:38.236995+0000","last_clean":"2023-10-01T11:09:38.236995+0000","last_became_active":"2023-10-01T11:06:11.791021+0000","last_became_peered":"2023-10-01T11:06:11.791021+0000","last_unstale":"2023-10-01T11:09:38.236995+0000","last_undegraded":"2023-10-01T11:09:38.236995+0000","last_fullsized":"2023-10-01T11:09:38.236995+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:14:03.385152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":351,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-01T11:09:37.853661+0000","last_change":"2023-10-01T10:37:42.762360+0000","last_active":"2023-10-01T11:09:37.853661+0000","last_peered":"2023-10-01T11:09:37.853661+0000","last_clean":"2023-10-01T11:09:37.853661+0000","last_became_active":"2023-10-01T10:37:42.761936+0000","last_became_peered":"2023-10-01T10:37:42.761936+0000","last_unstale":"2023-10-01T11:09:37.853661+0000","last_undegraded":"2023-10-01T11:09:37.853661+0000","last_fullsized":"2023-10-01T11:09:37.853661+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:50.637582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":343,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:22.157310+0000","last_change":"2023-10-01T10:38:16.037809+0000","last_active":"2023-10-01T11:09:22.157310+0000","last_peered":"2023-10-01T11:09:22.157310+0000","last_clean":"2023-10-01T11:09:22.157310+0000","last_became_active":"2023-10-01T10:38:16.037318+0000","last_became_peered":"2023-10-01T10:38:16.037318+0000","last_unstale":"2023-10-01T11:09:22.157310+0000","last_undegraded":"2023-10-01T11:09:22.157310+0000","last_fullsized":"2023-10-01T11:09:22.157310+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":343,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:22.151476+0000","last_change":"2023-10-01T11:04:02.726007+0000","last_active":"2023-10-01T11:09:22.151476+0000","last_peered":"2023-10-01T11:09:22.151476+0000","last_clean":"2023-10-01T11:09:22.151476+0000","last_became_active":"2023-10-01T11:04:02.725507+0000","last_became_peered":"2023-10-01T11:04:02.725507+0000","last_unstale":"2023-10-01T11:09:22.151476+0000","last_undegraded":"2023-10-01T11:09:22.151476+0000","last_fullsized":"2023-10-01T11:09:22.151476+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:32:04.611533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":406,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:22.157151+0000","last_change":"2023-10-01T11:05:33.830108+0000","last_active":"2023-10-01T11:09:22.157151+0000","last_peered":"2023-10-01T11:09:22.157151+0000","last_clean":"2023-10-01T11:09:22.157151+0000","last_became_active":"2023-10-01T11:05:33.791882+0000","last_became_peered":"2023-10-01T11:05:33.791882+0000","last_unstale":"2023-10-01T11:09:22.157151+0000","last_undegraded":"2023-10-01T11:09:22.157151+0000","last_fullsized":"2023-10-01T11:09:22.157151+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":343,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:22.150452+0000","last_change":"2023-10-01T11:04:02.733653+0000","last_active":"2023-10-01T11:09:22.150452+0000","last_peered":"2023-10-01T11:09:22.150452+0000","last_clean":"2023-10-01T11:09:22.150452+0000","last_became_active":"2023-10-01T11:04:02.733178+0000","last_became_peered":"2023-10-01T11:04:02.733178+0000","last_unstale":"2023-10-01T11:09:22.150452+0000","last_undegraded":"2023-10-01T11:09:22.150452+0000","last_fullsized":"2023-10-01T11:09:22.150452+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:18:25.249624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":404,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-01T11:09:37.853699+0000","last_change":"2023-10-01T11:05:33.777019+0000","last_active":"2023-10-01T11:09:37.853699+0000","last_peered":"2023-10-01T11:09:37.853699+0000","last_clean":"2023-10-01T11:09:37.853699+0000","last_became_active":"2023-10-01T11:05:33.776565+0000","last_became_peered":"2023-10-01T11:05:33.776565+0000","last_unstale":"2023-10-01T11:09:37.853699+0000","last_undegraded":"2023-10-01T11:09:37.853699+0000","last_fullsized":"2023-10-01T11:09:37.853699+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T22:00:18.890326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":360,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:23.203708+0000","last_change":"2023-10-01T10:37:42.765142+0000","last_active":"2023-10-01T11:09:23.203708+0000","last_peered":"2023-10-01T11:09:23.203708+0000","last_clean":"2023-10-01T11:09:23.203708+0000","last_became_active":"2023-10-01T10:37:42.764770+0000","last_became_peered":"2023-10-01T10:37:42.764770+0000","last_unstale":"2023-10-01T11:09:23.203708+0000","last_undegraded":"2023-10-01T11:09:23.203708+0000","last_fullsized":"2023-10-01T11:09:23.203708+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:23:33.527876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":73,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-10-01T11:09:22.157364+0000","last_change":"2023-10-01T11:04:40.889388+0000","last_active":"2023-10-01T11:09:22.157364+0000","last_peered":"2023-10-01T11:09:22.157364+0000","last_clean":"2023-10-01T11:09:22.157364+0000","last_became_active":"2023-10-01T11:04:40.888960+0000","last_became_peered":"2023-10-01T11:04:40.888960+0000","last_unstale":"2023-10-01T11:09:22.157364+0000","last_undegraded":"2023-10-01T11:09:22.157364+0000","last_fullsized":"2023-10-01T11:09:22.157364+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":379,"reported_epoch":359,"state":"active+clean","last_fresh":"2023-10-01T11:09:38.235884+0000","last_change":"2023-10-01T11:05:33.785774+0000","last_active":"2023-10-01T11:09:38.235884+0000","last_peered":"2023-10-01T11:09:38.235884+0000","last_clean":"2023-10-01T11:09:38.235884+0000","last_became_active":"2023-10-01T11:05:33.785426+0000","last_became_peered":"2023-10-01T11:05:33.785426+0000","last_unstale":"2023-10-01T11:09:38.235884+0000","last_undegraded":"2023-10-01T11:09:38.235884+0000","last_fullsized":"2023-10-01T11:09:38.235884+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:41:32.634339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":14,"num_pg":16,"stat_sum":{"num_bytes":2704539648,"num_objects":41268,"num_object_clones":0,"num_object_copies":123804,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":41268,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":41268,"num_write_kb":2641152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8111194112,"data_stored":8111194112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":41268,"ondisk_log_size":41268,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":82,"num_bytes_recovered":34354,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30319,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":5313312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430891,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1945896,"kb_used_data":471264,"kb_used_omap":0,"kb_used_meta":1474624,"kb_avail":91799256,"statfs":{"total":95995035648,"available":94002438144,"internally_reserved":0,"allocated":482574336,"data_stored":481991363,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1510014976},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 11:09:39 2023","interfaces":[{"interface":"back","average":{"1min":0.996,"5min":1.1839999999999999,"15min":1.1919999999999999},"min":{"1min":0.57399999999999995,"5min":0.27900000000000003,"15min":0.25700000000000001},"max":{"1min":1.284,"5min":10.013,"15min":11.166},"last":1.7529999999999999},{"interface":"front","average":{"1min":0.97399999999999998,"5min":1.218,"15min":1.2829999999999999},"min":{"1min":0.48899999999999999,"5min":0.34200000000000003,"15min":0.307},"max":{"1min":1.335,"5min":8.8949999999999996,"15min":11.18},"last":3.661}]},{"osd":1,"last update":"Sun Oct 1 11:09:39 2023","interfaces":[{"interface":"back","average":{"1min":0.875,"5min":1.1479999999999999,"15min":1.2769999999999999},"min":{"1min":0.503,"5min":0.30199999999999999,"15min":0.27400000000000002},"max":{"1min":1.2150000000000001,"5min":8.9309999999999992,"15min":14.132999999999999},"last":1.2490000000000001},{"interface":"front","average":{"1min":0.82399999999999995,"5min":1.2709999999999999,"15min":1.2829999999999999},"min":{"1min":0.497,"5min":0.33000000000000002,"15min":0.32400000000000001},"max":{"1min":1.236,"5min":9.8170000000000002,"15min":13.558999999999999},"last":7.2530000000000001}]},{"osd":2,"last update":"Sun Oct 1 11:09:53 2023","interfaces":[{"interface":"back","average":{"1min":0.874,"5min":1.4830000000000001,"15min":2.7000000000000002},"min":{"1min":0.44900000000000001,"5min":0.36899999999999999,"15min":0.33300000000000002},"max":{"1min":1.4870000000000001,"5min":9.7799999999999994,"15min":215.89699999999999},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.91900000000000004,"5min":1.2789999999999999,"15min":2.4239999999999999},"min":{"1min":0.40699999999999997,"5min":0.34899999999999998,"15min":0.307},"max":{"1min":1.218,"5min":8.3989999999999991,"15min":215.77600000000001},"last":0.45000000000000001}]},{"osd":3,"last update":"Sun Oct 1 11:09:39 2023","interfaces":[{"interface":"back","average":{"1min":0.89900000000000002,"5min":1.327,"15min":1.47},"min":{"1min":0.48999999999999999,"5min":0.40300000000000002,"15min":0.36299999999999999},"max":{"1min":1.329,"5min":12.358000000000001,"15min":12.358000000000001},"last":2.133},{"interface":"front","average":{"1min":0.99299999999999999,"5min":1.1499999999999999,"15min":1.282},"min":{"1min":0.66000000000000003,"5min":0.376,"15min":0.28299999999999997},"max":{"1min":1.296,"5min":8.3109999999999999,"15min":11.244999999999999},"last":0.39700000000000002}]},{"osd":4,"last update":"Sun Oct 1 11:09:13 2023","interfaces":[{"interface":"back","average":{"1min":1.091,"5min":1.492,"15min":1.4019999999999999},"min":{"1min":0.39200000000000002,"5min":0.34999999999999998,"15min":0.33100000000000002},"max":{"1min":2.8090000000000002,"5min":14.51,"15min":17.178999999999998},"last":0.42599999999999999},{"interface":"front","average":{"1min":1.0609999999999999,"5min":1.431,"15min":1.46},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.375},"max":{"1min":2.3029999999999999,"5min":12.414999999999999,"15min":17.25},"last":0.52100000000000002}]},{"osd":5,"last update":"Sun Oct 1 11:09:39 2023","interfaces":[{"interface":"back","average":{"1min":1.0089999999999999,"5min":1.9750000000000001,"15min":1.595},"min":{"1min":0.60099999999999998,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.3460000000000001,"5min":18.998999999999999,"15min":18.998999999999999},"last":0.58899999999999997},{"interface":"front","average":{"1min":1.024,"5min":1.4350000000000001,"15min":1.2629999999999999},"min":{"1min":0.71199999999999997,"5min":0.36899999999999999,"15min":0.33500000000000002},"max":{"1min":1.252,"5min":10.1,"15min":10.1},"last":0.66300000000000003}]},{"osd":6,"last update":"Sun Oct 1 11:09:39 2023","interfaces":[{"interface":"back","average":{"1min":1.0569999999999999,"5min":1.6080000000000001,"15min":1.4790000000000001},"min":{"1min":0.46999999999999997,"5min":0.39100000000000001,"15min":0.35299999999999998},"max":{"1min":1.341,"5min":13.038,"15min":15.433},"last":0.56100000000000005},{"interface":"front","average":{"1min":1.0489999999999999,"5min":1.508,"15min":1.425},"min":{"1min":0.44,"5min":0.36199999999999999,"15min":0.34899999999999998},"max":{"1min":1.458,"5min":11.536,"15min":11.536},"last":0.64600000000000002}]}]},{"osd":6,"up_from":42,"seq":180388627131,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1829336,"kb_used_data":643856,"kb_used_omap":0,"kb_used_meta":1185472,"kb_avail":91915816,"statfs":{"total":95995035648,"available":94121795584,"internally_reserved":0,"allocated":659308544,"data_stored":658741494,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1213923328},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 11:09:28 2023","interfaces":[{"interface":"back","average":{"1min":1.4390000000000001,"5min":1.145,"15min":1.528},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.26800000000000002},"max":{"1min":14.964,"5min":14.964,"15min":102.23},"last":2.702},{"interface":"front","average":{"1min":0.871,"5min":1.0109999999999999,"15min":1.506},"min":{"1min":0.52000000000000002,"5min":0.29999999999999999,"15min":0.29399999999999998},"max":{"1min":1.589,"5min":9.3849999999999998,"15min":101.217},"last":0.41299999999999998}]},{"osd":1,"last update":"Sun Oct 1 11:09:53 2023","interfaces":[{"interface":"back","average":{"1min":1.0589999999999999,"5min":1.1659999999999999,"15min":1.218},"min":{"1min":0.48899999999999999,"5min":0.31,"15min":0.27100000000000002},"max":{"1min":4.2460000000000004,"5min":14.946999999999999,"15min":14.946999999999999},"last":1.5389999999999999},{"interface":"front","average":{"1min":1.0489999999999999,"5min":1.071,"15min":1.153},"min":{"1min":0.40899999999999997,"5min":0.31,"15min":0.31},"max":{"1min":3.3210000000000002,"5min":6.835,"15min":7.0380000000000003},"last":0.57699999999999996}]},{"osd":2,"last update":"Sun Oct 1 11:09:59 2023","interfaces":[{"interface":"back","average":{"1min":3.5419999999999998,"5min":1.768,"15min":2.528},"min":{"1min":0.443,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":48.881999999999998,"5min":48.881999999999998,"15min":106.705},"last":0.46800000000000003},{"interface":"front","average":{"1min":3.7210000000000001,"5min":1.8080000000000001,"15min":2.8119999999999998},"min":{"1min":0.71699999999999997,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":48.948,"5min":48.948,"15min":106.806},"last":0.81200000000000006}]},{"osd":3,"last update":"Sun Oct 1 11:09:53 2023","interfaces":[{"interface":"back","average":{"1min":3.4239999999999999,"5min":1.6919999999999999,"15min":1.9890000000000001},"min":{"1min":0.62,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":50.225000000000001,"5min":50.225000000000001,"15min":102.273},"last":4.0940000000000003},{"interface":"front","average":{"1min":3.2280000000000002,"5min":1.6539999999999999,"15min":2.0569999999999999},"min":{"1min":0.46500000000000002,"5min":0.39000000000000001,"15min":0.36499999999999999},"max":{"1min":49.301000000000002,"5min":49.301000000000002,"15min":101.491},"last":3.3639999999999999}]},{"osd":4,"last update":"Sun Oct 1 11:09:20 2023","interfaces":[{"interface":"back","average":{"1min":1.1970000000000001,"5min":1.2050000000000001,"15min":1.137},"min":{"1min":0.47799999999999998,"5min":0.315,"15min":0.30299999999999999},"max":{"1min":5.4290000000000003,"5min":8.6289999999999996,"15min":8.6289999999999996},"last":0.871},{"interface":"front","average":{"1min":1.6719999999999999,"5min":1.722,"15min":1.615},"min":{"1min":0.42799999999999999,"5min":0.38700000000000001,"15min":0.36799999999999999},"max":{"1min":10.628,"5min":15.462,"15min":15.462},"last":0.67700000000000005}]},{"osd":5,"last update":"Sun Oct 1 11:09:53 2023","interfaces":[{"interface":"back","average":{"1min":1.121,"5min":1.744,"15min":1.345},"min":{"1min":0.39700000000000002,"5min":0.39400000000000002,"15min":0.33000000000000002},"max":{"1min":3.2440000000000002,"5min":26.379000000000001,"15min":26.379000000000001},"last":0.69399999999999995},{"interface":"front","average":{"1min":1.161,"5min":1.6839999999999999,"15min":1.5},"min":{"1min":0.46999999999999997,"5min":0.32000000000000001,"15min":0.315},"max":{"1min":3.339,"5min":28.010000000000002,"15min":28.010000000000002},"last":0.66600000000000004}]},{"osd":7,"last update":"Sun Oct 1 11:09:28 2023","interfaces":[{"interface":"back","average":{"1min":1.149,"5min":1.2410000000000001,"15min":1.27},"min":{"1min":0.50700000000000001,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":2.4470000000000001,"5min":6.7599999999999998,"15min":9.5419999999999998},"last":0.92900000000000005},{"interface":"front","average":{"1min":1.635,"5min":1.589,"15min":1.3740000000000001},"min":{"1min":0.47299999999999998,"5min":0.42499999999999999,"15min":0.38500000000000001},"max":{"1min":14.894,"5min":14.894,"15min":14.894},"last":0.63100000000000001}]}]},{"osd":1,"up_from":101,"seq":433791697349,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2242680,"kb_used_data":1338160,"kb_used_omap":4,"kb_used_meta":904507,"kb_avail":91502472,"statfs":{"total":95995035648,"available":93698531328,"internally_reserved":0,"allocated":1370275840,"data_stored":1368874553,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":926216078},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,2,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 11:09:52 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.58999999999999997,"15min":1.008},"min":{"1min":0.247,"5min":0.223,"15min":0.222},"max":{"1min":1.1579999999999999,"5min":4.54,"15min":12.888999999999999},"last":1.415},{"interface":"front","average":{"1min":0.745,"5min":0.60099999999999998,"15min":1.04},"min":{"1min":0.36499999999999999,"5min":0.23200000000000001,"15min":0.20699999999999999},"max":{"1min":3.4569999999999999,"5min":3.4790000000000001,"15min":15.416},"last":1.0409999999999999}]},{"osd":2,"last update":"Sun Oct 1 11:09:20 2023","interfaces":[{"interface":"back","average":{"1min":0.67800000000000005,"5min":0.69299999999999995,"15min":1.627},"min":{"1min":0.34899999999999998,"5min":0.24199999999999999,"15min":0.23200000000000001},"max":{"1min":2.077,"5min":5.0910000000000002,"15min":41.026000000000003},"last":2.9420000000000002},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.73099999999999998,"15min":1.7629999999999999},"min":{"1min":0.38200000000000001,"5min":0.29199999999999998,"15min":0.247},"max":{"1min":0.98399999999999999,"5min":5.1630000000000003,"15min":41.148000000000003},"last":5.0810000000000004}]},{"osd":3,"last update":"Sun Oct 1 11:09:52 2023","interfaces":[{"interface":"back","average":{"1min":0.90200000000000002,"5min":0.72099999999999997,"15min":1.0629999999999999},"min":{"1min":0.36899999999999999,"5min":0.27400000000000002,"15min":0.248},"max":{"1min":3.5899999999999999,"5min":3.5899999999999999,"15min":17.681999999999999},"last":1.3759999999999999},{"interface":"front","average":{"1min":0.72599999999999998,"5min":0.68500000000000005,"15min":1.1060000000000001},"min":{"1min":0.38400000000000001,"5min":0.28999999999999998,"15min":0.223},"max":{"1min":1.341,"5min":3.2629999999999999,"15min":10.983000000000001},"last":0.219}]},{"osd":4,"last update":"Sun Oct 1 11:09:20 2023","interfaces":[{"interface":"back","average":{"1min":1.125,"5min":3.306,"15min":1.887},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.307},"max":{"1min":5.7309999999999999,"5min":203.89500000000001,"15min":203.89500000000001},"last":0.82699999999999996},{"interface":"front","average":{"1min":1.1339999999999999,"5min":3.4420000000000002,"15min":1.982},"min":{"1min":0.39500000000000002,"5min":0.36699999999999999,"15min":0.34699999999999998},"max":{"1min":3.77,"5min":203.852,"15min":203.852},"last":0.32600000000000001}]},{"osd":5,"last update":"Sun Oct 1 11:09:52 2023","interfaces":[{"interface":"back","average":{"1min":1.0009999999999999,"5min":1.0760000000000001,"15min":1.153},"min":{"1min":0.622,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":1.333,"5min":8.2249999999999996,"15min":9.048},"last":1.1619999999999999},{"interface":"front","average":{"1min":1.331,"5min":1.194,"15min":1.179},"min":{"1min":0.71099999999999997,"5min":0.38,"15min":0.34200000000000003},"max":{"1min":5.5019999999999998,"5min":8.1110000000000007,"15min":9.8100000000000005},"last":0.86699999999999999}]},{"osd":6,"last update":"Sun Oct 1 11:09:52 2023","interfaces":[{"interface":"back","average":{"1min":1.1240000000000001,"5min":1.0509999999999999,"15min":1.214},"min":{"1min":0.65000000000000002,"5min":0.371,"15min":0.33800000000000002},"max":{"1min":2.1960000000000002,"5min":6.931,"15min":7.4509999999999996},"last":0.89700000000000002},{"interface":"front","average":{"1min":1.1220000000000001,"5min":1.1020000000000001,"15min":1.2110000000000001},"min":{"1min":0.69099999999999995,"5min":0.44900000000000001,"15min":0.33200000000000002},"max":{"1min":3.427,"5min":7.7569999999999997,"15min":7.7569999999999997},"last":1.7}]},{"osd":7,"last update":"Sun Oct 1 11:09:52 2023","interfaces":[{"interface":"back","average":{"1min":1.167,"5min":1.173,"15min":1.222},"min":{"1min":0.747,"5min":0.41299999999999998,"15min":0.36399999999999999},"max":{"1min":3.3639999999999999,"5min":9.5830000000000002,"15min":9.5830000000000002},"last":1.6100000000000001},{"interface":"front","average":{"1min":1.0529999999999999,"5min":1.1379999999999999,"15min":1.27},"min":{"1min":0.66600000000000004,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":1.734,"5min":6.0679999999999996,"15min":11.689},"last":0.42099999999999999}]}]},{"osd":0,"up_from":86,"seq":369367187959,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2392768,"kb_used_data":1426424,"kb_used_omap":5,"kb_used_meta":966330,"kb_avail":91352384,"statfs":{"total":95995035648,"available":93544841216,"internally_reserved":0,"allocated":1460658176,"data_stored":1459305698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":989522554},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 11:10:01 2023","interfaces":[{"interface":"back","average":{"1min":0.88100000000000001,"5min":0.72699999999999998,"15min":1.3799999999999999},"min":{"1min":0.26700000000000002,"5min":0.20000000000000001,"15min":0.19700000000000001},"max":{"1min":3.8090000000000002,"5min":7.8570000000000002,"15min":119.911},"last":0.60799999999999998},{"interface":"front","average":{"1min":1.4079999999999999,"5min":0.88100000000000001,"15min":1.552},"min":{"1min":0.40500000000000003,"5min":0.25700000000000001,"15min":0.20200000000000001},"max":{"1min":11.353,"5min":11.353,"15min":119.76300000000001},"last":1.5409999999999999}]},{"osd":2,"last update":"Sun Oct 1 11:10:03 2023","interfaces":[{"interface":"back","average":{"1min":0.85399999999999998,"5min":1.3089999999999999,"15min":2.0219999999999998},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.20699999999999999},"max":{"1min":2.5569999999999999,"5min":55.078000000000003,"15min":72.064999999999998},"last":0.40200000000000002},{"interface":"front","average":{"1min":1.1899999999999999,"5min":1.5700000000000001,"15min":2.194},"min":{"1min":0.45500000000000002,"5min":0.28399999999999997,"15min":0.252},"max":{"1min":3.2480000000000002,"5min":55.719000000000001,"15min":76.328999999999994},"last":0.45500000000000002}]},{"osd":3,"last update":"Sun Oct 1 11:10:01 2023","interfaces":[{"interface":"back","average":{"1min":1.2929999999999999,"5min":0.94899999999999995,"15min":1.756},"min":{"1min":0.36699999999999999,"5min":0.29799999999999999,"15min":0.214},"max":{"1min":4.6420000000000003,"5min":8.1560000000000006,"15min":81.674999999999997},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.95999999999999996,"5min":0.92700000000000005,"15min":1.5840000000000001},"min":{"1min":0.48399999999999999,"5min":0.28100000000000003,"15min":0.246},"max":{"1min":3.8239999999999998,"5min":8.1389999999999993,"15min":70.950999999999993},"last":0.53500000000000003}]},{"osd":4,"last update":"Sun Oct 1 11:09:30 2023","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":1.2490000000000001,"15min":1.327},"min":{"1min":0.627,"5min":0.38400000000000001,"15min":0.34300000000000003},"max":{"1min":1.3300000000000001,"5min":8.1080000000000005,"15min":9.3320000000000007},"last":0.64000000000000001},{"interface":"front","average":{"1min":1.0009999999999999,"5min":1.242,"15min":1.367},"min":{"1min":0.57599999999999996,"5min":0.36099999999999999,"15min":0.32300000000000001},"max":{"1min":1.544,"5min":8.0370000000000008,"15min":8.0530000000000008},"last":0.33600000000000002}]},{"osd":5,"last update":"Sun Oct 1 11:10:01 2023","interfaces":[{"interface":"back","average":{"1min":1.115,"5min":1.5629999999999999,"15min":1.577},"min":{"1min":0.41799999999999998,"5min":0.36699999999999999,"15min":0.309},"max":{"1min":2.7749999999999999,"5min":10.916,"15min":39.860999999999997},"last":1.1870000000000001},{"interface":"front","average":{"1min":1.0880000000000001,"5min":1.6040000000000001,"15min":1.5549999999999999},"min":{"1min":0.504,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":3.7589999999999999,"5min":10.930999999999999,"15min":39.646000000000001},"last":0.69899999999999995}]},{"osd":6,"last update":"Sun Oct 1 11:10:01 2023","interfaces":[{"interface":"back","average":{"1min":1.1160000000000001,"5min":1.2869999999999999,"15min":1.333},"min":{"1min":0.55200000000000005,"5min":0.373,"15min":0.373},"max":{"1min":3.8410000000000002,"5min":8.1660000000000004,"15min":9.4570000000000007},"last":0.67000000000000004},{"interface":"front","average":{"1min":1.1220000000000001,"5min":1.1899999999999999,"15min":1.349},"min":{"1min":0.57799999999999996,"5min":0.44800000000000001,"15min":0.378},"max":{"1min":3.9249999999999998,"5min":8.2379999999999995,"15min":11.032999999999999},"last":0.48799999999999999}]},{"osd":7,"last update":"Sun Oct 1 11:10:01 2023","interfaces":[{"interface":"back","average":{"1min":1.0169999999999999,"5min":1.22,"15min":1.3420000000000001},"min":{"1min":0.54900000000000004,"5min":0.45000000000000001,"15min":0.35499999999999998},"max":{"1min":1.5680000000000001,"5min":8.3109999999999999,"15min":12.680999999999999},"last":0.71999999999999997},{"interface":"front","average":{"1min":1.347,"5min":1.369,"15min":1.5269999999999999},"min":{"1min":0.67200000000000004,"5min":0.41499999999999998,"15min":0.32200000000000001},"max":{"1min":3.9449999999999998,"5min":7.9660000000000002,"15min":9.4659999999999993},"last":0.56599999999999995}]}]},{"osd":2,"up_from":116,"seq":498216206733,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1900908,"kb_used_data":1111844,"kb_used_omap":7,"kb_used_meta":789048,"kb_avail":91844244,"statfs":{"total":95995035648,"available":94048505856,"internally_reserved":0,"allocated":1138528256,"data_stored":1137192273,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":807985233},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,3,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 11:09:03 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.65800000000000003,"15min":1.542},"min":{"1min":0.248,"5min":0.21199999999999999,"15min":0.20200000000000001},"max":{"1min":0.90000000000000002,"5min":9.0310000000000006,"15min":104.42},"last":1.7410000000000001},{"interface":"front","average":{"1min":0.67400000000000004,"5min":0.65200000000000002,"15min":1.73},"min":{"1min":0.222,"5min":0.222,"15min":0.187},"max":{"1min":2.7410000000000001,"5min":2.8439999999999999,"15min":105.10899999999999},"last":0.32600000000000001}]},{"osd":1,"last update":"Sun Oct 1 11:09:03 2023","interfaces":[{"interface":"back","average":{"1min":0.79200000000000004,"5min":0.80000000000000004,"15min":1.3029999999999999},"min":{"1min":0.313,"5min":0.28799999999999998,"15min":0.20699999999999999},"max":{"1min":2.9289999999999998,"5min":7.1660000000000004,"15min":8.6440000000000001},"last":0.254},{"interface":"front","average":{"1min":0.92400000000000004,"5min":0.83599999999999997,"15min":1.5369999999999999},"min":{"1min":0.28299999999999997,"5min":0.26900000000000002,"15min":0.23999999999999999},"max":{"1min":3.0129999999999999,"5min":5.0970000000000004,"15min":11.554},"last":0.58199999999999996}]},{"osd":3,"last update":"Sun Oct 1 11:09:40 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.67800000000000005,"15min":1.371},"min":{"1min":0.39800000000000002,"5min":0.27300000000000002,"15min":0.23200000000000001},"max":{"1min":0.96299999999999997,"5min":3.4060000000000001,"15min":23.879000000000001},"last":1.9419999999999999},{"interface":"front","average":{"1min":0.70099999999999996,"5min":0.78700000000000003,"15min":1.468},"min":{"1min":0.49399999999999999,"5min":0.28599999999999998,"15min":0.26400000000000001},"max":{"1min":1.147,"5min":2.7909999999999999,"15min":23.704000000000001},"last":0.29299999999999998}]},{"osd":4,"last update":"Sun Oct 1 11:09:24 2023","interfaces":[{"interface":"back","average":{"1min":1.0149999999999999,"5min":1.2509999999999999,"15min":1.508},"min":{"1min":0.39700000000000002,"5min":0.34399999999999997,"15min":0.33000000000000002},"max":{"1min":2.8500000000000001,"5min":7.1189999999999998,"15min":9.5109999999999992},"last":1.5609999999999999},{"interface":"front","average":{"1min":1.296,"5min":1.3580000000000001,"15min":1.7350000000000001},"min":{"1min":0.51600000000000001,"5min":0.42599999999999999,"15min":0.317},"max":{"1min":5.1040000000000001,"5min":6.4589999999999996,"15min":11.574},"last":0.68300000000000005}]},{"osd":5,"last update":"Sun Oct 1 11:09:03 2023","interfaces":[{"interface":"back","average":{"1min":1.25,"5min":1.242,"15min":1.5740000000000001},"min":{"1min":0.41499999999999998,"5min":0.40100000000000002,"15min":0.36599999999999999},"max":{"1min":4.4000000000000004,"5min":5.0949999999999998,"15min":10.180999999999999},"last":0.39600000000000002},{"interface":"front","average":{"1min":2.0600000000000001,"5min":1.502,"15min":1.8660000000000001},"min":{"1min":0.52700000000000002,"5min":0.504,"15min":0.36199999999999999},"max":{"1min":12.518000000000001,"5min":12.518000000000001,"15min":12.835000000000001},"last":0.64900000000000002}]},{"osd":6,"last update":"Sun Oct 1 11:09:03 2023","interfaces":[{"interface":"back","average":{"1min":1.1850000000000001,"5min":1.288,"15min":1.905},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":3.335,"5min":6.1539999999999999,"15min":19.908000000000001},"last":0.71299999999999997},{"interface":"front","average":{"1min":1.121,"5min":1.044,"15min":1.4139999999999999},"min":{"1min":0.51200000000000001,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":3.4100000000000001,"5min":3.911,"15min":9.6709999999999994},"last":1.7729999999999999}]},{"osd":7,"last update":"Sun Oct 1 11:09:03 2023","interfaces":[{"interface":"back","average":{"1min":1.444,"5min":1.161,"15min":1.4319999999999999},"min":{"1min":0.53600000000000003,"5min":0.36699999999999999,"15min":0.35099999999999998},"max":{"1min":6.9169999999999998,"5min":6.9169999999999998,"15min":11.209},"last":1.802},{"interface":"front","average":{"1min":1.276,"5min":1.2290000000000001,"15min":1.573},"min":{"1min":0.44,"5min":0.44,"15min":0.40100000000000002},"max":{"1min":4.9560000000000004,"5min":5.016,"15min":9.5489999999999995},"last":0.42899999999999999}]}]},{"osd":3,"up_from":121,"seq":519691043207,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1957148,"kb_used_data":1197844,"kb_used_omap":2,"kb_used_meta":759293,"kb_avail":91788004,"statfs":{"total":95995035648,"available":93990916096,"internally_reserved":0,"allocated":1226592256,"data_stored":1225270227,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":777516085},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 11:09:45 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.65300000000000002,"15min":1.4590000000000001},"min":{"1min":0.23000000000000001,"5min":0.222,"15min":0.17199999999999999},"max":{"1min":1.012,"5min":5.4269999999999996,"15min":100.191},"last":0.623},{"interface":"front","average":{"1min":0.65700000000000003,"5min":0.63700000000000001,"15min":1.6120000000000001},"min":{"1min":0.33400000000000002,"5min":0.221,"15min":0.19600000000000001},"max":{"1min":1.2170000000000001,"5min":5.1719999999999997,"15min":99.852999999999994},"last":1.298}]},{"osd":1,"last update":"Sun Oct 1 11:09:45 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.61799999999999999,"15min":1.0589999999999999},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.22500000000000001},"max":{"1min":1.022,"5min":3.6139999999999999,"15min":10.574999999999999},"last":1.7110000000000001},{"interface":"front","average":{"1min":0.69999999999999996,"5min":0.69499999999999995,"15min":1.1890000000000001},"min":{"1min":0.40200000000000002,"5min":0.26600000000000001,"15min":0.23400000000000001},"max":{"1min":0.94999999999999996,"5min":3.5259999999999998,"15min":13.086},"last":1.679}]},{"osd":2,"last update":"Sun Oct 1 11:09:45 2023","interfaces":[{"interface":"back","average":{"1min":0.98599999999999999,"5min":1.387,"15min":2.0819999999999999},"min":{"1min":0.314,"5min":0.252,"15min":0.217},"max":{"1min":5.1970000000000001,"5min":53.948999999999998,"15min":81.212999999999994},"last":2.0129999999999999},{"interface":"front","average":{"1min":0.93700000000000006,"5min":1.4410000000000001,"15min":2.1219999999999999},"min":{"1min":0.45800000000000002,"5min":0.31900000000000001,"15min":0.27100000000000002},"max":{"1min":2.1800000000000002,"5min":53.899999999999999,"15min":80.819000000000003},"last":0.85299999999999998}]},{"osd":4,"last update":"Sun Oct 1 11:09:28 2023","interfaces":[{"interface":"back","average":{"1min":1.0920000000000001,"5min":1.103,"15min":1.4219999999999999},"min":{"1min":0.46999999999999997,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":3.5089999999999999,"5min":5.8929999999999998,"15min":10.808999999999999},"last":2.2450000000000001},{"interface":"front","average":{"1min":1.0880000000000001,"5min":1.093,"15min":1.4490000000000001},"min":{"1min":0.48599999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":3.492,"5min":5.2770000000000001,"15min":13.391999999999999},"last":2.4140000000000001}]},{"osd":5,"last update":"Sun Oct 1 11:09:45 2023","interfaces":[{"interface":"back","average":{"1min":0.98199999999999998,"5min":1.1930000000000001,"15min":1.381},"min":{"1min":0.56200000000000006,"5min":0.35599999999999998,"15min":0.34399999999999997},"max":{"1min":1.8939999999999999,"5min":7.3710000000000004,"15min":12.771000000000001},"last":1.26},{"interface":"front","average":{"1min":1.1559999999999999,"5min":1.292,"15min":1.5089999999999999},"min":{"1min":0.63100000000000001,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":5.2489999999999997,"5min":6.9130000000000003,"15min":12.335000000000001},"last":1.502}]},{"osd":6,"last update":"Sun Oct 1 11:09:45 2023","interfaces":[{"interface":"back","average":{"1min":1.008,"5min":1.222,"15min":1.4930000000000001},"min":{"1min":0.56299999999999994,"5min":0.38,"15min":0.38},"max":{"1min":1.706,"5min":8.9009999999999998,"15min":13.616},"last":1.331},{"interface":"front","average":{"1min":1.2330000000000001,"5min":1.2010000000000001,"15min":1.802},"min":{"1min":0.754,"5min":0.434,"15min":0.36799999999999999},"max":{"1min":4.2699999999999996,"5min":8.8849999999999998,"15min":73.542000000000002},"last":2.7360000000000002}]},{"osd":7,"last update":"Sun Oct 1 11:09:45 2023","interfaces":[{"interface":"back","average":{"1min":1.206,"5min":1.038,"15min":1.478},"min":{"1min":0.68600000000000005,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":4.3010000000000002,"5min":4.6959999999999997,"15min":12.416},"last":1.6419999999999999},{"interface":"front","average":{"1min":1.0229999999999999,"5min":1.0149999999999999,"15min":1.3799999999999999},"min":{"1min":0.57499999999999996,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":1.3839999999999999,"5min":5.6269999999999998,"15min":10.755000000000001},"last":1.994}]}]},{"osd":4,"up_from":126,"seq":541165879680,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1660544,"kb_used_data":993080,"kb_used_omap":10,"kb_used_meta":667445,"kb_avail":92084608,"statfs":{"total":95995035648,"available":94294638592,"internally_reserved":0,"allocated":1016913920,"data_stored":1015558323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":683464292},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 11:09:12 2023","interfaces":[{"interface":"back","average":{"1min":0.86899999999999999,"5min":1.099,"15min":1.6060000000000001},"min":{"1min":0.38,"5min":0.32600000000000001,"15min":0.26400000000000001},"max":{"1min":2.8580000000000001,"5min":8.8670000000000009,"15min":68.611000000000004},"last":0.30099999999999999},{"interface":"front","average":{"1min":0.98399999999999999,"5min":1.246,"15min":1.6240000000000001},"min":{"1min":0.35999999999999999,"5min":0.307,"15min":0.28799999999999998},"max":{"1min":3.726,"5min":12.369,"15min":68.480999999999995},"last":0.40400000000000003}]},{"osd":1,"last update":"Sun Oct 1 11:09:12 2023","interfaces":[{"interface":"back","average":{"1min":1.054,"5min":1.258,"15min":1.173},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.30199999999999999},"max":{"1min":4.843,"5min":9.1920000000000002,"15min":11.664},"last":0.68400000000000005},{"interface":"front","average":{"1min":0.98399999999999999,"5min":1.093,"15min":1.1719999999999999},"min":{"1min":0.34000000000000002,"5min":0.308,"15min":0.308},"max":{"1min":2.903,"5min":8.7240000000000002,"15min":11.914999999999999},"last":0.375}]},{"osd":2,"last update":"Sun Oct 1 11:09:12 2023","interfaces":[{"interface":"back","average":{"1min":0.99099999999999999,"5min":1.496,"15min":2.8570000000000002},"min":{"1min":0.34699999999999998,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":2.9289999999999998,"5min":24.995000000000001,"15min":133.321},"last":1.484},{"interface":"front","average":{"1min":0.97499999999999998,"5min":1.5840000000000001,"15min":2.665},"min":{"1min":0.39800000000000002,"5min":0.34499999999999997,"15min":0.33700000000000002},"max":{"1min":3.6720000000000002,"5min":24.832000000000001,"15min":132.875},"last":0.63100000000000001}]},{"osd":3,"last update":"Sun Oct 1 11:09:12 2023","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":1.319,"15min":1.5649999999999999},"min":{"1min":0.47699999999999998,"5min":0.34200000000000003,"15min":0.32700000000000001},"max":{"1min":3.7989999999999999,"5min":12.473000000000001,"15min":61.664999999999999},"last":0.70499999999999996},{"interface":"front","average":{"1min":1.375,"5min":1.298,"15min":1.9970000000000001},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.308},"max":{"1min":8.2309999999999999,"5min":9.1419999999999995,"15min":130.88300000000001},"last":0.45500000000000002}]},{"osd":5,"last update":"Sun Oct 1 11:09:12 2023","interfaces":[{"interface":"back","average":{"1min":2.0590000000000002,"5min":1.879,"15min":1.671},"min":{"1min":0.44800000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":12.457000000000001,"5min":25.175999999999998,"15min":84.307000000000002},"last":0.56000000000000005},{"interface":"front","average":{"1min":2.3180000000000001,"5min":1.9339999999999999,"15min":1.883},"min":{"1min":0.46899999999999997,"5min":0.35099999999999998,"15min":0.34899999999999998},"max":{"1min":9.1969999999999992,"5min":10.406000000000001,"15min":84.234999999999999},"last":0.42799999999999999}]},{"osd":6,"last update":"Sun Oct 1 11:09:12 2023","interfaces":[{"interface":"back","average":{"1min":1.6679999999999999,"5min":1.5529999999999999,"15min":1.2549999999999999},"min":{"1min":0.42399999999999999,"5min":0.38100000000000001,"15min":0.34999999999999998},"max":{"1min":8.1440000000000001,"5min":12.635999999999999,"15min":12.635999999999999},"last":0.66500000000000004},{"interface":"front","average":{"1min":1.534,"5min":1.474,"15min":1.268},"min":{"1min":0.41399999999999998,"5min":0.39300000000000002,"15min":0.36899999999999999},"max":{"1min":8.1769999999999996,"5min":9.4000000000000004,"15min":9.4000000000000004},"last":0.64500000000000002}]},{"osd":7,"last update":"Sun Oct 1 11:09:12 2023","interfaces":[{"interface":"back","average":{"1min":1.7190000000000001,"5min":1.7450000000000001,"15min":1.4019999999999999},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":9.2599999999999998,"5min":10.851000000000001,"15min":10.851000000000001},"last":0.58899999999999997},{"interface":"front","average":{"1min":2.528,"5min":1.7709999999999999,"15min":1.4119999999999999},"min":{"1min":0.42799999999999999,"5min":0.41799999999999998,"15min":0.40300000000000002},"max":{"1min":12.364000000000001,"5min":12.364000000000001,"15min":12.364000000000001},"last":0.57799999999999996}]}]},{"osd":5,"up_from":36,"seq":154618823361,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2144268,"kb_used_data":959812,"kb_used_omap":0,"kb_used_meta":1184448,"kb_avail":91600884,"statfs":{"total":95995035648,"available":93799305216,"internally_reserved":0,"allocated":982847488,"data_stored":982288002,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1212874752},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 11:09:41 2023","interfaces":[{"interface":"back","average":{"1min":0.85399999999999998,"5min":0.99299999999999999,"15min":1.526},"min":{"1min":0.56899999999999995,"5min":0.29799999999999999,"15min":0.24299999999999999},"max":{"1min":1.2589999999999999,"5min":11.933999999999999,"15min":115.057},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.80900000000000005,"5min":1.3740000000000001,"15min":1.7030000000000001},"min":{"1min":0.47199999999999998,"5min":0.28799999999999998,"15min":0.253},"max":{"1min":1.107,"5min":8.0399999999999991,"15min":115.53},"last":0.34899999999999998}]},{"osd":1,"last update":"Sun Oct 1 11:09:41 2023","interfaces":[{"interface":"back","average":{"1min":0.84799999999999998,"5min":0.94399999999999995,"15min":0.97599999999999998},"min":{"1min":0.58599999999999997,"5min":0.38,"15min":0.29599999999999999},"max":{"1min":1.1679999999999999,"5min":11.141999999999999,"15min":11.141999999999999},"last":3.2469999999999999},{"interface":"front","average":{"1min":0.86199999999999999,"5min":1.4510000000000001,"15min":1.2569999999999999},"min":{"1min":0.53900000000000003,"5min":0.318,"15min":0.318},"max":{"1min":1.1659999999999999,"5min":8.2260000000000009,"15min":8.2260000000000009},"last":0.94699999999999995}]},{"osd":2,"last update":"Sun Oct 1 11:09:11 2023","interfaces":[{"interface":"back","average":{"1min":1.403,"5min":1.294,"15min":2.3319999999999999},"min":{"1min":0.38900000000000001,"5min":0.32800000000000001,"15min":0.311},"max":{"1min":11.18,"5min":21.507000000000001,"15min":207.059},"last":0.56100000000000005},{"interface":"front","average":{"1min":1.54,"5min":1.712,"15min":2.5910000000000002},"min":{"1min":0.317,"5min":0.317,"15min":0.29299999999999998},"max":{"1min":8.9030000000000005,"5min":21.693999999999999,"15min":206.673},"last":2.004}]},{"osd":3,"last update":"Sun Oct 1 11:09:41 2023","interfaces":[{"interface":"back","average":{"1min":0.92200000000000004,"5min":1.46,"15min":1.5069999999999999},"min":{"1min":0.66800000000000004,"5min":0.378,"15min":0.33800000000000002},"max":{"1min":1.2030000000000001,"5min":8.0999999999999996,"15min":13.186999999999999},"last":0.68899999999999995},{"interface":"front","average":{"1min":0.88,"5min":1.2849999999999999,"15min":1.3720000000000001},"min":{"1min":0.50900000000000001,"5min":0.372,"15min":0.32000000000000001},"max":{"1min":1.222,"5min":9.2439999999999998,"15min":11.026999999999999},"last":2.5219999999999998}]},{"osd":4,"last update":"Sun Oct 1 11:09:11 2023","interfaces":[{"interface":"back","average":{"1min":1.927,"5min":1.595,"15min":1.181},"min":{"1min":0.61599999999999999,"5min":0.39000000000000001,"15min":0.311},"max":{"1min":8.2040000000000006,"5min":9.2720000000000002,"15min":9.2720000000000002},"last":0.58699999999999997},{"interface":"front","average":{"1min":2.3820000000000001,"5min":1.575,"15min":1.204},"min":{"1min":0.501,"5min":0.34100000000000003,"15min":0.30399999999999999},"max":{"1min":11.162000000000001,"5min":11.162000000000001,"15min":11.162000000000001},"last":0.60099999999999998}]},{"osd":6,"last update":"Sun Oct 1 11:09:41 2023","interfaces":[{"interface":"back","average":{"1min":1.0349999999999999,"5min":1.254,"15min":1.2},"min":{"1min":0.629,"5min":0.36599999999999999,"15min":0.35199999999999998},"max":{"1min":1.4279999999999999,"5min":11.196999999999999,"15min":11.196999999999999},"last":0.67600000000000005},{"interface":"front","average":{"1min":0.99399999999999999,"5min":1.4319999999999999,"15min":1.2010000000000001},"min":{"1min":0.65000000000000002,"5min":0.35499999999999998,"15min":0.33000000000000002},"max":{"1min":1.1879999999999999,"5min":12.189,"15min":12.189},"last":0.51100000000000001}]},{"osd":7,"last update":"Sun Oct 1 11:09:41 2023","interfaces":[{"interface":"back","average":{"1min":1.0600000000000001,"5min":1.496,"15min":1.2589999999999999},"min":{"1min":0.56899999999999995,"5min":0.378,"15min":0.34899999999999998},"max":{"1min":1.577,"5min":9.1959999999999997,"15min":9.1959999999999997},"last":1.708},{"interface":"front","average":{"1min":1.0900000000000001,"5min":1.3129999999999999,"15min":1.25},"min":{"1min":0.51100000000000001,"5min":0.39900000000000002,"15min":0.34699999999999998},"max":{"1min":1.5469999999999999,"5min":11.215,"15min":11.215},"last":0.70999999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1356464128,"data_stored":1356464128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1373437952,"data_stored":1373437952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1147404288,"data_stored":1147404288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1236402176,"data_stored":1236402176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1014824960,"data_stored":1014824960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":842334208,"data_stored":842334208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":650248192,"data_stored":650248192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":490078208,"data_stored":490078208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T11:10:13.646 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-10-01T11:10:13.646 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd pool set device_health_metrics pg_num 31 2023-10-01T11:10:13.652 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:13.679 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:13.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:13.682 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:13.783 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:13.817 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:13.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:13.821 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:10:13.921 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:13.949 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:13.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:13.952 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:13.968 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:13 smithi078 bash[75299]: cluster 2023-10-01T11:10:12.392417+0000 mgr.y (mgr.25072) 1588 : cluster 0 pgmap v1709: 89 pgs: 89 active+clean; 2.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-10-01T11:10:14.053 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:14.084 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:14.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:14.087 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:10:14.188 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:14.213 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:14.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:14.285 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:10:14.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:13 smithi078 bash[78274]: cluster 2023-10-01T11:10:12.392417+0000 mgr.y (mgr.25072) 1588 : cluster 0 pgmap v1709: 89 pgs: 89 active+clean; 2.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-10-01T11:10:14.354 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 32 16 53426 53410 104.307 69.375 0.0143042 0.00958132 2023-10-01T11:10:14.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:13 smithi173 bash[58816]: cluster 2023-10-01T11:10:12.392417+0000 mgr.y (mgr.25072) 1588 : cluster 0 pgmap v1709: 89 pgs: 89 active+clean; 2.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-10-01T11:10:14.386 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:14.412 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:14.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:14.415 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:14.515 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:14.543 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:14.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:14.547 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:14.648 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:14.676 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:14.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:14.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:14.779 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:14.809 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:14.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:14.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:10:14.913 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:14.939 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:14.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:14.942 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:10:15.043 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:15.070 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:15.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:15.073 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:10:15.174 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:15.210 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:15.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:15.212 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:15.313 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:15.342 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:15.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:15.344 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:10:15.354 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 33 16 54625 54609 103.416 74.9375 0.00414004 0.00966241 2023-10-01T11:10:15.445 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:15.476 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:15.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:15.479 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:10:15.580 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:15.609 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T11:10:15.616 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:15.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:15.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:10:15.657 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T11:10:15.721 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:15.745 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:15.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:15.749 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:15.849 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:15.905 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:15.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:15.910 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:10:16.011 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:16.042 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:16.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:16.045 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:16.146 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:16.159 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:15 smithi078 bash[75299]: cluster 2023-10-01T11:10:14.393088+0000 mgr.y (mgr.25072) 1589 : cluster 0 pgmap v1710: 89 pgs: 89 active+clean; 3.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-10-01T11:10:16.160 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:15 smithi078 bash[78274]: cluster 2023-10-01T11:10:14.393088+0000 mgr.y (mgr.25072) 1589 : cluster 0 pgmap v1710: 89 pgs: 89 active+clean; 3.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-10-01T11:10:16.178 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:16.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:16.182 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:10:16.283 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:16.293 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:15 smithi173 bash[58816]: cluster 2023-10-01T11:10:14.393088+0000 mgr.y (mgr.25072) 1589 : cluster 0 pgmap v1710: 89 pgs: 89 active+clean; 3.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-10-01T11:10:16.322 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:16.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:16.331 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:16.354 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 34 16 55626 55610 102.215 62.5625 0.0234297 0.00977784 2023-10-01T11:10:16.432 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:16.465 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:16.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:16.470 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:16.571 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:16.602 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:16.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:16.605 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:10:16.706 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:16.736 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:16.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:16.743 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:10:16.844 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:16.876 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:16.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:16.879 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:16.980 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:17.020 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:17.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:17.023 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:10:17.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:16 smithi173 bash[58816]: audit 2023-10-01T11:10:16.237434+0000 mon.a (mon.0) 2057 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T11:10:17.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:16 smithi173 bash[58816]: audit 2023-10-01T11:10:16.716991+0000 mon.a (mon.0) 2058 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:10:17.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:16 smithi173 bash[58816]: audit 2023-10-01T11:10:16.737707+0000 mon.a (mon.0) 2059 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:10:17.124 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:17.152 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:17.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:17.156 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:17.257 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:17.289 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:17.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:17.293 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:10:17.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:16 smithi078 bash[75299]: audit 2023-10-01T11:10:16.237434+0000 mon.a (mon.0) 2057 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T11:10:17.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:16 smithi078 bash[75299]: audit 2023-10-01T11:10:16.716991+0000 mon.a (mon.0) 2058 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:10:17.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:16 smithi078 bash[75299]: audit 2023-10-01T11:10:16.737707+0000 mon.a (mon.0) 2059 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:10:17.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:16 smithi078 bash[78274]: audit 2023-10-01T11:10:16.237434+0000 mon.a (mon.0) 2057 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T11:10:17.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:16 smithi078 bash[78274]: audit 2023-10-01T11:10:16.716991+0000 mon.a (mon.0) 2058 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:10:17.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:17 smithi078 bash[78274]: audit 2023-10-01T11:10:16.737707+0000 mon.a (mon.0) 2059 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:10:17.354 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 35 16 56706 56690 101.223 67.5 0.012391 0.00987231 2023-10-01T11:10:17.394 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:17.419 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:17.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:17.423 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:17.524 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:17.546 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:17.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:17.551 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:10:17.652 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:17.682 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:17.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:17.684 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:17.784 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:17.813 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:17.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:17.818 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:17.918 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:17.947 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:17.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:17.950 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:18.050 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:18.087 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:10:18.091 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:10:18.098 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:18.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:18.107 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:10:18.208 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:18.217 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:18 smithi173 bash[58816]: cluster 2023-10-01T11:10:16.393784+0000 mgr.y (mgr.25072) 1590 : cluster 0 pgmap v1711: 89 pgs: 89 active+clean; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-10-01T11:10:18.218 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:18 smithi173 bash[58816]: audit 2023-10-01T11:10:17.471436+0000 mon.a (mon.0) 2060 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:10:18.218 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:18 smithi173 bash[58816]: audit 2023-10-01T11:10:17.474007+0000 mon.a (mon.0) 2061 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:10:18.236 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:18.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:18.239 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:10:18.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:18 smithi078 bash[78274]: cluster 2023-10-01T11:10:16.393784+0000 mgr.y (mgr.25072) 1590 : cluster 0 pgmap v1711: 89 pgs: 89 active+clean; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-10-01T11:10:18.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:18 smithi078 bash[78274]: audit 2023-10-01T11:10:17.471436+0000 mon.a (mon.0) 2060 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:10:18.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:18 smithi078 bash[78274]: audit 2023-10-01T11:10:17.474007+0000 mon.a (mon.0) 2061 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:10:18.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:17 smithi078 bash[75299]: cluster 2023-10-01T11:10:16.393784+0000 mgr.y (mgr.25072) 1590 : cluster 0 pgmap v1711: 89 pgs: 89 active+clean; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-10-01T11:10:18.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:18 smithi078 bash[75299]: audit 2023-10-01T11:10:17.471436+0000 mon.a (mon.0) 2060 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:10:18.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:18 smithi078 bash[75299]: audit 2023-10-01T11:10:17.474007+0000 mon.a (mon.0) 2061 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:10:18.340 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:18.356 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 36 16 57791 57775 100.289 67.8125 0.0090863 0.0099634 2023-10-01T11:10:18.368 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:18.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:18.371 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:10:18.472 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:18.496 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:18.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:18.499 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:10:18.600 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:18.628 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:18.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:18.631 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:10:18.732 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:18.761 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:18.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:18.763 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:10:18.863 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:18.899 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:18.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:18.902 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:10:19.003 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:19.031 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:19.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:19.035 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:19.136 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:19.165 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:19.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:19.176 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:10:19.277 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:19.309 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:19.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:19.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:19.357 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 37 16 58960 58944 99.5527 73.0625 0.0241689 0.0100397 2023-10-01T11:10:19.416 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:19.439 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:19.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:19.441 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:10:19.542 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:19.578 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:19.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:19.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:10:19.681 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:19.709 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:19.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:19.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:19.812 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:19.849 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:19.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:19.853 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:10:19.954 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:19.982 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:19.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:19.991 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:10:20.092 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:20.116 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:20.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:20.118 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:10:20.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:20 smithi173 bash[58816]: cluster 2023-10-01T11:10:18.397597+0000 mgr.y (mgr.25072) 1591 : cluster 0 pgmap v1712: 89 pgs: 89 active+clean; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2023-10-01T11:10:20.129 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:10:20.129 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:10:20.219 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:20.245 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:20.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:20.248 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:10:20.271 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:20 smithi078 bash[75299]: cluster 2023-10-01T11:10:18.397597+0000 mgr.y (mgr.25072) 1591 : cluster 0 pgmap v1712: 89 pgs: 89 active+clean; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2023-10-01T11:10:20.271 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:20 smithi078 bash[78274]: cluster 2023-10-01T11:10:18.397597+0000 mgr.y (mgr.25072) 1591 : cluster 0 pgmap v1712: 89 pgs: 89 active+clean; 3.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 75 MiB/s wr, 1.21k op/s 2023-10-01T11:10:20.349 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:20.356 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 38 16 59998 59982 98.6401 64.875 0.0188216 0.0101317 2023-10-01T11:10:20.375 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:20.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:20.379 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:20.480 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:20.505 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:20.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:20.509 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:10:20.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:10:20 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:10:20] "GET /metrics HTTP/1.1" 200 36382 "" "Prometheus/2.33.4" 2023-10-01T11:10:20.611 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:20.644 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:20.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:20.646 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:10:20.747 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:20.774 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:20.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:20.779 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:20.880 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:20.906 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:20.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:20.910 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:10:21.010 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:21.040 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:21.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:21.043 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:21.144 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:21.176 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:21.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:21.179 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:21.280 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:21.311 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:21.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:21.314 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:10:21.356 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 39 16 61140 61124 97.9408 71.375 0.00632772 0.0102029 2023-10-01T11:10:21.415 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:21.441 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:21.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:21.444 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:21.545 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:21.571 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:21.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:21.574 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:21.675 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:21.707 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:21.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:21.712 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:21.813 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:21.839 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:21.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:21.845 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:10:21.945 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:21.977 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:21.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:21.981 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:10:22.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:22 smithi078 bash[78274]: cluster 2023-10-01T11:10:20.398296+0000 mgr.y (mgr.25072) 1592 : cluster 0 pgmap v1713: 89 pgs: 89 active+clean; 3.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-10-01T11:10:22.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:22 smithi078 bash[75299]: cluster 2023-10-01T11:10:20.398296+0000 mgr.y (mgr.25072) 1592 : cluster 0 pgmap v1713: 89 pgs: 89 active+clean; 3.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-10-01T11:10:22.083 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:22.094 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:22 smithi173 bash[58816]: cluster 2023-10-01T11:10:20.398296+0000 mgr.y (mgr.25072) 1592 : cluster 0 pgmap v1713: 89 pgs: 89 active+clean; 3.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-10-01T11:10:22.120 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:22.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:22.125 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:10:22.225 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:22.273 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:22.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:22.303 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:10:22.357 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:2023-10-01T11:10:22.356595+0000 min lat: 0.00136692 max lat: 0.206832 avg lat: 0.0102961 2023-10-01T11:10:22.357 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T11:10:22.357 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 40 16 62151 62135 97.0719 63.1875 0.00882501 0.0102961 2023-10-01T11:10:22.403 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:22.433 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:22.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:22.435 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:10:22.536 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:22.568 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:22.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:22.574 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:10:22.675 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:22.714 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:22.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:22.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:10:22.817 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:22.858 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:22.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:22.866 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:10:22.967 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:22.998 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:23.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:23.005 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:10:23.106 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:23.130 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:23.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:23.132 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:23.233 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:23.263 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:23.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:23.267 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:10:23.357 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 41 16 63116 63100 96.1752 60.3125 0.00831808 0.0103914 2023-10-01T11:10:23.368 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:23.409 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:23.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:23.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:10:23.514 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:23.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:23.549 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:23.549 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:10:23.650 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:23.659 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:23 smithi173 bash[58816]: cluster 2023-10-01T11:10:22.398820+0000 mgr.y (mgr.25072) 1593 : cluster 0 pgmap v1714: 89 pgs: 89 active+clean; 3.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-01T11:10:23.659 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:23 smithi173 bash[58816]: audit 2023-10-01T11:10:22.601762+0000 mon.a (mon.0) 2062 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:10:23.659 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:23 smithi173 bash[58816]: audit 2023-10-01T11:10:22.611079+0000 mon.a (mon.0) 2063 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:10:23.659 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:23 smithi173 bash[58816]: audit 2023-10-01T11:10:23.315236+0000 mon.a (mon.0) 2064 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:10:23.659 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:23 smithi173 bash[58816]: audit 2023-10-01T11:10:23.323796+0000 mon.a (mon.0) 2065 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:10:23.687 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:23.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:23.689 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:23.789 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:23.813 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:23.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:23.815 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:23.847 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:23 smithi078 bash[75299]: cluster 2023-10-01T11:10:22.398820+0000 mgr.y (mgr.25072) 1593 : cluster 0 pgmap v1714: 89 pgs: 89 active+clean; 3.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-01T11:10:23.847 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:23 smithi078 bash[75299]: audit 2023-10-01T11:10:22.601762+0000 mon.a (mon.0) 2062 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:10:23.847 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:23 smithi078 bash[75299]: audit 2023-10-01T11:10:22.611079+0000 mon.a (mon.0) 2063 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:10:23.847 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:23 smithi078 bash[75299]: audit 2023-10-01T11:10:23.315236+0000 mon.a (mon.0) 2064 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:10:23.847 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:23 smithi078 bash[75299]: audit 2023-10-01T11:10:23.323796+0000 mon.a (mon.0) 2065 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:10:23.847 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:23 smithi078 bash[78274]: cluster 2023-10-01T11:10:22.398820+0000 mgr.y (mgr.25072) 1593 : cluster 0 pgmap v1714: 89 pgs: 89 active+clean; 3.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-10-01T11:10:23.848 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:23 smithi078 bash[78274]: audit 2023-10-01T11:10:22.601762+0000 mon.a (mon.0) 2062 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:10:23.848 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:23 smithi078 bash[78274]: audit 2023-10-01T11:10:22.611079+0000 mon.a (mon.0) 2063 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:10:23.848 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:23 smithi078 bash[78274]: audit 2023-10-01T11:10:23.315236+0000 mon.a (mon.0) 2064 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:10:23.848 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:23 smithi078 bash[78274]: audit 2023-10-01T11:10:23.323796+0000 mon.a (mon.0) 2065 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:10:23.916 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:23.943 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:23.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:23.946 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:10:24.047 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:24.077 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:24.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:24.082 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:24.182 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:24.206 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:24.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:24.210 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:24.311 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:24.345 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:24.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:24.349 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:24.357 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 42 16 64068 64052 95.3019 59.5 0.0186434 0.0104871 2023-10-01T11:10:24.450 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:24.480 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:24.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:24.482 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:10:24.583 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:24.604 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:24 smithi173 bash[58816]: audit 2023-10-01T11:10:24.050285+0000 mon.a (mon.0) 2066 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T11:10:24.604 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:24 smithi173 bash[58816]: audit 2023-10-01T11:10:24.053232+0000 mon.a (mon.0) 2067 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T11:10:24.604 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:24 smithi173 bash[58816]: audit 2023-10-01T11:10:24.064764+0000 mon.a (mon.0) 2068 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:10:24.607 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:24.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:24.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:10:24.711 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:24.733 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:24.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:24.739 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:10:24.758 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:24 smithi078 bash[75299]: audit 2023-10-01T11:10:24.050285+0000 mon.a (mon.0) 2066 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T11:10:24.758 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:24 smithi078 bash[75299]: audit 2023-10-01T11:10:24.053232+0000 mon.a (mon.0) 2067 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T11:10:24.758 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:24 smithi078 bash[75299]: audit 2023-10-01T11:10:24.064764+0000 mon.a (mon.0) 2068 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:10:24.759 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:24 smithi078 bash[78274]: audit 2023-10-01T11:10:24.050285+0000 mon.a (mon.0) 2066 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T11:10:24.759 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:24 smithi078 bash[78274]: audit 2023-10-01T11:10:24.053232+0000 mon.a (mon.0) 2067 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T11:10:24.759 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:24 smithi078 bash[78274]: audit 2023-10-01T11:10:24.064764+0000 mon.a (mon.0) 2068 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:10:24.839 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:24.863 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:24.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:24.865 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:24.966 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:24.985 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:24.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:24.987 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:10:25.089 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:25.116 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:25.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:25.122 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:10:25.222 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:25.253 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:25.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:25.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:25.357 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 43 16 65054 65038 94.5186 61.625 0.0061367 0.010574 2023-10-01T11:10:25.359 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:25.393 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:25.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:25.397 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:25.498 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:25.523 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:25.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:25.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:25.627 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:25.662 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:25.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:25.665 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:25.766 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:25.802 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:25.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:25.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:10:25.908 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:25.934 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:25.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:25.937 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:10:26.037 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:26.070 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:26.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:26.072 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:10:26.173 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:26.205 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:26.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:26.212 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:10:26.313 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:26.321 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:26 smithi173 bash[58816]: cluster 2023-10-01T11:10:24.399509+0000 mgr.y (mgr.25072) 1594 : cluster 0 pgmap v1715: 89 pgs: 89 active+clean; 3.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s 2023-10-01T11:10:26.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:26 smithi078 bash[78274]: cluster 2023-10-01T11:10:24.399509+0000 mgr.y (mgr.25072) 1594 : cluster 0 pgmap v1715: 89 pgs: 89 active+clean; 3.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s 2023-10-01T11:10:26.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:26 smithi078 bash[75299]: cluster 2023-10-01T11:10:24.399509+0000 mgr.y (mgr.25072) 1594 : cluster 0 pgmap v1715: 89 pgs: 89 active+clean; 3.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s 2023-10-01T11:10:26.339 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:26.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:26.344 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:26.357 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 44 16 66183 66167 93.974 70.5625 0.0248446 0.0106349 2023-10-01T11:10:26.445 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:26.476 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:26.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:26.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:10:26.580 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:26.609 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:26.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:26.611 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:10:26.712 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:26.734 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:26.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:26.738 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:10:26.838 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:26.863 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:26.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:26.869 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:26.970 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:26.994 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:26.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:26.998 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:10:27.099 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:27.124 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:27.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:27.127 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:10:27.228 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:27.257 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:27.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:27.260 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:10:27.357 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 45 16 67305 67289 93.4439 70.125 0.00788213 0.0106958 2023-10-01T11:10:27.361 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:27.394 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:27.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:27.397 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:10:27.498 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:27.528 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:27.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:27.530 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:10:27.631 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:27.652 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:27.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:27.654 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:10:27.755 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:27.791 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:27.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:27.795 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:10:27.895 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:27.927 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:27.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:27.930 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:10:28.030 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:28.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:28.065 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:28.065 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:10:28.075 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:28 smithi173 bash[58816]: cluster 2023-10-01T11:10:26.400044+0000 mgr.y (mgr.25072) 1595 : cluster 0 pgmap v1716: 89 pgs: 89 active+clean; 3.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-01T11:10:28.166 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:28.191 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:28.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:28.197 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:10:28.298 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:28.324 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:28.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:28.327 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:28.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:28 smithi078 bash[78274]: cluster 2023-10-01T11:10:26.400044+0000 mgr.y (mgr.25072) 1595 : cluster 0 pgmap v1716: 89 pgs: 89 active+clean; 3.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-01T11:10:28.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:28 smithi078 bash[75299]: cluster 2023-10-01T11:10:26.400044+0000 mgr.y (mgr.25072) 1595 : cluster 0 pgmap v1716: 89 pgs: 89 active+clean; 3.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-10-01T11:10:28.357 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 46 16 68398 68382 92.8975 68.3125 0.0277499 0.0107577 2023-10-01T11:10:28.428 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:28.457 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:28.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:28.460 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:10:28.561 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:28.590 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:28.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:28.595 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:10:28.696 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:28.727 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:28.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:28.732 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:10:28.833 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:28.881 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:28.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:28.884 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:10:28.985 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:29.011 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:29.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:29.015 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:29.116 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:29.147 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:29.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:29.154 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:29.255 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:29.272 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:29.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:29.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:10:29.357 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 47 16 69418 69402 92.2772 63.75 0.0102295 0.0108309 2023-10-01T11:10:29.380 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:29.407 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:29.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:29.409 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:10:29.510 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:29.543 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:29.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:29.548 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:10:29.649 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:29.687 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:29.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:29.692 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:10:29.793 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:29.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:29.824 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:29.824 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:10:29.925 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:29.957 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:29.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:29.960 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:10:30.060 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:30.091 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:30.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:30.093 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:10:30.194 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:30.221 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:30.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:30.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:30.269 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:30 smithi078 bash[75299]: cluster 2023-10-01T11:10:28.400465+0000 mgr.y (mgr.25072) 1596 : cluster 0 pgmap v1717: 89 pgs: 89 active+clean; 3.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 56 MiB/s wr, 891 op/s 2023-10-01T11:10:30.270 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:10:30 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:10:30] "GET /metrics HTTP/1.1" 200 36382 "" "Prometheus/2.33.4" 2023-10-01T11:10:30.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:30 smithi078 bash[78274]: cluster 2023-10-01T11:10:28.400465+0000 mgr.y (mgr.25072) 1596 : cluster 0 pgmap v1717: 89 pgs: 89 active+clean; 3.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 56 MiB/s wr, 891 op/s 2023-10-01T11:10:30.324 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:30.358 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 48 16 70501 70485 91.764 67.6875 0.00854401 0.0108916 2023-10-01T11:10:30.361 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:30.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:30.366 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:10:30.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:30 smithi173 bash[58816]: cluster 2023-10-01T11:10:28.400465+0000 mgr.y (mgr.25072) 1596 : cluster 0 pgmap v1717: 89 pgs: 89 active+clean; 3.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 56 MiB/s wr, 891 op/s 2023-10-01T11:10:30.467 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:30.500 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:30.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:30.504 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:10:30.605 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:30.634 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:30.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:30.639 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:10:30.740 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:30.768 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:30.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:30.773 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:30.873 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:30.902 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:30.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:30.903 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:31.004 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:31.031 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:31.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:31.034 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:10:31.135 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:31.166 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:31.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:31.169 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:10:31.270 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:31.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:31.298 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:31.298 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:10:31.358 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 49 16 71503 71487 91.1692 62.625 0.0118231 0.0109625 2023-10-01T11:10:31.399 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:31.427 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:31.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:31.429 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:10:31.530 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:31.557 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:31.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:31.561 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:10:31.662 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:31.691 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:31.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:31.694 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:10:31.795 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:31.823 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:31.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:31.829 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:10:31.930 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:31.963 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:31.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:31.967 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:10:32.068 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:32.101 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:32.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:32.104 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:32.205 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:32.214 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:32 smithi078 bash[78274]: cluster 2023-10-01T11:10:30.401125+0000 mgr.y (mgr.25072) 1597 : cluster 0 pgmap v1718: 89 pgs: 89 active+clean; 4.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-10-01T11:10:32.214 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:32 smithi078 bash[75299]: cluster 2023-10-01T11:10:30.401125+0000 mgr.y (mgr.25072) 1597 : cluster 0 pgmap v1718: 89 pgs: 89 active+clean; 4.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-10-01T11:10:32.234 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:32.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:32.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:10:32.337 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:32.358 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 50 16 72473 72457 90.558 60.625 0.00425161 0.0110359 2023-10-01T11:10:32.367 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:32.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:32.369 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:10:32.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:32 smithi173 bash[58816]: cluster 2023-10-01T11:10:30.401125+0000 mgr.y (mgr.25072) 1597 : cluster 0 pgmap v1718: 89 pgs: 89 active+clean; 4.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-10-01T11:10:32.470 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:32.497 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:32.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:32.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:10:32.603 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:32.629 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:32.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:32.632 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:10:32.733 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:32.761 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:32.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:32.764 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:10:32.865 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:32.890 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:32.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:32.893 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:32.994 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:33.012 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:33.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:33.014 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:10:33.115 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:33.147 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:33.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:33.152 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:10:33.253 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:33.263 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:33 smithi078 bash[75299]: audit 2023-10-01T11:10:32.466285+0000 mon.a (mon.0) 2069 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:10:33.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:33 smithi078 bash[78274]: audit 2023-10-01T11:10:32.466285+0000 mon.a (mon.0) 2069 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:10:33.288 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:33.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:33.293 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:10:33.358 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 51 16 73315 73299 89.8142 52.625 0.0140357 0.0111287 2023-10-01T11:10:33.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:33 smithi173 bash[58816]: audit 2023-10-01T11:10:32.466285+0000 mon.a (mon.0) 2069 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:10:33.394 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:33.425 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:33.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:33.431 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:10:33.531 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:33.564 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:33.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:33.570 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:10:33.670 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:33.701 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:33.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:33.706 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:10:33.806 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:33.830 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:33.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:33.838 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:10:33.938 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:33.963 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:33.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:33.968 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:10:34.069 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:34.101 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:34.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:34.106 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:10:34.207 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:34.232 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:34.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:34.235 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:34.336 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:34.346 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:34 smithi173 bash[58816]: cluster 2023-10-01T11:10:32.401636+0000 mgr.y (mgr.25072) 1598 : cluster 0 pgmap v1719: 89 pgs: 89 active+clean; 4.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 60 MiB/s wr, 959 op/s 2023-10-01T11:10:34.358 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 52 16 74276 74260 89.242 60.0625 0.0236515 0.0111977 2023-10-01T11:10:34.360 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:34 smithi078 bash[78274]: cluster 2023-10-01T11:10:32.401636+0000 mgr.y (mgr.25072) 1598 : cluster 0 pgmap v1719: 89 pgs: 89 active+clean; 4.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 60 MiB/s wr, 959 op/s 2023-10-01T11:10:34.360 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:34.360 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:34 smithi078 bash[75299]: cluster 2023-10-01T11:10:32.401636+0000 mgr.y (mgr.25072) 1598 : cluster 0 pgmap v1719: 89 pgs: 89 active+clean; 4.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 60 MiB/s wr, 959 op/s 2023-10-01T11:10:34.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:34.362 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:10:34.462 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:34.486 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:34.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:34.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:34.591 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:34.616 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:34.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:34.618 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:34.712 INFO:teuthology.orchestra.run.smithi078.stdout:ERROR: (22) Invalid argument 2023-10-01T11:10:34.712 INFO:teuthology.orchestra.run.smithi078.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T11:10:34.719 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:34.752 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T11:10:34.752 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:34.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:34.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:10:34.797 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T11:10:34.847 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T11:10:34.857 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:34.884 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:34.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:34.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:34.901 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T11:10:34.945 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T11:10:34.988 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:34.991 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T11:10:35.020 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:35.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:35.023 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:35.037 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T11:10:35.078 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T11:10:35.084 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T11:10:35.085 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T11:10:35.085 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T11:10:35.085 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T11:10:35.085 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T11:10:35.085 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T11:10:35.085 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T11:10:35.085 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T11:10:35.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:35 smithi173 bash[58816]: audit 2023-10-01T11:10:34.635848+0000 mon.b (mon.1) 58 : audit 0 from='client.? 172.21.15.78:0/3790424113' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:10:35.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:35 smithi173 bash[58816]: audit 2023-10-01T11:10:35.036114+0000 mon.a (mon.0) 2070 : audit 1 from='client.? 172.21.15.78:0/2140516341' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T11:10:35.124 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:35.133 INFO:teuthology.orchestra.run.smithi078.stderr:noscrub is set 2023-10-01T11:10:35.155 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:35.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:35.159 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:35.164 INFO:teuthology.orchestra.run.smithi078.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-10-01T11:10:35.194 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:35 smithi078 bash[78274]: audit 2023-10-01T11:10:34.635848+0000 mon.b (mon.1) 58 : audit 0 from='client.? 172.21.15.78:0/3790424113' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:10:35.194 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:35 smithi078 bash[78274]: audit 2023-10-01T11:10:35.036114+0000 mon.a (mon.0) 2070 : audit 1 from='client.? 172.21.15.78:0/2140516341' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T11:10:35.196 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:35 smithi078 bash[75299]: audit 2023-10-01T11:10:34.635848+0000 mon.b (mon.1) 58 : audit 0 from='client.? 172.21.15.78:0/3790424113' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:10:35.196 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:35 smithi078 bash[75299]: audit 2023-10-01T11:10:35.036114+0000 mon.a (mon.0) 2070 : audit 1 from='client.? 172.21.15.78:0/2140516341' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T11:10:35.260 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:35.302 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:35.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:35.305 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:10:35.358 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 53 16 74984 74968 88.393 44.25 0.0109554 0.0113069 2023-10-01T11:10:35.406 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:35.432 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:35.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:35.434 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:10:35.535 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:35.558 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:35.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:35.563 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:10:35.663 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:35.688 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:35.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:35.691 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:35.792 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:35.824 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:35.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:35.826 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:10:35.927 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:35.956 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:35.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:35.959 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:10:36.059 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:36.094 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:36.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:36.098 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:10:36.118 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:36 smithi173 bash[58816]: cluster 2023-10-01T11:10:34.402302+0000 mgr.y (mgr.25072) 1599 : cluster 0 pgmap v1720: 89 pgs: 89 active+clean; 4.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2023-10-01T11:10:36.118 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:36 smithi173 bash[58816]: cluster 2023-10-01T11:10:35.101801+0000 mon.a (mon.0) 2071 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:10:36.118 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:36 smithi173 bash[58816]: cluster 2023-10-01T11:10:35.116925+0000 osd.7 (osd.7) 405 : cluster 3 failed to encode map e361 with expected crc 2023-10-01T11:10:36.118 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:36 smithi173 bash[58816]: cluster 2023-10-01T11:10:35.117327+0000 osd.5 (osd.5) 383 : cluster 3 failed to encode map e361 with expected crc 2023-10-01T11:10:36.118 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:36 smithi173 bash[58816]: audit 2023-10-01T11:10:35.121608+0000 mon.a (mon.0) 2072 : audit 1 from='client.? 172.21.15.78:0/2140516341' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T11:10:36.118 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:36 smithi173 bash[58816]: cluster 2023-10-01T11:10:35.121863+0000 mon.a (mon.0) 2073 : cluster 0 osdmap e361: 8 total, 8 up, 8 in 2023-10-01T11:10:36.118 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:36 smithi173 bash[58816]: audit 2023-10-01T11:10:35.160933+0000 mon.a (mon.0) 2074 : audit 1 from='client.? 172.21.15.78:0/3350572709' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "31"}]: dispatch 2023-10-01T11:10:36.199 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:36.238 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:36.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:36.240 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:10:36.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:36 smithi078 bash[78274]: cluster 2023-10-01T11:10:34.402302+0000 mgr.y (mgr.25072) 1599 : cluster 0 pgmap v1720: 89 pgs: 89 active+clean; 4.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2023-10-01T11:10:36.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:36 smithi078 bash[78274]: cluster 2023-10-01T11:10:35.101801+0000 mon.a (mon.0) 2071 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:10:36.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:36 smithi078 bash[78274]: cluster 2023-10-01T11:10:35.116925+0000 osd.7 (osd.7) 405 : cluster 3 failed to encode map e361 with expected crc 2023-10-01T11:10:36.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:36 smithi078 bash[78274]: cluster 2023-10-01T11:10:35.117327+0000 osd.5 (osd.5) 383 : cluster 3 failed to encode map e361 with expected crc 2023-10-01T11:10:36.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:36 smithi078 bash[78274]: audit 2023-10-01T11:10:35.121608+0000 mon.a (mon.0) 2072 : audit 1 from='client.? 172.21.15.78:0/2140516341' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T11:10:36.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:36 smithi078 bash[78274]: cluster 2023-10-01T11:10:35.121863+0000 mon.a (mon.0) 2073 : cluster 0 osdmap e361: 8 total, 8 up, 8 in 2023-10-01T11:10:36.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:36 smithi078 bash[78274]: audit 2023-10-01T11:10:35.160933+0000 mon.a (mon.0) 2074 : audit 1 from='client.? 172.21.15.78:0/3350572709' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "31"}]: dispatch 2023-10-01T11:10:36.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:36 smithi078 bash[75299]: cluster 2023-10-01T11:10:34.402302+0000 mgr.y (mgr.25072) 1599 : cluster 0 pgmap v1720: 89 pgs: 89 active+clean; 4.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2023-10-01T11:10:36.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:36 smithi078 bash[75299]: cluster 2023-10-01T11:10:35.101801+0000 mon.a (mon.0) 2071 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:10:36.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:36 smithi078 bash[75299]: cluster 2023-10-01T11:10:35.116925+0000 osd.7 (osd.7) 405 : cluster 3 failed to encode map e361 with expected crc 2023-10-01T11:10:36.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:36 smithi078 bash[75299]: cluster 2023-10-01T11:10:35.117327+0000 osd.5 (osd.5) 383 : cluster 3 failed to encode map e361 with expected crc 2023-10-01T11:10:36.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:36 smithi078 bash[75299]: audit 2023-10-01T11:10:35.121608+0000 mon.a (mon.0) 2072 : audit 1 from='client.? 172.21.15.78:0/2140516341' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T11:10:36.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:36 smithi078 bash[75299]: cluster 2023-10-01T11:10:35.121863+0000 mon.a (mon.0) 2073 : cluster 0 osdmap e361: 8 total, 8 up, 8 in 2023-10-01T11:10:36.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:36 smithi078 bash[75299]: audit 2023-10-01T11:10:35.160933+0000 mon.a (mon.0) 2074 : audit 1 from='client.? 172.21.15.78:0/3350572709' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pg_num", "val": "31"}]: dispatch 2023-10-01T11:10:36.341 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:36.358 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 54 16 75994 75978 87.925 63.125 0.0136481 0.0113674 2023-10-01T11:10:36.368 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:36.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:36.371 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:36.472 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:36.497 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:36.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:36.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:36.600 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:36.624 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:36.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:36.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:10:36.726 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:36.753 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:36.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:36.760 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:10:36.860 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:36.890 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:36.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:36.898 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:10:36.999 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:37.033 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:37.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:37.036 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:10:37.136 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:37.145 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:37 smithi078 bash[78274]: cluster 2023-10-01T11:10:36.402799+0000 mgr.y (mgr.25072) 1600 : cluster 0 pgmap v1722: 89 pgs: 89 active+clean; 4.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-01T11:10:37.147 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:37 smithi078 bash[75299]: cluster 2023-10-01T11:10:36.402799+0000 mgr.y (mgr.25072) 1600 : cluster 0 pgmap v1722: 89 pgs: 89 active+clean; 4.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-01T11:10:37.178 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:37.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:37.185 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:10:37.285 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:37.302 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:37.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:37.303 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:10:37.359 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 55 16 76750 76734 87.1854 47.25 0.0125007 0.0114641 2023-10-01T11:10:37.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:37 smithi173 bash[58816]: cluster 2023-10-01T11:10:36.402799+0000 mgr.y (mgr.25072) 1600 : cluster 0 pgmap v1722: 89 pgs: 89 active+clean; 4.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-01T11:10:37.404 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:37.434 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:37.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:37.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:10:37.519 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-01T11:10:37.537 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:37.568 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:37.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:37.571 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:37.672 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:37.708 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:37.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:37.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:10:37.811 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:37.842 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:37.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:37.846 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:37.946 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:37.983 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:37.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:37.986 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:10:38.086 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:38.120 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:38.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:38.126 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:38.227 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:38.255 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:38.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:38.261 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:38.359 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 56 16 77679 77663 86.6653 58.0625 0.010396 0.0115324 2023-10-01T11:10:38.361 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:38.392 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:38.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:38.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:10:38.495 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:38.520 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:38.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:38.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:10:38.626 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:38.659 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:38.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:38.662 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:10:38.762 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:38.793 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:38.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:38.795 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:10:38.896 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:38.939 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:38.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:38.947 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:10:39.047 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:39.072 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:39.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:39.076 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:10:39.176 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:39.211 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:39.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:39.216 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:10:39.316 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:39.351 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:39.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:39.357 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:10:39.359 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 57 16 78342 78326 85.8718 41.4375 0.0146944 0.0116396 2023-10-01T11:10:39.457 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:39.485 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:39.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:39.488 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:39.510 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-10-01T11:10:39.588 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:39.627 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:39.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:39.630 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:10:39.731 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:39.758 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:39.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:39.761 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:10:39.862 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:39.897 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:39.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:39.899 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:10:40.000 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:40.008 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:39 smithi078 bash[75299]: cluster 2023-10-01T11:10:38.403219+0000 mgr.y (mgr.25072) 1601 : cluster 0 pgmap v1723: 89 pgs: 89 active+clean; 4.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-01T11:10:40.009 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:39 smithi078 bash[78274]: cluster 2023-10-01T11:10:38.403219+0000 mgr.y (mgr.25072) 1601 : cluster 0 pgmap v1723: 89 pgs: 89 active+clean; 4.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-01T11:10:40.028 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:40.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:40.036 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:10:40.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:39 smithi173 bash[58816]: cluster 2023-10-01T11:10:38.403219+0000 mgr.y (mgr.25072) 1601 : cluster 0 pgmap v1723: 89 pgs: 89 active+clean; 4.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-01T11:10:40.136 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:40.161 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:40.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:40.167 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:40.268 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:40.270 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:10:40 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:10:40] "GET /metrics HTTP/1.1" 200 36377 "" "Prometheus/2.33.4" 2023-10-01T11:10:40.305 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:40.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:40.312 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:10:40.359 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 58 16 79255 79239 85.375 57.0625 0.0158792 0.0117057 2023-10-01T11:10:40.413 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:40.440 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:40.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:40.444 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:10:40.545 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:40.575 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:40.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:40.579 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:40.680 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:40.705 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:40.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:40.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:40.809 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:40.837 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:40.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:40.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:40.942 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:40.978 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:40.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:40.983 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:10:41.084 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:41.110 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:41.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:41.115 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:41.216 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:41.243 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:41.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:41.248 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:10:41.349 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:41.360 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 59 16 80175 80159 84.9015 57.5 0.011314 0.0117707 2023-10-01T11:10:41.392 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:41.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:41.395 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:10:41.473 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T11:10:41.496 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:41.498 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T11:10:41.560 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:41.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:41.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:10:41.666 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:41.697 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:41.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:41.703 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:10:41.804 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:41.835 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:41.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:41.842 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:10:41.943 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:41.952 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:41 smithi173 bash[58816]: cluster 2023-10-01T11:10:40.403959+0000 mgr.y (mgr.25072) 1602 : cluster 0 pgmap v1724: 89 pgs: 89 active+clean; 4.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 58 MiB/s wr, 924 op/s 2023-10-01T11:10:41.972 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:41.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:41.979 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:42.079 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:42.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:41 smithi078 bash[78274]: cluster 2023-10-01T11:10:40.403959+0000 mgr.y (mgr.25072) 1602 : cluster 0 pgmap v1724: 89 pgs: 89 active+clean; 4.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 58 MiB/s wr, 924 op/s 2023-10-01T11:10:42.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:41 smithi078 bash[75299]: cluster 2023-10-01T11:10:40.403959+0000 mgr.y (mgr.25072) 1602 : cluster 0 pgmap v1724: 89 pgs: 89 active+clean; 4.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 58 MiB/s wr, 924 op/s 2023-10-01T11:10:42.105 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:42.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:42.107 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:42.208 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:42.237 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:42.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:42.240 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:10:42.341 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:42.450 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:2023-10-01T11:10:42.359722+0000 min lat: 0.00136692 max lat: 0.367786 avg lat: 0.0118483 2023-10-01T11:10:42.450 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T11:10:42.450 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 60 16 81008 80992 84.3541 52.0625 0.0102025 0.0118483 2023-10-01T11:10:42.450 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:10:42.483 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:42.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:42.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:42.730 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:42.754 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:42.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:42.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:10:42.862 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:42.887 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:42.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:42.893 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:42.993 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:43.023 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:43.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:43.381 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:10:43.381 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 61 16 81915 81899 83.9005 56.6875 0.00909225 0.0119132 2023-10-01T11:10:43.482 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:43.511 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:43.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:43.514 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:43.615 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:43.648 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:43.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:43.649 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:10:43.750 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:43.782 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:43.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:43.785 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:10:43.845 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:43 smithi173 bash[58816]: cluster 2023-10-01T11:10:42.405026+0000 mgr.y (mgr.25072) 1603 : cluster 0 pgmap v1725: 89 pgs: 89 active+clean; 4.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 905 op/s 2023-10-01T11:10:43.886 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:43.912 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:43.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:43.915 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:10:44.016 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:44.050 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:44.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:44.059 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:10:44.160 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:44.192 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:44.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:44.196 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:10:44.297 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:44.306 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:44 smithi078 bash[75299]: cluster 2023-10-01T11:10:42.405026+0000 mgr.y (mgr.25072) 1603 : cluster 0 pgmap v1725: 89 pgs: 89 active+clean; 4.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 905 op/s 2023-10-01T11:10:44.307 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:44 smithi078 bash[78274]: cluster 2023-10-01T11:10:42.405026+0000 mgr.y (mgr.25072) 1603 : cluster 0 pgmap v1725: 89 pgs: 89 active+clean; 4.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 905 op/s 2023-10-01T11:10:44.346 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:44.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:44.351 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:10:44.360 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 62 16 82813 82797 83.4525 56.125 0.0188059 0.0119768 2023-10-01T11:10:44.453 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:44.480 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:44.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:44.482 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:10:44.512 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T11:10:44.512 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-01T11:10:44.583 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:44.606 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:44.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:44.611 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:10:44.712 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:44.738 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:44.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:44.741 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:10:44.842 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:44.868 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:44.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:44.870 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:10:44.973 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:45.000 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:45.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:45.005 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:10:45.106 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:45.135 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:45.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:45.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:10:45.241 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:45.269 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:45.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:45.271 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:10:45.360 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 63 16 83714 83698 83.0216 56.3125 0.0159606 0.0120392 2023-10-01T11:10:45.372 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:45.398 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:45.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:45.402 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:10:45.423 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:10:45.433 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:10:45.503 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:45.532 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:45.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:45.536 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:10:45.637 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:45.666 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:45.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:45.672 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:10:45.773 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:45.797 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:45.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:45.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:10:45.900 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:45.934 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:45.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:45.936 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:46.037 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:46.045 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:45 smithi078 bash[78274]: cluster 2023-10-01T11:10:44.405677+0000 mgr.y (mgr.25072) 1604 : cluster 0 pgmap v1726: 89 pgs: 89 active+clean; 5.0 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 54 MiB/s wr, 867 op/s 2023-10-01T11:10:46.045 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:45 smithi078 bash[75299]: cluster 2023-10-01T11:10:44.405677+0000 mgr.y (mgr.25072) 1604 : cluster 0 pgmap v1726: 89 pgs: 89 active+clean; 5.0 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 54 MiB/s wr, 867 op/s 2023-10-01T11:10:46.065 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:46.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:46.068 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:10:46.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:45 smithi173 bash[58816]: cluster 2023-10-01T11:10:44.405677+0000 mgr.y (mgr.25072) 1604 : cluster 0 pgmap v1726: 89 pgs: 89 active+clean; 5.0 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 54 MiB/s wr, 867 op/s 2023-10-01T11:10:46.169 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:46.192 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:46.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:46.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:10:46.295 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:46.329 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:46.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:46.332 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:10:46.360 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 64 16 84507 84491 82.4987 49.5625 0.00394892 0.0120932 2023-10-01T11:10:46.433 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:46.459 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:46.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:46.464 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:10:46.564 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:46.593 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:46.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:46.597 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:46.698 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:46.720 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:46.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:46.723 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:10:46.823 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:46.855 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:46.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:46.857 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:46.957 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:46.982 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:46.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:46.987 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:10:47.088 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:47.115 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:47.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:47.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:10:47.220 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:47.247 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:47.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:47.250 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:10:47.351 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:47.361 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 65 16 85450 85434 82.1348 58.9375 0.0157496 0.0121687 2023-10-01T11:10:47.375 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:47.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:47.380 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:10:47.481 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:47.511 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:47.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:47.515 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:10:47.616 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:47.642 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:47.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:47.811 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:47.856 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:47 smithi173 bash[58816]: cluster 2023-10-01T11:10:46.406249+0000 mgr.y (mgr.25072) 1605 : cluster 0 pgmap v1727: 89 pgs: 89 active+clean; 5.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 52 MiB/s wr, 836 op/s 2023-10-01T11:10:47.856 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:47 smithi173 bash[58816]: audit 2023-10-01T11:10:47.472690+0000 mon.a (mon.0) 2075 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:10:47.856 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:47 smithi173 bash[58816]: audit 2023-10-01T11:10:47.473811+0000 mon.a (mon.0) 2076 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:10:47.912 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:47.941 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:47.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:47.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:48.043 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:48.081 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:48.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:48.084 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:10:48.184 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:48.212 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:48.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:48.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:10:48.315 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:48.324 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:47 smithi078 bash[78274]: cluster 2023-10-01T11:10:46.406249+0000 mgr.y (mgr.25072) 1605 : cluster 0 pgmap v1727: 89 pgs: 89 active+clean; 5.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 52 MiB/s wr, 836 op/s 2023-10-01T11:10:48.324 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:47 smithi078 bash[78274]: audit 2023-10-01T11:10:47.472690+0000 mon.a (mon.0) 2075 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:10:48.324 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:47 smithi078 bash[78274]: audit 2023-10-01T11:10:47.473811+0000 mon.a (mon.0) 2076 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:10:48.325 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:47 smithi078 bash[75299]: cluster 2023-10-01T11:10:46.406249+0000 mgr.y (mgr.25072) 1605 : cluster 0 pgmap v1727: 89 pgs: 89 active+clean; 5.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 52 MiB/s wr, 836 op/s 2023-10-01T11:10:48.325 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:47 smithi078 bash[75299]: audit 2023-10-01T11:10:47.472690+0000 mon.a (mon.0) 2075 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:10:48.325 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:47 smithi078 bash[75299]: audit 2023-10-01T11:10:47.473811+0000 mon.a (mon.0) 2076 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:10:48.343 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:48.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:48.346 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:10:48.361 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 66 16 86314 86298 81.7085 54 0.0135236 0.0122316 2023-10-01T11:10:48.447 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:48.474 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:48.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:48.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:10:48.580 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:48.615 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:48.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:48.617 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:48.718 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:48.745 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:48.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:48.750 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:10:48.851 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:48.879 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:48.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:48.884 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:10:48.985 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:49.019 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:49.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:49.023 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:10:49.124 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:49.153 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:49.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:49.156 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:49.257 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:49.280 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:49.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:49.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:10:49.361 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 67 16 87180 87164 81.2967 54.125 0.0121568 0.0122933 2023-10-01T11:10:49.384 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:49.416 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:49.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:49.418 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:49.519 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:49.545 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:49.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:49.548 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:49.648 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:49.687 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:49.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:49.690 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:49.791 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:49.817 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:49.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:49.823 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:49.924 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:49.933 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:49 smithi173 bash[58816]: cluster 2023-10-01T11:10:48.409203+0000 mgr.y (mgr.25072) 1606 : cluster 0 pgmap v1728: 89 pgs: 89 active+clean; 5.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 45 MiB/s wr, 718 op/s 2023-10-01T11:10:49.960 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:49.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:49.962 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:10:50.063 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:50.102 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:50.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:50.106 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:10:50.206 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:50.215 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:49 smithi078 bash[75299]: cluster 2023-10-01T11:10:48.409203+0000 mgr.y (mgr.25072) 1606 : cluster 0 pgmap v1728: 89 pgs: 89 active+clean; 5.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 45 MiB/s wr, 718 op/s 2023-10-01T11:10:50.217 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:49 smithi078 bash[78274]: cluster 2023-10-01T11:10:48.409203+0000 mgr.y (mgr.25072) 1606 : cluster 0 pgmap v1728: 89 pgs: 89 active+clean; 5.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 45 MiB/s wr, 718 op/s 2023-10-01T11:10:50.231 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:10:50.236 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:50.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:50.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:10:50.347 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:50.361 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 68 16 87844 87828 80.7115 41.5 0.00562331 0.0123792 2023-10-01T11:10:50.381 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:50.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:50.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:50.487 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:50.522 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:50.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:50.525 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:10:50.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:10:50 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:10:50] "GET /metrics HTTP/1.1" 200 36380 "" "Prometheus/2.33.4" 2023-10-01T11:10:50.625 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:50.650 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:50.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:50.654 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:50.754 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:50.767 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2023-10-01T11:10:50.768 INFO:teuthology.orchestra.run.smithi078.stdout: "size": 20, 2023-10-01T11:10:50.768 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 600, 2023-10-01T11:10:50.768 INFO:teuthology.orchestra.run.smithi078.stdout: "ops": [ 2023-10-01T11:10:50.768 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.768 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.46454.0:143128 13.2 e346/284)", 2023-10-01T11:10:50.768 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:08:12.890054+0000", 2023-10-01T11:10:50.768 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 157.86989606700001, 2023-10-01T11:10:50.768 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.36247995, 2023-10-01T11:10:50.768 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:10:50.768 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "reached pg", 2023-10-01T11:10:50.768 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:10:50.769 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.769 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:10:50.769 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:08:12.890054+0000", 2023-10-01T11:10:50.769 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:10:50.769 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.769 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.769 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:10:50.769 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:08:12.890054+0000", 2023-10-01T11:10:50.769 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:10:50.769 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.769 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.769 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:10:50.770 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:08:12.890053+0000", 2023-10-01T11:10:50.770 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T11:10:50.770 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.770 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.770 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:10:50.770 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:08:12.890080+0000", 2023-10-01T11:10:50.770 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.6922000000000001e-05 2023-10-01T11:10:50.770 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.770 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.770 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:10:50.770 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:08:12.890095+0000", 2023-10-01T11:10:50.771 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.5119999999999999e-05 2023-10-01T11:10:50.771 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.771 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.771 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:10:50.771 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:08:12.890104+0000", 2023-10-01T11:10:50.771 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.3049999999999998e-06 2023-10-01T11:10:50.771 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.771 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.771 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:10:50.771 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:08:13.252014+0000", 2023-10-01T11:10:50.771 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.36190951799999999 2023-10-01T11:10:50.771 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.772 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.772 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:10:50.772 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:08:13.252534+0000", 2023-10-01T11:10:50.772 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00052010999999999995 2023-10-01T11:10:50.772 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:10:50.772 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:10:50.772 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:10:50.772 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.772 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.772 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.46454.0:143136 13.2 e346/284)", 2023-10-01T11:10:50.772 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:08:12.901871+0000", 2023-10-01T11:10:50.772 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 157.85807859900001, 2023-10-01T11:10:50.773 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.350832535, 2023-10-01T11:10:50.773 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:10:50.773 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "reached pg", 2023-10-01T11:10:50.773 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:10:50.773 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.773 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:10:50.773 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:08:12.901871+0000", 2023-10-01T11:10:50.773 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:10:50.773 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.773 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.773 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:10:50.773 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:08:12.901871+0000", 2023-10-01T11:10:50.774 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:10:50.774 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.774 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.774 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:10:50.774 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:08:12.901871+0000", 2023-10-01T11:10:50.774 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999995 2023-10-01T11:10:50.774 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.774 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.774 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:10:50.774 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:08:12.901878+0000", 2023-10-01T11:10:50.774 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.5399999999999998e-06 2023-10-01T11:10:50.775 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.775 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.775 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:10:50.775 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:08:12.901898+0000", 2023-10-01T11:10:50.775 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.9185999999999999e-05 2023-10-01T11:10:50.775 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.775 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.775 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:10:50.775 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:08:12.901907+0000", 2023-10-01T11:10:50.775 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.3009999999999995e-06 2023-10-01T11:10:50.775 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.776 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.776 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:10:50.776 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:08:13.252687+0000", 2023-10-01T11:10:50.776 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.35078046400000001 2023-10-01T11:10:50.776 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.776 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.776 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:10:50.776 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:08:13.252704+0000", 2023-10-01T11:10:50.776 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.6370000000000001e-05 2023-10-01T11:10:50.776 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:10:50.776 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:10:50.777 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:10:50.777 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.777 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.777 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.36819.0:33743 14.f 14:f4c680fc:::benchmark_data_smithi078_236918_object33742:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e359)", 2023-10-01T11:10:50.777 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:09:58.673007+0000", 2023-10-01T11:10:50.777 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 52.086942743000002, 2023-10-01T11:10:50.777 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.20249431800000001, 2023-10-01T11:10:50.777 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:10:50.777 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:10:50.777 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T11:10:50.777 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.36819", 2023-10-01T11:10:50.777 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1409915880", 2023-10-01T11:10:50.778 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 33743 2023-10-01T11:10:50.778 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.778 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:10:50.778 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.778 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:10:50.778 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.673007+0000", 2023-10-01T11:10:50.778 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:10:50.778 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.778 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.778 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:10:50.778 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.673007+0000", 2023-10-01T11:10:50.778 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:10:50.779 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.779 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.779 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:10:50.779 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.673006+0000", 2023-10-01T11:10:50.779 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999986 2023-10-01T11:10:50.779 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.779 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.779 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:10:50.779 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.673051+0000", 2023-10-01T11:10:50.779 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.5432999999999997e-05 2023-10-01T11:10:50.779 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.779 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.780 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:10:50.780 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.673063+0000", 2023-10-01T11:10:50.780 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.2269000000000001e-05 2023-10-01T11:10:50.780 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.780 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.780 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:10:50.780 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.673072+0000", 2023-10-01T11:10:50.780 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.1699999999999997e-06 2023-10-01T11:10:50.780 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.780 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.780 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:10:50.780 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.673438+0000", 2023-10-01T11:10:50.780 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00036636900000000002 2023-10-01T11:10:50.781 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.781 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.781 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:10:50.781 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.673615+0000", 2023-10-01T11:10:50.781 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000176861 2023-10-01T11:10:50.781 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.781 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.781 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 1,7", 2023-10-01T11:10:50.781 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.673762+0000", 2023-10-01T11:10:50.781 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00014771000000000001 2023-10-01T11:10:50.781 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.782 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.782 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:10:50.782 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.676887+0000", 2023-10-01T11:10:50.782 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.003124361 2023-10-01T11:10:50.782 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.782 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.782 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:10:50.782 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.679541+0000", 2023-10-01T11:10:50.782 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0026537119999999999 2023-10-01T11:10:50.782 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.782 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.783 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T11:10:50.783 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.875387+0000", 2023-10-01T11:10:50.783 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.19584690799999999 2023-10-01T11:10:50.783 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.783 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.783 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:10:50.783 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.875452+0000", 2023-10-01T11:10:50.783 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.4632000000000001e-05 2023-10-01T11:10:50.783 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.783 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.783 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:10:50.783 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.875501+0000", 2023-10-01T11:10:50.783 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.9317999999999998e-05 2023-10-01T11:10:50.784 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:10:50.784 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:10:50.784 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:10:50.784 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.784 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.784 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.36819.0:33746 14.9 e359/356 14:9aaff77a:::benchmark_data_smithi078_236918_object33745:head v 359'2099, mlcod=359'2095)", 2023-10-01T11:10:50.784 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:09:58.675517+0000", 2023-10-01T11:10:50.784 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 52.084432561, 2023-10-01T11:10:50.784 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.20385214500000001, 2023-10-01T11:10:50.784 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:10:50.784 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:10:50.785 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:10:50.785 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.785 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:10:50.785 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.675517+0000", 2023-10-01T11:10:50.785 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:10:50.785 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.785 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.785 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:10:50.785 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.675517+0000", 2023-10-01T11:10:50.785 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:10:50.785 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.786 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.786 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:10:50.786 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.675516+0000", 2023-10-01T11:10:50.786 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T11:10:50.786 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.786 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.786 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:10:50.786 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.675572+0000", 2023-10-01T11:10:50.786 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.6039999999999999e-05 2023-10-01T11:10:50.786 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.786 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.786 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:10:50.787 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.675581+0000", 2023-10-01T11:10:50.787 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.4919999999999993e-06 2023-10-01T11:10:50.787 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.787 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.787 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:10:50.787 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.675586+0000", 2023-10-01T11:10:50.787 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.8570000000000003e-06 2023-10-01T11:10:50.787 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.787 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.787 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:10:50.787 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.675626+0000", 2023-10-01T11:10:50.788 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.0092000000000002e-05 2023-10-01T11:10:50.788 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.788 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.788 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:10:50.788 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.675638+0000", 2023-10-01T11:10:50.788 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.2217e-05 2023-10-01T11:10:50.788 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.788 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.788 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:10:50.788 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.879328+0000", 2023-10-01T11:10:50.788 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.203689598 2023-10-01T11:10:50.788 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.788 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.789 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:10:50.789 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.879369+0000", 2023-10-01T11:10:50.789 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.1915000000000001e-05 2023-10-01T11:10:50.789 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:10:50.789 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:10:50.789 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:10:50.789 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.789 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.789 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.36819.0:33757 14.1 14:8473d534:::benchmark_data_smithi078_236918_object33756:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e359)", 2023-10-01T11:10:50.789 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:09:58.678911+0000", 2023-10-01T11:10:50.789 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 52.081038415999998, 2023-10-01T11:10:50.789 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.20218154499999999, 2023-10-01T11:10:50.789 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:10:50.790 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:10:50.790 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T11:10:50.790 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.36819", 2023-10-01T11:10:50.790 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1409915880", 2023-10-01T11:10:50.790 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 33757 2023-10-01T11:10:50.790 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.790 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:10:50.790 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.790 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:10:50.790 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.678911+0000", 2023-10-01T11:10:50.790 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:10:50.791 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.791 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.791 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:10:50.791 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.678911+0000", 2023-10-01T11:10:50.791 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:10:50.791 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.791 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.791 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:10:50.791 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.678910+0000", 2023-10-01T11:10:50.791 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999986 2023-10-01T11:10:50.791 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.791 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.792 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:10:50.792 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.678934+0000", 2023-10-01T11:10:50.792 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.4164000000000001e-05 2023-10-01T11:10:50.792 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.792 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.792 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:10:50.792 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.678948+0000", 2023-10-01T11:10:50.792 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.4234e-05 2023-10-01T11:10:50.792 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.792 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.792 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:10:50.792 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.678957+0000", 2023-10-01T11:10:50.793 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.2819999999999996e-06 2023-10-01T11:10:50.793 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.793 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.793 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:10:50.793 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.678996+0000", 2023-10-01T11:10:50.793 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.9106000000000001e-05 2023-10-01T11:10:50.793 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.793 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.793 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:10:50.793 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.679089+0000", 2023-10-01T11:10:50.793 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.3579000000000006e-05 2023-10-01T11:10:50.794 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.794 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.794 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 1,2", 2023-10-01T11:10:50.794 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.679179+0000", 2023-10-01T11:10:50.794 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.9976999999999996e-05 2023-10-01T11:10:50.794 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.794 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.794 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T11:10:50.794 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.879388+0000", 2023-10-01T11:10:50.794 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.200208356 2023-10-01T11:10:50.794 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.795 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.795 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:10:50.795 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.879422+0000", 2023-10-01T11:10:50.795 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.4450999999999999e-05 2023-10-01T11:10:50.795 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.795 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.795 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:10:50.795 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.879452+0000", 2023-10-01T11:10:50.795 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.0456000000000001e-05 2023-10-01T11:10:50.795 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.795 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.795 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:10:50.795 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.881030+0000", 2023-10-01T11:10:50.796 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.001577464 2023-10-01T11:10:50.796 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.796 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.796 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:10:50.796 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.881093+0000", 2023-10-01T11:10:50.796 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.3059999999999996e-05 2023-10-01T11:10:50.796 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:10:50.796 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:10:50.796 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:10:50.796 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.796 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.796 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.36819.0:33754 14.4 e359/356 14:2f35cd76:::benchmark_data_smithi078_236918_object33753:head v 359'2058, mlcod=359'2056)", 2023-10-01T11:10:50.797 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:09:58.679305+0000", 2023-10-01T11:10:50.797 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 52.080644997999997, 2023-10-01T11:10:50.797 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.19903825999999999, 2023-10-01T11:10:50.797 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:10:50.797 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:10:50.797 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:10:50.797 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.797 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:10:50.797 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.679305+0000", 2023-10-01T11:10:50.797 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:10:50.797 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.797 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.798 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:10:50.798 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.679305+0000", 2023-10-01T11:10:50.798 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:10:50.798 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.798 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.798 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:10:50.798 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.679304+0000", 2023-10-01T11:10:50.798 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T11:10:50.798 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.798 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.798 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:10:50.798 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.679342+0000", 2023-10-01T11:10:50.799 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.7685999999999998e-05 2023-10-01T11:10:50.799 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.799 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.799 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:10:50.799 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.679353+0000", 2023-10-01T11:10:50.799 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1471000000000001e-05 2023-10-01T11:10:50.799 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.799 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.799 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:10:50.799 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.679360+0000", 2023-10-01T11:10:50.799 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.1489999999999997e-06 2023-10-01T11:10:50.799 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.800 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.800 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:10:50.800 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.679402+0000", 2023-10-01T11:10:50.800 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.2080999999999999e-05 2023-10-01T11:10:50.800 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.800 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.800 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:10:50.800 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.679423+0000", 2023-10-01T11:10:50.800 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.0239999999999999e-05 2023-10-01T11:10:50.800 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.800 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.800 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:10:50.801 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.878295+0000", 2023-10-01T11:10:50.801 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.19887238099999999 2023-10-01T11:10:50.801 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.801 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.801 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:10:50.801 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.878343+0000", 2023-10-01T11:10:50.801 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.8097999999999999e-05 2023-10-01T11:10:50.801 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:10:50.801 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:10:50.802 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:10:50.802 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.802 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.802 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.36819.0:33758 14.a e359/356 14:5e76fe1d:::benchmark_data_smithi078_236918_object33757:head v 359'2189, mlcod=359'2187)", 2023-10-01T11:10:50.802 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:09:58.680437+0000", 2023-10-01T11:10:50.802 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 52.079512772999998, 2023-10-01T11:10:50.802 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.19995517400000001, 2023-10-01T11:10:50.802 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:10:50.802 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:10:50.802 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:10:50.802 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.803 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:10:50.803 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.680437+0000", 2023-10-01T11:10:50.803 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:10:50.803 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.803 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.803 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:10:50.803 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.680437+0000", 2023-10-01T11:10:50.803 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:10:50.803 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.803 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.803 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:10:50.803 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.680436+0000", 2023-10-01T11:10:50.804 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T11:10:50.804 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.804 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.804 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:10:50.804 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.680487+0000", 2023-10-01T11:10:50.804 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.0541000000000003e-05 2023-10-01T11:10:50.804 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.804 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.804 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:10:50.804 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.680498+0000", 2023-10-01T11:10:50.804 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1731e-05 2023-10-01T11:10:50.804 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.805 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.805 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:10:50.805 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.680505+0000", 2023-10-01T11:10:50.805 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.832e-06 2023-10-01T11:10:50.805 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.805 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.805 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:10:50.805 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.875340+0000", 2023-10-01T11:10:50.805 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.19483479000000001 2023-10-01T11:10:50.805 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.805 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.806 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:10:50.806 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.875375+0000", 2023-10-01T11:10:50.806 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.4612e-05 2023-10-01T11:10:50.806 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.806 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.806 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:10:50.806 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.880319+0000", 2023-10-01T11:10:50.806 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0049437889999999996 2023-10-01T11:10:50.806 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.806 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.806 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:10:50.806 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.880392+0000", 2023-10-01T11:10:50.807 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.3727999999999996e-05 2023-10-01T11:10:50.807 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:10:50.807 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:10:50.807 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:10:50.807 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.807 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.807 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.36819.0:33755 14.2 e359/356 14:45eabb94:::benchmark_data_smithi078_236918_object33754:head v 359'2114, mlcod=359'2112)", 2023-10-01T11:10:50.807 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:09:58.681137+0000", 2023-10-01T11:10:50.807 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 52.078812937999999, 2023-10-01T11:10:50.807 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.19929064699999999, 2023-10-01T11:10:50.807 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:10:50.807 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:10:50.808 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:10:50.808 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.808 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:10:50.808 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.681137+0000", 2023-10-01T11:10:50.808 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:10:50.808 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.808 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.808 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:10:50.808 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.681137+0000", 2023-10-01T11:10:50.808 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:10:50.808 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.808 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.809 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:10:50.809 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.681136+0000", 2023-10-01T11:10:50.809 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999995 2023-10-01T11:10:50.809 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.809 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.809 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:10:50.809 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.681158+0000", 2023-10-01T11:10:50.809 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.1988e-05 2023-10-01T11:10:50.809 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.809 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.809 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:10:50.809 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.875282+0000", 2023-10-01T11:10:50.810 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.19412410799999999 2023-10-01T11:10:50.810 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.810 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.810 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:10:50.810 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.875300+0000", 2023-10-01T11:10:50.810 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.7435000000000002e-05 2023-10-01T11:10:50.810 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.810 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.810 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:10:50.810 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.877629+0000", 2023-10-01T11:10:50.810 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0023291950000000001 2023-10-01T11:10:50.810 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.811 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.811 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:10:50.811 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.877669+0000", 2023-10-01T11:10:50.811 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.9962999999999997e-05 2023-10-01T11:10:50.811 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.811 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.811 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:10:50.811 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.880396+0000", 2023-10-01T11:10:50.811 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0027273150000000001 2023-10-01T11:10:50.811 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.811 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.811 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:10:50.812 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.880428+0000", 2023-10-01T11:10:50.812 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.1197e-05 2023-10-01T11:10:50.812 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:10:50.812 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:10:50.812 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:10:50.812 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.812 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.812 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.36819.0:33760 14.9 e359/356 14:925269e4:::benchmark_data_smithi078_236918_object33759:head v 359'2100, mlcod=359'2097)", 2023-10-01T11:10:50.812 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:09:58.683564+0000", 2023-10-01T11:10:50.812 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 52.076385915000003, 2023-10-01T11:10:50.812 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.19783094200000001, 2023-10-01T11:10:50.812 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:10:50.813 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:10:50.813 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:10:50.813 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.813 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:10:50.813 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.683564+0000", 2023-10-01T11:10:50.813 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:10:50.813 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.813 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.813 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:10:50.813 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.683564+0000", 2023-10-01T11:10:50.814 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:10:50.814 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.814 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.814 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:10:50.814 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.683563+0000", 2023-10-01T11:10:50.814 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T11:10:50.814 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.814 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.814 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:10:50.814 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.683614+0000", 2023-10-01T11:10:50.814 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.1223000000000001e-05 2023-10-01T11:10:50.815 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.815 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.815 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:10:50.815 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.875280+0000", 2023-10-01T11:10:50.815 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.19166576099999999 2023-10-01T11:10:50.815 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.815 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.815 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:10:50.815 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.875300+0000", 2023-10-01T11:10:50.815 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.9794000000000001e-05 2023-10-01T11:10:50.815 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.815 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.816 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:10:50.816 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.879544+0000", 2023-10-01T11:10:50.816 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0042437050000000004 2023-10-01T11:10:50.816 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.816 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.816 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:10:50.816 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.879655+0000", 2023-10-01T11:10:50.816 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000111086 2023-10-01T11:10:50.816 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.816 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.816 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:10:50.816 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.881355+0000", 2023-10-01T11:10:50.817 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.001700717 2023-10-01T11:10:50.817 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.817 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.817 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:10:50.817 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:09:58.881395+0000", 2023-10-01T11:10:50.817 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.9440000000000002e-05 2023-10-01T11:10:50.817 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:10:50.817 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:10:50.817 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:10:50.817 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.817 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.818 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.36819.0:38324 14.3 14:c3d3d8e5:::benchmark_data_smithi078_236918_object38323:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e359)", 2023-10-01T11:10:50.818 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:10:01.883224+0000", 2023-10-01T11:10:50.818 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 48.876725942, 2023-10-01T11:10:50.818 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.17789290399999999, 2023-10-01T11:10:50.818 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:10:50.818 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:10:50.818 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T11:10:50.818 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.36819", 2023-10-01T11:10:50.818 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1409915880", 2023-10-01T11:10:50.818 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 38324 2023-10-01T11:10:50.818 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.818 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:10:50.819 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.819 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:10:50.819 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.883224+0000", 2023-10-01T11:10:50.819 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:10:50.819 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.819 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.819 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:10:50.819 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.883224+0000", 2023-10-01T11:10:50.819 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:10:50.819 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.819 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.819 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:10:50.820 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.883222+0000", 2023-10-01T11:10:50.820 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999981 2023-10-01T11:10:50.820 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.820 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.820 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:10:50.820 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.883254+0000", 2023-10-01T11:10:50.820 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.2265999999999999e-05 2023-10-01T11:10:50.820 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.820 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.820 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:10:50.820 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.883278+0000", 2023-10-01T11:10:50.821 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.404e-05 2023-10-01T11:10:50.821 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.821 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.821 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:10:50.821 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.883289+0000", 2023-10-01T11:10:50.821 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.092e-05 2023-10-01T11:10:50.821 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.821 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.821 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:10:50.821 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.883462+0000", 2023-10-01T11:10:50.821 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00017236000000000001 2023-10-01T11:10:50.821 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.821 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.822 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:10:50.822 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.883622+0000", 2023-10-01T11:10:50.822 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000160579 2023-10-01T11:10:50.822 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.822 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.822 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 0,1", 2023-10-01T11:10:50.822 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.883788+0000", 2023-10-01T11:10:50.822 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000166105 2023-10-01T11:10:50.822 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.822 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.822 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T11:10:50.822 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.885644+0000", 2023-10-01T11:10:50.823 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0018551780000000001 2023-10-01T11:10:50.823 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.823 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.823 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:10:50.823 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.905674+0000", 2023-10-01T11:10:50.823 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.020030888 2023-10-01T11:10:50.823 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.823 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.823 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:10:50.823 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:02.060751+0000", 2023-10-01T11:10:50.823 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.15507683899999999 2023-10-01T11:10:50.823 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.824 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.824 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:10:50.824 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:02.061002+0000", 2023-10-01T11:10:50.824 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00025026300000000003 2023-10-01T11:10:50.824 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.824 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.824 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:10:50.824 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:02.061117+0000", 2023-10-01T11:10:50.824 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00011524800000000001 2023-10-01T11:10:50.824 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:10:50.824 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:10:50.824 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:10:50.824 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.825 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.825 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.36819.0:77783 14.1 14:85f91a45:::benchmark_data_smithi078_236918_object77782:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e361)", 2023-10-01T11:10:50.825 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:10:38.465691+0000", 2023-10-01T11:10:50.825 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 12.294258927, 2023-10-01T11:10:50.825 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.36551430800000001, 2023-10-01T11:10:50.825 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:10:50.825 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:10:50.825 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T11:10:50.825 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.36819", 2023-10-01T11:10:50.825 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1409915880", 2023-10-01T11:10:50.825 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 77783 2023-10-01T11:10:50.825 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.826 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:10:50.826 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.826 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:10:50.826 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.465691+0000", 2023-10-01T11:10:50.826 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:10:50.826 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.826 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.826 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:10:50.826 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.465691+0000", 2023-10-01T11:10:50.826 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:10:50.826 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.826 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.827 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:10:50.827 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.465690+0000", 2023-10-01T11:10:50.827 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T11:10:50.827 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.827 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.827 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:10:50.827 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.465710+0000", 2023-10-01T11:10:50.827 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.0395999999999999e-05 2023-10-01T11:10:50.827 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.827 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.827 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:10:50.827 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.465721+0000", 2023-10-01T11:10:50.828 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.0895e-05 2023-10-01T11:10:50.828 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.828 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.828 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:10:50.828 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.465727+0000", 2023-10-01T11:10:50.828 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.2940000000000002e-06 2023-10-01T11:10:50.828 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.828 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.828 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:10:50.828 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.473424+0000", 2023-10-01T11:10:50.828 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0076966109999999999 2023-10-01T11:10:50.829 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.829 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.829 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:10:50.829 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.473527+0000", 2023-10-01T11:10:50.829 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00010298699999999999 2023-10-01T11:10:50.829 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.829 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.829 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 1,2", 2023-10-01T11:10:50.829 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.473677+0000", 2023-10-01T11:10:50.829 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000149777 2023-10-01T11:10:50.829 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.829 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.830 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T11:10:50.830 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.477281+0000", 2023-10-01T11:10:50.830 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0036045230000000001 2023-10-01T11:10:50.830 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.830 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.830 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:10:50.830 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.477316+0000", 2023-10-01T11:10:50.830 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.4594000000000001e-05 2023-10-01T11:10:50.830 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.830 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.830 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:10:50.830 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.831074+0000", 2023-10-01T11:10:50.831 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.35375795500000001 2023-10-01T11:10:50.831 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.831 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.831 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:10:50.831 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.831127+0000", 2023-10-01T11:10:50.831 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.2951000000000002e-05 2023-10-01T11:10:50.831 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.831 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.831 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:10:50.831 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.831205+0000", 2023-10-01T11:10:50.831 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.8462999999999997e-05 2023-10-01T11:10:50.831 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:10:50.832 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:10:50.832 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:10:50.832 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.832 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.832 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.36819.0:77791 14.1 14:86ad5cd3:::benchmark_data_smithi078_236918_object77790:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e361)", 2023-10-01T11:10:50.832 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:10:38.473939+0000", 2023-10-01T11:10:50.832 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 12.286010906, 2023-10-01T11:10:50.832 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.35737125800000002, 2023-10-01T11:10:50.832 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:10:50.832 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:10:50.832 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T11:10:50.832 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.36819", 2023-10-01T11:10:50.833 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1409915880", 2023-10-01T11:10:50.833 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 77791 2023-10-01T11:10:50.833 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.833 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:10:50.833 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.833 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:10:50.833 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.473939+0000", 2023-10-01T11:10:50.833 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:10:50.833 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.833 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.833 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:10:50.834 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.473939+0000", 2023-10-01T11:10:50.834 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:10:50.834 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.834 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.834 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:10:50.834 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.473937+0000", 2023-10-01T11:10:50.834 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999986 2023-10-01T11:10:50.834 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.834 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.834 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:10:50.835 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.473960+0000", 2023-10-01T11:10:50.835 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.2724000000000001e-05 2023-10-01T11:10:50.835 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.835 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.835 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:10:50.835 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.473972+0000", 2023-10-01T11:10:50.835 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.2115e-05 2023-10-01T11:10:50.835 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.835 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.835 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:10:50.835 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.473980+0000", 2023-10-01T11:10:50.836 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.7219999999999999e-06 2023-10-01T11:10:50.836 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.836 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.836 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:10:50.836 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.477339+0000", 2023-10-01T11:10:50.836 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0033593080000000001 2023-10-01T11:10:50.836 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.836 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.836 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:10:50.836 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.477817+0000", 2023-10-01T11:10:50.836 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00047753900000000002 2023-10-01T11:10:50.836 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.837 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.837 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 1,2", 2023-10-01T11:10:50.837 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.478135+0000", 2023-10-01T11:10:50.837 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00031777999999999998 2023-10-01T11:10:50.837 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.837 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.837 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T11:10:50.837 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.481374+0000", 2023-10-01T11:10:50.837 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0032399149999999999 2023-10-01T11:10:50.837 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.837 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.837 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:10:50.838 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.830644+0000", 2023-10-01T11:10:50.838 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.349269677 2023-10-01T11:10:50.838 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.838 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.838 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:10:50.838 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.831243+0000", 2023-10-01T11:10:50.838 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000598757 2023-10-01T11:10:50.838 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.838 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.838 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:10:50.838 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.831280+0000", 2023-10-01T11:10:50.839 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.7005999999999998e-05 2023-10-01T11:10:50.839 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.839 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.839 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:10:50.839 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.831310+0000", 2023-10-01T11:10:50.839 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.0337000000000001e-05 2023-10-01T11:10:50.839 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:10:50.839 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:10:50.839 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:10:50.839 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.839 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.839 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.36819.0:77793 14.f 14:fc1fc176:::benchmark_data_smithi078_236918_object77792:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e361)", 2023-10-01T11:10:50.840 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:10:38.476884+0000", 2023-10-01T11:10:50.840 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 12.283066043, 2023-10-01T11:10:50.840 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.35839653199999999, 2023-10-01T11:10:50.840 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:10:50.840 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:10:50.840 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T11:10:50.840 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.36819", 2023-10-01T11:10:50.840 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1409915880", 2023-10-01T11:10:50.840 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 77793 2023-10-01T11:10:50.840 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.840 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:10:50.840 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.841 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:10:50.841 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.476884+0000", 2023-10-01T11:10:50.841 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:10:50.841 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.841 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.841 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:10:50.841 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.476884+0000", 2023-10-01T11:10:50.841 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:10:50.841 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.841 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.841 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:10:50.841 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.476882+0000", 2023-10-01T11:10:50.842 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999986 2023-10-01T11:10:50.842 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.842 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.842 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:10:50.842 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.476912+0000", 2023-10-01T11:10:50.842 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.9077e-05 2023-10-01T11:10:50.842 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.842 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.842 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:10:50.842 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.476924+0000", 2023-10-01T11:10:50.842 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.2121e-05 2023-10-01T11:10:50.842 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.843 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.843 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:10:50.843 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.476932+0000", 2023-10-01T11:10:50.843 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.3429999999999996e-06 2023-10-01T11:10:50.843 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.843 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.843 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:10:50.843 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.476973+0000", 2023-10-01T11:10:50.843 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.1007000000000002e-05 2023-10-01T11:10:50.843 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.843 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.844 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:10:50.844 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.477116+0000", 2023-10-01T11:10:50.844 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00014290099999999999 2023-10-01T11:10:50.844 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.844 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.844 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 1,7", 2023-10-01T11:10:50.844 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.477284+0000", 2023-10-01T11:10:50.844 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000167656 2023-10-01T11:10:50.844 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.844 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.844 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T11:10:50.845 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.480357+0000", 2023-10-01T11:10:50.845 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0030734439999999998 2023-10-01T11:10:50.845 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.845 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.845 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:10:50.845 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.481424+0000", 2023-10-01T11:10:50.845 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.001066532 2023-10-01T11:10:50.845 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.845 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.845 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:10:50.845 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.835172+0000", 2023-10-01T11:10:50.845 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.35374823100000002 2023-10-01T11:10:50.846 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.846 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.846 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:10:50.846 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.835234+0000", 2023-10-01T11:10:50.846 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.2633999999999997e-05 2023-10-01T11:10:50.846 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.846 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.846 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:10:50.846 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.835280+0000", 2023-10-01T11:10:50.846 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.5926e-05 2023-10-01T11:10:50.847 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:10:50.847 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:10:50.847 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:10:50.847 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.847 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.847 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.36819.0:77798 14.3 14:c9cda69d:::benchmark_data_smithi078_236918_object77797:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e361)", 2023-10-01T11:10:50.847 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:10:38.478025+0000", 2023-10-01T11:10:50.847 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 12.281925274000001, 2023-10-01T11:10:50.847 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.35335153499999999, 2023-10-01T11:10:50.847 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:10:50.847 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:10:50.847 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T11:10:50.848 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.36819", 2023-10-01T11:10:50.848 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1409915880", 2023-10-01T11:10:50.848 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 77798 2023-10-01T11:10:50.848 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.848 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:10:50.848 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.848 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:10:50.848 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.478025+0000", 2023-10-01T11:10:50.848 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:10:50.848 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.848 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.849 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:10:50.849 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.478025+0000", 2023-10-01T11:10:50.849 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:10:50.849 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.849 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.849 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:10:50.849 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.478023+0000", 2023-10-01T11:10:50.849 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T11:10:50.849 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.849 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.849 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:10:50.850 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.478048+0000", 2023-10-01T11:10:50.850 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.4349999999999999e-05 2023-10-01T11:10:50.850 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.850 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.850 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:10:50.850 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.478060+0000", 2023-10-01T11:10:50.850 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1841000000000001e-05 2023-10-01T11:10:50.850 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.850 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.850 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:10:50.850 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.478068+0000", 2023-10-01T11:10:50.851 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.5469999999999997e-06 2023-10-01T11:10:50.851 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.851 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.851 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:10:50.851 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.478830+0000", 2023-10-01T11:10:50.851 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00076192800000000004 2023-10-01T11:10:50.851 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.851 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.851 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:10:50.851 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.478947+0000", 2023-10-01T11:10:50.852 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000116518 2023-10-01T11:10:50.852 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.852 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.852 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 0,1", 2023-10-01T11:10:50.852 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.479069+0000", 2023-10-01T11:10:50.852 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00012222099999999999 2023-10-01T11:10:50.852 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.852 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.852 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T11:10:50.852 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.482308+0000", 2023-10-01T11:10:50.852 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00323954 2023-10-01T11:10:50.852 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.853 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.853 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:10:50.853 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.482432+0000", 2023-10-01T11:10:50.853 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000123466 2023-10-01T11:10:50.853 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.853 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.853 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:10:50.853 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.831294+0000", 2023-10-01T11:10:50.853 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.34886199200000001 2023-10-01T11:10:50.853 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.853 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.854 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:10:50.854 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.831339+0000", 2023-10-01T11:10:50.854 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.5061000000000001e-05 2023-10-01T11:10:50.854 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.854 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.854 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:10:50.854 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.831376+0000", 2023-10-01T11:10:50.854 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.7188999999999999e-05 2023-10-01T11:10:50.854 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:10:50.854 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:10:50.854 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:10:50.854 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.855 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.855 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.36819.0:77802 14.9 e361/356 14:9548ba1a:::benchmark_data_smithi078_236918_object77801:head v 361'4805, mlcod=361'4803)", 2023-10-01T11:10:50.855 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:10:38.483441+0000", 2023-10-01T11:10:50.855 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 12.276509244, 2023-10-01T11:10:50.855 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.35813577099999999, 2023-10-01T11:10:50.855 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:10:50.855 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:10:50.855 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:10:50.855 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.855 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:10:50.855 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.483441+0000", 2023-10-01T11:10:50.856 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:10:50.856 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.856 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.856 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:10:50.856 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.483441+0000", 2023-10-01T11:10:50.856 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:10:50.856 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.856 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.856 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:10:50.856 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.483440+0000", 2023-10-01T11:10:50.856 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999995 2023-10-01T11:10:50.856 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.856 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.857 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:10:50.857 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.483692+0000", 2023-10-01T11:10:50.857 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00025181800000000001 2023-10-01T11:10:50.857 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.857 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.857 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:10:50.857 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.483697+0000", 2023-10-01T11:10:50.857 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.4229999999999996e-06 2023-10-01T11:10:50.857 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.857 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.857 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:10:50.857 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.483701+0000", 2023-10-01T11:10:50.858 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.4570000000000001e-06 2023-10-01T11:10:50.858 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.858 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.858 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:10:50.858 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.830449+0000", 2023-10-01T11:10:50.858 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.34674812300000002 2023-10-01T11:10:50.858 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.858 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.858 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:10:50.858 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.830477+0000", 2023-10-01T11:10:50.858 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.813e-05 2023-10-01T11:10:50.858 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.859 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.859 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:10:50.859 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.841539+0000", 2023-10-01T11:10:50.859 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.011061757 2023-10-01T11:10:50.859 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.859 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.859 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:10:50.859 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.841576+0000", 2023-10-01T11:10:50.859 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.7623999999999998e-05 2023-10-01T11:10:50.859 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:10:50.859 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:10:50.860 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:10:50.860 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.860 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.860 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.36819.0:77805 14.f 14:f22d0655:::benchmark_data_smithi078_236918_object77804:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e361)", 2023-10-01T11:10:50.860 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:10:38.483627+0000", 2023-10-01T11:10:50.860 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 12.27632283, 2023-10-01T11:10:50.860 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.36450644700000001, 2023-10-01T11:10:50.860 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:10:50.860 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:10:50.860 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T11:10:50.860 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.36819", 2023-10-01T11:10:50.861 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1409915880", 2023-10-01T11:10:50.861 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 77805 2023-10-01T11:10:50.861 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.861 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:10:50.861 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.861 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:10:50.861 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.483627+0000", 2023-10-01T11:10:50.861 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:10:50.861 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.861 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.861 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:10:50.861 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.483627+0000", 2023-10-01T11:10:50.862 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:10:50.862 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.862 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.862 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:10:50.862 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.483625+0000", 2023-10-01T11:10:50.862 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999981 2023-10-01T11:10:50.862 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.862 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.862 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:10:50.862 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.483651+0000", 2023-10-01T11:10:50.862 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.6052000000000001e-05 2023-10-01T11:10:50.862 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.863 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.863 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:10:50.863 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.483661+0000", 2023-10-01T11:10:50.863 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.5629999999999998e-06 2023-10-01T11:10:50.863 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.863 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.863 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:10:50.863 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.483668+0000", 2023-10-01T11:10:50.863 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.9240000000000001e-06 2023-10-01T11:10:50.863 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.863 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.863 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:10:50.864 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.483742+0000", 2023-10-01T11:10:50.864 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.4585999999999994e-05 2023-10-01T11:10:50.864 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.864 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.864 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:10:50.864 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.483872+0000", 2023-10-01T11:10:50.864 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00012970700000000001 2023-10-01T11:10:50.864 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.864 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.864 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 1,7", 2023-10-01T11:10:50.864 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.484036+0000", 2023-10-01T11:10:50.865 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00016373200000000001 2023-10-01T11:10:50.865 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.865 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.865 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:10:50.865 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.835333+0000", 2023-10-01T11:10:50.865 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.35129682200000001 2023-10-01T11:10:50.865 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.865 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.865 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:10:50.865 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.835457+0000", 2023-10-01T11:10:50.866 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000124735 2023-10-01T11:10:50.866 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.866 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.866 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T11:10:50.866 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.848036+0000", 2023-10-01T11:10:50.866 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.012578397999999999 2023-10-01T11:10:50.866 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.866 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.866 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:10:50.866 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.848087+0000", 2023-10-01T11:10:50.867 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.1659999999999997e-05 2023-10-01T11:10:50.867 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.867 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.867 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:10:50.867 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.848133+0000", 2023-10-01T11:10:50.867 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.6035999999999998e-05 2023-10-01T11:10:50.867 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:10:50.867 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:10:50.867 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:10:50.867 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.867 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.868 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop_reply(client.36819.0:77793 14.f e361/356 ondisk, result = 0)", 2023-10-01T11:10:50.868 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:10:38.483734+0000", 2023-10-01T11:10:50.868 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 12.276215451000001, 2023-10-01T11:10:50.868 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.35155821300000001, 2023-10-01T11:10:50.868 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:10:50.868 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "started", 2023-10-01T11:10:50.868 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:10:50.868 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.868 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:10:50.868 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.483734+0000", 2023-10-01T11:10:50.868 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:10:50.869 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.869 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.869 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:10:50.869 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.483734+0000", 2023-10-01T11:10:50.869 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:10:50.869 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.869 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.869 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:10:50.869 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.483734+0000", 2023-10-01T11:10:50.869 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999995 2023-10-01T11:10:50.869 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.870 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.870 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:10:50.870 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.483739+0000", 2023-10-01T11:10:50.870 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.6110000000000001e-06 2023-10-01T11:10:50.870 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.870 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.870 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:10:50.870 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.483750+0000", 2023-10-01T11:10:50.870 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.0851e-05 2023-10-01T11:10:50.870 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.871 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.871 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:10:50.871 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.483757+0000", 2023-10-01T11:10:50.871 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.9639999999999997e-06 2023-10-01T11:10:50.871 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.871 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.871 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:10:50.871 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.835156+0000", 2023-10-01T11:10:50.871 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.35139868899999999 2023-10-01T11:10:50.871 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.871 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.872 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:10:50.872 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.835165+0000", 2023-10-01T11:10:50.872 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.268e-06 2023-10-01T11:10:50.872 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.872 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.872 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:10:50.872 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.835293+0000", 2023-10-01T11:10:50.872 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00012751 2023-10-01T11:10:50.872 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:10:50.872 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:10:50.872 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:10:50.872 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.873 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.873 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop_reply(client.36819.0:77791 14.1 e361/356 ondisk, result = 0)", 2023-10-01T11:10:50.873 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:10:38.486141+0000", 2023-10-01T11:10:50.873 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 12.273809277, 2023-10-01T11:10:50.873 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.34451200599999998, 2023-10-01T11:10:50.873 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:10:50.873 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "started", 2023-10-01T11:10:50.873 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:10:50.873 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.873 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:10:50.873 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.486141+0000", 2023-10-01T11:10:50.874 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:10:50.874 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.874 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.874 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:10:50.874 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.486141+0000", 2023-10-01T11:10:50.874 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:10:50.874 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.874 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.874 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:10:50.874 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.486140+0000", 2023-10-01T11:10:50.875 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T11:10:50.875 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.875 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.875 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:10:50.875 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.486146+0000", 2023-10-01T11:10:50.875 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.7259999999999996e-06 2023-10-01T11:10:50.875 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.875 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.875 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:10:50.875 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.830365+0000", 2023-10-01T11:10:50.875 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.34421905699999999 2023-10-01T11:10:50.876 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.876 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.876 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:10:50.876 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.830388+0000", 2023-10-01T11:10:50.876 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.2350000000000001e-05 2023-10-01T11:10:50.876 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.876 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.876 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:10:50.876 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.830605+0000", 2023-10-01T11:10:50.876 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00021721300000000001 2023-10-01T11:10:50.876 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.876 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.877 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:10:50.877 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.830627+0000", 2023-10-01T11:10:50.877 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.1676999999999999e-05 2023-10-01T11:10:50.877 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.877 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.877 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:10:50.877 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.830653+0000", 2023-10-01T11:10:50.877 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.5939e-05 2023-10-01T11:10:50.877 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:10:50.877 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:10:50.877 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:10:50.878 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.878 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.878 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop_reply(client.36819.0:77805 14.f e361/356 ondisk, result = 0)", 2023-10-01T11:10:50.878 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:10:38.487887+0000", 2023-10-01T11:10:50.878 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 12.272062449, 2023-10-01T11:10:50.878 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.34744752099999998, 2023-10-01T11:10:50.878 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:10:50.878 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "started", 2023-10-01T11:10:50.878 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:10:50.878 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.878 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:10:50.879 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.487887+0000", 2023-10-01T11:10:50.879 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:10:50.879 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.879 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.879 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:10:50.879 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.487887+0000", 2023-10-01T11:10:50.879 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:10:50.879 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.879 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.879 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:10:50.879 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.487886+0000", 2023-10-01T11:10:50.880 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T11:10:50.880 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.880 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.880 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:10:50.880 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.487896+0000", 2023-10-01T11:10:50.880 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.3200000000000006e-06 2023-10-01T11:10:50.880 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.880 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.880 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:10:50.880 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.830341+0000", 2023-10-01T11:10:50.880 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.342445308 2023-10-01T11:10:50.881 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.881 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.881 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:10:50.881 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.830365+0000", 2023-10-01T11:10:50.881 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.3632999999999999e-05 2023-10-01T11:10:50.881 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.881 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.881 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:10:50.881 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.835321+0000", 2023-10-01T11:10:50.881 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0049558520000000002 2023-10-01T11:10:50.882 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.882 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.882 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:10:50.882 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.835327+0000", 2023-10-01T11:10:50.882 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.7580000000000003e-06 2023-10-01T11:10:50.882 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.882 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.882 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:10:50.882 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.835335+0000", 2023-10-01T11:10:50.883 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.5939999999999996e-06 2023-10-01T11:10:50.883 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:10:50.883 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:10:50.883 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:10:50.883 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.883 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.883 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_op(client.36819.0:77810 14.f 14:f0dc6b88:::benchmark_data_smithi078_236918_object77809:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e361)", 2023-10-01T11:10:50.883 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:10:38.492012+0000", 2023-10-01T11:10:50.883 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 12.267937525000001, 2023-10-01T11:10:50.883 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.35992901500000002, 2023-10-01T11:10:50.883 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:10:50.883 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:10:50.884 INFO:teuthology.orchestra.run.smithi078.stdout: "client_info": { 2023-10-01T11:10:50.884 INFO:teuthology.orchestra.run.smithi078.stdout: "client": "client.36819", 2023-10-01T11:10:50.884 INFO:teuthology.orchestra.run.smithi078.stdout: "client_addr": "172.21.15.78:0/1409915880", 2023-10-01T11:10:50.884 INFO:teuthology.orchestra.run.smithi078.stdout: "tid": 77810 2023-10-01T11:10:50.884 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.884 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:10:50.884 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.884 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:10:50.884 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.492012+0000", 2023-10-01T11:10:50.884 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:10:50.884 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.885 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.885 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:10:50.885 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.492012+0000", 2023-10-01T11:10:50.885 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:10:50.885 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.885 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.885 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:10:50.885 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.492010+0000", 2023-10-01T11:10:50.885 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999981 2023-10-01T11:10:50.885 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.886 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.886 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:10:50.886 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.492041+0000", 2023-10-01T11:10:50.886 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.0635000000000001e-05 2023-10-01T11:10:50.886 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.886 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.886 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:10:50.886 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.830343+0000", 2023-10-01T11:10:50.886 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.33830252599999999 2023-10-01T11:10:50.886 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.886 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.887 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:10:50.887 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.830366+0000", 2023-10-01T11:10:50.887 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.3096999999999999e-05 2023-10-01T11:10:50.887 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.887 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.887 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:10:50.887 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.835487+0000", 2023-10-01T11:10:50.887 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0051204340000000001 2023-10-01T11:10:50.887 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.887 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.887 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:10:50.887 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.835629+0000", 2023-10-01T11:10:50.888 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00014220699999999999 2023-10-01T11:10:50.888 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.888 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.888 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "waiting for subops from 1,7", 2023-10-01T11:10:50.888 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.835767+0000", 2023-10-01T11:10:50.888 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000138085 2023-10-01T11:10:50.888 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.888 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.888 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:10:50.888 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.848028+0000", 2023-10-01T11:10:50.888 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.012260893 2023-10-01T11:10:50.889 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.889 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.889 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "op_commit", 2023-10-01T11:10:50.889 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.848143+0000", 2023-10-01T11:10:50.889 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00011487 2023-10-01T11:10:50.889 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.889 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.889 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:10:50.889 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.851854+0000", 2023-10-01T11:10:50.889 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0037111700000000002 2023-10-01T11:10:50.889 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.889 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.890 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:10:50.890 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.851902+0000", 2023-10-01T11:10:50.890 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.8371000000000002e-05 2023-10-01T11:10:50.890 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:10:50.890 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:10:50.890 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:10:50.890 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:38.851941+0000", 2023-10-01T11:10:50.890 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.8850999999999998e-05 2023-10-01T11:10:50.890 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:10:50.890 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:10:50.891 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:10:50.891 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:10:50.891 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:10:50.891 INFO:teuthology.orchestra.run.smithi078.stdout:} 2023-10-01T11:10:50.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:50.892 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:50.892 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:50.993 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:51.021 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:51.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:51.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:51.125 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:51.161 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:51.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:51.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:10:51.270 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:51.299 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:51.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:51.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:10:51.362 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 69 16 88726 88710 80.3406 55.125 0.0125109 0.0124414 2023-10-01T11:10:51.402 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:51.425 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:51.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:51.427 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:10:51.529 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:51.557 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:51.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:51.563 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:10:51.663 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:51.688 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:51.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:51.690 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:10:51.791 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:51.817 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:51.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:51.819 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:10:51.920 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:51.951 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:51.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:51.954 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:52.055 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:52.079 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:52.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:51 smithi078 bash[75299]: cluster 2023-10-01T11:10:50.409924+0000 mgr.y (mgr.25072) 1607 : cluster 0 pgmap v1729: 89 pgs: 89 active+clean; 5.3 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-01T11:10:52.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:51 smithi078 bash[78274]: cluster 2023-10-01T11:10:50.409924+0000 mgr.y (mgr.25072) 1607 : cluster 0 pgmap v1729: 89 pgs: 89 active+clean; 5.3 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-01T11:10:52.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:52.082 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:10:52.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:51 smithi173 bash[58816]: cluster 2023-10-01T11:10:50.409924+0000 mgr.y (mgr.25072) 1607 : cluster 0 pgmap v1729: 89 pgs: 89 active+clean; 5.3 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-10-01T11:10:52.183 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:52.210 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:52.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:52.214 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:10:52.315 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:52.349 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:52.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:52.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:52.362 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 70 16 89543 89527 79.9223 51.0625 0.0784864 0.0125044 2023-10-01T11:10:52.451 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:52.478 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:52.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:52.482 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:10:52.583 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:52.620 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:52.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:52.622 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:10:52.723 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:52.748 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:52.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:52.753 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:52.854 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:52.877 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:52.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:52.879 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:10:52.979 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:53.000 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:53.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:53.006 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:10:53.107 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:53.135 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:53.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:53.138 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:10:53.238 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:53.278 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:53.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:53.282 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:10:53.362 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 71 16 90441 90425 79.5871 56.125 0.0149434 0.0125594 2023-10-01T11:10:53.384 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:53.420 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:53.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:53.426 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:10:53.526 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:53.554 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:53.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:53.559 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:10:53.659 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:53.690 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:53.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:53.698 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:53.799 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:53.824 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:53.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:53.827 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:10:53.927 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:53.941 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:53 smithi173 bash[58816]: cluster 2023-10-01T11:10:52.410472+0000 mgr.y (mgr.25072) 1608 : cluster 0 pgmap v1730: 89 pgs: 89 active+clean; 5.3 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 50 MiB/s wr, 794 op/s 2023-10-01T11:10:53.956 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:53.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:53.958 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:10:54.059 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:54.086 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:54.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:54.092 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:10:54.194 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:54.218 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:54.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:54.222 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:10:54.269 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:53 smithi078 bash[78274]: cluster 2023-10-01T11:10:52.410472+0000 mgr.y (mgr.25072) 1608 : cluster 0 pgmap v1730: 89 pgs: 89 active+clean; 5.3 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 50 MiB/s wr, 794 op/s 2023-10-01T11:10:54.269 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:53 smithi078 bash[75299]: cluster 2023-10-01T11:10:52.410472+0000 mgr.y (mgr.25072) 1608 : cluster 0 pgmap v1730: 89 pgs: 89 active+clean; 5.3 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 50 MiB/s wr, 794 op/s 2023-10-01T11:10:54.323 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:54.346 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:54.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:54.348 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:10:54.362 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 72 16 91292 91276 79.2204 53.1875 0.0204375 0.0126172 2023-10-01T11:10:54.449 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:54.471 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:54.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:54.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:54.575 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:54.603 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:54.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:54.606 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:54.707 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:54.742 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:54.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:54.746 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:10:54.846 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:54.880 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:54.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:54.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:54.989 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:55.013 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:55.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:55.016 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:10:55.043 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-01T11:10:55.117 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:55.141 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:55.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:55.148 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:55.249 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:55.275 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:55.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:55.278 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:55.362 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 73 16 92060 92044 78.7926 48 0.00628516 0.0126778 2023-10-01T11:10:55.379 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:55.413 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:55.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:55.460 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:55.561 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:55.586 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:55.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:55.589 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:10:55.690 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:55.714 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:55.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:55.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:55.817 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:55.850 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:55.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:55.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:55.953 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:55.963 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:55 smithi173 bash[58816]: cluster 2023-10-01T11:10:54.411147+0000 mgr.y (mgr.25072) 1609 : cluster 0 pgmap v1731: 89 pgs: 89 active+clean; 5.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 60 MiB/s wr, 956 op/s 2023-10-01T11:10:55.982 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:55.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:55.987 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:56.088 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:56.121 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:56.124 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:55 smithi078 bash[75299]: cluster 2023-10-01T11:10:54.411147+0000 mgr.y (mgr.25072) 1609 : cluster 0 pgmap v1731: 89 pgs: 89 active+clean; 5.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 60 MiB/s wr, 956 op/s 2023-10-01T11:10:56.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:56.125 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:10:56.125 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:55 smithi078 bash[78274]: cluster 2023-10-01T11:10:54.411147+0000 mgr.y (mgr.25072) 1609 : cluster 0 pgmap v1731: 89 pgs: 89 active+clean; 5.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 60 MiB/s wr, 956 op/s 2023-10-01T11:10:56.226 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:56.258 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:56.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:56.266 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:10:56.362 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 74 16 92858 92842 78.4018 49.875 0.00826278 0.0127493 2023-10-01T11:10:56.367 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:56.389 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:56.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:56.391 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:56.492 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:56.518 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:56.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:56.522 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:10:56.623 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:56.657 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:56.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:56.660 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:10:56.760 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:56.803 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:56.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:56.805 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:10:56.906 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:56.937 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:56.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:56.944 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:10:57.045 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:57.071 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:57.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:57.076 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:10:57.176 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:57.205 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:57.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:57.210 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:10:57.311 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:57.336 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:57.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:57.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:57.362 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 75 16 93706 93690 78.0631 53 0.0153213 0.0128044 2023-10-01T11:10:57.440 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:57.471 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:10:57.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:57.477 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:10:57.578 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:57.601 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:57.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:57.604 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:10:57.705 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:57.731 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:57.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:57.734 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:10:57.835 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:57.869 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:57.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:57.874 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:10:57.975 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:58.006 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:58.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:58.009 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:10:58.110 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:58.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:57 smithi173 bash[58816]: cluster 2023-10-01T11:10:56.411719+0000 mgr.y (mgr.25072) 1610 : cluster 0 pgmap v1732: 89 pgs: 89 active+clean; 5.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 51 MiB/s wr, 816 op/s 2023-10-01T11:10:58.134 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:58.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:58.137 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:58.172 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:57 smithi078 bash[75299]: cluster 2023-10-01T11:10:56.411719+0000 mgr.y (mgr.25072) 1610 : cluster 0 pgmap v1732: 89 pgs: 89 active+clean; 5.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 51 MiB/s wr, 816 op/s 2023-10-01T11:10:58.172 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:57 smithi078 bash[78274]: cluster 2023-10-01T11:10:56.411719+0000 mgr.y (mgr.25072) 1610 : cluster 0 pgmap v1732: 89 pgs: 89 active+clean; 5.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 51 MiB/s wr, 816 op/s 2023-10-01T11:10:58.237 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:58.270 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:58.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:58.274 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:10:58.362 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 76 16 94571 94555 77.7473 54.0625 0.0183369 0.012856 2023-10-01T11:10:58.376 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:58.411 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:58.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:58.418 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:58.519 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:58.545 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:58.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:58.547 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:10:58.648 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:58.668 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:58.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:58.670 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:10:58.771 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:58.805 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:58.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:58.807 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:10:58.908 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:58.938 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:10:58.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:58.942 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:10:59.043 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:59.075 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:10:59.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:59.080 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:10:59.181 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:59.210 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:10:59.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:59.217 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:10:59.317 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:59.341 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:59.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:59.344 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:59.362 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 77 16 95390 95374 77.4023 51.1875 0.0303486 0.0129135 2023-10-01T11:10:59.445 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:59.472 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:10:59.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:59.474 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:10:59.575 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:59.600 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:10:59.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:59.606 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:10:59.706 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:59.732 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:10:59.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:59.736 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:10:59.836 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:59.864 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T11:10:59.869 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:10:59.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:10:59.872 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:10:59.973 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:00.002 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:00.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:00.007 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:11:00.108 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:00.135 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:00.137 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:10:59 smithi173 bash[58816]: cluster 2023-10-01T11:10:58.412060+0000 mgr.y (mgr.25072) 1611 : cluster 0 pgmap v1733: 89 pgs: 89 active+clean; 5.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 44 MiB/s wr, 703 op/s 2023-10-01T11:11:00.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:00.138 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:11:00.238 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:00.247 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:10:59 smithi078 bash[75299]: cluster 2023-10-01T11:10:58.412060+0000 mgr.y (mgr.25072) 1611 : cluster 0 pgmap v1733: 89 pgs: 89 active+clean; 5.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 44 MiB/s wr, 703 op/s 2023-10-01T11:11:00.247 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:10:59 smithi078 bash[78274]: cluster 2023-10-01T11:10:58.412060+0000 mgr.y (mgr.25072) 1611 : cluster 0 pgmap v1733: 89 pgs: 89 active+clean; 5.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 44 MiB/s wr, 703 op/s 2023-10-01T11:11:00.267 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:00.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:00.272 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:11:00.363 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 78 16 96247 96231 77.0966 53.5625 0.0130571 0.0129637 2023-10-01T11:11:00.373 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:00.400 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:00.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:00.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:11:00.507 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:00.533 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:00.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:00.536 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:11:00.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:11:00 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:11:00] "GET /metrics HTTP/1.1" 200 36380 "" "Prometheus/2.33.4" 2023-10-01T11:11:00.637 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:00.658 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:00.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:00.661 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:11:00.762 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:00.789 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:00.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:00.793 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:11:00.893 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:00.916 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:00.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:00.919 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:11:01.019 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:01.039 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:01.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:01.041 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:11:01.142 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:01.176 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:01.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:01.182 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:11:01.283 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:01.307 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:01.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:01.310 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:11:01.363 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 79 16 96822 96806 76.5756 35.9375 0.0422554 0.0130535 2023-10-01T11:11:01.411 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:01.440 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:01.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:01.443 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:11:01.543 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:01.561 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:01.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:01.563 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:11:01.663 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:01.696 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:01.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:01.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:11:01.805 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:01.830 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:01.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:01.834 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:11:01.935 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:01.944 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:01 smithi078 bash[78274]: cluster 2023-10-01T11:11:00.412754+0000 mgr.y (mgr.25072) 1612 : cluster 0 pgmap v1734: 89 pgs: 89 active+clean; 5.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 62 MiB/s wr, 984 op/s 2023-10-01T11:11:01.944 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:01 smithi078 bash[75299]: cluster 2023-10-01T11:11:00.412754+0000 mgr.y (mgr.25072) 1612 : cluster 0 pgmap v1734: 89 pgs: 89 active+clean; 5.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 62 MiB/s wr, 984 op/s 2023-10-01T11:11:01.962 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:01.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:01.967 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:11:02.067 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:02.095 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:02.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:02.100 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:11:02.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:01 smithi173 bash[58816]: cluster 2023-10-01T11:11:00.412754+0000 mgr.y (mgr.25072) 1612 : cluster 0 pgmap v1734: 89 pgs: 89 active+clean; 5.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 62 MiB/s wr, 984 op/s 2023-10-01T11:11:02.200 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:02.225 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:02.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:02.229 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:11:02.330 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:02.354 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:02.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:02.359 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:11:02.363 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:2023-10-01T11:11:02.362692+0000 min lat: 0.00136692 max lat: 0.367786 avg lat: 0.0131154 2023-10-01T11:11:02.363 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-10-01T11:11:02.363 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 80 16 97585 97569 76.2145 47.6875 0.0214835 0.0131154 2023-10-01T11:11:02.459 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:02.483 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:02.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:02.486 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:11:02.586 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:02.615 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:02.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:02.619 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:11:02.720 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:02.747 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:02.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:02.752 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:11:02.788 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T11:11:02.853 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:02.854 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T11:11:02.890 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T11:11:02.902 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:02.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:02.906 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:11:02.936 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T11:11:02.987 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T11:11:03.007 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:03.016 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:02 smithi078 bash[75299]: audit 2023-10-01T11:11:02.466956+0000 mon.a (mon.0) 2077 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:11:03.016 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:02 smithi078 bash[75299]: audit 2023-10-01T11:11:02.729114+0000 mon.a (mon.0) 2078 : audit 0 from='client.? 172.21.15.78:0/3420488929' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:11:03.016 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:02 smithi078 bash[78274]: audit 2023-10-01T11:11:02.466956+0000 mon.a (mon.0) 2077 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:11:03.017 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:02 smithi078 bash[78274]: audit 2023-10-01T11:11:02.729114+0000 mon.a (mon.0) 2078 : audit 0 from='client.? 172.21.15.78:0/3420488929' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:11:03.029 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T11:11:03.039 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:03.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:03.042 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:11:03.070 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T11:11:03.114 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T11:11:03.116 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T11:11:03.116 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T11:11:03.116 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T11:11:03.116 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T11:11:03.116 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T11:11:03.117 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T11:11:03.117 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T11:11:03.117 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T11:11:03.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:02 smithi173 bash[58816]: audit 2023-10-01T11:11:02.466956+0000 mon.a (mon.0) 2077 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:11:03.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:02 smithi173 bash[58816]: audit 2023-10-01T11:11:02.729114+0000 mon.a (mon.0) 2078 : audit 0 from='client.? 172.21.15.78:0/3420488929' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:11:03.143 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:03.195 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:03.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:03.200 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:11:03.300 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:03.325 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:03.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:03.332 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:11:03.363 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 81 16 98197 98181 75.7457 38.25 0.0238643 0.0131952 2023-10-01T11:11:03.434 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:03.458 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:03.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:03.462 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:11:03.563 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:03.588 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:03.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:03.591 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:11:03.692 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:03.720 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:03.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:03.726 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:11:03.826 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:03.854 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:03.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:03.860 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:11:03.923 INFO:teuthology.orchestra.run.smithi078.stderr:reweighted osd.0 to 1 (10000) 2023-10-01T11:11:03.923 INFO:teuthology.orchestra.run.smithi078.stderr:nodeep-scrub is set 2023-10-01T11:11:03.961 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:03.971 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:03 smithi173 bash[58816]: cluster 2023-10-01T11:11:02.413253+0000 mgr.y (mgr.25072) 1613 : cluster 0 pgmap v1735: 89 pgs: 89 active+clean; 5.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 47 MiB/s wr, 749 op/s 2023-10-01T11:11:03.971 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:03 smithi173 bash[58816]: audit 2023-10-01T11:11:03.294724+0000 mon.c (mon.2) 239 : audit 1 from='client.? 172.21.15.78:0/3626843067' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:11:03.971 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:03 smithi173 bash[58816]: audit 2023-10-01T11:11:03.295712+0000 mon.a (mon.0) 2079 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:11:03.971 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:03 smithi173 bash[58816]: audit 2023-10-01T11:11:03.474868+0000 mon.a (mon.0) 2080 : audit 1 from='client.? 172.21.15.78:0/1419802328' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T11:11:03.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:03.991 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:03.992 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:11:04.092 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:04.119 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:04.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:04.124 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:11:04.225 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:04.257 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:04.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:04.262 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:11:04.347 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:03 smithi078 bash[75299]: cluster 2023-10-01T11:11:02.413253+0000 mgr.y (mgr.25072) 1613 : cluster 0 pgmap v1735: 89 pgs: 89 active+clean; 5.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 47 MiB/s wr, 749 op/s 2023-10-01T11:11:04.348 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:03 smithi078 bash[75299]: audit 2023-10-01T11:11:03.294724+0000 mon.c (mon.2) 239 : audit 1 from='client.? 172.21.15.78:0/3626843067' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:11:04.348 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:03 smithi078 bash[75299]: audit 2023-10-01T11:11:03.295712+0000 mon.a (mon.0) 2079 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:11:04.348 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:03 smithi078 bash[75299]: audit 2023-10-01T11:11:03.474868+0000 mon.a (mon.0) 2080 : audit 1 from='client.? 172.21.15.78:0/1419802328' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T11:11:04.348 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:03 smithi078 bash[78274]: cluster 2023-10-01T11:11:02.413253+0000 mgr.y (mgr.25072) 1613 : cluster 0 pgmap v1735: 89 pgs: 89 active+clean; 5.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 47 MiB/s wr, 749 op/s 2023-10-01T11:11:04.348 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:03 smithi078 bash[78274]: audit 2023-10-01T11:11:03.294724+0000 mon.c (mon.2) 239 : audit 1 from='client.? 172.21.15.78:0/3626843067' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:11:04.348 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:03 smithi078 bash[78274]: audit 2023-10-01T11:11:03.295712+0000 mon.a (mon.0) 2079 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:11:04.348 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:03 smithi078 bash[78274]: audit 2023-10-01T11:11:03.474868+0000 mon.a (mon.0) 2080 : audit 1 from='client.? 172.21.15.78:0/1419802328' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T11:11:04.363 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:04.363 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 82 16 98960 98944 75.4035 47.6875 0.0182242 0.0132561 2023-10-01T11:11:04.391 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:04.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:04.396 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:11:04.496 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:04.534 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:04.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:04.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:11:04.639 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:04.666 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:04.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:04.671 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:11:04.772 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:04.802 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:04.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:04.808 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:11:04.908 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:04.935 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:04.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:04.939 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:11:05.040 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:05.066 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:05.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:05.071 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:04 smithi173 bash[58816]: cluster 2023-10-01T11:11:03.889208+0000 mon.a (mon.0) 2081 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:11:05.071 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:04 smithi173 bash[58816]: cluster 2023-10-01T11:11:03.912249+0000 osd.7 (osd.7) 406 : cluster 3 failed to encode map e362 with expected crc 2023-10-01T11:11:05.071 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:04 smithi173 bash[58816]: cluster 2023-10-01T11:11:03.913785+0000 osd.5 (osd.5) 384 : cluster 3 failed to encode map e362 with expected crc 2023-10-01T11:11:05.071 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:04 smithi173 bash[58816]: audit 2023-10-01T11:11:03.916685+0000 mon.a (mon.0) 2082 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T11:11:05.071 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:04 smithi173 bash[58816]: audit 2023-10-01T11:11:03.916759+0000 mon.a (mon.0) 2083 : audit 1 from='client.? 172.21.15.78:0/1419802328' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-01T11:11:05.071 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:04 smithi173 bash[58816]: cluster 2023-10-01T11:11:03.916856+0000 mon.a (mon.0) 2084 : cluster 0 osdmap e362: 8 total, 8 up, 8 in 2023-10-01T11:11:05.071 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:04 smithi173 bash[58816]: cluster 2023-10-01T11:11:03.923792+0000 osd.6 (osd.6) 347 : cluster 3 failed to encode map e362 with expected crc 2023-10-01T11:11:05.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:11:05.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:04 smithi078 bash[75299]: cluster 2023-10-01T11:11:03.889208+0000 mon.a (mon.0) 2081 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:11:05.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:04 smithi078 bash[75299]: cluster 2023-10-01T11:11:03.912249+0000 osd.7 (osd.7) 406 : cluster 3 failed to encode map e362 with expected crc 2023-10-01T11:11:05.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:04 smithi078 bash[75299]: cluster 2023-10-01T11:11:03.913785+0000 osd.5 (osd.5) 384 : cluster 3 failed to encode map e362 with expected crc 2023-10-01T11:11:05.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:04 smithi078 bash[75299]: audit 2023-10-01T11:11:03.916685+0000 mon.a (mon.0) 2082 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T11:11:05.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:04 smithi078 bash[75299]: audit 2023-10-01T11:11:03.916759+0000 mon.a (mon.0) 2083 : audit 1 from='client.? 172.21.15.78:0/1419802328' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-01T11:11:05.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:04 smithi078 bash[75299]: cluster 2023-10-01T11:11:03.916856+0000 mon.a (mon.0) 2084 : cluster 0 osdmap e362: 8 total, 8 up, 8 in 2023-10-01T11:11:05.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:04 smithi078 bash[75299]: cluster 2023-10-01T11:11:03.923792+0000 osd.6 (osd.6) 347 : cluster 3 failed to encode map e362 with expected crc 2023-10-01T11:11:05.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:04 smithi078 bash[78274]: cluster 2023-10-01T11:11:03.889208+0000 mon.a (mon.0) 2081 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:11:05.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:04 smithi078 bash[78274]: cluster 2023-10-01T11:11:03.912249+0000 osd.7 (osd.7) 406 : cluster 3 failed to encode map e362 with expected crc 2023-10-01T11:11:05.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:04 smithi078 bash[78274]: cluster 2023-10-01T11:11:03.913785+0000 osd.5 (osd.5) 384 : cluster 3 failed to encode map e362 with expected crc 2023-10-01T11:11:05.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:04 smithi078 bash[78274]: audit 2023-10-01T11:11:03.916685+0000 mon.a (mon.0) 2082 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T11:11:05.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:04 smithi078 bash[78274]: audit 2023-10-01T11:11:03.916759+0000 mon.a (mon.0) 2083 : audit 1 from='client.? 172.21.15.78:0/1419802328' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-01T11:11:05.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:04 smithi078 bash[78274]: cluster 2023-10-01T11:11:03.916856+0000 mon.a (mon.0) 2084 : cluster 0 osdmap e362: 8 total, 8 up, 8 in 2023-10-01T11:11:05.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:04 smithi078 bash[78274]: cluster 2023-10-01T11:11:03.923792+0000 osd.6 (osd.6) 347 : cluster 3 failed to encode map e362 with expected crc 2023-10-01T11:11:05.173 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:05.204 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:05.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:05.207 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:11:05.307 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:05.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:05.333 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:05.333 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:11:05.363 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 83 16 99778 99762 75.111 51.125 0.00481549 0.0133082 2023-10-01T11:11:05.434 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:05.461 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:05.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:05.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:11:05.570 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:05.597 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:05.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:05.601 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:11:05.701 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:05.726 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:05.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:05.729 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:11:05.830 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:05.854 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:05.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:05.862 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:11:05.963 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:05.996 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:05.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:05.998 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:11:06.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:05 smithi078 bash[78274]: cluster 2023-10-01T11:11:04.413797+0000 mgr.y (mgr.25072) 1614 : cluster 0 pgmap v1737: 89 pgs: 89 active+clean; 6.0 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 50 MiB/s wr, 794 op/s 2023-10-01T11:11:06.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:05 smithi078 bash[75299]: cluster 2023-10-01T11:11:04.413797+0000 mgr.y (mgr.25072) 1614 : cluster 0 pgmap v1737: 89 pgs: 89 active+clean; 6.0 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 50 MiB/s wr, 794 op/s 2023-10-01T11:11:06.099 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:06.126 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:06.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:06.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:11:06.235 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:06.242 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:05 smithi173 bash[58816]: cluster 2023-10-01T11:11:04.413797+0000 mgr.y (mgr.25072) 1614 : cluster 0 pgmap v1737: 89 pgs: 89 active+clean; 6.0 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 50 MiB/s wr, 794 op/s 2023-10-01T11:11:06.258 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:06.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:06.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:11:06.362 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:06.363 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 84 16 100575 100559 74.8098 49.8125 0.0172698 0.0133617 2023-10-01T11:11:06.389 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:06.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:06.393 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:11:06.478 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-01T11:11:06.493 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:06.539 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T11:11:06.563 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:06.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:06.571 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:11:06.671 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:06.697 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:06.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:06.704 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:11:06.805 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:06.827 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:06.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:06.829 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:11:06.930 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:06.955 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:06.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:06.959 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:11:07.060 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:07.085 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:07.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:07.090 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:11:07.191 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:07.221 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:07.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:07.224 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:11:07.325 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:07.353 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:07.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:07.357 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:11:07.363 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 85 16 101337 101321 74.4899 47.625 0.0185562 0.0134185 2023-10-01T11:11:07.458 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:07.483 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:07.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:07.488 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:11:07.588 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:07.626 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:07.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:07.628 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:11:07.686 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-10-01T11:11:07.686 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T11:11:07.729 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:07.752 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:07.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:07.757 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:11:07.858 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:07.899 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:07.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:07.904 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:11:08.005 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:08.032 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:08.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:08.036 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:11:08.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:07 smithi078 bash[78274]: cluster 2023-10-01T11:11:06.414270+0000 mgr.y (mgr.25072) 1615 : cluster 0 pgmap v1738: 89 pgs: 89 active+clean; 6.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 48 MiB/s wr, 768 op/s 2023-10-01T11:11:08.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:07 smithi078 bash[75299]: cluster 2023-10-01T11:11:06.414270+0000 mgr.y (mgr.25072) 1615 : cluster 0 pgmap v1738: 89 pgs: 89 active+clean; 6.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 48 MiB/s wr, 768 op/s 2023-10-01T11:11:08.137 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:08.148 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:07 smithi173 bash[58816]: cluster 2023-10-01T11:11:06.414270+0000 mgr.y (mgr.25072) 1615 : cluster 0 pgmap v1738: 89 pgs: 89 active+clean; 6.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 48 MiB/s wr, 768 op/s 2023-10-01T11:11:08.165 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:08.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:08.170 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:11:08.270 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:08.299 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:08.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:08.303 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:11:08.363 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 86 16 102105 102089 74.1819 48 0.0170482 0.0134738 2023-10-01T11:11:08.404 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:08.435 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T11:11:08.440 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:08.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:08.447 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:11:08.548 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:08.575 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:08.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:08.577 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:11:08.678 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:08.703 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:08.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:08.709 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:11:08.810 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:08.835 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:08.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:08.838 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:11:08.872 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-01T11:11:08.939 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:08.969 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:08.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:08.974 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:11:09.075 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:09.122 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:09.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:09.124 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:11:09.225 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:09.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:09.266 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:09.266 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:11:09.365 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 87 16 102842 102826 73.8586 46.0625 0.0371638 0.0135327 2023-10-01T11:11:09.366 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:09.394 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:09.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:09.397 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:11:09.497 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:09.540 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:09.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:09.543 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:11:09.643 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:09.675 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:09.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:09.679 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:11:09.779 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:09.819 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:09.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:09.826 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:11:09.927 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:09.955 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:09.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:09.958 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:11:10.059 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:10.087 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:10.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:10.092 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:11:10.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:09 smithi173 bash[58816]: cluster 2023-10-01T11:11:08.414756+0000 mgr.y (mgr.25072) 1616 : cluster 0 pgmap v1739: 89 pgs: 89 active+clean; 6.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 48 MiB/s wr, 768 op/s 2023-10-01T11:11:10.193 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:10.222 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:10.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:10.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:11:10.270 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:09 smithi078 bash[75299]: cluster 2023-10-01T11:11:08.414756+0000 mgr.y (mgr.25072) 1616 : cluster 0 pgmap v1739: 89 pgs: 89 active+clean; 6.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 48 MiB/s wr, 768 op/s 2023-10-01T11:11:10.271 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:11:10 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:11:10] "GET /metrics HTTP/1.1" 200 36378 "" "Prometheus/2.33.4" 2023-10-01T11:11:10.271 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:09 smithi078 bash[78274]: cluster 2023-10-01T11:11:08.414756+0000 mgr.y (mgr.25072) 1616 : cluster 0 pgmap v1739: 89 pgs: 89 active+clean; 6.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 48 MiB/s wr, 768 op/s 2023-10-01T11:11:10.325 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:10.350 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:10.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:10.353 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:11:10.363 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 88 16 103611 103595 73.5655 48.0625 0.0129624 0.0135863 2023-10-01T11:11:10.454 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:10.481 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:10.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:10.488 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:11:10.589 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:10.615 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:10.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:10.618 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:11:10.719 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:10.758 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:10.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:10.763 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:11:10.863 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:10.903 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:10.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:10.905 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:11:11.006 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:11.039 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:11.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:11.044 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:11:11.145 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:11.177 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:11.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:11.181 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:11:11.281 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:11.315 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:11.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:11.322 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:11:11.364 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 89 16 104371 104355 73.2726 47.5 0.0121818 0.0136413 2023-10-01T11:11:11.376 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:11:11.377 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:11:11.422 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:11.459 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:11.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:11.470 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:11:11.571 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:11.603 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:11.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:11.611 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:11:11.712 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:11.738 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:11.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:11.741 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:11:11.841 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:11.869 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:11.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:11.871 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:11:11.972 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:12.001 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:12.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:12.006 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:11:12.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:11 smithi078 bash[75299]: cluster 2023-10-01T11:11:10.415475+0000 mgr.y (mgr.25072) 1617 : cluster 0 pgmap v1740: 89 pgs: 89 active+clean; 6.2 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 47 MiB/s wr, 750 op/s 2023-10-01T11:11:12.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:11 smithi078 bash[78274]: cluster 2023-10-01T11:11:10.415475+0000 mgr.y (mgr.25072) 1617 : cluster 0 pgmap v1740: 89 pgs: 89 active+clean; 6.2 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 47 MiB/s wr, 750 op/s 2023-10-01T11:11:12.107 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:12.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:11 smithi173 bash[58816]: cluster 2023-10-01T11:11:10.415475+0000 mgr.y (mgr.25072) 1617 : cluster 0 pgmap v1740: 89 pgs: 89 active+clean; 6.2 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 47 MiB/s wr, 750 op/s 2023-10-01T11:11:12.136 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:12.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:12.140 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:11:12.241 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:12.266 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:12.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:12.273 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:11:12.364 INFO:tasks.radosbench.radosbench.0.smithi078.stdout: 90 4 105143 105139 73.0028 49 0.023873 0.0136929 2023-10-01T11:11:12.373 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:12.396 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:11:12.397 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:12.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:12.399 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:11:12.500 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:12.527 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:12.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:12.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:11:12.631 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:12.660 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:12.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:12.663 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:11:12.763 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:12.796 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:12.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:12.800 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:11:12.900 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:12.931 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:12.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:12.933 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:11:13.034 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:13.066 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:13.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:13.069 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:11:13.170 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:13.198 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:13.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:13.202 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:11:13.303 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:13.332 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:13.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:13.336 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:11:13.364 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Total time run: 90.0149 2023-10-01T11:11:13.364 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Total writes made: 105143 2023-10-01T11:11:13.364 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Write size: 65536 2023-10-01T11:11:13.364 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Object size: 65536 2023-10-01T11:11:13.364 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Bandwidth (MB/sec): 73.0039 2023-10-01T11:11:13.365 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Stddev Bandwidth: 30.6326 2023-10-01T11:11:13.365 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Max bandwidth (MB/sec): 167.938 2023-10-01T11:11:13.365 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Min bandwidth (MB/sec): 35.9375 2023-10-01T11:11:13.365 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Average IOPS: 1168 2023-10-01T11:11:13.365 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Stddev IOPS: 490.122 2023-10-01T11:11:13.365 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Max IOPS: 2687 2023-10-01T11:11:13.365 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Min IOPS: 575 2023-10-01T11:11:13.365 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Average Latency(s): 0.013693 2023-10-01T11:11:13.365 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Stddev Latency(s): 0.0126674 2023-10-01T11:11:13.365 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Max latency(s): 0.367786 2023-10-01T11:11:13.365 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Min latency(s): 0.00136692 2023-10-01T11:11:13.375 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Cleaning up (deleting benchmark objects) 2023-10-01T11:11:13.437 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:13.461 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:13.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:13.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:11:13.565 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:13.596 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:13.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:13.599 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:11:13.700 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:13.721 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T11:11:13.726 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:13.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:13.736 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:11:13.837 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:13.864 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:13.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:13.866 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:11:13.967 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:14.005 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:14.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:14.008 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:11:14.109 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:14.116 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:13 smithi173 bash[58816]: cluster 2023-10-01T11:11:12.415963+0000 mgr.y (mgr.25072) 1618 : cluster 0 pgmap v1741: 89 pgs: 89 active+clean; 6.3 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 47 MiB/s wr, 752 op/s 2023-10-01T11:11:14.128 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:14.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:14.131 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:11:14.198 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:13 smithi078 bash[78274]: cluster 2023-10-01T11:11:12.415963+0000 mgr.y (mgr.25072) 1618 : cluster 0 pgmap v1741: 89 pgs: 89 active+clean; 6.3 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 47 MiB/s wr, 752 op/s 2023-10-01T11:11:14.199 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:13 smithi078 bash[75299]: cluster 2023-10-01T11:11:12.415963+0000 mgr.y (mgr.25072) 1618 : cluster 0 pgmap v1741: 89 pgs: 89 active+clean; 6.3 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 47 MiB/s wr, 752 op/s 2023-10-01T11:11:14.231 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:14.262 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:14.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:14.265 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:11:14.366 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:14.390 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:14.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:14.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:11:14.495 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:14.524 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:14.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:14.534 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:11:14.635 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:14.658 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:14.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:14.662 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:11:14.763 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:14.785 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:14.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:14.788 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:11:14.888 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:14.919 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:14.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:14.921 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:11:15.022 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:15.047 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:15.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:15.049 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:11:15.150 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:15.174 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:15.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:15.179 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:11:15.280 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:15.302 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:15.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:15.306 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:11:15.407 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:15.435 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:15.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:15.437 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:11:15.538 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:15.581 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:15.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:15.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:11:15.684 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:15.719 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:15.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:15.723 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:11:15.823 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:15.859 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:15.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:15.863 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:11:15.963 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:15.986 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:15 smithi173 bash[58816]: cluster 2023-10-01T11:11:14.416587+0000 mgr.y (mgr.25072) 1619 : cluster 0 pgmap v1742: 89 pgs: 89 active+clean; 6.4 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 97 B/s rd, 45 MiB/s wr, 742 op/s 2023-10-01T11:11:15.990 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:15.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:15.993 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:11:16.093 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:16.117 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:16.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:16.124 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:11:16.225 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:16.233 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:15 smithi078 bash[78274]: cluster 2023-10-01T11:11:14.416587+0000 mgr.y (mgr.25072) 1619 : cluster 0 pgmap v1742: 89 pgs: 89 active+clean; 6.4 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 97 B/s rd, 45 MiB/s wr, 742 op/s 2023-10-01T11:11:16.234 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:15 smithi078 bash[75299]: cluster 2023-10-01T11:11:14.416587+0000 mgr.y (mgr.25072) 1619 : cluster 0 pgmap v1742: 89 pgs: 89 active+clean; 6.4 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 97 B/s rd, 45 MiB/s wr, 742 op/s 2023-10-01T11:11:16.255 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:16.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:16.259 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:11:16.359 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:16.385 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:16.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:16.391 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:11:16.491 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:16.522 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:16.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:16.527 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:11:16.628 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:16.655 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:16.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:16.658 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:11:16.758 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:16.805 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:16.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:16.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:11:16.912 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:16.948 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:16.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:16.952 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:11:17.052 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:17.081 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:17.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:17.086 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:11:17.186 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:17.213 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:17.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:17.218 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:11:17.319 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:17.349 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:17.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:17.352 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:11:17.452 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:17.486 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:17.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:17.489 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:11:17.589 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:17.615 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:17.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:17.618 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:11:17.719 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:17.770 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:17.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:17.775 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:11:17.875 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:17.909 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:17.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:17.915 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:11:18.015 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:18.039 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:18.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:18.041 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:11:18.142 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:18.172 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:18.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:18.176 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:11:18.212 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:17 smithi173 bash[58816]: cluster 2023-10-01T11:11:16.417120+0000 mgr.y (mgr.25072) 1620 : cluster 0 pgmap v1743: 89 pgs: 89 active+clean; 6.4 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 85 B/s rd, 38 MiB/s wr, 673 op/s 2023-10-01T11:11:18.212 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:17 smithi173 bash[58816]: audit 2023-10-01T11:11:17.467230+0000 mon.a (mon.0) 2085 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:11:18.277 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:18.285 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:17 smithi078 bash[78274]: cluster 2023-10-01T11:11:16.417120+0000 mgr.y (mgr.25072) 1620 : cluster 0 pgmap v1743: 89 pgs: 89 active+clean; 6.4 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 85 B/s rd, 38 MiB/s wr, 673 op/s 2023-10-01T11:11:18.285 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:17 smithi078 bash[78274]: audit 2023-10-01T11:11:17.467230+0000 mon.a (mon.0) 2085 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:11:18.286 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:17 smithi078 bash[75299]: cluster 2023-10-01T11:11:16.417120+0000 mgr.y (mgr.25072) 1620 : cluster 0 pgmap v1743: 89 pgs: 89 active+clean; 6.4 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 85 B/s rd, 38 MiB/s wr, 673 op/s 2023-10-01T11:11:18.286 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:17 smithi078 bash[75299]: audit 2023-10-01T11:11:17.467230+0000 mon.a (mon.0) 2085 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:11:18.307 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:18.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:18.313 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:11:18.414 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:18.458 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:18.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:18.464 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:11:18.518 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-10-01T11:11:18.518 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T11:11:18.565 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:18.606 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:18.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:18.611 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:11:18.712 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:18.737 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:18.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:18.743 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:11:18.844 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:18.868 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:18.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:18.873 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:11:18.974 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:18.997 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:18.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:18.999 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:11:19.100 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:19.132 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:19.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:19.134 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:11:19.235 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:19.269 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:19.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:19.276 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:11:19.377 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:19.404 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:19.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:19.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:11:19.507 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:19.528 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:19.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:19.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:11:19.632 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:19.655 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:19.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:19.659 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:11:19.760 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:19.793 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:19.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:19.799 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:11:19.900 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:19.925 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:19.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:19.927 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:11:19.983 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-01T11:11:20.029 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:20.064 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:20.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:20.066 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:11:20.167 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:20.176 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:20 smithi078 bash[75299]: cluster 2023-10-01T11:11:18.417542+0000 mgr.y (mgr.25072) 1621 : cluster 0 pgmap v1744: 89 pgs: 89 active+clean; 6.4 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 85 B/s rd, 33 MiB/s wr, 584 op/s 2023-10-01T11:11:20.177 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:20 smithi078 bash[78274]: cluster 2023-10-01T11:11:18.417542+0000 mgr.y (mgr.25072) 1621 : cluster 0 pgmap v1744: 89 pgs: 89 active+clean; 6.4 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 85 B/s rd, 33 MiB/s wr, 584 op/s 2023-10-01T11:11:20.196 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:20.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:20.207 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:11:20.308 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:20.316 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:19 smithi173 bash[58816]: cluster 2023-10-01T11:11:18.417542+0000 mgr.y (mgr.25072) 1621 : cluster 0 pgmap v1744: 89 pgs: 89 active+clean; 6.4 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 85 B/s rd, 33 MiB/s wr, 584 op/s 2023-10-01T11:11:20.333 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:20.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:20.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:11:20.440 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:20.448 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:11:20 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:11:20] "GET /metrics HTTP/1.1" 200 36385 "" "Prometheus/2.33.4" 2023-10-01T11:11:20.461 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:20.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:20.463 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:11:20.564 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:20.593 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:20.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:20.596 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:11:20.697 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:20.719 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:20.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:20.721 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:11:20.822 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:20.860 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:20.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:20.864 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:11:20.964 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:20.992 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:20.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:20.998 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:11:21.098 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:21.124 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:21.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:21.126 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:11:21.227 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:21.262 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:21.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:21.264 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:11:21.364 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:21.410 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:21.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:21.412 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:11:21.513 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:21.561 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:21.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:21.564 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:11:21.664 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:21.695 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:21.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:21.697 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:11:21.760 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T11:11:21.798 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:21.812 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T11:11:21.825 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:21.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:21.828 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:11:21.860 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T11:11:21.906 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T11:11:21.928 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:21.948 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T11:11:21.957 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:21.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:21.961 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:11:21.975 INFO:teuthology.orchestra.run.smithi078.stderr:reweighted osd.1 to 1 (10000) 2023-10-01T11:11:21.993 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T11:11:22.031 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T11:11:22.061 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T11:11:22.062 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:22.075 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T11:11:22.075 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T11:11:22.075 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T11:11:22.075 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T11:11:22.075 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T11:11:22.075 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T11:11:22.075 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T11:11:22.075 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T11:11:22.104 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:22.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:22.108 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:11:22.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:21 smithi173 bash[58816]: cluster 2023-10-01T11:11:20.418226+0000 mgr.y (mgr.25072) 1622 : cluster 0 pgmap v1745: 89 pgs: 89 active+clean; 6.2 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 85 B/s rd, 33 MiB/s wr, 860 op/s 2023-10-01T11:11:22.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:21 smithi173 bash[58816]: audit 2023-10-01T11:11:21.694832+0000 mon.a (mon.0) 2086 : audit 0 from='client.? 172.21.15.78:0/1471753399' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:11:22.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:21 smithi173 bash[58816]: audit 2023-10-01T11:11:21.838097+0000 mon.a (mon.0) 2087 : audit 1 from='client.? 172.21.15.78:0/3278650216' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T11:11:22.209 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:22.225 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:21 smithi078 bash[75299]: cluster 2023-10-01T11:11:20.418226+0000 mgr.y (mgr.25072) 1622 : cluster 0 pgmap v1745: 89 pgs: 89 active+clean; 6.2 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 85 B/s rd, 33 MiB/s wr, 860 op/s 2023-10-01T11:11:22.226 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:21 smithi078 bash[75299]: audit 2023-10-01T11:11:21.694832+0000 mon.a (mon.0) 2086 : audit 0 from='client.? 172.21.15.78:0/1471753399' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:11:22.226 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:21 smithi078 bash[75299]: audit 2023-10-01T11:11:21.838097+0000 mon.a (mon.0) 2087 : audit 1 from='client.? 172.21.15.78:0/3278650216' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T11:11:22.226 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:21 smithi078 bash[78274]: cluster 2023-10-01T11:11:20.418226+0000 mgr.y (mgr.25072) 1622 : cluster 0 pgmap v1745: 89 pgs: 89 active+clean; 6.2 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 85 B/s rd, 33 MiB/s wr, 860 op/s 2023-10-01T11:11:22.226 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:21 smithi078 bash[78274]: audit 2023-10-01T11:11:21.694832+0000 mon.a (mon.0) 2086 : audit 0 from='client.? 172.21.15.78:0/1471753399' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:11:22.227 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:21 smithi078 bash[78274]: audit 2023-10-01T11:11:21.838097+0000 mon.a (mon.0) 2087 : audit 1 from='client.? 172.21.15.78:0/3278650216' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T11:11:22.232 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:22.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:22.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:11:22.338 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:22.370 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:22.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:22.375 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:11:22.475 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:22.508 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:22.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:22.515 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:11:22.616 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:22.651 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:22.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:22.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:11:22.754 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:22.782 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:22.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:22.785 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:11:22.885 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:22.911 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:22.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:22.914 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:11:23.009 INFO:teuthology.orchestra.run.smithi078.stderr:noscrub is unset 2023-10-01T11:11:23.015 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:23.044 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:23.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:23.049 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:11:23.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:22 smithi173 bash[58816]: audit 2023-10-01T11:11:21.962775+0000 mon.a (mon.0) 2088 : audit 1 from='client.? 172.21.15.78:0/3278650216' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-01T11:11:23.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:22 smithi173 bash[58816]: cluster 2023-10-01T11:11:21.962890+0000 mon.a (mon.0) 2089 : cluster 0 osdmap e363: 8 total, 8 up, 8 in 2023-10-01T11:11:23.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:22 smithi173 bash[58816]: cluster 2023-10-01T11:11:21.963344+0000 osd.7 (osd.7) 407 : cluster 3 failed to encode map e363 with expected crc 2023-10-01T11:11:23.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:22 smithi173 bash[58816]: cluster 2023-10-01T11:11:21.968067+0000 osd.6 (osd.6) 348 : cluster 3 failed to encode map e363 with expected crc 2023-10-01T11:11:23.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:22 smithi173 bash[58816]: cluster 2023-10-01T11:11:21.972913+0000 osd.5 (osd.5) 385 : cluster 3 failed to encode map e363 with expected crc 2023-10-01T11:11:23.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:22 smithi173 bash[58816]: audit 2023-10-01T11:11:22.159833+0000 mon.b (mon.1) 59 : audit 1 from='client.? 172.21.15.78:0/3515743826' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T11:11:23.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:22 smithi173 bash[58816]: audit 2023-10-01T11:11:22.160187+0000 mon.a (mon.0) 2090 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T11:11:23.150 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:23.178 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:23.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:23.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:11:23.283 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:23.317 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:23.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:23.321 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:11:23.341 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:22 smithi078 bash[75299]: audit 2023-10-01T11:11:21.962775+0000 mon.a (mon.0) 2088 : audit 1 from='client.? 172.21.15.78:0/3278650216' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-01T11:11:23.341 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:22 smithi078 bash[75299]: cluster 2023-10-01T11:11:21.962890+0000 mon.a (mon.0) 2089 : cluster 0 osdmap e363: 8 total, 8 up, 8 in 2023-10-01T11:11:23.341 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:22 smithi078 bash[75299]: cluster 2023-10-01T11:11:21.963344+0000 osd.7 (osd.7) 407 : cluster 3 failed to encode map e363 with expected crc 2023-10-01T11:11:23.341 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:22 smithi078 bash[75299]: cluster 2023-10-01T11:11:21.968067+0000 osd.6 (osd.6) 348 : cluster 3 failed to encode map e363 with expected crc 2023-10-01T11:11:23.341 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:22 smithi078 bash[75299]: cluster 2023-10-01T11:11:21.972913+0000 osd.5 (osd.5) 385 : cluster 3 failed to encode map e363 with expected crc 2023-10-01T11:11:23.342 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:22 smithi078 bash[75299]: audit 2023-10-01T11:11:22.159833+0000 mon.b (mon.1) 59 : audit 1 from='client.? 172.21.15.78:0/3515743826' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T11:11:23.342 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:22 smithi078 bash[75299]: audit 2023-10-01T11:11:22.160187+0000 mon.a (mon.0) 2090 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T11:11:23.342 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:22 smithi078 bash[78274]: audit 2023-10-01T11:11:21.962775+0000 mon.a (mon.0) 2088 : audit 1 from='client.? 172.21.15.78:0/3278650216' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-01T11:11:23.342 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:22 smithi078 bash[78274]: cluster 2023-10-01T11:11:21.962890+0000 mon.a (mon.0) 2089 : cluster 0 osdmap e363: 8 total, 8 up, 8 in 2023-10-01T11:11:23.342 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:22 smithi078 bash[78274]: cluster 2023-10-01T11:11:21.963344+0000 osd.7 (osd.7) 407 : cluster 3 failed to encode map e363 with expected crc 2023-10-01T11:11:23.342 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:22 smithi078 bash[78274]: cluster 2023-10-01T11:11:21.968067+0000 osd.6 (osd.6) 348 : cluster 3 failed to encode map e363 with expected crc 2023-10-01T11:11:23.342 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:22 smithi078 bash[78274]: cluster 2023-10-01T11:11:21.972913+0000 osd.5 (osd.5) 385 : cluster 3 failed to encode map e363 with expected crc 2023-10-01T11:11:23.342 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:22 smithi078 bash[78274]: audit 2023-10-01T11:11:22.159833+0000 mon.b (mon.1) 59 : audit 1 from='client.? 172.21.15.78:0/3515743826' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T11:11:23.342 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:22 smithi078 bash[78274]: audit 2023-10-01T11:11:22.160187+0000 mon.a (mon.0) 2090 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T11:11:23.422 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:23.452 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:23.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:23.459 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:11:23.560 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:23.586 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:23.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:23.589 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:11:23.690 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:23.732 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:23.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:23.734 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:11:23.835 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:23.857 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:23.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:23.859 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:11:23.960 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:23.995 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:23.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:23.998 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:11:24.098 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:24.137 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:24.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:24.143 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:11:24.244 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:24.255 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:23 smithi173 bash[58816]: cluster 2023-10-01T11:11:22.418635+0000 mgr.y (mgr.25072) 1623 : cluster 0 pgmap v1747: 89 pgs: 89 active+clean; 6.1 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 102 B/s rd, 14 MiB/s wr, 709 op/s 2023-10-01T11:11:24.255 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:23 smithi173 bash[58816]: cluster 2023-10-01T11:11:22.970434+0000 mon.a (mon.0) 2091 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:11:24.255 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:23 smithi173 bash[58816]: cluster 2023-10-01T11:11:22.984950+0000 osd.5 (osd.5) 386 : cluster 3 failed to encode map e364 with expected crc 2023-10-01T11:11:24.255 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:23 smithi173 bash[58816]: cluster 2023-10-01T11:11:22.986490+0000 osd.7 (osd.7) 408 : cluster 3 failed to encode map e364 with expected crc 2023-10-01T11:11:24.256 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:23 smithi173 bash[58816]: cluster 2023-10-01T11:11:22.988833+0000 osd.7 (osd.7) 409 : cluster 3 failed to encode map e364 with expected crc 2023-10-01T11:11:24.256 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:23 smithi173 bash[58816]: audit 2023-10-01T11:11:22.997528+0000 mon.a (mon.0) 2092 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T11:11:24.256 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:23 smithi173 bash[58816]: cluster 2023-10-01T11:11:22.997592+0000 mon.a (mon.0) 2093 : cluster 0 osdmap e364: 8 total, 8 up, 8 in 2023-10-01T11:11:24.256 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:23 smithi173 bash[58816]: cluster 2023-10-01T11:11:23.012326+0000 osd.6 (osd.6) 349 : cluster 3 failed to encode map e364 with expected crc 2023-10-01T11:11:24.256 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:23 smithi173 bash[58816]: cluster 2023-10-01T11:11:23.013761+0000 osd.6 (osd.6) 350 : cluster 3 failed to encode map e364 with expected crc 2023-10-01T11:11:24.275 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:24.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:24.281 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:11:24.382 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:24.391 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:24 smithi078 bash[75299]: cluster 2023-10-01T11:11:22.418635+0000 mgr.y (mgr.25072) 1623 : cluster 0 pgmap v1747: 89 pgs: 89 active+clean; 6.1 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 102 B/s rd, 14 MiB/s wr, 709 op/s 2023-10-01T11:11:24.391 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:24 smithi078 bash[75299]: cluster 2023-10-01T11:11:22.970434+0000 mon.a (mon.0) 2091 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:11:24.391 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:24 smithi078 bash[75299]: cluster 2023-10-01T11:11:22.984950+0000 osd.5 (osd.5) 386 : cluster 3 failed to encode map e364 with expected crc 2023-10-01T11:11:24.391 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:24 smithi078 bash[75299]: cluster 2023-10-01T11:11:22.986490+0000 osd.7 (osd.7) 408 : cluster 3 failed to encode map e364 with expected crc 2023-10-01T11:11:24.391 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:24 smithi078 bash[75299]: cluster 2023-10-01T11:11:22.988833+0000 osd.7 (osd.7) 409 : cluster 3 failed to encode map e364 with expected crc 2023-10-01T11:11:24.391 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:24 smithi078 bash[75299]: audit 2023-10-01T11:11:22.997528+0000 mon.a (mon.0) 2092 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T11:11:24.391 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:24 smithi078 bash[75299]: cluster 2023-10-01T11:11:22.997592+0000 mon.a (mon.0) 2093 : cluster 0 osdmap e364: 8 total, 8 up, 8 in 2023-10-01T11:11:24.391 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:24 smithi078 bash[75299]: cluster 2023-10-01T11:11:23.012326+0000 osd.6 (osd.6) 349 : cluster 3 failed to encode map e364 with expected crc 2023-10-01T11:11:24.392 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:24 smithi078 bash[75299]: cluster 2023-10-01T11:11:23.013761+0000 osd.6 (osd.6) 350 : cluster 3 failed to encode map e364 with expected crc 2023-10-01T11:11:24.392 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:24 smithi078 bash[78274]: cluster 2023-10-01T11:11:22.418635+0000 mgr.y (mgr.25072) 1623 : cluster 0 pgmap v1747: 89 pgs: 89 active+clean; 6.1 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 102 B/s rd, 14 MiB/s wr, 709 op/s 2023-10-01T11:11:24.392 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:24 smithi078 bash[78274]: cluster 2023-10-01T11:11:22.970434+0000 mon.a (mon.0) 2091 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:11:24.392 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:24 smithi078 bash[78274]: cluster 2023-10-01T11:11:22.984950+0000 osd.5 (osd.5) 386 : cluster 3 failed to encode map e364 with expected crc 2023-10-01T11:11:24.392 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:24 smithi078 bash[78274]: cluster 2023-10-01T11:11:22.986490+0000 osd.7 (osd.7) 408 : cluster 3 failed to encode map e364 with expected crc 2023-10-01T11:11:24.392 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:24 smithi078 bash[78274]: cluster 2023-10-01T11:11:22.988833+0000 osd.7 (osd.7) 409 : cluster 3 failed to encode map e364 with expected crc 2023-10-01T11:11:24.392 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:24 smithi078 bash[78274]: audit 2023-10-01T11:11:22.997528+0000 mon.a (mon.0) 2092 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T11:11:24.392 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:24 smithi078 bash[78274]: cluster 2023-10-01T11:11:22.997592+0000 mon.a (mon.0) 2093 : cluster 0 osdmap e364: 8 total, 8 up, 8 in 2023-10-01T11:11:24.392 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:24 smithi078 bash[78274]: cluster 2023-10-01T11:11:23.012326+0000 osd.6 (osd.6) 349 : cluster 3 failed to encode map e364 with expected crc 2023-10-01T11:11:24.393 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:24 smithi078 bash[78274]: cluster 2023-10-01T11:11:23.013761+0000 osd.6 (osd.6) 350 : cluster 3 failed to encode map e364 with expected crc 2023-10-01T11:11:24.411 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:24.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:24.416 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:11:24.517 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:24.548 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:24.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:24.552 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:11:24.653 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:24.679 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:24.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:24.689 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:11:24.790 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:24.793 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T11:11:24.820 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:24.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:24.827 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:11:24.929 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:24.962 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:24.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:24.965 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:11:24.988 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:24 smithi078 bash[75299]: audit 2023-10-01T11:11:24.151250+0000 mon.a (mon.0) 2094 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T11:11:25.065 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:25.105 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:25.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:25.113 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:11:25.121 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:24 smithi173 bash[58816]: audit 2023-10-01T11:11:24.151250+0000 mon.a (mon.0) 2094 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T11:11:25.213 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:25.246 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:25.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:25.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:11:25.250 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:24 smithi078 bash[78274]: audit 2023-10-01T11:11:24.151250+0000 mon.a (mon.0) 2094 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T11:11:25.322 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-01T11:11:25.351 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:25.373 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:25.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:25.375 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:11:25.476 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:25.508 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:25.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:25.511 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:11:25.612 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:25.649 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:25.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:25.653 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:11:25.754 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:25.779 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:25.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:25.784 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:11:25.884 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:25.908 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:25.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:25.914 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:11:26.014 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:26.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:26.042 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:26.042 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:11:26.145 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:26.180 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:26.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:26.182 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:11:26.190 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:26 smithi078 bash[75299]: cluster 2023-10-01T11:11:24.420959+0000 mgr.y (mgr.25072) 1624 : cluster 0 pgmap v1749: 89 pgs: 89 active+clean; 6.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 817 op/s 2023-10-01T11:11:26.284 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:26.329 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:26.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:26.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:11:26.391 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:25 smithi173 bash[58816]: cluster 2023-10-01T11:11:24.420959+0000 mgr.y (mgr.25072) 1624 : cluster 0 pgmap v1749: 89 pgs: 89 active+clean; 6.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 817 op/s 2023-10-01T11:11:26.492 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:26.519 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:26.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:26.524 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:11:26.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:26 smithi078 bash[78274]: cluster 2023-10-01T11:11:24.420959+0000 mgr.y (mgr.25072) 1624 : cluster 0 pgmap v1749: 89 pgs: 89 active+clean; 6.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 817 op/s 2023-10-01T11:11:26.625 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:26.650 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:26.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:26.654 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:11:26.755 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:26.782 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:26.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:26.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:11:26.887 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:26.912 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:26.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:26.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:11:27.017 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:27.048 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:27.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:27.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:11:27.152 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:27.179 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:27.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:27.182 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:11:27.283 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:27.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:27.326 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:27.326 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:11:27.427 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:27.432 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T11:11:27.441 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T11:11:27.451 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:27.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:27.453 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:11:27.554 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:27.594 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:27.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:27.599 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:11:27.699 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:27.720 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:27.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:27.723 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:11:27.823 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:27.855 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:27.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:27.859 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:11:27.960 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:27.987 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:27.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:27.990 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:11:28.091 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:28.100 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:28 smithi078 bash[75299]: cluster 2023-10-01T11:11:26.421590+0000 mgr.y (mgr.25072) 1625 : cluster 0 pgmap v1750: 89 pgs: 89 active+clean; 5.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 961 op/s 2023-10-01T11:11:28.100 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:28 smithi078 bash[78274]: cluster 2023-10-01T11:11:26.421590+0000 mgr.y (mgr.25072) 1625 : cluster 0 pgmap v1750: 89 pgs: 89 active+clean; 5.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 961 op/s 2023-10-01T11:11:28.130 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:28.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:28.137 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:11:28.237 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:28.246 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:28 smithi173 bash[58816]: cluster 2023-10-01T11:11:26.421590+0000 mgr.y (mgr.25072) 1625 : cluster 0 pgmap v1750: 89 pgs: 89 active+clean; 5.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 961 op/s 2023-10-01T11:11:28.266 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:28.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:28.269 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:11:28.370 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:28.397 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:28.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:28.401 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:11:28.502 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:28.535 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:28.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:28.543 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:11:28.644 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:28.667 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:28.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:28.670 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:11:28.770 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:28.812 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:28.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:28.815 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:11:28.916 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:28.951 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:28.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:28.956 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:11:29.057 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:29.092 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:29.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:29.094 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:11:29.195 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:29.232 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:29.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:29.239 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:11:29.340 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:29.384 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:29.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:29.386 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:11:29.487 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:29.520 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:29.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:29.524 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:11:29.625 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:29.665 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:29.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:29.668 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:11:29.768 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:29.805 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:29.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:29.811 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:11:29.847 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-10-01T11:11:29.847 INFO:teuthology.orchestra.run.smithi173.stdout: "size": 20, 2023-10-01T11:11:29.848 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 600, 2023-10-01T11:11:29.848 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [ 2023-10-01T11:11:29.848 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.848 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:143114 13.5 13:a65b5d2f:::benchmark_data_smithi078_202524_object63556:head [delete] snapc 0=[] ondisk+write+known_if_redirected e346)", 2023-10-01T11:11:29.848 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:08:12.857123+0000", 2023-10-01T11:11:29.848 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 196.98480647299999, 2023-10-01T11:11:29.848 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.395807619, 2023-10-01T11:11:29.848 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:11:29.848 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "reached pg", 2023-10-01T11:11:29.848 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:11:29.848 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:11:29.849 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:11:29.849 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 143114 2023-10-01T11:11:29.849 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.849 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:11:29.849 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.849 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:11:29.849 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:12.857123+0000", 2023-10-01T11:11:29.849 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:11:29.849 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.849 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.849 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:11:29.849 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:12.857123+0000", 2023-10-01T11:11:29.850 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:11:29.850 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.850 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.850 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:11:29.850 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:12.857122+0000", 2023-10-01T11:11:29.850 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-10-01T11:11:29.850 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.850 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.850 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:11:29.850 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:12.857131+0000", 2023-10-01T11:11:29.850 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.6209999999999999e-06 2023-10-01T11:11:29.850 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.851 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.851 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:11:29.851 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:12.857141+0000", 2023-10-01T11:11:29.851 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0288e-05 2023-10-01T11:11:29.851 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.851 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.851 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:11:29.851 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:12.857149+0000", 2023-10-01T11:11:29.851 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.3529999999999997e-06 2023-10-01T11:11:29.851 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.851 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.851 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:11:29.852 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:12.857402+0000", 2023-10-01T11:11:29.852 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00025295599999999998 2023-10-01T11:11:29.852 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.852 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.852 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:11:29.852 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:12.857541+0000", 2023-10-01T11:11:29.852 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00013879499999999999 2023-10-01T11:11:29.852 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.852 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.852 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 0,5", 2023-10-01T11:11:29.852 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:12.857671+0000", 2023-10-01T11:11:29.852 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00013013900000000001 2023-10-01T11:11:29.853 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.853 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.853 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:11:29.853 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:12.879182+0000", 2023-10-01T11:11:29.853 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.021511200000000001 2023-10-01T11:11:29.853 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.853 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.853 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:11:29.853 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:12.894954+0000", 2023-10-01T11:11:29.853 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.015771681999999999 2023-10-01T11:11:29.853 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.853 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.854 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:11:29.854 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:13.252919+0000", 2023-10-01T11:11:29.854 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.35796557699999998 2023-10-01T11:11:29.854 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.854 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.854 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:11:29.854 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:13.252931+0000", 2023-10-01T11:11:29.854 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1778e-05 2023-10-01T11:11:29.854 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:11:29.854 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:11:29.854 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:11:29.855 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.855 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.855 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:143116 13.5 13:a05169f9:::benchmark_data_smithi078_202524_object63558:head [delete] snapc 0=[] ondisk+write+known_if_redirected e346)", 2023-10-01T11:11:29.855 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:08:12.857439+0000", 2023-10-01T11:11:29.855 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 196.98449093900001, 2023-10-01T11:11:29.855 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.39975269699999999, 2023-10-01T11:11:29.855 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:11:29.855 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "reached pg", 2023-10-01T11:11:29.855 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:11:29.855 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:11:29.855 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:11:29.855 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 143116 2023-10-01T11:11:29.856 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.856 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:11:29.856 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.856 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:11:29.856 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:12.857439+0000", 2023-10-01T11:11:29.856 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:11:29.856 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.856 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.856 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:11:29.856 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:12.857439+0000", 2023-10-01T11:11:29.856 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:11:29.856 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.857 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.857 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:11:29.857 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:12.857438+0000", 2023-10-01T11:11:29.857 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-10-01T11:11:29.857 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.857 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.857 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:11:29.857 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:12.857443+0000", 2023-10-01T11:11:29.857 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.1789999999999999e-06 2023-10-01T11:11:29.857 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.857 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.857 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:11:29.858 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:12.857454+0000", 2023-10-01T11:11:29.858 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0866000000000001e-05 2023-10-01T11:11:29.858 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.858 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.858 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:11:29.858 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:12.857461+0000", 2023-10-01T11:11:29.858 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.9410000000000001e-06 2023-10-01T11:11:29.858 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.858 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.858 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:11:29.858 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:12.874020+0000", 2023-10-01T11:11:29.858 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.016559398999999999 2023-10-01T11:11:29.859 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.859 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.859 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:11:29.859 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:12.874191+0000", 2023-10-01T11:11:29.859 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000170941 2023-10-01T11:11:29.859 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.859 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.859 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 0,5", 2023-10-01T11:11:29.859 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:12.874322+0000", 2023-10-01T11:11:29.859 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00013079199999999999 2023-10-01T11:11:29.859 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.859 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.859 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:11:29.860 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:12.882610+0000", 2023-10-01T11:11:29.860 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0082881889999999996 2023-10-01T11:11:29.860 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.860 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.860 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:11:29.860 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:12.894981+0000", 2023-10-01T11:11:29.860 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.012370594 2023-10-01T11:11:29.860 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.860 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.860 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:11:29.860 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:13.257174+0000", 2023-10-01T11:11:29.860 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.36219374700000001 2023-10-01T11:11:29.861 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.861 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.861 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:11:29.861 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:13.257192+0000", 2023-10-01T11:11:29.861 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.7062e-05 2023-10-01T11:11:29.861 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:11:29.861 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:11:29.861 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:11:29.861 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.861 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.861 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.46454.0:143131 13.a e346/284 13:50d16c00:::benchmark_data_smithi078_202524_object63573:head v 346'9303, mlcod=346'9300)", 2023-10-01T11:11:29.861 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:08:12.899248+0000", 2023-10-01T11:11:29.861 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 196.94268179400001, 2023-10-01T11:11:29.862 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.41391661499999999, 2023-10-01T11:11:29.862 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:11:29.862 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:11:29.862 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:11:29.862 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.862 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:11:29.862 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:12.899248+0000", 2023-10-01T11:11:29.862 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:11:29.862 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.862 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.862 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:11:29.862 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:12.899248+0000", 2023-10-01T11:11:29.863 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:11:29.863 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.863 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.863 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:11:29.863 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:12.899248+0000", 2023-10-01T11:11:29.863 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999995 2023-10-01T11:11:29.863 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.863 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.863 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:11:29.863 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:12.899252+0000", 2023-10-01T11:11:29.863 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.5990000000000004e-06 2023-10-01T11:11:29.864 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.864 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.864 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:11:29.864 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:12.899258+0000", 2023-10-01T11:11:29.864 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.4739999999999996e-06 2023-10-01T11:11:29.864 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.864 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.864 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:11:29.864 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:12.899262+0000", 2023-10-01T11:11:29.864 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.3630000000000004e-06 2023-10-01T11:11:29.864 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.864 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.864 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:11:29.865 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:12.905293+0000", 2023-10-01T11:11:29.865 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0060307750000000004 2023-10-01T11:11:29.865 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.865 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.865 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:11:29.865 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:12.905390+0000", 2023-10-01T11:11:29.865 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.7269000000000007e-05 2023-10-01T11:11:29.865 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.865 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.865 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:11:29.865 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:13.313078+0000", 2023-10-01T11:11:29.865 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.40768736900000002 2023-10-01T11:11:29.866 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.866 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.866 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:11:29.866 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:13.313165+0000", 2023-10-01T11:11:29.866 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.7027999999999994e-05 2023-10-01T11:11:29.866 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:11:29.866 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:11:29.866 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:11:29.866 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.866 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.866 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.46454.0:143136 13.2 e346/284)", 2023-10-01T11:11:29.866 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:08:12.902320+0000", 2023-10-01T11:11:29.866 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 196.939609496, 2023-10-01T11:11:29.867 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.404870014, 2023-10-01T11:11:29.867 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:11:29.867 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "reached pg", 2023-10-01T11:11:29.867 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:11:29.867 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.867 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:11:29.867 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:12.902320+0000", 2023-10-01T11:11:29.867 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:11:29.867 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.867 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.867 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:11:29.867 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:12.902320+0000", 2023-10-01T11:11:29.867 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:11:29.868 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.868 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.868 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:11:29.868 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:12.902319+0000", 2023-10-01T11:11:29.868 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-10-01T11:11:29.868 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.868 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.868 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:11:29.868 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:12.902328+0000", 2023-10-01T11:11:29.868 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.4209999999999995e-06 2023-10-01T11:11:29.868 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.868 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.868 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:11:29.869 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:12.902340+0000", 2023-10-01T11:11:29.869 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1603999999999999e-05 2023-10-01T11:11:29.869 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.869 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.869 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:11:29.869 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:12.902363+0000", 2023-10-01T11:11:29.869 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.3499999999999999e-05 2023-10-01T11:11:29.869 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.869 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.869 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:11:29.869 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:13.307165+0000", 2023-10-01T11:11:29.869 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.40480157300000003 2023-10-01T11:11:29.869 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.870 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.870 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:11:29.870 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:13.307190+0000", 2023-10-01T11:11:29.870 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.5724e-05 2023-10-01T11:11:29.870 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:11:29.870 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:11:29.870 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:11:29.870 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.870 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.870 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:143114 13.5 13:a65b5d2f:::benchmark_data_smithi078_202524_object63556:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e347)", 2023-10-01T11:11:29.870 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:08:12.973034+0000", 2023-10-01T11:11:29.870 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 196.868895815, 2023-10-01T11:11:29.871 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0149220889999999, 2023-10-01T11:11:29.871 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:11:29.871 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "reached pg", 2023-10-01T11:11:29.871 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:11:29.871 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:11:29.871 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:11:29.871 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 143114 2023-10-01T11:11:29.871 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.871 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:11:29.871 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.871 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:11:29.871 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:12.973034+0000", 2023-10-01T11:11:29.872 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:11:29.872 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.872 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.872 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:11:29.872 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:12.973034+0000", 2023-10-01T11:11:29.872 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:11:29.872 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.872 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.872 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:11:29.872 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:12.973031+0000", 2023-10-01T11:11:29.872 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999971 2023-10-01T11:11:29.873 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.873 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.873 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:11:29.873 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:12.973046+0000", 2023-10-01T11:11:29.873 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.5119999999999999e-05 2023-10-01T11:11:29.873 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.873 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.873 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:11:29.873 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:12.973068+0000", 2023-10-01T11:11:29.873 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.2079e-05 2023-10-01T11:11:29.873 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.873 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.874 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:11:29.874 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:12.973084+0000", 2023-10-01T11:11:29.874 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.5248999999999999e-05 2023-10-01T11:11:29.874 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.874 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.874 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:11:29.874 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:13.257269+0000", 2023-10-01T11:11:29.874 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.28418558100000002 2023-10-01T11:11:29.874 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.874 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.874 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:11:29.874 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:13.257281+0000", 2023-10-01T11:11:29.875 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1746000000000001e-05 2023-10-01T11:11:29.875 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.875 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.875 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:11:29.875 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:13.987820+0000", 2023-10-01T11:11:29.875 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.73053958500000005 2023-10-01T11:11:29.875 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.875 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.875 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:11:29.875 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:13.987956+0000", 2023-10-01T11:11:29.875 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00013560499999999999 2023-10-01T11:11:29.875 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:11:29.876 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:11:29.876 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:11:29.876 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.876 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.876 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:143116 13.5 13:a05169f9:::benchmark_data_smithi078_202524_object63558:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e347)", 2023-10-01T11:11:29.876 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:08:12.973097+0000", 2023-10-01T11:11:29.876 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 196.86883257599999, 2023-10-01T11:11:29.876 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.015213301, 2023-10-01T11:11:29.876 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:11:29.876 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "reached pg", 2023-10-01T11:11:29.876 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:11:29.876 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:11:29.877 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:11:29.877 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 143116 2023-10-01T11:11:29.877 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.877 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:11:29.877 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.877 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:11:29.877 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:12.973097+0000", 2023-10-01T11:11:29.877 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:11:29.877 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.877 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.877 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:11:29.877 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:12.973097+0000", 2023-10-01T11:11:29.878 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:11:29.878 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.878 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.878 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:11:29.878 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:12.973097+0000", 2023-10-01T11:11:29.878 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999995 2023-10-01T11:11:29.878 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.878 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.878 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:11:29.878 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:12.973101+0000", 2023-10-01T11:11:29.878 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.9009999999999996e-06 2023-10-01T11:11:29.878 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.879 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.879 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:11:29.879 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:12.973108+0000", 2023-10-01T11:11:29.879 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.3520000000000001e-06 2023-10-01T11:11:29.879 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.879 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.879 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:11:29.879 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:12.973114+0000", 2023-10-01T11:11:29.879 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.8950000000000003e-06 2023-10-01T11:11:29.879 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.879 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.879 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:11:29.880 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:13.257304+0000", 2023-10-01T11:11:29.880 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.28419050600000001 2023-10-01T11:11:29.880 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.880 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.880 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:11:29.880 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:13.257313+0000", 2023-10-01T11:11:29.880 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.4419999999999998e-06 2023-10-01T11:11:29.880 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.880 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.880 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:11:29.880 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:13.988123+0000", 2023-10-01T11:11:29.881 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.730810709 2023-10-01T11:11:29.881 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.881 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.881 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:11:29.881 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:13.988311+0000", 2023-10-01T11:11:29.881 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00018706800000000001 2023-10-01T11:11:29.881 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:11:29.881 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:11:29.881 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:11:29.881 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.881 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.881 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:143132 13.5 13:a6302a25:::benchmark_data_smithi078_202524_object63574:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e347)", 2023-10-01T11:11:29.882 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:08:12.973120+0000", 2023-10-01T11:11:29.882 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 196.86880997599999, 2023-10-01T11:11:29.882 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.07651857, 2023-10-01T11:11:29.882 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:11:29.882 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "reached pg", 2023-10-01T11:11:29.882 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:11:29.882 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:11:29.882 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:11:29.882 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 143132 2023-10-01T11:11:29.882 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.882 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:11:29.882 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.883 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:11:29.883 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:12.973120+0000", 2023-10-01T11:11:29.883 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:11:29.883 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.883 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.883 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:11:29.883 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:12.973120+0000", 2023-10-01T11:11:29.883 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:11:29.883 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.883 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.883 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:11:29.883 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:12.973119+0000", 2023-10-01T11:11:29.884 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999995 2023-10-01T11:11:29.884 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.884 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.884 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:11:29.884 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:12.973122+0000", 2023-10-01T11:11:29.884 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.0469999999999998e-06 2023-10-01T11:11:29.884 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.884 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.884 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:11:29.884 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:12.973129+0000", 2023-10-01T11:11:29.884 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.7649999999999996e-06 2023-10-01T11:11:29.884 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.885 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.885 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:11:29.885 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:12.973135+0000", 2023-10-01T11:11:29.885 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.879e-06 2023-10-01T11:11:29.885 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.885 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.885 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:11:29.885 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:13.257336+0000", 2023-10-01T11:11:29.885 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.28420111799999997 2023-10-01T11:11:29.885 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.885 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.885 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:11:29.886 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:13.257344+0000", 2023-10-01T11:11:29.886 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.4749999999999996e-06 2023-10-01T11:11:29.886 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.886 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.886 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:11:29.886 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:13.988337+0000", 2023-10-01T11:11:29.886 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.73099325599999998 2023-10-01T11:11:29.886 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.886 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.886 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for missing object", 2023-10-01T11:11:29.886 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:13.988453+0000", 2023-10-01T11:11:29.886 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00011612099999999999 2023-10-01T11:11:29.887 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.887 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.887 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:11:29.887 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.049553+0000", 2023-10-01T11:11:29.887 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.061100302000000002 2023-10-01T11:11:29.887 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.887 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.887 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:11:29.887 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.049638+0000", 2023-10-01T11:11:29.887 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.5086000000000005e-05 2023-10-01T11:11:29.887 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:11:29.887 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:11:29.887 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:11:29.888 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.888 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.888 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_repop(client.46454.0:143139 13.a e347/284 13:52381e1c:::benchmark_data_smithi078_202524_object63581:head v 347'9304, mlcod=346'9301)", 2023-10-01T11:11:29.888 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:08:12.977217+0000", 2023-10-01T11:11:29.888 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 196.86471282400001, 2023-10-01T11:11:29.888 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.43909309899999999, 2023-10-01T11:11:29.888 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:11:29.888 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:11:29.888 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:11:29.888 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.888 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:11:29.888 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:12.977217+0000", 2023-10-01T11:11:29.889 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:11:29.889 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.889 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.889 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:11:29.889 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:12.977217+0000", 2023-10-01T11:11:29.889 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:11:29.889 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.889 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.889 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:11:29.889 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:12.977217+0000", 2023-10-01T11:11:29.889 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999995 2023-10-01T11:11:29.889 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.890 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.890 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:11:29.890 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:12.977225+0000", 2023-10-01T11:11:29.890 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.8339999999999999e-06 2023-10-01T11:11:29.890 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.890 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.890 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:11:29.890 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:12.977251+0000", 2023-10-01T11:11:29.890 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.6735000000000001e-05 2023-10-01T11:11:29.890 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.890 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.890 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:11:29.891 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:12.977268+0000", 2023-10-01T11:11:29.891 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.6350000000000001e-05 2023-10-01T11:11:29.891 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.891 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.891 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:11:29.891 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:13.305122+0000", 2023-10-01T11:11:29.891 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.327854486 2023-10-01T11:11:29.891 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.891 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.891 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:11:29.891 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:13.305152+0000", 2023-10-01T11:11:29.891 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.0179e-05 2023-10-01T11:11:29.891 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.892 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.892 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:11:29.892 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:13.416250+0000", 2023-10-01T11:11:29.892 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.111097485 2023-10-01T11:11:29.892 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.892 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.892 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:11:29.892 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:13.416310+0000", 2023-10-01T11:11:29.892 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.0316999999999999e-05 2023-10-01T11:11:29.892 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:11:29.892 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:11:29.892 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:11:29.893 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.893 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.893 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:143151 13.7 13:e204e040:::benchmark_data_smithi078_202524_object63593:head [delete] snapc 0=[] ondisk+write+known_if_redirected e347)", 2023-10-01T11:11:29.893 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:08:13.440930+0000", 2023-10-01T11:11:29.893 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 196.40099949200001, 2023-10-01T11:11:29.893 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.5392972060000001, 2023-10-01T11:11:29.893 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:11:29.893 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:11:29.893 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:11:29.893 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:11:29.893 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:11:29.893 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 143151 2023-10-01T11:11:29.894 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.894 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:11:29.894 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.894 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:11:29.894 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:13.440930+0000", 2023-10-01T11:11:29.894 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:11:29.894 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.894 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.894 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:11:29.894 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:13.440930+0000", 2023-10-01T11:11:29.894 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:11:29.895 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.895 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.895 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:11:29.895 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:13.440928+0000", 2023-10-01T11:11:29.895 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999976 2023-10-01T11:11:29.895 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.895 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.895 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:11:29.895 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:13.440944+0000", 2023-10-01T11:11:29.895 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.6065e-05 2023-10-01T11:11:29.895 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.895 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.896 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:11:29.896 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:13.440978+0000", 2023-10-01T11:11:29.896 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.3868000000000002e-05 2023-10-01T11:11:29.896 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.896 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.896 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:11:29.896 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:13.440994+0000", 2023-10-01T11:11:29.896 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.6115000000000001e-05 2023-10-01T11:11:29.896 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.896 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.896 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:11:29.896 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:13.441089+0000", 2023-10-01T11:11:29.897 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.4987000000000004e-05 2023-10-01T11:11:29.897 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.897 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.897 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:11:29.897 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:13.441106+0000", 2023-10-01T11:11:29.897 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.6895999999999999e-05 2023-10-01T11:11:29.897 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.897 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.897 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:11:29.897 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:13.907092+0000", 2023-10-01T11:11:29.897 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.46598577299999999 2023-10-01T11:11:29.898 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.898 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.898 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:11:29.898 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:13.907111+0000", 2023-10-01T11:11:29.898 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.9644999999999999e-05 2023-10-01T11:11:29.898 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.898 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.898 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:11:29.898 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.948490+0000", 2023-10-01T11:11:29.898 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0413785820000001 2023-10-01T11:11:29.898 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.899 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.899 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:11:29.899 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.948655+0000", 2023-10-01T11:11:29.899 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00016519000000000001 2023-10-01T11:11:29.899 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.899 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.899 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 6,7", 2023-10-01T11:11:29.899 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.949027+0000", 2023-10-01T11:11:29.899 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00037164400000000002 2023-10-01T11:11:29.899 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.899 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.899 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:11:29.899 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.979784+0000", 2023-10-01T11:11:29.900 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.030756875999999999 2023-10-01T11:11:29.900 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.900 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.900 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:11:29.900 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.979845+0000", 2023-10-01T11:11:29.900 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.1209000000000003e-05 2023-10-01T11:11:29.900 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.900 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.900 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:11:29.900 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.979894+0000", 2023-10-01T11:11:29.900 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.9502999999999997e-05 2023-10-01T11:11:29.900 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.900 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.901 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:11:29.901 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.979960+0000", 2023-10-01T11:11:29.901 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.5309999999999996e-05 2023-10-01T11:11:29.901 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.901 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.901 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:11:29.901 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.980228+0000", 2023-10-01T11:11:29.901 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00026797600000000001 2023-10-01T11:11:29.901 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:11:29.901 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:11:29.901 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:11:29.901 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.902 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.902 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:143153 13.15 13:aa19e955:::benchmark_data_smithi078_202524_object63595:head [delete] snapc 0=[] ondisk+write+known_if_redirected e347)", 2023-10-01T11:11:29.902 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:08:13.477901+0000", 2023-10-01T11:11:29.902 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 196.36402833899999, 2023-10-01T11:11:29.902 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.65441090300000004, 2023-10-01T11:11:29.902 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:11:29.902 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:11:29.902 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:11:29.902 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:11:29.903 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:11:29.903 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 143153 2023-10-01T11:11:29.903 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.903 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:11:29.903 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.903 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:11:29.903 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:13.477901+0000", 2023-10-01T11:11:29.903 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:11:29.903 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.903 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.903 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:11:29.903 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:13.477901+0000", 2023-10-01T11:11:29.904 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:11:29.904 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.904 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.904 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:11:29.904 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:13.477899+0000", 2023-10-01T11:11:29.904 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999976 2023-10-01T11:11:29.904 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.904 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.904 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:11:29.904 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:13.477912+0000", 2023-10-01T11:11:29.904 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.3176000000000001e-05 2023-10-01T11:11:29.904 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.904 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.905 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:11:29.905 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:13.477942+0000", 2023-10-01T11:11:29.905 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.9587e-05 2023-10-01T11:11:29.905 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.905 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.905 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:11:29.905 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:13.477955+0000", 2023-10-01T11:11:29.905 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.3339999999999999e-05 2023-10-01T11:11:29.905 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.905 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.905 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:11:29.905 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:13.478042+0000", 2023-10-01T11:11:29.905 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.7268999999999994e-05 2023-10-01T11:11:29.906 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.906 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.906 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:11:29.906 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:13.478060+0000", 2023-10-01T11:11:29.906 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.7169000000000001e-05 2023-10-01T11:11:29.906 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.906 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.906 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:11:29.906 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.113071+0000", 2023-10-01T11:11:29.906 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.63501170299999998 2023-10-01T11:11:29.906 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.906 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.907 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:11:29.907 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.113221+0000", 2023-10-01T11:11:29.907 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000149417 2023-10-01T11:11:29.907 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.907 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.907 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 0,5", 2023-10-01T11:11:29.907 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.115830+0000", 2023-10-01T11:11:29.907 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0026091130000000001 2023-10-01T11:11:29.907 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.907 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.907 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:11:29.908 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.118214+0000", 2023-10-01T11:11:29.908 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0023841029999999998 2023-10-01T11:11:29.908 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.908 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.908 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:11:29.908 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.120061+0000", 2023-10-01T11:11:29.908 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0018471609999999999 2023-10-01T11:11:29.908 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.908 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.908 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:11:29.908 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.132201+0000", 2023-10-01T11:11:29.908 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.012139787000000001 2023-10-01T11:11:29.909 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.909 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.909 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:11:29.909 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.132263+0000", 2023-10-01T11:11:29.909 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.2260000000000004e-05 2023-10-01T11:11:29.909 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.909 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.909 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:11:29.909 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.132312+0000", 2023-10-01T11:11:29.909 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.9252999999999998e-05 2023-10-01T11:11:29.909 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:11:29.909 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:11:29.909 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:11:29.910 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.910 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.910 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:143170 13.7 13:e32077e5:::benchmark_data_smithi078_202524_object63612:head [delete] snapc 0=[] ondisk+write+known_if_redirected e348)", 2023-10-01T11:11:29.910 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:08:14.122851+0000", 2023-10-01T11:11:29.910 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 195.71907873800001, 2023-10-01T11:11:29.910 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.868347431, 2023-10-01T11:11:29.910 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:11:29.910 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:11:29.910 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:11:29.910 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:11:29.910 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:11:29.910 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 143170 2023-10-01T11:11:29.911 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.911 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:11:29.911 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.911 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:11:29.911 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.122851+0000", 2023-10-01T11:11:29.911 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:11:29.911 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.911 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.911 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:11:29.911 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.122851+0000", 2023-10-01T11:11:29.911 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:11:29.912 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.912 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.912 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:11:29.912 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.122850+0000", 2023-10-01T11:11:29.912 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-10-01T11:11:29.912 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.912 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.912 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:11:29.912 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.122857+0000", 2023-10-01T11:11:29.912 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.985e-06 2023-10-01T11:11:29.912 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.912 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.913 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:11:29.913 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.122869+0000", 2023-10-01T11:11:29.913 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2734999999999999e-05 2023-10-01T11:11:29.913 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.913 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.913 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:11:29.913 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.122878+0000", 2023-10-01T11:11:29.913 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.7059999999999993e-06 2023-10-01T11:11:29.913 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.913 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.913 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:11:29.913 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.142828+0000", 2023-10-01T11:11:29.914 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.019949840999999999 2023-10-01T11:11:29.914 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.914 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.914 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:11:29.914 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.142838+0000", 2023-10-01T11:11:29.914 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.4900000000000006e-06 2023-10-01T11:11:29.914 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.914 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.914 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:11:29.914 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.980285+0000", 2023-10-01T11:11:29.914 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.83744795900000002 2023-10-01T11:11:29.915 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.915 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.915 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:11:29.915 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.980452+0000", 2023-10-01T11:11:29.915 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000166319 2023-10-01T11:11:29.915 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.915 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.915 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 6,7", 2023-10-01T11:11:29.915 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.980842+0000", 2023-10-01T11:11:29.915 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00039006000000000002 2023-10-01T11:11:29.915 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.915 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.915 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:11:29.916 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.989179+0000", 2023-10-01T11:11:29.916 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0083370890000000007 2023-10-01T11:11:29.916 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.916 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.916 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:11:29.916 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.990703+0000", 2023-10-01T11:11:29.916 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.001524318 2023-10-01T11:11:29.916 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.916 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.916 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:11:29.916 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.990771+0000", 2023-10-01T11:11:29.916 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.7930999999999997e-05 2023-10-01T11:11:29.917 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.917 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.917 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:11:29.917 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.990844+0000", 2023-10-01T11:11:29.917 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.2979999999999996e-05 2023-10-01T11:11:29.917 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.917 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.917 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:11:29.917 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.991198+0000", 2023-10-01T11:11:29.917 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00035432500000000001 2023-10-01T11:11:29.917 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:11:29.917 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:11:29.918 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:11:29.918 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.918 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.918 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:143172 13.7 13:e0e3411e:::benchmark_data_smithi078_202524_object63614:head [delete] snapc 0=[] ondisk+write+known_if_redirected e348)", 2023-10-01T11:11:29.918 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:08:14.137655+0000", 2023-10-01T11:11:29.918 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 195.704274524, 2023-10-01T11:11:29.918 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.86621646699999999, 2023-10-01T11:11:29.918 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:11:29.918 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:11:29.918 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:11:29.918 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:11:29.918 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:11:29.919 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 143172 2023-10-01T11:11:29.919 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.919 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:11:29.919 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.919 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:11:29.919 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.137655+0000", 2023-10-01T11:11:29.919 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:11:29.919 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.919 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.919 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:11:29.919 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.137655+0000", 2023-10-01T11:11:29.919 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:11:29.920 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.920 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.920 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:11:29.920 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.137652+0000", 2023-10-01T11:11:29.920 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999971 2023-10-01T11:11:29.920 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.920 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.920 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:11:29.920 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.137669+0000", 2023-10-01T11:11:29.920 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.6838e-05 2023-10-01T11:11:29.920 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.920 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.921 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:11:29.921 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.137686+0000", 2023-10-01T11:11:29.921 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.7430000000000001e-05 2023-10-01T11:11:29.921 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.921 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.921 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:11:29.921 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.137701+0000", 2023-10-01T11:11:29.921 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.436e-05 2023-10-01T11:11:29.921 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.921 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.921 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:11:29.922 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.144391+0000", 2023-10-01T11:11:29.922 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0066905940000000002 2023-10-01T11:11:29.922 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.922 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.922 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:11:29.922 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.144403+0000", 2023-10-01T11:11:29.922 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1172000000000001e-05 2023-10-01T11:11:29.922 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.922 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.922 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:11:29.922 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.982467+0000", 2023-10-01T11:11:29.922 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.83806441799999998 2023-10-01T11:11:29.923 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.923 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.923 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:11:29.923 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.982624+0000", 2023-10-01T11:11:29.923 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00015672900000000001 2023-10-01T11:11:29.923 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.923 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.923 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 6,7", 2023-10-01T11:11:29.923 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.982865+0000", 2023-10-01T11:11:29.923 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000241353 2023-10-01T11:11:29.923 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.924 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.924 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:11:29.924 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.991224+0000", 2023-10-01T11:11:29.924 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0083592279999999998 2023-10-01T11:11:29.924 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.924 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.924 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:11:29.924 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.991312+0000", 2023-10-01T11:11:29.924 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.7435999999999994e-05 2023-10-01T11:11:29.924 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.924 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.924 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:11:29.925 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:15.003487+0000", 2023-10-01T11:11:29.925 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.012175305000000001 2023-10-01T11:11:29.925 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.925 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.925 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:11:29.925 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:15.003538+0000", 2023-10-01T11:11:29.925 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.1044999999999997e-05 2023-10-01T11:11:29.925 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.925 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.925 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:11:29.925 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:15.003872+0000", 2023-10-01T11:11:29.926 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00033364000000000001 2023-10-01T11:11:29.926 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:11:29.926 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:11:29.926 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:11:29.926 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.926 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.926 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:143184 13.7 13:e591764b:::benchmark_data_smithi078_202524_object63626:head [delete] snapc 0=[] ondisk+write+known_if_redirected e348)", 2023-10-01T11:11:29.926 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:08:14.162690+0000", 2023-10-01T11:11:29.926 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 195.67923951099999, 2023-10-01T11:11:29.926 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.84157462000000005, 2023-10-01T11:11:29.926 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:11:29.926 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:11:29.927 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:11:29.927 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:11:29.927 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:11:29.927 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 143184 2023-10-01T11:11:29.927 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.927 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:11:29.927 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.927 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:11:29.927 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.162690+0000", 2023-10-01T11:11:29.927 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:11:29.927 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.927 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.928 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:11:29.928 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.162690+0000", 2023-10-01T11:11:29.928 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:11:29.928 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.928 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.928 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:11:29.928 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.162689+0000", 2023-10-01T11:11:29.928 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-10-01T11:11:29.928 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.928 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.928 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:11:29.928 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.162697+0000", 2023-10-01T11:11:29.929 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.6299999999999998e-06 2023-10-01T11:11:29.929 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.929 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.929 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:11:29.929 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.162709+0000", 2023-10-01T11:11:29.929 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2085e-05 2023-10-01T11:11:29.929 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.929 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.929 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:11:29.929 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.162717+0000", 2023-10-01T11:11:29.929 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.0539999999999995e-06 2023-10-01T11:11:29.929 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.930 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.930 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:11:29.930 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.162807+0000", 2023-10-01T11:11:29.930 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.0418000000000001e-05 2023-10-01T11:11:29.930 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.930 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.930 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:11:29.930 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.162820+0000", 2023-10-01T11:11:29.930 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2591e-05 2023-10-01T11:11:29.930 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.930 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.930 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:11:29.931 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.984090+0000", 2023-10-01T11:11:29.931 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.82127066500000001 2023-10-01T11:11:29.931 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.931 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.931 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:11:29.931 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.984247+0000", 2023-10-01T11:11:29.931 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00015643300000000001 2023-10-01T11:11:29.931 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.931 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.931 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 6,7", 2023-10-01T11:11:29.931 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.984543+0000", 2023-10-01T11:11:29.931 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00029572000000000002 2023-10-01T11:11:29.932 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.932 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.932 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:11:29.932 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.991235+0000", 2023-10-01T11:11:29.932 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0066926049999999999 2023-10-01T11:11:29.932 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.932 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.932 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:11:29.932 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.999793+0000", 2023-10-01T11:11:29.932 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0085574329999999997 2023-10-01T11:11:29.932 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.933 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.933 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:11:29.933 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:15.003941+0000", 2023-10-01T11:11:29.933 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0041487479999999998 2023-10-01T11:11:29.933 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.933 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.933 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:11:29.933 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:15.004001+0000", 2023-10-01T11:11:29.933 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.9295000000000001e-05 2023-10-01T11:11:29.933 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.933 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.933 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:11:29.934 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:15.004265+0000", 2023-10-01T11:11:29.934 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00026424100000000002 2023-10-01T11:11:29.934 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:11:29.934 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:11:29.934 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:11:29.934 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.934 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.934 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:143186 13.7 13:e5209444:::benchmark_data_smithi078_202524_object63628:head [delete] snapc 0=[] ondisk+write+known_if_redirected e348)", 2023-10-01T11:11:29.934 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:08:14.163343+0000", 2023-10-01T11:11:29.934 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 195.67858664400001, 2023-10-01T11:11:29.934 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.84125156499999998, 2023-10-01T11:11:29.934 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:11:29.935 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:11:29.935 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:11:29.935 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:11:29.935 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:11:29.935 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 143186 2023-10-01T11:11:29.935 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.935 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:11:29.935 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.935 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:11:29.935 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.163343+0000", 2023-10-01T11:11:29.935 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:11:29.935 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.935 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.936 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:11:29.936 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.163343+0000", 2023-10-01T11:11:29.936 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:11:29.936 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.936 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.936 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:11:29.936 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.163342+0000", 2023-10-01T11:11:29.936 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-10-01T11:11:29.936 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.936 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.936 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:11:29.936 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.163348+0000", 2023-10-01T11:11:29.936 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.5910000000000003e-06 2023-10-01T11:11:29.937 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.937 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.937 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:11:29.937 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.163359+0000", 2023-10-01T11:11:29.937 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1079e-05 2023-10-01T11:11:29.937 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.937 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.937 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:11:29.937 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.163367+0000", 2023-10-01T11:11:29.937 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.2379999999999997e-06 2023-10-01T11:11:29.937 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.938 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.938 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:11:29.938 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.163421+0000", 2023-10-01T11:11:29.938 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.4196000000000001e-05 2023-10-01T11:11:29.938 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.938 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.938 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:11:29.938 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.163433+0000", 2023-10-01T11:11:29.938 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2115e-05 2023-10-01T11:11:29.938 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.938 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.938 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:11:29.939 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.985810+0000", 2023-10-01T11:11:29.939 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.82237642899999996 2023-10-01T11:11:29.939 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.939 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.939 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:11:29.939 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.985976+0000", 2023-10-01T11:11:29.939 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00016605799999999999 2023-10-01T11:11:29.939 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.939 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.939 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 6,7", 2023-10-01T11:11:29.939 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.986261+0000", 2023-10-01T11:11:29.939 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00028556000000000002 2023-10-01T11:11:29.939 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.940 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.940 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:11:29.940 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.991253+0000", 2023-10-01T11:11:29.940 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0049913479999999996 2023-10-01T11:11:29.940 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.940 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.940 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:11:29.940 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.999845+0000", 2023-10-01T11:11:29.940 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0085921999999999995 2023-10-01T11:11:29.940 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.940 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.940 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:11:29.940 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:15.004322+0000", 2023-10-01T11:11:29.940 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0044768159999999998 2023-10-01T11:11:29.941 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.941 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.941 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:11:29.941 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:15.004373+0000", 2023-10-01T11:11:29.941 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.1057000000000003e-05 2023-10-01T11:11:29.941 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.941 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.941 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:11:29.941 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:15.004595+0000", 2023-10-01T11:11:29.941 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000221831 2023-10-01T11:11:29.941 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:11:29.941 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:11:29.941 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:11:29.942 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.942 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.942 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:143203 13.7 13:e487dfcc:::benchmark_data_smithi078_202524_object63645:head [delete] snapc 0=[] ondisk+write+known_if_redirected e348)", 2023-10-01T11:11:29.942 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:08:14.188856+0000", 2023-10-01T11:11:29.942 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 195.65307426499999, 2023-10-01T11:11:29.942 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.81604188799999999, 2023-10-01T11:11:29.942 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:11:29.942 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:11:29.942 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:11:29.942 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:11:29.942 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:11:29.942 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 143203 2023-10-01T11:11:29.943 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.943 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:11:29.943 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.943 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:11:29.943 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.188856+0000", 2023-10-01T11:11:29.943 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:11:29.943 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.943 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.943 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:11:29.943 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.188856+0000", 2023-10-01T11:11:29.943 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:11:29.944 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.944 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.944 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:11:29.944 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.188854+0000", 2023-10-01T11:11:29.944 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-10-01T11:11:29.944 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.944 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.944 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:11:29.944 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.188861+0000", 2023-10-01T11:11:29.944 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.0029999999999997e-06 2023-10-01T11:11:29.944 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.945 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.945 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:11:29.945 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.188872+0000", 2023-10-01T11:11:29.945 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0586e-05 2023-10-01T11:11:29.945 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.945 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.945 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:11:29.945 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.188879+0000", 2023-10-01T11:11:29.945 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.8839999999999997e-06 2023-10-01T11:11:29.945 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.945 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.946 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:11:29.946 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.188944+0000", 2023-10-01T11:11:29.946 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.5535000000000002e-05 2023-10-01T11:11:29.946 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.946 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.946 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:11:29.946 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.188955+0000", 2023-10-01T11:11:29.946 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0315e-05 2023-10-01T11:11:29.946 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.946 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.946 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:11:29.946 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.987514+0000", 2023-10-01T11:11:29.947 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.79855892399999995 2023-10-01T11:11:29.947 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.947 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.947 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:11:29.947 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.987645+0000", 2023-10-01T11:11:29.947 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000131786 2023-10-01T11:11:29.947 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.947 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.947 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 6,7", 2023-10-01T11:11:29.947 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.987864+0000", 2023-10-01T11:11:29.947 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00021825800000000001 2023-10-01T11:11:29.947 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.947 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.948 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:11:29.948 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.993137+0000", 2023-10-01T11:11:29.948 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0052739040000000003 2023-10-01T11:11:29.948 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.948 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.948 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:11:29.948 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:15.002244+0000", 2023-10-01T11:11:29.948 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0091060429999999994 2023-10-01T11:11:29.948 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.948 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.948 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:11:29.948 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:15.004638+0000", 2023-10-01T11:11:29.948 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0023948620000000002 2023-10-01T11:11:29.949 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.949 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.949 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:11:29.949 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:15.004692+0000", 2023-10-01T11:11:29.949 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.3687999999999998e-05 2023-10-01T11:11:29.949 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.949 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.949 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:11:29.949 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:15.004897+0000", 2023-10-01T11:11:29.949 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00020536600000000001 2023-10-01T11:11:29.949 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:11:29.950 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:11:29.950 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:11:29.950 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.950 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.950 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:143209 13.7 13:e3302103:::benchmark_data_smithi078_202524_object63651:head [delete] snapc 0=[] ondisk+write+known_if_redirected e348)", 2023-10-01T11:11:29.950 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:08:14.195110+0000", 2023-10-01T11:11:29.950 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 195.64681985300001, 2023-10-01T11:11:29.950 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.81011687399999999, 2023-10-01T11:11:29.950 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:11:29.950 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:11:29.950 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:11:29.951 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:11:29.951 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:11:29.951 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 143209 2023-10-01T11:11:29.951 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.951 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:11:29.951 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.951 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:11:29.951 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.195110+0000", 2023-10-01T11:11:29.951 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:11:29.951 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.951 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.951 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:11:29.951 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.195110+0000", 2023-10-01T11:11:29.952 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:11:29.952 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.952 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.952 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:11:29.952 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.195109+0000", 2023-10-01T11:11:29.952 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-10-01T11:11:29.952 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.952 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.952 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:11:29.952 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.195115+0000", 2023-10-01T11:11:29.952 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.1290000000000001e-06 2023-10-01T11:11:29.952 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.952 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.953 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:11:29.953 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.195125+0000", 2023-10-01T11:11:29.953 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0378e-05 2023-10-01T11:11:29.953 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.953 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.953 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:11:29.953 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.195132+0000", 2023-10-01T11:11:29.953 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.3239999999999999e-06 2023-10-01T11:11:29.953 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.953 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.953 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:11:29.954 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.195185+0000", 2023-10-01T11:11:29.954 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.3635999999999999e-05 2023-10-01T11:11:29.954 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.954 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.954 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:11:29.954 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.195199+0000", 2023-10-01T11:11:29.954 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.3514e-05 2023-10-01T11:11:29.954 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.954 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.954 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:11:29.954 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.989237+0000", 2023-10-01T11:11:29.954 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.79403830399999997 2023-10-01T11:11:29.955 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.955 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.955 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:11:29.955 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.989348+0000", 2023-10-01T11:11:29.955 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00011096599999999999 2023-10-01T11:11:29.955 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.955 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.955 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 6,7", 2023-10-01T11:11:29.955 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.989543+0000", 2023-10-01T11:11:29.955 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00019511299999999999 2023-10-01T11:11:29.955 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.955 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.956 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:11:29.956 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.993147+0000", 2023-10-01T11:11:29.956 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0036039380000000001 2023-10-01T11:11:29.956 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.956 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.956 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:11:29.956 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:15.002290+0000", 2023-10-01T11:11:29.956 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0091432470000000002 2023-10-01T11:11:29.956 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.956 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.956 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:11:29.956 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:15.004951+0000", 2023-10-01T11:11:29.957 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0026607029999999999 2023-10-01T11:11:29.957 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.957 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.957 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:11:29.957 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:15.005025+0000", 2023-10-01T11:11:29.957 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.4079999999999995e-05 2023-10-01T11:11:29.957 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.957 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.957 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:11:29.957 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:15.005227+0000", 2023-10-01T11:11:29.957 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000201602 2023-10-01T11:11:29.957 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:11:29.958 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:11:29.958 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:11:29.958 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.958 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.958 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:143243 13.7 13:e7a829ca:::benchmark_data_smithi078_202524_object63685:head [delete] snapc 0=[] ondisk+write+known_if_redirected e348)", 2023-10-01T11:11:29.958 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:08:14.251678+0000", 2023-10-01T11:11:29.958 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 195.59025149600001, 2023-10-01T11:11:29.958 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.75376429, 2023-10-01T11:11:29.958 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:11:29.958 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:11:29.958 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:11:29.958 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:11:29.959 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:11:29.959 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 143243 2023-10-01T11:11:29.959 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.959 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:11:29.959 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.959 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:11:29.959 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.251678+0000", 2023-10-01T11:11:29.959 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:11:29.959 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.959 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.959 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:11:29.959 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.251678+0000", 2023-10-01T11:11:29.960 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:11:29.960 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.960 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.960 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:11:29.960 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.251677+0000", 2023-10-01T11:11:29.960 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-10-01T11:11:29.960 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.960 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.960 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:11:29.960 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.251683+0000", 2023-10-01T11:11:29.960 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.3990000000000003e-06 2023-10-01T11:11:29.960 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.961 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.961 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:11:29.961 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.251691+0000", 2023-10-01T11:11:29.961 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.1529999999999997e-06 2023-10-01T11:11:29.961 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.961 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.961 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:11:29.961 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.251696+0000", 2023-10-01T11:11:29.961 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.921e-06 2023-10-01T11:11:29.961 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.961 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.961 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:11:29.962 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.251735+0000", 2023-10-01T11:11:29.962 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.8915000000000003e-05 2023-10-01T11:11:29.962 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.962 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.962 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:11:29.962 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.251747+0000", 2023-10-01T11:11:29.962 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1763000000000001e-05 2023-10-01T11:11:29.962 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.962 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.962 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:11:29.962 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.991363+0000", 2023-10-01T11:11:29.963 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.73961652700000002 2023-10-01T11:11:29.963 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.963 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.963 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:11:29.963 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.991499+0000", 2023-10-01T11:11:29.963 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000135718 2023-10-01T11:11:29.963 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.963 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.963 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 6,7", 2023-10-01T11:11:29.963 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.991740+0000", 2023-10-01T11:11:29.963 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00024120200000000001 2023-10-01T11:11:29.963 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.964 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.964 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:11:29.964 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.996524+0000", 2023-10-01T11:11:29.964 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0047840929999999997 2023-10-01T11:11:29.964 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.964 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.964 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:11:29.964 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:15.002337+0000", 2023-10-01T11:11:29.964 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.005812841 2023-10-01T11:11:29.964 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.964 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.964 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:11:29.964 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:15.005256+0000", 2023-10-01T11:11:29.965 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0029195810000000001 2023-10-01T11:11:29.965 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.965 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.965 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:11:29.965 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:15.005286+0000", 2023-10-01T11:11:29.965 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.9294000000000001e-05 2023-10-01T11:11:29.965 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.965 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.965 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:11:29.965 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:15.005443+0000", 2023-10-01T11:11:29.965 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000156823 2023-10-01T11:11:29.965 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:11:29.965 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:11:29.966 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:11:29.966 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.966 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.966 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:143246 13.7 13:e5fe54ba:::benchmark_data_smithi078_202524_object63688:head [delete] snapc 0=[] ondisk+write+known_if_redirected e348)", 2023-10-01T11:11:29.966 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:08:14.259797+0000", 2023-10-01T11:11:29.966 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 195.582132543, 2023-10-01T11:11:29.966 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.74850766999999996, 2023-10-01T11:11:29.966 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:11:29.966 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:11:29.966 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:11:29.966 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:11:29.966 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:11:29.967 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 143246 2023-10-01T11:11:29.967 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.967 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:11:29.967 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.967 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:11:29.967 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.259797+0000", 2023-10-01T11:11:29.967 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:11:29.967 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.967 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.967 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:11:29.967 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.259797+0000", 2023-10-01T11:11:29.967 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:11:29.968 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.968 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.968 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:11:29.968 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.259795+0000", 2023-10-01T11:11:29.968 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999976 2023-10-01T11:11:29.968 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.968 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.968 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:11:29.968 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.259810+0000", 2023-10-01T11:11:29.968 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.5296999999999999e-05 2023-10-01T11:11:29.968 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.968 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.968 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:11:29.969 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.259842+0000", 2023-10-01T11:11:29.969 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.2177e-05 2023-10-01T11:11:29.969 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.969 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.969 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:11:29.969 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.259858+0000", 2023-10-01T11:11:29.969 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.5413999999999998e-05 2023-10-01T11:11:29.969 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.969 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.969 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:11:29.969 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.259935+0000", 2023-10-01T11:11:29.969 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.7715999999999999e-05 2023-10-01T11:11:29.969 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.970 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.970 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:11:29.970 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.259952+0000", 2023-10-01T11:11:29.970 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.6762999999999999e-05 2023-10-01T11:11:29.970 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.970 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.970 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:11:29.970 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.993190+0000", 2023-10-01T11:11:29.970 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.73323782999999998 2023-10-01T11:11:29.970 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.970 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.970 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:11:29.971 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.993344+0000", 2023-10-01T11:11:29.971 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00015383700000000001 2023-10-01T11:11:29.971 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.971 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.971 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 6,7", 2023-10-01T11:11:29.971 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.993558+0000", 2023-10-01T11:11:29.971 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000214549 2023-10-01T11:11:29.971 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.971 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.971 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:11:29.971 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.996529+0000", 2023-10-01T11:11:29.971 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.002970604 2023-10-01T11:11:29.971 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.972 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.972 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:11:29.972 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:15.005469+0000", 2023-10-01T11:11:29.972 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0089402849999999992 2023-10-01T11:11:29.972 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.972 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.972 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:11:29.972 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:15.007899+0000", 2023-10-01T11:11:29.972 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0024296159999999999 2023-10-01T11:11:29.972 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.972 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.972 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:11:29.972 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:15.007963+0000", 2023-10-01T11:11:29.973 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.4212000000000001e-05 2023-10-01T11:11:29.973 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.973 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.973 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:11:29.973 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:15.008305+0000", 2023-10-01T11:11:29.973 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000341978 2023-10-01T11:11:29.973 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:11:29.973 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:11:29.973 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:11:29.973 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.973 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.973 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:143308 13.7 13:e6767d8d:::benchmark_data_smithi078_202524_object63750:head [delete] snapc 0=[] ondisk+write+known_if_redirected e348)", 2023-10-01T11:11:29.973 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:08:14.439987+0000", 2023-10-01T11:11:29.974 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 195.401943037, 2023-10-01T11:11:29.974 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.56866824199999999, 2023-10-01T11:11:29.974 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:11:29.974 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:11:29.974 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:11:29.974 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:11:29.974 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:11:29.974 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 143308 2023-10-01T11:11:29.974 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.974 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:11:29.974 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.974 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:11:29.975 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.439987+0000", 2023-10-01T11:11:29.975 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:11:29.975 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.975 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.975 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:11:29.975 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.439987+0000", 2023-10-01T11:11:29.975 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:11:29.975 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.975 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.975 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:11:29.975 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.439985+0000", 2023-10-01T11:11:29.975 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-10-01T11:11:29.975 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.976 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.976 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:11:29.976 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.439994+0000", 2023-10-01T11:11:29.976 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.4909999999999994e-06 2023-10-01T11:11:29.976 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.976 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.976 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:11:29.976 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.440014+0000", 2023-10-01T11:11:29.976 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.9324e-05 2023-10-01T11:11:29.976 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.976 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.976 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:11:29.977 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.440024+0000", 2023-10-01T11:11:29.977 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0196e-05 2023-10-01T11:11:29.977 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.977 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.977 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:11:29.977 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.440090+0000", 2023-10-01T11:11:29.977 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.6225999999999999e-05 2023-10-01T11:11:29.977 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.977 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.977 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:11:29.977 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.440102+0000", 2023-10-01T11:11:29.977 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1472e-05 2023-10-01T11:11:29.978 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.978 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.978 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:11:29.978 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.994827+0000", 2023-10-01T11:11:29.978 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.55472525399999995 2023-10-01T11:11:29.978 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.978 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.978 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:11:29.978 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.994959+0000", 2023-10-01T11:11:29.978 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00013209499999999999 2023-10-01T11:11:29.978 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.978 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.979 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 6,7", 2023-10-01T11:11:29.979 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.995215+0000", 2023-10-01T11:11:29.979 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000256445 2023-10-01T11:11:29.979 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.979 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.979 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:11:29.979 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:14.996728+0000", 2023-10-01T11:11:29.979 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0015124209999999999 2023-10-01T11:11:29.979 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.979 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.979 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:11:29.980 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:15.007829+0000", 2023-10-01T11:11:29.980 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.011101408 2023-10-01T11:11:29.980 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.980 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.980 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:11:29.980 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:15.008364+0000", 2023-10-01T11:11:29.980 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00053432900000000005 2023-10-01T11:11:29.980 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.980 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.980 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:11:29.980 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:15.008414+0000", 2023-10-01T11:11:29.980 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.0506000000000001e-05 2023-10-01T11:11:29.981 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.981 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.981 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:11:29.981 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:08:15.008655+0000", 2023-10-01T11:11:29.981 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000240881 2023-10-01T11:11:29.981 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:11:29.981 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:11:29.981 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:11:29.981 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.981 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.981 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.36819.0:96561 14.e 14:72087225:::benchmark_data_smithi078_236918_object96560:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e361)", 2023-10-01T11:11:29.981 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:11:00.909628+0000", 2023-10-01T11:11:29.981 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 28.932302058000001, 2023-10-01T11:11:29.982 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.21198667199999999, 2023-10-01T11:11:29.982 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:11:29.982 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:11:29.982 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:11:29.982 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.36819", 2023-10-01T11:11:29.982 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/1409915880", 2023-10-01T11:11:29.982 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 96561 2023-10-01T11:11:29.982 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.982 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:11:29.982 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.982 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:11:29.983 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:11:00.909628+0000", 2023-10-01T11:11:29.983 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:11:29.983 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.983 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.983 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:11:29.983 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:11:00.909628+0000", 2023-10-01T11:11:29.983 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:11:29.983 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.983 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.983 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:11:29.983 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:11:00.909626+0000", 2023-10-01T11:11:29.983 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999976 2023-10-01T11:11:29.984 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.984 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.984 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:11:29.984 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:11:00.909723+0000", 2023-10-01T11:11:29.984 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.7849000000000005e-05 2023-10-01T11:11:29.984 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.984 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.984 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:11:29.984 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:11:00.909747+0000", 2023-10-01T11:11:29.984 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.3850999999999999e-05 2023-10-01T11:11:29.984 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.984 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.985 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:11:29.985 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:11:00.909763+0000", 2023-10-01T11:11:29.985 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.5608999999999999e-05 2023-10-01T11:11:29.985 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.985 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.985 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:11:29.985 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:11:00.909826+0000", 2023-10-01T11:11:29.985 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.3522999999999995e-05 2023-10-01T11:11:29.985 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.985 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.985 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:11:29.985 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:11:00.909956+0000", 2023-10-01T11:11:29.985 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000130016 2023-10-01T11:11:29.986 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.986 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.986 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 2,6", 2023-10-01T11:11:29.986 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:11:00.910077+0000", 2023-10-01T11:11:29.986 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000121079 2023-10-01T11:11:29.986 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.986 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.986 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:11:29.986 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:11:00.914525+0000", 2023-10-01T11:11:29.986 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0044479899999999998 2023-10-01T11:11:29.986 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.986 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.987 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:11:29.987 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:11:00.919795+0000", 2023-10-01T11:11:29.987 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0052697389999999998 2023-10-01T11:11:29.987 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.987 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.987 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:11:29.987 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:11:01.121485+0000", 2023-10-01T11:11:29.987 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.201690008 2023-10-01T11:11:29.987 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.987 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.987 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:11:29.987 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:11:01.121550+0000", 2023-10-01T11:11:29.988 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.4771999999999996e-05 2023-10-01T11:11:29.988 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:29.988 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:29.988 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:11:29.988 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:11:01.121614+0000", 2023-10-01T11:11:29.988 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.4472000000000002e-05 2023-10-01T11:11:29.988 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:11:29.988 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:11:29.988 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:11:29.988 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:11:29.988 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:11:29.988 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-10-01T11:11:29.989 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:30.010 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:30 smithi078 bash[75299]: cluster 2023-10-01T11:11:28.422011+0000 mgr.y (mgr.25072) 1626 : cluster 0 pgmap v1751: 89 pgs: 89 active+clean; 5.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 546 op/s 2023-10-01T11:11:30.019 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:30.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:30.023 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:11:30.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:30 smithi173 bash[58816]: cluster 2023-10-01T11:11:28.422011+0000 mgr.y (mgr.25072) 1626 : cluster 0 pgmap v1751: 89 pgs: 89 active+clean; 5.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 546 op/s 2023-10-01T11:11:30.124 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:30.157 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:11:30.168 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:30.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:30.174 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:11:30.273 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:11:30 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:11:30] "GET /metrics HTTP/1.1" 200 36385 "" "Prometheus/2.33.4" 2023-10-01T11:11:30.273 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:30 smithi078 bash[78274]: cluster 2023-10-01T11:11:28.422011+0000 mgr.y (mgr.25072) 1626 : cluster 0 pgmap v1751: 89 pgs: 89 active+clean; 5.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 546 op/s 2023-10-01T11:11:30.275 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:30.301 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:30.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:30.303 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:11:30.404 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:30.441 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:30.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:30.448 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:11:30.548 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:30.568 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:30.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:30.571 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:11:30.671 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:30.708 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:30.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:30.713 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:11:30.814 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:30.851 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:30.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:30.858 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:11:30.958 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:30.965 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-01T11:11:30.995 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:31.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:31.001 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:11:31.102 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:31.127 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:31.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:31.134 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:11:31.228 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:31 smithi078 bash[75299]: audit 2023-10-01T11:11:30.227567+0000 mon.a (mon.0) 2095 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:11:31.228 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:31 smithi078 bash[75299]: audit 2023-10-01T11:11:30.241799+0000 mon.a (mon.0) 2096 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:11:31.228 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:31 smithi078 bash[75299]: cluster 2023-10-01T11:11:30.422750+0000 mgr.y (mgr.25072) 1627 : cluster 0 pgmap v1752: 89 pgs: 89 active+clean; 5.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 773 op/s 2023-10-01T11:11:31.228 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:31 smithi078 bash[75299]: audit 2023-10-01T11:11:30.618126+0000 mon.a (mon.0) 2097 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:11:31.229 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:31 smithi078 bash[75299]: audit 2023-10-01T11:11:30.629712+0000 mon.a (mon.0) 2098 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:11:31.235 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:31.263 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:31.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:31.267 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:11:31.368 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:31.386 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:31.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:31.388 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:11:31.489 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:31.498 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:31 smithi173 bash[58816]: audit 2023-10-01T11:11:30.227567+0000 mon.a (mon.0) 2095 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:11:31.498 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:31 smithi173 bash[58816]: audit 2023-10-01T11:11:30.241799+0000 mon.a (mon.0) 2096 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:11:31.498 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:31 smithi173 bash[58816]: cluster 2023-10-01T11:11:30.422750+0000 mgr.y (mgr.25072) 1627 : cluster 0 pgmap v1752: 89 pgs: 89 active+clean; 5.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 773 op/s 2023-10-01T11:11:31.499 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:31 smithi173 bash[58816]: audit 2023-10-01T11:11:30.618126+0000 mon.a (mon.0) 2097 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:11:31.499 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:31 smithi173 bash[58816]: audit 2023-10-01T11:11:30.629712+0000 mon.a (mon.0) 2098 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:11:31.524 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:31.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:31.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:11:31.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:31 smithi078 bash[78274]: audit 2023-10-01T11:11:30.227567+0000 mon.a (mon.0) 2095 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:11:31.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:31 smithi078 bash[78274]: audit 2023-10-01T11:11:30.241799+0000 mon.a (mon.0) 2096 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:11:31.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:31 smithi078 bash[78274]: cluster 2023-10-01T11:11:30.422750+0000 mgr.y (mgr.25072) 1627 : cluster 0 pgmap v1752: 89 pgs: 89 active+clean; 5.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 773 op/s 2023-10-01T11:11:31.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:31 smithi078 bash[78274]: audit 2023-10-01T11:11:30.618126+0000 mon.a (mon.0) 2097 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:11:31.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:31 smithi078 bash[78274]: audit 2023-10-01T11:11:30.629712+0000 mon.a (mon.0) 2098 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:11:31.628 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:31.661 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:31.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:31.665 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:11:31.766 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:31.790 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:31.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:31.795 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:11:31.896 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:31.918 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:31.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:31.920 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:11:32.021 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:32.052 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:32.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:32.058 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:11:32.158 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:32.186 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:32.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:32.190 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:11:32.192 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:11:32.193 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:11:32.291 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:32.319 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:32.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:32.322 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:11:32.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:32 smithi078 bash[75299]: audit 2023-10-01T11:11:31.315353+0000 mon.a (mon.0) 2099 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T11:11:32.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:32 smithi078 bash[75299]: audit 2023-10-01T11:11:31.316347+0000 mon.a (mon.0) 2100 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T11:11:32.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:32 smithi078 bash[75299]: audit 2023-10-01T11:11:31.323935+0000 mon.a (mon.0) 2101 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:11:32.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:32 smithi078 bash[78274]: audit 2023-10-01T11:11:31.315353+0000 mon.a (mon.0) 2099 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T11:11:32.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:32 smithi078 bash[78274]: audit 2023-10-01T11:11:31.316347+0000 mon.a (mon.0) 2100 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T11:11:32.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:32 smithi078 bash[78274]: audit 2023-10-01T11:11:31.323935+0000 mon.a (mon.0) 2101 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:11:32.423 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:32.448 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:32.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:32.451 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:11:32.551 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:32.580 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:32.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:32.584 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:11:32.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:32 smithi173 bash[58816]: audit 2023-10-01T11:11:31.315353+0000 mon.a (mon.0) 2099 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T11:11:32.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:32 smithi173 bash[58816]: audit 2023-10-01T11:11:31.316347+0000 mon.a (mon.0) 2100 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T11:11:32.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:32 smithi173 bash[58816]: audit 2023-10-01T11:11:31.323935+0000 mon.a (mon.0) 2101 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:11:32.685 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:32.713 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:32.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:32.719 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:11:32.820 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:32.853 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:32.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:32.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:11:32.956 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:32.979 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:32.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:32.981 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:11:33.082 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:33.112 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:33.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:33.115 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:11:33.216 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:33.235 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:33.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:33.236 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:11:33.337 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:33.371 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:33.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:33.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:11:33.478 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:33.506 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:33.509 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:33 smithi173 bash[58816]: cluster 2023-10-01T11:11:32.423820+0000 mgr.y (mgr.25072) 1628 : cluster 0 pgmap v1753: 89 pgs: 89 active+clean; 5.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 718 op/s 2023-10-01T11:11:33.509 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:33 smithi173 bash[58816]: audit 2023-10-01T11:11:32.470300+0000 mon.a (mon.0) 2102 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:11:33.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:33.510 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:11:33.570 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:33 smithi078 bash[78274]: cluster 2023-10-01T11:11:32.423820+0000 mgr.y (mgr.25072) 1628 : cluster 0 pgmap v1753: 89 pgs: 89 active+clean; 5.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 718 op/s 2023-10-01T11:11:33.570 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:33 smithi078 bash[78274]: audit 2023-10-01T11:11:32.470300+0000 mon.a (mon.0) 2102 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:11:33.611 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:33.641 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:33.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:33.643 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:11:33.744 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:33.768 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:33.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:33.774 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:11:33.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:33 smithi078 bash[75299]: cluster 2023-10-01T11:11:32.423820+0000 mgr.y (mgr.25072) 1628 : cluster 0 pgmap v1753: 89 pgs: 89 active+clean; 5.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 718 op/s 2023-10-01T11:11:33.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:33 smithi078 bash[75299]: audit 2023-10-01T11:11:32.470300+0000 mon.a (mon.0) 2102 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:11:33.876 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:33.903 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:33.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:33.906 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:11:34.006 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:34.033 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:34.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:34.036 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:11:34.137 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:34.173 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:34.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:34.178 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:11:34.279 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:34.314 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:34.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:34.320 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:11:34.421 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:34.444 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:34.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:34.450 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:11:34.553 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:34.582 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:34.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:34.588 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:11:34.689 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:34.721 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:34.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:34.724 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:11:34.825 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:34.860 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:34.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:34.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:11:34.967 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:34.993 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:34.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:34.999 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:11:35.099 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:35.124 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:35.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:35.126 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:11:35.227 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:35.255 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:35.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:35.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:11:35.360 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:35.392 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:35.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:35.397 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:11:35.498 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:35.522 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:35.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:35.528 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:11:35.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:35 smithi173 bash[58816]: cluster 2023-10-01T11:11:34.424459+0000 mgr.y (mgr.25072) 1629 : cluster 0 pgmap v1754: 89 pgs: 89 active+clean; 5.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 828 op/s 2023-10-01T11:11:35.629 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:35.655 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:35.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:35.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:11:35.759 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:35.765 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T11:11:35.786 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:35.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:35.791 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:11:35.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:35 smithi078 bash[75299]: cluster 2023-10-01T11:11:34.424459+0000 mgr.y (mgr.25072) 1629 : cluster 0 pgmap v1754: 89 pgs: 89 active+clean; 5.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 828 op/s 2023-10-01T11:11:35.831 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:35 smithi078 bash[78274]: cluster 2023-10-01T11:11:34.424459+0000 mgr.y (mgr.25072) 1629 : cluster 0 pgmap v1754: 89 pgs: 89 active+clean; 5.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 828 op/s 2023-10-01T11:11:35.891 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:35.932 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:35.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:35.936 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:11:36.037 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:36.056 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:36.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:36.059 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:11:36.160 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:36.186 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:36.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:36.189 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:11:36.290 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:36.341 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:36.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:36.343 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:11:36.443 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:36.473 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:36.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:36.477 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:11:36.577 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:36.610 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:36.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:36.616 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:11:36.717 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:36.743 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:36.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:36.747 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:11:36.848 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:36.875 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:36.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:36.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:11:36.982 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:37.004 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:37.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:37.006 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:11:37.106 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:37.140 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:37.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:37.142 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:11:37.243 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:37.273 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:37.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:37.277 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:11:37.378 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:37.410 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:37.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:37.413 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:11:37.483 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:37 smithi078 bash[75299]: cluster 2023-10-01T11:11:36.424961+0000 mgr.y (mgr.25072) 1630 : cluster 0 pgmap v1755: 89 pgs: 89 active+clean; 5.5 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 676 op/s 2023-10-01T11:11:37.513 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:37.520 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:37 smithi173 bash[58816]: cluster 2023-10-01T11:11:36.424961+0000 mgr.y (mgr.25072) 1630 : cluster 0 pgmap v1755: 89 pgs: 89 active+clean; 5.5 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 676 op/s 2023-10-01T11:11:37.536 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:37.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:37.539 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:11:37.639 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:37.648 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:37 smithi078 bash[78274]: cluster 2023-10-01T11:11:36.424961+0000 mgr.y (mgr.25072) 1630 : cluster 0 pgmap v1755: 89 pgs: 89 active+clean; 5.5 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 676 op/s 2023-10-01T11:11:37.664 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:37.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:37.667 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:11:37.768 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:37.792 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:37.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:37.796 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:11:37.896 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:37.921 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:37.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:37.925 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:11:38.025 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:38.050 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:38.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:38.056 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:11:38.157 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:38.192 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:38.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:38.196 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:11:38.296 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:38.322 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:38.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:38.327 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:11:38.428 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:38.512 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:38.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:38.514 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:11:38.615 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:38.634 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:38.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:38.636 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:11:38.737 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:38.763 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:38.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:38.767 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:11:38.868 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:39.073 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:39.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:39.076 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:11:39.177 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:39.215 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:39.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:39.220 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:11:39.320 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:39.353 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:39.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:39.360 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:11:39.460 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:39.490 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:39.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:39.494 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:11:39.595 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:39.622 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:39.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:39.627 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:11:39.727 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:39.748 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:39.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:39.750 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:11:39.851 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:39.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:39 smithi173 bash[58816]: cluster 2023-10-01T11:11:38.425373+0000 mgr.y (mgr.25072) 1631 : cluster 0 pgmap v1756: 89 pgs: 89 active+clean; 5.5 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 580 op/s 2023-10-01T11:11:39.881 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:39.883 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:39 smithi078 bash[78274]: cluster 2023-10-01T11:11:38.425373+0000 mgr.y (mgr.25072) 1631 : cluster 0 pgmap v1756: 89 pgs: 89 active+clean; 5.5 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 580 op/s 2023-10-01T11:11:39.883 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:39 smithi078 bash[75299]: cluster 2023-10-01T11:11:38.425373+0000 mgr.y (mgr.25072) 1631 : cluster 0 pgmap v1756: 89 pgs: 89 active+clean; 5.5 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 580 op/s 2023-10-01T11:11:39.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:39.884 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:11:39.985 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:40.013 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:40.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:40.016 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:11:40.117 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:40.155 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:40.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:40.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:11:40.260 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:40.269 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:11:40 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:11:40] "GET /metrics HTTP/1.1" 200 36385 "" "Prometheus/2.33.4" 2023-10-01T11:11:40.290 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:40.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:40.294 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:11:40.395 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:40.421 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:40.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:40.427 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:11:40.528 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:40.561 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:40.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:40.563 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:11:40.567 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-10-01T11:11:40.567 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [ 2023-10-01T11:11:40.567 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:40.567 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.36819.0:124856 14.a 14:5f97cc46:::benchmark_data_smithi078_236918_object19710:head [delete] snapc 0=[] ondisk+write+known_if_redirected e364)", 2023-10-01T11:11:40.567 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:11:40.556457+0000", 2023-10-01T11:11:40.567 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 0.010302149, 2023-10-01T11:11:40.567 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.010326925000000001, 2023-10-01T11:11:40.567 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:11:40.568 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "waiting for sub ops", 2023-10-01T11:11:40.568 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:11:40.568 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.36819", 2023-10-01T11:11:40.568 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/1409915880", 2023-10-01T11:11:40.568 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 124856 2023-10-01T11:11:40.568 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:40.568 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:11:40.568 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:40.568 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:11:40.568 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:11:40.556457+0000", 2023-10-01T11:11:40.568 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:11:40.569 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:40.569 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:40.569 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:11:40.569 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:11:40.556457+0000", 2023-10-01T11:11:40.569 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999976 2023-10-01T11:11:40.569 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:40.569 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:40.569 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:11:40.569 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:11:40.556454+0000", 2023-10-01T11:11:40.569 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.6410000000000008e-06 2023-10-01T11:11:40.569 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:40.569 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:40.570 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:11:40.570 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:11:40.556463+0000", 2023-10-01T11:11:40.570 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.6877000000000001e-05 2023-10-01T11:11:40.570 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:40.570 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:40.570 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:11:40.570 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:11:40.556480+0000", 2023-10-01T11:11:40.570 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.0709999999999995e-06 2023-10-01T11:11:40.570 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:40.570 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:40.570 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:11:40.571 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:11:40.556488+0000", 2023-10-01T11:11:40.571 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.579e-05 2023-10-01T11:11:40.571 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:40.571 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:40.571 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:11:40.571 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:11:40.556534+0000", 2023-10-01T11:11:40.571 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00012272100000000001 2023-10-01T11:11:40.571 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:40.571 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:40.571 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:11:40.571 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:11:40.556657+0000", 2023-10-01T11:11:40.571 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00011972 2023-10-01T11:11:40.572 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:40.572 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:40.572 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1,3", 2023-10-01T11:11:40.572 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:11:40.556776+0000", 2023-10-01T11:11:40.572 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0041855 2023-10-01T11:11:40.572 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:40.572 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:40.572 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:11:40.572 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:11:40.560962+0000", 2023-10-01T11:11:40.572 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0045050610000000003 2023-10-01T11:11:40.572 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:11:40.573 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:11:40.573 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:11:40.573 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:40.573 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:40.573 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.36819.0:124861 14.a 14:54809c2b:::benchmark_data_smithi078_236918_object19715:head [delete] snapc 0=[] ondisk+write+known_if_redirected e364)", 2023-10-01T11:11:40.573 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:11:40.561627+0000", 2023-10-01T11:11:40.573 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 0.0051319199999999999, 2023-10-01T11:11:40.573 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0052161990000000004, 2023-10-01T11:11:40.573 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:11:40.573 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "waiting for sub ops", 2023-10-01T11:11:40.573 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:11:40.573 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.36819", 2023-10-01T11:11:40.574 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/1409915880", 2023-10-01T11:11:40.574 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 124861 2023-10-01T11:11:40.574 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:40.574 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:11:40.574 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:40.574 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:11:40.574 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:11:40.561627+0000", 2023-10-01T11:11:40.574 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:11:40.574 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:40.574 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:40.574 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:11:40.575 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:11:40.561627+0000", 2023-10-01T11:11:40.575 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-10-01T11:11:40.575 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:40.575 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:40.575 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:11:40.575 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:11:40.561625+0000", 2023-10-01T11:11:40.575 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.4609999999999996e-06 2023-10-01T11:11:40.575 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:40.575 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:40.575 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:11:40.575 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:11:40.561632+0000", 2023-10-01T11:11:40.575 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2841e-05 2023-10-01T11:11:40.575 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:40.576 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:40.576 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:11:40.576 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:11:40.561645+0000", 2023-10-01T11:11:40.576 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.4970000000000001e-06 2023-10-01T11:11:40.576 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:40.576 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:40.576 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:11:40.576 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:11:40.561650+0000", 2023-10-01T11:11:40.576 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.5988999999999997e-05 2023-10-01T11:11:40.576 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:40.576 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:40.577 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:11:40.577 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:11:40.561686+0000", 2023-10-01T11:11:40.577 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.2344999999999998e-05 2023-10-01T11:11:40.577 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:40.577 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:40.577 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:11:40.577 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:11:40.561778+0000", 2023-10-01T11:11:40.577 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.9489999999999999e-05 2023-10-01T11:11:40.577 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:40.577 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:40.577 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 1,3", 2023-10-01T11:11:40.578 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:11:40.561868+0000", 2023-10-01T11:11:40.578 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0034253259999999998 2023-10-01T11:11:40.578 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:11:40.578 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:11:40.578 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:11:40.578 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:11:40.565293+0000", 2023-10-01T11:11:40.578 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0036663949999999998 2023-10-01T11:11:40.578 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:11:40.578 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:11:40.578 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:11:40.579 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:11:40.579 INFO:teuthology.orchestra.run.smithi173.stdout: ], 2023-10-01T11:11:40.579 INFO:teuthology.orchestra.run.smithi173.stdout: "num_ops": 2 2023-10-01T11:11:40.579 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-10-01T11:11:40.664 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:40.687 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:40.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:40.689 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:11:40.790 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:40.818 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:40.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:40.823 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:11:40.924 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:40.960 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:40.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:40.965 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:11:41.066 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:41.097 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:41.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:41.101 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:11:41.202 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:41.231 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:41.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:41.233 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:11:41.334 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:41.360 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:41.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:41.363 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:11:41.464 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:41.484 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:41.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:41.486 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:11:41.587 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:41.592 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-01T11:11:41.620 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:41.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:41.626 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:11:41.726 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:41.766 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:41.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:41.773 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:11:41.852 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:41 smithi173 bash[58816]: cluster 2023-10-01T11:11:40.426392+0000 mgr.y (mgr.25072) 1632 : cluster 0 pgmap v1757: 89 pgs: 89 active+clean; 5.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 818 op/s 2023-10-01T11:11:41.874 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:41.905 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:41.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:41.912 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:11:42.013 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:42.036 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:42.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:42.040 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:11:42.141 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:42.170 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:42.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:42.173 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:11:42.273 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:42.299 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:42.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:42.301 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:11:42.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:41 smithi078 bash[75299]: cluster 2023-10-01T11:11:40.426392+0000 mgr.y (mgr.25072) 1632 : cluster 0 pgmap v1757: 89 pgs: 89 active+clean; 5.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 818 op/s 2023-10-01T11:11:42.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:41 smithi078 bash[78274]: cluster 2023-10-01T11:11:40.426392+0000 mgr.y (mgr.25072) 1632 : cluster 0 pgmap v1757: 89 pgs: 89 active+clean; 5.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 818 op/s 2023-10-01T11:11:42.402 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:42.435 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:42.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:42.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:11:42.538 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:42.566 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:42.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:42.572 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:11:42.672 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:42.701 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:42.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:42.706 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:11:42.806 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:42.833 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:42.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:42.834 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:11:42.871 INFO:teuthology.orchestra.run.smithi078.stderr:reweighted osd.2 to 1 (10000) 2023-10-01T11:11:42.935 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:42.981 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:42.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:42.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:11:43.086 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:43.095 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:42 smithi173 bash[58816]: audit 2023-10-01T11:11:42.534928+0000 mon.c (mon.2) 240 : audit 1 from='client.? 172.21.15.78:0/820529338' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-01T11:11:43.096 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:42 smithi173 bash[58816]: audit 2023-10-01T11:11:42.536613+0000 mon.a (mon.0) 2103 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-01T11:11:43.109 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:43.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:43.111 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:11:43.177 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:42 smithi078 bash[78274]: audit 2023-10-01T11:11:42.534928+0000 mon.c (mon.2) 240 : audit 1 from='client.? 172.21.15.78:0/820529338' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-01T11:11:43.177 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:42 smithi078 bash[78274]: audit 2023-10-01T11:11:42.536613+0000 mon.a (mon.0) 2103 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-01T11:11:43.177 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:42 smithi078 bash[75299]: audit 2023-10-01T11:11:42.534928+0000 mon.c (mon.2) 240 : audit 1 from='client.? 172.21.15.78:0/820529338' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-01T11:11:43.178 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:42 smithi078 bash[75299]: audit 2023-10-01T11:11:42.536613+0000 mon.a (mon.0) 2103 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-01T11:11:43.212 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:43.240 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:43.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:43.243 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:11:43.344 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:43.368 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:43.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:43.370 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:11:43.471 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:43.502 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:43.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:43.509 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:11:43.610 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:43.636 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:43.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:43.639 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:11:43.740 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:43.763 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:43.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:43.766 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:11:43.867 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:43.872 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:43 smithi078 bash[78274]: cluster 2023-10-01T11:11:42.426836+0000 mgr.y (mgr.25072) 1633 : cluster 0 pgmap v1758: 89 pgs: 89 active+clean; 5.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 629 op/s 2023-10-01T11:11:43.872 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:43 smithi078 bash[78274]: cluster 2023-10-01T11:11:42.856685+0000 osd.5 (osd.5) 387 : cluster 3 failed to encode map e365 with expected crc 2023-10-01T11:11:43.872 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:43 smithi078 bash[78274]: cluster 2023-10-01T11:11:42.858491+0000 osd.7 (osd.7) 410 : cluster 3 failed to encode map e365 with expected crc 2023-10-01T11:11:43.872 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:43 smithi078 bash[78274]: audit 2023-10-01T11:11:42.859500+0000 mon.a (mon.0) 2104 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-01T11:11:43.872 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:43 smithi078 bash[78274]: cluster 2023-10-01T11:11:42.859810+0000 mon.a (mon.0) 2105 : cluster 0 osdmap e365: 8 total, 8 up, 8 in 2023-10-01T11:11:43.872 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:43 smithi078 bash[78274]: cluster 2023-10-01T11:11:42.864925+0000 osd.6 (osd.6) 351 : cluster 3 failed to encode map e365 with expected crc 2023-10-01T11:11:43.872 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:43 smithi078 bash[78274]: cluster 2023-10-01T11:11:42.866173+0000 osd.5 (osd.5) 388 : cluster 3 failed to encode map e365 with expected crc 2023-10-01T11:11:43.872 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:43 smithi078 bash[75299]: cluster 2023-10-01T11:11:42.426836+0000 mgr.y (mgr.25072) 1633 : cluster 0 pgmap v1758: 89 pgs: 89 active+clean; 5.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 629 op/s 2023-10-01T11:11:43.873 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:43 smithi078 bash[75299]: cluster 2023-10-01T11:11:42.856685+0000 osd.5 (osd.5) 387 : cluster 3 failed to encode map e365 with expected crc 2023-10-01T11:11:43.873 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:43 smithi078 bash[75299]: cluster 2023-10-01T11:11:42.858491+0000 osd.7 (osd.7) 410 : cluster 3 failed to encode map e365 with expected crc 2023-10-01T11:11:43.873 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:43 smithi078 bash[75299]: audit 2023-10-01T11:11:42.859500+0000 mon.a (mon.0) 2104 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-01T11:11:43.873 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:43 smithi078 bash[75299]: cluster 2023-10-01T11:11:42.859810+0000 mon.a (mon.0) 2105 : cluster 0 osdmap e365: 8 total, 8 up, 8 in 2023-10-01T11:11:43.873 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:43 smithi078 bash[75299]: cluster 2023-10-01T11:11:42.864925+0000 osd.6 (osd.6) 351 : cluster 3 failed to encode map e365 with expected crc 2023-10-01T11:11:43.873 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:43 smithi078 bash[75299]: cluster 2023-10-01T11:11:42.866173+0000 osd.5 (osd.5) 388 : cluster 3 failed to encode map e365 with expected crc 2023-10-01T11:11:43.909 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:43.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:43.913 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:11:44.014 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:44.052 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:44.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:44.056 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:11:44.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:43 smithi173 bash[58816]: cluster 2023-10-01T11:11:42.426836+0000 mgr.y (mgr.25072) 1633 : cluster 0 pgmap v1758: 89 pgs: 89 active+clean; 5.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 629 op/s 2023-10-01T11:11:44.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:43 smithi173 bash[58816]: cluster 2023-10-01T11:11:42.856685+0000 osd.5 (osd.5) 387 : cluster 3 failed to encode map e365 with expected crc 2023-10-01T11:11:44.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:43 smithi173 bash[58816]: cluster 2023-10-01T11:11:42.858491+0000 osd.7 (osd.7) 410 : cluster 3 failed to encode map e365 with expected crc 2023-10-01T11:11:44.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:43 smithi173 bash[58816]: audit 2023-10-01T11:11:42.859500+0000 mon.a (mon.0) 2104 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-01T11:11:44.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:43 smithi173 bash[58816]: cluster 2023-10-01T11:11:42.859810+0000 mon.a (mon.0) 2105 : cluster 0 osdmap e365: 8 total, 8 up, 8 in 2023-10-01T11:11:44.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:43 smithi173 bash[58816]: cluster 2023-10-01T11:11:42.864925+0000 osd.6 (osd.6) 351 : cluster 3 failed to encode map e365 with expected crc 2023-10-01T11:11:44.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:43 smithi173 bash[58816]: cluster 2023-10-01T11:11:42.866173+0000 osd.5 (osd.5) 388 : cluster 3 failed to encode map e365 with expected crc 2023-10-01T11:11:44.156 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:44.182 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:44.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:44.185 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:11:44.286 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:44.313 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:44.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:44.315 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:11:44.416 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:44.444 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:44.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:44.447 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:11:44.548 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:44.573 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:44.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:44.579 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:11:44.680 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:44.704 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:44.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:44.708 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:11:44.809 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:44.834 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:44.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:44.838 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:11:44.938 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:44.961 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:44.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:44.969 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:11:45.069 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:45.092 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:45.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:45.094 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:11:45.195 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:45.219 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:45.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:45.222 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:11:45.322 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:45.347 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:45.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:45.353 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:11:45.453 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:45.484 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:45.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:45.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:11:45.587 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:45.608 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:45.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:45.617 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:11:45.717 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:45.752 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:45.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:45.754 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:11:45.793 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-01T11:11:45.855 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:45.875 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:45 smithi078 bash[75299]: cluster 2023-10-01T11:11:44.427450+0000 mgr.y (mgr.25072) 1634 : cluster 0 pgmap v1760: 89 pgs: 89 active+clean; 5.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2023-10-01T11:11:45.876 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:45 smithi078 bash[78274]: cluster 2023-10-01T11:11:44.427450+0000 mgr.y (mgr.25072) 1634 : cluster 0 pgmap v1760: 89 pgs: 89 active+clean; 5.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2023-10-01T11:11:45.916 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:45.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:45.919 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:11:46.020 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:46.053 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:46.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:46.063 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:11:46.164 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:46.201 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T11:11:46.213 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:46.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:46.216 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:11:46.270 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T11:11:46.306 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T11:11:46.317 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:46.345 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T11:11:46.348 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:46.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:46.351 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:11:46.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:45 smithi173 bash[58816]: cluster 2023-10-01T11:11:44.427450+0000 mgr.y (mgr.25072) 1634 : cluster 0 pgmap v1760: 89 pgs: 89 active+clean; 5.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2023-10-01T11:11:46.384 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T11:11:46.427 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T11:11:46.428 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T11:11:46.452 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:46.464 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T11:11:46.477 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:46.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:46.480 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:11:46.492 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T11:11:46.498 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T11:11:46.498 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T11:11:46.499 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T11:11:46.499 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T11:11:46.499 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T11:11:46.499 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T11:11:46.499 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T11:11:46.499 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T11:11:46.581 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:46.621 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:46.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:46.624 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:11:46.724 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:46.751 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:46.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:46.755 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:11:46.856 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:46.900 INFO:teuthology.orchestra.run.smithi078.stderr:nodeep-scrub is unset 2023-10-01T11:11:46.902 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:46.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:46.907 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:11:47.008 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:47.032 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:47.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:47.038 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:11:47.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:46 smithi078 bash[75299]: audit 2023-10-01T11:11:46.125946+0000 mon.a (mon.0) 2106 : audit 0 from='client.? 172.21.15.78:0/3108865331' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:11:47.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:46 smithi078 bash[75299]: audit 2023-10-01T11:11:46.546586+0000 mon.c (mon.2) 241 : audit 1 from='client.? 172.21.15.78:0/136568906' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:11:47.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:46 smithi078 bash[75299]: audit 2023-10-01T11:11:46.548289+0000 mon.a (mon.0) 2107 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:11:47.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:46 smithi078 bash[78274]: audit 2023-10-01T11:11:46.125946+0000 mon.a (mon.0) 2106 : audit 0 from='client.? 172.21.15.78:0/3108865331' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:11:47.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:46 smithi078 bash[78274]: audit 2023-10-01T11:11:46.546586+0000 mon.c (mon.2) 241 : audit 1 from='client.? 172.21.15.78:0/136568906' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:11:47.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:46 smithi078 bash[78274]: audit 2023-10-01T11:11:46.548289+0000 mon.a (mon.0) 2107 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:11:47.138 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:47.146 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:46 smithi173 bash[58816]: audit 2023-10-01T11:11:46.125946+0000 mon.a (mon.0) 2106 : audit 0 from='client.? 172.21.15.78:0/3108865331' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:11:47.146 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:46 smithi173 bash[58816]: audit 2023-10-01T11:11:46.546586+0000 mon.c (mon.2) 241 : audit 1 from='client.? 172.21.15.78:0/136568906' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:11:47.146 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:46 smithi173 bash[58816]: audit 2023-10-01T11:11:46.548289+0000 mon.a (mon.0) 2107 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:11:47.169 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:47.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:47.171 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:11:47.272 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:47.295 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:47.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:47.299 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:11:47.399 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:47.424 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:47.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:47.430 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:11:47.531 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:47.567 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:47.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:47.571 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:11:47.671 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:47.693 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:47.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:47.696 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:11:47.796 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:47.826 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:47.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:47.831 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:11:47.932 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:47.960 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:47.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:47.963 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:11:48.064 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:48.073 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:47 smithi173 bash[58816]: cluster 2023-10-01T11:11:46.428018+0000 mgr.y (mgr.25072) 1635 : cluster 0 pgmap v1761: 89 pgs: 89 active+clean; 5.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 682 op/s 2023-10-01T11:11:48.073 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:47 smithi173 bash[58816]: cluster 2023-10-01T11:11:46.869998+0000 mon.a (mon.0) 2108 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T11:11:48.073 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:47 smithi173 bash[58816]: cluster 2023-10-01T11:11:46.870037+0000 mon.a (mon.0) 2109 : cluster 1 Cluster is now healthy 2023-10-01T11:11:48.074 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:47 smithi173 bash[58816]: cluster 2023-10-01T11:11:46.883481+0000 osd.5 (osd.5) 389 : cluster 3 failed to encode map e366 with expected crc 2023-10-01T11:11:48.074 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:47 smithi173 bash[58816]: audit 2023-10-01T11:11:46.885769+0000 mon.a (mon.0) 2110 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T11:11:48.074 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:47 smithi173 bash[58816]: cluster 2023-10-01T11:11:46.886218+0000 mon.a (mon.0) 2111 : cluster 0 osdmap e366: 8 total, 8 up, 8 in 2023-10-01T11:11:48.074 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:47 smithi173 bash[58816]: cluster 2023-10-01T11:11:46.886529+0000 osd.7 (osd.7) 411 : cluster 3 failed to encode map e366 with expected crc 2023-10-01T11:11:48.074 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:47 smithi173 bash[58816]: cluster 2023-10-01T11:11:46.889989+0000 osd.6 (osd.6) 352 : cluster 3 failed to encode map e366 with expected crc 2023-10-01T11:11:48.074 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:47 smithi173 bash[58816]: audit 2023-10-01T11:11:47.474715+0000 mon.a (mon.0) 2112 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:11:48.074 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:47 smithi173 bash[58816]: audit 2023-10-01T11:11:47.475940+0000 mon.a (mon.0) 2113 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:11:48.092 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:48.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:48.094 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:11:48.195 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:48.204 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:47 smithi078 bash[78274]: cluster 2023-10-01T11:11:46.428018+0000 mgr.y (mgr.25072) 1635 : cluster 0 pgmap v1761: 89 pgs: 89 active+clean; 5.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 682 op/s 2023-10-01T11:11:48.204 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:47 smithi078 bash[78274]: cluster 2023-10-01T11:11:46.869998+0000 mon.a (mon.0) 2108 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T11:11:48.204 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:47 smithi078 bash[78274]: cluster 2023-10-01T11:11:46.870037+0000 mon.a (mon.0) 2109 : cluster 1 Cluster is now healthy 2023-10-01T11:11:48.204 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:47 smithi078 bash[78274]: cluster 2023-10-01T11:11:46.883481+0000 osd.5 (osd.5) 389 : cluster 3 failed to encode map e366 with expected crc 2023-10-01T11:11:48.204 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:47 smithi078 bash[78274]: audit 2023-10-01T11:11:46.885769+0000 mon.a (mon.0) 2110 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T11:11:48.204 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:47 smithi078 bash[78274]: cluster 2023-10-01T11:11:46.886218+0000 mon.a (mon.0) 2111 : cluster 0 osdmap e366: 8 total, 8 up, 8 in 2023-10-01T11:11:48.204 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:47 smithi078 bash[78274]: cluster 2023-10-01T11:11:46.886529+0000 osd.7 (osd.7) 411 : cluster 3 failed to encode map e366 with expected crc 2023-10-01T11:11:48.205 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:47 smithi078 bash[78274]: cluster 2023-10-01T11:11:46.889989+0000 osd.6 (osd.6) 352 : cluster 3 failed to encode map e366 with expected crc 2023-10-01T11:11:48.205 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:47 smithi078 bash[78274]: audit 2023-10-01T11:11:47.474715+0000 mon.a (mon.0) 2112 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:11:48.205 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:47 smithi078 bash[78274]: audit 2023-10-01T11:11:47.475940+0000 mon.a (mon.0) 2113 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:11:48.205 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:47 smithi078 bash[75299]: cluster 2023-10-01T11:11:46.428018+0000 mgr.y (mgr.25072) 1635 : cluster 0 pgmap v1761: 89 pgs: 89 active+clean; 5.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 682 op/s 2023-10-01T11:11:48.205 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:47 smithi078 bash[75299]: cluster 2023-10-01T11:11:46.869998+0000 mon.a (mon.0) 2108 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T11:11:48.205 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:47 smithi078 bash[75299]: cluster 2023-10-01T11:11:46.870037+0000 mon.a (mon.0) 2109 : cluster 1 Cluster is now healthy 2023-10-01T11:11:48.205 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:47 smithi078 bash[75299]: cluster 2023-10-01T11:11:46.883481+0000 osd.5 (osd.5) 389 : cluster 3 failed to encode map e366 with expected crc 2023-10-01T11:11:48.205 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:47 smithi078 bash[75299]: audit 2023-10-01T11:11:46.885769+0000 mon.a (mon.0) 2110 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T11:11:48.205 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:47 smithi078 bash[75299]: cluster 2023-10-01T11:11:46.886218+0000 mon.a (mon.0) 2111 : cluster 0 osdmap e366: 8 total, 8 up, 8 in 2023-10-01T11:11:48.205 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:47 smithi078 bash[75299]: cluster 2023-10-01T11:11:46.886529+0000 osd.7 (osd.7) 411 : cluster 3 failed to encode map e366 with expected crc 2023-10-01T11:11:48.205 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:47 smithi078 bash[75299]: cluster 2023-10-01T11:11:46.889989+0000 osd.6 (osd.6) 352 : cluster 3 failed to encode map e366 with expected crc 2023-10-01T11:11:48.206 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:47 smithi078 bash[75299]: audit 2023-10-01T11:11:47.474715+0000 mon.a (mon.0) 2112 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:11:48.206 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:47 smithi078 bash[75299]: audit 2023-10-01T11:11:47.475940+0000 mon.a (mon.0) 2113 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:11:48.227 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:48.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:48.234 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:11:48.335 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:48.368 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:48.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:48.370 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:11:48.471 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:48.496 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:48.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:48.498 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:11:48.599 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:48.627 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:48.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:48.633 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:11:48.734 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:48.766 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:48.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:48.771 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:11:48.871 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:48.906 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:48.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:48.910 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:11:49.011 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:49.058 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:49.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:49.066 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:11:49.167 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:49.198 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:49.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:49.203 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:11:49.304 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:49.332 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:49.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:49.341 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:11:49.442 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:49.465 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:49.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:49.468 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:11:49.568 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:49.597 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:49.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:49.600 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:11:49.700 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:49.730 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:49.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:49.734 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:11:49.835 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:49.878 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:49.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:49.884 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:11:49.984 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:49.993 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:49 smithi173 bash[58816]: cluster 2023-10-01T11:11:48.428453+0000 mgr.y (mgr.25072) 1636 : cluster 0 pgmap v1763: 89 pgs: 89 active+clean; 5.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 495 op/s 2023-10-01T11:11:50.025 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:50.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:50.031 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:11:50.132 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:50.168 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:50.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:50.171 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:11:50.271 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:50.272 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:49 smithi078 bash[75299]: cluster 2023-10-01T11:11:48.428453+0000 mgr.y (mgr.25072) 1636 : cluster 0 pgmap v1763: 89 pgs: 89 active+clean; 5.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 495 op/s 2023-10-01T11:11:50.273 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:11:50 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:11:50] "GET /metrics HTTP/1.1" 200 36385 "" "Prometheus/2.33.4" 2023-10-01T11:11:50.273 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:49 smithi078 bash[78274]: cluster 2023-10-01T11:11:48.428453+0000 mgr.y (mgr.25072) 1636 : cluster 0 pgmap v1763: 89 pgs: 89 active+clean; 5.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 495 op/s 2023-10-01T11:11:50.304 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:50.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:50.307 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:11:50.407 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:50.439 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:50.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:50.444 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:11:50.544 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:50.597 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:50.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:50.599 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:11:50.700 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:50.730 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:50.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:50.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:11:50.836 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:50.863 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:50.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:50.867 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:11:50.968 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:50.986 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T11:11:51.000 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:51.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:51.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:11:51.104 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:51.127 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:51.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:51.133 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:11:51.234 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:51.276 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:51.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:51.278 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:11:51.379 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:51.408 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:51.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:51.413 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:11:51.514 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:51.553 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:51.554 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T11:11:51.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:51.556 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:11:51.657 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:51.658 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-10-01T11:11:51.658 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T11:11:51.681 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:51.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:51.683 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:11:51.784 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:51.824 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:51.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:51.828 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:11:51.929 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:51.953 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:51.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:51.955 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:11:52.005 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:51 smithi173 bash[58816]: cluster 2023-10-01T11:11:50.429116+0000 mgr.y (mgr.25072) 1637 : cluster 0 pgmap v1764: 89 pgs: 89 active+clean; 4.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 779 op/s 2023-10-01T11:11:52.056 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:52.065 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:51 smithi078 bash[75299]: cluster 2023-10-01T11:11:50.429116+0000 mgr.y (mgr.25072) 1637 : cluster 0 pgmap v1764: 89 pgs: 89 active+clean; 4.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 779 op/s 2023-10-01T11:11:52.065 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:51 smithi078 bash[78274]: cluster 2023-10-01T11:11:50.429116+0000 mgr.y (mgr.25072) 1637 : cluster 0 pgmap v1764: 89 pgs: 89 active+clean; 4.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 779 op/s 2023-10-01T11:11:52.090 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:52.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:52.094 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:11:52.195 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:52.223 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:52.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:52.228 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:11:52.328 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:52.351 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:52.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:52.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:11:52.456 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:52.502 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:52.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:52.507 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:11:52.608 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:52.636 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:52.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:52.642 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:11:52.714 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-01T11:11:52.743 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:52.768 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:52.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:52.771 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:11:52.872 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:52.904 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:52.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:52.909 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:11:53.010 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:53.040 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:53.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:53.044 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:11:53.078 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:11:53.145 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:53.182 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:53.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:53.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:11:53.288 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:53.310 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:53.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:53.313 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:11:53.414 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:53.436 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:53.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:53.439 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:11:53.540 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:53.575 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:53.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:53.578 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:11:53.678 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:53.701 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:53.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:53.706 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:11:53.807 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:53.843 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:53.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:53.847 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:11:53.947 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:53.985 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:53.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:53.988 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:11:54.089 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:54.098 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:53 smithi173 bash[58816]: cluster 2023-10-01T11:11:52.429605+0000 mgr.y (mgr.25072) 1638 : cluster 0 pgmap v1765: 89 pgs: 89 active+clean; 4.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 718 op/s 2023-10-01T11:11:54.112 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:54.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:54.117 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:11:54.217 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:54.245 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:54.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:54.247 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:11:54.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:53 smithi078 bash[75299]: cluster 2023-10-01T11:11:52.429605+0000 mgr.y (mgr.25072) 1638 : cluster 0 pgmap v1765: 89 pgs: 89 active+clean; 4.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 718 op/s 2023-10-01T11:11:54.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:53 smithi078 bash[78274]: cluster 2023-10-01T11:11:52.429605+0000 mgr.y (mgr.25072) 1638 : cluster 0 pgmap v1765: 89 pgs: 89 active+clean; 4.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 718 op/s 2023-10-01T11:11:54.348 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:54.373 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:54.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:54.378 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:11:54.479 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:54.507 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:54.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:54.510 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:11:54.611 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:54.638 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:54.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:54.643 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:11:54.743 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:54.766 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:54.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:54.769 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:11:54.870 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:54.904 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:54.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:54.910 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:11:55.011 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:55.041 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:55.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:55.044 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:11:55.121 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:11:55.145 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:55.173 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:55.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:55.176 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:11:55.277 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:55.297 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:55.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:55.298 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:11:55.399 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:55.442 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:55.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:55.450 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:11:55.550 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:55.592 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:55.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:55.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:11:55.696 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:55.720 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:55.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:55.726 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:11:55.827 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:55.853 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:55.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:55.861 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:11:55.962 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:55.993 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:56.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:56.002 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:11:56.102 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:56.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:55 smithi173 bash[58816]: cluster 2023-10-01T11:11:54.430260+0000 mgr.y (mgr.25072) 1639 : cluster 0 pgmap v1766: 89 pgs: 89 active+clean; 4.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 696 op/s 2023-10-01T11:11:56.137 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:56.138 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:11:56.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:56.143 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:11:56.161 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:55 smithi078 bash[75299]: cluster 2023-10-01T11:11:54.430260+0000 mgr.y (mgr.25072) 1639 : cluster 0 pgmap v1766: 89 pgs: 89 active+clean; 4.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 696 op/s 2023-10-01T11:11:56.162 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:55 smithi078 bash[78274]: cluster 2023-10-01T11:11:54.430260+0000 mgr.y (mgr.25072) 1639 : cluster 0 pgmap v1766: 89 pgs: 89 active+clean; 4.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 696 op/s 2023-10-01T11:11:56.244 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:56.276 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:56.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:56.279 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:11:56.379 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:56.413 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:56.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:56.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:11:56.515 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:56.555 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:56.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:56.560 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:11:56.660 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:56.688 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:56.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:56.690 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:11:56.791 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:56.841 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:56.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:56.843 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:11:56.944 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:56.975 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:56.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:56.979 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:11:57.079 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:57.122 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:57.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:57.127 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:11:57.228 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:57.268 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:11:57.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:57.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:11:57.372 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:57.390 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:57.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:57.392 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:11:57.493 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:57.523 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:57.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:57.528 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:11:57.541 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T11:11:57.629 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:57.653 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:57.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:57.655 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:11:57.756 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:57.786 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:57.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:57.789 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:11:57.890 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:57.930 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:57.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:57.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:11:58.036 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:58.044 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:57 smithi078 bash[78274]: cluster 2023-10-01T11:11:56.430844+0000 mgr.y (mgr.25072) 1640 : cluster 0 pgmap v1767: 89 pgs: 89 active+clean; 4.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2023-10-01T11:11:58.045 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:57 smithi078 bash[75299]: cluster 2023-10-01T11:11:56.430844+0000 mgr.y (mgr.25072) 1640 : cluster 0 pgmap v1767: 89 pgs: 89 active+clean; 4.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2023-10-01T11:11:58.070 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:58.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:58.073 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:11:58.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:57 smithi173 bash[58816]: cluster 2023-10-01T11:11:56.430844+0000 mgr.y (mgr.25072) 1640 : cluster 0 pgmap v1767: 89 pgs: 89 active+clean; 4.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2023-10-01T11:11:58.174 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:58.200 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:58.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:58.203 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:11:58.304 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:58.339 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:58.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:58.344 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:11:58.444 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:58.477 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:58.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:58.483 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:11:58.583 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:58.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:58.618 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:58.618 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:11:58.719 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:58.757 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:11:58.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:58.762 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:11:58.863 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:58.886 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:58.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:58.888 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:11:58.989 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:59.020 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:59.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:59.022 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:11:59.123 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:59.172 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:11:59.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:59.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:11:59.275 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:59.304 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:59.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:59.311 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:11:59.411 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:59.438 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:11:59.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:59.442 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:11:59.545 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:59.577 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:11:59.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:59.580 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:11:59.681 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:59.707 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:11:59.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:59.711 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:11:59.812 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:59.841 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:11:59.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:59.844 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:11:59.945 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:59.974 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:11:59.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:11:59.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:12:00.079 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:00.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:11:59 smithi078 bash[75299]: cluster 2023-10-01T11:11:58.431283+0000 mgr.y (mgr.25072) 1641 : cluster 0 pgmap v1768: 89 pgs: 89 active+clean; 4.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 604 op/s 2023-10-01T11:12:00.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:11:59 smithi078 bash[78274]: cluster 2023-10-01T11:11:58.431283+0000 mgr.y (mgr.25072) 1641 : cluster 0 pgmap v1768: 89 pgs: 89 active+clean; 4.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 604 op/s 2023-10-01T11:12:00.115 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:00.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:00.118 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:12:00.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:11:59 smithi173 bash[58816]: cluster 2023-10-01T11:11:58.431283+0000 mgr.y (mgr.25072) 1641 : cluster 0 pgmap v1768: 89 pgs: 89 active+clean; 4.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 604 op/s 2023-10-01T11:12:00.219 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:00.251 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:00.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:00.255 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:00.356 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:00.382 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:00.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:00.387 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:12:00.488 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:00.497 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:12:00 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:12:00] "GET /metrics HTTP/1.1" 200 36385 "" "Prometheus/2.33.4" 2023-10-01T11:12:00.514 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:00.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:00.519 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:12:00.620 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:00.658 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:00.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:00.664 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:12:00.765 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:00.791 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:00.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:00.795 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:12:00.896 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:00.929 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:00.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:00.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:12:01.036 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:01.062 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:01.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:01.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:12:01.167 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:01.198 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:01.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:01.200 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:12:01.301 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:01.332 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:01.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:01.335 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:12:01.436 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:01.458 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:01.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:01.464 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:12:01.565 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:01.589 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:01.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:01.596 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:01.696 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:01.715 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:01.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:01.717 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:12:01.818 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:01.838 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:01.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:01.840 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:12:01.941 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:01.964 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:01.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:01.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:02.068 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:02.106 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:02.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:02.111 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:12:02.192 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:01 smithi173 bash[58816]: cluster 2023-10-01T11:12:00.431998+0000 mgr.y (mgr.25072) 1642 : cluster 0 pgmap v1769: 89 pgs: 89 active+clean; 4.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 805 op/s 2023-10-01T11:12:02.211 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:02.236 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:02.237 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:01 smithi078 bash[78274]: cluster 2023-10-01T11:12:00.431998+0000 mgr.y (mgr.25072) 1642 : cluster 0 pgmap v1769: 89 pgs: 89 active+clean; 4.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 805 op/s 2023-10-01T11:12:02.237 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:01 smithi078 bash[75299]: cluster 2023-10-01T11:12:00.431998+0000 mgr.y (mgr.25072) 1642 : cluster 0 pgmap v1769: 89 pgs: 89 active+clean; 4.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 805 op/s 2023-10-01T11:12:02.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:02.239 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:12:02.340 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:02.377 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:02.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:02.379 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:12:02.407 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-10-01T11:12:02.407 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T11:12:02.480 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:02.513 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:02.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:02.519 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:12:02.619 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:02.639 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:02.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:02.642 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:12:02.742 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:02.759 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:02.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:02.761 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:12:02.862 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:02.902 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:02.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:02.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:12:03.007 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:03.041 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:03.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:03.046 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:12:03.147 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:03.156 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:02 smithi173 bash[58816]: audit 2023-10-01T11:12:02.474842+0000 mon.a (mon.0) 2114 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:12:03.172 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:02 smithi078 bash[75299]: audit 2023-10-01T11:12:02.474842+0000 mon.a (mon.0) 2114 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:12:03.172 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:02 smithi078 bash[78274]: audit 2023-10-01T11:12:02.474842+0000 mon.a (mon.0) 2114 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:12:03.177 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:03.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:03.183 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:12:03.283 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:03.306 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:03.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:03.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:12:03.415 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:03.438 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:03.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:03.441 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:03.455 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-01T11:12:03.542 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:03.573 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:03.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:03.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:12:03.677 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:03.720 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:03.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:03.723 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:12:03.823 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:03.859 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:03.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:03.865 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:12:03.966 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:03.975 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:03 smithi173 bash[58816]: cluster 2023-10-01T11:12:02.432496+0000 mgr.y (mgr.25072) 1643 : cluster 0 pgmap v1770: 89 pgs: 89 active+clean; 4.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 617 op/s 2023-10-01T11:12:03.990 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:03.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:03.999 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:12:04.099 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:04.124 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:04.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:04.131 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:04.230 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:03 smithi078 bash[78274]: cluster 2023-10-01T11:12:02.432496+0000 mgr.y (mgr.25072) 1643 : cluster 0 pgmap v1770: 89 pgs: 89 active+clean; 4.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 617 op/s 2023-10-01T11:12:04.231 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:03 smithi078 bash[75299]: cluster 2023-10-01T11:12:02.432496+0000 mgr.y (mgr.25072) 1643 : cluster 0 pgmap v1770: 89 pgs: 89 active+clean; 4.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 617 op/s 2023-10-01T11:12:04.232 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:04.262 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:04.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:04.267 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:12:04.367 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:04.399 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:04.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:04.403 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:12:04.504 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:04.521 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:04.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:04.523 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:12:04.624 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:04.649 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:04.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:04.655 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:12:04.756 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:04.802 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:04.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:04.808 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:12:04.908 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:04.943 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:04.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:04.947 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:12:05.048 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:05.082 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:05.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:05.084 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:05.185 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:05.212 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:05.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:05.218 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:12:05.319 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:05.351 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:05.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:05.354 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:12:05.455 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:05.476 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:05.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:05.478 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:12:05.579 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:05.607 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:05.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:05.611 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:12:05.711 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:05.739 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:05.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:05.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:05.844 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:05.872 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:05.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:05.876 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:12:05.977 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:06.011 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:06.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:06.014 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:12:06.115 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:06.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:05 smithi173 bash[58816]: cluster 2023-10-01T11:12:04.433097+0000 mgr.y (mgr.25072) 1644 : cluster 0 pgmap v1771: 89 pgs: 89 active+clean; 4.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 742 op/s 2023-10-01T11:12:06.145 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:06.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:06.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:12:06.248 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:06.276 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:06.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:06.278 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:12:06.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:05 smithi078 bash[78274]: cluster 2023-10-01T11:12:04.433097+0000 mgr.y (mgr.25072) 1644 : cluster 0 pgmap v1771: 89 pgs: 89 active+clean; 4.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 742 op/s 2023-10-01T11:12:06.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:05 smithi078 bash[75299]: cluster 2023-10-01T11:12:04.433097+0000 mgr.y (mgr.25072) 1644 : cluster 0 pgmap v1771: 89 pgs: 89 active+clean; 4.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 742 op/s 2023-10-01T11:12:06.379 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:06.405 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:06.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:06.408 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:12:06.509 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:06.531 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:06.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:06.533 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:12:06.633 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:06.666 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:06.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:06.671 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:12:06.772 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:06.801 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:06.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:06.805 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:12:06.905 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:06.962 INFO:teuthology.orchestra.run.smithi078.stderr:reweighted osd.3 to 1 (10000) 2023-10-01T11:12:06.968 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:06.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:06.973 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:12:07.073 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:07.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:06 smithi078 bash[78274]: audit 2023-10-01T11:12:06.571058+0000 mon.c (mon.2) 242 : audit 1 from='client.? 172.21.15.78:0/2261567805' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-01T11:12:07.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:06 smithi078 bash[78274]: audit 2023-10-01T11:12:06.571417+0000 mon.a (mon.0) 2115 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-01T11:12:07.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:06 smithi078 bash[75299]: audit 2023-10-01T11:12:06.571058+0000 mon.c (mon.2) 242 : audit 1 from='client.? 172.21.15.78:0/2261567805' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-01T11:12:07.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:06 smithi078 bash[75299]: audit 2023-10-01T11:12:06.571417+0000 mon.a (mon.0) 2115 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-01T11:12:07.104 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:07.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:07.107 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:12:07.207 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:07.215 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:06 smithi173 bash[58816]: audit 2023-10-01T11:12:06.571058+0000 mon.c (mon.2) 242 : audit 1 from='client.? 172.21.15.78:0/2261567805' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-01T11:12:07.215 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:06 smithi173 bash[58816]: audit 2023-10-01T11:12:06.571417+0000 mon.a (mon.0) 2115 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-01T11:12:07.233 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:07.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:07.246 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:07.347 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:07.389 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:07.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:07.395 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:12:07.495 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:07.526 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:07.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:07.529 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:12:07.630 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:07.660 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:07.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:07.664 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:12:07.764 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:07.787 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:07.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:07.789 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:12:07.890 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:07.917 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:07.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:07.919 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:12:08.020 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:08.069 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:08.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:08.074 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:12:08.175 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:08.185 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:07 smithi078 bash[75299]: cluster 2023-10-01T11:12:06.433622+0000 mgr.y (mgr.25072) 1645 : cluster 0 pgmap v1772: 89 pgs: 89 active+clean; 4.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 620 op/s 2023-10-01T11:12:08.185 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:07 smithi078 bash[75299]: cluster 2023-10-01T11:12:06.946137+0000 osd.5 (osd.5) 390 : cluster 3 failed to encode map e367 with expected crc 2023-10-01T11:12:08.185 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:07 smithi078 bash[75299]: audit 2023-10-01T11:12:06.948393+0000 mon.a (mon.0) 2116 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-01T11:12:08.185 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:07 smithi078 bash[75299]: cluster 2023-10-01T11:12:06.948486+0000 mon.a (mon.0) 2117 : cluster 0 osdmap e367: 8 total, 8 up, 8 in 2023-10-01T11:12:08.185 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:07 smithi078 bash[75299]: cluster 2023-10-01T11:12:06.949930+0000 osd.5 (osd.5) 391 : cluster 3 failed to encode map e367 with expected crc 2023-10-01T11:12:08.185 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:07 smithi078 bash[75299]: cluster 2023-10-01T11:12:06.950111+0000 osd.7 (osd.7) 412 : cluster 3 failed to encode map e367 with expected crc 2023-10-01T11:12:08.185 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:07 smithi078 bash[75299]: cluster 2023-10-01T11:12:06.954895+0000 osd.6 (osd.6) 353 : cluster 3 failed to encode map e367 with expected crc 2023-10-01T11:12:08.185 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:07 smithi078 bash[78274]: cluster 2023-10-01T11:12:06.433622+0000 mgr.y (mgr.25072) 1645 : cluster 0 pgmap v1772: 89 pgs: 89 active+clean; 4.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 620 op/s 2023-10-01T11:12:08.186 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:07 smithi078 bash[78274]: cluster 2023-10-01T11:12:06.946137+0000 osd.5 (osd.5) 390 : cluster 3 failed to encode map e367 with expected crc 2023-10-01T11:12:08.186 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:07 smithi078 bash[78274]: audit 2023-10-01T11:12:06.948393+0000 mon.a (mon.0) 2116 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-01T11:12:08.186 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:07 smithi078 bash[78274]: cluster 2023-10-01T11:12:06.948486+0000 mon.a (mon.0) 2117 : cluster 0 osdmap e367: 8 total, 8 up, 8 in 2023-10-01T11:12:08.186 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:07 smithi078 bash[78274]: cluster 2023-10-01T11:12:06.949930+0000 osd.5 (osd.5) 391 : cluster 3 failed to encode map e367 with expected crc 2023-10-01T11:12:08.186 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:07 smithi078 bash[78274]: cluster 2023-10-01T11:12:06.950111+0000 osd.7 (osd.7) 412 : cluster 3 failed to encode map e367 with expected crc 2023-10-01T11:12:08.186 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:07 smithi078 bash[78274]: cluster 2023-10-01T11:12:06.954895+0000 osd.6 (osd.6) 353 : cluster 3 failed to encode map e367 with expected crc 2023-10-01T11:12:08.207 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:08.213 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:07 smithi173 bash[58816]: cluster 2023-10-01T11:12:06.433622+0000 mgr.y (mgr.25072) 1645 : cluster 0 pgmap v1772: 89 pgs: 89 active+clean; 4.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 620 op/s 2023-10-01T11:12:08.213 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:07 smithi173 bash[58816]: cluster 2023-10-01T11:12:06.946137+0000 osd.5 (osd.5) 390 : cluster 3 failed to encode map e367 with expected crc 2023-10-01T11:12:08.213 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:07 smithi173 bash[58816]: audit 2023-10-01T11:12:06.948393+0000 mon.a (mon.0) 2116 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-01T11:12:08.213 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:07 smithi173 bash[58816]: cluster 2023-10-01T11:12:06.948486+0000 mon.a (mon.0) 2117 : cluster 0 osdmap e367: 8 total, 8 up, 8 in 2023-10-01T11:12:08.213 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:07 smithi173 bash[58816]: cluster 2023-10-01T11:12:06.949930+0000 osd.5 (osd.5) 391 : cluster 3 failed to encode map e367 with expected crc 2023-10-01T11:12:08.213 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:07 smithi173 bash[58816]: cluster 2023-10-01T11:12:06.950111+0000 osd.7 (osd.7) 412 : cluster 3 failed to encode map e367 with expected crc 2023-10-01T11:12:08.213 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:07 smithi173 bash[58816]: cluster 2023-10-01T11:12:06.954895+0000 osd.6 (osd.6) 353 : cluster 3 failed to encode map e367 with expected crc 2023-10-01T11:12:08.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:08.214 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:12:08.309 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T11:12:08.315 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:08.354 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:08.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:08.356 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:12:08.456 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:08.484 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:08.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:08.487 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:12:08.588 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:08.608 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:08.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:08.611 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:12:08.711 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:08.739 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:08.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:08.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:12:08.845 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:08.867 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:08.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:08.871 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:12:08.972 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:08.999 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:09.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:09.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:12:09.101 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:09.133 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:09.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:09.136 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:09.237 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:09.265 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:09.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:09.267 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:12:09.368 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:09.392 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:09.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:09.397 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:12:09.498 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:09.523 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:09.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:09.528 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:12:09.629 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:09.646 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:09.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:09.648 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:12:09.749 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:09.780 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:09.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:09.787 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:12:09.875 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-01T11:12:09.887 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:09.910 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:09.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:09.918 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:12:10.018 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:10.050 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:10.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:10.053 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:12:10.153 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:10.194 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:10.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:10.203 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:12:10.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:09 smithi078 bash[78274]: cluster 2023-10-01T11:12:08.433974+0000 mgr.y (mgr.25072) 1646 : cluster 0 pgmap v1774: 89 pgs: 89 active+clean; 4.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 640 op/s 2023-10-01T11:12:10.270 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:09 smithi078 bash[75299]: cluster 2023-10-01T11:12:08.433974+0000 mgr.y (mgr.25072) 1646 : cluster 0 pgmap v1774: 89 pgs: 89 active+clean; 4.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 640 op/s 2023-10-01T11:12:10.270 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:12:10 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:12:10] "GET /metrics HTTP/1.1" 200 36380 "" "Prometheus/2.33.4" 2023-10-01T11:12:10.299 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T11:12:10.304 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:10.313 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:09 smithi173 bash[58816]: cluster 2023-10-01T11:12:08.433974+0000 mgr.y (mgr.25072) 1646 : cluster 0 pgmap v1774: 89 pgs: 89 active+clean; 4.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 640 op/s 2023-10-01T11:12:10.329 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:10.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:10.333 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:12:10.351 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T11:12:10.415 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T11:12:10.434 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:10.465 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T11:12:10.480 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:10.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:10.484 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:12:10.504 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T11:12:10.546 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T11:12:10.584 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:10.586 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T11:12:10.617 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:10.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:10.619 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:12:10.622 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T11:12:10.637 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T11:12:10.637 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T11:12:10.638 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T11:12:10.638 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T11:12:10.638 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T11:12:10.638 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T11:12:10.638 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T11:12:10.638 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T11:12:10.720 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:10.764 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:10.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:10.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:12:10.872 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:10.905 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:10.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:10.907 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:12:10.964 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:10 smithi078 bash[75299]: audit 2023-10-01T11:12:10.260603+0000 mon.c (mon.2) 243 : audit 0 from='client.? 172.21.15.78:0/2404365489' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:12:10.964 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:10 smithi078 bash[75299]: audit 2023-10-01T11:12:10.691751+0000 mon.a (mon.0) 2118 : audit 1 from='client.? 172.21.15.78:0/1806995236' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T11:12:10.983 INFO:teuthology.orchestra.run.smithi078.stderr:noscrub is set 2023-10-01T11:12:11.008 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:11.042 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:11.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:11.048 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:12:11.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:10 smithi173 bash[58816]: audit 2023-10-01T11:12:10.260603+0000 mon.c (mon.2) 243 : audit 0 from='client.? 172.21.15.78:0/2404365489' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:12:11.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:10 smithi173 bash[58816]: audit 2023-10-01T11:12:10.691751+0000 mon.a (mon.0) 2118 : audit 1 from='client.? 172.21.15.78:0/1806995236' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T11:12:11.149 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:11.174 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:11.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:11.177 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:11.185 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:10 smithi078 bash[78274]: audit 2023-10-01T11:12:10.260603+0000 mon.c (mon.2) 243 : audit 0 from='client.? 172.21.15.78:0/2404365489' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:12:11.185 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:10 smithi078 bash[78274]: audit 2023-10-01T11:12:10.691751+0000 mon.a (mon.0) 2118 : audit 1 from='client.? 172.21.15.78:0/1806995236' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T11:12:11.278 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:11.318 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:11.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:11.323 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:12:11.424 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:11.455 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:11.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:11.461 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:12:11.562 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:11.584 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:11.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:11.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:12:11.688 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:11.713 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:11.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:11.718 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:12:11.819 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:11.855 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:11.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:11.861 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:12:11.962 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:11.997 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:12.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:12.002 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:12:12.103 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:12.134 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:12.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:12.139 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:12:12.233 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:11 smithi078 bash[75299]: cluster 2023-10-01T11:12:10.434658+0000 mgr.y (mgr.25072) 1647 : cluster 0 pgmap v1775: 89 pgs: 89 active+clean; 4.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 629 op/s 2023-10-01T11:12:12.233 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:11 smithi078 bash[75299]: cluster 2023-10-01T11:12:10.959282+0000 mon.a (mon.0) 2119 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:12:12.233 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:11 smithi078 bash[75299]: cluster 2023-10-01T11:12:10.972834+0000 osd.7 (osd.7) 413 : cluster 3 failed to encode map e368 with expected crc 2023-10-01T11:12:12.233 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:11 smithi078 bash[75299]: audit 2023-10-01T11:12:10.973653+0000 mon.a (mon.0) 2120 : audit 1 from='client.? 172.21.15.78:0/1806995236' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T11:12:12.233 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:11 smithi078 bash[75299]: cluster 2023-10-01T11:12:10.974230+0000 osd.7 (osd.7) 414 : cluster 3 failed to encode map e368 with expected crc 2023-10-01T11:12:12.233 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:11 smithi078 bash[75299]: cluster 2023-10-01T11:12:10.974429+0000 osd.5 (osd.5) 392 : cluster 3 failed to encode map e368 with expected crc 2023-10-01T11:12:12.233 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:11 smithi078 bash[75299]: cluster 2023-10-01T11:12:10.975444+0000 mon.a (mon.0) 2121 : cluster 0 osdmap e368: 8 total, 8 up, 8 in 2023-10-01T11:12:12.233 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:11 smithi078 bash[75299]: cluster 2023-10-01T11:12:10.978207+0000 osd.5 (osd.5) 393 : cluster 3 failed to encode map e368 with expected crc 2023-10-01T11:12:12.234 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:11 smithi078 bash[75299]: cluster 2023-10-01T11:12:10.978587+0000 osd.6 (osd.6) 354 : cluster 3 failed to encode map e368 with expected crc 2023-10-01T11:12:12.234 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:11 smithi078 bash[75299]: cluster 2023-10-01T11:12:10.980128+0000 osd.5 (osd.5) 394 : cluster 3 failed to encode map e368 with expected crc 2023-10-01T11:12:12.234 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:11 smithi078 bash[75299]: cluster 2023-10-01T11:12:10.981118+0000 osd.6 (osd.6) 355 : cluster 3 failed to encode map e368 with expected crc 2023-10-01T11:12:12.234 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:11 smithi078 bash[78274]: cluster 2023-10-01T11:12:10.434658+0000 mgr.y (mgr.25072) 1647 : cluster 0 pgmap v1775: 89 pgs: 89 active+clean; 4.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 629 op/s 2023-10-01T11:12:12.234 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:11 smithi078 bash[78274]: cluster 2023-10-01T11:12:10.959282+0000 mon.a (mon.0) 2119 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:12:12.234 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:11 smithi078 bash[78274]: cluster 2023-10-01T11:12:10.972834+0000 osd.7 (osd.7) 413 : cluster 3 failed to encode map e368 with expected crc 2023-10-01T11:12:12.234 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:11 smithi078 bash[78274]: audit 2023-10-01T11:12:10.973653+0000 mon.a (mon.0) 2120 : audit 1 from='client.? 172.21.15.78:0/1806995236' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T11:12:12.234 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:11 smithi078 bash[78274]: cluster 2023-10-01T11:12:10.974230+0000 osd.7 (osd.7) 414 : cluster 3 failed to encode map e368 with expected crc 2023-10-01T11:12:12.234 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:11 smithi078 bash[78274]: cluster 2023-10-01T11:12:10.974429+0000 osd.5 (osd.5) 392 : cluster 3 failed to encode map e368 with expected crc 2023-10-01T11:12:12.235 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:11 smithi078 bash[78274]: cluster 2023-10-01T11:12:10.975444+0000 mon.a (mon.0) 2121 : cluster 0 osdmap e368: 8 total, 8 up, 8 in 2023-10-01T11:12:12.235 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:11 smithi078 bash[78274]: cluster 2023-10-01T11:12:10.978207+0000 osd.5 (osd.5) 393 : cluster 3 failed to encode map e368 with expected crc 2023-10-01T11:12:12.235 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:11 smithi078 bash[78274]: cluster 2023-10-01T11:12:10.978587+0000 osd.6 (osd.6) 354 : cluster 3 failed to encode map e368 with expected crc 2023-10-01T11:12:12.235 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:11 smithi078 bash[78274]: cluster 2023-10-01T11:12:10.980128+0000 osd.5 (osd.5) 394 : cluster 3 failed to encode map e368 with expected crc 2023-10-01T11:12:12.235 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:11 smithi078 bash[78274]: cluster 2023-10-01T11:12:10.981118+0000 osd.6 (osd.6) 355 : cluster 3 failed to encode map e368 with expected crc 2023-10-01T11:12:12.239 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:12.248 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:11 smithi173 bash[58816]: cluster 2023-10-01T11:12:10.434658+0000 mgr.y (mgr.25072) 1647 : cluster 0 pgmap v1775: 89 pgs: 89 active+clean; 4.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 629 op/s 2023-10-01T11:12:12.248 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:11 smithi173 bash[58816]: cluster 2023-10-01T11:12:10.959282+0000 mon.a (mon.0) 2119 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:12:12.248 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:11 smithi173 bash[58816]: cluster 2023-10-01T11:12:10.972834+0000 osd.7 (osd.7) 413 : cluster 3 failed to encode map e368 with expected crc 2023-10-01T11:12:12.248 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:11 smithi173 bash[58816]: audit 2023-10-01T11:12:10.973653+0000 mon.a (mon.0) 2120 : audit 1 from='client.? 172.21.15.78:0/1806995236' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T11:12:12.248 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:11 smithi173 bash[58816]: cluster 2023-10-01T11:12:10.974230+0000 osd.7 (osd.7) 414 : cluster 3 failed to encode map e368 with expected crc 2023-10-01T11:12:12.249 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:11 smithi173 bash[58816]: cluster 2023-10-01T11:12:10.974429+0000 osd.5 (osd.5) 392 : cluster 3 failed to encode map e368 with expected crc 2023-10-01T11:12:12.249 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:11 smithi173 bash[58816]: cluster 2023-10-01T11:12:10.975444+0000 mon.a (mon.0) 2121 : cluster 0 osdmap e368: 8 total, 8 up, 8 in 2023-10-01T11:12:12.249 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:11 smithi173 bash[58816]: cluster 2023-10-01T11:12:10.978207+0000 osd.5 (osd.5) 393 : cluster 3 failed to encode map e368 with expected crc 2023-10-01T11:12:12.249 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:11 smithi173 bash[58816]: cluster 2023-10-01T11:12:10.978587+0000 osd.6 (osd.6) 354 : cluster 3 failed to encode map e368 with expected crc 2023-10-01T11:12:12.249 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:11 smithi173 bash[58816]: cluster 2023-10-01T11:12:10.980128+0000 osd.5 (osd.5) 394 : cluster 3 failed to encode map e368 with expected crc 2023-10-01T11:12:12.249 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:11 smithi173 bash[58816]: cluster 2023-10-01T11:12:10.981118+0000 osd.6 (osd.6) 355 : cluster 3 failed to encode map e368 with expected crc 2023-10-01T11:12:12.261 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:12.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:12.268 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:12:12.369 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:12.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:12.396 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:12.396 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:12:12.497 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:12.518 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:12.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:12.520 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:12:12.621 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:12.649 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:12.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:12.655 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:12:12.756 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:12.778 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:12.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:12.781 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:12.882 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:12.912 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:12.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:12.915 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:12:13.015 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:13.046 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:13.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:13.052 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:13.153 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:13.176 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:13.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:13.179 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:12:13.280 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:13.304 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:13.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:13.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:13.412 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:13.437 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:13.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:13.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:12:13.542 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-10-01T11:12:13.543 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [ 2023-10-01T11:12:13.543 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:12:13.543 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.36819.0:146581 14.d 14:b74b9470:::benchmark_data_smithi078_236918_object41435:head [delete] snapc 0=[] ondisk+write+known_if_redirected e368)", 2023-10-01T11:12:13.543 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:12:13.537464+0000", 2023-10-01T11:12:13.543 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 0.0048472009999999998, 2023-10-01T11:12:13.543 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.004877592, 2023-10-01T11:12:13.543 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:12:13.543 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "waiting for sub ops", 2023-10-01T11:12:13.543 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:12:13.543 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.36819", 2023-10-01T11:12:13.544 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/1409915880", 2023-10-01T11:12:13.544 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 146581 2023-10-01T11:12:13.544 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:12:13.544 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:12:13.544 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:12:13.544 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:12:13.544 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:12:13.537464+0000", 2023-10-01T11:12:13.544 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:12:13.544 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:12:13.544 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:12:13.544 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:12:13.544 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:12:13.537464+0000", 2023-10-01T11:12:13.545 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-10-01T11:12:13.545 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:12:13.545 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:12:13.545 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:12:13.545 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:12:13.537463+0000", 2023-10-01T11:12:13.545 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.8019999999999992e-06 2023-10-01T11:12:13.545 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:12:13.545 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:12:13.545 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:12:13.545 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:12:13.537470+0000", 2023-10-01T11:12:13.545 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1787000000000001e-05 2023-10-01T11:12:13.546 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:12:13.546 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:12:13.546 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:12:13.546 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:12:13.537482+0000", 2023-10-01T11:12:13.546 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.9019999999999998e-06 2023-10-01T11:12:13.546 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:12:13.546 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:12:13.546 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:12:13.546 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:12:13.537487+0000", 2023-10-01T11:12:13.546 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.8862e-05 2023-10-01T11:12:13.546 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:12:13.546 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:12:13.547 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:12:13.547 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:12:13.537546+0000", 2023-10-01T11:12:13.547 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00016100500000000001 2023-10-01T11:12:13.547 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:12:13.547 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:12:13.547 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:12:13.547 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:12:13.537707+0000", 2023-10-01T11:12:13.547 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000170001 2023-10-01T11:12:13.547 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:12:13.547 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:12:13.547 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 4,5", 2023-10-01T11:12:13.547 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:12:13.537877+0000", 2023-10-01T11:12:13.548 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00041305699999999999 2023-10-01T11:12:13.548 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:12:13.548 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:12:13.548 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:12:13.548 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:12:13.548 INFO:teuthology.orchestra.run.smithi173.stdout: ], 2023-10-01T11:12:13.548 INFO:teuthology.orchestra.run.smithi173.stdout: "num_ops": 1 2023-10-01T11:12:13.548 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-10-01T11:12:13.548 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:13.586 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:13.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:13.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:13.690 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:13.714 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:13.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:13.720 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:13.820 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:13.853 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:13.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:13.856 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:12:13.956 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:13.988 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:13.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:13.993 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:13 smithi173 bash[58816]: cluster 2023-10-01T11:12:12.436130+0000 mgr.y (mgr.25072) 1648 : cluster 0 pgmap v1777: 89 pgs: 89 active+clean; 4.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 511 op/s 2023-10-01T11:12:13.993 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:12:14.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:13 smithi078 bash[75299]: cluster 2023-10-01T11:12:12.436130+0000 mgr.y (mgr.25072) 1648 : cluster 0 pgmap v1777: 89 pgs: 89 active+clean; 4.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 511 op/s 2023-10-01T11:12:14.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:13 smithi078 bash[78274]: cluster 2023-10-01T11:12:12.436130+0000 mgr.y (mgr.25072) 1648 : cluster 0 pgmap v1777: 89 pgs: 89 active+clean; 4.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 511 op/s 2023-10-01T11:12:14.094 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:14.121 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:14.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:14.123 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:12:14.224 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:14.256 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:14.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:14.262 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:12:14.358 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T11:12:14.363 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:14.448 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:14.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:14.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:12:14.554 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:14.602 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:14.603 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-01T11:12:14.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:14.605 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:12:14.706 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:14.733 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:14.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:14.735 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:12:14.836 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:14.864 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:14.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:14.868 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:12:14.968 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:15.019 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:15.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:15.025 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:12:15.126 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:15.156 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:15.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:15.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:12:15.262 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:15.302 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:15.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:15.307 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:12:15.393 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T11:12:15.408 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:15.435 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:15.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:15.438 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:12:15.541 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:15.576 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:15.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:15.579 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:15.679 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:15.715 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:15.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:15.722 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:12:15.822 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:15.862 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:15.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:15.864 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:12:15.965 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:15.998 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:16.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:16.006 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:12:16.106 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:16.115 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:15 smithi173 bash[58816]: cluster 2023-10-01T11:12:14.438721+0000 mgr.y (mgr.25072) 1649 : cluster 0 pgmap v1778: 89 pgs: 89 active+clean; 3.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2023-10-01T11:12:16.130 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:16.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:16.138 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:12:16.238 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:16.247 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:15 smithi078 bash[75299]: cluster 2023-10-01T11:12:14.438721+0000 mgr.y (mgr.25072) 1649 : cluster 0 pgmap v1778: 89 pgs: 89 active+clean; 3.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2023-10-01T11:12:16.248 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:15 smithi078 bash[78274]: cluster 2023-10-01T11:12:14.438721+0000 mgr.y (mgr.25072) 1649 : cluster 0 pgmap v1778: 89 pgs: 89 active+clean; 3.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2023-10-01T11:12:16.272 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:16.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:16.276 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:12:16.377 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:16.417 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:16.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:16.421 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:12:16.522 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:16.549 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:16.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:16.553 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:16.654 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:16.681 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:16.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:16.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:12:16.787 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:16.820 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:16.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:16.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:12:16.925 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:16.928 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:12:16.952 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:16.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:16.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:12:17.059 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:17.085 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:17.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:17.091 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:12:17.192 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:17.215 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:17.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:17.218 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:12:17.318 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:17.344 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:17.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:17.348 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:12:17.448 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:17.478 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:17.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:17.481 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:12:17.581 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:17.612 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:17.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:17.615 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:12:17.715 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:17.741 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:17.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:17.743 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:12:17.844 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:17.886 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:17.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:17.891 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:12:17.991 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:18.017 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:18.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:18.026 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:12:18.050 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:18 smithi078 bash[75299]: cluster 2023-10-01T11:12:16.439278+0000 mgr.y (mgr.25072) 1650 : cluster 0 pgmap v1779: 89 pgs: 89 active+clean; 3.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 678 op/s 2023-10-01T11:12:18.050 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:18 smithi078 bash[75299]: audit 2023-10-01T11:12:17.485745+0000 mon.a (mon.0) 2122 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:12:18.050 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:18 smithi078 bash[75299]: audit 2023-10-01T11:12:17.487599+0000 mon.a (mon.0) 2123 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:12:18.127 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:18.151 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:18.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:18.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:18.255 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:18.287 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:18.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:18.294 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:12:18.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:18 smithi078 bash[78274]: cluster 2023-10-01T11:12:16.439278+0000 mgr.y (mgr.25072) 1650 : cluster 0 pgmap v1779: 89 pgs: 89 active+clean; 3.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 678 op/s 2023-10-01T11:12:18.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:18 smithi078 bash[78274]: audit 2023-10-01T11:12:17.485745+0000 mon.a (mon.0) 2122 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:12:18.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:18 smithi078 bash[78274]: audit 2023-10-01T11:12:17.487599+0000 mon.a (mon.0) 2123 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:12:18.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:17 smithi173 bash[58816]: cluster 2023-10-01T11:12:16.439278+0000 mgr.y (mgr.25072) 1650 : cluster 0 pgmap v1779: 89 pgs: 89 active+clean; 3.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 678 op/s 2023-10-01T11:12:18.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:17 smithi173 bash[58816]: audit 2023-10-01T11:12:17.485745+0000 mon.a (mon.0) 2122 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:12:18.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:17 smithi173 bash[58816]: audit 2023-10-01T11:12:17.487599+0000 mon.a (mon.0) 2123 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:12:18.395 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:18.428 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:18.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:18.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:12:18.533 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:18.569 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:18.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:18.572 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:12:18.672 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:18.703 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:18.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:18.709 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:18.810 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:18.846 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:18.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:18.851 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:12:18.952 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:18.969 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:12:18.995 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:18.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:18.999 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:12:19.100 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:19.134 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:19.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:19.136 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:12:19.237 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:19.278 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:19.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:19.282 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:12:19.382 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:19.404 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:19.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:19.407 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:19.415 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T11:12:19.507 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:19.526 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:19.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:19.528 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:12:19.629 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:19.661 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:19.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:19.663 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:12:19.764 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:19.797 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:19.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:19.801 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:12:19.901 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:19.924 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:19.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:19.926 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:12:19.981 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:12:20.027 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:20.054 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:20.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:20.060 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:12:20.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:19 smithi173 bash[58816]: cluster 2023-10-01T11:12:18.439811+0000 mgr.y (mgr.25072) 1651 : cluster 0 pgmap v1780: 89 pgs: 89 active+clean; 3.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 644 op/s 2023-10-01T11:12:20.160 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:20.184 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:20.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:20.191 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:12:20.269 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:19 smithi078 bash[75299]: cluster 2023-10-01T11:12:18.439811+0000 mgr.y (mgr.25072) 1651 : cluster 0 pgmap v1780: 89 pgs: 89 active+clean; 3.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 644 op/s 2023-10-01T11:12:20.270 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:12:20 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:12:20] "GET /metrics HTTP/1.1" 200 36383 "" "Prometheus/2.33.4" 2023-10-01T11:12:20.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:19 smithi078 bash[78274]: cluster 2023-10-01T11:12:18.439811+0000 mgr.y (mgr.25072) 1651 : cluster 0 pgmap v1780: 89 pgs: 89 active+clean; 3.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 644 op/s 2023-10-01T11:12:20.291 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:20.328 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:20.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:20.332 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:12:20.433 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:20.473 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:20.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:20.475 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:12:20.576 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:20.600 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:20.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:20.602 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:12:20.703 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:20.729 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:20.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:20.731 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:12:20.831 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:20.855 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:20.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:20.857 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:20.958 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:20.991 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:20.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:20.995 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:12:21.096 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:21.119 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:21.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:21.123 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:12:21.224 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:21.244 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:21.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:21.248 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:12:21.348 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:21.386 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:21.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:21.390 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:12:21.491 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:21.519 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:21.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:21.522 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:12:21.623 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:21.653 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:21.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:21.658 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:12:21.759 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:21.786 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:21.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:21.794 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:12:21.895 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:21.926 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:21.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:21.931 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:12:22.032 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:22.070 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:22.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:22.077 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:12:22.178 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:22.186 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:21 smithi078 bash[78274]: cluster 2023-10-01T11:12:20.440508+0000 mgr.y (mgr.25072) 1652 : cluster 0 pgmap v1781: 89 pgs: 89 active+clean; 3.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 652 op/s 2023-10-01T11:12:22.187 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:21 smithi078 bash[75299]: cluster 2023-10-01T11:12:20.440508+0000 mgr.y (mgr.25072) 1652 : cluster 0 pgmap v1781: 89 pgs: 89 active+clean; 3.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 652 op/s 2023-10-01T11:12:22.206 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:22.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:22.209 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:12:22.310 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:22.337 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:22.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:22.345 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:12:22.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:21 smithi173 bash[58816]: cluster 2023-10-01T11:12:20.440508+0000 mgr.y (mgr.25072) 1652 : cluster 0 pgmap v1781: 89 pgs: 89 active+clean; 3.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 652 op/s 2023-10-01T11:12:22.445 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:22.482 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:22.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:22.490 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:12:22.591 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:22.621 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:22.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:22.624 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:12:22.725 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:22.757 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:22.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:22.759 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:12:22.860 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:22.886 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:22.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:22.889 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:12:22.990 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:23.011 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:23.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:23.014 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:12:23.115 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:23.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:23.173 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:23.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:12:23.275 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:23.305 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:23.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:23.312 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:12:23.413 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:23.530 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:23.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:23.533 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:12:23.634 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:23.660 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:23.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:23.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:12:23.767 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:23.802 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:23.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:23.804 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:12:23.905 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:23.930 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:23.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:23.937 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:12:24.038 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:24.047 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:23 smithi078 bash[75299]: cluster 2023-10-01T11:12:22.440998+0000 mgr.y (mgr.25072) 1653 : cluster 0 pgmap v1782: 89 pgs: 89 active+clean; 3.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 622 op/s 2023-10-01T11:12:24.048 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:23 smithi078 bash[78274]: cluster 2023-10-01T11:12:22.440998+0000 mgr.y (mgr.25072) 1653 : cluster 0 pgmap v1782: 89 pgs: 89 active+clean; 3.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 622 op/s 2023-10-01T11:12:24.066 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:24.067 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-10-01T11:12:24.067 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [], 2023-10-01T11:12:24.067 INFO:teuthology.orchestra.run.smithi173.stdout: "complaint_time": 30, 2023-10-01T11:12:24.068 INFO:teuthology.orchestra.run.smithi173.stdout: "num_blocked_ops": 0 2023-10-01T11:12:24.068 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-10-01T11:12:24.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:24.073 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:12:24.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:23 smithi173 bash[58816]: cluster 2023-10-01T11:12:22.440998+0000 mgr.y (mgr.25072) 1653 : cluster 0 pgmap v1782: 89 pgs: 89 active+clean; 3.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 622 op/s 2023-10-01T11:12:24.174 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:24.198 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:24.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:24.201 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:12:24.301 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:24.339 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:24.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:24.343 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:12:24.443 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:24.467 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:24.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:24.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:12:24.576 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:24.610 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:24.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:24.617 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:12:24.718 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:24.744 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:24.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:24.749 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:12:24.850 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:24.874 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:24.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:24.879 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:12:24.980 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:25.001 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:25.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:25.006 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:12:25.064 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-01T11:12:25.107 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:25.136 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:25.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:25.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:12:25.240 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:25.272 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:25.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:25.275 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:12:25.376 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:25.406 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:25.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:25.409 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:12:25.510 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:25.548 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:25.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:25.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:12:25.656 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:25.686 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:25.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:25.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:12:25.790 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:25.836 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:25.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:25.841 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:12:25.942 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:25.965 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:25.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:25.969 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:12:26.002 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:26 smithi173 bash[58816]: cluster 2023-10-01T11:12:24.441589+0000 mgr.y (mgr.25072) 1654 : cluster 0 pgmap v1783: 89 pgs: 89 active+clean; 3.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 732 op/s 2023-10-01T11:12:26.070 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:26.098 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:26.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:26.103 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:12:26.204 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:26.212 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:26 smithi078 bash[75299]: cluster 2023-10-01T11:12:24.441589+0000 mgr.y (mgr.25072) 1654 : cluster 0 pgmap v1783: 89 pgs: 89 active+clean; 3.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 732 op/s 2023-10-01T11:12:26.212 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:26 smithi078 bash[78274]: cluster 2023-10-01T11:12:24.441589+0000 mgr.y (mgr.25072) 1654 : cluster 0 pgmap v1783: 89 pgs: 89 active+clean; 3.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 732 op/s 2023-10-01T11:12:26.231 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:26.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:26.233 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:12:26.334 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:26.373 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:26.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:26.375 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:12:26.476 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:26.499 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:26.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:26.502 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:26.603 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:26.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:26.634 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:26.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:12:26.735 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:26.766 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:26.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:26.768 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:12:26.869 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:26.900 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:26.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:26.903 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:12:27.004 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:27.026 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:27.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:27.029 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:27.129 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:27.159 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:27.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:27.164 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:12:27.264 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:27.290 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:27.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:27.299 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:12:27.400 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:27.426 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:27.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:27.428 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:12:27.529 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:27.555 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:27.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:27.557 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:12:27.658 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:27.703 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:27.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:27.709 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:12:27.810 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:27.894 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:27.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:27.899 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:12:28.000 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:28.050 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:28.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:28.055 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:12:28.155 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:28.164 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:28 smithi078 bash[75299]: cluster 2023-10-01T11:12:26.442168+0000 mgr.y (mgr.25072) 1655 : cluster 0 pgmap v1784: 89 pgs: 89 active+clean; 3.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 633 op/s 2023-10-01T11:12:28.164 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:28 smithi078 bash[78274]: cluster 2023-10-01T11:12:26.442168+0000 mgr.y (mgr.25072) 1655 : cluster 0 pgmap v1784: 89 pgs: 89 active+clean; 3.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 633 op/s 2023-10-01T11:12:28.194 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:28.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:28.200 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:12:28.300 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:28.310 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:28 smithi173 bash[58816]: cluster 2023-10-01T11:12:26.442168+0000 mgr.y (mgr.25072) 1655 : cluster 0 pgmap v1784: 89 pgs: 89 active+clean; 3.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 633 op/s 2023-10-01T11:12:28.325 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:28.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:28.331 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:12:28.432 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:28.462 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:28.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:28.465 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:12:28.565 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:28.595 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:28.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:28.602 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:12:28.703 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:28.735 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:28.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:28.738 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:12:28.839 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:28.864 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:28.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:28.869 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:12:28.970 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:28.993 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:28.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:28.999 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:12:29.100 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:29.126 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:29.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:29.136 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:29.237 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:29.266 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:29.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:29.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:12:29.371 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:29.414 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:29.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:29.419 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:12:29.519 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:29.545 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:29.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:29.548 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:12:29.649 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:29.674 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:29.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:29.676 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:29.777 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:29.808 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:29.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:29.815 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:12:29.869 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T11:12:29.916 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:29.951 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:29.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:29.954 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:12:30.055 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:30.080 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:30.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:30.083 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:12:30.183 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:30.192 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:30 smithi173 bash[58816]: cluster 2023-10-01T11:12:28.445565+0000 mgr.y (mgr.25072) 1656 : cluster 0 pgmap v1785: 89 pgs: 89 active+clean; 3.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 545 op/s 2023-10-01T11:12:30.207 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:30.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:30.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:12:30.269 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:30 smithi078 bash[78274]: cluster 2023-10-01T11:12:28.445565+0000 mgr.y (mgr.25072) 1656 : cluster 0 pgmap v1785: 89 pgs: 89 active+clean; 3.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 545 op/s 2023-10-01T11:12:30.270 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:30 smithi078 bash[75299]: cluster 2023-10-01T11:12:28.445565+0000 mgr.y (mgr.25072) 1656 : cluster 0 pgmap v1785: 89 pgs: 89 active+clean; 3.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 545 op/s 2023-10-01T11:12:30.270 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:12:30 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:12:30] "GET /metrics HTTP/1.1" 200 36383 "" "Prometheus/2.33.4" 2023-10-01T11:12:30.309 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:30.340 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:30.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:30.343 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:12:30.443 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:30.475 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:30.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:30.479 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:12:30.579 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:30.622 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:30.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:30.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:12:30.728 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:30.755 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:30.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:30.762 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:30.863 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:30.886 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:30.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:30.889 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:12:30.989 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:31.017 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:31.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:31.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:12:31.037 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:31 smithi078 bash[75299]: audit 2023-10-01T11:12:30.226402+0000 mon.c (mon.2) 244 : audit 1 from='client.? 172.21.15.78:0/349480624' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-01T11:12:31.037 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:31 smithi078 bash[75299]: audit 2023-10-01T11:12:30.226918+0000 mon.a (mon.0) 2124 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-01T11:12:31.048 INFO:teuthology.orchestra.run.smithi078.stderr:reweighted osd.4 to 1 (10000) 2023-10-01T11:12:31.125 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:31.160 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:31.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:31.168 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:12:31.268 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:31.309 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:31.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:31.314 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:12:31.314 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:31 smithi078 bash[78274]: audit 2023-10-01T11:12:30.226402+0000 mon.c (mon.2) 244 : audit 1 from='client.? 172.21.15.78:0/349480624' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-01T11:12:31.314 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:31 smithi078 bash[78274]: audit 2023-10-01T11:12:30.226918+0000 mon.a (mon.0) 2124 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-01T11:12:31.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:31 smithi173 bash[58816]: audit 2023-10-01T11:12:30.226402+0000 mon.c (mon.2) 244 : audit 1 from='client.? 172.21.15.78:0/349480624' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-01T11:12:31.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:31 smithi173 bash[58816]: audit 2023-10-01T11:12:30.226918+0000 mon.a (mon.0) 2124 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-01T11:12:31.415 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:31.444 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:31.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:31.447 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:12:31.548 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:31.569 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:31.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:31.572 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:12:31.672 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:31.715 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:31.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:31.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:12:31.818 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:31.865 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:31.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:31.874 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:12:31.975 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:32.005 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:32.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:32.007 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:32.108 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:32.138 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:32.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:32.146 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:12:32.247 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:32.254 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:32 smithi173 bash[58816]: cluster 2023-10-01T11:12:30.446322+0000 mgr.y (mgr.25072) 1657 : cluster 0 pgmap v1786: 89 pgs: 89 active+clean; 3.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 768 op/s 2023-10-01T11:12:32.255 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:32 smithi173 bash[58816]: cluster 2023-10-01T11:12:31.036103+0000 osd.6 (osd.6) 356 : cluster 3 failed to encode map e369 with expected crc 2023-10-01T11:12:32.255 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:32 smithi173 bash[58816]: cluster 2023-10-01T11:12:31.036183+0000 osd.7 (osd.7) 415 : cluster 3 failed to encode map e369 with expected crc 2023-10-01T11:12:32.255 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:32 smithi173 bash[58816]: audit 2023-10-01T11:12:31.042596+0000 mon.a (mon.0) 2125 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-01T11:12:32.255 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:32 smithi173 bash[58816]: cluster 2023-10-01T11:12:31.042805+0000 mon.a (mon.0) 2126 : cluster 0 osdmap e369: 8 total, 8 up, 8 in 2023-10-01T11:12:32.255 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:32 smithi173 bash[58816]: cluster 2023-10-01T11:12:31.063602+0000 osd.5 (osd.5) 395 : cluster 3 failed to encode map e369 with expected crc 2023-10-01T11:12:32.255 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:32 smithi173 bash[58816]: audit 2023-10-01T11:12:31.381915+0000 mon.a (mon.0) 2127 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T11:12:32.270 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:32.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:32.275 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:12:32.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:32 smithi078 bash[75299]: cluster 2023-10-01T11:12:30.446322+0000 mgr.y (mgr.25072) 1657 : cluster 0 pgmap v1786: 89 pgs: 89 active+clean; 3.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 768 op/s 2023-10-01T11:12:32.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:32 smithi078 bash[75299]: cluster 2023-10-01T11:12:31.036103+0000 osd.6 (osd.6) 356 : cluster 3 failed to encode map e369 with expected crc 2023-10-01T11:12:32.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:32 smithi078 bash[75299]: cluster 2023-10-01T11:12:31.036183+0000 osd.7 (osd.7) 415 : cluster 3 failed to encode map e369 with expected crc 2023-10-01T11:12:32.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:32 smithi078 bash[75299]: audit 2023-10-01T11:12:31.042596+0000 mon.a (mon.0) 2125 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-01T11:12:32.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:32 smithi078 bash[75299]: cluster 2023-10-01T11:12:31.042805+0000 mon.a (mon.0) 2126 : cluster 0 osdmap e369: 8 total, 8 up, 8 in 2023-10-01T11:12:32.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:32 smithi078 bash[75299]: cluster 2023-10-01T11:12:31.063602+0000 osd.5 (osd.5) 395 : cluster 3 failed to encode map e369 with expected crc 2023-10-01T11:12:32.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:32 smithi078 bash[75299]: audit 2023-10-01T11:12:31.381915+0000 mon.a (mon.0) 2127 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T11:12:32.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:32 smithi078 bash[78274]: cluster 2023-10-01T11:12:30.446322+0000 mgr.y (mgr.25072) 1657 : cluster 0 pgmap v1786: 89 pgs: 89 active+clean; 3.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 768 op/s 2023-10-01T11:12:32.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:32 smithi078 bash[78274]: cluster 2023-10-01T11:12:31.036103+0000 osd.6 (osd.6) 356 : cluster 3 failed to encode map e369 with expected crc 2023-10-01T11:12:32.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:32 smithi078 bash[78274]: cluster 2023-10-01T11:12:31.036183+0000 osd.7 (osd.7) 415 : cluster 3 failed to encode map e369 with expected crc 2023-10-01T11:12:32.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:32 smithi078 bash[78274]: audit 2023-10-01T11:12:31.042596+0000 mon.a (mon.0) 2125 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-01T11:12:32.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:32 smithi078 bash[78274]: cluster 2023-10-01T11:12:31.042805+0000 mon.a (mon.0) 2126 : cluster 0 osdmap e369: 8 total, 8 up, 8 in 2023-10-01T11:12:32.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:32 smithi078 bash[78274]: cluster 2023-10-01T11:12:31.063602+0000 osd.5 (osd.5) 395 : cluster 3 failed to encode map e369 with expected crc 2023-10-01T11:12:32.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:32 smithi078 bash[78274]: audit 2023-10-01T11:12:31.381915+0000 mon.a (mon.0) 2127 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T11:12:32.375 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:32.400 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:32.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:32.407 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:12:32.508 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:32.529 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:32.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:32.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:32.631 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:32.669 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:32.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:32.670 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:12:32.771 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:32.799 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:32.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:32.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:12:32.905 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:32.932 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:32.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:32.938 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:33.039 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:33.045 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:33 smithi078 bash[75299]: audit 2023-10-01T11:12:32.475185+0000 mon.a (mon.0) 2128 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:12:33.074 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:33.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:33.082 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:12:33.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:33 smithi173 bash[58816]: audit 2023-10-01T11:12:32.475185+0000 mon.a (mon.0) 2128 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:12:33.182 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:33.211 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:33.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:33.213 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:33.247 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-01T11:12:33.298 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:33 smithi078 bash[78274]: audit 2023-10-01T11:12:32.475185+0000 mon.a (mon.0) 2128 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:12:33.314 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:33.338 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:33.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:33.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:12:33.444 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:33.493 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:33.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:33.500 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:12:33.601 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:33.607 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T11:12:33.624 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:33.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:33.626 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:12:33.661 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T11:12:33.709 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T11:12:33.727 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:33.747 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:33.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:33.749 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:33.754 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T11:12:33.803 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T11:12:33.850 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:33.851 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T11:12:33.881 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:33.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:33.884 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:33.891 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T11:12:33.926 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T11:12:33.933 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T11:12:33.933 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T11:12:33.933 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T11:12:33.933 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T11:12:33.933 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T11:12:33.933 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T11:12:33.933 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T11:12:33.934 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T11:12:33.985 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:34.007 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:34.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:34.011 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:12:34.072 INFO:teuthology.orchestra.run.smithi078.stderr:nodeep-scrub is set 2023-10-01T11:12:34.112 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:34.135 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:34.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:34.138 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:12:34.239 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:34.246 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:34 smithi078 bash[75299]: cluster 2023-10-01T11:12:32.446822+0000 mgr.y (mgr.25072) 1658 : cluster 0 pgmap v1788: 89 pgs: 89 active+clean; 3.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 658 op/s 2023-10-01T11:12:34.246 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:34 smithi078 bash[75299]: audit 2023-10-01T11:12:33.562749+0000 mon.b (mon.1) 60 : audit 0 from='client.? 172.21.15.78:0/936264664' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:12:34.246 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:34 smithi078 bash[75299]: audit 2023-10-01T11:12:33.971126+0000 mon.a (mon.0) 2129 : audit 1 from='client.? 172.21.15.78:0/4222260494' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:12:34.246 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:34 smithi078 bash[78274]: cluster 2023-10-01T11:12:32.446822+0000 mgr.y (mgr.25072) 1658 : cluster 0 pgmap v1788: 89 pgs: 89 active+clean; 3.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 658 op/s 2023-10-01T11:12:34.246 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:34 smithi078 bash[78274]: audit 2023-10-01T11:12:33.562749+0000 mon.b (mon.1) 60 : audit 0 from='client.? 172.21.15.78:0/936264664' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:12:34.247 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:34 smithi078 bash[78274]: audit 2023-10-01T11:12:33.971126+0000 mon.a (mon.0) 2129 : audit 1 from='client.? 172.21.15.78:0/4222260494' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:12:34.264 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:34.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:34.272 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:12:34.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:34 smithi173 bash[58816]: cluster 2023-10-01T11:12:32.446822+0000 mgr.y (mgr.25072) 1658 : cluster 0 pgmap v1788: 89 pgs: 89 active+clean; 3.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 658 op/s 2023-10-01T11:12:34.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:34 smithi173 bash[58816]: audit 2023-10-01T11:12:33.562749+0000 mon.b (mon.1) 60 : audit 0 from='client.? 172.21.15.78:0/936264664' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:12:34.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:34 smithi173 bash[58816]: audit 2023-10-01T11:12:33.971126+0000 mon.a (mon.0) 2129 : audit 1 from='client.? 172.21.15.78:0/4222260494' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:12:34.373 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:34.399 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:34.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:34.402 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:12:34.503 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:34.536 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:34.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:34.545 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:12:34.646 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:34.669 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:34.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:34.672 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:12:34.773 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:34.802 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:34.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:34.808 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:12:34.909 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:34.951 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:34.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:34.956 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:12:35.035 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-10-01T11:12:35.035 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T11:12:35.049 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:35 smithi078 bash[75299]: cluster 2023-10-01T11:12:34.044065+0000 mon.a (mon.0) 2130 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:12:35.049 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:35 smithi078 bash[75299]: cluster 2023-10-01T11:12:34.062956+0000 osd.7 (osd.7) 416 : cluster 3 failed to encode map e370 with expected crc 2023-10-01T11:12:35.049 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:35 smithi078 bash[75299]: audit 2023-10-01T11:12:34.066142+0000 mon.a (mon.0) 2131 : audit 1 from='client.? 172.21.15.78:0/4222260494' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T11:12:35.049 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:35 smithi078 bash[75299]: cluster 2023-10-01T11:12:34.066231+0000 mon.a (mon.0) 2132 : cluster 0 osdmap e370: 8 total, 8 up, 8 in 2023-10-01T11:12:35.049 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:35 smithi078 bash[75299]: cluster 2023-10-01T11:12:34.076988+0000 osd.5 (osd.5) 396 : cluster 3 failed to encode map e370 with expected crc 2023-10-01T11:12:35.049 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:35 smithi078 bash[75299]: cluster 2023-10-01T11:12:34.077556+0000 osd.6 (osd.6) 357 : cluster 3 failed to encode map e370 with expected crc 2023-10-01T11:12:35.049 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:35 smithi078 bash[78274]: cluster 2023-10-01T11:12:34.044065+0000 mon.a (mon.0) 2130 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:12:35.050 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:35 smithi078 bash[78274]: cluster 2023-10-01T11:12:34.062956+0000 osd.7 (osd.7) 416 : cluster 3 failed to encode map e370 with expected crc 2023-10-01T11:12:35.050 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:35 smithi078 bash[78274]: audit 2023-10-01T11:12:34.066142+0000 mon.a (mon.0) 2131 : audit 1 from='client.? 172.21.15.78:0/4222260494' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T11:12:35.050 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:35 smithi078 bash[78274]: cluster 2023-10-01T11:12:34.066231+0000 mon.a (mon.0) 2132 : cluster 0 osdmap e370: 8 total, 8 up, 8 in 2023-10-01T11:12:35.050 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:35 smithi078 bash[78274]: cluster 2023-10-01T11:12:34.076988+0000 osd.5 (osd.5) 396 : cluster 3 failed to encode map e370 with expected crc 2023-10-01T11:12:35.050 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:35 smithi078 bash[78274]: cluster 2023-10-01T11:12:34.077556+0000 osd.6 (osd.6) 357 : cluster 3 failed to encode map e370 with expected crc 2023-10-01T11:12:35.056 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:35.067 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:35 smithi173 bash[58816]: cluster 2023-10-01T11:12:34.044065+0000 mon.a (mon.0) 2130 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:12:35.068 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:35 smithi173 bash[58816]: cluster 2023-10-01T11:12:34.062956+0000 osd.7 (osd.7) 416 : cluster 3 failed to encode map e370 with expected crc 2023-10-01T11:12:35.068 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:35 smithi173 bash[58816]: audit 2023-10-01T11:12:34.066142+0000 mon.a (mon.0) 2131 : audit 1 from='client.? 172.21.15.78:0/4222260494' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T11:12:35.068 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:35 smithi173 bash[58816]: cluster 2023-10-01T11:12:34.066231+0000 mon.a (mon.0) 2132 : cluster 0 osdmap e370: 8 total, 8 up, 8 in 2023-10-01T11:12:35.068 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:35 smithi173 bash[58816]: cluster 2023-10-01T11:12:34.076988+0000 osd.5 (osd.5) 396 : cluster 3 failed to encode map e370 with expected crc 2023-10-01T11:12:35.068 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:35 smithi173 bash[58816]: cluster 2023-10-01T11:12:34.077556+0000 osd.6 (osd.6) 357 : cluster 3 failed to encode map e370 with expected crc 2023-10-01T11:12:35.093 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:35.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:35.099 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:12:35.199 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:35.223 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:35.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:35.227 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:35.328 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:35.354 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:35.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:35.358 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:12:35.459 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:35.483 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:35.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:35.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:12:35.586 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:35.614 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:35.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:35.617 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:35.718 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:35.747 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:35.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:35.752 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:12:35.853 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:35.880 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:35.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:35.885 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:12:35.986 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:36.016 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:36.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:36.024 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:12:36.124 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:36.182 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:36.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:36.184 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:12:36.200 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-01T11:12:36.285 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:36.334 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:36.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:36.340 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:36 smithi078 bash[75299]: cluster 2023-10-01T11:12:34.447342+0000 mgr.y (mgr.25072) 1659 : cluster 0 pgmap v1790: 89 pgs: 89 active+clean; 3.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 675 op/s 2023-10-01T11:12:36.340 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:36 smithi078 bash[78274]: cluster 2023-10-01T11:12:34.447342+0000 mgr.y (mgr.25072) 1659 : cluster 0 pgmap v1790: 89 pgs: 89 active+clean; 3.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 675 op/s 2023-10-01T11:12:36.340 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:12:36.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:36 smithi173 bash[58816]: cluster 2023-10-01T11:12:34.447342+0000 mgr.y (mgr.25072) 1659 : cluster 0 pgmap v1790: 89 pgs: 89 active+clean; 3.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 675 op/s 2023-10-01T11:12:36.442 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:36.466 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:36.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:36.468 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:12:36.569 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:36.609 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:36.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:36.614 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:12:36.714 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:36.753 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:36.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:36.759 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:12:36.860 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:36.886 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:36.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:36.891 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:12:36.991 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:37.027 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:37.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:37.031 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:12:37.132 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:37.192 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:37.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:37.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:12:37.297 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:37.333 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:37.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:37.342 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:12:37.443 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:37.477 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:37.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:37.481 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:12:37.582 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:37.616 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:37.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:37.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:12:37.720 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:37.746 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:37.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:37.750 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:12:37.851 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:37.881 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:37.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:37.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:12:37.984 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:38.014 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:38.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:38.020 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:12:38.095 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T11:12:38.113 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T11:12:38.120 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:38.213 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:38.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:38.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:12:38.317 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:38.330 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:38 smithi078 bash[75299]: cluster 2023-10-01T11:12:36.448021+0000 mgr.y (mgr.25072) 1660 : cluster 0 pgmap v1791: 89 pgs: 89 active+clean; 3.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 797 op/s 2023-10-01T11:12:38.330 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:38 smithi078 bash[78274]: cluster 2023-10-01T11:12:36.448021+0000 mgr.y (mgr.25072) 1660 : cluster 0 pgmap v1791: 89 pgs: 89 active+clean; 3.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 797 op/s 2023-10-01T11:12:38.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:38 smithi173 bash[58816]: cluster 2023-10-01T11:12:36.448021+0000 mgr.y (mgr.25072) 1660 : cluster 0 pgmap v1791: 89 pgs: 89 active+clean; 3.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 797 op/s 2023-10-01T11:12:38.382 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:38.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:38.388 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:12:38.489 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:38.519 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:38.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:38.522 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:12:38.623 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:38.650 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:12:38.660 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:38.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:38.663 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:12:38.764 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:38.817 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:38.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:38.823 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:12:38.923 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:38.952 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:38.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:38.954 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:12:39.055 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:39.107 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:39.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:39.112 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:12:39.213 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:39.254 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:39.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:39.260 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:12:39.360 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:39.418 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:39.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:39.424 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:12:39.464 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:39 smithi078 bash[78274]: audit 2023-10-01T11:12:38.199485+0000 mon.a (mon.0) 2133 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:12:39.464 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:39 smithi078 bash[78274]: audit 2023-10-01T11:12:38.220876+0000 mon.a (mon.0) 2134 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:12:39.464 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:39 smithi078 bash[78274]: cluster 2023-10-01T11:12:38.448492+0000 mgr.y (mgr.25072) 1661 : cluster 0 pgmap v1792: 89 pgs: 89 active+clean; 3.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 464 op/s 2023-10-01T11:12:39.464 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:39 smithi078 bash[75299]: audit 2023-10-01T11:12:38.199485+0000 mon.a (mon.0) 2133 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:12:39.464 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:39 smithi078 bash[75299]: audit 2023-10-01T11:12:38.220876+0000 mon.a (mon.0) 2134 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:12:39.464 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:39 smithi078 bash[75299]: cluster 2023-10-01T11:12:38.448492+0000 mgr.y (mgr.25072) 1661 : cluster 0 pgmap v1792: 89 pgs: 89 active+clean; 3.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 464 op/s 2023-10-01T11:12:39.525 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:39.534 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:39 smithi173 bash[58816]: audit 2023-10-01T11:12:38.199485+0000 mon.a (mon.0) 2133 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:12:39.534 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:39 smithi173 bash[58816]: audit 2023-10-01T11:12:38.220876+0000 mon.a (mon.0) 2134 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:12:39.534 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:39 smithi173 bash[58816]: cluster 2023-10-01T11:12:38.448492+0000 mgr.y (mgr.25072) 1661 : cluster 0 pgmap v1792: 89 pgs: 89 active+clean; 3.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 464 op/s 2023-10-01T11:12:39.551 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:39.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:39.553 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:12:39.654 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:39.678 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:39.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:39.682 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:12:39.782 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:39.810 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:39.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:39.812 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:39.913 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:39.950 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:39.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:39.954 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:12:40.055 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:40.080 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:40.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:40.085 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:40.186 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:40.211 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:40.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:40.213 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:12:40.314 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:40.335 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:40.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:40.340 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:12:40.440 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:40.477 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:40.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:40.487 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:12:40.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:12:40 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:12:40] "GET /metrics HTTP/1.1" 200 36382 "" "Prometheus/2.33.4" 2023-10-01T11:12:40.588 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:40.621 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:40.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:40.626 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:40.727 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:40.753 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T11:12:40.759 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:40.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:40.762 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:40.863 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:40.886 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:40.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:40.889 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:12:40.990 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:41.027 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:41.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:41.031 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:12:41.132 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:41.165 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:41.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:41.166 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:12:41.267 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:41.293 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:41.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:41.295 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:12:41.396 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:41.420 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:41.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:41.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:12:41.426 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:41 smithi173 bash[58816]: audit 2023-10-01T11:12:40.251892+0000 mon.a (mon.0) 2135 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:12:41.426 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:41 smithi173 bash[58816]: audit 2023-10-01T11:12:40.265511+0000 mon.a (mon.0) 2136 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:12:41.426 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:41 smithi173 bash[58816]: cluster 2023-10-01T11:12:40.449423+0000 mgr.y (mgr.25072) 1662 : cluster 0 pgmap v1793: 89 pgs: 89 active+clean; 2.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 649 op/s 2023-10-01T11:12:41.426 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:41 smithi173 bash[58816]: audit 2023-10-01T11:12:40.893380+0000 mon.a (mon.0) 2137 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T11:12:41.426 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:41 smithi173 bash[58816]: audit 2023-10-01T11:12:40.894553+0000 mon.a (mon.0) 2138 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T11:12:41.426 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:41 smithi173 bash[58816]: audit 2023-10-01T11:12:40.908500+0000 mon.a (mon.0) 2139 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:12:41.525 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:41.550 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:41.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:41.555 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:41.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:41 smithi078 bash[75299]: audit 2023-10-01T11:12:40.251892+0000 mon.a (mon.0) 2135 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:12:41.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:41 smithi078 bash[75299]: audit 2023-10-01T11:12:40.265511+0000 mon.a (mon.0) 2136 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:12:41.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:41 smithi078 bash[75299]: cluster 2023-10-01T11:12:40.449423+0000 mgr.y (mgr.25072) 1662 : cluster 0 pgmap v1793: 89 pgs: 89 active+clean; 2.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 649 op/s 2023-10-01T11:12:41.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:41 smithi078 bash[75299]: audit 2023-10-01T11:12:40.893380+0000 mon.a (mon.0) 2137 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T11:12:41.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:41 smithi078 bash[75299]: audit 2023-10-01T11:12:40.894553+0000 mon.a (mon.0) 2138 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T11:12:41.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:41 smithi078 bash[75299]: audit 2023-10-01T11:12:40.908500+0000 mon.a (mon.0) 2139 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:12:41.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:41 smithi078 bash[78274]: audit 2023-10-01T11:12:40.251892+0000 mon.a (mon.0) 2135 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:12:41.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:41 smithi078 bash[78274]: audit 2023-10-01T11:12:40.265511+0000 mon.a (mon.0) 2136 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:12:41.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:41 smithi078 bash[78274]: cluster 2023-10-01T11:12:40.449423+0000 mgr.y (mgr.25072) 1662 : cluster 0 pgmap v1793: 89 pgs: 89 active+clean; 2.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 649 op/s 2023-10-01T11:12:41.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:41 smithi078 bash[78274]: audit 2023-10-01T11:12:40.893380+0000 mon.a (mon.0) 2137 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T11:12:41.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:41 smithi078 bash[78274]: audit 2023-10-01T11:12:40.894553+0000 mon.a (mon.0) 2138 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T11:12:41.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:41 smithi078 bash[78274]: audit 2023-10-01T11:12:40.908500+0000 mon.a (mon.0) 2139 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:12:41.655 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:41.684 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:41.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:41.690 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:41.791 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:41.816 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:41.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:41.821 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:41.922 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:41.947 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:41.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:41.955 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:12:42.056 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:42.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:42.084 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:42.084 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:12:42.185 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:42.211 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:42.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:42.213 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:42.314 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:42.345 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:42.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:42.352 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:12:42.453 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:42.476 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:42.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:42.478 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:12:42.578 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:42.600 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:42.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:42.604 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:12:42.704 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:42.733 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:12:42.734 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:12:42.748 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:42.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:42.749 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:42.850 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:42.873 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:42.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:42.877 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:12:42.978 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:43.012 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:43.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:43.019 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:12:43.120 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:43.150 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:43.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:43.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:12:43.280 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:43.306 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:43.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:43.308 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:43.409 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:43.436 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:43.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:43.447 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:43.547 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:43.578 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:43.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:43.580 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:12:43.681 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:43.712 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:43.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:43.715 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:12:43.815 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:43.839 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:43.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:43.846 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:43.947 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:43.956 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:43 smithi173 bash[58816]: cluster 2023-10-01T11:12:42.449965+0000 mgr.y (mgr.25072) 1663 : cluster 0 pgmap v1794: 89 pgs: 89 active+clean; 2.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 604 op/s 2023-10-01T11:12:43.972 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:43.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:43.978 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:12:44.079 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:44.111 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:44.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:44.118 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:44.219 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:44.239 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:44.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:44.242 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:12:44.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:43 smithi078 bash[75299]: cluster 2023-10-01T11:12:42.449965+0000 mgr.y (mgr.25072) 1663 : cluster 0 pgmap v1794: 89 pgs: 89 active+clean; 2.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 604 op/s 2023-10-01T11:12:44.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:43 smithi078 bash[78274]: cluster 2023-10-01T11:12:42.449965+0000 mgr.y (mgr.25072) 1663 : cluster 0 pgmap v1794: 89 pgs: 89 active+clean; 2.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 604 op/s 2023-10-01T11:12:44.343 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:44.380 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:44.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:44.386 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:12:44.487 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:44.525 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:44.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:44.528 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:12:44.629 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:44.652 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:44.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:44.655 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:12:44.757 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:44.779 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:44.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:44.786 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:12:44.886 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:44.916 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:44.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:44.918 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:12:45.018 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:45.051 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:45.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:45.060 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:12:45.161 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:45.204 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:45.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:45.207 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:12:45.307 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:45.328 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:45.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:45.331 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:12:45.432 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:45.458 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:45.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:45.460 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:45.561 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:45.604 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:45.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:45.610 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:12:45.710 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:45.727 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-10-01T11:12:45.727 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T11:12:45.737 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:45.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:45.742 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:12:45.843 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:45.876 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:45.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:45.879 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:12:45.883 INFO:teuthology.orchestra.run.smithi078.stderr:reweighted osd.5 to 1 (10000) 2023-10-01T11:12:45.935 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:45 smithi078 bash[78274]: cluster 2023-10-01T11:12:44.450603+0000 mgr.y (mgr.25072) 1664 : cluster 0 pgmap v1795: 89 pgs: 89 active+clean; 2.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 643 op/s 2023-10-01T11:12:45.935 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:45 smithi078 bash[78274]: audit 2023-10-01T11:12:45.662067+0000 mon.a (mon.0) 2140 : audit 1 from='client.? 172.21.15.78:0/3887446642' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-01T11:12:45.935 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:45 smithi078 bash[75299]: cluster 2023-10-01T11:12:44.450603+0000 mgr.y (mgr.25072) 1664 : cluster 0 pgmap v1795: 89 pgs: 89 active+clean; 2.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 643 op/s 2023-10-01T11:12:45.935 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:45 smithi078 bash[75299]: audit 2023-10-01T11:12:45.662067+0000 mon.a (mon.0) 2140 : audit 1 from='client.? 172.21.15.78:0/3887446642' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-01T11:12:45.981 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:46.009 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:46.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:46.011 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:12:46.112 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:46.145 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:45 smithi173 bash[58816]: cluster 2023-10-01T11:12:44.450603+0000 mgr.y (mgr.25072) 1664 : cluster 0 pgmap v1795: 89 pgs: 89 active+clean; 2.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 643 op/s 2023-10-01T11:12:46.145 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:45 smithi173 bash[58816]: audit 2023-10-01T11:12:45.662067+0000 mon.a (mon.0) 2140 : audit 1 from='client.? 172.21.15.78:0/3887446642' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-01T11:12:46.146 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:46.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:46.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:12:46.256 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:46.290 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:46.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:46.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:12:46.397 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:46.422 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:46.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:46.428 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:12:46.528 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:46.569 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:46.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:46.571 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:12:46.672 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:46.705 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:46.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:46.712 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:12:46.812 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:46.818 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-01T11:12:46.835 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:46.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:46.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:12:46.940 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:46.971 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:46.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:46.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:12:47.074 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:47.086 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:46 smithi173 bash[58816]: cluster 2023-10-01T11:12:45.868834+0000 osd.7 (osd.7) 417 : cluster 3 failed to encode map e371 with expected crc 2023-10-01T11:12:47.086 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:46 smithi173 bash[58816]: audit 2023-10-01T11:12:45.872229+0000 mon.a (mon.0) 2141 : audit 1 from='client.? 172.21.15.78:0/3887446642' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-01T11:12:47.087 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:46 smithi173 bash[58816]: cluster 2023-10-01T11:12:45.872544+0000 mon.a (mon.0) 2142 : cluster 0 osdmap e371: 8 total, 8 up, 8 in 2023-10-01T11:12:47.087 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:46 smithi173 bash[58816]: cluster 2023-10-01T11:12:45.877243+0000 osd.5 (osd.5) 397 : cluster 3 failed to encode map e371 with expected crc 2023-10-01T11:12:47.087 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:46 smithi173 bash[58816]: cluster 2023-10-01T11:12:45.881236+0000 osd.6 (osd.6) 358 : cluster 3 failed to encode map e371 with expected crc 2023-10-01T11:12:47.087 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:46 smithi173 bash[58816]: cluster 2023-10-01T11:12:45.888593+0000 osd.5 (osd.5) 398 : cluster 3 failed to encode map e371 with expected crc 2023-10-01T11:12:47.112 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:47.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:47.117 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:12:47.218 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:47.226 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:46 smithi078 bash[78274]: cluster 2023-10-01T11:12:45.868834+0000 osd.7 (osd.7) 417 : cluster 3 failed to encode map e371 with expected crc 2023-10-01T11:12:47.226 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:46 smithi078 bash[78274]: audit 2023-10-01T11:12:45.872229+0000 mon.a (mon.0) 2141 : audit 1 from='client.? 172.21.15.78:0/3887446642' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-01T11:12:47.227 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:46 smithi078 bash[78274]: cluster 2023-10-01T11:12:45.872544+0000 mon.a (mon.0) 2142 : cluster 0 osdmap e371: 8 total, 8 up, 8 in 2023-10-01T11:12:47.227 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:46 smithi078 bash[78274]: cluster 2023-10-01T11:12:45.877243+0000 osd.5 (osd.5) 397 : cluster 3 failed to encode map e371 with expected crc 2023-10-01T11:12:47.227 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:46 smithi078 bash[78274]: cluster 2023-10-01T11:12:45.881236+0000 osd.6 (osd.6) 358 : cluster 3 failed to encode map e371 with expected crc 2023-10-01T11:12:47.227 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:46 smithi078 bash[78274]: cluster 2023-10-01T11:12:45.888593+0000 osd.5 (osd.5) 398 : cluster 3 failed to encode map e371 with expected crc 2023-10-01T11:12:47.227 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:46 smithi078 bash[75299]: cluster 2023-10-01T11:12:45.868834+0000 osd.7 (osd.7) 417 : cluster 3 failed to encode map e371 with expected crc 2023-10-01T11:12:47.227 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:46 smithi078 bash[75299]: audit 2023-10-01T11:12:45.872229+0000 mon.a (mon.0) 2141 : audit 1 from='client.? 172.21.15.78:0/3887446642' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-01T11:12:47.227 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:46 smithi078 bash[75299]: cluster 2023-10-01T11:12:45.872544+0000 mon.a (mon.0) 2142 : cluster 0 osdmap e371: 8 total, 8 up, 8 in 2023-10-01T11:12:47.227 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:46 smithi078 bash[75299]: cluster 2023-10-01T11:12:45.877243+0000 osd.5 (osd.5) 397 : cluster 3 failed to encode map e371 with expected crc 2023-10-01T11:12:47.227 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:46 smithi078 bash[75299]: cluster 2023-10-01T11:12:45.881236+0000 osd.6 (osd.6) 358 : cluster 3 failed to encode map e371 with expected crc 2023-10-01T11:12:47.228 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:46 smithi078 bash[75299]: cluster 2023-10-01T11:12:45.888593+0000 osd.5 (osd.5) 398 : cluster 3 failed to encode map e371 with expected crc 2023-10-01T11:12:47.249 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:47.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:47.255 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:12:47.355 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:47.395 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:47.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:47.398 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:12:47.498 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:47.523 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:47.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:47.528 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:12:47.628 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:47.657 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:47.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:47.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:12:47.760 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:47.781 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:47.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:47.784 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:12:47.884 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:47.890 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:47 smithi078 bash[78274]: cluster 2023-10-01T11:12:46.451224+0000 mgr.y (mgr.25072) 1665 : cluster 0 pgmap v1797: 89 pgs: 89 active+clean; 2.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 613 op/s 2023-10-01T11:12:47.891 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:47 smithi078 bash[78274]: audit 2023-10-01T11:12:47.475598+0000 mon.a (mon.0) 2143 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:12:47.891 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:47 smithi078 bash[75299]: cluster 2023-10-01T11:12:46.451224+0000 mgr.y (mgr.25072) 1665 : cluster 0 pgmap v1797: 89 pgs: 89 active+clean; 2.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 613 op/s 2023-10-01T11:12:47.891 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:47 smithi078 bash[75299]: audit 2023-10-01T11:12:47.475598+0000 mon.a (mon.0) 2143 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:12:47.916 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:47.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:47.923 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:12:48.024 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:48.033 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:47 smithi173 bash[58816]: cluster 2023-10-01T11:12:46.451224+0000 mgr.y (mgr.25072) 1665 : cluster 0 pgmap v1797: 89 pgs: 89 active+clean; 2.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 613 op/s 2023-10-01T11:12:48.033 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:47 smithi173 bash[58816]: audit 2023-10-01T11:12:47.475598+0000 mon.a (mon.0) 2143 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:12:48.054 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:48.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:48.058 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:12:48.158 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:48.195 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:48.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:48.199 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:12:48.300 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:48.345 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:48.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:48.348 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:12:48.448 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:48.476 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:48.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:48.483 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:12:48.584 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:48.629 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:48.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:48.637 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:12:48.723 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-01T11:12:48.737 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:48.810 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:48.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:48.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:12:48.921 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:48.947 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:48.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:48.954 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:12:49.055 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:49.097 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:49.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:49.104 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:12:49.205 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:49.232 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:49.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:49.234 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:12:49.335 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:49.374 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:49.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:49.377 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:12:49.477 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:49.510 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:49.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:49.512 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:12:49.613 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:49.650 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:49.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:49.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:12:49.753 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:49.789 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:49.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:49.796 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:12:49.894 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:49 smithi078 bash[75299]: cluster 2023-10-01T11:12:48.451656+0000 mgr.y (mgr.25072) 1666 : cluster 0 pgmap v1798: 89 pgs: 89 active+clean; 2.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 613 op/s 2023-10-01T11:12:49.897 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:49.925 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:49.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:49.931 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:12:50.032 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:50.042 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:49 smithi173 bash[58816]: cluster 2023-10-01T11:12:48.451656+0000 mgr.y (mgr.25072) 1666 : cluster 0 pgmap v1798: 89 pgs: 89 active+clean; 2.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 613 op/s 2023-10-01T11:12:50.059 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:50.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:50.066 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:50.166 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:50.174 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:49 smithi078 bash[78274]: cluster 2023-10-01T11:12:48.451656+0000 mgr.y (mgr.25072) 1666 : cluster 0 pgmap v1798: 89 pgs: 89 active+clean; 2.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 613 op/s 2023-10-01T11:12:50.194 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:50.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:50.202 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:12:50.302 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:50.334 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:50.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:50.338 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:12:50.439 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:50.464 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:50.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:50.472 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:12:50.572 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:50.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:12:50 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:12:50] "GET /metrics HTTP/1.1" 200 36384 "" "Prometheus/2.33.4" 2023-10-01T11:12:50.595 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:50.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:50.599 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:12:50.699 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:50.745 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:50.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:50.748 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:12:50.849 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:50.872 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:50.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:50.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:12:50.976 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:51.000 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:51.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:51.006 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:51.107 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:51.136 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:51.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:51.138 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:12:51.239 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:51.273 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:51.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:51.280 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:12:51.380 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:51.413 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:51.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:51.419 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:12:51.520 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:51.559 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:51.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:51.561 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:12:51.652 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T11:12:51.662 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:51.692 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:51.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:51.695 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:12:51.795 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:51.817 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:51.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:51.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:12:51.897 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:51 smithi078 bash[75299]: cluster 2023-10-01T11:12:50.452330+0000 mgr.y (mgr.25072) 1667 : cluster 0 pgmap v1799: 89 pgs: 89 active+clean; 2.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 642 op/s 2023-10-01T11:12:51.897 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:51 smithi078 bash[75299]: audit 2023-10-01T11:12:51.891224+0000 mon.b (mon.1) 61 : audit 0 from='client.? 172.21.15.78:0/1435629940' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:12:51.897 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:51 smithi078 bash[78274]: cluster 2023-10-01T11:12:50.452330+0000 mgr.y (mgr.25072) 1667 : cluster 0 pgmap v1799: 89 pgs: 89 active+clean; 2.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 642 op/s 2023-10-01T11:12:51.920 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:51.928 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:51 smithi173 bash[58816]: cluster 2023-10-01T11:12:50.452330+0000 mgr.y (mgr.25072) 1667 : cluster 0 pgmap v1799: 89 pgs: 89 active+clean; 2.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 642 op/s 2023-10-01T11:12:51.928 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:51 smithi173 bash[58816]: audit 2023-10-01T11:12:51.891224+0000 mon.b (mon.1) 61 : audit 0 from='client.? 172.21.15.78:0/1435629940' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:12:51.948 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:51.949 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T11:12:51.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:51.951 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:12:51.991 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T11:12:52.051 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:52.052 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T11:12:52.076 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:52.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:52.083 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:52.118 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T11:12:52.159 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T11:12:52.183 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:52.196 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:51 smithi078 bash[78274]: audit 2023-10-01T11:12:51.891224+0000 mon.b (mon.1) 61 : audit 0 from='client.? 172.21.15.78:0/1435629940' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:12:52.203 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T11:12:52.232 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:52.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:52.237 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:12:52.240 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T11:12:52.280 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T11:12:52.285 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T11:12:52.286 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T11:12:52.286 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T11:12:52.286 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T11:12:52.286 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T11:12:52.287 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T11:12:52.287 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T11:12:52.287 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T11:12:52.338 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:52.377 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:52.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:52.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:12:52.480 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:52.505 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:52.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:52.508 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:52.608 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:52.632 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:52.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:52.635 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:52.736 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:52.764 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:52.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:52.770 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:12:52.871 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:52.892 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:52.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:52.894 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:12:52.926 INFO:teuthology.orchestra.run.smithi078.stderr:noscrub is unset 2023-10-01T11:12:52.996 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:53.018 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:53.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:53.023 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:12:53.123 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:53.134 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:52 smithi173 bash[58816]: audit 2023-10-01T11:12:52.272751+0000 mon.a (mon.0) 2144 : audit 1 from='client.? 172.21.15.78:0/1064050550' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T11:12:53.153 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:53.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:53.156 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:53.257 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:53.263 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:52 smithi078 bash[75299]: audit 2023-10-01T11:12:52.272751+0000 mon.a (mon.0) 2144 : audit 1 from='client.? 172.21.15.78:0/1064050550' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T11:12:53.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:52 smithi078 bash[78274]: audit 2023-10-01T11:12:52.272751+0000 mon.a (mon.0) 2144 : audit 1 from='client.? 172.21.15.78:0/1064050550' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T11:12:53.274 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:53.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:53.276 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:12:53.376 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:53.410 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:53.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:53.418 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:53.518 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:53.544 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:53.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:53.550 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:12:53.651 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:53.676 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:53.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:53.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:53.779 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:53.804 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:53.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:53.810 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:53.911 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:53.920 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:53 smithi173 bash[58816]: cluster 2023-10-01T11:12:52.452866+0000 mgr.y (mgr.25072) 1668 : cluster 0 pgmap v1800: 89 pgs: 89 active+clean; 2.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 649 op/s 2023-10-01T11:12:53.920 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:53 smithi173 bash[58816]: cluster 2023-10-01T11:12:52.895797+0000 mon.a (mon.0) 2145 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:12:53.920 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:53 smithi173 bash[58816]: cluster 2023-10-01T11:12:52.911700+0000 osd.5 (osd.5) 399 : cluster 3 failed to encode map e372 with expected crc 2023-10-01T11:12:53.920 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:53 smithi173 bash[58816]: cluster 2023-10-01T11:12:52.913373+0000 osd.7 (osd.7) 418 : cluster 3 failed to encode map e372 with expected crc 2023-10-01T11:12:53.920 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:53 smithi173 bash[58816]: cluster 2023-10-01T11:12:52.916281+0000 osd.6 (osd.6) 359 : cluster 3 failed to encode map e372 with expected crc 2023-10-01T11:12:53.920 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:53 smithi173 bash[58816]: audit 2023-10-01T11:12:52.916405+0000 mon.a (mon.0) 2146 : audit 1 from='client.? 172.21.15.78:0/1064050550' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T11:12:53.920 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:53 smithi173 bash[58816]: cluster 2023-10-01T11:12:52.916590+0000 mon.a (mon.0) 2147 : cluster 0 osdmap e372: 8 total, 8 up, 8 in 2023-10-01T11:12:53.939 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:53.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:53.942 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:12:54.042 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:54.086 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:54.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:54.090 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:12:54.191 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:54.201 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:53 smithi078 bash[78274]: cluster 2023-10-01T11:12:52.452866+0000 mgr.y (mgr.25072) 1668 : cluster 0 pgmap v1800: 89 pgs: 89 active+clean; 2.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 649 op/s 2023-10-01T11:12:54.201 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:53 smithi078 bash[78274]: cluster 2023-10-01T11:12:52.895797+0000 mon.a (mon.0) 2145 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:12:54.201 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:53 smithi078 bash[78274]: cluster 2023-10-01T11:12:52.911700+0000 osd.5 (osd.5) 399 : cluster 3 failed to encode map e372 with expected crc 2023-10-01T11:12:54.201 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:53 smithi078 bash[78274]: cluster 2023-10-01T11:12:52.913373+0000 osd.7 (osd.7) 418 : cluster 3 failed to encode map e372 with expected crc 2023-10-01T11:12:54.201 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:53 smithi078 bash[78274]: cluster 2023-10-01T11:12:52.916281+0000 osd.6 (osd.6) 359 : cluster 3 failed to encode map e372 with expected crc 2023-10-01T11:12:54.202 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:53 smithi078 bash[78274]: audit 2023-10-01T11:12:52.916405+0000 mon.a (mon.0) 2146 : audit 1 from='client.? 172.21.15.78:0/1064050550' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T11:12:54.202 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:53 smithi078 bash[78274]: cluster 2023-10-01T11:12:52.916590+0000 mon.a (mon.0) 2147 : cluster 0 osdmap e372: 8 total, 8 up, 8 in 2023-10-01T11:12:54.202 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:53 smithi078 bash[75299]: cluster 2023-10-01T11:12:52.452866+0000 mgr.y (mgr.25072) 1668 : cluster 0 pgmap v1800: 89 pgs: 89 active+clean; 2.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 649 op/s 2023-10-01T11:12:54.203 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:53 smithi078 bash[75299]: cluster 2023-10-01T11:12:52.895797+0000 mon.a (mon.0) 2145 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:12:54.203 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:53 smithi078 bash[75299]: cluster 2023-10-01T11:12:52.911700+0000 osd.5 (osd.5) 399 : cluster 3 failed to encode map e372 with expected crc 2023-10-01T11:12:54.203 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:53 smithi078 bash[75299]: cluster 2023-10-01T11:12:52.913373+0000 osd.7 (osd.7) 418 : cluster 3 failed to encode map e372 with expected crc 2023-10-01T11:12:54.203 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:53 smithi078 bash[75299]: cluster 2023-10-01T11:12:52.916281+0000 osd.6 (osd.6) 359 : cluster 3 failed to encode map e372 with expected crc 2023-10-01T11:12:54.203 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:53 smithi078 bash[75299]: audit 2023-10-01T11:12:52.916405+0000 mon.a (mon.0) 2146 : audit 1 from='client.? 172.21.15.78:0/1064050550' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T11:12:54.204 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:53 smithi078 bash[75299]: cluster 2023-10-01T11:12:52.916590+0000 mon.a (mon.0) 2147 : cluster 0 osdmap e372: 8 total, 8 up, 8 in 2023-10-01T11:12:54.231 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:54.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:54.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:12:54.339 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:54.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:54.377 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:54.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:12:54.477 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:54.516 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:54.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:54.519 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:12:54.619 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:54.643 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:54.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:54.647 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:12:54.748 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:54.780 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:54.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:54.787 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:12:54.887 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:54.929 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:54.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:54.932 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:12:55.032 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:55.061 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:55.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:55.064 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:12:55.165 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:55.188 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:55.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:55.190 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:12:55.291 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:55.311 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:55.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:55.318 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:12:55.419 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:55.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:55.449 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:55.449 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:12:55.550 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:55.588 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:55.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:55.590 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:55.691 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:55.698 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:12:55.715 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:55.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:55.717 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:12:55.818 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:55.841 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:55.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:55.843 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:12:55.944 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:55.967 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:55.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:55.969 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:56.070 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:56.079 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:55 smithi173 bash[58816]: cluster 2023-10-01T11:12:54.453575+0000 mgr.y (mgr.25072) 1669 : cluster 0 pgmap v1802: 89 pgs: 89 active+clean; 2.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 705 op/s 2023-10-01T11:12:56.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:55 smithi078 bash[75299]: cluster 2023-10-01T11:12:54.453575+0000 mgr.y (mgr.25072) 1669 : cluster 0 pgmap v1802: 89 pgs: 89 active+clean; 2.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 705 op/s 2023-10-01T11:12:56.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:55 smithi078 bash[78274]: cluster 2023-10-01T11:12:54.453575+0000 mgr.y (mgr.25072) 1669 : cluster 0 pgmap v1802: 89 pgs: 89 active+clean; 2.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 705 op/s 2023-10-01T11:12:56.099 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:56.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:56.103 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:56.204 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:56.253 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:56.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:56.257 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:56.358 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:56.383 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T11:12:56.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:56.388 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:56.388 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:56.445 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-10-01T11:12:56.445 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [], 2023-10-01T11:12:56.445 INFO:teuthology.orchestra.run.smithi173.stdout: "complaint_time": 30, 2023-10-01T11:12:56.445 INFO:teuthology.orchestra.run.smithi173.stdout: "num_blocked_ops": 0 2023-10-01T11:12:56.445 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-10-01T11:12:56.489 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:56.530 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:56.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:56.536 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:12:56.637 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:56.662 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:56.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:56.668 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:12:56.769 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:56.791 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:56.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:56.795 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:12:56.896 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:56.917 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:56.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:56.920 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:12:57.021 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:57.050 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:57.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:57.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:12:57.158 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:57.179 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:57.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:57.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:12:57.215 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T11:12:57.283 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:57.307 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:57.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:57.312 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:57.413 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:57.445 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:57.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:57.450 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:12:57.550 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:57.581 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:57.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:57.586 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:12:57.615 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-01T11:12:57.686 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:57.710 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:57.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:57.714 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:12:57.815 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:57.845 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:57.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:57.855 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:12:57.956 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:57.979 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:57.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:57.986 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:12:58.087 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:58.120 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:12:58.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:58.126 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:12:58.226 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:58.234 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:57 smithi078 bash[78274]: cluster 2023-10-01T11:12:56.454149+0000 mgr.y (mgr.25072) 1670 : cluster 0 pgmap v1803: 89 pgs: 89 active+clean; 2.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 645 op/s 2023-10-01T11:12:58.235 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:57 smithi078 bash[75299]: cluster 2023-10-01T11:12:56.454149+0000 mgr.y (mgr.25072) 1670 : cluster 0 pgmap v1803: 89 pgs: 89 active+clean; 2.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 645 op/s 2023-10-01T11:12:58.255 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:12:58.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:58.266 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:12:58.367 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:58.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:57 smithi173 bash[58816]: cluster 2023-10-01T11:12:56.454149+0000 mgr.y (mgr.25072) 1670 : cluster 0 pgmap v1803: 89 pgs: 89 active+clean; 2.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 645 op/s 2023-10-01T11:12:58.395 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:58.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:58.403 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:12:58.503 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:58.535 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:58.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:58.537 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:12:58.638 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:58.676 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:58.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:58.683 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:12:58.784 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:58.808 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:58.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:58.812 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:12:58.912 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:58.938 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:58.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:58.940 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:12:59.041 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:59.074 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:12:59.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:59.076 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:12:59.177 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:59.200 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:12:59.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:59.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:12:59.308 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:59.333 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:12:59.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:59.335 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:12:59.436 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:59.467 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:12:59.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:59.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:12:59.571 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:59.598 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:59.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:59.600 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:12:59.700 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:59.746 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:12:59.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:59.748 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:12:59.849 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:59.883 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:12:59.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:12:59.886 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:12:59.986 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:00.022 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:00.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:00.027 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:13:00.127 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:00.136 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:12:59 smithi078 bash[75299]: cluster 2023-10-01T11:12:58.454576+0000 mgr.y (mgr.25072) 1671 : cluster 0 pgmap v1804: 89 pgs: 89 active+clean; 2.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 645 op/s 2023-10-01T11:13:00.136 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:12:59 smithi078 bash[78274]: cluster 2023-10-01T11:12:58.454576+0000 mgr.y (mgr.25072) 1671 : cluster 0 pgmap v1804: 89 pgs: 89 active+clean; 2.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 645 op/s 2023-10-01T11:13:00.153 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:00.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:00.156 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:13:00.256 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:00.302 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:00.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:00.308 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:13:00.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:12:59 smithi173 bash[58816]: cluster 2023-10-01T11:12:58.454576+0000 mgr.y (mgr.25072) 1671 : cluster 0 pgmap v1804: 89 pgs: 89 active+clean; 2.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 645 op/s 2023-10-01T11:13:00.409 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:00.449 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:00.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:00.459 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:13:00.560 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:00.568 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:13:00 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:13:00] "GET /metrics HTTP/1.1" 200 36384 "" "Prometheus/2.33.4" 2023-10-01T11:13:00.584 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:00.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:00.586 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:13:00.687 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:00.732 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:00.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:00.738 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:13:00.839 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:00.861 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:00.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:00.864 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:13:00.952 INFO:teuthology.orchestra.run.smithi078.stderr:reweighted osd.6 to 1 (10000) 2023-10-01T11:13:00.965 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:00.997 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:01.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:01.003 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:13:01.103 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:01.111 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:00 smithi173 bash[58816]: audit 2023-10-01T11:13:00.481034+0000 mon.a (mon.0) 2148 : audit 1 from='client.? 172.21.15.78:0/3998165626' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-01T11:13:01.127 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:01.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:01.129 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:13:01.204 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:00 smithi078 bash[78274]: audit 2023-10-01T11:13:00.481034+0000 mon.a (mon.0) 2148 : audit 1 from='client.? 172.21.15.78:0/3998165626' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-01T11:13:01.204 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:00 smithi078 bash[75299]: audit 2023-10-01T11:13:00.481034+0000 mon.a (mon.0) 2148 : audit 1 from='client.? 172.21.15.78:0/3998165626' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-01T11:13:01.230 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:01.259 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:01.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:01.261 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:13:01.362 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:01.388 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:01.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:01.391 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:13:01.491 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:01.518 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:01.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:01.522 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:13:01.623 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:01.646 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:01.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:01.653 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:13:01.754 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:01.776 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:01.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:01.779 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:13:01.880 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:01.904 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:01.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:01.906 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:13:02.006 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:02.035 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:02.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:02.040 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:13:02.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:01 smithi173 bash[58816]: cluster 2023-10-01T11:13:00.455243+0000 mgr.y (mgr.25072) 1672 : cluster 0 pgmap v1805: 89 pgs: 89 active+clean; 2.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 641 op/s 2023-10-01T11:13:02.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:01 smithi173 bash[58816]: cluster 2023-10-01T11:13:00.936560+0000 osd.7 (osd.7) 419 : cluster 3 failed to encode map e373 with expected crc 2023-10-01T11:13:02.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:01 smithi173 bash[58816]: cluster 2023-10-01T11:13:00.936850+0000 osd.5 (osd.5) 400 : cluster 3 failed to encode map e373 with expected crc 2023-10-01T11:13:02.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:01 smithi173 bash[58816]: cluster 2023-10-01T11:13:00.943011+0000 osd.6 (osd.6) 360 : cluster 3 failed to encode map e373 with expected crc 2023-10-01T11:13:02.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:01 smithi173 bash[58816]: audit 2023-10-01T11:13:00.943297+0000 mon.a (mon.0) 2149 : audit 1 from='client.? 172.21.15.78:0/3998165626' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-01T11:13:02.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:01 smithi173 bash[58816]: cluster 2023-10-01T11:13:00.949128+0000 mon.a (mon.0) 2150 : cluster 0 osdmap e373: 8 total, 8 up, 8 in 2023-10-01T11:13:02.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:01 smithi173 bash[58816]: cluster 2023-10-01T11:13:00.951410+0000 osd.5 (osd.5) 401 : cluster 3 failed to encode map e373 with expected crc 2023-10-01T11:13:02.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:01 smithi173 bash[58816]: cluster 2023-10-01T11:13:00.955008+0000 osd.6 (osd.6) 361 : cluster 3 failed to encode map e373 with expected crc 2023-10-01T11:13:02.140 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:02.175 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:02.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:02.183 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:13:02.284 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:02.292 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:02 smithi078 bash[78274]: cluster 2023-10-01T11:13:00.455243+0000 mgr.y (mgr.25072) 1672 : cluster 0 pgmap v1805: 89 pgs: 89 active+clean; 2.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 641 op/s 2023-10-01T11:13:02.292 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:02 smithi078 bash[78274]: cluster 2023-10-01T11:13:00.936560+0000 osd.7 (osd.7) 419 : cluster 3 failed to encode map e373 with expected crc 2023-10-01T11:13:02.292 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:02 smithi078 bash[78274]: cluster 2023-10-01T11:13:00.936850+0000 osd.5 (osd.5) 400 : cluster 3 failed to encode map e373 with expected crc 2023-10-01T11:13:02.292 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:02 smithi078 bash[78274]: cluster 2023-10-01T11:13:00.943011+0000 osd.6 (osd.6) 360 : cluster 3 failed to encode map e373 with expected crc 2023-10-01T11:13:02.292 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:02 smithi078 bash[78274]: audit 2023-10-01T11:13:00.943297+0000 mon.a (mon.0) 2149 : audit 1 from='client.? 172.21.15.78:0/3998165626' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-01T11:13:02.293 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:02 smithi078 bash[78274]: cluster 2023-10-01T11:13:00.949128+0000 mon.a (mon.0) 2150 : cluster 0 osdmap e373: 8 total, 8 up, 8 in 2023-10-01T11:13:02.293 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:02 smithi078 bash[78274]: cluster 2023-10-01T11:13:00.951410+0000 osd.5 (osd.5) 401 : cluster 3 failed to encode map e373 with expected crc 2023-10-01T11:13:02.293 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:02 smithi078 bash[78274]: cluster 2023-10-01T11:13:00.955008+0000 osd.6 (osd.6) 361 : cluster 3 failed to encode map e373 with expected crc 2023-10-01T11:13:02.293 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:02 smithi078 bash[75299]: cluster 2023-10-01T11:13:00.455243+0000 mgr.y (mgr.25072) 1672 : cluster 0 pgmap v1805: 89 pgs: 89 active+clean; 2.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 641 op/s 2023-10-01T11:13:02.293 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:02 smithi078 bash[75299]: cluster 2023-10-01T11:13:00.936560+0000 osd.7 (osd.7) 419 : cluster 3 failed to encode map e373 with expected crc 2023-10-01T11:13:02.293 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:02 smithi078 bash[75299]: cluster 2023-10-01T11:13:00.936850+0000 osd.5 (osd.5) 400 : cluster 3 failed to encode map e373 with expected crc 2023-10-01T11:13:02.293 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:02 smithi078 bash[75299]: cluster 2023-10-01T11:13:00.943011+0000 osd.6 (osd.6) 360 : cluster 3 failed to encode map e373 with expected crc 2023-10-01T11:13:02.293 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:02 smithi078 bash[75299]: audit 2023-10-01T11:13:00.943297+0000 mon.a (mon.0) 2149 : audit 1 from='client.? 172.21.15.78:0/3998165626' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-01T11:13:02.294 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:02 smithi078 bash[75299]: cluster 2023-10-01T11:13:00.949128+0000 mon.a (mon.0) 2150 : cluster 0 osdmap e373: 8 total, 8 up, 8 in 2023-10-01T11:13:02.294 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:02 smithi078 bash[75299]: cluster 2023-10-01T11:13:00.951410+0000 osd.5 (osd.5) 401 : cluster 3 failed to encode map e373 with expected crc 2023-10-01T11:13:02.294 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:02 smithi078 bash[75299]: cluster 2023-10-01T11:13:00.955008+0000 osd.6 (osd.6) 361 : cluster 3 failed to encode map e373 with expected crc 2023-10-01T11:13:02.328 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:02.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:02.332 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:13:02.424 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T11:13:02.432 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:02.458 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:02.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:02.460 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:13:02.541 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-01T11:13:02.561 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:02.631 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:02.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:02.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:13:02.735 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:02.788 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:02.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:02.795 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:13:02.896 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:02.925 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:02.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:02.932 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:13:02.965 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:13:02.965 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:13:03.033 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:03.064 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:03.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:03.070 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:13:03.171 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:03.191 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:03.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:03.193 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:13:03.293 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:03.304 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:02 smithi173 bash[58816]: audit 2023-10-01T11:13:02.476149+0000 mon.a (mon.0) 2151 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:13:03.320 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:03.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:03.323 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:13:03.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:02 smithi078 bash[75299]: audit 2023-10-01T11:13:02.476149+0000 mon.a (mon.0) 2151 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:13:03.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:02 smithi078 bash[78274]: audit 2023-10-01T11:13:02.476149+0000 mon.a (mon.0) 2151 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:13:03.424 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:03.453 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:03.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:03.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:13:03.555 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:03.580 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:03.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:03.582 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:13:03.683 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:03.714 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:03.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:03.718 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:13:03.819 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:03.845 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:03.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:03.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:13:03.949 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:03.952 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:03 smithi078 bash[75299]: cluster 2023-10-01T11:13:02.455768+0000 mgr.y (mgr.25072) 1673 : cluster 0 pgmap v1807: 89 pgs: 89 active+clean; 2.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 667 op/s 2023-10-01T11:13:03.975 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:03.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:03.978 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:13:04.079 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:04.112 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:04.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:04.117 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:13:04.218 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:04.227 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:03 smithi078 bash[78274]: cluster 2023-10-01T11:13:02.455768+0000 mgr.y (mgr.25072) 1673 : cluster 0 pgmap v1807: 89 pgs: 89 active+clean; 2.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 667 op/s 2023-10-01T11:13:04.256 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:04.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:04.260 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:13:04.361 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:04.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:03 smithi173 bash[58816]: cluster 2023-10-01T11:13:02.455768+0000 mgr.y (mgr.25072) 1673 : cluster 0 pgmap v1807: 89 pgs: 89 active+clean; 2.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 667 op/s 2023-10-01T11:13:04.385 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:04.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:04.388 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:13:04.488 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:04.514 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:04.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:04.519 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:13:04.620 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:04.643 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:04.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:04.650 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:13:04.751 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:04.783 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:04.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:04.786 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:13:04.887 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:04.906 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:04.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:04.910 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:13:05.011 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:05.042 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:05.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:05.045 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:13:05.146 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:05.168 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:05.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:05.170 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:13:05.271 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:05.296 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:05.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:05.303 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:13:05.403 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:05.432 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:05.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:05.438 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:13:05.539 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:05.562 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:05.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:05.564 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:13:05.665 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:05.700 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:05.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:05.703 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:13:05.804 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:05.829 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:05.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:05.831 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:13:05.932 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:05.954 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:05.957 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:05 smithi078 bash[75299]: cluster 2023-10-01T11:13:04.456293+0000 mgr.y (mgr.25072) 1674 : cluster 0 pgmap v1808: 89 pgs: 89 active+clean; 2.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 638 op/s 2023-10-01T11:13:05.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:05.958 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:13:06.059 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:06.083 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:06.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:06.086 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:13:06.186 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:06.198 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:05 smithi173 bash[58816]: cluster 2023-10-01T11:13:04.456293+0000 mgr.y (mgr.25072) 1674 : cluster 0 pgmap v1808: 89 pgs: 89 active+clean; 2.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 638 op/s 2023-10-01T11:13:06.212 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:06.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:06.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:13:06.378 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:05 smithi078 bash[78274]: cluster 2023-10-01T11:13:04.456293+0000 mgr.y (mgr.25072) 1674 : cluster 0 pgmap v1808: 89 pgs: 89 active+clean; 2.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 638 op/s 2023-10-01T11:13:06.478 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:06.504 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:06.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:06.508 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:13:06.609 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:06.642 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:06.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:06.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:13:06.745 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:06.780 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:06.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:06.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:13:06.884 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:06.910 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:06.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:06.918 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:13:07.019 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:07.047 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:07.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:07.051 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:13:07.152 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:07.185 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:07.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:07.190 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:13:07.282 INFO:teuthology.orchestra.run.smithi173.stdout:{ 2023-10-01T11:13:07.283 INFO:teuthology.orchestra.run.smithi173.stdout: "size": 20, 2023-10-01T11:13:07.283 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 600, 2023-10-01T11:13:07.283 INFO:teuthology.orchestra.run.smithi173.stdout: "ops": [ 2023-10-01T11:13:07.283 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.283 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:73718 13.7 13:eae49b7e:::benchmark_data_smithi078_202524_object73717:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e300)", 2023-10-01T11:13:07.283 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:03:07.717689+0000", 2023-10-01T11:13:07.283 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 599.55369023100002, 2023-10-01T11:13:07.283 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.6589488169999997, 2023-10-01T11:13:07.284 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:13:07.284 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "reached pg", 2023-10-01T11:13:07.284 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:13:07.284 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:13:07.284 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:13:07.284 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 73718 2023-10-01T11:13:07.284 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.284 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:13:07.284 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.285 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:13:07.285 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.717689+0000", 2023-10-01T11:13:07.285 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:13:07.285 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.285 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.285 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:13:07.285 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.717689+0000", 2023-10-01T11:13:07.285 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999976 2023-10-01T11:13:07.285 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.285 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.286 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:13:07.286 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.717687+0000", 2023-10-01T11:13:07.286 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00048091099999999999 2023-10-01T11:13:07.286 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.286 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.286 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:13:07.286 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.718168+0000", 2023-10-01T11:13:07.286 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.7263000000000001e-05 2023-10-01T11:13:07.286 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.286 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.286 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:13:07.287 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.718185+0000", 2023-10-01T11:13:07.287 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.2019999999999993e-06 2023-10-01T11:13:07.287 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.287 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.287 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:13:07.287 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.718194+0000", 2023-10-01T11:13:07.287 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0039130229999999998 2023-10-01T11:13:07.287 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.287 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.288 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.288 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.722107+0000", 2023-10-01T11:13:07.288 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2587e-05 2023-10-01T11:13:07.288 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.288 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.288 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:13:07.288 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.722120+0000", 2023-10-01T11:13:07.288 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.6514172919999996 2023-10-01T11:13:07.288 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.288 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.288 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.288 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.373537+0000", 2023-10-01T11:13:07.289 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0031010709999999999 2023-10-01T11:13:07.289 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.289 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.289 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:13:07.289 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.376638+0000", 2023-10-01T11:13:07.289 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.6589488169999997 2023-10-01T11:13:07.289 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:13:07.289 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:13:07.289 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:13:07.289 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.289 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.289 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:73722 13.7 13:efda010a:::benchmark_data_smithi078_202524_object73721:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e300)", 2023-10-01T11:13:07.290 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:03:07.718207+0000", 2023-10-01T11:13:07.290 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 599.553172754, 2023-10-01T11:13:07.290 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.8001951419999997, 2023-10-01T11:13:07.290 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:13:07.290 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "reached pg", 2023-10-01T11:13:07.290 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:13:07.290 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:13:07.290 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:13:07.290 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 73722 2023-10-01T11:13:07.291 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.291 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:13:07.291 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.291 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:13:07.291 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.718207+0000", 2023-10-01T11:13:07.291 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:13:07.291 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.291 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.291 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:13:07.291 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.718207+0000", 2023-10-01T11:13:07.291 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-10-01T11:13:07.292 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.292 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.292 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:13:07.292 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.718206+0000", 2023-10-01T11:13:07.292 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.4770000000000003e-05 2023-10-01T11:13:07.292 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.292 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.292 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:13:07.292 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.718290+0000", 2023-10-01T11:13:07.292 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.2629999999999992e-06 2023-10-01T11:13:07.292 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.292 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.293 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:13:07.293 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.718300+0000", 2023-10-01T11:13:07.293 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.0739999999999998e-06 2023-10-01T11:13:07.293 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.293 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.293 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:13:07.293 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.718306+0000", 2023-10-01T11:13:07.293 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0038408040000000002 2023-10-01T11:13:07.293 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.293 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.293 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.293 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.722147+0000", 2023-10-01T11:13:07.294 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.3969999999999994e-06 2023-10-01T11:13:07.294 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.294 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.294 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:13:07.294 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.722155+0000", 2023-10-01T11:13:07.294 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.6549701509999997 2023-10-01T11:13:07.294 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.294 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.294 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.294 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.377125+0000", 2023-10-01T11:13:07.294 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.014543443 2023-10-01T11:13:07.295 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.295 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.295 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for degraded object", 2023-10-01T11:13:07.295 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.391668+0000", 2023-10-01T11:13:07.295 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.122085764 2023-10-01T11:13:07.295 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.295 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.295 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.295 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.513754+0000", 2023-10-01T11:13:07.295 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.004647544 2023-10-01T11:13:07.295 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.295 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.296 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:13:07.296 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.518402+0000", 2023-10-01T11:13:07.296 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.8001951419999997 2023-10-01T11:13:07.296 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:13:07.296 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:13:07.296 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:13:07.296 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.296 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.296 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:73728 13.7 13:e775b1ab:::benchmark_data_smithi078_202524_object73727:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e300)", 2023-10-01T11:13:07.296 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:03:07.718312+0000", 2023-10-01T11:13:07.296 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 599.55306696499997, 2023-10-01T11:13:07.297 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.2149130760000002, 2023-10-01T11:13:07.297 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:13:07.297 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "reached pg", 2023-10-01T11:13:07.297 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:13:07.297 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:13:07.297 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:13:07.297 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 73728 2023-10-01T11:13:07.297 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.297 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:13:07.297 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.297 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:13:07.297 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.718312+0000", 2023-10-01T11:13:07.298 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:13:07.298 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.298 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.298 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:13:07.298 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.718312+0000", 2023-10-01T11:13:07.298 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-10-01T11:13:07.298 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.298 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.298 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:13:07.298 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.718312+0000", 2023-10-01T11:13:07.298 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.8604000000000003e-05 2023-10-01T11:13:07.298 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.299 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.299 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:13:07.299 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.718370+0000", 2023-10-01T11:13:07.299 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.2169999999999994e-06 2023-10-01T11:13:07.299 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.299 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.299 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:13:07.299 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.718378+0000", 2023-10-01T11:13:07.299 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.9350000000000002e-06 2023-10-01T11:13:07.299 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.299 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.300 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:13:07.300 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.718383+0000", 2023-10-01T11:13:07.300 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0037959560000000001 2023-10-01T11:13:07.300 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.300 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.300 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.300 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.722179+0000", 2023-10-01T11:13:07.300 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.661e-06 2023-10-01T11:13:07.300 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.300 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.300 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:13:07.301 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.722187+0000", 2023-10-01T11:13:07.301 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.6709630200000003 2023-10-01T11:13:07.301 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.301 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.301 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.301 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.393150+0000", 2023-10-01T11:13:07.301 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0071622500000000002 2023-10-01T11:13:07.301 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.301 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.301 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for missing object", 2023-10-01T11:13:07.301 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.400312+0000", 2023-10-01T11:13:07.302 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.142406428 2023-10-01T11:13:07.302 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.302 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.302 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.302 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.542719+0000", 2023-10-01T11:13:07.302 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0057209299999999999 2023-10-01T11:13:07.302 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.302 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.302 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for degraded object", 2023-10-01T11:13:07.302 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.548440+0000", 2023-10-01T11:13:07.302 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.38058401400000003 2023-10-01T11:13:07.302 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.303 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.303 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.303 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.929024+0000", 2023-10-01T11:13:07.303 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0042019179999999998 2023-10-01T11:13:07.303 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.303 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.303 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:13:07.303 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.933226+0000", 2023-10-01T11:13:07.303 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.2149130760000002 2023-10-01T11:13:07.303 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:13:07.303 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:13:07.304 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:13:07.304 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.304 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.304 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:73741 13.7 13:e43f8f33:::benchmark_data_smithi078_202524_object73740:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e300)", 2023-10-01T11:13:07.304 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:03:07.718389+0000", 2023-10-01T11:13:07.304 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 599.55299005400002, 2023-10-01T11:13:07.304 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.854059522, 2023-10-01T11:13:07.304 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:13:07.304 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:13:07.304 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:13:07.304 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:13:07.304 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:13:07.305 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 73741 2023-10-01T11:13:07.305 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.305 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:13:07.305 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.305 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:13:07.305 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.718389+0000", 2023-10-01T11:13:07.305 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:13:07.305 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.305 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.305 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:13:07.306 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.718389+0000", 2023-10-01T11:13:07.306 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.999999 2023-10-01T11:13:07.306 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.306 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.306 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:13:07.306 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.718389+0000", 2023-10-01T11:13:07.306 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.4270000000000001e-05 2023-10-01T11:13:07.306 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.306 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.306 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:13:07.306 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.718433+0000", 2023-10-01T11:13:07.307 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.0979999999999996e-06 2023-10-01T11:13:07.307 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.307 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.307 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:13:07.307 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.718440+0000", 2023-10-01T11:13:07.307 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.3209999999999998e-06 2023-10-01T11:13:07.307 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.307 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.307 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:13:07.307 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.718444+0000", 2023-10-01T11:13:07.307 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0037657110000000001 2023-10-01T11:13:07.307 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.308 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.308 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.308 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.722210+0000", 2023-10-01T11:13:07.308 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.5519999999999999e-06 2023-10-01T11:13:07.308 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.308 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.308 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:13:07.308 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.722218+0000", 2023-10-01T11:13:07.308 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.6786882990000001 2023-10-01T11:13:07.308 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.309 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.309 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.309 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.400906+0000", 2023-10-01T11:13:07.309 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0059755779999999996 2023-10-01T11:13:07.309 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.309 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.309 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:13:07.309 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.406881+0000", 2023-10-01T11:13:07.309 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00937435 2023-10-01T11:13:07.309 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.309 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.309 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 4,5", 2023-10-01T11:13:07.310 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.416256+0000", 2023-10-01T11:13:07.310 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.054214811000000002 2023-10-01T11:13:07.310 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.310 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.310 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:13:07.310 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.470471+0000", 2023-10-01T11:13:07.310 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.081062844999999994 2023-10-01T11:13:07.310 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.310 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.310 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:13:07.310 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.551533+0000", 2023-10-01T11:13:07.311 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0051759470000000002 2023-10-01T11:13:07.311 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.311 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.311 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:13:07.311 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.556709+0000", 2023-10-01T11:13:07.311 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.002901121 2023-10-01T11:13:07.311 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.311 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.311 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:13:07.311 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.559610+0000", 2023-10-01T11:13:07.311 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.01283839 2023-10-01T11:13:07.312 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.312 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.312 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:13:07.312 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.572449+0000", 2023-10-01T11:13:07.312 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.854059522 2023-10-01T11:13:07.312 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:13:07.312 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:13:07.312 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:13:07.312 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.312 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.312 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:73751 13.7 13:ebbcd948:::benchmark_data_smithi078_202524_object73750:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e300)", 2023-10-01T11:13:07.313 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:03:07.718450+0000", 2023-10-01T11:13:07.313 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 599.55292969000004, 2023-10-01T11:13:07.313 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.8718124840000003, 2023-10-01T11:13:07.313 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:13:07.313 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:13:07.313 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:13:07.313 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:13:07.313 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:13:07.313 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 73751 2023-10-01T11:13:07.313 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.313 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:13:07.313 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.313 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:13:07.314 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.718450+0000", 2023-10-01T11:13:07.314 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:13:07.314 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.314 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.314 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:13:07.314 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.718450+0000", 2023-10-01T11:13:07.314 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999995 2023-10-01T11:13:07.314 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.314 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.314 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:13:07.314 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.718449+0000", 2023-10-01T11:13:07.315 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.6961e-05 2023-10-01T11:13:07.315 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.315 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.315 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:13:07.315 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.718496+0000", 2023-10-01T11:13:07.315 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.9619999999999996e-06 2023-10-01T11:13:07.315 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.315 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.315 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:13:07.315 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.718503+0000", 2023-10-01T11:13:07.315 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.313e-06 2023-10-01T11:13:07.315 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.316 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.316 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:13:07.316 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.718507+0000", 2023-10-01T11:13:07.316 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0037338689999999999 2023-10-01T11:13:07.316 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.316 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.316 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.316 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.722241+0000", 2023-10-01T11:13:07.316 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.6210000000000004e-06 2023-10-01T11:13:07.316 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.316 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.317 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:13:07.317 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.722249+0000", 2023-10-01T11:13:07.317 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.7013689640000003 2023-10-01T11:13:07.317 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.317 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.317 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.317 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.423618+0000", 2023-10-01T11:13:07.317 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0067631710000000001 2023-10-01T11:13:07.317 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.317 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.317 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:13:07.318 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.430381+0000", 2023-10-01T11:13:07.318 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0096460780000000006 2023-10-01T11:13:07.318 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.318 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.318 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 4,5", 2023-10-01T11:13:07.318 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.440027+0000", 2023-10-01T11:13:07.318 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.030883622999999999 2023-10-01T11:13:07.318 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.318 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.318 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:13:07.318 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.470911+0000", 2023-10-01T11:13:07.319 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.104189745 2023-10-01T11:13:07.319 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.319 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.319 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:13:07.319 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.575100+0000", 2023-10-01T11:13:07.319 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.001515034 2023-10-01T11:13:07.319 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.319 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.319 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:13:07.319 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.576615+0000", 2023-10-01T11:13:07.319 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0022963380000000002 2023-10-01T11:13:07.320 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.320 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.320 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:13:07.320 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.578912+0000", 2023-10-01T11:13:07.320 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.011350442000000001 2023-10-01T11:13:07.320 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.320 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.320 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:13:07.320 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.590262+0000", 2023-10-01T11:13:07.320 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.8718124840000003 2023-10-01T11:13:07.320 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:13:07.320 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:13:07.321 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:13:07.321 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.321 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.321 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:73761 13.7 13:eea119d2:::benchmark_data_smithi078_202524_object73760:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected e300)", 2023-10-01T11:13:07.321 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:03:07.718513+0000", 2023-10-01T11:13:07.321 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 599.55286661000002, 2023-10-01T11:13:07.321 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.8908124619999995, 2023-10-01T11:13:07.321 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:13:07.321 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:13:07.321 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:13:07.321 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:13:07.321 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:13:07.322 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 73761 2023-10-01T11:13:07.322 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.322 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:13:07.322 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.322 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:13:07.322 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.718513+0000", 2023-10-01T11:13:07.322 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:13:07.322 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.322 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.322 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:13:07.322 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.718513+0000", 2023-10-01T11:13:07.323 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999995 2023-10-01T11:13:07.323 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.323 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.323 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:13:07.323 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.718512+0000", 2023-10-01T11:13:07.323 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.9983e-05 2023-10-01T11:13:07.323 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.323 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.323 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:13:07.323 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.718552+0000", 2023-10-01T11:13:07.323 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.8539999999999999e-06 2023-10-01T11:13:07.323 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.324 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.324 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:13:07.324 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.718559+0000", 2023-10-01T11:13:07.324 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.6020000000000002e-06 2023-10-01T11:13:07.324 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.324 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.324 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:13:07.324 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.718564+0000", 2023-10-01T11:13:07.324 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0037092290000000001 2023-10-01T11:13:07.324 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.324 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.325 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.325 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.722273+0000", 2023-10-01T11:13:07.325 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.8550000000000002e-06 2023-10-01T11:13:07.325 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.325 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.325 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:13:07.325 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:07.722281+0000", 2023-10-01T11:13:07.325 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.7253381279999997 2023-10-01T11:13:07.325 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.325 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.326 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.326 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.447619+0000", 2023-10-01T11:13:07.326 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0054270259999999997 2023-10-01T11:13:07.326 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.326 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.326 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:13:07.326 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.453046+0000", 2023-10-01T11:13:07.326 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.010633814 2023-10-01T11:13:07.326 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.326 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.327 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 4,5", 2023-10-01T11:13:07.327 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.463680+0000", 2023-10-01T11:13:07.327 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0081409099999999995 2023-10-01T11:13:07.327 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.327 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.327 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:13:07.327 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.471821+0000", 2023-10-01T11:13:07.327 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.121571225 2023-10-01T11:13:07.327 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.327 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.327 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:13:07.328 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.593392+0000", 2023-10-01T11:13:07.328 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0015781370000000001 2023-10-01T11:13:07.328 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.328 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.328 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:13:07.328 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.594970+0000", 2023-10-01T11:13:07.328 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0026622289999999999 2023-10-01T11:13:07.328 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.328 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.328 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:13:07.328 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.597632+0000", 2023-10-01T11:13:07.329 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.011693089 2023-10-01T11:13:07.329 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.329 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.329 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:13:07.329 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.609325+0000", 2023-10-01T11:13:07.329 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.8908124619999995 2023-10-01T11:13:07.329 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:13:07.329 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:13:07.329 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:13:07.329 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.329 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.330 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:73769 13.7 13:e97352b3:::benchmark_data_smithi078_202524_object73768:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e301)", 2023-10-01T11:13:07.330 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:03:09.066478+0000", 2023-10-01T11:13:07.330 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 598.204901791, 2023-10-01T11:13:07.330 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.6166157490000002, 2023-10-01T11:13:07.330 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:13:07.330 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:13:07.330 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:13:07.330 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:13:07.330 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:13:07.330 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 73769 2023-10-01T11:13:07.330 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.331 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:13:07.331 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.331 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:13:07.331 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:09.066478+0000", 2023-10-01T11:13:07.331 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:13:07.331 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.331 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.331 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:13:07.331 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:09.066478+0000", 2023-10-01T11:13:07.331 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999962 2023-10-01T11:13:07.331 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.331 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.332 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:13:07.332 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:09.066474+0000", 2023-10-01T11:13:07.332 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.000240323 2023-10-01T11:13:07.332 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.332 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.332 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:13:07.332 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:09.066714+0000", 2023-10-01T11:13:07.332 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.7214000000000001e-05 2023-10-01T11:13:07.332 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.332 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.332 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:13:07.333 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:09.066741+0000", 2023-10-01T11:13:07.333 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4734e-05 2023-10-01T11:13:07.333 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.333 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.333 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:13:07.333 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:09.066756+0000", 2023-10-01T11:13:07.333 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.5470384570000002 2023-10-01T11:13:07.333 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.333 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.333 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.333 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.613795+0000", 2023-10-01T11:13:07.334 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0062862300000000003 2023-10-01T11:13:07.334 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.334 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.334 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:13:07.334 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.620081+0000", 2023-10-01T11:13:07.334 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0097349370000000008 2023-10-01T11:13:07.334 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.334 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.334 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 4,5", 2023-10-01T11:13:07.334 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.629816+0000", 2023-10-01T11:13:07.334 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.035985132000000003 2023-10-01T11:13:07.335 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.335 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.335 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:13:07.335 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.665801+0000", 2023-10-01T11:13:07.335 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00065191399999999997 2023-10-01T11:13:07.335 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.335 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.335 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:13:07.335 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.666453+0000", 2023-10-01T11:13:07.335 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.001957164 2023-10-01T11:13:07.335 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.335 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.336 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:13:07.336 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.668410+0000", 2023-10-01T11:13:07.336 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.003187783 2023-10-01T11:13:07.336 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.336 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.336 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:13:07.336 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.671598+0000", 2023-10-01T11:13:07.336 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.011495613 2023-10-01T11:13:07.336 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.336 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.336 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:13:07.336 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.683093+0000", 2023-10-01T11:13:07.337 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.6166157490000002 2023-10-01T11:13:07.337 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:13:07.337 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:13:07.337 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:13:07.337 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.337 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.337 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:73773 13.7 13:e5ce5740:::benchmark_data_smithi078_202524_object73772:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e301)", 2023-10-01T11:13:07.337 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:03:09.086022+0000", 2023-10-01T11:13:07.337 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 598.185357146, 2023-10-01T11:13:07.337 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.6190876540000003, 2023-10-01T11:13:07.337 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:13:07.337 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:13:07.338 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:13:07.338 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:13:07.338 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:13:07.338 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 73773 2023-10-01T11:13:07.338 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.338 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:13:07.338 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.338 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:13:07.338 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:09.086022+0000", 2023-10-01T11:13:07.338 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:13:07.338 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.339 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.339 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:13:07.339 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:09.086022+0000", 2023-10-01T11:13:07.339 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999976 2023-10-01T11:13:07.339 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.339 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.339 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:13:07.339 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:09.086020+0000", 2023-10-01T11:13:07.339 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.7271999999999998e-05 2023-10-01T11:13:07.339 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.339 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.339 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:13:07.340 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:09.086057+0000", 2023-10-01T11:13:07.340 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.8853999999999999e-05 2023-10-01T11:13:07.340 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.340 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.340 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:13:07.340 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:09.086076+0000", 2023-10-01T11:13:07.340 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.1170000000000004e-06 2023-10-01T11:13:07.340 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.340 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.340 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:13:07.340 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:09.086084+0000", 2023-10-01T11:13:07.341 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.5514102870000004 2023-10-01T11:13:07.341 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.341 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.341 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.341 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.637494+0000", 2023-10-01T11:13:07.341 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0075558559999999997 2023-10-01T11:13:07.341 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.341 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.341 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:13:07.341 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.645050+0000", 2023-10-01T11:13:07.341 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.011704786 2023-10-01T11:13:07.341 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.342 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.342 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 4,5", 2023-10-01T11:13:07.342 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.656755+0000", 2023-10-01T11:13:07.342 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.026778924999999999 2023-10-01T11:13:07.342 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.342 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.342 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:13:07.342 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.683534+0000", 2023-10-01T11:13:07.342 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0013171280000000001 2023-10-01T11:13:07.342 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.342 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.343 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:13:07.343 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.684851+0000", 2023-10-01T11:13:07.343 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0021878449999999999 2023-10-01T11:13:07.343 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.343 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.343 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:13:07.343 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.687039+0000", 2023-10-01T11:13:07.343 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0031178260000000002 2023-10-01T11:13:07.343 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.343 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.343 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:13:07.343 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.690157+0000", 2023-10-01T11:13:07.344 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.014953331 2023-10-01T11:13:07.344 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.344 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.344 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:13:07.344 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.705110+0000", 2023-10-01T11:13:07.344 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.6190876540000003 2023-10-01T11:13:07.344 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:13:07.344 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:13:07.344 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:13:07.344 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.344 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.345 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:73795 13.7 13:ea704582:::benchmark_data_smithi078_202524_object73794:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e301)", 2023-10-01T11:13:07.345 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:03:09.165663+0000", 2023-10-01T11:13:07.345 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 598.10571605999996, 2023-10-01T11:13:07.345 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.6324160800000005, 2023-10-01T11:13:07.345 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:13:07.345 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:13:07.345 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:13:07.345 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:13:07.345 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:13:07.345 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 73795 2023-10-01T11:13:07.346 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.346 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:13:07.346 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.346 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:13:07.346 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:09.165663+0000", 2023-10-01T11:13:07.346 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:13:07.346 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.346 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.346 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:13:07.346 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:09.165663+0000", 2023-10-01T11:13:07.346 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999976 2023-10-01T11:13:07.347 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.347 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.347 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:13:07.347 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:09.165661+0000", 2023-10-01T11:13:07.347 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.2100999999999994e-05 2023-10-01T11:13:07.347 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.347 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.347 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:13:07.347 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:09.165723+0000", 2023-10-01T11:13:07.347 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.3499999999999999e-05 2023-10-01T11:13:07.347 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.347 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.348 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:13:07.348 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:09.165746+0000", 2023-10-01T11:13:07.348 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1450999999999999e-05 2023-10-01T11:13:07.348 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.348 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.348 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:13:07.348 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:09.165758+0000", 2023-10-01T11:13:07.348 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.5401505379999998 2023-10-01T11:13:07.348 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.348 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.348 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.348 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.705908+0000", 2023-10-01T11:13:07.349 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0081130030000000006 2023-10-01T11:13:07.349 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.349 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.349 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:13:07.349 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.714021+0000", 2023-10-01T11:13:07.349 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.009959318 2023-10-01T11:13:07.349 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.349 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.349 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 4,5", 2023-10-01T11:13:07.349 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.723981+0000", 2023-10-01T11:13:07.349 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.032189218999999998 2023-10-01T11:13:07.349 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.349 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.350 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:13:07.350 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.756170+0000", 2023-10-01T11:13:07.350 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.001291859 2023-10-01T11:13:07.350 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.350 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.350 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:13:07.350 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.757462+0000", 2023-10-01T11:13:07.350 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.028351995000000001 2023-10-01T11:13:07.350 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.350 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.350 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:13:07.351 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.785814+0000", 2023-10-01T11:13:07.351 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.002180431 2023-10-01T11:13:07.351 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.351 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.351 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:13:07.351 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.787994+0000", 2023-10-01T11:13:07.351 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.010085237 2023-10-01T11:13:07.351 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.351 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.351 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:13:07.351 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.798079+0000", 2023-10-01T11:13:07.352 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.6324160800000005 2023-10-01T11:13:07.352 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:13:07.352 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:13:07.352 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:13:07.352 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.352 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.352 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:73797 13.7 13:e3b312e8:::benchmark_data_smithi078_202524_object73796:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected e301)", 2023-10-01T11:13:07.352 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:03:09.175130+0000", 2023-10-01T11:13:07.352 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 598.09624928400001, 2023-10-01T11:13:07.352 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.6390158150000005, 2023-10-01T11:13:07.352 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:13:07.352 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:13:07.353 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:13:07.353 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:13:07.353 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:13:07.353 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 73797 2023-10-01T11:13:07.353 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.353 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:13:07.353 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.353 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:13:07.353 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:09.175130+0000", 2023-10-01T11:13:07.353 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:13:07.354 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.354 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.354 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:13:07.354 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:09.175130+0000", 2023-10-01T11:13:07.354 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999976 2023-10-01T11:13:07.354 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.354 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.354 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:13:07.354 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:09.175128+0000", 2023-10-01T11:13:07.354 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.3798000000000002e-05 2023-10-01T11:13:07.354 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.354 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.355 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:13:07.355 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:09.175182+0000", 2023-10-01T11:13:07.355 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.5231e-05 2023-10-01T11:13:07.355 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.355 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.355 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:13:07.355 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:09.175197+0000", 2023-10-01T11:13:07.355 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4278e-05 2023-10-01T11:13:07.355 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.355 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.355 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:13:07.356 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:09.175211+0000", 2023-10-01T11:13:07.356 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.5562459100000003 2023-10-01T11:13:07.356 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.356 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.356 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.356 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.731457+0000", 2023-10-01T11:13:07.356 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0084576970000000001 2023-10-01T11:13:07.356 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.356 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.356 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:13:07.356 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.739915+0000", 2023-10-01T11:13:07.356 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0091604259999999993 2023-10-01T11:13:07.356 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.357 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.357 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 4,5", 2023-10-01T11:13:07.357 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.749075+0000", 2023-10-01T11:13:07.357 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0097581089999999992 2023-10-01T11:13:07.357 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.357 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.357 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:13:07.357 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.758833+0000", 2023-10-01T11:13:07.357 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0012867180000000001 2023-10-01T11:13:07.357 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.357 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.357 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:13:07.358 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.760120+0000", 2023-10-01T11:13:07.358 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.040166564000000002 2023-10-01T11:13:07.358 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.358 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.358 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:13:07.358 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.800287+0000", 2023-10-01T11:13:07.358 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0030138259999999998 2023-10-01T11:13:07.358 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.358 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.358 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:13:07.359 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.803300+0000", 2023-10-01T11:13:07.359 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.010845594 2023-10-01T11:13:07.359 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.359 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.359 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:13:07.359 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:03:13.814146+0000", 2023-10-01T11:13:07.359 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.6390158150000005 2023-10-01T11:13:07.359 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:13:07.359 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:13:07.359 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:13:07.359 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.359 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.360 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:83246 13.7 13:e97770b5:::benchmark_data_smithi078_202524_object3688:head [delete] snapc 0=[] ondisk+write+known_if_redirected e320)", 2023-10-01T11:13:07.360 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:04:55.539579+0000", 2023-10-01T11:13:07.360 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 491.73180061800002, 2023-10-01T11:13:07.360 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.3631668770000003, 2023-10-01T11:13:07.360 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:13:07.360 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:13:07.360 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:13:07.360 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:13:07.360 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:13:07.360 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 83246 2023-10-01T11:13:07.360 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.361 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:13:07.361 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.361 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:13:07.361 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:55.539579+0000", 2023-10-01T11:13:07.361 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:13:07.361 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.361 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.361 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:13:07.361 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:55.539579+0000", 2023-10-01T11:13:07.361 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999971 2023-10-01T11:13:07.362 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.362 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.362 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:13:07.362 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:55.539576+0000", 2023-10-01T11:13:07.362 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.8807000000000001e-05 2023-10-01T11:13:07.362 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.362 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.362 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:13:07.362 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:55.539595+0000", 2023-10-01T11:13:07.362 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.9374e-05 2023-10-01T11:13:07.362 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.363 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.363 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:13:07.363 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:55.539624+0000", 2023-10-01T11:13:07.363 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.3293e-05 2023-10-01T11:13:07.363 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.363 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.363 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:13:07.363 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:55.539637+0000", 2023-10-01T11:13:07.363 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0147323530000001 2023-10-01T11:13:07.363 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.363 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.363 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.364 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:56.554370+0000", 2023-10-01T11:13:07.364 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0061905279999999998 2023-10-01T11:13:07.364 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.364 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.364 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for missing object", 2023-10-01T11:13:07.364 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:56.560560+0000", 2023-10-01T11:13:07.364 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.77892238199999997 2023-10-01T11:13:07.364 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.364 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.364 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.364 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:57.339482+0000", 2023-10-01T11:13:07.364 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0044609849999999998 2023-10-01T11:13:07.365 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.365 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.365 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for degraded object", 2023-10-01T11:13:07.365 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:57.343943+0000", 2023-10-01T11:13:07.365 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.3447326429999999 2023-10-01T11:13:07.365 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.365 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.365 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.365 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:59.688676+0000", 2023-10-01T11:13:07.365 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0059488020000000004 2023-10-01T11:13:07.366 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.366 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.366 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:13:07.366 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:59.694625+0000", 2023-10-01T11:13:07.366 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.012724657 2023-10-01T11:13:07.366 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.366 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.366 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 0,4,5", 2023-10-01T11:13:07.366 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:59.707350+0000", 2023-10-01T11:13:07.366 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.11274450799999999 2023-10-01T11:13:07.366 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.367 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.367 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:13:07.367 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:59.820094+0000", 2023-10-01T11:13:07.367 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.050915821 2023-10-01T11:13:07.367 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.367 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.367 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:13:07.367 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:59.871010+0000", 2023-10-01T11:13:07.367 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0070633570000000001 2023-10-01T11:13:07.367 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.367 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.367 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:13:07.368 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:59.878073+0000", 2023-10-01T11:13:07.368 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.001144087 2023-10-01T11:13:07.368 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.368 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.368 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:13:07.368 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:59.879217+0000", 2023-10-01T11:13:07.368 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0042778069999999998 2023-10-01T11:13:07.368 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.368 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.368 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:13:07.368 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:59.883495+0000", 2023-10-01T11:13:07.369 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.01925054 2023-10-01T11:13:07.369 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.369 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.369 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:13:07.369 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:59.902746+0000", 2023-10-01T11:13:07.369 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.3631668770000003 2023-10-01T11:13:07.369 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:13:07.369 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:13:07.369 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:13:07.369 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.369 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.370 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:83726 13.7 13:e30fefc9:::benchmark_data_smithi078_202524_object4168:head [delete] snapc 0=[] ondisk+write+known_if_redirected e321)", 2023-10-01T11:13:07.370 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:04:59.928121+0000", 2023-10-01T11:13:07.370 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 487.34325833399998, 2023-10-01T11:13:07.370 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.6283822749999999, 2023-10-01T11:13:07.370 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:13:07.370 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:13:07.370 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:13:07.370 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:13:07.370 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:13:07.370 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 83726 2023-10-01T11:13:07.371 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.371 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:13:07.371 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.371 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:13:07.371 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:59.928121+0000", 2023-10-01T11:13:07.371 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:13:07.371 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.371 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.371 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:13:07.371 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:59.928121+0000", 2023-10-01T11:13:07.372 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999976 2023-10-01T11:13:07.372 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.372 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.372 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:13:07.372 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:59.928119+0000", 2023-10-01T11:13:07.372 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0963e-05 2023-10-01T11:13:07.372 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.372 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.372 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:13:07.372 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:59.928130+0000", 2023-10-01T11:13:07.372 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.6269e-05 2023-10-01T11:13:07.373 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.373 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.373 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:13:07.373 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:59.928146+0000", 2023-10-01T11:13:07.373 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.7320000000000004e-06 2023-10-01T11:13:07.373 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.373 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.373 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:13:07.373 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:04:59.928155+0000", 2023-10-01T11:13:07.373 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.83608609300000003 2023-10-01T11:13:07.374 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.374 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.374 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.374 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:00.764241+0000", 2023-10-01T11:13:07.374 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.00493239 2023-10-01T11:13:07.374 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.374 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.374 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for missing object", 2023-10-01T11:13:07.374 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:00.769173+0000", 2023-10-01T11:13:07.374 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.836454166 2023-10-01T11:13:07.374 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.375 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.375 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.375 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:01.605627+0000", 2023-10-01T11:13:07.375 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0087184580000000001 2023-10-01T11:13:07.375 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.375 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.375 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for degraded object", 2023-10-01T11:13:07.375 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:01.614346+0000", 2023-10-01T11:13:07.375 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.7623074870000002 2023-10-01T11:13:07.375 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.375 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.376 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.376 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:04.376653+0000", 2023-10-01T11:13:07.376 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0065900769999999997 2023-10-01T11:13:07.376 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.376 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.376 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:13:07.376 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:04.383243+0000", 2023-10-01T11:13:07.376 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.012958179 2023-10-01T11:13:07.376 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.376 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.377 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 0,4,5", 2023-10-01T11:13:07.377 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:04.396202+0000", 2023-10-01T11:13:07.377 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.13163254699999999 2023-10-01T11:13:07.377 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.377 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.377 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:13:07.377 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:04.527834+0000", 2023-10-01T11:13:07.377 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0027133159999999999 2023-10-01T11:13:07.377 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.377 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.378 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:13:07.378 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:04.530547+0000", 2023-10-01T11:13:07.378 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0022698739999999999 2023-10-01T11:13:07.378 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.378 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.378 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:13:07.378 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:04.532817+0000", 2023-10-01T11:13:07.378 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.002300037 2023-10-01T11:13:07.378 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.378 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.379 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:13:07.379 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:04.535117+0000", 2023-10-01T11:13:07.379 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0037821479999999999 2023-10-01T11:13:07.379 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.379 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.379 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:13:07.379 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:04.538899+0000", 2023-10-01T11:13:07.379 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.017603931 2023-10-01T11:13:07.379 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.379 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.379 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:13:07.380 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:04.556503+0000", 2023-10-01T11:13:07.380 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4.6283822749999999 2023-10-01T11:13:07.380 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:13:07.380 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:13:07.380 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:13:07.380 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.380 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.380 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:86615 13.7 13:ead64f58:::benchmark_data_smithi078_202524_object7057:head [delete] snapc 0=[] ondisk+write+known_if_redirected e322)", 2023-10-01T11:13:07.380 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:05:30.140672+0000", 2023-10-01T11:13:07.380 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 457.13070767800002, 2023-10-01T11:13:07.381 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 14.388605118999999, 2023-10-01T11:13:07.381 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:13:07.381 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:13:07.381 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:13:07.381 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:13:07.381 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:13:07.381 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 86615 2023-10-01T11:13:07.381 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.381 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:13:07.381 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.381 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:13:07.381 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:30.140672+0000", 2023-10-01T11:13:07.382 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:13:07.382 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.382 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.382 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:13:07.382 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:30.140672+0000", 2023-10-01T11:13:07.382 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999976 2023-10-01T11:13:07.382 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.382 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.382 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:13:07.382 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:30.140670+0000", 2023-10-01T11:13:07.382 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1436e-05 2023-10-01T11:13:07.383 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.383 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.383 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:13:07.383 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:30.140681+0000", 2023-10-01T11:13:07.383 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.6262000000000001e-05 2023-10-01T11:13:07.383 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.383 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.383 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:13:07.383 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:30.140697+0000", 2023-10-01T11:13:07.383 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.5129999999999996e-06 2023-10-01T11:13:07.383 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.383 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.384 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:13:07.384 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:30.140706+0000", 2023-10-01T11:13:07.384 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.62280957699999995 2023-10-01T11:13:07.384 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.384 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.384 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.384 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:30.763515+0000", 2023-10-01T11:13:07.384 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0036732969999999998 2023-10-01T11:13:07.384 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.384 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.384 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for missing object", 2023-10-01T11:13:07.384 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:30.767189+0000", 2023-10-01T11:13:07.385 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.77364833200000005 2023-10-01T11:13:07.385 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.385 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.385 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.385 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:31.540837+0000", 2023-10-01T11:13:07.385 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0054527059999999999 2023-10-01T11:13:07.385 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.385 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.385 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for degraded object", 2023-10-01T11:13:07.385 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:31.546290+0000", 2023-10-01T11:13:07.385 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.328846092 2023-10-01T11:13:07.386 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.386 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.386 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.386 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:32.875136+0000", 2023-10-01T11:13:07.386 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1715e-05 2023-10-01T11:13:07.386 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.386 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.386 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:13:07.386 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:32.875147+0000", 2023-10-01T11:13:07.386 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.903379186 2023-10-01T11:13:07.386 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.387 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.387 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.387 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:33.778527+0000", 2023-10-01T11:13:07.387 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0341e-05 2023-10-01T11:13:07.387 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.387 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.387 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:13:07.387 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:33.778537+0000", 2023-10-01T11:13:07.387 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.8002640249999997 2023-10-01T11:13:07.387 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.387 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.387 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.388 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:40.578801+0000", 2023-10-01T11:13:07.388 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.008280819 2023-10-01T11:13:07.388 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.388 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.388 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:13:07.388 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:40.587082+0000", 2023-10-01T11:13:07.388 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0095164559999999995 2023-10-01T11:13:07.388 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.388 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.388 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 4,5,6", 2023-10-01T11:13:07.388 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:40.596598+0000", 2023-10-01T11:13:07.388 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.050241870000000001 2023-10-01T11:13:07.389 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.389 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.389 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:13:07.389 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:40.646840+0000", 2023-10-01T11:13:07.389 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.865497548 2023-10-01T11:13:07.389 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.389 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.389 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:13:07.389 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:44.512338+0000", 2023-10-01T11:13:07.389 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0018291080000000001 2023-10-01T11:13:07.389 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.390 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.390 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:13:07.390 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:44.514167+0000", 2023-10-01T11:13:07.390 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.001603908 2023-10-01T11:13:07.390 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.390 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.390 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:13:07.390 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:44.515771+0000", 2023-10-01T11:13:07.390 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.002782854 2023-10-01T11:13:07.390 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.390 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.391 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:13:07.391 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:44.518554+0000", 2023-10-01T11:13:07.391 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.01072329 2023-10-01T11:13:07.391 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.391 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.391 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:13:07.391 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:44.529277+0000", 2023-10-01T11:13:07.391 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 14.388605118999999 2023-10-01T11:13:07.393 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:13:07.393 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:13:07.393 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:13:07.393 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.393 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.393 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:86646 13.7 13:eebf072d:::benchmark_data_smithi078_202524_object7088:head [delete] snapc 0=[] ondisk+write+known_if_redirected e322)", 2023-10-01T11:13:07.393 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:05:30.975913+0000", 2023-10-01T11:13:07.394 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 456.29546668099999, 2023-10-01T11:13:07.394 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 13.493478633, 2023-10-01T11:13:07.394 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:13:07.394 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:13:07.394 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:13:07.394 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:13:07.394 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:13:07.394 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 86646 2023-10-01T11:13:07.394 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.394 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:13:07.394 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.395 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:13:07.395 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:30.975913+0000", 2023-10-01T11:13:07.395 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:13:07.395 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.395 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.395 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:13:07.395 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:30.975913+0000", 2023-10-01T11:13:07.395 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-10-01T11:13:07.395 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.395 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.395 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:13:07.396 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:30.975911+0000", 2023-10-01T11:13:07.396 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.5519999999999993e-06 2023-10-01T11:13:07.396 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.396 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.396 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:13:07.396 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:30.975920+0000", 2023-10-01T11:13:07.396 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.5242000000000001e-05 2023-10-01T11:13:07.396 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.396 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.396 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:13:07.396 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:30.975936+0000", 2023-10-01T11:13:07.396 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.7090000000000002e-06 2023-10-01T11:13:07.397 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.397 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.397 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:13:07.397 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:30.975943+0000", 2023-10-01T11:13:07.397 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.58928372699999998 2023-10-01T11:13:07.397 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.397 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.397 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.397 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:31.565227+0000", 2023-10-01T11:13:07.397 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0073614029999999999 2023-10-01T11:13:07.397 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.398 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.398 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for missing object", 2023-10-01T11:13:07.398 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:31.572589+0000", 2023-10-01T11:13:07.398 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.097891222999999999 2023-10-01T11:13:07.398 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.398 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.398 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.398 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:31.670480+0000", 2023-10-01T11:13:07.398 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0046611739999999997 2023-10-01T11:13:07.398 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.399 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.399 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for degraded object", 2023-10-01T11:13:07.399 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:31.675141+0000", 2023-10-01T11:13:07.399 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.199742466 2023-10-01T11:13:07.399 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.399 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.399 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.399 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:32.874883+0000", 2023-10-01T11:13:07.399 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.1440000000000001e-05 2023-10-01T11:13:07.399 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.399 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.399 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:13:07.400 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:32.874905+0000", 2023-10-01T11:13:07.400 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.90349204699999996 2023-10-01T11:13:07.400 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.400 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.400 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.400 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:33.778397+0000", 2023-10-01T11:13:07.400 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.7224999999999999e-05 2023-10-01T11:13:07.400 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.400 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.400 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:13:07.400 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:33.778414+0000", 2023-10-01T11:13:07.401 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.7314305340000002 2023-10-01T11:13:07.401 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.401 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.401 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.401 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:40.509845+0000", 2023-10-01T11:13:07.401 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0079409679999999996 2023-10-01T11:13:07.401 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.401 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.401 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:13:07.401 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:40.517786+0000", 2023-10-01T11:13:07.401 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0098886890000000009 2023-10-01T11:13:07.402 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.402 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.402 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 4,5,6", 2023-10-01T11:13:07.402 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:40.527674+0000", 2023-10-01T11:13:07.402 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.076099100000000003 2023-10-01T11:13:07.402 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.402 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.402 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:13:07.402 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:40.603773+0000", 2023-10-01T11:13:07.402 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.8486558290000001 2023-10-01T11:13:07.402 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.403 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.403 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:13:07.403 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:44.452429+0000", 2023-10-01T11:13:07.403 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0014831460000000001 2023-10-01T11:13:07.403 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.403 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.403 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:13:07.403 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:44.453912+0000", 2023-10-01T11:13:07.403 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.001504975 2023-10-01T11:13:07.403 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.403 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.404 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:13:07.404 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:44.455417+0000", 2023-10-01T11:13:07.404 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0022718360000000002 2023-10-01T11:13:07.404 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.404 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.404 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:13:07.404 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:44.457689+0000", 2023-10-01T11:13:07.404 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.011702198 2023-10-01T11:13:07.404 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.404 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.405 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:13:07.405 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:44.469391+0000", 2023-10-01T11:13:07.405 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 13.493478633 2023-10-01T11:13:07.405 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:13:07.405 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:13:07.405 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:13:07.405 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.405 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.405 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:86662 13.7 13:e284dbb5:::benchmark_data_smithi078_202524_object7104:head [delete] snapc 0=[] ondisk+write+known_if_redirected e322)", 2023-10-01T11:13:07.405 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:05:31.035224+0000", 2023-10-01T11:13:07.406 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 456.236155242, 2023-10-01T11:13:07.406 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 13.571343387000001, 2023-10-01T11:13:07.406 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:13:07.406 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:13:07.406 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:13:07.406 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:13:07.406 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:13:07.406 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 86662 2023-10-01T11:13:07.406 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.406 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:13:07.406 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.407 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:13:07.407 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:31.035224+0000", 2023-10-01T11:13:07.407 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:13:07.407 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.407 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.407 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:13:07.407 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:31.035224+0000", 2023-10-01T11:13:07.407 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-10-01T11:13:07.407 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.407 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.408 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:13:07.408 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:31.035222+0000", 2023-10-01T11:13:07.408 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.3919999999999992e-06 2023-10-01T11:13:07.408 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.408 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.408 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:13:07.408 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:31.035232+0000", 2023-10-01T11:13:07.408 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4538999999999999e-05 2023-10-01T11:13:07.408 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.408 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.408 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:13:07.408 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:31.035246+0000", 2023-10-01T11:13:07.409 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 7.8669999999999994e-06 2023-10-01T11:13:07.409 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.409 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.409 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:13:07.409 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:31.035254+0000", 2023-10-01T11:13:07.409 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.53814489399999998 2023-10-01T11:13:07.409 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.409 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.409 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.409 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:31.573399+0000", 2023-10-01T11:13:07.409 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0072359169999999997 2023-10-01T11:13:07.409 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.410 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.410 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for missing object", 2023-10-01T11:13:07.410 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:31.580635+0000", 2023-10-01T11:13:07.410 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.126385885 2023-10-01T11:13:07.410 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.410 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.410 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.410 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:31.707021+0000", 2023-10-01T11:13:07.410 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0043882749999999996 2023-10-01T11:13:07.410 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.410 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.411 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for degraded object", 2023-10-01T11:13:07.411 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:31.711409+0000", 2023-10-01T11:13:07.411 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.16392809 2023-10-01T11:13:07.411 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.411 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.411 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.411 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:32.875337+0000", 2023-10-01T11:13:07.411 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4117999999999999e-05 2023-10-01T11:13:07.411 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.411 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.411 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:13:07.411 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:32.875351+0000", 2023-10-01T11:13:07.412 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.90334182500000004 2023-10-01T11:13:07.412 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.412 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.412 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.412 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:33.778693+0000", 2023-10-01T11:13:07.412 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2795e-05 2023-10-01T11:13:07.412 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.412 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.412 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:13:07.412 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:33.778706+0000", 2023-10-01T11:13:07.412 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.8903261599999999 2023-10-01T11:13:07.413 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.413 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.413 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.413 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:40.669032+0000", 2023-10-01T11:13:07.413 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0048759670000000001 2023-10-01T11:13:07.413 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.413 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.413 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:13:07.413 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:40.673908+0000", 2023-10-01T11:13:07.413 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0074903449999999998 2023-10-01T11:13:07.413 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.413 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.414 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 4,5,6", 2023-10-01T11:13:07.414 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:40.681398+0000", 2023-10-01T11:13:07.414 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.011665429999999999 2023-10-01T11:13:07.414 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.414 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.414 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:13:07.414 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:40.693064+0000", 2023-10-01T11:13:07.414 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.8952961739999998 2023-10-01T11:13:07.414 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.414 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.414 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:13:07.414 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:44.588360+0000", 2023-10-01T11:13:07.415 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0019565659999999999 2023-10-01T11:13:07.415 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.415 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.415 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:13:07.415 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:44.590317+0000", 2023-10-01T11:13:07.415 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0014118080000000001 2023-10-01T11:13:07.415 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.415 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.415 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:13:07.415 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:44.591728+0000", 2023-10-01T11:13:07.415 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0025199710000000002 2023-10-01T11:13:07.415 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.416 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.416 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:13:07.416 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:44.594248+0000", 2023-10-01T11:13:07.416 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.012319251999999999 2023-10-01T11:13:07.416 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.416 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.416 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:13:07.416 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:44.606568+0000", 2023-10-01T11:13:07.416 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 13.571343387000001 2023-10-01T11:13:07.416 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:13:07.416 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:13:07.416 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:13:07.417 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.417 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.417 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:86685 13.7 13:ea6aa87c:::benchmark_data_smithi078_202524_object7127:head [delete] snapc 0=[] ondisk+write+known_if_redirected e322)", 2023-10-01T11:13:07.417 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:05:31.141059+0000", 2023-10-01T11:13:07.417 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 456.13032009199998, 2023-10-01T11:13:07.417 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 13.407561532000001, 2023-10-01T11:13:07.417 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:13:07.417 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:13:07.417 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:13:07.417 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:13:07.417 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:13:07.417 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 86685 2023-10-01T11:13:07.418 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.418 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:13:07.418 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.418 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:13:07.418 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:31.141059+0000", 2023-10-01T11:13:07.418 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:13:07.418 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.418 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.418 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:13:07.418 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:31.141059+0000", 2023-10-01T11:13:07.418 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999976 2023-10-01T11:13:07.418 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.419 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.419 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:13:07.419 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:31.141057+0000", 2023-10-01T11:13:07.419 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0761000000000001e-05 2023-10-01T11:13:07.419 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.419 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.419 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:13:07.419 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:31.141068+0000", 2023-10-01T11:13:07.419 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.7753000000000001e-05 2023-10-01T11:13:07.419 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.419 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.419 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:13:07.420 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:31.141086+0000", 2023-10-01T11:13:07.420 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0497e-05 2023-10-01T11:13:07.420 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.420 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.420 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:13:07.420 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:31.141096+0000", 2023-10-01T11:13:07.420 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.62369247100000003 2023-10-01T11:13:07.420 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.420 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.420 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.420 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:31.764789+0000", 2023-10-01T11:13:07.421 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0075553599999999997 2023-10-01T11:13:07.421 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.421 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.421 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for missing object", 2023-10-01T11:13:07.421 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:31.772344+0000", 2023-10-01T11:13:07.421 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.091291862000000001 2023-10-01T11:13:07.421 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.421 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.421 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.421 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:31.863636+0000", 2023-10-01T11:13:07.421 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0043329290000000001 2023-10-01T11:13:07.421 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.422 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.422 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for degraded object", 2023-10-01T11:13:07.422 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:31.867969+0000", 2023-10-01T11:13:07.422 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0072255429999999 2023-10-01T11:13:07.422 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.422 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.422 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.422 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:32.875194+0000", 2023-10-01T11:13:07.422 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2064e-05 2023-10-01T11:13:07.422 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.423 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.423 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:13:07.423 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:32.875206+0000", 2023-10-01T11:13:07.423 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.90335943699999999 2023-10-01T11:13:07.423 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.423 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.423 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.423 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:33.778566+0000", 2023-10-01T11:13:07.423 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.8039999999999996e-06 2023-10-01T11:13:07.423 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.423 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.423 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:13:07.424 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:33.778576+0000", 2023-10-01T11:13:07.424 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.8262578119999997 2023-10-01T11:13:07.424 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.424 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.424 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.424 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:40.604833+0000", 2023-10-01T11:13:07.424 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0047897069999999998 2023-10-01T11:13:07.424 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.424 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.424 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:13:07.424 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:40.609623+0000", 2023-10-01T11:13:07.424 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0071018549999999998 2023-10-01T11:13:07.425 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.425 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.425 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 4,5,6", 2023-10-01T11:13:07.425 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:40.616725+0000", 2023-10-01T11:13:07.425 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.051403620999999997 2023-10-01T11:13:07.425 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.425 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.425 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:13:07.425 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:40.668129+0000", 2023-10-01T11:13:07.425 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.8625854670000002 2023-10-01T11:13:07.425 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.426 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.426 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:13:07.426 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:44.530714+0000", 2023-10-01T11:13:07.426 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0020214410000000001 2023-10-01T11:13:07.426 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.426 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.426 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:13:07.426 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:44.532735+0000", 2023-10-01T11:13:07.426 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0014313749999999999 2023-10-01T11:13:07.426 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.426 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.427 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:13:07.427 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:44.534167+0000", 2023-10-01T11:13:07.427 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0031160670000000001 2023-10-01T11:13:07.427 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.427 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.427 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:13:07.427 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:44.537283+0000", 2023-10-01T11:13:07.427 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.011337942 2023-10-01T11:13:07.427 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.427 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.427 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:13:07.427 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:44.548621+0000", 2023-10-01T11:13:07.428 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 13.407561532000001 2023-10-01T11:13:07.428 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:13:07.428 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:13:07.428 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:13:07.428 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.428 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.428 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:86696 13.7 13:e3d5ae4a:::benchmark_data_smithi078_202524_object7138:head [delete] snapc 0=[] ondisk+write+known_if_redirected e322)", 2023-10-01T11:13:07.428 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:05:31.292893+0000", 2023-10-01T11:13:07.428 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 455.978486159, 2023-10-01T11:13:07.428 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 13.294007409000001, 2023-10-01T11:13:07.428 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:13:07.429 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:13:07.429 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:13:07.429 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:13:07.429 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:13:07.429 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 86696 2023-10-01T11:13:07.429 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.429 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:13:07.429 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.429 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:13:07.429 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:31.292893+0000", 2023-10-01T11:13:07.429 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:13:07.429 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.430 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.430 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:13:07.430 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:31.292893+0000", 2023-10-01T11:13:07.430 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-10-01T11:13:07.430 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.430 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.430 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:13:07.430 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:31.292891+0000", 2023-10-01T11:13:07.430 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1681999999999999e-05 2023-10-01T11:13:07.430 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.430 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.431 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:13:07.431 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:31.292903+0000", 2023-10-01T11:13:07.431 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.6076999999999999e-05 2023-10-01T11:13:07.431 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.431 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.431 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:13:07.431 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:31.292919+0000", 2023-10-01T11:13:07.431 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.0090000000000007e-06 2023-10-01T11:13:07.431 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.431 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.431 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:13:07.431 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:31.292927+0000", 2023-10-01T11:13:07.432 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.58868098700000004 2023-10-01T11:13:07.432 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.432 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.432 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.432 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:31.881608+0000", 2023-10-01T11:13:07.432 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0072607569999999996 2023-10-01T11:13:07.432 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.432 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.432 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for missing object", 2023-10-01T11:13:07.432 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:31.888869+0000", 2023-10-01T11:13:07.432 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.094290262999999999 2023-10-01T11:13:07.432 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.433 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.433 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.433 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:31.983159+0000", 2023-10-01T11:13:07.433 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0043375929999999998 2023-10-01T11:13:07.433 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.433 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.433 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for degraded object", 2023-10-01T11:13:07.433 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:31.987497+0000", 2023-10-01T11:13:07.433 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.88779457699999997 2023-10-01T11:13:07.433 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.433 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.434 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.434 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:32.875291+0000", 2023-10-01T11:13:07.434 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.9090000000000006e-06 2023-10-01T11:13:07.434 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.434 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.434 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:13:07.434 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:32.875300+0000", 2023-10-01T11:13:07.434 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.90334991099999995 2023-10-01T11:13:07.434 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.434 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.434 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.435 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:33.778650+0000", 2023-10-01T11:13:07.435 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.2904e-05 2023-10-01T11:13:07.435 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.435 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.435 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:13:07.435 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:33.778663+0000", 2023-10-01T11:13:07.435 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.8686369999999997 2023-10-01T11:13:07.435 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.435 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.435 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.435 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:40.647300+0000", 2023-10-01T11:13:07.435 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0058899269999999997 2023-10-01T11:13:07.436 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.436 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.436 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:13:07.436 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:40.653190+0000", 2023-10-01T11:13:07.436 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0075541239999999997 2023-10-01T11:13:07.436 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.436 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.436 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 4,5,6", 2023-10-01T11:13:07.436 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:40.660744+0000", 2023-10-01T11:13:07.436 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.026972561999999999 2023-10-01T11:13:07.436 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.436 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.437 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:13:07.437 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:40.687717+0000", 2023-10-01T11:13:07.437 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.880447089 2023-10-01T11:13:07.437 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.437 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.437 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:13:07.437 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:44.568164+0000", 2023-10-01T11:13:07.437 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0015400209999999999 2023-10-01T11:13:07.437 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.437 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.437 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:13:07.438 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:44.569704+0000", 2023-10-01T11:13:07.438 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0023502919999999999 2023-10-01T11:13:07.438 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.438 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.438 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:13:07.438 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:44.572054+0000", 2023-10-01T11:13:07.438 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0029631150000000001 2023-10-01T11:13:07.438 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.438 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.438 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:13:07.438 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:44.575017+0000", 2023-10-01T11:13:07.439 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.011883572 2023-10-01T11:13:07.439 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.439 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.439 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:13:07.439 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:44.586901+0000", 2023-10-01T11:13:07.439 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 13.294007409000001 2023-10-01T11:13:07.439 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:13:07.439 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:13:07.439 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:13:07.439 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.439 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.439 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:86703 13.7 13:e431ac7c:::benchmark_data_smithi078_202524_object7145:head [delete] snapc 0=[] ondisk+write+known_if_redirected e322)", 2023-10-01T11:13:07.440 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:05:31.326798+0000", 2023-10-01T11:13:07.440 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 455.94458123700002, 2023-10-01T11:13:07.440 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 13.239810693999999, 2023-10-01T11:13:07.440 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:13:07.440 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:13:07.440 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:13:07.440 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:13:07.440 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:13:07.440 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 86703 2023-10-01T11:13:07.440 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.440 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:13:07.441 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.441 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:13:07.441 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:31.326798+0000", 2023-10-01T11:13:07.441 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:13:07.441 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.441 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.441 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:13:07.441 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:31.326798+0000", 2023-10-01T11:13:07.441 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999986 2023-10-01T11:13:07.441 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.442 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.442 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:13:07.442 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:31.326797+0000", 2023-10-01T11:13:07.442 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.3610000000000007e-06 2023-10-01T11:13:07.442 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.442 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.442 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:13:07.442 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:31.326806+0000", 2023-10-01T11:13:07.442 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.3672000000000001e-05 2023-10-01T11:13:07.442 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.442 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.443 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:13:07.443 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:31.326820+0000", 2023-10-01T11:13:07.443 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 5.9649999999999996e-06 2023-10-01T11:13:07.443 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.443 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.443 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:13:07.443 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:31.326826+0000", 2023-10-01T11:13:07.443 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.56282140400000003 2023-10-01T11:13:07.443 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.443 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.443 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.443 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:31.889647+0000", 2023-10-01T11:13:07.444 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0071710580000000001 2023-10-01T11:13:07.444 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.444 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.444 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for missing object", 2023-10-01T11:13:07.444 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:31.896818+0000", 2023-10-01T11:13:07.444 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.11799372599999999 2023-10-01T11:13:07.444 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.444 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.444 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.444 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:32.014812+0000", 2023-10-01T11:13:07.444 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.004325906 2023-10-01T11:13:07.444 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.445 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.445 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for degraded object", 2023-10-01T11:13:07.445 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:32.019138+0000", 2023-10-01T11:13:07.445 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.85610778600000004 2023-10-01T11:13:07.445 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.445 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.445 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.445 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:32.875245+0000", 2023-10-01T11:13:07.445 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.7280000000000007e-06 2023-10-01T11:13:07.445 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.445 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.446 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:13:07.446 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:32.875255+0000", 2023-10-01T11:13:07.446 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.90334991799999997 2023-10-01T11:13:07.446 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.446 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.446 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.446 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:33.778605+0000", 2023-10-01T11:13:07.446 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.3151e-05 2023-10-01T11:13:07.446 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.446 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.446 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:13:07.446 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:33.778618+0000", 2023-10-01T11:13:07.447 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.8451439159999996 2023-10-01T11:13:07.447 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.447 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.447 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.447 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:40.623762+0000", 2023-10-01T11:13:07.447 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.006987727 2023-10-01T11:13:07.447 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.447 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.447 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:13:07.447 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:40.630750+0000", 2023-10-01T11:13:07.448 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0083540850000000007 2023-10-01T11:13:07.448 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.448 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.448 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 4,5,6", 2023-10-01T11:13:07.448 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:40.639104+0000", 2023-10-01T11:13:07.448 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.029460312999999998 2023-10-01T11:13:07.448 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.448 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.448 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:13:07.448 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:40.668564+0000", 2023-10-01T11:13:07.448 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.881846436 2023-10-01T11:13:07.448 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.448 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.449 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:13:07.449 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:44.550411+0000", 2023-10-01T11:13:07.449 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0016576710000000001 2023-10-01T11:13:07.449 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.449 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.449 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:13:07.449 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:44.552068+0000", 2023-10-01T11:13:07.449 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0014174050000000001 2023-10-01T11:13:07.449 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.449 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.449 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:13:07.449 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:44.553486+0000", 2023-10-01T11:13:07.450 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0023111849999999999 2023-10-01T11:13:07.450 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.450 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.450 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:13:07.450 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:44.555797+0000", 2023-10-01T11:13:07.450 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.010811932 2023-10-01T11:13:07.450 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.450 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.450 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:13:07.450 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:44.566609+0000", 2023-10-01T11:13:07.451 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 13.239810693999999 2023-10-01T11:13:07.451 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:13:07.451 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:13:07.451 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:13:07.451 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.451 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.451 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:86711 13.7 13:ed2385ac:::benchmark_data_smithi078_202524_object7153:head [delete] snapc 0=[] ondisk+write+known_if_redirected e322)", 2023-10-01T11:13:07.451 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:05:31.533982+0000", 2023-10-01T11:13:07.451 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 455.73739769299999, 2023-10-01T11:13:07.451 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 12.956297244, 2023-10-01T11:13:07.451 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:13:07.451 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:13:07.452 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:13:07.452 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:13:07.452 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:13:07.452 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 86711 2023-10-01T11:13:07.452 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.452 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:13:07.452 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.452 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:13:07.452 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:31.533982+0000", 2023-10-01T11:13:07.452 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:13:07.452 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.452 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.453 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:13:07.453 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:31.533982+0000", 2023-10-01T11:13:07.453 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999981 2023-10-01T11:13:07.453 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.453 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.453 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:13:07.453 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:31.533980+0000", 2023-10-01T11:13:07.453 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.1804000000000001e-05 2023-10-01T11:13:07.453 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.453 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.453 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:13:07.454 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:31.533992+0000", 2023-10-01T11:13:07.454 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.8580000000000002e-05 2023-10-01T11:13:07.454 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.454 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.454 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:13:07.454 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:31.534010+0000", 2023-10-01T11:13:07.454 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.4970000000000001e-06 2023-10-01T11:13:07.454 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.454 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.454 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:13:07.454 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:31.534019+0000", 2023-10-01T11:13:07.454 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.56321941499999995 2023-10-01T11:13:07.455 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.455 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.455 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.455 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:32.097238+0000", 2023-10-01T11:13:07.455 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.011057178000000001 2023-10-01T11:13:07.455 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.455 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.455 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for missing object", 2023-10-01T11:13:07.455 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:32.108295+0000", 2023-10-01T11:13:07.455 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.16429956800000001 2023-10-01T11:13:07.455 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.455 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.456 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.456 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:32.272595+0000", 2023-10-01T11:13:07.456 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0042883720000000004 2023-10-01T11:13:07.456 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.456 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.456 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for degraded object", 2023-10-01T11:13:07.456 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:32.276883+0000", 2023-10-01T11:13:07.456 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.59809309099999997 2023-10-01T11:13:07.456 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.456 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.456 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.457 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:32.874976+0000", 2023-10-01T11:13:07.457 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0589e-05 2023-10-01T11:13:07.457 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.457 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.457 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:13:07.457 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:32.874987+0000", 2023-10-01T11:13:07.457 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.90346108000000003 2023-10-01T11:13:07.457 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.457 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.457 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.457 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:33.778448+0000", 2023-10-01T11:13:07.458 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.0052e-05 2023-10-01T11:13:07.458 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.458 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.458 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:13:07.458 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:33.778458+0000", 2023-10-01T11:13:07.458 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.7564974859999998 2023-10-01T11:13:07.458 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.458 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.458 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.458 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:40.534956+0000", 2023-10-01T11:13:07.458 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0084769379999999998 2023-10-01T11:13:07.458 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.459 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.459 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:13:07.459 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:40.543432+0000", 2023-10-01T11:13:07.459 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0081610390000000001 2023-10-01T11:13:07.459 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.459 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.459 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 4,5,6", 2023-10-01T11:13:07.459 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:40.551594+0000", 2023-10-01T11:13:07.459 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.052620002999999999 2023-10-01T11:13:07.459 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.459 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.459 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:13:07.460 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:40.604214+0000", 2023-10-01T11:13:07.460 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.8688738790000001 2023-10-01T11:13:07.460 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.460 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.460 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:13:07.460 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:44.473087+0000", 2023-10-01T11:13:07.460 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0015026 2023-10-01T11:13:07.460 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.460 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.460 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:13:07.460 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:44.474590+0000", 2023-10-01T11:13:07.461 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0013584720000000001 2023-10-01T11:13:07.461 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.461 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.461 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:13:07.461 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:44.475948+0000", 2023-10-01T11:13:07.461 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.002972695 2023-10-01T11:13:07.461 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.461 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.461 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:13:07.461 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:44.478921+0000", 2023-10-01T11:13:07.461 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.011357785 2023-10-01T11:13:07.462 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.462 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.462 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:13:07.462 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:44.490279+0000", 2023-10-01T11:13:07.462 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 12.956297244 2023-10-01T11:13:07.462 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:13:07.462 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:13:07.462 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:13:07.462 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.462 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.462 INFO:teuthology.orchestra.run.smithi173.stdout: "description": "osd_op(client.46454.0:86731 13.7 13:eba259e6:::benchmark_data_smithi078_202524_object7173:head [delete] snapc 0=[] ondisk+write+known_if_redirected e322)", 2023-10-01T11:13:07.463 INFO:teuthology.orchestra.run.smithi173.stdout: "initiated_at": "2023-10-01T11:05:32.079898+0000", 2023-10-01T11:13:07.463 INFO:teuthology.orchestra.run.smithi173.stdout: "age": 455.19148132100003, 2023-10-01T11:13:07.463 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 12.430790841, 2023-10-01T11:13:07.463 INFO:teuthology.orchestra.run.smithi173.stdout: "type_data": { 2023-10-01T11:13:07.463 INFO:teuthology.orchestra.run.smithi173.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:13:07.463 INFO:teuthology.orchestra.run.smithi173.stdout: "client_info": { 2023-10-01T11:13:07.463 INFO:teuthology.orchestra.run.smithi173.stdout: "client": "client.46454", 2023-10-01T11:13:07.463 INFO:teuthology.orchestra.run.smithi173.stdout: "client_addr": "172.21.15.78:0/2230183457", 2023-10-01T11:13:07.463 INFO:teuthology.orchestra.run.smithi173.stdout: "tid": 86731 2023-10-01T11:13:07.463 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.463 INFO:teuthology.orchestra.run.smithi173.stdout: "events": [ 2023-10-01T11:13:07.464 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.464 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "initiated", 2023-10-01T11:13:07.464 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:32.079898+0000", 2023-10-01T11:13:07.464 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0 2023-10-01T11:13:07.464 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.464 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.464 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "throttled", 2023-10-01T11:13:07.464 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:32.079898+0000", 2023-10-01T11:13:07.464 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 4294967295.9999976 2023-10-01T11:13:07.464 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.464 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.464 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "header_read", 2023-10-01T11:13:07.464 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:32.079896+0000", 2023-10-01T11:13:07.465 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 1.4024e-05 2023-10-01T11:13:07.465 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.465 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.465 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "all_read", 2023-10-01T11:13:07.465 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:32.079910+0000", 2023-10-01T11:13:07.465 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 2.1471000000000002e-05 2023-10-01T11:13:07.465 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.465 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.465 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "dispatched", 2023-10-01T11:13:07.465 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:32.079931+0000", 2023-10-01T11:13:07.465 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 8.4379999999999995e-06 2023-10-01T11:13:07.466 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.466 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.466 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "queued_for_pg", 2023-10-01T11:13:07.466 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:32.079940+0000", 2023-10-01T11:13:07.466 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.639193758 2023-10-01T11:13:07.466 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.466 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.466 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.466 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:32.719133+0000", 2023-10-01T11:13:07.466 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.008022019 2023-10-01T11:13:07.466 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.466 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.467 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for missing object", 2023-10-01T11:13:07.467 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:32.727155+0000", 2023-10-01T11:13:07.467 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.084898662 2023-10-01T11:13:07.467 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.467 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.467 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.467 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:32.812054+0000", 2023-10-01T11:13:07.467 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0043227530000000004 2023-10-01T11:13:07.467 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.467 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.467 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for degraded object", 2023-10-01T11:13:07.467 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:32.816377+0000", 2023-10-01T11:13:07.468 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.058689674999999997 2023-10-01T11:13:07.468 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.468 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.468 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.468 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:32.875067+0000", 2023-10-01T11:13:07.468 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.8740000000000007e-06 2023-10-01T11:13:07.468 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.468 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.468 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:13:07.468 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:32.875076+0000", 2023-10-01T11:13:07.468 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.90341111100000004 2023-10-01T11:13:07.469 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.469 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.469 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.469 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:33.778488+0000", 2023-10-01T11:13:07.469 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 9.8179999999999999e-06 2023-10-01T11:13:07.469 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.469 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.469 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for peered", 2023-10-01T11:13:07.469 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:33.778497+0000", 2023-10-01T11:13:07.469 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 6.7800438310000004 2023-10-01T11:13:07.470 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.470 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.470 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "reached_pg", 2023-10-01T11:13:07.470 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:40.558541+0000", 2023-10-01T11:13:07.470 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0056168199999999998 2023-10-01T11:13:07.470 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.470 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.470 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "started", 2023-10-01T11:13:07.470 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:40.564158+0000", 2023-10-01T11:13:07.470 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0073831269999999997 2023-10-01T11:13:07.470 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.471 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.471 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "waiting for subops from 4,5,6", 2023-10-01T11:13:07.471 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:40.571541+0000", 2023-10-01T11:13:07.471 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.074866022000000004 2023-10-01T11:13:07.471 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.471 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.471 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "op_commit", 2023-10-01T11:13:07.471 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:40.646407+0000", 2023-10-01T11:13:07.471 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 3.8460276100000002 2023-10-01T11:13:07.471 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.471 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.471 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:13:07.472 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:44.492435+0000", 2023-10-01T11:13:07.472 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.002027198 2023-10-01T11:13:07.472 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.472 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.472 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:13:07.472 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:44.494462+0000", 2023-10-01T11:13:07.472 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.0020409949999999999 2023-10-01T11:13:07.472 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.472 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.472 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "sub_op_commit_rec", 2023-10-01T11:13:07.473 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:44.496503+0000", 2023-10-01T11:13:07.473 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.002851499 2023-10-01T11:13:07.473 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.473 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.473 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "commit_sent", 2023-10-01T11:13:07.473 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:44.499354+0000", 2023-10-01T11:13:07.473 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 0.011334479 2023-10-01T11:13:07.473 INFO:teuthology.orchestra.run.smithi173.stdout: }, 2023-10-01T11:13:07.473 INFO:teuthology.orchestra.run.smithi173.stdout: { 2023-10-01T11:13:07.473 INFO:teuthology.orchestra.run.smithi173.stdout: "event": "done", 2023-10-01T11:13:07.474 INFO:teuthology.orchestra.run.smithi173.stdout: "time": "2023-10-01T11:05:44.510689+0000", 2023-10-01T11:13:07.474 INFO:teuthology.orchestra.run.smithi173.stdout: "duration": 12.430790841 2023-10-01T11:13:07.474 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:13:07.474 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:13:07.474 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:13:07.474 INFO:teuthology.orchestra.run.smithi173.stdout: } 2023-10-01T11:13:07.474 INFO:teuthology.orchestra.run.smithi173.stdout: ] 2023-10-01T11:13:07.474 INFO:teuthology.orchestra.run.smithi173.stdout:} 2023-10-01T11:13:07.474 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:07.504 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:07.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:07.509 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:13:07.527 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:13:07.610 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:07.633 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:07.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:07.636 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:13:07.736 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:07.764 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:07.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:07.767 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:13:07.868 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:07.898 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:07.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:07.902 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:13:08.002 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:08.031 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:08.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:08.034 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:13:08.135 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:08.172 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:08.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:08.173 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:13:08.220 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:07 smithi078 bash[78274]: cluster 2023-10-01T11:13:06.456917+0000 mgr.y (mgr.25072) 1675 : cluster 0 pgmap v1809: 89 pgs: 89 active+clean; 1.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 636 op/s 2023-10-01T11:13:08.221 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:07 smithi078 bash[75299]: cluster 2023-10-01T11:13:06.456917+0000 mgr.y (mgr.25072) 1675 : cluster 0 pgmap v1809: 89 pgs: 89 active+clean; 1.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 636 op/s 2023-10-01T11:13:08.274 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:08.283 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:07 smithi173 bash[58816]: cluster 2023-10-01T11:13:06.456917+0000 mgr.y (mgr.25072) 1675 : cluster 0 pgmap v1809: 89 pgs: 89 active+clean; 1.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 636 op/s 2023-10-01T11:13:08.307 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:08.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:08.315 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:13:08.396 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-10-01T11:13:08.416 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:08.494 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:08.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:08.495 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:13:08.596 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:08.622 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:08.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:08.629 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:13:08.730 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:08.755 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:08.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:08.758 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:13:08.859 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:08.890 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:08.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:08.894 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:13:08.994 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:09.031 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:09.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:09.035 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:13:09.138 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:09.165 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:09.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:09.169 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:13:09.270 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:09.295 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:09.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:09.299 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:13:09.400 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:09.427 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:09.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:09.430 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:13:09.531 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:09.578 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:09.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:09.580 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:13:09.681 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:09.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:09.718 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:09.719 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:13:09.819 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:09.854 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:09.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:09.857 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:13:09.958 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:09.984 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:09.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:09.987 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:13:10.088 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:10.112 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:10.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:10.114 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:13:10.215 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:10.226 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:09 smithi173 bash[58816]: cluster 2023-10-01T11:13:08.457366+0000 mgr.y (mgr.25072) 1676 : cluster 0 pgmap v1810: 89 pgs: 89 active+clean; 1.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 636 op/s 2023-10-01T11:13:10.246 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:10.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:10.248 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:13:10.263 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:09 smithi078 bash[78274]: cluster 2023-10-01T11:13:08.457366+0000 mgr.y (mgr.25072) 1676 : cluster 0 pgmap v1810: 89 pgs: 89 active+clean; 1.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 636 op/s 2023-10-01T11:13:10.263 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:09 smithi078 bash[75299]: cluster 2023-10-01T11:13:08.457366+0000 mgr.y (mgr.25072) 1676 : cluster 0 pgmap v1810: 89 pgs: 89 active+clean; 1.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 636 op/s 2023-10-01T11:13:10.349 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:10.388 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:10.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:10.396 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:13:10.497 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:10.531 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:10.539 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:13:10 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:13:10] "GET /metrics HTTP/1.1" 200 36382 "" "Prometheus/2.33.4" 2023-10-01T11:13:10.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:10.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:13:10.581 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T11:13:10.626 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T11:13:10.641 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:10.685 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T11:13:10.693 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:10.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:10.696 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:13:10.748 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T11:13:10.790 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T11:13:10.797 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:10.838 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T11:13:10.842 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:10.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:10.846 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:13:10.889 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T11:13:10.922 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T11:13:10.929 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T11:13:10.929 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T11:13:10.929 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T11:13:10.929 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T11:13:10.929 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T11:13:10.929 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T11:13:10.929 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T11:13:10.929 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T11:13:10.948 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:10.991 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:10.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:10.995 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:13:11.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:10 smithi078 bash[75299]: audit 2023-10-01T11:13:10.518956+0000 mon.c (mon.2) 245 : audit 0 from='client.? 172.21.15.78:0/3819050887' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:13:11.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:10 smithi078 bash[78274]: audit 2023-10-01T11:13:10.518956+0000 mon.c (mon.2) 245 : audit 0 from='client.? 172.21.15.78:0/3819050887' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:13:11.096 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:11.135 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:11.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:11.139 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:13:11.239 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:11.247 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:10 smithi173 bash[58816]: audit 2023-10-01T11:13:10.518956+0000 mon.c (mon.2) 245 : audit 0 from='client.? 172.21.15.78:0/3819050887' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:13:11.270 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:11.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:11.275 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:13:11.375 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:11.397 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:11.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:11.407 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:13:11.508 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:11.538 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:11.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:11.543 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:13:11.644 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:11.670 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:11.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:11.674 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:13:11.774 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:11.792 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:11.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:11.793 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:13:11.894 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:11.921 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:11.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:11.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:13:12.023 INFO:teuthology.orchestra.run.smithi078.stderr:nodeep-scrub is unset 2023-10-01T11:13:12.024 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:12.033 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:11 smithi078 bash[78274]: cluster 2023-10-01T11:13:10.458043+0000 mgr.y (mgr.25072) 1677 : cluster 0 pgmap v1811: 89 pgs: 89 active+clean; 1.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 617 op/s 2023-10-01T11:13:12.033 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:11 smithi078 bash[78274]: audit 2023-10-01T11:13:11.085632+0000 mon.c (mon.2) 246 : audit 1 from='client.? 172.21.15.78:0/4207083129' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:13:12.033 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:11 smithi078 bash[78274]: audit 2023-10-01T11:13:11.086220+0000 mon.a (mon.0) 2152 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:13:12.033 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:11 smithi078 bash[75299]: cluster 2023-10-01T11:13:10.458043+0000 mgr.y (mgr.25072) 1677 : cluster 0 pgmap v1811: 89 pgs: 89 active+clean; 1.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 617 op/s 2023-10-01T11:13:12.033 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:11 smithi078 bash[75299]: audit 2023-10-01T11:13:11.085632+0000 mon.c (mon.2) 246 : audit 1 from='client.? 172.21.15.78:0/4207083129' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:13:12.033 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:11 smithi078 bash[75299]: audit 2023-10-01T11:13:11.086220+0000 mon.a (mon.0) 2152 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:13:12.058 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:12.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:12.062 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:13:12.163 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:12.200 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:12.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:12.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:13:12.305 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:12.334 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:12.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:12.338 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:13:12.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:11 smithi173 bash[58816]: cluster 2023-10-01T11:13:10.458043+0000 mgr.y (mgr.25072) 1677 : cluster 0 pgmap v1811: 89 pgs: 89 active+clean; 1.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 617 op/s 2023-10-01T11:13:12.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:11 smithi173 bash[58816]: audit 2023-10-01T11:13:11.085632+0000 mon.c (mon.2) 246 : audit 1 from='client.? 172.21.15.78:0/4207083129' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:13:12.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:11 smithi173 bash[58816]: audit 2023-10-01T11:13:11.086220+0000 mon.a (mon.0) 2152 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:13:12.439 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:12.474 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:12.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:12.476 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:13:12.577 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:12.606 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:12.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:12.614 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:13:12.715 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:12.750 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:12.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:12.755 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:13:12.856 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:12.892 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:12.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:12.895 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:13:12.996 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:13.034 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:13.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:13.039 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:13:13.140 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:13.163 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:13.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:13.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:13:13.269 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:13.278 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:12 smithi078 bash[78274]: cluster 2023-10-01T11:13:11.991536+0000 mon.a (mon.0) 2153 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T11:13:13.278 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:12 smithi078 bash[78274]: cluster 2023-10-01T11:13:11.991576+0000 mon.a (mon.0) 2154 : cluster 1 Cluster is now healthy 2023-10-01T11:13:13.278 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:12 smithi078 bash[78274]: cluster 2023-10-01T11:13:12.007189+0000 osd.5 (osd.5) 402 : cluster 3 failed to encode map e374 with expected crc 2023-10-01T11:13:13.278 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:12 smithi078 bash[78274]: cluster 2023-10-01T11:13:12.009598+0000 osd.7 (osd.7) 420 : cluster 3 failed to encode map e374 with expected crc 2023-10-01T11:13:13.278 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:12 smithi078 bash[78274]: audit 2023-10-01T11:13:12.009731+0000 mon.a (mon.0) 2155 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T11:13:13.279 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:12 smithi078 bash[78274]: cluster 2023-10-01T11:13:12.009799+0000 mon.a (mon.0) 2156 : cluster 0 osdmap e374: 8 total, 8 up, 8 in 2023-10-01T11:13:13.279 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:12 smithi078 bash[75299]: cluster 2023-10-01T11:13:11.991536+0000 mon.a (mon.0) 2153 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T11:13:13.279 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:12 smithi078 bash[75299]: cluster 2023-10-01T11:13:11.991576+0000 mon.a (mon.0) 2154 : cluster 1 Cluster is now healthy 2023-10-01T11:13:13.279 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:12 smithi078 bash[75299]: cluster 2023-10-01T11:13:12.007189+0000 osd.5 (osd.5) 402 : cluster 3 failed to encode map e374 with expected crc 2023-10-01T11:13:13.279 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:12 smithi078 bash[75299]: cluster 2023-10-01T11:13:12.009598+0000 osd.7 (osd.7) 420 : cluster 3 failed to encode map e374 with expected crc 2023-10-01T11:13:13.279 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:12 smithi078 bash[75299]: audit 2023-10-01T11:13:12.009731+0000 mon.a (mon.0) 2155 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T11:13:13.279 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:12 smithi078 bash[75299]: cluster 2023-10-01T11:13:12.009799+0000 mon.a (mon.0) 2156 : cluster 0 osdmap e374: 8 total, 8 up, 8 in 2023-10-01T11:13:13.293 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:13.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:13.298 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:13:13.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:12 smithi173 bash[58816]: cluster 2023-10-01T11:13:11.991536+0000 mon.a (mon.0) 2153 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T11:13:13.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:12 smithi173 bash[58816]: cluster 2023-10-01T11:13:11.991576+0000 mon.a (mon.0) 2154 : cluster 1 Cluster is now healthy 2023-10-01T11:13:13.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:12 smithi173 bash[58816]: cluster 2023-10-01T11:13:12.007189+0000 osd.5 (osd.5) 402 : cluster 3 failed to encode map e374 with expected crc 2023-10-01T11:13:13.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:12 smithi173 bash[58816]: cluster 2023-10-01T11:13:12.009598+0000 osd.7 (osd.7) 420 : cluster 3 failed to encode map e374 with expected crc 2023-10-01T11:13:13.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:12 smithi173 bash[58816]: audit 2023-10-01T11:13:12.009731+0000 mon.a (mon.0) 2155 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T11:13:13.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:12 smithi173 bash[58816]: cluster 2023-10-01T11:13:12.009799+0000 mon.a (mon.0) 2156 : cluster 0 osdmap e374: 8 total, 8 up, 8 in 2023-10-01T11:13:13.399 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:13.423 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:13.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:13.427 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:13:13.528 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:13.569 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:13.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:13.575 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:13:13.676 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:13.722 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:13.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:13.726 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:13:13.826 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:13.858 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:13.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:13.861 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:13:13.962 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:13.990 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:13.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:13.992 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:13:14.093 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:14.124 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:14.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:14.128 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:13:14.228 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:14.237 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:14 smithi173 bash[58816]: cluster 2023-10-01T11:13:12.458516+0000 mgr.y (mgr.25072) 1678 : cluster 0 pgmap v1813: 89 pgs: 89 active+clean; 1.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 618 op/s 2023-10-01T11:13:14.258 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:14.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:14.262 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:13:14.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:14 smithi078 bash[78274]: cluster 2023-10-01T11:13:12.458516+0000 mgr.y (mgr.25072) 1678 : cluster 0 pgmap v1813: 89 pgs: 89 active+clean; 1.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 618 op/s 2023-10-01T11:13:14.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:14 smithi078 bash[75299]: cluster 2023-10-01T11:13:12.458516+0000 mgr.y (mgr.25072) 1678 : cluster 0 pgmap v1813: 89 pgs: 89 active+clean; 1.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 618 op/s 2023-10-01T11:13:14.363 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:14.409 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:14.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:14.410 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:13:14.511 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:14.535 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:14.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:14.538 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:13:14.639 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:14.670 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:14.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:14.673 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:13:14.774 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:14.801 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:14.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:14.803 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:13:14.904 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:14.932 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:14.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:14.936 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:13:15.037 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:15.063 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:15.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:15.065 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:13:15.166 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:15.197 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:15.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:15.206 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:13:15.307 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:15.334 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:15.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:15.342 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:13:15.442 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:15.486 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:15.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:15.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:13:15.592 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:15.627 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:15.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:15.632 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:13:15.733 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:15.770 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:15.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:15.777 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:13:15.878 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:15.902 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:15.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:15.904 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:13:16.005 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:16.046 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:16.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:16.052 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:13:16.153 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:16.180 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:16.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:16.184 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:13:16.284 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:16.293 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:16 smithi078 bash[78274]: cluster 2023-10-01T11:13:14.459237+0000 mgr.y (mgr.25072) 1679 : cluster 0 pgmap v1814: 89 pgs: 89 active+clean; 1.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 612 op/s 2023-10-01T11:13:16.296 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:16 smithi078 bash[75299]: cluster 2023-10-01T11:13:14.459237+0000 mgr.y (mgr.25072) 1679 : cluster 0 pgmap v1814: 89 pgs: 89 active+clean; 1.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 612 op/s 2023-10-01T11:13:16.310 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:16.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:16.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:13:16.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:16 smithi173 bash[58816]: cluster 2023-10-01T11:13:14.459237+0000 mgr.y (mgr.25072) 1679 : cluster 0 pgmap v1814: 89 pgs: 89 active+clean; 1.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 612 op/s 2023-10-01T11:13:16.417 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:16.474 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:16.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:16.476 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:13:16.577 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:16.612 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:16.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:16.618 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:13:16.719 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:16.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:16.756 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:16.756 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:13:16.857 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:16.883 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:16.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:16.889 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:13:16.990 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:17.016 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:17.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:17.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:13:17.035 INFO:teuthology.orchestra.run.smithi078.stderr:reweighted osd.7 to 1 (10000) 2023-10-01T11:13:17.036 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:17 smithi078 bash[75299]: audit 2023-10-01T11:13:16.453091+0000 mon.a (mon.0) 2157 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T11:13:17.036 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:17 smithi078 bash[75299]: audit 2023-10-01T11:13:16.453100+0000 mon.b (mon.1) 62 : audit 1 from='client.? 172.21.15.78:0/2440334806' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T11:13:17.121 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:17.159 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:17.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:17.163 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:13:17.264 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:17.281 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T11:13:17.306 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:17.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:17.311 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:17 smithi078 bash[78274]: audit 2023-10-01T11:13:16.453091+0000 mon.a (mon.0) 2157 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T11:13:17.311 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:17 smithi078 bash[78274]: audit 2023-10-01T11:13:16.453100+0000 mon.b (mon.1) 62 : audit 1 from='client.? 172.21.15.78:0/2440334806' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T11:13:17.312 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:13:17.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:17 smithi173 bash[58816]: audit 2023-10-01T11:13:16.453091+0000 mon.a (mon.0) 2157 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T11:13:17.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:17 smithi173 bash[58816]: audit 2023-10-01T11:13:16.453100+0000 mon.b (mon.1) 62 : audit 1 from='client.? 172.21.15.78:0/2440334806' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T11:13:17.413 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:17.446 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:17.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:17.451 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:13:17.459 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T11:13:17.551 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:17.601 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:17.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:17.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:13:17.707 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:17.731 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:17.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:17.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:13:17.836 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:17.875 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:17.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:17.880 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:13:17.981 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:18.005 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:18.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:18.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:13:18.037 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:18 smithi173 bash[58816]: cluster 2023-10-01T11:13:16.459792+0000 mgr.y (mgr.25072) 1680 : cluster 0 pgmap v1815: 89 pgs: 89 active+clean; 1.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 614 op/s 2023-10-01T11:13:18.038 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:18 smithi173 bash[58816]: audit 2023-10-01T11:13:17.023617+0000 mon.a (mon.0) 2158 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-01T11:13:18.038 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:18 smithi173 bash[58816]: cluster 2023-10-01T11:13:17.023716+0000 mon.a (mon.0) 2159 : cluster 0 osdmap e375: 8 total, 8 up, 8 in 2023-10-01T11:13:18.038 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:18 smithi173 bash[58816]: cluster 2023-10-01T11:13:17.035427+0000 osd.7 (osd.7) 421 : cluster 3 failed to encode map e375 with expected crc 2023-10-01T11:13:18.038 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:18 smithi173 bash[58816]: cluster 2023-10-01T11:13:17.035482+0000 osd.5 (osd.5) 403 : cluster 3 failed to encode map e375 with expected crc 2023-10-01T11:13:18.038 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:18 smithi173 bash[58816]: cluster 2023-10-01T11:13:17.039400+0000 osd.6 (osd.6) 362 : cluster 3 failed to encode map e375 with expected crc 2023-10-01T11:13:18.038 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:18 smithi173 bash[58816]: audit 2023-10-01T11:13:17.486612+0000 mon.a (mon.0) 2160 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:13:18.038 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:18 smithi173 bash[58816]: audit 2023-10-01T11:13:17.494200+0000 mon.a (mon.0) 2161 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:13:18.111 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:18.142 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:18.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:18.146 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:13:18.247 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:18.259 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:18 smithi078 bash[75299]: cluster 2023-10-01T11:13:16.459792+0000 mgr.y (mgr.25072) 1680 : cluster 0 pgmap v1815: 89 pgs: 89 active+clean; 1.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 614 op/s 2023-10-01T11:13:18.259 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:18 smithi078 bash[75299]: audit 2023-10-01T11:13:17.023617+0000 mon.a (mon.0) 2158 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-01T11:13:18.259 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:18 smithi078 bash[75299]: cluster 2023-10-01T11:13:17.023716+0000 mon.a (mon.0) 2159 : cluster 0 osdmap e375: 8 total, 8 up, 8 in 2023-10-01T11:13:18.259 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:18 smithi078 bash[75299]: cluster 2023-10-01T11:13:17.035427+0000 osd.7 (osd.7) 421 : cluster 3 failed to encode map e375 with expected crc 2023-10-01T11:13:18.259 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:18 smithi078 bash[75299]: cluster 2023-10-01T11:13:17.035482+0000 osd.5 (osd.5) 403 : cluster 3 failed to encode map e375 with expected crc 2023-10-01T11:13:18.259 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:18 smithi078 bash[75299]: cluster 2023-10-01T11:13:17.039400+0000 osd.6 (osd.6) 362 : cluster 3 failed to encode map e375 with expected crc 2023-10-01T11:13:18.259 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:18 smithi078 bash[75299]: audit 2023-10-01T11:13:17.486612+0000 mon.a (mon.0) 2160 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:13:18.259 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:18 smithi078 bash[75299]: audit 2023-10-01T11:13:17.494200+0000 mon.a (mon.0) 2161 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:13:18.260 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:18 smithi078 bash[78274]: cluster 2023-10-01T11:13:16.459792+0000 mgr.y (mgr.25072) 1680 : cluster 0 pgmap v1815: 89 pgs: 89 active+clean; 1.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 614 op/s 2023-10-01T11:13:18.260 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:18 smithi078 bash[78274]: audit 2023-10-01T11:13:17.023617+0000 mon.a (mon.0) 2158 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-01T11:13:18.260 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:18 smithi078 bash[78274]: cluster 2023-10-01T11:13:17.023716+0000 mon.a (mon.0) 2159 : cluster 0 osdmap e375: 8 total, 8 up, 8 in 2023-10-01T11:13:18.260 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:18 smithi078 bash[78274]: cluster 2023-10-01T11:13:17.035427+0000 osd.7 (osd.7) 421 : cluster 3 failed to encode map e375 with expected crc 2023-10-01T11:13:18.260 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:18 smithi078 bash[78274]: cluster 2023-10-01T11:13:17.035482+0000 osd.5 (osd.5) 403 : cluster 3 failed to encode map e375 with expected crc 2023-10-01T11:13:18.260 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:18 smithi078 bash[78274]: cluster 2023-10-01T11:13:17.039400+0000 osd.6 (osd.6) 362 : cluster 3 failed to encode map e375 with expected crc 2023-10-01T11:13:18.260 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:18 smithi078 bash[78274]: audit 2023-10-01T11:13:17.486612+0000 mon.a (mon.0) 2160 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:13:18.260 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:18 smithi078 bash[78274]: audit 2023-10-01T11:13:17.494200+0000 mon.a (mon.0) 2161 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:13:18.277 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:18.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:18.280 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:13:18.381 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:18.413 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:18.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:18.416 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:13:18.517 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:18.553 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:18.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:18.555 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:13:18.656 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:18.691 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:18.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:18.696 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:13:18.718 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-01T11:13:18.719 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T11:13:18.798 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:18.849 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:18.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:18.854 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:13:18.955 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:18.987 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:18.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:18.993 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:13:19.093 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:19.117 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:19.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:19.119 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:13:19.184 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:13:19.219 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:19.258 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:19.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:19.261 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:13:19.362 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:19.388 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:19.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:19.397 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:13:19.497 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:19.523 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:19.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:19.531 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:13:19.631 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:19.659 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:19.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:19.661 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:13:19.762 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:19.805 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:19.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:19.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:13:19.908 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:19.939 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:19.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:19.951 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:13:20.041 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:20 smithi078 bash[75299]: cluster 2023-10-01T11:13:18.460232+0000 mgr.y (mgr.25072) 1681 : cluster 0 pgmap v1817: 89 pgs: 89 active+clean; 1.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 460 op/s 2023-10-01T11:13:20.052 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:20.082 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:20.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:20.087 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:13:20.188 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:20.196 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:20 smithi173 bash[58816]: cluster 2023-10-01T11:13:18.460232+0000 mgr.y (mgr.25072) 1681 : cluster 0 pgmap v1817: 89 pgs: 89 active+clean; 1.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 460 op/s 2023-10-01T11:13:20.210 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:20.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:20.217 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:13:20.318 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:20.326 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:20 smithi078 bash[78274]: cluster 2023-10-01T11:13:18.460232+0000 mgr.y (mgr.25072) 1681 : cluster 0 pgmap v1817: 89 pgs: 89 active+clean; 1.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 460 op/s 2023-10-01T11:13:20.326 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:13:20 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:13:20] "GET /metrics HTTP/1.1" 200 36383 "" "Prometheus/2.33.4" 2023-10-01T11:13:20.353 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:20.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:20.359 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:13:20.460 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:20.499 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:20.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:20.505 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:13:20.606 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:20.638 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:20.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:20.645 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:13:20.746 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:20.775 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:20.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:20.777 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:13:20.878 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:20.921 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:20.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:20.927 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:13:21.028 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:21.051 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:21.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:21.054 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:13:21.155 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:21.197 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:21.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:21.202 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:13:21.303 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:21.339 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:21.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:21.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:13:21.445 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:21.467 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:21.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:21.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:13:21.575 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:21.594 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:21.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:21.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:13:21.704 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:21.738 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:21.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:21.743 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:13:21.844 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:21.870 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:21.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:21.876 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:13:21.977 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:22.003 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:22.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:22.007 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:13:22.052 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:22 smithi173 bash[58816]: cluster 2023-10-01T11:13:20.460813+0000 mgr.y (mgr.25072) 1682 : cluster 0 pgmap v1818: 89 pgs: 89 active+clean; 1.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 667 op/s 2023-10-01T11:13:22.108 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:22.116 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:22 smithi078 bash[75299]: cluster 2023-10-01T11:13:20.460813+0000 mgr.y (mgr.25072) 1682 : cluster 0 pgmap v1818: 89 pgs: 89 active+clean; 1.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 667 op/s 2023-10-01T11:13:22.117 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:22 smithi078 bash[78274]: cluster 2023-10-01T11:13:20.460813+0000 mgr.y (mgr.25072) 1682 : cluster 0 pgmap v1818: 89 pgs: 89 active+clean; 1.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 667 op/s 2023-10-01T11:13:22.137 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:22.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:22.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:13:22.244 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:22.285 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:22.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:22.286 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:13:22.387 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:22.410 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:22.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:22.416 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:13:22.517 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:22.545 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:22.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:22.547 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:13:22.648 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:22.683 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:22.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:22.689 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:13:22.789 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:22.828 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:22.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:22.834 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:13:22.934 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:22.970 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:22.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:22.975 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:13:23.076 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:23.114 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:23.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:23.122 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:13:23.223 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:23.263 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:23.265 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:13:23.266 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:13:23.266 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:13:23.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:23.269 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:13:23.370 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:23.414 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:23.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:23.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:13:23.519 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:23.543 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:23.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:23.552 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:13:23.652 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:23.676 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:23.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:23.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:13:23.779 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:23.811 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:23.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:23.814 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:13:23.914 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:23.947 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:23.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:23.950 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:13:24.051 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:24.055 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:24 smithi173 bash[58816]: cluster 2023-10-01T11:13:22.461470+0000 mgr.y (mgr.25072) 1683 : cluster 0 pgmap v1819: 89 pgs: 89 active+clean; 1.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 624 op/s 2023-10-01T11:13:24.080 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:24.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:24.082 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:13:24.183 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:24.191 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:24 smithi078 bash[78274]: cluster 2023-10-01T11:13:22.461470+0000 mgr.y (mgr.25072) 1683 : cluster 0 pgmap v1819: 89 pgs: 89 active+clean; 1.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 624 op/s 2023-10-01T11:13:24.192 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:24 smithi078 bash[75299]: cluster 2023-10-01T11:13:22.461470+0000 mgr.y (mgr.25072) 1683 : cluster 0 pgmap v1819: 89 pgs: 89 active+clean; 1.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 624 op/s 2023-10-01T11:13:24.209 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:24.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:24.216 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:13:24.316 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:24.339 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:24.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:24.342 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:13:24.443 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:24.477 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:24.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:24.484 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:13:24.584 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:24.616 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:24.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:24.619 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:13:24.719 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:24.757 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:24.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:24.759 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:13:24.861 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:24.887 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:24.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:24.891 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:13:24.992 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:25.034 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:25.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:25.039 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:13:25.139 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:25.172 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:25.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:25.180 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:13:25.281 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:25.319 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:25.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:25.324 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:13:25.424 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:25.450 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:25.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:25.454 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:13:25.555 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:25.575 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:25.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:25.577 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:13:25.678 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:25.709 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:25.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:25.713 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:13:25.814 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:25.827 INFO:teuthology.orchestra.run.smithi078.stdout:ERROR: (22) Invalid argument 2023-10-01T11:13:25.828 INFO:teuthology.orchestra.run.smithi078.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T11:13:25.845 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:25.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:25.848 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:13:25.948 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:25.982 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:25.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:25.987 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:13:26.088 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:26.117 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:26.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:26.120 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:13:26.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:26 smithi173 bash[58816]: cluster 2023-10-01T11:13:24.464798+0000 mgr.y (mgr.25072) 1684 : cluster 0 pgmap v1820: 89 pgs: 89 active+clean; 1.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 642 op/s 2023-10-01T11:13:26.221 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:26.262 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:26.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:26.265 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:13:26.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:26 smithi078 bash[75299]: cluster 2023-10-01T11:13:24.464798+0000 mgr.y (mgr.25072) 1684 : cluster 0 pgmap v1820: 89 pgs: 89 active+clean; 1.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 642 op/s 2023-10-01T11:13:26.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:26 smithi078 bash[78274]: cluster 2023-10-01T11:13:24.464798+0000 mgr.y (mgr.25072) 1684 : cluster 0 pgmap v1820: 89 pgs: 89 active+clean; 1.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 642 op/s 2023-10-01T11:13:26.366 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:26.416 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:26.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:26.419 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:13:26.520 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:26.545 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:26.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:26.548 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:13:26.649 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:26.679 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:26.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:26.686 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:13:26.786 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:26.818 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:26.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:26.828 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:13:26.928 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:26.959 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:26.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:26.961 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:13:27.062 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:27.098 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:27.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:27.104 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:13:27.205 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:27.242 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:27.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:27.246 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:13:27.346 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:27.397 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:27.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:27.400 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:13:27.501 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:27.529 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:27.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:27.539 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:13:27.639 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:27.666 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:27.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:27.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:13:27.771 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:27.800 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:27.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:27.804 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:13:27.904 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:27.929 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:27.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:27.931 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:13:28.032 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:28.068 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:28.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:28.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:13:28.171 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:28.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:28.216 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:28.216 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:13:28.317 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:28.326 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:28 smithi173 bash[58816]: cluster 2023-10-01T11:13:26.465442+0000 mgr.y (mgr.25072) 1685 : cluster 0 pgmap v1821: 89 pgs: 89 active+clean; 1.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 646 op/s 2023-10-01T11:13:28.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:28 smithi078 bash[78274]: cluster 2023-10-01T11:13:26.465442+0000 mgr.y (mgr.25072) 1685 : cluster 0 pgmap v1821: 89 pgs: 89 active+clean; 1.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 646 op/s 2023-10-01T11:13:28.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:28 smithi078 bash[75299]: cluster 2023-10-01T11:13:26.465442+0000 mgr.y (mgr.25072) 1685 : cluster 0 pgmap v1821: 89 pgs: 89 active+clean; 1.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 646 op/s 2023-10-01T11:13:28.351 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:28.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:28.355 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:13:28.456 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:28.493 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:28.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:28.497 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:13:28.598 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:28.632 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:28.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:28.634 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:13:28.735 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:28.762 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:28.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:28.768 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:13:28.869 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:28.908 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:28.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:28.914 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:13:29.015 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:29.047 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:29.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:29.050 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:13:29.150 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:29.187 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:29.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:29.193 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:13:29.294 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:29.319 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-10-01T11:13:29.327 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:29.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:29.331 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:13:29.432 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:29.458 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:29.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:29.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:13:29.563 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:29.599 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:29.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:29.601 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:13:29.702 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:29.740 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:29.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:29.747 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:13:29.848 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:29.890 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:29.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:29.894 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:13:29.995 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:30.019 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:30.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:30.021 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:13:30.089 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:30 smithi078 bash[75299]: cluster 2023-10-01T11:13:28.465787+0000 mgr.y (mgr.25072) 1686 : cluster 0 pgmap v1822: 89 pgs: 89 active+clean; 1.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 564 op/s 2023-10-01T11:13:30.089 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:30 smithi078 bash[78274]: cluster 2023-10-01T11:13:28.465787+0000 mgr.y (mgr.25072) 1686 : cluster 0 pgmap v1822: 89 pgs: 89 active+clean; 1.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 564 op/s 2023-10-01T11:13:30.122 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:30.169 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:30.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:30.172 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:13:30.273 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:30.306 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:30.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:30.310 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:13:30.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:30 smithi173 bash[58816]: cluster 2023-10-01T11:13:28.465787+0000 mgr.y (mgr.25072) 1686 : cluster 0 pgmap v1822: 89 pgs: 89 active+clean; 1.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 564 op/s 2023-10-01T11:13:30.412 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:30.436 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:30.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:30.438 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:13:30.539 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:30.552 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:13:30 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:13:30] "GET /metrics HTTP/1.1" 200 36383 "" "Prometheus/2.33.4" 2023-10-01T11:13:30.576 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:30.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:30.582 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:13:30.683 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:30.717 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:30.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:30.724 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:13:30.824 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:30.862 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:30.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:30.866 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:13:30.967 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:31.003 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:31.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:31.006 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:13:31.107 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:31.133 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:31.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:31.139 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:13:31.240 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:31.280 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:31.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:31.282 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:13:31.383 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:31.415 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:31.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:31.418 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:13:31.519 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:31.544 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:31.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:31.550 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:13:31.650 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:31.681 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:31.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:31.687 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:13:31.788 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:31.811 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:31.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:31.813 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:13:31.914 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:31.953 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:31.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:31.955 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:13:32.056 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:32.093 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:32.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:32.099 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:13:32.200 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:32.208 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:32 smithi173 bash[58816]: cluster 2023-10-01T11:13:30.466616+0000 mgr.y (mgr.25072) 1687 : cluster 0 pgmap v1823: 89 pgs: 89 active+clean; 996 MiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 758 op/s 2023-10-01T11:13:32.235 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:32.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:32.240 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:13:32.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:32 smithi078 bash[75299]: cluster 2023-10-01T11:13:30.466616+0000 mgr.y (mgr.25072) 1687 : cluster 0 pgmap v1823: 89 pgs: 89 active+clean; 996 MiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 758 op/s 2023-10-01T11:13:32.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:32 smithi078 bash[78274]: cluster 2023-10-01T11:13:30.466616+0000 mgr.y (mgr.25072) 1687 : cluster 0 pgmap v1823: 89 pgs: 89 active+clean; 996 MiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 758 op/s 2023-10-01T11:13:32.340 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:32.371 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:32.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:32.376 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:13:32.477 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:32.506 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:32.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:32.515 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:13:32.616 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:32.642 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:32.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:32.649 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:13:32.750 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:32.780 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:32.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:32.786 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:13:32.887 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:32.912 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:32.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:32.923 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:13:33.023 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:33.055 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:33.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:33.060 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:13:33.161 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:33.173 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:33 smithi078 bash[75299]: audit 2023-10-01T11:13:32.481596+0000 mon.a (mon.0) 2162 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:13:33.174 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:33 smithi078 bash[78274]: audit 2023-10-01T11:13:32.481596+0000 mon.a (mon.0) 2162 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:13:33.200 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:33.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:33.207 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:13:33.308 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:33.357 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:33.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:33.360 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:13:33.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:33 smithi173 bash[58816]: audit 2023-10-01T11:13:32.481596+0000 mon.a (mon.0) 2162 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:13:33.460 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:33.494 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:33.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:33.499 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:13:33.599 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:33.623 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:33.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:33.626 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:13:33.727 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:33.756 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:33.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:33.762 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:13:33.862 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:33.887 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:33.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:33.890 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:13:33.990 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:34.017 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:34.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:34.022 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:13:34.123 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:34.171 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:34.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:34.175 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:13:34.275 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:34.283 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:34 smithi173 bash[58816]: cluster 2023-10-01T11:13:32.467149+0000 mgr.y (mgr.25072) 1688 : cluster 0 pgmap v1824: 89 pgs: 89 active+clean; 958 MiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 595 op/s 2023-10-01T11:13:34.301 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:34.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:34.306 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:13:34.407 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:34.436 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:34.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:34.442 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:13:34.543 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:34.548 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:34 smithi078 bash[75299]: cluster 2023-10-01T11:13:32.467149+0000 mgr.y (mgr.25072) 1688 : cluster 0 pgmap v1824: 89 pgs: 89 active+clean; 958 MiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 595 op/s 2023-10-01T11:13:34.549 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:34 smithi078 bash[78274]: cluster 2023-10-01T11:13:32.467149+0000 mgr.y (mgr.25072) 1688 : cluster 0 pgmap v1824: 89 pgs: 89 active+clean; 958 MiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 595 op/s 2023-10-01T11:13:34.577 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:34.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:34.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:13:34.680 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:34.720 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:34.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:34.722 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:13:34.823 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:34.846 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:34.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:34.851 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:13:34.951 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:34.978 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:34.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:34.981 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:13:35.081 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:35.114 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:35.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:35.116 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:13:35.216 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:35.238 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:35.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:35.241 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:13:35.341 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:35.361 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:35.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:35.364 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:13:35.465 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:35.487 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:35.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:35.489 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:13:35.590 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:35.629 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:35.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:35.632 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:13:35.711 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:13:35.733 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:35.781 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:35.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:35.788 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:13:35.889 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:35.930 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:35.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:35.936 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:13:36.037 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:36.066 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:36.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:36.071 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:13:36.172 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:36.203 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:36.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:36.206 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:13:36.307 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:36.317 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:36 smithi078 bash[75299]: cluster 2023-10-01T11:13:34.467804+0000 mgr.y (mgr.25072) 1689 : cluster 0 pgmap v1825: 89 pgs: 89 active+clean; 827 MiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 720 op/s 2023-10-01T11:13:36.318 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:36 smithi078 bash[78274]: cluster 2023-10-01T11:13:34.467804+0000 mgr.y (mgr.25072) 1689 : cluster 0 pgmap v1825: 89 pgs: 89 active+clean; 827 MiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 720 op/s 2023-10-01T11:13:36.341 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:36.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:36.344 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:13:36.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:36 smithi173 bash[58816]: cluster 2023-10-01T11:13:34.467804+0000 mgr.y (mgr.25072) 1689 : cluster 0 pgmap v1825: 89 pgs: 89 active+clean; 827 MiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 720 op/s 2023-10-01T11:13:36.445 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:36.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:36.482 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:36.482 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:13:36.583 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:36.605 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:36.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:36.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:13:36.711 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:36.738 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:36.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:36.742 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:13:36.843 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:36.867 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:36.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:36.870 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:13:36.970 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:36.998 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:37.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:37.006 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:13:37.107 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:37.137 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:37.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:37.141 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:13:37.242 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:37.267 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:37.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:37.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:13:37.372 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:37.397 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:37.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:37.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:13:37.500 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:37.523 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:37.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:37.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:13:37.626 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:37.656 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:37.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:37.663 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:13:37.763 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:37.788 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:37.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:37.793 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:13:37.894 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:37.919 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:37.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:37.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:13:38.024 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:38.049 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:38.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:38.054 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:13:38.155 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:38.180 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:38.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:38.192 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:13:38.292 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:38.300 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:38 smithi173 bash[58816]: cluster 2023-10-01T11:13:36.468399+0000 mgr.y (mgr.25072) 1690 : cluster 0 pgmap v1826: 89 pgs: 89 active+clean; 766 MiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 610 op/s 2023-10-01T11:13:38.318 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:38.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:38.321 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:13:38.380 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:38 smithi078 bash[78274]: cluster 2023-10-01T11:13:36.468399+0000 mgr.y (mgr.25072) 1690 : cluster 0 pgmap v1826: 89 pgs: 89 active+clean; 766 MiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 610 op/s 2023-10-01T11:13:38.380 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:38 smithi078 bash[75299]: cluster 2023-10-01T11:13:36.468399+0000 mgr.y (mgr.25072) 1690 : cluster 0 pgmap v1826: 89 pgs: 89 active+clean; 766 MiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 610 op/s 2023-10-01T11:13:38.422 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:38.478 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:38.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:38.485 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:13:38.535 INFO:teuthology.orchestra.run.smithi078.stdout: 2023-10-01T11:13:38.537 INFO:teuthology.orchestra.run.smithi078.stderr:dumped all 2023-10-01T11:13:38.586 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:38.611 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:38.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:38.614 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:13:38.715 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:38.743 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:38.743 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T11:13:38.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:38.750 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:13:38.786 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T11:13:38.834 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T11:13:38.851 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:38.885 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T11:13:38.897 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:38.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:38.900 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:13:38.919 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T11:13:38.950 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T11:13:38.994 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T11:13:39.000 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:39.036 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:39.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:39.039 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:13:39.048 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T11:13:39.055 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T11:13:39.055 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T11:13:39.055 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T11:13:39.055 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T11:13:39.055 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T11:13:39.055 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T11:13:39.055 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T11:13:39.055 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T11:13:39.130 INFO:teuthology.orchestra.run.smithi078.stderr:noscrub is set 2023-10-01T11:13:39.140 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:39.165 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:39.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:39.170 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:13:39.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:39 smithi078 bash[75299]: audit 2023-10-01T11:13:38.675924+0000 mon.a (mon.0) 2163 : audit 0 from='client.? 172.21.15.78:0/1917918205' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:13:39.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:39 smithi078 bash[75299]: audit 2023-10-01T11:13:39.063952+0000 mon.c (mon.2) 247 : audit 1 from='client.? 172.21.15.78:0/3369265641' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T11:13:39.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:39 smithi078 bash[75299]: audit 2023-10-01T11:13:39.064937+0000 mon.a (mon.0) 2164 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T11:13:39.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:39 smithi078 bash[78274]: audit 2023-10-01T11:13:38.675924+0000 mon.a (mon.0) 2163 : audit 0 from='client.? 172.21.15.78:0/1917918205' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:13:39.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:39 smithi078 bash[78274]: audit 2023-10-01T11:13:39.063952+0000 mon.c (mon.2) 247 : audit 1 from='client.? 172.21.15.78:0/3369265641' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T11:13:39.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:39 smithi078 bash[78274]: audit 2023-10-01T11:13:39.064937+0000 mon.a (mon.0) 2164 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T11:13:39.271 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:39.306 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:39.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:39.308 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:13:39.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:39 smithi173 bash[58816]: audit 2023-10-01T11:13:38.675924+0000 mon.a (mon.0) 2163 : audit 0 from='client.? 172.21.15.78:0/1917918205' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:13:39.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:39 smithi173 bash[58816]: audit 2023-10-01T11:13:39.063952+0000 mon.c (mon.2) 247 : audit 1 from='client.? 172.21.15.78:0/3369265641' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T11:13:39.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:39 smithi173 bash[58816]: audit 2023-10-01T11:13:39.064937+0000 mon.a (mon.0) 2164 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T11:13:39.409 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:39.433 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:39.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:39.436 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:13:39.537 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:39.563 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:39.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:39.568 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:13:39.668 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:39.700 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:39.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:39.703 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:13:39.804 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:39.833 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:39.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:39.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:13:39.938 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:39.968 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:39.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:39.972 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:13:40.072 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:40.096 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:40.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:40.103 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:13:40.204 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:40.213 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:40 smithi078 bash[75299]: cluster 2023-10-01T11:13:38.468826+0000 mgr.y (mgr.25072) 1691 : cluster 0 pgmap v1827: 89 pgs: 89 active+clean; 766 MiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 526 op/s 2023-10-01T11:13:40.213 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:40 smithi078 bash[75299]: audit 2023-10-01T11:13:38.505474+0000 mgr.y (mgr.25072) 1692 : audit 0 from='client.46919 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T11:13:40.213 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:40 smithi078 bash[75299]: cluster 2023-10-01T11:13:39.101701+0000 mon.a (mon.0) 2165 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:13:40.213 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:40 smithi078 bash[75299]: cluster 2023-10-01T11:13:39.119404+0000 osd.7 (osd.7) 422 : cluster 3 failed to encode map e376 with expected crc 2023-10-01T11:13:40.213 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:40 smithi078 bash[75299]: audit 2023-10-01T11:13:39.121399+0000 mon.a (mon.0) 2166 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T11:13:40.214 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:40 smithi078 bash[75299]: cluster 2023-10-01T11:13:39.121575+0000 mon.a (mon.0) 2167 : cluster 0 osdmap e376: 8 total, 8 up, 8 in 2023-10-01T11:13:40.214 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:40 smithi078 bash[75299]: cluster 2023-10-01T11:13:39.123183+0000 osd.7 (osd.7) 423 : cluster 3 failed to encode map e376 with expected crc 2023-10-01T11:13:40.214 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:40 smithi078 bash[75299]: cluster 2023-10-01T11:13:39.123814+0000 osd.6 (osd.6) 363 : cluster 3 failed to encode map e376 with expected crc 2023-10-01T11:13:40.214 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:40 smithi078 bash[75299]: cluster 2023-10-01T11:13:39.129010+0000 osd.5 (osd.5) 404 : cluster 3 failed to encode map e376 with expected crc 2023-10-01T11:13:40.214 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:40 smithi078 bash[78274]: cluster 2023-10-01T11:13:38.468826+0000 mgr.y (mgr.25072) 1691 : cluster 0 pgmap v1827: 89 pgs: 89 active+clean; 766 MiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 526 op/s 2023-10-01T11:13:40.214 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:40 smithi078 bash[78274]: audit 2023-10-01T11:13:38.505474+0000 mgr.y (mgr.25072) 1692 : audit 0 from='client.46919 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T11:13:40.214 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:40 smithi078 bash[78274]: cluster 2023-10-01T11:13:39.101701+0000 mon.a (mon.0) 2165 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:13:40.214 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:40 smithi078 bash[78274]: cluster 2023-10-01T11:13:39.119404+0000 osd.7 (osd.7) 422 : cluster 3 failed to encode map e376 with expected crc 2023-10-01T11:13:40.214 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:40 smithi078 bash[78274]: audit 2023-10-01T11:13:39.121399+0000 mon.a (mon.0) 2166 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T11:13:40.215 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:40 smithi078 bash[78274]: cluster 2023-10-01T11:13:39.121575+0000 mon.a (mon.0) 2167 : cluster 0 osdmap e376: 8 total, 8 up, 8 in 2023-10-01T11:13:40.215 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:40 smithi078 bash[78274]: cluster 2023-10-01T11:13:39.123183+0000 osd.7 (osd.7) 423 : cluster 3 failed to encode map e376 with expected crc 2023-10-01T11:13:40.215 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:40 smithi078 bash[78274]: cluster 2023-10-01T11:13:39.123814+0000 osd.6 (osd.6) 363 : cluster 3 failed to encode map e376 with expected crc 2023-10-01T11:13:40.215 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:40 smithi078 bash[78274]: cluster 2023-10-01T11:13:39.129010+0000 osd.5 (osd.5) 404 : cluster 3 failed to encode map e376 with expected crc 2023-10-01T11:13:40.232 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:40.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:40.234 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:13:40.331 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:13:40 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:13:40] "GET /metrics HTTP/1.1" 200 36381 "" "Prometheus/2.33.4" 2023-10-01T11:13:40.335 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:40.359 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:40.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:40.360 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:13:40.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:40 smithi173 bash[58816]: cluster 2023-10-01T11:13:38.468826+0000 mgr.y (mgr.25072) 1691 : cluster 0 pgmap v1827: 89 pgs: 89 active+clean; 766 MiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 526 op/s 2023-10-01T11:13:40.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:40 smithi173 bash[58816]: audit 2023-10-01T11:13:38.505474+0000 mgr.y (mgr.25072) 1692 : audit 0 from='client.46919 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T11:13:40.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:40 smithi173 bash[58816]: cluster 2023-10-01T11:13:39.101701+0000 mon.a (mon.0) 2165 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:13:40.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:40 smithi173 bash[58816]: cluster 2023-10-01T11:13:39.119404+0000 osd.7 (osd.7) 422 : cluster 3 failed to encode map e376 with expected crc 2023-10-01T11:13:40.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:40 smithi173 bash[58816]: audit 2023-10-01T11:13:39.121399+0000 mon.a (mon.0) 2166 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T11:13:40.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:40 smithi173 bash[58816]: cluster 2023-10-01T11:13:39.121575+0000 mon.a (mon.0) 2167 : cluster 0 osdmap e376: 8 total, 8 up, 8 in 2023-10-01T11:13:40.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:40 smithi173 bash[58816]: cluster 2023-10-01T11:13:39.123183+0000 osd.7 (osd.7) 423 : cluster 3 failed to encode map e376 with expected crc 2023-10-01T11:13:40.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:40 smithi173 bash[58816]: cluster 2023-10-01T11:13:39.123814+0000 osd.6 (osd.6) 363 : cluster 3 failed to encode map e376 with expected crc 2023-10-01T11:13:40.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:40 smithi173 bash[58816]: cluster 2023-10-01T11:13:39.129010+0000 osd.5 (osd.5) 404 : cluster 3 failed to encode map e376 with expected crc 2023-10-01T11:13:40.461 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:40.501 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:40.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:40.507 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:13:40.608 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:40.640 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:40.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:40.646 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:13:40.747 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:40.784 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:40.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:40.790 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:13:40.891 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:40.918 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:40.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:40.921 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:13:41.022 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:41.056 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:41.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:41.063 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:13:41.163 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:41.171 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:41 smithi173 bash[58816]: audit 2023-10-01T11:13:40.975530+0000 mon.a (mon.0) 2168 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T11:13:41.194 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:41.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:41.210 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:13:41.310 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:41.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:41 smithi078 bash[75299]: audit 2023-10-01T11:13:40.975530+0000 mon.a (mon.0) 2168 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T11:13:41.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:41 smithi078 bash[78274]: audit 2023-10-01T11:13:40.975530+0000 mon.a (mon.0) 2168 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T11:13:41.337 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:41.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:41.341 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:13:41.442 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:41.467 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:41.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:41.471 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:13:41.572 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:41.607 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:41.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:41.613 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:13:41.714 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:41.747 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:41.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:41.752 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:13:41.852 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:41.878 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:41.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:41.882 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:13:41.945 INFO:teuthology.orchestra.run.smithi078.stdout:{"pg_ready":true,"pg_map":{"version":1827,"stamp":"2023-10-01T11:13:38.468582+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":803215288,"num_objects":12272,"num_object_clones":0,"num_object_copies":36816,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":12272,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":198301,"num_write_kb":6731599,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":100,"num_bytes_recovered":5347666,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":143635,"ondisk_log_size":143635,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":11038684,"kb_used_data":2370140,"kb_used_omap":30,"kb_used_meta":8668257,"kb_avail":738922532,"statfs":{"total":767960285184,"available":756656672768,"internally_reserved":0,"allocated":2427023360,"data_stored":2418254119,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31502,"internal_metadata":8876295410},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-414318592,"num_objects":-6322,"num_object_clones":0,"num_object_copies":-18966,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-6322,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6322,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003606"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":358,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.014018+0000","last_change":"2023-10-01T11:04:02.725956+0000","last_active":"2023-10-01T11:13:12.014018+0000","last_peered":"2023-10-01T11:13:12.014018+0000","last_clean":"2023-10-01T11:13:12.014018+0000","last_became_active":"2023-10-01T11:04:02.725317+0000","last_became_peered":"2023-10-01T11:04:02.725317+0000","last_unstale":"2023-10-01T11:13:12.014018+0000","last_undegraded":"2023-10-01T11:13:12.014018+0000","last_fullsized":"2023-10-01T11:13:12.014018+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:34:07.675650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":363,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.025918+0000","last_change":"2023-10-01T10:38:16.025881+0000","last_active":"2023-10-01T11:13:12.025918+0000","last_peered":"2023-10-01T11:13:12.025918+0000","last_clean":"2023-10-01T11:13:12.025918+0000","last_became_active":"2023-10-01T10:38:16.025518+0000","last_became_peered":"2023-10-01T10:38:16.025518+0000","last_unstale":"2023-10-01T11:13:12.025918+0000","last_undegraded":"2023-10-01T11:13:12.025918+0000","last_fullsized":"2023-10-01T11:13:12.025918+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:48:37.826604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":172,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.039852+0000","last_change":"2023-10-01T11:08:46.870944+0000","last_active":"2023-10-01T11:13:12.039852+0000","last_peered":"2023-10-01T11:13:12.039852+0000","last_clean":"2023-10-01T11:13:12.039852+0000","last_became_active":"2023-10-01T11:08:46.870425+0000","last_became_peered":"2023-10-01T11:08:46.870425+0000","last_unstale":"2023-10-01T11:13:12.039852+0000","last_undegraded":"2023-10-01T11:13:12.039852+0000","last_fullsized":"2023-10-01T11:13:12.039852+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:28:03.064900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":381,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.026091+0000","last_change":"2023-10-01T11:06:11.786067+0000","last_active":"2023-10-01T11:13:12.026091+0000","last_peered":"2023-10-01T11:13:12.026091+0000","last_clean":"2023-10-01T11:13:12.026091+0000","last_became_active":"2023-10-01T11:06:11.785586+0000","last_became_peered":"2023-10-01T11:06:11.785586+0000","last_unstale":"2023-10-01T11:13:12.026091+0000","last_undegraded":"2023-10-01T11:13:12.026091+0000","last_fullsized":"2023-10-01T11:13:12.026091+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:09:53.087537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":364,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.026096+0000","last_change":"2023-10-01T10:37:42.766600+0000","last_active":"2023-10-01T11:13:12.026096+0000","last_peered":"2023-10-01T11:13:12.026096+0000","last_clean":"2023-10-01T11:13:12.026096+0000","last_became_active":"2023-10-01T10:37:42.766051+0000","last_became_peered":"2023-10-01T10:37:42.766051+0000","last_unstale":"2023-10-01T11:13:12.026096+0000","last_undegraded":"2023-10-01T11:13:12.026096+0000","last_fullsized":"2023-10-01T11:13:12.026096+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:51:17.707963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":61,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.025579+0000","last_change":"2023-10-01T11:05:33.800368+0000","last_active":"2023-10-01T11:13:12.025579+0000","last_peered":"2023-10-01T11:13:12.025579+0000","last_clean":"2023-10-01T11:13:12.025579+0000","last_became_active":"2023-10-01T11:05:33.799877+0000","last_became_peered":"2023-10-01T11:05:33.799877+0000","last_unstale":"2023-10-01T11:13:12.025579+0000","last_undegraded":"2023-10-01T11:13:12.025579+0000","last_fullsized":"2023-10-01T11:13:12.025579+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":135,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.026129+0000","last_change":"2023-10-01T11:04:40.881723+0000","last_active":"2023-10-01T11:13:12.026129+0000","last_peered":"2023-10-01T11:13:12.026129+0000","last_clean":"2023-10-01T11:13:12.026129+0000","last_became_active":"2023-10-01T11:04:40.881381+0000","last_became_peered":"2023-10-01T11:04:40.881381+0000","last_unstale":"2023-10-01T11:13:12.026129+0000","last_undegraded":"2023-10-01T11:13:12.026129+0000","last_fullsized":"2023-10-01T11:13:12.026129+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":403,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.026175+0000","last_change":"2023-10-01T11:05:33.782561+0000","last_active":"2023-10-01T11:13:12.026175+0000","last_peered":"2023-10-01T11:13:12.026175+0000","last_clean":"2023-10-01T11:13:12.026175+0000","last_became_active":"2023-10-01T11:05:33.782066+0000","last_became_peered":"2023-10-01T11:05:33.782066+0000","last_unstale":"2023-10-01T11:13:12.026175+0000","last_undegraded":"2023-10-01T11:13:12.026175+0000","last_fullsized":"2023-10-01T11:13:12.026175+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:57:06.451343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":378,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.025202+0000","last_change":"2023-10-01T11:06:11.792061+0000","last_active":"2023-10-01T11:13:12.025202+0000","last_peered":"2023-10-01T11:13:12.025202+0000","last_clean":"2023-10-01T11:13:12.025202+0000","last_became_active":"2023-10-01T11:06:11.789129+0000","last_became_peered":"2023-10-01T11:06:11.789129+0000","last_unstale":"2023-10-01T11:13:12.025202+0000","last_undegraded":"2023-10-01T11:13:12.025202+0000","last_fullsized":"2023-10-01T11:13:12.025202+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":423,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.028615+0000","last_change":"2023-10-01T11:06:11.807749+0000","last_active":"2023-10-01T11:13:12.028615+0000","last_peered":"2023-10-01T11:13:12.028615+0000","last_clean":"2023-10-01T11:13:12.028615+0000","last_became_active":"2023-10-01T11:06:11.807375+0000","last_became_peered":"2023-10-01T11:06:11.807375+0000","last_unstale":"2023-10-01T11:13:12.028615+0000","last_undegraded":"2023-10-01T11:13:12.028615+0000","last_fullsized":"2023-10-01T11:13:12.028615+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:44:05.945387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":282,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.025841+0000","last_change":"2023-10-01T11:06:11.816274+0000","last_active":"2023-10-01T11:13:12.025841+0000","last_peered":"2023-10-01T11:13:12.025841+0000","last_clean":"2023-10-01T11:13:12.025841+0000","last_became_active":"2023-10-01T11:06:11.790196+0000","last_became_peered":"2023-10-01T11:06:11.790196+0000","last_unstale":"2023-10-01T11:13:12.025841+0000","last_undegraded":"2023-10-01T11:13:12.025841+0000","last_fullsized":"2023-10-01T11:13:12.025841+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:09:46.957797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":158,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.022332+0000","last_change":"2023-10-01T11:08:46.887634+0000","last_active":"2023-10-01T11:13:17.022332+0000","last_peered":"2023-10-01T11:13:17.022332+0000","last_clean":"2023-10-01T11:13:17.022332+0000","last_became_active":"2023-10-01T11:08:46.887226+0000","last_became_peered":"2023-10-01T11:08:46.887226+0000","last_unstale":"2023-10-01T11:13:17.022332+0000","last_undegraded":"2023-10-01T11:13:17.022332+0000","last_fullsized":"2023-10-01T11:13:17.022332+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:41:08.771365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":487,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.040170+0000","last_change":"2023-10-01T11:06:11.836035+0000","last_active":"2023-10-01T11:13:17.040170+0000","last_peered":"2023-10-01T11:13:17.040170+0000","last_clean":"2023-10-01T11:13:17.040170+0000","last_became_active":"2023-10-01T11:06:11.794834+0000","last_became_peered":"2023-10-01T11:06:11.794834+0000","last_unstale":"2023-10-01T11:13:17.040170+0000","last_undegraded":"2023-10-01T11:13:17.040170+0000","last_fullsized":"2023-10-01T11:13:17.040170+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:25:13.103100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":33138,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":382,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.039778+0000","last_change":"2023-10-01T11:05:33.782898+0000","last_active":"2023-10-01T11:13:17.039778+0000","last_peered":"2023-10-01T11:13:17.039778+0000","last_clean":"2023-10-01T11:13:17.039778+0000","last_became_active":"2023-10-01T11:05:33.782135+0000","last_became_peered":"2023-10-01T11:05:33.782135+0000","last_unstale":"2023-10-01T11:13:17.039778+0000","last_undegraded":"2023-10-01T11:13:17.039778+0000","last_fullsized":"2023-10-01T11:13:17.039778+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:56:49.807416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":128,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.014256+0000","last_change":"2023-10-01T11:06:55.803414+0000","last_active":"2023-10-01T11:13:12.014256+0000","last_peered":"2023-10-01T11:13:12.014256+0000","last_clean":"2023-10-01T11:13:12.014256+0000","last_became_active":"2023-10-01T11:06:55.802738+0000","last_became_peered":"2023-10-01T11:06:55.802738+0000","last_unstale":"2023-10-01T11:13:12.014256+0000","last_undegraded":"2023-10-01T11:13:12.014256+0000","last_fullsized":"2023-10-01T11:13:12.014256+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:55:57.911799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":408,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.027093+0000","last_change":"2023-10-01T11:06:11.789794+0000","last_active":"2023-10-01T11:13:12.027093+0000","last_peered":"2023-10-01T11:13:12.027093+0000","last_clean":"2023-10-01T11:13:12.027093+0000","last_became_active":"2023-10-01T11:06:11.789452+0000","last_became_peered":"2023-10-01T11:06:11.789452+0000","last_unstale":"2023-10-01T11:13:12.027093+0000","last_undegraded":"2023-10-01T11:13:12.027093+0000","last_fullsized":"2023-10-01T11:13:12.027093+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:18:02.050694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":368,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.027735+0000","last_change":"2023-10-01T11:06:11.784794+0000","last_active":"2023-10-01T11:13:12.027735+0000","last_peered":"2023-10-01T11:13:12.027735+0000","last_clean":"2023-10-01T11:13:12.027735+0000","last_became_active":"2023-10-01T11:06:11.784368+0000","last_became_peered":"2023-10-01T11:06:11.784368+0000","last_unstale":"2023-10-01T11:13:12.027735+0000","last_undegraded":"2023-10-01T11:13:12.027735+0000","last_fullsized":"2023-10-01T11:13:12.027735+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:06:52.307449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":395,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.023254+0000","last_change":"2023-10-01T11:06:11.794671+0000","last_active":"2023-10-01T11:13:12.023254+0000","last_peered":"2023-10-01T11:13:12.023254+0000","last_clean":"2023-10-01T11:13:12.023254+0000","last_became_active":"2023-10-01T11:06:11.794121+0000","last_became_peered":"2023-10-01T11:06:11.794121+0000","last_unstale":"2023-10-01T11:13:12.023254+0000","last_undegraded":"2023-10-01T11:13:12.023254+0000","last_fullsized":"2023-10-01T11:13:12.023254+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":385,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.013764+0000","last_change":"2023-10-01T11:06:11.796259+0000","last_active":"2023-10-01T11:13:12.013764+0000","last_peered":"2023-10-01T11:13:12.013764+0000","last_clean":"2023-10-01T11:13:12.013764+0000","last_became_active":"2023-10-01T11:06:11.795870+0000","last_became_peered":"2023-10-01T11:06:11.795870+0000","last_unstale":"2023-10-01T11:13:12.013764+0000","last_undegraded":"2023-10-01T11:13:12.013764+0000","last_fullsized":"2023-10-01T11:13:12.013764+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:49:21.072163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":368,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.026020+0000","last_change":"2023-10-01T11:06:11.785871+0000","last_active":"2023-10-01T11:13:12.026020+0000","last_peered":"2023-10-01T11:13:12.026020+0000","last_clean":"2023-10-01T11:13:12.026020+0000","last_became_active":"2023-10-01T11:06:11.785183+0000","last_became_peered":"2023-10-01T11:06:11.785183+0000","last_unstale":"2023-10-01T11:13:12.026020+0000","last_undegraded":"2023-10-01T11:13:12.026020+0000","last_fullsized":"2023-10-01T11:13:12.026020+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:07:55.556404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":71,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.025134+0000","last_change":"2023-10-01T11:05:33.900352+0000","last_active":"2023-10-01T11:13:12.025134+0000","last_peered":"2023-10-01T11:13:12.025134+0000","last_clean":"2023-10-01T11:13:12.025134+0000","last_became_active":"2023-10-01T11:05:33.788906+0000","last_became_peered":"2023-10-01T11:05:33.788906+0000","last_unstale":"2023-10-01T11:13:12.025134+0000","last_undegraded":"2023-10-01T11:13:12.025134+0000","last_fullsized":"2023-10-01T11:13:12.025134+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":384,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.026051+0000","last_change":"2023-10-01T11:06:11.798727+0000","last_active":"2023-10-01T11:13:12.026051+0000","last_peered":"2023-10-01T11:13:12.026051+0000","last_clean":"2023-10-01T11:13:12.026051+0000","last_became_active":"2023-10-01T11:06:11.798388+0000","last_became_peered":"2023-10-01T11:06:11.798388+0000","last_unstale":"2023-10-01T11:13:12.026051+0000","last_undegraded":"2023-10-01T11:13:12.026051+0000","last_fullsized":"2023-10-01T11:13:12.026051+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:10:21.576627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.2","version":"375'12567","reported_seq":12597,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:35.271177+0000","last_change":"2023-10-01T11:09:21.161135+0000","last_active":"2023-10-01T11:13:35.271177+0000","last_peered":"2023-10-01T11:13:35.271177+0000","last_clean":"2023-10-01T11:13:35.271177+0000","last_became_active":"2023-10-01T11:09:21.160719+0000","last_became_peered":"2023-10-01T11:09:21.160719+0000","last_unstale":"2023-10-01T11:13:35.271177+0000","last_undegraded":"2023-10-01T11:13:35.271177+0000","last_fullsized":"2023-10-01T11:13:35.271177+0000","mapping_epoch":356,"log_start":"360'4600","ondisk_log_start":"360'4600","created":356,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_clean_scrub_stamp":"2023-10-01T11:09:20.132185+0000","objects_scrubbed":0,"log_size":7967,"log_dups_size":0,"ondisk_log_size":7967,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:53:24.785514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49348608,"num_objects":753,"num_object_clones":0,"num_object_copies":2259,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":753,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12567,"num_write_kb":426240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":274,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.021673+0000","last_change":"2023-10-01T11:06:11.840628+0000","last_active":"2023-10-01T11:13:12.021673+0000","last_peered":"2023-10-01T11:13:12.021673+0000","last_clean":"2023-10-01T11:13:12.021673+0000","last_became_active":"2023-10-01T11:06:11.800187+0000","last_became_peered":"2023-10-01T11:06:11.800187+0000","last_unstale":"2023-10-01T11:13:12.021673+0000","last_undegraded":"2023-10-01T11:13:12.021673+0000","last_fullsized":"2023-10-01T11:13:12.021673+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:24:48.900425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":416,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.018306+0000","last_change":"2023-10-01T11:06:11.830255+0000","last_active":"2023-10-01T11:13:12.018306+0000","last_peered":"2023-10-01T11:13:12.018306+0000","last_clean":"2023-10-01T11:13:12.018306+0000","last_became_active":"2023-10-01T11:06:11.829847+0000","last_became_peered":"2023-10-01T11:06:11.829847+0000","last_unstale":"2023-10-01T11:13:12.018306+0000","last_undegraded":"2023-10-01T11:13:12.018306+0000","last_fullsized":"2023-10-01T11:13:12.018306+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.3","version":"375'12535","reported_seq":12565,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:36.264718+0000","last_change":"2023-10-01T11:09:22.023134+0000","last_active":"2023-10-01T11:13:36.264718+0000","last_peered":"2023-10-01T11:13:36.264718+0000","last_clean":"2023-10-01T11:13:36.264718+0000","last_became_active":"2023-10-01T11:09:22.022727+0000","last_became_peered":"2023-10-01T11:09:22.022727+0000","last_unstale":"2023-10-01T11:13:36.264718+0000","last_undegraded":"2023-10-01T11:13:36.264718+0000","last_fullsized":"2023-10-01T11:13:36.264718+0000","mapping_epoch":356,"log_start":"360'4400","ondisk_log_start":"360'4400","created":356,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_clean_scrub_stamp":"2023-10-01T11:09:20.132185+0000","objects_scrubbed":0,"log_size":8135,"log_dups_size":0,"ondisk_log_size":8135,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:24:57.467745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":45547520,"num_objects":695,"num_object_clones":0,"num_object_copies":2085,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":695,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12535,"num_write_kb":423360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":233,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.022065+0000","last_change":"2023-10-01T11:06:11.813139+0000","last_active":"2023-10-01T11:13:12.022065+0000","last_peered":"2023-10-01T11:13:12.022065+0000","last_clean":"2023-10-01T11:13:12.022065+0000","last_became_active":"2023-10-01T11:06:11.790124+0000","last_became_peered":"2023-10-01T11:06:11.790124+0000","last_unstale":"2023-10-01T11:13:12.022065+0000","last_undegraded":"2023-10-01T11:13:12.022065+0000","last_fullsized":"2023-10-01T11:13:12.022065+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:30:46.077244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":362,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.013850+0000","last_change":"2023-10-01T11:04:02.724840+0000","last_active":"2023-10-01T11:13:12.013850+0000","last_peered":"2023-10-01T11:13:12.013850+0000","last_clean":"2023-10-01T11:13:12.013850+0000","last_became_active":"2023-10-01T11:04:02.724476+0000","last_became_peered":"2023-10-01T11:04:02.724476+0000","last_unstale":"2023-10-01T11:13:12.013850+0000","last_undegraded":"2023-10-01T11:13:12.013850+0000","last_fullsized":"2023-10-01T11:13:12.013850+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:05:08.882495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.0","version":"375'12462","reported_seq":12492,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:34.056803+0000","last_change":"2023-10-01T11:09:22.023014+0000","last_active":"2023-10-01T11:13:34.056803+0000","last_peered":"2023-10-01T11:13:34.056803+0000","last_clean":"2023-10-01T11:13:34.056803+0000","last_became_active":"2023-10-01T11:09:22.022611+0000","last_became_peered":"2023-10-01T11:09:22.022611+0000","last_unstale":"2023-10-01T11:13:34.056803+0000","last_undegraded":"2023-10-01T11:13:34.056803+0000","last_fullsized":"2023-10-01T11:13:34.056803+0000","mapping_epoch":356,"log_start":"359'2400","ondisk_log_start":"359'2400","created":356,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_clean_scrub_stamp":"2023-10-01T11:09:20.132185+0000","objects_scrubbed":0,"log_size":10062,"log_dups_size":0,"ondisk_log_size":10062,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:23:59.916230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50855936,"num_objects":776,"num_object_clones":0,"num_object_copies":2328,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":776,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12462,"num_write_kb":423616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":344,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.023299+0000","last_change":"2023-10-01T11:06:11.820664+0000","last_active":"2023-10-01T11:13:12.023299+0000","last_peered":"2023-10-01T11:13:12.023299+0000","last_clean":"2023-10-01T11:13:12.023299+0000","last_became_active":"2023-10-01T11:06:11.792931+0000","last_became_peered":"2023-10-01T11:06:11.792931+0000","last_unstale":"2023-10-01T11:13:12.023299+0000","last_undegraded":"2023-10-01T11:13:12.023299+0000","last_fullsized":"2023-10-01T11:13:12.023299+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":700,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":70,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.025725+0000","last_change":"2023-10-01T11:06:11.795096+0000","last_active":"2023-10-01T11:13:12.025725+0000","last_peered":"2023-10-01T11:13:12.025725+0000","last_clean":"2023-10-01T11:13:12.025725+0000","last_became_active":"2023-10-01T11:06:11.794716+0000","last_became_peered":"2023-10-01T11:06:11.794716+0000","last_unstale":"2023-10-01T11:13:12.025725+0000","last_undegraded":"2023-10-01T11:13:12.025725+0000","last_fullsized":"2023-10-01T11:13:12.025725+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1","version":"375'12385","reported_seq":12415,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:36.296534+0000","last_change":"2023-10-01T11:09:21.156985+0000","last_active":"2023-10-01T11:13:36.296534+0000","last_peered":"2023-10-01T11:13:36.296534+0000","last_clean":"2023-10-01T11:13:36.296534+0000","last_became_active":"2023-10-01T11:09:21.156452+0000","last_became_peered":"2023-10-01T11:09:21.156452+0000","last_unstale":"2023-10-01T11:13:36.296534+0000","last_undegraded":"2023-10-01T11:13:36.296534+0000","last_fullsized":"2023-10-01T11:13:36.296534+0000","mapping_epoch":356,"log_start":"360'4200","ondisk_log_start":"360'4200","created":356,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_clean_scrub_stamp":"2023-10-01T11:09:20.132185+0000","objects_scrubbed":0,"log_size":8185,"log_dups_size":0,"ondisk_log_size":8185,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:15:04.145939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":45154304,"num_objects":689,"num_object_clones":0,"num_object_copies":2067,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":689,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12385,"num_write_kb":418368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":396,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.018351+0000","last_change":"2023-10-01T11:05:33.788917+0000","last_active":"2023-10-01T11:13:12.018351+0000","last_peered":"2023-10-01T11:13:12.018351+0000","last_clean":"2023-10-01T11:13:12.018351+0000","last_became_active":"2023-10-01T11:05:33.788078+0000","last_became_peered":"2023-10-01T11:05:33.788078+0000","last_unstale":"2023-10-01T11:13:12.018351+0000","last_undegraded":"2023-10-01T11:13:12.018351+0000","last_fullsized":"2023-10-01T11:13:12.018351+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":126,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.024096+0000","last_change":"2023-10-01T11:08:46.873065+0000","last_active":"2023-10-01T11:13:17.024096+0000","last_peered":"2023-10-01T11:13:17.024096+0000","last_clean":"2023-10-01T11:13:17.024096+0000","last_became_active":"2023-10-01T11:08:46.872692+0000","last_became_peered":"2023-10-01T11:08:46.872692+0000","last_unstale":"2023-10-01T11:13:17.024096+0000","last_undegraded":"2023-10-01T11:13:17.024096+0000","last_fullsized":"2023-10-01T11:13:17.024096+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:27:32.806247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.6","version":"375'12472","reported_seq":12501,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:34.061432+0000","last_change":"2023-10-01T11:09:22.025630+0000","last_active":"2023-10-01T11:13:34.061432+0000","last_peered":"2023-10-01T11:13:34.061432+0000","last_clean":"2023-10-01T11:13:34.061432+0000","last_became_active":"2023-10-01T11:09:22.025164+0000","last_became_peered":"2023-10-01T11:09:22.025164+0000","last_unstale":"2023-10-01T11:13:34.061432+0000","last_undegraded":"2023-10-01T11:13:34.061432+0000","last_fullsized":"2023-10-01T11:13:34.061432+0000","mapping_epoch":356,"log_start":"359'2400","ondisk_log_start":"359'2400","created":356,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_clean_scrub_stamp":"2023-10-01T11:09:20.132185+0000","objects_scrubbed":0,"log_size":10072,"log_dups_size":0,"ondisk_log_size":10072,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:05:50.808903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56754176,"num_objects":866,"num_object_clones":0,"num_object_copies":2598,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":866,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12472,"num_write_kb":426816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":352,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.039109+0000","last_change":"2023-10-01T10:38:16.022493+0000","last_active":"2023-10-01T11:13:12.039109+0000","last_peered":"2023-10-01T11:13:12.039109+0000","last_clean":"2023-10-01T11:13:12.039109+0000","last_became_active":"2023-10-01T10:38:16.022033+0000","last_became_peered":"2023-10-01T10:38:16.022033+0000","last_unstale":"2023-10-01T11:13:12.039109+0000","last_undegraded":"2023-10-01T11:13:12.039109+0000","last_fullsized":"2023-10-01T11:13:12.039109+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:49:05.262763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":392,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.017458+0000","last_change":"2023-10-01T11:05:33.785133+0000","last_active":"2023-10-01T11:13:12.017458+0000","last_peered":"2023-10-01T11:13:12.017458+0000","last_clean":"2023-10-01T11:13:12.017458+0000","last_became_active":"2023-10-01T11:05:33.784749+0000","last_became_peered":"2023-10-01T11:05:33.784749+0000","last_unstale":"2023-10-01T11:13:12.017458+0000","last_undegraded":"2023-10-01T11:13:12.017458+0000","last_fullsized":"2023-10-01T11:13:12.017458+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:08:01.664432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.7","version":"375'12376","reported_seq":12400,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:33.948506+0000","last_change":"2023-10-01T11:09:22.022235+0000","last_active":"2023-10-01T11:13:33.948506+0000","last_peered":"2023-10-01T11:13:33.948506+0000","last_clean":"2023-10-01T11:13:33.948506+0000","last_became_active":"2023-10-01T11:09:22.021331+0000","last_became_peered":"2023-10-01T11:09:22.021331+0000","last_unstale":"2023-10-01T11:13:33.948506+0000","last_undegraded":"2023-10-01T11:13:33.948506+0000","last_fullsized":"2023-10-01T11:13:33.948506+0000","mapping_epoch":356,"log_start":"359'2300","ondisk_log_start":"359'2300","created":356,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_clean_scrub_stamp":"2023-10-01T11:09:20.132185+0000","objects_scrubbed":0,"log_size":10076,"log_dups_size":0,"ondisk_log_size":10076,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53084160,"num_objects":810,"num_object_clones":0,"num_object_copies":2430,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":810,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12376,"num_write_kb":421952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":365,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.013823+0000","last_change":"2023-10-01T11:05:33.782142+0000","last_active":"2023-10-01T11:13:12.013823+0000","last_peered":"2023-10-01T11:13:12.013823+0000","last_clean":"2023-10-01T11:13:12.013823+0000","last_became_active":"2023-10-01T11:05:33.781725+0000","last_became_peered":"2023-10-01T11:05:33.781725+0000","last_unstale":"2023-10-01T11:13:12.013823+0000","last_undegraded":"2023-10-01T11:13:12.013823+0000","last_fullsized":"2023-10-01T11:13:12.013823+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:58:37.958611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":352,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.024588+0000","last_change":"2023-10-01T11:04:02.727282+0000","last_active":"2023-10-01T11:13:12.024588+0000","last_peered":"2023-10-01T11:13:12.024588+0000","last_clean":"2023-10-01T11:13:12.024588+0000","last_became_active":"2023-10-01T11:04:02.726818+0000","last_became_peered":"2023-10-01T11:04:02.726818+0000","last_unstale":"2023-10-01T11:13:12.024588+0000","last_undegraded":"2023-10-01T11:13:12.024588+0000","last_fullsized":"2023-10-01T11:13:12.024588+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:11:11.868291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.4","version":"375'12221","reported_seq":12248,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:33.485303+0000","last_change":"2023-10-01T11:09:21.163719+0000","last_active":"2023-10-01T11:13:33.485303+0000","last_peered":"2023-10-01T11:13:33.485303+0000","last_clean":"2023-10-01T11:13:33.485303+0000","last_became_active":"2023-10-01T11:09:21.163294+0000","last_became_peered":"2023-10-01T11:09:21.163294+0000","last_unstale":"2023-10-01T11:13:33.485303+0000","last_undegraded":"2023-10-01T11:13:33.485303+0000","last_fullsized":"2023-10-01T11:13:33.485303+0000","mapping_epoch":356,"log_start":"359'2500","ondisk_log_start":"359'2500","created":356,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_clean_scrub_stamp":"2023-10-01T11:09:20.132185+0000","objects_scrubbed":0,"log_size":9721,"log_dups_size":0,"ondisk_log_size":9721,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50790400,"num_objects":775,"num_object_clones":0,"num_object_copies":2325,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":775,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12221,"num_write_kb":415872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":396,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.014155+0000","last_change":"2023-10-01T11:06:11.790944+0000","last_active":"2023-10-01T11:13:12.014155+0000","last_peered":"2023-10-01T11:13:12.014155+0000","last_clean":"2023-10-01T11:13:12.014155+0000","last_became_active":"2023-10-01T11:06:11.790151+0000","last_became_peered":"2023-10-01T11:06:11.790151+0000","last_unstale":"2023-10-01T11:13:12.014155+0000","last_undegraded":"2023-10-01T11:13:12.014155+0000","last_fullsized":"2023-10-01T11:13:12.014155+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:05:49.161868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":394,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.022120+0000","last_change":"2023-10-01T11:05:33.781934+0000","last_active":"2023-10-01T11:13:12.022120+0000","last_peered":"2023-10-01T11:13:12.022120+0000","last_clean":"2023-10-01T11:13:12.022120+0000","last_became_active":"2023-10-01T11:05:33.781555+0000","last_became_peered":"2023-10-01T11:05:33.781555+0000","last_unstale":"2023-10-01T11:13:12.022120+0000","last_undegraded":"2023-10-01T11:13:12.022120+0000","last_fullsized":"2023-10-01T11:13:12.022120+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:59:22.019053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.5","version":"375'12468","reported_seq":12498,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:34.453555+0000","last_change":"2023-10-01T11:09:22.023719+0000","last_active":"2023-10-01T11:13:34.453555+0000","last_peered":"2023-10-01T11:13:34.453555+0000","last_clean":"2023-10-01T11:13:34.453555+0000","last_became_active":"2023-10-01T11:09:22.023016+0000","last_became_peered":"2023-10-01T11:09:22.023016+0000","last_unstale":"2023-10-01T11:13:34.453555+0000","last_undegraded":"2023-10-01T11:13:34.453555+0000","last_fullsized":"2023-10-01T11:13:34.453555+0000","mapping_epoch":356,"log_start":"360'4700","ondisk_log_start":"360'4700","created":356,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_clean_scrub_stamp":"2023-10-01T11:09:20.132185+0000","objects_scrubbed":0,"log_size":7768,"log_dups_size":0,"ondisk_log_size":7768,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:55:17.819499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52297728,"num_objects":798,"num_object_clones":0,"num_object_copies":2394,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":798,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12468,"num_write_kb":424512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":387,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.026012+0000","last_change":"2023-10-01T11:06:11.784722+0000","last_active":"2023-10-01T11:13:12.026012+0000","last_peered":"2023-10-01T11:13:12.026012+0000","last_clean":"2023-10-01T11:13:12.026012+0000","last_became_active":"2023-10-01T11:06:11.784348+0000","last_became_peered":"2023-10-01T11:06:11.784348+0000","last_unstale":"2023-10-01T11:13:12.026012+0000","last_undegraded":"2023-10-01T11:13:12.026012+0000","last_fullsized":"2023-10-01T11:13:12.026012+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:40:29.283179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":369,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:14.023010+0000","last_change":"2023-10-01T10:38:16.019755+0000","last_active":"2023-10-01T11:13:14.023010+0000","last_peered":"2023-10-01T11:13:14.023010+0000","last_clean":"2023-10-01T11:13:14.023010+0000","last_became_active":"2023-10-01T10:38:16.019397+0000","last_became_peered":"2023-10-01T10:38:16.019397+0000","last_unstale":"2023-10-01T11:13:14.023010+0000","last_undegraded":"2023-10-01T11:13:14.023010+0000","last_fullsized":"2023-10-01T11:13:14.023010+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:42:56.623225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.8","version":"375'12439","reported_seq":12468,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:35.285383+0000","last_change":"2023-10-01T11:09:22.022986+0000","last_active":"2023-10-01T11:13:35.285383+0000","last_peered":"2023-10-01T11:13:35.285383+0000","last_clean":"2023-10-01T11:13:35.285383+0000","last_became_active":"2023-10-01T11:09:22.022553+0000","last_became_peered":"2023-10-01T11:09:22.022553+0000","last_unstale":"2023-10-01T11:13:35.285383+0000","last_undegraded":"2023-10-01T11:13:35.285383+0000","last_fullsized":"2023-10-01T11:13:35.285383+0000","mapping_epoch":356,"log_start":"360'4500","ondisk_log_start":"360'4500","created":356,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_clean_scrub_stamp":"2023-10-01T11:09:20.132185+0000","objects_scrubbed":0,"log_size":7939,"log_dups_size":0,"ondisk_log_size":7939,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T22:14:03.287229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":45809664,"num_objects":699,"num_object_clones":0,"num_object_copies":2097,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":699,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12439,"num_write_kb":420416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":279,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.045211+0000","last_change":"2023-10-01T11:06:11.788951+0000","last_active":"2023-10-01T11:13:12.045211+0000","last_peered":"2023-10-01T11:13:12.045211+0000","last_clean":"2023-10-01T11:13:12.045211+0000","last_became_active":"2023-10-01T11:06:11.788617+0000","last_became_peered":"2023-10-01T11:06:11.788617+0000","last_unstale":"2023-10-01T11:13:12.045211+0000","last_undegraded":"2023-10-01T11:13:12.045211+0000","last_fullsized":"2023-10-01T11:13:12.045211+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:07:46.629250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":357,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.023132+0000","last_change":"2023-10-01T10:37:42.766814+0000","last_active":"2023-10-01T11:13:12.023132+0000","last_peered":"2023-10-01T11:13:12.023132+0000","last_clean":"2023-10-01T11:13:12.023132+0000","last_became_active":"2023-10-01T10:37:42.766451+0000","last_became_peered":"2023-10-01T10:37:42.766451+0000","last_unstale":"2023-10-01T11:13:12.023132+0000","last_undegraded":"2023-10-01T11:13:12.023132+0000","last_fullsized":"2023-10-01T11:13:12.023132+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":159,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.041785+0000","last_change":"2023-10-01T11:08:46.870961+0000","last_active":"2023-10-01T11:13:12.041785+0000","last_peered":"2023-10-01T11:13:12.041785+0000","last_clean":"2023-10-01T11:13:12.041785+0000","last_became_active":"2023-10-01T11:08:46.870680+0000","last_became_peered":"2023-10-01T11:08:46.870680+0000","last_unstale":"2023-10-01T11:13:12.041785+0000","last_undegraded":"2023-10-01T11:13:12.041785+0000","last_fullsized":"2023-10-01T11:13:12.041785+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:36:45.187763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.e","version":"375'12353","reported_seq":12382,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:34.432104+0000","last_change":"2023-10-01T11:09:21.163877+0000","last_active":"2023-10-01T11:13:34.432104+0000","last_peered":"2023-10-01T11:13:34.432104+0000","last_clean":"2023-10-01T11:13:34.432104+0000","last_became_active":"2023-10-01T11:09:21.163510+0000","last_became_peered":"2023-10-01T11:09:21.163510+0000","last_unstale":"2023-10-01T11:13:34.432104+0000","last_undegraded":"2023-10-01T11:13:34.432104+0000","last_fullsized":"2023-10-01T11:13:34.432104+0000","mapping_epoch":356,"log_start":"360'4600","ondisk_log_start":"360'4600","created":356,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_clean_scrub_stamp":"2023-10-01T11:09:20.132185+0000","objects_scrubbed":0,"log_size":7753,"log_dups_size":0,"ondisk_log_size":7753,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:49:26.529322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49217536,"num_objects":751,"num_object_clones":0,"num_object_copies":2253,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":751,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12353,"num_write_kb":419328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":400,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.019081+0000","last_change":"2023-10-01T11:05:33.792877+0000","last_active":"2023-10-01T11:13:12.019081+0000","last_peered":"2023-10-01T11:13:12.019081+0000","last_clean":"2023-10-01T11:13:12.019081+0000","last_became_active":"2023-10-01T11:05:33.792054+0000","last_became_peered":"2023-10-01T11:05:33.792054+0000","last_unstale":"2023-10-01T11:13:12.019081+0000","last_undegraded":"2023-10-01T11:13:12.019081+0000","last_fullsized":"2023-10-01T11:13:12.019081+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":360,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.014446+0000","last_change":"2023-10-01T11:05:33.783100+0000","last_active":"2023-10-01T11:13:12.014446+0000","last_peered":"2023-10-01T11:13:12.014446+0000","last_clean":"2023-10-01T11:13:12.014446+0000","last_became_active":"2023-10-01T11:05:33.782557+0000","last_became_peered":"2023-10-01T11:05:33.782557+0000","last_unstale":"2023-10-01T11:13:12.014446+0000","last_undegraded":"2023-10-01T11:13:12.014446+0000","last_fullsized":"2023-10-01T11:13:12.014446+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:52:05.581697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":280,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:14.022943+0000","last_change":"2023-10-01T10:52:18.963437+0000","last_active":"2023-10-01T11:13:14.022943+0000","last_peered":"2023-10-01T11:13:14.022943+0000","last_clean":"2023-10-01T11:13:14.022943+0000","last_became_active":"2023-10-01T10:52:18.955165+0000","last_became_peered":"2023-10-01T10:52:18.955165+0000","last_unstale":"2023-10-01T11:13:14.022943+0000","last_undegraded":"2023-10-01T11:13:14.022943+0000","last_fullsized":"2023-10-01T11:13:14.022943+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:57:49.129100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.d","version":"375'12059","reported_seq":12087,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:33.494151+0000","last_change":"2023-10-01T11:09:21.165115+0000","last_active":"2023-10-01T11:13:33.494151+0000","last_peered":"2023-10-01T11:13:33.494151+0000","last_clean":"2023-10-01T11:13:33.494151+0000","last_became_active":"2023-10-01T11:09:21.164631+0000","last_became_peered":"2023-10-01T11:09:21.164631+0000","last_unstale":"2023-10-01T11:13:33.494151+0000","last_undegraded":"2023-10-01T11:13:33.494151+0000","last_fullsized":"2023-10-01T11:13:33.494151+0000","mapping_epoch":356,"log_start":"359'2300","ondisk_log_start":"359'2300","created":356,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_clean_scrub_stamp":"2023-10-01T11:09:20.132185+0000","objects_scrubbed":0,"log_size":9759,"log_dups_size":0,"ondisk_log_size":9759,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51904536,"num_objects":793,"num_object_clones":0,"num_object_copies":2379,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":793,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12059,"num_write_kb":411201,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":361,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.041225+0000","last_change":"2023-10-01T10:37:42.762524+0000","last_active":"2023-10-01T11:13:12.041225+0000","last_peered":"2023-10-01T11:13:12.041225+0000","last_clean":"2023-10-01T11:13:12.041225+0000","last_became_active":"2023-10-01T10:37:42.762209+0000","last_became_peered":"2023-10-01T10:37:42.762209+0000","last_unstale":"2023-10-01T11:13:12.041225+0000","last_undegraded":"2023-10-01T11:13:12.041225+0000","last_fullsized":"2023-10-01T11:13:12.041225+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:45:38.171397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":314,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.039988+0000","last_change":"2023-10-01T11:05:33.799413+0000","last_active":"2023-10-01T11:13:12.039988+0000","last_peered":"2023-10-01T11:13:12.039988+0000","last_clean":"2023-10-01T11:13:12.039988+0000","last_became_active":"2023-10-01T11:05:33.799158+0000","last_became_peered":"2023-10-01T11:05:33.799158+0000","last_unstale":"2023-10-01T11:13:12.039988+0000","last_undegraded":"2023-10-01T11:13:12.039988+0000","last_fullsized":"2023-10-01T11:13:12.039988+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:39:05.963181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":188,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.041526+0000","last_change":"2023-10-01T11:08:46.870350+0000","last_active":"2023-10-01T11:13:12.041526+0000","last_peered":"2023-10-01T11:13:12.041526+0000","last_clean":"2023-10-01T11:13:12.041526+0000","last_became_active":"2023-10-01T11:08:46.869990+0000","last_became_peered":"2023-10-01T11:08:46.869990+0000","last_unstale":"2023-10-01T11:13:12.041526+0000","last_undegraded":"2023-10-01T11:13:12.041526+0000","last_fullsized":"2023-10-01T11:13:12.041526+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:32:25.096683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.f","version":"375'12444","reported_seq":12473,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:36.277020+0000","last_change":"2023-10-01T11:09:21.156775+0000","last_active":"2023-10-01T11:13:36.277020+0000","last_peered":"2023-10-01T11:13:36.277020+0000","last_clean":"2023-10-01T11:13:36.277020+0000","last_became_active":"2023-10-01T11:09:21.156403+0000","last_became_peered":"2023-10-01T11:09:21.156403+0000","last_unstale":"2023-10-01T11:13:36.277020+0000","last_undegraded":"2023-10-01T11:13:36.277020+0000","last_fullsized":"2023-10-01T11:13:36.277020+0000","mapping_epoch":356,"log_start":"360'4300","ondisk_log_start":"360'4300","created":356,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_clean_scrub_stamp":"2023-10-01T11:09:20.132185+0000","objects_scrubbed":0,"log_size":8144,"log_dups_size":0,"ondisk_log_size":8144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:18:58.903554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":44433408,"num_objects":678,"num_object_clones":0,"num_object_copies":2034,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":678,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12444,"num_write_kb":419904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":382,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.018109+0000","last_change":"2023-10-01T11:06:11.791607+0000","last_active":"2023-10-01T11:13:12.018109+0000","last_peered":"2023-10-01T11:13:12.018109+0000","last_clean":"2023-10-01T11:13:12.018109+0000","last_became_active":"2023-10-01T11:06:11.789667+0000","last_became_peered":"2023-10-01T11:06:11.789667+0000","last_unstale":"2023-10-01T11:13:12.018109+0000","last_undegraded":"2023-10-01T11:13:12.018109+0000","last_fullsized":"2023-10-01T11:13:12.018109+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'139","reported_seq":656,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.021406+0000","last_change":"2023-10-01T11:08:15.927727+0000","last_active":"2023-10-01T11:13:12.021406+0000","last_peered":"2023-10-01T11:13:12.021406+0000","last_clean":"2023-10-01T11:13:12.021406+0000","last_became_active":"2023-10-01T11:08:15.927235+0000","last_became_peered":"2023-10-01T11:08:15.927235+0000","last_unstale":"2023-10-01T11:13:12.021406+0000","last_undegraded":"2023-10-01T11:13:12.021406+0000","last_fullsized":"2023-10-01T11:13:12.021406+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:09:17.221082+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:09:17.221082+0000","last_clean_scrub_stamp":"2023-10-01T10:09:17.221082+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":5313312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":388,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.027598+0000","last_change":"2023-10-01T11:05:33.788093+0000","last_active":"2023-10-01T11:13:12.027598+0000","last_peered":"2023-10-01T11:13:12.027598+0000","last_clean":"2023-10-01T11:13:12.027598+0000","last_became_active":"2023-10-01T11:05:33.787723+0000","last_became_peered":"2023-10-01T11:05:33.787723+0000","last_unstale":"2023-10-01T11:13:12.027598+0000","last_undegraded":"2023-10-01T11:13:12.027598+0000","last_fullsized":"2023-10-01T11:13:12.027598+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:04:55.138186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":410,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.024454+0000","last_change":"2023-10-01T11:06:11.788535+0000","last_active":"2023-10-01T11:13:12.024454+0000","last_peered":"2023-10-01T11:13:12.024454+0000","last_clean":"2023-10-01T11:13:12.024454+0000","last_became_active":"2023-10-01T11:06:11.788103+0000","last_became_peered":"2023-10-01T11:06:11.788103+0000","last_unstale":"2023-10-01T11:13:12.024454+0000","last_undegraded":"2023-10-01T11:13:12.024454+0000","last_fullsized":"2023-10-01T11:13:12.024454+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:02:49.482834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.c","version":"375'12332","reported_seq":12360,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:34.371804+0000","last_change":"2023-10-01T11:09:22.022461+0000","last_active":"2023-10-01T11:13:34.371804+0000","last_peered":"2023-10-01T11:13:34.371804+0000","last_clean":"2023-10-01T11:13:34.371804+0000","last_became_active":"2023-10-01T11:09:22.021657+0000","last_became_peered":"2023-10-01T11:09:22.021657+0000","last_unstale":"2023-10-01T11:13:34.371804+0000","last_undegraded":"2023-10-01T11:13:34.371804+0000","last_fullsized":"2023-10-01T11:13:34.371804+0000","mapping_epoch":356,"log_start":"360'4600","ondisk_log_start":"360'4600","created":356,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_clean_scrub_stamp":"2023-10-01T11:09:20.132185+0000","objects_scrubbed":0,"log_size":7732,"log_dups_size":0,"ondisk_log_size":7732,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:13:30.660488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54394880,"num_objects":830,"num_object_clones":0,"num_object_copies":2490,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":830,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12332,"num_write_kb":421184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":272,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.039821+0000","last_change":"2023-10-01T11:08:46.989647+0000","last_active":"2023-10-01T11:13:12.039821+0000","last_peered":"2023-10-01T11:13:12.039821+0000","last_clean":"2023-10-01T11:13:12.039821+0000","last_became_active":"2023-10-01T11:08:46.988979+0000","last_became_peered":"2023-10-01T11:08:46.988979+0000","last_unstale":"2023-10-01T11:13:12.039821+0000","last_undegraded":"2023-10-01T11:13:12.039821+0000","last_fullsized":"2023-10-01T11:13:12.039821+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:10:06.594141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":160,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.014072+0000","last_change":"2023-10-01T11:04:40.868156+0000","last_active":"2023-10-01T11:13:12.014072+0000","last_peered":"2023-10-01T11:13:12.014072+0000","last_clean":"2023-10-01T11:13:12.014072+0000","last_became_active":"2023-10-01T11:04:40.867864+0000","last_became_peered":"2023-10-01T11:04:40.867864+0000","last_unstale":"2023-10-01T11:13:12.014072+0000","last_undegraded":"2023-10-01T11:13:12.014072+0000","last_fullsized":"2023-10-01T11:13:12.014072+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:37:15.571777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":151,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.041478+0000","last_change":"2023-10-01T11:04:40.880090+0000","last_active":"2023-10-01T11:13:17.041478+0000","last_peered":"2023-10-01T11:13:17.041478+0000","last_clean":"2023-10-01T11:13:17.041478+0000","last_became_active":"2023-10-01T11:04:40.879768+0000","last_became_peered":"2023-10-01T11:04:40.879768+0000","last_unstale":"2023-10-01T11:13:17.041478+0000","last_undegraded":"2023-10-01T11:13:17.041478+0000","last_fullsized":"2023-10-01T11:13:17.041478+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:25:31.189338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.9","version":"375'12237","reported_seq":12264,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:33.964933+0000","last_change":"2023-10-01T11:09:21.165167+0000","last_active":"2023-10-01T11:13:33.964933+0000","last_peered":"2023-10-01T11:13:33.964933+0000","last_clean":"2023-10-01T11:13:33.964933+0000","last_became_active":"2023-10-01T11:09:21.164717+0000","last_became_peered":"2023-10-01T11:09:21.164717+0000","last_unstale":"2023-10-01T11:13:33.964933+0000","last_undegraded":"2023-10-01T11:13:33.964933+0000","last_fullsized":"2023-10-01T11:13:33.964933+0000","mapping_epoch":356,"log_start":"359'2200","ondisk_log_start":"359'2200","created":356,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_clean_scrub_stamp":"2023-10-01T11:09:20.132185+0000","objects_scrubbed":0,"log_size":10037,"log_dups_size":0,"ondisk_log_size":10037,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51707904,"num_objects":789,"num_object_clones":0,"num_object_copies":2367,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":789,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12237,"num_write_kb":416832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":203,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.043275+0000","last_change":"2023-10-01T11:06:55.803529+0000","last_active":"2023-10-01T11:13:17.043275+0000","last_peered":"2023-10-01T11:13:17.043275+0000","last_clean":"2023-10-01T11:13:17.043275+0000","last_became_active":"2023-10-01T11:06:55.803266+0000","last_became_peered":"2023-10-01T11:06:55.803266+0000","last_unstale":"2023-10-01T11:13:17.043275+0000","last_undegraded":"2023-10-01T11:13:17.043275+0000","last_fullsized":"2023-10-01T11:13:17.043275+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:57:06.218433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":260,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:14.022964+0000","last_change":"2023-10-01T10:52:18.964479+0000","last_active":"2023-10-01T11:13:14.022964+0000","last_peered":"2023-10-01T11:13:14.022964+0000","last_clean":"2023-10-01T11:13:14.022964+0000","last_became_active":"2023-10-01T10:52:18.963881+0000","last_became_peered":"2023-10-01T10:52:18.963881+0000","last_unstale":"2023-10-01T11:13:14.022964+0000","last_undegraded":"2023-10-01T11:13:14.022964+0000","last_fullsized":"2023-10-01T11:13:14.022964+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:36:30.405755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":352,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.018247+0000","last_change":"2023-10-01T11:06:11.791253+0000","last_active":"2023-10-01T11:13:12.018247+0000","last_peered":"2023-10-01T11:13:12.018247+0000","last_clean":"2023-10-01T11:13:12.018247+0000","last_became_active":"2023-10-01T11:06:11.789517+0000","last_became_peered":"2023-10-01T11:06:11.789517+0000","last_unstale":"2023-10-01T11:13:12.018247+0000","last_undegraded":"2023-10-01T11:13:12.018247+0000","last_fullsized":"2023-10-01T11:13:12.018247+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.a","version":"375'12302","reported_seq":12328,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:33.964376+0000","last_change":"2023-10-01T11:09:21.162350+0000","last_active":"2023-10-01T11:13:33.964376+0000","last_peered":"2023-10-01T11:13:33.964376+0000","last_clean":"2023-10-01T11:13:33.964376+0000","last_became_active":"2023-10-01T11:09:21.162009+0000","last_became_peered":"2023-10-01T11:09:21.162009+0000","last_unstale":"2023-10-01T11:13:33.964376+0000","last_undegraded":"2023-10-01T11:13:33.964376+0000","last_fullsized":"2023-10-01T11:13:33.964376+0000","mapping_epoch":356,"log_start":"359'2300","ondisk_log_start":"359'2300","created":356,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_clean_scrub_stamp":"2023-10-01T11:09:20.132185+0000","objects_scrubbed":0,"log_size":10002,"log_dups_size":0,"ondisk_log_size":10002,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50987008,"num_objects":778,"num_object_clones":0,"num_object_copies":2334,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":778,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12302,"num_write_kb":418560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":390,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.026140+0000","last_change":"2023-10-01T11:06:11.837719+0000","last_active":"2023-10-01T11:13:12.026140+0000","last_peered":"2023-10-01T11:13:12.026140+0000","last_clean":"2023-10-01T11:13:12.026140+0000","last_became_active":"2023-10-01T11:06:11.798523+0000","last_became_peered":"2023-10-01T11:06:11.798523+0000","last_unstale":"2023-10-01T11:13:12.026140+0000","last_undegraded":"2023-10-01T11:13:12.026140+0000","last_fullsized":"2023-10-01T11:13:12.026140+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:53:34.413543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":312,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.041770+0000","last_change":"2023-10-01T11:08:46.872392+0000","last_active":"2023-10-01T11:13:12.041770+0000","last_peered":"2023-10-01T11:13:12.041770+0000","last_clean":"2023-10-01T11:13:12.041770+0000","last_became_active":"2023-10-01T11:08:46.872108+0000","last_became_peered":"2023-10-01T11:08:46.872108+0000","last_unstale":"2023-10-01T11:13:12.041770+0000","last_undegraded":"2023-10-01T11:13:12.041770+0000","last_fullsized":"2023-10-01T11:13:12.041770+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:01:40.481690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":372,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.026291+0000","last_change":"2023-10-01T11:06:11.784630+0000","last_active":"2023-10-01T11:13:12.026291+0000","last_peered":"2023-10-01T11:13:12.026291+0000","last_clean":"2023-10-01T11:13:12.026291+0000","last_became_active":"2023-10-01T11:06:11.784264+0000","last_became_peered":"2023-10-01T11:06:11.784264+0000","last_unstale":"2023-10-01T11:13:12.026291+0000","last_undegraded":"2023-10-01T11:13:12.026291+0000","last_fullsized":"2023-10-01T11:13:12.026291+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:03:03.888001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.b","version":"375'12388","reported_seq":12416,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:34.049830+0000","last_change":"2023-10-01T11:09:22.024506+0000","last_active":"2023-10-01T11:13:34.049830+0000","last_peered":"2023-10-01T11:13:34.049830+0000","last_clean":"2023-10-01T11:13:34.049830+0000","last_became_active":"2023-10-01T11:09:22.023185+0000","last_became_peered":"2023-10-01T11:09:22.023185+0000","last_unstale":"2023-10-01T11:13:34.049830+0000","last_undegraded":"2023-10-01T11:13:34.049830+0000","last_fullsized":"2023-10-01T11:13:34.049830+0000","mapping_epoch":356,"log_start":"359'2300","ondisk_log_start":"359'2300","created":356,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_clean_scrub_stamp":"2023-10-01T11:09:20.132185+0000","objects_scrubbed":0,"log_size":10088,"log_dups_size":0,"ondisk_log_size":10088,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:06:56.128744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50331648,"num_objects":768,"num_object_clones":0,"num_object_copies":2304,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":768,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12388,"num_write_kb":420992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":70,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.025491+0000","last_change":"2023-10-01T11:06:11.818913+0000","last_active":"2023-10-01T11:13:12.025491+0000","last_peered":"2023-10-01T11:13:12.025491+0000","last_clean":"2023-10-01T11:13:12.025491+0000","last_became_active":"2023-10-01T11:06:11.818533+0000","last_became_peered":"2023-10-01T11:06:11.818533+0000","last_unstale":"2023-10-01T11:13:12.025491+0000","last_undegraded":"2023-10-01T11:13:12.025491+0000","last_fullsized":"2023-10-01T11:13:12.025491+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":56,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.041904+0000","last_change":"2023-10-01T11:08:46.872670+0000","last_active":"2023-10-01T11:13:12.041904+0000","last_peered":"2023-10-01T11:13:12.041904+0000","last_clean":"2023-10-01T11:13:12.041904+0000","last_became_active":"2023-10-01T11:08:46.872300+0000","last_became_peered":"2023-10-01T11:08:46.872300+0000","last_unstale":"2023-10-01T11:13:12.041904+0000","last_undegraded":"2023-10-01T11:13:12.041904+0000","last_fullsized":"2023-10-01T11:13:12.041904+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:57:28.746652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":358,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.013932+0000","last_change":"2023-10-01T11:05:33.784531+0000","last_active":"2023-10-01T11:13:12.013932+0000","last_peered":"2023-10-01T11:13:12.013932+0000","last_clean":"2023-10-01T11:13:12.013932+0000","last_became_active":"2023-10-01T11:05:33.784167+0000","last_became_peered":"2023-10-01T11:05:33.784167+0000","last_unstale":"2023-10-01T11:13:12.013932+0000","last_undegraded":"2023-10-01T11:13:12.013932+0000","last_fullsized":"2023-10-01T11:13:12.013932+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:34:16.434841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":90,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.014448+0000","last_change":"2023-10-01T11:06:11.791476+0000","last_active":"2023-10-01T11:13:12.014448+0000","last_peered":"2023-10-01T11:13:12.014448+0000","last_clean":"2023-10-01T11:13:12.014448+0000","last_became_active":"2023-10-01T11:06:11.791021+0000","last_became_peered":"2023-10-01T11:06:11.791021+0000","last_unstale":"2023-10-01T11:13:12.014448+0000","last_undegraded":"2023-10-01T11:13:12.014448+0000","last_fullsized":"2023-10-01T11:13:12.014448+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:14:03.385152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":368,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.040539+0000","last_change":"2023-10-01T10:37:42.762360+0000","last_active":"2023-10-01T11:13:12.040539+0000","last_peered":"2023-10-01T11:13:12.040539+0000","last_clean":"2023-10-01T11:13:12.040539+0000","last_became_active":"2023-10-01T10:37:42.761936+0000","last_became_peered":"2023-10-01T10:37:42.761936+0000","last_unstale":"2023-10-01T11:13:12.040539+0000","last_undegraded":"2023-10-01T11:13:12.040539+0000","last_fullsized":"2023-10-01T11:13:12.040539+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:50.637582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":359,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.018488+0000","last_change":"2023-10-01T10:38:16.037809+0000","last_active":"2023-10-01T11:13:12.018488+0000","last_peered":"2023-10-01T11:13:12.018488+0000","last_clean":"2023-10-01T11:13:12.018488+0000","last_became_active":"2023-10-01T10:38:16.037318+0000","last_became_peered":"2023-10-01T10:38:16.037318+0000","last_unstale":"2023-10-01T11:13:12.018488+0000","last_undegraded":"2023-10-01T11:13:12.018488+0000","last_fullsized":"2023-10-01T11:13:12.018488+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":360,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.014337+0000","last_change":"2023-10-01T11:04:02.726007+0000","last_active":"2023-10-01T11:13:12.014337+0000","last_peered":"2023-10-01T11:13:12.014337+0000","last_clean":"2023-10-01T11:13:12.014337+0000","last_became_active":"2023-10-01T11:04:02.725507+0000","last_became_peered":"2023-10-01T11:04:02.725507+0000","last_unstale":"2023-10-01T11:13:12.014337+0000","last_undegraded":"2023-10-01T11:13:12.014337+0000","last_fullsized":"2023-10-01T11:13:12.014337+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:32:04.611533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":422,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.023038+0000","last_change":"2023-10-01T11:05:33.830108+0000","last_active":"2023-10-01T11:13:12.023038+0000","last_peered":"2023-10-01T11:13:12.023038+0000","last_clean":"2023-10-01T11:13:12.023038+0000","last_became_active":"2023-10-01T11:05:33.791882+0000","last_became_peered":"2023-10-01T11:05:33.791882+0000","last_unstale":"2023-10-01T11:13:12.023038+0000","last_undegraded":"2023-10-01T11:13:12.023038+0000","last_fullsized":"2023-10-01T11:13:12.023038+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":360,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.013818+0000","last_change":"2023-10-01T11:04:02.733653+0000","last_active":"2023-10-01T11:13:12.013818+0000","last_peered":"2023-10-01T11:13:12.013818+0000","last_clean":"2023-10-01T11:13:12.013818+0000","last_became_active":"2023-10-01T11:04:02.733178+0000","last_became_peered":"2023-10-01T11:04:02.733178+0000","last_unstale":"2023-10-01T11:13:12.013818+0000","last_undegraded":"2023-10-01T11:13:12.013818+0000","last_fullsized":"2023-10-01T11:13:12.013818+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:18:25.249624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":421,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.026173+0000","last_change":"2023-10-01T11:05:33.777019+0000","last_active":"2023-10-01T11:13:12.026173+0000","last_peered":"2023-10-01T11:13:12.026173+0000","last_clean":"2023-10-01T11:13:12.026173+0000","last_became_active":"2023-10-01T11:05:33.776565+0000","last_became_peered":"2023-10-01T11:05:33.776565+0000","last_unstale":"2023-10-01T11:13:12.026173+0000","last_undegraded":"2023-10-01T11:13:12.026173+0000","last_fullsized":"2023-10-01T11:13:12.026173+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T22:00:18.890326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":377,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.044592+0000","last_change":"2023-10-01T10:37:42.765142+0000","last_active":"2023-10-01T11:13:12.044592+0000","last_peered":"2023-10-01T11:13:12.044592+0000","last_clean":"2023-10-01T11:13:12.044592+0000","last_became_active":"2023-10-01T10:37:42.764770+0000","last_became_peered":"2023-10-01T10:37:42.764770+0000","last_unstale":"2023-10-01T11:13:12.044592+0000","last_undegraded":"2023-10-01T11:13:12.044592+0000","last_fullsized":"2023-10-01T11:13:12.044592+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:23:33.527876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":89,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.026043+0000","last_change":"2023-10-01T11:04:40.889388+0000","last_active":"2023-10-01T11:13:12.026043+0000","last_peered":"2023-10-01T11:13:12.026043+0000","last_clean":"2023-10-01T11:13:12.026043+0000","last_became_active":"2023-10-01T11:04:40.888960+0000","last_became_peered":"2023-10-01T11:04:40.888960+0000","last_unstale":"2023-10-01T11:13:12.026043+0000","last_undegraded":"2023-10-01T11:13:12.026043+0000","last_fullsized":"2023-10-01T11:13:12.026043+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":396,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-10-01T11:13:12.013634+0000","last_change":"2023-10-01T11:05:33.785774+0000","last_active":"2023-10-01T11:13:12.013634+0000","last_peered":"2023-10-01T11:13:12.013634+0000","last_clean":"2023-10-01T11:13:12.013634+0000","last_became_active":"2023-10-01T11:05:33.785426+0000","last_became_peered":"2023-10-01T11:05:33.785426+0000","last_unstale":"2023-10-01T11:13:12.013634+0000","last_undegraded":"2023-10-01T11:13:12.013634+0000","last_fullsized":"2023-10-01T11:13:12.013634+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:41:32.634339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":14,"num_pg":16,"stat_sum":{"num_bytes":802619416,"num_objects":12248,"num_object_clones":0,"num_object_copies":36744,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":12248,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":198040,"num_write_kb":6729153,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2322018304,"data_stored":2322006088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":143440,"ondisk_log_size":143440,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":82,"num_bytes_recovered":34354,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30319,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":5313312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430933,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1643624,"kb_used_data":164896,"kb_used_omap":0,"kb_used_meta":1478720,"kb_avail":92101528,"statfs":{"total":95995035648,"available":94311964672,"internally_reserved":0,"allocated":168853504,"data_stored":168265363,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1514209280},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 11:12:43 2023","interfaces":[{"interface":"back","average":{"1min":2.476,"5min":1.681,"15min":1.2689999999999999},"min":{"1min":0.35399999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":8.6959999999999997,"5min":10.013,"15min":11.166},"last":2.827},{"interface":"front","average":{"1min":2.3620000000000001,"5min":1.5940000000000001,"15min":1.373},"min":{"1min":0.312,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":8.7119999999999997,"5min":8.8949999999999996,"15min":11.18},"last":2.8479999999999999}]},{"osd":1,"last update":"Sun Oct 1 11:12:43 2023","interfaces":[{"interface":"back","average":{"1min":1.9179999999999999,"5min":1.556,"15min":1.3160000000000001},"min":{"1min":0.29599999999999999,"5min":0.28000000000000003,"15min":0.27400000000000002},"max":{"1min":7.694,"5min":9.6259999999999994,"15min":14.132999999999999},"last":1.573},{"interface":"front","average":{"1min":2.1600000000000001,"5min":1.8500000000000001,"15min":1.4199999999999999},"min":{"1min":0.41699999999999998,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":5.2350000000000003,"5min":9.8170000000000002,"15min":13.558999999999999},"last":0.46100000000000002}]},{"osd":2,"last update":"Sun Oct 1 11:13:00 2023","interfaces":[{"interface":"back","average":{"1min":1.8240000000000001,"5min":1.6930000000000001,"15min":2.548},"min":{"1min":0.436,"5min":0.35199999999999998,"15min":0.33300000000000002},"max":{"1min":10.021000000000001,"5min":10.021000000000001,"15min":215.89699999999999},"last":3.4409999999999998},{"interface":"front","average":{"1min":1.532,"5min":1.819,"15min":2.3849999999999998},"min":{"1min":0.39300000000000002,"5min":0.32200000000000001,"15min":0.307},"max":{"1min":6.8810000000000002,"5min":14.663,"15min":215.77600000000001},"last":0.48099999999999998}]},{"osd":3,"last update":"Sun Oct 1 11:12:43 2023","interfaces":[{"interface":"back","average":{"1min":1.3140000000000001,"5min":1.6120000000000001,"15min":1.462},"min":{"1min":0.38,"5min":0.38,"15min":0.36299999999999999},"max":{"1min":6.9160000000000004,"5min":12.358000000000001,"15min":12.358000000000001},"last":2.734},{"interface":"front","average":{"1min":1.212,"5min":1.3999999999999999,"15min":1.256},"min":{"1min":0.38,"5min":0.38,"15min":0.28299999999999997},"max":{"1min":7.125,"5min":8.3109999999999999,"15min":8.516},"last":0.54800000000000004}]},{"osd":4,"last update":"Sun Oct 1 11:13:22 2023","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":1.095,"15min":1.302},"min":{"1min":0.372,"5min":0.35199999999999998,"15min":0.33100000000000002},"max":{"1min":4.7750000000000004,"5min":6.1130000000000004,"15min":17.178999999999998},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.872,"5min":1.0209999999999999,"15min":1.321},"min":{"1min":0.46100000000000002,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":4.1070000000000002,"5min":6.1840000000000002,"15min":17.25},"last":0.63400000000000001}]},{"osd":5,"last update":"Sun Oct 1 11:12:43 2023","interfaces":[{"interface":"back","average":{"1min":1.4950000000000001,"5min":1.7390000000000001,"15min":1.6100000000000001},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.33100000000000002},"max":{"1min":4.5170000000000003,"5min":18.998999999999999,"15min":18.998999999999999},"last":0.68500000000000005},{"interface":"front","average":{"1min":1.081,"5min":1.329,"15min":1.1910000000000001},"min":{"1min":0.42699999999999999,"5min":0.36899999999999999,"15min":0.33500000000000002},"max":{"1min":5.3360000000000003,"5min":10.1,"15min":10.1},"last":0.59499999999999997}]},{"osd":6,"last update":"Sun Oct 1 11:12:43 2023","interfaces":[{"interface":"back","average":{"1min":1.1479999999999999,"5min":1.7010000000000001,"15min":1.4570000000000001},"min":{"1min":0.44600000000000001,"5min":0.42199999999999999,"15min":0.35299999999999998},"max":{"1min":3.23,"5min":13.038,"15min":15.433},"last":0.72999999999999998},{"interface":"front","average":{"1min":0.82699999999999996,"5min":1.3140000000000001,"15min":1.323},"min":{"1min":0.38800000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":3.2629999999999999,"5min":11.318,"15min":11.536},"last":0.71799999999999997}]}]},{"osd":6,"up_from":42,"seq":180388627173,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1387676,"kb_used_data":210260,"kb_used_omap":0,"kb_used_meta":1177280,"kb_avail":92357476,"statfs":{"total":95995035648,"available":94574055424,"internally_reserved":0,"allocated":215306240,"data_stored":214729950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1205534720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 11:12:35 2023","interfaces":[{"interface":"back","average":{"1min":2.5630000000000002,"5min":1.8600000000000001,"15min":1.6439999999999999},"min":{"1min":0.29399999999999998,"5min":0.27000000000000002,"15min":0.26800000000000002},"max":{"1min":7.298,"5min":14.964,"15min":102.23},"last":0.82199999999999995},{"interface":"front","average":{"1min":2.0739999999999998,"5min":1.5780000000000001,"15min":1.5760000000000001},"min":{"1min":0.28599999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":9.0449999999999999,"5min":9.3849999999999998,"15min":101.217},"last":0.59099999999999997}]},{"osd":1,"last update":"Sun Oct 1 11:12:57 2023","interfaces":[{"interface":"back","average":{"1min":2.8519999999999999,"5min":1.8999999999999999,"15min":1.333},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":8.9770000000000003,"5min":14.946999999999999,"15min":14.946999999999999},"last":3.891},{"interface":"front","average":{"1min":1.321,"5min":1.466,"15min":1.2410000000000001},"min":{"1min":0.317,"5min":0.31,"15min":0.31},"max":{"1min":4.6520000000000001,"5min":8.1159999999999997,"15min":8.1159999999999997},"last":0.41399999999999998}]},{"osd":2,"last update":"Sun Oct 1 11:13:07 2023","interfaces":[{"interface":"back","average":{"1min":1.97,"5min":2.1890000000000001,"15min":2.4460000000000002},"min":{"1min":0.35399999999999998,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":5.7999999999999998,"5min":48.881999999999998,"15min":106.705},"last":1.2769999999999999},{"interface":"front","average":{"1min":1.8640000000000001,"5min":2.0870000000000002,"15min":2.5899999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":5.8970000000000002,"5min":48.948,"15min":106.806},"last":4.8869999999999996}]},{"osd":3,"last update":"Sun Oct 1 11:12:57 2023","interfaces":[{"interface":"back","average":{"1min":2.5009999999999999,"5min":2.1629999999999998,"15min":1.944},"min":{"1min":0.372,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":6.3659999999999997,"5min":50.225000000000001,"15min":102.273},"last":1.2989999999999999},{"interface":"front","average":{"1min":1.905,"5min":1.974,"15min":1.9399999999999999},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":4.7809999999999997,"5min":49.301000000000002,"15min":101.491},"last":1.3460000000000001}]},{"osd":4,"last update":"Sun Oct 1 11:13:28 2023","interfaces":[{"interface":"back","average":{"1min":1.196,"5min":1.2190000000000001,"15min":1.177},"min":{"1min":0.36499999999999999,"5min":0.34200000000000003,"15min":0.30299999999999999},"max":{"1min":6.9589999999999996,"5min":8.0820000000000007,"15min":8.6289999999999996},"last":2.8820000000000001},{"interface":"front","average":{"1min":1.5369999999999999,"5min":1.357,"15min":1.496},"min":{"1min":0.40000000000000002,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":4.1840000000000002,"5min":10.628,"15min":15.462},"last":1.3169999999999999}]},{"osd":5,"last update":"Sun Oct 1 11:12:57 2023","interfaces":[{"interface":"back","average":{"1min":1.5409999999999999,"5min":1.6970000000000001,"15min":1.2989999999999999},"min":{"1min":0.34200000000000003,"5min":0.33100000000000002,"15min":0.33000000000000002},"max":{"1min":6.444,"5min":26.379000000000001,"15min":26.379000000000001},"last":0.86199999999999999},{"interface":"front","average":{"1min":2.028,"5min":1.9830000000000001,"15min":1.5309999999999999},"min":{"1min":0.46800000000000003,"5min":0.32000000000000001,"15min":0.315},"max":{"1min":9.9700000000000006,"5min":28.010000000000002,"15min":28.010000000000002},"last":0.46000000000000002}]},{"osd":7,"last update":"Sun Oct 1 11:12:35 2023","interfaces":[{"interface":"back","average":{"1min":1.2230000000000001,"5min":1.458,"15min":1.278},"min":{"1min":0.39200000000000002,"5min":0.378,"15min":0.35499999999999998},"max":{"1min":4.9640000000000004,"5min":8.1370000000000005,"15min":9.5419999999999998},"last":0.49299999999999999},{"interface":"front","average":{"1min":1.482,"5min":1.4259999999999999,"15min":1.26},"min":{"1min":0.41599999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":6.3209999999999997,"5min":14.894,"15min":14.894},"last":0.89300000000000002}]}]},{"osd":1,"up_from":101,"seq":433791697391,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1354664,"kb_used_data":386144,"kb_used_omap":4,"kb_used_meta":968507,"kb_avail":92390488,"statfs":{"total":95995035648,"available":94607859712,"internally_reserved":0,"allocated":395411456,"data_stored":393960921,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":991752078},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 11:12:59 2023","interfaces":[{"interface":"back","average":{"1min":2.1379999999999999,"5min":1.357,"15min":1.1439999999999999},"min":{"1min":0.27000000000000002,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":11.756,"5min":11.756,"15min":11.756},"last":1.573},{"interface":"front","average":{"1min":3.4489999999999998,"5min":1.7769999999999999,"15min":1.2509999999999999},"min":{"1min":0.28499999999999998,"5min":0.215,"15min":0.215},"max":{"1min":11.943,"5min":11.943,"15min":11.943},"last":0.221}]},{"osd":2,"last update":"Sun Oct 1 11:13:27 2023","interfaces":[{"interface":"back","average":{"1min":2.5329999999999999,"5min":3.3809999999999998,"15min":2.2040000000000002},"min":{"1min":0.38300000000000001,"5min":0.23400000000000001,"15min":0.23200000000000001},"max":{"1min":10.143000000000001,"5min":130.90899999999999,"15min":130.90899999999999},"last":0.56599999999999995},{"interface":"front","average":{"1min":4.2670000000000003,"5min":3.6000000000000001,"15min":2.3959999999999999},"min":{"1min":0.307,"5min":0.307,"15min":0.247},"max":{"1min":12.085000000000001,"5min":130.72800000000001,"15min":130.72800000000001},"last":0.70899999999999996}]},{"osd":3,"last update":"Sun Oct 1 11:12:59 2023","interfaces":[{"interface":"back","average":{"1min":5.3929999999999998,"5min":4.6269999999999998,"15min":2.327},"min":{"1min":0.249,"5min":0.249,"15min":0.248},"max":{"1min":56.595999999999997,"5min":130.79599999999999,"15min":130.79599999999999},"last":0.36599999999999999},{"interface":"front","average":{"1min":7.1459999999999999,"5min":5.351,"15min":2.5310000000000001},"min":{"1min":0.34599999999999997,"5min":0.219,"15min":0.219},"max":{"1min":56.747,"5min":130.83500000000001,"15min":130.83500000000001},"last":0.34100000000000003}]},{"osd":4,"last update":"Sun Oct 1 11:13:27 2023","interfaces":[{"interface":"back","average":{"1min":1.6970000000000001,"5min":3.1280000000000001,"15min":2.5019999999999998},"min":{"1min":0.56999999999999995,"5min":0.33300000000000002,"15min":0.32600000000000001},"max":{"1min":5.5780000000000003,"5min":130.702,"15min":203.89500000000001},"last":0.51700000000000002},{"interface":"front","average":{"1min":4.4409999999999998,"5min":3.9510000000000001,"15min":2.8700000000000001},"min":{"1min":0.40500000000000003,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":13.138999999999999,"5min":130.572,"15min":203.852},"last":0.33100000000000002}]},{"osd":5,"last update":"Sun Oct 1 11:12:59 2023","interfaces":[{"interface":"back","average":{"1min":2.621,"5min":1.8160000000000001,"15min":1.3560000000000001},"min":{"1min":0.498,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":11.042999999999999,"5min":12.192,"15min":12.192},"last":0.38900000000000001},{"interface":"front","average":{"1min":1.5700000000000001,"5min":2.9380000000000002,"15min":1.8029999999999999},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.34200000000000003},"max":{"1min":5.6349999999999998,"5min":130.62299999999999,"15min":130.62299999999999},"last":0.54700000000000004}]},{"osd":6,"last update":"Sun Oct 1 11:12:59 2023","interfaces":[{"interface":"back","average":{"1min":1.393,"5min":2.9710000000000001,"15min":1.8300000000000001},"min":{"1min":0.48799999999999999,"5min":0.371,"15min":0.33800000000000002},"max":{"1min":5.6109999999999998,"5min":130.75299999999999,"15min":130.75299999999999},"last":0.48699999999999999},{"interface":"front","average":{"1min":3.4289999999999998,"5min":2.0150000000000001,"15min":1.472},"min":{"1min":0.441,"5min":0.38400000000000001,"15min":0.34699999999999998},"max":{"1min":12.01,"5min":12.01,"15min":12.01},"last":0.35599999999999998}]},{"osd":7,"last update":"Sun Oct 1 11:12:59 2023","interfaces":[{"interface":"back","average":{"1min":3.508,"5min":2.0529999999999999,"15min":1.49},"min":{"1min":0.40699999999999997,"5min":0.40100000000000002,"15min":0.38},"max":{"1min":12.055999999999999,"5min":12.055999999999999,"15min":12.055999999999999},"last":0.67100000000000004},{"interface":"front","average":{"1min":3.222,"5min":3.4369999999999998,"15min":2.0209999999999999},"min":{"1min":0.70299999999999996,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":8.6170000000000009,"5min":130.42099999999999,"15min":130.42099999999999},"last":0.44800000000000001}]}]},{"osd":0,"up_from":86,"seq":369367188002,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1366260,"kb_used_data":331820,"kb_used_omap":5,"kb_used_meta":1034426,"kb_avail":92378892,"statfs":{"total":95995035648,"available":94595985408,"internally_reserved":0,"allocated":339783680,"data_stored":338378268,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1059252858},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 11:13:06 2023","interfaces":[{"interface":"back","average":{"1min":2.4180000000000001,"5min":1.591,"15min":1.575},"min":{"1min":0.186,"5min":0.186,"15min":0.186},"max":{"1min":6.6719999999999997,"5min":7.8570000000000002,"15min":119.911},"last":0.60699999999999998},{"interface":"front","average":{"1min":4.0830000000000002,"5min":2.649,"15min":2.008},"min":{"1min":0.21099999999999999,"5min":0.189,"15min":0.189},"max":{"1min":11.983000000000001,"5min":13.989000000000001,"15min":119.76300000000001},"last":1.3420000000000001}]},{"osd":2,"last update":"Sun Oct 1 11:13:09 2023","interfaces":[{"interface":"back","average":{"1min":2.274,"5min":2.5310000000000001,"15min":2.23},"min":{"1min":0.23100000000000001,"5min":0.23000000000000001,"15min":0.20699999999999999},"max":{"1min":7.9240000000000004,"5min":67.076999999999998,"15min":72.064999999999998},"last":0.32000000000000001},{"interface":"front","average":{"1min":3.367,"5min":2.847,"15min":2.4660000000000002},"min":{"1min":0.26400000000000001,"5min":0.247,"15min":0.247},"max":{"1min":10.569000000000001,"5min":67.129999999999995,"15min":76.328999999999994},"last":0.64600000000000002}]},{"osd":3,"last update":"Sun Oct 1 11:13:06 2023","interfaces":[{"interface":"back","average":{"1min":2.278,"5min":2.5550000000000002,"15min":2.1779999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.214},"max":{"1min":10.614000000000001,"5min":67.706000000000003,"15min":81.674999999999997},"last":0.621},{"interface":"front","average":{"1min":2.5,"5min":2.5600000000000001,"15min":2.1139999999999999},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":6.2690000000000001,"5min":66.352000000000004,"15min":70.950999999999993},"last":0.63600000000000001}]},{"osd":4,"last update":"Sun Oct 1 11:13:35 2023","interfaces":[{"interface":"back","average":{"1min":3.0510000000000002,"5min":1.9419999999999999,"15min":1.5840000000000001},"min":{"1min":0.5,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":13.147,"5min":23.995000000000001,"15min":23.995000000000001},"last":3.1520000000000001},{"interface":"front","average":{"1min":3.4409999999999998,"5min":2.3180000000000001,"15min":1.7490000000000001},"min":{"1min":0.46800000000000003,"5min":0.33600000000000002,"15min":0.33500000000000002},"max":{"1min":12.015000000000001,"5min":21.451000000000001,"15min":21.451000000000001},"last":3.1339999999999999}]},{"osd":5,"last update":"Sun Oct 1 11:13:06 2023","interfaces":[{"interface":"back","average":{"1min":1.6259999999999999,"5min":1.9650000000000001,"15min":1.6859999999999999},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.309},"max":{"1min":6.2759999999999998,"5min":12.776999999999999,"15min":39.860999999999997},"last":0.67300000000000004},{"interface":"front","average":{"1min":2.0699999999999998,"5min":1.978,"15min":1.724},"min":{"1min":0.435,"5min":0.41599999999999998,"15min":0.38200000000000001},"max":{"1min":5.8369999999999997,"5min":14.529,"15min":39.646000000000001},"last":0.68799999999999994}]},{"osd":6,"last update":"Sun Oct 1 11:13:06 2023","interfaces":[{"interface":"back","average":{"1min":1.919,"5min":1.5800000000000001,"15min":1.466},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.373},"max":{"1min":5.8019999999999996,"5min":8.1660000000000004,"15min":9.4570000000000007},"last":0.66300000000000003},{"interface":"front","average":{"1min":2.6400000000000001,"5min":1.837,"15min":1.5309999999999999},"min":{"1min":0.41099999999999998,"5min":0.373,"15min":0.373},"max":{"1min":12.035,"5min":12.035,"15min":12.035},"last":0.55200000000000005}]},{"osd":7,"last update":"Sun Oct 1 11:13:06 2023","interfaces":[{"interface":"back","average":{"1min":1.742,"5min":1.458,"15min":1.429},"min":{"1min":0.45800000000000002,"5min":0.42999999999999999,"15min":0.35499999999999998},"max":{"1min":5.8780000000000001,"5min":8.3109999999999999,"15min":12.680999999999999},"last":0.71099999999999997},{"interface":"front","average":{"1min":3.302,"5min":2.0779999999999998,"15min":1.7130000000000001},"min":{"1min":0.55900000000000005,"5min":0.433,"15min":0.32200000000000001},"max":{"1min":12.053000000000001,"5min":12.053000000000001,"15min":12.053000000000001},"last":0.49399999999999999}]}]},{"osd":2,"up_from":116,"seq":498216206775,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1280740,"kb_used_data":369436,"kb_used_omap":7,"kb_used_meta":911288,"kb_avail":92464412,"statfs":{"total":95995035648,"available":94683557888,"internally_reserved":0,"allocated":378302464,"data_stored":376909877,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":933158993},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 11:13:15 2023","interfaces":[{"interface":"back","average":{"1min":1.472,"5min":1.1930000000000001,"15min":1.163},"min":{"1min":0.20899999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":4.4320000000000004,"5min":5.3630000000000004,"15min":9.4510000000000005},"last":0.23699999999999999},{"interface":"front","average":{"1min":3.0990000000000002,"5min":2.8980000000000001,"15min":1.8180000000000001},"min":{"1min":0.42699999999999999,"5min":0.222,"15min":0.187},"max":{"1min":8.8559999999999999,"5min":95.688999999999993,"15min":95.688999999999993},"last":0.95499999999999996}]},{"osd":1,"last update":"Sun Oct 1 11:13:15 2023","interfaces":[{"interface":"back","average":{"1min":4.694,"5min":2.641,"15min":1.744},"min":{"1min":0.26100000000000001,"5min":0.254,"15min":0.20699999999999999},"max":{"1min":34.753999999999998,"5min":34.753999999999998,"15min":34.753999999999998},"last":1.427},{"interface":"front","average":{"1min":4.5380000000000003,"5min":2.6059999999999999,"15min":1.865},"min":{"1min":0.39800000000000002,"5min":0.27200000000000002,"15min":0.23999999999999999},"max":{"1min":34.972999999999999,"5min":34.972999999999999,"15min":34.972999999999999},"last":1.9630000000000001}]},{"osd":3,"last update":"Sun Oct 1 11:12:50 2023","interfaces":[{"interface":"back","average":{"1min":1.8160000000000001,"5min":1.3520000000000001,"15min":1.4510000000000001},"min":{"1min":0.28100000000000003,"5min":0.24399999999999999,"15min":0.23200000000000001},"max":{"1min":4.843,"5min":8.2560000000000002,"15min":23.879000000000001},"last":0.376},{"interface":"front","average":{"1min":2.145,"5min":1.5660000000000001,"15min":1.6140000000000001},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":7.1630000000000003,"5min":10.102,"15min":23.704000000000001},"last":0.91900000000000004}]},{"osd":4,"last update":"Sun Oct 1 11:13:33 2023","interfaces":[{"interface":"back","average":{"1min":1.5069999999999999,"5min":1.546,"15min":1.556},"min":{"1min":0.36699999999999999,"5min":0.312,"15min":0.312},"max":{"1min":5.0739999999999998,"5min":10.004,"15min":10.004},"last":0.998},{"interface":"front","average":{"1min":2.988,"5min":2.2559999999999998,"15min":1.946},"min":{"1min":0.34100000000000003,"5min":0.33000000000000002,"15min":0.317},"max":{"1min":12.989000000000001,"5min":12.989000000000001,"15min":12.989000000000001},"last":1.9370000000000001}]},{"osd":5,"last update":"Sun Oct 1 11:13:15 2023","interfaces":[{"interface":"back","average":{"1min":2.0569999999999999,"5min":1.6279999999999999,"15min":1.593},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":5.6550000000000002,"5min":9.968,"15min":10.180999999999999},"last":1.0920000000000001},{"interface":"front","average":{"1min":1.931,"5min":2.121,"15min":1.8899999999999999},"min":{"1min":0.377,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":5.3390000000000004,"5min":12.518000000000001,"15min":12.518000000000001},"last":1.9810000000000001}]},{"osd":6,"last update":"Sun Oct 1 11:13:15 2023","interfaces":[{"interface":"back","average":{"1min":1.6850000000000001,"5min":2.7530000000000001,"15min":2.1720000000000002},"min":{"1min":0.40200000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":5.3159999999999998,"5min":95.802999999999997,"15min":95.802999999999997},"last":1.998},{"interface":"front","average":{"1min":2.004,"5min":1.5680000000000001,"15min":1.393},"min":{"1min":0.49199999999999999,"5min":0.40999999999999998,"15min":0.33700000000000002},"max":{"1min":6.3819999999999997,"5min":6.827,"15min":9.6709999999999994},"last":0.64900000000000002}]},{"osd":7,"last update":"Sun Oct 1 11:13:15 2023","interfaces":[{"interface":"back","average":{"1min":1.5860000000000001,"5min":1.458,"15min":1.419},"min":{"1min":0.442,"5min":0.40100000000000002,"15min":0.35099999999999998},"max":{"1min":4.5039999999999996,"5min":6.9169999999999998,"15min":11.209},"last":0.61199999999999999},{"interface":"front","average":{"1min":2.3799999999999999,"5min":2.6400000000000001,"15min":1.929},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":5.7000000000000002,"5min":95.228999999999999,"15min":95.228999999999999},"last":1.1679999999999999}]}]},{"osd":3,"up_from":121,"seq":519691043249,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1186832,"kb_used_data":329032,"kb_used_omap":2,"kb_used_meta":857725,"kb_avail":92558320,"statfs":{"total":95995035648,"available":94779719680,"internally_reserved":0,"allocated":336928768,"data_stored":335553813,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":878310453},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 11:12:50 2023","interfaces":[{"interface":"back","average":{"1min":2.4849999999999999,"5min":1.5429999999999999,"15min":1.6339999999999999},"min":{"1min":0.46400000000000002,"5min":0.222,"15min":0.17199999999999999},"max":{"1min":7.4859999999999998,"5min":9.625,"15min":100.191},"last":5.9900000000000002},{"interface":"front","average":{"1min":1.9390000000000001,"5min":1.6080000000000001,"15min":1.7669999999999999},"min":{"1min":0.313,"5min":0.217,"15min":0.20000000000000001},"max":{"1min":5.9359999999999999,"5min":12.196999999999999,"15min":99.852999999999994},"last":0.62}]},{"osd":1,"last update":"Sun Oct 1 11:12:50 2023","interfaces":[{"interface":"back","average":{"1min":2.1419999999999999,"5min":2.173,"15min":1.4670000000000001},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.22500000000000001},"max":{"1min":7.4450000000000003,"5min":51.070999999999998,"15min":51.070999999999998},"last":6.7370000000000001},{"interface":"front","average":{"1min":2.1259999999999999,"5min":2.0110000000000001,"15min":1.474},"min":{"1min":0.28399999999999997,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":7.5259999999999998,"5min":51.034999999999997,"15min":51.034999999999997},"last":2.7850000000000001}]},{"osd":2,"last update":"Sun Oct 1 11:12:50 2023","interfaces":[{"interface":"back","average":{"1min":7.8579999999999997,"5min":3.984,"15min":2.8090000000000002},"min":{"1min":0.32600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":90.540000000000006,"5min":90.540000000000006,"15min":90.540000000000006},"last":0.66500000000000004},{"interface":"front","average":{"1min":6.4770000000000003,"5min":3.427,"15min":2.673},"min":{"1min":0.32800000000000001,"5min":0.29299999999999998,"15min":0.27100000000000002},"max":{"1min":90.367000000000004,"5min":90.367000000000004,"15min":90.367000000000004},"last":6.7720000000000002}]},{"osd":4,"last update":"Sun Oct 1 11:12:35 2023","interfaces":[{"interface":"back","average":{"1min":1.1499999999999999,"5min":1.3260000000000001,"15min":1.3740000000000001},"min":{"1min":0.376,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":3.8100000000000001,"5min":7.3579999999999997,"15min":10.808999999999999},"last":1.5700000000000001},{"interface":"front","average":{"1min":1.4650000000000001,"5min":1.4039999999999999,"15min":1.407},"min":{"1min":0.40699999999999997,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":4.5579999999999998,"5min":6.7240000000000002,"15min":10.962999999999999},"last":0.86699999999999999}]},{"osd":5,"last update":"Sun Oct 1 11:12:50 2023","interfaces":[{"interface":"back","average":{"1min":1.6699999999999999,"5min":1.5640000000000001,"15min":1.484},"min":{"1min":0.42899999999999999,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":6.226,"5min":7.3710000000000004,"15min":12.771000000000001},"last":0.97199999999999998},{"interface":"front","average":{"1min":3.4809999999999999,"5min":2.2189999999999999,"15min":1.7190000000000001},"min":{"1min":0.48599999999999999,"5min":0.42899999999999999,"15min":0.35799999999999998},"max":{"1min":10.571999999999999,"5min":14.875999999999999,"15min":14.875999999999999},"last":0.75700000000000001}]},{"osd":6,"last update":"Sun Oct 1 11:12:50 2023","interfaces":[{"interface":"back","average":{"1min":1.659,"5min":1.6240000000000001,"15min":1.5720000000000001},"min":{"1min":0.46600000000000003,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":6.1929999999999996,"5min":6.9340000000000002,"15min":13.616},"last":1.339},{"interface":"front","average":{"1min":2.7669999999999999,"5min":1.9990000000000001,"15min":1.9379999999999999},"min":{"1min":0.52000000000000002,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":9.9309999999999992,"5min":14.906000000000001,"15min":73.542000000000002},"last":0.877}]},{"osd":7,"last update":"Sun Oct 1 11:12:50 2023","interfaces":[{"interface":"back","average":{"1min":3.415,"5min":2.0129999999999999,"15min":1.601},"min":{"1min":0.55000000000000004,"5min":0.39400000000000002,"15min":0.39100000000000001},"max":{"1min":13.617000000000001,"5min":14.929,"15min":14.929},"last":0.84999999999999998},{"interface":"front","average":{"1min":1.538,"5min":1.3680000000000001,"15min":1.3979999999999999},"min":{"1min":0.45800000000000002,"5min":0.38,"15min":0.38},"max":{"1min":5.8920000000000003,"5min":7.3819999999999997,"15min":8.7200000000000006},"last":1.6020000000000001}]}]},{"osd":4,"up_from":126,"seq":541165879722,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1058104,"kb_used_data":316336,"kb_used_omap":10,"kb_used_meta":741749,"kb_avail":92687048,"statfs":{"total":95995035648,"available":94911537152,"internally_reserved":0,"allocated":323928064,"data_stored":322515469,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":759551588},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 11:13:17 2023","interfaces":[{"interface":"back","average":{"1min":1.2190000000000001,"5min":1.143,"15min":1.573},"min":{"1min":0.29299999999999998,"5min":0.28699999999999998,"15min":0.26400000000000001},"max":{"1min":11.025,"5min":11.025,"15min":68.611000000000004},"last":0.28000000000000003},{"interface":"front","average":{"1min":1.353,"5min":1.194,"15min":1.5600000000000001},"min":{"1min":0.28199999999999997,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":4.5060000000000002,"5min":6.8940000000000001,"15min":68.480999999999995},"last":2.7629999999999999}]},{"osd":1,"last update":"Sun Oct 1 11:13:17 2023","interfaces":[{"interface":"back","average":{"1min":2.577,"5min":1.5429999999999999,"15min":1.282},"min":{"1min":0.41199999999999998,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":7.3360000000000003,"5min":7.3360000000000003,"15min":9.1920000000000002},"last":1.2929999999999999},{"interface":"front","average":{"1min":2.1619999999999999,"5min":1.3129999999999999,"15min":1.2609999999999999},"min":{"1min":0.33600000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":11.145,"5min":11.145,"15min":11.145},"last":1.3919999999999999}]},{"osd":2,"last update":"Sun Oct 1 11:13:17 2023","interfaces":[{"interface":"back","average":{"1min":13.215,"5min":4.0759999999999996,"15min":3.2280000000000002},"min":{"1min":0.374,"5min":0.34699999999999998,"15min":0.34100000000000003},"max":{"1min":219.05500000000001,"5min":219.05500000000001,"15min":219.05500000000001},"last":0.40400000000000003},{"interface":"front","average":{"1min":11.398999999999999,"5min":3.5329999999999999,"15min":2.8420000000000001},"min":{"1min":0.34799999999999998,"5min":0.33900000000000002,"15min":0.33700000000000002},"max":{"1min":218.81200000000001,"5min":218.81200000000001,"15min":218.81200000000001},"last":3.2559999999999998}]},{"osd":3,"last update":"Sun Oct 1 11:13:17 2023","interfaces":[{"interface":"back","average":{"1min":1.4430000000000001,"5min":1.8320000000000001,"15min":1.7430000000000001},"min":{"1min":0.34699999999999998,"5min":0.34499999999999997,"15min":0.34200000000000003},"max":{"1min":6.1440000000000001,"5min":18.475999999999999,"15min":61.664999999999999},"last":0.41299999999999998},{"interface":"front","average":{"1min":2.6269999999999998,"5min":2.2120000000000002,"15min":1.8200000000000001},"min":{"1min":0.38600000000000001,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":8.1980000000000004,"5min":18.303000000000001,"15min":61.707999999999998},"last":0.39300000000000002}]},{"osd":5,"last update":"Sun Oct 1 11:13:17 2023","interfaces":[{"interface":"back","average":{"1min":0.77900000000000003,"5min":1.256,"15min":1.3340000000000001},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":2.4809999999999999,"5min":12.457000000000001,"15min":25.175999999999998},"last":0.34200000000000003},{"interface":"front","average":{"1min":1.6870000000000001,"5min":1.4339999999999999,"15min":1.5129999999999999},"min":{"1min":0.378,"5min":0.307,"15min":0.307},"max":{"1min":11.084,"5min":11.084,"15min":13.629},"last":0.36199999999999999}]},{"osd":6,"last update":"Sun Oct 1 11:13:17 2023","interfaces":[{"interface":"back","average":{"1min":0.81799999999999995,"5min":1.151,"15min":1.2290000000000001},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":4.2800000000000002,"5min":8.1440000000000001,"15min":12.635999999999999},"last":0.441},{"interface":"front","average":{"1min":1.9510000000000001,"5min":1.3420000000000001,"15min":1.3049999999999999},"min":{"1min":0.38900000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":10.506,"5min":10.506,"15min":10.506},"last":0.432}]},{"osd":7,"last update":"Sun Oct 1 11:13:17 2023","interfaces":[{"interface":"back","average":{"1min":1.8049999999999999,"5min":1.373,"15min":1.4099999999999999},"min":{"1min":0.45100000000000001,"5min":0.376,"15min":0.376},"max":{"1min":7.4450000000000003,"5min":9.2599999999999998,"15min":10.851000000000001},"last":0.45300000000000001},{"interface":"front","average":{"1min":1.181,"5min":1.331,"15min":1.4019999999999999},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":11.117000000000001,"5min":12.364000000000001,"15min":12.364000000000001},"last":0.42399999999999999}]}]},{"osd":5,"up_from":36,"seq":154618823403,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1760784,"kb_used_data":262216,"kb_used_omap":0,"kb_used_meta":1498560,"kb_avail":91984368,"statfs":{"total":95995035648,"available":94191992832,"internally_reserved":0,"allocated":268509184,"data_stored":267940458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1534525440},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 11:12:46 2023","interfaces":[{"interface":"back","average":{"1min":9.0980000000000008,"5min":2.9460000000000002,"15min":2.1099999999999999},"min":{"1min":0.36199999999999999,"5min":0.27400000000000002,"15min":0.253},"max":{"1min":111.797,"5min":111.797,"15min":115.057},"last":1.177},{"interface":"front","average":{"1min":8.5500000000000007,"5min":2.9460000000000002,"15min":2.194},"min":{"1min":0.315,"5min":0.251,"15min":0.251},"max":{"1min":111.492,"5min":111.492,"15min":115.53},"last":1.004}]},{"osd":1,"last update":"Sun Oct 1 11:12:46 2023","interfaces":[{"interface":"back","average":{"1min":8.9779999999999998,"5min":2.9159999999999999,"15min":1.589},"min":{"1min":0.28699999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":111.14,"5min":111.14,"15min":111.14},"last":0.30299999999999999},{"interface":"front","average":{"1min":8.7400000000000002,"5min":3.1619999999999999,"15min":1.857},"min":{"1min":0.34799999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":111.095,"5min":111.095,"15min":111.095},"last":5.1120000000000001}]},{"osd":2,"last update":"Sun Oct 1 11:13:21 2023","interfaces":[{"interface":"back","average":{"1min":1.419,"5min":3.621,"15min":3.0419999999999998},"min":{"1min":0.36599999999999999,"5min":0.309,"15min":0.309},"max":{"1min":9.4879999999999995,"5min":129.70500000000001,"15min":207.059},"last":0.33400000000000002},{"interface":"front","average":{"1min":1.355,"5min":3.8410000000000002,"15min":3.3530000000000002},"min":{"1min":0.36699999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":2.9830000000000001,"5min":129.76300000000001,"15min":206.673},"last":3.694}]},{"osd":3,"last update":"Sun Oct 1 11:12:46 2023","interfaces":[{"interface":"back","average":{"1min":2.4300000000000002,"5min":3.0990000000000002,"15min":2.0499999999999998},"min":{"1min":0.48499999999999999,"5min":0.378,"15min":0.33800000000000002},"max":{"1min":6.9470000000000001,"5min":131.09399999999999,"15min":131.09399999999999},"last":3.431},{"interface":"front","average":{"1min":1.204,"5min":2.7629999999999999,"15min":1.8129999999999999},"min":{"1min":0.41299999999999998,"5min":0.36299999999999999,"15min":0.32000000000000001},"max":{"1min":3.5609999999999999,"5min":130.70699999999999,"15min":130.70699999999999},"last":0.89000000000000001}]},{"osd":4,"last update":"Sun Oct 1 11:13:21 2023","interfaces":[{"interface":"back","average":{"1min":1.048,"5min":1.1719999999999999,"15min":1.1850000000000001},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.311},"max":{"1min":5.2400000000000002,"5min":10.167,"15min":10.167},"last":0.70999999999999996},{"interface":"front","average":{"1min":1.6679999999999999,"5min":1.532,"15min":1.282},"min":{"1min":0.373,"5min":0.35999999999999999,"15min":0.30399999999999999},"max":{"1min":9.3390000000000004,"5min":11.162000000000001,"15min":11.162000000000001},"last":0.63400000000000001}]},{"osd":6,"last update":"Sun Oct 1 11:12:46 2023","interfaces":[{"interface":"back","average":{"1min":1.4990000000000001,"5min":1.444,"15min":1.1839999999999999},"min":{"1min":0.40899999999999997,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":9.3640000000000008,"5min":11.196999999999999,"15min":11.196999999999999},"last":0.69999999999999996},{"interface":"front","average":{"1min":0.95199999999999996,"5min":1.2450000000000001,"15min":1.171},"min":{"1min":0.38800000000000001,"5min":0.35499999999999998,"15min":0.34499999999999997},"max":{"1min":4.7089999999999996,"5min":8.234,"15min":12.189},"last":0.48899999999999999}]},{"osd":7,"last update":"Sun Oct 1 11:12:46 2023","interfaces":[{"interface":"back","average":{"1min":0.67200000000000004,"5min":1.167,"15min":1.198},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.34899999999999998},"max":{"1min":1.0920000000000001,"5min":8.1359999999999992,"15min":9.1959999999999997},"last":0.45100000000000001},{"interface":"front","average":{"1min":1.718,"5min":1.452,"15min":1.246},"min":{"1min":0.44500000000000001,"5min":0.40899999999999997,"15min":0.38600000000000001},"max":{"1min":9.3000000000000007,"5min":11.215,"15min":11.215},"last":0.66500000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":325386240,"data_stored":325386240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":385613824,"data_stored":385613824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":361627648,"data_stored":361627648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":316997632,"data_stored":316997632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":308613120,"data_stored":308609048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":257691648,"data_stored":257687576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":208408576,"data_stored":208404504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":157679616,"data_stored":157679616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T11:13:41.992 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T11:13:41.992 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:42.015 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:42.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:42.017 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:13:42.117 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:42.149 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:42.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:42.151 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:13:42.252 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:42.266 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:42 smithi078 bash[75299]: cluster 2023-10-01T11:13:40.469424+0000 mgr.y (mgr.25072) 1693 : cluster 0 pgmap v1829: 89 pgs: 89 active+clean; 603 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 627 op/s 2023-10-01T11:13:42.266 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:42 smithi078 bash[78274]: cluster 2023-10-01T11:13:40.469424+0000 mgr.y (mgr.25072) 1693 : cluster 0 pgmap v1829: 89 pgs: 89 active+clean; 603 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 627 op/s 2023-10-01T11:13:42.291 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:42.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:42.293 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:13:42.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:42 smithi173 bash[58816]: cluster 2023-10-01T11:13:40.469424+0000 mgr.y (mgr.25072) 1693 : cluster 0 pgmap v1829: 89 pgs: 89 active+clean; 603 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 627 op/s 2023-10-01T11:13:42.393 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:42.421 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:42.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:42.424 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:13:42.525 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:42.549 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:42.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:42.553 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:13:42.654 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:42.679 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:42.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:42.684 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:13:42.785 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:42.816 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:42.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:42.822 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:13:42.923 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:42.952 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:42.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:42.955 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:13:43.055 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:43.079 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:43.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:43.086 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:13:43.187 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:43.224 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:43.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:43.228 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:13:43.329 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:43.361 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:43.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:43.364 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:13:43.466 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:43.503 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:43.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:43.507 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:13:43.608 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:43.636 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:43.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:43.639 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:13:43.739 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:43.764 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:43.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:43.768 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:13:43.868 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:43.874 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2023-10-01T11:13:43.874 INFO:teuthology.orchestra.run.smithi078.stdout: "ops": [], 2023-10-01T11:13:43.874 INFO:teuthology.orchestra.run.smithi078.stdout: "complaint_time": 30, 2023-10-01T11:13:43.874 INFO:teuthology.orchestra.run.smithi078.stdout: "num_blocked_ops": 0 2023-10-01T11:13:43.874 INFO:teuthology.orchestra.run.smithi078.stdout:} 2023-10-01T11:13:43.896 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:43.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:43.900 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:13:44.000 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:44.033 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:44.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:44.038 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:13:44.061 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T11:13:44.075 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T11:13:44.139 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:44.145 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:44 smithi078 bash[75299]: cluster 2023-10-01T11:13:42.469971+0000 mgr.y (mgr.25072) 1694 : cluster 0 pgmap v1830: 89 pgs: 89 active+clean; 568 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2023-10-01T11:13:44.185 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:44.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:44.188 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:13:44.289 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:44.315 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:44.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:44.323 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:13:44.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:44 smithi078 bash[78274]: cluster 2023-10-01T11:13:42.469971+0000 mgr.y (mgr.25072) 1694 : cluster 0 pgmap v1830: 89 pgs: 89 active+clean; 568 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2023-10-01T11:13:44.424 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:44.522 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:44.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:44.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:13:44.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:44 smithi173 bash[58816]: cluster 2023-10-01T11:13:42.469971+0000 mgr.y (mgr.25072) 1694 : cluster 0 pgmap v1830: 89 pgs: 89 active+clean; 568 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 623 op/s 2023-10-01T11:13:44.703 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:44.732 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:44.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:44.745 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:13:44.846 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:44.874 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:44.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:44.878 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:13:44.979 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:45.017 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:45.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:45.019 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:13:45.120 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:45.143 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:45.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:45.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:13:45.192 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-10-01T11:13:45.247 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:45.281 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:45.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:45.284 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:13:45.384 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:45.420 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:45.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:45.427 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:13:45.527 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:45.562 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:45.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:45.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:13:45.665 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:45.692 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:45.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:45.695 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:13:45.795 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:45.842 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:45.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:45.846 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:13:45.947 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:45.966 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:13:45.980 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:45.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:45.982 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:13:46.083 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:46.115 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:46.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:46.116 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:13:46.217 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:46.253 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:46.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:46.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:13:46.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:46 smithi078 bash[78274]: cluster 2023-10-01T11:13:44.473750+0000 mgr.y (mgr.25072) 1695 : cluster 0 pgmap v1831: 89 pgs: 89 active+clean; 471 MiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 569 op/s 2023-10-01T11:13:46.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:46 smithi078 bash[75299]: cluster 2023-10-01T11:13:44.473750+0000 mgr.y (mgr.25072) 1695 : cluster 0 pgmap v1831: 89 pgs: 89 active+clean; 471 MiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 569 op/s 2023-10-01T11:13:46.356 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:46.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:46 smithi173 bash[58816]: cluster 2023-10-01T11:13:44.473750+0000 mgr.y (mgr.25072) 1695 : cluster 0 pgmap v1831: 89 pgs: 89 active+clean; 471 MiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 569 op/s 2023-10-01T11:13:46.399 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:46.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:46.404 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:13:46.505 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:46.536 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:46.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:46.546 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:13:46.647 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:46.692 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:46.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:46.699 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:13:46.799 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:46.826 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:46.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:46.832 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:13:46.933 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:46.958 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:46.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:46.991 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:13:47.092 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:47.121 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:47.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:47.123 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:13:47.224 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:47.253 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:47.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:47.263 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:13:47.364 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:47.401 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:47.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:47.406 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:13:47.507 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:47.549 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:47.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:47.550 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:13:47.650 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:47.673 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:47.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:47.676 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:13:47.776 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:47.799 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:47.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:47.802 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:13:47.902 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:47.931 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:47.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:47.938 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:13:48.002 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:13:48.002 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:13:48.038 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:48.066 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:48.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:48.068 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:13:48.169 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:48.197 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:48.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:48.203 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:13:48.303 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:48.312 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:48 smithi173 bash[58816]: cluster 2023-10-01T11:13:46.476743+0000 mgr.y (mgr.25072) 1696 : cluster 0 pgmap v1832: 89 pgs: 89 active+clean; 374 MiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 627 op/s 2023-10-01T11:13:48.312 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:48 smithi173 bash[58816]: audit 2023-10-01T11:13:47.348748+0000 mon.a (mon.0) 2169 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:13:48.313 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:48 smithi173 bash[58816]: audit 2023-10-01T11:13:47.373605+0000 mon.a (mon.0) 2170 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:13:48.313 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:48 smithi173 bash[58816]: audit 2023-10-01T11:13:47.494864+0000 mon.a (mon.0) 2171 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:13:48.313 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:48 smithi173 bash[58816]: audit 2023-10-01T11:13:47.496275+0000 mon.a (mon.0) 2172 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:13:48.313 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:48 smithi173 bash[58816]: audit 2023-10-01T11:13:47.516097+0000 mon.a (mon.0) 2173 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:13:48.313 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:48 smithi173 bash[58816]: audit 2023-10-01T11:13:47.540897+0000 mon.a (mon.0) 2174 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:13:48.327 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:48.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:48.330 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:13:48.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:48 smithi078 bash[75299]: cluster 2023-10-01T11:13:46.476743+0000 mgr.y (mgr.25072) 1696 : cluster 0 pgmap v1832: 89 pgs: 89 active+clean; 374 MiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 627 op/s 2023-10-01T11:13:48.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:48 smithi078 bash[75299]: audit 2023-10-01T11:13:47.348748+0000 mon.a (mon.0) 2169 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:13:48.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:48 smithi078 bash[75299]: audit 2023-10-01T11:13:47.373605+0000 mon.a (mon.0) 2170 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:13:48.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:48 smithi078 bash[75299]: audit 2023-10-01T11:13:47.494864+0000 mon.a (mon.0) 2171 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:13:48.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:48 smithi078 bash[75299]: audit 2023-10-01T11:13:47.496275+0000 mon.a (mon.0) 2172 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:13:48.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:48 smithi078 bash[75299]: audit 2023-10-01T11:13:47.516097+0000 mon.a (mon.0) 2173 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:13:48.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:48 smithi078 bash[75299]: audit 2023-10-01T11:13:47.540897+0000 mon.a (mon.0) 2174 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:13:48.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:48 smithi078 bash[78274]: cluster 2023-10-01T11:13:46.476743+0000 mgr.y (mgr.25072) 1696 : cluster 0 pgmap v1832: 89 pgs: 89 active+clean; 374 MiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 627 op/s 2023-10-01T11:13:48.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:48 smithi078 bash[78274]: audit 2023-10-01T11:13:47.348748+0000 mon.a (mon.0) 2169 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:13:48.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:48 smithi078 bash[78274]: audit 2023-10-01T11:13:47.373605+0000 mon.a (mon.0) 2170 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:13:48.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:48 smithi078 bash[78274]: audit 2023-10-01T11:13:47.494864+0000 mon.a (mon.0) 2171 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:13:48.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:48 smithi078 bash[78274]: audit 2023-10-01T11:13:47.496275+0000 mon.a (mon.0) 2172 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:13:48.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:48 smithi078 bash[78274]: audit 2023-10-01T11:13:47.516097+0000 mon.a (mon.0) 2173 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:13:48.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:48 smithi078 bash[78274]: audit 2023-10-01T11:13:47.540897+0000 mon.a (mon.0) 2174 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:13:48.431 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:48.460 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:48.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:48.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:13:48.564 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:48.599 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:48.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:48.602 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:13:48.702 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:48.729 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:48.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:48.732 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:13:48.833 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:48.861 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:48.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:48.866 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:13:48.966 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:49.001 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:49.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:49.005 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:13:49.106 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:49.138 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:49.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:49.141 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:13:49.241 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:49.267 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:49.271 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:49 smithi078 bash[75299]: audit 2023-10-01T11:13:48.229101+0000 mon.a (mon.0) 2175 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T11:13:49.271 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:49 smithi078 bash[75299]: audit 2023-10-01T11:13:48.230332+0000 mon.a (mon.0) 2176 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T11:13:49.271 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:49 smithi078 bash[75299]: audit 2023-10-01T11:13:48.235914+0000 mon.a (mon.0) 2177 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:13:49.271 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:49 smithi078 bash[78274]: audit 2023-10-01T11:13:48.229101+0000 mon.a (mon.0) 2175 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T11:13:49.272 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:49 smithi078 bash[78274]: audit 2023-10-01T11:13:48.230332+0000 mon.a (mon.0) 2176 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T11:13:49.272 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:49 smithi078 bash[78274]: audit 2023-10-01T11:13:48.235914+0000 mon.a (mon.0) 2177 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:13:49.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:49.273 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:13:49.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:49 smithi173 bash[58816]: audit 2023-10-01T11:13:48.229101+0000 mon.a (mon.0) 2175 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T11:13:49.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:49 smithi173 bash[58816]: audit 2023-10-01T11:13:48.230332+0000 mon.a (mon.0) 2176 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T11:13:49.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:49 smithi173 bash[58816]: audit 2023-10-01T11:13:48.235914+0000 mon.a (mon.0) 2177 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:13:49.373 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:49.398 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:49.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:49.400 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:13:49.501 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:49.533 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:49.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:49.537 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:13:49.637 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:49.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:49.669 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:49.669 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:13:49.770 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:49.804 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:49.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:49.809 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:13:49.910 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:49.942 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:49.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:49.948 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:13:50.049 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:50.072 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:50.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:50.081 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:13:50.182 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:50.218 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:50.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:50.227 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:13:50.327 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:50.337 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:50 smithi173 bash[58816]: cluster 2023-10-01T11:13:48.477274+0000 mgr.y (mgr.25072) 1697 : cluster 0 pgmap v1833: 89 pgs: 89 active+clean; 374 MiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 627 op/s 2023-10-01T11:13:50.359 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:50.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:50.363 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:13:50.464 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:50.474 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:50 smithi078 bash[75299]: cluster 2023-10-01T11:13:48.477274+0000 mgr.y (mgr.25072) 1697 : cluster 0 pgmap v1833: 89 pgs: 89 active+clean; 374 MiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 627 op/s 2023-10-01T11:13:50.474 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:13:50 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:13:50] "GET /metrics HTTP/1.1" 200 36379 "" "Prometheus/2.33.4" 2023-10-01T11:13:50.474 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:50 smithi078 bash[78274]: cluster 2023-10-01T11:13:48.477274+0000 mgr.y (mgr.25072) 1697 : cluster 0 pgmap v1833: 89 pgs: 89 active+clean; 374 MiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 627 op/s 2023-10-01T11:13:50.499 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:50.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:50.501 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:13:50.602 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:50.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:50.630 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:50.630 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:13:50.731 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:50.765 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:50.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:50.769 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:13:50.870 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:50.897 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:50.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:50.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:13:50.981 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:13:51.000 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:51.032 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:51.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:51.040 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:13:51.140 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:51.170 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:51.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:51.172 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:13:51.273 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:51.296 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:51.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:51.302 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:13:51.403 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:51.432 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:51.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:51.436 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:13:51.537 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:51.569 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:51.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:51.574 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:13:51.674 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:51.703 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:51.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:51.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:13:51.812 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:51.836 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:51.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:51.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:13:51.940 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:51.976 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:51.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:51.979 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:13:52.080 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:52.125 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:52.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:52.130 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:13:52.230 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:52.240 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:52 smithi173 bash[58816]: cluster 2023-10-01T11:13:50.479057+0000 mgr.y (mgr.25072) 1698 : cluster 0 pgmap v1834: 89 pgs: 89 active+clean; 223 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 678 op/s 2023-10-01T11:13:52.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:52.266 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:52.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:13:52.367 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:52.376 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:52 smithi078 bash[75299]: cluster 2023-10-01T11:13:50.479057+0000 mgr.y (mgr.25072) 1698 : cluster 0 pgmap v1834: 89 pgs: 89 active+clean; 223 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 678 op/s 2023-10-01T11:13:52.376 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:52 smithi078 bash[78274]: cluster 2023-10-01T11:13:50.479057+0000 mgr.y (mgr.25072) 1698 : cluster 0 pgmap v1834: 89 pgs: 89 active+clean; 223 MiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 678 op/s 2023-10-01T11:13:52.402 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:52.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:52.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:13:52.507 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:52.533 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:52.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:52.539 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:13:52.640 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:52.666 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:52.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:52.675 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:13:52.776 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:52.810 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:52.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:52.818 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:13:52.919 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:52.955 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:52.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:52.961 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:13:53.062 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:53.086 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:53.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:53.095 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:13:53.196 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:53.226 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:53.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:53.229 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:13:53.330 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:53.369 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:53.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:53.374 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:13:53.474 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:53.514 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:53.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:53.523 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:13:53.624 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:53.654 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:53.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:53.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:13:53.757 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:53.778 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:53.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:53.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:13:53.884 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:53.906 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:53.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:53.908 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:13:54.009 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:54.030 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:54.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:54.032 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:13:54.100 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Removed 105143 objects 2023-10-01T11:13:54.101 INFO:tasks.radosbench.radosbench.0.smithi078.stdout:Clean up completed and total clean up time :160.725 2023-10-01T11:13:54.133 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:54.160 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:54.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:54.162 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:13:54.263 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:54.270 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:54 smithi078 bash[75299]: cluster 2023-10-01T11:13:52.479597+0000 mgr.y (mgr.25072) 1699 : cluster 0 pgmap v1835: 89 pgs: 89 active+clean; 184 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 558 op/s 2023-10-01T11:13:54.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:54 smithi078 bash[78274]: cluster 2023-10-01T11:13:52.479597+0000 mgr.y (mgr.25072) 1699 : cluster 0 pgmap v1835: 89 pgs: 89 active+clean; 184 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 558 op/s 2023-10-01T11:13:54.287 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:54.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:54.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:13:54.390 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:54.423 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:54.425 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:54 smithi173 bash[58816]: cluster 2023-10-01T11:13:52.479597+0000 mgr.y (mgr.25072) 1699 : cluster 0 pgmap v1835: 89 pgs: 89 active+clean; 184 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 558 op/s 2023-10-01T11:13:54.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:54.426 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:13:54.527 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:54.559 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:54.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:54.562 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:13:54.662 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:54.685 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:54.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:54.687 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:13:54.697 INFO:teuthology.orchestra.run.smithi078.stdout: 2023-10-01T11:13:54.699 INFO:teuthology.orchestra.run.smithi078.stderr:dumped all 2023-10-01T11:13:54.788 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:54.805 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:54.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:54.807 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:13:54.908 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:54.938 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:54.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:54.941 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:13:55.042 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:55.074 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:55.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:55.078 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:13:55.178 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:55.206 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:55.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:55.208 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:13:55.309 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:55.334 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:55.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:55.336 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:13:55.436 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:55.465 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:55.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:55.468 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:13:55.568 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:55.599 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:55.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:55.601 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:13:55.702 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:55.736 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:55.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:55.739 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:13:55.839 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:55.868 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:55.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:55.873 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:13:55.973 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:56.006 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:56.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:56.008 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:13:56.109 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:56.136 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:56.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:56.138 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:13:56.239 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:56.260 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:56.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:56.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:13:56.363 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:56.385 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:56.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:56.388 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:13:56.489 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:56.502 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:56 smithi173 bash[58816]: cluster 2023-10-01T11:13:54.480090+0000 mgr.y (mgr.25072) 1700 : cluster 0 pgmap v1836: 89 pgs: 89 active+clean; 82 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 648 op/s 2023-10-01T11:13:56.502 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:56 smithi173 bash[58816]: audit 2023-10-01T11:13:54.681751+0000 mgr.y (mgr.25072) 1701 : audit 0 from='client.36957 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T11:13:56.524 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:56.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:56.528 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:13:56.616 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:56 smithi078 bash[75299]: cluster 2023-10-01T11:13:54.480090+0000 mgr.y (mgr.25072) 1700 : cluster 0 pgmap v1836: 89 pgs: 89 active+clean; 82 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 648 op/s 2023-10-01T11:13:56.617 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:56 smithi078 bash[75299]: audit 2023-10-01T11:13:54.681751+0000 mgr.y (mgr.25072) 1701 : audit 0 from='client.36957 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T11:13:56.617 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:56 smithi078 bash[78274]: cluster 2023-10-01T11:13:54.480090+0000 mgr.y (mgr.25072) 1700 : cluster 0 pgmap v1836: 89 pgs: 89 active+clean; 82 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 648 op/s 2023-10-01T11:13:56.617 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:56 smithi078 bash[78274]: audit 2023-10-01T11:13:54.681751+0000 mgr.y (mgr.25072) 1701 : audit 0 from='client.36957 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T11:13:56.628 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:56.660 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:56.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:56.663 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:13:56.763 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:56.790 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:56.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:56.793 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:13:56.894 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:56.918 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:56.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:56.920 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:13:57.021 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:57.051 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:57.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:57.055 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:13:57.156 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:57.188 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:57.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:57.191 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:13:57.292 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:57.323 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:57.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:57.326 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:13:57.353 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:57 smithi078 bash[75299]: cluster 2023-10-01T11:13:56.480803+0000 mgr.y (mgr.25072) 1702 : cluster 0 pgmap v1837: 89 pgs: 89 active+clean; 3.9 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 621 op/s 2023-10-01T11:13:57.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:57 smithi173 bash[58816]: cluster 2023-10-01T11:13:56.480803+0000 mgr.y (mgr.25072) 1702 : cluster 0 pgmap v1837: 89 pgs: 89 active+clean; 3.9 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 621 op/s 2023-10-01T11:13:57.428 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:57.436 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:57 smithi078 bash[78274]: cluster 2023-10-01T11:13:56.480803+0000 mgr.y (mgr.25072) 1702 : cluster 0 pgmap v1837: 89 pgs: 89 active+clean; 3.9 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 621 op/s 2023-10-01T11:13:57.451 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:57.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:57.453 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:13:57.554 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:57.581 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:57.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:57.583 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:13:57.684 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:57.712 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:57.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:57.715 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:13:57.816 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:57.847 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:57.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:57.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:13:57.951 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:57.976 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:57.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:57.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:13:58.080 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:58.110 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:58.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:58.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:13:58.213 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:58.241 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:58.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:58.244 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:13:58.345 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:58.368 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:58.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:58.372 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:13:58.472 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:58.505 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:58.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:58.507 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:13:58.608 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:58.630 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:58.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:58.633 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:13:58.720 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_9 2023-10-01T11:13:58.720 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd pool rm unique_pool_9 unique_pool_9 --yes-i-really-really-mean-it 2023-10-01T11:13:58.734 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:58.790 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:13:58.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:58.792 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:13:58.893 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:58.923 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:13:58.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:58.926 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:13:59.027 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:59.056 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:13:59.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:59.059 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:13:59.159 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:59.187 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:13:59.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:59.189 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:13:59.290 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:59.323 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:13:59.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:59.325 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:13:59.425 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:59.457 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:59.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:59.460 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:13:59.561 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:59.594 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:59.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:59.597 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:13:59.698 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:59.730 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:13:59.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:59.734 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:13:59.834 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:59.871 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:13:59.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:13:59.874 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:13:59.975 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:13:59.988 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:13:59 smithi173 bash[58816]: cluster 2023-10-01T11:13:58.481338+0000 mgr.y (mgr.25072) 1703 : cluster 0 pgmap v1838: 89 pgs: 89 active+clean; 3.9 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 492 op/s 2023-10-01T11:14:00.009 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:00.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:00.011 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:14:00.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:13:59 smithi078 bash[78274]: cluster 2023-10-01T11:13:58.481338+0000 mgr.y (mgr.25072) 1703 : cluster 0 pgmap v1838: 89 pgs: 89 active+clean; 3.9 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 492 op/s 2023-10-01T11:14:00.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:13:59 smithi078 bash[75299]: cluster 2023-10-01T11:13:58.481338+0000 mgr.y (mgr.25072) 1703 : cluster 0 pgmap v1838: 89 pgs: 89 active+clean; 3.9 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 492 op/s 2023-10-01T11:14:00.112 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:00.142 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:00.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:00.145 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:14:00.245 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:00.291 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:00.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:00.293 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:14:00.318 INFO:teuthology.orchestra.run.smithi078.stdout:{"pg_ready":true,"pg_map":{"version":1836,"stamp":"2023-10-01T11:13:54.479774+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":85465016,"num_objects":1320,"num_object_clones":0,"num_object_copies":3960,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1320,"num_whiteouts":0,"num_read":295,"num_read_kb":501,"num_write":209253,"num_write_kb":6731599,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":100,"num_bytes_recovered":5347666,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":143587,"ondisk_log_size":143587,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":9007216,"kb_used_data":334640,"kb_used_omap":30,"kb_used_meta":8672353,"kb_avail":740954000,"statfs":{"total":767960285184,"available":758736896000,"internally_reserved":0,"allocated":342671360,"data_stored":333881145,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31502,"internal_metadata":8880489714},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,3,4,6,10,3,7],"upper_bound":128},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-510197760,"num_objects":-7785,"num_object_clones":0,"num_object_copies":-23355,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-7785,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7785,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.010165"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":359,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.039768+0000","last_change":"2023-10-01T11:04:02.725956+0000","last_active":"2023-10-01T11:13:17.039768+0000","last_peered":"2023-10-01T11:13:17.039768+0000","last_clean":"2023-10-01T11:13:17.039768+0000","last_became_active":"2023-10-01T11:04:02.725317+0000","last_became_peered":"2023-10-01T11:04:02.725317+0000","last_unstale":"2023-10-01T11:13:17.039768+0000","last_undegraded":"2023-10-01T11:13:17.039768+0000","last_fullsized":"2023-10-01T11:13:17.039768+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:34:07.675650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":364,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.030122+0000","last_change":"2023-10-01T10:38:16.025881+0000","last_active":"2023-10-01T11:13:17.030122+0000","last_peered":"2023-10-01T11:13:17.030122+0000","last_clean":"2023-10-01T11:13:17.030122+0000","last_became_active":"2023-10-01T10:38:16.025518+0000","last_became_peered":"2023-10-01T10:38:16.025518+0000","last_unstale":"2023-10-01T11:13:17.030122+0000","last_undegraded":"2023-10-01T11:13:17.030122+0000","last_fullsized":"2023-10-01T11:13:17.030122+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:48:37.826604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":174,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-10-01T11:13:39.144720+0000","last_change":"2023-10-01T11:08:46.870944+0000","last_active":"2023-10-01T11:13:39.144720+0000","last_peered":"2023-10-01T11:13:39.144720+0000","last_clean":"2023-10-01T11:13:39.144720+0000","last_became_active":"2023-10-01T11:08:46.870425+0000","last_became_peered":"2023-10-01T11:08:46.870425+0000","last_unstale":"2023-10-01T11:13:39.144720+0000","last_undegraded":"2023-10-01T11:13:39.144720+0000","last_fullsized":"2023-10-01T11:13:39.144720+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:28:03.064900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":382,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.030274+0000","last_change":"2023-10-01T11:06:11.786067+0000","last_active":"2023-10-01T11:13:17.030274+0000","last_peered":"2023-10-01T11:13:17.030274+0000","last_clean":"2023-10-01T11:13:17.030274+0000","last_became_active":"2023-10-01T11:06:11.785586+0000","last_became_peered":"2023-10-01T11:06:11.785586+0000","last_unstale":"2023-10-01T11:13:17.030274+0000","last_undegraded":"2023-10-01T11:13:17.030274+0000","last_fullsized":"2023-10-01T11:13:17.030274+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:09:53.087537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":365,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.029857+0000","last_change":"2023-10-01T10:37:42.766600+0000","last_active":"2023-10-01T11:13:17.029857+0000","last_peered":"2023-10-01T11:13:17.029857+0000","last_clean":"2023-10-01T11:13:17.029857+0000","last_became_active":"2023-10-01T10:37:42.766051+0000","last_became_peered":"2023-10-01T10:37:42.766051+0000","last_unstale":"2023-10-01T11:13:17.029857+0000","last_undegraded":"2023-10-01T11:13:17.029857+0000","last_fullsized":"2023-10-01T11:13:17.029857+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:51:17.707963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":62,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.043311+0000","last_change":"2023-10-01T11:05:33.800368+0000","last_active":"2023-10-01T11:13:17.043311+0000","last_peered":"2023-10-01T11:13:17.043311+0000","last_clean":"2023-10-01T11:13:17.043311+0000","last_became_active":"2023-10-01T11:05:33.799877+0000","last_became_peered":"2023-10-01T11:05:33.799877+0000","last_unstale":"2023-10-01T11:13:17.043311+0000","last_undegraded":"2023-10-01T11:13:17.043311+0000","last_fullsized":"2023-10-01T11:13:17.043311+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":136,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.042584+0000","last_change":"2023-10-01T11:04:40.881723+0000","last_active":"2023-10-01T11:13:17.042584+0000","last_peered":"2023-10-01T11:13:17.042584+0000","last_clean":"2023-10-01T11:13:17.042584+0000","last_became_active":"2023-10-01T11:04:40.881381+0000","last_became_peered":"2023-10-01T11:04:40.881381+0000","last_unstale":"2023-10-01T11:13:17.042584+0000","last_undegraded":"2023-10-01T11:13:17.042584+0000","last_fullsized":"2023-10-01T11:13:17.042584+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":404,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.029921+0000","last_change":"2023-10-01T11:05:33.782561+0000","last_active":"2023-10-01T11:13:17.029921+0000","last_peered":"2023-10-01T11:13:17.029921+0000","last_clean":"2023-10-01T11:13:17.029921+0000","last_became_active":"2023-10-01T11:05:33.782066+0000","last_became_peered":"2023-10-01T11:05:33.782066+0000","last_unstale":"2023-10-01T11:13:17.029921+0000","last_undegraded":"2023-10-01T11:13:17.029921+0000","last_fullsized":"2023-10-01T11:13:17.029921+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:57:06.451343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":379,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.042311+0000","last_change":"2023-10-01T11:06:11.792061+0000","last_active":"2023-10-01T11:13:17.042311+0000","last_peered":"2023-10-01T11:13:17.042311+0000","last_clean":"2023-10-01T11:13:17.042311+0000","last_became_active":"2023-10-01T11:06:11.789129+0000","last_became_peered":"2023-10-01T11:06:11.789129+0000","last_unstale":"2023-10-01T11:13:17.042311+0000","last_undegraded":"2023-10-01T11:13:17.042311+0000","last_fullsized":"2023-10-01T11:13:17.042311+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":424,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.038672+0000","last_change":"2023-10-01T11:06:11.807749+0000","last_active":"2023-10-01T11:13:17.038672+0000","last_peered":"2023-10-01T11:13:17.038672+0000","last_clean":"2023-10-01T11:13:17.038672+0000","last_became_active":"2023-10-01T11:06:11.807375+0000","last_became_peered":"2023-10-01T11:06:11.807375+0000","last_unstale":"2023-10-01T11:13:17.038672+0000","last_undegraded":"2023-10-01T11:13:17.038672+0000","last_fullsized":"2023-10-01T11:13:17.038672+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:44:05.945387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":283,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.030109+0000","last_change":"2023-10-01T11:06:11.816274+0000","last_active":"2023-10-01T11:13:17.030109+0000","last_peered":"2023-10-01T11:13:17.030109+0000","last_clean":"2023-10-01T11:13:17.030109+0000","last_became_active":"2023-10-01T11:06:11.790196+0000","last_became_peered":"2023-10-01T11:06:11.790196+0000","last_unstale":"2023-10-01T11:13:17.030109+0000","last_undegraded":"2023-10-01T11:13:17.030109+0000","last_fullsized":"2023-10-01T11:13:17.030109+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:09:46.957797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":159,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:30.299537+0000","last_change":"2023-10-01T11:08:46.887634+0000","last_active":"2023-10-01T11:13:30.299537+0000","last_peered":"2023-10-01T11:13:30.299537+0000","last_clean":"2023-10-01T11:13:30.299537+0000","last_became_active":"2023-10-01T11:08:46.887226+0000","last_became_peered":"2023-10-01T11:08:46.887226+0000","last_unstale":"2023-10-01T11:13:30.299537+0000","last_undegraded":"2023-10-01T11:13:30.299537+0000","last_fullsized":"2023-10-01T11:13:30.299537+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:41:08.771365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":488,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:29.439214+0000","last_change":"2023-10-01T11:06:11.836035+0000","last_active":"2023-10-01T11:13:29.439214+0000","last_peered":"2023-10-01T11:13:29.439214+0000","last_clean":"2023-10-01T11:13:29.439214+0000","last_became_active":"2023-10-01T11:06:11.794834+0000","last_became_peered":"2023-10-01T11:06:11.794834+0000","last_unstale":"2023-10-01T11:13:29.439214+0000","last_undegraded":"2023-10-01T11:13:29.439214+0000","last_fullsized":"2023-10-01T11:13:29.439214+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:25:13.103100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":33138,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":383,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:24.438203+0000","last_change":"2023-10-01T11:05:33.782898+0000","last_active":"2023-10-01T11:13:24.438203+0000","last_peered":"2023-10-01T11:13:24.438203+0000","last_clean":"2023-10-01T11:13:24.438203+0000","last_became_active":"2023-10-01T11:05:33.782135+0000","last_became_peered":"2023-10-01T11:05:33.782135+0000","last_unstale":"2023-10-01T11:13:24.438203+0000","last_undegraded":"2023-10-01T11:13:24.438203+0000","last_fullsized":"2023-10-01T11:13:24.438203+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:56:49.807416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":129,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.039795+0000","last_change":"2023-10-01T11:06:55.803414+0000","last_active":"2023-10-01T11:13:17.039795+0000","last_peered":"2023-10-01T11:13:17.039795+0000","last_clean":"2023-10-01T11:13:17.039795+0000","last_became_active":"2023-10-01T11:06:55.802738+0000","last_became_peered":"2023-10-01T11:06:55.802738+0000","last_unstale":"2023-10-01T11:13:17.039795+0000","last_undegraded":"2023-10-01T11:13:17.039795+0000","last_fullsized":"2023-10-01T11:13:17.039795+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:55:57.911799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":409,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.029771+0000","last_change":"2023-10-01T11:06:11.789794+0000","last_active":"2023-10-01T11:13:17.029771+0000","last_peered":"2023-10-01T11:13:17.029771+0000","last_clean":"2023-10-01T11:13:17.029771+0000","last_became_active":"2023-10-01T11:06:11.789452+0000","last_became_peered":"2023-10-01T11:06:11.789452+0000","last_unstale":"2023-10-01T11:13:17.029771+0000","last_undegraded":"2023-10-01T11:13:17.029771+0000","last_fullsized":"2023-10-01T11:13:17.029771+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:18:02.050694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":369,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.037788+0000","last_change":"2023-10-01T11:06:11.784794+0000","last_active":"2023-10-01T11:13:17.037788+0000","last_peered":"2023-10-01T11:13:17.037788+0000","last_clean":"2023-10-01T11:13:17.037788+0000","last_became_active":"2023-10-01T11:06:11.784368+0000","last_became_peered":"2023-10-01T11:06:11.784368+0000","last_unstale":"2023-10-01T11:13:17.037788+0000","last_undegraded":"2023-10-01T11:13:17.037788+0000","last_fullsized":"2023-10-01T11:13:17.037788+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:06:52.307449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":396,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.042634+0000","last_change":"2023-10-01T11:06:11.794671+0000","last_active":"2023-10-01T11:13:17.042634+0000","last_peered":"2023-10-01T11:13:17.042634+0000","last_clean":"2023-10-01T11:13:17.042634+0000","last_became_active":"2023-10-01T11:06:11.794121+0000","last_became_peered":"2023-10-01T11:06:11.794121+0000","last_unstale":"2023-10-01T11:13:17.042634+0000","last_undegraded":"2023-10-01T11:13:17.042634+0000","last_fullsized":"2023-10-01T11:13:17.042634+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":387,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-10-01T11:13:39.127459+0000","last_change":"2023-10-01T11:06:11.796259+0000","last_active":"2023-10-01T11:13:39.127459+0000","last_peered":"2023-10-01T11:13:39.127459+0000","last_clean":"2023-10-01T11:13:39.127459+0000","last_became_active":"2023-10-01T11:06:11.795870+0000","last_became_peered":"2023-10-01T11:06:11.795870+0000","last_unstale":"2023-10-01T11:13:39.127459+0000","last_undegraded":"2023-10-01T11:13:39.127459+0000","last_fullsized":"2023-10-01T11:13:39.127459+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:49:21.072163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":369,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.030206+0000","last_change":"2023-10-01T11:06:11.785871+0000","last_active":"2023-10-01T11:13:17.030206+0000","last_peered":"2023-10-01T11:13:17.030206+0000","last_clean":"2023-10-01T11:13:17.030206+0000","last_became_active":"2023-10-01T11:06:11.785183+0000","last_became_peered":"2023-10-01T11:06:11.785183+0000","last_unstale":"2023-10-01T11:13:17.030206+0000","last_undegraded":"2023-10-01T11:13:17.030206+0000","last_fullsized":"2023-10-01T11:13:17.030206+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:07:55.556404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":72,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.042699+0000","last_change":"2023-10-01T11:05:33.900352+0000","last_active":"2023-10-01T11:13:17.042699+0000","last_peered":"2023-10-01T11:13:17.042699+0000","last_clean":"2023-10-01T11:13:17.042699+0000","last_became_active":"2023-10-01T11:05:33.788906+0000","last_became_peered":"2023-10-01T11:05:33.788906+0000","last_unstale":"2023-10-01T11:13:17.042699+0000","last_undegraded":"2023-10-01T11:13:17.042699+0000","last_fullsized":"2023-10-01T11:13:17.042699+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":385,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.033015+0000","last_change":"2023-10-01T11:06:11.798727+0000","last_active":"2023-10-01T11:13:17.033015+0000","last_peered":"2023-10-01T11:13:17.033015+0000","last_clean":"2023-10-01T11:13:17.033015+0000","last_became_active":"2023-10-01T11:06:11.798388+0000","last_became_peered":"2023-10-01T11:06:11.798388+0000","last_unstale":"2023-10-01T11:13:17.033015+0000","last_undegraded":"2023-10-01T11:13:17.033015+0000","last_fullsized":"2023-10-01T11:13:17.033015+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:10:21.576627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.2","version":"376'13146","reported_seq":13177,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-10-01T11:13:50.333354+0000","last_change":"2023-10-01T11:09:21.161135+0000","last_active":"2023-10-01T11:13:50.333354+0000","last_peered":"2023-10-01T11:13:50.333354+0000","last_clean":"2023-10-01T11:13:50.333354+0000","last_became_active":"2023-10-01T11:09:21.160719+0000","last_became_peered":"2023-10-01T11:09:21.160719+0000","last_unstale":"2023-10-01T11:13:50.333354+0000","last_undegraded":"2023-10-01T11:13:50.333354+0000","last_fullsized":"2023-10-01T11:13:50.333354+0000","mapping_epoch":356,"log_start":"361'5200","ondisk_log_start":"361'5200","created":356,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_clean_scrub_stamp":"2023-10-01T11:09:20.132185+0000","objects_scrubbed":0,"log_size":7946,"log_dups_size":0,"ondisk_log_size":7946,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:53:24.785514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11403264,"num_objects":174,"num_object_clones":0,"num_object_copies":522,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":174,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13146,"num_write_kb":426240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":275,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.040231+0000","last_change":"2023-10-01T11:06:11.840628+0000","last_active":"2023-10-01T11:13:17.040231+0000","last_peered":"2023-10-01T11:13:17.040231+0000","last_clean":"2023-10-01T11:13:17.040231+0000","last_became_active":"2023-10-01T11:06:11.800187+0000","last_became_peered":"2023-10-01T11:06:11.800187+0000","last_unstale":"2023-10-01T11:13:17.040231+0000","last_undegraded":"2023-10-01T11:13:17.040231+0000","last_fullsized":"2023-10-01T11:13:17.040231+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:24:48.900425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":417,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.042017+0000","last_change":"2023-10-01T11:06:11.830255+0000","last_active":"2023-10-01T11:13:17.042017+0000","last_peered":"2023-10-01T11:13:17.042017+0000","last_clean":"2023-10-01T11:13:17.042017+0000","last_became_active":"2023-10-01T11:06:11.829847+0000","last_became_peered":"2023-10-01T11:06:11.829847+0000","last_unstale":"2023-10-01T11:13:17.042017+0000","last_undegraded":"2023-10-01T11:13:17.042017+0000","last_fullsized":"2023-10-01T11:13:17.042017+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.3","version":"376'13116","reported_seq":13147,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-10-01T11:13:51.366914+0000","last_change":"2023-10-01T11:09:22.023134+0000","last_active":"2023-10-01T11:13:51.366914+0000","last_peered":"2023-10-01T11:13:51.366914+0000","last_clean":"2023-10-01T11:13:51.366914+0000","last_became_active":"2023-10-01T11:09:22.022727+0000","last_became_peered":"2023-10-01T11:09:22.022727+0000","last_unstale":"2023-10-01T11:13:51.366914+0000","last_undegraded":"2023-10-01T11:13:51.366914+0000","last_fullsized":"2023-10-01T11:13:51.366914+0000","mapping_epoch":356,"log_start":"361'5000","ondisk_log_start":"361'5000","created":356,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_clean_scrub_stamp":"2023-10-01T11:09:20.132185+0000","objects_scrubbed":0,"log_size":8116,"log_dups_size":0,"ondisk_log_size":8116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:24:57.467745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7471104,"num_objects":114,"num_object_clones":0,"num_object_copies":342,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":114,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13116,"num_write_kb":423360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":234,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.040329+0000","last_change":"2023-10-01T11:06:11.813139+0000","last_active":"2023-10-01T11:13:17.040329+0000","last_peered":"2023-10-01T11:13:17.040329+0000","last_clean":"2023-10-01T11:13:17.040329+0000","last_became_active":"2023-10-01T11:06:11.790124+0000","last_became_peered":"2023-10-01T11:06:11.790124+0000","last_unstale":"2023-10-01T11:13:17.040329+0000","last_undegraded":"2023-10-01T11:13:17.040329+0000","last_fullsized":"2023-10-01T11:13:17.040329+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:30:46.077244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":363,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.039618+0000","last_change":"2023-10-01T11:04:02.724840+0000","last_active":"2023-10-01T11:13:17.039618+0000","last_peered":"2023-10-01T11:13:17.039618+0000","last_clean":"2023-10-01T11:13:17.039618+0000","last_became_active":"2023-10-01T11:04:02.724476+0000","last_became_peered":"2023-10-01T11:04:02.724476+0000","last_unstale":"2023-10-01T11:13:17.039618+0000","last_undegraded":"2023-10-01T11:13:17.039618+0000","last_fullsized":"2023-10-01T11:13:17.039618+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:05:08.882495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.0","version":"376'13238","reported_seq":13269,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-10-01T11:13:54.047465+0000","last_change":"2023-10-01T11:09:22.023014+0000","last_active":"2023-10-01T11:13:54.047465+0000","last_peered":"2023-10-01T11:13:54.047465+0000","last_clean":"2023-10-01T11:13:54.047465+0000","last_became_active":"2023-10-01T11:09:22.022611+0000","last_became_peered":"2023-10-01T11:09:22.022611+0000","last_unstale":"2023-10-01T11:13:54.047465+0000","last_undegraded":"2023-10-01T11:13:54.047465+0000","last_fullsized":"2023-10-01T11:13:54.047465+0000","mapping_epoch":356,"log_start":"360'3200","ondisk_log_start":"360'3200","created":356,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_clean_scrub_stamp":"2023-10-01T11:09:20.132185+0000","objects_scrubbed":0,"log_size":10038,"log_dups_size":0,"ondisk_log_size":10038,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:23:59.916230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13238,"num_write_kb":423616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":345,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.042737+0000","last_change":"2023-10-01T11:06:11.820664+0000","last_active":"2023-10-01T11:13:17.042737+0000","last_peered":"2023-10-01T11:13:17.042737+0000","last_clean":"2023-10-01T11:13:17.042737+0000","last_became_active":"2023-10-01T11:06:11.792931+0000","last_became_peered":"2023-10-01T11:06:11.792931+0000","last_unstale":"2023-10-01T11:13:17.042737+0000","last_undegraded":"2023-10-01T11:13:17.042737+0000","last_fullsized":"2023-10-01T11:13:17.042737+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":700,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":71,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.042945+0000","last_change":"2023-10-01T11:06:11.795096+0000","last_active":"2023-10-01T11:13:17.042945+0000","last_peered":"2023-10-01T11:13:17.042945+0000","last_clean":"2023-10-01T11:13:17.042945+0000","last_became_active":"2023-10-01T11:06:11.794716+0000","last_became_peered":"2023-10-01T11:06:11.794716+0000","last_unstale":"2023-10-01T11:13:17.042945+0000","last_undegraded":"2023-10-01T11:13:17.042945+0000","last_fullsized":"2023-10-01T11:13:17.042945+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1","version":"376'12958","reported_seq":12989,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-10-01T11:13:51.317390+0000","last_change":"2023-10-01T11:09:21.156985+0000","last_active":"2023-10-01T11:13:51.317390+0000","last_peered":"2023-10-01T11:13:51.317390+0000","last_clean":"2023-10-01T11:13:51.317390+0000","last_became_active":"2023-10-01T11:09:21.156452+0000","last_became_peered":"2023-10-01T11:09:21.156452+0000","last_unstale":"2023-10-01T11:13:51.317390+0000","last_undegraded":"2023-10-01T11:13:51.317390+0000","last_fullsized":"2023-10-01T11:13:51.317390+0000","mapping_epoch":356,"log_start":"361'4800","ondisk_log_start":"361'4800","created":356,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_clean_scrub_stamp":"2023-10-01T11:09:20.132185+0000","objects_scrubbed":0,"log_size":8158,"log_dups_size":0,"ondisk_log_size":8158,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:15:04.145939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7602176,"num_objects":116,"num_object_clones":0,"num_object_copies":348,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":116,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12958,"num_write_kb":418368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":397,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.042200+0000","last_change":"2023-10-01T11:05:33.788917+0000","last_active":"2023-10-01T11:13:17.042200+0000","last_peered":"2023-10-01T11:13:17.042200+0000","last_clean":"2023-10-01T11:13:17.042200+0000","last_became_active":"2023-10-01T11:05:33.788078+0000","last_became_peered":"2023-10-01T11:05:33.788078+0000","last_unstale":"2023-10-01T11:13:17.042200+0000","last_undegraded":"2023-10-01T11:13:17.042200+0000","last_fullsized":"2023-10-01T11:13:17.042200+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":127,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:20.240005+0000","last_change":"2023-10-01T11:08:46.873065+0000","last_active":"2023-10-01T11:13:20.240005+0000","last_peered":"2023-10-01T11:13:20.240005+0000","last_clean":"2023-10-01T11:13:20.240005+0000","last_became_active":"2023-10-01T11:08:46.872692+0000","last_became_peered":"2023-10-01T11:08:46.872692+0000","last_unstale":"2023-10-01T11:13:20.240005+0000","last_undegraded":"2023-10-01T11:13:20.240005+0000","last_fullsized":"2023-10-01T11:13:20.240005+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:27:32.806247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.6","version":"376'13338","reported_seq":13369,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-10-01T11:13:54.097482+0000","last_change":"2023-10-01T11:09:22.025630+0000","last_active":"2023-10-01T11:13:54.097482+0000","last_peered":"2023-10-01T11:13:54.097482+0000","last_clean":"2023-10-01T11:13:54.097482+0000","last_became_active":"2023-10-01T11:09:22.025164+0000","last_became_peered":"2023-10-01T11:09:22.025164+0000","last_unstale":"2023-10-01T11:13:54.097482+0000","last_undegraded":"2023-10-01T11:13:54.097482+0000","last_fullsized":"2023-10-01T11:13:54.097482+0000","mapping_epoch":356,"log_start":"360'3300","ondisk_log_start":"360'3300","created":356,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_clean_scrub_stamp":"2023-10-01T11:09:20.132185+0000","objects_scrubbed":0,"log_size":10038,"log_dups_size":0,"ondisk_log_size":10038,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:05:50.808903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13338,"num_write_kb":426816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":354,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-10-01T11:13:39.129031+0000","last_change":"2023-10-01T10:38:16.022493+0000","last_active":"2023-10-01T11:13:39.129031+0000","last_peered":"2023-10-01T11:13:39.129031+0000","last_clean":"2023-10-01T11:13:39.129031+0000","last_became_active":"2023-10-01T10:38:16.022033+0000","last_became_peered":"2023-10-01T10:38:16.022033+0000","last_unstale":"2023-10-01T11:13:39.129031+0000","last_undegraded":"2023-10-01T11:13:39.129031+0000","last_fullsized":"2023-10-01T11:13:39.129031+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:49:05.262763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":393,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.040192+0000","last_change":"2023-10-01T11:05:33.785133+0000","last_active":"2023-10-01T11:13:17.040192+0000","last_peered":"2023-10-01T11:13:17.040192+0000","last_clean":"2023-10-01T11:13:17.040192+0000","last_became_active":"2023-10-01T11:05:33.784749+0000","last_became_peered":"2023-10-01T11:05:33.784749+0000","last_unstale":"2023-10-01T11:13:17.040192+0000","last_undegraded":"2023-10-01T11:13:17.040192+0000","last_fullsized":"2023-10-01T11:13:17.040192+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:08:01.664432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.7","version":"376'13181","reported_seq":13209,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-10-01T11:13:54.000281+0000","last_change":"2023-10-01T11:09:22.022235+0000","last_active":"2023-10-01T11:13:54.000281+0000","last_peered":"2023-10-01T11:13:54.000281+0000","last_clean":"2023-10-01T11:13:54.000281+0000","last_became_active":"2023-10-01T11:09:22.021331+0000","last_became_peered":"2023-10-01T11:09:22.021331+0000","last_unstale":"2023-10-01T11:13:54.000281+0000","last_undegraded":"2023-10-01T11:13:54.000281+0000","last_fullsized":"2023-10-01T11:13:54.000281+0000","mapping_epoch":356,"log_start":"360'3100","ondisk_log_start":"360'3100","created":356,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_clean_scrub_stamp":"2023-10-01T11:09:20.132185+0000","objects_scrubbed":0,"log_size":10081,"log_dups_size":0,"ondisk_log_size":10081,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13181,"num_write_kb":421952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":366,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.039654+0000","last_change":"2023-10-01T11:05:33.782142+0000","last_active":"2023-10-01T11:13:17.039654+0000","last_peered":"2023-10-01T11:13:17.039654+0000","last_clean":"2023-10-01T11:13:17.039654+0000","last_became_active":"2023-10-01T11:05:33.781725+0000","last_became_peered":"2023-10-01T11:05:33.781725+0000","last_unstale":"2023-10-01T11:13:17.039654+0000","last_undegraded":"2023-10-01T11:13:17.039654+0000","last_fullsized":"2023-10-01T11:13:17.039654+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:58:37.958611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":353,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.043599+0000","last_change":"2023-10-01T11:04:02.727282+0000","last_active":"2023-10-01T11:13:17.043599+0000","last_peered":"2023-10-01T11:13:17.043599+0000","last_clean":"2023-10-01T11:13:17.043599+0000","last_became_active":"2023-10-01T11:04:02.726818+0000","last_became_peered":"2023-10-01T11:04:02.726818+0000","last_unstale":"2023-10-01T11:13:17.043599+0000","last_undegraded":"2023-10-01T11:13:17.043599+0000","last_fullsized":"2023-10-01T11:13:17.043599+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:11:11.868291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.4","version":"376'12973","reported_seq":13001,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-10-01T11:13:53.451786+0000","last_change":"2023-10-01T11:09:21.163719+0000","last_active":"2023-10-01T11:13:53.451786+0000","last_peered":"2023-10-01T11:13:53.451786+0000","last_clean":"2023-10-01T11:13:53.451786+0000","last_became_active":"2023-10-01T11:09:21.163294+0000","last_became_peered":"2023-10-01T11:09:21.163294+0000","last_unstale":"2023-10-01T11:13:53.451786+0000","last_undegraded":"2023-10-01T11:13:53.451786+0000","last_fullsized":"2023-10-01T11:13:53.451786+0000","mapping_epoch":356,"log_start":"360'3200","ondisk_log_start":"360'3200","created":356,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_clean_scrub_stamp":"2023-10-01T11:09:20.132185+0000","objects_scrubbed":0,"log_size":9773,"log_dups_size":0,"ondisk_log_size":9773,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1507328,"num_objects":23,"num_object_clones":0,"num_object_copies":69,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":23,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12973,"num_write_kb":415872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":397,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.039927+0000","last_change":"2023-10-01T11:06:11.790944+0000","last_active":"2023-10-01T11:13:17.039927+0000","last_peered":"2023-10-01T11:13:17.039927+0000","last_clean":"2023-10-01T11:13:17.039927+0000","last_became_active":"2023-10-01T11:06:11.790151+0000","last_became_peered":"2023-10-01T11:06:11.790151+0000","last_unstale":"2023-10-01T11:13:17.039927+0000","last_undegraded":"2023-10-01T11:13:17.039927+0000","last_fullsized":"2023-10-01T11:13:17.039927+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:05:49.161868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":395,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.040376+0000","last_change":"2023-10-01T11:05:33.781934+0000","last_active":"2023-10-01T11:13:17.040376+0000","last_peered":"2023-10-01T11:13:17.040376+0000","last_clean":"2023-10-01T11:13:17.040376+0000","last_became_active":"2023-10-01T11:05:33.781555+0000","last_became_peered":"2023-10-01T11:05:33.781555+0000","last_unstale":"2023-10-01T11:13:17.040376+0000","last_undegraded":"2023-10-01T11:13:17.040376+0000","last_fullsized":"2023-10-01T11:13:17.040376+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:59:22.019053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.5","version":"376'13084","reported_seq":13115,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-10-01T11:13:49.438468+0000","last_change":"2023-10-01T11:09:22.023719+0000","last_active":"2023-10-01T11:13:49.438468+0000","last_peered":"2023-10-01T11:13:49.438468+0000","last_clean":"2023-10-01T11:13:49.438468+0000","last_became_active":"2023-10-01T11:09:22.023016+0000","last_became_peered":"2023-10-01T11:09:22.023016+0000","last_unstale":"2023-10-01T11:13:49.438468+0000","last_undegraded":"2023-10-01T11:13:49.438468+0000","last_fullsized":"2023-10-01T11:13:49.438468+0000","mapping_epoch":356,"log_start":"361'5300","ondisk_log_start":"361'5300","created":356,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_clean_scrub_stamp":"2023-10-01T11:09:20.132185+0000","objects_scrubbed":0,"log_size":7784,"log_dups_size":0,"ondisk_log_size":7784,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:55:17.819499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11927552,"num_objects":182,"num_object_clones":0,"num_object_copies":546,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":182,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13084,"num_write_kb":424512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":388,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.029789+0000","last_change":"2023-10-01T11:06:11.784722+0000","last_active":"2023-10-01T11:13:17.029789+0000","last_peered":"2023-10-01T11:13:17.029789+0000","last_clean":"2023-10-01T11:13:17.029789+0000","last_became_active":"2023-10-01T11:06:11.784348+0000","last_became_peered":"2023-10-01T11:06:11.784348+0000","last_unstale":"2023-10-01T11:13:17.029789+0000","last_undegraded":"2023-10-01T11:13:17.029789+0000","last_fullsized":"2023-10-01T11:13:17.029789+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:40:29.283179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":370,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.033555+0000","last_change":"2023-10-01T10:38:16.019755+0000","last_active":"2023-10-01T11:13:17.033555+0000","last_peered":"2023-10-01T11:13:17.033555+0000","last_clean":"2023-10-01T11:13:17.033555+0000","last_became_active":"2023-10-01T10:38:16.019397+0000","last_became_peered":"2023-10-01T10:38:16.019397+0000","last_unstale":"2023-10-01T11:13:17.033555+0000","last_undegraded":"2023-10-01T11:13:17.033555+0000","last_fullsized":"2023-10-01T11:13:17.033555+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:42:56.623225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.8","version":"376'13004","reported_seq":13035,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-10-01T11:13:50.307050+0000","last_change":"2023-10-01T11:09:22.022986+0000","last_active":"2023-10-01T11:13:50.307050+0000","last_peered":"2023-10-01T11:13:50.307050+0000","last_clean":"2023-10-01T11:13:50.307050+0000","last_became_active":"2023-10-01T11:09:22.022553+0000","last_became_peered":"2023-10-01T11:09:22.022553+0000","last_unstale":"2023-10-01T11:13:50.307050+0000","last_undegraded":"2023-10-01T11:13:50.307050+0000","last_fullsized":"2023-10-01T11:13:50.307050+0000","mapping_epoch":356,"log_start":"361'5100","ondisk_log_start":"361'5100","created":356,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_clean_scrub_stamp":"2023-10-01T11:09:20.132185+0000","objects_scrubbed":0,"log_size":7904,"log_dups_size":0,"ondisk_log_size":7904,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T22:14:03.287229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8781824,"num_objects":134,"num_object_clones":0,"num_object_copies":402,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":134,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13004,"num_write_kb":420416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":280,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.019272+0000","last_change":"2023-10-01T11:06:11.788951+0000","last_active":"2023-10-01T11:13:17.019272+0000","last_peered":"2023-10-01T11:13:17.019272+0000","last_clean":"2023-10-01T11:13:17.019272+0000","last_became_active":"2023-10-01T11:06:11.788617+0000","last_became_peered":"2023-10-01T11:06:11.788617+0000","last_unstale":"2023-10-01T11:13:17.019272+0000","last_undegraded":"2023-10-01T11:13:17.019272+0000","last_fullsized":"2023-10-01T11:13:17.019272+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:07:46.629250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":358,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.042551+0000","last_change":"2023-10-01T10:37:42.766814+0000","last_active":"2023-10-01T11:13:17.042551+0000","last_peered":"2023-10-01T11:13:17.042551+0000","last_clean":"2023-10-01T11:13:17.042551+0000","last_became_active":"2023-10-01T10:37:42.766451+0000","last_became_peered":"2023-10-01T10:37:42.766451+0000","last_unstale":"2023-10-01T11:13:17.042551+0000","last_undegraded":"2023-10-01T11:13:17.042551+0000","last_fullsized":"2023-10-01T11:13:17.042551+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":160,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.023298+0000","last_change":"2023-10-01T11:08:46.870961+0000","last_active":"2023-10-01T11:13:17.023298+0000","last_peered":"2023-10-01T11:13:17.023298+0000","last_clean":"2023-10-01T11:13:17.023298+0000","last_became_active":"2023-10-01T11:08:46.870680+0000","last_became_peered":"2023-10-01T11:08:46.870680+0000","last_unstale":"2023-10-01T11:13:17.023298+0000","last_undegraded":"2023-10-01T11:13:17.023298+0000","last_fullsized":"2023-10-01T11:13:17.023298+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:36:45.187763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.e","version":"376'12920","reported_seq":12950,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-10-01T11:13:49.429505+0000","last_change":"2023-10-01T11:09:21.163877+0000","last_active":"2023-10-01T11:13:49.429505+0000","last_peered":"2023-10-01T11:13:49.429505+0000","last_clean":"2023-10-01T11:13:49.429505+0000","last_became_active":"2023-10-01T11:09:21.163510+0000","last_became_peered":"2023-10-01T11:09:21.163510+0000","last_unstale":"2023-10-01T11:13:49.429505+0000","last_undegraded":"2023-10-01T11:13:49.429505+0000","last_fullsized":"2023-10-01T11:13:49.429505+0000","mapping_epoch":356,"log_start":"361'5200","ondisk_log_start":"361'5200","created":356,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_clean_scrub_stamp":"2023-10-01T11:09:20.132185+0000","objects_scrubbed":0,"log_size":7720,"log_dups_size":0,"ondisk_log_size":7720,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:49:26.529322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12058624,"num_objects":184,"num_object_clones":0,"num_object_copies":552,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":184,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12920,"num_write_kb":419328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":401,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.042419+0000","last_change":"2023-10-01T11:05:33.792877+0000","last_active":"2023-10-01T11:13:17.042419+0000","last_peered":"2023-10-01T11:13:17.042419+0000","last_clean":"2023-10-01T11:13:17.042419+0000","last_became_active":"2023-10-01T11:05:33.792054+0000","last_became_peered":"2023-10-01T11:05:33.792054+0000","last_unstale":"2023-10-01T11:13:17.042419+0000","last_undegraded":"2023-10-01T11:13:17.042419+0000","last_fullsized":"2023-10-01T11:13:17.042419+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":361,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.040053+0000","last_change":"2023-10-01T11:05:33.783100+0000","last_active":"2023-10-01T11:13:17.040053+0000","last_peered":"2023-10-01T11:13:17.040053+0000","last_clean":"2023-10-01T11:13:17.040053+0000","last_became_active":"2023-10-01T11:05:33.782557+0000","last_became_peered":"2023-10-01T11:05:33.782557+0000","last_unstale":"2023-10-01T11:13:17.040053+0000","last_undegraded":"2023-10-01T11:13:17.040053+0000","last_fullsized":"2023-10-01T11:13:17.040053+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:52:05.581697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":281,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.035534+0000","last_change":"2023-10-01T10:52:18.963437+0000","last_active":"2023-10-01T11:13:17.035534+0000","last_peered":"2023-10-01T11:13:17.035534+0000","last_clean":"2023-10-01T11:13:17.035534+0000","last_became_active":"2023-10-01T10:52:18.955165+0000","last_became_peered":"2023-10-01T10:52:18.955165+0000","last_unstale":"2023-10-01T11:13:17.035534+0000","last_undegraded":"2023-10-01T11:13:17.035534+0000","last_fullsized":"2023-10-01T11:13:17.035534+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:57:49.129100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.d","version":"376'12830","reported_seq":12859,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-10-01T11:13:53.555742+0000","last_change":"2023-10-01T11:09:21.165115+0000","last_active":"2023-10-01T11:13:53.555742+0000","last_peered":"2023-10-01T11:13:53.555742+0000","last_clean":"2023-10-01T11:13:53.555742+0000","last_became_active":"2023-10-01T11:09:21.164631+0000","last_became_peered":"2023-10-01T11:09:21.164631+0000","last_unstale":"2023-10-01T11:13:53.555742+0000","last_undegraded":"2023-10-01T11:13:53.555742+0000","last_fullsized":"2023-10-01T11:13:53.555742+0000","mapping_epoch":356,"log_start":"360'3100","ondisk_log_start":"360'3100","created":356,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_clean_scrub_stamp":"2023-10-01T11:09:20.132185+0000","objects_scrubbed":0,"log_size":9730,"log_dups_size":0,"ondisk_log_size":9730,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1376280,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12830,"num_write_kb":411201,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":362,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.033564+0000","last_change":"2023-10-01T10:37:42.762524+0000","last_active":"2023-10-01T11:13:17.033564+0000","last_peered":"2023-10-01T11:13:17.033564+0000","last_clean":"2023-10-01T11:13:17.033564+0000","last_became_active":"2023-10-01T10:37:42.762209+0000","last_became_peered":"2023-10-01T10:37:42.762209+0000","last_unstale":"2023-10-01T11:13:17.033564+0000","last_undegraded":"2023-10-01T11:13:17.033564+0000","last_fullsized":"2023-10-01T11:13:17.033564+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:45:38.171397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":315,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.036784+0000","last_change":"2023-10-01T11:05:33.799413+0000","last_active":"2023-10-01T11:13:17.036784+0000","last_peered":"2023-10-01T11:13:17.036784+0000","last_clean":"2023-10-01T11:13:17.036784+0000","last_became_active":"2023-10-01T11:05:33.799158+0000","last_became_peered":"2023-10-01T11:05:33.799158+0000","last_unstale":"2023-10-01T11:13:17.036784+0000","last_undegraded":"2023-10-01T11:13:17.036784+0000","last_fullsized":"2023-10-01T11:13:17.036784+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:39:05.963181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":189,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.019227+0000","last_change":"2023-10-01T11:08:46.870350+0000","last_active":"2023-10-01T11:13:17.019227+0000","last_peered":"2023-10-01T11:13:17.019227+0000","last_clean":"2023-10-01T11:13:17.019227+0000","last_became_active":"2023-10-01T11:08:46.869990+0000","last_became_peered":"2023-10-01T11:08:46.869990+0000","last_unstale":"2023-10-01T11:13:17.019227+0000","last_undegraded":"2023-10-01T11:13:17.019227+0000","last_fullsized":"2023-10-01T11:13:17.019227+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:32:25.096683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.f","version":"376'12998","reported_seq":13029,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-10-01T11:13:51.307905+0000","last_change":"2023-10-01T11:09:21.156775+0000","last_active":"2023-10-01T11:13:51.307905+0000","last_peered":"2023-10-01T11:13:51.307905+0000","last_clean":"2023-10-01T11:13:51.307905+0000","last_became_active":"2023-10-01T11:09:21.156403+0000","last_became_peered":"2023-10-01T11:09:21.156403+0000","last_unstale":"2023-10-01T11:13:51.307905+0000","last_undegraded":"2023-10-01T11:13:51.307905+0000","last_fullsized":"2023-10-01T11:13:51.307905+0000","mapping_epoch":356,"log_start":"361'4800","ondisk_log_start":"361'4800","created":356,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_clean_scrub_stamp":"2023-10-01T11:09:20.132185+0000","objects_scrubbed":0,"log_size":8198,"log_dups_size":0,"ondisk_log_size":8198,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:18:58.903554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8126464,"num_objects":124,"num_object_clones":0,"num_object_copies":372,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":124,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12998,"num_write_kb":419904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":383,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.042673+0000","last_change":"2023-10-01T11:06:11.791607+0000","last_active":"2023-10-01T11:13:17.042673+0000","last_peered":"2023-10-01T11:13:17.042673+0000","last_clean":"2023-10-01T11:13:17.042673+0000","last_became_active":"2023-10-01T11:06:11.789667+0000","last_became_peered":"2023-10-01T11:06:11.789667+0000","last_unstale":"2023-10-01T11:13:17.042673+0000","last_undegraded":"2023-10-01T11:13:17.042673+0000","last_fullsized":"2023-10-01T11:13:17.042673+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'139","reported_seq":657,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.044602+0000","last_change":"2023-10-01T11:08:15.927727+0000","last_active":"2023-10-01T11:13:17.044602+0000","last_peered":"2023-10-01T11:13:17.044602+0000","last_clean":"2023-10-01T11:13:17.044602+0000","last_became_active":"2023-10-01T11:08:15.927235+0000","last_became_peered":"2023-10-01T11:08:15.927235+0000","last_unstale":"2023-10-01T11:13:17.044602+0000","last_undegraded":"2023-10-01T11:13:17.044602+0000","last_fullsized":"2023-10-01T11:13:17.044602+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:09:17.221082+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:09:17.221082+0000","last_clean_scrub_stamp":"2023-10-01T10:09:17.221082+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":5313312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":389,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.037642+0000","last_change":"2023-10-01T11:05:33.788093+0000","last_active":"2023-10-01T11:13:17.037642+0000","last_peered":"2023-10-01T11:13:17.037642+0000","last_clean":"2023-10-01T11:13:17.037642+0000","last_became_active":"2023-10-01T11:05:33.787723+0000","last_became_peered":"2023-10-01T11:05:33.787723+0000","last_unstale":"2023-10-01T11:13:17.037642+0000","last_undegraded":"2023-10-01T11:13:17.037642+0000","last_fullsized":"2023-10-01T11:13:17.037642+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:04:55.138186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":411,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.043428+0000","last_change":"2023-10-01T11:06:11.788535+0000","last_active":"2023-10-01T11:13:17.043428+0000","last_peered":"2023-10-01T11:13:17.043428+0000","last_clean":"2023-10-01T11:13:17.043428+0000","last_became_active":"2023-10-01T11:06:11.788103+0000","last_became_peered":"2023-10-01T11:06:11.788103+0000","last_unstale":"2023-10-01T11:13:17.043428+0000","last_undegraded":"2023-10-01T11:13:17.043428+0000","last_fullsized":"2023-10-01T11:13:17.043428+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:02:49.482834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.c","version":"376'12948","reported_seq":12978,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-10-01T11:13:49.438854+0000","last_change":"2023-10-01T11:09:22.022461+0000","last_active":"2023-10-01T11:13:49.438854+0000","last_peered":"2023-10-01T11:13:49.438854+0000","last_clean":"2023-10-01T11:13:49.438854+0000","last_became_active":"2023-10-01T11:09:22.021657+0000","last_became_peered":"2023-10-01T11:09:22.021657+0000","last_unstale":"2023-10-01T11:13:49.438854+0000","last_undegraded":"2023-10-01T11:13:49.438854+0000","last_fullsized":"2023-10-01T11:13:49.438854+0000","mapping_epoch":356,"log_start":"361'5200","ondisk_log_start":"361'5200","created":356,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_clean_scrub_stamp":"2023-10-01T11:09:20.132185+0000","objects_scrubbed":0,"log_size":7748,"log_dups_size":0,"ondisk_log_size":7748,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:13:30.660488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14024704,"num_objects":214,"num_object_clones":0,"num_object_copies":642,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":214,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12948,"num_write_kb":421184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":273,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.019565+0000","last_change":"2023-10-01T11:08:46.989647+0000","last_active":"2023-10-01T11:13:17.019565+0000","last_peered":"2023-10-01T11:13:17.019565+0000","last_clean":"2023-10-01T11:13:17.019565+0000","last_became_active":"2023-10-01T11:08:46.988979+0000","last_became_peered":"2023-10-01T11:08:46.988979+0000","last_unstale":"2023-10-01T11:13:17.019565+0000","last_undegraded":"2023-10-01T11:13:17.019565+0000","last_fullsized":"2023-10-01T11:13:17.019565+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:10:06.594141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":161,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.039855+0000","last_change":"2023-10-01T11:04:40.868156+0000","last_active":"2023-10-01T11:13:17.039855+0000","last_peered":"2023-10-01T11:13:17.039855+0000","last_clean":"2023-10-01T11:13:17.039855+0000","last_became_active":"2023-10-01T11:04:40.867864+0000","last_became_peered":"2023-10-01T11:04:40.867864+0000","last_unstale":"2023-10-01T11:13:17.039855+0000","last_undegraded":"2023-10-01T11:13:17.039855+0000","last_fullsized":"2023-10-01T11:13:17.039855+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:37:15.571777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":152,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:24.068710+0000","last_change":"2023-10-01T11:04:40.880090+0000","last_active":"2023-10-01T11:13:24.068710+0000","last_peered":"2023-10-01T11:13:24.068710+0000","last_clean":"2023-10-01T11:13:24.068710+0000","last_became_active":"2023-10-01T11:04:40.879768+0000","last_became_peered":"2023-10-01T11:04:40.879768+0000","last_unstale":"2023-10-01T11:13:24.068710+0000","last_undegraded":"2023-10-01T11:13:24.068710+0000","last_fullsized":"2023-10-01T11:13:24.068710+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:25:31.189338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.9","version":"376'13024","reported_seq":13052,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-10-01T11:13:53.998111+0000","last_change":"2023-10-01T11:09:21.165167+0000","last_active":"2023-10-01T11:13:53.998111+0000","last_peered":"2023-10-01T11:13:53.998111+0000","last_clean":"2023-10-01T11:13:53.998111+0000","last_became_active":"2023-10-01T11:09:21.164717+0000","last_became_peered":"2023-10-01T11:09:21.164717+0000","last_unstale":"2023-10-01T11:13:53.998111+0000","last_undegraded":"2023-10-01T11:13:53.998111+0000","last_fullsized":"2023-10-01T11:13:53.998111+0000","mapping_epoch":356,"log_start":"360'3000","ondisk_log_start":"360'3000","created":356,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_clean_scrub_stamp":"2023-10-01T11:09:20.132185+0000","objects_scrubbed":0,"log_size":10024,"log_dups_size":0,"ondisk_log_size":10024,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131072,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13024,"num_write_kb":416832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":204,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:19.413865+0000","last_change":"2023-10-01T11:06:55.803529+0000","last_active":"2023-10-01T11:13:19.413865+0000","last_peered":"2023-10-01T11:13:19.413865+0000","last_clean":"2023-10-01T11:13:19.413865+0000","last_became_active":"2023-10-01T11:06:55.803266+0000","last_became_peered":"2023-10-01T11:06:55.803266+0000","last_unstale":"2023-10-01T11:13:19.413865+0000","last_undegraded":"2023-10-01T11:13:19.413865+0000","last_fullsized":"2023-10-01T11:13:19.413865+0000","mapping_epoch":338,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:57:06.218433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":261,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.035619+0000","last_change":"2023-10-01T10:52:18.964479+0000","last_active":"2023-10-01T11:13:17.035619+0000","last_peered":"2023-10-01T11:13:17.035619+0000","last_clean":"2023-10-01T11:13:17.035619+0000","last_became_active":"2023-10-01T10:52:18.963881+0000","last_became_peered":"2023-10-01T10:52:18.963881+0000","last_unstale":"2023-10-01T11:13:17.035619+0000","last_undegraded":"2023-10-01T11:13:17.035619+0000","last_fullsized":"2023-10-01T11:13:17.035619+0000","mapping_epoch":181,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:36:30.405755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":353,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.042605+0000","last_change":"2023-10-01T11:06:11.791253+0000","last_active":"2023-10-01T11:13:17.042605+0000","last_peered":"2023-10-01T11:13:17.042605+0000","last_clean":"2023-10-01T11:13:17.042605+0000","last_became_active":"2023-10-01T11:06:11.789517+0000","last_became_peered":"2023-10-01T11:06:11.789517+0000","last_unstale":"2023-10-01T11:13:17.042605+0000","last_undegraded":"2023-10-01T11:13:17.042605+0000","last_fullsized":"2023-10-01T11:13:17.042605+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"14.a","version":"376'13078","reported_seq":13104,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-10-01T11:13:54.014103+0000","last_change":"2023-10-01T11:09:21.162350+0000","last_active":"2023-10-01T11:13:54.014103+0000","last_peered":"2023-10-01T11:13:54.014103+0000","last_clean":"2023-10-01T11:13:54.014103+0000","last_became_active":"2023-10-01T11:09:21.162009+0000","last_became_peered":"2023-10-01T11:09:21.162009+0000","last_unstale":"2023-10-01T11:13:54.014103+0000","last_undegraded":"2023-10-01T11:13:54.014103+0000","last_fullsized":"2023-10-01T11:13:54.014103+0000","mapping_epoch":356,"log_start":"360'3000","ondisk_log_start":"360'3000","created":356,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_clean_scrub_stamp":"2023-10-01T11:09:20.132185+0000","objects_scrubbed":0,"log_size":10078,"log_dups_size":0,"ondisk_log_size":10078,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131072,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13078,"num_write_kb":418560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":391,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.033072+0000","last_change":"2023-10-01T11:06:11.837719+0000","last_active":"2023-10-01T11:13:17.033072+0000","last_peered":"2023-10-01T11:13:17.033072+0000","last_clean":"2023-10-01T11:13:17.033072+0000","last_became_active":"2023-10-01T11:06:11.798523+0000","last_became_peered":"2023-10-01T11:06:11.798523+0000","last_unstale":"2023-10-01T11:13:17.033072+0000","last_undegraded":"2023-10-01T11:13:17.033072+0000","last_fullsized":"2023-10-01T11:13:17.033072+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:53:34.413543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":313,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.019562+0000","last_change":"2023-10-01T11:08:46.872392+0000","last_active":"2023-10-01T11:13:17.019562+0000","last_peered":"2023-10-01T11:13:17.019562+0000","last_clean":"2023-10-01T11:13:17.019562+0000","last_became_active":"2023-10-01T11:08:46.872108+0000","last_became_peered":"2023-10-01T11:08:46.872108+0000","last_unstale":"2023-10-01T11:13:17.019562+0000","last_undegraded":"2023-10-01T11:13:17.019562+0000","last_fullsized":"2023-10-01T11:13:17.019562+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:01:40.481690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":374,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-10-01T11:13:39.128910+0000","last_change":"2023-10-01T11:06:11.784630+0000","last_active":"2023-10-01T11:13:39.128910+0000","last_peered":"2023-10-01T11:13:39.128910+0000","last_clean":"2023-10-01T11:13:39.128910+0000","last_became_active":"2023-10-01T11:06:11.784264+0000","last_became_peered":"2023-10-01T11:06:11.784264+0000","last_unstale":"2023-10-01T11:13:39.128910+0000","last_undegraded":"2023-10-01T11:13:39.128910+0000","last_fullsized":"2023-10-01T11:13:39.128910+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:03:03.888001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.b","version":"376'13156","reported_seq":13187,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-10-01T11:13:54.098136+0000","last_change":"2023-10-01T11:09:22.024506+0000","last_active":"2023-10-01T11:13:54.098136+0000","last_peered":"2023-10-01T11:13:54.098136+0000","last_clean":"2023-10-01T11:13:54.098136+0000","last_became_active":"2023-10-01T11:09:22.023185+0000","last_became_peered":"2023-10-01T11:09:22.023185+0000","last_unstale":"2023-10-01T11:13:54.098136+0000","last_undegraded":"2023-10-01T11:13:54.098136+0000","last_fullsized":"2023-10-01T11:13:54.098136+0000","mapping_epoch":356,"log_start":"360'3100","ondisk_log_start":"360'3100","created":356,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T11:09:20.132185+0000","last_clean_scrub_stamp":"2023-10-01T11:09:20.132185+0000","objects_scrubbed":0,"log_size":10056,"log_dups_size":0,"ondisk_log_size":10056,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:06:56.128744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13156,"num_write_kb":420992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":71,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.043218+0000","last_change":"2023-10-01T11:06:11.818913+0000","last_active":"2023-10-01T11:13:17.043218+0000","last_peered":"2023-10-01T11:13:17.043218+0000","last_clean":"2023-10-01T11:13:17.043218+0000","last_became_active":"2023-10-01T11:06:11.818533+0000","last_became_peered":"2023-10-01T11:06:11.818533+0000","last_unstale":"2023-10-01T11:13:17.043218+0000","last_undegraded":"2023-10-01T11:13:17.043218+0000","last_fullsized":"2023-10-01T11:13:17.043218+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":57,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.019773+0000","last_change":"2023-10-01T11:08:46.872670+0000","last_active":"2023-10-01T11:13:17.019773+0000","last_peered":"2023-10-01T11:13:17.019773+0000","last_clean":"2023-10-01T11:13:17.019773+0000","last_became_active":"2023-10-01T11:08:46.872300+0000","last_became_peered":"2023-10-01T11:08:46.872300+0000","last_unstale":"2023-10-01T11:13:17.019773+0000","last_undegraded":"2023-10-01T11:13:17.019773+0000","last_fullsized":"2023-10-01T11:13:17.019773+0000","mapping_epoch":352,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":353,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:57:28.746652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":359,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.039702+0000","last_change":"2023-10-01T11:05:33.784531+0000","last_active":"2023-10-01T11:13:17.039702+0000","last_peered":"2023-10-01T11:13:17.039702+0000","last_clean":"2023-10-01T11:13:17.039702+0000","last_became_active":"2023-10-01T11:05:33.784167+0000","last_became_peered":"2023-10-01T11:05:33.784167+0000","last_unstale":"2023-10-01T11:13:17.039702+0000","last_undegraded":"2023-10-01T11:13:17.039702+0000","last_fullsized":"2023-10-01T11:13:17.039702+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:34:16.434841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":91,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.040267+0000","last_change":"2023-10-01T11:06:11.791476+0000","last_active":"2023-10-01T11:13:17.040267+0000","last_peered":"2023-10-01T11:13:17.040267+0000","last_clean":"2023-10-01T11:13:17.040267+0000","last_became_active":"2023-10-01T11:06:11.791021+0000","last_became_peered":"2023-10-01T11:06:11.791021+0000","last_unstale":"2023-10-01T11:13:17.040267+0000","last_undegraded":"2023-10-01T11:13:17.040267+0000","last_fullsized":"2023-10-01T11:13:17.040267+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:14:03.385152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":369,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.035662+0000","last_change":"2023-10-01T10:37:42.762360+0000","last_active":"2023-10-01T11:13:17.035662+0000","last_peered":"2023-10-01T11:13:17.035662+0000","last_clean":"2023-10-01T11:13:17.035662+0000","last_became_active":"2023-10-01T10:37:42.761936+0000","last_became_peered":"2023-10-01T10:37:42.761936+0000","last_unstale":"2023-10-01T11:13:17.035662+0000","last_undegraded":"2023-10-01T11:13:17.035662+0000","last_fullsized":"2023-10-01T11:13:17.035662+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:50.637582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":360,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.042271+0000","last_change":"2023-10-01T10:38:16.037809+0000","last_active":"2023-10-01T11:13:17.042271+0000","last_peered":"2023-10-01T11:13:17.042271+0000","last_clean":"2023-10-01T11:13:17.042271+0000","last_became_active":"2023-10-01T10:38:16.037318+0000","last_became_peered":"2023-10-01T10:38:16.037318+0000","last_unstale":"2023-10-01T11:13:17.042271+0000","last_undegraded":"2023-10-01T11:13:17.042271+0000","last_fullsized":"2023-10-01T11:13:17.042271+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":361,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.039941+0000","last_change":"2023-10-01T11:04:02.726007+0000","last_active":"2023-10-01T11:13:17.039941+0000","last_peered":"2023-10-01T11:13:17.039941+0000","last_clean":"2023-10-01T11:13:17.039941+0000","last_became_active":"2023-10-01T11:04:02.725507+0000","last_became_peered":"2023-10-01T11:04:02.725507+0000","last_unstale":"2023-10-01T11:13:17.039941+0000","last_undegraded":"2023-10-01T11:13:17.039941+0000","last_fullsized":"2023-10-01T11:13:17.039941+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:32:04.611533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":423,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.042831+0000","last_change":"2023-10-01T11:05:33.830108+0000","last_active":"2023-10-01T11:13:17.042831+0000","last_peered":"2023-10-01T11:13:17.042831+0000","last_clean":"2023-10-01T11:13:17.042831+0000","last_became_active":"2023-10-01T11:05:33.791882+0000","last_became_peered":"2023-10-01T11:05:33.791882+0000","last_unstale":"2023-10-01T11:13:17.042831+0000","last_undegraded":"2023-10-01T11:13:17.042831+0000","last_fullsized":"2023-10-01T11:13:17.042831+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":361,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.039538+0000","last_change":"2023-10-01T11:04:02.733653+0000","last_active":"2023-10-01T11:13:17.039538+0000","last_peered":"2023-10-01T11:13:17.039538+0000","last_clean":"2023-10-01T11:13:17.039538+0000","last_became_active":"2023-10-01T11:04:02.733178+0000","last_became_peered":"2023-10-01T11:04:02.733178+0000","last_unstale":"2023-10-01T11:13:17.039538+0000","last_undegraded":"2023-10-01T11:13:17.039538+0000","last_fullsized":"2023-10-01T11:13:17.039538+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:18:25.249624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":423,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-10-01T11:13:39.130145+0000","last_change":"2023-10-01T11:05:33.777019+0000","last_active":"2023-10-01T11:13:39.130145+0000","last_peered":"2023-10-01T11:13:39.130145+0000","last_clean":"2023-10-01T11:13:39.130145+0000","last_became_active":"2023-10-01T11:05:33.776565+0000","last_became_peered":"2023-10-01T11:05:33.776565+0000","last_unstale":"2023-10-01T11:13:39.130145+0000","last_undegraded":"2023-10-01T11:13:39.130145+0000","last_fullsized":"2023-10-01T11:13:39.130145+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T22:00:18.890326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":378,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.032692+0000","last_change":"2023-10-01T10:37:42.765142+0000","last_active":"2023-10-01T11:13:17.032692+0000","last_peered":"2023-10-01T11:13:17.032692+0000","last_clean":"2023-10-01T11:13:17.032692+0000","last_became_active":"2023-10-01T10:37:42.764770+0000","last_became_peered":"2023-10-01T10:37:42.764770+0000","last_unstale":"2023-10-01T11:13:17.032692+0000","last_undegraded":"2023-10-01T11:13:17.032692+0000","last_fullsized":"2023-10-01T11:13:17.032692+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:23:33.527876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":90,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.042511+0000","last_change":"2023-10-01T11:04:40.889388+0000","last_active":"2023-10-01T11:13:17.042511+0000","last_peered":"2023-10-01T11:13:17.042511+0000","last_clean":"2023-10-01T11:13:17.042511+0000","last_became_active":"2023-10-01T11:04:40.888960+0000","last_became_peered":"2023-10-01T11:04:40.888960+0000","last_unstale":"2023-10-01T11:13:17.042511+0000","last_undegraded":"2023-10-01T11:13:17.042511+0000","last_fullsized":"2023-10-01T11:13:17.042511+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":397,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-10-01T11:13:17.040790+0000","last_change":"2023-10-01T11:05:33.785774+0000","last_active":"2023-10-01T11:13:17.040790+0000","last_peered":"2023-10-01T11:13:17.040790+0000","last_clean":"2023-10-01T11:13:17.040790+0000","last_became_active":"2023-10-01T11:05:33.785426+0000","last_became_peered":"2023-10-01T11:05:33.785426+0000","last_unstale":"2023-10-01T11:13:17.040790+0000","last_undegraded":"2023-10-01T11:13:17.040790+0000","last_fullsized":"2023-10-01T11:13:17.040790+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:41:32.634339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":14,"num_pg":16,"stat_sum":{"num_bytes":84869144,"num_objects":1296,"num_object_clones":0,"num_object_copies":3888,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1296,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":208992,"num_write_kb":6729153,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":239742976,"data_stored":239730760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":143392,"ondisk_log_size":143392,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":82,"num_bytes_recovered":34354,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30319,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":5313312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430937,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1501360,"kb_used_data":14440,"kb_used_omap":0,"kb_used_meta":1486912,"kb_avail":92243792,"statfs":{"total":95995035648,"available":94457643008,"internally_reserved":0,"allocated":14786560,"data_stored":14198096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1522597888},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 11:13:44 2023","interfaces":[{"interface":"back","average":{"1min":1.913,"5min":1.5940000000000001,"15min":1.3080000000000001},"min":{"1min":0.308,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":7.3700000000000001,"5min":8.6959999999999997,"15min":11.166},"last":6.5750000000000002},{"interface":"front","average":{"1min":1.9059999999999999,"5min":1.6020000000000001,"15min":1.4019999999999999},"min":{"1min":0.30399999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":7.3860000000000001,"5min":8.7119999999999997,"15min":11.18},"last":0.34799999999999998}]},{"osd":1,"last update":"Sun Oct 1 11:13:44 2023","interfaces":[{"interface":"back","average":{"1min":1.329,"5min":1.444,"15min":1.3500000000000001},"min":{"1min":0.316,"5min":0.28000000000000003,"15min":0.27400000000000002},"max":{"1min":4.8179999999999996,"5min":9.6259999999999994,"15min":14.132999999999999},"last":0.38800000000000001},{"interface":"front","average":{"1min":1.4670000000000001,"5min":1.5760000000000001,"15min":1.429},"min":{"1min":0.44400000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":5.1870000000000003,"5min":7.2530000000000001,"15min":13.558999999999999},"last":0.45900000000000002}]},{"osd":2,"last update":"Sun Oct 1 11:13:00 2023","interfaces":[{"interface":"back","average":{"1min":1.8240000000000001,"5min":1.6930000000000001,"15min":2.548},"min":{"1min":0.436,"5min":0.35199999999999998,"15min":0.33300000000000002},"max":{"1min":10.021000000000001,"5min":10.021000000000001,"15min":215.89699999999999},"last":0.51700000000000002},{"interface":"front","average":{"1min":1.532,"5min":1.819,"15min":2.3849999999999998},"min":{"1min":0.39300000000000002,"5min":0.32200000000000001,"15min":0.307},"max":{"1min":6.8810000000000002,"5min":14.663,"15min":215.77600000000001},"last":6.6200000000000001}]},{"osd":3,"last update":"Sun Oct 1 11:13:44 2023","interfaces":[{"interface":"back","average":{"1min":1.4019999999999999,"5min":1.2989999999999999,"15min":1.4430000000000001},"min":{"1min":0.38,"5min":0.38,"15min":0.36299999999999999},"max":{"1min":4.7199999999999998,"5min":6.9160000000000004,"15min":12.358000000000001},"last":0.49299999999999999},{"interface":"front","average":{"1min":1.5,"5min":1.3340000000000001,"15min":1.2589999999999999},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.28299999999999997},"max":{"1min":4.4619999999999997,"5min":7.7060000000000004,"15min":8.516},"last":3.3559999999999999}]},{"osd":4,"last update":"Sun Oct 1 11:13:22 2023","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":1.095,"15min":1.302},"min":{"1min":0.372,"5min":0.35199999999999998,"15min":0.33100000000000002},"max":{"1min":4.7750000000000004,"5min":6.1130000000000004,"15min":17.178999999999998},"last":1.417},{"interface":"front","average":{"1min":0.872,"5min":1.0209999999999999,"15min":1.321},"min":{"1min":0.46100000000000002,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":4.1070000000000002,"5min":6.1840000000000002,"15min":17.25},"last":1.296}]},{"osd":5,"last update":"Sun Oct 1 11:13:44 2023","interfaces":[{"interface":"back","average":{"1min":0.75900000000000001,"5min":1.1279999999999999,"15min":1.5609999999999999},"min":{"1min":0.435,"5min":0.40000000000000002,"15min":0.33100000000000002},"max":{"1min":2.8660000000000001,"5min":5.4850000000000003,"15min":18.998999999999999},"last":0.61899999999999999},{"interface":"front","average":{"1min":1.105,"5min":1.048,"15min":1.2110000000000001},"min":{"1min":0.40500000000000003,"5min":0.39200000000000002,"15min":0.33500000000000002},"max":{"1min":3.6840000000000002,"5min":5.3360000000000003,"15min":10.1},"last":2.8999999999999999}]},{"osd":6,"last update":"Sun Oct 1 11:13:44 2023","interfaces":[{"interface":"back","average":{"1min":0.70699999999999996,"5min":1.05,"15min":1.3979999999999999},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":1.6120000000000001,"5min":6.6479999999999997,"15min":15.433},"last":3.0470000000000002},{"interface":"front","average":{"1min":1.206,"5min":1.0640000000000001,"15min":1.3140000000000001},"min":{"1min":0.38500000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":7.1900000000000004,"5min":7.1900000000000004,"15min":11.536},"last":3.827}]}]},{"osd":6,"up_from":42,"seq":180388627177,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1201316,"kb_used_data":11740,"kb_used_omap":0,"kb_used_meta":1189568,"kb_avail":92543836,"statfs":{"total":95995035648,"available":94764888064,"internally_reserved":0,"allocated":12021760,"data_stored":11445147,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1218117632},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 11:13:35 2023","interfaces":[{"interface":"back","average":{"1min":1.478,"5min":1.8009999999999999,"15min":1.663},"min":{"1min":0.27100000000000002,"5min":0.27000000000000002,"15min":0.26800000000000002},"max":{"1min":3.8300000000000001,"5min":14.964,"15min":102.23},"last":5.016},{"interface":"front","average":{"1min":1.8620000000000001,"5min":1.593,"15min":1.597},"min":{"1min":0.35699999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":7.0140000000000002,"5min":9.0449999999999999,"15min":101.217},"last":0.69899999999999995}]},{"osd":1,"last update":"Sun Oct 1 11:12:57 2023","interfaces":[{"interface":"back","average":{"1min":2.8519999999999999,"5min":1.8999999999999999,"15min":1.333},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":8.9770000000000003,"5min":14.946999999999999,"15min":14.946999999999999},"last":1.111},{"interface":"front","average":{"1min":1.321,"5min":1.466,"15min":1.2410000000000001},"min":{"1min":0.317,"5min":0.31,"15min":0.31},"max":{"1min":4.6520000000000001,"5min":8.1159999999999997,"15min":8.1159999999999997},"last":0.62}]},{"osd":2,"last update":"Sun Oct 1 11:13:07 2023","interfaces":[{"interface":"back","average":{"1min":1.97,"5min":2.1890000000000001,"15min":2.4460000000000002},"min":{"1min":0.35399999999999998,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":5.7999999999999998,"5min":48.881999999999998,"15min":106.705},"last":0.56399999999999995},{"interface":"front","average":{"1min":1.8640000000000001,"5min":2.0870000000000002,"15min":2.5899999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":5.8970000000000002,"5min":48.948,"15min":106.806},"last":0.65300000000000002}]},{"osd":3,"last update":"Sun Oct 1 11:12:57 2023","interfaces":[{"interface":"back","average":{"1min":2.5009999999999999,"5min":2.1629999999999998,"15min":1.944},"min":{"1min":0.372,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":6.3659999999999997,"5min":50.225000000000001,"15min":102.273},"last":5.0570000000000004},{"interface":"front","average":{"1min":1.905,"5min":1.974,"15min":1.9399999999999999},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":4.7809999999999997,"5min":49.301000000000002,"15min":101.491},"last":0.629}]},{"osd":4,"last update":"Sun Oct 1 11:13:28 2023","interfaces":[{"interface":"back","average":{"1min":1.196,"5min":1.2190000000000001,"15min":1.177},"min":{"1min":0.36499999999999999,"5min":0.34200000000000003,"15min":0.30299999999999999},"max":{"1min":6.9589999999999996,"5min":8.0820000000000007,"15min":8.6289999999999996},"last":3.2090000000000001},{"interface":"front","average":{"1min":1.5369999999999999,"5min":1.357,"15min":1.496},"min":{"1min":0.40000000000000002,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":4.1840000000000002,"5min":10.628,"15min":15.462},"last":4.8049999999999997}]},{"osd":5,"last update":"Sun Oct 1 11:12:57 2023","interfaces":[{"interface":"back","average":{"1min":1.5409999999999999,"5min":1.6970000000000001,"15min":1.2989999999999999},"min":{"1min":0.34200000000000003,"5min":0.33100000000000002,"15min":0.33000000000000002},"max":{"1min":6.444,"5min":26.379000000000001,"15min":26.379000000000001},"last":5.0990000000000002},{"interface":"front","average":{"1min":2.028,"5min":1.9830000000000001,"15min":1.5309999999999999},"min":{"1min":0.46800000000000003,"5min":0.32000000000000001,"15min":0.315},"max":{"1min":9.9700000000000006,"5min":28.010000000000002,"15min":28.010000000000002},"last":2.3100000000000001}]},{"osd":7,"last update":"Sun Oct 1 11:13:35 2023","interfaces":[{"interface":"back","average":{"1min":0.89600000000000002,"5min":1.1970000000000001,"15min":1.256},"min":{"1min":0.41199999999999998,"5min":0.378,"15min":0.35499999999999998},"max":{"1min":3.29,"5min":8.1370000000000005,"15min":9.5419999999999998},"last":0.52300000000000002},{"interface":"front","average":{"1min":1.0169999999999999,"5min":1.169,"15min":1.2729999999999999},"min":{"1min":0.40300000000000002,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":3.7970000000000002,"5min":14.894,"15min":14.894},"last":5.0819999999999999}]}]},{"osd":1,"up_from":101,"seq":433791697394,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1057396,"kb_used_data":92972,"kb_used_omap":4,"kb_used_meta":964411,"kb_avail":92687756,"statfs":{"total":95995035648,"available":94912262144,"internally_reserved":0,"allocated":95203328,"data_stored":93748792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":987557774},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 11:12:59 2023","interfaces":[{"interface":"back","average":{"1min":2.1379999999999999,"5min":1.357,"15min":1.1439999999999999},"min":{"1min":0.27000000000000002,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":11.756,"5min":11.756,"15min":11.756},"last":3.4460000000000002},{"interface":"front","average":{"1min":3.4489999999999998,"5min":1.7769999999999999,"15min":1.2509999999999999},"min":{"1min":0.28499999999999998,"5min":0.215,"15min":0.215},"max":{"1min":11.943,"5min":11.943,"15min":11.943},"last":0.58299999999999996}]},{"osd":2,"last update":"Sun Oct 1 11:13:27 2023","interfaces":[{"interface":"back","average":{"1min":2.5329999999999999,"5min":3.3809999999999998,"15min":2.2040000000000002},"min":{"1min":0.38300000000000001,"5min":0.23400000000000001,"15min":0.23200000000000001},"max":{"1min":10.143000000000001,"5min":130.90899999999999,"15min":130.90899999999999},"last":1.9970000000000001},{"interface":"front","average":{"1min":4.2670000000000003,"5min":3.6000000000000001,"15min":2.3959999999999999},"min":{"1min":0.307,"5min":0.307,"15min":0.247},"max":{"1min":12.085000000000001,"5min":130.72800000000001,"15min":130.72800000000001},"last":1.927}]},{"osd":3,"last update":"Sun Oct 1 11:12:59 2023","interfaces":[{"interface":"back","average":{"1min":5.3929999999999998,"5min":4.6269999999999998,"15min":2.327},"min":{"1min":0.249,"5min":0.249,"15min":0.248},"max":{"1min":56.595999999999997,"5min":130.79599999999999,"15min":130.79599999999999},"last":3.7050000000000001},{"interface":"front","average":{"1min":7.1459999999999999,"5min":5.351,"15min":2.5310000000000001},"min":{"1min":0.34599999999999997,"5min":0.219,"15min":0.219},"max":{"1min":56.747,"5min":130.83500000000001,"15min":130.83500000000001},"last":0.67600000000000005}]},{"osd":4,"last update":"Sun Oct 1 11:13:27 2023","interfaces":[{"interface":"back","average":{"1min":1.6970000000000001,"5min":3.1280000000000001,"15min":2.5019999999999998},"min":{"1min":0.56999999999999995,"5min":0.33300000000000002,"15min":0.32600000000000001},"max":{"1min":5.5780000000000003,"5min":130.702,"15min":203.89500000000001},"last":3.6749999999999998},{"interface":"front","average":{"1min":4.4409999999999998,"5min":3.9510000000000001,"15min":2.8700000000000001},"min":{"1min":0.40500000000000003,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":13.138999999999999,"5min":130.572,"15min":203.852},"last":4.5640000000000001}]},{"osd":5,"last update":"Sun Oct 1 11:12:59 2023","interfaces":[{"interface":"back","average":{"1min":2.621,"5min":1.8160000000000001,"15min":1.3560000000000001},"min":{"1min":0.498,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":11.042999999999999,"5min":12.192,"15min":12.192},"last":0.89700000000000002},{"interface":"front","average":{"1min":1.5700000000000001,"5min":2.9380000000000002,"15min":1.8029999999999999},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.34200000000000003},"max":{"1min":5.6349999999999998,"5min":130.62299999999999,"15min":130.62299999999999},"last":6.7960000000000003}]},{"osd":6,"last update":"Sun Oct 1 11:12:59 2023","interfaces":[{"interface":"back","average":{"1min":1.393,"5min":2.9710000000000001,"15min":1.8300000000000001},"min":{"1min":0.48799999999999999,"5min":0.371,"15min":0.33800000000000002},"max":{"1min":5.6109999999999998,"5min":130.75299999999999,"15min":130.75299999999999},"last":3.4910000000000001},{"interface":"front","average":{"1min":3.4289999999999998,"5min":2.0150000000000001,"15min":1.472},"min":{"1min":0.441,"5min":0.38400000000000001,"15min":0.34699999999999998},"max":{"1min":12.01,"5min":12.01,"15min":12.01},"last":0.97299999999999998}]},{"osd":7,"last update":"Sun Oct 1 11:12:59 2023","interfaces":[{"interface":"back","average":{"1min":3.508,"5min":2.0529999999999999,"15min":1.49},"min":{"1min":0.40699999999999997,"5min":0.40100000000000002,"15min":0.38},"max":{"1min":12.055999999999999,"5min":12.055999999999999,"15min":12.055999999999999},"last":1.242},{"interface":"front","average":{"1min":3.222,"5min":3.4369999999999998,"15min":2.0209999999999999},"min":{"1min":0.70299999999999996,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":8.6170000000000009,"5min":130.42099999999999,"15min":130.42099999999999},"last":0.754}]}]},{"osd":0,"up_from":86,"seq":369367188005,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1060608,"kb_used_data":26104,"kb_used_omap":5,"kb_used_meta":1034426,"kb_avail":92684544,"statfs":{"total":95995035648,"available":94908973056,"internally_reserved":0,"allocated":26730496,"data_stored":25321083,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1059252858},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,2,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 11:13:06 2023","interfaces":[{"interface":"back","average":{"1min":2.4180000000000001,"5min":1.591,"15min":1.575},"min":{"1min":0.186,"5min":0.186,"15min":0.186},"max":{"1min":6.6719999999999997,"5min":7.8570000000000002,"15min":119.911},"last":0.505},{"interface":"front","average":{"1min":4.0830000000000002,"5min":2.649,"15min":2.008},"min":{"1min":0.21099999999999999,"5min":0.189,"15min":0.189},"max":{"1min":11.983000000000001,"5min":13.989000000000001,"15min":119.76300000000001},"last":6.5119999999999996}]},{"osd":2,"last update":"Sun Oct 1 11:13:09 2023","interfaces":[{"interface":"back","average":{"1min":2.274,"5min":2.5310000000000001,"15min":2.23},"min":{"1min":0.23100000000000001,"5min":0.23000000000000001,"15min":0.20699999999999999},"max":{"1min":7.9240000000000004,"5min":67.076999999999998,"15min":72.064999999999998},"last":4.4619999999999997},{"interface":"front","average":{"1min":3.367,"5min":2.847,"15min":2.4660000000000002},"min":{"1min":0.26400000000000001,"5min":0.247,"15min":0.247},"max":{"1min":10.569000000000001,"5min":67.129999999999995,"15min":76.328999999999994},"last":4.4950000000000001}]},{"osd":3,"last update":"Sun Oct 1 11:13:06 2023","interfaces":[{"interface":"back","average":{"1min":2.278,"5min":2.5550000000000002,"15min":2.1779999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.214},"max":{"1min":10.614000000000001,"5min":67.706000000000003,"15min":81.674999999999997},"last":0.76500000000000001},{"interface":"front","average":{"1min":2.5,"5min":2.5600000000000001,"15min":2.1139999999999999},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":6.2690000000000001,"5min":66.352000000000004,"15min":70.950999999999993},"last":0.67800000000000005}]},{"osd":4,"last update":"Sun Oct 1 11:13:35 2023","interfaces":[{"interface":"back","average":{"1min":3.0510000000000002,"5min":1.9419999999999999,"15min":1.5840000000000001},"min":{"1min":0.5,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":13.147,"5min":23.995000000000001,"15min":23.995000000000001},"last":0.53000000000000003},{"interface":"front","average":{"1min":3.4409999999999998,"5min":2.3180000000000001,"15min":1.7490000000000001},"min":{"1min":0.46800000000000003,"5min":0.33600000000000002,"15min":0.33500000000000002},"max":{"1min":12.015000000000001,"5min":21.451000000000001,"15min":21.451000000000001},"last":0.81699999999999995}]},{"osd":5,"last update":"Sun Oct 1 11:13:06 2023","interfaces":[{"interface":"back","average":{"1min":1.6259999999999999,"5min":1.9650000000000001,"15min":1.6859999999999999},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.309},"max":{"1min":6.2759999999999998,"5min":12.776999999999999,"15min":39.860999999999997},"last":0.73499999999999999},{"interface":"front","average":{"1min":2.0699999999999998,"5min":1.978,"15min":1.724},"min":{"1min":0.435,"5min":0.41599999999999998,"15min":0.38200000000000001},"max":{"1min":5.8369999999999997,"5min":14.529,"15min":39.646000000000001},"last":0.54700000000000004}]},{"osd":6,"last update":"Sun Oct 1 11:13:06 2023","interfaces":[{"interface":"back","average":{"1min":1.919,"5min":1.5800000000000001,"15min":1.466},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.373},"max":{"1min":5.8019999999999996,"5min":8.1660000000000004,"15min":9.4570000000000007},"last":0.69799999999999995},{"interface":"front","average":{"1min":2.6400000000000001,"5min":1.837,"15min":1.5309999999999999},"min":{"1min":0.41099999999999998,"5min":0.373,"15min":0.373},"max":{"1min":12.035,"5min":12.035,"15min":12.035},"last":0.79500000000000004}]},{"osd":7,"last update":"Sun Oct 1 11:13:06 2023","interfaces":[{"interface":"back","average":{"1min":1.742,"5min":1.458,"15min":1.429},"min":{"1min":0.45800000000000002,"5min":0.42999999999999999,"15min":0.35499999999999998},"max":{"1min":5.8780000000000001,"5min":8.3109999999999999,"15min":12.680999999999999},"last":0.56399999999999995},{"interface":"front","average":{"1min":3.302,"5min":2.0779999999999998,"15min":1.7130000000000001},"min":{"1min":0.55900000000000005,"5min":0.433,"15min":0.32200000000000001},"max":{"1min":12.053000000000001,"5min":12.053000000000001,"15min":12.053000000000001},"last":0.83699999999999997}]}]},{"osd":2,"up_from":116,"seq":498216206779,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":926768,"kb_used_data":15400,"kb_used_omap":7,"kb_used_meta":911288,"kb_avail":92818384,"statfs":{"total":95995035648,"available":95046025216,"internally_reserved":0,"allocated":15769600,"data_stored":14373012,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":933158993},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 11:13:15 2023","interfaces":[{"interface":"back","average":{"1min":1.472,"5min":1.1930000000000001,"15min":1.163},"min":{"1min":0.20899999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":4.4320000000000004,"5min":5.3630000000000004,"15min":9.4510000000000005},"last":1.579},{"interface":"front","average":{"1min":3.0990000000000002,"5min":2.8980000000000001,"15min":1.8180000000000001},"min":{"1min":0.42699999999999999,"5min":0.222,"15min":0.187},"max":{"1min":8.8559999999999999,"5min":95.688999999999993,"15min":95.688999999999993},"last":1.107}]},{"osd":1,"last update":"Sun Oct 1 11:13:15 2023","interfaces":[{"interface":"back","average":{"1min":4.694,"5min":2.641,"15min":1.744},"min":{"1min":0.26100000000000001,"5min":0.254,"15min":0.20699999999999999},"max":{"1min":34.753999999999998,"5min":34.753999999999998,"15min":34.753999999999998},"last":0.46300000000000002},{"interface":"front","average":{"1min":4.5380000000000003,"5min":2.6059999999999999,"15min":1.865},"min":{"1min":0.39800000000000002,"5min":0.27200000000000002,"15min":0.23999999999999999},"max":{"1min":34.972999999999999,"5min":34.972999999999999,"15min":34.972999999999999},"last":0.86299999999999999}]},{"osd":3,"last update":"Sun Oct 1 11:13:51 2023","interfaces":[{"interface":"back","average":{"1min":2.9100000000000001,"5min":1.8109999999999999,"15min":1.4470000000000001},"min":{"1min":0.35399999999999998,"5min":0.24399999999999999,"15min":0.23200000000000001},"max":{"1min":13.744,"5min":13.744,"15min":23.879000000000001},"last":5.1289999999999996},{"interface":"front","average":{"1min":2.5499999999999998,"5min":1.9239999999999999,"15min":1.552},"min":{"1min":0.308,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":8.2119999999999997,"5min":10.102,"15min":23.704000000000001},"last":0.68999999999999995}]},{"osd":4,"last update":"Sun Oct 1 11:13:33 2023","interfaces":[{"interface":"back","average":{"1min":1.5069999999999999,"5min":1.546,"15min":1.556},"min":{"1min":0.36699999999999999,"5min":0.312,"15min":0.312},"max":{"1min":5.0739999999999998,"5min":10.004,"15min":10.004},"last":0.71599999999999997},{"interface":"front","average":{"1min":2.988,"5min":2.2559999999999998,"15min":1.946},"min":{"1min":0.34100000000000003,"5min":0.33000000000000002,"15min":0.317},"max":{"1min":12.989000000000001,"5min":12.989000000000001,"15min":12.989000000000001},"last":6.0750000000000002}]},{"osd":5,"last update":"Sun Oct 1 11:13:15 2023","interfaces":[{"interface":"back","average":{"1min":2.0569999999999999,"5min":1.6279999999999999,"15min":1.593},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":5.6550000000000002,"5min":9.968,"15min":10.180999999999999},"last":0.59499999999999997},{"interface":"front","average":{"1min":1.931,"5min":2.121,"15min":1.8899999999999999},"min":{"1min":0.377,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":5.3390000000000004,"5min":12.518000000000001,"15min":12.518000000000001},"last":5.9950000000000001}]},{"osd":6,"last update":"Sun Oct 1 11:13:15 2023","interfaces":[{"interface":"back","average":{"1min":1.6850000000000001,"5min":2.7530000000000001,"15min":2.1720000000000002},"min":{"1min":0.40200000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":5.3159999999999998,"5min":95.802999999999997,"15min":95.802999999999997},"last":0.90000000000000002},{"interface":"front","average":{"1min":2.004,"5min":1.5680000000000001,"15min":1.393},"min":{"1min":0.49199999999999999,"5min":0.40999999999999998,"15min":0.33700000000000002},"max":{"1min":6.3819999999999997,"5min":6.827,"15min":9.6709999999999994},"last":1.6020000000000001}]},{"osd":7,"last update":"Sun Oct 1 11:13:15 2023","interfaces":[{"interface":"back","average":{"1min":1.5860000000000001,"5min":1.458,"15min":1.419},"min":{"1min":0.442,"5min":0.40100000000000002,"15min":0.35099999999999998},"max":{"1min":4.5039999999999996,"5min":6.9169999999999998,"15min":11.209},"last":0.5},{"interface":"front","average":{"1min":2.3799999999999999,"5min":2.6400000000000001,"15min":1.929},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":5.7000000000000002,"5min":95.228999999999999,"15min":95.228999999999999},"last":0.627}]}]},{"osd":3,"up_from":121,"seq":519691043252,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":925340,"kb_used_data":71700,"kb_used_omap":2,"kb_used_meta":853629,"kb_avail":92819812,"statfs":{"total":95995035648,"available":95047487488,"internally_reserved":0,"allocated":73420800,"data_stored":72041844,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":874116149},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,0,0,0,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 11:12:50 2023","interfaces":[{"interface":"back","average":{"1min":2.4849999999999999,"5min":1.5429999999999999,"15min":1.6339999999999999},"min":{"1min":0.46400000000000002,"5min":0.222,"15min":0.17199999999999999},"max":{"1min":7.4859999999999998,"5min":9.625,"15min":100.191},"last":4.0529999999999999},{"interface":"front","average":{"1min":1.9390000000000001,"5min":1.6080000000000001,"15min":1.7669999999999999},"min":{"1min":0.313,"5min":0.217,"15min":0.20000000000000001},"max":{"1min":5.9359999999999999,"5min":12.196999999999999,"15min":99.852999999999994},"last":0.50600000000000001}]},{"osd":1,"last update":"Sun Oct 1 11:12:50 2023","interfaces":[{"interface":"back","average":{"1min":2.1419999999999999,"5min":2.173,"15min":1.4670000000000001},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.22500000000000001},"max":{"1min":7.4450000000000003,"5min":51.070999999999998,"15min":51.070999999999998},"last":3.3410000000000002},{"interface":"front","average":{"1min":2.1259999999999999,"5min":2.0110000000000001,"15min":1.474},"min":{"1min":0.28399999999999997,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":7.5259999999999998,"5min":51.034999999999997,"15min":51.034999999999997},"last":0.22}]},{"osd":2,"last update":"Sun Oct 1 11:12:50 2023","interfaces":[{"interface":"back","average":{"1min":7.8579999999999997,"5min":3.984,"15min":2.8090000000000002},"min":{"1min":0.32600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":90.540000000000006,"5min":90.540000000000006,"15min":90.540000000000006},"last":15.789},{"interface":"front","average":{"1min":6.4770000000000003,"5min":3.427,"15min":2.673},"min":{"1min":0.32800000000000001,"5min":0.29299999999999998,"15min":0.27100000000000002},"max":{"1min":90.367000000000004,"5min":90.367000000000004,"15min":90.367000000000004},"last":15.747999999999999}]},{"osd":4,"last update":"Sun Oct 1 11:13:38 2023","interfaces":[{"interface":"back","average":{"1min":1.492,"5min":1.3360000000000001,"15min":1.3919999999999999},"min":{"1min":0.46100000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":5.931,"5min":7.3579999999999997,"15min":10.808999999999999},"last":0.43099999999999999},{"interface":"front","average":{"1min":2.681,"5min":1.657,"15min":1.4870000000000001},"min":{"1min":0.50900000000000001,"5min":0.32800000000000001,"15min":0.31900000000000001},"max":{"1min":9.8079999999999998,"5min":9.8079999999999998,"15min":10.962999999999999},"last":0.44500000000000001}]},{"osd":5,"last update":"Sun Oct 1 11:12:50 2023","interfaces":[{"interface":"back","average":{"1min":1.6699999999999999,"5min":1.5640000000000001,"15min":1.484},"min":{"1min":0.42899999999999999,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":6.226,"5min":7.3710000000000004,"15min":12.771000000000001},"last":0.53200000000000003},{"interface":"front","average":{"1min":3.4809999999999999,"5min":2.2189999999999999,"15min":1.7190000000000001},"min":{"1min":0.48599999999999999,"5min":0.42899999999999999,"15min":0.35799999999999998},"max":{"1min":10.571999999999999,"5min":14.875999999999999,"15min":14.875999999999999},"last":3.5819999999999999}]},{"osd":6,"last update":"Sun Oct 1 11:12:50 2023","interfaces":[{"interface":"back","average":{"1min":1.659,"5min":1.6240000000000001,"15min":1.5720000000000001},"min":{"1min":0.46600000000000003,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":6.1929999999999996,"5min":6.9340000000000002,"15min":13.616},"last":0.48099999999999998},{"interface":"front","average":{"1min":2.7669999999999999,"5min":1.9990000000000001,"15min":1.9379999999999999},"min":{"1min":0.52000000000000002,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":9.9309999999999992,"5min":14.906000000000001,"15min":73.542000000000002},"last":3.6499999999999999}]},{"osd":7,"last update":"Sun Oct 1 11:12:50 2023","interfaces":[{"interface":"back","average":{"1min":3.415,"5min":2.0129999999999999,"15min":1.601},"min":{"1min":0.55000000000000004,"5min":0.39400000000000002,"15min":0.39100000000000001},"max":{"1min":13.617000000000001,"5min":14.929,"15min":14.929},"last":3.6200000000000001},{"interface":"front","average":{"1min":1.538,"5min":1.3680000000000001,"15min":1.3979999999999999},"min":{"1min":0.45800000000000002,"5min":0.38,"15min":0.38},"max":{"1min":5.8920000000000003,"5min":7.3819999999999997,"15min":8.7200000000000006},"last":0.46200000000000002}]}]},{"osd":4,"up_from":126,"seq":541165879725,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":826948,"kb_used_data":89276,"kb_used_omap":10,"kb_used_meta":737653,"kb_avail":92918204,"statfs":{"total":95995035648,"available":95148240896,"internally_reserved":0,"allocated":91418624,"data_stored":90002028,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":755357284},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,2,3,1,5],"upper_bound":128},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 11:13:17 2023","interfaces":[{"interface":"back","average":{"1min":1.2190000000000001,"5min":1.143,"15min":1.573},"min":{"1min":0.29299999999999998,"5min":0.28699999999999998,"15min":0.26400000000000001},"max":{"1min":11.025,"5min":11.025,"15min":68.611000000000004},"last":4.867},{"interface":"front","average":{"1min":1.353,"5min":1.194,"15min":1.5600000000000001},"min":{"1min":0.28199999999999997,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":4.5060000000000002,"5min":6.8940000000000001,"15min":68.480999999999995},"last":5.9859999999999998}]},{"osd":1,"last update":"Sun Oct 1 11:13:17 2023","interfaces":[{"interface":"back","average":{"1min":2.577,"5min":1.5429999999999999,"15min":1.282},"min":{"1min":0.41199999999999998,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":7.3360000000000003,"5min":7.3360000000000003,"15min":9.1920000000000002},"last":3.4380000000000002},{"interface":"front","average":{"1min":2.1619999999999999,"5min":1.3129999999999999,"15min":1.2609999999999999},"min":{"1min":0.33600000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":11.145,"5min":11.145,"15min":11.145},"last":0.35099999999999998}]},{"osd":2,"last update":"Sun Oct 1 11:13:17 2023","interfaces":[{"interface":"back","average":{"1min":13.215,"5min":4.0759999999999996,"15min":3.2280000000000002},"min":{"1min":0.374,"5min":0.34699999999999998,"15min":0.34100000000000003},"max":{"1min":219.05500000000001,"5min":219.05500000000001,"15min":219.05500000000001},"last":1.9410000000000001},{"interface":"front","average":{"1min":11.398999999999999,"5min":3.5329999999999999,"15min":2.8420000000000001},"min":{"1min":0.34799999999999998,"5min":0.33900000000000002,"15min":0.33700000000000002},"max":{"1min":218.81200000000001,"5min":218.81200000000001,"15min":218.81200000000001},"last":5.8609999999999998}]},{"osd":3,"last update":"Sun Oct 1 11:13:17 2023","interfaces":[{"interface":"back","average":{"1min":1.4430000000000001,"5min":1.8320000000000001,"15min":1.7430000000000001},"min":{"1min":0.34699999999999998,"5min":0.34499999999999997,"15min":0.34200000000000003},"max":{"1min":6.1440000000000001,"5min":18.475999999999999,"15min":61.664999999999999},"last":5.9249999999999998},{"interface":"front","average":{"1min":2.6269999999999998,"5min":2.2120000000000002,"15min":1.8200000000000001},"min":{"1min":0.38600000000000001,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":8.1980000000000004,"5min":18.303000000000001,"15min":61.707999999999998},"last":3.4790000000000001}]},{"osd":5,"last update":"Sun Oct 1 11:13:17 2023","interfaces":[{"interface":"back","average":{"1min":0.77900000000000003,"5min":1.256,"15min":1.3340000000000001},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":2.4809999999999999,"5min":12.457000000000001,"15min":25.175999999999998},"last":5.899},{"interface":"front","average":{"1min":1.6870000000000001,"5min":1.4339999999999999,"15min":1.5129999999999999},"min":{"1min":0.378,"5min":0.307,"15min":0.307},"max":{"1min":11.084,"5min":11.084,"15min":13.629},"last":0.61299999999999999}]},{"osd":6,"last update":"Sun Oct 1 11:13:17 2023","interfaces":[{"interface":"back","average":{"1min":0.81799999999999995,"5min":1.151,"15min":1.2290000000000001},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":4.2800000000000002,"5min":8.1440000000000001,"15min":12.635999999999999},"last":5.9489999999999998},{"interface":"front","average":{"1min":1.9510000000000001,"5min":1.3420000000000001,"15min":1.3049999999999999},"min":{"1min":0.38900000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":10.506,"5min":10.506,"15min":10.506},"last":3.5070000000000001}]},{"osd":7,"last update":"Sun Oct 1 11:13:17 2023","interfaces":[{"interface":"back","average":{"1min":1.8049999999999999,"5min":1.373,"15min":1.4099999999999999},"min":{"1min":0.45100000000000001,"5min":0.376,"15min":0.376},"max":{"1min":7.4450000000000003,"5min":9.2599999999999998,"15min":10.851000000000001},"last":3.536},{"interface":"front","average":{"1min":1.181,"5min":1.331,"15min":1.4019999999999999},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":11.117000000000001,"5min":12.364000000000001,"15min":12.364000000000001},"last":0.45400000000000001}]}]},{"osd":5,"up_from":36,"seq":154618823407,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1507480,"kb_used_data":13008,"kb_used_omap":0,"kb_used_meta":1494464,"kb_avail":92237672,"statfs":{"total":95995035648,"available":94451376128,"internally_reserved":0,"allocated":13320192,"data_stored":12751143,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1530331136},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 11:13:50 2023","interfaces":[{"interface":"back","average":{"1min":1.3879999999999999,"5min":2.8679999999999999,"15min":2.073},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.253},"max":{"1min":6.8639999999999999,"5min":111.797,"15min":115.057},"last":0.28299999999999997},{"interface":"front","average":{"1min":1.5009999999999999,"5min":2.71,"15min":2.1800000000000002},"min":{"1min":0.29999999999999999,"5min":0.251,"15min":0.251},"max":{"1min":8.2279999999999998,"5min":111.492,"15min":115.53},"last":2.4249999999999998}]},{"osd":1,"last update":"Sun Oct 1 11:13:50 2023","interfaces":[{"interface":"back","average":{"1min":8.9700000000000006,"5min":4.423,"15min":2.133},"min":{"1min":0.30299999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":132.99600000000001,"5min":132.99600000000001,"15min":132.99600000000001},"last":2.4510000000000001},{"interface":"front","average":{"1min":8.8100000000000005,"5min":4.3689999999999998,"15min":2.367},"min":{"1min":0.35399999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":133.42400000000001,"5min":133.42400000000001,"15min":133.42400000000001},"last":2.6110000000000002}]},{"osd":2,"last update":"Sun Oct 1 11:13:21 2023","interfaces":[{"interface":"back","average":{"1min":1.419,"5min":3.621,"15min":3.0419999999999998},"min":{"1min":0.36599999999999999,"5min":0.309,"15min":0.309},"max":{"1min":9.4879999999999995,"5min":129.70500000000001,"15min":207.059},"last":2.4180000000000001},{"interface":"front","average":{"1min":1.355,"5min":3.8410000000000002,"15min":3.3530000000000002},"min":{"1min":0.36699999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":2.9830000000000001,"5min":129.76300000000001,"15min":206.673},"last":2.6309999999999998}]},{"osd":3,"last update":"Sun Oct 1 11:13:50 2023","interfaces":[{"interface":"back","average":{"1min":2.0630000000000002,"5min":2.9769999999999999,"15min":2.0859999999999999},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.33800000000000002},"max":{"1min":9.4339999999999993,"5min":131.09399999999999,"15min":131.09399999999999},"last":2.456},{"interface":"front","average":{"1min":1.516,"5min":2.6309999999999998,"15min":1.7949999999999999},"min":{"1min":0.40000000000000002,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":8.2349999999999994,"5min":130.70699999999999,"15min":130.70699999999999},"last":3.2160000000000002}]},{"osd":4,"last update":"Sun Oct 1 11:13:21 2023","interfaces":[{"interface":"back","average":{"1min":1.048,"5min":1.1719999999999999,"15min":1.1850000000000001},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.311},"max":{"1min":5.2400000000000002,"5min":10.167,"15min":10.167},"last":0.44900000000000001},{"interface":"front","average":{"1min":1.6679999999999999,"5min":1.532,"15min":1.282},"min":{"1min":0.373,"5min":0.35999999999999999,"15min":0.30399999999999999},"max":{"1min":9.3390000000000004,"5min":11.162000000000001,"15min":11.162000000000001},"last":0.52700000000000002}]},{"osd":6,"last update":"Sun Oct 1 11:13:50 2023","interfaces":[{"interface":"back","average":{"1min":0.90100000000000002,"5min":1.151,"15min":1.1950000000000001},"min":{"1min":0.371,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":2.0529999999999999,"5min":10.411,"15min":11.196999999999999},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.93700000000000006,"5min":0.91600000000000004,"15min":1.1779999999999999},"min":{"1min":0.46000000000000002,"5min":0.374,"15min":0.34499999999999997},"max":{"1min":2.4060000000000001,"5min":4.7089999999999996,"15min":12.189},"last":0.68100000000000005}]},{"osd":7,"last update":"Sun Oct 1 11:13:50 2023","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":0.79700000000000004,"15min":1.1699999999999999},"min":{"1min":0.41699999999999998,"5min":0.36699999999999999,"15min":0.34899999999999998},"max":{"1min":1.1559999999999999,"5min":3.9140000000000001,"15min":9.1959999999999997},"last":0.77600000000000002},{"interface":"front","average":{"1min":0.874,"5min":1.1950000000000001,"15min":1.2509999999999999},"min":{"1min":0.504,"5min":0.40899999999999997,"15min":0.38600000000000001},"max":{"1min":2.0299999999999998,"5min":10.457000000000001,"15min":11.215},"last":0.5}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16187392,"data_stored":16187392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":82509824,"data_stored":82509824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":52625408,"data_stored":52625408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":77467648,"data_stored":77463576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":856064,"data_stored":851992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":5640192,"data_stored":5636120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4456448,"data_stored":4456448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T11:14:00.324 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-01T11:14:00.324 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T11:14:00.324 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-01T11:14:00.325 INFO:tasks.thrashosds.thrasher:Setting osd 7 primary_affinity to 0.440963 2023-10-01T11:14:00.325 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 7 0.44096299215870594 2023-10-01T11:14:00.394 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:00.425 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:00.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:00.428 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:14:00.529 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:00.565 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:00.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:00.568 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:14:00.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:14:00 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:14:00] "GET /metrics HTTP/1.1" 200 36379 "" "Prometheus/2.33.4" 2023-10-01T11:14:00.669 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:00.704 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:00.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:00.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:00.807 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:00.835 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:00.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:00.838 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:00.940 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:00.969 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:00.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:00.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:01.072 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:01.105 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:01.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:01.108 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:14:01.209 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:01.238 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:01.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:01.240 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:01.341 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:01.376 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:01.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:01.380 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:14:01.481 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:01.510 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:01.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:01.513 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:14:01.614 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:01.637 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:01.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:01.640 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:01.741 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:01.760 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:01.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:01.764 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:14:01.796 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2023-10-01T11:14:01.796 INFO:teuthology.orchestra.run.smithi078.stdout: "size": 20, 2023-10-01T11:14:01.796 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 600, 2023-10-01T11:14:01.797 INFO:teuthology.orchestra.run.smithi078.stdout: "ops": [ 2023-10-01T11:14:01.797 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.797 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.46454.0:143127 13.9 e346/284 13:9ae4f8b3:::benchmark_data_smithi078_202524_object63569:head v 346'9040, mlcod=346'9038)", 2023-10-01T11:14:01.797 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:08:12.887516+0000", 2023-10-01T11:14:01.797 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 348.90408069, 2023-10-01T11:14:01.797 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.41120600800000001, 2023-10-01T11:14:01.797 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:14:01.798 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "started", 2023-10-01T11:14:01.798 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:14:01.798 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.798 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:14:01.798 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:08:12.887516+0000", 2023-10-01T11:14:01.798 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:14:01.798 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.798 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.798 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:14:01.798 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:08:12.887516+0000", 2023-10-01T11:14:01.798 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:14:01.799 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.799 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.799 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:14:01.799 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:08:12.887515+0000", 2023-10-01T11:14:01.799 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T11:14:01.799 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.799 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.799 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:14:01.799 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:08:12.887524+0000", 2023-10-01T11:14:01.799 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.5519999999999993e-06 2023-10-01T11:14:01.800 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.800 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.800 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:14:01.800 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:08:12.887536+0000", 2023-10-01T11:14:01.800 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.2133000000000001e-05 2023-10-01T11:14:01.800 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.800 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.800 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:14:01.800 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:08:12.887548+0000", 2023-10-01T11:14:01.800 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1223000000000001e-05 2023-10-01T11:14:01.801 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.801 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.801 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:14:01.801 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:08:12.887623+0000", 2023-10-01T11:14:01.801 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.538e-05 2023-10-01T11:14:01.801 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.801 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.801 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:14:01.801 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:08:12.887649+0000", 2023-10-01T11:14:01.802 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.6322999999999999e-05 2023-10-01T11:14:01.802 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.802 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.802 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:14:01.802 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:08:13.298722+0000", 2023-10-01T11:14:01.802 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.41107253599999999 2023-10-01T11:14:01.803 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:14:01.803 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:14:01.803 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:14:01.803 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.803 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.803 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.46454.0:143135 13.9 e346/284)", 2023-10-01T11:14:01.803 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:08:12.893919+0000", 2023-10-01T11:14:01.803 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 348.89767792100002, 2023-10-01T11:14:01.803 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.40589523799999999, 2023-10-01T11:14:01.804 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:14:01.804 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "reached pg", 2023-10-01T11:14:01.804 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:14:01.804 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.804 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:14:01.804 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:08:12.893919+0000", 2023-10-01T11:14:01.804 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:14:01.805 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.806 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.806 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:14:01.806 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:08:12.893919+0000", 2023-10-01T11:14:01.807 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:14:01.807 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.807 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.807 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:14:01.807 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:08:12.893918+0000", 2023-10-01T11:14:01.807 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T11:14:01.807 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.807 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.807 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:14:01.807 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:08:12.893927+0000", 2023-10-01T11:14:01.807 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.7880000000000001e-06 2023-10-01T11:14:01.808 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.808 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.808 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:14:01.808 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:08:12.901244+0000", 2023-10-01T11:14:01.808 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0073169029999999996 2023-10-01T11:14:01.808 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.808 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.808 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:14:01.808 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:08:12.901273+0000", 2023-10-01T11:14:01.808 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.9135000000000001e-05 2023-10-01T11:14:01.808 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.808 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.809 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:14:01.809 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:08:13.299790+0000", 2023-10-01T11:14:01.809 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.39851699600000001 2023-10-01T11:14:01.809 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.809 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.809 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:14:01.809 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:08:13.299814+0000", 2023-10-01T11:14:01.809 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.3394e-05 2023-10-01T11:14:01.809 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:14:01.809 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:14:01.809 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:14:01.809 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.810 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.810 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.46454.0:143132 13.5 e346/313)", 2023-10-01T11:14:01.810 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:08:12.894921+0000", 2023-10-01T11:14:01.810 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 348.896675034, 2023-10-01T11:14:01.810 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.33921986599999998, 2023-10-01T11:14:01.810 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:14:01.810 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "reached pg", 2023-10-01T11:14:01.810 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:14:01.810 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.810 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:14:01.810 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:08:12.894921+0000", 2023-10-01T11:14:01.811 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:14:01.811 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.811 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.811 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:14:01.811 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:08:12.894921+0000", 2023-10-01T11:14:01.811 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:14:01.811 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.811 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.811 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:14:01.811 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:08:12.894920+0000", 2023-10-01T11:14:01.811 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T11:14:01.812 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.812 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.812 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:14:01.812 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:08:12.894931+0000", 2023-10-01T11:14:01.812 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.0557000000000001e-05 2023-10-01T11:14:01.812 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.812 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.812 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:14:01.812 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:08:12.901282+0000", 2023-10-01T11:14:01.812 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.006351481 2023-10-01T11:14:01.812 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.812 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.813 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:14:01.813 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:08:12.901293+0000", 2023-10-01T11:14:01.813 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1099e-05 2023-10-01T11:14:01.813 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.813 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.813 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:14:01.813 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:08:13.234121+0000", 2023-10-01T11:14:01.813 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.33282715400000001 2023-10-01T11:14:01.813 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.813 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.813 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:14:01.814 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:08:13.234141+0000", 2023-10-01T11:14:01.814 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.0716e-05 2023-10-01T11:14:01.814 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:14:01.814 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:14:01.814 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:14:01.814 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.814 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.815 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.36819.0:38326 14.7 e359/356 14:e5366b28:::benchmark_data_smithi078_236918_object38325:head v 359'2457, mlcod=359'2452)", 2023-10-01T11:14:01.815 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:10:01.881309+0000", 2023-10-01T11:14:01.815 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 239.91028742699999, 2023-10-01T11:14:01.815 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.17794510799999999, 2023-10-01T11:14:01.815 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:14:01.815 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:14:01.815 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:14:01.815 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.815 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:14:01.815 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.881309+0000", 2023-10-01T11:14:01.815 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:14:01.816 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.816 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.816 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:14:01.816 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.881309+0000", 2023-10-01T11:14:01.816 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:14:01.816 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.816 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.816 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:14:01.816 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.881290+0000", 2023-10-01T11:14:01.816 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999809 2023-10-01T11:14:01.816 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.816 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.817 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:14:01.817 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.881386+0000", 2023-10-01T11:14:01.817 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.6576999999999994e-05 2023-10-01T11:14:01.817 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.817 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.817 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:14:01.817 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.881400+0000", 2023-10-01T11:14:01.817 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.3675999999999999e-05 2023-10-01T11:14:01.817 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.817 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.817 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:14:01.818 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.881408+0000", 2023-10-01T11:14:01.818 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.0239999999999997e-06 2023-10-01T11:14:01.818 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.818 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.818 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:14:01.818 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.881688+0000", 2023-10-01T11:14:01.818 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00027932500000000003 2023-10-01T11:14:01.818 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.818 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.818 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:14:01.819 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.881706+0000", 2023-10-01T11:14:01.819 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.8848999999999999e-05 2023-10-01T11:14:01.819 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.819 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.819 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:14:01.819 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.891084+0000", 2023-10-01T11:14:01.819 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0093772600000000001 2023-10-01T11:14:01.819 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.819 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.819 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:14:01.819 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:02.059254+0000", 2023-10-01T11:14:01.819 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.16817048500000001 2023-10-01T11:14:01.820 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:14:01.820 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:14:01.820 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:14:01.820 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.820 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.820 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.36819.0:38325 14.8 e359/356 14:15b7e639:::benchmark_data_smithi078_236918_object38324:head v 359'2376, mlcod=359'2374)", 2023-10-01T11:14:01.820 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:10:01.881674+0000", 2023-10-01T11:14:01.820 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 239.90992235499999, 2023-10-01T11:14:01.820 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.17761981900000001, 2023-10-01T11:14:01.820 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:14:01.820 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:14:01.821 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:14:01.821 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.821 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:14:01.821 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.881674+0000", 2023-10-01T11:14:01.821 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:14:01.821 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.821 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.821 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:14:01.821 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.881674+0000", 2023-10-01T11:14:01.821 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:14:01.821 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.821 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.822 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:14:01.822 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.881673+0000", 2023-10-01T11:14:01.822 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T11:14:01.822 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.822 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.822 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:14:01.822 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.881697+0000", 2023-10-01T11:14:01.822 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.3703e-05 2023-10-01T11:14:01.822 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.822 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.823 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:14:01.823 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.881708+0000", 2023-10-01T11:14:01.823 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1481000000000001e-05 2023-10-01T11:14:01.823 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.823 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.823 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:14:01.823 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.881715+0000", 2023-10-01T11:14:01.823 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.6359999999999997e-06 2023-10-01T11:14:01.823 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.823 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.823 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:14:01.824 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.881847+0000", 2023-10-01T11:14:01.824 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00013210199999999999 2023-10-01T11:14:01.824 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.825 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.825 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:14:01.825 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.881866+0000", 2023-10-01T11:14:01.825 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.8629000000000001e-05 2023-10-01T11:14:01.825 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.825 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.825 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:14:01.825 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.891160+0000", 2023-10-01T11:14:01.825 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0092945890000000007 2023-10-01T11:14:01.825 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.826 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.826 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:14:01.826 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:02.059294+0000", 2023-10-01T11:14:01.826 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.16813345299999999 2023-10-01T11:14:01.826 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:14:01.826 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:14:01.826 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:14:01.826 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.826 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.826 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.36819.0:38329 14.0 e359/356 14:088a0921:::benchmark_data_smithi078_236918_object38328:head v 359'2385, mlcod=359'2381)", 2023-10-01T11:14:01.827 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:10:01.883322+0000", 2023-10-01T11:14:01.827 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 239.90827457399999, 2023-10-01T11:14:01.827 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.17606540300000001, 2023-10-01T11:14:01.827 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:14:01.827 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:14:01.827 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:14:01.827 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.828 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:14:01.828 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.883322+0000", 2023-10-01T11:14:01.828 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:14:01.828 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.828 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.828 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:14:01.828 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.883322+0000", 2023-10-01T11:14:01.828 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:14:01.828 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.828 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.828 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:14:01.829 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.883321+0000", 2023-10-01T11:14:01.829 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T11:14:01.829 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.829 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.829 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:14:01.829 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.883345+0000", 2023-10-01T11:14:01.829 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.4311000000000002e-05 2023-10-01T11:14:01.829 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.829 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.830 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:14:01.830 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.883364+0000", 2023-10-01T11:14:01.830 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.8397999999999999e-05 2023-10-01T11:14:01.830 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.830 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.830 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:14:01.830 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.883371+0000", 2023-10-01T11:14:01.830 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.6939999999999995e-06 2023-10-01T11:14:01.830 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.830 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.831 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:14:01.831 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.883525+0000", 2023-10-01T11:14:01.831 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00015325700000000001 2023-10-01T11:14:01.831 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.831 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.831 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:14:01.831 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.883536+0000", 2023-10-01T11:14:01.831 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1799e-05 2023-10-01T11:14:01.831 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.832 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.832 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:14:01.832 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:02.059303+0000", 2023-10-01T11:14:01.832 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.17576650099999999 2023-10-01T11:14:01.832 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.832 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.832 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:14:01.832 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:02.059387+0000", 2023-10-01T11:14:01.832 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.4312000000000002e-05 2023-10-01T11:14:01.832 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:14:01.833 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:14:01.833 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:14:01.833 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.833 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.833 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.36819.0:38330 14.7 e359/356 14:e9142cb1:::benchmark_data_smithi078_236918_object38329:head v 359'2458, mlcod=359'2452)", 2023-10-01T11:14:01.833 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:10:01.883731+0000", 2023-10-01T11:14:01.833 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 239.90786542399999, 2023-10-01T11:14:01.833 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.175600591, 2023-10-01T11:14:01.833 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:14:01.834 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:14:01.834 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:14:01.834 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.834 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:14:01.834 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.883731+0000", 2023-10-01T11:14:01.834 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:14:01.834 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.834 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.835 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:14:01.835 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.883731+0000", 2023-10-01T11:14:01.835 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:14:01.835 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.835 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.836 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:14:01.836 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.883730+0000", 2023-10-01T11:14:01.836 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999995 2023-10-01T11:14:01.836 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.836 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.836 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:14:01.836 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.883759+0000", 2023-10-01T11:14:01.836 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.8643999999999999e-05 2023-10-01T11:14:01.837 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.837 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.837 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:14:01.837 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.883777+0000", 2023-10-01T11:14:01.837 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.8379000000000001e-05 2023-10-01T11:14:01.837 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.837 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.838 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:14:01.838 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.883783+0000", 2023-10-01T11:14:01.838 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.9529999999999996e-06 2023-10-01T11:14:01.838 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.838 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.838 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:14:01.838 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.883835+0000", 2023-10-01T11:14:01.838 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.1243999999999999e-05 2023-10-01T11:14:01.838 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.839 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.839 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:14:01.839 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.883851+0000", 2023-10-01T11:14:01.839 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.6741000000000001e-05 2023-10-01T11:14:01.839 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.839 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.839 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:14:01.840 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:02.059264+0000", 2023-10-01T11:14:01.840 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.17541248600000001 2023-10-01T11:14:01.840 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.840 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.840 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:14:01.840 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:02.059332+0000", 2023-10-01T11:14:01.840 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.7719000000000007e-05 2023-10-01T11:14:01.840 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:14:01.840 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:14:01.840 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:14:01.840 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.841 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.841 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.36819.0:38324 14.3 e359/356 14:c3d3d8e5:::benchmark_data_smithi078_236918_object38323:head v 359'2440, mlcod=359'2438)", 2023-10-01T11:14:01.841 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:10:01.884195+0000", 2023-10-01T11:14:01.841 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 239.90740122700001, 2023-10-01T11:14:01.841 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.175149417, 2023-10-01T11:14:01.841 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:14:01.841 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:14:01.841 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:14:01.841 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.841 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:14:01.841 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.884195+0000", 2023-10-01T11:14:01.841 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:14:01.842 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.842 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.842 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:14:01.842 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.884195+0000", 2023-10-01T11:14:01.842 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:14:01.842 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.842 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.842 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:14:01.842 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.884195+0000", 2023-10-01T11:14:01.842 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999995 2023-10-01T11:14:01.842 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.843 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.843 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:14:01.843 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.884217+0000", 2023-10-01T11:14:01.843 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.2115000000000001e-05 2023-10-01T11:14:01.843 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.843 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.843 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:14:01.843 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.884227+0000", 2023-10-01T11:14:01.843 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.0352e-05 2023-10-01T11:14:01.843 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.843 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.843 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:14:01.844 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.884233+0000", 2023-10-01T11:14:01.844 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.3280000000000002e-06 2023-10-01T11:14:01.844 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.844 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.844 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:14:01.844 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.884885+0000", 2023-10-01T11:14:01.844 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00065174000000000004 2023-10-01T11:14:01.844 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.844 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.844 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:14:01.844 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.884903+0000", 2023-10-01T11:14:01.844 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.7501000000000001e-05 2023-10-01T11:14:01.845 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.845 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.845 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:14:01.845 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.891057+0000", 2023-10-01T11:14:01.845 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0061543350000000004 2023-10-01T11:14:01.845 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.845 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.845 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:14:01.845 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:02.059345+0000", 2023-10-01T11:14:01.845 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.16828771100000001 2023-10-01T11:14:01.845 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:14:01.846 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:14:01.846 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:14:01.846 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.846 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.846 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.36819.0:38339 14.3 e359/356 14:c883e07d:::benchmark_data_smithi078_236918_object38338:head v 359'2441, mlcod=359'2438)", 2023-10-01T11:14:01.846 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:10:01.887940+0000", 2023-10-01T11:14:01.846 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 239.90365648400001, 2023-10-01T11:14:01.846 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.17149403599999999, 2023-10-01T11:14:01.846 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:14:01.846 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:14:01.847 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:14:01.847 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.847 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:14:01.847 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.887940+0000", 2023-10-01T11:14:01.847 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:14:01.847 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.847 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.847 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:14:01.847 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.887940+0000", 2023-10-01T11:14:01.847 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:14:01.847 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.848 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.848 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:14:01.848 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.887939+0000", 2023-10-01T11:14:01.848 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T11:14:01.848 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.848 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.848 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:14:01.848 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.887962+0000", 2023-10-01T11:14:01.848 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.2611000000000001e-05 2023-10-01T11:14:01.848 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.849 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.849 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:14:01.849 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.887972+0000", 2023-10-01T11:14:01.849 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.0179e-05 2023-10-01T11:14:01.849 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.849 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.849 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:14:01.849 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.887978+0000", 2023-10-01T11:14:01.849 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.5889999999999999e-06 2023-10-01T11:14:01.849 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.849 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.849 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:14:01.850 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.888037+0000", 2023-10-01T11:14:01.850 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.8428999999999999e-05 2023-10-01T11:14:01.850 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.850 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.850 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:14:01.850 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.888061+0000", 2023-10-01T11:14:01.850 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.4510000000000001e-05 2023-10-01T11:14:01.850 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.850 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.850 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:14:01.850 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:02.059359+0000", 2023-10-01T11:14:01.851 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.17129786399999999 2023-10-01T11:14:01.851 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.851 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.851 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:14:01.851 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:02.059434+0000", 2023-10-01T11:14:01.851 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.4682000000000001e-05 2023-10-01T11:14:01.851 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:14:01.851 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:14:01.851 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:14:01.851 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.851 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.851 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.36819.0:38335 14.b e359/356 14:de756c7a:::benchmark_data_smithi078_236918_object38334:head v 359'2351, mlcod=359'2348)", 2023-10-01T11:14:01.851 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:10:01.889244+0000", 2023-10-01T11:14:01.852 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 239.90235257000001, 2023-10-01T11:14:01.852 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.17188783799999999, 2023-10-01T11:14:01.852 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:14:01.852 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:14:01.852 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:14:01.852 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.852 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:14:01.852 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.889244+0000", 2023-10-01T11:14:01.852 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:14:01.852 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.852 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.853 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:14:01.853 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.889244+0000", 2023-10-01T11:14:01.853 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:14:01.853 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.853 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.853 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:14:01.853 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.889243+0000", 2023-10-01T11:14:01.853 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T11:14:01.853 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.853 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.853 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:14:01.853 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.889287+0000", 2023-10-01T11:14:01.853 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.4057000000000002e-05 2023-10-01T11:14:01.854 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.854 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.854 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:14:01.854 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.889299+0000", 2023-10-01T11:14:01.854 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.1640999999999999e-05 2023-10-01T11:14:01.854 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.854 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.854 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:14:01.854 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.889306+0000", 2023-10-01T11:14:01.854 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.029e-06 2023-10-01T11:14:01.855 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.855 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.855 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:14:01.855 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.889565+0000", 2023-10-01T11:14:01.855 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00025950499999999999 2023-10-01T11:14:01.855 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.855 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.855 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:14:01.855 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.889578+0000", 2023-10-01T11:14:01.855 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.3234000000000001e-05 2023-10-01T11:14:01.855 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.855 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.856 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:14:01.856 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:02.061085+0000", 2023-10-01T11:14:01.856 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.17150697200000001 2023-10-01T11:14:01.856 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.856 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.856 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:14:01.856 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:02.061132+0000", 2023-10-01T11:14:01.856 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.6310999999999998e-05 2023-10-01T11:14:01.856 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:14:01.856 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:14:01.856 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:14:01.857 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.857 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.857 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.36819.0:38337 14.6 e359/356 14:6c964297:::benchmark_data_smithi078_236918_object38336:head v 359'2446, mlcod=359'2444)", 2023-10-01T11:14:01.857 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:10:01.889317+0000", 2023-10-01T11:14:01.857 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 239.90227952399999, 2023-10-01T11:14:01.857 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.170194077, 2023-10-01T11:14:01.857 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:14:01.857 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:14:01.857 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:14:01.857 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.857 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:14:01.857 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.889317+0000", 2023-10-01T11:14:01.857 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:14:01.858 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.858 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.858 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:14:01.858 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.889317+0000", 2023-10-01T11:14:01.858 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:14:01.858 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.858 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.858 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:14:01.858 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.889317+0000", 2023-10-01T11:14:01.858 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999995 2023-10-01T11:14:01.858 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.859 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.859 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:14:01.859 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.889336+0000", 2023-10-01T11:14:01.859 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.9364e-05 2023-10-01T11:14:01.859 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.859 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.859 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:14:01.859 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.889342+0000", 2023-10-01T11:14:01.859 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.0610000000000001e-06 2023-10-01T11:14:01.859 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.859 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.859 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:14:01.860 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.889347+0000", 2023-10-01T11:14:01.860 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.566e-06 2023-10-01T11:14:01.860 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.860 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.860 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:14:01.860 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.889407+0000", 2023-10-01T11:14:01.860 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.0313999999999999e-05 2023-10-01T11:14:01.861 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.861 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.861 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:14:01.861 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.889426+0000", 2023-10-01T11:14:01.861 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.9289000000000001e-05 2023-10-01T11:14:01.861 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.861 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.861 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:14:01.861 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:02.059407+0000", 2023-10-01T11:14:01.861 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.16998091800000001 2023-10-01T11:14:01.861 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.862 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.862 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:14:01.862 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:02.059511+0000", 2023-10-01T11:14:01.862 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000103887 2023-10-01T11:14:01.862 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:14:01.862 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:14:01.862 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:14:01.862 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.862 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.862 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.36819.0:38343 14.8 e359/356 14:1e8daa62:::benchmark_data_smithi078_236918_object38342:head v 359'2377, mlcod=359'2374)", 2023-10-01T11:14:01.862 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:10:01.893444+0000", 2023-10-01T11:14:01.863 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 239.89815210899999, 2023-10-01T11:14:01.863 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.17182713199999999, 2023-10-01T11:14:01.863 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:14:01.863 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:14:01.863 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:14:01.863 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.863 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:14:01.863 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.893444+0000", 2023-10-01T11:14:01.863 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:14:01.863 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.864 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.864 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:14:01.864 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.893444+0000", 2023-10-01T11:14:01.864 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:14:01.864 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.864 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.864 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:14:01.864 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.893444+0000", 2023-10-01T11:14:01.864 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999995 2023-10-01T11:14:01.864 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.864 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.864 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:14:01.865 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.893467+0000", 2023-10-01T11:14:01.865 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.3113e-05 2023-10-01T11:14:01.865 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.865 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.865 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:14:01.865 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:02.059186+0000", 2023-10-01T11:14:01.865 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.16571961900000001 2023-10-01T11:14:01.866 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.866 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.866 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:14:01.866 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:02.059213+0000", 2023-10-01T11:14:01.866 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.6367e-05 2023-10-01T11:14:01.866 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.866 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.866 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:14:01.866 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:02.059449+0000", 2023-10-01T11:14:01.866 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00023646600000000001 2023-10-01T11:14:01.866 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.867 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.867 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:14:01.867 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:02.060098+0000", 2023-10-01T11:14:01.867 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00064928000000000002 2023-10-01T11:14:01.867 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.867 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.867 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:14:01.867 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:02.065209+0000", 2023-10-01T11:14:01.867 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0051109459999999999 2023-10-01T11:14:01.868 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.868 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.868 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:14:01.868 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:02.065271+0000", 2023-10-01T11:14:01.868 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.2032000000000005e-05 2023-10-01T11:14:01.868 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:14:01.868 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:14:01.868 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:14:01.868 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.869 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.869 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.36819.0:38341 14.3 e359/356 14:c6dc408a:::benchmark_data_smithi078_236918_object38340:head v 359'2442, mlcod=359'2438)", 2023-10-01T11:14:01.869 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:10:01.897649+0000", 2023-10-01T11:14:01.869 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 239.89394759199999, 2023-10-01T11:14:01.869 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.165081274, 2023-10-01T11:14:01.869 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:14:01.869 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:14:01.870 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:14:01.870 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.870 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:14:01.870 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.897649+0000", 2023-10-01T11:14:01.870 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:14:01.870 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.870 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.870 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:14:01.870 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.897649+0000", 2023-10-01T11:14:01.871 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:14:01.871 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.871 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.871 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:14:01.871 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.897647+0000", 2023-10-01T11:14:01.871 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999986 2023-10-01T11:14:01.871 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.872 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.872 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:14:01.872 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.897677+0000", 2023-10-01T11:14:01.872 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.0161000000000001e-05 2023-10-01T11:14:01.872 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.872 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.873 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:14:01.873 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:02.059227+0000", 2023-10-01T11:14:01.873 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.16155008700000001 2023-10-01T11:14:01.873 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.873 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.873 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:14:01.873 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:02.059251+0000", 2023-10-01T11:14:01.873 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.3048999999999999e-05 2023-10-01T11:14:01.874 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.874 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.874 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:14:01.874 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:02.059493+0000", 2023-10-01T11:14:01.874 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000242515 2023-10-01T11:14:01.875 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.875 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.875 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:14:01.875 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:02.059677+0000", 2023-10-01T11:14:01.875 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00018387199999999999 2023-10-01T11:14:01.875 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.876 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.876 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:14:01.876 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:02.062694+0000", 2023-10-01T11:14:01.876 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0030166490000000002 2023-10-01T11:14:01.876 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.876 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.876 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:14:01.876 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:02.062730+0000", 2023-10-01T11:14:01.876 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.6525e-05 2023-10-01T11:14:01.876 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:14:01.876 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:14:01.876 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:14:01.877 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.877 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.877 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.36819.0:38349 14.7 e359/356 14:efc6fbd2:::benchmark_data_smithi078_236918_object38348:head v 359'2459, mlcod=359'2455)", 2023-10-01T11:14:01.877 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:10:01.905339+0000", 2023-10-01T11:14:01.877 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 239.88625789100001, 2023-10-01T11:14:01.877 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.15878884600000001, 2023-10-01T11:14:01.877 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:14:01.877 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:14:01.877 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:14:01.877 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.878 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:14:01.878 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.905339+0000", 2023-10-01T11:14:01.878 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:14:01.878 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.878 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.878 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:14:01.878 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.905339+0000", 2023-10-01T11:14:01.878 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:14:01.878 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.878 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.878 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:14:01.878 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.905337+0000", 2023-10-01T11:14:01.881 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999986 2023-10-01T11:14:01.881 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.881 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.881 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:14:01.881 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:01.905385+0000", 2023-10-01T11:14:01.881 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.7540999999999998e-05 2023-10-01T11:14:01.881 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.881 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.882 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:14:01.882 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:02.059159+0000", 2023-10-01T11:14:01.882 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.15377407000000001 2023-10-01T11:14:01.882 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.882 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.882 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:14:01.882 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:02.059189+0000", 2023-10-01T11:14:01.882 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.0855000000000002e-05 2023-10-01T11:14:01.882 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.882 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.883 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:14:01.883 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:02.059513+0000", 2023-10-01T11:14:01.883 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000323448 2023-10-01T11:14:01.883 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.883 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.883 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:14:01.883 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:02.059537+0000", 2023-10-01T11:14:01.883 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.3989999999999999e-05 2023-10-01T11:14:01.883 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.883 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.884 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:14:01.884 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:02.064014+0000", 2023-10-01T11:14:01.884 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0044771300000000002 2023-10-01T11:14:01.884 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.884 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.884 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:14:01.884 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:10:02.064127+0000", 2023-10-01T11:14:01.884 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.000113365 2023-10-01T11:14:01.884 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:14:01.884 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:14:01.884 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:14:01.885 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.885 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.885 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.36819.0:113242 14.6 e364/356 14:6c8ac94f:::benchmark_data_smithi078_236918_object8096:head v 364'7180, mlcod=364'7177)", 2023-10-01T11:14:01.885 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:11:23.793604+0000", 2023-10-01T11:14:01.885 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 157.997992533, 2023-10-01T11:14:01.885 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.102899435, 2023-10-01T11:14:01.885 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:14:01.885 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:14:01.885 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:14:01.885 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.885 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:14:01.885 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:11:23.793604+0000", 2023-10-01T11:14:01.886 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:14:01.886 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.886 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.886 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:14:01.886 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:11:23.793604+0000", 2023-10-01T11:14:01.886 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:14:01.886 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.886 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.886 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:14:01.886 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:11:23.793603+0000", 2023-10-01T11:14:01.886 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T11:14:01.887 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.887 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.887 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:14:01.887 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:11:23.793610+0000", 2023-10-01T11:14:01.887 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.4559999999999997e-06 2023-10-01T11:14:01.887 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.887 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.887 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:14:01.887 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:11:23.793618+0000", 2023-10-01T11:14:01.887 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.7399999999999993e-06 2023-10-01T11:14:01.887 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.887 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.887 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:14:01.888 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:11:23.793624+0000", 2023-10-01T11:14:01.888 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.2900000000000002e-06 2023-10-01T11:14:01.888 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.888 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.888 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:14:01.888 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:11:23.793665+0000", 2023-10-01T11:14:01.888 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.1063000000000003e-05 2023-10-01T11:14:01.888 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.888 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.888 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:14:01.888 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:11:23.793684+0000", 2023-10-01T11:14:01.888 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.9706000000000001e-05 2023-10-01T11:14:01.889 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.889 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.889 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:14:01.889 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:11:23.800671+0000", 2023-10-01T11:14:01.889 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0069861929999999999 2023-10-01T11:14:01.889 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.889 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.889 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:14:01.889 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:11:23.896503+0000", 2023-10-01T11:14:01.889 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.095832804999999993 2023-10-01T11:14:01.889 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:14:01.890 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:14:01.890 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:14:01.890 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.890 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.890 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.36819.0:113243 14.b e364/356 14:d5923833:::benchmark_data_smithi078_236918_object8097:head v 364'7072, mlcod=364'7069)", 2023-10-01T11:14:01.890 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:11:23.793633+0000", 2023-10-01T11:14:01.890 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 157.99796388799999, 2023-10-01T11:14:01.890 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.103476159, 2023-10-01T11:14:01.890 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:14:01.890 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:14:01.890 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:14:01.890 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.891 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:14:01.891 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:11:23.793633+0000", 2023-10-01T11:14:01.891 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:14:01.891 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.891 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.891 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:14:01.891 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:11:23.793633+0000", 2023-10-01T11:14:01.891 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:14:01.891 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.891 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.891 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:14:01.891 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:11:23.793632+0000", 2023-10-01T11:14:01.891 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999995 2023-10-01T11:14:01.891 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.892 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.892 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:14:01.892 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:11:23.793636+0000", 2023-10-01T11:14:01.892 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.0690000000000003e-06 2023-10-01T11:14:01.892 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.892 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.892 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:14:01.892 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:11:23.793642+0000", 2023-10-01T11:14:01.892 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 6.0120000000000002e-06 2023-10-01T11:14:01.892 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.892 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.893 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:14:01.893 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:11:23.793646+0000", 2023-10-01T11:14:01.893 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.7309999999999999e-06 2023-10-01T11:14:01.893 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.893 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.893 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:14:01.893 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:11:23.793691+0000", 2023-10-01T11:14:01.893 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.4471999999999997e-05 2023-10-01T11:14:01.893 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.893 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.893 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:14:01.893 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:11:23.793713+0000", 2023-10-01T11:14:01.894 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.2585000000000001e-05 2023-10-01T11:14:01.894 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.894 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.894 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:14:01.894 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:11:23.800601+0000", 2023-10-01T11:14:01.894 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0068880800000000004 2023-10-01T11:14:01.894 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.894 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.894 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:14:01.894 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:11:23.897109+0000", 2023-10-01T11:14:01.894 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.096507482000000006 2023-10-01T11:14:01.895 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:14:01.895 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:14:01.895 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:14:01.895 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.895 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.895 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.36819.0:113247 14.5 e364/356 14:affc84c5:::benchmark_data_smithi078_236918_object8101:head v 364'7168, mlcod=364'7166)", 2023-10-01T11:14:01.895 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:11:23.798074+0000", 2023-10-01T11:14:01.895 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 157.99352240299999, 2023-10-01T11:14:01.895 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.12164799799999999, 2023-10-01T11:14:01.895 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:14:01.895 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:14:01.895 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:14:01.896 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.896 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:14:01.896 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:11:23.798074+0000", 2023-10-01T11:14:01.896 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:14:01.896 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.896 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.896 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:14:01.896 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:11:23.798074+0000", 2023-10-01T11:14:01.896 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:14:01.896 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.896 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.896 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:14:01.897 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:11:23.798073+0000", 2023-10-01T11:14:01.897 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T11:14:01.897 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.897 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.897 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:14:01.897 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:11:23.798083+0000", 2023-10-01T11:14:01.897 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.6069999999999997e-06 2023-10-01T11:14:01.897 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.897 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.897 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:14:01.897 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:11:23.798094+0000", 2023-10-01T11:14:01.897 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.0876000000000001e-05 2023-10-01T11:14:01.898 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.898 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.898 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:14:01.898 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:11:23.798101+0000", 2023-10-01T11:14:01.898 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 7.0149999999999998e-06 2023-10-01T11:14:01.898 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.898 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.898 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:14:01.898 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:11:23.798159+0000", 2023-10-01T11:14:01.898 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.8186000000000002e-05 2023-10-01T11:14:01.898 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.899 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.899 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:14:01.899 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:11:23.798177+0000", 2023-10-01T11:14:01.899 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.7693000000000002e-05 2023-10-01T11:14:01.899 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.899 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.899 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:14:01.899 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:11:23.919607+0000", 2023-10-01T11:14:01.899 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.121430356 2023-10-01T11:14:01.900 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.900 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.900 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:14:01.900 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:11:23.919722+0000", 2023-10-01T11:14:01.900 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00011506399999999999 2023-10-01T11:14:01.900 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:14:01.900 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:14:01.900 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:14:01.900 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.900 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.900 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.36819.0:113245 14.8 e364/356 14:15ecbb71:::benchmark_data_smithi078_236918_object8099:head v 364'7062, mlcod=364'7059)", 2023-10-01T11:14:01.901 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:11:23.799851+0000", 2023-10-01T11:14:01.901 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 157.99174543999999, 2023-10-01T11:14:01.901 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.103216238, 2023-10-01T11:14:01.901 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:14:01.901 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:14:01.901 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:14:01.901 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.901 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:14:01.901 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:11:23.799851+0000", 2023-10-01T11:14:01.901 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:14:01.902 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.902 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.902 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:14:01.902 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:11:23.799851+0000", 2023-10-01T11:14:01.902 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:14:01.902 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.902 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.902 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:14:01.902 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:11:23.799850+0000", 2023-10-01T11:14:01.903 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.999999 2023-10-01T11:14:01.903 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.903 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.903 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:14:01.903 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:11:23.799858+0000", 2023-10-01T11:14:01.903 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 8.3250000000000008e-06 2023-10-01T11:14:01.903 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.903 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.903 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:14:01.903 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:11:23.896417+0000", 2023-10-01T11:14:01.903 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.096558407999999998 2023-10-01T11:14:01.904 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.904 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.904 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:14:01.904 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:11:23.896426+0000", 2023-10-01T11:14:01.904 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 9.6660000000000003e-06 2023-10-01T11:14:01.904 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.904 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.904 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:14:01.905 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:11:23.898826+0000", 2023-10-01T11:14:01.905 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0023995370000000002 2023-10-01T11:14:01.905 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.905 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.905 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:14:01.905 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:11:23.898884+0000", 2023-10-01T11:14:01.905 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.7969e-05 2023-10-01T11:14:01.905 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.905 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.906 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:14:01.906 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:11:23.902875+0000", 2023-10-01T11:14:01.906 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0039909489999999997 2023-10-01T11:14:01.906 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.906 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.906 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:14:01.906 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:11:23.903067+0000", 2023-10-01T11:14:01.906 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00019256500000000001 2023-10-01T11:14:01.906 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:14:01.907 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:14:01.907 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:14:01.907 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.907 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.907 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.36819.0:113253 14.3 e364/356 14:c0081832:::benchmark_data_smithi078_236918_object8107:head v 364'7134, mlcod=364'7132)", 2023-10-01T11:14:01.907 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:11:23.804076+0000", 2023-10-01T11:14:01.907 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 157.987520647, 2023-10-01T11:14:01.907 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.106761826, 2023-10-01T11:14:01.908 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:14:01.908 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:14:01.908 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:14:01.908 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.908 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:14:01.908 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:11:23.804076+0000", 2023-10-01T11:14:01.908 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:14:01.908 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.908 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.908 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:14:01.909 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:11:23.804076+0000", 2023-10-01T11:14:01.909 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:14:01.909 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.909 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.909 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:14:01.909 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:11:23.804075+0000", 2023-10-01T11:14:01.909 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999986 2023-10-01T11:14:01.910 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.910 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.910 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:14:01.910 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:11:23.804085+0000", 2023-10-01T11:14:01.910 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.0757000000000001e-05 2023-10-01T11:14:01.910 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.910 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.911 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:14:01.911 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:11:23.896417+0000", 2023-10-01T11:14:01.911 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.0923321 2023-10-01T11:14:01.911 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.911 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.911 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:14:01.911 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:11:23.896443+0000", 2023-10-01T11:14:01.911 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 2.5636e-05 2023-10-01T11:14:01.912 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.912 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.912 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:14:01.912 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:11:23.896873+0000", 2023-10-01T11:14:01.912 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00042989699999999999 2023-10-01T11:14:01.912 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.912 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.912 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:14:01.912 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:11:23.896922+0000", 2023-10-01T11:14:01.912 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.9512999999999998e-05 2023-10-01T11:14:01.912 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.913 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.913 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:14:01.913 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:11:23.910426+0000", 2023-10-01T11:14:01.913 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.013503326 2023-10-01T11:14:01.913 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.913 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.913 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:14:01.913 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:11:23.910838+0000", 2023-10-01T11:14:01.913 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.00041182700000000003 2023-10-01T11:14:01.913 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:14:01.913 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:14:01.913 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:14:01.914 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.914 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.914 INFO:teuthology.orchestra.run.smithi078.stdout: "description": "osd_repop(client.36819.0:121834 14.c e364/356 14:315cf119:::benchmark_data_smithi078_236918_object16688:head v 364'7623, mlcod=364'7619)", 2023-10-01T11:14:01.914 INFO:teuthology.orchestra.run.smithi078.stdout: "initiated_at": "2023-10-01T11:11:36.145345+0000", 2023-10-01T11:14:01.914 INFO:teuthology.orchestra.run.smithi078.stdout: "age": 145.646251108, 2023-10-01T11:14:01.914 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.101937918, 2023-10-01T11:14:01.914 INFO:teuthology.orchestra.run.smithi078.stdout: "type_data": { 2023-10-01T11:14:01.914 INFO:teuthology.orchestra.run.smithi078.stdout: "flag_point": "commit sent; apply or cleanup", 2023-10-01T11:14:01.914 INFO:teuthology.orchestra.run.smithi078.stdout: "events": [ 2023-10-01T11:14:01.914 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.915 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "initiated", 2023-10-01T11:14:01.915 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:11:36.145345+0000", 2023-10-01T11:14:01.915 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:14:01.915 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.915 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.915 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "throttled", 2023-10-01T11:14:01.915 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:11:36.145345+0000", 2023-10-01T11:14:01.915 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0 2023-10-01T11:14:01.915 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.915 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.916 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "header_read", 2023-10-01T11:14:01.916 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:11:36.145345+0000", 2023-10-01T11:14:01.916 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4294967295.9999995 2023-10-01T11:14:01.916 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.916 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.916 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "all_read", 2023-10-01T11:14:01.916 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:11:36.145349+0000", 2023-10-01T11:14:01.916 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.2409999999999997e-06 2023-10-01T11:14:01.916 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.916 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.916 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "dispatched", 2023-10-01T11:14:01.917 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:11:36.145355+0000", 2023-10-01T11:14:01.917 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 5.7150000000000003e-06 2023-10-01T11:14:01.917 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.917 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.917 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "queued_for_pg", 2023-10-01T11:14:01.917 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:11:36.145359+0000", 2023-10-01T11:14:01.917 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 4.3630000000000004e-06 2023-10-01T11:14:01.917 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.917 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.917 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "reached_pg", 2023-10-01T11:14:01.917 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:11:36.146418+0000", 2023-10-01T11:14:01.917 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.001059078 2023-10-01T11:14:01.918 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.918 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.918 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "started", 2023-10-01T11:14:01.918 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:11:36.146438+0000", 2023-10-01T11:14:01.918 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 1.9724e-05 2023-10-01T11:14:01.918 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.918 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.918 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "commit_sent", 2023-10-01T11:14:01.918 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:11:36.247244+0000", 2023-10-01T11:14:01.919 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 0.100805388 2023-10-01T11:14:01.919 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2023-10-01T11:14:01.919 INFO:teuthology.orchestra.run.smithi078.stdout: { 2023-10-01T11:14:01.919 INFO:teuthology.orchestra.run.smithi078.stdout: "event": "done", 2023-10-01T11:14:01.919 INFO:teuthology.orchestra.run.smithi078.stdout: "time": "2023-10-01T11:11:36.247283+0000", 2023-10-01T11:14:01.919 INFO:teuthology.orchestra.run.smithi078.stdout: "duration": 3.9659000000000001e-05 2023-10-01T11:14:01.919 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:14:01.919 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:14:01.919 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:14:01.919 INFO:teuthology.orchestra.run.smithi078.stdout: } 2023-10-01T11:14:01.920 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2023-10-01T11:14:01.920 INFO:teuthology.orchestra.run.smithi078.stdout:} 2023-10-01T11:14:01.920 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:01.921 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T11:14:01.921 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T11:14:01.921 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T11:14:01.948 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T11:14:01.956 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:01.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:01.959 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:14:01.976 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T11:14:01.999 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T11:14:02.020 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T11:14:02.050 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T11:14:02.051 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T11:14:02.051 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T11:14:02.051 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T11:14:02.052 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T11:14:02.052 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T11:14:02.052 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T11:14:02.052 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T11:14:02.052 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T11:14:02.059 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:02.072 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:01 smithi078 bash[78274]: cluster 2023-10-01T11:14:00.481897+0000 mgr.y (mgr.25072) 1704 : cluster 0 pgmap v1839: 89 pgs: 89 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 497 op/s 2023-10-01T11:14:02.072 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:01 smithi078 bash[78274]: audit 2023-10-01T11:14:01.848091+0000 mon.c (mon.2) 248 : audit 0 from='client.? 172.21.15.78:0/1982804929' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:14:02.072 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:01 smithi078 bash[75299]: cluster 2023-10-01T11:14:00.481897+0000 mgr.y (mgr.25072) 1704 : cluster 0 pgmap v1839: 89 pgs: 89 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 497 op/s 2023-10-01T11:14:02.072 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:01 smithi078 bash[75299]: audit 2023-10-01T11:14:01.848091+0000 mon.c (mon.2) 248 : audit 0 from='client.? 172.21.15.78:0/1982804929' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:14:02.089 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:02.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:02.092 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:14:02.192 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:02.200 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:01 smithi173 bash[58816]: cluster 2023-10-01T11:14:00.481897+0000 mgr.y (mgr.25072) 1704 : cluster 0 pgmap v1839: 89 pgs: 89 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 497 op/s 2023-10-01T11:14:02.200 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:01 smithi173 bash[58816]: audit 2023-10-01T11:14:01.848091+0000 mon.c (mon.2) 248 : audit 0 from='client.? 172.21.15.78:0/1982804929' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:14:02.220 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:02.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:02.222 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:14:02.323 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:02.352 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:02.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:02.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:14:02.455 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:02.490 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:02.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:02.494 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:14:02.595 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:02.621 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:02.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:02.624 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:14:02.725 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:02.755 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:02.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:02.757 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:14:02.858 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:02.896 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:02.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:02.900 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:14:02.928 INFO:teuthology.orchestra.run.smithi078.stderr:nodeep-scrub is set 2023-10-01T11:14:03.001 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:03.034 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:03.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:03.038 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:14:03.138 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:03.152 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:02 smithi173 bash[58816]: audit 2023-10-01T11:14:02.039305+0000 mon.c (mon.2) 249 : audit 1 from='client.? 172.21.15.78:0/1751737955' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:14:03.152 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:02 smithi173 bash[58816]: audit 2023-10-01T11:14:02.039679+0000 mon.a (mon.0) 2178 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:14:03.152 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:02 smithi173 bash[58816]: audit 2023-10-01T11:14:02.482182+0000 mon.a (mon.0) 2179 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:14:03.170 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:03.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:03.173 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:14:03.274 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:03.287 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:02 smithi078 bash[78274]: audit 2023-10-01T11:14:02.039305+0000 mon.c (mon.2) 249 : audit 1 from='client.? 172.21.15.78:0/1751737955' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:14:03.287 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:02 smithi078 bash[78274]: audit 2023-10-01T11:14:02.039679+0000 mon.a (mon.0) 2178 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:14:03.288 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:02 smithi078 bash[78274]: audit 2023-10-01T11:14:02.482182+0000 mon.a (mon.0) 2179 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:14:03.288 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:02 smithi078 bash[75299]: audit 2023-10-01T11:14:02.039305+0000 mon.c (mon.2) 249 : audit 1 from='client.? 172.21.15.78:0/1751737955' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:14:03.288 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:02 smithi078 bash[75299]: audit 2023-10-01T11:14:02.039679+0000 mon.a (mon.0) 2178 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:14:03.288 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:02 smithi078 bash[75299]: audit 2023-10-01T11:14:02.482182+0000 mon.a (mon.0) 2179 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:14:03.306 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:03.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:03.308 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:14:03.409 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:03.445 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:03.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:03.448 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:14:03.549 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:03.579 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:03.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:03.583 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:14:03.684 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:03.718 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:03.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:03.720 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:14:03.821 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:03.862 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:03.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:03.865 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:14:03.966 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:03.982 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:03 smithi173 bash[58816]: cluster 2023-10-01T11:14:02.482371+0000 mgr.y (mgr.25072) 1705 : cluster 0 pgmap v1840: 89 pgs: 89 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 297 op/s 2023-10-01T11:14:03.984 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:03 smithi173 bash[58816]: cluster 2023-10-01T11:14:02.887174+0000 mon.a (mon.0) 2180 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:14:03.984 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:03 smithi173 bash[58816]: cluster 2023-10-01T11:14:02.918777+0000 osd.7 (osd.7) 424 : cluster 3 failed to encode map e377 with expected crc 2023-10-01T11:14:03.990 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:03 smithi173 bash[58816]: audit 2023-10-01T11:14:02.923357+0000 mon.a (mon.0) 2181 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T11:14:03.990 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:03 smithi173 bash[58816]: cluster 2023-10-01T11:14:02.923439+0000 mon.a (mon.0) 2182 : cluster 0 osdmap e377: 8 total, 8 up, 8 in 2023-10-01T11:14:03.991 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:03 smithi173 bash[58816]: cluster 2023-10-01T11:14:02.987890+0000 osd.6 (osd.6) 364 : cluster 3 failed to encode map e377 with expected crc 2023-10-01T11:14:03.991 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:03 smithi173 bash[58816]: cluster 2023-10-01T11:14:02.991082+0000 osd.6 (osd.6) 365 : cluster 3 failed to encode map e377 with expected crc 2023-10-01T11:14:04.001 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:04.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:04.011 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:14:04.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:03 smithi078 bash[75299]: cluster 2023-10-01T11:14:02.482371+0000 mgr.y (mgr.25072) 1705 : cluster 0 pgmap v1840: 89 pgs: 89 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 297 op/s 2023-10-01T11:14:04.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:03 smithi078 bash[75299]: cluster 2023-10-01T11:14:02.887174+0000 mon.a (mon.0) 2180 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:14:04.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:03 smithi078 bash[75299]: cluster 2023-10-01T11:14:02.918777+0000 osd.7 (osd.7) 424 : cluster 3 failed to encode map e377 with expected crc 2023-10-01T11:14:04.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:03 smithi078 bash[75299]: audit 2023-10-01T11:14:02.923357+0000 mon.a (mon.0) 2181 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T11:14:04.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:03 smithi078 bash[75299]: cluster 2023-10-01T11:14:02.923439+0000 mon.a (mon.0) 2182 : cluster 0 osdmap e377: 8 total, 8 up, 8 in 2023-10-01T11:14:04.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:03 smithi078 bash[75299]: cluster 2023-10-01T11:14:02.987890+0000 osd.6 (osd.6) 364 : cluster 3 failed to encode map e377 with expected crc 2023-10-01T11:14:04.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:03 smithi078 bash[75299]: cluster 2023-10-01T11:14:02.991082+0000 osd.6 (osd.6) 365 : cluster 3 failed to encode map e377 with expected crc 2023-10-01T11:14:04.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:03 smithi078 bash[78274]: cluster 2023-10-01T11:14:02.482371+0000 mgr.y (mgr.25072) 1705 : cluster 0 pgmap v1840: 89 pgs: 89 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 297 op/s 2023-10-01T11:14:04.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:03 smithi078 bash[78274]: cluster 2023-10-01T11:14:02.887174+0000 mon.a (mon.0) 2180 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:14:04.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:03 smithi078 bash[78274]: cluster 2023-10-01T11:14:02.918777+0000 osd.7 (osd.7) 424 : cluster 3 failed to encode map e377 with expected crc 2023-10-01T11:14:04.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:03 smithi078 bash[78274]: audit 2023-10-01T11:14:02.923357+0000 mon.a (mon.0) 2181 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T11:14:04.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:03 smithi078 bash[78274]: cluster 2023-10-01T11:14:02.923439+0000 mon.a (mon.0) 2182 : cluster 0 osdmap e377: 8 total, 8 up, 8 in 2023-10-01T11:14:04.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:03 smithi078 bash[78274]: cluster 2023-10-01T11:14:02.987890+0000 osd.6 (osd.6) 364 : cluster 3 failed to encode map e377 with expected crc 2023-10-01T11:14:04.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:03 smithi078 bash[78274]: cluster 2023-10-01T11:14:02.991082+0000 osd.6 (osd.6) 365 : cluster 3 failed to encode map e377 with expected crc 2023-10-01T11:14:04.112 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:04.144 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:04.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:04.146 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:14:04.247 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:04.279 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:04.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:04.282 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:14:04.383 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:04.410 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:04.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:04.413 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:14:04.513 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:04.541 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:04.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:04.543 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:14:04.644 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:04.678 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:04.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:04.682 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:14:04.734 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-10-01T11:14:04.782 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:04.829 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:04.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:04.830 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:14:04.931 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:04.964 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:04.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:04.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:05.068 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:05.106 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:05.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:05.109 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:14:05.210 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:05.244 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:05.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:05.247 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:14:05.348 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:05.385 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:05.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:05.388 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:05.489 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:05.520 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:05.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:05.523 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:14:05.624 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:05.655 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:05.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:05.659 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:14:05.760 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:05.793 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:05.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:05.796 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:05.897 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:05.931 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:05.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:05.933 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:14:06.034 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:06.046 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:05 smithi078 bash[75299]: cluster 2023-10-01T11:14:04.376669+0000 osd.5 (osd.5) 405 : cluster 3 failed to encode map e377 with expected crc 2023-10-01T11:14:06.047 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:05 smithi078 bash[75299]: cluster 2023-10-01T11:14:04.483256+0000 mgr.y (mgr.25072) 1706 : cluster 0 pgmap v1842: 89 pgs: 89 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 129 op/s 2023-10-01T11:14:06.047 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:05 smithi078 bash[78274]: cluster 2023-10-01T11:14:04.376669+0000 osd.5 (osd.5) 405 : cluster 3 failed to encode map e377 with expected crc 2023-10-01T11:14:06.047 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:05 smithi078 bash[78274]: cluster 2023-10-01T11:14:04.483256+0000 mgr.y (mgr.25072) 1706 : cluster 0 pgmap v1842: 89 pgs: 89 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 129 op/s 2023-10-01T11:14:06.069 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:06.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:06.073 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:06.174 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:06.204 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:06.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:06.207 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:14:06.301 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:14:06.308 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:06.337 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:06.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:06.340 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:06.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:05 smithi173 bash[58816]: cluster 2023-10-01T11:14:04.376669+0000 osd.5 (osd.5) 405 : cluster 3 failed to encode map e377 with expected crc 2023-10-01T11:14:06.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:05 smithi173 bash[58816]: cluster 2023-10-01T11:14:04.483256+0000 mgr.y (mgr.25072) 1706 : cluster 0 pgmap v1842: 89 pgs: 89 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 129 op/s 2023-10-01T11:14:06.441 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:06.466 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:06.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:06.469 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:14:06.569 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:06.600 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:06.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:06.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:14:06.704 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:06.736 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:06.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:06.739 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:14:06.791 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T11:14:06.808 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T11:14:06.840 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:06.877 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:06.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:06.881 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:14:06.981 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:07.015 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:07.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:07.019 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:14:07.119 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:07.157 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:07.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:07.160 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:14:07.261 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:07.291 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:07.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:07.295 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:14:07.321 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:14:07.395 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:07.423 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:07.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:07.427 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:14:07.527 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:07.559 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:07.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:07.561 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:07.661 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:07.692 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:07.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:07.695 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:14:07.795 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:07.832 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:07.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:07.836 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:14:07.936 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:07.961 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:07.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:07.963 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:14:08.064 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:08.092 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:08.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:08.094 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:08.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:07 smithi173 bash[58816]: cluster 2023-10-01T11:14:06.484255+0000 mgr.y (mgr.25072) 1707 : cluster 0 pgmap v1843: 89 pgs: 89 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 5 op/s 2023-10-01T11:14:08.195 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:08.228 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:08.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:08.230 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:14:08.331 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:08.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:07 smithi078 bash[78274]: cluster 2023-10-01T11:14:06.484255+0000 mgr.y (mgr.25072) 1707 : cluster 0 pgmap v1843: 89 pgs: 89 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 5 op/s 2023-10-01T11:14:08.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:07 smithi078 bash[75299]: cluster 2023-10-01T11:14:06.484255+0000 mgr.y (mgr.25072) 1707 : cluster 0 pgmap v1843: 89 pgs: 89 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 5 op/s 2023-10-01T11:14:08.355 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:08.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:08.358 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:08.459 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:08.499 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:08.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:08.502 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:14:08.603 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:08.626 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:08.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:08.628 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:08.728 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:08.758 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:08.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:08.760 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:14:08.861 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:08.893 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:08.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:08.895 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:14:08.996 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:09.025 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:09.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:09.028 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:14:09.129 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:09.157 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:09.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:09.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:14:09.260 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:09.292 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:09.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:09.296 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:14:09.350 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:14:09.396 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:09.421 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:09.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:09.423 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:09.524 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:09.548 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:09.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:09.551 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:09.651 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:09.682 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:09.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:09.685 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:14:09.786 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:09.811 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:09.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:09.813 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:09.914 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:09.939 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:09.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:09.942 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:14:10.043 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:10.069 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:10.070 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:09 smithi078 bash[75299]: cluster 2023-10-01T11:14:08.484915+0000 mgr.y (mgr.25072) 1708 : cluster 0 pgmap v1844: 89 pgs: 89 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 5 op/s 2023-10-01T11:14:10.071 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:09 smithi078 bash[78274]: cluster 2023-10-01T11:14:08.484915+0000 mgr.y (mgr.25072) 1708 : cluster 0 pgmap v1844: 89 pgs: 89 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 5 op/s 2023-10-01T11:14:10.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:10.071 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:14:10.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:09 smithi173 bash[58816]: cluster 2023-10-01T11:14:08.484915+0000 mgr.y (mgr.25072) 1708 : cluster 0 pgmap v1844: 89 pgs: 89 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 5 op/s 2023-10-01T11:14:10.173 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:10.204 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:10.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:10.207 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:14:10.308 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:10.341 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:10.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:10.343 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:14:10.382 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:14:10 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:14:10] "GET /metrics HTTP/1.1" 200 36330 "" "Prometheus/2.33.4" 2023-10-01T11:14:10.445 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:10.474 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:10.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:10.477 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:10.577 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:10.605 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:10.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:10.608 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:10.709 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:10.731 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:10.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:10.733 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:10.834 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:10.865 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:10.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:10.867 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:10.969 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:10.990 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:10.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:10.992 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:11.093 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:11.124 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:11.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:11.126 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:14:11.227 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:11.257 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:11.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:11.260 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:14:11.361 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:11.388 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:11.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:11.392 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:14:11.492 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:11.527 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:11.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:11.530 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:14:11.631 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:11.659 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:11.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:11.662 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:14:11.763 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:11.788 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:11.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:11.790 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:11.891 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:11.924 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:11.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:11.926 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:14:12.026 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:12.059 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:12.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:12.062 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:14:12.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:11 smithi078 bash[75299]: cluster 2023-10-01T11:14:10.485880+0000 mgr.y (mgr.25072) 1709 : cluster 0 pgmap v1845: 89 pgs: 89 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:14:12.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:11 smithi078 bash[78274]: cluster 2023-10-01T11:14:10.485880+0000 mgr.y (mgr.25072) 1709 : cluster 0 pgmap v1845: 89 pgs: 89 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:14:12.163 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:12.193 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:12.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:12.197 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:14:12.298 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:12.309 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:11 smithi173 bash[58816]: cluster 2023-10-01T11:14:10.485880+0000 mgr.y (mgr.25072) 1709 : cluster 0 pgmap v1845: 89 pgs: 89 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:14:12.326 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:12.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:12.329 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:14:12.430 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:12.459 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:12.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:12.462 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:14:12.563 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:12.584 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:12.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:12.586 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:14:12.687 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:12.714 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:12.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:12.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:12.817 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:12.848 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:12.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:12.851 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:14:12.952 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:12.984 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:12.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:12.986 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:14:13.087 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:13.123 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:13.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:13.126 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:14:13.227 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:13.249 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:13.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:13.252 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:13.353 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:13.380 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:13.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:13.383 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:13.484 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:13.519 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:13.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:13.522 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:14:13.623 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:13.657 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:13.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:13.660 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:14:13.761 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:13.785 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:13.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:13.787 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:14:13.888 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:13.914 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:13.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:13.917 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:13.943 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:14:13.943 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:14:14.018 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:14.031 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:13 smithi078 bash[78274]: cluster 2023-10-01T11:14:12.486651+0000 mgr.y (mgr.25072) 1710 : cluster 0 pgmap v1846: 89 pgs: 89 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:14:14.031 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:13 smithi078 bash[75299]: cluster 2023-10-01T11:14:12.486651+0000 mgr.y (mgr.25072) 1710 : cluster 0 pgmap v1846: 89 pgs: 89 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:14:14.046 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:14.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:14.048 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:14.149 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:14.182 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:14.184 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:13 smithi173 bash[58816]: cluster 2023-10-01T11:14:12.486651+0000 mgr.y (mgr.25072) 1710 : cluster 0 pgmap v1846: 89 pgs: 89 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:14:14.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:14.185 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:14:14.286 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:14.315 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:14.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:14.318 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:14:14.420 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:14.453 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:14.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:14.456 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:14:14.557 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:14.582 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:14.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:14.586 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:14.687 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:14.712 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:14.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:14.715 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:14.815 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:14.840 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:14.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:14.843 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:14.943 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:14.971 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:14.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:14.975 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:14:15.076 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:15.105 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:15.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:15.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:14:15.210 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:15.240 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:15.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:15.243 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:14:15.344 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:15.373 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:15.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:15.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:14:15.477 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:15.509 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:15.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:15.511 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:14:15.612 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:15.640 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:15.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:15.642 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:15.743 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:15.767 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:15.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:15.770 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:14:15.871 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:15.894 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:15.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:15.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:15.998 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:16.007 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:15 smithi078 bash[75299]: cluster 2023-10-01T11:14:14.487450+0000 mgr.y (mgr.25072) 1711 : cluster 0 pgmap v1847: 89 pgs: 89 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:14:16.007 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:15 smithi078 bash[78274]: cluster 2023-10-01T11:14:14.487450+0000 mgr.y (mgr.25072) 1711 : cluster 0 pgmap v1847: 89 pgs: 89 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:14:16.024 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:16.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:16.027 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:14:16.128 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:16.158 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:16.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:16.161 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:14:16.262 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:16.283 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:16.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:16.285 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:16.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:15 smithi173 bash[58816]: cluster 2023-10-01T11:14:14.487450+0000 mgr.y (mgr.25072) 1711 : cluster 0 pgmap v1847: 89 pgs: 89 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:14:16.386 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:16.416 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:16.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:16.419 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:16.520 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:16.548 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:16.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:16.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:14:16.652 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:16.681 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:16.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:16.683 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:14:16.784 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:16.822 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:16.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:16.825 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:14:16.926 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:16.951 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:16.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:16.954 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:14:17.054 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:17.087 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:17.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:17.090 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:14:17.191 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:17.214 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:17.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:17.217 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:17.318 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:17.352 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:17.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:17.355 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:14:17.456 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:17.489 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:17.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:17.493 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:14:17.594 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:17.625 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:17.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:17.628 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:14:17.729 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:17.755 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:17.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:17.758 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:17.858 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:17.880 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:17.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:17.882 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:17.983 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:18.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:17 smithi078 bash[75299]: cluster 2023-10-01T11:14:16.488353+0000 mgr.y (mgr.25072) 1712 : cluster 0 pgmap v1848: 89 pgs: 89 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:14:18.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:17 smithi078 bash[75299]: audit 2023-10-01T11:14:17.482727+0000 mon.a (mon.0) 2183 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:14:18.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:17 smithi078 bash[78274]: cluster 2023-10-01T11:14:16.488353+0000 mgr.y (mgr.25072) 1712 : cluster 0 pgmap v1848: 89 pgs: 89 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:14:18.014 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:17 smithi078 bash[78274]: audit 2023-10-01T11:14:17.482727+0000 mon.a (mon.0) 2183 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:14:18.017 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:18.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:18.021 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:14:18.122 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:18.146 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:18.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:18.148 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:18.249 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:18.260 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:17 smithi173 bash[58816]: cluster 2023-10-01T11:14:16.488353+0000 mgr.y (mgr.25072) 1712 : cluster 0 pgmap v1848: 89 pgs: 89 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:14:18.260 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:17 smithi173 bash[58816]: audit 2023-10-01T11:14:17.482727+0000 mon.a (mon.0) 2183 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:14:18.277 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:18.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:18.279 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:14:18.380 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:18.404 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:18.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:18.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:18.507 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:18.538 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:18.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:18.540 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:14:18.641 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:18.676 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:18.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:18.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:14:18.780 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:18.802 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:18.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:18.804 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:18.905 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:18.930 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:18.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:18.934 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:18.957 INFO:teuthology.orchestra.run.smithi078.stderr:pool 'unique_pool_9' removed 2023-10-01T11:14:19.035 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:19.054 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:18 smithi078 bash[78274]: audit 2023-10-01T11:14:18.463980+0000 mon.c (mon.2) 250 : audit 1 from='client.? 172.21.15.78:0/782727802' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T11:14:19.054 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:18 smithi078 bash[78274]: audit 2023-10-01T11:14:18.464656+0000 mon.a (mon.0) 2184 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T11:14:19.055 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:18 smithi078 bash[75299]: audit 2023-10-01T11:14:18.463980+0000 mon.c (mon.2) 250 : audit 1 from='client.? 172.21.15.78:0/782727802' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T11:14:19.055 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:18 smithi078 bash[75299]: audit 2023-10-01T11:14:18.464656+0000 mon.a (mon.0) 2184 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T11:14:19.055 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:19.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:19.058 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:19.158 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:19.187 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:19.189 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:18 smithi173 bash[58816]: audit 2023-10-01T11:14:18.463980+0000 mon.c (mon.2) 250 : audit 1 from='client.? 172.21.15.78:0/782727802' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T11:14:19.189 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:18 smithi173 bash[58816]: audit 2023-10-01T11:14:18.464656+0000 mon.a (mon.0) 2184 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]: dispatch 2023-10-01T11:14:19.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:19.190 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:14:19.290 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:19.323 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:19.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:19.331 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:14:19.432 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:19.461 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:19.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:19.464 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:14:19.565 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:19.588 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:19.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:19.591 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:14:19.692 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:19.712 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:19.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:19.714 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:19.814 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:19.845 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:19.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:19.848 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:14:19.949 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:19.976 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:19.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:19.979 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:20.079 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:20.099 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:20.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:20.101 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:20.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:19 smithi173 bash[58816]: cluster 2023-10-01T11:14:18.489069+0000 mgr.y (mgr.25072) 1713 : cluster 0 pgmap v1849: 89 pgs: 89 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:14:20.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:19 smithi173 bash[58816]: cluster 2023-10-01T11:14:18.942011+0000 osd.5 (osd.5) 406 : cluster 3 failed to encode map e378 with expected crc 2023-10-01T11:14:20.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:19 smithi173 bash[58816]: audit 2023-10-01T11:14:18.943187+0000 mon.a (mon.0) 2185 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]': finished 2023-10-01T11:14:20.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:19 smithi173 bash[58816]: cluster 2023-10-01T11:14:18.943267+0000 mon.a (mon.0) 2186 : cluster 0 osdmap e378: 8 total, 8 up, 8 in 2023-10-01T11:14:20.202 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:20.211 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:19 smithi078 bash[75299]: cluster 2023-10-01T11:14:18.489069+0000 mgr.y (mgr.25072) 1713 : cluster 0 pgmap v1849: 89 pgs: 89 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:14:20.211 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:19 smithi078 bash[75299]: cluster 2023-10-01T11:14:18.942011+0000 osd.5 (osd.5) 406 : cluster 3 failed to encode map e378 with expected crc 2023-10-01T11:14:20.211 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:19 smithi078 bash[75299]: audit 2023-10-01T11:14:18.943187+0000 mon.a (mon.0) 2185 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]': finished 2023-10-01T11:14:20.211 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:19 smithi078 bash[75299]: cluster 2023-10-01T11:14:18.943267+0000 mon.a (mon.0) 2186 : cluster 0 osdmap e378: 8 total, 8 up, 8 in 2023-10-01T11:14:20.211 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:19 smithi078 bash[78274]: cluster 2023-10-01T11:14:18.489069+0000 mgr.y (mgr.25072) 1713 : cluster 0 pgmap v1849: 89 pgs: 89 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:14:20.211 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:19 smithi078 bash[78274]: cluster 2023-10-01T11:14:18.942011+0000 osd.5 (osd.5) 406 : cluster 3 failed to encode map e378 with expected crc 2023-10-01T11:14:20.211 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:19 smithi078 bash[78274]: audit 2023-10-01T11:14:18.943187+0000 mon.a (mon.0) 2185 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]': finished 2023-10-01T11:14:20.211 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:19 smithi078 bash[78274]: cluster 2023-10-01T11:14:18.943267+0000 mon.a (mon.0) 2186 : cluster 0 osdmap e378: 8 total, 8 up, 8 in 2023-10-01T11:14:20.227 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:20.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:20.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:14:20.331 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:20.366 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:20.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:20.368 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:14:20.469 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:20.480 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:14:20 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:14:20] "GET /metrics HTTP/1.1" 200 36328 "" "Prometheus/2.33.4" 2023-10-01T11:14:20.506 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:20.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:20.510 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:14:20.610 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:20.643 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:20.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:20.646 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:14:20.747 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:20.770 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:20.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:20.773 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:20.873 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:20.892 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:20.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:20.894 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:14:20.995 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:21.020 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:21.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:21.021 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:21.122 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:21.148 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:21.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:21.150 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:14:21.251 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:21.284 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:21.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:21.286 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:14:21.388 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:21.420 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:21.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:21.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:14:21.524 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:21.558 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:21.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:21.559 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:14:21.660 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:21.682 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:21.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:21.684 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:21.785 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:21.807 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:21.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:21.810 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:21.910 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:21.935 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:21.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:21.938 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:21.958 INFO:teuthology.orchestra.run.smithi078.stderr:set osd.7 primary-affinity to 0.440963 (8288982) 2023-10-01T11:14:22.038 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:22.072 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:22.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:22.076 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:14:22.176 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:22.204 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:22.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:22.206 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:22.307 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:22.321 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:21 smithi173 bash[58816]: cluster 2023-10-01T11:14:20.489673+0000 mgr.y (mgr.25072) 1714 : cluster 0 pgmap v1851: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:14:22.322 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:21 smithi173 bash[58816]: audit 2023-10-01T11:14:20.967420+0000 mon.a (mon.0) 2187 : audit 1 from='client.? 172.21.15.78:0/2554808021' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.44096299215870594}]: dispatch 2023-10-01T11:14:22.342 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:22.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:22.346 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:14:22.365 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:22 smithi078 bash[78274]: cluster 2023-10-01T11:14:20.489673+0000 mgr.y (mgr.25072) 1714 : cluster 0 pgmap v1851: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:14:22.365 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:22 smithi078 bash[78274]: audit 2023-10-01T11:14:20.967420+0000 mon.a (mon.0) 2187 : audit 1 from='client.? 172.21.15.78:0/2554808021' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.44096299215870594}]: dispatch 2023-10-01T11:14:22.366 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:22 smithi078 bash[75299]: cluster 2023-10-01T11:14:20.489673+0000 mgr.y (mgr.25072) 1714 : cluster 0 pgmap v1851: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:14:22.366 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:22 smithi078 bash[75299]: audit 2023-10-01T11:14:20.967420+0000 mon.a (mon.0) 2187 : audit 1 from='client.? 172.21.15.78:0/2554808021' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.44096299215870594}]: dispatch 2023-10-01T11:14:22.411 INFO:teuthology.orchestra.run.smithi078.stdout:ERROR: (22) Invalid argument 2023-10-01T11:14:22.411 INFO:teuthology.orchestra.run.smithi078.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T11:14:22.447 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:22.470 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:22.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:22.472 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:22.573 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:22.596 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:22.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:22.598 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:22.699 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:22.725 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:22.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:22.728 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:14:22.829 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:22.860 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:22.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:22.864 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:14:22.961 INFO:teuthology.task.sequential:In sequential, running task print... 2023-10-01T11:14:22.962 INFO:teuthology.task.print:**** done end radosbench.yaml 2023-10-01T11:14:22.962 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-10-01T11:14:22.966 INFO:tasks.workunit:Pulling workunits from ref pacific 2023-10-01T11:14:22.966 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-10-01T11:14:22.966 DEBUG:teuthology.orchestra.run.smithi078:> stat -- /home/ubuntu/cephtest/mnt.0 2023-10-01T11:14:22.966 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:22.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:22.998 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:22.998 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:14:23.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:23.073 INFO:teuthology.orchestra.run.smithi078.stderr:stat: cannot stat '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-10-01T11:14:23.074 DEBUG:teuthology.orchestra.run.smithi078:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-10-01T11:14:23.099 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:23.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:22 smithi173 bash[58816]: cluster 2023-10-01T11:14:21.951536+0000 osd.7 (osd.7) 425 : cluster 3 failed to encode map e379 with expected crc 2023-10-01T11:14:23.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:22 smithi173 bash[58816]: cluster 2023-10-01T11:14:21.951580+0000 osd.5 (osd.5) 407 : cluster 3 failed to encode map e379 with expected crc 2023-10-01T11:14:23.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:22 smithi173 bash[58816]: audit 2023-10-01T11:14:21.952722+0000 mon.a (mon.0) 2188 : audit 1 from='client.? 172.21.15.78:0/2554808021' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.44096299215870594}]': finished 2023-10-01T11:14:23.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:22 smithi173 bash[58816]: cluster 2023-10-01T11:14:21.952814+0000 mon.a (mon.0) 2189 : cluster 0 osdmap e379: 8 total, 8 up, 8 in 2023-10-01T11:14:23.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:22 smithi173 bash[58816]: cluster 2023-10-01T11:14:21.954316+0000 osd.7 (osd.7) 426 : cluster 3 failed to encode map e379 with expected crc 2023-10-01T11:14:23.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:22 smithi173 bash[58816]: cluster 2023-10-01T11:14:22.413906+0000 osd.6 (osd.6) 366 : cluster 3 failed to encode map e379 with expected crc 2023-10-01T11:14:23.140 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:23.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:23.143 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:14:23.203 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-10-01T11:14:23.203 DEBUG:teuthology.orchestra.run.smithi078:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-10-01T11:14:23.210 INFO:tasks.workunit:timeout=3h 2023-10-01T11:14:23.210 INFO:tasks.workunit:cleanup=True 2023-10-01T11:14:23.211 DEBUG:teuthology.orchestra.run.smithi078:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch pacific https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-10-01T11:14:23.244 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:23.259 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:22 smithi078 bash[75299]: cluster 2023-10-01T11:14:21.951536+0000 osd.7 (osd.7) 425 : cluster 3 failed to encode map e379 with expected crc 2023-10-01T11:14:23.259 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:22 smithi078 bash[75299]: cluster 2023-10-01T11:14:21.951580+0000 osd.5 (osd.5) 407 : cluster 3 failed to encode map e379 with expected crc 2023-10-01T11:14:23.259 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:22 smithi078 bash[75299]: audit 2023-10-01T11:14:21.952722+0000 mon.a (mon.0) 2188 : audit 1 from='client.? 172.21.15.78:0/2554808021' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.44096299215870594}]': finished 2023-10-01T11:14:23.259 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:22 smithi078 bash[75299]: cluster 2023-10-01T11:14:21.952814+0000 mon.a (mon.0) 2189 : cluster 0 osdmap e379: 8 total, 8 up, 8 in 2023-10-01T11:14:23.259 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:22 smithi078 bash[75299]: cluster 2023-10-01T11:14:21.954316+0000 osd.7 (osd.7) 426 : cluster 3 failed to encode map e379 with expected crc 2023-10-01T11:14:23.260 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:22 smithi078 bash[75299]: cluster 2023-10-01T11:14:22.413906+0000 osd.6 (osd.6) 366 : cluster 3 failed to encode map e379 with expected crc 2023-10-01T11:14:23.260 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:22 smithi078 bash[78274]: cluster 2023-10-01T11:14:21.951536+0000 osd.7 (osd.7) 425 : cluster 3 failed to encode map e379 with expected crc 2023-10-01T11:14:23.260 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:22 smithi078 bash[78274]: cluster 2023-10-01T11:14:21.951580+0000 osd.5 (osd.5) 407 : cluster 3 failed to encode map e379 with expected crc 2023-10-01T11:14:23.260 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:22 smithi078 bash[78274]: audit 2023-10-01T11:14:21.952722+0000 mon.a (mon.0) 2188 : audit 1 from='client.? 172.21.15.78:0/2554808021' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.44096299215870594}]': finished 2023-10-01T11:14:23.260 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:22 smithi078 bash[78274]: cluster 2023-10-01T11:14:21.952814+0000 mon.a (mon.0) 2189 : cluster 0 osdmap e379: 8 total, 8 up, 8 in 2023-10-01T11:14:23.260 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:22 smithi078 bash[78274]: cluster 2023-10-01T11:14:21.954316+0000 osd.7 (osd.7) 426 : cluster 3 failed to encode map e379 with expected crc 2023-10-01T11:14:23.260 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:22 smithi078 bash[78274]: cluster 2023-10-01T11:14:22.413906+0000 osd.6 (osd.6) 366 : cluster 3 failed to encode map e379 with expected crc 2023-10-01T11:14:23.278 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:23.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:23.280 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:23.381 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:23.410 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:23.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:23.413 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:14:23.514 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:23.544 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:23.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:23.546 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:23.647 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:23.679 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:23.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:23.683 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:14:23.783 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:23.817 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:23.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:23.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:14:23.920 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:23.949 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:23.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:23.953 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:14:23.954 INFO:tasks.workunit.client.0.smithi078.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-10-01T11:14:24.054 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:24.076 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:24.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:24.078 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:24.179 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:24.209 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:24.210 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:23 smithi078 bash[78274]: cluster 2023-10-01T11:14:22.490216+0000 mgr.y (mgr.25072) 1715 : cluster 0 pgmap v1853: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:14:24.210 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:23 smithi078 bash[78274]: cluster 2023-10-01T11:14:22.960190+0000 mon.a (mon.0) 2190 : cluster 0 osdmap e380: 8 total, 8 up, 8 in 2023-10-01T11:14:24.210 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:23 smithi078 bash[78274]: cluster 2023-10-01T11:14:22.962816+0000 osd.7 (osd.7) 427 : cluster 3 failed to encode map e380 with expected crc 2023-10-01T11:14:24.210 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:23 smithi078 bash[78274]: cluster 2023-10-01T11:14:22.964312+0000 osd.7 (osd.7) 428 : cluster 3 failed to encode map e380 with expected crc 2023-10-01T11:14:24.211 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:23 smithi078 bash[78274]: cluster 2023-10-01T11:14:23.374233+0000 osd.6 (osd.6) 367 : cluster 3 failed to encode map e380 with expected crc 2023-10-01T11:14:24.211 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:23 smithi078 bash[75299]: cluster 2023-10-01T11:14:22.490216+0000 mgr.y (mgr.25072) 1715 : cluster 0 pgmap v1853: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:14:24.211 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:23 smithi078 bash[75299]: cluster 2023-10-01T11:14:22.960190+0000 mon.a (mon.0) 2190 : cluster 0 osdmap e380: 8 total, 8 up, 8 in 2023-10-01T11:14:24.211 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:23 smithi078 bash[75299]: cluster 2023-10-01T11:14:22.962816+0000 osd.7 (osd.7) 427 : cluster 3 failed to encode map e380 with expected crc 2023-10-01T11:14:24.211 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:23 smithi078 bash[75299]: cluster 2023-10-01T11:14:22.964312+0000 osd.7 (osd.7) 428 : cluster 3 failed to encode map e380 with expected crc 2023-10-01T11:14:24.211 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:23 smithi078 bash[75299]: cluster 2023-10-01T11:14:23.374233+0000 osd.6 (osd.6) 367 : cluster 3 failed to encode map e380 with expected crc 2023-10-01T11:14:24.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:24.212 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:14:24.313 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:24.339 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:24.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:24.342 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:24.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:23 smithi173 bash[58816]: cluster 2023-10-01T11:14:22.490216+0000 mgr.y (mgr.25072) 1715 : cluster 0 pgmap v1853: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:14:24.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:23 smithi173 bash[58816]: cluster 2023-10-01T11:14:22.960190+0000 mon.a (mon.0) 2190 : cluster 0 osdmap e380: 8 total, 8 up, 8 in 2023-10-01T11:14:24.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:23 smithi173 bash[58816]: cluster 2023-10-01T11:14:22.962816+0000 osd.7 (osd.7) 427 : cluster 3 failed to encode map e380 with expected crc 2023-10-01T11:14:24.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:23 smithi173 bash[58816]: cluster 2023-10-01T11:14:22.964312+0000 osd.7 (osd.7) 428 : cluster 3 failed to encode map e380 with expected crc 2023-10-01T11:14:24.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:23 smithi173 bash[58816]: cluster 2023-10-01T11:14:23.374233+0000 osd.6 (osd.6) 367 : cluster 3 failed to encode map e380 with expected crc 2023-10-01T11:14:24.443 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:24.466 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:24.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:24.468 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:24.569 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:24.603 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:24.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:24.606 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:14:24.706 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:24.734 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:24.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:24.737 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:24.838 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:24.871 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:24.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:24.874 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:14:24.975 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:25.009 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:25.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:25.012 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:14:25.113 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:25.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:24 smithi173 bash[58816]: cluster 2023-10-01T11:14:23.296705+0000 osd.5 (osd.5) 408 : cluster 3 failed to encode map e380 with expected crc 2023-10-01T11:14:25.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:24 smithi173 bash[58816]: cluster 2023-10-01T11:14:23.298980+0000 osd.5 (osd.5) 409 : cluster 3 failed to encode map e380 with expected crc 2023-10-01T11:14:25.122 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:24 smithi078 bash[78274]: cluster 2023-10-01T11:14:23.296705+0000 osd.5 (osd.5) 408 : cluster 3 failed to encode map e380 with expected crc 2023-10-01T11:14:25.122 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:24 smithi078 bash[78274]: cluster 2023-10-01T11:14:23.298980+0000 osd.5 (osd.5) 409 : cluster 3 failed to encode map e380 with expected crc 2023-10-01T11:14:25.122 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:24 smithi078 bash[75299]: cluster 2023-10-01T11:14:23.296705+0000 osd.5 (osd.5) 408 : cluster 3 failed to encode map e380 with expected crc 2023-10-01T11:14:25.122 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:24 smithi078 bash[75299]: cluster 2023-10-01T11:14:23.298980+0000 osd.5 (osd.5) 409 : cluster 3 failed to encode map e380 with expected crc 2023-10-01T11:14:25.139 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:25.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:25.142 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:14:25.242 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:25.270 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:25.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:25.272 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:25.373 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:25.407 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:25.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:25.411 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:14:25.511 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:25.545 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:25.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:25.548 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:25.650 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:25.684 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:25.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:25.686 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:14:25.787 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:25.815 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:25.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:25.818 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:25.919 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:25.945 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:25.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:25.948 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:14:26.048 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:26.062 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:25 smithi078 bash[78274]: cluster 2023-10-01T11:14:24.490877+0000 mgr.y (mgr.25072) 1716 : cluster 0 pgmap v1855: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:14:26.063 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:25 smithi078 bash[75299]: cluster 2023-10-01T11:14:24.490877+0000 mgr.y (mgr.25072) 1716 : cluster 0 pgmap v1855: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:14:26.081 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:26.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:26.084 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:26.185 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:26.212 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:26.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:26.214 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:14:26.315 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:26.336 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:26.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:26.337 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:26.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:25 smithi173 bash[58816]: cluster 2023-10-01T11:14:24.490877+0000 mgr.y (mgr.25072) 1716 : cluster 0 pgmap v1855: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:14:26.439 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:26.460 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:26.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:26.463 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:26.563 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:26.592 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:26.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:26.594 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:14:26.695 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:26.729 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:26.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:26.733 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:14:26.833 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:26.861 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:26.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:26.864 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:26.965 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:27.006 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:27.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:27.009 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:14:27.109 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:27.145 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:27.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:27.147 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:14:27.248 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:27.276 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:27.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:27.279 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:14:27.379 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:27.405 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:27.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:27.408 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:27.509 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:27.537 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:27.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:27.540 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:14:27.641 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:27.673 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:27.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:27.677 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:14:27.777 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:27.803 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:27.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:27.805 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:27.905 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:27.940 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:27.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:27.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:14:28.044 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:28.053 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:27 smithi078 bash[75299]: cluster 2023-10-01T11:14:26.491695+0000 mgr.y (mgr.25072) 1717 : cluster 0 pgmap v1856: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:14:28.053 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:27 smithi078 bash[78274]: cluster 2023-10-01T11:14:26.491695+0000 mgr.y (mgr.25072) 1717 : cluster 0 pgmap v1856: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:14:28.054 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-10-01T11:14:28.073 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:28.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:28.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:14:28.176 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:28.188 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:27 smithi173 bash[58816]: cluster 2023-10-01T11:14:26.491695+0000 mgr.y (mgr.25072) 1717 : cluster 0 pgmap v1856: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:14:28.208 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:28.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:28.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:14:28.311 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:28.347 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:28.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:28.349 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:14:28.450 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:28.686 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:28.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:28.689 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:14:28.790 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:28.822 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:28.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:28.824 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:14:28.925 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:28.950 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:28.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:28.952 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:29.053 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:29.077 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:29.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:29.199 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:14:29.300 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:29.323 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:29.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:29.326 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:29.427 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:29.557 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:29.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:29.559 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:29.660 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:29.685 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:29.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:29.687 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:29.788 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:29.866 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:29.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:29.868 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:14:29.969 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:30.003 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:30.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:30.456 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:29 smithi078 bash[75299]: cluster 2023-10-01T11:14:28.492482+0000 mgr.y (mgr.25072) 1718 : cluster 0 pgmap v1857: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:14:30.456 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:29 smithi078 bash[78274]: cluster 2023-10-01T11:14:28.492482+0000 mgr.y (mgr.25072) 1718 : cluster 0 pgmap v1857: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:14:30.457 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:14:30.457 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:29 smithi173 bash[58816]: cluster 2023-10-01T11:14:28.492482+0000 mgr.y (mgr.25072) 1718 : cluster 0 pgmap v1857: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:14:30.458 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T11:14:30.458 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T11:14:30.458 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T11:14:30.458 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T11:14:30.458 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T11:14:30.458 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T11:14:30.465 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T11:14:30.513 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T11:14:30.513 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T11:14:30.513 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T11:14:30.513 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T11:14:30.513 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T11:14:30.513 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T11:14:30.513 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T11:14:30.514 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T11:14:30.514 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T11:14:30.537 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:14:30 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:14:30] "GET /metrics HTTP/1.1" 200 36328 "" "Prometheus/2.33.4" 2023-10-01T11:14:30.568 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:30.599 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:30.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:30.822 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:30.923 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:30.959 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:30.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:30.976 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:14:31.077 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:31.097 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:30 smithi078 bash[75299]: audit 2023-10-01T11:14:30.295337+0000 mon.a (mon.0) 2191 : audit 0 from='client.? 172.21.15.78:0/2904099619' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:14:31.098 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:30 smithi078 bash[78274]: audit 2023-10-01T11:14:30.295337+0000 mon.a (mon.0) 2191 : audit 0 from='client.? 172.21.15.78:0/2904099619' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:14:31.129 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:31.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:31.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:14:31.284 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:31.298 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:30 smithi173 bash[58816]: audit 2023-10-01T11:14:30.295337+0000 mon.a (mon.0) 2191 : audit 0 from='client.? 172.21.15.78:0/2904099619' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:14:31.312 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:31.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:31.374 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:14:31.475 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:31.528 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:31.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:31.532 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:31.633 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:31.676 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:31.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:31.679 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:31.779 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:31.814 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:31.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:31.817 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:14:31.918 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:31.993 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:31.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:31.997 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:32.001 INFO:teuthology.orchestra.run.smithi078.stderr:noscrub is unset 2023-10-01T11:14:32.077 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:31 smithi078 bash[78274]: cluster 2023-10-01T11:14:30.493220+0000 mgr.y (mgr.25072) 1719 : cluster 0 pgmap v1858: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:14:32.077 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:31 smithi078 bash[78274]: audit 2023-10-01T11:14:31.093966+0000 mon.a (mon.0) 2192 : audit 1 from='client.? 172.21.15.78:0/3032723147' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T11:14:32.077 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:31 smithi078 bash[75299]: cluster 2023-10-01T11:14:30.493220+0000 mgr.y (mgr.25072) 1719 : cluster 0 pgmap v1858: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:14:32.077 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:31 smithi078 bash[75299]: audit 2023-10-01T11:14:31.093966+0000 mon.a (mon.0) 2192 : audit 1 from='client.? 172.21.15.78:0/3032723147' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T11:14:32.098 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:32.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:31 smithi173 bash[58816]: cluster 2023-10-01T11:14:30.493220+0000 mgr.y (mgr.25072) 1719 : cluster 0 pgmap v1858: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:14:32.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:31 smithi173 bash[58816]: audit 2023-10-01T11:14:31.093966+0000 mon.a (mon.0) 2192 : audit 1 from='client.? 172.21.15.78:0/3032723147' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T11:14:32.144 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:32.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:32.147 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:14:32.248 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:32.304 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:32.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:32.307 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:14:32.408 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:32.445 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:32.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:32.449 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:14:32.494 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T11:14:32.494 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T11:14:32.495 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-01T11:14:32.495 INFO:tasks.thrashosds.thrasher:Setting osd 0 primary_affinity to 0.121242 2023-10-01T11:14:32.495 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 0 0.12124209199891278 2023-10-01T11:14:32.550 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:32.595 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:32.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:32.598 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:14:32.699 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:32.762 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:32.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:32.765 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:14:32.866 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:32.934 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:32.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:32.937 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:33.038 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:33.066 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:33.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:33.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:14:33.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:32 smithi173 bash[58816]: cluster 2023-10-01T11:14:31.984400+0000 mon.a (mon.0) 2193 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:14:33.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:32 smithi173 bash[58816]: cluster 2023-10-01T11:14:31.994474+0000 osd.7 (osd.7) 429 : cluster 3 failed to encode map e381 with expected crc 2023-10-01T11:14:33.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:32 smithi173 bash[58816]: audit 2023-10-01T11:14:31.996058+0000 mon.a (mon.0) 2194 : audit 1 from='client.? 172.21.15.78:0/3032723147' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T11:14:33.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:32 smithi173 bash[58816]: cluster 2023-10-01T11:14:31.996124+0000 mon.a (mon.0) 2195 : cluster 0 osdmap e381: 8 total, 8 up, 8 in 2023-10-01T11:14:33.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:32 smithi173 bash[58816]: audit 2023-10-01T11:14:32.482258+0000 mon.a (mon.0) 2196 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:14:33.172 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:33.184 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:32 smithi078 bash[78274]: cluster 2023-10-01T11:14:31.984400+0000 mon.a (mon.0) 2193 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:14:33.184 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:32 smithi078 bash[78274]: cluster 2023-10-01T11:14:31.994474+0000 osd.7 (osd.7) 429 : cluster 3 failed to encode map e381 with expected crc 2023-10-01T11:14:33.184 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:32 smithi078 bash[78274]: audit 2023-10-01T11:14:31.996058+0000 mon.a (mon.0) 2194 : audit 1 from='client.? 172.21.15.78:0/3032723147' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T11:14:33.184 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:32 smithi078 bash[78274]: cluster 2023-10-01T11:14:31.996124+0000 mon.a (mon.0) 2195 : cluster 0 osdmap e381: 8 total, 8 up, 8 in 2023-10-01T11:14:33.184 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:32 smithi078 bash[78274]: audit 2023-10-01T11:14:32.482258+0000 mon.a (mon.0) 2196 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:14:33.184 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:32 smithi078 bash[75299]: cluster 2023-10-01T11:14:31.984400+0000 mon.a (mon.0) 2193 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:14:33.184 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:32 smithi078 bash[75299]: cluster 2023-10-01T11:14:31.994474+0000 osd.7 (osd.7) 429 : cluster 3 failed to encode map e381 with expected crc 2023-10-01T11:14:33.185 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:32 smithi078 bash[75299]: audit 2023-10-01T11:14:31.996058+0000 mon.a (mon.0) 2194 : audit 1 from='client.? 172.21.15.78:0/3032723147' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T11:14:33.185 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:32 smithi078 bash[75299]: cluster 2023-10-01T11:14:31.996124+0000 mon.a (mon.0) 2195 : cluster 0 osdmap e381: 8 total, 8 up, 8 in 2023-10-01T11:14:33.185 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:32 smithi078 bash[75299]: audit 2023-10-01T11:14:32.482258+0000 mon.a (mon.0) 2196 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:14:33.201 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:33.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:33.204 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:33.305 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:33.328 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:33.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:33.332 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:33.433 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:33.462 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:33.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:33.464 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:14:33.565 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:33.575 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:14:33.593 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:33.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:33.596 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:33.696 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:33.716 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:33.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:33.718 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:33.810 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T11:14:33.819 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:33.880 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:34.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:34.218 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:14:34.319 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:34.332 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:33 smithi173 bash[58816]: cluster 2023-10-01T11:14:32.493719+0000 mgr.y (mgr.25072) 1720 : cluster 0 pgmap v1860: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:14:34.353 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:34.353 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:33 smithi078 bash[75299]: cluster 2023-10-01T11:14:32.493719+0000 mgr.y (mgr.25072) 1720 : cluster 0 pgmap v1860: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:14:34.362 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:33 smithi078 bash[78274]: cluster 2023-10-01T11:14:32.493719+0000 mgr.y (mgr.25072) 1720 : cluster 0 pgmap v1860: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:14:34.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:34.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:14:34.464 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:34.743 DEBUG:teuthology.orchestra.run.smithi078:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-10-01T11:14:34.810 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:34.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:34.813 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:34.913 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:34.940 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:34.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:34.943 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:14:35.044 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:35.075 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:35.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:35.079 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:14:35.153 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T11:14:35.179 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:35.216 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:35.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:35.218 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:14:35.319 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:35.340 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:35.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:35.342 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:14:35.399 INFO:tasks.workunit.client.0.smithi078.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-10-01T11:14:35.442 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:35.475 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:35.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:35.478 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:14:35.579 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:35.588 INFO:tasks.workunit.client.0.smithi078.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-01T11:14:35.588 INFO:tasks.workunit.client.0.smithi078.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-10-01T11:14:35.615 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:35.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:35.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:14:35.719 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:35.756 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:35.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:35.759 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:14:35.860 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:35.886 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:35.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:35.891 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:35.992 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:36.016 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:36.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:36.020 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:36.120 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:36.155 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:36.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:36.158 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:36.259 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:36.295 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:36.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:36.297 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:36.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:36 smithi173 bash[58816]: cluster 2023-10-01T11:14:34.494290+0000 mgr.y (mgr.25072) 1721 : cluster 0 pgmap v1861: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:14:36.398 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:36.429 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:36.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:36.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:14:36.535 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:36.566 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:36.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:36.568 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:14:36.669 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:36.704 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:36.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:36.707 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:14:36.807 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:36.952 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:36 smithi078 bash[75299]: cluster 2023-10-01T11:14:34.494290+0000 mgr.y (mgr.25072) 1721 : cluster 0 pgmap v1861: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:14:36.952 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:36 smithi078 bash[78274]: cluster 2023-10-01T11:14:34.494290+0000 mgr.y (mgr.25072) 1721 : cluster 0 pgmap v1861: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:14:36.977 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:36.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:36.984 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:37.086 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:37.114 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:37.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:37.117 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:37.218 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:37.269 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:37.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:37.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:37.372 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:37.400 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:37.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:37.595 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:14:37.681 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:14:37.698 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:14:37.699 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:37.727 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:37.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:37.728 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:37.829 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:37.873 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:37.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:37.877 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:14:37.978 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:38.205 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:38 smithi173 bash[58816]: cluster 2023-10-01T11:14:36.495093+0000 mgr.y (mgr.25072) 1722 : cluster 0 pgmap v1862: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:14:38.205 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:38 smithi078 bash[75299]: cluster 2023-10-01T11:14:36.495093+0000 mgr.y (mgr.25072) 1722 : cluster 0 pgmap v1862: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:14:38.207 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:38 smithi078 bash[78274]: cluster 2023-10-01T11:14:36.495093+0000 mgr.y (mgr.25072) 1722 : cluster 0 pgmap v1862: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:14:38.229 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:38.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:38.231 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:38.332 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:38.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:38.362 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:38.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:14:38.463 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:38.489 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:38.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:38.492 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:14:38.593 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:38.619 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:38.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:38.622 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:38.723 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:38.756 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:38.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:38.762 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:14:38.863 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:38.899 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:38.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:38.902 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:14:39.003 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:39.034 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:39.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:39.038 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:14:39.139 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:39.166 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:39.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:39.170 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:14:39.270 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:39.303 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:39.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:39.306 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:14:39.407 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:39.440 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:39.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:39.444 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:14:39.544 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:39.568 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:39.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:39.571 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:14:39.672 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:39.707 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:39.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:39.710 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:14:39.811 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:39.835 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:39.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:39.838 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:14:39.938 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:39.974 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:39.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:39.976 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:14:40.078 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:40.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:40 smithi078 bash[75299]: cluster 2023-10-01T11:14:38.495575+0000 mgr.y (mgr.25072) 1723 : cluster 0 pgmap v1863: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:14:40.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:40 smithi078 bash[78274]: cluster 2023-10-01T11:14:38.495575+0000 mgr.y (mgr.25072) 1723 : cluster 0 pgmap v1863: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:14:40.111 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:40.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:40.115 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:14:40.216 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:40.247 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:40.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:40.249 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:14:40.350 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:40.362 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:40 smithi173 bash[58816]: cluster 2023-10-01T11:14:38.495575+0000 mgr.y (mgr.25072) 1723 : cluster 0 pgmap v1863: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:14:40.385 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:40.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:40.387 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:14:40.488 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:40.496 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:14:40 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:14:40] "GET /metrics HTTP/1.1" 200 33966 "" "Prometheus/2.33.4" 2023-10-01T11:14:40.509 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:40.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:40.512 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:14:40.613 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:40.638 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:40.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:40.640 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:40.688 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:14:40.742 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:40.768 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:40.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:40.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:40.872 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:40.907 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:40.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:40.910 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:14:41.011 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:41.042 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:41.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:41.045 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:14:41.145 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:41.175 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:41.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:41.177 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:14:41.278 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:41.312 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:41.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:41.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:14:41.416 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:41.449 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:41.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:41.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:14:41.552 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:41.587 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:41.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:41.590 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:14:41.691 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:41.716 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:41.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:41.718 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:41.819 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:41.851 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:41.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:41.854 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:14:41.955 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:41.985 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:41.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:41.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:14:42.089 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:42.115 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:42.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:42.118 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:14:42.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:42 smithi173 bash[58816]: cluster 2023-10-01T11:14:40.496159+0000 mgr.y (mgr.25072) 1724 : cluster 0 pgmap v1864: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:14:42.218 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:42.246 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:42.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:42.248 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:42.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:42 smithi078 bash[78274]: cluster 2023-10-01T11:14:40.496159+0000 mgr.y (mgr.25072) 1724 : cluster 0 pgmap v1864: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:14:42.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:42 smithi078 bash[75299]: cluster 2023-10-01T11:14:40.496159+0000 mgr.y (mgr.25072) 1724 : cluster 0 pgmap v1864: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:14:42.349 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:42.376 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:42.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:42.378 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:14:42.479 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:42.509 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:42.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:42.513 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:14:42.614 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:42.645 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:42.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:42.648 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:14:42.749 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:42.784 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:42.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:42.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:14:42.888 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:42.922 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:42.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:42.925 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:14:43.026 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:43.047 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:43.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:43.050 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:43.151 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:43.183 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:43.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:43.188 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:14:43.288 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:43.311 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:43.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:43.313 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:14:43.414 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:43.446 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:43.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:43.451 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:14:43.552 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:43.581 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:43.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:43.585 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:14:43.685 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:43.720 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:43.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:43.723 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:14:43.824 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:43.855 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:43.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:43.859 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:14:43.959 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:43.988 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:43.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:43.992 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:14:44.092 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:44.111 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:44.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:44.114 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:14:44.215 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:44.237 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:44.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:44.239 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:14:44.340 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:44.353 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:44 smithi173 bash[58816]: cluster 2023-10-01T11:14:42.496695+0000 mgr.y (mgr.25072) 1725 : cluster 0 pgmap v1865: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:14:44.362 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:44 smithi078 bash[75299]: cluster 2023-10-01T11:14:42.496695+0000 mgr.y (mgr.25072) 1725 : cluster 0 pgmap v1865: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:14:44.362 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:44 smithi078 bash[78274]: cluster 2023-10-01T11:14:42.496695+0000 mgr.y (mgr.25072) 1725 : cluster 0 pgmap v1865: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:14:44.373 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:44.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:44.376 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:14:44.477 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:44.510 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:44.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:44.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:14:44.614 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:44.649 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:44.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:44.651 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:14:44.752 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:44.773 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:44.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:44.775 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:14:44.875 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:44.901 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:44.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:44.903 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:45.004 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:45.025 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:45.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:45.028 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:14:45.129 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:45.149 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:45.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:45.151 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:14:45.252 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:45.273 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:45.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:45.275 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:14:45.376 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:45.408 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:45.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:45.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:14:45.512 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:45.539 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:45.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:45.542 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:45.642 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:45.674 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:45.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:45.677 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:14:45.778 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:45.805 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:45.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:45.808 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:45.909 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:45.931 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:45.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:45.933 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:46.034 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:46.045 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:46 smithi078 bash[75299]: cluster 2023-10-01T11:14:44.497479+0000 mgr.y (mgr.25072) 1726 : cluster 0 pgmap v1866: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:14:46.045 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:46 smithi078 bash[78274]: cluster 2023-10-01T11:14:44.497479+0000 mgr.y (mgr.25072) 1726 : cluster 0 pgmap v1866: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:14:46.060 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:46.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:46.062 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:46.163 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:46.201 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:46.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:46.204 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:14:46.305 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:46.320 INFO:tasks.workunit.client.0.smithi078.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-10-01T11:14:46.333 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:46.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:46.334 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:46.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:46 smithi173 bash[58816]: cluster 2023-10-01T11:14:44.497479+0000 mgr.y (mgr.25072) 1726 : cluster 0 pgmap v1866: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:14:46.435 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:46.468 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:46.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:46.470 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:14:46.571 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:46.599 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:46.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:46.601 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:14:46.702 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:46.725 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:46.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:46.727 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:14:46.780 INFO:tasks.workunit.client.0.smithi078.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-10-01T11:14:46.829 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:46.853 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:46.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:46.856 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:46.919 INFO:tasks.workunit.client.0.smithi078.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-01T11:14:46.920 INFO:tasks.workunit.client.0.smithi078.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-01T11:14:46.920 INFO:tasks.workunit.client.0.smithi078.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-10-01T11:14:46.957 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:46.988 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:46.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:46.990 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:14:47.091 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:47.107 INFO:tasks.workunit.client.0.smithi078.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-01T11:14:47.111 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2023-10-01T11:14:47.111 DEBUG:teuthology.orchestra.run.smithi078:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-10-01T11:14:47.111 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:47.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:47.112 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:14:47.213 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:47.244 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:47.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:47.247 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:14:47.333 INFO:tasks.workunit:Running workunits matching cls/test_cls_rbd.sh on client.0... 2023-10-01T11:14:47.334 INFO:tasks.workunit:Running workunit cls/test_cls_rbd.sh... 2023-10-01T11:14:47.334 DEBUG:teuthology.orchestra.run.smithi078:workunit test cls/test_cls_rbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=pacific TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh 2023-10-01T11:14:47.348 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:47.384 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:47.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:47.386 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:14:47.487 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:47.511 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:47.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:47.513 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:47.614 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:47.648 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:47.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:47.651 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:14:47.656 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2023-10-01T11:14:47.656 INFO:teuthology.orchestra.run.smithi078.stdout: "ops": [], 2023-10-01T11:14:47.657 INFO:teuthology.orchestra.run.smithi078.stdout: "complaint_time": 30, 2023-10-01T11:14:47.657 INFO:teuthology.orchestra.run.smithi078.stdout: "num_blocked_ops": 0 2023-10-01T11:14:47.657 INFO:teuthology.orchestra.run.smithi078.stdout:} 2023-10-01T11:14:47.751 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:47.775 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:47.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:47.777 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:47.877 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:47.906 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:47.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:47.910 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:14:48.011 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:48.031 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:48.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:48.033 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:48.133 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:48.146 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:48 smithi173 bash[58816]: cluster 2023-10-01T11:14:46.498077+0000 mgr.y (mgr.25072) 1727 : cluster 0 pgmap v1867: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:14:48.146 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:48 smithi173 bash[58816]: audit 2023-10-01T11:14:47.482356+0000 mon.a (mon.0) 2197 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:14:48.163 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:48.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:48.167 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:14:48.268 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:48.292 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:48.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:48.295 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:48.304 INFO:tasks.workunit.client.0.smithi078.stdout:Running main() from gmock_main.cc 2023-10-01T11:14:48.304 INFO:tasks.workunit.client.0.smithi078.stdout:[==========] Running 70 tests from 1 test suite. 2023-10-01T11:14:48.304 INFO:tasks.workunit.client.0.smithi078.stdout:[----------] Global test environment set-up. 2023-10-01T11:14:48.304 INFO:tasks.workunit.client.0.smithi078.stdout:[----------] 70 tests from TestClsRbd 2023-10-01T11:14:48.306 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:48 smithi078 bash[75299]: cluster 2023-10-01T11:14:46.498077+0000 mgr.y (mgr.25072) 1727 : cluster 0 pgmap v1867: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:14:48.307 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:48 smithi078 bash[75299]: audit 2023-10-01T11:14:47.482356+0000 mon.a (mon.0) 2197 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:14:48.307 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:48 smithi078 bash[78274]: cluster 2023-10-01T11:14:46.498077+0000 mgr.y (mgr.25072) 1727 : cluster 0 pgmap v1867: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:14:48.307 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:48 smithi078 bash[78274]: audit 2023-10-01T11:14:47.482356+0000 mon.a (mon.0) 2197 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:14:48.396 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:48.423 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:48.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:48.427 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:14:48.527 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:48.549 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:48.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:48.551 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:48.652 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:48.682 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:48.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:48.685 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:14:48.785 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:48.811 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:48.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:48.814 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:48.915 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:48.951 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:48.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:48.955 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:14:49.056 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:49.066 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:49 smithi078 bash[75299]: audit 2023-10-01T11:14:48.325130+0000 mon.a (mon.0) 2198 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T11:14:49.076 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:49.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:49.078 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:49.179 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:49.202 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:49.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:49.203 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:14:49.304 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:49.317 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:49 smithi173 bash[58816]: audit 2023-10-01T11:14:48.325130+0000 mon.a (mon.0) 2198 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T11:14:49.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:49 smithi078 bash[78274]: audit 2023-10-01T11:14:48.325130+0000 mon.a (mon.0) 2198 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T11:14:49.340 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:49.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:49.344 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:14:49.445 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:49.476 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:49.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:49.479 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:14:49.579 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:49.618 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:49.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:49.620 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:14:49.721 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:49.755 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:49.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:49.758 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:14:49.859 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:49.892 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:49.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:49.895 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:14:49.996 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:50.029 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:50.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:50.031 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:50.061 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.get_all_features 2023-10-01T11:14:50.133 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:50.163 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:50.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:50.167 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:14:50.267 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:50.269 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:50 smithi078 bash[78274]: cluster 2023-10-01T11:14:48.498474+0000 mgr.y (mgr.25072) 1728 : cluster 0 pgmap v1868: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:14:50.269 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:50 smithi078 bash[78274]: cluster 2023-10-01T11:14:49.038129+0000 osd.5 (osd.5) 410 : cluster 3 failed to encode map e382 with expected crc 2023-10-01T11:14:50.269 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:50 smithi078 bash[78274]: cluster 2023-10-01T11:14:49.039839+0000 mon.a (mon.0) 2199 : cluster 0 osdmap e382: 8 total, 8 up, 8 in 2023-10-01T11:14:50.269 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:50 smithi078 bash[78274]: cluster 2023-10-01T11:14:49.042992+0000 osd.7 (osd.7) 430 : cluster 3 failed to encode map e382 with expected crc 2023-10-01T11:14:50.269 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:50 smithi078 bash[78274]: cluster 2023-10-01T11:14:49.043087+0000 osd.5 (osd.5) 411 : cluster 3 failed to encode map e382 with expected crc 2023-10-01T11:14:50.269 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:50 smithi078 bash[78274]: cluster 2023-10-01T11:14:49.043521+0000 osd.6 (osd.6) 368 : cluster 3 failed to encode map e382 with expected crc 2023-10-01T11:14:50.269 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:50 smithi078 bash[78274]: cluster 2023-10-01T11:14:49.045831+0000 osd.7 (osd.7) 431 : cluster 3 failed to encode map e382 with expected crc 2023-10-01T11:14:50.269 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:50 smithi078 bash[78274]: cluster 2023-10-01T11:14:49.047234+0000 osd.6 (osd.6) 369 : cluster 3 failed to encode map e382 with expected crc 2023-10-01T11:14:50.269 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:50 smithi078 bash[78274]: cluster 2023-10-01T11:14:49.048404+0000 osd.6 (osd.6) 370 : cluster 3 failed to encode map e382 with expected crc 2023-10-01T11:14:50.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:50 smithi078 bash[78274]: cluster 2023-10-01T11:14:49.050705+0000 osd.6 (osd.6) 371 : cluster 3 failed to encode map e382 with expected crc 2023-10-01T11:14:50.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:50 smithi078 bash[78274]: audit 2023-10-01T11:14:49.065512+0000 mon.c (mon.2) 251 : audit 1 from='client.? 172.21.15.78:0/3518418639' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi078-257551-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-01T11:14:50.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:50 smithi078 bash[78274]: audit 2023-10-01T11:14:49.066096+0000 mon.a (mon.0) 2200 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi078-257551-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-01T11:14:50.270 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:50 smithi078 bash[75299]: cluster 2023-10-01T11:14:48.498474+0000 mgr.y (mgr.25072) 1728 : cluster 0 pgmap v1868: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:14:50.270 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:50 smithi078 bash[75299]: cluster 2023-10-01T11:14:49.038129+0000 osd.5 (osd.5) 410 : cluster 3 failed to encode map e382 with expected crc 2023-10-01T11:14:50.270 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:50 smithi078 bash[75299]: cluster 2023-10-01T11:14:49.039839+0000 mon.a (mon.0) 2199 : cluster 0 osdmap e382: 8 total, 8 up, 8 in 2023-10-01T11:14:50.270 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:50 smithi078 bash[75299]: cluster 2023-10-01T11:14:49.042992+0000 osd.7 (osd.7) 430 : cluster 3 failed to encode map e382 with expected crc 2023-10-01T11:14:50.270 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:50 smithi078 bash[75299]: cluster 2023-10-01T11:14:49.043087+0000 osd.5 (osd.5) 411 : cluster 3 failed to encode map e382 with expected crc 2023-10-01T11:14:50.271 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:50 smithi078 bash[75299]: cluster 2023-10-01T11:14:49.043521+0000 osd.6 (osd.6) 368 : cluster 3 failed to encode map e382 with expected crc 2023-10-01T11:14:50.271 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:50 smithi078 bash[75299]: cluster 2023-10-01T11:14:49.045831+0000 osd.7 (osd.7) 431 : cluster 3 failed to encode map e382 with expected crc 2023-10-01T11:14:50.271 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:50 smithi078 bash[75299]: cluster 2023-10-01T11:14:49.047234+0000 osd.6 (osd.6) 369 : cluster 3 failed to encode map e382 with expected crc 2023-10-01T11:14:50.271 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:50 smithi078 bash[75299]: cluster 2023-10-01T11:14:49.048404+0000 osd.6 (osd.6) 370 : cluster 3 failed to encode map e382 with expected crc 2023-10-01T11:14:50.271 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:50 smithi078 bash[75299]: cluster 2023-10-01T11:14:49.050705+0000 osd.6 (osd.6) 371 : cluster 3 failed to encode map e382 with expected crc 2023-10-01T11:14:50.271 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:50 smithi078 bash[75299]: audit 2023-10-01T11:14:49.065512+0000 mon.c (mon.2) 251 : audit 1 from='client.? 172.21.15.78:0/3518418639' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi078-257551-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-01T11:14:50.271 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:50 smithi078 bash[75299]: audit 2023-10-01T11:14:49.066096+0000 mon.a (mon.0) 2200 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi078-257551-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-01T11:14:50.293 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:50.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:50.295 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:14:50.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:50 smithi173 bash[58816]: cluster 2023-10-01T11:14:48.498474+0000 mgr.y (mgr.25072) 1728 : cluster 0 pgmap v1868: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:14:50.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:50 smithi173 bash[58816]: cluster 2023-10-01T11:14:49.038129+0000 osd.5 (osd.5) 410 : cluster 3 failed to encode map e382 with expected crc 2023-10-01T11:14:50.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:50 smithi173 bash[58816]: cluster 2023-10-01T11:14:49.039839+0000 mon.a (mon.0) 2199 : cluster 0 osdmap e382: 8 total, 8 up, 8 in 2023-10-01T11:14:50.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:50 smithi173 bash[58816]: cluster 2023-10-01T11:14:49.042992+0000 osd.7 (osd.7) 430 : cluster 3 failed to encode map e382 with expected crc 2023-10-01T11:14:50.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:50 smithi173 bash[58816]: cluster 2023-10-01T11:14:49.043087+0000 osd.5 (osd.5) 411 : cluster 3 failed to encode map e382 with expected crc 2023-10-01T11:14:50.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:50 smithi173 bash[58816]: cluster 2023-10-01T11:14:49.043521+0000 osd.6 (osd.6) 368 : cluster 3 failed to encode map e382 with expected crc 2023-10-01T11:14:50.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:50 smithi173 bash[58816]: cluster 2023-10-01T11:14:49.045831+0000 osd.7 (osd.7) 431 : cluster 3 failed to encode map e382 with expected crc 2023-10-01T11:14:50.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:50 smithi173 bash[58816]: cluster 2023-10-01T11:14:49.047234+0000 osd.6 (osd.6) 369 : cluster 3 failed to encode map e382 with expected crc 2023-10-01T11:14:50.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:50 smithi173 bash[58816]: cluster 2023-10-01T11:14:49.048404+0000 osd.6 (osd.6) 370 : cluster 3 failed to encode map e382 with expected crc 2023-10-01T11:14:50.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:50 smithi173 bash[58816]: cluster 2023-10-01T11:14:49.050705+0000 osd.6 (osd.6) 371 : cluster 3 failed to encode map e382 with expected crc 2023-10-01T11:14:50.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:50 smithi173 bash[58816]: audit 2023-10-01T11:14:49.065512+0000 mon.c (mon.2) 251 : audit 1 from='client.? 172.21.15.78:0/3518418639' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi078-257551-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-01T11:14:50.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:50 smithi173 bash[58816]: audit 2023-10-01T11:14:49.066096+0000 mon.a (mon.0) 2200 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi078-257551-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-10-01T11:14:50.396 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:50.400 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-10-01T11:14:50.419 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:50.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:50.422 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:14:50.457 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.get_all_features (396 ms) 2023-10-01T11:14:50.457 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.copyup 2023-10-01T11:14:50.522 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:50.534 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:14:50 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:14:50] "GET /metrics HTTP/1.1" 200 33964 "" "Prometheus/2.33.4" 2023-10-01T11:14:50.550 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:50.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:50.552 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:50.653 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:50.678 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:50.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:50.680 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:14:50.725 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.copyup (268 ms) 2023-10-01T11:14:50.725 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.sparse_copyup 2023-10-01T11:14:50.781 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:50.812 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:50.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:50.816 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:14:50.916 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:50.951 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:50.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:50.954 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:14:51.014 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.sparse_copyup (289 ms) 2023-10-01T11:14:51.014 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.get_and_set_id 2023-10-01T11:14:51.054 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:51.073 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:51.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:51.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:14:51.086 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.get_and_set_id (73 ms) 2023-10-01T11:14:51.086 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.add_remove_child 2023-10-01T11:14:51.099 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.add_remove_child (13 ms) 2023-10-01T11:14:51.100 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.directory_methods 2023-10-01T11:14:51.134 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.directory_methods (34 ms) 2023-10-01T11:14:51.134 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.create 2023-10-01T11:14:51.147 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T11:14:51.168 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T11:14:51.176 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:51.178 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.create (44 ms) 2023-10-01T11:14:51.178 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.get_features 2023-10-01T11:14:51.180 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.get_features (3 ms) 2023-10-01T11:14:51.180 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.get_object_prefix 2023-10-01T11:14:51.182 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.get_object_prefix (2 ms) 2023-10-01T11:14:51.183 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.get_create_timestamp 2023-10-01T11:14:51.185 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.get_create_timestamp (2 ms) 2023-10-01T11:14:51.185 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.get_access_timestamp 2023-10-01T11:14:51.188 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.get_access_timestamp (3 ms) 2023-10-01T11:14:51.188 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.get_modify_timestamp 2023-10-01T11:14:51.190 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.get_modify_timestamp (2 ms) 2023-10-01T11:14:51.190 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.get_data_pool 2023-10-01T11:14:51.192 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T11:14:51.197 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.get_data_pool (7 ms) 2023-10-01T11:14:51.197 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.get_size 2023-10-01T11:14:51.201 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:51.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:51.203 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:14:51.212 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T11:14:51.220 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.get_size (23 ms) 2023-10-01T11:14:51.220 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.set_size 2023-10-01T11:14:51.226 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.set_size (7 ms) 2023-10-01T11:14:51.227 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.protection_status 2023-10-01T11:14:51.236 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T11:14:51.253 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.protection_status (26 ms) 2023-10-01T11:14:51.254 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.snapshot_limits 2023-10-01T11:14:51.260 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T11:14:51.264 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.snapshot_limits (12 ms) 2023-10-01T11:14:51.264 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.parents_v1 2023-10-01T11:14:51.290 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T11:14:51.304 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:51.312 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:51 smithi078 bash[75299]: cluster 2023-10-01T11:14:50.041127+0000 osd.7 (osd.7) 432 : cluster 3 failed to encode map e383 with expected crc 2023-10-01T11:14:51.312 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:51 smithi078 bash[75299]: cluster 2023-10-01T11:14:50.041290+0000 osd.6 (osd.6) 372 : cluster 3 failed to encode map e383 with expected crc 2023-10-01T11:14:51.312 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:51 smithi078 bash[75299]: audit 2023-10-01T11:14:50.044340+0000 mon.a (mon.0) 2201 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi078-257551-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-10-01T11:14:51.312 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:51 smithi078 bash[75299]: cluster 2023-10-01T11:14:50.044565+0000 mon.a (mon.0) 2202 : cluster 0 osdmap e383: 8 total, 8 up, 8 in 2023-10-01T11:14:51.312 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:51 smithi078 bash[75299]: cluster 2023-10-01T11:14:50.048304+0000 osd.6 (osd.6) 373 : cluster 3 failed to encode map e383 with expected crc 2023-10-01T11:14:51.312 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:51 smithi078 bash[75299]: cluster 2023-10-01T11:14:50.048928+0000 osd.5 (osd.5) 412 : cluster 3 failed to encode map e383 with expected crc 2023-10-01T11:14:51.312 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:51 smithi078 bash[78274]: cluster 2023-10-01T11:14:50.041127+0000 osd.7 (osd.7) 432 : cluster 3 failed to encode map e383 with expected crc 2023-10-01T11:14:51.313 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:51 smithi078 bash[78274]: cluster 2023-10-01T11:14:50.041290+0000 osd.6 (osd.6) 372 : cluster 3 failed to encode map e383 with expected crc 2023-10-01T11:14:51.313 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:51 smithi078 bash[78274]: audit 2023-10-01T11:14:50.044340+0000 mon.a (mon.0) 2201 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi078-257551-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-10-01T11:14:51.313 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:51 smithi078 bash[78274]: cluster 2023-10-01T11:14:50.044565+0000 mon.a (mon.0) 2202 : cluster 0 osdmap e383: 8 total, 8 up, 8 in 2023-10-01T11:14:51.313 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:51 smithi078 bash[78274]: cluster 2023-10-01T11:14:50.048304+0000 osd.6 (osd.6) 373 : cluster 3 failed to encode map e383 with expected crc 2023-10-01T11:14:51.313 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:51 smithi078 bash[78274]: cluster 2023-10-01T11:14:50.048928+0000 osd.5 (osd.5) 412 : cluster 3 failed to encode map e383 with expected crc 2023-10-01T11:14:51.313 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T11:14:51.315 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T11:14:51.315 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T11:14:51.315 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T11:14:51.315 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T11:14:51.316 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T11:14:51.316 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T11:14:51.316 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T11:14:51.316 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T11:14:51.324 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:51.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:51.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:51.336 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.parents_v1 (69 ms) 2023-10-01T11:14:51.336 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.parents_v2 2023-10-01T11:14:51.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:51 smithi173 bash[58816]: cluster 2023-10-01T11:14:50.041127+0000 osd.7 (osd.7) 432 : cluster 3 failed to encode map e383 with expected crc 2023-10-01T11:14:51.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:51 smithi173 bash[58816]: cluster 2023-10-01T11:14:50.041290+0000 osd.6 (osd.6) 372 : cluster 3 failed to encode map e383 with expected crc 2023-10-01T11:14:51.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:51 smithi173 bash[58816]: audit 2023-10-01T11:14:50.044340+0000 mon.a (mon.0) 2201 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi078-257551-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-10-01T11:14:51.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:51 smithi173 bash[58816]: cluster 2023-10-01T11:14:50.044565+0000 mon.a (mon.0) 2202 : cluster 0 osdmap e383: 8 total, 8 up, 8 in 2023-10-01T11:14:51.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:51 smithi173 bash[58816]: cluster 2023-10-01T11:14:50.048304+0000 osd.6 (osd.6) 373 : cluster 3 failed to encode map e383 with expected crc 2023-10-01T11:14:51.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:51 smithi173 bash[58816]: cluster 2023-10-01T11:14:50.048928+0000 osd.5 (osd.5) 412 : cluster 3 failed to encode map e383 with expected crc 2023-10-01T11:14:51.387 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.parents_v2 (53 ms) 2023-10-01T11:14:51.387 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.snapshots 2023-10-01T11:14:51.422 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.snapshots (35 ms) 2023-10-01T11:14:51.422 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.snapid_race 2023-10-01T11:14:51.428 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:51.433 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.snapid_race (12 ms) 2023-10-01T11:14:51.434 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.stripingv2 2023-10-01T11:14:51.449 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.stripingv2 (15 ms) 2023-10-01T11:14:51.449 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.object_map_save 2023-10-01T11:14:51.454 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.object_map_save (5 ms) 2023-10-01T11:14:51.454 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.object_map_resize 2023-10-01T11:14:51.457 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:51.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:51.460 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:14:51.463 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.object_map_resize (9 ms) 2023-10-01T11:14:51.463 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.object_map_update 2023-10-01T11:14:51.471 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.object_map_update (8 ms) 2023-10-01T11:14:51.471 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.object_map_load_enoent 2023-10-01T11:14:51.471 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.object_map_load_enoent (1 ms) 2023-10-01T11:14:51.472 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.object_map_snap_add 2023-10-01T11:14:51.480 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.object_map_snap_add (8 ms) 2023-10-01T11:14:51.480 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.object_map_snap_remove 2023-10-01T11:14:51.488 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.object_map_snap_remove (8 ms) 2023-10-01T11:14:51.488 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.flags 2023-10-01T11:14:51.498 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.flags (10 ms) 2023-10-01T11:14:51.498 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.metadata 2023-10-01T11:14:51.530 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.metadata (32 ms) 2023-10-01T11:14:51.530 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.set_features 2023-10-01T11:14:51.545 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.set_features (15 ms) 2023-10-01T11:14:51.545 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.mirror 2023-10-01T11:14:51.562 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:51.593 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:51.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:51.596 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:51.612 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.mirror (67 ms) 2023-10-01T11:14:51.612 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.mirror_image 2023-10-01T11:14:51.642 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.mirror_image (30 ms) 2023-10-01T11:14:51.642 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.mirror_image_status 2023-10-01T11:14:51.697 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:51.727 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:51.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:51.730 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:51.831 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:51.857 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:51.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:51.860 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:14:51.960 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:51.988 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:51.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:51.992 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:14:52.093 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:52.101 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:52 smithi078 bash[78274]: cluster 2023-10-01T11:14:50.499111+0000 mgr.y (mgr.25072) 1729 : cluster 0 pgmap v1871: 105 pgs: 2 creating+activating, 18 creating+peering, 12 unknown, 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:14:52.101 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:52 smithi078 bash[78274]: cluster 2023-10-01T11:14:51.052343+0000 osd.5 (osd.5) 413 : cluster 3 failed to encode map e384 with expected crc 2023-10-01T11:14:52.101 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:52 smithi078 bash[78274]: cluster 2023-10-01T11:14:51.052675+0000 osd.7 (osd.7) 433 : cluster 3 failed to encode map e384 with expected crc 2023-10-01T11:14:52.101 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:52 smithi078 bash[78274]: cluster 2023-10-01T11:14:51.053254+0000 mon.a (mon.0) 2203 : cluster 0 osdmap e384: 8 total, 8 up, 8 in 2023-10-01T11:14:52.101 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:52 smithi078 bash[78274]: audit 2023-10-01T11:14:51.125470+0000 mon.a (mon.0) 2204 : audit 0 from='client.? 172.21.15.78:0/2874740370' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:14:52.102 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:52 smithi078 bash[78274]: cluster 2023-10-01T11:14:51.136891+0000 osd.6 (osd.6) 374 : cluster 3 failed to encode map e384 with expected crc 2023-10-01T11:14:52.102 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:52 smithi078 bash[75299]: cluster 2023-10-01T11:14:50.499111+0000 mgr.y (mgr.25072) 1729 : cluster 0 pgmap v1871: 105 pgs: 2 creating+activating, 18 creating+peering, 12 unknown, 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:14:52.102 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:52 smithi078 bash[75299]: cluster 2023-10-01T11:14:51.052343+0000 osd.5 (osd.5) 413 : cluster 3 failed to encode map e384 with expected crc 2023-10-01T11:14:52.103 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:52 smithi078 bash[75299]: cluster 2023-10-01T11:14:51.052675+0000 osd.7 (osd.7) 433 : cluster 3 failed to encode map e384 with expected crc 2023-10-01T11:14:52.103 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:52 smithi078 bash[75299]: cluster 2023-10-01T11:14:51.053254+0000 mon.a (mon.0) 2203 : cluster 0 osdmap e384: 8 total, 8 up, 8 in 2023-10-01T11:14:52.103 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:52 smithi078 bash[75299]: audit 2023-10-01T11:14:51.125470+0000 mon.a (mon.0) 2204 : audit 0 from='client.? 172.21.15.78:0/2874740370' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:14:52.103 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:52 smithi078 bash[75299]: cluster 2023-10-01T11:14:51.136891+0000 osd.6 (osd.6) 374 : cluster 3 failed to encode map e384 with expected crc 2023-10-01T11:14:52.116 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:52.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:52.118 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:14:52.219 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:52.246 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:52.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:52.248 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:14:52.349 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:52.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:52 smithi173 bash[58816]: cluster 2023-10-01T11:14:50.499111+0000 mgr.y (mgr.25072) 1729 : cluster 0 pgmap v1871: 105 pgs: 2 creating+activating, 18 creating+peering, 12 unknown, 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:14:52.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:52 smithi173 bash[58816]: cluster 2023-10-01T11:14:51.052343+0000 osd.5 (osd.5) 413 : cluster 3 failed to encode map e384 with expected crc 2023-10-01T11:14:52.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:52 smithi173 bash[58816]: cluster 2023-10-01T11:14:51.052675+0000 osd.7 (osd.7) 433 : cluster 3 failed to encode map e384 with expected crc 2023-10-01T11:14:52.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:52 smithi173 bash[58816]: cluster 2023-10-01T11:14:51.053254+0000 mon.a (mon.0) 2203 : cluster 0 osdmap e384: 8 total, 8 up, 8 in 2023-10-01T11:14:52.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:52 smithi173 bash[58816]: audit 2023-10-01T11:14:51.125470+0000 mon.a (mon.0) 2204 : audit 0 from='client.? 172.21.15.78:0/2874740370' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:14:52.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:52 smithi173 bash[58816]: cluster 2023-10-01T11:14:51.136891+0000 osd.6 (osd.6) 374 : cluster 3 failed to encode map e384 with expected crc 2023-10-01T11:14:52.373 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:52.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:52.375 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:52.476 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:52.505 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:52.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:52.507 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:14:52.608 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:52.633 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:52.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:52.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:14:52.736 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:52.760 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:52.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:52.762 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:52.863 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:52.885 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:52.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:52.887 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:14:52.988 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:53.016 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:53.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:53.019 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:53.119 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:53.149 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:53.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:53.152 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:14:53.253 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:53.281 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:53.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:53.284 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:14:53.385 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:53.416 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:53.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:53.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:14:53.519 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:53.545 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:53.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:53.548 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:53.648 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:53.706 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:53.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:53.709 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:14:53.810 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:53.836 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:53.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:53.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:14:53.940 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:53.964 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:53.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:53.967 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:14:54.068 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:54.091 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:54.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:54.094 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:54.195 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:54.221 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:54.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:54.224 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:14:54.325 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:54.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:54 smithi078 bash[75299]: cluster 2023-10-01T11:14:52.499590+0000 mgr.y (mgr.25072) 1730 : cluster 0 pgmap v1873: 105 pgs: 2 creating+activating, 18 creating+peering, 6 unknown, 79 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 5.2 KiB/s rd, 1.3 KiB/s wr, 6 op/s 2023-10-01T11:14:54.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:54 smithi078 bash[78274]: cluster 2023-10-01T11:14:52.499590+0000 mgr.y (mgr.25072) 1730 : cluster 0 pgmap v1873: 105 pgs: 2 creating+activating, 18 creating+peering, 6 unknown, 79 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 5.2 KiB/s rd, 1.3 KiB/s wr, 6 op/s 2023-10-01T11:14:54.347 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:54.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:54.350 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:54.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:54 smithi173 bash[58816]: cluster 2023-10-01T11:14:52.499590+0000 mgr.y (mgr.25072) 1730 : cluster 0 pgmap v1873: 105 pgs: 2 creating+activating, 18 creating+peering, 6 unknown, 79 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 5.2 KiB/s rd, 1.3 KiB/s wr, 6 op/s 2023-10-01T11:14:54.450 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:54.488 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:54.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:54.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:14:54.592 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:54.618 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:54.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:54.620 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:14:54.721 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:54.751 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:54.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:54.753 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:14:54.854 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:54.878 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:54.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:54.882 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:54.983 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:55.007 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:55.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:55.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:14:55.111 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:55.138 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:55.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:55.141 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:14:55.242 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:55.272 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:55.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:55.274 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:14:55.378 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:55.408 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:55.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:55.412 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:14:55.512 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:55.543 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:55.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:55.545 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:55.646 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:55.678 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:55.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:55.680 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:14:55.781 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:55.812 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:55.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:55.815 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:14:55.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:55 smithi078 bash[75299]: cluster 2023-10-01T11:14:54.500310+0000 mgr.y (mgr.25072) 1731 : cluster 0 pgmap v1874: 105 pgs: 2 creating+activating, 103 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 1.8 MiB/s rd, 1017 KiB/s wr, 878 op/s 2023-10-01T11:14:55.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:55 smithi078 bash[75299]: audit 2023-10-01T11:14:54.661075+0000 mon.a (mon.0) 2205 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:14:55.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:55 smithi078 bash[75299]: audit 2023-10-01T11:14:54.669330+0000 mon.a (mon.0) 2206 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:14:55.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:55 smithi078 bash[75299]: audit 2023-10-01T11:14:55.174089+0000 mon.a (mon.0) 2207 : audit 1 from='client.? 172.21.15.78:0/3659941222' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:14:55.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:55 smithi078 bash[75299]: audit 2023-10-01T11:14:55.210439+0000 mon.a (mon.0) 2208 : audit 1 from='client.? 172.21.15.78:0/531234110' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.12124209199891278}]: dispatch 2023-10-01T11:14:55.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:55 smithi078 bash[78274]: cluster 2023-10-01T11:14:54.500310+0000 mgr.y (mgr.25072) 1731 : cluster 0 pgmap v1874: 105 pgs: 2 creating+activating, 103 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 1.8 MiB/s rd, 1017 KiB/s wr, 878 op/s 2023-10-01T11:14:55.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:55 smithi078 bash[78274]: audit 2023-10-01T11:14:54.661075+0000 mon.a (mon.0) 2205 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:14:55.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:55 smithi078 bash[78274]: audit 2023-10-01T11:14:54.669330+0000 mon.a (mon.0) 2206 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:14:55.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:55 smithi078 bash[78274]: audit 2023-10-01T11:14:55.174089+0000 mon.a (mon.0) 2207 : audit 1 from='client.? 172.21.15.78:0/3659941222' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:14:55.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:55 smithi078 bash[78274]: audit 2023-10-01T11:14:55.210439+0000 mon.a (mon.0) 2208 : audit 1 from='client.? 172.21.15.78:0/531234110' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.12124209199891278}]: dispatch 2023-10-01T11:14:55.916 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:55.947 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:55.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:55.950 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:56.051 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:56.081 INFO:teuthology.orchestra.run.smithi078.stderr:nodeep-scrub is unset 2023-10-01T11:14:56.084 INFO:teuthology.orchestra.run.smithi078.stderr:set osd.0 primary-affinity to 0.121242 (879452) 2023-10-01T11:14:56.091 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:56.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:56.094 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:56.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:55 smithi173 bash[58816]: cluster 2023-10-01T11:14:54.500310+0000 mgr.y (mgr.25072) 1731 : cluster 0 pgmap v1874: 105 pgs: 2 creating+activating, 103 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 1.8 MiB/s rd, 1017 KiB/s wr, 878 op/s 2023-10-01T11:14:56.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:55 smithi173 bash[58816]: audit 2023-10-01T11:14:54.661075+0000 mon.a (mon.0) 2205 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:14:56.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:55 smithi173 bash[58816]: audit 2023-10-01T11:14:54.669330+0000 mon.a (mon.0) 2206 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:14:56.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:55 smithi173 bash[58816]: audit 2023-10-01T11:14:55.174089+0000 mon.a (mon.0) 2207 : audit 1 from='client.? 172.21.15.78:0/3659941222' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:14:56.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:55 smithi173 bash[58816]: audit 2023-10-01T11:14:55.210439+0000 mon.a (mon.0) 2208 : audit 1 from='client.? 172.21.15.78:0/531234110' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.12124209199891278}]: dispatch 2023-10-01T11:14:56.121 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.mirror_image_status (4479 ms) 2023-10-01T11:14:56.121 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.mirror_image_map 2023-10-01T11:14:56.194 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:56.219 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:56.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:56.222 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:14:56.266 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.mirror_image_map (145 ms) 2023-10-01T11:14:56.266 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.mirror_instances 2023-10-01T11:14:56.281 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.mirror_instances (15 ms) 2023-10-01T11:14:56.281 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.mirror_snapshot 2023-10-01T11:14:56.299 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.mirror_snapshot (18 ms) 2023-10-01T11:14:56.299 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.group_dir_list 2023-10-01T11:14:56.305 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.group_dir_list (7 ms) 2023-10-01T11:14:56.306 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.group_dir_add 2023-10-01T11:14:56.310 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.group_dir_add (4 ms) 2023-10-01T11:14:56.310 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.dir_add_already_existing 2023-10-01T11:14:56.316 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.dir_add_already_existing (6 ms) 2023-10-01T11:14:56.316 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.group_dir_rename 2023-10-01T11:14:56.323 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:56.327 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.group_dir_rename (11 ms) 2023-10-01T11:14:56.327 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.group_dir_remove 2023-10-01T11:14:56.334 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.group_dir_remove (7 ms) 2023-10-01T11:14:56.334 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.group_dir_remove_missing 2023-10-01T11:14:56.340 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.group_dir_remove_missing (7 ms) 2023-10-01T11:14:56.341 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.group_image_add 2023-10-01T11:14:56.344 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.group_image_add (3 ms) 2023-10-01T11:14:56.344 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.group_image_remove 2023-10-01T11:14:56.350 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.group_image_remove (6 ms) 2023-10-01T11:14:56.350 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.group_image_list 2023-10-01T11:14:56.352 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:56.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:56.355 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:14:56.357 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-10-01T11:14:56.358 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.group_image_list (6 ms) 2023-10-01T11:14:56.358 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.group_image_clean 2023-10-01T11:14:56.362 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.group_image_clean (7 ms) 2023-10-01T11:14:56.362 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.image_group_add 2023-10-01T11:14:56.367 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.image_group_add (5 ms) 2023-10-01T11:14:56.367 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.image_group_remove 2023-10-01T11:14:56.373 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.image_group_remove (5 ms) 2023-10-01T11:14:56.373 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.image_group_get 2023-10-01T11:14:56.378 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.image_group_get (6 ms) 2023-10-01T11:14:56.378 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_name 2023-10-01T11:14:56.381 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.group_snap_set_empty_name (2 ms) 2023-10-01T11:14:56.381 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_id 2023-10-01T11:14:56.384 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.group_snap_set_empty_id (3 ms) 2023-10-01T11:14:56.384 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_id 2023-10-01T11:14:56.389 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_id (5 ms) 2023-10-01T11:14:56.389 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_name 2023-10-01T11:14:56.394 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_name (6 ms) 2023-10-01T11:14:56.394 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.group_snap_set 2023-10-01T11:14:56.398 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.group_snap_set (3 ms) 2023-10-01T11:14:56.398 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.group_snap_list 2023-10-01T11:14:56.404 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.group_snap_list (6 ms) 2023-10-01T11:14:56.404 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.group_snap_list_max_return 2023-10-01T11:14:56.428 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.group_snap_list_max_return (24 ms) 2023-10-01T11:14:56.428 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.group_snap_list_max_read 2023-10-01T11:14:56.455 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:56.481 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:56.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:56.484 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:14:56.585 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:56.610 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:56.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:56.613 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:14:56.713 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:56.742 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:56.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:56.745 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:14:56.755 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:56 smithi078 bash[75299]: cluster 2023-10-01T11:14:56.063224+0000 mon.a (mon.0) 2209 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T11:14:56.755 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:56 smithi078 bash[75299]: cluster 2023-10-01T11:14:56.063289+0000 mon.a (mon.0) 2210 : cluster 1 Cluster is now healthy 2023-10-01T11:14:56.755 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:56 smithi078 bash[75299]: cluster 2023-10-01T11:14:56.068563+0000 osd.7 (osd.7) 434 : cluster 3 failed to encode map e385 with expected crc 2023-10-01T11:14:56.755 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:56 smithi078 bash[75299]: cluster 2023-10-01T11:14:56.068792+0000 osd.5 (osd.5) 414 : cluster 3 failed to encode map e385 with expected crc 2023-10-01T11:14:56.755 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:56 smithi078 bash[75299]: audit 2023-10-01T11:14:56.071431+0000 mon.a (mon.0) 2211 : audit 1 from='client.? 172.21.15.78:0/3659941222' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T11:14:56.755 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:56 smithi078 bash[75299]: audit 2023-10-01T11:14:56.071557+0000 mon.a (mon.0) 2212 : audit 1 from='client.? 172.21.15.78:0/531234110' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.12124209199891278}]': finished 2023-10-01T11:14:56.756 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:56 smithi078 bash[75299]: cluster 2023-10-01T11:14:56.071649+0000 mon.a (mon.0) 2213 : cluster 0 osdmap e385: 8 total, 8 up, 8 in 2023-10-01T11:14:56.756 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:56 smithi078 bash[78274]: cluster 2023-10-01T11:14:56.063224+0000 mon.a (mon.0) 2209 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T11:14:56.756 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:56 smithi078 bash[78274]: cluster 2023-10-01T11:14:56.063289+0000 mon.a (mon.0) 2210 : cluster 1 Cluster is now healthy 2023-10-01T11:14:56.756 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:56 smithi078 bash[78274]: cluster 2023-10-01T11:14:56.068563+0000 osd.7 (osd.7) 434 : cluster 3 failed to encode map e385 with expected crc 2023-10-01T11:14:56.757 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:56 smithi078 bash[78274]: cluster 2023-10-01T11:14:56.068792+0000 osd.5 (osd.5) 414 : cluster 3 failed to encode map e385 with expected crc 2023-10-01T11:14:56.757 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:56 smithi078 bash[78274]: audit 2023-10-01T11:14:56.071431+0000 mon.a (mon.0) 2211 : audit 1 from='client.? 172.21.15.78:0/3659941222' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T11:14:56.757 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:56 smithi078 bash[78274]: audit 2023-10-01T11:14:56.071557+0000 mon.a (mon.0) 2212 : audit 1 from='client.? 172.21.15.78:0/531234110' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.12124209199891278}]': finished 2023-10-01T11:14:56.757 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:56 smithi078 bash[78274]: cluster 2023-10-01T11:14:56.071649+0000 mon.a (mon.0) 2213 : cluster 0 osdmap e385: 8 total, 8 up, 8 in 2023-10-01T11:14:56.767 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.group_snap_list_max_read (339 ms) 2023-10-01T11:14:56.767 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.group_snap_remove 2023-10-01T11:14:56.774 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.group_snap_remove (7 ms) 2023-10-01T11:14:56.774 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.group_snap_get_by_id 2023-10-01T11:14:56.777 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.group_snap_get_by_id (4 ms) 2023-10-01T11:14:56.777 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.trash_methods 2023-10-01T11:14:56.792 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.trash_methods (14 ms) 2023-10-01T11:14:56.792 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.op_features 2023-10-01T11:14:56.803 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.op_features (11 ms) 2023-10-01T11:14:56.803 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.clone_parent 2023-10-01T11:14:56.832 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.clone_parent (29 ms) 2023-10-01T11:14:56.832 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.clone_parent_ns 2023-10-01T11:14:56.846 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:56.850 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.clone_parent_ns (18 ms) 2023-10-01T11:14:56.850 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.clone_child 2023-10-01T11:14:56.869 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.clone_child (19 ms) 2023-10-01T11:14:56.869 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.namespace_methods 2023-10-01T11:14:56.875 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:56.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:56.878 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:14:56.882 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.namespace_methods (13 ms) 2023-10-01T11:14:56.882 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.migration 2023-10-01T11:14:56.898 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.migration (16 ms) 2023-10-01T11:14:56.898 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.migration_v1 2023-10-01T11:14:56.917 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.migration_v1 (19 ms) 2023-10-01T11:14:56.917 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.assert_snapc_seq 2023-10-01T11:14:56.979 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:57.005 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:57.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:57.008 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:14:57.109 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:57.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:56 smithi173 bash[58816]: cluster 2023-10-01T11:14:56.063224+0000 mon.a (mon.0) 2209 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T11:14:57.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:56 smithi173 bash[58816]: cluster 2023-10-01T11:14:56.063289+0000 mon.a (mon.0) 2210 : cluster 1 Cluster is now healthy 2023-10-01T11:14:57.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:56 smithi173 bash[58816]: cluster 2023-10-01T11:14:56.068563+0000 osd.7 (osd.7) 434 : cluster 3 failed to encode map e385 with expected crc 2023-10-01T11:14:57.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:56 smithi173 bash[58816]: cluster 2023-10-01T11:14:56.068792+0000 osd.5 (osd.5) 414 : cluster 3 failed to encode map e385 with expected crc 2023-10-01T11:14:57.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:56 smithi173 bash[58816]: audit 2023-10-01T11:14:56.071431+0000 mon.a (mon.0) 2211 : audit 1 from='client.? 172.21.15.78:0/3659941222' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T11:14:57.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:56 smithi173 bash[58816]: audit 2023-10-01T11:14:56.071557+0000 mon.a (mon.0) 2212 : audit 1 from='client.? 172.21.15.78:0/531234110' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.12124209199891278}]': finished 2023-10-01T11:14:57.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:56 smithi173 bash[58816]: cluster 2023-10-01T11:14:56.071649+0000 mon.a (mon.0) 2213 : cluster 0 osdmap e385: 8 total, 8 up, 8 in 2023-10-01T11:14:57.137 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:57.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:57.140 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:14:57.241 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:57.269 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:14:57.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:57.273 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:14:57.373 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:57.407 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:57.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:57.410 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:14:57.511 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:57.541 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:57.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:57.545 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:14:57.645 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:57.680 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:57.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:57.683 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:14:57.784 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:57.813 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:57.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:57.817 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:57.917 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:57.930 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:57 smithi173 bash[58816]: cluster 2023-10-01T11:14:56.146460+0000 osd.6 (osd.6) 375 : cluster 3 failed to encode map e385 with expected crc 2023-10-01T11:14:57.930 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:57 smithi173 bash[58816]: cluster 2023-10-01T11:14:56.500771+0000 mgr.y (mgr.25072) 1732 : cluster 0 pgmap v1876: 105 pgs: 105 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 1.7 MiB/s rd, 947 KiB/s wr, 840 op/s 2023-10-01T11:14:57.931 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:57 smithi173 bash[58816]: cluster 2023-10-01T11:14:57.077324+0000 osd.7 (osd.7) 435 : cluster 3 failed to encode map e386 with expected crc 2023-10-01T11:14:57.931 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:57 smithi173 bash[58816]: cluster 2023-10-01T11:14:57.077621+0000 mon.a (mon.0) 2214 : cluster 0 osdmap e386: 8 total, 8 up, 8 in 2023-10-01T11:14:57.949 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:14:57.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:57.952 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:14:58.000 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:57 smithi078 bash[75299]: cluster 2023-10-01T11:14:56.146460+0000 osd.6 (osd.6) 375 : cluster 3 failed to encode map e385 with expected crc 2023-10-01T11:14:58.001 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:57 smithi078 bash[75299]: cluster 2023-10-01T11:14:56.500771+0000 mgr.y (mgr.25072) 1732 : cluster 0 pgmap v1876: 105 pgs: 105 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 1.7 MiB/s rd, 947 KiB/s wr, 840 op/s 2023-10-01T11:14:58.001 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:57 smithi078 bash[75299]: cluster 2023-10-01T11:14:57.077324+0000 osd.7 (osd.7) 435 : cluster 3 failed to encode map e386 with expected crc 2023-10-01T11:14:58.001 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:57 smithi078 bash[75299]: cluster 2023-10-01T11:14:57.077621+0000 mon.a (mon.0) 2214 : cluster 0 osdmap e386: 8 total, 8 up, 8 in 2023-10-01T11:14:58.001 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:57 smithi078 bash[78274]: cluster 2023-10-01T11:14:56.146460+0000 osd.6 (osd.6) 375 : cluster 3 failed to encode map e385 with expected crc 2023-10-01T11:14:58.001 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:57 smithi078 bash[78274]: cluster 2023-10-01T11:14:56.500771+0000 mgr.y (mgr.25072) 1732 : cluster 0 pgmap v1876: 105 pgs: 105 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 1.7 MiB/s rd, 947 KiB/s wr, 840 op/s 2023-10-01T11:14:58.001 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:57 smithi078 bash[78274]: cluster 2023-10-01T11:14:57.077324+0000 osd.7 (osd.7) 435 : cluster 3 failed to encode map e386 with expected crc 2023-10-01T11:14:58.001 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:57 smithi078 bash[78274]: cluster 2023-10-01T11:14:57.077621+0000 mon.a (mon.0) 2214 : cluster 0 osdmap e386: 8 total, 8 up, 8 in 2023-10-01T11:14:58.053 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:58.089 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.assert_snapc_seq (1172 ms) 2023-10-01T11:14:58.089 INFO:tasks.workunit.client.0.smithi078.stdout:[ RUN ] TestClsRbd.sparsify 2023-10-01T11:14:58.092 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:58.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:58.095 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:58.140 INFO:tasks.workunit.client.0.smithi078.stdout:[ OK ] TestClsRbd.sparsify (51 ms) 2023-10-01T11:14:58.197 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:58.227 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:58.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:58.229 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:58.330 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:58.361 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:58.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:58.364 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:58.464 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:58.493 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:14:58.493 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:58.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:58.497 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:58.598 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:58.629 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:14:58.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:58.633 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:14:58.734 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:58.758 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:58.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:58.760 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:58.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:58 smithi078 bash[75299]: cluster 2023-10-01T11:14:57.447674+0000 osd.5 (osd.5) 415 : cluster 3 failed to encode map e386 with expected crc 2023-10-01T11:14:58.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:58 smithi078 bash[75299]: cluster 2023-10-01T11:14:57.447797+0000 osd.6 (osd.6) 376 : cluster 3 failed to encode map e386 with expected crc 2023-10-01T11:14:58.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:58 smithi078 bash[75299]: cluster 2023-10-01T11:14:58.087305+0000 mon.a (mon.0) 2215 : cluster 0 osdmap e387: 8 total, 8 up, 8 in 2023-10-01T11:14:58.832 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:58 smithi078 bash[75299]: cluster 2023-10-01T11:14:58.447263+0000 osd.7 (osd.7) 436 : cluster 3 failed to encode map e387 with expected crc 2023-10-01T11:14:58.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:58 smithi078 bash[78274]: cluster 2023-10-01T11:14:57.447674+0000 osd.5 (osd.5) 415 : cluster 3 failed to encode map e386 with expected crc 2023-10-01T11:14:58.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:58 smithi078 bash[78274]: cluster 2023-10-01T11:14:57.447797+0000 osd.6 (osd.6) 376 : cluster 3 failed to encode map e386 with expected crc 2023-10-01T11:14:58.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:58 smithi078 bash[78274]: cluster 2023-10-01T11:14:58.087305+0000 mon.a (mon.0) 2215 : cluster 0 osdmap e387: 8 total, 8 up, 8 in 2023-10-01T11:14:58.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:58 smithi078 bash[78274]: cluster 2023-10-01T11:14:58.447263+0000 osd.7 (osd.7) 436 : cluster 3 failed to encode map e387 with expected crc 2023-10-01T11:14:58.862 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:58.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:58 smithi173 bash[58816]: cluster 2023-10-01T11:14:57.447674+0000 osd.5 (osd.5) 415 : cluster 3 failed to encode map e386 with expected crc 2023-10-01T11:14:58.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:58 smithi173 bash[58816]: cluster 2023-10-01T11:14:57.447797+0000 osd.6 (osd.6) 376 : cluster 3 failed to encode map e386 with expected crc 2023-10-01T11:14:58.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:58 smithi173 bash[58816]: cluster 2023-10-01T11:14:58.087305+0000 mon.a (mon.0) 2215 : cluster 0 osdmap e387: 8 total, 8 up, 8 in 2023-10-01T11:14:58.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:58 smithi173 bash[58816]: cluster 2023-10-01T11:14:58.447263+0000 osd.7 (osd.7) 436 : cluster 3 failed to encode map e387 with expected crc 2023-10-01T11:14:58.887 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:14:58.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:58.890 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:14:58.991 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:59.013 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:14:59.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:59.018 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:14:59.096 INFO:tasks.workunit.client.0.smithi078.stdout:[----------] 70 tests from TestClsRbd (8079 ms total) 2023-10-01T11:14:59.096 INFO:tasks.workunit.client.0.smithi078.stdout: 2023-10-01T11:14:59.096 INFO:tasks.workunit.client.0.smithi078.stdout:[----------] Global test environment tear-down 2023-10-01T11:14:59.096 INFO:tasks.workunit.client.0.smithi078.stdout:[==========] 70 tests from 1 test suite ran. (10792 ms total) 2023-10-01T11:14:59.096 INFO:tasks.workunit.client.0.smithi078.stdout:[ PASSED ] 70 tests. 2023-10-01T11:14:59.098 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-10-01T11:14:59.099 DEBUG:teuthology.orchestra.run.smithi078:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-10-01T11:14:59.119 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:59.135 INFO:tasks.workunit:Stopping ['cls/test_cls_rbd.sh'] on client.0... 2023-10-01T11:14:59.135 DEBUG:teuthology.orchestra.run.smithi078:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-10-01T11:14:59.151 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:59.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:59.154 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:14:59.255 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:59.290 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:59.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:59.293 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:14:59.394 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:59.415 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:59.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:59.417 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:59.518 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:59.537 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:14:59.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:59.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:14:59.640 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:59.673 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:14:59.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:59.677 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:14:59.777 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:59.811 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:59.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:59.814 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:14:59.915 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:59.927 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:59 smithi173 bash[58816]: cluster 2023-10-01T11:14:58.090798+0000 osd.6 (osd.6) 377 : cluster 3 failed to encode map e387 with expected crc 2023-10-01T11:14:59.927 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:59 smithi173 bash[58816]: cluster 2023-10-01T11:14:58.447201+0000 osd.5 (osd.5) 416 : cluster 3 failed to encode map e387 with expected crc 2023-10-01T11:14:59.927 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:59 smithi173 bash[58816]: cluster 2023-10-01T11:14:58.501452+0000 mgr.y (mgr.25072) 1733 : cluster 0 pgmap v1879: 105 pgs: 105 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 1.9 MiB/s rd, 1019 KiB/s wr, 898 op/s 2023-10-01T11:14:59.927 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:59 smithi173 bash[58816]: cluster 2023-10-01T11:14:59.087826+0000 osd.7 (osd.7) 437 : cluster 3 failed to encode map e388 with expected crc 2023-10-01T11:14:59.927 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:59 smithi173 bash[58816]: cluster 2023-10-01T11:14:59.090803+0000 mon.a (mon.0) 2216 : cluster 0 osdmap e388: 8 total, 8 up, 8 in 2023-10-01T11:14:59.927 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:59 smithi173 bash[58816]: audit 2023-10-01T11:14:59.478234+0000 mon.a (mon.0) 2217 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:14:59.927 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:14:59 smithi173 bash[58816]: audit 2023-10-01T11:14:59.483654+0000 mon.a (mon.0) 2218 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:14:59.945 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:14:59.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:14:59.949 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:15:00.003 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd set noscrub 2023-10-01T11:15:00.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:59 smithi078 bash[78274]: cluster 2023-10-01T11:14:58.090798+0000 osd.6 (osd.6) 377 : cluster 3 failed to encode map e387 with expected crc 2023-10-01T11:15:00.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:59 smithi078 bash[78274]: cluster 2023-10-01T11:14:58.447201+0000 osd.5 (osd.5) 416 : cluster 3 failed to encode map e387 with expected crc 2023-10-01T11:15:00.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:59 smithi078 bash[78274]: cluster 2023-10-01T11:14:58.501452+0000 mgr.y (mgr.25072) 1733 : cluster 0 pgmap v1879: 105 pgs: 105 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 1.9 MiB/s rd, 1019 KiB/s wr, 898 op/s 2023-10-01T11:15:00.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:59 smithi078 bash[78274]: cluster 2023-10-01T11:14:59.087826+0000 osd.7 (osd.7) 437 : cluster 3 failed to encode map e388 with expected crc 2023-10-01T11:15:00.012 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:59 smithi078 bash[78274]: cluster 2023-10-01T11:14:59.090803+0000 mon.a (mon.0) 2216 : cluster 0 osdmap e388: 8 total, 8 up, 8 in 2023-10-01T11:15:00.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:59 smithi078 bash[78274]: audit 2023-10-01T11:14:59.478234+0000 mon.a (mon.0) 2217 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:15:00.013 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:14:59 smithi078 bash[78274]: audit 2023-10-01T11:14:59.483654+0000 mon.a (mon.0) 2218 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:15:00.013 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:59 smithi078 bash[75299]: cluster 2023-10-01T11:14:58.090798+0000 osd.6 (osd.6) 377 : cluster 3 failed to encode map e387 with expected crc 2023-10-01T11:15:00.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:59 smithi078 bash[75299]: cluster 2023-10-01T11:14:58.447201+0000 osd.5 (osd.5) 416 : cluster 3 failed to encode map e387 with expected crc 2023-10-01T11:15:00.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:59 smithi078 bash[75299]: cluster 2023-10-01T11:14:58.501452+0000 mgr.y (mgr.25072) 1733 : cluster 0 pgmap v1879: 105 pgs: 105 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 1.9 MiB/s rd, 1019 KiB/s wr, 898 op/s 2023-10-01T11:15:00.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:59 smithi078 bash[75299]: cluster 2023-10-01T11:14:59.087826+0000 osd.7 (osd.7) 437 : cluster 3 failed to encode map e388 with expected crc 2023-10-01T11:15:00.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:59 smithi078 bash[75299]: cluster 2023-10-01T11:14:59.090803+0000 mon.a (mon.0) 2216 : cluster 0 osdmap e388: 8 total, 8 up, 8 in 2023-10-01T11:15:00.014 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:59 smithi078 bash[75299]: audit 2023-10-01T11:14:59.478234+0000 mon.a (mon.0) 2217 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:15:00.015 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:14:59 smithi078 bash[75299]: audit 2023-10-01T11:14:59.483654+0000 mon.a (mon.0) 2218 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:15:00.049 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:00.070 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:00.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:00.074 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:15:00.175 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:00.180 DEBUG:teuthology.parallel:result is None 2023-10-01T11:15:00.180 DEBUG:teuthology.orchestra.run.smithi078:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-10-01T11:15:00.199 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:00.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:00.201 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:15:00.207 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-10-01T11:15:00.207 DEBUG:teuthology.orchestra.run.smithi078:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-10-01T11:15:00.278 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-10-01T11:15:00.278 INFO:teuthology.task.sequential:In sequential, running task print... 2023-10-01T11:15:00.278 INFO:teuthology.task.print:**** done cls/test_cls_rbd.sh 5-workload 2023-10-01T11:15:00.278 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-10-01T11:15:00.279 INFO:tasks.workunit:Pulling workunits from ref pacific 2023-10-01T11:15:00.279 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-10-01T11:15:00.279 DEBUG:teuthology.orchestra.run.smithi078:> stat -- /home/ubuntu/cephtest/mnt.0 2023-10-01T11:15:00.302 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:00.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:00.329 INFO:teuthology.orchestra.run.smithi078.stderr:stat: cannot stat '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-10-01T11:15:00.329 DEBUG:teuthology.orchestra.run.smithi078:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-10-01T11:15:00.347 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:00.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:00.350 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:15:00.351 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-10-01T11:15:00.352 DEBUG:teuthology.orchestra.run.smithi078:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-10-01T11:15:00.399 INFO:tasks.workunit:timeout=3h 2023-10-01T11:15:00.399 INFO:tasks.workunit:cleanup=True 2023-10-01T11:15:00.399 DEBUG:teuthology.orchestra.run.smithi078:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch pacific https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-10-01T11:15:00.451 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:00.453 INFO:tasks.workunit.client.0.smithi078.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-10-01T11:15:00.477 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:00.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:00.480 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:15:00.581 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:00.582 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:15:00 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:15:00] "GET /metrics HTTP/1.1" 200 33964 "" "Prometheus/2.33.4" 2023-10-01T11:15:00.612 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:00.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:00.615 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:15:00.716 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:00.750 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:00.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:00.752 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:15:00.853 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:15:00.867 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:00 smithi078 bash[78274]: audit 2023-10-01T11:14:59.901888+0000 mon.a (mon.0) 2219 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T11:15:00.868 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:00 smithi078 bash[78274]: audit 2023-10-01T11:14:59.902906+0000 mon.a (mon.0) 2220 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T11:15:00.868 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:00 smithi078 bash[78274]: audit 2023-10-01T11:14:59.910645+0000 mon.a (mon.0) 2221 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:15:00.868 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:00 smithi078 bash[75299]: audit 2023-10-01T11:14:59.901888+0000 mon.a (mon.0) 2219 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T11:15:00.868 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:00 smithi078 bash[75299]: audit 2023-10-01T11:14:59.902906+0000 mon.a (mon.0) 2220 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T11:15:00.868 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:00 smithi078 bash[75299]: audit 2023-10-01T11:14:59.910645+0000 mon.a (mon.0) 2221 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:15:00.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:00 smithi173 bash[58816]: audit 2023-10-01T11:14:59.901888+0000 mon.a (mon.0) 2219 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T11:15:00.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:00 smithi173 bash[58816]: audit 2023-10-01T11:14:59.902906+0000 mon.a (mon.0) 2220 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T11:15:00.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:00 smithi173 bash[58816]: audit 2023-10-01T11:14:59.910645+0000 mon.a (mon.0) 2221 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:15:00.880 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:15:00.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:00.883 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:15:00.984 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:01.016 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:01.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:01.020 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:15:01.120 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:01.145 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:01.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:01.147 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:15:01.248 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:01.275 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:01.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:01.278 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:15:01.379 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:01.407 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:01.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:01.409 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:15:01.511 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:01.545 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:01.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:01.548 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:15:01.648 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:01.683 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:01.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:01.685 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:15:01.786 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:01.821 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:01.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:01.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:15:01.925 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:01.958 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:01.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:01.962 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:15:02.063 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:02.074 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:01 smithi173 bash[58816]: cluster 2023-10-01T11:15:00.502087+0000 mgr.y (mgr.25072) 1734 : cluster 0 pgmap v1881: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:15:02.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:01 smithi078 bash[75299]: cluster 2023-10-01T11:15:00.502087+0000 mgr.y (mgr.25072) 1734 : cluster 0 pgmap v1881: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:15:02.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:01 smithi078 bash[78274]: cluster 2023-10-01T11:15:00.502087+0000 mgr.y (mgr.25072) 1734 : cluster 0 pgmap v1881: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:15:02.093 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:02.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:02.095 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:15:02.196 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:02.226 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:02.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:02.228 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:15:02.330 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:02.359 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:02.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:02.361 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:15:02.461 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:02.492 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:02.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:02.495 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:15:02.545 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:15:02.596 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:02.629 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:02.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:02.631 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:15:02.732 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:02.759 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:02.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:02.761 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:15:02.862 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:02.894 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:02.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:02.897 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:15:02.998 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:03.011 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T11:15:03.011 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T11:15:03.011 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-10-01T11:15:03.011 INFO:tasks.thrashosds.thrasher:Setting osd 1 primary_affinity to 0.099957 2023-10-01T11:15:03.012 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 1 0.09995667370632022 2023-10-01T11:15:03.024 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:03.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:03.027 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:15:03.127 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:03.151 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:03.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:03.155 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:15:03.256 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:03.290 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:03.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:03.292 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:15:03.393 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:03.424 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:03.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:03.428 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:15:03.529 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:03.542 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:03 smithi173 bash[58816]: audit 2023-10-01T11:15:02.489531+0000 mon.a (mon.0) 2222 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:15:03.542 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:03 smithi173 bash[58816]: audit 2023-10-01T11:15:02.491136+0000 mon.a (mon.0) 2223 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:15:03.543 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:03 smithi173 bash[58816]: cluster 2023-10-01T11:15:02.502525+0000 mgr.y (mgr.25072) 1735 : cluster 0 pgmap v1882: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:15:03.543 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:03 smithi173 bash[58816]: audit 2023-10-01T11:15:03.422630+0000 mon.a (mon.0) 2224 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:15:03.560 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:03.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:03.564 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:15:03.665 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:03.684 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:03.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:03.686 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:15:03.787 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:03.795 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:03 smithi078 bash[75299]: audit 2023-10-01T11:15:02.489531+0000 mon.a (mon.0) 2222 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:15:03.796 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:03 smithi078 bash[75299]: audit 2023-10-01T11:15:02.491136+0000 mon.a (mon.0) 2223 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:15:03.796 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:03 smithi078 bash[75299]: cluster 2023-10-01T11:15:02.502525+0000 mgr.y (mgr.25072) 1735 : cluster 0 pgmap v1882: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:15:03.796 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:03 smithi078 bash[75299]: audit 2023-10-01T11:15:03.422630+0000 mon.a (mon.0) 2224 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:15:03.796 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:03 smithi078 bash[78274]: audit 2023-10-01T11:15:02.489531+0000 mon.a (mon.0) 2222 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:15:03.796 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:03 smithi078 bash[78274]: audit 2023-10-01T11:15:02.491136+0000 mon.a (mon.0) 2223 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:15:03.796 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:03 smithi078 bash[78274]: cluster 2023-10-01T11:15:02.502525+0000 mgr.y (mgr.25072) 1735 : cluster 0 pgmap v1882: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:15:03.796 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:03 smithi078 bash[78274]: audit 2023-10-01T11:15:03.422630+0000 mon.a (mon.0) 2224 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:15:03.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:03.812 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:03.812 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:15:03.913 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:03.941 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:03.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:03.943 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:15:04.044 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:04.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:04.077 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:04.077 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:15:04.178 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:04.212 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:04.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:04.215 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:15:04.316 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:04.339 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:04.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:04.342 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:15:04.442 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:04.470 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:04.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:04.473 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:15:04.574 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:04.575 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:15:04.612 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:04.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:04.616 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:15:04.627 INFO:teuthology.orchestra.run.smithi078.stdout:ERROR: (22) Invalid argument 2023-10-01T11:15:04.627 INFO:teuthology.orchestra.run.smithi078.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T11:15:04.717 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:04.753 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:04.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:04.759 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:15:04.860 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:04.893 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:04.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:04.896 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:15:04.997 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:05.026 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:05.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:05.028 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:15:05.130 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:05.166 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:05.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:05.168 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:15:05.269 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:05.307 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:05.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:05.311 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:15:05.411 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:05.439 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:05.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:05.442 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:15:05.543 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:05.576 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:05.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:05.578 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:15:05.679 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:05.695 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:05 smithi078 bash[78274]: cluster 2023-10-01T11:15:04.503011+0000 mgr.y (mgr.25072) 1736 : cluster 0 pgmap v1883: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:15:05.695 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:05 smithi078 bash[75299]: cluster 2023-10-01T11:15:04.503011+0000 mgr.y (mgr.25072) 1736 : cluster 0 pgmap v1883: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:15:05.710 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:05.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:05.712 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:15:05.813 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:15:05.839 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:15:05.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:05.842 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:15:05.869 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:05 smithi173 bash[58816]: cluster 2023-10-01T11:15:04.503011+0000 mgr.y (mgr.25072) 1736 : cluster 0 pgmap v1883: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:15:05.945 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:05.968 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:05.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:05.970 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:15:06.071 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:06.105 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:06.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:06.108 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:15:06.209 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:06.236 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:06.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:06.238 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:15:06.339 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:15:06.373 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:15:06.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:06.376 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:15:06.476 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:06.505 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:06.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:06.507 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:15:06.609 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:06.643 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:06.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:06.647 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:15:06.748 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:06.776 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:06.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:06.779 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:15:06.822 DEBUG:teuthology.orchestra.run.smithi078:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-10-01T11:15:06.829 INFO:tasks.workunit.client.0.smithi078.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-10-01T11:15:06.830 INFO:tasks.workunit.client.0.smithi078.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-01T11:15:06.831 INFO:tasks.workunit.client.0.smithi078.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-10-01T11:15:06.879 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:06.911 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:06.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:06.914 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:15:07.015 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:07.041 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:07.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:07.044 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:15:07.145 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:07.178 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:07.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:07.181 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:15:07.200 INFO:tasks.workunit.client.0.smithi078.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-10-01T11:15:07.275 INFO:tasks.workunit.client.0.smithi078.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-10-01T11:15:07.282 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:07.302 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:07.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:07.305 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:15:07.343 INFO:tasks.workunit.client.0.smithi078.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-10-01T11:15:07.345 INFO:tasks.workunit.client.0.smithi078.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-01T11:15:07.345 INFO:tasks.workunit.client.0.smithi078.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-10-01T11:15:07.401 INFO:tasks.workunit.client.0.smithi078.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-10-01T11:15:07.406 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:07.407 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2023-10-01T11:15:07.407 DEBUG:teuthology.orchestra.run.smithi078:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-10-01T11:15:07.461 INFO:tasks.workunit:Running workunits matching rbd/import_export.sh on client.0... 2023-10-01T11:15:07.462 INFO:tasks.workunit:Running workunit rbd/import_export.sh... 2023-10-01T11:15:07.462 DEBUG:teuthology.orchestra.run.smithi078:workunit test rbd/import_export.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=pacific TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_CREATE_ARGS=--new-format adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/import_export.sh 2023-10-01T11:15:07.484 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:07.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:07.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:15:07.491 INFO:tasks.workunit.client.0.smithi078.stderr:+ export RBD_FORCE_ALLOW_V1=1 2023-10-01T11:15:07.492 INFO:tasks.workunit.client.0.smithi078.stderr:+ TMPDIR=/tmp/rbd_import_export_259593 2023-10-01T11:15:07.492 INFO:tasks.workunit.client.0.smithi078.stderr:+ rm -rf /tmp/rbd_import_export_259593 2023-10-01T11:15:07.492 INFO:tasks.workunit.client.0.smithi078.stderr:+ mkdir /tmp/rbd_import_export_259593 2023-10-01T11:15:07.494 INFO:tasks.workunit.client.0.smithi078.stderr:+ trap rm -rf /tmp/rbd_import_export_259593 INT TERM EXIT 2023-10-01T11:15:07.494 INFO:tasks.workunit.client.0.smithi078.stderr:+ mkdir foo.259593 2023-10-01T11:15:07.496 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd import foo.259593 foo.dir 2023-10-01T11:15:07.587 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:15:07.615 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:15:07.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:07.618 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:15:07.719 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:07.747 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:07.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:07.750 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:15:07.831 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:07 smithi078 bash[75299]: cluster 2023-10-01T11:15:06.503455+0000 mgr.y (mgr.25072) 1737 : cluster 0 pgmap v1884: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:15:07.832 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:07 smithi078 bash[78274]: cluster 2023-10-01T11:15:06.503455+0000 mgr.y (mgr.25072) 1737 : cluster 0 pgmap v1884: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:15:07.852 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:07.864 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:07 smithi173 bash[58816]: cluster 2023-10-01T11:15:06.503455+0000 mgr.y (mgr.25072) 1737 : cluster 0 pgmap v1884: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:15:07.886 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:07.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:07.888 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:15:07.989 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:08.020 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:08.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:08.023 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:15:08.124 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:08.154 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:08.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:08.157 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:15:08.258 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:08.295 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:08.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:08.298 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:15:08.398 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:08.435 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:08.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:08.438 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:15:08.539 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:08.566 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:08.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:08.570 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:15:08.670 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:15:08.698 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:15:08.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:08.702 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:15:08.802 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:08.834 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:08.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:08.836 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:15:08.937 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:08.966 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:08.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:08.969 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:15:09.069 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:09.100 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:09.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:09.103 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:15:09.203 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:09.228 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:09.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:09.231 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:15:09.331 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:15:09.359 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:15:09.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:09.362 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:15:09.463 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:09.495 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:09.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:09.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:15:09.600 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:09.624 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:09.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:09.627 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:15:09.728 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:09.762 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:09.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:09.766 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:15:09.849 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-10-01T11:15:09.866 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:15:09.918 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:15:09.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:09.920 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:15:10.021 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:15:10.042 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:15:10.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:10.044 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:15:10.113 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:09 smithi078 bash[75299]: cluster 2023-10-01T11:15:08.504075+0000 mgr.y (mgr.25072) 1738 : cluster 0 pgmap v1885: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:15:10.113 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:09 smithi078 bash[78274]: cluster 2023-10-01T11:15:08.504075+0000 mgr.y (mgr.25072) 1738 : cluster 0 pgmap v1885: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:15:10.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:09 smithi173 bash[58816]: cluster 2023-10-01T11:15:08.504075+0000 mgr.y (mgr.25072) 1738 : cluster 0 pgmap v1885: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:15:10.144 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:10.179 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:10.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:10.182 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:15:10.282 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:10.315 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:10.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:10.319 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:15:10.420 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:10.448 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:10.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:10.450 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:15:10.551 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:10.564 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:15:10 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:15:10] "GET /metrics HTTP/1.1" 200 33960 "" "Prometheus/2.33.4" 2023-10-01T11:15:10.577 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:10.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:10.580 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:15:10.681 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:10.714 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:10.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:10.717 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:15:10.817 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:10.849 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:10.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:10.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:15:10.953 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:10.978 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:10.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:10.981 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:15:11.081 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:11.114 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:11.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:11.117 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:15:11.217 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:15:11.241 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:15:11.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:11.244 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:15:11.345 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:15:11.369 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:15:11.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:11.371 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:15:11.472 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:11.497 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:11.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:11.500 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:15:11.601 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:11.632 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:11.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:11.636 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:15:11.737 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:11.771 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:11.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:11.774 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:15:11.876 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:11.901 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:11.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:11.903 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:15:11.954 INFO:tasks.workunit.client.0.smithi078.stderr:rbd: cannot import a directory 2023-10-01T11:15:11.955 INFO:tasks.workunit.client.0.smithi078.stderr: Importing image: 0% complete...failed. 2023-10-01T11:15:11.955 INFO:tasks.workunit.client.0.smithi078.stderr:rbd: import failed: (21) Is a directory 2023-10-01T11:15:12.004 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:15:12.029 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:15:12.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:12.031 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:15:12.035 INFO:tasks.workunit.client.0.smithi078.stderr:+ true 2023-10-01T11:15:12.035 INFO:tasks.workunit.client.0.smithi078.stderr:+ rmdir foo.259593 2023-10-01T11:15:12.036 INFO:tasks.workunit.client.0.smithi078.stderr:+ dd if=/bin/sh of=/tmp/rbd_import_export_259593/img bs=1k count=1 seek=10 2023-10-01T11:15:12.038 INFO:tasks.workunit.client.0.smithi078.stderr:1+0 records in 2023-10-01T11:15:12.038 INFO:tasks.workunit.client.0.smithi078.stderr:1+0 records out 2023-10-01T11:15:12.038 INFO:tasks.workunit.client.0.smithi078.stderr:1024 bytes (1.0 kB, 1.0 KiB) copied, 0.000211039 s, 4.9 MB/s 2023-10-01T11:15:12.038 INFO:tasks.workunit.client.0.smithi078.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_259593/img bs=1k count=10 seek=100 2023-10-01T11:15:12.039 INFO:tasks.workunit.client.0.smithi078.stderr:10+0 records in 2023-10-01T11:15:12.039 INFO:tasks.workunit.client.0.smithi078.stderr:10+0 records out 2023-10-01T11:15:12.039 INFO:tasks.workunit.client.0.smithi078.stderr:10240 bytes (10 kB, 10 KiB) copied, 0.000103189 s, 99.2 MB/s 2023-10-01T11:15:12.039 INFO:tasks.workunit.client.0.smithi078.stderr:+ dd if=/bin/rm of=/tmp/rbd_import_export_259593/img bs=1k count=100 seek=1000 2023-10-01T11:15:12.040 INFO:tasks.workunit.client.0.smithi078.stderr:70+1 records in 2023-10-01T11:15:12.040 INFO:tasks.workunit.client.0.smithi078.stderr:70+1 records out 2023-10-01T11:15:12.040 INFO:tasks.workunit.client.0.smithi078.stderr:72056 bytes (72 kB, 70 KiB) copied, 0.000535906 s, 134 MB/s 2023-10-01T11:15:12.040 INFO:tasks.workunit.client.0.smithi078.stderr:+ dd if=/bin/ls of=/tmp/rbd_import_export_259593/img bs=1k seek=10000 2023-10-01T11:15:12.041 INFO:tasks.workunit.client.0.smithi078.stderr:138+1 records in 2023-10-01T11:15:12.042 INFO:tasks.workunit.client.0.smithi078.stderr:138+1 records out 2023-10-01T11:15:12.042 INFO:tasks.workunit.client.0.smithi078.stderr:142144 bytes (142 kB, 139 KiB) copied, 0.000465254 s, 306 MB/s 2023-10-01T11:15:12.042 INFO:tasks.workunit.client.0.smithi078.stderr:+ dd if=/bin/ln of=/tmp/rbd_import_export_259593/img bs=1k seek=100000 2023-10-01T11:15:12.134 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:12.147 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:11 smithi078 bash[78274]: cluster 2023-10-01T11:15:10.504731+0000 mgr.y (mgr.25072) 1739 : cluster 0 pgmap v1886: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:15:12.148 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:11 smithi078 bash[75299]: cluster 2023-10-01T11:15:10.504731+0000 mgr.y (mgr.25072) 1739 : cluster 0 pgmap v1886: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:15:12.165 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:12.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:12.168 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:15:12.215 INFO:tasks.workunit.client.0.smithi078.stderr:74+1 records in 2023-10-01T11:15:12.215 INFO:tasks.workunit.client.0.smithi078.stderr:74+1 records out 2023-10-01T11:15:12.215 INFO:tasks.workunit.client.0.smithi078.stderr:76160 bytes (76 kB, 74 KiB) copied, 0.172944 s, 440 kB/s 2023-10-01T11:15:12.216 INFO:tasks.workunit.client.0.smithi078.stderr:+ dd if=/bin/grep of=/tmp/rbd_import_export_259593/img bs=1k seek=1000000 2023-10-01T11:15:12.269 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:12.282 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:11 smithi173 bash[58816]: cluster 2023-10-01T11:15:10.504731+0000 mgr.y (mgr.25072) 1739 : cluster 0 pgmap v1886: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:15:12.283 INFO:tasks.workunit.client.0.smithi078.stderr:194+1 records in 2023-10-01T11:15:12.283 INFO:tasks.workunit.client.0.smithi078.stderr:194+1 records out 2023-10-01T11:15:12.283 INFO:tasks.workunit.client.0.smithi078.stderr:199136 bytes (199 kB, 194 KiB) copied, 0.0663942 s, 3.0 MB/s 2023-10-01T11:15:12.283 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd rm testimg 2023-10-01T11:15:12.297 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:12.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:12.302 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:15:12.403 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:12.433 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:12.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:12.437 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:15:12.537 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:12.559 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:12.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:12.561 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:15:12.615 INFO:tasks.workunit.client.0.smithi078.stderr: Removing image: 0% complete...failed. 2023-10-01T11:15:12.615 INFO:tasks.workunit.client.0.smithi078.stderr:rbd: delete error: (2) No such file or directory 2023-10-01T11:15:12.641 INFO:tasks.workunit.client.0.smithi078.stderr:+ true 2023-10-01T11:15:12.641 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd import --new-format /tmp/rbd_import_export_259593/img testimg 2023-10-01T11:15:12.662 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:12.687 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:12.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:12.690 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:15:12.790 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:15:12.816 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:15:12.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:12.819 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:15:12.920 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:12.949 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:12.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:12.950 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:15:13.051 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:13.076 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:13.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:13.079 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:15:13.180 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:13.215 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:13.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:13.218 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:15:13.319 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:15:13.343 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:15:13.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:13.346 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:15:13.447 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:13.478 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:13.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:13.482 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:15:13.583 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:13.619 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:13.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:13.622 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:15:13.723 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:13.744 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:13.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:13.746 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:15:13.818 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T11:15:13.837 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T11:15:13.848 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:13.857 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T11:15:13.877 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T11:15:13.879 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:13.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:13.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:15:13.904 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T11:15:13.934 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T11:15:13.956 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T11:15:13.979 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T11:15:13.981 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-10-01T11:15:13.982 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-10-01T11:15:13.982 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-10-01T11:15:13.982 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-10-01T11:15:13.982 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-10-01T11:15:13.982 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-10-01T11:15:13.982 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-10-01T11:15:13.982 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-10-01T11:15:13.984 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:13.991 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:13 smithi078 bash[78274]: cluster 2023-10-01T11:15:12.505533+0000 mgr.y (mgr.25072) 1740 : cluster 0 pgmap v1887: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:15:13.992 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:13 smithi078 bash[78274]: cluster 2023-10-01T11:15:12.911369+0000 osd.7 (osd.7) 438 : cluster 3 failed to encode map e389 with expected crc 2023-10-01T11:15:13.992 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:13 smithi078 bash[78274]: cluster 2023-10-01T11:15:12.913378+0000 osd.5 (osd.5) 417 : cluster 3 failed to encode map e389 with expected crc 2023-10-01T11:15:13.992 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:13 smithi078 bash[78274]: cluster 2023-10-01T11:15:12.914029+0000 mon.a (mon.0) 2225 : cluster 0 osdmap e389: 8 total, 8 up, 8 in 2023-10-01T11:15:13.992 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:13 smithi078 bash[78274]: audit 2023-10-01T11:15:13.790585+0000 mon.a (mon.0) 2226 : audit 0 from='client.? 172.21.15.78:0/2242417121' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:15:13.992 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:13 smithi078 bash[75299]: cluster 2023-10-01T11:15:12.505533+0000 mgr.y (mgr.25072) 1740 : cluster 0 pgmap v1887: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:15:13.993 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:13 smithi078 bash[75299]: cluster 2023-10-01T11:15:12.911369+0000 osd.7 (osd.7) 438 : cluster 3 failed to encode map e389 with expected crc 2023-10-01T11:15:13.993 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:13 smithi078 bash[75299]: cluster 2023-10-01T11:15:12.913378+0000 osd.5 (osd.5) 417 : cluster 3 failed to encode map e389 with expected crc 2023-10-01T11:15:13.993 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:13 smithi078 bash[75299]: cluster 2023-10-01T11:15:12.914029+0000 mon.a (mon.0) 2225 : cluster 0 osdmap e389: 8 total, 8 up, 8 in 2023-10-01T11:15:13.993 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:13 smithi078 bash[75299]: audit 2023-10-01T11:15:13.790585+0000 mon.a (mon.0) 2226 : audit 0 from='client.? 172.21.15.78:0/2242417121' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:15:14.004 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:14.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:14.007 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:15:14.107 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:14.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:13 smithi173 bash[58816]: cluster 2023-10-01T11:15:12.505533+0000 mgr.y (mgr.25072) 1740 : cluster 0 pgmap v1887: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:15:14.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:13 smithi173 bash[58816]: cluster 2023-10-01T11:15:12.911369+0000 osd.7 (osd.7) 438 : cluster 3 failed to encode map e389 with expected crc 2023-10-01T11:15:14.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:13 smithi173 bash[58816]: cluster 2023-10-01T11:15:12.913378+0000 osd.5 (osd.5) 417 : cluster 3 failed to encode map e389 with expected crc 2023-10-01T11:15:14.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:13 smithi173 bash[58816]: cluster 2023-10-01T11:15:12.914029+0000 mon.a (mon.0) 2225 : cluster 0 osdmap e389: 8 total, 8 up, 8 in 2023-10-01T11:15:14.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:13 smithi173 bash[58816]: audit 2023-10-01T11:15:13.790585+0000 mon.a (mon.0) 2226 : audit 0 from='client.? 172.21.15.78:0/2242417121' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:15:14.135 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:15:14.141 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:14.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:14.144 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:15:14.245 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:14.276 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:14.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:14.279 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:15:14.381 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:14.408 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:14.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:14.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:15:14.514 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:14.544 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:14.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:14.548 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:15:14.592 INFO:tasks.workunit.client.0.smithi078.stderr: Importing image: 1% complete... Importing image: 2% complete... Importing image: 3% complete... Importing image: 4% complete... Importing image: 5% complete... Importing image: 6% complete... Importing image: 7% complete... Importing image: 8% complete... Importing image: 9% complete... Importing image: 10% complete... Importing image: 11% complete... Importing image: 12% complete... Importing image: 13% complete... Importing image: 14% complete... Importing image: 15% complete... Importing image: 16% complete... Importing image: 17% complete... Importing image: 18% complete... Importing image: 19% complete... Importing image: 20% complete... Importing image: 21% complete... Importing image: 22% complete... Importing image: 23% complete... Importing image: 24% complete... Importing image: 25% complete... Importing image: 26% complete... Importing image: 27% complete... Importing image: 28% complete... Importing image: 29% complete... Importing image: 30% complete... Importing image: 31% complete... Importing image: 32% complete... Importing image: 33% complete... Importing image: 34% complete... Importing image: 35% complete... Importing image: 36% complete... Importing image: 37% complete... Importing image: 38% complete... Importing image: 39% complete... Importing image: 40% complete... Importing image: 41% complete... Importing image: 42% complete... Importing image: 43% complete... Importing image: 44% complete... Importing image: 45% complete... Importing image: 46% complete... Importing image: 47% complete... Importing image: 48% complete... Importing image: 49% complete... Importing image: 50% complete... Importing image: 51% complete... Importing image: 52% complete... Importing image: 53% complete... Importing image: 54% complete... Importing image: 55% complete... Importing image: 56% complete... Importing image: 57% complete... Importing image: 58% complete... Importing image: 59% complete... Importing image: 60% complete... Importing image: 61% complete... Importing image: 62% complete... Importing image: 63% complete... Importing image: 64% complete... Importing image: 65% complete... Importing image: 66% complete... Importing image: 67% complete... Importing image: 68% complete... Importing image: 69% complete... Importing image: 70% complete... Importing image: 71% complete... Importing image: 72% complete... Importing image: 73% complete... Importing image: 74% complete... Importing image: 75% complete... Importing image: 76% complete... Importing image: 77% complete... Importing image: 78% complete... Importing image: 79% complete... Importing image: 80% complete... Importing image: 81% complete... Importing image: 82% complete... Importing image: 83% complete... Importing image: 84% complete... Importing image: 85% complete... Importing image: 86% complete... Importing image: 87% complete... Importing image: 88% complete... Importing image: 89% complete... Importing image: 90% complete... Importing image: 91% complete... Importing image: 92% complete... Importing image: 93% complete... Importing image: 94% complete... Importing image: 95% complete... Importing image: 96% complete... Importing image: 97% complete... Importing image: 98% complete... Importing image: 99% complete... Importing image: 100% complete...done. 2023-10-01T11:15:14.592 INFO:tasks.workunit.client.0.smithi078.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-01T11:15:14.597 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd export testimg /tmp/rbd_import_export_259593/img2 2023-10-01T11:15:14.649 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:14.682 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:14.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:14.686 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:15:14.789 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:14.811 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:14.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:14.813 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:15:14.914 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:14.929 INFO:tasks.workunit.client.0.smithi078.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-01T11:15:14.935 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:14.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:14.937 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:15:14.939 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd export testimg - 2023-10-01T11:15:15.039 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:15.069 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:15.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:15.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:15:15.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:14 smithi078 bash[78274]: cluster 2023-10-01T11:15:13.913592+0000 osd.7 (osd.7) 439 : cluster 3 failed to encode map e390 with expected crc 2023-10-01T11:15:15.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:14 smithi078 bash[78274]: cluster 2023-10-01T11:15:13.914642+0000 mon.a (mon.0) 2227 : cluster 0 osdmap e390: 8 total, 8 up, 8 in 2023-10-01T11:15:15.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:14 smithi078 bash[78274]: cluster 2023-10-01T11:15:13.917962+0000 osd.5 (osd.5) 418 : cluster 3 failed to encode map e390 with expected crc 2023-10-01T11:15:15.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:14 smithi078 bash[75299]: cluster 2023-10-01T11:15:13.913592+0000 osd.7 (osd.7) 439 : cluster 3 failed to encode map e390 with expected crc 2023-10-01T11:15:15.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:14 smithi078 bash[75299]: cluster 2023-10-01T11:15:13.914642+0000 mon.a (mon.0) 2227 : cluster 0 osdmap e390: 8 total, 8 up, 8 in 2023-10-01T11:15:15.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:14 smithi078 bash[75299]: cluster 2023-10-01T11:15:13.917962+0000 osd.5 (osd.5) 418 : cluster 3 failed to encode map e390 with expected crc 2023-10-01T11:15:15.172 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:15.184 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:14 smithi173 bash[58816]: cluster 2023-10-01T11:15:13.913592+0000 osd.7 (osd.7) 439 : cluster 3 failed to encode map e390 with expected crc 2023-10-01T11:15:15.184 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:14 smithi173 bash[58816]: cluster 2023-10-01T11:15:13.914642+0000 mon.a (mon.0) 2227 : cluster 0 osdmap e390: 8 total, 8 up, 8 in 2023-10-01T11:15:15.184 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:14 smithi173 bash[58816]: cluster 2023-10-01T11:15:13.917962+0000 osd.5 (osd.5) 418 : cluster 3 failed to encode map e390 with expected crc 2023-10-01T11:15:15.203 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:15.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:15.206 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:15:15.308 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:15.339 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:15.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:15.342 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:15:15.443 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:15.475 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:15.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:15.479 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:15:15.580 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:15.851 INFO:tasks.workunit.client.0.smithi078.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-01T11:15:15.855 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd rm testimg 2023-10-01T11:15:15.975 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:15.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:15.977 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:15:16.054 INFO:tasks.workunit.client.0.smithi078.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-10-01T11:15:16.059 INFO:tasks.workunit.client.0.smithi078.stderr:+ cmp /tmp/rbd_import_export_259593/img /tmp/rbd_import_export_259593/img2 2023-10-01T11:15:16.078 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:16.111 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:16.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:16.114 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:15:16.220 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:16.232 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:15 smithi078 bash[75299]: cluster 2023-10-01T11:15:14.506107+0000 mgr.y (mgr.25072) 1741 : cluster 0 pgmap v1890: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 1023 B/s rd, 767 B/s wr, 2 op/s 2023-10-01T11:15:16.233 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:15 smithi078 bash[78274]: cluster 2023-10-01T11:15:14.506107+0000 mgr.y (mgr.25072) 1741 : cluster 0 pgmap v1890: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 1023 B/s rd, 767 B/s wr, 2 op/s 2023-10-01T11:15:16.254 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:16.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:16.256 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:15:16.357 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:16.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:15 smithi173 bash[58816]: cluster 2023-10-01T11:15:14.506107+0000 mgr.y (mgr.25072) 1741 : cluster 0 pgmap v1890: 73 pgs: 73 active+clean; 582 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 1023 B/s rd, 767 B/s wr, 2 op/s 2023-10-01T11:15:16.381 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:16.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:16.385 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:15:16.486 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:15:16.511 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:15:16.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:16.514 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:15:16.614 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:16.647 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:16.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:16.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:15:16.751 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:16.786 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:16.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:16.788 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:15:16.860 INFO:tasks.workunit.client.0.smithi078.stderr:+ cmp /tmp/rbd_import_export_259593/img /tmp/rbd_import_export_259593/img3 2023-10-01T11:15:16.889 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:16.917 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:16.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:16.920 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:15:17.020 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:17.136 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:17.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:17.139 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:15:17.240 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:17.276 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:17.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:17.279 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:15:17.380 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:17.383 INFO:tasks.workunit.client.0.smithi078.stderr:+ rm /tmp/rbd_import_export_259593/img2 /tmp/rbd_import_export_259593/img3 2023-10-01T11:15:17.412 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:17.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:17.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:15:17.516 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:17.691 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:17.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:17.705 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:15:17.807 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:17.843 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:17.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:17.846 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:15:17.947 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:17.979 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:17.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:17.982 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:15:18.083 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:18.094 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:17 smithi173 bash[58816]: cluster 2023-10-01T11:15:16.506579+0000 mgr.y (mgr.25072) 1742 : cluster 0 pgmap v1891: 73 pgs: 73 active+clean; 1.3 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 25 KiB/s rd, 25 KiB/s wr, 3 op/s 2023-10-01T11:15:18.152 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:17 smithi173 bash[58816]: audit 2023-10-01T11:15:17.483203+0000 mon.a (mon.0) 2228 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:15:18.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:18.153 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:18.153 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:15:18.254 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:18.265 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:17 smithi078 bash[78274]: cluster 2023-10-01T11:15:16.506579+0000 mgr.y (mgr.25072) 1742 : cluster 0 pgmap v1891: 73 pgs: 73 active+clean; 1.3 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 25 KiB/s rd, 25 KiB/s wr, 3 op/s 2023-10-01T11:15:18.266 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:17 smithi078 bash[78274]: audit 2023-10-01T11:15:17.483203+0000 mon.a (mon.0) 2228 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:15:18.266 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:17 smithi078 bash[75299]: cluster 2023-10-01T11:15:16.506579+0000 mgr.y (mgr.25072) 1742 : cluster 0 pgmap v1891: 73 pgs: 73 active+clean; 1.3 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 25 KiB/s rd, 25 KiB/s wr, 3 op/s 2023-10-01T11:15:18.266 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:17 smithi078 bash[75299]: audit 2023-10-01T11:15:17.483203+0000 mon.a (mon.0) 2228 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:15:18.281 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:18.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:18.284 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:15:18.385 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:18.417 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:18.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:18.420 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:15:18.521 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:18.554 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:18.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:18.556 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:15:18.564 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd import --new-format - testimg 2023-10-01T11:15:18.657 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:18.687 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:18.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:18.690 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:15:18.791 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:18.818 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:18.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:18.821 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:15:18.922 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:18.953 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:18.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:18.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:15:19.057 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:19.086 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:19.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:19.462 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:15:19.469 INFO:tasks.workunit.client.0.smithi078.stderr: Importing image: 100% complete...done. 2023-10-01T11:15:19.469 INFO:tasks.workunit.client.0.smithi078.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-01T11:15:19.470 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd export testimg /tmp/rbd_import_export_259593/img2 2023-10-01T11:15:19.470 INFO:tasks.workunit.client.0.smithi078.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-01T11:15:19.470 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd export testimg - 2023-10-01T11:15:19.564 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:19.589 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:19.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:19.591 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:15:19.692 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:19.724 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:19.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:19.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:15:19.827 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:19.863 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:19.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:19.866 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:15:19.967 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:15:19.993 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:15:19.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:19.995 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:15:20.096 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:20.116 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:20.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:20.123 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:15:20.124 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:19 smithi173 bash[58816]: cluster 2023-10-01T11:15:18.507229+0000 mgr.y (mgr.25072) 1743 : cluster 0 pgmap v1892: 73 pgs: 73 active+clean; 1.3 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 44 KiB/s rd, 25 KiB/s wr, 25 op/s 2023-10-01T11:15:20.224 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:20.257 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:20.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:20.260 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:15:20.268 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:19 smithi078 bash[78274]: cluster 2023-10-01T11:15:18.507229+0000 mgr.y (mgr.25072) 1743 : cluster 0 pgmap v1892: 73 pgs: 73 active+clean; 1.3 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 44 KiB/s rd, 25 KiB/s wr, 25 op/s 2023-10-01T11:15:20.268 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:19 smithi078 bash[75299]: cluster 2023-10-01T11:15:18.507229+0000 mgr.y (mgr.25072) 1743 : cluster 0 pgmap v1892: 73 pgs: 73 active+clean; 1.3 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 44 KiB/s rd, 25 KiB/s wr, 25 op/s 2023-10-01T11:15:20.348 INFO:tasks.workunit.client.0.smithi078.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-10-01T11:15:20.353 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd rm testimg 2023-10-01T11:15:20.360 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:20.391 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:20.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:20.393 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:15:20.485 INFO:tasks.workunit.client.0.smithi078.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-10-01T11:15:20.490 INFO:tasks.workunit.client.0.smithi078.stderr:+ cmp /tmp/rbd_import_export_259593/img /tmp/rbd_import_export_259593/img2 2023-10-01T11:15:20.494 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:20.529 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:20.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:20.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:15:20.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:15:20 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:15:20] "GET /metrics HTTP/1.1" 200 34006 "" "Prometheus/2.33.4" 2023-10-01T11:15:20.633 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:20.652 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:20.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:20.654 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:15:20.755 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:20.787 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:20.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:20.790 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:15:20.891 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:20.925 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:20.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:20.928 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:15:21.029 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:21.051 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:21.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:21.052 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:15:21.153 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:21.185 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:21.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:21.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:15:21.246 INFO:tasks.workunit.client.0.smithi078.stderr:+ cmp /tmp/rbd_import_export_259593/img /tmp/rbd_import_export_259593/img3 2023-10-01T11:15:21.288 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:21.312 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:21.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:21.314 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:15:21.415 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:21.447 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:21.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:21.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:15:21.551 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:15:21.581 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:15:21.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:21.584 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:15:21.684 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:21.713 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:21.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:21.715 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:15:21.740 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:15:21.775 INFO:tasks.workunit.client.0.smithi078.stderr:+ rm /tmp/rbd_import_export_259593/img /tmp/rbd_import_export_259593/img2 /tmp/rbd_import_export_259593/img3 2023-10-01T11:15:21.817 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:21.848 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:21.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:21.851 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:15:21.952 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:21.983 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:21.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:21.986 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:15:22.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:21 smithi078 bash[78274]: cluster 2023-10-01T11:15:20.508178+0000 mgr.y (mgr.25072) 1744 : cluster 0 pgmap v1893: 73 pgs: 73 active+clean; 4.3 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 174 KiB/s rd, 118 KiB/s wr, 41 op/s 2023-10-01T11:15:22.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:21 smithi078 bash[75299]: cluster 2023-10-01T11:15:20.508178+0000 mgr.y (mgr.25072) 1744 : cluster 0 pgmap v1893: 73 pgs: 73 active+clean; 4.3 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 174 KiB/s rd, 118 KiB/s wr, 41 op/s 2023-10-01T11:15:22.087 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:22.100 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:21 smithi173 bash[58816]: cluster 2023-10-01T11:15:20.508178+0000 mgr.y (mgr.25072) 1744 : cluster 0 pgmap v1893: 73 pgs: 73 active+clean; 4.3 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 174 KiB/s rd, 118 KiB/s wr, 41 op/s 2023-10-01T11:15:22.119 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:22.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:22.122 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:15:22.223 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:22.256 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:22.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:22.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:15:22.359 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:22.396 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:22.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:22.399 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:15:22.499 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:22.533 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:22.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:22.611 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:15:22.712 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:22.746 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:22.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:22.750 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:15:22.851 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:22.883 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:22.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:22.885 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:15:22.985 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:23.017 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:23.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:23.020 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:15:23.121 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:23.156 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:23.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:23.159 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:15:23.260 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:23.274 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-10-01T11:15:23.290 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:23.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:23.294 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:15:23.395 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:23.426 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:23.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:23.429 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:15:23.530 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:23.562 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:23.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:23.565 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:15:23.666 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:23.700 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:23.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:23.703 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:15:23.804 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:23.826 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:23.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:23.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:15:23.929 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:23.961 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:23.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:23.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:15:24.065 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:24.075 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:23 smithi078 bash[75299]: cluster 2023-10-01T11:15:22.508979+0000 mgr.y (mgr.25072) 1745 : cluster 0 pgmap v1894: 73 pgs: 73 active+clean; 4.3 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 145 KiB/s rd, 98 KiB/s wr, 34 op/s 2023-10-01T11:15:24.075 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:23 smithi078 bash[75299]: cluster 2023-10-01T11:15:22.903129+0000 mon.a (mon.0) 2229 : cluster 0 osdmap e391: 8 total, 8 up, 8 in 2023-10-01T11:15:24.075 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:23 smithi078 bash[75299]: cluster 2023-10-01T11:15:22.903351+0000 osd.7 (osd.7) 440 : cluster 3 failed to encode map e391 with expected crc 2023-10-01T11:15:24.075 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:23 smithi078 bash[75299]: cluster 2023-10-01T11:15:23.464255+0000 osd.6 (osd.6) 378 : cluster 3 failed to encode map e391 with expected crc 2023-10-01T11:15:24.076 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:23 smithi078 bash[78274]: cluster 2023-10-01T11:15:22.508979+0000 mgr.y (mgr.25072) 1745 : cluster 0 pgmap v1894: 73 pgs: 73 active+clean; 4.3 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 145 KiB/s rd, 98 KiB/s wr, 34 op/s 2023-10-01T11:15:24.076 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:23 smithi078 bash[78274]: cluster 2023-10-01T11:15:22.903129+0000 mon.a (mon.0) 2229 : cluster 0 osdmap e391: 8 total, 8 up, 8 in 2023-10-01T11:15:24.076 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:23 smithi078 bash[78274]: cluster 2023-10-01T11:15:22.903351+0000 osd.7 (osd.7) 440 : cluster 3 failed to encode map e391 with expected crc 2023-10-01T11:15:24.076 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:23 smithi078 bash[78274]: cluster 2023-10-01T11:15:23.464255+0000 osd.6 (osd.6) 378 : cluster 3 failed to encode map e391 with expected crc 2023-10-01T11:15:24.088 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:24.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:24.091 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:15:24.191 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:24.203 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:23 smithi173 bash[58816]: cluster 2023-10-01T11:15:22.508979+0000 mgr.y (mgr.25072) 1745 : cluster 0 pgmap v1894: 73 pgs: 73 active+clean; 4.3 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 145 KiB/s rd, 98 KiB/s wr, 34 op/s 2023-10-01T11:15:24.203 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:23 smithi173 bash[58816]: cluster 2023-10-01T11:15:22.903129+0000 mon.a (mon.0) 2229 : cluster 0 osdmap e391: 8 total, 8 up, 8 in 2023-10-01T11:15:24.203 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:23 smithi173 bash[58816]: cluster 2023-10-01T11:15:22.903351+0000 osd.7 (osd.7) 440 : cluster 3 failed to encode map e391 with expected crc 2023-10-01T11:15:24.204 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:23 smithi173 bash[58816]: cluster 2023-10-01T11:15:23.464255+0000 osd.6 (osd.6) 378 : cluster 3 failed to encode map e391 with expected crc 2023-10-01T11:15:24.221 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:24.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:24.224 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:15:24.325 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:24.357 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:24.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:24.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:15:24.370 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd help export 2023-10-01T11:15:24.370 INFO:tasks.workunit.client.0.smithi078.stderr:+ grep -q export-format 2023-10-01T11:15:24.460 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:24.493 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:24.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:24.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:15:24.500 INFO:tasks.workunit.client.0.smithi078.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_259593/img bs=1k count=10 seek=100 2023-10-01T11:15:24.500 INFO:tasks.workunit.client.0.smithi078.stderr:10+0 records in 2023-10-01T11:15:24.500 INFO:tasks.workunit.client.0.smithi078.stderr:10+0 records out 2023-10-01T11:15:24.500 INFO:tasks.workunit.client.0.smithi078.stderr:10240 bytes (10 kB, 10 KiB) copied, 9.7153e-05 s, 105 MB/s 2023-10-01T11:15:24.500 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd import --new-format /tmp/rbd_import_export_259593/img testimg 2023-10-01T11:15:24.565 INFO:tasks.workunit.client.0.smithi078.stderr: Importing image: 100% complete...done. 2023-10-01T11:15:24.566 INFO:tasks.workunit.client.0.smithi078.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-01T11:15:24.568 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd snap create testimg@snap 2023-10-01T11:15:24.600 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:24.632 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:24.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:24.635 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:15:24.735 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:24.758 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:24.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:24.760 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:15:24.861 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:15:24.883 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:15:24.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:24.886 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:15:24.922 INFO:teuthology.orchestra.run.smithi078.stderr:noscrub is set 2023-10-01T11:15:24.927 INFO:tasks.workunit.client.0.smithi078.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-10-01T11:15:24.937 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd image-meta set testimg key1 value1 2023-10-01T11:15:24.987 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:25.008 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:25.010 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd image-meta list testimg 2023-10-01T11:15:25.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:25.011 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:15:25.038 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:24 smithi078 bash[78274]: cluster 2023-10-01T11:15:23.459165+0000 osd.5 (osd.5) 419 : cluster 3 failed to encode map e391 with expected crc 2023-10-01T11:15:25.038 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:24 smithi078 bash[78274]: audit 2023-10-01T11:15:24.433584+0000 mon.a (mon.0) 2230 : audit 1 from='client.? 172.21.15.78:0/3218814586' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T11:15:25.038 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:24 smithi078 bash[75299]: cluster 2023-10-01T11:15:23.459165+0000 osd.5 (osd.5) 419 : cluster 3 failed to encode map e391 with expected crc 2023-10-01T11:15:25.038 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:24 smithi078 bash[75299]: audit 2023-10-01T11:15:24.433584+0000 mon.a (mon.0) 2230 : audit 1 from='client.? 172.21.15.78:0/3218814586' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T11:15:25.061 INFO:tasks.workunit.client.0.smithi078.stderr:+ IMAGEMETA_BEFORE=There is 1 metadatum on this image: 2023-10-01T11:15:25.061 INFO:tasks.workunit.client.0.smithi078.stderr: 2023-10-01T11:15:25.061 INFO:tasks.workunit.client.0.smithi078.stderr:Key Value 2023-10-01T11:15:25.061 INFO:tasks.workunit.client.0.smithi078.stderr:key1 value1 2023-10-01T11:15:25.062 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_259593/img_v2 2023-10-01T11:15:25.112 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:25.114 INFO:tasks.workunit.client.0.smithi078.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-01T11:15:25.120 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_259593/img_v2 testimg_import 2023-10-01T11:15:25.140 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:25.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:25.143 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:15:25.244 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:25.258 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:24 smithi173 bash[58816]: cluster 2023-10-01T11:15:23.459165+0000 osd.5 (osd.5) 419 : cluster 3 failed to encode map e391 with expected crc 2023-10-01T11:15:25.258 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:24 smithi173 bash[58816]: audit 2023-10-01T11:15:24.433584+0000 mon.a (mon.0) 2230 : audit 1 from='client.? 172.21.15.78:0/3218814586' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-10-01T11:15:25.275 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:25.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:25.278 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:15:25.378 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:25.401 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:25.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:25.403 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:15:25.504 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:25.532 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:25.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:25.535 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:15:25.635 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:25.670 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:25.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:25.674 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:15:25.774 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:25.796 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:25.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:25.797 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:15:25.898 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:25.933 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:25.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:25.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:15:25.941 INFO:tasks.workunit.client.0.smithi078.stderr: Importing image: 50% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-01T11:15:25.945 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd info testimg_import 2023-10-01T11:15:26.036 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:26.064 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:26.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:26.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:15:26.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:25 smithi078 bash[78274]: cluster 2023-10-01T11:15:24.509832+0000 mgr.y (mgr.25072) 1746 : cluster 0 pgmap v1896: 73 pgs: 73 active+clean; 1.6 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 255 KiB/s rd, 105 KiB/s wr, 77 op/s 2023-10-01T11:15:26.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:25 smithi078 bash[78274]: cluster 2023-10-01T11:15:24.903254+0000 mon.a (mon.0) 2231 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:15:26.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:25 smithi078 bash[78274]: cluster 2023-10-01T11:15:24.912861+0000 osd.7 (osd.7) 441 : cluster 3 failed to encode map e392 with expected crc 2023-10-01T11:15:26.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:25 smithi078 bash[78274]: audit 2023-10-01T11:15:24.915112+0000 mon.a (mon.0) 2232 : audit 1 from='client.? 172.21.15.78:0/3218814586' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T11:15:26.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:25 smithi078 bash[78274]: cluster 2023-10-01T11:15:24.915185+0000 mon.a (mon.0) 2233 : cluster 0 osdmap e392: 8 total, 8 up, 8 in 2023-10-01T11:15:26.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:25 smithi078 bash[78274]: cluster 2023-10-01T11:15:24.979479+0000 osd.5 (osd.5) 420 : cluster 3 failed to encode map e392 with expected crc 2023-10-01T11:15:26.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:25 smithi078 bash[75299]: cluster 2023-10-01T11:15:24.509832+0000 mgr.y (mgr.25072) 1746 : cluster 0 pgmap v1896: 73 pgs: 73 active+clean; 1.6 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 255 KiB/s rd, 105 KiB/s wr, 77 op/s 2023-10-01T11:15:26.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:25 smithi078 bash[75299]: cluster 2023-10-01T11:15:24.903254+0000 mon.a (mon.0) 2231 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:15:26.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:25 smithi078 bash[75299]: cluster 2023-10-01T11:15:24.912861+0000 osd.7 (osd.7) 441 : cluster 3 failed to encode map e392 with expected crc 2023-10-01T11:15:26.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:25 smithi078 bash[75299]: audit 2023-10-01T11:15:24.915112+0000 mon.a (mon.0) 2232 : audit 1 from='client.? 172.21.15.78:0/3218814586' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T11:15:26.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:25 smithi078 bash[75299]: cluster 2023-10-01T11:15:24.915185+0000 mon.a (mon.0) 2233 : cluster 0 osdmap e392: 8 total, 8 up, 8 in 2023-10-01T11:15:26.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:25 smithi078 bash[75299]: cluster 2023-10-01T11:15:24.979479+0000 osd.5 (osd.5) 420 : cluster 3 failed to encode map e392 with expected crc 2023-10-01T11:15:26.099 INFO:tasks.workunit.client.0.smithi078.stdout:rbd image 'testimg_import': 2023-10-01T11:15:26.099 INFO:tasks.workunit.client.0.smithi078.stdout: size 110 KiB in 1 objects 2023-10-01T11:15:26.099 INFO:tasks.workunit.client.0.smithi078.stdout: order 22 (4 MiB objects) 2023-10-01T11:15:26.099 INFO:tasks.workunit.client.0.smithi078.stdout: snapshot_count: 1 2023-10-01T11:15:26.099 INFO:tasks.workunit.client.0.smithi078.stdout: id: 9105ba7fd283 2023-10-01T11:15:26.100 INFO:tasks.workunit.client.0.smithi078.stdout: block_name_prefix: rbd_data.9105ba7fd283 2023-10-01T11:15:26.100 INFO:tasks.workunit.client.0.smithi078.stdout: format: 2 2023-10-01T11:15:26.100 INFO:tasks.workunit.client.0.smithi078.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-10-01T11:15:26.100 INFO:tasks.workunit.client.0.smithi078.stdout: op_features: 2023-10-01T11:15:26.100 INFO:tasks.workunit.client.0.smithi078.stdout: flags: 2023-10-01T11:15:26.100 INFO:tasks.workunit.client.0.smithi078.stdout: create_timestamp: Sun Oct 1 11:15:25 2023 2023-10-01T11:15:26.100 INFO:tasks.workunit.client.0.smithi078.stdout: access_timestamp: Sun Oct 1 11:15:25 2023 2023-10-01T11:15:26.100 INFO:tasks.workunit.client.0.smithi078.stdout: modify_timestamp: Sun Oct 1 11:15:25 2023 2023-10-01T11:15:26.105 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd info testimg_import@snap 2023-10-01T11:15:26.153 INFO:tasks.workunit.client.0.smithi078.stdout:rbd image 'testimg_import': 2023-10-01T11:15:26.153 INFO:tasks.workunit.client.0.smithi078.stdout: size 110 KiB in 1 objects 2023-10-01T11:15:26.153 INFO:tasks.workunit.client.0.smithi078.stdout: order 22 (4 MiB objects) 2023-10-01T11:15:26.153 INFO:tasks.workunit.client.0.smithi078.stdout: snapshot_count: 1 2023-10-01T11:15:26.153 INFO:tasks.workunit.client.0.smithi078.stdout: id: 9105ba7fd283 2023-10-01T11:15:26.153 INFO:tasks.workunit.client.0.smithi078.stdout: block_name_prefix: rbd_data.9105ba7fd283 2023-10-01T11:15:26.153 INFO:tasks.workunit.client.0.smithi078.stdout: format: 2 2023-10-01T11:15:26.153 INFO:tasks.workunit.client.0.smithi078.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-10-01T11:15:26.154 INFO:tasks.workunit.client.0.smithi078.stdout: op_features: 2023-10-01T11:15:26.154 INFO:tasks.workunit.client.0.smithi078.stdout: flags: 2023-10-01T11:15:26.154 INFO:tasks.workunit.client.0.smithi078.stdout: create_timestamp: Sun Oct 1 11:15:25 2023 2023-10-01T11:15:26.154 INFO:tasks.workunit.client.0.smithi078.stdout: access_timestamp: Sun Oct 1 11:15:25 2023 2023-10-01T11:15:26.154 INFO:tasks.workunit.client.0.smithi078.stdout: modify_timestamp: Sun Oct 1 11:15:25 2023 2023-10-01T11:15:26.154 INFO:tasks.workunit.client.0.smithi078.stdout: protected: False 2023-10-01T11:15:26.157 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd image-meta list testimg_import 2023-10-01T11:15:26.167 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:26.179 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:25 smithi173 bash[58816]: cluster 2023-10-01T11:15:24.509832+0000 mgr.y (mgr.25072) 1746 : cluster 0 pgmap v1896: 73 pgs: 73 active+clean; 1.6 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 255 KiB/s rd, 105 KiB/s wr, 77 op/s 2023-10-01T11:15:26.180 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:25 smithi173 bash[58816]: cluster 2023-10-01T11:15:24.903254+0000 mon.a (mon.0) 2231 : cluster 3 Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:15:26.180 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:25 smithi173 bash[58816]: cluster 2023-10-01T11:15:24.912861+0000 osd.7 (osd.7) 441 : cluster 3 failed to encode map e392 with expected crc 2023-10-01T11:15:26.180 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:25 smithi173 bash[58816]: audit 2023-10-01T11:15:24.915112+0000 mon.a (mon.0) 2232 : audit 1 from='client.? 172.21.15.78:0/3218814586' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-10-01T11:15:26.180 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:25 smithi173 bash[58816]: cluster 2023-10-01T11:15:24.915185+0000 mon.a (mon.0) 2233 : cluster 0 osdmap e392: 8 total, 8 up, 8 in 2023-10-01T11:15:26.180 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:25 smithi173 bash[58816]: cluster 2023-10-01T11:15:24.979479+0000 osd.5 (osd.5) 420 : cluster 3 failed to encode map e392 with expected crc 2023-10-01T11:15:26.199 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:26.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:26.202 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:15:26.210 INFO:tasks.workunit.client.0.smithi078.stderr:+ IMAGEMETA_AFTER=There is 1 metadatum on this image: 2023-10-01T11:15:26.211 INFO:tasks.workunit.client.0.smithi078.stderr: 2023-10-01T11:15:26.211 INFO:tasks.workunit.client.0.smithi078.stderr:Key Value 2023-10-01T11:15:26.211 INFO:tasks.workunit.client.0.smithi078.stderr:key1 value1 2023-10-01T11:15:26.211 INFO:tasks.workunit.client.0.smithi078.stderr:+ [ There is 1 metadatum on this image: 2023-10-01T11:15:26.211 INFO:tasks.workunit.client.0.smithi078.stderr: 2023-10-01T11:15:26.211 INFO:tasks.workunit.client.0.smithi078.stderr:Key Value 2023-10-01T11:15:26.211 INFO:tasks.workunit.client.0.smithi078.stderr:key1 value1 = There is 1 metadatum on this image: 2023-10-01T11:15:26.211 INFO:tasks.workunit.client.0.smithi078.stderr: 2023-10-01T11:15:26.211 INFO:tasks.workunit.client.0.smithi078.stderr:Key Value 2023-10-01T11:15:26.211 INFO:tasks.workunit.client.0.smithi078.stderr:key1 value1 ] 2023-10-01T11:15:26.211 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd export testimg_import /tmp/rbd_import_export_259593/img_import 2023-10-01T11:15:26.259 INFO:tasks.workunit.client.0.smithi078.stderr: Exporting image: 100% complete...done. 2023-10-01T11:15:26.265 INFO:tasks.workunit.client.0.smithi078.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_259593/img /tmp/rbd_import_export_259593/img_import 2023-10-01T11:15:26.266 INFO:tasks.workunit.client.0.smithi078.stderr:+ cmp -l /tmp/rbd_import_export_259593/img /tmp/rbd_import_export_259593/img_import 2023-10-01T11:15:26.267 INFO:tasks.workunit.client.0.smithi078.stderr:+ stat /tmp/rbd_import_export_259593/img --format %b 2023-10-01T11:15:26.303 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:26.305 INFO:tasks.workunit.client.0.smithi078.stderr:+ origsize=24 2023-10-01T11:15:26.305 INFO:tasks.workunit.client.0.smithi078.stderr:+ stat /tmp/rbd_import_export_259593/img_import --format %b 2023-10-01T11:15:26.306 INFO:tasks.workunit.client.0.smithi078.stderr:+ exportsize=224 2023-10-01T11:15:26.306 INFO:tasks.workunit.client.0.smithi078.stderr:+ difference=200 2023-10-01T11:15:26.306 INFO:tasks.workunit.client.0.smithi078.stderr:+ difference=200 2023-10-01T11:15:26.306 INFO:tasks.workunit.client.0.smithi078.stderr:+ test 200 -ge 0 -a 200 -lt 4096 2023-10-01T11:15:26.306 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd export testimg@snap /tmp/rbd_import_export_259593/img_snap 2023-10-01T11:15:26.334 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:26.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:26.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:15:26.351 INFO:tasks.workunit.client.0.smithi078.stderr: Exporting image: 100% complete...done. 2023-10-01T11:15:26.356 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd export testimg_import@snap /tmp/rbd_import_export_259593/img_snap_import 2023-10-01T11:15:26.405 INFO:tasks.workunit.client.0.smithi078.stderr: Exporting image: 100% complete...done. 2023-10-01T11:15:26.408 INFO:tasks.workunit.client.0.smithi078.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_259593/img_snap /tmp/rbd_import_export_259593/img_snap_import 2023-10-01T11:15:26.408 INFO:tasks.workunit.client.0.smithi078.stderr:+ cmp -l /tmp/rbd_import_export_259593/img_snap /tmp/rbd_import_export_259593/img_snap_import 2023-10-01T11:15:26.410 INFO:tasks.workunit.client.0.smithi078.stderr:+ stat /tmp/rbd_import_export_259593/img_snap --format %b 2023-10-01T11:15:26.411 INFO:tasks.workunit.client.0.smithi078.stderr:+ origsize=224 2023-10-01T11:15:26.411 INFO:tasks.workunit.client.0.smithi078.stderr:+ stat /tmp/rbd_import_export_259593/img_snap_import --format %b 2023-10-01T11:15:26.412 INFO:tasks.workunit.client.0.smithi078.stderr:+ exportsize=224 2023-10-01T11:15:26.413 INFO:tasks.workunit.client.0.smithi078.stderr:+ difference=0 2023-10-01T11:15:26.413 INFO:tasks.workunit.client.0.smithi078.stderr:+ difference=0 2023-10-01T11:15:26.413 INFO:tasks.workunit.client.0.smithi078.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-01T11:15:26.413 INFO:tasks.workunit.client.0.smithi078.stderr:+ rm /tmp/rbd_import_export_259593/img_v2 2023-10-01T11:15:26.413 INFO:tasks.workunit.client.0.smithi078.stderr:+ rm /tmp/rbd_import_export_259593/img_import 2023-10-01T11:15:26.414 INFO:tasks.workunit.client.0.smithi078.stderr:+ rm /tmp/rbd_import_export_259593/img_snap 2023-10-01T11:15:26.414 INFO:tasks.workunit.client.0.smithi078.stderr:+ rm /tmp/rbd_import_export_259593/img_snap_import 2023-10-01T11:15:26.415 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd snap rm testimg_import@snap 2023-10-01T11:15:26.438 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:26.473 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:26.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:26.476 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:15:26.577 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:26.607 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:26.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:26.610 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:15:26.710 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:26.747 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:26.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:26.751 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:15:26.851 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:26.873 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:26.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:26.875 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:15:26.947 INFO:tasks.workunit.client.0.smithi078.stderr: Removing snap: 100% complete...done. 2023-10-01T11:15:26.957 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd remove testimg_import 2023-10-01T11:15:26.976 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:26.985 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:26 smithi173 bash[58816]: cluster 2023-10-01T11:15:25.917666+0000 osd.5 (osd.5) 421 : cluster 3 failed to encode map e393 with expected crc 2023-10-01T11:15:26.986 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:26 smithi173 bash[58816]: cluster 2023-10-01T11:15:25.921125+0000 mon.a (mon.0) 2234 : cluster 0 osdmap e393: 8 total, 8 up, 8 in 2023-10-01T11:15:27.004 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:27.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:27.007 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:15:27.092 INFO:tasks.workunit.client.0.smithi078.stderr: Removing image: 100% complete...done. 2023-10-01T11:15:27.097 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd snap rm testimg@snap 2023-10-01T11:15:27.108 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:27.139 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:27.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:27.142 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:15:27.243 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:27.277 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:27.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:27.280 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:15:27.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:26 smithi078 bash[75299]: cluster 2023-10-01T11:15:25.917666+0000 osd.5 (osd.5) 421 : cluster 3 failed to encode map e393 with expected crc 2023-10-01T11:15:27.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:26 smithi078 bash[75299]: cluster 2023-10-01T11:15:25.921125+0000 mon.a (mon.0) 2234 : cluster 0 osdmap e393: 8 total, 8 up, 8 in 2023-10-01T11:15:27.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:26 smithi078 bash[78274]: cluster 2023-10-01T11:15:25.917666+0000 osd.5 (osd.5) 421 : cluster 3 failed to encode map e393 with expected crc 2023-10-01T11:15:27.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:26 smithi078 bash[78274]: cluster 2023-10-01T11:15:25.921125+0000 mon.a (mon.0) 2234 : cluster 0 osdmap e393: 8 total, 8 up, 8 in 2023-10-01T11:15:27.380 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:27.405 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:27.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:27.408 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:15:27.509 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:27.543 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:27.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:27.547 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:15:27.648 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:27.677 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:27.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:27.679 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:15:27.780 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:27.812 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:27.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:27.816 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:15:27.917 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:27.952 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:27.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:27.956 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:15:27.957 INFO:tasks.workunit.client.0.smithi078.stderr: Removing snap: 100% complete...done. 2023-10-01T11:15:27.969 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd rm testimg 2023-10-01T11:15:28.057 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:28.068 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:27 smithi173 bash[58816]: cluster 2023-10-01T11:15:26.510583+0000 mgr.y (mgr.25072) 1747 : cluster 0 pgmap v1899: 73 pgs: 73 active+clean; 692 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 198 KiB/s rd, 21 KiB/s wr, 86 op/s 2023-10-01T11:15:28.069 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:27 smithi173 bash[58816]: cluster 2023-10-01T11:15:26.936504+0000 osd.5 (osd.5) 422 : cluster 3 failed to encode map e394 with expected crc 2023-10-01T11:15:28.069 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:27 smithi173 bash[58816]: cluster 2023-10-01T11:15:26.940380+0000 mon.a (mon.0) 2235 : cluster 0 osdmap e394: 8 total, 8 up, 8 in 2023-10-01T11:15:28.087 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:28.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:28.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:15:28.118 INFO:tasks.workunit.client.0.smithi078.stderr: Removing image: 100% complete...done. 2023-10-01T11:15:28.122 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd import --order 20 /tmp/rbd_import_export_259593/img testimg 2023-10-01T11:15:28.191 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:28.194 INFO:tasks.workunit.client.0.smithi078.stderr: Importing image: 100% complete...done. 2023-10-01T11:15:28.194 INFO:tasks.workunit.client.0.smithi078.stderr:rbd: --order is deprecated, use --object-size 2023-10-01T11:15:28.197 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_259593/img_v2 2023-10-01T11:15:28.198 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:27 smithi078 bash[75299]: cluster 2023-10-01T11:15:26.510583+0000 mgr.y (mgr.25072) 1747 : cluster 0 pgmap v1899: 73 pgs: 73 active+clean; 692 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 198 KiB/s rd, 21 KiB/s wr, 86 op/s 2023-10-01T11:15:28.199 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:27 smithi078 bash[75299]: cluster 2023-10-01T11:15:26.936504+0000 osd.5 (osd.5) 422 : cluster 3 failed to encode map e394 with expected crc 2023-10-01T11:15:28.199 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:27 smithi078 bash[75299]: cluster 2023-10-01T11:15:26.940380+0000 mon.a (mon.0) 2235 : cluster 0 osdmap e394: 8 total, 8 up, 8 in 2023-10-01T11:15:28.199 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:27 smithi078 bash[78274]: cluster 2023-10-01T11:15:26.510583+0000 mgr.y (mgr.25072) 1747 : cluster 0 pgmap v1899: 73 pgs: 73 active+clean; 692 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 198 KiB/s rd, 21 KiB/s wr, 86 op/s 2023-10-01T11:15:28.199 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:27 smithi078 bash[78274]: cluster 2023-10-01T11:15:26.936504+0000 osd.5 (osd.5) 422 : cluster 3 failed to encode map e394 with expected crc 2023-10-01T11:15:28.199 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:27 smithi078 bash[78274]: cluster 2023-10-01T11:15:26.940380+0000 mon.a (mon.0) 2235 : cluster 0 osdmap e394: 8 total, 8 up, 8 in 2023-10-01T11:15:28.211 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:28.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:28.213 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:15:28.240 INFO:tasks.workunit.client.0.smithi078.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-01T11:15:28.244 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_259593/img_v2 testimg_import 2023-10-01T11:15:28.314 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:28.318 INFO:tasks.workunit.client.0.smithi078.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-01T11:15:28.323 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd info testimg_import 2023-10-01T11:15:28.323 INFO:tasks.workunit.client.0.smithi078.stderr:+ grep order 2023-10-01T11:15:28.323 INFO:tasks.workunit.client.0.smithi078.stderr:+ awk {print $2} 2023-10-01T11:15:28.323 INFO:tasks.workunit.client.0.smithi078.stderr:+ grep 20 2023-10-01T11:15:28.343 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:28.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:28.346 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:15:28.447 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:15:28.471 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:15:28.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:28.473 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:15:28.574 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:28.606 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:28.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:28.608 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:15:28.709 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:15:28.739 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:15:28.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:28.742 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:15:28.829 INFO:tasks.workunit.client.0.smithi078.stdout:20 2023-10-01T11:15:28.829 INFO:tasks.workunit.client.0.smithi078.stderr:+ rm /tmp/rbd_import_export_259593/img_v2 2023-10-01T11:15:28.830 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd remove testimg_import 2023-10-01T11:15:28.843 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:28.870 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:28.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:28.872 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:15:28.960 INFO:tasks.workunit.client.0.smithi078.stderr: Removing image: 100% complete...done. 2023-10-01T11:15:28.963 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd remove testimg 2023-10-01T11:15:28.974 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:28.986 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:28 smithi078 bash[78274]: cluster 2023-10-01T11:15:27.945574+0000 mon.a (mon.0) 2236 : cluster 0 osdmap e395: 8 total, 8 up, 8 in 2023-10-01T11:15:28.987 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:28 smithi078 bash[78274]: cluster 2023-10-01T11:15:27.950198+0000 osd.7 (osd.7) 442 : cluster 3 failed to encode map e395 with expected crc 2023-10-01T11:15:28.987 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:28 smithi078 bash[78274]: cluster 2023-10-01T11:15:27.959739+0000 osd.6 (osd.6) 379 : cluster 3 failed to encode map e395 with expected crc 2023-10-01T11:15:28.987 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:28 smithi078 bash[78274]: cluster 2023-10-01T11:15:28.010078+0000 osd.5 (osd.5) 423 : cluster 3 failed to encode map e395 with expected crc 2023-10-01T11:15:28.987 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:28 smithi078 bash[75299]: cluster 2023-10-01T11:15:27.945574+0000 mon.a (mon.0) 2236 : cluster 0 osdmap e395: 8 total, 8 up, 8 in 2023-10-01T11:15:28.987 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:28 smithi078 bash[75299]: cluster 2023-10-01T11:15:27.950198+0000 osd.7 (osd.7) 442 : cluster 3 failed to encode map e395 with expected crc 2023-10-01T11:15:28.987 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:28 smithi078 bash[75299]: cluster 2023-10-01T11:15:27.959739+0000 osd.6 (osd.6) 379 : cluster 3 failed to encode map e395 with expected crc 2023-10-01T11:15:28.987 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:28 smithi078 bash[75299]: cluster 2023-10-01T11:15:28.010078+0000 osd.5 (osd.5) 423 : cluster 3 failed to encode map e395 with expected crc 2023-10-01T11:15:28.998 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:29.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:29.000 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:15:29.091 INFO:tasks.workunit.client.0.smithi078.stderr: Removing image: 100% complete...done. 2023-10-01T11:15:29.094 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd import --image-feature layering /tmp/rbd_import_export_259593/img testimg 2023-10-01T11:15:29.101 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:15:29.123 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:15:29.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:29.125 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:15:29.151 INFO:tasks.workunit.client.0.smithi078.stderr: Importing image: 100% complete...done. 2023-10-01T11:15:29.155 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd info testimg 2023-10-01T11:15:29.156 INFO:tasks.workunit.client.0.smithi078.stderr:+ grep features 2023-10-01T11:15:29.196 INFO:tasks.workunit.client.0.smithi078.stderr:+ FEATURES_BEFORE= features: layering 2023-10-01T11:15:29.196 INFO:tasks.workunit.client.0.smithi078.stderr: op_features: 2023-10-01T11:15:29.196 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_259593/img_v2 2023-10-01T11:15:29.227 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:29.240 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:28 smithi173 bash[58816]: cluster 2023-10-01T11:15:27.945574+0000 mon.a (mon.0) 2236 : cluster 0 osdmap e395: 8 total, 8 up, 8 in 2023-10-01T11:15:29.240 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:28 smithi173 bash[58816]: cluster 2023-10-01T11:15:27.950198+0000 osd.7 (osd.7) 442 : cluster 3 failed to encode map e395 with expected crc 2023-10-01T11:15:29.241 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:28 smithi173 bash[58816]: cluster 2023-10-01T11:15:27.959739+0000 osd.6 (osd.6) 379 : cluster 3 failed to encode map e395 with expected crc 2023-10-01T11:15:29.241 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:28 smithi173 bash[58816]: cluster 2023-10-01T11:15:28.010078+0000 osd.5 (osd.5) 423 : cluster 3 failed to encode map e395 with expected crc 2023-10-01T11:15:29.242 INFO:tasks.workunit.client.0.smithi078.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-01T11:15:29.246 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_259593/img_v2 testimg_import 2023-10-01T11:15:29.261 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:29.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:29.265 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:15:29.338 INFO:tasks.workunit.client.0.smithi078.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-01T11:15:29.343 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd info testimg_import 2023-10-01T11:15:29.343 INFO:tasks.workunit.client.0.smithi078.stderr:+ grep features 2023-10-01T11:15:29.365 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:29.390 INFO:tasks.workunit.client.0.smithi078.stderr:+ FEATURES_AFTER= features: layering 2023-10-01T11:15:29.390 INFO:tasks.workunit.client.0.smithi078.stderr: op_features: 2023-10-01T11:15:29.390 INFO:tasks.workunit.client.0.smithi078.stderr:+ [ features: layering 2023-10-01T11:15:29.390 INFO:tasks.workunit.client.0.smithi078.stderr: op_features: != features: layering 2023-10-01T11:15:29.391 INFO:tasks.workunit.client.0.smithi078.stderr: op_features: ] 2023-10-01T11:15:29.391 INFO:tasks.workunit.client.0.smithi078.stderr:+ rm /tmp/rbd_import_export_259593/img_v2 2023-10-01T11:15:29.391 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd remove testimg_import 2023-10-01T11:15:29.396 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:29.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:29.399 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:15:29.500 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:29.501 INFO:tasks.workunit.client.0.smithi078.stderr: Removing image: 100% complete...done. 2023-10-01T11:15:29.504 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd remove testimg 2023-10-01T11:15:29.528 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:29.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:29.531 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:15:29.605 INFO:tasks.workunit.client.0.smithi078.stderr: Removing image: 100% complete...done. 2023-10-01T11:15:29.609 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd import --stripe-count 1000 --stripe-unit 4096 /tmp/rbd_import_export_259593/img testimg 2023-10-01T11:15:29.631 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:29.660 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:29.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:29.664 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:15:29.692 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-10-01T11:15:29.701 INFO:tasks.workunit.client.0.smithi078.stderr: Importing image: 100% complete...done. 2023-10-01T11:15:29.705 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_259593/img_v2 2023-10-01T11:15:29.765 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:29.786 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:29.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:29.789 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:15:29.890 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:29.922 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:29.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:29.924 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:15:30.025 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:30.041 INFO:tasks.workunit.client.0.smithi078.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-01T11:15:30.045 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_259593/img_v2 testimg_import 2023-10-01T11:15:30.061 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:30.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:30.064 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:15:30.127 INFO:tasks.workunit.client.0.smithi078.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-01T11:15:30.132 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd info testimg_import 2023-10-01T11:15:30.132 INFO:tasks.workunit.client.0.smithi078.stderr:+ grep stripe unit 2023-10-01T11:15:30.132 INFO:tasks.workunit.client.0.smithi078.stderr:+ grep -Ei (4 KiB|4096) 2023-10-01T11:15:30.165 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:30.175 INFO:tasks.workunit.client.0.smithi078.stdout: stripe unit: 4 KiB 2023-10-01T11:15:30.176 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd info testimg_import 2023-10-01T11:15:30.176 INFO:tasks.workunit.client.0.smithi078.stderr:+ grep stripe count 2023-10-01T11:15:30.176 INFO:tasks.workunit.client.0.smithi078.stderr:+ awk {print $3} 2023-10-01T11:15:30.176 INFO:tasks.workunit.client.0.smithi078.stderr:+ grep 1000 2023-10-01T11:15:30.188 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:30.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:30.190 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:15:30.217 INFO:tasks.workunit.client.0.smithi078.stdout:1000 2023-10-01T11:15:30.217 INFO:tasks.workunit.client.0.smithi078.stderr:+ rm /tmp/rbd_import_export_259593/img_v2 2023-10-01T11:15:30.217 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd remove testimg_import 2023-10-01T11:15:30.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:30 smithi078 bash[78274]: cluster 2023-10-01T11:15:28.511348+0000 mgr.y (mgr.25072) 1748 : cluster 0 pgmap v1902: 73 pgs: 73 active+clean; 802 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 82 KiB/s rd, 9.5 KiB/s wr, 103 op/s 2023-10-01T11:15:30.270 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:30 smithi078 bash[75299]: cluster 2023-10-01T11:15:28.511348+0000 mgr.y (mgr.25072) 1748 : cluster 0 pgmap v1902: 73 pgs: 73 active+clean; 802 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 82 KiB/s rd, 9.5 KiB/s wr, 103 op/s 2023-10-01T11:15:30.291 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:30.303 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:29 smithi173 bash[58816]: cluster 2023-10-01T11:15:28.511348+0000 mgr.y (mgr.25072) 1748 : cluster 0 pgmap v1902: 73 pgs: 73 active+clean; 802 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 82 KiB/s rd, 9.5 KiB/s wr, 103 op/s 2023-10-01T11:15:30.322 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:30.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:30.325 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:15:30.337 INFO:tasks.workunit.client.0.smithi078.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-10-01T11:15:30.342 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd remove testimg 2023-10-01T11:15:30.426 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:30.452 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:30.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:30.455 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:15:30.490 INFO:tasks.workunit.client.0.smithi078.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-10-01T11:15:30.496 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd import --image-format=2 /tmp/rbd_import_export_259593/img testimg 2023-10-01T11:15:30.555 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:30.576 INFO:tasks.workunit.client.0.smithi078.stderr: Importing image: 100% complete...done. 2023-10-01T11:15:30.580 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd snap create testimg@snap1 2023-10-01T11:15:30.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:15:30 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:15:30] "GET /metrics HTTP/1.1" 200 34006 "" "Prometheus/2.33.4" 2023-10-01T11:15:30.587 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:30.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:30.589 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:15:30.690 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:30.720 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:30.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:30.723 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:15:30.824 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:30.851 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:30.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:30.854 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:15:30.955 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:30.981 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:30.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:30.983 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:15:30.990 INFO:tasks.workunit.client.0.smithi078.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-10-01T11:15:31.002 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd snap create testimg@snap2 2023-10-01T11:15:31.084 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:31.117 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:31.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:31.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:15:31.220 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:15:31.247 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:15:31.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:31.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:15:31.351 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:31.384 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:31.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:31.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:15:31.488 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:31.520 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:31.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:31.523 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:15:31.624 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:31.657 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:31.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:31.660 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:15:31.761 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:15:31.786 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:15:31.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:31.790 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:15:31.891 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:31.924 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:31.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:31.927 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:15:31.942 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:15:31.996 INFO:tasks.workunit.client.0.smithi078.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-10-01T11:15:32.005 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd snap protect testimg@snap2 2023-10-01T11:15:32.027 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:32.058 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_259593/snap_protect 2023-10-01T11:15:32.062 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:32.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:32.065 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:15:32.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:31 smithi078 bash[75299]: cluster 2023-10-01T11:15:30.512087+0000 mgr.y (mgr.25072) 1749 : cluster 0 pgmap v1903: 73 pgs: 73 active+clean; 802 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 243 KiB/s rd, 25 KiB/s wr, 319 op/s 2023-10-01T11:15:32.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:31 smithi078 bash[75299]: cluster 2023-10-01T11:15:30.967031+0000 mon.a (mon.0) 2237 : cluster 0 osdmap e396: 8 total, 8 up, 8 in 2023-10-01T11:15:32.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:31 smithi078 bash[75299]: cluster 2023-10-01T11:15:30.968295+0000 osd.7 (osd.7) 443 : cluster 3 failed to encode map e396 with expected crc 2023-10-01T11:15:32.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:31 smithi078 bash[75299]: cluster 2023-10-01T11:15:30.976122+0000 osd.5 (osd.5) 424 : cluster 3 failed to encode map e396 with expected crc 2023-10-01T11:15:32.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:31 smithi078 bash[78274]: cluster 2023-10-01T11:15:30.512087+0000 mgr.y (mgr.25072) 1749 : cluster 0 pgmap v1903: 73 pgs: 73 active+clean; 802 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 243 KiB/s rd, 25 KiB/s wr, 319 op/s 2023-10-01T11:15:32.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:31 smithi078 bash[78274]: cluster 2023-10-01T11:15:30.967031+0000 mon.a (mon.0) 2237 : cluster 0 osdmap e396: 8 total, 8 up, 8 in 2023-10-01T11:15:32.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:31 smithi078 bash[78274]: cluster 2023-10-01T11:15:30.968295+0000 osd.7 (osd.7) 443 : cluster 3 failed to encode map e396 with expected crc 2023-10-01T11:15:32.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:31 smithi078 bash[78274]: cluster 2023-10-01T11:15:30.976122+0000 osd.5 (osd.5) 424 : cluster 3 failed to encode map e396 with expected crc 2023-10-01T11:15:32.109 INFO:tasks.workunit.client.0.smithi078.stderr: Exporting image: 33% complete... Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-10-01T11:15:32.114 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_259593/snap_protect testimg_import 2023-10-01T11:15:32.166 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:32.190 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:32.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:32.192 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:15:32.293 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:32.323 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:32.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:32.325 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:15:32.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:31 smithi173 bash[58816]: cluster 2023-10-01T11:15:30.512087+0000 mgr.y (mgr.25072) 1749 : cluster 0 pgmap v1903: 73 pgs: 73 active+clean; 802 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 243 KiB/s rd, 25 KiB/s wr, 319 op/s 2023-10-01T11:15:32.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:31 smithi173 bash[58816]: cluster 2023-10-01T11:15:30.967031+0000 mon.a (mon.0) 2237 : cluster 0 osdmap e396: 8 total, 8 up, 8 in 2023-10-01T11:15:32.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:31 smithi173 bash[58816]: cluster 2023-10-01T11:15:30.968295+0000 osd.7 (osd.7) 443 : cluster 3 failed to encode map e396 with expected crc 2023-10-01T11:15:32.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:31 smithi173 bash[58816]: cluster 2023-10-01T11:15:30.976122+0000 osd.5 (osd.5) 424 : cluster 3 failed to encode map e396 with expected crc 2023-10-01T11:15:32.426 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:15:32.452 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:15:32.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:32.455 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:15:32.555 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:32.584 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:32.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:32.586 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:15:32.687 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:32.713 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:32.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:32.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:15:32.816 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:32.842 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:32.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:32.844 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:15:32.945 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:32.979 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:32.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:32.982 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:15:33.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:32 smithi078 bash[75299]: cluster 2023-10-01T11:15:31.979362+0000 osd.7 (osd.7) 444 : cluster 3 failed to encode map e397 with expected crc 2023-10-01T11:15:33.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:32 smithi078 bash[75299]: cluster 2023-10-01T11:15:31.980650+0000 mon.a (mon.0) 2238 : cluster 0 osdmap e397: 8 total, 8 up, 8 in 2023-10-01T11:15:33.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:32 smithi078 bash[75299]: cluster 2023-10-01T11:15:31.986217+0000 osd.5 (osd.5) 425 : cluster 3 failed to encode map e397 with expected crc 2023-10-01T11:15:33.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:32 smithi078 bash[75299]: audit 2023-10-01T11:15:32.487307+0000 mon.a (mon.0) 2239 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:15:33.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:32 smithi078 bash[75299]: audit 2023-10-01T11:15:32.488391+0000 mon.a (mon.0) 2240 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:15:33.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:32 smithi078 bash[75299]: cluster 2023-10-01T11:15:32.907805+0000 mon.a (mon.0) 2241 : cluster 0 osdmap e398: 8 total, 8 up, 8 in 2023-10-01T11:15:33.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:33 smithi078 bash[78274]: cluster 2023-10-01T11:15:31.979362+0000 osd.7 (osd.7) 444 : cluster 3 failed to encode map e397 with expected crc 2023-10-01T11:15:33.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:33 smithi078 bash[78274]: cluster 2023-10-01T11:15:31.980650+0000 mon.a (mon.0) 2238 : cluster 0 osdmap e397: 8 total, 8 up, 8 in 2023-10-01T11:15:33.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:33 smithi078 bash[78274]: cluster 2023-10-01T11:15:31.986217+0000 osd.5 (osd.5) 425 : cluster 3 failed to encode map e397 with expected crc 2023-10-01T11:15:33.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:33 smithi078 bash[78274]: audit 2023-10-01T11:15:32.487307+0000 mon.a (mon.0) 2239 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:15:33.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:33 smithi078 bash[78274]: audit 2023-10-01T11:15:32.488391+0000 mon.a (mon.0) 2240 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:15:33.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:33 smithi078 bash[78274]: cluster 2023-10-01T11:15:32.907805+0000 mon.a (mon.0) 2241 : cluster 0 osdmap e398: 8 total, 8 up, 8 in 2023-10-01T11:15:33.083 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:33.114 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:33.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:33.116 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:15:33.217 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:33.239 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:33.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:33.241 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:15:33.342 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:33.356 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:32 smithi173 bash[58816]: cluster 2023-10-01T11:15:31.979362+0000 osd.7 (osd.7) 444 : cluster 3 failed to encode map e397 with expected crc 2023-10-01T11:15:33.356 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:32 smithi173 bash[58816]: cluster 2023-10-01T11:15:31.980650+0000 mon.a (mon.0) 2238 : cluster 0 osdmap e397: 8 total, 8 up, 8 in 2023-10-01T11:15:33.357 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:32 smithi173 bash[58816]: cluster 2023-10-01T11:15:31.986217+0000 osd.5 (osd.5) 425 : cluster 3 failed to encode map e397 with expected crc 2023-10-01T11:15:33.357 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:32 smithi173 bash[58816]: audit 2023-10-01T11:15:32.487307+0000 mon.a (mon.0) 2239 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:15:33.357 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:32 smithi173 bash[58816]: audit 2023-10-01T11:15:32.488391+0000 mon.a (mon.0) 2240 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:15:33.357 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:32 smithi173 bash[58816]: cluster 2023-10-01T11:15:32.907805+0000 mon.a (mon.0) 2241 : cluster 0 osdmap e398: 8 total, 8 up, 8 in 2023-10-01T11:15:33.374 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:33.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:33.378 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:15:33.478 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:33.503 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:33.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:33.506 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:15:33.606 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:33.632 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:33.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:33.634 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:15:33.735 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:33.759 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:33.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:33.762 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:15:33.862 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:33.894 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:33.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:33.898 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:15:33.944 INFO:tasks.workunit.client.0.smithi078.stderr: Importing image: 33% complete... Importing image: 66% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-10-01T11:15:33.948 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd info testimg_import@snap1 2023-10-01T11:15:33.948 INFO:tasks.workunit.client.0.smithi078.stderr:+ grep protected: False 2023-10-01T11:15:33.988 INFO:tasks.workunit.client.0.smithi078.stdout: protected: False 2023-10-01T11:15:33.989 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd info testimg_import@snap2 2023-10-01T11:15:33.989 INFO:tasks.workunit.client.0.smithi078.stderr:+ grep protected: True 2023-10-01T11:15:33.999 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:34.026 INFO:tasks.workunit.client.0.smithi078.stdout: protected: True 2023-10-01T11:15:34.026 INFO:tasks.workunit.client.0.smithi078.stderr:+ rm /tmp/rbd_import_export_259593/snap_protect 2023-10-01T11:15:34.027 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd snap unprotect testimg@snap2 2023-10-01T11:15:34.031 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:34.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:34.034 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:15:34.078 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd snap unprotect testimg_import@snap2 2023-10-01T11:15:34.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:33 smithi078 bash[75299]: cluster 2023-10-01T11:15:32.512696+0000 mgr.y (mgr.25072) 1750 : cluster 0 pgmap v1906: 73 pgs: 73 active+clean; 802 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 241 KiB/s rd, 22 KiB/s wr, 313 op/s 2023-10-01T11:15:34.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:33 smithi078 bash[75299]: cluster 2023-10-01T11:15:32.907482+0000 osd.7 (osd.7) 445 : cluster 3 failed to encode map e398 with expected crc 2023-10-01T11:15:34.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:33 smithi078 bash[75299]: cluster 2023-10-01T11:15:32.917811+0000 osd.7 (osd.7) 446 : cluster 3 failed to encode map e398 with expected crc 2023-10-01T11:15:34.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:33 smithi078 bash[75299]: cluster 2023-10-01T11:15:32.920707+0000 osd.5 (osd.5) 426 : cluster 3 failed to encode map e398 with expected crc 2023-10-01T11:15:34.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:33 smithi078 bash[75299]: cluster 2023-10-01T11:15:32.927381+0000 osd.6 (osd.6) 380 : cluster 3 failed to encode map e398 with expected crc 2023-10-01T11:15:34.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:33 smithi078 bash[75299]: cluster 2023-10-01T11:15:33.918182+0000 mon.a (mon.0) 2242 : cluster 0 osdmap e399: 8 total, 8 up, 8 in 2023-10-01T11:15:34.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:33 smithi078 bash[78274]: cluster 2023-10-01T11:15:32.512696+0000 mgr.y (mgr.25072) 1750 : cluster 0 pgmap v1906: 73 pgs: 73 active+clean; 802 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 241 KiB/s rd, 22 KiB/s wr, 313 op/s 2023-10-01T11:15:34.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:33 smithi078 bash[78274]: cluster 2023-10-01T11:15:32.907482+0000 osd.7 (osd.7) 445 : cluster 3 failed to encode map e398 with expected crc 2023-10-01T11:15:34.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:33 smithi078 bash[78274]: cluster 2023-10-01T11:15:32.917811+0000 osd.7 (osd.7) 446 : cluster 3 failed to encode map e398 with expected crc 2023-10-01T11:15:34.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:33 smithi078 bash[78274]: cluster 2023-10-01T11:15:32.920707+0000 osd.5 (osd.5) 426 : cluster 3 failed to encode map e398 with expected crc 2023-10-01T11:15:34.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:33 smithi078 bash[78274]: cluster 2023-10-01T11:15:32.927381+0000 osd.6 (osd.6) 380 : cluster 3 failed to encode map e398 with expected crc 2023-10-01T11:15:34.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:33 smithi078 bash[78274]: cluster 2023-10-01T11:15:33.918182+0000 mon.a (mon.0) 2242 : cluster 0 osdmap e399: 8 total, 8 up, 8 in 2023-10-01T11:15:34.127 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd snap purge testimg 2023-10-01T11:15:34.135 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:34.160 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:34.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:34.162 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:15:34.260 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:15:34.263 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:34.286 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:34.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:34.288 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:15:34.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:33 smithi173 bash[58816]: cluster 2023-10-01T11:15:32.512696+0000 mgr.y (mgr.25072) 1750 : cluster 0 pgmap v1906: 73 pgs: 73 active+clean; 802 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 241 KiB/s rd, 22 KiB/s wr, 313 op/s 2023-10-01T11:15:34.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:33 smithi173 bash[58816]: cluster 2023-10-01T11:15:32.907482+0000 osd.7 (osd.7) 445 : cluster 3 failed to encode map e398 with expected crc 2023-10-01T11:15:34.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:33 smithi173 bash[58816]: cluster 2023-10-01T11:15:32.917811+0000 osd.7 (osd.7) 446 : cluster 3 failed to encode map e398 with expected crc 2023-10-01T11:15:34.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:33 smithi173 bash[58816]: cluster 2023-10-01T11:15:32.920707+0000 osd.5 (osd.5) 426 : cluster 3 failed to encode map e398 with expected crc 2023-10-01T11:15:34.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:33 smithi173 bash[58816]: cluster 2023-10-01T11:15:32.927381+0000 osd.6 (osd.6) 380 : cluster 3 failed to encode map e398 with expected crc 2023-10-01T11:15:34.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:33 smithi173 bash[58816]: cluster 2023-10-01T11:15:33.918182+0000 mon.a (mon.0) 2242 : cluster 0 osdmap e399: 8 total, 8 up, 8 in 2023-10-01T11:15:34.388 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:34.415 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:34.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:34.419 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:15:34.519 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:34.549 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:34.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:34.552 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:15:34.653 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:34.685 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:34.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:34.688 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:15:34.789 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:34.819 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:34.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:34.821 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:15:34.922 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:34.958 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:34.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:34.962 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:15:34.998 INFO:teuthology.orchestra.run.smithi078.stderr:set osd.1 primary-affinity to 0.0999567 (865502) 2023-10-01T11:15:35.063 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:35.103 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:35.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:35.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:15:35.112 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:34 smithi078 bash[75299]: cluster 2023-10-01T11:15:33.917656+0000 osd.5 (osd.5) 427 : cluster 3 failed to encode map e399 with expected crc 2023-10-01T11:15:35.112 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:34 smithi078 bash[75299]: cluster 2023-10-01T11:15:33.918188+0000 osd.7 (osd.7) 447 : cluster 3 failed to encode map e399 with expected crc 2023-10-01T11:15:35.112 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:34 smithi078 bash[75299]: cluster 2023-10-01T11:15:33.929423+0000 osd.6 (osd.6) 381 : cluster 3 failed to encode map e399 with expected crc 2023-10-01T11:15:35.112 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:34 smithi078 bash[75299]: audit 2023-10-01T11:15:34.179955+0000 mon.c (mon.2) 252 : audit 1 from='client.? 172.21.15.78:0/39972016' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.09995667370632022}]: dispatch 2023-10-01T11:15:35.112 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:34 smithi078 bash[75299]: audit 2023-10-01T11:15:34.180330+0000 mon.a (mon.0) 2243 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.09995667370632022}]: dispatch 2023-10-01T11:15:35.112 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:34 smithi078 bash[78274]: cluster 2023-10-01T11:15:33.917656+0000 osd.5 (osd.5) 427 : cluster 3 failed to encode map e399 with expected crc 2023-10-01T11:15:35.113 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:34 smithi078 bash[78274]: cluster 2023-10-01T11:15:33.918188+0000 osd.7 (osd.7) 447 : cluster 3 failed to encode map e399 with expected crc 2023-10-01T11:15:35.113 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:34 smithi078 bash[78274]: cluster 2023-10-01T11:15:33.929423+0000 osd.6 (osd.6) 381 : cluster 3 failed to encode map e399 with expected crc 2023-10-01T11:15:35.113 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:34 smithi078 bash[78274]: audit 2023-10-01T11:15:34.179955+0000 mon.c (mon.2) 252 : audit 1 from='client.? 172.21.15.78:0/39972016' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.09995667370632022}]: dispatch 2023-10-01T11:15:35.113 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:34 smithi078 bash[78274]: audit 2023-10-01T11:15:34.180330+0000 mon.a (mon.0) 2243 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.09995667370632022}]: dispatch 2023-10-01T11:15:35.208 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:35.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:35.244 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:35.244 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:15:35.345 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:35.363 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:34 smithi173 bash[58816]: cluster 2023-10-01T11:15:33.917656+0000 osd.5 (osd.5) 427 : cluster 3 failed to encode map e399 with expected crc 2023-10-01T11:15:35.363 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:34 smithi173 bash[58816]: cluster 2023-10-01T11:15:33.918188+0000 osd.7 (osd.7) 447 : cluster 3 failed to encode map e399 with expected crc 2023-10-01T11:15:35.363 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:34 smithi173 bash[58816]: cluster 2023-10-01T11:15:33.929423+0000 osd.6 (osd.6) 381 : cluster 3 failed to encode map e399 with expected crc 2023-10-01T11:15:35.363 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:34 smithi173 bash[58816]: audit 2023-10-01T11:15:34.179955+0000 mon.c (mon.2) 252 : audit 1 from='client.? 172.21.15.78:0/39972016' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.09995667370632022}]: dispatch 2023-10-01T11:15:35.363 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:34 smithi173 bash[58816]: audit 2023-10-01T11:15:34.180330+0000 mon.a (mon.0) 2243 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.09995667370632022}]: dispatch 2023-10-01T11:15:35.381 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:35.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:35.384 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:15:35.484 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:35.520 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:35.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:35.523 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:15:35.624 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:35.662 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:35.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:35.664 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:15:35.765 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:35.810 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:35.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:35.814 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:15:35.916 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:35.959 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:35.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:35.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:15:36.063 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:36.078 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:35 smithi173 bash[58816]: cluster 2023-10-01T11:15:34.513633+0000 mgr.y (mgr.25072) 1751 : cluster 0 pgmap v1909: 73 pgs: 73 active+clean; 802 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 156 KiB/s rd, 24 KiB/s wr, 206 op/s 2023-10-01T11:15:36.078 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:35 smithi173 bash[58816]: cluster 2023-10-01T11:15:34.985612+0000 osd.6 (osd.6) 382 : cluster 3 failed to encode map e400 with expected crc 2023-10-01T11:15:36.078 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:35 smithi173 bash[58816]: cluster 2023-10-01T11:15:34.987877+0000 osd.6 (osd.6) 383 : cluster 3 failed to encode map e400 with expected crc 2023-10-01T11:15:36.078 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:35 smithi173 bash[58816]: audit 2023-10-01T11:15:34.988363+0000 mon.a (mon.0) 2244 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.09995667370632022}]': finished 2023-10-01T11:15:36.078 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:35 smithi173 bash[58816]: cluster 2023-10-01T11:15:34.988404+0000 osd.7 (osd.7) 448 : cluster 3 failed to encode map e400 with expected crc 2023-10-01T11:15:36.078 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:35 smithi173 bash[58816]: cluster 2023-10-01T11:15:34.988521+0000 mon.a (mon.0) 2245 : cluster 0 osdmap e400: 8 total, 8 up, 8 in 2023-10-01T11:15:36.078 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:35 smithi173 bash[58816]: cluster 2023-10-01T11:15:34.988855+0000 osd.5 (osd.5) 428 : cluster 3 failed to encode map e400 with expected crc 2023-10-01T11:15:36.078 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:35 smithi173 bash[58816]: cluster 2023-10-01T11:15:34.990054+0000 osd.6 (osd.6) 384 : cluster 3 failed to encode map e400 with expected crc 2023-10-01T11:15:36.097 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:36.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:36.099 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:15:36.200 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:36.232 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:36.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:36.235 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:15:36.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:35 smithi078 bash[78274]: cluster 2023-10-01T11:15:34.513633+0000 mgr.y (mgr.25072) 1751 : cluster 0 pgmap v1909: 73 pgs: 73 active+clean; 802 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 156 KiB/s rd, 24 KiB/s wr, 206 op/s 2023-10-01T11:15:36.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:35 smithi078 bash[78274]: cluster 2023-10-01T11:15:34.985612+0000 osd.6 (osd.6) 382 : cluster 3 failed to encode map e400 with expected crc 2023-10-01T11:15:36.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:35 smithi078 bash[78274]: cluster 2023-10-01T11:15:34.987877+0000 osd.6 (osd.6) 383 : cluster 3 failed to encode map e400 with expected crc 2023-10-01T11:15:36.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:35 smithi078 bash[78274]: audit 2023-10-01T11:15:34.988363+0000 mon.a (mon.0) 2244 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.09995667370632022}]': finished 2023-10-01T11:15:36.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:35 smithi078 bash[78274]: cluster 2023-10-01T11:15:34.988404+0000 osd.7 (osd.7) 448 : cluster 3 failed to encode map e400 with expected crc 2023-10-01T11:15:36.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:35 smithi078 bash[78274]: cluster 2023-10-01T11:15:34.988521+0000 mon.a (mon.0) 2245 : cluster 0 osdmap e400: 8 total, 8 up, 8 in 2023-10-01T11:15:36.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:35 smithi078 bash[78274]: cluster 2023-10-01T11:15:34.988855+0000 osd.5 (osd.5) 428 : cluster 3 failed to encode map e400 with expected crc 2023-10-01T11:15:36.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:35 smithi078 bash[78274]: cluster 2023-10-01T11:15:34.990054+0000 osd.6 (osd.6) 384 : cluster 3 failed to encode map e400 with expected crc 2023-10-01T11:15:36.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:35 smithi078 bash[75299]: cluster 2023-10-01T11:15:34.513633+0000 mgr.y (mgr.25072) 1751 : cluster 0 pgmap v1909: 73 pgs: 73 active+clean; 802 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 156 KiB/s rd, 24 KiB/s wr, 206 op/s 2023-10-01T11:15:36.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:35 smithi078 bash[75299]: cluster 2023-10-01T11:15:34.985612+0000 osd.6 (osd.6) 382 : cluster 3 failed to encode map e400 with expected crc 2023-10-01T11:15:36.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:35 smithi078 bash[75299]: cluster 2023-10-01T11:15:34.987877+0000 osd.6 (osd.6) 383 : cluster 3 failed to encode map e400 with expected crc 2023-10-01T11:15:36.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:35 smithi078 bash[75299]: audit 2023-10-01T11:15:34.988363+0000 mon.a (mon.0) 2244 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.09995667370632022}]': finished 2023-10-01T11:15:36.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:35 smithi078 bash[75299]: cluster 2023-10-01T11:15:34.988404+0000 osd.7 (osd.7) 448 : cluster 3 failed to encode map e400 with expected crc 2023-10-01T11:15:36.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:35 smithi078 bash[75299]: cluster 2023-10-01T11:15:34.988521+0000 mon.a (mon.0) 2245 : cluster 0 osdmap e400: 8 total, 8 up, 8 in 2023-10-01T11:15:36.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:35 smithi078 bash[75299]: cluster 2023-10-01T11:15:34.988855+0000 osd.5 (osd.5) 428 : cluster 3 failed to encode map e400 with expected crc 2023-10-01T11:15:36.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:35 smithi078 bash[75299]: cluster 2023-10-01T11:15:34.990054+0000 osd.6 (osd.6) 384 : cluster 3 failed to encode map e400 with expected crc 2023-10-01T11:15:36.335 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:36.368 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:36.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:36.370 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:15:36.471 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:15:36.505 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:15:36.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:36.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:15:36.608 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:15:36.636 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:15:36.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:36.638 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:15:36.739 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:15:36.772 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:15:36.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:36.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:15:36.875 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:36.911 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:36.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:36.914 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:15:37.015 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:37.024 INFO:tasks.workunit.client.0.smithi078.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-10-01T11:15:37.034 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd snap purge testimg_import 2023-10-01T11:15:37.045 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:37.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:37.048 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:15:37.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:37 smithi078 bash[78274]: cluster 2023-10-01T11:15:35.996345+0000 osd.6 (osd.6) 385 : cluster 3 failed to encode map e401 with expected crc 2023-10-01T11:15:37.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:37 smithi078 bash[78274]: cluster 2023-10-01T11:15:35.996386+0000 osd.7 (osd.7) 449 : cluster 3 failed to encode map e401 with expected crc 2023-10-01T11:15:37.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:37 smithi078 bash[78274]: cluster 2023-10-01T11:15:35.997337+0000 mon.a (mon.0) 2246 : cluster 0 osdmap e401: 8 total, 8 up, 8 in 2023-10-01T11:15:37.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:37 smithi078 bash[78274]: cluster 2023-10-01T11:15:35.999089+0000 osd.5 (osd.5) 429 : cluster 3 failed to encode map e401 with expected crc 2023-10-01T11:15:37.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:37 smithi078 bash[78274]: cluster 2023-10-01T11:15:35.999656+0000 osd.7 (osd.7) 450 : cluster 3 failed to encode map e401 with expected crc 2023-10-01T11:15:37.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:37 smithi078 bash[78274]: cluster 2023-10-01T11:15:35.999840+0000 osd.6 (osd.6) 386 : cluster 3 failed to encode map e401 with expected crc 2023-10-01T11:15:37.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:37 smithi078 bash[78274]: cluster 2023-10-01T11:15:36.001080+0000 osd.5 (osd.5) 430 : cluster 3 failed to encode map e401 with expected crc 2023-10-01T11:15:37.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:37 smithi078 bash[78274]: cluster 2023-10-01T11:15:36.002780+0000 osd.6 (osd.6) 387 : cluster 3 failed to encode map e401 with expected crc 2023-10-01T11:15:37.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:37 smithi078 bash[78274]: cluster 2023-10-01T11:15:36.003969+0000 osd.5 (osd.5) 431 : cluster 3 failed to encode map e401 with expected crc 2023-10-01T11:15:37.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:37 smithi078 bash[75299]: cluster 2023-10-01T11:15:35.996345+0000 osd.6 (osd.6) 385 : cluster 3 failed to encode map e401 with expected crc 2023-10-01T11:15:37.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:37 smithi078 bash[75299]: cluster 2023-10-01T11:15:35.996386+0000 osd.7 (osd.7) 449 : cluster 3 failed to encode map e401 with expected crc 2023-10-01T11:15:37.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:37 smithi078 bash[75299]: cluster 2023-10-01T11:15:35.997337+0000 mon.a (mon.0) 2246 : cluster 0 osdmap e401: 8 total, 8 up, 8 in 2023-10-01T11:15:37.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:37 smithi078 bash[75299]: cluster 2023-10-01T11:15:35.999089+0000 osd.5 (osd.5) 429 : cluster 3 failed to encode map e401 with expected crc 2023-10-01T11:15:37.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:37 smithi078 bash[75299]: cluster 2023-10-01T11:15:35.999656+0000 osd.7 (osd.7) 450 : cluster 3 failed to encode map e401 with expected crc 2023-10-01T11:15:37.083 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:37 smithi078 bash[75299]: cluster 2023-10-01T11:15:35.999840+0000 osd.6 (osd.6) 386 : cluster 3 failed to encode map e401 with expected crc 2023-10-01T11:15:37.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:37 smithi078 bash[75299]: cluster 2023-10-01T11:15:36.001080+0000 osd.5 (osd.5) 430 : cluster 3 failed to encode map e401 with expected crc 2023-10-01T11:15:37.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:37 smithi078 bash[75299]: cluster 2023-10-01T11:15:36.002780+0000 osd.6 (osd.6) 387 : cluster 3 failed to encode map e401 with expected crc 2023-10-01T11:15:37.084 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:37 smithi078 bash[75299]: cluster 2023-10-01T11:15:36.003969+0000 osd.5 (osd.5) 431 : cluster 3 failed to encode map e401 with expected crc 2023-10-01T11:15:37.149 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:37.166 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:37 smithi173 bash[58816]: cluster 2023-10-01T11:15:35.996345+0000 osd.6 (osd.6) 385 : cluster 3 failed to encode map e401 with expected crc 2023-10-01T11:15:37.166 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:37 smithi173 bash[58816]: cluster 2023-10-01T11:15:35.996386+0000 osd.7 (osd.7) 449 : cluster 3 failed to encode map e401 with expected crc 2023-10-01T11:15:37.166 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:37 smithi173 bash[58816]: cluster 2023-10-01T11:15:35.997337+0000 mon.a (mon.0) 2246 : cluster 0 osdmap e401: 8 total, 8 up, 8 in 2023-10-01T11:15:37.166 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:37 smithi173 bash[58816]: cluster 2023-10-01T11:15:35.999089+0000 osd.5 (osd.5) 429 : cluster 3 failed to encode map e401 with expected crc 2023-10-01T11:15:37.166 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:37 smithi173 bash[58816]: cluster 2023-10-01T11:15:35.999656+0000 osd.7 (osd.7) 450 : cluster 3 failed to encode map e401 with expected crc 2023-10-01T11:15:37.166 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:37 smithi173 bash[58816]: cluster 2023-10-01T11:15:35.999840+0000 osd.6 (osd.6) 386 : cluster 3 failed to encode map e401 with expected crc 2023-10-01T11:15:37.166 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:37 smithi173 bash[58816]: cluster 2023-10-01T11:15:36.001080+0000 osd.5 (osd.5) 430 : cluster 3 failed to encode map e401 with expected crc 2023-10-01T11:15:37.166 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:37 smithi173 bash[58816]: cluster 2023-10-01T11:15:36.002780+0000 osd.6 (osd.6) 387 : cluster 3 failed to encode map e401 with expected crc 2023-10-01T11:15:37.167 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:37 smithi173 bash[58816]: cluster 2023-10-01T11:15:36.003969+0000 osd.5 (osd.5) 431 : cluster 3 failed to encode map e401 with expected crc 2023-10-01T11:15:37.182 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:37.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:37.186 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:15:37.286 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:37.319 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:37.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:37.322 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:15:37.423 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:37.455 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:37.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:37.459 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:15:37.560 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:37.593 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:37.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:37.597 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:15:37.699 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:37.738 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:37.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:37.741 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:15:37.842 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:37.865 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:37.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:37.868 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:15:37.969 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:37.992 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:37.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:37.995 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:15:38.097 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:38.112 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:38 smithi078 bash[78274]: cluster 2023-10-01T11:15:36.514317+0000 mgr.y (mgr.25072) 1752 : cluster 0 pgmap v1912: 73 pgs: 73 active+clean; 802 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 156 KiB/s rd, 24 KiB/s wr, 206 op/s 2023-10-01T11:15:38.112 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:38 smithi078 bash[78274]: cluster 2023-10-01T11:15:37.009775+0000 osd.5 (osd.5) 432 : cluster 3 failed to encode map e402 with expected crc 2023-10-01T11:15:38.112 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:38 smithi078 bash[78274]: cluster 2023-10-01T11:15:37.012038+0000 mon.a (mon.0) 2247 : cluster 0 osdmap e402: 8 total, 8 up, 8 in 2023-10-01T11:15:38.113 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:38 smithi078 bash[78274]: cluster 2023-10-01T11:15:37.015142+0000 osd.7 (osd.7) 451 : cluster 3 failed to encode map e402 with expected crc 2023-10-01T11:15:38.113 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:38 smithi078 bash[78274]: cluster 2023-10-01T11:15:37.086722+0000 osd.6 (osd.6) 388 : cluster 3 failed to encode map e402 with expected crc 2023-10-01T11:15:38.113 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:38 smithi078 bash[75299]: cluster 2023-10-01T11:15:36.514317+0000 mgr.y (mgr.25072) 1752 : cluster 0 pgmap v1912: 73 pgs: 73 active+clean; 802 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 156 KiB/s rd, 24 KiB/s wr, 206 op/s 2023-10-01T11:15:38.114 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:38 smithi078 bash[75299]: cluster 2023-10-01T11:15:37.009775+0000 osd.5 (osd.5) 432 : cluster 3 failed to encode map e402 with expected crc 2023-10-01T11:15:38.114 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:38 smithi078 bash[75299]: cluster 2023-10-01T11:15:37.012038+0000 mon.a (mon.0) 2247 : cluster 0 osdmap e402: 8 total, 8 up, 8 in 2023-10-01T11:15:38.115 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:38 smithi078 bash[75299]: cluster 2023-10-01T11:15:37.015142+0000 osd.7 (osd.7) 451 : cluster 3 failed to encode map e402 with expected crc 2023-10-01T11:15:38.115 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:38 smithi078 bash[75299]: cluster 2023-10-01T11:15:37.086722+0000 osd.6 (osd.6) 388 : cluster 3 failed to encode map e402 with expected crc 2023-10-01T11:15:38.127 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:38.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:38.130 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:15:38.231 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:38.262 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:38.264 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:38 smithi173 bash[58816]: cluster 2023-10-01T11:15:36.514317+0000 mgr.y (mgr.25072) 1752 : cluster 0 pgmap v1912: 73 pgs: 73 active+clean; 802 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 156 KiB/s rd, 24 KiB/s wr, 206 op/s 2023-10-01T11:15:38.264 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:38 smithi173 bash[58816]: cluster 2023-10-01T11:15:37.009775+0000 osd.5 (osd.5) 432 : cluster 3 failed to encode map e402 with expected crc 2023-10-01T11:15:38.264 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:38 smithi173 bash[58816]: cluster 2023-10-01T11:15:37.012038+0000 mon.a (mon.0) 2247 : cluster 0 osdmap e402: 8 total, 8 up, 8 in 2023-10-01T11:15:38.265 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:38 smithi173 bash[58816]: cluster 2023-10-01T11:15:37.015142+0000 osd.7 (osd.7) 451 : cluster 3 failed to encode map e402 with expected crc 2023-10-01T11:15:38.265 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:38 smithi173 bash[58816]: cluster 2023-10-01T11:15:37.086722+0000 osd.6 (osd.6) 388 : cluster 3 failed to encode map e402 with expected crc 2023-10-01T11:15:38.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:38.266 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:15:38.366 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:38.399 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:38.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:38.403 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:15:38.504 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:38.523 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2023-10-01T11:15:38.523 INFO:teuthology.orchestra.run.smithi078.stdout: "ops": [], 2023-10-01T11:15:38.523 INFO:teuthology.orchestra.run.smithi078.stdout: "num_ops": 0 2023-10-01T11:15:38.523 INFO:teuthology.orchestra.run.smithi078.stdout:} 2023-10-01T11:15:38.541 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:38.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:38.544 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:15:38.644 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:38.675 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:38.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:38.677 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:15:38.778 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:38.806 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:38.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:38.809 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:15:38.910 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:38.931 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:38.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:38.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:15:39.034 INFO:tasks.workunit.client.0.smithi078.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-10-01T11:15:39.035 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:39.042 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:39 smithi078 bash[78274]: cluster 2023-10-01T11:15:38.021925+0000 mon.a (mon.0) 2248 : cluster 0 osdmap e403: 8 total, 8 up, 8 in 2023-10-01T11:15:39.042 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:39 smithi078 bash[78274]: cluster 2023-10-01T11:15:38.021974+0000 osd.7 (osd.7) 452 : cluster 3 failed to encode map e403 with expected crc 2023-10-01T11:15:39.042 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:39 smithi078 bash[78274]: cluster 2023-10-01T11:15:38.032938+0000 osd.5 (osd.5) 433 : cluster 3 failed to encode map e403 with expected crc 2023-10-01T11:15:39.043 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:39 smithi078 bash[78274]: cluster 2023-10-01T11:15:38.042455+0000 osd.6 (osd.6) 389 : cluster 3 failed to encode map e403 with expected crc 2023-10-01T11:15:39.043 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:39 smithi078 bash[75299]: cluster 2023-10-01T11:15:38.021925+0000 mon.a (mon.0) 2248 : cluster 0 osdmap e403: 8 total, 8 up, 8 in 2023-10-01T11:15:39.043 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:39 smithi078 bash[75299]: cluster 2023-10-01T11:15:38.021974+0000 osd.7 (osd.7) 452 : cluster 3 failed to encode map e403 with expected crc 2023-10-01T11:15:39.043 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:39 smithi078 bash[75299]: cluster 2023-10-01T11:15:38.032938+0000 osd.5 (osd.5) 433 : cluster 3 failed to encode map e403 with expected crc 2023-10-01T11:15:39.043 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:39 smithi078 bash[75299]: cluster 2023-10-01T11:15:38.042455+0000 osd.6 (osd.6) 389 : cluster 3 failed to encode map e403 with expected crc 2023-10-01T11:15:39.047 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd remove testimg 2023-10-01T11:15:39.059 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:39.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:39.061 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:15:39.162 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:15:39.195 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:15:39.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:39.198 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:15:39.201 INFO:tasks.workunit.client.0.smithi078.stderr: Removing image: 100% complete...done. 2023-10-01T11:15:39.205 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd remove testimg_import 2023-10-01T11:15:39.299 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:39.310 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:39 smithi173 bash[58816]: cluster 2023-10-01T11:15:38.021925+0000 mon.a (mon.0) 2248 : cluster 0 osdmap e403: 8 total, 8 up, 8 in 2023-10-01T11:15:39.311 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:39 smithi173 bash[58816]: cluster 2023-10-01T11:15:38.021974+0000 osd.7 (osd.7) 452 : cluster 3 failed to encode map e403 with expected crc 2023-10-01T11:15:39.311 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:39 smithi173 bash[58816]: cluster 2023-10-01T11:15:38.032938+0000 osd.5 (osd.5) 433 : cluster 3 failed to encode map e403 with expected crc 2023-10-01T11:15:39.311 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:39 smithi173 bash[58816]: cluster 2023-10-01T11:15:38.042455+0000 osd.6 (osd.6) 389 : cluster 3 failed to encode map e403 with expected crc 2023-10-01T11:15:39.329 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:39.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:39.331 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:15:39.345 INFO:tasks.workunit.client.0.smithi078.stderr: Removing image: 100% complete...done. 2023-10-01T11:15:39.349 INFO:tasks.workunit.client.0.smithi078.stderr:+ tiered=0 2023-10-01T11:15:39.350 INFO:tasks.workunit.client.0.smithi078.stderr:+ ceph osd dump 2023-10-01T11:15:39.350 INFO:tasks.workunit.client.0.smithi078.stderr:+ grep ^pool 2023-10-01T11:15:39.350 INFO:tasks.workunit.client.0.smithi078.stderr:+ grep 'rbd' 2023-10-01T11:15:39.350 INFO:tasks.workunit.client.0.smithi078.stderr:+ grep tier 2023-10-01T11:15:39.432 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:15:39.453 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:15:39.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:39.455 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:15:39.556 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:39.589 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:39.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:39.591 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:15:39.692 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:39.723 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:39.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:39.726 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:15:39.826 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:39.854 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:39.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:39.856 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:15:39.957 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:39.987 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:39.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:39.990 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:15:40.091 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:40.120 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:40.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:40.124 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:15:40.225 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:40.238 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:40 smithi078 bash[75299]: cluster 2023-10-01T11:15:38.514810+0000 mgr.y (mgr.25072) 1753 : cluster 0 pgmap v1915: 73 pgs: 73 active+clean; 802 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 139 KiB/s rd, 8.5 KiB/s wr, 177 op/s 2023-10-01T11:15:40.238 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:40 smithi078 bash[75299]: cluster 2023-10-01T11:15:39.025757+0000 osd.6 (osd.6) 390 : cluster 3 failed to encode map e404 with expected crc 2023-10-01T11:15:40.239 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:40 smithi078 bash[75299]: cluster 2023-10-01T11:15:39.026318+0000 osd.7 (osd.7) 453 : cluster 3 failed to encode map e404 with expected crc 2023-10-01T11:15:40.239 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:40 smithi078 bash[75299]: cluster 2023-10-01T11:15:39.027428+0000 mon.a (mon.0) 2249 : cluster 0 osdmap e404: 8 total, 8 up, 8 in 2023-10-01T11:15:40.239 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:40 smithi078 bash[75299]: cluster 2023-10-01T11:15:39.084227+0000 osd.5 (osd.5) 434 : cluster 3 failed to encode map e404 with expected crc 2023-10-01T11:15:40.239 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:40 smithi078 bash[78274]: cluster 2023-10-01T11:15:38.514810+0000 mgr.y (mgr.25072) 1753 : cluster 0 pgmap v1915: 73 pgs: 73 active+clean; 802 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 139 KiB/s rd, 8.5 KiB/s wr, 177 op/s 2023-10-01T11:15:40.239 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:40 smithi078 bash[78274]: cluster 2023-10-01T11:15:39.025757+0000 osd.6 (osd.6) 390 : cluster 3 failed to encode map e404 with expected crc 2023-10-01T11:15:40.239 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:40 smithi078 bash[78274]: cluster 2023-10-01T11:15:39.026318+0000 osd.7 (osd.7) 453 : cluster 3 failed to encode map e404 with expected crc 2023-10-01T11:15:40.239 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:40 smithi078 bash[78274]: cluster 2023-10-01T11:15:39.027428+0000 mon.a (mon.0) 2249 : cluster 0 osdmap e404: 8 total, 8 up, 8 in 2023-10-01T11:15:40.239 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:40 smithi078 bash[78274]: cluster 2023-10-01T11:15:39.084227+0000 osd.5 (osd.5) 434 : cluster 3 failed to encode map e404 with expected crc 2023-10-01T11:15:40.254 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:40.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:40.257 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:15:40.358 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:15:40.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:40 smithi173 bash[58816]: cluster 2023-10-01T11:15:38.514810+0000 mgr.y (mgr.25072) 1753 : cluster 0 pgmap v1915: 73 pgs: 73 active+clean; 802 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 139 KiB/s rd, 8.5 KiB/s wr, 177 op/s 2023-10-01T11:15:40.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:40 smithi173 bash[58816]: cluster 2023-10-01T11:15:39.025757+0000 osd.6 (osd.6) 390 : cluster 3 failed to encode map e404 with expected crc 2023-10-01T11:15:40.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:40 smithi173 bash[58816]: cluster 2023-10-01T11:15:39.026318+0000 osd.7 (osd.7) 453 : cluster 3 failed to encode map e404 with expected crc 2023-10-01T11:15:40.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:40 smithi173 bash[58816]: cluster 2023-10-01T11:15:39.027428+0000 mon.a (mon.0) 2249 : cluster 0 osdmap e404: 8 total, 8 up, 8 in 2023-10-01T11:15:40.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:40 smithi173 bash[58816]: cluster 2023-10-01T11:15:39.084227+0000 osd.5 (osd.5) 434 : cluster 3 failed to encode map e404 with expected crc 2023-10-01T11:15:40.387 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:15:40.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:40.389 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:15:40.490 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:40.497 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:15:40 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:15:40] "GET /metrics HTTP/1.1" 200 34009 "" "Prometheus/2.33.4" 2023-10-01T11:15:40.513 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:40.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:40.515 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:15:40.616 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:40.637 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:40.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:40.639 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:15:40.661 INFO:tasks.workunit.client.0.smithi078.stderr:+ dd if=/dev/urandom bs=1M seek=1 count=1 of=/tmp/rbd_import_export_259593/sparse1 2023-10-01T11:15:40.666 INFO:tasks.workunit.client.0.smithi078.stderr:1+0 records in 2023-10-01T11:15:40.666 INFO:tasks.workunit.client.0.smithi078.stderr:1+0 records out 2023-10-01T11:15:40.666 INFO:tasks.workunit.client.0.smithi078.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00351743 s, 298 MB/s 2023-10-01T11:15:40.666 INFO:tasks.workunit.client.0.smithi078.stderr:+ dd if=/dev/urandom bs=1M count=1 of=/tmp/rbd_import_export_259593/sparse2 2023-10-01T11:15:40.669 INFO:tasks.workunit.client.0.smithi078.stderr:1+0 records in 2023-10-01T11:15:40.670 INFO:tasks.workunit.client.0.smithi078.stderr:1+0 records out 2023-10-01T11:15:40.670 INFO:tasks.workunit.client.0.smithi078.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00323517 s, 324 MB/s 2023-10-01T11:15:40.670 INFO:tasks.workunit.client.0.smithi078.stderr:+ truncate /tmp/rbd_import_export_259593/sparse2 -s 2M 2023-10-01T11:15:40.739 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:40.773 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:40.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:40.777 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:15:40.877 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:40.901 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd rm sparse1 2023-10-01T11:15:40.908 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:40.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:40.911 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:15:40.951 INFO:tasks.workunit.client.0.smithi078.stderr: Removing image: 0% complete...failed. 2023-10-01T11:15:40.951 INFO:tasks.workunit.client.0.smithi078.stderr:rbd: delete error: (2) No such file or directory 2023-10-01T11:15:40.954 INFO:tasks.workunit.client.0.smithi078.stderr:+ true 2023-10-01T11:15:40.955 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_259593/sparse1 2023-10-01T11:15:41.012 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:41.037 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:41 smithi173 bash[58816]: audit 2023-10-01T11:15:40.539199+0000 mon.c (mon.2) 253 : audit 0 from='client.? 172.21.15.78:0/975621023' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-01T11:15:41.037 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:41 smithi173 bash[58816]: audit 2023-10-01T11:15:40.648398+0000 mon.c (mon.2) 254 : audit 0 from='client.? 172.21.15.78:0/975621023' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-01T11:15:41.039 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:41.041 INFO:tasks.workunit.client.0.smithi078.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-10-01T11:15:41.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:41.042 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:15:41.042 INFO:tasks.workunit.client.0.smithi078.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-01T11:15:41.042 INFO:tasks.workunit.client.0.smithi078.stderr:rbd: --order is deprecated, use --object-size 2023-10-01T11:15:41.044 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd ls -l 2023-10-01T11:15:41.045 INFO:tasks.workunit.client.0.smithi078.stderr:+ grep sparse1 2023-10-01T11:15:41.045 INFO:tasks.workunit.client.0.smithi078.stderr:+ grep -Ei (2 MiB|2048k) 2023-10-01T11:15:41.090 INFO:tasks.workunit.client.0.smithi078.stdout:sparse1 2 MiB 2 2023-10-01T11:15:41.091 INFO:tasks.workunit.client.0.smithi078.stderr:+ objects sparse1 2023-10-01T11:15:41.092 INFO:tasks.workunit.client.0.smithi078.stderr:+ image=sparse1 2023-10-01T11:15:41.092 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd info sparse1 2023-10-01T11:15:41.092 INFO:tasks.workunit.client.0.smithi078.stderr:+ grep block_name_prefix 2023-10-01T11:15:41.092 INFO:tasks.workunit.client.0.smithi078.stderr:+ awk {print $NF;} 2023-10-01T11:15:41.112 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-10-01T11:15:41.139 INFO:tasks.workunit.client.0.smithi078.stderr:+ prefix=rbd_data.b91e528ae4f9 2023-10-01T11:15:41.140 INFO:tasks.workunit.client.0.smithi078.stderr:+ grep rbd_data.b91e528ae4f9 2023-10-01T11:15:41.141 INFO:tasks.workunit.client.0.smithi078.stderr:+ get_image_data_pool sparse1 2023-10-01T11:15:41.141 INFO:tasks.workunit.client.0.smithi078.stderr:+ image=sparse1 2023-10-01T11:15:41.141 INFO:tasks.workunit.client.0.smithi078.stderr:+ sed -e s/rbd_data.b91e528ae4f9\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-10-01T11:15:41.141 INFO:tasks.workunit.client.0.smithi078.stderr:+ sort -u 2023-10-01T11:15:41.141 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd info sparse1 2023-10-01T11:15:41.142 INFO:tasks.workunit.client.0.smithi078.stderr:+ + grep data_pool: 2023-10-01T11:15:41.142 INFO:tasks.workunit.client.0.smithi078.stderr:awk -F: { print $NF } 2023-10-01T11:15:41.142 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:41.179 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:41.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:41.182 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:15:41.192 INFO:tasks.workunit.client.0.smithi078.stderr:+ data_pool= 2023-10-01T11:15:41.192 INFO:tasks.workunit.client.0.smithi078.stderr:+ [ -z ] 2023-10-01T11:15:41.192 INFO:tasks.workunit.client.0.smithi078.stderr:+ data_pool=rbd 2023-10-01T11:15:41.192 INFO:tasks.workunit.client.0.smithi078.stderr:+ echo rbd 2023-10-01T11:15:41.192 INFO:tasks.workunit.client.0.smithi078.stderr:+ rados ls -p rbd 2023-10-01T11:15:41.283 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:41.294 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:41 smithi078 bash[75299]: audit 2023-10-01T11:15:40.539199+0000 mon.c (mon.2) 253 : audit 0 from='client.? 172.21.15.78:0/975621023' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-01T11:15:41.294 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:41 smithi078 bash[75299]: audit 2023-10-01T11:15:40.648398+0000 mon.c (mon.2) 254 : audit 0 from='client.? 172.21.15.78:0/975621023' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-01T11:15:41.294 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:41 smithi078 bash[78274]: audit 2023-10-01T11:15:40.539199+0000 mon.c (mon.2) 253 : audit 0 from='client.? 172.21.15.78:0/975621023' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-01T11:15:41.294 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:41 smithi078 bash[78274]: audit 2023-10-01T11:15:40.648398+0000 mon.c (mon.2) 254 : audit 0 from='client.? 172.21.15.78:0/975621023' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-10-01T11:15:41.305 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:41.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:41.307 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:15:41.408 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:41.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:41.446 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:41.446 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:15:41.547 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:41.579 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:41.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:41.582 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:15:41.590 INFO:tasks.workunit.client.0.smithi078.stderr:+ objects=1 2023-10-01T11:15:41.590 INFO:tasks.workunit.client.0.smithi078.stderr:+ echo 1 2023-10-01T11:15:41.590 INFO:tasks.workunit.client.0.smithi078.stderr:+ [ 0 -eq 1 -o 1 = 1 ] 2023-10-01T11:15:41.590 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd export sparse1 /tmp/rbd_import_export_259593/sparse1.out 2023-10-01T11:15:41.642 INFO:tasks.workunit.client.0.smithi078.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-10-01T11:15:41.649 INFO:tasks.workunit.client.0.smithi078.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_259593/sparse1 /tmp/rbd_import_export_259593/sparse1.out 2023-10-01T11:15:41.650 INFO:tasks.workunit.client.0.smithi078.stderr:+ cmp -l /tmp/rbd_import_export_259593/sparse1 /tmp/rbd_import_export_259593/sparse1.out 2023-10-01T11:15:41.650 INFO:tasks.workunit.client.0.smithi078.stderr:+ stat /tmp/rbd_import_export_259593/sparse1 --format %b 2023-10-01T11:15:41.652 INFO:tasks.workunit.client.0.smithi078.stderr:+ origsize=2048 2023-10-01T11:15:41.652 INFO:tasks.workunit.client.0.smithi078.stderr:+ stat /tmp/rbd_import_export_259593/sparse1.out --format %b 2023-10-01T11:15:41.653 INFO:tasks.workunit.client.0.smithi078.stderr:+ exportsize=2048 2023-10-01T11:15:41.653 INFO:tasks.workunit.client.0.smithi078.stderr:+ difference=0 2023-10-01T11:15:41.653 INFO:tasks.workunit.client.0.smithi078.stderr:+ difference=0 2023-10-01T11:15:41.653 INFO:tasks.workunit.client.0.smithi078.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-01T11:15:41.654 INFO:tasks.workunit.client.0.smithi078.stderr:+ rm /tmp/rbd_import_export_259593/sparse1.out 2023-10-01T11:15:41.655 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd rm sparse1 2023-10-01T11:15:41.682 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:41.710 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:41.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:41.712 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:15:41.794 INFO:tasks.workunit.client.0.smithi078.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-10-01T11:15:41.798 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd rm sparse2 2023-10-01T11:15:41.813 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:41.847 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:41.848 INFO:tasks.workunit.client.0.smithi078.stderr: Removing image: 0% complete...failed. 2023-10-01T11:15:41.848 INFO:tasks.workunit.client.0.smithi078.stderr:rbd: delete error: (2) No such file or directory 2023-10-01T11:15:41.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:41.850 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:15:41.853 INFO:tasks.workunit.client.0.smithi078.stderr:+ true 2023-10-01T11:15:41.854 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_259593/sparse2 2023-10-01T11:15:41.933 INFO:tasks.workunit.client.0.smithi078.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-10-01T11:15:41.934 INFO:tasks.workunit.client.0.smithi078.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-01T11:15:41.934 INFO:tasks.workunit.client.0.smithi078.stderr:rbd: --order is deprecated, use --object-size 2023-10-01T11:15:41.938 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd ls -l 2023-10-01T11:15:41.938 INFO:tasks.workunit.client.0.smithi078.stderr:+ grep sparse2 2023-10-01T11:15:41.938 INFO:tasks.workunit.client.0.smithi078.stderr:+ grep -Ei (2 MiB|2048k) 2023-10-01T11:15:41.954 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:41.977 INFO:tasks.workunit.client.0.smithi078.stdout:sparse2 2 MiB 2 2023-10-01T11:15:41.978 INFO:tasks.workunit.client.0.smithi078.stderr:+ objects sparse2 2023-10-01T11:15:41.978 INFO:tasks.workunit.client.0.smithi078.stderr:+ image=sparse2 2023-10-01T11:15:41.978 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd info sparse2 2023-10-01T11:15:41.978 INFO:tasks.workunit.client.0.smithi078.stderr:+ grep+ block_name_prefix 2023-10-01T11:15:41.978 INFO:tasks.workunit.client.0.smithi078.stderr:awk {print $NF;} 2023-10-01T11:15:41.984 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:41.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:41.986 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:15:42.022 INFO:tasks.workunit.client.0.smithi078.stderr:+ prefix=rbd_data.b9516a96f148 2023-10-01T11:15:42.022 INFO:tasks.workunit.client.0.smithi078.stderr:+ grep rbd_data.b9516a96f148 2023-10-01T11:15:42.023 INFO:tasks.workunit.client.0.smithi078.stderr:+ + get_image_data_pool sparse2 2023-10-01T11:15:42.023 INFO:tasks.workunit.client.0.smithi078.stderr:sed+ -e s/rbd_data.b9516a96f148\.// -eimage=sparse2 s/^0*\([0-9a-f]\)/\1/ 2023-10-01T11:15:42.023 INFO:tasks.workunit.client.0.smithi078.stderr: 2023-10-01T11:15:42.023 INFO:tasks.workunit.client.0.smithi078.stderr:+ sort -u 2023-10-01T11:15:42.023 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd info sparse2 2023-10-01T11:15:42.023 INFO:tasks.workunit.client.0.smithi078.stderr:+ grep data_pool: 2023-10-01T11:15:42.023 INFO:tasks.workunit.client.0.smithi078.stderr:+ awk -F: { print $NF } 2023-10-01T11:15:42.065 INFO:tasks.workunit.client.0.smithi078.stderr:+ data_pool= 2023-10-01T11:15:42.065 INFO:tasks.workunit.client.0.smithi078.stderr:+ [ -z ] 2023-10-01T11:15:42.065 INFO:tasks.workunit.client.0.smithi078.stderr:+ data_pool=rbd 2023-10-01T11:15:42.065 INFO:tasks.workunit.client.0.smithi078.stderr:+ echo rbd 2023-10-01T11:15:42.066 INFO:tasks.workunit.client.0.smithi078.stderr:+ rados ls -p rbd 2023-10-01T11:15:42.087 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:42.099 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:42 smithi173 bash[58816]: cluster 2023-10-01T11:15:40.515528+0000 mgr.y (mgr.25072) 1754 : cluster 0 pgmap v1917: 73 pgs: 73 active+clean; 802 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 140 KiB/s rd, 11 KiB/s wr, 189 op/s 2023-10-01T11:15:42.105 INFO:tasks.workunit.client.0.smithi078.stderr:+ objects=0 2023-10-01T11:15:42.106 INFO:tasks.workunit.client.0.smithi078.stderr:+ echo 0 2023-10-01T11:15:42.106 INFO:tasks.workunit.client.0.smithi078.stderr:+ [ 0 -eq 1 -o 0 = 0 ] 2023-10-01T11:15:42.106 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd export sparse2 /tmp/rbd_import_export_259593/sparse2.out 2023-10-01T11:15:42.117 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:42.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:42.121 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:15:42.148 INFO:tasks.workunit.client.0.smithi078.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-10-01T11:15:42.152 INFO:tasks.workunit.client.0.smithi078.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_259593/sparse2 /tmp/rbd_import_export_259593/sparse2.out 2023-10-01T11:15:42.152 INFO:tasks.workunit.client.0.smithi078.stderr:+ cmp -l /tmp/rbd_import_export_259593/sparse2 /tmp/rbd_import_export_259593/sparse2.out 2023-10-01T11:15:42.156 INFO:tasks.workunit.client.0.smithi078.stderr:+ stat /tmp/rbd_import_export_259593/sparse2 --format %b 2023-10-01T11:15:42.159 INFO:tasks.workunit.client.0.smithi078.stderr:+ origsize=2048 2023-10-01T11:15:42.159 INFO:tasks.workunit.client.0.smithi078.stderr:+ stat /tmp/rbd_import_export_259593/sparse2.out --format %b 2023-10-01T11:15:42.159 INFO:tasks.workunit.client.0.smithi078.stderr:+ exportsize=2048 2023-10-01T11:15:42.159 INFO:tasks.workunit.client.0.smithi078.stderr:+ difference=0 2023-10-01T11:15:42.159 INFO:tasks.workunit.client.0.smithi078.stderr:+ difference=0 2023-10-01T11:15:42.160 INFO:tasks.workunit.client.0.smithi078.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-01T11:15:42.160 INFO:tasks.workunit.client.0.smithi078.stderr:+ rm /tmp/rbd_import_export_259593/sparse2.out 2023-10-01T11:15:42.160 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd rm sparse2 2023-10-01T11:15:42.221 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:42.254 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:42.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:42.256 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:15:42.294 INFO:tasks.workunit.client.0.smithi078.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-10-01T11:15:42.298 INFO:tasks.workunit.client.0.smithi078.stderr:+ truncate /tmp/rbd_import_export_259593/sparse1 -s 10M 2023-10-01T11:15:42.299 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd import --new-format --order 20 - sparse1 2023-10-01T11:15:42.299 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:42 smithi078 bash[75299]: cluster 2023-10-01T11:15:40.515528+0000 mgr.y (mgr.25072) 1754 : cluster 0 pgmap v1917: 73 pgs: 73 active+clean; 802 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 140 KiB/s rd, 11 KiB/s wr, 189 op/s 2023-10-01T11:15:42.300 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:42 smithi078 bash[78274]: cluster 2023-10-01T11:15:40.515528+0000 mgr.y (mgr.25072) 1754 : cluster 0 pgmap v1917: 73 pgs: 73 active+clean; 802 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 140 KiB/s rd, 11 KiB/s wr, 189 op/s 2023-10-01T11:15:42.357 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:42.383 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:42.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:42.386 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:15:42.427 INFO:tasks.workunit.client.0.smithi078.stderr: Importing image: 100% complete...done. 2023-10-01T11:15:42.428 INFO:tasks.workunit.client.0.smithi078.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-01T11:15:42.428 INFO:tasks.workunit.client.0.smithi078.stderr:rbd: --order is deprecated, use --object-size 2023-10-01T11:15:42.433 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd ls -l 2023-10-01T11:15:42.433 INFO:tasks.workunit.client.0.smithi078.stderr:+ grep sparse1 2023-10-01T11:15:42.433 INFO:tasks.workunit.client.0.smithi078.stderr:+ grep -Ei (10 MiB|10240k) 2023-10-01T11:15:42.488 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:42.489 INFO:tasks.workunit.client.0.smithi078.stdout:sparse1 10 MiB 2 2023-10-01T11:15:42.491 INFO:tasks.workunit.client.0.smithi078.stderr:+ objects sparse1 2023-10-01T11:15:42.491 INFO:tasks.workunit.client.0.smithi078.stderr:+ image=sparse1 2023-10-01T11:15:42.491 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd info sparse1 2023-10-01T11:15:42.491 INFO:tasks.workunit.client.0.smithi078.stderr:+ grep block_name_prefix 2023-10-01T11:15:42.491 INFO:tasks.workunit.client.0.smithi078.stderr:+ awk {print $NF;} 2023-10-01T11:15:42.511 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:42.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:42.514 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:15:42.551 INFO:tasks.workunit.client.0.smithi078.stderr:+ prefix=rbd_data.b97e80ef9e59 2023-10-01T11:15:42.553 INFO:tasks.workunit.client.0.smithi078.stderr:+ grep rbd_data.b97e80ef9e59 2023-10-01T11:15:42.553 INFO:tasks.workunit.client.0.smithi078.stderr:+ get_image_data_pool sparse1 2023-10-01T11:15:42.553 INFO:tasks.workunit.client.0.smithi078.stderr:+ image=sparse1 2023-10-01T11:15:42.553 INFO:tasks.workunit.client.0.smithi078.stderr:+ sed -e s/rbd_data.b97e80ef9e59\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-10-01T11:15:42.553 INFO:tasks.workunit.client.0.smithi078.stderr:+ sort -u 2023-10-01T11:15:42.553 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd info sparse1 2023-10-01T11:15:42.553 INFO:tasks.workunit.client.0.smithi078.stderr:+ grep data_pool: 2023-10-01T11:15:42.554 INFO:tasks.workunit.client.0.smithi078.stderr:+ awk -F: { print $NF } 2023-10-01T11:15:42.604 INFO:tasks.workunit.client.0.smithi078.stderr:+ data_pool= 2023-10-01T11:15:42.604 INFO:tasks.workunit.client.0.smithi078.stderr:+ [ -z ] 2023-10-01T11:15:42.604 INFO:tasks.workunit.client.0.smithi078.stderr:+ data_pool=rbd 2023-10-01T11:15:42.604 INFO:tasks.workunit.client.0.smithi078.stderr:+ echo rbd 2023-10-01T11:15:42.604 INFO:tasks.workunit.client.0.smithi078.stderr:+ rados ls -p rbd 2023-10-01T11:15:42.615 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:42.647 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:42.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:42.650 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:15:42.651 INFO:tasks.workunit.client.0.smithi078.stderr:+ objects=1 2023-10-01T11:15:42.651 INFO:tasks.workunit.client.0.smithi078.stderr:+ echo 1 2023-10-01T11:15:42.651 INFO:tasks.workunit.client.0.smithi078.stderr:+ [ 0 -eq 1 -o 1 = 1 ] 2023-10-01T11:15:42.651 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd export sparse1 /tmp/rbd_import_export_259593/sparse1.out 2023-10-01T11:15:42.718 INFO:tasks.workunit.client.0.smithi078.stderr: Exporting image: 10% complete... Exporting image: 20% complete... Exporting image: 30% complete... Exporting image: 40% complete... Exporting image: 50% complete... Exporting image: 60% complete... Exporting image: 70% complete... Exporting image: 80% complete... Exporting image: 90% complete... Exporting image: 100% complete...done. 2023-10-01T11:15:42.723 INFO:tasks.workunit.client.0.smithi078.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_259593/sparse1 /tmp/rbd_import_export_259593/sparse1.out 2023-10-01T11:15:42.723 INFO:tasks.workunit.client.0.smithi078.stderr:+ cmp -l /tmp/rbd_import_export_259593/sparse1 /tmp/rbd_import_export_259593/sparse1.out 2023-10-01T11:15:42.735 INFO:tasks.workunit.client.0.smithi078.stderr:+ stat /tmp/rbd_import_export_259593/sparse1 --format %b 2023-10-01T11:15:42.736 INFO:tasks.workunit.client.0.smithi078.stderr:+ origsize=2048 2023-10-01T11:15:42.736 INFO:tasks.workunit.client.0.smithi078.stderr:+ stat /tmp/rbd_import_export_259593/sparse1.out --format %b 2023-10-01T11:15:42.737 INFO:tasks.workunit.client.0.smithi078.stderr:+ exportsize=2048 2023-10-01T11:15:42.737 INFO:tasks.workunit.client.0.smithi078.stderr:+ difference=0 2023-10-01T11:15:42.737 INFO:tasks.workunit.client.0.smithi078.stderr:+ difference=0 2023-10-01T11:15:42.737 INFO:tasks.workunit.client.0.smithi078.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-01T11:15:42.737 INFO:tasks.workunit.client.0.smithi078.stderr:+ rm /tmp/rbd_import_export_259593/sparse1.out 2023-10-01T11:15:42.751 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:42.779 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:42.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:42.782 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:15:42.883 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:42.916 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:42.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:42.918 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:15:43.019 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:15:43.029 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd rm sparse1 2023-10-01T11:15:43.052 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1 2023-10-01T11:15:43.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:43.054 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.1' 2023-10-01T11:15:43.155 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:43.182 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:43.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:43.184 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:15:43.189 INFO:tasks.workunit.client.0.smithi078.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-10-01T11:15:43.193 INFO:tasks.workunit.client.0.smithi078.stderr:+ dd if=/dev/urandom bs=2M count=1 of=/tmp/rbd_import_export_259593/sparse2 oflag=append conv=notrunc 2023-10-01T11:15:43.201 INFO:tasks.workunit.client.0.smithi078.stderr:1+0 records in 2023-10-01T11:15:43.201 INFO:tasks.workunit.client.0.smithi078.stderr:1+0 records out 2023-10-01T11:15:43.201 INFO:tasks.workunit.client.0.smithi078.stderr:2097152 bytes (2.1 MB, 2.0 MiB) copied, 0.0071713 s, 292 MB/s 2023-10-01T11:15:43.201 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd import --new-format --order 20 - sparse2 2023-10-01T11:15:43.286 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:43.309 INFO:tasks.workunit.client.0.smithi078.stderr: Importing image: 100% complete...done. 2023-10-01T11:15:43.310 INFO:tasks.workunit.client.0.smithi078.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-01T11:15:43.310 INFO:tasks.workunit.client.0.smithi078.stderr:rbd: --order is deprecated, use --object-size 2023-10-01T11:15:43.314 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:43.315 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd ls -l 2023-10-01T11:15:43.315 INFO:tasks.workunit.client.0.smithi078.stderr:+ grep sparse2 2023-10-01T11:15:43.315 INFO:tasks.workunit.client.0.smithi078.stderr:+ grep -Ei (4 MiB|4096k) 2023-10-01T11:15:43.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:43.317 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:15:43.320 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T11:15:43.320 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-10-01T11:15:43.321 INFO:tasks.thrashosds.thrasher:Removing osd 0, in_osds are: [0, 1, 2, 3, 4, 5, 6, 7] 2023-10-01T11:15:43.321 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd out 0 2023-10-01T11:15:43.367 INFO:tasks.workunit.client.0.smithi078.stdout:sparse2 4 MiB 2 2023-10-01T11:15:43.368 INFO:tasks.workunit.client.0.smithi078.stderr:+ objects sparse2 2023-10-01T11:15:43.368 INFO:tasks.workunit.client.0.smithi078.stderr:+ image=sparse2 2023-10-01T11:15:43.368 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd info sparse2 2023-10-01T11:15:43.368 INFO:tasks.workunit.client.0.smithi078.stderr:+ grep block_name_prefix 2023-10-01T11:15:43.368 INFO:tasks.workunit.client.0.smithi078.stderr:+ awk {print $NF;} 2023-10-01T11:15:43.416 INFO:tasks.workunit.client.0.smithi078.stderr:+ prefix=rbd_data.b9abb12dc019 2023-10-01T11:15:43.417 INFO:tasks.workunit.client.0.smithi078.stderr:+ grep rbd_data.b9abb12dc019 2023-10-01T11:15:43.418 INFO:tasks.workunit.client.0.smithi078.stderr:+ sed -e s/rbd_data.b9abb12dc019\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-10-01T11:15:43.418 INFO:tasks.workunit.client.0.smithi078.stderr:+ sort -u 2023-10-01T11:15:43.418 INFO:tasks.workunit.client.0.smithi078.stderr:+ get_image_data_pool sparse2 2023-10-01T11:15:43.418 INFO:tasks.workunit.client.0.smithi078.stderr:+ image=sparse2 2023-10-01T11:15:43.418 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd info sparse2 2023-10-01T11:15:43.418 INFO:tasks.workunit.client.0.smithi078.stderr:+ grep data_pool: 2023-10-01T11:15:43.418 INFO:tasks.workunit.client.0.smithi078.stderr:+ awk -F: { print $NF } 2023-10-01T11:15:43.418 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:43.436 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:43.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:43.438 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:15:43.465 INFO:tasks.workunit.client.0.smithi078.stderr:+ data_pool= 2023-10-01T11:15:43.465 INFO:tasks.workunit.client.0.smithi078.stderr:+ [ -z ] 2023-10-01T11:15:43.465 INFO:tasks.workunit.client.0.smithi078.stderr:+ data_pool=rbd 2023-10-01T11:15:43.465 INFO:tasks.workunit.client.0.smithi078.stderr:+ echo rbd 2023-10-01T11:15:43.465 INFO:tasks.workunit.client.0.smithi078.stderr:+ rados ls -p rbd 2023-10-01T11:15:43.508 INFO:tasks.workunit.client.0.smithi078.stderr:+ objects=0 2023-10-01T11:15:43.508 INFO:tasks.workunit.client.0.smithi078.stderr:2 2023-10-01T11:15:43.508 INFO:tasks.workunit.client.0.smithi078.stderr:3 2023-10-01T11:15:43.509 INFO:tasks.workunit.client.0.smithi078.stderr:+ echo 0 2 3 2023-10-01T11:15:43.510 INFO:tasks.workunit.client.0.smithi078.stderr:+ [ 0 -eq 1 -o 0 2 3 = 0 2 3 ] 2023-10-01T11:15:43.510 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd export sparse2 /tmp/rbd_import_export_259593/sparse2.out 2023-10-01T11:15:43.540 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:43.554 INFO:tasks.workunit.client.0.smithi078.stderr: Exporting image: 25% complete... Exporting image: 50% complete... Exporting image: 75% complete... Exporting image: 100% complete...done. 2023-10-01T11:15:43.558 INFO:tasks.workunit.client.0.smithi078.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_259593/sparse2 /tmp/rbd_import_export_259593/sparse2.out 2023-10-01T11:15:43.559 INFO:tasks.workunit.client.0.smithi078.stderr:+ cmp -l /tmp/rbd_import_export_259593/sparse2 /tmp/rbd_import_export_259593/sparse2.out 2023-10-01T11:15:43.561 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:43.562 INFO:tasks.workunit.client.0.smithi078.stderr:+ stat /tmp/rbd_import_export_259593/sparse2 --format %b 2023-10-01T11:15:43.563 INFO:tasks.workunit.client.0.smithi078.stderr:+ origsize=6144 2023-10-01T11:15:43.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:43.563 INFO:tasks.workunit.client.0.smithi078.stderr:+ stat /tmp/rbd_import_export_259593/sparse2.out --format %b 2023-10-01T11:15:43.564 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:15:43.565 INFO:tasks.workunit.client.0.smithi078.stderr:+ exportsize=6144 2023-10-01T11:15:43.565 INFO:tasks.workunit.client.0.smithi078.stderr:+ difference=0 2023-10-01T11:15:43.565 INFO:tasks.workunit.client.0.smithi078.stderr:+ difference=0 2023-10-01T11:15:43.565 INFO:tasks.workunit.client.0.smithi078.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-10-01T11:15:43.565 INFO:tasks.workunit.client.0.smithi078.stderr:+ rm /tmp/rbd_import_export_259593/sparse2.out 2023-10-01T11:15:43.565 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd rm sparse2 2023-10-01T11:15:43.665 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:43.695 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:43.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:43.698 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:15:43.714 INFO:tasks.workunit.client.0.smithi078.stderr: Removing image: 25% complete... Removing image: 50% complete... Removing image: 75% complete... Removing image: 100% complete...done. 2023-10-01T11:15:43.718 INFO:tasks.workunit.client.0.smithi078.stdout:partially-sparse file imports to partially-sparse image 2023-10-01T11:15:43.719 INFO:tasks.workunit.client.0.smithi078.stderr:+ echo partially-sparse file imports to partially-sparse image 2023-10-01T11:15:43.719 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_259593/sparse1 sparse 2023-10-01T11:15:43.798 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:43.821 INFO:tasks.workunit.client.0.smithi078.stderr: Importing image: 10% complete... Importing image: 20% complete... Importing image: 30% complete... Importing image: 40% complete... Importing image: 50% complete... Importing image: 60% complete... Importing image: 70% complete... Importing image: 80% complete... Importing image: 90% complete... Importing image: 100% complete...done. 2023-10-01T11:15:43.822 INFO:tasks.workunit.client.0.smithi078.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-01T11:15:43.822 INFO:tasks.workunit.client.0.smithi078.stderr:rbd: --order is deprecated, use --object-size 2023-10-01T11:15:43.827 INFO:tasks.workunit.client.0.smithi078.stderr:+ objects sparse 2023-10-01T11:15:43.827 INFO:tasks.workunit.client.0.smithi078.stderr:+ image=sparse 2023-10-01T11:15:43.827 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:43.828 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd info sparse 2023-10-01T11:15:43.828 INFO:tasks.workunit.client.0.smithi078.stderr:+ grep block_name_prefix 2023-10-01T11:15:43.828 INFO:tasks.workunit.client.0.smithi078.stderr:+ awk {print $NF;} 2023-10-01T11:15:43.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:43.829 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:15:43.879 INFO:tasks.workunit.client.0.smithi078.stderr:+ prefix=rbd_data.92df1eca5400 2023-10-01T11:15:43.881 INFO:tasks.workunit.client.0.smithi078.stderr:+ grep rbd_data.92df1eca5400 2023-10-01T11:15:43.881 INFO:tasks.workunit.client.0.smithi078.stderr:+ + sed -e s/rbd_data.92df1eca5400\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-10-01T11:15:43.881 INFO:tasks.workunit.client.0.smithi078.stderr:get_image_data_pool sparse 2023-10-01T11:15:43.881 INFO:tasks.workunit.client.0.smithi078.stderr:+ image=sparse 2023-10-01T11:15:43.881 INFO:tasks.workunit.client.0.smithi078.stderr:+ sort -u 2023-10-01T11:15:43.881 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd info sparse 2023-10-01T11:15:43.881 INFO:tasks.workunit.client.0.smithi078.stderr:+ grep data_pool: 2023-10-01T11:15:43.882 INFO:tasks.workunit.client.0.smithi078.stderr:+ awk -F: { print $NF } 2023-10-01T11:15:43.930 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:43.936 INFO:tasks.workunit.client.0.smithi078.stderr:+ data_pool= 2023-10-01T11:15:43.936 INFO:tasks.workunit.client.0.smithi078.stderr:+ [ -z ] 2023-10-01T11:15:43.936 INFO:tasks.workunit.client.0.smithi078.stderr:+ data_pool=rbd 2023-10-01T11:15:43.936 INFO:tasks.workunit.client.0.smithi078.stderr:+ echo rbd 2023-10-01T11:15:43.937 INFO:tasks.workunit.client.0.smithi078.stderr:+ rados ls -p rbd 2023-10-01T11:15:43.960 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:43.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:43.963 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:15:43.980 INFO:tasks.workunit.client.0.smithi078.stderr:+ objects=1 2023-10-01T11:15:43.980 INFO:tasks.workunit.client.0.smithi078.stderr:+ echo 1 2023-10-01T11:15:43.980 INFO:tasks.workunit.client.0.smithi078.stderr:+ [ 0 -eq 1 -o 1 = 1 ] 2023-10-01T11:15:43.980 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd rm sparse 2023-10-01T11:15:44.064 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:44.077 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:43 smithi173 bash[58816]: cluster 2023-10-01T11:15:42.515984+0000 mgr.y (mgr.25072) 1755 : cluster 0 pgmap v1918: 73 pgs: 73 active+clean; 802 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 105 KiB/s rd, 8.5 KiB/s wr, 142 op/s 2023-10-01T11:15:44.077 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:43 smithi173 bash[58816]: cluster 2023-10-01T11:15:42.910986+0000 osd.6 (osd.6) 391 : cluster 3 failed to encode map e405 with expected crc 2023-10-01T11:15:44.077 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:43 smithi173 bash[58816]: cluster 2023-10-01T11:15:42.911494+0000 mon.a (mon.0) 2250 : cluster 0 osdmap e405: 8 total, 8 up, 8 in 2023-10-01T11:15:44.077 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:43 smithi173 bash[58816]: cluster 2023-10-01T11:15:42.913829+0000 osd.7 (osd.7) 454 : cluster 3 failed to encode map e405 with expected crc 2023-10-01T11:15:44.077 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:43 smithi173 bash[58816]: cluster 2023-10-01T11:15:43.066784+0000 osd.5 (osd.5) 435 : cluster 3 failed to encode map e405 with expected crc 2023-10-01T11:15:44.095 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:44.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:44.098 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:15:44.137 INFO:tasks.workunit.client.0.smithi078.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-10-01T11:15:44.143 INFO:tasks.workunit.client.0.smithi078.stdout:zeros import through stdin to sparse image 2023-10-01T11:15:44.144 INFO:tasks.workunit.client.0.smithi078.stderr:+ echo zeros import through stdin to sparse image 2023-10-01T11:15:44.144 INFO:tasks.workunit.client.0.smithi078.stderr:+ + dd if=/dev/zero bs=1M count=4 2023-10-01T11:15:44.144 INFO:tasks.workunit.client.0.smithi078.stderr:rbd import --new-format - sparse 2023-10-01T11:15:44.199 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:44.207 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:44 smithi078 bash[78274]: cluster 2023-10-01T11:15:42.515984+0000 mgr.y (mgr.25072) 1755 : cluster 0 pgmap v1918: 73 pgs: 73 active+clean; 802 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 105 KiB/s rd, 8.5 KiB/s wr, 142 op/s 2023-10-01T11:15:44.207 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:44 smithi078 bash[78274]: cluster 2023-10-01T11:15:42.910986+0000 osd.6 (osd.6) 391 : cluster 3 failed to encode map e405 with expected crc 2023-10-01T11:15:44.208 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:44 smithi078 bash[78274]: cluster 2023-10-01T11:15:42.911494+0000 mon.a (mon.0) 2250 : cluster 0 osdmap e405: 8 total, 8 up, 8 in 2023-10-01T11:15:44.208 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:44 smithi078 bash[78274]: cluster 2023-10-01T11:15:42.913829+0000 osd.7 (osd.7) 454 : cluster 3 failed to encode map e405 with expected crc 2023-10-01T11:15:44.208 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:44 smithi078 bash[78274]: cluster 2023-10-01T11:15:43.066784+0000 osd.5 (osd.5) 435 : cluster 3 failed to encode map e405 with expected crc 2023-10-01T11:15:44.208 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:43 smithi078 bash[75299]: cluster 2023-10-01T11:15:42.515984+0000 mgr.y (mgr.25072) 1755 : cluster 0 pgmap v1918: 73 pgs: 73 active+clean; 802 KiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 105 KiB/s rd, 8.5 KiB/s wr, 142 op/s 2023-10-01T11:15:44.208 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:43 smithi078 bash[75299]: cluster 2023-10-01T11:15:42.910986+0000 osd.6 (osd.6) 391 : cluster 3 failed to encode map e405 with expected crc 2023-10-01T11:15:44.208 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:43 smithi078 bash[75299]: cluster 2023-10-01T11:15:42.911494+0000 mon.a (mon.0) 2250 : cluster 0 osdmap e405: 8 total, 8 up, 8 in 2023-10-01T11:15:44.208 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:43 smithi078 bash[75299]: cluster 2023-10-01T11:15:42.913829+0000 osd.7 (osd.7) 454 : cluster 3 failed to encode map e405 with expected crc 2023-10-01T11:15:44.208 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:43 smithi078 bash[75299]: cluster 2023-10-01T11:15:43.066784+0000 osd.5 (osd.5) 435 : cluster 3 failed to encode map e405 with expected crc 2023-10-01T11:15:44.218 INFO:tasks.workunit.client.0.smithi078.stderr:4+0 records in 2023-10-01T11:15:44.218 INFO:tasks.workunit.client.0.smithi078.stderr:4+0 records out 2023-10-01T11:15:44.219 INFO:tasks.workunit.client.0.smithi078.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0740077 s, 56.7 MB/s 2023-10-01T11:15:44.222 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:44.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:44.225 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:15:44.239 INFO:tasks.workunit.client.0.smithi078.stderr: Importing image: 100% complete...done. 2023-10-01T11:15:44.240 INFO:tasks.workunit.client.0.smithi078.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-01T11:15:44.245 INFO:tasks.workunit.client.0.smithi078.stderr:+ objects sparse 2023-10-01T11:15:44.245 INFO:tasks.workunit.client.0.smithi078.stderr:+ image=sparse 2023-10-01T11:15:44.246 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd info sparse 2023-10-01T11:15:44.246 INFO:tasks.workunit.client.0.smithi078.stderr:+ grep block_name_prefix 2023-10-01T11:15:44.246 INFO:tasks.workunit.client.0.smithi078.stderr:+ awk {print $NF;} 2023-10-01T11:15:44.303 INFO:tasks.workunit.client.0.smithi078.stderr:+ prefix=rbd_data.b9f957b0ec03 2023-10-01T11:15:44.304 INFO:tasks.workunit.client.0.smithi078.stderr:+ + grep rbd_data.b9f957b0ec03 2023-10-01T11:15:44.304 INFO:tasks.workunit.client.0.smithi078.stderr:get_image_data_pool sparse 2023-10-01T11:15:44.304 INFO:tasks.workunit.client.0.smithi078.stderr:+ image=sparse 2023-10-01T11:15:44.304 INFO:tasks.workunit.client.0.smithi078.stderr:+ sed -e s/rbd_data.b9f957b0ec03\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-10-01T11:15:44.304 INFO:tasks.workunit.client.0.smithi078.stderr:+ sort -u 2023-10-01T11:15:44.304 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd info sparse 2023-10-01T11:15:44.304 INFO:tasks.workunit.client.0.smithi078.stderr:+ grep data_pool: 2023-10-01T11:15:44.305 INFO:tasks.workunit.client.0.smithi078.stderr:+ awk -F: { print $NF } 2023-10-01T11:15:44.326 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:44.354 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:44.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:44.357 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:15:44.359 INFO:tasks.workunit.client.0.smithi078.stderr:+ data_pool= 2023-10-01T11:15:44.359 INFO:tasks.workunit.client.0.smithi078.stderr:+ [ -z ] 2023-10-01T11:15:44.359 INFO:tasks.workunit.client.0.smithi078.stderr:+ data_pool=rbd 2023-10-01T11:15:44.359 INFO:tasks.workunit.client.0.smithi078.stderr:+ echo rbd 2023-10-01T11:15:44.359 INFO:tasks.workunit.client.0.smithi078.stderr:+ rados ls -p rbd 2023-10-01T11:15:44.403 INFO:tasks.workunit.client.0.smithi078.stderr:+ objects= 2023-10-01T11:15:44.404 INFO:tasks.workunit.client.0.smithi078.stderr:+ echo 2023-10-01T11:15:44.404 INFO:tasks.workunit.client.0.smithi078.stderr:+ [ 0 -eq 1 -o = ] 2023-10-01T11:15:44.404 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd rm sparse 2023-10-01T11:15:44.457 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:44.485 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:44.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:44.489 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:15:44.543 INFO:tasks.workunit.client.0.smithi078.stderr: Removing image: 100% complete...done. 2023-10-01T11:15:44.549 INFO:tasks.workunit.client.0.smithi078.stderr:+ echo zeros export to sparse file 2023-10-01T11:15:44.549 INFO:tasks.workunit.client.0.smithi078.stdout:zeros export to sparse file 2023-10-01T11:15:44.549 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd create --new-format sparse --size 4 2023-10-01T11:15:44.590 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:44.602 INFO:tasks.workunit.client.0.smithi078.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-10-01T11:15:44.606 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd info sparse 2023-10-01T11:15:44.607 INFO:tasks.workunit.client.0.smithi078.stderr:+ grep block_name_prefix 2023-10-01T11:15:44.607 INFO:tasks.workunit.client.0.smithi078.stderr:+ awk {print $NF;} 2023-10-01T11:15:44.622 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5 2023-10-01T11:15:44.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:44.625 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.5' 2023-10-01T11:15:44.659 INFO:tasks.workunit.client.0.smithi078.stderr:+ prefix=rbd_data.93218c3a0a87 2023-10-01T11:15:44.660 INFO:tasks.workunit.client.0.smithi078.stderr:+ dd if=/dev/zero bs=4M count=1 2023-10-01T11:15:44.660 INFO:tasks.workunit.client.0.smithi078.stderr:+ get_image_data_pool sparse 2023-10-01T11:15:44.660 INFO:tasks.workunit.client.0.smithi078.stderr:+ image=sparse 2023-10-01T11:15:44.660 INFO:tasks.workunit.client.0.smithi078.stderr:+ + rbd info sparse 2023-10-01T11:15:44.661 INFO:tasks.workunit.client.0.smithi078.stderr:grep data_pool: 2023-10-01T11:15:44.661 INFO:tasks.workunit.client.0.smithi078.stderr:+ awk -F: { print $NF } 2023-10-01T11:15:44.715 INFO:tasks.workunit.client.0.smithi078.stderr:+ data_pool= 2023-10-01T11:15:44.715 INFO:tasks.workunit.client.0.smithi078.stderr:+ [ -z ] 2023-10-01T11:15:44.715 INFO:tasks.workunit.client.0.smithi078.stderr:+ data_pool=rbd 2023-10-01T11:15:44.715 INFO:tasks.workunit.client.0.smithi078.stderr:+ echo rbd 2023-10-01T11:15:44.716 INFO:tasks.workunit.client.0.smithi078.stderr:+ rados -p rbd put rbd_data.93218c3a0a87.000000000000 - 2023-10-01T11:15:44.726 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:44.754 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7 2023-10-01T11:15:44.755 INFO:tasks.workunit.client.0.smithi078.stderr:1+0 records in 2023-10-01T11:15:44.755 INFO:tasks.workunit.client.0.smithi078.stderr:1+0 records out 2023-10-01T11:15:44.755 INFO:tasks.workunit.client.0.smithi078.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0943311 s, 44.5 MB/s 2023-10-01T11:15:44.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:44.757 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.7' 2023-10-01T11:15:44.789 INFO:tasks.workunit.client.0.smithi078.stderr:+ objects sparse 2023-10-01T11:15:44.789 INFO:tasks.workunit.client.0.smithi078.stderr:+ image=sparse 2023-10-01T11:15:44.790 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd info sparse 2023-10-01T11:15:44.790 INFO:tasks.workunit.client.0.smithi078.stderr:+ grep block_name_prefix 2023-10-01T11:15:44.790 INFO:tasks.workunit.client.0.smithi078.stderr:+ awk {print $NF;} 2023-10-01T11:15:44.849 INFO:tasks.workunit.client.0.smithi078.stderr:+ prefix=rbd_data.93218c3a0a87 2023-10-01T11:15:44.849 INFO:tasks.workunit.client.0.smithi078.stderr:+ grep rbd_data.93218c3a0a87 2023-10-01T11:15:44.849 INFO:tasks.workunit.client.0.smithi078.stderr:+ sed -e s/rbd_data.93218c3a0a87\.// -e s/^0*\([0-9a-f]\)/\1/ 2023-10-01T11:15:44.849 INFO:tasks.workunit.client.0.smithi078.stderr:+ sort -u 2023-10-01T11:15:44.850 INFO:tasks.workunit.client.0.smithi078.stderr:+ get_image_data_pool sparse 2023-10-01T11:15:44.850 INFO:tasks.workunit.client.0.smithi078.stderr:+ image=sparse 2023-10-01T11:15:44.851 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd info sparse 2023-10-01T11:15:44.851 INFO:tasks.workunit.client.0.smithi078.stderr:+ grep data_pool: 2023-10-01T11:15:44.851 INFO:tasks.workunit.client.0.smithi078.stderr:+ awk -F: { print $NF } 2023-10-01T11:15:44.859 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:44.883 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:44.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:44.885 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:15:44.903 INFO:tasks.workunit.client.0.smithi078.stderr:+ data_pool= 2023-10-01T11:15:44.904 INFO:tasks.workunit.client.0.smithi078.stderr:+ [ -z ] 2023-10-01T11:15:44.904 INFO:tasks.workunit.client.0.smithi078.stderr:+ data_pool=rbd 2023-10-01T11:15:44.904 INFO:tasks.workunit.client.0.smithi078.stderr:+ echo rbd 2023-10-01T11:15:44.904 INFO:tasks.workunit.client.0.smithi078.stderr:+ rados ls -p rbd 2023-10-01T11:15:44.947 INFO:tasks.workunit.client.0.smithi078.stderr:+ objects=0 2023-10-01T11:15:44.947 INFO:tasks.workunit.client.0.smithi078.stderr:+ echo 0 2023-10-01T11:15:44.947 INFO:tasks.workunit.client.0.smithi078.stderr:+ [ 0 -eq 1 -o 0 = 0 ] 2023-10-01T11:15:44.947 INFO:tasks.workunit.client.0.smithi078.stderr:+ rm /tmp/rbd_import_export_259593/sparse 2023-10-01T11:15:44.948 INFO:tasks.workunit.client.0.smithi078.stderr:rm: cannot remove '/tmp/rbd_import_export_259593/sparse': No such file or directory 2023-10-01T11:15:44.948 INFO:tasks.workunit.client.0.smithi078.stderr:+ true 2023-10-01T11:15:44.948 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd export sparse /tmp/rbd_import_export_259593/sparse 2023-10-01T11:15:44.986 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:44.986 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0: {} 2023-10-01T11:15:44.999 INFO:tasks.workunit.client.0.smithi078.stderr: Exporting image: 100% complete...done. 2023-10-01T11:15:45.006 INFO:tasks.workunit.client.0.smithi078.stderr:+ stat /tmp/rbd_import_export_259593/sparse --format=%b 2023-10-01T11:15:45.008 INFO:tasks.workunit.client.0.smithi078.stderr:+ [ 0 = 0 ] 2023-10-01T11:15:45.008 INFO:tasks.workunit.client.0.smithi078.stderr:+ rbd rm sparse 2023-10-01T11:15:45.009 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1: {} 2023-10-01T11:15:45.012 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3 2023-10-01T11:15:45.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:45.015 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.3' 2023-10-01T11:15:45.035 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2: {} 2023-10-01T11:15:45.062 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3: {} 2023-10-01T11:15:45.090 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4: {} 2023-10-01T11:15:45.116 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:45.118 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5: {} 2023-10-01T11:15:45.143 INFO:tasks.workunit.client.0.smithi078.stderr: Removing image: 100% complete...done. 2023-10-01T11:15:45.145 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6: {} 2023-10-01T11:15:45.147 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:45.149 INFO:tasks.workunit.client.0.smithi078.stderr:+ rm /tmp/rbd_import_export_259593/sparse /tmp/rbd_import_export_259593/sparse1 /tmp/rbd_import_export_259593/sparse2 /tmp/rbd_import_export_259593/sparse3 2023-10-01T11:15:45.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:45.150 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:15:45.151 INFO:tasks.workunit.client.0.smithi078.stderr:rm: cannot remove '/tmp/rbd_import_export_259593/sparse3': No such file or directory 2023-10-01T11:15:45.152 INFO:tasks.workunit.client.0.smithi078.stdout:OK 2023-10-01T11:15:45.152 INFO:tasks.workunit.client.0.smithi078.stderr:+ true 2023-10-01T11:15:45.152 INFO:tasks.workunit.client.0.smithi078.stderr:+ echo OK 2023-10-01T11:15:45.152 INFO:tasks.workunit.client.0.smithi078.stderr:+ rm -rf /tmp/rbd_import_export_259593 2023-10-01T11:15:45.153 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-10-01T11:15:45.153 DEBUG:teuthology.orchestra.run.smithi078:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-10-01T11:15:45.172 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7: {} 2023-10-01T11:15:45.174 INFO:teuthology.orchestra.run.smithi078.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-10-01T11:15:45.174 INFO:teuthology.orchestra.run.smithi078.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-10-01T11:15:45.174 INFO:teuthology.orchestra.run.smithi078.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-10-01T11:15:45.174 INFO:teuthology.orchestra.run.smithi078.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-10-01T11:15:45.174 INFO:teuthology.orchestra.run.smithi078.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-10-01T11:15:45.174 INFO:teuthology.orchestra.run.smithi078.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-10-01T11:15:45.175 INFO:teuthology.orchestra.run.smithi078.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-10-01T11:15:45.175 INFO:teuthology.orchestra.run.smithi078.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-10-01T11:15:45.181 INFO:tasks.workunit:Stopping ['rbd/import_export.sh'] on client.0... 2023-10-01T11:15:45.181 DEBUG:teuthology.orchestra.run.smithi078:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-10-01T11:15:45.251 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:45.284 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6 2023-10-01T11:15:45.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:45.287 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.6' 2023-10-01T11:15:45.388 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:45.414 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2 2023-10-01T11:15:45.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:45.417 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.2' 2023-10-01T11:15:45.456 DEBUG:teuthology.parallel:result is None 2023-10-01T11:15:45.456 DEBUG:teuthology.orchestra.run.smithi078:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-10-01T11:15:45.511 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-10-01T11:15:45.511 DEBUG:teuthology.orchestra.run.smithi078:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-10-01T11:15:45.518 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:45.561 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-10-01T11:15:45.561 INFO:teuthology.task.sequential:In sequential, running task print... 2023-10-01T11:15:45.561 INFO:teuthology.task.print:**** done rbd/import_export.sh 5-workload 2023-10-01T11:15:45.561 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-10-01T11:15:45.562 INFO:tasks.workunit:Pulling workunits from ref pacific 2023-10-01T11:15:45.562 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-10-01T11:15:45.562 DEBUG:teuthology.orchestra.run.smithi078:> stat -- /home/ubuntu/cephtest/mnt.0 2023-10-01T11:15:45.581 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:45.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:45.584 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:15:45.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:45.586 INFO:teuthology.orchestra.run.smithi078.stderr:stat: cannot stat '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-10-01T11:15:45.586 DEBUG:teuthology.orchestra.run.smithi078:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-10-01T11:15:45.685 DEBUG:teuthology.orchestra.run.smithi078:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:45.714 INFO:teuthology.orchestra.run.smithi078.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0 2023-10-01T11:15:45.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:45.716 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi078 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.0' 2023-10-01T11:15:45.817 DEBUG:teuthology.orchestra.run.smithi173:> sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:45.850 INFO:teuthology.orchestra.run.smithi173.stderr:Error response from daemon: Cannot kill container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4 2023-10-01T11:15:45.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:15:45.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi173 with status 1: 'sudo docker kill -s 1 ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd.4' 2023-10-01T11:15:45.929 INFO:teuthology.orchestra.run.smithi078.stderr:nodeep-scrub is set 2023-10-01T11:15:45.932 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-10-01T11:15:45.932 DEBUG:teuthology.orchestra.run.smithi078:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-10-01T11:15:45.937 INFO:tasks.thrashosds:joining thrashosds 2023-10-01T11:15:45.954 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:45 smithi078 bash[78274]: cluster 2023-10-01T11:15:44.516603+0000 mgr.y (mgr.25072) 1756 : cluster 0 pgmap v1920: 73 pgs: 73 active+clean; 1.6 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 1.1 MiB/s rd, 966 KiB/s wr, 403 op/s 2023-10-01T11:15:45.954 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:45 smithi078 bash[78274]: audit 2023-10-01T11:15:44.957818+0000 mon.a (mon.0) 2251 : audit 0 from='client.? 172.21.15.78:0/3643785611' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:15:45.954 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:45 smithi078 bash[78274]: audit 2023-10-01T11:15:45.223063+0000 mon.a (mon.0) 2252 : audit 1 from='client.? 172.21.15.78:0/3277978897' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:15:45.954 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:45 smithi078 bash[75299]: cluster 2023-10-01T11:15:44.516603+0000 mgr.y (mgr.25072) 1756 : cluster 0 pgmap v1920: 73 pgs: 73 active+clean; 1.6 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 1.1 MiB/s rd, 966 KiB/s wr, 403 op/s 2023-10-01T11:15:45.954 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:45 smithi078 bash[75299]: audit 2023-10-01T11:15:44.957818+0000 mon.a (mon.0) 2251 : audit 0 from='client.? 172.21.15.78:0/3643785611' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:15:45.954 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:45 smithi078 bash[75299]: audit 2023-10-01T11:15:45.223063+0000 mon.a (mon.0) 2252 : audit 1 from='client.? 172.21.15.78:0/3277978897' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:15:46.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:45 smithi173 bash[58816]: cluster 2023-10-01T11:15:44.516603+0000 mgr.y (mgr.25072) 1756 : cluster 0 pgmap v1920: 73 pgs: 73 active+clean; 1.6 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 1.1 MiB/s rd, 966 KiB/s wr, 403 op/s 2023-10-01T11:15:46.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:45 smithi173 bash[58816]: audit 2023-10-01T11:15:44.957818+0000 mon.a (mon.0) 2251 : audit 0 from='client.? 172.21.15.78:0/3643785611' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-10-01T11:15:46.120 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:45 smithi173 bash[58816]: audit 2023-10-01T11:15:45.223063+0000 mon.a (mon.0) 2252 : audit 1 from='client.? 172.21.15.78:0/3277978897' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:15:47.217 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:46 smithi078 bash[75299]: cluster 2023-10-01T11:15:45.913058+0000 mon.a (mon.0) 2253 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:15:47.217 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:46 smithi078 bash[75299]: cluster 2023-10-01T11:15:45.920842+0000 osd.5 (osd.5) 436 : cluster 3 failed to encode map e406 with expected crc 2023-10-01T11:15:47.217 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:46 smithi078 bash[75299]: audit 2023-10-01T11:15:45.922341+0000 mon.a (mon.0) 2254 : audit 1 from='client.? 172.21.15.78:0/3277978897' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T11:15:47.217 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:46 smithi078 bash[75299]: cluster 2023-10-01T11:15:45.922417+0000 mon.a (mon.0) 2255 : cluster 0 osdmap e406: 8 total, 8 up, 8 in 2023-10-01T11:15:47.217 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:46 smithi078 bash[78274]: cluster 2023-10-01T11:15:45.913058+0000 mon.a (mon.0) 2253 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:15:47.217 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:46 smithi078 bash[78274]: cluster 2023-10-01T11:15:45.920842+0000 osd.5 (osd.5) 436 : cluster 3 failed to encode map e406 with expected crc 2023-10-01T11:15:47.217 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:46 smithi078 bash[78274]: audit 2023-10-01T11:15:45.922341+0000 mon.a (mon.0) 2254 : audit 1 from='client.? 172.21.15.78:0/3277978897' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T11:15:47.218 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:46 smithi078 bash[78274]: cluster 2023-10-01T11:15:45.922417+0000 mon.a (mon.0) 2255 : cluster 0 osdmap e406: 8 total, 8 up, 8 in 2023-10-01T11:15:47.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:46 smithi173 bash[58816]: cluster 2023-10-01T11:15:45.913058+0000 mon.a (mon.0) 2253 : cluster 3 Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:15:47.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:46 smithi173 bash[58816]: cluster 2023-10-01T11:15:45.920842+0000 osd.5 (osd.5) 436 : cluster 3 failed to encode map e406 with expected crc 2023-10-01T11:15:47.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:46 smithi173 bash[58816]: audit 2023-10-01T11:15:45.922341+0000 mon.a (mon.0) 2254 : audit 1 from='client.? 172.21.15.78:0/3277978897' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-10-01T11:15:47.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:46 smithi173 bash[58816]: cluster 2023-10-01T11:15:45.922417+0000 mon.a (mon.0) 2255 : cluster 0 osdmap e406: 8 total, 8 up, 8 in 2023-10-01T11:15:47.699 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:15:48.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:47 smithi078 bash[75299]: cluster 2023-10-01T11:15:46.517079+0000 mgr.y (mgr.25072) 1757 : cluster 0 pgmap v1922: 73 pgs: 73 active+clean; 1.6 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 871 KiB/s rd, 833 KiB/s wr, 255 op/s 2023-10-01T11:15:48.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:47 smithi078 bash[75299]: audit 2023-10-01T11:15:47.484793+0000 mon.a (mon.0) 2256 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:15:48.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:47 smithi078 bash[75299]: cluster 2023-10-01T11:15:47.912626+0000 mon.a (mon.0) 2257 : cluster 0 osdmap e407: 8 total, 8 up, 8 in 2023-10-01T11:15:48.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:47 smithi078 bash[78274]: cluster 2023-10-01T11:15:46.517079+0000 mgr.y (mgr.25072) 1757 : cluster 0 pgmap v1922: 73 pgs: 73 active+clean; 1.6 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 871 KiB/s rd, 833 KiB/s wr, 255 op/s 2023-10-01T11:15:48.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:47 smithi078 bash[78274]: audit 2023-10-01T11:15:47.484793+0000 mon.a (mon.0) 2256 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:15:48.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:47 smithi078 bash[78274]: cluster 2023-10-01T11:15:47.912626+0000 mon.a (mon.0) 2257 : cluster 0 osdmap e407: 8 total, 8 up, 8 in 2023-10-01T11:15:48.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:47 smithi173 bash[58816]: cluster 2023-10-01T11:15:46.517079+0000 mgr.y (mgr.25072) 1757 : cluster 0 pgmap v1922: 73 pgs: 73 active+clean; 1.6 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 871 KiB/s rd, 833 KiB/s wr, 255 op/s 2023-10-01T11:15:48.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:47 smithi173 bash[58816]: audit 2023-10-01T11:15:47.484793+0000 mon.a (mon.0) 2256 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:15:48.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:47 smithi173 bash[58816]: cluster 2023-10-01T11:15:47.912626+0000 mon.a (mon.0) 2257 : cluster 0 osdmap e407: 8 total, 8 up, 8 in 2023-10-01T11:15:49.219 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd unset noscrub 2023-10-01T11:15:49.228 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:48 smithi078 bash[75299]: cluster 2023-10-01T11:15:48.056094+0000 osd.5 (osd.5) 437 : cluster 3 failed to encode map e407 with expected crc 2023-10-01T11:15:49.229 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:48 smithi078 bash[75299]: cluster 2023-10-01T11:15:48.056254+0000 osd.6 (osd.6) 392 : cluster 3 failed to encode map e407 with expected crc 2023-10-01T11:15:49.229 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:48 smithi078 bash[75299]: cluster 2023-10-01T11:15:48.056312+0000 osd.7 (osd.7) 455 : cluster 3 failed to encode map e407 with expected crc 2023-10-01T11:15:49.229 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:48 smithi078 bash[78274]: cluster 2023-10-01T11:15:48.056094+0000 osd.5 (osd.5) 437 : cluster 3 failed to encode map e407 with expected crc 2023-10-01T11:15:49.230 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:48 smithi078 bash[78274]: cluster 2023-10-01T11:15:48.056254+0000 osd.6 (osd.6) 392 : cluster 3 failed to encode map e407 with expected crc 2023-10-01T11:15:49.230 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:48 smithi078 bash[78274]: cluster 2023-10-01T11:15:48.056312+0000 osd.7 (osd.7) 455 : cluster 3 failed to encode map e407 with expected crc 2023-10-01T11:15:49.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:48 smithi173 bash[58816]: cluster 2023-10-01T11:15:48.056094+0000 osd.5 (osd.5) 437 : cluster 3 failed to encode map e407 with expected crc 2023-10-01T11:15:49.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:48 smithi173 bash[58816]: cluster 2023-10-01T11:15:48.056254+0000 osd.6 (osd.6) 392 : cluster 3 failed to encode map e407 with expected crc 2023-10-01T11:15:49.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:48 smithi173 bash[58816]: cluster 2023-10-01T11:15:48.056312+0000 osd.7 (osd.7) 455 : cluster 3 failed to encode map e407 with expected crc 2023-10-01T11:15:50.210 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:49 smithi078 bash[75299]: cluster 2023-10-01T11:15:48.517641+0000 mgr.y (mgr.25072) 1758 : cluster 0 pgmap v1924: 73 pgs: 73 active+clean; 4.6 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.7 MiB/s wr, 380 op/s 2023-10-01T11:15:50.210 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:49 smithi078 bash[78274]: cluster 2023-10-01T11:15:48.517641+0000 mgr.y (mgr.25072) 1758 : cluster 0 pgmap v1924: 73 pgs: 73 active+clean; 4.6 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.7 MiB/s wr, 380 op/s 2023-10-01T11:15:50.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:49 smithi173 bash[58816]: cluster 2023-10-01T11:15:48.517641+0000 mgr.y (mgr.25072) 1758 : cluster 0 pgmap v1924: 73 pgs: 73 active+clean; 4.6 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.7 MiB/s wr, 380 op/s 2023-10-01T11:15:50.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:15:50 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:15:50] "GET /metrics HTTP/1.1" 200 34020 "" "Prometheus/2.33.4" 2023-10-01T11:15:50.675 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:15:52.254 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:51 smithi078 bash[78274]: cluster 2023-10-01T11:15:50.518132+0000 mgr.y (mgr.25072) 1759 : cluster 0 pgmap v1925: 73 pgs: 73 active+clean; 4.6 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 1.0 MiB/s rd, 1.5 MiB/s wr, 335 op/s 2023-10-01T11:15:52.254 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:51 smithi078 bash[75299]: cluster 2023-10-01T11:15:50.518132+0000 mgr.y (mgr.25072) 1759 : cluster 0 pgmap v1925: 73 pgs: 73 active+clean; 4.6 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 1.0 MiB/s rd, 1.5 MiB/s wr, 335 op/s 2023-10-01T11:15:52.273 INFO:teuthology.orchestra.run.smithi078.stdout:ERROR: (22) Invalid argument 2023-10-01T11:15:52.273 INFO:teuthology.orchestra.run.smithi078.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T11:15:52.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:51 smithi173 bash[58816]: cluster 2023-10-01T11:15:50.518132+0000 mgr.y (mgr.25072) 1759 : cluster 0 pgmap v1925: 73 pgs: 73 active+clean; 4.6 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 1.0 MiB/s rd, 1.5 MiB/s wr, 335 op/s 2023-10-01T11:15:53.813 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-10-01T11:15:54.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:53 smithi078 bash[78274]: cluster 2023-10-01T11:15:52.518884+0000 mgr.y (mgr.25072) 1760 : cluster 0 pgmap v1926: 73 pgs: 73 active+clean; 4.6 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 202 KiB/s rd, 643 KiB/s wr, 98 op/s 2023-10-01T11:15:54.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:53 smithi078 bash[75299]: cluster 2023-10-01T11:15:52.518884+0000 mgr.y (mgr.25072) 1760 : cluster 0 pgmap v1926: 73 pgs: 73 active+clean; 4.6 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 202 KiB/s rd, 643 KiB/s wr, 98 op/s 2023-10-01T11:15:54.295 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:15:54.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:53 smithi173 bash[58816]: cluster 2023-10-01T11:15:52.518884+0000 mgr.y (mgr.25072) 1760 : cluster 0 pgmap v1926: 73 pgs: 73 active+clean; 4.6 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 202 KiB/s rd, 643 KiB/s wr, 98 op/s 2023-10-01T11:15:56.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:55 smithi078 bash[75299]: cluster 2023-10-01T11:15:54.519453+0000 mgr.y (mgr.25072) 1761 : cluster 0 pgmap v1927: 73 pgs: 73 active+clean; 4.6 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 188 KiB/s rd, 598 KiB/s wr, 91 op/s 2023-10-01T11:15:56.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:55 smithi078 bash[78274]: cluster 2023-10-01T11:15:54.519453+0000 mgr.y (mgr.25072) 1761 : cluster 0 pgmap v1927: 73 pgs: 73 active+clean; 4.6 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 188 KiB/s rd, 598 KiB/s wr, 91 op/s 2023-10-01T11:15:56.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:55 smithi173 bash[58816]: cluster 2023-10-01T11:15:54.519453+0000 mgr.y (mgr.25072) 1761 : cluster 0 pgmap v1927: 73 pgs: 73 active+clean; 4.6 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 188 KiB/s rd, 598 KiB/s wr, 91 op/s 2023-10-01T11:15:56.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:56 smithi078 bash[75299]: debug 2023-10-01T11:15:56.252+0000 7f31c8dd0700 -1 mon.a@0(leader).osd e407 definitely_dead 0 2023-10-01T11:15:56.991 INFO:teuthology.orchestra.run.smithi078.stderr:marked out osd.0. 2023-10-01T11:15:57.341 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:56 smithi078 bash[75299]: audit 2023-10-01T11:15:56.253388+0000 mon.c (mon.2) 255 : audit 1 from='client.? 172.21.15.78:0/3842977107' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-10-01T11:15:57.341 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:56 smithi078 bash[75299]: audit 2023-10-01T11:15:56.253871+0000 mon.a (mon.0) 2258 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-10-01T11:15:57.341 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:56 smithi078 bash[75299]: cluster 2023-10-01T11:15:56.253936+0000 mon.a (mon.0) 2259 : cluster 1 Client client.admin marked osd.0 out, while it was still marked up 2023-10-01T11:15:57.342 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:56 smithi078 bash[78274]: audit 2023-10-01T11:15:56.253388+0000 mon.c (mon.2) 255 : audit 1 from='client.? 172.21.15.78:0/3842977107' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-10-01T11:15:57.342 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:56 smithi078 bash[78274]: audit 2023-10-01T11:15:56.253871+0000 mon.a (mon.0) 2258 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-10-01T11:15:57.342 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:56 smithi078 bash[78274]: cluster 2023-10-01T11:15:56.253936+0000 mon.a (mon.0) 2259 : cluster 1 Client client.admin marked osd.0 out, while it was still marked up 2023-10-01T11:15:57.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:56 smithi173 bash[58816]: audit 2023-10-01T11:15:56.253388+0000 mon.c (mon.2) 255 : audit 1 from='client.? 172.21.15.78:0/3842977107' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-10-01T11:15:57.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:56 smithi173 bash[58816]: audit 2023-10-01T11:15:56.253871+0000 mon.a (mon.0) 2258 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-10-01T11:15:57.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:56 smithi173 bash[58816]: cluster 2023-10-01T11:15:56.253936+0000 mon.a (mon.0) 2259 : cluster 1 Client client.admin marked osd.0 out, while it was still marked up 2023-10-01T11:15:58.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:57 smithi078 bash[75299]: cluster 2023-10-01T11:15:56.520157+0000 mgr.y (mgr.25072) 1762 : cluster 0 pgmap v1928: 73 pgs: 73 active+clean; 4.6 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 161 KiB/s rd, 514 KiB/s wr, 78 op/s 2023-10-01T11:15:58.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:57 smithi078 bash[75299]: audit 2023-10-01T11:15:56.971768+0000 mon.a (mon.0) 2260 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2023-10-01T11:15:58.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:57 smithi078 bash[75299]: cluster 2023-10-01T11:15:56.971852+0000 osd.7 (osd.7) 456 : cluster 3 failed to encode map e408 with expected crc 2023-10-01T11:15:58.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:57 smithi078 bash[75299]: cluster 2023-10-01T11:15:56.971904+0000 mon.a (mon.0) 2261 : cluster 0 osdmap e408: 8 total, 8 up, 7 in 2023-10-01T11:15:58.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:57 smithi078 bash[75299]: cluster 2023-10-01T11:15:56.976238+0000 osd.5 (osd.5) 438 : cluster 3 failed to encode map e408 with expected crc 2023-10-01T11:15:58.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:57 smithi078 bash[75299]: cluster 2023-10-01T11:15:56.976323+0000 osd.7 (osd.7) 457 : cluster 3 failed to encode map e408 with expected crc 2023-10-01T11:15:58.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:57 smithi078 bash[75299]: cluster 2023-10-01T11:15:56.978216+0000 osd.6 (osd.6) 393 : cluster 3 failed to encode map e408 with expected crc 2023-10-01T11:15:58.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:57 smithi078 bash[75299]: cluster 2023-10-01T11:15:56.978280+0000 osd.5 (osd.5) 439 : cluster 3 failed to encode map e408 with expected crc 2023-10-01T11:15:58.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:57 smithi078 bash[75299]: cluster 2023-10-01T11:15:56.979022+0000 osd.7 (osd.7) 458 : cluster 3 failed to encode map e408 with expected crc 2023-10-01T11:15:58.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:57 smithi078 bash[75299]: cluster 2023-10-01T11:15:56.979901+0000 osd.5 (osd.5) 440 : cluster 3 failed to encode map e408 with expected crc 2023-10-01T11:15:58.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:57 smithi078 bash[75299]: cluster 2023-10-01T11:15:56.980553+0000 osd.7 (osd.7) 459 : cluster 3 failed to encode map e408 with expected crc 2023-10-01T11:15:58.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:57 smithi078 bash[75299]: cluster 2023-10-01T11:15:56.982452+0000 osd.7 (osd.7) 460 : cluster 3 failed to encode map e408 with expected crc 2023-10-01T11:15:58.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:57 smithi078 bash[75299]: cluster 2023-10-01T11:15:57.918465+0000 mon.a (mon.0) 2262 : cluster 0 osdmap e409: 8 total, 8 up, 7 in 2023-10-01T11:15:58.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:57 smithi078 bash[78274]: cluster 2023-10-01T11:15:56.520157+0000 mgr.y (mgr.25072) 1762 : cluster 0 pgmap v1928: 73 pgs: 73 active+clean; 4.6 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 161 KiB/s rd, 514 KiB/s wr, 78 op/s 2023-10-01T11:15:58.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:57 smithi078 bash[78274]: audit 2023-10-01T11:15:56.971768+0000 mon.a (mon.0) 2260 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2023-10-01T11:15:58.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:57 smithi078 bash[78274]: cluster 2023-10-01T11:15:56.971852+0000 osd.7 (osd.7) 456 : cluster 3 failed to encode map e408 with expected crc 2023-10-01T11:15:58.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:57 smithi078 bash[78274]: cluster 2023-10-01T11:15:56.971904+0000 mon.a (mon.0) 2261 : cluster 0 osdmap e408: 8 total, 8 up, 7 in 2023-10-01T11:15:58.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:57 smithi078 bash[78274]: cluster 2023-10-01T11:15:56.976238+0000 osd.5 (osd.5) 438 : cluster 3 failed to encode map e408 with expected crc 2023-10-01T11:15:58.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:57 smithi078 bash[78274]: cluster 2023-10-01T11:15:56.976323+0000 osd.7 (osd.7) 457 : cluster 3 failed to encode map e408 with expected crc 2023-10-01T11:15:58.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:57 smithi078 bash[78274]: cluster 2023-10-01T11:15:56.978216+0000 osd.6 (osd.6) 393 : cluster 3 failed to encode map e408 with expected crc 2023-10-01T11:15:58.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:57 smithi078 bash[78274]: cluster 2023-10-01T11:15:56.978280+0000 osd.5 (osd.5) 439 : cluster 3 failed to encode map e408 with expected crc 2023-10-01T11:15:58.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:57 smithi078 bash[78274]: cluster 2023-10-01T11:15:56.979022+0000 osd.7 (osd.7) 458 : cluster 3 failed to encode map e408 with expected crc 2023-10-01T11:15:58.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:57 smithi078 bash[78274]: cluster 2023-10-01T11:15:56.979901+0000 osd.5 (osd.5) 440 : cluster 3 failed to encode map e408 with expected crc 2023-10-01T11:15:58.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:57 smithi078 bash[78274]: cluster 2023-10-01T11:15:56.980553+0000 osd.7 (osd.7) 459 : cluster 3 failed to encode map e408 with expected crc 2023-10-01T11:15:58.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:57 smithi078 bash[78274]: cluster 2023-10-01T11:15:56.982452+0000 osd.7 (osd.7) 460 : cluster 3 failed to encode map e408 with expected crc 2023-10-01T11:15:58.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:57 smithi078 bash[78274]: cluster 2023-10-01T11:15:57.918465+0000 mon.a (mon.0) 2262 : cluster 0 osdmap e409: 8 total, 8 up, 7 in 2023-10-01T11:15:58.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:57 smithi173 bash[58816]: cluster 2023-10-01T11:15:56.520157+0000 mgr.y (mgr.25072) 1762 : cluster 0 pgmap v1928: 73 pgs: 73 active+clean; 4.6 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 161 KiB/s rd, 514 KiB/s wr, 78 op/s 2023-10-01T11:15:58.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:57 smithi173 bash[58816]: audit 2023-10-01T11:15:56.971768+0000 mon.a (mon.0) 2260 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2023-10-01T11:15:58.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:57 smithi173 bash[58816]: cluster 2023-10-01T11:15:56.971852+0000 osd.7 (osd.7) 456 : cluster 3 failed to encode map e408 with expected crc 2023-10-01T11:15:58.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:57 smithi173 bash[58816]: cluster 2023-10-01T11:15:56.971904+0000 mon.a (mon.0) 2261 : cluster 0 osdmap e408: 8 total, 8 up, 7 in 2023-10-01T11:15:58.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:57 smithi173 bash[58816]: cluster 2023-10-01T11:15:56.976238+0000 osd.5 (osd.5) 438 : cluster 3 failed to encode map e408 with expected crc 2023-10-01T11:15:58.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:57 smithi173 bash[58816]: cluster 2023-10-01T11:15:56.976323+0000 osd.7 (osd.7) 457 : cluster 3 failed to encode map e408 with expected crc 2023-10-01T11:15:58.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:57 smithi173 bash[58816]: cluster 2023-10-01T11:15:56.978216+0000 osd.6 (osd.6) 393 : cluster 3 failed to encode map e408 with expected crc 2023-10-01T11:15:58.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:57 smithi173 bash[58816]: cluster 2023-10-01T11:15:56.978280+0000 osd.5 (osd.5) 439 : cluster 3 failed to encode map e408 with expected crc 2023-10-01T11:15:58.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:57 smithi173 bash[58816]: cluster 2023-10-01T11:15:56.979022+0000 osd.7 (osd.7) 458 : cluster 3 failed to encode map e408 with expected crc 2023-10-01T11:15:58.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:57 smithi173 bash[58816]: cluster 2023-10-01T11:15:56.979901+0000 osd.5 (osd.5) 440 : cluster 3 failed to encode map e408 with expected crc 2023-10-01T11:15:58.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:57 smithi173 bash[58816]: cluster 2023-10-01T11:15:56.980553+0000 osd.7 (osd.7) 459 : cluster 3 failed to encode map e408 with expected crc 2023-10-01T11:15:58.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:57 smithi173 bash[58816]: cluster 2023-10-01T11:15:56.982452+0000 osd.7 (osd.7) 460 : cluster 3 failed to encode map e408 with expected crc 2023-10-01T11:15:58.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:57 smithi173 bash[58816]: cluster 2023-10-01T11:15:57.918465+0000 mon.a (mon.0) 2262 : cluster 0 osdmap e409: 8 total, 8 up, 7 in 2023-10-01T11:15:59.259 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:58 smithi078 bash[78274]: cluster 2023-10-01T11:15:57.915351+0000 osd.7 (osd.7) 461 : cluster 3 failed to encode map e409 with expected crc 2023-10-01T11:15:59.259 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:58 smithi078 bash[78274]: cluster 2023-10-01T11:15:57.920139+0000 osd.5 (osd.5) 441 : cluster 3 failed to encode map e409 with expected crc 2023-10-01T11:15:59.259 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:58 smithi078 bash[78274]: cluster 2023-10-01T11:15:57.921337+0000 osd.7 (osd.7) 462 : cluster 3 failed to encode map e409 with expected crc 2023-10-01T11:15:59.259 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:58 smithi078 bash[78274]: cluster 2023-10-01T11:15:57.922461+0000 osd.6 (osd.6) 394 : cluster 3 failed to encode map e409 with expected crc 2023-10-01T11:15:59.259 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:58 smithi078 bash[78274]: cluster 2023-10-01T11:15:57.924170+0000 osd.6 (osd.6) 395 : cluster 3 failed to encode map e409 with expected crc 2023-10-01T11:15:59.260 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:58 smithi078 bash[78274]: cluster 2023-10-01T11:15:58.932248+0000 mon.a (mon.0) 2263 : cluster 0 osdmap e410: 8 total, 8 up, 7 in 2023-10-01T11:15:59.260 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:58 smithi078 bash[75299]: cluster 2023-10-01T11:15:57.915351+0000 osd.7 (osd.7) 461 : cluster 3 failed to encode map e409 with expected crc 2023-10-01T11:15:59.260 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:58 smithi078 bash[75299]: cluster 2023-10-01T11:15:57.920139+0000 osd.5 (osd.5) 441 : cluster 3 failed to encode map e409 with expected crc 2023-10-01T11:15:59.260 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:58 smithi078 bash[75299]: cluster 2023-10-01T11:15:57.921337+0000 osd.7 (osd.7) 462 : cluster 3 failed to encode map e409 with expected crc 2023-10-01T11:15:59.260 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:58 smithi078 bash[75299]: cluster 2023-10-01T11:15:57.922461+0000 osd.6 (osd.6) 394 : cluster 3 failed to encode map e409 with expected crc 2023-10-01T11:15:59.260 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:58 smithi078 bash[75299]: cluster 2023-10-01T11:15:57.924170+0000 osd.6 (osd.6) 395 : cluster 3 failed to encode map e409 with expected crc 2023-10-01T11:15:59.260 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:58 smithi078 bash[75299]: cluster 2023-10-01T11:15:58.932248+0000 mon.a (mon.0) 2263 : cluster 0 osdmap e410: 8 total, 8 up, 7 in 2023-10-01T11:15:59.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:58 smithi173 bash[58816]: cluster 2023-10-01T11:15:57.915351+0000 osd.7 (osd.7) 461 : cluster 3 failed to encode map e409 with expected crc 2023-10-01T11:15:59.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:58 smithi173 bash[58816]: cluster 2023-10-01T11:15:57.920139+0000 osd.5 (osd.5) 441 : cluster 3 failed to encode map e409 with expected crc 2023-10-01T11:15:59.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:58 smithi173 bash[58816]: cluster 2023-10-01T11:15:57.921337+0000 osd.7 (osd.7) 462 : cluster 3 failed to encode map e409 with expected crc 2023-10-01T11:15:59.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:58 smithi173 bash[58816]: cluster 2023-10-01T11:15:57.922461+0000 osd.6 (osd.6) 394 : cluster 3 failed to encode map e409 with expected crc 2023-10-01T11:15:59.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:58 smithi173 bash[58816]: cluster 2023-10-01T11:15:57.924170+0000 osd.6 (osd.6) 395 : cluster 3 failed to encode map e409 with expected crc 2023-10-01T11:15:59.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:58 smithi173 bash[58816]: cluster 2023-10-01T11:15:58.932248+0000 mon.a (mon.0) 2263 : cluster 0 osdmap e410: 8 total, 8 up, 7 in 2023-10-01T11:15:59.691 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:15:59.998 INFO:teuthology.orchestra.run.smithi078.stderr:noscrub is unset 2023-10-01T11:16:00.269 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:59 smithi078 bash[75299]: cluster 2023-10-01T11:15:58.520601+0000 mgr.y (mgr.25072) 1763 : cluster 0 pgmap v1931: 73 pgs: 3 remapped+peering, 70 active+clean; 4.6 MiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 2 B/s, 0 objects/s recovering 2023-10-01T11:16:00.269 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:59 smithi078 bash[75299]: cluster 2023-10-01T11:15:58.928990+0000 osd.5 (osd.5) 442 : cluster 3 failed to encode map e410 with expected crc 2023-10-01T11:16:00.269 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:59 smithi078 bash[75299]: cluster 2023-10-01T11:15:58.929125+0000 osd.7 (osd.7) 463 : cluster 3 failed to encode map e410 with expected crc 2023-10-01T11:16:00.269 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:59 smithi078 bash[75299]: cluster 2023-10-01T11:15:58.933642+0000 osd.7 (osd.7) 464 : cluster 3 failed to encode map e410 with expected crc 2023-10-01T11:16:00.270 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:59 smithi078 bash[75299]: audit 2023-10-01T11:15:59.733232+0000 mon.c (mon.2) 256 : audit 1 from='client.? 172.21.15.78:0/3726909677' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T11:16:00.270 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:15:59 smithi078 bash[75299]: audit 2023-10-01T11:15:59.733858+0000 mon.a (mon.0) 2264 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T11:16:00.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:59 smithi078 bash[78274]: cluster 2023-10-01T11:15:58.520601+0000 mgr.y (mgr.25072) 1763 : cluster 0 pgmap v1931: 73 pgs: 3 remapped+peering, 70 active+clean; 4.6 MiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 2 B/s, 0 objects/s recovering 2023-10-01T11:16:00.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:59 smithi078 bash[78274]: cluster 2023-10-01T11:15:58.928990+0000 osd.5 (osd.5) 442 : cluster 3 failed to encode map e410 with expected crc 2023-10-01T11:16:00.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:59 smithi078 bash[78274]: cluster 2023-10-01T11:15:58.929125+0000 osd.7 (osd.7) 463 : cluster 3 failed to encode map e410 with expected crc 2023-10-01T11:16:00.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:59 smithi078 bash[78274]: cluster 2023-10-01T11:15:58.933642+0000 osd.7 (osd.7) 464 : cluster 3 failed to encode map e410 with expected crc 2023-10-01T11:16:00.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:59 smithi078 bash[78274]: audit 2023-10-01T11:15:59.733232+0000 mon.c (mon.2) 256 : audit 1 from='client.? 172.21.15.78:0/3726909677' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T11:16:00.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:15:59 smithi078 bash[78274]: audit 2023-10-01T11:15:59.733858+0000 mon.a (mon.0) 2264 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T11:16:00.271 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:16:00 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:16:00] "GET /metrics HTTP/1.1" 200 34020 "" "Prometheus/2.33.4" 2023-10-01T11:16:00.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:59 smithi173 bash[58816]: cluster 2023-10-01T11:15:58.520601+0000 mgr.y (mgr.25072) 1763 : cluster 0 pgmap v1931: 73 pgs: 3 remapped+peering, 70 active+clean; 4.6 MiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 2 B/s, 0 objects/s recovering 2023-10-01T11:16:00.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:59 smithi173 bash[58816]: cluster 2023-10-01T11:15:58.928990+0000 osd.5 (osd.5) 442 : cluster 3 failed to encode map e410 with expected crc 2023-10-01T11:16:00.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:59 smithi173 bash[58816]: cluster 2023-10-01T11:15:58.929125+0000 osd.7 (osd.7) 463 : cluster 3 failed to encode map e410 with expected crc 2023-10-01T11:16:00.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:59 smithi173 bash[58816]: cluster 2023-10-01T11:15:58.933642+0000 osd.7 (osd.7) 464 : cluster 3 failed to encode map e410 with expected crc 2023-10-01T11:16:00.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:59 smithi173 bash[58816]: audit 2023-10-01T11:15:59.733232+0000 mon.c (mon.2) 256 : audit 1 from='client.? 172.21.15.78:0/3726909677' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T11:16:00.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:15:59 smithi173 bash[58816]: audit 2023-10-01T11:15:59.733858+0000 mon.a (mon.0) 2264 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-10-01T11:16:01.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:00 smithi078 bash[75299]: cluster 2023-10-01T11:15:59.978923+0000 mon.a (mon.0) 2265 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:16:01.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:00 smithi078 bash[75299]: cluster 2023-10-01T11:15:59.990881+0000 osd.7 (osd.7) 465 : cluster 3 failed to encode map e411 with expected crc 2023-10-01T11:16:01.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:00 smithi078 bash[75299]: audit 2023-10-01T11:15:59.991390+0000 mon.a (mon.0) 2266 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T11:16:01.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:00 smithi078 bash[75299]: cluster 2023-10-01T11:15:59.991513+0000 mon.a (mon.0) 2267 : cluster 0 osdmap e411: 8 total, 8 up, 7 in 2023-10-01T11:16:01.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:00 smithi078 bash[75299]: audit 2023-10-01T11:15:59.991877+0000 mon.a (mon.0) 2268 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T11:16:01.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:00 smithi078 bash[75299]: cluster 2023-10-01T11:15:59.994034+0000 osd.7 (osd.7) 466 : cluster 3 failed to encode map e411 with expected crc 2023-10-01T11:16:01.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:00 smithi078 bash[75299]: cluster 2023-10-01T11:15:59.996467+0000 osd.7 (osd.7) 467 : cluster 3 failed to encode map e411 with expected crc 2023-10-01T11:16:01.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:00 smithi078 bash[75299]: cluster 2023-10-01T11:15:59.997803+0000 osd.7 (osd.7) 468 : cluster 3 failed to encode map e411 with expected crc 2023-10-01T11:16:01.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:00 smithi078 bash[75299]: cluster 2023-10-01T11:16:00.022770+0000 osd.5 (osd.5) 443 : cluster 3 failed to encode map e411 with expected crc 2023-10-01T11:16:01.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:00 smithi078 bash[75299]: cluster 2023-10-01T11:16:00.025568+0000 osd.5 (osd.5) 444 : cluster 3 failed to encode map e411 with expected crc 2023-10-01T11:16:01.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:00 smithi078 bash[78274]: cluster 2023-10-01T11:15:59.978923+0000 mon.a (mon.0) 2265 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:16:01.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:00 smithi078 bash[78274]: cluster 2023-10-01T11:15:59.990881+0000 osd.7 (osd.7) 465 : cluster 3 failed to encode map e411 with expected crc 2023-10-01T11:16:01.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:00 smithi078 bash[78274]: audit 2023-10-01T11:15:59.991390+0000 mon.a (mon.0) 2266 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T11:16:01.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:00 smithi078 bash[78274]: cluster 2023-10-01T11:15:59.991513+0000 mon.a (mon.0) 2267 : cluster 0 osdmap e411: 8 total, 8 up, 7 in 2023-10-01T11:16:01.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:00 smithi078 bash[78274]: audit 2023-10-01T11:15:59.991877+0000 mon.a (mon.0) 2268 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T11:16:01.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:00 smithi078 bash[78274]: cluster 2023-10-01T11:15:59.994034+0000 osd.7 (osd.7) 466 : cluster 3 failed to encode map e411 with expected crc 2023-10-01T11:16:01.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:00 smithi078 bash[78274]: cluster 2023-10-01T11:15:59.996467+0000 osd.7 (osd.7) 467 : cluster 3 failed to encode map e411 with expected crc 2023-10-01T11:16:01.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:00 smithi078 bash[78274]: cluster 2023-10-01T11:15:59.997803+0000 osd.7 (osd.7) 468 : cluster 3 failed to encode map e411 with expected crc 2023-10-01T11:16:01.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:00 smithi078 bash[78274]: cluster 2023-10-01T11:16:00.022770+0000 osd.5 (osd.5) 443 : cluster 3 failed to encode map e411 with expected crc 2023-10-01T11:16:01.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:00 smithi078 bash[78274]: cluster 2023-10-01T11:16:00.025568+0000 osd.5 (osd.5) 444 : cluster 3 failed to encode map e411 with expected crc 2023-10-01T11:16:01.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:00 smithi173 bash[58816]: cluster 2023-10-01T11:15:59.978923+0000 mon.a (mon.0) 2265 : cluster 3 Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-10-01T11:16:01.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:00 smithi173 bash[58816]: cluster 2023-10-01T11:15:59.990881+0000 osd.7 (osd.7) 465 : cluster 3 failed to encode map e411 with expected crc 2023-10-01T11:16:01.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:00 smithi173 bash[58816]: audit 2023-10-01T11:15:59.991390+0000 mon.a (mon.0) 2266 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-10-01T11:16:01.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:00 smithi173 bash[58816]: cluster 2023-10-01T11:15:59.991513+0000 mon.a (mon.0) 2267 : cluster 0 osdmap e411: 8 total, 8 up, 7 in 2023-10-01T11:16:01.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:00 smithi173 bash[58816]: audit 2023-10-01T11:15:59.991877+0000 mon.a (mon.0) 2268 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T11:16:01.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:00 smithi173 bash[58816]: cluster 2023-10-01T11:15:59.994034+0000 osd.7 (osd.7) 466 : cluster 3 failed to encode map e411 with expected crc 2023-10-01T11:16:01.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:00 smithi173 bash[58816]: cluster 2023-10-01T11:15:59.996467+0000 osd.7 (osd.7) 467 : cluster 3 failed to encode map e411 with expected crc 2023-10-01T11:16:01.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:00 smithi173 bash[58816]: cluster 2023-10-01T11:15:59.997803+0000 osd.7 (osd.7) 468 : cluster 3 failed to encode map e411 with expected crc 2023-10-01T11:16:01.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:00 smithi173 bash[58816]: cluster 2023-10-01T11:16:00.022770+0000 osd.5 (osd.5) 443 : cluster 3 failed to encode map e411 with expected crc 2023-10-01T11:16:01.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:00 smithi173 bash[58816]: cluster 2023-10-01T11:16:00.025568+0000 osd.5 (osd.5) 444 : cluster 3 failed to encode map e411 with expected crc 2023-10-01T11:16:02.151 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-10-01T11:16:02.290 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:01 smithi078 bash[75299]: cluster 2023-10-01T11:16:00.521292+0000 mgr.y (mgr.25072) 1764 : cluster 0 pgmap v1934: 73 pgs: 4 remapped+peering, 69 active+clean; 4.6 MiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 143 B/s, 4 objects/s recovering 2023-10-01T11:16:02.290 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:01 smithi078 bash[75299]: cluster 2023-10-01T11:16:01.000283+0000 osd.7 (osd.7) 469 : cluster 3 failed to encode map e412 with expected crc 2023-10-01T11:16:02.290 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:01 smithi078 bash[75299]: cluster 2023-10-01T11:16:01.000430+0000 mon.a (mon.0) 2269 : cluster 0 osdmap e412: 8 total, 8 up, 7 in 2023-10-01T11:16:02.290 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:01 smithi078 bash[75299]: cluster 2023-10-01T11:16:01.002343+0000 osd.7 (osd.7) 470 : cluster 3 failed to encode map e412 with expected crc 2023-10-01T11:16:02.291 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:01 smithi078 bash[75299]: cluster 2023-10-01T11:16:01.004071+0000 osd.7 (osd.7) 471 : cluster 3 failed to encode map e412 with expected crc 2023-10-01T11:16:02.291 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:01 smithi078 bash[75299]: cluster 2023-10-01T11:16:01.005678+0000 osd.7 (osd.7) 472 : cluster 3 failed to encode map e412 with expected crc 2023-10-01T11:16:02.291 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:01 smithi078 bash[75299]: cluster 2023-10-01T11:16:01.123874+0000 osd.5 (osd.5) 445 : cluster 3 failed to encode map e412 with expected crc 2023-10-01T11:16:02.291 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:01 smithi078 bash[75299]: cluster 2023-10-01T11:16:01.126479+0000 osd.5 (osd.5) 446 : cluster 3 failed to encode map e412 with expected crc 2023-10-01T11:16:02.291 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:01 smithi078 bash[78274]: cluster 2023-10-01T11:16:00.521292+0000 mgr.y (mgr.25072) 1764 : cluster 0 pgmap v1934: 73 pgs: 4 remapped+peering, 69 active+clean; 4.6 MiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 143 B/s, 4 objects/s recovering 2023-10-01T11:16:02.291 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:01 smithi078 bash[78274]: cluster 2023-10-01T11:16:01.000283+0000 osd.7 (osd.7) 469 : cluster 3 failed to encode map e412 with expected crc 2023-10-01T11:16:02.292 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:01 smithi078 bash[78274]: cluster 2023-10-01T11:16:01.000430+0000 mon.a (mon.0) 2269 : cluster 0 osdmap e412: 8 total, 8 up, 7 in 2023-10-01T11:16:02.292 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:01 smithi078 bash[78274]: cluster 2023-10-01T11:16:01.002343+0000 osd.7 (osd.7) 470 : cluster 3 failed to encode map e412 with expected crc 2023-10-01T11:16:02.292 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:01 smithi078 bash[78274]: cluster 2023-10-01T11:16:01.004071+0000 osd.7 (osd.7) 471 : cluster 3 failed to encode map e412 with expected crc 2023-10-01T11:16:02.292 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:01 smithi078 bash[78274]: cluster 2023-10-01T11:16:01.005678+0000 osd.7 (osd.7) 472 : cluster 3 failed to encode map e412 with expected crc 2023-10-01T11:16:02.292 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:01 smithi078 bash[78274]: cluster 2023-10-01T11:16:01.123874+0000 osd.5 (osd.5) 445 : cluster 3 failed to encode map e412 with expected crc 2023-10-01T11:16:02.292 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:01 smithi078 bash[78274]: cluster 2023-10-01T11:16:01.126479+0000 osd.5 (osd.5) 446 : cluster 3 failed to encode map e412 with expected crc 2023-10-01T11:16:02.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:01 smithi173 bash[58816]: cluster 2023-10-01T11:16:00.521292+0000 mgr.y (mgr.25072) 1764 : cluster 0 pgmap v1934: 73 pgs: 4 remapped+peering, 69 active+clean; 4.6 MiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 143 B/s, 4 objects/s recovering 2023-10-01T11:16:02.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:01 smithi173 bash[58816]: cluster 2023-10-01T11:16:01.000283+0000 osd.7 (osd.7) 469 : cluster 3 failed to encode map e412 with expected crc 2023-10-01T11:16:02.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:01 smithi173 bash[58816]: cluster 2023-10-01T11:16:01.000430+0000 mon.a (mon.0) 2269 : cluster 0 osdmap e412: 8 total, 8 up, 7 in 2023-10-01T11:16:02.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:01 smithi173 bash[58816]: cluster 2023-10-01T11:16:01.002343+0000 osd.7 (osd.7) 470 : cluster 3 failed to encode map e412 with expected crc 2023-10-01T11:16:02.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:01 smithi173 bash[58816]: cluster 2023-10-01T11:16:01.004071+0000 osd.7 (osd.7) 471 : cluster 3 failed to encode map e412 with expected crc 2023-10-01T11:16:02.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:01 smithi173 bash[58816]: cluster 2023-10-01T11:16:01.005678+0000 osd.7 (osd.7) 472 : cluster 3 failed to encode map e412 with expected crc 2023-10-01T11:16:02.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:01 smithi173 bash[58816]: cluster 2023-10-01T11:16:01.123874+0000 osd.5 (osd.5) 445 : cluster 3 failed to encode map e412 with expected crc 2023-10-01T11:16:02.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:01 smithi173 bash[58816]: cluster 2023-10-01T11:16:01.126479+0000 osd.5 (osd.5) 446 : cluster 3 failed to encode map e412 with expected crc 2023-10-01T11:16:03.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:03 smithi078 bash[78274]: audit 2023-10-01T11:16:02.485058+0000 mon.a (mon.0) 2270 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:16:03.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:03 smithi078 bash[75299]: audit 2023-10-01T11:16:02.485058+0000 mon.a (mon.0) 2270 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:16:03.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:03 smithi173 bash[58816]: audit 2023-10-01T11:16:02.485058+0000 mon.a (mon.0) 2270 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:16:04.211 INFO:tasks.thrashosds.thrasher:inning osd 2023-10-01T11:16:04.212 INFO:tasks.thrashosds.thrasher:Adding osd 0 2023-10-01T11:16:04.212 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd in 0 2023-10-01T11:16:04.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:04 smithi078 bash[75299]: cluster 2023-10-01T11:16:02.521763+0000 mgr.y (mgr.25072) 1765 : cluster 0 pgmap v1936: 73 pgs: 3 remapped+peering, 70 active+clean; 4.6 MiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 124 B/s, 6 objects/s recovering 2023-10-01T11:16:04.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:04 smithi078 bash[78274]: cluster 2023-10-01T11:16:02.521763+0000 mgr.y (mgr.25072) 1765 : cluster 0 pgmap v1936: 73 pgs: 3 remapped+peering, 70 active+clean; 4.6 MiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 124 B/s, 6 objects/s recovering 2023-10-01T11:16:04.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:04 smithi173 bash[58816]: cluster 2023-10-01T11:16:02.521763+0000 mgr.y (mgr.25072) 1765 : cluster 0 pgmap v1936: 73 pgs: 3 remapped+peering, 70 active+clean; 4.6 MiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 124 B/s, 6 objects/s recovering 2023-10-01T11:16:04.833 INFO:teuthology.orchestra.run.smithi078.stdout:ERROR: (22) Invalid argument 2023-10-01T11:16:04.834 INFO:teuthology.orchestra.run.smithi078.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T11:16:05.839 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-10-01T11:16:06.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:06 smithi078 bash[78274]: cluster 2023-10-01T11:16:04.522719+0000 mgr.y (mgr.25072) 1766 : cluster 0 pgmap v1937: 73 pgs: 73 active+clean; 4.6 MiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 779 KiB/s, 7 objects/s recovering 2023-10-01T11:16:06.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:06 smithi078 bash[78274]: audit 2023-10-01T11:16:05.936130+0000 mon.a (mon.0) 2271 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:16:06.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:06 smithi078 bash[78274]: audit 2023-10-01T11:16:05.943255+0000 mon.a (mon.0) 2272 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:16:06.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:06 smithi078 bash[75299]: cluster 2023-10-01T11:16:04.522719+0000 mgr.y (mgr.25072) 1766 : cluster 0 pgmap v1937: 73 pgs: 73 active+clean; 4.6 MiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 779 KiB/s, 7 objects/s recovering 2023-10-01T11:16:06.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:06 smithi078 bash[75299]: audit 2023-10-01T11:16:05.936130+0000 mon.a (mon.0) 2271 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:16:06.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:06 smithi078 bash[75299]: audit 2023-10-01T11:16:05.943255+0000 mon.a (mon.0) 2272 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:16:06.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:06 smithi173 bash[58816]: cluster 2023-10-01T11:16:04.522719+0000 mgr.y (mgr.25072) 1766 : cluster 0 pgmap v1937: 73 pgs: 73 active+clean; 4.6 MiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 779 KiB/s, 7 objects/s recovering 2023-10-01T11:16:06.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:06 smithi173 bash[58816]: audit 2023-10-01T11:16:05.936130+0000 mon.a (mon.0) 2271 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:16:06.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:06 smithi173 bash[58816]: audit 2023-10-01T11:16:05.943255+0000 mon.a (mon.0) 2272 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:16:07.334 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:16:07.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:07 smithi078 bash[78274]: audit 2023-10-01T11:16:06.286826+0000 mon.a (mon.0) 2273 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:16:07.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:07 smithi078 bash[78274]: audit 2023-10-01T11:16:06.293583+0000 mon.a (mon.0) 2274 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:16:07.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:07 smithi078 bash[78274]: cluster 2023-10-01T11:16:06.523536+0000 mgr.y (mgr.25072) 1767 : cluster 0 pgmap v1938: 73 pgs: 73 active+clean; 4.6 MiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 615 KiB/s, 5 objects/s recovering 2023-10-01T11:16:07.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:07 smithi078 bash[78274]: audit 2023-10-01T11:16:06.768258+0000 mon.a (mon.0) 2275 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T11:16:07.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:07 smithi078 bash[78274]: audit 2023-10-01T11:16:06.770059+0000 mon.a (mon.0) 2276 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T11:16:07.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:07 smithi078 bash[78274]: audit 2023-10-01T11:16:06.779306+0000 mon.a (mon.0) 2277 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:16:07.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:07 smithi078 bash[75299]: audit 2023-10-01T11:16:06.286826+0000 mon.a (mon.0) 2273 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:16:07.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:07 smithi078 bash[75299]: audit 2023-10-01T11:16:06.293583+0000 mon.a (mon.0) 2274 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:16:07.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:07 smithi078 bash[75299]: cluster 2023-10-01T11:16:06.523536+0000 mgr.y (mgr.25072) 1767 : cluster 0 pgmap v1938: 73 pgs: 73 active+clean; 4.6 MiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 615 KiB/s, 5 objects/s recovering 2023-10-01T11:16:07.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:07 smithi078 bash[75299]: audit 2023-10-01T11:16:06.768258+0000 mon.a (mon.0) 2275 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T11:16:07.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:07 smithi078 bash[75299]: audit 2023-10-01T11:16:06.770059+0000 mon.a (mon.0) 2276 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T11:16:07.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:07 smithi078 bash[75299]: audit 2023-10-01T11:16:06.779306+0000 mon.a (mon.0) 2277 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:16:07.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:07 smithi173 bash[58816]: audit 2023-10-01T11:16:06.286826+0000 mon.a (mon.0) 2273 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:16:07.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:07 smithi173 bash[58816]: audit 2023-10-01T11:16:06.293583+0000 mon.a (mon.0) 2274 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:16:07.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:07 smithi173 bash[58816]: cluster 2023-10-01T11:16:06.523536+0000 mgr.y (mgr.25072) 1767 : cluster 0 pgmap v1938: 73 pgs: 73 active+clean; 4.6 MiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 615 KiB/s, 5 objects/s recovering 2023-10-01T11:16:07.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:07 smithi173 bash[58816]: audit 2023-10-01T11:16:06.768258+0000 mon.a (mon.0) 2275 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T11:16:07.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:07 smithi173 bash[58816]: audit 2023-10-01T11:16:06.770059+0000 mon.a (mon.0) 2276 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T11:16:07.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:07 smithi173 bash[58816]: audit 2023-10-01T11:16:06.779306+0000 mon.a (mon.0) 2277 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:16:09.969 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:16:10.270 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:09 smithi078 bash[75299]: cluster 2023-10-01T11:16:08.524232+0000 mgr.y (mgr.25072) 1768 : cluster 0 pgmap v1939: 73 pgs: 73 active+clean; 4.6 MiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 547 KiB/s, 3 objects/s recovering 2023-10-01T11:16:10.271 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:09 smithi078 bash[78274]: cluster 2023-10-01T11:16:08.524232+0000 mgr.y (mgr.25072) 1768 : cluster 0 pgmap v1939: 73 pgs: 73 active+clean; 4.6 MiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 547 KiB/s, 3 objects/s recovering 2023-10-01T11:16:10.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:09 smithi173 bash[58816]: cluster 2023-10-01T11:16:08.524232+0000 mgr.y (mgr.25072) 1768 : cluster 0 pgmap v1939: 73 pgs: 73 active+clean; 4.6 MiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 547 KiB/s, 3 objects/s recovering 2023-10-01T11:16:10.381 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:16:10.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:16:10 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:16:10] "GET /metrics HTTP/1.1" 200 34025 "" "Prometheus/2.33.4" 2023-10-01T11:16:12.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:11 smithi078 bash[75299]: cluster 2023-10-01T11:16:10.524837+0000 mgr.y (mgr.25072) 1769 : cluster 0 pgmap v1940: 73 pgs: 73 active+clean; 4.6 MiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 467 KiB/s, 2 objects/s recovering 2023-10-01T11:16:12.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:11 smithi078 bash[78274]: cluster 2023-10-01T11:16:10.524837+0000 mgr.y (mgr.25072) 1769 : cluster 0 pgmap v1940: 73 pgs: 73 active+clean; 4.6 MiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 467 KiB/s, 2 objects/s recovering 2023-10-01T11:16:12.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:11 smithi173 bash[58816]: cluster 2023-10-01T11:16:10.524837+0000 mgr.y (mgr.25072) 1769 : cluster 0 pgmap v1940: 73 pgs: 73 active+clean; 4.6 MiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 467 KiB/s, 2 objects/s recovering 2023-10-01T11:16:12.940 INFO:teuthology.orchestra.run.smithi078.stderr:nodeep-scrub is unset 2023-10-01T11:16:13.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:12 smithi078 bash[78274]: audit 2023-10-01T11:16:12.463681+0000 mon.a (mon.0) 2278 : audit 1 from='client.? 172.21.15.78:0/1433296814' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:16:13.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:12 smithi078 bash[75299]: audit 2023-10-01T11:16:12.463681+0000 mon.a (mon.0) 2278 : audit 1 from='client.? 172.21.15.78:0/1433296814' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:16:13.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:12 smithi173 bash[58816]: audit 2023-10-01T11:16:12.463681+0000 mon.a (mon.0) 2278 : audit 1 from='client.? 172.21.15.78:0/1433296814' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-10-01T11:16:14.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:13 smithi078 bash[75299]: cluster 2023-10-01T11:16:12.525431+0000 mgr.y (mgr.25072) 1770 : cluster 0 pgmap v1941: 73 pgs: 73 active+clean; 4.6 MiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 405 KiB/s, 2 objects/s recovering 2023-10-01T11:16:14.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:13 smithi078 bash[75299]: cluster 2023-10-01T11:16:12.924944+0000 mon.a (mon.0) 2279 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T11:16:14.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:13 smithi078 bash[75299]: cluster 2023-10-01T11:16:12.924972+0000 mon.a (mon.0) 2280 : cluster 1 Cluster is now healthy 2023-10-01T11:16:14.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:13 smithi078 bash[75299]: cluster 2023-10-01T11:16:12.932608+0000 osd.6 (osd.6) 396 : cluster 3 failed to encode map e413 with expected crc 2023-10-01T11:16:14.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:13 smithi078 bash[75299]: cluster 2023-10-01T11:16:12.933053+0000 osd.7 (osd.7) 473 : cluster 3 failed to encode map e413 with expected crc 2023-10-01T11:16:14.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:13 smithi078 bash[75299]: audit 2023-10-01T11:16:12.934113+0000 mon.a (mon.0) 2281 : audit 1 from='client.? 172.21.15.78:0/1433296814' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T11:16:14.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:13 smithi078 bash[75299]: cluster 2023-10-01T11:16:12.934157+0000 mon.a (mon.0) 2282 : cluster 0 osdmap e413: 8 total, 8 up, 7 in 2023-10-01T11:16:14.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:13 smithi078 bash[75299]: audit 2023-10-01T11:16:13.450031+0000 mon.a (mon.0) 2283 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:16:14.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:13 smithi078 bash[78274]: cluster 2023-10-01T11:16:12.525431+0000 mgr.y (mgr.25072) 1770 : cluster 0 pgmap v1941: 73 pgs: 73 active+clean; 4.6 MiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 405 KiB/s, 2 objects/s recovering 2023-10-01T11:16:14.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:13 smithi078 bash[78274]: cluster 2023-10-01T11:16:12.924944+0000 mon.a (mon.0) 2279 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T11:16:14.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:13 smithi078 bash[78274]: cluster 2023-10-01T11:16:12.924972+0000 mon.a (mon.0) 2280 : cluster 1 Cluster is now healthy 2023-10-01T11:16:14.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:13 smithi078 bash[78274]: cluster 2023-10-01T11:16:12.932608+0000 osd.6 (osd.6) 396 : cluster 3 failed to encode map e413 with expected crc 2023-10-01T11:16:14.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:13 smithi078 bash[78274]: cluster 2023-10-01T11:16:12.933053+0000 osd.7 (osd.7) 473 : cluster 3 failed to encode map e413 with expected crc 2023-10-01T11:16:14.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:13 smithi078 bash[78274]: audit 2023-10-01T11:16:12.934113+0000 mon.a (mon.0) 2281 : audit 1 from='client.? 172.21.15.78:0/1433296814' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T11:16:14.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:13 smithi078 bash[78274]: cluster 2023-10-01T11:16:12.934157+0000 mon.a (mon.0) 2282 : cluster 0 osdmap e413: 8 total, 8 up, 7 in 2023-10-01T11:16:14.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:13 smithi078 bash[78274]: audit 2023-10-01T11:16:13.450031+0000 mon.a (mon.0) 2283 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:16:14.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:13 smithi173 bash[58816]: cluster 2023-10-01T11:16:12.525431+0000 mgr.y (mgr.25072) 1770 : cluster 0 pgmap v1941: 73 pgs: 73 active+clean; 4.6 MiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 405 KiB/s, 2 objects/s recovering 2023-10-01T11:16:14.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:13 smithi173 bash[58816]: cluster 2023-10-01T11:16:12.924944+0000 mon.a (mon.0) 2279 : cluster 1 Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-10-01T11:16:14.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:13 smithi173 bash[58816]: cluster 2023-10-01T11:16:12.924972+0000 mon.a (mon.0) 2280 : cluster 1 Cluster is now healthy 2023-10-01T11:16:14.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:13 smithi173 bash[58816]: cluster 2023-10-01T11:16:12.932608+0000 osd.6 (osd.6) 396 : cluster 3 failed to encode map e413 with expected crc 2023-10-01T11:16:14.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:13 smithi173 bash[58816]: cluster 2023-10-01T11:16:12.933053+0000 osd.7 (osd.7) 473 : cluster 3 failed to encode map e413 with expected crc 2023-10-01T11:16:14.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:13 smithi173 bash[58816]: audit 2023-10-01T11:16:12.934113+0000 mon.a (mon.0) 2281 : audit 1 from='client.? 172.21.15.78:0/1433296814' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-10-01T11:16:14.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:13 smithi173 bash[58816]: cluster 2023-10-01T11:16:12.934157+0000 mon.a (mon.0) 2282 : cluster 0 osdmap e413: 8 total, 8 up, 7 in 2023-10-01T11:16:14.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:13 smithi173 bash[58816]: audit 2023-10-01T11:16:13.450031+0000 mon.a (mon.0) 2283 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:16:15.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:14 smithi078 bash[75299]: cluster 2023-10-01T11:16:13.481332+0000 osd.5 (osd.5) 447 : cluster 3 failed to encode map e413 with expected crc 2023-10-01T11:16:15.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:14 smithi078 bash[78274]: cluster 2023-10-01T11:16:13.481332+0000 osd.5 (osd.5) 447 : cluster 3 failed to encode map e413 with expected crc 2023-10-01T11:16:15.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:14 smithi173 bash[58816]: cluster 2023-10-01T11:16:13.481332+0000 osd.5 (osd.5) 447 : cluster 3 failed to encode map e413 with expected crc 2023-10-01T11:16:16.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:15 smithi078 bash[75299]: cluster 2023-10-01T11:16:14.526298+0000 mgr.y (mgr.25072) 1771 : cluster 0 pgmap v1943: 73 pgs: 73 active+clean; 4.6 MiB data, 7.3 GiB used, 618 GiB / 626 GiB avail 2023-10-01T11:16:16.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:15 smithi078 bash[78274]: cluster 2023-10-01T11:16:14.526298+0000 mgr.y (mgr.25072) 1771 : cluster 0 pgmap v1943: 73 pgs: 73 active+clean; 4.6 MiB data, 7.3 GiB used, 618 GiB / 626 GiB avail 2023-10-01T11:16:16.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:15 smithi173 bash[58816]: cluster 2023-10-01T11:16:14.526298+0000 mgr.y (mgr.25072) 1771 : cluster 0 pgmap v1943: 73 pgs: 73 active+clean; 4.6 MiB data, 7.3 GiB used, 618 GiB / 626 GiB avail 2023-10-01T11:16:16.896 INFO:teuthology.orchestra.run.smithi078.stdout:ERROR: (22) Invalid argument 2023-10-01T11:16:16.896 INFO:teuthology.orchestra.run.smithi078.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T11:16:17.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:17 smithi078 bash[75299]: debug 2023-10-01T11:16:17.164+0000 7f31c8dd0700 -1 mon.a@0(leader).osd e413 definitely_dead 0 2023-10-01T11:16:17.836 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-10-01T11:16:17.967 INFO:teuthology.orchestra.run.smithi078.stderr:marked in osd.0. 2023-10-01T11:16:18.220 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:17 smithi078 bash[78274]: cluster 2023-10-01T11:16:16.527050+0000 mgr.y (mgr.25072) 1772 : cluster 0 pgmap v1944: 73 pgs: 73 active+clean; 4.6 MiB data, 7.3 GiB used, 618 GiB / 626 GiB avail 2023-10-01T11:16:18.220 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:17 smithi078 bash[78274]: audit 2023-10-01T11:16:17.166925+0000 mon.a (mon.0) 2284 : audit 0 from='client.? 172.21.15.78:0/1169930568' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-10-01T11:16:18.220 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:17 smithi078 bash[78274]: audit 2023-10-01T11:16:17.491084+0000 mon.a (mon.0) 2285 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:16:18.220 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:17 smithi078 bash[78274]: audit 2023-10-01T11:16:17.492281+0000 mon.a (mon.0) 2286 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:16:18.220 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:17 smithi078 bash[75299]: cluster 2023-10-01T11:16:16.527050+0000 mgr.y (mgr.25072) 1772 : cluster 0 pgmap v1944: 73 pgs: 73 active+clean; 4.6 MiB data, 7.3 GiB used, 618 GiB / 626 GiB avail 2023-10-01T11:16:18.220 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:17 smithi078 bash[75299]: audit 2023-10-01T11:16:17.166925+0000 mon.a (mon.0) 2284 : audit 0 from='client.? 172.21.15.78:0/1169930568' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-10-01T11:16:18.221 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:17 smithi078 bash[75299]: audit 2023-10-01T11:16:17.491084+0000 mon.a (mon.0) 2285 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:16:18.221 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:17 smithi078 bash[75299]: audit 2023-10-01T11:16:17.492281+0000 mon.a (mon.0) 2286 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:16:18.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:17 smithi173 bash[58816]: cluster 2023-10-01T11:16:16.527050+0000 mgr.y (mgr.25072) 1772 : cluster 0 pgmap v1944: 73 pgs: 73 active+clean; 4.6 MiB data, 7.3 GiB used, 618 GiB / 626 GiB avail 2023-10-01T11:16:18.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:17 smithi173 bash[58816]: audit 2023-10-01T11:16:17.166925+0000 mon.a (mon.0) 2284 : audit 0 from='client.? 172.21.15.78:0/1169930568' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-10-01T11:16:18.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:17 smithi173 bash[58816]: audit 2023-10-01T11:16:17.491084+0000 mon.a (mon.0) 2285 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:16:18.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:17 smithi173 bash[58816]: audit 2023-10-01T11:16:17.492281+0000 mon.a (mon.0) 2286 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:16:19.089 INFO:tasks.thrashosds.thrasher:Added osd 0 2023-10-01T11:16:19.089 INFO:tasks.thrashosds.thrasher:fixing pg num pool device_health_metrics 2023-10-01T11:16:19.089 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd pool set device_health_metrics pgp_num 31 2023-10-01T11:16:19.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:18 smithi078 bash[75299]: cluster 2023-10-01T11:16:17.957756+0000 osd.7 (osd.7) 474 : cluster 3 failed to encode map e414 with expected crc 2023-10-01T11:16:19.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:18 smithi078 bash[75299]: audit 2023-10-01T11:16:17.958506+0000 mon.a (mon.0) 2287 : audit 1 from='client.? 172.21.15.78:0/1169930568' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2023-10-01T11:16:19.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:18 smithi078 bash[75299]: cluster 2023-10-01T11:16:17.958587+0000 mon.a (mon.0) 2288 : cluster 0 osdmap e414: 8 total, 8 up, 8 in 2023-10-01T11:16:19.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:18 smithi078 bash[75299]: cluster 2023-10-01T11:16:17.962015+0000 osd.7 (osd.7) 475 : cluster 3 failed to encode map e414 with expected crc 2023-10-01T11:16:19.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:18 smithi078 bash[75299]: cluster 2023-10-01T11:16:17.962892+0000 osd.5 (osd.5) 448 : cluster 3 failed to encode map e414 with expected crc 2023-10-01T11:16:19.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:18 smithi078 bash[75299]: cluster 2023-10-01T11:16:17.963932+0000 osd.7 (osd.7) 476 : cluster 3 failed to encode map e414 with expected crc 2023-10-01T11:16:19.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:18 smithi078 bash[75299]: cluster 2023-10-01T11:16:17.965667+0000 osd.7 (osd.7) 477 : cluster 3 failed to encode map e414 with expected crc 2023-10-01T11:16:19.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:18 smithi078 bash[75299]: cluster 2023-10-01T11:16:17.965690+0000 osd.6 (osd.6) 397 : cluster 3 failed to encode map e414 with expected crc 2023-10-01T11:16:19.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:18 smithi078 bash[75299]: cluster 2023-10-01T11:16:17.966091+0000 osd.5 (osd.5) 449 : cluster 3 failed to encode map e414 with expected crc 2023-10-01T11:16:19.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:18 smithi078 bash[75299]: cluster 2023-10-01T11:16:17.968945+0000 osd.6 (osd.6) 398 : cluster 3 failed to encode map e414 with expected crc 2023-10-01T11:16:19.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:18 smithi078 bash[78274]: cluster 2023-10-01T11:16:17.957756+0000 osd.7 (osd.7) 474 : cluster 3 failed to encode map e414 with expected crc 2023-10-01T11:16:19.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:18 smithi078 bash[78274]: audit 2023-10-01T11:16:17.958506+0000 mon.a (mon.0) 2287 : audit 1 from='client.? 172.21.15.78:0/1169930568' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2023-10-01T11:16:19.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:18 smithi078 bash[78274]: cluster 2023-10-01T11:16:17.958587+0000 mon.a (mon.0) 2288 : cluster 0 osdmap e414: 8 total, 8 up, 8 in 2023-10-01T11:16:19.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:18 smithi078 bash[78274]: cluster 2023-10-01T11:16:17.962015+0000 osd.7 (osd.7) 475 : cluster 3 failed to encode map e414 with expected crc 2023-10-01T11:16:19.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:18 smithi078 bash[78274]: cluster 2023-10-01T11:16:17.962892+0000 osd.5 (osd.5) 448 : cluster 3 failed to encode map e414 with expected crc 2023-10-01T11:16:19.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:18 smithi078 bash[78274]: cluster 2023-10-01T11:16:17.963932+0000 osd.7 (osd.7) 476 : cluster 3 failed to encode map e414 with expected crc 2023-10-01T11:16:19.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:18 smithi078 bash[78274]: cluster 2023-10-01T11:16:17.965667+0000 osd.7 (osd.7) 477 : cluster 3 failed to encode map e414 with expected crc 2023-10-01T11:16:19.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:18 smithi078 bash[78274]: cluster 2023-10-01T11:16:17.965690+0000 osd.6 (osd.6) 397 : cluster 3 failed to encode map e414 with expected crc 2023-10-01T11:16:19.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:18 smithi078 bash[78274]: cluster 2023-10-01T11:16:17.966091+0000 osd.5 (osd.5) 449 : cluster 3 failed to encode map e414 with expected crc 2023-10-01T11:16:19.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:18 smithi078 bash[78274]: cluster 2023-10-01T11:16:17.968945+0000 osd.6 (osd.6) 398 : cluster 3 failed to encode map e414 with expected crc 2023-10-01T11:16:19.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:18 smithi173 bash[58816]: cluster 2023-10-01T11:16:17.957756+0000 osd.7 (osd.7) 474 : cluster 3 failed to encode map e414 with expected crc 2023-10-01T11:16:19.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:18 smithi173 bash[58816]: audit 2023-10-01T11:16:17.958506+0000 mon.a (mon.0) 2287 : audit 1 from='client.? 172.21.15.78:0/1169930568' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2023-10-01T11:16:19.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:18 smithi173 bash[58816]: cluster 2023-10-01T11:16:17.958587+0000 mon.a (mon.0) 2288 : cluster 0 osdmap e414: 8 total, 8 up, 8 in 2023-10-01T11:16:19.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:18 smithi173 bash[58816]: cluster 2023-10-01T11:16:17.962015+0000 osd.7 (osd.7) 475 : cluster 3 failed to encode map e414 with expected crc 2023-10-01T11:16:19.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:18 smithi173 bash[58816]: cluster 2023-10-01T11:16:17.962892+0000 osd.5 (osd.5) 448 : cluster 3 failed to encode map e414 with expected crc 2023-10-01T11:16:19.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:18 smithi173 bash[58816]: cluster 2023-10-01T11:16:17.963932+0000 osd.7 (osd.7) 476 : cluster 3 failed to encode map e414 with expected crc 2023-10-01T11:16:19.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:18 smithi173 bash[58816]: cluster 2023-10-01T11:16:17.965667+0000 osd.7 (osd.7) 477 : cluster 3 failed to encode map e414 with expected crc 2023-10-01T11:16:19.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:18 smithi173 bash[58816]: cluster 2023-10-01T11:16:17.965690+0000 osd.6 (osd.6) 397 : cluster 3 failed to encode map e414 with expected crc 2023-10-01T11:16:19.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:18 smithi173 bash[58816]: cluster 2023-10-01T11:16:17.966091+0000 osd.5 (osd.5) 449 : cluster 3 failed to encode map e414 with expected crc 2023-10-01T11:16:19.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:18 smithi173 bash[58816]: cluster 2023-10-01T11:16:17.968945+0000 osd.6 (osd.6) 398 : cluster 3 failed to encode map e414 with expected crc 2023-10-01T11:16:20.269 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:19 smithi078 bash[78274]: cluster 2023-10-01T11:16:18.527790+0000 mgr.y (mgr.25072) 1773 : cluster 0 pgmap v1946: 73 pgs: 5 peering, 68 active+clean; 4.6 MiB data, 7.3 GiB used, 618 GiB / 626 GiB avail 2023-10-01T11:16:20.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:19 smithi078 bash[78274]: cluster 2023-10-01T11:16:18.967127+0000 osd.7 (osd.7) 478 : cluster 3 failed to encode map e415 with expected crc 2023-10-01T11:16:20.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:19 smithi078 bash[78274]: cluster 2023-10-01T11:16:18.969985+0000 mon.a (mon.0) 2289 : cluster 0 osdmap e415: 8 total, 8 up, 8 in 2023-10-01T11:16:20.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:19 smithi078 bash[78274]: cluster 2023-10-01T11:16:18.971206+0000 osd.7 (osd.7) 479 : cluster 3 failed to encode map e415 with expected crc 2023-10-01T11:16:20.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:19 smithi078 bash[78274]: cluster 2023-10-01T11:16:18.971531+0000 osd.6 (osd.6) 399 : cluster 3 failed to encode map e415 with expected crc 2023-10-01T11:16:20.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:19 smithi078 bash[78274]: cluster 2023-10-01T11:16:18.971901+0000 osd.5 (osd.5) 450 : cluster 3 failed to encode map e415 with expected crc 2023-10-01T11:16:20.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:19 smithi078 bash[78274]: cluster 2023-10-01T11:16:18.972391+0000 osd.7 (osd.7) 480 : cluster 3 failed to encode map e415 with expected crc 2023-10-01T11:16:20.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:19 smithi078 bash[78274]: cluster 2023-10-01T11:16:18.973692+0000 osd.7 (osd.7) 481 : cluster 3 failed to encode map e415 with expected crc 2023-10-01T11:16:20.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:19 smithi078 bash[78274]: cluster 2023-10-01T11:16:18.973924+0000 osd.5 (osd.5) 451 : cluster 3 failed to encode map e415 with expected crc 2023-10-01T11:16:20.270 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:19 smithi078 bash[75299]: cluster 2023-10-01T11:16:18.527790+0000 mgr.y (mgr.25072) 1773 : cluster 0 pgmap v1946: 73 pgs: 5 peering, 68 active+clean; 4.6 MiB data, 7.3 GiB used, 618 GiB / 626 GiB avail 2023-10-01T11:16:20.271 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:19 smithi078 bash[75299]: cluster 2023-10-01T11:16:18.967127+0000 osd.7 (osd.7) 478 : cluster 3 failed to encode map e415 with expected crc 2023-10-01T11:16:20.271 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:19 smithi078 bash[75299]: cluster 2023-10-01T11:16:18.969985+0000 mon.a (mon.0) 2289 : cluster 0 osdmap e415: 8 total, 8 up, 8 in 2023-10-01T11:16:20.271 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:19 smithi078 bash[75299]: cluster 2023-10-01T11:16:18.971206+0000 osd.7 (osd.7) 479 : cluster 3 failed to encode map e415 with expected crc 2023-10-01T11:16:20.271 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:19 smithi078 bash[75299]: cluster 2023-10-01T11:16:18.971531+0000 osd.6 (osd.6) 399 : cluster 3 failed to encode map e415 with expected crc 2023-10-01T11:16:20.271 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:19 smithi078 bash[75299]: cluster 2023-10-01T11:16:18.971901+0000 osd.5 (osd.5) 450 : cluster 3 failed to encode map e415 with expected crc 2023-10-01T11:16:20.271 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:19 smithi078 bash[75299]: cluster 2023-10-01T11:16:18.972391+0000 osd.7 (osd.7) 480 : cluster 3 failed to encode map e415 with expected crc 2023-10-01T11:16:20.271 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:19 smithi078 bash[75299]: cluster 2023-10-01T11:16:18.973692+0000 osd.7 (osd.7) 481 : cluster 3 failed to encode map e415 with expected crc 2023-10-01T11:16:20.271 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:19 smithi078 bash[75299]: cluster 2023-10-01T11:16:18.973924+0000 osd.5 (osd.5) 451 : cluster 3 failed to encode map e415 with expected crc 2023-10-01T11:16:20.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:19 smithi173 bash[58816]: cluster 2023-10-01T11:16:18.527790+0000 mgr.y (mgr.25072) 1773 : cluster 0 pgmap v1946: 73 pgs: 5 peering, 68 active+clean; 4.6 MiB data, 7.3 GiB used, 618 GiB / 626 GiB avail 2023-10-01T11:16:20.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:19 smithi173 bash[58816]: cluster 2023-10-01T11:16:18.967127+0000 osd.7 (osd.7) 478 : cluster 3 failed to encode map e415 with expected crc 2023-10-01T11:16:20.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:19 smithi173 bash[58816]: cluster 2023-10-01T11:16:18.969985+0000 mon.a (mon.0) 2289 : cluster 0 osdmap e415: 8 total, 8 up, 8 in 2023-10-01T11:16:20.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:19 smithi173 bash[58816]: cluster 2023-10-01T11:16:18.971206+0000 osd.7 (osd.7) 479 : cluster 3 failed to encode map e415 with expected crc 2023-10-01T11:16:20.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:19 smithi173 bash[58816]: cluster 2023-10-01T11:16:18.971531+0000 osd.6 (osd.6) 399 : cluster 3 failed to encode map e415 with expected crc 2023-10-01T11:16:20.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:19 smithi173 bash[58816]: cluster 2023-10-01T11:16:18.971901+0000 osd.5 (osd.5) 450 : cluster 3 failed to encode map e415 with expected crc 2023-10-01T11:16:20.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:19 smithi173 bash[58816]: cluster 2023-10-01T11:16:18.972391+0000 osd.7 (osd.7) 480 : cluster 3 failed to encode map e415 with expected crc 2023-10-01T11:16:20.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:19 smithi173 bash[58816]: cluster 2023-10-01T11:16:18.973692+0000 osd.7 (osd.7) 481 : cluster 3 failed to encode map e415 with expected crc 2023-10-01T11:16:20.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:19 smithi173 bash[58816]: cluster 2023-10-01T11:16:18.973924+0000 osd.5 (osd.5) 451 : cluster 3 failed to encode map e415 with expected crc 2023-10-01T11:16:20.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:16:20 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:16:20] "GET /metrics HTTP/1.1" 200 34023 "" "Prometheus/2.33.4" 2023-10-01T11:16:21.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:20 smithi078 bash[75299]: cluster 2023-10-01T11:16:19.970762+0000 osd.5 (osd.5) 452 : cluster 3 failed to encode map e416 with expected crc 2023-10-01T11:16:21.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:20 smithi078 bash[75299]: cluster 2023-10-01T11:16:19.974706+0000 mon.a (mon.0) 2290 : cluster 0 osdmap e416: 8 total, 8 up, 8 in 2023-10-01T11:16:21.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:20 smithi078 bash[75299]: cluster 2023-10-01T11:16:19.975011+0000 osd.7 (osd.7) 482 : cluster 3 failed to encode map e416 with expected crc 2023-10-01T11:16:21.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:20 smithi078 bash[75299]: cluster 2023-10-01T11:16:19.977733+0000 osd.7 (osd.7) 483 : cluster 3 failed to encode map e416 with expected crc 2023-10-01T11:16:21.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:20 smithi078 bash[78274]: cluster 2023-10-01T11:16:19.970762+0000 osd.5 (osd.5) 452 : cluster 3 failed to encode map e416 with expected crc 2023-10-01T11:16:21.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:20 smithi078 bash[78274]: cluster 2023-10-01T11:16:19.974706+0000 mon.a (mon.0) 2290 : cluster 0 osdmap e416: 8 total, 8 up, 8 in 2023-10-01T11:16:21.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:20 smithi078 bash[78274]: cluster 2023-10-01T11:16:19.975011+0000 osd.7 (osd.7) 482 : cluster 3 failed to encode map e416 with expected crc 2023-10-01T11:16:21.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:20 smithi078 bash[78274]: cluster 2023-10-01T11:16:19.977733+0000 osd.7 (osd.7) 483 : cluster 3 failed to encode map e416 with expected crc 2023-10-01T11:16:21.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:20 smithi173 bash[58816]: cluster 2023-10-01T11:16:19.970762+0000 osd.5 (osd.5) 452 : cluster 3 failed to encode map e416 with expected crc 2023-10-01T11:16:21.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:20 smithi173 bash[58816]: cluster 2023-10-01T11:16:19.974706+0000 mon.a (mon.0) 2290 : cluster 0 osdmap e416: 8 total, 8 up, 8 in 2023-10-01T11:16:21.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:20 smithi173 bash[58816]: cluster 2023-10-01T11:16:19.975011+0000 osd.7 (osd.7) 482 : cluster 3 failed to encode map e416 with expected crc 2023-10-01T11:16:21.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:20 smithi173 bash[58816]: cluster 2023-10-01T11:16:19.977733+0000 osd.7 (osd.7) 483 : cluster 3 failed to encode map e416 with expected crc 2023-10-01T11:16:22.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:22 smithi078 bash[78274]: cluster 2023-10-01T11:16:20.528583+0000 mgr.y (mgr.25072) 1774 : cluster 0 pgmap v1949: 73 pgs: 1 active+recovery_wait+degraded, 9 peering, 63 active+clean; 4.6 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 2/84 objects degraded (2.381%); 0 B/s, 0 objects/s recovering 2023-10-01T11:16:22.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:22 smithi078 bash[78274]: cluster 2023-10-01T11:16:20.978511+0000 mon.a (mon.0) 2291 : cluster 3 Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-10-01T11:16:22.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:22 smithi078 bash[78274]: cluster 2023-10-01T11:16:20.978571+0000 mon.a (mon.0) 2292 : cluster 3 Health check failed: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded (PG_DEGRADED) 2023-10-01T11:16:22.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:22 smithi078 bash[78274]: cluster 2023-10-01T11:16:20.994597+0000 osd.7 (osd.7) 484 : cluster 3 failed to encode map e417 with expected crc 2023-10-01T11:16:22.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:22 smithi078 bash[78274]: cluster 2023-10-01T11:16:20.997242+0000 osd.7 (osd.7) 485 : cluster 3 failed to encode map e417 with expected crc 2023-10-01T11:16:22.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:22 smithi078 bash[78274]: cluster 2023-10-01T11:16:20.999226+0000 mon.a (mon.0) 2293 : cluster 0 osdmap e417: 8 total, 8 up, 8 in 2023-10-01T11:16:22.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:22 smithi078 bash[78274]: cluster 2023-10-01T11:16:20.999866+0000 osd.7 (osd.7) 486 : cluster 3 failed to encode map e417 with expected crc 2023-10-01T11:16:22.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:22 smithi078 bash[78274]: cluster 2023-10-01T11:16:21.003723+0000 osd.7 (osd.7) 487 : cluster 3 failed to encode map e417 with expected crc 2023-10-01T11:16:22.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:22 smithi078 bash[78274]: cluster 2023-10-01T11:16:21.136475+0000 osd.5 (osd.5) 453 : cluster 3 failed to encode map e417 with expected crc 2023-10-01T11:16:22.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:22 smithi078 bash[75299]: cluster 2023-10-01T11:16:20.528583+0000 mgr.y (mgr.25072) 1774 : cluster 0 pgmap v1949: 73 pgs: 1 active+recovery_wait+degraded, 9 peering, 63 active+clean; 4.6 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 2/84 objects degraded (2.381%); 0 B/s, 0 objects/s recovering 2023-10-01T11:16:22.334 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:22 smithi078 bash[75299]: cluster 2023-10-01T11:16:20.978511+0000 mon.a (mon.0) 2291 : cluster 3 Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-10-01T11:16:22.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:22 smithi078 bash[75299]: cluster 2023-10-01T11:16:20.978571+0000 mon.a (mon.0) 2292 : cluster 3 Health check failed: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded (PG_DEGRADED) 2023-10-01T11:16:22.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:22 smithi078 bash[75299]: cluster 2023-10-01T11:16:20.994597+0000 osd.7 (osd.7) 484 : cluster 3 failed to encode map e417 with expected crc 2023-10-01T11:16:22.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:22 smithi078 bash[75299]: cluster 2023-10-01T11:16:20.997242+0000 osd.7 (osd.7) 485 : cluster 3 failed to encode map e417 with expected crc 2023-10-01T11:16:22.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:22 smithi078 bash[75299]: cluster 2023-10-01T11:16:20.999226+0000 mon.a (mon.0) 2293 : cluster 0 osdmap e417: 8 total, 8 up, 8 in 2023-10-01T11:16:22.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:22 smithi078 bash[75299]: cluster 2023-10-01T11:16:20.999866+0000 osd.7 (osd.7) 486 : cluster 3 failed to encode map e417 with expected crc 2023-10-01T11:16:22.335 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:22 smithi078 bash[75299]: cluster 2023-10-01T11:16:21.003723+0000 osd.7 (osd.7) 487 : cluster 3 failed to encode map e417 with expected crc 2023-10-01T11:16:22.336 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:22 smithi078 bash[75299]: cluster 2023-10-01T11:16:21.136475+0000 osd.5 (osd.5) 453 : cluster 3 failed to encode map e417 with expected crc 2023-10-01T11:16:22.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:22 smithi173 bash[58816]: cluster 2023-10-01T11:16:20.528583+0000 mgr.y (mgr.25072) 1774 : cluster 0 pgmap v1949: 73 pgs: 1 active+recovery_wait+degraded, 9 peering, 63 active+clean; 4.6 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 2/84 objects degraded (2.381%); 0 B/s, 0 objects/s recovering 2023-10-01T11:16:22.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:22 smithi173 bash[58816]: cluster 2023-10-01T11:16:20.978511+0000 mon.a (mon.0) 2291 : cluster 3 Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-10-01T11:16:22.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:22 smithi173 bash[58816]: cluster 2023-10-01T11:16:20.978571+0000 mon.a (mon.0) 2292 : cluster 3 Health check failed: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded (PG_DEGRADED) 2023-10-01T11:16:22.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:22 smithi173 bash[58816]: cluster 2023-10-01T11:16:20.994597+0000 osd.7 (osd.7) 484 : cluster 3 failed to encode map e417 with expected crc 2023-10-01T11:16:22.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:22 smithi173 bash[58816]: cluster 2023-10-01T11:16:20.997242+0000 osd.7 (osd.7) 485 : cluster 3 failed to encode map e417 with expected crc 2023-10-01T11:16:22.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:22 smithi173 bash[58816]: cluster 2023-10-01T11:16:20.999226+0000 mon.a (mon.0) 2293 : cluster 0 osdmap e417: 8 total, 8 up, 8 in 2023-10-01T11:16:22.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:22 smithi173 bash[58816]: cluster 2023-10-01T11:16:20.999866+0000 osd.7 (osd.7) 486 : cluster 3 failed to encode map e417 with expected crc 2023-10-01T11:16:22.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:22 smithi173 bash[58816]: cluster 2023-10-01T11:16:21.003723+0000 osd.7 (osd.7) 487 : cluster 3 failed to encode map e417 with expected crc 2023-10-01T11:16:22.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:22 smithi173 bash[58816]: cluster 2023-10-01T11:16:21.136475+0000 osd.5 (osd.5) 453 : cluster 3 failed to encode map e417 with expected crc 2023-10-01T11:16:22.627 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:16:23.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:23 smithi078 bash[75299]: cluster 2023-10-01T11:16:21.996827+0000 osd.5 (osd.5) 454 : cluster 3 failed to encode map e418 with expected crc 2023-10-01T11:16:23.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:23 smithi078 bash[75299]: cluster 2023-10-01T11:16:21.999223+0000 osd.7 (osd.7) 488 : cluster 3 failed to encode map e418 with expected crc 2023-10-01T11:16:23.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:23 smithi078 bash[75299]: cluster 2023-10-01T11:16:21.999354+0000 mon.a (mon.0) 2294 : cluster 0 osdmap e418: 8 total, 8 up, 8 in 2023-10-01T11:16:23.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:23 smithi078 bash[75299]: cluster 2023-10-01T11:16:22.002522+0000 osd.7 (osd.7) 489 : cluster 3 failed to encode map e418 with expected crc 2023-10-01T11:16:23.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:23 smithi078 bash[75299]: cluster 2023-10-01T11:16:22.004910+0000 osd.7 (osd.7) 490 : cluster 3 failed to encode map e418 with expected crc 2023-10-01T11:16:23.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:23 smithi078 bash[78274]: cluster 2023-10-01T11:16:21.996827+0000 osd.5 (osd.5) 454 : cluster 3 failed to encode map e418 with expected crc 2023-10-01T11:16:23.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:23 smithi078 bash[78274]: cluster 2023-10-01T11:16:21.999223+0000 osd.7 (osd.7) 488 : cluster 3 failed to encode map e418 with expected crc 2023-10-01T11:16:23.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:23 smithi078 bash[78274]: cluster 2023-10-01T11:16:21.999354+0000 mon.a (mon.0) 2294 : cluster 0 osdmap e418: 8 total, 8 up, 8 in 2023-10-01T11:16:23.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:23 smithi078 bash[78274]: cluster 2023-10-01T11:16:22.002522+0000 osd.7 (osd.7) 489 : cluster 3 failed to encode map e418 with expected crc 2023-10-01T11:16:23.334 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:23 smithi078 bash[78274]: cluster 2023-10-01T11:16:22.004910+0000 osd.7 (osd.7) 490 : cluster 3 failed to encode map e418 with expected crc 2023-10-01T11:16:23.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:23 smithi173 bash[58816]: cluster 2023-10-01T11:16:21.996827+0000 osd.5 (osd.5) 454 : cluster 3 failed to encode map e418 with expected crc 2023-10-01T11:16:23.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:23 smithi173 bash[58816]: cluster 2023-10-01T11:16:21.999223+0000 osd.7 (osd.7) 488 : cluster 3 failed to encode map e418 with expected crc 2023-10-01T11:16:23.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:23 smithi173 bash[58816]: cluster 2023-10-01T11:16:21.999354+0000 mon.a (mon.0) 2294 : cluster 0 osdmap e418: 8 total, 8 up, 8 in 2023-10-01T11:16:23.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:23 smithi173 bash[58816]: cluster 2023-10-01T11:16:22.002522+0000 osd.7 (osd.7) 489 : cluster 3 failed to encode map e418 with expected crc 2023-10-01T11:16:23.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:23 smithi173 bash[58816]: cluster 2023-10-01T11:16:22.004910+0000 osd.7 (osd.7) 490 : cluster 3 failed to encode map e418 with expected crc 2023-10-01T11:16:23.645 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:16:24.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:24 smithi078 bash[75299]: cluster 2023-10-01T11:16:22.529386+0000 mgr.y (mgr.25072) 1775 : cluster 0 pgmap v1952: 73 pgs: 1 active+recovery_wait+degraded, 9 peering, 63 active+clean; 4.6 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 2/84 objects degraded (2.381%); 0 B/s, 2 objects/s recovering 2023-10-01T11:16:24.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:24 smithi078 bash[75299]: cluster 2023-10-01T11:16:23.015285+0000 osd.7 (osd.7) 491 : cluster 3 failed to encode map e419 with expected crc 2023-10-01T11:16:24.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:24 smithi078 bash[75299]: cluster 2023-10-01T11:16:23.016371+0000 osd.6 (osd.6) 400 : cluster 3 failed to encode map e419 with expected crc 2023-10-01T11:16:24.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:24 smithi078 bash[75299]: cluster 2023-10-01T11:16:23.017190+0000 mon.a (mon.0) 2295 : cluster 0 osdmap e419: 8 total, 8 up, 8 in 2023-10-01T11:16:24.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:24 smithi078 bash[75299]: cluster 2023-10-01T11:16:23.017946+0000 osd.6 (osd.6) 401 : cluster 3 failed to encode map e419 with expected crc 2023-10-01T11:16:24.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:24 smithi078 bash[75299]: cluster 2023-10-01T11:16:23.019084+0000 osd.6 (osd.6) 402 : cluster 3 failed to encode map e419 with expected crc 2023-10-01T11:16:24.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:24 smithi078 bash[75299]: cluster 2023-10-01T11:16:23.019116+0000 osd.7 (osd.7) 492 : cluster 3 failed to encode map e419 with expected crc 2023-10-01T11:16:24.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:24 smithi078 bash[75299]: cluster 2023-10-01T11:16:23.022036+0000 osd.7 (osd.7) 493 : cluster 3 failed to encode map e419 with expected crc 2023-10-01T11:16:24.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:24 smithi078 bash[78274]: cluster 2023-10-01T11:16:22.529386+0000 mgr.y (mgr.25072) 1775 : cluster 0 pgmap v1952: 73 pgs: 1 active+recovery_wait+degraded, 9 peering, 63 active+clean; 4.6 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 2/84 objects degraded (2.381%); 0 B/s, 2 objects/s recovering 2023-10-01T11:16:24.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:24 smithi078 bash[78274]: cluster 2023-10-01T11:16:23.015285+0000 osd.7 (osd.7) 491 : cluster 3 failed to encode map e419 with expected crc 2023-10-01T11:16:24.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:24 smithi078 bash[78274]: cluster 2023-10-01T11:16:23.016371+0000 osd.6 (osd.6) 400 : cluster 3 failed to encode map e419 with expected crc 2023-10-01T11:16:24.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:24 smithi078 bash[78274]: cluster 2023-10-01T11:16:23.017190+0000 mon.a (mon.0) 2295 : cluster 0 osdmap e419: 8 total, 8 up, 8 in 2023-10-01T11:16:24.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:24 smithi078 bash[78274]: cluster 2023-10-01T11:16:23.017946+0000 osd.6 (osd.6) 401 : cluster 3 failed to encode map e419 with expected crc 2023-10-01T11:16:24.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:24 smithi078 bash[78274]: cluster 2023-10-01T11:16:23.019084+0000 osd.6 (osd.6) 402 : cluster 3 failed to encode map e419 with expected crc 2023-10-01T11:16:24.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:24 smithi078 bash[78274]: cluster 2023-10-01T11:16:23.019116+0000 osd.7 (osd.7) 492 : cluster 3 failed to encode map e419 with expected crc 2023-10-01T11:16:24.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:24 smithi078 bash[78274]: cluster 2023-10-01T11:16:23.022036+0000 osd.7 (osd.7) 493 : cluster 3 failed to encode map e419 with expected crc 2023-10-01T11:16:24.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:24 smithi173 bash[58816]: cluster 2023-10-01T11:16:22.529386+0000 mgr.y (mgr.25072) 1775 : cluster 0 pgmap v1952: 73 pgs: 1 active+recovery_wait+degraded, 9 peering, 63 active+clean; 4.6 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 2/84 objects degraded (2.381%); 0 B/s, 2 objects/s recovering 2023-10-01T11:16:24.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:24 smithi173 bash[58816]: cluster 2023-10-01T11:16:23.015285+0000 osd.7 (osd.7) 491 : cluster 3 failed to encode map e419 with expected crc 2023-10-01T11:16:24.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:24 smithi173 bash[58816]: cluster 2023-10-01T11:16:23.016371+0000 osd.6 (osd.6) 400 : cluster 3 failed to encode map e419 with expected crc 2023-10-01T11:16:24.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:24 smithi173 bash[58816]: cluster 2023-10-01T11:16:23.017190+0000 mon.a (mon.0) 2295 : cluster 0 osdmap e419: 8 total, 8 up, 8 in 2023-10-01T11:16:24.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:24 smithi173 bash[58816]: cluster 2023-10-01T11:16:23.017946+0000 osd.6 (osd.6) 401 : cluster 3 failed to encode map e419 with expected crc 2023-10-01T11:16:24.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:24 smithi173 bash[58816]: cluster 2023-10-01T11:16:23.019084+0000 osd.6 (osd.6) 402 : cluster 3 failed to encode map e419 with expected crc 2023-10-01T11:16:24.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:24 smithi173 bash[58816]: cluster 2023-10-01T11:16:23.019116+0000 osd.7 (osd.7) 492 : cluster 3 failed to encode map e419 with expected crc 2023-10-01T11:16:24.372 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:24 smithi173 bash[58816]: cluster 2023-10-01T11:16:23.022036+0000 osd.7 (osd.7) 493 : cluster 3 failed to encode map e419 with expected crc 2023-10-01T11:16:25.183 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:25 smithi078 bash[78274]: cluster 2023-10-01T11:16:25.011638+0000 mon.a (mon.0) 2296 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2023-10-01T11:16:25.184 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:25 smithi078 bash[78274]: cluster 2023-10-01T11:16:25.011698+0000 mon.a (mon.0) 2297 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded) 2023-10-01T11:16:25.184 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:25 smithi078 bash[78274]: cluster 2023-10-01T11:16:25.011726+0000 mon.a (mon.0) 2298 : cluster 1 Cluster is now healthy 2023-10-01T11:16:25.185 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:25 smithi078 bash[75299]: cluster 2023-10-01T11:16:25.011638+0000 mon.a (mon.0) 2296 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2023-10-01T11:16:25.185 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:25 smithi078 bash[75299]: cluster 2023-10-01T11:16:25.011698+0000 mon.a (mon.0) 2297 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded) 2023-10-01T11:16:25.185 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:25 smithi078 bash[75299]: cluster 2023-10-01T11:16:25.011726+0000 mon.a (mon.0) 2298 : cluster 1 Cluster is now healthy 2023-10-01T11:16:25.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:25 smithi173 bash[58816]: cluster 2023-10-01T11:16:25.011638+0000 mon.a (mon.0) 2296 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2023-10-01T11:16:25.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:25 smithi173 bash[58816]: cluster 2023-10-01T11:16:25.011698+0000 mon.a (mon.0) 2297 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded) 2023-10-01T11:16:25.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:25 smithi173 bash[58816]: cluster 2023-10-01T11:16:25.011726+0000 mon.a (mon.0) 2298 : cluster 1 Cluster is now healthy 2023-10-01T11:16:26.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:26 smithi173 bash[58816]: cluster 2023-10-01T11:16:24.530358+0000 mgr.y (mgr.25072) 1776 : cluster 0 pgmap v1954: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 1.0 MiB/s, 8 objects/s recovering 2023-10-01T11:16:26.655 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:26 smithi078 bash[75299]: cluster 2023-10-01T11:16:24.530358+0000 mgr.y (mgr.25072) 1776 : cluster 0 pgmap v1954: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 1.0 MiB/s, 8 objects/s recovering 2023-10-01T11:16:26.655 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:26 smithi078 bash[78274]: cluster 2023-10-01T11:16:24.530358+0000 mgr.y (mgr.25072) 1776 : cluster 0 pgmap v1954: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 1.0 MiB/s, 8 objects/s recovering 2023-10-01T11:16:28.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:28 smithi078 bash[75299]: cluster 2023-10-01T11:16:26.531140+0000 mgr.y (mgr.25072) 1777 : cluster 0 pgmap v1955: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 779 KiB/s, 6 objects/s recovering 2023-10-01T11:16:28.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:28 smithi078 bash[78274]: cluster 2023-10-01T11:16:26.531140+0000 mgr.y (mgr.25072) 1777 : cluster 0 pgmap v1955: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 779 KiB/s, 6 objects/s recovering 2023-10-01T11:16:28.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:28 smithi173 bash[58816]: cluster 2023-10-01T11:16:26.531140+0000 mgr.y (mgr.25072) 1777 : cluster 0 pgmap v1955: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 779 KiB/s, 6 objects/s recovering 2023-10-01T11:16:29.340 INFO:teuthology.orchestra.run.smithi078.stdout:ERROR: (22) Invalid argument 2023-10-01T11:16:29.340 INFO:teuthology.orchestra.run.smithi078.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T11:16:29.617 INFO:teuthology.orchestra.run.smithi078.stderr:Error ENOENT: unrecognized pool 'device_health_metrics' 2023-10-01T11:16:30.331 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:16:30 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:16:30] "GET /metrics HTTP/1.1" 200 34023 "" "Prometheus/2.33.4" 2023-10-01T11:16:30.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:30 smithi078 bash[78274]: cluster 2023-10-01T11:16:28.531802+0000 mgr.y (mgr.25072) 1778 : cluster 0 pgmap v1956: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 620 KiB/s, 5 objects/s recovering 2023-10-01T11:16:30.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:30 smithi078 bash[78274]: audit 2023-10-01T11:16:29.613348+0000 mon.c (mon.2) 257 : audit 1 from='client.? 172.21.15.78:0/4146954090' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "31"}]: dispatch 2023-10-01T11:16:30.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:30 smithi078 bash[78274]: audit 2023-10-01T11:16:29.613754+0000 mon.a (mon.0) 2299 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "31"}]: dispatch 2023-10-01T11:16:30.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:30 smithi078 bash[75299]: cluster 2023-10-01T11:16:28.531802+0000 mgr.y (mgr.25072) 1778 : cluster 0 pgmap v1956: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 620 KiB/s, 5 objects/s recovering 2023-10-01T11:16:30.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:30 smithi078 bash[75299]: audit 2023-10-01T11:16:29.613348+0000 mon.c (mon.2) 257 : audit 1 from='client.? 172.21.15.78:0/4146954090' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "31"}]: dispatch 2023-10-01T11:16:30.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:30 smithi078 bash[75299]: audit 2023-10-01T11:16:29.613754+0000 mon.a (mon.0) 2299 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "31"}]: dispatch 2023-10-01T11:16:30.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:30 smithi173 bash[58816]: cluster 2023-10-01T11:16:28.531802+0000 mgr.y (mgr.25072) 1778 : cluster 0 pgmap v1956: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 620 KiB/s, 5 objects/s recovering 2023-10-01T11:16:30.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:30 smithi173 bash[58816]: audit 2023-10-01T11:16:29.613348+0000 mon.c (mon.2) 257 : audit 1 from='client.? 172.21.15.78:0/4146954090' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "31"}]: dispatch 2023-10-01T11:16:30.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:30 smithi173 bash[58816]: audit 2023-10-01T11:16:29.613754+0000 mon.a (mon.0) 2299 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "device_health_metrics", "var": "pgp_num", "val": "31"}]: dispatch 2023-10-01T11:16:30.650 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-10-01T11:16:31.280 DEBUG:teuthology.orchestra.run:got remote process result: 2 2023-10-01T11:16:31.281 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=600 2023-10-01T11:16:32.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:32 smithi173 bash[58816]: cluster 2023-10-01T11:16:30.532411+0000 mgr.y (mgr.25072) 1779 : cluster 0 pgmap v1957: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 547 KiB/s, 4 objects/s recovering 2023-10-01T11:16:32.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:32 smithi078 bash[75299]: cluster 2023-10-01T11:16:30.532411+0000 mgr.y (mgr.25072) 1779 : cluster 0 pgmap v1957: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 547 KiB/s, 4 objects/s recovering 2023-10-01T11:16:32.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:32 smithi078 bash[78274]: cluster 2023-10-01T11:16:30.532411+0000 mgr.y (mgr.25072) 1779 : cluster 0 pgmap v1957: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 547 KiB/s, 4 objects/s recovering 2023-10-01T11:16:33.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:33 smithi173 bash[58816]: audit 2023-10-01T11:16:32.485651+0000 mon.a (mon.0) 2300 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:16:33.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:33 smithi078 bash[75299]: audit 2023-10-01T11:16:32.485651+0000 mon.a (mon.0) 2300 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:16:33.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:33 smithi078 bash[78274]: audit 2023-10-01T11:16:32.485651+0000 mon.a (mon.0) 2300 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:16:34.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:34 smithi173 bash[58816]: cluster 2023-10-01T11:16:32.533065+0000 mgr.y (mgr.25072) 1780 : cluster 0 pgmap v1958: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 467 KiB/s, 3 objects/s recovering 2023-10-01T11:16:34.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:34 smithi173 bash[58816]: audit 2023-10-01T11:16:33.470409+0000 mon.a (mon.0) 2301 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:16:34.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:34 smithi078 bash[75299]: cluster 2023-10-01T11:16:32.533065+0000 mgr.y (mgr.25072) 1780 : cluster 0 pgmap v1958: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 467 KiB/s, 3 objects/s recovering 2023-10-01T11:16:34.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:34 smithi078 bash[75299]: audit 2023-10-01T11:16:33.470409+0000 mon.a (mon.0) 2301 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:16:34.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:34 smithi078 bash[78274]: cluster 2023-10-01T11:16:32.533065+0000 mgr.y (mgr.25072) 1780 : cluster 0 pgmap v1958: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 467 KiB/s, 3 objects/s recovering 2023-10-01T11:16:34.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:34 smithi078 bash[78274]: audit 2023-10-01T11:16:33.470409+0000 mon.a (mon.0) 2301 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:16:35.409 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:16:35.409 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:16:36.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:36 smithi173 bash[58816]: cluster 2023-10-01T11:16:34.534130+0000 mgr.y (mgr.25072) 1781 : cluster 0 pgmap v1959: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 406 KiB/s, 3 objects/s recovering 2023-10-01T11:16:36.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:36 smithi078 bash[75299]: cluster 2023-10-01T11:16:34.534130+0000 mgr.y (mgr.25072) 1781 : cluster 0 pgmap v1959: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 406 KiB/s, 3 objects/s recovering 2023-10-01T11:16:36.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:36 smithi078 bash[78274]: cluster 2023-10-01T11:16:34.534130+0000 mgr.y (mgr.25072) 1781 : cluster 0 pgmap v1959: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 406 KiB/s, 3 objects/s recovering 2023-10-01T11:16:38.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:38 smithi078 bash[78274]: cluster 2023-10-01T11:16:36.534900+0000 mgr.y (mgr.25072) 1782 : cluster 0 pgmap v1960: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:16:38.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:38 smithi078 bash[75299]: cluster 2023-10-01T11:16:36.534900+0000 mgr.y (mgr.25072) 1782 : cluster 0 pgmap v1960: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:16:38.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:38 smithi173 bash[58816]: cluster 2023-10-01T11:16:36.534900+0000 mgr.y (mgr.25072) 1782 : cluster 0 pgmap v1960: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:16:40.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:40 smithi173 bash[58816]: cluster 2023-10-01T11:16:38.535573+0000 mgr.y (mgr.25072) 1783 : cluster 0 pgmap v1961: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:16:40.383 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:40 smithi078 bash[78274]: cluster 2023-10-01T11:16:38.535573+0000 mgr.y (mgr.25072) 1783 : cluster 0 pgmap v1961: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:16:40.384 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:40 smithi078 bash[75299]: cluster 2023-10-01T11:16:38.535573+0000 mgr.y (mgr.25072) 1783 : cluster 0 pgmap v1961: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:16:40.384 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:16:40 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:16:40] "GET /metrics HTTP/1.1" 200 34046 "" "Prometheus/2.33.4" 2023-10-01T11:16:40.617 INFO:teuthology.orchestra.run.smithi078.stdout:mon.a: {} 2023-10-01T11:16:40.632 INFO:teuthology.orchestra.run.smithi078.stdout:mon.b: {} 2023-10-01T11:16:40.646 INFO:teuthology.orchestra.run.smithi078.stdout:mon.c: {} 2023-10-01T11:16:40.648 INFO:teuthology.orchestra.run.smithi078.stderr:mon.a: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-10-01T11:16:40.648 INFO:teuthology.orchestra.run.smithi078.stderr:mon.b: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-10-01T11:16:40.648 INFO:teuthology.orchestra.run.smithi078.stderr:mon.c: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-10-01T11:16:40.725 INFO:teuthology.orchestra.run.smithi078.stdout:ERROR: (22) Invalid argument 2023-10-01T11:16:40.726 INFO:teuthology.orchestra.run.smithi078.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T11:16:41.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:41 smithi078 bash[75299]: audit 2023-10-01T11:16:40.602746+0000 mon.c (mon.2) 258 : audit 0 from='client.? 172.21.15.78:0/1812345281' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-01T11:16:41.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:41 smithi078 bash[78274]: audit 2023-10-01T11:16:40.602746+0000 mon.c (mon.2) 258 : audit 0 from='client.? 172.21.15.78:0/1812345281' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-01T11:16:41.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:41 smithi173 bash[58816]: audit 2023-10-01T11:16:40.602746+0000 mon.c (mon.2) 258 : audit 0 from='client.? 172.21.15.78:0/1812345281' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-10-01T11:16:42.206 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-10-01T11:16:42.225 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd reweight 0 1 2023-10-01T11:16:42.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:42 smithi173 bash[58816]: cluster 2023-10-01T11:16:40.536105+0000 mgr.y (mgr.25072) 1784 : cluster 0 pgmap v1962: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:16:42.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:42 smithi078 bash[75299]: cluster 2023-10-01T11:16:40.536105+0000 mgr.y (mgr.25072) 1784 : cluster 0 pgmap v1962: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:16:42.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:42 smithi078 bash[78274]: cluster 2023-10-01T11:16:40.536105+0000 mgr.y (mgr.25072) 1784 : cluster 0 pgmap v1962: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:16:44.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:44 smithi078 bash[75299]: cluster 2023-10-01T11:16:42.536828+0000 mgr.y (mgr.25072) 1785 : cluster 0 pgmap v1963: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:16:44.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:44 smithi078 bash[78274]: cluster 2023-10-01T11:16:42.536828+0000 mgr.y (mgr.25072) 1785 : cluster 0 pgmap v1963: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:16:44.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:44 smithi173 bash[58816]: cluster 2023-10-01T11:16:42.536828+0000 mgr.y (mgr.25072) 1785 : cluster 0 pgmap v1963: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:16:46.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:46 smithi078 bash[75299]: cluster 2023-10-01T11:16:44.537877+0000 mgr.y (mgr.25072) 1786 : cluster 0 pgmap v1964: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:16:46.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:46 smithi078 bash[78274]: cluster 2023-10-01T11:16:44.537877+0000 mgr.y (mgr.25072) 1786 : cluster 0 pgmap v1964: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:16:46.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:46 smithi173 bash[58816]: cluster 2023-10-01T11:16:44.537877+0000 mgr.y (mgr.25072) 1786 : cluster 0 pgmap v1964: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:16:46.986 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T11:16:47.044 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:16:48.407 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:48 smithi078 bash[78274]: cluster 2023-10-01T11:16:46.538474+0000 mgr.y (mgr.25072) 1787 : cluster 0 pgmap v1965: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:16:48.407 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:48 smithi078 bash[78274]: audit 2023-10-01T11:16:47.486001+0000 mon.a (mon.0) 2302 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:16:48.407 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:48 smithi078 bash[75299]: cluster 2023-10-01T11:16:46.538474+0000 mgr.y (mgr.25072) 1787 : cluster 0 pgmap v1965: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:16:48.407 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:48 smithi078 bash[75299]: audit 2023-10-01T11:16:47.486001+0000 mon.a (mon.0) 2302 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:16:48.442 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:48 smithi173 bash[58816]: cluster 2023-10-01T11:16:46.538474+0000 mgr.y (mgr.25072) 1787 : cluster 0 pgmap v1965: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:16:48.442 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:48 smithi173 bash[58816]: audit 2023-10-01T11:16:47.486001+0000 mon.a (mon.0) 2302 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:16:50.187 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-10-01T11:16:50.187 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T11:16:50.232 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:50 smithi173 bash[58816]: cluster 2023-10-01T11:16:48.539164+0000 mgr.y (mgr.25072) 1788 : cluster 0 pgmap v1966: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:16:50.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:50 smithi078 bash[75299]: cluster 2023-10-01T11:16:48.539164+0000 mgr.y (mgr.25072) 1788 : cluster 0 pgmap v1966: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:16:50.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:50 smithi078 bash[78274]: cluster 2023-10-01T11:16:48.539164+0000 mgr.y (mgr.25072) 1788 : cluster 0 pgmap v1966: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:16:50.582 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:16:50 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:16:50] "GET /metrics HTTP/1.1" 200 34021 "" "Prometheus/2.33.4" 2023-10-01T11:16:51.004 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-10-01T11:16:51.152 INFO:teuthology.orchestra.run.smithi078.stderr:reweighted osd.0 to 1 (10000) 2023-10-01T11:16:51.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:51 smithi173 bash[58816]: audit 2023-10-01T11:16:50.679675+0000 mon.c (mon.2) 259 : audit 1 from='client.? 172.21.15.78:0/2673518698' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T11:16:51.454 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:51 smithi173 bash[58816]: audit 2023-10-01T11:16:50.680270+0000 mon.a (mon.0) 2303 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T11:16:51.455 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:51 smithi078 bash[78274]: audit 2023-10-01T11:16:50.679675+0000 mon.c (mon.2) 259 : audit 1 from='client.? 172.21.15.78:0/2673518698' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T11:16:51.455 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:51 smithi078 bash[78274]: audit 2023-10-01T11:16:50.680270+0000 mon.a (mon.0) 2303 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T11:16:51.455 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:51 smithi078 bash[75299]: audit 2023-10-01T11:16:50.679675+0000 mon.c (mon.2) 259 : audit 1 from='client.? 172.21.15.78:0/2673518698' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T11:16:51.456 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:51 smithi078 bash[75299]: audit 2023-10-01T11:16:50.680270+0000 mon.a (mon.0) 2303 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T11:16:52.184 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 0 1 2023-10-01T11:16:52.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:52 smithi078 bash[75299]: cluster 2023-10-01T11:16:50.540049+0000 mgr.y (mgr.25072) 1789 : cluster 0 pgmap v1967: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:16:52.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:52 smithi078 bash[75299]: cluster 2023-10-01T11:16:51.144122+0000 osd.7 (osd.7) 494 : cluster 3 failed to encode map e420 with expected crc 2023-10-01T11:16:52.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:52 smithi078 bash[75299]: audit 2023-10-01T11:16:51.145043+0000 mon.a (mon.0) 2304 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-01T11:16:52.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:52 smithi078 bash[75299]: cluster 2023-10-01T11:16:51.145138+0000 mon.a (mon.0) 2305 : cluster 0 osdmap e420: 8 total, 8 up, 8 in 2023-10-01T11:16:52.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:52 smithi078 bash[75299]: cluster 2023-10-01T11:16:51.569806+0000 osd.6 (osd.6) 403 : cluster 3 failed to encode map e420 with expected crc 2023-10-01T11:16:52.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:52 smithi078 bash[78274]: cluster 2023-10-01T11:16:50.540049+0000 mgr.y (mgr.25072) 1789 : cluster 0 pgmap v1967: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:16:52.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:52 smithi078 bash[78274]: cluster 2023-10-01T11:16:51.144122+0000 osd.7 (osd.7) 494 : cluster 3 failed to encode map e420 with expected crc 2023-10-01T11:16:52.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:52 smithi078 bash[78274]: audit 2023-10-01T11:16:51.145043+0000 mon.a (mon.0) 2304 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-01T11:16:52.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:52 smithi078 bash[78274]: cluster 2023-10-01T11:16:51.145138+0000 mon.a (mon.0) 2305 : cluster 0 osdmap e420: 8 total, 8 up, 8 in 2023-10-01T11:16:52.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:52 smithi078 bash[78274]: cluster 2023-10-01T11:16:51.569806+0000 osd.6 (osd.6) 403 : cluster 3 failed to encode map e420 with expected crc 2023-10-01T11:16:52.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:52 smithi173 bash[58816]: cluster 2023-10-01T11:16:50.540049+0000 mgr.y (mgr.25072) 1789 : cluster 0 pgmap v1967: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:16:52.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:52 smithi173 bash[58816]: cluster 2023-10-01T11:16:51.144122+0000 osd.7 (osd.7) 494 : cluster 3 failed to encode map e420 with expected crc 2023-10-01T11:16:52.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:52 smithi173 bash[58816]: audit 2023-10-01T11:16:51.145043+0000 mon.a (mon.0) 2304 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-10-01T11:16:52.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:52 smithi173 bash[58816]: cluster 2023-10-01T11:16:51.145138+0000 mon.a (mon.0) 2305 : cluster 0 osdmap e420: 8 total, 8 up, 8 in 2023-10-01T11:16:52.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:52 smithi173 bash[58816]: cluster 2023-10-01T11:16:51.569806+0000 osd.6 (osd.6) 403 : cluster 3 failed to encode map e420 with expected crc 2023-10-01T11:16:53.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:53 smithi078 bash[75299]: cluster 2023-10-01T11:16:51.569917+0000 osd.5 (osd.5) 455 : cluster 3 failed to encode map e420 with expected crc 2023-10-01T11:16:53.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:53 smithi078 bash[78274]: cluster 2023-10-01T11:16:51.569917+0000 osd.5 (osd.5) 455 : cluster 3 failed to encode map e420 with expected crc 2023-10-01T11:16:53.622 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:53 smithi173 bash[58816]: cluster 2023-10-01T11:16:51.569917+0000 osd.5 (osd.5) 455 : cluster 3 failed to encode map e420 with expected crc 2023-10-01T11:16:54.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:54 smithi078 bash[75299]: cluster 2023-10-01T11:16:52.540884+0000 mgr.y (mgr.25072) 1790 : cluster 0 pgmap v1969: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:16:54.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:54 smithi078 bash[78274]: cluster 2023-10-01T11:16:52.540884+0000 mgr.y (mgr.25072) 1790 : cluster 0 pgmap v1969: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:16:54.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:54 smithi173 bash[58816]: cluster 2023-10-01T11:16:52.540884+0000 mgr.y (mgr.25072) 1790 : cluster 0 pgmap v1969: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:16:55.809 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T11:16:56.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:56 smithi078 bash[75299]: cluster 2023-10-01T11:16:54.541842+0000 mgr.y (mgr.25072) 1791 : cluster 0 pgmap v1970: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:16:56.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:56 smithi078 bash[78274]: cluster 2023-10-01T11:16:54.541842+0000 mgr.y (mgr.25072) 1791 : cluster 0 pgmap v1970: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:16:56.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:56 smithi173 bash[58816]: cluster 2023-10-01T11:16:54.541842+0000 mgr.y (mgr.25072) 1791 : cluster 0 pgmap v1970: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:16:56.964 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:16:58.425 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:16:58 smithi078 bash[75299]: cluster 2023-10-01T11:16:56.542738+0000 mgr.y (mgr.25072) 1792 : cluster 0 pgmap v1971: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:16:58.425 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:16:58 smithi078 bash[78274]: cluster 2023-10-01T11:16:56.542738+0000 mgr.y (mgr.25072) 1792 : cluster 0 pgmap v1971: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:16:58.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:16:58 smithi173 bash[58816]: cluster 2023-10-01T11:16:56.542738+0000 mgr.y (mgr.25072) 1792 : cluster 0 pgmap v1971: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:16:58.862 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-10-01T11:16:58.862 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T11:16:59.720 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-10-01T11:17:00.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:00 smithi078 bash[75299]: cluster 2023-10-01T11:16:58.543431+0000 mgr.y (mgr.25072) 1793 : cluster 0 pgmap v1972: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:00.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:00 smithi078 bash[78274]: cluster 2023-10-01T11:16:58.543431+0000 mgr.y (mgr.25072) 1793 : cluster 0 pgmap v1972: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:00.582 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:17:00 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:17:00] "GET /metrics HTTP/1.1" 200 34021 "" "Prometheus/2.33.4" 2023-10-01T11:17:00.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:00 smithi173 bash[58816]: cluster 2023-10-01T11:16:58.543431+0000 mgr.y (mgr.25072) 1793 : cluster 0 pgmap v1972: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:01.187 INFO:teuthology.orchestra.run.smithi078.stderr:set osd.0 primary-affinity to 1 (8655362) 2023-10-01T11:17:01.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:01 smithi078 bash[75299]: audit 2023-10-01T11:17:00.739468+0000 mon.c (mon.2) 260 : audit 1 from='client.? 172.21.15.78:0/1316017283' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T11:17:01.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:01 smithi078 bash[75299]: audit 2023-10-01T11:17:00.740222+0000 mon.a (mon.0) 2306 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T11:17:01.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:01 smithi078 bash[78274]: audit 2023-10-01T11:17:00.739468+0000 mon.c (mon.2) 260 : audit 1 from='client.? 172.21.15.78:0/1316017283' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T11:17:01.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:01 smithi078 bash[78274]: audit 2023-10-01T11:17:00.740222+0000 mon.a (mon.0) 2306 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T11:17:01.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:01 smithi173 bash[58816]: audit 2023-10-01T11:17:00.739468+0000 mon.c (mon.2) 260 : audit 1 from='client.? 172.21.15.78:0/1316017283' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T11:17:01.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:01 smithi173 bash[58816]: audit 2023-10-01T11:17:00.740222+0000 mon.a (mon.0) 2306 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-10-01T11:17:02.282 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd reweight 1 1 2023-10-01T11:17:02.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:02 smithi078 bash[75299]: cluster 2023-10-01T11:17:00.544128+0000 mgr.y (mgr.25072) 1794 : cluster 0 pgmap v1973: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:02.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:02 smithi078 bash[75299]: cluster 2023-10-01T11:17:01.172699+0000 osd.7 (osd.7) 495 : cluster 3 failed to encode map e421 with expected crc 2023-10-01T11:17:02.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:02 smithi078 bash[75299]: audit 2023-10-01T11:17:01.176040+0000 mon.a (mon.0) 2307 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-10-01T11:17:02.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:02 smithi078 bash[75299]: cluster 2023-10-01T11:17:01.176152+0000 mon.a (mon.0) 2308 : cluster 0 osdmap e421: 8 total, 8 up, 8 in 2023-10-01T11:17:02.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:02 smithi078 bash[75299]: cluster 2023-10-01T11:17:01.180304+0000 osd.5 (osd.5) 456 : cluster 3 failed to encode map e421 with expected crc 2023-10-01T11:17:02.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:02 smithi078 bash[75299]: cluster 2023-10-01T11:17:01.182261+0000 osd.6 (osd.6) 404 : cluster 3 failed to encode map e421 with expected crc 2023-10-01T11:17:02.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:02 smithi078 bash[78274]: cluster 2023-10-01T11:17:00.544128+0000 mgr.y (mgr.25072) 1794 : cluster 0 pgmap v1973: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:02.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:02 smithi078 bash[78274]: cluster 2023-10-01T11:17:01.172699+0000 osd.7 (osd.7) 495 : cluster 3 failed to encode map e421 with expected crc 2023-10-01T11:17:02.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:02 smithi078 bash[78274]: audit 2023-10-01T11:17:01.176040+0000 mon.a (mon.0) 2307 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-10-01T11:17:02.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:02 smithi078 bash[78274]: cluster 2023-10-01T11:17:01.176152+0000 mon.a (mon.0) 2308 : cluster 0 osdmap e421: 8 total, 8 up, 8 in 2023-10-01T11:17:02.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:02 smithi078 bash[78274]: cluster 2023-10-01T11:17:01.180304+0000 osd.5 (osd.5) 456 : cluster 3 failed to encode map e421 with expected crc 2023-10-01T11:17:02.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:02 smithi078 bash[78274]: cluster 2023-10-01T11:17:01.182261+0000 osd.6 (osd.6) 404 : cluster 3 failed to encode map e421 with expected crc 2023-10-01T11:17:02.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:02 smithi173 bash[58816]: cluster 2023-10-01T11:17:00.544128+0000 mgr.y (mgr.25072) 1794 : cluster 0 pgmap v1973: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:02.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:02 smithi173 bash[58816]: cluster 2023-10-01T11:17:01.172699+0000 osd.7 (osd.7) 495 : cluster 3 failed to encode map e421 with expected crc 2023-10-01T11:17:02.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:02 smithi173 bash[58816]: audit 2023-10-01T11:17:01.176040+0000 mon.a (mon.0) 2307 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-10-01T11:17:02.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:02 smithi173 bash[58816]: cluster 2023-10-01T11:17:01.176152+0000 mon.a (mon.0) 2308 : cluster 0 osdmap e421: 8 total, 8 up, 8 in 2023-10-01T11:17:02.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:02 smithi173 bash[58816]: cluster 2023-10-01T11:17:01.180304+0000 osd.5 (osd.5) 456 : cluster 3 failed to encode map e421 with expected crc 2023-10-01T11:17:02.621 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:02 smithi173 bash[58816]: cluster 2023-10-01T11:17:01.182261+0000 osd.6 (osd.6) 404 : cluster 3 failed to encode map e421 with expected crc 2023-10-01T11:17:03.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:03 smithi078 bash[78274]: cluster 2023-10-01T11:17:02.193577+0000 osd.7 (osd.7) 496 : cluster 3 failed to encode map e422 with expected crc 2023-10-01T11:17:03.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:03 smithi078 bash[78274]: cluster 2023-10-01T11:17:02.193587+0000 osd.5 (osd.5) 457 : cluster 3 failed to encode map e422 with expected crc 2023-10-01T11:17:03.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:03 smithi078 bash[78274]: cluster 2023-10-01T11:17:02.194493+0000 mon.a (mon.0) 2309 : cluster 0 osdmap e422: 8 total, 8 up, 8 in 2023-10-01T11:17:03.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:03 smithi078 bash[78274]: cluster 2023-10-01T11:17:02.199757+0000 osd.6 (osd.6) 405 : cluster 3 failed to encode map e422 with expected crc 2023-10-01T11:17:03.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:03 smithi078 bash[78274]: audit 2023-10-01T11:17:02.485677+0000 mon.a (mon.0) 2310 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:17:03.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:03 smithi078 bash[75299]: cluster 2023-10-01T11:17:02.193577+0000 osd.7 (osd.7) 496 : cluster 3 failed to encode map e422 with expected crc 2023-10-01T11:17:03.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:03 smithi078 bash[75299]: cluster 2023-10-01T11:17:02.193587+0000 osd.5 (osd.5) 457 : cluster 3 failed to encode map e422 with expected crc 2023-10-01T11:17:03.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:03 smithi078 bash[75299]: cluster 2023-10-01T11:17:02.194493+0000 mon.a (mon.0) 2309 : cluster 0 osdmap e422: 8 total, 8 up, 8 in 2023-10-01T11:17:03.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:03 smithi078 bash[75299]: cluster 2023-10-01T11:17:02.199757+0000 osd.6 (osd.6) 405 : cluster 3 failed to encode map e422 with expected crc 2023-10-01T11:17:03.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:03 smithi078 bash[75299]: audit 2023-10-01T11:17:02.485677+0000 mon.a (mon.0) 2310 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:17:03.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:03 smithi173 bash[58816]: cluster 2023-10-01T11:17:02.193577+0000 osd.7 (osd.7) 496 : cluster 3 failed to encode map e422 with expected crc 2023-10-01T11:17:03.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:03 smithi173 bash[58816]: cluster 2023-10-01T11:17:02.193587+0000 osd.5 (osd.5) 457 : cluster 3 failed to encode map e422 with expected crc 2023-10-01T11:17:03.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:03 smithi173 bash[58816]: cluster 2023-10-01T11:17:02.194493+0000 mon.a (mon.0) 2309 : cluster 0 osdmap e422: 8 total, 8 up, 8 in 2023-10-01T11:17:03.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:03 smithi173 bash[58816]: cluster 2023-10-01T11:17:02.199757+0000 osd.6 (osd.6) 405 : cluster 3 failed to encode map e422 with expected crc 2023-10-01T11:17:03.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:03 smithi173 bash[58816]: audit 2023-10-01T11:17:02.485677+0000 mon.a (mon.0) 2310 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:17:04.500 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T11:17:04.530 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:04 smithi173 bash[58816]: cluster 2023-10-01T11:17:02.544787+0000 mgr.y (mgr.25072) 1795 : cluster 0 pgmap v1976: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:04.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:04 smithi078 bash[75299]: cluster 2023-10-01T11:17:02.544787+0000 mgr.y (mgr.25072) 1795 : cluster 0 pgmap v1976: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:04.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:04 smithi078 bash[78274]: cluster 2023-10-01T11:17:02.544787+0000 mgr.y (mgr.25072) 1795 : cluster 0 pgmap v1976: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:05.516 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:05 smithi173 bash[58816]: cluster 2023-10-01T11:17:04.545942+0000 mgr.y (mgr.25072) 1796 : cluster 0 pgmap v1977: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:05.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:05 smithi078 bash[78274]: cluster 2023-10-01T11:17:04.545942+0000 mgr.y (mgr.25072) 1796 : cluster 0 pgmap v1977: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:05.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:05 smithi078 bash[75299]: cluster 2023-10-01T11:17:04.545942+0000 mgr.y (mgr.25072) 1796 : cluster 0 pgmap v1977: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:07.060 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:17:07.741 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-10-01T11:17:07.741 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T11:17:07.766 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:07 smithi173 bash[58816]: cluster 2023-10-01T11:17:06.546680+0000 mgr.y (mgr.25072) 1797 : cluster 0 pgmap v1978: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:07.766 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:07 smithi173 bash[58816]: audit 2023-10-01T11:17:06.862545+0000 mon.a (mon.0) 2311 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T11:17:08.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:07 smithi078 bash[78274]: cluster 2023-10-01T11:17:06.546680+0000 mgr.y (mgr.25072) 1797 : cluster 0 pgmap v1978: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:08.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:07 smithi078 bash[78274]: audit 2023-10-01T11:17:06.862545+0000 mon.a (mon.0) 2311 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T11:17:08.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:07 smithi078 bash[75299]: cluster 2023-10-01T11:17:06.546680+0000 mgr.y (mgr.25072) 1797 : cluster 0 pgmap v1978: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:08.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:07 smithi078 bash[75299]: audit 2023-10-01T11:17:06.862545+0000 mon.a (mon.0) 2311 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T11:17:08.683 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-10-01T11:17:10.214 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:09 smithi078 bash[75299]: cluster 2023-10-01T11:17:08.547319+0000 mgr.y (mgr.25072) 1798 : cluster 0 pgmap v1979: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:10.214 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:09 smithi078 bash[78274]: cluster 2023-10-01T11:17:08.547319+0000 mgr.y (mgr.25072) 1798 : cluster 0 pgmap v1979: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:10.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:09 smithi173 bash[58816]: cluster 2023-10-01T11:17:08.547319+0000 mgr.y (mgr.25072) 1798 : cluster 0 pgmap v1979: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:10.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:17:10 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:17:10] "GET /metrics HTTP/1.1" 200 34022 "" "Prometheus/2.33.4" 2023-10-01T11:17:10.960 INFO:teuthology.orchestra.run.smithi078.stderr:reweighted osd.1 to 1 (10000) 2023-10-01T11:17:11.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:10 smithi078 bash[78274]: audit 2023-10-01T11:17:10.663666+0000 mon.c (mon.2) 261 : audit 1 from='client.? 172.21.15.78:0/4245015111' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T11:17:11.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:10 smithi078 bash[78274]: audit 2023-10-01T11:17:10.664357+0000 mon.a (mon.0) 2312 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T11:17:11.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:10 smithi078 bash[75299]: audit 2023-10-01T11:17:10.663666+0000 mon.c (mon.2) 261 : audit 1 from='client.? 172.21.15.78:0/4245015111' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T11:17:11.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:10 smithi078 bash[75299]: audit 2023-10-01T11:17:10.664357+0000 mon.a (mon.0) 2312 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T11:17:11.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:10 smithi173 bash[58816]: audit 2023-10-01T11:17:10.663666+0000 mon.c (mon.2) 261 : audit 1 from='client.? 172.21.15.78:0/4245015111' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T11:17:11.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:10 smithi173 bash[58816]: audit 2023-10-01T11:17:10.664357+0000 mon.a (mon.0) 2312 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T11:17:11.967 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 1 1 2023-10-01T11:17:12.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:11 smithi078 bash[75299]: cluster 2023-10-01T11:17:10.548245+0000 mgr.y (mgr.25072) 1799 : cluster 0 pgmap v1980: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:12.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:11 smithi078 bash[75299]: audit 2023-10-01T11:17:10.954779+0000 mon.a (mon.0) 2313 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-01T11:17:12.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:11 smithi078 bash[75299]: cluster 2023-10-01T11:17:10.954850+0000 mon.a (mon.0) 2314 : cluster 0 osdmap e423: 8 total, 8 up, 8 in 2023-10-01T11:17:12.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:11 smithi078 bash[78274]: cluster 2023-10-01T11:17:10.548245+0000 mgr.y (mgr.25072) 1799 : cluster 0 pgmap v1980: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:12.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:11 smithi078 bash[78274]: audit 2023-10-01T11:17:10.954779+0000 mon.a (mon.0) 2313 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-01T11:17:12.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:11 smithi078 bash[78274]: cluster 2023-10-01T11:17:10.954850+0000 mon.a (mon.0) 2314 : cluster 0 osdmap e423: 8 total, 8 up, 8 in 2023-10-01T11:17:12.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:11 smithi173 bash[58816]: cluster 2023-10-01T11:17:10.548245+0000 mgr.y (mgr.25072) 1799 : cluster 0 pgmap v1980: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:12.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:11 smithi173 bash[58816]: audit 2023-10-01T11:17:10.954779+0000 mon.a (mon.0) 2313 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-10-01T11:17:12.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:11 smithi173 bash[58816]: cluster 2023-10-01T11:17:10.954850+0000 mon.a (mon.0) 2314 : cluster 0 osdmap e423: 8 total, 8 up, 8 in 2023-10-01T11:17:13.226 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T11:17:13.255 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:12 smithi173 bash[58816]: cluster 2023-10-01T11:17:11.695469+0000 osd.5 (osd.5) 458 : cluster 3 failed to encode map e423 with expected crc 2023-10-01T11:17:13.255 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:12 smithi173 bash[58816]: cluster 2023-10-01T11:17:11.877764+0000 osd.7 (osd.7) 497 : cluster 3 failed to encode map e423 with expected crc 2023-10-01T11:17:13.256 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:12 smithi173 bash[58816]: cluster 2023-10-01T11:17:12.552413+0000 osd.6 (osd.6) 406 : cluster 3 failed to encode map e423 with expected crc 2023-10-01T11:17:13.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:12 smithi078 bash[75299]: cluster 2023-10-01T11:17:11.695469+0000 osd.5 (osd.5) 458 : cluster 3 failed to encode map e423 with expected crc 2023-10-01T11:17:13.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:12 smithi078 bash[75299]: cluster 2023-10-01T11:17:11.877764+0000 osd.7 (osd.7) 497 : cluster 3 failed to encode map e423 with expected crc 2023-10-01T11:17:13.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:12 smithi078 bash[75299]: cluster 2023-10-01T11:17:12.552413+0000 osd.6 (osd.6) 406 : cluster 3 failed to encode map e423 with expected crc 2023-10-01T11:17:13.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:12 smithi078 bash[78274]: cluster 2023-10-01T11:17:11.695469+0000 osd.5 (osd.5) 458 : cluster 3 failed to encode map e423 with expected crc 2023-10-01T11:17:13.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:12 smithi078 bash[78274]: cluster 2023-10-01T11:17:11.877764+0000 osd.7 (osd.7) 497 : cluster 3 failed to encode map e423 with expected crc 2023-10-01T11:17:13.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:12 smithi078 bash[78274]: cluster 2023-10-01T11:17:12.552413+0000 osd.6 (osd.6) 406 : cluster 3 failed to encode map e423 with expected crc 2023-10-01T11:17:14.238 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:13 smithi173 bash[58816]: cluster 2023-10-01T11:17:12.548909+0000 mgr.y (mgr.25072) 1800 : cluster 0 pgmap v1982: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:14.239 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:13 smithi173 bash[58816]: audit 2023-10-01T11:17:13.556530+0000 mon.a (mon.0) 2315 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:17:14.239 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:13 smithi173 bash[58816]: audit 2023-10-01T11:17:13.563919+0000 mon.a (mon.0) 2316 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:17:14.239 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:13 smithi173 bash[58816]: audit 2023-10-01T11:17:13.717804+0000 mon.a (mon.0) 2317 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:17:14.239 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:13 smithi173 bash[58816]: audit 2023-10-01T11:17:13.725914+0000 mon.a (mon.0) 2318 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:17:14.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:13 smithi078 bash[75299]: cluster 2023-10-01T11:17:12.548909+0000 mgr.y (mgr.25072) 1800 : cluster 0 pgmap v1982: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:14.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:13 smithi078 bash[75299]: audit 2023-10-01T11:17:13.556530+0000 mon.a (mon.0) 2315 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:17:14.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:13 smithi078 bash[75299]: audit 2023-10-01T11:17:13.563919+0000 mon.a (mon.0) 2316 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:17:14.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:13 smithi078 bash[75299]: audit 2023-10-01T11:17:13.717804+0000 mon.a (mon.0) 2317 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:17:14.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:13 smithi078 bash[75299]: audit 2023-10-01T11:17:13.725914+0000 mon.a (mon.0) 2318 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:17:14.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:13 smithi078 bash[78274]: cluster 2023-10-01T11:17:12.548909+0000 mgr.y (mgr.25072) 1800 : cluster 0 pgmap v1982: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:14.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:13 smithi078 bash[78274]: audit 2023-10-01T11:17:13.556530+0000 mon.a (mon.0) 2315 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:17:14.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:13 smithi078 bash[78274]: audit 2023-10-01T11:17:13.563919+0000 mon.a (mon.0) 2316 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:17:14.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:13 smithi078 bash[78274]: audit 2023-10-01T11:17:13.717804+0000 mon.a (mon.0) 2317 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:17:14.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:13 smithi078 bash[78274]: audit 2023-10-01T11:17:13.725914+0000 mon.a (mon.0) 2318 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:17:15.323 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:14 smithi173 bash[58816]: audit 2023-10-01T11:17:14.200488+0000 mon.a (mon.0) 2319 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T11:17:15.324 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:14 smithi173 bash[58816]: audit 2023-10-01T11:17:14.202019+0000 mon.a (mon.0) 2320 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T11:17:15.324 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:14 smithi173 bash[58816]: audit 2023-10-01T11:17:14.210449+0000 mon.a (mon.0) 2321 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:17:15.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:14 smithi078 bash[75299]: audit 2023-10-01T11:17:14.200488+0000 mon.a (mon.0) 2319 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T11:17:15.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:14 smithi078 bash[75299]: audit 2023-10-01T11:17:14.202019+0000 mon.a (mon.0) 2320 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T11:17:15.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:14 smithi078 bash[75299]: audit 2023-10-01T11:17:14.210449+0000 mon.a (mon.0) 2321 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:17:15.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:14 smithi078 bash[78274]: audit 2023-10-01T11:17:14.200488+0000 mon.a (mon.0) 2319 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T11:17:15.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:14 smithi078 bash[78274]: audit 2023-10-01T11:17:14.202019+0000 mon.a (mon.0) 2320 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T11:17:15.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:14 smithi078 bash[78274]: audit 2023-10-01T11:17:14.210449+0000 mon.a (mon.0) 2321 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:17:16.291 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:15 smithi173 bash[58816]: cluster 2023-10-01T11:17:14.549890+0000 mgr.y (mgr.25072) 1801 : cluster 0 pgmap v1983: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:16.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:15 smithi078 bash[75299]: cluster 2023-10-01T11:17:14.549890+0000 mgr.y (mgr.25072) 1801 : cluster 0 pgmap v1983: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:16.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:15 smithi078 bash[78274]: cluster 2023-10-01T11:17:14.549890+0000 mgr.y (mgr.25072) 1801 : cluster 0 pgmap v1983: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:16.511 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:17:16.545 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-10-01T11:17:16.545 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T11:17:17.374 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-10-01T11:17:18.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:17 smithi078 bash[75299]: cluster 2023-10-01T11:17:16.550700+0000 mgr.y (mgr.25072) 1802 : cluster 0 pgmap v1984: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:18.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:17 smithi078 bash[75299]: audit 2023-10-01T11:17:17.486954+0000 mon.a (mon.0) 2322 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:17:18.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:17 smithi078 bash[78274]: cluster 2023-10-01T11:17:16.550700+0000 mgr.y (mgr.25072) 1802 : cluster 0 pgmap v1984: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:18.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:17 smithi078 bash[78274]: audit 2023-10-01T11:17:17.486954+0000 mon.a (mon.0) 2322 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:17:18.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:17 smithi173 bash[58816]: cluster 2023-10-01T11:17:16.550700+0000 mgr.y (mgr.25072) 1802 : cluster 0 pgmap v1984: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:18.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:17 smithi173 bash[58816]: audit 2023-10-01T11:17:17.486954+0000 mon.a (mon.0) 2322 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:17:20.271 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:19 smithi078 bash[78274]: cluster 2023-10-01T11:17:18.551308+0000 mgr.y (mgr.25072) 1803 : cluster 0 pgmap v1985: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:20.271 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:19 smithi078 bash[75299]: cluster 2023-10-01T11:17:18.551308+0000 mgr.y (mgr.25072) 1803 : cluster 0 pgmap v1985: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:20.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:19 smithi173 bash[58816]: cluster 2023-10-01T11:17:18.551308+0000 mgr.y (mgr.25072) 1803 : cluster 0 pgmap v1985: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:20.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:17:20 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:17:20] "GET /metrics HTTP/1.1" 200 34020 "" "Prometheus/2.33.4" 2023-10-01T11:17:21.002 INFO:teuthology.orchestra.run.smithi078.stderr:set osd.1 primary-affinity to 1 (8655362) 2023-10-01T11:17:21.261 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:20 smithi078 bash[75299]: audit 2023-10-01T11:17:20.124311+0000 mon.c (mon.2) 262 : audit 1 from='client.? 172.21.15.78:0/2464737482' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T11:17:21.262 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:20 smithi078 bash[75299]: audit 2023-10-01T11:17:20.124981+0000 mon.a (mon.0) 2323 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T11:17:21.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:20 smithi078 bash[78274]: audit 2023-10-01T11:17:20.124311+0000 mon.c (mon.2) 262 : audit 1 from='client.? 172.21.15.78:0/2464737482' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T11:17:21.262 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:20 smithi078 bash[78274]: audit 2023-10-01T11:17:20.124981+0000 mon.a (mon.0) 2323 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T11:17:21.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:20 smithi173 bash[58816]: audit 2023-10-01T11:17:20.124311+0000 mon.c (mon.2) 262 : audit 1 from='client.? 172.21.15.78:0/2464737482' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T11:17:21.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:20 smithi173 bash[58816]: audit 2023-10-01T11:17:20.124981+0000 mon.a (mon.0) 2323 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-10-01T11:17:22.056 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd reweight 2 1 2023-10-01T11:17:22.149 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T11:17:22.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:21 smithi078 bash[78274]: cluster 2023-10-01T11:17:20.552337+0000 mgr.y (mgr.25072) 1804 : cluster 0 pgmap v1986: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:22.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:21 smithi078 bash[78274]: cluster 2023-10-01T11:17:20.991544+0000 osd.7 (osd.7) 498 : cluster 3 failed to encode map e424 with expected crc 2023-10-01T11:17:22.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:21 smithi078 bash[78274]: cluster 2023-10-01T11:17:20.991941+0000 osd.6 (osd.6) 407 : cluster 3 failed to encode map e424 with expected crc 2023-10-01T11:17:22.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:21 smithi078 bash[78274]: audit 2023-10-01T11:17:20.991946+0000 mon.a (mon.0) 2324 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-01T11:17:22.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:21 smithi078 bash[78274]: cluster 2023-10-01T11:17:20.992083+0000 mon.a (mon.0) 2325 : cluster 0 osdmap e424: 8 total, 8 up, 8 in 2023-10-01T11:17:22.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:21 smithi078 bash[78274]: cluster 2023-10-01T11:17:20.992252+0000 osd.5 (osd.5) 459 : cluster 3 failed to encode map e424 with expected crc 2023-10-01T11:17:22.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:21 smithi078 bash[78274]: cluster 2023-10-01T11:17:20.993208+0000 osd.7 (osd.7) 499 : cluster 3 failed to encode map e424 with expected crc 2023-10-01T11:17:22.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:21 smithi078 bash[75299]: cluster 2023-10-01T11:17:20.552337+0000 mgr.y (mgr.25072) 1804 : cluster 0 pgmap v1986: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:22.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:21 smithi078 bash[75299]: cluster 2023-10-01T11:17:20.991544+0000 osd.7 (osd.7) 498 : cluster 3 failed to encode map e424 with expected crc 2023-10-01T11:17:22.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:21 smithi078 bash[75299]: cluster 2023-10-01T11:17:20.991941+0000 osd.6 (osd.6) 407 : cluster 3 failed to encode map e424 with expected crc 2023-10-01T11:17:22.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:21 smithi078 bash[75299]: audit 2023-10-01T11:17:20.991946+0000 mon.a (mon.0) 2324 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-01T11:17:22.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:21 smithi078 bash[75299]: cluster 2023-10-01T11:17:20.992083+0000 mon.a (mon.0) 2325 : cluster 0 osdmap e424: 8 total, 8 up, 8 in 2023-10-01T11:17:22.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:21 smithi078 bash[75299]: cluster 2023-10-01T11:17:20.992252+0000 osd.5 (osd.5) 459 : cluster 3 failed to encode map e424 with expected crc 2023-10-01T11:17:22.333 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:21 smithi078 bash[75299]: cluster 2023-10-01T11:17:20.993208+0000 osd.7 (osd.7) 499 : cluster 3 failed to encode map e424 with expected crc 2023-10-01T11:17:22.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:21 smithi173 bash[58816]: cluster 2023-10-01T11:17:20.552337+0000 mgr.y (mgr.25072) 1804 : cluster 0 pgmap v1986: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:22.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:21 smithi173 bash[58816]: cluster 2023-10-01T11:17:20.991544+0000 osd.7 (osd.7) 498 : cluster 3 failed to encode map e424 with expected crc 2023-10-01T11:17:22.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:21 smithi173 bash[58816]: cluster 2023-10-01T11:17:20.991941+0000 osd.6 (osd.6) 407 : cluster 3 failed to encode map e424 with expected crc 2023-10-01T11:17:22.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:21 smithi173 bash[58816]: audit 2023-10-01T11:17:20.991946+0000 mon.a (mon.0) 2324 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-10-01T11:17:22.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:21 smithi173 bash[58816]: cluster 2023-10-01T11:17:20.992083+0000 mon.a (mon.0) 2325 : cluster 0 osdmap e424: 8 total, 8 up, 8 in 2023-10-01T11:17:22.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:21 smithi173 bash[58816]: cluster 2023-10-01T11:17:20.992252+0000 osd.5 (osd.5) 459 : cluster 3 failed to encode map e424 with expected crc 2023-10-01T11:17:22.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:21 smithi173 bash[58816]: cluster 2023-10-01T11:17:20.993208+0000 osd.7 (osd.7) 499 : cluster 3 failed to encode map e424 with expected crc 2023-10-01T11:17:23.286 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:23 smithi173 bash[58816]: cluster 2023-10-01T11:17:21.992018+0000 osd.7 (osd.7) 500 : cluster 3 failed to encode map e425 with expected crc 2023-10-01T11:17:23.286 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:23 smithi173 bash[58816]: cluster 2023-10-01T11:17:21.993114+0000 mon.a (mon.0) 2326 : cluster 0 osdmap e425: 8 total, 8 up, 8 in 2023-10-01T11:17:23.286 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:23 smithi173 bash[58816]: cluster 2023-10-01T11:17:21.994619+0000 osd.7 (osd.7) 501 : cluster 3 failed to encode map e425 with expected crc 2023-10-01T11:17:23.286 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:23 smithi173 bash[58816]: cluster 2023-10-01T11:17:21.997169+0000 osd.6 (osd.6) 408 : cluster 3 failed to encode map e425 with expected crc 2023-10-01T11:17:23.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:22 smithi078 bash[75299]: cluster 2023-10-01T11:17:21.992018+0000 osd.7 (osd.7) 500 : cluster 3 failed to encode map e425 with expected crc 2023-10-01T11:17:23.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:22 smithi078 bash[75299]: cluster 2023-10-01T11:17:21.993114+0000 mon.a (mon.0) 2326 : cluster 0 osdmap e425: 8 total, 8 up, 8 in 2023-10-01T11:17:23.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:22 smithi078 bash[75299]: cluster 2023-10-01T11:17:21.994619+0000 osd.7 (osd.7) 501 : cluster 3 failed to encode map e425 with expected crc 2023-10-01T11:17:23.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:22 smithi078 bash[75299]: cluster 2023-10-01T11:17:21.997169+0000 osd.6 (osd.6) 408 : cluster 3 failed to encode map e425 with expected crc 2023-10-01T11:17:23.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:23 smithi078 bash[78274]: cluster 2023-10-01T11:17:21.992018+0000 osd.7 (osd.7) 500 : cluster 3 failed to encode map e425 with expected crc 2023-10-01T11:17:23.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:23 smithi078 bash[78274]: cluster 2023-10-01T11:17:21.993114+0000 mon.a (mon.0) 2326 : cluster 0 osdmap e425: 8 total, 8 up, 8 in 2023-10-01T11:17:23.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:23 smithi078 bash[78274]: cluster 2023-10-01T11:17:21.994619+0000 osd.7 (osd.7) 501 : cluster 3 failed to encode map e425 with expected crc 2023-10-01T11:17:23.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:23 smithi078 bash[78274]: cluster 2023-10-01T11:17:21.997169+0000 osd.6 (osd.6) 408 : cluster 3 failed to encode map e425 with expected crc 2023-10-01T11:17:24.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:24 smithi078 bash[75299]: cluster 2023-10-01T11:17:22.395226+0000 osd.5 (osd.5) 460 : cluster 3 failed to encode map e425 with expected crc 2023-10-01T11:17:24.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:24 smithi078 bash[75299]: cluster 2023-10-01T11:17:22.552945+0000 mgr.y (mgr.25072) 1805 : cluster 0 pgmap v1989: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:24.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:24 smithi078 bash[78274]: cluster 2023-10-01T11:17:22.395226+0000 osd.5 (osd.5) 460 : cluster 3 failed to encode map e425 with expected crc 2023-10-01T11:17:24.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:24 smithi078 bash[78274]: cluster 2023-10-01T11:17:22.552945+0000 mgr.y (mgr.25072) 1805 : cluster 0 pgmap v1989: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:24.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:24 smithi173 bash[58816]: cluster 2023-10-01T11:17:22.395226+0000 osd.5 (osd.5) 460 : cluster 3 failed to encode map e425 with expected crc 2023-10-01T11:17:24.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:24 smithi173 bash[58816]: cluster 2023-10-01T11:17:22.552945+0000 mgr.y (mgr.25072) 1805 : cluster 0 pgmap v1989: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:25.440 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-10-01T11:17:25.441 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T11:17:26.209 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-10-01T11:17:26.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:26 smithi078 bash[78274]: cluster 2023-10-01T11:17:24.553898+0000 mgr.y (mgr.25072) 1806 : cluster 0 pgmap v1990: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:26.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:26 smithi078 bash[75299]: cluster 2023-10-01T11:17:24.553898+0000 mgr.y (mgr.25072) 1806 : cluster 0 pgmap v1990: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:26.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:26 smithi173 bash[58816]: cluster 2023-10-01T11:17:24.553898+0000 mgr.y (mgr.25072) 1806 : cluster 0 pgmap v1990: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:26.835 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:17:28.327 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:28 smithi078 bash[78274]: cluster 2023-10-01T11:17:26.554746+0000 mgr.y (mgr.25072) 1807 : cluster 0 pgmap v1991: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:28.327 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:28 smithi078 bash[75299]: cluster 2023-10-01T11:17:26.554746+0000 mgr.y (mgr.25072) 1807 : cluster 0 pgmap v1991: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:28.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:28 smithi173 bash[58816]: cluster 2023-10-01T11:17:26.554746+0000 mgr.y (mgr.25072) 1807 : cluster 0 pgmap v1991: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:30.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:30 smithi078 bash[78274]: cluster 2023-10-01T11:17:28.555381+0000 mgr.y (mgr.25072) 1808 : cluster 0 pgmap v1992: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:30.271 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:30 smithi078 bash[75299]: cluster 2023-10-01T11:17:28.555381+0000 mgr.y (mgr.25072) 1808 : cluster 0 pgmap v1992: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:30.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:30 smithi173 bash[58816]: cluster 2023-10-01T11:17:28.555381+0000 mgr.y (mgr.25072) 1808 : cluster 0 pgmap v1992: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:30.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:17:30 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:17:30] "GET /metrics HTTP/1.1" 200 34020 "" "Prometheus/2.33.4" 2023-10-01T11:17:30.984 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T11:17:31.039 INFO:teuthology.orchestra.run.smithi078.stderr:reweighted osd.2 to 1 (10000) 2023-10-01T11:17:31.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:31 smithi078 bash[78274]: audit 2023-10-01T11:17:30.552941+0000 mon.c (mon.2) 263 : audit 1 from='client.? 172.21.15.78:0/31835131' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-01T11:17:31.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:31 smithi078 bash[78274]: audit 2023-10-01T11:17:30.553597+0000 mon.a (mon.0) 2327 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-01T11:17:31.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:31 smithi078 bash[75299]: audit 2023-10-01T11:17:30.552941+0000 mon.c (mon.2) 263 : audit 1 from='client.? 172.21.15.78:0/31835131' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-01T11:17:31.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:31 smithi078 bash[75299]: audit 2023-10-01T11:17:30.553597+0000 mon.a (mon.0) 2327 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-01T11:17:31.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:31 smithi173 bash[58816]: audit 2023-10-01T11:17:30.552941+0000 mon.c (mon.2) 263 : audit 1 from='client.? 172.21.15.78:0/31835131' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-01T11:17:31.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:31 smithi173 bash[58816]: audit 2023-10-01T11:17:30.553597+0000 mon.a (mon.0) 2327 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-10-01T11:17:31.983 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 2 1 2023-10-01T11:17:32.293 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:32 smithi173 bash[58816]: cluster 2023-10-01T11:17:30.556283+0000 mgr.y (mgr.25072) 1809 : cluster 0 pgmap v1993: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:32.293 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:32 smithi173 bash[58816]: cluster 2023-10-01T11:17:31.029582+0000 osd.5 (osd.5) 461 : cluster 3 failed to encode map e426 with expected crc 2023-10-01T11:17:32.293 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:32 smithi173 bash[58816]: cluster 2023-10-01T11:17:31.031059+0000 osd.7 (osd.7) 502 : cluster 3 failed to encode map e426 with expected crc 2023-10-01T11:17:32.293 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:32 smithi173 bash[58816]: audit 2023-10-01T11:17:31.032513+0000 mon.a (mon.0) 2328 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-01T11:17:32.294 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:32 smithi173 bash[58816]: cluster 2023-10-01T11:17:31.032609+0000 mon.a (mon.0) 2329 : cluster 0 osdmap e426: 8 total, 8 up, 8 in 2023-10-01T11:17:32.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:32 smithi078 bash[75299]: cluster 2023-10-01T11:17:30.556283+0000 mgr.y (mgr.25072) 1809 : cluster 0 pgmap v1993: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:32.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:32 smithi078 bash[75299]: cluster 2023-10-01T11:17:31.029582+0000 osd.5 (osd.5) 461 : cluster 3 failed to encode map e426 with expected crc 2023-10-01T11:17:32.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:32 smithi078 bash[75299]: cluster 2023-10-01T11:17:31.031059+0000 osd.7 (osd.7) 502 : cluster 3 failed to encode map e426 with expected crc 2023-10-01T11:17:32.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:32 smithi078 bash[75299]: audit 2023-10-01T11:17:31.032513+0000 mon.a (mon.0) 2328 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-01T11:17:32.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:32 smithi078 bash[75299]: cluster 2023-10-01T11:17:31.032609+0000 mon.a (mon.0) 2329 : cluster 0 osdmap e426: 8 total, 8 up, 8 in 2023-10-01T11:17:32.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:32 smithi078 bash[78274]: cluster 2023-10-01T11:17:30.556283+0000 mgr.y (mgr.25072) 1809 : cluster 0 pgmap v1993: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:32.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:32 smithi078 bash[78274]: cluster 2023-10-01T11:17:31.029582+0000 osd.5 (osd.5) 461 : cluster 3 failed to encode map e426 with expected crc 2023-10-01T11:17:32.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:32 smithi078 bash[78274]: cluster 2023-10-01T11:17:31.031059+0000 osd.7 (osd.7) 502 : cluster 3 failed to encode map e426 with expected crc 2023-10-01T11:17:32.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:32 smithi078 bash[78274]: audit 2023-10-01T11:17:31.032513+0000 mon.a (mon.0) 2328 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-10-01T11:17:32.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:32 smithi078 bash[78274]: cluster 2023-10-01T11:17:31.032609+0000 mon.a (mon.0) 2329 : cluster 0 osdmap e426: 8 total, 8 up, 8 in 2023-10-01T11:17:33.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:33 smithi078 bash[78274]: audit 2023-10-01T11:17:32.487267+0000 mon.a (mon.0) 2330 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:17:33.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:33 smithi078 bash[75299]: audit 2023-10-01T11:17:32.487267+0000 mon.a (mon.0) 2330 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:17:33.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:33 smithi173 bash[58816]: audit 2023-10-01T11:17:32.487267+0000 mon.a (mon.0) 2330 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:17:34.324 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-10-01T11:17:34.325 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T11:17:34.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:34 smithi078 bash[75299]: cluster 2023-10-01T11:17:32.556984+0000 mgr.y (mgr.25072) 1810 : cluster 0 pgmap v1995: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:34.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:34 smithi078 bash[78274]: cluster 2023-10-01T11:17:32.556984+0000 mgr.y (mgr.25072) 1810 : cluster 0 pgmap v1995: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:34.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:34 smithi173 bash[58816]: cluster 2023-10-01T11:17:32.556984+0000 mgr.y (mgr.25072) 1810 : cluster 0 pgmap v1995: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:35.135 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-10-01T11:17:36.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:36 smithi078 bash[75299]: cluster 2023-10-01T11:17:34.557984+0000 mgr.y (mgr.25072) 1811 : cluster 0 pgmap v1996: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:36.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:36 smithi078 bash[78274]: cluster 2023-10-01T11:17:34.557984+0000 mgr.y (mgr.25072) 1811 : cluster 0 pgmap v1996: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:36.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:36 smithi173 bash[58816]: cluster 2023-10-01T11:17:34.557984+0000 mgr.y (mgr.25072) 1811 : cluster 0 pgmap v1996: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:36.762 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:17:38.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:38 smithi078 bash[75299]: cluster 2023-10-01T11:17:36.558844+0000 mgr.y (mgr.25072) 1812 : cluster 0 pgmap v1997: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:38.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:38 smithi078 bash[78274]: cluster 2023-10-01T11:17:36.558844+0000 mgr.y (mgr.25072) 1812 : cluster 0 pgmap v1997: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:38.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:38 smithi173 bash[58816]: cluster 2023-10-01T11:17:36.558844+0000 mgr.y (mgr.25072) 1812 : cluster 0 pgmap v1997: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:39.908 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T11:17:40.269 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:40 smithi078 bash[75299]: cluster 2023-10-01T11:17:38.559492+0000 mgr.y (mgr.25072) 1813 : cluster 0 pgmap v1998: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:40.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:40 smithi078 bash[78274]: cluster 2023-10-01T11:17:38.559492+0000 mgr.y (mgr.25072) 1813 : cluster 0 pgmap v1998: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:40.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:40 smithi173 bash[58816]: cluster 2023-10-01T11:17:38.559492+0000 mgr.y (mgr.25072) 1813 : cluster 0 pgmap v1998: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:40.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:17:40 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:17:40] "GET /metrics HTTP/1.1" 200 34021 "" "Prometheus/2.33.4" 2023-10-01T11:17:41.082 INFO:teuthology.orchestra.run.smithi078.stderr:set osd.2 primary-affinity to 1 (8655362) 2023-10-01T11:17:41.325 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:41 smithi078 bash[75299]: audit 2023-10-01T11:17:40.313417+0000 mon.c (mon.2) 264 : audit 1 from='client.? 172.21.15.78:0/4118259703' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-01T11:17:41.325 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:41 smithi078 bash[75299]: audit 2023-10-01T11:17:40.314158+0000 mon.a (mon.0) 2331 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-01T11:17:41.325 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:41 smithi078 bash[78274]: audit 2023-10-01T11:17:40.313417+0000 mon.c (mon.2) 264 : audit 1 from='client.? 172.21.15.78:0/4118259703' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-01T11:17:41.325 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:41 smithi078 bash[78274]: audit 2023-10-01T11:17:40.314158+0000 mon.a (mon.0) 2331 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-01T11:17:41.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:41 smithi173 bash[58816]: audit 2023-10-01T11:17:40.313417+0000 mon.c (mon.2) 264 : audit 1 from='client.? 172.21.15.78:0/4118259703' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-01T11:17:41.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:41 smithi173 bash[58816]: audit 2023-10-01T11:17:40.314158+0000 mon.a (mon.0) 2331 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-10-01T11:17:42.047 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd reweight 3 1 2023-10-01T11:17:42.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:42 smithi078 bash[75299]: cluster 2023-10-01T11:17:40.560487+0000 mgr.y (mgr.25072) 1814 : cluster 0 pgmap v1999: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:42.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:42 smithi078 bash[75299]: cluster 2023-10-01T11:17:41.069508+0000 osd.5 (osd.5) 462 : cluster 3 failed to encode map e427 with expected crc 2023-10-01T11:17:42.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:42 smithi078 bash[75299]: audit 2023-10-01T11:17:41.070257+0000 mon.a (mon.0) 2332 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-10-01T11:17:42.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:42 smithi078 bash[75299]: cluster 2023-10-01T11:17:41.070347+0000 mon.a (mon.0) 2333 : cluster 0 osdmap e427: 8 total, 8 up, 8 in 2023-10-01T11:17:42.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:42 smithi078 bash[75299]: cluster 2023-10-01T11:17:41.072457+0000 osd.7 (osd.7) 503 : cluster 3 failed to encode map e427 with expected crc 2023-10-01T11:17:42.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:42 smithi078 bash[78274]: cluster 2023-10-01T11:17:40.560487+0000 mgr.y (mgr.25072) 1814 : cluster 0 pgmap v1999: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:42.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:42 smithi078 bash[78274]: cluster 2023-10-01T11:17:41.069508+0000 osd.5 (osd.5) 462 : cluster 3 failed to encode map e427 with expected crc 2023-10-01T11:17:42.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:42 smithi078 bash[78274]: audit 2023-10-01T11:17:41.070257+0000 mon.a (mon.0) 2332 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-10-01T11:17:42.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:42 smithi078 bash[78274]: cluster 2023-10-01T11:17:41.070347+0000 mon.a (mon.0) 2333 : cluster 0 osdmap e427: 8 total, 8 up, 8 in 2023-10-01T11:17:42.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:42 smithi078 bash[78274]: cluster 2023-10-01T11:17:41.072457+0000 osd.7 (osd.7) 503 : cluster 3 failed to encode map e427 with expected crc 2023-10-01T11:17:42.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:42 smithi173 bash[58816]: cluster 2023-10-01T11:17:40.560487+0000 mgr.y (mgr.25072) 1814 : cluster 0 pgmap v1999: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:42.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:42 smithi173 bash[58816]: cluster 2023-10-01T11:17:41.069508+0000 osd.5 (osd.5) 462 : cluster 3 failed to encode map e427 with expected crc 2023-10-01T11:17:42.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:42 smithi173 bash[58816]: audit 2023-10-01T11:17:41.070257+0000 mon.a (mon.0) 2332 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-10-01T11:17:42.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:42 smithi173 bash[58816]: cluster 2023-10-01T11:17:41.070347+0000 mon.a (mon.0) 2333 : cluster 0 osdmap e427: 8 total, 8 up, 8 in 2023-10-01T11:17:42.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:42 smithi173 bash[58816]: cluster 2023-10-01T11:17:41.072457+0000 osd.7 (osd.7) 503 : cluster 3 failed to encode map e427 with expected crc 2023-10-01T11:17:43.302 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-10-01T11:17:43.303 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T11:17:44.124 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-10-01T11:17:44.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:44 smithi078 bash[78274]: cluster 2023-10-01T11:17:42.561245+0000 mgr.y (mgr.25072) 1815 : cluster 0 pgmap v2001: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:44.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:44 smithi078 bash[75299]: cluster 2023-10-01T11:17:42.561245+0000 mgr.y (mgr.25072) 1815 : cluster 0 pgmap v2001: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:44.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:44 smithi173 bash[58816]: cluster 2023-10-01T11:17:42.561245+0000 mgr.y (mgr.25072) 1815 : cluster 0 pgmap v2001: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:46.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:46 smithi078 bash[75299]: cluster 2023-10-01T11:17:44.562163+0000 mgr.y (mgr.25072) 1816 : cluster 0 pgmap v2002: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:46.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:46 smithi078 bash[78274]: cluster 2023-10-01T11:17:44.562163+0000 mgr.y (mgr.25072) 1816 : cluster 0 pgmap v2002: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:46.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:46 smithi173 bash[58816]: cluster 2023-10-01T11:17:44.562163+0000 mgr.y (mgr.25072) 1816 : cluster 0 pgmap v2002: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:46.823 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:17:48.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:48 smithi078 bash[75299]: cluster 2023-10-01T11:17:46.562982+0000 mgr.y (mgr.25072) 1817 : cluster 0 pgmap v2003: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:48.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:48 smithi078 bash[75299]: audit 2023-10-01T11:17:47.487717+0000 mon.a (mon.0) 2334 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:17:48.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:48 smithi078 bash[78274]: cluster 2023-10-01T11:17:46.562982+0000 mgr.y (mgr.25072) 1817 : cluster 0 pgmap v2003: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:48.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:48 smithi078 bash[78274]: audit 2023-10-01T11:17:47.487717+0000 mon.a (mon.0) 2334 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:17:48.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:48 smithi173 bash[58816]: cluster 2023-10-01T11:17:46.562982+0000 mgr.y (mgr.25072) 1817 : cluster 0 pgmap v2003: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:48.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:48 smithi173 bash[58816]: audit 2023-10-01T11:17:47.487717+0000 mon.a (mon.0) 2334 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:17:48.901 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T11:17:50.167 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:50 smithi173 bash[58816]: cluster 2023-10-01T11:17:48.563661+0000 mgr.y (mgr.25072) 1818 : cluster 0 pgmap v2004: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:50.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:50 smithi078 bash[78274]: cluster 2023-10-01T11:17:48.563661+0000 mgr.y (mgr.25072) 1818 : cluster 0 pgmap v2004: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:50.270 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:50 smithi078 bash[75299]: cluster 2023-10-01T11:17:48.563661+0000 mgr.y (mgr.25072) 1818 : cluster 0 pgmap v2004: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:50.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:17:50 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:17:50] "GET /metrics HTTP/1.1" 200 34023 "" "Prometheus/2.33.4" 2023-10-01T11:17:51.117 INFO:teuthology.orchestra.run.smithi078.stderr:reweighted osd.3 to 1 (10000) 2023-10-01T11:17:51.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:51 smithi173 bash[58816]: audit 2023-10-01T11:17:50.365014+0000 mon.c (mon.2) 265 : audit 1 from='client.? 172.21.15.78:0/922727250' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-01T11:17:51.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:51 smithi173 bash[58816]: audit 2023-10-01T11:17:50.365617+0000 mon.a (mon.0) 2335 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-01T11:17:51.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:51 smithi078 bash[75299]: audit 2023-10-01T11:17:50.365014+0000 mon.c (mon.2) 265 : audit 1 from='client.? 172.21.15.78:0/922727250' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-01T11:17:51.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:51 smithi078 bash[75299]: audit 2023-10-01T11:17:50.365617+0000 mon.a (mon.0) 2335 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-01T11:17:51.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:51 smithi078 bash[78274]: audit 2023-10-01T11:17:50.365014+0000 mon.c (mon.2) 265 : audit 1 from='client.? 172.21.15.78:0/922727250' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-01T11:17:51.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:51 smithi078 bash[78274]: audit 2023-10-01T11:17:50.365617+0000 mon.a (mon.0) 2335 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-10-01T11:17:51.992 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-10-01T11:17:51.993 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T11:17:52.084 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 3 1 2023-10-01T11:17:52.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:52 smithi173 bash[58816]: cluster 2023-10-01T11:17:50.564713+0000 mgr.y (mgr.25072) 1819 : cluster 0 pgmap v2005: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:52.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:52 smithi173 bash[58816]: audit 2023-10-01T11:17:51.109132+0000 mon.a (mon.0) 2336 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-01T11:17:52.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:52 smithi173 bash[58816]: cluster 2023-10-01T11:17:51.109232+0000 mon.a (mon.0) 2337 : cluster 0 osdmap e428: 8 total, 8 up, 8 in 2023-10-01T11:17:52.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:52 smithi173 bash[58816]: cluster 2023-10-01T11:17:51.681305+0000 osd.6 (osd.6) 409 : cluster 3 failed to encode map e428 with expected crc 2023-10-01T11:17:52.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:52 smithi078 bash[75299]: cluster 2023-10-01T11:17:50.564713+0000 mgr.y (mgr.25072) 1819 : cluster 0 pgmap v2005: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:52.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:52 smithi078 bash[75299]: audit 2023-10-01T11:17:51.109132+0000 mon.a (mon.0) 2336 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-01T11:17:52.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:52 smithi078 bash[75299]: cluster 2023-10-01T11:17:51.109232+0000 mon.a (mon.0) 2337 : cluster 0 osdmap e428: 8 total, 8 up, 8 in 2023-10-01T11:17:52.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:52 smithi078 bash[75299]: cluster 2023-10-01T11:17:51.681305+0000 osd.6 (osd.6) 409 : cluster 3 failed to encode map e428 with expected crc 2023-10-01T11:17:52.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:52 smithi078 bash[78274]: cluster 2023-10-01T11:17:50.564713+0000 mgr.y (mgr.25072) 1819 : cluster 0 pgmap v2005: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:52.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:52 smithi078 bash[78274]: audit 2023-10-01T11:17:51.109132+0000 mon.a (mon.0) 2336 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-10-01T11:17:52.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:52 smithi078 bash[78274]: cluster 2023-10-01T11:17:51.109232+0000 mon.a (mon.0) 2337 : cluster 0 osdmap e428: 8 total, 8 up, 8 in 2023-10-01T11:17:52.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:52 smithi078 bash[78274]: cluster 2023-10-01T11:17:51.681305+0000 osd.6 (osd.6) 409 : cluster 3 failed to encode map e428 with expected crc 2023-10-01T11:17:52.841 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-10-01T11:17:53.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:53 smithi078 bash[78274]: cluster 2023-10-01T11:17:51.681186+0000 osd.7 (osd.7) 504 : cluster 3 failed to encode map e428 with expected crc 2023-10-01T11:17:53.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:53 smithi078 bash[78274]: cluster 2023-10-01T11:17:51.681284+0000 osd.5 (osd.5) 463 : cluster 3 failed to encode map e428 with expected crc 2023-10-01T11:17:53.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:53 smithi078 bash[75299]: cluster 2023-10-01T11:17:51.681186+0000 osd.7 (osd.7) 504 : cluster 3 failed to encode map e428 with expected crc 2023-10-01T11:17:53.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:53 smithi078 bash[75299]: cluster 2023-10-01T11:17:51.681284+0000 osd.5 (osd.5) 463 : cluster 3 failed to encode map e428 with expected crc 2023-10-01T11:17:53.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:53 smithi173 bash[58816]: cluster 2023-10-01T11:17:51.681186+0000 osd.7 (osd.7) 504 : cluster 3 failed to encode map e428 with expected crc 2023-10-01T11:17:53.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:53 smithi173 bash[58816]: cluster 2023-10-01T11:17:51.681284+0000 osd.5 (osd.5) 463 : cluster 3 failed to encode map e428 with expected crc 2023-10-01T11:17:54.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:54 smithi078 bash[75299]: cluster 2023-10-01T11:17:52.565494+0000 mgr.y (mgr.25072) 1820 : cluster 0 pgmap v2007: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:54.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:54 smithi078 bash[78274]: cluster 2023-10-01T11:17:52.565494+0000 mgr.y (mgr.25072) 1820 : cluster 0 pgmap v2007: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:54.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:54 smithi173 bash[58816]: cluster 2023-10-01T11:17:52.565494+0000 mgr.y (mgr.25072) 1820 : cluster 0 pgmap v2007: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:56.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:56 smithi078 bash[75299]: cluster 2023-10-01T11:17:54.566456+0000 mgr.y (mgr.25072) 1821 : cluster 0 pgmap v2008: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:56.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:56 smithi078 bash[78274]: cluster 2023-10-01T11:17:54.566456+0000 mgr.y (mgr.25072) 1821 : cluster 0 pgmap v2008: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:56.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:56 smithi173 bash[58816]: cluster 2023-10-01T11:17:54.566456+0000 mgr.y (mgr.25072) 1821 : cluster 0 pgmap v2008: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:56.875 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:17:57.623 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T11:17:58.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:17:58 smithi078 bash[75299]: cluster 2023-10-01T11:17:56.567204+0000 mgr.y (mgr.25072) 1822 : cluster 0 pgmap v2009: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:58.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:17:58 smithi078 bash[78274]: cluster 2023-10-01T11:17:56.567204+0000 mgr.y (mgr.25072) 1822 : cluster 0 pgmap v2009: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:17:58.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:17:58 smithi173 bash[58816]: cluster 2023-10-01T11:17:56.567204+0000 mgr.y (mgr.25072) 1822 : cluster 0 pgmap v2009: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:00.331 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:18:00 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:18:00] "GET /metrics HTTP/1.1" 200 34023 "" "Prometheus/2.33.4" 2023-10-01T11:18:00.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:00 smithi078 bash[78274]: cluster 2023-10-01T11:17:58.567844+0000 mgr.y (mgr.25072) 1823 : cluster 0 pgmap v2010: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:00.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:00 smithi078 bash[75299]: cluster 2023-10-01T11:17:58.567844+0000 mgr.y (mgr.25072) 1823 : cluster 0 pgmap v2010: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:00.418 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:00 smithi173 bash[58816]: cluster 2023-10-01T11:17:58.567844+0000 mgr.y (mgr.25072) 1823 : cluster 0 pgmap v2010: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:00.579 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-10-01T11:18:00.580 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T11:18:01.157 INFO:teuthology.orchestra.run.smithi078.stderr:set osd.3 primary-affinity to 1 (8655362) 2023-10-01T11:18:01.437 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:01 smithi173 bash[58816]: audit 2023-10-01T11:18:00.485407+0000 mon.a (mon.0) 2338 : audit 1 from='client.? 172.21.15.78:0/1657448482' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-01T11:18:01.439 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-10-01T11:18:01.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:01 smithi078 bash[75299]: audit 2023-10-01T11:18:00.485407+0000 mon.a (mon.0) 2338 : audit 1 from='client.? 172.21.15.78:0/1657448482' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-01T11:18:01.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:01 smithi078 bash[78274]: audit 2023-10-01T11:18:00.485407+0000 mon.a (mon.0) 2338 : audit 1 from='client.? 172.21.15.78:0/1657448482' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-10-01T11:18:02.104 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd reweight 4 1 2023-10-01T11:18:02.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:02 smithi078 bash[75299]: cluster 2023-10-01T11:18:00.568790+0000 mgr.y (mgr.25072) 1824 : cluster 0 pgmap v2011: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:02.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:02 smithi078 bash[75299]: audit 2023-10-01T11:18:01.150507+0000 mon.a (mon.0) 2339 : audit 1 from='client.? 172.21.15.78:0/1657448482' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-10-01T11:18:02.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:02 smithi078 bash[75299]: cluster 2023-10-01T11:18:01.150572+0000 mon.a (mon.0) 2340 : cluster 0 osdmap e429: 8 total, 8 up, 8 in 2023-10-01T11:18:02.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:02 smithi078 bash[75299]: cluster 2023-10-01T11:18:01.457926+0000 osd.6 (osd.6) 410 : cluster 3 failed to encode map e429 with expected crc 2023-10-01T11:18:02.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:02 smithi078 bash[75299]: cluster 2023-10-01T11:18:01.458388+0000 osd.7 (osd.7) 505 : cluster 3 failed to encode map e429 with expected crc 2023-10-01T11:18:02.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:02 smithi078 bash[78274]: cluster 2023-10-01T11:18:00.568790+0000 mgr.y (mgr.25072) 1824 : cluster 0 pgmap v2011: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:02.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:02 smithi078 bash[78274]: audit 2023-10-01T11:18:01.150507+0000 mon.a (mon.0) 2339 : audit 1 from='client.? 172.21.15.78:0/1657448482' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-10-01T11:18:02.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:02 smithi078 bash[78274]: cluster 2023-10-01T11:18:01.150572+0000 mon.a (mon.0) 2340 : cluster 0 osdmap e429: 8 total, 8 up, 8 in 2023-10-01T11:18:02.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:02 smithi078 bash[78274]: cluster 2023-10-01T11:18:01.457926+0000 osd.6 (osd.6) 410 : cluster 3 failed to encode map e429 with expected crc 2023-10-01T11:18:02.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:02 smithi078 bash[78274]: cluster 2023-10-01T11:18:01.458388+0000 osd.7 (osd.7) 505 : cluster 3 failed to encode map e429 with expected crc 2023-10-01T11:18:02.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:02 smithi173 bash[58816]: cluster 2023-10-01T11:18:00.568790+0000 mgr.y (mgr.25072) 1824 : cluster 0 pgmap v2011: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:02.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:02 smithi173 bash[58816]: audit 2023-10-01T11:18:01.150507+0000 mon.a (mon.0) 2339 : audit 1 from='client.? 172.21.15.78:0/1657448482' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-10-01T11:18:02.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:02 smithi173 bash[58816]: cluster 2023-10-01T11:18:01.150572+0000 mon.a (mon.0) 2340 : cluster 0 osdmap e429: 8 total, 8 up, 8 in 2023-10-01T11:18:02.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:02 smithi173 bash[58816]: cluster 2023-10-01T11:18:01.457926+0000 osd.6 (osd.6) 410 : cluster 3 failed to encode map e429 with expected crc 2023-10-01T11:18:02.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:02 smithi173 bash[58816]: cluster 2023-10-01T11:18:01.458388+0000 osd.7 (osd.7) 505 : cluster 3 failed to encode map e429 with expected crc 2023-10-01T11:18:03.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:03 smithi078 bash[75299]: cluster 2023-10-01T11:18:01.458237+0000 osd.5 (osd.5) 464 : cluster 3 failed to encode map e429 with expected crc 2023-10-01T11:18:03.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:03 smithi078 bash[75299]: audit 2023-10-01T11:18:02.488291+0000 mon.a (mon.0) 2341 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:18:03.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:03 smithi078 bash[78274]: cluster 2023-10-01T11:18:01.458237+0000 osd.5 (osd.5) 464 : cluster 3 failed to encode map e429 with expected crc 2023-10-01T11:18:03.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:03 smithi078 bash[78274]: audit 2023-10-01T11:18:02.488291+0000 mon.a (mon.0) 2341 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:18:03.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:03 smithi173 bash[58816]: cluster 2023-10-01T11:18:01.458237+0000 osd.5 (osd.5) 464 : cluster 3 failed to encode map e429 with expected crc 2023-10-01T11:18:03.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:03 smithi173 bash[58816]: audit 2023-10-01T11:18:02.488291+0000 mon.a (mon.0) 2341 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:18:04.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:04 smithi078 bash[75299]: cluster 2023-10-01T11:18:02.569533+0000 mgr.y (mgr.25072) 1825 : cluster 0 pgmap v2013: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:04.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:04 smithi078 bash[78274]: cluster 2023-10-01T11:18:02.569533+0000 mgr.y (mgr.25072) 1825 : cluster 0 pgmap v2013: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:04.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:04 smithi173 bash[58816]: cluster 2023-10-01T11:18:02.569533+0000 mgr.y (mgr.25072) 1825 : cluster 0 pgmap v2013: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:06.219 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T11:18:06.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:06 smithi078 bash[75299]: cluster 2023-10-01T11:18:04.570509+0000 mgr.y (mgr.25072) 1826 : cluster 0 pgmap v2014: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:06.603 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:06 smithi078 bash[78274]: cluster 2023-10-01T11:18:04.570509+0000 mgr.y (mgr.25072) 1826 : cluster 0 pgmap v2014: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:06.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:06 smithi173 bash[58816]: cluster 2023-10-01T11:18:04.570509+0000 mgr.y (mgr.25072) 1826 : cluster 0 pgmap v2014: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:06.887 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:18:08.413 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:08 smithi078 bash[78274]: cluster 2023-10-01T11:18:06.571290+0000 mgr.y (mgr.25072) 1827 : cluster 0 pgmap v2015: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:08.413 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:08 smithi078 bash[75299]: cluster 2023-10-01T11:18:06.571290+0000 mgr.y (mgr.25072) 1827 : cluster 0 pgmap v2015: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:08.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:08 smithi173 bash[58816]: cluster 2023-10-01T11:18:06.571290+0000 mgr.y (mgr.25072) 1827 : cluster 0 pgmap v2015: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:09.307 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-10-01T11:18:09.308 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T11:18:10.089 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-10-01T11:18:10.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:10 smithi078 bash[75299]: cluster 2023-10-01T11:18:08.572024+0000 mgr.y (mgr.25072) 1828 : cluster 0 pgmap v2016: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:10.341 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:18:10 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:18:10] "GET /metrics HTTP/1.1" 200 34024 "" "Prometheus/2.33.4" 2023-10-01T11:18:10.342 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:10 smithi078 bash[78274]: cluster 2023-10-01T11:18:08.572024+0000 mgr.y (mgr.25072) 1828 : cluster 0 pgmap v2016: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:10.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:10 smithi173 bash[58816]: cluster 2023-10-01T11:18:08.572024+0000 mgr.y (mgr.25072) 1828 : cluster 0 pgmap v2016: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:11.184 INFO:teuthology.orchestra.run.smithi078.stderr:reweighted osd.4 to 1 (10000) 2023-10-01T11:18:11.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:11 smithi078 bash[75299]: audit 2023-10-01T11:18:10.492918+0000 mon.a (mon.0) 2342 : audit 1 from='client.? 172.21.15.78:0/3433430743' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-01T11:18:11.591 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:11 smithi078 bash[78274]: audit 2023-10-01T11:18:10.492918+0000 mon.a (mon.0) 2342 : audit 1 from='client.? 172.21.15.78:0/3433430743' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-01T11:18:11.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:11 smithi173 bash[58816]: audit 2023-10-01T11:18:10.492918+0000 mon.a (mon.0) 2342 : audit 1 from='client.? 172.21.15.78:0/3433430743' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-10-01T11:18:12.089 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 4 1 2023-10-01T11:18:12.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:12 smithi078 bash[78274]: cluster 2023-10-01T11:18:10.572971+0000 mgr.y (mgr.25072) 1829 : cluster 0 pgmap v2017: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:12.617 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:12 smithi078 bash[78274]: cluster 2023-10-01T11:18:11.177618+0000 osd.6 (osd.6) 411 : cluster 3 failed to encode map e430 with expected crc 2023-10-01T11:18:12.617 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:12 smithi078 bash[78274]: audit 2023-10-01T11:18:11.179337+0000 mon.a (mon.0) 2343 : audit 1 from='client.? 172.21.15.78:0/3433430743' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-01T11:18:12.617 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:12 smithi078 bash[78274]: cluster 2023-10-01T11:18:11.179408+0000 mon.a (mon.0) 2344 : cluster 0 osdmap e430: 8 total, 8 up, 8 in 2023-10-01T11:18:12.617 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:12 smithi078 bash[75299]: cluster 2023-10-01T11:18:10.572971+0000 mgr.y (mgr.25072) 1829 : cluster 0 pgmap v2017: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:12.617 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:12 smithi078 bash[75299]: cluster 2023-10-01T11:18:11.177618+0000 osd.6 (osd.6) 411 : cluster 3 failed to encode map e430 with expected crc 2023-10-01T11:18:12.618 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:12 smithi078 bash[75299]: audit 2023-10-01T11:18:11.179337+0000 mon.a (mon.0) 2343 : audit 1 from='client.? 172.21.15.78:0/3433430743' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-01T11:18:12.618 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:12 smithi078 bash[75299]: cluster 2023-10-01T11:18:11.179408+0000 mon.a (mon.0) 2344 : cluster 0 osdmap e430: 8 total, 8 up, 8 in 2023-10-01T11:18:12.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:12 smithi173 bash[58816]: cluster 2023-10-01T11:18:10.572971+0000 mgr.y (mgr.25072) 1829 : cluster 0 pgmap v2017: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:12.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:12 smithi173 bash[58816]: cluster 2023-10-01T11:18:11.177618+0000 osd.6 (osd.6) 411 : cluster 3 failed to encode map e430 with expected crc 2023-10-01T11:18:12.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:12 smithi173 bash[58816]: audit 2023-10-01T11:18:11.179337+0000 mon.a (mon.0) 2343 : audit 1 from='client.? 172.21.15.78:0/3433430743' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-10-01T11:18:12.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:12 smithi173 bash[58816]: cluster 2023-10-01T11:18:11.179408+0000 mon.a (mon.0) 2344 : cluster 0 osdmap e430: 8 total, 8 up, 8 in 2023-10-01T11:18:13.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:13 smithi078 bash[75299]: cluster 2023-10-01T11:18:11.911370+0000 osd.5 (osd.5) 465 : cluster 3 failed to encode map e430 with expected crc 2023-10-01T11:18:14.089 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:13 smithi078 bash[75299]: cluster 2023-10-01T11:18:11.911518+0000 osd.7 (osd.7) 506 : cluster 3 failed to encode map e430 with expected crc 2023-10-01T11:18:14.089 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:13 smithi078 bash[78274]: cluster 2023-10-01T11:18:11.911370+0000 osd.5 (osd.5) 465 : cluster 3 failed to encode map e430 with expected crc 2023-10-01T11:18:14.090 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:13 smithi078 bash[78274]: cluster 2023-10-01T11:18:11.911518+0000 osd.7 (osd.7) 506 : cluster 3 failed to encode map e430 with expected crc 2023-10-01T11:18:14.095 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:13 smithi173 bash[58816]: cluster 2023-10-01T11:18:11.911370+0000 osd.5 (osd.5) 465 : cluster 3 failed to encode map e430 with expected crc 2023-10-01T11:18:14.097 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:13 smithi173 bash[58816]: cluster 2023-10-01T11:18:11.911518+0000 osd.7 (osd.7) 506 : cluster 3 failed to encode map e430 with expected crc 2023-10-01T11:18:14.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:14 smithi078 bash[75299]: cluster 2023-10-01T11:18:12.573770+0000 mgr.y (mgr.25072) 1830 : cluster 0 pgmap v2019: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:15.214 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:14 smithi078 bash[78274]: cluster 2023-10-01T11:18:12.573770+0000 mgr.y (mgr.25072) 1830 : cluster 0 pgmap v2019: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:15.215 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:14 smithi173 bash[58816]: cluster 2023-10-01T11:18:12.573770+0000 mgr.y (mgr.25072) 1830 : cluster 0 pgmap v2019: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:15.215 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T11:18:15.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:15 smithi078 bash[75299]: audit 2023-10-01T11:18:14.292876+0000 mon.a (mon.0) 2345 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T11:18:15.826 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:15 smithi078 bash[75299]: audit 2023-10-01T11:18:14.668254+0000 mon.a (mon.0) 2346 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:18:15.826 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:15 smithi078 bash[75299]: audit 2023-10-01T11:18:14.675521+0000 mon.a (mon.0) 2347 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:18:15.827 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:15 smithi078 bash[78274]: audit 2023-10-01T11:18:14.292876+0000 mon.a (mon.0) 2345 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T11:18:15.827 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:15 smithi078 bash[78274]: audit 2023-10-01T11:18:14.668254+0000 mon.a (mon.0) 2346 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:18:15.827 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:15 smithi078 bash[78274]: audit 2023-10-01T11:18:14.675521+0000 mon.a (mon.0) 2347 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:18:15.828 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:15 smithi173 bash[58816]: audit 2023-10-01T11:18:14.292876+0000 mon.a (mon.0) 2345 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T11:18:15.872 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:15 smithi173 bash[58816]: audit 2023-10-01T11:18:14.668254+0000 mon.a (mon.0) 2346 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:18:15.872 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:15 smithi173 bash[58816]: audit 2023-10-01T11:18:14.675521+0000 mon.a (mon.0) 2347 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:18:16.462 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:16 smithi173 bash[58816]: cluster 2023-10-01T11:18:14.574815+0000 mgr.y (mgr.25072) 1831 : cluster 0 pgmap v2020: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:16.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:16 smithi078 bash[75299]: cluster 2023-10-01T11:18:14.574815+0000 mgr.y (mgr.25072) 1831 : cluster 0 pgmap v2020: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:16.781 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:16 smithi078 bash[78274]: cluster 2023-10-01T11:18:14.574815+0000 mgr.y (mgr.25072) 1831 : cluster 0 pgmap v2020: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:16.890 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:18:18.221 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-10-01T11:18:18.221 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T11:18:18.246 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:18 smithi173 bash[58816]: cluster 2023-10-01T11:18:16.575578+0000 mgr.y (mgr.25072) 1832 : cluster 0 pgmap v2021: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:18.246 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:18 smithi173 bash[58816]: audit 2023-10-01T11:18:17.488061+0000 mon.a (mon.0) 2348 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:18:18.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:18 smithi078 bash[78274]: cluster 2023-10-01T11:18:16.575578+0000 mgr.y (mgr.25072) 1832 : cluster 0 pgmap v2021: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:18.353 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:18 smithi078 bash[78274]: audit 2023-10-01T11:18:17.488061+0000 mon.a (mon.0) 2348 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:18:18.353 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:18 smithi078 bash[75299]: cluster 2023-10-01T11:18:16.575578+0000 mgr.y (mgr.25072) 1832 : cluster 0 pgmap v2021: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:18.353 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:18 smithi078 bash[75299]: audit 2023-10-01T11:18:17.488061+0000 mon.a (mon.0) 2348 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:18:19.046 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-10-01T11:18:20.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:20 smithi078 bash[75299]: cluster 2023-10-01T11:18:18.576074+0000 mgr.y (mgr.25072) 1833 : cluster 0 pgmap v2022: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:20.344 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:20 smithi078 bash[78274]: cluster 2023-10-01T11:18:18.576074+0000 mgr.y (mgr.25072) 1833 : cluster 0 pgmap v2022: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:20.344 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:18:20 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:18:20] "GET /metrics HTTP/1.1" 200 34021 "" "Prometheus/2.33.4" 2023-10-01T11:18:20.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:20 smithi173 bash[58816]: cluster 2023-10-01T11:18:18.576074+0000 mgr.y (mgr.25072) 1833 : cluster 0 pgmap v2022: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:21.229 INFO:teuthology.orchestra.run.smithi078.stderr:set osd.4 primary-affinity to 1 (8655362) 2023-10-01T11:18:21.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:21 smithi078 bash[75299]: audit 2023-10-01T11:18:20.542102+0000 mon.a (mon.0) 2349 : audit 1 from='client.? 172.21.15.78:0/2949740459' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-10-01T11:18:21.602 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:21 smithi078 bash[75299]: audit 2023-10-01T11:18:20.767285+0000 mon.a (mon.0) 2350 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:18:21.603 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:21 smithi078 bash[75299]: audit 2023-10-01T11:18:20.774118+0000 mon.a (mon.0) 2351 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:18:21.603 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:21 smithi078 bash[78274]: audit 2023-10-01T11:18:20.542102+0000 mon.a (mon.0) 2349 : audit 1 from='client.? 172.21.15.78:0/2949740459' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-10-01T11:18:21.603 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:21 smithi078 bash[78274]: audit 2023-10-01T11:18:20.767285+0000 mon.a (mon.0) 2350 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:18:21.603 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:21 smithi078 bash[78274]: audit 2023-10-01T11:18:20.774118+0000 mon.a (mon.0) 2351 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:18:21.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:21 smithi173 bash[58816]: audit 2023-10-01T11:18:20.542102+0000 mon.a (mon.0) 2349 : audit 1 from='client.? 172.21.15.78:0/2949740459' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-10-01T11:18:21.624 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:21 smithi173 bash[58816]: audit 2023-10-01T11:18:20.767285+0000 mon.a (mon.0) 2350 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:18:21.624 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:21 smithi173 bash[58816]: audit 2023-10-01T11:18:20.774118+0000 mon.a (mon.0) 2351 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:18:22.276 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd reweight 5 1 2023-10-01T11:18:22.502 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:22 smithi078 bash[78274]: cluster 2023-10-01T11:18:20.576740+0000 mgr.y (mgr.25072) 1834 : cluster 0 pgmap v2023: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:22.502 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:22 smithi078 bash[78274]: audit 2023-10-01T11:18:21.222027+0000 mon.a (mon.0) 2352 : audit 1 from='client.? 172.21.15.78:0/2949740459' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-10-01T11:18:22.502 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:22 smithi078 bash[78274]: cluster 2023-10-01T11:18:21.222098+0000 mon.a (mon.0) 2353 : cluster 0 osdmap e431: 8 total, 8 up, 8 in 2023-10-01T11:18:22.502 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:22 smithi078 bash[78274]: audit 2023-10-01T11:18:21.998308+0000 mon.a (mon.0) 2354 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:18:22.502 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:22 smithi078 bash[78274]: audit 2023-10-01T11:18:22.007605+0000 mon.a (mon.0) 2355 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:18:22.503 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:22 smithi078 bash[75299]: cluster 2023-10-01T11:18:20.576740+0000 mgr.y (mgr.25072) 1834 : cluster 0 pgmap v2023: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:22.503 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:22 smithi078 bash[75299]: audit 2023-10-01T11:18:21.222027+0000 mon.a (mon.0) 2352 : audit 1 from='client.? 172.21.15.78:0/2949740459' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-10-01T11:18:22.503 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:22 smithi078 bash[75299]: cluster 2023-10-01T11:18:21.222098+0000 mon.a (mon.0) 2353 : cluster 0 osdmap e431: 8 total, 8 up, 8 in 2023-10-01T11:18:22.503 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:22 smithi078 bash[75299]: audit 2023-10-01T11:18:21.998308+0000 mon.a (mon.0) 2354 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:18:22.503 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:22 smithi078 bash[75299]: audit 2023-10-01T11:18:22.007605+0000 mon.a (mon.0) 2355 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:18:22.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:22 smithi173 bash[58816]: cluster 2023-10-01T11:18:20.576740+0000 mgr.y (mgr.25072) 1834 : cluster 0 pgmap v2023: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:22.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:22 smithi173 bash[58816]: audit 2023-10-01T11:18:21.222027+0000 mon.a (mon.0) 2352 : audit 1 from='client.? 172.21.15.78:0/2949740459' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-10-01T11:18:22.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:22 smithi173 bash[58816]: cluster 2023-10-01T11:18:21.222098+0000 mon.a (mon.0) 2353 : cluster 0 osdmap e431: 8 total, 8 up, 8 in 2023-10-01T11:18:22.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:22 smithi173 bash[58816]: audit 2023-10-01T11:18:21.998308+0000 mon.a (mon.0) 2354 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:18:22.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:22 smithi173 bash[58816]: audit 2023-10-01T11:18:22.007605+0000 mon.a (mon.0) 2355 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:18:23.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:23 smithi078 bash[75299]: cluster 2023-10-01T11:18:21.847519+0000 osd.6 (osd.6) 412 : cluster 3 failed to encode map e431 with expected crc 2023-10-01T11:18:23.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:23 smithi078 bash[75299]: cluster 2023-10-01T11:18:21.851306+0000 osd.6 (osd.6) 413 : cluster 3 failed to encode map e431 with expected crc 2023-10-01T11:18:23.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:23 smithi078 bash[75299]: cluster 2023-10-01T11:18:22.089363+0000 osd.5 (osd.5) 466 : cluster 3 failed to encode map e431 with expected crc 2023-10-01T11:18:23.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:23 smithi078 bash[75299]: cluster 2023-10-01T11:18:22.089459+0000 osd.7 (osd.7) 507 : cluster 3 failed to encode map e431 with expected crc 2023-10-01T11:18:23.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:23 smithi078 bash[75299]: audit 2023-10-01T11:18:22.478521+0000 mon.a (mon.0) 2356 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T11:18:23.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:23 smithi078 bash[75299]: audit 2023-10-01T11:18:22.479891+0000 mon.a (mon.0) 2357 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T11:18:23.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:23 smithi078 bash[75299]: audit 2023-10-01T11:18:22.489872+0000 mon.a (mon.0) 2358 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:18:23.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:23 smithi078 bash[75299]: cluster 2023-10-01T11:18:22.577435+0000 mgr.y (mgr.25072) 1835 : cluster 0 pgmap v2025: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:23.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:23 smithi078 bash[78274]: cluster 2023-10-01T11:18:21.847519+0000 osd.6 (osd.6) 412 : cluster 3 failed to encode map e431 with expected crc 2023-10-01T11:18:23.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:23 smithi078 bash[78274]: cluster 2023-10-01T11:18:21.851306+0000 osd.6 (osd.6) 413 : cluster 3 failed to encode map e431 with expected crc 2023-10-01T11:18:23.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:23 smithi078 bash[78274]: cluster 2023-10-01T11:18:22.089363+0000 osd.5 (osd.5) 466 : cluster 3 failed to encode map e431 with expected crc 2023-10-01T11:18:23.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:23 smithi078 bash[78274]: cluster 2023-10-01T11:18:22.089459+0000 osd.7 (osd.7) 507 : cluster 3 failed to encode map e431 with expected crc 2023-10-01T11:18:23.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:23 smithi078 bash[78274]: audit 2023-10-01T11:18:22.478521+0000 mon.a (mon.0) 2356 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T11:18:23.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:23 smithi078 bash[78274]: audit 2023-10-01T11:18:22.479891+0000 mon.a (mon.0) 2357 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T11:18:23.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:23 smithi078 bash[78274]: audit 2023-10-01T11:18:22.489872+0000 mon.a (mon.0) 2358 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:18:23.584 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:23 smithi078 bash[78274]: cluster 2023-10-01T11:18:22.577435+0000 mgr.y (mgr.25072) 1835 : cluster 0 pgmap v2025: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:23.588 INFO:teuthology.orchestra.run.smithi173.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.b/config 2023-10-01T11:18:23.618 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:23 smithi173 bash[58816]: cluster 2023-10-01T11:18:21.847519+0000 osd.6 (osd.6) 412 : cluster 3 failed to encode map e431 with expected crc 2023-10-01T11:18:23.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:23 smithi173 bash[58816]: cluster 2023-10-01T11:18:21.851306+0000 osd.6 (osd.6) 413 : cluster 3 failed to encode map e431 with expected crc 2023-10-01T11:18:23.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:23 smithi173 bash[58816]: cluster 2023-10-01T11:18:22.089363+0000 osd.5 (osd.5) 466 : cluster 3 failed to encode map e431 with expected crc 2023-10-01T11:18:23.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:23 smithi173 bash[58816]: cluster 2023-10-01T11:18:22.089459+0000 osd.7 (osd.7) 507 : cluster 3 failed to encode map e431 with expected crc 2023-10-01T11:18:23.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:23 smithi173 bash[58816]: audit 2023-10-01T11:18:22.478521+0000 mon.a (mon.0) 2356 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T11:18:23.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:23 smithi173 bash[58816]: audit 2023-10-01T11:18:22.479891+0000 mon.a (mon.0) 2357 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T11:18:23.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:23 smithi173 bash[58816]: audit 2023-10-01T11:18:22.489872+0000 mon.a (mon.0) 2358 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:18:23.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:23 smithi173 bash[58816]: cluster 2023-10-01T11:18:22.577435+0000 mgr.y (mgr.25072) 1835 : cluster 0 pgmap v2025: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:26.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:25 smithi078 bash[75299]: cluster 2023-10-01T11:18:24.578355+0000 mgr.y (mgr.25072) 1836 : cluster 0 pgmap v2026: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:26.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:25 smithi078 bash[78274]: cluster 2023-10-01T11:18:24.578355+0000 mgr.y (mgr.25072) 1836 : cluster 0 pgmap v2026: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:26.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:25 smithi173 bash[58816]: cluster 2023-10-01T11:18:24.578355+0000 mgr.y (mgr.25072) 1836 : cluster 0 pgmap v2026: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:26.895 INFO:teuthology.orchestra.run.smithi173.stdout:ERROR: (22) Invalid argument 2023-10-01T11:18:26.896 INFO:teuthology.orchestra.run.smithi173.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-10-01T11:18:27.265 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:18:28.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:27 smithi078 bash[75299]: cluster 2023-10-01T11:18:26.579090+0000 mgr.y (mgr.25072) 1837 : cluster 0 pgmap v2027: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:28.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:27 smithi078 bash[78274]: cluster 2023-10-01T11:18:26.579090+0000 mgr.y (mgr.25072) 1837 : cluster 0 pgmap v2027: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:28.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:27 smithi173 bash[58816]: cluster 2023-10-01T11:18:26.579090+0000 mgr.y (mgr.25072) 1837 : cluster 0 pgmap v2027: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:30.270 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:29 smithi078 bash[75299]: cluster 2023-10-01T11:18:28.579821+0000 mgr.y (mgr.25072) 1838 : cluster 0 pgmap v2028: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:30.551 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:29 smithi078 bash[78274]: cluster 2023-10-01T11:18:28.579821+0000 mgr.y (mgr.25072) 1838 : cluster 0 pgmap v2028: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:30.551 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:29 smithi173 bash[58816]: cluster 2023-10-01T11:18:28.579821+0000 mgr.y (mgr.25072) 1838 : cluster 0 pgmap v2028: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:30.551 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:18:30 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:18:30] "GET /metrics HTTP/1.1" 200 34021 "" "Prometheus/2.33.4" 2023-10-01T11:18:30.988 INFO:teuthology.orchestra.run.smithi078.stderr:reweighted osd.5 to 1 (10000) 2023-10-01T11:18:31.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:30 smithi078 bash[75299]: audit 2023-10-01T11:18:30.949761+0000 mon.a (mon.0) 2359 : audit 1 from='client.? 172.21.15.78:0/1434009871' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-01T11:18:31.382 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:30 smithi078 bash[78274]: audit 2023-10-01T11:18:30.949761+0000 mon.a (mon.0) 2359 : audit 1 from='client.? 172.21.15.78:0/1434009871' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-01T11:18:31.383 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:30 smithi173 bash[58816]: audit 2023-10-01T11:18:30.949761+0000 mon.a (mon.0) 2359 : audit 1 from='client.? 172.21.15.78:0/1434009871' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-10-01T11:18:31.940 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 5 1 2023-10-01T11:18:32.296 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:31 smithi078 bash[78274]: cluster 2023-10-01T11:18:30.580881+0000 mgr.y (mgr.25072) 1839 : cluster 0 pgmap v2029: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:32.359 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:31 smithi078 bash[78274]: audit 2023-10-01T11:18:30.981207+0000 mon.a (mon.0) 2360 : audit 1 from='client.? 172.21.15.78:0/1434009871' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-01T11:18:32.359 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:31 smithi078 bash[78274]: cluster 2023-10-01T11:18:30.981268+0000 mon.a (mon.0) 2361 : cluster 0 osdmap e432: 8 total, 8 up, 8 in 2023-10-01T11:18:32.359 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:31 smithi078 bash[78274]: cluster 2023-10-01T11:18:30.981524+0000 osd.7 (osd.7) 508 : cluster 3 failed to encode map e432 with expected crc 2023-10-01T11:18:32.359 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:31 smithi078 bash[78274]: cluster 2023-10-01T11:18:30.992882+0000 osd.6 (osd.6) 414 : cluster 3 failed to encode map e432 with expected crc 2023-10-01T11:18:32.361 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:31 smithi078 bash[78274]: cluster 2023-10-01T11:18:30.993042+0000 osd.5 (osd.5) 467 : cluster 3 failed to encode map e432 with expected crc 2023-10-01T11:18:32.361 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:31 smithi078 bash[75299]: cluster 2023-10-01T11:18:30.580881+0000 mgr.y (mgr.25072) 1839 : cluster 0 pgmap v2029: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:32.361 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:31 smithi078 bash[75299]: audit 2023-10-01T11:18:30.981207+0000 mon.a (mon.0) 2360 : audit 1 from='client.? 172.21.15.78:0/1434009871' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-01T11:18:32.361 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:31 smithi078 bash[75299]: cluster 2023-10-01T11:18:30.981268+0000 mon.a (mon.0) 2361 : cluster 0 osdmap e432: 8 total, 8 up, 8 in 2023-10-01T11:18:32.361 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:31 smithi078 bash[75299]: cluster 2023-10-01T11:18:30.981524+0000 osd.7 (osd.7) 508 : cluster 3 failed to encode map e432 with expected crc 2023-10-01T11:18:32.361 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:31 smithi078 bash[75299]: cluster 2023-10-01T11:18:30.992882+0000 osd.6 (osd.6) 414 : cluster 3 failed to encode map e432 with expected crc 2023-10-01T11:18:32.361 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:31 smithi078 bash[75299]: cluster 2023-10-01T11:18:30.993042+0000 osd.5 (osd.5) 467 : cluster 3 failed to encode map e432 with expected crc 2023-10-01T11:18:32.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:31 smithi173 bash[58816]: cluster 2023-10-01T11:18:30.580881+0000 mgr.y (mgr.25072) 1839 : cluster 0 pgmap v2029: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:32.702 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:31 smithi173 bash[58816]: audit 2023-10-01T11:18:30.981207+0000 mon.a (mon.0) 2360 : audit 1 from='client.? 172.21.15.78:0/1434009871' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-10-01T11:18:32.702 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:31 smithi173 bash[58816]: cluster 2023-10-01T11:18:30.981268+0000 mon.a (mon.0) 2361 : cluster 0 osdmap e432: 8 total, 8 up, 8 in 2023-10-01T11:18:32.702 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:31 smithi173 bash[58816]: cluster 2023-10-01T11:18:30.981524+0000 osd.7 (osd.7) 508 : cluster 3 failed to encode map e432 with expected crc 2023-10-01T11:18:32.702 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:31 smithi173 bash[58816]: cluster 2023-10-01T11:18:30.992882+0000 osd.6 (osd.6) 414 : cluster 3 failed to encode map e432 with expected crc 2023-10-01T11:18:32.703 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:31 smithi173 bash[58816]: cluster 2023-10-01T11:18:30.993042+0000 osd.5 (osd.5) 467 : cluster 3 failed to encode map e432 with expected crc 2023-10-01T11:18:33.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:32 smithi078 bash[75299]: audit 2023-10-01T11:18:32.488678+0000 mon.a (mon.0) 2362 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:18:33.696 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:32 smithi078 bash[78274]: audit 2023-10-01T11:18:32.488678+0000 mon.a (mon.0) 2362 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:18:33.698 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:32 smithi173 bash[58816]: audit 2023-10-01T11:18:32.488678+0000 mon.a (mon.0) 2362 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:18:34.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:33 smithi078 bash[75299]: cluster 2023-10-01T11:18:32.581564+0000 mgr.y (mgr.25072) 1840 : cluster 0 pgmap v2031: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:34.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:33 smithi078 bash[78274]: cluster 2023-10-01T11:18:32.581564+0000 mgr.y (mgr.25072) 1840 : cluster 0 pgmap v2031: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:34.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:33 smithi173 bash[58816]: cluster 2023-10-01T11:18:32.581564+0000 mgr.y (mgr.25072) 1840 : cluster 0 pgmap v2031: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:36.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:35 smithi078 bash[75299]: cluster 2023-10-01T11:18:34.582389+0000 mgr.y (mgr.25072) 1841 : cluster 0 pgmap v2032: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:36.341 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:35 smithi078 bash[78274]: cluster 2023-10-01T11:18:34.582389+0000 mgr.y (mgr.25072) 1841 : cluster 0 pgmap v2032: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:36.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:35 smithi173 bash[58816]: cluster 2023-10-01T11:18:34.582389+0000 mgr.y (mgr.25072) 1841 : cluster 0 pgmap v2032: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:37.057 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:18:38.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:37 smithi078 bash[75299]: cluster 2023-10-01T11:18:36.583153+0000 mgr.y (mgr.25072) 1842 : cluster 0 pgmap v2033: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:38.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:37 smithi078 bash[78274]: cluster 2023-10-01T11:18:36.583153+0000 mgr.y (mgr.25072) 1842 : cluster 0 pgmap v2033: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:38.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:38 smithi173 bash[58816]: cluster 2023-10-01T11:18:36.583153+0000 mgr.y (mgr.25072) 1842 : cluster 0 pgmap v2033: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:40.271 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:40 smithi078 bash[75299]: cluster 2023-10-01T11:18:38.583727+0000 mgr.y (mgr.25072) 1843 : cluster 0 pgmap v2034: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:40.271 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:40 smithi078 bash[78274]: cluster 2023-10-01T11:18:38.583727+0000 mgr.y (mgr.25072) 1843 : cluster 0 pgmap v2034: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:40.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:40 smithi173 bash[58816]: cluster 2023-10-01T11:18:38.583727+0000 mgr.y (mgr.25072) 1843 : cluster 0 pgmap v2034: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:40.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:18:40 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:18:40] "GET /metrics HTTP/1.1" 200 34014 "" "Prometheus/2.33.4" 2023-10-01T11:18:41.018 INFO:teuthology.orchestra.run.smithi078.stderr:set osd.5 primary-affinity to 1 (8655362) 2023-10-01T11:18:41.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:41 smithi078 bash[75299]: audit 2023-10-01T11:18:40.835937+0000 mon.c (mon.2) 266 : audit 1 from='client.? 172.21.15.78:0/1934181752' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-01T11:18:41.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:41 smithi078 bash[75299]: audit 2023-10-01T11:18:40.836790+0000 mon.a (mon.0) 2363 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-01T11:18:41.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:41 smithi078 bash[78274]: audit 2023-10-01T11:18:40.835937+0000 mon.c (mon.2) 266 : audit 1 from='client.? 172.21.15.78:0/1934181752' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-01T11:18:41.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:41 smithi078 bash[78274]: audit 2023-10-01T11:18:40.836790+0000 mon.a (mon.0) 2363 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-01T11:18:41.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:41 smithi173 bash[58816]: audit 2023-10-01T11:18:40.835937+0000 mon.c (mon.2) 266 : audit 1 from='client.? 172.21.15.78:0/1934181752' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-01T11:18:41.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:41 smithi173 bash[58816]: audit 2023-10-01T11:18:40.836790+0000 mon.a (mon.0) 2363 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-10-01T11:18:41.995 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd reweight 6 1 2023-10-01T11:18:42.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:42 smithi078 bash[75299]: cluster 2023-10-01T11:18:40.584684+0000 mgr.y (mgr.25072) 1844 : cluster 0 pgmap v2035: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:42.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:42 smithi078 bash[75299]: cluster 2023-10-01T11:18:41.009203+0000 osd.7 (osd.7) 509 : cluster 3 failed to encode map e433 with expected crc 2023-10-01T11:18:42.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:42 smithi078 bash[75299]: audit 2023-10-01T11:18:41.010432+0000 mon.a (mon.0) 2364 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-10-01T11:18:42.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:42 smithi078 bash[75299]: cluster 2023-10-01T11:18:41.010545+0000 mon.a (mon.0) 2365 : cluster 0 osdmap e433: 8 total, 8 up, 8 in 2023-10-01T11:18:42.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:42 smithi078 bash[78274]: cluster 2023-10-01T11:18:40.584684+0000 mgr.y (mgr.25072) 1844 : cluster 0 pgmap v2035: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:42.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:42 smithi078 bash[78274]: cluster 2023-10-01T11:18:41.009203+0000 osd.7 (osd.7) 509 : cluster 3 failed to encode map e433 with expected crc 2023-10-01T11:18:42.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:42 smithi078 bash[78274]: audit 2023-10-01T11:18:41.010432+0000 mon.a (mon.0) 2364 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-10-01T11:18:42.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:42 smithi078 bash[78274]: cluster 2023-10-01T11:18:41.010545+0000 mon.a (mon.0) 2365 : cluster 0 osdmap e433: 8 total, 8 up, 8 in 2023-10-01T11:18:42.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:42 smithi173 bash[58816]: cluster 2023-10-01T11:18:40.584684+0000 mgr.y (mgr.25072) 1844 : cluster 0 pgmap v2035: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:42.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:42 smithi173 bash[58816]: cluster 2023-10-01T11:18:41.009203+0000 osd.7 (osd.7) 509 : cluster 3 failed to encode map e433 with expected crc 2023-10-01T11:18:42.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:42 smithi173 bash[58816]: audit 2023-10-01T11:18:41.010432+0000 mon.a (mon.0) 2364 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-10-01T11:18:42.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:42 smithi173 bash[58816]: cluster 2023-10-01T11:18:41.010545+0000 mon.a (mon.0) 2365 : cluster 0 osdmap e433: 8 total, 8 up, 8 in 2023-10-01T11:18:43.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:43 smithi078 bash[78274]: cluster 2023-10-01T11:18:41.965801+0000 osd.5 (osd.5) 468 : cluster 3 failed to encode map e433 with expected crc 2023-10-01T11:18:43.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:43 smithi078 bash[78274]: cluster 2023-10-01T11:18:41.968598+0000 osd.5 (osd.5) 469 : cluster 3 failed to encode map e433 with expected crc 2023-10-01T11:18:43.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:43 smithi078 bash[78274]: cluster 2023-10-01T11:18:42.619128+0000 osd.6 (osd.6) 415 : cluster 3 failed to encode map e433 with expected crc 2023-10-01T11:18:43.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:43 smithi078 bash[75299]: cluster 2023-10-01T11:18:41.965801+0000 osd.5 (osd.5) 468 : cluster 3 failed to encode map e433 with expected crc 2023-10-01T11:18:43.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:43 smithi078 bash[75299]: cluster 2023-10-01T11:18:41.968598+0000 osd.5 (osd.5) 469 : cluster 3 failed to encode map e433 with expected crc 2023-10-01T11:18:43.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:43 smithi078 bash[75299]: cluster 2023-10-01T11:18:42.619128+0000 osd.6 (osd.6) 415 : cluster 3 failed to encode map e433 with expected crc 2023-10-01T11:18:43.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:43 smithi173 bash[58816]: cluster 2023-10-01T11:18:41.965801+0000 osd.5 (osd.5) 468 : cluster 3 failed to encode map e433 with expected crc 2023-10-01T11:18:43.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:43 smithi173 bash[58816]: cluster 2023-10-01T11:18:41.968598+0000 osd.5 (osd.5) 469 : cluster 3 failed to encode map e433 with expected crc 2023-10-01T11:18:43.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:43 smithi173 bash[58816]: cluster 2023-10-01T11:18:42.619128+0000 osd.6 (osd.6) 415 : cluster 3 failed to encode map e433 with expected crc 2023-10-01T11:18:44.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:44 smithi078 bash[75299]: cluster 2023-10-01T11:18:42.585478+0000 mgr.y (mgr.25072) 1845 : cluster 0 pgmap v2037: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:44.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:44 smithi078 bash[78274]: cluster 2023-10-01T11:18:42.585478+0000 mgr.y (mgr.25072) 1845 : cluster 0 pgmap v2037: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:44.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:44 smithi173 bash[58816]: cluster 2023-10-01T11:18:42.585478+0000 mgr.y (mgr.25072) 1845 : cluster 0 pgmap v2037: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:46.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:46 smithi078 bash[78274]: cluster 2023-10-01T11:18:44.586558+0000 mgr.y (mgr.25072) 1846 : cluster 0 pgmap v2038: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:46.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:46 smithi078 bash[75299]: cluster 2023-10-01T11:18:44.586558+0000 mgr.y (mgr.25072) 1846 : cluster 0 pgmap v2038: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:46.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:46 smithi173 bash[58816]: cluster 2023-10-01T11:18:44.586558+0000 mgr.y (mgr.25072) 1846 : cluster 0 pgmap v2038: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:46.815 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:18:48.217 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:48 smithi078 bash[75299]: cluster 2023-10-01T11:18:46.587372+0000 mgr.y (mgr.25072) 1847 : cluster 0 pgmap v2039: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:48.217 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:48 smithi078 bash[75299]: audit 2023-10-01T11:18:47.488688+0000 mon.a (mon.0) 2366 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:18:48.217 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:48 smithi078 bash[78274]: cluster 2023-10-01T11:18:46.587372+0000 mgr.y (mgr.25072) 1847 : cluster 0 pgmap v2039: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:48.217 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:48 smithi078 bash[78274]: audit 2023-10-01T11:18:47.488688+0000 mon.a (mon.0) 2366 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:18:48.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:48 smithi173 bash[58816]: cluster 2023-10-01T11:18:46.587372+0000 mgr.y (mgr.25072) 1847 : cluster 0 pgmap v2039: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:48.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:48 smithi173 bash[58816]: audit 2023-10-01T11:18:47.488688+0000 mon.a (mon.0) 2366 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:18:50.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:50 smithi078 bash[75299]: cluster 2023-10-01T11:18:48.588025+0000 mgr.y (mgr.25072) 1848 : cluster 0 pgmap v2040: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:50.332 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:18:50 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:18:50] "GET /metrics HTTP/1.1" 200 34021 "" "Prometheus/2.33.4" 2023-10-01T11:18:50.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:50 smithi078 bash[78274]: cluster 2023-10-01T11:18:48.588025+0000 mgr.y (mgr.25072) 1848 : cluster 0 pgmap v2040: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:50.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:50 smithi173 bash[58816]: cluster 2023-10-01T11:18:48.588025+0000 mgr.y (mgr.25072) 1848 : cluster 0 pgmap v2040: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:51.066 INFO:teuthology.orchestra.run.smithi078.stderr:reweighted osd.6 to 1 (10000) 2023-10-01T11:18:51.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:51 smithi078 bash[75299]: audit 2023-10-01T11:18:50.092432+0000 mon.a (mon.0) 2367 : audit 1 from='client.? 172.21.15.78:0/4013451712' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-01T11:18:51.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:51 smithi078 bash[78274]: audit 2023-10-01T11:18:50.092432+0000 mon.a (mon.0) 2367 : audit 1 from='client.? 172.21.15.78:0/4013451712' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-01T11:18:51.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:51 smithi173 bash[58816]: audit 2023-10-01T11:18:50.092432+0000 mon.a (mon.0) 2367 : audit 1 from='client.? 172.21.15.78:0/4013451712' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-10-01T11:18:52.019 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 6 1 2023-10-01T11:18:52.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:52 smithi078 bash[75299]: cluster 2023-10-01T11:18:50.589060+0000 mgr.y (mgr.25072) 1849 : cluster 0 pgmap v2041: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:52.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:52 smithi078 bash[75299]: cluster 2023-10-01T11:18:51.058360+0000 osd.5 (osd.5) 470 : cluster 3 failed to encode map e434 with expected crc 2023-10-01T11:18:52.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:52 smithi078 bash[75299]: audit 2023-10-01T11:18:51.059751+0000 mon.a (mon.0) 2368 : audit 1 from='client.? 172.21.15.78:0/4013451712' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-01T11:18:52.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:52 smithi078 bash[75299]: cluster 2023-10-01T11:18:51.059839+0000 mon.a (mon.0) 2369 : cluster 0 osdmap e434: 8 total, 8 up, 8 in 2023-10-01T11:18:52.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:52 smithi078 bash[75299]: cluster 2023-10-01T11:18:51.098895+0000 osd.7 (osd.7) 510 : cluster 3 failed to encode map e434 with expected crc 2023-10-01T11:18:52.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:52 smithi078 bash[75299]: cluster 2023-10-01T11:18:51.098909+0000 osd.6 (osd.6) 416 : cluster 3 failed to encode map e434 with expected crc 2023-10-01T11:18:52.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:52 smithi078 bash[78274]: cluster 2023-10-01T11:18:50.589060+0000 mgr.y (mgr.25072) 1849 : cluster 0 pgmap v2041: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:52.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:52 smithi078 bash[78274]: cluster 2023-10-01T11:18:51.058360+0000 osd.5 (osd.5) 470 : cluster 3 failed to encode map e434 with expected crc 2023-10-01T11:18:52.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:52 smithi078 bash[78274]: audit 2023-10-01T11:18:51.059751+0000 mon.a (mon.0) 2368 : audit 1 from='client.? 172.21.15.78:0/4013451712' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-01T11:18:52.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:52 smithi078 bash[78274]: cluster 2023-10-01T11:18:51.059839+0000 mon.a (mon.0) 2369 : cluster 0 osdmap e434: 8 total, 8 up, 8 in 2023-10-01T11:18:52.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:52 smithi078 bash[78274]: cluster 2023-10-01T11:18:51.098895+0000 osd.7 (osd.7) 510 : cluster 3 failed to encode map e434 with expected crc 2023-10-01T11:18:52.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:52 smithi078 bash[78274]: cluster 2023-10-01T11:18:51.098909+0000 osd.6 (osd.6) 416 : cluster 3 failed to encode map e434 with expected crc 2023-10-01T11:18:52.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:52 smithi173 bash[58816]: cluster 2023-10-01T11:18:50.589060+0000 mgr.y (mgr.25072) 1849 : cluster 0 pgmap v2041: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:52.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:52 smithi173 bash[58816]: cluster 2023-10-01T11:18:51.058360+0000 osd.5 (osd.5) 470 : cluster 3 failed to encode map e434 with expected crc 2023-10-01T11:18:52.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:52 smithi173 bash[58816]: audit 2023-10-01T11:18:51.059751+0000 mon.a (mon.0) 2368 : audit 1 from='client.? 172.21.15.78:0/4013451712' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-10-01T11:18:52.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:52 smithi173 bash[58816]: cluster 2023-10-01T11:18:51.059839+0000 mon.a (mon.0) 2369 : cluster 0 osdmap e434: 8 total, 8 up, 8 in 2023-10-01T11:18:52.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:52 smithi173 bash[58816]: cluster 2023-10-01T11:18:51.098895+0000 osd.7 (osd.7) 510 : cluster 3 failed to encode map e434 with expected crc 2023-10-01T11:18:52.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:52 smithi173 bash[58816]: cluster 2023-10-01T11:18:51.098909+0000 osd.6 (osd.6) 416 : cluster 3 failed to encode map e434 with expected crc 2023-10-01T11:18:54.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:54 smithi078 bash[75299]: cluster 2023-10-01T11:18:52.589952+0000 mgr.y (mgr.25072) 1850 : cluster 0 pgmap v2043: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:54.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:54 smithi078 bash[78274]: cluster 2023-10-01T11:18:52.589952+0000 mgr.y (mgr.25072) 1850 : cluster 0 pgmap v2043: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:54.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:54 smithi173 bash[58816]: cluster 2023-10-01T11:18:52.589952+0000 mgr.y (mgr.25072) 1850 : cluster 0 pgmap v2043: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:56.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:56 smithi078 bash[75299]: cluster 2023-10-01T11:18:54.591040+0000 mgr.y (mgr.25072) 1851 : cluster 0 pgmap v2044: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:56.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:56 smithi078 bash[78274]: cluster 2023-10-01T11:18:54.591040+0000 mgr.y (mgr.25072) 1851 : cluster 0 pgmap v2044: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:56.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:56 smithi173 bash[58816]: cluster 2023-10-01T11:18:54.591040+0000 mgr.y (mgr.25072) 1851 : cluster 0 pgmap v2044: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:56.793 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:18:58.073 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:18:58 smithi078 bash[78274]: cluster 2023-10-01T11:18:56.591895+0000 mgr.y (mgr.25072) 1852 : cluster 0 pgmap v2045: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:58.074 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:18:58 smithi078 bash[75299]: cluster 2023-10-01T11:18:56.591895+0000 mgr.y (mgr.25072) 1852 : cluster 0 pgmap v2045: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:18:58.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:18:58 smithi173 bash[58816]: cluster 2023-10-01T11:18:56.591895+0000 mgr.y (mgr.25072) 1852 : cluster 0 pgmap v2045: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:00.270 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:00 smithi078 bash[75299]: cluster 2023-10-01T11:18:58.592593+0000 mgr.y (mgr.25072) 1853 : cluster 0 pgmap v2046: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:00.271 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:00 smithi078 bash[78274]: cluster 2023-10-01T11:18:58.592593+0000 mgr.y (mgr.25072) 1853 : cluster 0 pgmap v2046: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:00.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:00 smithi173 bash[58816]: cluster 2023-10-01T11:18:58.592593+0000 mgr.y (mgr.25072) 1853 : cluster 0 pgmap v2046: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:00.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:19:00 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:19:00] "GET /metrics HTTP/1.1" 200 34021 "" "Prometheus/2.33.4" 2023-10-01T11:19:01.091 INFO:teuthology.orchestra.run.smithi078.stderr:set osd.6 primary-affinity to 1 (8655362) 2023-10-01T11:19:01.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:01 smithi078 bash[75299]: audit 2023-10-01T11:19:00.333873+0000 mon.a (mon.0) 2370 : audit 1 from='client.? 172.21.15.78:0/682600321' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-01T11:19:01.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:01 smithi078 bash[78274]: audit 2023-10-01T11:19:00.333873+0000 mon.a (mon.0) 2370 : audit 1 from='client.? 172.21.15.78:0/682600321' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-01T11:19:01.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:01 smithi173 bash[58816]: audit 2023-10-01T11:19:00.333873+0000 mon.a (mon.0) 2370 : audit 1 from='client.? 172.21.15.78:0/682600321' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-10-01T11:19:02.110 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd reweight 7 1 2023-10-01T11:19:02.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:02 smithi173 bash[58816]: cluster 2023-10-01T11:19:00.593680+0000 mgr.y (mgr.25072) 1854 : cluster 0 pgmap v2047: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:02.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:02 smithi173 bash[58816]: audit 2023-10-01T11:19:01.086293+0000 mon.a (mon.0) 2371 : audit 1 from='client.? 172.21.15.78:0/682600321' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-10-01T11:19:02.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:02 smithi173 bash[58816]: cluster 2023-10-01T11:19:01.086385+0000 mon.a (mon.0) 2372 : cluster 0 osdmap e435: 8 total, 8 up, 8 in 2023-10-01T11:19:02.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:02 smithi173 bash[58816]: cluster 2023-10-01T11:19:01.088524+0000 osd.6 (osd.6) 417 : cluster 3 failed to encode map e435 with expected crc 2023-10-01T11:19:02.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:02 smithi078 bash[78274]: cluster 2023-10-01T11:19:00.593680+0000 mgr.y (mgr.25072) 1854 : cluster 0 pgmap v2047: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:02.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:02 smithi078 bash[78274]: audit 2023-10-01T11:19:01.086293+0000 mon.a (mon.0) 2371 : audit 1 from='client.? 172.21.15.78:0/682600321' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-10-01T11:19:02.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:02 smithi078 bash[78274]: cluster 2023-10-01T11:19:01.086385+0000 mon.a (mon.0) 2372 : cluster 0 osdmap e435: 8 total, 8 up, 8 in 2023-10-01T11:19:02.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:02 smithi078 bash[78274]: cluster 2023-10-01T11:19:01.088524+0000 osd.6 (osd.6) 417 : cluster 3 failed to encode map e435 with expected crc 2023-10-01T11:19:02.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:02 smithi078 bash[75299]: cluster 2023-10-01T11:19:00.593680+0000 mgr.y (mgr.25072) 1854 : cluster 0 pgmap v2047: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:02.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:02 smithi078 bash[75299]: audit 2023-10-01T11:19:01.086293+0000 mon.a (mon.0) 2371 : audit 1 from='client.? 172.21.15.78:0/682600321' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-10-01T11:19:02.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:02 smithi078 bash[75299]: cluster 2023-10-01T11:19:01.086385+0000 mon.a (mon.0) 2372 : cluster 0 osdmap e435: 8 total, 8 up, 8 in 2023-10-01T11:19:02.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:02 smithi078 bash[75299]: cluster 2023-10-01T11:19:01.088524+0000 osd.6 (osd.6) 417 : cluster 3 failed to encode map e435 with expected crc 2023-10-01T11:19:03.371 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:03 smithi173 bash[58816]: cluster 2023-10-01T11:19:02.094314+0000 mon.a (mon.0) 2373 : cluster 0 osdmap e436: 8 total, 8 up, 8 in 2023-10-01T11:19:03.372 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:03 smithi173 bash[58816]: cluster 2023-10-01T11:19:02.096158+0000 osd.6 (osd.6) 418 : cluster 3 failed to encode map e436 with expected crc 2023-10-01T11:19:03.372 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:03 smithi173 bash[58816]: cluster 2023-10-01T11:19:02.098229+0000 osd.6 (osd.6) 419 : cluster 3 failed to encode map e436 with expected crc 2023-10-01T11:19:03.372 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:03 smithi173 bash[58816]: audit 2023-10-01T11:19:02.489568+0000 mon.a (mon.0) 2374 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:19:03.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:03 smithi078 bash[78274]: cluster 2023-10-01T11:19:02.094314+0000 mon.a (mon.0) 2373 : cluster 0 osdmap e436: 8 total, 8 up, 8 in 2023-10-01T11:19:03.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:03 smithi078 bash[78274]: cluster 2023-10-01T11:19:02.096158+0000 osd.6 (osd.6) 418 : cluster 3 failed to encode map e436 with expected crc 2023-10-01T11:19:03.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:03 smithi078 bash[78274]: cluster 2023-10-01T11:19:02.098229+0000 osd.6 (osd.6) 419 : cluster 3 failed to encode map e436 with expected crc 2023-10-01T11:19:03.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:03 smithi078 bash[78274]: audit 2023-10-01T11:19:02.489568+0000 mon.a (mon.0) 2374 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:19:03.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:03 smithi078 bash[75299]: cluster 2023-10-01T11:19:02.094314+0000 mon.a (mon.0) 2373 : cluster 0 osdmap e436: 8 total, 8 up, 8 in 2023-10-01T11:19:03.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:03 smithi078 bash[75299]: cluster 2023-10-01T11:19:02.096158+0000 osd.6 (osd.6) 418 : cluster 3 failed to encode map e436 with expected crc 2023-10-01T11:19:03.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:03 smithi078 bash[75299]: cluster 2023-10-01T11:19:02.098229+0000 osd.6 (osd.6) 419 : cluster 3 failed to encode map e436 with expected crc 2023-10-01T11:19:03.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:03 smithi078 bash[75299]: audit 2023-10-01T11:19:02.489568+0000 mon.a (mon.0) 2374 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:19:04.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:04 smithi173 bash[58816]: cluster 2023-10-01T11:19:02.594273+0000 mgr.y (mgr.25072) 1855 : cluster 0 pgmap v2050: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:04.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:04 smithi078 bash[75299]: cluster 2023-10-01T11:19:02.594273+0000 mgr.y (mgr.25072) 1855 : cluster 0 pgmap v2050: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:04.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:04 smithi078 bash[78274]: cluster 2023-10-01T11:19:02.594273+0000 mgr.y (mgr.25072) 1855 : cluster 0 pgmap v2050: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:06.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:06 smithi173 bash[58816]: cluster 2023-10-01T11:19:04.595254+0000 mgr.y (mgr.25072) 1856 : cluster 0 pgmap v2051: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:06.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:06 smithi078 bash[75299]: cluster 2023-10-01T11:19:04.595254+0000 mgr.y (mgr.25072) 1856 : cluster 0 pgmap v2051: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:06.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:06 smithi078 bash[78274]: cluster 2023-10-01T11:19:04.595254+0000 mgr.y (mgr.25072) 1856 : cluster 0 pgmap v2051: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:06.886 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:19:08.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:08 smithi078 bash[78274]: cluster 2023-10-01T11:19:06.595984+0000 mgr.y (mgr.25072) 1857 : cluster 0 pgmap v2052: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:08.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:08 smithi078 bash[75299]: cluster 2023-10-01T11:19:06.595984+0000 mgr.y (mgr.25072) 1857 : cluster 0 pgmap v2052: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:08.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:08 smithi173 bash[58816]: cluster 2023-10-01T11:19:06.595984+0000 mgr.y (mgr.25072) 1857 : cluster 0 pgmap v2052: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:10.269 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:10 smithi078 bash[75299]: cluster 2023-10-01T11:19:08.596608+0000 mgr.y (mgr.25072) 1858 : cluster 0 pgmap v2053: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:10.269 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:10 smithi078 bash[78274]: cluster 2023-10-01T11:19:08.596608+0000 mgr.y (mgr.25072) 1858 : cluster 0 pgmap v2053: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:10.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:10 smithi173 bash[58816]: cluster 2023-10-01T11:19:08.596608+0000 mgr.y (mgr.25072) 1858 : cluster 0 pgmap v2053: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:10.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:19:10 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:19:10] "GET /metrics HTTP/1.1" 200 34023 "" "Prometheus/2.33.4" 2023-10-01T11:19:11.134 INFO:teuthology.orchestra.run.smithi078.stderr:reweighted osd.7 to 1 (10000) 2023-10-01T11:19:11.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:11 smithi173 bash[58816]: audit 2023-10-01T11:19:10.468384+0000 mon.c (mon.2) 267 : audit 1 from='client.? 172.21.15.78:0/3600855039' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T11:19:11.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:11 smithi173 bash[58816]: audit 2023-10-01T11:19:10.469023+0000 mon.a (mon.0) 2375 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T11:19:11.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:11 smithi078 bash[75299]: audit 2023-10-01T11:19:10.468384+0000 mon.c (mon.2) 267 : audit 1 from='client.? 172.21.15.78:0/3600855039' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T11:19:11.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:11 smithi078 bash[75299]: audit 2023-10-01T11:19:10.469023+0000 mon.a (mon.0) 2375 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T11:19:11.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:11 smithi078 bash[78274]: audit 2023-10-01T11:19:10.468384+0000 mon.c (mon.2) 267 : audit 1 from='client.? 172.21.15.78:0/3600855039' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T11:19:11.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:11 smithi078 bash[78274]: audit 2023-10-01T11:19:10.469023+0000 mon.a (mon.0) 2375 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T11:19:12.057 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd primary-affinity 7 1 2023-10-01T11:19:12.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:12 smithi078 bash[75299]: cluster 2023-10-01T11:19:10.597615+0000 mgr.y (mgr.25072) 1859 : cluster 0 pgmap v2054: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:12.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:12 smithi078 bash[75299]: audit 2023-10-01T11:19:11.126022+0000 mon.a (mon.0) 2376 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-01T11:19:12.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:12 smithi078 bash[75299]: cluster 2023-10-01T11:19:11.126120+0000 mon.a (mon.0) 2377 : cluster 0 osdmap e437: 8 total, 8 up, 8 in 2023-10-01T11:19:12.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:12 smithi078 bash[75299]: cluster 2023-10-01T11:19:11.302427+0000 osd.6 (osd.6) 420 : cluster 3 failed to encode map e437 with expected crc 2023-10-01T11:19:12.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:12 smithi078 bash[75299]: cluster 2023-10-01T11:19:11.302651+0000 osd.7 (osd.7) 511 : cluster 3 failed to encode map e437 with expected crc 2023-10-01T11:19:12.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:12 smithi078 bash[78274]: cluster 2023-10-01T11:19:10.597615+0000 mgr.y (mgr.25072) 1859 : cluster 0 pgmap v2054: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:12.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:12 smithi078 bash[78274]: audit 2023-10-01T11:19:11.126022+0000 mon.a (mon.0) 2376 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-01T11:19:12.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:12 smithi078 bash[78274]: cluster 2023-10-01T11:19:11.126120+0000 mon.a (mon.0) 2377 : cluster 0 osdmap e437: 8 total, 8 up, 8 in 2023-10-01T11:19:12.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:12 smithi078 bash[78274]: cluster 2023-10-01T11:19:11.302427+0000 osd.6 (osd.6) 420 : cluster 3 failed to encode map e437 with expected crc 2023-10-01T11:19:12.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:12 smithi078 bash[78274]: cluster 2023-10-01T11:19:11.302651+0000 osd.7 (osd.7) 511 : cluster 3 failed to encode map e437 with expected crc 2023-10-01T11:19:12.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:12 smithi173 bash[58816]: cluster 2023-10-01T11:19:10.597615+0000 mgr.y (mgr.25072) 1859 : cluster 0 pgmap v2054: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:12.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:12 smithi173 bash[58816]: audit 2023-10-01T11:19:11.126022+0000 mon.a (mon.0) 2376 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-10-01T11:19:12.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:12 smithi173 bash[58816]: cluster 2023-10-01T11:19:11.126120+0000 mon.a (mon.0) 2377 : cluster 0 osdmap e437: 8 total, 8 up, 8 in 2023-10-01T11:19:12.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:12 smithi173 bash[58816]: cluster 2023-10-01T11:19:11.302427+0000 osd.6 (osd.6) 420 : cluster 3 failed to encode map e437 with expected crc 2023-10-01T11:19:12.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:12 smithi173 bash[58816]: cluster 2023-10-01T11:19:11.302651+0000 osd.7 (osd.7) 511 : cluster 3 failed to encode map e437 with expected crc 2023-10-01T11:19:13.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:13 smithi078 bash[75299]: cluster 2023-10-01T11:19:11.302402+0000 osd.5 (osd.5) 471 : cluster 3 failed to encode map e437 with expected crc 2023-10-01T11:19:13.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:13 smithi078 bash[78274]: cluster 2023-10-01T11:19:11.302402+0000 osd.5 (osd.5) 471 : cluster 3 failed to encode map e437 with expected crc 2023-10-01T11:19:13.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:13 smithi173 bash[58816]: cluster 2023-10-01T11:19:11.302402+0000 osd.5 (osd.5) 471 : cluster 3 failed to encode map e437 with expected crc 2023-10-01T11:19:14.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:14 smithi078 bash[75299]: cluster 2023-10-01T11:19:12.598285+0000 mgr.y (mgr.25072) 1860 : cluster 0 pgmap v2056: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:14.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:14 smithi078 bash[78274]: cluster 2023-10-01T11:19:12.598285+0000 mgr.y (mgr.25072) 1860 : cluster 0 pgmap v2056: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:14.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:14 smithi173 bash[58816]: cluster 2023-10-01T11:19:12.598285+0000 mgr.y (mgr.25072) 1860 : cluster 0 pgmap v2056: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:16.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:16 smithi078 bash[75299]: cluster 2023-10-01T11:19:14.599287+0000 mgr.y (mgr.25072) 1861 : cluster 0 pgmap v2057: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:16.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:16 smithi078 bash[78274]: cluster 2023-10-01T11:19:14.599287+0000 mgr.y (mgr.25072) 1861 : cluster 0 pgmap v2057: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:16.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:16 smithi173 bash[58816]: cluster 2023-10-01T11:19:14.599287+0000 mgr.y (mgr.25072) 1861 : cluster 0 pgmap v2057: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:16.837 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:19:18.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:18 smithi078 bash[78274]: cluster 2023-10-01T11:19:16.600037+0000 mgr.y (mgr.25072) 1862 : cluster 0 pgmap v2058: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:18.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:18 smithi078 bash[78274]: audit 2023-10-01T11:19:17.490013+0000 mon.a (mon.0) 2378 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:19:18.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:18 smithi078 bash[75299]: cluster 2023-10-01T11:19:16.600037+0000 mgr.y (mgr.25072) 1862 : cluster 0 pgmap v2058: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:18.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:18 smithi078 bash[75299]: audit 2023-10-01T11:19:17.490013+0000 mon.a (mon.0) 2378 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:19:18.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:18 smithi173 bash[58816]: cluster 2023-10-01T11:19:16.600037+0000 mgr.y (mgr.25072) 1862 : cluster 0 pgmap v2058: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:18.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:18 smithi173 bash[58816]: audit 2023-10-01T11:19:17.490013+0000 mon.a (mon.0) 2378 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:19:20.269 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:20 smithi078 bash[78274]: cluster 2023-10-01T11:19:18.600757+0000 mgr.y (mgr.25072) 1863 : cluster 0 pgmap v2059: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:20.634 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:20 smithi078 bash[75299]: cluster 2023-10-01T11:19:18.600757+0000 mgr.y (mgr.25072) 1863 : cluster 0 pgmap v2059: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:20.634 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:19:20 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:19:20] "GET /metrics HTTP/1.1" 200 34019 "" "Prometheus/2.33.4" 2023-10-01T11:19:20.635 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:20 smithi173 bash[58816]: cluster 2023-10-01T11:19:18.600757+0000 mgr.y (mgr.25072) 1863 : cluster 0 pgmap v2059: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:21.172 INFO:teuthology.orchestra.run.smithi078.stderr:set osd.7 primary-affinity to 1 (8655362) 2023-10-01T11:19:21.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:21 smithi078 bash[78274]: audit 2023-10-01T11:19:20.413781+0000 mon.c (mon.2) 268 : audit 1 from='client.? 172.21.15.78:0/3929253835' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T11:19:21.743 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:21 smithi078 bash[78274]: audit 2023-10-01T11:19:20.414537+0000 mon.a (mon.0) 2379 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T11:19:21.744 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:21 smithi078 bash[75299]: audit 2023-10-01T11:19:20.413781+0000 mon.c (mon.2) 268 : audit 1 from='client.? 172.21.15.78:0/3929253835' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T11:19:21.744 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:21 smithi078 bash[75299]: audit 2023-10-01T11:19:20.414537+0000 mon.a (mon.0) 2379 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T11:19:21.744 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:21 smithi173 bash[58816]: audit 2023-10-01T11:19:20.413781+0000 mon.c (mon.2) 268 : audit 1 from='client.? 172.21.15.78:0/3929253835' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T11:19:21.744 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:21 smithi173 bash[58816]: audit 2023-10-01T11:19:20.414537+0000 mon.a (mon.0) 2379 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-10-01T11:19:22.134 INFO:tasks.thrashosds.thrasher:joining the do_sighup greenlet 2023-10-01T11:19:22.138 INFO:tasks.thrashosds.thrasher:joining the do_optrack_toggle greenlet 2023-10-01T11:19:22.138 INFO:tasks.thrashosds.thrasher:joining the do_dump_ops greenlet 2023-10-01T11:19:22.138 INFO:tasks.thrashosds.thrasher:joining the do_noscrub_toggle greenlet 2023-10-01T11:19:22.138 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-10-01T11:19:22.138 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-01T11:19:22.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:22 smithi078 bash[78274]: cluster 2023-10-01T11:19:20.601765+0000 mgr.y (mgr.25072) 1864 : cluster 0 pgmap v2060: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:22.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:22 smithi078 bash[78274]: audit 2023-10-01T11:19:21.165367+0000 mon.a (mon.0) 2380 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-10-01T11:19:22.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:22 smithi078 bash[78274]: cluster 2023-10-01T11:19:21.165479+0000 mon.a (mon.0) 2381 : cluster 0 osdmap e438: 8 total, 8 up, 8 in 2023-10-01T11:19:22.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:22 smithi078 bash[78274]: cluster 2023-10-01T11:19:21.168151+0000 osd.7 (osd.7) 512 : cluster 3 failed to encode map e438 with expected crc 2023-10-01T11:19:22.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:22 smithi078 bash[75299]: cluster 2023-10-01T11:19:20.601765+0000 mgr.y (mgr.25072) 1864 : cluster 0 pgmap v2060: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:22.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:22 smithi078 bash[75299]: audit 2023-10-01T11:19:21.165367+0000 mon.a (mon.0) 2380 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-10-01T11:19:22.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:22 smithi078 bash[75299]: cluster 2023-10-01T11:19:21.165479+0000 mon.a (mon.0) 2381 : cluster 0 osdmap e438: 8 total, 8 up, 8 in 2023-10-01T11:19:22.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:22 smithi078 bash[75299]: cluster 2023-10-01T11:19:21.168151+0000 osd.7 (osd.7) 512 : cluster 3 failed to encode map e438 with expected crc 2023-10-01T11:19:22.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:22 smithi173 bash[58816]: cluster 2023-10-01T11:19:20.601765+0000 mgr.y (mgr.25072) 1864 : cluster 0 pgmap v2060: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:22.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:22 smithi173 bash[58816]: audit 2023-10-01T11:19:21.165367+0000 mon.a (mon.0) 2380 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-10-01T11:19:22.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:22 smithi173 bash[58816]: cluster 2023-10-01T11:19:21.165479+0000 mon.a (mon.0) 2381 : cluster 0 osdmap e438: 8 total, 8 up, 8 in 2023-10-01T11:19:22.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:22 smithi173 bash[58816]: cluster 2023-10-01T11:19:21.168151+0000 osd.7 (osd.7) 512 : cluster 3 failed to encode map e438 with expected crc 2023-10-01T11:19:23.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:23 smithi078 bash[75299]: cluster 2023-10-01T11:19:22.173870+0000 osd.7 (osd.7) 513 : cluster 3 failed to encode map e439 with expected crc 2023-10-01T11:19:23.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:23 smithi078 bash[75299]: cluster 2023-10-01T11:19:22.176304+0000 mon.a (mon.0) 2382 : cluster 0 osdmap e439: 8 total, 8 up, 8 in 2023-10-01T11:19:23.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:23 smithi078 bash[75299]: cluster 2023-10-01T11:19:22.176627+0000 osd.7 (osd.7) 514 : cluster 3 failed to encode map e439 with expected crc 2023-10-01T11:19:23.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:23 smithi078 bash[75299]: audit 2023-10-01T11:19:22.571283+0000 mon.a (mon.0) 2383 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T11:19:23.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:23 smithi078 bash[78274]: cluster 2023-10-01T11:19:22.173870+0000 osd.7 (osd.7) 513 : cluster 3 failed to encode map e439 with expected crc 2023-10-01T11:19:23.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:23 smithi078 bash[78274]: cluster 2023-10-01T11:19:22.176304+0000 mon.a (mon.0) 2382 : cluster 0 osdmap e439: 8 total, 8 up, 8 in 2023-10-01T11:19:23.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:23 smithi078 bash[78274]: cluster 2023-10-01T11:19:22.176627+0000 osd.7 (osd.7) 514 : cluster 3 failed to encode map e439 with expected crc 2023-10-01T11:19:23.583 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:23 smithi078 bash[78274]: audit 2023-10-01T11:19:22.571283+0000 mon.a (mon.0) 2383 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T11:19:23.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:23 smithi173 bash[58816]: cluster 2023-10-01T11:19:22.173870+0000 osd.7 (osd.7) 513 : cluster 3 failed to encode map e439 with expected crc 2023-10-01T11:19:23.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:23 smithi173 bash[58816]: cluster 2023-10-01T11:19:22.176304+0000 mon.a (mon.0) 2382 : cluster 0 osdmap e439: 8 total, 8 up, 8 in 2023-10-01T11:19:23.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:23 smithi173 bash[58816]: cluster 2023-10-01T11:19:22.176627+0000 osd.7 (osd.7) 514 : cluster 3 failed to encode map e439 with expected crc 2023-10-01T11:19:23.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:23 smithi173 bash[58816]: audit 2023-10-01T11:19:22.571283+0000 mon.a (mon.0) 2383 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T11:19:24.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:24 smithi078 bash[78274]: cluster 2023-10-01T11:19:22.602260+0000 mgr.y (mgr.25072) 1865 : cluster 0 pgmap v2063: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:24.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:24 smithi078 bash[75299]: cluster 2023-10-01T11:19:22.602260+0000 mgr.y (mgr.25072) 1865 : cluster 0 pgmap v2063: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:24.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:24 smithi173 bash[58816]: cluster 2023-10-01T11:19:22.602260+0000 mgr.y (mgr.25072) 1865 : cluster 0 pgmap v2063: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:26.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:26 smithi078 bash[75299]: cluster 2023-10-01T11:19:24.603279+0000 mgr.y (mgr.25072) 1866 : cluster 0 pgmap v2064: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-10-01T11:19:26.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:26 smithi078 bash[78274]: cluster 2023-10-01T11:19:24.603279+0000 mgr.y (mgr.25072) 1866 : cluster 0 pgmap v2064: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-10-01T11:19:26.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:26 smithi173 bash[58816]: cluster 2023-10-01T11:19:24.603279+0000 mgr.y (mgr.25072) 1866 : cluster 0 pgmap v2064: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-10-01T11:19:26.923 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:19:28.470 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:28 smithi078 bash[75299]: cluster 2023-10-01T11:19:26.604015+0000 mgr.y (mgr.25072) 1867 : cluster 0 pgmap v2065: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-10-01T11:19:28.470 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:28 smithi078 bash[75299]: audit 2023-10-01T11:19:28.116889+0000 mon.a (mon.0) 2384 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:19:28.470 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:28 smithi078 bash[75299]: audit 2023-10-01T11:19:28.123721+0000 mon.a (mon.0) 2385 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:19:28.470 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:28 smithi078 bash[78274]: cluster 2023-10-01T11:19:26.604015+0000 mgr.y (mgr.25072) 1867 : cluster 0 pgmap v2065: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-10-01T11:19:28.470 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:28 smithi078 bash[78274]: audit 2023-10-01T11:19:28.116889+0000 mon.a (mon.0) 2384 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:19:28.470 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:28 smithi078 bash[78274]: audit 2023-10-01T11:19:28.123721+0000 mon.a (mon.0) 2385 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:19:28.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:28 smithi173 bash[58816]: cluster 2023-10-01T11:19:26.604015+0000 mgr.y (mgr.25072) 1867 : cluster 0 pgmap v2065: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-10-01T11:19:28.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:28 smithi173 bash[58816]: audit 2023-10-01T11:19:28.116889+0000 mon.a (mon.0) 2384 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:19:28.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:28 smithi173 bash[58816]: audit 2023-10-01T11:19:28.123721+0000 mon.a (mon.0) 2385 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:19:30.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:29 smithi078 bash[78274]: cluster 2023-10-01T11:19:28.604762+0000 mgr.y (mgr.25072) 1868 : cluster 0 pgmap v2066: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-10-01T11:19:30.141 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:29 smithi078 bash[78274]: audit 2023-10-01T11:19:28.821709+0000 mon.a (mon.0) 2386 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:19:30.141 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:29 smithi078 bash[78274]: audit 2023-10-01T11:19:28.829802+0000 mon.a (mon.0) 2387 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:19:30.141 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:29 smithi078 bash[78274]: audit 2023-10-01T11:19:29.355881+0000 mon.a (mon.0) 2388 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T11:19:30.141 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:29 smithi078 bash[78274]: audit 2023-10-01T11:19:29.357155+0000 mon.a (mon.0) 2389 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T11:19:30.141 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:29 smithi078 bash[78274]: audit 2023-10-01T11:19:29.365335+0000 mon.a (mon.0) 2390 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:19:30.141 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:29 smithi078 bash[75299]: cluster 2023-10-01T11:19:28.604762+0000 mgr.y (mgr.25072) 1868 : cluster 0 pgmap v2066: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-10-01T11:19:30.141 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:29 smithi078 bash[75299]: audit 2023-10-01T11:19:28.821709+0000 mon.a (mon.0) 2386 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:19:30.142 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:29 smithi078 bash[75299]: audit 2023-10-01T11:19:28.829802+0000 mon.a (mon.0) 2387 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:19:30.142 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:29 smithi078 bash[75299]: audit 2023-10-01T11:19:29.355881+0000 mon.a (mon.0) 2388 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T11:19:30.142 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:29 smithi078 bash[75299]: audit 2023-10-01T11:19:29.357155+0000 mon.a (mon.0) 2389 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T11:19:30.142 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:29 smithi078 bash[75299]: audit 2023-10-01T11:19:29.365335+0000 mon.a (mon.0) 2390 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:19:30.142 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:29 smithi173 bash[58816]: cluster 2023-10-01T11:19:28.604762+0000 mgr.y (mgr.25072) 1868 : cluster 0 pgmap v2066: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-10-01T11:19:30.143 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:29 smithi173 bash[58816]: audit 2023-10-01T11:19:28.821709+0000 mon.a (mon.0) 2386 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:19:30.143 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:29 smithi173 bash[58816]: audit 2023-10-01T11:19:28.829802+0000 mon.a (mon.0) 2387 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:19:30.143 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:29 smithi173 bash[58816]: audit 2023-10-01T11:19:29.355881+0000 mon.a (mon.0) 2388 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T11:19:30.143 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:29 smithi173 bash[58816]: audit 2023-10-01T11:19:29.357155+0000 mon.a (mon.0) 2389 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T11:19:30.143 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:29 smithi173 bash[58816]: audit 2023-10-01T11:19:29.365335+0000 mon.a (mon.0) 2390 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:19:30.577 INFO:teuthology.orchestra.run.smithi078.stdout: 2023-10-01T11:19:30.577 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":439,"fsid":"d8016480-6041-11ee-8db5-212e2dc638e7","created":"2023-10-01T10:04:59.345633+0000","modified":"2023-10-01T11:19:22.162185+0000","last_up_change":"2023-10-01T10:38:15.000941+0000","last_in_change":"2023-10-01T11:16:17.942848+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":23,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":15,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-01T10:09:16.689172+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"70","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{},"mgr_devicehealth":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-01T10:13:43.595790+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"404","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":404,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-01T10:15:38.225217+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-01T10:15:38.676998+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"3cdf50f0-cae3-4e38-a7ca-c24c4a07b615","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":82,"up_from":86,"up_thru":421,"down_at":83,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6802","nonce":2878175461},{"type":"v1","addr":"172.21.15.78:6803","nonce":2878175461}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6804","nonce":2878175461},{"type":"v1","addr":"172.21.15.78:6805","nonce":2878175461}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6808","nonce":2878175461},{"type":"v1","addr":"172.21.15.78:6809","nonce":2878175461}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6806","nonce":2878175461},{"type":"v1","addr":"172.21.15.78:6807","nonce":2878175461}]},"public_addr":"172.21.15.78:6803/2878175461","cluster_addr":"172.21.15.78:6805/2878175461","heartbeat_back_addr":"172.21.15.78:6809/2878175461","heartbeat_front_addr":"172.21.15.78:6807/2878175461","state":["exists","up"]},{"osd":1,"uuid":"4e6bea65-4109-43d2-b5be-3811fdd2a803","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":96,"up_from":101,"up_thru":424,"down_at":97,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6810","nonce":1115426690},{"type":"v1","addr":"172.21.15.78:6811","nonce":1115426690}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6812","nonce":1115426690},{"type":"v1","addr":"172.21.15.78:6813","nonce":1115426690}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6816","nonce":1115426690},{"type":"v1","addr":"172.21.15.78:6817","nonce":1115426690}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6814","nonce":1115426690},{"type":"v1","addr":"172.21.15.78:6815","nonce":1115426690}]},"public_addr":"172.21.15.78:6811/1115426690","cluster_addr":"172.21.15.78:6813/1115426690","heartbeat_back_addr":"172.21.15.78:6817/1115426690","heartbeat_front_addr":"172.21.15.78:6815/1115426690","state":["exists","up"]},{"osd":2,"uuid":"9f830594-31d6-4994-85fa-0615c280d5c3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":112,"up_from":116,"up_thru":414,"down_at":113,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6818","nonce":4228246677},{"type":"v1","addr":"172.21.15.78:6819","nonce":4228246677}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6820","nonce":4228246677},{"type":"v1","addr":"172.21.15.78:6821","nonce":4228246677}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6824","nonce":4228246677},{"type":"v1","addr":"172.21.15.78:6825","nonce":4228246677}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6822","nonce":4228246677},{"type":"v1","addr":"172.21.15.78:6823","nonce":4228246677}]},"public_addr":"172.21.15.78:6819/4228246677","cluster_addr":"172.21.15.78:6821/4228246677","heartbeat_back_addr":"172.21.15.78:6825/4228246677","heartbeat_front_addr":"172.21.15.78:6823/4228246677","state":["exists","up"]},{"osd":3,"uuid":"2780842b-2f4f-4190-b4a5-279a0444adf2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":117,"up_from":121,"up_thru":414,"down_at":118,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6826","nonce":2365141372},{"type":"v1","addr":"172.21.15.78:6827","nonce":2365141372}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6828","nonce":2365141372},{"type":"v1","addr":"172.21.15.78:6829","nonce":2365141372}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6832","nonce":2365141372},{"type":"v1","addr":"172.21.15.78:6833","nonce":2365141372}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6830","nonce":2365141372},{"type":"v1","addr":"172.21.15.78:6831","nonce":2365141372}]},"public_addr":"172.21.15.78:6827/2365141372","cluster_addr":"172.21.15.78:6829/2365141372","heartbeat_back_addr":"172.21.15.78:6833/2365141372","heartbeat_front_addr":"172.21.15.78:6831/2365141372","state":["exists","up"]},{"osd":4,"uuid":"57c30778-fd7e-4f3e-afd9-389ef377cb62","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":30,"last_clean_end":122,"up_from":126,"up_thru":414,"down_at":123,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6800","nonce":3220463704},{"type":"v1","addr":"172.21.15.173:6801","nonce":3220463704}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6802","nonce":3220463704},{"type":"v1","addr":"172.21.15.173:6803","nonce":3220463704}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6806","nonce":3220463704},{"type":"v1","addr":"172.21.15.173:6807","nonce":3220463704}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6804","nonce":3220463704},{"type":"v1","addr":"172.21.15.173:6805","nonce":3220463704}]},"public_addr":"172.21.15.173:6801/3220463704","cluster_addr":"172.21.15.173:6803/3220463704","heartbeat_back_addr":"172.21.15.173:6807/3220463704","heartbeat_front_addr":"172.21.15.173:6805/3220463704","state":["exists","up"]},{"osd":5,"uuid":"c62aa7c3-2d09-4088-ba36-c84588dc3549","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":414,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6808","nonce":2798643181},{"type":"v1","addr":"172.21.15.173:6809","nonce":2798643181}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6810","nonce":2798643181},{"type":"v1","addr":"172.21.15.173:6811","nonce":2798643181}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6814","nonce":2798643181},{"type":"v1","addr":"172.21.15.173:6815","nonce":2798643181}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6812","nonce":2798643181},{"type":"v1","addr":"172.21.15.173:6813","nonce":2798643181}]},"public_addr":"172.21.15.173:6809/2798643181","cluster_addr":"172.21.15.173:6811/2798643181","heartbeat_back_addr":"172.21.15.173:6815/2798643181","heartbeat_front_addr":"172.21.15.173:6813/2798643181","state":["exists","up"]},{"osd":6,"uuid":"b140330d-dd93-4ce4-9b9a-2ec6321419b3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":435,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6816","nonce":1568190351},{"type":"v1","addr":"172.21.15.173:6817","nonce":1568190351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6818","nonce":1568190351},{"type":"v1","addr":"172.21.15.173:6819","nonce":1568190351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6822","nonce":1568190351},{"type":"v1","addr":"172.21.15.173:6823","nonce":1568190351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6820","nonce":1568190351},{"type":"v1","addr":"172.21.15.173:6821","nonce":1568190351}]},"public_addr":"172.21.15.173:6817/1568190351","cluster_addr":"172.21.15.173:6819/1568190351","heartbeat_back_addr":"172.21.15.173:6823/1568190351","heartbeat_front_addr":"172.21.15.173:6821/1568190351","state":["exists","up"]},{"osd":7,"uuid":"7590027f-50c4-4457-843d-cebaa734d2ea","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":438,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6824","nonce":1958390745},{"type":"v1","addr":"172.21.15.173:6825","nonce":1958390745}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6826","nonce":1958390745},{"type":"v1","addr":"172.21.15.173:6827","nonce":1958390745}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6830","nonce":1958390745},{"type":"v1","addr":"172.21.15.173:6831","nonce":1958390745}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6828","nonce":1958390745},{"type":"v1","addr":"172.21.15.173:6829","nonce":1958390745}]},"public_addr":"172.21.15.173:6825/1958390745","cluster_addr":"172.21.15.173:6827/1958390745","heartbeat_back_addr":"172.21.15.173:6831/1958390745","heartbeat_front_addr":"172.21.15.173:6829/1958390745","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:07:47.950416+0000","dead_epoch":82},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:08:31.368397+0000","dead_epoch":96},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:09:14.866866+0000","dead_epoch":112},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:09:59.445542+0000","dead_epoch":117},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:10:39.550033+0000","dead_epoch":122},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:11:20.934732+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:12:01.654044+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:12:43.447412+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.d","mappings":[{"from":1,"to":5}]},{"pgid":"4.f","mappings":[{"from":1,"to":2}]},{"pgid":"4.14","mappings":[{"from":1,"to":2}]},{"pgid":"4.15","mappings":[{"from":3,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.78:0/3939471097":"2023-10-02T10:05:57.727322+0000","172.21.15.78:0/590730483":"2023-10-02T10:22:47.233952+0000","172.21.15.78:6800/1700562157":"2023-10-02T10:05:57.727322+0000","172.21.15.78:6801/1700562157":"2023-10-02T10:05:57.727322+0000","172.21.15.78:0/3635389419":"2023-10-02T10:05:57.727322+0000","172.21.15.78:0/3335202251":"2023-10-02T10:22:47.233952+0000","172.21.15.78:0/308254153":"2023-10-02T10:05:27.007550+0000","172.21.15.173:6834/1928273846":"2023-10-02T10:21:31.381928+0000","172.21.15.173:0/4047131805":"2023-10-02T10:21:31.381928+0000","172.21.15.78:6801/1224621968":"2023-10-02T10:13:17.135564+0000","172.21.15.78:0/158328292":"2023-10-02T10:05:27.007550+0000","172.21.15.78:0/1297753620":"2023-10-02T10:22:47.233952+0000","172.21.15.78:0/1585217033":"2023-10-02T10:22:47.233952+0000","172.21.15.173:6835/1928273846":"2023-10-02T10:21:31.381928+0000","172.21.15.173:0/4059345053":"2023-10-02T10:21:31.381928+0000","172.21.15.78:6800/516444166":"2023-10-02T10:19:23.134044+0000","172.21.15.78:0/2751026363":"2023-10-02T10:22:47.233952+0000","172.21.15.78:0/702157702":"2023-10-02T10:19:23.134044+0000","172.21.15.78:6801/516444166":"2023-10-02T10:19:23.134044+0000","172.21.15.78:6801/449971457":"2023-10-02T10:22:47.233952+0000","172.21.15.78:0/3274439700":"2023-10-02T10:19:23.134044+0000","172.21.15.173:0/2382630687":"2023-10-02T10:21:31.381928+0000","172.21.15.78:0/3655343641":"2023-10-02T10:13:17.135564+0000","172.21.15.78:6800/3027286102":"2023-10-02T10:05:27.007550+0000","172.21.15.78:0/2359078223":"2023-10-02T10:13:17.135564+0000","172.21.15.78:6801/3027286102":"2023-10-02T10:05:27.007550+0000","172.21.15.78:0/3611942206":"2023-10-02T10:22:47.233952+0000","172.21.15.173:0/1462478964":"2023-10-02T10:21:31.381928+0000","172.21.15.78:6800/449971457":"2023-10-02T10:22:47.233952+0000","172.21.15.78:6800/1224621968":"2023-10-02T10:13:17.135564+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":15,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-01T11:19:30.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:19:30 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:19:30] "GET /metrics HTTP/1.1" 200 34019 "" "Prometheus/2.33.4" 2023-10-01T11:19:31.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:30 smithi078 bash[75299]: audit 2023-10-01T11:19:30.573125+0000 mon.c (mon.2) 269 : audit 0 from='client.? 172.21.15.78:0/3579752058' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T11:19:31.081 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:30 smithi078 bash[78274]: audit 2023-10-01T11:19:30.573125+0000 mon.c (mon.2) 269 : audit 0 from='client.? 172.21.15.78:0/3579752058' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T11:19:31.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:30 smithi173 bash[58816]: audit 2023-10-01T11:19:30.573125+0000 mon.c (mon.2) 269 : audit 0 from='client.? 172.21.15.78:0/3579752058' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T11:19:31.644 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-10-01T11:19:31.645 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd dump --format=json 2023-10-01T11:19:32.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:31 smithi078 bash[75299]: cluster 2023-10-01T11:19:30.605546+0000 mgr.y (mgr.25072) 1869 : cluster 0 pgmap v2067: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 61 KiB/s, 0 objects/s recovering 2023-10-01T11:19:32.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:31 smithi078 bash[78274]: cluster 2023-10-01T11:19:30.605546+0000 mgr.y (mgr.25072) 1869 : cluster 0 pgmap v2067: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 61 KiB/s, 0 objects/s recovering 2023-10-01T11:19:32.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:31 smithi173 bash[58816]: cluster 2023-10-01T11:19:30.605546+0000 mgr.y (mgr.25072) 1869 : cluster 0 pgmap v2067: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 61 KiB/s, 0 objects/s recovering 2023-10-01T11:19:33.119 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:32 smithi173 bash[58816]: audit 2023-10-01T11:19:32.490536+0000 mon.a (mon.0) 2391 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:19:33.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:32 smithi078 bash[75299]: audit 2023-10-01T11:19:32.490536+0000 mon.a (mon.0) 2391 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:19:33.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:32 smithi078 bash[78274]: audit 2023-10-01T11:19:32.490536+0000 mon.a (mon.0) 2391 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:19:34.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:33 smithi078 bash[75299]: cluster 2023-10-01T11:19:32.606194+0000 mgr.y (mgr.25072) 1870 : cluster 0 pgmap v2068: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 55 KiB/s, 0 objects/s recovering 2023-10-01T11:19:34.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:33 smithi078 bash[78274]: cluster 2023-10-01T11:19:32.606194+0000 mgr.y (mgr.25072) 1870 : cluster 0 pgmap v2068: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 55 KiB/s, 0 objects/s recovering 2023-10-01T11:19:34.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:33 smithi173 bash[58816]: cluster 2023-10-01T11:19:32.606194+0000 mgr.y (mgr.25072) 1870 : cluster 0 pgmap v2068: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 55 KiB/s, 0 objects/s recovering 2023-10-01T11:19:36.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:35 smithi078 bash[75299]: cluster 2023-10-01T11:19:34.607280+0000 mgr.y (mgr.25072) 1871 : cluster 0 pgmap v2069: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-10-01T11:19:36.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:35 smithi078 bash[78274]: cluster 2023-10-01T11:19:34.607280+0000 mgr.y (mgr.25072) 1871 : cluster 0 pgmap v2069: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-10-01T11:19:36.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:35 smithi173 bash[58816]: cluster 2023-10-01T11:19:34.607280+0000 mgr.y (mgr.25072) 1871 : cluster 0 pgmap v2069: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-10-01T11:19:36.427 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:19:38.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:37 smithi078 bash[75299]: cluster 2023-10-01T11:19:36.608001+0000 mgr.y (mgr.25072) 1872 : cluster 0 pgmap v2070: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:38.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:37 smithi078 bash[78274]: cluster 2023-10-01T11:19:36.608001+0000 mgr.y (mgr.25072) 1872 : cluster 0 pgmap v2070: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:38.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:37 smithi173 bash[58816]: cluster 2023-10-01T11:19:36.608001+0000 mgr.y (mgr.25072) 1872 : cluster 0 pgmap v2070: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:40.128 INFO:teuthology.orchestra.run.smithi078.stdout: 2023-10-01T11:19:40.128 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":439,"fsid":"d8016480-6041-11ee-8db5-212e2dc638e7","created":"2023-10-01T10:04:59.345633+0000","modified":"2023-10-01T11:19:22.162185+0000","last_up_change":"2023-10-01T10:38:15.000941+0000","last_in_change":"2023-10-01T11:16:17.942848+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":23,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":15,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"pacific","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-10-01T10:09:16.689172+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"70","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{},"mgr_devicehealth":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-10-01T10:13:43.595790+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"404","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":404,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-10-01T10:15:38.225217+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-10-01T10:15:38.676998+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"60","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"3cdf50f0-cae3-4e38-a7ca-c24c4a07b615","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":82,"up_from":86,"up_thru":421,"down_at":83,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6802","nonce":2878175461},{"type":"v1","addr":"172.21.15.78:6803","nonce":2878175461}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6804","nonce":2878175461},{"type":"v1","addr":"172.21.15.78:6805","nonce":2878175461}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6808","nonce":2878175461},{"type":"v1","addr":"172.21.15.78:6809","nonce":2878175461}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6806","nonce":2878175461},{"type":"v1","addr":"172.21.15.78:6807","nonce":2878175461}]},"public_addr":"172.21.15.78:6803/2878175461","cluster_addr":"172.21.15.78:6805/2878175461","heartbeat_back_addr":"172.21.15.78:6809/2878175461","heartbeat_front_addr":"172.21.15.78:6807/2878175461","state":["exists","up"]},{"osd":1,"uuid":"4e6bea65-4109-43d2-b5be-3811fdd2a803","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":96,"up_from":101,"up_thru":424,"down_at":97,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6810","nonce":1115426690},{"type":"v1","addr":"172.21.15.78:6811","nonce":1115426690}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6812","nonce":1115426690},{"type":"v1","addr":"172.21.15.78:6813","nonce":1115426690}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6816","nonce":1115426690},{"type":"v1","addr":"172.21.15.78:6817","nonce":1115426690}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6814","nonce":1115426690},{"type":"v1","addr":"172.21.15.78:6815","nonce":1115426690}]},"public_addr":"172.21.15.78:6811/1115426690","cluster_addr":"172.21.15.78:6813/1115426690","heartbeat_back_addr":"172.21.15.78:6817/1115426690","heartbeat_front_addr":"172.21.15.78:6815/1115426690","state":["exists","up"]},{"osd":2,"uuid":"9f830594-31d6-4994-85fa-0615c280d5c3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":112,"up_from":116,"up_thru":414,"down_at":113,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6818","nonce":4228246677},{"type":"v1","addr":"172.21.15.78:6819","nonce":4228246677}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6820","nonce":4228246677},{"type":"v1","addr":"172.21.15.78:6821","nonce":4228246677}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6824","nonce":4228246677},{"type":"v1","addr":"172.21.15.78:6825","nonce":4228246677}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6822","nonce":4228246677},{"type":"v1","addr":"172.21.15.78:6823","nonce":4228246677}]},"public_addr":"172.21.15.78:6819/4228246677","cluster_addr":"172.21.15.78:6821/4228246677","heartbeat_back_addr":"172.21.15.78:6825/4228246677","heartbeat_front_addr":"172.21.15.78:6823/4228246677","state":["exists","up"]},{"osd":3,"uuid":"2780842b-2f4f-4190-b4a5-279a0444adf2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":117,"up_from":121,"up_thru":414,"down_at":118,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6826","nonce":2365141372},{"type":"v1","addr":"172.21.15.78:6827","nonce":2365141372}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6828","nonce":2365141372},{"type":"v1","addr":"172.21.15.78:6829","nonce":2365141372}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6832","nonce":2365141372},{"type":"v1","addr":"172.21.15.78:6833","nonce":2365141372}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6830","nonce":2365141372},{"type":"v1","addr":"172.21.15.78:6831","nonce":2365141372}]},"public_addr":"172.21.15.78:6827/2365141372","cluster_addr":"172.21.15.78:6829/2365141372","heartbeat_back_addr":"172.21.15.78:6833/2365141372","heartbeat_front_addr":"172.21.15.78:6831/2365141372","state":["exists","up"]},{"osd":4,"uuid":"57c30778-fd7e-4f3e-afd9-389ef377cb62","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":30,"last_clean_end":122,"up_from":126,"up_thru":414,"down_at":123,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6800","nonce":3220463704},{"type":"v1","addr":"172.21.15.173:6801","nonce":3220463704}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6802","nonce":3220463704},{"type":"v1","addr":"172.21.15.173:6803","nonce":3220463704}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6806","nonce":3220463704},{"type":"v1","addr":"172.21.15.173:6807","nonce":3220463704}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6804","nonce":3220463704},{"type":"v1","addr":"172.21.15.173:6805","nonce":3220463704}]},"public_addr":"172.21.15.173:6801/3220463704","cluster_addr":"172.21.15.173:6803/3220463704","heartbeat_back_addr":"172.21.15.173:6807/3220463704","heartbeat_front_addr":"172.21.15.173:6805/3220463704","state":["exists","up"]},{"osd":5,"uuid":"c62aa7c3-2d09-4088-ba36-c84588dc3549","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":414,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6808","nonce":2798643181},{"type":"v1","addr":"172.21.15.173:6809","nonce":2798643181}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6810","nonce":2798643181},{"type":"v1","addr":"172.21.15.173:6811","nonce":2798643181}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6814","nonce":2798643181},{"type":"v1","addr":"172.21.15.173:6815","nonce":2798643181}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6812","nonce":2798643181},{"type":"v1","addr":"172.21.15.173:6813","nonce":2798643181}]},"public_addr":"172.21.15.173:6809/2798643181","cluster_addr":"172.21.15.173:6811/2798643181","heartbeat_back_addr":"172.21.15.173:6815/2798643181","heartbeat_front_addr":"172.21.15.173:6813/2798643181","state":["exists","up"]},{"osd":6,"uuid":"b140330d-dd93-4ce4-9b9a-2ec6321419b3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":435,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6816","nonce":1568190351},{"type":"v1","addr":"172.21.15.173:6817","nonce":1568190351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6818","nonce":1568190351},{"type":"v1","addr":"172.21.15.173:6819","nonce":1568190351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6822","nonce":1568190351},{"type":"v1","addr":"172.21.15.173:6823","nonce":1568190351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6820","nonce":1568190351},{"type":"v1","addr":"172.21.15.173:6821","nonce":1568190351}]},"public_addr":"172.21.15.173:6817/1568190351","cluster_addr":"172.21.15.173:6819/1568190351","heartbeat_back_addr":"172.21.15.173:6823/1568190351","heartbeat_front_addr":"172.21.15.173:6821/1568190351","state":["exists","up"]},{"osd":7,"uuid":"7590027f-50c4-4457-843d-cebaa734d2ea","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":438,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6824","nonce":1958390745},{"type":"v1","addr":"172.21.15.173:6825","nonce":1958390745}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6826","nonce":1958390745},{"type":"v1","addr":"172.21.15.173:6827","nonce":1958390745}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6830","nonce":1958390745},{"type":"v1","addr":"172.21.15.173:6831","nonce":1958390745}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.173:6828","nonce":1958390745},{"type":"v1","addr":"172.21.15.173:6829","nonce":1958390745}]},"public_addr":"172.21.15.173:6825/1958390745","cluster_addr":"172.21.15.173:6827/1958390745","heartbeat_back_addr":"172.21.15.173:6831/1958390745","heartbeat_front_addr":"172.21.15.173:6829/1958390745","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:07:47.950416+0000","dead_epoch":82},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:08:31.368397+0000","dead_epoch":96},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:09:14.866866+0000","dead_epoch":112},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:09:59.445542+0000","dead_epoch":117},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:10:39.550033+0000","dead_epoch":122},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:11:20.934732+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:12:01.654044+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138314316775423,"old_weight":0,"last_purged_snaps_scrub":"2023-10-01T10:12:43.447412+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.d","mappings":[{"from":1,"to":5}]},{"pgid":"4.f","mappings":[{"from":1,"to":2}]},{"pgid":"4.14","mappings":[{"from":1,"to":2}]},{"pgid":"4.15","mappings":[{"from":3,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.78:0/590730483":"2023-10-02T10:22:47.233952+0000","172.21.15.78:0/3335202251":"2023-10-02T10:22:47.233952+0000","172.21.15.78:0/1585217033":"2023-10-02T10:22:47.233952+0000","172.21.15.78:6801/449971457":"2023-10-02T10:22:47.233952+0000","172.21.15.78:0/3939471097":"2023-10-02T10:05:57.727322+0000","172.21.15.78:6800/1700562157":"2023-10-02T10:05:57.727322+0000","172.21.15.78:6801/1700562157":"2023-10-02T10:05:57.727322+0000","172.21.15.78:0/3635389419":"2023-10-02T10:05:57.727322+0000","172.21.15.78:0/1297753620":"2023-10-02T10:22:47.233952+0000","172.21.15.78:0/158328292":"2023-10-02T10:05:27.007550+0000","172.21.15.78:0/308254153":"2023-10-02T10:05:27.007550+0000","172.21.15.173:0/2382630687":"2023-10-02T10:21:31.381928+0000","172.21.15.78:0/3274439700":"2023-10-02T10:19:23.134044+0000","172.21.15.78:0/2751026363":"2023-10-02T10:22:47.233952+0000","172.21.15.78:0/702157702":"2023-10-02T10:19:23.134044+0000","172.21.15.78:0/3655343641":"2023-10-02T10:13:17.135564+0000","172.21.15.78:6800/3027286102":"2023-10-02T10:05:27.007550+0000","172.21.15.78:6801/516444166":"2023-10-02T10:19:23.134044+0000","172.21.15.78:0/2359078223":"2023-10-02T10:13:17.135564+0000","172.21.15.78:6801/3027286102":"2023-10-02T10:05:27.007550+0000","172.21.15.173:0/4047131805":"2023-10-02T10:21:31.381928+0000","172.21.15.78:0/3611942206":"2023-10-02T10:22:47.233952+0000","172.21.15.173:0/1462478964":"2023-10-02T10:21:31.381928+0000","172.21.15.78:6800/449971457":"2023-10-02T10:22:47.233952+0000","172.21.15.78:6800/1224621968":"2023-10-02T10:13:17.135564+0000","172.21.15.78:6801/1224621968":"2023-10-02T10:13:17.135564+0000","172.21.15.173:6834/1928273846":"2023-10-02T10:21:31.381928+0000","172.21.15.78:6800/516444166":"2023-10-02T10:19:23.134044+0000","172.21.15.173:0/4059345053":"2023-10-02T10:21:31.381928+0000","172.21.15.173:6835/1928273846":"2023-10-02T10:21:31.381928+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":15,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-10-01T11:19:40.268 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:40 smithi078 bash[78274]: cluster 2023-10-01T11:19:38.608516+0000 mgr.y (mgr.25072) 1873 : cluster 0 pgmap v2071: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:40.269 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:39 smithi078 bash[75299]: cluster 2023-10-01T11:19:38.608516+0000 mgr.y (mgr.25072) 1873 : cluster 0 pgmap v2071: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:40.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:40 smithi173 bash[58816]: cluster 2023-10-01T11:19:38.608516+0000 mgr.y (mgr.25072) 1873 : cluster 0 pgmap v2071: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:40.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:19:40 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:19:40] "GET /metrics HTTP/1.1" 200 34028 "" "Prometheus/2.33.4" 2023-10-01T11:19:41.214 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-10-01T11:19:41.218 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-10-01T11:19:41.218 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-10-01T11:19:41.219 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-10-01T11:19:41.219 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-10-01T11:19:41.220 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-10-01T11:19:41.220 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-10-01T11:19:41.221 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-10-01T11:19:41.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:40 smithi078 bash[75299]: audit 2023-10-01T11:19:40.123654+0000 mon.a (mon.0) 2392 : audit 0 from='client.? 172.21.15.78:0/3246236831' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T11:19:41.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:41 smithi078 bash[78274]: audit 2023-10-01T11:19:40.123654+0000 mon.a (mon.0) 2392 : audit 0 from='client.? 172.21.15.78:0/3246236831' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T11:19:41.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:41 smithi173 bash[58816]: audit 2023-10-01T11:19:40.123654+0000 mon.a (mon.0) 2392 : audit 0 from='client.? 172.21.15.78:0/3246236831' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-10-01T11:19:42.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:42 smithi078 bash[75299]: cluster 2023-10-01T11:19:40.609450+0000 mgr.y (mgr.25072) 1874 : cluster 0 pgmap v2072: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:42.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:42 smithi078 bash[78274]: cluster 2023-10-01T11:19:40.609450+0000 mgr.y (mgr.25072) 1874 : cluster 0 pgmap v2072: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:42.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:42 smithi173 bash[58816]: cluster 2023-10-01T11:19:40.609450+0000 mgr.y (mgr.25072) 1874 : cluster 0 pgmap v2072: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:44.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:44 smithi078 bash[75299]: cluster 2023-10-01T11:19:42.610181+0000 mgr.y (mgr.25072) 1875 : cluster 0 pgmap v2073: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:44.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:44 smithi078 bash[78274]: cluster 2023-10-01T11:19:42.610181+0000 mgr.y (mgr.25072) 1875 : cluster 0 pgmap v2073: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:44.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:44 smithi173 bash[58816]: cluster 2023-10-01T11:19:42.610181+0000 mgr.y (mgr.25072) 1875 : cluster 0 pgmap v2073: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:46.098 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:19:46.099 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:19:46.099 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:19:46.100 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:19:46.100 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:19:46.105 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:19:46.106 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:19:46.107 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:19:46.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:46 smithi078 bash[75299]: cluster 2023-10-01T11:19:44.611234+0000 mgr.y (mgr.25072) 1876 : cluster 0 pgmap v2074: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:46.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:46 smithi078 bash[78274]: cluster 2023-10-01T11:19:44.611234+0000 mgr.y (mgr.25072) 1876 : cluster 0 pgmap v2074: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:46.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:46 smithi173 bash[58816]: cluster 2023-10-01T11:19:44.611234+0000 mgr.y (mgr.25072) 1876 : cluster 0 pgmap v2074: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:48.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:48 smithi078 bash[75299]: cluster 2023-10-01T11:19:46.611822+0000 mgr.y (mgr.25072) 1877 : cluster 0 pgmap v2075: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:48.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:48 smithi078 bash[75299]: audit 2023-10-01T11:19:47.490873+0000 mon.a (mon.0) 2393 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:19:48.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:48 smithi078 bash[78274]: cluster 2023-10-01T11:19:46.611822+0000 mgr.y (mgr.25072) 1877 : cluster 0 pgmap v2075: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:48.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:48 smithi078 bash[78274]: audit 2023-10-01T11:19:47.490873+0000 mon.a (mon.0) 2393 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:19:48.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:48 smithi173 bash[58816]: cluster 2023-10-01T11:19:46.611822+0000 mgr.y (mgr.25072) 1877 : cluster 0 pgmap v2075: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:48.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:48 smithi173 bash[58816]: audit 2023-10-01T11:19:47.490873+0000 mon.a (mon.0) 2393 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:19:50.270 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:50 smithi078 bash[78274]: cluster 2023-10-01T11:19:48.612593+0000 mgr.y (mgr.25072) 1878 : cluster 0 pgmap v2076: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:50.270 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:50 smithi078 bash[75299]: cluster 2023-10-01T11:19:48.612593+0000 mgr.y (mgr.25072) 1878 : cluster 0 pgmap v2076: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:50.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:50 smithi173 bash[58816]: cluster 2023-10-01T11:19:48.612593+0000 mgr.y (mgr.25072) 1878 : cluster 0 pgmap v2076: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:50.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:19:50 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:19:50] "GET /metrics HTTP/1.1" 200 34019 "" "Prometheus/2.33.4" 2023-10-01T11:19:52.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:52 smithi078 bash[75299]: cluster 2023-10-01T11:19:50.613641+0000 mgr.y (mgr.25072) 1879 : cluster 0 pgmap v2077: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:52.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:52 smithi078 bash[78274]: cluster 2023-10-01T11:19:50.613641+0000 mgr.y (mgr.25072) 1879 : cluster 0 pgmap v2077: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:52.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:52 smithi173 bash[58816]: cluster 2023-10-01T11:19:50.613641+0000 mgr.y (mgr.25072) 1879 : cluster 0 pgmap v2077: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:54.031 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:54 smithi078 bash[75299]: cluster 2023-10-01T11:19:52.614094+0000 mgr.y (mgr.25072) 1880 : cluster 0 pgmap v2078: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:54.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:54 smithi078 bash[78274]: cluster 2023-10-01T11:19:52.614094+0000 mgr.y (mgr.25072) 1880 : cluster 0 pgmap v2078: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:54.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:54 smithi173 bash[58816]: cluster 2023-10-01T11:19:52.614094+0000 mgr.y (mgr.25072) 1880 : cluster 0 pgmap v2078: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:56.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:56 smithi078 bash[75299]: cluster 2023-10-01T11:19:54.614642+0000 mgr.y (mgr.25072) 1881 : cluster 0 pgmap v2079: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:56.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:56 smithi078 bash[78274]: cluster 2023-10-01T11:19:54.614642+0000 mgr.y (mgr.25072) 1881 : cluster 0 pgmap v2079: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:56.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:56 smithi173 bash[58816]: cluster 2023-10-01T11:19:54.614642+0000 mgr.y (mgr.25072) 1881 : cluster 0 pgmap v2079: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:58.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:19:58 smithi078 bash[75299]: cluster 2023-10-01T11:19:56.615114+0000 mgr.y (mgr.25072) 1882 : cluster 0 pgmap v2080: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:58.333 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:19:58 smithi078 bash[78274]: cluster 2023-10-01T11:19:56.615114+0000 mgr.y (mgr.25072) 1882 : cluster 0 pgmap v2080: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:58.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:19:58 smithi173 bash[58816]: cluster 2023-10-01T11:19:56.615114+0000 mgr.y (mgr.25072) 1882 : cluster 0 pgmap v2080: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:19:59.696 INFO:teuthology.orchestra.run.smithi078.stdout:519691043326 2023-10-01T11:19:59.696 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-10-01T11:20:00.331 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:20:00 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:20:00] "GET /metrics HTTP/1.1" 200 34019 "" "Prometheus/2.33.4" 2023-10-01T11:20:00.429 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:20:00 smithi078 bash[75299]: cluster 2023-10-01T11:19:58.616036+0000 mgr.y (mgr.25072) 1883 : cluster 0 pgmap v2081: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:00.429 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:20:00 smithi078 bash[75299]: cluster 2023-10-01T11:20:00.000125+0000 mon.a (mon.0) 2394 : cluster 1 overall HEALTH_OK 2023-10-01T11:20:00.430 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:20:00 smithi078 bash[78274]: cluster 2023-10-01T11:19:58.616036+0000 mgr.y (mgr.25072) 1883 : cluster 0 pgmap v2081: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:00.430 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:20:00 smithi078 bash[78274]: cluster 2023-10-01T11:20:00.000125+0000 mon.a (mon.0) 2394 : cluster 1 overall HEALTH_OK 2023-10-01T11:20:00.431 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:20:00 smithi173 bash[58816]: cluster 2023-10-01T11:19:58.616036+0000 mgr.y (mgr.25072) 1883 : cluster 0 pgmap v2081: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:00.431 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:20:00 smithi173 bash[58816]: cluster 2023-10-01T11:20:00.000125+0000 mon.a (mon.0) 2394 : cluster 1 overall HEALTH_OK 2023-10-01T11:20:02.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:20:02 smithi078 bash[78274]: cluster 2023-10-01T11:20:00.616639+0000 mgr.y (mgr.25072) 1884 : cluster 0 pgmap v2082: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:02.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:20:02 smithi078 bash[75299]: cluster 2023-10-01T11:20:00.616639+0000 mgr.y (mgr.25072) 1884 : cluster 0 pgmap v2082: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:02.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:20:02 smithi173 bash[58816]: cluster 2023-10-01T11:20:00.616639+0000 mgr.y (mgr.25072) 1884 : cluster 0 pgmap v2082: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:03.368 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:20:03 smithi078 bash[78274]: audit 2023-10-01T11:20:02.490433+0000 mon.a (mon.0) 2395 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:20:03.369 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:20:03 smithi078 bash[75299]: audit 2023-10-01T11:20:02.490433+0000 mon.a (mon.0) 2395 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:20:03.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:20:03 smithi173 bash[58816]: audit 2023-10-01T11:20:02.490433+0000 mon.a (mon.0) 2395 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:20:04.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:20:04 smithi078 bash[78274]: cluster 2023-10-01T11:20:02.617655+0000 mgr.y (mgr.25072) 1885 : cluster 0 pgmap v2083: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:04.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:20:04 smithi078 bash[75299]: cluster 2023-10-01T11:20:02.617655+0000 mgr.y (mgr.25072) 1885 : cluster 0 pgmap v2083: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:04.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:20:04 smithi173 bash[58816]: cluster 2023-10-01T11:20:02.617655+0000 mgr.y (mgr.25072) 1885 : cluster 0 pgmap v2083: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:04.917 INFO:teuthology.orchestra.run.smithi078.stdout:180388627251 2023-10-01T11:20:04.917 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-10-01T11:20:05.621 INFO:teuthology.orchestra.run.smithi078.stdout:498216206853 2023-10-01T11:20:05.621 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-10-01T11:20:06.059 INFO:teuthology.orchestra.run.smithi078.stdout:154618823481 2023-10-01T11:20:06.059 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-10-01T11:20:06.161 INFO:teuthology.orchestra.run.smithi078.stdout:369367188079 2023-10-01T11:20:06.161 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-10-01T11:20:06.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:20:06 smithi078 bash[75299]: cluster 2023-10-01T11:20:04.618475+0000 mgr.y (mgr.25072) 1886 : cluster 0 pgmap v2084: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:06.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:20:06 smithi078 bash[78274]: cluster 2023-10-01T11:20:04.618475+0000 mgr.y (mgr.25072) 1886 : cluster 0 pgmap v2084: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:06.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:20:06 smithi173 bash[58816]: cluster 2023-10-01T11:20:04.618475+0000 mgr.y (mgr.25072) 1886 : cluster 0 pgmap v2084: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:06.505 INFO:teuthology.orchestra.run.smithi078.stdout:541165879800 2023-10-01T11:20:06.506 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-10-01T11:20:07.582 INFO:teuthology.orchestra.run.smithi078.stdout:206158431012 2023-10-01T11:20:07.582 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-10-01T11:20:07.595 INFO:teuthology.orchestra.run.smithi078.stdout:433791697469 2023-10-01T11:20:07.596 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-10-01T11:20:08.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:20:08 smithi078 bash[75299]: cluster 2023-10-01T11:20:06.618863+0000 mgr.y (mgr.25072) 1887 : cluster 0 pgmap v2085: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:08.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:20:08 smithi078 bash[78274]: cluster 2023-10-01T11:20:06.618863+0000 mgr.y (mgr.25072) 1887 : cluster 0 pgmap v2085: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:08.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:20:08 smithi173 bash[58816]: cluster 2023-10-01T11:20:06.618863+0000 mgr.y (mgr.25072) 1887 : cluster 0 pgmap v2085: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:10.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:20:10 smithi078 bash[75299]: cluster 2023-10-01T11:20:08.619489+0000 mgr.y (mgr.25072) 1888 : cluster 0 pgmap v2086: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:10.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:20:10 smithi078 bash[78274]: cluster 2023-10-01T11:20:08.619489+0000 mgr.y (mgr.25072) 1888 : cluster 0 pgmap v2086: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:10.332 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:20:10 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:20:10] "GET /metrics HTTP/1.1" 200 34020 "" "Prometheus/2.33.4" 2023-10-01T11:20:10.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:20:10 smithi173 bash[58816]: cluster 2023-10-01T11:20:08.619489+0000 mgr.y (mgr.25072) 1888 : cluster 0 pgmap v2086: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:10.481 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:20:10.481 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:20:10.482 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:20:10.482 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:20:10.483 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:20:10.483 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:20:11.491 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:20:11.491 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:20:12.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:20:12 smithi078 bash[75299]: cluster 2023-10-01T11:20:10.620485+0000 mgr.y (mgr.25072) 1889 : cluster 0 pgmap v2087: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:12.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:20:12 smithi078 bash[78274]: cluster 2023-10-01T11:20:10.620485+0000 mgr.y (mgr.25072) 1889 : cluster 0 pgmap v2087: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:12.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:20:12 smithi173 bash[58816]: cluster 2023-10-01T11:20:10.620485+0000 mgr.y (mgr.25072) 1889 : cluster 0 pgmap v2087: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:14.335 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:20:14 smithi078 bash[78274]: cluster 2023-10-01T11:20:12.621278+0000 mgr.y (mgr.25072) 1890 : cluster 0 pgmap v2088: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:14.336 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:20:14 smithi078 bash[75299]: cluster 2023-10-01T11:20:12.621278+0000 mgr.y (mgr.25072) 1890 : cluster 0 pgmap v2088: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:14.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:20:14 smithi173 bash[58816]: cluster 2023-10-01T11:20:12.621278+0000 mgr.y (mgr.25072) 1890 : cluster 0 pgmap v2088: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:16.177 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:20:16 smithi078 bash[75299]: cluster 2023-10-01T11:20:14.621929+0000 mgr.y (mgr.25072) 1891 : cluster 0 pgmap v2089: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:16.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:20:16 smithi173 bash[58816]: cluster 2023-10-01T11:20:14.621929+0000 mgr.y (mgr.25072) 1891 : cluster 0 pgmap v2089: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:16.454 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:20:16 smithi078 bash[78274]: cluster 2023-10-01T11:20:14.621929+0000 mgr.y (mgr.25072) 1891 : cluster 0 pgmap v2089: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:18.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:20:18 smithi173 bash[58816]: cluster 2023-10-01T11:20:16.622371+0000 mgr.y (mgr.25072) 1892 : cluster 0 pgmap v2090: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:18.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:20:18 smithi173 bash[58816]: audit 2023-10-01T11:20:17.490788+0000 mon.a (mon.0) 2396 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:20:18.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:20:18 smithi078 bash[75299]: cluster 2023-10-01T11:20:16.622371+0000 mgr.y (mgr.25072) 1892 : cluster 0 pgmap v2090: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:18.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:20:18 smithi078 bash[75299]: audit 2023-10-01T11:20:17.490788+0000 mon.a (mon.0) 2396 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:20:18.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:20:18 smithi078 bash[78274]: cluster 2023-10-01T11:20:16.622371+0000 mgr.y (mgr.25072) 1892 : cluster 0 pgmap v2090: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:18.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:20:18 smithi078 bash[78274]: audit 2023-10-01T11:20:17.490788+0000 mon.a (mon.0) 2396 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:20:20.191 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:20:20 smithi078 bash[78274]: cluster 2023-10-01T11:20:18.622811+0000 mgr.y (mgr.25072) 1893 : cluster 0 pgmap v2091: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:20.191 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:20:20 smithi078 bash[75299]: cluster 2023-10-01T11:20:18.622811+0000 mgr.y (mgr.25072) 1893 : cluster 0 pgmap v2091: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:20.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:20:20 smithi173 bash[58816]: cluster 2023-10-01T11:20:18.622811+0000 mgr.y (mgr.25072) 1893 : cluster 0 pgmap v2091: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:20.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:20:20 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:20:20] "GET /metrics HTTP/1.1" 200 34016 "" "Prometheus/2.33.4" 2023-10-01T11:20:22.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:20:22 smithi173 bash[58816]: cluster 2023-10-01T11:20:20.623527+0000 mgr.y (mgr.25072) 1894 : cluster 0 pgmap v2092: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:22.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:20:22 smithi078 bash[75299]: cluster 2023-10-01T11:20:20.623527+0000 mgr.y (mgr.25072) 1894 : cluster 0 pgmap v2092: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:22.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:20:22 smithi078 bash[78274]: cluster 2023-10-01T11:20:20.623527+0000 mgr.y (mgr.25072) 1894 : cluster 0 pgmap v2092: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:23.047 INFO:teuthology.orchestra.run.smithi078.stdout:180388627255 2023-10-01T11:20:24.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:20:24 smithi078 bash[75299]: cluster 2023-10-01T11:20:22.624229+0000 mgr.y (mgr.25072) 1895 : cluster 0 pgmap v2093: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:24.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:20:24 smithi078 bash[75299]: audit 2023-10-01T11:20:23.046221+0000 mon.a (mon.0) 2397 : audit 0 from='client.? 172.21.15.78:0/2198484209' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-01T11:20:24.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:20:24 smithi078 bash[78274]: cluster 2023-10-01T11:20:22.624229+0000 mgr.y (mgr.25072) 1895 : cluster 0 pgmap v2093: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:24.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:20:24 smithi078 bash[78274]: audit 2023-10-01T11:20:23.046221+0000 mon.a (mon.0) 2397 : audit 0 from='client.? 172.21.15.78:0/2198484209' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-01T11:20:24.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:20:24 smithi173 bash[58816]: cluster 2023-10-01T11:20:22.624229+0000 mgr.y (mgr.25072) 1895 : cluster 0 pgmap v2093: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:24.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:20:24 smithi173 bash[58816]: audit 2023-10-01T11:20:23.046221+0000 mon.a (mon.0) 2397 : audit 0 from='client.? 172.21.15.78:0/2198484209' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-10-01T11:20:26.000 INFO:tasks.cephadm.ceph_manager.ceph:need seq 180388627251 got 180388627255 for osd.6 2023-10-01T11:20:26.000 DEBUG:teuthology.parallel:result is None 2023-10-01T11:20:26.079 INFO:teuthology.orchestra.run.smithi078.stdout:498216206858 2023-10-01T11:20:26.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:20:26 smithi078 bash[75299]: cluster 2023-10-01T11:20:24.625278+0000 mgr.y (mgr.25072) 1896 : cluster 0 pgmap v2094: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:26.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:20:26 smithi078 bash[78274]: cluster 2023-10-01T11:20:24.625278+0000 mgr.y (mgr.25072) 1896 : cluster 0 pgmap v2094: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:26.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:20:26 smithi173 bash[58816]: cluster 2023-10-01T11:20:24.625278+0000 mgr.y (mgr.25072) 1896 : cluster 0 pgmap v2094: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:26.865 INFO:teuthology.orchestra.run.smithi078.stdout:541165879805 2023-10-01T11:20:26.879 INFO:teuthology.orchestra.run.smithi078.stdout:154618823486 2023-10-01T11:20:27.304 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:20:27 smithi078 bash[78274]: audit 2023-10-01T11:20:26.077808+0000 mon.a (mon.0) 2398 : audit 0 from='client.? 172.21.15.78:0/3040286050' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-01T11:20:27.304 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:20:27 smithi078 bash[78274]: audit 2023-10-01T11:20:26.864265+0000 mon.a (mon.0) 2399 : audit 0 from='client.? 172.21.15.78:0/2891829396' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-01T11:20:27.305 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:20:27 smithi078 bash[78274]: audit 2023-10-01T11:20:26.879005+0000 mon.a (mon.0) 2400 : audit 0 from='client.? 172.21.15.78:0/2360082045' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-01T11:20:27.305 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:20:27 smithi078 bash[75299]: audit 2023-10-01T11:20:26.077808+0000 mon.a (mon.0) 2398 : audit 0 from='client.? 172.21.15.78:0/3040286050' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-01T11:20:27.305 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:20:27 smithi078 bash[75299]: audit 2023-10-01T11:20:26.864265+0000 mon.a (mon.0) 2399 : audit 0 from='client.? 172.21.15.78:0/2891829396' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-01T11:20:27.305 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:20:27 smithi078 bash[75299]: audit 2023-10-01T11:20:26.879005+0000 mon.a (mon.0) 2400 : audit 0 from='client.? 172.21.15.78:0/2360082045' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-01T11:20:27.307 INFO:teuthology.orchestra.run.smithi078.stdout:369367188084 2023-10-01T11:20:27.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:20:27 smithi173 bash[58816]: audit 2023-10-01T11:20:26.077808+0000 mon.a (mon.0) 2398 : audit 0 from='client.? 172.21.15.78:0/3040286050' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-10-01T11:20:27.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:20:27 smithi173 bash[58816]: audit 2023-10-01T11:20:26.864265+0000 mon.a (mon.0) 2399 : audit 0 from='client.? 172.21.15.78:0/2891829396' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-10-01T11:20:27.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:20:27 smithi173 bash[58816]: audit 2023-10-01T11:20:26.879005+0000 mon.a (mon.0) 2400 : audit 0 from='client.? 172.21.15.78:0/2360082045' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-10-01T11:20:27.687 INFO:teuthology.orchestra.run.smithi078.stdout:519691043332 2023-10-01T11:20:28.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:20:28 smithi078 bash[78274]: cluster 2023-10-01T11:20:26.625738+0000 mgr.y (mgr.25072) 1897 : cluster 0 pgmap v2095: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:28.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:20:28 smithi078 bash[78274]: audit 2023-10-01T11:20:27.306844+0000 mon.a (mon.0) 2401 : audit 0 from='client.? 172.21.15.78:0/1175988679' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-01T11:20:28.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:20:28 smithi078 bash[78274]: audit 2023-10-01T11:20:27.686666+0000 mon.a (mon.0) 2402 : audit 0 from='client.? 172.21.15.78:0/3285017586' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-01T11:20:28.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:20:28 smithi078 bash[75299]: cluster 2023-10-01T11:20:26.625738+0000 mgr.y (mgr.25072) 1897 : cluster 0 pgmap v2095: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:28.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:20:28 smithi078 bash[75299]: audit 2023-10-01T11:20:27.306844+0000 mon.a (mon.0) 2401 : audit 0 from='client.? 172.21.15.78:0/1175988679' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-01T11:20:28.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:20:28 smithi078 bash[75299]: audit 2023-10-01T11:20:27.686666+0000 mon.a (mon.0) 2402 : audit 0 from='client.? 172.21.15.78:0/3285017586' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-01T11:20:28.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:20:28 smithi173 bash[58816]: cluster 2023-10-01T11:20:26.625738+0000 mgr.y (mgr.25072) 1897 : cluster 0 pgmap v2095: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:28.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:20:28 smithi173 bash[58816]: audit 2023-10-01T11:20:27.306844+0000 mon.a (mon.0) 2401 : audit 0 from='client.? 172.21.15.78:0/1175988679' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-10-01T11:20:28.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:20:28 smithi173 bash[58816]: audit 2023-10-01T11:20:27.686666+0000 mon.a (mon.0) 2402 : audit 0 from='client.? 172.21.15.78:0/3285017586' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-10-01T11:20:29.088 INFO:teuthology.orchestra.run.smithi078.stdout:206158431017 2023-10-01T11:20:29.305 INFO:teuthology.orchestra.run.smithi078.stdout:433791697474 2023-10-01T11:20:29.902 INFO:tasks.cephadm.ceph_manager.ceph:need seq 498216206853 got 498216206858 for osd.2 2023-10-01T11:20:29.903 DEBUG:teuthology.parallel:result is None 2023-10-01T11:20:30.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:20:30 smithi078 bash[75299]: cluster 2023-10-01T11:20:28.626132+0000 mgr.y (mgr.25072) 1898 : cluster 0 pgmap v2096: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:30.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:20:30 smithi078 bash[75299]: audit 2023-10-01T11:20:29.087148+0000 mon.c (mon.2) 270 : audit 0 from='client.? 172.21.15.78:0/999466648' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-01T11:20:30.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:20:30 smithi078 bash[75299]: audit 2023-10-01T11:20:29.304046+0000 mon.c (mon.2) 271 : audit 0 from='client.? 172.21.15.78:0/3615895526' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-01T11:20:30.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:20:30 smithi078 bash[75299]: audit 2023-10-01T11:20:29.441043+0000 mon.a (mon.0) 2403 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T11:20:30.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:20:30 smithi078 bash[75299]: audit 2023-10-01T11:20:29.815964+0000 mon.a (mon.0) 2404 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:20:30.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:20:30 smithi078 bash[75299]: audit 2023-10-01T11:20:29.822285+0000 mon.a (mon.0) 2405 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:20:30.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:20:30 smithi078 bash[78274]: cluster 2023-10-01T11:20:28.626132+0000 mgr.y (mgr.25072) 1898 : cluster 0 pgmap v2096: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:30.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:20:30 smithi078 bash[78274]: audit 2023-10-01T11:20:29.087148+0000 mon.c (mon.2) 270 : audit 0 from='client.? 172.21.15.78:0/999466648' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-01T11:20:30.082 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:20:30 smithi078 bash[78274]: audit 2023-10-01T11:20:29.304046+0000 mon.c (mon.2) 271 : audit 0 from='client.? 172.21.15.78:0/3615895526' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-01T11:20:30.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:20:30 smithi078 bash[78274]: audit 2023-10-01T11:20:29.441043+0000 mon.a (mon.0) 2403 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T11:20:30.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:20:30 smithi078 bash[78274]: audit 2023-10-01T11:20:29.815964+0000 mon.a (mon.0) 2404 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:20:30.083 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:20:30 smithi078 bash[78274]: audit 2023-10-01T11:20:29.822285+0000 mon.a (mon.0) 2405 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:20:30.333 INFO:tasks.cephadm.ceph_manager.ceph:need seq 154618823481 got 154618823486 for osd.5 2023-10-01T11:20:30.333 DEBUG:teuthology.parallel:result is None 2023-10-01T11:20:30.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:20:30 smithi173 bash[58816]: cluster 2023-10-01T11:20:28.626132+0000 mgr.y (mgr.25072) 1898 : cluster 0 pgmap v2096: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:30.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:20:30 smithi173 bash[58816]: audit 2023-10-01T11:20:29.087148+0000 mon.c (mon.2) 270 : audit 0 from='client.? 172.21.15.78:0/999466648' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-10-01T11:20:30.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:20:30 smithi173 bash[58816]: audit 2023-10-01T11:20:29.304046+0000 mon.c (mon.2) 271 : audit 0 from='client.? 172.21.15.78:0/3615895526' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-10-01T11:20:30.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:20:30 smithi173 bash[58816]: audit 2023-10-01T11:20:29.441043+0000 mon.a (mon.0) 2403 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-10-01T11:20:30.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:20:30 smithi173 bash[58816]: audit 2023-10-01T11:20:29.815964+0000 mon.a (mon.0) 2404 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:20:30.370 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:20:30 smithi173 bash[58816]: audit 2023-10-01T11:20:29.822285+0000 mon.a (mon.0) 2405 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:20:30.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:20:30 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:20:30] "GET /metrics HTTP/1.1" 200 34016 "" "Prometheus/2.33.4" 2023-10-01T11:20:31.866 INFO:tasks.cephadm.ceph_manager.ceph:need seq 541165879800 got 541165879805 for osd.4 2023-10-01T11:20:31.866 DEBUG:teuthology.parallel:result is None 2023-10-01T11:20:32.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:20:32 smithi078 bash[78274]: cluster 2023-10-01T11:20:30.626998+0000 mgr.y (mgr.25072) 1899 : cluster 0 pgmap v2097: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:32.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:20:32 smithi078 bash[75299]: cluster 2023-10-01T11:20:30.626998+0000 mgr.y (mgr.25072) 1899 : cluster 0 pgmap v2097: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:32.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:20:32 smithi173 bash[58816]: cluster 2023-10-01T11:20:30.626998+0000 mgr.y (mgr.25072) 1899 : cluster 0 pgmap v2097: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:33.096 INFO:tasks.cephadm.ceph_manager.ceph:need seq 369367188079 got 369367188084 for osd.0 2023-10-01T11:20:33.096 DEBUG:teuthology.parallel:result is None 2023-10-01T11:20:33.228 INFO:tasks.cephadm.ceph_manager.ceph:need seq 519691043326 got 519691043332 for osd.3 2023-10-01T11:20:33.229 DEBUG:teuthology.parallel:result is None 2023-10-01T11:20:33.267 INFO:tasks.cephadm.ceph_manager.ceph:need seq 433791697469 got 433791697474 for osd.1 2023-10-01T11:20:33.267 DEBUG:teuthology.parallel:result is None 2023-10-01T11:20:33.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:20:33 smithi078 bash[75299]: audit 2023-10-01T11:20:32.491741+0000 mon.a (mon.0) 2406 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:20:33.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:20:33 smithi078 bash[78274]: audit 2023-10-01T11:20:32.491741+0000 mon.a (mon.0) 2406 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:20:33.362 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158431012 got 206158431017 for osd.7 2023-10-01T11:20:33.363 DEBUG:teuthology.parallel:result is None 2023-10-01T11:20:33.363 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-10-01T11:20:33.363 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T11:20:33.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:20:33 smithi173 bash[58816]: audit 2023-10-01T11:20:32.491741+0000 mon.a (mon.0) 2406 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:20:34.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:20:34 smithi078 bash[75299]: cluster 2023-10-01T11:20:32.627769+0000 mgr.y (mgr.25072) 1900 : cluster 0 pgmap v2098: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:34.332 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:20:34 smithi078 bash[78274]: cluster 2023-10-01T11:20:32.627769+0000 mgr.y (mgr.25072) 1900 : cluster 0 pgmap v2098: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:34.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:20:34 smithi173 bash[58816]: cluster 2023-10-01T11:20:32.627769+0000 mgr.y (mgr.25072) 1900 : cluster 0 pgmap v2098: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:36.331 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:20:36 smithi078 bash[75299]: cluster 2023-10-01T11:20:34.628835+0000 mgr.y (mgr.25072) 1901 : cluster 0 pgmap v2099: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:36.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:20:36 smithi078 bash[78274]: cluster 2023-10-01T11:20:34.628835+0000 mgr.y (mgr.25072) 1901 : cluster 0 pgmap v2099: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:36.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:20:36 smithi173 bash[58816]: cluster 2023-10-01T11:20:34.628835+0000 mgr.y (mgr.25072) 1901 : cluster 0 pgmap v2099: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:37.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:20:37 smithi173 bash[58816]: audit 2023-10-01T11:20:36.117913+0000 mon.a (mon.0) 2407 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:20:37.369 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:20:37 smithi173 bash[58816]: audit 2023-10-01T11:20:36.126480+0000 mon.a (mon.0) 2408 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:20:37.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:20:37 smithi078 bash[75299]: audit 2023-10-01T11:20:36.117913+0000 mon.a (mon.0) 2407 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:20:37.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:20:37 smithi078 bash[75299]: audit 2023-10-01T11:20:36.126480+0000 mon.a (mon.0) 2408 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:20:37.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:20:37 smithi078 bash[78274]: audit 2023-10-01T11:20:36.117913+0000 mon.a (mon.0) 2407 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:20:37.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:20:37 smithi078 bash[78274]: audit 2023-10-01T11:20:36.126480+0000 mon.a (mon.0) 2408 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:20:38.135 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:20:38.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:20:38 smithi078 bash[78274]: cluster 2023-10-01T11:20:36.629427+0000 mgr.y (mgr.25072) 1902 : cluster 0 pgmap v2100: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:38.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:20:38 smithi078 bash[78274]: audit 2023-10-01T11:20:37.288834+0000 mon.a (mon.0) 2409 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:20:38.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:20:38 smithi078 bash[78274]: audit 2023-10-01T11:20:37.298792+0000 mon.a (mon.0) 2410 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:20:38.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:20:38 smithi078 bash[78274]: audit 2023-10-01T11:20:37.770485+0000 mon.a (mon.0) 2411 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T11:20:38.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:20:38 smithi078 bash[78274]: audit 2023-10-01T11:20:37.772196+0000 mon.a (mon.0) 2412 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T11:20:38.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:20:38 smithi078 bash[78274]: audit 2023-10-01T11:20:37.781226+0000 mon.a (mon.0) 2413 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:20:38.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:20:38 smithi078 bash[75299]: cluster 2023-10-01T11:20:36.629427+0000 mgr.y (mgr.25072) 1902 : cluster 0 pgmap v2100: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:38.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:20:38 smithi078 bash[75299]: audit 2023-10-01T11:20:37.288834+0000 mon.a (mon.0) 2409 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:20:38.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:20:38 smithi078 bash[75299]: audit 2023-10-01T11:20:37.298792+0000 mon.a (mon.0) 2410 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:20:38.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:20:38 smithi078 bash[75299]: audit 2023-10-01T11:20:37.770485+0000 mon.a (mon.0) 2411 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T11:20:38.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:20:38 smithi078 bash[75299]: audit 2023-10-01T11:20:37.772196+0000 mon.a (mon.0) 2412 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T11:20:38.583 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:20:38 smithi078 bash[75299]: audit 2023-10-01T11:20:37.781226+0000 mon.a (mon.0) 2413 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:20:38.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:20:38 smithi173 bash[58816]: cluster 2023-10-01T11:20:36.629427+0000 mgr.y (mgr.25072) 1902 : cluster 0 pgmap v2100: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:38.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:20:38 smithi173 bash[58816]: audit 2023-10-01T11:20:37.288834+0000 mon.a (mon.0) 2409 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:20:38.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:20:38 smithi173 bash[58816]: audit 2023-10-01T11:20:37.298792+0000 mon.a (mon.0) 2410 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:20:38.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:20:38 smithi173 bash[58816]: audit 2023-10-01T11:20:37.770485+0000 mon.a (mon.0) 2411 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-10-01T11:20:38.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:20:38 smithi173 bash[58816]: audit 2023-10-01T11:20:37.772196+0000 mon.a (mon.0) 2412 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-10-01T11:20:38.620 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:20:38 smithi173 bash[58816]: audit 2023-10-01T11:20:37.781226+0000 mon.a (mon.0) 2413 : audit 1 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' 2023-10-01T11:20:40.566 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:20:40 smithi078 bash[78274]: cluster 2023-10-01T11:20:38.630233+0000 mgr.y (mgr.25072) 1903 : cluster 0 pgmap v2101: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:40.566 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:20:40 smithi078 bash[75299]: cluster 2023-10-01T11:20:38.630233+0000 mgr.y (mgr.25072) 1903 : cluster 0 pgmap v2101: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:40.567 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:20:40 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:20:40] "GET /metrics HTTP/1.1" 200 34022 "" "Prometheus/2.33.4" 2023-10-01T11:20:40.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:20:40 smithi173 bash[58816]: cluster 2023-10-01T11:20:38.630233+0000 mgr.y (mgr.25072) 1903 : cluster 0 pgmap v2101: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:42.064 INFO:teuthology.orchestra.run.smithi078.stdout: 2023-10-01T11:20:42.066 INFO:teuthology.orchestra.run.smithi078.stderr:dumped all 2023-10-01T11:20:42.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:20:42 smithi078 bash[75299]: cluster 2023-10-01T11:20:40.630922+0000 mgr.y (mgr.25072) 1904 : cluster 0 pgmap v2102: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:42.581 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:20:42 smithi078 bash[78274]: cluster 2023-10-01T11:20:40.630922+0000 mgr.y (mgr.25072) 1904 : cluster 0 pgmap v2102: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:42.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:20:42 smithi173 bash[58816]: cluster 2023-10-01T11:20:40.630922+0000 mgr.y (mgr.25072) 1904 : cluster 0 pgmap v2102: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:42.988 INFO:teuthology.orchestra.run.smithi078.stdout:{"pg_ready":true,"pg_map":{"version":2102,"stamp":"2023-10-01T11:20:40.630532+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6279,"num_read_kb":13744,"num_write":1107,"num_write_kb":15160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":162,"num_bytes_recovered":6528818,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1172,"ondisk_log_size":1172,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":8758076,"kb_used_data":49532,"kb_used_omap":30,"kb_used_meta":8708449,"kb_avail":741203140,"statfs":{"total":767960285184,"available":758992015360,"internally_reserved":0,"allocated":50720768,"data_stored":40686313,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31502,"internal_metadata":8917452018},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004628"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":425,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.170753+0000","last_change":"2023-10-01T11:04:02.725956+0000","last_active":"2023-10-01T11:19:21.170753+0000","last_peered":"2023-10-01T11:19:21.170753+0000","last_clean":"2023-10-01T11:19:21.170753+0000","last_became_active":"2023-10-01T11:04:02.725317+0000","last_became_peered":"2023-10-01T11:04:02.725317+0000","last_unstale":"2023-10-01T11:19:21.170753+0000","last_undegraded":"2023-10-01T11:19:21.170753+0000","last_fullsized":"2023-10-01T11:19:21.170753+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:34:07.675650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":429,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.174594+0000","last_change":"2023-10-01T10:38:16.025881+0000","last_active":"2023-10-01T11:19:22.174594+0000","last_peered":"2023-10-01T11:19:22.174594+0000","last_clean":"2023-10-01T11:19:22.174594+0000","last_became_active":"2023-10-01T10:38:16.025518+0000","last_became_peered":"2023-10-01T10:38:16.025518+0000","last_unstale":"2023-10-01T11:19:22.174594+0000","last_undegraded":"2023-10-01T11:19:22.174594+0000","last_fullsized":"2023-10-01T11:19:22.174594+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:48:37.826604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":285,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.182342+0000","last_change":"2023-10-01T11:17:22.000180+0000","last_active":"2023-10-01T11:19:22.182342+0000","last_peered":"2023-10-01T11:19:22.182342+0000","last_clean":"2023-10-01T11:19:22.182342+0000","last_became_active":"2023-10-01T11:17:21.999806+0000","last_became_peered":"2023-10-01T11:17:21.999806+0000","last_unstale":"2023-10-01T11:19:22.182342+0000","last_undegraded":"2023-10-01T11:19:22.182342+0000","last_fullsized":"2023-10-01T11:19:22.182342+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:17:10.953153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":460,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.176417+0000","last_change":"2023-10-01T11:17:02.473305+0000","last_active":"2023-10-01T11:19:21.176417+0000","last_peered":"2023-10-01T11:19:21.176417+0000","last_clean":"2023-10-01T11:19:21.176417+0000","last_became_active":"2023-10-01T11:17:02.472151+0000","last_became_peered":"2023-10-01T11:17:02.472151+0000","last_unstale":"2023-10-01T11:19:21.176417+0000","last_undegraded":"2023-10-01T11:19:21.176417+0000","last_fullsized":"2023-10-01T11:19:21.176417+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:23:59.546636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":448,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.164052+0000","last_change":"2023-10-01T11:16:18.984005+0000","last_active":"2023-10-01T11:19:21.164052+0000","last_peered":"2023-10-01T11:19:21.164052+0000","last_clean":"2023-10-01T11:19:21.164052+0000","last_became_active":"2023-10-01T11:16:18.983549+0000","last_became_peered":"2023-10-01T11:16:18.983549+0000","last_unstale":"2023-10-01T11:19:21.164052+0000","last_undegraded":"2023-10-01T11:19:21.164052+0000","last_fullsized":"2023-10-01T11:19:21.164052+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:10:36.758334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":125,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.176708+0000","last_change":"2023-10-01T11:05:33.800368+0000","last_active":"2023-10-01T11:19:21.176708+0000","last_peered":"2023-10-01T11:19:21.176708+0000","last_clean":"2023-10-01T11:19:21.176708+0000","last_became_active":"2023-10-01T11:05:33.799877+0000","last_became_peered":"2023-10-01T11:05:33.799877+0000","last_unstale":"2023-10-01T11:19:21.176708+0000","last_undegraded":"2023-10-01T11:19:21.176708+0000","last_fullsized":"2023-10-01T11:19:21.176708+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":214,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.184393+0000","last_change":"2023-10-01T11:17:02.474326+0000","last_active":"2023-10-01T11:19:22.184393+0000","last_peered":"2023-10-01T11:19:22.184393+0000","last_clean":"2023-10-01T11:19:22.184393+0000","last_became_active":"2023-10-01T11:17:02.473842+0000","last_became_peered":"2023-10-01T11:17:02.473842+0000","last_unstale":"2023-10-01T11:19:22.184393+0000","last_undegraded":"2023-10-01T11:19:22.184393+0000","last_fullsized":"2023-10-01T11:19:22.184393+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:58:58.628130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":469,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.174026+0000","last_change":"2023-10-01T11:05:33.782561+0000","last_active":"2023-10-01T11:19:22.174026+0000","last_peered":"2023-10-01T11:19:22.174026+0000","last_clean":"2023-10-01T11:19:22.174026+0000","last_became_active":"2023-10-01T11:05:33.782066+0000","last_became_peered":"2023-10-01T11:05:33.782066+0000","last_unstale":"2023-10-01T11:19:22.174026+0000","last_undegraded":"2023-10-01T11:19:22.174026+0000","last_fullsized":"2023-10-01T11:19:22.174026+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:57:06.451343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":442,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.177281+0000","last_change":"2023-10-01T11:06:11.792061+0000","last_active":"2023-10-01T11:19:21.177281+0000","last_peered":"2023-10-01T11:19:21.177281+0000","last_clean":"2023-10-01T11:19:21.177281+0000","last_became_active":"2023-10-01T11:06:11.789129+0000","last_became_peered":"2023-10-01T11:06:11.789129+0000","last_unstale":"2023-10-01T11:19:21.177281+0000","last_undegraded":"2023-10-01T11:19:21.177281+0000","last_fullsized":"2023-10-01T11:19:21.177281+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":489,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.174090+0000","last_change":"2023-10-01T11:06:11.807749+0000","last_active":"2023-10-01T11:19:22.174090+0000","last_peered":"2023-10-01T11:19:22.174090+0000","last_clean":"2023-10-01T11:19:22.174090+0000","last_became_active":"2023-10-01T11:06:11.807375+0000","last_became_peered":"2023-10-01T11:06:11.807375+0000","last_unstale":"2023-10-01T11:19:22.174090+0000","last_undegraded":"2023-10-01T11:19:22.174090+0000","last_fullsized":"2023-10-01T11:19:22.174090+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:44:05.945387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":233,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.185239+0000","last_change":"2023-10-01T11:19:22.185239+0000","last_active":"2023-10-01T11:19:22.185239+0000","last_peered":"2023-10-01T11:19:22.185239+0000","last_clean":"2023-10-01T11:19:22.185239+0000","last_became_active":"2023-10-01T11:19:22.184399+0000","last_became_peered":"2023-10-01T11:19:22.184399+0000","last_unstale":"2023-10-01T11:19:22.185239+0000","last_undegraded":"2023-10-01T11:19:22.185239+0000","last_fullsized":"2023-10-01T11:19:22.185239+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":208,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.182591+0000","last_change":"2023-10-01T11:17:22.399197+0000","last_active":"2023-10-01T11:19:22.182591+0000","last_peered":"2023-10-01T11:19:22.182591+0000","last_clean":"2023-10-01T11:19:22.182591+0000","last_became_active":"2023-10-01T11:17:22.398388+0000","last_became_peered":"2023-10-01T11:17:22.398388+0000","last_unstale":"2023-10-01T11:19:22.182591+0000","last_undegraded":"2023-10-01T11:19:22.182591+0000","last_fullsized":"2023-10-01T11:19:22.182591+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:24:22.497645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":554,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.180215+0000","last_change":"2023-10-01T11:06:11.836035+0000","last_active":"2023-10-01T11:19:22.180215+0000","last_peered":"2023-10-01T11:19:22.180215+0000","last_clean":"2023-10-01T11:19:22.180215+0000","last_became_active":"2023-10-01T11:06:11.794834+0000","last_became_peered":"2023-10-01T11:06:11.794834+0000","last_unstale":"2023-10-01T11:19:22.180215+0000","last_undegraded":"2023-10-01T11:19:22.180215+0000","last_fullsized":"2023-10-01T11:19:22.180215+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:25:13.103100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":33138,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":458,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.179754+0000","last_change":"2023-10-01T11:15:57.937317+0000","last_active":"2023-10-01T11:19:22.179754+0000","last_peered":"2023-10-01T11:19:22.179754+0000","last_clean":"2023-10-01T11:19:22.179754+0000","last_became_active":"2023-10-01T11:15:57.936299+0000","last_became_peered":"2023-10-01T11:15:57.936299+0000","last_unstale":"2023-10-01T11:19:22.179754+0000","last_undegraded":"2023-10-01T11:19:22.179754+0000","last_fullsized":"2023-10-01T11:19:22.179754+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:04:32.042221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":91,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.173605+0000","last_change":"2023-10-01T11:14:22.974249+0000","last_active":"2023-10-01T11:19:21.173605+0000","last_peered":"2023-10-01T11:19:21.173605+0000","last_clean":"2023-10-01T11:19:21.173605+0000","last_became_active":"2023-10-01T11:14:22.973777+0000","last_became_peered":"2023-10-01T11:14:22.973777+0000","last_unstale":"2023-10-01T11:19:21.173605+0000","last_undegraded":"2023-10-01T11:19:21.173605+0000","last_fullsized":"2023-10-01T11:19:21.173605+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":474,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.174437+0000","last_change":"2023-10-01T11:06:11.789794+0000","last_active":"2023-10-01T11:19:22.174437+0000","last_peered":"2023-10-01T11:19:22.174437+0000","last_clean":"2023-10-01T11:19:22.174437+0000","last_became_active":"2023-10-01T11:06:11.789452+0000","last_became_peered":"2023-10-01T11:06:11.789452+0000","last_unstale":"2023-10-01T11:19:22.174437+0000","last_undegraded":"2023-10-01T11:19:22.174437+0000","last_fullsized":"2023-10-01T11:19:22.174437+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:18:02.050694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":447,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.183907+0000","last_change":"2023-10-01T11:17:02.472419+0000","last_active":"2023-10-01T11:19:22.183907+0000","last_peered":"2023-10-01T11:19:22.183907+0000","last_clean":"2023-10-01T11:19:22.183907+0000","last_became_active":"2023-10-01T11:17:02.472063+0000","last_became_peered":"2023-10-01T11:17:02.472063+0000","last_unstale":"2023-10-01T11:19:22.183907+0000","last_undegraded":"2023-10-01T11:19:22.183907+0000","last_fullsized":"2023-10-01T11:19:22.183907+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:00:18.363944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":459,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.177184+0000","last_change":"2023-10-01T11:06:11.794671+0000","last_active":"2023-10-01T11:19:21.177184+0000","last_peered":"2023-10-01T11:19:21.177184+0000","last_clean":"2023-10-01T11:19:21.177184+0000","last_became_active":"2023-10-01T11:06:11.794121+0000","last_became_peered":"2023-10-01T11:06:11.794121+0000","last_unstale":"2023-10-01T11:19:21.177184+0000","last_undegraded":"2023-10-01T11:19:21.177184+0000","last_fullsized":"2023-10-01T11:19:21.177184+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":342,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.190563+0000","last_change":"2023-10-01T11:19:22.190563+0000","last_active":"2023-10-01T11:19:22.190563+0000","last_peered":"2023-10-01T11:19:22.190563+0000","last_clean":"2023-10-01T11:19:22.190563+0000","last_became_active":"2023-10-01T11:19:22.190197+0000","last_became_peered":"2023-10-01T11:19:22.190197+0000","last_unstale":"2023-10-01T11:19:22.190563+0000","last_undegraded":"2023-10-01T11:19:22.190563+0000","last_fullsized":"2023-10-01T11:19:22.190563+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":447,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.184450+0000","last_change":"2023-10-01T11:17:02.474479+0000","last_active":"2023-10-01T11:19:22.184450+0000","last_peered":"2023-10-01T11:19:22.184450+0000","last_clean":"2023-10-01T11:19:22.184450+0000","last_became_active":"2023-10-01T11:17:02.473987+0000","last_became_peered":"2023-10-01T11:17:02.473987+0000","last_unstale":"2023-10-01T11:19:22.184450+0000","last_undegraded":"2023-10-01T11:19:22.184450+0000","last_fullsized":"2023-10-01T11:19:22.184450+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:09:04.119899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":173,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.178743+0000","last_change":"2023-10-01T11:16:18.994830+0000","last_active":"2023-10-01T11:19:21.178743+0000","last_peered":"2023-10-01T11:19:21.178743+0000","last_clean":"2023-10-01T11:19:21.178743+0000","last_became_active":"2023-10-01T11:16:18.984619+0000","last_became_peered":"2023-10-01T11:16:18.984619+0000","last_unstale":"2023-10-01T11:19:21.178743+0000","last_undegraded":"2023-10-01T11:19:21.178743+0000","last_fullsized":"2023-10-01T11:19:21.178743+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":450,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.166861+0000","last_change":"2023-10-01T11:06:11.798727+0000","last_active":"2023-10-01T11:19:21.166861+0000","last_peered":"2023-10-01T11:19:21.166861+0000","last_clean":"2023-10-01T11:19:21.166861+0000","last_became_active":"2023-10-01T11:06:11.798388+0000","last_became_peered":"2023-10-01T11:06:11.798388+0000","last_unstale":"2023-10-01T11:19:21.166861+0000","last_undegraded":"2023-10-01T11:19:21.166861+0000","last_fullsized":"2023-10-01T11:19:21.166861+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:10:21.576627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":311,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.173725+0000","last_change":"2023-10-01T11:16:19.047127+0000","last_active":"2023-10-01T11:19:21.173725+0000","last_peered":"2023-10-01T11:19:21.173725+0000","last_clean":"2023-10-01T11:19:21.173725+0000","last_became_active":"2023-10-01T11:16:18.991314+0000","last_became_peered":"2023-10-01T11:16:18.991314+0000","last_unstale":"2023-10-01T11:19:21.173725+0000","last_undegraded":"2023-10-01T11:19:21.173725+0000","last_fullsized":"2023-10-01T11:19:21.173725+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":110,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":480,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.176510+0000","last_change":"2023-10-01T11:06:11.830255+0000","last_active":"2023-10-01T11:19:21.176510+0000","last_peered":"2023-10-01T11:19:21.176510+0000","last_clean":"2023-10-01T11:19:21.176510+0000","last_became_active":"2023-10-01T11:06:11.829847+0000","last_became_peered":"2023-10-01T11:06:11.829847+0000","last_unstale":"2023-10-01T11:19:21.176510+0000","last_undegraded":"2023-10-01T11:19:21.176510+0000","last_fullsized":"2023-10-01T11:19:21.176510+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":244,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.173888+0000","last_change":"2023-10-01T11:14:22.974529+0000","last_active":"2023-10-01T11:19:21.173888+0000","last_peered":"2023-10-01T11:19:21.173888+0000","last_clean":"2023-10-01T11:19:21.173888+0000","last_became_active":"2023-10-01T11:14:22.974060+0000","last_became_peered":"2023-10-01T11:14:22.974060+0000","last_unstale":"2023-10-01T11:19:21.173888+0000","last_undegraded":"2023-10-01T11:19:21.173888+0000","last_fullsized":"2023-10-01T11:19:21.173888+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":429,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.170121+0000","last_change":"2023-10-01T11:04:02.724840+0000","last_active":"2023-10-01T11:19:21.170121+0000","last_peered":"2023-10-01T11:19:21.170121+0000","last_clean":"2023-10-01T11:19:21.170121+0000","last_became_active":"2023-10-01T11:04:02.724476+0000","last_became_peered":"2023-10-01T11:04:02.724476+0000","last_unstale":"2023-10-01T11:19:21.170121+0000","last_undegraded":"2023-10-01T11:19:21.170121+0000","last_fullsized":"2023-10-01T11:19:21.170121+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:05:08.882495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":300,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.190703+0000","last_change":"2023-10-01T11:19:22.190703+0000","last_active":"2023-10-01T11:19:22.190703+0000","last_peered":"2023-10-01T11:19:22.190703+0000","last_clean":"2023-10-01T11:19:22.190703+0000","last_became_active":"2023-10-01T11:19:22.190327+0000","last_became_peered":"2023-10-01T11:19:22.190327+0000","last_unstale":"2023-10-01T11:19:22.190703+0000","last_undegraded":"2023-10-01T11:19:22.190703+0000","last_fullsized":"2023-10-01T11:19:22.190703+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":630,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":134,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.176067+0000","last_change":"2023-10-01T11:06:11.795096+0000","last_active":"2023-10-01T11:19:21.176067+0000","last_peered":"2023-10-01T11:19:21.176067+0000","last_clean":"2023-10-01T11:19:21.176067+0000","last_became_active":"2023-10-01T11:06:11.794716+0000","last_became_peered":"2023-10-01T11:06:11.794716+0000","last_unstale":"2023-10-01T11:19:21.176067+0000","last_undegraded":"2023-10-01T11:19:21.176067+0000","last_fullsized":"2023-10-01T11:19:21.176067+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":460,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.176813+0000","last_change":"2023-10-01T11:05:33.788917+0000","last_active":"2023-10-01T11:19:21.176813+0000","last_peered":"2023-10-01T11:19:21.176813+0000","last_clean":"2023-10-01T11:19:21.176813+0000","last_became_active":"2023-10-01T11:05:33.788078+0000","last_became_peered":"2023-10-01T11:05:33.788078+0000","last_unstale":"2023-10-01T11:19:21.176813+0000","last_undegraded":"2023-10-01T11:19:21.176813+0000","last_fullsized":"2023-10-01T11:19:21.176813+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":36,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.175639+0000","last_change":"2023-10-01T11:16:18.991614+0000","last_active":"2023-10-01T11:19:21.175639+0000","last_peered":"2023-10-01T11:19:21.175639+0000","last_clean":"2023-10-01T11:19:21.175639+0000","last_became_active":"2023-10-01T11:16:18.990345+0000","last_became_peered":"2023-10-01T11:16:18.990345+0000","last_unstale":"2023-10-01T11:19:21.175639+0000","last_undegraded":"2023-10-01T11:19:21.175639+0000","last_fullsized":"2023-10-01T11:19:21.175639+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:40:31.751016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":445,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.174438+0000","last_change":"2023-10-01T11:16:19.054310+0000","last_active":"2023-10-01T11:19:22.174438+0000","last_peered":"2023-10-01T11:19:22.174438+0000","last_clean":"2023-10-01T11:19:22.174438+0000","last_became_active":"2023-10-01T11:16:18.997794+0000","last_became_peered":"2023-10-01T11:16:18.997794+0000","last_unstale":"2023-10-01T11:19:22.174438+0000","last_undegraded":"2023-10-01T11:19:22.174438+0000","last_fullsized":"2023-10-01T11:19:22.174438+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:48:17.243891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":459,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.180461+0000","last_change":"2023-10-01T11:05:33.785133+0000","last_active":"2023-10-01T11:19:22.180461+0000","last_peered":"2023-10-01T11:19:22.180461+0000","last_clean":"2023-10-01T11:19:22.180461+0000","last_became_active":"2023-10-01T11:05:33.784749+0000","last_became_peered":"2023-10-01T11:05:33.784749+0000","last_unstale":"2023-10-01T11:19:22.180461+0000","last_undegraded":"2023-10-01T11:19:22.180461+0000","last_fullsized":"2023-10-01T11:19:22.180461+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:08:01.664432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":316,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.175853+0000","last_change":"2023-10-01T11:19:02.131086+0000","last_active":"2023-10-01T11:19:21.175853+0000","last_peered":"2023-10-01T11:19:21.175853+0000","last_clean":"2023-10-01T11:19:21.175853+0000","last_became_active":"2023-10-01T11:19:02.130633+0000","last_became_peered":"2023-10-01T11:19:02.130633+0000","last_unstale":"2023-10-01T11:19:21.175853+0000","last_undegraded":"2023-10-01T11:19:21.175853+0000","last_fullsized":"2023-10-01T11:19:21.175853+0000","mapping_epoch":435,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":436,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":419,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.180706+0000","last_change":"2023-10-01T11:04:02.727282+0000","last_active":"2023-10-01T11:19:22.180706+0000","last_peered":"2023-10-01T11:19:22.180706+0000","last_clean":"2023-10-01T11:19:22.180706+0000","last_became_active":"2023-10-01T11:04:02.726818+0000","last_became_peered":"2023-10-01T11:04:02.726818+0000","last_unstale":"2023-10-01T11:19:22.180706+0000","last_undegraded":"2023-10-01T11:19:22.180706+0000","last_fullsized":"2023-10-01T11:19:22.180706+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:11:11.868291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":463,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.179771+0000","last_change":"2023-10-01T11:06:11.790944+0000","last_active":"2023-10-01T11:19:22.179771+0000","last_peered":"2023-10-01T11:19:22.179771+0000","last_clean":"2023-10-01T11:19:22.179771+0000","last_became_active":"2023-10-01T11:06:11.790151+0000","last_became_peered":"2023-10-01T11:06:11.790151+0000","last_unstale":"2023-10-01T11:19:22.179771+0000","last_undegraded":"2023-10-01T11:19:22.179771+0000","last_fullsized":"2023-10-01T11:19:22.179771+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:05:49.161868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":470,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.180832+0000","last_change":"2023-10-01T11:15:57.939847+0000","last_active":"2023-10-01T11:19:22.180832+0000","last_peered":"2023-10-01T11:19:22.180832+0000","last_clean":"2023-10-01T11:19:22.180832+0000","last_became_active":"2023-10-01T11:15:57.939154+0000","last_became_peered":"2023-10-01T11:15:57.939154+0000","last_unstale":"2023-10-01T11:19:22.180832+0000","last_undegraded":"2023-10-01T11:19:22.180832+0000","last_fullsized":"2023-10-01T11:19:22.180832+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:25:56.931688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":453,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.164001+0000","last_change":"2023-10-01T11:06:11.784722+0000","last_active":"2023-10-01T11:19:21.164001+0000","last_peered":"2023-10-01T11:19:21.164001+0000","last_clean":"2023-10-01T11:19:21.164001+0000","last_became_active":"2023-10-01T11:06:11.784348+0000","last_became_peered":"2023-10-01T11:06:11.784348+0000","last_unstale":"2023-10-01T11:19:21.164001+0000","last_undegraded":"2023-10-01T11:19:21.164001+0000","last_fullsized":"2023-10-01T11:19:21.164001+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:40:29.283179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":436,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.326699+0000","last_change":"2023-10-01T10:38:16.019755+0000","last_active":"2023-10-01T11:19:22.326699+0000","last_peered":"2023-10-01T11:19:22.326699+0000","last_clean":"2023-10-01T11:19:22.326699+0000","last_became_active":"2023-10-01T10:38:16.019397+0000","last_became_peered":"2023-10-01T10:38:16.019397+0000","last_unstale":"2023-10-01T11:19:22.326699+0000","last_undegraded":"2023-10-01T11:19:22.326699+0000","last_fullsized":"2023-10-01T11:19:22.326699+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:42:56.623225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"405'127","reported_seq":693,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.183164+0000","last_change":"2023-10-01T11:17:22.006372+0000","last_active":"2023-10-01T11:19:22.183164+0000","last_peered":"2023-10-01T11:19:22.183164+0000","last_clean":"2023-10-01T11:19:22.183164+0000","last_became_active":"2023-10-01T11:17:22.005936+0000","last_became_peered":"2023-10-01T11:17:22.005936+0000","last_unstale":"2023-10-01T11:19:22.183164+0000","last_undegraded":"2023-10-01T11:19:22.183164+0000","last_fullsized":"2023-10-01T11:19:22.183164+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":127,"log_dups_size":0,"ondisk_log_size":127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:55:00.251169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":597,"num_read_kb":514,"num_write":134,"num_write_kb":4165,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":421,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.177200+0000","last_change":"2023-10-01T10:37:42.766814+0000","last_active":"2023-10-01T11:19:21.177200+0000","last_peered":"2023-10-01T11:19:21.177200+0000","last_clean":"2023-10-01T11:19:21.177200+0000","last_became_active":"2023-10-01T10:37:42.766451+0000","last_became_peered":"2023-10-01T10:37:42.766451+0000","last_unstale":"2023-10-01T11:19:21.177200+0000","last_undegraded":"2023-10-01T11:19:21.177200+0000","last_fullsized":"2023-10-01T11:19:21.177200+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":265,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.182552+0000","last_change":"2023-10-01T11:17:22.401658+0000","last_active":"2023-10-01T11:19:22.182552+0000","last_peered":"2023-10-01T11:19:22.182552+0000","last_clean":"2023-10-01T11:19:22.182552+0000","last_became_active":"2023-10-01T11:17:22.401009+0000","last_became_peered":"2023-10-01T11:17:22.401009+0000","last_unstale":"2023-10-01T11:19:22.182552+0000","last_undegraded":"2023-10-01T11:19:22.182552+0000","last_fullsized":"2023-10-01T11:19:22.182552+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:15:42.663028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"405'80","reported_seq":1099,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.177273+0000","last_change":"2023-10-01T11:15:39.089563+0000","last_active":"2023-10-01T11:19:21.177273+0000","last_peered":"2023-10-01T11:19:21.177273+0000","last_clean":"2023-10-01T11:19:21.177273+0000","last_became_active":"2023-10-01T11:05:33.792054+0000","last_became_peered":"2023-10-01T11:05:33.792054+0000","last_unstale":"2023-10-01T11:19:21.177273+0000","last_undegraded":"2023-10-01T11:19:21.177273+0000","last_fullsized":"2023-10-01T11:19:21.177273+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":80,"log_dups_size":0,"ondisk_log_size":80,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":994,"num_read_kb":2114,"num_write":54,"num_write_kb":1154,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":445,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.171656+0000","last_change":"2023-10-01T11:16:18.993464+0000","last_active":"2023-10-01T11:19:21.171656+0000","last_peered":"2023-10-01T11:19:21.171656+0000","last_clean":"2023-10-01T11:19:21.171656+0000","last_became_active":"2023-10-01T11:16:18.992859+0000","last_became_peered":"2023-10-01T11:16:18.992859+0000","last_unstale":"2023-10-01T11:19:21.171656+0000","last_undegraded":"2023-10-01T11:19:21.171656+0000","last_fullsized":"2023-10-01T11:19:21.171656+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:03:59.015456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":165,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.182980+0000","last_change":"2023-10-01T11:17:22.398815+0000","last_active":"2023-10-01T11:19:22.182980+0000","last_peered":"2023-10-01T11:19:22.182980+0000","last_clean":"2023-10-01T11:19:22.182980+0000","last_became_active":"2023-10-01T11:17:22.398275+0000","last_became_peered":"2023-10-01T11:17:22.398275+0000","last_unstale":"2023-10-01T11:19:22.182980+0000","last_undegraded":"2023-10-01T11:19:22.182980+0000","last_fullsized":"2023-10-01T11:19:22.182980+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:40:17.815750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"405'64","reported_seq":813,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.164699+0000","last_change":"2023-10-01T11:16:19.070021+0000","last_active":"2023-10-01T11:19:21.164699+0000","last_peered":"2023-10-01T11:19:21.164699+0000","last_clean":"2023-10-01T11:19:21.164699+0000","last_became_active":"2023-10-01T11:16:18.988806+0000","last_became_peered":"2023-10-01T11:16:18.988806+0000","last_unstale":"2023-10-01T11:19:21.164699+0000","last_undegraded":"2023-10-01T11:19:21.164699+0000","last_fullsized":"2023-10-01T11:19:21.164699+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":64,"log_dups_size":0,"ondisk_log_size":64,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:53:19.662794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042224999999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":500,"num_read_kb":2336,"num_write":43,"num_write_kb":1518,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":277,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.183552+0000","last_change":"2023-10-01T11:17:22.398544+0000","last_active":"2023-10-01T11:19:22.183552+0000","last_peered":"2023-10-01T11:19:22.183552+0000","last_clean":"2023-10-01T11:19:22.183552+0000","last_became_active":"2023-10-01T11:17:22.397941+0000","last_became_peered":"2023-10-01T11:17:22.397941+0000","last_unstale":"2023-10-01T11:19:22.183552+0000","last_undegraded":"2023-10-01T11:19:22.183552+0000","last_fullsized":"2023-10-01T11:19:22.183552+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:10:48.292501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":299,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.182761+0000","last_change":"2023-10-01T11:17:22.401230+0000","last_active":"2023-10-01T11:19:22.182761+0000","last_peered":"2023-10-01T11:19:22.182761+0000","last_clean":"2023-10-01T11:19:22.182761+0000","last_became_active":"2023-10-01T11:17:22.400774+0000","last_became_peered":"2023-10-01T11:17:22.400774+0000","last_unstale":"2023-10-01T11:19:22.182761+0000","last_undegraded":"2023-10-01T11:19:22.182761+0000","last_fullsized":"2023-10-01T11:19:22.182761+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:03:49.081904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"405'227","reported_seq":1130,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.176413+0000","last_change":"2023-10-01T11:15:39.089380+0000","last_active":"2023-10-01T11:19:21.176413+0000","last_peered":"2023-10-01T11:19:21.176413+0000","last_clean":"2023-10-01T11:19:21.176413+0000","last_became_active":"2023-10-01T11:06:11.789667+0000","last_became_peered":"2023-10-01T11:06:11.789667+0000","last_unstale":"2023-10-01T11:19:21.176413+0000","last_undegraded":"2023-10-01T11:19:21.176413+0000","last_fullsized":"2023-10-01T11:19:21.176413+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":227,"log_dups_size":0,"ondisk_log_size":227,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1045,"num_read_kb":1067,"num_write":205,"num_write_kb":189,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'139","reported_seq":758,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.190630+0000","last_change":"2023-10-01T11:19:22.190630+0000","last_active":"2023-10-01T11:19:22.190630+0000","last_peered":"2023-10-01T11:19:22.190630+0000","last_clean":"2023-10-01T11:19:22.190630+0000","last_became_active":"2023-10-01T11:19:22.190203+0000","last_became_peered":"2023-10-01T11:19:22.190203+0000","last_unstale":"2023-10-01T11:19:22.190630+0000","last_undegraded":"2023-10-01T11:19:22.190630+0000","last_fullsized":"2023-10-01T11:19:22.190630+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:09:17.221082+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:09:17.221082+0000","last_clean_scrub_stamp":"2023-10-01T10:09:17.221082+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":6494048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":454,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.437635+0000","last_change":"2023-10-01T11:05:33.788093+0000","last_active":"2023-10-01T11:19:21.437635+0000","last_peered":"2023-10-01T11:19:21.437635+0000","last_clean":"2023-10-01T11:19:21.437635+0000","last_became_active":"2023-10-01T11:05:33.787723+0000","last_became_peered":"2023-10-01T11:05:33.787723+0000","last_unstale":"2023-10-01T11:19:21.437635+0000","last_undegraded":"2023-10-01T11:19:21.437635+0000","last_fullsized":"2023-10-01T11:19:21.437635+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:04:55.138186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":369,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.176982+0000","last_change":"2023-10-01T11:19:02.159604+0000","last_active":"2023-10-01T11:19:21.176982+0000","last_peered":"2023-10-01T11:19:21.176982+0000","last_clean":"2023-10-01T11:19:21.176982+0000","last_became_active":"2023-10-01T11:19:02.158991+0000","last_became_peered":"2023-10-01T11:19:02.158991+0000","last_unstale":"2023-10-01T11:19:21.176982+0000","last_undegraded":"2023-10-01T11:19:21.176982+0000","last_fullsized":"2023-10-01T11:19:21.176982+0000","mapping_epoch":435,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":436,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"405'183","reported_seq":1149,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.188449+0000","last_change":"2023-10-01T11:19:22.188449+0000","last_active":"2023-10-01T11:19:22.188449+0000","last_peered":"2023-10-01T11:19:22.188449+0000","last_clean":"2023-10-01T11:19:22.188449+0000","last_became_active":"2023-10-01T11:19:22.187455+0000","last_became_peered":"2023-10-01T11:19:22.187455+0000","last_unstale":"2023-10-01T11:19:22.188449+0000","last_undegraded":"2023-10-01T11:19:22.188449+0000","last_fullsized":"2023-10-01T11:19:22.188449+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1514,"num_read_kb":1489,"num_write":145,"num_write_kb":150,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":240,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.175406+0000","last_change":"2023-10-01T11:17:02.474010+0000","last_active":"2023-10-01T11:19:21.175406+0000","last_peered":"2023-10-01T11:19:21.175406+0000","last_clean":"2023-10-01T11:19:21.175406+0000","last_became_active":"2023-10-01T11:17:02.473678+0000","last_became_peered":"2023-10-01T11:17:02.473678+0000","last_unstale":"2023-10-01T11:19:21.175406+0000","last_undegraded":"2023-10-01T11:19:21.175406+0000","last_fullsized":"2023-10-01T11:19:21.175406+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:18:59.211258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":37,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.184339+0000","last_change":"2023-10-01T11:16:18.985906+0000","last_active":"2023-10-01T11:19:22.184339+0000","last_peered":"2023-10-01T11:19:22.184339+0000","last_clean":"2023-10-01T11:19:22.184339+0000","last_became_active":"2023-10-01T11:16:18.985541+0000","last_became_peered":"2023-10-01T11:16:18.985541+0000","last_unstale":"2023-10-01T11:19:22.184339+0000","last_undegraded":"2023-10-01T11:19:22.184339+0000","last_fullsized":"2023-10-01T11:19:22.184339+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:49:19.304679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"405'105","reported_seq":739,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.174603+0000","last_change":"2023-10-01T11:16:22.012650+0000","last_active":"2023-10-01T11:19:21.174603+0000","last_peered":"2023-10-01T11:19:21.174603+0000","last_clean":"2023-10-01T11:19:21.174603+0000","last_became_active":"2023-10-01T11:16:22.011729+0000","last_became_peered":"2023-10-01T11:16:22.011729+0000","last_unstale":"2023-10-01T11:19:21.174603+0000","last_undegraded":"2023-10-01T11:19:21.174603+0000","last_fullsized":"2023-10-01T11:19:21.174603+0000","mapping_epoch":417,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":418,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":105,"log_dups_size":0,"ondisk_log_size":105,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":738,"num_read_kb":1646,"num_write":82,"num_write_kb":1055,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":150,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.183086+0000","last_change":"2023-10-01T11:17:22.401284+0000","last_active":"2023-10-01T11:19:22.183086+0000","last_peered":"2023-10-01T11:19:22.183086+0000","last_clean":"2023-10-01T11:19:22.183086+0000","last_became_active":"2023-10-01T11:17:22.400619+0000","last_became_peered":"2023-10-01T11:17:22.400619+0000","last_unstale":"2023-10-01T11:19:22.183086+0000","last_undegraded":"2023-10-01T11:19:22.183086+0000","last_fullsized":"2023-10-01T11:19:22.183086+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:20:40.635133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":431,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.175542+0000","last_change":"2023-10-01T11:17:02.200206+0000","last_active":"2023-10-01T11:19:21.175542+0000","last_peered":"2023-10-01T11:19:21.175542+0000","last_clean":"2023-10-01T11:19:21.175542+0000","last_became_active":"2023-10-01T11:17:02.199670+0000","last_became_peered":"2023-10-01T11:17:02.199670+0000","last_unstale":"2023-10-01T11:19:21.175542+0000","last_undegraded":"2023-10-01T11:19:21.175542+0000","last_fullsized":"2023-10-01T11:19:21.175542+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:11:51.310314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":482,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.166964+0000","last_change":"2023-10-01T11:16:18.999845+0000","last_active":"2023-10-01T11:19:21.166964+0000","last_peered":"2023-10-01T11:19:21.166964+0000","last_clean":"2023-10-01T11:19:21.166964+0000","last_became_active":"2023-10-01T11:16:18.978956+0000","last_became_peered":"2023-10-01T11:16:18.978956+0000","last_unstale":"2023-10-01T11:19:21.166964+0000","last_undegraded":"2023-10-01T11:19:21.166964+0000","last_fullsized":"2023-10-01T11:19:21.166964+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:33:27.070624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"405'163","reported_seq":725,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.182789+0000","last_change":"2023-10-01T11:17:22.007789+0000","last_active":"2023-10-01T11:19:22.182789+0000","last_peered":"2023-10-01T11:19:22.182789+0000","last_clean":"2023-10-01T11:19:22.182789+0000","last_became_active":"2023-10-01T11:17:22.007230+0000","last_became_peered":"2023-10-01T11:17:22.007230+0000","last_unstale":"2023-10-01T11:19:22.182789+0000","last_undegraded":"2023-10-01T11:19:22.182789+0000","last_fullsized":"2023-10-01T11:19:22.182789+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":163,"log_dups_size":0,"ondisk_log_size":163,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:43:53.181499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001122088,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":536,"num_read_kb":2851,"num_write":150,"num_write_kb":2326,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":457,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.173865+0000","last_change":"2023-10-01T11:16:18.980825+0000","last_active":"2023-10-01T11:19:22.173865+0000","last_peered":"2023-10-01T11:19:22.173865+0000","last_clean":"2023-10-01T11:19:22.173865+0000","last_became_active":"2023-10-01T11:16:18.979768+0000","last_became_peered":"2023-10-01T11:16:18.979768+0000","last_unstale":"2023-10-01T11:19:22.173865+0000","last_undegraded":"2023-10-01T11:19:22.173865+0000","last_fullsized":"2023-10-01T11:19:22.173865+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:45:26.288769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"405'28","reported_seq":206,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.176718+0000","last_change":"2023-10-01T11:15:39.032714+0000","last_active":"2023-10-01T11:19:21.176718+0000","last_peered":"2023-10-01T11:19:21.176718+0000","last_clean":"2023-10-01T11:19:21.176718+0000","last_became_active":"2023-10-01T11:06:11.818533+0000","last_became_peered":"2023-10-01T11:06:11.818533+0000","last_unstale":"2023-10-01T11:19:21.176718+0000","last_undegraded":"2023-10-01T11:19:21.176718+0000","last_fullsized":"2023-10-01T11:19:21.176718+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":61,"num_read_kb":1227,"num_write":33,"num_write_kb":2157,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":41,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.184526+0000","last_change":"2023-10-01T11:16:18.995911+0000","last_active":"2023-10-01T11:19:22.184526+0000","last_peered":"2023-10-01T11:19:22.184526+0000","last_clean":"2023-10-01T11:19:22.184526+0000","last_became_active":"2023-10-01T11:16:18.991018+0000","last_became_peered":"2023-10-01T11:16:18.991018+0000","last_unstale":"2023-10-01T11:19:22.184526+0000","last_undegraded":"2023-10-01T11:19:22.184526+0000","last_fullsized":"2023-10-01T11:19:22.184526+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:33:34.834635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":425,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.170316+0000","last_change":"2023-10-01T11:05:33.784531+0000","last_active":"2023-10-01T11:19:21.170316+0000","last_peered":"2023-10-01T11:19:21.170316+0000","last_clean":"2023-10-01T11:19:21.170316+0000","last_became_active":"2023-10-01T11:05:33.784167+0000","last_became_peered":"2023-10-01T11:05:33.784167+0000","last_unstale":"2023-10-01T11:19:21.170316+0000","last_undegraded":"2023-10-01T11:19:21.170316+0000","last_fullsized":"2023-10-01T11:19:21.170316+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:34:16.434841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":170,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.184205+0000","last_change":"2023-10-01T11:17:02.199637+0000","last_active":"2023-10-01T11:19:22.184205+0000","last_peered":"2023-10-01T11:19:22.184205+0000","last_clean":"2023-10-01T11:19:22.184205+0000","last_became_active":"2023-10-01T11:17:02.199115+0000","last_became_peered":"2023-10-01T11:17:02.199115+0000","last_unstale":"2023-10-01T11:19:22.184205+0000","last_undegraded":"2023-10-01T11:19:22.184205+0000","last_fullsized":"2023-10-01T11:19:22.184205+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:16:50.614626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":435,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.326848+0000","last_change":"2023-10-01T10:37:42.762360+0000","last_active":"2023-10-01T11:19:22.326848+0000","last_peered":"2023-10-01T11:19:22.326848+0000","last_clean":"2023-10-01T11:19:22.326848+0000","last_became_active":"2023-10-01T10:37:42.761936+0000","last_became_peered":"2023-10-01T10:37:42.761936+0000","last_unstale":"2023-10-01T11:19:22.326848+0000","last_undegraded":"2023-10-01T11:19:22.326848+0000","last_fullsized":"2023-10-01T11:19:22.326848+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:50.637582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":423,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.176914+0000","last_change":"2023-10-01T10:38:16.037809+0000","last_active":"2023-10-01T11:19:21.176914+0000","last_peered":"2023-10-01T11:19:21.176914+0000","last_clean":"2023-10-01T11:19:21.176914+0000","last_became_active":"2023-10-01T10:38:16.037318+0000","last_became_peered":"2023-10-01T10:38:16.037318+0000","last_unstale":"2023-10-01T11:19:21.176914+0000","last_undegraded":"2023-10-01T11:19:21.176914+0000","last_fullsized":"2023-10-01T11:19:21.176914+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":427,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.171222+0000","last_change":"2023-10-01T11:04:02.726007+0000","last_active":"2023-10-01T11:19:21.171222+0000","last_peered":"2023-10-01T11:19:21.171222+0000","last_clean":"2023-10-01T11:19:21.171222+0000","last_became_active":"2023-10-01T11:04:02.725507+0000","last_became_peered":"2023-10-01T11:04:02.725507+0000","last_unstale":"2023-10-01T11:19:21.171222+0000","last_undegraded":"2023-10-01T11:19:21.171222+0000","last_fullsized":"2023-10-01T11:19:21.171222+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:32:04.611533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":486,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.177015+0000","last_change":"2023-10-01T11:05:33.830108+0000","last_active":"2023-10-01T11:19:21.177015+0000","last_peered":"2023-10-01T11:19:21.177015+0000","last_clean":"2023-10-01T11:19:21.177015+0000","last_became_active":"2023-10-01T11:05:33.791882+0000","last_became_peered":"2023-10-01T11:05:33.791882+0000","last_unstale":"2023-10-01T11:19:21.177015+0000","last_undegraded":"2023-10-01T11:19:21.177015+0000","last_fullsized":"2023-10-01T11:19:21.177015+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":445,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.170488+0000","last_change":"2023-10-01T11:16:18.980782+0000","last_active":"2023-10-01T11:19:21.170488+0000","last_peered":"2023-10-01T11:19:21.170488+0000","last_clean":"2023-10-01T11:19:21.170488+0000","last_became_active":"2023-10-01T11:16:18.980097+0000","last_became_peered":"2023-10-01T11:16:18.980097+0000","last_unstale":"2023-10-01T11:19:21.170488+0000","last_undegraded":"2023-10-01T11:19:21.170488+0000","last_fullsized":"2023-10-01T11:19:21.170488+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:37:17.022077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":488,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.174783+0000","last_change":"2023-10-01T11:05:33.777019+0000","last_active":"2023-10-01T11:19:22.174783+0000","last_peered":"2023-10-01T11:19:22.174783+0000","last_clean":"2023-10-01T11:19:22.174783+0000","last_became_active":"2023-10-01T11:05:33.776565+0000","last_became_peered":"2023-10-01T11:05:33.776565+0000","last_unstale":"2023-10-01T11:19:22.174783+0000","last_undegraded":"2023-10-01T11:19:22.174783+0000","last_fullsized":"2023-10-01T11:19:22.174783+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T22:00:18.890326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":461,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.164141+0000","last_change":"2023-10-01T11:16:18.982759+0000","last_active":"2023-10-01T11:19:21.164141+0000","last_peered":"2023-10-01T11:19:21.164141+0000","last_clean":"2023-10-01T11:19:21.164141+0000","last_became_active":"2023-10-01T11:16:18.982413+0000","last_became_peered":"2023-10-01T11:16:18.982413+0000","last_unstale":"2023-10-01T11:19:21.164141+0000","last_undegraded":"2023-10-01T11:19:21.164141+0000","last_fullsized":"2023-10-01T11:19:21.164141+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:30:23.401194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":174,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.175807+0000","last_change":"2023-10-01T11:17:02.200873+0000","last_active":"2023-10-01T11:19:21.175807+0000","last_peered":"2023-10-01T11:19:21.175807+0000","last_clean":"2023-10-01T11:19:21.175807+0000","last_became_active":"2023-10-01T11:17:02.200271+0000","last_became_peered":"2023-10-01T11:17:02.200271+0000","last_unstale":"2023-10-01T11:19:21.175807+0000","last_undegraded":"2023-10-01T11:19:21.175807+0000","last_fullsized":"2023-10-01T11:19:21.175807+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:54:31.586661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":463,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.170093+0000","last_change":"2023-10-01T11:05:33.785774+0000","last_active":"2023-10-01T11:19:21.170093+0000","last_peered":"2023-10-01T11:19:21.170093+0000","last_clean":"2023-10-01T11:19:21.170093+0000","last_became_active":"2023-10-01T11:05:33.785426+0000","last_became_peered":"2023-10-01T11:05:33.785426+0000","last_unstale":"2023-10-01T11:19:21.170093+0000","last_undegraded":"2023-10-01T11:19:21.170093+0000","last_fullsized":"2023-10-01T11:19:21.170093+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:41:32.634339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":94,"num_bytes_recovered":34770,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30319,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":6494048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5985,"num_read_kb":13244,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":46,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158431019,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1487152,"kb_used_data":8424,"kb_used_omap":0,"kb_used_meta":1478720,"kb_avail":92258000,"statfs":{"total":95995035648,"available":94472192000,"internally_reserved":0,"allocated":8626176,"data_stored":7990409,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1514209280},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 11:19:56 2023","interfaces":[{"interface":"back","average":{"1min":0.92200000000000004,"5min":0.91400000000000003,"15min":1.242},"min":{"1min":0.55100000000000005,"5min":0.44,"15min":0.27100000000000002},"max":{"1min":1.208,"5min":1.2649999999999999,"15min":10.013},"last":1.1100000000000001},{"interface":"front","average":{"1min":0.98499999999999999,"5min":0.95499999999999996,"15min":1.2549999999999999},"min":{"1min":0.64600000000000002,"5min":0.60099999999999998,"15min":0.29599999999999999},"max":{"1min":1.2769999999999999,"5min":1.288,"15min":8.8949999999999996},"last":0.85399999999999998}]},{"osd":1,"last update":"Sun Oct 1 11:19:56 2023","interfaces":[{"interface":"back","average":{"1min":0.84799999999999998,"5min":0.89100000000000001,"15min":1.1559999999999999},"min":{"1min":0.63800000000000001,"5min":0.60899999999999999,"15min":0.28000000000000003},"max":{"1min":1.196,"5min":1.2270000000000001,"15min":9.6259999999999994},"last":0.751},{"interface":"front","average":{"1min":0.99199999999999999,"5min":0.99099999999999999,"15min":1.286},"min":{"1min":0.73199999999999998,"5min":0.61399999999999999,"15min":0.30499999999999999},"max":{"1min":1.3149999999999999,"5min":1.4990000000000001,"15min":9.8170000000000002},"last":1.198}]},{"osd":2,"last update":"Sun Oct 1 11:20:17 2023","interfaces":[{"interface":"back","average":{"1min":1.0129999999999999,"5min":0.996,"15min":1.325},"min":{"1min":0.68100000000000005,"5min":0.62,"15min":0.33200000000000002},"max":{"1min":1.339,"5min":1.5149999999999999,"15min":10.021000000000001},"last":1.139},{"interface":"front","average":{"1min":0.89100000000000001,"5min":0.96199999999999997,"15min":1.304},"min":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.32200000000000001},"max":{"1min":1.1579999999999999,"5min":1.3009999999999999,"15min":14.663},"last":1.042}]},{"osd":3,"last update":"Sun Oct 1 11:19:56 2023","interfaces":[{"interface":"back","average":{"1min":0.998,"5min":1.028,"15min":1.22},"min":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.34499999999999997},"max":{"1min":1.3049999999999999,"5min":1.403,"15min":12.358000000000001},"last":1.2410000000000001},{"interface":"front","average":{"1min":1.024,"5min":0.97799999999999998,"15min":1.1559999999999999},"min":{"1min":0.71999999999999997,"5min":0.57799999999999996,"15min":0.32500000000000001},"max":{"1min":1.2569999999999999,"5min":1.327,"15min":8.3109999999999999},"last":0.96799999999999997}]},{"osd":4,"last update":"Sun Oct 1 11:20:30 2023","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":0.96299999999999997,"15min":1.1739999999999999},"min":{"1min":0.61199999999999999,"5min":0.503,"15min":0.34999999999999998},"max":{"1min":1.1779999999999999,"5min":1.861,"15min":14.51},"last":1.0189999999999999},{"interface":"front","average":{"1min":1.151,"5min":1.1879999999999999,"15min":1.232},"min":{"1min":0.91600000000000004,"5min":0.66500000000000004,"15min":0.34799999999999998},"max":{"1min":1.4179999999999999,"5min":1.98,"15min":12.414999999999999},"last":1.2829999999999999}]},{"osd":5,"last update":"Sun Oct 1 11:19:56 2023","interfaces":[{"interface":"back","average":{"1min":1.083,"5min":1.0880000000000001,"15min":1.397},"min":{"1min":0.79600000000000004,"5min":0.52000000000000002,"15min":0.33100000000000002},"max":{"1min":1.391,"5min":1.583,"15min":18.998999999999999},"last":1.0880000000000001},{"interface":"front","average":{"1min":1.054,"5min":1.056,"15min":1.1859999999999999},"min":{"1min":0.79400000000000004,"5min":0.64800000000000002,"15min":0.36899999999999999},"max":{"1min":1.367,"5min":1.4490000000000001,"15min":10.1},"last":1.169}]},{"osd":6,"last update":"Sun Oct 1 11:19:56 2023","interfaces":[{"interface":"back","average":{"1min":1.1439999999999999,"5min":1.153,"15min":1.282},"min":{"1min":0.89800000000000002,"5min":0.60399999999999998,"15min":0.38500000000000001},"max":{"1min":1.361,"5min":1.601,"15min":13.038},"last":1.323},{"interface":"front","average":{"1min":1.099,"5min":1.1020000000000001,"15min":1.2310000000000001},"min":{"1min":0.81599999999999995,"5min":0.56999999999999995,"15min":0.32400000000000001},"max":{"1min":1.3660000000000001,"5min":1.393,"15min":11.536},"last":0.94199999999999995}]}]},{"osd":6,"up_from":42,"seq":180388627259,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1181604,"kb_used_data":4316,"kb_used_omap":0,"kb_used_meta":1177280,"kb_avail":92563548,"statfs":{"total":95995035648,"available":94785073152,"internally_reserved":0,"allocated":4419584,"data_stored":3795663,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1205534720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 11:19:53 2023","interfaces":[{"interface":"back","average":{"1min":0.94599999999999995,"5min":0.88600000000000001,"15min":1.2470000000000001},"min":{"1min":0.61499999999999999,"5min":0.435,"15min":0.27000000000000002},"max":{"1min":1.2270000000000001,"5min":1.4199999999999999,"15min":14.964},"last":0.79100000000000004},{"interface":"front","average":{"1min":0.95299999999999996,"5min":0.94599999999999995,"15min":1.1799999999999999},"min":{"1min":0.79000000000000004,"5min":0.46200000000000002,"15min":0.26500000000000001},"max":{"1min":1.5580000000000001,"5min":3.327,"15min":9.3849999999999998},"last":1.093}]},{"osd":1,"last update":"Sun Oct 1 11:20:16 2023","interfaces":[{"interface":"back","average":{"1min":1.012,"5min":0.93400000000000005,"15min":1.2649999999999999},"min":{"1min":0.63800000000000001,"5min":0.56799999999999995,"15min":0.28199999999999997},"max":{"1min":1.264,"5min":1.3560000000000001,"15min":14.946999999999999},"last":0.98099999999999998},{"interface":"front","average":{"1min":0.97599999999999998,"5min":0.97299999999999998,"15min":1.115},"min":{"1min":0.63300000000000001,"5min":0.59699999999999998,"15min":0.31},"max":{"1min":1.278,"5min":1.5469999999999999,"15min":8.1159999999999997},"last":1.2809999999999999}]},{"osd":2,"last update":"Sun Oct 1 11:20:26 2023","interfaces":[{"interface":"back","average":{"1min":0.84899999999999998,"5min":0.90700000000000003,"15min":1.718},"min":{"1min":0.58999999999999997,"5min":0.47399999999999998,"15min":0.29799999999999999},"max":{"1min":1.1479999999999999,"5min":1.3300000000000001,"15min":87.287999999999997},"last":0.89600000000000002},{"interface":"front","average":{"1min":0.97699999999999998,"5min":0.97399999999999998,"15min":1.738},"min":{"1min":0.71999999999999997,"5min":0.54600000000000004,"15min":0.32300000000000001},"max":{"1min":1.1879999999999999,"5min":1.7110000000000001,"15min":88.028000000000006},"last":1.024}]},{"osd":3,"last update":"Sun Oct 1 11:20:16 2023","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":0.98299999999999998,"15min":1.3899999999999999},"min":{"1min":0.61599999999999999,"5min":0.55100000000000005,"15min":0.30399999999999999},"max":{"1min":1.304,"5min":1.4379999999999999,"15min":50.225000000000001},"last":1.0660000000000001},{"interface":"front","average":{"1min":0.998,"5min":0.98399999999999999,"15min":1.3859999999999999},"min":{"1min":0.79100000000000004,"5min":0.624,"15min":0.33900000000000002},"max":{"1min":1.22,"5min":1.4419999999999999,"15min":49.301000000000002},"last":1.1619999999999999}]},{"osd":4,"last update":"Sun Oct 1 11:19:38 2023","interfaces":[{"interface":"back","average":{"1min":0.98699999999999999,"5min":0.95699999999999996,"15min":1.1519999999999999},"min":{"1min":0.50800000000000001,"5min":0.47799999999999998,"15min":0.315},"max":{"1min":1.3480000000000001,"5min":1.4450000000000001,"15min":8.7210000000000001},"last":1.214},{"interface":"front","average":{"1min":1.0369999999999999,"5min":1.052,"15min":1.3460000000000001},"min":{"1min":0.67300000000000004,"5min":0.59099999999999997,"15min":0.35399999999999998},"max":{"1min":1.258,"5min":1.5169999999999999,"15min":15.462},"last":1.2390000000000001}]},{"osd":5,"last update":"Sun Oct 1 11:20:16 2023","interfaces":[{"interface":"back","average":{"1min":1.1000000000000001,"5min":1.0409999999999999,"15min":1.306},"min":{"1min":0.70899999999999996,"5min":0.49099999999999999,"15min":0.33100000000000002},"max":{"1min":1.3600000000000001,"5min":1.488,"15min":26.379000000000001},"last":1.1379999999999999},{"interface":"front","average":{"1min":1.1180000000000001,"5min":1.085,"15min":1.4359999999999999},"min":{"1min":0.88600000000000001,"5min":0.65200000000000002,"15min":0.32000000000000001},"max":{"1min":1.3959999999999999,"5min":1.468,"15min":28.010000000000002},"last":1.379}]},{"osd":7,"last update":"Sun Oct 1 11:19:53 2023","interfaces":[{"interface":"back","average":{"1min":1.1279999999999999,"5min":1.103,"15min":1.173},"min":{"1min":0.89600000000000002,"5min":0.49099999999999999,"15min":0.35499999999999998},"max":{"1min":1.4890000000000001,"5min":1.585,"15min":8.1370000000000005},"last":1.333},{"interface":"front","average":{"1min":1.151,"5min":1.115,"15min":1.274},"min":{"1min":0.84099999999999997,"5min":0.51000000000000001,"15min":0.34999999999999998},"max":{"1min":1.3979999999999999,"5min":1.5589999999999999,"15min":14.894},"last":1.266}]}]},{"osd":1,"up_from":101,"seq":433791697477,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1003772,"kb_used_data":9012,"kb_used_omap":4,"kb_used_meta":994747,"kb_avail":92741380,"statfs":{"total":95995035648,"available":94967173120,"internally_reserved":0,"allocated":9228288,"data_stored":7548147,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":1018621838},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 11:20:13 2023","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.66800000000000004,"15min":0.92300000000000004},"min":{"1min":0.307,"5min":0.307,"15min":0.19800000000000001},"max":{"1min":0.91400000000000003,"5min":1.244,"15min":11.756},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.71399999999999997,"5min":0.70499999999999996,"15min":1.161},"min":{"1min":0.46899999999999997,"5min":0.309,"15min":0.215},"max":{"1min":1.7190000000000001,"5min":1.7190000000000001,"15min":11.943},"last":0.66900000000000004}]},{"osd":2,"last update":"Sun Oct 1 11:20:38 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.72999999999999998,"15min":2.0070000000000001},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.23400000000000001},"max":{"1min":0.97999999999999998,"5min":1.1579999999999999,"15min":130.90899999999999},"last":0.97999999999999998},{"interface":"front","average":{"1min":0.73699999999999999,"5min":0.80100000000000005,"15min":2.117},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.29199999999999998},"max":{"1min":1.526,"5min":1.526,"15min":130.72800000000001},"last":1.0720000000000001}]},{"osd":3,"last update":"Sun Oct 1 11:20:13 2023","interfaces":[{"interface":"back","average":{"1min":0.72399999999999998,"5min":0.72499999999999998,"15min":2.0419999999999998},"min":{"1min":0.379,"5min":0.379,"15min":0.249},"max":{"1min":1.0049999999999999,"5min":1.1519999999999999,"15min":130.79599999999999},"last":0.74099999999999999},{"interface":"front","average":{"1min":0.68400000000000005,"5min":0.75700000000000001,"15min":2.323},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.219},"max":{"1min":1.319,"5min":1.47,"15min":130.83500000000001},"last":0.77400000000000002}]},{"osd":4,"last update":"Sun Oct 1 11:20:38 2023","interfaces":[{"interface":"back","average":{"1min":0.90900000000000003,"5min":0.97599999999999998,"15min":2.4670000000000001},"min":{"1min":0.66200000000000003,"5min":0.56699999999999995,"15min":0.33300000000000002},"max":{"1min":1.2589999999999999,"5min":1.456,"15min":203.89500000000001},"last":0.82099999999999995},{"interface":"front","average":{"1min":0.93500000000000005,"5min":0.998,"15min":2.8210000000000002},"min":{"1min":0.71599999999999997,"5min":0.67600000000000005,"15min":0.32600000000000001},"max":{"1min":1.6579999999999999,"5min":1.6579999999999999,"15min":203.852},"last":1.002}]},{"osd":5,"last update":"Sun Oct 1 11:20:13 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":1.038,"15min":1.321},"min":{"1min":0.67800000000000005,"5min":0.59699999999999998,"15min":0.33600000000000002},"max":{"1min":1.5860000000000001,"5min":1.696,"15min":12.192},"last":1.04},{"interface":"front","average":{"1min":1.0580000000000001,"5min":1.069,"15min":1.7330000000000001},"min":{"1min":0.79700000000000004,"5min":0.627,"15min":0.35299999999999998},"max":{"1min":1.363,"5min":1.5669999999999999,"15min":130.62299999999999},"last":1.1160000000000001}]},{"osd":6,"last update":"Sun Oct 1 11:20:13 2023","interfaces":[{"interface":"back","average":{"1min":1.0800000000000001,"5min":1.093,"15min":1.7569999999999999},"min":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.371},"max":{"1min":1.3480000000000001,"5min":1.6619999999999999,"15min":130.75299999999999},"last":0.93500000000000005},{"interface":"front","average":{"1min":1.071,"5min":1.1080000000000001,"15min":1.696},"min":{"1min":0.73399999999999999,"5min":0.52400000000000002,"15min":0.35599999999999998},"max":{"1min":1.772,"5min":1.772,"15min":58.238},"last":1.103}]},{"osd":7,"last update":"Sun Oct 1 11:20:13 2023","interfaces":[{"interface":"back","average":{"1min":1.113,"5min":1.1259999999999999,"15min":1.496},"min":{"1min":0.77300000000000002,"5min":0.64100000000000001,"15min":0.40100000000000002},"max":{"1min":1.8169999999999999,"5min":1.8169999999999999,"15min":12.055999999999999},"last":1.1539999999999999},{"interface":"front","average":{"1min":1.0840000000000001,"5min":1.1339999999999999,"15min":1.913},"min":{"1min":0.78200000000000003,"5min":0.68899999999999995,"15min":0.40500000000000003},"max":{"1min":1.677,"5min":1.7669999999999999,"15min":130.42099999999999},"last":0.88800000000000001}]}]},{"osd":0,"up_from":86,"seq":369367188087,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1061996,"kb_used_data":9508,"kb_used_omap":5,"kb_used_meta":1052474,"kb_avail":92683156,"statfs":{"total":95995035648,"available":94907551744,"internally_reserved":0,"allocated":9736192,"data_stored":8125705,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1077734010},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 11:20:31 2023","interfaces":[{"interface":"back","average":{"1min":0.67500000000000004,"5min":0.68899999999999995,"15min":1.0820000000000001},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.186},"max":{"1min":1.6639999999999999,"5min":1.6639999999999999,"15min":12.433},"last":0.73299999999999998},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.68600000000000005,"15min":1.4990000000000001},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.189},"max":{"1min":1.0980000000000001,"5min":1.0980000000000001,"15min":13.989000000000001},"last":0.80800000000000005}]},{"osd":2,"last update":"Sun Oct 1 11:20:31 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.68899999999999995,"15min":1.595},"min":{"1min":0.248,"5min":0.248,"15min":0.23000000000000001},"max":{"1min":1.484,"5min":1.484,"15min":67.076999999999998},"last":1.2350000000000001},{"interface":"front","average":{"1min":0.71399999999999997,"5min":0.79000000000000004,"15min":1.7949999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.247},"max":{"1min":1.2809999999999999,"5min":1.3029999999999999,"15min":67.129999999999995},"last":0.94199999999999995}]},{"osd":3,"last update":"Sun Oct 1 11:20:31 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":0.81100000000000005,"15min":1.417},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.251},"max":{"1min":1.0169999999999999,"5min":1.232,"15min":67.706000000000003},"last":0.76800000000000002},{"interface":"front","average":{"1min":0.74299999999999999,"5min":0.79300000000000004,"15min":1.536},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.254},"max":{"1min":1.722,"5min":1.722,"15min":66.352000000000004},"last":0.85799999999999998}]},{"osd":4,"last update":"Sun Oct 1 11:19:48 2023","interfaces":[{"interface":"back","average":{"1min":1.022,"5min":1.032,"15min":1.4339999999999999},"min":{"1min":0.78300000000000003,"5min":0.40400000000000003,"15min":0.35299999999999998},"max":{"1min":1.5429999999999999,"5min":1.675,"15min":23.995000000000001},"last":1.2689999999999999},{"interface":"front","average":{"1min":0.995,"5min":0.996,"15min":1.5620000000000001},"min":{"1min":0.80200000000000005,"5min":0.54900000000000004,"15min":0.33600000000000002},"max":{"1min":1.431,"5min":1.7649999999999999,"15min":21.451000000000001},"last":0.91400000000000003}]},{"osd":5,"last update":"Sun Oct 1 11:20:31 2023","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":1.0569999999999999,"15min":1.4550000000000001},"min":{"1min":0.44,"5min":0.44,"15min":0.35099999999999998},"max":{"1min":1.3400000000000001,"5min":1.728,"15min":12.776999999999999},"last":1.0669999999999999},{"interface":"front","average":{"1min":1.048,"5min":1.1140000000000001,"15min":1.5880000000000001},"min":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.38200000000000001},"max":{"1min":1.9079999999999999,"5min":1.9079999999999999,"15min":14.529},"last":1.099}]},{"osd":6,"last update":"Sun Oct 1 11:20:31 2023","interfaces":[{"interface":"back","average":{"1min":1.048,"5min":1.1120000000000001,"15min":1.3899999999999999},"min":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.373},"max":{"1min":1.853,"5min":1.853,"15min":10.430999999999999},"last":1.345},{"interface":"front","average":{"1min":1.002,"5min":1.0569999999999999,"15min":1.4239999999999999},"min":{"1min":0.64400000000000002,"5min":0.51400000000000001,"15min":0.373},"max":{"1min":1.482,"5min":1.5900000000000001,"15min":12.035},"last":1.038}]},{"osd":7,"last update":"Sun Oct 1 11:20:31 2023","interfaces":[{"interface":"back","average":{"1min":1.0840000000000001,"5min":1.1759999999999999,"15min":1.3520000000000001},"min":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.42999999999999999},"max":{"1min":1.9510000000000001,"5min":1.9510000000000001,"15min":10.553000000000001},"last":1.3939999999999999},{"interface":"front","average":{"1min":1.042,"5min":1.111,"15min":1.5309999999999999},"min":{"1min":0.76800000000000002,"5min":0.65400000000000003,"15min":0.41499999999999998},"max":{"1min":1.448,"5min":1.6970000000000001,"15min":12.053000000000001},"last":1.1439999999999999}]}]},{"osd":2,"up_from":116,"seq":498216206861,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":920228,"kb_used_data":4828,"kb_used_omap":7,"kb_used_meta":915384,"kb_avail":92824924,"statfs":{"total":95995035648,"available":95052722176,"internally_reserved":0,"allocated":4943872,"data_stored":3340074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":937353297},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 11:20:27 2023","interfaces":[{"interface":"back","average":{"1min":0.67300000000000004,"5min":0.66100000000000003,"15min":0.90900000000000003},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.20799999999999999},"max":{"1min":2.0760000000000001,"5min":2.0760000000000001,"15min":13.429},"last":0.73499999999999999},{"interface":"front","average":{"1min":0.68100000000000005,"5min":0.73699999999999999,"15min":1.444},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.20899999999999999},"max":{"1min":1.748,"5min":1.748,"15min":95.688999999999993},"last":0.67900000000000005}]},{"osd":1,"last update":"Sun Oct 1 11:20:27 2023","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.73099999999999998,"15min":1.429},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.254},"max":{"1min":1.123,"5min":1.24,"15min":34.753999999999998},"last":0.89500000000000002},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.70299999999999996,"15min":1.4370000000000001},"min":{"1min":0.314,"5min":0.314,"15min":0.27200000000000002},"max":{"1min":0.95099999999999996,"5min":1.175,"15min":34.972999999999999},"last":0.82999999999999996}]},{"osd":3,"last update":"Sun Oct 1 11:20:03 2023","interfaces":[{"interface":"back","average":{"1min":0.71799999999999997,"5min":0.72099999999999997,"15min":1.0680000000000001},"min":{"1min":0.35999999999999999,"5min":0.34799999999999998,"15min":0.24399999999999999},"max":{"1min":1.0069999999999999,"5min":1.141,"15min":13.744},"last":0.872},{"interface":"front","average":{"1min":0.85599999999999998,"5min":0.83599999999999997,"15min":1.1779999999999999},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.21099999999999999},"max":{"1min":1.1930000000000001,"5min":1.4199999999999999,"15min":10.102},"last":0.97199999999999998}]},{"osd":4,"last update":"Sun Oct 1 11:19:43 2023","interfaces":[{"interface":"back","average":{"1min":1.081,"5min":1.0189999999999999,"15min":1.2769999999999999},"min":{"1min":0.77100000000000002,"5min":0.51400000000000001,"15min":0.312},"max":{"1min":1.2669999999999999,"5min":1.575,"15min":10.004},"last":1.0449999999999999},{"interface":"front","average":{"1min":0.97599999999999998,"5min":0.94399999999999995,"15min":1.538},"min":{"1min":0.78800000000000003,"5min":0.44700000000000001,"15min":0.33000000000000002},"max":{"1min":1.2370000000000001,"5min":1.327,"15min":12.989000000000001},"last":0.995}]},{"osd":5,"last update":"Sun Oct 1 11:20:27 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":1.0669999999999999,"15min":1.3120000000000001},"min":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.33600000000000002},"max":{"1min":1.383,"5min":1.694,"15min":9.968},"last":1.147},{"interface":"front","average":{"1min":1.032,"5min":1.0389999999999999,"15min":1.55},"min":{"1min":0.504,"5min":0.504,"15min":0.33100000000000002},"max":{"1min":2.0009999999999999,"5min":2.0009999999999999,"15min":12.518000000000001},"last":1.1619999999999999}]},{"osd":6,"last update":"Sun Oct 1 11:20:27 2023","interfaces":[{"interface":"back","average":{"1min":1.0329999999999999,"5min":1.048,"15min":1.742},"min":{"1min":0.70699999999999996,"5min":0.69399999999999995,"15min":0.39300000000000002},"max":{"1min":1.885,"5min":1.885,"15min":95.802999999999997},"last":1.081},{"interface":"front","average":{"1min":1.034,"5min":1.0820000000000001,"15min":1.278},"min":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.33700000000000002},"max":{"1min":1.573,"5min":1.573,"15min":10.007},"last":1.355}]},{"osd":7,"last update":"Sun Oct 1 11:20:27 2023","interfaces":[{"interface":"back","average":{"1min":1.0900000000000001,"5min":1.1339999999999999,"15min":1.276},"min":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.36699999999999999},"max":{"1min":1.6830000000000001,"5min":1.6830000000000001,"15min":10.053000000000001},"last":1.4359999999999999},{"interface":"front","average":{"1min":1.0720000000000001,"5min":1.1539999999999999,"15min":1.671},"min":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.35199999999999998},"max":{"1min":1.639,"5min":1.754,"15min":95.228999999999999},"last":1.238}]}]},{"osd":3,"up_from":121,"seq":519691043334,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":858484,"kb_used_data":4844,"kb_used_omap":2,"kb_used_meta":853629,"kb_avail":92886668,"statfs":{"total":95995035648,"available":95115948032,"internally_reserved":0,"allocated":4960256,"data_stored":3341934,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":874116149},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 11:19:58 2023","interfaces":[{"interface":"back","average":{"1min":0.71999999999999997,"5min":0.69899999999999995,"15min":2.0059999999999998},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.222},"max":{"1min":1.333,"5min":1.333,"15min":259.98599999999999},"last":0.89200000000000002},{"interface":"front","average":{"1min":0.72499999999999998,"5min":0.69299999999999995,"15min":2.0640000000000001},"min":{"1min":0.44500000000000001,"5min":0.32500000000000001,"15min":0.217},"max":{"1min":1.115,"5min":1.115,"15min":259.93000000000001},"last":1.008}]},{"osd":1,"last update":"Sun Oct 1 11:19:58 2023","interfaces":[{"interface":"back","average":{"1min":0.75700000000000001,"5min":0.754,"15min":2.2629999999999999},"min":{"1min":0.45200000000000001,"5min":0.37,"15min":0.23999999999999999},"max":{"1min":1.1499999999999999,"5min":1.3979999999999999,"15min":259.80200000000002},"last":0.78000000000000003},{"interface":"front","average":{"1min":0.77400000000000002,"5min":0.83799999999999997,"15min":2.2040000000000002},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.22},"max":{"1min":1.4730000000000001,"5min":1.4730000000000001,"15min":260.08499999999998},"last":1.163}]},{"osd":2,"last update":"Sun Oct 1 11:19:58 2023","interfaces":[{"interface":"back","average":{"1min":0.755,"5min":0.71899999999999997,"15min":2.9710000000000001},"min":{"1min":0.42299999999999999,"5min":0.32300000000000001,"15min":0.22600000000000001},"max":{"1min":1.2569999999999999,"5min":1.2569999999999999,"15min":259.83600000000001},"last":0.84099999999999997},{"interface":"front","average":{"1min":0.89900000000000002,"5min":0.91600000000000004,"15min":2.8180000000000001},"min":{"1min":0.436,"5min":0.436,"15min":0.29299999999999998},"max":{"1min":1.3320000000000001,"5min":1.389,"15min":260.13200000000001},"last":1.288}]},{"osd":4,"last update":"Sun Oct 1 11:19:49 2023","interfaces":[{"interface":"back","average":{"1min":1.121,"5min":3.8130000000000002,"15min":2.105},"min":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.32100000000000001},"max":{"1min":1.6040000000000001,"5min":260.173,"15min":260.173},"last":1.355},{"interface":"front","average":{"1min":1.097,"5min":3.7429999999999999,"15min":2.1680000000000001},"min":{"1min":0.74199999999999999,"5min":0.51300000000000001,"15min":0.31900000000000001},"max":{"1min":1.466,"5min":260.18400000000003,"15min":260.18400000000003},"last":1.3939999999999999}]},{"osd":5,"last update":"Sun Oct 1 11:19:58 2023","interfaces":[{"interface":"back","average":{"1min":1.0349999999999999,"5min":0.996,"15min":2.2250000000000001},"min":{"1min":0.57399999999999995,"5min":0.53900000000000003,"15min":0.35599999999999998},"max":{"1min":1.391,"5min":1.514,"15min":260.11500000000001},"last":1.2030000000000001},{"interface":"front","average":{"1min":1.0720000000000001,"5min":1.0089999999999999,"15min":2.4670000000000001},"min":{"1min":0.61099999999999999,"5min":0.59899999999999998,"15min":0.35799999999999998},"max":{"1min":2.3500000000000001,"5min":2.3500000000000001,"15min":260.38799999999998},"last":1.0900000000000001}]},{"osd":6,"last update":"Sun Oct 1 11:19:58 2023","interfaces":[{"interface":"back","average":{"1min":1.1819999999999999,"5min":1.081,"15min":2.2890000000000001},"min":{"1min":0.75600000000000001,"5min":0.57999999999999996,"15min":0.32100000000000001},"max":{"1min":2.3100000000000001,"5min":2.3100000000000001,"15min":260.24000000000001},"last":1.4850000000000001},{"interface":"front","average":{"1min":1.048,"5min":1.0880000000000001,"15min":2.399},"min":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.40400000000000003},"max":{"1min":1.544,"5min":1.5660000000000001,"15min":260.45100000000002},"last":1.452}]},{"osd":7,"last update":"Sun Oct 1 11:19:58 2023","interfaces":[{"interface":"back","average":{"1min":1.121,"5min":1.0940000000000001,"15min":2.3730000000000002},"min":{"1min":0.76700000000000002,"5min":0.59099999999999997,"15min":0.39100000000000001},"max":{"1min":1.603,"5min":1.603,"15min":260.51799999999997},"last":1.365},{"interface":"front","average":{"1min":1.159,"5min":1.1439999999999999,"15min":2.1680000000000001},"min":{"1min":0.69399999999999995,"5min":0.61199999999999999,"15min":0.38},"max":{"1min":1.6180000000000001,"5min":1.7070000000000001,"15min":260.48200000000003},"last":1.429}]}]},{"osd":4,"up_from":126,"seq":541165879808,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":750736,"kb_used_data":4872,"kb_used_omap":10,"kb_used_meta":745845,"kb_avail":92994416,"statfs":{"total":95995035648,"available":95226281984,"internally_reserved":0,"allocated":4988928,"data_stored":3343266,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":763745892},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 11:20:33 2023","interfaces":[{"interface":"back","average":{"1min":0.97299999999999998,"5min":0.94399999999999995,"15min":1.0980000000000001},"min":{"1min":0.55300000000000005,"5min":0.47599999999999998,"15min":0.28000000000000003},"max":{"1min":1.2130000000000001,"5min":1.8859999999999999,"15min":11.025},"last":1.0309999999999999},{"interface":"front","average":{"1min":1.018,"5min":0.97399999999999998,"15min":1.2},"min":{"1min":0.68700000000000006,"5min":0.48999999999999999,"15min":0.26400000000000001},"max":{"1min":1.548,"5min":1.8109999999999999,"15min":12.369},"last":1.2809999999999999}]},{"osd":1,"last update":"Sun Oct 1 11:20:33 2023","interfaces":[{"interface":"back","average":{"1min":0.91000000000000003,"5min":0.95499999999999996,"15min":1.274},"min":{"1min":0.69099999999999995,"5min":0.56399999999999995,"15min":0.30199999999999999},"max":{"1min":1.252,"5min":1.3300000000000001,"15min":9.1920000000000002},"last":0.96999999999999997},{"interface":"front","average":{"1min":1.0109999999999999,"5min":0.99199999999999999,"15min":1.1719999999999999},"min":{"1min":0.59399999999999997,"5min":0.50600000000000001,"15min":0.26300000000000001},"max":{"1min":1.4570000000000001,"5min":2.0550000000000002,"15min":11.145},"last":1.204}]},{"osd":2,"last update":"Sun Oct 1 11:20:33 2023","interfaces":[{"interface":"back","average":{"1min":0.98199999999999998,"5min":1.0069999999999999,"15min":2.2410000000000001},"min":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.34699999999999998},"max":{"1min":1.3420000000000001,"5min":1.972,"15min":219.05500000000001},"last":0.91000000000000003},{"interface":"front","average":{"1min":0.99399999999999999,"5min":0.99299999999999999,"15min":2.1139999999999999},"min":{"1min":0.63600000000000001,"5min":0.55800000000000005,"15min":0.33900000000000002},"max":{"1min":1.595,"5min":2.02,"15min":218.81200000000001},"last":0.68700000000000006}]},{"osd":3,"last update":"Sun Oct 1 11:20:33 2023","interfaces":[{"interface":"back","average":{"1min":1.0389999999999999,"5min":1.0549999999999999,"15min":1.4790000000000001},"min":{"1min":0.69299999999999995,"5min":0.495,"15min":0.34200000000000003},"max":{"1min":1.431,"5min":2.1320000000000001,"15min":18.475999999999999},"last":0.81599999999999995},{"interface":"front","average":{"1min":1.04,"5min":0.99099999999999999,"15min":1.518},"min":{"1min":0.67700000000000005,"5min":0.61299999999999999,"15min":0.33500000000000002},"max":{"1min":1.4139999999999999,"5min":1.4139999999999999,"15min":18.303000000000001},"last":1.0509999999999999}]},{"osd":5,"last update":"Sun Oct 1 11:20:33 2023","interfaces":[{"interface":"back","average":{"1min":1.073,"5min":1.0369999999999999,"15min":1.3759999999999999},"min":{"1min":0.65200000000000002,"5min":0.57099999999999995,"15min":0.30299999999999999},"max":{"1min":1.502,"5min":2.165,"15min":25.175999999999998},"last":1.411},{"interface":"front","average":{"1min":1.077,"5min":1.0589999999999999,"15min":1.413},"min":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.307},"max":{"1min":1.379,"5min":2.097,"15min":11.084},"last":1.3280000000000001}]},{"osd":6,"last update":"Sun Oct 1 11:20:33 2023","interfaces":[{"interface":"back","average":{"1min":1.125,"5min":1.0840000000000001,"15min":1.3009999999999999},"min":{"1min":0.78400000000000003,"5min":0.66600000000000004,"15min":0.35199999999999998},"max":{"1min":1.5740000000000001,"5min":2.1899999999999999,"15min":12.635999999999999},"last":1.087},{"interface":"front","average":{"1min":1.0649999999999999,"5min":1.0669999999999999,"15min":1.343},"min":{"1min":0.71099999999999997,"5min":0.63,"15min":0.32500000000000001},"max":{"1min":1.363,"5min":2.3199999999999998,"15min":10.506},"last":1.171}]},{"osd":7,"last update":"Sun Oct 1 11:20:33 2023","interfaces":[{"interface":"back","average":{"1min":1.1040000000000001,"5min":1.0980000000000001,"15min":1.4059999999999999},"min":{"1min":0.83099999999999996,"5min":0.64900000000000002,"15min":0.376},"max":{"1min":1.4059999999999999,"5min":1.879,"15min":10.851000000000001},"last":1.2430000000000001},{"interface":"front","average":{"1min":1.2529999999999999,"5min":1.2050000000000001,"15min":1.359},"min":{"1min":0.71499999999999997,"5min":0.61599999999999999,"15min":0.40500000000000003},"max":{"1min":1.579,"5min":2.1600000000000001,"15min":12.364000000000001},"last":1.383}]}]},{"osd":5,"up_from":36,"seq":154618823489,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1494104,"kb_used_data":3728,"kb_used_omap":0,"kb_used_meta":1490368,"kb_avail":92251048,"statfs":{"total":95995035648,"available":94465073152,"internally_reserved":0,"allocated":3817472,"data_stored":3201115,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1526136832},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 11:19:59 2023","interfaces":[{"interface":"back","average":{"1min":0.93100000000000005,"5min":0.93999999999999995,"15min":1.599},"min":{"1min":0.56200000000000006,"5min":0.53200000000000003,"15min":0.27100000000000002},"max":{"1min":1.2370000000000001,"5min":4.2619999999999996,"15min":111.797},"last":0.90300000000000002},{"interface":"front","average":{"1min":0.88800000000000001,"5min":0.93500000000000005,"15min":1.677},"min":{"1min":0.60199999999999998,"5min":0.51800000000000002,"15min":0.251},"max":{"1min":1.1930000000000001,"5min":4.2229999999999999,"15min":111.492},"last":1.069}]},{"osd":1,"last update":"Sun Oct 1 11:19:59 2023","interfaces":[{"interface":"back","average":{"1min":0.97499999999999998,"5min":1.0049999999999999,"15min":2.1339999999999999},"min":{"1min":0.57699999999999996,"5min":0.46200000000000002,"15min":0.26300000000000001},"max":{"1min":1.6020000000000001,"5min":4.375,"15min":132.99600000000001},"last":1.0389999999999999},{"interface":"front","average":{"1min":1.0600000000000001,"5min":1.024,"15min":2.2919999999999998},"min":{"1min":0.70299999999999996,"5min":0.49199999999999999,"15min":0.27500000000000002},"max":{"1min":1.3160000000000001,"5min":4.3369999999999997,"15min":133.42400000000001},"last":0.99199999999999999}]},{"osd":2,"last update":"Sun Oct 1 11:20:26 2023","interfaces":[{"interface":"back","average":{"1min":0.98199999999999998,"5min":1.044,"15min":2.2519999999999998},"min":{"1min":0.63600000000000001,"5min":0.59899999999999998,"15min":0.309},"max":{"1min":1.351,"5min":4.327,"15min":129.70500000000001},"last":0.75900000000000001},{"interface":"front","average":{"1min":0.95499999999999996,"5min":1.0129999999999999,"15min":2.5219999999999998},"min":{"1min":0.61399999999999999,"5min":0.61099999999999999,"15min":0.28799999999999998},"max":{"1min":1.2430000000000001,"5min":4.3890000000000002,"15min":129.76300000000001},"last":1.179}]},{"osd":3,"last update":"Sun Oct 1 11:19:59 2023","interfaces":[{"interface":"back","average":{"1min":1.022,"5min":1.0189999999999999,"15min":1.8300000000000001},"min":{"1min":0.82099999999999995,"5min":0.57199999999999995,"15min":0.34499999999999997},"max":{"1min":1.232,"5min":4.444,"15min":131.09399999999999},"last":1.248},{"interface":"front","average":{"1min":1.0049999999999999,"5min":1.026,"15min":1.6519999999999999},"min":{"1min":0.75600000000000001,"5min":0.67100000000000004,"15min":0.36299999999999999},"max":{"1min":1.4079999999999999,"5min":4.7489999999999997,"15min":130.70699999999999},"last":0.83099999999999996}]},{"osd":4,"last update":"Sun Oct 1 11:20:26 2023","interfaces":[{"interface":"back","average":{"1min":1.054,"5min":1.0509999999999999,"15min":1.236},"min":{"1min":0.80400000000000005,"5min":0.48599999999999999,"15min":0.36799999999999999},"max":{"1min":1.28,"5min":4.1820000000000004,"15min":10.167},"last":1.0129999999999999},{"interface":"front","average":{"1min":1.147,"5min":1.107,"15min":1.3400000000000001},"min":{"1min":0.74399999999999999,"5min":0.54200000000000004,"15min":0.34100000000000003},"max":{"1min":1.784,"5min":5.444,"15min":11.162000000000001},"last":1.143}]},{"osd":6,"last update":"Sun Oct 1 11:19:59 2023","interfaces":[{"interface":"back","average":{"1min":1.113,"5min":1.123,"15min":1.1870000000000001},"min":{"1min":0.82899999999999996,"5min":0.52200000000000002,"15min":0.33100000000000002},"max":{"1min":1.3460000000000001,"5min":5.0419999999999998,"15min":11.196999999999999},"last":1.296},{"interface":"front","average":{"1min":1.115,"5min":1.1000000000000001,"15min":1.1519999999999999},"min":{"1min":0.91700000000000004,"5min":0.629,"15min":0.35499999999999998},"max":{"1min":1.6399999999999999,"5min":4.8019999999999996,"15min":12.189},"last":1.109}]},{"osd":7,"last update":"Sun Oct 1 11:19:59 2023","interfaces":[{"interface":"back","average":{"1min":1.1419999999999999,"5min":1.1319999999999999,"15min":1.143},"min":{"1min":0.94899999999999995,"5min":0.60599999999999998,"15min":0.36699999999999999},"max":{"1min":1.569,"5min":4.9850000000000003,"15min":9.1959999999999997},"last":0.72099999999999997},{"interface":"front","average":{"1min":1.2310000000000001,"5min":1.2250000000000001,"15min":1.2569999999999999},"min":{"1min":0.88300000000000001,"5min":0.56599999999999995,"15min":0.39900000000000002},"max":{"1min":1.413,"5min":5.1100000000000003,"15min":11.215},"last":1.2270000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T11:20:42.994 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:pacific shell --fsid d8016480-6041-11ee-8db5-212e2dc638e7 -- ceph pg dump --format=json 2023-10-01T11:20:43.331 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:20:43 smithi078 bash[78274]: audit 2023-10-01T11:20:42.045506+0000 mgr.y (mgr.25072) 1905 : audit 0 from='client.47960 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T11:20:43.332 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:20:43 smithi078 bash[75299]: audit 2023-10-01T11:20:42.045506+0000 mgr.y (mgr.25072) 1905 : audit 0 from='client.47960 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T11:20:43.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:20:43 smithi173 bash[58816]: audit 2023-10-01T11:20:42.045506+0000 mgr.y (mgr.25072) 1905 : audit 0 from='client.47960 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-10-01T11:20:44.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:20:44 smithi078 bash[75299]: cluster 2023-10-01T11:20:42.631544+0000 mgr.y (mgr.25072) 1906 : cluster 0 pgmap v2103: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:44.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:20:44 smithi078 bash[78274]: cluster 2023-10-01T11:20:42.631544+0000 mgr.y (mgr.25072) 1906 : cluster 0 pgmap v2103: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:44.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:20:44 smithi173 bash[58816]: cluster 2023-10-01T11:20:42.631544+0000 mgr.y (mgr.25072) 1906 : cluster 0 pgmap v2103: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:46.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:20:46 smithi078 bash[75299]: cluster 2023-10-01T11:20:44.632570+0000 mgr.y (mgr.25072) 1907 : cluster 0 pgmap v2104: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:46.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:20:46 smithi078 bash[78274]: cluster 2023-10-01T11:20:44.632570+0000 mgr.y (mgr.25072) 1907 : cluster 0 pgmap v2104: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:46.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:20:46 smithi173 bash[58816]: cluster 2023-10-01T11:20:44.632570+0000 mgr.y (mgr.25072) 1907 : cluster 0 pgmap v2104: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:47.804 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/mon.a/config 2023-10-01T11:20:48.581 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:20:48 smithi078 bash[75299]: cluster 2023-10-01T11:20:46.633375+0000 mgr.y (mgr.25072) 1908 : cluster 0 pgmap v2105: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:48.582 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:20:48 smithi078 bash[75299]: audit 2023-10-01T11:20:47.492419+0000 mon.a (mon.0) 2414 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:20:48.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:20:48 smithi078 bash[78274]: cluster 2023-10-01T11:20:46.633375+0000 mgr.y (mgr.25072) 1908 : cluster 0 pgmap v2105: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:48.582 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:20:48 smithi078 bash[78274]: audit 2023-10-01T11:20:47.492419+0000 mon.a (mon.0) 2414 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:20:48.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:20:48 smithi173 bash[58816]: cluster 2023-10-01T11:20:46.633375+0000 mgr.y (mgr.25072) 1908 : cluster 0 pgmap v2105: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:48.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:20:48 smithi173 bash[58816]: audit 2023-10-01T11:20:47.492419+0000 mon.a (mon.0) 2414 : audit 0 from='mgr.25072 172.21.15.78:0/2101720204' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-10-01T11:20:50.269 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:20:50 smithi078 bash[78274]: cluster 2023-10-01T11:20:48.634109+0000 mgr.y (mgr.25072) 1909 : cluster 0 pgmap v2106: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:50.269 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:20:50 smithi078 bash[75299]: cluster 2023-10-01T11:20:48.634109+0000 mgr.y (mgr.25072) 1909 : cluster 0 pgmap v2106: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:50.581 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:20:50 smithi078 bash[71214]: ::ffff:172.21.15.173 - - [01/Oct/2023:11:20:50] "GET /metrics HTTP/1.1" 200 34016 "" "Prometheus/2.33.4" 2023-10-01T11:20:50.619 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:20:50 smithi173 bash[58816]: cluster 2023-10-01T11:20:48.634109+0000 mgr.y (mgr.25072) 1909 : cluster 0 pgmap v2106: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:51.388 INFO:teuthology.orchestra.run.smithi078.stdout: 2023-10-01T11:20:51.390 INFO:teuthology.orchestra.run.smithi078.stderr:dumped all 2023-10-01T11:20:52.310 INFO:teuthology.orchestra.run.smithi078.stdout:{"pg_ready":true,"pg_map":{"version":2107,"stamp":"2023-10-01T11:20:50.634365+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6279,"num_read_kb":13744,"num_write":1107,"num_write_kb":15160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":162,"num_bytes_recovered":6528818,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1172,"ondisk_log_size":1172,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":8758076,"kb_used_data":49532,"kb_used_omap":30,"kb_used_meta":8708449,"kb_avail":741203140,"statfs":{"total":767960285184,"available":758992015360,"internally_reserved":0,"allocated":50720768,"data_stored":40686313,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31502,"internal_metadata":8917452018},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004658"},"pg_stats":[{"pgid":"3.1a","version":"60'1","reported_seq":425,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.170753+0000","last_change":"2023-10-01T11:04:02.725956+0000","last_active":"2023-10-01T11:19:21.170753+0000","last_peered":"2023-10-01T11:19:21.170753+0000","last_clean":"2023-10-01T11:19:21.170753+0000","last_became_active":"2023-10-01T11:04:02.725317+0000","last_became_peered":"2023-10-01T11:04:02.725317+0000","last_unstale":"2023-10-01T11:19:21.170753+0000","last_undegraded":"2023-10-01T11:19:21.170753+0000","last_fullsized":"2023-10-01T11:19:21.170753+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:34:07.675650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":429,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.174594+0000","last_change":"2023-10-01T10:38:16.025881+0000","last_active":"2023-10-01T11:19:22.174594+0000","last_peered":"2023-10-01T11:19:22.174594+0000","last_clean":"2023-10-01T11:19:22.174594+0000","last_became_active":"2023-10-01T10:38:16.025518+0000","last_became_peered":"2023-10-01T10:38:16.025518+0000","last_unstale":"2023-10-01T11:19:22.174594+0000","last_undegraded":"2023-10-01T11:19:22.174594+0000","last_fullsized":"2023-10-01T11:19:22.174594+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:48:37.826604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":285,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.182342+0000","last_change":"2023-10-01T11:17:22.000180+0000","last_active":"2023-10-01T11:19:22.182342+0000","last_peered":"2023-10-01T11:19:22.182342+0000","last_clean":"2023-10-01T11:19:22.182342+0000","last_became_active":"2023-10-01T11:17:21.999806+0000","last_became_peered":"2023-10-01T11:17:21.999806+0000","last_unstale":"2023-10-01T11:19:22.182342+0000","last_undegraded":"2023-10-01T11:19:22.182342+0000","last_fullsized":"2023-10-01T11:19:22.182342+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:17:10.953153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":460,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.176417+0000","last_change":"2023-10-01T11:17:02.473305+0000","last_active":"2023-10-01T11:19:21.176417+0000","last_peered":"2023-10-01T11:19:21.176417+0000","last_clean":"2023-10-01T11:19:21.176417+0000","last_became_active":"2023-10-01T11:17:02.472151+0000","last_became_peered":"2023-10-01T11:17:02.472151+0000","last_unstale":"2023-10-01T11:19:21.176417+0000","last_undegraded":"2023-10-01T11:19:21.176417+0000","last_fullsized":"2023-10-01T11:19:21.176417+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:23:59.546636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":448,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.164052+0000","last_change":"2023-10-01T11:16:18.984005+0000","last_active":"2023-10-01T11:19:21.164052+0000","last_peered":"2023-10-01T11:19:21.164052+0000","last_clean":"2023-10-01T11:19:21.164052+0000","last_became_active":"2023-10-01T11:16:18.983549+0000","last_became_peered":"2023-10-01T11:16:18.983549+0000","last_unstale":"2023-10-01T11:19:21.164052+0000","last_undegraded":"2023-10-01T11:19:21.164052+0000","last_fullsized":"2023-10-01T11:19:21.164052+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:10:36.758334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":125,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.176708+0000","last_change":"2023-10-01T11:05:33.800368+0000","last_active":"2023-10-01T11:19:21.176708+0000","last_peered":"2023-10-01T11:19:21.176708+0000","last_clean":"2023-10-01T11:19:21.176708+0000","last_became_active":"2023-10-01T11:05:33.799877+0000","last_became_peered":"2023-10-01T11:05:33.799877+0000","last_unstale":"2023-10-01T11:19:21.176708+0000","last_undegraded":"2023-10-01T11:19:21.176708+0000","last_fullsized":"2023-10-01T11:19:21.176708+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":214,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.184393+0000","last_change":"2023-10-01T11:17:02.474326+0000","last_active":"2023-10-01T11:19:22.184393+0000","last_peered":"2023-10-01T11:19:22.184393+0000","last_clean":"2023-10-01T11:19:22.184393+0000","last_became_active":"2023-10-01T11:17:02.473842+0000","last_became_peered":"2023-10-01T11:17:02.473842+0000","last_unstale":"2023-10-01T11:19:22.184393+0000","last_undegraded":"2023-10-01T11:19:22.184393+0000","last_fullsized":"2023-10-01T11:19:22.184393+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:58:58.628130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":469,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.174026+0000","last_change":"2023-10-01T11:05:33.782561+0000","last_active":"2023-10-01T11:19:22.174026+0000","last_peered":"2023-10-01T11:19:22.174026+0000","last_clean":"2023-10-01T11:19:22.174026+0000","last_became_active":"2023-10-01T11:05:33.782066+0000","last_became_peered":"2023-10-01T11:05:33.782066+0000","last_unstale":"2023-10-01T11:19:22.174026+0000","last_undegraded":"2023-10-01T11:19:22.174026+0000","last_fullsized":"2023-10-01T11:19:22.174026+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:57:06.451343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":442,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.177281+0000","last_change":"2023-10-01T11:06:11.792061+0000","last_active":"2023-10-01T11:19:21.177281+0000","last_peered":"2023-10-01T11:19:21.177281+0000","last_clean":"2023-10-01T11:19:21.177281+0000","last_became_active":"2023-10-01T11:06:11.789129+0000","last_became_peered":"2023-10-01T11:06:11.789129+0000","last_unstale":"2023-10-01T11:19:21.177281+0000","last_undegraded":"2023-10-01T11:19:21.177281+0000","last_fullsized":"2023-10-01T11:19:21.177281+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":489,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.174090+0000","last_change":"2023-10-01T11:06:11.807749+0000","last_active":"2023-10-01T11:19:22.174090+0000","last_peered":"2023-10-01T11:19:22.174090+0000","last_clean":"2023-10-01T11:19:22.174090+0000","last_became_active":"2023-10-01T11:06:11.807375+0000","last_became_peered":"2023-10-01T11:06:11.807375+0000","last_unstale":"2023-10-01T11:19:22.174090+0000","last_undegraded":"2023-10-01T11:19:22.174090+0000","last_fullsized":"2023-10-01T11:19:22.174090+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:44:05.945387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"60'3","reported_seq":233,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.185239+0000","last_change":"2023-10-01T11:19:22.185239+0000","last_active":"2023-10-01T11:19:22.185239+0000","last_peered":"2023-10-01T11:19:22.185239+0000","last_clean":"2023-10-01T11:19:22.185239+0000","last_became_active":"2023-10-01T11:19:22.184399+0000","last_became_peered":"2023-10-01T11:19:22.184399+0000","last_unstale":"2023-10-01T11:19:22.185239+0000","last_undegraded":"2023-10-01T11:19:22.185239+0000","last_fullsized":"2023-10-01T11:19:22.185239+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":208,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.182591+0000","last_change":"2023-10-01T11:17:22.399197+0000","last_active":"2023-10-01T11:19:22.182591+0000","last_peered":"2023-10-01T11:19:22.182591+0000","last_clean":"2023-10-01T11:19:22.182591+0000","last_became_active":"2023-10-01T11:17:22.398388+0000","last_became_peered":"2023-10-01T11:17:22.398388+0000","last_unstale":"2023-10-01T11:19:22.182591+0000","last_undegraded":"2023-10-01T11:19:22.182591+0000","last_fullsized":"2023-10-01T11:19:22.182591+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:24:22.497645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"77'23","reported_seq":554,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.180215+0000","last_change":"2023-10-01T11:06:11.836035+0000","last_active":"2023-10-01T11:19:22.180215+0000","last_peered":"2023-10-01T11:19:22.180215+0000","last_clean":"2023-10-01T11:19:22.180215+0000","last_became_active":"2023-10-01T11:06:11.794834+0000","last_became_peered":"2023-10-01T11:06:11.794834+0000","last_unstale":"2023-10-01T11:19:22.180215+0000","last_undegraded":"2023-10-01T11:19:22.180215+0000","last_fullsized":"2023-10-01T11:19:22.180215+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:25:13.103100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":33138,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":458,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.179754+0000","last_change":"2023-10-01T11:15:57.937317+0000","last_active":"2023-10-01T11:19:22.179754+0000","last_peered":"2023-10-01T11:19:22.179754+0000","last_clean":"2023-10-01T11:19:22.179754+0000","last_became_active":"2023-10-01T11:15:57.936299+0000","last_became_peered":"2023-10-01T11:15:57.936299+0000","last_unstale":"2023-10-01T11:19:22.179754+0000","last_undegraded":"2023-10-01T11:19:22.179754+0000","last_fullsized":"2023-10-01T11:19:22.179754+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:04:32.042221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"60'2","reported_seq":91,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.173605+0000","last_change":"2023-10-01T11:14:22.974249+0000","last_active":"2023-10-01T11:19:21.173605+0000","last_peered":"2023-10-01T11:19:21.173605+0000","last_clean":"2023-10-01T11:19:21.173605+0000","last_became_active":"2023-10-01T11:14:22.973777+0000","last_became_peered":"2023-10-01T11:14:22.973777+0000","last_unstale":"2023-10-01T11:19:21.173605+0000","last_undegraded":"2023-10-01T11:19:21.173605+0000","last_fullsized":"2023-10-01T11:19:21.173605+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":474,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.174437+0000","last_change":"2023-10-01T11:06:11.789794+0000","last_active":"2023-10-01T11:19:22.174437+0000","last_peered":"2023-10-01T11:19:22.174437+0000","last_clean":"2023-10-01T11:19:22.174437+0000","last_became_active":"2023-10-01T11:06:11.789452+0000","last_became_peered":"2023-10-01T11:06:11.789452+0000","last_unstale":"2023-10-01T11:19:22.174437+0000","last_undegraded":"2023-10-01T11:19:22.174437+0000","last_fullsized":"2023-10-01T11:19:22.174437+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:18:02.050694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":447,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.183907+0000","last_change":"2023-10-01T11:17:02.472419+0000","last_active":"2023-10-01T11:19:22.183907+0000","last_peered":"2023-10-01T11:19:22.183907+0000","last_clean":"2023-10-01T11:19:22.183907+0000","last_became_active":"2023-10-01T11:17:02.472063+0000","last_became_peered":"2023-10-01T11:17:02.472063+0000","last_unstale":"2023-10-01T11:19:22.183907+0000","last_undegraded":"2023-10-01T11:19:22.183907+0000","last_fullsized":"2023-10-01T11:19:22.183907+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:00:18.363944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":459,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.177184+0000","last_change":"2023-10-01T11:06:11.794671+0000","last_active":"2023-10-01T11:19:21.177184+0000","last_peered":"2023-10-01T11:19:21.177184+0000","last_clean":"2023-10-01T11:19:21.177184+0000","last_became_active":"2023-10-01T11:06:11.794121+0000","last_became_peered":"2023-10-01T11:06:11.794121+0000","last_unstale":"2023-10-01T11:19:21.177184+0000","last_undegraded":"2023-10-01T11:19:21.177184+0000","last_fullsized":"2023-10-01T11:19:21.177184+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":342,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.190563+0000","last_change":"2023-10-01T11:19:22.190563+0000","last_active":"2023-10-01T11:19:22.190563+0000","last_peered":"2023-10-01T11:19:22.190563+0000","last_clean":"2023-10-01T11:19:22.190563+0000","last_became_active":"2023-10-01T11:19:22.190197+0000","last_became_peered":"2023-10-01T11:19:22.190197+0000","last_unstale":"2023-10-01T11:19:22.190563+0000","last_undegraded":"2023-10-01T11:19:22.190563+0000","last_fullsized":"2023-10-01T11:19:22.190563+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":447,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.184450+0000","last_change":"2023-10-01T11:17:02.474479+0000","last_active":"2023-10-01T11:19:22.184450+0000","last_peered":"2023-10-01T11:19:22.184450+0000","last_clean":"2023-10-01T11:19:22.184450+0000","last_became_active":"2023-10-01T11:17:02.473987+0000","last_became_peered":"2023-10-01T11:17:02.473987+0000","last_unstale":"2023-10-01T11:19:22.184450+0000","last_undegraded":"2023-10-01T11:19:22.184450+0000","last_fullsized":"2023-10-01T11:19:22.184450+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:09:04.119899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"60'8","reported_seq":173,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.178743+0000","last_change":"2023-10-01T11:16:18.994830+0000","last_active":"2023-10-01T11:19:21.178743+0000","last_peered":"2023-10-01T11:19:21.178743+0000","last_clean":"2023-10-01T11:19:21.178743+0000","last_became_active":"2023-10-01T11:16:18.984619+0000","last_became_peered":"2023-10-01T11:16:18.984619+0000","last_unstale":"2023-10-01T11:19:21.178743+0000","last_undegraded":"2023-10-01T11:19:21.178743+0000","last_fullsized":"2023-10-01T11:19:21.178743+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":450,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.166861+0000","last_change":"2023-10-01T11:06:11.798727+0000","last_active":"2023-10-01T11:19:21.166861+0000","last_peered":"2023-10-01T11:19:21.166861+0000","last_clean":"2023-10-01T11:19:21.166861+0000","last_became_active":"2023-10-01T11:06:11.798388+0000","last_became_peered":"2023-10-01T11:06:11.798388+0000","last_unstale":"2023-10-01T11:19:21.166861+0000","last_undegraded":"2023-10-01T11:19:21.166861+0000","last_fullsized":"2023-10-01T11:19:21.166861+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:10:21.576627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"60'1","reported_seq":311,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.173725+0000","last_change":"2023-10-01T11:16:19.047127+0000","last_active":"2023-10-01T11:19:21.173725+0000","last_peered":"2023-10-01T11:19:21.173725+0000","last_clean":"2023-10-01T11:19:21.173725+0000","last_became_active":"2023-10-01T11:16:18.991314+0000","last_became_peered":"2023-10-01T11:16:18.991314+0000","last_unstale":"2023-10-01T11:19:21.173725+0000","last_undegraded":"2023-10-01T11:19:21.173725+0000","last_fullsized":"2023-10-01T11:19:21.173725+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":110,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":480,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.176510+0000","last_change":"2023-10-01T11:06:11.830255+0000","last_active":"2023-10-01T11:19:21.176510+0000","last_peered":"2023-10-01T11:19:21.176510+0000","last_clean":"2023-10-01T11:19:21.176510+0000","last_became_active":"2023-10-01T11:06:11.829847+0000","last_became_peered":"2023-10-01T11:06:11.829847+0000","last_unstale":"2023-10-01T11:19:21.176510+0000","last_undegraded":"2023-10-01T11:19:21.176510+0000","last_fullsized":"2023-10-01T11:19:21.176510+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"60'2","reported_seq":244,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.173888+0000","last_change":"2023-10-01T11:14:22.974529+0000","last_active":"2023-10-01T11:19:21.173888+0000","last_peered":"2023-10-01T11:19:21.173888+0000","last_clean":"2023-10-01T11:19:21.173888+0000","last_became_active":"2023-10-01T11:14:22.974060+0000","last_became_peered":"2023-10-01T11:14:22.974060+0000","last_unstale":"2023-10-01T11:19:21.173888+0000","last_undegraded":"2023-10-01T11:19:21.173888+0000","last_fullsized":"2023-10-01T11:19:21.173888+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":429,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.170121+0000","last_change":"2023-10-01T11:04:02.724840+0000","last_active":"2023-10-01T11:19:21.170121+0000","last_peered":"2023-10-01T11:19:21.170121+0000","last_clean":"2023-10-01T11:19:21.170121+0000","last_became_active":"2023-10-01T11:04:02.724476+0000","last_became_peered":"2023-10-01T11:04:02.724476+0000","last_unstale":"2023-10-01T11:19:21.170121+0000","last_undegraded":"2023-10-01T11:19:21.170121+0000","last_fullsized":"2023-10-01T11:19:21.170121+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:05:08.882495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"60'1","reported_seq":300,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.190703+0000","last_change":"2023-10-01T11:19:22.190703+0000","last_active":"2023-10-01T11:19:22.190703+0000","last_peered":"2023-10-01T11:19:22.190703+0000","last_clean":"2023-10-01T11:19:22.190703+0000","last_became_active":"2023-10-01T11:19:22.190327+0000","last_became_peered":"2023-10-01T11:19:22.190327+0000","last_unstale":"2023-10-01T11:19:22.190703+0000","last_undegraded":"2023-10-01T11:19:22.190703+0000","last_fullsized":"2023-10-01T11:19:22.190703+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":630,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":134,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.176067+0000","last_change":"2023-10-01T11:06:11.795096+0000","last_active":"2023-10-01T11:19:21.176067+0000","last_peered":"2023-10-01T11:19:21.176067+0000","last_clean":"2023-10-01T11:19:21.176067+0000","last_became_active":"2023-10-01T11:06:11.794716+0000","last_became_peered":"2023-10-01T11:06:11.794716+0000","last_unstale":"2023-10-01T11:19:21.176067+0000","last_undegraded":"2023-10-01T11:19:21.176067+0000","last_fullsized":"2023-10-01T11:19:21.176067+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":460,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.176813+0000","last_change":"2023-10-01T11:05:33.788917+0000","last_active":"2023-10-01T11:19:21.176813+0000","last_peered":"2023-10-01T11:19:21.176813+0000","last_clean":"2023-10-01T11:19:21.176813+0000","last_became_active":"2023-10-01T11:05:33.788078+0000","last_became_peered":"2023-10-01T11:05:33.788078+0000","last_unstale":"2023-10-01T11:19:21.176813+0000","last_undegraded":"2023-10-01T11:19:21.176813+0000","last_fullsized":"2023-10-01T11:19:21.176813+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":37,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.184309+0000","last_change":"2023-10-01T11:16:18.991614+0000","last_active":"2023-10-01T11:19:22.184309+0000","last_peered":"2023-10-01T11:19:22.184309+0000","last_clean":"2023-10-01T11:19:22.184309+0000","last_became_active":"2023-10-01T11:16:18.990345+0000","last_became_peered":"2023-10-01T11:16:18.990345+0000","last_unstale":"2023-10-01T11:19:22.184309+0000","last_undegraded":"2023-10-01T11:19:22.184309+0000","last_fullsized":"2023-10-01T11:19:22.184309+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:40:31.751016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"60'2","reported_seq":445,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.174438+0000","last_change":"2023-10-01T11:16:19.054310+0000","last_active":"2023-10-01T11:19:22.174438+0000","last_peered":"2023-10-01T11:19:22.174438+0000","last_clean":"2023-10-01T11:19:22.174438+0000","last_became_active":"2023-10-01T11:16:18.997794+0000","last_became_peered":"2023-10-01T11:16:18.997794+0000","last_unstale":"2023-10-01T11:19:22.174438+0000","last_undegraded":"2023-10-01T11:19:22.174438+0000","last_fullsized":"2023-10-01T11:19:22.174438+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:48:17.243891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":459,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.180461+0000","last_change":"2023-10-01T11:05:33.785133+0000","last_active":"2023-10-01T11:19:22.180461+0000","last_peered":"2023-10-01T11:19:22.180461+0000","last_clean":"2023-10-01T11:19:22.180461+0000","last_became_active":"2023-10-01T11:05:33.784749+0000","last_became_peered":"2023-10-01T11:05:33.784749+0000","last_unstale":"2023-10-01T11:19:22.180461+0000","last_undegraded":"2023-10-01T11:19:22.180461+0000","last_fullsized":"2023-10-01T11:19:22.180461+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:08:01.664432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":316,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.175853+0000","last_change":"2023-10-01T11:19:02.131086+0000","last_active":"2023-10-01T11:19:21.175853+0000","last_peered":"2023-10-01T11:19:21.175853+0000","last_clean":"2023-10-01T11:19:21.175853+0000","last_became_active":"2023-10-01T11:19:02.130633+0000","last_became_peered":"2023-10-01T11:19:02.130633+0000","last_unstale":"2023-10-01T11:19:21.175853+0000","last_undegraded":"2023-10-01T11:19:21.175853+0000","last_fullsized":"2023-10-01T11:19:21.175853+0000","mapping_epoch":435,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":436,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":419,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.180706+0000","last_change":"2023-10-01T11:04:02.727282+0000","last_active":"2023-10-01T11:19:22.180706+0000","last_peered":"2023-10-01T11:19:22.180706+0000","last_clean":"2023-10-01T11:19:22.180706+0000","last_became_active":"2023-10-01T11:04:02.726818+0000","last_became_peered":"2023-10-01T11:04:02.726818+0000","last_unstale":"2023-10-01T11:19:22.180706+0000","last_undegraded":"2023-10-01T11:19:22.180706+0000","last_fullsized":"2023-10-01T11:19:22.180706+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:11:11.868291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":463,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.179771+0000","last_change":"2023-10-01T11:06:11.790944+0000","last_active":"2023-10-01T11:19:22.179771+0000","last_peered":"2023-10-01T11:19:22.179771+0000","last_clean":"2023-10-01T11:19:22.179771+0000","last_became_active":"2023-10-01T11:06:11.790151+0000","last_became_peered":"2023-10-01T11:06:11.790151+0000","last_unstale":"2023-10-01T11:19:22.179771+0000","last_undegraded":"2023-10-01T11:19:22.179771+0000","last_fullsized":"2023-10-01T11:19:22.179771+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:05:49.161868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":470,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.180832+0000","last_change":"2023-10-01T11:15:57.939847+0000","last_active":"2023-10-01T11:19:22.180832+0000","last_peered":"2023-10-01T11:19:22.180832+0000","last_clean":"2023-10-01T11:19:22.180832+0000","last_became_active":"2023-10-01T11:15:57.939154+0000","last_became_peered":"2023-10-01T11:15:57.939154+0000","last_unstale":"2023-10-01T11:19:22.180832+0000","last_undegraded":"2023-10-01T11:19:22.180832+0000","last_fullsized":"2023-10-01T11:19:22.180832+0000","mapping_epoch":408,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":409,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:25:56.931688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":453,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.164001+0000","last_change":"2023-10-01T11:06:11.784722+0000","last_active":"2023-10-01T11:19:21.164001+0000","last_peered":"2023-10-01T11:19:21.164001+0000","last_clean":"2023-10-01T11:19:21.164001+0000","last_became_active":"2023-10-01T11:06:11.784348+0000","last_became_peered":"2023-10-01T11:06:11.784348+0000","last_unstale":"2023-10-01T11:19:21.164001+0000","last_undegraded":"2023-10-01T11:19:21.164001+0000","last_fullsized":"2023-10-01T11:19:21.164001+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:40:29.283179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":436,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.326699+0000","last_change":"2023-10-01T10:38:16.019755+0000","last_active":"2023-10-01T11:19:22.326699+0000","last_peered":"2023-10-01T11:19:22.326699+0000","last_clean":"2023-10-01T11:19:22.326699+0000","last_became_active":"2023-10-01T10:38:16.019397+0000","last_became_peered":"2023-10-01T10:38:16.019397+0000","last_unstale":"2023-10-01T11:19:22.326699+0000","last_undegraded":"2023-10-01T11:19:22.326699+0000","last_fullsized":"2023-10-01T11:19:22.326699+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:42:56.623225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"405'127","reported_seq":693,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.183164+0000","last_change":"2023-10-01T11:17:22.006372+0000","last_active":"2023-10-01T11:19:22.183164+0000","last_peered":"2023-10-01T11:19:22.183164+0000","last_clean":"2023-10-01T11:19:22.183164+0000","last_became_active":"2023-10-01T11:17:22.005936+0000","last_became_peered":"2023-10-01T11:17:22.005936+0000","last_unstale":"2023-10-01T11:19:22.183164+0000","last_undegraded":"2023-10-01T11:19:22.183164+0000","last_fullsized":"2023-10-01T11:19:22.183164+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":127,"log_dups_size":0,"ondisk_log_size":127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:55:00.251169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":597,"num_read_kb":514,"num_write":134,"num_write_kb":4165,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":421,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.177200+0000","last_change":"2023-10-01T10:37:42.766814+0000","last_active":"2023-10-01T11:19:21.177200+0000","last_peered":"2023-10-01T11:19:21.177200+0000","last_clean":"2023-10-01T11:19:21.177200+0000","last_became_active":"2023-10-01T10:37:42.766451+0000","last_became_peered":"2023-10-01T10:37:42.766451+0000","last_unstale":"2023-10-01T11:19:21.177200+0000","last_undegraded":"2023-10-01T11:19:21.177200+0000","last_fullsized":"2023-10-01T11:19:21.177200+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":265,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.182552+0000","last_change":"2023-10-01T11:17:22.401658+0000","last_active":"2023-10-01T11:19:22.182552+0000","last_peered":"2023-10-01T11:19:22.182552+0000","last_clean":"2023-10-01T11:19:22.182552+0000","last_became_active":"2023-10-01T11:17:22.401009+0000","last_became_peered":"2023-10-01T11:17:22.401009+0000","last_unstale":"2023-10-01T11:19:22.182552+0000","last_undegraded":"2023-10-01T11:19:22.182552+0000","last_fullsized":"2023-10-01T11:19:22.182552+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:15:42.663028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"405'80","reported_seq":1099,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.177273+0000","last_change":"2023-10-01T11:15:39.089563+0000","last_active":"2023-10-01T11:19:21.177273+0000","last_peered":"2023-10-01T11:19:21.177273+0000","last_clean":"2023-10-01T11:19:21.177273+0000","last_became_active":"2023-10-01T11:05:33.792054+0000","last_became_peered":"2023-10-01T11:05:33.792054+0000","last_unstale":"2023-10-01T11:19:21.177273+0000","last_undegraded":"2023-10-01T11:19:21.177273+0000","last_fullsized":"2023-10-01T11:19:21.177273+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":80,"log_dups_size":0,"ondisk_log_size":80,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":994,"num_read_kb":2114,"num_write":54,"num_write_kb":1154,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":446,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.180646+0000","last_change":"2023-10-01T11:16:18.993464+0000","last_active":"2023-10-01T11:19:22.180646+0000","last_peered":"2023-10-01T11:19:22.180646+0000","last_clean":"2023-10-01T11:19:22.180646+0000","last_became_active":"2023-10-01T11:16:18.992859+0000","last_became_peered":"2023-10-01T11:16:18.992859+0000","last_unstale":"2023-10-01T11:19:22.180646+0000","last_undegraded":"2023-10-01T11:19:22.180646+0000","last_fullsized":"2023-10-01T11:19:22.180646+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:03:59.015456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":165,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.182980+0000","last_change":"2023-10-01T11:17:22.398815+0000","last_active":"2023-10-01T11:19:22.182980+0000","last_peered":"2023-10-01T11:19:22.182980+0000","last_clean":"2023-10-01T11:19:22.182980+0000","last_became_active":"2023-10-01T11:17:22.398275+0000","last_became_peered":"2023-10-01T11:17:22.398275+0000","last_unstale":"2023-10-01T11:19:22.182980+0000","last_undegraded":"2023-10-01T11:19:22.182980+0000","last_fullsized":"2023-10-01T11:19:22.182980+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T13:40:17.815750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"405'64","reported_seq":814,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.326370+0000","last_change":"2023-10-01T11:16:19.070021+0000","last_active":"2023-10-01T11:19:22.326370+0000","last_peered":"2023-10-01T11:19:22.326370+0000","last_clean":"2023-10-01T11:19:22.326370+0000","last_became_active":"2023-10-01T11:16:18.988806+0000","last_became_peered":"2023-10-01T11:16:18.988806+0000","last_unstale":"2023-10-01T11:19:22.326370+0000","last_undegraded":"2023-10-01T11:19:22.326370+0000","last_fullsized":"2023-10-01T11:19:22.326370+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":64,"log_dups_size":0,"ondisk_log_size":64,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:53:19.662794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042224999999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":500,"num_read_kb":2336,"num_write":43,"num_write_kb":1518,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":277,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.183552+0000","last_change":"2023-10-01T11:17:22.398544+0000","last_active":"2023-10-01T11:19:22.183552+0000","last_peered":"2023-10-01T11:19:22.183552+0000","last_clean":"2023-10-01T11:19:22.183552+0000","last_became_active":"2023-10-01T11:17:22.397941+0000","last_became_peered":"2023-10-01T11:17:22.397941+0000","last_unstale":"2023-10-01T11:19:22.183552+0000","last_undegraded":"2023-10-01T11:19:22.183552+0000","last_fullsized":"2023-10-01T11:19:22.183552+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:10:48.292501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":299,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.182761+0000","last_change":"2023-10-01T11:17:22.401230+0000","last_active":"2023-10-01T11:19:22.182761+0000","last_peered":"2023-10-01T11:19:22.182761+0000","last_clean":"2023-10-01T11:19:22.182761+0000","last_became_active":"2023-10-01T11:17:22.400774+0000","last_became_peered":"2023-10-01T11:17:22.400774+0000","last_unstale":"2023-10-01T11:19:22.182761+0000","last_undegraded":"2023-10-01T11:19:22.182761+0000","last_fullsized":"2023-10-01T11:19:22.182761+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:03:49.081904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"405'227","reported_seq":1130,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.176413+0000","last_change":"2023-10-01T11:15:39.089380+0000","last_active":"2023-10-01T11:19:21.176413+0000","last_peered":"2023-10-01T11:19:21.176413+0000","last_clean":"2023-10-01T11:19:21.176413+0000","last_became_active":"2023-10-01T11:06:11.789667+0000","last_became_peered":"2023-10-01T11:06:11.789667+0000","last_unstale":"2023-10-01T11:19:21.176413+0000","last_undegraded":"2023-10-01T11:19:21.176413+0000","last_fullsized":"2023-10-01T11:19:21.176413+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":227,"log_dups_size":0,"ondisk_log_size":227,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1045,"num_read_kb":1067,"num_write":205,"num_write_kb":189,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"76'139","reported_seq":758,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.190630+0000","last_change":"2023-10-01T11:19:22.190630+0000","last_active":"2023-10-01T11:19:22.190630+0000","last_peered":"2023-10-01T11:19:22.190630+0000","last_clean":"2023-10-01T11:19:22.190630+0000","last_became_active":"2023-10-01T11:19:22.190203+0000","last_became_peered":"2023-10-01T11:19:22.190203+0000","last_unstale":"2023-10-01T11:19:22.190630+0000","last_undegraded":"2023-10-01T11:19:22.190630+0000","last_fullsized":"2023-10-01T11:19:22.190630+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:09:17.221082+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:09:17.221082+0000","last_clean_scrub_stamp":"2023-10-01T10:09:17.221082+0000","objects_scrubbed":0,"log_size":139,"log_dups_size":0,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":6494048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":454,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.437635+0000","last_change":"2023-10-01T11:05:33.788093+0000","last_active":"2023-10-01T11:19:21.437635+0000","last_peered":"2023-10-01T11:19:21.437635+0000","last_clean":"2023-10-01T11:19:21.437635+0000","last_became_active":"2023-10-01T11:05:33.787723+0000","last_became_peered":"2023-10-01T11:05:33.787723+0000","last_unstale":"2023-10-01T11:19:21.437635+0000","last_undegraded":"2023-10-01T11:19:21.437635+0000","last_fullsized":"2023-10-01T11:19:21.437635+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:04:55.138186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":369,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.176982+0000","last_change":"2023-10-01T11:19:02.159604+0000","last_active":"2023-10-01T11:19:21.176982+0000","last_peered":"2023-10-01T11:19:21.176982+0000","last_clean":"2023-10-01T11:19:21.176982+0000","last_became_active":"2023-10-01T11:19:02.158991+0000","last_became_peered":"2023-10-01T11:19:02.158991+0000","last_unstale":"2023-10-01T11:19:21.176982+0000","last_undegraded":"2023-10-01T11:19:21.176982+0000","last_fullsized":"2023-10-01T11:19:21.176982+0000","mapping_epoch":435,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":436,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"405'183","reported_seq":1149,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.188449+0000","last_change":"2023-10-01T11:19:22.188449+0000","last_active":"2023-10-01T11:19:22.188449+0000","last_peered":"2023-10-01T11:19:22.188449+0000","last_clean":"2023-10-01T11:19:22.188449+0000","last_became_active":"2023-10-01T11:19:22.187455+0000","last_became_peered":"2023-10-01T11:19:22.187455+0000","last_unstale":"2023-10-01T11:19:22.188449+0000","last_undegraded":"2023-10-01T11:19:22.188449+0000","last_fullsized":"2023-10-01T11:19:22.188449+0000","mapping_epoch":438,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":183,"log_dups_size":0,"ondisk_log_size":183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1514,"num_read_kb":1489,"num_write":145,"num_write_kb":150,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":240,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.175406+0000","last_change":"2023-10-01T11:17:02.474010+0000","last_active":"2023-10-01T11:19:21.175406+0000","last_peered":"2023-10-01T11:19:21.175406+0000","last_clean":"2023-10-01T11:19:21.175406+0000","last_became_active":"2023-10-01T11:17:02.473678+0000","last_became_peered":"2023-10-01T11:17:02.473678+0000","last_unstale":"2023-10-01T11:19:21.175406+0000","last_undegraded":"2023-10-01T11:19:21.175406+0000","last_fullsized":"2023-10-01T11:19:21.175406+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T21:18:59.211258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":37,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.184339+0000","last_change":"2023-10-01T11:16:18.985906+0000","last_active":"2023-10-01T11:19:22.184339+0000","last_peered":"2023-10-01T11:19:22.184339+0000","last_clean":"2023-10-01T11:19:22.184339+0000","last_became_active":"2023-10-01T11:16:18.985541+0000","last_became_peered":"2023-10-01T11:16:18.985541+0000","last_unstale":"2023-10-01T11:19:22.184339+0000","last_undegraded":"2023-10-01T11:19:22.184339+0000","last_fullsized":"2023-10-01T11:19:22.184339+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:49:19.304679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"405'105","reported_seq":739,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.174603+0000","last_change":"2023-10-01T11:16:22.012650+0000","last_active":"2023-10-01T11:19:21.174603+0000","last_peered":"2023-10-01T11:19:21.174603+0000","last_clean":"2023-10-01T11:19:21.174603+0000","last_became_active":"2023-10-01T11:16:22.011729+0000","last_became_peered":"2023-10-01T11:16:22.011729+0000","last_unstale":"2023-10-01T11:19:21.174603+0000","last_undegraded":"2023-10-01T11:19:21.174603+0000","last_fullsized":"2023-10-01T11:19:21.174603+0000","mapping_epoch":417,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":418,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":105,"log_dups_size":0,"ondisk_log_size":105,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":738,"num_read_kb":1646,"num_write":82,"num_write_kb":1055,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":150,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.183086+0000","last_change":"2023-10-01T11:17:22.401284+0000","last_active":"2023-10-01T11:19:22.183086+0000","last_peered":"2023-10-01T11:19:22.183086+0000","last_clean":"2023-10-01T11:19:22.183086+0000","last_became_active":"2023-10-01T11:17:22.400619+0000","last_became_peered":"2023-10-01T11:17:22.400619+0000","last_unstale":"2023-10-01T11:19:22.183086+0000","last_undegraded":"2023-10-01T11:19:22.183086+0000","last_fullsized":"2023-10-01T11:19:22.183086+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:20:40.635133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":431,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.175542+0000","last_change":"2023-10-01T11:17:02.200206+0000","last_active":"2023-10-01T11:19:21.175542+0000","last_peered":"2023-10-01T11:19:21.175542+0000","last_clean":"2023-10-01T11:19:21.175542+0000","last_became_active":"2023-10-01T11:17:02.199670+0000","last_became_peered":"2023-10-01T11:17:02.199670+0000","last_unstale":"2023-10-01T11:19:21.175542+0000","last_undegraded":"2023-10-01T11:19:21.175542+0000","last_fullsized":"2023-10-01T11:19:21.175542+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T17:11:51.310314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"60'2","reported_seq":482,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.166964+0000","last_change":"2023-10-01T11:16:18.999845+0000","last_active":"2023-10-01T11:19:21.166964+0000","last_peered":"2023-10-01T11:19:21.166964+0000","last_clean":"2023-10-01T11:19:21.166964+0000","last_became_active":"2023-10-01T11:16:18.978956+0000","last_became_peered":"2023-10-01T11:16:18.978956+0000","last_unstale":"2023-10-01T11:19:21.166964+0000","last_undegraded":"2023-10-01T11:19:21.166964+0000","last_fullsized":"2023-10-01T11:19:21.166964+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:33:27.070624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"405'163","reported_seq":725,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.182789+0000","last_change":"2023-10-01T11:17:22.007789+0000","last_active":"2023-10-01T11:19:22.182789+0000","last_peered":"2023-10-01T11:19:22.182789+0000","last_clean":"2023-10-01T11:19:22.182789+0000","last_became_active":"2023-10-01T11:17:22.007230+0000","last_became_peered":"2023-10-01T11:17:22.007230+0000","last_unstale":"2023-10-01T11:19:22.182789+0000","last_undegraded":"2023-10-01T11:19:22.182789+0000","last_fullsized":"2023-10-01T11:19:22.182789+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":163,"log_dups_size":0,"ondisk_log_size":163,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T15:43:53.181499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001122088,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":536,"num_read_kb":2851,"num_write":150,"num_write_kb":2326,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":457,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.173865+0000","last_change":"2023-10-01T11:16:18.980825+0000","last_active":"2023-10-01T11:19:22.173865+0000","last_peered":"2023-10-01T11:19:22.173865+0000","last_clean":"2023-10-01T11:19:22.173865+0000","last_became_active":"2023-10-01T11:16:18.979768+0000","last_became_peered":"2023-10-01T11:16:18.979768+0000","last_unstale":"2023-10-01T11:19:22.173865+0000","last_undegraded":"2023-10-01T11:19:22.173865+0000","last_fullsized":"2023-10-01T11:19:22.173865+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:45:26.288769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"405'28","reported_seq":206,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.176718+0000","last_change":"2023-10-01T11:15:39.032714+0000","last_active":"2023-10-01T11:19:21.176718+0000","last_peered":"2023-10-01T11:19:21.176718+0000","last_clean":"2023-10-01T11:19:21.176718+0000","last_became_active":"2023-10-01T11:06:11.818533+0000","last_became_peered":"2023-10-01T11:06:11.818533+0000","last_unstale":"2023-10-01T11:19:21.176718+0000","last_undegraded":"2023-10-01T11:19:21.176718+0000","last_fullsized":"2023-10-01T11:19:21.176718+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:13:43.645968+0000","last_clean_scrub_stamp":"2023-10-01T10:13:43.645968+0000","objects_scrubbed":0,"log_size":28,"log_dups_size":0,"ondisk_log_size":28,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":61,"num_read_kb":1227,"num_write":33,"num_write_kb":2157,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"60'1","reported_seq":41,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.184526+0000","last_change":"2023-10-01T11:16:18.995911+0000","last_active":"2023-10-01T11:19:22.184526+0000","last_peered":"2023-10-01T11:19:22.184526+0000","last_clean":"2023-10-01T11:19:22.184526+0000","last_became_active":"2023-10-01T11:16:18.991018+0000","last_became_peered":"2023-10-01T11:16:18.991018+0000","last_unstale":"2023-10-01T11:19:22.184526+0000","last_undegraded":"2023-10-01T11:19:22.184526+0000","last_fullsized":"2023-10-01T11:19:22.184526+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:33:34.834635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":426,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.179268+0000","last_change":"2023-10-01T11:05:33.784531+0000","last_active":"2023-10-01T11:19:22.179268+0000","last_peered":"2023-10-01T11:19:22.179268+0000","last_clean":"2023-10-01T11:19:22.179268+0000","last_became_active":"2023-10-01T11:05:33.784167+0000","last_became_peered":"2023-10-01T11:05:33.784167+0000","last_unstale":"2023-10-01T11:19:22.179268+0000","last_undegraded":"2023-10-01T11:19:22.179268+0000","last_fullsized":"2023-10-01T11:19:22.179268+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T12:34:16.434841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"60'1","reported_seq":170,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.184205+0000","last_change":"2023-10-01T11:17:02.199637+0000","last_active":"2023-10-01T11:19:22.184205+0000","last_peered":"2023-10-01T11:19:22.184205+0000","last_clean":"2023-10-01T11:19:22.184205+0000","last_became_active":"2023-10-01T11:17:02.199115+0000","last_became_peered":"2023-10-01T11:17:02.199115+0000","last_unstale":"2023-10-01T11:19:22.184205+0000","last_undegraded":"2023-10-01T11:19:22.184205+0000","last_fullsized":"2023-10-01T11:19:22.184205+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T18:16:50.614626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":435,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.326848+0000","last_change":"2023-10-01T10:37:42.762360+0000","last_active":"2023-10-01T11:19:22.326848+0000","last_peered":"2023-10-01T11:19:22.326848+0000","last_clean":"2023-10-01T11:19:22.326848+0000","last_became_active":"2023-10-01T10:37:42.761936+0000","last_became_peered":"2023-10-01T10:37:42.761936+0000","last_unstale":"2023-10-01T11:19:22.326848+0000","last_undegraded":"2023-10-01T11:19:22.326848+0000","last_fullsized":"2023-10-01T11:19:22.326848+0000","mapping_epoch":121,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T20:39:50.637582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"60'2","reported_seq":423,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.176914+0000","last_change":"2023-10-01T10:38:16.037809+0000","last_active":"2023-10-01T11:19:21.176914+0000","last_peered":"2023-10-01T11:19:21.176914+0000","last_clean":"2023-10-01T11:19:21.176914+0000","last_became_active":"2023-10-01T10:38:16.037318+0000","last_became_peered":"2023-10-01T10:38:16.037318+0000","last_unstale":"2023-10-01T11:19:21.176914+0000","last_undegraded":"2023-10-01T11:19:21.176914+0000","last_fullsized":"2023-10-01T11:19:21.176914+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":427,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.171222+0000","last_change":"2023-10-01T11:04:02.726007+0000","last_active":"2023-10-01T11:19:21.171222+0000","last_peered":"2023-10-01T11:19:21.171222+0000","last_clean":"2023-10-01T11:19:21.171222+0000","last_became_active":"2023-10-01T11:04:02.725507+0000","last_became_peered":"2023-10-01T11:04:02.725507+0000","last_unstale":"2023-10-01T11:19:21.171222+0000","last_undegraded":"2023-10-01T11:19:21.171222+0000","last_fullsized":"2023-10-01T11:19:21.171222+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T14:32:04.611533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"60'2","reported_seq":486,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.177015+0000","last_change":"2023-10-01T11:05:33.830108+0000","last_active":"2023-10-01T11:19:21.177015+0000","last_peered":"2023-10-01T11:19:21.177015+0000","last_clean":"2023-10-01T11:19:21.177015+0000","last_became_active":"2023-10-01T11:05:33.791882+0000","last_became_peered":"2023-10-01T11:05:33.791882+0000","last_unstale":"2023-10-01T11:19:21.177015+0000","last_undegraded":"2023-10-01T11:19:21.177015+0000","last_fullsized":"2023-10-01T11:19:21.177015+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":445,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.170488+0000","last_change":"2023-10-01T11:16:18.980782+0000","last_active":"2023-10-01T11:19:21.170488+0000","last_peered":"2023-10-01T11:19:21.170488+0000","last_clean":"2023-10-01T11:19:21.170488+0000","last_became_active":"2023-10-01T11:16:18.980097+0000","last_became_peered":"2023-10-01T11:16:18.980097+0000","last_unstale":"2023-10-01T11:19:21.170488+0000","last_undegraded":"2023-10-01T11:19:21.170488+0000","last_fullsized":"2023-10-01T11:19:21.170488+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T10:37:17.022077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"60'1","reported_seq":488,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-10-01T11:19:22.174783+0000","last_change":"2023-10-01T11:05:33.777019+0000","last_active":"2023-10-01T11:19:22.174783+0000","last_peered":"2023-10-01T11:19:22.174783+0000","last_clean":"2023-10-01T11:19:22.174783+0000","last_became_active":"2023-10-01T11:05:33.776565+0000","last_became_peered":"2023-10-01T11:05:33.776565+0000","last_unstale":"2023-10-01T11:19:22.174783+0000","last_undegraded":"2023-10-01T11:19:22.174783+0000","last_fullsized":"2023-10-01T11:19:22.174783+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T22:00:18.890326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":461,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.164141+0000","last_change":"2023-10-01T11:16:18.982759+0000","last_active":"2023-10-01T11:19:21.164141+0000","last_peered":"2023-10-01T11:19:21.164141+0000","last_clean":"2023-10-01T11:19:21.164141+0000","last_became_active":"2023-10-01T11:16:18.982413+0000","last_became_peered":"2023-10-01T11:16:18.982413+0000","last_unstale":"2023-10-01T11:19:21.164141+0000","last_undegraded":"2023-10-01T11:19:21.164141+0000","last_fullsized":"2023-10-01T11:19:21.164141+0000","mapping_epoch":414,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T11:30:23.401194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"60'4","reported_seq":174,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.175807+0000","last_change":"2023-10-01T11:17:02.200873+0000","last_active":"2023-10-01T11:19:21.175807+0000","last_peered":"2023-10-01T11:19:21.175807+0000","last_clean":"2023-10-01T11:19:21.175807+0000","last_became_active":"2023-10-01T11:17:02.200271+0000","last_became_peered":"2023-10-01T11:17:02.200271+0000","last_unstale":"2023-10-01T11:19:21.175807+0000","last_undegraded":"2023-10-01T11:19:21.175807+0000","last_fullsized":"2023-10-01T11:19:21.175807+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:38.252489+0000","last_clean_scrub_stamp":"2023-10-01T10:15:38.252489+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T19:54:31.586661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":463,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-10-01T11:19:21.170093+0000","last_change":"2023-10-01T11:05:33.785774+0000","last_active":"2023-10-01T11:19:21.170093+0000","last_peered":"2023-10-01T11:19:21.170093+0000","last_clean":"2023-10-01T11:19:21.170093+0000","last_became_active":"2023-10-01T11:05:33.785426+0000","last_became_peered":"2023-10-01T11:05:33.785426+0000","last_unstale":"2023-10-01T11:19:21.170093+0000","last_undegraded":"2023-10-01T11:19:21.170093+0000","last_fullsized":"2023-10-01T11:19:21.170093+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":57,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-10-01T10:15:39.263253+0000","last_clean_scrub_stamp":"2023-10-01T10:15:39.263253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-10-02T16:41:32.634339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":94,"num_bytes_recovered":34770,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30319,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":294,"num_read_kb":500,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":6494048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5985,"num_read_kb":13244,"num_write":846,"num_write_kb":12714,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":46,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":977,"ondisk_log_size":977,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158431021,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1487152,"kb_used_data":8424,"kb_used_omap":0,"kb_used_meta":1478720,"kb_avail":92258000,"statfs":{"total":95995035648,"available":94472192000,"internally_reserved":0,"allocated":8626176,"data_stored":7990409,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1514209280},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 11:19:56 2023","interfaces":[{"interface":"back","average":{"1min":0.92200000000000004,"5min":0.91400000000000003,"15min":1.242},"min":{"1min":0.55100000000000005,"5min":0.44,"15min":0.27100000000000002},"max":{"1min":1.208,"5min":1.2649999999999999,"15min":10.013},"last":0.92200000000000004},{"interface":"front","average":{"1min":0.98499999999999999,"5min":0.95499999999999996,"15min":1.2549999999999999},"min":{"1min":0.64600000000000002,"5min":0.60099999999999998,"15min":0.29599999999999999},"max":{"1min":1.2769999999999999,"5min":1.288,"15min":8.8949999999999996},"last":1.034}]},{"osd":1,"last update":"Sun Oct 1 11:19:56 2023","interfaces":[{"interface":"back","average":{"1min":0.84799999999999998,"5min":0.89100000000000001,"15min":1.1559999999999999},"min":{"1min":0.63800000000000001,"5min":0.60899999999999999,"15min":0.28000000000000003},"max":{"1min":1.196,"5min":1.2270000000000001,"15min":9.6259999999999994},"last":0.94399999999999995},{"interface":"front","average":{"1min":0.99199999999999999,"5min":0.99099999999999999,"15min":1.286},"min":{"1min":0.73199999999999998,"5min":0.61399999999999999,"15min":0.30499999999999999},"max":{"1min":1.3149999999999999,"5min":1.4990000000000001,"15min":9.8170000000000002},"last":0.76900000000000002}]},{"osd":2,"last update":"Sun Oct 1 11:20:17 2023","interfaces":[{"interface":"back","average":{"1min":1.0129999999999999,"5min":0.996,"15min":1.325},"min":{"1min":0.68100000000000005,"5min":0.62,"15min":0.33200000000000002},"max":{"1min":1.339,"5min":1.5149999999999999,"15min":10.021000000000001},"last":0.98299999999999998},{"interface":"front","average":{"1min":0.89100000000000001,"5min":0.96199999999999997,"15min":1.304},"min":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.32200000000000001},"max":{"1min":1.1579999999999999,"5min":1.3009999999999999,"15min":14.663},"last":0.88300000000000001}]},{"osd":3,"last update":"Sun Oct 1 11:19:56 2023","interfaces":[{"interface":"back","average":{"1min":0.998,"5min":1.028,"15min":1.22},"min":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.34499999999999997},"max":{"1min":1.3049999999999999,"5min":1.403,"15min":12.358000000000001},"last":0.93100000000000005},{"interface":"front","average":{"1min":1.024,"5min":0.97799999999999998,"15min":1.1559999999999999},"min":{"1min":0.71999999999999997,"5min":0.57799999999999996,"15min":0.32500000000000001},"max":{"1min":1.2569999999999999,"5min":1.327,"15min":8.3109999999999999},"last":0.54300000000000004}]},{"osd":4,"last update":"Sun Oct 1 11:20:30 2023","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":0.96299999999999997,"15min":1.1739999999999999},"min":{"1min":0.61199999999999999,"5min":0.503,"15min":0.34999999999999998},"max":{"1min":1.1779999999999999,"5min":1.861,"15min":14.51},"last":0.66900000000000004},{"interface":"front","average":{"1min":1.151,"5min":1.1879999999999999,"15min":1.232},"min":{"1min":0.91600000000000004,"5min":0.66500000000000004,"15min":0.34799999999999998},"max":{"1min":1.4179999999999999,"5min":1.98,"15min":12.414999999999999},"last":1.0549999999999999}]},{"osd":5,"last update":"Sun Oct 1 11:19:56 2023","interfaces":[{"interface":"back","average":{"1min":1.083,"5min":1.0880000000000001,"15min":1.397},"min":{"1min":0.79600000000000004,"5min":0.52000000000000002,"15min":0.33100000000000002},"max":{"1min":1.391,"5min":1.583,"15min":18.998999999999999},"last":0.70699999999999996},{"interface":"front","average":{"1min":1.054,"5min":1.056,"15min":1.1859999999999999},"min":{"1min":0.79400000000000004,"5min":0.64800000000000002,"15min":0.36899999999999999},"max":{"1min":1.367,"5min":1.4490000000000001,"15min":10.1},"last":0.78600000000000003}]},{"osd":6,"last update":"Sun Oct 1 11:19:56 2023","interfaces":[{"interface":"back","average":{"1min":1.1439999999999999,"5min":1.153,"15min":1.282},"min":{"1min":0.89800000000000002,"5min":0.60399999999999998,"15min":0.38500000000000001},"max":{"1min":1.361,"5min":1.601,"15min":13.038},"last":0.85699999999999998},{"interface":"front","average":{"1min":1.099,"5min":1.1020000000000001,"15min":1.2310000000000001},"min":{"1min":0.81599999999999995,"5min":0.56999999999999995,"15min":0.32400000000000001},"max":{"1min":1.3660000000000001,"5min":1.393,"15min":11.536},"last":0.80300000000000005}]}]},{"osd":6,"up_from":42,"seq":180388627261,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1181604,"kb_used_data":4316,"kb_used_omap":0,"kb_used_meta":1177280,"kb_avail":92563548,"statfs":{"total":95995035648,"available":94785073152,"internally_reserved":0,"allocated":4419584,"data_stored":3795663,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1205534720},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 11:19:53 2023","interfaces":[{"interface":"back","average":{"1min":0.94599999999999995,"5min":0.88600000000000001,"15min":1.2470000000000001},"min":{"1min":0.61499999999999999,"5min":0.435,"15min":0.27000000000000002},"max":{"1min":1.2270000000000001,"5min":1.4199999999999999,"15min":14.964},"last":1.073},{"interface":"front","average":{"1min":0.95299999999999996,"5min":0.94599999999999995,"15min":1.1799999999999999},"min":{"1min":0.79000000000000004,"5min":0.46200000000000002,"15min":0.26500000000000001},"max":{"1min":1.5580000000000001,"5min":3.327,"15min":9.3849999999999998},"last":1.143}]},{"osd":1,"last update":"Sun Oct 1 11:20:16 2023","interfaces":[{"interface":"back","average":{"1min":1.012,"5min":0.93400000000000005,"15min":1.2649999999999999},"min":{"1min":0.63800000000000001,"5min":0.56799999999999995,"15min":0.28199999999999997},"max":{"1min":1.264,"5min":1.3560000000000001,"15min":14.946999999999999},"last":1.21},{"interface":"front","average":{"1min":0.97599999999999998,"5min":0.97299999999999998,"15min":1.115},"min":{"1min":0.63300000000000001,"5min":0.59699999999999998,"15min":0.31},"max":{"1min":1.278,"5min":1.5469999999999999,"15min":8.1159999999999997},"last":1.042}]},{"osd":2,"last update":"Sun Oct 1 11:20:26 2023","interfaces":[{"interface":"back","average":{"1min":0.84899999999999998,"5min":0.90700000000000003,"15min":1.718},"min":{"1min":0.58999999999999997,"5min":0.47399999999999998,"15min":0.29799999999999999},"max":{"1min":1.1479999999999999,"5min":1.3300000000000001,"15min":87.287999999999997},"last":0.85899999999999999},{"interface":"front","average":{"1min":0.97699999999999998,"5min":0.97399999999999998,"15min":1.738},"min":{"1min":0.71999999999999997,"5min":0.54600000000000004,"15min":0.32300000000000001},"max":{"1min":1.1879999999999999,"5min":1.7110000000000001,"15min":88.028000000000006},"last":1.236}]},{"osd":3,"last update":"Sun Oct 1 11:20:16 2023","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":0.98299999999999998,"15min":1.3899999999999999},"min":{"1min":0.61599999999999999,"5min":0.55100000000000005,"15min":0.30399999999999999},"max":{"1min":1.304,"5min":1.4379999999999999,"15min":50.225000000000001},"last":0.92300000000000004},{"interface":"front","average":{"1min":0.998,"5min":0.98399999999999999,"15min":1.3859999999999999},"min":{"1min":0.79100000000000004,"5min":0.624,"15min":0.33900000000000002},"max":{"1min":1.22,"5min":1.4419999999999999,"15min":49.301000000000002},"last":1.1140000000000001}]},{"osd":4,"last update":"Sun Oct 1 11:20:40 2023","interfaces":[{"interface":"back","average":{"1min":0.97799999999999998,"5min":0.95099999999999996,"15min":1.1659999999999999},"min":{"1min":0.625,"5min":0.47799999999999998,"15min":0.315},"max":{"1min":1.2589999999999999,"5min":1.3979999999999999,"15min":8.7210000000000001},"last":0.77800000000000002},{"interface":"front","average":{"1min":1.0620000000000001,"5min":1.044,"15min":1.3560000000000001},"min":{"1min":0.73499999999999999,"5min":0.59099999999999997,"15min":0.35399999999999998},"max":{"1min":1.242,"5min":1.5169999999999999,"15min":15.462},"last":0.97499999999999998}]},{"osd":5,"last update":"Sun Oct 1 11:20:16 2023","interfaces":[{"interface":"back","average":{"1min":1.1000000000000001,"5min":1.0409999999999999,"15min":1.306},"min":{"1min":0.70899999999999996,"5min":0.49099999999999999,"15min":0.33100000000000002},"max":{"1min":1.3600000000000001,"5min":1.488,"15min":26.379000000000001},"last":0.82199999999999995},{"interface":"front","average":{"1min":1.1180000000000001,"5min":1.085,"15min":1.4359999999999999},"min":{"1min":0.88600000000000001,"5min":0.65200000000000002,"15min":0.32000000000000001},"max":{"1min":1.3959999999999999,"5min":1.468,"15min":28.010000000000002},"last":0.89000000000000001}]},{"osd":7,"last update":"Sun Oct 1 11:19:53 2023","interfaces":[{"interface":"back","average":{"1min":1.1279999999999999,"5min":1.103,"15min":1.173},"min":{"1min":0.89600000000000002,"5min":0.49099999999999999,"15min":0.35499999999999998},"max":{"1min":1.4890000000000001,"5min":1.585,"15min":8.1370000000000005},"last":1.2649999999999999},{"interface":"front","average":{"1min":1.151,"5min":1.115,"15min":1.274},"min":{"1min":0.84099999999999997,"5min":0.51000000000000001,"15min":0.34999999999999998},"max":{"1min":1.3979999999999999,"5min":1.5589999999999999,"15min":14.894},"last":1.171}]}]},{"osd":1,"up_from":101,"seq":433791697479,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1003772,"kb_used_data":9012,"kb_used_omap":4,"kb_used_meta":994747,"kb_avail":92741380,"statfs":{"total":95995035648,"available":94967173120,"internally_reserved":0,"allocated":9228288,"data_stored":7548147,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":1018621838},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 11:20:13 2023","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.66800000000000004,"15min":0.92300000000000004},"min":{"1min":0.307,"5min":0.307,"15min":0.19800000000000001},"max":{"1min":0.91400000000000003,"5min":1.244,"15min":11.756},"last":0.72199999999999998},{"interface":"front","average":{"1min":0.71399999999999997,"5min":0.70499999999999996,"15min":1.161},"min":{"1min":0.46899999999999997,"5min":0.309,"15min":0.215},"max":{"1min":1.7190000000000001,"5min":1.7190000000000001,"15min":11.943},"last":0.76600000000000001}]},{"osd":2,"last update":"Sun Oct 1 11:20:38 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.72999999999999998,"15min":2.0070000000000001},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.23400000000000001},"max":{"1min":0.97999999999999998,"5min":1.1579999999999999,"15min":130.90899999999999},"last":0.82999999999999996},{"interface":"front","average":{"1min":0.73699999999999999,"5min":0.80100000000000005,"15min":2.117},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.29199999999999998},"max":{"1min":1.526,"5min":1.526,"15min":130.72800000000001},"last":1.0409999999999999}]},{"osd":3,"last update":"Sun Oct 1 11:20:13 2023","interfaces":[{"interface":"back","average":{"1min":0.72399999999999998,"5min":0.72499999999999998,"15min":2.0419999999999998},"min":{"1min":0.379,"5min":0.379,"15min":0.249},"max":{"1min":1.0049999999999999,"5min":1.1519999999999999,"15min":130.79599999999999},"last":0.89700000000000002},{"interface":"front","average":{"1min":0.68400000000000005,"5min":0.75700000000000001,"15min":2.323},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.219},"max":{"1min":1.319,"5min":1.47,"15min":130.83500000000001},"last":0.92300000000000004}]},{"osd":4,"last update":"Sun Oct 1 11:20:38 2023","interfaces":[{"interface":"back","average":{"1min":0.90900000000000003,"5min":0.97599999999999998,"15min":2.4670000000000001},"min":{"1min":0.66200000000000003,"5min":0.56699999999999995,"15min":0.33300000000000002},"max":{"1min":1.2589999999999999,"5min":1.456,"15min":203.89500000000001},"last":0.95599999999999996},{"interface":"front","average":{"1min":0.93500000000000005,"5min":0.998,"15min":2.8210000000000002},"min":{"1min":0.71599999999999997,"5min":0.67600000000000005,"15min":0.32600000000000001},"max":{"1min":1.6579999999999999,"5min":1.6579999999999999,"15min":203.852},"last":0.96599999999999997}]},{"osd":5,"last update":"Sun Oct 1 11:20:13 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":1.038,"15min":1.321},"min":{"1min":0.67800000000000005,"5min":0.59699999999999998,"15min":0.33600000000000002},"max":{"1min":1.5860000000000001,"5min":1.696,"15min":12.192},"last":1.073},{"interface":"front","average":{"1min":1.0580000000000001,"5min":1.069,"15min":1.7330000000000001},"min":{"1min":0.79700000000000004,"5min":0.627,"15min":0.35299999999999998},"max":{"1min":1.363,"5min":1.5669999999999999,"15min":130.62299999999999},"last":1.137}]},{"osd":6,"last update":"Sun Oct 1 11:20:13 2023","interfaces":[{"interface":"back","average":{"1min":1.0800000000000001,"5min":1.093,"15min":1.7569999999999999},"min":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.371},"max":{"1min":1.3480000000000001,"5min":1.6619999999999999,"15min":130.75299999999999},"last":1.228},{"interface":"front","average":{"1min":1.071,"5min":1.1080000000000001,"15min":1.696},"min":{"1min":0.73399999999999999,"5min":0.52400000000000002,"15min":0.35599999999999998},"max":{"1min":1.772,"5min":1.772,"15min":58.238},"last":1.252}]},{"osd":7,"last update":"Sun Oct 1 11:20:13 2023","interfaces":[{"interface":"back","average":{"1min":1.113,"5min":1.1259999999999999,"15min":1.496},"min":{"1min":0.77300000000000002,"5min":0.64100000000000001,"15min":0.40100000000000002},"max":{"1min":1.8169999999999999,"5min":1.8169999999999999,"15min":12.055999999999999},"last":1.321},{"interface":"front","average":{"1min":1.0840000000000001,"5min":1.1339999999999999,"15min":1.913},"min":{"1min":0.78200000000000003,"5min":0.68899999999999995,"15min":0.40500000000000003},"max":{"1min":1.677,"5min":1.7669999999999999,"15min":130.42099999999999},"last":1.351}]}]},{"osd":0,"up_from":86,"seq":369367188089,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1061996,"kb_used_data":9508,"kb_used_omap":5,"kb_used_meta":1052474,"kb_avail":92683156,"statfs":{"total":95995035648,"available":94907551744,"internally_reserved":0,"allocated":9736192,"data_stored":8125705,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":1077734010},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Oct 1 11:20:31 2023","interfaces":[{"interface":"back","average":{"1min":0.67500000000000004,"5min":0.68899999999999995,"15min":1.0820000000000001},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.186},"max":{"1min":1.6639999999999999,"5min":1.6639999999999999,"15min":12.433},"last":0.73999999999999999},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.68600000000000005,"15min":1.4990000000000001},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.189},"max":{"1min":1.0980000000000001,"5min":1.0980000000000001,"15min":13.989000000000001},"last":0.70699999999999996}]},{"osd":2,"last update":"Sun Oct 1 11:20:31 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.68899999999999995,"15min":1.595},"min":{"1min":0.248,"5min":0.248,"15min":0.23000000000000001},"max":{"1min":1.484,"5min":1.484,"15min":67.076999999999998},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.71399999999999997,"5min":0.79000000000000004,"15min":1.7949999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.247},"max":{"1min":1.2809999999999999,"5min":1.3029999999999999,"15min":67.129999999999995},"last":0.91800000000000004}]},{"osd":3,"last update":"Sun Oct 1 11:20:31 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":0.81100000000000005,"15min":1.417},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.251},"max":{"1min":1.0169999999999999,"5min":1.232,"15min":67.706000000000003},"last":0.85699999999999998},{"interface":"front","average":{"1min":0.74299999999999999,"5min":0.79300000000000004,"15min":1.536},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.254},"max":{"1min":1.722,"5min":1.722,"15min":66.352000000000004},"last":0.82699999999999996}]},{"osd":4,"last update":"Sun Oct 1 11:20:48 2023","interfaces":[{"interface":"back","average":{"1min":0.95899999999999996,"5min":1.014,"15min":1.444},"min":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.35299999999999998},"max":{"1min":1.8089999999999999,"5min":1.8089999999999999,"15min":23.995000000000001},"last":1.349},{"interface":"front","average":{"1min":0.97699999999999998,"5min":0.995,"15min":1.575},"min":{"1min":0.71699999999999997,"5min":0.54900000000000004,"15min":0.33600000000000002},"max":{"1min":1.5660000000000001,"5min":1.7649999999999999,"15min":21.451000000000001},"last":1.5660000000000001}]},{"osd":5,"last update":"Sun Oct 1 11:20:31 2023","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":1.0569999999999999,"15min":1.4550000000000001},"min":{"1min":0.44,"5min":0.44,"15min":0.35099999999999998},"max":{"1min":1.3400000000000001,"5min":1.728,"15min":12.776999999999999},"last":1.3169999999999999},{"interface":"front","average":{"1min":1.048,"5min":1.1140000000000001,"15min":1.5880000000000001},"min":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.38200000000000001},"max":{"1min":1.9079999999999999,"5min":1.9079999999999999,"15min":14.529},"last":1.0489999999999999}]},{"osd":6,"last update":"Sun Oct 1 11:20:31 2023","interfaces":[{"interface":"back","average":{"1min":1.048,"5min":1.1120000000000001,"15min":1.3899999999999999},"min":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.373},"max":{"1min":1.853,"5min":1.853,"15min":10.430999999999999},"last":1.1060000000000001},{"interface":"front","average":{"1min":1.002,"5min":1.0569999999999999,"15min":1.4239999999999999},"min":{"1min":0.64400000000000002,"5min":0.51400000000000001,"15min":0.373},"max":{"1min":1.482,"5min":1.5900000000000001,"15min":12.035},"last":1.284}]},{"osd":7,"last update":"Sun Oct 1 11:20:31 2023","interfaces":[{"interface":"back","average":{"1min":1.0840000000000001,"5min":1.1759999999999999,"15min":1.3520000000000001},"min":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.42999999999999999},"max":{"1min":1.9510000000000001,"5min":1.9510000000000001,"15min":10.553000000000001},"last":1.1399999999999999},{"interface":"front","average":{"1min":1.042,"5min":1.111,"15min":1.5309999999999999},"min":{"1min":0.76800000000000002,"5min":0.65400000000000003,"15min":0.41499999999999998},"max":{"1min":1.448,"5min":1.6970000000000001,"15min":12.053000000000001},"last":1.363}]}]},{"osd":2,"up_from":116,"seq":498216206863,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":920228,"kb_used_data":4828,"kb_used_omap":7,"kb_used_meta":915384,"kb_avail":92824924,"statfs":{"total":95995035648,"available":95052722176,"internally_reserved":0,"allocated":4943872,"data_stored":3340074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8111,"internal_metadata":937353297},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 11:20:27 2023","interfaces":[{"interface":"back","average":{"1min":0.67300000000000004,"5min":0.66100000000000003,"15min":0.90900000000000003},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.20799999999999999},"max":{"1min":2.0760000000000001,"5min":2.0760000000000001,"15min":13.429},"last":0.98799999999999999},{"interface":"front","average":{"1min":0.68100000000000005,"5min":0.73699999999999999,"15min":1.444},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.20899999999999999},"max":{"1min":1.748,"5min":1.748,"15min":95.688999999999993},"last":0.78200000000000003}]},{"osd":1,"last update":"Sun Oct 1 11:20:27 2023","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.73099999999999998,"15min":1.429},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.254},"max":{"1min":1.123,"5min":1.24,"15min":34.753999999999998},"last":0.91300000000000003},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.70299999999999996,"15min":1.4370000000000001},"min":{"1min":0.314,"5min":0.314,"15min":0.27200000000000002},"max":{"1min":0.95099999999999996,"5min":1.175,"15min":34.972999999999999},"last":0.82199999999999995}]},{"osd":3,"last update":"Sun Oct 1 11:20:03 2023","interfaces":[{"interface":"back","average":{"1min":0.71799999999999997,"5min":0.72099999999999997,"15min":1.0680000000000001},"min":{"1min":0.35999999999999999,"5min":0.34799999999999998,"15min":0.24399999999999999},"max":{"1min":1.0069999999999999,"5min":1.141,"15min":13.744},"last":0.86099999999999999},{"interface":"front","average":{"1min":0.85599999999999998,"5min":0.83599999999999997,"15min":1.1779999999999999},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.21099999999999999},"max":{"1min":1.1930000000000001,"5min":1.4199999999999999,"15min":10.102},"last":1.038}]},{"osd":4,"last update":"Sun Oct 1 11:20:46 2023","interfaces":[{"interface":"back","average":{"1min":0.91800000000000004,"5min":1.0289999999999999,"15min":1.2789999999999999},"min":{"1min":0.63,"5min":0.63,"15min":0.312},"max":{"1min":1.2130000000000001,"5min":1.575,"15min":10.004},"last":1.181},{"interface":"front","average":{"1min":0.89800000000000002,"5min":0.96199999999999997,"15min":1.54},"min":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.33000000000000002},"max":{"1min":1.7110000000000001,"5min":1.7110000000000001,"15min":12.989000000000001},"last":1.0169999999999999}]},{"osd":5,"last update":"Sun Oct 1 11:20:27 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":1.0669999999999999,"15min":1.3120000000000001},"min":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.33600000000000002},"max":{"1min":1.383,"5min":1.694,"15min":9.968},"last":1.119},{"interface":"front","average":{"1min":1.032,"5min":1.0389999999999999,"15min":1.55},"min":{"1min":0.504,"5min":0.504,"15min":0.33100000000000002},"max":{"1min":2.0009999999999999,"5min":2.0009999999999999,"15min":12.518000000000001},"last":0.95099999999999996}]},{"osd":6,"last update":"Sun Oct 1 11:20:27 2023","interfaces":[{"interface":"back","average":{"1min":1.0329999999999999,"5min":1.048,"15min":1.742},"min":{"1min":0.70699999999999996,"5min":0.69399999999999995,"15min":0.39300000000000002},"max":{"1min":1.885,"5min":1.885,"15min":95.802999999999997},"last":1.0660000000000001},{"interface":"front","average":{"1min":1.034,"5min":1.0820000000000001,"15min":1.278},"min":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.33700000000000002},"max":{"1min":1.573,"5min":1.573,"15min":10.007},"last":1.0840000000000001}]},{"osd":7,"last update":"Sun Oct 1 11:20:27 2023","interfaces":[{"interface":"back","average":{"1min":1.0900000000000001,"5min":1.1339999999999999,"15min":1.276},"min":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.36699999999999999},"max":{"1min":1.6830000000000001,"5min":1.6830000000000001,"15min":10.053000000000001},"last":1.1519999999999999},{"interface":"front","average":{"1min":1.0720000000000001,"5min":1.1539999999999999,"15min":1.671},"min":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.35199999999999998},"max":{"1min":1.639,"5min":1.754,"15min":95.228999999999999},"last":1.2869999999999999}]}]},{"osd":3,"up_from":121,"seq":519691043336,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":858484,"kb_used_data":4844,"kb_used_omap":2,"kb_used_meta":853629,"kb_avail":92886668,"statfs":{"total":95995035648,"available":95115948032,"internally_reserved":0,"allocated":4960256,"data_stored":3341934,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":874116149},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 11:19:58 2023","interfaces":[{"interface":"back","average":{"1min":0.71999999999999997,"5min":0.69899999999999995,"15min":2.0059999999999998},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.222},"max":{"1min":1.333,"5min":1.333,"15min":259.98599999999999},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.72499999999999998,"5min":0.69299999999999995,"15min":2.0640000000000001},"min":{"1min":0.44500000000000001,"5min":0.32500000000000001,"15min":0.217},"max":{"1min":1.115,"5min":1.115,"15min":259.93000000000001},"last":0.69399999999999995}]},{"osd":1,"last update":"Sun Oct 1 11:19:58 2023","interfaces":[{"interface":"back","average":{"1min":0.75700000000000001,"5min":0.754,"15min":2.2629999999999999},"min":{"1min":0.45200000000000001,"5min":0.37,"15min":0.23999999999999999},"max":{"1min":1.1499999999999999,"5min":1.3979999999999999,"15min":259.80200000000002},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.77400000000000002,"5min":0.83799999999999997,"15min":2.2040000000000002},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.22},"max":{"1min":1.4730000000000001,"5min":1.4730000000000001,"15min":260.08499999999998},"last":1.0349999999999999}]},{"osd":2,"last update":"Sun Oct 1 11:19:58 2023","interfaces":[{"interface":"back","average":{"1min":0.755,"5min":0.71899999999999997,"15min":2.9710000000000001},"min":{"1min":0.42299999999999999,"5min":0.32300000000000001,"15min":0.22600000000000001},"max":{"1min":1.2569999999999999,"5min":1.2569999999999999,"15min":259.83600000000001},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.89900000000000002,"5min":0.91600000000000004,"15min":2.8180000000000001},"min":{"1min":0.436,"5min":0.436,"15min":0.29299999999999998},"max":{"1min":1.3320000000000001,"5min":1.389,"15min":260.13200000000001},"last":0.66200000000000003}]},{"osd":4,"last update":"Sun Oct 1 11:19:49 2023","interfaces":[{"interface":"back","average":{"1min":1.121,"5min":3.8130000000000002,"15min":2.105},"min":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.32100000000000001},"max":{"1min":1.6040000000000001,"5min":260.173,"15min":260.173},"last":0.90200000000000002},{"interface":"front","average":{"1min":1.097,"5min":3.7429999999999999,"15min":2.1680000000000001},"min":{"1min":0.74199999999999999,"5min":0.51300000000000001,"15min":0.31900000000000001},"max":{"1min":1.466,"5min":260.18400000000003,"15min":260.18400000000003},"last":0.86399999999999999}]},{"osd":5,"last update":"Sun Oct 1 11:19:58 2023","interfaces":[{"interface":"back","average":{"1min":1.0349999999999999,"5min":0.996,"15min":2.2250000000000001},"min":{"1min":0.57399999999999995,"5min":0.53900000000000003,"15min":0.35599999999999998},"max":{"1min":1.391,"5min":1.514,"15min":260.11500000000001},"last":0.81100000000000005},{"interface":"front","average":{"1min":1.0720000000000001,"5min":1.0089999999999999,"15min":2.4670000000000001},"min":{"1min":0.61099999999999999,"5min":0.59899999999999998,"15min":0.35799999999999998},"max":{"1min":2.3500000000000001,"5min":2.3500000000000001,"15min":260.38799999999998},"last":0.74399999999999999}]},{"osd":6,"last update":"Sun Oct 1 11:19:58 2023","interfaces":[{"interface":"back","average":{"1min":1.1819999999999999,"5min":1.081,"15min":2.2890000000000001},"min":{"1min":0.75600000000000001,"5min":0.57999999999999996,"15min":0.32100000000000001},"max":{"1min":2.3100000000000001,"5min":2.3100000000000001,"15min":260.24000000000001},"last":0.96199999999999997},{"interface":"front","average":{"1min":1.048,"5min":1.0880000000000001,"15min":2.399},"min":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.40400000000000003},"max":{"1min":1.544,"5min":1.5660000000000001,"15min":260.45100000000002},"last":0.94899999999999995}]},{"osd":7,"last update":"Sun Oct 1 11:19:58 2023","interfaces":[{"interface":"back","average":{"1min":1.121,"5min":1.0940000000000001,"15min":2.3730000000000002},"min":{"1min":0.76700000000000002,"5min":0.59099999999999997,"15min":0.39100000000000001},"max":{"1min":1.603,"5min":1.603,"15min":260.51799999999997},"last":0.85299999999999998},{"interface":"front","average":{"1min":1.159,"5min":1.1439999999999999,"15min":2.1680000000000001},"min":{"1min":0.69399999999999995,"5min":0.61199999999999999,"15min":0.38},"max":{"1min":1.6180000000000001,"5min":1.7070000000000001,"15min":260.48200000000003},"last":1.111}]}]},{"osd":4,"up_from":126,"seq":541165879810,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":750736,"kb_used_data":4872,"kb_used_omap":10,"kb_used_meta":745845,"kb_avail":92994416,"statfs":{"total":95995035648,"available":95226281984,"internally_reserved":0,"allocated":4988928,"data_stored":3343266,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":763745892},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 11:20:33 2023","interfaces":[{"interface":"back","average":{"1min":0.97299999999999998,"5min":0.94399999999999995,"15min":1.0980000000000001},"min":{"1min":0.55300000000000005,"5min":0.47599999999999998,"15min":0.28000000000000003},"max":{"1min":1.2130000000000001,"5min":1.8859999999999999,"15min":11.025},"last":1.1220000000000001},{"interface":"front","average":{"1min":1.018,"5min":0.97399999999999998,"15min":1.2},"min":{"1min":0.68700000000000006,"5min":0.48999999999999999,"15min":0.26400000000000001},"max":{"1min":1.548,"5min":1.8109999999999999,"15min":12.369},"last":0.82599999999999996}]},{"osd":1,"last update":"Sun Oct 1 11:20:33 2023","interfaces":[{"interface":"back","average":{"1min":0.91000000000000003,"5min":0.95499999999999996,"15min":1.274},"min":{"1min":0.69099999999999995,"5min":0.56399999999999995,"15min":0.30199999999999999},"max":{"1min":1.252,"5min":1.3300000000000001,"15min":9.1920000000000002},"last":0.95299999999999996},{"interface":"front","average":{"1min":1.0109999999999999,"5min":0.99199999999999999,"15min":1.1719999999999999},"min":{"1min":0.59399999999999997,"5min":0.50600000000000001,"15min":0.26300000000000001},"max":{"1min":1.4570000000000001,"5min":2.0550000000000002,"15min":11.145},"last":1.014}]},{"osd":2,"last update":"Sun Oct 1 11:20:33 2023","interfaces":[{"interface":"back","average":{"1min":0.98199999999999998,"5min":1.0069999999999999,"15min":2.2410000000000001},"min":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.34699999999999998},"max":{"1min":1.3420000000000001,"5min":1.972,"15min":219.05500000000001},"last":0.73299999999999998},{"interface":"front","average":{"1min":0.99399999999999999,"5min":0.99299999999999999,"15min":2.1139999999999999},"min":{"1min":0.63600000000000001,"5min":0.55800000000000005,"15min":0.33900000000000002},"max":{"1min":1.595,"5min":2.02,"15min":218.81200000000001},"last":1.0649999999999999}]},{"osd":3,"last update":"Sun Oct 1 11:20:33 2023","interfaces":[{"interface":"back","average":{"1min":1.0389999999999999,"5min":1.0549999999999999,"15min":1.4790000000000001},"min":{"1min":0.69299999999999995,"5min":0.495,"15min":0.34200000000000003},"max":{"1min":1.431,"5min":2.1320000000000001,"15min":18.475999999999999},"last":0.98999999999999999},{"interface":"front","average":{"1min":1.04,"5min":0.99099999999999999,"15min":1.518},"min":{"1min":0.67700000000000005,"5min":0.61299999999999999,"15min":0.33500000000000002},"max":{"1min":1.4139999999999999,"5min":1.4139999999999999,"15min":18.303000000000001},"last":1.1759999999999999}]},{"osd":5,"last update":"Sun Oct 1 11:20:33 2023","interfaces":[{"interface":"back","average":{"1min":1.073,"5min":1.0369999999999999,"15min":1.3759999999999999},"min":{"1min":0.65200000000000002,"5min":0.57099999999999995,"15min":0.30299999999999999},"max":{"1min":1.502,"5min":2.165,"15min":25.175999999999998},"last":0.92200000000000004},{"interface":"front","average":{"1min":1.077,"5min":1.0589999999999999,"15min":1.413},"min":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.307},"max":{"1min":1.379,"5min":2.097,"15min":11.084},"last":1.212}]},{"osd":6,"last update":"Sun Oct 1 11:20:33 2023","interfaces":[{"interface":"back","average":{"1min":1.125,"5min":1.0840000000000001,"15min":1.3009999999999999},"min":{"1min":0.78400000000000003,"5min":0.66600000000000004,"15min":0.35199999999999998},"max":{"1min":1.5740000000000001,"5min":2.1899999999999999,"15min":12.635999999999999},"last":1.1539999999999999},{"interface":"front","average":{"1min":1.0649999999999999,"5min":1.0669999999999999,"15min":1.343},"min":{"1min":0.71099999999999997,"5min":0.63,"15min":0.32500000000000001},"max":{"1min":1.363,"5min":2.3199999999999998,"15min":10.506},"last":1.0980000000000001}]},{"osd":7,"last update":"Sun Oct 1 11:20:33 2023","interfaces":[{"interface":"back","average":{"1min":1.1040000000000001,"5min":1.0980000000000001,"15min":1.4059999999999999},"min":{"1min":0.83099999999999996,"5min":0.64900000000000002,"15min":0.376},"max":{"1min":1.4059999999999999,"5min":1.879,"15min":10.851000000000001},"last":1.2509999999999999},{"interface":"front","average":{"1min":1.2529999999999999,"5min":1.2050000000000001,"15min":1.359},"min":{"1min":0.71499999999999997,"5min":0.61599999999999999,"15min":0.40500000000000003},"max":{"1min":1.579,"5min":2.1600000000000001,"15min":12.364000000000001},"last":1.292}]}]},{"osd":5,"up_from":36,"seq":154618823491,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1494104,"kb_used_data":3728,"kb_used_omap":0,"kb_used_meta":1490368,"kb_avail":92251048,"statfs":{"total":95995035648,"available":94465073152,"internally_reserved":0,"allocated":3817472,"data_stored":3201115,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":1526136832},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Oct 1 11:19:59 2023","interfaces":[{"interface":"back","average":{"1min":0.93100000000000005,"5min":0.93999999999999995,"15min":1.599},"min":{"1min":0.56200000000000006,"5min":0.53200000000000003,"15min":0.27100000000000002},"max":{"1min":1.2370000000000001,"5min":4.2619999999999996,"15min":111.797},"last":1.232},{"interface":"front","average":{"1min":0.88800000000000001,"5min":0.93500000000000005,"15min":1.677},"min":{"1min":0.60199999999999998,"5min":0.51800000000000002,"15min":0.251},"max":{"1min":1.1930000000000001,"5min":4.2229999999999999,"15min":111.492},"last":0.85099999999999998}]},{"osd":1,"last update":"Sun Oct 1 11:19:59 2023","interfaces":[{"interface":"back","average":{"1min":0.97499999999999998,"5min":1.0049999999999999,"15min":2.1339999999999999},"min":{"1min":0.57699999999999996,"5min":0.46200000000000002,"15min":0.26300000000000001},"max":{"1min":1.6020000000000001,"5min":4.375,"15min":132.99600000000001},"last":0.95499999999999996},{"interface":"front","average":{"1min":1.0600000000000001,"5min":1.024,"15min":2.2919999999999998},"min":{"1min":0.70299999999999996,"5min":0.49199999999999999,"15min":0.27500000000000002},"max":{"1min":1.3160000000000001,"5min":4.3369999999999997,"15min":133.42400000000001},"last":0.92500000000000004}]},{"osd":2,"last update":"Sun Oct 1 11:20:26 2023","interfaces":[{"interface":"back","average":{"1min":0.98199999999999998,"5min":1.044,"15min":2.2519999999999998},"min":{"1min":0.63600000000000001,"5min":0.59899999999999998,"15min":0.309},"max":{"1min":1.351,"5min":4.327,"15min":129.70500000000001},"last":1.103},{"interface":"front","average":{"1min":0.95499999999999996,"5min":1.0129999999999999,"15min":2.5219999999999998},"min":{"1min":0.61399999999999999,"5min":0.61099999999999999,"15min":0.28799999999999998},"max":{"1min":1.2430000000000001,"5min":4.3890000000000002,"15min":129.76300000000001},"last":0.72499999999999998}]},{"osd":3,"last update":"Sun Oct 1 11:19:59 2023","interfaces":[{"interface":"back","average":{"1min":1.022,"5min":1.0189999999999999,"15min":1.8300000000000001},"min":{"1min":0.82099999999999995,"5min":0.57199999999999995,"15min":0.34499999999999997},"max":{"1min":1.232,"5min":4.444,"15min":131.09399999999999},"last":0.996},{"interface":"front","average":{"1min":1.0049999999999999,"5min":1.026,"15min":1.6519999999999999},"min":{"1min":0.75600000000000001,"5min":0.67100000000000004,"15min":0.36299999999999999},"max":{"1min":1.4079999999999999,"5min":4.7489999999999997,"15min":130.70699999999999},"last":0.877}]},{"osd":4,"last update":"Sun Oct 1 11:20:26 2023","interfaces":[{"interface":"back","average":{"1min":1.054,"5min":1.0509999999999999,"15min":1.236},"min":{"1min":0.80400000000000005,"5min":0.48599999999999999,"15min":0.36799999999999999},"max":{"1min":1.28,"5min":4.1820000000000004,"15min":10.167},"last":1.1910000000000001},{"interface":"front","average":{"1min":1.147,"5min":1.107,"15min":1.3400000000000001},"min":{"1min":0.74399999999999999,"5min":0.54200000000000004,"15min":0.34100000000000003},"max":{"1min":1.784,"5min":5.444,"15min":11.162000000000001},"last":1.0449999999999999}]},{"osd":6,"last update":"Sun Oct 1 11:19:59 2023","interfaces":[{"interface":"back","average":{"1min":1.113,"5min":1.123,"15min":1.1870000000000001},"min":{"1min":0.82899999999999996,"5min":0.52200000000000002,"15min":0.33100000000000002},"max":{"1min":1.3460000000000001,"5min":5.0419999999999998,"15min":11.196999999999999},"last":1.179},{"interface":"front","average":{"1min":1.115,"5min":1.1000000000000001,"15min":1.1519999999999999},"min":{"1min":0.91700000000000004,"5min":0.629,"15min":0.35499999999999998},"max":{"1min":1.6399999999999999,"5min":4.8019999999999996,"15min":12.189},"last":1.0700000000000001}]},{"osd":7,"last update":"Sun Oct 1 11:19:59 2023","interfaces":[{"interface":"back","average":{"1min":1.1419999999999999,"5min":1.1319999999999999,"15min":1.143},"min":{"1min":0.94899999999999995,"5min":0.60599999999999998,"15min":0.36699999999999999},"max":{"1min":1.569,"5min":4.9850000000000003,"15min":9.1959999999999997},"last":1.3009999999999999},{"interface":"front","average":{"1min":1.2310000000000001,"5min":1.2250000000000001,"15min":1.2569999999999999},"min":{"1min":0.88300000000000001,"5min":0.56599999999999995,"15min":0.39900000000000002},"max":{"1min":1.413,"5min":5.1100000000000003,"15min":11.215},"last":1.3169999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5510,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4210,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6928,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10652,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-10-01T11:20:52.315 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-10-01T11:20:52.315 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph_fc1e90a395a6a5631142eb70fd9bf3d22740a9da/qa/tasks/workunit.py", line 119, in task assert 'client' in role AssertionError 2023-10-01T11:20:52.379 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=bab83aa8362245d789538165696fe576 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph_fc1e90a395a6a5631142eb70fd9bf3d22740a9da/qa/tasks/workunit.py", line 119, in task assert 'client' in role AssertionError 2023-10-01T11:20:52.381 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2023-10-01T11:20:52.392 INFO:tasks.cephadm:Teardown begin 2023-10-01T11:20:52.392 DEBUG:teuthology.orchestra.run.smithi078:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-01T11:20:52.439 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:20:52 smithi078 bash[75299]: cluster 2023-10-01T11:20:50.635171+0000 mgr.y (mgr.25072) 1910 : cluster 0 pgmap v2107: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:52.439 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:20:52 smithi078 bash[78274]: cluster 2023-10-01T11:20:50.635171+0000 mgr.y (mgr.25072) 1910 : cluster 0 pgmap v2107: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:52.440 DEBUG:teuthology.orchestra.run.smithi173:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-01T11:20:52.453 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:20:52 smithi173 bash[58816]: cluster 2023-10-01T11:20:50.635171+0000 mgr.y (mgr.25072) 1910 : cluster 0 pgmap v2107: 73 pgs: 73 active+clean; 4.6 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail 2023-10-01T11:20:52.550 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2023-10-01T11:20:52.550 DEBUG:teuthology.orchestra.run.smithi078:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-10-01T11:20:52.562 DEBUG:teuthology.orchestra.run.smithi173:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-10-01T11:20:52.599 INFO:tasks.cephadm:Stopping all daemons... 2023-10-01T11:20:52.599 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2023-10-01T11:20:52.599 DEBUG:teuthology.orchestra.run.smithi078:> sudo systemctl stop ceph-d8016480-6041-11ee-8db5-212e2dc638e7@mon.a 2023-10-01T11:20:52.811 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:20:52 smithi078 systemd[1]: Stopping Ceph mon.a for d8016480-6041-11ee-8db5-212e2dc638e7... 2023-10-01T11:20:53.081 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:20:52 smithi078 bash[75299]: debug 2023-10-01T11:20:52.808+0000 7f31ce5db700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-10-01T11:20:53.082 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:20:52 smithi078 bash[75299]: debug 2023-10-01T11:20:52.808+0000 7f31ce5db700 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2023-10-01T11:20:53.082 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:20:52 smithi078 bash[71214]: [01/Oct/2023:11:20:52] ENGINE Bus STOPPING 2023-10-01T11:20:53.082 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:20:52 smithi078 bash[71214]: [01/Oct/2023:11:20:52] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-10-01T11:20:53.082 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:20:52 smithi078 bash[71214]: [01/Oct/2023:11:20:52] ENGINE Bus STOPPED 2023-10-01T11:20:53.082 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:20:52 smithi078 bash[71214]: [01/Oct/2023:11:20:52] ENGINE Bus STARTING 2023-10-01T11:20:53.083 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:20:52 smithi078 bash[71214]: [01/Oct/2023:11:20:52] ENGINE Serving on http://:::9283 2023-10-01T11:20:53.083 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:20:52 smithi078 bash[71214]: [01/Oct/2023:11:20:52] ENGINE Bus STARTED 2023-10-01T11:20:53.982 INFO:journalctl@ceph.mon.a.smithi078.stdout:Oct 01 11:20:53 smithi078 bash[276213]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-mon-a 2023-10-01T11:20:54.006 DEBUG:teuthology.orchestra.run.smithi078:> sudo pkill -f 'journalctl -f -n 0 -u ceph-d8016480-6041-11ee-8db5-212e2dc638e7@mon.a.service' 2023-10-01T11:20:54.148 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-01T11:20:54.148 INFO:tasks.cephadm.mon.a:Stopped mon.a 2023-10-01T11:20:54.149 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2023-10-01T11:20:54.149 DEBUG:teuthology.orchestra.run.smithi078:> sudo systemctl stop ceph-d8016480-6041-11ee-8db5-212e2dc638e7@mon.c 2023-10-01T11:20:54.308 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:20:54 smithi078 systemd[1]: Stopping Ceph mon.c for d8016480-6041-11ee-8db5-212e2dc638e7... 2023-10-01T11:20:54.579 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:20:54 smithi078 bash[78274]: debug 2023-10-01T11:20:54.304+0000 7fe053c93700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-10-01T11:20:54.579 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:20:54 smithi078 bash[78274]: debug 2023-10-01T11:20:54.304+0000 7fe053c93700 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2023-10-01T11:20:55.560 INFO:journalctl@ceph.mon.c.smithi078.stdout:Oct 01 11:20:55 smithi078 bash[276331]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-mon-c 2023-10-01T11:20:55.570 DEBUG:teuthology.orchestra.run.smithi078:> sudo pkill -f 'journalctl -f -n 0 -u ceph-d8016480-6041-11ee-8db5-212e2dc638e7@mon.c.service' 2023-10-01T11:20:55.666 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-01T11:20:55.666 INFO:tasks.cephadm.mon.b:Stopped mon.c 2023-10-01T11:20:55.666 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2023-10-01T11:20:55.666 DEBUG:teuthology.orchestra.run.smithi173:> sudo systemctl stop ceph-d8016480-6041-11ee-8db5-212e2dc638e7@mon.b 2023-10-01T11:20:55.982 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:20:55 smithi173 systemd[1]: Stopping Ceph mon.b for d8016480-6041-11ee-8db5-212e2dc638e7... 2023-10-01T11:20:56.271 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:20:55 smithi173 bash[58816]: debug 2023-10-01T11:20:55.978+0000 7f84d7047700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-10-01T11:20:56.272 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:20:55 smithi173 bash[58816]: debug 2023-10-01T11:20:55.978+0000 7f84d7047700 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2023-10-01T11:20:57.254 INFO:journalctl@ceph.mon.b.smithi173.stdout:Oct 01 11:20:56 smithi173 bash[166556]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-mon-b 2023-10-01T11:20:57.262 DEBUG:teuthology.orchestra.run.smithi173:> sudo pkill -f 'journalctl -f -n 0 -u ceph-d8016480-6041-11ee-8db5-212e2dc638e7@mon.b.service' 2023-10-01T11:20:57.340 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-01T11:20:57.340 INFO:tasks.cephadm.mon.b:Stopped mon.b 2023-10-01T11:20:57.340 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2023-10-01T11:20:57.341 DEBUG:teuthology.orchestra.run.smithi078:> sudo systemctl stop ceph-d8016480-6041-11ee-8db5-212e2dc638e7@mgr.y 2023-10-01T11:20:57.615 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:20:57 smithi078 systemd[1]: Stopping Ceph mgr.y for d8016480-6041-11ee-8db5-212e2dc638e7... 2023-10-01T11:20:57.615 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:20:57 smithi078 bash[71214]: debug 2023-10-01T11:20:57.516+0000 7fd9fef73700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-01T11:20:57.615 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:20:57 smithi078 bash[71214]: debug 2023-10-01T11:20:57.516+0000 7fd9fef73700 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2023-10-01T11:20:58.580 INFO:journalctl@ceph.mgr.y.smithi078.stdout:Oct 01 11:20:58 smithi078 bash[276431]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-mgr-y 2023-10-01T11:20:58.591 DEBUG:teuthology.orchestra.run.smithi078:> sudo pkill -f 'journalctl -f -n 0 -u ceph-d8016480-6041-11ee-8db5-212e2dc638e7@mgr.y.service' 2023-10-01T11:20:58.674 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-01T11:20:58.674 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2023-10-01T11:20:58.674 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2023-10-01T11:20:58.674 DEBUG:teuthology.orchestra.run.smithi173:> sudo systemctl stop ceph-d8016480-6041-11ee-8db5-212e2dc638e7@mgr.x 2023-10-01T11:20:59.087 INFO:journalctl@ceph.mgr.x.smithi173.stdout:Oct 01 11:20:58 smithi173 systemd[1]: Stopping Ceph mgr.x for d8016480-6041-11ee-8db5-212e2dc638e7... 2023-10-01T11:20:59.868 DEBUG:teuthology.orchestra.run.smithi173:> sudo pkill -f 'journalctl -f -n 0 -u ceph-d8016480-6041-11ee-8db5-212e2dc638e7@mgr.x.service' 2023-10-01T11:20:59.896 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-01T11:20:59.897 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2023-10-01T11:20:59.897 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2023-10-01T11:20:59.897 DEBUG:teuthology.orchestra.run.smithi078:> sudo systemctl stop ceph-d8016480-6041-11ee-8db5-212e2dc638e7@osd.0 2023-10-01T11:21:00.344 INFO:journalctl@ceph.osd.0.smithi078.stdout:Oct 01 11:20:59 smithi078 systemd[1]: Stopping Ceph osd.0 for d8016480-6041-11ee-8db5-212e2dc638e7... 2023-10-01T11:21:00.345 INFO:journalctl@ceph.osd.0.smithi078.stdout:Oct 01 11:21:00 smithi078 bash[84784]: debug 2023-10-01T11:21:00.084+0000 7f3481c35700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-01T11:21:00.345 INFO:journalctl@ceph.osd.0.smithi078.stdout:Oct 01 11:21:00 smithi078 bash[84784]: debug 2023-10-01T11:21:00.084+0000 7f3481c35700 -1 osd.0 439 *** Got signal Terminated *** 2023-10-01T11:21:00.345 INFO:journalctl@ceph.osd.0.smithi078.stdout:Oct 01 11:21:00 smithi078 bash[84784]: debug 2023-10-01T11:21:00.084+0000 7f3481c35700 -1 osd.0 439 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-01T11:21:02.369 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 11:21:02 smithi173 bash[54497]: ts=2023-10-01T11:21:02.077Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.78:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.78:8765: connect: connection refused" 2023-10-01T11:21:02.369 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 11:21:02 smithi173 bash[54497]: ts=2023-10-01T11:21:02.077Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.78:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.78:8765: connect: connection refused" 2023-10-01T11:21:02.369 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 11:21:02 smithi173 bash[54497]: ts=2023-10-01T11:21:02.068Z caller=refresh.go:98 level=error component="discovery manager notify" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.78:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.78:8765: connect: connection refused" 2023-10-01T11:21:02.370 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 11:21:02 smithi173 bash[54497]: ts=2023-10-01T11:21:02.071Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.78:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.78:8765: connect: connection refused" 2023-10-01T11:21:06.538 INFO:journalctl@ceph.osd.0.smithi078.stdout:Oct 01 11:21:06 smithi078 bash[276535]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd-0 2023-10-01T11:21:06.815 INFO:journalctl@ceph.osd.0.smithi078.stdout:Oct 01 11:21:06 smithi078 bash[276601]: Error response from daemon: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd-0 2023-10-01T11:21:09.254 DEBUG:teuthology.orchestra.run.smithi078:> sudo pkill -f 'journalctl -f -n 0 -u ceph-d8016480-6041-11ee-8db5-212e2dc638e7@osd.0.service' 2023-10-01T11:21:09.362 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-01T11:21:09.363 INFO:tasks.cephadm.osd.0:Stopped osd.0 2023-10-01T11:21:09.363 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2023-10-01T11:21:09.363 DEBUG:teuthology.orchestra.run.smithi078:> sudo systemctl stop ceph-d8016480-6041-11ee-8db5-212e2dc638e7@osd.1 2023-10-01T11:21:09.560 INFO:journalctl@ceph.osd.1.smithi078.stdout:Oct 01 11:21:09 smithi078 systemd[1]: Stopping Ceph osd.1 for d8016480-6041-11ee-8db5-212e2dc638e7... 2023-10-01T11:21:09.561 INFO:journalctl@ceph.osd.1.smithi078.stdout:Oct 01 11:21:09 smithi078 bash[91532]: debug 2023-10-01T11:21:09.543+0000 7fe940e8d700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-01T11:21:09.561 INFO:journalctl@ceph.osd.1.smithi078.stdout:Oct 01 11:21:09 smithi078 bash[91532]: debug 2023-10-01T11:21:09.543+0000 7fe940e8d700 -1 osd.1 439 *** Got signal Terminated *** 2023-10-01T11:21:09.561 INFO:journalctl@ceph.osd.1.smithi078.stdout:Oct 01 11:21:09 smithi078 bash[91532]: debug 2023-10-01T11:21:09.543+0000 7fe940e8d700 -1 osd.1 439 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-01T11:21:16.081 INFO:journalctl@ceph.osd.1.smithi078.stdout:Oct 01 11:21:15 smithi078 bash[276743]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd-1 2023-10-01T11:21:16.081 INFO:journalctl@ceph.osd.1.smithi078.stdout:Oct 01 11:21:16 smithi078 bash[276810]: Error response from daemon: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd-1 2023-10-01T11:21:18.113 DEBUG:teuthology.orchestra.run.smithi078:> sudo pkill -f 'journalctl -f -n 0 -u ceph-d8016480-6041-11ee-8db5-212e2dc638e7@osd.1.service' 2023-10-01T11:21:18.200 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-01T11:21:18.200 INFO:tasks.cephadm.osd.1:Stopped osd.1 2023-10-01T11:21:18.200 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2023-10-01T11:21:18.201 DEBUG:teuthology.orchestra.run.smithi078:> sudo systemctl stop ceph-d8016480-6041-11ee-8db5-212e2dc638e7@osd.2 2023-10-01T11:21:18.389 INFO:journalctl@ceph.osd.2.smithi078.stdout:Oct 01 11:21:18 smithi078 systemd[1]: Stopping Ceph osd.2 for d8016480-6041-11ee-8db5-212e2dc638e7... 2023-10-01T11:21:18.390 INFO:journalctl@ceph.osd.2.smithi078.stdout:Oct 01 11:21:18 smithi078 bash[97994]: debug 2023-10-01T11:21:18.375+0000 7f9c92ad1700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-01T11:21:18.390 INFO:journalctl@ceph.osd.2.smithi078.stdout:Oct 01 11:21:18 smithi078 bash[97994]: debug 2023-10-01T11:21:18.375+0000 7f9c92ad1700 -1 osd.2 439 *** Got signal Terminated *** 2023-10-01T11:21:18.390 INFO:journalctl@ceph.osd.2.smithi078.stdout:Oct 01 11:21:18 smithi078 bash[97994]: debug 2023-10-01T11:21:18.375+0000 7f9c92ad1700 -1 osd.2 439 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-01T11:21:24.977 INFO:journalctl@ceph.osd.2.smithi078.stdout:Oct 01 11:21:24 smithi078 bash[276953]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd-2 2023-10-01T11:21:25.331 INFO:journalctl@ceph.osd.2.smithi078.stdout:Oct 01 11:21:24 smithi078 bash[277017]: Error response from daemon: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd-2 2023-10-01T11:21:26.514 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:26 smithi173 bash[43488]: debug 2023-10-01T11:21:26.142+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:02.351468+0000 front 2023-10-01T11:21:02.351568+0000 (oldest deadline 2023-10-01T11:21:25.851364+0000) 2023-10-01T11:21:26.869 INFO:journalctl@ceph.osd.4.smithi173.stdout:Oct 01 11:21:26 smithi173 bash[64546]: debug 2023-10-01T11:21:26.510+0000 7f33f5637700 -1 osd.4 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:20:59.955684+0000 front 2023-10-01T11:20:59.955644+0000 (oldest deadline 2023-10-01T11:21:25.855636+0000) 2023-10-01T11:21:27.155 DEBUG:teuthology.orchestra.run.smithi078:> sudo pkill -f 'journalctl -f -n 0 -u ceph-d8016480-6041-11ee-8db5-212e2dc638e7@osd.2.service' 2023-10-01T11:21:27.217 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-01T11:21:27.217 INFO:tasks.cephadm.osd.2:Stopped osd.2 2023-10-01T11:21:27.218 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2023-10-01T11:21:27.218 DEBUG:teuthology.orchestra.run.smithi078:> sudo systemctl stop ceph-d8016480-6041-11ee-8db5-212e2dc638e7@osd.3 2023-10-01T11:21:27.448 INFO:journalctl@ceph.osd.3.smithi078.stdout:Oct 01 11:21:27 smithi078 systemd[1]: Stopping Ceph osd.3 for d8016480-6041-11ee-8db5-212e2dc638e7... 2023-10-01T11:21:27.537 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:27 smithi173 bash[43488]: debug 2023-10-01T11:21:27.166+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:02.351468+0000 front 2023-10-01T11:21:02.351568+0000 (oldest deadline 2023-10-01T11:21:25.851364+0000) 2023-10-01T11:21:27.537 INFO:journalctl@ceph.osd.5.smithi173.stdout:Oct 01 11:21:27 smithi173 bash[35238]: debug 2023-10-01T11:21:27.378+0000 7f62e9807700 -1 osd.5 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:04.800418+0000 front 2023-10-01T11:21:04.800449+0000 (oldest deadline 2023-10-01T11:21:27.099965+0000) 2023-10-01T11:21:27.831 INFO:journalctl@ceph.osd.3.smithi078.stdout:Oct 01 11:21:27 smithi078 bash[100923]: debug 2023-10-01T11:21:27.443+0000 7f2e798bb700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-01T11:21:27.831 INFO:journalctl@ceph.osd.3.smithi078.stdout:Oct 01 11:21:27 smithi078 bash[100923]: debug 2023-10-01T11:21:27.443+0000 7f2e798bb700 -1 osd.3 439 *** Got signal Terminated *** 2023-10-01T11:21:27.832 INFO:journalctl@ceph.osd.3.smithi078.stdout:Oct 01 11:21:27 smithi078 bash[100923]: debug 2023-10-01T11:21:27.443+0000 7f2e798bb700 -1 osd.3 439 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-01T11:21:27.869 INFO:journalctl@ceph.osd.4.smithi173.stdout:Oct 01 11:21:27 smithi173 bash[64546]: debug 2023-10-01T11:21:27.534+0000 7f33f5637700 -1 osd.4 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:20:59.955684+0000 front 2023-10-01T11:20:59.955644+0000 (oldest deadline 2023-10-01T11:21:25.855636+0000) 2023-10-01T11:21:28.529 INFO:journalctl@ceph.osd.5.smithi173.stdout:Oct 01 11:21:28 smithi173 bash[35238]: debug 2023-10-01T11:21:28.330+0000 7f62e9807700 -1 osd.5 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:04.800418+0000 front 2023-10-01T11:21:04.800449+0000 (oldest deadline 2023-10-01T11:21:27.099965+0000) 2023-10-01T11:21:28.529 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:28 smithi173 bash[43488]: debug 2023-10-01T11:21:28.154+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:02.351468+0000 front 2023-10-01T11:21:02.351568+0000 (oldest deadline 2023-10-01T11:21:25.851364+0000) 2023-10-01T11:21:28.869 INFO:journalctl@ceph.osd.4.smithi173.stdout:Oct 01 11:21:28 smithi173 bash[64546]: debug 2023-10-01T11:21:28.526+0000 7f33f5637700 -1 osd.4 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:20:59.955684+0000 front 2023-10-01T11:20:59.955644+0000 (oldest deadline 2023-10-01T11:21:25.855636+0000) 2023-10-01T11:21:28.869 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:28 smithi173 bash[39321]: debug 2023-10-01T11:21:28.590+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:04.883741+0000 front 2023-10-01T11:21:04.883677+0000 (oldest deadline 2023-10-01T11:21:27.783163+0000) 2023-10-01T11:21:29.503 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:29 smithi173 bash[43488]: debug 2023-10-01T11:21:29.170+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:02.351468+0000 front 2023-10-01T11:21:02.351568+0000 (oldest deadline 2023-10-01T11:21:25.851364+0000) 2023-10-01T11:21:29.503 INFO:journalctl@ceph.osd.5.smithi173.stdout:Oct 01 11:21:29 smithi173 bash[35238]: debug 2023-10-01T11:21:29.322+0000 7f62e9807700 -1 osd.5 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:04.800418+0000 front 2023-10-01T11:21:04.800449+0000 (oldest deadline 2023-10-01T11:21:27.099965+0000) 2023-10-01T11:21:29.869 INFO:journalctl@ceph.osd.4.smithi173.stdout:Oct 01 11:21:29 smithi173 bash[64546]: debug 2023-10-01T11:21:29.498+0000 7f33f5637700 -1 osd.4 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:20:59.955684+0000 front 2023-10-01T11:20:59.955644+0000 (oldest deadline 2023-10-01T11:21:25.855636+0000) 2023-10-01T11:21:29.869 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:29 smithi173 bash[39321]: debug 2023-10-01T11:21:29.618+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:04.883741+0000 front 2023-10-01T11:21:04.883677+0000 (oldest deadline 2023-10-01T11:21:27.783163+0000) 2023-10-01T11:21:30.527 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:30 smithi173 bash[43488]: debug 2023-10-01T11:21:30.137+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:02.351468+0000 front 2023-10-01T11:21:02.351568+0000 (oldest deadline 2023-10-01T11:21:25.851364+0000) 2023-10-01T11:21:30.528 INFO:journalctl@ceph.osd.5.smithi173.stdout:Oct 01 11:21:30 smithi173 bash[35238]: debug 2023-10-01T11:21:30.277+0000 7f62e9807700 -1 osd.5 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:04.800418+0000 front 2023-10-01T11:21:04.800449+0000 (oldest deadline 2023-10-01T11:21:27.099965+0000) 2023-10-01T11:21:30.581 INFO:journalctl@ceph.osd.3.smithi078.stdout:Oct 01 11:21:30 smithi078 bash[100923]: debug 2023-10-01T11:21:30.203+0000 7f2e756b0700 -1 osd.3 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:04.699605+0000 front 2023-10-01T11:21:04.699481+0000 (oldest deadline 2023-10-01T11:21:29.999598+0000) 2023-10-01T11:21:30.869 INFO:journalctl@ceph.osd.4.smithi173.stdout:Oct 01 11:21:30 smithi173 bash[64546]: debug 2023-10-01T11:21:30.525+0000 7f33f5637700 -1 osd.4 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:20:59.955684+0000 front 2023-10-01T11:20:59.955644+0000 (oldest deadline 2023-10-01T11:21:25.855636+0000) 2023-10-01T11:21:30.869 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:30 smithi173 bash[39321]: debug 2023-10-01T11:21:30.617+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:04.883741+0000 front 2023-10-01T11:21:04.883677+0000 (oldest deadline 2023-10-01T11:21:27.783163+0000) 2023-10-01T11:21:31.539 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:31 smithi173 bash[43488]: debug 2023-10-01T11:21:31.133+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:02.351468+0000 front 2023-10-01T11:21:02.351568+0000 (oldest deadline 2023-10-01T11:21:25.851364+0000) 2023-10-01T11:21:31.539 INFO:journalctl@ceph.osd.5.smithi173.stdout:Oct 01 11:21:31 smithi173 bash[35238]: debug 2023-10-01T11:21:31.325+0000 7f62e9807700 -1 osd.5 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:04.800418+0000 front 2023-10-01T11:21:04.800449+0000 (oldest deadline 2023-10-01T11:21:27.099965+0000) 2023-10-01T11:21:31.581 INFO:journalctl@ceph.osd.3.smithi078.stdout:Oct 01 11:21:31 smithi078 bash[100923]: debug 2023-10-01T11:21:31.175+0000 7f2e756b0700 -1 osd.3 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:04.699605+0000 front 2023-10-01T11:21:04.699481+0000 (oldest deadline 2023-10-01T11:21:29.999598+0000) 2023-10-01T11:21:31.869 INFO:journalctl@ceph.osd.4.smithi173.stdout:Oct 01 11:21:31 smithi173 bash[64546]: debug 2023-10-01T11:21:31.537+0000 7f33f5637700 -1 osd.4 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:20:59.955684+0000 front 2023-10-01T11:20:59.955644+0000 (oldest deadline 2023-10-01T11:21:25.855636+0000) 2023-10-01T11:21:31.870 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:31 smithi173 bash[39321]: debug 2023-10-01T11:21:31.665+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:04.883741+0000 front 2023-10-01T11:21:04.883677+0000 (oldest deadline 2023-10-01T11:21:27.783163+0000) 2023-10-01T11:21:32.369 INFO:journalctl@ceph.osd.5.smithi173.stdout:Oct 01 11:21:32 smithi173 bash[35238]: debug 2023-10-01T11:21:32.317+0000 7f62e9807700 -1 osd.5 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:04.800418+0000 front 2023-10-01T11:21:04.800449+0000 (oldest deadline 2023-10-01T11:21:27.099965+0000) 2023-10-01T11:21:32.369 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:32 smithi173 bash[43488]: debug 2023-10-01T11:21:32.093+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:02.351468+0000 front 2023-10-01T11:21:02.351568+0000 (oldest deadline 2023-10-01T11:21:25.851364+0000) 2023-10-01T11:21:32.581 INFO:journalctl@ceph.osd.3.smithi078.stdout:Oct 01 11:21:32 smithi078 bash[100923]: debug 2023-10-01T11:21:32.195+0000 7f2e756b0700 -1 osd.3 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:04.699605+0000 front 2023-10-01T11:21:04.699481+0000 (oldest deadline 2023-10-01T11:21:29.999598+0000) 2023-10-01T11:21:32.869 INFO:journalctl@ceph.osd.4.smithi173.stdout:Oct 01 11:21:32 smithi173 bash[64546]: debug 2023-10-01T11:21:32.573+0000 7f33f5637700 -1 osd.4 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:20:59.955684+0000 front 2023-10-01T11:20:59.955644+0000 (oldest deadline 2023-10-01T11:21:25.855636+0000) 2023-10-01T11:21:32.869 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:32 smithi173 bash[39321]: debug 2023-10-01T11:21:32.661+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:04.883741+0000 front 2023-10-01T11:21:04.883677+0000 (oldest deadline 2023-10-01T11:21:27.783163+0000) 2023-10-01T11:21:33.332 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:33 smithi173 bash[43488]: debug 2023-10-01T11:21:33.057+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:02.351468+0000 front 2023-10-01T11:21:02.351568+0000 (oldest deadline 2023-10-01T11:21:25.851364+0000) 2023-10-01T11:21:33.619 INFO:journalctl@ceph.osd.5.smithi173.stdout:Oct 01 11:21:33 smithi173 bash[35238]: debug 2023-10-01T11:21:33.329+0000 7f62e9807700 -1 osd.5 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:04.800418+0000 front 2023-10-01T11:21:04.800449+0000 (oldest deadline 2023-10-01T11:21:27.099965+0000) 2023-10-01T11:21:33.619 INFO:journalctl@ceph.osd.4.smithi173.stdout:Oct 01 11:21:33 smithi173 bash[64546]: debug 2023-10-01T11:21:33.561+0000 7f33f5637700 -1 osd.4 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:20:59.955684+0000 front 2023-10-01T11:20:59.955644+0000 (oldest deadline 2023-10-01T11:21:25.855636+0000) 2023-10-01T11:21:34.009 INFO:journalctl@ceph.osd.3.smithi078.stdout:Oct 01 11:21:33 smithi078 bash[277161]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd-3 2023-10-01T11:21:34.009 INFO:journalctl@ceph.osd.3.smithi078.stdout:Oct 01 11:21:33 smithi078 bash[277228]: Error response from daemon: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd-3 2023-10-01T11:21:34.062 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:33 smithi173 bash[39321]: debug 2023-10-01T11:21:33.653+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:04.883741+0000 front 2023-10-01T11:21:04.883677+0000 (oldest deadline 2023-10-01T11:21:27.783163+0000) 2023-10-01T11:21:34.361 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:34 smithi173 bash[43488]: debug 2023-10-01T11:21:34.057+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:02.351468+0000 front 2023-10-01T11:21:02.351568+0000 (oldest deadline 2023-10-01T11:21:25.851364+0000) 2023-10-01T11:21:34.619 INFO:journalctl@ceph.osd.5.smithi173.stdout:Oct 01 11:21:34 smithi173 bash[35238]: debug 2023-10-01T11:21:34.357+0000 7f62e9807700 -1 osd.5 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:04.800418+0000 front 2023-10-01T11:21:04.800449+0000 (oldest deadline 2023-10-01T11:21:27.099965+0000) 2023-10-01T11:21:34.619 INFO:journalctl@ceph.osd.4.smithi173.stdout:Oct 01 11:21:34 smithi173 bash[64546]: debug 2023-10-01T11:21:34.557+0000 7f33f5637700 -1 osd.4 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:20:59.955684+0000 front 2023-10-01T11:20:59.955644+0000 (oldest deadline 2023-10-01T11:21:25.855636+0000) 2023-10-01T11:21:35.084 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:34 smithi173 bash[39321]: debug 2023-10-01T11:21:34.629+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:04.883741+0000 front 2023-10-01T11:21:04.883677+0000 (oldest deadline 2023-10-01T11:21:27.783163+0000) 2023-10-01T11:21:35.369 INFO:journalctl@ceph.osd.5.smithi173.stdout:Oct 01 11:21:35 smithi173 bash[35238]: debug 2023-10-01T11:21:35.309+0000 7f62e9807700 -1 osd.5 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:04.800418+0000 front 2023-10-01T11:21:04.800449+0000 (oldest deadline 2023-10-01T11:21:27.099965+0000) 2023-10-01T11:21:35.369 INFO:journalctl@ceph.osd.5.smithi173.stdout:Oct 01 11:21:35 smithi173 bash[35238]: debug 2023-10-01T11:21:35.309+0000 7f62e9807700 -1 osd.5 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:10.001910+0000 front 2023-10-01T11:21:10.002143+0000 (oldest deadline 2023-10-01T11:21:35.301741+0000) 2023-10-01T11:21:35.369 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:35 smithi173 bash[43488]: debug 2023-10-01T11:21:35.081+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:02.351468+0000 front 2023-10-01T11:21:02.351568+0000 (oldest deadline 2023-10-01T11:21:25.851364+0000) 2023-10-01T11:21:35.869 INFO:journalctl@ceph.osd.4.smithi173.stdout:Oct 01 11:21:35 smithi173 bash[64546]: debug 2023-10-01T11:21:35.561+0000 7f33f5637700 -1 osd.4 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:20:59.955684+0000 front 2023-10-01T11:20:59.955644+0000 (oldest deadline 2023-10-01T11:21:25.855636+0000) 2023-10-01T11:21:35.869 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:35 smithi173 bash[39321]: debug 2023-10-01T11:21:35.641+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:04.883741+0000 front 2023-10-01T11:21:04.883677+0000 (oldest deadline 2023-10-01T11:21:27.783163+0000) 2023-10-01T11:21:35.869 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:35 smithi173 bash[39321]: debug 2023-10-01T11:21:35.641+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:11.884876+0000 front 2023-10-01T11:21:11.884681+0000 (oldest deadline 2023-10-01T11:21:35.384313+0000) 2023-10-01T11:21:36.045 DEBUG:teuthology.orchestra.run.smithi078:> sudo pkill -f 'journalctl -f -n 0 -u ceph-d8016480-6041-11ee-8db5-212e2dc638e7@osd.3.service' 2023-10-01T11:21:36.082 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-01T11:21:36.083 INFO:tasks.cephadm.osd.3:Stopped osd.3 2023-10-01T11:21:36.083 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2023-10-01T11:21:36.083 DEBUG:teuthology.orchestra.run.smithi173:> sudo systemctl stop ceph-d8016480-6041-11ee-8db5-212e2dc638e7@osd.4 2023-10-01T11:21:36.341 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:36 smithi173 bash[43488]: debug 2023-10-01T11:21:36.073+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:02.351468+0000 front 2023-10-01T11:21:02.351568+0000 (oldest deadline 2023-10-01T11:21:25.851364+0000) 2023-10-01T11:21:36.341 INFO:journalctl@ceph.osd.4.smithi173.stdout:Oct 01 11:21:36 smithi173 systemd[1]: Stopping Ceph osd.4 for d8016480-6041-11ee-8db5-212e2dc638e7... 2023-10-01T11:21:36.341 INFO:journalctl@ceph.osd.4.smithi173.stdout:Oct 01 11:21:36 smithi173 bash[64546]: debug 2023-10-01T11:21:36.289+0000 7f33fd049700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-01T11:21:36.342 INFO:journalctl@ceph.osd.4.smithi173.stdout:Oct 01 11:21:36 smithi173 bash[64546]: debug 2023-10-01T11:21:36.289+0000 7f33fd049700 -1 osd.4 439 *** Got signal Terminated *** 2023-10-01T11:21:36.342 INFO:journalctl@ceph.osd.4.smithi173.stdout:Oct 01 11:21:36 smithi173 bash[64546]: debug 2023-10-01T11:21:36.289+0000 7f33fd049700 -1 osd.4 439 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-01T11:21:36.615 INFO:journalctl@ceph.osd.4.smithi173.stdout:Oct 01 11:21:36 smithi173 bash[64546]: debug 2023-10-01T11:21:36.517+0000 7f33f5637700 -1 osd.4 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:20:59.955684+0000 front 2023-10-01T11:20:59.955644+0000 (oldest deadline 2023-10-01T11:21:25.855636+0000) 2023-10-01T11:21:36.617 INFO:journalctl@ceph.osd.5.smithi173.stdout:Oct 01 11:21:36 smithi173 bash[35238]: debug 2023-10-01T11:21:36.337+0000 7f62e9807700 -1 osd.5 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:04.800418+0000 front 2023-10-01T11:21:04.800449+0000 (oldest deadline 2023-10-01T11:21:27.099965+0000) 2023-10-01T11:21:36.617 INFO:journalctl@ceph.osd.5.smithi173.stdout:Oct 01 11:21:36 smithi173 bash[35238]: debug 2023-10-01T11:21:36.337+0000 7f62e9807700 -1 osd.5 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:10.001910+0000 front 2023-10-01T11:21:10.002143+0000 (oldest deadline 2023-10-01T11:21:35.301741+0000) 2023-10-01T11:21:36.869 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:36 smithi173 bash[39321]: debug 2023-10-01T11:21:36.609+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:04.883741+0000 front 2023-10-01T11:21:04.883677+0000 (oldest deadline 2023-10-01T11:21:27.783163+0000) 2023-10-01T11:21:36.869 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:36 smithi173 bash[39321]: debug 2023-10-01T11:21:36.609+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:11.884876+0000 front 2023-10-01T11:21:11.884681+0000 (oldest deadline 2023-10-01T11:21:35.384313+0000) 2023-10-01T11:21:37.369 INFO:journalctl@ceph.osd.5.smithi173.stdout:Oct 01 11:21:37 smithi173 bash[35238]: debug 2023-10-01T11:21:37.321+0000 7f62e9807700 -1 osd.5 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:04.800418+0000 front 2023-10-01T11:21:04.800449+0000 (oldest deadline 2023-10-01T11:21:27.099965+0000) 2023-10-01T11:21:37.369 INFO:journalctl@ceph.osd.5.smithi173.stdout:Oct 01 11:21:37 smithi173 bash[35238]: debug 2023-10-01T11:21:37.321+0000 7f62e9807700 -1 osd.5 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:10.001910+0000 front 2023-10-01T11:21:10.002143+0000 (oldest deadline 2023-10-01T11:21:35.301741+0000) 2023-10-01T11:21:37.370 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:37 smithi173 bash[43488]: debug 2023-10-01T11:21:37.105+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:02.351468+0000 front 2023-10-01T11:21:02.351568+0000 (oldest deadline 2023-10-01T11:21:25.851364+0000) 2023-10-01T11:21:37.370 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:37 smithi173 bash[43488]: debug 2023-10-01T11:21:37.105+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:12.754237+0000 front 2023-10-01T11:21:12.754426+0000 (oldest deadline 2023-10-01T11:21:36.853921+0000) 2023-10-01T11:21:37.869 INFO:journalctl@ceph.osd.4.smithi173.stdout:Oct 01 11:21:37 smithi173 bash[64546]: debug 2023-10-01T11:21:37.561+0000 7f33f5637700 -1 osd.4 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:20:59.955684+0000 front 2023-10-01T11:20:59.955644+0000 (oldest deadline 2023-10-01T11:21:25.855636+0000) 2023-10-01T11:21:37.869 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:37 smithi173 bash[39321]: debug 2023-10-01T11:21:37.589+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:04.883741+0000 front 2023-10-01T11:21:04.883677+0000 (oldest deadline 2023-10-01T11:21:27.783163+0000) 2023-10-01T11:21:37.870 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:37 smithi173 bash[39321]: debug 2023-10-01T11:21:37.589+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:11.884876+0000 front 2023-10-01T11:21:11.884681+0000 (oldest deadline 2023-10-01T11:21:35.384313+0000) 2023-10-01T11:21:38.565 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:38 smithi173 bash[43488]: debug 2023-10-01T11:21:38.141+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:02.351468+0000 front 2023-10-01T11:21:02.351568+0000 (oldest deadline 2023-10-01T11:21:25.851364+0000) 2023-10-01T11:21:38.565 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:38 smithi173 bash[43488]: debug 2023-10-01T11:21:38.141+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:12.754237+0000 front 2023-10-01T11:21:12.754426+0000 (oldest deadline 2023-10-01T11:21:36.853921+0000) 2023-10-01T11:21:38.566 INFO:journalctl@ceph.osd.5.smithi173.stdout:Oct 01 11:21:38 smithi173 bash[35238]: debug 2023-10-01T11:21:38.289+0000 7f62e9807700 -1 osd.5 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:04.800418+0000 front 2023-10-01T11:21:04.800449+0000 (oldest deadline 2023-10-01T11:21:27.099965+0000) 2023-10-01T11:21:38.566 INFO:journalctl@ceph.osd.5.smithi173.stdout:Oct 01 11:21:38 smithi173 bash[35238]: debug 2023-10-01T11:21:38.289+0000 7f62e9807700 -1 osd.5 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:10.001910+0000 front 2023-10-01T11:21:10.002143+0000 (oldest deadline 2023-10-01T11:21:35.301741+0000) 2023-10-01T11:21:38.869 INFO:journalctl@ceph.osd.4.smithi173.stdout:Oct 01 11:21:38 smithi173 bash[64546]: debug 2023-10-01T11:21:38.609+0000 7f33f5637700 -1 osd.4 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:20:59.955684+0000 front 2023-10-01T11:20:59.955644+0000 (oldest deadline 2023-10-01T11:21:25.855636+0000) 2023-10-01T11:21:38.869 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:38 smithi173 bash[39321]: debug 2023-10-01T11:21:38.561+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:04.883741+0000 front 2023-10-01T11:21:04.883677+0000 (oldest deadline 2023-10-01T11:21:27.783163+0000) 2023-10-01T11:21:38.869 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:38 smithi173 bash[39321]: debug 2023-10-01T11:21:38.561+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:11.884876+0000 front 2023-10-01T11:21:11.884681+0000 (oldest deadline 2023-10-01T11:21:35.384313+0000) 2023-10-01T11:21:39.369 INFO:journalctl@ceph.osd.5.smithi173.stdout:Oct 01 11:21:39 smithi173 bash[35238]: debug 2023-10-01T11:21:39.265+0000 7f62e9807700 -1 osd.5 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:04.800418+0000 front 2023-10-01T11:21:04.800449+0000 (oldest deadline 2023-10-01T11:21:27.099965+0000) 2023-10-01T11:21:39.369 INFO:journalctl@ceph.osd.5.smithi173.stdout:Oct 01 11:21:39 smithi173 bash[35238]: debug 2023-10-01T11:21:39.265+0000 7f62e9807700 -1 osd.5 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:10.001910+0000 front 2023-10-01T11:21:10.002143+0000 (oldest deadline 2023-10-01T11:21:35.301741+0000) 2023-10-01T11:21:39.369 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:39 smithi173 bash[43488]: debug 2023-10-01T11:21:39.097+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:02.351468+0000 front 2023-10-01T11:21:02.351568+0000 (oldest deadline 2023-10-01T11:21:25.851364+0000) 2023-10-01T11:21:39.370 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:39 smithi173 bash[43488]: debug 2023-10-01T11:21:39.097+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:12.754237+0000 front 2023-10-01T11:21:12.754426+0000 (oldest deadline 2023-10-01T11:21:36.853921+0000) 2023-10-01T11:21:39.869 INFO:journalctl@ceph.osd.4.smithi173.stdout:Oct 01 11:21:39 smithi173 bash[64546]: debug 2023-10-01T11:21:39.593+0000 7f33f5637700 -1 osd.4 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:20:59.955684+0000 front 2023-10-01T11:20:59.955644+0000 (oldest deadline 2023-10-01T11:21:25.855636+0000) 2023-10-01T11:21:39.869 INFO:journalctl@ceph.osd.4.smithi173.stdout:Oct 01 11:21:39 smithi173 bash[64546]: debug 2023-10-01T11:21:39.593+0000 7f33f5637700 -1 osd.4 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:13.458023+0000 front 2023-10-01T11:21:13.457815+0000 (oldest deadline 2023-10-01T11:21:38.757552+0000) 2023-10-01T11:21:39.869 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:39 smithi173 bash[39321]: debug 2023-10-01T11:21:39.561+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:04.883741+0000 front 2023-10-01T11:21:04.883677+0000 (oldest deadline 2023-10-01T11:21:27.783163+0000) 2023-10-01T11:21:39.870 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:39 smithi173 bash[39321]: debug 2023-10-01T11:21:39.561+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:11.884876+0000 front 2023-10-01T11:21:11.884681+0000 (oldest deadline 2023-10-01T11:21:35.384313+0000) 2023-10-01T11:21:40.307 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:40 smithi173 bash[43488]: debug 2023-10-01T11:21:40.053+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:02.351468+0000 front 2023-10-01T11:21:02.351568+0000 (oldest deadline 2023-10-01T11:21:25.851364+0000) 2023-10-01T11:21:40.307 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:40 smithi173 bash[43488]: debug 2023-10-01T11:21:40.053+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:12.754237+0000 front 2023-10-01T11:21:12.754426+0000 (oldest deadline 2023-10-01T11:21:36.853921+0000) 2023-10-01T11:21:40.586 INFO:journalctl@ceph.osd.5.smithi173.stdout:Oct 01 11:21:40 smithi173 bash[35238]: debug 2023-10-01T11:21:40.305+0000 7f62e9807700 -1 osd.5 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:04.800418+0000 front 2023-10-01T11:21:04.800449+0000 (oldest deadline 2023-10-01T11:21:27.099965+0000) 2023-10-01T11:21:40.587 INFO:journalctl@ceph.osd.5.smithi173.stdout:Oct 01 11:21:40 smithi173 bash[35238]: debug 2023-10-01T11:21:40.305+0000 7f62e9807700 -1 osd.5 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:10.001910+0000 front 2023-10-01T11:21:10.002143+0000 (oldest deadline 2023-10-01T11:21:35.301741+0000) 2023-10-01T11:21:40.869 INFO:journalctl@ceph.osd.4.smithi173.stdout:Oct 01 11:21:40 smithi173 bash[64546]: debug 2023-10-01T11:21:40.585+0000 7f33f5637700 -1 osd.4 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:20:59.955684+0000 front 2023-10-01T11:20:59.955644+0000 (oldest deadline 2023-10-01T11:21:25.855636+0000) 2023-10-01T11:21:40.869 INFO:journalctl@ceph.osd.4.smithi173.stdout:Oct 01 11:21:40 smithi173 bash[64546]: debug 2023-10-01T11:21:40.585+0000 7f33f5637700 -1 osd.4 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:13.458023+0000 front 2023-10-01T11:21:13.457815+0000 (oldest deadline 2023-10-01T11:21:38.757552+0000) 2023-10-01T11:21:40.869 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:40 smithi173 bash[39321]: debug 2023-10-01T11:21:40.585+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:04.883741+0000 front 2023-10-01T11:21:04.883677+0000 (oldest deadline 2023-10-01T11:21:27.783163+0000) 2023-10-01T11:21:40.870 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:40 smithi173 bash[39321]: debug 2023-10-01T11:21:40.585+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:11.884876+0000 front 2023-10-01T11:21:11.884681+0000 (oldest deadline 2023-10-01T11:21:35.384313+0000) 2023-10-01T11:21:41.351 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:41 smithi173 bash[43488]: debug 2023-10-01T11:21:41.009+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:02.351468+0000 front 2023-10-01T11:21:02.351568+0000 (oldest deadline 2023-10-01T11:21:25.851364+0000) 2023-10-01T11:21:41.352 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:41 smithi173 bash[43488]: debug 2023-10-01T11:21:41.009+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:12.754237+0000 front 2023-10-01T11:21:12.754426+0000 (oldest deadline 2023-10-01T11:21:36.853921+0000) 2023-10-01T11:21:41.619 INFO:journalctl@ceph.osd.5.smithi173.stdout:Oct 01 11:21:41 smithi173 bash[35238]: debug 2023-10-01T11:21:41.345+0000 7f62e9807700 -1 osd.5 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:04.800418+0000 front 2023-10-01T11:21:04.800449+0000 (oldest deadline 2023-10-01T11:21:27.099965+0000) 2023-10-01T11:21:41.619 INFO:journalctl@ceph.osd.5.smithi173.stdout:Oct 01 11:21:41 smithi173 bash[35238]: debug 2023-10-01T11:21:41.345+0000 7f62e9807700 -1 osd.5 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:10.001910+0000 front 2023-10-01T11:21:10.002143+0000 (oldest deadline 2023-10-01T11:21:35.301741+0000) 2023-10-01T11:21:41.619 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:41 smithi173 bash[39321]: debug 2023-10-01T11:21:41.541+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:04.883741+0000 front 2023-10-01T11:21:04.883677+0000 (oldest deadline 2023-10-01T11:21:27.783163+0000) 2023-10-01T11:21:41.620 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:41 smithi173 bash[39321]: debug 2023-10-01T11:21:41.541+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:11.884876+0000 front 2023-10-01T11:21:11.884681+0000 (oldest deadline 2023-10-01T11:21:35.384313+0000) 2023-10-01T11:21:42.227 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:42 smithi173 bash[43488]: debug 2023-10-01T11:21:42.001+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:02.351468+0000 front 2023-10-01T11:21:02.351568+0000 (oldest deadline 2023-10-01T11:21:25.851364+0000) 2023-10-01T11:21:42.227 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:42 smithi173 bash[43488]: debug 2023-10-01T11:21:42.001+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:12.754237+0000 front 2023-10-01T11:21:12.754426+0000 (oldest deadline 2023-10-01T11:21:36.853921+0000) 2023-10-01T11:21:42.369 INFO:journalctl@ceph.osd.5.smithi173.stdout:Oct 01 11:21:42 smithi173 bash[35238]: debug 2023-10-01T11:21:42.325+0000 7f62e9807700 -1 osd.5 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:04.800418+0000 front 2023-10-01T11:21:04.800449+0000 (oldest deadline 2023-10-01T11:21:27.099965+0000) 2023-10-01T11:21:42.369 INFO:journalctl@ceph.osd.5.smithi173.stdout:Oct 01 11:21:42 smithi173 bash[35238]: debug 2023-10-01T11:21:42.325+0000 7f62e9807700 -1 osd.5 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:10.001910+0000 front 2023-10-01T11:21:10.002143+0000 (oldest deadline 2023-10-01T11:21:35.301741+0000) 2023-10-01T11:21:42.869 INFO:journalctl@ceph.osd.4.smithi173.stdout:Oct 01 11:21:42 smithi173 bash[166765]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd-4 2023-10-01T11:21:42.869 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:42 smithi173 bash[39321]: debug 2023-10-01T11:21:42.565+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:04.883741+0000 front 2023-10-01T11:21:04.883677+0000 (oldest deadline 2023-10-01T11:21:27.783163+0000) 2023-10-01T11:21:42.870 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:42 smithi173 bash[39321]: debug 2023-10-01T11:21:42.565+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:11.884876+0000 front 2023-10-01T11:21:11.884681+0000 (oldest deadline 2023-10-01T11:21:35.384313+0000) 2023-10-01T11:21:43.369 INFO:journalctl@ceph.osd.4.smithi173.stdout:Oct 01 11:21:42 smithi173 bash[166831]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd-4 2023-10-01T11:21:43.369 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:43 smithi173 bash[43488]: debug 2023-10-01T11:21:43.017+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:02.351468+0000 front 2023-10-01T11:21:02.351568+0000 (oldest deadline 2023-10-01T11:21:25.851364+0000) 2023-10-01T11:21:43.370 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:43 smithi173 bash[43488]: debug 2023-10-01T11:21:43.017+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:12.754237+0000 front 2023-10-01T11:21:12.754426+0000 (oldest deadline 2023-10-01T11:21:36.853921+0000) 2023-10-01T11:21:43.869 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:43 smithi173 bash[39321]: debug 2023-10-01T11:21:43.585+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:04.883741+0000 front 2023-10-01T11:21:04.883677+0000 (oldest deadline 2023-10-01T11:21:27.783163+0000) 2023-10-01T11:21:43.869 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:43 smithi173 bash[39321]: debug 2023-10-01T11:21:43.585+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:11.884876+0000 front 2023-10-01T11:21:11.884681+0000 (oldest deadline 2023-10-01T11:21:35.384313+0000) 2023-10-01T11:21:43.869 INFO:journalctl@ceph.osd.5.smithi173.stdout:Oct 01 11:21:43 smithi173 bash[35238]: debug 2023-10-01T11:21:43.365+0000 7f62e9807700 -1 osd.5 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:04.800418+0000 front 2023-10-01T11:21:04.800449+0000 (oldest deadline 2023-10-01T11:21:27.099965+0000) 2023-10-01T11:21:43.870 INFO:journalctl@ceph.osd.5.smithi173.stdout:Oct 01 11:21:43 smithi173 bash[35238]: debug 2023-10-01T11:21:43.365+0000 7f62e9807700 -1 osd.5 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:10.001910+0000 front 2023-10-01T11:21:10.002143+0000 (oldest deadline 2023-10-01T11:21:35.301741+0000) 2023-10-01T11:21:44.369 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:44 smithi173 bash[43488]: debug 2023-10-01T11:21:44.041+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:02.351468+0000 front 2023-10-01T11:21:02.351568+0000 (oldest deadline 2023-10-01T11:21:25.851364+0000) 2023-10-01T11:21:44.369 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:44 smithi173 bash[43488]: debug 2023-10-01T11:21:44.041+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:12.754237+0000 front 2023-10-01T11:21:12.754426+0000 (oldest deadline 2023-10-01T11:21:36.853921+0000) 2023-10-01T11:21:44.847 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:44 smithi173 bash[39321]: debug 2023-10-01T11:21:44.561+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:04.883741+0000 front 2023-10-01T11:21:04.883677+0000 (oldest deadline 2023-10-01T11:21:27.783163+0000) 2023-10-01T11:21:44.848 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:44 smithi173 bash[39321]: debug 2023-10-01T11:21:44.561+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:11.884876+0000 front 2023-10-01T11:21:11.884681+0000 (oldest deadline 2023-10-01T11:21:35.384313+0000) 2023-10-01T11:21:44.848 INFO:journalctl@ceph.osd.5.smithi173.stdout:Oct 01 11:21:44 smithi173 bash[35238]: debug 2023-10-01T11:21:44.369+0000 7f62e9807700 -1 osd.5 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:04.800418+0000 front 2023-10-01T11:21:04.800449+0000 (oldest deadline 2023-10-01T11:21:27.099965+0000) 2023-10-01T11:21:44.848 INFO:journalctl@ceph.osd.5.smithi173.stdout:Oct 01 11:21:44 smithi173 bash[35238]: debug 2023-10-01T11:21:44.369+0000 7f62e9807700 -1 osd.5 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:10.001910+0000 front 2023-10-01T11:21:10.002143+0000 (oldest deadline 2023-10-01T11:21:35.301741+0000) 2023-10-01T11:21:45.119 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:44 smithi173 bash[43488]: debug 2023-10-01T11:21:44.993+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:02.351468+0000 front 2023-10-01T11:21:02.351568+0000 (oldest deadline 2023-10-01T11:21:25.851364+0000) 2023-10-01T11:21:45.119 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:44 smithi173 bash[43488]: debug 2023-10-01T11:21:44.993+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:12.754237+0000 front 2023-10-01T11:21:12.754426+0000 (oldest deadline 2023-10-01T11:21:36.853921+0000) 2023-10-01T11:21:45.590 DEBUG:teuthology.orchestra.run.smithi173:> sudo pkill -f 'journalctl -f -n 0 -u ceph-d8016480-6041-11ee-8db5-212e2dc638e7@osd.4.service' 2023-10-01T11:21:45.619 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-01T11:21:45.619 INFO:tasks.cephadm.osd.4:Stopped osd.4 2023-10-01T11:21:45.620 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2023-10-01T11:21:45.620 DEBUG:teuthology.orchestra.run.smithi173:> sudo systemctl stop ceph-d8016480-6041-11ee-8db5-212e2dc638e7@osd.5 2023-10-01T11:21:45.671 INFO:journalctl@ceph.osd.5.smithi173.stdout:Oct 01 11:21:45 smithi173 bash[35238]: debug 2023-10-01T11:21:45.405+0000 7f62e9807700 -1 osd.5 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:04.800418+0000 front 2023-10-01T11:21:04.800449+0000 (oldest deadline 2023-10-01T11:21:27.099965+0000) 2023-10-01T11:21:45.671 INFO:journalctl@ceph.osd.5.smithi173.stdout:Oct 01 11:21:45 smithi173 bash[35238]: debug 2023-10-01T11:21:45.405+0000 7f62e9807700 -1 osd.5 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:10.001910+0000 front 2023-10-01T11:21:10.002143+0000 (oldest deadline 2023-10-01T11:21:35.301741+0000) 2023-10-01T11:21:45.672 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:45 smithi173 bash[39321]: debug 2023-10-01T11:21:45.577+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:04.883741+0000 front 2023-10-01T11:21:04.883677+0000 (oldest deadline 2023-10-01T11:21:27.783163+0000) 2023-10-01T11:21:45.672 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:45 smithi173 bash[39321]: debug 2023-10-01T11:21:45.577+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:11.884876+0000 front 2023-10-01T11:21:11.884681+0000 (oldest deadline 2023-10-01T11:21:35.384313+0000) 2023-10-01T11:21:45.993 INFO:journalctl@ceph.osd.5.smithi173.stdout:Oct 01 11:21:45 smithi173 systemd[1]: Stopping Ceph osd.5 for d8016480-6041-11ee-8db5-212e2dc638e7... 2023-10-01T11:21:45.993 INFO:journalctl@ceph.osd.5.smithi173.stdout:Oct 01 11:21:45 smithi173 bash[35238]: debug 2023-10-01T11:21:45.841+0000 7f62ed88c700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-01T11:21:45.993 INFO:journalctl@ceph.osd.5.smithi173.stdout:Oct 01 11:21:45 smithi173 bash[35238]: debug 2023-10-01T11:21:45.841+0000 7f62ed88c700 -1 osd.5 439 *** Got signal Terminated *** 2023-10-01T11:21:45.994 INFO:journalctl@ceph.osd.5.smithi173.stdout:Oct 01 11:21:45 smithi173 bash[35238]: debug 2023-10-01T11:21:45.841+0000 7f62ed88c700 -1 osd.5 439 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-01T11:21:46.369 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:45 smithi173 bash[43488]: debug 2023-10-01T11:21:45.989+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:02.351468+0000 front 2023-10-01T11:21:02.351568+0000 (oldest deadline 2023-10-01T11:21:25.851364+0000) 2023-10-01T11:21:46.369 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:45 smithi173 bash[43488]: debug 2023-10-01T11:21:45.989+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:12.754237+0000 front 2023-10-01T11:21:12.754426+0000 (oldest deadline 2023-10-01T11:21:36.853921+0000) 2023-10-01T11:21:46.869 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:46 smithi173 bash[39321]: debug 2023-10-01T11:21:46.593+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:04.883741+0000 front 2023-10-01T11:21:04.883677+0000 (oldest deadline 2023-10-01T11:21:27.783163+0000) 2023-10-01T11:21:46.869 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:46 smithi173 bash[39321]: debug 2023-10-01T11:21:46.593+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:11.884876+0000 front 2023-10-01T11:21:11.884681+0000 (oldest deadline 2023-10-01T11:21:35.384313+0000) 2023-10-01T11:21:46.869 INFO:journalctl@ceph.osd.5.smithi173.stdout:Oct 01 11:21:46 smithi173 bash[35238]: debug 2023-10-01T11:21:46.429+0000 7f62e9807700 -1 osd.5 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:04.800418+0000 front 2023-10-01T11:21:04.800449+0000 (oldest deadline 2023-10-01T11:21:27.099965+0000) 2023-10-01T11:21:46.869 INFO:journalctl@ceph.osd.5.smithi173.stdout:Oct 01 11:21:46 smithi173 bash[35238]: debug 2023-10-01T11:21:46.429+0000 7f62e9807700 -1 osd.5 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:10.001910+0000 front 2023-10-01T11:21:10.002143+0000 (oldest deadline 2023-10-01T11:21:35.301741+0000) 2023-10-01T11:21:47.369 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:46 smithi173 bash[43488]: debug 2023-10-01T11:21:46.957+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:02.351468+0000 front 2023-10-01T11:21:02.351568+0000 (oldest deadline 2023-10-01T11:21:25.851364+0000) 2023-10-01T11:21:47.369 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:46 smithi173 bash[43488]: debug 2023-10-01T11:21:46.957+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:12.754237+0000 front 2023-10-01T11:21:12.754426+0000 (oldest deadline 2023-10-01T11:21:36.853921+0000) 2023-10-01T11:21:47.869 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:47 smithi173 bash[39321]: debug 2023-10-01T11:21:47.565+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:04.883741+0000 front 2023-10-01T11:21:04.883677+0000 (oldest deadline 2023-10-01T11:21:27.783163+0000) 2023-10-01T11:21:47.869 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:47 smithi173 bash[39321]: debug 2023-10-01T11:21:47.565+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:11.884876+0000 front 2023-10-01T11:21:11.884681+0000 (oldest deadline 2023-10-01T11:21:35.384313+0000) 2023-10-01T11:21:47.869 INFO:journalctl@ceph.osd.5.smithi173.stdout:Oct 01 11:21:47 smithi173 bash[35238]: debug 2023-10-01T11:21:47.413+0000 7f62e9807700 -1 osd.5 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:04.800418+0000 front 2023-10-01T11:21:04.800449+0000 (oldest deadline 2023-10-01T11:21:27.099965+0000) 2023-10-01T11:21:47.870 INFO:journalctl@ceph.osd.5.smithi173.stdout:Oct 01 11:21:47 smithi173 bash[35238]: debug 2023-10-01T11:21:47.413+0000 7f62e9807700 -1 osd.5 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:10.001910+0000 front 2023-10-01T11:21:10.002143+0000 (oldest deadline 2023-10-01T11:21:35.301741+0000) 2023-10-01T11:21:47.870 INFO:journalctl@ceph.osd.5.smithi173.stdout:Oct 01 11:21:47 smithi173 bash[35238]: debug 2023-10-01T11:21:47.413+0000 7f62e9807700 -1 osd.5 439 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2023-10-01T11:21:22.903916+0000 front 2023-10-01T11:21:22.903946+0000 (oldest deadline 2023-10-01T11:21:47.003721+0000) 2023-10-01T11:21:48.369 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:47 smithi173 bash[43488]: debug 2023-10-01T11:21:47.949+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:02.351468+0000 front 2023-10-01T11:21:02.351568+0000 (oldest deadline 2023-10-01T11:21:25.851364+0000) 2023-10-01T11:21:48.369 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:47 smithi173 bash[43488]: debug 2023-10-01T11:21:47.949+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:12.754237+0000 front 2023-10-01T11:21:12.754426+0000 (oldest deadline 2023-10-01T11:21:36.853921+0000) 2023-10-01T11:21:48.369 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:47 smithi173 bash[43488]: debug 2023-10-01T11:21:47.949+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2023-10-01T11:21:23.057715+0000 front 2023-10-01T11:21:23.057415+0000 (oldest deadline 2023-10-01T11:21:47.757342+0000) 2023-10-01T11:21:48.869 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:48 smithi173 bash[39321]: debug 2023-10-01T11:21:48.545+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:04.883741+0000 front 2023-10-01T11:21:04.883677+0000 (oldest deadline 2023-10-01T11:21:27.783163+0000) 2023-10-01T11:21:48.869 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:48 smithi173 bash[39321]: debug 2023-10-01T11:21:48.545+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:11.884876+0000 front 2023-10-01T11:21:11.884681+0000 (oldest deadline 2023-10-01T11:21:35.384313+0000) 2023-10-01T11:21:48.869 INFO:journalctl@ceph.osd.5.smithi173.stdout:Oct 01 11:21:48 smithi173 bash[35238]: debug 2023-10-01T11:21:48.441+0000 7f62e9807700 -1 osd.5 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:04.800418+0000 front 2023-10-01T11:21:04.800449+0000 (oldest deadline 2023-10-01T11:21:27.099965+0000) 2023-10-01T11:21:48.870 INFO:journalctl@ceph.osd.5.smithi173.stdout:Oct 01 11:21:48 smithi173 bash[35238]: debug 2023-10-01T11:21:48.441+0000 7f62e9807700 -1 osd.5 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:10.001910+0000 front 2023-10-01T11:21:10.002143+0000 (oldest deadline 2023-10-01T11:21:35.301741+0000) 2023-10-01T11:21:48.870 INFO:journalctl@ceph.osd.5.smithi173.stdout:Oct 01 11:21:48 smithi173 bash[35238]: debug 2023-10-01T11:21:48.441+0000 7f62e9807700 -1 osd.5 439 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2023-10-01T11:21:22.903916+0000 front 2023-10-01T11:21:22.903946+0000 (oldest deadline 2023-10-01T11:21:47.003721+0000) 2023-10-01T11:21:49.369 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:48 smithi173 bash[43488]: debug 2023-10-01T11:21:48.969+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:02.351468+0000 front 2023-10-01T11:21:02.351568+0000 (oldest deadline 2023-10-01T11:21:25.851364+0000) 2023-10-01T11:21:49.369 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:48 smithi173 bash[43488]: debug 2023-10-01T11:21:48.969+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:12.754237+0000 front 2023-10-01T11:21:12.754426+0000 (oldest deadline 2023-10-01T11:21:36.853921+0000) 2023-10-01T11:21:49.369 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:48 smithi173 bash[43488]: debug 2023-10-01T11:21:48.969+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2023-10-01T11:21:23.057715+0000 front 2023-10-01T11:21:23.057415+0000 (oldest deadline 2023-10-01T11:21:47.757342+0000) 2023-10-01T11:21:49.869 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:49 smithi173 bash[39321]: debug 2023-10-01T11:21:49.497+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:04.883741+0000 front 2023-10-01T11:21:04.883677+0000 (oldest deadline 2023-10-01T11:21:27.783163+0000) 2023-10-01T11:21:49.869 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:49 smithi173 bash[39321]: debug 2023-10-01T11:21:49.497+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:11.884876+0000 front 2023-10-01T11:21:11.884681+0000 (oldest deadline 2023-10-01T11:21:35.384313+0000) 2023-10-01T11:21:49.869 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:49 smithi173 bash[39321]: debug 2023-10-01T11:21:49.497+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2023-10-01T11:21:22.887368+0000 front 2023-10-01T11:21:22.887565+0000 (oldest deadline 2023-10-01T11:21:48.787210+0000) 2023-10-01T11:21:49.869 INFO:journalctl@ceph.osd.5.smithi173.stdout:Oct 01 11:21:49 smithi173 bash[35238]: debug 2023-10-01T11:21:49.441+0000 7f62e9807700 -1 osd.5 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:04.800418+0000 front 2023-10-01T11:21:04.800449+0000 (oldest deadline 2023-10-01T11:21:27.099965+0000) 2023-10-01T11:21:49.870 INFO:journalctl@ceph.osd.5.smithi173.stdout:Oct 01 11:21:49 smithi173 bash[35238]: debug 2023-10-01T11:21:49.441+0000 7f62e9807700 -1 osd.5 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:10.001910+0000 front 2023-10-01T11:21:10.002143+0000 (oldest deadline 2023-10-01T11:21:35.301741+0000) 2023-10-01T11:21:49.870 INFO:journalctl@ceph.osd.5.smithi173.stdout:Oct 01 11:21:49 smithi173 bash[35238]: debug 2023-10-01T11:21:49.441+0000 7f62e9807700 -1 osd.5 439 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2023-10-01T11:21:22.903916+0000 front 2023-10-01T11:21:22.903946+0000 (oldest deadline 2023-10-01T11:21:47.003721+0000) 2023-10-01T11:21:50.369 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:49 smithi173 bash[43488]: debug 2023-10-01T11:21:49.945+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:02.351468+0000 front 2023-10-01T11:21:02.351568+0000 (oldest deadline 2023-10-01T11:21:25.851364+0000) 2023-10-01T11:21:50.369 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:49 smithi173 bash[43488]: debug 2023-10-01T11:21:49.945+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:12.754237+0000 front 2023-10-01T11:21:12.754426+0000 (oldest deadline 2023-10-01T11:21:36.853921+0000) 2023-10-01T11:21:50.369 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:49 smithi173 bash[43488]: debug 2023-10-01T11:21:49.945+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2023-10-01T11:21:23.057715+0000 front 2023-10-01T11:21:23.057415+0000 (oldest deadline 2023-10-01T11:21:47.757342+0000) 2023-10-01T11:21:50.869 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:50 smithi173 bash[39321]: debug 2023-10-01T11:21:50.533+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:04.883741+0000 front 2023-10-01T11:21:04.883677+0000 (oldest deadline 2023-10-01T11:21:27.783163+0000) 2023-10-01T11:21:50.869 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:50 smithi173 bash[39321]: debug 2023-10-01T11:21:50.533+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:11.884876+0000 front 2023-10-01T11:21:11.884681+0000 (oldest deadline 2023-10-01T11:21:35.384313+0000) 2023-10-01T11:21:50.869 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:50 smithi173 bash[39321]: debug 2023-10-01T11:21:50.533+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2023-10-01T11:21:22.887368+0000 front 2023-10-01T11:21:22.887565+0000 (oldest deadline 2023-10-01T11:21:48.787210+0000) 2023-10-01T11:21:50.869 INFO:journalctl@ceph.osd.5.smithi173.stdout:Oct 01 11:21:50 smithi173 bash[35238]: debug 2023-10-01T11:21:50.405+0000 7f62e9807700 -1 osd.5 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:04.800418+0000 front 2023-10-01T11:21:04.800449+0000 (oldest deadline 2023-10-01T11:21:27.099965+0000) 2023-10-01T11:21:50.870 INFO:journalctl@ceph.osd.5.smithi173.stdout:Oct 01 11:21:50 smithi173 bash[35238]: debug 2023-10-01T11:21:50.405+0000 7f62e9807700 -1 osd.5 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:10.001910+0000 front 2023-10-01T11:21:10.002143+0000 (oldest deadline 2023-10-01T11:21:35.301741+0000) 2023-10-01T11:21:50.870 INFO:journalctl@ceph.osd.5.smithi173.stdout:Oct 01 11:21:50 smithi173 bash[35238]: debug 2023-10-01T11:21:50.405+0000 7f62e9807700 -1 osd.5 439 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2023-10-01T11:21:22.903916+0000 front 2023-10-01T11:21:22.903946+0000 (oldest deadline 2023-10-01T11:21:47.003721+0000) 2023-10-01T11:21:51.266 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:50 smithi173 bash[43488]: debug 2023-10-01T11:21:50.917+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:02.351468+0000 front 2023-10-01T11:21:02.351568+0000 (oldest deadline 2023-10-01T11:21:25.851364+0000) 2023-10-01T11:21:51.266 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:50 smithi173 bash[43488]: debug 2023-10-01T11:21:50.917+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:12.754237+0000 front 2023-10-01T11:21:12.754426+0000 (oldest deadline 2023-10-01T11:21:36.853921+0000) 2023-10-01T11:21:51.266 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:50 smithi173 bash[43488]: debug 2023-10-01T11:21:50.917+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2023-10-01T11:21:23.057715+0000 front 2023-10-01T11:21:23.057415+0000 (oldest deadline 2023-10-01T11:21:47.757342+0000) 2023-10-01T11:21:51.869 INFO:journalctl@ceph.osd.5.smithi173.stdout:Oct 01 11:21:51 smithi173 bash[166968]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd-5 2023-10-01T11:21:51.869 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:51 smithi173 bash[39321]: debug 2023-10-01T11:21:51.573+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:04.883741+0000 front 2023-10-01T11:21:04.883677+0000 (oldest deadline 2023-10-01T11:21:27.783163+0000) 2023-10-01T11:21:51.869 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:51 smithi173 bash[39321]: debug 2023-10-01T11:21:51.573+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:11.884876+0000 front 2023-10-01T11:21:11.884681+0000 (oldest deadline 2023-10-01T11:21:35.384313+0000) 2023-10-01T11:21:51.870 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:51 smithi173 bash[39321]: debug 2023-10-01T11:21:51.573+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2023-10-01T11:21:22.887368+0000 front 2023-10-01T11:21:22.887565+0000 (oldest deadline 2023-10-01T11:21:48.787210+0000) 2023-10-01T11:21:52.369 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:51 smithi173 bash[43488]: debug 2023-10-01T11:21:51.945+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:02.351468+0000 front 2023-10-01T11:21:02.351568+0000 (oldest deadline 2023-10-01T11:21:25.851364+0000) 2023-10-01T11:21:52.369 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:51 smithi173 bash[43488]: debug 2023-10-01T11:21:51.945+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:12.754237+0000 front 2023-10-01T11:21:12.754426+0000 (oldest deadline 2023-10-01T11:21:36.853921+0000) 2023-10-01T11:21:52.369 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:51 smithi173 bash[43488]: debug 2023-10-01T11:21:51.945+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2023-10-01T11:21:23.057715+0000 front 2023-10-01T11:21:23.057415+0000 (oldest deadline 2023-10-01T11:21:47.757342+0000) 2023-10-01T11:21:52.369 INFO:journalctl@ceph.osd.5.smithi173.stdout:Oct 01 11:21:51 smithi173 bash[167045]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd-5 2023-10-01T11:21:52.866 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:52 smithi173 bash[39321]: debug 2023-10-01T11:21:52.577+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:04.883741+0000 front 2023-10-01T11:21:04.883677+0000 (oldest deadline 2023-10-01T11:21:27.783163+0000) 2023-10-01T11:21:52.866 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:52 smithi173 bash[39321]: debug 2023-10-01T11:21:52.577+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:11.884876+0000 front 2023-10-01T11:21:11.884681+0000 (oldest deadline 2023-10-01T11:21:35.384313+0000) 2023-10-01T11:21:52.866 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:52 smithi173 bash[39321]: debug 2023-10-01T11:21:52.577+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2023-10-01T11:21:22.887368+0000 front 2023-10-01T11:21:22.887565+0000 (oldest deadline 2023-10-01T11:21:48.787210+0000) 2023-10-01T11:21:53.369 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:52 smithi173 bash[43488]: debug 2023-10-01T11:21:52.953+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:02.351468+0000 front 2023-10-01T11:21:02.351568+0000 (oldest deadline 2023-10-01T11:21:25.851364+0000) 2023-10-01T11:21:53.369 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:52 smithi173 bash[43488]: debug 2023-10-01T11:21:52.953+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:12.754237+0000 front 2023-10-01T11:21:12.754426+0000 (oldest deadline 2023-10-01T11:21:36.853921+0000) 2023-10-01T11:21:53.369 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:52 smithi173 bash[43488]: debug 2023-10-01T11:21:52.953+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2023-10-01T11:21:23.057715+0000 front 2023-10-01T11:21:23.057415+0000 (oldest deadline 2023-10-01T11:21:47.757342+0000) 2023-10-01T11:21:53.869 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:53 smithi173 bash[39321]: debug 2023-10-01T11:21:53.577+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:04.883741+0000 front 2023-10-01T11:21:04.883677+0000 (oldest deadline 2023-10-01T11:21:27.783163+0000) 2023-10-01T11:21:53.869 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:53 smithi173 bash[39321]: debug 2023-10-01T11:21:53.577+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:11.884876+0000 front 2023-10-01T11:21:11.884681+0000 (oldest deadline 2023-10-01T11:21:35.384313+0000) 2023-10-01T11:21:53.870 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:53 smithi173 bash[39321]: debug 2023-10-01T11:21:53.577+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2023-10-01T11:21:22.887368+0000 front 2023-10-01T11:21:22.887565+0000 (oldest deadline 2023-10-01T11:21:48.787210+0000) 2023-10-01T11:21:53.870 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:53 smithi173 bash[39321]: debug 2023-10-01T11:21:53.577+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2023-10-01T11:21:28.788178+0000 front 2023-10-01T11:21:28.788114+0000 (oldest deadline 2023-10-01T11:21:53.487765+0000) 2023-10-01T11:21:54.119 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:53 smithi173 bash[43488]: debug 2023-10-01T11:21:53.933+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:02.351468+0000 front 2023-10-01T11:21:02.351568+0000 (oldest deadline 2023-10-01T11:21:25.851364+0000) 2023-10-01T11:21:54.119 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:53 smithi173 bash[43488]: debug 2023-10-01T11:21:53.933+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:12.754237+0000 front 2023-10-01T11:21:12.754426+0000 (oldest deadline 2023-10-01T11:21:36.853921+0000) 2023-10-01T11:21:54.119 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:53 smithi173 bash[43488]: debug 2023-10-01T11:21:53.933+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2023-10-01T11:21:23.057715+0000 front 2023-10-01T11:21:23.057415+0000 (oldest deadline 2023-10-01T11:21:47.757342+0000) 2023-10-01T11:21:54.213 DEBUG:teuthology.orchestra.run.smithi173:> sudo pkill -f 'journalctl -f -n 0 -u ceph-d8016480-6041-11ee-8db5-212e2dc638e7@osd.5.service' 2023-10-01T11:21:54.289 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-01T11:21:54.289 INFO:tasks.cephadm.osd.5:Stopped osd.5 2023-10-01T11:21:54.289 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2023-10-01T11:21:54.289 DEBUG:teuthology.orchestra.run.smithi173:> sudo systemctl stop ceph-d8016480-6041-11ee-8db5-212e2dc638e7@osd.6 2023-10-01T11:21:54.518 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:54 smithi173 systemd[1]: Stopping Ceph osd.6 for d8016480-6041-11ee-8db5-212e2dc638e7... 2023-10-01T11:21:54.518 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:54 smithi173 bash[39321]: debug 2023-10-01T11:21:54.513+0000 7f212f434700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-01T11:21:54.869 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:54 smithi173 bash[39321]: debug 2023-10-01T11:21:54.513+0000 7f212f434700 -1 osd.6 439 *** Got signal Terminated *** 2023-10-01T11:21:54.869 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:54 smithi173 bash[39321]: debug 2023-10-01T11:21:54.513+0000 7f212f434700 -1 osd.6 439 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-01T11:21:54.869 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:54 smithi173 bash[39321]: debug 2023-10-01T11:21:54.613+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:04.883741+0000 front 2023-10-01T11:21:04.883677+0000 (oldest deadline 2023-10-01T11:21:27.783163+0000) 2023-10-01T11:21:54.869 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:54 smithi173 bash[39321]: debug 2023-10-01T11:21:54.613+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:11.884876+0000 front 2023-10-01T11:21:11.884681+0000 (oldest deadline 2023-10-01T11:21:35.384313+0000) 2023-10-01T11:21:54.870 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:54 smithi173 bash[39321]: debug 2023-10-01T11:21:54.613+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2023-10-01T11:21:22.887368+0000 front 2023-10-01T11:21:22.887565+0000 (oldest deadline 2023-10-01T11:21:48.787210+0000) 2023-10-01T11:21:54.870 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:54 smithi173 bash[39321]: debug 2023-10-01T11:21:54.613+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2023-10-01T11:21:28.788178+0000 front 2023-10-01T11:21:28.788114+0000 (oldest deadline 2023-10-01T11:21:53.487765+0000) 2023-10-01T11:21:55.369 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:54 smithi173 bash[43488]: debug 2023-10-01T11:21:54.909+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:02.351468+0000 front 2023-10-01T11:21:02.351568+0000 (oldest deadline 2023-10-01T11:21:25.851364+0000) 2023-10-01T11:21:55.369 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:54 smithi173 bash[43488]: debug 2023-10-01T11:21:54.909+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:12.754237+0000 front 2023-10-01T11:21:12.754426+0000 (oldest deadline 2023-10-01T11:21:36.853921+0000) 2023-10-01T11:21:55.369 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:54 smithi173 bash[43488]: debug 2023-10-01T11:21:54.909+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2023-10-01T11:21:23.057715+0000 front 2023-10-01T11:21:23.057415+0000 (oldest deadline 2023-10-01T11:21:47.757342+0000) 2023-10-01T11:21:55.869 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:55 smithi173 bash[39321]: debug 2023-10-01T11:21:55.613+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:04.883741+0000 front 2023-10-01T11:21:04.883677+0000 (oldest deadline 2023-10-01T11:21:27.783163+0000) 2023-10-01T11:21:55.870 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:55 smithi173 bash[39321]: debug 2023-10-01T11:21:55.613+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:11.884876+0000 front 2023-10-01T11:21:11.884681+0000 (oldest deadline 2023-10-01T11:21:35.384313+0000) 2023-10-01T11:21:55.870 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:55 smithi173 bash[39321]: debug 2023-10-01T11:21:55.613+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2023-10-01T11:21:22.887368+0000 front 2023-10-01T11:21:22.887565+0000 (oldest deadline 2023-10-01T11:21:48.787210+0000) 2023-10-01T11:21:55.870 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:55 smithi173 bash[39321]: debug 2023-10-01T11:21:55.613+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2023-10-01T11:21:28.788178+0000 front 2023-10-01T11:21:28.788114+0000 (oldest deadline 2023-10-01T11:21:53.487765+0000) 2023-10-01T11:21:56.369 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:55 smithi173 bash[43488]: debug 2023-10-01T11:21:55.905+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:02.351468+0000 front 2023-10-01T11:21:02.351568+0000 (oldest deadline 2023-10-01T11:21:25.851364+0000) 2023-10-01T11:21:56.369 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:55 smithi173 bash[43488]: debug 2023-10-01T11:21:55.905+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:12.754237+0000 front 2023-10-01T11:21:12.754426+0000 (oldest deadline 2023-10-01T11:21:36.853921+0000) 2023-10-01T11:21:56.369 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:55 smithi173 bash[43488]: debug 2023-10-01T11:21:55.905+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2023-10-01T11:21:23.057715+0000 front 2023-10-01T11:21:23.057415+0000 (oldest deadline 2023-10-01T11:21:47.757342+0000) 2023-10-01T11:21:56.869 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:56 smithi173 bash[39321]: debug 2023-10-01T11:21:56.581+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:04.883741+0000 front 2023-10-01T11:21:04.883677+0000 (oldest deadline 2023-10-01T11:21:27.783163+0000) 2023-10-01T11:21:56.869 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:56 smithi173 bash[39321]: debug 2023-10-01T11:21:56.581+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:11.884876+0000 front 2023-10-01T11:21:11.884681+0000 (oldest deadline 2023-10-01T11:21:35.384313+0000) 2023-10-01T11:21:56.869 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:56 smithi173 bash[39321]: debug 2023-10-01T11:21:56.581+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2023-10-01T11:21:22.887368+0000 front 2023-10-01T11:21:22.887565+0000 (oldest deadline 2023-10-01T11:21:48.787210+0000) 2023-10-01T11:21:56.869 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:56 smithi173 bash[39321]: debug 2023-10-01T11:21:56.581+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2023-10-01T11:21:28.788178+0000 front 2023-10-01T11:21:28.788114+0000 (oldest deadline 2023-10-01T11:21:53.487765+0000) 2023-10-01T11:21:57.369 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:56 smithi173 bash[43488]: debug 2023-10-01T11:21:56.901+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:02.351468+0000 front 2023-10-01T11:21:02.351568+0000 (oldest deadline 2023-10-01T11:21:25.851364+0000) 2023-10-01T11:21:57.369 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:56 smithi173 bash[43488]: debug 2023-10-01T11:21:56.901+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:12.754237+0000 front 2023-10-01T11:21:12.754426+0000 (oldest deadline 2023-10-01T11:21:36.853921+0000) 2023-10-01T11:21:57.369 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:56 smithi173 bash[43488]: debug 2023-10-01T11:21:56.901+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2023-10-01T11:21:23.057715+0000 front 2023-10-01T11:21:23.057415+0000 (oldest deadline 2023-10-01T11:21:47.757342+0000) 2023-10-01T11:21:57.369 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:56 smithi173 bash[43488]: debug 2023-10-01T11:21:56.901+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2023-10-01T11:21:27.758239+0000 front 2023-10-01T11:21:27.758156+0000 (oldest deadline 2023-10-01T11:21:55.958572+0000) 2023-10-01T11:21:57.869 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:57 smithi173 bash[39321]: debug 2023-10-01T11:21:57.569+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:04.883741+0000 front 2023-10-01T11:21:04.883677+0000 (oldest deadline 2023-10-01T11:21:27.783163+0000) 2023-10-01T11:21:57.869 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:57 smithi173 bash[39321]: debug 2023-10-01T11:21:57.569+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:11.884876+0000 front 2023-10-01T11:21:11.884681+0000 (oldest deadline 2023-10-01T11:21:35.384313+0000) 2023-10-01T11:21:57.869 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:57 smithi173 bash[39321]: debug 2023-10-01T11:21:57.569+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2023-10-01T11:21:22.887368+0000 front 2023-10-01T11:21:22.887565+0000 (oldest deadline 2023-10-01T11:21:48.787210+0000) 2023-10-01T11:21:57.869 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:57 smithi173 bash[39321]: debug 2023-10-01T11:21:57.569+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2023-10-01T11:21:28.788178+0000 front 2023-10-01T11:21:28.788114+0000 (oldest deadline 2023-10-01T11:21:53.487765+0000) 2023-10-01T11:21:58.369 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:57 smithi173 bash[43488]: debug 2023-10-01T11:21:57.953+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:02.351468+0000 front 2023-10-01T11:21:02.351568+0000 (oldest deadline 2023-10-01T11:21:25.851364+0000) 2023-10-01T11:21:58.369 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:57 smithi173 bash[43488]: debug 2023-10-01T11:21:57.953+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:12.754237+0000 front 2023-10-01T11:21:12.754426+0000 (oldest deadline 2023-10-01T11:21:36.853921+0000) 2023-10-01T11:21:58.369 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:57 smithi173 bash[43488]: debug 2023-10-01T11:21:57.953+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2023-10-01T11:21:23.057715+0000 front 2023-10-01T11:21:23.057415+0000 (oldest deadline 2023-10-01T11:21:47.757342+0000) 2023-10-01T11:21:58.370 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:57 smithi173 bash[43488]: debug 2023-10-01T11:21:57.953+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2023-10-01T11:21:27.758239+0000 front 2023-10-01T11:21:27.758156+0000 (oldest deadline 2023-10-01T11:21:55.958572+0000) 2023-10-01T11:21:58.869 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:58 smithi173 bash[39321]: debug 2023-10-01T11:21:58.605+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:04.883741+0000 front 2023-10-01T11:21:04.883677+0000 (oldest deadline 2023-10-01T11:21:27.783163+0000) 2023-10-01T11:21:58.869 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:58 smithi173 bash[39321]: debug 2023-10-01T11:21:58.605+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:11.884876+0000 front 2023-10-01T11:21:11.884681+0000 (oldest deadline 2023-10-01T11:21:35.384313+0000) 2023-10-01T11:21:58.869 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:58 smithi173 bash[39321]: debug 2023-10-01T11:21:58.605+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2023-10-01T11:21:22.887368+0000 front 2023-10-01T11:21:22.887565+0000 (oldest deadline 2023-10-01T11:21:48.787210+0000) 2023-10-01T11:21:58.870 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:21:58 smithi173 bash[39321]: debug 2023-10-01T11:21:58.605+0000 7f2129bac700 -1 osd.6 439 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2023-10-01T11:21:28.788178+0000 front 2023-10-01T11:21:28.788114+0000 (oldest deadline 2023-10-01T11:21:53.487765+0000) 2023-10-01T11:21:59.369 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:58 smithi173 bash[43488]: debug 2023-10-01T11:21:58.961+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:02.351468+0000 front 2023-10-01T11:21:02.351568+0000 (oldest deadline 2023-10-01T11:21:25.851364+0000) 2023-10-01T11:21:59.369 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:58 smithi173 bash[43488]: debug 2023-10-01T11:21:58.961+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:12.754237+0000 front 2023-10-01T11:21:12.754426+0000 (oldest deadline 2023-10-01T11:21:36.853921+0000) 2023-10-01T11:21:59.370 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:58 smithi173 bash[43488]: debug 2023-10-01T11:21:58.961+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2023-10-01T11:21:23.057715+0000 front 2023-10-01T11:21:23.057415+0000 (oldest deadline 2023-10-01T11:21:47.757342+0000) 2023-10-01T11:21:59.370 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:58 smithi173 bash[43488]: debug 2023-10-01T11:21:58.961+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2023-10-01T11:21:27.758239+0000 front 2023-10-01T11:21:27.758156+0000 (oldest deadline 2023-10-01T11:21:55.958572+0000) 2023-10-01T11:22:00.119 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:59 smithi173 bash[43488]: debug 2023-10-01T11:21:59.929+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:02.351468+0000 front 2023-10-01T11:21:02.351568+0000 (oldest deadline 2023-10-01T11:21:25.851364+0000) 2023-10-01T11:22:00.119 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:59 smithi173 bash[43488]: debug 2023-10-01T11:21:59.929+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:12.754237+0000 front 2023-10-01T11:21:12.754426+0000 (oldest deadline 2023-10-01T11:21:36.853921+0000) 2023-10-01T11:22:00.120 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:59 smithi173 bash[43488]: debug 2023-10-01T11:21:59.929+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2023-10-01T11:21:23.057715+0000 front 2023-10-01T11:21:23.057415+0000 (oldest deadline 2023-10-01T11:21:47.757342+0000) 2023-10-01T11:22:00.120 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:21:59 smithi173 bash[43488]: debug 2023-10-01T11:21:59.929+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2023-10-01T11:21:27.758239+0000 front 2023-10-01T11:21:27.758156+0000 (oldest deadline 2023-10-01T11:21:55.958572+0000) 2023-10-01T11:22:00.869 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:22:00 smithi173 bash[167186]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd-6 2023-10-01T11:22:00.869 INFO:journalctl@ceph.osd.6.smithi173.stdout:Oct 01 11:22:00 smithi173 bash[167249]: Error response from daemon: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd-6 2023-10-01T11:22:01.369 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:22:00 smithi173 bash[43488]: debug 2023-10-01T11:22:00.933+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:02.351468+0000 front 2023-10-01T11:21:02.351568+0000 (oldest deadline 2023-10-01T11:21:25.851364+0000) 2023-10-01T11:22:01.369 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:22:00 smithi173 bash[43488]: debug 2023-10-01T11:22:00.933+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:12.754237+0000 front 2023-10-01T11:21:12.754426+0000 (oldest deadline 2023-10-01T11:21:36.853921+0000) 2023-10-01T11:22:01.369 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:22:00 smithi173 bash[43488]: debug 2023-10-01T11:22:00.933+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2023-10-01T11:21:23.057715+0000 front 2023-10-01T11:21:23.057415+0000 (oldest deadline 2023-10-01T11:21:47.757342+0000) 2023-10-01T11:22:01.369 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:22:00 smithi173 bash[43488]: debug 2023-10-01T11:22:00.933+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2023-10-01T11:21:27.758239+0000 front 2023-10-01T11:21:27.758156+0000 (oldest deadline 2023-10-01T11:21:55.958572+0000) 2023-10-01T11:22:02.369 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:22:01 smithi173 bash[43488]: debug 2023-10-01T11:22:01.941+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:02.351468+0000 front 2023-10-01T11:21:02.351568+0000 (oldest deadline 2023-10-01T11:21:25.851364+0000) 2023-10-01T11:22:02.369 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:22:01 smithi173 bash[43488]: debug 2023-10-01T11:22:01.941+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:12.754237+0000 front 2023-10-01T11:21:12.754426+0000 (oldest deadline 2023-10-01T11:21:36.853921+0000) 2023-10-01T11:22:02.369 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:22:01 smithi173 bash[43488]: debug 2023-10-01T11:22:01.941+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2023-10-01T11:21:23.057715+0000 front 2023-10-01T11:21:23.057415+0000 (oldest deadline 2023-10-01T11:21:47.757342+0000) 2023-10-01T11:22:02.370 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:22:01 smithi173 bash[43488]: debug 2023-10-01T11:22:01.941+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2023-10-01T11:21:27.758239+0000 front 2023-10-01T11:21:27.758156+0000 (oldest deadline 2023-10-01T11:21:55.958572+0000) 2023-10-01T11:22:02.370 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 11:22:02 smithi173 bash[54497]: ts=2023-10-01T11:22:02.067Z caller=refresh.go:98 level=error component="discovery manager notify" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.78:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.78:8765: connect: connection refused" 2023-10-01T11:22:02.370 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 11:22:02 smithi173 bash[54497]: ts=2023-10-01T11:22:02.070Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.78:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.78:8765: connect: connection refused" 2023-10-01T11:22:02.370 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 11:22:02 smithi173 bash[54497]: ts=2023-10-01T11:22:02.077Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.78:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.78:8765: connect: connection refused" 2023-10-01T11:22:02.370 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 11:22:02 smithi173 bash[54497]: ts=2023-10-01T11:22:02.077Z caller=refresh.go:98 level=error component="discovery manager scrape" discovery=http msg="Unable to refresh target groups" err="Get \"http://172.21.15.78:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.78:8765: connect: connection refused" 2023-10-01T11:22:02.659 DEBUG:teuthology.orchestra.run.smithi173:> sudo pkill -f 'journalctl -f -n 0 -u ceph-d8016480-6041-11ee-8db5-212e2dc638e7@osd.6.service' 2023-10-01T11:22:02.689 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-01T11:22:02.689 INFO:tasks.cephadm.osd.6:Stopped osd.6 2023-10-01T11:22:02.689 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2023-10-01T11:22:02.690 DEBUG:teuthology.orchestra.run.smithi173:> sudo systemctl stop ceph-d8016480-6041-11ee-8db5-212e2dc638e7@osd.7 2023-10-01T11:22:02.928 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:22:02 smithi173 systemd[1]: Stopping Ceph osd.7 for d8016480-6041-11ee-8db5-212e2dc638e7... 2023-10-01T11:22:02.928 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:22:02 smithi173 bash[43488]: debug 2023-10-01T11:22:02.897+0000 7fb4ee1c2700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2023-10-01T11:22:02.928 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:22:02 smithi173 bash[43488]: debug 2023-10-01T11:22:02.897+0000 7fb4ee1c2700 -1 osd.7 439 *** Got signal Terminated *** 2023-10-01T11:22:02.928 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:22:02 smithi173 bash[43488]: debug 2023-10-01T11:22:02.897+0000 7fb4ee1c2700 -1 osd.7 439 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-10-01T11:22:02.929 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:22:02 smithi173 bash[43488]: debug 2023-10-01T11:22:02.921+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:02.351468+0000 front 2023-10-01T11:21:02.351568+0000 (oldest deadline 2023-10-01T11:21:25.851364+0000) 2023-10-01T11:22:02.929 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:22:02 smithi173 bash[43488]: debug 2023-10-01T11:22:02.921+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:12.754237+0000 front 2023-10-01T11:21:12.754426+0000 (oldest deadline 2023-10-01T11:21:36.853921+0000) 2023-10-01T11:22:03.369 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:22:02 smithi173 bash[43488]: debug 2023-10-01T11:22:02.921+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2023-10-01T11:21:23.057715+0000 front 2023-10-01T11:21:23.057415+0000 (oldest deadline 2023-10-01T11:21:47.757342+0000) 2023-10-01T11:22:03.369 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:22:02 smithi173 bash[43488]: debug 2023-10-01T11:22:02.921+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2023-10-01T11:21:27.758239+0000 front 2023-10-01T11:21:27.758156+0000 (oldest deadline 2023-10-01T11:21:55.958572+0000) 2023-10-01T11:22:03.369 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:22:02 smithi173 bash[43488]: debug 2023-10-01T11:22:02.921+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.173:6804 osd.4 since back 2023-10-01T11:21:39.360920+0000 front 2023-10-01T11:21:39.360951+0000 (oldest deadline 2023-10-01T11:22:02.260517+0000) 2023-10-01T11:22:04.369 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:22:03 smithi173 bash[43488]: debug 2023-10-01T11:22:03.881+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:02.351468+0000 front 2023-10-01T11:21:02.351568+0000 (oldest deadline 2023-10-01T11:21:25.851364+0000) 2023-10-01T11:22:04.369 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:22:03 smithi173 bash[43488]: debug 2023-10-01T11:22:03.881+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:12.754237+0000 front 2023-10-01T11:21:12.754426+0000 (oldest deadline 2023-10-01T11:21:36.853921+0000) 2023-10-01T11:22:04.369 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:22:03 smithi173 bash[43488]: debug 2023-10-01T11:22:03.881+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2023-10-01T11:21:23.057715+0000 front 2023-10-01T11:21:23.057415+0000 (oldest deadline 2023-10-01T11:21:47.757342+0000) 2023-10-01T11:22:04.370 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:22:03 smithi173 bash[43488]: debug 2023-10-01T11:22:03.881+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2023-10-01T11:21:27.758239+0000 front 2023-10-01T11:21:27.758156+0000 (oldest deadline 2023-10-01T11:21:55.958572+0000) 2023-10-01T11:22:04.370 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:22:03 smithi173 bash[43488]: debug 2023-10-01T11:22:03.881+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.173:6804 osd.4 since back 2023-10-01T11:21:39.360920+0000 front 2023-10-01T11:21:39.360951+0000 (oldest deadline 2023-10-01T11:22:02.260517+0000) 2023-10-01T11:22:05.369 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:22:04 smithi173 bash[43488]: debug 2023-10-01T11:22:04.869+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:02.351468+0000 front 2023-10-01T11:21:02.351568+0000 (oldest deadline 2023-10-01T11:21:25.851364+0000) 2023-10-01T11:22:05.369 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:22:04 smithi173 bash[43488]: debug 2023-10-01T11:22:04.869+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:12.754237+0000 front 2023-10-01T11:21:12.754426+0000 (oldest deadline 2023-10-01T11:21:36.853921+0000) 2023-10-01T11:22:05.369 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:22:04 smithi173 bash[43488]: debug 2023-10-01T11:22:04.869+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2023-10-01T11:21:23.057715+0000 front 2023-10-01T11:21:23.057415+0000 (oldest deadline 2023-10-01T11:21:47.757342+0000) 2023-10-01T11:22:05.369 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:22:04 smithi173 bash[43488]: debug 2023-10-01T11:22:04.869+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2023-10-01T11:21:27.758239+0000 front 2023-10-01T11:21:27.758156+0000 (oldest deadline 2023-10-01T11:21:55.958572+0000) 2023-10-01T11:22:05.370 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:22:04 smithi173 bash[43488]: debug 2023-10-01T11:22:04.869+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.173:6804 osd.4 since back 2023-10-01T11:21:39.360920+0000 front 2023-10-01T11:21:39.360951+0000 (oldest deadline 2023-10-01T11:22:02.260517+0000) 2023-10-01T11:22:06.119 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:22:05 smithi173 bash[43488]: debug 2023-10-01T11:22:05.849+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:02.351468+0000 front 2023-10-01T11:21:02.351568+0000 (oldest deadline 2023-10-01T11:21:25.851364+0000) 2023-10-01T11:22:06.120 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:22:05 smithi173 bash[43488]: debug 2023-10-01T11:22:05.849+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:12.754237+0000 front 2023-10-01T11:21:12.754426+0000 (oldest deadline 2023-10-01T11:21:36.853921+0000) 2023-10-01T11:22:06.120 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:22:05 smithi173 bash[43488]: debug 2023-10-01T11:22:05.849+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2023-10-01T11:21:23.057715+0000 front 2023-10-01T11:21:23.057415+0000 (oldest deadline 2023-10-01T11:21:47.757342+0000) 2023-10-01T11:22:06.120 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:22:05 smithi173 bash[43488]: debug 2023-10-01T11:22:05.849+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2023-10-01T11:21:27.758239+0000 front 2023-10-01T11:21:27.758156+0000 (oldest deadline 2023-10-01T11:21:55.958572+0000) 2023-10-01T11:22:06.120 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:22:05 smithi173 bash[43488]: debug 2023-10-01T11:22:05.849+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.173:6804 osd.4 since back 2023-10-01T11:21:39.360920+0000 front 2023-10-01T11:21:39.360951+0000 (oldest deadline 2023-10-01T11:22:02.260517+0000) 2023-10-01T11:22:07.119 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:22:06 smithi173 bash[43488]: debug 2023-10-01T11:22:06.849+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:02.351468+0000 front 2023-10-01T11:21:02.351568+0000 (oldest deadline 2023-10-01T11:21:25.851364+0000) 2023-10-01T11:22:07.119 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:22:06 smithi173 bash[43488]: debug 2023-10-01T11:22:06.849+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:12.754237+0000 front 2023-10-01T11:21:12.754426+0000 (oldest deadline 2023-10-01T11:21:36.853921+0000) 2023-10-01T11:22:07.119 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:22:06 smithi173 bash[43488]: debug 2023-10-01T11:22:06.849+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2023-10-01T11:21:23.057715+0000 front 2023-10-01T11:21:23.057415+0000 (oldest deadline 2023-10-01T11:21:47.757342+0000) 2023-10-01T11:22:07.119 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:22:06 smithi173 bash[43488]: debug 2023-10-01T11:22:06.849+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2023-10-01T11:21:27.758239+0000 front 2023-10-01T11:21:27.758156+0000 (oldest deadline 2023-10-01T11:21:55.958572+0000) 2023-10-01T11:22:07.120 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:22:06 smithi173 bash[43488]: debug 2023-10-01T11:22:06.849+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.173:6804 osd.4 since back 2023-10-01T11:21:39.360920+0000 front 2023-10-01T11:21:39.360951+0000 (oldest deadline 2023-10-01T11:22:02.260517+0000) 2023-10-01T11:22:08.119 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:22:07 smithi173 bash[43488]: debug 2023-10-01T11:22:07.861+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2023-10-01T11:21:02.351468+0000 front 2023-10-01T11:21:02.351568+0000 (oldest deadline 2023-10-01T11:21:25.851364+0000) 2023-10-01T11:22:08.119 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:22:07 smithi173 bash[43488]: debug 2023-10-01T11:22:07.861+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2023-10-01T11:21:12.754237+0000 front 2023-10-01T11:21:12.754426+0000 (oldest deadline 2023-10-01T11:21:36.853921+0000) 2023-10-01T11:22:08.119 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:22:07 smithi173 bash[43488]: debug 2023-10-01T11:22:07.861+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2023-10-01T11:21:23.057715+0000 front 2023-10-01T11:21:23.057415+0000 (oldest deadline 2023-10-01T11:21:47.757342+0000) 2023-10-01T11:22:08.119 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:22:07 smithi173 bash[43488]: debug 2023-10-01T11:22:07.861+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2023-10-01T11:21:27.758239+0000 front 2023-10-01T11:21:27.758156+0000 (oldest deadline 2023-10-01T11:21:55.958572+0000) 2023-10-01T11:22:08.119 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:22:07 smithi173 bash[43488]: debug 2023-10-01T11:22:07.861+0000 7fb4e893a700 -1 osd.7 439 heartbeat_check: no reply from 172.21.15.173:6804 osd.4 since back 2023-10-01T11:21:39.360920+0000 front 2023-10-01T11:21:39.360951+0000 (oldest deadline 2023-10-01T11:22:02.260517+0000) 2023-10-01T11:22:09.119 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:22:08 smithi173 bash[167387]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd-7 2023-10-01T11:22:09.119 INFO:journalctl@ceph.osd.7.smithi173.stdout:Oct 01 11:22:08 smithi173 bash[167455]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-osd-7 2023-10-01T11:22:10.833 DEBUG:teuthology.orchestra.run.smithi173:> sudo pkill -f 'journalctl -f -n 0 -u ceph-d8016480-6041-11ee-8db5-212e2dc638e7@osd.7.service' 2023-10-01T11:22:10.862 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-01T11:22:10.863 INFO:tasks.cephadm.osd.7:Stopped osd.7 2023-10-01T11:22:10.863 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2023-10-01T11:22:10.863 DEBUG:teuthology.orchestra.run.smithi173:> sudo systemctl stop ceph-d8016480-6041-11ee-8db5-212e2dc638e7@prometheus.a 2023-10-01T11:22:11.086 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 11:22:10 smithi173 systemd[1]: Stopping Ceph prometheus.a for d8016480-6041-11ee-8db5-212e2dc638e7... 2023-10-01T11:22:11.372 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 11:22:11 smithi173 bash[54497]: ts=2023-10-01T11:22:11.085Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-10-01T11:22:11.372 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 11:22:11 smithi173 bash[54497]: ts=2023-10-01T11:22:11.085Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2023-10-01T11:22:11.372 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 11:22:11 smithi173 bash[54497]: ts=2023-10-01T11:22:11.086Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2023-10-01T11:22:11.373 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 11:22:11 smithi173 bash[54497]: ts=2023-10-01T11:22:11.086Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2023-10-01T11:22:11.373 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 11:22:11 smithi173 bash[54497]: ts=2023-10-01T11:22:11.086Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2023-10-01T11:22:11.373 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 11:22:11 smithi173 bash[54497]: ts=2023-10-01T11:22:11.086Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2023-10-01T11:22:11.373 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 11:22:11 smithi173 bash[54497]: ts=2023-10-01T11:22:11.086Z caller=main.go:828 level=info msg="Scrape manager stopped" 2023-10-01T11:22:11.373 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 11:22:11 smithi173 bash[54497]: ts=2023-10-01T11:22:11.086Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2023-10-01T11:22:11.373 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 11:22:11 smithi173 bash[54497]: ts=2023-10-01T11:22:11.086Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2023-10-01T11:22:11.373 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 11:22:11 smithi173 bash[54497]: ts=2023-10-01T11:22:11.090Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2023-10-01T11:22:11.374 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 11:22:11 smithi173 bash[54497]: ts=2023-10-01T11:22:11.090Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2023-10-01T11:22:11.374 INFO:journalctl@ceph.prometheus.a.smithi173.stdout:Oct 01 11:22:11 smithi173 bash[54497]: ts=2023-10-01T11:22:11.090Z caller=main.go:1066 level=info msg="See you next time!" 2023-10-01T11:22:11.961 DEBUG:teuthology.orchestra.run.smithi173:> sudo pkill -f 'journalctl -f -n 0 -u ceph-d8016480-6041-11ee-8db5-212e2dc638e7@prometheus.a.service' 2023-10-01T11:22:11.992 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-10-01T11:22:11.992 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2023-10-01T11:22:11.992 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid d8016480-6041-11ee-8db5-212e2dc638e7 --force --keep-logs 2023-10-01T11:22:12.006 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:Oct 01 11:22:11 smithi078 bash[70209]: level=warn ts=2023-10-01T11:22:11.626Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi078.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.78:8443: connect: connection refused" 2023-10-01T11:22:12.006 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:Oct 01 11:22:11 smithi078 bash[70209]: level=warn ts=2023-10-01T11:22:11.627Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi173.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.173:8443: connect: connection refused" 2023-10-01T11:22:19.084 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 11:22:18 smithi078 systemd[1]: Stopping Ceph node-exporter.a for d8016480-6041-11ee-8db5-212e2dc638e7... 2023-10-01T11:22:19.831 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 11:22:19 smithi078 bash[277587]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-node-exporter-a 2023-10-01T11:22:19.831 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 11:22:19 smithi078 systemd[1]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2023-10-01T11:22:19.832 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 11:22:19 smithi078 bash[277651]: Error response from daemon: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-node-exporter-a 2023-10-01T11:22:19.832 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 11:22:19 smithi078 systemd[1]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7@node-exporter.a.service: Failed with result 'exit-code'. 2023-10-01T11:22:19.832 INFO:journalctl@ceph.node-exporter.a.smithi078.stdout:Oct 01 11:22:19 smithi078 systemd[1]: Stopped Ceph node-exporter.a for d8016480-6041-11ee-8db5-212e2dc638e7. 2023-10-01T11:22:20.240 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:Oct 01 11:22:19 smithi078 systemd[1]: Stopping Ceph alertmanager.a for d8016480-6041-11ee-8db5-212e2dc638e7... 2023-10-01T11:22:20.240 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:Oct 01 11:22:20 smithi078 bash[70209]: level=info ts=2023-10-01T11:22:20.113Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2023-10-01T11:22:21.081 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:Oct 01 11:22:20 smithi078 bash[277712]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-alertmanager-a 2023-10-01T11:22:21.081 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:Oct 01 11:22:20 smithi078 bash[277774]: Error response from daemon: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-alertmanager-a 2023-10-01T11:22:21.081 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:Oct 01 11:22:20 smithi078 systemd[1]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-10-01T11:22:21.081 INFO:journalctl@ceph.alertmanager.a.smithi078.stdout:Oct 01 11:22:20 smithi078 systemd[1]: Stopped Ceph alertmanager.a for d8016480-6041-11ee-8db5-212e2dc638e7. 2023-10-01T11:22:32.622 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid d8016480-6041-11ee-8db5-212e2dc638e7 --force --keep-logs 2023-10-01T11:22:39.369 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 11:22:39 smithi173 systemd[1]: Stopping Ceph node-exporter.b for d8016480-6041-11ee-8db5-212e2dc638e7... 2023-10-01T11:22:40.119 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 11:22:39 smithi173 bash[167919]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-node-exporter-b 2023-10-01T11:22:40.119 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 11:22:39 smithi173 systemd[1]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2023-10-01T11:22:40.119 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 11:22:39 smithi173 bash[167980]: Error response from daemon: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-node-exporter-b 2023-10-01T11:22:40.119 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 11:22:39 smithi173 systemd[1]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7@node-exporter.b.service: Failed with result 'exit-code'. 2023-10-01T11:22:40.120 INFO:journalctl@ceph.node-exporter.b.smithi173.stdout:Oct 01 11:22:39 smithi173 systemd[1]: Stopped Ceph node-exporter.b for d8016480-6041-11ee-8db5-212e2dc638e7. 2023-10-01T11:22:40.619 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 11:22:40 smithi173 systemd[1]: Stopping Ceph grafana.a for d8016480-6041-11ee-8db5-212e2dc638e7... 2023-10-01T11:22:40.619 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 11:22:40 smithi173 bash[53242]: t=2023-10-01T11:22:40+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2023-10-01T11:22:41.369 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 11:22:40 smithi173 bash[168041]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-grafana-a 2023-10-01T11:22:41.369 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 11:22:41 smithi173 bash[168104]: Error response from daemon: No such container: ceph-d8016480-6041-11ee-8db5-212e2dc638e7-grafana-a 2023-10-01T11:22:41.369 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 11:22:41 smithi173 systemd[1]: ceph-d8016480-6041-11ee-8db5-212e2dc638e7@grafana.a.service: Succeeded. 2023-10-01T11:22:41.369 INFO:journalctl@ceph.grafana.a.smithi173.stdout:Oct 01 11:22:41 smithi173 systemd[1]: Stopped Ceph grafana.a for d8016480-6041-11ee-8db5-212e2dc638e7. 2023-10-01T11:22:44.238 DEBUG:teuthology.orchestra.run.smithi078:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-01T11:22:44.254 DEBUG:teuthology.orchestra.run.smithi173:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-10-01T11:22:44.269 INFO:tasks.cephadm:Archiving crash dumps... 2023-10-01T11:22:44.271 DEBUG:teuthology.misc:Transferring archived files from smithi078:/var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/crash to /home/teuthworker/archive/teuthology-2023-09-30_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7407998/remote/smithi078/crash 2023-10-01T11:22:44.272 DEBUG:teuthology.orchestra.run.smithi078:> sudo tar c -f - -C /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/crash -- . 2023-10-01T11:22:44.325 INFO:teuthology.orchestra.run.smithi078.stderr:tar: /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/crash: Cannot open: No such file or directory 2023-10-01T11:22:44.325 INFO:teuthology.orchestra.run.smithi078.stderr:tar: Error is not recoverable: exiting now 2023-10-01T11:22:44.327 DEBUG:teuthology.misc:Transferring archived files from smithi173:/var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/crash to /home/teuthworker/archive/teuthology-2023-09-30_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7407998/remote/smithi173/crash 2023-10-01T11:22:44.328 DEBUG:teuthology.orchestra.run.smithi173:> sudo tar c -f - -C /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/crash -- . 2023-10-01T11:22:44.364 INFO:teuthology.orchestra.run.smithi173.stderr:tar: /var/lib/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/crash: Cannot open: No such file or directory 2023-10-01T11:22:44.364 INFO:teuthology.orchestra.run.smithi173.stderr:tar: Error is not recoverable: exiting now 2023-10-01T11:22:44.366 INFO:tasks.cephadm:Checking cluster log for badness... 2023-10-01T11:22:44.366 DEBUG:teuthology.orchestra.run.smithi078:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2023-10-01T11:22:44.386 INFO:teuthology.orchestra.run.smithi078.stderr:grep: /var/log/ceph/d8016480-6041-11ee-8db5-212e2dc638e7/ceph.log: No such file or directory 2023-10-01T11:22:44.387 INFO:tasks.cephadm:Compressing logs... 2023-10-01T11:22:44.388 DEBUG:teuthology.orchestra.run.smithi078:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-10-01T11:22:44.435 DEBUG:teuthology.orchestra.run.smithi173:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-10-01T11:22:44.452 INFO:teuthology.orchestra.run.smithi173.stderr:find: '/var/log/rbd-target-api': No such file or directory 2023-10-01T11:22:44.457 INFO:teuthology.orchestra.run.smithi078.stderr:find: '/var/log/rbd-target-api': No such file or directory 2023-10-01T11:32:27.391 INFO:tasks.cephadm:Archiving logs... 2023-10-01T11:32:27.391 DEBUG:teuthology.misc:Transferring archived files from smithi078:/var/log/ceph to /home/teuthworker/archive/teuthology-2023-09-30_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7407998/remote/smithi078/log 2023-10-01T11:32:27.393 DEBUG:teuthology.orchestra.run.smithi078:> sudo tar c -f - -C /var/log/ceph -- . 2023-10-01T11:33:36.306 DEBUG:teuthology.misc:Transferring archived files from smithi173:/var/log/ceph to /home/teuthworker/archive/teuthology-2023-09-30_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7407998/remote/smithi173/log 2023-10-01T11:33:36.307 DEBUG:teuthology.orchestra.run.smithi173:> sudo tar c -f - -C /var/log/ceph -- . 2023-10-01T11:34:35.241 INFO:tasks.cephadm:Removing cluster... 2023-10-01T11:34:35.241 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid d8016480-6041-11ee-8db5-212e2dc638e7 --force 2023-10-01T11:34:37.302 DEBUG:teuthology.orchestra.run.smithi173:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid d8016480-6041-11ee-8db5-212e2dc638e7 --force 2023-10-01T11:34:39.678 INFO:tasks.cephadm:Removing cephadm ... 2023-10-01T11:34:39.678 DEBUG:teuthology.orchestra.run.smithi078:> rm -rf /home/ubuntu/cephtest/cephadm 2023-10-01T11:34:39.703 DEBUG:teuthology.orchestra.run.smithi173:> rm -rf /home/ubuntu/cephtest/cephadm 2023-10-01T11:34:39.729 INFO:tasks.cephadm:Teardown complete 2023-10-01T11:34:39.729 DEBUG:teuthology.run_tasks:Unwinding manager install 2023-10-01T11:34:39.740 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2023-10-01T11:34:39.740 DEBUG:teuthology.orchestra.run.smithi078:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2023-10-01T11:34:39.746 DEBUG:teuthology.orchestra.run.smithi173:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2023-10-01T11:34:39.856 DEBUG:teuthology.run_tasks:Unwinding manager clock 2023-10-01T11:34:39.866 INFO:teuthology.task.clock:Checking final clock skew... 2023-10-01T11:34:39.867 DEBUG:teuthology.orchestra.run.smithi078:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-01T11:34:39.869 DEBUG:teuthology.orchestra.run.smithi173:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-10-01T11:34:39.902 INFO:teuthology.orchestra.run.smithi078.stdout: remote refid st t when poll reach delay offset jitter 2023-10-01T11:34:39.903 INFO:teuthology.orchestra.run.smithi078.stdout:============================================================================== 2023-10-01T11:34:39.903 INFO:teuthology.orchestra.run.smithi078.stdout:*hv01.front.sepi 44.4.53.2 2 u 170 256 377 0.190 0.860 0.392 2023-10-01T11:34:39.903 INFO:teuthology.orchestra.run.smithi078.stdout:+hv02.front.sepi 63.231.80.2 3 u 159 256 377 0.157 1.423 0.333 2023-10-01T11:34:39.903 INFO:teuthology.orchestra.run.smithi078.stdout:+hv03.front.sepi 74.6.168.72 3 u 150 256 377 0.182 1.303 0.326 2023-10-01T11:34:39.903 INFO:teuthology.orchestra.run.smithi078.stdout: hv04.front.sepi .INIT. 16 u - 1024 0 0.000 0.000 0.000 2023-10-01T11:34:39.957 INFO:teuthology.orchestra.run.smithi173.stdout: remote refid st t when poll reach delay offset jitter 2023-10-01T11:34:39.957 INFO:teuthology.orchestra.run.smithi173.stdout:============================================================================== 2023-10-01T11:34:39.957 INFO:teuthology.orchestra.run.smithi173.stdout:*hv01.front.sepi 44.4.53.2 2 u 165 256 377 0.182 -0.001 0.143 2023-10-01T11:34:39.957 INFO:teuthology.orchestra.run.smithi173.stdout:+hv02.front.sepi 63.231.80.2 3 u 131 256 377 0.163 0.599 0.347 2023-10-01T11:34:39.958 INFO:teuthology.orchestra.run.smithi173.stdout:+hv03.front.sepi 74.6.168.72 3 u 150 256 377 0.168 0.513 0.072 2023-10-01T11:34:39.958 INFO:teuthology.orchestra.run.smithi173.stdout: hv04.front.sepi .INIT. 16 u - 1024 0 0.000 0.000 0.000 2023-10-01T11:34:39.959 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2023-10-01T11:34:39.970 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2023-10-01T11:34:39.970 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2023-10-01T11:34:39.981 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2023-10-01T11:34:39.993 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2023-10-01T11:34:40.005 INFO:teuthology.task.internal:Duration was 5892.337919 seconds 2023-10-01T11:34:40.006 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2023-10-01T11:34:40.017 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2023-10-01T11:34:40.017 DEBUG:teuthology.orchestra.run.smithi078:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-10-01T11:34:40.020 DEBUG:teuthology.orchestra.run.smithi173:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-10-01T11:34:40.145 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2023-10-01T11:34:40.145 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi078.front.sepia.ceph.com 2023-10-01T11:34:40.145 DEBUG:teuthology.orchestra.run.smithi078:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-10-01T11:34:40.160 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi173.front.sepia.ceph.com 2023-10-01T11:34:40.160 DEBUG:teuthology.orchestra.run.smithi173:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-10-01T11:34:40.175 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2023-10-01T11:34:40.176 DEBUG:teuthology.orchestra.run.smithi078:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-10-01T11:34:40.202 DEBUG:teuthology.orchestra.run.smithi173:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-10-01T11:34:40.423 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2023-10-01T11:34:40.423 DEBUG:teuthology.orchestra.run.smithi078:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-10-01T11:34:40.425 DEBUG:teuthology.orchestra.run.smithi173:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-10-01T11:34:41.024 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2023-10-01T11:34:41.037 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2023-10-01T11:34:41.037 DEBUG:teuthology.orchestra.run.smithi078:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-10-01T11:34:41.078 DEBUG:teuthology.orchestra.run.smithi173:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-10-01T11:34:41.117 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2023-10-01T11:34:41.131 DEBUG:teuthology.orchestra.run.smithi078:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-10-01T11:34:41.134 DEBUG:teuthology.orchestra.run.smithi173:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-10-01T11:34:41.145 INFO:teuthology.orchestra.run.smithi078.stdout:kernel.core_pattern = core 2023-10-01T11:34:41.179 INFO:teuthology.orchestra.run.smithi173.stdout:kernel.core_pattern = core 2023-10-01T11:34:41.224 DEBUG:teuthology.orchestra.run.smithi078:> test -e /home/ubuntu/cephtest/archive/coredump 2023-10-01T11:34:41.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:34:41.230 DEBUG:teuthology.orchestra.run.smithi173:> test -e /home/ubuntu/cephtest/archive/coredump 2023-10-01T11:34:41.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:34:41.274 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2023-10-01T11:34:41.287 INFO:teuthology.task.internal:Transferring archived files... 2023-10-01T11:34:41.288 DEBUG:teuthology.misc:Transferring archived files from smithi078:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2023-09-30_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7407998/remote/smithi078 2023-10-01T11:34:41.289 DEBUG:teuthology.orchestra.run.smithi078:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-10-01T11:34:41.343 DEBUG:teuthology.misc:Transferring archived files from smithi173:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2023-09-30_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7407998/remote/smithi173 2023-10-01T11:34:41.344 DEBUG:teuthology.orchestra.run.smithi173:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-10-01T11:34:41.383 INFO:teuthology.task.internal:Removing archive directory... 2023-10-01T11:34:41.383 DEBUG:teuthology.orchestra.run.smithi078:> rm -rf -- /home/ubuntu/cephtest/archive 2023-10-01T11:34:41.385 DEBUG:teuthology.orchestra.run.smithi173:> rm -rf -- /home/ubuntu/cephtest/archive 2023-10-01T11:34:41.423 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2023-10-01T11:34:41.434 INFO:teuthology.task.internal:Not uploading archives. 2023-10-01T11:34:41.434 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2023-10-01T11:34:41.447 INFO:teuthology.task.internal:Tidying up after the test... 2023-10-01T11:34:41.447 DEBUG:teuthology.orchestra.run.smithi078:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-10-01T11:34:41.449 DEBUG:teuthology.orchestra.run.smithi173:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-10-01T11:34:41.455 INFO:teuthology.orchestra.run.smithi078.stdout: 1048601 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Oct 1 11:34 /home/ubuntu/cephtest 2023-10-01T11:34:41.455 INFO:teuthology.orchestra.run.smithi078.stdout: 1572870 4 drwxrwxr-x 3 ubuntu ubuntu 4096 Oct 1 11:15 /home/ubuntu/cephtest/mnt.0 2023-10-01T11:34:41.455 INFO:teuthology.orchestra.run.smithi078.stdout: 1572874 4 drwxrwxr-x 2 ubuntu ubuntu 4096 Oct 1 11:15 /home/ubuntu/cephtest/mnt.0/client.0 2023-10-01T11:34:41.457 INFO:teuthology.orchestra.run.smithi078.stderr:rmdir: failed to remove '/home/ubuntu/cephtest': Directory not empty 2023-10-01T11:34:41.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-10-01T11:34:41.465 ERROR:teuthology.run_tasks:Manager failed: internal.base Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 154, in run_tasks suppress = manager.__exit__(*exc_info) File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__ next(self.gen) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/internal/__init__.py", line 53, in base run.wait( File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 479, in wait proc.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi078 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2023-10-01T11:34:41.472 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2023-10-01T11:34:41.485 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2023-10-01T11:34:41.500 DEBUG:teuthology.run_tasks:Exception was not quenched, exiting: CommandFailedError: Command failed on smithi078 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2023-10-01T11:34:41.500 INFO:teuthology.nuke:Checking targets against current locks 2023-10-01T11:34:41.515 INFO:teuthology.orchestra.run.smithi173.stdout: 1048603 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Oct 1 11:34 /home/ubuntu/cephtest 2023-10-01T11:34:41.540 DEBUG:teuthology.nuke:shortname: smithi078 2023-10-01T11:34:41.540 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-10-01T11:34:41.563 DEBUG:teuthology.nuke:shortname: smithi173 2023-10-01T11:34:41.563 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-10-01T11:34:41.568 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi078.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-09-30_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7407998', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '20.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-01 09:47:34.191666', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFyT/r7xuKfmg50w0OdQebr4A/ptJo1Zk6DyZNZNZM/q5rW+sznh+wvHqeT3e4YtCdZLaXyvMms8o8u8H5nXOYM='} 2023-10-01T11:34:41.584 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi173.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-09-30_14:23:11-upgrade:pacific-x-reef-distro-default-smithi/7407998', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '20.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-10-01 09:47:34.189897', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFyT/r7xuKfmg50w0OdQebr4A/ptJo1Zk6DyZNZNZM/q5rW+sznh+wvHqeT3e4YtCdZLaXyvMms8o8u8H5nXOYM='} 2023-10-01T11:34:41.616 INFO:teuthology.orchestra.console.smithi078:Power off 2023-10-01T11:34:41.617 DEBUG:teuthology.orchestra.console.smithi078:pexpect command: ipmitool -H smithi078.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-10-01T11:34:41.633 INFO:teuthology.orchestra.console.smithi173:Power off 2023-10-01T11:34:41.633 DEBUG:teuthology.orchestra.console.smithi173:pexpect command: ipmitool -H smithi173.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-10-01T11:34:41.643 DEBUG:teuthology.orchestra.console.smithi078:power off output: Chassis Power Control: Down/Off 2023-10-01T11:34:41.643 DEBUG:teuthology.orchestra.console.smithi078:pexpect command: ipmitool -H smithi078.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-01T11:34:41.656 DEBUG:teuthology.orchestra.console.smithi173:power off output: Chassis Power Control: Down/Off 2023-10-01T11:34:41.656 DEBUG:teuthology.orchestra.console.smithi173:pexpect command: ipmitool -H smithi173.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-01T11:34:41.666 DEBUG:teuthology.orchestra.console.smithi078:check power output: Chassis Power is on 2023-10-01T11:34:41.679 DEBUG:teuthology.orchestra.console.smithi173:check power output: Chassis Power is on 2023-10-01T11:34:45.667 DEBUG:teuthology.orchestra.console.smithi078:pexpect command: ipmitool -H smithi078.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-01T11:34:45.680 DEBUG:teuthology.orchestra.console.smithi173:pexpect command: ipmitool -H smithi173.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-01T11:34:45.777 DEBUG:teuthology.orchestra.console.smithi078:check power output: Chassis Power is on 2023-10-01T11:34:45.791 DEBUG:teuthology.orchestra.console.smithi173:check power output: Chassis Power is on 2023-10-01T11:34:49.782 DEBUG:teuthology.orchestra.console.smithi078:pexpect command: ipmitool -H smithi078.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-01T11:34:49.792 DEBUG:teuthology.orchestra.console.smithi173:pexpect command: ipmitool -H smithi173.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-01T11:34:49.893 DEBUG:teuthology.orchestra.console.smithi078:check power output: Chassis Power is on 2023-10-01T11:34:49.902 DEBUG:teuthology.orchestra.console.smithi173:check power output: Chassis Power is on 2023-10-01T11:34:53.898 DEBUG:teuthology.orchestra.console.smithi078:pexpect command: ipmitool -H smithi078.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-01T11:34:53.908 DEBUG:teuthology.orchestra.console.smithi173:pexpect command: ipmitool -H smithi173.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-10-01T11:34:54.010 DEBUG:teuthology.orchestra.console.smithi078:check power output: Chassis Power is off 2023-10-01T11:34:54.017 DEBUG:teuthology.orchestra.console.smithi173:check power output: Chassis Power is off 2023-10-01T11:34:54.112 INFO:teuthology.orchestra.console.smithi078:Power off completed 2023-10-01T11:34:54.118 INFO:teuthology.orchestra.console.smithi173:Power off completed 2023-10-01T11:34:54.224 INFO:teuthology.run:Summary data: description: upgrade:pacific-x/stress-split/{0-distro/ubuntu_20.04 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity} duration: 5892.337918758392 failure_reason: '' flavor: default owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=bab83aa8362245d789538165696fe576 status: fail success: false 2023-10-01T11:34:54.225 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-10-01T11:34:54.320 INFO:teuthology.run:FAIL